├── .gitattributes ├── readme.md ├── go-path └── src │ └── dsmnko.org │ └── hlc18 │ ├── vector_test.go │ ├── LICENSE │ ├── bitset_test.go │ ├── vector.go │ ├── indexes_test.go │ ├── evio-http-server_test.go │ ├── bitset.go │ ├── evio-http-server.go │ ├── cache.go │ ├── suggest_test.go │ ├── suggest.go │ ├── data.go │ ├── model_test.go │ ├── data_easyjson.go │ ├── recommend.go │ ├── recommend_test.go │ ├── filter_test.go │ ├── group.go │ ├── indexes.go │ └── groupindex.go ├── LICENSE ├── Dockerfile ├── main_test.go ├── evio.diff └── main.go /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | ## Highload Cup 2018 3rd grade prize winner (https://highloadcup.ru/ru/rating/) 2 | Best GO solution )) 3 | 4 | 5 | 1. В GOPATH должна быть добавлена папка ./go-path 6 | 1. get`ы всех зависимостей есть в Dockerfile 7 | 1. в ./go-path/src/tidwall/evio скопировать исходники tidwall/evio, все "хаки" для evio лежат в evio.diff 8 | 9 | 10 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/vector_test.go: -------------------------------------------------------------------------------- 1 | package hlc18 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | ) 7 | 8 | func TestVector(t *testing.T) { 9 | 10 | r := rand.New(rand.NewSource(99)) 11 | 12 | vec := makeVector(8) 13 | 14 | var refSum int 15 | 16 | for i := 0; i < 642; i++ { 17 | v := r.Uint32() 18 | refSum += int(v) 19 | if i != vec.Push(v) { 20 | t.Errorf("push %d error %v", i, vec) 21 | } 22 | } 23 | 24 | var vecSum int 25 | 26 | sum := func(i int, v uint32) bool { 27 | vecSum += int(v) 28 | return true 29 | } 30 | 31 | vec.Iterate(sum) 32 | 33 | if vecSum != refSum { 34 | t.Errorf("summs diff") 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 Joshua J Baker 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 Joshua J Baker 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Наследуемся от CentOS 7 2 | FROM centos:7 3 | 4 | # Выбираем рабочую папку 5 | WORKDIR /root 6 | 7 | # Устанавливаем wget и скачиваем Go 8 | RUN yum install -y wget && \ 9 | yum install -y git && \ 10 | # yum install -y net-tools && \ 11 | wget https://storage.googleapis.com/golang/go1.11.4.linux-amd64.tar.gz 12 | 13 | # Устанавливаем Go, создаем workspace и папку проекта 14 | RUN tar -C /usr/local -xzf go1.11.4.linux-amd64.tar.gz && \ 15 | mkdir go && mkdir go/src && mkdir go/bin && mkdir go/pkg && \ 16 | mkdir go/src/dumb 17 | 18 | # Задаем переменные окружения для работы Go 19 | ENV PATH=${PATH}:/usr/local/go/bin GOROOT=/usr/local/go GOPATH=/root/go 20 | 21 | # Копируем наш исходный main.go внутрь контейнера, в папку go/src/dumb 22 | ADD main.go go/src/dumb 23 | 24 | # Копируем пакеты 25 | COPY go-path/src go/src 26 | 27 | # Устанавливаем пакеты 28 | # Компилируем и устанавливаем наш сервер 29 | RUN go get -u github.com/valyala/fasthttp && \ 30 | go get -u github.com/mailru/easyjson && \ 31 | go get -u github.com/valyala/fastjson && \ 32 | go build dumb && go install dumb 33 | 34 | # Открываем 80-й порт наружу 35 | EXPOSE 80 36 | 37 | 38 | # sysctl -a | grep 'net.ipv4.tcp_window_scaling\|tcp_low_latency\|tcp_sack\|tcp_timestamps\|tcp_fastopen' &&\ 39 | 40 | # Запускаем наш сервер 41 | #CMD ifconfig | grep mtu && head -n 26 /proc/cpuinfo && GODEBUG=memprofilerate=0 ./go/bin/dumb 42 | CMD grep "model name" /proc/cpuinfo | head -n 1 && \ 43 | cat /proc/version &&\ 44 | GODEBUG=memprofilerate=0 ./go/bin/dumb 45 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/bitset_test.go: -------------------------------------------------------------------------------- 1 | package hlc18 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestBitSet(t *testing.T) { 8 | 9 | var bitset Bitset256 10 | 11 | if bitset.IsEmpty() != true { 12 | t.Errorf("IsEmpty failed, %v", bitset) 13 | } 14 | 15 | bitset.Set(0) 16 | bitset.Set(1) 17 | 18 | bitset.Set(127) 19 | bitset.Set(64) 20 | bitset.Set(63) 21 | 22 | if bitset.Get(0) != true { 23 | t.Errorf("0 failed, %v", bitset) 24 | } 25 | if bitset.Get(1) != true { 26 | t.Errorf("1 failed, %v", bitset) 27 | } 28 | if bitset.Get(63) != true { 29 | t.Errorf("63 failed, %v", bitset) 30 | } 31 | if bitset.Get(64) != true { 32 | t.Errorf("64 failed, %v", bitset) 33 | } 34 | if bitset.Get(127) != true { 35 | t.Errorf("127 failed, %v", bitset) 36 | } 37 | 38 | bitset.Set(127) // have to be idempotent 39 | if bitset.Get(127) != true { 40 | t.Errorf("127 failed, %v", bitset) 41 | } 42 | 43 | if bitset.Get(2) == true { 44 | t.Errorf("2 failed, %v", bitset) 45 | } 46 | if bitset.Get(255) == true { 47 | t.Errorf("255 failed, %v", bitset) 48 | } 49 | 50 | var bitset2 Bitset256 51 | 52 | bitset2.Set(0) 53 | bitset2.Set(1) 54 | bitset2.Set(64) 55 | bitset2.Set(63) 56 | 57 | if bitset.Contains(&bitset2) != true { 58 | t.Errorf("Contains failed, %v, %v", bitset, bitset2) 59 | } 60 | 61 | if bitset.Any(&bitset2) != true { 62 | t.Errorf("Any failed, %v, %v", bitset, bitset2) 63 | } 64 | 65 | bitset2.Set(42) 66 | 67 | if bitset.Any(&bitset2) != true { 68 | t.Errorf("Any (42) failed, %v, %v", bitset, bitset2) 69 | } 70 | if bitset.Contains(&bitset2) == true { 71 | t.Errorf("non Contains failed, %v, %v", bitset, bitset2) 72 | } 73 | 74 | var bitset3 Bitset256 75 | 76 | bitset3.Set(42) 77 | 78 | if bitset.Any(&bitset3) == true { 79 | t.Errorf("non Any failed, %v, %v", bitset, bitset3) 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/vector.go: -------------------------------------------------------------------------------- 1 | package hlc18 2 | 3 | type VectorUint32 struct { 4 | blockSize, len int 5 | blocks [][]uint32 6 | } 7 | 8 | func makeVector(block int) (vect *VectorUint32) { 9 | return &VectorUint32{ 10 | blockSize: block, 11 | len: 0, 12 | blocks: make([][]uint32, 0)} 13 | } 14 | 15 | func makeInitialVector(block int, value uint32) (vect *VectorUint32) { 16 | vector := &VectorUint32{ 17 | blockSize: block, 18 | len: 0, 19 | blocks: make([][]uint32, 0)} 20 | vector.Push(value) 21 | return vector 22 | } 23 | 24 | func (p *VectorUint32) Push(value uint32) int { 25 | b := p.len / p.blockSize 26 | if b >= len(p.blocks) { 27 | // expand, copy, swap 28 | tmp := make([][]uint32, b+1) 29 | tmp[b] = make([]uint32, p.blockSize) 30 | copy(tmp, p.blocks) 31 | p.blocks = tmp 32 | } 33 | p.blocks[b][p.len%p.blockSize] = value 34 | p.len++ 35 | return p.len - 1 36 | } 37 | 38 | func (p *VectorUint32) Get(index int) uint32 { 39 | return p.blocks[index/p.blockSize][index%p.blockSize] 40 | } 41 | 42 | // возвращаем предыдущее знаение 43 | func (p *VectorUint32) Len() int { 44 | return p.len 45 | } 46 | 47 | // iter func(i int,v uint32) bool - return true for continue, false to exit Iteration 48 | 49 | /** 50 | Iterate() returns last index iterated 51 | */ 52 | func (p *VectorUint32) Iterate(iter func(i int, v uint32) bool) int { 53 | var index = 0 54 | for ib := 0; ib < len(p.blocks) && index < p.len; ib++ { 55 | for ii := 0; ii < p.blockSize && index < p.len; ii++ { 56 | index = (ib * p.blockSize) + ii 57 | if index < p.len { 58 | if !iter(index, p.blocks[ib][ii]) { 59 | return index 60 | } 61 | } 62 | } 63 | } 64 | return index 65 | } 66 | 67 | func (p *VectorUint32) CopyTo(out []uint32) { 68 | for i, b := range p.blocks { 69 | if i+1 < len(p.blocks) { 70 | copy(out[i*p.blockSize:], b) 71 | } else { 72 | copy(out[i*p.blockSize:], b[:(p.len-1)%p.blockSize+1]) 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/indexes_test.go: -------------------------------------------------------------------------------- 1 | package hlc18 2 | 3 | import ( 4 | "math" 5 | "testing" 6 | ) 7 | 8 | func TestIndexOrIterator(t *testing.T) { 9 | 10 | iterator := makeIndexOrIterator() 11 | 12 | iterator.push([]uint32{1}) 13 | iterator.push([]uint32{1}) 14 | iterator.push([]uint32{}) 15 | iterator.push([]uint32{}) 16 | 17 | if 1 != iterator.Next() { 18 | t.Error("failed") 19 | } 20 | if math.MaxUint32 != iterator.Next() { 21 | t.Error("failed") 22 | } 23 | 24 | iterator = makeIndexOrIterator() 25 | iterator.push([]uint32{1, 3}) 26 | iterator.push([]uint32{2, 4}) 27 | iterator.push([]uint32{2, 3}) 28 | iterator.push([]uint32{1, 2, 3, 4}) 29 | 30 | if 4 != iterator.Next() { 31 | t.Error("failed") 32 | } 33 | if 3 != iterator.Next() { 34 | t.Error("failed") 35 | } 36 | if 2 != iterator.Next() { 37 | t.Error("failed") 38 | } 39 | if 1 != iterator.Next() { 40 | t.Error("failed") 41 | } 42 | if math.MaxUint32 != iterator.Next() { 43 | t.Error("failed") 44 | } 45 | 46 | } 47 | 48 | func TestIndexAndIterator(t *testing.T) { 49 | 50 | iterator := makeIndexAndIterator() 51 | iterator.push([]uint32{1}) 52 | iterator.push([]uint32{1}) 53 | iterator.Prepare() 54 | if 1 != iterator.Next() { 55 | t.Error("failed") 56 | } 57 | if math.MaxUint32 != iterator.Next() { 58 | t.Error("failed") 59 | } 60 | 61 | iterator = makeIndexAndIterator() 62 | iterator.push([]uint32{1}) 63 | iterator.push([]uint32{}) 64 | iterator.Prepare() 65 | if math.MaxUint32 != iterator.Next() { 66 | t.Error("failed") 67 | } 68 | 69 | iterator = makeIndexAndIterator() 70 | iterator.push([]uint32{1, 3}) 71 | iterator.push([]uint32{2, 4}) 72 | iterator.push([]uint32{2, 3}) 73 | iterator.push([]uint32{1, 2, 3, 4}) 74 | iterator.Prepare() 75 | if math.MaxUint32 != iterator.Next() { 76 | t.Error("failed") 77 | } 78 | 79 | iterator = makeIndexAndIterator() 80 | iterator.push([]uint32{1, 2}) 81 | iterator.push([]uint32{2, 4}) 82 | iterator.push([]uint32{2, 3}) 83 | iterator.push([]uint32{1, 2, 3, 4}) 84 | iterator.Prepare() 85 | if 2 != iterator.Next() { 86 | t.Error("failed") 87 | } 88 | if math.MaxUint32 != iterator.Next() { 89 | t.Error("failed") 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/evio-http-server_test.go: -------------------------------------------------------------------------------- 1 | package hlc18 2 | 3 | import ( 4 | // "github.com/valyala/fasthttp" 5 | "testing" 6 | ) 7 | 8 | func TestParseQuery(t *testing.T) { 9 | //queryArgs.ParseBytes(u.queryString) 10 | // order=-1&query_id=120&keys=status&birth=1983&limit=35&likes=26242 11 | // joined=2016&order=-1&query_id=0&keys=interests&limit=35&likes=21514 12 | // city=%D0%9F%D0%B5%D1%80%D0%B5%D0%B3%D0%B0%D0%BC%D0%B0&query_id=2040&limit=10 13 | // joined=2016&order=-1&query_id=1800&limit=30&keys=interests 14 | // sname_null=0&query_id=2160&limit=18&sex_eq=m 15 | // interests_any=%D0%A1%D0%B8%D0%BC%D0%BF%D1%81%D0%BE%D0%BD%D1%8B%2C%D0%90%D0%BF%D0%B5%D0%BB%D1%8C%D1%81%D0%B8%D0%BD%D0%BE%D0%B2%D1%8B%D0%B9+%D1%81%D0%BE%D0%BA%2C%D0%A0%D1%8D%D0%BF%2C%D0%A1%D1%82%D0%B5%D0%B9%D0%BA&query_id=2280&limit=26&likes_contains=26566 16 | // query_id=960&limit=14&likes_contains=401 17 | // city=%D0%9B%D0%B5%D0%B9%D0%BF%D0%BE%D1%80%D0%B8%D0%B6&query_id=840&limit=4 18 | // query_id=360&birth_lt=691597982&city_any=%D0%9C%D0%BE%D1%81%D0%BE%D0%BB%D0%B5%D1%81%D1%81%D0%BA%2C%D0%A0%D0%BE%D1%82%D1%82%D0%B5%D1%80%D0%BE%D1%88%D1%82%D0%B0%D0%B4%D1%82%2C%D0%9D%D0%BE%D0%B2%D0%BE%D0%BA%D0%B5%D0%BD%D1%81%D0%BA%2C%D0%9B%D0%B8%D1%81%D1%81%D0%B0%D0%BA%D0%BE%D0%B1%D1%81%D0%BA%2C%D0%97%D0%B5%D0%BB%D0%B5%D0%BD%D0%BE%D0%BB%D0%B0%D0%BC%D1%81%D0%BA&limit=22&sex_eq=f 19 | // city=%D0%91%D0%B0%D1%80%D1%81%D0%BE%D0%B3%D0%B0%D0%BC%D0%B0&query_id=480&limit=18 20 | // order=1&query_id=1200&keys=city%2Cstatus&interests=%D0%9E%D0%B1%D1%89%D0%B5%D0%BD%D0%B8%D0%B5&birth=1990&limit=20 21 | // interests=%D0%98%D0%BD%D1%82%D0%B5%D1%80%D0%BD%D0%B5%D1%82&order=1&query_id=1320&limit=50&keys=city%2Csex 22 | // email_gt=wa&country_null=1&query_id=720&limit=32&sex_eq=f 23 | // joined=2013&order=1&query_id=1680&keys=country&interests=%D0%9F%D0%BB%D0%B0%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5&limit=30 24 | // country=%D0%A0%D0%BE%D1%81%D0%B0%D1%82%D1%80%D0%B8%D1%81&query_id=240&limit=16 25 | // country=%D0%9C%D0%B0%D0%BB%D0%BC%D0%B0%D0%BB%D1%8C&query_id=1440&limit=2 26 | // query_id=600&order=-1&sex=f&limit=10&keys=country%2Cstatus 27 | // query_id=1560&interests_contains=%D0%9C%D0%B0%D1%81%D1%81%D0%B0%D0%B6%2C%D0%A2%D1%8F%D0%B6%D1%91%D0%BB%D0%B0%D1%8F+%D0%B0%D1%82%D0%BB%D0%B5%D1%82%D0%B8%D0%BA%D0%B0%2C%D0%9A%D0%B8%D0%BD%D0%BE%D0%BA%D0%B8%D1%84%D0%B8%D0%BB%D1%8C%D0%BC%D1%8B%2C%D0%A7%D1%83%D0%B4%D0%B0%D0%BA%2C%D0%9F%D0%BE%D0%BF+%D1%80%D0%BE%D0%BA&status_neq=%D0%B7%D0%B0%D0%BD%D1%8F%D1%82%D1%8B&limit=2 28 | // country_null=0&query_id=1920&interests_any=%D0%9C%D0%BE%D1%80%D0%BE%D0%B6%D0%B5%D0%BD%D0%BE%D0%B5%2C%D0%A1%D0%BE%D0%BD&sex_eq=f&limit=18&status_eq=%D1%81%D0%B2%D0%BE%D0%B1%D0%BE%D0%B4%D0%BD%D1%8B 29 | 30 | } 31 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/bitset.go: -------------------------------------------------------------------------------- 1 | package hlc18 2 | 3 | type Bitset256 [4]uint64 4 | type Bitset128 [2]uint64 5 | 6 | func (p *Bitset256) Set(bit byte) { 7 | p[bit/64] = p[bit/64] | (1 << (bit % 64)) 8 | } 9 | 10 | func (p *Bitset256) Get(bit byte) bool { 11 | return 1 == ((p[bit/64] >> (bit % 64)) & 0x01) 12 | } 13 | 14 | func (p *Bitset256) Contains(p2 *Bitset256) bool { 15 | return (p[0]&p2[0] == p2[0]) && 16 | (p[1]&p2[1] == p2[1]) && 17 | (p[2]&p2[2] == p2[2]) && 18 | (p[3]&p2[3] == p2[3]) 19 | } 20 | 21 | func (p *Bitset256) Any(p2 *Bitset256) bool { 22 | return (p[0]&p2[0] != 0) || 23 | (p[1]&p2[1] != 0) || 24 | (p[2]&p2[2] != 0) || 25 | (p[3]&p2[3] != 0) 26 | } 27 | 28 | func (p *Bitset256) IsNotEmpty() bool { 29 | return p[0] != 0 || p[1] != 0 || p[2] != 0 || p[3] != 0 30 | } 31 | 32 | func (p *Bitset256) IsEmpty() bool { 33 | return !p.IsNotEmpty() 34 | } 35 | 36 | func (p *Bitset128) Set(bit byte) { 37 | p[bit/64] = p[bit/64] | (1 << (bit % 64)) 38 | } 39 | 40 | func (p *Bitset128) Get(bit byte) bool { 41 | return 1 == ((p[bit/64] >> (bit % 64)) & 0x01) 42 | } 43 | 44 | func (p *Bitset128) Contains(p2 *Bitset128) bool { 45 | return (p[0]&p2[0] == p2[0]) && (p[1]&p2[1] == p2[1]) 46 | } 47 | 48 | func (p *Bitset128) Any(p2 *Bitset128) bool { 49 | return (p[0]&p2[0] != 0) || (p[1]&p2[1] != 0) 50 | } 51 | 52 | func (p *Bitset128) IsNotEmpty() bool { 53 | return p[0] != 0 || p[1] != 0 54 | } 55 | 56 | func (p *Bitset128) IsEmpty() bool { 57 | return !p.IsNotEmpty() 58 | } 59 | 60 | func (p *Bitset128) Count() (count int) { 61 | n := p[0] 62 | for n != 0 { 63 | count++ 64 | n &= n - 1 // Zero the lowest-order one-bit 65 | } 66 | n = p[1] 67 | for n != 0 { 68 | count++ 69 | n &= n - 1 // Zero the lowest-order one-bit 70 | } 71 | return count 72 | } 73 | 74 | func (p *Bitset128) Reset() { 75 | p[0] = 0 76 | p[1] = 0 77 | } 78 | 79 | func Bitset128Set(bit byte, t0 *uint64, t1 *uint64) { 80 | //p[bit / 64] = p[bit / 64] | (1 << (bit % 64)) 81 | if 0 == bit/64 { 82 | *t0 = *t0 | (1 << (bit % 64)) 83 | } else { 84 | *t1 = *t1 | (1 << (bit % 64)) 85 | } 86 | 87 | } 88 | 89 | func Bitset128Get(bit byte, t0 *uint64, t1 *uint64) bool { 90 | if 0 == bit/64 { 91 | return 1 == ((*t0 >> (bit % 64)) & 0x01) 92 | } else { 93 | return 1 == ((*t1 >> (bit % 64)) & 0x01) 94 | } 95 | } 96 | 97 | func Bitset128Contains(t0 *uint64, t1 *uint64, p2 *Bitset128) bool { 98 | return (*t0&p2[0] == p2[0]) && (*t1&p2[1] == p2[1]) 99 | } 100 | 101 | func Bitset128Any(t0 *uint64, t1 *uint64, p2 *Bitset128) bool { 102 | return (*t0&p2[0] != 0) || (*t1&p2[1] != 0) 103 | } 104 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/evio-http-server.go: -------------------------------------------------------------------------------- 1 | package hlc18 2 | 3 | import ( 4 | "fmt" 5 | "github.com/valyala/fasthttp" 6 | "log" 7 | "strconv" 8 | "strings" 9 | "tidwall/evio" 10 | ) 11 | 12 | type Request struct { 13 | Proto, Method, Path, Query, Head, Body string 14 | } 15 | 16 | type Context struct { 17 | Is evio.InputStream 18 | Out [8192]byte 19 | } 20 | 21 | func EvioServer(port int, handler func(c evio.Conn, in []byte) (out []byte, action evio.Action)) { 22 | 23 | var events evio.Events 24 | 25 | events.NumLoops = 4 26 | events.LoadBalance = evio.LeastConnections // todo: ??? 27 | 28 | events.Serving = func(srv evio.Server) (action evio.Action) { 29 | fmt.Printf("%v\thttp server started on port %d (loops: %d)\n", Timenow(), port, srv.NumLoops) 30 | return 31 | } 32 | 33 | events.Opened = func(c evio.Conn) (out []byte, opts evio.Options, action evio.Action) { 34 | c.SetContext(&Context{}) 35 | opts.ReuseInputBuffer = true 36 | //opts.TCPKeepAlive 37 | //log.Printf("closed: %s: %s", c.LocalAddr().String(), c.RemoteAddr().String()) 38 | return 39 | } 40 | 41 | events.Data = handler 42 | 43 | if err := evio.Serve(events, fmt.Sprintf("tcp4://:%d", port)); err != nil { // ?reuseport=true 44 | log.Fatal(err) 45 | } 46 | } 47 | 48 | func AppendHttpResponse(b []byte, status, headers string, body []byte) []byte { 49 | b = append(b, "HTTP/1.1 "...) 50 | b = append(b, status...) 51 | b = append(b, "\r\nS: e\r\nD: S\r\nC: a\r\n"...) 52 | if len(body) > 0 { 53 | b = append(b, "Content-Length: "...) 54 | b = fasthttp.AppendUint(b, len(body)) 55 | b = append(b, '\r', '\n') 56 | } else { 57 | b = append(b, "Content-Length: 0\r\n"...) 58 | } 59 | b = append(b, '\r', '\n') 60 | if len(body) > 0 { 61 | b = append(b, body...) 62 | } 63 | return b 64 | } 65 | 66 | func Parsereq(data []byte, req *Request) (leftover []byte, err error) { 67 | 68 | if len(data) < 6 { 69 | // not enough data 70 | return data, nil 71 | } 72 | 73 | sdata := B2s(data) // string(data) 74 | var i, s int 75 | var top string 76 | var clen int 77 | var q = -1 78 | 79 | // Method, Path, Proto line 80 | for ; i < len(sdata); i++ { 81 | if sdata[i] == ' ' { 82 | req.Method = sdata[s:i] 83 | for i, s = i+1, i+1; i < len(sdata); i++ { 84 | if sdata[i] == '?' && q == -1 { 85 | q = i - s 86 | } else if sdata[i] == ' ' { 87 | if q != -1 { 88 | req.Path = sdata[s : s+q] //sdata[s:q] 89 | req.Query = sdata[s+q+1 : i] // req.Path[q+1 : i] 90 | } else { 91 | req.Path = sdata[s:i] 92 | } 93 | for i, s = i+1, i+1; i < len(sdata); i++ { 94 | if sdata[i] == '\n' && sdata[i-1] == '\r' { 95 | req.Proto = sdata[s:i] 96 | i, s = i+1, i+1 97 | break 98 | } 99 | } 100 | break 101 | } 102 | } 103 | break 104 | } 105 | } 106 | if req.Proto == "" { 107 | return data, fmt.Errorf("malformed Request - empty proto, data len %d, sdata: '%s'", len(sdata), sdata) 108 | } 109 | top = sdata[:s] 110 | for ; i < len(sdata); i++ { 111 | if i > 1 && sdata[i] == '\n' && sdata[i-1] == '\r' { 112 | line := sdata[s : i-1] 113 | s = i + 1 114 | if line == "" { 115 | req.Head = sdata[len(top)+2 : i+1] 116 | i++ 117 | if clen > 0 { 118 | if len(sdata[i:]) < clen { 119 | break 120 | } 121 | req.Body = sdata[i : i+clen] 122 | i += clen 123 | } 124 | return data[i:], nil 125 | } 126 | if strings.HasPrefix(line, "Content-Length:") { 127 | n, err := strconv.ParseInt(strings.TrimSpace(line[len("Content-Length:"):]), 10, 64) 128 | if err == nil { 129 | clen = int(n) 130 | } 131 | } 132 | } 133 | } 134 | // not enough data 135 | return data, nil 136 | } 137 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "dsmnko.org/hlc18" 5 | "fmt" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestDatesTs(t *testing.T) { 11 | fmt.Printf("date %v\n", time.Unix(1391299200, 0).UTC()) 12 | fmt.Printf("bitrh %v\n", time.Unix(596264959, 0).UTC()) 13 | 14 | fmt.Printf("like.ts %v\n", time.Unix(1480017567, 0).UTC()) 15 | 16 | fmt.Printf("joined %v\n", time.Unix(1332115200, 0).UTC()) 17 | fmt.Printf("now %v\n", time.Unix(1545834028, 0).UTC()) 18 | fmt.Printf("now-real %d\n", time.Now().UTC().Unix()) 19 | 20 | fmt.Printf("joined-max %d, %v\n", hlc18.MaxJoinedMinPremium, time.Unix(int64(hlc18.MaxJoinedMinPremium), 0).UTC()) 21 | fmt.Printf("joined-min %d, %v\n", hlc18.MinJoined, time.Unix(int64(hlc18.MinJoined), 0).UTC()) 22 | 23 | //fmt.Printf("min-premium %v\n", timeStampOf("2018-01-01T00:00:00Z")) 24 | //parsed := timeStampOf("1950-01-01T00:00:00Z") 25 | //fmt.Printf("%v, %v, %v\n", time.Unix(803238391, 0).UTC(), parsed, uint32(parsed)) 26 | //fmt.Printf("min %v\n", time.Unix(int64(parsed), 0).UTC()) 27 | //fmt.Printf("uint32 min %v\n", time.Unix(int64(uint32(parsed)), 0).UTC()) 28 | 29 | //fmt.Printf("joined-min %d, %v\n", minJoined, time.Unix(int64(minJoined), 0).UTC()) 30 | //fmt.Printf("joined-max %d, %v\n", maxJoinedMinPremium, time.Unix(int64(maxJoinedMinPremium), 0).UTC()) 31 | 32 | // fmt.Println("Time parsing"); 33 | // dateString := "2014-11-12T00:00:00.00 0Z" 34 | // t, e := time.Parse(time.RFC3339,dateString) { 35 | // } 36 | } 37 | 38 | /* 39 | func TestLikes(t *testing.T) { 40 | 41 | json := `{"likes":[ 42 | {"likee": 3929, "ts": 1464869768, "liker": 25486}, 43 | {"likee": 13239, "ts": 1431103000, "liker": 26727}, 44 | {"likee": 2407, "ts": 1439604510, "liker": 6403}, 45 | {"likee": 26677, "ts": 1454719940, "liker": 22248}, 46 | {"likee": 22411, "ts": 1481309376, "liker": 32820}, 47 | {"likee": 9747, "ts": 1431850118, "liker": 43794}, 48 | {"likee": 43575, "ts": 1499496173, "liker": 16134}, 49 | {"likee": 29725, "ts": 1479087147, "liker": 22248} 50 | ]}` 51 | 52 | likes := hlc18.LikesJson{} 53 | if err := likes.UnmarshalJSON([]byte(json)); err != nil { 54 | t.Error("UnmarshalJSON failed", err) 55 | } 56 | 57 | json_e1 := `{"likes":[{"likeS":1090354,"likee":1265187,"ts":1539147049},{"liker":170169,"likee":1218842,"ts":1466399211},{"liker":1230885,"likee":1241432,"ts":1502073097},{"liker":159840,"likee":780037,"ts":1454571186},{"liker":170169,"likee":197962,"ts":1530321211},{"liker":21501,"likee":370380,"ts":1489574354},{"liker":917010,"likee":1277359,"ts":1541567544},{"liker":1041411,"likee":924176,"ts":1528403302},{"liker":917010,"likee":1143455,"ts":1528370621},{"liker":917010,"likee":770291,"ts":1502086250},{"liker":917010,"likee":165963,"ts":1531346942},{"liker":170169,"likee":260866,"ts":1469019928},{"liker":159840,"likee":325585,"ts":1460007344},{"liker":917010,"likee":46193,"ts":1496548847},{"liker":170169,"likee":107890,"ts":1475811909},{"liker":159840,"likee":351503,"ts":1501461434},{"liker":917010,"likee":378435,"ts":1474619345},{"liker":1041411,"likee":143608,"ts":1498102770},{"liker":1090354,"likee":862719,"ts":1470049747},{"liker":917010,"likee":646907,"ts":1468621932},{"liker":21501,"likee":852484,"ts":1485352282},{"liker":917010,"likee":91715,"ts":1501137002},{"liker":21501,"likee":631966,"ts":1512659286},{"liker":21501,"likee":530578,"ts":1498079095},{"liker":151910,"likee":1630163,"ts":1535876732},{"liker":1090354,"likee":1118859,"ts":1528514719},{"liker":1090354,"likee":1148473,"ts":1488293405},{"liker":1230885,"likee":505838,"ts":1524076400},{"liker":21501,"likee":105292,"ts":1490557679},{"liker":159840,"likee":263047,"ts":1540004458},{"liker":1117672,"likee":226741,"ts":1453448609}]}` 58 | if err := likes.UnmarshalJSON([]byte(json_e1)); err == nil { 59 | t.Error("UnmarshalJSON failed", err) 60 | } else { 61 | fmt.Print(err) 62 | } 63 | } 64 | 65 | func TestFilter(t *testing.T) { 66 | 67 | var accounts = make([]Account, 30000) 68 | maxId, e := loadDataArray(accounts) 69 | 70 | if maxId != 30000 || e != nil { 71 | t.Error("load fail") 72 | } 73 | params := make(map[string]string) 74 | params["sname_null"] = "0" 75 | params["sex_eq"] = "m" 76 | filter, e := makeFilter(params) 77 | account := &accounts[29999-1] 78 | writeAccount(os.Stdout, false, filter, account, 29999) 79 | if b, e := filter.test(account); !b || e != nil { 80 | t.Errorf("29999 test failed: %v,%v, for filter: %#v", b, e, filter) 81 | } 82 | } 83 | */ 84 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/cache.go: -------------------------------------------------------------------------------- 1 | package hlc18 2 | 3 | import ( 4 | "errors" 5 | "sync" 6 | ) 7 | 8 | // todo: по suggest & recommend можно попробовать считать (и кешировать) всегда по 20, но, есть сомнение что это окажется эффективнее чем сейчас... 9 | 10 | type CacheKey struct { 11 | id uint32 12 | cityCode uint16 13 | countryCode byte 14 | } 15 | 16 | var recommendL2CacheMutex sync.RWMutex 17 | var recommendL2Cache map[CacheKey][]uint32 18 | 19 | var suggestL2CacheMutex sync.RWMutex 20 | var suggestL2Cache map[CacheKey][]uint32 21 | 22 | var groupL2Cache map[GroupCacheKey][]GroupItem 23 | var groupL2CacheMutex sync.RWMutex 24 | 25 | type Reply struct { 26 | Status int 27 | Body []byte 28 | } 29 | 30 | var filterCache map[string][]byte 31 | var filterCacheMutex sync.Mutex 32 | 33 | var groupCache map[string][]byte 34 | var groupCacheMutex sync.Mutex 35 | 36 | var suggestCache map[string][]byte 37 | var suggestCacheMutex sync.Mutex 38 | 39 | var recommendCache map[string][]byte 40 | var recommendCacheMutex sync.Mutex 41 | 42 | var errorNotFound = errors.New("not found") 43 | 44 | func ResetCaches() { 45 | recommendL2CacheMutex.Lock() 46 | recommendL2Cache = make(map[CacheKey][]uint32, 22000) 47 | recommendL2CacheMutex.Unlock() 48 | 49 | suggestL2CacheMutex.Lock() 50 | suggestL2Cache = make(map[CacheKey][]uint32, 13000) 51 | suggestL2CacheMutex.Unlock() 52 | 53 | filterCacheMutex.Lock() 54 | filterCache = make(map[string][]byte, 32000) 55 | filterCacheMutex.Unlock() 56 | 57 | groupCacheMutex.Lock() 58 | groupCache = make(map[string][]byte, 12500) 59 | groupCacheMutex.Unlock() 60 | 61 | groupL2CacheMutex.Lock() 62 | groupL2Cache = make(map[GroupCacheKey][]GroupItem, 25000) 63 | groupL2CacheMutex.Unlock() 64 | 65 | recommendCacheMutex.Lock() 66 | recommendCache = make(map[string][]byte, 11000) 67 | recommendCacheMutex.Unlock() 68 | 69 | suggestCacheMutex.Lock() 70 | suggestCache = make(map[string][]byte, 6500) 71 | suggestCacheMutex.Unlock() 72 | } 73 | 74 | func getRecommendL2Cache(token CacheKey, limit int) ([]uint32, error) { 75 | recommendL2CacheMutex.RLock() 76 | if res, found := recommendL2Cache[token]; found && res != nil { 77 | recommendL2CacheMutex.RUnlock() 78 | if len(res) >= limit { 79 | return res[:limit], nil 80 | } 81 | return nil, errorNotFound 82 | } 83 | recommendL2CacheMutex.RUnlock() 84 | return nil, errorNotFound 85 | } 86 | 87 | func putRecommendL2Cache(token CacheKey, value []uint32) { 88 | if value != nil { 89 | recommendL2CacheMutex.Lock() 90 | old := recommendL2Cache[token] 91 | if len(old) < len(value) { 92 | valueCopy := append([]uint32{}, value...) 93 | recommendL2Cache[token] = valueCopy 94 | } 95 | recommendL2CacheMutex.Unlock() 96 | } 97 | } 98 | 99 | func getSuggestL2Cache(token CacheKey, limit int) ([]uint32, error) { 100 | suggestL2CacheMutex.RLock() 101 | if res, found := suggestL2Cache[token]; found && res != nil { 102 | suggestL2CacheMutex.RUnlock() 103 | if len(res) >= limit { 104 | return res[:limit], nil 105 | } 106 | return nil, errorNotFound 107 | } 108 | suggestL2CacheMutex.RUnlock() 109 | return nil, errorNotFound 110 | } 111 | 112 | func putSuggestL2Cache(token CacheKey, value []uint32) { 113 | if value != nil { 114 | suggestL2CacheMutex.Lock() 115 | old := suggestL2Cache[token] 116 | if len(old) < len(value) { 117 | valueCopy := append([]uint32{}, value...) 118 | suggestL2Cache[token] = valueCopy 119 | } 120 | suggestL2CacheMutex.Unlock() 121 | } 122 | } 123 | 124 | func GetFilterCache(token string) []byte { 125 | return filterCache[token] 126 | } 127 | 128 | func PutFilterCache(token string, value []byte) { 129 | reply := append([]byte{}, value...) 130 | filterCacheMutex.Lock() 131 | filterCache[token] = reply 132 | filterCacheMutex.Unlock() 133 | } 134 | 135 | func GetGroupCache(token string) []byte { 136 | return groupCache[token] 137 | } 138 | 139 | func PutGroupCache(token string, value []byte) { 140 | reply := append([]byte{}, value...) 141 | groupCacheMutex.Lock() 142 | groupCache[token] = reply 143 | groupCacheMutex.Unlock() 144 | } 145 | 146 | func GetSuggestCache(token string) []byte { 147 | return suggestCache[token] 148 | } 149 | 150 | func PutSuggestCache(token string, value []byte) { 151 | reply := append([]byte{}, value...) 152 | suggestCacheMutex.Lock() 153 | suggestCache[token] = reply 154 | suggestCacheMutex.Unlock() 155 | } 156 | 157 | func GetRecommendCache(token string) []byte { 158 | return recommendCache[token] 159 | } 160 | 161 | func PutRecommendCache(token string, value []byte) { 162 | reply := append([]byte{}, value...) 163 | recommendCacheMutex.Lock() 164 | recommendCache[token] = reply 165 | recommendCacheMutex.Unlock() 166 | } 167 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/suggest_test.go: -------------------------------------------------------------------------------- 1 | package hlc18 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "reflect" 7 | "testing" 8 | ) 9 | 10 | func TestSuggest(t *testing.T) { 11 | accounts := loadTestData(t) 12 | if l := len(accounts); l != 30000 { 13 | t.Fatal("load failed") 14 | } 15 | 16 | var rec []uint32 17 | var str string 18 | 19 | str, _ = url.PathUnescape("/accounts/20126/suggest/?query_id=518&limit=6") 20 | fmt.Println(str) 21 | 22 | ///accounts/20126/suggest/?query_id=518&limit=6 23 | //REQUEST URI: /accounts/20126/suggest/?query_id=518&limit=6 24 | //REQUEST BODY: 25 | //BODY GOT: {"accounts":[]} 26 | //BODY EXP: {"accounts":[{"sname":"\u0421\u0442\u0430\u043c\u044b\u043a\u0430\u043a\u0438\u0439","status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0414\u0430\u043d\u0438\u043b\u0430","email":"ilhetahidvesitylit@inbox.com","id":20049}, 27 | // {"sname":"\u0414\u0430\u043d\u043e\u043b\u043e\u0432\u0438\u0447","status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u041e\u043b\u0435\u0433","email":"utnele@yandex.ru","id":19487}, 28 | // {"sname":"\u0422\u0435\u0440\u0430\u0448\u0435\u0432\u0438\u0447","status":"\u0432\u0441\u0451 \u0441\u043b\u043e\u0436\u043d\u043e","fname":"\u0415\u0433\u043e\u0440","email":"odtuqwatudetve@yahoo.com","id":16585}, 29 | // {"sname":"\u0425\u043e\u043f\u0430\u0442\u043e\u0441\u044f\u043d","status":"\u0437\u0430\u043d\u044f\u0442\u044b","fname":"\u0412\u044f\u0447\u0435\u0441\u043b\u0430\u0432","email":"takletgobnattepadreti@ya.ru","id":15317}, 30 | // {"sname":"\u0414\u0430\u043d\u044b\u043a\u0430\u0442\u0438\u043d","status":"\u0432\u0441\u0451 \u0441\u043b\u043e\u0436\u043d\u043e","fname":"\u0421\u0435\u043c\u0451\u043d","email":"fehase@ya.ru","id":13417}, 31 | // {"sname":"\u0424\u0435\u0442\u0435\u0442\u0430\u0447\u0430\u043d","status":"\u0437\u0430\u043d\u044f\u0442\u044b","fname":"\u0415\u0433\u043e\u0440","email":"vadebroawasam@list.ru","id":8169}]} 32 | 33 | rec = Suggest(20126, 6, map[string]string{}) 34 | if len(rec) != 6 { 35 | t.Error("20126 failed, len(rec) != 6") 36 | } 37 | if len(rec) > 0 && rec[0] != 20049 { 38 | t.Error("20126 failed, rec[0] != 20049") 39 | } 40 | fmt.Printf("%v\n", rec) 41 | 42 | // REQUEST URI: /accounts/5784/suggest/?country=%D0%98%D1%81%D0%BF%D0%B5%D0%B7%D0%B8%D1%8F&query_id=1850&limit=10 43 | // REQUEST BODY: 44 | // BODY GOT: {"accounts":[{"id":29999,"email":"fyshaffatenodladha@yandex.ru","sname":"Стамашелан","status":"свободны","fname":"Алексей"},{"id":29513,"email":"tagredlavemo@mail.ru","status":"свободны","fname":"Роман","sname":"Колленпов"},{"id":29477,"email":"datalsenunpi@yandex.ru","status":"свободны","fname":"Даниил","sname":"Хопатотин"},{"id":29043,"email":"ehnesavtar@mail.ru","status":"свободны","fname":"Евгений"},{"id":28937,"email":"datososercaives@inbox.com","status":"свободны","fname":"Владимир","sname":"Фаашекий"},{"id":28937,"email":"datososercaives@inbox.com","status":"свободны","fname":"Владимир","sname":"Фаашекий"},{"id":28877,"email":"amsidehuhnarwinac@mail.ru","fname":"Степан","sname":"Пенушусян","status":"свободны"},{"id":28653,"email":"egnanysmasomrotow@me.com","status":"свободны","fname":"Никита","sname":"Терушусян"},{"id":28511,"email":"petetietguwnafan@inbox.ru","status":"свободны","fname":"Алексей","sname":"Колыкако"},{"id":28505,"email":"mysdyllotfacugit@me.com","sname":"Данашело","status":"всё сложно","fname":"Леонид"}]} 45 | // BODY EXP: {"accounts":[{"sname":"\u0421\u0442\u0430\u043c\u0430\u0448\u0435\u043b\u0430\u043d","status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0410\u043b\u0435\u043a\u0441\u0435\u0439","email":"fyshaffatenodladha@yandex.ru", 46 | // "id":29999},{"sname":"\u041a\u043e\u043b\u043b\u0435\u043d\u043f\u043e\u0432","status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0420\u043e\u043c\u0430\u043d","email":"tagredlavemo@mail.ru" 47 | // "id":29513},{"sname":"\u0425\u043e\u043f\u0430\u0442\u043e\u0442\u0438\u043d","status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0414\u0430\u043d\u0438\u0438\u043b","email":"datalsenunpi@yandex.ru", 48 | // "id":29477}, 49 | // "id":29043,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0415\u0432\u0433\u0435\u043d\u0438\u0439","email":"ehnesavtar@mail.ru"},{"sname":"\u0424\u0430\u0430\u0448\u0435\u043a\u0438\u0439","status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0412\u043b\u0430\u0434\u0438\u043c\u0438\u0440","email":"datososercaives@inbox.com", 50 | // "id":28937},{"sname":"\u041f\u0435\u043d\u0443\u0448\u0443\u0441\u044f\u043d","status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0421\u0442\u0435\u043f\u0430\u043d","email":"amsidehuhnarwinac@mail.ru", 51 | // "id":28877},{"sname":"\u0422\u0435\u0440\u0443\u0448\u0443\u0441\u044f\u043d","status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u041d\u0438\u043a\u0438\u0442\u0430","email":"egnanysmasomrotow@me.com", 52 | // "id":28653},{"sname":"\u041a\u043e\u043b\u044b\u043a\u0430\u043a\u043e","status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0410\u043b\u0435\u043a\u0441\u0435\u0439","email":"petetietguwnafan@inbox.ru", 53 | // "id":28511},{"sname":"\u0414\u0430\u043d\u0430\u0448\u0435\u043b\u043e","status":"\u0432\u0441\u0451 \u0441\u043b\u043e\u0436\u043d\u043e","fname":"\u041b\u0435\u043e\u043d\u0438\u0434","email":"mysdyllotfacugit@me.com", 54 | // "id":28505},{"sname":"\u041f\u0435\u043d\u0443\u0448\u0443\u0447\u0430\u043d","status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0421\u0435\u0440\u0433\u0435\u0439","email":"romotgalenic@email.com", 55 | // "id":27775}]} 56 | 57 | str, _ = url.PathUnescape("/accounts/5784/suggest/?country=%D0%98%D1%81%D0%BF%D0%B5%D0%B7%D0%B8%D1%8F&query_id=1850&limit=10") 58 | fmt.Println(str) 59 | 60 | rec = Suggest(5784, 10, map[string]string{"country": "Испезия"}) 61 | if len(rec) != 10 { 62 | t.Error("5784 failed, len(rec) != 10") 63 | } 64 | if len(rec) > 0 && rec[0] != 29999 { 65 | t.Error("5784 failed, rec[0] != 29999") 66 | } 67 | 68 | awaited := []uint32{ 69 | 29999, 70 | 29513, 71 | 29477, 72 | 29043, 73 | 28937, 74 | 28877, 75 | 28653, 76 | 28511, 77 | 28505, 78 | 27775, 79 | } 80 | for i, id := range rec { 81 | if id != awaited[i] { 82 | t.Errorf("5784 failed, rec[%d]%d != %d\n", i, id, awaited[i]) 83 | } 84 | } 85 | fmt.Printf("%v\n", rec) 86 | 87 | // cache 88 | rec2 := Suggest(5784, 10, map[string]string{"country": "Испезия"}) 89 | if !reflect.DeepEqual(rec2, rec) { 90 | t.Error("cache failed, ", str) 91 | } 92 | 93 | } 94 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/suggest.go: -------------------------------------------------------------------------------- 1 | package hlc18 2 | 3 | import ( 4 | "sort" 5 | "sync" 6 | ) 7 | 8 | /* 9 | Подбор по похожим симпатиям: /accounts//suggest/ 10 | 11 | Этот тип запросов похож на предыдущий тем, что он тоже про поиск "вторых половинок". 12 | Аналогично пересылается id пользователя, для которого мы ищем вторую половинку и аналогично используется GET-параметер limit. 13 | Различия в реализации. Теперь мы ищем, кого лайкают пользователи того же пола с похожими "симпатиями" и предлагаем тех, 14 | кого они недавно лайкали сами. В случае, если в запросе передан GET-параметр country или city, 15 | то искать "похожие симпатии" нужно только в определённой локации. 16 | 17 | Похожесть симпатий определим как функцию: similarity = f (me, account), 18 | которая вычисляется однозначно как сумма из дробей 1 / abs(my_like['ts'] - like['ts']), 19 | где my_like и like - это симпатии к одному и тому же пользователю. 20 | Для дроби, где my_like['ts'] == like['ts'], заменяем дробь на 1. 21 | Если общих лайков нет, то стоит считать пользователей абсолютно непохожими с similarity = 0. 22 | Если у одного аккаунта есть несколько лайков на одного и того же пользователя с разными датами, 23 | то в формуле используется среднее арифметическое их дат. 24 | 25 | В ответе возвращается список тех, кого ещё не лайкал пользователь с указанным id, 26 | но кого лайкали пользователи с самыми похожими симпатиями. 27 | Сортировка по убыванию похожести, 28 | а между лайками одного такого пользователя - по убыванию id лайка. 29 | 30 | */ 31 | 32 | // todo: попробовать float32 ? 33 | func similarity(likeId, likeTs uint32, other []Like) float32 { 34 | if otherTs := likeTsOf(other, likeId); otherTs != likeTs { 35 | return float32(1.0) / float32(abs(int32(likeTs)-int32(otherTs))) 36 | } 37 | return 1.0 38 | } 39 | 40 | type Other struct { 41 | id uint32 42 | weight float32 43 | //index int 44 | } 45 | 46 | var othersMapPool = sync.Pool{ 47 | New: func() interface{} { 48 | return make(map[uint32]float32, 3200) 49 | }, 50 | } 51 | 52 | func othersMapBorrow() map[uint32]float32 { 53 | return othersMapPool.Get().(map[uint32]float32) 54 | } 55 | 56 | func othersMapRelease(buffer map[uint32]float32) { 57 | for key := range buffer { 58 | delete(buffer, key) 59 | } 60 | othersMapPool.Put(buffer) 61 | } 62 | 63 | // todo: !!! "утоптать" лайки после фазы-2 64 | func Suggest(myId uint32, limit int, params map[string]string) []uint32 { 65 | 66 | var cityCode uint16 67 | var countryCode byte 68 | 69 | if v, _ := params["city"]; v != "" { 70 | if cityCode = CityDict.ids[v]; cityCode == 0 { 71 | return []uint32{} 72 | } 73 | } else if v, _ := params["country"]; v != "" { 74 | if countryCode = CountryDict.ids[v]; countryCode == 0 { 75 | return []uint32{} 76 | } 77 | } 78 | 79 | cacheKey := CacheKey{myId, cityCode, countryCode} 80 | if cache, e := getSuggestL2Cache(cacheKey, limit); e == nil { 81 | return cache 82 | } 83 | 84 | me := Store.Likes[myId-1] 85 | 86 | if len(me) == 0 { 87 | putSuggestL2Cache(cacheKey, []uint32{}) 88 | return []uint32{} 89 | } 90 | 91 | // найдем всех с кем одинаково лакайли 92 | // кто -> вес 93 | othersMap := othersMapBorrow() // make(map[uint32]float32, 3200) 94 | 95 | // likeid -> ts (unique) 96 | mylikes := make(map[uint32]uint32, len(me)) 97 | 98 | //index := 0 99 | //othersMapIndex := make(map[uint32]int) 100 | 101 | for i := 0; i < len(me); i++ { 102 | likeId := me[i].Id 103 | mylikes[likeId] = likeTsOf(me, likeId) 104 | } 105 | 106 | for likeId, likeTs := range mylikes { //i := 0; i < me.likesId.Len(); i++ { 107 | // if vector := LikesIndex[likeId-1]; vector != nil { 108 | // vector.Iterate(func(_ int, otherId uint32) bool { 109 | // otherAccount := &Store.Accounts[otherId-1] 110 | // if otherId != myId && 111 | // (countryCode == 0 || countryCode == otherAccount.getCountry()) && 112 | // (cityCode == 0 || cityCode == otherAccount.getCity()) { 113 | // othersMap[otherId] += similarity(likeId, likeTs, &Store.Accounts2[otherId-1]) 114 | // } 115 | // return true 116 | // }) 117 | // } 118 | 119 | var prevOther uint32 120 | for _, otherId := range LikesIndexCompact[likeId-1] { 121 | if otherId != myId && prevOther != otherId { 122 | prevOther = otherId 123 | otherAccount := &Store.Accounts[otherId-1] 124 | if (countryCode == 0 || countryCode == otherAccount.getCountry()) && 125 | (cityCode == 0 || cityCode == otherAccount.getCity()) { 126 | othersMap[otherId] += similarity(likeId, likeTs, Store.Likes[otherId-1]) 127 | //if _, found := othersMapIndex[otherId]; !found { 128 | // othersMapIndex[otherId] = index 129 | // index++ 130 | //} 131 | } 132 | } 133 | } 134 | 135 | } 136 | 137 | if len(othersMap) == 0 { 138 | othersMapRelease(othersMap) 139 | putSuggestL2Cache(cacheKey, []uint32{}) 140 | return []uint32{} 141 | } 142 | 143 | others := make([]Other, 0, len(othersMap)) 144 | 145 | for k, v := range othersMap { 146 | others = append(others, Other{k, v}) 147 | } 148 | 149 | othersMapRelease(othersMap) 150 | 151 | sort.Slice(others, func(i, j int) bool { 152 | //return others[i].weight > others[j].weight 153 | if diff := others[i].weight - others[j].weight; diff > 0 { 154 | return true 155 | } else if diff < 0 { 156 | return false 157 | } else { 158 | return others[i].id < others[j].id 159 | } 160 | }) 161 | 162 | //fmt.Printf("%v\n", others) 163 | 164 | result := make([]uint32, 20)[:0] 165 | 166 | //seen := make(map[uint32]bool) 167 | 168 | suggestedBuffer := make([]uint32, 0, 128) 169 | 170 | //println(len(others)) 3200 max 171 | 172 | for _, otherId := range others { 173 | suggested := suggestedBuffer 174 | other := Store.Likes[otherId.id-1] 175 | for i := 0; i < len(other); i++ { 176 | if likeId := other[i].Id; !containsLike(me, likeId) { 177 | suggested = append(suggested, likeId) 178 | } 179 | } 180 | // В ответе возвращается список тех, кого ещё не лайкал пользователь с указанным id, 181 | // но кого лайкали пользователи с самыми похожими симпатиями. 182 | // Сортировка по убыванию похожести, а между лайками одного такого пользователя - по убыванию id лайка. 183 | sort.Slice(suggested, func(i, j int) bool { 184 | return suggested[i] > suggested[j] 185 | }) 186 | 187 | for _, s := range suggested { 188 | if !seen(result, s) { // защита от дублей 189 | result = append(result, s) 190 | if rlen := len(result); rlen > 19 || rlen >= limit { 191 | putSuggestL2Cache(cacheKey, result) 192 | return result 193 | } 194 | } 195 | } 196 | } 197 | 198 | finalResult := result[:min(limit, len(result))] 199 | putSuggestL2Cache(cacheKey, finalResult) 200 | return finalResult 201 | } 202 | 203 | func seen(array []uint32, value uint32) bool { 204 | for _, v := range array { 205 | if v == value { 206 | return true 207 | } 208 | } 209 | return false 210 | } 211 | 212 | // todo: вычислять при переиндексации и т.д. 213 | func likeTsOf(likes []Like, likeId uint32) uint32 { 214 | count := int64(0) 215 | sum := int64(0) 216 | 217 | // for i := 0; i < len(likes); i++ { 218 | // if likeId == likes[i].Id { 219 | // count++ 220 | // sum += int64(likes[i].Ts) 221 | // } else if likes[i].Id > likeId { 222 | // break 223 | // } 224 | // } 225 | 226 | for i := 0; i < len(likes) && likes[i].Id <= likeId; i++ { 227 | if likeId == likes[i].Id { 228 | count++ 229 | sum += int64(likes[i].Ts) 230 | } 231 | } 232 | 233 | if count == 0 { 234 | println("todo: likeTsOf:count == 0") // todo 235 | return 0 236 | } 237 | return uint32(sum / count) 238 | } 239 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/data.go: -------------------------------------------------------------------------------- 1 | package hlc18 2 | 3 | import ( 4 | "errors" 5 | "github.com/valyala/fastjson" 6 | "sync" 7 | ) 8 | 9 | type LikeUpdate struct { 10 | Likee uint32 11 | Liker uint32 12 | Ts uint32 13 | } 14 | 15 | type Premium struct { 16 | Start int32 `json:"start"` 17 | Finish int32 `json:"finish"` 18 | } 19 | 20 | type Like struct { 21 | Id uint32 `json:"id"` 22 | Ts uint32 `json:"ts"` 23 | } 24 | 25 | //easyjson:json 26 | type AccountJson struct { 27 | Id uint32 `json:"id"` 28 | Fname string `json:"fname"` 29 | Sname string `json:"sname"` 30 | Email string `json:"email"` 31 | Interests []string `json:"interests"` 32 | Status string `json:"status"` 33 | Premium Premium `json:"premium"` 34 | Sex string `json:"sex"` 35 | Phone string `json:"phone"` 36 | Likes []Like `json:"likes"` 37 | Birth int32 `json:"birth"` 38 | City string `json:"city"` 39 | Country string `json:"country"` 40 | Joined int32 `json:"joined"` 41 | } 42 | 43 | var parserPool = sync.Pool{ 44 | New: func() interface{} { return &fastjson.Parser{} }, 45 | } 46 | 47 | var likeUpdatePool = sync.Pool{ 48 | New: func() interface{} { 49 | buffer := [1024]LikeUpdate{} 50 | return buffer[:] 51 | }, 52 | } 53 | 54 | func _LikesBufferBorrow() []LikeUpdate { 55 | return likeUpdatePool.Get().([]LikeUpdate) 56 | } 57 | 58 | func _LikesBufferRelease(buffer []LikeUpdate) { 59 | likeUpdatePool.Put(buffer) 60 | } 61 | 62 | func ParseLikesUpdate(bytes []byte) ([]LikeUpdate, error) { 63 | var parser = parserPool.Get().(*fastjson.Parser) 64 | defer parserPool.Put(parser) 65 | if value, err := parser.ParseBytes(bytes); err == nil { 66 | array := value.GetArray("likes") 67 | likes := make([]LikeUpdate, len(array)) 68 | for i, v := range array { 69 | likes[i].Ts = uint32(v.GetUint("ts")) 70 | likes[i].Liker = uint32(v.GetUint("liker")) 71 | likes[i].Likee = uint32(v.GetUint("likee")) 72 | } 73 | return likes, nil 74 | } else { 75 | return nil, err 76 | } 77 | } 78 | 79 | /* 80 | { 81 | "sname": "Хопетачан", 82 | "email": "orhograanenor@yahoo.com", 83 | "country": "Голция", 84 | "interests": [], 85 | "birth": 736598811, 86 | "id": 50000, 87 | "sex": "f", 88 | "likes": [ 89 | {"ts": 1475619112, "id": 38753}, 90 | {"ts": 1464366718, "id": 14893}, 91 | {"ts": 1510257477, "id": 37967}, 92 | {"ts": 1431722263, "id": 38933} 93 | ], 94 | "premium": {"start": 1519661251, "finish": 1522253251}, 95 | "status": "всё сложно", 96 | "fname": "Полина", 97 | "joined": 1466035200 98 | } 99 | */ 100 | var umnarshallError = errors.New("umnarshallError") 101 | 102 | func UnmarshalNew(account *AccountJson, bytes []byte) error { 103 | var parser = parserPool.Get().(*fastjson.Parser) 104 | defer parserPool.Put(parser) 105 | if value, err := parser.ParseBytes(bytes); err == nil { 106 | if account.Id = uint32(value.GetUint("id")); account.Id == 0 { 107 | return umnarshallError 108 | } 109 | account.Birth = int32(value.GetInt("birth")) 110 | account.Joined = int32(value.GetInt("joined")) 111 | if account.Birth == 0 || account.Joined == 0 { 112 | return umnarshallError 113 | } 114 | if vp := value.Get("premium"); vp != nil { 115 | account.Premium.Start = int32(vp.GetInt("start")) 116 | account.Premium.Finish = int32(vp.GetInt("finish")) 117 | if account.Premium.Start == 0 || account.Premium.Finish == 0 { 118 | return umnarshallError 119 | } 120 | } 121 | vls := value.GetArray("likes") 122 | account.Likes = make([]Like, len(vls), len(vls)) 123 | for i, vl := range vls { 124 | account.Likes[i].Id = uint32(vl.GetUint("id")) 125 | account.Likes[i].Ts = uint32(vl.GetUint("ts")) 126 | if account.Likes[i].Id == 0 || account.Likes[i].Ts == 0 { 127 | return umnarshallError 128 | } 129 | } 130 | 131 | // todo: по уму, все строки надо сразу херачить в словари 132 | account.Email = safeString(value.Get("email")) 133 | account.Phone = safeString(value.Get("phone")) 134 | account.Sex = safeString(value.Get("sex")) 135 | account.Fname = toString(value.Get("fname")) 136 | account.Sname = toString(value.Get("sname")) 137 | account.Status = toString(value.Get("status")) 138 | account.City = toString(value.Get("city")) 139 | account.Country = toString(value.Get("country")) 140 | 141 | vis := value.GetArray("interests") 142 | if len(vis) > 0 { 143 | account.Interests = make([]string, len(vis), len(vis)) 144 | for i, vi := range vis { 145 | account.Interests[i] = toString(vi) 146 | } 147 | } 148 | } else { 149 | return err 150 | } 151 | return nil 152 | } 153 | 154 | func UnmarshalUpdate(account *AccountJson, bytes []byte) error { 155 | var parser = parserPool.Get().(*fastjson.Parser) 156 | defer parserPool.Put(parser) 157 | if value, err := parser.ParseBytes(bytes); err == nil { 158 | 159 | if v := value.Get("birth"); v != nil { 160 | if account.Birth = int32(v.GetInt()); account.Birth == 0 { 161 | return umnarshallError 162 | } 163 | } 164 | if v := value.Get("joined"); v != nil { 165 | if account.Joined = int32(v.GetInt()); account.Joined == 0 { 166 | return umnarshallError 167 | } 168 | } 169 | if vp := value.Get("premium"); vp != nil { 170 | account.Premium.Start = int32(vp.GetInt("start")) 171 | account.Premium.Finish = int32(vp.GetInt("finish")) 172 | if account.Premium.Start == 0 || account.Premium.Finish == 0 { 173 | return umnarshallError 174 | } 175 | } 176 | 177 | // todo: по уму, все строки надо сразу херачить в словари 178 | account.Email = safeString(value.Get("email")) 179 | account.Phone = safeString(value.Get("phone")) 180 | account.Sex = safeString(value.Get("sex")) 181 | account.Fname = toString(value.Get("fname")) 182 | account.Sname = toString(value.Get("sname")) 183 | account.Status = toString(value.Get("status")) 184 | account.City = toString(value.Get("city")) 185 | account.Country = toString(value.Get("country")) 186 | 187 | if vis := value.GetArray("interests"); len(vis) > 0 { 188 | account.Interests = make([]string, len(vis), len(vis)) 189 | for i, vi := range vis { 190 | account.Interests[i] = toString(vi) 191 | } 192 | } 193 | 194 | if vls := value.GetArray("likes"); len(vls) > 0 { 195 | for _, vl := range vls { 196 | if vl.GetUint("id") == 0 || uint32(vl.GetUint("ts")) == 0 { 197 | return umnarshallError 198 | } 199 | } 200 | // todo : ? проверить что сюда не попадаем 201 | panic("oops!") 202 | 203 | } 204 | 205 | } else { 206 | return err 207 | } 208 | return nil 209 | } 210 | 211 | /* 212 | func (p *Storage) CompressToAccountDirect(value *fastjson.Value) (uint32, error) { 213 | 214 | id := uint32(value.GetUint("id")) 215 | 216 | acc2 := &p.Accounts2[id-1] 217 | acc := &p.Accounts[id-1] 218 | 219 | if email := safeString(value.Get("email")); email != "" { 220 | EmailMap[email] = id 221 | acc2.email = email 222 | if domain, _ := domainOf(email); domain == 0 { 223 | return 0, fmt.Errorf("invalid email: %v", value) 224 | } else { 225 | acc.setDomain(domain) 226 | } 227 | } else { 228 | acc2.email = "" 229 | acc.setDomain(0) 230 | } 231 | 232 | if phone := safeString(value.Get("phone")); phone != "" { 233 | PhoneMap[phone] = id 234 | acc2.phone = phone 235 | acc.setPhoneCode(phoneCodeOf(phone)) 236 | } else { 237 | acc2.phone = "" 238 | acc.setPhoneCode(0) 239 | } 240 | 241 | sex := safeString(value.Get("sex")) 242 | nameCode := FnameDict.put(toString(value.Get("fname"))) 243 | acc.setFname(nameCode) 244 | SexNames[sexCode(sex)-1][nameCode] = 1 245 | 246 | acc.setSname(SnameDict.put(toString(value.Get("sname")))) 247 | 248 | i := (id - 1) * 2 249 | setInterestsDirect(value.GetArray("interests"), &Store.interests[i], &Store.interests[i+1]) 250 | 251 | acc.setSex(sex) 252 | acc.setStatus(statusCode(toString(value.Get("status")))) 253 | 254 | if premium := value.Get("premium"); premium != nil { 255 | acc2.premiumStart = int32(premium.GetInt("start")) 256 | acc2.premiumFinish = int32(premium.GetInt("finish")) 257 | } 258 | 259 | acc.setPremium(acc2.IsPremium(p.Now)) 260 | 261 | acc.birth = int32(value.GetInt("birth")) 262 | acc.joined = int32(value.GetInt("joined")) 263 | acc.setBirthYear(yearOf(acc.birth)) 264 | acc.setJoinedYear(yearOf(acc.joined)) 265 | 266 | // сделал индекс для лайков и пока непонятно зачем их вообще сортировать 267 | //LikeSlice(value.Likes).Sort() 268 | 269 | if likes := value.GetArray("likes"); len(likes) != 0 { 270 | acc2.likesId = makeIdArray(len(likes)) 271 | acc2.likesTs = make([]uint32, len(likes)) 272 | for i, like := range likes { 273 | acc2.likesId.Put(i, uint32(like.GetUint("id"))) 274 | acc2.likesTs[i] = uint32(like.GetUint("ts")) 275 | } 276 | // indexing 277 | AppendAccountLikesIndexDirect(id, acc2.likesId) 278 | //appendInterestsIndex(value.Id, acc) 279 | } 280 | 281 | return id, nil 282 | }*/ 283 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/model_test.go: -------------------------------------------------------------------------------- 1 | package hlc18 2 | 3 | import ( 4 | "archive/zip" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/bcicen/jstream" 8 | "github.com/pkg/profile" 9 | "github.com/valyala/fastjson" 10 | "log" 11 | "runtime" 12 | "testing" 13 | ) 14 | 15 | func BenchmarkJsons(b *testing.B) { 16 | 17 | bytes := []byte(`{"premium":{"finish":1569783897,"start":1538247897},"birth":692070602,"likes":[{"id":1066502,"ts":1457224588},{"id":78150,"ts":1510460183},{"id":853672,"ts":1515150863},{"id":733556,"ts":1481890895},{"id":856740,"ts":1487805837},{"id":1047142,"ts":1458068896},{"id":629144,"ts":1497234539},{"id":300826,"ts":1487754288},{"id":452932,"ts":1461098717},{"id":1195182,"ts":1530979034},{"id":30712,"ts":1461763696},{"id":454620,"ts":1481582929},{"id":407760,"ts":1529800373},{"id":371160,"ts":1501625150},{"id":241380,"ts":1534382164},{"id":803700,"ts":1519479388},{"id":685966,"ts":1486322921},{"id":705718,"ts":1520958917},{"id":626422,"ts":1510751192},{"id":804108,"ts":1493807254},{"id":129572,"ts":1497212902},{"id":778756,"ts":1510678372},{"id":773756,"ts":1532030052},{"id":492876,"ts":1502403536},{"id":142110,"ts":1519654549},{"id":1160208,"ts":1492142179},{"id":102868,"ts":1488957199},{"id":684186,"ts":1489454147},{"id":234630,"ts":1535043000},{"id":973762,"ts":1458619587},{"id":172290,"ts":1486412904},{"id":559182,"ts":1521790424},{"id":1181378,"ts":1495685605},{"id":1213814,"ts":1476655968},{"id":664086,"ts":1501684354},{"id":1270304,"ts":1517398037},{"id":804610,"ts":1525566197},{"id":593016,"ts":1504872249},{"id":239696,"ts":1535239086},{"id":579844,"ts":1525424588},{"id":986152,"ts":1494908263},{"id":799462,"ts":1468367050},{"id":519462,"ts":1470576898},{"id":824660,"ts":1453419877},{"id":738136,"ts":1488428182},{"id":683884,"ts":1468286120},{"id":218188,"ts":1506629719},{"id":1008478,"ts":1533144960},{"id":1090292,"ts":1515672077},{"id":1230224,"ts":1526088509},{"id":298636,"ts":1522805966},{"id":540050,"ts":1532083398},{"id":1200456,"ts":1455636126},{"id":342036,"ts":1467363135},{"id":810644,"ts":1515156323}],"sex":"m","status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","country":"\u0424\u0438\u043d\u043c\u0430\u043b\u044c","interests":["\u0420\u0435\u0433\u0433\u0438","\u041f\u0430\u0441\u0442\u0430","\u0414\u0440\u0443\u0437\u044c\u044f"],"joined":1342828800,"city":"\u041a\u0440\u043e\u043d\u043e\u0433\u043e\u0440\u0441\u043a","email":"hiteher@inbox.ru"}`) 18 | var p fastjson.Parser 19 | 20 | b.Run("FastJson", func(b *testing.B) { 21 | for i := 0; i < b.N; i++ { 22 | acc, err := p.ParseBytes(bytes) 23 | if err != nil { 24 | log.Fatal(err) 25 | } 26 | 27 | s := toString(acc.Get("city")) 28 | if s != "Кроногорск" { 29 | log.Fatal("FastJson Кроногорск") 30 | } 31 | toString(acc.Get("status")) 32 | toString(acc.Get("country")) 33 | for _, interest := range acc.GetArray("interests") { 34 | toString(interest) 35 | } 36 | 37 | } 38 | }) 39 | 40 | b.Run("UnmarshalJSON", func(b *testing.B) { 41 | for i := 0; i < b.N; i++ { 42 | var accountJson AccountJson 43 | if err := json.Unmarshal(bytes, &accountJson); err != nil { 44 | log.Fatal(err) 45 | } 46 | if accountJson.City != "Кроногорск" { 47 | log.Fatal("UnmarshalJSON Кроногорск") 48 | } 49 | } 50 | }) 51 | 52 | //bytes := acc.GetStringBytes("city") 53 | //s, err := strconv.Unquote() 54 | } 55 | 56 | func TestFastJsonLoad2(t *testing.T) { 57 | const capacity = 30000 58 | MakeIndexes(capacity) 59 | InitStore(capacity) 60 | if err := Store.LoadData("/tmp/data-test-2612/"); err != nil { 61 | t.Fatal(err) 62 | } 63 | } 64 | 65 | func TestBcicenJstream(t *testing.T) { 66 | 67 | defer profile.Start(profile.MemProfileRate(2048), profile.ProfilePath("c:/tmp")).Stop() 68 | 69 | r, err := zip.OpenReader("/tmp/data/data.zip") 70 | if err != nil { 71 | t.Error(err) 72 | } 73 | defer r.Close() 74 | 75 | arraysCount := 0 76 | accounts := 0 77 | 78 | maxid := 0 79 | 80 | for _, f := range r.File { 81 | rc, err := f.Open() 82 | if err != nil { 83 | t.Error(err) 84 | } 85 | decoder := jstream.NewDecoder(rc, 2) // extract JSON values at a depth level of 1 86 | for mv := range decoder.Stream() { 87 | switch mv.Value.(type) { 88 | case []interface{}: 89 | arraysCount++ 90 | //case float64: 91 | // label = "float " 92 | //case jstream.KV: 93 | // label = "kv " 94 | //case string: 95 | // label = "string " 96 | case map[string]interface{}: 97 | accounts++ 98 | //m := mv.Value.(map[string]interface{}) 99 | //i := m["id"].(int) 100 | //if maxid < i { 101 | // maxid = i 102 | //} 103 | //println(i.(int)) 104 | } 105 | } 106 | 107 | fmt.Printf("accouns %d, maxid so far %d\n", accounts, maxid) 108 | 109 | rc.Close() 110 | 111 | /* value, err := p.ParseBytes(buffer) 112 | if err != nil { 113 | t.Error(err) 114 | } 115 | fmt.Printf("%s, %d, %d\n", f.Name, len(buffer), len(value.GetArray("accounts"))) 116 | 117 | for _, a := range value.GetArray("accounts") { 118 | 119 | acc := AccountJson{ 120 | Id: uint32(a.GetUint("id")), 121 | Fname: toString(a.Get("fname")), 122 | Sname: toString(a.Get("sname")), 123 | Email: toString(a.Get("email")), 124 | //Interests : toString(a.Get("interests")), 125 | Status: toString(a.Get("status")), 126 | //Premium : { Start: }toString(a.Get("premium")), 127 | Sex: toString(a.Get("sex")), 128 | Phone: toString(a.Get("phone")), 129 | //Likes : toString(a.Get("likes")), 130 | Birth: int32(a.GetInt("birth")), 131 | City: toString(a.Get("city")), 132 | Country: toString(a.Get("country")), 133 | Joined: int32(a.GetInt("joined")), 134 | } 135 | 136 | if interests := a.GetArray("interests"); len(interests) > 0 { 137 | acc.Interests = make([]string, len(interests)) 138 | for i, v := range interests { 139 | acc.Interests[i] = toString(v) 140 | } 141 | } 142 | //if likes := a.GetArray("likes"); len(likes) > 0 { 143 | //} 144 | 145 | emails[acc.Email] = acc.Id 146 | }*/ 147 | /* 148 | acc, err := p.ParseBytes(bytes) 149 | if err != nil { 150 | log.Fatal(err) 151 | } 152 | for _, v := range acc.GetArray("accounts") { 153 | id := uint32(v.GetInt("id")) 154 | if maxId < id { 155 | maxId = id 156 | } 157 | if err = CompressJsonToAccount(id, &(*arr)[id-1], v); err != nil { 158 | return 0, runType, err 159 | } 160 | } 161 | */ 162 | //_ = rc.Close() 163 | } 164 | 165 | println("maxid ", maxid) 166 | 167 | PrintMemUsage("") 168 | runtime.GC() 169 | PrintMemUsage("") 170 | 171 | //for k,v := range emails { 172 | // fmt.Println(k,v) 173 | //} 174 | } 175 | 176 | func TestFastJsonsLoad(t *testing.T) { 177 | 178 | defer profile.Start(profile.MemProfileRate(2048), profile.ProfilePath("c:/tmp")).Stop() 179 | 180 | r, err := zip.OpenReader("/tmp/data/data.zip") 181 | if err != nil { 182 | t.Error(err) 183 | } 184 | defer r.Close() 185 | 186 | //bytes := make([]byte, 20*1024*1024) 187 | // Iterate through the files in the archive, 188 | 189 | var p fastjson.Parser 190 | var bufferBytes [20 * 1024 * 1024]byte 191 | 192 | var emails = make(map[string]uint32) 193 | 194 | for _, f := range r.File { 195 | 196 | rc, err := f.Open() 197 | if err != nil { 198 | t.Error(err) 199 | } 200 | buffer, err := read(rc, bufferBytes[:]) 201 | if err != nil { 202 | t.Error(err) 203 | } 204 | 205 | value, err := p.ParseBytes(buffer) 206 | if err != nil { 207 | t.Error(err) 208 | } 209 | fmt.Printf("%s, %d, %d\n", f.Name, len(buffer), len(value.GetArray("accounts"))) 210 | 211 | for _, a := range value.GetArray("accounts") { 212 | 213 | acc := AccountJson{ 214 | Id: uint32(a.GetUint("id")), 215 | Fname: toString(a.Get("fname")), 216 | Sname: toString(a.Get("sname")), 217 | Email: toString(a.Get("email")), 218 | //Interests : toString(a.Get("interests")), 219 | Status: toString(a.Get("status")), 220 | //Premium : { Start: }toString(a.Get("premium")), 221 | Sex: toString(a.Get("sex")), 222 | Phone: toString(a.Get("phone")), 223 | //Likes : toString(a.Get("likes")), 224 | Birth: int32(a.GetInt("birth")), 225 | City: toString(a.Get("city")), 226 | Country: toString(a.Get("country")), 227 | Joined: int32(a.GetInt("joined")), 228 | } 229 | 230 | if interests := a.GetArray("interests"); len(interests) > 0 { 231 | acc.Interests = make([]string, len(interests)) 232 | for i, v := range interests { 233 | acc.Interests[i] = toString(v) 234 | } 235 | } 236 | //if likes := a.GetArray("likes"); len(likes) > 0 { 237 | //} 238 | 239 | emails[acc.Email] = acc.Id 240 | } 241 | /* 242 | acc, err := p.ParseBytes(bytes) 243 | if err != nil { 244 | log.Fatal(err) 245 | } 246 | for _, v := range acc.GetArray("accounts") { 247 | id := uint32(v.GetInt("id")) 248 | if maxId < id { 249 | maxId = id 250 | } 251 | if err = CompressJsonToAccount(id, &(*arr)[id-1], v); err != nil { 252 | return 0, runType, err 253 | } 254 | } 255 | */_ = rc.Close() 256 | } 257 | 258 | //for k,v := range emails { 259 | // fmt.Println(k,v) 260 | //} 261 | } 262 | 263 | func TestLoad(t *testing.T) { 264 | //defer profile.Start(profile.MemProfileRate(2048), profile.ProfilePath("c:/tmp")).Stop() 265 | 266 | r, _ := zip.OpenReader("/tmp/data/data.zip") 267 | defer r.Close() 268 | 269 | var fileCounter = 0 270 | // Iterate through the files in the archive, 271 | // printing some of their contents. 272 | fmt.Printf("%v\tloading\n", Timenow()) 273 | for _, f := range r.File { 274 | rc, _ := f.Open() 275 | 276 | dec := json.NewDecoder(rc) 277 | //json.Delim: { 278 | _, _ = dec.Token() 279 | //string: accounts 280 | _, _ = dec.Token() 281 | //json.Delim: [ 282 | _, _ = dec.Token() 283 | //fmt.Printf("%T: %v\n", t, t) 284 | 285 | // while the array contains values 286 | for dec.More() { 287 | var account AccountJson 288 | // decode an array value (Message) 289 | err := dec.Decode(&account) 290 | if err != nil { 291 | t.Fatal(err) 292 | } 293 | 294 | } 295 | rc.Close() 296 | fileCounter++ 297 | } 298 | } 299 | 300 | func TestLikesJson(t *testing.T) { 301 | bytes := []byte(`{"likes":[ 302 | {"likee": 3929, "ts": 1464869768, "liker": 25486}, 303 | {"likee": 13239, "ts": 1431103000, "liker": 26727}, 304 | {"likee": 2407, "ts": 1439604510, "liker": 6403}, 305 | {"likee": 26677, "ts": 1454719940, "liker": 22248}, 306 | {"likee": 22411, "ts": 1481309376, "liker": 32820}, 307 | {"likee": 9747, "ts": 1431850118, "liker": 43794}, 308 | {"likee": 43575, "ts": 1499496173, "liker": 16134}, 309 | {"likee": 29725, "ts": 1479087147, "liker": 22248} 310 | ]}`) 311 | 312 | //likesBuffer := LikesBufferBorrow(); defer LikesBufferRelease(likesBuffer) 313 | likes, _ := ParseLikesUpdate(bytes) 314 | println(len(likes)) 315 | 316 | if len(likes) != 8 { 317 | t.Fatal("parseLikesUpdate failed") 318 | } 319 | if likes[7].Ts != 1479087147 || likes[0].Likee != 3929 { 320 | t.Fatal("parseLikesUpdate failed") 321 | } 322 | 323 | } 324 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/data_easyjson.go: -------------------------------------------------------------------------------- 1 | // Code generated by easyjson for marshaling/unmarshaling. DO NOT EDIT. 2 | 3 | package hlc18 4 | 5 | import ( 6 | json "encoding/json" 7 | easyjson "github.com/mailru/easyjson" 8 | jlexer "github.com/mailru/easyjson/jlexer" 9 | jwriter "github.com/mailru/easyjson/jwriter" 10 | ) 11 | 12 | // suppress unused package warning 13 | var ( 14 | _ *json.RawMessage 15 | _ *jlexer.Lexer 16 | _ *jwriter.Writer 17 | _ easyjson.Marshaler 18 | ) 19 | 20 | func easyjson794297d0DecodeDsmnkoOrgHlc18(in *jlexer.Lexer, out *Premium) { 21 | isTopLevel := in.IsStart() 22 | if in.IsNull() { 23 | if isTopLevel { 24 | in.Consumed() 25 | } 26 | in.Skip() 27 | return 28 | } 29 | in.Delim('{') 30 | for !in.IsDelim('}') { 31 | key := in.UnsafeString() 32 | in.WantColon() 33 | if in.IsNull() { 34 | in.Skip() 35 | in.WantComma() 36 | continue 37 | } 38 | switch key { 39 | case "start": 40 | out.Start = int32(in.Int32()) 41 | case "finish": 42 | out.Finish = int32(in.Int32()) 43 | default: 44 | in.AddError(&jlexer.LexerError{ 45 | Offset: in.GetPos(), 46 | Reason: "unknown field", 47 | Data: key, 48 | }) 49 | } 50 | in.WantComma() 51 | } 52 | in.Delim('}') 53 | if isTopLevel { 54 | in.Consumed() 55 | } 56 | } 57 | func easyjson794297d0EncodeDsmnkoOrgHlc18(out *jwriter.Writer, in Premium) { 58 | out.RawByte('{') 59 | first := true 60 | _ = first 61 | { 62 | const prefix string = ",\"start\":" 63 | if first { 64 | first = false 65 | out.RawString(prefix[1:]) 66 | } else { 67 | out.RawString(prefix) 68 | } 69 | out.Int32(int32(in.Start)) 70 | } 71 | { 72 | const prefix string = ",\"finish\":" 73 | if first { 74 | first = false 75 | out.RawString(prefix[1:]) 76 | } else { 77 | out.RawString(prefix) 78 | } 79 | out.Int32(int32(in.Finish)) 80 | } 81 | out.RawByte('}') 82 | } 83 | 84 | // MarshalJSON supports json.Marshaler interface 85 | func (v Premium) MarshalJSON() ([]byte, error) { 86 | w := jwriter.Writer{} 87 | easyjson794297d0EncodeDsmnkoOrgHlc18(&w, v) 88 | return w.Buffer.BuildBytes(), w.Error 89 | } 90 | 91 | // MarshalEasyJSON supports easyjson.Marshaler interface 92 | func (v Premium) MarshalEasyJSON(w *jwriter.Writer) { 93 | easyjson794297d0EncodeDsmnkoOrgHlc18(w, v) 94 | } 95 | 96 | // UnmarshalJSON supports json.Unmarshaler interface 97 | func (v *Premium) UnmarshalJSON(data []byte) error { 98 | r := jlexer.Lexer{Data: data} 99 | easyjson794297d0DecodeDsmnkoOrgHlc18(&r, v) 100 | return r.Error() 101 | } 102 | 103 | // UnmarshalEasyJSON supports easyjson.Unmarshaler interface 104 | func (v *Premium) UnmarshalEasyJSON(l *jlexer.Lexer) { 105 | easyjson794297d0DecodeDsmnkoOrgHlc18(l, v) 106 | } 107 | 108 | func easyjson794297d0DecodeDsmnkoOrgHlc183(in *jlexer.Lexer, out *Like) { 109 | isTopLevel := in.IsStart() 110 | if in.IsNull() { 111 | if isTopLevel { 112 | in.Consumed() 113 | } 114 | in.Skip() 115 | return 116 | } 117 | in.Delim('{') 118 | for !in.IsDelim('}') { 119 | key := in.UnsafeString() 120 | in.WantColon() 121 | if in.IsNull() { 122 | in.Skip() 123 | in.WantComma() 124 | continue 125 | } 126 | switch key { 127 | case "id": 128 | out.Id = uint32(in.Uint32()) 129 | case "ts": 130 | out.Ts = uint32(in.Uint32()) 131 | default: 132 | in.AddError(&jlexer.LexerError{ 133 | Offset: in.GetPos(), 134 | Reason: "unknown field", 135 | Data: key, 136 | }) 137 | } 138 | in.WantComma() 139 | } 140 | in.Delim('}') 141 | if isTopLevel { 142 | in.Consumed() 143 | } 144 | } 145 | func easyjson794297d0EncodeDsmnkoOrgHlc183(out *jwriter.Writer, in Like) { 146 | out.RawByte('{') 147 | first := true 148 | _ = first 149 | { 150 | const prefix string = ",\"id\":" 151 | if first { 152 | first = false 153 | out.RawString(prefix[1:]) 154 | } else { 155 | out.RawString(prefix) 156 | } 157 | out.Uint32(uint32(in.Id)) 158 | } 159 | { 160 | const prefix string = ",\"ts\":" 161 | if first { 162 | first = false 163 | out.RawString(prefix[1:]) 164 | } else { 165 | out.RawString(prefix) 166 | } 167 | out.Int32(int32(in.Ts)) 168 | } 169 | out.RawByte('}') 170 | } 171 | 172 | // MarshalJSON supports json.Marshaler interface 173 | func (v Like) MarshalJSON() ([]byte, error) { 174 | w := jwriter.Writer{} 175 | easyjson794297d0EncodeDsmnkoOrgHlc183(&w, v) 176 | return w.Buffer.BuildBytes(), w.Error 177 | } 178 | 179 | // MarshalEasyJSON supports easyjson.Marshaler interface 180 | func (v Like) MarshalEasyJSON(w *jwriter.Writer) { 181 | easyjson794297d0EncodeDsmnkoOrgHlc183(w, v) 182 | } 183 | 184 | // UnmarshalJSON supports json.Unmarshaler interface 185 | func (v *Like) UnmarshalJSON(data []byte) error { 186 | r := jlexer.Lexer{Data: data} 187 | easyjson794297d0DecodeDsmnkoOrgHlc183(&r, v) 188 | return r.Error() 189 | } 190 | 191 | // UnmarshalEasyJSON supports easyjson.Unmarshaler interface 192 | func (v *Like) UnmarshalEasyJSON(l *jlexer.Lexer) { 193 | easyjson794297d0DecodeDsmnkoOrgHlc183(l, v) 194 | } 195 | func easyjson794297d0DecodeDsmnkoOrgHlc184(in *jlexer.Lexer, out *AccountJson) { 196 | isTopLevel := in.IsStart() 197 | if in.IsNull() { 198 | if isTopLevel { 199 | in.Consumed() 200 | } 201 | in.Skip() 202 | return 203 | } 204 | in.Delim('{') 205 | for !in.IsDelim('}') { 206 | key := in.UnsafeString() 207 | in.WantColon() 208 | if in.IsNull() { 209 | in.Skip() 210 | in.WantComma() 211 | continue 212 | } 213 | switch key { 214 | case "id": 215 | out.Id = uint32(in.Uint32()) 216 | case "fname": 217 | out.Fname = string(in.String()) 218 | case "sname": 219 | out.Sname = string(in.String()) 220 | case "email": 221 | out.Email = string(in.String()) 222 | case "interests": 223 | if in.IsNull() { 224 | in.Skip() 225 | out.Interests = nil 226 | } else { 227 | in.Delim('[') 228 | if out.Interests == nil { 229 | if !in.IsDelim(']') { 230 | out.Interests = make([]string, 0, 4) 231 | } else { 232 | out.Interests = []string{} 233 | } 234 | } else { 235 | out.Interests = (out.Interests)[:0] 236 | } 237 | for !in.IsDelim(']') { 238 | var v4 string 239 | v4 = string(in.String()) 240 | out.Interests = append(out.Interests, v4) 241 | in.WantComma() 242 | } 243 | in.Delim(']') 244 | } 245 | case "status": 246 | out.Status = string(in.String()) 247 | case "premium": 248 | (out.Premium).UnmarshalEasyJSON(in) 249 | case "sex": 250 | out.Sex = string(in.String()) 251 | case "phone": 252 | out.Phone = string(in.String()) 253 | case "likes": 254 | if in.IsNull() { 255 | in.Skip() 256 | out.Likes = nil 257 | } else { 258 | in.Delim('[') 259 | if out.Likes == nil { 260 | if !in.IsDelim(']') { 261 | out.Likes = make([]Like, 0, 8) 262 | } else { 263 | out.Likes = []Like{} 264 | } 265 | } else { 266 | out.Likes = (out.Likes)[:0] 267 | } 268 | for !in.IsDelim(']') { 269 | var v5 Like 270 | (v5).UnmarshalEasyJSON(in) 271 | out.Likes = append(out.Likes, v5) 272 | in.WantComma() 273 | } 274 | in.Delim(']') 275 | } 276 | case "birth": 277 | out.Birth = int32(in.Int32()) 278 | case "city": 279 | out.City = string(in.String()) 280 | case "country": 281 | out.Country = string(in.String()) 282 | case "joined": 283 | out.Joined = int32(in.Int32()) 284 | default: 285 | in.AddError(&jlexer.LexerError{ 286 | Offset: in.GetPos(), 287 | Reason: "unknown field", 288 | Data: key, 289 | }) 290 | } 291 | in.WantComma() 292 | } 293 | in.Delim('}') 294 | if isTopLevel { 295 | in.Consumed() 296 | } 297 | } 298 | func easyjson794297d0EncodeDsmnkoOrgHlc184(out *jwriter.Writer, in AccountJson) { 299 | out.RawByte('{') 300 | first := true 301 | _ = first 302 | { 303 | const prefix string = ",\"id\":" 304 | if first { 305 | first = false 306 | out.RawString(prefix[1:]) 307 | } else { 308 | out.RawString(prefix) 309 | } 310 | out.Uint32(uint32(in.Id)) 311 | } 312 | { 313 | const prefix string = ",\"fname\":" 314 | if first { 315 | first = false 316 | out.RawString(prefix[1:]) 317 | } else { 318 | out.RawString(prefix) 319 | } 320 | out.String(string(in.Fname)) 321 | } 322 | { 323 | const prefix string = ",\"sname\":" 324 | if first { 325 | first = false 326 | out.RawString(prefix[1:]) 327 | } else { 328 | out.RawString(prefix) 329 | } 330 | out.String(string(in.Sname)) 331 | } 332 | { 333 | const prefix string = ",\"email\":" 334 | if first { 335 | first = false 336 | out.RawString(prefix[1:]) 337 | } else { 338 | out.RawString(prefix) 339 | } 340 | out.String(string(in.Email)) 341 | } 342 | { 343 | const prefix string = ",\"interests\":" 344 | if first { 345 | first = false 346 | out.RawString(prefix[1:]) 347 | } else { 348 | out.RawString(prefix) 349 | } 350 | if in.Interests == nil && (out.Flags&jwriter.NilSliceAsEmpty) == 0 { 351 | out.RawString("null") 352 | } else { 353 | out.RawByte('[') 354 | for v6, v7 := range in.Interests { 355 | if v6 > 0 { 356 | out.RawByte(',') 357 | } 358 | out.String(string(v7)) 359 | } 360 | out.RawByte(']') 361 | } 362 | } 363 | { 364 | const prefix string = ",\"status\":" 365 | if first { 366 | first = false 367 | out.RawString(prefix[1:]) 368 | } else { 369 | out.RawString(prefix) 370 | } 371 | out.String(string(in.Status)) 372 | } 373 | { 374 | const prefix string = ",\"premium\":" 375 | if first { 376 | first = false 377 | out.RawString(prefix[1:]) 378 | } else { 379 | out.RawString(prefix) 380 | } 381 | (in.Premium).MarshalEasyJSON(out) 382 | } 383 | { 384 | const prefix string = ",\"sex\":" 385 | if first { 386 | first = false 387 | out.RawString(prefix[1:]) 388 | } else { 389 | out.RawString(prefix) 390 | } 391 | out.String(string(in.Sex)) 392 | } 393 | { 394 | const prefix string = ",\"phone\":" 395 | if first { 396 | first = false 397 | out.RawString(prefix[1:]) 398 | } else { 399 | out.RawString(prefix) 400 | } 401 | out.String(string(in.Phone)) 402 | } 403 | { 404 | const prefix string = ",\"likes\":" 405 | if first { 406 | first = false 407 | out.RawString(prefix[1:]) 408 | } else { 409 | out.RawString(prefix) 410 | } 411 | if in.Likes == nil && (out.Flags&jwriter.NilSliceAsEmpty) == 0 { 412 | out.RawString("null") 413 | } else { 414 | out.RawByte('[') 415 | for v8, v9 := range in.Likes { 416 | if v8 > 0 { 417 | out.RawByte(',') 418 | } 419 | (v9).MarshalEasyJSON(out) 420 | } 421 | out.RawByte(']') 422 | } 423 | } 424 | { 425 | const prefix string = ",\"birth\":" 426 | if first { 427 | first = false 428 | out.RawString(prefix[1:]) 429 | } else { 430 | out.RawString(prefix) 431 | } 432 | out.Int32(int32(in.Birth)) 433 | } 434 | { 435 | const prefix string = ",\"city\":" 436 | if first { 437 | first = false 438 | out.RawString(prefix[1:]) 439 | } else { 440 | out.RawString(prefix) 441 | } 442 | out.String(string(in.City)) 443 | } 444 | { 445 | const prefix string = ",\"country\":" 446 | if first { 447 | first = false 448 | out.RawString(prefix[1:]) 449 | } else { 450 | out.RawString(prefix) 451 | } 452 | out.String(string(in.Country)) 453 | } 454 | { 455 | const prefix string = ",\"joined\":" 456 | if first { 457 | first = false 458 | out.RawString(prefix[1:]) 459 | } else { 460 | out.RawString(prefix) 461 | } 462 | out.Int32(int32(in.Joined)) 463 | } 464 | out.RawByte('}') 465 | } 466 | 467 | // MarshalJSON supports json.Marshaler interface 468 | func (v AccountJson) MarshalJSON() ([]byte, error) { 469 | w := jwriter.Writer{} 470 | easyjson794297d0EncodeDsmnkoOrgHlc184(&w, v) 471 | return w.Buffer.BuildBytes(), w.Error 472 | } 473 | 474 | // MarshalEasyJSON supports easyjson.Marshaler interface 475 | func (v AccountJson) MarshalEasyJSON(w *jwriter.Writer) { 476 | easyjson794297d0EncodeDsmnkoOrgHlc184(w, v) 477 | } 478 | 479 | // UnmarshalJSON supports json.Unmarshaler interface 480 | func (v *AccountJson) UnmarshalJSON(data []byte) error { 481 | r := jlexer.Lexer{Data: data} 482 | easyjson794297d0DecodeDsmnkoOrgHlc184(&r, v) 483 | return r.Error() 484 | } 485 | 486 | // UnmarshalEasyJSON supports easyjson.Unmarshaler interface 487 | func (v *AccountJson) UnmarshalEasyJSON(l *jlexer.Lexer) { 488 | easyjson794297d0DecodeDsmnkoOrgHlc184(l, v) 489 | } 490 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/recommend.go: -------------------------------------------------------------------------------- 1 | package hlc18 2 | 3 | import ( 4 | "math" 5 | "sort" 6 | "sync" 7 | ) 8 | 9 | /* 10 | Решение должно проверять совместимость только с противоположным полом 11 | Если в GET-запросе передана страна или город с ключами country и city соответственно, 12 | то нужно искать только среди живущих в указанном месте. 13 | 14 | В ответе ожидается код 200 и структура {"accounts": [ ... ]} либо код 404 , 15 | если пользователя с искомым id не обнаружено в хранимых данных. По ключу "accounts" должны быть N пользователей, 16 | сортированных по убыванию их совместимости с обозначенным id. 17 | Число N задаётся в запросе GET-параметром limit и не бывает больше 20. 18 | 19 | 1. Наибольший вклад в совместимость даёт наличие статуса "свободны". 20 | Те кто "всё сложно" идут во вторую очередь, а "занятые" в третью и последнюю 21 | (очень вероятно их вообще не будет в ответе). 22 | 23 | 2. Далее идёт совместимость по интересам. 24 | Чем больше совпавших интересов у пользователей, тем более они совместимы. 25 | 26 | 3. Третий по значению параметр - различие в возрасте. Чем больше разница, тем меньше совместимость. 27 | 28 | 4. Те, у кого активирован премиум-аккаунт, пропихиваются в самый верх, вперёд обычных пользователей. 29 | Если таких несколько, то они сортируются по совместимости между собой. 30 | 31 | 5. Если общих интересов нет, то стоит считать пользователей абсолютно несовместимыми с compatibility = 0. 32 | 33 | Число N задаётся в запросе GET-параметром limit и не бывает больше 20. 34 | 35 | */ 36 | 37 | func abs(n int32) int32 { 38 | y := n >> 31 39 | return (n ^ y) - y 40 | } 41 | 42 | func min(a, b int) int { 43 | if a < b { 44 | return a 45 | } 46 | return b 47 | } 48 | 49 | func commonInterests2(me0, me1 uint64, somebodyId uint32) (count uint64) { 50 | 51 | pos := (somebodyId - 1) * 2 52 | n := me0 & Store.interests[pos] //somebody.interests0 53 | for n != 0 { 54 | count++ 55 | n &= n - 1 // Zero the lowest-order one-bit 56 | } 57 | n = me1 & Store.interests[pos+1] 58 | for n != 0 { 59 | count++ 60 | n &= n - 1 // Zero the lowest-order one-bit 61 | } 62 | return count 63 | } 64 | 65 | func commonInterests2_new(me0, me1 uint64, somebodyId uint32) byte { 66 | var count uint64 67 | pos := (somebodyId - 1) * 2 68 | n := me0 & Store.interests[pos] 69 | 70 | if n != 0 { 71 | n -= (n >> 1) & m1 //put count of each 2 bits into those 2 bits 72 | n = (n & m2) + ((n >> 2) & m2) //put count of each 4 bits into those 4 bits 73 | n = (n + (n >> 4)) & m4 //put count of each 8 bits into those 8 bits 74 | count += (n * h01) >> (TEST_BITS_64 - 8) 75 | } 76 | 77 | n = me1 & Store.interests[pos+1] 78 | if n != 0 { 79 | n -= (n >> 1) & m1 //put count of each 2 bits into those 2 bits 80 | n = (n & m2) + ((n >> 2) & m2) //put count of each 4 bits into those 4 bits 81 | n = (n + (n >> 4)) & m4 //put count of each 8 bits into those 8 bits 82 | count += (n * h01) >> (TEST_BITS_64 - 8) 83 | } 84 | 85 | return byte(count) 86 | } 87 | 88 | func containsSorted(array []uint32, id uint32) bool { 89 | i := sort.Search(len(array), func(i int) bool { return array[i] >= id }) 90 | return i < len(array) && array[i] == id 91 | } 92 | 93 | func contains(array []uint32, id uint32) bool { 94 | for i := 0; i < len(array); i++ { 95 | if id == array[i] { 96 | return true 97 | } 98 | } 99 | return false 100 | } 101 | 102 | func containsLike(array []Like, id uint32) bool { 103 | for i := 0; i < len(array); i++ { 104 | if id == array[i].Id { 105 | return true 106 | } else if array[i].Id > id { // т.к. теперь сортированные 107 | return false 108 | } 109 | } 110 | return false 111 | } 112 | 113 | func recommendKey(sex, premium, status, interest byte) int { 114 | return int(sex)*6*InterestsCount + int(premium)*3*InterestsCount + int(status)*InterestsCount + int(interest) 115 | } 116 | 117 | var CountryCount int 118 | var CityCount int 119 | var InterestsCount int 120 | 121 | func recommendKeyCountry(key int, country byte) int { 122 | return key*CountryCount + int(country) 123 | } 124 | 125 | func recommendKeyCity(key int, city uint16) int { 126 | return key*CityCount + int(city) 127 | } 128 | 129 | var selectionPool = sync.Pool{ 130 | New: func() interface{} { return make([]uint64, 1024*16) }, 131 | } 132 | 133 | func token(id uint32, common uint64, myBirth int32) uint64 { 134 | return uint64(id) | (common << 24) | (uint64(abs(myBirth-Store.Accounts[id-1].birth)) << 32) 135 | } 136 | 137 | func tokenCommon(token uint64) uint32 { 138 | return uint32((token << 32) >> (24 + 32)) 139 | } 140 | 141 | var TOKEN_MASK = uint64(mask(0, 24)) 142 | 143 | func tokenId(token uint64) uint32 { 144 | return uint32(token & TOKEN_MASK) 145 | } 146 | 147 | func tokenBirthDiff(token uint64) uint64 { 148 | return token >> 32 149 | } 150 | 151 | func Recommend(myId uint32, limit int, params map[string]string) []uint32 { 152 | 153 | var cityCode uint16 154 | var countryCode byte 155 | 156 | if v, _ := params["city"]; v != "" { 157 | if cityCode = CityDict.ids[v]; cityCode == 0 { 158 | return []uint32{} 159 | } 160 | } else if v, _ := params["country"]; v != "" { 161 | if countryCode = CountryDict.ids[v]; countryCode == 0 { 162 | return []uint32{} 163 | } 164 | } 165 | 166 | cacheKey := CacheKey{myId, cityCode, countryCode} 167 | if cache, e := getRecommendL2Cache(cacheKey, limit); e == nil { 168 | return cache 169 | } 170 | 171 | interests0 := Store.interests[(myId-1)*2] 172 | interests1 := Store.interests[(myId-1)*2+1] 173 | hasInterests := interests0 != 0 || interests1 != 0 174 | 175 | if !hasInterests { 176 | return []uint32{} 177 | } 178 | 179 | me := &Store.Accounts[myId-1] 180 | 181 | // какой максимум может быть в принципе 182 | // maxInterests := int(bitcount(me.interests0) + bitcount(me.interests1)) 183 | // какой максимум видели 184 | maxSeenInterests := uint64(0) 185 | // сколько таких максимумов видели 186 | maxSeenInterestsCount := 0 187 | 188 | sex := byte(0) // 0 - m, 1 - f 189 | if 1 == me.getSex() { 190 | sex = 1 191 | } 192 | 193 | resultBuff := [20]uint32{} 194 | result := resultBuff[:] 195 | foundCount := 0 196 | 197 | poolBytes := selectionPool.Get().([]uint64) 198 | 199 | interestsBuffer := make([]byte, 16) // todo: ? sync.Pool 200 | myInterests := interestsToArray(interests0, interests1, interestsBuffer) 201 | for _, premium := range []byte{1, 0} { 202 | for _, status := range []byte{0, 1, 2} { 203 | var selected = poolBytes[:0] 204 | iter := makeIndexOrIterator() 205 | for _, anInterest := range myInterests { 206 | key := recommendKey(sex, premium, status, anInterest) 207 | if countryCode != 0 { 208 | iter.push(RecommendIndexCountryCompact[recommendKeyCountry(int(key), countryCode)]) 209 | } else if cityCode != 0 { 210 | iter.push(RecommendIndexCityCompact[recommendKeyCity(int(key), cityCode)]) 211 | } else { 212 | iter.push(RecommendIndexCompact[key]) 213 | } 214 | } 215 | 216 | for id := iter.Next(); id != math.MaxUint32 && limit > 0; id = iter.Next() { 217 | var common uint64 218 | 219 | pos := (id - 1) * 2 220 | n := interests0 & Store.interests[pos] 221 | n -= (n >> 1) & m1 //put count of each 2 bits into those 2 bits 222 | n = (n & m2) + ((n >> 2) & m2) //put count of each 4 bits into those 4 bits 223 | n = (n + (n >> 4)) & m4 //put count of each 8 bits into those 8 bits 224 | common += (n * h01) >> (TEST_BITS_64 - 8) 225 | n = interests1 & Store.interests[pos+1] 226 | n -= (n >> 1) & m1 //put count of each 2 bits into those 2 bits 227 | n = (n & m2) + ((n >> 2) & m2) //put count of each 4 bits into those 4 bits 228 | n = (n + (n >> 4)) & m4 //put count of each 8 bits into those 8 bits 229 | common += (n * h01) >> (TEST_BITS_64 - 8) 230 | 231 | if common == 0 { 232 | continue 233 | 234 | } else if common > maxSeenInterests { 235 | maxSeenInterests = common 236 | maxSeenInterestsCount = 1 237 | selected = append(selected, token(id, common, me.birth)) 238 | } else if common == maxSeenInterests { 239 | maxSeenInterestsCount++ 240 | selected = append(selected, token(id, common, me.birth)) 241 | } else if common > 0 && maxSeenInterestsCount < limit { 242 | // с 0 совпавшими интересами нас не интересуют совсем 243 | selected = append(selected, token(id, common, me.birth)) 244 | } 245 | } 246 | 247 | if len(selected) > 0 { 248 | var order = func(i int, j int) bool { 249 | // обратная сортировка, в начале массива будут "лучшие" записи 250 | // Далее идёт совместимость по интересам. 251 | // Чем больше совпавших интересов у пользователей, тем более они совместимы 252 | ti := selected[i] 253 | tj := selected[j] 254 | commonI := tokenCommon(ti) 255 | commonJ := tokenCommon(tj) 256 | if commonI > commonJ { 257 | return true 258 | } else if commonI == commonJ { 259 | // Третий по значению параметр - различие в возрасте. Чем больше разница, тем меньше совместимость 260 | diffI := tokenBirthDiff(ti) 261 | diffJ := tokenBirthDiff(tj) 262 | if diffI < diffJ { 263 | return true 264 | } else if diffI == diffJ { 265 | return tokenId(ti) < tokenId(tj) 266 | } 267 | /* 268 | iid := tokenId(selected[i]) 269 | jid := tokenId(selected[j]) 270 | ai := &Store.Accounts[iid-1] 271 | aj := &Store.Accounts[jid-1] 272 | diffI := abs(me.birth - ai.birth) 273 | diffJ := abs(me.birth - aj.birth) 274 | if diffI < diffJ { 275 | return true 276 | } else if diffI == diffJ { 277 | return iid < jid 278 | } 279 | */ 280 | return false 281 | } 282 | return false 283 | } 284 | sort.Slice(selected, order) 285 | 286 | for i := 0; i < len(result)-foundCount && i < len(selected); i++ { 287 | result[i+foundCount] = tokenId(selected[i]) 288 | } 289 | //copy(result[foundCount:], selected) 290 | foundCount += len(selected) 291 | } 292 | if foundCount >= limit { 293 | selectionPool.Put(poolBytes) 294 | l := min(limit, len(result)) 295 | //for i := 0; i < l; i++ { 296 | // result[i] = tokenId(result[i]) 297 | //} 298 | putRecommendL2Cache(cacheKey, result[:l]) 299 | return result[:l] 300 | } 301 | } 302 | } 303 | 304 | // todo: сюда мы попадем только если нашли меньше чеm limit? 305 | selectionPool.Put(poolBytes) 306 | l := min(limit, foundCount) 307 | //for i := 0; i < l; i++ { 308 | // result[i] = tokenId(result[i]) 309 | //} 310 | putRecommendL2Cache(cacheKey, result[:l]) 311 | return result[:l] 312 | } 313 | 314 | func interestsToArray(interests0 uint64, interests1 uint64, buffer []byte) []byte { 315 | 316 | var myInterests = buffer[:0] 317 | 318 | // + unroll interests0 319 | interest := interests0 >> 1 320 | for i := byte(0); i < 63 && interest != 0; i++ { 321 | if interest&1 == 1 { 322 | myInterests = append(myInterests, i) 323 | } 324 | interest >>= 1 325 | } 326 | // + unroll interests1 327 | interest = interests1 328 | for i := byte(0); i < 64 && interest != 0; i++ { 329 | if interest&1 == 1 { 330 | myInterests = append(myInterests, i+63) 331 | } 332 | interest >>= 1 333 | } 334 | return myInterests 335 | } 336 | 337 | /* 338 | func commonInterests(meId, somebodyId uint32) byte { 339 | pos := (meId-1) * 2 340 | return commonInterests2(Store.interests[pos], Store.interests[pos+1], somebodyId ) 341 | } 342 | */ 343 | 344 | func bitcountNaive(n uint64) (count byte) { 345 | count = 0 346 | for n != 0 { 347 | count += byte(n & 1) 348 | n >>= 1 349 | } 350 | return count 351 | } 352 | 353 | // Sparse-ones: Only iterates as many times as there are 1-bits in the integer. 354 | func bitcountSparse(n uint64) (count byte) { 355 | for n != 0 { 356 | count++ 357 | n &= n - 1 // Zero the lowest-order one-bit 358 | } 359 | return count 360 | } 361 | 362 | const TEST_BITS_64 = 8 * 8 363 | const TEST_BITS_32 = 4 * 8 364 | 365 | const m1 = ^uint64(0) / 3 366 | const m2 = ^uint64(0) / 5 367 | const m4 = ^uint64(0) / 17 368 | const h01 = ^uint64(0) / 255 369 | 370 | func bitcountWP3(n uint64) (count byte) { 371 | n -= (n >> 1) & m1 //put count of each 2 bits into those 2 bits 372 | n = (n & m2) + ((n >> 2) & m2) //put count of each 4 bits into those 4 bits 373 | n = (n + (n >> 4)) & m4 //put count of each 8 bits into those 8 bits 374 | return byte((n * h01) >> (TEST_BITS_64 - 8)) 375 | } 376 | 377 | /* 378 | // https://bisqwit.iki.fi/source/misc/bitcounting/#Wp3NiftyRevised 379 | const unsigned TEST_BITS = sizeof(TestType) * CHAR_BITS; 380 | 381 | //This uses fewer arithmetic operations than any other known 382 | //implementation on machines with fast multiplication. 383 | //It uses 12 arithmetic operations, one of which is a multiply. 384 | static unsigned bitcount (TestType n) 385 | { 386 | TestType m1 = (~(TestType)0) / 3; 387 | TestType m2 = (~(TestType)0) / 5; 388 | TestType m4 = (~(TestType)0) / 17; 389 | TestType h01 = (~(TestType)0) / 255; 390 | 391 | n -= (n >> 1) & m1; //put count of each 2 bits into those 2 bits 392 | n = (n & m2) + ((n >> 2) & m2); //put count of each 4 bits into those 4 bits 393 | n = (n + (n >> 4)) & m4; //put count of each 8 bits into those 8 bits 394 | 395 | return (n * h01) >> (TEST_BITS-8); 396 | // returns left 8 bits of x + (x<<8) + (x<<16) + (x<<24) + ... 397 | } 398 | */ 399 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/recommend_test.go: -------------------------------------------------------------------------------- 1 | package hlc18 2 | 3 | import ( 4 | "fmt" 5 | "math/bits" 6 | "net/url" 7 | "reflect" 8 | "testing" 9 | ) 10 | 11 | func BenchmarkBitCount(b *testing.B) { 12 | 13 | var me Bitset128 14 | var other Bitset128 15 | 16 | me.Set(1) 17 | me.Set(16) 18 | me.Set(31) 19 | 20 | me.Set(63) 21 | me.Set(64) 22 | me.Set(65) 23 | me.Set(42) 24 | me.Set(89) 25 | 26 | other.Set(16) 27 | other.Set(31) 28 | other.Set(8) 29 | other.Set(21) 30 | 31 | other.Set(63) 32 | other.Set(64) 33 | other.Set(65) 34 | 35 | test0 := me[0] & other[0] 36 | //test1 := me[1] & other[1] 37 | 38 | //println(bitcountNaive(test0)) 39 | //println(bitcountNaive(test1)) 40 | //println(bitcountSparse(test0)) 41 | //println(bitcountSparse(test1)) 42 | //println(bitcountWP3(test0)) 43 | //println(bitcountWP3(test1)) 44 | 45 | //test0 = test1 46 | 47 | b.Run("math.OnesCount64", func(b *testing.B) { 48 | for i := 0; i < b.N; i++ { 49 | bits.OnesCount64(test0) 50 | } 51 | 52 | }) 53 | 54 | b.Run("bitcountWP3", func(b *testing.B) { 55 | for i := 0; i < b.N; i++ { 56 | bitcountWP3(test0) 57 | } 58 | }) 59 | 60 | b.Run("bitcountNaive", func(b *testing.B) { 61 | for i := 0; i < b.N; i++ { 62 | bitcountNaive(test0) 63 | } 64 | 65 | }) 66 | /* b.Run("bitcountNaive", func(b *testing.B) { 67 | for i := 0; i < b.N; i++ { 68 | for i := 0; i < b.N; i++ { 69 | bitcountNaive(test1) 70 | } 71 | } 72 | }) 73 | */ 74 | b.Run("bitcountSparse[0]", func(b *testing.B) { 75 | for i := 0; i < b.N; i++ { 76 | bitcountSparse(test0) 77 | } 78 | 79 | }) 80 | 81 | /*b.Run("bitcountSparse[1]", func(b *testing.B) { 82 | for i := 0; i < b.N; i++ { 83 | for i := 0; i < b.N; i++ { 84 | bitcountSparse(test1) 85 | } 86 | } 87 | }) 88 | */ 89 | 90 | /* b.Run("bitcountWP3[1]", func(b *testing.B) { 91 | for i := 0; i < b.N; i++ { 92 | for i := 0; i < b.N; i++ { 93 | bitcountWP3(test1) 94 | } 95 | } 96 | }) 97 | */ 98 | 99 | } 100 | 101 | func BenchmarkLoad(b *testing.B) { 102 | const capacity = 1310000 103 | MakeIndexes(capacity) 104 | InitStore(capacity) 105 | _ = Store.LoadData("/Users/den/tmp/data/") 106 | 107 | b.Run("Recommend", func(b *testing.B) { 108 | for i := 0; i < b.N; i++ { 109 | //for j := 1; j < 100; j++ { 110 | _ = Recommend(uint32(i+1), 20, map[string]string{}) 111 | //} 112 | } 113 | 114 | }) 115 | 116 | } 117 | 118 | func TestRecommend(t *testing.T) { 119 | const capacity = 30000 120 | MakeIndexes(capacity) 121 | InitStore(capacity) 122 | _ = Store.LoadData("/tmp/data-test-2612/") 123 | if l := len(Store.Accounts); l != 30000 { 124 | t.Fatal("load failed") 125 | } 126 | 127 | var rec []uint32 128 | var str string 129 | 130 | str, _ = url.PathUnescape("/accounts/18222/recommend/?city=%D0%9B%D0%B5%D0%B9%D0%BF%D0%BE%D1%80%D0%B8%D0%B6&query_id=840&limit=4") 131 | fmt.Println(str) 132 | 133 | // REQUEST URI: /accounts/18222/recommend/?city=%D0%9B%D0%B5%D0%B9%D0%BF%D0%BE%D1%80%D0%B8%D0%B6&query_id=840&limit=4 134 | // REQUEST BODY: 135 | // BODY GOT: {"accounts":[]} 136 | // BODY EXP: {"accounts":[{"premium":{"finish":1558306307,"start":1526770307},"status":"\u0432\u0441\u0451 \u0441\u043b\u043e\u0436\u043d\u043e","fname":"\u0418\u043b\u044c\u044f","email":"irywiharfisiorte@yandex.ru","id":20899,"sname":"\u0414\u0430\u043d\u0430\u0442\u043e\u0432\u0435\u043d","birth":742486436}, 137 | // {"id":20073,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0410\u043d\u0430\u0442\u043e\u043b\u0438\u0439","birth":737114436,"email":"ahnieletsontocesnyn@yandex.ru"}, 138 | // {"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0420\u0443\u0441\u043b\u0430\u043d","email":"segoevdarlapitiet@list.ru","id":7461,"sname":"\u0414\u0430\u043d\u0430\u0442\u043e\u0442\u0435\u0432","birth":737518627}, 139 | // {"id":24597,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u041d\u0438\u043a\u0438\u0442\u0430","birth":622702446,"email":"ihaltet@inbox.com"}]} 140 | 141 | accounts := Store.Accounts 142 | 143 | fmt.Printf("Now: %d\n", Store.Now) 144 | 145 | _ = recommendKey(0, 1, 1, 10) 146 | //keyCity := recommendKeyCity(key, 72) 147 | 148 | //countries := CityMap[CityDict.values[72]] 149 | //for c := range countries { 150 | // keyCountry := recommendKeyCountry(key, CountryDict.ids[c]) 151 | // fmt.Printf("%d: %v\n%d: %v\n", key, RecommendIndexCompact[key], keyCountry, RecommendIndexCountryCompact[keyCountry]) 152 | //} 153 | 154 | //fmt.Printf("%d: %v\n%d: %v\n",key,RecommendIndexCompact[key], keyCity, RecommendIndexCityCompact[keyCity]) 155 | 156 | rec = Recommend(18222, 4, map[string]string{"city": "Лейпориж"}) 157 | 158 | print("city code: ") 159 | println(CityDict.ids["Лейпориж"]) 160 | 161 | print("me: ") 162 | accounts[18221].print(18222) 163 | print("await[0]: ") 164 | accounts[20899-1].print(20899) 165 | print("await[3]: ") 166 | accounts[24597-1].print(24597) 167 | 168 | fmt.Println("------------------") 169 | 170 | for _, a := range rec { 171 | accounts[a-1].print(a) 172 | } 173 | 174 | if rec[0] != 20899 { 175 | t.Error("failed [0]-20899: /accounts/18222/recommend/?city=Лейпориж&query_id=840&limit=4") 176 | } 177 | if rec[1] != 20073 { 178 | t.Error("failed [1]-20073: /accounts/18222/recommend/?city=Лейпориж&query_id=840&limit=4") 179 | } 180 | if rec[2] != 7461 { 181 | t.Error("failed [2]-7461: /accounts/18222/recommend/?city=Лейпориж&query_id=840&limit=4") 182 | } 183 | if rec[3] != 24597 { 184 | t.Error("failed [3]-24597: /accounts/18222/recommend/?city=Лейпориж&query_id=840&limit=4") 185 | } 186 | 187 | // 18222 : 0, f, заняты, etatrem@yandex.ru, 1991-09-01 10:07:08 +0000 UTC, 100000000000000000000100000000000000000000000100000000000:0 188 | // ------------------ 189 | // 20899 : 1, m, всё сложно, irywiharfisiorte@yandex.ru, 1993-07-12 14:13:56 +0000 UTC, 100000100000:1000000000000 190 | // 191 | // 20073 : 0, m, свободны, ahnieletsontocesnyn@yandex.ru, 1993-05-11 10:00:36 +0000 UTC, 100000000000:10000000000000100000000 192 | // 7461 : 0, m, свободны, segoevdarlapitiet@list.ru, 1993-05-16 02:17:07 +0000 UTC, 100000001000010100000001000:0 193 | // 24597 : 0, m, свободны, ihaltet@inbox.com, 1989-09-25 04:54:06 +0000 UTC, 10000000000000000000000100000000000000000000000000000000000:0 194 | // 195 | // 196 | // 13235 : 0, m, свободны, inpezoh@gmail.com, 1987-12-02 16:19:15 +0000 UTC, 1000000000000000100000001010100000000000000100000000000:10000000000000000000000000 197 | // 27871 : 0, m, свободны, luhorog@gmail.com, 1994-04-25 17:26:46 +0000 UTC, 1000000000000000000000000000100000000000000000100000000010:1 198 | // 8323 : 0, m, свободны, vipofennysah@icloud.com, 1994-02-06 14:39:05 +0000 UTC, 10000000000100000000000100000000000000000000001000000000000:0 199 | // 13687 : 0, m, свободны, atsaihod@yandex.ru, 1989-08-09 15:45:04 +0000 UTC, 10000000100000000000000000010000000000000000100000000000:0 200 | 201 | // REQUEST URI: /accounts/1741/recommend/?query_id=1080&limit=6 202 | // REQUEST BODY: 203 | // BODY GOT: {"accounts":[{"id":13591,"email":"seynfelusylninip@email.com","status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0415\u0433\u043e\u0440","birth":665264004,"premium":{"finish":1551155844,"start":1519619844}},{"id":25201,"email":"imrevheen@yandex.ru","fname":"\u0421\u0442\u0435\u043f\u0430\u043d","sname":"\u041f\u0435\u043d\u043e\u043b\u043e\u0442\u0438\u043d","birth":651935709,"premium":{"finish":1559445559,"start":1527909559},"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b"},{"id":29811,"email":"redsiptontas@ymail.com","status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0412\u0438\u043a\u0442\u043e\u0440","sname":"\u041b\u0435\u0431\u0430\u0448\u0435\u043b\u0430\u043d","birth":769468347,"premium":{"finish":1563218399,"start":1531682399}},{"id":4363,"email":"awenkertad@icloud.com","status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0421\u0430\u0432\u0432\u0430","sname":"\u0425\u043e\u043f\u043e\u043b\u043e\u043f\u043e\u0432","birth":547700732,"premium":{"finish":1549230002,"start":1533505202}},{"id":26753,"email":"rivevocnat@gmail.com","status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u041d\u0438\u043a\u0438\u0442\u0430","sname":"\u0422\u0435\u0440\u0443\u0448\u0443\u0432\u0435\u043d","birth":494307371,"premium":{"finish":1561510239,"start":1529974239}},{"id":16025,"email":"miohco@rambler.ru","status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0410\u043b\u0435\u043a\u0441\u0435\u0439","sname":"\u0414\u0430\u043d\u044b\u043a\u0430\u0432\u0435\u043d","birth":704701991,"premium":{"finish":1565880301,"start":1534344301}}]} 204 | // BODY EXP: {"accounts":[{"premium":{"finish":1551955204,"start":1520419204},"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0412\u0430\u0441\u0438\u043b\u0438\u0441\u0430","email":"fivewevheecwes@ya.ru","id":7840,"sname":"\u041a\u043b\u0435\u0440\u044b\u043a\u0430\u0442\u0435\u0432\u0430","birth":664858853},{"premium":{"finish":1550960915,"start":1543098515},"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0410\u043b\u0451\u043d\u0430","email":"inenselrecwes@list.ru","id":15918,"sname":"\u041f\u0435\u043d\u043e\u043b\u043e\u043d\u0430\u044f","birth":760873559},{"premium":{"finish":1552365635,"start":1536640835},"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u041c\u0430\u0440\u0438\u044f","email":"efweminehsehelifa@yandex.ru","id":8666,"sname":"\u0424\u0430\u043e\u043b\u043e\u0447\u0430\u043d","birth":609534278},{"premium":{"finish":1570280743,"start":1538744743},"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u041b\u0435\u0440\u0430","email":"oscetedewavetegta@gmail.com","id":17496,"sname":"\u0422\u0435\u0440\u043b\u0435\u043d\u0432\u0438\u0447","birth":475439729},{"premium":{"finish":1562589674,"start":1531053674},"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0415\u0432\u0433\u0435\u043d\u0438\u044f","email":"ehhanettotsaed@inbox.ru","id":29932,"sname":"\u041a\u0438\u0441\u043b\u0435\u043d\u0441\u044f\u043d","birth":704475019},{"premium":{"finish":1552687515,"start":1536962715},"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","birth":702625295,"email":"tuwleqvohagonarerle@inbox.ru","id":28910}]} 205 | 206 | rec = Recommend(1741, 6, map[string]string{}) 207 | 208 | print("\nme and awaited-----------------------------------\n") 209 | 210 | accounts[1741-1].print(1741) 211 | accounts[7840-1].print(7840) 212 | 213 | print("\ngot -----------------------------------\n") 214 | 215 | for _, a := range rec { 216 | accounts[a-1].print(a) 217 | } 218 | 219 | if rec[0] != 7840 { 220 | t.Error("failed [0]-7840: /accounts/1741/recommend/?query_id=1080&limit=6") 221 | } 222 | 223 | str, _ = url.PathUnescape("/accounts/21987/recommend/?country=%D0%9C%D0%B0%D0%BB%D0%BC%D0%B0%D0%BB%D1%8C&query_id=1440&limit=2") 224 | fmt.Println("\n", str) 225 | 226 | //REQUEST URI: /accounts/21987/recommend/?country=%D0%9C%D0%B0%D0%BB%D0%BC%D0%B0%D0%BB%D1%8C&query_id=1440&limit=2 227 | //REQUEST BODY: 228 | //BODY GOT: {"accounts":[{"id":6045,"email":"intibinon@inbox.ru","status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0415\u0433\u043e\u0440","sname":"\u041b\u0435\u0431\u043e\u043b\u043e\u0441\u044f\u043d","birth":657448097,"premium":{"finish":1551792615,"start":1520256615}},{"id":8797,"email":"ehdatdaehrererdaas@inbox.com","status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u0414\u0435\u043d\u0438\u0441","sname":"\u0424\u0435\u0442\u043e\u043b\u043e\u0447\u0430\u043d","birth":767405048,"premium":{"finish":1558658432,"start":1542933632}}]} 229 | //BODY EXP: {"accounts":[{"premium":{"finish":1570444883,"start":1538908883},"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u041c\u0438\u043b\u0435\u043d\u0430","email":"pepicrinhi@yandex.ru","id":20490,"sname":"\u041b\u0435\u0431\u043e\u043b\u043e\u0432\u0438\u0447","birth":732823869},{"premium":{"finish":1565249231,"start":1533713231},"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","fname":"\u041d\u0430\u0442\u0430\u043b\u044c\u044f","email":"tawnatpawyd@ymail.com","id":4666,"sname":"\u041a\u0438\u0441\u0430\u0448\u0435\u0442\u0435\u0432\u0430","birth":744371189}]} 230 | 231 | rec = Recommend(21987, 2, map[string]string{"country": "Малмаль"}) 232 | 233 | accounts[21987-1].print(21987) 234 | accounts[20490-1].print(20490) 235 | accounts[4666-1].print(4666) 236 | 237 | fmt.Println("------------------") 238 | 239 | for _, a := range rec { 240 | accounts[a-1].print(a) 241 | } 242 | 243 | if rec[0] != 20490 { 244 | t.Error("failed [0]-21987: /accounts/21987/recommend/?country=Малмаль&query_id=1440&limit=2") 245 | } 246 | if rec[1] != 4666 { 247 | t.Error("failed [1]-4666: /accounts/21987/recommend/?country=Малмаль&query_id=1440&limit=2") 248 | 249 | } 250 | 251 | // cache check 252 | rec2 := Recommend(21987, 2, map[string]string{"country": "Малмаль"}) 253 | if !reflect.DeepEqual(rec2, rec) { 254 | t.Error("cache failed, ", str) 255 | } 256 | 257 | str, _ = url.PathUnescape("/accounts/2222/recommend/?city=%D0%A1%D0%B0%D0%BD%D0%BA%D1%82%D0%BE%D0%B3%D1%80%D0%B0%D0%B4&query_id=2054&limit=8") 258 | fmt.Println("\n", str) 259 | //REQUEST URI: /accounts/2222/recommend/?city=%D0%A1%D0%B0%D0%BD%D0%BA%D1%82%D0%BE%D0%B3%D1%80%D0%B0%D0%B4&query_id=2054&limit=8 260 | rec = Recommend(2222, 8, map[string]string{"city": "Санктоград"}) 261 | accounts[2222-1].print(2222) 262 | fmt.Println("------------------") 263 | for _, a := range rec { 264 | accounts[a-1].print(a) 265 | } 266 | if len(rec) != 0 { 267 | t.Error("failed, ", str) 268 | } 269 | 270 | } 271 | -------------------------------------------------------------------------------- /evio.diff: -------------------------------------------------------------------------------- 1 | diff --git a/go-path/src/tidwall/evio/evio.go b/go-path/src/tidwall/evio/evio.go 2 | index 8e31287..d222514 100644 3 | --- a/go-path/src/tidwall/evio/evio.go 4 | +++ b/go-path/src/tidwall/evio/evio.go 5 | @@ -62,6 +62,8 @@ type Conn interface { 6 | RemoteAddr() net.Addr 7 | // Wake triggers a Data event for this connection. 8 | Wake() 9 | + 10 | + WriteAhead([]byte) []byte 11 | } 12 | 13 | // LoadBalance sets the load balancing method. 14 | @@ -113,8 +115,10 @@ type Events struct { 15 | // underlying socket connection. It can be freely used in goroutines 16 | // and should be closed when it's no longer needed. 17 | Detached func(c Conn, rwc io.ReadWriteCloser) (action Action) 18 | + 19 | // PreWrite fires just before any data is written to any client socket. 20 | - PreWrite func() 21 | + // PreWrite func() 22 | + 23 | // Data fires when a connection sends the server data. 24 | // The in parameter is the incoming data. 25 | // Use the out return value to write data to the connection. 26 | diff --git a/go-path/src/tidwall/evio/evio_other.go b/go-path/src/tidwall/evio/evio_other.go 27 | index 31b9231..7b66b3a 100644 28 | --- a/go-path/src/tidwall/evio/evio_other.go 29 | +++ b/go-path/src/tidwall/evio/evio_other.go 30 | @@ -10,6 +10,7 @@ import ( 31 | "errors" 32 | "net" 33 | "os" 34 | + "sync/atomic" 35 | ) 36 | 37 | func (ln *listener) close() { 38 | @@ -39,3 +40,14 @@ func reuseportListenPacket(proto, addr string) (l net.PacketConn, err error) { 39 | func reuseportListen(proto, addr string) (l net.Listener, err error) { 40 | return nil, errors.New("reuseport is not available") 41 | } 42 | + 43 | +var EpollWaitTimeout = int32(-1) 44 | + 45 | +func SetEpollWait(value int32) { 46 | + println("EpollWaitTimeout other ", value) 47 | + atomic.StoreInt32(&EpollWaitTimeout, value) 48 | +} 49 | + 50 | +func GetEpollWait() int32 { 51 | + return atomic.LoadInt32(&EpollWaitTimeout) 52 | +} 53 | diff --git a/go-path/src/tidwall/evio/evio_std.go b/go-path/src/tidwall/evio/evio_std.go 54 | index 4eb3e27..17684b2 100644 55 | --- a/go-path/src/tidwall/evio/evio_std.go 56 | +++ b/go-path/src/tidwall/evio/evio_std.go 57 | @@ -35,12 +35,13 @@ type stdudpconn struct { 58 | in []byte 59 | } 60 | 61 | -func (c *stdudpconn) Context() interface{} { return nil } 62 | -func (c *stdudpconn) SetContext(ctx interface{}) {} 63 | -func (c *stdudpconn) AddrIndex() int { return c.addrIndex } 64 | -func (c *stdudpconn) LocalAddr() net.Addr { return c.localAddr } 65 | -func (c *stdudpconn) RemoteAddr() net.Addr { return c.remoteAddr } 66 | -func (c *stdudpconn) Wake() {} 67 | +func (c *stdudpconn) Context() interface{} { return nil } 68 | +func (c *stdudpconn) SetContext(ctx interface{}) {} 69 | +func (c *stdudpconn) AddrIndex() int { return c.addrIndex } 70 | +func (c *stdudpconn) LocalAddr() net.Addr { return c.localAddr } 71 | +func (c *stdudpconn) RemoteAddr() net.Addr { return c.remoteAddr } 72 | +func (c *stdudpconn) Wake() {} 73 | +func (c *stdudpconn) WriteAhead(out []byte) []byte { return out } 74 | 75 | type stdloop struct { 76 | idx int // loop index 77 | @@ -64,12 +65,13 @@ type wakeReq struct { 78 | c *stdconn 79 | } 80 | 81 | -func (c *stdconn) Context() interface{} { return c.ctx } 82 | -func (c *stdconn) SetContext(ctx interface{}) { c.ctx = ctx } 83 | -func (c *stdconn) AddrIndex() int { return c.addrIndex } 84 | -func (c *stdconn) LocalAddr() net.Addr { return c.localAddr } 85 | -func (c *stdconn) RemoteAddr() net.Addr { return c.remoteAddr } 86 | -func (c *stdconn) Wake() { c.loop.ch <- wakeReq{c} } 87 | +func (c *stdconn) Context() interface{} { return c.ctx } 88 | +func (c *stdconn) SetContext(ctx interface{}) { c.ctx = ctx } 89 | +func (c *stdconn) AddrIndex() int { return c.addrIndex } 90 | +func (c *stdconn) LocalAddr() net.Addr { return c.localAddr } 91 | +func (c *stdconn) RemoteAddr() net.Addr { return c.remoteAddr } 92 | +func (c *stdconn) Wake() { c.loop.ch <- wakeReq{c} } 93 | +func (c *stdconn) WriteAhead(out []byte) []byte { return out } 94 | 95 | type stdin struct { 96 | c *stdconn 97 | @@ -380,9 +382,6 @@ func stdloopRead(s *stdserver, l *stdloop, c *stdconn, in []byte) error { 98 | if s.events.Data != nil { 99 | out, action := s.events.Data(c, in) 100 | if len(out) > 0 { 101 | - if s.events.PreWrite != nil { 102 | - s.events.PreWrite() 103 | - } 104 | c.conn.Write(out) 105 | } 106 | switch action { 107 | @@ -401,9 +400,6 @@ func stdloopReadUDP(s *stdserver, l *stdloop, c *stdudpconn) error { 108 | if s.events.Data != nil { 109 | out, action := s.events.Data(c, c.in) 110 | if len(out) > 0 { 111 | - if s.events.PreWrite != nil { 112 | - s.events.PreWrite() 113 | - } 114 | s.lns[c.addrIndex].pconn.WriteTo(out, c.remoteAddr) 115 | } 116 | switch action { 117 | @@ -435,9 +431,6 @@ func stdloopAccept(s *stdserver, l *stdloop, c *stdconn) error { 118 | if s.events.Opened != nil { 119 | out, opts, action := s.events.Opened(c) 120 | if len(out) > 0 { 121 | - if s.events.PreWrite != nil { 122 | - s.events.PreWrite() 123 | - } 124 | c.conn.Write(out) 125 | } 126 | if opts.TCPKeepAlive > 0 { 127 | diff --git a/go-path/src/tidwall/evio/evio_unix.go b/go-path/src/tidwall/evio/evio_unix.go 128 | index 16c696e..a69d137 100644 129 | --- a/go-path/src/tidwall/evio/evio_unix.go 130 | +++ b/go-path/src/tidwall/evio/evio_unix.go 131 | @@ -8,6 +8,7 @@ package evio 132 | 133 | import ( 134 | "io" 135 | + "log" 136 | "net" 137 | "os" 138 | "runtime" 139 | @@ -15,9 +16,10 @@ import ( 140 | "sync/atomic" 141 | "syscall" 142 | "time" 143 | + "unsafe" 144 | 145 | reuseport "github.com/kavu/go_reuseport" 146 | - "github.com/tidwall/evio/internal" 147 | + "tidwall/evio/internal" 148 | ) 149 | 150 | type conn struct { 151 | @@ -46,6 +48,26 @@ func (c *conn) Wake() { 152 | } 153 | } 154 | 155 | +func (c *conn) WriteAhead(out []byte) []byte { 156 | + if len(out) != 0 { 157 | + n, err := write(c.fd, out) 158 | + if err != nil { 159 | + if err != syscall.EAGAIN { 160 | + //return loopCloseConn(s, l, c, err) 161 | + log.Fatal("WriteAhead err != syscall.EAGAIN") 162 | + } 163 | + n = 0 164 | + //println("wtite EAGAIN") 165 | + } 166 | + if n == len(out) { 167 | + return nil 168 | + } else { 169 | + return out[n:] 170 | + } 171 | + } 172 | + return out 173 | +} 174 | + 175 | type server struct { 176 | events Events // user events 177 | loops []*loop // all the loops 178 | @@ -162,13 +184,6 @@ func loopCloseConn(s *server, l *loop, c *conn, err error) error { 179 | atomic.AddInt32(&l.count, -1) 180 | delete(l.fdconns, c.fd) 181 | syscall.Close(c.fd) 182 | - if s.events.Closed != nil { 183 | - switch s.events.Closed(c, err) { 184 | - case None: 185 | - case Shutdown: 186 | - return errClosing 187 | - } 188 | - } 189 | return nil 190 | } 191 | 192 | @@ -328,9 +343,6 @@ func loopUDPRead(s *server, l *loop, lnidx, fd int) error { 193 | in := append([]byte{}, l.packet[:n]...) 194 | out, action := s.events.Data(c, in) 195 | if len(out) > 0 { 196 | - if s.events.PreWrite != nil { 197 | - s.events.PreWrite() 198 | - } 199 | syscall.Sendto(fd, out, 0, sa) 200 | } 201 | switch action { 202 | @@ -353,10 +365,15 @@ func loopOpened(s *server, l *loop, c *conn) error { 203 | } 204 | c.action = action 205 | c.reuse = opts.ReuseInputBuffer 206 | - if opts.TCPKeepAlive > 0 { 207 | - if _, ok := s.lns[c.lnidx].ln.(*net.TCPListener); ok { 208 | - internal.SetKeepAlive(c.fd, int(opts.TCPKeepAlive/time.Second)) 209 | - } 210 | + 211 | + if err := syscall.SetsockoptInt(c.fd, syscall.IPPROTO_TCP, syscall.TCP_NODELAY, 1); err != nil { 212 | + log.Fatal("syscall.IPPROTO_TCP, syscall.TCP_NODELAY") 213 | + } 214 | + if err := syscall.SetsockoptInt(c.fd, syscall.SOL_SOCKET, syscall.SO_SNDBUF, 16*1024); err != nil { 215 | + log.Fatal("syscall.SOL_SOCKET, syscall.SO_SNDBUF") 216 | + } 217 | + if err := syscall.SetsockoptInt(c.fd, syscall.SOL_SOCKET, syscall.SO_RCVBUF, 16*1024); err != nil { 218 | + log.Fatal("syscall.SOL_SOCKET, syscall.SO_RCVBUF") 219 | } 220 | } 221 | if len(c.out) == 0 && c.action == None { 222 | @@ -366,9 +383,6 @@ func loopOpened(s *server, l *loop, c *conn) error { 223 | } 224 | 225 | func loopWrite(s *server, l *loop, c *conn) error { 226 | - if s.events.PreWrite != nil { 227 | - s.events.PreWrite() 228 | - } 229 | n, err := syscall.Write(c.fd, c.out) 230 | if err != nil { 231 | if err == syscall.EAGAIN { 232 | @@ -419,9 +433,62 @@ func loopWake(s *server, l *loop, c *conn) error { 233 | return nil 234 | } 235 | 236 | +var _zero uintptr 237 | + 238 | +var ( 239 | + errEAGAIN error = syscall.EAGAIN 240 | + errEINVAL error = syscall.EINVAL 241 | + errENOENT error = syscall.ENOENT 242 | +) 243 | + 244 | +// errnoErr returns common boxed Errno values, to prevent 245 | +// allocations at runtime. 246 | +func errnoErr(e syscall.Errno) error { 247 | + switch e { 248 | + case 0: 249 | + return nil 250 | + case syscall.EAGAIN: 251 | + return errEAGAIN 252 | + case syscall.EINVAL: 253 | + return errEINVAL 254 | + case syscall.ENOENT: 255 | + return errENOENT 256 | + } 257 | + return e 258 | +} 259 | +func read(fd int, p []byte) (n int, err error) { 260 | + var _p0 unsafe.Pointer 261 | + if len(p) > 0 { 262 | + _p0 = unsafe.Pointer(&p[0]) 263 | + } else { 264 | + _p0 = unsafe.Pointer(&_zero) 265 | + } 266 | + r0, _, e1 := syscall.Syscall(syscall.SYS_READ, uintptr(fd), uintptr(_p0), uintptr(len(p))) 267 | + n = int(r0) 268 | + if e1 != 0 { 269 | + err = errnoErr(e1) 270 | + } 271 | + return 272 | +} 273 | + 274 | +func write(fd int, p []byte) (n int, err error) { 275 | + var _p0 unsafe.Pointer 276 | + if len(p) > 0 { 277 | + _p0 = unsafe.Pointer(&p[0]) 278 | + } else { 279 | + _p0 = unsafe.Pointer(&_zero) 280 | + } 281 | + r0, _, e1 := syscall.Syscall(syscall.SYS_WRITE, uintptr(fd), uintptr(_p0), uintptr(len(p))) 282 | + n = int(r0) 283 | + if e1 != 0 { 284 | + err = errnoErr(e1) 285 | + } 286 | + return 287 | +} 288 | + 289 | func loopRead(s *server, l *loop, c *conn) error { 290 | var in []byte 291 | - n, err := syscall.Read(c.fd, l.packet) 292 | + n, err := read(c.fd, l.packet) //syscall.Read(c.fd, l.packet) 293 | if n == 0 || err != nil { 294 | if err == syscall.EAGAIN { 295 | return nil 296 | @@ -429,14 +496,25 @@ func loopRead(s *server, l *loop, c *conn) error { 297 | return loopCloseConn(s, l, c, err) 298 | } 299 | in = l.packet[:n] 300 | - if !c.reuse { 301 | - in = append([]byte{}, in...) 302 | - } 303 | if s.events.Data != nil { 304 | out, action := s.events.Data(c, in) 305 | c.action = action 306 | - if len(out) > 0 { 307 | - c.out = append([]byte{}, out...) 308 | + if len(out) != 0 { 309 | + n, err = write(c.fd, out) 310 | + if err != nil { 311 | + if err != syscall.EAGAIN { 312 | + return loopCloseConn(s, l, c, err) 313 | + } 314 | + n = 0 315 | + println("wtite EAGAIN") 316 | + } 317 | + if n == len(out) { 318 | + c.out = nil 319 | + } else { 320 | + c.out = out[n:] 321 | + println("wtite tail ", len(c.out)) 322 | + } 323 | + //c.out = out // append([]byte{}, out...) // todo : can I just copy pointer and write only once? Ramen? 324 | } 325 | } 326 | if len(c.out) != 0 || c.action != None { 327 | @@ -532,3 +610,12 @@ func reuseportListenPacket(proto, addr string) (l net.PacketConn, err error) { 328 | func reuseportListen(proto, addr string) (l net.Listener, err error) { 329 | return reuseport.Listen(proto, addr) 330 | } 331 | + 332 | +func SetEpollWait(value int32) { 333 | + println("EpollWaitTimeout unix ", value) 334 | + atomic.StoreInt32(&internal.EpollWaitTimeout, value) 335 | +} 336 | + 337 | +func GetEpollWait() int32 { 338 | + return atomic.LoadInt32(&internal.EpollWaitTimeout) 339 | +} 340 | diff --git a/go-path/src/tidwall/evio/internal/internal_linux.go b/go-path/src/tidwall/evio/internal/internal_linux.go 341 | index 6bbaffa..b79e695 100644 342 | --- a/go-path/src/tidwall/evio/internal/internal_linux.go 343 | +++ b/go-path/src/tidwall/evio/internal/internal_linux.go 344 | @@ -5,6 +5,7 @@ 345 | package internal 346 | 347 | import ( 348 | + "sync/atomic" 349 | "syscall" 350 | ) 351 | 352 | @@ -48,11 +49,14 @@ func (p *Poll) Trigger(note interface{}) error { 353 | return err 354 | } 355 | 356 | +var EpollWaitTimeout = int32(-1) 357 | + 358 | // Wait ... 359 | func (p *Poll) Wait(iter func(fd int, note interface{}) error) error { 360 | - events := make([]syscall.EpollEvent, 64) 361 | + events := make([]syscall.EpollEvent, 256) 362 | for { 363 | - n, err := syscall.EpollWait(p.fd, events, -1) 364 | + wait := int(atomic.LoadInt32(&EpollWaitTimeout)) 365 | + n, err := syscall.EpollWait(p.fd, events, wait) 366 | if err != nil && err != syscall.EINTR { 367 | return err 368 | } 369 | diff --git a/go-path/src/tidwall/evio/vendor/github.com/kavu/go_reuseport/tcp.go b/go-path/src/tidwall/evio/vendor/github.com/kavu/go_reuseport/tcp.go 370 | index 76540a1..e491dd5 100644 371 | --- a/go-path/src/tidwall/evio/vendor/github.com/kavu/go_reuseport/tcp.go 372 | +++ b/go-path/src/tidwall/evio/vendor/github.com/kavu/go_reuseport/tcp.go 373 | @@ -113,6 +113,12 @@ func NewReusablePortListener(proto, addr string) (l net.Listener, err error) { 374 | return nil, err 375 | } 376 | 377 | + 378 | + if err := syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_DEFER_ACCEPT, 1); err != nil { 379 | + syscall.Close(fd) 380 | + return nil, err 381 | + } 382 | + 383 | if err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, reusePort, 1); err != nil { 384 | syscall.Close(fd) 385 | return nil, err 386 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/filter_test.go: -------------------------------------------------------------------------------- 1 | package hlc18 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/url" 7 | "testing" 8 | ) 9 | 10 | const TMP_DATA_DIR = "/tmp/data-test-2612/" 11 | 12 | func BenchmarkFilterInretestsContains(b *testing.B) { 13 | 14 | const capacity = 30000 15 | MakeIndexes(capacity) 16 | InitStore(capacity) 17 | _ = Store.LoadData(TMP_DATA_DIR) 18 | if l := len(Store.Accounts); l != 30000 { 19 | log.Fatal("load failed") 20 | } 21 | 22 | b.Run("interests_contains[2], country_null=1, sex_eq=f", func(b *testing.B) { 23 | for i := 0; i < b.N; i++ { 24 | _, _ = filterSample(nil, 28, "interests_contains", "Выходные,Рубашки", "country_null", "1", "sex_eq", "f") 25 | } 26 | }) 27 | 28 | b.Run("interests_contains[1], status_eq=свободны, sex_eq=m", func(b *testing.B) { 29 | for i := 0; i < b.N; i++ { 30 | _, _ = filterSample(nil, 20, "interests_contains", "Рубашки", "status_eq", "свободны", "sex_eq", "m") 31 | } 32 | }) 33 | 34 | } 35 | 36 | func TestFilterInretestsContains(t *testing.T) { 37 | const capacity = 30000 38 | MakeIndexes(capacity) 39 | InitStore(capacity) 40 | 41 | _ = Store.LoadData(TMP_DATA_DIR) 42 | if l := len(Store.Accounts); l != 30000 { 43 | t.Fatal("load failed") 44 | } 45 | var result []uint32 46 | 47 | //REQUEST URI: /accounts/filter/?interests_contains=%D0%9F%D0%BE%D1%86%D0%B5%D0%BB%D1%83%D0%B8%2C%D0%A2%D0%B5%D0%BB%D0%B5%D0%B2%D0%B8%D0%B4%D0%B5%D0%BD%D0%B8%D0%B5&premium_null=0&query_id=1635&limit=10 48 | //REQUEST BODY: 49 | //BODY GOT: {"accounts":[{"id":10387,"email":"hedenfeh@yandex.ru","premium":{"finish":1549850513,"start":1541988113}}]} 50 | //BODY EXP: {"accounts":[{"premium":{"finish":1557667257,"start":1526131257},"email":"ahtininebuson@ymail.com","id":28962},{"premium":{"finish":1532007885,"start":1516283085},"email":"losedluihet@yandex.ru","id":28389},{"premium":{"finish":1545107928,"start":1537245528},"email":"syehollo@me.com","id":28010},{"premium":{"finish":1552946982,"start":1537222182},"email":"avitalnedi@email.com","id":27629},{"premium":{"finish":1532313560,"start":1529721560},"email":"toeltolnobcain@email.com","id":26147},{"premium":{"finish":1572869912,"start":1541333912},"email":"poncihednehpeogty@yandex.ru","id":20811},{"premium":{"finish":1572409241,"start":1540873241},"email":"nenatititreleer@email.com","id":20209},{"premium":{"finish":1526155357,"start":1518292957},"email":"heohevecdotket@yahoo.com","id":17168},{"premium":{"finish":1555770461,"start":1524234461},"email":"tirehsuqnerohunro@ya.ru","id":14573},{"premium":{"finish":1549850513,"start":1541988113},"email":"hedenfeh@yandex.ru","id":10387}]} 51 | print_path("/accounts/filter/?interests_contains=%D0%9F%D0%BE%D1%86%D0%B5%D0%BB%D1%83%D0%B8%2C%D0%A2%D0%B5%D0%BB%D0%B5%D0%B2%D0%B8%D0%B4%D0%B5%D0%BD%D0%B8%D0%B5&premium_null=0&query_id=1635&limit=10") 52 | ///accounts/filter/?interests_contains=Поцелуи,Телевидение&premium_null=0&query_id=1635&limit=10 53 | result, _ = filterSample(t, 10, "interests_contains", "Поцелуи,Телевидение", "premium_null", "0") 54 | if len(result) != 10 || result[0] != 28962 || result[9] != 10387 { 55 | t.Errorf("invalid reply: %v\n", result) 56 | } 57 | 58 | //REQUEST URI: /accounts/filter/?interests_contains=%D0%92%D1%8B%D1%85%D0%BE%D0%B4%D0%BD%D1%8B%D0%B5%2C%D0%A0%D1%83%D0%B1%D0%B0%D1%88%D0%BA%D0%B8&country_null=1&query_id=1646&limit=28&sex_eq=f 59 | //REQUEST BODY: 60 | //BODY GOT: {"accounts":[]} 61 | //BODY EXP: {"accounts":[{"id":27942,"sex":"f","email":"ytahtomen@inbox.ru"}]} 62 | print_path("/accounts/filter/?interests_contains=%D0%92%D1%8B%D1%85%D0%BE%D0%B4%D0%BD%D1%8B%D0%B5%2C%D0%A0%D1%83%D0%B1%D0%B0%D1%88%D0%BA%D0%B8&country_null=1&query_id=1646&limit=28&sex_eq=f") 63 | ///accounts/filter/?interests_contains=Выходные,Рубашки&country_null=1&query_id=1646&limit=28&sex_eq=f 64 | result, _ = filterSample(t, 28, "interests_contains", "Выходные,Рубашки", "country_null", "1", "sex_eq", "f") 65 | if len(result) != 1 || result[0] != 27942 { 66 | t.Errorf("invalid reply: %v\n", result) 67 | } 68 | 69 | // REQUEST URI: /accounts/filter/?interests_contains=%D0%A0%D1%83%D0%B1%D0%B0%D1%88%D0%BA%D0%B8&status_eq=%D1%81%D0%B2%D0%BE%D0%B1%D0%BE%D0%B4%D0%BD%D1%8B&query_id=1028&limit=20&sex_eq=m 70 | // REQUEST BODY: 71 | // BODY GOT: {"accounts":[]} 72 | // BODY EXP: {"accounts":[ 73 | // {"id":29755,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"edehemtisdawi@rambler.ru"}, 74 | // {"id":29547,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"ettehoeslabo@mail.ru"}, 75 | // {"id":29199,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"dytetafinep@me.com"}, 76 | // {"id":29147,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"tenaedonontigtes@icloud.com"}, 77 | // {"id":29089,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"mynensedtoohru@icloud.com"}, 78 | // {"id":28961,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"hirodun@ya.ru"}, 79 | // {"id":28915,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"terohehesiratten@email.com"}, 80 | // {"id":28887,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"gamocvoirhaloag@inbox.com"}, 81 | // {"id":28541,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"posiet@gmail.com"}, 82 | // {"id":28405,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"ofatnerlinutas@ymail.com"}, 83 | // {"id":28379,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"ehterernetna@list.ru"}, 84 | // {"id":28375,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"homistudu@ya.ru"}, 85 | // {"id":28351,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"letnytuclinsincetobas@list.ru"}, 86 | // {"id":28095,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"omacpetgawsawyh@me.com"}, 87 | // {"id":28081,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"netdigosoasweah@email.com"}, 88 | // {"id":28069,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"hewawgoeg@me.com"}, 89 | // {"id":27839,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"hedparersistalve@list.ru"}, 90 | // {"id":27559,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"ehetiftaitotih@email.com"}, 91 | // {"id":27541,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"soletit@mail.ru"}, 92 | // {"id":27517,"status":"\u0441\u0432\u043e\u0431\u043e\u0434\u043d\u044b","sex":"m","email":"tinesseetireh@ya.ru"}]} 93 | print_path("/accounts/filter/?interests_contains=%D0%A0%D1%83%D0%B1%D0%B0%D1%88%D0%BA%D0%B8&status_eq=%D1%81%D0%B2%D0%BE%D0%B1%D0%BE%D0%B4%D0%BD%D1%8B&query_id=1028&limit=20&sex_eq=m") 94 | ///accounts/filter/?interests_contains=Рубашки&status_eq=свободны&query_id=1028&limit=20&sex_eq=m 95 | result, _ = filterSample(t, 20, "interests_contains", "Рубашки", "status_eq", "свободны", "sex_eq", "m") 96 | if len(result) != 20 || result[0] != 29755 || result[19] != 27517 { 97 | t.Errorf("invalid reply: %v\n", result) 98 | } 99 | 100 | } 101 | 102 | func filterSample(t *testing.T, limit int, conditions ...string) ([]uint32, *Filter) { 103 | var params = make(map[string]string) 104 | for i, cond := range conditions { 105 | if i%2 == 0 { 106 | params[cond] = conditions[i+1] 107 | } 108 | } 109 | var filter *Filter 110 | var err error 111 | if filter, err = MakeFilter(params); err != nil { 112 | if t != nil { 113 | t.Errorf("MakeFilter failed, %v, %v\n", params, err) 114 | } else { 115 | log.Fatalf("MakeFilter failed, %v, %v\n", params, err) 116 | } 117 | return nil, nil 118 | } 119 | result := make([]uint32, 50)[:0] 120 | filter.Process(limit, func(_ bool, _ map[string]bool, _ *Account, id uint32) { 121 | result = append(result, id) 122 | }) 123 | return result, filter 124 | } 125 | 126 | func print_path(path string) { 127 | var str string 128 | str, _ = url.PathUnescape(path) 129 | fmt.Println(str) 130 | } 131 | 132 | /* 133 | 134 | /accounts/filter/?interests_contains=Выходные,Рубашки&country_null=1&query_id=1646&limit=28&sex_eq=f 135 | /accounts/filter/?interests_contains=Горы,Пляжный+отдых,Апельсиновый+сок,Фотография&limit=38&sex_eq=m&status_eq=свободны&query_id=13582&city_null=1 136 | /accounts/filter/?interests_contains=Спортивные+машины,Салаты,Курица,AC/DC,Симпсоны&limit=36&sex_eq=f&status_eq=свободны&query_id=14789&city_null=1 137 | /accounts/filter/?interests_contains=Пиво,Туфли,Симпсоны&limit=40&sex_eq=f&status_eq=свободны&query_id=15631&city_null=1 138 | /accounts/filter/?interests_contains=Регги,Юмор,Фрукты&limit=16&sex_eq=m&status_neq=свободны&query_id=13031&city_null=1 139 | /accounts/filter/?interests_contains=Телевидение,Автомобили,Pitbull,Ужин+с+друзьями,Обнимашки&limit=28&sex_eq=f&status_eq=свободны&query_id=2603&city_null=1 140 | /accounts/filter/?interests_contains=Стейк,Боевые+искусства,Сон&limit=40&sex_eq=m&status_eq=всё+сложно&query_id=12288&city_null=1 141 | /accounts/filter/?interests_contains=Pitbull,Бокс,Клубника&limit=28&city_null=1&status_neq=свободны&query_id=3236 142 | /accounts/filter/?interests_contains=Еда+и+Напитки,Салаты,Бокс,Мороженое,Рыба&limit=38&sex_eq=m&status_neq=всё+сложно&query_id=5878&city_null=1 143 | /accounts/filter/?interests_contains=Вкусно+поесть,Мясо,Фитнес&limit=40&city_null=1&sex_eq=m&query_id=1596 144 | /accounts/filter/?interests_contains=Любовь,Красное+вино,Фильмы,Клубника&limit=32&sex_eq=m&status_neq=всё+сложно&query_id=15312&city_null=1 145 | /accounts/filter/?interests_contains=Хип+Хоп,South+Park,Боевые+искусства&limit=26&sex_eq=m&status_neq=свободны&query_id=29441&city_null=1 146 | /accounts/filter/?interests_contains=50+Cent,Красное+вино,Компьютеры,Матрица&limit=14&sex_eq=m&status_neq=свободны&query_id=5956&city_null=1 147 | /accounts/filter/?interests_contains=Выходные,Матрица,Девушки&limit=6&sex_eq=f&status_eq=свободны&query_id=26264&city_null=1 148 | /accounts/filter/?interests_contains=Вечер+с+друзьями,Целоваться,Поцелуи,Знакомство,Путешествия&limit=2&query_id=5343&city_null=1&status_eq=заняты 149 | /accounts/filter/?interests_contains=Танцевальная,Автомобили,Плавание&limit=26&sex_eq=m&status_neq=заняты&query_id=21349&city_null=1 150 | /accounts/filter/?interests_contains=Стейк,Любовь,Овощи,Люди+Икс&limit=34&sex_eq=m&status_eq=заняты&query_id=15066&city_null=1 151 | /accounts/filter/?interests_contains=Боевые+искусства,Знакомство,Итальянская+кухня&limit=28&city_null=1&sex_eq=f&query_id=18110 152 | /accounts/filter/?interests_contains=Регги,Металлика,South+Park,Фитнес&limit=16&city_null=1&status_neq=заняты&query_id=10122 153 | 154 | 155 | 11.7446ms:/accounts/filter/?interests_contains=%D0%93%D0%BE%D1%80%D1%8B%2C%D0%9F%D0%BB%D1%8F%D0%B6%D0%BD%D1%8B%D0%B9+%D0%BE%D1%82%D0%B4%D1%8B%D1%85%2C%D0%90%D0%BF%D0%B5%D0%BB%D1%8C%D1%81%D0%B8%D0%BD%D0%BE%D0%B2%D1%8B%D0%B9+%D1%81%D0%BE%D0%BA%2C%D0%A4%D0%BE%D1%82%D0%BE%D0%B3%D1%80%D0%B0%D1%84%D0%B8%D1%8F&limit=38&sex_eq=m&status_eq=%D1%81%D0%B2%D0%BE%D0%B1%D0%BE%D0%B4%D0%BD%D1%8B&query_id=13582&city_null=1 156 | 11.7392ms:/accounts/filter/?interests_contains=%D0%A1%D0%BF%D0%BE%D1%80%D1%82%D0%B8%D0%B2%D0%BD%D1%8B%D0%B5+%D0%BC%D0%B0%D1%88%D0%B8%D0%BD%D1%8B%2C%D0%A1%D0%B0%D0%BB%D0%B0%D1%82%D1%8B%2C%D0%9A%D1%83%D1%80%D0%B8%D1%86%D0%B0%2CAC%2FDC%2C%D0%A1%D0%B8%D0%BC%D0%BF%D1%81%D0%BE%D0%BD%D1%8B&limit=36&sex_eq=f&status_eq=%D1%81%D0%B2%D0%BE%D0%B1%D0%BE%D0%B4%D0%BD%D1%8B&query_id=14789&city_null=1 157 | 11.7118ms:/accounts/filter/?interests_contains=%D0%9F%D0%B8%D0%B2%D0%BE%2C%D0%A2%D1%83%D1%84%D0%BB%D0%B8%2C%D0%A1%D0%B8%D0%BC%D0%BF%D1%81%D0%BE%D0%BD%D1%8B&limit=40&sex_eq=f&status_eq=%D1%81%D0%B2%D0%BE%D0%B1%D0%BE%D0%B4%D0%BD%D1%8B&query_id=15631&city_null=1 158 | 11.6907ms:/accounts/filter/?interests_contains=%D0%A0%D0%B5%D0%B3%D0%B3%D0%B8%2C%D0%AE%D0%BC%D0%BE%D1%80%2C%D0%A4%D1%80%D1%83%D0%BA%D1%82%D1%8B&limit=16&sex_eq=m&status_neq=%D1%81%D0%B2%D0%BE%D0%B1%D0%BE%D0%B4%D0%BD%D1%8B&query_id=13031&city_null=1 159 | 10.7657ms:/accounts/filter/?interests_contains=%D0%A2%D0%B5%D0%BB%D0%B5%D0%B2%D0%B8%D0%B4%D0%B5%D0%BD%D0%B8%D0%B5%2C%D0%90%D0%B2%D1%82%D0%BE%D0%BC%D0%BE%D0%B1%D0%B8%D0%BB%D0%B8%2CPitbull%2C%D0%A3%D0%B6%D0%B8%D0%BD+%D1%81+%D0%B4%D1%80%D1%83%D0%B7%D1%8C%D1%8F%D0%BC%D0%B8%2C%D0%9E%D0%B1%D0%BD%D0%B8%D0%BC%D0%B0%D1%88%D0%BA%D0%B8&limit=28&sex_eq=f&status_eq=%D1%81%D0%B2%D0%BE%D0%B1%D0%BE%D0%B4%D0%BD%D1%8B&query_id=2603&city_null=1 160 | 10.7428ms:/accounts/filter/?interests_contains=%D0%A1%D1%82%D0%B5%D0%B9%D0%BA%2C%D0%91%D0%BE%D0%B5%D0%B2%D1%8B%D0%B5+%D0%B8%D1%81%D0%BA%D1%83%D1%81%D1%81%D1%82%D0%B2%D0%B0%2C%D0%A1%D0%BE%D0%BD&limit=40&sex_eq=m&status_eq=%D0%B2%D1%81%D1%91+%D1%81%D0%BB%D0%BE%D0%B6%D0%BD%D0%BE&query_id=12288&city_null=1 161 | 10.7357ms:/accounts/filter/?interests_contains=Pitbull%2C%D0%91%D0%BE%D0%BA%D1%81%2C%D0%9A%D0%BB%D1%83%D0%B1%D0%BD%D0%B8%D0%BA%D0%B0&limit=28&city_null=1&status_neq=%D1%81%D0%B2%D0%BE%D0%B1%D0%BE%D0%B4%D0%BD%D1%8B&query_id=3236 162 | 10.7346ms:/accounts/filter/?interests_contains=%D0%95%D0%B4%D0%B0+%D0%B8+%D0%9D%D0%B0%D0%BF%D0%B8%D1%82%D0%BA%D0%B8%2C%D0%A1%D0%B0%D0%BB%D0%B0%D1%82%D1%8B%2C%D0%91%D0%BE%D0%BA%D1%81%2C%D0%9C%D0%BE%D1%80%D0%BE%D0%B6%D0%B5%D0%BD%D0%BE%D0%B5%2C%D0%A0%D1%8B%D0%B1%D0%B0&limit=38&sex_eq=m&status_neq=%D0%B2%D1%81%D1%91+%D1%81%D0%BB%D0%BE%D0%B6%D0%BD%D0%BE&query_id=5878&city_null=1 163 | 9.7898ms:/accounts/filter/?interests_contains=%D0%92%D0%BA%D1%83%D1%81%D0%BD%D0%BE+%D0%BF%D0%BE%D0%B5%D1%81%D1%82%D1%8C%2C%D0%9C%D1%8F%D1%81%D0%BE%2C%D0%A4%D0%B8%D1%82%D0%BD%D0%B5%D1%81&limit=40&city_null=1&sex_eq=m&query_id=1596 164 | 9.7874ms:/accounts/filter/?interests_contains=%D0%9B%D1%8E%D0%B1%D0%BE%D0%B2%D1%8C%2C%D0%9A%D1%80%D0%B0%D1%81%D0%BD%D0%BE%D0%B5+%D0%B2%D0%B8%D0%BD%D0%BE%2C%D0%A4%D0%B8%D0%BB%D1%8C%D0%BC%D1%8B%2C%D0%9A%D0%BB%D1%83%D0%B1%D0%BD%D0%B8%D0%BA%D0%B0&limit=32&sex_eq=m&status_neq=%D0%B2%D1%81%D1%91+%D1%81%D0%BB%D0%BE%D0%B6%D0%BD%D0%BE&query_id=15312&city_null=1 165 | 166 | 59.5562ms:/accounts/filter/?email_lt=ab&limit=4&query_id=28609 167 | 38.0642ms:/accounts/filter/?email_lt=ab&limit=38&sex_eq=f&query_id=5971 168 | 11.7405ms:/accounts/filter/?interests_contains=%D0%A5%D0%B8%D0%BF+%D0%A5%D0%BE%D0%BF%2CSouth+Park%2C%D0%91%D0%BE%D0%B5%D0%B2%D1%8B%D0%B5+%D0%B8%D1%81%D0%BA%D1%83%D1%81%D1%81%D1%82%D0%B2%D0%B0&limit=26&sex_eq=m&status_neq=%D1%81%D0%B2%D0%BE%D0%B1%D0%BE%D0%B4%D0%BD%D1%8B&query_id=29441&city_null=1 169 | 11.7118ms:/accounts/filter/?interests_contains=50+Cent%2C%D0%9A%D1%80%D0%B0%D1%81%D0%BD%D0%BE%D0%B5+%D0%B2%D0%B8%D0%BD%D0%BE%2C%D0%9A%D0%BE%D0%BC%D0%BF%D1%8C%D1%8E%D1%82%D0%B5%D1%80%D1%8B%2C%D0%9C%D0%B0%D1%82%D1%80%D0%B8%D1%86%D0%B0&limit=14&sex_eq=m&status_neq=%D1%81%D0%B2%D0%BE%D0%B1%D0%BE%D0%B4%D0%BD%D1%8B&query_id=5956&city_null=1 170 | 11.6801ms:/accounts/filter/?interests_contains=%D0%92%D1%8B%D1%85%D0%BE%D0%B4%D0%BD%D1%8B%D0%B5%2C%D0%9C%D0%B0%D1%82%D1%80%D0%B8%D1%86%D0%B0%2C%D0%94%D0%B5%D0%B2%D1%83%D1%88%D0%BA%D0%B8&limit=6&sex_eq=f&status_eq=%D1%81%D0%B2%D0%BE%D0%B1%D0%BE%D0%B4%D0%BD%D1%8B&query_id=26264&city_null=1 171 | 10.7646ms:/accounts/filter/?interests_contains=%D0%92%D0%B5%D1%87%D0%B5%D1%80+%D1%81+%D0%B4%D1%80%D1%83%D0%B7%D1%8C%D1%8F%D0%BC%D0%B8%2C%D0%A6%D0%B5%D0%BB%D0%BE%D0%B2%D0%B0%D1%82%D1%8C%D1%81%D1%8F%2C%D0%9F%D0%BE%D1%86%D0%B5%D0%BB%D1%83%D0%B8%2C%D0%97%D0%BD%D0%B0%D0%BA%D0%BE%D0%BC%D1%81%D1%82%D0%B2%D0%BE%2C%D0%9F%D1%83%D1%82%D0%B5%D1%88%D0%B5%D1%81%D1%82%D0%B2%D0%B8%D1%8F&limit=2&query_id=5343&city_null=1&status_eq=%D0%B7%D0%B0%D0%BD%D1%8F%D1%82%D1%8B 172 | 10.7643ms:/accounts/filter/?interests_contains=%D0%A2%D0%B0%D0%BD%D1%86%D0%B5%D0%B2%D0%B0%D0%BB%D1%8C%D0%BD%D0%B0%D1%8F%2C%D0%90%D0%B2%D1%82%D0%BE%D0%BC%D0%BE%D0%B1%D0%B8%D0%BB%D0%B8%2C%D0%9F%D0%BB%D0%B0%D0%B2%D0%B0%D0%BD%D0%B8%D0%B5&limit=26&sex_eq=m&status_neq=%D0%B7%D0%B0%D0%BD%D1%8F%D1%82%D1%8B&query_id=21349&city_null=1 173 | 10.7633ms:/accounts/filter/?interests_contains=%D0%A1%D1%82%D0%B5%D0%B9%D0%BA%2C%D0%9B%D1%8E%D0%B1%D0%BE%D0%B2%D1%8C%2C%D0%9E%D0%B2%D0%BE%D1%89%D0%B8%2C%D0%9B%D1%8E%D0%B4%D0%B8+%D0%98%D0%BA%D1%81&limit=34&sex_eq=m&status_eq=%D0%B7%D0%B0%D0%BD%D1%8F%D1%82%D1%8B&query_id=15066&city_null=1 174 | 10.7632ms:/accounts/filter/?interests_contains=%D0%91%D0%BE%D0%B5%D0%B2%D1%8B%D0%B5+%D0%B8%D1%81%D0%BA%D1%83%D1%81%D1%81%D1%82%D0%B2%D0%B0%2C%D0%97%D0%BD%D0%B0%D0%BA%D0%BE%D0%BC%D1%81%D1%82%D0%B2%D0%BE%2C%D0%98%D1%82%D0%B0%D0%BB%D1%8C%D1%8F%D0%BD%D1%81%D0%BA%D0%B0%D1%8F+%D0%BA%D1%83%D1%85%D0%BD%D1%8F&limit=28&city_null=1&sex_eq=f&query_id=18110 175 | 10.7538ms:/accounts/filter/?interests_contains=%D0%A0%D0%B5%D0%B3%D0%B3%D0%B8%2C%D0%9C%D0%B5%D1%82%D0%B0%D0%BB%D0%BB%D0%B8%D0%BA%D0%B0%2CSouth+Park%2C%D0%A4%D0%B8%D1%82%D0%BD%D0%B5%D1%81&limit=16&city_null=1&status_neq=%D0%B7%D0%B0%D0%BD%D1%8F%D1%82%D1%8B&query_id=10122 176 | */ 177 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/group.go: -------------------------------------------------------------------------------- 1 | package hlc18 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/valyala/fasthttp" 7 | "io" 8 | "log" 9 | "math" 10 | "sort" 11 | "strconv" 12 | "strings" 13 | ) 14 | 15 | // Полей для группировки всего пять - sex, status, interests, country, city. 16 | type Group struct { 17 | Limit int 18 | GroupBy []string 19 | FilterBy map[string]bool 20 | Filter Filter 21 | dataMask uint64 22 | Order int8 23 | interestGroup bool // надо ли группировать по интересам 24 | interestKeyCode byte // значение кода интереса если его надо учитывать в группах 25 | likes bool // группа с лайками 26 | 27 | } 28 | 29 | func unpackSex(data uint64) byte { 30 | return byte(1&data) + 1 31 | } 32 | 33 | func unpackInterest(data uint64) byte { 34 | return byte((data & SNAME_MASK) >> SNAME_OFFSET) 35 | } 36 | 37 | func packInterest(data *uint64, code byte) { 38 | if code > 127 { 39 | log.Fatalf("invalid interest '%d'", code) 40 | } 41 | *data = (*data & ^SNAME_MASK) | (uint64(code) << SNAME_OFFSET) 42 | } 43 | 44 | func calcInterestKeyPart(code byte) uint64 { 45 | if code > 127 { 46 | log.Fatalf("invalid interest '%d'", code) 47 | } 48 | return (uint64(code) << SNAME_OFFSET) 49 | } 50 | 51 | func (g *Group) FromParams(params map[string]string) error { 52 | 53 | g.Filter.birth_gt = math.MaxInt32 54 | g.Filter.birth_lt = math.MaxInt32 55 | 56 | g.FilterBy = make(map[string]bool) 57 | 58 | var found = false 59 | 60 | for key, val := range params { 61 | switch key { 62 | case "order": 63 | if "1" == val { 64 | g.Order = 1 65 | } else if "-1" == val { 66 | g.Order = -1 67 | } else { 68 | return fmt.Errorf("invalid %s=%s", key, val) 69 | } 70 | case "keys": 71 | g.GroupBy = strings.Split(val, ",") 72 | for _, group := range g.GroupBy { 73 | switch group { 74 | case "sex": 75 | g.dataMask |= 1 76 | case "status": 77 | g.dataMask |= STATUS_MASK 78 | case "country": 79 | g.dataMask |= COUNTRY_MASK 80 | case "city": 81 | g.dataMask |= CITY_MASK 82 | case "interests": 83 | g.interestGroup = true 84 | 85 | default: 86 | return fmt.Errorf("invalid %s=%s", key, val) 87 | } 88 | } 89 | 90 | case "fname": 91 | g.FilterBy[key] = true 92 | if g.Filter.fname_eq, found = FnameDict.ids[val]; !found { 93 | g.Filter.ShortCircuit = true 94 | return nil 95 | } 96 | 97 | case "sname": 98 | g.FilterBy[key] = true 99 | if g.Filter.sname_eq, found = SnameDict.ids[val]; !found { 100 | g.Filter.ShortCircuit = true 101 | return nil 102 | } 103 | case "status": 104 | g.FilterBy[key] = true 105 | if g.Filter.status_eq = statusCode(val); g.Filter.status_eq == 0 { 106 | return fmt.Errorf("invalid %s=%s", key, val) 107 | } 108 | 109 | case "sex": 110 | g.FilterBy[key] = true 111 | if g.Filter.sex_eq = sexCode(val); g.Filter.sex_eq == 0 { 112 | return fmt.Errorf("invalid %s=%s", key, val) 113 | } 114 | case "city": 115 | g.FilterBy[key] = true 116 | if g.Filter.city_eq, found = CityDict.ids[val]; !found { 117 | g.Filter.ShortCircuit = true 118 | return nil 119 | } 120 | case "country": 121 | g.FilterBy[key] = true 122 | if g.Filter.country_eq, found = CountryDict.ids[val]; !found { 123 | g.Filter.ShortCircuit = true 124 | return nil 125 | } 126 | 127 | //для likes будет только один id, для interests только одна строка, для birth и joined - будет одно число - год). 128 | case "likes": 129 | g.likes = true 130 | g.FilterBy[key] = true 131 | if i, e := strconv.Atoi(val); e == nil { 132 | if uint32(i) > Store.MaxId || i == 0 { 133 | g.Filter.ShortCircuit = true 134 | return nil 135 | } 136 | g.Filter.likes_contains = []uint32{uint32(i)} 137 | } else { 138 | return fmt.Errorf("invalid %s=%s, %v\n", key, val, e) 139 | } 140 | 141 | case "interests": 142 | g.FilterBy[key] = true 143 | if code, ok := InterestDict.ids[val]; ok && code > 0 { 144 | g.Filter.interests_any.Set(code) 145 | g.interestKeyCode = code 146 | } else { 147 | g.Filter.ShortCircuit = true 148 | return nil 149 | } 150 | 151 | case "birth": 152 | g.FilterBy[key] = true 153 | if year, e := strconv.Atoi(val); e == nil { 154 | if year < 1950 || year > 2005 { 155 | g.Filter.ShortCircuit = true 156 | return nil 157 | } 158 | g.Filter.birth_year = uint16(year) 159 | } else { 160 | return fmt.Errorf("invalid %s=%s", key, val) 161 | } 162 | 163 | case "joined": 164 | g.FilterBy[key] = true 165 | if year, e := strconv.Atoi(val); e == nil { 166 | if year < 2011 || year > 2018 { 167 | g.Filter.ShortCircuit = true 168 | return nil 169 | } 170 | g.Filter.joined_year = uint16(year) 171 | } else { 172 | return fmt.Errorf("invalid %s=%s", key, val) 173 | } 174 | 175 | // что это за щщит ? 176 | case "phone": 177 | case "email": 178 | case "premium": 179 | return fmt.Errorf("ORLY? %s=%s", key, val) 180 | default: 181 | return fmt.Errorf("unknown param %s=%s", key, val) 182 | } 183 | } 184 | 185 | if len(g.GroupBy) == 0 || g.Order == 0 { 186 | return errors.New("empty keys or order") 187 | } 188 | 189 | g.Filter.updateCheckFlags() 190 | return nil 191 | } 192 | 193 | var errorInvalidParam = errors.New("invalid param") 194 | 195 | func (p *Group) FromArgs(args *fasthttp.Args) error { 196 | 197 | p.Filter.birth_gt = math.MaxInt32 198 | p.Filter.birth_lt = math.MaxInt32 199 | p.FilterBy = make(map[string]bool) 200 | 201 | var found = false 202 | 203 | var err error 204 | 205 | args.VisitAll(func(keyBytes, valueBytes []byte) { 206 | if err != nil { 207 | return 208 | } 209 | val := B2s(valueBytes) 210 | key := B2s(keyBytes) 211 | switch key { 212 | case "limit": 213 | if p.Limit, err = fasthttp.ParseUint(valueBytes); err != nil || p.Limit < 1 { 214 | err = errorInvalidParam 215 | return 216 | } 217 | case "query_id": 218 | return // ignored 219 | 220 | case "order": 221 | if "1" == val { 222 | p.Order = 1 223 | } else if "-1" == val { 224 | p.Order = -1 225 | } else { 226 | err = errorInvalidParam 227 | return 228 | } 229 | case "keys": 230 | p.GroupBy = strings.Split(val, ",") 231 | for _, group := range p.GroupBy { 232 | switch group { 233 | case "sex": 234 | p.dataMask |= 1 235 | case "status": 236 | p.dataMask |= STATUS_MASK 237 | case "country": 238 | p.dataMask |= COUNTRY_MASK 239 | case "city": 240 | p.dataMask |= CITY_MASK 241 | case "interests": 242 | p.interestGroup = true 243 | default: 244 | err = errorInvalidParam 245 | return 246 | } 247 | } 248 | 249 | case "fname": 250 | p.FilterBy[key] = true 251 | if p.Filter.fname_eq, found = FnameDict.ids[val]; !found { 252 | p.Filter.ShortCircuit = true 253 | err = errorInvalidParam 254 | return 255 | } 256 | 257 | case "sname": 258 | p.FilterBy[key] = true 259 | if p.Filter.sname_eq, found = SnameDict.ids[val]; !found { 260 | p.Filter.ShortCircuit = true 261 | err = errorInvalidParam 262 | return 263 | } 264 | case "status": 265 | p.FilterBy[key] = true 266 | if p.Filter.status_eq = statusCode(val); p.Filter.status_eq == 0 { 267 | err = errorInvalidParam 268 | return 269 | } 270 | 271 | case "sex": 272 | p.FilterBy[key] = true 273 | if p.Filter.sex_eq = sexCode(val); p.Filter.sex_eq == 0 { 274 | err = errorInvalidParam 275 | return 276 | } 277 | case "city": 278 | p.FilterBy[key] = true 279 | if p.Filter.city_eq, found = CityDict.ids[val]; !found { 280 | p.Filter.ShortCircuit = true 281 | //return nil 282 | } 283 | case "country": 284 | p.FilterBy[key] = true 285 | if p.Filter.country_eq, found = CountryDict.ids[val]; !found { 286 | p.Filter.ShortCircuit = true 287 | //return nil 288 | } 289 | 290 | //для likes будет только один id, для interests только одна строка, для birth и joined - будет одно число - год). 291 | case "likes": 292 | p.FilterBy[key] = true 293 | p.likes = true 294 | if i, e := fasthttp.ParseUint(valueBytes); e == nil { 295 | if uint32(i) > Store.MaxId || i == 0 { 296 | p.Filter.ShortCircuit = true 297 | //return nil 298 | } 299 | p.Filter.likes_contains = []uint32{uint32(i)} 300 | } else { 301 | err = errorInvalidParam 302 | return 303 | } 304 | 305 | case "interests": 306 | p.FilterBy[key] = true 307 | if code, ok := InterestDict.ids[val]; ok && code > 0 { 308 | p.Filter.interests_any.Set(code) 309 | p.interestKeyCode = code 310 | } else { 311 | p.Filter.ShortCircuit = true 312 | //return nil 313 | } 314 | 315 | case "birth": 316 | p.FilterBy[key] = true 317 | if year, e := strconv.Atoi(val); e == nil { 318 | if year < 1950 || year > 2005 { 319 | p.Filter.ShortCircuit = true 320 | //return nil 321 | } 322 | p.Filter.birth_year = uint16(year) 323 | } else { 324 | err = errorInvalidParam 325 | return 326 | } 327 | 328 | case "joined": 329 | p.FilterBy[key] = true 330 | if year, e := strconv.Atoi(val); e == nil { 331 | if year < 2011 || year > 2018 { 332 | p.Filter.ShortCircuit = true 333 | //return nil 334 | } 335 | p.Filter.joined_year = uint16(year) 336 | } else { 337 | err = errorInvalidParam 338 | return 339 | } 340 | 341 | default: 342 | //err = fmt.Errorf("unknown param %s=%s", key, val) // что это за щщит? 343 | err = errorInvalidParam 344 | return 345 | } 346 | }) 347 | 348 | if len(p.GroupBy) == 0 { 349 | return errorInvalidParam 350 | } 351 | 352 | p.Filter.updateCheckFlags() 353 | return err 354 | } 355 | 356 | /** 357 | передается функция для учета ключа классификации 358 | */ 359 | func (g *Group) ClassifyAccount(id uint32, collect func(uint64)) { 360 | account := &Store.Accounts[id-1] 361 | if g.Filter.Test(id, account) { 362 | // todo - unroll всех или учет единичного интереса, может вернуть список ключей 363 | key := g.dataMask & account.data 364 | if g.interestGroup { 365 | if g.interestKeyCode == 0 { 366 | // + unroll interests0 367 | pos := (id - 1) * 2 368 | interest := Store.interests[pos] >> 1 369 | for i := byte(0); i < 63 && interest != 0; i++ { 370 | if interest&1 == 1 { 371 | collect(key | calcInterestKeyPart(i+1)) 372 | } 373 | interest >>= 1 374 | } 375 | // + unroll interests1 376 | interest = Store.interests[pos+1] 377 | for i := byte(0); i < 64 && interest != 0; i++ { 378 | if interest&1 == 1 { 379 | collect(key | calcInterestKeyPart(i+64)) 380 | } 381 | interest >>= 1 382 | } 383 | // todo ? надо ли считать "пустой интерес" 384 | // if account.interests0 == 0 && account.interests1 == 0 { } 385 | return 386 | } else { 387 | packInterest(&key, g.interestKeyCode) 388 | } 389 | } 390 | collect(key) 391 | } 392 | } 393 | 394 | func (p *Group) tails(counters map[uint64]int32, maxLimit int) ([]GroupItem, []GroupItem) { 395 | items := make([]GroupItem, len(counters))[:] 396 | var index = 0 397 | for k, c := range counters { 398 | items[index] = GroupItem{Key: k, Count: c} 399 | index++ 400 | } 401 | 402 | less := func(i, j int) bool { 403 | if items[i].Count < items[j].Count { 404 | return true 405 | } else if items[i].Count == items[j].Count { 406 | // если одинаковые то сортируем по ключам группировки 407 | //return p.decodeWithCache(items[i].Key) < p.decodeWithCache(items[j].Key) 408 | for _, v := range p.GroupBy { 409 | if cmp := strings.Compare(decode(v, items[i].Key), decode(v, items[j].Key)); cmp == -1 { 410 | return true 411 | } else if cmp == 1 { 412 | return false 413 | } 414 | } 415 | } 416 | return false 417 | } 418 | sort.Slice(items[:], less) 419 | var head []GroupItem 420 | var tail []GroupItem 421 | if maxLimit < len(items) { 422 | head = append([]GroupItem{}, items[:maxLimit]...) 423 | tail = append([]GroupItem{}, items[len(items)-maxLimit:]...) 424 | } else { 425 | head = items 426 | tail = append([]GroupItem{}, head...) 427 | } 428 | for i, j := 0, len(tail)-1; i < j; i, j = i+1, j-1 { 429 | tail[i], tail[j] = tail[j], tail[i] 430 | } 431 | return head, tail 432 | } 433 | 434 | type GroupCacheKey struct { 435 | sex, status, interests, country, city byte // для сортировки важен порядок задания ключей 436 | 437 | order int8 438 | 439 | sex_eq byte 440 | status_eq byte 441 | country_eq byte 442 | interests_eq byte 443 | city_eq uint16 444 | birth_year uint16 445 | joined_year uint16 446 | } 447 | 448 | func (p *Group) getCachedGroup(key GroupCacheKey) []GroupItem { 449 | groupL2CacheMutex.RLock() 450 | if res, found := groupL2Cache[key]; found { 451 | groupL2CacheMutex.RUnlock() 452 | return res 453 | } 454 | groupL2CacheMutex.RUnlock() 455 | return nil 456 | } 457 | 458 | func (p *Group) toCachedItems(key GroupCacheKey, counters map[uint64]int32) (xs []GroupItem) { 459 | head, tail := p.tails(counters, 50) 460 | countersRelease(counters) 461 | if key.order == 1 { 462 | if p.Limit < len(head) { 463 | xs = head[:p.Limit] 464 | } else { 465 | xs = head 466 | } 467 | groupL2CacheMutex.Lock() 468 | groupL2Cache[key] = head 469 | key.order = 0 - key.order 470 | groupL2Cache[key] = tail 471 | groupL2CacheMutex.Unlock() 472 | } else { 473 | if p.Limit < len(tail) { 474 | xs = tail[:p.Limit] 475 | } else { 476 | xs = tail 477 | } 478 | groupL2CacheMutex.Lock() 479 | groupL2Cache[key] = tail 480 | key.order = 0 - key.order 481 | groupL2Cache[key] = head 482 | groupL2CacheMutex.Unlock() 483 | } 484 | return xs 485 | } 486 | 487 | func (p *Group) CacheKey() GroupCacheKey { 488 | key := GroupCacheKey{order: p.Order, 489 | sex_eq: p.Filter.sex_eq, 490 | status_eq: p.Filter.status_eq, 491 | country_eq: p.Filter.country_eq, 492 | interests_eq: p.interestKeyCode, 493 | city_eq: p.Filter.city_eq, 494 | birth_year: p.Filter.birth_year, 495 | joined_year: p.Filter.joined_year, 496 | } 497 | 498 | for i, s := range p.GroupBy { 499 | switch s { 500 | case "sex": 501 | key.sex = byte(i + 1) 502 | case "status": 503 | key.status = byte(i + 1) 504 | case "country": 505 | key.country = byte(i + 1) 506 | case "city": 507 | key.city = byte(i + 1) 508 | case "interests": 509 | key.interests = byte(i + 1) 510 | } 511 | } 512 | return key 513 | } 514 | 515 | func (p *Group) aggregate(updateCache bool) ([]GroupItem, bool) { 516 | key := p.CacheKey() 517 | if xs := p.getCachedGroup(key); xs != nil { 518 | return xs, true 519 | } 520 | for _, aggregate := range _aggregates { 521 | if counters, ok := aggregate.Aggregate(p); ok { 522 | if updateCache { 523 | return p.toCachedItems(key, counters), ok 524 | } else { 525 | return p.Sort(counters, p.Limit), ok 526 | } 527 | } 528 | } 529 | return nil, false 530 | } 531 | 532 | func (g *Group) Classify(accounts *[]Account, updateCache bool) []GroupItem { 533 | 534 | if !g.likes { 535 | if items, ok := g.aggregate(updateCache); ok { 536 | if len(items) <= g.Limit { 537 | return items 538 | } else { 539 | return items[:g.Limit] 540 | } 541 | } 542 | log.Printf("wtf?: %v\n", g) 543 | } 544 | 545 | // brute force fallback, более-менее приемлем для индексированных фильтров 546 | counters := countersBorrow() // make(map[uint64]int32) 547 | 548 | collect := func(groupKey uint64) { counters[groupKey] += 1 } 549 | // идем в обычном порядке (хм, а так будет быстрее или пофиг?) 550 | if index, iter := g.Filter.Index(false); index != nil || iter != nil { 551 | if iter != nil { 552 | for id := iter.Next(); id != math.MaxUint32; id = iter.Next() { 553 | g.ClassifyAccount(id, collect) 554 | } 555 | } else { 556 | for _, id := range index { 557 | g.ClassifyAccount(id, collect) 558 | } 559 | } 560 | } else { 561 | log.Printf("orly?: %v\n", g) 562 | // жестокий харкор: тупо идем снизу и набираем данные пока не упремся в лимит, в финале сюда не должны попадать 563 | for id := uint32(0); id < Store.MaxId; id++ { 564 | g.ClassifyAccount(id+1, collect) 565 | } 566 | } 567 | // todo: likes не кешируем, а остальной fallback? 568 | return g.Sort(counters, g.Limit) 569 | } 570 | 571 | type GroupItem struct { 572 | Key uint64 573 | Count int32 574 | } 575 | 576 | func (g *Group) Sort(counters map[uint64]int32, limit int) []GroupItem { 577 | items := make([]GroupItem, len(counters))[:] 578 | var index = 0 579 | for k, c := range counters { 580 | items[index] = GroupItem{k, c} 581 | index++ 582 | } 583 | countersRelease(counters) 584 | 585 | less := func(i, j int) bool { 586 | if items[i].Count < items[j].Count { 587 | return true 588 | } else if items[i].Count == items[j].Count { 589 | // если одинаковые то сортируем по ключам группировки 590 | //return g.decodeWithCache(items[i].Key) < g.decodeWithCache(items[j].Key) 591 | for _, v := range g.GroupBy { 592 | if cmp := strings.Compare(decode(v, items[i].Key), decode(v, items[j].Key)); cmp == -1 { 593 | return true 594 | } else if cmp == 1 { 595 | return false 596 | } 597 | } 598 | } 599 | return false 600 | } 601 | 602 | if g.Order == 1 { 603 | sort.Slice(items[:], less) 604 | } else { 605 | sort.Slice(items[:], func(i, j int) bool { return less(j, i) }) 606 | } 607 | 608 | if limit < len(items) { 609 | return items[:limit] 610 | } 611 | 612 | return items 613 | } 614 | 615 | func decode(field string, groupKey uint64) string { 616 | switch field { 617 | case "sex": 618 | return sexOf(unpackSex(groupKey)) 619 | case "status": 620 | return statusOf(unpackStatus(groupKey)) 621 | case "country": 622 | return CountryDict.values[unpackCountry(groupKey)] 623 | case "city": 624 | return CityDict.values[unpackCity(groupKey)] 625 | case "interests": 626 | return InterestDict.values[unpackInterest(groupKey)] 627 | default: 628 | return "" 629 | } 630 | } 631 | 632 | func (g *Group) WriteJsonGroupItemOut(out []byte, groupKey uint64, count int32) []byte { 633 | out = append(out, '{') 634 | for _, name := range g.GroupBy { 635 | if str := decode(name, groupKey); str != "" { //todo: ? unroll to switch 636 | out = append(out, '"') 637 | out = append(out, name...) 638 | out = append(out, "\":\""...) 639 | out = append(out, str...) 640 | out = append(out, "\","...) 641 | } 642 | } 643 | out = append(out, "\"count\":"...) 644 | out = fasthttp.AppendUint(out, int(count)) 645 | out = append(out, '}') 646 | return out 647 | } 648 | 649 | func (g *Group) DebugWriteJsonGroupItem(w io.Writer, groupKey uint64, count int32) { 650 | _, _ = fmt.Fprintf(w, "{") 651 | for _, name := range g.GroupBy { 652 | if str := decode(name, groupKey); str != "" { 653 | _, _ = fmt.Fprintf(w, "\"%s\":%s,", name, str) 654 | } 655 | } 656 | _, _ = fmt.Fprintf(w, "\"count\":%d}", count) 657 | } 658 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/indexes.go: -------------------------------------------------------------------------------- 1 | package hlc18 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | ) 7 | 8 | const ( 9 | HUGE_BLOCK_SIZE = 32 * 1024 // для очень неселективных индексов (пол) 10 | LARGE_BLOCK_SIZE = 4 * 1024 // для средне-селективных индексов (десятки-сотни бакетов) 11 | MEDIUM_BLOCK_SIZE = 1024 // для высоко-средне-селективных 12 | SMALL_BLOCK_SIZE = 32 // для высоко-селективных 13 | 14 | LIKES_INDEX_VECTOR_CAPACITY = 24 15 | ) 16 | 17 | var ( 18 | //var LikesIndex []*VectorUint32 19 | 20 | LikesIndexCompact [][]uint32 21 | 22 | InterestsIndexCompact [][]uint32 23 | InterestsSexIndexCompact [][]uint32 24 | Interests2xIndexCompact [][]uint32 25 | 26 | BirthYearIndexCompact [][]uint32 27 | CityIndexCompact [][]uint32 28 | CountryIndexCompact [][]uint32 29 | FnameIndexCompact [][]uint32 30 | SexIndexCompact [][]uint32 31 | StatusIndexCompact [][]uint32 32 | 33 | //sex, premium, status, interest [,country|city] 34 | RecommendIndexCompact map[int][]uint32 35 | RecommendIndexCountryCompact map[int][]uint32 36 | RecommendIndexCityCompact map[int][]uint32 37 | 38 | // имена которые мы видели для полов [пол][имя] - 1:0 39 | SexNames [2][]byte 40 | 41 | BirthYearCityIndexCompact [][]uint32 42 | EmailPrefixIndexCompact [][]uint32 43 | 44 | CountryPhoneCodeSexIndex map[int][]uint32 45 | CityPhoneCodeSexIndex map[int][]uint32 46 | ) 47 | 48 | var interestsIndexContainer = make([][]uint32, 90) 49 | var interestsSexIndexContainer = make([][]uint32, 90*2) 50 | var interests2xIndexContainer = make([][]uint32, (90+1)*90) 51 | 52 | func MakeIndexes(capacity int) { 53 | 54 | SexNames[0] = make([]byte, 256) 55 | SexNames[1] = make([]byte, 256) 56 | 57 | for x := range interestsIndexContainer { 58 | interestsIndexContainer[x] = make([]uint32, 44000) 59 | } 60 | for x := range interestsSexIndexContainer { 61 | interestsSexIndexContainer[x] = make([]uint32, 22000) 62 | } 63 | 64 | for x := range interests2xIndexContainer { 65 | interests2xIndexContainer[x] = make([]uint32, 1300) //1430 max, 1300 med 66 | } 67 | 68 | LikesIndexCompact = make([][]uint32, capacity) 69 | for x := range LikesIndexCompact { 70 | LikesIndexCompact[x] = make([]uint32, 0, 32) 71 | } 72 | 73 | } 74 | 75 | func ResetIndexes() { 76 | 77 | // update all counters 78 | InterestsCount = len(InterestDict.ids) 79 | CountryCount = len(CountryDict.ids) + 1 80 | CityCount = len(CityDict.ids) + 1 81 | 82 | // -------------- RebuildIndexes() 83 | 84 | BirthYearIndexCompact = nil 85 | BirthYearCityIndexCompact = nil 86 | CityIndexCompact = nil 87 | CountryIndexCompact = nil 88 | FnameIndexCompact = nil 89 | SexIndexCompact = nil 90 | StatusIndexCompact = nil 91 | EmailPrefixIndexCompact = nil 92 | 93 | CountryPhoneCodeSexIndex = nil 94 | CityPhoneCodeSexIndex = nil 95 | 96 | // ------------- RebuildRecommendIndexes() 97 | 98 | resetRecommendIndex(&RecommendIndexCompact) 99 | resetRecommendIndex(&RecommendIndexCityCompact) 100 | resetRecommendIndex(&RecommendIndexCountryCompact) 101 | 102 | resetWithContainer(&InterestsIndexCompact, &interestsIndexContainer) 103 | resetWithContainer(&InterestsSexIndexCompact, &interestsSexIndexContainer) 104 | resetWithContainer(&Interests2xIndexCompact, &interests2xIndexContainer) 105 | 106 | } 107 | 108 | func resetRecommendIndex(index *map[int][]uint32) { 109 | 110 | if len(*index) == 0 { 111 | *index = make(map[int][]uint32) 112 | } else { 113 | for i, xs := range *index { 114 | (*index)[i] = xs[:0] //index[i][:0] 115 | } 116 | } 117 | } 118 | 119 | const BIRTH_YEARS_LEN = 56 120 | 121 | func RebuildRecommendIndexes(accounts *[]Account, maxId uint32) { 122 | 123 | //recommendTemp := make(map[int]*VectorUint32/*, 2*2*3*InterestsCount*/) 124 | //recommendTempCity := make(map[int]*VectorUint32 /*, 2*2*3*InterestsCount*CityCount*/) 125 | //recommendTempCountry := make(map[int]*VectorUint32 /*, 2*2*3*InterestsCount*CountryCount*/) 126 | 127 | //interests2xIndexTemp := make([]*VectorUint32, (90+1)*90) 128 | interestsBuffer := make([]byte, 32) // 129 | for i := uint32(0); i < maxId; i++ { 130 | accPtr := &((*accounts)[i]) 131 | id := i + 1 132 | appendRecommendAndInterestsIndex(id, accPtr, interestsBuffer) 133 | } 134 | 135 | //RecommendIndexCityCompact = make(map[int][]uint32, len(recommendTempCity)) 136 | //compact_map(recommendTempCity, RecommendIndexCityCompact) 137 | //recommendTempCity = nil 138 | //indexMapStat("RecommendIndexCityCompact", RecommendIndexCityCompact) 139 | 140 | //RecommendIndexCountryCompact = make(map[int][]uint32, len(recommendTempCountry)) 141 | //compact_map(recommendTempCountry, RecommendIndexCountryCompact) 142 | //recommendTempCountry = nil 143 | //indexMapStat("RecommendIndexCountryCompact", RecommendIndexCountryCompact) 144 | 145 | //RecommendIndexCompact = make(map[int][]uint32, len(recommendTemp)) 146 | //compact_map(recommendTemp, RecommendIndexCompact) 147 | //recommendTemp = nil 148 | //indexMapStat("RecommendIndexCompact", RecommendIndexCompact) 149 | 150 | } 151 | 152 | // sex -1|2 153 | func countryPhoneCodeSexKey(country byte, phoneCode uint16, sex byte) int { 154 | return (int(sex-1)*1000+int(phoneCode))*CountryCount + int(country) 155 | } 156 | 157 | func cityPhoneCodeSexKey(city uint16, phoneCode uint16, sex byte) int { 158 | return (int(sex-1)*1000+int(phoneCode))*CityCount + int(city) 159 | } 160 | 161 | func RebuildIndexes(accounts *[]Account, maxId uint32) { 162 | 163 | birthYearTemp := make([]*VectorUint32, BIRTH_YEARS_LEN) 164 | birthYearCityTemp := make([]*VectorUint32, BIRTH_YEARS_LEN*(CityCount+1)) 165 | cityTemp := make([]*VectorUint32, CityCount) 166 | sexTemp := make([]*VectorUint32, 2) 167 | statusTemp := make([]*VectorUint32, 3) 168 | 169 | countryIndices := make([]*VectorUint32, (CountryCount + 1)) 170 | fnameIndices := make([]*VectorUint32, len(FnameDict.values)+1) 171 | emailPrefixTemp := make([]*VectorUint32, 26*(26+1)) 172 | 173 | CountryPhoneCodeSexIndex = make(map[int][]uint32, CountryCount*100*2) 174 | CityPhoneCodeSexIndex = make(map[int][]uint32, CityCount*100*2) 175 | 176 | for i := uint32(0); i < maxId; i++ { 177 | 178 | accPtr := &((*accounts)[i]) 179 | id := i + 1 180 | 181 | //compactLikes(id) 182 | 183 | appendBirthYearCityIndex(id, accPtr, birthYearTemp, cityTemp, birthYearCityTemp) 184 | 185 | appendCountryIndex(id, accPtr, countryIndices) 186 | appendFnameIndex(id, accPtr, fnameIndices) 187 | appendSexIndex(id, accPtr, sexTemp) 188 | appendStatusIndex(id, accPtr, statusTemp) 189 | 190 | appendEmailIndex(id, accPtr, emailPrefixTemp) 191 | 192 | phoneCode := accPtr.getPhoneCode() 193 | if phoneCode != 0 { 194 | key := countryPhoneCodeSexKey(accPtr.getCountry(), phoneCode, accPtr.getSex()) 195 | CountryPhoneCodeSexIndex[key] = append(CountryPhoneCodeSexIndex[key], id) 196 | 197 | key = cityPhoneCodeSexKey(accPtr.getCity(), phoneCode, accPtr.getSex()) 198 | CityPhoneCodeSexIndex[key] = append(CityPhoneCodeSexIndex[key], id) 199 | } 200 | 201 | } 202 | 203 | //indexMapStat("CountryPhoneCodeSexIndex",CountryPhoneCodeSexIndex) 204 | //indexMapStat("CityPhoneCodeSexIndex",CityPhoneCodeSexIndex) 205 | 206 | BirthYearIndexCompact = make([][]uint32, len(birthYearTemp)) 207 | compact(birthYearTemp, BirthYearIndexCompact, false) 208 | birthYearTemp = nil 209 | 210 | CityIndexCompact = make([][]uint32, len(cityTemp)) 211 | compact(cityTemp, CityIndexCompact, false) 212 | cityTemp = nil 213 | //indexStat("CityIndexCompact",BirthYearIndexCompact) 214 | 215 | BirthYearCityIndexCompact = make([][]uint32, len(birthYearCityTemp)) 216 | compact(birthYearCityTemp, BirthYearCityIndexCompact, false) 217 | birthYearCityTemp = nil 218 | //indexStat("BirthYearCityIndexCompact",BirthYearCityIndexCompact) 219 | 220 | CountryIndexCompact = make([][]uint32, len(countryIndices)) 221 | compact(countryIndices, CountryIndexCompact, false) 222 | countryIndices = nil 223 | 224 | FnameIndexCompact = make([][]uint32, len(fnameIndices)) 225 | compact(fnameIndices, FnameIndexCompact, false) 226 | fnameIndices = nil 227 | 228 | SexIndexCompact = make([][]uint32, len(sexTemp)) 229 | compact(sexTemp, SexIndexCompact, false) 230 | sexTemp = nil 231 | 232 | StatusIndexCompact = make([][]uint32, len(statusTemp)) 233 | compact(statusTemp, StatusIndexCompact, false) 234 | statusTemp = nil 235 | 236 | EmailPrefixIndexCompact = make([][]uint32, len(emailPrefixTemp)) 237 | compact(emailPrefixTemp, EmailPrefixIndexCompact, false) 238 | emailPrefixTemp = nil 239 | 240 | } 241 | 242 | func resetWithContainer(index, cont *[][]uint32) { 243 | *index = make([][]uint32, len(*cont)) 244 | for x := range *cont { 245 | (*index)[x] = (*cont)[x][:0] 246 | } 247 | } 248 | 249 | /* 250 | func compactLikes(id uint32 ) { 251 | account := &Store.Accounts2[id-1] 252 | if account.updatedLikes != nil { 253 | added := len(*account.updatedLikes) / 2 //account.updatedLikes.Len() / 2 254 | current := account.likesId.Len() 255 | newLikesId := makeIdArray(current + added) 256 | copy(newLikesId, account.likesId) 257 | account.likesId = newLikesId 258 | // stat[added] += 1 259 | //} 260 | newLikeTs := make([]uint32, current+added) 261 | copy(newLikeTs, account.likesTs) 262 | account.likesTs = newLikeTs 263 | for i := 0; i < added; i++ { 264 | account.likesId.Put(current+i, (*account.updatedLikes)[i*2]) 265 | account.likesTs[current+i] = (*account.updatedLikes)[i*2+1] 266 | } 267 | account.updatedLikes = nil 268 | } 269 | } 270 | */ 271 | 272 | func compact_map(src map[int]*VectorUint32, dest map[int][]uint32) { 273 | for key, val := range src { 274 | array := make([]uint32, val.Len(), val.Len()) 275 | val.CopyTo(array) 276 | dest[key] = array 277 | delete(src, key) 278 | } 279 | } 280 | 281 | func compact(src []*VectorUint32, dest [][]uint32, sort bool) { 282 | for i := 0; i < len(src); i++ { 283 | if (src)[i] != nil { 284 | dest[i] = make([]uint32, src[i].len, src[i].len) 285 | src[i].CopyTo(dest[i]) 286 | if sort { 287 | Uint32Slice(dest[i]).Sort() 288 | } 289 | src[i] = nil 290 | } else { 291 | dest[i] = make([]uint32, 0) 292 | } 293 | } 294 | } 295 | 296 | const LIKE_MOD_FLAG = 2000000 297 | 298 | func AppendAccountLikesIndex(id uint32, likes []Like) { 299 | for _, like := range likes { 300 | LikesIndexCompact[like.Id-1] = append(LikesIndexCompact[like.Id-1], id) 301 | if len(LikesIndexCompact[like.Id-1]) > 1 && LikesIndexCompact[like.Id-1][0] < LIKE_MOD_FLAG { 302 | LikesIndexCompact[like.Id-1][0] += LIKE_MOD_FLAG 303 | } 304 | } 305 | } 306 | 307 | /* 308 | func appendInterestsIndex(id uint32, account *Account) { 309 | // + unroll interests0 310 | pos := (id - 1) * 2 311 | interest := Store.interests[pos] >> 1 312 | for i := 0; i < 63 && interest != 0; i++ { 313 | if interest&1 == 1 { 314 | InterestsIndexCompact[i] = append(InterestsIndexCompact[i],id) 315 | } 316 | interest >>= 1 317 | } 318 | // + unroll interests1 319 | interest = Store.interests[pos+1] 320 | for i := byte(0); i < 64 && interest != 0; i++ { 321 | if interest&1 == 1 { 322 | InterestsIndexCompact[i+63] = append(InterestsIndexCompact[i+63],id) 323 | } 324 | interest >>= 1 325 | } 326 | } 327 | */ 328 | 329 | func appendBirthYearCityIndex(id uint32, account *Account, dstBirth, dstCity, dstBirthCity []*VectorUint32) { 330 | year := account.getBirthYear() - 1950 331 | { 332 | m := dstBirth[year] 333 | if m == nil { 334 | m = makeVector(LARGE_BLOCK_SIZE) 335 | dstBirth[year] = m 336 | } 337 | m.Push(id) 338 | } 339 | city := account.getCity() 340 | { 341 | m := dstCity[city] 342 | if m == nil { 343 | m = makeVector(LARGE_BLOCK_SIZE) 344 | dstCity[city] = m 345 | } 346 | m.Push(id) 347 | } 348 | birthCity := city*BIRTH_YEARS_LEN + year 349 | { 350 | m := dstBirthCity[birthCity] 351 | if m == nil { 352 | m = makeVector(512) 353 | dstBirthCity[birthCity] = m 354 | } 355 | m.Push(id) 356 | } 357 | 358 | } 359 | 360 | func appendCountryIndex(id uint32, account *Account, dst []*VectorUint32) { 361 | city := account.getCountry() 362 | m := dst[city] 363 | if m == nil { 364 | m = makeVector(MEDIUM_BLOCK_SIZE) 365 | dst[city] = m 366 | } 367 | m.Push(id) 368 | } 369 | 370 | func appendFnameIndex(id uint32, account *Account, dst []*VectorUint32) { 371 | city := account.getFname() 372 | m := dst[city] 373 | if m == nil { 374 | m = makeVector(MEDIUM_BLOCK_SIZE) 375 | dst[city] = m 376 | } 377 | m.Push(id) 378 | } 379 | 380 | func appendSexIndex(id uint32, account *Account, dst []*VectorUint32) { 381 | sex := account.data & 1 382 | m := dst[sex] 383 | if m == nil { 384 | m = makeVector(HUGE_BLOCK_SIZE) 385 | dst[sex] = m 386 | } 387 | m.Push(id) 388 | } 389 | 390 | func appendEmailIndex(id uint32, account *Account, dst []*VectorUint32) { 391 | i := emailIndex(Store.Accounts2[id-1].email) 392 | m := dst[i] 393 | if m == nil { 394 | m = makeVector(HUGE_BLOCK_SIZE) 395 | dst[i] = m 396 | } 397 | m.Push(id) 398 | } 399 | 400 | func emailIndex(s string) int { 401 | bytes := S2b(s) 402 | return int(bytes[0]-'a')*26 + int(bytes[1]-'a') 403 | } 404 | 405 | func appendStatusIndex(id uint32, account *Account, dst []*VectorUint32) { 406 | status := account.getStatus() - 1 407 | m := dst[status] 408 | if m == nil { 409 | m = makeVector(HUGE_BLOCK_SIZE) 410 | dst[status] = m 411 | } 412 | m.Push(id) 413 | } 414 | 415 | func appendRecommendAndInterestsIndex(id uint32, account *Account, buffer []byte) { 416 | pos := (id - 1) * 2 417 | sex := byte(account.data & 1) 418 | // + unroll interests0 419 | interests := interestsToArray(Store.interests[pos], Store.interests[pos+1], buffer) 420 | 421 | for i, code := range interests { 422 | addRecommendKeys(id, code, account) 423 | InterestsIndexCompact[code] = append(InterestsIndexCompact[code], id) 424 | InterestsSexIndexCompact[code*2+sex] = append(InterestsSexIndexCompact[code*2+sex], id) 425 | 426 | for j := i + 1; j < len(interests); j++ { 427 | code2x := int(code)*90 + int(interests[j]) 428 | Interests2xIndexCompact[code2x] = append(Interests2xIndexCompact[code2x], id) 429 | 430 | //if m, ok := inter2x[key]; ok { 431 | // m.Push(id) 432 | //} else { 433 | // inter2x[key] = makeInitialVector(MEDIUM_BLOCK_SIZE, id) 434 | //} 435 | } 436 | } 437 | 438 | /* 439 | interest := Store.interests[pos] >> 1 440 | for i := byte(0); i < 63 && interest != 0; i++ { 441 | if interest&1 == 1 { 442 | addRecommendKeys(id, i, account, dst, dstCountry, dstCity) 443 | InterestsIndexCompact[i] = append(InterestsIndexCompact[i], id) 444 | InterestsSexIndexCompact[i*2+sex] = append(InterestsSexIndexCompact[i*2+sex], id) 445 | } 446 | interest >>= 1 447 | } 448 | // + unroll interests1 449 | interest = Store.interests[pos+1] 450 | for i := byte(0); i < 64 && interest != 0; i++ { 451 | if interest&1 == 1 { 452 | addRecommendKeys(id, i+63, account, dst, dstCountry, dstCity) 453 | InterestsIndexCompact[i+63] = append(InterestsIndexCompact[i+63], id) 454 | InterestsSexIndexCompact[(i+63)*2+sex] = append(InterestsSexIndexCompact[(i+63)*2+sex], id) 455 | } 456 | interest >>= 1 457 | } 458 | */ 459 | 460 | } 461 | 462 | func addRecommendKeys(id uint32, interest byte, account *Account) { 463 | 464 | key := recommendKey(byte(account.data&1), account.IsPremium(), account.getStatus()-1, interest) 465 | if xs, ok := RecommendIndexCompact[key]; ok { 466 | RecommendIndexCompact[key] = append(xs, id) 467 | } else { 468 | RecommendIndexCompact[key] = append(make([]uint32, 0, 9500), id) 469 | } 470 | 471 | keyCity := recommendKeyCity(key, account.getCity()) 472 | 473 | if xs, ok := RecommendIndexCityCompact[keyCity]; ok { 474 | RecommendIndexCityCompact[keyCity] = append(xs, id) 475 | } else { 476 | RecommendIndexCityCompact[keyCity] = append([]uint32{}, id) 477 | } 478 | 479 | keyCountry := recommendKeyCountry(key, account.getCountry()) 480 | if xs, ok := RecommendIndexCountryCompact[keyCountry]; ok { 481 | RecommendIndexCountryCompact[keyCountry] = append(xs, id) 482 | } else { 483 | RecommendIndexCountryCompact[keyCountry] = append([]uint32{}, id) 484 | } 485 | 486 | } 487 | 488 | type IndexIterator interface { 489 | Next() uint32 490 | // возвращает индекс если он один или nil если индексов нет или их несколько 491 | ToSingle() []uint32 492 | Len() int 493 | } 494 | 495 | const DEFAULT_ITER_CAPACITY = 16 496 | 497 | type IndexOrIterator struct { 498 | //current int 499 | indexes [][]uint32 500 | positions []int 501 | } 502 | 503 | func makeIndexOrIterator() *IndexOrIterator { 504 | return &IndexOrIterator{ 505 | indexes: make([][]uint32, 0, DEFAULT_ITER_CAPACITY), 506 | positions: make([]int, 0, DEFAULT_ITER_CAPACITY), 507 | } 508 | } 509 | 510 | func (p *IndexOrIterator) push(index []uint32) { 511 | if len(index) != 0 { 512 | p.indexes = append(p.indexes, index) 513 | p.positions = append(p.positions, len(index)-1) 514 | } 515 | } 516 | func (p *IndexOrIterator) ToSingle() (index []uint32) { 517 | if len(p.indexes) == 1 { 518 | return p.indexes[0] 519 | } else if len(p.indexes) == 0 { 520 | return emptyIndex 521 | } 522 | return nil 523 | } 524 | 525 | func (p *IndexOrIterator) Len() (l int) { 526 | for _, i := range p.indexes { 527 | l += len(i) 528 | } 529 | return l 530 | } 531 | 532 | func (p *IndexOrIterator) Next() uint32 { 533 | 534 | nextVal := uint32(0) 535 | nextFound := false 536 | 537 | // найдем след значение для возврата 538 | for i := range p.indexes { 539 | pos := p.positions[i] 540 | if pos >= 0 && nextVal <= p.indexes[i][pos] { 541 | nextVal = p.indexes[i][pos] 542 | nextFound = true 543 | } 544 | } 545 | 546 | if !nextFound { 547 | return math.MaxUint32 548 | } 549 | 550 | // сдвинемся в позициях индексов 551 | for i := range p.indexes { 552 | pos := p.positions[i] 553 | if pos >= 0 && nextVal == p.indexes[i][pos] { 554 | p.positions[i] = p.positions[i] - 1 555 | } 556 | } 557 | 558 | return nextVal 559 | } 560 | 561 | type IndexAndIterator struct { 562 | base []uint32 563 | basePos int 564 | indexes [][]uint32 565 | positions []int 566 | } 567 | 568 | func makeIndexAndIterator() *IndexAndIterator { 569 | return &IndexAndIterator{ 570 | indexes: make([][]uint32, 0, DEFAULT_ITER_CAPACITY), 571 | positions: make([]int, 0, DEFAULT_ITER_CAPACITY), 572 | } 573 | } 574 | 575 | func (p *IndexAndIterator) push(index []uint32) { 576 | p.indexes = append(p.indexes, index) 577 | p.positions = append(p.positions, len(index)-1) 578 | } 579 | 580 | func (p *IndexAndIterator) ToSingle() (index []uint32) { 581 | if len(p.indexes) == 1 { 582 | return p.indexes[0] 583 | } else if len(p.indexes) == 0 { 584 | return emptyIndex 585 | } 586 | return nil 587 | } 588 | 589 | func (p *IndexAndIterator) Prepare() *IndexAndIterator { 590 | // найдем мин индекс 591 | ml := math.MaxInt32 592 | i := 0 593 | for x, xs := range p.indexes { 594 | if l := len(xs); l < ml { 595 | ml = l 596 | i = x 597 | } 598 | } 599 | p.base = p.indexes[i] 600 | p.basePos = len(p.base) - 1 601 | // delete trick, no leaks, https://github.com/golang/go/wiki/SliceTricks 602 | copy(p.indexes[i:], p.indexes[i+1:]) 603 | p.indexes[len(p.indexes)-1] = nil // or the zero value of T 604 | p.indexes = p.indexes[:len(p.indexes)-1] 605 | p.positions = append(p.positions[:i], p.positions[i+1:]...) 606 | return p 607 | } 608 | 609 | // todo: брать самый короткий индекс и пятится по нему назад 610 | func (p *IndexAndIterator) Next() uint32 { 611 | for b := p.basePos; b >= 0; b-- { 612 | max := p.base[b] 613 | nextFound := true 614 | for i, xs := range p.indexes { 615 | //var found = p.moveTo(i, max) 616 | //index := p.indexes[i] 617 | pos := p.positions[i] 618 | found := false 619 | for pos >= 0 && xs[pos] >= max { 620 | pos-- 621 | p.positions[i] = pos 622 | found = xs[pos+1] == max // можно nextFound здесь считать 623 | } 624 | nextFound = nextFound && found 625 | } 626 | if nextFound { 627 | p.basePos = b - 1 628 | return max 629 | } 630 | } 631 | return math.MaxUint32 632 | } 633 | 634 | func (p *IndexAndIterator) Len() (l int) { 635 | for _, i := range p.indexes { 636 | l += len(i) 637 | } 638 | return l 639 | } 640 | 641 | func indexVectorStat(name string, index []*VectorUint32) { 642 | 643 | stat := make(map[int]int) 644 | for _, li := range index { 645 | if li != nil { 646 | stat[li.Len()] += 1 647 | } 648 | } 649 | fmt.Printf("\nindex stat for '%s'\nbucket;size\n", name) 650 | for k, v := range stat { 651 | fmt.Printf("%d;%d\n", k, v) 652 | } 653 | 654 | } 655 | 656 | func indexStat(name string, index [][]uint32) { 657 | stat := make(map[int]int) 658 | for _, li := range index { 659 | if li != nil { 660 | stat[len(li)] += 1 661 | } 662 | } 663 | fmt.Printf("\nindex stat for '%s'\nbucket;size\n", name) 664 | for k, v := range stat { 665 | fmt.Printf("%d;%d\n", k, v) 666 | } 667 | 668 | } 669 | 670 | func indexMapStat(name string, index map[int][]uint32) { 671 | 672 | stat := make(map[int]int) 673 | for _, lk := range index { 674 | stat[len(lk)] += 1 675 | } 676 | fmt.Printf("\nindex stat for '%s'\nbucket;size\n", name) 677 | for k, v := range stat { 678 | fmt.Printf("%d;%d\n", k, v) 679 | } 680 | 681 | } 682 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | h "dsmnko.org/hlc18" 5 | "fmt" 6 | "github.com/valyala/fasthttp" 7 | "log" 8 | "math" 9 | "runtime" 10 | "runtime/debug" 11 | "sort" 12 | "strings" 13 | "sync" 14 | "sync/atomic" 15 | "tidwall/evio" 16 | "time" 17 | ) 18 | 19 | const CAPACITY = 1320000 20 | 21 | // rating`s 22 | const ( 23 | DataDir = "/tmp/data/" 24 | ListenPort = 80 25 | Phase1Count = 27000 26 | Phase2Count = 90000 27 | Phase3Count = 60000 28 | Phase1TestCount = 3000 29 | Phase2TestCount = 10000 30 | Phase3TestCount = 15000 31 | collectStat = false 32 | postOperations = true 33 | ) 34 | 35 | const ( 36 | //DataDir = "/var/data/data2/" //data-test-2612/" //data-test-2401/" 37 | //ListenPort = 8080 38 | //Phase1Count = 16000 //27000 39 | //Phase2Count = 90000 40 | //Phase3Count = 30000 //60000 41 | //Phase1TestCount = 2400 //3000 42 | //Phase2TestCount = 10000 43 | //Phase3TestCount = 10000 //15000 44 | //collectStat = false 45 | //postOperations = true 46 | 47 | Phase1Unique = 16000 48 | Phase3Unique = 30000 49 | 50 | Phase1TestUnique = 2400 51 | Phase3TestUnique = 10000 52 | ) 53 | 54 | var ( 55 | getOverall int64 56 | getDone int64 57 | postOverall int64 58 | postDone int64 59 | 60 | cacheRead int64 61 | 62 | mutate sync.Mutex 63 | 64 | phase = int64(1) 65 | 66 | queryStatMutex sync.Mutex 67 | ) 68 | 69 | const Status200 = "200 OK" 70 | const Status400 = "400 Bad Request" 71 | const Status404 = "404 Not Found" 72 | const HeaderConnectionKeepAlive = "Connection: keep-alive\r\n" 73 | 74 | type QueryStat struct { 75 | key string 76 | count int 77 | sum int64 // nano 78 | max int64 // nano 79 | } 80 | 81 | var queryStat = make(map[string]QueryStat, 64*1024) 82 | 83 | func maxNano(a, b int64) int64 { 84 | if a > b { 85 | return a 86 | } else { 87 | return b 88 | } 89 | } 90 | 91 | func printQueryStatReport() { 92 | if len(queryStat) > 0 { 93 | report := make([]QueryStat, len(queryStat))[:0] 94 | queryStatMutex.Lock() 95 | for _, v := range queryStat { 96 | report = append(report, v) 97 | } 98 | //todo: reset --- queryStat = make(map[string]QueryStat) 99 | queryStatMutex.Unlock() 100 | sort.Slice(report, func(i, j int) bool { 101 | return report[i].sum > report[j].sum 102 | //return report[i].max > report[j].max 103 | }) 104 | fmt.Println("\nrequest;params;count;sum(mls);avg;max") 105 | for _, v := range report[:] { 106 | count := v.count 107 | if count > 0 { 108 | fmt.Printf("%s;%d;%d;%d;%d\n", v.key, count, v.sum/int64(time.Millisecond), v.sum/int64(time.Microsecond)/int64(count), v.max/int64(time.Microsecond)) 109 | } else { 110 | fmt.Printf("%s;%d\n", v.key, count) 111 | } 112 | } 113 | fmt.Println() 114 | } 115 | } 116 | 117 | func updateQueryStat(key string, start int64) { 118 | nano := time.Now().UnixNano() - start 119 | queryStatMutex.Lock() 120 | if stat, ok := queryStat[key]; ok { 121 | queryStat[key] = QueryStat{key, stat.count + 1, stat.sum + nano, maxNano(nano, stat.max)} 122 | } else { 123 | queryStat[key] = QueryStat{key, 1, nano, nano} 124 | } 125 | queryStatMutex.Unlock() 126 | } 127 | 128 | func filterQueryKey(args *fasthttp.Args) string { 129 | keys := make([]string, args.Len())[:0] 130 | args.VisitAll(func(keyBytes, _ []byte) { 131 | k := h.B2s(keyBytes) 132 | if k != "limit" && k != "query_id" { 133 | keys = append(keys, k) 134 | } 135 | }) 136 | key := "filter;" 137 | sort.StringSlice(keys).Sort() 138 | for _, k := range keys { 139 | key += k + "," 140 | } 141 | return key 142 | } 143 | 144 | func sortedParams(params map[string]string) (key string) { 145 | keys := make([]string, len(params))[:0] 146 | for k, _ := range params { 147 | if k != "order" && k != "keys" { 148 | keys = append(keys, k) 149 | } 150 | } 151 | sort.StringSlice(keys).Sort() 152 | for _, k := range keys { 153 | key += k + "," 154 | } 155 | return key 156 | } 157 | 158 | func groupQueryKey(p *h.Group) string { 159 | key := fmt.Sprintf("group;%v/%d/", p.GroupBy, p.Order) 160 | return key + sortedGroupFilters(&p.FilterBy) 161 | } 162 | 163 | func sortedGroupFilters(p *map[string]bool) (key string) { 164 | keys := make([]string, len(*p))[:0] 165 | for k := range *p { 166 | keys = append(keys, k) 167 | } 168 | sort.StringSlice(keys).Sort() 169 | for _, k := range keys { 170 | key += k + "," 171 | } 172 | return key 173 | } 174 | 175 | func recommendQueryKey(params map[string]string) string { 176 | return "recommend;" + sortedParams(params) 177 | } 178 | 179 | func suggestQueryKey(params map[string]string) string { 180 | return "suggest;" + sortedParams(params) 181 | } 182 | 183 | var EMPTY_ACCOUNTS = []byte("{\"accounts\":[]}") 184 | var EMPTY_GROUPS = []byte("{\"groups\":[]}") 185 | var EMPTY_POST = []byte("{}") 186 | 187 | func FilterHandler(c evio.Conn, args *fasthttp.Args, out /*, body */ []byte, query string, readCache bool) []byte { 188 | var cacheToken = string(h.S2b(query)) 189 | 190 | if readCache { 191 | if bytes := h.GetFilterCache(cacheToken); bytes != nil { 192 | return c.WriteAhead(bytes) 193 | } 194 | println("GetFilterCache ", "miss") 195 | } 196 | filter, err := h.MakeFilterArgs(args) 197 | if err != nil || filter.Limit < 1 { 198 | out = h.AppendHttpResponse(out, Status400, "", nil) 199 | tail := c.WriteAhead(out) 200 | h.PutFilterCache(cacheToken, out) 201 | return tail 202 | } 203 | if filter.ShortCircuit || filter.Limit > 50 { 204 | out = h.AppendHttpResponse(out, Status200, "", EMPTY_ACCOUNTS) 205 | tail := c.WriteAhead(out) 206 | h.PutFilterCache(cacheToken, out) 207 | return tail 208 | } 209 | if collectStat { 210 | nanoStart := time.Now().UnixNano() 211 | defer updateQueryStat(filterQueryKey(args), nanoStart) 212 | } 213 | bodyBuffer := bodyBufferPool.Get().([]byte) 214 | body := bodyBuffer[:0] 215 | 216 | var consumer = func(separate bool, fields map[string]bool, account *h.Account, accountId uint32) { 217 | body = h.WriteAccountOut(body, separate, filter.Fields, account, accountId) 218 | } 219 | 220 | body = append(body, "{\"accounts\":["...) 221 | filter.Process(filter.Limit, consumer) 222 | body = append(body, "]}"...) 223 | 224 | out = h.AppendHttpResponse(out, Status200, "", body) 225 | tail := c.WriteAhead(out) 226 | bodyBufferPool.Put(bodyBuffer) 227 | h.PutFilterCache(cacheToken, out) 228 | return tail 229 | } 230 | 231 | func GroupHandler(c evio.Conn, args *fasthttp.Args, out []byte, query string, readCache bool) []byte { 232 | var cacheToken = string(h.S2b(query)) 233 | if readCache { 234 | if bytes := h.GetGroupCache(cacheToken); bytes != nil { 235 | return c.WriteAhead(bytes) 236 | } 237 | println("GetGroupCache ", "miss") 238 | } 239 | var group h.Group 240 | if err := group.FromArgs(args); err != nil || group.Limit < 1 { 241 | out = h.AppendHttpResponse(out, Status400, "", nil) 242 | tail := c.WriteAhead(out) 243 | h.PutGroupCache(cacheToken, out) 244 | return tail 245 | } 246 | if group.Filter.ShortCircuit { 247 | out = h.AppendHttpResponse(out, Status200, "", EMPTY_GROUPS) 248 | tail := c.WriteAhead(out) 249 | h.PutGroupCache(cacheToken, out) 250 | return tail 251 | } 252 | if collectStat { 253 | nanoStart := time.Now().UnixNano() 254 | defer updateQueryStat(groupQueryKey(&group), nanoStart) 255 | } 256 | 257 | bodyBuffer := bodyBufferPool.Get().([]byte) 258 | body := bodyBuffer[:0] 259 | 260 | if items := group.Classify(&h.Store.Accounts, true); len(items) > 0 { 261 | body = append(body, "{\"groups\":["...) 262 | for i, item := range items { 263 | if i > 0 { 264 | body = append(body, ',') 265 | } 266 | body = group.WriteJsonGroupItemOut(body, item.Key, item.Count) 267 | } 268 | body = append(body, "]}"...) 269 | } else { 270 | out = h.AppendHttpResponse(out, Status200, "", EMPTY_GROUPS) 271 | tail := c.WriteAhead(out) 272 | bodyBufferPool.Put(bodyBuffer) 273 | h.PutGroupCache(cacheToken, out) 274 | return tail 275 | } 276 | out = h.AppendHttpResponse(out, Status200, "", body) 277 | tail := c.WriteAhead(out) 278 | bodyBufferPool.Put(bodyBuffer) 279 | h.PutGroupCache(cacheToken, out) 280 | return tail 281 | } 282 | 283 | // Особенность 8. Если в хранимых данных не существует пользователя с переданным id, то ожидается код 404 с пустым телом ответа. 284 | func RecommendHandler(c evio.Conn, args *fasthttp.Args, id uint32, out []byte, query string, readCache bool) []byte { 285 | var cacheToken = string(h.S2b(query)) 286 | if readCache { 287 | if bytes := h.GetRecommendCache(cacheToken); bytes != nil { 288 | return c.WriteAhead(bytes) 289 | } 290 | println("GetRecommendCache ", "miss") 291 | } 292 | limit, _, params := ParseParams(args) 293 | if limit < 1 { 294 | out = h.AppendHttpResponse(out, Status400, "", nil) 295 | tail := c.WriteAhead(out) 296 | h.PutRecommendCache(cacheToken, out) 297 | return tail 298 | } 299 | if v, ok := params["city"]; ok && v == "" { 300 | out = h.AppendHttpResponse(out, Status400, "", nil) 301 | tail := c.WriteAhead(out) 302 | h.PutRecommendCache(cacheToken, out) 303 | return tail 304 | } 305 | if v, ok := params["country"]; ok && v == "" { 306 | out = h.AppendHttpResponse(out, Status400, "", nil) 307 | tail := c.WriteAhead(out) 308 | h.PutRecommendCache(cacheToken, out) 309 | return tail 310 | } 311 | if id > h.Store.MaxId || id < 1 || h.Store.Accounts[id-1].IsEmpty() { 312 | out = h.AppendHttpResponse(out, Status404, "", nil) 313 | tail := c.WriteAhead(out) 314 | h.PutRecommendCache(cacheToken, out) 315 | return tail 316 | } 317 | if collectStat { 318 | nanoStart := time.Now().UnixNano() 319 | defer updateQueryStat(recommendQueryKey(params), nanoStart) 320 | } 321 | 322 | bodyBuffer := bodyBufferPool.Get().([]byte) 323 | body := bodyBuffer[:0] 324 | 325 | body = append(body, "{\"accounts\":["...) 326 | recommend := h.Recommend(id, limit, params) 327 | for i, accId := range recommend { 328 | body = h.WriteAccountOutRecommend(body, i != 0, &h.Store.Accounts[accId-1], accId) 329 | } 330 | body = append(body, "]}"...) 331 | 332 | out = h.AppendHttpResponse(out, Status200, "", body) 333 | tail := c.WriteAhead(out) 334 | bodyBufferPool.Put(bodyBuffer) 335 | h.PutRecommendCache(cacheToken, out) 336 | return tail 337 | } 338 | 339 | func SuggestHandler(c evio.Conn, args *fasthttp.Args, id uint32, out []byte, query string, readCache bool) []byte { 340 | var cacheToken = string(h.S2b(query)) 341 | if readCache { 342 | if bytes := h.GetSuggestCache(cacheToken); bytes != nil { 343 | return c.WriteAhead(bytes) 344 | } 345 | println("GetSuggestCache ", "miss") 346 | } 347 | limit, _, params := ParseParams(args) 348 | if limit < 1 { 349 | out = h.AppendHttpResponse(out, Status400, "", nil) 350 | tail := c.WriteAhead(out) 351 | h.PutSuggestCache(cacheToken, out) 352 | return tail 353 | } 354 | if v, ok := params["city"]; ok && v == "" { 355 | out = h.AppendHttpResponse(out, Status400, "", nil) 356 | tail := c.WriteAhead(out) 357 | h.PutSuggestCache(cacheToken, out) 358 | return tail 359 | } 360 | if v, ok := params["country"]; ok && v == "" { 361 | out = h.AppendHttpResponse(out, Status400, "", nil) 362 | tail := c.WriteAhead(out) 363 | h.PutSuggestCache(cacheToken, out) 364 | return tail 365 | } 366 | if id > h.Store.MaxId || id < 1 || h.Store.Accounts[id-1].IsEmpty() { 367 | out = h.AppendHttpResponse(out, Status404, "", nil) 368 | tail := c.WriteAhead(out) 369 | h.PutSuggestCache(cacheToken, out) 370 | return tail 371 | } 372 | if collectStat { 373 | nanoStart := time.Now().UnixNano() 374 | defer updateQueryStat(suggestQueryKey(params), nanoStart) 375 | } 376 | bodyBuffer := bodyBufferPool.Get().([]byte) 377 | body := bodyBuffer[:0] 378 | 379 | body = append(body, "{\"accounts\":["...) 380 | suggested := h.Suggest(id, limit, params) 381 | for i, accId := range suggested { 382 | body = h.WriteAccountOutSuggest(body, i != 0, &h.Store.Accounts[accId-1], accId) 383 | } 384 | body = append(body, "]}"...) 385 | 386 | out = h.AppendHttpResponse(out, Status200, "", body) 387 | tail := c.WriteAhead(out) 388 | bodyBufferPool.Put(bodyBuffer) 389 | h.PutSuggestCache(cacheToken, out) 390 | return tail 391 | } 392 | 393 | /** 394 | В ответе ожидается код 201 с пустым json-ом в теле ответа ({}), если создание нового пользователя прошло успешно. 395 | В случае некорректных типов данных или неизвестных ключей нужно вернуть код 400 с пустым телом. 396 | */ 397 | // /accounts/new/ -> 201 398 | func NewHandler(c evio.Conn, out, body []byte) []byte { 399 | var accountJson h.AccountJson 400 | if err := h.UnmarshalNew(&accountJson, body); /*accountJson.UnmarshalJSON(body)*/ err != nil { 401 | return c.WriteAhead(h.AppendHttpResponse(out, Status400, HeaderConnectionKeepAlive, nil)) 402 | } 403 | if accountJson.Id > CAPACITY || !h.Store.Accounts[accountJson.Id-1].IsEmpty() { 404 | return c.WriteAhead(h.AppendHttpResponse(out, Status400, HeaderConnectionKeepAlive, nil)) 405 | } 406 | mutate.Lock() 407 | if err := h.ValidateNew(&accountJson); err != nil { 408 | mutate.Unlock() 409 | return c.WriteAhead(h.AppendHttpResponse(out, Status400, HeaderConnectionKeepAlive, nil)) 410 | } 411 | mutate.Unlock() 412 | out = c.WriteAhead(h.AppendHttpResponse(out, "201 Created", HeaderConnectionKeepAlive, EMPTY_POST)) 413 | if h.Store.MaxId < accountJson.Id { 414 | h.Store.MaxId = accountJson.Id 415 | } 416 | mutate.Lock() 417 | if err := h.Store.CompressToAccount(&accountJson); err != nil { 418 | log.Fatalf("new account fatal: %v, accountJson: %v", err, accountJson) 419 | } 420 | mutate.Unlock() 421 | return out 422 | } 423 | 424 | /** 425 | В ответе ожидается код 202 с пустым json-ом в теле ответа ({}), если обновление прошло успешно. 426 | Если запись с указанным id не существует в имеющихся данных, то ожидается код 404 с пустым телом. 427 | Если запись существует, но в теле запроса переданы неизвестные поля или типы значений неверны, то ожидается код 400. 428 | */ 429 | 430 | // /accounts// (/accounts/46133/?query_id=308) -> 202 431 | func UpdateHandler(c evio.Conn, out, body []byte, id uint32) []byte { 432 | //nanoStart := time.Now().UnixNano();defer updateQueryStat("update;full;", nanoStart) 433 | if id < 1 || id > atomic.LoadUint32(&h.Store.MaxId) /*|| h.Store.Accounts[id-1].sexStatus == 0*/ { 434 | return h.AppendHttpResponse(out, Status404, HeaderConnectionKeepAlive, nil) 435 | } 436 | var accountJson h.AccountJson 437 | accountJson.Birth = math.MaxInt32 438 | accountJson.Id = id 439 | if err := h.UnmarshalUpdate(&accountJson, body); /*accountJson.UnmarshalJSON(body)*/ err != nil { 440 | return c.WriteAhead(h.AppendHttpResponse(out, Status400, HeaderConnectionKeepAlive, nil)) 441 | } 442 | // проверить уникальность телефона и email 443 | mutate.Lock() 444 | if err := h.ValidateUpdate(&accountJson, id); err != nil { 445 | mutate.Unlock() 446 | return c.WriteAhead(h.AppendHttpResponse(out, Status400, HeaderConnectionKeepAlive, nil)) 447 | } 448 | mutate.Unlock() 449 | out = c.WriteAhead(h.AppendHttpResponse(out, "202 Accepted", HeaderConnectionKeepAlive, EMPTY_POST)) 450 | //nanoStart2 := time.Now().UnixNano(); defer updateQueryStat("update;update;", nanoStart2) 451 | accountJson.Id = id 452 | mutate.Lock() 453 | if err := h.Store.CompressUpdateToAccount(&accountJson); err != nil { 454 | log.Fatalf("update account fatal: %v, accountJson: %v", err, accountJson) 455 | } 456 | mutate.Unlock() 457 | return out 458 | } 459 | 460 | /** 461 | В ответе ожидается код 202 с пустым json-ом в теле ответа ({}), если обновление прошло успешно. 462 | Если в теле запроса переданы неизвестные поля или типы значений неверны, то ожидается код 400. 463 | */ 464 | func LikesHandler(c evio.Conn, out, body []byte) []byte { 465 | //nanoStart := time.Now().UnixNano();defer updateQueryStat("like;full;", nanoStart) 466 | var err error 467 | var likes []h.LikeUpdate 468 | if likes, err = h.ParseLikesUpdate(body); err != nil { 469 | return c.WriteAhead(h.AppendHttpResponse(out, Status400, "", nil)) 470 | } 471 | if err := h.ValidateLikes(likes); err != nil { 472 | return c.WriteAhead(h.AppendHttpResponse(out, Status400, "", nil)) 473 | } 474 | 475 | out = c.WriteAhead(h.AppendHttpResponse(out, "202 Accepted", HeaderConnectionKeepAlive, EMPTY_POST)) 476 | //nanoStart2 := time.Now().UnixNano();defer updateQueryStat("like;;", nanoStart2) 477 | mutate.Lock() // todo: ? отдельные локи на обновление индекса лайков и данных store/счетов 478 | h.Store.LikesUpdate(likes) 479 | mutate.Unlock() 480 | return out 481 | } 482 | 483 | func main() { 484 | 485 | debug.SetGCPercent(10) 486 | h.InitStore(CAPACITY) 487 | h.MakeIndexes(CAPACITY) 488 | var err error 489 | if err = h.Store.LoadData(DataDir); err != nil { 490 | log.Fatal(err) 491 | } 492 | fmt.Printf("%v\truntype: %d, now: %d\n", h.Timenow(), h.Store.RunType, h.Store.Now) 493 | runtime.GC() 494 | PrintReport() 495 | 496 | evio.SetEpollWait(-1) 497 | debug.SetGCPercent(-1) 498 | h.PrintMemUsage("GC off") 499 | 500 | //defer profile.Start(profile.MemProfileRate(512), profile.ProfilePath(".")).Stop() 501 | //defer profile.Start(profile.CPUProfile, profile.ProfilePath(".")).Stop() 502 | //defer profile.Start(profile.MutexProfile, profile.ProfilePath(".")).Stop() 503 | 504 | h.EvioServer(ListenPort, EvioHandler) 505 | return 506 | 507 | } 508 | 509 | //var prof interface {Stop()} 510 | func PrePost() int64 { 511 | counter := atomic.AddInt64(&postOverall, 1) 512 | if counter == 1 { 513 | // начало фазы 2 514 | //prof = profile.Start(profile.MutexProfile/*profile.MemProfileRate(512)*/, profile.ProfilePath(".")) 515 | evio.SetEpollWait(0) 516 | if atomic.CompareAndSwapInt64(&phase, 1, 2) { 517 | _, _ = fmt.Printf("%v\t* phase-2 started, getOverall %d\n", h.Timenow(), atomic.LoadInt64(&getOverall)) 518 | } 519 | } 520 | return counter 521 | } 522 | 523 | func PostPost(_ int64) { 524 | done := atomic.AddInt64(&postDone, 1) 525 | if (done == Phase2TestCount && h.Store.RunType == 0) || (done == Phase2Count && h.Store.RunType == 1) { 526 | evio.SetEpollWait(-1) 527 | //prof.Stop() 528 | if postOperations { 529 | Phase_2_Copmplete() 530 | } 531 | } 532 | 533 | // todo: spikes hunt 534 | // if done == 1000 { 535 | // printQueryStatReport() 536 | // } 537 | } 538 | 539 | func PreGet() int64 { 540 | counter := atomic.AddInt64(&getOverall, 1) 541 | if 1 == counter { 542 | evio.SetEpollWait(0) 543 | _, _ = fmt.Printf("%v\t* phase-1 started, postOverall %d\n", h.Timenow(), atomic.LoadInt64(&postOverall)) 544 | } 545 | if 2 == atomic.LoadInt64(&phase) { 546 | //evio.SetEpollWait(0) 547 | if atomic.CompareAndSwapInt64(&phase, 2, 3) { 548 | _, _ = fmt.Printf("%v\t* phase-3 started, postOverall %d\n", h.Timenow(), atomic.LoadInt64(&postOverall)) 549 | } 550 | } 551 | return counter 552 | } 553 | 554 | func PostGet(_ int64) { 555 | done := atomic.AddInt64(&getDone, 1) 556 | 557 | if h.Store.RunType == 1 { 558 | if done == Phase1Unique || done == (Phase1Count+Phase3Unique) { 559 | println("cache on") 560 | atomic.StoreInt64(&cacheRead, 1) 561 | } 562 | } else { 563 | if done == Phase1TestUnique || done == (Phase1TestCount+Phase3TestUnique) { 564 | println("cache on") 565 | atomic.StoreInt64(&cacheRead, 1) 566 | } 567 | } 568 | 569 | if (h.Store.RunType == 1 && done == Phase1Count) || 570 | (h.Store.RunType == 0 && done == Phase1TestCount) { 571 | evio.SetEpollWait(-1) 572 | if postOperations { 573 | Phase_1_Complete(done) 574 | } 575 | } else if (h.Store.RunType == 1 && done == (Phase1Count+Phase3Count)) || 576 | (h.Store.RunType == 0 && done == (Phase1TestCount+Phase3TestCount)) { 577 | evio.SetEpollWait(-1) 578 | atomic.StoreInt64(&cacheRead, 0) 579 | println("cache off") 580 | if postOperations { 581 | Phase_3_Complete(done) 582 | } 583 | } 584 | 585 | } 586 | 587 | func ParseParams(args *fasthttp.Args) (limit, queryId int, params map[string]string) { 588 | limit = -1 589 | params = make(map[string]string) 590 | 591 | args.VisitAll(func(keyBytes, valueBytes []byte) { 592 | key := h.B2s(keyBytes) 593 | switch key { 594 | case "limit": 595 | limit, _ = fasthttp.ParseUint(valueBytes) 596 | case "query_id": 597 | queryId, _ = fasthttp.ParseUint(valueBytes) 598 | default: 599 | params[key] = h.B2s(valueBytes) 600 | } 601 | }) 602 | return limit, queryId, params 603 | } 604 | 605 | func PrintReport() { 606 | mutate.Lock() 607 | info := fmt.Sprintf("capacity %v accounts, maxId %v, ", len(h.Store.Accounts), h.Store.MaxId) 608 | info += fmt.Sprintf("fnames:%v,", h.FnameDict.Len()) 609 | info += fmt.Sprintf("snames:%v,", h.SnameDict.Len()) 610 | info += fmt.Sprintf("interests:%v,", h.InterestDict.Len()) 611 | info += fmt.Sprintf("cities:%v,", h.CityDict.Len()) 612 | info += fmt.Sprintf("countries:%v,", h.CountryDict.Len()) 613 | info += fmt.Sprintf("domains:%v,", h.DomainDict.Len()) 614 | info += fmt.Sprintf("emails:%v,", len(h.EmailMap)) 615 | info += fmt.Sprintf("phones:%v", len(h.PhoneMap)) 616 | mutate.Unlock() 617 | h.PrintMemUsage(info) 618 | } 619 | 620 | var bodyBufferPool = sync.Pool{ 621 | New: func() interface{} { 622 | return make([]byte, 4096) 623 | }, 624 | } 625 | 626 | func EvioHandler(c evio.Conn, in []byte) (out []byte, action evio.Action) { 627 | if len(in) == 0 { // len(in) 628 | //todo: wtf ? 629 | log.Printf("empty in\n") 630 | return 631 | } 632 | 633 | ctx := c.Context().(*h.Context) 634 | data := ctx.Is.Begin(in) 635 | out = ctx.Out[:0] 636 | var req h.Request 637 | for { 638 | leftover, err := h.Parsereq(data, &req) 639 | if err != nil { 640 | log.Printf("500 Error: %v, %v\n", err, req) 641 | out = h.AppendHttpResponse(out, "500 Error", "", nil) 642 | action = evio.Close 643 | break 644 | } else if len(leftover) == len(data) { 645 | // Request not ready, yet 646 | // log.Printf("leftover (%d) %s\n", len(leftover), req.Query) 647 | break 648 | } 649 | 650 | // handle the Request 651 | if req.Method == "GET" { 652 | //out = c.WriteAhead(h.AppendHttpResponse(out, Status404, "", nil)) 653 | counter := PreGet() 654 | readCache := 1 == atomic.LoadInt64(&cacheRead) 655 | var args = fasthttp.AcquireArgs() 656 | args.ParseBytes(h.S2b(req.Query)) 657 | //println(req.Path, req.Query) 658 | if "/accounts/filter/" == req.Path { 659 | out = FilterHandler(c, args, out, req.Query, readCache) 660 | //updateQueryStat(fmt.Sprintf("%d:%s%s", counter, req.Path, req.Query), nanoStart) 661 | } else if "/accounts/group/" == req.Path { 662 | out = GroupHandler(c, args, out, req.Query, readCache) 663 | //updateQueryStat(fmt.Sprintf("%d:%s%s", counter, req.Path, req.Query), nanoStart) 664 | } else if strings.HasSuffix(req.Path, "/recommend/") { 665 | if id, err := fasthttp.ParseUint(h.S2b(req.Path)[10 : len(req.Path)-11]); err == nil && id > 0 { 666 | out = RecommendHandler(c, args, uint32(id), out, req.Query, readCache) 667 | } else { 668 | // todo: cache this shit too? 669 | out = c.WriteAhead(h.AppendHttpResponse(out, Status404, "", nil)) 670 | } 671 | } else if strings.HasSuffix(req.Path, "/suggest/") { 672 | if id, err := fasthttp.ParseUint(h.S2b(req.Path)[10 : len(req.Path)-9]); err == nil && id > 0 { 673 | out = SuggestHandler(c, args, uint32(id), out, req.Query, readCache) 674 | } else { 675 | // todo: cache this shit too? 676 | out = c.WriteAhead(h.AppendHttpResponse(out, Status404, "", nil)) 677 | } 678 | 679 | } else { 680 | out = c.WriteAhead(h.AppendHttpResponse(out, Status404, "", nil)) 681 | } 682 | fasthttp.ReleaseArgs(args) 683 | PostGet(counter) 684 | } else { // POST 685 | counter := PrePost() 686 | if strings.HasSuffix(req.Path, "/new/") { 687 | out = NewHandler(c, out, h.S2b(req.Body)) 688 | } else if strings.HasSuffix(req.Path, "/likes/") { 689 | out = LikesHandler(c, out, h.S2b(req.Body)) 690 | } else if strings.HasPrefix(req.Path, "/accounts/") { 691 | if id, err := fasthttp.ParseUint(h.S2b(req.Path)[10 : len(req.Path)-1]); err == nil && id > 0 { 692 | out = UpdateHandler(c, out, h.S2b(req.Body), uint32(id)) 693 | } else { 694 | out = c.WriteAhead(h.AppendHttpResponse(out, Status404, HeaderConnectionKeepAlive, nil)) 695 | } 696 | } else { 697 | out = c.WriteAhead(h.AppendHttpResponse(out, Status400, HeaderConnectionKeepAlive, nil)) 698 | } 699 | PostPost(counter) 700 | //updateQueryStat(fmt.Sprintf("%d:%s%s", counter, req.Path, req.Query), nanoStart) 701 | } 702 | if len(leftover) == 2 && leftover[0] == '\r' && leftover[1] == '\n' { 703 | data = leftover[2:] 704 | } else { 705 | data = leftover 706 | } 707 | if len(data) == 0 { 708 | //updateQueryStat(string(h.S2b(req.Query)), nanoStart) 709 | break 710 | } 711 | } 712 | 713 | ctx.Is.End(data) 714 | return 715 | } 716 | 717 | func Phase_2_Copmplete() { 718 | 719 | timer := time.NewTimer(50 * time.Millisecond) 720 | go func() { 721 | <-timer.C 722 | h.PrintMemUsage(fmt.Sprintf("* phase-2 finished, getOverall %d, fire rebuild timer", atomic.LoadInt64(&getOverall))) 723 | debug.SetGCPercent(70) 724 | println("GC on 70") 725 | h.Store.Rebuild(true) 726 | PrintReport() 727 | debug.SetGCPercent(-1) 728 | h.PrintMemUsage("GC off") 729 | }() 730 | } 731 | 732 | func Phase_3_Complete(counter int64) { 733 | _, _ = fmt.Printf("%v\t* phase-3 complete, getOverall %d, postOverall %d\n", h.Timenow(), counter, postOverall) 734 | timer := time.NewTimer(250 * time.Millisecond) 735 | go func() { 736 | <-timer.C 737 | h.PrintMemUsage("GC on 10") 738 | debug.SetGCPercent(10) 739 | printQueryStatReport() 740 | }() 741 | } 742 | 743 | func Phase_1_Complete(counter int64) { 744 | timer := time.NewTimer(500 * time.Millisecond) 745 | go func() { 746 | <-timer.C 747 | atomic.StoreInt64(&cacheRead, 0) 748 | println("cache off") 749 | h.PrintMemUsage(fmt.Sprintf("* phase-1 complete, getOverall %d", counter)) 750 | debug.SetGCPercent(70) 751 | fmt.Printf("%v\tGC on 70, and ResetIndexes()/h.ResetCaches()\n", h.Timenow()) 752 | h.ResetIndexes() 753 | h.ResetCaches() 754 | runtime.GC() 755 | debug.SetGCPercent(-1) 756 | h.PrintMemUsage("GC off") 757 | //evio.SetEpollWait(0) 758 | }() 759 | } 760 | -------------------------------------------------------------------------------- /go-path/src/dsmnko.org/hlc18/groupindex.go: -------------------------------------------------------------------------------- 1 | package hlc18 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "strconv" 7 | "sync" 8 | "tidwall/evio" 9 | "unsafe" 10 | ) 11 | 12 | const ( 13 | BIRTH_YEARS = 56 14 | JOINED_YEARS = 8 15 | STATUSES = 3 16 | ) 17 | 18 | var _aggregates []Aggregate 19 | 20 | type Aggregate interface { 21 | Index(id uint32, account *Account) 22 | Aggregate(group *Group) (map[uint64]int32, bool) 23 | GroupKey(x int, y int, groups []bool) uint64 24 | } 25 | 26 | func capacityOf(name string) int { 27 | switch name { 28 | case "status": 29 | return STATUSES 30 | case "country": 31 | return len(CountryDict.values) + 1 32 | case "city": 33 | return len(CityDict.values) + 1 34 | case "sex": 35 | return 2 36 | case "joined": 37 | return JOINED_YEARS 38 | case "birth": 39 | return BIRTH_YEARS 40 | 41 | default: 42 | log.Fatal("capacityOf " + name) 43 | return -1 44 | } 45 | } 46 | 47 | func offsetOf(name string) uint { 48 | switch name { 49 | case "status": 50 | return STATUS_OFFSET 51 | case "country": 52 | return COUNTRY_OFFSET 53 | case "city": 54 | return CITY_OFFSET 55 | case "sex": 56 | return 0 57 | case "joined": 58 | return JOINED_Y_OFFSET 59 | case "birth": 60 | return BIRTH_Y_OFFSET 61 | 62 | default: 63 | log.Fatal("offsetOf " + name) 64 | return 0 65 | } 66 | } 67 | 68 | func filterValueOf(name string, group *Group) int { 69 | switch name { 70 | case "status": 71 | return int(group.Filter.status_eq) - 1 72 | case "country": 73 | return int(group.Filter.country_eq) 74 | case "city": 75 | return int(group.Filter.city_eq) 76 | case "sex": 77 | return int(group.Filter.sex_eq) - 1 78 | case "joined": 79 | return int(group.Filter.joined_year) - 2011 80 | case "birth": 81 | return int(group.Filter.birth_year) - 1950 82 | case "interests": 83 | return int(group.interestKeyCode) - 1 84 | default: 85 | log.Fatal("filterValueOf " + name) 86 | return -1 87 | } 88 | } 89 | 90 | func sexIndexFunc(a *Account) uint { return uint(a.getSex() - 1) } 91 | 92 | func indexFuncOf(name string) func(*Account) uint { 93 | switch name { 94 | case "status": 95 | return func(a *Account) uint { return uint(a.getStatus() - 1) } 96 | case "country": 97 | return func(a *Account) uint { return uint(a.getCountry()) } 98 | case "city": 99 | return func(a *Account) uint { return uint(a.getCity()) } 100 | case "sex": 101 | return sexIndexFunc 102 | case "joined": 103 | return func(a *Account) uint { return uint(a.getJoinedYear() - 2011) } 104 | case "birth": 105 | return func(a *Account) uint { return uint(a.getBirthYear() - 1950) } 106 | default: 107 | log.Fatal("indexFuncOf " + name) 108 | return nil 109 | } 110 | } 111 | 112 | var rangeBackend []int 113 | 114 | func RebuildAggregates(accounts *[]Account, maxId uint32) { 115 | rangeBackend = make([]int, 8192*2) 116 | for i := range rangeBackend { 117 | rangeBackend[i] = i 118 | } 119 | 120 | _aggregates = []Aggregate{ 121 | 122 | makeInterestBy("country", "sex"), 123 | makeInterestBy("country", "status"), 124 | makeInterestBy("country", "joined"), 125 | makeInterestBy("country", "birth"), 126 | 127 | makeInterestBy("city", "sex"), 128 | makeInterestBy("city", "status"), 129 | makeInterestBy("city", "birth"), 130 | makeInterestBy("city", "joined"), 131 | 132 | makeTrippleAggregate("birth", "sex", "status"), 133 | 134 | makeTrippleAggregate("country", "sex", "status"), 135 | makeTrippleAggregate("country", "sex", "joined"), 136 | makeTrippleAggregate("country", "sex", "birth"), 137 | makeTrippleAggregate("country", "status", "joined"), 138 | makeTrippleAggregate("country", "status", "birth"), 139 | 140 | makeTrippleAggregate("city", "sex", "status"), 141 | makeTrippleAggregate("city", "sex", "joined"), 142 | makeTrippleAggregate("city", "sex", "birth"), 143 | makeTrippleAggregate("city", "status", "joined"), 144 | makeTrippleAggregate("city", "status", "birth"), 145 | 146 | makeQuadAggregate("country", "sex", "status", "joined"), 147 | makeQuadAggregate("country", "sex", "status", "birth"), 148 | 149 | makeQuadAggregate("city", "sex", "status", "birth"), 150 | makeQuadAggregate("city", "sex", "status", "joined"), 151 | 152 | makeInterestBy4("country", "sex", "joined"), 153 | makeInterestBy4("country", "sex", "birth"), 154 | makeInterestBy4("country", "status", "joined"), 155 | makeInterestBy4("country", "status", "birth"), 156 | 157 | makeInterestBy4("city", "sex", "joined"), 158 | makeInterestBy4("city", "sex", "birth"), 159 | makeInterestBy4("city", "status", "joined"), 160 | makeInterestBy4("city", "status", "birth"), 161 | } 162 | for i := uint32(0); i < maxId; i++ { 163 | account := &((*accounts)[i]) 164 | for _, aggregate := range _aggregates { 165 | aggregate.Index(i+1, account) 166 | } 167 | } 168 | } 169 | 170 | /** interestBy */ 171 | type interestBy struct { 172 | data [][]int32 173 | first, second, third string 174 | 175 | stripe int 176 | firstOffset, secondOffset, thirdOffset uint 177 | firstIndex, secondIndex, thirdIndex func(*Account) uint 178 | names map[string]int 179 | backData []int32 180 | } 181 | 182 | func makeInterestBy(second, third string) *interestBy { 183 | p := interestBy{ 184 | first: "interests", 185 | firstOffset: SNAME_OFFSET, 186 | second: second, 187 | secondOffset: offsetOf(second), 188 | secondIndex: indexFuncOf(second), 189 | stripe: 1, 190 | names: map[string]int{"interests": 0, second: 1}, 191 | } 192 | 193 | p.stripe = capacityOf(third) 194 | p.third = third 195 | p.thirdOffset = offsetOf(third) 196 | p.thirdIndex = indexFuncOf(third) 197 | p.names[third] = 2 198 | 199 | p.data = make([][]int32, capacityOf(second)*p.stripe) 200 | 201 | columnCapacity := len(InterestDict.values) 202 | p.backData = make([]int32, len(p.data)*columnCapacity) 203 | 204 | for i := 0; i < len(p.data); i++ { 205 | p.data[i] = p.backData[i*columnCapacity : (i+1)*columnCapacity] 206 | //p.data[i] = make([]int32, columnCapacity) 207 | } 208 | 209 | return &p 210 | } 211 | 212 | func (p *interestBy) Index(id uint32, account *Account) { 213 | j := p.secondIndex(account)*uint(p.stripe) + p.thirdIndex(account) 214 | 215 | pos := (id - 1) * 2 216 | interest0 := Store.interests[pos] >> 1 217 | interest1 := Store.interests[pos+1] 218 | 219 | xs := p.data[j] 220 | pxs0 := uintptr(unsafe.Pointer(&xs[0])) 221 | 222 | // + unroll interests0 223 | for i := uintptr(0); i < 63 && interest0 != 0; i++ { 224 | if 1 == interest0&1 { 225 | //xs[i] += 1 226 | *(*uint32)(unsafe.Pointer(pxs0 + i*4)) += 1 227 | 228 | } 229 | interest0 >>= 1 230 | } 231 | // + unroll interests1 232 | for i := uintptr(63); i < 127 && interest1 != 0; i++ { 233 | if 1 == interest1&1 { 234 | //xs[i] += 1 235 | *(*uint32)(unsafe.Pointer(pxs0 + i*4)) += 1 236 | } 237 | interest1 >>= 1 238 | } 239 | } 240 | 241 | func (p *interestBy) GroupKey(i, j int, groups []bool) uint64 { 242 | var key uint64 243 | if groups[0] { 244 | key |= keyBits(i+1, p.firstOffset) 245 | } 246 | if groups[1] { 247 | key |= keyBits(j/p.stripe, p.secondOffset) 248 | } 249 | if groups[2] { 250 | key |= keyBits(j%p.stripe, p.thirdOffset) 251 | } 252 | return key 253 | } 254 | 255 | func (p *interestBy) Aggregate(group *Group) (map[uint64]int32, bool) { 256 | if len(group.GroupBy) > 3 || len(group.FilterBy) > 3 { 257 | return nil, false 258 | } 259 | 260 | groups := []bool{false, false, false} 261 | for _, n := range group.GroupBy { 262 | if v, ok := p.names[n]; ok { 263 | groups[v] = true 264 | } else { 265 | return nil, false 266 | } 267 | } 268 | 269 | filters := []int{-1, -1, -1} 270 | for n, _ := range group.FilterBy { 271 | if v, ok := p.names[n]; ok { 272 | filters[v] = filterValueOf(n, group) 273 | } else { 274 | return nil, false 275 | } 276 | } 277 | 278 | // interestBy используем только если в условиях были интересы 279 | if groups[0] == false && filters[0] == -1 { 280 | return nil, false 281 | } 282 | counters := countersBorrow() // make(map[uint64]int32) 283 | 284 | var listJ []int 285 | if filters[1] != -1 && filters[2] != -1 { 286 | listJ = []int{filters[1]*p.stripe + filters[2]} 287 | } else if filters[1] != -1 { 288 | jLen := capacityOf(p.third) 289 | listJ = make([]int, jLen) 290 | f1 := filters[1] * p.stripe 291 | for jk := 0; jk < jLen; jk++ { 292 | listJ[jk] = f1 + jk 293 | } 294 | } else if filters[2] != -1 { 295 | jLen := capacityOf(p.second) 296 | listJ = make([]int, jLen) 297 | for jk := 0; jk < jLen; jk++ { 298 | listJ[jk] = jk*p.stripe + filters[2] 299 | } 300 | } 301 | 302 | if len(listJ) == 0 && filters[0] == -1 { 303 | for j, ds := range p.data { 304 | for i, val := range ds { 305 | if val > 0 { 306 | counters[p.GroupKey(i, j, groups)] += val 307 | } 308 | } 309 | } 310 | } else if len(listJ) == 0 && filters[0] != -1 { 311 | i := filters[0] 312 | for j, ds := range p.data { 313 | if val := ds[i]; val > 0 { 314 | counters[p.GroupKey(i, j, groups)] += val 315 | } 316 | } 317 | } else if len(listJ) != 0 && filters[0] == -1 { 318 | for _, j := range listJ { 319 | ds := p.data[j] 320 | for i, val := range ds { 321 | if val > 0 { 322 | counters[p.GroupKey(i, j, groups)] += val 323 | } 324 | } 325 | } 326 | } else if len(listJ) != 0 && filters[0] != -1 { 327 | i := filters[0] 328 | for _, j := range listJ { 329 | if val := p.data[j][i]; val > 0 { 330 | counters[p.GroupKey(i, j, groups)] += val 331 | } 332 | } 333 | } 334 | //println(len(counters)) 335 | return counters, true 336 | } 337 | 338 | /* quadAggregate */ 339 | type quadAggregate struct { 340 | data [][]int32 341 | first, second, third, forth string 342 | cap2, stripe, stripe2 int 343 | firstOffset, secondOffset, thirdOffset, forthOffset uint 344 | firstIndex, secondIndex, thirdIndex, forthIndex func(*Account) uint 345 | names map[string]int 346 | } 347 | 348 | func makeQuadAggregate(first, second, third, forth string) *quadAggregate { 349 | p := quadAggregate{ 350 | data: make([][]int32, capacityOf(first)), 351 | 352 | first: first, 353 | second: second, 354 | third: third, 355 | forth: forth, 356 | 357 | firstOffset: offsetOf(first), 358 | secondOffset: offsetOf(second), 359 | thirdOffset: offsetOf(third), 360 | forthOffset: offsetOf(forth), 361 | 362 | //secondCapacity: capacityOf(second), 363 | cap2: capacityOf(second), 364 | stripe: capacityOf(third), 365 | stripe2: capacityOf(forth), 366 | 367 | firstIndex: indexFuncOf(first), 368 | secondIndex: indexFuncOf(second), 369 | thirdIndex: indexFuncOf(third), 370 | forthIndex: indexFuncOf(forth), 371 | 372 | names: map[string]int{first: 0, second: 1, third: 2, forth: 3}} 373 | 374 | columnCapacity := p.cap2 * p.stripe * p.stripe2 375 | for i := 0; i < len(p.data); i++ { 376 | p.data[i] = make([]int32, columnCapacity) 377 | } 378 | return &p 379 | } 380 | 381 | func (p *quadAggregate) Index(id uint32, account *Account) { 382 | p.data[p.firstIndex(account)][(uint(p.stripe)*p.secondIndex(account)+p.thirdIndex(account))*uint(p.stripe2)+p.forthIndex(account)] += 1 383 | } 384 | 385 | func (p *quadAggregate) GroupKey(i int, j int, groups []bool) uint64 { 386 | var key uint64 387 | if groups[0] { 388 | key |= keyBits(i, p.firstOffset) 389 | } 390 | if groups[1] { 391 | key |= keyBits(j/p.stripe/p.stripe2, p.secondOffset) 392 | } 393 | if groups[2] { 394 | key |= keyBits((j/p.stripe2)%p.stripe, p.thirdOffset) 395 | } 396 | if groups[3] { 397 | key |= keyBits(j%p.stripe2, p.forthOffset) 398 | } 399 | return key 400 | } 401 | 402 | var countersPool = sync.Pool{ 403 | New: func() interface{} { 404 | return make(map[uint64]int32, 2048) 405 | }, 406 | } 407 | 408 | func countersBorrow() map[uint64]int32 { 409 | return countersPool.Get().(map[uint64]int32) 410 | } 411 | 412 | func countersRelease(buffer map[uint64]int32) { 413 | for key := range buffer { 414 | delete(buffer, key) 415 | } 416 | countersPool.Put(buffer) 417 | } 418 | 419 | func (p *quadAggregate) Aggregate(group *Group) (map[uint64]int32, bool) { 420 | 421 | if len(group.GroupBy) > 3 || len(group.FilterBy) > 3 { 422 | return nil, false 423 | } 424 | 425 | groups := []bool{false, false, false, false} 426 | for _, n := range group.GroupBy { 427 | if v, ok := p.names[n]; ok { 428 | groups[v] = true 429 | } else { 430 | return nil, false 431 | } 432 | } 433 | filters := []int{-1, -1, -1, -1} 434 | for n, _ := range group.FilterBy { 435 | if v, ok := p.names[n]; ok { 436 | filters[v] = filterValueOf(n, group) 437 | } else { 438 | return nil, false 439 | } 440 | } 441 | 442 | counters := countersBorrow() // make(map[uint64]int32) 443 | 444 | var listJ []int 445 | if filters[1] != -1 || filters[2] != -1 || filters[3] != -1 { 446 | xs := filters[1:2] 447 | if xs[0] == -1 { 448 | xs = rangeBackend[0:p.cap2] 449 | } 450 | ys := filters[2:3] 451 | if ys[0] == -1 { 452 | ys = rangeBackend[0:p.stripe] 453 | } 454 | zs := filters[3:4] 455 | if zs[0] == -1 { 456 | zs = rangeBackend[0:p.stripe2] 457 | } 458 | listJ = make([]int, len(xs)*len(ys)*len(zs))[:0] 459 | for _, x := range xs { 460 | for _, y := range ys { 461 | for _, z := range zs { 462 | listJ = append(listJ, (x*p.stripe+y)*p.stripe2+z) 463 | } 464 | } 465 | } 466 | } 467 | 468 | if filters[0] == -1 { 469 | for i := 0; i < len(p.data); i++ { 470 | if len(listJ) == 0 { 471 | for j := 0; j < len(p.data[i]); j++ { 472 | if val := p.data[i][j]; val > 0 { 473 | counters[p.GroupKey(i, j, groups)] += val 474 | } 475 | } 476 | } else { 477 | for _, j := range listJ { 478 | if val := p.data[i][j]; val > 0 { 479 | counters[p.GroupKey(i, j, groups)] += val 480 | } 481 | } 482 | } 483 | } 484 | } else { 485 | i := filters[0] 486 | if len(listJ) == 0 { 487 | for j := 0; j < len(p.data[i]); j++ { 488 | if val := p.data[i][j]; val > 0 { 489 | counters[p.GroupKey(i, j, groups)] += val 490 | } 491 | } 492 | } else { 493 | for _, j := range listJ { 494 | if val := p.data[i][j]; val > 0 { 495 | counters[p.GroupKey(i, j, groups)] += val 496 | } 497 | } 498 | } 499 | 500 | } 501 | //println(len(counters)) 502 | return counters, true 503 | } 504 | 505 | /* tippleAggregate */ 506 | type trippleAggregate struct { 507 | data [][]int32 508 | first, second, third string 509 | //secondCapacity, 510 | stripe int 511 | firstOffset, secondOffset, thirdOffset uint 512 | firstIndex, secondIndex, thirdIndex func(*Account) uint 513 | names map[string]int 514 | } 515 | 516 | func makeTrippleAggregate(first, second, third string) *trippleAggregate { 517 | p := trippleAggregate{ 518 | data: make([][]int32, capacityOf(first)), 519 | 520 | first: first, 521 | second: second, 522 | third: third, 523 | 524 | firstOffset: offsetOf(first), 525 | secondOffset: offsetOf(second), 526 | thirdOffset: offsetOf(third), 527 | 528 | //secondCapacity: capacityOf(second), 529 | stripe: capacityOf(third), 530 | 531 | firstIndex: indexFuncOf(first), 532 | secondIndex: indexFuncOf(second), 533 | thirdIndex: indexFuncOf(third), 534 | 535 | names: map[string]int{first: 0, second: 1, third: 2}} 536 | 537 | columnCapacity := capacityOf(second) * p.stripe 538 | for i := 0; i < len(p.data); i++ { 539 | p.data[i] = make([]int32, columnCapacity) 540 | } 541 | return &p 542 | } 543 | 544 | func (p *trippleAggregate) Index(id uint32, account *Account) { 545 | p.data[p.firstIndex(account)][uint(p.stripe)*p.secondIndex(account)+p.thirdIndex(account)] += 1 546 | } 547 | 548 | func (p *trippleAggregate) GroupKey(i int, j int, groups []bool) uint64 { 549 | var key uint64 550 | if groups[0] { 551 | key |= keyBits(i, p.firstOffset) 552 | } 553 | if groups[1] { 554 | key |= keyBits(j/p.stripe, p.secondOffset) 555 | } 556 | if groups[2] { 557 | key |= keyBits(j%p.stripe, p.thirdOffset) 558 | } 559 | return key 560 | } 561 | 562 | func (p *trippleAggregate) Aggregate(group *Group) (map[uint64]int32, bool) { 563 | 564 | if len(group.GroupBy) > 3 || len(group.FilterBy) > 3 { 565 | return nil, false 566 | } 567 | 568 | groups := []bool{false, false, false} 569 | for _, n := range group.GroupBy { 570 | if v, ok := p.names[n]; ok { 571 | groups[v] = true 572 | } else { 573 | return nil, false 574 | } 575 | } 576 | filters := []int{-1, -1, -1} 577 | for n, _ := range group.FilterBy { 578 | if v, ok := p.names[n]; ok { 579 | filters[v] = filterValueOf(n, group) 580 | } else { 581 | return nil, false 582 | } 583 | } 584 | counters := countersBorrow() // make(map[uint64]int32) 585 | 586 | var listJ []int 587 | if filters[1] != -1 && filters[2] != -1 { 588 | listJ = []int{filters[1]*p.stripe + filters[2]} 589 | } else if filters[1] != -1 { 590 | jLen := capacityOf(p.third) 591 | listJ = make([]int, jLen) 592 | f1 := filters[1] * p.stripe 593 | for jk := 0; jk < jLen; jk++ { 594 | listJ[jk] = f1 + jk 595 | } 596 | } else if filters[2] != -1 { 597 | jLen := capacityOf(p.second) 598 | listJ = make([]int, jLen) 599 | for jk := 0; jk < jLen; jk++ { 600 | listJ[jk] = jk*p.stripe + filters[2] 601 | } 602 | } 603 | 604 | if filters[0] == -1 { 605 | for i := 0; i < len(p.data); i++ { 606 | if len(listJ) == 0 { 607 | for j := 0; j < len(p.data[i]); j++ { 608 | if val := p.data[i][j]; val > 0 { 609 | counters[p.GroupKey(i, j, groups)] += val 610 | } 611 | } 612 | } else { 613 | for _, j := range listJ { 614 | if val := p.data[i][j]; val > 0 { 615 | counters[p.GroupKey(i, j, groups)] += val 616 | } 617 | } 618 | } 619 | } 620 | } else { 621 | i := filters[0] 622 | if len(listJ) == 0 { 623 | for j := 0; j < len(p.data[i]); j++ { 624 | if val := p.data[i][j]; val > 0 { 625 | counters[p.GroupKey(i, j, groups)] += val 626 | } 627 | } 628 | } else { 629 | for _, j := range listJ { 630 | if val := p.data[i][j]; val > 0 { 631 | counters[p.GroupKey(i, j, groups)] += val 632 | } 633 | } 634 | } 635 | 636 | } 637 | //println(len(counters)) 638 | return counters, true 639 | } 640 | 641 | /* doubleAggregate */ 642 | /* 643 | type doubleAggregate struct { 644 | data [][]int32 645 | first, second string 646 | firstOffset, secondOffset uint 647 | firstIndex, secondIndex func(*Account) uint 648 | } 649 | 650 | func makeDoubleAggregate(first, second string) *doubleAggregate { 651 | aggregate := doubleAggregate{ 652 | data: make([][]int32, capacityOf(first)), 653 | first: first, 654 | second: second, 655 | firstOffset: offsetOf(first), 656 | secondOffset: offsetOf(second), 657 | firstIndex: indexFuncOf(first), 658 | secondIndex: indexFuncOf(second)} 659 | 660 | columnCapacity := capacityOf(second) 661 | for i := 0; i < len(aggregate.data); i++ { 662 | aggregate.data[i] = make([]int32, columnCapacity) 663 | } 664 | return &aggregate 665 | } 666 | 667 | func (p *doubleAggregate) Index(id uint32, account *Account) { 668 | p.data[p.firstIndex(account)][p.secondIndex(account)] += 1 669 | } 670 | 671 | func (p *doubleAggregate) GroupKey(x int, y int, groups []bool) uint64 { 672 | if groups[0] && groups[1] { 673 | return keyBits(x, p.firstOffset) | keyBits(y, p.secondOffset) 674 | } else if groups[0] { 675 | return keyBits(x, p.firstOffset) 676 | } else { 677 | return keyBits(y, p.secondOffset) 678 | } 679 | } 680 | 681 | func (p *doubleAggregate) Aggregate(group *Group) (map[uint64]int32, bool) { 682 | 683 | xFilter := -1 684 | yFilter := -1 685 | var firstGroup, secondGroup bool 686 | 687 | if len(group.GroupBy) == 1 && p.first == group.GroupBy[0] { 688 | // x -> .. 689 | firstGroup = true 690 | if len(group.FilterBy) == 1 && group.FilterBy[p.second] { 691 | yFilter = filterValueOf(p.second, group) 692 | } else if len(group.FilterBy) != 0 { 693 | return nil, false 694 | } 695 | } else if len(group.GroupBy) == 1 && p.second == group.GroupBy[0] { 696 | // y -> .. 697 | secondGroup = true 698 | if len(group.FilterBy) == 1 && group.FilterBy[p.first] { 699 | xFilter = filterValueOf(p.first, group) 700 | } else if len(group.FilterBy) != 0 { 701 | return nil, false 702 | } 703 | } else if len(group.GroupBy) == 2 && ((p.second == group.GroupBy[0] && p.first == group.GroupBy[1]) || (p.second == group.GroupBy[1] && p.first == group.GroupBy[0])) { 704 | // x,y -> .. 705 | firstGroup = true 706 | secondGroup = true 707 | if len(group.FilterBy) == 1 && group.FilterBy[p.first] { 708 | xFilter = filterValueOf(p.first, group) 709 | } else if len(group.FilterBy) == 1 && group.FilterBy[p.second] { 710 | yFilter = filterValueOf(p.second, group) 711 | } else if len(group.FilterBy) != 0 { 712 | return nil, false 713 | } 714 | } else { 715 | return nil, false 716 | } 717 | 718 | counters := make(map[uint64]int32) 719 | // todo: написать 3 цикла в зависимости от x_filter/y_filter 720 | for i := 0; i < len(p.data); i++ { 721 | if xFilter == -1 || xFilter == i { 722 | for j := 0; j < len(p.data[i]); j++ { 723 | if yFilter == -1 || yFilter == j { 724 | if val := p.data[i][j]; val > 0 { 725 | counters[p.GroupKey(i, j, []bool{firstGroup, secondGroup})] += val 726 | } 727 | } 728 | } 729 | } 730 | } 731 | 732 | return counters, true 733 | } 734 | */ 735 | 736 | func keyBits(value int, offset uint) uint64 { 737 | if offset == 0 { 738 | return uint64(value) 739 | } else { 740 | return uint64(value) << offset 741 | } 742 | } 743 | 744 | /** interestBy4 */ 745 | type interestBy4 struct { 746 | data [][]int32 747 | first, second, third, forth string 748 | 749 | cap2, stripe, stripe2, fullStripe int 750 | 751 | firstOffset, secondOffset, thirdOffset, forthOffset uint 752 | firstIndex, secondIndex, thirdIndex, forthIndex func(*Account) uint 753 | names map[string]int 754 | backData []int32 755 | } 756 | 757 | func makeInterestBy4(second, third, forth string) *interestBy4 { 758 | p := interestBy4{ 759 | first: "interests", 760 | firstOffset: SNAME_OFFSET, 761 | second: second, 762 | secondOffset: offsetOf(second), 763 | secondIndex: indexFuncOf(second), 764 | stripe: 1, 765 | names: map[string]int{"interests": 0, second: 1}, 766 | } 767 | 768 | p.stripe = capacityOf(third) 769 | p.third = third 770 | p.thirdOffset = offsetOf(third) 771 | p.thirdIndex = indexFuncOf(third) 772 | p.names[third] = 2 773 | 774 | p.cap2 = capacityOf(second) 775 | p.stripe2 = capacityOf(forth) 776 | p.forth = forth 777 | p.forthOffset = offsetOf(forth) 778 | p.forthIndex = indexFuncOf(forth) 779 | p.names[forth] = 3 780 | 781 | p.fullStripe = p.stripe * p.stripe2 782 | 783 | columnCapacity := len(InterestDict.values) 784 | p.data = make([][]int32, p.cap2*p.fullStripe) 785 | p.backData = make([]int32, len(p.data)*columnCapacity) 786 | for i := 0; i < len(p.data); i++ { 787 | p.data[i] = p.backData[i*columnCapacity : (i+1)*columnCapacity] 788 | //p.data[i] = make([]int32, columnCapacity) 789 | } 790 | return &p 791 | } 792 | 793 | func (p *interestBy4) Index(id uint32, account *Account) { 794 | j := p.secondIndex(account)*uint(p.fullStripe) + p.thirdIndex(account)*uint(p.stripe2) + p.forthIndex(account) 795 | 796 | pos := (id - 1) * 2 797 | interest0 := Store.interests[pos] >> 1 798 | interest1 := Store.interests[pos+1] 799 | 800 | xs := p.data[j] 801 | pxs0 := uintptr(unsafe.Pointer(&xs[0])) 802 | 803 | // + unroll interests0 804 | for i := uintptr(0); i < 63 && interest0 != 0; i++ { 805 | if 1 == interest0&1 { 806 | //xs[i] += 1 807 | *(*uint32)(unsafe.Pointer(pxs0 + i*4)) += 1 808 | 809 | } 810 | interest0 >>= 1 811 | } 812 | // + unroll interests1 813 | for i := uintptr(63); i < 127 && interest1 != 0; i++ { 814 | if 1 == interest1&1 { 815 | //xs[i] += 1 816 | *(*uint32)(unsafe.Pointer(pxs0 + i*4)) += 1 817 | } 818 | interest1 >>= 1 819 | } 820 | } 821 | 822 | func (p *interestBy4) GroupKey(i, j int, groups []bool) uint64 { 823 | var key uint64 824 | if groups[0] { 825 | key |= keyBits(i+1, p.firstOffset) 826 | } 827 | if groups[1] { 828 | key |= keyBits(j/p.fullStripe, p.secondOffset) 829 | } 830 | if groups[2] { 831 | key |= keyBits((j/p.stripe2)%p.stripe, p.thirdOffset) 832 | } 833 | if groups[3] { 834 | key |= keyBits(j%p.stripe, p.thirdOffset) 835 | } 836 | return key 837 | } 838 | 839 | func (p *interestBy4) Aggregate(group *Group) (map[uint64]int32, bool) { 840 | if len(group.GroupBy) > 3 || len(group.FilterBy) > 3 { 841 | return nil, false 842 | } 843 | 844 | groups := []bool{false, false, false, false} 845 | for _, n := range group.GroupBy { 846 | if v, ok := p.names[n]; ok { 847 | groups[v] = true 848 | } else { 849 | return nil, false 850 | } 851 | } 852 | 853 | filters := []int{-1, -1, -1, -1} 854 | for n, _ := range group.FilterBy { 855 | if v, ok := p.names[n]; ok { 856 | filters[v] = filterValueOf(n, group) 857 | } else { 858 | return nil, false 859 | } 860 | } 861 | 862 | // interestBy* агрегат используем только если в условиях или ключе группы были интересы 863 | if groups[0] == false && filters[0] == -1 { 864 | return nil, false 865 | } 866 | 867 | counters := countersBorrow() // make(map[uint64]int32) 868 | 869 | var listJ []int 870 | if filters[1] != -1 || filters[2] != -1 || filters[3] != -1 { 871 | xs := filters[1:2] 872 | if xs[0] == -1 { 873 | xs = rangeBackend[0:p.cap2] 874 | } 875 | ys := filters[2:3] 876 | if ys[0] == -1 { 877 | ys = rangeBackend[0:p.stripe] 878 | } 879 | zs := filters[3:4] 880 | if zs[0] == -1 { 881 | zs = rangeBackend[0:p.stripe2] 882 | } 883 | listJ = make([]int, 0, len(xs)*len(ys)*len(zs)) 884 | for _, x := range xs { 885 | for _, y := range ys { 886 | for _, z := range zs { 887 | listJ = append(listJ, (x*p.stripe+y)*p.stripe2+z) 888 | } 889 | } 890 | } 891 | } 892 | /* 893 | // (3) Conversion of a Pointer to a uintptr and back, with arithmetic. 894 | // 895 | // If p points into an allocated object, it can be advanced through the object 896 | // by conversion to uintptr, addition of an offset, and conversion back to Pointer. 897 | // 898 | // p = unsafe.Pointer(uintptr(p) + offset) 899 | // 900 | // The most common use of this pattern is to access fields in a struct 901 | // or elements of an array: 902 | // 903 | // // equivalent to f := unsafe.Pointer(&s.f) 904 | // f := unsafe.Pointer(uintptr(unsafe.Pointer(&s)) + unsafe.Offsetof(s.f)) 905 | // 906 | // // equivalent to e := unsafe.Pointer(&x[i]) 907 | // e := unsafe.Pointer(uintptr(unsafe.Pointer(&x[0])) + i*unsafe.Sizeof(x[0])) 908 | */ 909 | if len(listJ) == 0 && filters[0] == -1 { 910 | for j, ds := range p.data { 911 | for i, val := range ds { 912 | if val > 0 { 913 | counters[p.GroupKey(i, j, groups)] += val 914 | } 915 | } 916 | } 917 | } else if len(listJ) == 0 && filters[0] != -1 { 918 | i := filters[0] 919 | for j, ds := range p.data { 920 | if val := ds[i]; val > 0 { 921 | counters[p.GroupKey(i, j, groups)] += val 922 | } 923 | } 924 | } else if len(listJ) != 0 && filters[0] == -1 { 925 | for _, j := range listJ { 926 | ds := p.data[j] 927 | for i, val := range ds { 928 | if val > 0 { 929 | counters[p.GroupKey(i, j, groups)] += val 930 | } 931 | } 932 | } 933 | } else if len(listJ) != 0 && filters[0] != -1 { 934 | i := filters[0] 935 | for _, j := range listJ { 936 | if val := p.data[j][i]; val > 0 { 937 | counters[p.GroupKey(i, j, groups)] += val 938 | } 939 | } 940 | } 941 | //println(len(counters)) 942 | return counters, true 943 | } 944 | 945 | // birthYear uint16 // Ограничено снизу 01.01.1950 и сверху 01.01.2005-ым. 946 | // joinedYear uint16 // 2011-01-01 : 2018-01-01 947 | 948 | func CalculateGroups() { 949 | for _, key := range []string{ 950 | "city,status", 951 | "interests", 952 | "country,sex", 953 | "city,sex", 954 | } { 955 | groupOf("keys", key).aggregate(true) 956 | } 957 | 958 | groupOf("keys", "city,status", "sex", "m").aggregate(true) 959 | groupOf("keys", "city,status", "sex", "f").aggregate(true) 960 | 961 | groupOf("keys", "country,sex", "sex", "m").aggregate(true) 962 | groupOf("keys", "country,sex", "sex", "f").aggregate(true) 963 | 964 | //fmt.Printf("%v\tCalculateGroups 1\n", Timenow()) 965 | 966 | for joined := 2011; joined <= 2018; joined++ { 967 | if evio.GetEpollWait() == 0 { 968 | fmt.Printf("%v\tCalculateGroups stopped\n", Timenow()) 969 | return 970 | } 971 | joinedStr := strconv.Itoa(joined) 972 | groupOf("keys", "city", "joined", joinedStr).aggregate(true) 973 | groupOf("keys", "city,status", "joined", joinedStr).aggregate(true) 974 | groupOf("keys", "city,status", "joined", joinedStr, "sex", "m").aggregate(true) 975 | groupOf("keys", "city,status", "joined", joinedStr, "sex", "f").aggregate(true) 976 | groupOf("keys", "city,sex", "joined", joinedStr).aggregate(true) 977 | for interest := range InterestDict.ids { 978 | if evio.GetEpollWait() == 0 { 979 | fmt.Printf("%v\tCalculateGroups stopped\n", Timenow()) 980 | return 981 | } 982 | if interest != "" { 983 | groupOf("keys", "country,sex", "joined", joinedStr, "interests", interest, "joined", joinedStr).aggregate(true) 984 | groupOf("keys", "city,status", "joined", joinedStr, "interests", interest).aggregate(true) 985 | groupOf("keys", "city,sex", "joined", joinedStr, "interests", interest).aggregate(true) 986 | groupOf("keys", "country,status", "joined", joinedStr, "interests", interest).aggregate(true) 987 | } 988 | } 989 | //group;[city]/1/joined,status,;50;1;39;998 990 | } 991 | //fmt.Printf("%v\tCalculateGroups 2\n", Timenow()) 992 | 993 | for interest := range InterestDict.ids { 994 | if evio.GetEpollWait() == 0 { 995 | fmt.Printf("%v\tCalculateGroups stopped\n", Timenow()) 996 | return 997 | } 998 | if interest != "" { 999 | groupOf("keys", "country,sex", "interests", interest).aggregate(true) 1000 | groupOf("keys", "city,status", "interests", interest).aggregate(true) 1001 | groupOf("keys", "city", "interests", interest).aggregate(true) 1002 | groupOf("keys", "city,sex", "interests", interest).aggregate(true) 1003 | } 1004 | } 1005 | //fmt.Printf("%v\tCalculateGroups 3\n", Timenow()) 1006 | 1007 | for birth := 1950; birth <= 2005; birth++ { 1008 | if evio.GetEpollWait() == 0 { 1009 | fmt.Printf("%v\tCalculateGroups stopped\n", Timenow()) 1010 | return 1011 | } 1012 | birthStr := strconv.Itoa(birth) 1013 | 1014 | groupOf("keys", "country,status", "birth", birthStr).aggregate(true) 1015 | groupOf("keys", "city,status", "birth", birthStr).aggregate(true) 1016 | groupOf("keys", "city,status", "birth", birthStr, "sex", "m").aggregate(true) 1017 | groupOf("keys", "city,status", "birth", birthStr, "sex", "f").aggregate(true) 1018 | 1019 | groupOf("keys", "city", "birth", birthStr).aggregate(true) 1020 | groupOf("keys", "interests", "birth", birthStr).aggregate(true) 1021 | groupOf("keys", "city", "birth", birthStr, "status", "свободны").aggregate(true) 1022 | groupOf("keys", "city", "birth", birthStr, "status", "заняты").aggregate(true) 1023 | groupOf("keys", "city", "birth", birthStr, "status", "всё сложно").aggregate(true) 1024 | 1025 | } 1026 | 1027 | fmt.Printf("%v\tCalculateGroups done\n", Timenow()) 1028 | 1029 | // [city status]/1/joined,sex, 16 38 2378 32207 1030 | // [city status]/joined, 6 2 488 979 1031 | // [city sex]/1/interests,joined, 17 10 632 994 1032 | // [city status]/interests, 16 13 854 1952 1033 | // [city status]/birth,sex, 15 3 261 997 1034 | // [city]/birth,sex, 33 7 236 977 1035 | // [city]/interests,joined, 27 7 289 977 1036 | // [city]/interests, 44 7 177 997 1037 | 1038 | // [city]/1/birth,sex, 25 6 273 977 1039 | // [city status]/-1/birth,status, 29 4 168 977 1040 | // [city status]/-1/birth, 13 4 375 977 1041 | // [status]/-1/likes, 90 2 32 977 1042 | // [country status]/-1/joined,status, 24 2 122 977 1043 | // [city sex]/1/joined,status, 11 2 266 977 1044 | // [city]/-1/joined,status, 33 2 88 977 1045 | // [city]/1/sex, 43 2 68 977 1046 | // [country]/-1/joined,status, 33 1 59 977 1047 | // [city status]/1/birth, 8 1 244 977 1048 | // [interests]/1/ 28 1 69 977 1049 | // [city status]/-1/interests, 21 14 697 976 1050 | // [city]/1/interests, 35 10 306 976 1051 | // [city status]/1/interests,joined, 14 8 627 976 1052 | // [city sex]/-1/interests, 13 8 675 976 1053 | // [city]/1/birth,status, 25 7 312 976 1054 | // [city sex]/1/interests, 17 6 402 976 1055 | // [city sex]/1/birth,status, 15 5 390 976 1056 | // [city status]/-1/birth,interests, 20 5 292 976 1057 | // [city status]/1/birth,sex, 13 5 450 976 1058 | // [city]/-1/interests,joined, 30 5 194 976 1059 | // [city sex]/-1/joined,status, 12 4 406 976 1060 | // [city]/-1/birth,status, 38 4 128 976 1061 | 1062 | } 1063 | 1064 | func CalculateGroupsParallel(wg *sync.WaitGroup) { 1065 | for birth := 1950; birth <= 2005; birth++ { 1066 | birthStr := strconv.Itoa(birth) 1067 | for interest := range InterestDict.ids { 1068 | if evio.GetEpollWait() == 0 { 1069 | fmt.Printf("%v\tCalculateGroupsParallel stopped\n", Timenow()) 1070 | return 1071 | } 1072 | if interest != "" { 1073 | groupOf("keys", "city,status", "birth", birthStr, "interests", interest).aggregate(true) 1074 | } 1075 | } 1076 | } 1077 | 1078 | wg.Done() 1079 | fmt.Printf("%v\tCalculateGroupsParallel done\n", Timenow()) 1080 | } 1081 | 1082 | func groupOf(conditions ...string) *Group { 1083 | var params = make(map[string]string) 1084 | for i, cond := range conditions { 1085 | if i%2 == 0 { 1086 | params[cond] = conditions[i+1] 1087 | } 1088 | } 1089 | params["order"] = "1" 1090 | 1091 | var group Group 1092 | if err := group.FromParams(params); err != nil { 1093 | log.Printf("FromParams failed, %v, %v\n", params, err) 1094 | return nil 1095 | } 1096 | return &group 1097 | } 1098 | --------------------------------------------------------------------------------