├── .gitignore ├── .vscode └── launch.json ├── Dockerfile ├── Gopkg.lock ├── Gopkg.toml ├── LICENSE ├── README.md ├── Task2018.pdf ├── account ├── account.go ├── account_json.go └── jwriter.go ├── avltree ├── avltree_universal.go └── avltreeidint32.go ├── db ├── bitmappool.go ├── coliter.go ├── column.go ├── heap.go ├── likesl.go ├── smacc.go └── values.go ├── dict ├── email.go ├── email.stat └── leaf.go ├── dockersend.sh ├── docs ├── 102.pdf ├── 1509.05053.pdf ├── 1683220.pdf ├── 1768-1835-1-PB.pdf ├── Dictionary-based_order-preserving_string_compressi.pdf ├── Gin.pdf ├── Go assembly language complementary reference · Iskander Sharipov technical blog.pdf ├── Hasso_Plattner_A_Course_In-Memory_DB.pdf ├── JumpHash_1406.2294.pdf ├── P19.pdf ├── Principles of DBM.pdf ├── Writing and Optimizing Go code.pdf ├── colomnindices-master.zip ├── dremel.pdf ├── fts_anatomy_pgsql.pdf ├── index-search-modern-cpus.pdf ├── kolonochnyy-soprotsessor-baz-dannyh-dlya-klasternyh-vychislitelnyh-sistem.pdf ├── parallelnaya-dekompozitsiya-relyatsionnyh-operatsiy-na-osnove-raspredelennyh-kolonochnyh-indeksov.pdf ├── quad search memorization.pdf ├── quad search.pdf ├── rolling hash.pdf ├── structures for inmemory db.PDF ├── ДЕКОМПОЗИЦИЯ ОПЕРАЦИИ ГРУППИРОВКИ.pdf ├── ДЕКОМПОЗИЦИЯ ОПЕРАЦИЙ ПЕРЕСЕЧЕНИЯ И СОЕДИНЕНИЯ.pdf ├── КОЛОНОЧНЫЕ ХЕШ-ИНДЕКСЫ.pdf ├── МЕТОДЫ ПАРАЛЛЕЛЬНОЙ ОБРАБОТКИ.pdf ├── Обновление распределенных сжатых колоночных индексов.pdf ├── Операции естественного соединения.pdf ├── Презентация МЕТОДЫ ПАРАЛЛЕЛЬНОЙ ОБРАБОТКИ.pdf ├── Распределенные колоночные индексы.pdf └── хэш рабин карп.pdf ├── flysort ├── benchresults ├── binlinesort.go ├── flysort_test.go ├── heapinlined.go ├── pheapsort.go └── pheapsort_test.go ├── gentest ├── gentest ├── main.go └── stat.txt ├── groupset ├── const.go ├── groupset.go ├── grresult.go └── описание.txt ├── handlers ├── filter.go ├── filter.stat ├── filter_stats.go ├── group.go ├── handlers.go ├── likes.go ├── new.go ├── recommend.go ├── suggest.go └── update.go ├── intsearch ├── benchresults ├── ints │ └── main.go ├── intsearch.go ├── intsearch_amd64.s ├── intsearch_asm.go ├── intsearch_test.go ├── intunsafe.go ├── intunsafe.o └── unsafeasm ├── jumphash └── jumphash.go ├── main.go ├── parser └── dataparser.go ├── queryset ├── chans.go ├── const.go └── queryset.go ├── recommend └── recommend.go ├── run.sh ├── run0.sh ├── suggest ├── heap.go ├── sort.go └── suggest.go ├── treebidimap └── treebidimap.go └── vendor └── github.com ├── RoaringBitmap └── roaring │ ├── .gitignore │ ├── .gitmodules │ ├── .travis.yml │ ├── AUTHORS │ ├── CONTRIBUTORS │ ├── LICENSE │ ├── LICENSE-2.0.txt │ ├── Makefile │ ├── README.md │ ├── arraycontainer.go │ ├── arraycontainer_gen.go │ ├── bitmapcontainer.go │ ├── bitmapcontainer_gen.go │ ├── clz.go │ ├── clz_compat.go │ ├── ctz.go │ ├── ctz_compat.go │ ├── fastaggregation.go │ ├── manyiterator.go │ ├── parallel.go │ ├── popcnt.go │ ├── popcnt_amd64.s │ ├── popcnt_asm.go │ ├── popcnt_compat.go │ ├── popcnt_generic.go │ ├── popcnt_slices.go │ ├── priorityqueue.go │ ├── roaring.go │ ├── roaringarray.go │ ├── roaringarray_gen.go │ ├── runcontainer.go │ ├── runcontainer_gen.go │ ├── serialization.go │ ├── serialization_generic.go │ ├── serialization_littleendian.go │ ├── serializationfuzz.go │ ├── setutil.go │ ├── shortiterator.go │ ├── smat.go │ └── util.go ├── buaazp └── fasthttprouter │ ├── .gitignore │ ├── .travis.yml │ ├── HttpRouterLicense │ ├── LICENSE │ ├── README.md │ ├── path.go │ ├── router.go │ └── tree.go ├── glycerine └── go-unsnap-stream │ ├── .gitignore │ ├── LICENSE │ ├── README.md │ ├── binary.dat │ ├── binary.dat.snappy │ ├── rbuf.go │ ├── snap.go │ ├── unenc.txt │ ├── unenc.txt.snappy │ └── unsnap.go ├── golang └── snappy │ ├── .gitignore │ ├── AUTHORS │ ├── CONTRIBUTORS │ ├── LICENSE │ ├── README │ ├── decode.go │ ├── decode_amd64.go │ ├── decode_amd64.s │ ├── decode_other.go │ ├── encode.go │ ├── encode_amd64.go │ ├── encode_amd64.s │ ├── encode_other.go │ └── snappy.go ├── klauspost ├── compress │ ├── LICENSE │ ├── flate │ │ ├── copy.go │ │ ├── crc32_amd64.go │ │ ├── crc32_amd64.s │ │ ├── crc32_noasm.go │ │ ├── deflate.go │ │ ├── dict_decoder.go │ │ ├── gen.go │ │ ├── huffman_bit_writer.go │ │ ├── huffman_code.go │ │ ├── inflate.go │ │ ├── reverse_bits.go │ │ ├── snappy.go │ │ └── token.go │ ├── gzip │ │ ├── gunzip.go │ │ └── gzip.go │ ├── snappy │ │ ├── AUTHORS │ │ ├── CONTRIBUTORS │ │ └── LICENSE │ └── zlib │ │ ├── reader.go │ │ └── writer.go └── cpuid │ ├── .gitignore │ ├── .travis.yml │ ├── CONTRIBUTING.txt │ ├── LICENSE │ ├── README.md │ ├── cpuid.go │ ├── cpuid_386.s │ ├── cpuid_amd64.s │ ├── detect_intel.go │ ├── detect_ref.go │ ├── generate.go │ └── private-gen.go ├── mailru └── easyjson │ ├── .gitignore │ ├── .travis.yml │ ├── LICENSE │ ├── Makefile │ ├── README.md │ ├── buffer │ └── pool.go │ ├── helpers.go │ ├── jlexer │ ├── bytestostr.go │ ├── bytestostr_nounsafe.go │ ├── error.go │ └── lexer.go │ ├── jwriter │ └── writer.go │ └── raw.go ├── mschoch └── smat │ ├── .gitignore │ ├── .travis.yml │ ├── LICENSE │ ├── README.md │ ├── actionseq.go │ └── smat.go ├── philhofer └── fwd │ ├── LICENSE.md │ ├── README.md │ ├── reader.go │ ├── writer.go │ ├── writer_appengine.go │ └── writer_unsafe.go ├── tinylib └── msgp │ ├── LICENSE │ └── msgp │ ├── advise_linux.go │ ├── advise_other.go │ ├── circular.go │ ├── defs.go │ ├── edit.go │ ├── elsize.go │ ├── errors.go │ ├── extension.go │ ├── file.go │ ├── file_port.go │ ├── integers.go │ ├── json.go │ ├── json_bytes.go │ ├── number.go │ ├── purego.go │ ├── read.go │ ├── read_bytes.go │ ├── size.go │ ├── unsafe.go │ ├── write.go │ └── write_bytes.go ├── valyala ├── bytebufferpool │ ├── .travis.yml │ ├── LICENSE │ ├── README.md │ ├── bytebuffer.go │ ├── doc.go │ └── pool.go └── fasthttp │ ├── .gitignore │ ├── .travis.yml │ ├── LICENSE │ ├── README.md │ ├── TODO │ ├── args.go │ ├── bytesconv.go │ ├── bytesconv_32.go │ ├── bytesconv_64.go │ ├── client.go │ ├── coarseTime.go │ ├── compress.go │ ├── cookie.go │ ├── doc.go │ ├── fasthttputil │ ├── doc.go │ ├── ecdsa.key │ ├── ecdsa.pem │ ├── inmemory_listener.go │ ├── pipeconns.go │ ├── rsa.key │ └── rsa.pem │ ├── fs.go │ ├── go.mod │ ├── go.sum │ ├── header.go │ ├── http.go │ ├── lbclient.go │ ├── nocopy.go │ ├── peripconn.go │ ├── reuseport │ └── LICENSE │ ├── server.go │ ├── ssl-cert-snakeoil.key │ ├── ssl-cert-snakeoil.pem │ ├── stackless │ ├── doc.go │ ├── func.go │ └── writer.go │ ├── status.go │ ├── stream.go │ ├── strings.go │ ├── tcpdialer.go │ ├── timer.go │ ├── uri.go │ ├── uri_unix.go │ ├── uri_windows.go │ ├── userdata.go │ └── workerpool.go └── willf └── bitset ├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── VERSION ├── bitset.go ├── popcnt.go ├── popcnt_19.go ├── popcnt_amd64.go ├── popcnt_amd64.s ├── popcnt_generic.go ├── trailing_zeros_18.go └── trailing_zeros_19.go /.gitignore: -------------------------------------------------------------------------------- 1 | # ---> Go 2 | # Binaries for programs and plugins 3 | *.exe 4 | *.exe~ 5 | *.dll 6 | *.so 7 | *.dylib 8 | 9 | # Test binary, build with `go test -c` 10 | *.test 11 | 12 | # Output of the go coverage tool, specifically when used with LiteIDE 13 | *.out 14 | 15 | dockercreds 16 | highloadcup2018 17 | debug 18 | test_accounts/data/data.zip 19 | test_accounts/* 20 | test_accounts0/data/data.zip 21 | test_accounts1/data/data.zip 22 | test_accounts0/* 23 | test_accounts1/* 24 | test_accounts2/data/data.zip 25 | test_accounts2/* 26 | test_accountsA/data/data.zip 27 | test_accountsA/* 28 | intsearch.o 29 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Используйте IntelliSense, чтобы узнать о возможных атрибутах. 3 | // Наведите указатель мыши, чтобы просмотреть описания существующих атрибутов. 4 | // Для получения дополнительной информации посетите: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Launch Package", 9 | "type": "go", 10 | "request": "launch", 11 | "mode": "debug", 12 | "program": "${workspaceFolder}", 13 | "args": ["-addr",":8000"] 14 | } 15 | ] 16 | } -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:7 2 | COPY highloadcup2018 / 3 | EXPOSE 80 4 | CMD ["/highloadcup2018"] 5 | -------------------------------------------------------------------------------- /Gopkg.toml: -------------------------------------------------------------------------------- 1 | # Gopkg.toml example 2 | # 3 | # Refer to https://golang.github.io/dep/docs/Gopkg.toml.html 4 | # for detailed Gopkg.toml documentation. 5 | # 6 | # required = ["github.com/user/thing/cmd/thing"] 7 | # ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] 8 | # 9 | # [[constraint]] 10 | # name = "github.com/user/project" 11 | # version = "1.0.0" 12 | # 13 | # [[constraint]] 14 | # name = "github.com/user/project2" 15 | # branch = "dev" 16 | # source = "github.com/myfork/project2" 17 | # 18 | # [[override]] 19 | # name = "github.com/x/y" 20 | # version = "2.4.0" 21 | # 22 | # [prune] 23 | # non-go = false 24 | # go-tests = true 25 | # unused-packages = true 26 | 27 | 28 | [[constraint]] 29 | name = "github.com/buaazp/fasthttprouter" 30 | branch = "master" 31 | 32 | [[constraint]] 33 | branch = "master" 34 | name = "github.com/mailru/easyjson" 35 | 36 | [[constraint]] 37 | name = "github.com/valyala/fasthttp" 38 | branch = "master" 39 | 40 | [prune] 41 | go-tests = true 42 | unused-packages = true 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | Copyright (c) 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 5 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # github.com/covrom/highloadcup2018 2 | 3 | Мое решение для Highload Cup 2018. 4 | 5 | [Техническое задание](Task2018.pdf) 6 | 7 | В финале https://highloadcup.ru/ru/rating/ мое решение занимает 28 место с ~410 сек. штрафа и потреблением памяти 1,6-1,7 Гб. 8 | 9 | Ноу-хау решения стал разработанный универсальный движок columnstore (`db/column.go`) для любого содержимого колонок 10 | и быстрые intersect/merge-итераторы без аллокации памяти (`db/coliter.go`). 11 | 12 | Все алгоритмы хэш или основанные на деревьях - не пригодились по причине того, что они сильно тратят память на указатели (именно на сами указатели, а не на данные), которые дополнительно еще обрабатывает и GC. Из-за этого было принято решение реализовывать алгоритмы на массивах, без указателей. 13 | 14 | Для сети использовался `fasthttp` и `fasthttprouter` без переделок. 15 | 16 | В папке `docs` содержатся материалы по inmemory column - базам данных. 17 | 18 | Содержимое некоторых пакетов: 19 | * `avltree` - содержатся хорошие примеры реализации avl-деревьев, но они не пригодились при решении задачи. 20 | * `flysort` - содержится бенчмарк алгоритмов добавления и сортировки "на лету" - линейный поиск и вставка, бинарный поиски и вставка, мин-хип. 21 | Мин-хип был использован в пакете suggest, однако надо помнить, что из-за отсутствия упорядоченности по уровням он выстраивает неточный список первых минимальных при принудительном ограничении длины массива, на котором он строится, и подходит только для поиска максимальных N элементов. Т.о. ограниченный мин-хип подходит только для поиска максимальных и упорядоченных по возрастанию N значений, а не минимальных. 22 | * `gentest` - лежит утилита обработки исходных данных, для подсчета некоторых статистик. 23 | * `intsearch` - содержится бенчмарк различных вариантов реализации бинарного поиска в сортированном массиве. Безусловный победитель - алгоритм на ассемблере. В ходе исследований выяснились разные интересные особенности компилятора, см. например, https://github.com/golang/go/issues/30366 24 | * `jumphash` - содержится хэш-функция, которая не пригодилась. 25 | * `treebidimap` - содержится алгоритм двунаправленной мапы, который тоже не пригодился. 26 | 27 | Отдельная благодарность Сергею https://github.com/bochsdbg за участие в исследовании некоторых особенностей компилятора при boundary check. -------------------------------------------------------------------------------- /Task2018.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/Task2018.pdf -------------------------------------------------------------------------------- /db/bitmappool.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/RoaringBitmap/roaring" 7 | ) 8 | 9 | var bitPool = sync.Pool{} 10 | 11 | func GetBitmap() *roaring.Bitmap { 12 | sl := bitPool.Get() 13 | if sl != nil { 14 | vsl := sl.(*roaring.Bitmap) 15 | return vsl 16 | } 17 | return roaring.New() 18 | } 19 | 20 | func PutBitmap(sl *roaring.Bitmap) { 21 | if sl == nil { 22 | return 23 | } 24 | sl.Clear() 25 | bitPool.Put(sl) 26 | } 27 | -------------------------------------------------------------------------------- /db/heap.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | type ElemHeapIDAcc struct { 4 | ID IDAcc // Value of this element. 5 | Iterator IDIterator // Which list this element comes from. 6 | } 7 | 8 | type IDAccHeap struct { 9 | reverse bool 10 | Elems []ElemHeapIDAcc 11 | } 12 | 13 | func NewIDAccHeap(reverse bool, capacity int) *IDAccHeap { 14 | return &IDAccHeap{ 15 | reverse: reverse, 16 | Elems: make([]ElemHeapIDAcc, 0, capacity), 17 | } 18 | } 19 | 20 | func (h *IDAccHeap) Clone() *IDAccHeap { 21 | rv := NewIDAccHeap(h.reverse, cap(h.Elems)) 22 | for _, el := range h.Elems { 23 | v := ElemHeapIDAcc{ 24 | ID: el.ID, 25 | Iterator: el.Iterator.Clone(), 26 | } 27 | rv.Elems = append(rv.Elems, v) 28 | } 29 | return rv 30 | } 31 | 32 | func (h *IDAccHeap) Len() int { return len(h.Elems) } 33 | func (h *IDAccHeap) Less(i, j int) bool { 34 | if h.reverse { 35 | return h.Elems[i].ID > h.Elems[j].ID 36 | } else { 37 | return h.Elems[i].ID < h.Elems[j].ID 38 | } 39 | } 40 | func (h *IDAccHeap) Swap(i, j int) { h.Elems[i], h.Elems[j] = h.Elems[j], h.Elems[i] } 41 | func (h *IDAccHeap) Push(x ElemHeapIDAcc) { 42 | h.Elems = append(h.Elems, x) 43 | } 44 | 45 | func (h *IDAccHeap) Pop() ElemHeapIDAcc { 46 | old := h.Elems 47 | n := len(old) 48 | x := old[n-1] 49 | h.Elems = old[0 : n-1] 50 | return x 51 | } 52 | 53 | func InitIDAccHeap(h *IDAccHeap) { 54 | n := h.Len() 55 | for i := n/2 - 1; i >= 0; i-- { 56 | downIDAccHeap(h, i, n) 57 | } 58 | } 59 | 60 | func PushIDAccHeap(h *IDAccHeap, x ElemHeapIDAcc) { 61 | h.Push(x) 62 | upIDAccHeap(h, h.Len()-1) 63 | } 64 | 65 | func PopIDAccHeap(h *IDAccHeap) ElemHeapIDAcc { 66 | n := h.Len() - 1 67 | h.Swap(0, n) 68 | downIDAccHeap(h, 0, n) 69 | return h.Pop() 70 | } 71 | 72 | func RemoveIDAccHeap(h *IDAccHeap, i int) ElemHeapIDAcc { 73 | n := h.Len() - 1 74 | if n != i { 75 | h.Swap(i, n) 76 | if !downIDAccHeap(h, i, n) { 77 | upIDAccHeap(h, i) 78 | } 79 | } 80 | return h.Pop() 81 | } 82 | 83 | func FixIDAccHeap(h *IDAccHeap, i int) { 84 | if !downIDAccHeap(h, i, h.Len()) { 85 | upIDAccHeap(h, i) 86 | } 87 | } 88 | 89 | func upIDAccHeap(h *IDAccHeap, j int) { 90 | for { 91 | i := (j - 1) / 2 // parent 92 | if i == j || !h.Less(j, i) { 93 | break 94 | } 95 | h.Swap(i, j) 96 | j = i 97 | } 98 | } 99 | 100 | func downIDAccHeap(h *IDAccHeap, i0, n int) bool { 101 | i := i0 102 | for { 103 | j1 := 2*i + 1 104 | if j1 >= n || j1 < 0 { // j1 < 0 after int overflow 105 | break 106 | } 107 | j := j1 // left child 108 | if j2 := j1 + 1; j2 < n && h.Less(j2, j1) { 109 | j = j2 // = 2*i + 2 // right child 110 | } 111 | if !h.Less(j, i) { 112 | break 113 | } 114 | h.Swap(i, j) 115 | i = j 116 | } 117 | return i > i0 118 | } 119 | -------------------------------------------------------------------------------- /db/values.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "errors" 5 | "strconv" 6 | "time" 7 | ) 8 | 9 | var NullTime int32 10 | 11 | type IDAccount []byte 12 | 13 | func (idacc IDAccount) Int() IDAcc { 14 | var us uint64 15 | for _, digit := range idacc { 16 | d := digit - '0' 17 | if d > 9 { 18 | break 19 | } 20 | us = us*10 + uint64(d) 21 | } 22 | return IDAcc(us) 23 | } 24 | 25 | type TimeStamp []byte 26 | 27 | func (ts TimeStamp) Time() time.Time { 28 | return time.Unix(int64(ts.Int()), 0).UTC() 29 | } 30 | 31 | func (ts TimeStamp) Int() int32 { 32 | if len(ts) == 0 { 33 | return NullTime 34 | } 35 | var us int64 36 | mul := int32(1) 37 | for i, digit := range ts { 38 | if i == 0 && digit == '-' { 39 | mul = -1 40 | continue 41 | } 42 | d := digit - '0' 43 | if d > 9 { 44 | break 45 | } 46 | us = us*10 + int64(d) 47 | } 48 | 49 | return mul * int32(us) 50 | } 51 | 52 | var ErrNotTimeStamp = errors.New("not a timestamp") 53 | 54 | func (ts TimeStamp) Validate() error { 55 | for i, digit := range ts { 56 | if i == 0 && digit == '-' { 57 | continue 58 | } 59 | d := digit - '0' 60 | if d > 9 { 61 | return ErrNotTimeStamp 62 | } 63 | } 64 | return nil 65 | } 66 | 67 | func (ts TimeStamp) MarshalJSON() ([]byte, error) { 68 | return []byte(strconv.FormatUint(uint64(ts.Int()), 10)), nil 69 | } 70 | 71 | func (ts *TimeStamp) UnmarshalJSON(data []byte) error { 72 | *ts = TimeStamp(data) 73 | return ts.Validate() 74 | } 75 | 76 | type IDAcc uint32 77 | 78 | func (id IDAcc) MarshalJSON() ([]byte, error) { 79 | return []byte(strconv.FormatUint(uint64(id), 10)), nil 80 | } 81 | 82 | func (id *IDAcc) UnmarshalJSON(data []byte) error { 83 | i, err := strconv.ParseUint(string(data), 10, 32) 84 | if err != nil { 85 | return err 86 | } 87 | *id = IDAcc(i) 88 | return nil 89 | } 90 | -------------------------------------------------------------------------------- /dict/leaf.go: -------------------------------------------------------------------------------- 1 | package dict 2 | 3 | import ( 4 | "strings" 5 | "sync" 6 | ) 7 | 8 | type LeafDictonary struct { 9 | sync.RWMutex 10 | mm map[string]uint32 11 | ms []string 12 | } 13 | 14 | func NewDictonary(c int) *LeafDictonary { 15 | return &LeafDictonary{ 16 | mm: make(map[string]uint32, c), 17 | ms: make([]string, 0, c), 18 | } 19 | } 20 | 21 | func (ld *LeafDictonary) Length() int { 22 | ld.RLock() 23 | l := len(ld.ms) 24 | ld.RUnlock() 25 | return l 26 | } 27 | 28 | func (ld *LeafDictonary) Put(b string) uint32 { 29 | ld.Lock() 30 | if i, ok := ld.mm[b]; ok { 31 | ld.Unlock() 32 | return i 33 | } 34 | i := len(ld.ms) 35 | ld.ms = append(ld.ms, b) 36 | ld.mm[b] = uint32(i) 37 | ld.Unlock() 38 | return uint32(i) 39 | } 40 | 41 | func (ld *LeafDictonary) In(b string) (uint32, bool) { 42 | ld.RLock() 43 | if i, ok := ld.mm[b]; ok { 44 | ld.RUnlock() 45 | return i, true 46 | } 47 | ld.RUnlock() 48 | return 0, false 49 | } 50 | 51 | func (ld *LeafDictonary) Get(n uint32) string { 52 | ld.RLock() 53 | if int(n) < len(ld.ms) { 54 | ld.RUnlock() 55 | return ld.ms[int(n)] 56 | } 57 | ld.RUnlock() 58 | return "" 59 | } 60 | 61 | func (ld *LeafDictonary) Compare(x, y uint32) int { 62 | return strings.Compare(ld.Get(x), ld.Get(y)) 63 | } 64 | -------------------------------------------------------------------------------- /dockersend.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | go generate ./... 4 | go build -a -ldflags="-w -s" . 5 | docker build --no-cache --pull -t highloadcup2018 . 6 | docker login stor.highloadcup.ru 7 | docker tag highloadcup2018 stor.highloadcup.ru/accounts/maltese_builder 8 | docker push stor.highloadcup.ru/accounts/maltese_builder 9 | -------------------------------------------------------------------------------- /docs/102.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/102.pdf -------------------------------------------------------------------------------- /docs/1509.05053.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/1509.05053.pdf -------------------------------------------------------------------------------- /docs/1683220.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/1683220.pdf -------------------------------------------------------------------------------- /docs/1768-1835-1-PB.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/1768-1835-1-PB.pdf -------------------------------------------------------------------------------- /docs/Dictionary-based_order-preserving_string_compressi.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/Dictionary-based_order-preserving_string_compressi.pdf -------------------------------------------------------------------------------- /docs/Gin.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/Gin.pdf -------------------------------------------------------------------------------- /docs/Go assembly language complementary reference · Iskander Sharipov technical blog.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/Go assembly language complementary reference · Iskander Sharipov technical blog.pdf -------------------------------------------------------------------------------- /docs/Hasso_Plattner_A_Course_In-Memory_DB.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/Hasso_Plattner_A_Course_In-Memory_DB.pdf -------------------------------------------------------------------------------- /docs/JumpHash_1406.2294.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/JumpHash_1406.2294.pdf -------------------------------------------------------------------------------- /docs/P19.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/P19.pdf -------------------------------------------------------------------------------- /docs/Principles of DBM.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/Principles of DBM.pdf -------------------------------------------------------------------------------- /docs/Writing and Optimizing Go code.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/Writing and Optimizing Go code.pdf -------------------------------------------------------------------------------- /docs/colomnindices-master.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/colomnindices-master.zip -------------------------------------------------------------------------------- /docs/dremel.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/dremel.pdf -------------------------------------------------------------------------------- /docs/fts_anatomy_pgsql.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/fts_anatomy_pgsql.pdf -------------------------------------------------------------------------------- /docs/index-search-modern-cpus.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/index-search-modern-cpus.pdf -------------------------------------------------------------------------------- /docs/kolonochnyy-soprotsessor-baz-dannyh-dlya-klasternyh-vychislitelnyh-sistem.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/kolonochnyy-soprotsessor-baz-dannyh-dlya-klasternyh-vychislitelnyh-sistem.pdf -------------------------------------------------------------------------------- /docs/parallelnaya-dekompozitsiya-relyatsionnyh-operatsiy-na-osnove-raspredelennyh-kolonochnyh-indeksov.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/parallelnaya-dekompozitsiya-relyatsionnyh-operatsiy-na-osnove-raspredelennyh-kolonochnyh-indeksov.pdf -------------------------------------------------------------------------------- /docs/quad search memorization.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/quad search memorization.pdf -------------------------------------------------------------------------------- /docs/quad search.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/quad search.pdf -------------------------------------------------------------------------------- /docs/rolling hash.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/rolling hash.pdf -------------------------------------------------------------------------------- /docs/structures for inmemory db.PDF: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/structures for inmemory db.PDF -------------------------------------------------------------------------------- /docs/ДЕКОМПОЗИЦИЯ ОПЕРАЦИИ ГРУППИРОВКИ.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/ДЕКОМПОЗИЦИЯ ОПЕРАЦИИ ГРУППИРОВКИ.pdf -------------------------------------------------------------------------------- /docs/ДЕКОМПОЗИЦИЯ ОПЕРАЦИЙ ПЕРЕСЕЧЕНИЯ И СОЕДИНЕНИЯ.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/ДЕКОМПОЗИЦИЯ ОПЕРАЦИЙ ПЕРЕСЕЧЕНИЯ И СОЕДИНЕНИЯ.pdf -------------------------------------------------------------------------------- /docs/КОЛОНОЧНЫЕ ХЕШ-ИНДЕКСЫ.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/КОЛОНОЧНЫЕ ХЕШ-ИНДЕКСЫ.pdf -------------------------------------------------------------------------------- /docs/МЕТОДЫ ПАРАЛЛЕЛЬНОЙ ОБРАБОТКИ.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/МЕТОДЫ ПАРАЛЛЕЛЬНОЙ ОБРАБОТКИ.pdf -------------------------------------------------------------------------------- /docs/Обновление распределенных сжатых колоночных индексов.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/Обновление распределенных сжатых колоночных индексов.pdf -------------------------------------------------------------------------------- /docs/Операции естественного соединения.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/Операции естественного соединения.pdf -------------------------------------------------------------------------------- /docs/Презентация МЕТОДЫ ПАРАЛЛЕЛЬНОЙ ОБРАБОТКИ.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/Презентация МЕТОДЫ ПАРАЛЛЕЛЬНОЙ ОБРАБОТКИ.pdf -------------------------------------------------------------------------------- /docs/Распределенные колоночные индексы.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/Распределенные колоночные индексы.pdf -------------------------------------------------------------------------------- /docs/хэш рабин карп.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/docs/хэш рабин карп.pdf -------------------------------------------------------------------------------- /flysort/benchresults: -------------------------------------------------------------------------------- 1 | goos: linux 2 | goarch: amd64 3 | pkg: github.com/covrom/highloadcup2018/flysort 4 | 5 | BenchmarkInserts/SortInsert100-12 50000000 29.8 ns/op 6 | BenchmarkInserts/LineInsert100-12 50000000 26.5 ns/op 7 | 8 | BenchmarkInserts/SortInsert1000-12 10000000 154 ns/op 9 | BenchmarkInserts/LineInsert1000-12 10000000 146 ns/op 10 | 11 | BenchmarkInserts/SortInsert10000-12 1000000 1950 ns/op 12 | BenchmarkInserts/LineInsert10000-12 1000000 1932 ns/op 13 | 14 | PASS 15 | ok github.com/covrom/highloadcup2018/flysort 10.092s 16 | -------------------------------------------------------------------------------- /flysort/heapinlined.go: -------------------------------------------------------------------------------- 1 | package flysort 2 | 3 | var heapLimit int 4 | 5 | // limited min-heap 6 | type heapInts []int 7 | 8 | func (h heapInts) Less(i, j int) bool { 9 | return h[i] < h[j] 10 | } 11 | 12 | func InitHeap(h *heapInts) { 13 | n := len(*h) 14 | for i := n/2 - 1; i >= 0; i-- { 15 | downHeap(h, i, n) 16 | } 17 | } 18 | 19 | func PushHeap(h *heapInts, x int) { 20 | n := len(*h) 21 | if heapLimit <= 0 || n < heapLimit { 22 | *h = append(*h, x) 23 | upHeap(h, n) 24 | } else if x >= (*h)[0] { 25 | // collect only max N 26 | (*h)[0] = x 27 | downHeap(h, 0, n) 28 | } 29 | } 30 | 31 | func PopHeap(h *heapInts) int { 32 | n := len(*h) - 1 33 | (*h)[0], (*h)[n] = (*h)[n], (*h)[0] 34 | downHeap(h, 0, n) 35 | x := (*h)[n] 36 | *h = (*h)[:n] 37 | return x 38 | } 39 | 40 | func RemoveHeap(h *heapInts, i int) int { 41 | n := len(*h) - 1 42 | if n != i { 43 | (*h)[i], (*h)[n] = (*h)[n], (*h)[i] 44 | if !downHeap(h, i, n) { 45 | upHeap(h, i) 46 | } 47 | } 48 | x := (*h)[n] 49 | *h = (*h)[0 : n-1] 50 | return x 51 | } 52 | 53 | func FixHeap(h *heapInts, i int) { 54 | if !downHeap(h, i, len(*h)) { 55 | upHeap(h, i) 56 | } 57 | } 58 | 59 | func upHeap(h *heapInts, j int) { 60 | for { 61 | i := (j - 1) / 2 // parent 62 | if i == j || !h.Less(j, i) { 63 | break 64 | } 65 | (*h)[i], (*h)[j] = (*h)[j], (*h)[i] 66 | j = i 67 | } 68 | } 69 | 70 | func downHeap(h *heapInts, i0, n int) bool { 71 | i := i0 72 | for { 73 | j1 := 2*i + 1 74 | if j1 >= n || j1 < 0 { // j1 < 0 after int overflow 75 | break 76 | } 77 | j := j1 // left child 78 | if j2 := j1 + 1; j2 < n && h.Less(j2, j1) { 79 | j = j2 // = 2*i + 2 // right child 80 | } 81 | if !h.Less(j, i) { 82 | break 83 | } 84 | (*h)[i], (*h)[j] = (*h)[j], (*h)[i] 85 | i = j 86 | } 87 | return i > i0 88 | } 89 | -------------------------------------------------------------------------------- /flysort/pheapsort.go: -------------------------------------------------------------------------------- 1 | package flysort 2 | 3 | const heap_workers = 3 4 | 5 | func ParallelHeapSort(ints []int) []int { 6 | chres := make(chan int, heap_workers*3) 7 | workSize := len(ints) / heap_workers 8 | for i := 0; i < len(ints); i += workSize { 9 | right := i + workSize 10 | if right >= len(ints) { 11 | right = len(ints) 12 | } 13 | go heapSort(ints[i:right], chres) 14 | } 15 | h := make(heapInts, 0, len(ints)) 16 | heapLimit = 0 17 | for v := range chres { 18 | PushHeap(&h, v) 19 | if len(h) == len(ints) { 20 | break 21 | } 22 | } 23 | res := make([]int, len(h)) 24 | for i := range res { 25 | res[i] = PopHeap(&h) 26 | } 27 | return res 28 | } 29 | 30 | func heapSort(ints []int, chout chan int) { 31 | h := make(heapInts, 0, len(ints)) 32 | for _, v := range ints { 33 | PushHeap(&h, v) 34 | } 35 | for len(h) > 0 { 36 | chout <- PopHeap(&h) 37 | } 38 | } 39 | 40 | func NormalHeapSort(ints []int) []int { 41 | h := make(heapInts, 0, len(ints)) 42 | for _, v := range ints { 43 | PushHeap(&h, v) 44 | } 45 | res := make([]int, 0, len(h)) 46 | for len(h) > 0 { 47 | res = append(res, PopHeap(&h)) 48 | } 49 | return res 50 | } 51 | -------------------------------------------------------------------------------- /flysort/pheapsort_test.go: -------------------------------------------------------------------------------- 1 | package flysort 2 | 3 | import ( 4 | "math/rand" 5 | "sort" 6 | "testing" 7 | ) 8 | 9 | func BenchmarkParallelHeapSort100(b *testing.B) { 10 | if Ints == nil { 11 | fillInts() 12 | } 13 | a := Ints[:100] 14 | b.ResetTimer() 15 | for i := 0; i < b.N; i++ { 16 | ParallelHeapSort(a) 17 | } 18 | } 19 | 20 | func BenchmarkParallelHeapSort1000(b *testing.B) { 21 | if Ints == nil { 22 | fillInts() 23 | } 24 | a := Ints[:1000] 25 | b.ResetTimer() 26 | for i := 0; i < b.N; i++ { 27 | ParallelHeapSort(a) 28 | } 29 | } 30 | 31 | func BenchmarkParallelHeapSort100000(b *testing.B) { 32 | if Ints == nil { 33 | fillInts() 34 | } 35 | a := Ints[:100000] 36 | b.ResetTimer() 37 | for i := 0; i < b.N; i++ { 38 | ParallelHeapSort(a) 39 | } 40 | } 41 | 42 | func BenchmarkNormalHeapSort100(b *testing.B) { 43 | if Ints == nil { 44 | fillInts() 45 | } 46 | a := Ints[:100] 47 | b.ResetTimer() 48 | for i := 0; i < b.N; i++ { 49 | NormalHeapSort(a) 50 | } 51 | } 52 | 53 | func BenchmarkNormalHeapSort1000(b *testing.B) { 54 | if Ints == nil { 55 | fillInts() 56 | } 57 | a := Ints[:1000] 58 | b.ResetTimer() 59 | for i := 0; i < b.N; i++ { 60 | NormalHeapSort(a) 61 | } 62 | } 63 | 64 | func BenchmarkNormalHeapSort100000(b *testing.B) { 65 | if Ints == nil { 66 | fillInts() 67 | } 68 | a := Ints[:100000] 69 | b.ResetTimer() 70 | for i := 0; i < b.N; i++ { 71 | NormalHeapSort(a) 72 | } 73 | } 74 | 75 | func BenchmarkStdSort100(b *testing.B) { 76 | if Ints == nil { 77 | fillInts() 78 | } 79 | a := make([]int, 100) 80 | b.ResetTimer() 81 | for i := 0; i < b.N; i++ { 82 | copy(a, Ints) 83 | sort.Ints(a) 84 | } 85 | } 86 | 87 | func BenchmarkStdSort1000(b *testing.B) { 88 | if Ints == nil { 89 | fillInts() 90 | } 91 | a := make([]int, 1000) 92 | b.ResetTimer() 93 | for i := 0; i < b.N; i++ { 94 | copy(a, Ints) 95 | sort.Ints(a) 96 | } 97 | } 98 | 99 | func BenchmarkStdSort100000(b *testing.B) { 100 | if Ints == nil { 101 | fillInts() 102 | } 103 | a := make([]int, 100000) 104 | b.ResetTimer() 105 | for i := 0; i < b.N; i++ { 106 | copy(a, Ints) 107 | sort.Ints(a) 108 | } 109 | } 110 | 111 | func TestParallelHeapSort(t *testing.T) { 112 | rand.Seed(0) 113 | if Ints == nil { 114 | fillInts() 115 | } 116 | h1 := make([]int, 100) 117 | h2 := make([]int, 100) 118 | h3 := make([]int, 100) 119 | copy(h1, Ints) 120 | copy(h2, Ints) 121 | copy(h3, Ints) 122 | sort.Ints(h1) 123 | h2 = ParallelHeapSort(h2) 124 | h3 = NormalHeapSort(h3) 125 | for i, v := range h1 { 126 | if v != h2[i] || v != h3[i] { 127 | t.Errorf("idx=%v, v1=%v, v2=%v, v3=%v", i, v, h2[i], h3[i]) 128 | } 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /gentest/gentest: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/gentest/gentest -------------------------------------------------------------------------------- /gentest/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "archive/zip" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/covrom/highloadcup2018/account" 8 | "log" 9 | "os" 10 | "sort" 11 | "strings" 12 | ) 13 | 14 | func main() { 15 | 16 | if len(os.Args) < 2 { 17 | log.Fatal("Укажите путь к data.zip") 18 | } 19 | dataFName := os.Args[1] 20 | r, err := zip.OpenReader(dataFName) 21 | if err != nil { 22 | log.Fatal(err) 23 | } 24 | defer r.Close() 25 | 26 | data_accs := []account.Account{} 27 | 28 | maxid := uint32(0) 29 | for _, f := range r.File { 30 | if strings.HasPrefix(f.Name, "accounts_") && strings.HasSuffix(f.Name, ".json") { 31 | rc, err := f.Open() 32 | if err != nil { 33 | log.Println(err) 34 | continue 35 | } 36 | 37 | accs := &account.Accounts{} 38 | dec := json.NewDecoder(rc) 39 | 40 | err = dec.Decode(accs) 41 | if err != nil { 42 | log.Println(err) 43 | continue 44 | } 45 | 46 | data_accs = append(data_accs, accs.Accounts...) 47 | for _, acc := range accs.Accounts { 48 | if acc.ID > maxid { 49 | maxid = acc.ID 50 | } 51 | } 52 | 53 | rc.Close() 54 | } 55 | } 56 | 57 | minLen := 1000 58 | maxLen := 0 59 | runesEmail := make(map[string]int) 60 | for _, acc := range data_accs { 61 | if len(acc.Email) > maxLen { 62 | maxLen = len(acc.Email) 63 | } 64 | if len(acc.Email) < minLen { 65 | minLen = len(acc.Email) 66 | } 67 | s := "" 68 | for i, r := range acc.Email { 69 | if i < 3 { 70 | s += string(r) 71 | } else if i == 3 { 72 | s += string(r) 73 | cnt := runesEmail[s] 74 | cnt++ 75 | runesEmail[s] = cnt 76 | } else { 77 | break 78 | } 79 | } 80 | } 81 | type remlstr struct { 82 | r string 83 | cnt int 84 | } 85 | rems := make([]remlstr, 0, len(runesEmail)) 86 | for k, v := range runesEmail { 87 | rems = append(rems, remlstr{k, v}) 88 | 89 | } 90 | sort.Slice(rems, func(i, j int) bool { return rems[i].cnt > rems[j].cnt }) 91 | 92 | fmt.Println("minLen:", minLen, "maxLen:", maxLen) 93 | for _, r := range rems { 94 | fmt.Printf("%q cnt=%d\n", r.r, r.cnt) 95 | } 96 | // go build . && ./gentest ../test_accounts/data/data.zip >> ./stat.txt 97 | } 98 | -------------------------------------------------------------------------------- /groupset/const.go: -------------------------------------------------------------------------------- 1 | package groupset 2 | 3 | type GroupKey uint8 4 | 5 | const ( 6 | G_sex GroupKey = iota 7 | G_status 8 | G_interests 9 | G_country 10 | G_city 11 | 12 | glength 13 | ) 14 | 15 | const ( 16 | F_query_id uint = iota 17 | F_limit 18 | F_order 19 | F_keys 20 | 21 | F_sex 22 | F_email 23 | F_status 24 | F_fname 25 | F_sname 26 | F_phone 27 | F_country 28 | F_city 29 | F_birth 30 | F_joined 31 | F_interests 32 | F_likes 33 | F_premium 34 | 35 | Flength 36 | ) 37 | 38 | var mapQueryKeys = map[string]GroupKey{ 39 | "sex": G_sex, 40 | "status": G_status, 41 | "interests": G_interests, 42 | "country": G_country, 43 | "city": G_city, 44 | } 45 | 46 | var mapQueryFilter = map[string]uint{ 47 | "query_id": F_query_id, 48 | "limit": F_limit, 49 | "order": F_order, 50 | "keys": F_keys, 51 | 52 | "sex": F_sex, 53 | "email": F_email, 54 | "status": F_status, 55 | "fname": F_fname, 56 | "sname": F_sname, 57 | "phone": F_phone, 58 | "country": F_country, 59 | "city": F_city, 60 | "birth": F_birth, 61 | "joined": F_joined, 62 | "interests": F_interests, 63 | "likes": F_likes, 64 | } 65 | -------------------------------------------------------------------------------- /groupset/описание.txt: -------------------------------------------------------------------------------- 1 | Выборка может идти по полям sex, status, country, city, birth, interests, likes, joined, 2 | но значение в нём будет только одно (для likes будет только один id, для interests только одна строка, для birth и joined - будет одно число - год). 3 | Возможные значения для данных полей перечислены в таблице. 4 | 5 | Название поля Возможные значения 6 | sex выбрать всех, чей пол равен указанному; 7 | status выбрать всех, чей статус равен указанному; 8 | country выбрать всех, кто живёт в данной стране; 9 | city выбрать всех, кто живёт в данном городе; 10 | birth выбрать всех, кто родился в указанном году; 11 | interests выбрать всех, у кого есть данный интерес; (в значении - одна строка); 12 | likes выбрать всех, кто лайкал данного пользователя; (в значении - один id); 13 | joined выбрать всех, кто зарегистрировался в указанном году; 14 | 15 | Порядок сортировки - в начале по count, затем по полям в keys, в порядке, в котором они перечислены в keys. 16 | 17 | -------------------------------------------------------------------------------- /handlers/filter.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "github.com/covrom/highloadcup2018/account" 5 | "github.com/covrom/highloadcup2018/db" 6 | "github.com/covrom/highloadcup2018/queryset" 7 | "io" 8 | 9 | "github.com/valyala/fasthttp" 10 | ) 11 | 12 | func Filter(ctx *fasthttp.RequestCtx) { 13 | // статистика 14 | // if FilterStats != nil { 15 | // FilterStats.AddQueryArgs(ctx.QueryArgs()) 16 | // } 17 | 18 | // выстраиваем цепочку индексов 19 | // сначала жадно ищем покрывающий индекс, и если не нашли - добавляем одинарный обработчик 20 | // индекс является функционалом - он производит переработку текущего сета, переданного ему на вход 21 | // если индекс может отдать биткарту для And-операции, 22 | // то он это делает, и затем результаты нескольких таких фильтров обединяются в одну биткарту, 23 | // которая применяется к текущему сету 24 | 25 | retStatus := 200 26 | 27 | qs := queryset.NewQuerySet() 28 | 29 | ctx.QueryArgs().VisitAll(func(k, v []byte) { 30 | if retStatus == 400 { 31 | return 32 | } 33 | if ok := qs.SetKVFilter(k, v); !ok { 34 | retStatus = 400 35 | } 36 | }) 37 | 38 | if retStatus == 200 { 39 | qs.Validate() 40 | if !qs.HasBadVals { 41 | queryset.OneScan(qs) 42 | } 43 | if qs.HasBadVals { 44 | ctx.Error("", 400) 45 | } else { 46 | ctx.SetContentType("application/json") 47 | io.WriteString(ctx, `{"accounts":[`) 48 | // проходим в нужном порядке через пул воркеров 49 | for j, i := range qs.SortedResult { 50 | if j > 0 { 51 | io.WriteString(ctx, ",") 52 | } 53 | db.SmAcc.RLock() 54 | acc, ok := db.SmAcc.GetById(i) 55 | if ok { 56 | account.WriteSmallAccountJSON(ctx, db.SmAcc, acc, qs.FieldMap) //lks 57 | } 58 | db.SmAcc.RUnlock() 59 | } 60 | io.WriteString(ctx, `]}`) 61 | ctx.SetStatusCode(200) 62 | } 63 | } else { 64 | ctx.Error("", retStatus) 65 | } 66 | 67 | qs.Close() 68 | } 69 | -------------------------------------------------------------------------------- /handlers/filter_stats.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "sort" 7 | "strconv" 8 | "strings" 9 | "sync" 10 | "time" 11 | 12 | "github.com/valyala/fasthttp" 13 | ) 14 | 15 | var FilterStats *FilterStat 16 | 17 | type statRec struct { 18 | cnt int 19 | dmsec int 20 | } 21 | 22 | type FilterStat struct { 23 | mu sync.Mutex 24 | statMap map[string]statRec 25 | maxLimit int 26 | GroupKeys bool 27 | keysMap map[string]int 28 | } 29 | 30 | func NewFilterStat() *FilterStat { 31 | fst := &FilterStat{ 32 | statMap: make(map[string]statRec), 33 | keysMap: make(map[string]int), 34 | } 35 | return fst 36 | } 37 | 38 | func (fst *FilterStat) Add(ss []string, d time.Duration) { 39 | fst.mu.Lock() 40 | defer fst.mu.Unlock() 41 | 42 | sort.Strings(ss) 43 | 44 | name := strings.Join(ss, ",") 45 | cnt := fst.statMap[name] 46 | cnt.cnt++ 47 | cnt.dmsec += int(d / time.Millisecond) 48 | fst.statMap[name] = cnt 49 | 50 | } 51 | 52 | type fstLine struct { 53 | name string 54 | cnt int 55 | dmsec int 56 | } 57 | 58 | func (fst *FilterStat) String() string { 59 | fst.mu.Lock() 60 | defer fst.mu.Unlock() 61 | 62 | s := &strings.Builder{} 63 | strs := make([]fstLine, 0, len(fst.statMap)) 64 | for k, v := range fst.statMap { 65 | strs = append(strs, fstLine{name: k, cnt: v.cnt, dmsec: v.dmsec}) 66 | } 67 | sort.Slice(strs, func(i, j int) bool { 68 | return strs[i].dmsec > strs[j].dmsec || (strs[i].dmsec == strs[j].dmsec && strs[i].cnt > strs[j].cnt) 69 | }) 70 | fmt.Fprintf(s, "Max limit : %d\n", fst.maxLimit) 71 | if fst.GroupKeys { 72 | strs2 := make([]fstLine, 0, len(fst.keysMap)) 73 | for k, v := range fst.keysMap { 74 | strs2 = append(strs2, fstLine{name: k, cnt: v}) 75 | } 76 | sort.Slice(strs2, func(i, j int) bool { 77 | return strs2[i].cnt > strs2[j].cnt 78 | }) 79 | for _, v := range strs2 { 80 | fmt.Fprintf(s, "keys[%s] : %d\n", v.name, v.cnt) 81 | } 82 | } 83 | for _, v := range strs { 84 | fmt.Fprintf(s, "[%s] : %d %dms\n", v.name, v.cnt, v.dmsec) 85 | } 86 | 87 | return s.String() 88 | } 89 | 90 | var b_keys = []byte("keys") 91 | 92 | func (fst *FilterStat) AddQueryArgs(qa *fasthttp.Args, d time.Duration) { 93 | args := make([]string, 0) 94 | qa.VisitAll(func(k, v []byte) { 95 | if !bytes.Equal(k, b_query_id) { 96 | if fst.GroupKeys && bytes.Equal(k, b_keys) { 97 | keys := strings.Split(string(v), ",") 98 | sort.Strings(keys) 99 | ks := strings.Join(keys, ",") 100 | args = append(args, string(k)+"["+ks+"]") 101 | cnt := fst.keysMap[ks] 102 | cnt++ 103 | fst.keysMap[ks] = cnt 104 | } else { 105 | args = append(args, string(k)) 106 | } 107 | } 108 | if bytes.Equal(k, b_limit) { 109 | lim, err := strconv.Atoi(string(v)) 110 | if err == nil && lim > fst.maxLimit { 111 | fst.maxLimit = lim 112 | } 113 | } 114 | }) 115 | fst.Add(args, d) 116 | } 117 | -------------------------------------------------------------------------------- /handlers/group.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "github.com/covrom/highloadcup2018/groupset" 5 | 6 | "github.com/valyala/fasthttp" 7 | ) 8 | 9 | func Group(ctx *fasthttp.RequestCtx) { 10 | retStatus := 200 11 | 12 | gs := groupset.NewGroupSet() 13 | 14 | ctx.QueryArgs().VisitAll(func(k, v []byte) { 15 | if retStatus == 400 { 16 | return 17 | } 18 | if ok := gs.SetKVFilter(k, v); !ok { 19 | retStatus = 400 20 | } 21 | if gs.HasBadVals { 22 | retStatus = 400 23 | } 24 | }) 25 | 26 | if retStatus == 200 { 27 | gs.Validate() 28 | if !gs.HasBadVals { 29 | groupset.OneScan(gs) 30 | } 31 | if gs.HasBadVals { 32 | ctx.Error("", 400) 33 | } else { 34 | ctx.SetContentType("application/json") 35 | gs.MarshalResult(ctx) 36 | ctx.SetStatusCode(200) 37 | } 38 | } else { 39 | ctx.Error("", retStatus) 40 | } 41 | 42 | gs.Close() 43 | } 44 | -------------------------------------------------------------------------------- /handlers/handlers.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "fmt" 5 | "github.com/covrom/highloadcup2018/db" 6 | "strconv" 7 | "time" 8 | 9 | "github.com/valyala/fasthttp" 10 | ) 11 | 12 | func SelectGet(ctx *fasthttp.RequestCtx) { 13 | // done := make(chan struct{}) 14 | // defer close(done) 15 | // go monitorTime(ctx, done) 16 | 17 | if FilterStats != nil { 18 | tnow := time.Now() 19 | defer func() { FilterStats.AddQueryArgs(ctx.QueryArgs(), time.Since(tnow)) }() 20 | } 21 | 22 | s, ok := ctx.UserValue("uid").(string) 23 | if !ok { 24 | ctx.SetStatusCode(404) 25 | return 26 | } 27 | switch s { 28 | case "filter": 29 | if ctx.UserValue("pth") != nil { 30 | ctx.SetStatusCode(404) 31 | return 32 | } 33 | Filter(ctx) 34 | case "group": 35 | if ctx.UserValue("pth") != nil { 36 | ctx.SetStatusCode(404) 37 | return 38 | } 39 | Group(ctx) 40 | default: 41 | i, err := strconv.Atoi(s) 42 | if err != nil { 43 | ctx.SetStatusCode(404) 44 | return 45 | } 46 | switch ctx.UserValue("pth") { 47 | case "suggest": 48 | Suggest(ctx, db.IDAcc(i)) 49 | case "recommend": 50 | Recommend(ctx, db.IDAcc(i)) 51 | default: 52 | ctx.SetStatusCode(404) 53 | return 54 | } 55 | } 56 | } 57 | 58 | func SelectPost(ctx *fasthttp.RequestCtx) { 59 | // done := make(chan struct{}) 60 | // defer close(done) 61 | // go monitorTime(ctx, done) 62 | 63 | if FilterStats != nil { 64 | tnow := time.Now() 65 | defer func() { FilterStats.AddQueryArgs(ctx.QueryArgs(), time.Since(tnow)) }() 66 | } 67 | 68 | s, ok := ctx.UserValue("uid").(string) 69 | if !ok { 70 | ctx.SetStatusCode(404) 71 | return 72 | } 73 | if ctx.UserValue("wrong") != nil { 74 | ctx.SetStatusCode(404) 75 | return 76 | } 77 | switch s { 78 | case "new": 79 | NewAcc(ctx) 80 | case "likes": 81 | Likes(ctx) 82 | default: 83 | i, err := strconv.Atoi(s) 84 | if err != nil { 85 | ctx.SetStatusCode(404) 86 | return 87 | } 88 | UpdateAcc(ctx, db.IDAcc(i)) 89 | } 90 | } 91 | 92 | func monitorTime(ctx *fasthttp.RequestCtx, done chan struct{}) { 93 | const d = 1000 * time.Millisecond 94 | tmr := time.NewTimer(d) 95 | defer tmr.Stop() 96 | for { 97 | select { 98 | case <-tmr.C: 99 | // if FilterStats != nil { 100 | // FilterStats.AddQueryArgs(ctx.QueryArgs(), d) 101 | // } 102 | fmt.Println("overload:", ctx.RequestURI()) 103 | return 104 | case <-done: 105 | return 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /handlers/likes.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "github.com/covrom/highloadcup2018/account" 5 | "github.com/covrom/highloadcup2018/db" 6 | "io" 7 | 8 | "github.com/valyala/fasthttp" 9 | ) 10 | 11 | type oneLike struct { 12 | likee db.IDAcc 13 | ts int32 14 | } 15 | 16 | func Likes(ctx *fasthttp.RequestCtx) { 17 | retStatus := 202 18 | js := ctx.PostBody() 19 | likes := account.UpdateLikes{} 20 | err := (&likes).UnmarshalJSON(js) 21 | if err != nil { 22 | retStatus = 400 23 | } else { 24 | accs := make(map[int32][]oneLike, len(likes.Likes)) 25 | for _, like := range likes.Likes { 26 | db.SmAcc.RLock() 27 | ifrom := db.SmAcc.GetIdx(db.IDAcc(like.Liker)) 28 | db.SmAcc.RUnlock() 29 | if ifrom >= 0 { 30 | db.SmAcc.RLock() 31 | ok := db.SmAcc.Contains(db.IDAcc(like.Likee)) 32 | db.SmAcc.RUnlock() 33 | if ok { 34 | lks, ok := accs[ifrom] 35 | if !ok { 36 | lks = make([]oneLike, 0, 10) 37 | } 38 | lks = append(lks, oneLike{ 39 | likee: db.IDAcc(like.Likee), 40 | ts: like.Stamp.Int(), 41 | }) 42 | accs[ifrom] = lks 43 | } else { 44 | retStatus = 400 45 | break 46 | } 47 | } else { 48 | retStatus = 400 49 | break 50 | } 51 | } 52 | if retStatus == 202 { 53 | for idx, ls := range accs { 54 | db.SmAcc.Lock() 55 | acc := db.SmAcc.Get(idx) 56 | acclikes := make([]db.Like, len(ls)) 57 | for i, l := range ls { 58 | acclikes[i] = db.NewLike( 59 | acc.ID, 60 | l.likee, 61 | l.ts, 62 | ) 63 | } 64 | db.SmAcc.Append(acc, acclikes) 65 | db.SmAcc.Unlock() 66 | } 67 | } 68 | } 69 | 70 | ctx.SetStatusCode(retStatus) 71 | if retStatus == 202 { 72 | ctx.SetContentType("application/json") 73 | io.WriteString(ctx, `{}`) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /handlers/new.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "github.com/covrom/highloadcup2018/account" 5 | "github.com/covrom/highloadcup2018/db" 6 | "io" 7 | 8 | "github.com/valyala/fasthttp" 9 | ) 10 | 11 | func NewAcc(ctx *fasthttp.RequestCtx) { 12 | retStatus := 201 13 | js := ctx.PostBody() 14 | acc := account.Account{} 15 | err := (&acc).UnmarshalJSON(js) 16 | if err != nil { 17 | retStatus = 400 18 | } else { 19 | db.SmAcc.RLock() 20 | ok := db.SmAcc.Contains(db.IDAcc(acc.ID)) 21 | db.SmAcc.RUnlock() 22 | if ok { 23 | retStatus = 400 24 | } else { 25 | db.SmAcc.Lock() 26 | smacc, likes := account.ConvertAccountToSmall(acc, db.SmAcc, false) 27 | db.SmAcc.Append(smacc, likes) 28 | db.SmAcc.Unlock() 29 | } 30 | } 31 | 32 | ctx.SetStatusCode(retStatus) 33 | if retStatus == 201 { 34 | ctx.SetContentType("application/json") 35 | io.WriteString(ctx, `{}`) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /handlers/recommend.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "bytes" 5 | "github.com/covrom/highloadcup2018/account" 6 | "github.com/covrom/highloadcup2018/db" 7 | "github.com/covrom/highloadcup2018/recommend" 8 | "io" 9 | "strconv" 10 | 11 | "github.com/valyala/fasthttp" 12 | ) 13 | 14 | var ( 15 | b_country = []byte("country") 16 | b_city = []byte("city") 17 | b_query_id = []byte("query_id") 18 | b_limit = []byte("limit") 19 | ) 20 | 21 | type eventRecommend struct { 22 | limit int // -1 без лимита 23 | id db.IDAcc 24 | country, 25 | city db.DataEntry 26 | } 27 | 28 | func Recommend(ctx *fasthttp.RequestCtx, id db.IDAcc) { 29 | 30 | retStatus := 200 31 | 32 | limit := 20 33 | 34 | ev := eventRecommend{ 35 | id: id, 36 | } 37 | 38 | idx := db.SmAcc.GetIdx(id) 39 | if idx < 0 { 40 | retStatus = 404 41 | } else { 42 | // обходим в порядке указания параметров 43 | ctx.QueryArgs().VisitAll(func(k, v []byte) { 44 | if retStatus == 400 { 45 | // если была ошибка - возвращаемся 46 | return 47 | } 48 | if bytes.Equal(k, b_country) { 49 | if vidx, ok := db.SmAcc.Country.InDictonary(string(v)); ok && !db.SmAcc.Country.IsZero(vidx) { 50 | ev.country = vidx 51 | } else { 52 | retStatus = 400 53 | } 54 | } else if bytes.Equal(k, b_city) { 55 | if vidx, ok := db.SmAcc.City.InDictonary(string(v)); ok && !db.SmAcc.City.IsZero(vidx) { 56 | ev.city = vidx 57 | } else { 58 | retStatus = 400 59 | } 60 | } else if bytes.Equal(k, b_query_id) { 61 | // ничего 62 | } else if bytes.Equal(k, b_limit) { 63 | var err error 64 | limit, err = strconv.Atoi(string(v)) 65 | if err != nil || limit <= 0 { 66 | retStatus = 400 67 | } 68 | } else { 69 | retStatus = 400 70 | } 71 | // здесь нельзя ничего делать, только возвращаемся 72 | }) 73 | ev.limit = limit 74 | } 75 | 76 | if retStatus == 200 { 77 | ctx.SetContentType("application/json") 78 | io.WriteString(ctx, `{"accounts":[`) 79 | // проходим в нужном порядке через пул воркеров 80 | res := recommend.OneScan(ev.id, ev.country, ev.city, ev.limit) 81 | for j, i := range res { 82 | if j > 0 { 83 | io.WriteString(ctx, ",") 84 | } 85 | db.SmAcc.RLock() 86 | acc, ok := db.SmAcc.GetById(i) 87 | if ok { 88 | account.WriteSmallAccountJSON(ctx, db.SmAcc, acc, account.MaskRecommend) 89 | } 90 | db.SmAcc.RUnlock() 91 | } 92 | recommend.PutCndsmSlice(res) 93 | io.WriteString(ctx, `]}`) 94 | ctx.SetStatusCode(200) 95 | 96 | } else { 97 | ctx.Error("", retStatus) 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /handlers/update.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "github.com/covrom/highloadcup2018/account" 5 | "github.com/covrom/highloadcup2018/db" 6 | "github.com/covrom/highloadcup2018/dict" 7 | "io" 8 | "strings" 9 | 10 | "github.com/valyala/fasthttp" 11 | ) 12 | 13 | func UpdateAcc(ctx *fasthttp.RequestCtx, id db.IDAcc) { 14 | 15 | retStatus := 202 16 | db.SmAcc.RLock() 17 | i := db.SmAcc.GetIdx(id) 18 | db.SmAcc.RUnlock() 19 | if i < 0 { 20 | retStatus = 404 21 | } else { 22 | js := ctx.PostBody() 23 | acc := account.Account{} 24 | err := (&acc).UnmarshalJSON(js) 25 | if err != nil && err != account.ErrEmptyID { 26 | retStatus = 400 27 | } else { 28 | acc.ID = uint32(id) 29 | 30 | //валидация 31 | if len(acc.Email) > 0 { 32 | if !strings.ContainsRune(acc.Email, '@') { 33 | retStatus = 400 34 | } 35 | } 36 | 37 | if len(acc.Status) > 0 { 38 | if !(acc.Status == "свободны" || acc.Status == "заняты" || acc.Status == "всё сложно") { 39 | retStatus = 400 40 | } 41 | } 42 | 43 | if len(acc.Sex) > 0 { 44 | if !(acc.Sex == "f" || acc.Sex == "m") { 45 | retStatus = 400 46 | } 47 | } 48 | 49 | if retStatus == 202 { 50 | // есть такой же email для другого id? 51 | emlidx, ok := dict.DictonaryEml.In(acc.Email) 52 | if ok && emlidx != uint32(id) { 53 | retStatus = 400 54 | } else { 55 | db.SmAcc.Lock() 56 | smacc := db.SmAcc.Get(i) 57 | likes := db.GetLikesSlice(len(acc.Likes)) 58 | smacc, likes = account.ConvertAccountUpdateSmall(acc, smacc, likes, db.SmAcc) 59 | db.SmAcc.Append(smacc, likes) 60 | db.PutLikesSlice(likes) 61 | db.SmAcc.Unlock() 62 | } 63 | } 64 | } 65 | } 66 | 67 | ctx.SetStatusCode(retStatus) 68 | if retStatus == 202 { 69 | ctx.SetContentType("application/json") 70 | io.WriteString(ctx, `{}`) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /intsearch/benchresults: -------------------------------------------------------------------------------- 1 | goos: linux 2 | goarch: amd64 3 | pkg: github.com/covrom/highloadcup2018/intsearch 4 | 5 | BenchmarkStd100-12 50000000 36.3 ns/op 6 | BenchmarkStd200-12 30000000 41.8 ns/op 7 | BenchmarkStd500-12 30000000 52.8 ns/op 8 | BenchmarkStd1000-12 30000000 58.0 ns/op 9 | BenchmarkStd10000-12 20000000 75.2 ns/op 10 | BenchmarkStd1e5-12 20000000 83.8 ns/op 11 | BenchmarkStd1e6-12 20000000 91.8 ns/op 12 | 13 | BenchmarkInterp100-12 50000000 21.4 ns/op 14 | BenchmarkInterp200-12 50000000 33.4 ns/op 15 | BenchmarkInterp500-12 50000000 35.0 ns/op 16 | BenchmarkInterp1000-12 50000000 37.8 ns/op 17 | BenchmarkInterp10000-12 30000000 45.4 ns/op 18 | BenchmarkInterp1e5-12 30000000 48.8 ns/op 19 | BenchmarkInterp1e6-12 20000000 62.1 ns/op 20 | 21 | BenchmarkAsm100-12 100000000 14.5 ns/op 22 | BenchmarkAsm200-12 100000000 16.5 ns/op 23 | BenchmarkAsm500-12 100000000 19.7 ns/op 24 | BenchmarkAsm1000-12 100000000 23.1 ns/op 25 | BenchmarkAsm10000-12 50000000 33.2 ns/op 26 | BenchmarkAsm1e5-12 50000000 36.6 ns/op 27 | BenchmarkAsm1e6-12 50000000 37.0 ns/op 28 | 29 | BenchmarkBin100-12 100000000 15.0 ns/op 30 | BenchmarkBin200-12 100000000 16.9 ns/op 31 | BenchmarkBin500-12 100000000 20.9 ns/op 32 | BenchmarkBin1000-12 50000000 24.7 ns/op 33 | BenchmarkBin10000-12 50000000 35.7 ns/op 34 | BenchmarkBin1e5-12 50000000 36.8 ns/op 35 | BenchmarkBin1e6-12 50000000 37.9 ns/op 36 | 37 | BenchmarkBinApprox100-12 50000000 25.3 ns/op 38 | BenchmarkBinApprox200-12 50000000 27.3 ns/op 39 | BenchmarkBinApprox500-12 50000000 31.1 ns/op 40 | BenchmarkBinApprox1000-12 50000000 32.4 ns/op 41 | BenchmarkBinApprox10000-12 50000000 35.2 ns/op 42 | BenchmarkBinApprox1e5-12 50000000 37.9 ns/op 43 | BenchmarkBinApprox1e6-12 30000000 42.0 ns/op 44 | 45 | BenchmarkLine100-12 30000000 47.0 ns/op 46 | BenchmarkLine200-12 20000000 73.1 ns/op 47 | BenchmarkLine500-12 10000000 151 ns/op 48 | BenchmarkLine1000-12 5000000 278 ns/op 49 | BenchmarkLine10000-12 500000 2625 ns/op 50 | BenchmarkLine1e5-12 50000 25200 ns/op 51 | BenchmarkLine1e6-12 10000 142944 ns/op 52 | 53 | PASS 54 | ok github.com/covrom/highloadcup2018/intsearch 75.429s 55 | -------------------------------------------------------------------------------- /intsearch/ints/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/covrom/highloadcup2018/intsearch" 6 | "math/rand" 7 | ) 8 | 9 | // go build -gcflags -S ./main.go 10 | 11 | func main() { 12 | rand.Seed(0) 13 | 14 | const limit = 100 15 | 16 | ints := make([]uint32, limit) 17 | 18 | for i := range ints { 19 | ints[i] = uint32(i) 20 | } 21 | 22 | for want, q := range ints { 23 | if idx := intsearch.AsmSearchInts(ints, q); idx != uint32(want) { 24 | fmt.Printf("StdSearchInts(ints, %v)=%v, want %v\n", q, idx, want) 25 | } 26 | if idx := intsearch.StdSearchInts(ints, q); idx != uint32(want) { 27 | fmt.Printf("SearchInts(ints, %v)=%v, want %v\n", q, idx, want) 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /intsearch/intsearch_amd64.s: -------------------------------------------------------------------------------- 1 | // +build amd64 2 | 3 | TEXT ·stdSearch(SB), 4, $0-40 4 | MOVQ a+0(FP), AX 5 | MOVQ a_len+8(FP), DI 6 | MOVL x+24(FP), DX 7 | XORL SI, SI 8 | other1: 9 | MOVL DI, CX 10 | loop1: 11 | CMPL SI, CX 12 | JCC return1 13 | LEAL (SI)(CX*1), DI 14 | SHRL $1, DI 15 | MOVL (AX)(DI*4), R8 16 | CMPL R8, DX 17 | JCC other1 18 | LEAL 1(DI), SI 19 | JMP loop1 20 | return1: 21 | MOVL SI, ret+32(FP) 22 | RET 23 | -------------------------------------------------------------------------------- /intsearch/intsearch_asm.go: -------------------------------------------------------------------------------- 1 | // +build amd64 2 | 3 | package intsearch 4 | 5 | //go:noescape 6 | 7 | func stdSearch(a []uint32, x uint32) uint32 8 | 9 | // { 10 | // n := uint32(len(a)) 11 | // i, j := uint32(0), n 12 | // for i < j { 13 | // h := (i + j) >> 1 14 | // if a[h] < x { 15 | // i = h + 1 16 | // } else { 17 | // j = h 18 | // } 19 | // } 20 | // return i 21 | // } 22 | -------------------------------------------------------------------------------- /intsearch/intunsafe.go: -------------------------------------------------------------------------------- 1 | package intsearch 2 | 3 | import "unsafe" 4 | 5 | // see https://github.com/golang/go/issues/30366 6 | func unsafeBinSearch(a []uint32, x uint32) uint32 { 7 | n := uint32(len(a)) 8 | 9 | p := (*[1 << 31]uint32)(unsafe.Pointer(&a[0])) 10 | if n == 0 || p == nil { 11 | return 0 12 | } 13 | 14 | i, j := uint32(0), n 15 | for i < j { 16 | h := (i + j) >> 1 17 | if p[h] < x { 18 | i = h + 1 19 | } else { 20 | j = h 21 | } 22 | } 23 | return i 24 | } 25 | -------------------------------------------------------------------------------- /intsearch/intunsafe.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/intsearch/intunsafe.o -------------------------------------------------------------------------------- /jumphash/jumphash.go: -------------------------------------------------------------------------------- 1 | package jumphash 2 | 3 | // Package jump implements Google's Jump Consistent Hash 4 | /* 5 | From the paper "A Fast, Minimal Memory, Consistent Hash Algorithm" by John Lamping, Eric Veach (2014). 6 | http://arxiv.org/abs/1406.2294 7 | */ 8 | 9 | // Hash consistently chooses a hash bucket number in the range [0, numBuckets) for the given key. numBuckets must be >= 1. 10 | func Hash(key uint64, numBuckets int) int32 { 11 | 12 | var b int64 = -1 13 | var j int64 14 | 15 | for j < int64(numBuckets) { 16 | b = j 17 | key = key*2862933555777941757 + 1 18 | j = int64(float64(b+1) * (float64(int64(1)<<31) / float64((key>>33)+1))) 19 | } 20 | 21 | return int32(b) 22 | } 23 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "github.com/covrom/highloadcup2018/account" 6 | "github.com/covrom/highloadcup2018/db" 7 | "github.com/covrom/highloadcup2018/handlers" 8 | "github.com/covrom/highloadcup2018/parser" 9 | "github.com/covrom/highloadcup2018/recommend" 10 | "log" 11 | "os" 12 | "os/signal" 13 | "regexp" 14 | "sync" 15 | "syscall" 16 | "time" 17 | 18 | // "net/http" 19 | // _ "net/http/pprof" 20 | 21 | // "github.com/pkg/profile" 22 | 23 | "github.com/buaazp/fasthttprouter" 24 | "github.com/valyala/fasthttp" 25 | ) 26 | 27 | var ( 28 | // Важно - сервер должен слушать 80-й порт, чтобы обстрел прошел успешно! 29 | // Запросы идут с заголовком Host: accounts.com 30 | // по протоколу HTTP/1.1 с переиспользуемыми соединениями (keep-alive). 31 | // Сетевые потери полностью отсутствуют. 32 | addr = flag.String("addr", ":80", "addr:port") 33 | ) 34 | 35 | func onShutdown(f func()) { 36 | once := &sync.Once{} 37 | sigc := make(chan os.Signal, 3) 38 | signal.Notify(sigc, os.Interrupt, os.Kill, syscall.SIGTERM) 39 | go func() { 40 | <-sigc 41 | once.Do(f) 42 | }() 43 | } 44 | 45 | func main() { 46 | 47 | flag.Parse() 48 | 49 | zt, err := time.Parse(time.RFC3339, "1949-12-31T23:59:59Z") 50 | if err != nil { 51 | log.Fatal(err) 52 | } 53 | db.NullTime = int32(zt.Unix()) 54 | 55 | account.RePhoneCode = regexp.MustCompile(`\([0-9]+\)`) 56 | 57 | recommend.StatusBusy = db.SmAcc.Status.ToDictonary("заняты") //1 58 | recommend.StatusTricky = db.SmAcc.Status.ToDictonary("всё сложно") //2 59 | recommend.StatusFree = db.SmAcc.Status.ToDictonary("свободны") //3 60 | 61 | // go tool pprof -seconds 180 ./highloadcup2018 http://127.0.0.1:8080/debug/pprof/profile 62 | // go tool pprof -alloc_objects ./highloadcup2018 http://127.0.0.1:8080/debug/pprof/heap 63 | // go tool pprof -inuse_space ./highloadcup2018 http://127.0.0.1:8080/debug/pprof/heap 64 | 65 | // go http.ListenAndServe(":8080", nil) 66 | 67 | parser.ParseData() 68 | 69 | // profiler := profile.Start(profile.TraceProfile, profile.ProfilePath(".")) 70 | // defer profiler.Stop() 71 | 72 | router := fasthttprouter.New() 73 | router.GET("/accounts/:uid/", handlers.SelectGet) 74 | router.GET("/accounts/:uid/:pth/", handlers.SelectGet) 75 | router.POST("/accounts/:uid/", handlers.SelectPost) 76 | router.POST("/accounts/:uid/:wrong/", handlers.SelectPost) 77 | 78 | s := &fasthttp.Server{ 79 | Handler: router.Handler, 80 | // Concurrency: 4000, 81 | // ReadTimeout: 3 * time.Second, 82 | // WriteTimeout: 3 * time.Second, 83 | // MaxConnsPerIP: 4000, 84 | // MaxRequestsPerConn: 1, 85 | // MaxKeepaliveDuration: 60 * time.Minute, 86 | ReduceMemoryUsage: true, 87 | // LogAllErrors: true, 88 | } 89 | if err := s.ListenAndServe(*addr); err != nil { 90 | log.Fatalf("error in ListenAndServe: %s", err) 91 | } 92 | 93 | // go get -u github.com/atercattus/highloadcup_tester 94 | // highloadcup_tester -addr http://127.0.0.1:8000 -hlcupdocs ./test_accounts/ -test -phase 1 -utf8 -filter "filter" 95 | // highloadcup_tester -addr http://127.0.0.1:8000 -hlcupdocs ./test_accounts/ -tank 300 -time 180s -phase 1 -utf8 -filter "filter" 96 | 97 | } 98 | -------------------------------------------------------------------------------- /queryset/chans.go: -------------------------------------------------------------------------------- 1 | package queryset 2 | 3 | import "github.com/covrom/highloadcup2018/db" 4 | 5 | func FilterLimit(qs *QuerySet) { 6 | iter := db.SmAcc.Iterator() 7 | limit := qs.Limit 8 | for iter.HasNext() { 9 | if limit <= 0 || qs.HasBadVals { 10 | break 11 | } 12 | id := iter.NextID() 13 | qs.SortedResult = append(qs.SortedResult, id) 14 | limit-- 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /queryset/const.go: -------------------------------------------------------------------------------- 1 | package queryset 2 | 3 | const ( 4 | F_query_id uint = iota 5 | F_limit 6 | F_sex_eq 7 | F_email_domain 8 | F_email_lt 9 | F_email_gt 10 | F_status_eq 11 | F_status_neq 12 | F_fname_eq 13 | F_fname_any 14 | F_fname_null 15 | F_sname_eq 16 | F_sname_starts 17 | F_sname_null 18 | F_phone_code 19 | F_phone_null 20 | F_country_eq 21 | F_country_null 22 | F_city_eq 23 | F_city_any 24 | F_city_null 25 | F_birth_year 26 | F_birth_lt 27 | F_birth_gt 28 | F_interests_contains 29 | F_interests_any 30 | F_likes_contains 31 | F_premium_now 32 | F_premium_null 33 | 34 | F_country 35 | F_city 36 | 37 | Flength 38 | ) 39 | 40 | var mapQueryKeysFilter = map[string]uint{ 41 | "query_id": F_query_id, 42 | "limit": F_limit, 43 | "sex_eq": F_sex_eq, 44 | "email_domain": F_email_domain, 45 | "email_lt": F_email_lt, 46 | "email_gt": F_email_gt, 47 | "status_eq": F_status_eq, 48 | "status_neq": F_status_neq, 49 | "fname_eq": F_fname_eq, 50 | "fname_any": F_fname_any, 51 | "fname_null": F_fname_null, 52 | "sname_eq": F_sname_eq, 53 | "sname_starts": F_sname_starts, 54 | "sname_null": F_sname_null, 55 | "phone_code": F_phone_code, 56 | "phone_null": F_phone_null, 57 | "country_eq": F_country_eq, 58 | "country_null": F_country_null, 59 | "city_eq": F_city_eq, 60 | "city_any": F_city_any, 61 | "city_null": F_city_null, 62 | "birth_year": F_birth_year, 63 | "birth_lt": F_birth_lt, 64 | "birth_gt": F_birth_gt, 65 | "interests_contains": F_interests_contains, 66 | "interests_any": F_interests_any, 67 | "likes_contains": F_likes_contains, 68 | "premium_now": F_premium_now, 69 | "premium_null": F_premium_null, 70 | } 71 | 72 | var mapQueryKeysRecommend = map[string]uint{ 73 | "query_id": F_query_id, 74 | "limit": F_limit, 75 | "country": F_country, 76 | "city": F_city, 77 | } 78 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | go generate ./... 4 | go build -a . 5 | #docker build --no-cache --pull -t highloadcup2018 . 6 | #docker run --rm -it -p 8000:80 -t highloadcup2018 7 | mkdir -p /tmp/data 8 | cp ./test_accounts/data/data.zip /tmp/data/ 9 | cp ./test_accounts/data/options.txt /tmp/data/ 10 | ./highloadcup2018 -addr ":8000" -------------------------------------------------------------------------------- /run0.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | go generate ./... 4 | go build -a . 5 | #docker build --no-cache --pull -t highloadcup2018 . 6 | #docker run --rm -it -p 8000:80 -t highloadcup2018 7 | mkdir -p /tmp/data 8 | cp ./test_accounts0/data/data.zip /tmp/data/ 9 | cp ./test_accounts0/data/options.txt /tmp/data/ 10 | ./highloadcup2018 -addr ":8000" -------------------------------------------------------------------------------- /suggest/heap.go: -------------------------------------------------------------------------------- 1 | package suggest 2 | 3 | const heapLimit = 21 4 | 5 | // limited min-heap 6 | type heapSims []simrec 7 | 8 | func (h heapSims) Len() int { return len(h) } 9 | func (h heapSims) Less(i, j int) bool { 10 | return (h[i].sim < h[j].sim) || ((h[i].sim == h[j].sim) && (h[i].like < h[j].like)) 11 | } 12 | func (h heapSims) Swap(i, j int) { h[i], h[j] = h[j], h[i] } 13 | func (h *heapSims) Push(x simrec) { *h = append(*h, x) } 14 | func (h *heapSims) Pop() simrec { 15 | old := *h 16 | n := len(old) 17 | x := old[n-1] 18 | *h = old[0 : n-1] 19 | return x 20 | } 21 | 22 | func InitHeap(h *heapSims) { 23 | n := h.Len() 24 | for i := n/2 - 1; i >= 0; i-- { 25 | downHeap(h, i, n) 26 | } 27 | } 28 | 29 | func PushHeap(h *heapSims, x simrec) { 30 | n := h.Len() 31 | if n < heapLimit { 32 | h.Push(x) 33 | upHeap(h, n) 34 | } else { 35 | (*h)[0] = x 36 | downHeap(h, 0, n) 37 | } 38 | } 39 | 40 | func PopHeap(h *heapSims) simrec { 41 | n := h.Len() - 1 42 | h.Swap(0, n) 43 | downHeap(h, 0, n) 44 | return h.Pop() 45 | } 46 | 47 | func RemoveHeap(h *heapSims, i int) simrec { 48 | n := h.Len() - 1 49 | if n != i { 50 | h.Swap(i, n) 51 | if !downHeap(h, i, n) { 52 | upHeap(h, i) 53 | } 54 | } 55 | return h.Pop() 56 | } 57 | 58 | func FixHeap(h *heapSims, i int) { 59 | if !downHeap(h, i, h.Len()) { 60 | upHeap(h, i) 61 | } 62 | } 63 | 64 | func upHeap(h *heapSims, j int) { 65 | for { 66 | i := (j - 1) / 2 // parent 67 | if i == j || !h.Less(j, i) { 68 | break 69 | } 70 | h.Swap(i, j) 71 | j = i 72 | } 73 | } 74 | 75 | func downHeap(h *heapSims, i0, n int) bool { 76 | i := i0 77 | for { 78 | j1 := 2*i + 1 79 | if j1 >= n || j1 < 0 { // j1 < 0 after int overflow 80 | break 81 | } 82 | j := j1 // left child 83 | if j2 := j1 + 1; j2 < n && h.Less(j2, j1) { 84 | j = j2 // = 2*i + 2 // right child 85 | } 86 | if !h.Less(j, i) { 87 | break 88 | } 89 | h.Swap(i, j) 90 | i = j 91 | } 92 | return i > i0 93 | } 94 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | roaring-fuzz.zip 3 | workdir 4 | coverage.out 5 | testdata/all3.classic 6 | testdata/all3.msgp.snappy 7 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/vendor/github.com/RoaringBitmap/roaring/.gitmodules -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | sudo: false 3 | install: 4 | - go get -t github.com/RoaringBitmap/roaring 5 | - go get -t golang.org/x/tools/cmd/cover 6 | - go get -t github.com/mattn/goveralls 7 | - go get -t github.com/mschoch/smat 8 | notifications: 9 | email: false 10 | go: 11 | - "1.7.x" 12 | - "1.8.x" 13 | - "1.9.x" 14 | - "1.10.x" 15 | - tip 16 | 17 | # whitelist 18 | branches: 19 | only: 20 | - master 21 | script: 22 | - goveralls -v -service travis-ci -ignore arraycontainer_gen.go,bitmapcontainer_gen.go,rle16_gen.go,rle_gen.go,roaringarray_gen.go,rle.go || go test 23 | - go test -race -run TestConcurrent* 24 | - GOARCH=arm64 go build 25 | - GOARCH=386 go build 26 | - GOARCH=386 go test 27 | - GOARCH=arm go build 28 | matrix: 29 | allow_failures: 30 | - go: tip 31 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the official list of roaring authors for copyright purposes. 2 | 3 | Todd Gruben (@tgruben), 4 | Daniel Lemire (@lemire), 5 | Elliot Murphy (@statik), 6 | Bob Potter (@bpot), 7 | Tyson Maly (@tvmaly), 8 | Will Glynn (@willglynn), 9 | Brent Pedersen (@brentp) 10 | Maciej Biłas (@maciej), 11 | Joe Nall (@joenall) 12 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | # This is the official list of roaring contributors 2 | 3 | Todd Gruben (@tgruben), 4 | Daniel Lemire (@lemire), 5 | Elliot Murphy (@statik), 6 | Bob Potter (@bpot), 7 | Tyson Maly (@tvmaly), 8 | Will Glynn (@willglynn), 9 | Brent Pedersen (@brentp), 10 | Jason E. Aten (@glycerine), 11 | Vali Malinoiu (@0x4139), 12 | Forud Ghafouri (@fzerorubigd), 13 | Joe Nall (@joenall), 14 | (@fredim) 15 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/arraycontainer_gen.go: -------------------------------------------------------------------------------- 1 | package roaring 2 | 3 | // NOTE: THIS FILE WAS PRODUCED BY THE 4 | // MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) 5 | // DO NOT EDIT 6 | 7 | import "github.com/tinylib/msgp/msgp" 8 | 9 | // Deprecated: DecodeMsg implements msgp.Decodable 10 | func (z *arrayContainer) DecodeMsg(dc *msgp.Reader) (err error) { 11 | var field []byte 12 | _ = field 13 | var zbzg uint32 14 | zbzg, err = dc.ReadMapHeader() 15 | if err != nil { 16 | return 17 | } 18 | for zbzg > 0 { 19 | zbzg-- 20 | field, err = dc.ReadMapKeyPtr() 21 | if err != nil { 22 | return 23 | } 24 | switch msgp.UnsafeString(field) { 25 | case "content": 26 | var zbai uint32 27 | zbai, err = dc.ReadArrayHeader() 28 | if err != nil { 29 | return 30 | } 31 | if cap(z.content) >= int(zbai) { 32 | z.content = (z.content)[:zbai] 33 | } else { 34 | z.content = make([]uint16, zbai) 35 | } 36 | for zxvk := range z.content { 37 | z.content[zxvk], err = dc.ReadUint16() 38 | if err != nil { 39 | return 40 | } 41 | } 42 | default: 43 | err = dc.Skip() 44 | if err != nil { 45 | return 46 | } 47 | } 48 | } 49 | return 50 | } 51 | 52 | // Deprecated: EncodeMsg implements msgp.Encodable 53 | func (z *arrayContainer) EncodeMsg(en *msgp.Writer) (err error) { 54 | // map header, size 1 55 | // write "content" 56 | err = en.Append(0x81, 0xa7, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74) 57 | if err != nil { 58 | return err 59 | } 60 | err = en.WriteArrayHeader(uint32(len(z.content))) 61 | if err != nil { 62 | return 63 | } 64 | for zxvk := range z.content { 65 | err = en.WriteUint16(z.content[zxvk]) 66 | if err != nil { 67 | return 68 | } 69 | } 70 | return 71 | } 72 | 73 | // Deprecated: MarshalMsg implements msgp.Marshaler 74 | func (z *arrayContainer) MarshalMsg(b []byte) (o []byte, err error) { 75 | o = msgp.Require(b, z.Msgsize()) 76 | // map header, size 1 77 | // string "content" 78 | o = append(o, 0x81, 0xa7, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74) 79 | o = msgp.AppendArrayHeader(o, uint32(len(z.content))) 80 | for zxvk := range z.content { 81 | o = msgp.AppendUint16(o, z.content[zxvk]) 82 | } 83 | return 84 | } 85 | 86 | // Deprecated: UnmarshalMsg implements msgp.Unmarshaler 87 | func (z *arrayContainer) UnmarshalMsg(bts []byte) (o []byte, err error) { 88 | var field []byte 89 | _ = field 90 | var zcmr uint32 91 | zcmr, bts, err = msgp.ReadMapHeaderBytes(bts) 92 | if err != nil { 93 | return 94 | } 95 | for zcmr > 0 { 96 | zcmr-- 97 | field, bts, err = msgp.ReadMapKeyZC(bts) 98 | if err != nil { 99 | return 100 | } 101 | switch msgp.UnsafeString(field) { 102 | case "content": 103 | var zajw uint32 104 | zajw, bts, err = msgp.ReadArrayHeaderBytes(bts) 105 | if err != nil { 106 | return 107 | } 108 | if cap(z.content) >= int(zajw) { 109 | z.content = (z.content)[:zajw] 110 | } else { 111 | z.content = make([]uint16, zajw) 112 | } 113 | for zxvk := range z.content { 114 | z.content[zxvk], bts, err = msgp.ReadUint16Bytes(bts) 115 | if err != nil { 116 | return 117 | } 118 | } 119 | default: 120 | bts, err = msgp.Skip(bts) 121 | if err != nil { 122 | return 123 | } 124 | } 125 | } 126 | o = bts 127 | return 128 | } 129 | 130 | // Deprecated: Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message 131 | func (z *arrayContainer) Msgsize() (s int) { 132 | s = 1 + 8 + msgp.ArrayHeaderSize + (len(z.content) * (msgp.Uint16Size)) 133 | return 134 | } 135 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/clz.go: -------------------------------------------------------------------------------- 1 | // +build go1.9 2 | // "go1.9", from Go version 1.9 onward 3 | // See https://golang.org/pkg/go/build/#hdr-Build_Constraints 4 | 5 | package roaring 6 | 7 | import "math/bits" 8 | 9 | func countLeadingZeros(x uint64) int { 10 | return bits.LeadingZeros64(x) 11 | } 12 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/clz_compat.go: -------------------------------------------------------------------------------- 1 | // +build !go1.9 2 | 3 | package roaring 4 | 5 | // LeadingZeroBits returns the number of consecutive most significant zero 6 | // bits of x. 7 | func countLeadingZeros(i uint64) int { 8 | if i == 0 { 9 | return 64 10 | } 11 | n := 1 12 | x := uint32(i >> 32) 13 | if x == 0 { 14 | n += 32 15 | x = uint32(i) 16 | } 17 | if (x >> 16) == 0 { 18 | n += 16 19 | x <<= 16 20 | } 21 | if (x >> 24) == 0 { 22 | n += 8 23 | x <<= 8 24 | } 25 | if x>>28 == 0 { 26 | n += 4 27 | x <<= 4 28 | } 29 | if x>>30 == 0 { 30 | n += 2 31 | x <<= 2 32 | 33 | } 34 | n -= int(x >> 31) 35 | return n 36 | } 37 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/ctz.go: -------------------------------------------------------------------------------- 1 | // +build go1.9 2 | // "go1.9", from Go version 1.9 onward 3 | // See https://golang.org/pkg/go/build/#hdr-Build_Constraints 4 | 5 | package roaring 6 | 7 | import "math/bits" 8 | 9 | func countTrailingZeros(x uint64) int { 10 | return bits.TrailingZeros64(x) 11 | } 12 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/ctz_compat.go: -------------------------------------------------------------------------------- 1 | // +build !go1.9 2 | 3 | package roaring 4 | 5 | // Reuse of portions of go/src/math/big standard lib code 6 | // under this license: 7 | /* 8 | Copyright (c) 2009 The Go Authors. All rights reserved. 9 | 10 | Redistribution and use in source and binary forms, with or without 11 | modification, are permitted provided that the following conditions are 12 | met: 13 | 14 | * Redistributions of source code must retain the above copyright 15 | notice, this list of conditions and the following disclaimer. 16 | * Redistributions in binary form must reproduce the above 17 | copyright notice, this list of conditions and the following disclaimer 18 | in the documentation and/or other materials provided with the 19 | distribution. 20 | * Neither the name of Google Inc. nor the names of its 21 | contributors may be used to endorse or promote products derived from 22 | this software without specific prior written permission. 23 | 24 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 27 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 28 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 29 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 30 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 31 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 32 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 33 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 34 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 | */ 36 | 37 | const deBruijn32 = 0x077CB531 38 | 39 | var deBruijn32Lookup = []byte{ 40 | 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, 41 | 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9, 42 | } 43 | 44 | const deBruijn64 = 0x03f79d71b4ca8b09 45 | 46 | var deBruijn64Lookup = []byte{ 47 | 0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4, 48 | 62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5, 49 | 63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11, 50 | 54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, 51 | } 52 | 53 | // trailingZeroBits returns the number of consecutive least significant zero 54 | // bits of x. 55 | func countTrailingZeros(x uint64) int { 56 | // x & -x leaves only the right-most bit set in the word. Let k be the 57 | // index of that bit. Since only a single bit is set, the value is two 58 | // to the power of k. Multiplying by a power of two is equivalent to 59 | // left shifting, in this case by k bits. The de Bruijn constant is 60 | // such that all six bit, consecutive substrings are distinct. 61 | // Therefore, if we have a left shifted version of this constant we can 62 | // find by how many bits it was shifted by looking at which six bit 63 | // substring ended up at the top of the word. 64 | // (Knuth, volume 4, section 7.3.1) 65 | if x == 0 { 66 | // We have to special case 0; the fomula 67 | // below doesn't work for 0. 68 | return 64 69 | } 70 | return int(deBruijn64Lookup[((x&-x)*(deBruijn64))>>58]) 71 | } 72 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/manyiterator.go: -------------------------------------------------------------------------------- 1 | package roaring 2 | 3 | type manyIterable interface { 4 | nextMany(hs uint32, buf []uint32) int 5 | } 6 | 7 | type manyIterator struct { 8 | slice []uint16 9 | loc int 10 | } 11 | 12 | func (si *manyIterator) nextMany(hs uint32, buf []uint32) int { 13 | n := 0 14 | l := si.loc 15 | s := si.slice 16 | for n < len(buf) && l < len(s) { 17 | buf[n] = uint32(s[l]) | hs 18 | l++ 19 | n++ 20 | } 21 | si.loc = l 22 | return n 23 | } 24 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/popcnt.go: -------------------------------------------------------------------------------- 1 | // +build go1.9 2 | // "go1.9", from Go version 1.9 onward 3 | // See https://golang.org/pkg/go/build/#hdr-Build_Constraints 4 | 5 | package roaring 6 | 7 | import "math/bits" 8 | 9 | func popcount(x uint64) uint64 { 10 | return uint64(bits.OnesCount64(x)) 11 | } 12 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/popcnt_amd64.s: -------------------------------------------------------------------------------- 1 | // +build amd64,!appengine,!go1.9 2 | 3 | TEXT ·hasAsm(SB),4,$0-1 4 | MOVQ $1, AX 5 | CPUID 6 | SHRQ $23, CX 7 | ANDQ $1, CX 8 | MOVB CX, ret+0(FP) 9 | RET 10 | 11 | #define POPCNTQ_DX_DX BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0xd2 12 | 13 | TEXT ·popcntSliceAsm(SB),4,$0-32 14 | XORQ AX, AX 15 | MOVQ s+0(FP), SI 16 | MOVQ s_len+8(FP), CX 17 | TESTQ CX, CX 18 | JZ popcntSliceEnd 19 | popcntSliceLoop: 20 | BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0x16 // POPCNTQ (SI), DX 21 | ADDQ DX, AX 22 | ADDQ $8, SI 23 | LOOP popcntSliceLoop 24 | popcntSliceEnd: 25 | MOVQ AX, ret+24(FP) 26 | RET 27 | 28 | TEXT ·popcntMaskSliceAsm(SB),4,$0-56 29 | XORQ AX, AX 30 | MOVQ s+0(FP), SI 31 | MOVQ s_len+8(FP), CX 32 | TESTQ CX, CX 33 | JZ popcntMaskSliceEnd 34 | MOVQ m+24(FP), DI 35 | popcntMaskSliceLoop: 36 | MOVQ (DI), DX 37 | NOTQ DX 38 | ANDQ (SI), DX 39 | POPCNTQ_DX_DX 40 | ADDQ DX, AX 41 | ADDQ $8, SI 42 | ADDQ $8, DI 43 | LOOP popcntMaskSliceLoop 44 | popcntMaskSliceEnd: 45 | MOVQ AX, ret+48(FP) 46 | RET 47 | 48 | TEXT ·popcntAndSliceAsm(SB),4,$0-56 49 | XORQ AX, AX 50 | MOVQ s+0(FP), SI 51 | MOVQ s_len+8(FP), CX 52 | TESTQ CX, CX 53 | JZ popcntAndSliceEnd 54 | MOVQ m+24(FP), DI 55 | popcntAndSliceLoop: 56 | MOVQ (DI), DX 57 | ANDQ (SI), DX 58 | POPCNTQ_DX_DX 59 | ADDQ DX, AX 60 | ADDQ $8, SI 61 | ADDQ $8, DI 62 | LOOP popcntAndSliceLoop 63 | popcntAndSliceEnd: 64 | MOVQ AX, ret+48(FP) 65 | RET 66 | 67 | TEXT ·popcntOrSliceAsm(SB),4,$0-56 68 | XORQ AX, AX 69 | MOVQ s+0(FP), SI 70 | MOVQ s_len+8(FP), CX 71 | TESTQ CX, CX 72 | JZ popcntOrSliceEnd 73 | MOVQ m+24(FP), DI 74 | popcntOrSliceLoop: 75 | MOVQ (DI), DX 76 | ORQ (SI), DX 77 | POPCNTQ_DX_DX 78 | ADDQ DX, AX 79 | ADDQ $8, SI 80 | ADDQ $8, DI 81 | LOOP popcntOrSliceLoop 82 | popcntOrSliceEnd: 83 | MOVQ AX, ret+48(FP) 84 | RET 85 | 86 | TEXT ·popcntXorSliceAsm(SB),4,$0-56 87 | XORQ AX, AX 88 | MOVQ s+0(FP), SI 89 | MOVQ s_len+8(FP), CX 90 | TESTQ CX, CX 91 | JZ popcntXorSliceEnd 92 | MOVQ m+24(FP), DI 93 | popcntXorSliceLoop: 94 | MOVQ (DI), DX 95 | XORQ (SI), DX 96 | POPCNTQ_DX_DX 97 | ADDQ DX, AX 98 | ADDQ $8, SI 99 | ADDQ $8, DI 100 | LOOP popcntXorSliceLoop 101 | popcntXorSliceEnd: 102 | MOVQ AX, ret+48(FP) 103 | RET 104 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/popcnt_asm.go: -------------------------------------------------------------------------------- 1 | // +build amd64,!appengine,!go1.9 2 | 3 | package roaring 4 | 5 | // *** the following functions are defined in popcnt_amd64.s 6 | 7 | //go:noescape 8 | 9 | func hasAsm() bool 10 | 11 | // useAsm is a flag used to select the GO or ASM implementation of the popcnt function 12 | var useAsm = hasAsm() 13 | 14 | //go:noescape 15 | 16 | func popcntSliceAsm(s []uint64) uint64 17 | 18 | //go:noescape 19 | 20 | func popcntMaskSliceAsm(s, m []uint64) uint64 21 | 22 | //go:noescape 23 | 24 | func popcntAndSliceAsm(s, m []uint64) uint64 25 | 26 | //go:noescape 27 | 28 | func popcntOrSliceAsm(s, m []uint64) uint64 29 | 30 | //go:noescape 31 | 32 | func popcntXorSliceAsm(s, m []uint64) uint64 33 | 34 | func popcntSlice(s []uint64) uint64 { 35 | if useAsm { 36 | return popcntSliceAsm(s) 37 | } 38 | return popcntSliceGo(s) 39 | } 40 | 41 | func popcntMaskSlice(s, m []uint64) uint64 { 42 | if useAsm { 43 | return popcntMaskSliceAsm(s, m) 44 | } 45 | return popcntMaskSliceGo(s, m) 46 | } 47 | 48 | func popcntAndSlice(s, m []uint64) uint64 { 49 | if useAsm { 50 | return popcntAndSliceAsm(s, m) 51 | } 52 | return popcntAndSliceGo(s, m) 53 | } 54 | 55 | func popcntOrSlice(s, m []uint64) uint64 { 56 | if useAsm { 57 | return popcntOrSliceAsm(s, m) 58 | } 59 | return popcntOrSliceGo(s, m) 60 | } 61 | 62 | func popcntXorSlice(s, m []uint64) uint64 { 63 | if useAsm { 64 | return popcntXorSliceAsm(s, m) 65 | } 66 | return popcntXorSliceGo(s, m) 67 | } 68 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/popcnt_compat.go: -------------------------------------------------------------------------------- 1 | // +build !go1.9 2 | 3 | package roaring 4 | 5 | // bit population count, take from 6 | // https://code.google.com/p/go/issues/detail?id=4988#c11 7 | // credit: https://code.google.com/u/arnehormann/ 8 | // credit: https://play.golang.org/p/U7SogJ7psJ 9 | // credit: http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel 10 | func popcount(x uint64) uint64 { 11 | x -= (x >> 1) & 0x5555555555555555 12 | x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 13 | x += x >> 4 14 | x &= 0x0f0f0f0f0f0f0f0f 15 | x *= 0x0101010101010101 16 | return x >> 56 17 | } 18 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/popcnt_generic.go: -------------------------------------------------------------------------------- 1 | // +build !amd64 appengine go1.9 2 | 3 | package roaring 4 | 5 | func popcntSlice(s []uint64) uint64 { 6 | return popcntSliceGo(s) 7 | } 8 | 9 | func popcntMaskSlice(s, m []uint64) uint64 { 10 | return popcntMaskSliceGo(s, m) 11 | } 12 | 13 | func popcntAndSlice(s, m []uint64) uint64 { 14 | return popcntAndSliceGo(s, m) 15 | } 16 | 17 | func popcntOrSlice(s, m []uint64) uint64 { 18 | return popcntOrSliceGo(s, m) 19 | } 20 | 21 | func popcntXorSlice(s, m []uint64) uint64 { 22 | return popcntXorSliceGo(s, m) 23 | } 24 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/popcnt_slices.go: -------------------------------------------------------------------------------- 1 | package roaring 2 | 3 | func popcntSliceGo(s []uint64) uint64 { 4 | cnt := uint64(0) 5 | for _, x := range s { 6 | cnt += popcount(x) 7 | } 8 | return cnt 9 | } 10 | 11 | func popcntMaskSliceGo(s, m []uint64) uint64 { 12 | cnt := uint64(0) 13 | for i := range s { 14 | cnt += popcount(s[i] &^ m[i]) 15 | } 16 | return cnt 17 | } 18 | 19 | func popcntAndSliceGo(s, m []uint64) uint64 { 20 | cnt := uint64(0) 21 | for i := range s { 22 | cnt += popcount(s[i] & m[i]) 23 | } 24 | return cnt 25 | } 26 | 27 | func popcntOrSliceGo(s, m []uint64) uint64 { 28 | cnt := uint64(0) 29 | for i := range s { 30 | cnt += popcount(s[i] | m[i]) 31 | } 32 | return cnt 33 | } 34 | 35 | func popcntXorSliceGo(s, m []uint64) uint64 { 36 | cnt := uint64(0) 37 | for i := range s { 38 | cnt += popcount(s[i] ^ m[i]) 39 | } 40 | return cnt 41 | } 42 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/priorityqueue.go: -------------------------------------------------------------------------------- 1 | package roaring 2 | 3 | import "container/heap" 4 | 5 | ///////////// 6 | // The priorityQueue is used to keep Bitmaps sorted. 7 | //////////// 8 | 9 | type item struct { 10 | value *Bitmap 11 | index int 12 | } 13 | 14 | type priorityQueue []*item 15 | 16 | func (pq priorityQueue) Len() int { return len(pq) } 17 | 18 | func (pq priorityQueue) Less(i, j int) bool { 19 | return pq[i].value.GetSizeInBytes() < pq[j].value.GetSizeInBytes() 20 | } 21 | 22 | func (pq priorityQueue) Swap(i, j int) { 23 | pq[i], pq[j] = pq[j], pq[i] 24 | pq[i].index = i 25 | pq[j].index = j 26 | } 27 | 28 | func (pq *priorityQueue) Push(x interface{}) { 29 | n := len(*pq) 30 | item := x.(*item) 31 | item.index = n 32 | *pq = append(*pq, item) 33 | } 34 | 35 | func (pq *priorityQueue) Pop() interface{} { 36 | old := *pq 37 | n := len(old) 38 | item := old[n-1] 39 | item.index = -1 // for safety 40 | *pq = old[0 : n-1] 41 | return item 42 | } 43 | 44 | func (pq *priorityQueue) update(item *item, value *Bitmap) { 45 | item.value = value 46 | heap.Fix(pq, item.index) 47 | } 48 | 49 | ///////////// 50 | // The containerPriorityQueue is used to keep the containers of various Bitmaps sorted. 51 | //////////// 52 | 53 | type containeritem struct { 54 | value *Bitmap 55 | keyindex int 56 | index int 57 | } 58 | 59 | type containerPriorityQueue []*containeritem 60 | 61 | func (pq containerPriorityQueue) Len() int { return len(pq) } 62 | 63 | func (pq containerPriorityQueue) Less(i, j int) bool { 64 | k1 := pq[i].value.highlowcontainer.getKeyAtIndex(pq[i].keyindex) 65 | k2 := pq[j].value.highlowcontainer.getKeyAtIndex(pq[j].keyindex) 66 | if k1 != k2 { 67 | return k1 < k2 68 | } 69 | c1 := pq[i].value.highlowcontainer.getContainerAtIndex(pq[i].keyindex) 70 | c2 := pq[j].value.highlowcontainer.getContainerAtIndex(pq[j].keyindex) 71 | 72 | return c1.getCardinality() > c2.getCardinality() 73 | } 74 | 75 | func (pq containerPriorityQueue) Swap(i, j int) { 76 | pq[i], pq[j] = pq[j], pq[i] 77 | pq[i].index = i 78 | pq[j].index = j 79 | } 80 | 81 | func (pq *containerPriorityQueue) Push(x interface{}) { 82 | n := len(*pq) 83 | item := x.(*containeritem) 84 | item.index = n 85 | *pq = append(*pq, item) 86 | } 87 | 88 | func (pq *containerPriorityQueue) Pop() interface{} { 89 | old := *pq 90 | n := len(old) 91 | item := old[n-1] 92 | item.index = -1 // for safety 93 | *pq = old[0 : n-1] 94 | return item 95 | } 96 | 97 | //func (pq *containerPriorityQueue) update(item *containeritem, value *Bitmap, keyindex int) { 98 | // item.value = value 99 | // item.keyindex = keyindex 100 | // heap.Fix(pq, item.index) 101 | //} 102 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/serialization.go: -------------------------------------------------------------------------------- 1 | package roaring 2 | 3 | import ( 4 | "encoding/binary" 5 | "errors" 6 | "fmt" 7 | "io" 8 | 9 | "github.com/tinylib/msgp/msgp" 10 | ) 11 | 12 | // writeTo for runContainer16 follows this 13 | // spec: https://github.com/RoaringBitmap/RoaringFormatSpec 14 | // 15 | func (b *runContainer16) writeTo(stream io.Writer) (int, error) { 16 | buf := make([]byte, 2+4*len(b.iv)) 17 | binary.LittleEndian.PutUint16(buf[0:], uint16(len(b.iv))) 18 | for i, v := range b.iv { 19 | binary.LittleEndian.PutUint16(buf[2+i*4:], v.start) 20 | binary.LittleEndian.PutUint16(buf[2+2+i*4:], v.length) 21 | } 22 | return stream.Write(buf) 23 | } 24 | 25 | func (b *runContainer16) writeToMsgpack(stream io.Writer) (int, error) { 26 | bts, err := b.MarshalMsg(nil) 27 | if err != nil { 28 | return 0, err 29 | } 30 | return stream.Write(bts) 31 | } 32 | 33 | func (b *runContainer16) readFromMsgpack(stream io.Reader) (int, error) { 34 | err := msgp.Decode(stream, b) 35 | return 0, err 36 | } 37 | 38 | var errCorruptedStream = errors.New("insufficient/odd number of stored bytes, corrupted stream detected") 39 | 40 | func (b *runContainer16) readFrom(stream io.Reader) (int, error) { 41 | b.iv = b.iv[:0] 42 | b.card = 0 43 | var numRuns uint16 44 | err := binary.Read(stream, binary.LittleEndian, &numRuns) 45 | if err != nil { 46 | return 0, err 47 | } 48 | nr := int(numRuns) 49 | encRun := make([]uint16, 2*nr) 50 | by := make([]byte, 4*nr) 51 | err = binary.Read(stream, binary.LittleEndian, &by) 52 | if err != nil { 53 | return 0, err 54 | } 55 | for i := range encRun { 56 | if len(by) < 2 { 57 | return 0, errCorruptedStream 58 | } 59 | encRun[i] = binary.LittleEndian.Uint16(by) 60 | by = by[2:] 61 | } 62 | for i := 0; i < nr; i++ { 63 | if i > 0 && b.iv[i-1].last() >= encRun[i*2] { 64 | return 0, fmt.Errorf("error: stored runContainer had runs that were not in sorted order!! (b.iv[i-1=%v].last = %v >= encRun[i=%v] = %v)", i-1, b.iv[i-1].last(), i, encRun[i*2]) 65 | } 66 | b.iv = append(b.iv, interval16{start: encRun[i*2], length: encRun[i*2+1]}) 67 | b.card += int64(encRun[i*2+1]) + 1 68 | } 69 | return 0, err 70 | } 71 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/serialization_generic.go: -------------------------------------------------------------------------------- 1 | // +build !amd64,!386 appengine 2 | 3 | package roaring 4 | 5 | import ( 6 | "encoding/binary" 7 | "errors" 8 | "io" 9 | ) 10 | 11 | func (b *arrayContainer) writeTo(stream io.Writer) (int, error) { 12 | buf := make([]byte, 2*len(b.content)) 13 | for i, v := range b.content { 14 | base := i * 2 15 | buf[base] = byte(v) 16 | buf[base+1] = byte(v >> 8) 17 | } 18 | return stream.Write(buf) 19 | } 20 | 21 | func (b *arrayContainer) readFrom(stream io.Reader) (int, error) { 22 | err := binary.Read(stream, binary.LittleEndian, b.content) 23 | if err != nil { 24 | return 0, err 25 | } 26 | return 2 * len(b.content), nil 27 | } 28 | 29 | func (b *bitmapContainer) writeTo(stream io.Writer) (int, error) { 30 | if b.cardinality <= arrayDefaultMaxSize { 31 | return 0, errors.New("refusing to write bitmap container with cardinality of array container") 32 | } 33 | 34 | // Write set 35 | buf := make([]byte, 8*len(b.bitmap)) 36 | for i, v := range b.bitmap { 37 | base := i * 8 38 | buf[base] = byte(v) 39 | buf[base+1] = byte(v >> 8) 40 | buf[base+2] = byte(v >> 16) 41 | buf[base+3] = byte(v >> 24) 42 | buf[base+4] = byte(v >> 32) 43 | buf[base+5] = byte(v >> 40) 44 | buf[base+6] = byte(v >> 48) 45 | buf[base+7] = byte(v >> 56) 46 | } 47 | return stream.Write(buf) 48 | } 49 | 50 | func (b *bitmapContainer) readFrom(stream io.Reader) (int, error) { 51 | err := binary.Read(stream, binary.LittleEndian, b.bitmap) 52 | if err != nil { 53 | return 0, err 54 | } 55 | b.computeCardinality() 56 | return 8 * len(b.bitmap), nil 57 | } 58 | 59 | func (bc *bitmapContainer) asLittleEndianByteSlice() []byte { 60 | by := make([]byte, len(bc.bitmap)*8) 61 | for i := range bc.bitmap { 62 | binary.LittleEndian.PutUint64(by[i*8:], bc.bitmap[i]) 63 | } 64 | return by 65 | } 66 | 67 | func uint64SliceAsByteSlice(slice []uint64) []byte { 68 | by := make([]byte, len(slice)*8) 69 | 70 | for i, v := range slice { 71 | binary.LittleEndian.PutUint64(by[i*8:], v) 72 | } 73 | 74 | return by 75 | } 76 | 77 | func byteSliceAsUint16Slice(slice []byte) []uint16 { 78 | if len(slice)%2 != 0 { 79 | panic("Slice size should be divisible by 2") 80 | } 81 | 82 | b := make([]uint16, len(slice)/2) 83 | 84 | for i := range b { 85 | b[i] = binary.LittleEndian.Uint16(slice[2*i:]) 86 | } 87 | 88 | return b 89 | } 90 | 91 | func byteSliceAsUint64Slice(slice []byte) []uint64 { 92 | if len(slice)%8 != 0 { 93 | panic("Slice size should be divisible by 8") 94 | } 95 | 96 | b := make([]uint64, len(slice)/8) 97 | 98 | for i := range b { 99 | b[i] = binary.LittleEndian.Uint64(slice[8*i:]) 100 | } 101 | 102 | return b 103 | } 104 | 105 | // Converts a byte slice to a interval16 slice. 106 | // The function assumes that the slice byte buffer is run container data 107 | // encoded according to Roaring Format Spec 108 | func byteSliceAsInterval16Slice(byteSlice []byte) []interval16 { 109 | if len(byteSlice)%4 != 0 { 110 | panic("Slice size should be divisible by 4") 111 | } 112 | 113 | intervalSlice := make([]interval16, len(byteSlice)/4) 114 | 115 | for i := range intervalSlice { 116 | intervalSlice[i] = interval16{ 117 | start: binary.LittleEndian.Uint16(byteSlice[i*4:]), 118 | length: binary.LittleEndian.Uint16(byteSlice[i*4+2:]), 119 | } 120 | } 121 | 122 | return intervalSlice 123 | } 124 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/serialization_littleendian.go: -------------------------------------------------------------------------------- 1 | // +build 386 amd64,!appengine 2 | 3 | package roaring 4 | 5 | import ( 6 | "errors" 7 | "io" 8 | "reflect" 9 | "unsafe" 10 | ) 11 | 12 | func (ac *arrayContainer) writeTo(stream io.Writer) (int, error) { 13 | buf := uint16SliceAsByteSlice(ac.content) 14 | return stream.Write(buf) 15 | } 16 | 17 | func (bc *bitmapContainer) writeTo(stream io.Writer) (int, error) { 18 | if bc.cardinality <= arrayDefaultMaxSize { 19 | return 0, errors.New("refusing to write bitmap container with cardinality of array container") 20 | } 21 | buf := uint64SliceAsByteSlice(bc.bitmap) 22 | return stream.Write(buf) 23 | } 24 | 25 | // readFrom reads an arrayContainer from stream. 26 | // PRE-REQUISITE: you must size the arrayContainer correctly (allocate b.content) 27 | // *before* you call readFrom. We can't guess the size in the stream 28 | // by this point. 29 | func (ac *arrayContainer) readFrom(stream io.Reader) (int, error) { 30 | buf := uint16SliceAsByteSlice(ac.content) 31 | return io.ReadFull(stream, buf) 32 | } 33 | 34 | func (bc *bitmapContainer) readFrom(stream io.Reader) (int, error) { 35 | buf := uint64SliceAsByteSlice(bc.bitmap) 36 | n, err := io.ReadFull(stream, buf) 37 | bc.computeCardinality() 38 | return n, err 39 | } 40 | 41 | func uint64SliceAsByteSlice(slice []uint64) []byte { 42 | // make a new slice header 43 | header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) 44 | 45 | // update its capacity and length 46 | header.Len *= 8 47 | header.Cap *= 8 48 | 49 | // return it 50 | return *(*[]byte)(unsafe.Pointer(&header)) 51 | } 52 | 53 | func uint16SliceAsByteSlice(slice []uint16) []byte { 54 | // make a new slice header 55 | header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) 56 | 57 | // update its capacity and length 58 | header.Len *= 2 59 | header.Cap *= 2 60 | 61 | // return it 62 | return *(*[]byte)(unsafe.Pointer(&header)) 63 | } 64 | 65 | func (bc *bitmapContainer) asLittleEndianByteSlice() []byte { 66 | return uint64SliceAsByteSlice(bc.bitmap) 67 | } 68 | 69 | // Deserialization code follows 70 | 71 | func byteSliceAsUint16Slice(slice []byte) []uint16 { 72 | if len(slice)%2 != 0 { 73 | panic("Slice size should be divisible by 2") 74 | } 75 | 76 | // make a new slice header 77 | header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) 78 | 79 | // update its capacity and length 80 | header.Len /= 2 81 | header.Cap /= 2 82 | 83 | // return it 84 | return *(*[]uint16)(unsafe.Pointer(&header)) 85 | } 86 | 87 | func byteSliceAsUint64Slice(slice []byte) []uint64 { 88 | if len(slice)%8 != 0 { 89 | panic("Slice size should be divisible by 8") 90 | } 91 | 92 | // make a new slice header 93 | header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) 94 | 95 | // update its capacity and length 96 | header.Len /= 8 97 | header.Cap /= 8 98 | 99 | // return it 100 | return *(*[]uint64)(unsafe.Pointer(&header)) 101 | } 102 | 103 | func byteSliceAsInterval16Slice(slice []byte) []interval16 { 104 | if len(slice)%4 != 0 { 105 | panic("Slice size should be divisible by 4") 106 | } 107 | 108 | // make a new slice header 109 | header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice)) 110 | 111 | // update its capacity and length 112 | header.Len /= 4 113 | header.Cap /= 4 114 | 115 | // return it 116 | return *(*[]interval16)(unsafe.Pointer(&header)) 117 | } 118 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/serializationfuzz.go: -------------------------------------------------------------------------------- 1 | // +build gofuzz 2 | 3 | package roaring 4 | 5 | import "bytes" 6 | 7 | func FuzzSerializationStream(data []byte) int { 8 | newrb := NewBitmap() 9 | if _, err := newrb.ReadFrom(bytes.NewReader(data)); err != nil { 10 | return 0 11 | } 12 | return 1 13 | } 14 | 15 | func FuzzSerializationBuffer(data []byte) int { 16 | newrb := NewBitmap() 17 | if _, err := newrb.FromBuffer(data); err != nil { 18 | return 0 19 | } 20 | return 1 21 | } 22 | -------------------------------------------------------------------------------- /vendor/github.com/RoaringBitmap/roaring/shortiterator.go: -------------------------------------------------------------------------------- 1 | package roaring 2 | 3 | type shortIterable interface { 4 | hasNext() bool 5 | next() uint16 6 | } 7 | 8 | type shortIterator struct { 9 | slice []uint16 10 | loc int 11 | } 12 | 13 | func (si *shortIterator) hasNext() bool { 14 | return si.loc < len(si.slice) 15 | } 16 | 17 | func (si *shortIterator) next() uint16 { 18 | a := si.slice[si.loc] 19 | si.loc++ 20 | return a 21 | } 22 | 23 | type reverseIterator struct { 24 | slice []uint16 25 | loc int 26 | } 27 | 28 | func (si *reverseIterator) hasNext() bool { 29 | return si.loc >= 0 30 | } 31 | 32 | func (si *reverseIterator) next() uint16 { 33 | a := si.slice[si.loc] 34 | si.loc-- 35 | return a 36 | } 37 | -------------------------------------------------------------------------------- /vendor/github.com/buaazp/fasthttprouter/.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | 3 | coverage.out 4 | examples/basic/basic 5 | examples/hosts/hosts 6 | examples/auth/auth 7 | -------------------------------------------------------------------------------- /vendor/github.com/buaazp/fasthttprouter/.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: go 3 | 4 | go: 5 | - 1.5 6 | - 1.6 7 | - 1.7 8 | - tip 9 | 10 | before_install: 11 | - go get -v github.com/axw/gocov/gocov 12 | - go get -v github.com/mattn/goveralls 13 | # - go get -v github.com/golang/lint/golint 14 | 15 | install: 16 | - go get -d -t -v ./... 17 | - go install -v 18 | 19 | script: 20 | - go vet ./... 21 | # - $HOME/gopath/bin/golint ./... 22 | - go test -v -covermode=count -coverprofile=coverage.out 23 | - $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci 24 | 25 | -after_success: 26 | - coveralls -------------------------------------------------------------------------------- /vendor/github.com/buaazp/fasthttprouter/HttpRouterLicense: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 Julien Schmidt. All rights reserved. 2 | 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | * The names of the contributors may not be used to endorse or promote 12 | products derived from this software without specific prior written 13 | permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL JULIEN SCHMIDT BE LIABLE FOR ANY 19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /vendor/github.com/buaazp/fasthttprouter/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015-2016, 招牌疯子 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of uq nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | -------------------------------------------------------------------------------- /vendor/github.com/buaazp/fasthttprouter/path.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Julien Schmidt. All rights reserved. 2 | // Based on the path package, Copyright 2009 The Go Authors. 3 | // Use of this source code is governed by a BSD-style license that can be found 4 | // in the LICENSE file. 5 | 6 | package fasthttprouter 7 | 8 | // CleanPath is the URL version of path.Clean, it returns a canonical URL path 9 | // for p, eliminating . and .. elements. 10 | // 11 | // The following rules are applied iteratively until no further processing can 12 | // be done: 13 | // 1. Replace multiple slashes with a single slash. 14 | // 2. Eliminate each . path name element (the current directory). 15 | // 3. Eliminate each inner .. path name element (the parent directory) 16 | // along with the non-.. element that precedes it. 17 | // 4. Eliminate .. elements that begin a rooted path: 18 | // that is, replace "/.." by "/" at the beginning of a path. 19 | // 20 | // If the result of this process is an empty string, "/" is returned 21 | func CleanPath(p string) string { 22 | // Turn empty string into "/" 23 | if p == "" { 24 | return "/" 25 | } 26 | 27 | n := len(p) 28 | var buf []byte 29 | 30 | // Invariants: 31 | // reading from path; r is index of next byte to process. 32 | // writing to buf; w is index of next byte to write. 33 | 34 | // path must start with '/' 35 | r := 1 36 | w := 1 37 | 38 | if p[0] != '/' { 39 | r = 0 40 | buf = make([]byte, n+1) 41 | buf[0] = '/' 42 | } 43 | 44 | trailing := n > 2 && p[n-1] == '/' 45 | 46 | // A bit more clunky without a 'lazybuf' like the path package, but the loop 47 | // gets completely inlined (bufApp). So in contrast to the path package this 48 | // loop has no expensive function calls (except 1x make) 49 | 50 | for r < n { 51 | switch { 52 | case p[r] == '/': 53 | // empty path element, trailing slash is added after the end 54 | r++ 55 | 56 | case p[r] == '.' && r+1 == n: 57 | trailing = true 58 | r++ 59 | 60 | case p[r] == '.' && p[r+1] == '/': 61 | // . element 62 | r++ 63 | 64 | case p[r] == '.' && p[r+1] == '.' && (r+2 == n || p[r+2] == '/'): 65 | // .. element: remove to last / 66 | r += 2 67 | 68 | if w > 1 { 69 | // can backtrack 70 | w-- 71 | 72 | if buf == nil { 73 | for w > 1 && p[w] != '/' { 74 | w-- 75 | } 76 | } else { 77 | for w > 1 && buf[w] != '/' { 78 | w-- 79 | } 80 | } 81 | } 82 | 83 | default: 84 | // real path element. 85 | // add slash if needed 86 | if w > 1 { 87 | bufApp(&buf, p, w, '/') 88 | w++ 89 | } 90 | 91 | // copy element 92 | for r < n && p[r] != '/' { 93 | bufApp(&buf, p, w, p[r]) 94 | w++ 95 | r++ 96 | } 97 | } 98 | } 99 | 100 | // re-append trailing slash 101 | if trailing && w > 1 { 102 | bufApp(&buf, p, w, '/') 103 | w++ 104 | } 105 | 106 | if buf == nil { 107 | return p[:w] 108 | } 109 | return string(buf[:w]) 110 | } 111 | 112 | // internal helper to lazily create a buffer if necessary 113 | func bufApp(buf *[]byte, s string, w int, c byte) { 114 | if *buf == nil { 115 | if s[w] == c { 116 | return 117 | } 118 | 119 | *buf = make([]byte, len(s)) 120 | copy(*buf, s[:w]) 121 | } 122 | (*buf)[w] = c 123 | } 124 | -------------------------------------------------------------------------------- /vendor/github.com/glycerine/go-unsnap-stream/.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | -------------------------------------------------------------------------------- /vendor/github.com/glycerine/go-unsnap-stream/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT license. 2 | 3 | Copyright (c) 2014 the go-unsnap-stream authors. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | 22 | -------------------------------------------------------------------------------- /vendor/github.com/glycerine/go-unsnap-stream/README.md: -------------------------------------------------------------------------------- 1 | go-unsnap-stream 2 | ================ 3 | 4 | This is a small golang library for decoding and encoding the snappy *streaming* format, specified here: https://github.com/google/snappy/blob/master/framing_format.txt 5 | 6 | Note that the *streaming or framing format* for snappy is different from snappy itself. Think of it as a train of boxcars: the streaming format breaks your data in chunks, applies snappy to each chunk alone, then puts a thin wrapper around the chunk, and sends it along in turn. You can begin decoding before receiving everything. And memory requirements for decoding are sane. 7 | 8 | Strangely, though the streaming format was first proposed in Go[1][2], it was never upated, and I could not locate any other library for Go that would handle the streaming/framed snappy format. Hence this implementation of the spec. There is a command line tool[3] that has a C implementation, but this is the only Go implementation that I am aware of. The reference for the framing/streaming spec seems to be the python implementation[4]. 9 | 10 | For binary compatibility with the python implementation, one could use the C-snappy compressor/decompressor code directly; using github.com/dgryski/go-csnappy. In fact we did this for a while to verify byte-for-byte compatiblity, as the native Go implementation produces slightly different binary compression (still conformant with the standard of course), which made test-diffs harder, and some have complained about it being slower than the C. 11 | 12 | However, while the c-snappy was useful for checking compatibility, it introduced dependencies on external C libraries (both the c-snappy library and the C standard library). Our go binary executable that used the go-unsnap-stream library was no longer standalone, and deployment was painful if not impossible if the target had a different C standard library. So we've gone back to using the snappy-go implementation (entirely in Go) for ease of deployment. See the comments at the top of unsnap.go if you wish to use c-snappy instead. 13 | 14 | [1] https://groups.google.com/forum/#!msg/snappy-compression/qvLNe2cSH9s/R19oBC-p7g4J 15 | 16 | [2] https://codereview.appspot.com/5167058 17 | 18 | [3] https://github.com/kubo/snzip 19 | 20 | [4] https://pypi.python.org/pypi/python-snappy -------------------------------------------------------------------------------- /vendor/github.com/glycerine/go-unsnap-stream/binary.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/vendor/github.com/glycerine/go-unsnap-stream/binary.dat -------------------------------------------------------------------------------- /vendor/github.com/glycerine/go-unsnap-stream/binary.dat.snappy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/vendor/github.com/glycerine/go-unsnap-stream/binary.dat.snappy -------------------------------------------------------------------------------- /vendor/github.com/glycerine/go-unsnap-stream/snap.go: -------------------------------------------------------------------------------- 1 | package unsnap 2 | 3 | import ( 4 | "encoding/binary" 5 | 6 | // no c lib dependency 7 | snappy "github.com/golang/snappy" 8 | // or, use the C wrapper for speed 9 | //snappy "github.com/dgryski/go-csnappy" 10 | ) 11 | 12 | // add Write() method for SnappyFile (see unsnap.go) 13 | 14 | // reference for snappy framing/streaming format: 15 | // http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt 16 | // ?spec=svn68&r=71 17 | 18 | // 19 | // Write writes len(p) bytes from p to the underlying data stream. 20 | // It returns the number of bytes written from p (0 <= n <= len(p)) and 21 | // any error encountered that caused the write to stop early. Write 22 | // must return a non-nil error if it returns n < len(p). 23 | // 24 | func (sf *SnappyFile) Write(p []byte) (n int, err error) { 25 | 26 | if sf.SnappyEncodeDecodeOff { 27 | return sf.Writer.Write(p) 28 | } 29 | 30 | if !sf.Writing { 31 | panic("Writing on a read-only SnappyFile") 32 | } 33 | 34 | // encoding in snappy can apparently go beyond the original size, beware. 35 | // so our buffers must be sized 2*max snappy chunk => 2 * CHUNK_MAX(65536) 36 | 37 | sf.DecBuf.Reset() 38 | sf.EncBuf.Reset() 39 | 40 | if !sf.HeaderChunkWritten { 41 | sf.HeaderChunkWritten = true 42 | _, err = sf.Writer.Write(SnappyStreamHeaderMagic) 43 | if err != nil { 44 | return 45 | } 46 | } 47 | var chunk []byte 48 | var chunk_type byte 49 | var crc uint32 50 | 51 | for len(p) > 0 { 52 | 53 | // chunk points to input p by default, unencoded input. 54 | chunk = p[:IntMin(len(p), CHUNK_MAX)] 55 | crc = masked_crc32c(chunk) 56 | 57 | writeme := chunk[:] 58 | 59 | // first write to EncBuf, as a temp, in case we want 60 | // to discard and send uncompressed instead. 61 | compressed_chunk := snappy.Encode(sf.EncBuf.GetEndmostWritableSlice(), chunk) 62 | 63 | if len(compressed_chunk) <= int((1-_COMPRESSION_THRESHOLD)*float64(len(chunk))) { 64 | writeme = compressed_chunk 65 | chunk_type = _COMPRESSED_CHUNK 66 | } else { 67 | // keep writeme pointing at original chunk (uncompressed) 68 | chunk_type = _UNCOMPRESSED_CHUNK 69 | } 70 | 71 | const crc32Sz = 4 72 | var tag32 uint32 = uint32(chunk_type) + (uint32(len(writeme)+crc32Sz) << 8) 73 | 74 | err = binary.Write(sf.Writer, binary.LittleEndian, tag32) 75 | if err != nil { 76 | return 77 | } 78 | 79 | err = binary.Write(sf.Writer, binary.LittleEndian, crc) 80 | if err != nil { 81 | return 82 | } 83 | 84 | _, err = sf.Writer.Write(writeme) 85 | if err != nil { 86 | return 87 | } 88 | 89 | n += len(chunk) 90 | p = p[len(chunk):] 91 | } 92 | return n, nil 93 | } 94 | 95 | func IntMin(a int, b int) int { 96 | if a < b { 97 | return a 98 | } 99 | return b 100 | } 101 | -------------------------------------------------------------------------------- /vendor/github.com/glycerine/go-unsnap-stream/unenc.txt: -------------------------------------------------------------------------------- 1 | hello_snappy 2 | -------------------------------------------------------------------------------- /vendor/github.com/glycerine/go-unsnap-stream/unenc.txt.snappy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/covrom/highloadcup2018/e87183ecd952a7405eb76af4a3831883e053672c/vendor/github.com/glycerine/go-unsnap-stream/unenc.txt.snappy -------------------------------------------------------------------------------- /vendor/github.com/golang/snappy/.gitignore: -------------------------------------------------------------------------------- 1 | cmd/snappytool/snappytool 2 | testdata/bench 3 | 4 | # These explicitly listed benchmark data files are for an obsolete version of 5 | # snappy_test.go. 6 | testdata/alice29.txt 7 | testdata/asyoulik.txt 8 | testdata/fireworks.jpeg 9 | testdata/geo.protodata 10 | testdata/html 11 | testdata/html_x_4 12 | testdata/kppkn.gtb 13 | testdata/lcet10.txt 14 | testdata/paper-100k.pdf 15 | testdata/plrabn12.txt 16 | testdata/urls.10K 17 | -------------------------------------------------------------------------------- /vendor/github.com/golang/snappy/AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the official list of Snappy-Go authors for copyright purposes. 2 | # This file is distinct from the CONTRIBUTORS files. 3 | # See the latter for an explanation. 4 | 5 | # Names should be added to this file as 6 | # Name or Organization 7 | # The email address is not required for organizations. 8 | 9 | # Please keep the list sorted. 10 | 11 | Damian Gryski 12 | Google Inc. 13 | Jan Mercl <0xjnml@gmail.com> 14 | Rodolfo Carvalho 15 | Sebastien Binet 16 | -------------------------------------------------------------------------------- /vendor/github.com/golang/snappy/CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | # This is the official list of people who can contribute 2 | # (and typically have contributed) code to the Snappy-Go repository. 3 | # The AUTHORS file lists the copyright holders; this file 4 | # lists people. For example, Google employees are listed here 5 | # but not in AUTHORS, because Google holds the copyright. 6 | # 7 | # The submission process automatically checks to make sure 8 | # that people submitting code are listed in this file (by email address). 9 | # 10 | # Names should be added to this file only after verifying that 11 | # the individual or the individual's organization has agreed to 12 | # the appropriate Contributor License Agreement, found here: 13 | # 14 | # http://code.google.com/legal/individual-cla-v1.0.html 15 | # http://code.google.com/legal/corporate-cla-v1.0.html 16 | # 17 | # The agreement for individuals can be filled out on the web. 18 | # 19 | # When adding J Random Contributor's name to this file, 20 | # either J's name or J's organization's name should be 21 | # added to the AUTHORS file, depending on whether the 22 | # individual or corporate CLA was used. 23 | 24 | # Names should be added to this file like so: 25 | # Name 26 | 27 | # Please keep the list sorted. 28 | 29 | Damian Gryski 30 | Jan Mercl <0xjnml@gmail.com> 31 | Kai Backman 32 | Marc-Antoine Ruel 33 | Nigel Tao 34 | Rob Pike 35 | Rodolfo Carvalho 36 | Russ Cox 37 | Sebastien Binet 38 | -------------------------------------------------------------------------------- /vendor/github.com/golang/snappy/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /vendor/github.com/golang/snappy/decode_amd64.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Snappy-Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build !appengine 6 | // +build gc 7 | // +build !noasm 8 | 9 | package snappy 10 | 11 | // decode has the same semantics as in decode_other.go. 12 | // 13 | //go:noescape 14 | func decode(dst, src []byte) int 15 | -------------------------------------------------------------------------------- /vendor/github.com/golang/snappy/decode_other.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Snappy-Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build !amd64 appengine !gc noasm 6 | 7 | package snappy 8 | 9 | // decode writes the decoding of src to dst. It assumes that the varint-encoded 10 | // length of the decompressed bytes has already been read, and that len(dst) 11 | // equals that length. 12 | // 13 | // It returns 0 on success or a decodeErrCodeXxx error code on failure. 14 | func decode(dst, src []byte) int { 15 | var d, s, offset, length int 16 | for s < len(src) { 17 | switch src[s] & 0x03 { 18 | case tagLiteral: 19 | x := uint32(src[s] >> 2) 20 | switch { 21 | case x < 60: 22 | s++ 23 | case x == 60: 24 | s += 2 25 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. 26 | return decodeErrCodeCorrupt 27 | } 28 | x = uint32(src[s-1]) 29 | case x == 61: 30 | s += 3 31 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. 32 | return decodeErrCodeCorrupt 33 | } 34 | x = uint32(src[s-2]) | uint32(src[s-1])<<8 35 | case x == 62: 36 | s += 4 37 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. 38 | return decodeErrCodeCorrupt 39 | } 40 | x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 41 | case x == 63: 42 | s += 5 43 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. 44 | return decodeErrCodeCorrupt 45 | } 46 | x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 47 | } 48 | length = int(x) + 1 49 | if length <= 0 { 50 | return decodeErrCodeUnsupportedLiteralLength 51 | } 52 | if length > len(dst)-d || length > len(src)-s { 53 | return decodeErrCodeCorrupt 54 | } 55 | copy(dst[d:], src[s:s+length]) 56 | d += length 57 | s += length 58 | continue 59 | 60 | case tagCopy1: 61 | s += 2 62 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. 63 | return decodeErrCodeCorrupt 64 | } 65 | length = 4 + int(src[s-2])>>2&0x7 66 | offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) 67 | 68 | case tagCopy2: 69 | s += 3 70 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. 71 | return decodeErrCodeCorrupt 72 | } 73 | length = 1 + int(src[s-3])>>2 74 | offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8) 75 | 76 | case tagCopy4: 77 | s += 5 78 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. 79 | return decodeErrCodeCorrupt 80 | } 81 | length = 1 + int(src[s-5])>>2 82 | offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24) 83 | } 84 | 85 | if offset <= 0 || d < offset || length > len(dst)-d { 86 | return decodeErrCodeCorrupt 87 | } 88 | // Copy from an earlier sub-slice of dst to a later sub-slice. Unlike 89 | // the built-in copy function, this byte-by-byte copy always runs 90 | // forwards, even if the slices overlap. Conceptually, this is: 91 | // 92 | // d += forwardCopy(dst[d:d+length], dst[d-offset:]) 93 | for end := d + length; d != end; d++ { 94 | dst[d] = dst[d-offset] 95 | } 96 | } 97 | if d != len(dst) { 98 | return decodeErrCodeCorrupt 99 | } 100 | return 0 101 | } 102 | -------------------------------------------------------------------------------- /vendor/github.com/golang/snappy/encode_amd64.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Snappy-Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // +build !appengine 6 | // +build gc 7 | // +build !noasm 8 | 9 | package snappy 10 | 11 | // emitLiteral has the same semantics as in encode_other.go. 12 | // 13 | //go:noescape 14 | func emitLiteral(dst, lit []byte) int 15 | 16 | // emitCopy has the same semantics as in encode_other.go. 17 | // 18 | //go:noescape 19 | func emitCopy(dst []byte, offset, length int) int 20 | 21 | // extendMatch has the same semantics as in encode_other.go. 22 | // 23 | //go:noescape 24 | func extendMatch(src []byte, i, j int) int 25 | 26 | // encodeBlock has the same semantics as in encode_other.go. 27 | // 28 | //go:noescape 29 | func encodeBlock(dst, src []byte) (d int) 30 | -------------------------------------------------------------------------------- /vendor/github.com/klauspost/compress/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 The Go Authors. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /vendor/github.com/klauspost/compress/flate/copy.go: -------------------------------------------------------------------------------- 1 | // Copyright 2012 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package flate 6 | 7 | // forwardCopy is like the built-in copy function except that it always goes 8 | // forward from the start, even if the dst and src overlap. 9 | // It is equivalent to: 10 | // for i := 0; i < n; i++ { 11 | // mem[dst+i] = mem[src+i] 12 | // } 13 | func forwardCopy(mem []byte, dst, src, n int) { 14 | if dst <= src { 15 | copy(mem[dst:dst+n], mem[src:src+n]) 16 | return 17 | } 18 | for { 19 | if dst >= src+n { 20 | copy(mem[dst:dst+n], mem[src:src+n]) 21 | return 22 | } 23 | // There is some forward overlap. The destination 24 | // will be filled with a repeated pattern of mem[src:src+k]. 25 | // We copy one instance of the pattern here, then repeat. 26 | // Each time around this loop k will double. 27 | k := dst - src 28 | copy(mem[dst:dst+k], mem[src:src+k]) 29 | n -= k 30 | dst += k 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /vendor/github.com/klauspost/compress/flate/crc32_amd64.go: -------------------------------------------------------------------------------- 1 | //+build !noasm 2 | //+build !appengine 3 | //+build !gccgo 4 | 5 | // Copyright 2015, Klaus Post, see LICENSE for details. 6 | 7 | package flate 8 | 9 | import ( 10 | "github.com/klauspost/cpuid" 11 | ) 12 | 13 | // crc32sse returns a hash for the first 4 bytes of the slice 14 | // len(a) must be >= 4. 15 | //go:noescape 16 | func crc32sse(a []byte) uint32 17 | 18 | // crc32sseAll calculates hashes for each 4-byte set in a. 19 | // dst must be east len(a) - 4 in size. 20 | // The size is not checked by the assembly. 21 | //go:noescape 22 | func crc32sseAll(a []byte, dst []uint32) 23 | 24 | // matchLenSSE4 returns the number of matching bytes in a and b 25 | // up to length 'max'. Both slices must be at least 'max' 26 | // bytes in size. 27 | // 28 | // TODO: drop the "SSE4" name, since it doesn't use any SSE instructions. 29 | // 30 | //go:noescape 31 | func matchLenSSE4(a, b []byte, max int) int 32 | 33 | // histogram accumulates a histogram of b in h. 34 | // h must be at least 256 entries in length, 35 | // and must be cleared before calling this function. 36 | //go:noescape 37 | func histogram(b []byte, h []int32) 38 | 39 | // Detect SSE 4.2 feature. 40 | func init() { 41 | useSSE42 = cpuid.CPU.SSE42() 42 | } 43 | -------------------------------------------------------------------------------- /vendor/github.com/klauspost/compress/flate/crc32_noasm.go: -------------------------------------------------------------------------------- 1 | //+build !amd64 noasm appengine gccgo 2 | 3 | // Copyright 2015, Klaus Post, see LICENSE for details. 4 | 5 | package flate 6 | 7 | func init() { 8 | useSSE42 = false 9 | } 10 | 11 | // crc32sse should never be called. 12 | func crc32sse(a []byte) uint32 { 13 | panic("no assembler") 14 | } 15 | 16 | // crc32sseAll should never be called. 17 | func crc32sseAll(a []byte, dst []uint32) { 18 | panic("no assembler") 19 | } 20 | 21 | // matchLenSSE4 should never be called. 22 | func matchLenSSE4(a, b []byte, max int) int { 23 | panic("no assembler") 24 | return 0 25 | } 26 | 27 | // histogram accumulates a histogram of b in h. 28 | // 29 | // len(h) must be >= 256, and h's elements must be all zeroes. 30 | func histogram(b []byte, h []int32) { 31 | h = h[:256] 32 | for _, t := range b { 33 | h[t]++ 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /vendor/github.com/klauspost/compress/flate/reverse_bits.go: -------------------------------------------------------------------------------- 1 | // Copyright 2009 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package flate 6 | 7 | var reverseByte = [256]byte{ 8 | 0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0, 9 | 0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0, 10 | 0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8, 11 | 0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8, 12 | 0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4, 13 | 0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4, 14 | 0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec, 15 | 0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc, 16 | 0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2, 17 | 0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2, 18 | 0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea, 19 | 0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa, 20 | 0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6, 21 | 0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6, 22 | 0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee, 23 | 0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe, 24 | 0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1, 25 | 0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1, 26 | 0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9, 27 | 0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9, 28 | 0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5, 29 | 0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5, 30 | 0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed, 31 | 0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd, 32 | 0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3, 33 | 0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3, 34 | 0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb, 35 | 0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb, 36 | 0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7, 37 | 0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7, 38 | 0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef, 39 | 0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff, 40 | } 41 | 42 | func reverseUint16(v uint16) uint16 { 43 | return uint16(reverseByte[v>>8]) | uint16(reverseByte[v&0xFF])<<8 44 | } 45 | 46 | func reverseBits(number uint16, bitLength byte) uint16 { 47 | return reverseUint16(number << uint8(16-bitLength)) 48 | } 49 | -------------------------------------------------------------------------------- /vendor/github.com/klauspost/compress/snappy/AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the official list of Snappy-Go authors for copyright purposes. 2 | # This file is distinct from the CONTRIBUTORS files. 3 | # See the latter for an explanation. 4 | 5 | # Names should be added to this file as 6 | # Name or Organization 7 | # The email address is not required for organizations. 8 | 9 | # Please keep the list sorted. 10 | 11 | Damian Gryski 12 | Google Inc. 13 | Jan Mercl <0xjnml@gmail.com> 14 | Rodolfo Carvalho 15 | Sebastien Binet 16 | -------------------------------------------------------------------------------- /vendor/github.com/klauspost/compress/snappy/CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | # This is the official list of people who can contribute 2 | # (and typically have contributed) code to the Snappy-Go repository. 3 | # The AUTHORS file lists the copyright holders; this file 4 | # lists people. For example, Google employees are listed here 5 | # but not in AUTHORS, because Google holds the copyright. 6 | # 7 | # The submission process automatically checks to make sure 8 | # that people submitting code are listed in this file (by email address). 9 | # 10 | # Names should be added to this file only after verifying that 11 | # the individual or the individual's organization has agreed to 12 | # the appropriate Contributor License Agreement, found here: 13 | # 14 | # http://code.google.com/legal/individual-cla-v1.0.html 15 | # http://code.google.com/legal/corporate-cla-v1.0.html 16 | # 17 | # The agreement for individuals can be filled out on the web. 18 | # 19 | # When adding J Random Contributor's name to this file, 20 | # either J's name or J's organization's name should be 21 | # added to the AUTHORS file, depending on whether the 22 | # individual or corporate CLA was used. 23 | 24 | # Names should be added to this file like so: 25 | # Name 26 | 27 | # Please keep the list sorted. 28 | 29 | Damian Gryski 30 | Jan Mercl <0xjnml@gmail.com> 31 | Kai Backman 32 | Marc-Antoine Ruel 33 | Nigel Tao 34 | Rob Pike 35 | Rodolfo Carvalho 36 | Russ Cox 37 | Sebastien Binet 38 | -------------------------------------------------------------------------------- /vendor/github.com/klauspost/compress/snappy/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /vendor/github.com/klauspost/cpuid/.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | -------------------------------------------------------------------------------- /vendor/github.com/klauspost/cpuid/.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | sudo: false 4 | 5 | os: 6 | - linux 7 | - osx 8 | go: 9 | - 1.8.x 10 | - 1.9.x 11 | - 1.10.x 12 | - master 13 | 14 | script: 15 | - go vet ./... 16 | - go test -v ./... 17 | - go test -race ./... 18 | - diff <(gofmt -d .) <("") 19 | 20 | matrix: 21 | allow_failures: 22 | - go: 'master' 23 | fast_finish: true 24 | -------------------------------------------------------------------------------- /vendor/github.com/klauspost/cpuid/CONTRIBUTING.txt: -------------------------------------------------------------------------------- 1 | Developer Certificate of Origin 2 | Version 1.1 3 | 4 | Copyright (C) 2015- Klaus Post & Contributors. 5 | Email: klauspost@gmail.com 6 | 7 | Everyone is permitted to copy and distribute verbatim copies of this 8 | license document, but changing it is not allowed. 9 | 10 | 11 | Developer's Certificate of Origin 1.1 12 | 13 | By making a contribution to this project, I certify that: 14 | 15 | (a) The contribution was created in whole or in part by me and I 16 | have the right to submit it under the open source license 17 | indicated in the file; or 18 | 19 | (b) The contribution is based upon previous work that, to the best 20 | of my knowledge, is covered under an appropriate open source 21 | license and I have the right under that license to submit that 22 | work with modifications, whether created in whole or in part 23 | by me, under the same open source license (unless I am 24 | permitted to submit under a different license), as indicated 25 | in the file; or 26 | 27 | (c) The contribution was provided directly to me by some other 28 | person who certified (a), (b) or (c) and I have not modified 29 | it. 30 | 31 | (d) I understand and agree that this project and the contribution 32 | are public and that a record of the contribution (including all 33 | personal information I submit with it, including my sign-off) is 34 | maintained indefinitely and may be redistributed consistent with 35 | this project or the open source license(s) involved. 36 | -------------------------------------------------------------------------------- /vendor/github.com/klauspost/cpuid/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Klaus Post 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /vendor/github.com/klauspost/cpuid/cpuid_386.s: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. 2 | 3 | // +build 386,!gccgo 4 | 5 | // func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32) 6 | TEXT ·asmCpuid(SB), 7, $0 7 | XORL CX, CX 8 | MOVL op+0(FP), AX 9 | CPUID 10 | MOVL AX, eax+4(FP) 11 | MOVL BX, ebx+8(FP) 12 | MOVL CX, ecx+12(FP) 13 | MOVL DX, edx+16(FP) 14 | RET 15 | 16 | // func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32) 17 | TEXT ·asmCpuidex(SB), 7, $0 18 | MOVL op+0(FP), AX 19 | MOVL op2+4(FP), CX 20 | CPUID 21 | MOVL AX, eax+8(FP) 22 | MOVL BX, ebx+12(FP) 23 | MOVL CX, ecx+16(FP) 24 | MOVL DX, edx+20(FP) 25 | RET 26 | 27 | // func xgetbv(index uint32) (eax, edx uint32) 28 | TEXT ·asmXgetbv(SB), 7, $0 29 | MOVL index+0(FP), CX 30 | BYTE $0x0f; BYTE $0x01; BYTE $0xd0 // XGETBV 31 | MOVL AX, eax+4(FP) 32 | MOVL DX, edx+8(FP) 33 | RET 34 | 35 | // func asmRdtscpAsm() (eax, ebx, ecx, edx uint32) 36 | TEXT ·asmRdtscpAsm(SB), 7, $0 37 | BYTE $0x0F; BYTE $0x01; BYTE $0xF9 // RDTSCP 38 | MOVL AX, eax+0(FP) 39 | MOVL BX, ebx+4(FP) 40 | MOVL CX, ecx+8(FP) 41 | MOVL DX, edx+12(FP) 42 | RET 43 | -------------------------------------------------------------------------------- /vendor/github.com/klauspost/cpuid/cpuid_amd64.s: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. 2 | 3 | //+build amd64,!gccgo 4 | 5 | // func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32) 6 | TEXT ·asmCpuid(SB), 7, $0 7 | XORQ CX, CX 8 | MOVL op+0(FP), AX 9 | CPUID 10 | MOVL AX, eax+8(FP) 11 | MOVL BX, ebx+12(FP) 12 | MOVL CX, ecx+16(FP) 13 | MOVL DX, edx+20(FP) 14 | RET 15 | 16 | // func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32) 17 | TEXT ·asmCpuidex(SB), 7, $0 18 | MOVL op+0(FP), AX 19 | MOVL op2+4(FP), CX 20 | CPUID 21 | MOVL AX, eax+8(FP) 22 | MOVL BX, ebx+12(FP) 23 | MOVL CX, ecx+16(FP) 24 | MOVL DX, edx+20(FP) 25 | RET 26 | 27 | // func asmXgetbv(index uint32) (eax, edx uint32) 28 | TEXT ·asmXgetbv(SB), 7, $0 29 | MOVL index+0(FP), CX 30 | BYTE $0x0f; BYTE $0x01; BYTE $0xd0 // XGETBV 31 | MOVL AX, eax+8(FP) 32 | MOVL DX, edx+12(FP) 33 | RET 34 | 35 | // func asmRdtscpAsm() (eax, ebx, ecx, edx uint32) 36 | TEXT ·asmRdtscpAsm(SB), 7, $0 37 | BYTE $0x0F; BYTE $0x01; BYTE $0xF9 // RDTSCP 38 | MOVL AX, eax+0(FP) 39 | MOVL BX, ebx+4(FP) 40 | MOVL CX, ecx+8(FP) 41 | MOVL DX, edx+12(FP) 42 | RET 43 | -------------------------------------------------------------------------------- /vendor/github.com/klauspost/cpuid/detect_intel.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. 2 | 3 | // +build 386,!gccgo amd64,!gccgo 4 | 5 | package cpuid 6 | 7 | func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32) 8 | func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32) 9 | func asmXgetbv(index uint32) (eax, edx uint32) 10 | func asmRdtscpAsm() (eax, ebx, ecx, edx uint32) 11 | 12 | func initCPU() { 13 | cpuid = asmCpuid 14 | cpuidex = asmCpuidex 15 | xgetbv = asmXgetbv 16 | rdtscpAsm = asmRdtscpAsm 17 | } 18 | -------------------------------------------------------------------------------- /vendor/github.com/klauspost/cpuid/detect_ref.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. 2 | 3 | // +build !amd64,!386 gccgo 4 | 5 | package cpuid 6 | 7 | func initCPU() { 8 | cpuid = func(op uint32) (eax, ebx, ecx, edx uint32) { 9 | return 0, 0, 0, 0 10 | } 11 | 12 | cpuidex = func(op, op2 uint32) (eax, ebx, ecx, edx uint32) { 13 | return 0, 0, 0, 0 14 | } 15 | 16 | xgetbv = func(index uint32) (eax, edx uint32) { 17 | return 0, 0 18 | } 19 | 20 | rdtscpAsm = func() (eax, ebx, ecx, edx uint32) { 21 | return 0, 0, 0, 0 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /vendor/github.com/klauspost/cpuid/generate.go: -------------------------------------------------------------------------------- 1 | package cpuid 2 | 3 | //go:generate go run private-gen.go 4 | //go:generate gofmt -w ./private 5 | -------------------------------------------------------------------------------- /vendor/github.com/mailru/easyjson/.gitignore: -------------------------------------------------------------------------------- 1 | .root 2 | *_easyjson.go 3 | *.iml 4 | .idea 5 | *.swp 6 | -------------------------------------------------------------------------------- /vendor/github.com/mailru/easyjson/.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - tip 5 | install: 6 | - go get github.com/ugorji/go/codec 7 | - go get github.com/pquerna/ffjson/fflib/v1 8 | - go get github.com/json-iterator/go 9 | - go get github.com/golang/lint/golint 10 | -------------------------------------------------------------------------------- /vendor/github.com/mailru/easyjson/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Mail.Ru Group 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /vendor/github.com/mailru/easyjson/Makefile: -------------------------------------------------------------------------------- 1 | PKG=github.com/mailru/easyjson 2 | GOPATH:=$(PWD)/.root:$(GOPATH) 3 | export GOPATH 4 | 5 | all: test 6 | 7 | .root/src/$(PKG): 8 | mkdir -p $@ 9 | for i in $$PWD/* ; do ln -s $$i $@/`basename $$i` ; done 10 | 11 | root: .root/src/$(PKG) 12 | 13 | clean: 14 | rm -rf .root 15 | rm -rf tests/*_easyjson.go 16 | 17 | build: 18 | go build -i -o .root/bin/easyjson $(PKG)/easyjson 19 | 20 | generate: root build 21 | .root/bin/easyjson -stubs \ 22 | .root/src/$(PKG)/tests/snake.go \ 23 | .root/src/$(PKG)/tests/data.go \ 24 | .root/src/$(PKG)/tests/omitempty.go \ 25 | .root/src/$(PKG)/tests/nothing.go \ 26 | .root/src/$(PKG)/tests/named_type.go \ 27 | .root/src/$(PKG)/tests/custom_map_key_type.go \ 28 | .root/src/$(PKG)/tests/embedded_type.go 29 | 30 | .root/bin/easyjson -all .root/src/$(PKG)/tests/data.go 31 | .root/bin/easyjson -all .root/src/$(PKG)/tests/nothing.go 32 | .root/bin/easyjson -all .root/src/$(PKG)/tests/errors.go 33 | .root/bin/easyjson -snake_case .root/src/$(PKG)/tests/snake.go 34 | .root/bin/easyjson -omit_empty .root/src/$(PKG)/tests/omitempty.go 35 | .root/bin/easyjson -build_tags=use_easyjson .root/src/$(PKG)/benchmark/data.go 36 | .root/bin/easyjson .root/src/$(PKG)/tests/nested_easy.go 37 | .root/bin/easyjson .root/src/$(PKG)/tests/named_type.go 38 | .root/bin/easyjson .root/src/$(PKG)/tests/custom_map_key_type.go 39 | .root/bin/easyjson .root/src/$(PKG)/tests/embedded_type.go 40 | .root/bin/easyjson -disallow_unknown_fields .root/src/$(PKG)/tests/disallow_unknown.go 41 | 42 | test: generate root 43 | go test \ 44 | $(PKG)/tests \ 45 | $(PKG)/jlexer \ 46 | $(PKG)/gen \ 47 | $(PKG)/buffer 48 | go test -benchmem -tags use_easyjson -bench . $(PKG)/benchmark 49 | golint -set_exit_status .root/src/$(PKG)/tests/*_easyjson.go 50 | 51 | bench-other: generate root 52 | @go test -benchmem -bench . $(PKG)/benchmark 53 | @go test -benchmem -tags use_ffjson -bench . $(PKG)/benchmark 54 | @go test -benchmem -tags use_jsoniter -bench . $(PKG)/benchmark 55 | @go test -benchmem -tags use_codec -bench . $(PKG)/benchmark 56 | 57 | bench-python: 58 | benchmark/ujson.sh 59 | 60 | 61 | .PHONY: root clean generate test build 62 | -------------------------------------------------------------------------------- /vendor/github.com/mailru/easyjson/helpers.go: -------------------------------------------------------------------------------- 1 | // Package easyjson contains marshaler/unmarshaler interfaces and helper functions. 2 | package easyjson 3 | 4 | import ( 5 | "io" 6 | "io/ioutil" 7 | "net/http" 8 | "strconv" 9 | 10 | "github.com/mailru/easyjson/jlexer" 11 | "github.com/mailru/easyjson/jwriter" 12 | ) 13 | 14 | // Marshaler is an easyjson-compatible marshaler interface. 15 | type Marshaler interface { 16 | MarshalEasyJSON(w *jwriter.Writer) 17 | } 18 | 19 | // Marshaler is an easyjson-compatible unmarshaler interface. 20 | type Unmarshaler interface { 21 | UnmarshalEasyJSON(w *jlexer.Lexer) 22 | } 23 | 24 | // Optional defines an undefined-test method for a type to integrate with 'omitempty' logic. 25 | type Optional interface { 26 | IsDefined() bool 27 | } 28 | 29 | // Marshal returns data as a single byte slice. Method is suboptimal as the data is likely to be copied 30 | // from a chain of smaller chunks. 31 | func Marshal(v Marshaler) ([]byte, error) { 32 | w := jwriter.Writer{} 33 | v.MarshalEasyJSON(&w) 34 | return w.BuildBytes() 35 | } 36 | 37 | // MarshalToWriter marshals the data to an io.Writer. 38 | func MarshalToWriter(v Marshaler, w io.Writer) (written int, err error) { 39 | jw := jwriter.Writer{} 40 | v.MarshalEasyJSON(&jw) 41 | return jw.DumpTo(w) 42 | } 43 | 44 | // MarshalToHTTPResponseWriter sets Content-Length and Content-Type headers for the 45 | // http.ResponseWriter, and send the data to the writer. started will be equal to 46 | // false if an error occurred before any http.ResponseWriter methods were actually 47 | // invoked (in this case a 500 reply is possible). 48 | func MarshalToHTTPResponseWriter(v Marshaler, w http.ResponseWriter) (started bool, written int, err error) { 49 | jw := jwriter.Writer{} 50 | v.MarshalEasyJSON(&jw) 51 | if jw.Error != nil { 52 | return false, 0, jw.Error 53 | } 54 | w.Header().Set("Content-Type", "application/json") 55 | w.Header().Set("Content-Length", strconv.Itoa(jw.Size())) 56 | 57 | started = true 58 | written, err = jw.DumpTo(w) 59 | return 60 | } 61 | 62 | // Unmarshal decodes the JSON in data into the object. 63 | func Unmarshal(data []byte, v Unmarshaler) error { 64 | l := jlexer.Lexer{Data: data} 65 | v.UnmarshalEasyJSON(&l) 66 | return l.Error() 67 | } 68 | 69 | // UnmarshalFromReader reads all the data in the reader and decodes as JSON into the object. 70 | func UnmarshalFromReader(r io.Reader, v Unmarshaler) error { 71 | data, err := ioutil.ReadAll(r) 72 | if err != nil { 73 | return err 74 | } 75 | l := jlexer.Lexer{Data: data} 76 | v.UnmarshalEasyJSON(&l) 77 | return l.Error() 78 | } 79 | -------------------------------------------------------------------------------- /vendor/github.com/mailru/easyjson/jlexer/bytestostr.go: -------------------------------------------------------------------------------- 1 | // This file will only be included to the build if neither 2 | // easyjson_nounsafe nor appengine build tag is set. See README notes 3 | // for more details. 4 | 5 | //+build !easyjson_nounsafe 6 | //+build !appengine 7 | 8 | package jlexer 9 | 10 | import ( 11 | "reflect" 12 | "unsafe" 13 | ) 14 | 15 | // bytesToStr creates a string pointing at the slice to avoid copying. 16 | // 17 | // Warning: the string returned by the function should be used with care, as the whole input data 18 | // chunk may be either blocked from being freed by GC because of a single string or the buffer.Data 19 | // may be garbage-collected even when the string exists. 20 | func bytesToStr(data []byte) string { 21 | h := (*reflect.SliceHeader)(unsafe.Pointer(&data)) 22 | shdr := reflect.StringHeader{Data: h.Data, Len: h.Len} 23 | return *(*string)(unsafe.Pointer(&shdr)) 24 | } 25 | -------------------------------------------------------------------------------- /vendor/github.com/mailru/easyjson/jlexer/bytestostr_nounsafe.go: -------------------------------------------------------------------------------- 1 | // This file is included to the build if any of the buildtags below 2 | // are defined. Refer to README notes for more details. 3 | 4 | //+build easyjson_nounsafe appengine 5 | 6 | package jlexer 7 | 8 | // bytesToStr creates a string normally from []byte 9 | // 10 | // Note that this method is roughly 1.5x slower than using the 'unsafe' method. 11 | func bytesToStr(data []byte) string { 12 | return string(data) 13 | } 14 | -------------------------------------------------------------------------------- /vendor/github.com/mailru/easyjson/jlexer/error.go: -------------------------------------------------------------------------------- 1 | package jlexer 2 | 3 | import "fmt" 4 | 5 | // LexerError implements the error interface and represents all possible errors that can be 6 | // generated during parsing the JSON data. 7 | type LexerError struct { 8 | Reason string 9 | Offset int 10 | Data string 11 | } 12 | 13 | func (l *LexerError) Error() string { 14 | return fmt.Sprintf("parse error: %s near offset %d of '%s'", l.Reason, l.Offset, l.Data) 15 | } 16 | -------------------------------------------------------------------------------- /vendor/github.com/mailru/easyjson/raw.go: -------------------------------------------------------------------------------- 1 | package easyjson 2 | 3 | import ( 4 | "github.com/mailru/easyjson/jlexer" 5 | "github.com/mailru/easyjson/jwriter" 6 | ) 7 | 8 | // RawMessage is a raw piece of JSON (number, string, bool, object, array or 9 | // null) that is extracted without parsing and output as is during marshaling. 10 | type RawMessage []byte 11 | 12 | // MarshalEasyJSON does JSON marshaling using easyjson interface. 13 | func (v *RawMessage) MarshalEasyJSON(w *jwriter.Writer) { 14 | if len(*v) == 0 { 15 | w.RawString("null") 16 | } else { 17 | w.Raw(*v, nil) 18 | } 19 | } 20 | 21 | // UnmarshalEasyJSON does JSON unmarshaling using easyjson interface. 22 | func (v *RawMessage) UnmarshalEasyJSON(l *jlexer.Lexer) { 23 | *v = RawMessage(l.Raw()) 24 | } 25 | 26 | // UnmarshalJSON implements encoding/json.Unmarshaler interface. 27 | func (v *RawMessage) UnmarshalJSON(data []byte) error { 28 | *v = data 29 | return nil 30 | } 31 | 32 | var nullBytes = []byte("null") 33 | 34 | // MarshalJSON implements encoding/json.Marshaler interface. 35 | func (v RawMessage) MarshalJSON() ([]byte, error) { 36 | if len(v) == 0 { 37 | return nullBytes, nil 38 | } 39 | return v, nil 40 | } 41 | 42 | // IsDefined is required for integration with omitempty easyjson logic. 43 | func (v *RawMessage) IsDefined() bool { 44 | return len(*v) > 0 45 | } 46 | -------------------------------------------------------------------------------- /vendor/github.com/mschoch/smat/.gitignore: -------------------------------------------------------------------------------- 1 | #* 2 | *.sublime-* 3 | *~ 4 | .#* 5 | .project 6 | .settings 7 | **/.idea/ 8 | **/*.iml 9 | /examples/bolt/boltsmat-fuzz.zip 10 | /examples/bolt/workdir/ 11 | .DS_Store 12 | coverage.out 13 | *.test 14 | tags 15 | -------------------------------------------------------------------------------- /vendor/github.com/mschoch/smat/.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: go 3 | go: 4 | - 1.6 5 | script: 6 | - go get golang.org/x/tools/cmd/cover 7 | - go get github.com/mattn/goveralls 8 | - go get github.com/kisielk/errcheck 9 | - go test -v -race 10 | - go vet 11 | - errcheck ./... 12 | - go test -coverprofile=profile.out -covermode=count 13 | - goveralls -service=travis-ci -coverprofile=profile.out -repotoken $COVERALLS 14 | notifications: 15 | email: 16 | - marty.schoch@gmail.com 17 | -------------------------------------------------------------------------------- /vendor/github.com/mschoch/smat/actionseq.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Marty Schoch 2 | 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the 5 | // License. You may obtain a copy of the License at 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // Unless required by applicable law or agreed to in writing, 8 | // software distributed under the License is distributed on an "AS 9 | // IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 10 | // express or implied. See the License for the specific language 11 | // governing permissions and limitations under the License. 12 | 13 | package smat 14 | 15 | // ActionSeq represents a sequence of actions, used for populating a corpus 16 | // of byte sequences for the corresponding fuzz tests 17 | type ActionSeq []ActionID 18 | 19 | // ByteEncoding runs the FSM to produce a byte sequence to trigger the 20 | // desired action 21 | func (a ActionSeq) ByteEncoding(ctx Context, setup, teardown ActionID, actionMap ActionMap) ([]byte, error) { 22 | setupFunc, teardownFunc, err := actionMap.findSetupTeardown(setup, teardown) 23 | if err != nil { 24 | return nil, err 25 | } 26 | state, err := setupFunc(ctx) 27 | if err != nil { 28 | return nil, err 29 | } 30 | defer func() { 31 | _, _ = teardownFunc(ctx) 32 | }() 33 | 34 | var rv []byte 35 | for _, actionID := range a { 36 | b, err := probeStateForAction(state, actionID) 37 | if err != nil { 38 | return nil, err 39 | } 40 | rv = append(rv, b) 41 | action, ok := actionMap[actionID] 42 | if !ok { 43 | continue 44 | } 45 | state, err = action(ctx) 46 | if err != nil { 47 | return nil, err 48 | } 49 | } 50 | return rv, nil 51 | } 52 | 53 | func probeStateForAction(state State, actionID ActionID) (byte, error) { 54 | for i := 0; i < 256; i++ { 55 | nextActionID := state(byte(i)) 56 | if nextActionID == actionID { 57 | return byte(i), nil 58 | } 59 | } 60 | return 0, ErrActionNotPossible 61 | } 62 | -------------------------------------------------------------------------------- /vendor/github.com/philhofer/fwd/LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014-2015, Philip Hofer 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /vendor/github.com/philhofer/fwd/writer_appengine.go: -------------------------------------------------------------------------------- 1 | // +build appengine 2 | 3 | package fwd 4 | 5 | func unsafestr(s string) []byte { return []byte(s) } 6 | -------------------------------------------------------------------------------- /vendor/github.com/philhofer/fwd/writer_unsafe.go: -------------------------------------------------------------------------------- 1 | // +build !appengine 2 | 3 | package fwd 4 | 5 | import ( 6 | "reflect" 7 | "unsafe" 8 | ) 9 | 10 | // unsafe cast string as []byte 11 | func unsafestr(b string) []byte { 12 | l := len(b) 13 | return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ 14 | Len: l, 15 | Cap: l, 16 | Data: (*reflect.StringHeader)(unsafe.Pointer(&b)).Data, 17 | })) 18 | } 19 | -------------------------------------------------------------------------------- /vendor/github.com/tinylib/msgp/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Philip Hofer 2 | Portions Copyright (c) 2009 The Go Authors (license at http://golang.org) where indicated 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 5 | 6 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 7 | 8 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /vendor/github.com/tinylib/msgp/msgp/advise_linux.go: -------------------------------------------------------------------------------- 1 | // +build linux,!appengine 2 | 3 | package msgp 4 | 5 | import ( 6 | "os" 7 | "syscall" 8 | ) 9 | 10 | func adviseRead(mem []byte) { 11 | syscall.Madvise(mem, syscall.MADV_SEQUENTIAL|syscall.MADV_WILLNEED) 12 | } 13 | 14 | func adviseWrite(mem []byte) { 15 | syscall.Madvise(mem, syscall.MADV_SEQUENTIAL) 16 | } 17 | 18 | func fallocate(f *os.File, sz int64) error { 19 | err := syscall.Fallocate(int(f.Fd()), 0, 0, sz) 20 | if err == syscall.ENOTSUP { 21 | return f.Truncate(sz) 22 | } 23 | return err 24 | } 25 | -------------------------------------------------------------------------------- /vendor/github.com/tinylib/msgp/msgp/advise_other.go: -------------------------------------------------------------------------------- 1 | // +build !linux appengine 2 | 3 | package msgp 4 | 5 | import ( 6 | "os" 7 | ) 8 | 9 | // TODO: darwin, BSD support 10 | 11 | func adviseRead(mem []byte) {} 12 | 13 | func adviseWrite(mem []byte) {} 14 | 15 | func fallocate(f *os.File, sz int64) error { 16 | return f.Truncate(sz) 17 | } 18 | -------------------------------------------------------------------------------- /vendor/github.com/tinylib/msgp/msgp/circular.go: -------------------------------------------------------------------------------- 1 | package msgp 2 | 3 | type timer interface { 4 | StartTimer() 5 | StopTimer() 6 | } 7 | 8 | // EndlessReader is an io.Reader 9 | // that loops over the same data 10 | // endlessly. It is used for benchmarking. 11 | type EndlessReader struct { 12 | tb timer 13 | data []byte 14 | offset int 15 | } 16 | 17 | // NewEndlessReader returns a new endless reader 18 | func NewEndlessReader(b []byte, tb timer) *EndlessReader { 19 | return &EndlessReader{tb: tb, data: b, offset: 0} 20 | } 21 | 22 | // Read implements io.Reader. In practice, it 23 | // always returns (len(p), nil), although it 24 | // fills the supplied slice while the benchmark 25 | // timer is stopped. 26 | func (c *EndlessReader) Read(p []byte) (int, error) { 27 | c.tb.StopTimer() 28 | var n int 29 | l := len(p) 30 | m := len(c.data) 31 | for n < l { 32 | nn := copy(p[n:], c.data[c.offset:]) 33 | n += nn 34 | c.offset += nn 35 | c.offset %= m 36 | } 37 | c.tb.StartTimer() 38 | return n, nil 39 | } 40 | -------------------------------------------------------------------------------- /vendor/github.com/tinylib/msgp/msgp/file.go: -------------------------------------------------------------------------------- 1 | // +build linux darwin dragonfly freebsd netbsd openbsd 2 | // +build !appengine 3 | 4 | package msgp 5 | 6 | import ( 7 | "os" 8 | "syscall" 9 | ) 10 | 11 | // ReadFile reads a file into 'dst' using 12 | // a read-only memory mapping. Consequently, 13 | // the file must be mmap-able, and the 14 | // Unmarshaler should never write to 15 | // the source memory. (Methods generated 16 | // by the msgp tool obey that constraint, but 17 | // user-defined implementations may not.) 18 | // 19 | // Reading and writing through file mappings 20 | // is only efficient for large files; small 21 | // files are best read and written using 22 | // the ordinary streaming interfaces. 23 | // 24 | func ReadFile(dst Unmarshaler, file *os.File) error { 25 | stat, err := file.Stat() 26 | if err != nil { 27 | return err 28 | } 29 | data, err := syscall.Mmap(int(file.Fd()), 0, int(stat.Size()), syscall.PROT_READ, syscall.MAP_SHARED) 30 | if err != nil { 31 | return err 32 | } 33 | adviseRead(data) 34 | _, err = dst.UnmarshalMsg(data) 35 | uerr := syscall.Munmap(data) 36 | if err == nil { 37 | err = uerr 38 | } 39 | return err 40 | } 41 | 42 | // MarshalSizer is the combination 43 | // of the Marshaler and Sizer 44 | // interfaces. 45 | type MarshalSizer interface { 46 | Marshaler 47 | Sizer 48 | } 49 | 50 | // WriteFile writes a file from 'src' using 51 | // memory mapping. It overwrites the entire 52 | // contents of the previous file. 53 | // The mapping size is calculated 54 | // using the `Msgsize()` method 55 | // of 'src', so it must produce a result 56 | // equal to or greater than the actual encoded 57 | // size of the object. Otherwise, 58 | // a fault (SIGBUS) will occur. 59 | // 60 | // Reading and writing through file mappings 61 | // is only efficient for large files; small 62 | // files are best read and written using 63 | // the ordinary streaming interfaces. 64 | // 65 | // NOTE: The performance of this call 66 | // is highly OS- and filesystem-dependent. 67 | // Users should take care to test that this 68 | // performs as expected in a production environment. 69 | // (Linux users should run a kernel and filesystem 70 | // that support fallocate(2) for the best results.) 71 | func WriteFile(src MarshalSizer, file *os.File) error { 72 | sz := src.Msgsize() 73 | err := fallocate(file, int64(sz)) 74 | if err != nil { 75 | return err 76 | } 77 | data, err := syscall.Mmap(int(file.Fd()), 0, sz, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED) 78 | if err != nil { 79 | return err 80 | } 81 | adviseWrite(data) 82 | chunk := data[:0] 83 | chunk, err = src.MarshalMsg(chunk) 84 | if err != nil { 85 | return err 86 | } 87 | uerr := syscall.Munmap(data) 88 | if uerr != nil { 89 | return uerr 90 | } 91 | return file.Truncate(int64(len(chunk))) 92 | } 93 | -------------------------------------------------------------------------------- /vendor/github.com/tinylib/msgp/msgp/file_port.go: -------------------------------------------------------------------------------- 1 | // +build windows appengine 2 | 3 | package msgp 4 | 5 | import ( 6 | "io/ioutil" 7 | "os" 8 | ) 9 | 10 | // MarshalSizer is the combination 11 | // of the Marshaler and Sizer 12 | // interfaces. 13 | type MarshalSizer interface { 14 | Marshaler 15 | Sizer 16 | } 17 | 18 | func ReadFile(dst Unmarshaler, file *os.File) error { 19 | if u, ok := dst.(Decodable); ok { 20 | return u.DecodeMsg(NewReader(file)) 21 | } 22 | 23 | data, err := ioutil.ReadAll(file) 24 | if err != nil { 25 | return err 26 | } 27 | _, err = dst.UnmarshalMsg(data) 28 | return err 29 | } 30 | 31 | func WriteFile(src MarshalSizer, file *os.File) error { 32 | if e, ok := src.(Encodable); ok { 33 | w := NewWriter(file) 34 | err := e.EncodeMsg(w) 35 | if err == nil { 36 | err = w.Flush() 37 | } 38 | return err 39 | } 40 | 41 | raw, err := src.MarshalMsg(nil) 42 | if err != nil { 43 | return err 44 | } 45 | _, err = file.Write(raw) 46 | return err 47 | } 48 | -------------------------------------------------------------------------------- /vendor/github.com/tinylib/msgp/msgp/purego.go: -------------------------------------------------------------------------------- 1 | // +build purego appengine 2 | 3 | package msgp 4 | 5 | // let's just assume appengine 6 | // uses 64-bit hardware... 7 | const smallint = false 8 | 9 | func UnsafeString(b []byte) string { 10 | return string(b) 11 | } 12 | 13 | func UnsafeBytes(s string) []byte { 14 | return []byte(s) 15 | } 16 | -------------------------------------------------------------------------------- /vendor/github.com/tinylib/msgp/msgp/size.go: -------------------------------------------------------------------------------- 1 | package msgp 2 | 3 | // The sizes provided 4 | // are the worst-case 5 | // encoded sizes for 6 | // each type. For variable- 7 | // length types ([]byte, string), 8 | // the total encoded size is 9 | // the prefix size plus the 10 | // length of the object. 11 | const ( 12 | Int64Size = 9 13 | IntSize = Int64Size 14 | UintSize = Int64Size 15 | Int8Size = 2 16 | Int16Size = 3 17 | Int32Size = 5 18 | Uint8Size = 2 19 | ByteSize = Uint8Size 20 | Uint16Size = 3 21 | Uint32Size = 5 22 | Uint64Size = Int64Size 23 | Float64Size = 9 24 | Float32Size = 5 25 | Complex64Size = 10 26 | Complex128Size = 18 27 | 28 | TimeSize = 15 29 | BoolSize = 1 30 | NilSize = 1 31 | 32 | MapHeaderSize = 5 33 | ArrayHeaderSize = 5 34 | 35 | BytesPrefixSize = 5 36 | StringPrefixSize = 5 37 | ExtensionPrefixSize = 6 38 | ) 39 | -------------------------------------------------------------------------------- /vendor/github.com/tinylib/msgp/msgp/unsafe.go: -------------------------------------------------------------------------------- 1 | // +build !purego,!appengine 2 | 3 | package msgp 4 | 5 | import ( 6 | "reflect" 7 | "unsafe" 8 | ) 9 | 10 | // NOTE: 11 | // all of the definition in this file 12 | // should be repeated in appengine.go, 13 | // but without using unsafe 14 | 15 | const ( 16 | // spec says int and uint are always 17 | // the same size, but that int/uint 18 | // size may not be machine word size 19 | smallint = unsafe.Sizeof(int(0)) == 4 20 | ) 21 | 22 | // UnsafeString returns the byte slice as a volatile string 23 | // THIS SHOULD ONLY BE USED BY THE CODE GENERATOR. 24 | // THIS IS EVIL CODE. 25 | // YOU HAVE BEEN WARNED. 26 | func UnsafeString(b []byte) string { 27 | sh := (*reflect.SliceHeader)(unsafe.Pointer(&b)) 28 | return *(*string)(unsafe.Pointer(&reflect.StringHeader{Data: sh.Data, Len: sh.Len})) 29 | } 30 | 31 | // UnsafeBytes returns the string as a byte slice 32 | // THIS SHOULD ONLY BE USED BY THE CODE GENERATOR. 33 | // THIS IS EVIL CODE. 34 | // YOU HAVE BEEN WARNED. 35 | func UnsafeBytes(s string) []byte { 36 | return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ 37 | Len: len(s), 38 | Cap: len(s), 39 | Data: (*(*reflect.StringHeader)(unsafe.Pointer(&s))).Data, 40 | })) 41 | } 42 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/bytebufferpool/.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.6 5 | 6 | script: 7 | # build test for supported platforms 8 | - GOOS=linux go build 9 | - GOOS=darwin go build 10 | - GOOS=freebsd go build 11 | - GOOS=windows go build 12 | - GOARCH=386 go build 13 | 14 | # run tests on a standard platform 15 | - go test -v ./... 16 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/bytebufferpool/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Aliaksandr Valialkin, VertaMedia 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/bytebufferpool/README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/valyala/bytebufferpool.svg)](https://travis-ci.org/valyala/bytebufferpool) 2 | [![GoDoc](https://godoc.org/github.com/valyala/bytebufferpool?status.svg)](http://godoc.org/github.com/valyala/bytebufferpool) 3 | [![Go Report](http://goreportcard.com/badge/valyala/bytebufferpool)](http://goreportcard.com/report/valyala/bytebufferpool) 4 | 5 | # bytebufferpool 6 | 7 | An implementation of a pool of byte buffers with anti-memory-waste protection. 8 | 9 | The pool may waste limited amount of memory due to fragmentation. 10 | This amount equals to the maximum total size of the byte buffers 11 | in concurrent use. 12 | 13 | # Benchmark results 14 | Currently bytebufferpool is fastest and most effective buffer pool written in Go. 15 | 16 | You can find results [here](https://omgnull.github.io/go-benchmark/buffer/). 17 | 18 | # bytebufferpool users 19 | 20 | * [fasthttp](https://github.com/valyala/fasthttp) 21 | * [quicktemplate](https://github.com/valyala/quicktemplate) 22 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/bytebufferpool/bytebuffer.go: -------------------------------------------------------------------------------- 1 | package bytebufferpool 2 | 3 | import "io" 4 | 5 | // ByteBuffer provides byte buffer, which can be used for minimizing 6 | // memory allocations. 7 | // 8 | // ByteBuffer may be used with functions appending data to the given []byte 9 | // slice. See example code for details. 10 | // 11 | // Use Get for obtaining an empty byte buffer. 12 | type ByteBuffer struct { 13 | 14 | // B is a byte buffer to use in append-like workloads. 15 | // See example code for details. 16 | B []byte 17 | } 18 | 19 | // Len returns the size of the byte buffer. 20 | func (b *ByteBuffer) Len() int { 21 | return len(b.B) 22 | } 23 | 24 | // ReadFrom implements io.ReaderFrom. 25 | // 26 | // The function appends all the data read from r to b. 27 | func (b *ByteBuffer) ReadFrom(r io.Reader) (int64, error) { 28 | p := b.B 29 | nStart := int64(len(p)) 30 | nMax := int64(cap(p)) 31 | n := nStart 32 | if nMax == 0 { 33 | nMax = 64 34 | p = make([]byte, nMax) 35 | } else { 36 | p = p[:nMax] 37 | } 38 | for { 39 | if n == nMax { 40 | nMax *= 2 41 | bNew := make([]byte, nMax) 42 | copy(bNew, p) 43 | p = bNew 44 | } 45 | nn, err := r.Read(p[n:]) 46 | n += int64(nn) 47 | if err != nil { 48 | b.B = p[:n] 49 | n -= nStart 50 | if err == io.EOF { 51 | return n, nil 52 | } 53 | return n, err 54 | } 55 | } 56 | } 57 | 58 | // WriteTo implements io.WriterTo. 59 | func (b *ByteBuffer) WriteTo(w io.Writer) (int64, error) { 60 | n, err := w.Write(b.B) 61 | return int64(n), err 62 | } 63 | 64 | // Bytes returns b.B, i.e. all the bytes accumulated in the buffer. 65 | // 66 | // The purpose of this function is bytes.Buffer compatibility. 67 | func (b *ByteBuffer) Bytes() []byte { 68 | return b.B 69 | } 70 | 71 | // Write implements io.Writer - it appends p to ByteBuffer.B 72 | func (b *ByteBuffer) Write(p []byte) (int, error) { 73 | b.B = append(b.B, p...) 74 | return len(p), nil 75 | } 76 | 77 | // WriteByte appends the byte c to the buffer. 78 | // 79 | // The purpose of this function is bytes.Buffer compatibility. 80 | // 81 | // The function always returns nil. 82 | func (b *ByteBuffer) WriteByte(c byte) error { 83 | b.B = append(b.B, c) 84 | return nil 85 | } 86 | 87 | // WriteString appends s to ByteBuffer.B. 88 | func (b *ByteBuffer) WriteString(s string) (int, error) { 89 | b.B = append(b.B, s...) 90 | return len(s), nil 91 | } 92 | 93 | // Set sets ByteBuffer.B to p. 94 | func (b *ByteBuffer) Set(p []byte) { 95 | b.B = append(b.B[:0], p...) 96 | } 97 | 98 | // SetString sets ByteBuffer.B to s. 99 | func (b *ByteBuffer) SetString(s string) { 100 | b.B = append(b.B[:0], s...) 101 | } 102 | 103 | // String returns string representation of ByteBuffer.B. 104 | func (b *ByteBuffer) String() string { 105 | return string(b.B) 106 | } 107 | 108 | // Reset makes ByteBuffer.B empty. 109 | func (b *ByteBuffer) Reset() { 110 | b.B = b.B[:0] 111 | } 112 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/bytebufferpool/doc.go: -------------------------------------------------------------------------------- 1 | // Package bytebufferpool implements a pool of byte buffers 2 | // with anti-fragmentation protection. 3 | // 4 | // The pool may waste limited amount of memory due to fragmentation. 5 | // This amount equals to the maximum total size of the byte buffers 6 | // in concurrent use. 7 | package bytebufferpool 8 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/.gitignore: -------------------------------------------------------------------------------- 1 | tags 2 | *.pprof 3 | *.fasthttp.gz 4 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - tip 5 | - 1.11.x 6 | - 1.10.x 7 | - 1.9.x 8 | 9 | os: 10 | - linux 11 | - osx 12 | 13 | matrix: 14 | allow_failures: 15 | - tip 16 | fast_finish: true 17 | 18 | before_install: 19 | - go get -t -v ./... 20 | # - go get -v golang.org/x/tools/cmd/goimports 21 | 22 | script: 23 | # TODO(@kirilldanshin) 24 | # - test -z "$(goimports -d $(find . -type f -name '*.go' -not -path "./vendor/*"))" 25 | # build test for supported platforms 26 | - GOOS=linux go build 27 | - GOOS=darwin go build 28 | - GOOS=freebsd go build 29 | - GOOS=windows go build 30 | - GOARCH=386 go build 31 | 32 | # run tests on a standard platform 33 | - go test -v ./... 34 | 35 | # run tests with the race detector as well 36 | - go test -race -v ./... 37 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015-present Aliaksandr Valialkin, VertaMedia 4 | Copyright (c) 2018-present Kirill Danshin 5 | Copyright (c) 2018-present Erik Dubbelboer 6 | Copyright (c) 2018-present FastHTTP Authors 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining a copy 9 | of this software and associated documentation files (the "Software"), to deal 10 | in the Software without restriction, including without limitation the rights 11 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | copies of the Software, and to permit persons to whom the Software is 13 | furnished to do so, subject to the following conditions: 14 | 15 | The above copyright notice and this permission notice shall be included in all 16 | copies or substantial portions of the Software. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | SOFTWARE. 25 | 26 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/TODO: -------------------------------------------------------------------------------- 1 | - SessionClient with referer and cookies support. 2 | - ProxyHandler similar to FSHandler. 3 | - WebSockets. See https://tools.ietf.org/html/rfc6455 . 4 | - HTTP/2.0. See https://tools.ietf.org/html/rfc7540 . 5 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/bytesconv_32.go: -------------------------------------------------------------------------------- 1 | // +build !amd64,!arm64,!ppc64 2 | 3 | package fasthttp 4 | 5 | const ( 6 | maxHexIntChars = 7 7 | ) 8 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/bytesconv_64.go: -------------------------------------------------------------------------------- 1 | // +build amd64 arm64 ppc64 2 | 3 | package fasthttp 4 | 5 | const ( 6 | maxHexIntChars = 15 7 | ) 8 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/coarseTime.go: -------------------------------------------------------------------------------- 1 | package fasthttp 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // CoarseTimeNow returns the current time truncated to the nearest second. 8 | // 9 | // Deprecated: This is slower than calling time.Now() directly. 10 | // This is now time.Now().Truncate(time.Second) shortcut. 11 | func CoarseTimeNow() time.Time { 12 | return time.Now().Truncate(time.Second) 13 | } 14 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package fasthttp provides fast HTTP server and client API. 3 | 4 | Fasthttp provides the following features: 5 | 6 | * Optimized for speed. Easily handles more than 100K qps and more than 1M 7 | concurrent keep-alive connections on modern hardware. 8 | * Optimized for low memory usage. 9 | * Easy 'Connection: Upgrade' support via RequestCtx.Hijack. 10 | * Server provides the following anti-DoS limits: 11 | 12 | * The number of concurrent connections. 13 | * The number of concurrent connections per client IP. 14 | * The number of requests per connection. 15 | * Request read timeout. 16 | * Response write timeout. 17 | * Maximum request header size. 18 | * Maximum request body size. 19 | * Maximum request execution time. 20 | * Maximum keep-alive connection lifetime. 21 | * Early filtering out non-GET requests. 22 | 23 | * A lot of additional useful info is exposed to request handler: 24 | 25 | * Server and client address. 26 | * Per-request logger. 27 | * Unique request id. 28 | * Request start time. 29 | * Connection start time. 30 | * Request sequence number for the current connection. 31 | 32 | * Client supports automatic retry on idempotent requests' failure. 33 | * Fasthttp API is designed with the ability to extend existing client 34 | and server implementations or to write custom client and server 35 | implementations from scratch. 36 | */ 37 | package fasthttp 38 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/fasthttputil/doc.go: -------------------------------------------------------------------------------- 1 | // Package fasthttputil provides utility functions for fasthttp. 2 | package fasthttputil 3 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/fasthttputil/ecdsa.key: -------------------------------------------------------------------------------- 1 | -----BEGIN EC PRIVATE KEY----- 2 | MHcCAQEEIBpQbZ6a5jL1Yh4wdP6yZk4MKjYWArD/QOLENFw8vbELoAoGCCqGSM49 3 | AwEHoUQDQgAEKQCZWgE2IBhb47ot8MIs1D4KSisHYlZ41IWyeutpjb0fjwwIhimh 4 | pl1Qld1/d2j3Z3vVyfa5yD+ncV7qCFZuSg== 5 | -----END EC PRIVATE KEY----- 6 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/fasthttputil/ecdsa.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIBbTCCAROgAwIBAgIQPo718S+K+G7hc1SgTEU4QDAKBggqhkjOPQQDAjASMRAw 3 | DgYDVQQKEwdBY21lIENvMB4XDTE3MDQyMDIxMDExNFoXDTE4MDQyMDIxMDExNFow 4 | EjEQMA4GA1UEChMHQWNtZSBDbzBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABCkA 5 | mVoBNiAYW+O6LfDCLNQ+CkorB2JWeNSFsnrraY29H48MCIYpoaZdUJXdf3do92d7 6 | 1cn2ucg/p3Fe6ghWbkqjSzBJMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggr 7 | BgEFBQcDATAMBgNVHRMBAf8EAjAAMBQGA1UdEQQNMAuCCWxvY2FsaG9zdDAKBggq 8 | hkjOPQQDAgNIADBFAiEAoLAIQkvSuIcHUqyWroA6yWYw2fznlRH/uO9/hMCxUCEC 9 | IClRYb/5O9eD/Eq/ozPnwNpsQHOeYefEhadJ/P82y0lG 10 | -----END CERTIFICATE----- 11 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/fasthttputil/inmemory_listener.go: -------------------------------------------------------------------------------- 1 | package fasthttputil 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "sync" 7 | ) 8 | 9 | // InmemoryListener provides in-memory dialer<->net.Listener implementation. 10 | // 11 | // It may be used either for fast in-process client<->server communications 12 | // without network stack overhead or for client<->server tests. 13 | type InmemoryListener struct { 14 | lock sync.Mutex 15 | closed bool 16 | conns chan net.Conn 17 | } 18 | 19 | // NewInmemoryListener returns new in-memory dialer<->net.Listener. 20 | func NewInmemoryListener() *InmemoryListener { 21 | return &InmemoryListener{ 22 | conns: make(chan net.Conn, 1024), 23 | } 24 | } 25 | 26 | // Accept implements net.Listener's Accept. 27 | // 28 | // It is safe calling Accept from concurrently running goroutines. 29 | // 30 | // Accept returns new connection per each Dial call. 31 | func (ln *InmemoryListener) Accept() (net.Conn, error) { 32 | c, ok := <-ln.conns 33 | if !ok { 34 | return nil, fmt.Errorf("InmemoryListener is already closed: use of closed network connection") 35 | } 36 | return c, nil 37 | } 38 | 39 | // Close implements net.Listener's Close. 40 | func (ln *InmemoryListener) Close() error { 41 | var err error 42 | 43 | ln.lock.Lock() 44 | if !ln.closed { 45 | close(ln.conns) 46 | ln.closed = true 47 | } else { 48 | err = fmt.Errorf("InmemoryListener is already closed") 49 | } 50 | ln.lock.Unlock() 51 | return err 52 | } 53 | 54 | // Addr implements net.Listener's Addr. 55 | func (ln *InmemoryListener) Addr() net.Addr { 56 | return &net.UnixAddr{ 57 | Name: "InmemoryListener", 58 | Net: "memory", 59 | } 60 | } 61 | 62 | // Dial creates new client<->server connection, enqueues server side 63 | // of the connection to Accept and returns client side of the connection. 64 | // 65 | // It is safe calling Dial from concurrently running goroutines. 66 | func (ln *InmemoryListener) Dial() (net.Conn, error) { 67 | pc := NewPipeConns() 68 | cConn := pc.Conn1() 69 | sConn := pc.Conn2() 70 | ln.lock.Lock() 71 | if !ln.closed { 72 | ln.conns <- sConn 73 | } else { 74 | sConn.Close() 75 | cConn.Close() 76 | cConn = nil 77 | } 78 | ln.lock.Unlock() 79 | 80 | if cConn == nil { 81 | return nil, fmt.Errorf("InmemoryListener is already closed") 82 | } 83 | return cConn, nil 84 | } 85 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/fasthttputil/rsa.key: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQD4IQusAs8PJdnG 3 | 3mURt/AXtgC+ceqLOatJ49JJE1VPTkMAy+oE1f1XvkMrYsHqmDf6GWVzgVXryL4U 4 | wq2/nJSm56ddhN55nI8oSN3dtywUB8/ShelEN73nlN77PeD9tl6NksPwWaKrqxq0 5 | FlabRPZSQCfmgZbhDV8Sa8mfCkFU0G0lit6kLGceCKMvmW+9Bz7ebsYmVdmVMxmf 6 | IJStFD44lWFTdUc65WISKEdW2ELcUefb0zOLw+0PCbXFGJH5x5ktksW8+BBk2Hkg 7 | GeQRL/qPCccthbScO0VgNj3zJ3ZZL0ObSDAbvNDG85joeNjDNq5DT/BAZ0bOSbEF 8 | sh+f9BAzAgMBAAECggEBAJWv2cq7Jw6MVwSRxYca38xuD6TUNBopgBvjREixURW2 9 | sNUaLuMb9Omp7fuOaE2N5rcJ+xnjPGIxh/oeN5MQctz9gwn3zf6vY+15h97pUb4D 10 | uGvYPRDaT8YVGS+X9NMZ4ZCmqW2lpWzKnCFoGHcy8yZLbcaxBsRdvKzwOYGoPiFb 11 | K2QuhXZ/1UPmqK9i2DFKtj40X6vBszTNboFxOVpXrPu0FJwLVSDf2hSZ4fMM0DH3 12 | YqwKcYf5te+hxGKgrqRA3tn0NCWii0in6QIwXMC+kMw1ebg/tZKqyDLMNptAK8J+ 13 | DVw9m5X1seUHS5ehU/g2jrQrtK5WYn7MrFK4lBzlRwECgYEA/d1TeANYECDWRRDk 14 | B0aaRZs87Rwl/J9PsvbsKvtU/bX+OfSOUjOa9iQBqn0LmU8GqusEET/QVUfocVwV 15 | Bggf/5qDLxz100Rj0ags/yE/kNr0Bb31kkkKHFMnCT06YasR7qKllwrAlPJvQv9x 16 | IzBKq+T/Dx08Wep9bCRSFhzRCnsCgYEA+jdeZXTDr/Vz+D2B3nAw1frqYFfGnEVY 17 | wqmoK3VXMDkGuxsloO2rN+SyiUo3JNiQNPDub/t7175GH5pmKtZOlftePANsUjBj 18 | wZ1D0rI5Bxu/71ibIUYIRVmXsTEQkh/ozoh3jXCZ9+bLgYiYx7789IUZZSokFQ3D 19 | FICUT9KJ36kCgYAGoq9Y1rWJjmIrYfqj2guUQC+CfxbbGIrrwZqAsRsSmpwvhZ3m 20 | tiSZxG0quKQB+NfSxdvQW5ulbwC7Xc3K35F+i9pb8+TVBdeaFkw+yu6vaZmxQLrX 21 | fQM/pEjD7A7HmMIaO7QaU5SfEAsqdCTP56Y8AftMuNXn/8IRfo2KuGwaWwKBgFpU 22 | ILzJoVdlad9E/Rw7LjYhZfkv1uBVXIyxyKcfrkEXZSmozDXDdxsvcZCEfVHM6Ipk 23 | K/+7LuMcqp4AFEAEq8wTOdq6daFaHLkpt/FZK6M4TlruhtpFOPkoNc3e45eM83OT 24 | 6mziKINJC1CQ6m65sQHpBtjxlKMRG8rL/D6wx9s5AoGBAMRlqNPMwglT3hvDmsAt 25 | 9Lf9pdmhERUlHhD8bj8mDaBj2Aqv7f6VRJaYZqP403pKKQexuqcn80mtjkSAPFkN 26 | Cj7BVt/RXm5uoxDTnfi26RF9F6yNDEJ7UU9+peBr99aazF/fTgW/1GcMkQnum8uV 27 | c257YgaWmjK9uB0Y2r2VxS0G 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/fasthttputil/rsa.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICujCCAaKgAwIBAgIJAMbXnKZ/cikUMA0GCSqGSIb3DQEBCwUAMBUxEzARBgNV 3 | BAMTCnVidW50dS5uYW4wHhcNMTUwMjA0MDgwMTM5WhcNMjUwMjAxMDgwMTM5WjAV 4 | MRMwEQYDVQQDEwp1YnVudHUubmFuMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB 5 | CgKCAQEA+CELrALPDyXZxt5lEbfwF7YAvnHqizmrSePSSRNVT05DAMvqBNX9V75D 6 | K2LB6pg3+hllc4FV68i+FMKtv5yUpuenXYTeeZyPKEjd3bcsFAfP0oXpRDe955Te 7 | +z3g/bZejZLD8Fmiq6satBZWm0T2UkAn5oGW4Q1fEmvJnwpBVNBtJYrepCxnHgij 8 | L5lvvQc+3m7GJlXZlTMZnyCUrRQ+OJVhU3VHOuViEihHVthC3FHn29Mzi8PtDwm1 9 | xRiR+ceZLZLFvPgQZNh5IBnkES/6jwnHLYW0nDtFYDY98yd2WS9Dm0gwG7zQxvOY 10 | 6HjYwzauQ0/wQGdGzkmxBbIfn/QQMwIDAQABow0wCzAJBgNVHRMEAjAAMA0GCSqG 11 | SIb3DQEBCwUAA4IBAQBQjKm/4KN/iTgXbLTL3i7zaxYXFLXsnT1tF+ay4VA8aj98 12 | L3JwRTciZ3A5iy/W4VSCt3eASwOaPWHKqDBB5RTtL73LoAqsWmO3APOGQAbixcQ2 13 | 45GXi05OKeyiYRi1Nvq7Unv9jUkRDHUYVPZVSAjCpsXzPhFkmZoTRxmx5l0ZF7Li 14 | K91lI5h+eFq0dwZwrmlPambyh1vQUi70VHv8DNToVU29kel7YLbxGbuqETfhrcy6 15 | X+Mha6RYITkAn5FqsZcKMsc9eYGEF4l3XV+oS7q6xfTxktYJMFTI18J0lQ2Lv/CI 16 | whdMnYGntDQBE/iFCrJEGNsKGc38796GBOb5j+zd 17 | -----END CERTIFICATE----- 18 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/valyala/fasthttp 2 | 3 | require ( 4 | github.com/klauspost/compress v1.4.0 5 | github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e // indirect 6 | github.com/valyala/bytebufferpool v1.0.0 7 | github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a 8 | golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 9 | ) 10 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/go.sum: -------------------------------------------------------------------------------- 1 | github.com/klauspost/compress v1.4.0 h1:8nsMz3tWa9SWWPL60G1V6CUsf4lLjWLTNEtibhe8gh8= 2 | github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 3 | github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e h1:+lIPJOWl+jSiJOc70QXJ07+2eg2Jy2EC7Mi11BWujeM= 4 | github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 5 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 6 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 7 | github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= 8 | github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= 9 | golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 h1:czFLhve3vsQetD6JOJ8NZZvGQIXlnN3/yXxbT6/awxI= 10 | golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 11 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/nocopy.go: -------------------------------------------------------------------------------- 1 | package fasthttp 2 | 3 | // Embed this type into a struct, which mustn't be copied, 4 | // so `go vet` gives a warning if this struct is copied. 5 | // 6 | // See https://github.com/golang/go/issues/8005#issuecomment-190753527 for details. 7 | // and also: https://stackoverflow.com/questions/52494458/nocopy-minimal-example 8 | type noCopy struct{} 9 | 10 | func (*noCopy) Lock() {} 11 | func (*noCopy) Unlock() {} 12 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/peripconn.go: -------------------------------------------------------------------------------- 1 | package fasthttp 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "sync" 7 | ) 8 | 9 | type perIPConnCounter struct { 10 | pool sync.Pool 11 | lock sync.Mutex 12 | m map[uint32]int 13 | } 14 | 15 | func (cc *perIPConnCounter) Register(ip uint32) int { 16 | cc.lock.Lock() 17 | if cc.m == nil { 18 | cc.m = make(map[uint32]int) 19 | } 20 | n := cc.m[ip] + 1 21 | cc.m[ip] = n 22 | cc.lock.Unlock() 23 | return n 24 | } 25 | 26 | func (cc *perIPConnCounter) Unregister(ip uint32) { 27 | cc.lock.Lock() 28 | if cc.m == nil { 29 | cc.lock.Unlock() 30 | panic("BUG: perIPConnCounter.Register() wasn't called") 31 | } 32 | n := cc.m[ip] - 1 33 | if n < 0 { 34 | cc.lock.Unlock() 35 | panic(fmt.Sprintf("BUG: negative per-ip counter=%d for ip=%d", n, ip)) 36 | } 37 | cc.m[ip] = n 38 | cc.lock.Unlock() 39 | } 40 | 41 | type perIPConn struct { 42 | net.Conn 43 | 44 | ip uint32 45 | perIPConnCounter *perIPConnCounter 46 | } 47 | 48 | func acquirePerIPConn(conn net.Conn, ip uint32, counter *perIPConnCounter) *perIPConn { 49 | v := counter.pool.Get() 50 | if v == nil { 51 | v = &perIPConn{ 52 | perIPConnCounter: counter, 53 | } 54 | } 55 | c := v.(*perIPConn) 56 | c.Conn = conn 57 | c.ip = ip 58 | return c 59 | } 60 | 61 | func releasePerIPConn(c *perIPConn) { 62 | c.Conn = nil 63 | c.perIPConnCounter.pool.Put(c) 64 | } 65 | 66 | func (c *perIPConn) Close() error { 67 | err := c.Conn.Close() 68 | c.perIPConnCounter.Unregister(c.ip) 69 | releasePerIPConn(c) 70 | return err 71 | } 72 | 73 | func getUint32IP(c net.Conn) uint32 { 74 | return ip2uint32(getConnIP4(c)) 75 | } 76 | 77 | func getConnIP4(c net.Conn) net.IP { 78 | addr := c.RemoteAddr() 79 | ipAddr, ok := addr.(*net.TCPAddr) 80 | if !ok { 81 | return net.IPv4zero 82 | } 83 | return ipAddr.IP.To4() 84 | } 85 | 86 | func ip2uint32(ip net.IP) uint32 { 87 | if len(ip) != 4 { 88 | return 0 89 | } 90 | return uint32(ip[0])<<24 | uint32(ip[1])<<16 | uint32(ip[2])<<8 | uint32(ip[3]) 91 | } 92 | 93 | func uint322ip(ip uint32) net.IP { 94 | b := make([]byte, 4) 95 | b[0] = byte(ip >> 24) 96 | b[1] = byte(ip >> 16) 97 | b[2] = byte(ip >> 8) 98 | b[3] = byte(ip) 99 | return b 100 | } 101 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/reuseport/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Max Riveiro 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/ssl-cert-snakeoil.key: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQD4IQusAs8PJdnG 3 | 3mURt/AXtgC+ceqLOatJ49JJE1VPTkMAy+oE1f1XvkMrYsHqmDf6GWVzgVXryL4U 4 | wq2/nJSm56ddhN55nI8oSN3dtywUB8/ShelEN73nlN77PeD9tl6NksPwWaKrqxq0 5 | FlabRPZSQCfmgZbhDV8Sa8mfCkFU0G0lit6kLGceCKMvmW+9Bz7ebsYmVdmVMxmf 6 | IJStFD44lWFTdUc65WISKEdW2ELcUefb0zOLw+0PCbXFGJH5x5ktksW8+BBk2Hkg 7 | GeQRL/qPCccthbScO0VgNj3zJ3ZZL0ObSDAbvNDG85joeNjDNq5DT/BAZ0bOSbEF 8 | sh+f9BAzAgMBAAECggEBAJWv2cq7Jw6MVwSRxYca38xuD6TUNBopgBvjREixURW2 9 | sNUaLuMb9Omp7fuOaE2N5rcJ+xnjPGIxh/oeN5MQctz9gwn3zf6vY+15h97pUb4D 10 | uGvYPRDaT8YVGS+X9NMZ4ZCmqW2lpWzKnCFoGHcy8yZLbcaxBsRdvKzwOYGoPiFb 11 | K2QuhXZ/1UPmqK9i2DFKtj40X6vBszTNboFxOVpXrPu0FJwLVSDf2hSZ4fMM0DH3 12 | YqwKcYf5te+hxGKgrqRA3tn0NCWii0in6QIwXMC+kMw1ebg/tZKqyDLMNptAK8J+ 13 | DVw9m5X1seUHS5ehU/g2jrQrtK5WYn7MrFK4lBzlRwECgYEA/d1TeANYECDWRRDk 14 | B0aaRZs87Rwl/J9PsvbsKvtU/bX+OfSOUjOa9iQBqn0LmU8GqusEET/QVUfocVwV 15 | Bggf/5qDLxz100Rj0ags/yE/kNr0Bb31kkkKHFMnCT06YasR7qKllwrAlPJvQv9x 16 | IzBKq+T/Dx08Wep9bCRSFhzRCnsCgYEA+jdeZXTDr/Vz+D2B3nAw1frqYFfGnEVY 17 | wqmoK3VXMDkGuxsloO2rN+SyiUo3JNiQNPDub/t7175GH5pmKtZOlftePANsUjBj 18 | wZ1D0rI5Bxu/71ibIUYIRVmXsTEQkh/ozoh3jXCZ9+bLgYiYx7789IUZZSokFQ3D 19 | FICUT9KJ36kCgYAGoq9Y1rWJjmIrYfqj2guUQC+CfxbbGIrrwZqAsRsSmpwvhZ3m 20 | tiSZxG0quKQB+NfSxdvQW5ulbwC7Xc3K35F+i9pb8+TVBdeaFkw+yu6vaZmxQLrX 21 | fQM/pEjD7A7HmMIaO7QaU5SfEAsqdCTP56Y8AftMuNXn/8IRfo2KuGwaWwKBgFpU 22 | ILzJoVdlad9E/Rw7LjYhZfkv1uBVXIyxyKcfrkEXZSmozDXDdxsvcZCEfVHM6Ipk 23 | K/+7LuMcqp4AFEAEq8wTOdq6daFaHLkpt/FZK6M4TlruhtpFOPkoNc3e45eM83OT 24 | 6mziKINJC1CQ6m65sQHpBtjxlKMRG8rL/D6wx9s5AoGBAMRlqNPMwglT3hvDmsAt 25 | 9Lf9pdmhERUlHhD8bj8mDaBj2Aqv7f6VRJaYZqP403pKKQexuqcn80mtjkSAPFkN 26 | Cj7BVt/RXm5uoxDTnfi26RF9F6yNDEJ7UU9+peBr99aazF/fTgW/1GcMkQnum8uV 27 | c257YgaWmjK9uB0Y2r2VxS0G 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/ssl-cert-snakeoil.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICujCCAaKgAwIBAgIJAMbXnKZ/cikUMA0GCSqGSIb3DQEBCwUAMBUxEzARBgNV 3 | BAMTCnVidW50dS5uYW4wHhcNMTUwMjA0MDgwMTM5WhcNMjUwMjAxMDgwMTM5WjAV 4 | MRMwEQYDVQQDEwp1YnVudHUubmFuMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB 5 | CgKCAQEA+CELrALPDyXZxt5lEbfwF7YAvnHqizmrSePSSRNVT05DAMvqBNX9V75D 6 | K2LB6pg3+hllc4FV68i+FMKtv5yUpuenXYTeeZyPKEjd3bcsFAfP0oXpRDe955Te 7 | +z3g/bZejZLD8Fmiq6satBZWm0T2UkAn5oGW4Q1fEmvJnwpBVNBtJYrepCxnHgij 8 | L5lvvQc+3m7GJlXZlTMZnyCUrRQ+OJVhU3VHOuViEihHVthC3FHn29Mzi8PtDwm1 9 | xRiR+ceZLZLFvPgQZNh5IBnkES/6jwnHLYW0nDtFYDY98yd2WS9Dm0gwG7zQxvOY 10 | 6HjYwzauQ0/wQGdGzkmxBbIfn/QQMwIDAQABow0wCzAJBgNVHRMEAjAAMA0GCSqG 11 | SIb3DQEBCwUAA4IBAQBQjKm/4KN/iTgXbLTL3i7zaxYXFLXsnT1tF+ay4VA8aj98 12 | L3JwRTciZ3A5iy/W4VSCt3eASwOaPWHKqDBB5RTtL73LoAqsWmO3APOGQAbixcQ2 13 | 45GXi05OKeyiYRi1Nvq7Unv9jUkRDHUYVPZVSAjCpsXzPhFkmZoTRxmx5l0ZF7Li 14 | K91lI5h+eFq0dwZwrmlPambyh1vQUi70VHv8DNToVU29kel7YLbxGbuqETfhrcy6 15 | X+Mha6RYITkAn5FqsZcKMsc9eYGEF4l3XV+oS7q6xfTxktYJMFTI18J0lQ2Lv/CI 16 | whdMnYGntDQBE/iFCrJEGNsKGc38796GBOb5j+zd 17 | -----END CERTIFICATE----- 18 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/stackless/doc.go: -------------------------------------------------------------------------------- 1 | // Package stackless provides functionality that may save stack space 2 | // for high number of concurrently running goroutines. 3 | package stackless 4 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/stackless/func.go: -------------------------------------------------------------------------------- 1 | package stackless 2 | 3 | import ( 4 | "runtime" 5 | "sync" 6 | ) 7 | 8 | // NewFunc returns stackless wrapper for the function f. 9 | // 10 | // Unlike f, the returned stackless wrapper doesn't use stack space 11 | // on the goroutine that calls it. 12 | // The wrapper may save a lot of stack space if the following conditions 13 | // are met: 14 | // 15 | // - f doesn't contain blocking calls on network, I/O or channels; 16 | // - f uses a lot of stack space; 17 | // - the wrapper is called from high number of concurrent goroutines. 18 | // 19 | // The stackless wrapper returns false if the call cannot be processed 20 | // at the moment due to high load. 21 | func NewFunc(f func(ctx interface{})) func(ctx interface{}) bool { 22 | if f == nil { 23 | panic("BUG: f cannot be nil") 24 | } 25 | 26 | funcWorkCh := make(chan *funcWork, runtime.GOMAXPROCS(-1)*2048) 27 | onceInit := func() { 28 | n := runtime.GOMAXPROCS(-1) 29 | for i := 0; i < n; i++ { 30 | go funcWorker(funcWorkCh, f) 31 | } 32 | } 33 | var once sync.Once 34 | 35 | return func(ctx interface{}) bool { 36 | once.Do(onceInit) 37 | fw := getFuncWork() 38 | fw.ctx = ctx 39 | 40 | select { 41 | case funcWorkCh <- fw: 42 | default: 43 | putFuncWork(fw) 44 | return false 45 | } 46 | <-fw.done 47 | putFuncWork(fw) 48 | return true 49 | } 50 | } 51 | 52 | func funcWorker(funcWorkCh <-chan *funcWork, f func(ctx interface{})) { 53 | for fw := range funcWorkCh { 54 | f(fw.ctx) 55 | fw.done <- struct{}{} 56 | } 57 | } 58 | 59 | func getFuncWork() *funcWork { 60 | v := funcWorkPool.Get() 61 | if v == nil { 62 | v = &funcWork{ 63 | done: make(chan struct{}, 1), 64 | } 65 | } 66 | return v.(*funcWork) 67 | } 68 | 69 | func putFuncWork(fw *funcWork) { 70 | fw.ctx = nil 71 | funcWorkPool.Put(fw) 72 | } 73 | 74 | var funcWorkPool sync.Pool 75 | 76 | type funcWork struct { 77 | ctx interface{} 78 | done chan struct{} 79 | } 80 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/stackless/writer.go: -------------------------------------------------------------------------------- 1 | package stackless 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | 8 | "github.com/valyala/bytebufferpool" 9 | ) 10 | 11 | // Writer is an interface stackless writer must conform to. 12 | // 13 | // The interface contains common subset for Writers from compress/* packages. 14 | type Writer interface { 15 | Write(p []byte) (int, error) 16 | Flush() error 17 | Close() error 18 | Reset(w io.Writer) 19 | } 20 | 21 | // NewWriterFunc must return new writer that will be wrapped into 22 | // stackless writer. 23 | type NewWriterFunc func(w io.Writer) Writer 24 | 25 | // NewWriter creates a stackless writer around a writer returned 26 | // from newWriter. 27 | // 28 | // The returned writer writes data to dstW. 29 | // 30 | // Writers that use a lot of stack space may be wrapped into stackless writer, 31 | // thus saving stack space for high number of concurrently running goroutines. 32 | func NewWriter(dstW io.Writer, newWriter NewWriterFunc) Writer { 33 | w := &writer{ 34 | dstW: dstW, 35 | } 36 | w.zw = newWriter(&w.xw) 37 | return w 38 | } 39 | 40 | type writer struct { 41 | dstW io.Writer 42 | zw Writer 43 | xw xWriter 44 | 45 | err error 46 | n int 47 | 48 | p []byte 49 | op op 50 | } 51 | 52 | type op int 53 | 54 | const ( 55 | opWrite op = iota 56 | opFlush 57 | opClose 58 | opReset 59 | ) 60 | 61 | func (w *writer) Write(p []byte) (int, error) { 62 | w.p = p 63 | err := w.do(opWrite) 64 | w.p = nil 65 | return w.n, err 66 | } 67 | 68 | func (w *writer) Flush() error { 69 | return w.do(opFlush) 70 | } 71 | 72 | func (w *writer) Close() error { 73 | return w.do(opClose) 74 | } 75 | 76 | func (w *writer) Reset(dstW io.Writer) { 77 | w.xw.Reset() 78 | w.do(opReset) 79 | w.dstW = dstW 80 | } 81 | 82 | func (w *writer) do(op op) error { 83 | w.op = op 84 | if !stacklessWriterFunc(w) { 85 | return errHighLoad 86 | } 87 | err := w.err 88 | if err != nil { 89 | return err 90 | } 91 | if w.xw.bb != nil && len(w.xw.bb.B) > 0 { 92 | _, err = w.dstW.Write(w.xw.bb.B) 93 | } 94 | w.xw.Reset() 95 | 96 | return err 97 | } 98 | 99 | var errHighLoad = errors.New("cannot compress data due to high load") 100 | 101 | var stacklessWriterFunc = NewFunc(writerFunc) 102 | 103 | func writerFunc(ctx interface{}) { 104 | w := ctx.(*writer) 105 | switch w.op { 106 | case opWrite: 107 | w.n, w.err = w.zw.Write(w.p) 108 | case opFlush: 109 | w.err = w.zw.Flush() 110 | case opClose: 111 | w.err = w.zw.Close() 112 | case opReset: 113 | w.zw.Reset(&w.xw) 114 | w.err = nil 115 | default: 116 | panic(fmt.Sprintf("BUG: unexpected op: %d", w.op)) 117 | } 118 | } 119 | 120 | type xWriter struct { 121 | bb *bytebufferpool.ByteBuffer 122 | } 123 | 124 | func (w *xWriter) Write(p []byte) (int, error) { 125 | if w.bb == nil { 126 | w.bb = bufferPool.Get() 127 | } 128 | w.bb.Write(p) 129 | return len(p), nil 130 | } 131 | 132 | func (w *xWriter) Reset() { 133 | if w.bb != nil { 134 | bufferPool.Put(w.bb) 135 | w.bb = nil 136 | } 137 | } 138 | 139 | var bufferPool bytebufferpool.Pool 140 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/stream.go: -------------------------------------------------------------------------------- 1 | package fasthttp 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "sync" 7 | 8 | "github.com/valyala/fasthttp/fasthttputil" 9 | ) 10 | 11 | // StreamWriter must write data to w. 12 | // 13 | // Usually StreamWriter writes data to w in a loop (aka 'data streaming'). 14 | // 15 | // StreamWriter must return immediately if w returns error. 16 | // 17 | // Since the written data is buffered, do not forget calling w.Flush 18 | // when the data must be propagated to reader. 19 | type StreamWriter func(w *bufio.Writer) 20 | 21 | // NewStreamReader returns a reader, which replays all the data generated by sw. 22 | // 23 | // The returned reader may be passed to Response.SetBodyStream. 24 | // 25 | // Close must be called on the returned reader after all the required data 26 | // has been read. Otherwise goroutine leak may occur. 27 | // 28 | // See also Response.SetBodyStreamWriter. 29 | func NewStreamReader(sw StreamWriter) io.ReadCloser { 30 | pc := fasthttputil.NewPipeConns() 31 | pw := pc.Conn1() 32 | pr := pc.Conn2() 33 | 34 | var bw *bufio.Writer 35 | v := streamWriterBufPool.Get() 36 | if v == nil { 37 | bw = bufio.NewWriter(pw) 38 | } else { 39 | bw = v.(*bufio.Writer) 40 | bw.Reset(pw) 41 | } 42 | 43 | go func() { 44 | sw(bw) 45 | bw.Flush() 46 | pw.Close() 47 | 48 | streamWriterBufPool.Put(bw) 49 | }() 50 | 51 | return pr 52 | } 53 | 54 | var streamWriterBufPool sync.Pool 55 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/strings.go: -------------------------------------------------------------------------------- 1 | package fasthttp 2 | 3 | var ( 4 | defaultServerName = []byte("fasthttp") 5 | defaultUserAgent = []byte("fasthttp") 6 | defaultContentType = []byte("text/plain; charset=utf-8") 7 | ) 8 | 9 | var ( 10 | strSlash = []byte("/") 11 | strSlashSlash = []byte("//") 12 | strSlashDotDot = []byte("/..") 13 | strSlashDotSlash = []byte("/./") 14 | strSlashDotDotSlash = []byte("/../") 15 | strCRLF = []byte("\r\n") 16 | strHTTP = []byte("http") 17 | strHTTPS = []byte("https") 18 | strHTTP11 = []byte("HTTP/1.1") 19 | strColonSlashSlash = []byte("://") 20 | strColonSpace = []byte(": ") 21 | strGMT = []byte("GMT") 22 | 23 | strResponseContinue = []byte("HTTP/1.1 100 Continue\r\n\r\n") 24 | 25 | strGet = []byte("GET") 26 | strHead = []byte("HEAD") 27 | strPost = []byte("POST") 28 | strPut = []byte("PUT") 29 | strDelete = []byte("DELETE") 30 | strConnect = []byte("CONNECT") 31 | strOptions = []byte("OPTIONS") 32 | strTrace = []byte("TRACE") 33 | strPatch = []byte("PATCH") 34 | 35 | strExpect = []byte("Expect") 36 | strConnection = []byte("Connection") 37 | strContentLength = []byte("Content-Length") 38 | strContentType = []byte("Content-Type") 39 | strDate = []byte("Date") 40 | strHost = []byte("Host") 41 | strReferer = []byte("Referer") 42 | strServer = []byte("Server") 43 | strTransferEncoding = []byte("Transfer-Encoding") 44 | strContentEncoding = []byte("Content-Encoding") 45 | strAcceptEncoding = []byte("Accept-Encoding") 46 | strUserAgent = []byte("User-Agent") 47 | strCookie = []byte("Cookie") 48 | strSetCookie = []byte("Set-Cookie") 49 | strLocation = []byte("Location") 50 | strIfModifiedSince = []byte("If-Modified-Since") 51 | strLastModified = []byte("Last-Modified") 52 | strAcceptRanges = []byte("Accept-Ranges") 53 | strRange = []byte("Range") 54 | strContentRange = []byte("Content-Range") 55 | 56 | strCookieExpires = []byte("expires") 57 | strCookieDomain = []byte("domain") 58 | strCookiePath = []byte("path") 59 | strCookieHTTPOnly = []byte("HttpOnly") 60 | strCookieSecure = []byte("secure") 61 | strCookieMaxAge = []byte("max-age") 62 | strCookieSameSite = []byte("SameSite") 63 | strCookieSameSiteLax = []byte("Lax") 64 | strCookieSameSiteStrict = []byte("Strict") 65 | 66 | strClose = []byte("close") 67 | strGzip = []byte("gzip") 68 | strDeflate = []byte("deflate") 69 | strKeepAlive = []byte("keep-alive") 70 | strKeepAliveCamelCase = []byte("Keep-Alive") 71 | strUpgrade = []byte("Upgrade") 72 | strChunked = []byte("chunked") 73 | strIdentity = []byte("identity") 74 | str100Continue = []byte("100-continue") 75 | strPostArgsContentType = []byte("application/x-www-form-urlencoded") 76 | strMultipartFormData = []byte("multipart/form-data") 77 | strBoundary = []byte("boundary") 78 | strBytes = []byte("bytes") 79 | strTextSlash = []byte("text/") 80 | strApplicationSlash = []byte("application/") 81 | ) 82 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/timer.go: -------------------------------------------------------------------------------- 1 | package fasthttp 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | func initTimer(t *time.Timer, timeout time.Duration) *time.Timer { 9 | if t == nil { 10 | return time.NewTimer(timeout) 11 | } 12 | if t.Reset(timeout) { 13 | panic("BUG: active timer trapped into initTimer()") 14 | } 15 | return t 16 | } 17 | 18 | func stopTimer(t *time.Timer) { 19 | if !t.Stop() { 20 | // Collect possibly added time from the channel 21 | // if timer has been stopped and nobody collected its' value. 22 | select { 23 | case <-t.C: 24 | default: 25 | } 26 | } 27 | } 28 | 29 | func acquireTimer(timeout time.Duration) *time.Timer { 30 | v := timerPool.Get() 31 | if v == nil { 32 | return time.NewTimer(timeout) 33 | } 34 | t := v.(*time.Timer) 35 | initTimer(t, timeout) 36 | return t 37 | } 38 | 39 | func releaseTimer(t *time.Timer) { 40 | stopTimer(t) 41 | timerPool.Put(t) 42 | } 43 | 44 | var timerPool sync.Pool 45 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/uri_unix.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | package fasthttp 4 | 5 | func addLeadingSlash(dst, src []byte) []byte { 6 | // add leading slash for unix paths 7 | if len(src) == 0 || src[0] != '/' { 8 | dst = append(dst, '/') 9 | } 10 | 11 | return dst 12 | } 13 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/uri_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | package fasthttp 4 | 5 | func addLeadingSlash(dst, src []byte) []byte { 6 | // zero length and "C:/" case 7 | if len(src) == 0 || (len(src) > 2 && src[1] != ':') { 8 | dst = append(dst, '/') 9 | } 10 | 11 | return dst 12 | } 13 | -------------------------------------------------------------------------------- /vendor/github.com/valyala/fasthttp/userdata.go: -------------------------------------------------------------------------------- 1 | package fasthttp 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | type userDataKV struct { 8 | key []byte 9 | value interface{} 10 | } 11 | 12 | type userData []userDataKV 13 | 14 | func (d *userData) Set(key string, value interface{}) { 15 | args := *d 16 | n := len(args) 17 | for i := 0; i < n; i++ { 18 | kv := &args[i] 19 | if string(kv.key) == key { 20 | kv.value = value 21 | return 22 | } 23 | } 24 | 25 | c := cap(args) 26 | if c > n { 27 | args = args[:n+1] 28 | kv := &args[n] 29 | kv.key = append(kv.key[:0], key...) 30 | kv.value = value 31 | *d = args 32 | return 33 | } 34 | 35 | kv := userDataKV{} 36 | kv.key = append(kv.key[:0], key...) 37 | kv.value = value 38 | *d = append(args, kv) 39 | } 40 | 41 | func (d *userData) SetBytes(key []byte, value interface{}) { 42 | d.Set(b2s(key), value) 43 | } 44 | 45 | func (d *userData) Get(key string) interface{} { 46 | args := *d 47 | n := len(args) 48 | for i := 0; i < n; i++ { 49 | kv := &args[i] 50 | if string(kv.key) == key { 51 | return kv.value 52 | } 53 | } 54 | return nil 55 | } 56 | 57 | func (d *userData) GetBytes(key []byte) interface{} { 58 | return d.Get(b2s(key)) 59 | } 60 | 61 | func (d *userData) Reset() { 62 | args := *d 63 | n := len(args) 64 | for i := 0; i < n; i++ { 65 | v := args[i].value 66 | if vc, ok := v.(io.Closer); ok { 67 | vc.Close() 68 | } 69 | } 70 | *d = (*d)[:0] 71 | } 72 | -------------------------------------------------------------------------------- /vendor/github.com/willf/bitset/.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | target 27 | -------------------------------------------------------------------------------- /vendor/github.com/willf/bitset/.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | sudo: false 4 | 5 | branches: 6 | except: 7 | - release 8 | 9 | branches: 10 | only: 11 | - master 12 | - travis 13 | 14 | go: 15 | - "1.9.x" 16 | - "1.10.x" 17 | - tip 18 | 19 | matrix: 20 | allow_failures: 21 | - go: tip 22 | 23 | before_install: 24 | - if [ -n "$GH_USER" ]; then git config --global github.user ${GH_USER}; fi; 25 | - if [ -n "$GH_TOKEN" ]; then git config --global github.token ${GH_TOKEN}; fi; 26 | - go get github.com/mattn/goveralls 27 | 28 | before_script: 29 | - make deps 30 | 31 | script: 32 | - make qa 33 | 34 | after_failure: 35 | - cat ./target/test/report.xml 36 | 37 | after_success: 38 | - if [ "$TRAVIS_GO_VERSION" = "1.8" ]; then $HOME/gopath/bin/goveralls -covermode=count -coverprofile=target/report/coverage.out -service=travis-ci; fi; 39 | -------------------------------------------------------------------------------- /vendor/github.com/willf/bitset/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Will Fitzgerald. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /vendor/github.com/willf/bitset/README.md: -------------------------------------------------------------------------------- 1 | # bitset 2 | 3 | *Go language library to map between non-negative integers and boolean values* 4 | 5 | [![Master Build Status](https://secure.travis-ci.org/willf/bitset.png?branch=master)](https://travis-ci.org/willf/bitset?branch=master) 6 | [![Master Coverage Status](https://coveralls.io/repos/willf/bitset/badge.svg?branch=master&service=github)](https://coveralls.io/github/willf/bitset?branch=master) 7 | [![Go Report Card](https://goreportcard.com/badge/github.com/willf/bitset)](https://goreportcard.com/report/github.com/willf/bitset) 8 | [![GoDoc](https://godoc.org/github.com/willf/bitset?status.svg)](http://godoc.org/github.com/willf/bitset) 9 | 10 | 11 | ## Description 12 | 13 | Package bitset implements bitsets, a mapping between non-negative integers and boolean values. 14 | It should be more efficient than map[uint] bool. 15 | 16 | It provides methods for setting, clearing, flipping, and testing individual integers. 17 | 18 | But it also provides set intersection, union, difference, complement, and symmetric operations, as well as tests to check whether any, all, or no bits are set, and querying a bitset's current length and number of positive bits. 19 | 20 | BitSets are expanded to the size of the largest set bit; the memory allocation is approximately Max bits, where Max is the largest set bit. BitSets are never shrunk. On creation, a hint can be given for the number of bits that will be used. 21 | 22 | Many of the methods, including Set, Clear, and Flip, return a BitSet pointer, which allows for chaining. 23 | 24 | ### Example use: 25 | 26 | ```go 27 | package main 28 | 29 | import ( 30 | "fmt" 31 | "math/rand" 32 | 33 | "github.com/willf/bitset" 34 | ) 35 | 36 | func main() { 37 | fmt.Printf("Hello from BitSet!\n") 38 | var b bitset.BitSet 39 | // play some Go Fish 40 | for i := 0; i < 100; i++ { 41 | card1 := uint(rand.Intn(52)) 42 | card2 := uint(rand.Intn(52)) 43 | b.Set(card1) 44 | if b.Test(card2) { 45 | fmt.Println("Go Fish!") 46 | } 47 | b.Clear(card1) 48 | } 49 | 50 | // Chaining 51 | b.Set(10).Set(11) 52 | 53 | for i, e := b.NextSet(0); e; i, e = b.NextSet(i + 1) { 54 | fmt.Println("The following bit is set:", i) 55 | } 56 | if b.Intersection(bitset.New(100).Set(10)).Count() == 1 { 57 | fmt.Println("Intersection works.") 58 | } else { 59 | fmt.Println("Intersection doesn't work???") 60 | } 61 | } 62 | ``` 63 | 64 | As an alternative to BitSets, one should check out the 'big' package, which provides a (less set-theoretical) view of bitsets. 65 | 66 | Godoc documentation is at: https://godoc.org/github.com/willf/bitset 67 | 68 | 69 | ## Implementation Note 70 | 71 | Go 1.9 introduced a native `math/bits` library. We provide backward compatibility to Go 1.7, which might be removed. 72 | 73 | It is possible that a later version will match the `math/bits` return signature for counts (which is `int`, rather than our library's `unit64`). If so, the version will be bumped. 74 | 75 | ## Installation 76 | 77 | ```bash 78 | go get github.com/willf/bitset 79 | ``` 80 | 81 | ## Contributing 82 | 83 | If you wish to contribute to this project, please branch and issue a pull request against master ("[GitHub Flow](https://guides.github.com/introduction/flow/)") 84 | 85 | This project include a Makefile that allows you to test and build the project with simple commands. 86 | To see all available options: 87 | ```bash 88 | make help 89 | ``` 90 | 91 | ## Running all tests 92 | 93 | Before committing the code, please check if it passes all tests using (note: this will install some dependencies): 94 | ```bash 95 | make qa 96 | ``` 97 | -------------------------------------------------------------------------------- /vendor/github.com/willf/bitset/VERSION: -------------------------------------------------------------------------------- 1 | 1.1.3 2 | -------------------------------------------------------------------------------- /vendor/github.com/willf/bitset/popcnt.go: -------------------------------------------------------------------------------- 1 | package bitset 2 | 3 | // bit population count, take from 4 | // https://code.google.com/p/go/issues/detail?id=4988#c11 5 | // credit: https://code.google.com/u/arnehormann/ 6 | func popcount(x uint64) (n uint64) { 7 | x -= (x >> 1) & 0x5555555555555555 8 | x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 9 | x += x >> 4 10 | x &= 0x0f0f0f0f0f0f0f0f 11 | x *= 0x0101010101010101 12 | return x >> 56 13 | } 14 | 15 | func popcntSliceGo(s []uint64) uint64 { 16 | cnt := uint64(0) 17 | for _, x := range s { 18 | cnt += popcount(x) 19 | } 20 | return cnt 21 | } 22 | 23 | func popcntMaskSliceGo(s, m []uint64) uint64 { 24 | cnt := uint64(0) 25 | for i := range s { 26 | cnt += popcount(s[i] &^ m[i]) 27 | } 28 | return cnt 29 | } 30 | 31 | func popcntAndSliceGo(s, m []uint64) uint64 { 32 | cnt := uint64(0) 33 | for i := range s { 34 | cnt += popcount(s[i] & m[i]) 35 | } 36 | return cnt 37 | } 38 | 39 | func popcntOrSliceGo(s, m []uint64) uint64 { 40 | cnt := uint64(0) 41 | for i := range s { 42 | cnt += popcount(s[i] | m[i]) 43 | } 44 | return cnt 45 | } 46 | 47 | func popcntXorSliceGo(s, m []uint64) uint64 { 48 | cnt := uint64(0) 49 | for i := range s { 50 | cnt += popcount(s[i] ^ m[i]) 51 | } 52 | return cnt 53 | } 54 | -------------------------------------------------------------------------------- /vendor/github.com/willf/bitset/popcnt_19.go: -------------------------------------------------------------------------------- 1 | // +build go1.9 2 | 3 | package bitset 4 | 5 | import "math/bits" 6 | 7 | func popcntSlice(s []uint64) uint64 { 8 | var cnt int 9 | for _, x := range s { 10 | cnt += bits.OnesCount64(x) 11 | } 12 | return uint64(cnt) 13 | } 14 | 15 | func popcntMaskSlice(s, m []uint64) uint64 { 16 | var cnt int 17 | for i := range s { 18 | cnt += bits.OnesCount64(s[i] &^ m[i]) 19 | } 20 | return uint64(cnt) 21 | } 22 | 23 | func popcntAndSlice(s, m []uint64) uint64 { 24 | var cnt int 25 | for i := range s { 26 | cnt += bits.OnesCount64(s[i] & m[i]) 27 | } 28 | return uint64(cnt) 29 | } 30 | 31 | func popcntOrSlice(s, m []uint64) uint64 { 32 | var cnt int 33 | for i := range s { 34 | cnt += bits.OnesCount64(s[i] | m[i]) 35 | } 36 | return uint64(cnt) 37 | } 38 | 39 | func popcntXorSlice(s, m []uint64) uint64 { 40 | var cnt int 41 | for i := range s { 42 | cnt += bits.OnesCount64(s[i] ^ m[i]) 43 | } 44 | return uint64(cnt) 45 | } 46 | -------------------------------------------------------------------------------- /vendor/github.com/willf/bitset/popcnt_amd64.go: -------------------------------------------------------------------------------- 1 | // +build !go1.9 2 | // +build amd64,!appengine 3 | 4 | package bitset 5 | 6 | // *** the following functions are defined in popcnt_amd64.s 7 | 8 | //go:noescape 9 | 10 | func hasAsm() bool 11 | 12 | // useAsm is a flag used to select the GO or ASM implementation of the popcnt function 13 | var useAsm = hasAsm() 14 | 15 | //go:noescape 16 | 17 | func popcntSliceAsm(s []uint64) uint64 18 | 19 | //go:noescape 20 | 21 | func popcntMaskSliceAsm(s, m []uint64) uint64 22 | 23 | //go:noescape 24 | 25 | func popcntAndSliceAsm(s, m []uint64) uint64 26 | 27 | //go:noescape 28 | 29 | func popcntOrSliceAsm(s, m []uint64) uint64 30 | 31 | //go:noescape 32 | 33 | func popcntXorSliceAsm(s, m []uint64) uint64 34 | 35 | func popcntSlice(s []uint64) uint64 { 36 | if useAsm { 37 | return popcntSliceAsm(s) 38 | } 39 | return popcntSliceGo(s) 40 | } 41 | 42 | func popcntMaskSlice(s, m []uint64) uint64 { 43 | if useAsm { 44 | return popcntMaskSliceAsm(s, m) 45 | } 46 | return popcntMaskSliceGo(s, m) 47 | } 48 | 49 | func popcntAndSlice(s, m []uint64) uint64 { 50 | if useAsm { 51 | return popcntAndSliceAsm(s, m) 52 | } 53 | return popcntAndSliceGo(s, m) 54 | } 55 | 56 | func popcntOrSlice(s, m []uint64) uint64 { 57 | if useAsm { 58 | return popcntOrSliceAsm(s, m) 59 | } 60 | return popcntOrSliceGo(s, m) 61 | } 62 | 63 | func popcntXorSlice(s, m []uint64) uint64 { 64 | if useAsm { 65 | return popcntXorSliceAsm(s, m) 66 | } 67 | return popcntXorSliceGo(s, m) 68 | } 69 | -------------------------------------------------------------------------------- /vendor/github.com/willf/bitset/popcnt_amd64.s: -------------------------------------------------------------------------------- 1 | // +build !go1.9 2 | // +build amd64,!appengine 3 | 4 | TEXT ·hasAsm(SB),4,$0-1 5 | MOVQ $1, AX 6 | CPUID 7 | SHRQ $23, CX 8 | ANDQ $1, CX 9 | MOVB CX, ret+0(FP) 10 | RET 11 | 12 | #define POPCNTQ_DX_DX BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0xd2 13 | 14 | TEXT ·popcntSliceAsm(SB),4,$0-32 15 | XORQ AX, AX 16 | MOVQ s+0(FP), SI 17 | MOVQ s_len+8(FP), CX 18 | TESTQ CX, CX 19 | JZ popcntSliceEnd 20 | popcntSliceLoop: 21 | BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0x16 // POPCNTQ (SI), DX 22 | ADDQ DX, AX 23 | ADDQ $8, SI 24 | LOOP popcntSliceLoop 25 | popcntSliceEnd: 26 | MOVQ AX, ret+24(FP) 27 | RET 28 | 29 | TEXT ·popcntMaskSliceAsm(SB),4,$0-56 30 | XORQ AX, AX 31 | MOVQ s+0(FP), SI 32 | MOVQ s_len+8(FP), CX 33 | TESTQ CX, CX 34 | JZ popcntMaskSliceEnd 35 | MOVQ m+24(FP), DI 36 | popcntMaskSliceLoop: 37 | MOVQ (DI), DX 38 | NOTQ DX 39 | ANDQ (SI), DX 40 | POPCNTQ_DX_DX 41 | ADDQ DX, AX 42 | ADDQ $8, SI 43 | ADDQ $8, DI 44 | LOOP popcntMaskSliceLoop 45 | popcntMaskSliceEnd: 46 | MOVQ AX, ret+48(FP) 47 | RET 48 | 49 | TEXT ·popcntAndSliceAsm(SB),4,$0-56 50 | XORQ AX, AX 51 | MOVQ s+0(FP), SI 52 | MOVQ s_len+8(FP), CX 53 | TESTQ CX, CX 54 | JZ popcntAndSliceEnd 55 | MOVQ m+24(FP), DI 56 | popcntAndSliceLoop: 57 | MOVQ (DI), DX 58 | ANDQ (SI), DX 59 | POPCNTQ_DX_DX 60 | ADDQ DX, AX 61 | ADDQ $8, SI 62 | ADDQ $8, DI 63 | LOOP popcntAndSliceLoop 64 | popcntAndSliceEnd: 65 | MOVQ AX, ret+48(FP) 66 | RET 67 | 68 | TEXT ·popcntOrSliceAsm(SB),4,$0-56 69 | XORQ AX, AX 70 | MOVQ s+0(FP), SI 71 | MOVQ s_len+8(FP), CX 72 | TESTQ CX, CX 73 | JZ popcntOrSliceEnd 74 | MOVQ m+24(FP), DI 75 | popcntOrSliceLoop: 76 | MOVQ (DI), DX 77 | ORQ (SI), DX 78 | POPCNTQ_DX_DX 79 | ADDQ DX, AX 80 | ADDQ $8, SI 81 | ADDQ $8, DI 82 | LOOP popcntOrSliceLoop 83 | popcntOrSliceEnd: 84 | MOVQ AX, ret+48(FP) 85 | RET 86 | 87 | TEXT ·popcntXorSliceAsm(SB),4,$0-56 88 | XORQ AX, AX 89 | MOVQ s+0(FP), SI 90 | MOVQ s_len+8(FP), CX 91 | TESTQ CX, CX 92 | JZ popcntXorSliceEnd 93 | MOVQ m+24(FP), DI 94 | popcntXorSliceLoop: 95 | MOVQ (DI), DX 96 | XORQ (SI), DX 97 | POPCNTQ_DX_DX 98 | ADDQ DX, AX 99 | ADDQ $8, SI 100 | ADDQ $8, DI 101 | LOOP popcntXorSliceLoop 102 | popcntXorSliceEnd: 103 | MOVQ AX, ret+48(FP) 104 | RET 105 | -------------------------------------------------------------------------------- /vendor/github.com/willf/bitset/popcnt_generic.go: -------------------------------------------------------------------------------- 1 | // +build !go1.9 2 | // +build !amd64 appengine 3 | 4 | package bitset 5 | 6 | func popcntSlice(s []uint64) uint64 { 7 | return popcntSliceGo(s) 8 | } 9 | 10 | func popcntMaskSlice(s, m []uint64) uint64 { 11 | return popcntMaskSliceGo(s, m) 12 | } 13 | 14 | func popcntAndSlice(s, m []uint64) uint64 { 15 | return popcntAndSliceGo(s, m) 16 | } 17 | 18 | func popcntOrSlice(s, m []uint64) uint64 { 19 | return popcntOrSliceGo(s, m) 20 | } 21 | 22 | func popcntXorSlice(s, m []uint64) uint64 { 23 | return popcntXorSliceGo(s, m) 24 | } 25 | -------------------------------------------------------------------------------- /vendor/github.com/willf/bitset/trailing_zeros_18.go: -------------------------------------------------------------------------------- 1 | // +build !go1.9 2 | 3 | package bitset 4 | 5 | var deBruijn = [...]byte{ 6 | 0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4, 7 | 62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5, 8 | 63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11, 9 | 54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, 10 | } 11 | 12 | func trailingZeroes64(v uint64) uint { 13 | return uint(deBruijn[((v&-v)*0x03f79d71b4ca8b09)>>58]) 14 | } 15 | -------------------------------------------------------------------------------- /vendor/github.com/willf/bitset/trailing_zeros_19.go: -------------------------------------------------------------------------------- 1 | // +build go1.9 2 | 3 | package bitset 4 | 5 | import "math/bits" 6 | 7 | func trailingZeroes64(v uint64) uint { 8 | return uint(bits.TrailingZeros64(v)) 9 | } 10 | --------------------------------------------------------------------------------