├── .dockerignore ├── .gitignore ├── Dockerfile ├── README.md ├── http.go ├── index_country.go ├── index_location.go ├── index_user.go ├── index_visit.go ├── json.go ├── location.go ├── locations_avg.go ├── main.go ├── request.go ├── socket.go ├── stats.go ├── string_constants.go ├── user.go ├── user_visits.go ├── utils.go └── visit.go /.dockerignore: -------------------------------------------------------------------------------- 1 | .git/ 2 | .idea/ 3 | tmp/ 4 | epoll_server/ 5 | *.sh 6 | *.go 7 | highloadcup_tester 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | mr_hlc 2 | tmp/ 3 | .idea/ 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.5 2 | ADD mr_hlc . 3 | EXPOSE 80 4 | CMD ./mr_hlc 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bicycle-mrhlc 2 | Мое решение для [mail.ru'шного HighLoad Cup 2017](https://highloadcup.ru/round/1/) 3 | 4 | Версия "как есть", как была залита в последний раз. 5 | 6 | Комментариев не много, а некоторые еще могут быть и не актуальными, все как в жизни. 7 | -------------------------------------------------------------------------------- /http.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "log" 6 | "runtime" 7 | "strconv" 8 | "sync" 9 | "sync/atomic" 10 | "syscall" 11 | ) 12 | 13 | type ( 14 | Method int 15 | parseRequestStatus int 16 | parseRequestState int 17 | ) 18 | 19 | const ( 20 | MethodGET = Method(iota) 21 | MethodPOST 22 | ) 23 | 24 | const ( 25 | parseRequestStatusOk = parseRequestStatus(iota) 26 | parseRequestStatusBadRequest 27 | parseRequestStatusNeedMore 28 | ) 29 | 30 | const ( 31 | parseRequestStateBegin = parseRequestState(iota) 32 | parseRequestStateHeaders 33 | parseRequestStateBody 34 | ) 35 | 36 | type ( 37 | RequestCtx struct { 38 | state parseRequestState 39 | contentLength int 40 | keepAlive bool 41 | 42 | Method Method 43 | Path []byte 44 | Body []byte 45 | 46 | ResponseStatus int 47 | ResponseBody []byte 48 | 49 | inputBuf []byte 50 | inputBufOffs int 51 | outputBuf []byte 52 | 53 | UserBuf []byte // может использоваться внутри RequestHandler как угодно, сервер его не трогает 54 | } 55 | 56 | RequestHandler func(ctx *RequestCtx) 57 | 58 | HTTPServer struct { 59 | Handler RequestHandler 60 | httpCurrentConnections int32 61 | } 62 | ) 63 | 64 | func (c *RequestCtx) Reset() { 65 | c.state = parseRequestStateBegin 66 | c.contentLength = 0 67 | c.keepAlive = false 68 | 69 | c.Method = MethodGET 70 | c.Path = nil 71 | c.Body = nil 72 | 73 | c.ResponseStatus = 200 74 | c.ResponseBody = nil 75 | 76 | c.inputBufOffs = 0 77 | if cap(c.inputBuf) == 0 { 78 | c.inputBuf = make([]byte, 16*1024, 16*1024) 79 | } else { 80 | c.inputBuf = c.inputBuf[:] 81 | } 82 | 83 | if cap(c.outputBuf) == 0 { 84 | c.outputBuf = make([]byte, 0, 16*1024) 85 | } else { 86 | c.outputBuf = c.outputBuf[:0] 87 | } 88 | 89 | if cap(c.UserBuf) == 0 { 90 | c.UserBuf = make([]byte, 0, 16*1024) 91 | } else { 92 | c.UserBuf = c.UserBuf[:0] 93 | } 94 | } 95 | 96 | func (s *HTTPServer) GetCurrentConnections() int32 { 97 | return atomic.LoadInt32(&s.httpCurrentConnections) 98 | } 99 | 100 | func (s *HTTPServer) ListenAndServe(port int) error { 101 | cpu := runtime.NumCPU() 102 | if cpu > 4 { 103 | cpu = 4 104 | } 105 | log.Println(`Use`, cpu, `listeners`) 106 | 107 | serverFds := make([]int, 0, cpu) 108 | epollFds := make([]int, 0, cpu) 109 | 110 | defer func() { 111 | for i := range serverFds { 112 | serverFd := serverFds[i] 113 | epollFd := epollFds[i] 114 | 115 | syscall.Close(epollFd) 116 | syscall.Close(serverFd) 117 | } 118 | }() 119 | 120 | for i := 1; i <= cpu; i++ { 121 | if serverFd, err := socketCreateListener(port); err != nil { 122 | return err 123 | } else if epollFd, err := socketCreateListenerEpoll(serverFd); err != nil { 124 | syscall.Close(serverFd) 125 | return err 126 | } else { 127 | serverFds = append(serverFds, serverFd) 128 | epollFds = append(epollFds, epollFd) 129 | } 130 | } 131 | 132 | var wg sync.WaitGroup 133 | wg.Add(len(serverFds)) 134 | for i := range serverFds { 135 | serverFd := serverFds[i] 136 | epollFd := epollFds[i] 137 | go s.epollLoop(epollFd, serverFd) 138 | } 139 | wg.Wait() 140 | 141 | return nil 142 | } 143 | 144 | func (s *HTTPServer) epollLoop(epollFd, serverFd int) { 145 | var ( 146 | epollEvent syscall.EpollEvent 147 | epollEvents [MaxEpollEvents]syscall.EpollEvent 148 | 149 | activeCtx = make(map[int]*RequestCtx, 10) // 2000 ? 150 | usedCtx []*RequestCtx 151 | 152 | ctx *RequestCtx 153 | ) 154 | 155 | closeClientFd := func(fd int) { 156 | if ctx, ok := activeCtx[fd]; !ok { 157 | return 158 | } else { 159 | syscall.Close(fd) 160 | delete(activeCtx, fd) 161 | usedCtx = append(usedCtx, ctx) 162 | } 163 | } 164 | 165 | for { 166 | nEvents, err := syscall.EpollWait(epollFd, epollEvents[:], -1) 167 | if err != nil { 168 | if errno, ok := err.(syscall.Errno); ok && (errno == syscall.EINTR) { 169 | continue 170 | } 171 | 172 | log.Println(`EpollWait: `, err) 173 | break 174 | } 175 | 176 | for ev := 0; ev < nEvents; ev++ { 177 | fd := int(epollEvents[ev].Fd) 178 | events := epollEvents[ev].Events 179 | 180 | if (events&syscall.EPOLLERR != 0) || (events&syscall.EPOLLHUP != 0) || (events&syscall.EPOLLIN == 0) { 181 | closeClientFd(fd) 182 | continue 183 | } else if fd == serverFd { 184 | for { 185 | connFd, _, err := syscall.Accept(serverFd) 186 | 187 | if err != nil { 188 | if errno, ok := err.(syscall.Errno); ok { 189 | if errno == syscall.EAGAIN { 190 | // обработаны все новые коннекты 191 | } else { 192 | log.Printf("Accept: errno: %v\n", errno) 193 | } 194 | } else { 195 | log.Printf("Accept: %T %s\n", err, err) 196 | } 197 | break 198 | } else if err := socketSetNonBlocking(connFd); err != nil { 199 | log.Println("setSocketNonBlocking: ", err) 200 | break 201 | } 202 | 203 | epollEvent.Events = syscall.EPOLLIN | EPOLLET // | syscall.EPOLLOUT 204 | epollEvent.Fd = int32(connFd) 205 | if err := syscall.EpollCtl(epollFd, syscall.EPOLL_CTL_ADD, connFd, &epollEvent); err != nil { 206 | log.Println("EpollCtl: ", err) 207 | break 208 | } 209 | } 210 | 211 | continue 212 | } 213 | 214 | // обработка основых соединений 215 | 216 | var ok bool 217 | if ctx, ok = activeCtx[fd]; !ok { 218 | if l := len(usedCtx); l > 0 { 219 | ctx = usedCtx[l-1] 220 | usedCtx = usedCtx[:l-1] 221 | } else { 222 | ctx = &RequestCtx{} 223 | } 224 | ctx.Reset() 225 | activeCtx[fd] = ctx 226 | } 227 | 228 | for { 229 | nbytes, err := syscall.Read(fd, ctx.inputBuf[ctx.inputBufOffs:]) 230 | 231 | if err != nil { 232 | allOk := false 233 | if errno, ok := err.(syscall.Errno); ok { 234 | if errno == syscall.EAGAIN { 235 | // обработаны все новые данные 236 | allOk = true 237 | } else if errno == syscall.EBADF { 238 | // видимо, соединение уже закрылось и так чуть раньше по другому условию 239 | } else { 240 | log.Printf("Read: unknown errno: %v\n", errno) 241 | } 242 | } else { 243 | log.Printf("Read: unknown error type %T: %s\n", err, err) 244 | } 245 | 246 | if !allOk { 247 | closeClientFd(fd) 248 | } 249 | 250 | break 251 | } 252 | 253 | if nbytes > 0 { 254 | status, bufNew := s.parseRequest(ctx.inputBuf[ctx.inputBufOffs:ctx.inputBufOffs+nbytes], ctx) 255 | 256 | switch status { 257 | case parseRequestStatusOk: 258 | if s.Handler != nil { 259 | s.Handler(ctx) 260 | } 261 | case parseRequestStatusNeedMore: 262 | ctx.inputBufOffs += nbytes 263 | continue 264 | case parseRequestStatusBadRequest: 265 | ctx.ResponseStatus = 400 266 | } 267 | 268 | buf := s.buildResponse(ctx) 269 | if n, err := syscall.Write(fd, buf); err != nil { 270 | log.Println(`Write`, err) 271 | } else if n != len(buf) { 272 | log.Println(`Write`, n, `!=`, len(buf)) 273 | } 274 | 275 | if ctx.Method == MethodPOST { 276 | // яндекс.Танк не умеет в нормальные POST запросы, присылая два лишних байта "\r\n" 277 | closeClientFd(fd) 278 | // ToDo: вычитывать их и работать дальше? ;) 279 | break 280 | } else { 281 | ctx.Reset() 282 | 283 | if len(bufNew) > 0 { 284 | log.Println(`buffer tail`, len(bufNew), string(bufNew)) 285 | copy(ctx.inputBuf, bufNew) 286 | ctx.inputBufOffs = len(bufNew) 287 | } else { 288 | ctx.inputBufOffs = 0 289 | } 290 | } 291 | 292 | } else { 293 | // соединение закрылось 294 | closeClientFd(fd) 295 | } 296 | } 297 | } 298 | } 299 | } 300 | 301 | func (s *HTTPServer) buildResponse(ctx *RequestCtx) []byte { 302 | var line []byte 303 | switch ctx.ResponseStatus { 304 | case 200: 305 | line = line200 306 | case 400: 307 | line = line400 308 | case 404: 309 | line = line404 310 | default: 311 | line = line500 312 | } 313 | 314 | tmpBuf := ctx.outputBuf 315 | 316 | // формирование ответа 317 | tmpBuf = append(tmpBuf[:0], line...) 318 | 319 | tmpBuf = append(tmpBuf, "Content-Type: application/json\r\nServer: yocto_http\r\n"...) 320 | 321 | tmpBuf = append(tmpBuf, "Content-Length: "...) 322 | tmpBuf = strconv.AppendUint(tmpBuf, uint64(len(ctx.ResponseBody)), 10) 323 | tmpBuf = append(tmpBuf, "\r\n"...) 324 | 325 | if ctx.keepAlive { 326 | tmpBuf = append(tmpBuf, "Connection: keep-alive\r\n"...) 327 | } 328 | 329 | tmpBuf = append(tmpBuf, "\r\n"...) 330 | 331 | if ctx.ResponseBody != nil { 332 | tmpBuf = append(tmpBuf, ctx.ResponseBody...) 333 | } 334 | 335 | ctx.outputBuf = tmpBuf // на случай расширения буфера 336 | 337 | return tmpBuf 338 | } 339 | 340 | func (s *HTTPServer) parseRequest(buf []byte, ctx *RequestCtx) (parseRequestStatus, []byte) { 341 | var idx int 342 | 343 | for { 344 | switch ctx.state { 345 | case parseRequestStateBegin: 346 | if idx = bytes.IndexByte(buf, '\n'); idx == -1 { 347 | return parseRequestStatusNeedMore, buf 348 | } 349 | 350 | ctx.state = parseRequestStateHeaders 351 | 352 | // GET /path/to/file HTTP/1.1 353 | line := buf[:idx-1] // \r\n 354 | buf = buf[idx+1:] 355 | if idx = bytes.IndexByte(line, ' '); idx == -1 { 356 | return parseRequestStatusBadRequest, buf 357 | } 358 | 359 | // GET 360 | if method := line[0:idx]; bytes.Equal(method, strGET) { 361 | ctx.Method = MethodGET 362 | } else if bytes.Equal(method, strPOST) { 363 | ctx.Method = MethodPOST 364 | } else { 365 | return parseRequestStatusBadRequest, buf 366 | } 367 | line = line[idx+1:] 368 | 369 | // /path/to/file 370 | if idx = bytes.IndexByte(line, ' '); idx == -1 { 371 | return parseRequestStatusBadRequest, buf 372 | } 373 | ctx.Path = line[:idx] 374 | 375 | // HTTP/1.1 поддерживает keep-alive по умолчанию 376 | if bytes.HasSuffix(line, str11) { 377 | ctx.keepAlive = true 378 | } 379 | 380 | case parseRequestStateHeaders: 381 | if idx = bytes.IndexByte(buf, '\n'); idx == -1 { 382 | return parseRequestStatusNeedMore, buf 383 | } 384 | 385 | line := buf[:idx-1] // \r\n 386 | buf = buf[idx+1:] 387 | if len(line) == 0 { 388 | if ctx.contentLength == 0 { 389 | // все, распарсили запрос 390 | return parseRequestStatusOk, buf 391 | } else { 392 | ctx.state = parseRequestStateBody 393 | } 394 | } else if idx = bytes.IndexByte(line, ':'); idx == -1 { 395 | return parseRequestStatusBadRequest, buf 396 | } else { 397 | key, value := line[:idx], bytesTrimLeftInplace(line[idx+1:]) 398 | bytesToLowerInplace(key) 399 | 400 | if bytes.Equal(key, strContentLength) { 401 | if i64, ok := byteSliceToInt64(value); !ok { 402 | return parseRequestStatusBadRequest, buf 403 | } else { 404 | ctx.contentLength = int(i64) 405 | } 406 | } else if bytes.Equal(key, strConnection) { 407 | bytesToLowerInplace(value) 408 | if bytes.Equal(value, strKeepAlive) { 409 | ctx.keepAlive = true 410 | } else if bytes.Equal(value, strClose) { 411 | ctx.keepAlive = false 412 | } 413 | } 414 | } 415 | 416 | case parseRequestStateBody: 417 | l := ctx.contentLength 418 | if len(buf) < l { 419 | return parseRequestStatusNeedMore, buf 420 | } 421 | 422 | ctx.Body = buf[:l] 423 | buf = buf[l:] 424 | 425 | return parseRequestStatusOk, buf 426 | 427 | default: 428 | log.Println(`Bug in code: unexpected parse state`) 429 | return parseRequestStatusBadRequest, buf 430 | } 431 | } 432 | 433 | log.Println(`Bug in code: after of HTTPServer.parseRequest loop`) 434 | return parseRequestStatusBadRequest, buf 435 | } 436 | -------------------------------------------------------------------------------- /index_country.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "sync" 6 | ) 7 | 8 | type ( 9 | IndexCountry struct { 10 | names [][]byte 11 | rwLock sync.RWMutex 12 | } 13 | ) 14 | 15 | func MakeIndexCountry() *IndexCountry { 16 | index := &IndexCountry{} 17 | index.names = make([][]byte, 0, 300) 18 | index.names = append(index.names, nil) // резервирую 0 индекс, чтобы не выдавался 19 | return index 20 | } 21 | 22 | func (ic *IndexCountry) Add(name []byte) (int, bool) { 23 | ic.rwLock.Lock() 24 | for idx, n := range ic.names { 25 | if bytes.Equal(n, name) { 26 | ic.rwLock.Unlock() 27 | return idx, false 28 | } 29 | } 30 | ic.names = append(ic.names, name) 31 | idx := len(ic.names) - 1 32 | ic.rwLock.Unlock() 33 | return idx, true 34 | } 35 | 36 | func (ic *IndexCountry) Find(name []byte) (idx int) { // 0 - если не найден 37 | ic.rwLock.RLock() 38 | for i, n := range ic.names { 39 | if bytes.Equal(n, name) { 40 | idx = i 41 | break 42 | } 43 | } 44 | ic.rwLock.RUnlock() 45 | return 46 | } 47 | 48 | func (ic *IndexCountry) GetByIdx(idx int) (name []byte) { 49 | ic.rwLock.RLock() 50 | if (idx >= 0) && (idx < len(ic.names)) { 51 | name = ic.names[idx] 52 | } 53 | ic.rwLock.RUnlock() 54 | return 55 | } 56 | -------------------------------------------------------------------------------- /index_location.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | const ( 8 | locationsPreallocCount = 800 * 1000 9 | ) 10 | 11 | type ( 12 | IndexLocation struct { 13 | locations []Location 14 | locationsExtra map[int32]*Location // для тех, чей id не вписывается в норму 15 | rwLock sync.RWMutex 16 | } 17 | ) 18 | 19 | func MakeIndexLocation() *IndexLocation { 20 | return &IndexLocation{ 21 | locations: make([]Location, locationsPreallocCount), 22 | locationsExtra: make(map[int32]*Location), 23 | } 24 | } 25 | 26 | func (il *IndexLocation) Add(location *Location) bool { 27 | var ok bool 28 | 29 | if location.Id < locationsPreallocCount { 30 | il.rwLock.RLock() 31 | if il.locations[location.Id].Id == location.Id { 32 | ok = true 33 | } 34 | il.rwLock.RUnlock() 35 | if ok { 36 | return false 37 | } 38 | 39 | il.rwLock.Lock() 40 | if il.locations[location.Id].Id == location.Id { 41 | il.rwLock.Unlock() 42 | return false 43 | } 44 | 45 | il.locations[location.Id] = *location 46 | 47 | il.rwLock.Unlock() 48 | 49 | return true 50 | } 51 | 52 | // id вне заранее выделенного диапазона. задействуем map 53 | 54 | il.rwLock.RLock() 55 | _, ok = il.locationsExtra[location.Id] 56 | il.rwLock.RUnlock() 57 | if ok { 58 | return false 59 | } 60 | 61 | il.rwLock.Lock() 62 | if _, ok = il.locationsExtra[location.Id]; ok { 63 | il.rwLock.Unlock() 64 | return false 65 | } 66 | 67 | il.locationsExtra[location.Id] = location 68 | 69 | il.rwLock.Unlock() 70 | 71 | return true 72 | } 73 | 74 | func (il *IndexLocation) Update(id int32, update *Location) bool { 75 | var ( 76 | location *Location 77 | ok bool 78 | ) 79 | 80 | il.rwLock.RLock() 81 | if id < locationsPreallocCount { 82 | location = &il.locations[id] 83 | ok = location.Id == id 84 | } else { 85 | location, ok = il.locationsExtra[id] 86 | } 87 | il.rwLock.RUnlock() 88 | if !ok { 89 | return false 90 | } 91 | 92 | return location.Update(update) 93 | } 94 | 95 | func (il *IndexLocation) Get(id int32) *Location { 96 | var ( 97 | location *Location 98 | ok bool 99 | ) 100 | 101 | il.rwLock.RLock() 102 | if id < locationsPreallocCount { 103 | location = &il.locations[id] 104 | ok = location.Id == id 105 | } else { 106 | location, ok = il.locationsExtra[id] 107 | } 108 | il.rwLock.RUnlock() 109 | if !ok { 110 | return nil 111 | } 112 | return location 113 | } 114 | -------------------------------------------------------------------------------- /index_user.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | const ( 8 | usersPreallocCount = 1.2 * 1000 * 1000 9 | ) 10 | 11 | type ( 12 | IndexUser struct { 13 | users []User 14 | usersExtra map[int32]*User 15 | rwLock sync.RWMutex 16 | } 17 | ) 18 | 19 | func MakeIndexUser() *IndexUser { 20 | return &IndexUser{ 21 | users: make([]User, usersPreallocCount), 22 | usersExtra: make(map[int32]*User), 23 | } 24 | } 25 | 26 | func (iu *IndexUser) Add(user *User) bool { 27 | var ok bool 28 | 29 | if user.Id < usersPreallocCount { 30 | iu.rwLock.RLock() 31 | if iu.users[user.Id].Id == user.Id { 32 | ok = true 33 | } 34 | iu.rwLock.RUnlock() 35 | if ok { 36 | return false 37 | } 38 | 39 | iu.rwLock.Lock() 40 | if iu.users[user.Id].Id == user.Id { 41 | iu.rwLock.Unlock() 42 | return false 43 | } 44 | 45 | iu.users[user.Id] = *user 46 | 47 | iu.rwLock.Unlock() 48 | 49 | return true 50 | } 51 | 52 | // id вне заранее выделенного диапазона. задействуем map 53 | 54 | iu.rwLock.RLock() 55 | _, ok = iu.usersExtra[user.Id] 56 | iu.rwLock.RUnlock() 57 | if ok { 58 | return false 59 | } 60 | 61 | iu.rwLock.Lock() 62 | if _, ok := iu.usersExtra[user.Id]; ok { 63 | iu.rwLock.Unlock() 64 | return false 65 | } 66 | 67 | iu.usersExtra[user.Id] = user 68 | 69 | iu.rwLock.Unlock() 70 | 71 | return true 72 | } 73 | 74 | func (iu *IndexUser) Update(id int32, update *User) bool { 75 | var ( 76 | user *User 77 | ok bool 78 | ) 79 | 80 | iu.rwLock.RLock() 81 | if id < usersPreallocCount { 82 | user = &iu.users[id] 83 | ok = user.Id == id 84 | } else { 85 | user, ok = iu.usersExtra[id] 86 | } 87 | iu.rwLock.RUnlock() 88 | if !ok { 89 | return false 90 | } 91 | 92 | return user.Update(update) 93 | } 94 | 95 | func (iu *IndexUser) Get(id int32) *User { 96 | var ( 97 | user *User 98 | ok bool 99 | ) 100 | 101 | iu.rwLock.RLock() 102 | if id < usersPreallocCount { 103 | user = &iu.users[id] 104 | ok = user.Id == id 105 | } else { 106 | user, ok = iu.usersExtra[id] 107 | } 108 | iu.rwLock.RUnlock() 109 | if !ok { 110 | return nil 111 | } 112 | return user 113 | } 114 | -------------------------------------------------------------------------------- /index_visit.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | const ( 8 | visitsPreallocCount = 12 * 1000 * 1000 9 | ) 10 | 11 | type ( 12 | IndexVisit struct { 13 | visits []Visit 14 | visitsExtra map[int32]*Visit 15 | rwLock sync.RWMutex 16 | } 17 | ) 18 | 19 | func MakeIndexVisit() *IndexVisit { 20 | return &IndexVisit{ 21 | visits: make([]Visit, visitsPreallocCount), 22 | visitsExtra: make(map[int32]*Visit), 23 | } 24 | } 25 | 26 | func (iv *IndexVisit) Add(visit *Visit) bool { 27 | var ok bool 28 | 29 | if visit.Id < visitsPreallocCount { 30 | iv.rwLock.RLock() 31 | if iv.visits[visit.Id].Id == visit.Id { 32 | ok = true 33 | } 34 | iv.rwLock.RUnlock() 35 | if ok { 36 | return false 37 | } 38 | 39 | iv.rwLock.Lock() 40 | if iv.visits[visit.Id].Id == visit.Id { 41 | iv.rwLock.Unlock() 42 | return false 43 | } 44 | 45 | iv.visits[visit.Id] = *visit 46 | 47 | iv.rwLock.Unlock() 48 | 49 | return true 50 | } 51 | 52 | // id вне заранее выделенного диапазона. задействуем map 53 | 54 | iv.rwLock.RLock() 55 | _, ok = iv.visitsExtra[visit.Id] 56 | iv.rwLock.RUnlock() 57 | if ok { 58 | return false 59 | } 60 | 61 | iv.rwLock.Lock() 62 | if _, ok := iv.visitsExtra[visit.Id]; ok { 63 | iv.rwLock.Unlock() 64 | return false 65 | } 66 | 67 | iv.visitsExtra[visit.Id] = visit 68 | 69 | iv.rwLock.Unlock() 70 | 71 | return true 72 | } 73 | 74 | func (iv *IndexVisit) Update(id int32, update *Visit) bool { 75 | var ( 76 | visit *Visit 77 | ok bool 78 | ) 79 | 80 | iv.rwLock.RLock() 81 | if id < visitsPreallocCount { 82 | visit = &iv.visits[id] 83 | ok = visit.Id == id 84 | } else { 85 | visit, ok = iv.visitsExtra[id] 86 | } 87 | iv.rwLock.RUnlock() 88 | if !ok { 89 | return false 90 | } 91 | 92 | return visit.Update(update) 93 | } 94 | 95 | func (iv *IndexVisit) Get(id int32) *Visit { 96 | var ( 97 | visit *Visit 98 | ok bool 99 | ) 100 | 101 | iv.rwLock.RLock() 102 | if id < visitsPreallocCount { 103 | visit = &iv.visits[id] 104 | ok = visit.Id == id 105 | } else { 106 | visit, ok = iv.visitsExtra[id] 107 | } 108 | iv.rwLock.RUnlock() 109 | if !ok { 110 | return nil 111 | } 112 | return visit 113 | } 114 | -------------------------------------------------------------------------------- /json.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | ) 6 | 7 | type ( 8 | JSValueType int 9 | ) 10 | 11 | const ( 12 | jsValueTypeString = JSValueType(iota) 13 | jsValueTypeNumeric 14 | ) 15 | 16 | func isSpace(ch byte) bool { 17 | return ch == ' ' || ch == '\t' || ch == '\n' 18 | } 19 | 20 | func isNum(ch byte) bool { 21 | return ch >= '0' && ch <= '9' 22 | } 23 | 24 | // {"type": [{item}, {item}, ...]} 25 | func ParseData(buf []byte, expectedType []byte, itemCallback func(item []byte) bool) error { 26 | const ( 27 | stateBegin = iota 28 | stateWaitTypeKey 29 | stateTypeKey 30 | stateWaitTypeColon 31 | stateWaitArray 32 | stateWaitHashmapBegin 33 | stateWaitHashmapEnd 34 | stateAfterHashmapWaitCommaOrArrayEnd 35 | stateWaitEnd 36 | ) 37 | 38 | var ( 39 | state = stateBegin 40 | keyFrom int 41 | hashmapFrom int 42 | ) 43 | 44 | for idx, ch := range buf { 45 | switch state { 46 | case stateBegin: 47 | if ch == '{' { 48 | state = stateWaitTypeKey 49 | } else if !isSpace(ch) { 50 | return ErrWrongData 51 | } 52 | case stateWaitTypeKey: 53 | if ch == '"' { 54 | state = stateTypeKey 55 | keyFrom = idx 56 | } else if !isSpace(ch) { 57 | return ErrWrongData 58 | } 59 | case stateTypeKey: 60 | if ch == '"' { 61 | dataType := buf[keyFrom+1 : idx] 62 | if !bytes.Equal(dataType, expectedType) { 63 | return ErrWrongData 64 | } 65 | state = stateWaitTypeColon 66 | } 67 | case stateWaitTypeColon: 68 | if ch == ':' { 69 | state = stateWaitArray 70 | } else if !isSpace(ch) { 71 | return ErrWrongData 72 | } 73 | case stateWaitArray: 74 | if ch == '[' { 75 | state = stateWaitHashmapBegin 76 | } else if !isSpace(ch) { 77 | return ErrWrongData 78 | } 79 | case stateWaitHashmapBegin: 80 | if ch == '{' { 81 | state = stateWaitHashmapEnd 82 | hashmapFrom = idx 83 | } else if !isSpace(ch) { 84 | return ErrWrongData 85 | } 86 | case stateWaitHashmapEnd: 87 | if ch == '}' { 88 | item := buf[hashmapFrom : idx+1] 89 | if !itemCallback(item) { 90 | return ErrWrongData 91 | } 92 | state = stateAfterHashmapWaitCommaOrArrayEnd 93 | } 94 | case stateAfterHashmapWaitCommaOrArrayEnd: 95 | if ch == ',' { 96 | state = stateWaitHashmapBegin 97 | } else if ch == ']' { 98 | state = stateWaitEnd 99 | } else if !isSpace(ch) { 100 | return ErrWrongData 101 | } 102 | case stateWaitEnd: 103 | if ch == '}' { 104 | return nil 105 | } else if !isSpace(ch) { 106 | return ErrWrongData 107 | } 108 | default: 109 | return ErrWrongData 110 | } 111 | } 112 | 113 | return nil 114 | } 115 | 116 | func ParseItem(buf []byte, fieldCallback func(key, value []byte, valueType JSValueType) bool) error { 117 | const ( 118 | stateBegin = iota 119 | stateWaitKey 120 | stateKey 121 | stateWaitColon 122 | stateWaitValue 123 | stateStringValue 124 | stateNumericValue 125 | statePositiveIntegerValue 126 | stateWaitCommaOrHashmapEnd 127 | ) 128 | 129 | var ( 130 | state = stateBegin 131 | keyFrom, keyTo int 132 | valueFrom int 133 | ) 134 | 135 | for idx, ch := range buf { 136 | switch state { 137 | case stateBegin: 138 | if ch == '{' { 139 | state = stateWaitKey 140 | } else if !isSpace(ch) { 141 | return ErrWrongData 142 | } 143 | case stateWaitKey: 144 | if ch == '"' { 145 | state = stateKey 146 | keyFrom = idx 147 | } else if !isSpace(ch) { 148 | return ErrWrongData 149 | } 150 | case stateKey: 151 | if ch == '"' { 152 | state = stateWaitColon 153 | keyTo = idx 154 | } 155 | case stateWaitColon: 156 | if ch == ':' { 157 | state = stateWaitValue 158 | } else if !isSpace(ch) { 159 | return ErrWrongData 160 | } 161 | case stateWaitValue: 162 | valueFrom = idx 163 | if ch == '"' { 164 | state = stateStringValue 165 | } else if isNum(ch) { 166 | state = statePositiveIntegerValue 167 | } else if ch == '-' { 168 | state = stateNumericValue 169 | } else if !isSpace(ch) { 170 | return ErrWrongData 171 | } 172 | case stateNumericValue: 173 | if isNum(ch) { 174 | state = statePositiveIntegerValue 175 | } else { 176 | return ErrWrongData 177 | } 178 | case stateStringValue: 179 | if ch == '"' { 180 | key := buf[keyFrom+1 : keyTo] 181 | value := buf[valueFrom+1 : idx] 182 | if !fieldCallback(key, value, jsValueTypeString) { 183 | return ErrWrongData 184 | } 185 | state = stateWaitCommaOrHashmapEnd 186 | } 187 | case statePositiveIntegerValue: 188 | if isNum(ch) { 189 | // do nothing 190 | } else if (ch == ',') || (ch == '}') || isSpace(ch) { 191 | key := buf[keyFrom+1 : keyTo] 192 | value := buf[valueFrom:idx] 193 | if !fieldCallback(key, value, jsValueTypeNumeric) { 194 | return ErrWrongData 195 | } 196 | 197 | if ch == ',' { 198 | state = stateWaitKey 199 | } else if ch == '}' { 200 | return nil 201 | } else { 202 | state = stateWaitCommaOrHashmapEnd 203 | } 204 | } else { 205 | return ErrWrongData 206 | } 207 | case stateWaitCommaOrHashmapEnd: 208 | if ch == ',' { 209 | state = stateWaitKey 210 | } else if ch == '}' { 211 | return nil 212 | } else if !isSpace(ch) { 213 | return ErrWrongData 214 | } 215 | default: 216 | return ErrWrongData 217 | } 218 | } 219 | 220 | return nil 221 | } 222 | -------------------------------------------------------------------------------- /location.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "log" 6 | "strconv" 7 | ) 8 | 9 | type ( 10 | Location struct { 11 | Id int32 12 | Place []byte 13 | CountryIdx int32 14 | City []byte 15 | Distance int32 16 | 17 | cache LocationsAvg 18 | } 19 | ) 20 | 21 | func (l *Location) Reset() { 22 | l.Id = 0 23 | l.Place = l.Place[:0] 24 | l.CountryIdx = 0 25 | l.City = l.City[:0] 26 | l.Distance = 0 27 | l.cache.locations = l.cache.locations[:0] 28 | } 29 | 30 | func (l *Location) Parse(buf []byte) bool { 31 | changed := false 32 | 33 | err := ParseItem(buf, func(key, value []byte, valueType JSValueType) bool { 34 | changed = true // проверка, что не совсем пустой buf пришел 35 | 36 | if bytes.Equal(key, strId) { 37 | if valueType != jsValueTypeNumeric { 38 | return false 39 | } else if i64, ok := byteSliceToInt64(value); !ok { 40 | return false 41 | } else { 42 | l.Id = int32(i64) 43 | } 44 | } else if bytes.Equal(key, strPlace) { 45 | if valueType != jsValueTypeString { 46 | return false 47 | } 48 | l.Place = append(l.Place[0:0], value...) 49 | } else if bytes.Equal(key, strCountry) { 50 | if valueType != jsValueTypeString { 51 | return false 52 | } 53 | 54 | country := utf8Unescaped(value) 55 | idx, _ := indexCountry.Add(country) 56 | l.CountryIdx = int32(idx) 57 | } else if bytes.Equal(key, strCity) { 58 | if valueType != jsValueTypeString { 59 | return false 60 | } 61 | l.City = append(l.City[0:0], value...) 62 | } else if bytes.Equal(key, strDistance) { 63 | if valueType != jsValueTypeNumeric { 64 | return false 65 | } else if i64, ok := byteSliceToInt64(value); !ok { 66 | return false 67 | } else { 68 | l.Distance = int32(i64) 69 | } 70 | } // else { // неизвестный ключ 71 | 72 | return true 73 | }) 74 | 75 | return (err == nil) && changed 76 | } 77 | 78 | func (l *Location) Serialize(buf []byte) []byte { 79 | country := indexCountry.GetByIdx(int(l.CountryIdx)) 80 | 81 | buf = append(buf, `{"id":`...) 82 | buf = strconv.AppendInt(buf, int64(l.Id), 10) 83 | buf = append(buf, `,"place":"`...) 84 | buf = append(buf, l.Place...) 85 | buf = append(buf, `","country":"`...) 86 | buf = append(buf, country...) 87 | buf = append(buf, `","city":"`...) 88 | buf = append(buf, l.City...) 89 | buf = append(buf, `","distance":`...) 90 | buf = strconv.AppendInt(buf, int64(l.Distance), 10) 91 | buf = append(buf, '}') 92 | 93 | return buf 94 | } 95 | 96 | func (l *Location) CheckFields(update bool) bool { 97 | /*if utf8EscapedStringLen(l.Country) > 50 { 98 | return false 99 | } else if utf8EscapedStringLen(l.City) > 50 { 100 | return false 101 | }*/ 102 | 103 | if !update && (len(l.City) == 0 || l.CountryIdx == 0 || l.Distance == 0 || len(l.Place) == 0) { 104 | // при создании должны передаваться все поля 105 | return false 106 | } 107 | 108 | if update && (l.Id != 0) { 109 | // id не может обновляться 110 | return false 111 | } 112 | 113 | return true 114 | } 115 | 116 | func (l *Location) Update(update *Location) bool { 117 | /* 118 | Если меняется Distance: 119 | - в UserVisits (Visit(Location.cache.visitId) => User(visit.User).cache.distance) 120 | поменять поля distance, countryIdx 121 | */ 122 | 123 | placeChanged := (len(update.Place) != 0) && !bytes.Equal(l.Place, update.Place) 124 | if placeChanged { 125 | l.Place = append(l.Place[:0], update.Place...) 126 | } 127 | 128 | countryChanged := (update.CountryIdx != 0) && (l.CountryIdx != update.CountryIdx) 129 | if countryChanged { 130 | l.CountryIdx = update.CountryIdx 131 | } 132 | 133 | if len(update.City) != 0 { 134 | l.City = append(l.City[:0], update.City...) 135 | } 136 | 137 | distanceChanged := (update.Distance != 0) && (l.Distance != update.Distance) 138 | if distanceChanged { 139 | l.Distance = update.Distance 140 | } 141 | 142 | if placeChanged || distanceChanged || countryChanged { 143 | l.cacheUpdateDistanceAndCountryIdxAndPlace() 144 | } 145 | 146 | return true 147 | } 148 | 149 | func (l *Location) cacheUpdateDistanceAndCountryIdxAndPlace() { 150 | for _, la := range l.cache.locations { 151 | if visit := indexVisit.Get(la.visitId); visit == nil { 152 | log.Println(`WTF visit nil in location cache`, la.visitId) 153 | } else if user := indexUser.Get(visit.User); user == nil { 154 | log.Println(`WTF user nil in location cache`, la.visitId, visit.User) 155 | } else { 156 | for i, uv := range user.cache.visits { 157 | if uv.visitId != visit.Id { 158 | continue 159 | } 160 | 161 | if l.Distance != 0 { 162 | user.cache.visits[i].distance = l.Distance 163 | } 164 | if l.CountryIdx != 0 { 165 | user.cache.visits[i].countryIdx = l.CountryIdx 166 | } 167 | if len(l.Place) != 0 { 168 | user.cache.visits[i].place = l.Place 169 | } 170 | } 171 | } 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /locations_avg.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | /* 4 | /locations//avg 5 | 6 | Возможные GET-параметры: 7 | fromDate - учитывать оценки только с visited_at > fromDate 8 | toDate - учитывать оценки только с visited_at < toDate 9 | fromAge - учитывать только путешественников, у которых возраст (считается от текущего timestamp) больше этого параметра 10 | toAge - как предыдущее, но наоборот 11 | gender - учитывать оценки только мужчин или женщин 12 | */ 13 | 14 | type ( 15 | LocationsAvg struct { 16 | locations []LocationAvg 17 | } 18 | 19 | LocationAvg struct { 20 | visitId int32 21 | visitedAt int32 22 | birthdate int64 23 | gender byte 24 | 25 | mark uint8 26 | } 27 | ) 28 | 29 | func (la *LocationsAvg) Add(location *Location, visit *Visit, user *User) bool { 30 | la.locations = append(la.locations, LocationAvg{ 31 | visitId: visit.Id, 32 | visitedAt: visit.VisitedAt, 33 | birthdate: user.BirthDate, 34 | gender: user.Gender, 35 | mark: visit.Mark, 36 | }) 37 | return true 38 | } 39 | 40 | func (la *LocationsAvg) MoveByVisitId(target *Location, visitId int32) bool { 41 | currentPos := -1 42 | for i, laItem := range la.locations { 43 | if laItem.visitId == visitId { 44 | currentPos = i 45 | break 46 | } 47 | } 48 | if currentPos == -1 { 49 | return false 50 | } 51 | 52 | bak := la.locations[currentPos] 53 | 54 | lastIdx := len(la.locations) - 1 55 | if currentPos < lastIdx { 56 | la.locations[currentPos] = la.locations[lastIdx] 57 | } 58 | la.locations = la.locations[:lastIdx] 59 | 60 | target.cache.locations = append(target.cache.locations, bak) 61 | 62 | return true 63 | } 64 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "archive/zip" 5 | "bufio" 6 | "bytes" 7 | "encoding/json" 8 | "errors" 9 | "flag" 10 | "fmt" 11 | "io" 12 | "log" 13 | "os" 14 | "os/signal" 15 | "path" 16 | "runtime" 17 | "runtime/debug" 18 | "runtime/pprof" 19 | "strconv" 20 | "strings" 21 | "sync" 22 | "sync/atomic" 23 | "syscall" 24 | "time" 25 | ) 26 | 27 | var ( 28 | ErrWrongData = errors.New(`Wrong data`) 29 | ) 30 | 31 | var ( 32 | indexUser = MakeIndexUser() 33 | indexLocation = MakeIndexLocation() 34 | indexVisit = MakeIndexVisit() 35 | 36 | indexCountry = MakeIndexCountry() 37 | ) 38 | 39 | var ( 40 | BuildInfo string = `` 41 | 42 | argv struct { 43 | port uint 44 | help bool 45 | pprof bool 46 | zipPath string 47 | } 48 | 49 | dictStatistics struct { 50 | Users int64 `json:"users"` 51 | Locations int64 `json:"locations"` 52 | Visits int64 `json:"visits"` 53 | LocationMaxId int32 `json:"location_max_id"` 54 | UserMaxId int32 `json:"user_max_id"` 55 | VisitMaxId int32 `json:"visit_max_id"` 56 | Elapsed int64 `json:"elapsed"` 57 | } 58 | 59 | // текущее время (или реальное, или полученное из архива с данными) 60 | timeNow time.Time 61 | 62 | currentPhase int 63 | 64 | buf bytes.Buffer 65 | 66 | queries, prevQPS int64 67 | 68 | poolLocation = sync.Pool{ 69 | New: func() interface{} { 70 | return &Location{} 71 | }, 72 | } 73 | ) 74 | 75 | func init() { 76 | flag.UintVar(&argv.port, `port`, 80, `port to listen`) 77 | flag.BoolVar(&argv.help, `h`, false, `show this help`) 78 | flag.BoolVar(&argv.pprof, `pprof`, false, `enable pprof`) 79 | flag.StringVar(&argv.zipPath, `zip`, `/tmp/data/data.zip`, `path to zip file`) 80 | flag.Parse() 81 | } 82 | 83 | func main() { 84 | if argv.help { 85 | fmt.Printf("Builded from %s\n", BuildInfo) 86 | flag.Usage() 87 | return 88 | } 89 | 90 | log.Printf("Started on %d CPUs\n", runtime.NumCPU()) 91 | 92 | srv := HTTPServer{ 93 | Handler: requestHandler, 94 | } 95 | 96 | log.SetFlags(log.LstdFlags | log.Lmicroseconds) 97 | debug.SetGCPercent(100) 98 | 99 | determineCurrentTime() 100 | 101 | log.Println(`Load DB...`) 102 | 103 | mt := time.Now().UnixNano() 104 | if err := loadDB(); err != nil { 105 | log.Fatalf(`load DB fail: %s`, err) 106 | } 107 | dictStatistics.Elapsed = (time.Now().UnixNano() - mt) / int64(time.Millisecond) 108 | 109 | buf.Reset() 110 | json.NewEncoder(&buf).Encode(dictStatistics) 111 | log.Println(`DB loaded stats:`, string(bytes.TrimSpace(buf.Bytes()))) 112 | 113 | if argv.pprof { 114 | if fd, err := os.Create(`pprof.cpu`); err == nil { 115 | pprof.StartCPUProfile(fd) 116 | defer func() { 117 | pprof.StopCPUProfile() 118 | fd.Close() 119 | }() 120 | } 121 | 122 | defer func() { 123 | if fd, err := os.Create(`pprof.mem`); err == nil { 124 | pprof.WriteHeapProfile(fd) 125 | fd.Close() 126 | } 127 | }() 128 | } 129 | 130 | warming() 131 | 132 | go func() { 133 | for { 134 | time.Sleep(1 * time.Second) 135 | qps := atomic.SwapInt64(&queries, 0) 136 | endPhase := (qps == 0) && (prevQPS != 0) 137 | newPhase := (qps != 0) && (prevQPS == 0) 138 | prevQPS = qps 139 | 140 | if endPhase { 141 | if currentPhase >= 3 { 142 | gcFrom := time.Now() 143 | runtime.GC() 144 | gcElapsed := time.Now().Sub(gcFrom) 145 | 146 | log.Printf("Phase ended. Goroutines: %d GC elapsed (ms): %d RSS: %dMB GC pauses (ms): %s\n", 147 | runtime.NumGoroutine(), 148 | int64(gcElapsed/time.Millisecond), 149 | getRSSMemory()/1024/1024, 150 | getGCStats(), 151 | ) 152 | } else { 153 | log.Printf("Phase ended. RSS: %dMB\n", 154 | getRSSMemory()/1024/1024, 155 | ) 156 | } 157 | } else if newPhase { 158 | currentPhase++ 159 | log.Println(`Phase`, currentPhase, `started`) 160 | } 161 | } 162 | }() 163 | 164 | go func() { 165 | if err := srv.ListenAndServe(int(argv.port)); err != nil { 166 | log.Fatalf(`ListenAndServe fail: %s`, err) 167 | } 168 | }() 169 | 170 | runtime.GC() 171 | debug.SetGCPercent(-1) 172 | 173 | mlockallErr := syscall.Mlockall(syscall.MCL_CURRENT) 174 | 175 | log.Printf("Ready for work. Build: %s. RSS:% dMB. GC pauses before (ms): %s. GC disabled. mlockall: %s\n", 176 | BuildInfo, 177 | getRSSMemory()/1024/1024, 178 | getGCStats(), 179 | mlockallErr, 180 | ) 181 | 182 | ch := make(chan os.Signal, 10) 183 | signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) 184 | <-ch 185 | 186 | log.Printf("Bye. RSS: %dMB GC pauses (ms): %s\n", getRSSMemory()/1024/1024, getGCStats()) 187 | } 188 | 189 | func requestHandler(ctx *RequestCtx) { 190 | atomic.AddInt64(&queries, 1) 191 | 192 | req := requestParamsPool.Get().(*RequestParams) 193 | defer requestParamsPool.Put(req) 194 | 195 | if !parseRequest(ctx, req) { 196 | ctx.ResponseStatus = 400 197 | return 198 | } else if req.entity == nil { 199 | ctx.ResponseStatus = 400 200 | return 201 | } 202 | 203 | if !req.isGET && req.isNew { 204 | // POST //new на создание 205 | reqNew(ctx, req) 206 | } else if req.id <= 0 { 207 | ctx.ResponseStatus = 404 208 | return 209 | } else if !req.isGET { 210 | // POST // на обновление 211 | reqUpdate(ctx, req) 212 | } else if req.action == nil { 213 | // GET // для получения данных о сущности 214 | reqGet(ctx, req) 215 | } else if bytes.Equal(req.entity, strUsers) && bytes.Equal(req.action, strVisits) { 216 | // GET /users//visits для получения списка посещений пользователем 217 | reqUserVisits(ctx, req) 218 | } else if bytes.Equal(req.entity, strLocations) && bytes.Equal(req.action, strAvg) { 219 | // GET /locations//avg для получения средней оценки достопримечательности 220 | reqLocationAvg(ctx, req) 221 | } else { 222 | ctx.ResponseStatus = 400 223 | return 224 | } 225 | } 226 | 227 | func reqGet(ctx *RequestCtx, req *RequestParams) { 228 | // GET // для получения данных о сущности 229 | 230 | if bytes.Equal(req.entity, strUsers) { 231 | if user := indexUser.Get(req.id); user == nil { 232 | ctx.ResponseStatus = 404 233 | } else { 234 | ctx.ResponseBody = user.Serialize(ctx.UserBuf[:0]) 235 | } 236 | } else if bytes.Equal(req.entity, strLocations) { 237 | if location := indexLocation.Get(req.id); location == nil { 238 | ctx.ResponseStatus = 404 239 | } else { 240 | ctx.ResponseBody = location.Serialize(ctx.UserBuf[:0]) 241 | } 242 | } else if bytes.Equal(req.entity, strVisits) { 243 | if visit := indexVisit.Get(req.id); visit == nil { 244 | ctx.ResponseStatus = 404 245 | } else { 246 | ctx.ResponseBody = visit.Serialize(ctx.UserBuf[:0]) 247 | } 248 | } else { 249 | ctx.ResponseStatus = 400 250 | } 251 | } 252 | 253 | func reqUserVisits(ctx *RequestCtx, req *RequestParams) { 254 | // GET /users//visits для получения списка посещений пользователем 255 | 256 | user := indexUser.Get(req.id) 257 | 258 | if user == nil { 259 | ctx.ResponseStatus = 404 260 | return 261 | } 262 | 263 | fromDate := req.fromDate 264 | toDate := req.toDate 265 | countryIdx := int32(req.countryIdx) 266 | toDistance := req.toDistance 267 | 268 | buf := ctx.UserBuf[:0] 269 | 270 | bufWithData := false 271 | 272 | buf = append(buf, `{"visits":[`...) 273 | 274 | for _, cacheItem := range user.cache.visits { 275 | if (fromDate > 0) && (cacheItem.visitedAt <= fromDate) { 276 | continue 277 | } else if (toDate > 0) && (cacheItem.visitedAt >= toDate) { 278 | continue 279 | } else if (toDistance > 0) && (cacheItem.distance >= toDistance) { 280 | continue 281 | } else if (countryIdx > 0) && (cacheItem.countryIdx != countryIdx) { 282 | continue 283 | } 284 | 285 | bufWithData = true 286 | 287 | buf = append(buf, `{"mark":`...) 288 | buf = append(buf, cacheItem.markChar) 289 | buf = append(buf, `,"visited_at":`...) 290 | buf = append(buf, cacheItem.visitedAtStr[:cacheItem.visitedAtStrLen]...) 291 | buf = append(buf, `,"place":"`...) 292 | buf = append(buf, cacheItem.place...) 293 | buf = append(buf, `"},`...) 294 | } 295 | 296 | if bufWithData { 297 | buf = buf[:len(buf)-1] // убираю последнюю запятую 298 | } 299 | 300 | buf = append(buf, `]}`...) 301 | 302 | ctx.ResponseBody = buf 303 | } 304 | 305 | func reqLocationAvg(ctx *RequestCtx, req *RequestParams) { 306 | // GET /locations//avg для получения средней оценки достопримечательности 307 | 308 | location := indexLocation.Get(req.id) 309 | 310 | if location == nil { 311 | ctx.ResponseStatus = 404 312 | return 313 | } 314 | 315 | var ( 316 | markCnt, markSum int64 317 | avg float64 318 | ) 319 | 320 | fromDate := req.fromDate 321 | toDate := req.toDate 322 | fromAge := req.fromAge 323 | toAge := req.toAge 324 | 325 | fromAgeTimestamp := ageToTimestamp(fromAge) 326 | toAgeTimestamp := ageToTimestamp(toAge) 327 | 328 | gender := req.gender 329 | 330 | for _, cacheItem := range location.cache.locations { 331 | if (fromDate > 0) && (cacheItem.visitedAt <= fromDate) { 332 | continue 333 | } else if (toDate > 0) && (cacheItem.visitedAt >= toDate) { 334 | continue 335 | } else if (fromAge > 0) && (cacheItem.birthdate > fromAgeTimestamp) { 336 | continue 337 | } else if (toAge > 0) && (cacheItem.birthdate < toAgeTimestamp) { 338 | continue 339 | } else if (gender != 0) && (gender != cacheItem.gender) { 340 | continue 341 | } 342 | 343 | markSum += int64(cacheItem.mark) 344 | markCnt++ 345 | } 346 | 347 | if markCnt > 0 { 348 | avg = float64(markSum) / float64(markCnt) 349 | } 350 | avg += 1e-10 // +eps как костыль для округления 351 | 352 | buf := ctx.UserBuf[:0] 353 | 354 | buf = append(buf, `{"avg":`...) 355 | buf = strconv.AppendFloat(buf, avg, 'f', 5, 64) 356 | buf = append(buf, '}') 357 | 358 | ctx.ResponseBody = buf 359 | } 360 | 361 | func reqNew(ctx *RequestCtx, req *RequestParams) { 362 | // POST //new на создание 363 | 364 | if bytes.Equal(req.entity, strUsers) { 365 | var user User 366 | if !user.Parse(ctx.Body) { 367 | ctx.ResponseStatus = 400 368 | return 369 | } else if !user.CheckFields(false) { 370 | ctx.ResponseStatus = 400 371 | return 372 | } else if !indexUser.Add(&user) { 373 | ctx.ResponseStatus = 400 374 | return 375 | } 376 | 377 | } else if bytes.Equal(req.entity, strLocations) { 378 | location := poolLocation.Get().(*Location) 379 | location.Reset() 380 | if !location.Parse(ctx.Body) { 381 | ctx.ResponseStatus = 400 382 | poolLocation.Put(location) 383 | return 384 | } else if !location.CheckFields(false) { 385 | ctx.ResponseStatus = 400 386 | poolLocation.Put(location) 387 | return 388 | } else if !indexLocation.Add(location) { 389 | ctx.ResponseStatus = 400 390 | poolLocation.Put(location) 391 | return 392 | } 393 | // если все хорошо, то не возвращаю location в пул 394 | 395 | } else if bytes.Equal(req.entity, strVisits) { 396 | var visit Visit 397 | if !visit.Parse(ctx.Body) { 398 | ctx.ResponseStatus = 400 399 | return 400 | } else if !visit.CheckFields(false) { 401 | ctx.ResponseStatus = 400 402 | return 403 | } else if !indexVisit.Add(&visit) { 404 | ctx.ResponseStatus = 400 405 | return 406 | } else if user := indexUser.Get(visit.User); user == nil { 407 | ctx.ResponseStatus = 404 408 | return 409 | } else if location := indexLocation.Get(visit.Location); location == nil { 410 | ctx.ResponseStatus = 404 411 | return 412 | } else { 413 | user.cache.Add(location, &visit) 414 | location.cache.Add(location, &visit, user) 415 | } 416 | 417 | } else { 418 | ctx.ResponseStatus = 400 419 | return 420 | } 421 | 422 | ctx.ResponseBody = emptyResponseBody 423 | } 424 | 425 | func reqUpdate(ctx *RequestCtx, req *RequestParams) { 426 | // POST // на обновление 427 | 428 | if bytes.Equal(req.entity, strUsers) { 429 | var user User 430 | 431 | if !user.Parse(ctx.Body) { 432 | ctx.ResponseStatus = 400 433 | return 434 | } 435 | 436 | if !user.CheckFields(true) { 437 | // 404 приоритетнее, чем 400 438 | if indexUser.Get(req.id) == nil { 439 | ctx.ResponseStatus = 404 440 | } else { 441 | ctx.ResponseStatus = 400 442 | } 443 | return 444 | } else if !indexUser.Update(req.id, &user) { 445 | ctx.ResponseStatus = 404 446 | return 447 | } 448 | 449 | } else if bytes.Equal(req.entity, strLocations) { 450 | location := poolLocation.Get().(*Location) 451 | location.Reset() 452 | 453 | if !location.Parse(ctx.Body) { 454 | ctx.ResponseStatus = 400 455 | poolLocation.Put(location) 456 | return 457 | } 458 | 459 | if !location.CheckFields(true) { 460 | // 404 приоритетнее, чем 400 461 | if indexLocation.Get(req.id) == nil { 462 | ctx.ResponseStatus = 404 463 | } else { 464 | ctx.ResponseStatus = 400 465 | } 466 | poolLocation.Put(location) 467 | return 468 | } else if !indexLocation.Update(req.id, location) { 469 | ctx.ResponseStatus = 404 470 | poolLocation.Put(location) 471 | return 472 | } 473 | poolLocation.Put(location) 474 | 475 | } else if bytes.Equal(req.entity, strVisits) { 476 | var visit Visit 477 | 478 | if !visit.Parse(ctx.Body) { 479 | ctx.ResponseStatus = 400 480 | return 481 | } 482 | 483 | if !visit.CheckFields(true) { 484 | // 404 приоритетнее, чем 400 485 | if indexVisit.Get(req.id) == nil { 486 | ctx.ResponseStatus = 404 487 | } else { 488 | ctx.ResponseStatus = 400 489 | } 490 | return 491 | } else if !indexVisit.Update(req.id, &visit) { 492 | ctx.ResponseStatus = 404 493 | return 494 | } 495 | 496 | } else { 497 | ctx.ResponseStatus = 400 498 | return 499 | } 500 | 501 | ctx.ResponseBody = emptyResponseBody 502 | } 503 | 504 | func loadDB() error { 505 | r, err := zip.OpenReader(argv.zipPath) 506 | if err != nil { 507 | return err 508 | } 509 | defer r.Close() 510 | 511 | // для корректной загрузки нужно обходить файлы в определенном порядке типов 512 | 513 | priority := []string{`locations_`, `users_`, `visits_`} 514 | 515 | queue := map[string][]*zip.File{} 516 | 517 | for _, prefix := range priority { 518 | queue[prefix] = []*zip.File{} 519 | } 520 | 521 | for _, f := range r.File { 522 | for prefix := range queue { 523 | if strings.HasPrefix(f.Name, prefix) { 524 | queue[prefix] = append(queue[prefix], f) 525 | } 526 | } 527 | } 528 | 529 | for _, prefix := range priority { 530 | for _, f := range queue[prefix] { 531 | fd, err := f.Open() 532 | if err != nil { 533 | return err 534 | } 535 | 536 | switch prefix { 537 | case `locations_`: 538 | err = loadLocations(fd) 539 | case `users_`: 540 | err = loadUsers(fd) 541 | case `visits_`: 542 | err = loadVisits(fd) 543 | } 544 | 545 | fd.Close() 546 | 547 | if err != nil { 548 | return err 549 | } 550 | } 551 | } 552 | 553 | return nil 554 | } 555 | 556 | func determineCurrentTime() { 557 | timeNow = time.Now() 558 | 559 | optionsTxt := path.Join(path.Dir(argv.zipPath), `options.txt`) 560 | if fd, err := os.Open(optionsTxt); err == nil { 561 | if err := loadOptions(fd); err != nil { 562 | log.Println(`Cannot read options.txt:`, err) 563 | } 564 | fd.Close() 565 | } 566 | 567 | timeNow = timeFloor(timeNow.In(time.UTC)) 568 | log.Printf("NOW %d (%s)\n", timeNow.Unix(), timeNow.String()) 569 | } 570 | 571 | func loadOptions(src io.Reader) (err error) { 572 | if line, err := bufio.NewReader(src).ReadString('\n'); err != nil { 573 | return err 574 | } else if ts, err := strconv.ParseUint(strings.TrimSpace(line), 10, 64); err != nil { 575 | return err 576 | } else { 577 | newTimeNow := time.Unix(int64(ts), 0) 578 | timeNow = newTimeNow 579 | } 580 | 581 | return nil 582 | } 583 | 584 | func loadUsers(src io.Reader) (err error) { 585 | buf.Reset() 586 | buf.ReadFrom(src) 587 | 588 | err = ParseData(buf.Bytes(), strUsers, func(item []byte) bool { 589 | var user User 590 | if !user.Parse(item) { 591 | return false 592 | } else if !indexUser.Add(&user) { 593 | return false 594 | } 595 | 596 | if dictStatistics.UserMaxId < user.Id { 597 | dictStatistics.UserMaxId = user.Id 598 | } 599 | dictStatistics.Users++ 600 | 601 | return true 602 | }) 603 | 604 | return 605 | } 606 | 607 | func loadLocations(src io.Reader) (err error) { 608 | buf.Reset() 609 | buf.ReadFrom(src) 610 | 611 | err = ParseData(buf.Bytes(), strLocations, func(item []byte) bool { 612 | var location Location 613 | if !location.Parse(item) { 614 | return false 615 | } else if !indexLocation.Add(&location) { 616 | return false 617 | } 618 | 619 | if dictStatistics.LocationMaxId < location.Id { 620 | dictStatistics.LocationMaxId = location.Id 621 | } 622 | dictStatistics.Locations++ 623 | 624 | return true 625 | }) 626 | 627 | return 628 | } 629 | 630 | func loadVisits(src io.Reader) (err error) { 631 | buf.Reset() 632 | buf.ReadFrom(src) 633 | 634 | err = ParseData(buf.Bytes(), strVisits, func(item []byte) bool { 635 | var visit Visit 636 | if !visit.Parse(item) { 637 | return false 638 | } else if !indexVisit.Add(&visit) { 639 | return false 640 | } else if user := indexUser.Get(visit.User); user == nil { 641 | // кривые даннные в исходной выборке? 642 | return false 643 | } else if location := indexLocation.Get(visit.Location); location == nil { 644 | // кривые даннные в исходной выборке? 645 | return false 646 | } else { 647 | user.cache.Add(location, &visit) 648 | location.cache.Add(location, &visit, user) 649 | } 650 | 651 | if dictStatistics.VisitMaxId < visit.Id { 652 | dictStatistics.VisitMaxId = visit.Id 653 | } 654 | dictStatistics.Visits++ 655 | 656 | return true 657 | }) 658 | 659 | return 660 | } 661 | 662 | func warming() { 663 | ageToTimestampWarming() 664 | } 665 | -------------------------------------------------------------------------------- /request.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "sync" 6 | ) 7 | 8 | type ( 9 | RequestParams struct { 10 | isGET bool 11 | isNew bool 12 | id int32 13 | entity []byte 14 | action []byte 15 | 16 | fromDate int32 // visited_at > fromDate 17 | toDate int32 // visited_at < toDate 18 | countryIdx int // название страны (индекс в indexCountry), в которой находятся интересующие достопримечательности 19 | toDistance int32 // возвращать только те места, у которых расстояние от города меньше этого параметра 20 | fromAge int32 // учитывать только путешественников, у которых возраст (в годах) (считается от текущего timestamp) больше этого параметра 21 | toAge int32 // как предыдущее, но наоборот 22 | gender byte // учитывать оценки только мужчин или женщин 23 | } 24 | ) 25 | 26 | var ( 27 | requestParamsPool = sync.Pool{ 28 | New: func() interface{} { 29 | return &RequestParams{} 30 | }, 31 | } 32 | ) 33 | 34 | func parseArgs(ctx *RequestCtx, args []byte, params *RequestParams) bool { 35 | if len(args) == 0 { 36 | return true 37 | } 38 | 39 | var arg, val []byte 40 | 41 | for len(args) > 0 { 42 | pEq := bytes.IndexByte(args, '=') 43 | pAmp := bytes.IndexByte(args, '&') 44 | if pEq == -1 { 45 | return false 46 | } else if pAmp > -1 && pAmp < pEq { 47 | return false 48 | } 49 | 50 | arg = args[:pEq] 51 | if pAmp == -1 { 52 | val = args[pEq+1:] 53 | args = nil 54 | } else { 55 | val = args[pEq+1 : pAmp] 56 | args = args[pAmp+1:] 57 | } 58 | 59 | if len(val) == 0 { 60 | return false 61 | } 62 | 63 | if bytes.Equal(arg, strFromDate) { 64 | if i64, ok := byteSliceToInt64(val); !ok { 65 | return false 66 | } else { 67 | params.fromDate = int32(i64) 68 | } 69 | } else if bytes.Equal(arg, strToDate) { 70 | if i64, ok := byteSliceToInt64(val); !ok { 71 | return false 72 | } else { 73 | params.toDate = int32(i64) 74 | } 75 | } else if bytes.Equal(arg, strCountry) { 76 | country := urlDecode(ctx.UserBuf[:0], val) 77 | params.countryIdx = indexCountry.Find(country) 78 | } else if bytes.Equal(arg, strToDistance) { 79 | if i64, ok := byteSliceToInt64(val); !ok { 80 | return false 81 | } else { 82 | params.toDistance = int32(i64) 83 | } 84 | } else if bytes.Equal(arg, strFromAge) { 85 | if i64, ok := byteSliceToInt64(val); !ok { 86 | return false 87 | } else { 88 | params.fromAge = int32(i64) 89 | } 90 | } else if bytes.Equal(arg, strToAge) { 91 | if i64, ok := byteSliceToInt64(val); !ok { 92 | return false 93 | } else { 94 | params.toAge = int32(i64) 95 | } 96 | } else if bytes.Equal(arg, strGender) { 97 | if (len(val) != 1) || ((val[0] != 'm') && (val[0] != 'f')) { 98 | return false 99 | } 100 | params.gender = val[0] 101 | } 102 | } 103 | 104 | return true 105 | } 106 | 107 | func parseRequest(ctx *RequestCtx, params *RequestParams) bool { 108 | // method 109 | params.isGET = ctx.Method == MethodGET 110 | 111 | uri := ctx.Path[1:] // убираю начальный / 112 | 113 | var args []byte 114 | 115 | // path?args 116 | if idx := bytes.IndexByte(uri, '?'); idx > 0 { 117 | args = uri[idx+1:] 118 | uri = uri[:idx] 119 | } 120 | 121 | // entity 122 | if idx := bytes.IndexByte(uri, '/'); idx > 0 { 123 | params.entity = uri[0:idx] 124 | uri = uri[idx+1:] 125 | } else if idx == -1 { 126 | params.entity = uri 127 | uri = nil 128 | } else { 129 | return false 130 | } 131 | 132 | params.isNew = false 133 | params.action = nil 134 | params.id = 0 135 | params.fromDate = 0 136 | params.toDate = 0 137 | params.countryIdx = 0 138 | params.toDistance = 0 139 | params.fromAge = 0 140 | params.toAge = 0 141 | params.gender = 0 142 | 143 | if len(uri) == 0 { 144 | return true 145 | } 146 | 147 | // id 148 | if bytes.Equal(uri, strNew) { 149 | // //new 150 | params.isNew = true 151 | uri = nil 152 | } else if idx := bytes.IndexByte(uri, '/'); idx == 0 { 153 | return false 154 | } else { 155 | // // 156 | tail := uri 157 | to := idx 158 | if to == -1 { 159 | to = len(uri) 160 | tail = nil 161 | } else { 162 | tail = uri[to+1:] 163 | } 164 | 165 | if i64, ok := byteSliceToInt64(uri[0:to]); ok { 166 | params.id = int32(i64) 167 | } // else // могут быть всякие "/users/bad". это корректный запрос, просто 404 168 | 169 | uri = tail 170 | } 171 | 172 | // action 173 | if len(uri) > 0 { 174 | // /users//visits 175 | params.action = uri 176 | } 177 | 178 | return parseArgs(ctx, args, params) 179 | } 180 | -------------------------------------------------------------------------------- /socket.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net" 5 | "syscall" 6 | ) 7 | 8 | const ( 9 | EPOLLET = 1 << 31 // в stdlib идет не того типа, как хотелось бы 10 | MaxEpollEvents = 64 11 | SO_REUSEPORT = 15 // нет в stdlib 12 | ) 13 | 14 | func socketCreateListener(port int) (serverFd int, err error) { 15 | addr := syscall.SockaddrInet4{Port: port} 16 | copy(addr.Addr[:], net.ParseIP(`0.0.0.0`).To4()) 17 | 18 | serverFd, err = syscall.Socket(syscall.AF_INET, syscall.O_NONBLOCK|syscall.SOCK_STREAM, 0) 19 | if err != nil { 20 | return 21 | } 22 | 23 | if err = socketSetNonBlocking(serverFd); err != nil { 24 | syscall.Close(serverFd) 25 | return 26 | } 27 | 28 | if err = syscall.SetsockoptInt(serverFd, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1); err != nil { 29 | syscall.Close(serverFd) 30 | return 31 | } 32 | 33 | if err = syscall.SetsockoptInt(serverFd, syscall.SOL_SOCKET, SO_REUSEPORT, 1); err != nil { 34 | syscall.Close(serverFd) 35 | return 36 | } 37 | 38 | if err = syscall.Bind(serverFd, &addr); err != nil { 39 | syscall.Close(serverFd) 40 | return 41 | } else if err = syscall.Listen(serverFd, syscall.SOMAXCONN); err != nil { 42 | syscall.Close(serverFd) 43 | return 44 | } 45 | 46 | return 47 | } 48 | 49 | func socketSetNonBlocking(fd int) error { 50 | return syscall.SetNonblock(fd, true) 51 | } 52 | 53 | func socketCreateListenerEpoll(serverFd int) (epollFd int, err error) { 54 | var event syscall.EpollEvent 55 | event.Events = syscall.EPOLLIN | EPOLLET 56 | event.Fd = int32(serverFd) 57 | 58 | epollFd, err = syscall.EpollCreate1(0) 59 | if err != nil { 60 | return 0, err 61 | } else if err = syscall.EpollCtl(epollFd, syscall.EPOLL_CTL_ADD, serverFd, &event); err != nil { 62 | syscall.Close(epollFd) 63 | return 0, err 64 | } 65 | 66 | return epollFd, nil 67 | } 68 | -------------------------------------------------------------------------------- /stats.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | /* 4 | #include 5 | */ 6 | import "C" 7 | 8 | import ( 9 | "bytes" 10 | "fmt" 11 | "os" 12 | "runtime/debug" 13 | "time" 14 | ) 15 | 16 | var ( 17 | gcStatsBuf bytes.Buffer 18 | gcStats debug.GCStats 19 | ) 20 | 21 | func getGCStats() []byte { 22 | debug.ReadGCStats(&gcStats) 23 | 24 | gcStatsBuf.Reset() 25 | 26 | for _, p := range gcStats.Pause { 27 | fmt.Fprint(&gcStatsBuf, int64(p/time.Millisecond), ` `) 28 | } 29 | 30 | return gcStatsBuf.Bytes() 31 | } 32 | 33 | func getRSSMemory() (rss int64) { 34 | fd, err := os.Open(`/proc/self/statm`) 35 | if err != nil { 36 | return 0 37 | } 38 | defer fd.Close() 39 | 40 | var tmp int64 41 | 42 | if _, err := fmt.Fscanf(fd, `%d %d`, &tmp, &rss); err != nil { 43 | return 0 44 | } 45 | 46 | pagesize := int64(C.sysconf(C._SC_PAGESIZE)) 47 | 48 | rss *= pagesize 49 | 50 | return 51 | } 52 | -------------------------------------------------------------------------------- /string_constants.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | var ( 4 | strGET = []byte(`GET`) 5 | strPOST = []byte(`POST`) 6 | strContentLength = []byte(`content-length`) 7 | strConnection = []byte(`connection`) 8 | strClose = []byte(`close`) 9 | strKeepAlive = []byte(`keep-alive`) 10 | str11 = []byte(`/1.1`) 11 | 12 | line200 = []byte("HTTP/1.1 200 OK\r\n") 13 | line400 = []byte("HTTP/1.1 400 Bad Request\r\n") 14 | line404 = []byte("HTTP/1.1 404 Not Found\r\n") 15 | line500 = []byte("HTTP/1.1 500 Internal Server Error\r\n") 16 | 17 | emptyResponseBody = []byte(`{}`) 18 | strUsers = []byte(`users`) 19 | strVisits = []byte(`visits`) 20 | strLocations = []byte(`locations`) 21 | strAvg = []byte(`avg`) 22 | strNew = []byte(`new`) 23 | 24 | strId = []byte(`id`) 25 | strLocation = []byte(`location`) 26 | strUser = []byte(`user`) 27 | strVisitedAt = []byte(`visited_at`) 28 | strMark = []byte(`mark`) 29 | strPlace = []byte(`place`) 30 | strCountry = []byte(`country`) 31 | strCity = []byte(`city`) 32 | strDistance = []byte(`distance`) 33 | strEmail = []byte(`email`) 34 | strFirstName = []byte(`first_name`) 35 | strLastName = []byte(`last_name`) 36 | strGender = []byte(`gender`) 37 | strBirthdate = []byte(`birth_date`) 38 | strFromDate = []byte(`fromDate`) 39 | strToDate = []byte(`toDate`) 40 | strToDistance = []byte(`toDistance`) 41 | strFromAge = []byte(`fromAge`) 42 | strToAge = []byte(`toAge`) 43 | ) 44 | -------------------------------------------------------------------------------- /user.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "log" 6 | "strconv" 7 | ) 8 | 9 | type ( 10 | User struct { 11 | Id int32 12 | Email []byte 13 | FirstName []byte 14 | LastName []byte 15 | Gender byte 16 | BirthDate int64 17 | 18 | birthdateSetted bool 19 | 20 | cache UserVisits 21 | } 22 | ) 23 | 24 | func (u *User) Parse(buf []byte) bool { 25 | changed := false 26 | 27 | u.birthdateSetted = false 28 | 29 | err := ParseItem(buf, func(key, value []byte, valueType JSValueType) bool { 30 | changed = true // проверка, что не совсем пустой buf пришел 31 | 32 | if bytes.Equal(key, strId) { 33 | if valueType != jsValueTypeNumeric { 34 | return false 35 | } else if i64, ok := byteSliceToInt64(value); !ok { 36 | return false 37 | } else { 38 | u.Id = int32(i64) 39 | } 40 | } else if bytes.Equal(key, strEmail) { 41 | if valueType != jsValueTypeString { 42 | return false 43 | } 44 | u.Email = append(u.Email[0:0], value...) 45 | } else if bytes.Equal(key, strFirstName) { 46 | if valueType != jsValueTypeString { 47 | return false 48 | } 49 | u.FirstName = append(u.FirstName[0:0], value...) 50 | } else if bytes.Equal(key, strLastName) { 51 | if valueType != jsValueTypeString { 52 | return false 53 | } 54 | u.LastName = append(u.LastName[0:0], value...) 55 | } else if bytes.Equal(key, strGender) { 56 | if (valueType != jsValueTypeString) || (len(value) != 1) { 57 | return false 58 | } 59 | u.Gender = value[0] 60 | } else if bytes.Equal(key, strBirthdate) { 61 | if valueType != jsValueTypeNumeric { 62 | return false 63 | } else if i64, ok := byteSliceToInt64(value); !ok { 64 | return false 65 | } else { 66 | u.BirthDate = i64 67 | u.birthdateSetted = true 68 | } 69 | } // else { // неизвестный ключ 70 | 71 | return true 72 | }) 73 | 74 | return (err == nil) && changed 75 | } 76 | 77 | func (u *User) Serialize(buf []byte) []byte { 78 | buf = append(buf, `{"id":`...) 79 | buf = strconv.AppendInt(buf, int64(u.Id), 10) 80 | buf = append(buf, `,"email":"`...) 81 | buf = append(buf, u.Email...) 82 | buf = append(buf, `","first_name":"`...) 83 | buf = append(buf, u.FirstName...) 84 | buf = append(buf, `","last_name":"`...) 85 | buf = append(buf, u.LastName...) 86 | buf = append(buf, `","gender":"`...) 87 | buf = append(buf, u.Gender) 88 | buf = append(buf, `","birth_date":`...) 89 | buf = strconv.AppendInt(buf, int64(u.BirthDate), 10) 90 | buf = append(buf, '}') 91 | 92 | return buf 93 | } 94 | 95 | func (u *User) CheckFields(update bool) bool { 96 | /*if utf8EscapedStringLen(u.Email) > 100 { 97 | return false 98 | } else if utf8EscapedStringLen(u.FirstName) > 50 { 99 | return false 100 | } else if utf8EscapedStringLen(u.LastName) > 50 { 101 | return false 102 | } else*/ 103 | if u.Gender != 0 && u.Gender != 'm' && u.Gender != 'f' { 104 | return false 105 | } 106 | 107 | if !update && (len(u.Email) == 0 || len(u.FirstName) == 0 || len(u.LastName) == 0 || u.Gender == 0 || !u.birthdateSetted) { 108 | // при создании должны передаваться все поля 109 | return false 110 | } 111 | 112 | if update && (u.Id != 0) { 113 | // id не может обновляться 114 | return false 115 | } 116 | 117 | return true 118 | } 119 | 120 | func (u *User) Update(update *User) bool { 121 | /* 122 | Если меняется Gender: 123 | - в LocationsAvg обновить поле gender для: Visit(User.cache.visitId) => Location(Visit.Location).cache.gender 124 | Если меняется birthdate: 125 | - в LocationsAvg обновить поле birthdate для: Visit(User.cache.visitId) => Location(Visit.Location).cache.birthdate 126 | */ 127 | 128 | if len(update.Email) != 0 { 129 | u.Email = update.Email 130 | } 131 | 132 | if len(update.FirstName) != 0 { 133 | u.FirstName = update.FirstName 134 | } 135 | 136 | if len(update.LastName) != 0 { 137 | u.LastName = update.LastName 138 | } 139 | 140 | if update.Gender != 0 && (u.Gender != update.Gender) { 141 | u.Gender = update.Gender 142 | u.cacheUpdateGender() 143 | } 144 | 145 | if update.birthdateSetted && (u.BirthDate != update.BirthDate) { 146 | u.BirthDate = update.BirthDate 147 | u.cacheUpdateBirthdate() 148 | } 149 | 150 | return true 151 | } 152 | 153 | func (u *User) cacheUpdateBirthdate() { 154 | for _, uv := range u.cache.visits { 155 | if visit := indexVisit.Get(uv.visitId); visit == nil { 156 | log.Println(`WTF visit nil in user cache`, uv.visitId) 157 | } else if location := indexLocation.Get(visit.Location); location == nil { 158 | log.Println(`WTF location nil in user cache`, visit.Location) 159 | } else { 160 | for i, la := range location.cache.locations { 161 | if la.visitId == visit.Id { 162 | location.cache.locations[i].birthdate = u.BirthDate 163 | } 164 | } 165 | } 166 | } 167 | } 168 | 169 | func (u *User) cacheUpdateGender() { 170 | for _, uv := range u.cache.visits { 171 | if visit := indexVisit.Get(uv.visitId); visit == nil { 172 | log.Println(`WTF visit nil in user cache`, uv.visitId) 173 | } else if location := indexLocation.Get(visit.Location); location == nil { 174 | log.Println(`WTF location nil in user cache`, visit.Location) 175 | } else { 176 | for i, la := range location.cache.locations { 177 | if la.visitId == visit.Id { 178 | location.cache.locations[i].gender = u.Gender 179 | } 180 | } 181 | } 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /user_visits.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strconv" 5 | ) 6 | 7 | /* 8 | /users//visits 9 | отсортированная по возрастанию дат 10 | 11 | Возможные GET-параметры: 12 | fromDate - посещения с visited_at > fromDate 13 | toDate - посещения с visited_at < toDate 14 | country - название страны, в которой находятся интересующие достопримечательности 15 | toDistance - возвращать только те места, у которых расстояние от города меньше этого параметра 16 | 17 | Пример корректного ответа на запрос: 18 | { 19 | "mark": 2, 20 | "visited_at": 1223268286, 21 | "place": "Кольский полуостров" 22 | }, 23 | */ 24 | 25 | type ( 26 | UserVisits struct { 27 | visits []UserVisit 28 | } 29 | 30 | UserVisit struct { 31 | visitId int32 32 | visitedAt int32 33 | visitedAtStr [12]byte 34 | visitedAtStrLen int32 35 | distance int32 36 | countryIdx int32 37 | markChar byte // уже символом, не 0..5 38 | place []byte 39 | } 40 | ) 41 | 42 | func (uv *UserVisits) allocSpaceByVisitedAt(visitedAt int32) (idx int) { 43 | l := len(uv.visits) 44 | 45 | for idx = 0; idx < l; idx++ { 46 | if uv.visits[idx].visitedAt > visitedAt { 47 | break 48 | } 49 | } 50 | 51 | uv.visits = append(uv.visits, UserVisit{}) 52 | 53 | if idx == l { 54 | // добавление в конец или в пустой список 55 | // мы уже выделили место на +1 элемент. больше ничего делать не надо 56 | } else { 57 | // добавление в середину или начало, нужно сдвигать 58 | copy(uv.visits[idx+1:], uv.visits[idx:]) 59 | } 60 | 61 | return idx 62 | } 63 | 64 | func (uv *UserVisits) Add(location *Location, visit *Visit) bool { 65 | idx := uv.allocSpaceByVisitedAt(visit.VisitedAt) 66 | 67 | item := &uv.visits[idx] 68 | item.visitId = visit.Id 69 | item.visitedAt = visit.VisitedAt 70 | buf := strconv.AppendInt(item.visitedAtStr[:0], int64(visit.VisitedAt), 10) 71 | item.visitedAtStrLen = int32(len(buf)) 72 | item.distance = location.Distance 73 | item.countryIdx = location.CountryIdx 74 | item.markChar = visit.Mark + '0' 75 | item.place = location.Place 76 | 77 | return true 78 | } 79 | 80 | func (uv *UserVisits) MoveByVisitId(target *User, visitId int32) bool { 81 | currentPos := -1 82 | for i, uvItem := range uv.visits { 83 | if uvItem.visitId == visitId { 84 | currentPos = i 85 | break 86 | } 87 | } 88 | if currentPos == -1 { 89 | return false 90 | } 91 | 92 | bak := uv.visits[currentPos] 93 | 94 | // удаляем из себя 95 | l := len(uv.visits) 96 | if currentPos < l-1 { 97 | copy(uv.visits[currentPos:], uv.visits[currentPos+1:]) 98 | } 99 | uv.visits = uv.visits[:l-1] 100 | 101 | // добавляем в новый список 102 | idx := target.cache.allocSpaceByVisitedAt(bak.visitedAt) 103 | target.cache.visits[idx] = bak 104 | 105 | return true 106 | } 107 | 108 | func (uv *UserVisits) ChangeVisitedAtByVisitId(visitId int32, visitedAt int32) { 109 | visitPos := -1 110 | for i, v := range uv.visits { 111 | if v.visitId == visitId { 112 | // тут поменять visitedAt еще нельзя, чтобы не сломался поиск новой позиции 113 | visitPos = i 114 | break 115 | } 116 | } 117 | if visitPos < 0 { 118 | return 119 | } 120 | 121 | // ищем, с кем поменяться местами 122 | switchPos := 0 123 | l := len(uv.visits) 124 | for switchPos = 0; switchPos < l; switchPos++ { 125 | if uv.visits[switchPos].visitedAt > visitedAt { 126 | break 127 | } 128 | } 129 | 130 | uv.visits[visitPos].visitedAt = visitedAt 131 | buf := strconv.AppendInt(uv.visits[visitPos].visitedAtStr[:0], int64(visitedAt), 10) 132 | uv.visits[visitPos].visitedAtStrLen = int32(len(buf)) 133 | 134 | // меняемся местами 135 | bak := uv.visits[visitPos] 136 | if switchPos > visitPos { 137 | cnt := switchPos - visitPos - 1 138 | copy(uv.visits[visitPos:visitPos+cnt], uv.visits[visitPos+1:]) 139 | uv.visits[switchPos-1] = bak 140 | } else if switchPos < visitPos { 141 | cnt := visitPos - switchPos 142 | copy(uv.visits[switchPos+1:], uv.visits[switchPos:switchPos+cnt]) 143 | uv.visits[switchPos] = bak 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "strconv" 6 | "sync" 7 | "time" 8 | "unicode/utf8" 9 | ) 10 | 11 | var ( 12 | bytesPool = sync.Pool{ 13 | New: func() interface{} { 14 | return make([]byte, 4094) 15 | }, 16 | } 17 | ) 18 | 19 | func byteSliceToInt64(s []byte) (res int64, ok bool) { 20 | sign := len(s) > 0 && s[0] == '-' 21 | if sign { 22 | s = s[1:] 23 | } 24 | 25 | ok = true 26 | 27 | res = 0 28 | for _, c := range s { 29 | if v := int64(c - '0'); v < 0 || v > 9 { 30 | ok = false 31 | break 32 | } else { 33 | res = res*10 + v 34 | } 35 | } 36 | 37 | if sign { 38 | res = -res 39 | } 40 | 41 | return 42 | } 43 | 44 | func utf8UnescapedLen(b []byte) int { 45 | if len(b) == 0 { 46 | return 0 47 | } 48 | return utf8.RuneCount(utf8Unescaped(b)) 49 | } 50 | 51 | // хак для перевода экранированных строк вида "\u1234\u5678" в нормальный юникод 52 | // выделяет память под ответ 53 | func utf8Unescaped(b []byte) []byte { 54 | buf := bytesPool.Get().([]byte)[:0] 55 | 56 | var tmp [4]byte 57 | 58 | i, l := 0, len(b) 59 | for i < l { 60 | ch := b[i] 61 | 62 | // \u1234 63 | 64 | // в случае любых ошибок просто пропускаем один байт 65 | if ch != '\\' { 66 | } else if (i >= l-4) || b[i+1] != 'u' { 67 | } else if r, err := strconv.ParseUint(string(b[i+2:i+6]), 16, 64); err != nil { 68 | } else { 69 | n := utf8.EncodeRune(tmp[:], rune(r)) 70 | buf = append(buf, tmp[:n]...) 71 | i += 6 72 | continue 73 | } 74 | 75 | buf = append(buf, ch) 76 | i++ 77 | } 78 | 79 | res := append([]byte{}, buf...) 80 | 81 | bytesPool.Put(buf) 82 | 83 | return res 84 | } 85 | 86 | func timeFloor(t time.Time) time.Time { 87 | year, month, day := t.Date() 88 | t = time.Date(year, month, day, 0, 0, 0, 0, t.Location()) 89 | return t 90 | } 91 | 92 | var agesTimestamps []int64 93 | 94 | func ageToTimestampWarming() { 95 | for age := 0; age <= 200; age++ { 96 | agesTimestamps = append(agesTimestamps, ageToTimestamp(int32(age))) 97 | } 98 | } 99 | 100 | func ageToTimestamp(age int32) int64 { 101 | if (age >= 0) && int(age) < len(agesTimestamps) { 102 | return agesTimestamps[age] 103 | } 104 | 105 | year, month, day := timeNow.Date() 106 | ts := time.Date(year-int(age), month, day, 0, 0, 0, 0, timeNow.Location()) 107 | return ts.Unix() 108 | } 109 | 110 | func bytesToLowerInplace(buf []byte) { 111 | for i, ch := range buf { 112 | if ch >= 'A' && ch <= 'Z' { 113 | buf[i] += 'a' - 'A' 114 | } 115 | } 116 | } 117 | 118 | func bytesTrimLeftInplace(buf []byte) []byte { 119 | i, l := 0, len(buf) 120 | for ; i < l && buf[i] == ' '; i++ { 121 | } 122 | return buf[i:] 123 | } 124 | 125 | // скопировано из github.com/valyala/fasthttp 126 | var hex2intTable = func() []byte { 127 | b := make([]byte, 255) 128 | for i := byte(0); i < 255; i++ { 129 | c := byte(16) 130 | if i >= '0' && i <= '9' { 131 | c = i - '0' 132 | } else if i >= 'a' && i <= 'f' { 133 | c = i - 'a' + 10 134 | } else if i >= 'A' && i <= 'F' { 135 | c = i - 'A' + 10 136 | } 137 | b[i] = c 138 | } 139 | return b 140 | }() 141 | 142 | // скопировано из github.com/valyala/fasthttp 143 | func urlDecode(dst, src []byte) []byte { 144 | if bytes.IndexByte(src, '%') < 0 && bytes.IndexByte(src, '+') < 0 { 145 | // fast path: src doesn't contain encoded chars 146 | return append(dst, src...) 147 | } 148 | 149 | // slow path 150 | for i := 0; i < len(src); i++ { 151 | c := src[i] 152 | if c == '%' { 153 | if i+2 >= len(src) { 154 | return append(dst, src[i:]...) 155 | } 156 | x2 := hex2intTable[src[i+2]] 157 | x1 := hex2intTable[src[i+1]] 158 | if x1 == 16 || x2 == 16 { 159 | dst = append(dst, '%') 160 | } else { 161 | dst = append(dst, x1<<4|x2) 162 | i += 2 163 | } 164 | } else if c == '+' { 165 | dst = append(dst, ' ') 166 | } else { 167 | dst = append(dst, c) 168 | } 169 | } 170 | return dst 171 | } 172 | -------------------------------------------------------------------------------- /visit.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "log" 6 | "strconv" 7 | ) 8 | 9 | const ( 10 | minMarkValue = 0 11 | maxMarkValue = 5 12 | ) 13 | 14 | type ( 15 | Visit struct { 16 | Id int32 17 | Location int32 18 | User int32 19 | VisitedAt int32 20 | Mark uint8 21 | markSetted bool 22 | } 23 | ) 24 | 25 | func (v *Visit) Parse(buf []byte) bool { 26 | changed := false 27 | 28 | err := ParseItem(buf, func(key, value []byte, valueType JSValueType) bool { 29 | changed = true // проверка, что не совсем пустой buf пришел 30 | 31 | if bytes.Equal(key, strId) { 32 | if valueType != jsValueTypeNumeric { 33 | return false 34 | } else if i64, ok := byteSliceToInt64(value); !ok { 35 | return false 36 | } else { 37 | v.Id = int32(i64) 38 | } 39 | } else if bytes.Equal(key, strLocation) { 40 | if valueType != jsValueTypeNumeric { 41 | return false 42 | } else if i64, ok := byteSliceToInt64(value); !ok { 43 | return false 44 | } else { 45 | v.Location = int32(i64) 46 | } 47 | } else if bytes.Equal(key, strUser) { 48 | if valueType != jsValueTypeNumeric { 49 | return false 50 | } else if i64, ok := byteSliceToInt64(value); !ok { 51 | return false 52 | } else { 53 | v.User = int32(i64) 54 | } 55 | } else if bytes.Equal(key, strVisitedAt) { 56 | if valueType != jsValueTypeNumeric { 57 | return false 58 | } else if i64, ok := byteSliceToInt64(value); !ok { 59 | return false 60 | } else { 61 | v.VisitedAt = int32(i64) 62 | } 63 | } else if bytes.Equal(key, strMark) { 64 | if valueType != jsValueTypeNumeric { 65 | return false 66 | } else if mark, ok := byteSliceToInt64(value); !ok || (mark < minMarkValue) || (mark > maxMarkValue) { 67 | return false 68 | } else { 69 | v.Mark = uint8(mark) 70 | v.markSetted = true 71 | } 72 | } // else { // неизвестный ключ 73 | 74 | return true 75 | }) 76 | 77 | return (err == nil) && changed 78 | } 79 | 80 | func (v *Visit) Serialize(buf []byte) []byte { 81 | buf = append(buf, `{"id":`...) 82 | buf = strconv.AppendInt(buf, int64(v.Id), 10) 83 | buf = append(buf, `,"location":`...) 84 | buf = strconv.AppendInt(buf, int64(v.Location), 10) 85 | buf = append(buf, `,"user":`...) 86 | buf = strconv.AppendInt(buf, int64(v.User), 10) 87 | buf = append(buf, `,"visited_at":`...) 88 | buf = strconv.AppendInt(buf, int64(v.VisitedAt), 10) 89 | buf = append(buf, `,"mark":`...) 90 | buf = strconv.AppendInt(buf, int64(v.Mark), 10) 91 | buf = append(buf, '}') 92 | 93 | return buf 94 | } 95 | 96 | func (v *Visit) CheckFields(update bool) bool { 97 | // ToDo: location - id 98 | // ToDo: user - id 99 | if v.Mark > maxMarkValue { 100 | return false 101 | } 102 | 103 | if !update && (!v.markSetted || v.VisitedAt == 0 || v.User == 0 || v.Location == 0) { 104 | // при создании должны передаваться все поля 105 | return false 106 | } 107 | 108 | if update && (v.Id != 0) { 109 | // id не может обновляться 110 | return false 111 | } 112 | 113 | return true 114 | } 115 | 116 | func (v *Visit) Update(update *Visit) bool { 117 | /* 118 | Если меняется Location: 119 | - удалить (по LocationAvg.visitId) из старого Location.cache и добавить в новый 120 | - в UserVisits обновить поля distance, countryIdx 121 | Если меняется User: 122 | - удалить (по UserVisit.visitId) из старого User.cache и добавить в новый 123 | - в LocationsAvg обновить поля birthdate и gender 124 | Если меняется VisitedAt: 125 | - в LocationsAvg обновить visitedAt 126 | - в UserVisits обновить visitedAt 127 | Если меняется Mark: 128 | - в LocationsAvg обновить mark 129 | */ 130 | 131 | if update.Location != 0 && (v.Location != update.Location) { 132 | old := v.Location 133 | v.Location = update.Location 134 | v.cacheUpdateLocation(old) 135 | } 136 | 137 | if update.User != 0 && (v.User != update.User) { 138 | old := v.User 139 | v.User = update.User 140 | v.cacheUpdateUser(old) 141 | } 142 | 143 | if update.VisitedAt != 0 && (v.VisitedAt != update.VisitedAt) { 144 | v.VisitedAt = update.VisitedAt 145 | v.cacheUpdateVisitedAt() 146 | } 147 | 148 | if update.markSetted && (v.Mark != update.Mark) { 149 | v.Mark = update.Mark 150 | v.cacheUpdateMark() 151 | } 152 | 153 | return true 154 | } 155 | 156 | func (v *Visit) cacheUpdateLocation(oldLocationId int32) { 157 | if location := indexLocation.Get(v.Location); location == nil { 158 | log.Println(`WTF location nil in visit`, v.Location) 159 | } else if user := indexUser.Get(v.User); user == nil { 160 | log.Println(`WTF user nil in visit`, v.User) 161 | } else if locationOld := indexLocation.Get(oldLocationId); locationOld == nil { 162 | log.Println(`WTF location nil in visit`, oldLocationId) 163 | } else { 164 | locationOld.cache.MoveByVisitId(location, v.Id) 165 | 166 | for i, uv := range user.cache.visits { 167 | if uv.visitId == v.Id { 168 | user.cache.visits[i].place = location.Place 169 | user.cache.visits[i].distance = location.Distance 170 | user.cache.visits[i].countryIdx = location.CountryIdx 171 | } 172 | } 173 | } 174 | } 175 | 176 | func (v *Visit) cacheUpdateUser(oldUserId int32) { 177 | if user := indexUser.Get(v.User); user == nil { 178 | log.Println(`WTF user nil in visit`, v.User) 179 | } else if location := indexLocation.Get(v.Location); location == nil { 180 | log.Println(`WTF location nil in visit`, v.Location) 181 | } else if userOld := indexUser.Get(oldUserId); userOld == nil { 182 | log.Println(`WTF user nil in visit`, oldUserId) 183 | } else { 184 | userOld.cache.MoveByVisitId(user, v.Id) 185 | 186 | for i, la := range location.cache.locations { 187 | if la.visitId == v.Id { 188 | location.cache.locations[i].birthdate = user.BirthDate 189 | location.cache.locations[i].gender = user.Gender 190 | } 191 | } 192 | } 193 | } 194 | 195 | func (v *Visit) cacheUpdateVisitedAt() { 196 | if location := indexLocation.Get(v.Location); location == nil { 197 | log.Println(`WTF location nil in visit`, v.Location) 198 | } else { 199 | for i, la := range location.cache.locations { 200 | if la.visitId == v.Id { 201 | location.cache.locations[i].visitedAt = v.VisitedAt 202 | } 203 | } 204 | } 205 | 206 | if user := indexUser.Get(v.User); user == nil { 207 | log.Println(`WTF user nil in visit`, v.User) 208 | } else { 209 | user.cache.ChangeVisitedAtByVisitId(v.Id, v.VisitedAt) 210 | } 211 | } 212 | 213 | func (v *Visit) cacheUpdateMark() { 214 | if location := indexLocation.Get(v.Location); location == nil { 215 | log.Println(`WTF location nil in visit`, v.Location) 216 | } else { 217 | for i, la := range location.cache.locations { 218 | if la.visitId == v.Id { 219 | location.cache.locations[i].mark = v.Mark 220 | } 221 | } 222 | } 223 | 224 | if user := indexUser.Get(v.User); user == nil { 225 | log.Println(`WTF user nil in visit`, v.User) 226 | } else { 227 | for i, uv := range user.cache.visits { 228 | if uv.visitId == v.Id { 229 | user.cache.visits[i].markChar = v.Mark + '0' 230 | } 231 | } 232 | } 233 | } 234 | --------------------------------------------------------------------------------