├── .gitignore ├── example ├── view │ ├── static │ │ └── .gitkeep │ ├── src │ │ ├── router │ │ │ ├── _import_production.js │ │ │ ├── _import_development.js │ │ │ └── index.js │ │ ├── assets │ │ │ └── logo.png │ │ ├── api │ │ │ └── example.js │ │ ├── App.vue │ │ ├── main.js │ │ ├── utils │ │ │ └── fetch.js │ │ └── views │ │ │ └── index.vue │ ├── config │ │ ├── prod.env.js │ │ ├── dev.env.js │ │ └── index.js │ ├── .editorconfig │ ├── .gitignore │ ├── .postcssrc.js │ ├── build │ │ ├── dev-client.js │ │ ├── vue-loader.conf.js │ │ ├── build.js │ │ ├── webpack.dev.conf.js │ │ ├── check-versions.js │ │ ├── webpack.base.conf.js │ │ ├── utils.js │ │ ├── dev-server.js │ │ └── webpack.prod.conf.js │ ├── .babelrc │ ├── index.html │ ├── README.md │ └── package.json ├── .gitignore └── main.go ├── lbsengine.pdf ├── types ├── search_response.go ├── scored_document.go ├── indexed_document.go ├── search_request.go ├── options.go ├── indexed_document_gen_test.go └── indexed_document_gen.go ├── dbtest ├── cachego_test.go ├── gocache_test.go ├── redis_test.go ├── tidb_test.go └── Earthpoint_test.go ├── spider ├── spider_test.go ├── jsonOperate.go └── spider.go ├── core ├── cacher.go ├── geohash_test.go ├── geohash.go ├── indexer_test.go └── indexer.go ├── Makefile ├── README.md ├── LICENSE ├── engine ├── worker.go ├── engine_test.go └── engine.go └── distanceMeasure ├── distanceMeasure_test.go └── distanceMeasure.go /.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | -------------------------------------------------------------------------------- /example/view/static/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /example/.gitignore: -------------------------------------------------------------------------------- 1 | example 2 | -------------------------------------------------------------------------------- /lbsengine.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sillydong/lbsengine/HEAD/lbsengine.pdf -------------------------------------------------------------------------------- /example/view/src/router/_import_production.js: -------------------------------------------------------------------------------- 1 | module.exports = file => () => import('@/views/' + file + '.vue') 2 | -------------------------------------------------------------------------------- /example/view/src/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sillydong/lbsengine/HEAD/example/view/src/assets/logo.png -------------------------------------------------------------------------------- /example/view/config/prod.env.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | module.exports = { 3 | NODE_ENV: '"production"', 4 | BASE_API: '"/"' 5 | } 6 | -------------------------------------------------------------------------------- /example/view/src/router/_import_development.js: -------------------------------------------------------------------------------- 1 | module.exports = file => require('@/views/' + file + '.vue').default // vue-loader at least v13.0.0+ 2 | -------------------------------------------------------------------------------- /types/search_response.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | //搜索结果返回 4 | type SearchResponse struct { 5 | Docs ScoredDocuments //排序好的结果 6 | Timeout bool //是否超时 7 | Count int //数量 8 | } 9 | -------------------------------------------------------------------------------- /example/view/.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | -------------------------------------------------------------------------------- /example/view/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules/ 3 | dist/ 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Editor directories and files 9 | .idea 10 | .vscode 11 | *.suo 12 | *.ntvs* 13 | *.njsproj 14 | *.sln 15 | -------------------------------------------------------------------------------- /example/view/.postcssrc.js: -------------------------------------------------------------------------------- 1 | // https://github.com/michael-ciniawsky/postcss-load-config 2 | 3 | module.exports = { 4 | "plugins": { 5 | // to edit target browsers: use "browserslist" field in package.json 6 | "autoprefixer": {} 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /example/view/config/dev.env.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | const merge = require('webpack-merge') 3 | const prodEnv = require('./prod.env') 4 | 5 | module.exports = merge(prodEnv, { 6 | NODE_ENV: '"development"', 7 | BASE_API: '"http://localhost:8081"' 8 | }) 9 | -------------------------------------------------------------------------------- /example/view/build/dev-client.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | 'use strict' 3 | require('eventsource-polyfill') 4 | var hotClient = require('webpack-hot-middleware/client?noInfo=true&reload=true') 5 | 6 | hotClient.subscribe(function (event) { 7 | if (event.action === 'reload') { 8 | window.location.reload() 9 | } 10 | }) 11 | -------------------------------------------------------------------------------- /example/view/src/api/example.js: -------------------------------------------------------------------------------- 1 | import {get,post,del} from "@/utils/fetch"; 2 | 3 | export function api_add(form){ 4 | return post('/api/add',form) 5 | } 6 | 7 | export function api_del(id){ 8 | return del('/api/del/'+id) 9 | } 10 | 11 | export function api_query(params){ 12 | return get("/api/query",params) 13 | } 14 | 15 | -------------------------------------------------------------------------------- /example/view/src/router/index.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import Router from 'vue-router' 3 | 4 | const _import = require('./_import_' + process.env.NODE_ENV) 5 | 6 | Vue.use(Router) 7 | 8 | export default new Router({ 9 | routes: [ 10 | { 11 | path: '/', 12 | name: 'LBS Example', 13 | component: _import('index') 14 | } 15 | ] 16 | }) 17 | -------------------------------------------------------------------------------- /example/view/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | ["env", { 4 | "modules": false, 5 | "targets": { 6 | "browsers": ["> 1%", "last 2 versions", "not ie <= 8"] 7 | } 8 | }], 9 | "stage-2" 10 | ], 11 | "plugins": ["transform-runtime"], 12 | "env": { 13 | "test": { 14 | "presets": ["env", "stage-2",["es2015", { "modules": false }]], 15 | "plugins": ["istanbul"] 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /example/view/src/App.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 12 | 13 | 24 | -------------------------------------------------------------------------------- /example/view/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | lbsexample 9 | 10 | 11 |
12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /types/scored_document.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | //排序的struct 4 | type ScoredDocument struct { 5 | Distance float64 6 | DocId uint64 7 | Model interface{} //为外围组装数据提供 8 | } 9 | 10 | //排序 11 | type ScoredDocuments []*ScoredDocument 12 | 13 | func (docs ScoredDocuments) Len() int { 14 | return len(docs) 15 | } 16 | 17 | func (docs ScoredDocuments) Swap(i, j int) { 18 | docs[i], docs[j] = docs[j], docs[i] 19 | } 20 | 21 | func (docs ScoredDocuments) Less(i, j int) bool { 22 | return docs[i].Distance < docs[j].Distance 23 | } 24 | -------------------------------------------------------------------------------- /example/view/build/vue-loader.conf.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | const utils = require('./utils') 3 | const config = require('../config') 4 | const isProduction = process.env.NODE_ENV === 'production' 5 | 6 | module.exports = { 7 | loaders: utils.cssLoaders({ 8 | sourceMap: isProduction 9 | ? config.build.productionSourceMap 10 | : config.dev.cssSourceMap, 11 | extract: isProduction 12 | }), 13 | transformToRequire: { 14 | video: 'src', 15 | source: 'src', 16 | img: 'src', 17 | image: 'xlink:href' 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /types/indexed_document.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | //go:generate msgp 4 | 5 | //固态存储中存储的struct 6 | type IndexedDocument struct { 7 | DocId uint64 `msg:"id"` 8 | Latitude float64 `msg:"lat"` 9 | Longitude float64 `msg:"long"` 10 | Fields interface{} `msg:"f"` 11 | } 12 | 13 | //marshal 14 | func (z *IndexedDocument) MarshalBinary() (data []byte, err error) { 15 | return z.MarshalMsg(nil) 16 | } 17 | 18 | //unmarshal 19 | func (z *IndexedDocument) UnmarshalBinary(data []byte) error { 20 | _, err := z.UnmarshalMsg(data) 21 | return err 22 | } 23 | -------------------------------------------------------------------------------- /example/view/README.md: -------------------------------------------------------------------------------- 1 | # view 2 | 3 | > A Vue.js project 4 | 5 | ## Build Setup 6 | 7 | ``` bash 8 | # install dependencies 9 | npm install 10 | 11 | # serve with hot reload at localhost:8080 12 | npm run dev 13 | 14 | # build for production with minification 15 | npm run build 16 | 17 | # build for production and view the bundle analyzer report 18 | npm run build --report 19 | ``` 20 | 21 | For a detailed explanation on how things work, check out the [guide](http://vuejs-templates.github.io/webpack/) and [docs for vue-loader](http://vuejs.github.io/vue-loader). 22 | -------------------------------------------------------------------------------- /dbtest/cachego_test.go: -------------------------------------------------------------------------------- 1 | package dbtest 2 | 3 | import ( 4 | "fmt" 5 | "github.com/muesli/cache2go" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | var cachegoclient *cache2go.CacheTable 11 | 12 | func init() { 13 | cachegoclient = cache2go.Cache("mycache") 14 | } 15 | 16 | func BenchmarkBSet(b *testing.B) { 17 | for i := 0; i < b.N; i++ { 18 | key := fmt.Sprintf("key%v", i) 19 | value := fmt.Sprintf("value%v", i) 20 | cachegoclient.Add(key, 5*time.Minute, value) 21 | } 22 | } 23 | 24 | func BenchmarkBGet(b *testing.B) { 25 | for i := 0; i < b.N; i++ { 26 | key := fmt.Sprintf("key%v", i) 27 | cachegoclient.Value(key) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /dbtest/gocache_test.go: -------------------------------------------------------------------------------- 1 | package dbtest 2 | 3 | import ( 4 | "fmt" 5 | "github.com/patrickmn/go-cache" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | var gocacheclient *cache.Cache 11 | 12 | func init() { 13 | gocacheclient = cache.New(5*time.Minute, 10*time.Minute) 14 | } 15 | 16 | func BenchmarkASet(b *testing.B) { 17 | for i := 0; i < b.N; i++ { 18 | key := fmt.Sprintf("key%v", i) 19 | value := fmt.Sprintf("value%v", i) 20 | gocacheclient.Set(key, value, 5*time.Minute) 21 | } 22 | } 23 | 24 | func BenchmarkAGet(b *testing.B) { 25 | for i := 0; i < b.N; i++ { 26 | key := fmt.Sprintf("key%v", i) 27 | gocacheclient.Get(key) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /example/view/src/main.js: -------------------------------------------------------------------------------- 1 | // The Vue build version to load with the `import` command 2 | // (runtime-only or standalone) has been set in webpack.base.conf with an alias. 3 | import Vue from 'vue' 4 | import App from './App' 5 | import router from './router' 6 | 7 | import Mint from 'mint-ui' 8 | import 'mint-ui/lib/style.css' 9 | import VueAMap from 'vue-amap' 10 | 11 | Vue.use(Mint) 12 | Vue.use(VueAMap) 13 | VueAMap.initAMapApiLoader({ 14 | key: '8cc3327f86ecdbc7840b2bee9e0997da', 15 | plugin: ['Geolocation','Autocomplete','Scale','OverView','ToolBar'] 16 | }); 17 | 18 | Vue.config.productionTip = false 19 | 20 | /* eslint-disable no-new */ 21 | new Vue({ 22 | el: '#app', 23 | router, 24 | template: '', 25 | components: { App } 26 | }) 27 | -------------------------------------------------------------------------------- /spider/spider_test.go: -------------------------------------------------------------------------------- 1 | package spider 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/3zheng/railgun/PoolAndAgent" 8 | ) 9 | 10 | func TestGetGaodeData(t *testing.T) { 11 | //初始化并连接mysql数据库 12 | pDBProcess := PoolAndAgent.CreateADODatabase("root:123456@tcp(localhost:3306)/gotest?charset=utf8") 13 | pDBProcess.InitDB() 14 | 15 | for i := 1; i <= 40; i++ { 16 | str := fmt.Sprintf("%d", i) 17 | arrData := GetPOIData(str) 18 | fmt.Println("长度=", len(arrData), arrData) 19 | t.Log(arrData) 20 | 21 | for _, value := range arrData { 22 | sqlExpress := fmt.Sprintf("insert into poi_data(id, name, location, longitude, latitude) values( '%s', '%s', '%s', %f, %f)", value.ID, value.Name, value.Location, value.Coordinate.Longitude, value.Coordinate.Latitude) 23 | pDBProcess.WriteToDB(sqlExpress) 24 | } 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /types/search_request.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import "time" 4 | 5 | //搜索请求 6 | type SearchRequest struct { 7 | Latitude float64 8 | Longitude float64 9 | CountOnly bool 10 | Offset int 11 | Limit int 12 | SearchOption *SearchOptions //可留空，使用引擎默认参数 13 | } 14 | 15 | //可不设置的搜索参数 16 | type SearchOptions struct { 17 | Refresh bool 18 | OrderDesc bool 19 | Timeout time.Duration 20 | Accuracy int //计算进度 21 | Circles int //圈数，默认1，不扩散 22 | Excepts map[uint64]bool //排除指定ID 23 | Filter func(doc IndexedDocument) bool 24 | } 25 | 26 | func (o *SearchOptions) Init() { 27 | if o.Accuracy == 0 { 28 | o.Accuracy = STANDARD 29 | } 30 | if o.Circles == 0 { 31 | o.Circles = 1 32 | } 33 | } 34 | 35 | const ( 36 | _ = iota 37 | STANDARD //传统计算方法 38 | MEITUAN //美团开放计算方法 39 | IMPROVED //优化的计算方法 40 | ) 41 | -------------------------------------------------------------------------------- /core/cacher.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "github.com/patrickmn/go-cache" 5 | "github.com/sillydong/lbsengine/types" 6 | "time" 7 | ) 8 | 9 | //缓存器 10 | type Cacher struct { 11 | client *cache.Cache 12 | } 13 | 14 | func (c *Cacher) Init() { 15 | c.client = cache.New(5*time.Minute, 10*time.Minute) 16 | } 17 | 18 | //读缓存 19 | func (c *Cacher) Get(key string, offset, limit int) (types.ScoredDocuments, int) { 20 | if val, ok := c.client.Get(key); ok { 21 | docs := val.(types.ScoredDocuments) 22 | size := len(docs) 23 | if offset > size { 24 | return nil, size 25 | } else if offset+limit > size { 26 | return docs[offset:], size 27 | } else { 28 | return docs[offset : offset+limit], size 29 | } 30 | } 31 | return nil, 0 32 | } 33 | 34 | //写缓存 35 | func (c *Cacher) Set(key string, value types.ScoredDocuments) { 36 | c.client.Set(key, value, 5*time.Minute) 37 | } 38 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Go parameters 2 | GOCMD=go 3 | GOBUILD=$(GOCMD) build 4 | GOCLEAN=$(GOCMD) clean 5 | GOTEST=$(GOCMD) test 6 | GOGET=$(GOCMD) get 7 | BINARY_NAME=bin/example 8 | BINARY_UNIX=$(BINARY_NAME)_unix 9 | 10 | all: build-view build build-linux 11 | build-view: 12 | cd example/view && npm run build 13 | cd example && go-bindata-assetfs view/dist/... 14 | build: 15 | $(GOBUILD) -o $(BINARY_NAME) -v ./example/... 16 | clean: 17 | $(GOCLEAN) 18 | rm -f $(BINARY_NAME) 19 | rm -f $(BINARY_UNIX) 20 | rm -rf example/view/dist/* 21 | run-view: 22 | cd example/view && npm run dev 23 | run: 24 | $(GOBUILD) -o $(BINARY_NAME) -v example/main.go 25 | ./$(BINARY_NAME) 26 | analysis: 27 | @echo "engine" 28 | @ cloc --exclude-dir=example ./ 29 | @echo "example" 30 | @ cloc example/main.go 31 | @echo "view" 32 | @ cloc example/view/src 33 | 34 | # Cross compilation 35 | build-linux: 36 | CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GOBUILD) -o $(BINARY_UNIX) ./example/... 37 | -------------------------------------------------------------------------------- /dbtest/redis_test.go: -------------------------------------------------------------------------------- 1 | package dbtest 2 | 3 | import ( 4 | "github.com/go-redis/redis" 5 | "github.com/sillydong/goczd/godata" 6 | "math/rand" 7 | "strconv" 8 | "testing" 9 | ) 10 | 11 | var rclient *redis.Client 12 | 13 | func init() { 14 | rclient = redis.NewClient(&redis.Options{ 15 | Addr: "127.0.0.1:6379", 16 | DB: 4, 17 | }) 18 | } 19 | 20 | func BenchmarkRAdd(b *testing.B) { 21 | for i := 0; i < b.N; i++ { 22 | lat, lng := RandomPoint() 23 | rclient.GeoAdd("geo", &redis.GeoLocation{ 24 | Name: strconv.Itoa(i), 25 | Longitude: lng, 26 | Latitude: lat, 27 | }) 28 | } 29 | } 30 | 31 | func BenchmarkRSearch(b *testing.B) { 32 | for i := 0; i < b.N; i++ { 33 | lat, lng := RandomPoint() 34 | rclient.GeoRadius("geo", lng, lat, &redis.GeoRadiusQuery{ 35 | Unit: "m", 36 | WithDist: true, 37 | Sort: "ASC", 38 | Count: 10, 39 | }) 40 | } 41 | } 42 | 43 | func RandomPoint() (lat, lng float64) { 44 | lat = 40.8137674 + godata.Round(rand.Float64()*0.01, 7) 45 | lng = -73.8525142 + godata.Round(rand.Float64()*0.01, 7) 46 | return 47 | } 48 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | LBSENGINE 2 | ----- 3 | 4 | ## What 5 | 6 | LBSENGINE实现了一个通用的GEO索引引擎,[2017Go基金会中国黑客马拉松](http://gohack2017.golangfoundation.org/)参赛项目，获得一等奖。 7 | 8 | ## How 9 | 10 | 项目成于2017Go基金会中国黑客马拉松，演示PPT见[lbsengine.pdf](https://github.com/sillydong/lbsengine/blob/master/lbsengine.pdf) 11 | 12 | 开发可参考`example`中的代码实现 13 | 14 | ## Who 15 | 16 | - [sillydong](https://github.com/sillydong) 17 | 18 | - [3zhen](https://github.com/3zheng) 19 | 20 | ## Todo 21 | 22 | 1. 维护一份内存索引，启动时从永久存储初始化内存索引 23 | 2. 优化序列化/反序列化方法 24 | 3. 参考Redis优化geohash算法 25 | 4. ... 26 | 27 | ## Thanks 28 | 29 | 1. [redis](http://github.com/go-redis/redis) 30 | 2. [geohash](http://github.com/mmcloughlin/geohash) 31 | 3. [go-cache](http://github.com/patrickmn/go-cache) 32 | 4. [murmur hash](https://github.com/huichen/murmur) 33 | 5. [gorp](https://github.com/go-gorp/gorp) 34 | 6. [echo](https://github.com/labstack/echo) 35 | 7. [go-bindata-assetfs](https://github.com/elazarl/go-bindata-assetfs) 36 | 8. [echo-static](https://github.com/Code-Hex/echo-static) 37 | 9. [msgp](https://github.com/tinylib/msgp) 38 | 10. [tidb](https://github.com/pingcap/tidb) 39 | -------------------------------------------------------------------------------- /core/geohash_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "github.com/mmcloughlin/geohash" 5 | "testing" 6 | ) 7 | 8 | func TestEncode(t *testing.T) { 9 | latitude := 31.286305 10 | longitude := 121.448463 11 | hash := geohash.EncodeWithPrecision(latitude, longitude, 5) 12 | t.Log(hash) 13 | } 14 | 15 | func TestEncodeInt(t *testing.T) { 16 | latitude := 31.286305 17 | longitude := 121.448463 18 | hash := geohash.EncodeIntWithPrecision(latitude, longitude, 6) 19 | t.Log(hash) 20 | } 21 | 22 | func TestNeighbour(t *testing.T) { 23 | hashs := geohash.Neighbors("wtw3g") 24 | t.Logf("%+v", hashs) 25 | } 26 | 27 | func BenchmarkNeighbour(b *testing.B) { 28 | for i := 0; i < b.N; i++ { 29 | geohash.Neighbors("wtw3g") 30 | } 31 | } 32 | 33 | func TestLoop(t *testing.T) { 34 | latitude := 31.286305 35 | longitude := 121.448463 36 | neightbours := LoopNeighbours(latitude, longitude, 5, 3) 37 | t.Logf("%+v", len(neightbours)) 38 | } 39 | 40 | func BenchmarkLoop(b *testing.B) { 41 | latitude := 31.286305 42 | longitude := 121.448463 43 | for i := 0; i < b.N; i++ { 44 | LoopNeighbours(latitude, longitude, 5, 1) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Reda Lemeden 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /engine/worker.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "github.com/sillydong/lbsengine/types" 5 | ) 6 | 7 | type indexerSearchRequest struct { 8 | countonly bool 9 | hash string 10 | latitude float64 11 | longitude float64 12 | option *types.SearchOptions 13 | indexerReturnChannel chan *indexerSearchResponse 14 | } 15 | 16 | type indexerSearchResponse struct { 17 | docs types.ScoredDocuments 18 | count int 19 | } 20 | 21 | func (e *Engine) indexerAddWorker(shard int) { 22 | for { 23 | request := <-e.indexerAddChannels[shard] 24 | e.indexer.Add(request) 25 | } 26 | } 27 | 28 | func (e *Engine) indexerRemoveWorker(shard int) { 29 | for { 30 | request := <-e.indexerRemoveChannels[shard] 31 | e.indexer.Remove(request) 32 | } 33 | } 34 | 35 | func (e *Engine) indexerSearchWorker(shard int) { 36 | for { 37 | request := <-e.indexerSearchChannels[shard] 38 | docs, count := e.indexer.Search(request.countonly, request.hash, request.latitude, request.longitude, request.option) 39 | request.indexerReturnChannel <- &indexerSearchResponse{docs: docs, count: count} 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /example/view/build/build.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | require('./check-versions')() 3 | 4 | process.env.NODE_ENV = 'production' 5 | 6 | const ora = require('ora') 7 | const rm = require('rimraf') 8 | const path = require('path') 9 | const chalk = require('chalk') 10 | const webpack = require('webpack') 11 | const config = require('../config') 12 | const webpackConfig = require('./webpack.prod.conf') 13 | 14 | const spinner = ora('building for production...') 15 | spinner.start() 16 | 17 | rm(path.join(config.build.assetsRoot, config.build.assetsSubDirectory), err => { 18 | if (err) throw err 19 | webpack(webpackConfig, function (err, stats) { 20 | spinner.stop() 21 | if (err) throw err 22 | process.stdout.write(stats.toString({ 23 | colors: true, 24 | modules: false, 25 | children: false, 26 | chunks: false, 27 | chunkModules: false 28 | }) + '\n\n') 29 | 30 | if (stats.hasErrors()) { 31 | console.log(chalk.red(' Build failed with errors.\n')) 32 | process.exit(1) 33 | } 34 | 35 | console.log(chalk.cyan(' Build complete.\n')) 36 | console.log(chalk.yellow( 37 | ' Tip: built files are meant to be served over an HTTP server.\n' + 38 | ' Opening index.html over file:// won\'t work.\n' 39 | )) 40 | }) 41 | }) 42 | -------------------------------------------------------------------------------- /example/view/build/webpack.dev.conf.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | const utils = require('./utils') 3 | const webpack = require('webpack') 4 | const config = require('../config') 5 | const merge = require('webpack-merge') 6 | const baseWebpackConfig = require('./webpack.base.conf') 7 | const HtmlWebpackPlugin = require('html-webpack-plugin') 8 | const FriendlyErrorsPlugin = require('friendly-errors-webpack-plugin') 9 | 10 | // add hot-reload related code to entry chunks 11 | Object.keys(baseWebpackConfig.entry).forEach(function (name) { 12 | baseWebpackConfig.entry[name] = ['./build/dev-client'].concat(baseWebpackConfig.entry[name]) 13 | }) 14 | 15 | module.exports = merge(baseWebpackConfig, { 16 | module: { 17 | rules: utils.styleLoaders({ sourceMap: config.dev.cssSourceMap }) 18 | }, 19 | // cheap-module-eval-source-map is faster for development 20 | devtool: '#cheap-module-eval-source-map', 21 | plugins: [ 22 | new webpack.DefinePlugin({ 23 | 'process.env': config.dev.env 24 | }), 25 | // https://github.com/glenjamin/webpack-hot-middleware#installation--usage 26 | new webpack.HotModuleReplacementPlugin(), 27 | new webpack.NoEmitOnErrorsPlugin(), 28 | // https://github.com/ampedandwired/html-webpack-plugin 29 | new HtmlWebpackPlugin({ 30 | filename: 'index.html', 31 | template: 'index.html', 32 | inject: true 33 | }), 34 | new FriendlyErrorsPlugin() 35 | ] 36 | }) 37 | -------------------------------------------------------------------------------- /example/view/build/check-versions.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | const chalk = require('chalk') 3 | const semver = require('semver') 4 | const packageConfig = require('../package.json') 5 | const shell = require('shelljs') 6 | function exec (cmd) { 7 | return require('child_process').execSync(cmd).toString().trim() 8 | } 9 | 10 | const versionRequirements = [ 11 | { 12 | name: 'node', 13 | currentVersion: semver.clean(process.version), 14 | versionRequirement: packageConfig.engines.node 15 | } 16 | ] 17 | 18 | if (shell.which('npm')) { 19 | versionRequirements.push({ 20 | name: 'npm', 21 | currentVersion: exec('npm --version'), 22 | versionRequirement: packageConfig.engines.npm 23 | }) 24 | } 25 | 26 | module.exports = function () { 27 | const warnings = [] 28 | for (let i = 0; i < versionRequirements.length; i++) { 29 | const mod = versionRequirements[i] 30 | if (!semver.satisfies(mod.currentVersion, mod.versionRequirement)) { 31 | warnings.push(mod.name + ': ' + 32 | chalk.red(mod.currentVersion) + ' should be ' + 33 | chalk.green(mod.versionRequirement) 34 | ) 35 | } 36 | } 37 | 38 | if (warnings.length) { 39 | console.log('') 40 | console.log(chalk.yellow('To use this template, you must update following to modules:')) 41 | console.log() 42 | for (let i = 0; i < warnings.length; i++) { 43 | const warning = warnings[i] 44 | console.log(' ' + warning) 45 | } 46 | console.log() 47 | process.exit(1) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /dbtest/tidb_test.go: -------------------------------------------------------------------------------- 1 | package dbtest 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/go-gorp/gorp" 6 | "github.com/go-sql-driver/mysql" 7 | "log" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | var client *gorp.DbMap 13 | 14 | func init() { 15 | dbconfig := &mysql.Config{ 16 | User: "root", 17 | Passwd: "", 18 | DBName: "lbsexample", 19 | Net: "tcp", 20 | Addr: "127.0.0.1:4000", 21 | } 22 | db, err := sql.Open("mysql", dbconfig.FormatDSN()) 23 | if err != nil { 24 | log.Fatal(err) 25 | } 26 | if err := db.Ping(); err != nil { 27 | log.Fatal(err) 28 | } 29 | db.SetConnMaxLifetime(time.Minute) 30 | db.SetMaxIdleConns(10) 31 | db.SetMaxOpenConns(10) 32 | 33 | client = &gorp.DbMap{Db: db, Dialect: gorp.MySQLDialect{}} 34 | 35 | client.AddTable(Tidbtest{}).SetKeys(true, "id") 36 | } 37 | 38 | type Tidbtest struct { 39 | Id int64 `db:"id"` 40 | Test string `db:"test"` 41 | } 42 | 43 | func TestAdd(t *testing.T) { 44 | data := &Tidbtest{Test: "hello"} 45 | err := client.Insert(data) 46 | if err != nil { 47 | t.Error(err) 48 | } else { 49 | t.Logf("%+v", data) 50 | } 51 | } 52 | 53 | func TestRemove(t *testing.T) { 54 | data := &Tidbtest{Id: 1} 55 | i, err := client.Delete(data) 56 | if err != nil { 57 | t.Error(err) 58 | } else { 59 | t.Log(i) 60 | } 61 | } 62 | 63 | func TestSelect(t *testing.T) { 64 | var datas []Tidbtest 65 | x, err := client.Select(&datas, "select * from tidbtest") 66 | if err != nil { 67 | t.Error(err) 68 | } else { 69 | t.Logf("%+v", x) 70 | t.Logf("%+v", datas) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /core/geohash.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import "github.com/mmcloughlin/geohash" 4 | 5 | func LoopNeighbours(latitude, longitude float64, precision uint, loop int) (neighbours []string) { 6 | if loop == 0 { 7 | loop = 1 8 | } 9 | hash := geohash.EncodeWithPrecision(latitude, longitude, precision) 10 | if loop == 1 { 11 | neighbours = geohash.Neighbors(hash) 12 | neighbours = append(neighbours, hash) 13 | return neighbours 14 | } else { 15 | neighbours = append(neighbours, hash) 16 | 17 | box := geohash.BoundingBox(hash) 18 | centerlat, centerlng := box.Center() 19 | height := box.MaxLat - box.MinLat 20 | width := box.MaxLng - box.MinLng 21 | 22 | for i := 1; i <= loop; i++ { 23 | side := 2 * i 24 | fi := float64(i) 25 | latup := centerlat + width*fi 26 | lngright := centerlng + width*fi 27 | latdown := centerlat - height*fi 28 | lngleft := centerlng - width*fi 29 | 30 | for k := 0; k < side; k++ { 31 | //上 32 | uplng := centerlng + width*float64(k-i+1) 33 | neighbours = append(neighbours, geohash.EncodeWithPrecision(latup, uplng, precision)) 34 | 35 | //右 36 | rightlat := centerlat - height*float64(k-i+1) 37 | neighbours = append(neighbours, geohash.EncodeWithPrecision(rightlat, lngright, precision)) 38 | 39 | //下 40 | downlng := centerlng - width*float64(k-i+1) 41 | neighbours = append(neighbours, geohash.EncodeWithPrecision(latdown, downlng, precision)) 42 | 43 | //左 44 | leftlat := centerlat + height*float64(k-i+1) 45 | neighbours = append(neighbours, geohash.EncodeWithPrecision(leftlat, lngleft, precision)) 46 | } 47 | } 48 | 49 | return neighbours 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /spider/jsonOperate.go: -------------------------------------------------------------------------------- 1 | package spider 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "reflect" 7 | "strconv" 8 | "strings" 9 | ) 10 | 11 | func ReadFromJson(body []byte) (datas []PoiData) { 12 | datas = make([]PoiData, 0) 13 | var dat map[string]interface{} 14 | dat = make(map[string]interface{}) 15 | err := json.Unmarshal(body, &dat) 16 | if err != nil { 17 | fmt.Println("json解析错误:", err) 18 | return 19 | } 20 | 21 | if suggestion, ok := dat["suggestion"]; ok { 22 | fmt.Println("type of suggestion: ", reflect.TypeOf(suggestion)) 23 | } 24 | 25 | if pois, ok := dat["pois"]; ok { 26 | fmt.Println("type of pois: ", reflect.TypeOf(pois)) 27 | switch data := pois.(type) { 28 | case []interface{}: 29 | fmt.Println("POI数组长度:", len(data)) 30 | for _, elem := range data { 31 | switch elem := elem.(type) { 32 | case map[string]interface{}: 33 | id, _ := elem["id"] 34 | name, _ := elem["name"] 35 | location, _ := elem["location"] 36 | fmt.Println(name, location) 37 | data := PoiData{} 38 | switch id := id.(type) { 39 | case string: 40 | data.ID = id 41 | } 42 | switch name := name.(type) { 43 | case string: 44 | data.Name = name 45 | } 46 | switch location := location.(type) { 47 | case string: 48 | data.Location = location 49 | coord := strings.Split(location, ",") 50 | data.Coordinate.Longitude, _ = strconv.ParseFloat(coord[0], 64) 51 | data.Coordinate.Latitude, _ = strconv.ParseFloat(coord[1], 64) 52 | } 53 | datas = append(datas, data) 54 | } 55 | } 56 | } 57 | } 58 | 59 | fmt.Println("datas =", datas) 60 | return 61 | } 62 | -------------------------------------------------------------------------------- /dbtest/Earthpoint_test.go: -------------------------------------------------------------------------------- 1 | package dbtest 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "testing" 7 | ) 8 | 9 | const RADIUS = 1.0 10 | 11 | type EarthCoordinate struct { 12 | longitude float64 //经度坐标 东经为正数，西经为负数 13 | latitude float64 //纬度坐标 北纬为正数，南纬为负数 14 | } 15 | 16 | func ChangeAngleToRadian(angle float64) float64 { 17 | return angle / 180.0 * math.Pi 18 | } 19 | 20 | func CalEarthPoint(pt1, pt2 EarthCoordinate) { 21 | x1 := RADIUS * math.Cos(ChangeAngleToRadian(pt1.latitude)) * math.Cos(ChangeAngleToRadian(pt1.longitude)) 22 | y1 := RADIUS * math.Cos(ChangeAngleToRadian(pt1.latitude)) * math.Sin(ChangeAngleToRadian(pt1.longitude)) 23 | z1 := RADIUS * math.Sin(ChangeAngleToRadian(pt1.latitude)) 24 | 25 | x2 := RADIUS * math.Cos(ChangeAngleToRadian(pt2.latitude)) * math.Cos(ChangeAngleToRadian(pt2.longitude)) 26 | y2 := RADIUS * math.Cos(ChangeAngleToRadian(pt2.latitude)) * math.Sin(ChangeAngleToRadian(pt2.longitude)) 27 | z2 := RADIUS * math.Sin(ChangeAngleToRadian(pt2.latitude)) 28 | 29 | distance := math.Sqrt(math.Pow(x1-x2, 2.0) + math.Pow(y1-y2, 2.0) + math.Pow(z1-z2, 2.0)) 30 | fmt.Printf("x1 = %f, y1 = %f, z1 = %f\nx2 = %f, y2 = %f, z2 = %f\ndistance = %f", x1, y1, z1, x2, y2, z2, distance) 31 | } 32 | 33 | func Test_EarthPoint(t *testing.T) { 34 | p1a, p2a := EarthCoordinate{longitude: 100.0, latitude: 20.0}, EarthCoordinate{longitude: 130.0, latitude: 40.0} 35 | p1b, p2b := EarthCoordinate{longitude: 170.0, latitude: 20.0}, EarthCoordinate{longitude: -160.0, latitude: 40.0} 36 | p1c, p2c := EarthCoordinate{longitude: 170.0, latitude: -20.0}, EarthCoordinate{longitude: -160.0, latitude: -40.0} 37 | CalEarthPoint(p1a, p2a) 38 | CalEarthPoint(p1b, p2b) 39 | CalEarthPoint(p1c, p2c) 40 | } 41 | -------------------------------------------------------------------------------- /types/options.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "runtime" 5 | "time" 6 | ) 7 | 8 | type EngineOptions struct { 9 | NumShards int //channel分片 10 | AddBuffer int //add channel长度 11 | RemoveBuffer int //remove channel长度 12 | SearchBuffer int //channel长度 13 | SearchWorkerThreads int //每个搜索channel worker数量 14 | 15 | DefaultSearchOption *SearchOptions //默认搜索配置 16 | IndexerOption *IndexerOptions //索引器配置项 17 | } 18 | 19 | func (o *EngineOptions) Init() { 20 | if o.NumShards == 0 { 21 | o.NumShards = runtime.NumCPU() 22 | } 23 | if o.AddBuffer == 0 { 24 | o.AddBuffer = runtime.NumCPU() 25 | } 26 | if o.RemoveBuffer == 0 { 27 | o.RemoveBuffer = runtime.NumCPU() 28 | } 29 | if o.SearchBuffer == 0 { 30 | o.SearchBuffer = 10 * runtime.NumCPU() 31 | } 32 | if o.SearchWorkerThreads == 0 { 33 | o.SearchWorkerThreads = 9 //9个neighbor格子 34 | } 35 | if o.DefaultSearchOption == nil { 36 | o.DefaultSearchOption = &SearchOptions{ 37 | Refresh: false, 38 | OrderDesc: false, 39 | Timeout: 2 * time.Second, 40 | Accuracy: STANDARD, 41 | Circles: 1, 42 | Excepts: nil, 43 | Filter: nil, 44 | } 45 | } 46 | if o.IndexerOption == nil { 47 | o.IndexerOption = &IndexerOptions{ 48 | RedisHost: "127.0.0.1:6379", 49 | RedisPassword: "", 50 | RedisDb: 3, 51 | HashSize: 1000, 52 | GeoShard: 5, 53 | GeoPrecious: 5, 54 | } 55 | } 56 | } 57 | 58 | type IndexerOptions struct { 59 | RedisHost string 60 | RedisPassword string 61 | RedisDb int 62 | HashSize uint64 //hash分片大小 63 | GeoShard uint64 //GEOHASH分片大小 64 | GeoPrecious uint //GEOHASH位数 65 | CenterLatitude float64 //城市中心纬度 66 | CenterLongitude float64 //城市中心经度 67 | Location string //城市 68 | } 69 | -------------------------------------------------------------------------------- /example/view/build/webpack.base.conf.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | const path = require('path') 3 | const utils = require('./utils') 4 | const config = require('../config') 5 | const vueLoaderConfig = require('./vue-loader.conf') 6 | 7 | function resolve (dir) { 8 | return path.join(__dirname, '..', dir) 9 | } 10 | 11 | module.exports = { 12 | entry: { 13 | app: './src/main.js' 14 | }, 15 | output: { 16 | path: config.build.assetsRoot, 17 | filename: '[name].js', 18 | publicPath: process.env.NODE_ENV === 'production' 19 | ? config.build.assetsPublicPath 20 | : config.dev.assetsPublicPath 21 | }, 22 | resolve: { 23 | extensions: ['.js', '.vue', '.json'], 24 | alias: { 25 | 'vue$': 'vue/dist/vue.esm.js', 26 | '@': resolve('src'), 27 | } 28 | }, 29 | module: { 30 | rules: [ 31 | { 32 | test: /\.vue$/, 33 | loader: 'vue-loader', 34 | options: vueLoaderConfig 35 | }, 36 | { 37 | test: /\.js$/, 38 | loader: 'babel-loader', 39 | include: [resolve('src'), resolve('test')] 40 | }, 41 | { 42 | test: /\.(png|jpe?g|gif|svg)(\?.*)?$/, 43 | loader: 'url-loader', 44 | options: { 45 | limit: 10000, 46 | name: utils.assetsPath('img/[name].[hash:7].[ext]') 47 | } 48 | }, 49 | { 50 | test: /\.(mp4|webm|ogg|mp3|wav|flac|aac)(\?.*)?$/, 51 | loader: 'url-loader', 52 | options: { 53 | limit: 10000, 54 | name: utils.assetsPath('media/[name].[hash:7].[ext]') 55 | } 56 | }, 57 | { 58 | test: /\.(woff2?|eot|ttf|otf)(\?.*)?$/, 59 | loader: 'url-loader', 60 | options: { 61 | limit: 10000, 62 | name: utils.assetsPath('fonts/[name].[hash:7].[ext]') 63 | } 64 | } 65 | ] 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /example/view/config/index.js: -------------------------------------------------------------------------------- 1 | 2 | 'use strict' 3 | // Template version: 1.1.3 4 | // see http://vuejs-templates.github.io/webpack for documentation. 5 | 6 | const path = require('path') 7 | 8 | module.exports = { 9 | build: { 10 | env: require('./prod.env'), 11 | index: path.resolve(__dirname, '../dist/lbs.html'), 12 | assetsRoot: path.resolve(__dirname, '../dist'), 13 | assetsSubDirectory: 'static', 14 | assetsPublicPath: '/', 15 | productionSourceMap: true, 16 | // Gzip off by default as many popular static hosts such as 17 | // Surge or Netlify already gzip all static assets for you. 18 | // Before setting to `true`, make sure to: 19 | // npm install --save-dev compression-webpack-plugin 20 | productionGzip: false, 21 | productionGzipExtensions: ['js', 'css'], 22 | // Run the build command with an extra argument to 23 | // View the bundle analyzer report after build finishes: 24 | // `npm run build --report` 25 | // Set to `true` or `false` to always turn it on or off 26 | bundleAnalyzerReport: process.env.npm_config_report 27 | }, 28 | dev: { 29 | env: require('./dev.env'), 30 | port: process.env.PORT || 8080, 31 | autoOpenBrowser: true, 32 | assetsSubDirectory: 'static', 33 | assetsPublicPath: '/', 34 | proxyTable: { 35 | '/api':{ 36 | target:'http://localhost:8877', 37 | changeOrigin: true, 38 | logLeve: 'debug', 39 | pathRewrite:{ 40 | '^/api':'/api' 41 | } 42 | } 43 | }, 44 | // CSS Sourcemaps off by default because relative paths are "buggy" 45 | // with this option, according to the CSS-Loader README 46 | // (https://github.com/webpack/css-loader#sourcemaps) 47 | // In our experience, they generally work as expected, 48 | // just be aware of this issue when enabling this option. 49 | cssSourceMap: false 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /example/view/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "view", 3 | "version": "1.0.0", 4 | "description": "A Vue.js project", 5 | "author": "Chen.Zhidong ", 6 | "private": true, 7 | "scripts": { 8 | "dev": "node build/dev-server.js", 9 | "start": "npm run dev", 10 | "build": "node build/build.js" 11 | }, 12 | "dependencies": { 13 | "axios": "^0.16.2", 14 | "mint-ui": "^2.2.9", 15 | "qs": "^6.5.1", 16 | "vue": "^2.5.2", 17 | "vue-amap": "^0.3.1", 18 | "vue-router": "^3.0.1" 19 | }, 20 | "devDependencies": { 21 | "autoprefixer": "^7.1.2", 22 | "babel-core": "^6.22.1", 23 | "babel-loader": "^7.1.1", 24 | "babel-plugin-transform-runtime": "^6.22.0", 25 | "babel-preset-env": "^1.3.2", 26 | "babel-preset-stage-2": "^6.22.0", 27 | "babel-register": "^6.22.0", 28 | "chalk": "^2.0.1", 29 | "connect-history-api-fallback": "^1.3.0", 30 | "copy-webpack-plugin": "^4.0.1", 31 | "css-loader": "^0.28.0", 32 | "eventsource-polyfill": "^0.9.6", 33 | "express": "^4.14.1", 34 | "extract-text-webpack-plugin": "^3.0.0", 35 | "file-loader": "^1.1.4", 36 | "friendly-errors-webpack-plugin": "^1.6.1", 37 | "html-webpack-plugin": "^2.30.1", 38 | "http-proxy-middleware": "^0.17.3", 39 | "webpack-bundle-analyzer": "^2.9.0", 40 | "semver": "^5.3.0", 41 | "shelljs": "^0.7.6", 42 | "opn": "^5.1.0", 43 | "optimize-css-assets-webpack-plugin": "^3.2.0", 44 | "ora": "^1.2.0", 45 | "rimraf": "^2.6.0", 46 | "url-loader": "^0.5.8", 47 | "vue-loader": "^13.3.0", 48 | "vue-style-loader": "^3.0.1", 49 | "vue-template-compiler": "^2.5.2", 50 | "portfinder": "^1.0.13", 51 | "webpack": "^3.6.0", 52 | "webpack-dev-middleware": "^1.12.0", 53 | "webpack-hot-middleware": "^2.18.2", 54 | "webpack-merge": "^4.1.0" 55 | }, 56 | "engines": { 57 | "node": ">= 4.0.0", 58 | "npm": ">= 3.0.0" 59 | }, 60 | "browserslist": [ 61 | "> 1%", 62 | "last 2 versions", 63 | "not ie <= 8" 64 | ] 65 | } 66 | -------------------------------------------------------------------------------- /example/view/build/utils.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | const path = require('path') 3 | const config = require('../config') 4 | const ExtractTextPlugin = require('extract-text-webpack-plugin') 5 | 6 | exports.assetsPath = function (_path) { 7 | const assetsSubDirectory = process.env.NODE_ENV === 'production' 8 | ? config.build.assetsSubDirectory 9 | : config.dev.assetsSubDirectory 10 | return path.posix.join(assetsSubDirectory, _path) 11 | } 12 | 13 | exports.cssLoaders = function (options) { 14 | options = options || {} 15 | 16 | const cssLoader = { 17 | loader: 'css-loader', 18 | options: { 19 | minimize: process.env.NODE_ENV === 'production', 20 | sourceMap: options.sourceMap 21 | } 22 | } 23 | 24 | // generate loader string to be used with extract text plugin 25 | function generateLoaders (loader, loaderOptions) { 26 | const loaders = [cssLoader] 27 | if (loader) { 28 | loaders.push({ 29 | loader: loader + '-loader', 30 | options: Object.assign({}, loaderOptions, { 31 | sourceMap: options.sourceMap 32 | }) 33 | }) 34 | } 35 | 36 | // Extract CSS when that option is specified 37 | // (which is the case during production build) 38 | if (options.extract) { 39 | return ExtractTextPlugin.extract({ 40 | use: loaders, 41 | fallback: 'vue-style-loader' 42 | }) 43 | } else { 44 | return ['vue-style-loader'].concat(loaders) 45 | } 46 | } 47 | 48 | // https://vue-loader.vuejs.org/en/configurations/extract-css.html 49 | return { 50 | css: generateLoaders(), 51 | postcss: generateLoaders(), 52 | less: generateLoaders('less'), 53 | sass: generateLoaders('sass', { indentedSyntax: true }), 54 | scss: generateLoaders('sass'), 55 | stylus: generateLoaders('stylus'), 56 | styl: generateLoaders('stylus') 57 | } 58 | } 59 | 60 | // Generate loaders for standalone style files (outside of .vue) 61 | exports.styleLoaders = function (options) { 62 | const output = [] 63 | const loaders = exports.cssLoaders(options) 64 | for (const extension in loaders) { 65 | const loader = loaders[extension] 66 | output.push({ 67 | test: new RegExp('\\.' + extension + '$'), 68 | use: loader 69 | }) 70 | } 71 | return output 72 | } 73 | -------------------------------------------------------------------------------- /engine/engine_test.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "fmt" 5 | "github.com/huichen/murmur" 6 | "github.com/sillydong/goczd/godata" 7 | "github.com/sillydong/lbsengine/types" 8 | "math/rand" 9 | "testing" 10 | "time" 11 | "unsafe" 12 | ) 13 | 14 | var e *Engine 15 | 16 | func init() { 17 | rand.Seed(time.Now().Unix()) 18 | e = &Engine{} 19 | e.Init(nil) 20 | } 21 | 22 | func BenchmarkAdd(b *testing.B) { 23 | for i := 0; i < b.N; i++ { 24 | lat, lng := RandomPoint() 25 | e.Add(&types.IndexedDocument{ 26 | DocId: uint64(i), 27 | Latitude: lat, 28 | Longitude: lng, 29 | Fields: map[string]string{ 30 | "a": "b", 31 | }, 32 | }) 33 | } 34 | } 35 | 36 | func TestSearch(t *testing.T) { 37 | lat, lng := RandomPoint() 38 | fmt.Println(lat, lng) 39 | result := e.Search(&types.SearchRequest{ 40 | Latitude: lat, 41 | Longitude: lng, 42 | Offset: 0, 43 | Limit: 100, 44 | }) 45 | fmt.Printf("%+v\n", result) 46 | //x,_ := json.Marshal(result) 47 | //fmt.Println(string(x)) 48 | } 49 | 50 | func BenchmarkSearch(b *testing.B) { 51 | for i := 0; i < b.N; i++ { 52 | lat, lng := RandomPoint() 53 | e.Search(&types.SearchRequest{ 54 | Latitude: lat, 55 | Longitude: lng, 56 | Offset: 0, 57 | Limit: 10, 58 | //SearchOption:&types.SearchOptions{ 59 | // Refresh:false, 60 | // Circles:2, 61 | //}, 62 | }) 63 | //fmt.Println(resp.Count) 64 | } 65 | } 66 | 67 | func TestPoint(t *testing.T) { 68 | lat, lng := RandomPoint() 69 | fmt.Printf("%+v - %+v", lat, lng) 70 | } 71 | 72 | func RandomPoint() (lat, lng float64) { 73 | lat = 40.8137674 + godata.Round(rand.Float64()*0.01, 7) 74 | lng = -73.8525142 + godata.Round(rand.Float64()*0.01, 7) 75 | return 76 | } 77 | 78 | func TestParse(t *testing.T) { 79 | t.Logf("%+v", []byte(fmt.Sprintf("%d", 1234567890))) 80 | t.Logf("%+v", parse(1234567890)) 81 | } 82 | 83 | func BenchmarkStringShard(b *testing.B) { 84 | for i := 0; i < b.N; i++ { 85 | x := uint64(i) 86 | murmur.Murmur3([]byte(fmt.Sprintf("%d", x))) 87 | } 88 | } 89 | 90 | func BenchmarkUintShard(b *testing.B) { 91 | for i := 0; i < b.N; i++ { 92 | x := uint64(i) 93 | murmur.Murmur3(parse(x)) 94 | } 95 | } 96 | 97 | func parse(d uint64) []byte { 98 | s := fmt.Sprintf("%d", d) 99 | x := (*[2]uintptr)(unsafe.Pointer(&s)) 100 | h := [3]uintptr{x[0], x[1], x[1]} 101 | return *(*[]byte)(unsafe.Pointer(&h)) 102 | } 103 | -------------------------------------------------------------------------------- /spider/spider.go: -------------------------------------------------------------------------------- 1 | package spider 2 | 3 | import ( 4 | "crypto/md5" 5 | "fmt" 6 | "github.com/sillydong/lbsengine/distanceMeasure" 7 | "io/ioutil" 8 | "net/http" 9 | "sort" 10 | ) 11 | 12 | type PoiData struct { 13 | ID string //唯一标识符 14 | Name string //兴趣点的名字 15 | Location string //字符串形式的坐标 16 | Coordinate distanceMeasure.EarthCoordinate //EarthCoordinate形式的坐标 17 | } 18 | 19 | type URL struct { 20 | UrlHead string //url的开始部分，类似http://restapi.amap.com/v3/place/text? 21 | MapParam map[string]string //url的参数部分 22 | } 23 | 24 | func (this *URL) Init(urlHead string) { 25 | this.MapParam = make(map[string]string, 50) 26 | this.UrlHead = urlHead 27 | } 28 | 29 | func (this *URL) AddParam(key, value string) { 30 | this.MapParam[key] = value 31 | } 32 | 33 | func (this *URL) GetFinalURL() string { 34 | str := this.UrlHead 35 | for key, value := range this.MapParam { 36 | str += "&" + key + "=" + value 37 | } 38 | return str 39 | } 40 | 41 | //根据已有的参数获取数字签名的MD5值,privateKey为私钥 42 | func (this *URL) GetMD5Sign(privateKey string) string { 43 | keys := make([]string, 0) 44 | for key, _ := range this.MapParam { 45 | keys = append(keys, key) 46 | } 47 | //升序排序 48 | sort.Strings(keys) 49 | //根据排序后的结果取key-value值 50 | var sign string 51 | length := len(keys) 52 | for i, key := range keys { 53 | elem, _ := this.MapParam[key] 54 | sign += key + "=" + elem 55 | if i < length-1 { 56 | //不是最后一个的话要+& 57 | sign += "&" 58 | } else { 59 | //最后一个则直接+私钥 60 | sign += privateKey 61 | } 62 | } 63 | //进行MD5加密 64 | fmt.Println("被加密的MD5串：", sign) 65 | byteMD5 := md5.Sum([]byte(sign)) 66 | sigMD5 := fmt.Sprintf("%x", byteMD5) 67 | return sigMD5 68 | } 69 | 70 | func GetPOIData(pageId string) []PoiData { 71 | //var url string = "http://restapi.amap.com/v3/place/text?&keywords=超市&city=shanghai&key=7ffc2abae565cdb9302ebaef2e45c572&extensions=all&sig=" 72 | pUrl := new(URL) 73 | pUrl.Init("http://restapi.amap.com/v3/place/text?") 74 | pUrl.AddParam("keywords", "超市") 75 | pUrl.AddParam("city", "shanghai") 76 | pUrl.AddParam("key", "7ffc2abae565cdb9302ebaef2e45c572") 77 | pUrl.AddParam("extensions", "all") 78 | pUrl.AddParam("page", pageId) 79 | //获取数字签名的MD5值 80 | sig := pUrl.GetMD5Sign("f66d435f57cb361630f2110a97aa4fd9") 81 | pUrl.AddParam("sig", sig) 82 | finalUrl := pUrl.GetFinalURL() 83 | fmt.Println("final url: ", finalUrl) 84 | resp, err := http.Get(finalUrl) 85 | if err != nil { 86 | fmt.Println("post错误:", err) 87 | return nil 88 | } 89 | fmt.Println("resp结果值：", resp) 90 | 91 | body, err := ioutil.ReadAll(resp.Body) 92 | fmt.Println("返回的body为：", string(body[:])) 93 | return ReadFromJson(body) 94 | } 95 | -------------------------------------------------------------------------------- /example/view/src/utils/fetch.js: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | import {MessageBox,Indicator} from "mint-ui"; 3 | import router from "../router/index" 4 | import * as qs from "qs"; 5 | 6 | // 创建axios实例 7 | const service = axios.create({ 8 | baseURL: process.env.BASE_API, // api的base_url 9 | timeout: 5000, // 请求超时时间 10 | }); 11 | 12 | // request拦截器 13 | service.interceptors.request.use((config) => { 14 | Indicator.open(); 15 | // Do something before request is sent 16 | config.headers.post['Content-Type'] = 'application/x-www-form-urlencoded'; 17 | if (config.method === 'post') { 18 | config.data = qs.stringify(config.data); 19 | } 20 | return config; 21 | }, (error) => { 22 | // Do something with request error 23 | console.log(error); // for debug 24 | MessageBox.alert("请求失败5",error); 25 | return Promise.reject(error); 26 | }); 27 | 28 | // respone拦截器 29 | service.interceptors.response.use( 30 | (response) => { 31 | Indicator.close() 32 | console.log(response.data) 33 | const status = response.data.status; 34 | if (status == 1) { 35 | return response 36 | } else { 37 | const err = response.data.error; 38 | switch (typeof(err)) { 39 | case "string": 40 | MessageBox.alert("请求失败4", err); 41 | break; 42 | case "object": 43 | MessageBox.alert("请求失败3", Object.values(err).join()); 44 | break; 45 | case "array": 46 | MessageBox.alert("请求失败2", Object.values(err).join()); 47 | break; 48 | } 49 | return Promise.reject(response); 50 | } 51 | }, error => { 52 | Indicator.close(); 53 | console.log('err' + error);// for debug 54 | MessageBox.alert("请求失败1", error.message); 55 | return Promise.reject(error); 56 | } 57 | ); 58 | 59 | export function get(url, params) { 60 | return new Promise((resolve, reject) => { 61 | service.get(url, {params: params}).then(response => { 62 | resolve(response.data); 63 | }, err => { 64 | reject(err) 65 | }).catch((error) => { 66 | reject(error) 67 | }); 68 | }) 69 | } 70 | 71 | export function post(url, params) { 72 | return new Promise((resolve, reject) => { 73 | service.post(url, params).then(response => { 74 | resolve(response.data); 75 | }, err => { 76 | reject(err) 77 | }).catch((error) => { 78 | reject(error) 79 | }); 80 | }) 81 | } 82 | 83 | export function del(url){ 84 | return new Promise((resolve,reject)=>{ 85 | service.del(url).then(response => { 86 | resolve(response.data); 87 | },err=>{ 88 | reject(err) 89 | }).catch(err => { 90 | reject(err) 91 | }); 92 | }) 93 | } 94 | -------------------------------------------------------------------------------- /example/view/src/views/index.vue: -------------------------------------------------------------------------------- 1 | 21 | 22 | 93 | 94 | 105 | -------------------------------------------------------------------------------- /core/indexer_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | "github.com/sillydong/lbsengine/types" 6 | "math/rand" 7 | "testing" 8 | ) 9 | 10 | var indexer *Indexer 11 | 12 | func init() { 13 | indexer = &Indexer{} 14 | indexer.Init(&types.IndexerOptions{ 15 | RedisHost: "127.0.0.1:6379", 16 | RedisDb: 3, 17 | HashSize: 1000, 18 | GeoShard: 5, 19 | GeoPrecious: 5, 20 | }) 21 | } 22 | 23 | func TestAddDocument(t *testing.T) { 24 | doc := &types.IndexedDocument{ 25 | DocId: 1, 26 | Latitude: 40.7137674, 27 | Longitude: -73.9525142, 28 | Fields: map[string]string{ 29 | "a": "b", 30 | "c": "d", 31 | }, 32 | } 33 | indexer.Add(doc) 34 | } 35 | 36 | func TestRemoveDocument(t *testing.T) { 37 | indexer.Remove(1) 38 | } 39 | 40 | func TestStore(t *testing.T) { 41 | strs, err := indexer.client.HVals("h_dr5rt_1").Result() 42 | if err != nil { 43 | t.Error(err) 44 | } 45 | docs := make([]types.IndexedDocument, 0) 46 | for _, str := range strs { 47 | document := types.IndexedDocument{} 48 | document.UnmarshalMsg([]byte(str)) 49 | docs = append(docs, document) 50 | } 51 | fmt.Printf("%+v\n", docs) 52 | } 53 | 54 | func BenchmarkStore(b *testing.B) { 55 | str, _ := indexer.client.HGet("h_dr5rt_1", "1").Result() 56 | for i := 0; i < b.N; i++ { 57 | document := types.IndexedDocument{} 58 | document.UnmarshalMsg(tobytes(str)) 59 | } 60 | } 61 | 62 | func TestSearch(t *testing.T) { 63 | option := &types.SearchOptions{ 64 | Refresh: false, 65 | OrderDesc: false, 66 | Accuracy: types.STANDARD, 67 | Circles: 1, 68 | Excepts: map[uint64]bool{ 69 | 1: true, 70 | }, 71 | Filter: func(doc types.IndexedDocument) bool { 72 | fmt.Println("filtering") 73 | if doc.Fields.(map[string]interface{})["a"] == "b" { 74 | return true 75 | } 76 | return false 77 | }, 78 | } 79 | docs, num := indexer.Search(false, "h_dr5rt_1", 40.7137674, -73.9525142, option) 80 | fmt.Println(num) 81 | if num > 0 { 82 | for _, doc := range docs { 83 | fmt.Printf("%+v\n", doc) 84 | } 85 | } 86 | } 87 | 88 | func BenchmarkSearch(b *testing.B) { 89 | option := &types.SearchOptions{ 90 | Refresh: false, 91 | OrderDesc: false, 92 | Accuracy: types.MEITUAN, 93 | Circles: 1, 94 | //Excepts: map[uint64]bool{ 95 | // 1: true, 96 | //}, 97 | //Filter: func(doc types.IndexedDocument) bool { 98 | // fmt.Println("filtering") 99 | // if doc.Fields.(map[string]interface{})["a"] == "b" { 100 | // return true 101 | // } 102 | // return false 103 | //}, 104 | } 105 | for i := 0; i < b.N; i++ { 106 | indexer.Search(false, "h_dr5rt_1", 40.7137674, -73.9525142, option) 107 | } 108 | } 109 | 110 | func TestIdShard(t *testing.T) { 111 | for i := 0; i < 10; i++ { 112 | x := rand.Int() 113 | fmt.Printf("%v -> %v\n", x, indexer.hashshard(uint64(x))) 114 | } 115 | } 116 | 117 | func TestGeoShard(t *testing.T) { 118 | for i := 0; i < 10; i++ { 119 | fmt.Printf("%v -> %v\n", i, indexer.geoshard("asdfg", uint64(i))) 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /types/indexed_document_gen_test.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | // NOTE: THIS FILE WAS PRODUCED BY THE 4 | // MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) 5 | // DO NOT EDIT 6 | 7 | import ( 8 | "bytes" 9 | "testing" 10 | 11 | "github.com/tinylib/msgp/msgp" 12 | ) 13 | 14 | func TestMarshalUnmarshalIndexedDocument(t *testing.T) { 15 | v := IndexedDocument{} 16 | bts, err := v.MarshalMsg(nil) 17 | if err != nil { 18 | t.Fatal(err) 19 | } 20 | left, err := v.UnmarshalMsg(bts) 21 | if err != nil { 22 | t.Fatal(err) 23 | } 24 | if len(left) > 0 { 25 | t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) 26 | } 27 | 28 | left, err = msgp.Skip(bts) 29 | if err != nil { 30 | t.Fatal(err) 31 | } 32 | if len(left) > 0 { 33 | t.Errorf("%d bytes left over after Skip(): %q", len(left), left) 34 | } 35 | } 36 | 37 | func BenchmarkMarshalMsgIndexedDocument(b *testing.B) { 38 | v := IndexedDocument{} 39 | b.ReportAllocs() 40 | b.ResetTimer() 41 | for i := 0; i < b.N; i++ { 42 | v.MarshalMsg(nil) 43 | } 44 | } 45 | 46 | func BenchmarkAppendMsgIndexedDocument(b *testing.B) { 47 | v := IndexedDocument{} 48 | bts := make([]byte, 0, v.Msgsize()) 49 | bts, _ = v.MarshalMsg(bts[0:0]) 50 | b.SetBytes(int64(len(bts))) 51 | b.ReportAllocs() 52 | b.ResetTimer() 53 | for i := 0; i < b.N; i++ { 54 | bts, _ = v.MarshalMsg(bts[0:0]) 55 | } 56 | } 57 | 58 | func BenchmarkUnmarshalIndexedDocument(b *testing.B) { 59 | v := IndexedDocument{} 60 | bts, _ := v.MarshalMsg(nil) 61 | b.ReportAllocs() 62 | b.SetBytes(int64(len(bts))) 63 | b.ResetTimer() 64 | for i := 0; i < b.N; i++ { 65 | _, err := v.UnmarshalMsg(bts) 66 | if err != nil { 67 | b.Fatal(err) 68 | } 69 | } 70 | } 71 | 72 | func TestEncodeDecodeIndexedDocument(t *testing.T) { 73 | v := IndexedDocument{} 74 | var buf bytes.Buffer 75 | msgp.Encode(&buf, &v) 76 | 77 | m := v.Msgsize() 78 | if buf.Len() > m { 79 | t.Logf("WARNING: Msgsize() for %v is inaccurate", v) 80 | } 81 | 82 | vn := IndexedDocument{} 83 | err := msgp.Decode(&buf, &vn) 84 | if err != nil { 85 | t.Error(err) 86 | } 87 | 88 | buf.Reset() 89 | msgp.Encode(&buf, &v) 90 | err = msgp.NewReader(&buf).Skip() 91 | if err != nil { 92 | t.Error(err) 93 | } 94 | } 95 | 96 | func BenchmarkEncodeIndexedDocument(b *testing.B) { 97 | v := IndexedDocument{} 98 | var buf bytes.Buffer 99 | msgp.Encode(&buf, &v) 100 | b.SetBytes(int64(buf.Len())) 101 | en := msgp.NewWriter(msgp.Nowhere) 102 | b.ReportAllocs() 103 | b.ResetTimer() 104 | for i := 0; i < b.N; i++ { 105 | v.EncodeMsg(en) 106 | } 107 | en.Flush() 108 | } 109 | 110 | func BenchmarkDecodeIndexedDocument(b *testing.B) { 111 | v := IndexedDocument{} 112 | var buf bytes.Buffer 113 | msgp.Encode(&buf, &v) 114 | b.SetBytes(int64(buf.Len())) 115 | rd := msgp.NewEndlessReader(buf.Bytes(), b) 116 | dc := msgp.NewReader(rd) 117 | b.ReportAllocs() 118 | b.ResetTimer() 119 | for i := 0; i < b.N; i++ { 120 | err := v.DecodeMsg(dc) 121 | if err != nil { 122 | b.Fatal(err) 123 | } 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /example/view/build/dev-server.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | require('./check-versions')() 3 | 4 | const config = require('../config') 5 | if (!process.env.NODE_ENV) { 6 | process.env.NODE_ENV = JSON.parse(config.dev.env.NODE_ENV) 7 | } 8 | 9 | const opn = require('opn') 10 | const path = require('path') 11 | const express = require('express') 12 | const webpack = require('webpack') 13 | const proxyMiddleware = require('http-proxy-middleware') 14 | const webpackConfig = require('./webpack.dev.conf') 15 | 16 | // default port where dev server listens for incoming traffic 17 | const port = process.env.PORT || config.dev.port 18 | // automatically open browser, if not set will be false 19 | const autoOpenBrowser = !!config.dev.autoOpenBrowser 20 | // Define HTTP proxies to your custom API backend 21 | // https://github.com/chimurai/http-proxy-middleware 22 | const proxyTable = config.dev.proxyTable 23 | 24 | const app = express() 25 | const compiler = webpack(webpackConfig) 26 | 27 | const devMiddleware = require('webpack-dev-middleware')(compiler, { 28 | publicPath: webpackConfig.output.publicPath, 29 | quiet: true 30 | }) 31 | 32 | const hotMiddleware = require('webpack-hot-middleware')(compiler, { 33 | log: false, 34 | heartbeat: 2000 35 | }) 36 | // force page reload when html-webpack-plugin template changes 37 | // currently disabled until this is resolved: 38 | // https://github.com/jantimon/html-webpack-plugin/issues/680 39 | // compiler.plugin('compilation', function (compilation) { 40 | // compilation.plugin('html-webpack-plugin-after-emit', function (data, cb) { 41 | // hotMiddleware.publish({ action: 'reload' }) 42 | // cb() 43 | // }) 44 | // }) 45 | 46 | // enable hot-reload and state-preserving 47 | // compilation error display 48 | app.use(hotMiddleware) 49 | 50 | // proxy api requests 51 | Object.keys(proxyTable).forEach(function (context) { 52 | let options = proxyTable[context] 53 | if (typeof options === 'string') { 54 | options = { target: options } 55 | } 56 | app.use(proxyMiddleware(options.filter || context, options)) 57 | }) 58 | 59 | // handle fallback for HTML5 history API 60 | app.use(require('connect-history-api-fallback')()) 61 | 62 | // serve webpack bundle output 63 | app.use(devMiddleware) 64 | 65 | // serve pure static assets 66 | const staticPath = path.posix.join(config.dev.assetsPublicPath, config.dev.assetsSubDirectory) 67 | app.use(staticPath, express.static('./static')) 68 | 69 | const uri = 'http://localhost:' + port 70 | 71 | var _resolve 72 | var _reject 73 | var readyPromise = new Promise((resolve, reject) => { 74 | _resolve = resolve 75 | _reject = reject 76 | }) 77 | 78 | var server 79 | var portfinder = require('portfinder') 80 | portfinder.basePort = port 81 | 82 | console.log('> Starting dev server...') 83 | devMiddleware.waitUntilValid(() => { 84 | portfinder.getPort((err, port) => { 85 | if (err) { 86 | _reject(err) 87 | } 88 | process.env.PORT = port 89 | var uri = 'http://localhost:' + port 90 | console.log('> Listening at ' + uri + '\n') 91 | // when env is testing, don't need open it 92 | if (autoOpenBrowser && process.env.NODE_ENV !== 'testing') { 93 | opn(uri) 94 | } 95 | server = app.listen(port) 96 | _resolve() 97 | }) 98 | }) 99 | 100 | module.exports = { 101 | ready: readyPromise, 102 | close: () => { 103 | server.close() 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /distanceMeasure/distanceMeasure_test.go: -------------------------------------------------------------------------------- 1 | package distanceMeasure 2 | 3 | import ( 4 | "log" 5 | "math" 6 | "math/rand" 7 | "os" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | const TEST_DATA_NUM = 9000000 13 | 14 | var arrPoint1 [TEST_DATA_NUM]*EarthCoordinate //测试数据组1 15 | var arrPoint2 [TEST_DATA_NUM]*EarthCoordinate //测试数据组2 16 | var precision float64 = 0.1 //随机数生成的精度大小 17 | var benchmarkPoint EarthCoordinate = EarthCoordinate{Longitude: 121.0, Latitude: 31.0} //基准值 18 | var resultStardard, resultQuick, resultQuick2 [TEST_DATA_NUM]float64 //各个算法的测距后得到的结果 19 | 20 | func init() { 21 | //使用随机数生成测试数据 22 | //var tmpPoint EarthCoordinate 23 | now := time.Now().Unix() 24 | rand.Seed(now) 25 | 26 | for i := 0; i < TEST_DATA_NUM; i++ { 27 | //随机生成1000000组经纬度坐标 28 | //以1000000为除数求余，然后把得到的余数再与除以1000000作为benchmarkPoint的小数部分 29 | tmpPoint := new(EarthCoordinate) 30 | tmpPoint.Latitude = benchmarkPoint.Latitude + float64(rand.Intn(1000000))/1000000*precision 31 | tmpPoint.Longitude = benchmarkPoint.Longitude + float64(rand.Intn(1000000))/1000000*precision 32 | arrPoint1[i] = tmpPoint 33 | tmpPoint = new(EarthCoordinate) 34 | tmpPoint.Latitude = benchmarkPoint.Latitude + float64(rand.Intn(1000000))/1000000*precision 35 | tmpPoint.Longitude = benchmarkPoint.Longitude + float64(rand.Intn(1000000))/1000000*precision 36 | arrPoint2[i] = tmpPoint 37 | } 38 | } 39 | 40 | //标准球体算法 41 | func TestStardard(t *testing.T) { 42 | measure := GetInstance() 43 | //计算标准三维距离算法的时间 44 | before := time.Now().UnixNano() 45 | for i := 0; i < TEST_DATA_NUM; i++ { 46 | resultStardard[i] = measure.MeasureByStardardMethod(arrPoint1[i], arrPoint2[i]) 47 | } 48 | after := time.Now().UnixNano() 49 | t.Logf("Haversine公式标准球体算法耗时：%E ns\n", float64(after-before)) 50 | } 51 | 52 | //没有预设本地经纬度坐标的快速测距算法 53 | func TestQuickWithoutSetLocation(t *testing.T) { 54 | measure := GetInstance() 55 | //通过三维近似为二维计算距离算法的时间 56 | before := time.Now().UnixNano() 57 | for i := 0; i < TEST_DATA_NUM; i++ { 58 | resultQuick[i] = measure.MeasureByQuickMethodWithoutLocation(arrPoint1[i], arrPoint2[i]) 59 | } 60 | after := time.Now().UnixNano() 61 | t.Logf("平面近似二维距离算法（勾股定理，未预设城市经纬度）的耗时：%E ns\n", float64(after-before)) 62 | } 63 | 64 | func TestQuick(t *testing.T) { 65 | measure := GetInstance() 66 | //先来个错误的示范 67 | if _, err := measure.MeasureByQuickMethod(arrPoint1[0], arrPoint2[0]); err != nil { 68 | t.Log(err) 69 | } 70 | 71 | //通过三维近似为二维计算距离算法的时间 72 | before := time.Now().UnixNano() 73 | measure.SetLocalEarthCoordinate(&benchmarkPoint, "上海") 74 | for i := 0; i < TEST_DATA_NUM; i++ { 75 | if value, err := measure.MeasureByQuickMethod(arrPoint1[i], arrPoint2[i]); err != nil { 76 | t.Error("出错了") 77 | } else { 78 | resultQuick2[i] = value 79 | } 80 | } 81 | after := time.Now().UnixNano() 82 | t.Logf("平面近似二维距离算法（勾股定理,已预设城市经纬度）的耗时：%E ns\n", float64(after-before)) 83 | } 84 | 85 | //计算标准差 86 | func TestCompare(t *testing.T) { 87 | var variance, variance2 float64 = 0.0, 0.0 88 | logFile, err := os.OpenFile("distance.log", os.O_APPEND|os.O_CREATE, 0666) 89 | if err != nil { 90 | t.Log("打开distance.log失败") 91 | return 92 | } 93 | log.SetOutput(logFile) 94 | for i := 0; i < TEST_DATA_NUM/100; i++ { 95 | log.Printf("第%d组数据: EarthPointPair = %v 和 %v, stardard = %f, quick = %f, quick2 = %f\n", 96 | i, arrPoint1[i], arrPoint2[i], resultStardard[i], resultQuick[i], resultQuick[i]) 97 | } 98 | 99 | for i := 0; i < TEST_DATA_NUM; i++ { 100 | // log.Printf("第%d组数据: EarthPointPair = %v, stardard = %f, quick = %f, quick2 = %f\n", 101 | // i, arrPoint[i], resultStardard[i], resultQuick[i], resultQuick[i]) 102 | variance += math.Pow(resultQuick[i]-resultStardard[i], 2) 103 | variance2 += math.Pow(resultQuick2[i]-resultStardard[i], 2) 104 | } 105 | t.Log(variance, variance2) 106 | t.Logf("精度为%f度时，换算以米为单位时表示搜索范围距离在%f米内时:\n标准差组1为%f米,组2为%f米\n", 107 | precision, DIST_PER_DEGREE*precision, math.Sqrt(variance/TEST_DATA_NUM), math.Sqrt(variance2/TEST_DATA_NUM)) 108 | return 109 | } 110 | -------------------------------------------------------------------------------- /types/indexed_document_gen.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | // NOTE: THIS FILE WAS PRODUCED BY THE 4 | // MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) 5 | // DO NOT EDIT 6 | 7 | import ( 8 | "github.com/tinylib/msgp/msgp" 9 | ) 10 | 11 | // DecodeMsg implements msgp.Decodable 12 | func (z *IndexedDocument) DecodeMsg(dc *msgp.Reader) (err error) { 13 | var field []byte 14 | _ = field 15 | var zxvk uint32 16 | zxvk, err = dc.ReadMapHeader() 17 | if err != nil { 18 | return 19 | } 20 | for zxvk > 0 { 21 | zxvk-- 22 | field, err = dc.ReadMapKeyPtr() 23 | if err != nil { 24 | return 25 | } 26 | switch msgp.UnsafeString(field) { 27 | case "id": 28 | z.DocId, err = dc.ReadUint64() 29 | if err != nil { 30 | return 31 | } 32 | case "lat": 33 | z.Latitude, err = dc.ReadFloat64() 34 | if err != nil { 35 | return 36 | } 37 | case "long": 38 | z.Longitude, err = dc.ReadFloat64() 39 | if err != nil { 40 | return 41 | } 42 | case "f": 43 | z.Fields, err = dc.ReadIntf() 44 | if err != nil { 45 | return 46 | } 47 | default: 48 | err = dc.Skip() 49 | if err != nil { 50 | return 51 | } 52 | } 53 | } 54 | return 55 | } 56 | 57 | // EncodeMsg implements msgp.Encodable 58 | func (z *IndexedDocument) EncodeMsg(en *msgp.Writer) (err error) { 59 | // map header, size 4 60 | // write "id" 61 | err = en.Append(0x84, 0xa2, 0x69, 0x64) 62 | if err != nil { 63 | return err 64 | } 65 | err = en.WriteUint64(z.DocId) 66 | if err != nil { 67 | return 68 | } 69 | // write "lat" 70 | err = en.Append(0xa3, 0x6c, 0x61, 0x74) 71 | if err != nil { 72 | return err 73 | } 74 | err = en.WriteFloat64(z.Latitude) 75 | if err != nil { 76 | return 77 | } 78 | // write "long" 79 | err = en.Append(0xa4, 0x6c, 0x6f, 0x6e, 0x67) 80 | if err != nil { 81 | return err 82 | } 83 | err = en.WriteFloat64(z.Longitude) 84 | if err != nil { 85 | return 86 | } 87 | // write "f" 88 | err = en.Append(0xa1, 0x66) 89 | if err != nil { 90 | return err 91 | } 92 | err = en.WriteIntf(z.Fields) 93 | if err != nil { 94 | return 95 | } 96 | return 97 | } 98 | 99 | // MarshalMsg implements msgp.Marshaler 100 | func (z *IndexedDocument) MarshalMsg(b []byte) (o []byte, err error) { 101 | o = msgp.Require(b, z.Msgsize()) 102 | // map header, size 4 103 | // string "id" 104 | o = append(o, 0x84, 0xa2, 0x69, 0x64) 105 | o = msgp.AppendUint64(o, z.DocId) 106 | // string "lat" 107 | o = append(o, 0xa3, 0x6c, 0x61, 0x74) 108 | o = msgp.AppendFloat64(o, z.Latitude) 109 | // string "long" 110 | o = append(o, 0xa4, 0x6c, 0x6f, 0x6e, 0x67) 111 | o = msgp.AppendFloat64(o, z.Longitude) 112 | // string "f" 113 | o = append(o, 0xa1, 0x66) 114 | o, err = msgp.AppendIntf(o, z.Fields) 115 | if err != nil { 116 | return 117 | } 118 | return 119 | } 120 | 121 | // UnmarshalMsg implements msgp.Unmarshaler 122 | func (z *IndexedDocument) UnmarshalMsg(bts []byte) (o []byte, err error) { 123 | var field []byte 124 | _ = field 125 | var zbzg uint32 126 | zbzg, bts, err = msgp.ReadMapHeaderBytes(bts) 127 | if err != nil { 128 | return 129 | } 130 | for zbzg > 0 { 131 | zbzg-- 132 | field, bts, err = msgp.ReadMapKeyZC(bts) 133 | if err != nil { 134 | return 135 | } 136 | switch msgp.UnsafeString(field) { 137 | case "id": 138 | z.DocId, bts, err = msgp.ReadUint64Bytes(bts) 139 | if err != nil { 140 | return 141 | } 142 | case "lat": 143 | z.Latitude, bts, err = msgp.ReadFloat64Bytes(bts) 144 | if err != nil { 145 | return 146 | } 147 | case "long": 148 | z.Longitude, bts, err = msgp.ReadFloat64Bytes(bts) 149 | if err != nil { 150 | return 151 | } 152 | case "f": 153 | z.Fields, bts, err = msgp.ReadIntfBytes(bts) 154 | if err != nil { 155 | return 156 | } 157 | default: 158 | bts, err = msgp.Skip(bts) 159 | if err != nil { 160 | return 161 | } 162 | } 163 | } 164 | o = bts 165 | return 166 | } 167 | 168 | // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message 169 | func (z *IndexedDocument) Msgsize() (s int) { 170 | s = 1 + 3 + msgp.Uint64Size + 4 + msgp.Float64Size + 5 + msgp.Float64Size + 2 + msgp.GuessSize(z.Fields) 171 | return 172 | } 173 | -------------------------------------------------------------------------------- /example/view/build/webpack.prod.conf.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | const path = require('path') 3 | const utils = require('./utils') 4 | const webpack = require('webpack') 5 | const config = require('../config') 6 | const merge = require('webpack-merge') 7 | const baseWebpackConfig = require('./webpack.base.conf') 8 | const CopyWebpackPlugin = require('copy-webpack-plugin') 9 | const HtmlWebpackPlugin = require('html-webpack-plugin') 10 | const ExtractTextPlugin = require('extract-text-webpack-plugin') 11 | const OptimizeCSSPlugin = require('optimize-css-assets-webpack-plugin') 12 | 13 | const env = config.build.env 14 | 15 | const webpackConfig = merge(baseWebpackConfig, { 16 | module: { 17 | rules: utils.styleLoaders({ 18 | sourceMap: config.build.productionSourceMap, 19 | extract: true 20 | }) 21 | }, 22 | devtool: config.build.productionSourceMap ? '#source-map' : false, 23 | output: { 24 | path: config.build.assetsRoot, 25 | filename: utils.assetsPath('js/[name].[chunkhash].js'), 26 | chunkFilename: utils.assetsPath('js/[id].[chunkhash].js') 27 | }, 28 | plugins: [ 29 | // http://vuejs.github.io/vue-loader/en/workflow/production.html 30 | new webpack.DefinePlugin({ 31 | 'process.env': env 32 | }), 33 | // UglifyJs do not support ES6+, you can also use babel-minify for better treeshaking: https://github.com/babel/minify 34 | new webpack.optimize.UglifyJsPlugin({ 35 | compress: { 36 | warnings: false 37 | }, 38 | sourceMap: true 39 | }), 40 | // extract css into its own file 41 | new ExtractTextPlugin({ 42 | filename: utils.assetsPath('css/[name].[contenthash].css') 43 | }), 44 | // Compress extracted CSS. We are using this plugin so that possible 45 | // duplicated CSS from different components can be deduped. 46 | new OptimizeCSSPlugin({ 47 | cssProcessorOptions: { 48 | safe: true 49 | } 50 | }), 51 | // generate dist index.html with correct asset hash for caching. 52 | // you can customize output by editing /index.html 53 | // see https://github.com/ampedandwired/html-webpack-plugin 54 | new HtmlWebpackPlugin({ 55 | filename: config.build.index, 56 | template: 'index.html', 57 | inject: true, 58 | minify: { 59 | removeComments: true, 60 | collapseWhitespace: true, 61 | removeAttributeQuotes: true 62 | // more options: 63 | // https://github.com/kangax/html-minifier#options-quick-reference 64 | }, 65 | // necessary to consistently work with multiple chunks via CommonsChunkPlugin 66 | chunksSortMode: 'dependency' 67 | }), 68 | // keep module.id stable when vender modules does not change 69 | new webpack.HashedModuleIdsPlugin(), 70 | // split vendor js into its own file 71 | new webpack.optimize.CommonsChunkPlugin({ 72 | name: 'vendor', 73 | minChunks: function (module) { 74 | // any required modules inside node_modules are extracted to vendor 75 | return ( 76 | module.resource && 77 | /\.js$/.test(module.resource) && 78 | module.resource.indexOf( 79 | path.join(__dirname, '../node_modules') 80 | ) === 0 81 | ) 82 | } 83 | }), 84 | // extract webpack runtime and module manifest to its own file in order to 85 | // prevent vendor hash from being updated whenever app bundle is updated 86 | new webpack.optimize.CommonsChunkPlugin({ 87 | name: 'manifest', 88 | chunks: ['vendor'] 89 | }), 90 | // copy custom static assets 91 | new CopyWebpackPlugin([ 92 | { 93 | from: path.resolve(__dirname, '../static'), 94 | to: config.build.assetsSubDirectory, 95 | ignore: ['.*'] 96 | } 97 | ]) 98 | ] 99 | }) 100 | 101 | if (config.build.productionGzip) { 102 | const CompressionWebpackPlugin = require('compression-webpack-plugin') 103 | 104 | webpackConfig.plugins.push( 105 | new CompressionWebpackPlugin({ 106 | asset: '[path].gz[query]', 107 | algorithm: 'gzip', 108 | test: new RegExp( 109 | '\\.(' + 110 | config.build.productionGzipExtensions.join('|') + 111 | ')$' 112 | ), 113 | threshold: 10240, 114 | minRatio: 0.8 115 | }) 116 | ) 117 | } 118 | 119 | if (config.build.bundleAnalyzerReport) { 120 | const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin 121 | webpackConfig.plugins.push(new BundleAnalyzerPlugin()) 122 | } 123 | 124 | module.exports = webpackConfig 125 | -------------------------------------------------------------------------------- /distanceMeasure/distanceMeasure.go: -------------------------------------------------------------------------------- 1 | package distanceMeasure 2 | 3 | /* 4 | 使用本package的DistanceMeasure结构体建议使用GetInstance和CreateNewMeasure来获取对象， 5 | 而不建议自己直接new。因为如果自己new可能会在切换城市的时候忘记设置本地的location 6 | GetInstance和CreateNewMeasure的区别是： 7 | 1.如果一个进程同时只为一个城市或一个区（比如上海这么人口密集的城市可以一个区一个基准坐标，这样误差也会小）服务， 8 | 那么建议使用GetInstance通常只需要在启动时以读入配置文件的配置项的方式来设置就行了。 9 | 2.如果一个进程同时为多个城市或者多个区服务，那么建议使用CreateNewMeasure，这样调用SetLocalEarthCoordinate 10 | 设置本地城市坐标时才不会互相覆盖和影响 11 | MeasureByStardardMethod的数学推导：http://blog.csdn.net/liminlu0314/article/details/8553926 12 | MeasureByQuickMethodWithoutLocation的参考链接：https://tech.meituan.com/lucene-distance.html 13 | */ 14 | 15 | import ( 16 | "fmt" 17 | "math" 18 | ) 19 | 20 | const RADIUS = 6378137.0 //地球半径，单位米 21 | const DIST_PER_DEGREE = math.Pi * 35433.88889 // πr/180° 解释为每度所表示的实际长度 22 | 23 | var localMeasure *DistanceMeasure = nil 24 | 25 | type DistanceMeasure struct { 26 | IsSetLocation bool //是否设置了本地基准经纬度 27 | Benchmark *EarthCoordinate //基准坐标 28 | cosLatitude float64 //math.Cos(Benchmark.latitude) 基准维度的cos值 29 | cityName string //城市名，可以不填，打印使用的 30 | IsFirstUse bool //是否是第一次使用，在GetInstance使用后置为ture 31 | } 32 | 33 | type MeasureError struct { 34 | } 35 | 36 | func (e MeasureError) Error() string { 37 | return "尚未输入本地城市的基准经纬度坐标" 38 | } 39 | 40 | type EarthCoordinate struct { 41 | Longitude float64 //经度坐标 东经为正数，西经为负数 42 | Latitude float64 //纬度坐标 北纬为正数，南纬为负数 43 | } 44 | 45 | //返回单例 46 | func GetInstance() *DistanceMeasure { 47 | if localMeasure == nil { 48 | localMeasure = new(DistanceMeasure) 49 | localMeasure.IsSetLocation = false 50 | localMeasure.Benchmark = &EarthCoordinate{0.0, 0.0} 51 | } 52 | localMeasure.IsFirstUse = true 53 | return localMeasure 54 | } 55 | 56 | //创建一个新的DistanceMeasure对象 57 | func CreateNewMeasure() *DistanceMeasure{ 58 | measure := new(DistanceMeasure) 59 | measure.IsSetLocation = false 60 | measure.Benchmark = &EarthCoordinate{0.0, 0.0} 61 | measure.IsFirstUse = true 62 | return measure 63 | } 64 | 65 | //设置本地的基准坐标，可以取一个城市的市中心，EarthCoordinate必须要输入所在城市的经纬度坐标，name值（城市名）可以为空 66 | func (this *DistanceMeasure) SetLocalEarthCoordinate(location *EarthCoordinate, name string) { 67 | this.IsSetLocation = true //已设置坐标 68 | this.Benchmark = location //用于QuickMethod的本地经纬度坐标 69 | this.cityName = name //城市名，打印使用 70 | this.cosLatitude = math.Cos(this.ChangeAngleToRadian(location.Latitude)) //基准维度的cos值 71 | fmt.Println("设置了城市坐标，城市名：", name, "经纬度坐标：", location) 72 | } 73 | 74 | func (this *DistanceMeasure) ChangeAngleToRadian(angle float64) float64 { 75 | return angle / 180.0 * math.Pi 76 | } 77 | 78 | //标准球体测距算法，基准算法，基于球面模型来处理的（立体几何），即Haversine公式 79 | func (this *DistanceMeasure) MeasureByStardardMethod(pt1, pt2 *EarthCoordinate) float64 { 80 | //先把角度转成弧度 81 | lon1 := this.ChangeAngleToRadian(pt1.Longitude) 82 | lat1 := this.ChangeAngleToRadian(pt1.Latitude) 83 | lon2 := this.ChangeAngleToRadian(pt2.Longitude) 84 | lat2 := this.ChangeAngleToRadian(pt2.Latitude) 85 | //因为cos余弦函数是偶函数，所以lon2-lon1还是lon1-lon2都一样 86 | //得到距离的弧度值 87 | radDist := math.Acos(math.Sin(lat1)*math.Sin(lat2) + math.Cos(lat1)*math.Cos(lat2)*math.Cos(lon2-lon1)) 88 | return radDist * RADIUS //乘以地球半径，返回实际长度 89 | } 90 | 91 | //未提前设置本地城市经纬度坐标的快速测距算法 92 | func (this *DistanceMeasure) MeasureByQuickMethodWithoutLocation(pt1, pt2 *EarthCoordinate) float64 { 93 | diffLat := (pt1.Latitude - pt2.Latitude) //纬度差的实际距离，单位m 94 | radLat := this.ChangeAngleToRadian((pt1.Latitude + pt2.Latitude) / 2) 95 | diffLog := math.Cos(radLat) * (pt1.Longitude - pt2.Longitude) //经度差的实际距离，单位m,需要乘以cos(所在纬度),因为纬度越高时经度差的表示的实际距离越短 96 | //根据勾股定理 97 | dist := DIST_PER_DEGREE * math.Sqrt(diffLat*diffLat+diffLog*diffLog) 98 | return dist 99 | } 100 | 101 | //已提前设置本地城市经纬度坐标的快速测距算法 102 | func (this *DistanceMeasure) MeasureByQuickMethod(pt1, pt2 *EarthCoordinate) (float64, error) { 103 | //先判断是否已经输入了本地基准经纬度坐标 104 | 105 | if this.IsSetLocation == false { 106 | return 0.0, MeasureError{} 107 | } 108 | if this.IsFirstUse { 109 | this.IsFirstUse = false 110 | //第一次调用这个函数的时候打印城市名进行提示 111 | var strLon, strLat string 112 | if this.Benchmark.Longitude >= 0 { 113 | strLon = fmt.Sprintf("东经%f°", this.Benchmark.Longitude) 114 | } else { 115 | strLon = fmt.Sprintf("西经%f°", -this.Benchmark.Longitude) 116 | } 117 | if this.Benchmark.Latitude >= 0 { 118 | strLat = fmt.Sprintf("北纬%f°", this.Benchmark.Latitude) 119 | } else { 120 | strLat = fmt.Sprintf("南纬%f°", -this.Benchmark.Latitude) 121 | } 122 | fmt.Printf("本次快速测距算法设置的城市为%s,输入的基准经纬度坐标为[%s, %s]\n", this.cityName, strLon, strLat) 123 | fmt.Println("如果当前保存的城市与你期望的城市不符，请重新调用SetLocalEarthCoordinate来设置") 124 | } 125 | 126 | diffLat := (pt1.Latitude - pt2.Latitude) //纬度差的实际距离，单位m 127 | diffLog := this.cosLatitude * (pt1.Longitude - pt2.Longitude) //经度差的实际距离，单位m,需要乘以cos(所在纬度),因为纬度越高时经度差的表示的实际距离越短 128 | distance := DIST_PER_DEGREE * math.Sqrt(diffLat*diffLat+diffLog*diffLog) 129 | return distance, nil 130 | } 131 | -------------------------------------------------------------------------------- /core/indexer.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "github.com/go-redis/redis" 5 | "github.com/mmcloughlin/geohash" 6 | "github.com/sillydong/lbsengine/distanceMeasure" 7 | "github.com/sillydong/lbsengine/types" 8 | "log" 9 | "strconv" 10 | "unsafe" 11 | ) 12 | 13 | type Indexer struct { 14 | option *types.IndexerOptions 15 | client *redis.Client 16 | measure *distanceMeasure.DistanceMeasure 17 | } 18 | 19 | func (i *Indexer) Init(option *types.IndexerOptions) { 20 | i.option = option 21 | i.client = redis.NewClient(&redis.Options{ 22 | Addr: option.RedisHost, 23 | DB: option.RedisDb, 24 | Password: option.RedisPassword, 25 | }) 26 | err := i.client.Ping().Err() 27 | if err != nil { 28 | log.Fatal(err) 29 | } 30 | //距离计算 31 | i.measure = distanceMeasure.GetInstance() 32 | if option.CenterLongitude != 0.0 && option.CenterLatitude != 0.0 { 33 | i.measure.SetLocalEarthCoordinate(&distanceMeasure.EarthCoordinate{Latitude: option.CenterLatitude, Longitude: option.CenterLongitude}, option.Location) 34 | } 35 | } 36 | 37 | func (i *Indexer) Add(doc *types.IndexedDocument) { 38 | sdocid := strconv.FormatUint(doc.DocId, 10) 39 | hashs := i.hashshard(doc.DocId) 40 | pip := i.client.Pipeline() 41 | //删除旧数据 42 | oldhash, _ := i.client.HGet(hashs, sdocid).Result() 43 | if len(oldhash) > 0 { 44 | pip.HDel(i.geoshard(oldhash, doc.DocId), sdocid) 45 | } 46 | newhash := geohash.EncodeWithPrecision(doc.Latitude, doc.Longitude, i.option.GeoPrecious) 47 | pip.HSet(hashs, sdocid, newhash) 48 | pip.HSet(i.geoshard(newhash, doc.DocId), sdocid, doc) 49 | _, err := pip.Exec() 50 | if err != nil { 51 | log.Print(err) 52 | } 53 | } 54 | 55 | func (i *Indexer) Remove(docid uint64) { 56 | sdocid := strconv.FormatUint(docid, 10) 57 | hashs := i.hashshard(docid) 58 | hash, _ := i.client.HGet(hashs, sdocid).Result() 59 | if len(hash) > 0 { 60 | geos := i.geoshard(hash, docid) 61 | pip := i.client.Pipeline() 62 | pip.HDel(hashs, sdocid) 63 | pip.HDel(geos, sdocid) 64 | _, err := pip.Exec() 65 | if err != nil { 66 | log.Print(err) 67 | } 68 | } 69 | } 70 | 71 | func (i *Indexer) Search(countonly bool, hash string, latitude, longitude float64, options *types.SearchOptions) (docs types.ScoredDocuments, count int) { 72 | strs := i.client.HVals(hash).Val() 73 | if len(strs) > 0 { 74 | if countonly { 75 | //仅计算数量 76 | if options.Excepts == nil && options.Filter == nil { 77 | count += len(strs) 78 | } else { 79 | for _, str := range strs { 80 | document := types.IndexedDocument{} 81 | document.UnmarshalMsg(tobytes(str)) 82 | if document.DocId != 0 { 83 | //判断是否排除 84 | if options.Excepts == nil { 85 | //判断是否过滤 86 | if options.Filter == nil || options.Filter(document) { 87 | count++ 88 | } 89 | } else { 90 | if _, ok := options.Excepts[document.DocId]; !ok { 91 | if options.Filter == nil || options.Filter(document) { 92 | count++ 93 | } 94 | } 95 | } 96 | } 97 | } 98 | } 99 | } else { 100 | //需要数据 101 | for _, str := range strs { 102 | document := types.IndexedDocument{} 103 | document.UnmarshalMsg(tobytes(str)) 104 | if document.DocId != 0 { 105 | //判断是否排除 106 | if options.Excepts == nil { 107 | //判断是否过滤 108 | if options.Filter == nil || options.Filter(document) { 109 | doc := types.ScoredDocument{ 110 | DocId: document.DocId, 111 | Distance: i.distance(options.Accuracy, document.Latitude, document.Longitude, latitude, longitude), 112 | } 113 | docs = append(docs, &doc) 114 | count++ 115 | } 116 | } else { 117 | if _, ok := options.Excepts[document.DocId]; !ok { 118 | if options.Filter == nil || options.Filter(document) { 119 | doc := types.ScoredDocument{ 120 | DocId: document.DocId, 121 | Distance: i.distance(options.Accuracy, document.Latitude, document.Longitude, latitude, longitude), 122 | } 123 | docs = append(docs, &doc) 124 | count++ 125 | } 126 | } 127 | } 128 | } 129 | } 130 | } 131 | } 132 | return 133 | } 134 | 135 | //距离计算 136 | func (i *Indexer) distance(accuracy int, alatitude, alongitude, blatitude, blongitude float64) float64 { 137 | //fmt.Printf("%+v - %+v ------ %+v - %+v",alatitude,alongitude,blatitude,blongitude) 138 | a := &distanceMeasure.EarthCoordinate{Latitude: alatitude, Longitude: alongitude} 139 | b := &distanceMeasure.EarthCoordinate{Latitude: blatitude, Longitude: blongitude} 140 | switch accuracy { 141 | case types.STANDARD: 142 | return i.measure.MeasureByStardardMethod(a, b) 143 | case types.MEITUAN: 144 | return i.measure.MeasureByQuickMethodWithoutLocation(a, b) 145 | case types.IMPROVED: 146 | distance, err := i.measure.MeasureByQuickMethod(a, b) 147 | if err != nil { 148 | return i.measure.MeasureByQuickMethodWithoutLocation(a, b) 149 | } 150 | return distance 151 | } 152 | return 0.0 153 | } 154 | 155 | func (i *Indexer) hashshard(docid uint64) string { 156 | return "g_" + strconv.FormatUint((uint64(docid/i.option.HashSize)+1)*i.option.HashSize, 10) 157 | } 158 | 159 | func (i *Indexer) geoshard(hash string, docid uint64) string { 160 | return "h_" + hash + "_" + strconv.FormatUint(docid-docid/i.option.GeoShard*i.option.GeoShard, 10) 161 | } 162 | 163 | func tobytes(s string) []byte { 164 | x := (*[2]uintptr)(unsafe.Pointer(&s)) 165 | h := [3]uintptr{x[0], x[1], x[1]} 166 | return *(*[]byte)(unsafe.Pointer(&h)) 167 | } 168 | -------------------------------------------------------------------------------- /engine/engine.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "fmt" 5 | "github.com/huichen/murmur" 6 | "github.com/sillydong/lbsengine/core" 7 | "github.com/sillydong/lbsengine/types" 8 | "sort" 9 | "strconv" 10 | "time" 11 | "unsafe" 12 | ) 13 | 14 | type Engine struct { 15 | option *types.EngineOptions 16 | 17 | cacher *core.Cacher 18 | indexer *core.Indexer 19 | 20 | indexerAddChannels []chan *types.IndexedDocument 21 | indexerRemoveChannels []chan uint64 22 | indexerSearchChannels []chan *indexerSearchRequest 23 | } 24 | 25 | func (e *Engine) Init(option *types.EngineOptions) { 26 | if option == nil { 27 | option = &types.EngineOptions{} 28 | } 29 | option.Init() 30 | e.option = option 31 | 32 | //初始化缓存器 33 | e.cacher = &core.Cacher{} 34 | e.cacher.Init() 35 | //初始化索引器 36 | e.indexer = &core.Indexer{} 37 | e.indexer.Init(e.option.IndexerOption) 38 | 39 | e.indexerAddChannels = make([]chan *types.IndexedDocument, e.option.NumShards) 40 | e.indexerRemoveChannels = make([]chan uint64, e.option.NumShards) 41 | e.indexerSearchChannels = make([]chan *indexerSearchRequest, e.option.NumShards) 42 | 43 | for i := 0; i < int(e.option.NumShards); i++ { 44 | //初始化channel 45 | e.indexerAddChannels[i] = make(chan *types.IndexedDocument, e.option.AddBuffer) 46 | e.indexerRemoveChannels[i] = make(chan uint64, e.option.RemoveBuffer) 47 | e.indexerSearchChannels[i] = make(chan *indexerSearchRequest, e.option.SearchBuffer) 48 | 49 | go e.indexerAddWorker(i) 50 | go e.indexerRemoveWorker(i) 51 | for k := 0; k < e.option.SearchWorkerThreads; k++ { 52 | go e.indexerSearchWorker(i) 53 | } 54 | } 55 | } 56 | 57 | func (e *Engine) Add(doc *types.IndexedDocument) { 58 | shard := e.shardid(doc.DocId) 59 | e.indexerAddChannels[shard] <- doc 60 | } 61 | 62 | func (e *Engine) Remove(docid uint64) { 63 | shard := e.shardid(docid) 64 | e.indexerRemoveChannels[shard] <- docid 65 | } 66 | 67 | func (e *Engine) Search(request *types.SearchRequest) *types.SearchResponse { 68 | content := fmt.Sprintf("%v", request) 69 | hash := murmur.Murmur3(tobytes(content)) 70 | 71 | shard := int(hash - hash/uint32(e.option.NumShards)*uint32(e.option.NumShards)) 72 | 73 | cachekey := fmt.Sprintf("%v", hash) 74 | 75 | if request.SearchOption == nil { 76 | request.SearchOption = e.option.DefaultSearchOption 77 | } 78 | request.SearchOption.Init() 79 | 80 | //是否刷新缓存 81 | if !request.SearchOption.Refresh { 82 | //从缓存取数据 83 | docs, count := e.cacher.Get(cachekey, request.Offset, request.Limit) 84 | if docs != nil && count > 0 { 85 | return &types.SearchResponse{Docs: docs, Count: count, Timeout: false} 86 | } 87 | } 88 | 89 | //获取neighbour 90 | neighbours := core.LoopNeighbours(request.Latitude, request.Longitude, e.option.IndexerOption.GeoPrecious, request.SearchOption.Circles) 91 | loops := len(neighbours) * int(e.option.IndexerOption.GeoShard) //geohash分片 92 | indexerReturnChannel := make(chan *indexerSearchResponse, loops) 93 | 94 | //下发任务 95 | for _, geo := range neighbours { 96 | for i := 0; i < int(e.option.IndexerOption.GeoShard); i++ { 97 | geoshard := "h_" + geo + "_" + strconv.Itoa(i) 98 | e.indexerSearchChannels[shard] <- &indexerSearchRequest{ 99 | countonly: request.CountOnly, 100 | hash: geoshard, 101 | latitude: request.Latitude, 102 | longitude: request.Longitude, 103 | option: request.SearchOption, 104 | indexerReturnChannel: indexerReturnChannel, 105 | } 106 | } 107 | } 108 | 109 | //整理数据 110 | docs := types.ScoredDocuments{} 111 | count := 0 112 | istimeout := false 113 | if request.SearchOption.Timeout > 0 { 114 | deadline := time.Now().Add(request.SearchOption.Timeout) 115 | for i := 0; i < loops; i++ { 116 | select { 117 | case lresponse := <-indexerReturnChannel: 118 | if lresponse.count == 0 { 119 | continue 120 | } 121 | if !request.CountOnly { 122 | docs = append(docs, lresponse.docs...) 123 | } 124 | count += lresponse.count 125 | case <-time.After(deadline.Sub(time.Now())): 126 | istimeout = true 127 | break 128 | } 129 | 130 | } 131 | } else { 132 | for i := 0; i < loops; i++ { 133 | lresponse := <-indexerReturnChannel 134 | if lresponse.count == 0 { 135 | continue 136 | } 137 | if !request.CountOnly { 138 | docs = append(docs, lresponse.docs...) 139 | } 140 | count += lresponse.count 141 | } 142 | } 143 | 144 | //最终排序 145 | if !request.CountOnly { 146 | if request.SearchOption.OrderDesc { 147 | sort.Sort(sort.Reverse(docs)) 148 | } else { 149 | sort.Sort(docs) 150 | } 151 | } 152 | 153 | //写缓存 154 | if len(docs) > 0 { 155 | e.cacher.Set(cachekey, docs) 156 | } 157 | 158 | //拼返回数据 159 | response := &types.SearchResponse{} 160 | response.Count = count 161 | if !request.CountOnly { 162 | start := request.Offset 163 | stop := request.Offset + request.Limit 164 | if start > count { 165 | response.Docs = nil 166 | } else if stop > count { 167 | response.Docs = docs[start:] 168 | } else { 169 | response.Docs = docs[start:stop] 170 | } 171 | } else { 172 | response.Docs = nil 173 | } 174 | response.Timeout = istimeout 175 | 176 | return response 177 | 178 | } 179 | 180 | func (e *Engine) shardid(docid uint64) int { 181 | content := fmt.Sprintf("%d", docid) 182 | hash := murmur.Murmur3(tobytes(content)) 183 | return int(hash - hash/uint32(e.option.NumShards)*uint32(e.option.NumShards)) 184 | } 185 | 186 | func tobytes(s string) []byte { 187 | x := (*[2]uintptr)(unsafe.Pointer(&s)) 188 | h := [3]uintptr{x[0], x[1], x[1]} 189 | return *(*[]byte)(unsafe.Pointer(&h)) 190 | } 191 | -------------------------------------------------------------------------------- /example/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/Code-Hex/echo-static" 6 | "github.com/elazarl/go-bindata-assetfs" 7 | "github.com/go-gorp/gorp" 8 | "github.com/go-sql-driver/mysql" 9 | "github.com/labstack/echo" 10 | "github.com/labstack/echo/middleware" 11 | "github.com/sillydong/lbsengine/engine" 12 | "github.com/sillydong/lbsengine/types" 13 | "log" 14 | "net/http" 15 | "strconv" 16 | "strings" 17 | "time" 18 | ) 19 | 20 | func main() { 21 | //tidb 22 | dbconfig := &mysql.Config{ 23 | User: "root", 24 | Passwd: "", 25 | DBName: "lbsexample", 26 | Net: "tcp", 27 | Addr: "127.0.0.1:4000", 28 | } 29 | db, err := sql.Open("mysql", dbconfig.FormatDSN()) 30 | if err != nil { 31 | log.Fatal(err) 32 | } 33 | if err := db.Ping(); err != nil { 34 | log.Fatal(err) 35 | } 36 | db.SetConnMaxLifetime(time.Minute) 37 | db.SetMaxIdleConns(10) 38 | db.SetMaxOpenConns(10) 39 | 40 | client := &gorp.DbMap{Db: db, Dialect: gorp.MySQLDialect{}} 41 | 42 | client.AddTableWithName(PoiData{}, "poi_data").SetKeys(true, "id") 43 | 44 | //engine 45 | eg := &engine.Engine{} 46 | eg.Init(nil) 47 | 48 | //echo 49 | addr := ":8877" 50 | e := echo.New() 51 | e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{ 52 | Skipper: ResourceSkipper, 53 | })) 54 | e.Use(middleware.Recover()) 55 | 56 | e.Use(static.ServeRoot("/", &assetfs.AssetFS{ 57 | Asset: Asset, 58 | AssetDir: AssetDir, 59 | AssetInfo: AssetInfo, 60 | Prefix: "view/dist", 61 | })) 62 | api := e.Group("/api") 63 | api.POST("/add", func(context echo.Context) error { 64 | //解析对象 65 | item := new(PoiData) 66 | if err := context.Bind(item); err != nil { 67 | return context.JSON(http.StatusInternalServerError, Response{ 68 | Status: 0, 69 | Error: "请求的参数不正确", 70 | }) 71 | } 72 | 73 | //写tidb 74 | err := client.Insert(item) 75 | if err != nil { 76 | context.Logger().Error(err) 77 | return context.JSON(http.StatusInternalServerError, Response{ 78 | Status: 0, 79 | Error: err.Error(), 80 | }) 81 | } 82 | if item.Id > 0 { 83 | //写engine 84 | eg.Add(&types.IndexedDocument{ 85 | DocId: uint64(item.Id), 86 | Latitude: item.Latitude, 87 | Longitude: item.Longitude, 88 | Fields: nil, 89 | }) 90 | return context.JSON(http.StatusOK, Response{ 91 | Status: 1, 92 | Data: "添加成功", 93 | }) 94 | } else { 95 | return context.JSON(http.StatusInternalServerError, Response{ 96 | Status: 0, 97 | Error: "数据库写入失败", 98 | }) 99 | } 100 | 101 | }) 102 | api.DELETE("/del/:id", func(context echo.Context) error { 103 | id := context.Param("id") 104 | fid, _ := strconv.Atoi(id) 105 | //删tidb 106 | _, err := client.Delete(&PoiData{Id: int64(fid)}) 107 | if err != nil { 108 | context.Logger().Error(err) 109 | } 110 | 111 | //删engine 112 | eg.Remove(uint64(fid)) 113 | 114 | return context.JSON(http.StatusOK, Response{ 115 | Status: 1, 116 | Data: "删除成功", 117 | }) 118 | }) 119 | api.GET("/query", func(context echo.Context) error { 120 | latitude := context.QueryParam("latitude") 121 | longitude := context.QueryParam("longitude") 122 | offset := context.QueryParam("offset") 123 | 124 | flat, _ := strconv.ParseFloat(latitude, 10) 125 | flng, _ := strconv.ParseFloat(longitude, 10) 126 | foff, _ := strconv.Atoi(offset) 127 | 128 | //查engine 129 | resp := eg.Search(&types.SearchRequest{ 130 | Latitude: flat, 131 | Longitude: flng, 132 | CountOnly: false, 133 | Offset: foff, 134 | Limit: 10, 135 | }) 136 | if resp.Count > 0 && resp.Docs != nil { 137 | //拼数据 138 | ids := make([]interface{}, len(resp.Docs)) 139 | for i, doc := range resp.Docs { 140 | ids[i] = doc.DocId 141 | //context.Logger().Printf("%+v\n",doc) 142 | } 143 | var datas []PoiData 144 | _, err := client.Select(&datas, "select * from poi_data where id in (?"+strings.Repeat(",?", len(resp.Docs)-1)+")", ids...) 145 | if err != nil { 146 | context.Logger().Error(err) 147 | } 148 | 149 | datasmap := make(map[uint64]PoiData, len(datas)) 150 | for _, data := range datas { 151 | datasmap[uint64(data.Id)] = data 152 | } 153 | for _, doc := range resp.Docs { 154 | doc.Model = datasmap[doc.DocId] 155 | } 156 | 157 | return context.JSON(http.StatusOK, Response{ 158 | Status: 1, 159 | Data: resp, 160 | }) 161 | } else { 162 | return context.JSON(http.StatusOK, Response{ 163 | Status: 0, 164 | Error: "无结果", 165 | }) 166 | } 167 | }) 168 | api.GET("/init", func(context echo.Context) error { 169 | var datas []PoiData 170 | _, err := client.Select(&datas, "select * from poi_data") 171 | if err != nil { 172 | context.Logger().Error(err) 173 | } 174 | 175 | context.Logger().Printf("about to import %v lines", len(datas)) 176 | 177 | for _, item := range datas { 178 | context.Logger().Print(item.Id) 179 | eg.Add(&types.IndexedDocument{ 180 | DocId: uint64(item.Id), 181 | Latitude: item.Latitude, 182 | Longitude: item.Longitude, 183 | Fields: nil, 184 | }) 185 | } 186 | 187 | return context.JSON(http.StatusOK, "导入完成") 188 | }) 189 | e.Logger.Fatal(e.Start(addr)) 190 | } 191 | 192 | func ResourceSkipper(c echo.Context) bool { 193 | if strings.HasPrefix(c.Path(), "/static") { 194 | return true 195 | } 196 | return false 197 | } 198 | 199 | type PoiData struct { 200 | Id int64 `db:"id" form:"-"` 201 | Name string `db:"name" form:"name"` 202 | AmapId string `db:"amapid" form:"amapid"` 203 | Location string `db:"location" form:"location"` 204 | Latitude float64 `db:"latitude" form:"latitude"` 205 | Longitude float64 `db:"longitude" form:"longitude"` 206 | CreateTime string `db:"create_time" form:"create_time"` 207 | } 208 | 209 | type Response struct { 210 | Status int `json:"status"` 211 | Error string `json:"error"` 212 | Data interface{} `json:"data"` 213 | } 214 | --------------------------------------------------------------------------------