├── .gitignore ├── .travis.yml ├── Makefile ├── README.md ├── client ├── README.md └── client.go ├── collector └── collector.go ├── config-sample.json ├── config-sample.yml ├── config ├── config.go └── config_test.go ├── go.mod ├── go.sum ├── main.go ├── orderbook.png ├── output ├── .gitkeep ├── csv │ └── .gitkeep └── tar │ └── .gitkeep ├── parser ├── data_csv_depth.go └── data_csv_depth_test.go ├── storage ├── csv │ ├── csv.go │ ├── csv_test.go │ ├── tar.go │ └── tar_test.go ├── influxdb │ ├── influxdb.go │ └── influxdb_test.go └── storage.go ├── ticker.png ├── tool ├── csv2crex │ ├── README.md │ └── main.go └── influx2csv │ ├── README.md │ └── main.go └── ws ├── builder.go ├── builder_test.go └── ws.go /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | /market_data_collector* 3 | /*.csv 4 | /csv/*.csv 5 | /tar/*.tar.gz 6 | /config.json 7 | /config.yml 8 | /output/csv/ 9 | /output/tar/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: # 指明使用的语言 2 | - go 3 | 4 | go: # 语言版本号 5 | - "1.12" # 默认使用最新版本,注意，需要 "1.10" 版本的时候必须表示为字符串形式，如果写成 1.10 则会使用 1.1 版本;x表示对应前缀的最新版本 6 | 7 | os: 8 | - linux 9 | - osx 10 | 11 | script: # 执行的脚步，但是go默认会执行下面的这些命令的，所以可以不用写 12 | #- go get -v 13 | - ls 14 | - export GO111MODULE=on 15 | - echo $GOPATH 16 | - echo $PWD 17 | - cd $GOPATH/src/github.com/goex-top/market_data_collector 18 | - echo $PWD 19 | - make 20 | # - go test -v -race -coverprofile=coverage.txt -covermode=atomic ./... 21 | # 22 | #after_success: 23 | # - bash <(curl -s https://codecov.io/bash) -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VERSION := $(shell git describe --tags) 2 | BUILD := $(shell git rev-parse --short HEAD) 3 | PROJECTNAME := $(shell basename "$(PWD)") 4 | 5 | # Basic go commands 6 | GOCMD=go 7 | GOBUILD=$(GOCMD) build 8 | GOCLEAN=$(GOCMD) clean 9 | GOTEST=$(GOCMD) test 10 | GOGET=$(GOCMD) get 11 | GOMOD=$(GOCMD) mod 12 | 13 | # Binary names 14 | BINARY_NAME=market_data_collector 15 | BINARY_UNIX=$(BINARY_NAME)_unix 16 | 17 | all: build 18 | build: 19 | # $(GOMOD) tidy 20 | $(GOBUILD) -o $(BINARY_NAME) -v -ldflags '-s -w' 21 | test: 22 | $(GOTEST) -v ./... 23 | clean: 24 | $(GOCLEAN) 25 | rm -f $(BINARY_NAME) 26 | rm -f $(BINARY_UNIX) 27 | run: 28 | $(GOBUILD) -o $(BINARY_NAME) -v ./... 29 | ./$(BINARY_NAME) 30 | # Cross compilation 31 | build-unix: 32 | CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 $(GOBUILD) -o $(BINARY_UNIX) -v -ldflags '-s -w' 33 | build-linux: 34 | CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GOBUILD) -o ${BINARY_NAME} -v -ldflags '-s -w' 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Market Data Collector 2 | ![HitCount](http://hits.dwyl.io/goex-top/market_data_collector.svg) 3 | [![Build Status](https://travis-ci.org/goex-top/market_data_collector.png)](https://travis-ci.org/goex-top/market_data_collector) 4 | 5 | 6 | Collect market data for quant analysis 7 | ## Quick Start 8 | ### Installation 9 | 10 | `go install github.com/goex-top/market_data_collector` 11 | 12 | ### Configure 13 | create a configure file `config.json` , or `config.yml`, ( `cp config-sample.json config.json` or `cp config-sample.yml config.yml` ) 14 | ```json 15 | { 16 | "subs": [ 17 | { 18 | "exchange_name": "binance.com", 19 | "currency_pair": "BTC_USDT", 20 | "period": 100, 21 | "flag": 3 22 | }, 23 | { 24 | "exchange_name": "binance.com_swap", 25 | "currency_pair": "BTC_USDT", 26 | "contract_type": "swap", 27 | "period": 100, 28 | "flag": 3 29 | } 30 | ], 31 | "store": { 32 | "csv": false, 33 | "influx_db": true, 34 | "influx_db_cfg": { 35 | "url": "http://localhost:8086", 36 | "database": "market_data" 37 | } 38 | }, 39 | "with_market_center": false, 40 | "market_center_path": "/tmp/goex.market.center" 41 | } 42 | ``` 43 | 44 | ```yaml 45 | --- 46 | subs: 47 | - exchange_name: binance.com 48 | currency_pair: BTC_USDT 49 | period: 100 50 | flag: 2 51 | - exchange_name: okex.com_swap 52 | currency_pair: BTC_USDT 53 | contract_type: quarter 54 | period: 100 55 | flag: 2 56 | with_market_center: true 57 | store: 58 | csv: false 59 | influx_db: true 60 | influx_db_cfg: 61 | url: http://localhost:8086 62 | database: market_data 63 | username: 64 | password: 65 | market_center_path: "/tmp/goex.market.center" 66 | ``` 67 | 68 | **Description** 69 | 70 | ``` 71 | { 72 | "subs": [ // subscribs, it's a array for multi-exchanges 73 | { 74 | "exchange_name": "binance.com_swap", // exchange name, ref to https://github.com/goex-top/market_center#support-exchanges 75 | "currency_pair": "BTC_USDT", // pair with `_` 76 | "period": 100, // period 77 | "flag": 2, // flag, is a mask for market, 1: depth, 2: ticker, 3: depth and ticker 78 | } 79 | ], 80 | "store": { // storage 81 | "csv": true // store data to csv 82 | "influx_db": true, // store data to influxdb 83 | "influx_db_cfg": { 84 | "url": "http://localhost:8086", // influxdb url 85 | "database": "market_data" // influxdb database name 86 | } 87 | }, 88 | "with_market_center: true // market data from exchange or not, if true, it will get market data from market center 89 | "market_center_path": "/tmp/goex.market.center" // market center path 90 | } 91 | 92 | ``` 93 | 94 | ## Flag 95 | only one command flag `-c` to input configure file, for example `market_data_collector -c config.json` 96 | 97 | 98 | ### Run 99 | `market_data_collector -c config.json` 100 | 101 | ## Storage 102 | ### CSV 103 | Store daily data in different `csv` files in `csv` folder, compress it to `tar` folder 104 | 105 | `csv` folder(older file was removed automatically) 106 | ``` 107 | ├── depth_binance.com_BTC_USDT_2020-01-26.csv 108 | ├── depth_fcoin.com_BTC_USDT_2020-01-26.csv 109 | ├── depth_huobi.pro_BTC_USDT_2020-01-26.csv 110 | └── depth_okex.com_BTC_USDT_2020-01-26.csv 111 | ``` 112 | 113 | `tar` folder 114 | ``` 115 | . 116 | ├── binance.com_BTC_USDT_2020-01-24.tar.gz 117 | ├── binance.com_BTC_USDT_2020-01-25.tar.gz 118 | ├── fcoin.com_BTC_USDT_2020-01-24.tar.gz 119 | ├── fcoin.com_BTC_USDT_2020-01-25.tar.gz 120 | ├── huobi.pro_BTC_USDT_2020-01-24.tar.gz 121 | ├── huobi.pro_BTC_USDT_2020-01-25.tar.gz 122 | ├── okex.com_BTC_USDT_2020-01-24.tar.gz 123 | └── okex.com_BTC_USDT_2020-01-25.tar.gz 124 | ``` 125 | 126 | #### Format 127 | ##### ticker 128 | ![ticker](ticker.png) 129 | 130 | | symbol | type | description | 131 | | ---- | ---- | ---- | 132 | | t | int | timestamp | 133 | | b | float | best bid | 134 | | s | float | best ask | 135 | | h | float | high price | 136 | | l | float | low price | 137 | | v | float | volume | 138 | 139 | ##### orderbook 140 | ![orderbook](orderbook.png) 141 | 142 | | symbol | type | description | 143 | | ---- | ---- | ---- | 144 | | t | int | timestamp | 145 | | a | array | asks list with size 20, each element is [p,q], p:price, q:qty | 146 | | b | array | bids list with size 20, each element is [p,q], p:price, q:qty | 147 | 148 | ### InfluxDB 149 | 150 | influxDB schema 151 | 152 | |MEASUREMENT | TAGS | FIELDS| 153 | | ---- | ---- | ---- | 154 | |ticker | exchangeName_spot=pair | xxx| 155 | |kline | exchangeName_future_contractType=pair | xxx| 156 | |depth | exchangeName_swap=pair | xxx | 157 | 158 | > docker: https://github.com/philhawthorne/docker-influxdb-grafana 159 | 160 | ## Support Data 161 | * Ticker 162 | * Depth(Orderbook) 163 | * ~~Kline~~ 164 | 165 | ## TODO 166 | * SQLite 167 | 168 | 169 | -------------------------------------------------------------------------------- /client/README.md: -------------------------------------------------------------------------------- 1 | # Client for market 2 | 3 | Get market data directly 4 | ```go 5 | c := client.NewClient(ExchangeName, CurrencyPair, "", nil) 6 | c.GetTicker() 7 | ``` 8 | 9 | Get market data with market center 10 | ```go 11 | 12 | import ( 13 | mcc "github.com/goex-top/market_center_client" 14 | ) 15 | 16 | mccc := mcc.NewClient() 17 | isSpot := market_center.IsFutureExchange(ExchangeName) 18 | if Flag&market_center.DataFlag_Depth != 0 { 19 | if isSpot { 20 | mccc.SubscribeSpotDepth(ExchangeName, CurrencyPair, Period) 21 | } else { 22 | mccc.SubscribeFutureDepth(ExchangeName, ContractType, CurrencyPair, Period) 23 | } 24 | } 25 | if Flag&market_center.DataFlag_Ticker != 0 { 26 | if isSpot { 27 | mccc.SubscribeSpotTicker(ExchangeName, CurrencyPair, Period) 28 | } else { 29 | mccc.SubscribeFutureTicker(ExchangeName, ContractType, CurrencyPair, Period) 30 | } 31 | } 32 | c := client.NewClient(ExchangeName, CurrencyPair, "", mccc) 33 | c.GetTicker() 34 | ``` -------------------------------------------------------------------------------- /client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "github.com/goex-top/market_center" 5 | mcc "github.com/goex-top/market_center/client" 6 | "github.com/nntaoli-project/goex" 7 | "github.com/nntaoli-project/goex/builder" 8 | "os" 9 | "sort" 10 | ) 11 | 12 | type Client struct { 13 | ExchangeName string 14 | CurrencyPair string 15 | ContractType string 16 | spotApi goex.API 17 | futureApi goex.FutureRestAPI 18 | c *mcc.Client 19 | isDirect bool 20 | isSpot bool 21 | } 22 | 23 | func NewClient(exchangeName, currencyPair, contractType string, c *mcc.Client) *Client { 24 | proxy := os.Getenv("HTTP_PROXY") 25 | var spotApi goex.API 26 | var futureApi goex.FutureRestAPI 27 | var direct = false 28 | var isSpot = !market_center.IsFutureExchange(exchangeName) 29 | if c == nil { 30 | if isSpot { 31 | spotApi = builder.NewAPIBuilder().HttpProxy(proxy).Build(market_center.SupportAdapter[exchangeName]) 32 | } else { 33 | futureApi = builder.NewAPIBuilder().HttpProxy(proxy).BuildFuture(market_center.SupportAdapter[exchangeName]) 34 | } 35 | direct = true 36 | } 37 | return &Client{ 38 | ExchangeName: exchangeName, 39 | CurrencyPair: currencyPair, 40 | spotApi: spotApi, 41 | futureApi: futureApi, 42 | ContractType: contractType, 43 | c: c, 44 | isDirect: direct, 45 | isSpot: isSpot, 46 | } 47 | } 48 | 49 | func (c *Client) Close() { 50 | if c.c != nil { 51 | c.c.Close() 52 | } 53 | } 54 | 55 | func (c *Client) Name() string { 56 | return c.ExchangeName 57 | } 58 | 59 | func (c *Client) GetTicker() *goex.Ticker { 60 | var tick *goex.Ticker 61 | var err error 62 | if c.isDirect { 63 | if c.isSpot { 64 | tick, err = c.spotApi.GetTicker(goex.NewCurrencyPair2(c.CurrencyPair)) 65 | } else { 66 | tick, err = c.futureApi.GetFutureTicker(goex.NewCurrencyPair2(c.CurrencyPair), c.ContractType) 67 | } 68 | } else { 69 | if c.isSpot { 70 | tick, err = c.c.GetSpotTicker(c.ExchangeName, c.CurrencyPair) 71 | } else { 72 | tick, err = c.c.GetFutureTicker(c.ExchangeName, c.ContractType, c.CurrencyPair) 73 | } 74 | } 75 | if err != nil { 76 | return nil 77 | } 78 | return tick 79 | } 80 | 81 | func (c *Client) GetDepth() *goex.Depth { 82 | var depth *goex.Depth 83 | var err error 84 | if c.isDirect { 85 | if c.isSpot { 86 | depth, err = c.spotApi.GetDepth(20, goex.NewCurrencyPair2(c.CurrencyPair)) 87 | } else { 88 | depth, err = c.futureApi.GetFutureDepth(goex.NewCurrencyPair2(c.CurrencyPair), c.ContractType, 20) 89 | } 90 | } else { 91 | if c.isSpot { 92 | depth, err = c.c.GetSpotDepth(c.ExchangeName, c.CurrencyPair) 93 | } else { 94 | depth, err = c.c.GetFutureDepth(c.ExchangeName, c.ContractType, c.CurrencyPair) 95 | } 96 | } 97 | if err != nil || depth.AskList.Len() < 2 || depth.BidList.Len() < 2 { 98 | return nil 99 | } 100 | 101 | if depth.AskList[0].Price > depth.AskList[1].Price { 102 | sort.Slice(depth.AskList, func(i, j int) bool { 103 | return depth.AskList[i].Price < depth.AskList[j].Price 104 | }) 105 | } 106 | if depth.BidList[0].Price < depth.BidList[1].Price { 107 | sort.Slice(depth.BidList, func(i, j int) bool { 108 | return depth.BidList[i].Price > depth.BidList[j].Price 109 | }) 110 | } 111 | 112 | if depth.AskList.Len() > 20 { 113 | depth.AskList = depth.AskList[:20] 114 | } 115 | if depth.BidList.Len() > 20 { 116 | depth.BidList = depth.BidList[:20] 117 | } 118 | 119 | return depth 120 | } 121 | 122 | func (c *Client) GetKline() *goex.Kline { 123 | panic("not support yet") 124 | } 125 | -------------------------------------------------------------------------------- /collector/collector.go: -------------------------------------------------------------------------------- 1 | package collector 2 | 3 | import ( 4 | "context" 5 | "github.com/goex-top/market_center" 6 | "github.com/goex-top/market_data_collector/client" 7 | "github.com/goex-top/market_data_collector/storage" 8 | "log" 9 | "time" 10 | ) 11 | 12 | func NewCollector(ctx context.Context, c *client.Client, period int64, flag market_center.DataFlag, store storage.Storage) { 13 | log.Printf("(%s) %s %s new collector with flag[%d]\n", c.ExchangeName, c.ContractType, c.CurrencyPair, flag) 14 | go func() { 15 | tick := time.NewTicker(time.Millisecond * time.Duration(period)) 16 | for { 17 | select { 18 | case <-ctx.Done(): 19 | log.Printf("(%s) %s collector exit\n", c.ExchangeName, c.CurrencyPair) 20 | return 21 | case <-tick.C: 22 | if flag&market_center.DataFlag_Depth != 0 { 23 | depth := c.GetDepth() 24 | if depth != nil { 25 | if depth.UTime.IsZero() { 26 | depth.UTime = time.Now() 27 | } 28 | store.SaveDepth(depth) 29 | } 30 | } 31 | 32 | if flag&market_center.DataFlag_Ticker != 0 { 33 | ticker := c.GetTicker() 34 | if ticker != nil { 35 | if ticker.Date == 0 { 36 | ticker.Date = uint64(time.Now().UnixNano() / int64(time.Millisecond)) 37 | } 38 | store.SaveTicker(ticker) 39 | } 40 | } 41 | } 42 | } 43 | }() 44 | } 45 | -------------------------------------------------------------------------------- /config-sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "subs": [ 3 | { 4 | "exchange_name": "Binance", 5 | "currency_pair": "BTC_USDT", 6 | "period": 100, 7 | "flag": 3 8 | }, 9 | { 10 | "exchange_name": "Swap_Binance", 11 | "currency_pair": "BTC_USDT", 12 | "contract_type": "swap", 13 | "period": 100, 14 | "flag": 3 15 | } 16 | ], 17 | "store": { 18 | "csv": false, 19 | "influx_db": true, 20 | "influx_db_cfg": { 21 | "url": "http://localhost:8086", 22 | "database": "market_data" 23 | } 24 | }, 25 | "with_market_center": false, 26 | "market_center_path": "/tmp/goex.market.center" 27 | } 28 | -------------------------------------------------------------------------------- /config-sample.yml: -------------------------------------------------------------------------------- 1 | --- 2 | subs: 3 | - exchange_name: Binance 4 | currency_pair: BTC_USDT 5 | period: 100 6 | flag: 2 7 | - exchange_name: Swap_Binance 8 | currency_pair: BTC_USDT 9 | contract_type: swap 10 | period: 100 11 | flag: 2 12 | store: 13 | csv: false 14 | influx_db: true 15 | influx_db_cfg: 16 | url: http://localhost:8086 17 | database: market_data 18 | with_market_center: true 19 | market_center_path: "/tmp/goex.market.center" 20 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "github.com/goex-top/market_center" 4 | 5 | type Subscribe struct { 6 | ExchangeName string `json:"exchange_name" yaml:"exchange_name" default:"Binance"` 7 | CurrencyPair string `json:"currency_pair" yaml:"currency_pair" default:"BTC_USDT"` 8 | ContractType string `json:"contract_type,omitempty" yaml:"contract_type" default:""` 9 | Period int64 `json:"period" yaml:"period" default:"100"` 10 | Flag market_center.DataFlag `json:"flag" yaml:"flag" default:"1"` 11 | } 12 | 13 | type Storage struct { 14 | Csv bool `json:"csv" yaml:"csv" ` 15 | InfluxDB bool `json:"influx_db" yaml:"influx_db" ` 16 | InfluxDbCfg struct { 17 | Url string `json:"url" yaml:"url" default:"http://localhost:8086"` 18 | Database string `json:"database" yaml:"database" default:"market_data"` 19 | Username string `json:"username" yaml:"username" default:""` 20 | Password string `json:"password" yaml:"password" default:""` 21 | } `json:"influx_db_cfg" yaml:"influx_db_cfg"` 22 | // TBD 23 | } 24 | type Config struct { 25 | Subs []Subscribe `json:"subs" yaml:"subs" default:"subs"` 26 | Store Storage `json:"store" yaml:"store" default:""` 27 | WithMarketCenter bool `json:"with_market_center" yaml:"with_market_center" ` 28 | MarketCenterPath string `json:"market_center_path" yaml:"market_center_path" default:"/tmp/goex.market.center"` 29 | } 30 | -------------------------------------------------------------------------------- /config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/jinzhu/configor" 5 | "testing" 6 | ) 7 | 8 | func TestConfig(t *testing.T) { 9 | c1 := &Config{} 10 | t.Log(configor.Load(c1, "../config-sample.json")) 11 | t.Log(c1) 12 | c2 := &Config{} 13 | t.Log(configor.Load(c2, "../config-sample.yml")) 14 | t.Log(c2) 15 | } 16 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/goex-top/market_data_collector 2 | 3 | go 1.13 4 | 5 | replace github.com/nntaoli-project/goex v1.1.1 => github.com/beaquant/goex v1.0.5-0.20200522013427-ef4423112b89 6 | 7 | require ( 8 | github.com/go-gota/gota v0.10.1 9 | github.com/goex-top/market_center v1.0.6-alpha 10 | github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d 11 | github.com/jinzhu/configor v1.2.0 12 | github.com/json-iterator/go v1.1.8 13 | github.com/nntaoli-project/goex v1.1.1 14 | gonum.org/v1/gonum v0.7.0 // indirect 15 | ) 16 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/Kucoin/kucoin-go-sdk v1.2.7 h1:lh74YnCmcswmnvkk0nMeodw+y17UEjMhyEzrIS14SDs= 4 | github.com/Kucoin/kucoin-go-sdk v1.2.7/go.mod h1:Wz3fTuM5gIct9chN6H6OBCXbku10XEcAjH5g/FL3wIY= 5 | github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= 6 | github.com/beaquant/goex v1.0.5-0.20200522013427-ef4423112b89 h1:qKyETMqRdLHN/NfmXIvFww+rUn892xyAQUadosGCaO0= 7 | github.com/beaquant/goex v1.0.5-0.20200522013427-ef4423112b89/go.mod h1:sCb1e95fe/WBvGA8JVQc3r2XTFawUZro39mb9/qY3To= 8 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 10 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 11 | github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= 12 | github.com/go-gota/gota v0.10.1 h1:BWci+R5dE28GnXoD1EWoQqe7WCQHAPJ996mK7LZrB4U= 13 | github.com/go-gota/gota v0.10.1/go.mod h1:NZLQccXn0rABmkXjsaugRY6l+UH2dDZSgIgF8E2ipmA= 14 | github.com/go-openapi/errors v0.19.4 h1:fSGwO1tSYHFu70NKaWJt5Qh0qoBRtCm/mXS1yhf+0W0= 15 | github.com/go-openapi/errors v0.19.4/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= 16 | github.com/goex-top/market_center v1.0.6-alpha h1:t/2MMyyXE94Qrvk3WtrvEFd6bW1uz/CnM+FbiLtIzZo= 17 | github.com/goex-top/market_center v1.0.6-alpha/go.mod h1:wUIcwJyaRlwOVliuu3EDTN0YGWYZKA4PkWihwhWEYdg= 18 | github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= 19 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 20 | github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= 21 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 22 | github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 23 | github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= 24 | github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 25 | github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d h1:/WZQPMZNsjZ7IlCpsLGdQBINg5bxKQ1K1sh6awxLtkA= 26 | github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= 27 | github.com/jinzhu/configor v1.2.0 h1:u78Jsrxw2+3sGbGMgpY64ObKU4xWCNmNRJIjGVqxYQA= 28 | github.com/jinzhu/configor v1.2.0/go.mod h1:nX89/MOmDba7ZX7GCyU/VIaQ2Ar2aizBl2d3JLF/rDc= 29 | github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 30 | github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= 31 | github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 32 | github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= 33 | github.com/klauspost/compress v1.8.2 h1:Bx0qjetmNjdFXASH02NSAREKpiaDwkO1DRZ3dV2KCcs= 34 | github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 35 | github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w= 36 | github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= 37 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= 38 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 39 | github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= 40 | github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 41 | github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= 42 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 43 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= 44 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 45 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= 46 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 47 | github.com/nubo/jwt v0.0.0-20150918093313-da5b79c3bbaf h1:mP7zQzhCrNQgSdCpxFxyZV/JMHbz4LJsyppAZMQVrI0= 48 | github.com/nubo/jwt v0.0.0-20150918093313-da5b79c3bbaf/go.mod h1:LuR7jHS+7SJ6EywD7zZiO6h0vwTBSevFk5wunVt3gf4= 49 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 50 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 51 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 52 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 53 | github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= 54 | github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= 55 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 56 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 57 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 58 | github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= 59 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 60 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 61 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 62 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 63 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 64 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 65 | github.com/valyala/fasthttp v1.6.0 h1:uWF8lgKmeaIewWVPwi4GRq2P6+R46IgYZdxWtM+GtEY= 66 | github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= 67 | github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= 68 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 69 | golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 70 | golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 71 | golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2 h1:y102fOLFqhV41b+4GPiJoa0k/x+pJcEi2/HB1Y5T6fU= 72 | golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 73 | golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= 74 | golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM= 75 | golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 76 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 77 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 78 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= 79 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 80 | golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9 h1:1/DFK4b7JH8DmkqhUk48onnSfrPzImPoVxuomtbT2nk= 81 | golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 82 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 83 | golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 84 | golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 85 | gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= 86 | gonum.org/v1/gonum v0.7.0 h1:Hdks0L0hgznZLG9nzXb8vZ0rRvqNvAcgAp84y7Mwkgw= 87 | gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM= 88 | gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= 89 | gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= 90 | gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= 91 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 92 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 93 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 94 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 95 | rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= 96 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "github.com/goex-top/market_center" 8 | mcc "github.com/goex-top/market_center/client" 9 | "github.com/goex-top/market_data_collector/client" 10 | "github.com/goex-top/market_data_collector/collector" 11 | "github.com/goex-top/market_data_collector/config" 12 | "github.com/goex-top/market_data_collector/storage" 13 | "github.com/goex-top/market_data_collector/storage/csv" 14 | "github.com/goex-top/market_data_collector/storage/influxdb" 15 | "github.com/jinzhu/configor" 16 | "log" 17 | "os" 18 | "os/signal" 19 | "syscall" 20 | "time" 21 | ) 22 | 23 | var ( 24 | cfg config.Config 25 | ) 26 | 27 | func usage() { 28 | fmt.Fprintf(os.Stderr, `market data collector version: v1.0.0 29 | Usage: market_data_collector [-h] [-c config.json] 30 | Options: 31 | `) 32 | flag.PrintDefaults() 33 | } 34 | 35 | func main() { 36 | var c string 37 | var help bool 38 | flag.StringVar(&c, "c", "config.yml", "set configuration `json/yml file`") 39 | flag.BoolVar(&help, "h", false, "this help") 40 | flag.Usage = usage 41 | 42 | flag.Parse() 43 | if help { 44 | flag.Usage() 45 | return 46 | } 47 | err := configor.Load(&cfg, c) 48 | 49 | if err != nil { 50 | panic(err) 51 | } 52 | 53 | if cfg.Store.Csv == cfg.Store.InfluxDB { 54 | panic("currently only support csv, please check your configure") 55 | } 56 | 57 | ctx, cancel := context.WithCancel(context.Background()) 58 | for _, v := range cfg.Subs { 59 | var sto storage.Storage 60 | if cfg.Store.Csv { 61 | sto = csv.NewCsvStorage(ctx, v.ExchangeName, v.CurrencyPair, v.ContractType, v.Flag, "output/csv", "output/tar") 62 | } 63 | if cfg.Store.InfluxDB { 64 | sto = influxdb.NewInfluxdb(ctx, v.ExchangeName, v.CurrencyPair, v.ContractType, cfg.Store.InfluxDbCfg.Url, cfg.Store.InfluxDbCfg.Database, cfg.Store.InfluxDbCfg.Username, cfg.Store.InfluxDbCfg.Password) 65 | } 66 | go sto.SaveWorker() 67 | cl := &client.Client{} 68 | if !cfg.WithMarketCenter { 69 | cl = client.NewClient(v.ExchangeName, v.CurrencyPair, v.ContractType, nil) 70 | } else { 71 | mccc := mcc.NewClient() 72 | isSpot := market_center.IsFutureExchange(v.ExchangeName) 73 | if v.Flag&market_center.DataFlag_Depth != 0 { 74 | if isSpot { 75 | mccc.SubscribeSpotDepth(v.ExchangeName, v.CurrencyPair, v.Period) 76 | } else { 77 | mccc.SubscribeFutureDepth(v.ExchangeName, v.ContractType, v.CurrencyPair, v.Period) 78 | } 79 | } 80 | if v.Flag&market_center.DataFlag_Ticker != 0 { 81 | if isSpot { 82 | mccc.SubscribeSpotTicker(v.ExchangeName, v.CurrencyPair, v.Period) 83 | } else { 84 | mccc.SubscribeFutureTicker(v.ExchangeName, v.ContractType, v.CurrencyPair, v.Period) 85 | } 86 | } 87 | cl = client.NewClient(v.ExchangeName, v.CurrencyPair, v.ContractType, mccc) 88 | } 89 | 90 | collector.NewCollector(ctx, cl, v.Period, v.Flag, sto) 91 | } 92 | 93 | exitSignal := make(chan os.Signal, 1) 94 | sigs := []os.Signal{os.Interrupt, syscall.SIGILL, syscall.SIGINT, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM} 95 | signal.Notify(exitSignal, sigs...) 96 | <-exitSignal 97 | cancel() 98 | time.Sleep(time.Second) 99 | log.Println("market data collector exit") 100 | } 101 | -------------------------------------------------------------------------------- /orderbook.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/goex-top/market_data_collector/5d444d185a6bd9f00ea37400e92cc65adba7d2eb/orderbook.png -------------------------------------------------------------------------------- /output/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/goex-top/market_data_collector/5d444d185a6bd9f00ea37400e92cc65adba7d2eb/output/.gitkeep -------------------------------------------------------------------------------- /output/csv/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/goex-top/market_data_collector/5d444d185a6bd9f00ea37400e92cc65adba7d2eb/output/csv/.gitkeep -------------------------------------------------------------------------------- /output/tar/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/goex-top/market_data_collector/5d444d185a6bd9f00ea37400e92cc65adba7d2eb/output/tar/.gitkeep -------------------------------------------------------------------------------- /parser/data_csv_depth.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "encoding/csv" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "github.com/go-gota/gota/dataframe" 9 | "github.com/nntaoli-project/goex" 10 | "io/ioutil" 11 | "log" 12 | "os" 13 | "path/filepath" 14 | "strings" 15 | "time" 16 | ) 17 | 18 | // DepthDataFromCSVeData loads the market data from a csv file. 19 | // It expands the underlying data struct. 20 | type DepthDataFromCSVeData struct { 21 | FileDir string 22 | } 23 | 24 | func NewDepthDataParser(fileDir string) *DepthDataFromCSVeData { 25 | return &DepthDataFromCSVeData{FileDir: fileDir} 26 | } 27 | 28 | // Load single data events into a stream ordered by date (latest first). 29 | func (d *DepthDataFromCSVeData) Load(filename string) ([]goex.Depth, error) { 30 | // check file location 31 | if len(d.FileDir) == 0 { 32 | return nil, errors.New("no directory for data provided: ") 33 | } 34 | 35 | if len(filename) == 0 { 36 | return nil, errors.New("no filename for data provided: ") 37 | } 38 | if !strings.Contains(filename, "depth") { 39 | return nil, errors.New("no a standard depth filename") 40 | } 41 | if d.FileDir[len(d.FileDir)-1] != '/' { 42 | d.FileDir += "/" 43 | } 44 | 45 | df := LoadDataFrameFromCSV(d.FileDir + filename) 46 | 47 | if df.Err != nil { 48 | return nil, df.Err 49 | } 50 | 51 | depths := make([]goex.Depth, 0) 52 | 53 | size := df.Nrow() 54 | lastTs := 0 55 | for i := 0; i < size; i++ { 56 | ts, _ := df.Elem(i, 0).Int() 57 | if ts == lastTs { 58 | continue 59 | } 60 | lastTs = ts 61 | 62 | str1 := df.Elem(i, 1).String() 63 | asks := make([][]float64, 0) 64 | err := json.Unmarshal([]byte(str1), &asks) 65 | if err != nil { 66 | panic(err) 67 | } 68 | str2 := df.Elem(i, 2).String() 69 | bids := make([][]float64, 0) 70 | err = json.Unmarshal([]byte(str2), &bids) 71 | if err != nil { 72 | panic(err) 73 | } 74 | 75 | asklist := make(goex.DepthRecords, 0) 76 | for _, v := range asks { 77 | asklist = append(asklist, goex.DepthRecord{ 78 | Price: v[0], 79 | Amount: v[1], 80 | }) 81 | } 82 | bidlist := make(goex.DepthRecords, 0) 83 | for _, v := range bids { 84 | bidlist = append(bidlist, goex.DepthRecord{ 85 | Price: v[0], 86 | Amount: v[1], 87 | }) 88 | } 89 | 90 | depths = append(depths, goex.Depth{ 91 | UTime: time.Unix(0, int64(ts)*int64(time.Millisecond)), 92 | AskList: asklist, 93 | BidList: bidlist, 94 | }) 95 | } 96 | return depths, nil 97 | } 98 | 99 | func LoadDataFrameFromCSV(filename string) *dataframe.DataFrame { 100 | fmt.Println("load csv file: ", filename) 101 | f, err := os.Open(filename) 102 | defer f.Close() 103 | 104 | if err != nil { 105 | return nil 106 | } 107 | df := dataframe.ReadCSV(f) 108 | return &df 109 | } 110 | 111 | // readDepthFromCSVFile opens and reads a csv file line by line 112 | // and returns a slice with a key/value map for each line. 113 | func readDepthFromCSVFile(path string) (lines []map[string]string, err error) { 114 | log.Printf("Loading from %s.\n", path) 115 | // open file 116 | file, err := os.Open(path) 117 | if err != nil { 118 | return nil, err 119 | } 120 | defer file.Close() 121 | 122 | // create scanner on top of file 123 | reader := csv.NewReader(file) 124 | // set delimiter 125 | reader.Comma = ',' 126 | // read first line for keys and fill in array 127 | keys, err := reader.Read() 128 | 129 | // read each line and create a map of values combined to the keys 130 | for line, err := reader.Read(); err == nil; line, err = reader.Read() { 131 | l := make(map[string]string) 132 | for i, v := range line { 133 | l[keys[i]] = v 134 | } 135 | // put found line as map into stream holder item 136 | lines = append(lines, l) 137 | } 138 | 139 | return lines, nil 140 | } 141 | 142 | // fetchFilesFromDir returns a map of all filenames in a directory, 143 | func fetchFilesFromDir(dir string) (m map[string]string, err error) { 144 | // read filenames from directory 145 | files, err := ioutil.ReadDir(dir) 146 | if err != nil { 147 | return m, err 148 | } 149 | 150 | // initialise the map 151 | m = make(map[string]string) 152 | 153 | // read filenames from directory 154 | for _, file := range files { 155 | // file is directory 156 | if file.IsDir() { 157 | continue 158 | } 159 | 160 | filename := file.Name() 161 | extension := filepath.Ext(filename) 162 | // file is not CSV 163 | if extension != ".csv" { 164 | continue 165 | } 166 | 167 | name := filename[0 : len(filename)-len(extension)] 168 | m[name] = filename 169 | } 170 | return m, nil 171 | } 172 | -------------------------------------------------------------------------------- /parser/data_csv_depth_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import "testing" 4 | 5 | func TestDepthDataFromCSVeData_Load(t *testing.T) { 6 | dd := NewDepthDataParser("../output/csv/") 7 | t.Log(dd.Load("depth_binance.com_BTC_USDT_2020-01-30.csv")) 8 | } 9 | -------------------------------------------------------------------------------- /storage/csv/csv.go: -------------------------------------------------------------------------------- 1 | package csv 2 | 3 | import ( 4 | "context" 5 | "encoding/csv" 6 | "fmt" 7 | "github.com/goex-top/market_center" 8 | jsoniter "github.com/json-iterator/go" 9 | "github.com/nntaoli-project/goex" 10 | "log" 11 | "os" 12 | "time" 13 | ) 14 | 15 | var json = jsoniter.ConfigCompatibleWithStandardLibrary 16 | 17 | type CsvStorage struct { 18 | exchangeName string 19 | pair string 20 | contractType string 21 | flag market_center.DataFlag 22 | prefix string 23 | outputPath string 24 | saveDepthChan chan goex.Depth 25 | saveTickerChan chan goex.Ticker 26 | saveKlineChan chan goex.Kline 27 | fileTimestamp time.Time 28 | ctx context.Context 29 | depthFile *os.File 30 | tickerFile *os.File 31 | klineFile *os.File 32 | depthCsv *csv.Writer 33 | tickerCsv *csv.Writer 34 | klineCsv *csv.Writer 35 | } 36 | 37 | func NewCsvStorage( 38 | ctx context.Context, 39 | exchangeName string, 40 | pair string, 41 | contractType string, 42 | flag market_center.DataFlag, 43 | prefix string, 44 | outputPath string, 45 | ) *CsvStorage { 46 | var saveDepthChan chan goex.Depth 47 | var saveTickerChan chan goex.Ticker 48 | var saveKlineChan chan goex.Kline 49 | var depthFile *os.File 50 | var tickerFile *os.File 51 | var klineFile *os.File 52 | var depthCsv *csv.Writer 53 | var tickerCsv *csv.Writer 54 | var klineCsv *csv.Writer 55 | 56 | fileTimestamp := time.Now() 57 | ts := fileTimestamp.Format("2006-01-02") 58 | isNew := false 59 | 60 | if flag&market_center.DataFlag_Depth != 0 { 61 | isNew, depthFile = OpenCsvFile(fmt.Sprintf("%s/depth_%s_%s_%s%s.csv", prefix, exchangeName, pair, contractType, ts)) 62 | depthCsv = csv.NewWriter(depthFile) 63 | if isNew { 64 | data := []string{"t", "a", "b"} 65 | depthCsv.Write(data) 66 | depthCsv.Flush() 67 | } 68 | saveDepthChan = make(chan goex.Depth) 69 | } 70 | 71 | if flag&market_center.DataFlag_Ticker != 0 { 72 | isNew, tickerFile = OpenCsvFile(fmt.Sprintf("%s/ticker_%s_%s_%s%s.csv", prefix, exchangeName, pair, contractType, ts)) 73 | tickerCsv = csv.NewWriter(tickerFile) 74 | if isNew { 75 | data := []string{"t", "b", "s", "h", "l", "v"} 76 | tickerCsv.Write(data) 77 | tickerCsv.Flush() 78 | } 79 | saveTickerChan = make(chan goex.Ticker) 80 | } 81 | 82 | if flag&market_center.DataFlag_Kline != 0 { 83 | isNew, klineFile = OpenCsvFile(fmt.Sprintf("%s/depth_%s_%s_%s%s.csv", prefix, exchangeName, pair, contractType, ts)) 84 | klineCsv = csv.NewWriter(klineFile) 85 | if isNew { 86 | data := []string{"t", "o", "h", "l", "c", "v"} 87 | klineCsv.Write(data) 88 | klineCsv.Flush() 89 | } 90 | saveKlineChan = make(chan goex.Kline) 91 | } 92 | 93 | return &CsvStorage{ 94 | ctx: ctx, 95 | exchangeName: exchangeName, 96 | pair: pair, 97 | contractType: contractType, 98 | flag: flag, 99 | prefix: prefix, 100 | outputPath: outputPath, 101 | saveDepthChan: saveDepthChan, 102 | saveTickerChan: saveTickerChan, 103 | saveKlineChan: saveKlineChan, 104 | fileTimestamp: fileTimestamp, 105 | depthFile: depthFile, 106 | tickerFile: tickerFile, 107 | klineFile: klineFile, 108 | depthCsv: depthCsv, 109 | tickerCsv: tickerCsv, 110 | klineCsv: klineCsv, 111 | } 112 | } 113 | 114 | func (s *CsvStorage) SaveDepth(depth *goex.Depth) { 115 | if s.saveDepthChan == nil { 116 | return 117 | } 118 | s.saveDepthChan <- *depth 119 | } 120 | 121 | func (s *CsvStorage) SaveTicker(ticker *goex.Ticker) { 122 | if s.saveTickerChan == nil { 123 | return 124 | } 125 | 126 | s.saveTickerChan <- *ticker 127 | } 128 | 129 | func (s *CsvStorage) SaveKline(kline *goex.Kline) { 130 | if s.saveKlineChan == nil { 131 | return 132 | } 133 | 134 | s.saveKlineChan <- *kline 135 | } 136 | 137 | func (s *CsvStorage) Close() { 138 | if s.depthCsv != nil { 139 | s.depthCsv.Flush() 140 | s.depthFile.Close() 141 | } 142 | if s.tickerCsv != nil { 143 | s.tickerCsv.Flush() 144 | s.tickerFile.Close() 145 | } 146 | if s.klineCsv != nil { 147 | s.klineCsv.Flush() 148 | s.klineFile.Close() 149 | } 150 | //close(s.saveDepthChan) 151 | //close(s.saveTickerChan) 152 | //close(s.saveKlineChan) 153 | } 154 | 155 | func (s *CsvStorage) compress(fileTimestamp time.Time) { 156 | ts := fileTimestamp.Format("2006-01-02") 157 | //src := fmt.Sprintf("%s_%s_%s.csv", s.exchangeName, s.pair, ts) 158 | filters := []string{s.exchangeName, s.pair, ts, ".csv"} 159 | dst := fmt.Sprintf("%s/%s_%s_%s.tar.gz", s.outputPath, s.exchangeName, s.pair, ts) 160 | 161 | csvs := GetSrcFileName(s.prefix, filters) 162 | log.Println("start to compress *.csv to *.tar.gz, ts:", ts) 163 | err := CompressFile(s.prefix, csvs, dst) 164 | if err != nil { 165 | log.Println(err) 166 | return 167 | } 168 | for _, v := range csvs { 169 | err := os.Remove(s.prefix + "/" + v) 170 | if err != nil { 171 | log.Printf("remove file %s fail:%s\n", s.prefix+"/"+v, err.Error()) 172 | } else { 173 | log.Printf("remove file %s success\n", s.prefix+"/"+v) 174 | } 175 | } 176 | } 177 | 178 | func (s *CsvStorage) reNewFile() { 179 | now := time.Now() 180 | if now.Day() == s.fileTimestamp.Day() { 181 | return 182 | } 183 | s.Close() 184 | log.Printf("now day:%d, file timestamp day:%d", now.Day(), s.fileTimestamp.Day()) 185 | go s.compress(s.fileTimestamp) 186 | 187 | s.fileTimestamp = now 188 | 189 | ts := s.fileTimestamp.Format("2006-01-02") 190 | isNew := false 191 | 192 | if s.flag&market_center.DataFlag_Depth != 0 { 193 | isNew, s.depthFile = OpenCsvFile(fmt.Sprintf("%s/depth_%s_%s_%s%s.csv", s.prefix, s.exchangeName, s.pair, s.contractType, ts)) 194 | s.depthCsv = csv.NewWriter(s.depthFile) 195 | if isNew { 196 | data := []string{"t", "a", "b"} 197 | s.depthCsv.Write(data) 198 | s.depthCsv.Flush() 199 | } 200 | } 201 | 202 | if s.flag&market_center.DataFlag_Ticker != 0 { 203 | isNew, s.tickerFile = OpenCsvFile(fmt.Sprintf("%s/ticker_%s_%s_%s%s.csv", s.prefix, s.exchangeName, s.pair, s.contractType, ts)) 204 | s.tickerCsv = csv.NewWriter(s.tickerFile) 205 | if isNew { 206 | data := []string{"t", "b", "s", "h", "l", "v"} 207 | s.tickerCsv.Write(data) 208 | s.tickerCsv.Flush() 209 | } 210 | } 211 | 212 | if s.flag&market_center.DataFlag_Kline != 0 { 213 | isNew, s.klineFile = OpenCsvFile(fmt.Sprintf("%s/depth_%s_%s_%s%s.csv", s.prefix, s.exchangeName, s.pair, s.contractType, ts)) 214 | s.klineCsv = csv.NewWriter(s.klineFile) 215 | if isNew { 216 | data := []string{"t", "o", "h", "l", "c", "v"} 217 | s.klineCsv.Write(data) 218 | s.klineCsv.Flush() 219 | } 220 | } 221 | } 222 | 223 | func (s *CsvStorage) SaveWorker() { 224 | 225 | tick := time.NewTicker(time.Second) 226 | 227 | for { 228 | select { 229 | case <-tick.C: 230 | s.reNewFile() 231 | case o := <-s.saveDepthChan: 232 | asks := make([][]float64, 0) 233 | bids := make([][]float64, 0) 234 | for _, v := range o.AskList { 235 | ask := make([]float64, 0) 236 | ask = append(ask, v.Price, v.Amount) 237 | asks = append(asks, ask) 238 | } 239 | a, _ := json.Marshal(asks) 240 | 241 | for _, v := range o.BidList { 242 | bid := make([]float64, 0) 243 | bid = append(bid, v.Price, v.Amount) 244 | bids = append(bids, bid) 245 | } 246 | b, _ := json.Marshal(bids) 247 | 248 | data := []string{ 249 | fmt.Sprint(o.UTime.UnixNano() / int64(time.Millisecond)), 250 | string(a), 251 | string(b), 252 | } 253 | 254 | s.depthCsv.Write(data) 255 | s.depthCsv.Flush() 256 | 257 | case o := <-s.saveTickerChan: 258 | data := []string{ 259 | fmt.Sprint(o.Date), 260 | fmt.Sprint(o.Buy), 261 | fmt.Sprint(o.Sell), 262 | fmt.Sprint(o.High), 263 | fmt.Sprint(o.Low), 264 | fmt.Sprint(o.Vol), 265 | } 266 | s.tickerCsv.Write(data) 267 | s.tickerCsv.Flush() 268 | 269 | case o := <-s.saveKlineChan: 270 | data := []string{ 271 | fmt.Sprint(o.Timestamp), 272 | fmt.Sprint(o.Open), 273 | fmt.Sprint(o.High), 274 | fmt.Sprint(o.Low), 275 | fmt.Sprint(o.Close), 276 | fmt.Sprint(o.Vol), 277 | } 278 | 279 | s.klineCsv.Write(data) 280 | s.klineCsv.Flush() 281 | 282 | case <-s.ctx.Done(): 283 | s.Close() 284 | log.Printf("(%s) %s saveWorker exit\n", s.exchangeName, s.pair) 285 | return 286 | } 287 | } 288 | } 289 | 290 | func OpenCsvFile(fileName string) (bool, *os.File) { 291 | var file *os.File 292 | var err1 error 293 | var isNew = false 294 | checkFileIsExist := func(fileName string) bool { 295 | var exist = true 296 | if _, err := os.Stat(fileName); os.IsNotExist(err) { 297 | exist = false 298 | } 299 | return exist 300 | } 301 | if checkFileIsExist(fileName) { 302 | file, err1 = os.OpenFile(fileName, os.O_CREATE|os.O_APPEND|os.O_RDWR, 666) 303 | } else { 304 | file, err1 = os.Create(fileName) 305 | isNew = true 306 | } 307 | if err1 != nil { 308 | fmt.Fprintf(os.Stderr, "unable to write file on filehook %v", err1) 309 | panic(err1) 310 | } 311 | return isNew, file 312 | } 313 | -------------------------------------------------------------------------------- /storage/csv/csv_test.go: -------------------------------------------------------------------------------- 1 | package csv 2 | 3 | import ( 4 | "github.com/goex-top/market_center" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | var s = &CsvStorage{prefix: "../output/csv", outputPath: "../output/tar", exchangeName: market_center.BINANCE, pair: "BTC_USDT"} 10 | 11 | func TestCsvStorage_Compress(t *testing.T) { 12 | ts, _ := time.Parse("2006-01-02", "2020-01-29") 13 | 14 | s.compress(ts) 15 | } 16 | -------------------------------------------------------------------------------- /storage/csv/tar.go: -------------------------------------------------------------------------------- 1 | package csv 2 | 3 | import ( 4 | "archive/tar" 5 | "compress/gzip" 6 | "errors" 7 | "io" 8 | "io/ioutil" 9 | "log" 10 | "os" 11 | "strings" 12 | ) 13 | 14 | func GetSrcFileName(inputPath string, filters []string) []string { 15 | files := make([]string, 0) 16 | finfos, err := ioutil.ReadDir(inputPath) 17 | if err != nil { 18 | log.Printf("err : %s \n", err) 19 | return files 20 | } 21 | for _, fi := range finfos { 22 | if fi.IsDir() { 23 | continue 24 | } 25 | notMatch := false 26 | for _, filter := range filters { 27 | if !strings.Contains(fi.Name(), filter) { 28 | notMatch = true 29 | } 30 | } 31 | if !notMatch { 32 | files = append(files, fi.Name()) 33 | } 34 | } 35 | return files 36 | } 37 | 38 | func CompressFile(inputPath string, src []string, dest string) error { 39 | csvFiles := make([]*os.File, 0) 40 | if len(src) == 0 { 41 | log.Println("no file will compress") 42 | return errors.New("no file will compress") 43 | } 44 | 45 | for _, fi := range src { 46 | log.Printf("%s\n", fi) 47 | 48 | file, err1 := os.OpenFile(inputPath+"/"+fi, os.O_RDONLY, 666) 49 | if err1 != nil { 50 | log.Printf("open %s error:%v", inputPath+fi, err1) 51 | continue 52 | } 53 | csvFiles = append(csvFiles, file) 54 | } 55 | 56 | return Compress(csvFiles, dest) 57 | } 58 | 59 | //压缩 使用gzip压缩成tar.gz 60 | func Compress(files []*os.File, dest string) error { 61 | d, _ := os.Create(dest) 62 | defer d.Close() 63 | gw := gzip.NewWriter(d) 64 | defer gw.Close() 65 | tw := tar.NewWriter(gw) 66 | defer tw.Close() 67 | for _, file := range files { 68 | err := compress(file, "", tw) 69 | if err != nil { 70 | return err 71 | } 72 | } 73 | return nil 74 | } 75 | 76 | func compress(file *os.File, prefix string, tw *tar.Writer) error { 77 | info, err := file.Stat() 78 | if err != nil { 79 | return err 80 | } 81 | if info.IsDir() { 82 | prefix = prefix + "/" + info.Name() 83 | fileInfos, err := file.Readdir(-1) 84 | if err != nil { 85 | return err 86 | } 87 | for _, fi := range fileInfos { 88 | f, err := os.Open(file.Name() + "/" + fi.Name()) 89 | if err != nil { 90 | return err 91 | } 92 | err = compress(f, prefix, tw) 93 | if err != nil { 94 | return err 95 | } 96 | } 97 | } else { 98 | header, err := tar.FileInfoHeader(info, "") 99 | header.Name = prefix + "/" + header.Name 100 | if err != nil { 101 | return err 102 | } 103 | err = tw.WriteHeader(header) 104 | if err != nil { 105 | return err 106 | } 107 | _, err = io.Copy(tw, file) 108 | file.Close() 109 | if err != nil { 110 | return err 111 | } 112 | } 113 | return nil 114 | } 115 | 116 | //解压 tar.gz 117 | func DeCompress(tarFile, dest string) error { 118 | srcFile, err := os.Open(tarFile) 119 | if err != nil { 120 | return err 121 | } 122 | defer srcFile.Close() 123 | gr, err := gzip.NewReader(srcFile) 124 | if err != nil { 125 | return err 126 | } 127 | defer gr.Close() 128 | tr := tar.NewReader(gr) 129 | for { 130 | hdr, err := tr.Next() 131 | if err != nil { 132 | if err == io.EOF { 133 | break 134 | } else { 135 | return err 136 | } 137 | } 138 | filename := dest + hdr.Name 139 | file, err := createFile(filename) 140 | if err != nil { 141 | return err 142 | } 143 | io.Copy(file, tr) 144 | } 145 | return nil 146 | } 147 | 148 | func createFile(name string) (*os.File, error) { 149 | err := os.MkdirAll(string([]rune(name)[0:strings.LastIndex(name, "/")]), 0755) 150 | if err != nil { 151 | return nil, err 152 | } 153 | return os.Create(name) 154 | } 155 | -------------------------------------------------------------------------------- /storage/csv/tar_test.go: -------------------------------------------------------------------------------- 1 | package csv 2 | 3 | import ( 4 | "log" 5 | "testing" 6 | ) 7 | 8 | func TestGetAllFileName(t *testing.T) { 9 | csvs := GetSrcFileName("../output/csv/", []string{"csv"}) 10 | for _, v := range csvs { 11 | log.Println(v) 12 | //t.Log(os.Remove("../output/csv/" + v)) 13 | } 14 | } 15 | 16 | func TestCompressAllCsv(t *testing.T) { 17 | CompressFile("../output/csv", []string{"2006-01-02"}, "../output/tar/aaa.tar.gz") 18 | } 19 | -------------------------------------------------------------------------------- /storage/influxdb/influxdb.go: -------------------------------------------------------------------------------- 1 | package influxdb 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | _ "github.com/influxdata/influxdb1-client" // this is important because of the bug in go mod 7 | client "github.com/influxdata/influxdb1-client/v2" 8 | "github.com/nntaoli-project/goex" 9 | "log" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | type InfluxdbStorage struct { 15 | ctx context.Context 16 | exchangeName string 17 | pair string 18 | contractType string 19 | tag map[string]string 20 | Url string 21 | DatabaseName string 22 | Username string 23 | Password string 24 | cli client.Client 25 | saveDepthChan chan goex.Depth 26 | saveTickerChan chan goex.Ticker 27 | saveKlineChan chan goex.Kline 28 | } 29 | 30 | func NewInfluxdb(ctx context.Context, 31 | exchangeName string, 32 | pair string, 33 | contractType string, 34 | url, 35 | databaseName, 36 | username, 37 | password string, 38 | ) *InfluxdbStorage { 39 | s := &InfluxdbStorage{ 40 | ctx: ctx, 41 | Username: username, 42 | Password: password, 43 | Url: url, 44 | DatabaseName: databaseName, 45 | exchangeName: exchangeName, 46 | pair: pair, 47 | contractType: contractType, 48 | } 49 | cli, err := client.NewHTTPClient(client.HTTPConfig{ 50 | Addr: url, 51 | Username: username, 52 | Password: password, 53 | }) 54 | if err != nil { 55 | log.Fatal(err) 56 | } 57 | s.tag = make(map[string]string) 58 | if strings.Contains(exchangeName, "Future") { 59 | s.tag[exchangeName+"_"+contractType] = pair 60 | } else if strings.Contains(exchangeName, "Swap") { 61 | s.tag[exchangeName+"_"+contractType] = pair 62 | } else { 63 | s.tag[exchangeName+"_spot"] = pair 64 | } 65 | s.cli = cli 66 | s.saveTickerChan = make(chan goex.Ticker) 67 | s.saveDepthChan = make(chan goex.Depth) 68 | s.saveKlineChan = make(chan goex.Kline) 69 | return s 70 | } 71 | 72 | func (s *InfluxdbStorage) SaveDepth(depth *goex.Depth) { 73 | if s.saveDepthChan == nil { 74 | return 75 | } 76 | s.saveDepthChan <- *depth 77 | } 78 | 79 | func (s *InfluxdbStorage) SaveTicker(ticker *goex.Ticker) { 80 | if s.saveTickerChan == nil { 81 | return 82 | } 83 | 84 | s.saveTickerChan <- *ticker 85 | } 86 | 87 | func (s *InfluxdbStorage) SaveKline(kline *goex.Kline) { 88 | if s.saveKlineChan == nil { 89 | return 90 | } 91 | 92 | s.saveKlineChan <- *kline 93 | } 94 | 95 | func (s *InfluxdbStorage) Close() { 96 | //close(s.saveDepthChan) 97 | //close(s.saveTickerChan) 98 | //close(s.saveKlineChan) 99 | s.cli.Close() 100 | } 101 | 102 | func (s *InfluxdbStorage) SaveWorker() { 103 | /* 104 | |MEASUREMENT | TAGS | FIELDS| 105 | | ---- | ---- | ---- | 106 | |ticker | exchangeName_spot=pair | xxx| 107 | |kline | exchangeName_future_contractType=pair | xxx| 108 | |depth | exchangeName_swap=pair | xxx | 109 | */ 110 | 111 | for { 112 | select { 113 | case o := <-s.saveDepthChan: 114 | fields := make(map[string]interface{}) 115 | fields["ts"] = o.UTime.UnixNano() / int64(time.Millisecond) //unit ms 116 | for k, v := range o.AskList { 117 | fields[fmt.Sprintf("ask%d_price", k)] = v.Price 118 | fields[fmt.Sprintf("ask%d_amount", k)] = v.Amount 119 | } 120 | for k, v := range o.BidList { 121 | fields[fmt.Sprintf("bid%d_price", k)] = v.Price 122 | fields[fmt.Sprintf("bid%d_amount", k)] = v.Amount 123 | } 124 | s.WritesPoints("depth", s.tag, fields) 125 | case o := <-s.saveTickerChan: 126 | fields := make(map[string]interface{}) 127 | fields["ts"] = int64(o.Date) 128 | fields["last"] = o.Last 129 | fields["buy"] = o.Buy 130 | fields["sell"] = o.Sell 131 | fields["vol"] = o.Vol 132 | fields["high"] = o.High 133 | fields["low"] = o.Low 134 | 135 | s.WritesPoints("ticker", s.tag, fields) 136 | 137 | case o := <-s.saveKlineChan: 138 | fields := make(map[string]interface{}) 139 | fields["ts"] = o.Timestamp 140 | fields["open"] = o.Open 141 | fields["high"] = o.High 142 | fields["low"] = o.Low 143 | fields["close"] = o.Close 144 | fields["vol"] = o.Vol 145 | s.WritesPoints("kline", s.tag, fields) 146 | 147 | case <-s.ctx.Done(): 148 | s.Close() 149 | log.Printf("(%s) %s saveWorker exit\n", s.exchangeName, s.pair) 150 | return 151 | } 152 | } 153 | } 154 | 155 | //Insert 156 | func (s *InfluxdbStorage) WritesPoints(table string, tags map[string]string, fields map[string]interface{}) { 157 | bp, err := client.NewBatchPoints(client.BatchPointsConfig{ 158 | Database: s.DatabaseName, 159 | Precision: "ms", 160 | }) 161 | 162 | if err != nil { 163 | log.Println(s.exchangeName, s.pair, s.contractType, "Influxdb NewBatchPoints err:", err) 164 | return 165 | } 166 | 167 | pt, err := client.NewPoint( 168 | table, 169 | tags, 170 | fields, 171 | time.Now(), 172 | ) 173 | if err != nil { 174 | log.Println(s.exchangeName, s.pair, s.contractType, "Influxdb NewPoint err:", err) 175 | return 176 | } 177 | bp.AddPoint(pt) 178 | 179 | if err := s.cli.Write(bp); err != nil { 180 | log.Println(s.exchangeName, s.pair, s.contractType, "Influxdb Write err:", err) 181 | return 182 | } 183 | } 184 | 185 | //query 186 | func (s *InfluxdbStorage) QueryDB(cli client.Client, cmd string) (res []client.Result, err error) { 187 | q := client.Query{ 188 | Command: cmd, 189 | Database: s.DatabaseName, 190 | } 191 | if response, err := cli.Query(q); err == nil { 192 | if response.Error() != nil { 193 | return res, response.Error() 194 | } 195 | res = response.Results 196 | } else { 197 | return res, err 198 | } 199 | return res, nil 200 | } 201 | -------------------------------------------------------------------------------- /storage/influxdb/influxdb_test.go: -------------------------------------------------------------------------------- 1 | package influxdb 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/nntaoli-project/goex" 7 | "github.com/nntaoli-project/goex/binance" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | var s = NewInfluxdb(context.Background(), "binance", "btc_usdt", "", 13 | "http://localhost:8086", "testdb", "admin", "") 14 | 15 | func TestNewInfluxdb(t *testing.T) { 16 | 17 | } 18 | 19 | func storeTicker(ticker *goex.Ticker) { 20 | fmt.Println(ticker) 21 | s.WritesPoints("okex_future", map[string]string{"this_week": "btc_usd"}, map[string]interface{}{ 22 | "Pair": ticker.Pair.String(), 23 | "Last": ticker.Last, 24 | "Buy": ticker.Buy, 25 | "Sell": ticker.Sell, 26 | }) 27 | 28 | } 29 | 30 | func TestInfluxdbStorage_WritesPoints(t *testing.T) { 31 | ws := binance.NewBinanceWs() 32 | ws.ProxyUrl("socks5://127.0.0.1:1080") 33 | ws.SetCallbacks(storeTicker, nil, nil, nil) 34 | ws.SubscribeTicker(goex.BTC_USDT) 35 | time.Sleep(time.Minute) 36 | } 37 | -------------------------------------------------------------------------------- /storage/storage.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import "github.com/nntaoli-project/goex" 4 | 5 | type Storage interface { 6 | SaveDepth(depth *goex.Depth) 7 | SaveTicker(ticker *goex.Ticker) 8 | SaveKline(kline *goex.Kline) 9 | SaveWorker() 10 | Close() 11 | } 12 | -------------------------------------------------------------------------------- /ticker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/goex-top/market_data_collector/5d444d185a6bd9f00ea37400e92cc65adba7d2eb/ticker.png -------------------------------------------------------------------------------- /tool/csv2crex/README.md: -------------------------------------------------------------------------------- 1 | 2 | convert depth data to [crex format](https://github.com/coinrust/crex#%E5%9B%9E%E6%B5%8B%E6%95%B0%E6%8D%AE) 3 | 4 | ` ./csv2crex -p=../../output/csv -f=depth_huobi.pro_BTC_USDT_2020-01-30.csv` -------------------------------------------------------------------------------- /tool/csv2crex/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/csv" 5 | "flag" 6 | "fmt" 7 | "github.com/goex-top/market_data_collector/parser" 8 | csvsto "github.com/goex-top/market_data_collector/storage/csv" 9 | "os" 10 | "strconv" 11 | "time" 12 | ) 13 | 14 | func usage() { 15 | fmt.Fprintf(os.Stderr, `csv2crex version: v1.0.0 16 | Usage: csv2crex [-h] [-p path/xxx] [-f depth_xxx.csv] 17 | Options: 18 | `) 19 | flag.PrintDefaults() 20 | } 21 | 22 | func main() { 23 | var p string 24 | var f string 25 | var o string 26 | var help bool 27 | flag.StringVar(&p, "p", "./", "path") 28 | flag.StringVar(&f, "f", "*.csv", "csv file") 29 | flag.StringVar(&o, "o", "./", "output path") 30 | flag.BoolVar(&help, "h", false, "this help") 31 | flag.Usage = usage 32 | 33 | flag.Parse() 34 | if help { 35 | flag.Usage() 36 | return 37 | } 38 | if p[len(p)-1] != '/' { 39 | p += "/" 40 | } 41 | fmt.Printf("input path:%s\nfile:%s\noutput path:%s\n", p, f, o) 42 | var target = f 43 | if p == o { 44 | target = "new_" + target 45 | } 46 | csvParser := parser.NewDepthDataParser(p) 47 | depth, err := csvParser.Load(f) 48 | if err != nil { 49 | fmt.Printf("load file(%s) fail:%v", p+f, err) 50 | return 51 | } 52 | if o[len(o)-1] != '/' { 53 | o += "/" 54 | } 55 | isNew, targetCsvFile := csvsto.OpenCsvFile(o + target) 56 | targetCsv := csv.NewWriter(targetCsvFile) 57 | if isNew { 58 | data := []string{"t"} 59 | for k := range depth[0].AskList { 60 | if k >= 20 { 61 | break 62 | } 63 | data = append(data, fmt.Sprintf("asks[%d].price", k)) 64 | data = append(data, fmt.Sprintf("asks[%d].amount", k)) 65 | } 66 | for k := range depth[0].BidList { 67 | if k >= 20 { 68 | break 69 | } 70 | data = append(data, fmt.Sprintf("bids[%d].price", k)) 71 | data = append(data, fmt.Sprintf("bids[%d].amount", k)) 72 | } 73 | targetCsv.Write(data) 74 | targetCsv.Flush() 75 | } 76 | 77 | for k := range depth { 78 | data := []string{strconv.Itoa(int(depth[k].UTime.UnixNano() / int64(time.Millisecond)))} 79 | for kk, asks := range depth[k].AskList { 80 | if kk >= 20 { 81 | break 82 | } 83 | data = append(data, strconv.FormatFloat(asks.Price, 'f', -1, 64)) 84 | data = append(data, strconv.FormatFloat(asks.Amount, 'f', -1, 64)) 85 | } 86 | for kk, bids := range depth[k].BidList { 87 | if kk >= 20 { 88 | break 89 | } 90 | data = append(data, strconv.FormatFloat(bids.Price, 'f', -1, 64)) 91 | data = append(data, strconv.FormatFloat(bids.Amount, 'f', -1, 64)) 92 | } 93 | targetCsv.Write(data) 94 | } 95 | targetCsv.Flush() 96 | } 97 | -------------------------------------------------------------------------------- /tool/influx2csv/README.md: -------------------------------------------------------------------------------- 1 | `influx2csv.exe -o= -u=http://localhost:8086 -t=Huobi_spot=BTC_USDT -s=2020-05-10T00:00:00Z -e=2020-05-25T00:00:00Z` -------------------------------------------------------------------------------- /tool/influx2csv/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/csv" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | csvsto "github.com/goex-top/market_data_collector/storage/csv" 9 | client "github.com/influxdata/influxdb1-client/v2" 10 | "os" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | func usage() { 16 | fmt.Fprintf(os.Stderr, `influx2csv version: v1.0.0 17 | Usage: influx2csv [-h] [-p path/xxx] [-f depth_xxx.csv] 18 | Options: 19 | `) 20 | flag.PrintDefaults() 21 | } 22 | 23 | //Url, 24 | //DatabaseName, 25 | //Username, 26 | //Password string 27 | //exchangeName string 28 | //pair string 29 | //contractType string 30 | 31 | func main() { 32 | var databasename string 33 | var username string 34 | var password string 35 | var url string 36 | var tag string 37 | var o string 38 | var start string 39 | var end string 40 | var help bool 41 | flag.StringVar(&o, "o", "./", "output path") 42 | flag.StringVar(&url, "u", "http://localhost:8086", "influxdb url, http://localhost:8086") 43 | flag.StringVar(&databasename, "d", "market_data", "database name, market_data") 44 | flag.StringVar(&username, "n", "", "username") 45 | flag.StringVar(&password, "p", "", "password") 46 | flag.StringVar(&tag, "t", "", "tag, Future_Okex_next_week=BTC_USDT") 47 | flag.StringVar(&start, "s", "", "start time, 2020-05-22T23:00:00Z") 48 | flag.StringVar(&end, "e", "", "end time, 2020-05-22T23:00:00Z") 49 | flag.BoolVar(&help, "h", false, "this help") 50 | flag.Usage = usage 51 | 52 | flag.Parse() 53 | 54 | if help { 55 | flag.Usage() 56 | return 57 | } 58 | if url == "" { 59 | fmt.Println("please input url") 60 | return 61 | } 62 | if databasename == "" { 63 | fmt.Println("please input databasename") 64 | return 65 | } 66 | if tag == "" { 67 | fmt.Println("please input tag") 68 | return 69 | } 70 | fmt.Printf("url:%s\ndatabase:%s\ntag:%s\nstart:%s\nend:%s\n", url, databasename, tag, start, end) 71 | cli, err := client.NewHTTPClient(client.HTTPConfig{ 72 | Addr: url, 73 | Username: username, 74 | Password: password, 75 | }) 76 | if err != nil { 77 | fmt.Printf("new influxdb client fail:%s, exit!\n", err) 78 | return 79 | } 80 | _, _, err = cli.Ping(time.Second * 5) 81 | if err != nil { 82 | fmt.Printf("ping infuxdb fail:%s, exit!\n", err) 83 | return 84 | } 85 | fmt.Printf("ping %s is ok\n", url) 86 | if len(o) > 0 && o[len(o)-1] != '/' { 87 | o += "/" 88 | } 89 | t := strings.Split(tag, "=") 90 | target := fmt.Sprintf("depth_%s_%s_%s_%s.csv", t[0], t[1], start, end) 91 | target = strings.Replace(target, ":", ".", -1) 92 | isNew, targetCsvFile := csvsto.OpenCsvFile(o + target) 93 | defer targetCsvFile.Close() 94 | 95 | targetCsv := csv.NewWriter(targetCsvFile) 96 | if isNew { 97 | data := []string{"t"} 98 | for k := 0; k < 20; k++ { 99 | data = append(data, fmt.Sprintf("asks[%d].price", k)) 100 | data = append(data, fmt.Sprintf("asks[%d].amount", k)) 101 | } 102 | for k := 0; k < 20; k++ { 103 | data = append(data, fmt.Sprintf("bids[%d].price", k)) 104 | data = append(data, fmt.Sprintf("bids[%d].amount", k)) 105 | } 106 | targetCsv.Write(data) 107 | targetCsv.Flush() 108 | } else { 109 | fmt.Printf("file exist:%s, exit!\n", target) 110 | return 111 | } 112 | 113 | // verify tag 114 | q := fmt.Sprintf("SHOW TAG VALUES ON %s FROM depth WITH KEY = %s", databasename, t[0]) 115 | fmt.Println(q) 116 | ret, err := QueryDB(cli, databasename, q) 117 | if err != nil { 118 | fmt.Println(err) 119 | return 120 | } 121 | if len(ret) == 0 || len(ret[0].Series) == 0 { 122 | fmt.Println("tag query no data") 123 | return 124 | } 125 | 126 | // start query data 127 | st, err := time.Parse("2006-01-02T15:04:05Z", start) 128 | if err != nil { 129 | fmt.Printf("start time(%s) format error:%s\n", start, err) 130 | return 131 | } 132 | et, err := time.Parse("2006-01-02T15:04:05Z", end) 133 | if err != nil { 134 | fmt.Printf("end time(%s) format error:%s\n", end, err) 135 | return 136 | } 137 | minutes := 10 138 | count := 0 139 | n := int(et.Sub(st).Minutes()/float64(minutes)) + 1 140 | for k := 0; k < n; k++ { 141 | q = "SELECT last(ts) AS ts," 142 | for i := 0; i < 20; i++ { 143 | q += fmt.Sprintf(" last(\"ask%d_price\") AS \"asks[%d].price\", last(\"ask%d_amount\") AS \"asks[%d].amount\",", i, i, i, i) 144 | } 145 | for i := 0; i < 20; i++ { 146 | q += fmt.Sprintf(" last(\"bid%d_price\") AS \"bids[%d].price\", last(\"bid%d_amount\") AS \"bids[%d].amount\",", i, i, i, i) 147 | } 148 | q = q[:len(q)-1] 149 | q += fmt.Sprintf(" FROM \"%s\".\"autogen\".\"depth\" WHERE \"%s\"='%s' AND time >= '%s' AND time <= '%s' GROUP BY time(1ms) FILL(null)", 150 | databasename, t[0], t[1], st.Format("2006-01-02T15:04:05Z"), st.Add(time.Hour*4).Format("2006-01-02T15:04:05Z")) 151 | 152 | st = st.Add(time.Duration(minutes) * time.Minute) 153 | 154 | fmt.Println(q) 155 | ret, err = QueryDB(cli, databasename, q) 156 | if err != nil { 157 | fmt.Printf("[%s - %s] error:%s\n", st.Format("2006-01-02T15:04:05Z"), st.Add(time.Hour*4).Format("2006-01-02T15:04:05Z"), err) 158 | continue 159 | } 160 | if len(ret) == 0 || len(ret[0].Series) == 0 { 161 | fmt.Printf("[%s - %s] query no data %s\n", st.Format("2006-01-02T15:04:05Z"), st.Add(time.Hour*4).Format("2006-01-02T15:04:05Z"), ret[0].Err) 162 | continue 163 | } 164 | 165 | for _, row := range ret[0].Series { 166 | for _, value := range row.Values { 167 | if value[1] != nil { 168 | count++ 169 | data := make([]string, 0) 170 | data = append(data, string(value[1].(json.Number))) 171 | for k := 2; k < len(value); k += 2 { 172 | data = append(data, string(value[k].(json.Number))) 173 | data = append(data, string(value[k+1].(json.Number))) 174 | } 175 | targetCsv.Write(data) 176 | } 177 | } 178 | } 179 | targetCsv.Flush() 180 | fmt.Printf("%d rows has been exported sucuessful!\n", len(ret[0].Series)) 181 | } 182 | 183 | fmt.Printf("\ntotal %d rows has been exported to %s sucuessful!\n", count, target) 184 | } 185 | 186 | //query 187 | func QueryDB(cli client.Client, database, cmd string) (res []client.Result, err error) { 188 | q := client.Query{ 189 | Command: cmd, 190 | Database: database, 191 | } 192 | if response, err := cli.Query(q); err == nil { 193 | if response.Error() != nil { 194 | return res, response.Error() 195 | } 196 | res = response.Results 197 | } else { 198 | return res, err 199 | } 200 | return res, nil 201 | } 202 | -------------------------------------------------------------------------------- /ws/builder.go: -------------------------------------------------------------------------------- 1 | package ws 2 | 3 | import ( 4 | "github.com/goex-top/market_center" 5 | "github.com/nntaoli-project/goex" 6 | "github.com/nntaoli-project/goex/binance" 7 | //"github.com/nntaoli-project/goex/huobi" 8 | "github.com/nntaoli-project/goex/okex" 9 | ) 10 | 11 | func BuilderSpot(exchangeName string) WS { 12 | switch exchangeName { 13 | case market_center.OKEX: 14 | return NewOKExSpotV3Ws() 15 | case market_center.BINANCE: 16 | return binance.NewBinanceWs() 17 | default: 18 | return nil 19 | } 20 | 21 | } 22 | 23 | func BuilderFuture(exchangeName string) FutureWS { 24 | switch exchangeName { 25 | case market_center.FUTURE_OKEX: 26 | return NewOKExFutureV3Ws() 27 | //case market_center.FUTURE_HBDM: 28 | // return huobi.NewHbdmWs() 29 | default: 30 | return nil 31 | } 32 | } 33 | 34 | type OKExSpotV3Ws struct { 35 | ok *okex.OKExV3SpotWs 36 | } 37 | 38 | func NewOKExSpotV3Ws() *OKExSpotV3Ws { 39 | return &OKExSpotV3Ws{ 40 | ok: okex.NewOKExSpotV3Ws(nil), 41 | } 42 | } 43 | 44 | func (o *OKExSpotV3Ws) SubscribeTicker(pair goex.CurrencyPair) error { 45 | return o.ok.SubscribeTicker(pair) 46 | } 47 | 48 | func (o *OKExSpotV3Ws) SubscribeDepth(pair goex.CurrencyPair, size int) error { 49 | return o.ok.SubscribeDepth(pair, size) 50 | } 51 | 52 | func (o *OKExSpotV3Ws) SubscribeTrade(pair goex.CurrencyPair) error { 53 | return o.ok.SubscribeTrade(pair) 54 | } 55 | 56 | func (o *OKExSpotV3Ws) SubscribeKline(pair goex.CurrencyPair, period int) error { 57 | return o.ok.SubscribeKline(pair, period) 58 | } 59 | 60 | func (o *OKExSpotV3Ws) SetCallbacks( 61 | tickerCallback func(ticker *goex.Ticker), 62 | depthCallback func(*goex.Depth), 63 | tradeCallback func(*goex.Trade), 64 | klineCallback func(*goex.Kline, int)) { 65 | kcb := func(kline *goex.Kline, period goex.KlinePeriod) { 66 | klineCallback(kline, int(period)) 67 | } 68 | o.ok.SetCallbacks(tickerCallback, depthCallback, tradeCallback, kcb, nil) 69 | } 70 | 71 | type OKExFutureV3Ws struct { 72 | ok *okex.OKExV3FuturesWs 73 | } 74 | 75 | func NewOKExFutureV3Ws() *OKExFutureV3Ws { 76 | return &OKExFutureV3Ws{ 77 | ok: okex.NewOKExV3FuturesWs(nil), 78 | } 79 | } 80 | 81 | func (o *OKExFutureV3Ws) SubscribeTicker(pair goex.CurrencyPair, contract string) error { 82 | return o.ok.SubscribeTicker(pair, contract) 83 | } 84 | 85 | func (o *OKExFutureV3Ws) SubscribeDepth(pair goex.CurrencyPair, contract string, size int) error { 86 | return o.ok.SubscribeDepth(pair, contract, size) 87 | } 88 | 89 | func (o *OKExFutureV3Ws) SubscribeTrade(pair goex.CurrencyPair, contract string) error { 90 | return o.ok.SubscribeTrade(pair, contract) 91 | } 92 | 93 | func (o *OKExFutureV3Ws) SubscribeKline(pair goex.CurrencyPair, contract string, period int) error { 94 | return o.ok.SubscribeKline(pair, contract, period) 95 | } 96 | 97 | func (o *OKExFutureV3Ws) SetCallbacks( 98 | tickerCallback func(ticker *goex.FutureTicker), 99 | depthCallback func(*goex.Depth), 100 | tradeCallback func(*goex.Trade, string), 101 | klineCallback func(*goex.FutureKline, int)) { 102 | o.ok.SetCallbacks(tickerCallback, depthCallback, tradeCallback, klineCallback, nil) 103 | } 104 | -------------------------------------------------------------------------------- /ws/builder_test.go: -------------------------------------------------------------------------------- 1 | package ws 2 | 3 | import ( 4 | "github.com/goex-top/market_center" 5 | "testing" 6 | ) 7 | 8 | func TestBuilderSpot(t *testing.T) { 9 | t.Log(BuilderSpot(market_center.BINANCE)) 10 | } 11 | -------------------------------------------------------------------------------- /ws/ws.go: -------------------------------------------------------------------------------- 1 | package ws 2 | 3 | import "github.com/nntaoli-project/goex" 4 | 5 | type WS interface { 6 | SetCallbacks( 7 | tickerCallback func(ticker *goex.Ticker), 8 | depthCallback func(*goex.Depth), 9 | tradeCallback func(*goex.Trade), 10 | klineCallback func(*goex.Kline, int)) 11 | SubscribeTicker(pair goex.CurrencyPair) error 12 | SubscribeDepth(pair goex.CurrencyPair, size int) error 13 | SubscribeTrade(pair goex.CurrencyPair) error 14 | SubscribeKline(pair goex.CurrencyPair, period int) error 15 | } 16 | 17 | type FutureWS interface { 18 | SetCallbacks(tickerCallback func(ticker *goex.FutureTicker), 19 | depthCallback func(*goex.Depth), 20 | tradeCallback func(*goex.Trade, string), 21 | klineCallback func(*goex.FutureKline, int)) 22 | SubscribeTicker(pair goex.CurrencyPair, contract string) error 23 | SubscribeDepth(pair goex.CurrencyPair, contract string, size int) error 24 | SubscribeTrade(pair goex.CurrencyPair, contract string) error 25 | SubscribeKline(pair goex.CurrencyPair, contract string, period int) error 26 | } 27 | --------------------------------------------------------------------------------