├── .gitignore ├── LICENSE ├── README.md ├── go.mod ├── quote.go ├── quote └── main.go └── quote_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | *.csv 22 | *.txt 23 | *.json 24 | *.exe 25 | *.test 26 | *.prof 27 | *.log 28 | 29 | .DS_Store 30 | .vscode 31 | 32 | debug 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016-2024 Mark Chenoweth 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-quote 2 | 3 | [![GoDoc](http://godoc.org/github.com/markcheno/go-quote?status.svg)](http://godoc.org/github.com/markcheno/go-quote) 4 | 5 | A free quote downloader library and cli 6 | 7 | Downloads daily historical price quotes from Tiingo and daily/intraday data from various api's. Written in pure Go. No external dependencies. Now downloads crypto coin historical data from various exchanges. 8 | 9 | - Update: 04/01/2025 - added markets flag, wildcard input files, multiple inputs 10 | 11 | - Update: 03/02/2025 - Removed obsolete Yahoo support 12 | 13 | - Update: 02/15/2024 - Major update: updated to Go 1.22, removed bittrex/binance support, fixed nasdaq/tiingo markets 14 | 15 | - Update: 11/15/2021 - Removed obsolete markets, converted to go modules 16 | 17 | - Update: 7/18/2021 - Removed obsolete Google support 18 | 19 | - Update: 6/26/2019 - updated GDAX to Coinbase, added coinbase market 20 | 21 | - Update: 4/26/2018 - Added preliminary [tiingo](https://api.tiingo.com/) CRYPTO support. Use -source=tiingo-crypto -token= You can also set env variable TIINGO_API_TOKEN. To get symbol lists, use market: tiingo-btc, tiingo-eth or tiingo-usd 22 | 23 | - Update: 12/21/2017 - Added Amibroker format option (creates csv file with separate date and time). Use -format=ami 24 | 25 | - Update: 12/20/2017 - Added [Binance](https://www.binance.com/trade.html) exchange support. Use -source=binance 26 | 27 | - Update: 12/18/2017 - Added [Bittrex](https://bittrex.com/home/markets) exchange support. Use -source=bittrex 28 | 29 | - Update: 10/21/2017 - Added Coinbase [GDAX](https://www.gdax.com/trade/BTC-USD) exchange support. Use -source=gdax All times are in UTC. Automatically rate limited. 30 | 31 | - Update: 7/19/2017 - Added preliminary [tiingo](https://api.tiingo.com/) support. Use -source=tiingo -token= You can also set env variable TIINGO_API_TOKEN 32 | 33 | - Update: 5/24/2017 - Now works with the new Yahoo download format. Beware - Yahoo data quality is now questionable and the free Yahoo quotes are likely to permanently go away in the near future. Use with caution! 34 | 35 | Still very much in alpha mode. Expect bugs and API changes. Comments/suggestions/pull requests welcome! 36 | 37 | Copyright 2024 Mark Chenoweth 38 | 39 | Install CLI utility (quote) with: 40 | 41 | ```bash 42 | go install github.com/markcheno/go-quote/quote@latest 43 | ``` 44 | 45 | ``` 46 | Usage: 47 | quote -h | -help 48 | quote -v | -version 49 | quote [-output=] 50 | quote [-years=|(-start= [-end=])] [options] [-infile=| ...] 51 | 52 | Options: 53 | -h -help show help 54 | -v -version show version 55 | -years= number of years to download [default=5] 56 | -start= yyyy[-[mm-[dd]]] 57 | -end= yyyy[-[mm-[dd]]] [default=today] 58 | -infile= list of symbols to download 59 | -markets= list of markets to download (comma separated) 60 | -outfile= output filename 61 | -period= 1m|3m|5m|15m|30m|1h|2h|4h|6h|8h|12h|d|3d|w|m [default=d] 62 | -source= tiingo|tiingo-crypto|coinbase [default=tiingo] 63 | -token= tingo api token [default=TIINGO_API_TOKEN] 64 | -format= (csv|json|hs|ami) [default=csv] 65 | -all= all in one file (true|false) [default=false] 66 | -log= filename|stdout|stderr|discard [default=stdout] 67 | -delay= delay in milliseconds between quote requests 68 | 69 | Note: not all periods work with all sources 70 | 71 | Valid markets: 72 | etf,nasdaq,nasdaq100,amex,nyse,megacap,largecap,midcap,smallcap,microcap,nanocap, 73 | telecommunications,health_care,finance,real_estate,consumer_discretionary, 74 | consumer_staples,industrials,basic_materials,energy,utilities 75 | coinbase,tiingo-usd,tiingo-btc,tiingo-eth 76 | ``` 77 | 78 | ## CLI Examples 79 | 80 | ```bash 81 | # display usage 82 | quote -help 83 | 84 | # downloads 10 years of smallcap, midcap, largecap and megacap stocks to stocks.csv 85 | quote -markets=smallcap,midcap,largecap,megacap -all=true -years=10 -outfile=stocks.csv 86 | 87 | # downloads 10 years of spy,qqq and djia to indexes.csv 88 | quote -years=10 -outfile=indexes.csv -all=true spy qqq djia 89 | 90 | # downloads 5 years of Tiingo SPY history to spy.csv (TIINGO_API_TOKEN must be set) 91 | quote spy 92 | 93 | # downloads 1 year of bitcoin history to BTC-USD.csv 94 | quote -years=1 -source=coinbase BTC-USD 95 | 96 | 97 | # downloads full etf symbol list to etf.txt, also works for nasdaq,nasdaq100,nyse,amex 98 | quote etf 99 | 100 | # download fresh etf list and 5 years of etf data all in one file 101 | quote -markets=etf -all=true -outfile=etf.csv 102 | ``` 103 | 104 | ## Install library 105 | 106 | Install the package with: 107 | 108 | ```bash 109 | go get github.com/markcheno/go-quote@latest 110 | ``` 111 | 112 | ## Library example 113 | 114 | ```go 115 | package main 116 | 117 | import ( 118 | "fmt" 119 | "github.com/markcheno/go-quote" 120 | "github.com/markcheno/go-talib" 121 | ) 122 | 123 | func main() { 124 | spy, _ := quote.NewQuoteFromTiingo("spy", "2016-01-01", "2016-04-01", quote.Daily, true) 125 | fmt.Print(spy.CSV()) 126 | rsi2 := talib.Rsi(spy.Close, 2) 127 | fmt.Println(rsi2) 128 | } 129 | ``` 130 | 131 | ## License 132 | 133 | MIT License - see LICENSE for more details 134 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/markcheno/go-quote 2 | 3 | go 1.22 4 | -------------------------------------------------------------------------------- /quote.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package quote is free quote downloader library and cli 3 | 4 | # Downloads historical price quotes from Tiingo and Coinbase 5 | 6 | Copyright 2025 Mark Chenoweth 7 | Licensed under terms of MIT license (see LICENSE) 8 | */ 9 | package quote 10 | 11 | import ( 12 | "bufio" 13 | "bytes" 14 | "encoding/json" 15 | "fmt" 16 | "io" 17 | "log" 18 | "math/rand" 19 | "net" 20 | "net/http" 21 | "net/textproto" 22 | "net/url" 23 | "os" 24 | "sort" 25 | "strconv" 26 | "strings" 27 | "time" 28 | ) 29 | 30 | // Quote - structure for historical price data 31 | type Quote struct { 32 | Symbol string `json:"symbol"` 33 | Precision int64 `json:"-"` 34 | Date []time.Time `json:"date"` 35 | Open []float64 `json:"open"` 36 | High []float64 `json:"high"` 37 | Low []float64 `json:"low"` 38 | Close []float64 `json:"close"` 39 | Volume []float64 `json:"volume"` 40 | } 41 | 42 | // Quotes - an array of historical price data 43 | type Quotes []Quote 44 | 45 | // Period - for quote history 46 | type Period string 47 | 48 | // ClientTimeout - connect/read timeout for client requests 49 | const ClientTimeout = 10 * time.Second 50 | 51 | const ( 52 | // Min1 - 1 Minute time period 53 | Min1 Period = "60" 54 | // Min3 - 3 Minute time period 55 | Min3 Period = "3m" 56 | // Min5 - 5 Minute time period 57 | Min5 Period = "300" 58 | // Min15 - 15 Minute time period 59 | Min15 Period = "900" 60 | // Min30 - 30 Minute time period 61 | Min30 Period = "1800" 62 | // Min60 - 60 Minute time period 63 | Min60 Period = "3600" 64 | // Hour2 - 2 hour time period 65 | Hour2 Period = "2h" 66 | // Hour4 - 4 hour time period 67 | Hour4 Period = "4h" 68 | // Hour6 - 6 hour time period 69 | Hour6 Period = "6h" 70 | // Hour8 - 8 hour time period 71 | Hour8 Period = "8h" 72 | // Hour12 - 12 hour time period 73 | Hour12 Period = "12h" 74 | // Daily time period 75 | Daily Period = "d" 76 | // Day3 - 3 day time period 77 | Day3 Period = "3d" 78 | // Weekly time period 79 | Weekly Period = "w" 80 | // Monthly time period 81 | Monthly Period = "m" 82 | ) 83 | 84 | // Log - standard logger, disabled by default 85 | var Log *log.Logger 86 | 87 | // Delay - time delay in milliseconds between quote requests (default=100) 88 | // Be nice, don't get blocked 89 | var Delay time.Duration 90 | 91 | func init() { 92 | Log = log.New(io.Discard, "quote: ", log.Ldate|log.Ltime|log.Lshortfile) 93 | Delay = 100 94 | } 95 | 96 | // NewQuote - new empty Quote struct 97 | func NewQuote(symbol string, bars int) Quote { 98 | return Quote{ 99 | Symbol: symbol, 100 | Date: make([]time.Time, bars), 101 | Open: make([]float64, bars), 102 | High: make([]float64, bars), 103 | Low: make([]float64, bars), 104 | Close: make([]float64, bars), 105 | Volume: make([]float64, bars), 106 | } 107 | } 108 | 109 | // ParseDateString - parse a potentially partial date string to Time 110 | func ParseDateString(dt string) time.Time { 111 | if dt == "" { 112 | return time.Now() 113 | } 114 | t, _ := time.Parse("2006-01-02 15:04", dt+"0000-01-01 00:00"[len(dt):]) 115 | return t 116 | } 117 | 118 | func getPrecision(symbol string) int { 119 | var precision int 120 | precision = 2 121 | if strings.Contains(strings.ToUpper(symbol), "BTC") || 122 | strings.Contains(strings.ToUpper(symbol), "ETH") || 123 | strings.Contains(strings.ToUpper(symbol), "USD") { 124 | precision = 8 125 | } 126 | return precision 127 | } 128 | 129 | // CSV - convert Quote structure to csv string 130 | func (q Quote) CSV() string { 131 | 132 | precision := getPrecision(q.Symbol) 133 | 134 | var buffer bytes.Buffer 135 | buffer.WriteString("datetime,open,high,low,close,volume\n") 136 | for bar := range q.Close { 137 | str := fmt.Sprintf("%s,%.*f,%.*f,%.*f,%.*f,%.*f\n", q.Date[bar].Format("2006-01-02 15:04"), 138 | precision, q.Open[bar], precision, q.High[bar], precision, q.Low[bar], precision, q.Close[bar], precision, q.Volume[bar]) 139 | buffer.WriteString(str) 140 | } 141 | return buffer.String() 142 | } 143 | 144 | // Highstock - convert Quote structure to Highstock json format 145 | func (q Quote) Highstock() string { 146 | 147 | precision := getPrecision(q.Symbol) 148 | 149 | var buffer bytes.Buffer 150 | buffer.WriteString("[\n") 151 | for bar := range q.Close { 152 | comma := "," 153 | if bar == len(q.Close)-1 { 154 | comma = "" 155 | } 156 | str := fmt.Sprintf("[%d,%.*f,%.*f,%.*f,%.*f,%.*f]%s\n", 157 | q.Date[bar].UnixNano()/1000000, precision, q.Open[bar], precision, q.High[bar], precision, q.Low[bar], precision, q.Close[bar], precision, q.Volume[bar], comma) 158 | buffer.WriteString(str) 159 | 160 | } 161 | buffer.WriteString("]\n") 162 | return buffer.String() 163 | } 164 | 165 | // Amibroker - convert Quote structure to csv string 166 | func (q Quote) Amibroker() string { 167 | 168 | precision := getPrecision(q.Symbol) 169 | 170 | var buffer bytes.Buffer 171 | buffer.WriteString("date,time,open,high,low,close,volume\n") 172 | for bar := range q.Close { 173 | str := fmt.Sprintf("%s,%s,%.*f,%.*f,%.*f,%.*f,%.*f\n", q.Date[bar].Format("2006-01-02"), q.Date[bar].Format("15:04"), 174 | precision, q.Open[bar], precision, q.High[bar], precision, q.Low[bar], precision, q.Close[bar], precision, q.Volume[bar]) 175 | buffer.WriteString(str) 176 | } 177 | return buffer.String() 178 | } 179 | 180 | // WriteCSV - write Quote struct to csv file 181 | func (q Quote) WriteCSV(filename string) error { 182 | if filename == "" { 183 | if q.Symbol != "" { 184 | filename = q.Symbol + ".csv" 185 | } else { 186 | filename = "quote.csv" 187 | } 188 | } 189 | csv := q.CSV() 190 | return os.WriteFile(filename, []byte(csv), 0644) 191 | } 192 | 193 | // WriteAmibroker - write Quote struct to csv file 194 | func (q Quote) WriteAmibroker(filename string) error { 195 | if filename == "" { 196 | if q.Symbol != "" { 197 | filename = q.Symbol + ".csv" 198 | } else { 199 | filename = "quote.csv" 200 | } 201 | } 202 | csv := q.Amibroker() 203 | return os.WriteFile(filename, []byte(csv), 0644) 204 | } 205 | 206 | // WriteHighstock - write Quote struct to Highstock json format 207 | func (q Quote) WriteHighstock(filename string) error { 208 | if filename == "" { 209 | if q.Symbol != "" { 210 | filename = q.Symbol + ".json" 211 | } else { 212 | filename = "quote.json" 213 | } 214 | } 215 | csv := q.Highstock() 216 | return os.WriteFile(filename, []byte(csv), 0644) 217 | } 218 | 219 | // NewQuoteFromCSV - parse csv quote string into Quote structure 220 | func NewQuoteFromCSV(symbol, csv string) (Quote, error) { 221 | 222 | tmp := strings.Split(csv, "\n") 223 | numrows := len(tmp) 224 | q := NewQuote(symbol, numrows-1) 225 | 226 | for row, bar := 1, 0; row < numrows; row, bar = row+1, bar+1 { 227 | line := strings.Split(tmp[row], ",") 228 | if len(line) != 6 { 229 | break 230 | } 231 | q.Date[bar], _ = time.Parse("2006-01-02 15:04", line[0]) 232 | q.Open[bar], _ = strconv.ParseFloat(line[1], 64) 233 | q.High[bar], _ = strconv.ParseFloat(line[2], 64) 234 | q.Low[bar], _ = strconv.ParseFloat(line[3], 64) 235 | q.Close[bar], _ = strconv.ParseFloat(line[4], 64) 236 | q.Volume[bar], _ = strconv.ParseFloat(line[5], 64) 237 | } 238 | return q, nil 239 | } 240 | 241 | // NewQuoteFromCSVDateFormat - parse csv quote string into Quote structure 242 | // with specified DateTime format 243 | func NewQuoteFromCSVDateFormat(symbol, csv string, format string) (Quote, error) { 244 | 245 | tmp := strings.Split(csv, "\n") 246 | numrows := len(tmp) 247 | q := NewQuote("", numrows-1) 248 | 249 | if len(strings.TrimSpace(format)) == 0 { 250 | format = "2006-01-02 15:04" 251 | } 252 | 253 | for row, bar := 1, 0; row < numrows; row, bar = row+1, bar+1 { 254 | line := strings.Split(tmp[row], ",") 255 | q.Date[bar], _ = time.Parse(format, line[0]) 256 | q.Open[bar], _ = strconv.ParseFloat(line[1], 64) 257 | q.High[bar], _ = strconv.ParseFloat(line[2], 64) 258 | q.Low[bar], _ = strconv.ParseFloat(line[3], 64) 259 | q.Close[bar], _ = strconv.ParseFloat(line[4], 64) 260 | q.Volume[bar], _ = strconv.ParseFloat(line[5], 64) 261 | } 262 | return q, nil 263 | } 264 | 265 | // NewQuoteFromCSVFile - parse csv quote file into Quote structure 266 | func NewQuoteFromCSVFile(symbol, filename string) (Quote, error) { 267 | csv, err := os.ReadFile(filename) 268 | if err != nil { 269 | return NewQuote("", 0), err 270 | } 271 | return NewQuoteFromCSV(symbol, string(csv)) 272 | } 273 | 274 | // NewQuoteFromCSVFileDateFormat - parse csv quote file into Quote structure 275 | // with specified DateTime format 276 | func NewQuoteFromCSVFileDateFormat(symbol, filename string, format string) (Quote, error) { 277 | csv, err := os.ReadFile(filename) 278 | if err != nil { 279 | return NewQuote("", 0), err 280 | } 281 | return NewQuoteFromCSVDateFormat(symbol, string(csv), format) 282 | } 283 | 284 | // JSON - convert Quote struct to json string 285 | func (q Quote) JSON(indent bool) string { 286 | var j []byte 287 | if indent { 288 | j, _ = json.MarshalIndent(q, "", " ") 289 | } else { 290 | j, _ = json.Marshal(q) 291 | } 292 | return string(j) 293 | } 294 | 295 | // WriteJSON - write Quote struct to json file 296 | func (q Quote) WriteJSON(filename string, indent bool) error { 297 | if filename == "" { 298 | filename = q.Symbol + ".json" 299 | } 300 | json := q.JSON(indent) 301 | return os.WriteFile(filename, []byte(json), 0644) 302 | 303 | } 304 | 305 | // NewQuoteFromJSON - parse json quote string into Quote structure 306 | func NewQuoteFromJSON(jsn string) (Quote, error) { 307 | q := Quote{} 308 | err := json.Unmarshal([]byte(jsn), &q) 309 | if err != nil { 310 | return q, err 311 | } 312 | return q, nil 313 | } 314 | 315 | // NewQuoteFromJSONFile - parse json quote string into Quote structure 316 | func NewQuoteFromJSONFile(filename string) (Quote, error) { 317 | jsn, err := os.ReadFile(filename) 318 | if err != nil { 319 | return NewQuote("", 0), err 320 | } 321 | return NewQuoteFromJSON(string(jsn)) 322 | } 323 | 324 | // CSV - convert Quotes structure to csv string 325 | func (q Quotes) CSV() string { 326 | 327 | var buffer bytes.Buffer 328 | 329 | buffer.WriteString("symbol,datetime,open,high,low,close,volume\n") 330 | 331 | for sym := 0; sym < len(q); sym++ { 332 | quote := q[sym] 333 | precision := getPrecision(quote.Symbol) 334 | for bar := range quote.Close { 335 | str := fmt.Sprintf("%s,%s,%.*f,%.*f,%.*f,%.*f,%.*f\n", 336 | quote.Symbol, quote.Date[bar].Format("2006-01-02 15:04"), precision, quote.Open[bar], precision, quote.High[bar], precision, quote.Low[bar], precision, quote.Close[bar], precision, quote.Volume[bar]) 337 | buffer.WriteString(str) 338 | } 339 | } 340 | 341 | return buffer.String() 342 | } 343 | 344 | // Highstock - convert Quotes structure to Highstock json format 345 | func (q Quotes) Highstock() string { 346 | 347 | var buffer bytes.Buffer 348 | 349 | buffer.WriteString("{") 350 | 351 | for sym := 0; sym < len(q); sym++ { 352 | quote := q[sym] 353 | precision := getPrecision(quote.Symbol) 354 | for bar := range quote.Close { 355 | comma := "," 356 | if bar == len(quote.Close)-1 { 357 | comma = "" 358 | } 359 | if bar == 0 { 360 | buffer.WriteString(fmt.Sprintf("\"%s\":[\n", quote.Symbol)) 361 | } 362 | str := fmt.Sprintf("[%d,%.*f,%.*f,%.*f,%.*f,%.*f]%s\n", 363 | quote.Date[bar].UnixNano()/1000000, precision, quote.Open[bar], precision, quote.High[bar], precision, quote.Low[bar], precision, quote.Close[bar], precision, quote.Volume[bar], comma) 364 | buffer.WriteString(str) 365 | } 366 | if sym < len(q)-1 { 367 | buffer.WriteString("],\n") 368 | } else { 369 | buffer.WriteString("]\n") 370 | } 371 | } 372 | 373 | buffer.WriteString("}") 374 | 375 | return buffer.String() 376 | } 377 | 378 | // Amibroker - convert Quotes structure to csv string 379 | func (q Quotes) Amibroker() string { 380 | 381 | var buffer bytes.Buffer 382 | 383 | buffer.WriteString("symbol,date,time,open,high,low,close,volume\n") 384 | 385 | for sym := 0; sym < len(q); sym++ { 386 | quote := q[sym] 387 | precision := getPrecision(quote.Symbol) 388 | for bar := range quote.Close { 389 | str := fmt.Sprintf("%s,%s,%s,%.*f,%.*f,%.*f,%.*f,%.*f\n", 390 | quote.Symbol, quote.Date[bar].Format("2006-01-02"), quote.Date[bar].Format("15:04"), precision, quote.Open[bar], precision, quote.High[bar], precision, quote.Low[bar], precision, quote.Close[bar], precision, quote.Volume[bar]) 391 | buffer.WriteString(str) 392 | } 393 | } 394 | 395 | return buffer.String() 396 | } 397 | 398 | // WriteCSV - write Quotes structure to file 399 | func (q Quotes) WriteCSV(filename string) error { 400 | if filename == "" { 401 | filename = "quotes.csv" 402 | } 403 | csv := q.CSV() 404 | ba := []byte(csv) 405 | return os.WriteFile(filename, ba, 0644) 406 | } 407 | 408 | // WriteAmibroker - write Quotes structure to file 409 | func (q Quotes) WriteAmibroker(filename string) error { 410 | if filename == "" { 411 | filename = "quotes.csv" 412 | } 413 | csv := q.Amibroker() 414 | ba := []byte(csv) 415 | return os.WriteFile(filename, ba, 0644) 416 | } 417 | 418 | // NewQuotesFromCSV - parse csv quote string into Quotes array 419 | func NewQuotesFromCSV(csv string) (Quotes, error) { 420 | 421 | quotes := Quotes{} 422 | tmp := strings.Split(csv, "\n") 423 | numrows := len(tmp) 424 | 425 | var index = make(map[string]int) 426 | for idx := 1; idx < numrows; idx++ { 427 | sym := strings.Split(tmp[idx], ",")[0] 428 | index[sym]++ 429 | } 430 | 431 | row := 1 432 | for sym, len := range index { 433 | q := NewQuote(sym, len) 434 | for bar := 0; bar < len; bar++ { 435 | line := strings.Split(tmp[row], ",") 436 | q.Date[bar], _ = time.Parse("2006-01-02 15:04", line[1]) 437 | q.Open[bar], _ = strconv.ParseFloat(line[2], 64) 438 | q.High[bar], _ = strconv.ParseFloat(line[3], 64) 439 | q.Low[bar], _ = strconv.ParseFloat(line[4], 64) 440 | q.Close[bar], _ = strconv.ParseFloat(line[5], 64) 441 | q.Volume[bar], _ = strconv.ParseFloat(line[6], 64) 442 | row++ 443 | } 444 | quotes = append(quotes, q) 445 | } 446 | return quotes, nil 447 | } 448 | 449 | // NewQuotesFromCSVFile - parse csv quote file into Quotes array 450 | func NewQuotesFromCSVFile(filename string) (Quotes, error) { 451 | csv, err := os.ReadFile(filename) 452 | if err != nil { 453 | return Quotes{}, err 454 | } 455 | return NewQuotesFromCSV(string(csv)) 456 | } 457 | 458 | // JSON - convert Quotes struct to json string 459 | func (q Quotes) JSON(indent bool) string { 460 | var j []byte 461 | if indent { 462 | j, _ = json.MarshalIndent(q, "", " ") 463 | } else { 464 | j, _ = json.Marshal(q) 465 | } 466 | return string(j) 467 | } 468 | 469 | // WriteJSON - write Quote struct to json file 470 | func (q Quotes) WriteJSON(filename string, indent bool) error { 471 | if filename == "" { 472 | filename = "quotes.json" 473 | } 474 | jsn := q.JSON(indent) 475 | return os.WriteFile(filename, []byte(jsn), 0644) 476 | } 477 | 478 | // WriteHighstock - write Quote struct to json file in Highstock format 479 | func (q Quotes) WriteHighstock(filename string) error { 480 | if filename == "" { 481 | filename = "quotes.json" 482 | } 483 | hc := q.Highstock() 484 | return os.WriteFile(filename, []byte(hc), 0644) 485 | } 486 | 487 | // NewQuotesFromJSON - parse json quote string into Quote structure 488 | func NewQuotesFromJSON(jsn string) (Quotes, error) { 489 | quotes := Quotes{} 490 | err := json.Unmarshal([]byte(jsn), "es) 491 | if err != nil { 492 | return quotes, err 493 | } 494 | return quotes, nil 495 | } 496 | 497 | // NewQuotesFromJSONFile - parse json quote string into Quote structure 498 | func NewQuotesFromJSONFile(filename string) (Quotes, error) { 499 | jsn, err := os.ReadFile(filename) 500 | if err != nil { 501 | return Quotes{}, err 502 | } 503 | return NewQuotesFromJSON(string(jsn)) 504 | } 505 | 506 | // pickRandomUserAgent selects a random user agent from the list 507 | func pickRandomUserAgent() string { 508 | var USER_AGENTS = []string{ 509 | // Chrome 510 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36", 511 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36", 512 | "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36", 513 | 514 | // Firefox 515 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:135.0) Gecko/20100101 Firefox/135.0", 516 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 14.7; rv:135.0) Gecko/20100101 Firefox/135.0", 517 | "Mozilla/5.0 (X11; Linux i686; rv:135.0) Gecko/20100101 Firefox/135.0", 518 | 519 | // Safari 520 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.3 Safari/605.1.15", 521 | 522 | // Edge 523 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36 Edg/131.0.2903.86", 524 | } 525 | 526 | rand.NewSource(time.Now().UnixNano()) 527 | return USER_AGENTS[rand.Intn(len(USER_AGENTS))] 528 | } 529 | 530 | func tiingoDaily(symbol string, from, to time.Time, token string) (Quote, error) { 531 | 532 | type tquote struct { 533 | AdjClose float64 `json:"adjClose"` 534 | AdjHigh float64 `json:"adjHigh"` 535 | AdjLow float64 `json:"adjLow"` 536 | AdjOpen float64 `json:"adjOpen"` 537 | AdjVolume float64 `json:"adjVolume"` 538 | Close float64 `json:"close"` 539 | Date string `json:"date"` 540 | DivCash float64 `json:"divCash"` 541 | High float64 `json:"high"` 542 | Low float64 `json:"low"` 543 | Open float64 `json:"open"` 544 | SplitFactor float64 `json:"splitFactor"` 545 | Volume float64 `json:"volume"` 546 | } 547 | 548 | var tiingo []tquote 549 | 550 | url := fmt.Sprintf( 551 | "https://api.tiingo.com/tiingo/daily/%s/prices?startDate=%s&endDate=%s", 552 | strings.TrimSpace(strings.Replace(symbol, "/", "-", -1)), 553 | url.QueryEscape(from.Format("2006-1-2")), 554 | url.QueryEscape(to.Format("2006-1-2"))) 555 | 556 | client := &http.Client{Timeout: ClientTimeout} 557 | req, _ := http.NewRequest("GET", url, nil) 558 | req.Header.Set("Authorization", fmt.Sprintf("Token %s", token)) 559 | resp, err := client.Do(req) 560 | 561 | if err != nil { 562 | Log.Printf("tiingo error: %v\n", err) 563 | return NewQuote("", 0), err 564 | } 565 | defer resp.Body.Close() 566 | 567 | if resp.StatusCode == http.StatusOK { 568 | contents, _ := io.ReadAll(resp.Body) 569 | err = json.Unmarshal(contents, &tiingo) 570 | if err != nil { 571 | Log.Printf("tiingo error: %v\n", err) 572 | return NewQuote("", 0), err 573 | } 574 | } else if resp.StatusCode == http.StatusNotFound { 575 | Log.Printf("symbol '%s' not found\n", symbol) 576 | return NewQuote("", 0), err 577 | } 578 | 579 | numrows := len(tiingo) 580 | quote := NewQuote(symbol, numrows) 581 | 582 | for bar := 0; bar < numrows; bar++ { 583 | quote.Date[bar], _ = time.Parse("2006-01-02", tiingo[bar].Date[0:10]) 584 | quote.Open[bar] = tiingo[bar].AdjOpen 585 | quote.High[bar] = tiingo[bar].AdjHigh 586 | quote.Low[bar] = tiingo[bar].AdjLow 587 | quote.Close[bar] = tiingo[bar].AdjClose 588 | quote.Volume[bar] = float64(tiingo[bar].Volume) 589 | } 590 | 591 | return quote, nil 592 | } 593 | 594 | func tiingoCrypto(symbol string, from, to time.Time, period Period, token string) (Quote, error) { 595 | 596 | resampleFreq := "1day" 597 | switch period { 598 | case Min1: 599 | resampleFreq = "1min" 600 | case Min3: 601 | resampleFreq = "3min" 602 | case Min5: 603 | resampleFreq = "5min" 604 | case Min15: 605 | resampleFreq = "15min" 606 | case Min30: 607 | resampleFreq = "30min" 608 | case Min60: 609 | resampleFreq = "1hour" 610 | case Hour2: 611 | resampleFreq = "2hour" 612 | case Hour4: 613 | resampleFreq = "4hour" 614 | case Hour6: 615 | resampleFreq = "6hour" 616 | case Hour8: 617 | resampleFreq = "8hour" 618 | case Hour12: 619 | resampleFreq = "12hour" 620 | case Daily: 621 | resampleFreq = "1day" 622 | } 623 | 624 | type priceData struct { 625 | TradesDone float64 `json:"tradesDone"` 626 | Close float64 `json:"close"` 627 | VolumeNotional float64 `json:"volumeNotional"` 628 | Low float64 `json:"low"` 629 | Open float64 `json:"open"` 630 | Date string `json:"date"` // "2017-12-19T00:00:00Z" 631 | High float64 `json:"high"` 632 | Volume float64 `json:"volume"` 633 | } 634 | 635 | type cryptoData struct { 636 | Ticker string `json:"ticker"` 637 | BaseCurrency string `json:"baseCurrency"` 638 | QuoteCurrency string `json:"quoteCurrency"` 639 | PriceData []priceData `json:"priceData"` 640 | } 641 | 642 | var crypto []cryptoData 643 | 644 | url := fmt.Sprintf( 645 | "https://api.tiingo.com/tiingo/crypto/prices?tickers=%s&startDate=%s&endDate=%s&resampleFreq=%s", 646 | symbol, 647 | url.QueryEscape(from.Format("2006-1-2")), 648 | url.QueryEscape(to.Format("2006-1-2")), 649 | resampleFreq) 650 | 651 | client := &http.Client{Timeout: ClientTimeout} 652 | req, _ := http.NewRequest("GET", url, nil) 653 | req.Header.Set("Authorization", fmt.Sprintf("Token %s", token)) 654 | resp, err := client.Do(req) 655 | 656 | if err != nil { 657 | Log.Printf("symbol '%s' not found\n", symbol) 658 | return NewQuote("", 0), err 659 | } 660 | defer resp.Body.Close() 661 | 662 | contents, _ := io.ReadAll(resp.Body) 663 | err = json.Unmarshal(contents, &crypto) 664 | if err != nil { 665 | Log.Printf("tiingo crypto symbol '%s' error: %v\n", symbol, err) 666 | return NewQuote("", 0), err 667 | } 668 | if len(crypto) < 1 { 669 | Log.Printf("tiingo crypto symbol '%s' No data returned", symbol) 670 | return NewQuote("", 0), err 671 | } 672 | 673 | numrows := len(crypto[0].PriceData) 674 | quote := NewQuote(symbol, numrows) 675 | 676 | for bar := 0; bar < numrows; bar++ { 677 | quote.Date[bar], _ = time.Parse(time.RFC3339, crypto[0].PriceData[bar].Date) 678 | quote.Open[bar] = crypto[0].PriceData[bar].Open 679 | quote.High[bar] = crypto[0].PriceData[bar].High 680 | quote.Low[bar] = crypto[0].PriceData[bar].Low 681 | quote.Close[bar] = crypto[0].PriceData[bar].Close 682 | quote.Volume[bar] = float64(crypto[0].PriceData[bar].Volume) 683 | } 684 | 685 | return quote, nil 686 | } 687 | 688 | // NewQuoteFromTiingo - Tiingo daily historical prices for a symbol 689 | func NewQuoteFromTiingo(symbol, startDate, endDate string, token string) (Quote, error) { 690 | 691 | from := ParseDateString(startDate) 692 | to := ParseDateString(endDate) 693 | 694 | return tiingoDaily(symbol, from, to, token) 695 | } 696 | 697 | // NewQuoteFromTiingoCrypto - Tiingo crypto historical prices for a symbol 698 | func NewQuoteFromTiingoCrypto(symbol, startDate, endDate string, period Period, token string) (Quote, error) { 699 | 700 | from := ParseDateString(startDate) 701 | to := ParseDateString(endDate) 702 | 703 | return tiingoCrypto(symbol, from, to, period, token) 704 | } 705 | 706 | // NewQuotesFromTiingoSyms - create a list of prices from symbols in string array 707 | func NewQuotesFromTiingoSyms(symbols []string, startDate, endDate string, token string) (Quotes, error) { 708 | 709 | quotes := Quotes{} 710 | for _, symbol := range symbols { 711 | quote, err := NewQuoteFromTiingo(symbol, startDate, endDate, token) 712 | if err == nil { 713 | quotes = append(quotes, quote) 714 | } else { 715 | Log.Println("error downloading " + symbol) 716 | } 717 | time.Sleep(Delay * time.Millisecond) 718 | } 719 | return quotes, nil 720 | } 721 | 722 | // NewQuotesFromTiingoCryptoSyms - create a list of prices from symbols in string array 723 | func NewQuotesFromTiingoCryptoSyms(symbols []string, startDate, endDate string, period Period, token string) (Quotes, error) { 724 | 725 | quotes := Quotes{} 726 | for _, symbol := range symbols { 727 | quote, err := NewQuoteFromTiingoCrypto(symbol, startDate, endDate, period, token) 728 | if err == nil { 729 | quotes = append(quotes, quote) 730 | } else { 731 | Log.Println("error downloading " + symbol) 732 | } 733 | time.Sleep(Delay * time.Millisecond) 734 | } 735 | return quotes, nil 736 | } 737 | 738 | // NewQuoteFromCoinbase - Coinbase Pro historical prices for a symbol 739 | func NewQuoteFromCoinbase(symbol, startDate, endDate string, period Period) (Quote, error) { 740 | 741 | start := ParseDateString(startDate) //.In(time.Now().Location()) 742 | end := ParseDateString(endDate) //.In(time.Now().Location()) 743 | 744 | var granularity int // seconds 745 | 746 | switch period { 747 | case Min1: 748 | granularity = 60 749 | case Min5: 750 | granularity = 5 * 60 751 | case Min15: 752 | granularity = 15 * 60 753 | case Min30: 754 | granularity = 30 * 60 755 | case Min60: 756 | granularity = 60 * 60 757 | case Daily: 758 | granularity = 24 * 60 * 60 759 | case Weekly: 760 | granularity = 7 * 24 * 60 * 60 761 | default: 762 | granularity = 24 * 60 * 60 763 | } 764 | 765 | var quote Quote 766 | quote.Symbol = symbol 767 | 768 | maxBars := 200 769 | var step = time.Second * time.Duration(granularity) 770 | 771 | startBar := start 772 | endBar := startBar.Add(time.Duration(maxBars) * step) 773 | 774 | if endBar.After(end) { 775 | endBar = end 776 | } 777 | 778 | //Log.Printf("startBar=%v, endBar=%v\n", startBar, endBar) 779 | 780 | for startBar.Before(end) { 781 | 782 | url := fmt.Sprintf( 783 | "https://api.exchange.coinbase.com/products/%s/candles?start=%s&end=%s&granularity=%d", 784 | symbol, 785 | url.QueryEscape(startBar.Format(time.RFC3339)), 786 | url.QueryEscape(endBar.Format(time.RFC3339)), 787 | granularity) 788 | 789 | client := &http.Client{Timeout: ClientTimeout} 790 | req, _ := http.NewRequest("GET", url, nil) 791 | resp, err := client.Do(req) 792 | 793 | if err != nil { 794 | Log.Printf("coinbase error: %v\n", err) 795 | return NewQuote("", 0), err 796 | } 797 | defer resp.Body.Close() 798 | 799 | contents, _ := io.ReadAll(resp.Body) 800 | 801 | type cb [6]float64 802 | var bars []cb 803 | err = json.Unmarshal(contents, &bars) 804 | if err != nil { 805 | Log.Printf("coinbase error: %v\n", err) 806 | } 807 | 808 | numrows := len(bars) 809 | q := NewQuote(symbol, numrows) 810 | 811 | //Log.Printf("numrows=%d, bars=%v\n", numrows, bars) 812 | 813 | for row := 0; row < numrows; row++ { 814 | bar := numrows - 1 - row // reverse the order 815 | q.Date[bar] = time.Unix(int64(bars[row][0]), 0) 816 | q.Low[bar] = bars[row][1] 817 | q.High[bar] = bars[row][2] 818 | q.Open[bar] = bars[row][3] 819 | q.Close[bar] = bars[row][4] 820 | q.Volume[bar] = bars[row][5] 821 | } 822 | quote.Date = append(quote.Date, q.Date...) 823 | quote.Low = append(quote.Low, q.Low...) 824 | quote.High = append(quote.High, q.High...) 825 | quote.Open = append(quote.Open, q.Open...) 826 | quote.Close = append(quote.Close, q.Close...) 827 | quote.Volume = append(quote.Volume, q.Volume...) 828 | 829 | time.Sleep(time.Second) 830 | startBar = endBar.Add(step) 831 | endBar = startBar.Add(time.Duration(maxBars) * step) 832 | 833 | } 834 | 835 | return quote, nil 836 | } 837 | 838 | // NewQuotesFromCoinbase - create a list of prices from symbols in file 839 | func NewQuotesFromCoinbase(filename, startDate, endDate string, period Period) (Quotes, error) { 840 | 841 | quotes := Quotes{} 842 | inFile, err := os.Open(filename) 843 | if err != nil { 844 | return quotes, err 845 | } 846 | defer inFile.Close() 847 | scanner := bufio.NewScanner(inFile) 848 | scanner.Split(bufio.ScanLines) 849 | 850 | for scanner.Scan() { 851 | sym := scanner.Text() 852 | quote, err := NewQuoteFromCoinbase(sym, startDate, endDate, period) 853 | if err == nil { 854 | quotes = append(quotes, quote) 855 | } else { 856 | Log.Println("error downloading " + sym) 857 | } 858 | time.Sleep(Delay * time.Millisecond) 859 | } 860 | return quotes, nil 861 | } 862 | 863 | // NewQuotesFromCoinbaseSyms - create a list of prices from symbols in string array 864 | func NewQuotesFromCoinbaseSyms(symbols []string, startDate, endDate string, period Period) (Quotes, error) { 865 | 866 | quotes := Quotes{} 867 | for _, symbol := range symbols { 868 | quote, err := NewQuoteFromCoinbase(symbol, startDate, endDate, period) 869 | if err == nil { 870 | quotes = append(quotes, quote) 871 | } else { 872 | Log.Println("error downloading " + symbol) 873 | } 874 | time.Sleep(Delay * time.Millisecond) 875 | } 876 | return quotes, nil 877 | } 878 | 879 | // NewEtfList - download a list of etf symbols to an array of strings 880 | func NewEtfList() ([]string, error) { 881 | 882 | var symbols []string 883 | 884 | buf, err := getAnonFTP("ftp.nasdaqtrader.com", "21", "symboldirectory", "otherlisted.txt") 885 | if err != nil { 886 | Log.Println(err) 887 | return symbols, err 888 | } 889 | 890 | for _, line := range strings.Split(string(buf), "\n") { 891 | // ACT Symbol|Security Name|Exchange|CQS Symbol|ETF|Round Lot Size|Test Issue|NASDAQ Symbol 892 | cols := strings.Split(line, "|") 893 | if len(cols) > 5 && cols[4] == "Y" && cols[6] == "N" { 894 | symbols = append(symbols, strings.ToLower(cols[0])) 895 | } 896 | } 897 | sort.Strings(symbols) 898 | return symbols, nil 899 | } 900 | 901 | // NewEtfFile - download a list of etf symbols to a file 902 | func NewEtfFile(filename string) error { 903 | if filename == "" { 904 | filename = "etf.txt" 905 | } 906 | etfs, err := NewEtfList() 907 | if err != nil { 908 | return err 909 | } 910 | ba := []byte(strings.Join(etfs, "\n")) 911 | return os.WriteFile(filename, ba, 0644) 912 | } 913 | 914 | // ValidMarkets list of markets that can be downloaded 915 | var ValidMarkets = [...]string{ 916 | "etf", 917 | "nasdaq", 918 | "nasdaq100", 919 | "amex", 920 | "nyse", 921 | "megacap", 922 | "largecap", 923 | "midcap", 924 | "smallcap", 925 | "microcap", 926 | "nanocap", 927 | "telecommunications", 928 | "health_care", 929 | "finance", 930 | "real_estate", 931 | "consumer_discretionary", 932 | "consumer_staples", 933 | "industrials", 934 | "basic_materials", 935 | "energy", 936 | "utilities", 937 | "technology", 938 | "tiingo-btc", 939 | "tiingo-eth", 940 | "tiingo-usd", 941 | "coinbase", 942 | } 943 | 944 | // ValidMarket - validate market string 945 | func ValidMarket(market string) bool { 946 | if strings.HasPrefix(market, "tiingo") { 947 | if os.Getenv("TIINGO_API_TOKEN") == "" { 948 | fmt.Println("ERROR: Requires TIINGO_API_TOKEN to be set") 949 | return false 950 | } 951 | } 952 | for _, v := range ValidMarkets { 953 | if v == market { 954 | return true 955 | } 956 | } 957 | return false 958 | } 959 | 960 | // NewMarketList - download a list of market symbols to an array of strings 961 | func NewMarketList(market string) ([]string, error) { 962 | 963 | var symbols []string 964 | if !ValidMarket(market) { 965 | return symbols, fmt.Errorf("invalid market") 966 | } 967 | var url string 968 | switch market { 969 | case "nasdaq": 970 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true&exchange=NASDAQ" 971 | case "nasdaq100": 972 | url = "https://api.nasdaq.com/api/quote/list-type/nasdaq100" 973 | case "amex": 974 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true&exchange=AMEX" 975 | case "nyse": 976 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true&exchange=NYSE" 977 | case "megacap": 978 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true&marketcap=mega" 979 | case "largecap": 980 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true&marketcap=large" 981 | case "midcap": 982 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true&marketcap=mid" 983 | case "smallcap": 984 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true&marketcap=small" 985 | case "microcap": 986 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true&marketcap=micro" 987 | case "nanocap": 988 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true&marketcap=nano" 989 | case "telecommunications": 990 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true§or=telecommunications" 991 | case "health_care": 992 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true§or=health_care" 993 | case "finance": 994 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true§or=finance" 995 | case "real_estate": 996 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true§or=real_estate" 997 | case "consumer_discretionary": 998 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true§or=consumer_discretionary" 999 | case "consumer_staples": 1000 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true§or=consumer_staples" 1001 | case "industrials": 1002 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true§or=industrials" 1003 | case "basic_materials": 1004 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true§or=basic_materials" 1005 | case "energy": 1006 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true§or=energy" 1007 | case "utilities": 1008 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true§or=utilities" 1009 | case "technology": 1010 | url = "https://api.nasdaq.com/api/screener/stocks?tableonly=true&offset=0&download=true§or=technology" 1011 | case "tiingo-btc": 1012 | url = fmt.Sprintf("https://api.tiingo.com/tiingo/crypto?token=%s", os.Getenv("TIINGO_API_TOKEN")) 1013 | case "tiingo-eth": 1014 | url = fmt.Sprintf("https://api.tiingo.com/tiingo/crypto?token=%s", os.Getenv("TIINGO_API_TOKEN")) 1015 | case "tiingo-usd": 1016 | url = fmt.Sprintf("https://api.tiingo.com/tiingo/crypto?token=%s", os.Getenv("TIINGO_API_TOKEN")) 1017 | case "coinbase": 1018 | url = "https://api.exchange.coinbase.com/products" 1019 | } 1020 | 1021 | req, _ := http.NewRequest("GET", url, nil) 1022 | req.Header.Add("User-Agent", "markcheno/go-quote") 1023 | req.Header.Add("Accept", "application/json") 1024 | req.Header.Add("Content-Type", "application/json; charset=utf-8") 1025 | client := &http.Client{} 1026 | resp, err := client.Do(req) 1027 | if err != nil { 1028 | return symbols, err 1029 | } 1030 | defer resp.Body.Close() 1031 | 1032 | buf := new(bytes.Buffer) 1033 | buf.ReadFrom(resp.Body) 1034 | newStr := buf.String() 1035 | 1036 | if strings.HasPrefix(market, "tiingo") { 1037 | return getTiingoCryptoMarket(market, newStr) 1038 | } 1039 | 1040 | if strings.HasPrefix(market, "coinbase") { 1041 | return getCoinbaseMarket(market, newStr) 1042 | } 1043 | 1044 | if market == "nasdaq100" { 1045 | return getNasdaq100Market(market, newStr) 1046 | } 1047 | 1048 | return getNasdaqMarket(market, newStr) 1049 | 1050 | } 1051 | 1052 | func getTiingoCryptoMarket(market, rawdata string) ([]string, error) { 1053 | 1054 | type Symbol struct { 1055 | Ticker string `json:"ticker"` 1056 | Name string `json:"name"` 1057 | BaseCurrency string `json:"baseCurrency"` 1058 | QuoteCurrency string `json:"quoteCurrency"` 1059 | } 1060 | 1061 | var markets []Symbol 1062 | 1063 | err := json.Unmarshal([]byte(rawdata), &markets) 1064 | if err != nil { 1065 | fmt.Println(err) 1066 | } 1067 | 1068 | var symbols []string 1069 | for _, mkt := range markets { 1070 | if strings.HasSuffix(market, "btc") && mkt.QuoteCurrency == "btc" { 1071 | symbols = append(symbols, mkt.Ticker) 1072 | } else if strings.HasSuffix(market, "eth") && mkt.QuoteCurrency == "eth" { 1073 | symbols = append(symbols, mkt.Ticker) 1074 | } else if strings.HasSuffix(market, "usd") && mkt.QuoteCurrency == "usd" { 1075 | symbols = append(symbols, mkt.Ticker) 1076 | } 1077 | } 1078 | 1079 | return symbols, err 1080 | } 1081 | 1082 | func getNasdaqMarket(market, rawdata string) ([]string, error) { 1083 | 1084 | // https://www.nasdaq.com/market-activity/stocks/screener 1085 | 1086 | type Headers struct { 1087 | Symbol string `json:"symbol"` 1088 | Name string `json:"name"` 1089 | LastSale string `json:"lastsale"` 1090 | NetChange string `json:"netchange"` 1091 | PctChange string `json:"pctchange"` 1092 | MarketCap string `json:"marketCap"` 1093 | } 1094 | 1095 | type Row struct { 1096 | Symbol string `json:"symbol"` 1097 | Name string `json:"name"` 1098 | LastSale string `json:"lastsale"` 1099 | NetChange string `json:"netchange"` 1100 | PctChange string `json:"pctchange"` 1101 | MarketCap string `json:"marketCap"` 1102 | URL string `json:"url"` 1103 | } 1104 | 1105 | type Table struct { 1106 | AsOf *string `json:"asOf"` 1107 | Headers Headers `json:"headers"` 1108 | Rows []Row `json:"rows"` 1109 | } 1110 | 1111 | type Status struct { 1112 | RCode int `json:"rCode"` 1113 | BCodeMessage *string `json:"bCodeMessage"` 1114 | DeveloperMessage *string `json:"developerMessage"` 1115 | } 1116 | 1117 | type ApiResponse struct { 1118 | Data Table `json:"data"` 1119 | Message *string `json:"message"` 1120 | Status Status `json:"status"` 1121 | } 1122 | 1123 | // Unmarshal the JSON into our structs 1124 | var apiResponse ApiResponse 1125 | err := json.Unmarshal([]byte(rawdata), &apiResponse) 1126 | if err != nil { 1127 | log.Fatalf("Error parsing JSON: %v", err) 1128 | } 1129 | 1130 | var symbols []string 1131 | for _, row := range apiResponse.Data.Rows { 1132 | symbols = append(symbols, strings.ToLower(row.Symbol)) 1133 | //fmt.Printf("Symbol: %s\n", row.Symbol) 1134 | } 1135 | 1136 | sort.Strings(symbols) 1137 | 1138 | return symbols, err 1139 | } 1140 | 1141 | func getNasdaq100Market(market, rawdata string) ([]string, error) { 1142 | 1143 | // https://api.nasdaq.com/api/quote/list-type/nasdaq100 1144 | 1145 | type Headers struct { 1146 | Symbol string `json:"symbol"` 1147 | Name string `json:"companyName"` 1148 | MarketCap string `json:"marketCap"` 1149 | LastSale string `json:"lastSalePrice"` 1150 | NetChange string `json:"netChange"` 1151 | PctChange string `json:"percentageChange"` 1152 | } 1153 | 1154 | type Row struct { 1155 | Symbol string `json:"symbol"` 1156 | Sector string `json:"sector"` 1157 | Name string `json:"companyName"` 1158 | MarketCap string `json:"marketCap"` 1159 | LastSalePrice string `json:"lastSalePrice"` 1160 | NetChange string `json:"netChange"` 1161 | PctChange string `json:"percentageChange"` 1162 | Delta string `json:"deltaIndicator"` 1163 | } 1164 | 1165 | type Table struct { 1166 | AsOf *string `json:"asOf"` 1167 | Headers Headers `json:"headers"` 1168 | Rows []Row `json:"rows"` 1169 | } 1170 | 1171 | type Data struct { 1172 | TotalRecords int `json:"totalrecords"` 1173 | Limit int `json:"limit"` 1174 | Offset int `json:"offset"` 1175 | Date string `json:"date"` 1176 | Data Table `json:"data"` 1177 | } 1178 | 1179 | type Status struct { 1180 | RCode int `json:"rCode"` 1181 | BCodeMessage *string `json:"bCodeMessage"` 1182 | DeveloperMessage *string `json:"developerMessage"` 1183 | } 1184 | 1185 | type ApiResponse struct { 1186 | Data Data `json:"data"` 1187 | Message *string `json:"message"` 1188 | Status Status `json:"status"` 1189 | } 1190 | 1191 | // Unmarshal the JSON into our structs 1192 | var apiResponse ApiResponse 1193 | err := json.Unmarshal([]byte(rawdata), &apiResponse) 1194 | if err != nil { 1195 | log.Fatalf("Error parsing JSON: %v", err) 1196 | } 1197 | 1198 | var symbols []string 1199 | for _, row := range apiResponse.Data.Data.Rows { 1200 | symbols = append(symbols, strings.ToLower(row.Symbol)) 1201 | //fmt.Printf("Symbol: %s\n", row.Symbol) 1202 | } 1203 | 1204 | sort.Strings(symbols) 1205 | 1206 | return symbols, err 1207 | } 1208 | 1209 | func getCoinbaseMarket(market, rawdata string) ([]string, error) { 1210 | 1211 | type Symbol struct { 1212 | ID string `json:"id"` 1213 | BaseCurrency string `json:"base_currency"` 1214 | QuoteCurrency string `json:"quote_currency"` 1215 | QuoteIncrement string `json:"quote_increment"` 1216 | BaseIncrement string `json:"base_increment"` 1217 | DisplayName string `json:"display_name"` 1218 | MinMarketFunds string `json:"min_market_funds"` 1219 | MarginEnabled bool `json:"margin_enabled"` 1220 | PostOnly bool `json:"post_only"` 1221 | LimitOnly bool `json:"limit_only"` 1222 | CancelOnly bool `json:"cancel_only"` 1223 | Status string `json:"status"` 1224 | StatusMessage string `json:"status_message"` 1225 | TradingDisabled bool `json:"trading_disabled"` 1226 | FxStablecoin bool `json:"fx_stablecoin"` 1227 | MaxSlippagePercentage string `json:"max_slippage_percentage"` 1228 | AuctionMode bool `json:"auction_mode"` 1229 | HighBidLimitPercentage string `json:"high_bid_limit_percentage"` 1230 | } 1231 | 1232 | var markets []Symbol 1233 | 1234 | err := json.Unmarshal([]byte(rawdata), &markets) 1235 | if err != nil { 1236 | fmt.Println(err) 1237 | } 1238 | 1239 | var symbols []string 1240 | for _, mkt := range markets { 1241 | if !mkt.TradingDisabled { 1242 | symbols = append(symbols, mkt.ID) 1243 | } 1244 | } 1245 | 1246 | sort.Strings(symbols) 1247 | 1248 | return symbols, err 1249 | } 1250 | 1251 | // NewMarketFile - download a list of market symbols to a file 1252 | func NewMarketFile(market, filename string) error { 1253 | if !ValidMarket(market) { 1254 | return fmt.Errorf("invalid market") 1255 | } 1256 | // default filename 1257 | if filename == "" { 1258 | filename = market + ".txt" 1259 | } 1260 | syms, err := NewMarketList(market) 1261 | if err != nil { 1262 | return err 1263 | } 1264 | 1265 | // Trim whitespace from each symbol 1266 | for i := range syms { 1267 | syms[i] = strings.TrimSpace(syms[i]) 1268 | } 1269 | 1270 | ba := []byte(strings.Join(syms, "\n")) 1271 | return os.WriteFile(filename, ba, 0644) 1272 | } 1273 | 1274 | // NewSymbolsFromFile - read symbols from a file 1275 | func NewSymbolsFromFile(filename string) ([]string, error) { 1276 | raw, err := os.ReadFile(filename) 1277 | if err != nil { 1278 | return []string{}, err 1279 | } 1280 | 1281 | a := strings.Split(strings.ToLower(string(raw)), "\n") 1282 | 1283 | return deleteEmpty(a), nil 1284 | } 1285 | 1286 | // delete empty strings from a string array 1287 | func deleteEmpty(s []string) []string { 1288 | var r []string 1289 | for _, str := range s { 1290 | if str != "" { 1291 | r = append(r, str) 1292 | } 1293 | } 1294 | return r 1295 | } 1296 | 1297 | // Grab a file via anonymous FTP 1298 | func getAnonFTP(addr, port string, dir string, fname string) ([]byte, error) { 1299 | 1300 | var err error 1301 | var contents []byte 1302 | const timeout = 5 * time.Second 1303 | 1304 | nconn, err := net.DialTimeout("tcp", addr+":"+port, timeout) 1305 | if err != nil { 1306 | return contents, err 1307 | } 1308 | defer nconn.Close() 1309 | 1310 | conn := textproto.NewConn(nconn) 1311 | _, _, _ = conn.ReadResponse(2) 1312 | defer conn.Close() 1313 | 1314 | _ = conn.PrintfLine("USER anonymous") 1315 | _, _, _ = conn.ReadResponse(0) 1316 | 1317 | _ = conn.PrintfLine("PASS anonymous") 1318 | _, _, _ = conn.ReadResponse(230) 1319 | 1320 | _ = conn.PrintfLine("CWD %s", dir) 1321 | _, _, _ = conn.ReadResponse(250) 1322 | 1323 | _ = conn.PrintfLine("PASV") 1324 | _, message, _ := conn.ReadResponse(1) 1325 | 1326 | // PASV response format : 227 Entering Passive Mode (h1,h2,h3,h4,p1,p2). 1327 | start, end := strings.Index(message, "("), strings.Index(message, ")") 1328 | s := strings.Split(message[start:end], ",") 1329 | l1, _ := strconv.Atoi(s[len(s)-2]) 1330 | l2, _ := strconv.Atoi(s[len(s)-1]) 1331 | dport := l1*256 + l2 1332 | 1333 | _ = conn.PrintfLine("RETR %s", fname) 1334 | _, _, _ = conn.ReadResponse(1) 1335 | dconn, err := net.DialTimeout("tcp", addr+":"+strconv.Itoa(dport), timeout) 1336 | if err == nil { 1337 | defer dconn.Close() 1338 | } 1339 | 1340 | contents, err = io.ReadAll(dconn) 1341 | if err != nil { 1342 | return contents, err 1343 | } 1344 | 1345 | _ = dconn.Close() 1346 | _, _, _ = conn.ReadResponse(2) 1347 | 1348 | return contents, nil 1349 | } 1350 | -------------------------------------------------------------------------------- /quote/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package quote is free quote downloader library and cli 3 | 4 | Downloads historical price quotes from Tiingo and Coinbase 5 | 6 | Copyright 2025 Mark Chenoweth 7 | Licensed under terms of MIT license 8 | 9 | */ 10 | 11 | package main 12 | 13 | import ( 14 | "flag" 15 | "fmt" 16 | "io" 17 | "os" 18 | "path/filepath" 19 | "strings" 20 | "time" 21 | 22 | "github.com/markcheno/go-quote" 23 | ) 24 | 25 | var usage = `Usage: 26 | quote -h | -help 27 | quote -v | -version 28 | quote [-output=] 29 | quote [-years=|(-start= [-end=])] [options] [-infile=| ...] 30 | 31 | Options: 32 | -h -help show help 33 | -v -version show version 34 | -years= number of years to download [default=5] 35 | -start= yyyy[-[mm-[dd]]] 36 | -end= yyyy[-[mm-[dd]]] [default=today] 37 | -markets= list of valid markets to download (comma separated) 38 | -infile= list of symbols to download 39 | -outfile= output filename 40 | -period= 1m|3m|5m|15m|30m|1h|2h|4h|6h|8h|12h|d|3d|w|m [default=d] 41 | -source= tiingo|tiingo-crypto|coinbase [default=tiingo] 42 | -token= tingo api token [default=TIINGO_API_TOKEN] 43 | -format= (csv|json|hs|ami) [default=csv] 44 | -all= all in one file (true|false) [default=false] 45 | -log= filename|stdout|stderr|discard [default=stdout] 46 | -delay= delay in milliseconds between quote requests 47 | 48 | Note: not all periods work with all sources 49 | 50 | Valid markets: 51 | etf,nasdaq,nasdaq100,amex,nyse,megacap,largecap,midcap,smallcap,microcap,nanocap, 52 | telecommunications,health_care,finance,real_estate,consumer_discretionary, 53 | consumer_staples,industrials,basic_materials,energy,utilities,technology 54 | coinbase,tiingo-usd,tiingo-btc,tiingo-eth 55 | ` 56 | 57 | const ( 58 | version = "0.4" 59 | dateFormat = "2006-01-02" 60 | ) 61 | 62 | type quoteflags struct { 63 | years int 64 | delay int 65 | start string 66 | end string 67 | period string 68 | source string 69 | token string 70 | markets string 71 | infile string 72 | outfile string 73 | format string 74 | log string 75 | all bool 76 | version bool 77 | } 78 | 79 | func check(e error) { 80 | if e != nil { 81 | fmt.Printf("\nerror: %v\n\n", e) 82 | fmt.Println(usage) 83 | os.Exit(0) 84 | //panic(e) 85 | } 86 | } 87 | 88 | func checkFlags(flags quoteflags) error { 89 | 90 | // validate source 91 | if flags.source != "tiingo" && 92 | flags.source != "tiingo-crypto" && 93 | flags.source != "coinbase" { 94 | return fmt.Errorf("invalid source, must be either 'tiingo', 'tiingo-crypto', or 'coinbase'") 95 | } 96 | 97 | // validate period 98 | if flags.source == "tiingo" { 99 | // check period 100 | if flags.period != "d" { 101 | return fmt.Errorf("invalid period for tiingo, must be 'd'") 102 | } 103 | // check token 104 | if flags.token == "" { 105 | return fmt.Errorf("missing token for tiingo, must be passed or TIINGO_API_TOKEN must be set") 106 | } 107 | } 108 | 109 | if flags.source == "tiingo-crypto" && 110 | !(flags.period == "1m" || 111 | flags.period == "3m" || 112 | flags.period == "5m" || 113 | flags.period == "15m" || 114 | flags.period == "30m" || 115 | flags.period == "1h" || 116 | flags.period == "2h" || 117 | flags.period == "4h" || 118 | flags.period == "6h" || 119 | flags.period == "8h" || 120 | flags.period == "12h" || 121 | flags.period == "d") { 122 | return fmt.Errorf("invalid source for tiingo-crypto, must be '1m', '3m', '5m', '15m', '30m', '1h', '2h', '4h', '6h', '8h', '12h', '1d', '3d', '1w', or '1M'") 123 | } 124 | 125 | if flags.source == "tiingo-crypto" && flags.token == "" { 126 | return fmt.Errorf("missing token for tiingo-crypto, must be passed or TIINGO_API_TOKEN must be set") 127 | } 128 | 129 | return nil 130 | } 131 | 132 | func setOutput(flags quoteflags) error { 133 | var err error 134 | if flags.log == "stdout" { 135 | quote.Log.SetOutput(os.Stdout) 136 | } else if flags.log == "stderr" { 137 | quote.Log.SetOutput(os.Stderr) 138 | } else if flags.log == "discard" { 139 | quote.Log.SetOutput(io.Discard) 140 | } else { 141 | var f *os.File 142 | f, err = os.OpenFile(flags.log, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) 143 | if err != nil { 144 | quote.Log.Println(err) 145 | } 146 | defer f.Close() 147 | quote.Log.SetOutput(f) 148 | } 149 | return err 150 | } 151 | 152 | // func getSymbols(flags quoteflags, args []string) ([]string, error) { 153 | // var err error 154 | // var symbols []string 155 | // if flags.infile != "" { 156 | // symbols, err = quote.NewSymbolsFromFile(flags.infile) 157 | // if err != nil { 158 | // return symbols, err 159 | // } 160 | // } else { 161 | // symbols = args 162 | // } 163 | // // make sure we found some symbols 164 | // if len(symbols) == 0 { 165 | // return symbols, fmt.Errorf("no symbols specified") 166 | // } 167 | // // validate outfileFlag 168 | // if len(symbols) > 1 && flags.outfile != "" && !flags.all { 169 | // return symbols, fmt.Errorf("outfile not valid with multiple symbols\nuse -all=true") 170 | // } 171 | // return symbols, nil 172 | // } 173 | 174 | func getSymbols(flags quoteflags, args []string) ([]string, error) { 175 | var err error 176 | var symbols []string 177 | 178 | if flags.infile != "" { 179 | // Check if infile contains wildcard characters 180 | if strings.Contains(flags.infile, "*") || strings.Contains(flags.infile, "?") { 181 | // Find all matching files 182 | matches, err := filepath.Glob(flags.infile) 183 | if err != nil { 184 | return nil, fmt.Errorf("error processing wildcard pattern: %v", err) 185 | } 186 | 187 | if len(matches) == 0 { 188 | return nil, fmt.Errorf("no files match pattern: %s", flags.infile) 189 | } 190 | 191 | // Read symbols from all matching files 192 | for _, file := range matches { 193 | fileSymbols, err := quote.NewSymbolsFromFile(file) 194 | if err != nil { 195 | return nil, fmt.Errorf("error reading symbols from %s: %v", file, err) 196 | } 197 | symbols = append(symbols, fileSymbols...) 198 | } 199 | } else { 200 | // Regular file handling 201 | symbols, err = quote.NewSymbolsFromFile(flags.infile) 202 | if err != nil { 203 | return symbols, err 204 | } 205 | } 206 | } else if flags.markets != "" { 207 | 208 | markets := strings.Split(flags.markets, ",") 209 | for _, cmd := range markets { 210 | if !quote.ValidMarket(cmd) { 211 | return symbols, fmt.Errorf("invalid market specified: " + cmd) 212 | } 213 | file := cmd + ".csv" 214 | switch cmd { 215 | case "etf": 216 | quote.NewEtfFile(file) 217 | 218 | default: 219 | quote.NewMarketFile(cmd, file) 220 | } 221 | fileSymbols, err := quote.NewSymbolsFromFile(file) 222 | if err != nil { 223 | return nil, fmt.Errorf("error reading symbols from %s: %v", file, err) 224 | } 225 | symbols = append(symbols, fileSymbols...) 226 | } 227 | return symbols, nil 228 | } else { 229 | symbols = args 230 | } 231 | 232 | // make sure we found some symbols 233 | if len(symbols) == 0 { 234 | return symbols, fmt.Errorf("no symbols specified") 235 | } 236 | 237 | // validate outfileFlag 238 | if len(symbols) > 1 && flags.outfile != "" && !flags.all { 239 | return symbols, fmt.Errorf("outfile not valid with multiple symbols\nuse -all=true") 240 | } 241 | 242 | return symbols, nil 243 | } 244 | 245 | func getPeriod(periodFlag string) quote.Period { 246 | period := quote.Daily 247 | switch periodFlag { 248 | case "1m": 249 | period = quote.Min1 250 | case "3m": 251 | period = quote.Min3 252 | case "5m": 253 | period = quote.Min5 254 | case "15m": 255 | period = quote.Min15 256 | case "30m": 257 | period = quote.Min30 258 | case "1h": 259 | period = quote.Min60 260 | case "2h": 261 | period = quote.Hour2 262 | case "4h": 263 | period = quote.Hour4 264 | case "6h": 265 | period = quote.Hour6 266 | case "8h": 267 | period = quote.Hour8 268 | case "12h": 269 | period = quote.Hour12 270 | case "d": 271 | period = quote.Daily 272 | case "1d": 273 | period = quote.Daily 274 | case "3d": 275 | period = quote.Day3 276 | case "w": 277 | period = quote.Weekly 278 | case "1w": 279 | period = quote.Weekly 280 | case "m": 281 | period = quote.Monthly 282 | case "1M": 283 | period = quote.Monthly 284 | } 285 | return period 286 | } 287 | 288 | func getTimes(flags quoteflags) (time.Time, time.Time) { 289 | // determine start/end times 290 | to := quote.ParseDateString(flags.end) 291 | var from time.Time 292 | if flags.start != "" { 293 | from = quote.ParseDateString(flags.start) 294 | } else { // use years 295 | from = to.Add(-time.Duration(int(time.Hour) * 24 * 365 * flags.years)) 296 | } 297 | return from, to 298 | } 299 | 300 | func outputAll(symbols []string, flags quoteflags) error { 301 | // output all in one file 302 | from, to := getTimes(flags) 303 | period := getPeriod(flags.period) 304 | quotes := quote.Quotes{} 305 | var err error 306 | if flags.source == "tiingo" { 307 | quotes, err = quote.NewQuotesFromTiingoSyms(symbols, from.Format(dateFormat), to.Format(dateFormat), flags.token) 308 | } else if flags.source == "tiingo-crypto" { 309 | quotes, err = quote.NewQuotesFromTiingoCryptoSyms(symbols, from.Format(dateFormat), to.Format(dateFormat), period, flags.token) 310 | } else if flags.source == "coinbase" { 311 | quotes, err = quote.NewQuotesFromCoinbaseSyms(symbols, from.Format(dateFormat), to.Format(dateFormat), period) 312 | } 313 | if err != nil { 314 | return err 315 | } 316 | 317 | if flags.format == "csv" { 318 | err = quotes.WriteCSV(flags.outfile) 319 | } else if flags.format == "json" { 320 | err = quotes.WriteJSON(flags.outfile, false) 321 | } else if flags.format == "hs" { 322 | err = quotes.WriteHighstock(flags.outfile) 323 | } else if flags.format == "ami" { 324 | err = quotes.WriteAmibroker(flags.outfile) 325 | } 326 | return err 327 | } 328 | 329 | func outputIndividual(symbols []string, flags quoteflags) error { 330 | // output individual symbol files 331 | 332 | from, to := getTimes(flags) 333 | period := getPeriod(flags.period) 334 | 335 | for _, sym := range symbols { 336 | var q quote.Quote 337 | if flags.source == "tiingo" { 338 | q, _ = quote.NewQuoteFromTiingo(sym, from.Format(dateFormat), to.Format(dateFormat), flags.token) 339 | } else if flags.source == "tiingo-crypto" { 340 | q, _ = quote.NewQuoteFromTiingoCrypto(sym, from.Format(dateFormat), to.Format(dateFormat), period, flags.token) 341 | } else if flags.source == "coinbase" { 342 | q, _ = quote.NewQuoteFromCoinbase(sym, from.Format(dateFormat), to.Format(dateFormat), period) 343 | } 344 | var err error 345 | if flags.format == "csv" { 346 | err = q.WriteCSV(flags.outfile) 347 | } else if flags.format == "json" { 348 | err = q.WriteJSON(flags.outfile, false) 349 | } else if flags.format == "hs" { 350 | err = q.WriteHighstock(flags.outfile) 351 | } else if flags.format == "ami" { 352 | err = q.WriteAmibroker(flags.outfile) 353 | } 354 | if err != nil { 355 | fmt.Printf("Error writing file: %v\n", err) 356 | } 357 | time.Sleep(quote.Delay * time.Millisecond) 358 | } 359 | return nil 360 | } 361 | 362 | func handleCommand(symbols []string, flags quoteflags) bool { 363 | 364 | if flags.markets != "" { 365 | return false 366 | } 367 | 368 | // handle market special commands 369 | for _, cmd := range symbols { 370 | if !quote.ValidMarket(cmd) { 371 | return false 372 | } 373 | switch cmd { 374 | case "etf": 375 | quote.NewEtfFile(flags.outfile) 376 | default: 377 | quote.NewMarketFile(cmd, flags.outfile) 378 | } 379 | } 380 | return true 381 | } 382 | 383 | func main() { 384 | 385 | var err error 386 | var symbols []string 387 | var flags quoteflags 388 | 389 | flag.IntVar(&flags.years, "years", 5, "number of years to download") 390 | flag.IntVar(&flags.delay, "delay", 100, "milliseconds to delay between requests") 391 | flag.StringVar(&flags.start, "start", "", "start date (yyyy[-mm[-dd]])") 392 | flag.StringVar(&flags.end, "end", "", "end date (yyyy[-mm[-dd]])") 393 | flag.StringVar(&flags.period, "period", "d", "1m|5m|15m|30m|1h|d") 394 | flag.StringVar(&flags.source, "source", "tiingo", "tiingo|tiingo-crypto|coinbase") 395 | flag.StringVar(&flags.token, "token", os.Getenv("TIINGO_API_TOKEN"), "tiingo api token") 396 | flag.StringVar(&flags.infile, "infile", "", "input filename") 397 | flag.StringVar(&flags.outfile, "outfile", "", "output filename") 398 | flag.StringVar(&flags.markets, "markets", "", "list of valid markets (comma separated)") 399 | flag.StringVar(&flags.format, "format", "csv", "csv|json") 400 | flag.StringVar(&flags.log, "log", "stdout", "|stdout") 401 | flag.BoolVar(&flags.all, "all", false, "all output in one file") 402 | flag.BoolVar(&flags.version, "v", false, "show version") 403 | flag.BoolVar(&flags.version, "version", false, "show version") 404 | flag.Parse() 405 | 406 | if flags.version { 407 | fmt.Println(version) 408 | os.Exit(0) 409 | } 410 | 411 | quote.Delay = time.Duration(flags.delay) 412 | 413 | err = setOutput(flags) 414 | check(err) 415 | 416 | err = checkFlags(flags) 417 | check(err) 418 | 419 | symbols, err = getSymbols(flags, flag.Args()) 420 | check(err) 421 | 422 | // check for and handle special commands 423 | if handleCommand(symbols, flags) { 424 | os.Exit(0) 425 | } 426 | 427 | //fmt.Println("Downloading quotes for", len(symbols), "symbols") 428 | 429 | // main output 430 | if flags.all { 431 | outputAll(symbols, flags) 432 | } else { 433 | outputIndividual(symbols, flags) 434 | } 435 | } 436 | -------------------------------------------------------------------------------- /quote_test.go: -------------------------------------------------------------------------------- 1 | package quote 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "reflect" 7 | "runtime" 8 | "testing" 9 | ) 10 | 11 | // assert fails the test if the condition is false. 12 | func assert(t *testing.T, condition bool, msg string, v ...interface{}) { 13 | if !condition { 14 | _, file, line, _ := runtime.Caller(1) 15 | fmt.Printf("%s:%d: "+msg+"\n", append([]interface{}{filepath.Base(file), line}, v...)...) 16 | t.FailNow() 17 | } 18 | } 19 | 20 | // ok fails the test if an err is not nil. 21 | func ok(t *testing.T, err error) { 22 | if err != nil { 23 | _, file, line, _ := runtime.Caller(1) 24 | fmt.Printf("%s:%d: unexpected error: %s\n", filepath.Base(file), line, err.Error()) 25 | t.FailNow() 26 | } 27 | } 28 | 29 | // equals fails the test if exp is not equal to act. 30 | func equals(t *testing.T, exp, act interface{}) { 31 | if !reflect.DeepEqual(exp, act) { 32 | _, file, line, _ := runtime.Caller(1) 33 | fmt.Printf("%s:%d:\n\texp: %#v\n\tgot: %#v\n", filepath.Base(file), line, exp, act) 34 | t.FailNow() 35 | } 36 | } 37 | 38 | // TODO - everything 39 | 40 | func TestNewQuoteFromCSV(t *testing.T) { 41 | symbol := "aapl" 42 | csv := `datetime,open,high,low,close,volume 43 | 2014-07-14 00:00,95.86,96.89,95.65,88.40,42810000.00 44 | 2014-07-15 00:00,96.80,96.85,95.03,87.36,45477900.00 45 | 2014-07-16 00:00,96.97,97.10,94.74,86.87,53396300.00 46 | 2014-07-17 00:00,95.03,95.28,92.57,85.32,57298000.00 47 | 2014-07-18 00:00,93.62,94.74,93.02,86.55,49988000.00 48 | 2014-07-21 00:00,94.99,95.00,93.72,86.10,39079000.00 49 | 2014-07-22 00:00,94.68,94.89,94.12,86.81,55197000.00 50 | 2014-07-23 00:00,95.42,97.88,95.17,89.08,92918000.00 51 | 2014-07-24 00:00,97.04,97.32,96.42,88.93,45729000.00 52 | 2014-07-25 00:00,96.85,97.84,96.64,89.52,43469000.00 53 | 2014-07-28 00:00,97.82,99.24,97.55,90.75,55318000.00 54 | 2014-07-29 00:00,99.33,99.44,98.25,90.17,43143000.00 55 | 2014-07-30 00:00,98.44,98.70,97.67,89.96,33010000.00 56 | 2014-07-31 00:00,97.16,97.45,95.33,87.62,56843000.00` 57 | q, _ := NewQuoteFromCSV(symbol, csv) 58 | //fmt.Println(q) 59 | if len(q.Close) != 14 { 60 | t.Error("Invalid length") 61 | } 62 | if q.Close[len(q.Close)-1] != 87.62 { 63 | t.Error("Invalid last value") 64 | } 65 | } 66 | 67 | func TestNewQuotesFromCSV(t *testing.T) { 68 | csv := `symbol,datetime,open,high,low,close,volume 69 | spy,2018-07-12 00:00,278.28,279.43,277.60,273.95,60124700.00 70 | spy,2018-07-13 00:00,279.17,279.93,278.66,274.17,48216000.00 71 | spy,2018-07-16 00:00,279.64,279.80,278.84,273.92,48201000.00 72 | spy,2018-07-17 00:00,278.47,280.91,278.41,275.03,52315500.00 73 | spy,2018-07-18 00:00,280.56,281.18,280.06,275.61,44593500.00 74 | spy,2018-07-19 00:00,280.31,280.74,279.46,274.57,61412100.00 75 | spy,2018-07-20 00:00,279.77,280.48,279.50,274.26,82337700.00 76 | aapl,2018-07-12 00:00,189.53,191.41,189.31,188.17,18041100.00 77 | aapl,2018-07-13 00:00,191.08,191.84,190.90,188.46,12513900.00 78 | aapl,2018-07-16 00:00,191.52,192.65,190.42,188.05,15043100.00 79 | aapl,2018-07-17 00:00,189.75,191.87,189.20,188.58,15534500.00 80 | aapl,2018-07-18 00:00,191.78,191.80,189.93,187.55,16393400.00 81 | aapl,2018-07-19 00:00,189.69,192.55,189.69,189.00,20286800.00 82 | aapl,2018-07-20 00:00,191.78,192.43,190.17,188.57,20676200.00` 83 | q, _ := NewQuotesFromCSV(csv) 84 | //fmt.Println(q) 85 | if len(q) != 2 { 86 | t.Error("Invalid length") 87 | } 88 | if q[0].Symbol != "spy" { 89 | t.Error("Invalid symbol") 90 | } 91 | if q[0].Close[len(q[0].Close)-1] != 274.26 { 92 | t.Error("Invalid last value") 93 | } 94 | if q[1].Symbol != "aapl" { 95 | t.Error("Invalid symbol") 96 | } 97 | if q[1].Close[len(q[1].Close)-1] != 188.57 { 98 | t.Error("Invalid last value") 99 | } 100 | } 101 | --------------------------------------------------------------------------------