├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── TODO.md ├── build.sbt ├── project └── build.properties └── src ├── main ├── resources │ └── reference.conf └── scala │ └── openquant │ └── yahoofinance │ ├── Fundamentals.scala │ ├── Logging.scala │ ├── Quote.scala │ ├── Resolution.scala │ ├── YahooFinance.scala │ └── impl │ ├── FundamentalsParser.scala │ └── QuoteParser.scala └── test └── scala └── openquant └── yahoofinance └── YahooFinanceSpec.scala /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.log 3 | 4 | # sbt specific 5 | .cache 6 | .history 7 | .lib/ 8 | dist/* 9 | target/ 10 | lib_managed/ 11 | src_managed/ 12 | project/boot/ 13 | project/plugins/project/ 14 | 15 | # Scala-IDE specific 16 | .scala_dependencies 17 | .worksheet 18 | .idea 19 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | scala: 3 | - 2.11.8 4 | jdk: 5 | - oraclejdk8 6 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | 4 | ## 0.2 5 | Added fundamentals 6 | 7 | ## 0.1 8 | Initial release 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Open Quant 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # YahooFinanceScala 2 | A non-blocking Yahoo Finance Scala client based on Akka streams. 3 | 4 | [![Twitter Follow](https://img.shields.io/twitter/follow/openquantfin.svg?style=social)](https://twitter.com/intent/user?screen_name=openquantfin) 5 | [![Build Status](https://travis-ci.org/openquant/YahooFinanceScala.svg?branch=master)](https://travis-ci.org/openquant/YahooFinanceScala) 6 | [![Join the chat at https://gitter.im/openquant/algotrading](https://badges.gitter.im/openquant/common.svg)](https://gitter.im/openquant/algotrading?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 7 | 8 | PayPal donate button 12 | 13 | 14 | ## Add YahooFinanceScala to your build.sbt file 15 | 16 | 17 | ```scala 18 | libraryDependencies ++= Seq( 19 | "com.larroy.openquant" %% "yahoofinancescala" % "0.3" 20 | ) 21 | ``` 22 | 23 | [Check it out in Central](http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22com.larroy.openquant%22) 24 | 25 | ## Usage 26 | 27 | 28 | `$ sbt console` 29 | 30 | ```scala 31 | import scala.concurrent.duration.Duration 32 | import openquant.yahoofinance.{YahooFinance, Quote, Fundamentals} 33 | import akka.actor.ActorSystem 34 | import java.time.ZonedDateTime 35 | import scala.concurrent.Await 36 | 37 | implicit val system = ActorSystem() 38 | 39 | val yahooFinance = new YahooFinance() 40 | val quotes: IndexedSeq[Quote] = Await.result(yahooFinance.quotes("MSFT", Some(ZonedDateTime.now().minusDays(5))), Duration.Inf) 41 | // Quote(2016-04-01T00:00-04:00[America/New_York],55.049999,55.57,55.610001,54.57,24298600,55.57) 42 | val fundamentals: IndexedSeq[Fundamentals] = Await.result(yahooFinance.fundamentals("IBM"), Duration.Inf) 43 | // fundamentals: IndexedSeq[openquant.yahoofinance.Fundamentals] = Vector(Fundamentals(true,IBM,International Business Machines)) 44 | ``` 45 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | # TODO 2 | 3 | 4 | - Improve fundamentals 5 | - Handle HTTP redirections 6 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | lazy val commonSettings = Seq( 2 | version := "0.3", 3 | organization := "com.larroy.openquant", 4 | name := "YahooFinanceScala", 5 | scalaVersion := "2.11.8", 6 | scalacOptions := Seq( 7 | "-target:jvm-1.8", 8 | "-unchecked", 9 | "-deprecation", 10 | "-feature", 11 | "-encoding", "utf8", 12 | "-Xlint" 13 | 14 | ), 15 | resolvers ++= Seq(Resolver.sonatypeRepo("releases"), 16 | Resolver.sonatypeRepo("snapshots"), 17 | Resolver.bintrayRepo("scalaz", "releases"), 18 | Resolver.bintrayRepo("megamsys", "scala"), 19 | "Akka Snapshot Repository" at "http://repo.akka.io/snapshots/" 20 | ), 21 | 22 | // Sonatype publishing 23 | publishMavenStyle := true, 24 | publishTo := { 25 | val nexus = "https://oss.sonatype.org/" 26 | if (isSnapshot.value) 27 | Some("snapshots" at nexus + "content/repositories/snapshots") 28 | else 29 | Some("releases" at nexus + "service/local/staging/deploy/maven2") 30 | }, 31 | autoScalaLibrary := false, 32 | autoScalaLibrary in test := false, 33 | publishArtifact in Test := false, 34 | pomIncludeRepository := { _ => false }, 35 | pomExtra := ( 36 | https://github.com/openquant 37 | 38 | 39 | MIT 40 | http://opensource.org/licenses/MIT 41 | repo 42 | 43 | 44 | 45 | https://github.com/openquant/YahooFinanceScala.git 46 | scm:git@github.com:openquant/YahooFinanceScala.git 47 | 48 | 49 | 50 | larroy 51 | Pedro Larroy 52 | https://github.com/larroy 53 | 54 | 55 | ) 56 | ) 57 | 58 | lazy val commonDependencies = Seq( 59 | "org.slf4j" % "jcl-over-slf4j" % "1.7.7", 60 | "commons-logging" % "commons-logging" % "1.2", 61 | "ch.qos.logback" % "logback-classic" % "1.1.3", 62 | "net.ceedubs" %% "ficus" % "1.1.2", 63 | "com.typesafe.akka" %% "akka-stream" % "2.4.3", 64 | //"com.typesafe.akka" %% "akka-http-core" % "2.4-SNAPSHOT", 65 | "com.typesafe.akka" %% "akka-http-core" % "2.4.3", 66 | "com.github.tototoshi" %% "scala-csv" % "1.+", 67 | "ch.qos.logback" % "logback-classic" % "1.1.3" 68 | ) 69 | 70 | lazy val testDependencies = Seq( 71 | "org.specs2" %% "specs2" % "3.+" % "test", 72 | "com.typesafe.akka" %% "akka-testkit" % "2.4.3" % "test" 73 | ) 74 | 75 | lazy val yahoofinancescala = project.in(file(".")) 76 | .settings(commonSettings: _*) 77 | .settings(libraryDependencies ++= commonDependencies) 78 | .settings(libraryDependencies ++= testDependencies) 79 | 80 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.9 2 | -------------------------------------------------------------------------------- /src/main/resources/reference.conf: -------------------------------------------------------------------------------- 1 | openquant { 2 | yahoofinance { 3 | quotes { 4 | scheme = "http" 5 | host = "ichart.yahoo.com" 6 | path = "/table.csv" 7 | } 8 | fundamentals { 9 | # http://finance.yahoo.com/d/quotes.csv?s=GE+PTR+MSFT&f=snd1l1yr 10 | scheme = "http" 11 | host = "download.finance.yahoo.com" 12 | path = "/d/quotes.csv" 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /src/main/scala/openquant/yahoofinance/Fundamentals.scala: -------------------------------------------------------------------------------- 1 | package openquant.yahoofinance 2 | 3 | case class Fundamentals( 4 | looksValid: Boolean, 5 | symbol: String, 6 | name: String = "" 7 | ) 8 | 9 | -------------------------------------------------------------------------------- /src/main/scala/openquant/yahoofinance/Logging.scala: -------------------------------------------------------------------------------- 1 | package openquant.yahoofinance 2 | 3 | import org.slf4j.{Logger, LoggerFactory} 4 | 5 | trait Logging { 6 | val log: Logger = LoggerFactory.getLogger(this.getClass) 7 | } 8 | -------------------------------------------------------------------------------- /src/main/scala/openquant/yahoofinance/Quote.scala: -------------------------------------------------------------------------------- 1 | package openquant.yahoofinance 2 | 3 | import java.time.ZonedDateTime 4 | 5 | case class Quote( 6 | date: ZonedDateTime, 7 | open: BigDecimal, 8 | close: BigDecimal, 9 | high: BigDecimal, 10 | low: BigDecimal, 11 | volume: BigDecimal, 12 | adjClose: BigDecimal 13 | ) 14 | -------------------------------------------------------------------------------- /src/main/scala/openquant/yahoofinance/Resolution.scala: -------------------------------------------------------------------------------- 1 | package openquant.yahoofinance 2 | 3 | object Resolution extends Enumeration { 4 | type Enum = Value 5 | val Day, Month, Week = Value 6 | 7 | def parameter(resolution: Enum): String = resolution match { 8 | case Day ⇒ "d" 9 | case Week ⇒ "w" 10 | case Month ⇒ "m" 11 | } 12 | } 13 | 14 | 15 | -------------------------------------------------------------------------------- /src/main/scala/openquant/yahoofinance/YahooFinance.scala: -------------------------------------------------------------------------------- 1 | package openquant.yahoofinance 2 | 3 | import java.time.ZonedDateTime 4 | import java.time.temporal.ChronoField 5 | 6 | import akka.actor.ActorSystem 7 | import akka.http.scaladsl.Http 8 | import akka.http.scaladsl.model.Uri.Query 9 | import akka.http.scaladsl.model.{HttpRequest, HttpResponse, Uri} 10 | import akka.stream.ActorMaterializer 11 | import akka.stream.scaladsl.Sink 12 | import akka.util.ByteString 13 | import com.typesafe.config.{Config, ConfigFactory} 14 | import net.ceedubs.ficus.Ficus._ 15 | import openquant.yahoofinance.impl.{FundamentalsParser, QuoteParser} 16 | 17 | import scala.collection.immutable.IndexedSeq 18 | import scala.concurrent.Future 19 | 20 | /** 21 | * Fetch quotes from Yahoo Finance asynchronously 22 | * 23 | * @param actorSystem an akka ActorSystem to run on 24 | */ 25 | class YahooFinance(implicit actorSystem: ActorSystem, config: Config = ConfigFactory.load().getConfig("openquant.yahoofinance")) { 26 | implicit val materializer = ActorMaterializer() 27 | implicit val executionContext = actorSystem.dispatcher 28 | val QuotesScheme = config.as[String]("quotes.scheme") 29 | val QuotesHost = config.as[String]("quotes.host") 30 | val QuotesPath = config.as[String]("quotes.path") 31 | 32 | val FundamentalsScheme = config.as[String]("fundamentals.scheme") 33 | val FundamentalsHost = config.as[String]("fundamentals.host") 34 | val FundamentalsPath = config.as[String]("fundamentals.path") 35 | 36 | /** 37 | * Get quotes from Yahoo Finance 38 | * 39 | * @param symbol for example MSFT, IBM, etc. 40 | * @param fromOpt initial time 41 | * @param toOpt end time 42 | * @param resolution quotes [[Resolution]], default is Day 43 | * @return future quotes or a failed future on error 44 | */ 45 | def quotes( 46 | symbol: String, 47 | fromOpt: Option[ZonedDateTime] = None, 48 | toOpt: Option[ZonedDateTime] = None, 49 | resolution: Resolution.Enum = Resolution.Day): Future[IndexedSeq[Quote]] = { 50 | // s: symbol 51 | // abc: from 52 | // def: to 53 | // g: resolution 54 | 55 | val params = { 56 | Vector( 57 | "s" → symbol, 58 | "g" → Resolution.parameter(resolution) 59 | ) ++ 60 | fromOpt.toList.flatMap { from => 61 | Vector( 62 | "a" → (from.get(ChronoField.MONTH_OF_YEAR) - 1).toString, 63 | "b" → from.get(ChronoField.DAY_OF_MONTH).toString, 64 | "c" → from.get(ChronoField.YEAR).toString 65 | ) 66 | } ++ 67 | toOpt.toList.flatMap { to => 68 | Vector( 69 | "d" → (to.get(ChronoField.MONTH_OF_YEAR) - 1).toString, 70 | "e" → to.get(ChronoField.DAY_OF_MONTH).toString, 71 | "f" → to.get(ChronoField.YEAR).toString 72 | ) 73 | } 74 | } 75 | 76 | val query = Query(params: _*) 77 | 78 | val uri = Uri(scheme = QuotesScheme).withQuery(query).withHost(QuotesHost).withPath(Uri.Path(QuotesPath)) 79 | 80 | val request = HttpRequest(uri = uri) 81 | val res = Http().singleRequest(request) 82 | res.flatMap(handleQuotesResponse) 83 | } 84 | 85 | private def handleQuotesResponse(response: HttpResponse): Future[Vector[Quote]] = { 86 | if (response.status.isFailure || response.status.isRedirection) 87 | Future.failed(new RuntimeException(response.status.reason)) 88 | else { 89 | val parser = QuoteParser() 90 | val concat = Sink.fold[ByteString, ByteString](ByteString())(_ ++ _) 91 | val content: Future[String] = response.entity.dataBytes.runWith(concat).map(_.utf8String) 92 | val res = content.map { x ⇒ 93 | parser.parse(x) 94 | } 95 | res 96 | } 97 | } 98 | 99 | def fundamentals(symbols: String*): Future[IndexedSeq[Fundamentals]] = { 100 | fundamentals(symbols.toIndexedSeq) 101 | } 102 | 103 | def fundamentals(symbols: IndexedSeq[String]): Future[IndexedSeq[Fundamentals]] = { 104 | val params: Vector[(String, String)] = Vector( 105 | "s" → symbols.mkString("+"), 106 | "f" → "sn" 107 | ) 108 | val query = Query(params: _*) 109 | 110 | val uri = Uri(scheme = FundamentalsScheme).withQuery(query).withHost(FundamentalsHost).withPath(Uri.Path(FundamentalsPath)) 111 | val request = HttpRequest(uri = uri) 112 | val res = Http().singleRequest(request) 113 | res.flatMap(handleFundamentalsResponse) 114 | } 115 | 116 | private def handleFundamentalsResponse(response: HttpResponse): Future[Vector[Fundamentals]] = { 117 | // FIXME: handle redirects 118 | if (response.status.isFailure || response.status.isRedirection) 119 | Future.failed(new RuntimeException(response.status.reason)) 120 | else { 121 | val concat = Sink.fold[ByteString, ByteString](ByteString())(_ ++ _) 122 | val content: Future[String] = response.entity.dataBytes.runWith(concat).map(_.utf8String) 123 | val res = content.map(FundamentalsParser) 124 | res 125 | } 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /src/main/scala/openquant/yahoofinance/impl/FundamentalsParser.scala: -------------------------------------------------------------------------------- 1 | package openquant.yahoofinance.impl 2 | 3 | import java.time.format.DateTimeFormatter 4 | import java.time.{LocalDate, ZoneId, ZonedDateTime} 5 | 6 | import com.github.tototoshi.csv._ 7 | import openquant.yahoofinance.Fundamentals 8 | 9 | import scala.io.Source 10 | 11 | /** 12 | * Parses fundamental data in CSV format from Yahoo Finance into [[Fundamentals]] 13 | */ 14 | object FundamentalsParser extends Function1[String, Vector[Fundamentals]] { 15 | def apply(content: String): Vector[Fundamentals] = { 16 | val csvReader = CSVReader.open(Source.fromString(content)) 17 | val fundamentals: Vector[Fundamentals] = csvReader.toStream.map { fields ⇒ 18 | parseCSVLine(fields.toVector) 19 | }.toVector 20 | fundamentals 21 | } 22 | 23 | private def parseCSVLine(field: Vector[String]): Fundamentals = { 24 | require(field.length >= 2, "number of fields") 25 | val name = field(1) 26 | if (name == "N/A") 27 | Fundamentals( 28 | looksValid = false, 29 | symbol = field(0), 30 | name = name 31 | ) 32 | else 33 | Fundamentals( 34 | looksValid = true, 35 | symbol = field(0), 36 | name = name 37 | ) 38 | } 39 | } -------------------------------------------------------------------------------- /src/main/scala/openquant/yahoofinance/impl/QuoteParser.scala: -------------------------------------------------------------------------------- 1 | package openquant.yahoofinance.impl 2 | 3 | import java.time.format.DateTimeFormatter 4 | import java.time.{LocalDate, ZoneId, ZonedDateTime} 5 | 6 | import com.github.tototoshi.csv._ 7 | import openquant.yahoofinance.Quote 8 | 9 | import scala.io.Source 10 | 11 | /** 12 | * Parses historical data in CSV format from Yahoo Finance into [[Quote]] 13 | */ 14 | class QuoteParser { 15 | private[this] val df = DateTimeFormatter.ofPattern("yyyy-MM-dd") 16 | private[this] val zoneId = ZoneId.of("America/New_York") 17 | 18 | def parse(content: String): Vector[Quote] = { 19 | val csvReader = CSVReader.open(Source.fromString(content)) 20 | val quotes: Vector[Quote] = csvReader.toStream.drop(1).map { fields ⇒ 21 | parseCSVLine(fields.toVector) 22 | }.toVector 23 | quotes 24 | } 25 | 26 | private def parseCSVLine(field: Vector[String]): Quote = { 27 | require(field.length >= 7) 28 | Quote( 29 | parseDate(field(0)), 30 | BigDecimal(field(1)), 31 | BigDecimal(field(4)), 32 | BigDecimal(field(2)), 33 | BigDecimal(field(3)), 34 | BigDecimal(field(5)), 35 | BigDecimal(field(6)) 36 | ) 37 | } 38 | 39 | private def parseDate(date: String): ZonedDateTime = { 40 | LocalDate.parse(date, df).atStartOfDay().atZone(zoneId) 41 | } 42 | } 43 | 44 | object QuoteParser { 45 | def apply() = new QuoteParser 46 | } 47 | -------------------------------------------------------------------------------- /src/test/scala/openquant/yahoofinance/YahooFinanceSpec.scala: -------------------------------------------------------------------------------- 1 | package openquant.yahoofinance 2 | 3 | import java.time.ZonedDateTime 4 | 5 | import akka.actor.ActorSystem 6 | import akka.testkit.TestKit 7 | import org.specs2.matcher.{FutureMatchers, Matchers} 8 | import org.specs2.mutable._ 9 | 10 | import scala.concurrent.Await 11 | import scala.concurrent.duration.Duration 12 | 13 | class YahooFinanceSpec extends TestKit(ActorSystem()) with SpecificationLike with Matchers with Logging { 14 | "get quotes" in { 15 | val yahooFinance = new YahooFinance() 16 | val res = Await.result(yahooFinance.quotes("MSFT", Some(ZonedDateTime.now().minusDays(5))), Duration.Inf) 17 | res.length must be_>=(3) 18 | res.length must be_<=(5) 19 | } 20 | "get full history" in { 21 | val yahooFinance = new YahooFinance() 22 | val res = Await.result(yahooFinance.quotes("MSFT"), Duration.Inf) 23 | res.length must be_>=(1000) 24 | } 25 | "non-existent symbol" in { 26 | val yahooFinance = new YahooFinance() 27 | Await.result(yahooFinance.quotes("qwertyasdf"), Duration.Inf) must throwA[RuntimeException] 28 | } 29 | "invalid fundamentals" in { 30 | val yahooFinance = new YahooFinance() 31 | val invalids = Await.result(yahooFinance.fundamentals(Vector("qwertyasdf")), Duration.Inf) 32 | invalids must have size (1) 33 | invalids.head.looksValid must beFalse 34 | } 35 | 36 | "valid fundamentals" in { 37 | val yahooFinance = new YahooFinance() 38 | val syms = Vector("MSFT", "IBM") 39 | val valids = Await.result(yahooFinance.fundamentals(syms), Duration.Inf) 40 | valids must have size(2) 41 | valids.foreach { x ⇒ 42 | x.looksValid must beTrue 43 | x.name must not beEmpty 44 | } 45 | valids.map { _.symbol } must contain(exactly(syms:_*)) 46 | ok 47 | } 48 | } --------------------------------------------------------------------------------