├── .gitignore ├── .project ├── .pydevproject ├── README.md ├── conf ├── backtest_period.ini ├── backtest_sma.ini ├── backtest_smaPortfolio.ini ├── backtest_zscorePortfolio.ini ├── stock.list └── test.ini ├── data ├── hoursing_interestRate.xls ├── longTerm_1871.xls └── symbols │ ├── SPY500.list │ ├── china10.list │ ├── china544.list │ └── nasdaq.list ├── deprecated ├── example │ ├── chinaReturn.py │ ├── enumerateWeeklyTrading.py │ ├── plotGoogle.py │ ├── plotInterestHoursingStock.py │ └── usTradingStrategy.py └── ultrafinance │ ├── lib │ ├── __init__.py │ ├── dataType.py │ ├── errors.py │ ├── excelLib.py │ ├── exception.py │ ├── googleFinance.py │ ├── hbaseClient.py │ ├── historicalDataStorage.py │ ├── plotDateValueDict.py │ ├── plotPortfolio.py │ ├── stockMeasurement.py │ ├── tradingStrategy │ │ ├── __init__.py │ │ ├── automaticInvestmentPlan.py │ │ └── tradeInWeek.py │ ├── tradingStrategyFactory.py │ ├── util.py │ └── yahooFinance.py │ └── processChain │ ├── __init__.py │ ├── baseModule.py │ ├── config │ ├── historicalStock.ini │ ├── interestHoursingStock.ini │ ├── tradingInWeekStrategy.ini │ └── usTradingStrategy.ini │ ├── configuration.py │ ├── feeder │ ├── __init__.py │ ├── defaultFeeder.py │ ├── excelDataFeeder.py │ ├── historicalDataFeeder.py │ └── indexDataFeeder.py │ ├── outputer │ ├── __init__.py │ ├── defaultOutputer.py │ ├── emailOutputer.py │ ├── plotStockOutputer.py │ └── plotYearlyOutputer.py │ ├── pluginManager.py │ ├── processChain.py │ └── processor │ ├── __init__.py │ ├── avgDivProcessor.py │ ├── defaultProcessor.py │ └── tradingStrategyProcessor.py ├── examples ├── __init__.py ├── backTester.py ├── fundamentalCrawler.py ├── stockCrawler.py └── stockPicker.py ├── hadoop ├── conf │ ├── core-site.xml │ ├── hbase-site.xml │ ├── hdfs-site.xml │ └── mapred-site.xml └── installHadoopHbaseHive.sh ├── lib └── PyDispatcher-2.0.1.zip ├── result └── graph │ ├── GOOG graph.png │ ├── hoursing.png │ ├── housing_interest_stock.png │ └── portfolios.png ├── setup.py ├── setupCommand.py ├── tests ├── __init__.py └── unit │ ├── __init__.py │ ├── stock.list │ ├── test_account.py │ ├── test_excel_dam.py │ ├── test_excel_lib.py │ ├── test_google_dam.py │ ├── test_google_finance.py │ ├── test_observable.py │ ├── test_plot_date_value_dict.py │ ├── test_py_talib.py │ ├── test_pyconfig.py │ ├── test_singleton.py │ ├── test_sql_dam.py │ ├── test_tick_feeder.py │ ├── test_trading_center.py │ ├── test_util.py │ ├── test_yahoo_dam.py │ └── test_yahoo_finance.py └── ultrafinance ├── __init__.py ├── backTest ├── __init__.py ├── account.py ├── accountManager.py ├── appGlobal.py ├── btUtil.py ├── constant.py ├── history.py ├── indexHelper.py ├── metric.py ├── stateSaver │ ├── __init__.py │ ├── hbaseSaver.py │ ├── sqlSaver.py │ └── stateSaverFactory.py ├── tickFeeder.py ├── tickSubscriber │ ├── __init__.py │ └── strategies │ │ ├── __init__.py │ │ ├── baseStrategy.py │ │ ├── periodStrategy.py │ │ ├── smaPortfolioStrategy.py │ │ ├── smaStrategy.py │ │ ├── strategyFactory.py │ │ ├── zscoreMomentumPortfolioStrategy.py │ │ └── zscorePortfolioStrategy.py ├── tradingCenter.py └── tradingEngine.py ├── dam ├── DAMFactory.py ├── __init__.py ├── baseDAM.py ├── excelDAM.py ├── excelLib.py ├── googleDAM.py ├── googleFinance.py ├── hbaseDAM.py ├── hbaseLib.py ├── sqlDAM.py ├── yahooDAM.py └── yahooFinance.py ├── designPattern ├── __init__.py ├── observable.py └── singleton.py ├── lib ├── __init__.py ├── errors.py ├── plotDateValueDict.py └── util.py ├── model └── __init__.py ├── module ├── __init__.py ├── backTester.py └── googleCrawler.py ├── pyTaLib ├── __init__.py ├── indicator.py └── pandasImpl.py └── ufConfig ├── __init__.py └── pyConfig.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | ultraFinance 4 | 5 | 6 | 7 | 8 | 9 | org.python.pydev.PyDevBuilder 10 | 11 | 12 | 13 | 14 | 15 | org.python.pydev.pythonNature 16 | 17 | 18 | -------------------------------------------------------------------------------- /.pydevproject: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Default 6 | python 2.7 7 | 8 | /ultraFinance/ultrafinance 9 | /ultraFinance 10 | 11 | 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ultrafinance 2 | ============ 3 | Python project for real-time financial data collection, analyzing && backtesting trading strategies. 4 | 5 | Project Home: https://code.google.com/p/ultra-finance/ 6 | 7 | -------------------------------------------------------------------------------- /conf/backtest_period.ini: -------------------------------------------------------------------------------- 1 | [app_main] 2 | 3 | [ultrafinance] 4 | backtest.strategy_name = period 5 | backtest.index = NYSEARCA:SPY 6 | backtest.tradetype = quote 7 | backtest.symbolfile = stock.list 8 | 9 | backtest.output_saver = true 10 | backtest.input_dam = sql 11 | backtest.input_db = sqlite:////data/stock.sqlite 12 | backtest.output_saver = sql 13 | backtest.output_db_prefix = sqlite:////data/ 14 | backtest.strategy.period = 30 15 | 16 | [loggers] 17 | keys = root 18 | 19 | [handlers] 20 | keys = console 21 | 22 | [formatters] 23 | keys = generic 24 | 25 | [logger_root] 26 | level = DEBUG 27 | handlers = console 28 | 29 | [handler_console] 30 | class = StreamHandler 31 | level = DEBUG 32 | formatter = generic 33 | args = (sys.stdout,) 34 | 35 | [formatter_generic] 36 | format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(threadName)s] %(message)s 37 | datefmt = %y/%m/%d %H:%M:%S -------------------------------------------------------------------------------- /conf/backtest_sma.ini: -------------------------------------------------------------------------------- 1 | [app_main] 2 | 3 | [ultrafinance] 4 | backtest.strategy_name = sma 5 | backtest.index = NYSEARCA:SPY 6 | backtest.tradetype = quote 7 | backtest.symbolfile = stock.list 8 | 9 | backtest.output_saver = true 10 | backtest.input_dam = sql 11 | backtest.input_db = sqlite:////data/stock.sqlite 12 | backtest.output_saver = sql 13 | backtest.output_db_prefix = sqlite:////data/ 14 | 15 | [loggers] 16 | keys = root 17 | 18 | [handlers] 19 | keys = console 20 | 21 | [formatters] 22 | keys = generic 23 | 24 | [logger_root] 25 | level = INFO 26 | handlers = console 27 | 28 | [handler_console] 29 | class = StreamHandler 30 | level = INFO 31 | formatter = generic 32 | args = (sys.stdout,) 33 | 34 | [formatter_generic] 35 | format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(threadName)s] %(message)s 36 | datefmt = %y/%m/%d %H:%M:%S -------------------------------------------------------------------------------- /conf/backtest_smaPortfolio.ini: -------------------------------------------------------------------------------- 1 | [app_main] 2 | 3 | [ultrafinance] 4 | backtest.strategy_name = smaPortfolio 5 | backtest.index = NYSEARCA:SPY 6 | backtest.tradetype = quote 7 | backtest.symbolfile = stock.list 8 | backtest.buying_ratio = 1 9 | 10 | backtest.output_saver = true 11 | backtest.input_dam = sql 12 | backtest.input_db = sqlite:////data/stock.sqlite 13 | backtest.output_saver = sql 14 | backtest.output_db_prefix = sqlite:////data/ 15 | 16 | 17 | [loggers] 18 | keys = root 19 | 20 | [handlers] 21 | keys = console 22 | 23 | [formatters] 24 | keys = generic 25 | 26 | [logger_root] 27 | level = DEBUG 28 | handlers = console 29 | 30 | [handler_console] 31 | class = StreamHandler 32 | level = DEBUG 33 | formatter = generic 34 | args = (sys.stdout,) 35 | 36 | [formatter_generic] 37 | format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(threadName)s] %(message)s 38 | datefmt = %y/%m/%d %H:%M:%S -------------------------------------------------------------------------------- /conf/backtest_zscorePortfolio.ini: -------------------------------------------------------------------------------- 1 | [app_main] 2 | 3 | [ultrafinance] 4 | backtest.strategy_name = zscorePortfolio 5 | backtest.index = NYSEARCA:SPY 6 | backtest.tradetype = quote 7 | backtest.symbolfile = stock.list 8 | backtest.buying_ratio = 1 9 | 10 | backtest.output_saver = true 11 | backtest.input_dam = sql 12 | backtest.input_db = sqlite:////data/stock.sqlite 13 | backtest.output_saver = sql 14 | backtest.output_db_prefix = sqlite:////data/ 15 | 16 | 17 | [loggers] 18 | keys = root 19 | 20 | [handlers] 21 | keys = console 22 | 23 | [formatters] 24 | keys = generic 25 | 26 | [logger_root] 27 | level = DEBUG 28 | handlers = console 29 | 30 | [handler_console] 31 | class = StreamHandler 32 | level = DEBUG 33 | formatter = generic 34 | args = (sys.stdout,) 35 | 36 | [formatter_generic] 37 | format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(threadName)s] %(message)s 38 | datefmt = %y/%m/%d %H:%M:%S -------------------------------------------------------------------------------- /conf/stock.list: -------------------------------------------------------------------------------- 1 | EBAY 2 | GOOG 3 | C 4 | MSFT 5 | F -------------------------------------------------------------------------------- /conf/test.ini: -------------------------------------------------------------------------------- 1 | [app_main] 2 | field1 = value1 3 | field2 = value2 4 | field3 = 5 | 6 | [log] 7 | file=test.log -------------------------------------------------------------------------------- /data/hoursing_interestRate.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panpanpandas/ultrafinance/ce3594dbfef747cba0c39f059316df7c208aac40/data/hoursing_interestRate.xls -------------------------------------------------------------------------------- /data/longTerm_1871.xls: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panpanpandas/ultrafinance/ce3594dbfef747cba0c39f059316df7c208aac40/data/longTerm_1871.xls -------------------------------------------------------------------------------- /data/symbols/SPY500.list: -------------------------------------------------------------------------------- 1 | MMM 2 | ACE 3 | ABT 4 | ANF 5 | ADBE 6 | AMD 7 | AES 8 | AET 9 | AFL 10 | A 11 | APD 12 | ARG 13 | AKS 14 | AKAM 15 | AA 16 | ATI 17 | AGN 18 | ALL 19 | ALTR 20 | MO 21 | AMZN 22 | AEE 23 | AEP 24 | AXP 25 | AIG 26 | AMT 27 | AMP 28 | ABC 29 | AMGN 30 | APH 31 | APC 32 | ADI 33 | AON 34 | APA 35 | AIV 36 | APOL 37 | AAPL 38 | AMAT 39 | ADM 40 | AIZ 41 | T 42 | ADSK 43 | ADP 44 | AN 45 | AZO 46 | AVB 47 | AVY 48 | AVP 49 | BHI 50 | BLL 51 | BAC 52 | BK 53 | BCR 54 | BAX 55 | BBT 56 | BDX 57 | BBBY 58 | BMS 59 | BRKb 60 | BBY 61 | BIG 62 | BIIB 63 | HRB 64 | BMC 65 | BA 66 | BXP 67 | BSX 68 | BMY 69 | BRCM 70 | BFb 71 | CHRW 72 | CA 73 | CVC 74 | COG 75 | CAM 76 | CPB 77 | COF 78 | CAH 79 | CFN 80 | KMX 81 | CCL 82 | CAT 83 | CBG 84 | CBS 85 | CELG 86 | CNP 87 | CTL 88 | CEPH 89 | CERN 90 | CF 91 | SCHW 92 | CHK 93 | CVX 94 | CB 95 | CI 96 | CINF 97 | CTAS 98 | CSCO 99 | C 100 | CTXS 101 | CLF 102 | CLX 103 | CME 104 | CMS 105 | COH 106 | KO 107 | CCE 108 | CTSH 109 | CL 110 | CMCSA 111 | CMA 112 | CSC 113 | CPWR 114 | CAG 115 | COP 116 | CNX 117 | ED 118 | STZ 119 | CEG 120 | GLW 121 | COST 122 | CVH 123 | COV 124 | CSX 125 | CMI 126 | CVS 127 | DHI 128 | DHR 129 | DRI 130 | DVA 131 | DF 132 | DE 133 | DELL 134 | DNR 135 | XRAY 136 | DVN 137 | DV 138 | DO 139 | DTV 140 | DFS 141 | DISCA 142 | D 143 | RRD 144 | DOV 145 | DOW 146 | DPS 147 | DTE 148 | DD 149 | DUK 150 | DNB 151 | ETFC 152 | EMN 153 | ETN 154 | EBAY 155 | ECL 156 | EIX 157 | EP 158 | ERTS 159 | EMC 160 | EMR 161 | ETR 162 | EOG 163 | EQT 164 | EFX 165 | EQR 166 | EL 167 | EXC 168 | EXPE 169 | EXPD 170 | ESRX 171 | XOM 172 | FFIV 173 | FDO 174 | FAST 175 | FII 176 | FDX 177 | FIS 178 | FITB 179 | FHN 180 | FSLR 181 | FE 182 | FISV 183 | FLIR 184 | FLS 185 | FLR 186 | FMC 187 | FTI 188 | F 189 | FRX 190 | FO 191 | BEN 192 | FCX 193 | FTR 194 | GME 195 | GCI 196 | GPS 197 | GD 198 | GE 199 | GIS 200 | GPC 201 | GNW 202 | GENZ 203 | GILD 204 | GS 205 | GR 206 | GT 207 | GOOG 208 | GWW 209 | HAL 210 | HOG 211 | HAR 212 | HRS 213 | HIG 214 | HAS 215 | HCP 216 | HCN 217 | HNZ 218 | HP 219 | HES 220 | HPQ 221 | HD 222 | HON 223 | HRL 224 | HSP 225 | HST 226 | HCBK 227 | HUM 228 | HBAN 229 | ITW 230 | TEG 231 | INTC 232 | ICE 233 | IBM 234 | IFF 235 | IGT 236 | IP 237 | IPG 238 | INTU 239 | ISRG 240 | IVZ 241 | IRM 242 | ITT 243 | JBL 244 | JEC 245 | JNS 246 | JDSU 247 | JNJ 248 | JCI 249 | JOYG 250 | JPM 251 | JNPR 252 | K 253 | KEY 254 | KMB 255 | KIM 256 | KLAC 257 | KSS 258 | KFT 259 | KR 260 | LLL 261 | LH 262 | LM 263 | LEG 264 | LEN 265 | LUK 266 | LXK 267 | LIFE 268 | LLY 269 | LTD 270 | LNC 271 | LLTC 272 | LMT 273 | L 274 | LO 275 | LOW 276 | LSI 277 | MTB 278 | M 279 | MRO 280 | MAR 281 | MMC 282 | MI 283 | MAS 284 | MEE 285 | MA 286 | MAT 287 | MKC 288 | MCD 289 | MHP 290 | MCK 291 | MJN 292 | MWV 293 | MHS 294 | MDT 295 | WFR 296 | MRK 297 | MET 298 | PCS 299 | MCHP 300 | MU 301 | MSFT 302 | MOLX 303 | TAP 304 | MON 305 | MWW 306 | MCO 307 | MS 308 | MMI 309 | MSI 310 | MUR 311 | MYL 312 | NBR 313 | NDAQ 314 | NOV 315 | NSM 316 | NTAP 317 | NFLX 318 | NWL 319 | NFX 320 | NEM 321 | NWSA 322 | NEE 323 | GAS 324 | NKE 325 | NI 326 | NE 327 | NBL 328 | JWN 329 | NSC 330 | NTRS 331 | NOC 332 | NU 333 | NOVL 334 | NVLS 335 | NRG 336 | NUE 337 | NVDA 338 | NYX 339 | ORLY 340 | OXY 341 | OMC 342 | OKE 343 | ORCL 344 | OI 345 | PCAR 346 | IR 347 | PLL 348 | PH 349 | PDCO 350 | PAYX 351 | BTU 352 | JCP 353 | PBCT 354 | POM 355 | PEP 356 | PKI 357 | PFE 358 | PCG 359 | PM 360 | PNW 361 | PXD 362 | PBI 363 | PCL 364 | PNC 365 | RL 366 | PPG 367 | PPL 368 | PX 369 | PCP 370 | PCLN 371 | PFG 372 | PG 373 | PGN 374 | PGR 375 | PLD 376 | PRU 377 | PEG 378 | PSA 379 | PHM 380 | QEP 381 | PWR 382 | QCOM 383 | DGX 384 | Q 385 | RSH 386 | RRC 387 | RTN 388 | RHT 389 | RF 390 | RSG 391 | RAI 392 | RHI 393 | ROK 394 | COL 395 | ROP 396 | ROST 397 | RDC 398 | R 399 | SWY 400 | SAI 401 | CRM 402 | SNDK 403 | SLE 404 | SCG 405 | SLB 406 | SNI 407 | SEE 408 | SHLD 409 | SRE 410 | SHW 411 | SIAL 412 | SPG 413 | SLM 414 | SJM 415 | SNA 416 | SO 417 | LUV 418 | SWN 419 | SE 420 | S 421 | STJ 422 | SWK 423 | SPLS 424 | SBUX 425 | HOT 426 | STT 427 | SRCL 428 | SYK 429 | SUN 430 | STI 431 | SVU 432 | SYMC 433 | SYY 434 | TROW 435 | TGT 436 | TE 437 | TLAB 438 | THC 439 | TDC 440 | TER 441 | TSO 442 | TXN 443 | TXT 444 | HSY 445 | TRV 446 | TMO 447 | TIF 448 | TWX 449 | TWC 450 | TIE 451 | TJX 452 | TMK 453 | TSS 454 | TSN 455 | TYC 456 | USB 457 | UNP 458 | UNH 459 | UPS 460 | X 461 | UTX 462 | UNM 463 | URBN 464 | VFC 465 | VLO 466 | VAR 467 | VTR 468 | VRSN 469 | VZ 470 | VIAb 471 | V 472 | VNO 473 | VMC 474 | WMT 475 | WAG 476 | DIS 477 | WPO 478 | WM 479 | WAT 480 | WPI 481 | WLP 482 | WFC 483 | WDC 484 | WU 485 | WY 486 | WHR 487 | WFMI 488 | WMB 489 | WIN 490 | WEC 491 | WYN 492 | WYNN 493 | XEL 494 | XRX 495 | XLNX 496 | XL 497 | YHOO 498 | YUM 499 | ZMH 500 | ZION -------------------------------------------------------------------------------- /data/symbols/china10.list: -------------------------------------------------------------------------------- 1 | DL 2 | DYP 3 | EDU 4 | EJ 5 | GA 6 | GRO 7 | GSI 8 | GU 9 | LDK 10 | LFT -------------------------------------------------------------------------------- /deprecated/example/chinaReturn.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Apr 24, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from operator import itemgetter 7 | import time 8 | import os 9 | 10 | from ultrafinance.lib.stockMeasurement import StockMeasurement 11 | from ultrafinance.lib.historicalDataStorage import HistoricalDataStorage 12 | from ultrafinance.lib.yahooFinance import YahooFinance 13 | from ultrafinance.lib.dataType import StockDailyType 14 | from ultrafinance.lib.excelLib import ExcelLib 15 | 16 | import logging 17 | LOG = logging.getLogger() 18 | 19 | benchmarks = {'DEFAULT': '^GSPC', 20 | 'L': '^STI', 21 | 'HK': '^HSI', 22 | 'SI': '^FTSE'} 23 | benchmarkValues = {} 24 | 25 | def buildBenchmarkValues(): 26 | print "Building BenchmarkValues" 27 | for key in benchmarks.values(): 28 | benchmarkValues[key] = YahooFinance().getHistoricalPrices(key, '19010101', '20130101') 29 | time.sleep(1) 30 | 31 | print "BenchmarkValues %s built" % benchmarkValues.keys() 32 | 33 | class ChinaReturn(): 34 | def __init__(self, fullTest=False): 35 | ''' constructor ''' 36 | #folder path ../dataSource/CHINA' 37 | self.__workingDir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 38 | 'dataSource', 39 | 'CHINA') 40 | self.dayIntervals = [1, 3, 30, 90, 250, 500, 750] 41 | if fullTest: 42 | self.__stocklistFile = os.path.join(self.__workingDir, 'china544.list') 43 | else: 44 | self.__stocklistFile = os.path.join(self.__workingDir, 'china10.list') 45 | 46 | def saveHistroyDataIntoExcel(self): 47 | ''' read stock list, get historical data, and save to excel ''' 48 | print 'Save HistroyData Into Excel' 49 | storage = HistoricalDataStorage(os.path.join(self.__workingDir, 'china') ) 50 | storage.buildExlsFromFile(fileName=self.__stocklistFile, div=5) 51 | 52 | def analyze(self): 53 | ''' analyze ''' 54 | print 'Start analyzing' 55 | buildBenchmarkValues() 56 | 57 | returnRates = [[], [], [], [], [], [], []] 58 | alphas = [[], [], [], [], [], [], []] 59 | relativeReturnRates = [[], [], [], [], [], [], []] 60 | 61 | for fileName in filter( lambda f: f.endswith('.xls'), os.listdir(self.__workingDir) ): 62 | excelFile = os.path.join(self.__workingDir, fileName) 63 | sheetNames = ExcelLib.getSheetNames(excelFile) 64 | 65 | for sheetName in sheetNames: 66 | with ExcelLib(excelFile) as excel: 67 | excel.openSheet(sheetName=sheetName) 68 | 69 | contry = sheetName.split('.')[-1] if len( sheetName.split('.') ) != 1 else 'DEFAULT' 70 | benchmark = benchmarks[contry] 71 | print 'Processing %s with benchmark %s' % (sheetName, benchmark) 72 | 73 | for index, duration in enumerate(self.dayIntervals): 74 | data = [] 75 | broke = False 76 | for i in range(1, duration + 1): 77 | #print "row %d, duration %d" % (i, duration) 78 | if broke: 79 | break 80 | 81 | try: 82 | values = excel.readRow(i) 83 | for j in range( len(values) ): 84 | values[j] = float(values[j]) if j != 0 else values[j] 85 | 86 | data.append( StockDailyType( *values ) ) 87 | except Exception: 88 | print 'Analyzing %s break at %d' % (sheetName, i) 89 | broke = True 90 | break 91 | if data: 92 | dateValues = sorted(data, key=itemgetter(0)) 93 | 94 | #print benchmarkValues[benchmark] 95 | stockMeasurement = StockMeasurement(dateValues, benchmark, benchmarkValues[benchmark]) 96 | stockMeasurement.linearRegression() 97 | returnRates[index].append( stockMeasurement.returnRate() ) 98 | alphas[index].append( stockMeasurement.alpha() ) 99 | relativeReturnRates[index].append( stockMeasurement.relativeReturnRate() ) 100 | 101 | with open(os.path.join(self.__workingDir, 'output.txt'), 'w') as outputFile: 102 | outputReturnRates = map(lambda x: sum(x)/len(x), returnRates) 103 | outputAlphas = map(lambda x: sum(x)/len(x), alphas) 104 | outputRelativeReturnRates = map(lambda x: sum(x)/len(x), relativeReturnRates) 105 | print "Days since going public %s" % self.dayIntervals 106 | print "returnRates: %s" % outputReturnRates 107 | print "alphas: %s" % outputAlphas 108 | print "relativeReturnRates: %s" % outputRelativeReturnRates 109 | outputFile.write("outputReturnRates %s\n" % outputReturnRates) 110 | outputFile.write("outputAlphas %s\n" % outputAlphas) 111 | outputFile.write("outputRelativeReturnRates %s\n" % outputRelativeReturnRates) 112 | outputFile.write("returnRates %s\n" % returnRates) 113 | outputFile.write("alphas %s\n" % alphas) 114 | outputFile.write("relativeReturnRates %s\n" % relativeReturnRates) 115 | 116 | if __name__ == '__main__': 117 | app = ChinaReturn() 118 | app.saveHistroyDataIntoExcel() 119 | app.analyze() -------------------------------------------------------------------------------- /deprecated/example/enumerateWeeklyTrading.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on July 18, 2010 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.lib.tradingStrategy.tradeInWeek import TradeInWeek 7 | from ultrafinance.lib.historicalDataStorage import HistoricalDataStorage 8 | from os.path import join, dirname, abspath 9 | 10 | if __name__ == '__main__': 11 | outputPrefix = join(dirname(dirname(abspath(__file__))), 'dataSource', 'SPY', 'spy' ) 12 | print outputPrefix 13 | storage = HistoricalDataStorage(outputPrefix) 14 | storage.buildExls(['SPY'], 1) 15 | 16 | tradeInWeek = TradeInWeek('%s0.xls' % outputPrefix, 0) 17 | for value in tradeInWeek.tryAllStrategy(): 18 | print value -------------------------------------------------------------------------------- /deprecated/example/plotGoogle.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on July 17, 2010 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.processChain.processChain import runProcessChain 7 | 8 | if __name__ == '__main__': 9 | configFile1 = 'historicalStock.ini' 10 | runProcessChain(configFile1) 11 | -------------------------------------------------------------------------------- /deprecated/example/plotInterestHoursingStock.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on July 17, 2010 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.processChain.processChain import runProcessChain 7 | 8 | if __name__ == '__main__': 9 | configFile1 = 'interestHoursingStock.ini' 10 | runProcessChain(configFile1) -------------------------------------------------------------------------------- /deprecated/example/usTradingStrategy.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on July 17, 2010 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.processChain.processChain import runProcessChain 7 | 8 | if __name__ == '__main__': 9 | configFile1 = 'usTradingStrategy.ini' 10 | runProcessChain(configFile1) -------------------------------------------------------------------------------- /deprecated/ultrafinance/lib/__init__.py: -------------------------------------------------------------------------------- 1 | ''' ultraFinance lib ''' 2 | import os, sys 3 | 4 | import logging 5 | LOG = logging.getLogger(__name__) 6 | 7 | ''' 8 | mainSrc = os.path.join(os.path.dirname( (os.path.dirname(os.path.abspath(__file__))))) 9 | sys.path.append(mainSrc) 10 | 11 | logging.basicConfig(level=logging.DEBUG, 12 | format='%(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] [%(threadName)s] %(message)s', 13 | filename=os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'ultraFinance.log'), 14 | filemode='w') 15 | ''' 16 | -------------------------------------------------------------------------------- /deprecated/ultrafinance/lib/dataType.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Feb 5, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from collections import namedtuple 7 | 8 | DateValueType = namedtuple('DateValue', 'date, value') 9 | StockDailyType = namedtuple('StockDailyType', 'date, open, high, low, close, volume, adjClose') 10 | 11 | -------------------------------------------------------------------------------- /deprecated/ultrafinance/lib/errors.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on May 6, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import traceback 7 | 8 | class Errors(object): 9 | """ class hosts error code constants """ 10 | # general errors 11 | UNKNOWN_ERROR = 1 12 | FILE_NOT_EXIST = 2 13 | FILE_EXIST = 3 14 | UNDEFINED_METHOD = 4 15 | 16 | NETWORK_ERROR = 100 17 | NETWORK_400_ERROR = 101 18 | 19 | INDEX_RANGE_ERROR = 200 20 | INVALID_DAM_NAME = 201 21 | 22 | STOCK_SYMBOL_ERROR = 300 23 | STOCK_PARSING_ERROR = 301 24 | 25 | HBASE_CREATE_ERROR=401 26 | HBASE_UPDATE_ERROR=402 27 | 28 | #type eroor 29 | SIDE_TYPE_ERROR=500 30 | ORDER_TYPE_ERROR=501 31 | TRANSITION_TYPE_ERROR=502 32 | 33 | #tickFeeder 34 | FEEDER_INVALID_ERROR = 600 35 | SYMBOL_EXIST = 601 36 | INVALID_TYPE = 602 37 | SYMBOL_NOT_IN_SOURCE = 604 38 | 39 | #account error 40 | ORDER_INVALID_ERROR = 700 41 | MISSING_SYMBOL = 701 42 | 43 | #excelLib error 44 | SHEET_NAME_EXIST = 800 45 | SHEET_NAME_INVALID = 801 46 | INVALID_EXCEL_MODE = 802 47 | 48 | #trading error 49 | INVALID_ACCOUNT = 901 50 | 51 | #metric 52 | INVALID_METRIC_NAME = 1001 53 | ACCOUNT_ALEADY_SET = 1002 54 | ACCOUNT_NOT_SET = 1003 55 | INVALID_RISK_FREE_RETURN = 1004 56 | 57 | #strategy 58 | INVALID_STRATEGY_NAME = 1200 59 | NONE_ACCOUNT_ID = 1201 60 | 61 | class UfException(Exception): 62 | """ Ultra-Finance exception """ 63 | def __init__(self, error, errorMsg): 64 | """ constructor """ 65 | Exception.__init__(self) 66 | self.__error = error 67 | self.__errorMsg = errorMsg 68 | 69 | def __str__(self): 70 | """ string """ 71 | return repr(self.__errorMsg) 72 | 73 | def getCode(self): 74 | """ accessor """ 75 | return self.__error 76 | 77 | def getMsg(self): 78 | """ accessor """ 79 | return "%s: %s" % (self.__errorMsg, traceback.format_exc(5)) -------------------------------------------------------------------------------- /deprecated/ultrafinance/lib/excelLib.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Jan 30, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from xlrd import open_workbook 7 | from ultrafinance.lib.errors import UfException, Errors 8 | 9 | import logging 10 | LOG = logging.getLogger(__name__) 11 | 12 | class ExcelLib(): 13 | ''' lib for aceesing excel ''' 14 | def __init__(self, fileName=None, sheetNumber=0, sheetName=None): 15 | ''' 16 | constructor 17 | 18 | ''' 19 | self.book = None 20 | self.sheet = None 21 | if fileName is not None: 22 | self.book = open_workbook(fileName) 23 | self.sheet = self.book.sheet_by_name(sheetName) if sheetName \ 24 | else self.book.sheet_by_index(sheetNumber) 25 | 26 | def __enter__(self): 27 | ''' 28 | do nothing because xlrd handle file close in Book constructor, 29 | which make reading multiple sheets slow 30 | ''' 31 | return self 32 | 33 | def __exit__(self, type, value, traceback): 34 | ''' do nothing ''' 35 | pass 36 | 37 | def openSheet(self, sheetNumber=0, sheetName=None): 38 | self.sheet = self.book.sheet_by_name(sheetName) if sheetName \ 39 | else self.book.sheet_by_index(sheetNumber) 40 | 41 | @staticmethod 42 | def getTotalSheetNumber(fileName): 43 | return open_workbook(fileName).nsheets 44 | 45 | @staticmethod 46 | def getSheetNames(fileName): 47 | return open_workbook(fileName).sheet_names() 48 | 49 | def setSheetNumber(self, sheetNumber): 50 | self.sheet = self.book.sheet_by_index(sheetNumber) 51 | 52 | def readRow(self, rowNumber, startCol=0, endCol=-1): 53 | if abs(rowNumber) >= self.sheet.nrows: 54 | raise UfException(Errors.INDEX_RANGE_ERROR, 55 | "Excellib.readRow: row number too big: row %s, max %s" % (rowNumber, self.sheet.nrows) ) 56 | if max(abs(startCol), abs(endCol)) > self.sheet.ncols: 57 | raise UfException(Errors.INDEX_RANGE_ERROR, 58 | "Excellib.readRow: col number too big: col %s, max %s" % (max(abs(startCol), abs(endCol)), self.sheet.ncols) ) 59 | if -1 == endCol: 60 | endCol = self.sheet.ncols 61 | 62 | return [self.readCell(rowNumber, i) for i in range(startCol, endCol)] 63 | 64 | def readCol(self, colNumber, startRow=0, endRow=-1): 65 | if abs(colNumber) > self.sheet.ncols: 66 | raise UfException(Errors.INDEX_RANGE_ERROR, 67 | "Excellib.readCol: col number too big: col %s, max %s" % (colNumber, self.sheet.ncols) ) 68 | if max(abs(startRow), abs(endRow)) > self.sheet.nrows: 69 | raise UfException(Errors.INDEX_RANGE_ERROR, 70 | "Excellib.readCol: row number too big: row %s, max %s" % (max(abs(startRow), abs(endRow)), self.sheet.nrows) ) 71 | if -1 == endRow: 72 | endRow = self.sheet.nrows 73 | 74 | return [self.readCell(i, colNumber) for i in range(startRow, endRow)] 75 | 76 | def readCell(self, rowNumber, colNumber): 77 | ''' read a cell''' 78 | try: 79 | return self.sheet.cell(rowNumber, colNumber).value 80 | except BaseException as excep: 81 | raise UfException(Errors.UNKNOWN_ERROR, "Unknown Error in Excellib.readCell %s" % excep) -------------------------------------------------------------------------------- /deprecated/ultrafinance/lib/exception.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on May 6, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import traceback 7 | 8 | class Errors(object): 9 | """ class hosts error code constants """ 10 | # general errors 11 | UNKNOWN_ERROR = 1 12 | 13 | NETWORK_ERROR = 100 14 | NETWORK_400_ERROR = 101 15 | 16 | INDEX_RANGE_ERROR = 200 17 | 18 | STOCK_SYMBOL_ERROR = 300 19 | STOCK_PARSING_ERROR = 301 20 | 21 | HBASE_CREATE_ERROR=401 22 | HBASE_UPDATE_ERROR=402 23 | 24 | #type eroor 25 | SIDE_TYPE_ERROR=500 26 | ORDER_TYPE_ERROR=501 27 | TRANSITION_TYPE_ERROR=502 28 | 29 | #tickFeeder 30 | FEEDER_INVALID_ERROR = 600 31 | 32 | #account error 33 | TRANSITION_INVALID_ERROR = 700 34 | 35 | 36 | class UfException(Exception): 37 | """ Ultra-Finance exception """ 38 | def __init__(self, error, errorMsg): 39 | """ constructor """ 40 | Exception.__init__(self) 41 | self.__error = error 42 | self.__errorMsg = errorMsg 43 | 44 | def __str__(self): 45 | """ string """ 46 | return repr(self.__errorMsg) 47 | 48 | def getCode(self): 49 | """ accessor """ 50 | return self.__error 51 | 52 | def getMsg(self): 53 | """ accessor """ 54 | return "%s: %s" % (self.__errorMsg, traceback.format_exc(5)) -------------------------------------------------------------------------------- /deprecated/ultrafinance/lib/hbaseClient.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Sep 27, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import sys 7 | 8 | from thrift import Thrift 9 | from thrift.transport import TSocket, TTransport 10 | from thrift.protocol import TBinaryProtocol 11 | from hbase import ttypes 12 | from hbase.Hbase import Client, ColumnDescriptor, Mutation 13 | 14 | from ultrafinance.lib.errors import UfException, Errors 15 | 16 | import logging 17 | LOG = logging.getLogger(__name__) 18 | 19 | class HBaseClient: 20 | ''' Hbase client ''' 21 | def __init__(self): 22 | transport = TSocket.TSocket('localhost', 9090) 23 | transport = TTransport.TBufferedTransport(transport) 24 | protocol = TBinaryProtocol.TBinaryProtocol(transport) 25 | 26 | self.__client = Client(protocol) 27 | transport.open() 28 | 29 | def getTableNames(self): 30 | ''' get table names ''' 31 | return self.__client.getTableNames() 32 | 33 | def deleteTable(self, tName): 34 | ''' delete table name ''' 35 | if self.__client.isTableEnabled(tName): 36 | self.__client.disableTable(tName) 37 | 38 | self.__client.deleteTable(tName) 39 | 40 | def createTable(self, tName, ColumnDescriptors): 41 | try: 42 | self.__client.createTable(tName, ColumnDescriptors) 43 | except ttypes.AlreadyExists as excp: 44 | raise UfException(Errors.HBASE_CREATE_ERROR, 45 | "AlreadyExists Error when creating table %s with cols: %s): %s" % \ 46 | (tName, [col.name for col in ColumnDescriptors], excp.message)) 47 | 48 | def getColumnDescriptors(self, tName): 49 | try: 50 | return self.__client.getColumnDescriptors(tName) 51 | except: 52 | raise UfException(Errors.UNKNOWN_ERROR, 53 | "Error when getting column descriptors table %s" % tName) 54 | 55 | def updateRow(self, tName, rowName, mutations, timestamp=None): 56 | ''' add row to table ''' 57 | try: 58 | if timestamp is None: 59 | self.__client.mutateRow(tName, rowName, mutations) 60 | else: 61 | self.__client.mutateRowTs(tName, rowName, mutations, timestamp) 62 | except Exception as excp: 63 | raise UfException(Errors.HBASE_UPDATE_ERROR, 64 | "Error when updating table %s - rowName %s - mutations %s: %s" % \ 65 | (tName, rowName, mutations, excp)) 66 | 67 | def getRow(self, tName, rowName): 68 | ''' get row ''' 69 | result = self.__client.getRow(tName, rowName) 70 | if not result: 71 | return result 72 | else: 73 | return result[0] 74 | 75 | def scanTable(self, tName, columns, startRow="", endRow=None): 76 | ''' scan a table ''' 77 | if endRow is None: 78 | scanner = self.__client.scannerOpen(tName, startRow, columns) 79 | else: 80 | scanner = self.__client.scannerOpenWithStop(tName, startRow, endRow, columns) 81 | ret = [] 82 | 83 | row = self.__client.scannerGet(scanner) 84 | while row: 85 | ret.append(row[0]) 86 | row = self.__client.scannerGet(scanner) 87 | 88 | return ret 89 | 90 | def getClient(self): 91 | ''' return client, in case low level api is needed ''' 92 | return self.__client 93 | 94 | 95 | if __name__ == '__main__': 96 | h = HBaseClient() 97 | 98 | #delete all exiting tables 99 | for tName in h.getTableNames(): 100 | print "deleting %s" % tName 101 | h.deleteTable(tName) 102 | 103 | assert not h.getTableNames() 104 | 105 | #create table 106 | tName = 'testTable' 107 | 108 | 109 | h.createTable(tName, [ColumnDescriptor(name='col1', maxVersions=5), ColumnDescriptor(name='col2', maxVersions=5)]) 110 | print h.getTableNames() 111 | assert h.getTableNames() 112 | 113 | print "column families in %s" %(tName) 114 | print h.getColumnDescriptors(tName) 115 | 116 | #updateRow 117 | h.updateRows(tName, "bar", [Mutation(column="col1:bar", value='12345'), Mutation(column="col2:", value="67890")]) 118 | h.updateRows(tName, "foo", [Mutation(column="col1:foo", value='12345')]) 119 | print h.getRow(tName, 'bar') 120 | print h.getRow(tName, 'foo') 121 | 122 | #scan table 123 | rows = h.scanTable(tName, columns=["col1", "col2"]) 124 | print rows 125 | assert 2 == len(rows) 126 | 127 | rows = h.scanTable(tName, columns=["col1", "col2"], startRow="foo") 128 | print rows 129 | assert 1 == len(rows) 130 | 131 | rows = h.scanTable(tName, columns=["col1", "col2"], endRow="foo") 132 | print rows 133 | assert 1 == len(rows) -------------------------------------------------------------------------------- /deprecated/ultrafinance/lib/historicalDataStorage.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Mar 20, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from datetime import date 7 | from xlwt import Workbook 8 | import time 9 | import traceback 10 | 11 | from ultrafinance.lib.errors import UfException, Errors 12 | from ultrafinance.lib.yahooFinance import YahooFinance 13 | 14 | import logging 15 | LOG = logging.getLogger(__name__) 16 | 17 | class HistoricalDataStorage(): 18 | ''' class that store historical data from Yahoo finance ''' 19 | def __init__(self, outputPrefix, startDate = '1900-01-01', endDate = date.today()): 20 | ''' constructor ''' 21 | self.__outputPrefix = outputPrefix 22 | self.__startDate = startDate 23 | self.__endDate = endDate 24 | 25 | def buildExls(self, stocks, div=1): 26 | ''' get a list of stock data and store ''' 27 | print "BuildExls %s, div %s" % (stocks, div) 28 | 29 | if div < 1: 30 | raise UfException(Errors.INDEX_RANGE_ERROR, "div need to be at least 1, %s are given" % div) 31 | 32 | for i in range(div): 33 | workbook = Workbook() 34 | stocksToProcess = stocks[i * len(stocks)/div: (i+1) * len(stocks)/div] 35 | for stock in stocksToProcess: 36 | try: 37 | self.__buildExl(stock, workbook) 38 | except Exception: 39 | print "failed buildExl for stock %s: %s" % (stock, traceback.print_exc()) 40 | 41 | #sleep for 2 seconds, or Yahoo server will throw exception 42 | time.sleep(2) 43 | 44 | fileName = '%s%d.xls' % (self.__outputPrefix, i) 45 | print "Saved %s to %s" % (stocksToProcess, fileName) 46 | workbook.save(fileName) 47 | 48 | def buildExlsFromFile(self, fileName, div=1): 49 | ''' read a file with stock names, get data and store''' 50 | print "buildExlsFromFile %s, div %s" % (fileName, div) 51 | 52 | f = open(fileName) 53 | lines = [line.rstrip() for line in f] 54 | self.buildExls(lines, div) 55 | 56 | def __buildExl(self, stock, workbook): 57 | ''' get one stock historical data and store it ''' 58 | try: 59 | ws = workbook.add_sheet(stock) 60 | 61 | #get data 62 | yahooFinance = YahooFinance() 63 | allData = yahooFinance.getHistoricalPrices(stock, self.__startDate, self.__endDate) 64 | for col, field in enumerate(['date', 'open', 'high', 'low', 'close', 'volume', 'adjClose']): 65 | ws.write(0, col, field) 66 | 67 | for row, data in enumerate(allData): 68 | for col, field in enumerate(['date', 'open', 'high', 'low', 'close', 'volume', 'adjClose']): 69 | ws.write(row+1, col, getattr(data, field) ) 70 | 71 | except UfException as excp: 72 | raise excp 73 | except Exception: 74 | raise UfException(Errors.UNKNOWN_ERROR, "historicalStorage.__buildExl got unknown error %s" 75 | % traceback.print_exc()) -------------------------------------------------------------------------------- /deprecated/ultrafinance/lib/plotDateValueDict.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Jan 3, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from matplotlib import pyplot 7 | from datetime import datetime 8 | from ultrafinance.lib.errors import UfException, Errors 9 | 10 | import logging 11 | LOG = logging.getLogger(__name__) 12 | 13 | class PlotDateValueDict(object): 14 | ''' plot dict with date value ''' 15 | def __init__(self, dateValueDict, dateFormat='%Y', lowMargin=0.05, upMargin=0.05, rightMargin=0.05, leftMargin=0.05, betweenMargin=0.05): 16 | ''' constructor ''' 17 | self.dateValueDict = dateValueDict 18 | self.length = len(dateValueDict.keys()) 19 | self.lowMargin = lowMargin 20 | self.upMargin = upMargin 21 | self.leftMargin = leftMargin 22 | self.rightMargin = rightMargin 23 | self.dateFormat = dateFormat 24 | 25 | self.rect = [] 26 | height = float(1 - self.lowMargin - self.upMargin - (self.length-1)*betweenMargin)/self.length 27 | pre = self.lowMargin 28 | for _ in range(self.length): 29 | self.rect.append([self.leftMargin, pre, 1 - self.leftMargin - self.rightMargin , height]) 30 | pre = pre + height + betweenMargin 31 | 32 | pyplot.rc('axes', grid=True) 33 | pyplot.rc('grid', color='0.75', linestyle='-', linewidth=0.5) 34 | 35 | def plot(self): 36 | ''' plot dataValue ''' 37 | try: 38 | fig = pyplot.figure() 39 | 40 | i = 0 41 | ax0 = None 42 | for label, dateValues in self.dateValueDict.items(): 43 | if 0 == i: 44 | ax = fig.add_axes(self.rect[i]) 45 | ax0 = ax 46 | else: 47 | ax = fig.add_axes(self.rect[i], sharex=ax0) 48 | i += 1 49 | ax.plot_date([datetime.strptime(dateValue.date, self.dateFormat) for dateValue in dateValues], 50 | [dateValue.value for dateValue in dateValues], fmt='b-') 51 | ax.set_ylabel(label) 52 | ax.set_ylim(min([int(dateValue.value) for dateValue in dateValues]) /1.1, max([int(dateValue.value) for dateValue in dateValues]) * 1.1 ) 53 | #ax.set_ylim(0, 1000) 54 | 55 | pyplot.show() 56 | 57 | except UfException as excep: 58 | raise excep 59 | except BaseException as excep: 60 | raise UfException(Errors.UNKNOWN_ERROR, "plotDateValueDict.plot got unknown error %s" % excep) -------------------------------------------------------------------------------- /deprecated/ultrafinance/lib/plotPortfolio.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Feb 20, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from matplotlib import pyplot 7 | import math 8 | from ultrafinance.lib.errors import UfException, Errors 9 | 10 | import logging 11 | LOG = logging.getLogger(__name__) 12 | 13 | class PlotPortfolio(object): 14 | ''' plot portfolio curve. only two are supported ''' 15 | def __init__(self, labelReturnDeviations): 16 | ''' constructor ''' 17 | self.labelReturnDeviations = labelReturnDeviations 18 | pyplot.rc('axes', grid=True) 19 | pyplot.rc('grid', color='0.75', linestyle='-', linewidth=0.5) 20 | 21 | def plot(self): 22 | ''' plot ''' 23 | try: 24 | returns = [] 25 | deviations = [] 26 | portfolio1 = self.labelReturnDeviations[0] 27 | portfolio2 = self.labelReturnDeviations[1] 28 | 29 | for i in range(100): 30 | returns.append(portfolio1['return']*i/100 + portfolio2['return']*(100-i)/100) 31 | deviations.append(pow(portfolio1['deviation']*i/100, 2) + pow(portfolio2['deviation']*(100-i)/100, 2) + 2*(i/100)*((100-i)/100)*portfolio1['cov']) 32 | 33 | pyplot.plot([math.sqrt(deviation) for deviation in deviations], returns,'b-') 34 | pyplot.ylabel('Returns') 35 | pyplot.xlabel('Deviations') 36 | 37 | except UfException as excep: 38 | raise excep 39 | except BaseException as excep: 40 | raise UfException(Errors.UNKNOWN_ERROR, "plotPortfolio.plot got unknown error %s" % excep) 41 | 42 | def show(self): 43 | ''' 44 | show the graph. 45 | NOTICE: after show(), You can't do savefig() anymore. 46 | This is a known bug: http://code.google.com/p/ultra-finance/issues/detail?id=3 47 | ''' 48 | pyplot.show() 49 | 50 | def savefig(self, fileName): 51 | LOG.debug('Save portfolio to %s' % fileName) 52 | pyplot.savefig(fileName) -------------------------------------------------------------------------------- /deprecated/ultrafinance/lib/stockMeasurement.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Jan 3, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import copy 7 | import numpy 8 | 9 | from ultrafinance.lib.yahooFinance import YahooFinance 10 | from ultrafinance.lib.errors import UfException, Errors 11 | 12 | import logging 13 | LOG = logging.getLogger(__name__) 14 | 15 | class StockMeasurement(): 16 | ''' measurement of a single stock/index ''' 17 | def __init__(self, dateValues, benchmark='^GSPC', benchmarkValues=None): 18 | ''' constructor ''' 19 | self.__dateValues = dateValues 20 | self.__benchmark = benchmark 21 | self.__benchmarkValues = copy.deepcopy(benchmarkValues) 22 | self.__alpha = None 23 | self.__beta = None 24 | self.__regressioned = False 25 | 26 | def mean(self): 27 | ''' get average ''' 28 | return numpy.mean([float(dateValues.close) for dateValues in self.__dateValues], axis=0) 29 | 30 | def std(self): 31 | ''' get standard deviation ''' 32 | return numpy.std([float(dateValues.close) for dateValues in self.__dateValues], axis=0) 33 | 34 | def linearRegression(self): 35 | if self.__regressioned: 36 | return 37 | 38 | if not self.__benchmarkValues: 39 | self.__benchmarkValues = YahooFinance().getHistoricalPrices(self.__benchmark, self.__dateValues[0].date, self.__dateValues[-1].date) 40 | 41 | tradeSuspended = False 42 | if 0 in map(lambda x: float(x.adjClose), self.__dateValues): 43 | tradeSuspended = True 44 | 45 | #filter out date tha't not in both stock and benchmark 46 | dateSet = set([dateValue.date for dateValue in self.__dateValues]) & set([dateValue.date for dateValue in self.__benchmarkValues]) 47 | self.__dateValues = filter(lambda dateValue: dateValue.date in dateSet, self.__dateValues) 48 | self.__benchmarkValues = filter(lambda dateValue: dateValue.date in dateSet, self.__benchmarkValues) 49 | 50 | if len(self.__dateValues) <= 1 or tradeSuspended: 51 | msg = "Not enought dateValues" if len(self.__dateValues) <= 1 else "trade suspended" 52 | LOG.debug(msg) 53 | 54 | self.__beta = 0 55 | self.__alpha = 0 56 | self.__regressioned = True 57 | return 58 | 59 | try: 60 | x = [float(self.__benchmarkValues[index + 1].adjClose)/float(self.__benchmarkValues[index].adjClose) for index in range(len(self.__benchmarkValues) - 1)] 61 | y = [float(self.__dateValues[index + 1].adjClose)/float(self.__dateValues[index].adjClose) for index in range(len(self.__dateValues) - 1)] 62 | (self.__beta, self.__alpha) = numpy.polyfit(x, y, 1) 63 | self.__regressioned = True 64 | except BaseException as excep: 65 | raise UfException(Errors.UNKNOWN_ERROR, "stockMeasurement.linearRegression got unknown error %s" % excep) 66 | 67 | def marketReturnRate(self): 68 | if not self.__regressioned: 69 | self.linearRegression() 70 | return (float(self.__benchmarkValues[-1].adjClose) - float(self.__benchmarkValues[0].adjClose)) / float(self.__benchmarkValues[0].adjClose) \ 71 | if self.__benchmarkValues and float(self.__benchmarkValues[0].adjClose) \ 72 | else 0 73 | 74 | def returnRate(self): 75 | return (float(self.__dateValues[-1].adjClose) - float(self.__dateValues[0].adjClose)) / float(self.__dateValues[0].adjClose) \ 76 | if self.__dateValues and float(self.__dateValues[0].adjClose) \ 77 | else 0 78 | 79 | def relativeReturnRate(self): 80 | return self.returnRate() - self.marketReturnRate() 81 | 82 | def alpha(self): 83 | if not self.__regressioned: 84 | self.linearRegression() 85 | return self.__alpha 86 | 87 | def beta(self): 88 | if not self.__regressioned: 89 | self.linearRegression() 90 | return self.__beta 91 | -------------------------------------------------------------------------------- /deprecated/ultrafinance/lib/tradingStrategy/__init__.py: -------------------------------------------------------------------------------- 1 | ''' trading strategy ''' 2 | import os, sys 3 | 4 | mainSrc = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 5 | sys.path.append(mainSrc) -------------------------------------------------------------------------------- /deprecated/ultrafinance/lib/tradingStrategy/automaticInvestmentPlan.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Feb 26, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import logging 7 | LOG = logging.getLogger(__name__) 8 | 9 | def fixAmountPerPeriod(dateValues, interval): 10 | ''' fix amount investment for each time period''' 11 | amount = float(100) 12 | share = sum([amount/dateValues[index].value for index in range(len(dateValues)/interval)]) 13 | return share * dateValues[-1].value / (amount * len(dateValues) / interval) 14 | 15 | def fixAmountPerPeriodWithAddtionWhenDrop(dateValues, interval, slidingWindow): 16 | ''' fix amount investment for each time period''' 17 | amount = float(100) 18 | fixShare = sum([amount/dateValues[index].value for index in range(len(dateValues)/interval)]) 19 | #adjustShare = sum([amount/dateValues[index].value for index in range(len(dateValues)) if index >= slidingWindow and min([dateValue.value for dateValue in dateValues[index-slidingWindow:index+1]]) == dateValues[index]]) 20 | adjustShare = sum([amount/dateValues[index].value for index in range(len(dateValues)) if index >= slidingWindow and min([dateValue.value for dateValue in dateValues[index-slidingWindow:index+1]]) == dateValues[index].value]) 21 | share = adjustShare + fixShare 22 | return share * dateValues[-1].value / (amount * len(dateValues) / interval) 23 | 24 | def adjustFixAmountPerPeriod(dateValues, interval, slidingWindow): 25 | ''' fix amount investment for each time period''' 26 | amount = float(100) 27 | share = 0 28 | for frame in range(len(dateValues)/interval): 29 | bought = False 30 | for i in range(interval): 31 | index = frame*interval + i 32 | if index >= slidingWindow and min([dateValue.value for dateValue in dateValues[index-slidingWindow:index+1]]) == dateValues[index].value: 33 | share += amount/dateValues[index].value 34 | bought = True 35 | break 36 | 37 | if not bought: 38 | share += amount/dateValues[(frame+1) * interval - 1].value 39 | 40 | return share * dateValues[-1].value / (amount * len(dateValues) / interval) -------------------------------------------------------------------------------- /deprecated/ultrafinance/lib/tradingStrategy/tradeInWeek.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Mar 27, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from datetime import date 7 | from operator import itemgetter 8 | from ultrafinance.lib.excelLib import ExcelLib 9 | 10 | import logging 11 | LOG = logging.getLogger(__name__) 12 | 13 | class TradeInWeek(): 14 | ''' 15 | trade in one week. Which day to buy and sell is best 16 | ''' 17 | WEEKDAY = ['SUN.', 'MON.', 'TUE.', 'WED.', 'THU.', 'FRI.'] 18 | def __init__(self, fileName, sheetNumber = 0): 19 | ''' Constructor ''' 20 | self.returnRate = {} 21 | self.fileName = fileName 22 | self.sheetNumber = sheetNumber 23 | 24 | def tryAllStrategy(self): 25 | ''' preparing data''' 26 | for buyWeekDate in range(1, 6): 27 | for sellWeekDate in range(1, 6): 28 | self.returnRate[(TradeInWeek.WEEKDAY[buyWeekDate], TradeInWeek.WEEKDAY[sellWeekDate])] = \ 29 | '%.2f' % self.oneStrategy(buyWeekDate, sellWeekDate) 30 | 31 | return sorted(self.returnRate.items(), key=itemgetter(1)) 32 | 33 | def oneStrategy(self, buyWeekDate, sellWeekDate): 34 | totalReturn = 0 35 | with ExcelLib(self.fileName, self.sheetNumber) as excel: 36 | stockDates = excel.readCol(0, 1, -1) 37 | open = excel.readCol(1, 1, -1) 38 | close = excel.readCol(4, 1, -1) 39 | 40 | for i, stockDate in enumerate([date(int(stockDate.split('-')[0]), int(stockDate.split('-')[1]), int(stockDate.split('-')[2])) 41 | for stockDate in stockDates]): 42 | if buyWeekDate == stockDate.isoweekday(): 43 | for holdDay in range(1, 5): 44 | if i + holdDay < len(stockDates): 45 | stockDate = stockDates[i + holdDay] 46 | if sellWeekDate == date(int(stockDate.split('-')[0]), int(stockDate.split('-')[1]), int(stockDate.split('-')[2])).isoweekday(): 47 | #print float(close[i + holdDay]) - 48 | totalReturn += float(close[i + holdDay]) 49 | totalReturn -= float(open[i]) 50 | 51 | return totalReturn 52 | 53 | if __name__ == '__main__': 54 | tradeInWeek = TradeInWeek('../../dataSource/SPY/SPYINDEX0.xls', 0) 55 | tradeInWeek.tryAllStrategy() 56 | print tradeInWeek.returnRate -------------------------------------------------------------------------------- /deprecated/ultrafinance/lib/tradingStrategyFactory.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Feb 26, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.lib.errors import UfException, Errors 7 | import traceback 8 | 9 | import logging 10 | LOG = logging.getLogger() 11 | 12 | class TradingStrategyFactory(): 13 | ''' Factory method for trading Strategies ''' 14 | def __init__(self, strategy): 15 | ''' constructor ''' 16 | self.strategy = strategy 17 | 18 | def calculateReturn(self, dateValues, *args): 19 | try: 20 | return self.strategy(dateValues, *args) 21 | 22 | except UfException as excep: 23 | raise excep 24 | except BaseException as excep: 25 | raise UfException(Errors.UNKNOWN_ERROR, "tradingStrategyFactory.calculateReturn got unknown error %s" % traceback.print_exc()) -------------------------------------------------------------------------------- /deprecated/ultrafinance/lib/util.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 18, 2010 3 | 4 | @author: ppa 5 | ''' 6 | import sys 7 | from BeautifulSoup import BeautifulSoup 8 | from datetime import datetime 9 | import time 10 | 11 | import logging 12 | LOG = logging.getLogger() 13 | 14 | googCSVDateformat = "%d-%b-%y" 15 | 16 | def import_class(path, fileName, className=None): 17 | ''' dynamically import class ''' 18 | if not className: 19 | className = capitalize(fileName) 20 | sys.path.append(path) 21 | 22 | mod = __import__(fileName) 23 | return getattr(mod, className) 24 | 25 | def capitalize(inputString): 26 | ''' capitalize first letter ''' 27 | return inputString[0].upper() + inputString[1:] if len(inputString) > 1 else inputString[0].upper() 28 | 29 | def deCapitalize(inputString): 30 | ''' de-capitalize first letter ''' 31 | return inputString[0].lower() + inputString[1:] if len(inputString) > 1 else inputString[0].lower() 32 | 33 | def splitByComma(inputString): 34 | ''' split string by comma ''' 35 | return [name.strip() for name in inputString.split(',')] 36 | 37 | def convertGoogCSVDate(googCSVDate): 38 | ''' convert date 25-Jul-2010 to 20100725''' 39 | d = str(datetime.strptime(googCSVDate, googCSVDateformat).date() ) 40 | return d.replace("-", "") 41 | 42 | def findPatthen(page, pList): 43 | datas = [BeautifulSoup(page)] 44 | index = 0 45 | for key, pattern in pList: 46 | newDatas = [] 47 | for data in datas: 48 | if 'id' == key: 49 | newDatas.extend(data.findAll(id=pattern, recursive=True)) 50 | if 'text' == key: 51 | newDatas.extend(data.findAll(text=pattern, recursive=True)) 52 | 53 | datas = newDatas 54 | index += 1 55 | if not datas: 56 | break 57 | 58 | return datas 59 | 60 | def string2EpochTime(stingTime, format='%Y%m%d'): 61 | ''' convert string time to epoch time ''' 62 | return int(time.mktime(datetime.strptime(stingTime, '%Y%m%d').timetuple())) 63 | 64 | def string2datetime(stringTime, format='%Y%m%d'): 65 | ''' convert string time to epoch time''' 66 | return datetime.strptime(stringTime, '%Y%m%d') 67 | -------------------------------------------------------------------------------- /deprecated/ultrafinance/lib/yahooFinance.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 21, 2010 3 | 4 | @author: ppa 5 | 6 | Thanks to Corey Goldberg, this module is based http://www.goldb.org/ystockquote.html 7 | sample usage: 8 | >>> import YahooFinance 9 | >>> print YahooFinance.get_price('GOOG') 10 | 529.46 11 | ''' 12 | import urllib 13 | import traceback 14 | from operator import itemgetter 15 | from ultrafinance.lib.dataType import StockDailyType 16 | from ultrafinance.lib.errors import UfException, Errors 17 | 18 | import logging 19 | LOG = logging.getLogger(__name__) 20 | 21 | class YahooFinance(object): 22 | def __request(self, symbol, stat): 23 | try: 24 | url = 'http://finance.yahoo.com/d/quotes.csv?s=%s&f=%s' % (symbol, stat) 25 | return urllib.urlopen(url).read().strip().strip('"') 26 | except IOError: 27 | raise UfException(Errors.NETWORK_ERROR, "Can't connect to Yahoo server") 28 | except BaseException: 29 | raise UfException(Errors.UNKNOWN_ERROR, "Unknown Error in YahooFinance.__request %s" % traceback.format_exc()) 30 | 31 | def get_all(self, symbol): 32 | """ 33 | Get all available quote data for the given ticker symbol. 34 | Returns a dictionary. 35 | """ 36 | values = self.__request(symbol, 'l1c1va2xj1b4j4dyekjm3m4rr5p5p6s7').split(',') 37 | data = {} 38 | data['price'] = values[0] 39 | data['change'] = values[1] 40 | data['volume'] = values[2] 41 | data['avg_daily_volume'] = values[3] 42 | data['stock_exchange'] = values[4] 43 | data['market_cap'] = values[5] 44 | data['book_value'] = values[6] 45 | data['ebitda'] = values[7] 46 | data['dividend_per_share'] = values[8] 47 | data['dividend_yield'] = values[9] 48 | data['earnings_per_share'] = values[10] 49 | data['52_week_high'] = values[11] 50 | data['52_week_low'] = values[12] 51 | data['50day_moving_avg'] = values[13] 52 | data['200day_moving_avg'] = values[14] 53 | data['price_earnings_ratio'] = values[15] 54 | data['price_earnings_growth_ratio'] = values[16] 55 | data['price_sales_ratio'] = values[17] 56 | data['price_book_ratio'] = values[18] 57 | data['short_ratio'] = values[19] 58 | return data 59 | 60 | def get_price(self, symbol): 61 | return self.__request(symbol, 'l1') 62 | 63 | def get_change(self, symbol): 64 | return self.__request(symbol, 'c1') 65 | 66 | def get_volume(self, symbol): 67 | return self.__request(symbol, 'v') 68 | 69 | def get_avg_daily_volume(self, symbol): 70 | return self.__request(symbol, 'a2') 71 | 72 | def get_stock_exchange(self, symbol): 73 | return self.__request(symbol, 'x') 74 | 75 | def get_market_cap(self, symbol): 76 | return self.__request(symbol, 'j1') 77 | 78 | def get_book_value(self, symbol): 79 | return self.__request(symbol, 'b4') 80 | 81 | def get_ebitda(self, symbol): 82 | return self.__request(symbol, 'j4') 83 | 84 | def get_dividend_per_share(self, symbol): 85 | return self.__request(symbol, 'd') 86 | 87 | def get_dividend_yield(self, symbol): 88 | return self.__request(symbol, 'y') 89 | 90 | def get_earnings_per_share(self, symbol): 91 | return self.__request(symbol, 'e') 92 | 93 | def get_52_week_high(self, symbol): 94 | return self.__request(symbol, 'k') 95 | 96 | def get_52_week_low(self, symbol): 97 | return self.__request(symbol, 'j') 98 | 99 | def get_50day_moving_avg(self, symbol): 100 | return self.__request(symbol, 'm3') 101 | 102 | def get_200day_moving_avg(self, symbol): 103 | return self.__request(symbol, 'm4') 104 | 105 | def get_price_earnings_ratio(self, symbol): 106 | return self.__request(symbol, 'r') 107 | 108 | def get_price_earnings_growth_ratio(self, symbol): 109 | return self.__request(symbol, 'r5') 110 | 111 | def get_price_sales_ratio(self, symbol): 112 | return self.__request(symbol, 'p5') 113 | 114 | def get_price_book_ratio(self, symbol): 115 | return self.__request(symbol, 'p6') 116 | 117 | def get_short_ratio(self, symbol): 118 | return self.__request(symbol, 's7') 119 | 120 | def getHistoricalPrices(self, symbol, start_date, end_date): 121 | """ 122 | Get historical prices for the given ticker symbol. 123 | Date format is 'YYYY-MM-DD' 124 | 125 | Returns a nested list. 126 | """ 127 | try: 128 | start_date = str(start_date).replace('-', '') 129 | end_date = str(end_date).replace('-', '') 130 | 131 | url = 'http://ichart.yahoo.com/table.csv?s=%s&' % symbol + \ 132 | 'd=%s&' % str(int(end_date[4:6]) - 1) + \ 133 | 'e=%s&' % str(int(end_date[6:8])) + \ 134 | 'f=%s&' % str(int(end_date[0:4])) + \ 135 | 'g=d&' + \ 136 | 'a=%s&' % str(int(start_date[4:6]) - 1) + \ 137 | 'b=%s&' % str(int(start_date[6:8])) + \ 138 | 'c=%s&' % str(int(start_date[0:4])) + \ 139 | 'ignore=.csv' 140 | days = urllib.urlopen(url).readlines() 141 | values = [day[:-2].split(',') for day in days] 142 | # sample values:[['Date', 'Open', 'High', 'Low', 'Close', 'Volume', 'Adj Clos'], \ 143 | # ['2009-12-31', '112.77', '112.80', '111.39', '111.44', '90637900', '109.7']...] 144 | data = [] 145 | for value in values[1:]: 146 | data.append(StockDailyType(value[0], value[1], value[2], value[3], value[4], value[5], value[6])) 147 | 148 | dateValues = sorted(data, key=itemgetter(0)) 149 | return dateValues 150 | 151 | except IOError: 152 | raise UfException(Errors.NETWORK_ERROR, "Can't connect to Yahoo server") 153 | except BaseException as excep: 154 | raise UfException(Errors.UNKNOWN_ERROR, "Unknown Error in YahooFinance.getHistoricalPrices %s" % excep) 155 | #sample output 156 | #[stockDaylyData(date='2010-01-04, open='112.37', high='113.39', low='111.51', close='113.33', volume='118944600', adjClose='111.6'))...] -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/__init__.py: -------------------------------------------------------------------------------- 1 | ''' processChain ''' -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/baseModule.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 18, 2010 3 | 4 | @author: ppa 5 | ''' 6 | import abc 7 | from threading import Thread 8 | from pydispatch.dispatcher import send 9 | from ultrafinance.lib.util import deCapitalize 10 | 11 | import logging 12 | LOG = logging.getLogger(__name__) 13 | 14 | class BaseModule(object): 15 | ''' 16 | Base class for a feeder, processor and outputer 17 | ''' 18 | __metaclass__ = abc.ABCMeta 19 | 20 | def __init__(self): 21 | ''' constructor''' 22 | self.input = None 23 | self.output = None 24 | 25 | def before(self): 26 | ''' init operation for processing data''' 27 | pass 28 | 29 | def after(self): 30 | ''' operation for cleaing up(e.g. close connection, database ...) ''' 31 | signal = deCapitalize(self.__class__.__name__) 32 | print 'send out signal %s' % signal 33 | send(signal, self, input=self.output) 34 | 35 | def execute(self, input): 36 | ''' processsing data''' 37 | self.input = input 38 | 39 | def run(self, input): 40 | ''' full execution''' 41 | print 'running %s' % self.__class__.__name__ 42 | def runFunc(input): 43 | self.before() 44 | self.output = self.execute(input) 45 | self.after() 46 | 47 | thread = Thread(target=runFunc, args=(input,)) 48 | thread.start() 49 | thread.join() 50 | 51 | def __getName(self): 52 | ''' retrun name ''' 53 | return self.__class__.__name__ 54 | 55 | name = property(__getName) -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/config/historicalStock.ini: -------------------------------------------------------------------------------- 1 | [app_main] 2 | feeder=historicalDataFeeder 3 | processor=defaultProcessor 4 | outputer=plotStockOutputer 5 | 6 | [historicalDataFeeder_mapping] 7 | receiver=defaultProcessor 8 | 9 | [defaultProcessor_mapping] 10 | receiver=plotStockOutputer 11 | 12 | [log] 13 | file=ultraFinance.log -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/config/interestHoursingStock.ini: -------------------------------------------------------------------------------- 1 | [app_main] 2 | feeder=excelDataFeeder 3 | processor=defaultProcessor 4 | outputer=plotYearlyOutputer 5 | 6 | [excelDataFeeder_mapping] 7 | receiver=defaultProcessor 8 | 9 | [defaultProcessor_mapping] 10 | receiver=plotYearlyOutputer 11 | 12 | [log] 13 | file=ultraFinance.log -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/config/tradingInWeekStrategy.ini: -------------------------------------------------------------------------------- 1 | [app_main] 2 | feeder=historicalDataFeeder 3 | processor=defaultProcessor 4 | outputer=plotStockOutputer 5 | 6 | [historicalDataFeeder_mapping] 7 | receiver=defaultProcessor 8 | 9 | [defaultProcessor_mapping] 10 | receiver=plotStockOutputer 11 | 12 | [log] 13 | file=ultraFinance.log -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/config/usTradingStrategy.ini: -------------------------------------------------------------------------------- 1 | [app_main] 2 | feeder=indexDataFeeder 3 | processor=tradingStrategyProcessor 4 | outputer=defaultOutputer 5 | 6 | [indexDataFeeder_mapping] 7 | receiver=tradingStrategyProcessor 8 | 9 | [tradingStrategyProcessor_mapping] 10 | receiver=defaultOutputer 11 | 12 | [log] 13 | file=ultraFinance.log -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/configuration.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 18, 2010 3 | 4 | @author: ppa 5 | ''' 6 | import ConfigParser 7 | from os import path 8 | 9 | import logging 10 | LOG = logging.getLogger(__name__) 11 | 12 | class Configuration(object): 13 | ''' class that handles configuration ''' 14 | def __init__(self, configFileName=None): 15 | ''' Constructor ''' 16 | if configFileName: 17 | self.configFilePath = path.join(path.dirname(path.abspath(__file__)), 'config', configFileName) 18 | else: 19 | self.configFilePath = path.join(path.dirname(path.abspath(__file__)), 'config', 'dev.ini') 20 | 21 | print "Using configure file: %s" % self.configFilePath 22 | 23 | def getConfiguration(self, section): 24 | ''' load all configuration ''' 25 | configs = {} 26 | parser = ConfigParser.SafeConfigParser() 27 | parser.read(self.configFilePath) 28 | if parser.has_section(section): 29 | for name, value in parser.items(section): 30 | configs[name] = value 31 | return configs 32 | else: 33 | return None 34 | 35 | def getOption(self, section, option): 36 | ''' whether a option exists in the section ''' 37 | parser = ConfigParser.SafeConfigParser() 38 | parser.read(self.configFilePath) 39 | if parser.has_option(section, option): 40 | return parser.get(section, option) 41 | else: 42 | return None 43 | 44 | def getAppMain(self): 45 | ''' get app_main ''' 46 | self.getConfiguration('app_main') 47 | #app_global = Configuration().getConfiguration("app_main") 48 | 49 | if __name__ == '__main__': 50 | print Configuration().getOption('app_main', 'feeder') -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/feeder/__init__.py: -------------------------------------------------------------------------------- 1 | """ feeder package """ -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/feeder/defaultFeeder.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 18, 2010 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.processChain.baseModule import BaseModule 7 | 8 | import logging 9 | LOG = logging.getLogger(__name__) 10 | 11 | class DefaultFeeder(BaseModule): 12 | ''' Default feeder ''' 13 | def __init__(self): 14 | ''' Constructor ''' 15 | super(DefaultFeeder, self).__init__() 16 | 17 | def execute(self, input): 18 | ''' preparing data''' 19 | super(DefaultFeeder, self).execute(input) 20 | data = {'defaultStock':('12/18/2010', '$20')} 21 | return data -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/feeder/excelDataFeeder.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Jan 30, 2010 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.lib.excelLib import ExcelLib 7 | from ultrafinance.processChain.baseModule import BaseModule 8 | from ultrafinance.lib.dataType import DateValueType 9 | from os.path import join 10 | 11 | import logging 12 | LOG = logging.getLogger(__name__) 13 | 14 | class ExcelDataFeeder(BaseModule): 15 | ''' 16 | feeder that get stock, hoursing and interest rate from excel 17 | ''' 18 | def __init__(self): 19 | ''' Constructor ''' 20 | super(ExcelDataFeeder, self).__init__() 21 | self.stockData = [] 22 | self.hoursingData = [] 23 | self.interestData = [] 24 | 25 | def execute(self, input): 26 | ''' preparing data''' 27 | with ExcelLib(join('dataSource', 'hoursing_interestRate.xls'), 0) as excel: 28 | year = excel.readCol(0, 7, 127) 29 | hoursing = excel.readCol(1, 7, 127) 30 | interest = excel.readCol(5, 7, 127) 31 | for i in range(len(year)): 32 | self.hoursingData.append(DateValueType(str(int(year[i])), hoursing[i])) 33 | self.interestData.append(DateValueType(str(int(year[i])), interest[i])) 34 | 35 | with ExcelLib(join('dataSource', 'longTerm_1871.xls'), 0) as excel: 36 | year = excel.readCol(0, 8, 147) 37 | stock = excel.readCol(1, 8, 147) 38 | for i in range(len(year)): 39 | self.stockData.append(DateValueType(str(int(year[i])), stock[i])) 40 | 41 | ret = {'stock': self.stockData, 'hoursing': self.hoursingData, 'interest': self.interestData} 42 | return ret 43 | 44 | if __name__ == '__main__': 45 | feeder = ExcelDataFeeder() 46 | feeder.execute("") 47 | print feeder.hoursingData 48 | print feeder.interestData 49 | print feeder.stockData -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/feeder/historicalDataFeeder.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 18, 2010 3 | 4 | @author: ppa 5 | ''' 6 | from datetime import date 7 | 8 | from ultrafinance.lib.yahooFinance import YahooFinance 9 | from ultrafinance.processChain.baseModule import BaseModule 10 | 11 | import logging 12 | LOG = logging.getLogger(__name__) 13 | 14 | class HistoricalDataFeeder(BaseModule): 15 | ''' 16 | feeder that get stock data from Yahoo Finance 17 | ''' 18 | def __init__(self): 19 | ''' Constructor ''' 20 | super(HistoricalDataFeeder, self).__init__() 21 | self.__yahooFinance = YahooFinance() 22 | 23 | def execute(self, input): 24 | ''' preparing data''' 25 | super(HistoricalDataFeeder, self).execute(input) 26 | return {'history stock': self.__yahooFinance.getHistoricalPrices(input, '1990-01-01', date.today())} -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/feeder/indexDataFeeder.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Jan 30, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.lib.excelLib import ExcelLib 7 | from ultrafinance.processChain.baseModule import BaseModule 8 | from ultrafinance.lib.dataType import DateValueType 9 | from os.path import join 10 | 11 | import logging 12 | LOG = logging.getLogger(__name__) 13 | 14 | class IndexDataFeeder(BaseModule): 15 | ''' 16 | feeder that get stock, hoursing and interest rate from excel 17 | ''' 18 | def __init__(self): 19 | ''' Constructor ''' 20 | super(IndexDataFeeder, self).__init__() 21 | self.stockData = [] 22 | 23 | def execute(self, input): 24 | ''' preparing data''' 25 | with ExcelLib(join('dataSource', 'longTerm_1871.xls'), 0) as excel: 26 | year = excel.readCol(0, 8, 147) 27 | stock = excel.readCol(1, 8, 147) 28 | for i in range(len(year)): 29 | self.stockData.append(DateValueType(str(int(year[i])), stock[i])) 30 | 31 | ret = self.stockData 32 | return ret 33 | 34 | if __name__ == '__main__': 35 | feeder = IndexDataFeeder() 36 | feeder.execute("") 37 | print feeder.stockData -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/outputer/__init__.py: -------------------------------------------------------------------------------- 1 | """ outputer package """ -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/outputer/defaultOutputer.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 18, 2010 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.processChain.baseModule import BaseModule 7 | 8 | import logging 9 | LOG = logging.getLogger(__name__) 10 | 11 | class DefaultOutputer(BaseModule): 12 | ''' Default feeder ''' 13 | def __init__(self): 14 | ''' constructor ''' 15 | super(DefaultOutputer, self).__init__() 16 | 17 | def execute(self, input): 18 | ''' do output''' 19 | super(DefaultOutputer, self).execute(input) 20 | print input -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/outputer/emailOutputer.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 19, 2010 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.processChain.baseModule import BaseModule 7 | 8 | from email.MIMEMultipart import MIMEMultipart 9 | from email.MIMEText import MIMEText 10 | import json, smtplib 11 | 12 | import logging 13 | LOG = logging.getLogger(__name__) 14 | 15 | gmail_user = "your@gmail.com" 16 | gmail_pwd = "your_password" 17 | 18 | def mail(to, text): 19 | msg = MIMEMultipart() 20 | msg['From'] = gmail_user 21 | msg['To'] = to 22 | msg['Subject'] = 'Ultra-Finance' 23 | 24 | msg.attach(MIMEText(text)) 25 | mailServer = smtplib.SMTP("smtp.gmail.com", 587) 26 | mailServer.ehlo() 27 | mailServer.starttls() 28 | mailServer.ehlo() 29 | mailServer.login(gmail_user, gmail_pwd) 30 | mailServer.sendmail(gmail_user, to, msg.as_string()) 31 | # Should be mailServer.quit(), but that crashes... 32 | mailServer.close() 33 | 34 | class EmailOutputer(BaseModule): 35 | ''' Default feeder ''' 36 | def __init__(self): 37 | ''' constructor ''' 38 | super(EmailOutputer, self).__init__() 39 | 40 | def execute(self, input): 41 | ''' do output''' 42 | super(EmailOutputer, self).execute(input) 43 | print 'sending email' 44 | mail("des@gmail.com", json.dumps(input)) 45 | return None -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/outputer/plotStockOutputer.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 18, 2010 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.processChain.baseModule import BaseModule 7 | from matplotlib import pyplot 8 | from datetime import datetime 9 | import pylab 10 | 11 | import logging 12 | LOG = logging.getLogger(__name__) 13 | 14 | class PlotStockOutputer(BaseModule): 15 | ''' Default feeder ''' 16 | def __init__(self): 17 | ''' constructor ''' 18 | super(PlotStockOutputer, self).__init__() 19 | 20 | def execute(self, dateValuesDict): 21 | ''' do output ''' 22 | super(PlotStockOutputer, self).execute(input) 23 | #pyplot.ion() # turns interactive mode on 24 | fig = pylab.figure() 25 | ax = fig.gca() 26 | 27 | # Plotting here ... 28 | for dateValues in dateValuesDict.values(): 29 | ax.plot_date([datetime.strptime(dateValue.date, '%Y-%m-%d') for dateValue in dateValues], 30 | [dateValue.adjClose for dateValue in dateValues], fmt='b-') 31 | pylab.grid() 32 | pyplot.show() -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/outputer/plotYearlyOutputer.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 18, 2010 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.processChain.baseModule import BaseModule 7 | from ultrafinance.lib.plotDateValueDict import PlotDateValueDict 8 | 9 | import logging 10 | LOG = logging.getLogger(__name__) 11 | 12 | class PlotYearlyOutputer(BaseModule): 13 | ''' Default feeder ''' 14 | def __init__(self): 15 | ''' constructor ''' 16 | super(PlotYearlyOutputer, self).__init__() 17 | 18 | def execute(self, dateValuesDict): 19 | ''' do output ''' 20 | super(PlotYearlyOutputer, self).execute(input) 21 | p = PlotDateValueDict(dateValuesDict) 22 | p.plot() -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/pluginManager.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 25, 2010 3 | 4 | @author: ppa 5 | ''' 6 | 7 | import os 8 | import fnmatch 9 | from pydispatch.dispatcher import connect, send 10 | from ultrafinance.lib.util import import_class, splitByComma 11 | 12 | import logging 13 | LOG = logging.getLogger(__name__) 14 | 15 | class PluginManager(object): 16 | ''' plugin manager that init and run all plugins ''' 17 | groupNames = ['feeder', 'processor', 'outputer'] 18 | 19 | def __init__(self, configure): 20 | ''' constructor ''' 21 | self.plugins = {} 22 | self.configure = configure 23 | for groupName in PluginManager.groupNames: 24 | self.plugins[groupName] = {} 25 | 26 | def __loadGroupPlugins(self, groupName): 27 | ''' load a group of plugins under a folder ''' 28 | path = os.path.join(os.path.dirname(os.path.abspath(__file__)), groupName) 29 | print "Loading plugins under %s" % path 30 | for pluginName in [os.path.splitext(filename)[0] for filename in fnmatch.filter(os.listdir(path), '*.py')]: 31 | # __init__ will not be loaded 32 | if '__init__' != pluginName: 33 | print "Plugin loaded: %s" % pluginName 34 | self.plugins[groupName][pluginName] = import_class(path, pluginName)() 35 | 36 | def runFeederPlugins(self): 37 | print 'Start running feeders......' 38 | feederStrings = self.configure.getOption('app_main', 'feeder') 39 | if not feederStrings: 40 | print "No feeders provided: %s" % feederStrings 41 | 42 | for feederName in splitByComma(feederStrings): 43 | if feederName in self.plugins['feeder']: 44 | self.runPlugin('feeder', feederName) 45 | 46 | def setup(self): 47 | ''' setup plugins ''' 48 | self.loadPlugins() 49 | self.setupDispatchers(self.configure) 50 | 51 | def loadPlugins(self): 52 | ''' load all plugins ''' 53 | print "Start loading plugins....." 54 | for groupName in PluginManager.groupNames: 55 | self.__loadGroupPlugins(groupName) 56 | 57 | def setupDispatchers(self, configure): 58 | ''' setup dispatchers ''' 59 | print "Start setting up dispatcher......" 60 | for groupName in PluginManager.groupNames: 61 | for pluginName in self.plugins[groupName]: 62 | receiverStrings = configure.getOption(pluginName + '_mapping', 'receiver') 63 | if receiverStrings: 64 | self.__setup_dispatcher(groupName, pluginName, splitByComma(receiverStrings)) 65 | 66 | def __setup_dispatcher(self, groupName, pluginName, receiverNames): 67 | ''' setup dispatcher according to config file ''' 68 | if 'outputer' != groupName: #can not be the last one in groupNames 69 | gIndex = PluginManager.groupNames.index(groupName) 70 | nextGroupName = PluginManager.groupNames[gIndex + 1] 71 | for receiverName in receiverNames: 72 | if pluginName in self.plugins[groupName] and receiverName in self.plugins[nextGroupName]: 73 | print 'connect %s to receiver %s' % (pluginName, receiverName) 74 | connect( 75 | self.plugins[nextGroupName][receiverName].run, 76 | signal = pluginName, 77 | sender = self.plugins[groupName][pluginName] 78 | ) 79 | else: 80 | print "Can't set dispatcher for outputer plugin: %s" % pluginName 81 | 82 | def runPlugin(self, groupName, pluginName): 83 | ''' run plugin ''' 84 | print "Running plugin: %s" % pluginName 85 | self.plugins[groupName][pluginName].run(self.plugins[groupName][pluginName].input) 86 | 87 | #def triggerDispatcher(self, groupName, pluginName): 88 | # ''' trigger dispatcher ''' 89 | # print "Trigger dispatcher: %s" % pluginName 90 | # send(pluginName, pluginName, input = self.plugins[groupName][pluginName].input) 91 | 92 | def setInput(self, groupName, pluginName, input): 93 | self.plugins[groupName][pluginName].input = input -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/processChain.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 18, 2010 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.processChain.pluginManager import PluginManager 7 | from ultrafinance.processChain.configuration import Configuration 8 | import threading 9 | import time 10 | 11 | import logging 12 | LOG = logging.getLogger(__name__) 13 | 14 | #TODO: use singleton pattern 15 | class ProcessChain(threading.Thread): 16 | ''' class processChain ''' 17 | def __init__(self, configFile): 18 | ''' constructor ''' 19 | super(ProcessChain, self).__init__() 20 | self.configure = Configuration(configFile) 21 | #self.configure = Configuration() 22 | self.pluginManager = PluginManager(self.configure) 23 | 24 | def run(self): 25 | ''' thread run function ''' 26 | self.pluginManager.setup() 27 | self.pluginManager.setInput('feeder', 'historicalDataFeeder', 'GOOG') 28 | self.pluginManager.runFeederPlugins() 29 | 30 | def runProcessChain(configFile): 31 | ''' run process chain ''' 32 | print "Using configuration file %s" % configFile 33 | processChain = ProcessChain(configFile) 34 | processChain.start() -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/processor/__init__.py: -------------------------------------------------------------------------------- 1 | """ processor package """ -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/processor/avgDivProcessor.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 18, 2010 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.processChain.baseModule import BaseModule 7 | from ultrafinance.lib.stockMeasurement import StockMeasurement 8 | 9 | import logging 10 | LOG = logging.getLogger(__name__) 11 | 12 | class AvgDivProcessor(BaseModule): 13 | ''' Calculate average and standard deviation ''' 14 | def __init__(self): 15 | ''' constructor ''' 16 | super(AvgDivProcessor, self).__init__() 17 | 18 | def execute(self, dateValuesDict): 19 | ''' processing input''' 20 | super(AvgDivProcessor, self).execute(dateValuesDict) 21 | for dateValues in dateValuesDict.values(): 22 | stockMeasurement = StockMeasurement(dateValues) 23 | data = {'days': len(dateValues), 'avg': stockMeasurement.mean(), 'standard deviation': stockMeasurement.std(), 24 | 'alpha': stockMeasurement.alpha(), 'beta': stockMeasurement.beta()} 25 | return data -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/processor/defaultProcessor.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 18, 2010 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.processChain.baseModule import BaseModule 7 | 8 | import logging 9 | LOG = logging.getLogger(__name__) 10 | 11 | class DefaultProcessor(BaseModule): 12 | def __init__(self): 13 | ''' constructor ''' 14 | super(DefaultProcessor, self).__init__() 15 | 16 | def execute(self, input): 17 | ''' processing data''' 18 | super(DefaultProcessor, self).execute(input) 19 | return input -------------------------------------------------------------------------------- /deprecated/ultrafinance/processChain/processor/tradingStrategyProcessor.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Feb 26, 2010 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.processChain.baseModule import BaseModule 7 | from ultrafinance.lib.tradingStrategyFactory import TradingStrategyFactory 8 | from ultrafinance.lib.tradingStrategy.automaticInvestmentPlan import adjustFixAmountPerPeriod 9 | from ultrafinance.lib.tradingStrategy.automaticInvestmentPlan import fixAmountPerPeriod 10 | from ultrafinance.lib.tradingStrategy.automaticInvestmentPlan import fixAmountPerPeriodWithAddtionWhenDrop 11 | 12 | import logging 13 | LOG = logging.getLogger(__name__) 14 | 15 | class TradingStrategyProcessor(BaseModule): 16 | ''' Calculate average and standard deviation ''' 17 | def __init__(self): 18 | ''' constructor ''' 19 | super(TradingStrategyProcessor, self).__init__() 20 | 21 | def execute(self, dateValueList): 22 | ''' processing input''' 23 | output = {} 24 | tradingStrategyFactory = TradingStrategyFactory(fixAmountPerPeriod) 25 | output['fixAmountPerPeriod'] = tradingStrategyFactory.calculateReturn(dateValueList, 1) 26 | 27 | tradingStrategyFactory = TradingStrategyFactory(fixAmountPerPeriodWithAddtionWhenDrop) 28 | output['fixAmountPerPeriodWithAddtionWhenDrop'] = tradingStrategyFactory.calculateReturn(dateValueList, 1, 5) 29 | 30 | tradingStrategyFactory = TradingStrategyFactory(adjustFixAmountPerPeriod) 31 | output['adjustFixAmountPerPeriod'] = tradingStrategyFactory.calculateReturn(dateValueList, 5, 5) 32 | return output -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- 1 | """ application package """ 2 | import os, sys 3 | 4 | mainSrc = os.path.dirname(os.path.abspath(__file__)) 5 | sys.path.append(mainSrc) 6 | -------------------------------------------------------------------------------- /examples/backTester.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 3, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.module.backTester import BackTester 7 | 8 | if __name__ == "__main__": 9 | 10 | backTester = BackTester("backtest_smaPortfolio.ini", startTickDate = 20101010, startTradeDate = 20111220) 11 | backTester.setup() 12 | backTester.runTests() 13 | backTester.printMetrics() 14 | -------------------------------------------------------------------------------- /examples/fundamentalCrawler.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 4, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.dam.DAMFactory import DAMFactory 7 | from os import path 8 | import optparse 9 | 10 | from threading import Thread 11 | from threading import Lock 12 | 13 | BATCH_SIZE = 30 14 | THREAD_TIMEOUT = 5 15 | MAX_TRY = 3 16 | 17 | class FundamentalCrawler(object): 18 | ''' collect fundamental for a list of symbol ''' 19 | def __init__(self): 20 | ''' constructor ''' 21 | self.symbols = [] 22 | self.outputDAM = None 23 | self.googleDAM = None 24 | self.readLock = Lock() 25 | self.writeLock = Lock() 26 | self.failed = [] 27 | self.succeeded = [] 28 | 29 | def getOutputSql(self): 30 | ''' output path for sql database''' 31 | return path.join(path.dirname(path.dirname(path.realpath(__file__))), 32 | "data", 33 | "fundamental.sqlite") 34 | 35 | def getOptions(self): 36 | ''' crawling data and save to hbase ''' 37 | parser = optparse.OptionParser("Usage: %prog [options]") 38 | parser.add_option("-f", "--symbolFile", dest = "symbolFile", type = "string", 39 | help = "file that contains symbols for each line") 40 | parser.add_option("-o", "--outputDAM", dest = "outputDAM", 41 | default = 'sql', type = "string", 42 | help = "output dam, e.g. sql|hbase") 43 | 44 | (options, _) = parser.parse_args() 45 | 46 | # get symbols 47 | if options.symbolFile is None or not path.exists(options.symbolFile): 48 | print("Please provide valid file: %s" % options.symbolFile) 49 | exit(4) 50 | 51 | # get all symbols 52 | with open(options.symbolFile, 'r') as f: 53 | for line in f.readlines(): 54 | self.symbols.append(line.strip()) 55 | 56 | if not self.symbols: 57 | print("No symbols provided in file %s" % options.symbolFile) 58 | exit(4) 59 | 60 | # set output dam 61 | if options.outputDAM not in ["hbase", "sql"]: 62 | print("Please provide valid outputDAM %s" % options.outputDAM) 63 | exit(4) 64 | 65 | if 'sql' == options.outputDAM: 66 | sqlLocation = 'sqlite:///%s' % self.getOutputSql() 67 | print("Sqlite location: %s" % sqlLocation) 68 | setting = {'db': sqlLocation} 69 | 70 | 71 | # set google and output dam 72 | self.googleDAM = DAMFactory.createDAM("google") 73 | self.outputDAM = DAMFactory.createDAM(options.outputDAM, setting) 74 | 75 | def __getSaveOneSymbol(self, symbol): 76 | ''' get and save data for one symbol ''' 77 | try: 78 | with self.readLock: #dam is not thread safe 79 | failCount = 0 80 | #try several times since it may fail 81 | while failCount < MAX_TRY: 82 | try: 83 | self.googleDAM.setSymbol(symbol) 84 | keyTimeValueDict = self.googleDAM.readFundamental() 85 | 86 | except BaseException as excp: 87 | failCount += 1 88 | else: 89 | break 90 | 91 | if failCount >= MAX_TRY: 92 | raise BaseException("Can't retrive historical data") 93 | 94 | with self.writeLock: #dam is not thread safe 95 | self.outputDAM.setSymbol(symbol) 96 | self.outputDAM.writeFundamental(keyTimeValueDict) 97 | 98 | except BaseException as excp: 99 | print("Error while processing %s: %s" % (symbol, excp)) 100 | self.failed.append(symbol) 101 | else: 102 | print("Processed %s" % symbol) 103 | self.succeeded.append(symbol) 104 | 105 | def getSaveSymbols(self): 106 | ''' get and save data ''' 107 | counter = 0 108 | rounds = 0 109 | 110 | while counter < len(self.symbols): 111 | size = len(self.symbols) - counter 112 | if BATCH_SIZE < size: 113 | size = BATCH_SIZE 114 | symbols = self.symbols[counter: counter + size] 115 | 116 | threads = [] 117 | for symbol in symbols: 118 | thread = Thread(name = symbol, target = self.__getSaveOneSymbol, args = [symbol]) 119 | thread.daemon = True 120 | thread.start() 121 | 122 | threads.append(thread) 123 | 124 | for thread in threads: 125 | thread.join(THREAD_TIMEOUT) # no need to block, because thread should complete at last 126 | 127 | #can't start another thread to do commit because for sqlLite, only object for the same thread can be commited 128 | if 0 == rounds % 3: 129 | self.outputDAM.commit() 130 | 131 | counter += size 132 | rounds += 1 133 | 134 | def printFailedSucceeded(self): 135 | ''' print out which ones fails''' 136 | print("Succeeded: %s" % self.succeeded) 137 | print("Failed: %s" % self.failed) 138 | 139 | if __name__ == '__main__': 140 | crawler = FundamentalCrawler() 141 | crawler.getOptions() 142 | crawler.getSaveSymbols() 143 | crawler.printFailedSucceeded() 144 | 145 | -------------------------------------------------------------------------------- /examples/stockCrawler.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 4, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.module.googleCrawler import GoogleCrawler 7 | 8 | from os import path 9 | import datetime 10 | from dateutil.relativedelta import relativedelta 11 | import optparse 12 | 13 | BATCH_SIZE = 30 14 | THREAD_TIMEOUT = 5 15 | MAX_TRY = 3 16 | 17 | class StockCrawler(object): 18 | ''' collect quotes/ticks for a list of symbol ''' 19 | def __init__(self): 20 | ''' constructor ''' 21 | self.symbols = [] 22 | self.start = None 23 | 24 | def getOptions(self): 25 | ''' crawling data and save to hbase ''' 26 | parser = optparse.OptionParser("Usage: %prog [options]") 27 | parser.add_option("-f", "--symbolFile", dest = "symbolFile", type = "string", 28 | help = "file that contains symbols for each line") 29 | parser.add_option("-s", "--start", dest = "start", 30 | default = 'month', type = "string", 31 | help = "start date, all|month") 32 | 33 | (options, _) = parser.parse_args() 34 | 35 | # get symbols 36 | if options.symbolFile is None or not path.exists(options.symbolFile): 37 | print("Please provide valid file: %s" % options.symbolFile) 38 | exit(4) 39 | 40 | # get all symbols 41 | with open(options.symbolFile, 'r') as f: 42 | for line in f.readlines(): 43 | self.symbols.append(line.strip()) 44 | 45 | if not self.symbols: 46 | print("No symbols provided in file %s" % options.symbolFile) 47 | exit(4) 48 | 49 | if options.start not in ['all', 'month']: 50 | print("Please provide valid start option(all|month): %s" % options.outputDAM) 51 | exit(4) 52 | 53 | # set start date and end date 54 | if 'all' == options.start: 55 | self.start = '19800101' 56 | else: 57 | self.start = (datetime.datetime.now() + relativedelta(months = -1)).strftime("%Y%m%d") 58 | self.end = datetime.datetime.now().strftime("%Y%m%d") 59 | print("Retrieving quotes start from %s" % self.start) 60 | 61 | def retrieveQuotes(self): 62 | ''' retrieve quotes ''' 63 | googleCrawler = GoogleCrawler(self.symbols, self.start) 64 | googleCrawler.getAndSaveSymbols() 65 | print("Sqlite location: %s" % googleCrawler.sqlLocation) 66 | print("Succeeded: %s" % googleCrawler.succeeded) 67 | print("Failed: %s" % googleCrawler.failed) 68 | 69 | if __name__ == '__main__': 70 | crawler = StockCrawler() 71 | crawler.getOptions() 72 | crawler.retrieveQuotes() 73 | -------------------------------------------------------------------------------- /examples/stockPicker.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Aug 11, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import os 7 | import sys 8 | from ultrafinance.lib.googleFinance import GoogleFinance 9 | 10 | class StockPicker: 11 | ''' sort stocks ''' 12 | def __init__(self): 13 | ''' constructor ''' 14 | self.__workingDir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 15 | 'dataSource', 16 | 'SPY') 17 | self.__stockList = os.path.join(self.__workingDir, 'SPY500.list') 18 | self.__outputFile = os.path.join(self.__workingDir, 'SPY500.output') 19 | 20 | def getStockFinancials(self, fields, writeOutput=False): 21 | ''' get stock financials ''' 22 | with open(self.__stockList) as inFile: 23 | stockFinancials = {} 24 | 25 | for stockName in inFile.readlines(): 26 | try: 27 | stockName = stockName.strip() 28 | googleFinance = GoogleFinance() 29 | financials = googleFinance.getFinancials(stockName, fields) 30 | print "%s %s" % (stockName, financials) 31 | stockFinancials[stockName] = financials 32 | except Exception as excp: 33 | print '%s' % excp 34 | 35 | if writeOutput: 36 | with open(self.__outputFile, 'w') as outFile: 37 | outFile.write(str(stockFinancials)) 38 | 39 | return stockFinancials 40 | 41 | return {} 42 | 43 | def readStockFinancials(self): 44 | ''' read financials from file ''' 45 | with open(self.__outputFile) as file: 46 | # dangerous to use eval 47 | return eval(file.read()) 48 | 49 | def calDeceasingRate(self, value): 50 | try: 51 | firstIncRate = (float(value[0]) - float(value[1])) / float(value[1]) if float(value[1]) else 0 52 | secondIncRate = (float(value[1]) - float(value[2])) / float(value[2]) if float(value[2]) else 0 53 | thirdIncRate = (float(value[2]) - float(value[3])) / float(value[3]) if float(value[3]) else 0 54 | incRate = (firstIncRate + secondIncRate + thirdIncRate) / 3 55 | 56 | return incRate 57 | except: 58 | return 0 - sys.maxint 59 | 60 | def calAverage(self, value): 61 | try: 62 | return (float(value[0]) + float(value[1]) + float(value[2]) + float(value[3])) / 4 63 | except: 64 | return 0 - sys.maxint 65 | 66 | def sortStocks(self, stockFinancials, field, topNum=20): 67 | def sortByFieldDecreasing(stockFinancials): 68 | values = stockFinancials[1] 69 | value = values[field] 70 | if not value: 71 | return 0 - sys.maxint 72 | 73 | # return self.calDeceasingRate(value) 74 | return self.calAverage(value) 75 | 76 | ret = {} 77 | for name, values in sorted(stockFinancials.iteritems(), key=sortByFieldDecreasing, reverse=True)[0:topNum]: 78 | #ret[name] = (self.calDeceasingRate(values[field]), values[field]) 79 | ret[name] = ('%.2f' % self.calAverage(values[field]), values[field]) 80 | #ret[name] = '%.2f' % self.calAverage(values[field]) 81 | 82 | return ret 83 | 84 | if __name__ == '__main__': 85 | app = StockPicker() 86 | #app.getStockFinancials(['Total Revenue', 87 | # 'Diluted Normalized EPS', 'Net Income', 'Gross Profit', 'Total Operating Expense', 'Income After Tax'], 88 | # True) 89 | stockFinancials = app.readStockFinancials() 90 | topNum = 20 91 | field1 = 'Total Revenue' 92 | field2 = 'Diluted Normalized EPS' 93 | print 'Top %d sorted by %s Increasing' % (topNum, field1) 94 | for key, value in app.sortStocks(stockFinancials, field1, topNum).items(): 95 | print '||%s||%s||' % (key, value) 96 | print 'Top %d sorted by %s Increasing' % (topNum, field2) 97 | for key, value in app.sortStocks(stockFinancials, field2, topNum).items(): 98 | print '||%s||%s||' % (key, value) -------------------------------------------------------------------------------- /hadoop/conf/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | fs.default.name 9 | hdfs://localhost:9000 10 | 11 | 12 | -------------------------------------------------------------------------------- /hadoop/conf/hbase-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 24 | 25 | 26 | hbase.rootdir 27 | file:///DIRECTORY/hbase 28 | 29 | 30 | -------------------------------------------------------------------------------- /hadoop/conf/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | dfs.replication 9 | 1 10 | 11 | 12 | -------------------------------------------------------------------------------- /hadoop/conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | mapred.job.tracker 9 | localhost:9001 10 | 11 | 12 | -------------------------------------------------------------------------------- /hadoop/installHadoopHbaseHive.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eua 2 | 3 | ABS_PATH=$(cd ${0%/*} && echo $PWD/${0##*/}) 4 | ROOT=$(dirname "$ABS_PATH") 5 | DOWNLOAD_DIR=$ROOT/download 6 | TARGET_DIR=$ROOT/software 7 | JAVA_HOME="/Library/Java/Home" 8 | CLEANUP="ALL" #ALL|SOFTWARE|NONE 9 | #hadoop config 10 | HADOOP_VERSION="hadoop-0.20.203.0" 11 | HADOOP_URL="http://ftp.wayne.edu/apache//hadoop/common/${HADOOP_VERSION}/${HADOOP_VERSION}rc1.tar.gz" 12 | #hive config 13 | HIVE_VERSION="hive-0.7.1" 14 | HIVE_URL="http://mirror.candidhosting.com/pub/apache//hive/stable/${HIVE_VERSION}.tar.gz" 15 | #hbase config 16 | HBASE_VERSION="hbase-0.90.4" 17 | HBASE_URL="http://www.takeyellow.com/apachemirror//hbase/stable/${HBASE_VERSION}.tar.gz" 18 | HBASE_ROOTDIR="$TARGET_DIR/$HBASE_VERSION/rootdir" 19 | 20 | function cleanup { 21 | echo "======================Cleanup=====================================" 22 | mkdir -p $DOWNLOAD_DIR 23 | mkdir -p $TARGET_DIR 24 | if [ $CLEANUP="ALL" ]; then 25 | rm -rf $DOWNLOAD_DIR/* 26 | rm -rf $TARGET_DIR/* 27 | elif [ $CLEANUP="SOFTWARE" ]; then 28 | rm -rf $DOWNLOAD_DIR/* 29 | fi 30 | echo "======================Done: Cleanup=====================================" 31 | } 32 | 33 | function changeHadoopConfig { 34 | HADOOP_HOME=$TARGET_DIR/$HADOOP_VERSION 35 | sed -i -e "s%.*export JAVA_HOME.*%export JAVA_HOME=${JAVA_HOME}%g" $HADOOP_HOME/conf/hadoop-env.sh 36 | cp $ROOT/conf/core-site.xml $HADOOP_HOME/conf/ 37 | cp $ROOT/conf/hdfs-site.xml $HADOOP_HOME/conf/ 38 | cp $ROOT/conf/mapred-site.xml $HADOOP_HOME/conf/ 39 | } 40 | 41 | function setupPassphraseless { 42 | ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa 43 | cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys 44 | } 45 | 46 | 47 | function installHadoop { 48 | echo "======================Install Hadoop=====================================" 49 | cd $TARGET_DIR 50 | if [ ! -f ${DOWNLOAD_DIR}/${HADOOP_VERSION}.tar.gz ]; then 51 | curl $HADOOP_URL --O ${DOWNLOAD_DIR}/${HADOOP_VERSION}.tar.gz 52 | fi 53 | 54 | tar -xvzf ${DOWNLOAD_DIR}/${HADOOP_VERSION}.tar.gz 55 | export HADOOP_HOME=$TARGET_DIR/$HADOOP_VERSION 56 | cd $HADOOP_HOME 57 | changeHadoopConfig 58 | echo "======================Done: Install Hadoop=====================================" 59 | } 60 | 61 | function installHive { 62 | echo "======================Install Hive=====================================" 63 | cd $TARGET_DIR 64 | if [ ! -f ${DOWNLOAD_DIR}/${HIVE_VERSION}.tar.gz ]; then 65 | curl $HIVE_URL --O ${DOWNLOAD_DIR}/${HIVE_VERSION}.tar.gz 66 | fi 67 | 68 | tar -xvzf ${DOWNLOAD_DIR}/${HIVE_VERSION}.tar.gz 69 | export HIVE_HOME=$TARGET_DIR/$HIVE_VERSION 70 | export PATH=$HIVE_HOME/bin:$PATH 71 | echo "======================Done: Install Hive=====================================" 72 | } 73 | 74 | function changeHbaseConfig { 75 | HBASE_HOME=$TARGET_DIR/$HBASE_VERSION 76 | cp $ROOT/conf/hbase-site.xml $HBASE_HOME/conf/ 77 | sed -i -e "s%.*%$HBASE_ROOTDIR%g" $HBASE_HOME/conf/hbase-site.xml 78 | } 79 | 80 | function installHBase { 81 | echo "======================Install HBase=====================================" 82 | cd $TARGET_DIR 83 | if [ ! -f ${DOWNLOAD_DIR}/${HBASE_VERSION}.tar.gz ]; then 84 | curl $HBASE_URL --O ${DOWNLOAD_DIR}/${HBASE_VERSION}.tar.gz 85 | fi 86 | 87 | tar -xvzf ${DOWNLOAD_DIR}/${HBASE_VERSION}.tar.gz 88 | export HBASE_HOME=$TARGET_DIR/$HBASE_VERSION 89 | changeHbaseConfig 90 | echo "======================Done: Install Hadoop=====================================" 91 | } 92 | 93 | function echoHadoopUsage { 94 | echo "============================Hadoop Usage==========================" 95 | echo "To start hadoop: $HADOOP_HOME/bin/start-all.sh" 96 | echo "Hit NameNode at http://localhost:50070/" 97 | echo "Hit JobTracker - http://localhost:50030/" 98 | echo "Stop deamon: $HADOOP_HOME/bin/stop-all.sh" 99 | echo "Format FS: $HADOOP_HOME/bin/hadoop namenode -format" 100 | echo "============================Example==========================" 101 | echo "Copy the input files into the distributed filesystem:" 102 | echo "$ $HADOOP_HOME/bin/hadoop fs -put conf input" 103 | echo "Run some of the examples provided:" 104 | echo "$ $HADOOP_HOME/bin/hadoop jar hadoop-*-examples.jar grep input output 'dfs[a-z.]+'" 105 | echo "Examine the output files:" 106 | echo "$ $HADOOP_HOME/bin/hadoop fs -get output output " 107 | echo "$ cat output/*" 108 | echo "Or" 109 | echo "$ $HADOOP_HOME/bin/hadoop fs -cat output/*" 110 | echo "======================Done: Hadoop Usage=====================================" 111 | } 112 | 113 | function echoHbaseUsage { 114 | echo "============================HBase Usage==========================" 115 | echo "To start hbase: $HBASE_HOME/bin/start-hbase.sh" 116 | echo "To stop hbase: $HBASE_HOME/bin/stop-hbase.sh" 117 | echo "To start hbase shell: $HBASE_HOME/bin/hbase shell" 118 | echo "Create: create 'test', 'cf'" 119 | echo "Insert: put 'test', 'row1', 'cf:a', 'value1'" 120 | echo "List: scan 'test'" 121 | echo "Query: get 'test', 'row1'" 122 | echo "Delete: disable 'test'; drop 'test'" 123 | echo "$HBASE_HOME/bin/hbase-daemon.sh start thrift" 124 | echo "======================Done: HBase Usage=====================================" 125 | } 126 | 127 | cleanup 128 | installHadoop 129 | installHBase 130 | echoHadoopUsage 131 | echoHbaseUsage 132 | -------------------------------------------------------------------------------- /lib/PyDispatcher-2.0.1.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panpanpandas/ultrafinance/ce3594dbfef747cba0c39f059316df7c208aac40/lib/PyDispatcher-2.0.1.zip -------------------------------------------------------------------------------- /result/graph/GOOG graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panpanpandas/ultrafinance/ce3594dbfef747cba0c39f059316df7c208aac40/result/graph/GOOG graph.png -------------------------------------------------------------------------------- /result/graph/hoursing.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panpanpandas/ultrafinance/ce3594dbfef747cba0c39f059316df7c208aac40/result/graph/hoursing.png -------------------------------------------------------------------------------- /result/graph/housing_interest_stock.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panpanpandas/ultrafinance/ce3594dbfef747cba0c39f059316df7c208aac40/result/graph/housing_interest_stock.png -------------------------------------------------------------------------------- /result/graph/portfolios.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/panpanpandas/ultrafinance/ce3594dbfef747cba0c39f059316df7c208aac40/result/graph/portfolios.png -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on July 1, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from setuptools import setup 7 | from setupCommand import TestCommand, CleanCommand 8 | 9 | version = '1.0.1' 10 | 11 | setup(name='ultrafinance', 12 | version=version, 13 | description="python project for finance: realtime data collection, analyze, algorithmic trading", 14 | long_description="""""", 15 | classifiers=[ 16 | "Development Status :: 1 - Beta", 17 | "Intended Audience :: Developers", 18 | "License :: OSI Approved :: MIT License", 19 | ], 20 | keywords='python, Finance, Algorithm, Trading, Realtime, QuantLib, pydispather', 21 | author='Pan', 22 | author_email='panpandas@gmail.com', 23 | url='http://code.google.com/p/ultra-finance/', 24 | license='MIT', 25 | 26 | install_not_requires=[ 27 | 'hbase-thrift>=0.20.4', 28 | 'pandas', 29 | 'xlwt', 30 | 'xlrd', 31 | 'matplotlib>=1.1.0' 32 | ], 33 | packages=['ultrafinance'], 34 | include_package_data=True, 35 | install_requires=[ 36 | 'scipy>=0.13.0', 37 | 'numpy>=1.5.1', 38 | 'beautifulsoup4', 39 | 'SQLAlchemy>=0.8', 40 | 'mox' 41 | ], 42 | cmdclass = {'test': TestCommand, 'clean': CleanCommand } 43 | ) 44 | -------------------------------------------------------------------------------- /setupCommand.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on July 4, 2011 3 | 4 | @author: ppa 5 | copied from http://da44en.wordpress.com/2002/11/22/using-distutils/ 6 | ''' 7 | from distutils.core import Command 8 | from unittest import TextTestRunner, TestLoader 9 | from glob import glob 10 | from os.path import splitext, basename, join as pjoin 11 | import os 12 | 13 | class TestCommand(Command): 14 | ''' test command for setup.py ''' 15 | user_options = [ ] 16 | 17 | def initialize_options(self): 18 | self._dir = os.getcwd() 19 | 20 | def finalize_options(self): 21 | pass 22 | 23 | def run(self): 24 | ''' 25 | Finds all the tests modules in tests/, and runs them. 26 | ''' 27 | testfiles = [] 28 | for t in glob(pjoin(self._dir, 'tests', 'unit', '*.py')): 29 | if not t.endswith('__init__.py'): 30 | testfiles.append('.'.join( 31 | ['tests', 'unit', splitext(basename(t))[0]]) 32 | ) 33 | 34 | print('Running tests: %s' % testfiles) 35 | tests = TestLoader().loadTestsFromNames(testfiles) 36 | t = TextTestRunner(verbosity = 1) 37 | t.run(tests) 38 | 39 | class CleanCommand(Command): 40 | ''' clean command for setup.py ''' 41 | user_options = [ ] 42 | 43 | def initialize_options(self): 44 | self._clean_me = [ ] 45 | for root, _, files in os.walk('.'): 46 | for f in files: 47 | if f.endswith('.pyc'): 48 | self._clean_me.append(pjoin(root, f)) 49 | 50 | def finalize_options(self): 51 | pass 52 | 53 | def run(self): 54 | for clean_me in self._clean_me: 55 | try: 56 | os.unlink(clean_me) 57 | except: 58 | pass 59 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """ test package """ -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | """ unittest package """ 2 | import os, sys 3 | 4 | mainSrc = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), 'ultrafinance') 5 | sys.path.append(mainSrc) -------------------------------------------------------------------------------- /tests/unit/stock.list: -------------------------------------------------------------------------------- 1 | MMM 2 | ACE 3 | ABT 4 | ANF -------------------------------------------------------------------------------- /tests/unit/test_account.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Jan 08, 2012 3 | 4 | @author: ppa 5 | ''' 6 | import unittest 7 | from ultrafinance.backTest.account import Account 8 | from ultrafinance.model import Order, Side, Quote 9 | 10 | class testAccount(unittest.TestCase): 11 | def setUp(self): 12 | pass 13 | 14 | def tearDown(self): 15 | pass 16 | 17 | def testGetCash(self): 18 | account = Account(1000, 1) 19 | self.assertEquals(1000, account.cash) 20 | 21 | def testGetHoldingCost(self): 22 | share = 10 23 | price = 9.1 24 | account = Account(1000, 1) 25 | account._Account__holdings = {'stock1': (share, price)} 26 | print(account.holdings) 27 | 28 | holdingCost = account.getHoldingCost() 29 | print(holdingCost) 30 | 31 | self.assertAlmostEquals(share * price, holdingCost) 32 | 33 | def testGetHoldingValue(self): 34 | share = 10 35 | price = 9.1 36 | curPrice = 10.1 37 | account = Account(1000, 1) 38 | account._Account__holdings = {'stock1': (share, price)} 39 | account.setLastTickDict({'stock1': Quote(0, 0, 0, 0, curPrice, 0, 0)}) 40 | 41 | holdingValue = account.getHoldingValue() 42 | print(holdingValue) 43 | self.assertAlmostEqual(share * curPrice, holdingValue) 44 | 45 | def testTotalValue(self): 46 | share = 10 47 | price = 9.1 48 | curPrice = 10.1 49 | account = Account(1000, 1) 50 | account._Account__holdings = {'stock1': (share, price)} 51 | account.setLastTickDict({'stock1': Quote(0, 0, 0, 0, curPrice, 0, 0)}) 52 | 53 | totalValue = account.getTotalValue() 54 | print(totalValue) 55 | self.assertAlmostEquals(1000 + share * curPrice, totalValue) 56 | 57 | def testValidate(self): 58 | share = 10 59 | price = 9.1 60 | symbol = 'stock1' 61 | account = Account(1000, 1) 62 | account._Account__holdings = {symbol: (share, price)} 63 | 64 | # can't buy because price too high 65 | order1 = Order(accountId = 'id1', side = Side.BUY, symbol = symbol, price = 10000, share = 100000) 66 | self.assertEquals(False, account.validate(order1) ) 67 | 68 | # can't buy because of commission fee 69 | order1 = Order(accountId = 'id1', side = Side.BUY, symbol = symbol, price = 100, share = 10) 70 | self.assertEquals(False, account.validate(order1) ) 71 | 72 | # buy it 73 | order1 = Order(accountId = 'id1', side = Side.BUY, symbol = symbol, price = 100, share = 9) 74 | self.assertEquals(True, account.validate(order1) ) 75 | 76 | # can't sell because don't have the stock 77 | order1 = Order(accountId = 'id1', side = Side.SELL, symbol = 'fas89ph2', price = 100, share = 9) 78 | self.assertEquals(False, account.validate(order1) ) 79 | 80 | # can't sell because don't have the enough share 81 | order1 = Order(accountId = 'id1', side = Side.SELL, symbol = symbol, price = 100, share = 9000) 82 | self.assertEquals(False, account.validate(order1) ) 83 | 84 | # sell it 85 | order1 = Order(accountId = 'id1', side = Side.SELL, symbol = symbol, price = 100, share = 9) 86 | self.assertEquals(True, account.validate(order1) ) 87 | 88 | -------------------------------------------------------------------------------- /tests/unit/test_excel_dam.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 27, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import unittest 7 | 8 | import os 9 | from ultrafinance.model import Tick, Quote 10 | from ultrafinance.dam.excelDAM import ExcelDAM 11 | 12 | class testExcelDAM(unittest.TestCase): 13 | 14 | def setUp(self): 15 | self.targetPath = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'output') 16 | self.symbol = 'ebay' 17 | 18 | def tearDown(self): 19 | pass 20 | 21 | def testWriteExcel(self): 22 | writeDam = ExcelDAM() 23 | writeDam.setDir(self.targetPath) 24 | writeDam.setSymbol(self.symbol) 25 | 26 | for f in [writeDam.targetPath(ExcelDAM.QUOTE), writeDam.targetPath(ExcelDAM.TICK)]: 27 | if os.path.exists(f): 28 | os.remove(f) 29 | 30 | quote1 = Quote('1320676200', '32.58', '32.58', '32.57', '32.57', '65212', None) 31 | quote2 = Quote('1320676201', '32.59', '32.59', '32.58', '32.58', '65213', None) 32 | tick1 = Tick('1320676200', '32.58', '32.58', '32.57', '32.57', '65212') 33 | tick2 = Tick('1320676201', '32.59', '32.59', '32.58', '32.58', '65213') 34 | writeDam.writeQuotes([quote1, quote2]) 35 | writeDam.writeTicks([tick1, tick2]) 36 | 37 | def testReadExcel(self): 38 | self.testWriteExcel() 39 | readDam = ExcelDAM() 40 | readDam.setDir(self.targetPath) 41 | readDam.setSymbol(self.symbol) 42 | 43 | print(readDam.readQuotes(0, 10000000000)) 44 | print(readDam.readTicks(1320676201, 1320676201)) 45 | -------------------------------------------------------------------------------- /tests/unit/test_excel_lib.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on May 6, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import unittest 7 | 8 | import os 9 | from ultrafinance.dam.excelLib import ExcelLib 10 | 11 | class testExcelLib(unittest.TestCase): 12 | 13 | def setUp(self): 14 | pass 15 | 16 | def tearDown(self): 17 | pass 18 | 19 | def testReadExcel(self): 20 | dataSourcePath = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), 'dataSource') 21 | with ExcelLib( fileName = os.path.join(dataSourcePath, 'hoursing_interestRate.xls'), 22 | mode = ExcelLib.READ_MODE ) as excel: 23 | print("sheet numbers: %s" % excel.getOperation().getTotalSheetNumber()) 24 | print("sheetNames %s" % excel.getOperation().getSheetNames()) 25 | excel.openSheet('Data') 26 | data = excel.readRow(0) 27 | print(data) 28 | self.assertNotEqual(0, len(data)) 29 | 30 | data = excel.readCol(0, 7) 31 | print(data) 32 | self.assertNotEqual(0, len(data)) 33 | 34 | def testWriteExcel(self): 35 | targetPath = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'output') 36 | targetFile = os.path.join(targetPath, "writeTest.xls") 37 | sheetName = "testSheet" 38 | 39 | if os.path.exists(targetFile): 40 | os.remove(targetFile) 41 | 42 | with ExcelLib(fileName = targetFile, 43 | mode = ExcelLib.WRITE_MODE ) as excel: 44 | excel.openSheet(sheetName) 45 | excel.writeRow(0, [1, 2, 3, "4", "5"]) 46 | 47 | self.assertTrue(os.path.exists(targetFile)) 48 | -------------------------------------------------------------------------------- /tests/unit/test_google_dam.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 27, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import unittest 7 | from ultrafinance.dam.googleDAM import GoogleDAM 8 | 9 | class testGoogleDam(unittest.TestCase): 10 | 11 | def setUp(self): 12 | pass 13 | 14 | def tearDown(self): 15 | pass 16 | 17 | def testReadQuotes(self): 18 | dam = GoogleDAM() 19 | dam.setSymbol('NASDAQ:EBAY') 20 | data = dam.readQuotes('20131101', '20131110') 21 | print([str(q) for q in data]) 22 | self.assertNotEqual(0, len(data)) 23 | 24 | def testReadTicks(self): 25 | dam = GoogleDAM() 26 | dam.setSymbol('EBAY') 27 | data = dam.readTicks('20111120', '20111201') 28 | print(data) 29 | self.assertNotEqual(0, len(data)) 30 | 31 | def testReadFundamental(self): 32 | dam = GoogleDAM() 33 | dam.setSymbol('EBAY') 34 | keyTimeValueDict = dam.readFundamental() 35 | print(keyTimeValueDict) 36 | self.assertNotEqual(0, len(keyTimeValueDict)) 37 | -------------------------------------------------------------------------------- /tests/unit/test_google_finance.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on July 30, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import unittest 7 | from ultrafinance.dam.googleFinance import GoogleFinance 8 | from ultrafinance.lib.errors import UfException 9 | 10 | import logging 11 | LOG = logging.getLogger(__name__) 12 | 13 | class testGoogleFinance(unittest.TestCase): 14 | 15 | def setUp(self): 16 | pass 17 | 18 | def tearDown(self): 19 | pass 20 | 21 | def testGetQuotes(self): 22 | googleFinance = GoogleFinance() 23 | data = googleFinance.getQuotes('NASDAQ:EBAY', '20131101', None) 24 | print [str(q) for q in data] 25 | assert len(data) 26 | 27 | def testGetAll(self): 28 | googleFinance = GoogleFinance() 29 | data = googleFinance.getAll('EBAY') 30 | print(data) 31 | self.assertNotEqual(0, len(data)) 32 | 33 | def testGetAll_badSymbol(self): 34 | googleFinance = GoogleFinance() 35 | self.assertRaises(UfException, googleFinance.getAll, 'fasfdsdfasf') 36 | 37 | def testGetQuotes_badSymbol(self): 38 | googleFinance = GoogleFinance() 39 | self.assertRaises(UfException, googleFinance.getQuotes, *['AFSDFASDFASDFS', '20110101', '20110110']) 40 | 41 | def testGetFinancials(self): 42 | googleFinance = GoogleFinance() 43 | #ret = googleFinance.getFinancials('NASDAQ:EBAY', ['Net Income', 'Total Revenue', 'Diluted Normalized EPS', 'Total Common Shares Outstanding'], False) 44 | ret = googleFinance.getFinancials('NASDAQ:EBAY') 45 | print(ret) 46 | 47 | def testGetTicks(self): 48 | googleFinance = GoogleFinance() 49 | ret = googleFinance.getTicks('EBAY', start = '20110101', end = '20110110') 50 | print(ret) 51 | -------------------------------------------------------------------------------- /tests/unit/test_observable.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on July 30, 2011 3 | 4 | @author: ppa 5 | please refer to http://code.activestate.com/recipes/131499-observer-pattern/ 6 | ''' 7 | import unittest 8 | from ultrafinance.designPattern.observable import Observable 9 | 10 | class Data(Observable): 11 | def __init__(self, name=''): 12 | Observable.__init__(self) 13 | self.name = name 14 | self.data = 0 15 | 16 | def setData(self, data): 17 | self.data = data 18 | self.notify() 19 | 20 | def getData(self): 21 | return self.data 22 | 23 | 24 | class HexViewer: 25 | def __init__(self): 26 | self.data = 0 27 | 28 | def update(self, subject): 29 | self.data = subject.data 30 | print('HexViewer: Subject %s has data 0x%x' % (subject.name, subject.getData())) 31 | 32 | class DecimalViewer: 33 | def __init__(self): 34 | self.data = 0 35 | 36 | def update(self, subject): 37 | self.data = subject.data 38 | print('DecimalViewer: Subject %s has data %d' % (subject.name, subject.getData())) 39 | 40 | class testObservable(unittest.TestCase): 41 | def setUp(self): 42 | pass 43 | 44 | def tearDown(self): 45 | pass 46 | 47 | def testAttachAndUnattach(self): 48 | data1 = Data('Data 1') 49 | view1 = DecimalViewer() 50 | view2 = HexViewer() 51 | data1.attach(view1) 52 | data1.attach(view2) 53 | 54 | print("Setting Data 1 = 10") 55 | data1.setData(10) 56 | self.assertEquals(10, data1.data) 57 | self.assertEquals(10, view1.data) 58 | self.assertEquals(10, view2.data) 59 | print("Setting Data 1 = 3") 60 | data1.setData(3) 61 | self.assertEquals(3, data1.data) 62 | self.assertEquals(3, view1.data) 63 | self.assertEquals(3, view2.data) 64 | print("Detach HexViewer from data1") 65 | data1.detach(view2) 66 | print("Setting Data 1 = 10") 67 | data1.setData(10) 68 | self.assertEquals(10, data1.data) 69 | self.assertEquals(10, view1.data) 70 | self.assertEquals(3, view2.data) -------------------------------------------------------------------------------- /tests/unit/test_plot_date_value_dict.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on May 6, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import unittest 7 | 8 | from ultrafinance.lib.plotDateValueDict import PlotDateValueDict 9 | 10 | class testPlotDateValueDict(unittest.TestCase): 11 | def setUp(self): 12 | pass 13 | 14 | def tearDown(self): 15 | pass 16 | 17 | def testPlotOneDateValue(self): 18 | stock1 = [('20101101', 1), ('20101102', 2), ('20101103', 3)] 19 | stock2 = [('20101101', 3), ('20101102', 2), ('20101103', 1)] 20 | stock3 = [('20101101', 1), ('20101102', 2), ('20101103', 3)] 21 | stock4 = [('20101101', 3), ('20101102', 2), ('20101103', 1)] 22 | dateValues = {'testStock1': stock1, 'testStock2': stock2, 'testStock3': stock3, 'testStock4': stock4} 23 | p = PlotDateValueDict(dateValues, '%Y%m%d') 24 | p.plot() 25 | -------------------------------------------------------------------------------- /tests/unit/test_py_talib.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 18, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import unittest 7 | from ultrafinance.pyTaLib.indicator import Sma 8 | 9 | class testPyTaLib(unittest.TestCase): 10 | def setUp(self): 11 | pass 12 | 13 | def tearDown(self): 14 | pass 15 | 16 | def testSma(self): 17 | sma = Sma(period = 3) 18 | expectedAvgs = [1, 1.5, 2, 3, 4] 19 | for index, number in enumerate(range(1, 6) ): 20 | self.assertEqual(expectedAvgs[index], sma(number)) 21 | -------------------------------------------------------------------------------- /tests/unit/test_pyconfig.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 30, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import unittest 7 | from ultrafinance.ufConfig.pyConfig import PyConfig 8 | 9 | class testPyConfig(unittest.TestCase): 10 | def setUp(self): 11 | self.config = PyConfig() 12 | self.config.setSource("test.ini") 13 | 14 | def tearDown(self): 15 | pass 16 | 17 | def testGetSession(self): 18 | keyValues = self.config.getSection("app_main") 19 | print(keyValues) 20 | self.assertNotEqual(0, len(keyValues)) 21 | 22 | if not keyValues['field3']: 23 | print("field3 is None") 24 | 25 | def testGetOption(self): 26 | option = self.config.getOption("log", "file") 27 | print(option) 28 | self.assertEqual("test.log", option) 29 | -------------------------------------------------------------------------------- /tests/unit/test_sql_dam.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 27, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import unittest 7 | 8 | import os 9 | from ultrafinance.model import Tick, Quote 10 | from ultrafinance.dam.sqlDAM import SqlDAM 11 | 12 | class testSqlDAM(unittest.TestCase): 13 | 14 | def setUp(self): 15 | self.targetPath = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'output') 16 | self.symbol = 'ebay' 17 | 18 | def tearDown(self): 19 | pass 20 | 21 | def testSqlDam(self): 22 | dam = SqlDAM(echo = False) 23 | dam.setup({"db": 'sqlite:////tmp/sqldam.sqlite'}) 24 | dam.setSymbol("test") 25 | 26 | quotes = [Quote(*['1320676200', '32.59', '32.59', '32.58', '32.58', '65213', None]), 27 | Quote(*['1320676201', '32.60', '32.60', '32.59', '32.59', '65214', None])] 28 | ticks = [Tick(*['1320676200', '32.59', '32.59', '32.58', '32.58', '65213']), 29 | Tick(*['1320676201', '32.60', '32.60', '32.59', '32.59', '65214'])] 30 | 31 | dam.writeQuotes(quotes) 32 | dam.writeTicks(ticks) 33 | dam.commit() 34 | print([str(quotes) for symbol, quotes in dam.readBatchTupleQuotes(["test"], 0, None).items() ]) 35 | print([str(quote) for quote in dam.readQuotes(0, None) ]) 36 | print([str(tick) for tick in dam.readTicks(0, "1320676201")]) 37 | print([str(tick) for tick in dam.readTicks(0, "1320676202")]) 38 | 39 | 40 | def testFundamental(self): 41 | dam = SqlDAM(echo = False) 42 | dam.setup({"db": 'sqlite:////tmp/sqldam.sqlite'}) 43 | dam.setSymbol("test") 44 | 45 | keyTimeValueDict = {'Total Debt': {'As of 2011-12-31': 2089.6500000000001, 'As of 2012-03-31': 2085.0, 'As of 2010-12-31': 1794.23, 'As of 2009-12-31': 0.0, 'As of 2008-12-31': 1000.0, 'As of 2011-09-30': 2543.9899999999998, 'As of 2011-06-30': 2545.6999999999998, 'As of 2011-03-31': 1794.48}, 'Effect of Special Items on Income Taxes': {'12 months ending 2010-12-31': None, '3 months ending 2011-09-30': None, '12 months ending 2008-12-31': None, '3 months ending 2012-03-31': None, '3 months ending 2011-12-31': None, '12 months ending 2011-12-31': None, '12 months ending 2009-12-31': None, '3 months ending 2011-03-31': None, '3 months ending 2011-06-30': None}} 46 | dam.writeFundamental(keyTimeValueDict) 47 | dam.commit() 48 | 49 | ret = dam.readFundamental() 50 | print(ret) 51 | self.assertEqual(keyTimeValueDict, ret) 52 | -------------------------------------------------------------------------------- /tests/unit/test_tick_feeder.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Jan 18, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import mox 7 | import unittest 8 | from ultrafinance.backTest.tickSubscriber import TickSubsriber 9 | from ultrafinance.dam.baseDAM import BaseDAM 10 | from ultrafinance.model import Tick 11 | 12 | from ultrafinance.backTest.tickFeeder import TickFeeder 13 | from ultrafinance.lib.errors import UfException 14 | 15 | class testTickFeeder(unittest.TestCase): 16 | def setUp(self): 17 | self.mock = mox.Mox() 18 | 19 | def tearDown(self): 20 | pass 21 | 22 | def testAddSource(self): 23 | dam = self.mock.CreateMockAnything('dam') 24 | dam.symbol = 's1' 25 | 26 | tf = TickFeeder() 27 | tf.addSource(dam) 28 | 29 | print(tf._TickFeeder__source) 30 | self.assertEquals({'s1': dam}, tf._TickFeeder__source) 31 | 32 | def testGetSymbolsByRe(self): 33 | tf = TickFeeder() 34 | tf._TickFeeder__source = {'s1': 'dam1', 's11': 'dam2', 's2': 'dam3'} 35 | 36 | symbols = tf.getSymbolsByRe('s3') 37 | print(symbols) 38 | self.assertEquals([], symbols) 39 | 40 | symbols = tf.getSymbolsByRe('s1') 41 | print(symbols) 42 | self.assertEquals(['s1'], symbols) 43 | 44 | symbols = tf.getSymbolsByRe('.*') 45 | print(symbols) 46 | self.assertEquals(set(symbols), set(['s1', 's11', 's2'])) 47 | 48 | def testValidate_Normal(self): 49 | sub = self.mock.CreateMock(TickSubsriber) 50 | sub.subRules().AndReturn(['s1', 'mockRule']) 51 | 52 | tf = TickFeeder() 53 | tf._TickFeeder__source = {'s1': 'dam1', 's11': 'dam2', 's2': 'dam3'} 54 | 55 | self.mock.ReplayAll() 56 | symbols, sub = tf.validate(sub) 57 | self.mock.VerifyAll() 58 | 59 | print(symbols) 60 | 61 | 62 | def testValidate_Exception(self): 63 | sub = self.mock.CreateMock(TickSubsriber) 64 | sub.subRules().AndReturn(['s3', 'mockRule']) 65 | 66 | tf = TickFeeder() 67 | tf._TickFeeder__source = {'s1': 'dam1', 's11': 'dam2', 's2': 'dam3'} 68 | 69 | self.mock.ReplayAll() 70 | self.assertRaises(UfException, tf.validate, sub) 71 | self.mock.VerifyAll() 72 | 73 | def testRegister_Normal(self): 74 | sub = self.mock.CreateMock(TickSubsriber) 75 | sub.subRules().AndReturn(['s1', 'mockRule']) 76 | 77 | tf = TickFeeder() 78 | tf._TickFeeder__source = {'s1': 'dam1', 's11': 'dam2', 's2': 'dam3'} 79 | 80 | self.mock.ReplayAll() 81 | tf.register(sub) 82 | self.mock.VerifyAll() 83 | 84 | subs = tf.getSubs() 85 | print(subs) 86 | self.assertEquals({sub: {'symbols': ['s1'], 'fail': 0} }, 87 | subs) 88 | 89 | def testRegister_Exception(self): 90 | sub = self.mock.CreateMock(TickSubsriber) 91 | sub.subRules().AndReturn(['s3', 'mockRule']) 92 | 93 | tf = TickFeeder() 94 | tf._TickFeeder__source = {'s1': 'dam1', 's11': 'dam2', 's2': 'dam3'} 95 | 96 | self.mock.ReplayAll() 97 | self.assertRaises(UfException, tf.register, sub) 98 | self.mock.VerifyAll() 99 | 100 | def testInputType(self): 101 | tf = TickFeeder() 102 | 103 | #invalid type - test assignment 104 | self.assertRaises(UfException, tf._TickFeeder__setInputType, 'adafsdf') 105 | 106 | #valid type - test assignment 107 | tf.inputType = TickFeeder.TICK_TYPE 108 | 109 | self.assertEquals(TickFeeder.TICK_TYPE, tf.inputType) 110 | 111 | def testIndexTicks_quote(self): 112 | tickTime1Dam1 = Tick('time1', 'open1', 'high1', 'low1', 'close1', 'volume1') 113 | tickTime2Dam1 = Tick('time2', 'open2', 'high2', 'low2', 'close2', 'volume2') 114 | tickTime1Dam2 = Tick('time1', 'open11', 'high11', 'low11', 'close11', 'volume11') 115 | tickTime2Dam2 = Tick('time2', 'open22', 'high22', 'low22', 'close22', 'volume22') 116 | 117 | dam1 = self.mock.CreateMock(BaseDAM) 118 | dam1.readQuotes(mox.IgnoreArg(), mox.IgnoreArg()).AndReturn([tickTime1Dam1, tickTime2Dam1]) 119 | 120 | dam2 = self.mock.CreateMock(BaseDAM) 121 | dam2.readQuotes(mox.IgnoreArg(), mox.IgnoreArg()).AndReturn([tickTime1Dam2, tickTime2Dam2]) 122 | 123 | tf = TickFeeder() 124 | tf.inputType = TickFeeder.QUOTE_TYPE 125 | tf._TickFeeder__source = {'s1': dam1, 's2': dam2} 126 | 127 | self.mock.ReplayAll() 128 | timeTicks = tf.indexTicks() 129 | self.mock.VerifyAll() 130 | 131 | print(timeTicks) 132 | self.assertEquals({'time1': {'s1': tickTime1Dam1, 's2': tickTime1Dam2}, 133 | 'time2': {'s1': tickTime2Dam1, 's2': tickTime2Dam2}}, 134 | timeTicks) 135 | 136 | 137 | def testIndexTicks_tick(self): 138 | tickTime1Dam1 = Tick('time1', 'open1', 'high1', 'low1', 'close1', 'volume1') 139 | tickTime2Dam1 = Tick('time2', 'open2', 'high2', 'low2', 'close2', 'volume2') 140 | tickTime1Dam2 = Tick('time1', 'open11', 'high11', 'low11', 'close11', 'volume11') 141 | tickTime2Dam2 = Tick('time2', 'open22', 'high22', 'low22', 'close22', 'volume22') 142 | 143 | dam1 = self.mock.CreateMock(BaseDAM) 144 | dam1.readTicks(mox.IgnoreArg(), mox.IgnoreArg()).AndReturn([tickTime1Dam1, tickTime2Dam1]) 145 | 146 | dam2 = self.mock.CreateMock(BaseDAM) 147 | dam2.readTicks(mox.IgnoreArg(), mox.IgnoreArg()).AndReturn([tickTime1Dam2, tickTime2Dam2]) 148 | 149 | tf = TickFeeder() 150 | tf.inputType = TickFeeder.TICK_TYPE 151 | tf._TickFeeder__source = {'s1': dam1, 's2': dam2} 152 | 153 | self.mock.ReplayAll() 154 | timeTicks = tf.indexTicks() 155 | self.mock.VerifyAll() 156 | 157 | print(timeTicks) 158 | self.assertEquals({'time1': {'s1': tickTime1Dam1, 's2': tickTime1Dam2}, 159 | 'time2': {'s1': tickTime2Dam1, 's2': tickTime2Dam2}}, 160 | timeTicks) 161 | 162 | def testPubTicks(self): 163 | sub = self.mock.CreateMock(TickSubsriber) 164 | sub.runConsume(['ticks']) 165 | 166 | tf = TickFeeder() 167 | self.mock.ReplayAll() 168 | thread = tf.pubTicks(['ticks'], sub) 169 | self.mock.VerifyAll() 170 | 171 | print (thread) 172 | 173 | #TODO, too lazy to write this one........ 174 | def testExecute(self): 175 | pass -------------------------------------------------------------------------------- /tests/unit/test_util.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Jan 08, 2012 3 | 4 | @author: ppa 5 | ''' 6 | import unittest 7 | import os 8 | 9 | from ultrafinance.lib.util import capitalize, importClass 10 | 11 | class Test_util(object): 12 | ''' test class for importClass test ''' 13 | pass 14 | 15 | class Class1(object): 16 | ''' test class for importClass test ''' 17 | pass 18 | 19 | class testUtil(unittest.TestCase): 20 | def setUp(self): 21 | pass 22 | 23 | def tearDown(self): 24 | pass 25 | 26 | def testCapitalize(self): 27 | ''' test capitalize() ''' 28 | self.assertEquals("_123abc", capitalize("_123abc")) 29 | self.assertEquals("Abc123", capitalize("abc123")) 30 | self.assertEquals("A", capitalize("a")) 31 | self.assertEquals("", capitalize("")) 32 | 33 | def testImportClass(self): 34 | ''' test importClass() ''' 35 | curDir = os.path.dirname(os.path.abspath(__file__)) 36 | cl1 = importClass(curDir, "test_util") 37 | self.assertNotEquals(None, cl1) 38 | 39 | cl2 = importClass(curDir, "test_util", 'Class1') 40 | self.assertNotEquals(None, cl2) 41 | 42 | self.assertRaises(AttributeError, importClass, curDir, "test_util", 'Class2') 43 | -------------------------------------------------------------------------------- /tests/unit/test_yahoo_dam.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 28, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import unittest 7 | from ultrafinance.dam.yahooDAM import YahooDAM 8 | 9 | class testYahooDAM(unittest.TestCase): 10 | 11 | def setUp(self): 12 | pass 13 | 14 | def tearDown(self): 15 | pass 16 | 17 | def testReadQuotes(self): 18 | dam = YahooDAM() 19 | dam.setSymbol('^STI') 20 | data = dam.readQuotes('20110101', '20110110') 21 | print(data) 22 | self.assertNotEquals(len(data), 0) 23 | -------------------------------------------------------------------------------- /tests/unit/test_yahoo_finance.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on May 6, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import unittest 7 | from ultrafinance.dam.yahooFinance import YahooFinance 8 | 9 | class testYahooFinance(unittest.TestCase): 10 | 11 | def setUp(self): 12 | pass 13 | 14 | def tearDown(self): 15 | pass 16 | 17 | def testGetHistoricalPrices(self): 18 | yahooFinance = YahooFinance() 19 | data = yahooFinance.getQuotes('^STI', '20110101', '20110110') 20 | self.assertNotEqual(0, len(data)) -------------------------------------------------------------------------------- /ultrafinance/__init__.py: -------------------------------------------------------------------------------- 1 | """ ultrafinance package """ 2 | import os, re, sys 3 | 4 | import logging 5 | LOG = logging.getLogger() 6 | 7 | # library paths 8 | """ 9 | LIB = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'lib') 10 | for entry in os.listdir(LIB): 11 | filePath = os.path.join(LIB, entry) 12 | if os.path.isfile(filePath) and re.search(r".*\.(zip|egg|tar\.gz|tgz)$", filePath): 13 | LOG.debug("...appending library %s to sys.path" % filePath) 14 | sys.path.append(filePath) 15 | """ -------------------------------------------------------------------------------- /ultrafinance/backTest/__init__.py: -------------------------------------------------------------------------------- 1 | """ backtest package """ 2 | -------------------------------------------------------------------------------- /ultrafinance/backTest/accountManager.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Jan 29, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.backTest.account import Account 7 | from ultrafinance.backTest.constant import STATE_SAVER_ACCOUNT, STATE_SAVER_HOLDING_VALUE 8 | 9 | import copy 10 | 11 | import logging 12 | LOG = logging.getLogger() 13 | 14 | class AccountManager(object): 15 | ''' 16 | account manager 17 | Note: set metricNames before creating accounts 18 | ''' 19 | def __init__(self): 20 | ''' constructor ''' 21 | self.__accounts = {} 22 | self.__accountPositions = {} 23 | self.saver = None 24 | 25 | def createAccount(self, cash, commission = 0): 26 | ''' create account ''' 27 | account = Account(cash, commission) 28 | self.__accounts[account.accountId] = account 29 | self.__accountPositions[account.accountId] = [] # list contains tuple (time, position) 30 | 31 | return account.accountId 32 | 33 | def getAccountCopy(self, accountId): 34 | ''' get shallow copy of an account ''' 35 | return copy.copy(self.__accounts.get(accountId)) 36 | 37 | def getAccount(self, accountId): 38 | ''' get account ''' 39 | return self.__accounts.get(accountId) 40 | 41 | def getAccounts(self): 42 | ''' get accounts ''' 43 | return self.__accounts.values() 44 | 45 | def updateAccountsPosition(self, tickDict): 46 | ''' update account position based on new tick ''' 47 | curTime = tickDict.values()[0].time 48 | 49 | for accountId, account in self.__accounts.items(): 50 | account.setLastTickDict(tickDict) 51 | position = account.getTotalValue() 52 | holdingValue = account.getHoldingValue() 53 | 54 | self.__accountPositions[accountId].append((curTime, position)) 55 | #record 56 | if self.saver: 57 | self.saver.write(curTime, STATE_SAVER_ACCOUNT, position) 58 | self.saver.write(curTime, STATE_SAVER_HOLDING_VALUE, holdingValue) 59 | 60 | if position < 0: 61 | raise Exception("account %s value %s less than 0" % (accountId, position)) 62 | 63 | def getAccountPostions(self, accountId): 64 | ''' get account positions ''' 65 | return self.__accountPositions.get(accountId) 66 | -------------------------------------------------------------------------------- /ultrafinance/backTest/appGlobal.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Jan 24, 2012 3 | 4 | @author: ppa 5 | ''' 6 | 7 | from ultrafinance.designPattern.singleton import Singleton 8 | 9 | class AppGlobal(dict, Singleton): 10 | ''' appGlobal class ''' 11 | def __init__(self): 12 | super(AppGlobal, self).__init__() 13 | 14 | appGlobal = AppGlobal.getInstance() 15 | 16 | -------------------------------------------------------------------------------- /ultrafinance/backTest/btUtil.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Jan 6, 2012 3 | 4 | @author: ppa 5 | ''' 6 | import re 7 | 8 | OUTPUT_PREFIX = 'output' 9 | OUTPUT_FIELDS = ['placedOrder', 'accountValue', 'executedOrder'] 10 | 11 | def findInListbyRe(itemList, reString): 12 | ''' find items in list by re ''' 13 | items = [] 14 | pair = re.compile(reString) 15 | 16 | for item in itemList: 17 | result = pair.match(item) 18 | if result and result.group(0) == item: 19 | items.append(item) 20 | 21 | return items 22 | -------------------------------------------------------------------------------- /ultrafinance/backTest/constant.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Jan 20, 2012 3 | 4 | @author: ppa 5 | ''' 6 | 7 | #config constant 8 | 9 | 10 | CONF_APP_MAIN = 'app_main' 11 | 12 | CONF_ULTRAFINANCE_SECTION = 'ultrafinance' 13 | CONF_TRADE_TYPE = 'backtest.tradetype' 14 | CONF_SYMBOL_FILE = 'backtest.symbolfile' 15 | CONF_INDEX = 'backtest.index' 16 | CONF_INIT_CASH = "backtest.initcash" 17 | CONF_INPUT_DAM = 'backtest.input_dam' 18 | CONF_INPUT_DB = 'backtest.input_db' 19 | CONF_STRATEGY_NAME = 'backtest.strategy_name' 20 | CONF_SAVER = 'backtest.output_saver' 21 | CONF_OUTPUT_DB_PREFIX = 'backtest.output_db_prefix' 22 | CONF_START_TRADE_DATE = 'backtest.start_trade_date' 23 | CONF_END_TRADE_DATE = 'backtest.end_trade_date' 24 | CONF_BUYING_RATIO = 'backtest.buying_ratio' 25 | 26 | #period strategy 27 | CONF_STRATEGY_PERIOD = 'backtest.strategy.period' 28 | 29 | #app global 30 | TICK = 'tick' 31 | QUOTE = 'quote' 32 | TRADE_TYPE = 'tradetype' 33 | STOP_FLAG = "stopFlag" 34 | 35 | #EVENT LIST 36 | EVENT_TICK_UPDATE = 'tickUpdate' 37 | EVENT_ORDER_EXECUTED = "orderExecuted" 38 | 39 | #state saver 40 | STATE_SAVER_ACCOUNT = "account" 41 | STATE_SAVER_HOLDING_VALUE = "holdingValue" 42 | STATE_SAVER_INDEX_PRICE = "indexPrice" 43 | STATE_SAVER_UPDATED_ORDERS = "updatedOrders" 44 | STATE_SAVER_PLACED_ORDERS = "placedOrders" 45 | STATE_SAVER_METRICS_START_DATE = "startDate" 46 | STATE_SAVER_METRICS_END_DATE = "endDate" 47 | STATE_SAVER_METRICS_LOWEST_VALUE = "lowestValue" 48 | STATE_SAVER_METRICS_HIGHEST_VALUE = "highestValue" 49 | STATE_SAVER_METRICS_SHARPE_RATIO = "sharpeRatio" 50 | STATE_SAVER_METRICS_MAX_DRAW_DOWN = "maxDrawDown" 51 | STATE_SAVER_METRICS_R_SQUARED = "rSquared" 52 | -------------------------------------------------------------------------------- /ultrafinance/backTest/history.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on May 26, 2012 3 | 4 | @author: ppa 5 | ''' 6 | import logging 7 | LOG = logging.getLogger() 8 | 9 | class History(object): 10 | ''' helper class providing history info ''' 11 | INDEX = "I" 12 | 13 | def __init__(self): 14 | ''' constructor ''' 15 | self.timeSymbolTick = {} 16 | 17 | def update(self, timeStamp, symbolTickDict, indexTick): 18 | ''' do update ''' 19 | #each timestamp can only be updated once 20 | if timeStamp not in self.timeSymbolTick: 21 | self.timeSymbolTick[timeStamp] = {} 22 | 23 | for symbol, tick in symbolTickDict.iteritems(): 24 | self.timeSymbolTick[timeStamp][symbol] = tick 25 | 26 | self.timeSymbolTick[timeStamp][History.INDEX] = indexTick 27 | -------------------------------------------------------------------------------- /ultrafinance/backTest/indexHelper.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on May 26, 2012 3 | 4 | @author: ppa 5 | ''' 6 | import logging 7 | LOG = logging.getLogger() 8 | 9 | class IndexHelper(object): 10 | ''' helper class providing index info ''' 11 | 12 | def __init__(self): 13 | ''' constructor ''' 14 | self.__tick = [] 15 | 16 | def appendTick(self, tick): 17 | ''' self current tick ''' 18 | self.__tick.append(tick) 19 | 20 | if len(self.__tick) >= 252: 21 | self.__tick.pop(0) 22 | -------------------------------------------------------------------------------- /ultrafinance/backTest/metric.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Apr 29, 2012 3 | 4 | @author: ppa 5 | ''' 6 | import abc 7 | from ultrafinance.pyTaLib.indicator import stddev, sharpeRatio, mean, rsquared 8 | 9 | class BaseMetric(object): 10 | ''' base metric class ''' 11 | __metaclass__ = abc.ABCMeta 12 | 13 | @abc.abstractmethod 14 | def calculate(self, timePositions): 15 | ''' keep record of the account ''' 16 | return 17 | 18 | @abc.abstractmethod 19 | def formatResult(self): 20 | ''' print result ''' 21 | return 22 | 23 | class BasicMetric(BaseMetric): 24 | ''' basic metrics ''' 25 | MAX_TIME_VALUE = 'maxTimeValue' 26 | MIN_TIME_VALUE = 'minTimeValue' 27 | MAX_DRAW_DOWN = "maxDrawDown" 28 | STDDEV = 'stddev' 29 | SRATIO = 'sharpeRatio' 30 | START_TIME = "startTime" 31 | END_TIME="endTime" 32 | END_VALUE="endValue" 33 | R_SQUARED = "rSquared" 34 | 35 | def __init__(self): 36 | super(BasicMetric, self).__init__() 37 | self.result = {BasicMetric.MAX_TIME_VALUE: (None, -1), 38 | BasicMetric.MIN_TIME_VALUE: (None, -1), 39 | BasicMetric.STDDEV:-1, 40 | BasicMetric.SRATIO:-1, 41 | BasicMetric.R_SQUARED:-1, 42 | BasicMetric.START_TIME:-1, 43 | BasicMetric.END_TIME:-1, 44 | BasicMetric.END_VALUE:-1} 45 | 46 | def calculate(self, timePositions, iTimePositionDict): 47 | ''' calculate basic metrics ''' 48 | if not timePositions: 49 | return self.result 50 | 51 | # get max value, min value, max draw down 52 | lastHigest = 0 53 | maxDrawDownTimeStamp = 0 54 | maxDrawDownPosition = 0 55 | for (timeStamp, position) in timePositions: 56 | if self.result[BasicMetric.MAX_TIME_VALUE][0] is None or self.result[BasicMetric.MAX_TIME_VALUE][1] < position: 57 | self.result[BasicMetric.MAX_TIME_VALUE] = timeStamp, position 58 | if self.result[BasicMetric.MIN_TIME_VALUE][0] is None or self.result[BasicMetric.MIN_TIME_VALUE][1] > position: 59 | self.result[BasicMetric.MIN_TIME_VALUE] = timeStamp, position 60 | if position > lastHigest: 61 | lastHigest = position 62 | maxDrawDownPosition = position 63 | maxDrawDownTimeStamp = timeStamp 64 | elif maxDrawDownPosition > position: 65 | maxDrawDownPosition = position 66 | maxDrawDownTimeStamp = timeStamp 67 | 68 | self.result[BasicMetric.MAX_DRAW_DOWN] = (0, 0) if lastHigest == 0 else \ 69 | (maxDrawDownTimeStamp, 1 - (maxDrawDownPosition / lastHigest)) 70 | self.result[BasicMetric.START_TIME] = timePositions[0][0] 71 | self.result[BasicMetric.END_TIME] = timePositions[-1][0] 72 | self.result[BasicMetric.END_VALUE] = timePositions[-1][1] 73 | self.result[BasicMetric.STDDEV] = stddev([timePosition[1] for timePosition in timePositions]) 74 | self.result[BasicMetric.SRATIO] = sharpeRatio([timePosition[1] for timePosition in timePositions]) 75 | self.result[BasicMetric.R_SQUARED] = rsquared([tp[1] for tp in timePositions], [iTimePositionDict.get(tp[0], tp[1]) for tp in timePositions]) 76 | 77 | return self.result 78 | 79 | def formatResult(self): 80 | ''' format result ''' 81 | return "Lowest value %.2f at %s; Highest %.2f at %s; %s - %s end values %.1f; %s - %s end values %.1f; Sharpe ratio is %.2f; r squared is %.2f" % \ 82 | (self.result[BasicMetric.MIN_TIME_VALUE][1], self.result[BasicMetric.MIN_TIME_VALUE][0], 83 | self.result[BasicMetric.MAX_TIME_VALUE][1], self.result[BasicMetric.MAX_TIME_VALUE][0], 84 | self.result[BasicMetric.START_TIME], self.result[BasicMetric.END_TIME], self.result[BasicMetric.END_VALUE], 85 | self.result[BasicMetric.MAX_DRAW_DOWN][1], self.result[BaseMetric.MAX_DRAW_DOWN][0], 86 | self.result[BasicMetric.SRATIO], self.result[BasicMetric.R_SQUARED]) 87 | 88 | class MetricManager(object): 89 | ''' TODO: make it more generic for more metrics ''' 90 | def __init__(self): 91 | ''' constructor ''' 92 | self.__calculated = {} 93 | 94 | def calculate(self, symbols, timePositions, iTimePositionDict): 95 | ''' calculate metric base on positions ''' 96 | metric = BasicMetric() 97 | metric.calculate(timePositions, iTimePositionDict) 98 | self.__calculated['_'.join(symbols)] = metric.result 99 | return metric.result 100 | 101 | def formatMetrics(self): 102 | ''' output all calculated metrics ''' 103 | bestSymbol = None 104 | bestMetric = None 105 | worstSymbol = None 106 | worstMetric = None 107 | 108 | output = [] 109 | for symbols, metric in self.__calculated.items(): 110 | output.append("%s: %s" % (symbols, metric.formatResult())) 111 | 112 | if bestSymbol == None or metric.result[BasicMetric.END_VALUE] > bestMetric.result[BasicMetric.END_VALUE]: 113 | bestSymbol = symbols 114 | bestMetric = metric 115 | 116 | if worstSymbol == None or metric.result[BasicMetric.END_VALUE] < worstMetric.result[BasicMetric.END_VALUE]: 117 | worstSymbol = symbols 118 | worstMetric = metric 119 | 120 | output.append("MEAN end value: %.1f, mean sharp ratio: %.2f" % (mean([m[BasicMetric.END_VALUE] for m in self.__calculated.values() if m[BasicMetric.END_VALUE] > 0]), 121 | mean([m[BasicMetric.SRATIO] for m in self.__calculated.values() if m[BasicMetric.SRATIO] > -1]))) 122 | output.append("Best %s: %s" % (bestSymbol, bestMetric.formatResult())) 123 | output.append("Worst %s: %s" % (worstSymbol, worstMetric.formatResult())) 124 | return '\n'.join(output) 125 | 126 | 127 | def getMetrics(self): 128 | ''' get metrics ''' 129 | return self.__calculated 130 | -------------------------------------------------------------------------------- /ultrafinance/backTest/stateSaver/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 6, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.lib.errors import Errors, UfException 7 | import abc 8 | 9 | import logging 10 | LOG = logging.getLogger() 11 | 12 | class StateSaver(object): 13 | ''' state saver ''' 14 | __metaclass__ = abc.ABCMeta 15 | 16 | def __init__(self): 17 | ''' constructor ''' 18 | self.__tableName = None 19 | 20 | @abc.abstractmethod 21 | def getStates(self, start, end): 22 | ''' read value with row ''' 23 | pass 24 | 25 | @abc.abstractmethod 26 | def write(self, row, col, value): 27 | ''' write value with row and col ''' 28 | pass 29 | 30 | def commit(self): 31 | ''' complete write operation ''' 32 | pass 33 | 34 | def setup(self, setting): 35 | ''' setup ''' 36 | pass 37 | 38 | def getTableName(self): 39 | ''' return table name ''' 40 | return self.__tableName 41 | 42 | def setTableName(self, tableName): 43 | ''' set table name, table name can only be set once ''' 44 | if self.__tableName: 45 | raise UfException(Errors.TABLENAME_ALREADY_SET, 46 | "table name %s already set" % self.__tableName) 47 | 48 | self.__tableName = tableName 49 | 50 | tableName = property(getTableName, setTableName) 51 | -------------------------------------------------------------------------------- /ultrafinance/backTest/stateSaver/hbaseSaver.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 6, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.lib.errors import Errors, UfException 7 | from ultrafinance.dam.hbaseLib import HBaseLib 8 | from ultrafinance.backTest.stateSaver import StateSaver 9 | from hbase.Hbase import Mutation, ColumnDescriptor 10 | 11 | import logging 12 | LOG = logging.getLogger() 13 | 14 | class HbaseSaver(StateSaver): 15 | ''' hbase saver ''' 16 | def __init__(self, ip = "localhost", port = 9090): 17 | ''' constructor ''' 18 | super(HbaseSaver, self).__init__() 19 | self.__hbase = HBaseLib(ip, port) 20 | self.__writeCache = {} 21 | 22 | def resetCols(self, cols): 23 | ''' create cols ''' 24 | if self.tableName in self.__hbase.getTableNames(): 25 | self.__hbase.deleteTable(self.tableName) 26 | 27 | LOG.debug("create table %s with cols %s" % (self.tableName, cols)) 28 | self.__hbase.createTable(self.tableName, [ColumnDescriptor(name = str(col), maxVersions = 5) for col in cols]) 29 | 30 | def read(self, row, col): 31 | ''' read value with row and col ''' 32 | oneRow = self.__hbase.getRow(self.tableName, row) 33 | keyValues = oneRow.columns 34 | key = "%s:" % col 35 | 36 | if key in keyValues: 37 | return keyValues[key].value 38 | else: 39 | return None 40 | 41 | def write(self, row, col, value): 42 | ''' write value with row and col ''' 43 | self.__writeCache[(row, col)] = value 44 | 45 | def commit(self): 46 | ''' complete write operation ''' 47 | if not self.tableName: 48 | raise UfException(Errors.TABLENAME_NOT_SET, 49 | "Table name not set") 50 | 51 | # reset table with all cols first 52 | cols = set() 53 | for (row, col) in self.__writeCache.iterkeys(): 54 | cols.add(col) 55 | 56 | self.resetCols(cols) 57 | 58 | # write values 59 | for (row, col), value in self.__writeCache.iteritems(): 60 | self.__hbase.updateRow(self.tableName, 61 | row, 62 | [Mutation(column = "%s:" % col, value = str(value))]) 63 | 64 | def setup(self, setting): 65 | ''' setup ''' 66 | pass 67 | 68 | if __name__ == '__main__': 69 | h = HbaseSaver() 70 | h.tableName = 'unittest_outputSaver' 71 | #h.resetCols(['accountValue', '1']) 72 | for i in range(5): 73 | h.write('time1', 'accountValue', 10000) 74 | h.commit() 75 | accountValue = h.read('time1', 'accountValue') 76 | print(accountValue) 77 | assert str(10000) == accountValue 78 | assert None == h.read('time1', 'EDFASNONdafs') 79 | -------------------------------------------------------------------------------- /ultrafinance/backTest/stateSaver/stateSaverFactory.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 6, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.lib.errors import Errors, UfException 7 | from ultrafinance.designPattern.singleton import Singleton 8 | 9 | import logging 10 | LOG = logging.getLogger() 11 | 12 | class StateSaverFactory(Singleton): 13 | ''' factory for output saver ''' 14 | @staticmethod 15 | def createStateSaver(name, setting): 16 | ''' create state saver ''' 17 | if 'sql' == name: 18 | from ultrafinance.backTest.stateSaver.sqlSaver import SqlSaver 19 | saver = SqlSaver() 20 | else: 21 | raise UfException(Errors.INVALID_SAVER_NAME, 22 | "Saver name is invalid %s" % name) 23 | 24 | saver.setup(setting) 25 | return saver -------------------------------------------------------------------------------- /ultrafinance/backTest/tickFeeder.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 6, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.lib.errors import UfException, Errors 7 | from threading import Thread 8 | 9 | from ultrafinance.backTest.appGlobal import appGlobal 10 | from ultrafinance.backTest.constant import TRADE_TYPE, TICK, QUOTE 11 | from ultrafinance.backTest.constant import STATE_SAVER_INDEX_PRICE 12 | 13 | import traceback 14 | import time 15 | import logging 16 | LOG = logging.getLogger() 17 | 18 | class TickFeeder(object): 19 | ''' constructor 20 | no tick operation should take more that 2 second 21 | threadMaxFails indicates how many times thread for a subscriber can timeout, 22 | if it exceeds, them unregister that subscriber 23 | ''' 24 | def __init__(self, intervalTimeout = 2, start = 0, end = None): 25 | self.__subs = {} # securityIds: sub 26 | self.__symbols = [] 27 | self.__indexSymbol = None 28 | self.__dam = None 29 | self.__intervalTimeout = intervalTimeout 30 | self.start = start 31 | self.end = end 32 | self.tradingCenter = None 33 | self.saver = None 34 | self.__updatedTick = None 35 | self.timeTicksDict = {} 36 | self.iTimePositionDict = {} 37 | 38 | def getUpdatedTick(self): 39 | ''' return timeTickTuple with status changes ''' 40 | timeTicksTuple = self.__updatedTick 41 | 42 | return timeTicksTuple 43 | 44 | def clearUpdateTick(self): 45 | ''' clear current ticks ''' 46 | self.__updatedTick = None 47 | 48 | 49 | def _getSymbolTicksDict(self, symbols): 50 | ''' get ticks from one dam''' 51 | ticks = [] 52 | if TICK == appGlobal[TRADE_TYPE]: 53 | ticks = self.__dam.readBatchTupleTicks(symbols, self.start, self.end) 54 | elif QUOTE == appGlobal[TRADE_TYPE]: 55 | ticks = self.__dam.readBatchTupleQuotes(symbols, self.start, self.end) 56 | else: 57 | raise UfException(Errors.INVALID_TYPE, 58 | 'Type %s is not accepted' % appGlobal[TRADE_TYPE]) 59 | 60 | return ticks 61 | 62 | def __loadTicks(self): 63 | ''' generate timeTicksDict based on source DAM''' 64 | LOG.info('Start loading ticks, it may take a while......') 65 | 66 | LOG.info('Indexing ticks for %s' % self.__symbols) 67 | try: 68 | self.timeTicksDict = self._getSymbolTicksDict(self.__symbols) 69 | 70 | except KeyboardInterrupt as ki: 71 | LOG.warn("Interrupted by user when loading ticks for %s" % self.__symbols) 72 | raise ki 73 | except BaseException as excp: 74 | LOG.warn("Unknown exception when loading ticks for %s: except %s, traceback %s" % (self.__symbols, excp, traceback.format_exc(8))) 75 | 76 | 77 | def __loadIndex(self): 78 | ''' generate timeTicksDict based on source DAM''' 79 | LOG.debug('Start loading index ticks, it may take a while......') 80 | try: 81 | return self._getSymbolTicksDict([self.__indexSymbol]) 82 | 83 | except KeyboardInterrupt as ki: 84 | LOG.warn("Interrupted by user when loading ticks for %s" % self.__indexSymbol) 85 | raise ki 86 | except BaseException as excp: 87 | LOG.warn("Unknown exception when loading ticks for %s: except %s, traceback %s" % (self.__indexSymbol, excp, traceback.format_exc(8))) 88 | 89 | return {} 90 | 91 | def execute(self): 92 | ''' execute func ''' 93 | self.__loadTicks() 94 | 95 | for timeStamp in sorted(self.timeTicksDict.iterkeys()): 96 | # make sure trading center finish updating first 97 | self._freshTradingCenter(self.timeTicksDict[timeStamp]) 98 | 99 | self._freshUpdatedTick(timeStamp, self.timeTicksDict[timeStamp]) 100 | #self._updateHistory(timeStamp, self.timeTicksDict[timeStamp], self.indexTicksDict.get(timeStamp)) 101 | 102 | while self.__updatedTick: 103 | time.sleep(0) 104 | 105 | def _freshUpdatedTick(self, timeStamp, symbolTicksDict): 106 | ''' update self.__updatedTick ''' 107 | self.__updatedTick = (timeStamp, symbolTicksDict) 108 | 109 | def _freshTradingCenter(self, symbolTicksDict): 110 | ''' feed trading center ticks ''' 111 | self.tradingCenter.consumeTicks(symbolTicksDict) 112 | 113 | def complete(self): 114 | ''' 115 | call when complete feeding ticks 116 | write history to saver 117 | ''' 118 | try: 119 | if not self.saver: 120 | return 121 | 122 | timeITicksDict = self.__loadIndex() 123 | if timeITicksDict: 124 | for time, symbolDict in timeITicksDict.iteritems(): 125 | for symbol in symbolDict.keys(): 126 | self.saver.write(time, STATE_SAVER_INDEX_PRICE, symbolDict[symbol].close) 127 | self.iTimePositionDict[time] = symbolDict[symbol].close 128 | break # should only have one benchmark 129 | 130 | 131 | except Exception as ex: 132 | LOG.warn("Unknown error when recording index info:" + str(ex)) 133 | 134 | def setSymbols(self, symbols): 135 | ''' set symbols ''' 136 | self.__symbols = symbols 137 | 138 | def setIndexSymbol(self, indexSymbol): 139 | ''' set symbols ''' 140 | self.__indexSymbol = indexSymbol 141 | 142 | 143 | def setDam(self, dam): 144 | ''' set source dam ''' 145 | self.__dam = dam 146 | 147 | def pubTicks(self, ticks, sub): 148 | ''' publish ticks to sub ''' 149 | thread = Thread(target = sub.doConsume, args = (ticks,)) 150 | thread.setDaemon(False) 151 | thread.start() 152 | return thread 153 | -------------------------------------------------------------------------------- /ultrafinance/backTest/tickSubscriber/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 6, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import abc 7 | import uuid 8 | import threading 9 | 10 | class TickSubsriber(object): 11 | ''' tick subscriber ''' 12 | __metaclass__ = abc.ABCMeta 13 | 14 | def __init__(self, name): 15 | ''' constructor ''' 16 | self.__id = self.__generateId() 17 | self.__name = name 18 | self.__threadLock = threading.Lock() 19 | 20 | def __generateId(self): 21 | ''' generate id ''' 22 | return uuid.uuid4() 23 | 24 | def __getId(self): 25 | ''' get id ''' 26 | return self.__id 27 | 28 | def __getName(self): 29 | ''' get name ''' 30 | return self.__name 31 | 32 | def preConsume(self, ticks): 33 | ''' override function ''' 34 | pass 35 | 36 | @abc.abstractmethod 37 | def tickUpdate(self, ticks): 38 | ''' consume ticks ''' 39 | return 40 | 41 | def orderExecuted(self, orderDict): 42 | ''' call back for executed order with order id, should be overridden ''' 43 | return 44 | 45 | def complete(self): 46 | ''' complete operation ''' 47 | pass 48 | 49 | @abc.abstractmethod 50 | def subRules(self): 51 | ''' call back from framework 52 | return (symbolRe, rules) 53 | ''' 54 | return 55 | 56 | subId = property(__getId) 57 | name = property(__getName) 58 | -------------------------------------------------------------------------------- /ultrafinance/backTest/tickSubscriber/strategies/__init__.py: -------------------------------------------------------------------------------- 1 | ''' strategy ''' -------------------------------------------------------------------------------- /ultrafinance/backTest/tickSubscriber/strategies/baseStrategy.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 25, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import abc 7 | from ultrafinance.backTest.tickSubscriber import TickSubsriber 8 | from ultrafinance.lib.errors import Errors, UfException 9 | from ultrafinance.backTest.constant import EVENT_TICK_UPDATE, EVENT_ORDER_EXECUTED 10 | 11 | import logging 12 | LOG = logging.getLogger() 13 | 14 | class BaseStrategy(TickSubsriber): 15 | ''' trading center ''' 16 | __meta__ = abc.ABCMeta 17 | 18 | def __init__(self, name): 19 | ''' constructor ''' 20 | super(BaseStrategy, self).__init__(name) 21 | self.accountId = None 22 | self.tradingEngine = None 23 | self.configDict = {} 24 | self.symbols = [] 25 | self.__curTime = '' 26 | self.indexHelper = None 27 | self.history = None 28 | self.accountManager = None 29 | 30 | 31 | def subRules(self): 32 | ''' override function ''' 33 | return (self.symbols, [EVENT_TICK_UPDATE, EVENT_ORDER_EXECUTED]) 34 | 35 | def checkReady(self): 36 | ''' 37 | whether strategy has been set up and ready to run 38 | TODO: check trading engine 39 | ''' 40 | if self.accountId is None: 41 | raise UfException(Errors.NONE_ACCOUNT_ID, 42 | "Account id is none") 43 | 44 | return True 45 | 46 | def placeOrder(self, order): 47 | ''' place order and keep record''' 48 | orderId = self.tradingEngine.placeOrder(order) 49 | 50 | return orderId 51 | 52 | def complete(self): 53 | ''' complete operation ''' 54 | pass 55 | 56 | def setSymbols(self, symbols): 57 | '''set symbols ''' 58 | if list != type(symbols): 59 | raise UfException(Errors.INVALID_SYMBOLS, 60 | "symbols %s is not a list" % symbols) 61 | 62 | self.symbols = symbols 63 | 64 | def getAccountCopy(self): 65 | ''' get copy of account info ''' 66 | return self.accountManager.getAccountCopy(self.accountId) 67 | -------------------------------------------------------------------------------- /ultrafinance/backTest/tickSubscriber/strategies/periodStrategy.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 25, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.model import Type, Action, Order 7 | from ultrafinance.backTest.tickSubscriber.strategies.baseStrategy import BaseStrategy 8 | from ultrafinance.backTest.constant import CONF_STRATEGY_PERIOD, CONF_INIT_CASH 9 | 10 | import logging 11 | LOG = logging.getLogger() 12 | 13 | class PeriodStrategy(BaseStrategy): 14 | ''' period strategy ''' 15 | def __init__(self, configDict): 16 | ''' constructor ''' 17 | super(PeriodStrategy, self).__init__("periodStrategy") 18 | self.configDict = configDict 19 | 20 | assert int(configDict[CONF_STRATEGY_PERIOD]) >= 1 21 | 22 | self.perAmount = max(1, round(int(configDict[CONF_INIT_CASH]) / 100)) #buy 1/100 per time 23 | self.period = int(configDict[CONF_STRATEGY_PERIOD]) 24 | self.symbols = None 25 | self.counter = 0 26 | 27 | def increaseAndCheckCounter(self): 28 | ''' increase counter by one and check whether a period is end ''' 29 | self.counter += 1 30 | self.counter %= self.period 31 | if not self.counter: 32 | return True 33 | else: 34 | return False 35 | 36 | def tickUpdate(self, tickDict): 37 | ''' consume ticks ''' 38 | assert self.symbols 39 | assert self.symbols[0] in tickDict.keys() 40 | symbol = self.symbols[0] 41 | tick = tickDict[symbol] 42 | 43 | if self.increaseAndCheckCounter(): 44 | self.placeOrder(Order(accountId = self.accountId, 45 | action = Action.BUY, 46 | type = Type.MARKET, 47 | symbol = symbol, 48 | price = tick.close, 49 | share = self.perAmount / float(tick.close))) 50 | 51 | 52 | -------------------------------------------------------------------------------- /ultrafinance/backTest/tickSubscriber/strategies/strategyFactory.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 26, 2011 3 | 4 | @author: ppa 5 | ''' 6 | 7 | from ultrafinance.backTest.tickSubscriber.strategies.periodStrategy import PeriodStrategy 8 | from ultrafinance.backTest.tickSubscriber.strategies.smaStrategy import SMAStrategy 9 | from ultrafinance.backTest.tickSubscriber.strategies.smaPortfolioStrategy import SMAPortfolioStrategy 10 | from ultrafinance.backTest.tickSubscriber.strategies.zscorePortfolioStrategy import ZscorePortfolioStrategy 11 | from ultrafinance.backTest.tickSubscriber.strategies.zscoreMomentumPortfolioStrategy import ZscoreMomentumPortfolioStrategy 12 | 13 | from ultrafinance.lib.errors import Errors, UfException 14 | 15 | class StrategyFactory(object): 16 | ''' Strategy factory ''' 17 | STRATEGY_DICT = {'period': PeriodStrategy, 18 | 'sma': SMAStrategy, 19 | 'smaPortfolio': SMAPortfolioStrategy, 20 | 'zscorePortfolio': ZscorePortfolioStrategy, 21 | 'zscoreMomentumPortfolio': ZscoreMomentumPortfolioStrategy} 22 | 23 | @staticmethod 24 | def createStrategy(name, configDict): 25 | ''' create a metric ''' 26 | if name not in StrategyFactory.STRATEGY_DICT: 27 | raise UfException(Errors.INVALID_STRATEGY_NAME, 28 | "Strategy name is invalid %s" % name) 29 | return StrategyFactory.STRATEGY_DICT[name](configDict) 30 | 31 | @staticmethod 32 | def getAvailableTypes(): 33 | ''' return all available types ''' 34 | return StrategyFactory.STRATEGY_DICT.keys() 35 | -------------------------------------------------------------------------------- /ultrafinance/backTest/tickSubscriber/strategies/zscoreMomentumPortfolioStrategy.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 09, 2013 3 | 4 | This strategy use zscore to trade stocks 5 | 6 | When to Buy/Short: 7 | if zsocore is > 2 8 | 9 | When to Sell/Buy to cover: 10 | 1 after 10 days 11 | 2 or stop order is met(5%) 12 | 13 | @author: ppa 14 | ''' 15 | from ultrafinance.model import Type, Action, Order 16 | from ultrafinance.backTest.tickSubscriber.strategies.baseStrategy import BaseStrategy 17 | from ultrafinance.pyTaLib.indicator import ZScore, Momentum 18 | from ultrafinance.backTest.constant import CONF_START_TRADE_DATE, CONF_BUYING_RATIO 19 | import math 20 | 21 | import logging 22 | LOG = logging.getLogger() 23 | 24 | class ZscoreMomentumPortfolioStrategy(BaseStrategy): 25 | ''' period strategy ''' 26 | def __init__(self, configDict): 27 | ''' constructor ''' 28 | super(ZscoreMomentumPortfolioStrategy, self).__init__("zscoreMomentumPortfolioStrategy") 29 | self.__trakers = {} 30 | self.startDate = int(configDict.get(CONF_START_TRADE_DATE)) 31 | self.buyingRatio = int(configDict.get(CONF_BUYING_RATIO) if CONF_BUYING_RATIO in configDict else 2) 32 | 33 | def __setUpTrakers(self): 34 | ''' set symbols ''' 35 | for symbol in self.symbols: 36 | self.__trakers[symbol] = OneTraker(symbol, self, self.buyingRatio) 37 | 38 | def orderExecuted(self, orderDict): 39 | ''' call back for executed order ''' 40 | for orderId, order in orderDict.items(): 41 | if order.symbol in self.__trakers.keys(): 42 | self.__trakers[order.symbol].orderExecuted(orderId) 43 | 44 | def tickUpdate(self, tickDict): 45 | ''' consume ticks ''' 46 | if not self.__trakers: 47 | self.__setUpTrakers() 48 | 49 | for symbol, tick in tickDict.items(): 50 | if symbol in self.__trakers: 51 | self.__trakers[symbol].tickUpdate(tick) 52 | 53 | class OneTraker(object): 54 | ''' tracker for one stock ''' 55 | def __init__(self, symbol, strategy, buyingRatio): 56 | ''' constructor ''' 57 | self.__symbol = symbol 58 | self.__strategy = strategy 59 | self.__startDate = strategy.startDate 60 | self.__buyingRatio = buyingRatio 61 | self.__buyThreshold = 1.5 62 | self.__sellThreshold = 0.5 63 | self.__preZscore = None 64 | self.__priceZscore = ZScore(150) 65 | self.__volumeZscore = ZScore(150) 66 | self.__dayCounter = 0 67 | self.__dayCounterThreshold = 5 68 | 69 | # order id 70 | self.__position = 0 71 | self.__buyPrice = 0 72 | 73 | 74 | def __getCashToBuyStock(self): 75 | ''' calculate the amount of money to buy stock ''' 76 | account = self.__strategy.getAccountCopy() 77 | 78 | if (account.buyingPower >= account.getTotalValue() / self.__buyingRatio): 79 | return account.getTotalValue() / self.__buyingRatio 80 | else: 81 | return 0 82 | 83 | def __placeBuyOrder(self, tick): 84 | ''' place buy order''' 85 | cash = self.__getCashToBuyStock() 86 | if cash == 0: 87 | return 88 | 89 | share = math.floor(cash / float(tick.close)) 90 | order = Order(accountId = self.__strategy.accountId, 91 | action = Action.BUY, 92 | type = Type.MARKET, 93 | symbol = self.__symbol, 94 | share = share) 95 | if self.__strategy.placeOrder(order): 96 | self.__position = share 97 | self.__buyPrice = tick.close 98 | 99 | def __placeSellOrder(self, tick): 100 | ''' place sell order ''' 101 | if self.__position < 0: 102 | return 103 | 104 | share = self.__position 105 | order = Order(accountId = self.__strategy.accountId, 106 | action = Action.SELL, 107 | type = Type.MARKET, 108 | symbol = self.__symbol, 109 | share = -share) 110 | if self.__strategy.placeOrder(order): 111 | self.__position = 0 112 | self.__buyPrice = 0 113 | 114 | 115 | def orderExecuted(self, orderId): 116 | ''' call back for executed order ''' 117 | return 118 | 119 | def tickUpdate(self, tick): 120 | ''' consume ticks ''' 121 | LOG.debug("tickUpdate %s with tick %s, price %s" % (self.__symbol, tick.time, tick.close)) 122 | self.__priceZscore(tick.close) 123 | self.__volumeZscore(tick.volume) 124 | 125 | # get zscore 126 | priceZscore = self.__priceZscore.getLastValue() 127 | volumeZscore = self.__volumeZscore.getLastValue() 128 | 129 | #if haven't started, don't do any trading 130 | if tick.time <= self.__startDate: 131 | return 132 | 133 | # if not enough data, skip to reduce risk 134 | if priceZscore is None or volumeZscore is None: 135 | return 136 | 137 | if self.__position > 0: 138 | self.__dayCounter += 1 139 | 140 | if priceZscore > self.__buyThreshold and self.__preZscore and self.__preZscore < self.__buyThreshold and self.__position <= 0 and abs(volumeZscore) > 1: 141 | self.__placeBuyOrder(tick) 142 | elif self.__position > 0: 143 | if (self.__dayCounter > self.__dayCounterThreshold and priceZscore < self.__sellThreshold)\ 144 | or priceZscore < 0 or self.__buyPrice * 0.9 > tick.close: 145 | self.__placeSellOrder(tick) 146 | self.__dayCounter = 0 147 | 148 | self.__preZscore = priceZscore 149 | -------------------------------------------------------------------------------- /ultrafinance/backTest/tickSubscriber/strategies/zscorePortfolioStrategy.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 09, 2013 3 | 4 | This strategy use zscore to trade stocks 5 | 6 | When to Buy/Short: 7 | if zsocore is > 2.5 8 | 9 | When to Sell/Buy to cover: 10 | 1 after 10 days 11 | 2 or stop order is met(5%) 12 | 13 | @author: ppa 14 | ''' 15 | from ultrafinance.model import Type, Action, Order 16 | from ultrafinance.backTest.tickSubscriber.strategies.baseStrategy import BaseStrategy 17 | from ultrafinance.pyTaLib.indicator import ZScore 18 | from ultrafinance.backTest.constant import CONF_START_TRADE_DATE, CONF_BUYING_RATIO 19 | import math 20 | 21 | import logging 22 | LOG = logging.getLogger() 23 | 24 | class ZscorePortfolioStrategy(BaseStrategy): 25 | ''' period strategy ''' 26 | def __init__(self, configDict): 27 | ''' constructor ''' 28 | super(ZscorePortfolioStrategy, self).__init__("zscorePortfolioStrategy") 29 | self.__trakers = {} 30 | self.startDate = int(configDict.get(CONF_START_TRADE_DATE)) 31 | self.buyingRatio = int(configDict.get(CONF_BUYING_RATIO) if CONF_BUYING_RATIO in configDict else 2) 32 | 33 | def __setUpTrakers(self): 34 | ''' set symbols ''' 35 | for symbol in self.symbols: 36 | self.__trakers[symbol] = OneTraker(symbol, self, self.buyingRatio) 37 | 38 | def orderExecuted(self, orderDict): 39 | ''' call back for executed order ''' 40 | for orderId, order in orderDict.items(): 41 | if order.symbol in self.__trakers.keys(): 42 | self.__trakers[order.symbol].orderExecuted(orderId) 43 | 44 | def tickUpdate(self, tickDict): 45 | ''' consume ticks ''' 46 | if not self.__trakers: 47 | self.__setUpTrakers() 48 | 49 | for symbol, tick in tickDict.items(): 50 | if symbol in self.__trakers: 51 | self.__trakers[symbol].tickUpdate(tick) 52 | 53 | class OneTraker(object): 54 | ''' tracker for one stock ''' 55 | def __init__(self, symbol, strategy, buyingRatio): 56 | ''' constructor ''' 57 | self.__symbol = symbol 58 | self.__strategy = strategy 59 | self.__startDate = strategy.startDate 60 | self.__buyingRatio = buyingRatio 61 | self.__buyThreshold = -2 62 | self.__sellThreshold = 0.5 63 | self.__priceZscore = ZScore(120) 64 | self.__volumeZscore = ZScore(120) 65 | self.__toSell = False 66 | self.__toBuy = False 67 | 68 | # order id 69 | self.__position = 0 70 | 71 | 72 | def __getCashToBuyStock(self): 73 | ''' calculate the amount of money to buy stock ''' 74 | account = self.__strategy.getAccountCopy() 75 | 76 | if (account.buyingPower >= account.getTotalValue() / self.__buyingRatio): 77 | return account.getTotalValue() / self.__buyingRatio 78 | else: 79 | return 0 80 | 81 | def __placeBuyOrder(self, tick): 82 | ''' place buy order''' 83 | cash = self.__getCashToBuyStock() 84 | if cash == 0 or self.__position > 0: 85 | return 86 | 87 | share = math.floor(cash / float(tick.close)) - self.__position 88 | order = Order(accountId = self.__strategy.accountId, 89 | action = Action.BUY, 90 | type = Type.MARKET, 91 | symbol = self.__symbol, 92 | share = share) 93 | if self.__strategy.placeOrder(order): 94 | self.__position = math.floor(cash / float(tick.close)) 95 | 96 | def __placeSellOrder(self, tick): 97 | ''' place sell order ''' 98 | if self.__position < 0: 99 | return 100 | 101 | share = self.__position 102 | order = Order(accountId = self.__strategy.accountId, 103 | action = Action.SELL, 104 | type = Type.MARKET, 105 | symbol = self.__symbol, 106 | share = -share) 107 | if self.__strategy.placeOrder(order): 108 | self.__position = 0 109 | 110 | 111 | def orderExecuted(self, orderId): 112 | ''' call back for executed order ''' 113 | return 114 | 115 | def tickUpdate(self, tick): 116 | ''' consume ticks ''' 117 | LOG.debug("tickUpdate %s with tick %s, price %s" % (self.__symbol, tick.time, tick.close)) 118 | self.__priceZscore(tick.close) 119 | self.__volumeZscore(tick.volume) 120 | 121 | #if haven't started, don't do any trading 122 | if tick.time <= self.__startDate: 123 | return 124 | 125 | # if not enough data, skip to reduce risk 126 | if not self.__priceZscore.getLastValue() or not self.__volumeZscore.getLastValue(): 127 | return 128 | 129 | # get zscore 130 | priceZscore = self.__priceZscore.getLastValue() 131 | volumeZscore = self.__volumeZscore.getLastValue() 132 | if priceZscore is None or volumeZscore is None: 133 | return 134 | 135 | if priceZscore < self.__buyThreshold and self.__position <= 0 and abs(volumeZscore) > 1: 136 | self.__placeBuyOrder(tick) 137 | 138 | elif priceZscore > self.__sellThreshold and self.__position > 0 and abs(volumeZscore) > 1: 139 | self.__placeSellOrder(tick) 140 | """ 141 | if self.__toBuy: 142 | self.__placeBuyOrder(tick) 143 | self.__toBuy = False 144 | return 145 | 146 | if self.__toSell: 147 | self.__placeSellOrder(tick) 148 | self.__toSell = False 149 | return 150 | 151 | if priceZscore < (-self.__threshold) and not self.__buyOrder and abs(volumeZscore) > 1.5: 152 | self.__toBuy = True 153 | 154 | elif self.__buyOrder and priceZscore > 0.5: 155 | self.__toSell = True 156 | """ -------------------------------------------------------------------------------- /ultrafinance/dam/DAMFactory.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 9, 2011 3 | 4 | @author: ppa 5 | ''' 6 | 7 | from ultrafinance.lib.errors import Errors, UfException 8 | 9 | class DAMFactory(object): 10 | ''' DAM factory ''' 11 | @staticmethod 12 | def createDAM(damType, settings = None): 13 | ''' create DAM ''' 14 | if 'yahoo' == damType: 15 | from ultrafinance.dam.yahooDAM import YahooDAM 16 | dam = YahooDAM() 17 | elif 'google' == damType: 18 | from ultrafinance.dam.googleDAM import GoogleDAM 19 | dam = GoogleDAM() 20 | elif 'excel' == damType: 21 | from ultrafinance.dam.excelDAM import ExcelDAM 22 | dam = ExcelDAM() 23 | elif 'hbase' == damType: 24 | from ultrafinance.dam.hbaseDAM import HBaseDAM 25 | dam = HBaseDAM() 26 | elif 'sql' == damType: 27 | from ultrafinance.dam.sqlDAM import SqlDAM 28 | dam = SqlDAM() 29 | else: 30 | raise UfException(Errors.INVALID_DAM_TYPE, 31 | "DAM type is invalid %s" % damType) 32 | 33 | dam.setup(settings) 34 | return dam 35 | 36 | @staticmethod 37 | def getAvailableTypes(): 38 | ''' return all available types ''' 39 | return ['yahoo', 'google', 'excel', 'hbase', 'sql'] 40 | -------------------------------------------------------------------------------- /ultrafinance/dam/__init__.py: -------------------------------------------------------------------------------- 1 | ''' data access module ''' -------------------------------------------------------------------------------- /ultrafinance/dam/baseDAM.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 9, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import abc 7 | from ultrafinance.lib.errors import UfException, Errors 8 | 9 | class BaseDAM(object): 10 | ''' base class for DAO ''' 11 | __metaclass__ = abc.ABCMeta 12 | 13 | def __init__(self): 14 | ''' constructor ''' 15 | self.__symbol = None 16 | 17 | def readQuotes(self, start, end): 18 | ''' read quotes ''' 19 | raise UfException(Errors.UNDEFINED_METHOD, "readQuotes method is not defined") 20 | 21 | def readTupleQuotes(self, start, end): 22 | ''' read quotes as tuple to save memory ''' 23 | raise UfException(Errors.UNDEFINED_METHOD, "readTupleQuotes method is not defined") 24 | 25 | def readBatchTupleQuotes(self, symbols, start, end): 26 | ''' 27 | TODO: revisit API for batch and single symbol operation 28 | read batch quotes as tuple to save memory 29 | ''' 30 | raise UfException(Errors.UNDEFINED_METHOD, "readBatchTupleQuotes method is not defined") 31 | 32 | def writeQuotes(self, quotes): 33 | ''' write quotes ''' 34 | raise UfException(Errors.UNDEFINED_METHOD, "writeQuotes method is not defined") 35 | 36 | def readTicks(self, start, end): 37 | ''' read ticks ''' 38 | raise UfException(Errors.UNDEFINED_METHOD, "readTicks method is not defined") 39 | 40 | def writeTicks(self, ticks): 41 | ''' read quotes ''' 42 | raise UfException(Errors.UNDEFINED_METHOD, "writeTicks method is not defined") 43 | 44 | def readFundamental(self): 45 | ''' read fundamental ''' 46 | raise UfException(Errors.UNDEFINED_METHOD, "readFundamental method is not defined") 47 | 48 | def writeFundamental(self, keyTimeValueDict): 49 | ''' write fundamental ''' 50 | raise UfException(Errors.UNDEFINED_METHOD, "writeFundamental method is not defined") 51 | 52 | def setSymbol(self, symbol): 53 | ''' set symbol ''' 54 | self.__symbol = symbol 55 | 56 | def destruct(self): 57 | ''' destructor ''' 58 | pass 59 | 60 | def __getSymbol(self): 61 | ''' get symbol ''' 62 | return self.__symbol 63 | 64 | def setup(self, settings): 65 | ''' setup dam ''' 66 | pass 67 | 68 | def commit(self): 69 | ''' commit write changes ''' 70 | pass 71 | 72 | symbol = property(__getSymbol, setSymbol) 73 | -------------------------------------------------------------------------------- /ultrafinance/dam/excelDAM.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 9, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.dam.baseDAM import BaseDAM 7 | from ultrafinance.dam.excelLib import ExcelLib 8 | from ultrafinance.model import TICK_FIELDS, QUOTE_FIELDS, Quote, Tick 9 | from ultrafinance.lib.errors import UfException, Errors 10 | 11 | from os import path 12 | 13 | import logging 14 | LOG = logging.getLogger() 15 | 16 | class ExcelDAM(BaseDAM): 17 | ''' Excel DAO ''' 18 | QUOTE = 'quote' 19 | TICK = 'tick' 20 | 21 | def __init__(self): 22 | ''' constructor ''' 23 | super(ExcelDAM, self).__init__() 24 | self.__dir = None 25 | 26 | def targetPath(self, kind): 27 | return path.join(self.__dir, "%s-%s.xls" % (self.symbol, kind) ) 28 | 29 | def __findRange(self, excelLib, start, end): 30 | ''' return low and high as excel range ''' 31 | inc = 1 32 | low = 0 33 | high = 0 34 | dates = excelLib.readCol(0, 1) 35 | 36 | for index, date in enumerate(dates): 37 | if int(start) <= int(date): 38 | low = index + inc 39 | break 40 | 41 | if low: 42 | for index, date in reversed(list(enumerate(dates))): 43 | if int(date) <= int(end): 44 | high = index + inc 45 | break 46 | 47 | return low, high 48 | 49 | def __readData(self, targetPath, start, end): 50 | ''' read data ''' 51 | ret = [] 52 | if not path.exists(targetPath): 53 | LOG.error("Target file doesn't exist: %s" % path.abspath(targetPath) ) 54 | return ret 55 | 56 | with ExcelLib(fileName = targetPath, mode = ExcelLib.READ_MODE) as excel: 57 | low, high = self.__findRange(excel, start, end) 58 | 59 | for index in range(low, high + 1): 60 | ret.append(excel.readRow(index)) 61 | 62 | return ret 63 | 64 | def __writeData(self, targetPath, fields, rows): 65 | ''' write data ''' 66 | if path.exists(targetPath): 67 | LOG.error("Target file exists: %s" % path.abspath(targetPath) ) 68 | raise UfException(Errors.FILE_EXIST, "can't write to a existing file") #because xlwt doesn't support it 69 | 70 | with ExcelLib(fileName = targetPath, mode = ExcelLib.WRITE_MODE) as excel: 71 | excel.writeRow(0, fields) 72 | for index, row in enumerate(rows): 73 | excel.writeRow(index+1, row) 74 | 75 | def readQuotes(self, start, end): 76 | ''' read quotes ''' 77 | quotes = self.__readData(self.targetPath(ExcelDAM.QUOTE), start, end) 78 | return [Quote(*quote) for quote in quotes] 79 | 80 | def writeQuotes(self, quotes): 81 | ''' write quotes ''' 82 | self.__writeData(self.targetPath(ExcelDAM.QUOTE), 83 | QUOTE_FIELDS, 84 | [[getattr(quote, field) for field in QUOTE_FIELDS] for quote in quotes]) 85 | 86 | def readTicks(self, start, end): 87 | ''' read ticks ''' 88 | ticks = self.__readData(self.targetPath(ExcelDAM.TICK), start, end) 89 | return [Tick(*tick) for tick in ticks] 90 | 91 | def writeTicks(self, ticks): 92 | ''' read quotes ''' 93 | self.__writeData(self.targetPath(ExcelDAM.TICK), 94 | TICK_FIELDS, 95 | [[getattr(tick, field) for field in TICK_FIELDS] for tick in ticks]) 96 | 97 | def setDir(self, path): 98 | ''' set dir ''' 99 | self.__dir = path -------------------------------------------------------------------------------- /ultrafinance/dam/googleDAM.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 9, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.dam.baseDAM import BaseDAM 7 | from ultrafinance.dam.googleFinance import GoogleFinance 8 | 9 | import logging 10 | LOG = logging.getLogger() 11 | 12 | class GoogleDAM(BaseDAM): 13 | ''' Google DAO ''' 14 | 15 | def __init__(self): 16 | ''' constructor ''' 17 | super(GoogleDAM, self).__init__() 18 | self.__gf = GoogleFinance() 19 | 20 | def readQuotes(self, start, end): 21 | ''' read quotes from google Financial''' 22 | if self.symbol is None: 23 | LOG.debug('Symbol is None') 24 | return [] 25 | 26 | return self.__gf.getQuotes(self.symbol, start, end) 27 | 28 | def readTicks(self, start, end): 29 | ''' read ticks from google Financial''' 30 | if self.symbol is None: 31 | LOG.debug('Symbol is None') 32 | return [] 33 | 34 | return self.__gf.getTicks(self.symbol, start, end) 35 | 36 | def readFundamental(self): 37 | ''' read fundamental ''' 38 | if self.symbol is None: 39 | LOG.debug('Symbol is None') 40 | return {} 41 | 42 | return self.__gf.getFinancials(self.symbol) 43 | -------------------------------------------------------------------------------- /ultrafinance/dam/hbaseDAM.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 9, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.dam.baseDAM import BaseDAM 7 | from ultrafinance.dam.hbaseLib import HBaseLib 8 | from ultrafinance.model import Quote, Tick, TICK_FIELDS, QUOTE_FIELDS 9 | from hbase.Hbase import Mutation, ColumnDescriptor 10 | 11 | class HBaseDAM(BaseDAM): 12 | ''' HBase DAO ''' 13 | QUOTE = 'quote' 14 | TICK = 'tick' 15 | 16 | def __init__(self, ip="localhost", port=9090): 17 | ''' constructor ''' 18 | super(HBaseDAM, self).__init__() 19 | self.__hbase = HBaseLib(ip, port) 20 | 21 | def tableName(self, kind): 22 | return "%s-%s" % (self.symbol, kind) 23 | 24 | def __rowResultToQuote(self, row): 25 | ''' convert rowResult from Hbase to Quote''' 26 | keyValues = row.columns 27 | for field in QUOTE_FIELDS: 28 | key = "%s:%s" % (HBaseDAM.QUOTE, field) 29 | if 'time' != field and keyValues[key].value: 30 | keyValues[key].value = float(keyValues[key].value) 31 | 32 | return Quote(*[keyValues["%s:%s" % (HBaseDAM.QUOTE, field)].value for field in QUOTE_FIELDS]) 33 | 34 | def __rowResultToTick(self, row): 35 | ''' convert rowResult from Hbase to Tick''' 36 | keyValues = row.columns 37 | for field in TICK_FIELDS: 38 | key = "%s:%s" % (HBaseDAM.TICK, field) 39 | if 'time' != field and keyValues[key].value: 40 | keyValues[key].value = float(keyValues[key].value) 41 | 42 | return Tick(*[keyValues["%s:%s" % (HBaseDAM.TICK, field)].value for field in TICK_FIELDS]) 43 | 44 | def readQuotes(self, start, end): 45 | ''' read quotes ''' 46 | rows = self.__hbase.scanTable(self.tableName(HBaseDAM.QUOTE), [HBaseDAM.QUOTE], start, end) 47 | 48 | return [self.__rowResultToQuote(row) for row in rows] 49 | 50 | def writeQuotes(self, quotes): 51 | ''' write quotes ''' 52 | tName = self.tableName(HBaseDAM.QUOTE) 53 | if tName not in self.__hbase.getTableNames(): 54 | self.__hbase.createTable(tName, [ColumnDescriptor(name=HBaseDAM.QUOTE, maxVersions=5)]) 55 | 56 | for quote in quotes: 57 | self.__hbase.updateRow(self.tableName(HBaseDAM.QUOTE), 58 | quote.time, 59 | [Mutation(column = "%s:%s" % (HBaseDAM.QUOTE, field), 60 | value = getattr(quote, field) ) for field in QUOTE_FIELDS]) 61 | 62 | def readTicks(self, start, end): 63 | ''' read ticks ''' 64 | rows = self.__hbase.scanTable(self.tableName(HBaseDAM.TICK), [HBaseDAM.TICK], start, end) 65 | return [self.__rowResultToTick(row) for row in rows] 66 | 67 | def writeTicks(self, ticks): 68 | ''' read quotes ''' 69 | tName = self.tableName(HBaseDAM.TICK) 70 | if tName not in self.__hbase.getTableNames(): 71 | self.__hbase.createTable(tName, [ColumnDescriptor(name=HBaseDAM.TICK, maxVersions=5)]) 72 | 73 | for tick in ticks: 74 | self.__hbase.updateRow(self.tableName(HBaseDAM.TICK), 75 | tick.time, 76 | [Mutation(column = "%s:%s" % (HBaseDAM.TICK, field), 77 | value = getattr(tick, field) ) for field in TICK_FIELDS]) 78 | 79 | if __name__ == '__main__': 80 | dam = HBaseDAM() 81 | dam.setSymbol("test") 82 | quotes = [Quote(*['1320676200', '32.59', '32.59', '32.58', '32.58', '65213', None]), 83 | Quote(*['1320676201', '32.60', '32.60', '32.59', '32.59', '65214', None])] 84 | ticks = [Tick(*['1320676200', '32.59', '32.59', '32.58', '32.58', '65213']), 85 | Tick(*['1320676201', '32.60', '32.60', '32.59', '32.59', '65214'])] 86 | 87 | dam.writeQuotes(quotes) 88 | dam.writeTicks(ticks) 89 | print(dam.readQuotes("0", None)) 90 | print(dam.readTicks("0", "1320676201")) 91 | print(dam.readTicks("0", "1320676202")) 92 | -------------------------------------------------------------------------------- /ultrafinance/dam/hbaseLib.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Sep 27, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from thrift.transport import TSocket, TTransport 7 | from thrift.protocol import TBinaryProtocol 8 | from hbase import ttypes 9 | from hbase.Hbase import Client, ColumnDescriptor, Mutation 10 | 11 | from ultrafinance.lib.errors import UfException, Errors 12 | 13 | import logging 14 | LOG = logging.getLogger() 15 | 16 | class HBaseLib: 17 | ''' Hbase client ''' 18 | def __init__(self, ip='localhost', port=9090, timeout = 10): 19 | transport = TSocket.TSocket(ip, int(port)) 20 | transport.setTimeout(timeout * 1000) 21 | transport = TTransport.TBufferedTransport(transport) 22 | protocol = TBinaryProtocol.TBinaryProtocol(transport) 23 | 24 | self.__client = Client(protocol) 25 | transport.open() 26 | 27 | def getTableNames(self): 28 | ''' get table names ''' 29 | return self.__client.getTableNames() 30 | 31 | def deleteTable(self, tName): 32 | ''' delete table name ''' 33 | if self.__client.isTableEnabled(tName): 34 | self.__client.disableTable(tName) 35 | 36 | self.__client.deleteTable(tName) 37 | 38 | def disableTable(self, tName): 39 | ''' disable table ''' 40 | self.__client.disableTable(tName) 41 | 42 | def enableTable(self, tName): 43 | ''' enable table ''' 44 | self.__client.enableTable(tName) 45 | 46 | def createTable(self, tName, ColumnDescriptors): 47 | try: 48 | self.__client.createTable(tName, ColumnDescriptors) 49 | except ttypes.AlreadyExists as excp: 50 | raise UfException(Errors.HBASE_CREATE_ERROR, 51 | "AlreadyExists Error when creating table %s with cols: %s): %s" % \ 52 | (tName, [col.name for col in ColumnDescriptors], excp.message)) 53 | 54 | def getColumnDescriptors(self, tName): 55 | try: 56 | return self.__client.getColumnDescriptors(tName) 57 | except: 58 | raise UfException(Errors.UNKNOWN_ERROR, 59 | "Error when getting column descriptors table %s" % tName) 60 | 61 | def updateRow(self, tName, rowName, mutations, timestamp=None): 62 | ''' add row to table ''' 63 | try: 64 | if timestamp is None: 65 | self.__client.mutateRow(tName, rowName, mutations) 66 | else: 67 | self.__client.mutateRowTs(tName, rowName, mutations, timestamp) 68 | except Exception as excp: 69 | raise UfException(Errors.HBASE_UPDATE_ERROR, 70 | "Error when updating table %s - rowName %s - mutations %s: %s" % \ 71 | (tName, rowName, mutations, excp)) 72 | 73 | def getRow(self, tName, rowName): 74 | ''' get row ''' 75 | result = self.__client.getRow(tName, rowName) 76 | if not result: 77 | return result 78 | else: 79 | return result[0] 80 | 81 | def scanTable(self, tName, columns, startRow="", endRow=None): 82 | ''' scan a table ''' 83 | if endRow is None: 84 | scanner = self.__client.scannerOpen(tName, str(startRow), columns) 85 | else: 86 | scanner = self.__client.scannerOpenWithStop(tName, startRow, str(endRow), columns) 87 | ret = [] 88 | 89 | row = self.__client.scannerGet(scanner) 90 | while row: 91 | ret.append(row[0]) 92 | row = self.__client.scannerGet(scanner) 93 | 94 | return ret 95 | 96 | def getClient(self): 97 | ''' return client, in case low level api is needed ''' 98 | return self.__client 99 | 100 | # testing 101 | if __name__ == '__main__': 102 | h = HBaseLib() 103 | 104 | #delete all exiting tables 105 | for tName in h.getTableNames(): 106 | print "disable %s" % tName 107 | h.disableTable(tName) 108 | 109 | #assert not h.getTableNames() 110 | 111 | #create table 112 | tName = 'testTable' 113 | 114 | h.createTable(tName, [ColumnDescriptor(name='col1', maxVersions=5), ColumnDescriptor(name='col2', maxVersions=5)]) 115 | print h.getTableNames() 116 | assert h.getTableNames() 117 | 118 | print "column families in %s" % tName 119 | print h.getColumnDescriptors(tName) 120 | 121 | #updateRow 122 | h.updateRow(tName, "bar", [Mutation(column="col1:bar", value='12345'), Mutation(column="col2:", value="67890")]) 123 | h.updateRow(tName, "foo", [Mutation(column="col1:foo", value='12345')]) 124 | print h.getRow(tName, 'bar') 125 | print h.getRow(tName, 'foo') 126 | 127 | #scan table 128 | rows = h.scanTable(tName, columns=["col1", "col2"]) 129 | print rows 130 | assert 2 == len(rows) 131 | 132 | rows = h.scanTable(tName, columns=["col1", "col2"], startRow="foo") 133 | print rows 134 | assert 1 == len(rows) 135 | 136 | rows = h.scanTable(tName, columns=["col1", "col2"], endRow="foo") 137 | print rows 138 | assert 1 == len(rows) -------------------------------------------------------------------------------- /ultrafinance/dam/yahooDAM.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 9, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.dam.baseDAM import BaseDAM 7 | from ultrafinance.dam.yahooFinance import YahooFinance 8 | 9 | import logging 10 | LOG = logging.getLogger() 11 | 12 | class YahooDAM(BaseDAM): 13 | ''' Yahoo DAM ''' 14 | 15 | def __init__(self): 16 | ''' constructor ''' 17 | super(YahooDAM, self).__init__() 18 | self.__yf = YahooFinance() 19 | 20 | def readQuotes(self, start, end): 21 | ''' read quotes from Yahoo Financial''' 22 | if self.symbol is None: 23 | LOG.debug('Symbol is None') 24 | return [] 25 | 26 | return self.__yf.getQuotes(self.symbol, start, end) 27 | -------------------------------------------------------------------------------- /ultrafinance/dam/yahooFinance.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 21, 2010 3 | 4 | @author: ppa 5 | 6 | Thanks to Corey Goldberg, this module is based http://www.goldb.org/ystockquote.html 7 | sample usage: 8 | >>> import YahooFinance 9 | >>> print YahooFinance.get_price('GOOG') 10 | 529.46 11 | ''' 12 | import urllib 13 | import traceback 14 | from ultrafinance.model import Quote 15 | from ultrafinance.lib.errors import UfException, Errors 16 | 17 | import logging 18 | LOG = logging.getLogger() 19 | 20 | class YahooFinance(object): 21 | def __request(self, symbol, stat): 22 | try: 23 | url = 'http://finance.yahoo.com/d/quotes.csv?s=%s&f=%s' % (symbol, stat) 24 | return urllib.urlopen(url).read().strip().strip('"') 25 | except IOError: 26 | raise UfException(Errors.NETWORK_ERROR, "Can't connect to Yahoo server") 27 | except BaseException: 28 | raise UfException(Errors.UNKNOWN_ERROR, "Unknown Error in YahooFinance.__request %s" % traceback.format_exc()) 29 | 30 | def getAll(self, symbol): 31 | """ 32 | Get all available quote data for the given ticker symbol. 33 | Returns a dictionary. 34 | """ 35 | values = self.__request(symbol, 'l1c1va2xj1b4j4dyekjm3m4rr5p5p6s7').split(',') 36 | data = {} 37 | data['price'] = values[0] 38 | data['change'] = values[1] 39 | data['volume'] = values[2] 40 | data['avg_daily_volume'] = values[3] 41 | data['stock_exchange'] = values[4] 42 | data['market_cap'] = values[5] 43 | data['book_value'] = values[6] 44 | data['ebitda'] = values[7] 45 | data['dividend_per_share'] = values[8] 46 | data['dividend_yield'] = values[9] 47 | data['earnings_per_share'] = values[10] 48 | data['52_week_high'] = values[11] 49 | data['52_week_low'] = values[12] 50 | data['50day_moving_avg'] = values[13] 51 | data['200day_moving_avg'] = values[14] 52 | data['price_earnings_ratio'] = values[15] 53 | data['price_earnings_growth_ratio'] = values[16] 54 | data['price_sales_ratio'] = values[17] 55 | data['price_book_ratio'] = values[18] 56 | data['short_ratio'] = values[19] 57 | return data 58 | 59 | def getQuotes(self, symbol, start, end): 60 | """ 61 | Get historical prices for the given ticker symbol. 62 | Date format is 'YYYY-MM-DD' 63 | 64 | Returns a nested list. 65 | """ 66 | try: 67 | start = str(start).replace('-', '') 68 | end = str(end).replace('-', '') 69 | 70 | url = 'http://ichart.yahoo.com/table.csv?s=%s&' % symbol + \ 71 | 'd=%s&' % str(int(end[4:6]) - 1) + \ 72 | 'e=%s&' % str(int(end[6:8])) + \ 73 | 'f=%s&' % str(int(end[0:4])) + \ 74 | 'g=d&' + \ 75 | 'a=%s&' % str(int(start[4:6]) - 1) + \ 76 | 'b=%s&' % str(int(start[6:8])) + \ 77 | 'c=%s&' % str(int(start[0:4])) + \ 78 | 'ignore=.csv' 79 | days = urllib.urlopen(url).readlines() 80 | values = [day[:-2].split(',') for day in days] 81 | # sample values:[['Date', 'Open', 'High', 'Low', 'Close', 'Volume', 'Adj Clos'], \ 82 | # ['2009-12-31', '112.77', '112.80', '111.39', '111.44', '90637900', '109.7']...] 83 | data = [] 84 | for value in values[1:]: 85 | data.append(Quote(value[0], value[1], value[2], value[3], value[4], value[5], value[6])) 86 | 87 | dateValues = sorted(data, key = lambda q: q.time) 88 | return dateValues 89 | 90 | except IOError: 91 | raise UfException(Errors.NETWORK_ERROR, "Can't connect to Yahoo server") 92 | except BaseException: 93 | raise UfException(Errors.UNKNOWN_ERROR, "Unknown Error in YahooFinance.getHistoricalPrices %s" % traceback.format_exc()) 94 | #sample output 95 | #[stockDaylyData(date='2010-01-04, open='112.37', high='113.39', low='111.51', close='113.33', volume='118944600', adjClose='111.6'))...] -------------------------------------------------------------------------------- /ultrafinance/designPattern/__init__.py: -------------------------------------------------------------------------------- 1 | ''' design patterns ''' -------------------------------------------------------------------------------- /ultrafinance/designPattern/observable.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on July 31, 2011 3 | 4 | @author: ppa 5 | 6 | copyied from http://code.activestate.com/recipes/131499-observer-pattern/ 7 | Not sure whether it's a good idea to remove observer class 8 | ''' 9 | class Observable: 10 | ''' class for observer design pattern ''' 11 | def __init__(self): 12 | ''' constructor ''' 13 | self._observers = [] 14 | 15 | def attach(self, observer): 16 | ''' add observer ''' 17 | if not observer in self._observers: 18 | self._observers.append(observer) 19 | 20 | def detach(self, observer): 21 | ''' remove observer ''' 22 | try: 23 | self._observers.remove(observer) 24 | except ValueError: 25 | pass 26 | 27 | def notify(self, modifier=None): 28 | ''' call update() method of observer ''' 29 | for observer in self._observers: 30 | if modifier != observer: 31 | observer.update(self) -------------------------------------------------------------------------------- /ultrafinance/lib/__init__.py: -------------------------------------------------------------------------------- 1 | ''' ultraFinance lib ''' 2 | import os, sys 3 | 4 | import logging 5 | LOG = logging.getLogger(__name__) 6 | 7 | ''' 8 | mainSrc = os.path.join(os.path.dirname( (os.path.dirname(os.path.abspath(__file__))))) 9 | sys.path.append(mainSrc) 10 | 11 | logging.basicConfig(level=logging.DEBUG, 12 | format='%(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] [%(threadName)s] %(message)s', 13 | filename=os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'ultraFinance.log'), 14 | filemode='w') 15 | ''' 16 | -------------------------------------------------------------------------------- /ultrafinance/lib/errors.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on May 6, 2011 3 | 4 | @author: ppa 5 | ''' 6 | import traceback 7 | 8 | class Errors(object): 9 | """ class hosts error code constants """ 10 | # general errors 11 | UNKNOWN_ERROR = 1 12 | FILE_NOT_EXIST = 2 13 | FILE_EXIST = 3 14 | UNDEFINED_METHOD = 4 15 | 16 | NETWORK_ERROR = 100 17 | NETWORK_400_ERROR = 101 18 | 19 | INDEX_RANGE_ERROR = 200 20 | INVALID_DAM_TYPE = 201 21 | 22 | STOCK_SYMBOL_ERROR = 300 23 | STOCK_PARSING_ERROR = 301 24 | 25 | HBASE_CREATE_ERROR = 401 26 | HBASE_UPDATE_ERROR = 402 27 | 28 | #type eroor 29 | SIDE_TYPE_ERROR = 500 30 | ORDER_TYPE_ERROR = 501 31 | TRANSITION_TYPE_ERROR = 502 32 | 33 | #tickFeeder 34 | FEEDER_INVALID_ERROR = 600 35 | SYMBOL_EXIST = 601 36 | INVALID_TYPE = 602 37 | SYMBOL_NOT_IN_SOURCE = 604 38 | FEEDER_TIMEOUT = 605 39 | 40 | #account error 41 | ORDER_INVALID_ERROR = 700 42 | MISSING_SYMBOL = 701 43 | 44 | #excelLib error 45 | SHEET_NAME_EXIST = 800 46 | SHEET_NAME_INVALID = 801 47 | INVALID_EXCEL_MODE = 802 48 | 49 | #trading error 50 | INVALID_ACCOUNT = 901 51 | 52 | #metric 53 | INVALID_METRIC_NAME = 1001 54 | ACCOUNT_ALEADY_SET = 1002 55 | ACCOUNT_NOT_SET = 1003 56 | INVALID_RISK_FREE_RETURN = 1004 57 | 58 | #strategy 59 | INVALID_STRATEGY_NAME = 1200 60 | NONE_ACCOUNT_ID = 1201 61 | NONE_TRADING_CENTER = 1202 62 | INVALID_SYMBOLS = 1203 63 | 64 | #outputSaver 65 | TABLENAME_NOT_SET = 1300 66 | TABLENAME_ALREADY_SET = 1301 67 | INVALID_SAVER_NAME = 1302 68 | 69 | class UfException(Exception): 70 | """ Ultra-Finance exception """ 71 | def __init__(self, error, errorMsg): 72 | """ constructor """ 73 | super(UfException, self).__init__() 74 | self.__error = error 75 | self.__errorMsg = errorMsg 76 | 77 | def __str__(self): 78 | """ string """ 79 | return repr(self.__errorMsg) 80 | 81 | def getCode(self): 82 | """ accessor """ 83 | return self.__error 84 | 85 | def getMsg(self): 86 | """ accessor """ 87 | return "%s: %s" % (self.__errorMsg, traceback.format_exc(5)) 88 | -------------------------------------------------------------------------------- /ultrafinance/lib/plotDateValueDict.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Jan 3, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from matplotlib import pyplot 7 | from datetime import datetime 8 | from ultrafinance.lib.errors import UfException, Errors 9 | 10 | import logging 11 | LOG = logging.getLogger() 12 | 13 | class PlotDateValueDict(object): 14 | ''' plot dict with date value ''' 15 | def __init__(self, dateValueDict, dateFormat = '%Y%m%d', lowMargin = 0.05, upMargin = 0.05, rightMargin = 0.05, leftMargin = 0.05, betweenMargin = 0.05): 16 | ''' constructor ''' 17 | self.dateValueDict = dateValueDict 18 | self.length = len(dateValueDict.keys()) 19 | self.lowMargin = lowMargin 20 | self.upMargin = upMargin 21 | self.leftMargin = leftMargin 22 | self.rightMargin = rightMargin 23 | self.dateFormat = dateFormat 24 | 25 | self.rect = [] 26 | height = float(1 - self.lowMargin - self.upMargin - (self.length - 1) * betweenMargin) / self.length 27 | pre = self.lowMargin 28 | for _ in range(self.length): 29 | self.rect.append([self.leftMargin, pre, 1 - self.leftMargin - self.rightMargin , height]) 30 | pre = pre + height + betweenMargin 31 | 32 | pyplot.rc('axes', grid = True) 33 | pyplot.rc('grid', color = '0.75', linestyle = '-', linewidth = 0.5) 34 | 35 | def plot(self): 36 | ''' plot dataValue ''' 37 | try: 38 | fig = pyplot.figure() 39 | 40 | i = 0 41 | ax0 = None 42 | for label, dateValues in self.dateValueDict.items(): 43 | if 0 == i: 44 | ax = fig.add_axes(self.rect[i]) 45 | ax0 = ax 46 | else: 47 | ax = fig.add_axes(self.rect[i], sharex = ax0) 48 | i += 1 49 | ax.plot_date([datetime.strptime(dateValue[0], self.dateFormat) for dateValue in dateValues], 50 | [dateValue[1] for dateValue in dateValues], fmt = 'b-') 51 | ax.set_ylabel(label) 52 | ax.set_ylim(min([int(dateValue[1]) for dateValue in dateValues]) / 1.1, max([int(dateValue[1]) for dateValue in dateValues]) * 1.1) 53 | #ax.set_ylim(0, 1000) 54 | 55 | #pyplot.legend() 56 | pyplot.show() 57 | 58 | except UfException as excep: 59 | raise excep 60 | except BaseException as excep: 61 | raise UfException(Errors.UNKNOWN_ERROR, "plotDateValueDict.plot got unknown error %s" % excep) 62 | -------------------------------------------------------------------------------- /ultrafinance/lib/util.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 18, 2010 3 | 4 | @author: ppa 5 | ''' 6 | import sys 7 | from bs4 import BeautifulSoup 8 | from datetime import datetime 9 | from datetime import timedelta 10 | from datetime import date 11 | import time 12 | from time import gmtime, strftime 13 | 14 | import logging 15 | LOG = logging.getLogger() 16 | 17 | googCSVDateformat = "%d-%b-%y" 18 | 19 | def importClass(path, moduleName, className = None): 20 | ''' dynamically import class ''' 21 | if not className: 22 | className = capitalize(moduleName) 23 | sys.path.append(path) 24 | 25 | mod = __import__(moduleName) 26 | return getattr(mod, className) 27 | 28 | def capitalize(inputString): 29 | ''' capitalize first letter ''' 30 | if not inputString: 31 | return inputString 32 | elif 1 == len(inputString): 33 | return inputString[0].upper() 34 | else: 35 | return inputString[0].upper() + inputString[1:] 36 | 37 | def deCapitalize(inputString): 38 | ''' capitalize first letter ''' 39 | if not inputString: 40 | return inputString 41 | elif 1 == len(inputString): 42 | return inputString[0].lower() 43 | else: 44 | return inputString[0].lower() + inputString[1:] 45 | 46 | def splitByComma(inputString): 47 | ''' split string by comma ''' 48 | return [name.strip() for name in inputString.split(',')] 49 | 50 | def convertGoogCSVDate(googCSVDate): 51 | ''' convert date 25-Jul-2010 to 20100725''' 52 | d = str(datetime.strptime(googCSVDate, googCSVDateformat).date()) 53 | return d.replace("-", "") 54 | 55 | def findPatthen(page, pList): 56 | datas = [BeautifulSoup(page)] 57 | for key, pattern in pList: 58 | newDatas = findPattern(datas, key, pattern) 59 | 60 | datas = newDatas 61 | if not datas: 62 | break 63 | 64 | return datas 65 | 66 | def findPattern(datas, key, pattern): 67 | newDatas = [] 68 | for data in datas: 69 | if 'id' == key: 70 | newDatas.extend(data.findAll(id = pattern, recursive = True)) 71 | if 'text' == key: 72 | newDatas.extend(data.findAll(text = pattern, recursive = True)) 73 | 74 | return newDatas 75 | def string2EpochTime(stingTime, format = '%Y%m%d'): 76 | ''' convert string time to epoch time ''' 77 | return int(time.mktime(datetime.strptime(stingTime, '%Y%m%d').timetuple())) 78 | 79 | def string2datetime(stringTime, format = '%Y%m%d'): 80 | ''' convert string time to epoch time''' 81 | return datetime.strptime(stringTime, '%Y%m%d') 82 | 83 | 84 | def splitListEqually(inputList, chunks): 85 | return [inputList[i : i + chunks] for i in range(0, len(inputList), chunks)] 86 | 87 | def splitDictEqually(inputDict, chunks): 88 | '''Splits dict by keys. Returns a list of dictionaries. 89 | from http://enginepewpew.blogspot.com/2012/03/splitting-dictionary-into-equal-chunks.html 90 | ''' 91 | return_list = [dict() for idx in xrange(chunks)] 92 | idx = 0 93 | for k,v in inputDict.iteritems(): 94 | return_list[idx][k] = v 95 | if idx < chunks-1: # indexes start at 0 96 | idx += 1 97 | else: 98 | idx = 0 99 | return return_list 100 | 101 | def getDateString(numDaysBefore): 102 | ''' return string represent date of n days ago ''' 103 | t = date.today() - timedelta(days=numDaysBefore) 104 | return t.strftime("%Y%m%d") 105 | -------------------------------------------------------------------------------- /ultrafinance/module/__init__.py: -------------------------------------------------------------------------------- 1 | ''' high level module that can be used directly ''' -------------------------------------------------------------------------------- /ultrafinance/module/googleCrawler.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Dec 4, 2011 3 | 4 | @author: ppa 5 | ''' 6 | from ultrafinance.dam.DAMFactory import DAMFactory 7 | 8 | from os import path 9 | import time 10 | 11 | from threading import Thread 12 | from threading import Lock 13 | 14 | import logging 15 | LOG = logging.getLogger() 16 | 17 | 18 | THREAD_TIMEOUT = 5 19 | MAX_TRY = 3 20 | 21 | class GoogleCrawler(object): 22 | ''' collect quotes/ticks for a list of symbol ''' 23 | def __init__(self, symbols, start, poolsize = 5): 24 | ''' constructor ''' 25 | self.symbols = symbols 26 | self.sqlLocation = None 27 | self.outputDAM = DAMFactory.createDAM("sql", self.__getOutputDamSetting()) 28 | self.googleDAM = DAMFactory.createDAM("google") 29 | self.start = start 30 | self.end = None 31 | self.poolsize = poolsize 32 | self.readLock = Lock() 33 | self.writeLock = Lock() 34 | self.failed = [] 35 | self.succeeded = [] 36 | 37 | def __getOutputDamSetting(self): 38 | self.sqlLocation = 'sqlite:///%s' % self.__getOutputSql() 39 | LOG.info("Sqlite location: %s" % self.sqlLocation) 40 | return {'db': self.sqlLocation} 41 | 42 | def __getOutputSql(self): 43 | return path.join("/" 44 | "data", 45 | "stock.sqlite") 46 | 47 | def __getSaveOneSymbol(self, symbol): 48 | ''' get and save data for one symbol ''' 49 | try: 50 | lastExcp = None 51 | with self.readLock: #dam is not thread safe 52 | failCount = 0 53 | #try several times since it may fail 54 | while failCount < MAX_TRY: 55 | try: 56 | self.googleDAM.setSymbol(symbol) 57 | quotes = self.googleDAM.readQuotes(self.start, self.end) 58 | 59 | except BaseException as excp: 60 | failCount += 1 61 | lastExcp = excp 62 | else: 63 | break 64 | 65 | if failCount >= MAX_TRY: 66 | raise BaseException("Can't retrieve historical data %s" % lastExcp) 67 | 68 | with self.writeLock: #dam is not thread safe 69 | self.outputDAM.setSymbol(symbol) 70 | self.outputDAM.writeQuotes(quotes) 71 | 72 | except BaseException as excp: 73 | LOG.info("Error while processing %s: %s" % (symbol, excp)) 74 | self.failed.append(symbol) 75 | else: 76 | LOG.info("Processed %s" % symbol) 77 | self.succeeded.append(symbol) 78 | 79 | def getAndSaveSymbols(self): 80 | ''' get and save data ''' 81 | counter = 0 82 | rounds = 0 83 | 84 | while counter < len(self.symbols): 85 | size = len(self.symbols) - counter 86 | if self.poolsize < size: 87 | size = self.poolsize 88 | symbols = self.symbols[counter: counter + size] 89 | 90 | threads = [] 91 | for symbol in symbols: 92 | thread = Thread(name = symbol, target = self.__getSaveOneSymbol, args = [symbol]) 93 | thread.daemon = True 94 | thread.start() 95 | 96 | threads.append(thread) 97 | 98 | for thread in threads: 99 | thread.join(THREAD_TIMEOUT) # no need to block, because thread should complete at last 100 | 101 | #can't start another thread to do commit because for sqlLite, only object for the same thread can be commited 102 | if 0 == rounds % 3: 103 | self.outputDAM.commit() 104 | 105 | counter += size 106 | rounds += 1 107 | 108 | # sleep for 3 second to avoid being blocked by google... 109 | time.sleep(5) 110 | 111 | if __name__ == '__main__': 112 | crawler = GoogleCrawler(["AAPL", "EBAY", "GOOG"], "20131101") 113 | crawler.getAndSaveSymbols() 114 | print("Sqlite location: %s" % crawler.sqlLocation) 115 | print("Succeeded: %s" % crawler.succeeded) 116 | print("Failed: %s" % crawler.failed) 117 | 118 | -------------------------------------------------------------------------------- /ultrafinance/pyTaLib/__init__.py: -------------------------------------------------------------------------------- 1 | """ talib python impl""" -------------------------------------------------------------------------------- /ultrafinance/ufConfig/__init__.py: -------------------------------------------------------------------------------- 1 | ''' config module ''' -------------------------------------------------------------------------------- /ultrafinance/ufConfig/pyConfig.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 30, 2010 3 | 4 | @author: ppa 5 | ''' 6 | import ConfigParser 7 | from os import path 8 | from ultrafinance.lib.errors import UfException, Errors 9 | 10 | import logging 11 | LOG = logging.getLogger() 12 | 13 | class PyConfig(object): 14 | ''' class that handles configuration ''' 15 | def __init__(self): 16 | ''' Constructor ''' 17 | self.__dir = None 18 | self.__parser = None 19 | self.__fullPath = None 20 | 21 | def setSource(self, fileName): 22 | ''' 23 | set source file name 24 | assume the fileName is full path first, if can't find it, use conf directory 25 | ''' 26 | fullPath = path.abspath(fileName) 27 | 28 | if not path.exists(fullPath): 29 | fullPath = path.join(path.join(path.dirname(path.dirname(path.dirname(path.abspath(__file__)))), 'conf'), 30 | fileName) 31 | if not path.exists(fullPath): 32 | msg = "config file doesn't exist at: %s or %s" % (fileName, fullPath) 33 | LOG.error(msg) 34 | raise UfException(Errors.FILE_NOT_EXIST, msg) 35 | 36 | self.__parser = ConfigParser.SafeConfigParser(defaults={"here": self.__dir}) 37 | self.__parser.read(fullPath) 38 | self.__fullPath = fullPath 39 | 40 | def getDir(self): 41 | ''' get directory of conf file''' 42 | self.__validateConfig() 43 | return path.dirname(self.__fullPath) 44 | 45 | 46 | def getSection(self, section): 47 | ''' load all configuration ''' 48 | self.__validateConfig() 49 | 50 | configs = {} 51 | if self.__parser and self.__parser.has_section(section): 52 | for name, value in self.__parser.items(section): 53 | configs[name] = value 54 | return configs 55 | 56 | return configs 57 | 58 | def getOption(self, section, option): 59 | ''' whether an option exists in the section ''' 60 | self.__validateConfig() 61 | 62 | if self.__parser and self.__parser.has_option(section, option): 63 | return self.__parser.get(section, option) 64 | else: 65 | return None 66 | 67 | def getFullPath(self): 68 | ''' get full path of config ''' 69 | return self.__fullPath 70 | 71 | def override(self, section, key, value): 72 | ''' override/set a key value pair''' 73 | if not self.__parser.has_section(section): 74 | self.__parser.add_section(section) 75 | 76 | self.__parser.set(section, key, str(value)) 77 | 78 | def __validateConfig(self): 79 | ''' validate config is ok ''' 80 | if self.__parser is None: 81 | msg = "No config file is loaded, please use setSource method first" 82 | LOG.error(msg) 83 | raise UfException(Errors.FILE_NOT_EXIST, msg) 84 | 85 | 86 | 87 | if __name__ == '__main__': 88 | config = PyConfig() 89 | config.setSource('test.ini') 90 | config.override("testSection", "123", "456") 91 | config.override("testSection", "123", "567") 92 | print(config.getOption('app_main', 'feeder')) 93 | print(config.getSection('app_main')) 94 | print(config.getOption("testSection", "123")) 95 | --------------------------------------------------------------------------------