├── .gitignore ├── script ├── create_db ├── drop_db ├── setup_db ├── reset_db ├── console ├── migrate ├── delete_security_name_dbs ├── seed └── import ├── app ├── domain │ ├── currency.rb │ ├── time_zone.rb │ ├── find_security_classification.rb │ ├── find_corporate_action.rb │ ├── find_eod_bar.rb │ ├── create_security.rb │ ├── corporate_action_adjustment.rb │ ├── lookup_fundamentals.rb │ ├── find_security.rb │ ├── find_fundamentals.rb │ └── find_time_series.rb ├── importers │ ├── quandl_fred.rb │ ├── quandl_cme.rb │ ├── quandl_fed.rb │ ├── quandl_us_census.rb │ ├── quandl_us_treasury.rb │ ├── quandl_bls.rb │ ├── bsym_exchanges.rb │ ├── yahoo_eod.rb │ ├── yahoo_splits_and_dividends.rb │ ├── bsym_securities.rb │ ├── quandl_eod.rb │ ├── quandl_time_series_importer.rb │ ├── exchanges.rb │ ├── quandl_fundamentals.rb │ ├── optiondata.rb │ └── csidata.rb ├── lru_cache.rb ├── corporate_action_loader.rb ├── time.rb ├── clients │ ├── browser.rb │ ├── csidata.rb │ ├── quandl_fundamentals.rb │ ├── quandl_eod.rb │ └── bsym.rb ├── eod_bar_loader.rb ├── database.rb ├── security_classification_loader.rb ├── time_series_map.rb ├── time_series_map_loader.rb ├── security_name_database.rb ├── time_series_observation_loader.rb ├── stats.rb └── data_model.rb ├── migrations ├── 003_lengthen_time_series_name.rb ├── 006_add_date_to_security_classifications.rb ├── 004_add_value_field_to_corporate_actions.rb ├── 002_lengthen_listed_security_symbol.rb ├── 007_remove_default_and_add_uniqueness_constraint_to_security_classifications.rb ├── 005_replace_sector_and_industry_relations_with_classification_relation.rb ├── 008_create_classification_stats.rb └── 001_create_initial_schema.rb ├── config └── application.sample.yml ├── test.rb ├── app_config.rb ├── Gemfile ├── LICENSE ├── Gemfile.lock ├── application.rb └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea 3 | *.log 4 | data 5 | config/application.yml 6 | -------------------------------------------------------------------------------- /script/create_db: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Creating securitiesdb database on localhost." 4 | createdb securitiesdb -h localhost 5 | -------------------------------------------------------------------------------- /script/drop_db: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Dropping securitiesdb database from localhost." 4 | dropdb securitiesdb -h localhost 5 | -------------------------------------------------------------------------------- /script/setup_db: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | MYDIR="$(dirname "$(which "$0")")" 4 | 5 | $MYDIR/create_db && $MYDIR/migrate && $MYDIR/seed 6 | -------------------------------------------------------------------------------- /script/reset_db: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | MYDIR="$(dirname "$(which "$0")")" 4 | 5 | ($MYDIR/drop_db 2> /dev/null || true) && $MYDIR/delete_security_name_dbs && $MYDIR/setup_db 6 | -------------------------------------------------------------------------------- /app/domain/currency.rb: -------------------------------------------------------------------------------- 1 | # represents an ISO 4217 currency code (see https://en.wikipedia.org/wiki/ISO_4217; e.g. USD, EUR, CHF, etc.) 2 | module Currency 3 | USD = "USD" 4 | EUR = "EUR" 5 | end 6 | -------------------------------------------------------------------------------- /migrations/003_lengthen_time_series_name.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | alter_table :time_series do 4 | set_column_type :name, String, text: true 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /migrations/006_add_date_to_security_classifications.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | alter_table :security_classifications do 4 | add_column :date, Integer, null: false, default: 19000101 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /app/domain/time_zone.rb: -------------------------------------------------------------------------------- 1 | # represents a named time zone from the IANA time zone database (see http://www.joda.org/joda-time/timezones.html) 2 | module TimeZone 3 | US_CENTRAL_TIME = "America/Chicago" 4 | US_EASTERN_TIME = "America/New_York" 5 | end 6 | -------------------------------------------------------------------------------- /migrations/004_add_value_field_to_corporate_actions.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | alter_table :corporate_actions do 4 | add_column :value, BigDecimal, null: true # for dividends, this will hold the dividend amount 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /migrations/002_lengthen_listed_security_symbol.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | alter_table :listed_securities do 4 | set_column_type :symbol, String, size: 21 # we need to support OCC option symbols, which are 21 chars long 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /app/importers/quandl_fred.rb: -------------------------------------------------------------------------------- 1 | class QuandlFredImporter < QuandlTimeSeriesImporter 2 | def import 3 | # import FRED datasets from https://www.quandl.com/data/FRED 4 | 5 | import_quandl_time_series_database("FRED") # Federal Reserve Economic Data - https://www.quandl.com/data/FRED 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /app/importers/quandl_cme.rb: -------------------------------------------------------------------------------- 1 | class QuandlCmeImporter < QuandlTimeSeriesImporter 2 | def import 3 | # import CME datasets from https://www.quandl.com/data/CME 4 | 5 | import_quandl_time_series_database("CME") # Chicago Mercantile Exchange Futures Data - https://www.quandl.com/data/CME 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /app/importers/quandl_fed.rb: -------------------------------------------------------------------------------- 1 | class QuandlFedImporter < QuandlTimeSeriesImporter 2 | def import 3 | # Official US figures on money supply, interest rates, mortgages, government finances, bank assets and debt, exchange rates, industrial production. 4 | import_quandl_time_series_database("FED") 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /config/application.sample.yml: -------------------------------------------------------------------------------- 1 | company_name_search_database_dir: data/security_name_dbs 2 | database: 3 | connection_string: postgres://username@localhost/database_name OR jdbc:postgresql://localhost/database_name?user=username 4 | log_level: info 5 | database_log_level: error 6 | quandl: 7 | api_key: abc123 8 | api_version: "2015-04-09" 9 | -------------------------------------------------------------------------------- /migrations/007_remove_default_and_add_uniqueness_constraint_to_security_classifications.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | alter_table :security_classifications do 4 | set_column_default :date, nil 5 | 6 | add_index [:classification_id, :security_id, :date], unique: true 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /script/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'irb' 4 | require_relative '../application' 5 | 6 | def main 7 | # Application.load_config(ARGV.first || Application::DEFAULT_CONFIG_FILE_PATH) 8 | Application.load(ARGV.first || Application::DEFAULT_CONFIG_FILE_PATH) 9 | IRB.start 10 | end 11 | 12 | main if __FILE__ == $0 13 | -------------------------------------------------------------------------------- /test.rb: -------------------------------------------------------------------------------- 1 | require 'pp' 2 | require_relative 'application' 3 | 4 | def main 5 | Application.load(ARGV.first || Application::DEFAULT_CONFIG_FILE_PATH) 6 | 7 | # pp CsiData::Client.new.amex 8 | QuandlEod::Client.new(Application.logger).send(:enumerate_rowsets_in_csv) {|symbol, bars| puts symbol, bars.inspect; break } 9 | end 10 | 11 | main 12 | -------------------------------------------------------------------------------- /app/lru_cache.rb: -------------------------------------------------------------------------------- 1 | require 'lru_redux' 2 | 3 | class LruCache 4 | def initialize(size) 5 | @cache = LruRedux::Cache.new(size) 6 | end 7 | 8 | def get(key) 9 | @cache[key] 10 | end 11 | 12 | def set(key, value) 13 | @cache[key] = value 14 | end 15 | 16 | def get_or_set(key, &blk) 17 | @cache.getset(key, &blk) 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /app/importers/quandl_us_census.rb: -------------------------------------------------------------------------------- 1 | class QuandlUsCensusImporter < QuandlTimeSeriesImporter 2 | def import 3 | # Data on the American people, places and economy. It provides many data on U.S. imports/exports, domestic production, and other key national indicators. 4 | import_quandl_time_series_database("USCENSUS") # Census stats - https://www.quandl.com/data/USCENSUS 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /app/domain/find_security_classification.rb: -------------------------------------------------------------------------------- 1 | class FindSecurityClassification 2 | class << self 3 | def at_or_earlier_than(security, major = nil, minor = nil, micro = nil, datestamp) 4 | security_classification_time_series_map = SecurityClassificationLoader.get(security, major, minor, micro) 5 | security_classification_time_series_map.latest_value_at_or_earlier_than(datestamp) 6 | end 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /script/migrate: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require_relative '../application' 4 | 5 | def migrate 6 | # puts "sequel -m migrations/ #{AppConfig.database.connection_string}" 7 | `sequel -m migrations/ #{AppConfig.database.connection_string}` 8 | end 9 | 10 | def main 11 | Application.load_config(ARGV.first || Application::DEFAULT_CONFIG_FILE_PATH) 12 | puts "Migrating database." 13 | migrate 14 | end 15 | 16 | main if __FILE__ == $0 17 | -------------------------------------------------------------------------------- /app/importers/quandl_us_treasury.rb: -------------------------------------------------------------------------------- 1 | class QuandlUsTreasuryImporter < QuandlTimeSeriesImporter 2 | def import 3 | # The U.S. Treasury ensures the nation's financial security, manages the nation's debt, collects tax revenues, and issues currency, provides data on yield rates. 4 | import_quandl_time_series_database("USTREASURY") # Treasury rates, yield curve rates, debt, tax revenues, etc. - https://www.quandl.com/data/USTREASURY 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /app_config.rb: -------------------------------------------------------------------------------- 1 | require 'settingslogic' 2 | require 'uri' 3 | 4 | class AppConfig < Settingslogic 5 | def self.load(config_file_path) 6 | source config_file_path 7 | # namespace environment 8 | suppress_errors true 9 | load! 10 | 11 | # connection_string = self.database.connection_string 12 | # uri = URI(connection_string) if connection_string 13 | # self["database_adapter"] = uri ? uri.scheme : nil 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /app/domain/find_corporate_action.rb: -------------------------------------------------------------------------------- 1 | class FindCorporateAction 2 | class << self 3 | def between(security, start_datestamp, end_datestamp, inclusive_of_start_datestamp = true, inclusive_of_end_datestamp = false) 4 | corporate_action_time_series_map = CorporateActionLoader.get(security) 5 | corporate_action_time_series_map.between(start_datestamp, end_datestamp, inclusive_of_start_datestamp, inclusive_of_end_datestamp).values 6 | end 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /script/delete_security_name_dbs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require_relative '../application' 4 | 5 | def delete_security_name_dbs 6 | files_to_delete = Dir.glob(File.join(AppConfig.company_name_search_database_dir, "*.db")) 7 | files_to_delete.each {|file_path| File.delete(file_path) } 8 | end 9 | 10 | def main 11 | Application.load_config(ARGV.first || Application::DEFAULT_CONFIG_FILE_PATH) 12 | puts "Deleting Security Names DBs." 13 | delete_security_name_dbs 14 | end 15 | 16 | main if __FILE__ == $0 17 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gem "sequel" 4 | 5 | # for ruby 2.3.0 6 | gem "pg" 7 | gem "sequel_pg" 8 | 9 | # for JRuby 10 | # gem "jdbc-postgres" 11 | 12 | gem "settingslogic" 13 | gem "logging" 14 | gem "slop" 15 | gem "json" 16 | # gem "beefcake" 17 | 18 | gem "nokogiri" 19 | gem "watir-webdriver" 20 | gem "quandl" 21 | gem "simple-spreadsheet" # for reading xls and xlsx files 22 | gem "rubyzip" 23 | gem "simstring_pure" 24 | gem "text" # for Double Metaphone algorihtm 25 | gem "lru_redux" 26 | gem "treemap" 27 | -------------------------------------------------------------------------------- /app/importers/quandl_bls.rb: -------------------------------------------------------------------------------- 1 | class QuandlBlsImporter < QuandlTimeSeriesImporter 2 | def import 3 | # import BLS datasets from https://www.quandl.com/data/BLSE 4 | 5 | # BLSE/CEU0000000001 -> Employment - All employees, thousands; Total nonfarm industry 6 | # import_quandl_time_series("BLSE", "CEU0000000001") 7 | 8 | import_quandl_time_series_database("BLSE") # BLS Employment & Unemployment - https://www.quandl.com/data/BLSE 9 | import_quandl_time_series_database("BLSI") # BLS Inflation & Prices - https://www.quandl.com/data/BLSI 10 | import_quandl_time_series_database("BLSB") # BLS Pay & Benefits - https://www.quandl.com/data/BLSB 11 | import_quandl_time_series_database("BLSP") # BLS Productivity - https://www.quandl.com/data/BLSP 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /app/corporate_action_loader.rb: -------------------------------------------------------------------------------- 1 | require 'singleton' 2 | 3 | class CorporateActionLoader < TimeSeriesMapLoader 4 | include Singleton 5 | 6 | def self.get(security) 7 | instance.get(security) 8 | end 9 | 10 | 11 | def initialize() 12 | super(100) # 100 TimeSeriesMap objects - one per security 13 | end 14 | 15 | protected 16 | 17 | # compute cache key that identifies a unique Security 18 | def cache_key(security) 19 | security.id 20 | end 21 | 22 | # query the database all the corporate actions associated with 23 | def find_observations(security) 24 | security.corporate_actions.to_a 25 | end 26 | 27 | def extract_observation_time(corporate_action) 28 | corporate_action.ex_date 29 | end 30 | 31 | def extract_observation_value(corporate_action) 32 | corporate_action 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /app/time.rb: -------------------------------------------------------------------------------- 1 | require 'date' 2 | 3 | require_relative 'date' 4 | 5 | LocalTime = Struct.new(:hour, :minute, :second) 6 | 7 | class DateTime 8 | class << self 9 | def timestamp_to_dt(timestamp) 10 | ts_str = timestamp.to_s 11 | year = ts_str.slice(0, 4).to_i 12 | month = ts_str.slice(4, 2).to_i 13 | day = ts_str.slice(6, 2).to_i 14 | hour = ts_str.slice(8, 2).to_i 15 | min = ts_str.slice(10, 2).to_i 16 | sec = ts_str.slice(12, 2).to_i 17 | DateTime.new(year, month, day, hour, min, sec) 18 | end 19 | 20 | def to_timestamp(t) 21 | ((((t.year * 100 + t.month) * 100 + t.day) * 100 + t.hour) * 100 + t.minute) * 100 + t.second 22 | end 23 | end 24 | end 25 | 26 | module DateTimeExtensions 27 | refine DateTime do 28 | def to_timestamp 29 | DateTime.to_timestamp(self) 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /app/clients/browser.rb: -------------------------------------------------------------------------------- 1 | require 'watir-webdriver' 2 | 3 | module Browser 4 | class << self 5 | def open(download_directory = Dir.pwd) 6 | download_directory.gsub!("/", "\\") if Selenium::WebDriver::Platform.windows? 7 | 8 | profile = Selenium::WebDriver::Firefox::Profile.new 9 | profile['browser.download.dir'] = download_directory 10 | profile['browser.download.folderList'] = 2 # When set to 2, the location specified for the most recent download is utilized again. 11 | profile['browser.helperApps.neverAsk.saveToDisk'] = "application/octet-stream" 12 | Watir::Browser.new :firefox, :profile => profile 13 | 14 | # profile = Selenium::WebDriver::Chrome::Profile.new 15 | # profile['download.prompt_for_download'] = false 16 | # profile['download.default_directory'] = download_directory 17 | # Watir::Browser.new :chrome, :profile => profile 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /migrations/005_replace_sector_and_industry_relations_with_classification_relation.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | alter_table :securities do 4 | drop_foreign_key :industry_id 5 | drop_foreign_key :sector_id 6 | end 7 | 8 | drop_table :industries 9 | drop_table :sectors 10 | 11 | create_table :classifications do 12 | primary_key :id 13 | String :major, text: true, null: false # primary classification 14 | String :minor, text: true, null: false # sub-classification 15 | String :micro, text: true, null: false # sub-sub-classification 16 | 17 | index :id, unique: true 18 | index [:major, :minor, :micro], unique: true 19 | index :minor 20 | index :micro 21 | end 22 | 23 | create_table :security_classifications do 24 | primary_key :id 25 | foreign_key :classification_id, :classifications, null: false 26 | foreign_key :security_id, :securities, null: false 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /app/domain/find_eod_bar.rb: -------------------------------------------------------------------------------- 1 | class FindEodBar 2 | class << self 3 | def at_or_earlier_than(security, datestamp) 4 | # first attempt to find the eod_bar - search this year's worth of eod bars 5 | tsmap = EodBarLoader.get(security, datestamp) 6 | eod_bar = tsmap.latest_value_at_or_earlier_than(datestamp) 7 | return eod_bar if eod_bar 8 | 9 | # second attempt to find the eod_bar - search previous year's worth of eod bars 10 | year, month, day = *Date.datestamp_components(datestamp) 11 | tsmap = EodBarLoader.get(security, Date.build_datestamp(year - 1, month, day)) 12 | eod_bar = tsmap.latest_value_at_or_earlier_than(datestamp) 13 | return eod_bar if eod_bar 14 | 15 | # third attempt to find the eod bar - query DB for latest eod bar observed on or earlier than 16 | security.eod_bars_dataset. 17 | where { date <= datestamp }. 18 | order(Sequel.desc(:date)). 19 | limit(1). 20 | first 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /app/domain/create_security.rb: -------------------------------------------------------------------------------- 1 | require 'singleton' 2 | 3 | class CreateSecurity 4 | include Singleton 5 | 6 | class << self 7 | extend Forwardable 8 | def_delegators :instance, :run 9 | end 10 | 11 | 12 | def run(name, security_type_name) 13 | security_type = find_or_create_security_type(security_type_name) 14 | 15 | security = Security.create( 16 | security_type_id: security_type && security_type.id, 17 | name: name, 18 | search_key: extract_search_key_from_security_name(name) 19 | ) 20 | 21 | db = SecurityNameDatabaseRegistry.get(security_type_name) 22 | search_key = extract_search_key_from_security_name(name) 23 | db.add(search_key) 24 | 25 | security 26 | end 27 | 28 | private 29 | 30 | def extract_search_key_from_security_name(security_name) 31 | security_name.downcase 32 | end 33 | 34 | def find_or_create_security_type(security_type_name) 35 | if security_type_name && !security_type_name.empty? 36 | SecurityType.first(name: security_type_name) || SecurityType.create(name: security_type_name) 37 | end 38 | end 39 | 40 | end 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 David Ellis 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /app/eod_bar_loader.rb: -------------------------------------------------------------------------------- 1 | require 'singleton' 2 | 3 | class EodBarLoader < TimeSeriesMapLoader 4 | include Singleton 5 | 6 | def self.get(security, datestamp) 7 | instance.get(security, datestamp) 8 | end 9 | 10 | 11 | def initialize() 12 | super(50) # 50 TimeSeriesMap objects - one per unique (Security, year) pair 13 | end 14 | 15 | protected 16 | 17 | # compute cache key that identifies a unique (TimeSeries, year) pair 18 | def cache_key(security, datestamp) 19 | year = datestamp / 10000 20 | "#{security.id}-#{year}" 21 | end 22 | 23 | # query the database for a year's worth of EodBars associated with and covering the date given by 24 | def find_observations(security, datestamp) 25 | year, _, _ = Date.datestamp_components(datestamp) 26 | first_datestamp_of_year = Date.build_datestamp(year, 1, 1) 27 | last_datestamp_of_year = Date.build_datestamp(year, 12, 31) 28 | security.eod_bars_dataset.where { (date >= first_datestamp_of_year) & (date <= last_datestamp_of_year) }.to_a 29 | end 30 | 31 | def extract_observation_time(eod_bar) 32 | eod_bar.date 33 | end 34 | 35 | def extract_observation_value(eod_bar) 36 | eod_bar 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /app/database.rb: -------------------------------------------------------------------------------- 1 | require 'forwardable' 2 | require 'sequel' 3 | require 'uri' 4 | 5 | if RUBY_PLATFORM == "java" # check to see if we're running jruby for JRuby 6 | require 'jdbc/postgres' 7 | end 8 | 9 | class Database 10 | attr_reader :connection 11 | 12 | class << self 13 | extend Forwardable 14 | 15 | def instance 16 | @instance ||= self.new 17 | end 18 | 19 | def_delegator :instance, :connect 20 | def_delegator :instance, :connection 21 | end 22 | 23 | def connect(connection_string, logger = Application.database_logger) 24 | @connection ||= begin 25 | uri = URI(connection_string) 26 | case uri.scheme 27 | when "postgres", "mysql", "sqlite" 28 | connection = Sequel.connect(connection_string, :logger => logger) 29 | Sequel::Model.raise_on_save_failure = true 30 | connection 31 | when "jdbc" 32 | connection = Sequel.connect(connection_string, :logger => logger) 33 | Sequel::Model.raise_on_save_failure = true 34 | connection 35 | else 36 | raise "There is no database adapter for the following connection scheme: #{uri.scheme}" 37 | end 38 | end 39 | end 40 | 41 | def disconnect 42 | connection.disconnect 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /app/security_classification_loader.rb: -------------------------------------------------------------------------------- 1 | require 'singleton' 2 | 3 | class SecurityClassificationLoader < TimeSeriesMapLoader 4 | include Singleton 5 | 6 | # a nil value for major, minor, or macro means that parameter is a wildcard (*) 7 | def self.get(security, major = nil, minor = nil, micro = nil) 8 | instance.get(security, major, minor, micro) 9 | end 10 | 11 | 12 | def initialize() 13 | super(100) # 100 TimeSeriesMap objects - one per (Security, major, minor, micro) tuple 14 | end 15 | 16 | protected 17 | 18 | # compute cache key that identifies a unique (Security, major, minor, micro) tuple 19 | def cache_key(security, major, minor, micro) 20 | "#{security.id}-#{major}-#{minor}-#{micro}" 21 | end 22 | 23 | # query the database for all the SecurityClassifications that are associated with the given (Security, major, minor, micro) tuple 24 | def find_observations(security, major, minor, micro) 25 | query = {} 26 | query[:major] = major if major 27 | query[:minor] = minor if minor 28 | query[:micro] = micro if micro 29 | security.security_classifications.where(query).to_a 30 | end 31 | 32 | def extract_observation_time(security_classification) 33 | security_classification.date 34 | end 35 | 36 | def extract_observation_value(security_classification) 37 | security_classification 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /migrations/008_create_classification_stats.rb: -------------------------------------------------------------------------------- 1 | Sequel.migration do 2 | change do 3 | create_table :security_variables do 4 | primary_key :id 5 | String :name, text: true, null: false 6 | end 7 | 8 | create_table :classification_summaries do 9 | primary_key :id 10 | foreign_key :classification_id, :classifications, null: false 11 | foreign_key :security_variable_id, :security_variables, null: false 12 | Integer :date, null: false 13 | 14 | Integer :n 15 | BigDecimal :mean 16 | BigDecimal :variance 17 | BigDecimal :min 18 | BigDecimal :max 19 | BigDecimal :percentile_1 20 | BigDecimal :percentile_5 21 | BigDecimal :percentile_10 22 | BigDecimal :percentile_15 23 | BigDecimal :percentile_20 24 | BigDecimal :percentile_25 25 | BigDecimal :percentile_30 26 | BigDecimal :percentile_35 27 | BigDecimal :percentile_40 28 | BigDecimal :percentile_45 29 | BigDecimal :percentile_50 30 | BigDecimal :percentile_55 31 | BigDecimal :percentile_60 32 | BigDecimal :percentile_65 33 | BigDecimal :percentile_70 34 | BigDecimal :percentile_75 35 | BigDecimal :percentile_80 36 | BigDecimal :percentile_85 37 | BigDecimal :percentile_90 38 | BigDecimal :percentile_95 39 | BigDecimal :percentile_99 40 | end 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /script/seed: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require_relative '../application' 4 | 5 | def seed 6 | CorporateActionType.create(name: "Cash Dividend") 7 | CorporateActionType.create(name: "Split") 8 | 9 | DataVendor.create(name: "Quandl") 10 | 11 | FundamentalDimension.create(name: "INST", description: "Instantaneous point in time snapshot.") 12 | FundamentalDimension.create(name: "ARY", description: "As reported, annually.") 13 | FundamentalDimension.create(name: "ARQ", description: "As reported, quarterly.") 14 | FundamentalDimension.create(name: "ART-Q", description: "As reported, trailing twelve months (TTM). Aggregated over quarterly observations.") 15 | FundamentalDimension.create(name: "MRY", description: "Most-recent reported, annually.") 16 | FundamentalDimension.create(name: "MRQ", description: "Most-recent reported, quarterly.") 17 | FundamentalDimension.create(name: "MRT-Q", description: "Most-recent reported, trailing twelve months (TTM). Aggregated over quarterly observations.") 18 | 19 | SecurityType.create(name: "Equity Option", :classification => "Option") 20 | 21 | UpdateFrequency.create(label: "Daily") 22 | UpdateFrequency.create(label: "Weekly") 23 | UpdateFrequency.create(label: "Monthly") 24 | UpdateFrequency.create(label: "Quarterly") 25 | UpdateFrequency.create(label: "Yearly") 26 | UpdateFrequency.create(label: "Irregular") 27 | end 28 | 29 | def main 30 | Application.load(ARGV.first || Application::DEFAULT_CONFIG_FILE_PATH) 31 | puts "Seeding database." 32 | seed 33 | end 34 | 35 | main if __FILE__ == $0 36 | -------------------------------------------------------------------------------- /app/time_series_map.rb: -------------------------------------------------------------------------------- 1 | require 'treemap' 2 | 3 | class TimeSeriesMap 4 | def initialize(navigable_map = nil) 5 | @navigable_map = navigable_map || TreeMap.new 6 | end 7 | 8 | def add(time, value) 9 | @navigable_map.put(time, value) 10 | end 11 | 12 | def remove(time) 13 | @navigable_map.remove(time) 14 | end 15 | 16 | # returns array of keys 17 | def keys 18 | @navigable_map.key_set.to_a 19 | end 20 | 21 | # returns array of values 22 | def values 23 | @navigable_map.values.to_a 24 | end 25 | 26 | def [](time) 27 | @navigable_map[time] 28 | end 29 | 30 | def get(time) 31 | @navigable_map[time] 32 | end 33 | 34 | def between(start_time, end_time, inclusive_of_start_time = true, inclusive_of_end_time = false) 35 | TimeSeriesMap.new(@navigable_map.sub_map(start_time, inclusive_of_start_time, end_time, inclusive_of_end_time)) 36 | end 37 | 38 | def latest_value_at_or_earlier_than(time) 39 | entry = @navigable_map.floor_entry(time) # floorKey returns the greatest key less than or equal to the given key, or null if there is no such key. 40 | entry.value if entry 41 | end 42 | 43 | def latest_value_earlier_than(time) 44 | key = @navigable_map.lower_entry(time) # lowerKey returns the greatest key strictly less than the given key, or null if there is no such key. 45 | entry.value if entry 46 | end 47 | 48 | def earliest_value_at_or_later_than(time) 49 | key = @navigable_map.ceiling_entry(time) # ceilingKey returns the least key greater than or equal to the given key, or null if there is no such key. 50 | entry.value if entry 51 | end 52 | 53 | def earliest_value_later_than(time) 54 | key = @navigable_map.higher_entry(time) # higherKey returns the least key strictly greater than the given key, or null if there is no such key. 55 | entry.value if entry 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /app/time_series_map_loader.rb: -------------------------------------------------------------------------------- 1 | class TimeSeriesMapLoader 2 | attr_accessor :cache 3 | 4 | def initialize(cache_size = 100) 5 | self.cache = LruCache.new(cache_size) 6 | end 7 | 8 | def get(aggregate_entity, *observation_selection_criteria) 9 | key = cache_key(aggregate_entity, *observation_selection_criteria) 10 | cache.get(key) || load_observations_into_cache(key, aggregate_entity, observation_selection_criteria) 11 | end 12 | 13 | protected 14 | 15 | # compute cache key that will identify the subset of observations belonging to 16 | # that were relevant per the given observation_selection_criteria 17 | def cache_key(aggregate_entity, *observation_selection_criteria) 18 | raise "TimeSeriesMapLoader#cache_key not implemented." 19 | end 20 | 21 | # query the database for a subset of observations belonging to that were relevant at time