├── .gitignore ├── License.txt ├── Makefile ├── README.md ├── THANKS.txt ├── backtester ├── .cargo │ └── config ├── .gitignore ├── Cargo.toml ├── README.md ├── bench.sh ├── profile.sh ├── src │ ├── backtest.rs │ └── main.rs ├── test-specific.sh └── test.sh ├── broker_shims ├── .gitignore ├── FXCM │ ├── java_bridge │ │ ├── .gitignore │ │ ├── build.xml │ │ ├── manifest.mf │ │ ├── nbproject │ │ │ ├── build-impl.xml │ │ │ ├── configs │ │ │ │ └── FXCM.properties │ │ │ ├── genfiles.properties │ │ │ ├── project.properties │ │ │ └── project.xml │ │ └── src │ │ │ └── tickrecorder │ │ │ ├── Config.java │ │ │ ├── HistoryDownloader.java │ │ │ ├── IMPORTANT.md │ │ │ ├── MySessionStatusListener.java │ │ │ ├── PrivateConfig.java │ │ │ ├── PrivateConfig.sample │ │ │ ├── PublicConfig.java │ │ │ ├── ResponseListener.java │ │ │ ├── TableListener.java │ │ │ ├── TableManagerListener.java │ │ │ ├── TickRecorder.java │ │ │ └── redisPubsubListener.java │ └── native │ │ ├── .cargo │ │ └── config │ │ ├── .gitignore │ │ ├── Cargo.toml │ │ ├── README.txt │ │ ├── bench.sh │ │ ├── core.sh │ │ ├── debug.sh │ │ ├── install.sh │ │ ├── native │ │ ├── .vscode │ │ │ ├── c_cpp_properties.json │ │ │ ├── extensions.json │ │ │ └── settings.json │ │ ├── README.txt │ │ ├── build.sh │ │ ├── install.sh │ │ └── src │ │ │ ├── GlobalResponseListener.cpp │ │ │ ├── GlobalResponseListener.h │ │ │ ├── GlobalTableListener.cpp │ │ │ ├── GlobalTableListener.h │ │ │ ├── SessionStatusListener.cpp │ │ │ ├── SessionStatusListener.h │ │ │ ├── broker_server.cpp │ │ │ ├── libfxcm_ffi.cpp │ │ │ ├── libfxcm_ffi.h │ │ │ ├── offers.cpp │ │ │ └── trade_execution.cpp │ │ ├── r2_release.sh │ │ ├── release_debug.sh │ │ ├── release_test.sh │ │ ├── src │ │ ├── helper_objects.rs │ │ └── lib.rs │ │ ├── test.sh │ │ └── valgrind.sh ├── README.md └── simbroker │ ├── .cargo │ └── config │ ├── Cargo.toml │ ├── README.md │ ├── install.sh │ ├── install_release.sh │ └── src │ ├── client.rs │ ├── helpers.rs │ ├── lib.rs │ ├── superlog.rs │ └── tests.rs ├── configurator ├── .cargo │ └── config ├── .gitignore ├── Cargo.toml └── src │ ├── directory.rs │ ├── main.rs │ ├── misc.rs │ ├── schema.rs │ └── theme.rs ├── data ├── .gitignore ├── documents │ ├── .gitignore │ └── reference │ │ ├── 53b34794-22c8-4cdb-97af-6b28b70111ae.json │ │ └── README.md ├── historical_ticks │ └── .gitignore ├── notebooks │ └── .gitignore └── plot_poloniex_trades.py ├── data_downloaders ├── fxcm_flatfile │ ├── .cargo │ │ └── config │ ├── .gitignore │ ├── Cargo.toml │ ├── install.sh │ ├── install_release.sh │ └── src │ │ └── main.rs ├── fxcm_java │ ├── .gitignore │ ├── README.md │ ├── download.js │ └── package.json ├── fxcm_native │ ├── .cargo │ │ └── config │ ├── .gitignore │ ├── Cargo.toml │ ├── README.txt │ ├── debug.sh │ ├── run.sh │ ├── src │ │ └── main.rs │ └── test.sh ├── iex │ ├── .babelrc │ ├── .eslintrc.json │ ├── .flowconfig │ ├── .gitignore │ ├── README.txt │ ├── iex.js │ ├── package.json │ ├── run.sh │ └── src │ │ └── commands.js └── poloniex │ ├── .babelrc │ ├── .eslintrc.json │ ├── .flowconfig │ ├── .gitignore │ ├── debug.sh │ ├── index.js │ ├── inspect.sh │ ├── package.json │ ├── run.sh │ └── src │ ├── historical.js │ └── streaming.js ├── dev.sh ├── gource-recent.sh ├── gource.sh ├── logger ├── .cargo │ └── config ├── .gitignore ├── Cargo.toml ├── README.txt ├── install.sh ├── install_release.sh └── src │ └── main.rs ├── mm-react ├── .eslintrc.json ├── .flowconfig ├── .gitignore ├── .roadhogrc ├── .roadhogrc.mock.js ├── package.json ├── public │ ├── .gitignore │ ├── index.html │ └── staticStyle.css └── src │ ├── components │ ├── AppPage.js │ ├── ContentContainer.js │ ├── DashHeader.js │ ├── MacroBuilder.js │ ├── MacroManager.js │ ├── backtest │ │ ├── BacktestMonitor.js │ │ ├── BacktestStarter.js │ │ └── BacktesterSpawner.js │ ├── data │ │ ├── DataDownloader.js │ │ ├── DataDownloaderSpawner.js │ │ ├── DataManager.js │ │ ├── DownloadControl.js │ │ ├── DstSelector.js │ │ └── RunningDownloads.js │ ├── docs │ │ ├── Ckeditor.js │ │ ├── DocCreator.js │ │ ├── DocSearcher.js │ │ └── DocViewer.js │ ├── instances │ │ ├── Instance.js │ │ ├── InstanceSpawner.js │ │ └── LiveInstances.js │ └── logging │ │ ├── LiveLog.js │ │ └── LogLine.js │ ├── index.js │ ├── models │ ├── Data.js │ ├── Documents.js │ ├── GlobalState.js │ ├── InstanceManagement.js │ ├── Logging.js │ ├── PlatformCommunication.js │ └── macros.js │ ├── router.js │ ├── routes │ ├── Backtest.js │ ├── DataManagement.js │ ├── Documentation.js │ ├── IndexPage.js │ ├── InstanceManagement.js │ ├── Logging.js │ └── NotFound.js │ ├── services │ └── example.js │ ├── static │ └── css │ │ ├── IndexPage.css │ │ ├── globalStyle.css │ │ ├── instances.css │ │ └── logging.css │ └── utils │ ├── commands.js │ ├── data_util.js │ ├── request.js │ └── spawner_macro.js ├── mm ├── .gitignore ├── README.md ├── db.js ├── manager.js ├── package.json ├── routes │ ├── data.js │ └── index.js ├── sources │ ├── backtest.css │ ├── backtest.js │ ├── commands.css │ ├── data_downloaders.js │ ├── global_script.js │ ├── global_style.css │ ├── hc_theme.js │ ├── instance_css.css │ ├── instance_script.js │ ├── log.css │ ├── log.js │ ├── monitor_css.css │ ├── monitor_script.js │ └── plotting.js └── views │ ├── backtest.ejs │ ├── data_downloaders.ejs │ ├── instances.ejs │ ├── log.ejs │ ├── monitor.ejs │ └── partials │ └── header.ejs ├── optimizer ├── .cargo │ └── config ├── .gitignore ├── Cargo.toml └── src │ ├── main.rs │ ├── tests │ ├── benchmarks │ │ ├── misc.rs │ │ ├── mod.rs │ │ └── transport.rs │ └── mod.rs │ └── transport │ ├── mod.rs │ └── test.rs ├── private ├── .cargo │ └── config ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── README.md ├── build.sh ├── install.sh ├── install_release.sh ├── src │ ├── indicators │ │ ├── .gitignore │ │ ├── mod.rs │ │ └── sma.rs │ ├── lib.rs │ ├── sinks │ │ └── mod.rs │ ├── strategies │ │ ├── fuzzer │ │ │ ├── README.md │ │ │ ├── extern │ │ │ │ ├── bindings.cpp │ │ │ │ └── build.sh │ │ │ └── mod.rs │ │ ├── mod.rs │ │ └── sma_cross │ │ │ └── mod.rs │ └── trading_conditions │ │ ├── mod.rs │ │ └── sma_cross.rs └── test.sh ├── run.sh ├── scripts ├── .gitignore └── fuzz │ ├── .cargo │ └── config │ ├── .gitignore │ ├── Cargo.toml │ ├── core.sh │ ├── gdb.sh │ ├── gdb_release.sh │ ├── profile.sh │ ├── rebuild.sh │ ├── rebuild_release.sh │ ├── release.sh │ ├── run.sh │ └── src │ └── main.rs ├── spawner ├── .cargo │ └── config ├── .gitignore ├── Cargo.toml ├── README.md ├── install.sh ├── install_release.sh └── src │ ├── documents.rs │ ├── main.rs │ └── redis_proxy.rs ├── tick_parser ├── .cargo │ └── config ├── .gitignore ├── Cargo.toml ├── src │ ├── main.rs │ ├── processor.rs │ └── transport │ │ ├── mod.rs │ │ └── test.rs └── test.sh ├── tick_writer └── writer.js └── util ├── .cargo └── config ├── .gitignore ├── Cargo.toml ├── from_hashmap ├── .gitignore ├── Cargo.toml └── src │ └── lib.rs ├── js ├── .babelrc ├── .eslintrc.json ├── .flowconfig ├── .gitignore ├── README.txt ├── index.js ├── package.json └── src │ └── ffi.js └── src ├── instance.rs ├── lib.rs ├── strategies.rs ├── trading ├── broker.rs ├── datafield.rs ├── indicators.rs ├── mod.rs ├── objects.rs ├── tick.rs └── trading_condition.rs └── transport ├── command_server.rs ├── commands.rs ├── data.rs ├── ffi ├── mod.rs └── poloniex.rs ├── mod.rs ├── postgres.rs ├── query_server.rs ├── redis.rs ├── textlog.rs └── tickstream ├── generators ├── flatfile_reader.rs ├── mod.rs ├── postgres_reader.rs ├── random_reader.rs └── redis_reader.rs ├── generics.rs ├── maps ├── mod.rs └── poloniex.rs ├── mod.rs └── sinks ├── console_sink.rs ├── csv_sink.rs ├── mod.rs ├── null_sink.rs ├── redis_sink.rs └── stream_sink.rs /.gitignore: -------------------------------------------------------------------------------- 1 | dist 2 | core 3 | vgcore.* 4 | *.dot 5 | Cargo.lock 6 | *.dump 7 | callgrind.* 8 | *.so 9 | 10 | # vim swap files 11 | *.swp 12 | 13 | # wordgrinder documents 14 | *.wg 15 | npm-debug.log 16 | -------------------------------------------------------------------------------- /License.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Casey Primozic 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /THANKS.txt: -------------------------------------------------------------------------------- 1 | This project wouldn't have happened were it not for a wide range of people. 2 | 3 | - Alex Jensen (https://github.com/dalexj): For helping me learn programming and always being there for me to spam screenshots to 4 | - Prof. Rosasco (http://www.valpo.edu/computing-information-sciences/about-us/faculty/nicholas-rosasco/): For being a great professor and lending some awesome ideas to the platform 5 | - Shepmaster (Jake Goulding; https://github.com/shepmaster): For practically running the Rust stack overflow and keeping everything high-quality 6 | -------------------------------------------------------------------------------- /backtester/.cargo/config: -------------------------------------------------------------------------------- 1 | [build] 2 | rustflags = [ 3 | "-L", "../broker_shims/simbroker/target/release", 4 | "-L", "../../broker_shims/simbroker/target/release", 5 | "-L", "../../../broker_shims/simbroker/target/release", 6 | "-L", "../util/target/release/deps", 7 | "-L", "../../util/target/release/deps", 8 | "-L", "../../../util/target/release/deps", 9 | "-L", "../../../../util/target/release/deps", 10 | "-L", "../dist/lib", 11 | "-C", "prefer-dynamic" 12 | ] 13 | -------------------------------------------------------------------------------- /backtester/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /backtester/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "backtester" 3 | version = "0.1.0" 4 | authors = ["Casey Primozic "] 5 | 6 | [profile.release] 7 | opt-level = 3 8 | debug = true 9 | debug-assertions = false 10 | -------------------------------------------------------------------------------- /backtester/README.md: -------------------------------------------------------------------------------- 1 | # TickGrinder Backtester 2 | The backtester is an external application that integrates into the TickGrinder platform and simulates live market data in order to test the platform's trading activities. Its primary purpose is to allow users to test out strategies and view their performance based on historical data downloaded elsewhere. 3 | 4 | # Backtest Modes 5 | There are different modes that the backtester can use, each acting a bit differently than the others. This allows for the backtester to be used in unique situations where different parts of the platform's performance are important. 6 | 7 | # Live Mode 8 | Live mode attempts to simulate live trading conditions as closely as possible. It reads ticks out at the rate that they were recorded, simulating the exact conditions that would be experienced in a live trading environment. This main purpose of this mode is to verify platform integrity and verify that it acts as expected in a live environment. 9 | 10 | # Fast Mode 11 | In Fast Mode, ticks are sent through the system as fast as it can handle them. The ideas is that the platform will still act in the same way it would in a live trading environment but the rate at which it processes the data is greatly amplified. This mode is best suited for profiling strategies and determining their runtime characteristics, profitability, and other statistics. 12 | -------------------------------------------------------------------------------- /backtester/bench.sh: -------------------------------------------------------------------------------- 1 | CARGO_INCREMENTAL=1 RUST_BACKTRACE=1 LD_LIBRARY_PATH=native/dist:../util/target/release/deps cargo bench -- --nocapture 2 | -------------------------------------------------------------------------------- /backtester/profile.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=/home/casey/.multirust/toolchains/nightly-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/lib/:native/dist:../util/target/release/deps \ 2 | valgrind --dump-instr=yes --tool=callgrind $(find | rg -N release/deps/backtester-) retran 3 | -------------------------------------------------------------------------------- /backtester/src/backtest.rs: -------------------------------------------------------------------------------- 1 | //! Defines a backtest, which determines what data is sent and the 2 | //! conditions that trigger it to be sent. 3 | 4 | use std::sync::mpsc; 5 | use uuid::Uuid; 6 | 7 | use {BacktestType, DataSource, DataDest}; 8 | use simbroker::SimBrokerSettings; 9 | use tickgrinder_util::transport::tickstream::TickstreamCommand; 10 | 11 | /// Contains controls for pausing, resuming, and stopping a backtest as well as 12 | /// some data about it. 13 | pub struct BacktestHandle { 14 | pub symbol: String, 15 | pub backtest_type: BacktestType, 16 | pub data_source: DataSource, 17 | pub endpoint: DataDest, 18 | pub handle: mpsc::SyncSender, 19 | } 20 | 21 | #[derive(Serialize, Deserialize, Debug, Clone)] 22 | pub struct SerializableBacktestHandle { 23 | pub uuid: Uuid, 24 | pub symbol: String, 25 | pub backtest_type: BacktestType, 26 | pub data_source: DataSource, 27 | pub endpoint: DataDest, 28 | } 29 | 30 | impl SerializableBacktestHandle { 31 | pub fn from_handle(handle: &BacktestHandle, uuid: Uuid) -> SerializableBacktestHandle { 32 | SerializableBacktestHandle { 33 | uuid: uuid, 34 | symbol: handle.symbol.clone(), 35 | backtest_type: handle.backtest_type.clone(), 36 | data_source: handle.data_source.clone(), 37 | endpoint: handle.endpoint.clone(), 38 | } 39 | } 40 | } 41 | 42 | /// Contains all the information necessary to start a backtest 43 | #[derive(Serialize, Deserialize, Clone, Debug)] 44 | pub struct BacktestDefinition { 45 | pub start_time: Option, 46 | /// Stop backtest after timestamp reached or None 47 | pub max_timestamp: Option, 48 | /// Stop backtest after `max_tick_n` ticks have been processed or None 49 | pub max_tick_n: Option, 50 | pub symbol: String, 51 | pub backtest_type: BacktestType, 52 | pub data_source: DataSource, 53 | pub data_dest: DataDest, 54 | pub broker_settings: SimBrokerSettings, 55 | } 56 | 57 | /// Ticks sent to the SimBroker should be re-broadcast to the client. 58 | #[test] 59 | fn tick_retransmission() { 60 | use std::collections::HashMap; 61 | 62 | use futures::{Future, Stream}; 63 | 64 | use tickgrinder_util::trading::tick::Tick; 65 | use simbroker::*; 66 | 67 | // create the SimBroker 68 | let symbol = "TEST".to_string(); 69 | let mut sim_client = SimBrokerClient::init(HashMap::new()).wait().unwrap().unwrap(); 70 | 71 | // subscribe to ticks from the SimBroker for the test pair 72 | let subbed_ticks = sim_client.sub_ticks(symbol).unwrap(); 73 | 74 | let res: Vec = subbed_ticks 75 | .wait() 76 | .take(10) 77 | .map(|t| { 78 | println!("Received tick: {:?}", t); 79 | t.unwrap() 80 | }) 81 | .collect(); 82 | assert_eq!(res.len(), 10); 83 | } 84 | -------------------------------------------------------------------------------- /backtester/test-specific.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=../dist/lib RUSTFLAGS="-L ../util/target/debug/deps -L ../dist/lib -C prefer-dynamic" cargo test retrans -- --nocapture 2 | -------------------------------------------------------------------------------- /backtester/test.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=../dist/lib RUSTFLAGS="-L ../util/target/debug/deps -L ../dist/lib -C prefer-dynamic" cargo test -- --nocapture 2 | -------------------------------------------------------------------------------- /broker_shims/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock 3 | src/conf.rs 4 | -------------------------------------------------------------------------------- /broker_shims/FXCM/java_bridge/.gitignore: -------------------------------------------------------------------------------- 1 | # NetBeans specific 2 | nbproject/private/ 3 | build/ 4 | nbbuild/ 5 | dist/ 6 | nbdist/ 7 | nbactions.xml 8 | nb-configuration.xml 9 | 10 | # Class Files 11 | *.class 12 | 13 | # Package Files 14 | *.jar 15 | *.war 16 | *.ear 17 | -------------------------------------------------------------------------------- /broker_shims/FXCM/java_bridge/manifest.mf: -------------------------------------------------------------------------------- 1 | Manifest-Version: 1.0 2 | X-COMMENT: Main-Class will be added automatically by build 3 | 4 | -------------------------------------------------------------------------------- /broker_shims/FXCM/java_bridge/nbproject/configs/FXCM.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ameobea/tickgrinder/f2ccd640b173cb17b0af4e04df9b203935eb2dc2/broker_shims/FXCM/java_bridge/nbproject/configs/FXCM.properties -------------------------------------------------------------------------------- /broker_shims/FXCM/java_bridge/nbproject/genfiles.properties: -------------------------------------------------------------------------------- 1 | build.xml.data.CRC32=1d66b6e6 2 | build.xml.script.CRC32=043cd532 3 | build.xml.stylesheet.CRC32=8064a381@1.79.1.48 4 | # This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml. 5 | # Do not edit this file. You may delete it but then the IDE will never regenerate such files for you. 6 | nbproject/build-impl.xml.data.CRC32=1d66b6e6 7 | nbproject/build-impl.xml.script.CRC32=cb4ae7f4 8 | nbproject/build-impl.xml.stylesheet.CRC32=05530350@1.79.1.48 9 | -------------------------------------------------------------------------------- /broker_shims/FXCM/java_bridge/nbproject/project.properties: -------------------------------------------------------------------------------- 1 | annotation.processing.enabled=true 2 | annotation.processing.enabled.in.editor=false 3 | annotation.processing.processors.list= 4 | annotation.processing.run.all.processors=true 5 | annotation.processing.source.output=${build.generated.sources.dir}/ap-source-output 6 | application.title=TickRecorder 7 | application.vendor=Casey 8 | auxiliary.org-netbeans-spi-editor-hints-projects.perProjectHintSettingsFile=nbproject/cfg_hints.xml 9 | build.classes.dir=${build.dir}/classes 10 | build.classes.excludes=**/*.java,**/*.form 11 | # This directory is removed when the project is cleaned: 12 | build.dir=build 13 | build.generated.dir=${build.dir}/generated 14 | build.generated.sources.dir=${build.dir}/generated-sources 15 | # Only compile against the classpath explicitly listed here: 16 | build.sysclasspath=ignore 17 | build.test.classes.dir=${build.dir}/test/classes 18 | build.test.results.dir=${build.dir}/test/results 19 | # Uncomment to specify the preferred debugger connection transport: 20 | #debug.transport=dt_socket 21 | debug.classpath=\ 22 | ${run.classpath} 23 | debug.test.classpath=\ 24 | ${run.test.classpath} 25 | # Files in build.classes.dir which should be excluded from distribution jar 26 | dist.archive.excludes= 27 | # This directory is removed when the project is cleaned: 28 | dist.dir=dist 29 | dist.jar=${dist.dir}/TickRecorder.jar 30 | dist.javadoc.dir=${dist.dir}/javadoc 31 | endorsed.classpath= 32 | excludes= 33 | file.reference.fxcore2.jar=C:\\Program Files\\Candleworks\\ForexConnectAPIx64\\bin\\java\\fxcore2.jar 34 | file.reference.jedis-2.8.0.jar=C:\\Users\\Casey\\Desktop\\jedis-2.8.0.jar 35 | includes=** 36 | jar.compress=false 37 | javac.classpath=\ 38 | ${file.reference.fxcore2.jar}:\ 39 | ${file.reference.jedis-2.8.0.jar}:\ 40 | ${libs.json-simple.classpath}:\ 41 | ${libs.apache-commons-pool2.classpath} 42 | # Space-separated list of extra javac options 43 | javac.compilerargs= 44 | javac.deprecation=false 45 | javac.external.vm=true 46 | javac.processorpath=\ 47 | ${javac.classpath} 48 | javac.source=1.8 49 | javac.target=1.8 50 | javac.test.classpath=\ 51 | ${javac.classpath}:\ 52 | ${build.classes.dir} 53 | javac.test.processorpath=\ 54 | ${javac.test.classpath} 55 | javadoc.additionalparam= 56 | javadoc.author=false 57 | javadoc.encoding=${source.encoding} 58 | javadoc.noindex=false 59 | javadoc.nonavbar=false 60 | javadoc.notree=false 61 | javadoc.private=false 62 | javadoc.splitindex=true 63 | javadoc.use=true 64 | javadoc.version=false 65 | javadoc.windowtitle= 66 | main.class=tickrecorder.TickRecorder 67 | manifest.file=manifest.mf 68 | meta.inf.dir=${src.dir}/META-INF 69 | mkdist.disabled=false 70 | platform.active=default_platform 71 | run.classpath=\ 72 | ${javac.classpath}:\ 73 | ${build.classes.dir} 74 | # Space-separated list of JVM arguments used when running the project. 75 | # You may also define separate properties like run-sys-prop.name=value instead of -Dname=value. 76 | # To set system properties for unit tests define test-sys-prop.name=value: 77 | run.jvmargs=-Djava.library.path=.;./java 78 | run.test.classpath=\ 79 | ${javac.test.classpath}:\ 80 | ${build.test.classes.dir} 81 | source.encoding=UTF-8 82 | src.dir=src 83 | test.src.dir=test 84 | -------------------------------------------------------------------------------- /broker_shims/FXCM/java_bridge/nbproject/project.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | org.netbeans.modules.java.j2seproject 4 | 5 | 6 | TickRecorder 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /broker_shims/FXCM/java_bridge/src/tickrecorder/Config.java: -------------------------------------------------------------------------------- 1 | package tickrecorder; 2 | 3 | public class Config implements PublicConfig { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /broker_shims/FXCM/java_bridge/src/tickrecorder/HistoryDownloader.java: -------------------------------------------------------------------------------- 1 | package tickrecorder; 2 | 3 | import com.fxcore2.*; 4 | import java.util.Calendar; 5 | 6 | public class HistoryDownloader { 7 | public static void downloadHistory(O2GSession session, String pair, String resolution, Calendar startTime, Calendar endTime, String uuid){ 8 | O2GRequestFactory factory = session.getRequestFactory(); 9 | try{ 10 | O2GTimeframeCollection timeFrames = factory.getTimeFrameCollection(); 11 | O2GTimeframe timeFrame = timeFrames.get(resolution); 12 | O2GRequest marketDataRequest = factory.createMarketDataSnapshotRequestInstrument(pair, timeFrame, 300); 13 | factory.fillMarketDataSnapshotRequestTime(marketDataRequest, startTime, endTime, true); 14 | TickRecorder.redisPublish("historicalPrices", "{\"type\": \"segmentID\", \"id\": \"" + 15 | marketDataRequest.getRequestId() + "\", \"uuid\": \"" + uuid + "\"}"); 16 | session.sendRequest(marketDataRequest); 17 | }catch(java.lang.NullPointerException ex){ 18 | downloadHistory(session, pair, resolution, startTime, endTime, uuid); 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /broker_shims/FXCM/java_bridge/src/tickrecorder/IMPORTANT.md: -------------------------------------------------------------------------------- 1 | ***SETTING THIS SHIT UP*** 2 | 3 | So there is a lot of work that goes into setting up this particular piece of software. First of all, you need to add the FXCM trading API as a library. 4 | The guide for this can be found here: http://fxcodebase.com/bin/forexconnect/1.4.1/help/Java/order2go2_Java.html and I've downloaded a copy in case of disaster here: https://ameo.link/u/1oe.png 5 | 6 | The FXCM Jar needs to be in the lib directory, and all the DLLs need to be in the system PATH. I'm not sure at this point how to set that up for Linux, but for Windows I found it easiest to just 7 | put them into a folder that's already in the PATH, such as system32. That makes the step about adding the -D parameter for the JVM unnecessary. 8 | 9 | I read somewhere on the website that openJDK or something like that for linux is very unadvised and to use Oracle Java instead. 10 | 11 | For Redis, I used the Jedis library: https://github.com/xetorthio/jedis/releases. I just added it as a library. 12 | 13 | You'll need to include a JSON library as well; I used json-simple: https://code.google.com/archive/p/json-simple/downloads. Require the jar as a library. -------------------------------------------------------------------------------- /broker_shims/FXCM/java_bridge/src/tickrecorder/MySessionStatusListener.java: -------------------------------------------------------------------------------- 1 | package tickrecorder; 2 | 3 | import com.fxcore2.*; 4 | 5 | public class MySessionStatusListener implements IO2GSessionStatus { 6 | private boolean isConnected = false; 7 | private boolean hasError = false; 8 | 9 | public void onSessionStatusChanged(O2GSessionStatusCode ogssc){ 10 | System.out.println("Session Status Changed with code " + ogssc.toString()); 11 | if(ogssc.toString() == "CONNECTED"){ 12 | isConnected = true; 13 | } 14 | } 15 | 16 | public void onLoginFailed(String string){ 17 | System.out.println("Login failed!"); 18 | hasError = true; 19 | } 20 | 21 | public boolean isConnected(){ 22 | if(isConnected){ 23 | return true; 24 | }else{ 25 | return false; 26 | } 27 | } 28 | 29 | public boolean hasError(){ 30 | if(hasError){ 31 | return true; 32 | }else{ 33 | return false; 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /broker_shims/FXCM/java_bridge/src/tickrecorder/PrivateConfig.java: -------------------------------------------------------------------------------- 1 | package tickrecorder; 2 | 3 | public interface PrivateConfig { 4 | public String FXCMUsername = "D102506685001"; 5 | public String FXCMPassword = "6251"; 6 | public String FXCMHostsURL = "http://www.fxcorporate.com/Hosts.jsp"; 7 | public String redisIP = "chitara.ameo.link"; 8 | public int redisPort = 6379; 9 | } 10 | -------------------------------------------------------------------------------- /broker_shims/FXCM/java_bridge/src/tickrecorder/PrivateConfig.sample: -------------------------------------------------------------------------------- 1 | package tickrecorder; 2 | 3 | public class PrivateConfig { 4 | public String FXCMUsername = "D**********"; 5 | public String FXCMPassword = "1234"; 6 | public String FXCMHostsURL = "http://www.fxcorporate.com/Hosts.jsp"; 7 | public String redisIP = "example.com"; 8 | public int redisPort = 1234; 9 | } 10 | -------------------------------------------------------------------------------- /broker_shims/FXCM/java_bridge/src/tickrecorder/PublicConfig.java: -------------------------------------------------------------------------------- 1 | package tickrecorder; 2 | 3 | public interface PublicConfig extends PrivateConfig { 4 | public String[] monitoredPairs = {"USD/CAD", "EUR/USD", "EUR/JPY", "AUD/USD"}; 5 | public String connectionType = "Demo"; 6 | } 7 | -------------------------------------------------------------------------------- /broker_shims/FXCM/java_bridge/src/tickrecorder/ResponseListener.java: -------------------------------------------------------------------------------- 1 | package tickrecorder; 2 | 3 | import com.fxcore2.*; 4 | import java.util.Calendar; 5 | import java.math.BigDecimal; 6 | 7 | public class ResponseListener implements IO2GResponseListener { 8 | private String requestID; 9 | private O2GSession session; 10 | 11 | public void onRequestCompleted(String requestID, O2GResponse ogr){ 12 | System.out.println("New Response recieved of type " + ogr.getType().toString() + ": " + ogr.toString()); 13 | O2GResponseReaderFactory readerFactory = session.getResponseReaderFactory(); 14 | if(ogr.getType().toString() == "MARKET_DATA_SNAPSHOT"){ 15 | O2GMarketDataSnapshotResponseReader marketSnapshotReader = readerFactory.createMarketDataSnapshotReader(ogr); 16 | Long lastTimestamp = null; 17 | 18 | if(marketSnapshotReader.size() == 300){ 19 | TickRecorder.redisPublish("historicalPrices", "{\"status\": \">300 data\"}"); 20 | } 21 | 22 | String response = "{\"type\": \"segment\", \"id\": \"" + requestID; 23 | response += "\", \"data\": ["; 24 | for (int i = 0; i < marketSnapshotReader.size(); i++) { 25 | response += "{\"timestamp\": "; 26 | Calendar timestampCalendar = marketSnapshotReader.getDate(i); 27 | lastTimestamp = timestampCalendar.getTimeInMillis(); 28 | response += String.valueOf(lastTimestamp); 29 | response += ", \"bid\": "; 30 | response += String.valueOf(marketSnapshotReader.getBid(i)); 31 | response += ", \"ask\": "; 32 | response += String.valueOf(marketSnapshotReader.getAsk(i)); 33 | response += "}"; 34 | if(i < marketSnapshotReader.size()-1){ 35 | response += ", "; 36 | } 37 | } 38 | response += "]}"; 39 | 40 | TickRecorder.redisPublish("historicalPrices", response); 41 | //TickRecorder.redisPublish("historicalPrices", "{\"status\": \"segmentDone\", \"lastTimestamp\": " + String.valueOf(lastTimestamp) + "}"); 42 | } 43 | } 44 | 45 | public void onRequestFailed(String requestID, String err){ 46 | System.out.println("Request failed, " + err); 47 | if(err.contains("unsupported scope")){ 48 | TickRecorder.redisPublish("historicalPrices", "{\"error\": \"No ticks in range\", \"id\": \"" + requestID + "\"}"); 49 | } 50 | } 51 | 52 | public void onTablesUpdates(O2GResponse ogr){ 53 | 54 | } 55 | 56 | public ResponseListener(O2GSession session){ 57 | this.session = session; 58 | } 59 | 60 | public void setRequestID(String requestID){ 61 | this.requestID = requestID; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /broker_shims/FXCM/java_bridge/src/tickrecorder/TableListener.java: -------------------------------------------------------------------------------- 1 | package tickrecorder; 2 | 3 | import com.fxcore2.*; 4 | import static tickrecorder.TickRecorder.redisPublish; 5 | import java.util.Date; 6 | import java.text.DecimalFormat; 7 | 8 | public class TableListener implements IO2GTableListener { 9 | public void onAdded(String string, O2GRow o2grow){ 10 | 11 | } 12 | 13 | public void onChanged(String rowID, O2GRow rowData){ 14 | O2GOfferTableRow offerTableRow = (O2GOfferTableRow)(rowData); 15 | if (offerTableRow!=null){ 16 | for(int i=0;i"] 5 | 6 | [lib] 7 | name = "fxcm" 8 | crate-type = ["dylib"] 9 | 10 | [profile.release] 11 | opt-level = 3 12 | debug = true 13 | debug-assertions = false 14 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/README.txt: -------------------------------------------------------------------------------- 1 | This is a broker shim that uses the native FXCM ForexConnect C++ API to communicate with the broker and exposes its functionality in Rust. 2 | 3 | The code in the `native` directory creates a C++ library that is used to actually communicate with the broker (further documented in the README file inside that directory) and the `src` directory of this Cargo project contains Rust code implementing the Broker trait and communicating with the C++ API via FFI. 4 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/bench.sh: -------------------------------------------------------------------------------- 1 | #cd native && ./build.sh 2 | # LD_LIBRARY_PATH=native/dist strace -f -e trace=network -s 10000 cargo test 3 | LD_LIBRARY_PATH=native/dist:~/tickgrinder/util/target/release/deps cargo bench -- --nocapture 4 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/core.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=~/tickgrinder/dist/lib/:~/tickgrinder/util/target/release/deps/:~/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/ ddd $(find | grep debug/fxcm-) core 2 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/debug.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=~/tickgrinder/dist/lib/:~/tickgrinder/util/target/release/deps/:~/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/ gdbgui $(find | grep debug/fxcm-) 2 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/install.sh: -------------------------------------------------------------------------------- 1 | cargo build 2 | cp target/debug/libfxcm.so ../../../dist/lib 3 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/native/.vscode/c_cpp_properties.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": [ 3 | { 4 | "name": "Mac", 5 | "includePath": ["/usr/include"], 6 | "browse" : { 7 | "limitSymbolsToIncludedHeaders" : true, 8 | "databaseFilename" : "" 9 | } 10 | }, 11 | { 12 | "name": "Linux", 13 | "includePath": ["/usr/include", "./include/ForexConnectAPI-Linux-x86_64/include"], 14 | "browse" : { 15 | "limitSymbolsToIncludedHeaders" : true, 16 | "databaseFilename" : "" 17 | } 18 | }, 19 | { 20 | "name": "Win32", 21 | "includePath": ["c:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/include"], 22 | "browse" : { 23 | "limitSymbolsToIncludedHeaders" : true, 24 | "databaseFilename" : "" 25 | } 26 | } 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/native/.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | // See http://go.microsoft.com/fwlink/?LinkId=827846 3 | // for the documentation about the extensions.json format 4 | "recommendations": [ 5 | // Extension identifier format: ${publisher}.${name}. Example: vscode.csharp 6 | 7 | ] 8 | } -------------------------------------------------------------------------------- /broker_shims/FXCM/native/native/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | // Place your settings in this file to overwrite default and user settings. 2 | { 3 | "C_Cpp.clang_format_style": "LLVM", 4 | "C_Cpp.clang_format_path": "/usr/bin/clang-format", 5 | } -------------------------------------------------------------------------------- /broker_shims/FXCM/native/native/README.txt: -------------------------------------------------------------------------------- 1 | This is the C++ portion of the application. Its purpose is to create all of the utility functions needed to communicate with the broker and perform tasks such as log in, open trades, subscribe to prices, etc. and expose them to the Rust application in a consolidated way that fits the platform's needs. 2 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/native/build.sh: -------------------------------------------------------------------------------- 1 | rm dist -r 2 | mkdir -p dist 3 | cp -u include/ForexConnectAPI-Linux-x86_64/lib/*.so dist 4 | cp include/ForexConnectAPI-Linux-x86_64/samples/cpp/sample_tools/lib/libsample_tools.so dist 5 | g++ -g -rdynamic -shared -fPIC -std=c++11 -O3 src/libfxcm_ffi.cpp src/broker_server.cpp src/GlobalResponseListener.cpp \ 6 | include/ForexConnectAPI-Linux-x86_64/samples/cpp/NonTableManagerSamples/GetHistPrices/source/CommonSources.cpp \ 7 | -I include/ForexConnectAPI-Linux-x86_64/include/ -I include/ForexConnectAPI-Linux-x86_64/samples/cpp/TableManagerSamples/GetOffers/source/ \ 8 | -lboost_system -lboost_thread include/ForexConnectAPI-Linux-x86_64/samples/cpp/NonTableManagerSamples/GetHistPrices/source/ResponseListener.cpp \ 9 | -Wl,--no-undefined -o dist/libfxcm_ffi.so src/GlobalTableListener.cpp src/SessionStatusListener.cpp src/trade_execution.cpp \ 10 | src/offers.cpp include/ForexConnectAPI-Linux-x86_64/samples/cpp/NonTableManagerSamples/GetHistPrices/source/LoginParams.cpp \ 11 | -Iinclude/ForexConnectAPI-Linux-x86_64/samples/cpp/NonTableManagerSamples/Login/source \ 12 | -Iinclude/ForexConnectAPI-Linux-x86_64/samples/cpp/sample_tools/include/ include/ForexConnectAPI-Linux-x86_64/include/ForexConnect.h \ 13 | -Iinclude/ForexConnectAPI-Linux-x86_64/samples/cpp/NonTableManagerSamples/GetHistPrices/source/ \ 14 | dist/libForexConnect.so dist/libfxmsg.so dist/libsample_tools.so 15 | cp dist/libfxcm_ffi.so ../../../../dist/lib 16 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/native/install.sh: -------------------------------------------------------------------------------- 1 | ./build.sh 2 | cp dist/* ../../../../dist/lib 3 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/native/src/GlobalResponseListener.cpp: -------------------------------------------------------------------------------- 1 | #include "stdafx.h" 2 | #include 3 | #include 4 | 5 | #include 6 | #include 7 | #include "GlobalResponseListener.h" 8 | 9 | GlobalResponseListener::GlobalResponseListener() { 10 | mRefCount = 1; 11 | mResponseEvent = CreateEvent(0, FALSE, FALSE, 0); 12 | } 13 | 14 | GlobalResponseListener::~GlobalResponseListener() { 15 | CloseHandle(mResponseEvent); 16 | } 17 | 18 | long GlobalResponseListener::addRef() { 19 | return InterlockedIncrement(&mRefCount); 20 | } 21 | 22 | long GlobalResponseListener::release() { 23 | long rc = InterlockedDecrement(&mRefCount); 24 | if (rc == 0) 25 | delete this; 26 | return rc; 27 | } 28 | 29 | void GlobalResponseListener::setRequestIDs(std::vector &requestIDs) { 30 | mRequestIDs.resize(requestIDs.size()); 31 | std::copy(requestIDs.begin(), requestIDs.end(), mRequestIDs.begin()); 32 | ResetEvent(mResponseEvent); 33 | } 34 | 35 | bool GlobalResponseListener::waitEvents() { 36 | return WaitForSingleObject(mResponseEvent, _TIMEOUT) == 0; 37 | } 38 | 39 | void GlobalResponseListener::stopWaiting() { 40 | SetEvent(mResponseEvent); 41 | } 42 | 43 | void GlobalResponseListener::onRequestCompleted(const char* requestId, IO2GResponse *response) { 44 | switch(response->getType()) { 45 | case GetOffers: { 46 | break; 47 | } 48 | default: { 49 | // TOOD: log 50 | break; 51 | } 52 | } 53 | } 54 | 55 | void GlobalResponseListener::onRequestFailed(const char* requestId , const char* error) { 56 | if (std::find(mRequestIDs.begin(), mRequestIDs.end(), requestId) != mRequestIDs.end()) { 57 | // std::cout << "The request has been failed. ID: " << requestId << " : " << error << std::endl; 58 | // TODO: Log to Rust 59 | stopWaiting(); 60 | } 61 | } 62 | 63 | void GlobalResponseListener::onTablesUpdates(IO2GResponse* data) { 64 | // You should also capture IO2GResponse in the onTablesUpdates function of a response listener class. 65 | // TODO: just log to Rust for now 66 | } 67 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/native/src/GlobalResponseListener.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | class GlobalResponseListener : public IO2GResponseListener { 4 | public: 5 | GlobalResponseListener(); 6 | virtual long addRef(); 7 | virtual long release(); 8 | 9 | void setRequestIDs(std::vector &orderIDs); 10 | 11 | bool waitEvents(); 12 | void stopWaiting(); 13 | 14 | virtual void onRequestCompleted(const char *requestId, IO2GResponse *response = 0); 15 | 16 | virtual void onRequestFailed(const char *requestId , const char *error); 17 | 18 | virtual void onTablesUpdates(IO2GResponse *data); 19 | 20 | private: 21 | long mRefCount; 22 | std::vector mRequestIDs; 23 | HANDLE mResponseEvent; 24 | 25 | protected: 26 | /** Destructor. */ 27 | virtual ~GlobalResponseListener(); 28 | }; 29 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/native/src/GlobalTableListener.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | class GlobalTableListener : public IO2GTableListener { 4 | public: 5 | GlobalTableListener(TickCallback tcb, void* tce, LogCallback lcb, void* lcbe); 6 | 7 | virtual long addRef(); 8 | virtual long release(); 9 | 10 | void setRequestIDs(std::vector &orderIDs); 11 | 12 | void onStatusChanged(O2GTableStatus); 13 | void onAdded(const char*, IO2GRow*); 14 | void onChanged(const char*, IO2GRow*); 15 | void onDeleted(const char*, IO2GRow*); 16 | 17 | void setTickCallback(TickCallback _tcb, void* _tcbe); 18 | void setResponseCallback(ResponseCallback rcb, void* rcbe); 19 | 20 | void subscribeTradingEvents(IO2GTableManager *manager); 21 | void unsubscribeTradingEvents(IO2GTableManager *manager); 22 | void subscribeNewOffers(IO2GTableManager *manager); 23 | void unsubscribeNewOffers(IO2GTableManager *manager); 24 | 25 | private: 26 | long mRefCount; 27 | 28 | // tick callback 29 | TickCallback tick_cb; 30 | void* tick_cb_env; 31 | 32 | // log callback 33 | LogCallback log_cb; 34 | void* log_cb_env; 35 | 36 | // response callback 37 | ResponseCallback res_cb; 38 | void* res_cb_env; 39 | }; 40 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/native/src/SessionStatusListener.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | class SessionStatusListener : public IO2GSessionStatus { 4 | private: 5 | long mRefCount; 6 | bool mError; 7 | bool mConnected; 8 | bool mDisconnected; 9 | IO2GSession *mSession; 10 | /** Event handle. */ 11 | HANDLE mSessionEvent; 12 | LogCallback log_cb; 13 | void* log_cb_env; 14 | std::string mSessionID; 15 | 16 | protected: 17 | ~SessionStatusListener(); 18 | 19 | public: 20 | SessionStatusListener(IO2GSession *session, LogCallback log_cb, void* log_cb_env); 21 | 22 | virtual long addRef(); 23 | virtual long release(); 24 | 25 | virtual void onLoginFailed(const char *error); 26 | virtual void onSessionStatusChanged(IO2GSessionStatus::O2GSessionStatus status); 27 | 28 | bool hasError() const; 29 | bool isConnected() const; 30 | bool isDisconnected() const; 31 | void reset(); 32 | bool waitEvents(); 33 | 34 | void _log(char* msg, CLogLevel severity); 35 | }; 36 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/native/src/offers.cpp: -------------------------------------------------------------------------------- 1 | //! Functions for dealing with offer rows, getting information from/about them and things like that. 2 | 3 | #include "stdafx.h" 4 | #include "CommonSources.h" 5 | 6 | // For all the below functions, see the IO2GOfferRow documentation for their usage. 7 | // http://www.fxcodebase.com/documents/ForexConnectAPI/IO2GOfferRow.html 8 | 9 | double getBid(void* void_row) { 10 | IO2GOfferRow * row = (IO2GOfferRow*)void_row; 11 | return row->getBid(); 12 | } 13 | 14 | const char *getBidTradable(void* void_row) { 15 | IO2GOfferRow * row = (IO2GOfferRow*)void_row; 16 | return row->getBidTradable(); 17 | } 18 | 19 | double getAsk(void* void_row) { 20 | IO2GOfferRow * row = (IO2GOfferRow*)void_row; 21 | return row->getAsk(); 22 | } 23 | 24 | const char *getAskTradable(void* void_row) { 25 | IO2GOfferRow * row = (IO2GOfferRow*)void_row; 26 | return row->getAskTradable(); 27 | } 28 | 29 | extern "C" int getDigits(void* void_row) { 30 | IO2GOfferRow * row = (IO2GOfferRow*)void_row; 31 | return row->getDigits(); 32 | } 33 | 34 | double getHigh(void* void_row) { 35 | IO2GOfferRow * row = (IO2GOfferRow*)void_row; 36 | return row->getHigh(); 37 | } 38 | 39 | double getLow(void* void_row) { 40 | IO2GOfferRow * row = (IO2GOfferRow*)void_row; 41 | return row->getLow(); 42 | } 43 | 44 | int getVolume(void* void_row) { 45 | IO2GOfferRow * row = (IO2GOfferRow*)void_row; 46 | return row->getVolume(); 47 | } 48 | 49 | const char *getTradingStatus(void* void_row) { 50 | IO2GOfferRow * row = (IO2GOfferRow*)void_row; 51 | return row->getAskTradable(); 52 | } 53 | 54 | double getPointSize(void* void_row) { 55 | IO2GOfferRow * row = (IO2GOfferRow*)void_row; 56 | return row->getPointSize(); 57 | } 58 | 59 | double getPipSize(void* void_row) { 60 | IO2GOfferRow * row = (IO2GOfferRow*)void_row; 61 | return getPointSize(row); 62 | } 63 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/native/src/trade_execution.cpp: -------------------------------------------------------------------------------- 1 | //! Functions for opening, closing, modifying, and monitoring trades and positions. 2 | 3 | #include "stdafx.h" 4 | #include "CommonSources.h" 5 | #include "libfxcm_ffi.h" 6 | 7 | void open_market(IO2GSession* session, const char* symbol, int quantity, bool is_long, const char* uuid) { 8 | 9 | } 10 | 11 | void close_market(IO2GSession* session, const char* position_uuid, int quantity, bool was_long) { 12 | 13 | } 14 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/r2_release.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=~/tickgrinder/dist/lib/:~/tickgrinder/util/target/release/deps/:~/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/ r2 $(find | grep release/deps/fxcm-) 2 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/release_debug.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=~/tickgrinder/dist/lib/:~/tickgrinder/util/target/release/deps/:~/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/ ddd $(find | grep release/deps/fxcm-) 2 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/release_test.sh: -------------------------------------------------------------------------------- 1 | #cd native && ./build.sh 2 | # LD_LIBRARY_PATH=native/dist strace -f -e trace=network -s 10000 cargo test 3 | LD_LIBRARY_PATH=native/dist:~/tickgrinder/util/target/release/deps cargo test --release -- --nocapture 4 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/test.sh: -------------------------------------------------------------------------------- 1 | #cd native && ./build.sh 2 | # LD_LIBRARY_PATH=native/dist strace -f -e trace=network -s 10000 cargo test 3 | LD_LIBRARY_PATH=native/dist:../../../util/target/release/deps cargo test -- --nocapture 4 | -------------------------------------------------------------------------------- /broker_shims/FXCM/native/valgrind.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=~/tickgrinder/dist/lib/:~/tickgrinder/util/target/release/deps/:~/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/ valgrind $(find | grep debug/fxcm-) 2 | -------------------------------------------------------------------------------- /broker_shims/README.md: -------------------------------------------------------------------------------- 1 | # Broker Shims 2 | Broker Shims are API wrappers that are written for individual brokers allowing them to interact with the platform in a generic way. They wrap the remote broker API into an internal one utilizing futures. -------------------------------------------------------------------------------- /broker_shims/simbroker/.cargo/config: -------------------------------------------------------------------------------- 1 | [build] 2 | rustflags = [ 3 | "-L", "../../util/target/release/deps", 4 | "-L", "../../../util/target/release/deps", 5 | "-L", "../../dist/lib", 6 | "-C", "prefer-dynamic" 7 | ] 8 | -------------------------------------------------------------------------------- /broker_shims/simbroker/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "simbroker" 3 | version = "0.1.0" 4 | authors = ["Casey Primozic "] 5 | 6 | [lib] 7 | name = "simbroker" 8 | crate-type = ["dylib"] 9 | 10 | [profile.release] 11 | opt-level = 3 12 | debug = true 13 | debug-assertions = false 14 | 15 | [features] 16 | default = [] 17 | superlog = [] 18 | -------------------------------------------------------------------------------- /broker_shims/simbroker/install.sh: -------------------------------------------------------------------------------- 1 | CARGO_INCREMENTAL=1 cargo build 2 | cp target/debug/libsimbroker.so ../../dist/lib 3 | -------------------------------------------------------------------------------- /broker_shims/simbroker/install_release.sh: -------------------------------------------------------------------------------- 1 | CARGO_INCREMENTAL=1 cargo build --release 2 | cp target/release/libsimbroker.so ../../dist/lib 3 | -------------------------------------------------------------------------------- /configurator/.cargo/config: -------------------------------------------------------------------------------- 1 | [build] 2 | rustflags = [ 3 | "-L", "../util/target/release/deps", 4 | "-L", "../../util/target/release/deps", 5 | "-L", "../../../util/target/release/deps", 6 | "-C", "prefer-dynamic" 7 | ] 8 | -------------------------------------------------------------------------------- /configurator/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock 3 | conf.rs 4 | conf.js 5 | settings.json 6 | -------------------------------------------------------------------------------- /configurator/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "configurator" 3 | version = "0.1.0" 4 | authors = ["Casey Primozic "] 5 | 6 | [dependencies] 7 | cursive = { version = "0.6.1", default-features = false, features = ["termion-backend"] } 8 | serde_json = "0.8.4" 9 | indoc = "^0.1" 10 | termion = "1.5.0" 11 | 12 | [profile.release] 13 | opt-level = 3 14 | debug = true 15 | debug-assertions = false 16 | -------------------------------------------------------------------------------- /configurator/src/theme.rs: -------------------------------------------------------------------------------- 1 | //! Defines the custom theme for Cursive 2 | 3 | use cursive::theme::{Theme, BorderStyle, Palette, Color, BaseColor}; 4 | 5 | pub const THEME: Theme = Theme { 6 | shadow: false, 7 | borders: BorderStyle::Simple, 8 | colors: Palette { 9 | background: Color::RgbLowRes(0,1,1), 10 | shadow: Color::Dark(BaseColor::Magenta), 11 | view: Color::Dark(BaseColor::White), 12 | primary: Color::Dark(BaseColor::Black), 13 | secondary: Color::Dark(BaseColor::Blue), 14 | tertiary: Color::Dark(BaseColor::White), 15 | title_primary: Color::Dark(BaseColor::Red), 16 | title_secondary: Color::Dark(BaseColor::Yellow), 17 | highlight: Color::Dark(BaseColor::Red), 18 | highlight_inactive: Color::Dark(BaseColor::Blue), 19 | }, 20 | }; 21 | -------------------------------------------------------------------------------- /data/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | -------------------------------------------------------------------------------- /data/documents/.gitignore: -------------------------------------------------------------------------------- 1 | tantivy_index/ 2 | user_documents/ 3 | 4 | -------------------------------------------------------------------------------- /data/documents/reference/53b34794-22c8-4cdb-97af-6b28b70111ae.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Welcome to TickGrinder", 3 | "body": "# Welcome to TickGrinder\nTickGrinder is a platform.", 4 | "tags": ["intro", "welcome", "start"], 5 | "creation_date": "Sat Feb 18 19:37:52 CST 2017", 6 | "modification_date": "Sat Feb 18 19:39:52 CST 2017", 7 | "id": "53b34794-22c8-4cdb-97af-6b28b70111ae" 8 | } 9 | -------------------------------------------------------------------------------- /data/documents/reference/README.md: -------------------------------------------------------------------------------- 1 | # Documentation 2 | This is the main place where the platform's documentation is kept. However, looking through it here directly is a bad idea since it's unformatted and unorganized! Instead, launch the platform's MM and navigate to the "Docs" tab. 3 | 4 | There, you can browse the entire collection of reference and documentation, search it for ideas you may want to learn about, and add your own notes to be indexed alongside it. 5 | -------------------------------------------------------------------------------- /data/historical_ticks/.gitignore: -------------------------------------------------------------------------------- 1 | *.csv 2 | -------------------------------------------------------------------------------- /data/notebooks/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | -------------------------------------------------------------------------------- /data/plot_poloniex_trades.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import matplotlib.pyplot as plt 3 | import math 4 | 5 | df = pd.read_csv("polo_book_rew_trade.csv") 6 | df.columns = ['timestamp', 'tradeID', 'rate', 'amount', 'date', 'total', 'isBuy'] 7 | ax = plt.gca() 8 | ax.scatter(x=df['timestamp'], y=df['rate'], marker='o', c='b', s=df['amount'].map(lambda x: math.sqrt(x * 100))) 9 | 10 | ax.set_autoscaley_on(False) 11 | ax.set_ylim([min(df['rate']), max(df['rate'])]) 12 | 13 | plt.show() 14 | 15 | -------------------------------------------------------------------------------- /data_downloaders/fxcm_flatfile/.cargo/config: -------------------------------------------------------------------------------- 1 | [build] 2 | rustflags = [ 3 | "-L", "../../util/target/release/deps", 4 | "-L", "../../../util/target/release/deps", 5 | "-L", "../../dist/lib", 6 | "-L", "../../../dist/lib" 7 | ] 8 | -------------------------------------------------------------------------------- /data_downloaders/fxcm_flatfile/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | -------------------------------------------------------------------------------- /data_downloaders/fxcm_flatfile/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fxcm_flatfile" 3 | version = "0.1.0" 4 | authors = ["Casey Primozic "] 5 | description = "Data downloader to download the officially hosted FXCM flatfile tick archives" 6 | 7 | [profile.release] 8 | opt-level = 3 9 | debug = true 10 | debug-assertions = false 11 | -------------------------------------------------------------------------------- /data_downloaders/fxcm_flatfile/install.sh: -------------------------------------------------------------------------------- 1 | CARGO_INCREMENTAL=1 LD_LIBRARY_PATH=../dist/lib:../util/target/release/deps cargo build 2 | cp target/debug/fxcm_flatfile ../dist/fxcm_flatfile_downloader 3 | -------------------------------------------------------------------------------- /data_downloaders/fxcm_flatfile/install_release.sh: -------------------------------------------------------------------------------- 1 | CARGO_INCREMENTAL=1 LD_LIBRARY_PATH=../dist/lib:../util/target/release/deps cargo build --release 2 | cp target/release/fxcm_flatfile ../dist/fxcm_flatfile_downloader 3 | -------------------------------------------------------------------------------- /data_downloaders/fxcm_java/.gitignore: -------------------------------------------------------------------------------- 1 | *.csv 2 | node_modules 3 | -------------------------------------------------------------------------------- /data_downloaders/fxcm_java/README.md: -------------------------------------------------------------------------------- 1 | # FXCM Tick Downloader 2 | This tool integrates with a remote Java server that serves as a link between the FXCM API and this NodeJS application. The Java application can be found in the `tick_recorder` directory. This tool and the Java server are linked using Redis pubsub and all data is passed over a shared channel. 3 | 4 | ## Download Method 5 | The tool requests ticks in 10-second chunks which are queued up all at once, the default being 50 10-second chunks at a time. These requests are relayed through the Java application to the FXCM API servers and return the responses over the pubsub channel as soon as they arrive. If no response is received in a certain amount of time (default 300ms), the missing chunks are resent. 6 | 7 | Each chunk is assigned a `uuid` before being transmitted to the server which is sent back along with the response since responses do not come in order. The server sends a response (`type: segmentID`) as soon as it receives the request that contains the The server also creates an `id` for the request that is used for identification if the request fails (the failure doesn't preserve the `uuid`). 8 | 9 | Once a result is received from the FXCM API, it is relayed back to the client (`type: segment`) and stored. The segment is marked as successful so it is not resent. 10 | 11 | ## Error handling 12 | For cases where no ticks exist within a requested range, the error (`No ticks in range`) is transmitted to the client along with the FXCM request ID. This is matched with the UUID which is then marked as successful so it isn't retransmitted. 13 | 14 | ## Processing result data 15 | The result data is not sorted and may contain duplicate rows. It should be sorted in order of the first column (timestamp) and duplicate rows should be removed. This can be done easily with Linux command line tools such as `sort`; `sort usdcad.csv | uniq -u > usdcad_sorted.csv` should do the trick. 16 | -------------------------------------------------------------------------------- /data_downloaders/fxcm_java/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "fxcm_data_downloader", 3 | "version": "1.0.0", 4 | "description": "Client that interfaces with the FXCM Java server (../TickRecorder) to download historical data.", 5 | "main": "download.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "Casey Primozic ", 10 | "license": "UNLICENSED", 11 | "dependencies": { 12 | "redis": "2.4.2", 13 | "uuid64": "0.0.1", 14 | "bluebird": "3.3.4" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /data_downloaders/fxcm_native/.cargo/config: -------------------------------------------------------------------------------- 1 | [build] 2 | rustflags = [ 3 | "-L", "../../util/target/release/deps", 4 | "-L", "../../../util/target/release/deps", 5 | "-L", "../../dist/lib", 6 | "-L", "../../../dist/lib" 7 | ] 8 | -------------------------------------------------------------------------------- /data_downloaders/fxcm_native/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock 3 | conf.rs 4 | -------------------------------------------------------------------------------- /data_downloaders/fxcm_native/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fxcm_native" 3 | version = "0.1.0" 4 | authors = ["Casey Primozic "] 5 | 6 | [profile.release] 7 | opt-level = 3 8 | debug = true 9 | debug-assertions = false 10 | -------------------------------------------------------------------------------- /data_downloaders/fxcm_native/README.txt: -------------------------------------------------------------------------------- 1 | FXCM Native Data Downloader 2 | 3 | This data downloader makes use of the FXCM native API (located in libfxcm_ffi.so with source code in the broker shims directory of the util library) to enable data downloading from FXCM without requiring a remote Java client. 4 | 5 | The remote API works by passing a callback function to the C++ code which is run for each downloaded tick. The C++ downloader code is responsible for downloading all data in the specified range and calling the callback for each individual tick received. There is no guarentee that the received ticks will be in order, only that they will all be received. 6 | -------------------------------------------------------------------------------- /data_downloaders/fxcm_native/debug.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=../../dist/lib:../../util/target/release/deps cargo build && LD_LIBRARY_PATH=~/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/:../../dist/lib ddd $(find | grep debug/fxcm_native-) 2 | -------------------------------------------------------------------------------- /data_downloaders/fxcm_native/run.sh: -------------------------------------------------------------------------------- 1 | cargo build 2 | LD_LIBRARY_PATH=../../dist/lib:../../util/target/release/deps RUST_BACKTRACE=1 ./target/debug/fxcm_native df193da7-ce00-464a-b2d7-98ffd7bc51f5 3 | -------------------------------------------------------------------------------- /data_downloaders/fxcm_native/test.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=../../dist/lib:../../util/target/release/deps cargo test 2 | -------------------------------------------------------------------------------- /data_downloaders/iex/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["es2015-node5", "stage-3"], 3 | "plugins": [ 4 | "transform-flow-strip-types" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /data_downloaders/iex/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "rules": { 3 | "flowtype/boolean-style": [2, "boolean"], 4 | "flowtype/define-flow-type": 1, 5 | "flowtype/delimiter-dangle": [2, "never"], 6 | "flowtype/generic-spacing": [2, "never"], 7 | "flowtype/no-primitive-constructor-types": 2, 8 | "flowtype/object-type-delimiter": [2, "comma"], 9 | "flowtype/require-parameter-type": 2, 10 | "flowtype/require-return-type": [2, "always", {"annotateUndefined": "never"}], 11 | "flowtype/require-valid-file-annotation": 2, 12 | "flowtype/semi": [2, "always"], 13 | "flowtype/space-after-type-colon": [2, "always"], 14 | "flowtype/space-before-generic-bracket": [2, "never"], 15 | "flowtype/space-before-type-colon": [2, "never"], 16 | "flowtype/type-id-match": [2, "^([A-Z][a-z0-9]+)+Type$"], 17 | "flowtype/union-intersection-spacing": [2, "always"], 18 | "flowtype/use-flow-type": 1, 19 | "flowtype/valid-syntax": 1, 20 | "indent": [2, 2], 21 | "quotes": [2, "single"], 22 | "linebreak-style": [2, "unix"], 23 | "semi": [2, "always"], 24 | "comma-dangle": [2, "only-multiline"], 25 | "no-console": 0, 26 | "no-unused-vars": [2, {"vars": "all", "args": "none"}], 27 | "no-multiple-empty-lines": [2, {"max": 1}] 28 | }, 29 | "env": { 30 | "es6": true, 31 | "browser": false, 32 | "node": true 33 | }, 34 | "extends": ["eslint:recommended"], 35 | "parser": "babel-eslint", 36 | "parserOptions": { 37 | "sourceType": "module", 38 | "ecmaFeatures": { 39 | "experimentalObjectRestSpread": true, 40 | "jsx": false 41 | }, 42 | "ecmaVersion": 7 43 | }, 44 | "plugins": [ 45 | "flowtype" 46 | ], 47 | "globals": { 48 | "require": true, 49 | "module": true 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /data_downloaders/iex/.flowconfig: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ameobea/tickgrinder/f2ccd640b173cb17b0af4e04df9b203935eb2dc2/data_downloaders/iex/.flowconfig -------------------------------------------------------------------------------- /data_downloaders/iex/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | src/conf.js 3 | -------------------------------------------------------------------------------- /data_downloaders/iex/README.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ameobea/tickgrinder/f2ccd640b173cb17b0af4e04df9b203935eb2dc2/data_downloaders/iex/README.txt -------------------------------------------------------------------------------- /data_downloaders/iex/iex.js: -------------------------------------------------------------------------------- 1 | //! The IEX streaming data downloader. Hooks into the free IEX socket.io API to retrieve live top-of-book information 2 | // @flow 3 | 4 | const WebSocket = require('ws'); 5 | 6 | const io = require('socket.io-client'); 7 | const ref = require('ref'); 8 | 9 | import { initWs, handleCommand } from './src/commands'; 10 | import util from 'tickgrinder_util'; 11 | const { TickgrinderUtil, Log, getRxClosure } = util.ffi; 12 | const CONF = require('./src/conf.js'); 13 | 14 | // set up environment for the download 15 | const socket = io(CONF.iex_data_downloader_tops_url); 16 | const cs = TickgrinderUtil.get_command_server(ref.allocCString('IEX Data Downloader')); 17 | if(cs.isNull()) { 18 | console.error('Attempt to create a `CommandServer` returned a null pointer.'); 19 | process.exit(1); 20 | } 21 | 22 | // usage: node manager.js uuid 23 | let ourUuid: string = process.argv[2]; 24 | 25 | if(!ourUuid) { 26 | console.error('Usage: node manager.js uuid'); 27 | process.exit(0); 28 | } else { 29 | Log.notice(cs, '', `IEX Data Downloader now listening for commands on ${CONF.redis_control_channel} and ${ourUuid}`); 30 | } 31 | 32 | // start listening for commands from the platform and responding to them 33 | initWs(handleCommand, null, ourUuid, (err: string) => { 34 | Log.error('', `Error in WebSocket connection: ${err}`); 35 | }).then((socket: WebSocket) => { 36 | Log.notice('', 'WebSocket connection successfully opened to platform communication system'); 37 | }); 38 | 39 | /** 40 | * Starts recording live ticks from the IEX exchange and writing them to the supplied endpiont. 41 | */ 42 | function startDownload(symbols: string[]) { 43 | const rxClosure = getRxClosure(); // TODO 44 | 45 | /** 46 | * Processing an incoming message from the IEX socket server which contains a new top of book data update 47 | */ 48 | const handleWsMessage = (msg: { 49 | symbol: string, 50 | market_percent: number, 51 | bidSize: number, 52 | bidPrice: number, 53 | askSize: number, 54 | askPrice: number, 55 | volume: number, 56 | lastSalePrice: number, 57 | lastSaleSize: number, 58 | lastSaleTime: number, 59 | lastUpdated: number 60 | }) => { 61 | // TickgrinderUtil.c_cs_debug(cs, ref.allocCString(''), ref.allocCString(JSON.stringify(msg)));// + JSON.stringify(msg)); 62 | // console.log(msg); 63 | TickgrinderUtil.exec_c_rx_closure(rxClosure, msg.lastUpdated, msg.bidPrice * 100, msg.askPrice * 100); 64 | }; 65 | 66 | socket.on('message', handleWsMessage); 67 | 68 | socket.on('connect_failed', function() { 69 | Log.error(cs, '', 'Unable to connect to the IEX socket.io API server!'); 70 | }); 71 | 72 | socket.on('connecting', function() { 73 | Log.notice(cs, '', 'Connecting to IEX socket.io API server...'); 74 | }); 75 | 76 | socket.on('connected', function() { 77 | Log.notice(cs, '', 'Successfully connected to IEX socket.io API server.'); 78 | }); 79 | 80 | socket.emit('subscribe', symbols.join(',')); 81 | } 82 | -------------------------------------------------------------------------------- /data_downloaders/iex/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "iex", 3 | "version": "1.0.0", 4 | "dependencies": { 5 | "socket.io": "1.7.3" 6 | }, 7 | "description": "Data Downloader using the public IEX data feed as a data source", 8 | "devDependencies": { 9 | "babel-cli": "*", 10 | "babel-core": "*", 11 | "babel-eslint": "*", 12 | "babel-plugin-transform-flow-strip-types": "*", 13 | "babel-preset-es2015-node5": "*", 14 | "babel-preset-stage-3": "*", 15 | "babel-register": "*", 16 | "eslint": ">=2.0.0", 17 | "eslint-plugin-flowtype": "*", 18 | "flow-bin": "*" 19 | }, 20 | "main": "iex.js", 21 | "scripts": { 22 | "flow": "flow; test $? -eq 0 -o $? -eq 2", 23 | "run": "babel-node iex.js", 24 | "transpile": "rm ../../dist/iex_dd -rf && babel --out-dir ../../dist/iex_dd iex.js && babel --out-dir ../../dist/iex_dd/src ./src && cp ./node_modules ../../dist/iex_dd -r", 25 | "test": "echo \"Error: no test specified\" && exit 1", 26 | "start": "sh run.sh" 27 | }, 28 | "author": "Casey Primozic ", 29 | "license": "MIT" 30 | } 31 | -------------------------------------------------------------------------------- /data_downloaders/iex/run.sh: -------------------------------------------------------------------------------- 1 | RUST_BACKTRACE=1 LD_LIBRARY_PATH=../../dist/lib/ npm run-script run 2 | -------------------------------------------------------------------------------- /data_downloaders/iex/src/commands.js: -------------------------------------------------------------------------------- 1 | //! Functions useful for interfacing with the platform's command and response communication system 2 | /* global WebSocket */ 3 | // @flow 4 | 5 | const CONF = require('./conf.js'); 6 | 7 | // TODO: Swap to using the functions from the JS util library 8 | 9 | /** 10 | * Generates a new V4 UUID in hyphenated form 11 | */ 12 | export function v4(): string { 13 | function s4(): string { 14 | return Math.floor((1 + Math.random()) * 0x10000) 15 | .toString(16) 16 | .substring(1); 17 | } 18 | return s4() + s4() + '-' + s4() + '-' + s4() + '-' + 19 | s4() + '-' + s4() + s4() + s4(); 20 | } 21 | 22 | /** 23 | * Starts the WS listening for new messages sets up processing callback 24 | */ 25 | export function initWs(callback: (msg: {uuid: string, msg: any}) => void, ourUuid: string): WebSocket { 26 | let socketUrl = 'ws://localhost:7037'; 27 | let socket = new WebSocket(socketUrl); 28 | socket.onmessage = (message: any) => { 29 | let parsed = JSON.parse(message.data); 30 | // throw away messages we're transmitting to channels we don't care about 31 | if ([CONF.redis_control_channel, CONF.redis_responses_channel, CONF.redis_log_channel, ourUuid].indexOf(parsed.channel) !== -1) { 32 | callback(parsed); 33 | } 34 | }; 35 | 36 | socket.onerror = () => { 37 | // TODO 38 | }; 39 | 40 | return socket; 41 | } 42 | 43 | /** 44 | * Processes a command from the platform and returns a response to be sent back, also taking any effects that the command commands us to take. 45 | */ 46 | export function handleCommand(command: any, our_uuid: string): any { 47 | switch (command) { 48 | case 'Ping': 49 | var temp = [our_uuid]; 50 | return {Pong: {args: temp.splice(2)}}; 51 | case 'Kill': 52 | return {Error: {status: 'We\'re client side, we don\'t take orders from you.'}}; 53 | case 'Type': 54 | return {Info: {info: 'IEX Data Downloader'}}; 55 | default: 56 | return {Error: {status: 'Command not recognized.'}}; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /data_downloaders/poloniex/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["es2015-node5", "stage-3"], 3 | "plugins": [ 4 | "transform-flow-strip-types" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /data_downloaders/poloniex/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "rules": { 3 | "flowtype/boolean-style": [2, "boolean"], 4 | "flowtype/define-flow-type": 1, 5 | "flowtype/delimiter-dangle": [2, "never"], 6 | "flowtype/generic-spacing": [2, "never"], 7 | "flowtype/no-primitive-constructor-types": 2, 8 | "flowtype/object-type-delimiter": [2, "comma"], 9 | "flowtype/require-valid-file-annotation": 2, 10 | "flowtype/semi": [2, "always"], 11 | "flowtype/space-before-generic-bracket": [2, "never"], 12 | "flowtype/type-id-match": [2, "[A-Z][a-z0-9]+"], 13 | "flowtype/union-intersection-spacing": [2, "always"], 14 | "flowtype/use-flow-type": 1, 15 | "flowtype/valid-syntax": 1, 16 | "indent": [2, 2], 17 | "quotes": [2, "single"], 18 | "linebreak-style": [2, "unix"], 19 | "semi": [2, "always"], 20 | "comma-dangle": [2, "only-multiline"], 21 | "no-console": 0, 22 | "no-unused-vars": [2, {"vars": "all", "args": "none"}], 23 | "no-multiple-empty-lines": [2, {"max": 1}] 24 | }, 25 | "env": { 26 | "es6": true, 27 | "browser": false, 28 | "node": true 29 | }, 30 | "extends": ["eslint:recommended"], 31 | "parser": "babel-eslint", 32 | "parserOptions": { 33 | "sourceType": "module", 34 | "ecmaFeatures": { 35 | "experimentalObjectRestSpread": true, 36 | "jsx": false 37 | }, 38 | "ecmaVersion": 7 39 | }, 40 | "plugins": [ 41 | "flowtype" 42 | ], 43 | "globals": { 44 | "require": true, 45 | "module": true 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /data_downloaders/poloniex/.flowconfig: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ameobea/tickgrinder/f2ccd640b173cb17b0af4e04df9b203935eb2dc2/data_downloaders/poloniex/.flowconfig -------------------------------------------------------------------------------- /data_downloaders/poloniex/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | src/conf.js 3 | yarn.lock 4 | log.txt 5 | -------------------------------------------------------------------------------- /data_downloaders/poloniex/debug.sh: -------------------------------------------------------------------------------- 1 | RUST_BACKTRACE=1 LD_LIBRARY_PATH=../../dist/lib/ npm run-script debug 2 | -------------------------------------------------------------------------------- /data_downloaders/poloniex/inspect.sh: -------------------------------------------------------------------------------- 1 | RUST_BACKTRACE=1 LD_LIBRARY_PATH=../../dist/lib/ npm run-script inspect 2 | -------------------------------------------------------------------------------- /data_downloaders/poloniex/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "poloniex", 3 | "version": "1.0.0", 4 | "dependencies": { 5 | "autobahn": "*", 6 | "lodash": "4.17.4" 7 | }, 8 | "description": "A data downloader using the Poloniex API to download data", 9 | "devDependencies": { 10 | "babel-cli": "*", 11 | "babel-core": "*", 12 | "babel-eslint": "*", 13 | "babel-plugin-transform-flow-strip-types": "*", 14 | "babel-preset-es2015-node5": "*", 15 | "babel-preset-stage-3": "*", 16 | "babel-register": "*", 17 | "eslint": ">=2.0.0", 18 | "eslint-plugin-flowtype": "*", 19 | "flow-bin": "*" 20 | }, 21 | "main": "index.js", 22 | "scripts": { 23 | "flow": "flow; test $? -eq 0 -o $? -eq 2", 24 | "run": "babel-node index.js 13474c8b-5b0b-408e-b799-3a0de0254486", 25 | "transpile": "rm ../../dist/poloniex_dd -rf && babel --out-dir ../../dist/poloniex_dd index.js && babel --out-dir ../../dist/poloniex_dd/src ./src && cp ./node_modules ../../dist/poloniex_dd -r", 26 | "debug": "babel-node debug index.js 5d4fdb9d-1b49-458d-ad03-b5f1935f0000", 27 | "inspect": "babel-node --inspect index.js a2a8de10-8b93-e0ae-a675-6f720cd4d888", 28 | "test": "echo \"Error: no test specified\" && exit 1" 29 | }, 30 | "author": "Casey Primozic ", 31 | "license": "MIT" 32 | } 33 | -------------------------------------------------------------------------------- /data_downloaders/poloniex/run.sh: -------------------------------------------------------------------------------- 1 | RUST_BACKTRACE=1 LD_LIBRARY_PATH=../../dist/lib/ npm run-script run -- $1 2 | -------------------------------------------------------------------------------- /dev.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script symlinks the mm directory into the dist directory so that changes made to the 4 | # mm directory during development will be reflected when running the platform. 5 | # 6 | # You only need to run this script if you're working on the MM. 7 | 8 | make debug && make dev && ./run.sh 9 | -------------------------------------------------------------------------------- /gource-recent.sh: -------------------------------------------------------------------------------- 1 | gource --start-date '2017-01-01' -s .4 2 | -------------------------------------------------------------------------------- /gource.sh: -------------------------------------------------------------------------------- 1 | gource -s .4 2 | -------------------------------------------------------------------------------- /logger/.cargo/config: -------------------------------------------------------------------------------- 1 | [build] 2 | rustflags = [ 3 | "-L", "../util/target/release/deps", 4 | "-L", "../../util/target/release/deps", 5 | "-L", "../../../util/target/release/deps", 6 | "-C", "prefer-dynamic" 7 | ] 8 | -------------------------------------------------------------------------------- /logger/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /logger/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "logger" 3 | version = "0.1.0" 4 | authors = ["Casey Primozic "] 5 | 6 | [profile.release] 7 | opt-level = 3 8 | debug = true 9 | debug-assertions = false 10 | -------------------------------------------------------------------------------- /logger/README.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ameobea/tickgrinder/f2ccd640b173cb17b0af4e04df9b203935eb2dc2/logger/README.txt -------------------------------------------------------------------------------- /logger/install.sh: -------------------------------------------------------------------------------- 1 | CARGO_INCREMENTAL=1 LD_LIBRARY_PATH=../dist/lib:../util/target/release/deps cargo build 2 | cp target/debug/logger ../dist/ 3 | -------------------------------------------------------------------------------- /logger/install_release.sh: -------------------------------------------------------------------------------- 1 | CARGO_INCREMENTAL=1 LD_LIBRARY_PATH=../dist/lib:../util/target/release/deps cargo build --release 2 | cp target/debug/logger ../dist/ 3 | -------------------------------------------------------------------------------- /mm-react/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "rules": { 3 | "flowtype/boolean-style": [2, "boolean"], 4 | "flowtype/define-flow-type": 1, 5 | "flowtype/delimiter-dangle": [2, "never"], 6 | "flowtype/generic-spacing": [2, "never"], 7 | "flowtype/no-primitive-constructor-types": 2, 8 | "flowtype/object-type-delimiter": [2, "comma"], 9 | "flowtype/require-valid-file-annotation": 2, 10 | "flowtype/semi": [2, "always"], 11 | "flowtype/space-before-generic-bracket": [2, "never"], 12 | "flowtype/type-id-match": [2, "[A-Z][a-z0-9]+"], 13 | "flowtype/union-intersection-spacing": [2, "always"], 14 | "flowtype/use-flow-type": 1, 15 | "flowtype/valid-syntax": 1, 16 | "indent": [2, 2], 17 | "quotes": [2, "single"], 18 | "react/forbid-component-props": 0, 19 | "react/jsx-filename-extension": [1, { "extensions": [".js", ".jsx"] }], 20 | "react/jsx-indent": [1, 2], 21 | "react/jsx-indent-props": [1, 2], 22 | "react/no-multi-comp": 0, 23 | "react/display-name": 0, 24 | "react/jsx-max-props-per-line": [1, {"maximum": 3, "when": "multiline"}], 25 | "linebreak-style": [2, "unix"], 26 | "semi": [2, "always"], 27 | "comma-dangle": [2, "only-multiline"], 28 | "no-console":0, 29 | "no-unused-vars": [2, {"vars": "all", "args": "none"}], 30 | "no-multiple-empty-lines": [2, {"max": 1}] 31 | }, 32 | "env": { 33 | "es6": true, 34 | "browser": true 35 | }, 36 | "extends": ["eslint:recommended", "plugin:react/all"], 37 | "parser": "babel-eslint", 38 | "parserOptions": { 39 | "sourceType": "module", 40 | "ecmaFeatures": { 41 | "experimentalObjectRestSpread": true, 42 | "jsx": true 43 | }, 44 | "ecmaVersion": 7 45 | }, 46 | "plugins": [ 47 | "flowtype", 48 | "react" 49 | ], 50 | "globals": { 51 | "require": true, 52 | "module": true 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /mm-react/.flowconfig: -------------------------------------------------------------------------------- 1 | [ignore] 2 | .*/node_modules/* 3 | 4 | [include] 5 | 6 | [libs] 7 | 8 | [options] 9 | -------------------------------------------------------------------------------- /mm-react/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | 6 | # production 7 | /dist 8 | 9 | # misc 10 | .DS_Store 11 | npm-debug.log* 12 | 13 | conf.js 14 | -------------------------------------------------------------------------------- /mm-react/.roadhogrc: -------------------------------------------------------------------------------- 1 | { 2 | "entry": "src/index.js", 3 | "env": { 4 | "development": { 5 | "extraBabelPlugins": [ 6 | "dva-hmr", 7 | "transform-runtime" 8 | ] 9 | }, 10 | "production": { 11 | "extraBabelPlugins": [ 12 | "transform-runtime" 13 | ] 14 | } 15 | }, 16 | "extraBabelPlugins": [ 17 | "transform-runtime", 18 | ["import", { "libraryName": "antd", "style": true }] 19 | ], 20 | "theme": { 21 | "@primary-color": "#AB074B", 22 | "@component-background": "#313130", 23 | "@text-color": "fade(#fff, 65%)", 24 | "@text-color-secondary": "fade(#fff, 43%)", 25 | "@disabled-color": "fade(#fff, 25%)", 26 | "@heading-color-dark": "fade(#000, 97%)", 27 | "@text-color-dark": "fade(#000, 91%)", 28 | "@text-color-secondary-dark": "fade(#000, 67%)", 29 | "@disabled-color-dark": "fade(#000, 35%)", 30 | "@input-bg": "#313130", 31 | "@menu-dark-bg": "#404040", 32 | "@layout-header-background": "#050505", 33 | "@popover-bg": "#21191b", 34 | "@background-color-base": "#303030", 35 | "@layout-body-background": "#383636", 36 | "@normal-color": "#1c1c1c", 37 | "@primary-1": "#1c1c1c" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /mm-react/.roadhogrc.mock.js: -------------------------------------------------------------------------------- 1 | 2 | export default { 3 | }; 4 | -------------------------------------------------------------------------------- /mm-react/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "scripts": { 4 | "start": "roadhog server", 5 | "build": "roadhog build", 6 | "flow": "flow; test $? -eq 0 -o $? -eq 2", 7 | "test": "roadhog test", 8 | "build-debug": "roadhog build --debug" 9 | }, 10 | "engines": { 11 | "install-node": "6.9.2" 12 | }, 13 | "dependencies": { 14 | "antd": "^2.7.0", 15 | "babel-plugin-import": "^1.4.0", 16 | "babel-runtime": "^6.9.2", 17 | "dva": "^1.2.1", 18 | "dva-loading": "^0.2.1", 19 | "eslint-plugin-promise": "^3.4.1", 20 | "flow-bin": "*", 21 | "html-to-react": "^1.2.4", 22 | "react": "^15.4.0", 23 | "react-dom": "^15.4.0" 24 | }, 25 | "devDependencies": { 26 | "babel-eslint": "^7.1.1", 27 | "babel-plugin-transform-flow-strip-types": "*", 28 | "babel-plugin-dva-hmr": "^0.3.2", 29 | "babel-plugin-transform-runtime": "^6.9.0", 30 | "babel-plugin-flow-react-proptypes": "0.21.0", 31 | "eslint": "^3.12.2", 32 | "eslint-config-airbnb": "^13.0.0", 33 | "eslint-plugin-flowtype": "*", 34 | "eslint-config-standard": "^6.2.1", 35 | "eslint-plugin-import": "^2.2.0", 36 | "eslint-plugin-jsx-a11y": "^2.2.3", 37 | "eslint-plugin-promise": "^3.4.1", 38 | "eslint-plugin-react": "^6.8.0", 39 | "eslint-plugin-standard": "^2.0.1", 40 | "expect": "^1.20.2", 41 | "husky": "^0.12.0", 42 | "lodash": "4.17.4", 43 | "redbox-react": "^1.3.2", 44 | "roadhog": "^0.5.2" 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /mm-react/public/.gitignore: -------------------------------------------------------------------------------- 1 | ckeditor/ 2 | -------------------------------------------------------------------------------- /mm-react/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | TickGrinder Dashboard 7 | 8 | 9 | 10 | 11 |
12 | 13 | 14 |
15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /mm-react/public/staticStyle.css: -------------------------------------------------------------------------------- 1 | html, body, #root { 2 | height:100%; 3 | } 4 | 5 | input { 6 | background-color: #313130 !important; 7 | } 8 | -------------------------------------------------------------------------------- /mm-react/src/components/AppPage.js: -------------------------------------------------------------------------------- 1 | //! A page of the application. Contains the basic structure of the GUI including title, header, navigation, and footer. 2 | 3 | import React from 'react'; 4 | import { connect } from 'dva'; 5 | import { Layout } from 'antd'; 6 | const { Header, Content, Footer, Sider } = Layout; 7 | 8 | import DashHeader from './DashHeader'; 9 | import gstyles from '../static/css/globalStyle.css'; 10 | 11 | class AppPage extends React.Component { 12 | render() { 13 | return ( 14 | 15 | 16 | 17 | {this.props.children} 18 | 19 |
20 | TickGrinder Algorithmic Trading Platform; Created by Casey Primozic ©2017 21 |
22 | 23 | ); 24 | } 25 | } 26 | 27 | function mapProps(state) { 28 | return { title: state.global.title }; 29 | } 30 | 31 | export default connect(mapProps)(AppPage); 32 | -------------------------------------------------------------------------------- /mm-react/src/components/ContentContainer.js: -------------------------------------------------------------------------------- 1 | //! A wrapper for the main content that is unique to the currently selected page. 2 | 3 | import React, { Component, PropTypes } from 'react' 4 | import { connect } from 'dva'; 5 | 6 | import gstyles from '../static/css/globalStyle.css'; 7 | 8 | class ContentContainer extends Component { 9 | static propTypes = { 10 | title: PropTypes.string.isRequired, 11 | } 12 | 13 | render() { 14 | return ( 15 |
16 | { this.props.children } 17 |
18 | ); 19 | } 20 | 21 | componentWillMount() { 22 | this.props.dispatch({type: 'global/pageChange', title: this.props.title}); 23 | } 24 | } 25 | 26 | export default connect()(ContentContainer); 27 | -------------------------------------------------------------------------------- /mm-react/src/components/DashHeader.js: -------------------------------------------------------------------------------- 1 | //! Contains nav information as well as the title 2 | 3 | import React from 'react'; 4 | import { Link } from 'react-router' 5 | import { connect } from 'dva'; 6 | import { Layout, Menu } from 'antd'; 7 | const { Submenu } = Menu; 8 | const { Header } = Layout; 9 | 10 | import gstyles from '../static/css/globalStyle.css'; 11 | 12 | class DashHeader extends React.Component { 13 | propTypes: { 14 | title: string 15 | } 16 | 17 | render() { 18 | return ( 19 |
20 | 21 | Home 22 | Backtesting 23 | Data 24 | Instance Management 25 | Logging + Monitoring 26 | Docs 27 | 28 |
29 | ); 30 | } 31 | } 32 | 33 | function mapState(state) { 34 | return { title: state.global.title } 35 | } 36 | 37 | export default connect(mapState)(DashHeader); 38 | -------------------------------------------------------------------------------- /mm-react/src/components/MacroBuilder.js: -------------------------------------------------------------------------------- 1 | //! Macro builder component. Allows users to visually create spawner macros and edit spawner macros that they 2 | //! have already created. 3 | 4 | import React from 'react'; 5 | import { connect } from 'dva'; 6 | import { Select } from 'antd'; 7 | 8 | const MacroBuilder = ({dispatch}) => { 9 | return ( 10 | 13 | ); 14 | }; 15 | 16 | MacroBuilder.propTypes = { 17 | dispatch: React.PropTypes.func.isRequired, 18 | }; 19 | 20 | export default connect()(MacroBuilder); 21 | -------------------------------------------------------------------------------- /mm-react/src/components/MacroManager.js: -------------------------------------------------------------------------------- 1 | //! The macro manager supplies an interface for listing, creating, and executing spawner macros. 2 | //! Spawner macros are the highest level of user control over the platform itself, allowing the automation 3 | //! of tasks such as instance spawning, strategy deployment, backtesting, and other high-level control over 4 | //! the platform and its modules. 5 | 6 | import React from 'react'; 7 | import { connect } from 'dva'; 8 | import { Select } from 'antd'; 9 | 10 | import { MacroShape } from '../utils/commands'; 11 | 12 | const MacroManager = ({dispatch, definedMacros}) => { 13 | return ( 14 | 17 | ); 18 | }; 19 | 20 | MacroManager.propTypes = { 21 | definedMacros: React.PropTypes.arrayOf(React.PropTypes.shape(MacroShape)).isRequired, 22 | dispatch: React.PropTypes.func.isRequired, 23 | }; 24 | 25 | function mapProps(state) { 26 | return { 27 | definedMacros: state.macros.definedMacros, 28 | }; 29 | } 30 | 31 | export default connect(mapProps)(MacroManager); 32 | -------------------------------------------------------------------------------- /mm-react/src/components/backtest/BacktestMonitor.js: -------------------------------------------------------------------------------- 1 | //! Lists running backtests and has options for stopping them, monitoring their progress, etc. 2 | 3 | import React from 'react'; 4 | import { connect } from 'dva'; 5 | import { Table } from 'antd'; 6 | 7 | const BacktestMonitor = ({dispatch}) => { 8 | return ( 9 |
10 | ); 11 | }; 12 | 13 | export default connect()(BacktestMonitor); 14 | -------------------------------------------------------------------------------- /mm-react/src/components/backtest/BacktestStarter.js: -------------------------------------------------------------------------------- 1 | //! Component to initialize backtests 2 | 3 | import React from 'react'; 4 | import { connect } from 'dva'; 5 | 6 | const BacktestStarter = ({dispatch}) => { 7 | return ( 8 |
9 | ); 10 | }; 11 | 12 | export default connect()(BacktestStarter); 13 | -------------------------------------------------------------------------------- /mm-react/src/components/backtest/BacktesterSpawner.js: -------------------------------------------------------------------------------- 1 | //! Component for spawning in new backtester instances 2 | 3 | import React from 'react'; 4 | import { connect } from 'dva'; 5 | 6 | const BacktesterSpawner = ({dispatch}) => { 7 | return ( 8 |
9 | ); 10 | }; 11 | 12 | export default connect()(BacktesterSpawner); 13 | -------------------------------------------------------------------------------- /mm-react/src/components/data/DataDownloaderSpawner.js: -------------------------------------------------------------------------------- 1 | //! A component to spawn a data downloader. 2 | 3 | import React from 'react'; 4 | import { connect } from 'dva'; 5 | import { Select, Button, Form } from 'antd'; 6 | const FormItem = Form.Item; 7 | const Option = Select.Option; 8 | 9 | import { dataDownloaders } from '../../utils/data_util'; 10 | 11 | class DataDownloaderSpawner extends React.Component { 12 | handleSubmit = e => { 13 | e.preventDefault(); 14 | this.props.form.validateFields((err, values) => { 15 | if (!err) { 16 | // get the value of the selected downloader's command from the form 17 | let cmd = this.props.form.getFieldValue('downloaderName'); 18 | // send the command to the spawner to be executed 19 | this.props.dispatch({ 20 | cmd: cmd, 21 | cb_action: 'instances/instanceSpawnCallback', 22 | instance_name: 'Spawner', 23 | type: 'platform_communication/sendCommandToInstance', 24 | }); 25 | } 26 | }); 27 | } 28 | 29 | render() { 30 | // creation `Option`s for each of the available data downloaders 31 | let availableDownloaders = []; 32 | for(var i=0; i 38 | {this.props.dataDownloaders[i].name} 39 | 40 | ); 41 | } 42 | 43 | const { getFieldDecorator } = this.props.form; 44 | return ( 45 |
46 | 47 | {getFieldDecorator('downloaderName', { 48 | rules: [ 49 | { required: true, message: 'Please select a downloader to spawn!' }, 50 | ], 51 | })( 52 | 55 | )} 56 | 57 | 58 | 59 | {getFieldDecorator('button', {})( 60 | 63 | )} 64 | 65 |
66 | ); 67 | } 68 | } 69 | 70 | DataDownloaderSpawner.propTypes = { 71 | dataDownloaders: React.PropTypes.arrayOf(React.PropTypes.shape({ 72 | name: React.PropTypes.string.isRequired, 73 | command: React.PropTypes.string.isRequired, 74 | description: React.PropTypes.string.isRequired, 75 | })).isRequired, 76 | dispatch: React.PropTypes.func.isRequired, 77 | form: React.PropTypes.any.isRequired, 78 | }; 79 | 80 | function mapProps(state) { 81 | return { 82 | dataDownloaders: dataDownloaders, 83 | }; 84 | } 85 | 86 | export default connect(mapProps)(Form.create()(DataDownloaderSpawner)); 87 | -------------------------------------------------------------------------------- /mm-react/src/components/data/DataManager.js: -------------------------------------------------------------------------------- 1 | //! Tool used to facilitate the management of stored data. Includes functionality to list what data is currently 2 | //! stored and apply transformations and analysis to it. 3 | 4 | import React from 'react'; 5 | import { connect } from 'dva'; 6 | import { Table } from 'antd'; 7 | 8 | const DataManager = ({dispatch, downloadedData}) => { 9 | const columns = [{ 10 | // TODO 11 | }]; 12 | 13 | return ( 14 |
15 |

Downloaded Data

16 | 17 | 18 | ); 19 | }; 20 | 21 | DataManager.propTypes = { 22 | dispatch: React.PropTypes.func.isRequired, 23 | downloadedData: React.PropTypes.any.isRequired, // TODO: Update to official schema once it's established 24 | }; 25 | 26 | function mapProps(state) { 27 | return { 28 | downloadedData: state.data.downloadedData, 29 | }; 30 | } 31 | 32 | export default connect(mapProps)(DataManager); 33 | -------------------------------------------------------------------------------- /mm-react/src/components/data/DownloadControl.js: -------------------------------------------------------------------------------- 1 | //! Controls for managing a single data download. 2 | 3 | import React from 'react'; 4 | import { connect } from 'dva'; 5 | import { Button, Popconfirm } from 'antd'; 6 | 7 | /** 8 | * Returns a function that sends a command to a data downloader instance to cancel a running data download 9 | */ 10 | const handleClick = (dispatch, downloaderId, downloadId) => { 11 | return () => { 12 | let cmd = {CancelDataDownload: {download_id: downloadId}}; 13 | dispatch({type: 'platform_communication/sendCommand', channel: downloaderId, cmd: cmd}); 14 | }; 15 | }; 16 | 17 | /** 18 | * Creates a set of buttons for controlling a running data download 19 | */ 20 | const DownloadControl = ({dispatch, downloaderId, downloadId}) => { 21 | return ( 22 |
23 | 27 | 28 | 29 |
30 | ); 31 | }; 32 | 33 | DownloadControl.propTypes = { 34 | dispatch: React.PropTypes.func.isRequired, 35 | downloadId: React.PropTypes.string.isRequired, 36 | downloaderId: React.PropTypes.string.isRequired, 37 | }; 38 | 39 | export default connect()(DownloadControl); 40 | -------------------------------------------------------------------------------- /mm-react/src/components/data/DstSelector.js: -------------------------------------------------------------------------------- 1 | //! A component that's meant to be included in a form. It contains a `Select` dropdown box that can be used to pick a data 2 | //! downloader. Depending on the current selection, it also renders a `TickSink` component which contains input fields 3 | //! for the user to fill out and create a `HistTickDst`. 4 | 5 | import React from 'react'; 6 | import { connect } from 'dva'; 7 | import { Select } from 'antd'; 8 | const Option = Select.Option; 9 | 10 | import { tickSinkDefs, TickSink } from '../../utils/data_util'; 11 | 12 | const dstOpts = []; 13 | for(let dstName in tickSinkDefs) { 14 | dstOpts.push(); 15 | } 16 | 17 | class DstSelector extends React.Component { 18 | constructor(props) { 19 | super(props); 20 | let firstSinkId = null; 21 | for(let dstName in tickSinkDefs) { 22 | firstSinkId = dstName; 23 | break; 24 | } 25 | 26 | this.handleSinkChange = this.handleSinkChange.bind(this); 27 | this.handleUpdate = this.handleUpdate.bind(this); 28 | this.bindRef = this.bindRef.bind(this) 29 | this.state = { 30 | sinkJsonName: firstSinkId, 31 | }; 32 | } 33 | 34 | /** 35 | * Called every time the user selects a different sink from the `Select`. 36 | */ 37 | handleSinkChange(sinkJsonName: string) { 38 | this.setState({sinkJsonName: sinkJsonName, value: this.input.getHistTickDst()}); 39 | } 40 | 41 | /** 42 | * Called every time the sink parameters are changed by the user. Stores the dst in the store so that it can be retrieved 43 | * from higher up in the scope in order to get the value in the form. 44 | */ 45 | handleUpdate(dst) { 46 | this.props.dispatch({type: 'data/newDst', dst: dst}); 47 | } 48 | 49 | bindRef(child) { 50 | this.input = child; 51 | } 52 | 53 | render() { 54 | return ( 55 |
56 | 59 | 60 | 61 |
62 | ); 63 | } 64 | } 65 | 66 | DstSelector.PropTypes = { 67 | value: React.PropTypes.string, 68 | }; 69 | 70 | export default connect()(DstSelector); 71 | -------------------------------------------------------------------------------- /mm-react/src/components/docs/Ckeditor.js: -------------------------------------------------------------------------------- 1 | //! Creates a ckeditor instance. Contains options for taking callbacks involved with saving changes. 2 | /* global CKEDITOR */ 3 | 4 | import React from 'react'; 5 | import { connect } from 'dva'; 6 | 7 | /** 8 | * After the CKEditor plugin has loaded, initialize the editor 9 | */ 10 | function awaitCk(rand) { 11 | setTimeout(() => { 12 | let ckeditorLoaded = true; 13 | try{ CKEDITOR; } 14 | catch(e) { 15 | if(e.name == 'ReferenceError') { 16 | ckeditorLoaded = false; 17 | } 18 | } 19 | 20 | if(ckeditorLoaded) { 21 | CKEDITOR.replace( `ckeditor-${rand}` ); 22 | } else { 23 | awaitCk(rand); 24 | } 25 | }, 50); 26 | } 27 | 28 | class CKEditor extends React.Component { 29 | componentDidMount() { 30 | // add a script tag onto the document that loads the CKEditor script 31 | let ckeditor_src = document.createElement('script'); 32 | ckeditor_src.type = 'text/javascript'; 33 | ckeditor_src.async = true; 34 | ckeditor_src.src='/ckeditor/ckeditor.js'; 35 | document.getElementById('ckeditor-' + this.props.rand).appendChild(ckeditor_src); 36 | 37 | // wait for the CKEditor script to load and then initialize the editor 38 | awaitCk(this.props.rand); 39 | 40 | // register our id as the active editor instance 41 | this.props.dispatch({type: 'documents/setEditorId', id: this.props.rand}); 42 | } 43 | 44 | shouldComponentUpdate(...args) { 45 | return false; 46 | } 47 | 48 | render() { 49 | return ( 50 | 48 | 49 |
50 | 51 |

Load a Chart

52 | 53 | 54 | 55 | 56 |

Name of Chart

57 | 58 |
59 | 60 | 61 | -------------------------------------------------------------------------------- /mm/views/partials/header.ejs: -------------------------------------------------------------------------------- 1 | 11 | -------------------------------------------------------------------------------- /optimizer/.cargo/config: -------------------------------------------------------------------------------- 1 | [build] 2 | rustflags = [ 3 | "-L", "../util/target/release/deps", 4 | "-L", "../../util/target/release/deps", 5 | "-L", "../dist/lib", 6 | "-L", "../../dist/lib" 7 | ] 8 | -------------------------------------------------------------------------------- /optimizer/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /optimizer/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "optimizer" 3 | version = "0.1.0" 4 | authors = ["Casey Primozic "] 5 | 6 | [profile.release] 7 | opt-level = 3 8 | debug = true 9 | debug-assertions = false 10 | -------------------------------------------------------------------------------- /optimizer/src/tests/benchmarks/misc.rs: -------------------------------------------------------------------------------- 1 | use test; 2 | use uuid::Uuid; 3 | 4 | #[bench] 5 | fn uuid_generation(b: &mut test::Bencher) { 6 | b.iter(|| { 7 | let x = Uuid::new_v4(); 8 | }); 9 | } 10 | -------------------------------------------------------------------------------- /optimizer/src/tests/benchmarks/mod.rs: -------------------------------------------------------------------------------- 1 | #[allow(unused_variables)] 2 | pub mod transport; 3 | #[allow(unused_variables)] 4 | pub mod misc; 5 | -------------------------------------------------------------------------------- /optimizer/src/tests/benchmarks/transport.rs: -------------------------------------------------------------------------------- 1 | use test; 2 | 3 | use serde_json; 4 | use uuid::Uuid; 5 | use tickgrinder_util::transport::commands::*; 6 | 7 | #[bench] 8 | fn wrappedcmd_to_string(b: &mut test::Bencher) { 9 | let cmd = Command::AddSMA{period: 42.23423f64}; 10 | let wr_cmd = WrappedCommand{uuid: Uuid::new_v4(), cmd: cmd}; 11 | b.iter(|| { 12 | let wr_cmd = &wr_cmd; 13 | let _ = serde_json::to_string(wr_cmd); 14 | }); 15 | } 16 | 17 | #[bench] 18 | fn string_to_wrappedcmd(b: &mut test::Bencher) { 19 | let raw = "{\"uuid\":\"2f663301-5b73-4fa0-b201-09ab196ec5fd\",\"cmd\":{\"RemoveSMA\":{\"period\":5.2342}}}"; 20 | b.iter(|| { 21 | let raw = &raw; 22 | let _: WrappedCommand = serde_json::from_str(raw).unwrap(); 23 | }); 24 | } 25 | -------------------------------------------------------------------------------- /optimizer/src/tests/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod benchmarks; 2 | -------------------------------------------------------------------------------- /optimizer/src/transport/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod postgres; 2 | pub mod tick_processor; 3 | #[allow(unused_imports)] 4 | mod test; 5 | -------------------------------------------------------------------------------- /optimizer/src/transport/test.rs: -------------------------------------------------------------------------------- 1 | use serde_json; 2 | 3 | use tickgrinder_util::transport::commands::*; 4 | 5 | #[test] 6 | fn command_serialization() { 7 | let cmd_str = "{\"AddSMA\": {\"period\": 6.64} }"; 8 | let cmd: Command = serde_json::from_str(cmd_str).unwrap(); 9 | assert_eq!(cmd, Command::AddSMA{period: 6.64f64}); 10 | } 11 | 12 | #[test] 13 | fn command_deserialization() { 14 | let cmd = Command::RemoveSMA{period: 6.64f64}; 15 | let cmd_string = serde_json::to_string(&cmd).unwrap(); 16 | assert_eq!("{\"RemoveSMA\":{\"period\":6.64}}", cmd_string.as_str()); 17 | } 18 | 19 | #[test] 20 | fn response_serialization() { 21 | let res_str = "\"Ok\""; 22 | let res: Response = serde_json::from_str(res_str).unwrap(); 23 | assert_eq!(res, Response::Ok); 24 | } 25 | 26 | #[test] 27 | fn response_deserialization() { 28 | let res = Response::Ok; 29 | let res_string = serde_json::to_string(&res).unwrap(); 30 | assert_eq!("\"Ok\"", res_string.as_str()); 31 | } 32 | -------------------------------------------------------------------------------- /private/.cargo/config: -------------------------------------------------------------------------------- 1 | [build] 2 | rustflags = [ 3 | "-L", "../util/target/release/deps", 4 | "-L", "../../util/target/release/deps", 5 | "-L", "../../../util/target/release/deps", 6 | "-L", "../../../../util/target/release/deps", 7 | "-L", "../dist/lib", 8 | "-L", "../../dist/lib", 9 | "-L", "../../../dist/lib", 10 | "-L", "../../../../dist/lib", 11 | "-C", "prefer-dynamic" 12 | ] 13 | -------------------------------------------------------------------------------- /private/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | *.so 3 | -------------------------------------------------------------------------------- /private/Cargo.lock: -------------------------------------------------------------------------------- 1 | [root] 2 | name = "tickgrinder-private" 3 | version = "0.1.0" 4 | 5 | -------------------------------------------------------------------------------- /private/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tickgrinder-private" 3 | version = "0.1.0" 4 | authors = ["Casey Primozic "] 5 | description = "Contains the user-defined strategies, indicators, and other custom proprietary parts of the platform." 6 | 7 | [lib] 8 | name = "private" 9 | crate-type = ["dylib"] 10 | 11 | [profile.release] 12 | opt-level = 3 13 | debug = true 14 | debug-assertions = false 15 | -------------------------------------------------------------------------------- /private/README.md: -------------------------------------------------------------------------------- 1 | # Private Directory 2 | 3 | This directory contains all of the custom, proprietary code for the platform that's meant to be supplied by the user. It should be maintained as a separate git repository and, in general, kept private. It compiles down to a library that can be plugged into the main platform in order to implement custom functionality. 4 | 5 | ## Overview 6 | 7 | The platform is just that - a platform on top of which you can build a functioning trading system. Due to the fact that successful trading strategies are by their very nature secret, this functionality has been built in so that you can use and contribute to the public platform without having to sacrifice the security and privacy of your own strategies, indicators, and data analysis techniques. 8 | 9 | ## Usage 10 | 11 | There are several subdirectories inside this directory that each contain files pertaining to a different proprietary part of the algorithmic trading process. By default, they are symlinked into the platform's code directories so that their contents are automatically compiled into the platform during build. 12 | 13 | ### Strategies 14 | Strategies are the main logic for the trading system. It is their job to ingest live data from the platform and use it to create trade conditions which are in turn fed to the Tick Processors. 15 | 16 | ### Indicators 17 | Indicators are rules for transforming data. Some examples of classic trading indicators include the SMA, RSI, and Stochastic. You can write your own indicators (which can also be used for plotting stored data). 18 | -------------------------------------------------------------------------------- /private/build.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=../dist/lib:../util/target/release/deps cargo build 2 | -------------------------------------------------------------------------------- /private/install.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=../dist/lib:../util/target/release/deps cargo build 2 | cp target/debug/libprivate.so ../dist/lib 3 | -------------------------------------------------------------------------------- /private/install_release.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=../dist/lib:../util/target/release/deps cargo build --release 2 | cp target/release/libprivate.so ../dist/lib 3 | -------------------------------------------------------------------------------- /private/src/indicators/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ameobea/tickgrinder/f2ccd640b173cb17b0af4e04df9b203935eb2dc2/private/src/indicators/.gitignore -------------------------------------------------------------------------------- /private/src/indicators/mod.rs: -------------------------------------------------------------------------------- 1 | //! Contains all private indicators you may devise for your system. 2 | 3 | // use tickgrinder_util::transport::postgres::*; 4 | // use tickgrinder_util::trading::trading_condition::*; 5 | // use tickgrinder_util::conf::CONF; 6 | 7 | mod sma; 8 | 9 | pub use self::sma::Sma; 10 | -------------------------------------------------------------------------------- /private/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Private data directory for the platform. See README.md for more information. 2 | 3 | #![feature(libc, test, plugin, custom_derive, conservative_impl_trait)] 4 | 5 | extern crate test; 6 | extern crate postgres; 7 | extern crate serde; 8 | extern crate serde_json; 9 | #[macro_use] 10 | extern crate serde_derive; 11 | extern crate tickgrinder_util; 12 | extern crate futures; 13 | extern crate fxcm; 14 | extern crate libc; 15 | extern crate uuid; 16 | extern crate rand; 17 | extern crate time; 18 | 19 | pub mod indicators; 20 | pub mod trading_conditions; 21 | pub mod strategies; 22 | pub mod sinks; 23 | 24 | // Sets up the defaults for your application 25 | pub use fxcm::FXCMNative as ActiveBroker; 26 | -------------------------------------------------------------------------------- /private/src/sinks/mod.rs: -------------------------------------------------------------------------------- 1 | //! Sinks are destinations for ticks. To create a custom destination for ticks of some kind, you can implement `GenTickSink` 2 | //! for it and export it here. 3 | 4 | use tickgrinder_util::transport::tickstream::generics::GenTickSink; 5 | 6 | /// Contains all created tick sinks 7 | pub enum Sinks { 8 | 9 | } 10 | -------------------------------------------------------------------------------- /private/src/strategies/fuzzer/README.md: -------------------------------------------------------------------------------- 1 | # Broker Fuzzer 2 | The broker fuzzer is not so such a strategy as a tool used for testing the functionality of the platform's broker shims. It was originally designed to verify the functionality of the Backtester module's SimBroker 3 | 4 | ## Functionality 5 | Traditional fuzzing works by trying to find issues, bugs, or vulnerabilities in programs by subjecting them to a large amount of semi-random inputs with the goal of making it break or misbehave in some way. 6 | 7 | The broker fuzzer works in a similar way. It genereates a large number of messages that it then sends to the specified broker. These messages consist of all possible trading actions, status queries, and other broker commands that are available through the Broker API. In addition, it will log in precise detail the order of events that take place from its view. The fuzzer can be configured to be deterministic in its fuzzing activities so that particular tests can be repeated exactly. When the log output from the fuzzer is compared to the advanced logs produced by the SimBroker itself, it's possible to verify that events happen in precisely the right order and that no race conditions or unordered events take place. 8 | 9 | ## Usage 10 | The Fuzzer is still in active development and is currently not ready for real use. This section will be updated once development has progressed to the point that the tool is ready for use. // TODO 11 | -------------------------------------------------------------------------------- /private/src/strategies/fuzzer/extern/bindings.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | /// Creates a new deterministic psuedorandom number generator given the provided seed and returns a reference to it. 5 | extern "C" void* init_rng(unsigned int seed) { 6 | boost::random::mt19937* gen = new boost::random::mt19937{seed}; 7 | return (void*)gen; 8 | } 9 | 10 | /// Given a reference to a random number generator, returns a random integer from within the range [min, max]. 11 | extern "C" int rand_int_range(void* void_rng, int min, int max) { 12 | boost::random::mt19937* gen = (boost::random::mt19937*)void_rng; 13 | boost::random::uniform_int_distribution<> dist{min, max}; 14 | return dist(*gen); 15 | } 16 | -------------------------------------------------------------------------------- /private/src/strategies/fuzzer/extern/build.sh: -------------------------------------------------------------------------------- 1 | g++ -g -rdynamic -shared -fPIC -std=c++11 -O3 bindings.cpp -lboost_random -o librand_bindings.so -Wl,--no-undefined 2 | -------------------------------------------------------------------------------- /private/src/strategies/mod.rs: -------------------------------------------------------------------------------- 1 | //! Strategies are the core pieces of log that tell the platform how to process data. 2 | 3 | use std::collections::HashMap; 4 | 5 | use tickgrinder_util::strategies::Strategy; 6 | 7 | pub mod sma_cross; 8 | pub mod fuzzer; 9 | 10 | // Set this to whichever strategy you want to use. 11 | pub use self::sma_cross::SmaCross as ActiveStrategy; 12 | 13 | // Returns K:V settings to be sent to the broker during initialization 14 | pub fn get_broker_settings() -> HashMap { 15 | HashMap::new() // TODO 16 | } 17 | -------------------------------------------------------------------------------- /private/src/strategies/sma_cross/mod.rs: -------------------------------------------------------------------------------- 1 | //! A basic "hello world" strategy to show the platform's functionality. The strategy places buy orders when 2 | //! the price crosses over the SMA and places sell orders when it crosses back. 3 | //! 4 | //! See /util/src/strategies.rs for more detailed documentation on how to implement the Strategy trait. 5 | 6 | use std::thread; 7 | use std::time::Duration; 8 | use std::collections::HashMap; 9 | use std::fmt::Debug; 10 | 11 | use futures::{Future, Complete}; 12 | 13 | use tickgrinder_util::strategies::{StrategyManager, ManagedStrategy, Helper, StrategyAction, Tickstream, Merged}; 14 | use tickgrinder_util::transport::command_server::CommandServer; 15 | use tickgrinder_util::transport::query_server::QueryServer; 16 | use tickgrinder_util::trading::broker::Broker; 17 | use tickgrinder_util::trading::objects::BrokerAction; 18 | use tickgrinder_util::trading::tick::{Tick, GenTick}; 19 | 20 | use super::get_broker_settings; 21 | 22 | #[derive(Clone)] 23 | pub struct SmaCross {} 24 | 25 | impl ManagedStrategy for SmaCross { 26 | /// Called when we are to start actively trading this strategy and initialize trading activity. 27 | fn init(&mut self, helper: &mut Helper, subscriptions: &[Tickstream]) { 28 | helper.cs.notice(Some("Startup"), "SMA Cross strategy is being initialized..."); 29 | let accounts = unwrap_log_panic(helper.broker.execute(BrokerAction::ListAccounts).wait().unwrap(), &mut helper.cs); 30 | let accounts_dbg = format!("Accounts on the broker: {:?}", accounts); 31 | println!("{}", accounts_dbg); 32 | } 33 | 34 | fn tick(&mut self, helper: &mut Helper, t: &GenTick>) -> Option { 35 | unimplemented!(); 36 | } 37 | 38 | /// Indicates that the platform is shutting down and that we need to do anything necessary (closing positions) 39 | /// before that happens. Includes a future to complete once we're ready. 40 | fn abort(&mut self) {} 41 | } 42 | 43 | /// Unwraps a Result and returns the inner value if it is Ok; `panic!()`s after logging a critical error otherwise. 44 | fn unwrap_log_panic(res: Result, cs: &mut CommandServer) -> T where E:Debug { 45 | match res { 46 | Ok(val) => val, 47 | Err(err) => { 48 | let err_string = format!("{:?}", err); 49 | cs.critical(None, &format!("There was an error when unwrapping Result: {}", err_string)); 50 | // make sure that the message actually gets sent before dying 51 | thread::sleep(Duration::from_secs(1)); 52 | panic!(err_string); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /private/src/trading_conditions/mod.rs: -------------------------------------------------------------------------------- 1 | //! Conditions that are evaluated for every new Tick received by a Tick Processor. They return 2 | //! `TradingAction`s that are used to actually execute trades, modify positions, etc. 3 | 4 | pub mod sma_cross; 5 | 6 | use tickgrinder_util::trading::trading_condition::*; 7 | 8 | // use indicators::*; 9 | use self::sma_cross::*; 10 | 11 | /// Contains every indicator that you may want to use in your platform. 12 | pub enum TradingConditions { 13 | SmaCross{period: usize}, 14 | } 15 | 16 | impl TradingConditions { 17 | fn get(&self) -> impl TradingCondition { 18 | match *self { 19 | TradingConditions::SmaCross{period} => SmaCross::new(period) 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /private/src/trading_conditions/sma_cross.rs: -------------------------------------------------------------------------------- 1 | //! Basic trading condition that checks whether the price crosses a SMA upwards or downwards 2 | //! and creates orders when it does. 3 | 4 | use tickgrinder_util::trading::tick::*; 5 | use tickgrinder_util::trading::trading_condition::*; 6 | 7 | pub struct SmaCross { 8 | period: usize, 9 | } 10 | 11 | impl TradingCondition for SmaCross { 12 | fn eval(&mut self, t: &Tick) -> Option { 13 | unimplemented!(); 14 | } 15 | } 16 | 17 | impl SmaCross { 18 | pub fn new(period: usize) -> SmaCross { 19 | SmaCross { 20 | period: period, 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /private/test.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=../dist/lib:../util/target/release/deps cargo test -- --nocapture 2 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script starts the whole platform and is the only official way to start the platform. 4 | # It initializes a spawner instance which in turn spawns a Logger and a MM instance. 5 | # Once you run this script, you can view the MM web console in your web browser (default port 8002). 6 | 7 | LD_LIBRARY_PATH="$(pwd)/dist/lib" 8 | export LD_LIBRARY_PATH 9 | cd dist && RUST_BACKTRACE=1 RUST_BACKTRACE=1 ./spawner 10 | -------------------------------------------------------------------------------- /scripts/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock 3 | *.so 4 | -------------------------------------------------------------------------------- /scripts/fuzz/.cargo/config: -------------------------------------------------------------------------------- 1 | [build] 2 | rustflags = [ 3 | "-L", "../../util/target/release/deps", 4 | "-L", "../../../util/target/release/deps", 5 | "-L", "../../dist/lib", 6 | "-L", "../../../dist/lib", 7 | "-C", "prefer-dynamic" 8 | ] 9 | -------------------------------------------------------------------------------- /scripts/fuzz/.gitignore: -------------------------------------------------------------------------------- 1 | gdb.txt 2 | -------------------------------------------------------------------------------- /scripts/fuzz/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fuzz" 3 | version = "0.1.0" 4 | authors = ["Casey Primozic "] 5 | 6 | [profile.release] 7 | opt-level = 3 8 | debug = true 9 | debug-assertions = false 10 | -------------------------------------------------------------------------------- /scripts/fuzz/core.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=~/tickgrinder/dist/lib/:~/tickgrinder/util/target/release/deps/:~/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/ ddd $(find | grep debug/fuzz) core 2 | -------------------------------------------------------------------------------- /scripts/fuzz/gdb.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=~/tickgrinder/dist/lib/:~/tickgrinder/util/target/release/deps/:~/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/ ddd target/debug/fuzz 2 | -------------------------------------------------------------------------------- /scripts/fuzz/gdb_release.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=~/tickgrinder/dist/lib/:~/tickgrinder/util/target/release/deps/:~/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/ ddd $(find | grep debug/fuzz) 2 | -------------------------------------------------------------------------------- /scripts/fuzz/profile.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=../../dist/lib valgrind --tool=callgrind target/release/fuzz --dump-instr=yes 2 | -------------------------------------------------------------------------------- /scripts/fuzz/rebuild.sh: -------------------------------------------------------------------------------- 1 | cd ../../broker_shims/FXCM/native && cargo clean && ./install.sh 2 | rm ../../simbroker/target -rf 3 | cd ../../../ && make dev 4 | cd private && cargo clean && ./install.sh 5 | cd ../scripts/fuzz && cargo clean && cargo build 6 | -------------------------------------------------------------------------------- /scripts/fuzz/rebuild_release.sh: -------------------------------------------------------------------------------- 1 | cd ../../broker_shims/FXCM/native && cargo clean && ./install_release.sh 2 | rm ../../simbroker/target -rf 3 | cd ../../../ && make dev_release 4 | cd private && cargo clean && ./install_release.sh 5 | cd ../scripts/fuzz && cargo clean && cargo build --release 6 | -------------------------------------------------------------------------------- /scripts/fuzz/release.sh: -------------------------------------------------------------------------------- 1 | CARGO_INCREMENTAL=1 LD_LIBRARY_PATH=../../dist/lib $(find target | rg target/release/deps/fuzz- --no-line-number) 2 | -------------------------------------------------------------------------------- /scripts/fuzz/run.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=../../dist/lib $(find target | rg target/debug/deps/fuzz- --no-line-number) 2 | -------------------------------------------------------------------------------- /spawner/.cargo/config: -------------------------------------------------------------------------------- 1 | [build] 2 | rustflags = [ 3 | "-L", "../util/target/release/deps", 4 | "-L", "../../util/target/release/deps", 5 | "-L", "../../../util/target/release/deps", 6 | "-C", "prefer-dynamic" 7 | ] 8 | -------------------------------------------------------------------------------- /spawner/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /spawner/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "spawner" 3 | version = "0.1.0" 4 | authors = ["Casey Primozic "] 5 | description = "Instance spawner and management system for the TickGrinder algorithmic trading platform" 6 | 7 | [profile.release] 8 | opt-level = 3 9 | debug = true 10 | debug-assertions = false 11 | 12 | [dependencies] 13 | tantivy = { git = "https://github.com/tantivy-search/tantivy.git", rev = "27c373d26df7ba89e04f596dc76507436f90d89c" } 14 | -------------------------------------------------------------------------------- /spawner/README.md: -------------------------------------------------------------------------------- 1 | # Instance Spawner 2 | 3 | This is the main entry point to the TickGrinder platform. It is responsible for spawning and managing all instances of the platform. 4 | 5 | By default, it spawns the MM interface, which can be used to initiate trading activity, run backtests, and perform all other bot functionality, and a Logger instance. 6 | -------------------------------------------------------------------------------- /spawner/install.sh: -------------------------------------------------------------------------------- 1 | CARGO_INCREMENTAL=1 LD_LIBRARY_PATH=../dist/lib:../util/target/release/deps cargo build 2 | cp target/debug/spawner ../dist/ 3 | -------------------------------------------------------------------------------- /spawner/install_release.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=../dist/lib:../util/target/release/deps cargo build --release 2 | cp target/release/spawner ../dist/ 3 | -------------------------------------------------------------------------------- /tick_parser/.cargo/config: -------------------------------------------------------------------------------- 1 | [build] 2 | rustflags = [ 3 | "-L", "../util/target/release/deps", 4 | "-L", "../../util/target/release/deps", 5 | "-L", "../../../util/target/release/deps", 6 | "-C", "prefer-dynamic" 7 | ] 8 | -------------------------------------------------------------------------------- /tick_parser/.gitignore: -------------------------------------------------------------------------------- 1 | *.lock 2 | target 3 | -------------------------------------------------------------------------------- /tick_parser/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tick_processor" 3 | version = "4.0.3" 4 | authors = ["Casey Primozic "] 5 | 6 | [profile.release] 7 | opt-level = 3 8 | debug = true 9 | debug-assertions = false 10 | -------------------------------------------------------------------------------- /tick_parser/src/transport/mod.rs: -------------------------------------------------------------------------------- 1 | #[allow(unused_imports)] 2 | mod test; 3 | -------------------------------------------------------------------------------- /tick_parser/src/transport/test.rs: -------------------------------------------------------------------------------- 1 | use futures::Future; 2 | use futures::stream::Stream; 3 | use redis; 4 | use uuid::Uuid; 5 | 6 | use std::thread; 7 | use std::time::Duration; 8 | 9 | use tickgrinder_util::transport; 10 | use tickgrinder_util::transport::redis::*; 11 | use tickgrinder_util::transport::commands::*; 12 | use tickgrinder_util::transport::postgres; 13 | use tickgrinder_util::transport::query_server::QueryServer; 14 | use tickgrinder_util::transport::command_server::*; 15 | use tickgrinder_util::trading::tick::{Tick, SymbolTick}; 16 | use tickgrinder_util::conf::CONF; 17 | use processor::Processor; 18 | 19 | #[test] 20 | fn postgres_tick_insertion() { 21 | let mut qs = QueryServer::new(5); 22 | for i in 0..10 { 23 | let t = Tick {timestamp: i, bid: 1, ask: 1}; 24 | t.store("test0", &mut qs); 25 | } 26 | // todo 🔜: make sure they were actually inserted 27 | // ^^ 3 months later 28 | } 29 | 30 | #[test] 31 | fn postgres_db_reset() { 32 | let client = postgres::get_client().unwrap(); 33 | postgres::reset_db(&client, CONF.postgres_user).unwrap(); 34 | } 35 | 36 | /// Subscribe to Redis PubSub channel, then send some ticks 37 | /// through and make sure they're stored and processed. 38 | #[test] 39 | fn tick_ingestion() { 40 | let mut processor = Processor::new("test8".to_string(), &Uuid::new_v4()); 41 | let rx = sub_channel(CONF.redis_host, "TEST_ticks_ii"); 42 | let mut client = get_client(CONF.redis_host); 43 | 44 | // send 5 ticks to through the redis channel 45 | for timestamp in 1..6 { 46 | let client = &mut client; 47 | let tick_string = format!("{{\"bid\": 1, \"ask\": 1, \"timestamp\": {}}}", timestamp); 48 | println!("{}", tick_string); 49 | redis::cmd("PUBLISH") 50 | .arg("TEST_ticks_ii") 51 | .arg(tick_string) 52 | .execute(client); 53 | } 54 | 55 | // process the 5 ticks 56 | for json_tick in rx.wait().take(5) { 57 | processor.process(Tick::from_json_string(json_tick.expect("unable to unwrap json_tick"))); 58 | } 59 | // assert_eq!(processor.ticks.len(), 5); 60 | // TODO: Update to modern tick processing stuff 61 | } 62 | 63 | #[test] 64 | fn command_server_broadcast() { 65 | use std::str::FromStr; 66 | 67 | let cmds_channel_string = String::from("test_channel_998"); 68 | let mut cs = CommandServer::new(Uuid::new_v4(), "Tick Processor Test"); 69 | let mut client = get_client(CONF.redis_host); 70 | let rx = sub_channel(CONF.redis_host, &cmds_channel_string); 71 | 72 | let cmd = Command::Ping; 73 | let responses_future = cs.broadcast(cmd, cmds_channel_string); 74 | 75 | let recvd_cmd_str = rx.wait().next().unwrap().unwrap(); 76 | let recvd_cmd = WrappedCommand::from_str(recvd_cmd_str.as_str()).unwrap(); 77 | let res = Response::Pong{args: vec!("1".to_string(), "2".to_string())}; 78 | for _ in 0..2 { 79 | redis::cmd("PUBLISH") 80 | .arg(CONF.redis_responses_channel) 81 | .arg(res.wrap(recvd_cmd.uuid).to_string().unwrap().as_str()) 82 | .execute(&mut client); 83 | } 84 | 85 | let responses = responses_future.wait().unwrap(); 86 | assert_eq!(responses.len(), 2); 87 | thread::sleep(Duration::new(3,0)); 88 | } 89 | -------------------------------------------------------------------------------- /tick_parser/test.sh: -------------------------------------------------------------------------------- 1 | LD_LIBRARY_PATH=native/dist:../util/target/release/deps cargo test -- --nocapture 2 | -------------------------------------------------------------------------------- /tick_writer/writer.js: -------------------------------------------------------------------------------- 1 | //! Tick Writer 2 | //! 3 | //! Listens for new, live, real ticks on redis and records them to flatfile storage for backtesting etc. 4 | 5 | /*jslint node: true */ 6 | "use strict"; 7 | 8 | var redis = require("redis"); 9 | var fs = require("fs"); 10 | 11 | //var conf = require("../../conf/conf"); 12 | 13 | var redisClient = redis.createClient(); 14 | redisClient.subscribe("ticks"); 15 | 16 | var initFile = (symbol, callback)=>{ 17 | fs.writeFile("./" + symbol + ".csv", "timestamp, bid, ask", (err, res)=>{ 18 | callback(); 19 | }); 20 | }; 21 | 22 | var existingFiles = {}; 23 | 24 | var parsed; 25 | redisClient.on("message", (channel, message)=>{ 26 | parsed = JSON.parse(message); 27 | console.log(message); 28 | 29 | if(parsed.real){ 30 | new Promise((fulfill, reject)=>{ 31 | if(!existingFiles[parsed.symbol]){ 32 | fs.stat("./" + parsed.symbol + ".csv", (err, stat)=>{ 33 | if(err){ 34 | initFile(parsed.symbol, ()=>{ 35 | existingFiles[parsed.symbol] = true; 36 | fulfill(); 37 | }); 38 | }else{ 39 | existingFiles[parsed.symbol] = true; 40 | fulfill(); 41 | } 42 | }); 43 | }else{ 44 | fulfill(); 45 | } 46 | }).then(()=>{ 47 | var appendString = "\n" + parsed.timestamp.toString() + ", " + parsed.bid.toString() + ", " + parsed.ask.toString(); 48 | fs.appendFile("./" + parsed.symbol + ".csv", appendString, (err, res)=>{}); 49 | }); 50 | } 51 | }); 52 | 53 | -------------------------------------------------------------------------------- /util/.cargo/config: -------------------------------------------------------------------------------- 1 | [build] 2 | rustflags = ["-C", "prefer-dynamic"] 3 | -------------------------------------------------------------------------------- /util/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | Cargo.lock 3 | src/conf.rs 4 | -------------------------------------------------------------------------------- /util/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tickgrinder-util" 3 | version = "0.1.0" 4 | authors = ["Casey Primozic "] 5 | description = "Code used in all of TickGrinder's modules." 6 | 7 | [dependencies] 8 | redis = "0.8.0" 9 | futures = "=0.1.14" 10 | postgres = "0.15.1" 11 | ws = "0.7.3" 12 | serde = "1.0.11" 13 | serde_json = "1.0.2" 14 | serde_derive = "1.0.11" 15 | libflate = "0.1.10" 16 | hyper = "0.11.2" 17 | # rustc-serialize = "0.3" 18 | csv = "1.0.0-beta.4" 19 | tempdir = "0.3.5" 20 | lazy_static = "0.1.*" 21 | uuid = { version = "=0.5.1", features = ["serde", "v4",] } 22 | indoc = "^0.1.15" 23 | time = "0.1.38" 24 | chrono = "0.4.0" 25 | rand = "0.3.16" 26 | from_hashmap = { path = "from_hashmap" } 27 | clippy = { git = "https://github.com/Manishearth/rust-clippy.git", optional = true } 28 | 29 | [lib] 30 | name = "tickgrinder_util" 31 | crate-type = ["dylib"] 32 | 33 | [profile.release] 34 | opt-level = 3 35 | debug = true 36 | debug-assertions = false 37 | -------------------------------------------------------------------------------- /util/from_hashmap/.gitignore: -------------------------------------------------------------------------------- 1 | *.lock 2 | target 3 | -------------------------------------------------------------------------------- /util/from_hashmap/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "from_hashmap" 3 | version = "0.1.0" 4 | authors = ["Casey Primozic "] 5 | description = "Defines a procedural macro to build a struct from a HashMap containing values for its fields" 6 | 7 | [dependencies] 8 | syn = "0.10" 9 | quote = "0.3" 10 | 11 | [lib] 12 | proc-macro = true 13 | -------------------------------------------------------------------------------- /util/js/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["es2015-node5", "stage-3"], 3 | "plugins": [ 4 | "transform-flow-strip-types" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /util/js/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "rules": { 3 | "flowtype/boolean-style": [2, "boolean"], 4 | "flowtype/define-flow-type": 1, 5 | "flowtype/delimiter-dangle": [2, "never"], 6 | "flowtype/generic-spacing": [2, "never"], 7 | "flowtype/no-primitive-constructor-types": 2, 8 | "flowtype/object-type-delimiter": [2, "comma"], 9 | "flowtype/require-valid-file-annotation": 2, 10 | "flowtype/semi": [2, "always"], 11 | "flowtype/space-before-generic-bracket": [2, "never"], 12 | "flowtype/type-id-match": [2, "[A-Z][a-z0-9]+"], 13 | "flowtype/union-intersection-spacing": [2, "always"], 14 | "flowtype/use-flow-type": 1, 15 | "flowtype/valid-syntax": 1, 16 | "indent": [2, 2], 17 | "quotes": [2, "single"], 18 | "linebreak-style": [2, "unix"], 19 | "semi": [2, "always"], 20 | "comma-dangle": [2, "only-multiline"], 21 | "no-console": 0, 22 | "no-unused-vars": [2, {"vars": "all", "args": "none"}], 23 | "no-multiple-empty-lines": [2, {"max": 1}] 24 | }, 25 | "env": { 26 | "es6": true, 27 | "browser": false, 28 | "node": true 29 | }, 30 | "extends": ["eslint:recommended"], 31 | "parser": "babel-eslint", 32 | "parserOptions": { 33 | "sourceType": "module", 34 | "ecmaFeatures": { 35 | "experimentalObjectRestSpread": true, 36 | "jsx": false 37 | }, 38 | "ecmaVersion": 7 39 | }, 40 | "plugins": [ 41 | "flowtype" 42 | ], 43 | "globals": { 44 | "require": true, 45 | "module": true 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /util/js/.flowconfig: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Ameobea/tickgrinder/f2ccd640b173cb17b0af4e04df9b203935eb2dc2/util/js/.flowconfig -------------------------------------------------------------------------------- /util/js/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | stripped/ 3 | yarn.lock 4 | src/conf.js 5 | -------------------------------------------------------------------------------- /util/js/README.txt: -------------------------------------------------------------------------------- 1 | == JavaScript Utility Library == 2 | 3 | Contains FFI bindings to the TickGrinder utility library as well as some common functions that are shared among multiple modules. 4 | -------------------------------------------------------------------------------- /util/js/index.js: -------------------------------------------------------------------------------- 1 | //! JavaScript utility library. See README.txt for more information. 2 | // @flow 3 | 4 | const WebSocket = require('ws'); 5 | 6 | const ffi = require('./src/ffi'); 7 | const CONF = require('./src/conf'); 8 | 9 | /** 10 | * Generates a new V4 UUID in hyphenated form 11 | */ 12 | function v4(): string { 13 | function s4(): string { 14 | return Math.floor((1 + Math.random()) * 0x10000) 15 | .toString(16) 16 | .substring(1); 17 | } 18 | return s4() + s4() + '-' + s4() + '-' + s4() + '-' + 19 | s4() + '-' + s4() + s4() + s4(); 20 | } 21 | 22 | /** 23 | * Starts the WS listening for new messages sets up processing callback 24 | * @param {function} callback - The function invoked when a message is received over the websocket connection 25 | * @param {dispatch} dispatch - A piece of state passed along with the message to the callback function. 26 | * @param {string} ourUuid - The UUID of the instance creating this websocket server 27 | * @param {function} wsError - A callback invoked when the websocket encounters an error 28 | */ 29 | function initWs(callback: (dispatch: any, parsed: any) => void, dispatch: any, ourUuid: string, wsError: (e: string) => void): WebSocket { 30 | let socketUrl = 'ws://localhost:7037'; 31 | let socket = new WebSocket(socketUrl); 32 | socket.onmessage = message => { 33 | if(typeof(message.data) != 'string') { 34 | wsError('Received non-string data over websocket connection!'); 35 | return; 36 | } 37 | let parsed = JSON.parse(message.data); 38 | // throw away messages we're transmitting to channels we don't care about 39 | if ([CONF.redis_control_channel, CONF.redis_responses_channel, CONF.redis_log_channel, ourUuid].indexOf(parsed.channel) !== -1) { 40 | callback(dispatch, parsed); 41 | } 42 | }; 43 | 44 | socket.onerror = () => { 45 | wsError('Unhandled error occured on the websocket connection!'); 46 | }; 47 | 48 | return socket; 49 | } 50 | 51 | module.exports = { 52 | ffi: ffi, 53 | v4: v4, 54 | initWs: initWs, 55 | }; 56 | -------------------------------------------------------------------------------- /util/js/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "util-js", 3 | "version": "1.0.0", 4 | "description": "JavaScript TickGrinder utility library", 5 | "main": "index.js", 6 | "scripts": { 7 | "flow": "flow; test $? -eq 0 -o $? -eq 2", 8 | "strip": "rm -rf stripped && babel --out-dir stripped index.js && babel --out-dir stripped/src src/", 9 | "test": "echo \"Error: no test specified\" && exit 1" 10 | }, 11 | "author": "Casey Primozic ", 12 | "license": "MIT", 13 | "dependencies": { 14 | "ffi": "2.2.0", 15 | "ref": "1.3.4", 16 | "ws": "2.2.0" 17 | }, 18 | "devDependencies": { 19 | "babel-cli": "^6.23.0", 20 | "babel-preset-es2015-node5": "*", 21 | "babel-preset-stage-3": "*", 22 | "babel-plugin-transform-flow-strip-types": "*", 23 | "flow-bin": "*" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /util/src/instance.rs: -------------------------------------------------------------------------------- 1 | //! Defines the `PlatformInstance` trait which can be implemented to create instances that communicate on the platform, 2 | //! sending/receiving commands and interacting with other instances. 3 | 4 | use std::str::FromStr; 5 | 6 | use uuid::Uuid; 7 | use futures::Stream; 8 | use redis; 9 | use serde_json; 10 | 11 | use transport::commands::{Command, Response, WrappedCommand, send_command}; 12 | use transport::command_server::CommandServer; 13 | use transport::redis::{get_client, sub_multiple}; 14 | use conf::CONF; 15 | 16 | pub trait PlatformInstance { 17 | /// Instructs the instance to start listening for and responding to commands. 18 | fn listen(mut self, uuid: Uuid, cs: &mut CommandServer) where Self:Sized { 19 | // subscribe to the command channels 20 | let rx = sub_multiple( 21 | CONF.redis_host, &[CONF.redis_control_channel, uuid.hyphenated().to_string().as_str()] 22 | ); 23 | let redis_client = get_client(CONF.redis_host); 24 | 25 | // Signal to the platform that we're ready to receive commands 26 | let _ = send_command(&WrappedCommand::from_command( 27 | Command::Ready{instance_type: "Backtester".to_string(), uuid: uuid}), &redis_client, "control" 28 | ); 29 | 30 | for res in rx.wait() { 31 | let (_, msg) = res.expect("Received err in the listen() event loop for the backtester!"); 32 | let wr_cmd = match WrappedCommand::from_str(msg.as_str()) { 33 | Ok(wr) => wr, 34 | Err(e) => { 35 | cs.error(Some("CommandProcessing"), &format!("Unable to parse WrappedCommand from String: {:?}", e)); 36 | return; 37 | } 38 | }; 39 | 40 | let res: Option = self.handle_command(wr_cmd.cmd); 41 | if res.is_some() { 42 | redis::cmd("PUBLISH") 43 | .arg(CONF.redis_responses_channel) 44 | .arg(&res.unwrap().wrap(wr_cmd.uuid).to_string().unwrap()) 45 | .execute(&redis_client); 46 | } 47 | } 48 | } 49 | 50 | /// Given a `Command` from the platform, process it and optionally return a `Response` to be sent as a reply. 51 | fn handle_command(&mut self, cmd: Command) -> Option; 52 | } 53 | -------------------------------------------------------------------------------- /util/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! Code shared by all modules of the platform 2 | 3 | #![feature(rustc_attrs, plugin, conservative_impl_trait, test, fn_traits, core, unboxed_closures, libc, raw)] 4 | 5 | #![allow(unknown_lints)] 6 | #![cfg_attr(feature="clippy", feature(plugin))] 7 | #![cfg_attr(feature="clippy", plugin(clippy))] 8 | 9 | extern crate redis; 10 | extern crate futures; 11 | extern crate uuid; 12 | extern crate serde; 13 | extern crate serde_json; 14 | #[macro_use] 15 | extern crate serde_derive; 16 | extern crate postgres; 17 | extern crate csv; 18 | extern crate rand; 19 | extern crate time; 20 | extern crate test; 21 | extern crate libc; 22 | 23 | pub mod transport; 24 | pub mod strategies; 25 | pub mod trading; 26 | pub mod instance; 27 | pub mod conf; 28 | -------------------------------------------------------------------------------- /util/src/trading/broker.rs: -------------------------------------------------------------------------------- 1 | //! Represents a broker - the endpoint for all trading activity on the platform. 2 | //! Also contains helper functions for managing accounts. 3 | 4 | use std::collections::HashMap; 5 | 6 | use futures::sync::oneshot::Receiver; 7 | use futures::stream::Stream; 8 | 9 | use trading::tick::Tick; 10 | pub use trading::objects::*; 11 | 12 | /// A broker is the endpoint for all trading actions taken by the platform. It processes 13 | /// trades and supplies information about the condition of portfolios. The Broker trait 14 | /// acts as a wrapper for individual broker APIs. 15 | pub trait Broker { 16 | /// Creates a connection to the broker and initializes its internal environment. 17 | /// Takes a Key:Value HashMap containing configuration settings. 18 | fn init(settings: HashMap) -> Receiver> where Self:Sized; 19 | 20 | /// Executes a BrokerAction on the broker, returning its response. 21 | fn execute(&mut self, action: BrokerAction) -> PendingResult; 22 | 23 | /// Returns a stream of messages pushed from the broker that do not originate from an 24 | /// action sent to the broker. These can be things like notifications of closed positions, 25 | /// orders being filled, etc. 26 | fn get_stream(&mut self) -> Result + Send>, BrokerError>; 27 | 28 | /// Returns a stream of live ticks for a symbol. 29 | fn sub_ticks(&mut self, symbol: String) -> Result + Send>, BrokerError>; 30 | } 31 | 32 | /// Utility type for a broker response that may fail 33 | pub type BrokerResult = Result; 34 | 35 | /// Utility type for a currently pending broker action 36 | pub type PendingResult = Receiver; 37 | -------------------------------------------------------------------------------- /util/src/trading/datafield.rs: -------------------------------------------------------------------------------- 1 | use std::ops::Index; 2 | 3 | #[derive(Debug)] 4 | pub struct DataField { 5 | pub data: Vec 6 | } 7 | 8 | #[allow(dead_code)] 9 | impl DataField { 10 | pub fn new() -> DataField { 11 | DataField { 12 | data: Vec::new() 13 | } 14 | } 15 | 16 | pub fn push(&mut self, d: T) { 17 | self.data.push(d); 18 | } 19 | 20 | pub fn first(&mut self) -> Option<&T> { 21 | self.data.first() 22 | } 23 | 24 | pub fn last(&mut self) -> Option<&T> { 25 | self.data.last() 26 | } 27 | 28 | pub fn len(&self) -> usize { 29 | self.data.len() 30 | } 31 | 32 | pub fn is_empty(&self) -> bool { 33 | self.data.is_empty() 34 | } 35 | } 36 | 37 | impl Index for DataField { 38 | type Output = T; 39 | 40 | fn index(&self, index: usize) -> &T { 41 | &self.data[index] 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /util/src/trading/indicators.rs: -------------------------------------------------------------------------------- 1 | //! Indicators transform data from a live data source, database, or other indicator into some 2 | //! metric that can be used to create trading signals or analyze data. 3 | 4 | use std::collections::HashMap; 5 | 6 | use trading::tick::*; 7 | 8 | /// This trait is used to use an indicator to read historical data from the database or some 9 | /// other source and return in in a format that the Monitor plotting API understands (JSON). 10 | pub trait HistQuery { 11 | /// Returns a JSON-formatted string containing an Array of timestamped indicator values 12 | /// through the specified time range. Args is any additional indicator-specific 13 | /// arguments that need to be given. Period is the minimum time that must elapse between 14 | /// two distinct indicator values returned. 15 | fn get(start_time: u64, end_time: u64, period: u64, args: HashMap) -> Result; 16 | } 17 | 18 | /// Implemented for indicators that can process live ticks. The `tick()` function should be 19 | /// called for every live tick and an arbitrary data type returned. 20 | pub trait LiveQuery { 21 | /// Called every new tick received; optionally returns an arbitrary piece of data every time 22 | /// a new one is received. 23 | fn tick(t: Tick) -> Option; 24 | } 25 | -------------------------------------------------------------------------------- /util/src/trading/mod.rs: -------------------------------------------------------------------------------- 1 | //! Defines structs and methods for trading primitives (trades, positions, etc.) 2 | //! used for interacting with brokers and backtesting. 3 | 4 | pub mod tick; 5 | pub mod broker; 6 | pub mod indicators; 7 | pub mod trading_condition; 8 | pub mod datafield; 9 | pub mod objects; 10 | -------------------------------------------------------------------------------- /util/src/trading/trading_condition.rs: -------------------------------------------------------------------------------- 1 | //! Trading conditions are expressions that the Tick Processor evaluates for every received tick. 2 | //! If the condition returns a `TradingAction` when evaluated, that action is executed. 3 | 4 | use uuid::Uuid; 5 | 6 | use trading::tick::Tick; 7 | 8 | pub trait TradingCondition { 9 | /// Evaulate a new Tick with the condition. Returns a TradingAction to take or None. 10 | fn eval(&mut self, t: &Tick) -> Option; 11 | } 12 | 13 | #[derive(Clone, Debug, PartialEq)] 14 | pub enum TradingAction { 15 | /// Opens an order at market price +-max_range pips. 16 | MarketOrder { 17 | symbol: String, long: bool, size: usize, stop: Option, 18 | take_profit: Option, max_range: Option, 19 | }, 20 | /// Opens an order at a price equal or better to `entry_price` as soon as possible. 21 | LimitOrder{ 22 | symbol: String, long: bool, size: usize, stop: Option, 23 | take_profit: Option, entry_price: usize, 24 | }, 25 | /// Closes `size` units of a position with the specified UUID at the current market rate. 26 | MarketClose{ uuid: Uuid, size: usize, }, 27 | /// Places an order to close `size` units of a position with the specified UUID. 28 | LimitClose{ uuid: Uuid, size: usize, exit_price: usize, }, 29 | /// Modifies an order without taking any trading action 30 | ModifyOrder{ uuid: Uuid, size: usize, entry_price: usize, stop: Option, take_profit: Option,}, 31 | /// Modifies a position without taking any trading action. 32 | ModifyPosition{ uuid: Uuid, stop: Option, take_profit: Option }, 33 | /// Attempts to cancel an order 34 | CancelOrder{ uuid: Uuid }, 35 | } 36 | -------------------------------------------------------------------------------- /util/src/transport/mod.rs: -------------------------------------------------------------------------------- 1 | //! Defines how the different modules of the platform talk to each other and interact with the outside world. 2 | 3 | pub mod redis; 4 | pub mod postgres; 5 | pub mod commands; 6 | pub mod query_server; 7 | pub mod command_server; 8 | pub mod tickstream; 9 | pub mod textlog; 10 | pub mod data; 11 | pub mod ffi; 12 | -------------------------------------------------------------------------------- /util/src/transport/postgres.rs: -------------------------------------------------------------------------------- 1 | // functions for communicating with the postgresql database 2 | 3 | #![allow(unused_must_use)] 4 | 5 | use postgres::{Connection, Error, TlsMode}; 6 | 7 | use conf::CONF; 8 | 9 | pub fn get_client() -> Result { 10 | let conn_string = format!("postgres://{}:{}@{}:{}/{}", 11 | CONF.postgres_user, 12 | CONF.postgres_password, 13 | CONF.postgres_host, 14 | CONF.postgres_port, 15 | CONF.postgres_db 16 | ); 17 | 18 | // TODO: Look into setting up TLS 19 | Connection::connect(conn_string.as_str(), TlsMode::None) 20 | } 21 | 22 | /**************************\ 23 | * TICK-RELATED FUNCTIONS * 24 | \**************************/ 25 | 26 | pub fn csv_to_tick_table(filename: &str, table_name: &str, client: &Connection) { 27 | let query = format!( 28 | "COPY {}(tick_time, bid, ask) FROM {} DELIMETER ', ' CSV HEADER", 29 | table_name, 30 | filename 31 | ); 32 | client.execute(&query, &[]); 33 | } 34 | 35 | /// Creates a new table for ticks with given symbol if such a table doesn't already exist. 36 | pub fn init_tick_table(symbol: &str, client: &Connection, pg_user: &str) -> Result<(), String> { 37 | tick_table_inner(format!("ticks_{}", symbol).as_str(), client, pg_user) 38 | } 39 | 40 | /// Initializes a table in which historical ticks can be stored if such a table doesn't already exist. 41 | pub fn init_hist_data_table(table_name: &str, client: &Connection, pg_user: &str) -> Result<(), String> { 42 | tick_table_inner(table_name, client, pg_user) 43 | } 44 | 45 | fn tick_table_inner(table_name: &str, client: &Connection, pg_user: &str) -> Result<(), String> { 46 | let query1 = format!( 47 | "CREATE TABLE IF NOT EXISTS {} 48 | ( 49 | tick_time BIGINT NOT NULL PRIMARY KEY UNIQUE, 50 | bid BIGINT NOT NULL, 51 | ask BIGINT NOT NULL 52 | ) 53 | WITH ( 54 | OIDS=FALSE 55 | );", table_name); 56 | let query2 = format!( 57 | "ALTER TABLE {} 58 | OWNER TO {};", table_name, pg_user); 59 | client.execute(&query1, &[]) 60 | .map_err(|_| "Error while querying postgres to set up tick table" ); 61 | client.execute(&query2, &[]) 62 | .map_err(|_| "Error while querying postgres to set up tick table" ); 63 | 64 | Ok(()) 65 | } 66 | 67 | /*************************** 68 | * ADMINISTRATIVE FUNCTIONS * 69 | ***************************/ 70 | 71 | /// Drops all tables in the database, resetting it to defaults 72 | pub fn reset_db(client: &Connection, pg_user: &'static str) -> Result<(), Error> { 73 | let query = format!("DROP SCHEMA public CASCADE; 74 | CREATE SCHEMA public AUTHORIZATION {}; 75 | ALTER SCHEMA public OWNER TO {}; 76 | GRANT ALL ON SCHEMA public TO {};", 77 | pg_user, pg_user, pg_user); 78 | client.batch_execute(query.as_str()) 79 | } 80 | -------------------------------------------------------------------------------- /util/src/transport/textlog.rs: -------------------------------------------------------------------------------- 1 | //! Helper functions for logging to file. 2 | 3 | use std::thread; 4 | use std::path::PathBuf; 5 | use std::fs::{DirBuilder, File}; 6 | use std::io::Write; 7 | use std::fmt::Debug; 8 | use time::now; 9 | 10 | use futures::Stream; 11 | use futures::sync::mpsc::{channel, Sender}; 12 | 13 | use conf::CONF; 14 | 15 | /// Returns a `Sender` that writes all `String`s sent to it to file. `dirname` is the name of the subdirectory of the 16 | /// `[CONF.data_dir]/logs` directory where the file is written and `chunk_size` is how many lines are stored in memory 17 | /// before being written to disc. 18 | pub fn get_logger_handle(dirname: String, chunk_size: usize) -> Sender { 19 | let (tx, rx) = channel(chunk_size); 20 | 21 | // spawn the logger thread and initialize the logging loop 22 | thread::spawn(move || { 23 | let dirname = &dirname; 24 | // if the directories don't exist in the logging directory, create them 25 | let log_dir: PathBuf = PathBuf::from(CONF.data_dir).join("logs").join(dirname); 26 | if !log_dir.is_dir() { 27 | let mut builder = DirBuilder::new(); 28 | builder.recursive(true).create(log_dir.clone()) 29 | .expect("Unable to create directory to hold the log files; permission issue or bad data dir configured?"); 30 | } 31 | 32 | println!("Attempting to find valid filename..."); 33 | let mut attempts = 1; 34 | let curtime = now(); 35 | let mut datestring = format!("{}-{}_{}.log", curtime.tm_mon + 1, curtime.tm_mday, attempts); 36 | while PathBuf::from(CONF.data_dir).join("logs").join(dirname).join(&datestring).exists() { 37 | attempts += 1; 38 | datestring = format!("{}-{}_{}.log", curtime.tm_mon + 1, curtime.tm_mday, attempts); 39 | } 40 | 41 | println!("creating log file..."); 42 | let datestring = format!("{}-{}_{}.log", curtime.tm_mon + 1, curtime.tm_mday, attempts); 43 | let mut file = File::create(PathBuf::from(CONF.data_dir).join("logs").join(dirname).join(&datestring)) 44 | .expect("Unable to create log file!"); 45 | 46 | // buffer up chunk_size log lines before writing to disk 47 | for msg in rx.chunks(chunk_size).wait() { 48 | // println!("Logging message chunk..."); 49 | let text: String = match msg { 50 | Ok(lines) => lines.as_slice().join("\n") + "\n", 51 | // World is likely dropping due to a crash or shutdown 52 | Err(_) => unimplemented!(), 53 | }; 54 | 55 | // write the chunk_size lines into the file 56 | write!(&mut file, "{}", text).expect("Unable to write lines into log file!"); 57 | } 58 | }); 59 | 60 | tx 61 | } 62 | 63 | /// Given a type that can be debug-formatted, returns a String that contains its debug-formatted version. 64 | pub fn debug(x: T) -> String where T:Debug { 65 | format!("{:?}", x) 66 | } 67 | -------------------------------------------------------------------------------- /util/src/transport/tickstream/generators/mod.rs: -------------------------------------------------------------------------------- 1 | //! Tick generators take data from an external source and create a stream of time series data out of it that can be saved to storage, used 2 | //! for a backtest, or fed into strategies during a live trading system. 3 | 4 | pub mod flatfile_reader; 5 | pub mod postgres_reader; 6 | pub mod random_reader; 7 | pub mod redis_reader; 8 | -------------------------------------------------------------------------------- /util/src/transport/tickstream/generators/random_reader.rs: -------------------------------------------------------------------------------- 1 | //! A `TickGenerator` that generates random ticks. 2 | 3 | use std::thread; 4 | use std::sync::{Arc, Mutex}; 5 | use std::sync::atomic::AtomicBool; 6 | use rand::{thread_rng, ThreadRng}; 7 | use rand::distributions::{IndependentSample, Range}; 8 | 9 | use futures::sync::mpsc::channel; 10 | use futures::{Future, Stream, Sink}; 11 | use futures::stream::BoxStream; 12 | 13 | use trading::tick::Tick; 14 | 15 | use super::super::*; 16 | 17 | pub struct RandomReader {} 18 | 19 | impl TickGenerator for RandomReader { 20 | fn get( 21 | &mut self, mut map: Box, cmd_handle: CommandStream 22 | ) -> Result, String> { 23 | let (mut tx, rx) = channel::(1); 24 | let mut timestamp = 0; 25 | 26 | // small atomic communication bus between the handle listener and worker threads 27 | let internal_message: Arc> = Arc::new(Mutex::new(TickstreamCommand::Stop)); 28 | let _internal_message = internal_message.clone(); 29 | let got_mail = Arc::new(AtomicBool::new(false)); 30 | let mut _got_mail = got_mail.clone(); 31 | 32 | let reader_handle = thread::spawn(move || { 33 | thread::park(); 34 | 35 | let mut rng = thread_rng(); 36 | loop { 37 | if check_mail(&*got_mail, &*_internal_message) { 38 | println!("Stop command received; killing reader"); 39 | break; 40 | } 41 | timestamp += 1; 42 | 43 | let tick = get_rand_tick(&mut rng, timestamp); 44 | 45 | // apply the map 46 | let mod_t = map.map(tick); 47 | if mod_t.is_some() { 48 | tx = tx.send(mod_t.unwrap()).wait().expect("Unable to send tick to sink in random_reader.rs"); 49 | } 50 | } 51 | }).thread().clone(); 52 | 53 | // spawn the handle listener thread that awaits commands 54 | spawn_listener_thread(_got_mail, cmd_handle, internal_message, reader_handle); 55 | 56 | Ok(rx.boxed()) 57 | } 58 | 59 | fn get_raw(&mut self) -> Result, String> { 60 | let (mut tx, rx) = channel(1); 61 | let mut timestamp = 0; 62 | 63 | thread::spawn(move || { 64 | let mut rng = thread_rng(); 65 | loop { 66 | let t = get_rand_tick(&mut rng, timestamp); 67 | tx = tx.send(t).wait().expect("Unable to send through tx in `get_raw` in random_reader!"); 68 | timestamp += 1; 69 | } 70 | }); 71 | 72 | Ok(rx.boxed()) 73 | } 74 | } 75 | 76 | fn get_rand_tick(mut rng: &mut ThreadRng, timestamp: u64) -> Tick { 77 | let price_range = Range::new(10, 99); 78 | let spread_range = Range::new(0, 5); 79 | 80 | let price = price_range.ind_sample(rng); 81 | let spread = spread_range.ind_sample(rng); 82 | 83 | Tick { 84 | timestamp: timestamp, 85 | bid: price, 86 | ask: price-spread, 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /util/src/transport/tickstream/generators/redis_reader.rs: -------------------------------------------------------------------------------- 1 | //! A `TickGenerator` that reads ticks out of a Redis channel. 2 | 3 | use std::thread; 4 | 5 | use futures::sync::mpsc::channel; 6 | use futures::{Future, Stream, Sink}; 7 | use futures::stream::BoxStream; 8 | 9 | use trading::tick::Tick; 10 | use transport::redis::sub_channel; 11 | 12 | use super::super::*; 13 | 14 | pub struct RedisReader { 15 | pub symbol: String, 16 | pub redis_host: String, 17 | pub channel: String 18 | } 19 | 20 | impl TickGenerator for RedisReader { 21 | fn get( 22 | &mut self, mut map: Box, cmd_handle: CommandStream 23 | ) -> Result, String> { 24 | let host = self.redis_host.clone(); 25 | let input_channel = self.channel.clone(); 26 | 27 | // small atomic communication bus between the handle listener and worker threads 28 | let internal_message: Arc> = Arc::new(Mutex::new(TickstreamCommand::Stop)); 29 | let _internal_message = internal_message.clone(); 30 | let got_mail = Arc::new(AtomicBool::new(false)); 31 | let mut _got_mail = got_mail.clone(); 32 | 33 | let (mut tx, rx) = channel::(1); 34 | 35 | let reader_handle = thread::spawn(move || { 36 | let in_rx = sub_channel(host.as_str(), input_channel.as_str()); 37 | 38 | for t_string in in_rx.wait() { 39 | if check_mail(&*got_mail, &*_internal_message) { 40 | println!("Stop command received; killing reader"); 41 | break; 42 | } 43 | let t = Tick::from_json_string(t_string.unwrap()); 44 | 45 | // apply map 46 | let t_mod = map.map(t); 47 | if t_mod.is_some() { 48 | tx = tx.send(t_mod.unwrap()).wait().expect("Unable to send through tx in `get` redis_reader!"); 49 | } 50 | } 51 | }).thread().clone(); 52 | 53 | // spawn the handle listener thread that awaits commands 54 | spawn_listener_thread(_got_mail, cmd_handle, internal_message, reader_handle); 55 | 56 | Ok(rx.boxed()) 57 | } 58 | 59 | fn get_raw(&mut self) -> Result, String> { 60 | let (mut tx, rx) = channel(1); 61 | 62 | let input_channel = self.channel.clone(); 63 | thread::spawn(move || { 64 | let in_rx = sub_channel(CONF.redis_host, input_channel.as_str()); 65 | 66 | for t_string in in_rx.wait() { 67 | let t = Tick::from_json_string(t_string.unwrap()); 68 | tx = tx.send(t).wait().expect("Unable to send through tx in `get_raw` in redis_reader!"); 69 | } 70 | }); 71 | 72 | Ok(rx.boxed()) 73 | } 74 | } 75 | 76 | impl RedisReader { 77 | pub fn new(symbol: String, host: String, channel: String) -> RedisReader { 78 | RedisReader { 79 | symbol: symbol, 80 | redis_host: host, 81 | channel: channel 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /util/src/transport/tickstream/maps/mod.rs: -------------------------------------------------------------------------------- 1 | //! Contains all the `TickMap`s for the platform. 2 | 3 | use super::*; 4 | 5 | pub mod poloniex; 6 | 7 | pub use self::poloniex::*; 8 | 9 | /// Inserts a static delay between each tick. 10 | pub struct FastMap { 11 | pub delay_ms: usize 12 | } 13 | 14 | impl TickMap for FastMap { 15 | fn map(&mut self, t: Tick) -> Option { 16 | // block for the delay then return the tick 17 | thread::sleep(Duration::from_millis(self.delay_ms as u64)); 18 | Some(t) 19 | } 20 | } 21 | 22 | /// Plays ticks back at the rate that they were recorded. 23 | pub struct LiveMap { 24 | pub last_tick_timestamp: u64 25 | } 26 | 27 | impl TickMap for LiveMap { 28 | fn map(&mut self, t: Tick) -> Option { 29 | if self.last_tick_timestamp == 0 { 30 | self.last_tick_timestamp = t.timestamp; 31 | return None 32 | } 33 | 34 | let diff = t.timestamp - self.last_tick_timestamp; 35 | thread::sleep(Duration::from_millis(diff as u64)); 36 | Some(t) 37 | } 38 | } 39 | 40 | impl LiveMap { 41 | pub fn new() -> LiveMap { 42 | LiveMap { 43 | last_tick_timestamp: 0 44 | } 45 | } 46 | } 47 | 48 | pub struct NullMap {} 49 | 50 | impl TickMap for NullMap { 51 | fn map(&mut self, t: Tick) -> Option { Some(t) } 52 | } 53 | -------------------------------------------------------------------------------- /util/src/transport/tickstream/sinks/console_sink.rs: -------------------------------------------------------------------------------- 1 | //! Simply `println!()`s ticks to the console; useful for debugging. 2 | 3 | use std::fmt::Debug; 4 | use std::collections::HashMap; 5 | 6 | use trading::tick::{Tick, GenTick}; 7 | use transport::tickstream::{TickSink, GenTickSink}; 8 | 9 | pub struct ConsoleSink {} 10 | 11 | impl TickSink for ConsoleSink { 12 | fn tick(&mut self, t: Tick) { 13 | println!("{:?}", t); 14 | } 15 | } 16 | 17 | impl GenTickSink for ConsoleSink where T:Debug, T:Sized { 18 | fn new(_: HashMap) -> Result { 19 | Ok(ConsoleSink {}) 20 | } 21 | 22 | fn tick(&mut self, t: GenTick) { 23 | println!("{:?}", t); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /util/src/transport/tickstream/sinks/csv_sink.rs: -------------------------------------------------------------------------------- 1 | //! Saves data to a CSV flatfile. 2 | 3 | // TODO: Enable compression/decompression and transferring to/from archives 4 | 5 | use std::collections::HashMap; 6 | use std::fs::File; 7 | use std::path::Path; 8 | use std::marker::PhantomData; 9 | 10 | use csv::Writer; 11 | use serde::{Serialize, Deserialize}; 12 | 13 | use trading::tick::GenTick; 14 | use transport::tickstream::GenTickSink; 15 | use transport::textlog::debug; 16 | 17 | // TODO: Non `CommandServer`-Based Logging 18 | 19 | pub struct CsvSink { 20 | writer: Writer, 21 | ghost: PhantomData, 22 | } 23 | 24 | /// A tick sink that writes data to a CSV file. As long as the data is able to be split up into columns and serialized, this sink should be able to handle it. 25 | /// Requires that the setting `output_path` be supplied in the settings `HashMap`. 26 | impl GenTickSink for CsvSink where T : Serialize, T : for<'de> Deserialize<'de> { 27 | fn new(settings: HashMap) -> Result { 28 | let output_path = match settings.get("output_path") { 29 | Some(p) => p, 30 | None => { return Err(String::from("You must supply an `output_path` argument in the input `HashMap`~")) }, 31 | }; 32 | 33 | Ok(CsvSink { 34 | writer: Writer::from_path(Path::new(output_path)).map_err(debug)?, 35 | ghost: PhantomData{}, 36 | }) 37 | } 38 | 39 | fn tick(&mut self, t: GenTick) { 40 | if let Err(e) = self.writer.serialize((t.timestamp, t.data)) { 41 | println!("Error while writing line to file: {:?}", e); 42 | } 43 | let _ = self.writer.flush(); // TODO: find a better way to handle this 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /util/src/transport/tickstream/sinks/mod.rs: -------------------------------------------------------------------------------- 1 | //! Tick sinks server as the consumers of time series data streams within the platform. They can be things such as databases where the data 2 | //! is stored, backtests, or strategy executors. 3 | 4 | pub mod console_sink; 5 | pub mod csv_sink; 6 | pub mod null_sink; 7 | pub mod redis_sink; 8 | pub mod stream_sink; 9 | -------------------------------------------------------------------------------- /util/src/transport/tickstream/sinks/null_sink.rs: -------------------------------------------------------------------------------- 1 | //! Pipes data into the abyss. 2 | 3 | #[allow(unused_imports)] 4 | use test; 5 | 6 | use trading::tick::Tick; 7 | 8 | use transport::tickstream::TickSink; 9 | 10 | pub struct NullSink {} 11 | 12 | impl TickSink for NullSink { 13 | #[allow(unused_variables)] 14 | fn tick(&mut self, t: Tick) {} 15 | } 16 | 17 | /// I'd like to imagine this is optimized out but you never know... 18 | #[bench] 19 | fn null_sink(b: &mut test::Bencher) { 20 | let mut ns = NullSink{}; 21 | let t = Tick::null(); 22 | b.iter(|| ns.tick(t)) 23 | } 24 | -------------------------------------------------------------------------------- /util/src/transport/tickstream/sinks/redis_sink.rs: -------------------------------------------------------------------------------- 1 | //! Send the output ticks of the backtest through a Redis channel 2 | 3 | use redis::{Client, cmd}; 4 | 5 | use transport::redis::get_client; 6 | use trading::tick::Tick; 7 | use transport::tickstream::TickSink; 8 | 9 | pub struct RedisSink { 10 | pub symbol: String, 11 | pub tx_channel: String, 12 | pub client: Client 13 | } 14 | 15 | impl TickSink for RedisSink { 16 | fn tick(&mut self, t: Tick) { 17 | cmd("PUBLISH") 18 | .arg(self.tx_channel.clone()) 19 | .arg(t.to_json_string(self.symbol.clone())) 20 | .execute(&self.client); 21 | } 22 | } 23 | 24 | impl RedisSink { 25 | pub fn new(symbol: String, tx_channel: String, redis_host: &str) -> RedisSink { 26 | RedisSink { 27 | symbol: symbol, 28 | tx_channel: tx_channel, 29 | client: get_client(redis_host) 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /util/src/transport/tickstream/sinks/stream_sink.rs: -------------------------------------------------------------------------------- 1 | //! Re-routes the output of the Generator's stream through another stream 2 | 3 | use std::thread; 4 | use std::sync::mpsc; 5 | 6 | use futures::sync::mpsc::UnboundedSender; 7 | 8 | use trading::tick::Tick; 9 | use transport::tickstream::TickSink; 10 | 11 | pub struct StreamSink { 12 | mpsc_tx: mpsc::Sender, 13 | } 14 | 15 | // We map the input stream into a MPSC channel which then sends them through a different 16 | // futures channel due to the fact that futures-rs is terrible but we're WAY too 17 | // commmitted to go back now. 18 | impl StreamSink { 19 | #[allow(unused_variables)] 20 | pub fn new(symbol: String, dst_tx: UnboundedSender) -> StreamSink { 21 | let (tx, rx) = mpsc::channel::(); 22 | thread::spawn(move || { 23 | let dst_tx = dst_tx; 24 | for t in rx.iter() { 25 | dst_tx.send(t).expect("Unable to send tick to sink in stream_sink.rs"); 26 | } 27 | }); 28 | 29 | StreamSink { 30 | mpsc_tx: tx, 31 | } 32 | } 33 | } 34 | 35 | impl TickSink for StreamSink { 36 | fn tick(&mut self, t: Tick) { 37 | self.mpsc_tx.send(t).unwrap(); 38 | } 39 | } 40 | --------------------------------------------------------------------------------