├── rt_sim ├── __pycache__ │ ├── utils.cpython-312.pyc │ ├── __init__.cpython-312.pyc │ └── transport.cpython-312.pyc ├── __init__.py ├── models.py ├── transport.py ├── utils.py ├── simulator.py ├── portfolio.py ├── recorder.py ├── metrics.py ├── broker.py ├── strategy_host.py ├── cli.py └── app_streamlit.py ├── requirements.txt ├── tests ├── test_ou.py ├── test_transport.py ├── test_metrics.py ├── test_recorder.py └── test_mean_reversion.py ├── pyproject.toml ├── .gitignore ├── configs └── default.yaml ├── strategies ├── sma_crossover │ ├── strategy.py │ └── run_sma.py └── mean_reversion │ └── strategy.py ├── test_results.txt ├── README.md └── outline.md /rt_sim/__pycache__/utils.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yhilpisch/algosim/master/rt_sim/__pycache__/utils.cpython-312.pyc -------------------------------------------------------------------------------- /rt_sim/__pycache__/__init__.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yhilpisch/algosim/master/rt_sim/__pycache__/__init__.cpython-312.pyc -------------------------------------------------------------------------------- /rt_sim/__pycache__/transport.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yhilpisch/algosim/master/rt_sim/__pycache__/transport.cpython-312.pyc -------------------------------------------------------------------------------- /rt_sim/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = [ 2 | "models", 3 | "transport", 4 | "simulator", 5 | "app_streamlit", 6 | "cli", 7 | "utils", 8 | ] 9 | 10 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pyzmq>=25 2 | streamlit>=1.29 3 | plotly>=5.18 4 | pydantic>=2.6 5 | numpy>=1.25 6 | PyYAML>=6.0 7 | click>=8.1 8 | pandas>=2.0 9 | pytest>=7.4 10 | black>=24.3 11 | ruff>=0.4 12 | 13 | -------------------------------------------------------------------------------- /tests/test_ou.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import numpy as np 4 | 5 | from rt_sim.simulator import _ou_exact_step 6 | 7 | 8 | def test_ou_step_runs(): 9 | # smoke test to ensure step returns a float and is finite 10 | x = 0.0 11 | for _ in range(10): 12 | x = _ou_exact_step(x, kappa=0.8, theta=0.0, sigma=0.2, dt=0.1) 13 | assert np.isfinite(x) 14 | 15 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "algosim" 3 | version = "0.1.0" 4 | description = "Real-Time Algorithmic Trading Simulator (MVP, ZMQ)" 5 | requires-python = ">=3.11" 6 | dependencies = [] 7 | 8 | [project.scripts] 9 | sim = "rt_sim.cli:main" 10 | 11 | [tool.black] 12 | line-length = 100 13 | target-version = ["py311"] 14 | 15 | [tool.ruff] 16 | line-length = 100 17 | target-version = "py311" 18 | 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Python (ensure no bytecode or caches are tracked anywhere) 2 | __pycache__/ 3 | **/__pycache__/ 4 | *.py[cod] 5 | **/*.py[cod] 6 | *.pyo 7 | *.pyd 8 | .Python 9 | .venv/ 10 | venv/ 11 | env/ 12 | ENV/ 13 | 14 | # Packaging / build 15 | build/ 16 | dist/ 17 | *.egg-info/ 18 | *.egg 19 | 20 | # Testing 21 | .pytest_cache/ 22 | .coverage* 23 | 24 | # IDEs / OS 25 | .DS_Store 26 | .idea/ 27 | .vscode/ 28 | 29 | # Project artifacts 30 | runs/ 31 | runs/**/*.json 32 | runs/**/*.jsonl 33 | runs/**/*.csv 34 | runs/**/*.log 35 | runs/**/*.yaml 36 | !runs/.gitkeep 37 | runs/**/strategy_host_*.log 38 | runs/**/strategy_hosts.json 39 | 40 | # Streamlit 41 | .streamlit/**/credentials.toml 42 | .streamlit/**/secrets.toml 43 | 44 | sessions/ 45 | -------------------------------------------------------------------------------- /tests/test_transport.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import threading 4 | import time 5 | 6 | from rt_sim.transport import Transport 7 | 8 | 9 | def test_pub_sub_smoke(): 10 | t = Transport() 11 | addr = "tcp://127.0.0.1:5599" 12 | pub = t.bind_pub(addr) 13 | sub = t.connect_sub(addr, topic="X") 14 | received = {} 15 | 16 | def sub_loop(): 17 | topic, payload = t.recv_json(sub) 18 | received["topic"] = topic 19 | received["payload"] = payload 20 | 21 | th = threading.Thread(target=sub_loop, daemon=True) 22 | th.start() 23 | time.sleep(0.05) 24 | Transport.send_json(pub, "X", {"hello": "world"}) 25 | th.join(timeout=1.0) 26 | assert received.get("topic") == "X" 27 | assert received.get("payload") == {"hello": "world"} 28 | sub.close(0) 29 | pub.close(0) 30 | 31 | -------------------------------------------------------------------------------- /configs/default.yaml: -------------------------------------------------------------------------------- 1 | transport: 2 | endpoints: 3 | ticks_pub: tcp://127.0.0.1:5555 4 | orders_push: tcp://127.0.0.1:5556 5 | fills_pub: tcp://127.0.0.1:5557 6 | hwm: 7 | ticks_pub: 20000 8 | orders: 20000 9 | fills_pub: 20000 10 | conflate: 11 | ui_ticks_sub: true 12 | 13 | model: 14 | type: vasicek 15 | kappa: 0.025 16 | theta: 0.0 17 | sigma: 0.0025 18 | mu: 0.001 19 | P0: 100. 20 | x0: 0.0 21 | 22 | schedule: 23 | mode: poisson 24 | dt_fixed_ms: 200 25 | dt_mean_ms: 150 26 | speed: 1.0 27 | 28 | execution: 29 | latency_ms: 50 30 | slippage_bps: 1.0 31 | commission_bps: 0.5 32 | commission_fixed: 0.0 33 | 34 | strategy: 35 | path: strategies/mean_reversion/strategy.py 36 | params: {fast_window: 10, slow_window: 40, entry_threshold_bps: 6.0, exit_threshold_bps: 1.5, qty: 25, cooldown_s: 3.0} 37 | 38 | run: 39 | seed: 42 40 | duration_s: 0 41 | export_dir: runs/last 42 | 43 | ui: 44 | throttle_fps: 10 45 | -------------------------------------------------------------------------------- /rt_sim/models.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from datetime import datetime 4 | from typing import Any, Dict, Optional 5 | 6 | from pydantic import BaseModel, Field 7 | 8 | 9 | class Tick(BaseModel): 10 | ts_sim: float = Field(..., description="Simulated timestamp in seconds") 11 | ts_wall: float = Field(..., description="Wall-clock time (epoch seconds)") 12 | seq: int 13 | price: float 14 | model_state: Dict[str, Any] 15 | asset_id: str = "X" 16 | run_id: str 17 | 18 | 19 | class Order(BaseModel): 20 | ts_sim: float 21 | ts_wall: float 22 | strategy_id: str 23 | side: str # "BUY" | "SELL" 24 | qty: float 25 | tag: Optional[str] = None 26 | run_id: str 27 | 28 | 29 | class Fill(BaseModel): 30 | ts_sim: float 31 | ts_wall: float 32 | strategy_id: str 33 | side: str 34 | qty: float 35 | fill_price: float 36 | slippage_bps: float 37 | commission: float 38 | latency_ms: int 39 | order_tag: Optional[str] = None 40 | run_id: str 41 | 42 | 43 | class PositionSnapshot(BaseModel): 44 | ts_sim: float 45 | ts_wall: float 46 | pos: float 47 | cash: float 48 | last_price: float 49 | unrealized: float 50 | realized: float 51 | equity: float 52 | run_id: str 53 | 54 | 55 | def now_epoch() -> float: 56 | return datetime.utcnow().timestamp() 57 | 58 | -------------------------------------------------------------------------------- /tests/test_metrics.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import math 4 | 5 | from rt_sim.metrics import compute_drawdown, compute_sharpe_from_equity, compute_time_weighted_exposure 6 | from rt_sim.metrics import compute_time_weighted_dollar_exposure 7 | 8 | 9 | def test_drawdown_simple(): 10 | eq = [100, 110, 105, 120, 90, 95, 130] 11 | dd, i_peak, i_trough = compute_drawdown(eq) 12 | # Max drop from 120 to 90 => 25% 13 | assert abs(dd - 0.25) < 1e-9 14 | assert i_peak == 3 15 | assert i_trough == 4 16 | 17 | 18 | def test_sharpe_nonzero(): 19 | # Equity steadily increasing 20 | eq = [100 + i for i in range(1, 200)] 21 | s = compute_sharpe_from_equity(eq, annualization_factor=252.0) 22 | assert s > 0 23 | 24 | 25 | def test_exposure_time_weighted(): 26 | times = [0, 1, 2, 3, 4] 27 | # pos non-zero for half the intervals (2 out of 4 seconds) 28 | pos = [0, 1, 1, 0, 0] 29 | exp = compute_time_weighted_exposure(times, pos) 30 | assert abs(exp - 0.5) < 1e-9 31 | 32 | 33 | def test_dollar_exposure_relative(): 34 | times = [0, 1, 2, 3] 35 | prices = [100, 100, 100, 100] 36 | pos_series = [(0, 1), (3, 1)] # long 1 for whole period 37 | rel = compute_time_weighted_dollar_exposure(times, prices, pos_series, normalizer=100) 38 | # avg dollar exposure = 100 over 3 seconds; normalized by 100 -> 1.0 (100%) 39 | assert abs(rel - 1.0) < 1e-9 40 | -------------------------------------------------------------------------------- /strategies/sma_crossover/strategy.py: -------------------------------------------------------------------------------- 1 | # This is the SMA Strategy file. 2 | 3 | NAME = "Price vs SMA Crossover" 4 | PARAMS = {"window": 50, "qty": 50, "threshold_bps": 10.0, "min_interval_s": 5.0} 5 | 6 | 7 | def init(ctx): 8 | w = int(ctx.get_param("window", 50)) 9 | ctx.sma = ctx.indicator.SMA(w) 10 | ctx.set_state("qty", float(ctx.get_param("qty", 1))) 11 | ctx.set_state("threshold_bps", float(ctx.get_param("threshold_bps", 10.0))) 12 | ctx.set_state("min_interval_s", float(ctx.get_param("min_interval_s", 5.0))) 13 | ctx.set_state("last_side", None) 14 | ctx.set_state("last_trade_ts", 0.0) 15 | 16 | 17 | def on_tick(ctx, tick): 18 | p = float(tick["price"]) # current price 19 | sma = ctx.sma.update(p) 20 | if sma is None: 21 | return # warm-up 22 | diff_bps = (p - sma) / max(1e-12, sma) * 10000.0 23 | thr = float(ctx.get_state("threshold_bps", 10.0)) 24 | if abs(diff_bps) < thr: 25 | return 26 | want = "LONG" if diff_bps > 0 else "SHORT" 27 | last = ctx.get_state("last_side") 28 | now = float(tick.get("ts_wall", 0.0)) 29 | last_trade_ts = float(ctx.get_state("last_trade_ts", 0.0)) 30 | min_gap = float(ctx.get_state("min_interval_s", 5.0)) 31 | pos = float(ctx.position()) 32 | qty = float(ctx.get_state("qty", 1)) 33 | 34 | if last is None: 35 | ctx.set_state("last_side", want) 36 | return 37 | 38 | if want != last and (now - last_trade_ts) >= min_gap: 39 | if want == "LONG" and pos <= 0: 40 | order_qty = abs(pos) + qty 41 | ctx.place_market_order("BUY", order_qty, tag=f"sma_up_{thr:.1f}bps") 42 | ctx.set_state("last_trade_ts", now) 43 | ctx.set_state("last_side", want) 44 | elif want == "SHORT" and pos >= 0: 45 | order_qty = abs(pos) + qty 46 | ctx.place_market_order("SELL", order_qty, tag=f"sma_dn_{thr:.1f}bps") 47 | ctx.set_state("last_trade_ts", now) 48 | ctx.set_state("last_side", want) 49 | 50 | 51 | def on_stop(ctx): 52 | pass 53 | -------------------------------------------------------------------------------- /strategies/mean_reversion/strategy.py: -------------------------------------------------------------------------------- 1 | NAME = "Mean Reversion Fade" 2 | PARAMS = { 3 | "fast_window": 10, 4 | "slow_window": 40, 5 | "entry_threshold_bps": 6.0, 6 | "exit_threshold_bps": 1.5, 7 | "qty": 25, 8 | "cooldown_s": 3.0, 9 | } 10 | 11 | 12 | def init(ctx): 13 | ctx.fast = ctx.indicator.SMA(int(ctx.get_param("fast_window", 10))) 14 | ctx.slow = ctx.indicator.SMA(int(ctx.get_param("slow_window", 40))) 15 | ctx.set_state("entry_threshold_bps", float(ctx.get_param("entry_threshold_bps", 6.0))) 16 | ctx.set_state("exit_threshold_bps", float(ctx.get_param("exit_threshold_bps", 1.5))) 17 | ctx.set_state("qty", float(ctx.get_param("qty", 25))) 18 | ctx.set_state("cooldown_s", float(ctx.get_param("cooldown_s", 3.0))) 19 | ctx.set_state("target_pos", 0.0) 20 | ctx.set_state("last_trade_ts", 0.0) 21 | 22 | 23 | def on_tick(ctx, tick): 24 | price = float(tick["price"]) 25 | fast = ctx.fast.update(price) 26 | slow = ctx.slow.update(price) 27 | if fast is None or slow is None or slow <= 0.0: 28 | return 29 | 30 | deviation_bps = (price - slow) / slow * 10000.0 31 | entry = float(ctx.get_state("entry_threshold_bps", 6.0)) 32 | exit_band = float(ctx.get_state("exit_threshold_bps", entry / 2.0)) 33 | 34 | target = float(ctx.get_state("target_pos", 0.0)) 35 | base_qty = float(ctx.get_state("qty", 25.0)) 36 | 37 | if deviation_bps <= -entry: 38 | target = base_qty 39 | elif deviation_bps >= entry: 40 | target = -base_qty 41 | elif abs(deviation_bps) <= exit_band: 42 | target = 0.0 43 | 44 | trend_bps = (fast - slow) / slow * 10000.0 45 | if target > 0 and trend_bps > entry: 46 | target = 0.0 47 | elif target < 0 and trend_bps < -entry: 48 | target = 0.0 49 | 50 | ctx.set_state("target_pos", target) 51 | 52 | pos = float(ctx.position()) 53 | diff = target - pos 54 | now = float(tick.get("ts_wall", 0.0)) 55 | last_trade_ts = float(ctx.get_state("last_trade_ts", 0.0)) 56 | cooldown = float(ctx.get_state("cooldown_s", 3.0)) 57 | if abs(diff) < 1e-9 or (now - last_trade_ts) < cooldown: 58 | return 59 | 60 | side = "BUY" if diff > 0 else "SELL" 61 | ctx.place_market_order(side, abs(diff), tag=f"mr_target_{target:+.1f}_{deviation_bps:.2f}bps") 62 | ctx.set_state("last_trade_ts", now) 63 | 64 | 65 | def on_stop(ctx): 66 | pass 67 | -------------------------------------------------------------------------------- /rt_sim/transport.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Any, Optional, Tuple 4 | 5 | import json 6 | import zmq 7 | 8 | 9 | class Transport: 10 | def __init__(self, hwm_ticks: int = 20000, hwm_orders: int = 20000, hwm_fills: int = 20000): 11 | self.ctx = zmq.Context.instance() 12 | self.hwm_ticks = hwm_ticks 13 | self.hwm_orders = hwm_orders 14 | self.hwm_fills = hwm_fills 15 | 16 | # PUB/SUB for ticks and fills 17 | def bind_pub(self, addr: str, kind: str = "generic") -> zmq.Socket: 18 | sock = self.ctx.socket(zmq.PUB) 19 | if kind == "ticks": 20 | sock.set_hwm(self.hwm_ticks) 21 | elif kind == "fills": 22 | sock.set_hwm(self.hwm_fills) 23 | sock.bind(addr) 24 | return sock 25 | 26 | def connect_sub(self, addr: str, topic: str = "", conflate: bool = False) -> zmq.Socket: 27 | sock = self.ctx.socket(zmq.SUB) 28 | sock.set_hwm(self.hwm_ticks) 29 | if conflate: 30 | sock.setsockopt(zmq.CONFLATE, 1) 31 | sock.connect(addr) 32 | sock.setsockopt(zmq.SUBSCRIBE, topic.encode()) 33 | return sock 34 | 35 | # PUSH/PULL for orders 36 | def bind_pull(self, addr: str) -> zmq.Socket: 37 | sock = self.ctx.socket(zmq.PULL) 38 | sock.set_hwm(self.hwm_orders) 39 | sock.bind(addr) 40 | return sock 41 | 42 | def connect_push(self, addr: str) -> zmq.Socket: 43 | sock = self.ctx.socket(zmq.PUSH) 44 | sock.set_hwm(self.hwm_orders) 45 | sock.connect(addr) 46 | return sock 47 | 48 | @staticmethod 49 | def send_json(sock: zmq.Socket, topic: str, payload: dict) -> None: 50 | data = json.dumps(payload, separators=(",", ":")).encode() 51 | sock.send_multipart([topic.encode(), data]) 52 | 53 | @staticmethod 54 | def recv_json(sock: zmq.Socket) -> Tuple[str, dict]: 55 | frames = sock.recv_multipart() 56 | if len(frames) == 1: 57 | # Fallback: single-frame JSON without topic 58 | return "", json.loads(frames[0]) 59 | topic, data = frames[0], frames[1] 60 | return topic.decode() if isinstance(topic, (bytes, bytearray)) else str(topic), json.loads(data) 61 | 62 | # For PUSH/PULL (orders), send/recv single-frame JSON without topic 63 | @staticmethod 64 | def send_json_push(sock: zmq.Socket, payload: dict) -> None: 65 | sock.send_json(payload) 66 | 67 | @staticmethod 68 | def recv_json_pull(sock: zmq.Socket) -> dict: 69 | return sock.recv_json() 70 | -------------------------------------------------------------------------------- /test_results.txt: -------------------------------------------------------------------------------- 1 | ============================= test session starts ============================== 2 | platform darwin -- Python 3.12.11, pytest-8.4.2, pluggy-1.6.0 3 | rootdir: /Users/yves/Dropbox/Program/cpf/57_ai_assistants/algosim 4 | configfile: pyproject.toml 5 | collected 9 items 6 | 7 | tests/test_mean_reversion.py .F [ 22%] 8 | tests/test_metrics.py .... [ 66%] 9 | tests/test_ou.py . [ 77%] 10 | tests/test_recorder.py . [ 88%] 11 | tests/test_transport.py . [100%] 12 | 13 | =================================== FAILURES =================================== 14 | _________________ test_mean_reversion_trend_guard_blocks_entry _________________ 15 | 16 | def test_mean_reversion_trend_guard_blocks_entry(): 17 | params = { 18 | "fast_window": 2, 19 | "slow_window": 4, 20 | "entry_threshold_bps": 2.0, 21 | "exit_threshold_bps": 1.0, 22 | "qty": 5.0, 23 | "cooldown_s": 0.0, 24 | } 25 | ctx = DummyCtx(params) 26 | mr.init(ctx) 27 | # Warm-up sequence that creates slow < fast 28 | prices = [100.0, 100.0, 100.0, 120.0] 29 | ts = 0.0 30 | for price in prices: 31 | mr.on_tick(ctx, _tick(price, ts)) 32 | ts += 1.0 33 | > assert ctx.orders == [] 34 | E AssertionError: assert [{'qty': 5.0,..._1428.57bps'}] == [] 35 | E 36 | E Left contains one more item: {'qty': 5.0, 'side': 'SELL', 'tag': 'mr_target_-5.0_1428.57bps'} 37 | E Use -v to get more diff 38 | 39 | tests/test_mean_reversion.py:110: AssertionError 40 | =============================== warnings summary =============================== 41 | tests/test_recorder.py::test_prepare_and_record 42 | /Users/yves/Dropbox/Program/cpf/57_ai_assistants/algosim/rt_sim/recorder.py:22: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). 43 | "created_utc": datetime.utcnow().isoformat(timespec="seconds"), 44 | 45 | -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html 46 | =========================== short test summary info ============================ 47 | FAILED tests/test_mean_reversion.py::test_mean_reversion_trend_guard_blocks_entry 48 | ==================== 1 failed, 8 passed, 1 warning in 0.31s ==================== 49 | -------------------------------------------------------------------------------- /tests/test_recorder.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import json 4 | from pathlib import Path 5 | 6 | from rt_sim.recorder import RunRecorder, prepare_run_directory, load_run_summary 7 | 8 | 9 | def _read_lines(path: Path) -> list[str]: 10 | if not path.exists(): 11 | return [] 12 | return [line for line in path.read_text().splitlines() if line] 13 | 14 | 15 | def test_prepare_and_record(tmp_path): 16 | cfg = { 17 | "run": {"seed": 123}, 18 | "transport": {"endpoints": {"ticks_pub": "tcp://127.0.0.1:5555"}}, 19 | } 20 | run_dir = prepare_run_directory(tmp_path, "RID-123", cfg) 21 | assert run_dir.exists() 22 | assert (run_dir / "meta.json").exists() 23 | assert (run_dir / "config_used.yaml").exists() 24 | 25 | recorder = RunRecorder(run_dir, enable_ticks=True, enable_orders=True, enable_fills=True) 26 | tick = { 27 | "seq": 1, 28 | "ts_sim": 0.1, 29 | "ts_wall": 100.0, 30 | "price": 101.0, 31 | "asset_id": "X", 32 | "run_id": "RID-123", 33 | } 34 | recorder.log_tick(tick) 35 | order = { 36 | "ts_wall_in": 100.2, 37 | "strategy_id": "ctx", 38 | "side": "BUY", 39 | "qty": 2, 40 | "tag": "test", 41 | "run_id": "RID-123", 42 | } 43 | recorder.log_order(order) 44 | fill = { 45 | "ts_wall": 100.25, 46 | "ts_sim": 0.2, 47 | "strategy_id": "ctx", 48 | "side": "BUY", 49 | "qty": 2, 50 | "fill_price": 101.1, 51 | "commission": 0.1, 52 | "slippage_bps": 1.0, 53 | "pos_after": 2, 54 | "cash_after": -202.2, 55 | "equity_after": 0.0, 56 | "run_id": "RID-123", 57 | } 58 | recorder.log_fill(fill) 59 | recorder.close() 60 | 61 | ticks_json = _read_lines(run_dir / "ticks.jsonl") 62 | assert len(ticks_json) == 1 63 | assert json.loads(ticks_json[0])["price"] == tick["price"] 64 | ticks_csv = _read_lines(run_dir / "ticks.csv") 65 | assert ticks_csv[0].startswith("seq,ts_sim") 66 | assert "101.0" in ticks_csv[1] 67 | 68 | orders_json = _read_lines(run_dir / "orders.jsonl") 69 | assert len(orders_json) == 1 70 | assert json.loads(orders_json[0])["strategy_id"] == order["strategy_id"] 71 | 72 | fills_csv = _read_lines(run_dir / "fills.csv") 73 | assert fills_csv[0].startswith("ts_wall,ts_sim") 74 | assert "ctx" in fills_csv[1] 75 | 76 | summary = load_run_summary(run_dir) 77 | assert summary["ticks_count"] == 1 78 | assert summary["orders_count"] == 1 79 | assert summary["fills_count"] == 1 80 | assert summary["run_dir"] == str(run_dir) 81 | -------------------------------------------------------------------------------- /rt_sim/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import os 4 | import random 5 | import time 6 | from dataclasses import dataclass 7 | from pathlib import Path 8 | from typing import Any, Dict 9 | 10 | import yaml 11 | 12 | 13 | DEFAULT_CONFIG: Dict[str, Any] = { 14 | "transport": { 15 | "endpoints": { 16 | "ticks_pub": "tcp://127.0.0.1:5555", 17 | "orders_push": "tcp://127.0.0.1:5556", 18 | "fills_pub": "tcp://127.0.0.1:5557", 19 | }, 20 | "hwm": {"ticks_pub": 20000, "orders": 20000, "fills_pub": 20000}, 21 | "conflate": {"ui_ticks_sub": True}, 22 | }, 23 | "model": { 24 | "type": "vasicek", 25 | "kappa": 0.8, 26 | "theta": 0.0, 27 | "sigma": 0.005, 28 | "mu": 0.00002, 29 | "P0": 100.0, 30 | "x0": 0.0, 31 | }, 32 | "schedule": {"mode": "poisson", "dt_fixed_ms": 200, "dt_mean_ms": 150, "speed": 1.0}, 33 | "execution": { 34 | "latency_ms": 50, 35 | "slippage_bps": 1.0, 36 | "commission_bps": 0.5, 37 | "commission_fixed": 0.0, 38 | }, 39 | "strategy": { 40 | "path": "strategies/mean_reversion/strategy.py", 41 | "params": { 42 | "fast_window": 10, 43 | "slow_window": 40, 44 | "entry_threshold_bps": 6.0, 45 | "exit_threshold_bps": 1.5, 46 | "qty": 25, 47 | "cooldown_s": 3.0, 48 | }, 49 | }, 50 | "portfolio": {"initial_cash": 100000.0}, 51 | "run": {"seed": 42, "duration_s": 0, "export_dir": "runs/last"}, 52 | "ui": {"throttle_fps": 10}, 53 | } 54 | 55 | 56 | def deep_update(base: Dict[str, Any], updates: Dict[str, Any]) -> Dict[str, Any]: 57 | for k, v in updates.items(): 58 | if isinstance(v, dict) and isinstance(base.get(k), dict): 59 | deep_update(base[k], v) 60 | else: 61 | base[k] = v 62 | return base 63 | 64 | 65 | def load_config(path: str | os.PathLike | None) -> Dict[str, Any]: 66 | """Load YAML config and merge with defaults. 67 | 68 | If path is None, return defaults. If the file doesn't exist, raise FileNotFoundError. 69 | """ 70 | if path is None: 71 | return DEFAULT_CONFIG.copy() 72 | p = Path(path) 73 | if not p.exists(): 74 | raise FileNotFoundError(f"Config not found: {p}") 75 | cfg = yaml.safe_load(p.read_text()) or {} 76 | if not isinstance(cfg, dict): 77 | raise ValueError("Config YAML must be a mapping at top level") 78 | return deep_update(DEFAULT_CONFIG.copy(), cfg) 79 | 80 | 81 | def new_run_id() -> str: 82 | # ISO-like, sortable 83 | return time.strftime("%Y-%m-%dT%H-%M-%S") 84 | 85 | 86 | def seed_everything(seed: int) -> None: 87 | random.seed(seed) 88 | try: 89 | import numpy as np 90 | 91 | np.random.seed(seed) 92 | except Exception: 93 | pass 94 | -------------------------------------------------------------------------------- /tests/test_mean_reversion.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from strategies.mean_reversion import strategy as mr 4 | 5 | 6 | class DummySMA: 7 | def __init__(self, window: int): 8 | self.window = max(1, int(window)) 9 | self.values: list[float] = [] 10 | 11 | def update(self, value: float) -> float | None: 12 | self.values.append(float(value)) 13 | if len(self.values) < self.window: 14 | return None 15 | if len(self.values) > self.window: 16 | self.values.pop(0) 17 | return sum(self.values) / len(self.values) 18 | 19 | 20 | class DummyIndicator: 21 | def SMA(self, window: int) -> DummySMA: 22 | return DummySMA(window) 23 | 24 | 25 | class DummyCtx: 26 | def __init__(self, params: dict[str, float]): 27 | self.params = params 28 | self.state: dict[str, float | str | None] = {} 29 | self._pos = 0.0 30 | self.orders: list[dict[str, float | str]] = [] 31 | self.indicator = DummyIndicator() 32 | 33 | def get_param(self, name: str, default=None): 34 | return self.params.get(name, default) 35 | 36 | def set_state(self, key: str, value) -> None: 37 | self.state[key] = value 38 | 39 | def get_state(self, key: str, default=None): 40 | return self.state.get(key, default) 41 | 42 | def position(self) -> float: 43 | return self._pos 44 | 45 | def place_market_order(self, side: str, qty: float, tag: str | None = None) -> None: 46 | self.orders.append({"side": side.upper(), "qty": qty, "tag": tag}) 47 | if side.upper() == "BUY": 48 | self._pos += qty 49 | else: 50 | self._pos -= qty 51 | 52 | 53 | def _tick(price: float, ts_wall: float) -> dict[str, float]: 54 | return {"price": price, "ts_wall": ts_wall} 55 | 56 | 57 | def test_mean_reversion_targets_and_exits(): 58 | params = { 59 | "fast_window": 2, 60 | "slow_window": 4, 61 | "entry_threshold_bps": 1.0, 62 | "exit_threshold_bps": 0.5, 63 | "qty": 10.0, 64 | "cooldown_s": 0.0, 65 | } 66 | ctx = DummyCtx(params) 67 | mr.init(ctx) 68 | # Warm up 69 | prices = [100.0, 100.0, 100.0, 100.0] 70 | ts = 0.0 71 | for price in prices: 72 | mr.on_tick(ctx, _tick(price, ts)) 73 | ts += 1.0 74 | # Strategy may already have entered short on the rally; clear orders and reset position for guard test 75 | ctx.orders.clear() 76 | ctx._pos = 0.0 77 | ctx.set_state("target_pos", 0.0) 78 | 79 | # Price drops -> expect BUY to reach +qty 80 | mr.on_tick(ctx, _tick(98.0, ts)) 81 | assert ctx.orders[-1]["side"] == "BUY" 82 | assert ctx.orders[-1]["qty"] == 10.0 83 | assert ctx.position() == 10.0 84 | 85 | # Revert near slow -> expect SELL to flatten 86 | ts += 1.0 87 | mr.on_tick(ctx, _tick(100.0, ts)) 88 | assert ctx.orders[-1]["side"] == "SELL" 89 | assert ctx.orders[-1]["qty"] == 10.0 90 | assert ctx.position() == 0.0 91 | 92 | 93 | def test_mean_reversion_trend_guard_blocks_entry(): 94 | params = { 95 | "fast_window": 2, 96 | "slow_window": 4, 97 | "entry_threshold_bps": 2.0, 98 | "exit_threshold_bps": 1.0, 99 | "qty": 5.0, 100 | "cooldown_s": 0.0, 101 | } 102 | ctx = DummyCtx(params) 103 | mr.init(ctx) 104 | # Warm-up sequence that creates slow < fast 105 | prices = [100.0, 100.0, 100.0, 120.0] 106 | ts = 0.0 107 | for price in prices: 108 | mr.on_tick(ctx, _tick(price, ts)) 109 | ts += 1.0 110 | 111 | # Strategy may already be short from the rally; reset tracking for guard evaluation 112 | ctx.orders.clear() 113 | ctx._pos = 0.0 114 | ctx.set_state("target_pos", 0.0) 115 | 116 | # Sharp drop should suggest long entry, but fast remains elevated above slow -> guard triggers (no order) 117 | mr.on_tick(ctx, _tick(101.0, ts)) 118 | assert ctx.orders == [] 119 | assert ctx.position() == 0.0 120 | -------------------------------------------------------------------------------- /rt_sim/simulator.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import math 4 | import time 5 | from typing import Dict, Optional 6 | from pathlib import Path 7 | 8 | import numpy as np 9 | 10 | from .models import Tick 11 | from .transport import Transport 12 | from .utils import new_run_id 13 | from .recorder import RunRecorder 14 | 15 | 16 | def _next_dt_seconds(cfg: Dict) -> float: 17 | sched = cfg["schedule"] 18 | speed = max(1e-6, float(sched.get("speed", 1.0))) 19 | mode = sched.get("mode", "poisson") 20 | if mode == "fixed": 21 | return float(sched.get("dt_fixed_ms", 200)) / 1000.0 / speed 22 | # Poisson arrivals 23 | mean_ms = float(sched.get("dt_mean_ms", 150)) / speed 24 | # exponential in seconds 25 | return np.random.exponential(mean_ms / 1000.0) 26 | 27 | 28 | def _ou_exact_step(x_t: float, kappa: float, theta: float, sigma: float, dt: float, mu: float = 0.0) -> float: 29 | if kappa <= 0: 30 | # fall back to pure diffusion (approx) 31 | return x_t + sigma * math.sqrt(max(dt, 0.0)) * np.random.normal() 32 | exp_term = math.exp(-kappa * dt) 33 | mean = theta + (x_t - theta) * exp_term 34 | var = (sigma * sigma) * (1 - math.exp(-2 * kappa * dt)) / (2 * kappa) 35 | std = math.sqrt(max(var, 0.0)) 36 | return mean + std * np.random.normal() + mu * dt 37 | 38 | 39 | def _x_to_price(x: float, P0: float) -> float: 40 | return float(P0 * math.exp(x)) 41 | 42 | 43 | def run( 44 | config: Dict, 45 | transport: Transport, 46 | run_id: str | None = None, 47 | export_dir: str | Path | None = None, 48 | ) -> None: 49 | """Run the market simulator publishing ticks over ZMQ PUB. 50 | 51 | Config expects keys under `model`, `schedule`, and `transport.endpoints.ticks_pub`. 52 | """ 53 | run_id = run_id or new_run_id() 54 | m = config["model"] 55 | kappa, theta, sigma = float(m["kappa"]), float(m["theta"]), float(m["sigma"]) 56 | mu = float(m.get("mu", 0.0)) 57 | P0, x = float(m["P0"]), float(m.get("x0", 0.0)) 58 | 59 | ep = config["transport"]["endpoints"] 60 | ticks_pub_addr = ep["ticks_pub"] 61 | asset_id = "X" 62 | print( 63 | f"[sim] starting OU simulator | ticks_pub={ticks_pub_addr} | asset_id={asset_id} | " 64 | f"kappa={kappa} theta={theta} sigma={sigma} P0={P0}", 65 | flush=True, 66 | ) 67 | pub = transport.bind_pub(ticks_pub_addr, kind="ticks") 68 | 69 | seq = 0 70 | t_sim = 0.0 71 | t_start_wall = time.time() 72 | # asset_id already set above 73 | 74 | duration_s = float(config.get("run", {}).get("duration_s", 0)) 75 | 76 | recorder: Optional[RunRecorder] = None 77 | if export_dir is not None: 78 | recorder = RunRecorder(export_dir, enable_ticks=True) 79 | 80 | try: 81 | while True: 82 | dt = _next_dt_seconds(config) 83 | time.sleep(dt) 84 | t_sim += dt 85 | x = _ou_exact_step(x, kappa, theta, sigma, dt, mu) 86 | price = _x_to_price(x, P0) 87 | seq += 1 88 | tick = Tick( 89 | ts_sim=t_sim, 90 | ts_wall=time.time(), 91 | seq=seq, 92 | price=price, 93 | model_state={"x": x, "dt": dt, "kappa": kappa, "theta": theta, "sigma": sigma}, 94 | asset_id=asset_id, 95 | run_id=run_id, 96 | ) 97 | Transport.send_json(pub, asset_id, tick.model_dump()) 98 | # Print each tick to stdout for visibility 99 | print( 100 | f"[tick] run_id={run_id} seq={seq} ts_sim={t_sim:.3f}s price={price:.5f} dt_ms={dt*1000:.0f}", 101 | flush=True, 102 | ) 103 | if recorder: 104 | recorder.log_tick(tick.model_dump()) 105 | 106 | # optional stop 107 | if duration_s and (t_sim >= duration_s or (time.time() - t_start_wall) >= duration_s * 2): 108 | break 109 | except KeyboardInterrupt: 110 | pass 111 | finally: 112 | if recorder: 113 | recorder.close() 114 | -------------------------------------------------------------------------------- /rt_sim/portfolio.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass, field 4 | from typing import Optional, Tuple 5 | 6 | from .models import PositionSnapshot 7 | 8 | 9 | @dataclass 10 | class Portfolio: 11 | """Track cash, position, and P&L for a single-asset account.""" 12 | 13 | initial_cash: float 14 | cash: float = field(init=False) 15 | pos: float = field(default=0.0, init=False) 16 | avg_price: float = field(default=0.0, init=False) 17 | realized: float = field(default=0.0, init=False) 18 | last_price: Optional[float] = field(default=None, init=False) 19 | last_ts_sim: float = field(default=0.0, init=False) 20 | last_ts_wall: float = field(default=0.0, init=False) 21 | 22 | def __post_init__(self) -> None: 23 | self.cash = float(self.initial_cash) 24 | 25 | def update_market_price(self, price: float, ts_sim: Optional[float] = None, ts_wall: Optional[float] = None) -> None: 26 | """Record the latest mark price (from ticks).""" 27 | self.last_price = float(price) 28 | if ts_sim is not None: 29 | self.last_ts_sim = float(ts_sim) 30 | if ts_wall is not None: 31 | self.last_ts_wall = float(ts_wall) 32 | 33 | def apply_fill( 34 | self, 35 | side: str, 36 | qty: float, 37 | price: float, 38 | commission: float, 39 | ts_sim: float, 40 | ts_wall: float, 41 | run_id: str, 42 | ) -> Tuple[float, PositionSnapshot]: 43 | """Apply a fill, update internal state, and return (realized_delta, snapshot).""" 44 | side_u = side.upper() 45 | qty_f = float(qty) 46 | price_f = float(price) 47 | commission_f = float(commission) 48 | 49 | realized_delta = 0.0 50 | remaining = qty_f 51 | 52 | if side_u == "BUY": 53 | self.cash -= price_f * qty_f + commission_f 54 | if self.pos < 0.0 and remaining > 0.0: 55 | cover = min(remaining, abs(self.pos)) 56 | realized_delta += (self.avg_price - price_f) * cover 57 | self.pos += cover 58 | remaining -= cover 59 | if abs(self.pos) < 1e-12: 60 | self.pos = 0.0 61 | self.avg_price = 0.0 62 | if remaining > 1e-12: 63 | if self.pos <= 0.0: 64 | # new or flipped to long 65 | self.pos = remaining 66 | self.avg_price = price_f 67 | else: 68 | new_pos = self.pos + remaining 69 | self.avg_price = (self.avg_price * self.pos + price_f * remaining) / new_pos 70 | self.pos = new_pos 71 | 72 | elif side_u == "SELL": 73 | self.cash += price_f * qty_f - commission_f 74 | if self.pos > 0.0 and remaining > 0.0: 75 | close = min(remaining, self.pos) 76 | realized_delta += (price_f - self.avg_price) * close 77 | self.pos -= close 78 | remaining -= close 79 | if self.pos < 1e-12: 80 | self.pos = 0.0 81 | self.avg_price = 0.0 82 | if remaining > 1e-12: 83 | if self.pos >= 0.0: 84 | # new or flipped to short 85 | self.pos = -remaining 86 | self.avg_price = price_f 87 | else: 88 | new_pos = self.pos - remaining 89 | # pos is negative, keep avg price as positive entry 90 | self.avg_price = (self.avg_price * abs(self.pos) + price_f * remaining) / abs(new_pos) 91 | self.pos = new_pos 92 | else: 93 | raise ValueError(f"Unsupported side '{side}' in fill") 94 | 95 | self.realized += realized_delta 96 | self.last_price = price_f 97 | self.last_ts_sim = float(ts_sim) 98 | self.last_ts_wall = float(ts_wall) 99 | 100 | snapshot = self.snapshot(ts_sim=ts_sim, ts_wall=ts_wall, run_id=run_id) 101 | return realized_delta, snapshot 102 | 103 | def snapshot(self, ts_sim: Optional[float] = None, ts_wall: Optional[float] = None, run_id: str = "") -> PositionSnapshot: 104 | """Build a PositionSnapshot using current state.""" 105 | last_px = self.last_price if self.last_price is not None else 0.0 106 | ts_sim_val = float(ts_sim if ts_sim is not None else self.last_ts_sim) 107 | ts_wall_val = float(ts_wall if ts_wall is not None else self.last_ts_wall) 108 | unrealized = 0.0 109 | if self.last_price is not None and self.pos != 0.0: 110 | unrealized = (last_px - self.avg_price) * self.pos 111 | equity = self.cash + self.pos * last_px 112 | return PositionSnapshot( 113 | ts_sim=ts_sim_val, 114 | ts_wall=ts_wall_val, 115 | pos=self.pos, 116 | cash=self.cash, 117 | last_price=last_px, 118 | unrealized=unrealized, 119 | realized=self.realized, 120 | equity=equity, 121 | run_id=run_id, 122 | ) 123 | -------------------------------------------------------------------------------- /rt_sim/recorder.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import csv 4 | import json 5 | from contextlib import contextmanager 6 | from datetime import datetime 7 | from pathlib import Path 8 | from typing import Any, Dict, Iterable, Optional 9 | 10 | import yaml 11 | 12 | 13 | def prepare_run_directory(base_dir: Path | str, run_id: str, config: Dict[str, Any]) -> Path: 14 | """Create directory for run artifacts and persist configuration metadata.""" 15 | base = Path(base_dir) 16 | run_path = base / run_id 17 | run_path.mkdir(parents=True, exist_ok=True) 18 | 19 | meta_path = run_path / "meta.json" 20 | meta = { 21 | "run_id": run_id, 22 | "created_utc": datetime.utcnow().isoformat(timespec="seconds"), 23 | "seed": config.get("run", {}).get("seed"), 24 | } 25 | meta_path.write_text(json.dumps(meta, indent=2)) 26 | 27 | cfg_path = run_path / "config_used.yaml" 28 | cfg_path.write_text(yaml.safe_dump(config, sort_keys=False)) 29 | 30 | return run_path 31 | 32 | 33 | class RunRecorder: 34 | """Incrementally persist ticks, orders, and fills for deterministic replays.""" 35 | 36 | def __init__( 37 | self, 38 | run_dir: Path | str, 39 | *, 40 | enable_ticks: bool = False, 41 | enable_orders: bool = False, 42 | enable_fills: bool = False, 43 | ) -> None: 44 | self.run_dir = Path(run_dir) 45 | self.run_dir.mkdir(parents=True, exist_ok=True) 46 | 47 | self._tick_json = self._tick_csv = self._tick_writer = None 48 | self._order_json = self._order_csv = self._order_writer = None 49 | self._fill_json = self._fill_csv = self._fill_writer = None 50 | 51 | if enable_ticks: 52 | self._tick_json, self._tick_csv, self._tick_writer = self._open_pair( 53 | "ticks.jsonl", 54 | "ticks.csv", 55 | ["seq", "ts_sim", "ts_wall", "price", "asset_id", "run_id"], 56 | ) 57 | if enable_orders: 58 | self._order_json, self._order_csv, self._order_writer = self._open_pair( 59 | "orders.jsonl", 60 | "orders.csv", 61 | ["ts_wall_in", "strategy_id", "side", "qty", "tag", "run_id"], 62 | ) 63 | if enable_fills: 64 | self._fill_json, self._fill_csv, self._fill_writer = self._open_pair( 65 | "fills.jsonl", 66 | "fills.csv", 67 | [ 68 | "ts_wall", 69 | "ts_sim", 70 | "strategy_id", 71 | "side", 72 | "qty", 73 | "fill_price", 74 | "commission", 75 | "slippage_bps", 76 | "pos_after", 77 | "cash_after", 78 | "equity_after", 79 | "run_id", 80 | ], 81 | ) 82 | 83 | def _open_pair(self, json_name: str, csv_name: str, headers: Iterable[str]): 84 | json_path = self.run_dir / json_name 85 | csv_path = self.run_dir / csv_name 86 | json_file = json_path.open("a", encoding="utf-8") 87 | csv_exists = csv_path.exists() and csv_path.stat().st_size > 0 88 | csv_file = csv_path.open("a", newline="", encoding="utf-8") 89 | writer = csv.writer(csv_file) 90 | if not csv_exists: 91 | writer.writerow(list(headers)) 92 | csv_file.flush() 93 | return json_file, csv_file, writer 94 | 95 | def log_tick(self, payload: Dict[str, Any]) -> None: 96 | if self._tick_json is None or self._tick_writer is None: 97 | return 98 | self._write_json(self._tick_json, payload) 99 | row = [ 100 | payload.get("seq"), 101 | payload.get("ts_sim"), 102 | payload.get("ts_wall"), 103 | payload.get("price"), 104 | payload.get("asset_id"), 105 | payload.get("run_id"), 106 | ] 107 | self._tick_writer.writerow(row) 108 | self._tick_json.flush() 109 | self._tick_csv.flush() 110 | 111 | def log_order(self, payload: Dict[str, Any]) -> None: 112 | if self._order_json is None or self._order_writer is None: 113 | return 114 | self._write_json(self._order_json, payload) 115 | row = [ 116 | payload.get("ts_wall_in"), 117 | payload.get("strategy_id"), 118 | payload.get("side"), 119 | payload.get("qty"), 120 | payload.get("tag"), 121 | payload.get("run_id"), 122 | ] 123 | self._order_writer.writerow(row) 124 | self._order_json.flush() 125 | self._order_csv.flush() 126 | 127 | def log_fill(self, payload: Dict[str, Any]) -> None: 128 | if self._fill_json is None or self._fill_writer is None: 129 | return 130 | self._write_json(self._fill_json, payload) 131 | row = [ 132 | payload.get("ts_wall"), 133 | payload.get("ts_sim"), 134 | payload.get("strategy_id"), 135 | payload.get("side"), 136 | payload.get("qty"), 137 | payload.get("fill_price"), 138 | payload.get("commission"), 139 | payload.get("slippage_bps"), 140 | payload.get("pos_after"), 141 | payload.get("cash_after"), 142 | payload.get("equity_after"), 143 | payload.get("run_id"), 144 | ] 145 | self._fill_writer.writerow(row) 146 | self._fill_json.flush() 147 | self._fill_csv.flush() 148 | 149 | def close(self) -> None: 150 | for handle in ( 151 | self._tick_json, 152 | self._tick_csv, 153 | self._order_json, 154 | self._order_csv, 155 | self._fill_json, 156 | self._fill_csv, 157 | ): 158 | try: 159 | if handle: 160 | handle.close() 161 | except Exception: 162 | pass 163 | 164 | def __enter__(self) -> "RunRecorder": 165 | return self 166 | 167 | def __exit__(self, exc_type, exc, tb) -> None: 168 | self.close() 169 | 170 | @staticmethod 171 | def _write_json(handle, payload: Dict[str, Any]) -> None: 172 | handle.write(json.dumps(payload, separators=(",", ":")) + "\n") 173 | handle.flush() 174 | 175 | 176 | def load_run_summary(run_dir: Path | str) -> Dict[str, Any]: 177 | run_path = Path(run_dir) 178 | summary: Dict[str, Any] = {"run_dir": str(run_path)} 179 | 180 | meta_path = run_path / "meta.json" 181 | if meta_path.exists(): 182 | try: 183 | summary.update(json.loads(meta_path.read_text())) 184 | except json.JSONDecodeError: 185 | summary["meta_error"] = "Invalid meta.json" 186 | 187 | for kind in ("ticks", "orders", "fills"): 188 | file_path = run_path / f"{kind}.jsonl" 189 | count = 0 190 | if file_path.exists(): 191 | with file_path.open("r", encoding="utf-8") as fh: 192 | for _ in fh: 193 | count += 1 194 | summary[f"{kind}_count"] = count 195 | 196 | return summary 197 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # algosim — Real‑Time Algorithmic Trading Simulator (MVP, ZMQ) 2 | 3 | The Python Quants — TPQ Logo 4 | 5 | algosim is a small, local-first teaching tool that simulates real‑time prices, lets you plug in tiny Python strategies, and streams events over ZeroMQ. A Streamlit UI provides controls and live views; a CLI supports headless runs and reproducible replays. 6 | 7 | Author: Dr. Yves J. Hilpisch — The Python Quants GmbH 8 | 9 | ## Features (MVP) 10 | 11 | - Simulator: OU/Vasicek price process (small moves + gentle upward drift), Poisson/fixed tick schedule 12 | - Broker: MARKET fills with latency, slippage, commissions; publishes fills; simple position/cash tracking 13 | - Transport: ZMQ PUB/SUB for ticks and fills; PUSH/PULL for orders (JSON) 14 | - Streamlit App: 15 | - Tabs 16 | - Ticks: real-time Plotly chart with persistent streaming updates, trade overlays (green up-triangles for buys, red down-triangles for sells), optional text mode, and live tick stats 17 | - Fills / Orders: manual BUY/SELL on orders_push, scrollable fill history, contextual warnings 18 | - P&L: dense 6-per-row KPI grid (Position, Value, Cash, Equity, MaxDD, Sharpe, Exposure, Win Rate, Avg Trade P/L, Avg Hold, Dollar Exposure) + equity chart 19 | - Strategy: inline code editor for `strategy.py`, Start/Stop strategy host, PARAMS override (JSON), tick topic, conflation toggle, auto-flatten option on stop, and timestamped live logs in a scrollable pane 20 | - Admin: listener controls, diagnostics (3s receive/fill probes), status dashboard, local process management, metrics settings, config loader, strategy host registry tools, latest recording path hint 21 | - Start/Stop SUB quick controls remain in the sidebar for convenience 22 | - Status/diagnostics distinguish between listener-derived metrics and test probes 23 | - Built-in Strategies: 24 | - Mean-reversion fade (`strategies/mean_reversion/strategy.py`): targets ±qty positions when price deviates from a slow SMA beyond configurable entry/exit bands, with trend-aware guardrails and cooldown 25 | - SMA crossover (`strategies/sma_crossover/strategy.py` / `strategies/sma_crossover/run_sma.py`) 26 | - Recorder: captures ticks, orders, fills, config, and seed to `runs///` (JSONL + CSV) for deterministic replays (`sim report`, `sim replay`) 27 | - Config: YAML (`configs/default.yaml`) incl. `portfolio.initial_cash` (default 100,000) 28 | 29 | See `outline.md` for the full specification and roadmap. 30 | 31 | ## Quickstart 32 | 33 | - Create a virtual environment and install dependencies 34 | 35 | ``` 36 | python -m venv .venv && source .venv/bin/activate 37 | pip install -r requirements.txt 38 | ``` 39 | 40 | - Start simulator and broker (inline to see logs) 41 | 42 | ``` 43 | python -m rt_sim.cli run --config configs/default.yaml --inline 44 | ``` 45 | 46 | - In another terminal, start the UI 47 | 48 | ``` 49 | streamlit run rt_sim/app_streamlit.py 50 | ``` 51 | 52 | In the UI sidebar: 53 | - Local Processes: Start local simulator and broker (if not already running) 54 | - Status: verify ticks are flowing (ticks/sec > 0) 55 | - Ticks tab: Chart (ISO timestamps) or Text mode 56 | - Fills / Orders tab: send manual BUY/SELL; fills appear below 57 | - P&L tab: Position | Cash | Equity (single line), live equity chart 58 | 59 | ### Strategy Host (via UI) 60 | 61 | Use the Strategy tab to edit and run a strategy with the built‑in host: 62 | 63 | - Strategy path: defaults to `strategies/mean_reversion/strategy.py` (resolved relative to project root); SMA crossover remains available at `strategies/sma_crossover/strategy.py` 64 | - Load file / Save file: edit the file inline 65 | - PARAMS override (JSON): e.g. `{ "fast_window": 10, "slow_window": 40, "entry_threshold_bps": 6, "exit_threshold_bps": 2, "qty": 20, "cooldown_s": 4 }` 66 | - Strategy ID: topic used for fills (e.g., `sma1`) 67 | - Tick topic: `X` (default asset) or empty to subscribe to all 68 | - Start strategy host: launches a subprocess and shows Live Logs (written to `runs/strategy_host_*.log`) 69 | - Stop strategy host: terminates the subprocess (optionally auto-flattens the position first) 70 | - Stop ALL strategy hosts: sends SIGTERM to all tracked strategy host PIDs (`runs/strategy_hosts.json`) 71 | 72 | The example strategy template implements price‑vs‑SMA crossover with a no‑trade band (threshold_bps) and a cooldown (min_interval_s) to limit churn. 73 | 74 | ### Recording & Replay 75 | 76 | - Every `sim run` stores artifacts under `run.export_dir//` (default `runs/last/`) 77 | - Inspect a run with `python -m rt_sim.cli report runs/last/` — prints counts and metadata 78 | - Re-broadcast ticks using `python -m rt_sim.cli replay runs/last/ --speed 2.0 --echo` 79 | - Artifacts include JSONL + CSV for ticks/orders/fills plus `config_used.yaml` and `meta.json` (seed + timestamps) 80 | 81 | ### Testing 82 | 83 | - Install deps: `pip install -r requirements.txt` 84 | - Run the suite: `python -m pytest` 85 | - Coverage includes metrics, OU stepper, transport smoke tests, recorder persistence, and mean-reversion signal logic 86 | 87 | ## Strategy Runner (SMA Crossover) 88 | 89 | Run a simple price vs SMA crossover strategy that places orders automatically over ZMQ: 90 | 91 | ``` 92 | python strategies/sma_crossover/run_sma.py \ 93 | --config configs/default.yaml \ 94 | --strategy-id sma1 \ 95 | --window 100 \ 96 | --qty 1 \ 97 | --threshold-bps 15 \ 98 | --min-interval-s 10 99 | ``` 100 | 101 | Useful options: 102 | - `--print-each` prints every tick (warm‑up and px/SMA) 103 | - `--report-sec N` prints tick counts every N seconds (default 1s) 104 | - `--topic X` subscribe to a specific tick topic (default subscribes to all) 105 | - `--no-conflate` process all ticks (default behavior already avoids conflation) 106 | 107 | The app will display resulting fills and live P&L. Trades also appear on the live tick chart. 108 | 109 | ## Configuration Notes 110 | 111 | - Edit `configs/default.yaml` for model params (e.g., `sigma`, `mu`), schedule, execution, endpoints, and `portfolio.initial_cash`. 112 | - The app’s Config section (bottom of sidebar) resets Position/Cash/P&L to reflect the loaded config. 113 | - In the P&L tab, you can set the Sharpe annualization factor (e.g., trading seconds per year) in the sidebar under “Metrics Settings”. 114 | 115 | ## Troubleshooting 116 | 117 | - No fills: start broker and simulator first; verify “Fills listener alive: True” and try “Test fills (3s)”. 118 | - Orders deferred: broker prints “defer fill: no price yet” until it receives the first tick. 119 | - Strategy runner sees 0 ticks: subscribe to all topics (`--topic ""`), avoid conflation, and wait for the runner’s “first tick …” message. 120 | - Strategy host via UI shows no logs: use the Strategy tab’s “Stop ALL strategy hosts”, start local simulator/broker, then Start strategy host again; logs are tailed from `runs/strategy_host_*.log`. 121 | 122 | ## Status (MVP) 123 | 124 | - Implemented: OU simulator, broker with execution costs + portfolio snapshots, ZeroMQ transport, Streamlit UI (ticks/fills/P&L/strategy/admin), CLI (`run`, `run-strategy`, `new-strategy`, `report`, `replay`), SMA strategy host, SMA & mean-reversion example strategies, recorder exports, metrics helpers, and unit/smoke tests (metrics, transport, recorder, strategies) 125 | - Next: headless evaluation workflow (`sim eval`) for deterministic comparisons, strategy linting (`sim doctor`), richer analytics/metrics dashboards, CI integration for tests, and additional strategy templates 126 | -------------------------------------------------------------------------------- /rt_sim/metrics.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import math 4 | from typing import Iterable, List, Sequence, Tuple 5 | 6 | 7 | def compute_drawdown(equity: Sequence[float]) -> Tuple[float, int, int]: 8 | """Compute max drawdown from an equity curve. 9 | 10 | Returns (max_drawdown, peak_index, trough_index) where drawdown is expressed as a fraction (0.1 = 10%). 11 | If equity has <2 points, returns (0.0, 0, 0). 12 | """ 13 | if not equity or len(equity) < 2: 14 | return 0.0, 0, 0 15 | peak = equity[0] 16 | peak_i = 0 17 | max_dd = 0.0 18 | max_i = 0 19 | max_peak_i = 0 20 | for i, v in enumerate(equity): 21 | if v > peak: 22 | peak = v 23 | peak_i = i 24 | dd = 0.0 if peak <= 0 else (peak - v) / peak 25 | if dd > max_dd: 26 | max_dd = dd 27 | max_i = i 28 | max_peak_i = peak_i 29 | return max_dd, max_peak_i, max_i 30 | 31 | 32 | def compute_sharpe_from_equity(equity: Sequence[float], rf_per_period: float = 0.0, annualization_factor: float = 252.0) -> float: 33 | """Compute a basic Sharpe ratio from an equity curve by using simple per-step returns. 34 | 35 | annualization_factor scales from per-step to annualized (e.g., 252 for daily, or seconds_in_year / sample_period_seconds). 36 | """ 37 | if not equity or len(equity) < 3: 38 | return 0.0 39 | rets: List[float] = [] 40 | for i in range(1, len(equity)): 41 | if equity[i - 1] != 0: 42 | rets.append((equity[i] - equity[i - 1]) / equity[i - 1]) 43 | if not rets: 44 | return 0.0 45 | mean = sum(rets) / len(rets) 46 | # subtract risk-free per period 47 | mean -= rf_per_period 48 | var = sum((r - (sum(rets) / len(rets))) ** 2 for r in rets) / max(1, (len(rets) - 1)) 49 | std = math.sqrt(var) 50 | if std == 0: 51 | return 0.0 52 | return (mean / std) * math.sqrt(annualization_factor) 53 | 54 | 55 | def compute_time_weighted_exposure(times: Sequence[float], positions: Sequence[float]) -> float: 56 | """Compute time-weighted exposure: fraction of time with non-zero position. 57 | 58 | times must be monotonically increasing and align with positions. 59 | Returns a value in [0,1]. If insufficient data, returns 0. 60 | """ 61 | if not times or len(times) < 2 or len(times) != len(positions): 62 | return 0.0 63 | total = 0.0 64 | active = 0.0 65 | for i in range(1, len(times)): 66 | dt = max(0.0, times[i] - times[i - 1]) 67 | total += dt 68 | if positions[i - 1] != 0.0: 69 | active += dt 70 | if total == 0: 71 | return 0.0 72 | return active / total 73 | 74 | 75 | def compute_trade_stats(fills: Sequence[dict]) -> dict: 76 | """Compute simple trade statistics from a sequence of fills. 77 | 78 | Uses FIFO lot matching to compute realized P&L segments and hold times. 79 | Returns dict with keys: trade_count, win_rate, avg_trade_pl, avg_hold_s. 80 | """ 81 | if not fills: 82 | return {"trade_count": 0, "win_rate": 0.0, "avg_trade_pl": 0.0, "avg_hold_s": 0.0} 83 | 84 | # Ensure chronological order by timestamp if present 85 | fills_sorted = sorted( 86 | fills, 87 | key=lambda f: (float(f.get("ts_wall", 0.0)), float(f.get("ts_sim", 0.0))), 88 | ) 89 | 90 | # Inventory lots for long and short 91 | long_lots: List[Tuple[float, float, float, float]] = [] # (qty, price, ts, comm_per_unit) 92 | short_lots: List[Tuple[float, float, float, float]] = [] 93 | 94 | trade_pnls: List[float] = [] 95 | trade_holds: List[float] = [] 96 | 97 | for f in fills_sorted: 98 | side = str(f.get("side", "")).upper() 99 | qty = float(f.get("qty", 0.0)) 100 | price = float(f.get("fill_price", 0.0)) 101 | ts = float(f.get("ts_wall", 0.0)) 102 | comm_per_unit = float(f.get("commission", 0.0)) / qty if qty else 0.0 103 | 104 | if side == "BUY": 105 | # First close shorts 106 | remain = qty 107 | while remain > 0 and short_lots: 108 | sqty, sprice, sts, scomm = short_lots[0] 109 | matched = min(remain, sqty) 110 | # Short P&L = entry_price - exit_price (minus commissions) 111 | pnl = (sprice - price) * matched - (scomm + comm_per_unit) * matched 112 | hold = max(0.0, ts - sts) 113 | trade_pnls.append(pnl) 114 | trade_holds.append(hold) 115 | sqty -= matched 116 | remain -= matched 117 | if sqty <= 1e-12: 118 | short_lots.pop(0) 119 | else: 120 | short_lots[0] = (sqty, sprice, sts, scomm) 121 | # Remaining creates/extends long 122 | if remain > 1e-12: 123 | long_lots.append((remain, price, ts, comm_per_unit)) 124 | 125 | elif side == "SELL": 126 | # First close longs 127 | remain = qty 128 | while remain > 0 and long_lots: 129 | lqty, lprice, lts, lcomm = long_lots[0] 130 | matched = min(remain, lqty) 131 | # Long P&L = exit_price - entry_price (minus commissions) 132 | pnl = (price - lprice) * matched - (lcomm + comm_per_unit) * matched 133 | hold = max(0.0, ts - lts) 134 | trade_pnls.append(pnl) 135 | trade_holds.append(hold) 136 | lqty -= matched 137 | remain -= matched 138 | if lqty <= 1e-12: 139 | long_lots.pop(0) 140 | else: 141 | long_lots[0] = (lqty, lprice, lts, lcomm) 142 | # Remaining creates/extends short 143 | if remain > 1e-12: 144 | short_lots.append((remain, price, ts, comm_per_unit)) 145 | 146 | n = len(trade_pnls) 147 | if n == 0: 148 | return {"trade_count": 0, "win_rate": 0.0, "avg_trade_pl": 0.0, "avg_hold_s": 0.0} 149 | wins = sum(1 for p in trade_pnls if p > 0) 150 | return { 151 | "trade_count": n, 152 | "win_rate": wins / n, 153 | "avg_trade_pl": sum(trade_pnls) / n, 154 | "avg_hold_s": sum(trade_holds) / n if trade_holds else 0.0, 155 | } 156 | 157 | 158 | def compute_time_weighted_dollar_exposure( 159 | tick_times: Sequence[float], 160 | tick_prices: Sequence[float], 161 | pos_series: Sequence[Tuple[float, float]], 162 | normalizer: float, 163 | ) -> float: 164 | """Compute time-weighted average dollar exposure relative to `normalizer`. 165 | 166 | - Integrates |pos| * price over tick intervals using the position at the start of each interval. 167 | - Returns avg exposure as a fraction of `normalizer` (e.g., initial cash). Can exceed 1.0 if leveraged. 168 | """ 169 | n = len(tick_times) 170 | if n < 2 or n != len(tick_prices) or normalizer <= 0: 171 | return 0.0 172 | # Ensure pos_series is sorted by time 173 | ps = sorted(list(pos_series), key=lambda x: x[0]) 174 | if not ps: 175 | return 0.0 176 | pos_idx = 0 177 | total = 0.0 178 | dollar = 0.0 179 | for i in range(1, n): 180 | t0, t1 = float(tick_times[i - 1]), float(tick_times[i]) 181 | dt = max(0.0, t1 - t0) 182 | if dt == 0.0: 183 | continue 184 | # Advance position index up to t0 185 | while pos_idx + 1 < len(ps) and ps[pos_idx + 1][0] <= t0: 186 | pos_idx += 1 187 | pos_t0 = float(ps[pos_idx][1]) 188 | px_t0 = float(tick_prices[i - 1]) 189 | dollar += abs(pos_t0) * px_t0 * dt 190 | total += dt 191 | if total == 0.0: 192 | return 0.0 193 | return dollar / (total * normalizer) 194 | -------------------------------------------------------------------------------- /rt_sim/broker.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import time 4 | from dataclasses import dataclass 5 | from typing import Deque, Dict, Optional 6 | from collections import deque 7 | from pathlib import Path 8 | 9 | from .transport import Transport 10 | from .portfolio import Portfolio 11 | from .recorder import RunRecorder 12 | 13 | 14 | @dataclass 15 | class PendingOrder: 16 | ts_wall_in: float 17 | strategy_id: str 18 | side: str # BUY/SELL 19 | qty: float 20 | tag: Optional[str] 21 | 22 | 23 | class Broker: 24 | def __init__(self, config: Dict, transport: Transport, run_id: str, export_dir: str | Path | None = None): 25 | self.cfg = config 26 | self.t = transport 27 | self.run_id = run_id 28 | ep = config["transport"]["endpoints"] 29 | self.addr_orders = ep["orders_push"] 30 | self.addr_ticks = ep["ticks_pub"] 31 | self.addr_fills = ep["fills_pub"] 32 | 33 | # Execution params 34 | ex = config["execution"] 35 | self.latency_ms = int(ex.get("latency_ms", 50)) 36 | self.slippage_bps = float(ex.get("slippage_bps", 1.0)) 37 | self.commission_bps = float(ex.get("commission_bps", 0.5)) 38 | self.commission_fixed = float(ex.get("commission_fixed", 0.0)) 39 | 40 | initial_cash = float(config.get("portfolio", {}).get("initial_cash", 100000.0)) 41 | self.portfolio = Portfolio(initial_cash=initial_cash) 42 | 43 | self.last_price: Optional[float] = None 44 | self.last_ts_sim: float = 0.0 45 | self.pending: Deque[PendingOrder] = deque() 46 | self.recorder: Optional[RunRecorder] = ( 47 | RunRecorder(export_dir, enable_orders=True, enable_fills=True) if export_dir is not None else None 48 | ) 49 | 50 | def start(self) -> None: 51 | # Bind/Connect sockets 52 | pull = self.t.bind_pull(self.addr_orders) 53 | # Subscribe to all tick topics to avoid topic mismatches; avoid ZMQ conflation here 54 | sub = self.t.connect_sub(self.addr_ticks, topic="", conflate=False) 55 | pub = self.t.bind_pub(self.addr_fills, kind="fills") 56 | 57 | poller = __import__("zmq").Poller() 58 | poller.register(pull, __import__("zmq").POLLIN) 59 | poller.register(sub, __import__("zmq").POLLIN) 60 | print( 61 | f"[broker] listening orders@{self.addr_orders} | ticks@{self.addr_ticks} | fills@{self.addr_fills}", 62 | flush=True, 63 | ) 64 | 65 | try: 66 | while True: 67 | socks = dict(poller.poll(timeout=50)) 68 | # Ticks update last price and simulated time 69 | if sub in socks and socks[sub] == __import__("zmq").POLLIN: 70 | _, payload = self.t.recv_json(sub) 71 | self.last_price = float(payload.get("price")) 72 | self.last_ts_sim = float(payload.get("ts_sim", self.last_ts_sim)) 73 | ts_wall = float(payload.get("ts_wall", time.time())) 74 | self.portfolio.update_market_price(self.last_price, ts_sim=self.last_ts_sim, ts_wall=ts_wall) 75 | # Debug tick reception 76 | try: 77 | print(f"[broker] tick seq={int(payload.get('seq', -1))} price={self.last_price:.5f}") 78 | except Exception: 79 | pass 80 | snap_dict = self.portfolio.snapshot(ts_sim=self.last_ts_sim, ts_wall=ts_wall, run_id=self.run_id).model_dump() 81 | Transport.send_json(pub, "portfolio", {"type": "portfolio", "source": "mark", **snap_dict}) 82 | 83 | # Orders 84 | if pull in socks and socks[pull] == __import__("zmq").POLLIN: 85 | order = self.t.recv_json_pull(pull) 86 | print(f"[order] recv side={order.get('side')} qty={order.get('qty')} from={order.get('strategy_id')}", flush=True) 87 | po = PendingOrder( 88 | ts_wall_in=time.time(), 89 | strategy_id=order.get("strategy_id", "unknown"), 90 | side=order.get("side", "BUY"), 91 | qty=float(order.get("qty", 0.0)), 92 | tag=order.get("tag"), 93 | ) 94 | self.pending.append(po) 95 | if self.recorder: 96 | self.recorder.log_order( 97 | { 98 | "ts_wall_in": po.ts_wall_in, 99 | "strategy_id": po.strategy_id, 100 | "side": po.side, 101 | "qty": po.qty, 102 | "tag": po.tag, 103 | "run_id": self.run_id, 104 | } 105 | ) 106 | 107 | # Attempt fills (wall-clock latency for MVP) 108 | now = time.time() 109 | while self.pending and (now - self.pending[0].ts_wall_in) * 1000.0 >= self.latency_ms: 110 | po = self.pending.popleft() 111 | if self.last_price is None: 112 | # No price yet; defer 113 | self.pending.appendleft(po) 114 | print("[broker] defer fill: no price yet", flush=True) 115 | break 116 | price = float(self.last_price) 117 | slip = self.slippage_bps / 10000.0 118 | fill_price = price * (1.0 + slip) if po.side.upper() == "BUY" else price * (1.0 - slip) 119 | notional = fill_price * po.qty 120 | commission = self.commission_fixed + self.commission_bps / 10000.0 * notional 121 | realized_delta, snapshot = self.portfolio.apply_fill( 122 | po.side, 123 | po.qty, 124 | fill_price, 125 | commission, 126 | self.last_ts_sim, 127 | now, 128 | self.run_id, 129 | ) 130 | snap_dict = snapshot.model_dump() 131 | 132 | # Publish fill 133 | payload = { 134 | "ts_sim": self.last_ts_sim, 135 | "ts_wall": now, 136 | "strategy_id": po.strategy_id, 137 | "side": po.side, 138 | "qty": po.qty, 139 | "fill_price": fill_price, 140 | "slippage_bps": self.slippage_bps, 141 | "commission": commission, 142 | "latency_ms": self.latency_ms, 143 | "order_tag": po.tag, 144 | "run_id": self.run_id, 145 | "pos_after": snap_dict["pos"], 146 | "cash_after": snap_dict["cash"], 147 | "equity_after": snap_dict["equity"], 148 | "realized_pl": snap_dict["realized"], 149 | "unrealized_pl": snap_dict["unrealized"], 150 | "portfolio_ts_sim": snap_dict["ts_sim"], 151 | "portfolio_ts_wall": snap_dict["ts_wall"], 152 | } 153 | Transport.send_json(pub, po.strategy_id, payload) 154 | if self.recorder: 155 | self.recorder.log_fill(payload) 156 | print( 157 | f"[fill] strat={po.strategy_id} side={po.side} qty={po.qty} price={fill_price:.5f} pos={self.portfolio.pos:.2f} cash={self.portfolio.cash:.2f} " 158 | f"realized={self.portfolio.realized:.2f} delta={realized_delta:.2f}", 159 | flush=True, 160 | ) 161 | # Broadcast snapshot on dedicated topic for UI/consumers 162 | snap_payload = {"type": "portfolio", "source": "fill", **snap_dict} 163 | Transport.send_json(pub, "portfolio", snap_payload) 164 | except KeyboardInterrupt: 165 | pass 166 | finally: 167 | if self.recorder: 168 | self.recorder.close() 169 | 170 | 171 | def run(config: Dict, transport: Transport, run_id: str, export_dir: str | Path | None = None) -> None: 172 | Broker(config, transport, run_id, export_dir=export_dir).start() 173 | -------------------------------------------------------------------------------- /rt_sim/strategy_host.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import importlib.util 4 | import json 5 | import signal 6 | import sys 7 | import time 8 | from dataclasses import dataclass 9 | from pathlib import Path 10 | from types import ModuleType 11 | from typing import Any, Callable, Dict, Optional 12 | 13 | import zmq 14 | 15 | import os 16 | 17 | from .transport import Transport 18 | 19 | _LOG_FILE = None 20 | _TICK_REPORT_INTERVAL = 5.0 21 | 22 | 23 | def _log(msg: str) -> None: 24 | global _LOG_FILE 25 | from datetime import datetime 26 | 27 | stamp = datetime.utcnow().isoformat(timespec="seconds") 28 | formatted = f"[{stamp}] {msg}" 29 | try: 30 | print(formatted, flush=True) 31 | except Exception: 32 | pass 33 | try: 34 | path = os.environ.get("STRAT_LOG_FILE") 35 | if path: 36 | if _LOG_FILE is None: 37 | _LOG_FILE = open(path, "a", buffering=1) 38 | _LOG_FILE.write(formatted + "\n") 39 | except Exception: 40 | pass 41 | 42 | 43 | class SMA: 44 | def __init__(self, window: int): 45 | self.w = max(1, int(window)) 46 | self.buf: list[float] = [] 47 | self.sum = 0.0 48 | 49 | def update(self, x: float) -> Optional[float]: 50 | if len(self.buf) == self.w: 51 | self.sum -= self.buf[0] 52 | self.buf = self.buf[1:] 53 | self.buf.append(float(x)) 54 | self.sum += float(x) 55 | if len(self.buf) < self.w: 56 | return None 57 | return self.sum / self.w 58 | 59 | 60 | class IndicatorsNS: 61 | def SMA(self, window: int) -> SMA: 62 | return SMA(window) 63 | 64 | 65 | class StrategyContext: 66 | def __init__(self, strategy_id: str, params: Dict[str, Any], transport: Transport, endpoints: Dict[str, str]): 67 | self._sid = strategy_id 68 | self._params = params or {} 69 | self._state: Dict[str, Any] = {} 70 | self._t = transport 71 | self._endpoints = endpoints 72 | self.indicator = IndicatorsNS() 73 | # runtime/position tracking 74 | self._pos: float = 0.0 75 | # sockets 76 | self._push = self._t.connect_push(self._endpoints["orders_push"]) 77 | self._push.setsockopt(zmq.LINGER, 500) 78 | 79 | # params/state 80 | def get_param(self, name: str, default: Any = None) -> Any: 81 | return self._params.get(name, default) 82 | 83 | def set_state(self, key: str, value: Any) -> None: 84 | self._state[key] = value 85 | 86 | def get_state(self, key: str, default: Any = None) -> Any: 87 | return self._state.get(key, default) 88 | 89 | # trading helpers 90 | def position(self) -> float: 91 | return self._pos 92 | 93 | def place_market_order(self, side: str, qty: float, tag: Optional[str] = None) -> None: 94 | payload = { 95 | "strategy_id": self._sid, 96 | "side": str(side).upper(), 97 | "qty": float(qty), 98 | "tag": tag, 99 | } 100 | Transport.send_json_push(self._push, payload) 101 | _log(f"[host] order placed: {payload}") 102 | 103 | # internal update from fills 104 | def _on_fill(self, fill: Dict[str, Any]) -> None: 105 | side = str(fill.get("side", "")).upper() 106 | qty = float(fill.get("qty", 0.0)) 107 | if side == "BUY": 108 | self._pos += qty 109 | elif side == "SELL": 110 | self._pos -= qty 111 | 112 | def close(self) -> None: 113 | try: 114 | self._push.close(0) 115 | except Exception: 116 | pass 117 | 118 | 119 | def _load_module(path: Path) -> ModuleType: 120 | spec = importlib.util.spec_from_file_location(path.stem, str(path)) 121 | if spec is None or spec.loader is None: 122 | raise ImportError(f"Cannot load strategy module from {path}") 123 | mod = importlib.util.module_from_spec(spec) 124 | sys.modules[path.stem] = mod 125 | spec.loader.exec_module(mod) # type: ignore[assignment] 126 | return mod 127 | 128 | 129 | def run( 130 | config: Dict[str, Any], 131 | transport: Transport, 132 | run_id: str, 133 | strategy_path: str, 134 | strategy_id: Optional[str] = None, 135 | params_override: Optional[Dict[str, Any]] = None, 136 | topic: str = "", 137 | conflate: bool = False, 138 | ) -> None: 139 | endpoints = config["transport"]["endpoints"] 140 | ticks_addr = endpoints["ticks_pub"] 141 | fills_addr = endpoints["fills_pub"] 142 | 143 | mod_path = Path(strategy_path).resolve() 144 | mod = _load_module(mod_path) 145 | sid = strategy_id or getattr(mod, "NAME", None) or mod_path.stem 146 | base_params = getattr(mod, "PARAMS", {}) or {} 147 | user_params = params_override or {} 148 | params = {**base_params, **user_params} 149 | 150 | ctx = StrategyContext(sid, params, transport, endpoints) 151 | if hasattr(mod, "init") and callable(mod.init): # type: ignore[attr-defined] 152 | mod.init(ctx) # type: ignore[attr-defined] 153 | 154 | sub_ticks = transport.connect_sub(ticks_addr, topic=topic, conflate=conflate) 155 | sub_fills = transport.connect_sub(fills_addr, topic=sid, conflate=False) 156 | 157 | poller = zmq.Poller() 158 | poller.register(sub_ticks, zmq.POLLIN) 159 | poller.register(sub_fills, zmq.POLLIN) 160 | 161 | _log( 162 | f"[host] strategy_id={sid} | file={mod_path} | ticks={ticks_addr} topic='{topic or '*'}' conflate={conflate} | fills={fills_addr}" 163 | ) 164 | 165 | last_mtime = mod_path.stat().st_mtime 166 | running = True 167 | 168 | def _sigint(_sig, _frm): 169 | nonlocal running 170 | running = False 171 | 172 | signal.signal(signal.SIGINT, _sigint) 173 | 174 | try: 175 | tick_count = 0 176 | t_last = time.time() 177 | saw_first_tick = False 178 | while running: 179 | socks = dict(poller.poll(timeout=200)) 180 | # fills update position 181 | if sub_fills in socks and socks[sub_fills] == zmq.POLLIN: 182 | _, fill = transport.recv_json(sub_fills) 183 | ctx._on_fill(fill) 184 | 185 | # ticks -> on_tick 186 | if sub_ticks in socks and socks[sub_ticks] == zmq.POLLIN: 187 | _, tick = transport.recv_json(sub_ticks) 188 | tick_count += 1 189 | if not saw_first_tick: 190 | saw_first_tick = True 191 | _log(f"[host] first tick price={float(tick.get('price', 0.0)):.5f}") 192 | if hasattr(mod, "on_tick") and callable(mod.on_tick): # type: ignore[attr-defined] 193 | try: 194 | mod.on_tick(ctx, tick) # type: ignore[attr-defined] 195 | except Exception as e: 196 | _log(f"[host] on_tick error: {e}") 197 | 198 | now = time.time() 199 | if now - t_last >= _TICK_REPORT_INTERVAL: 200 | elapsed = now - t_last 201 | _log(f"[host] ticks last {elapsed:.1f}s: {tick_count}") 202 | tick_count = 0 203 | t_last = now 204 | 205 | # hot reload check 206 | try: 207 | mtime = mod_path.stat().st_mtime 208 | if mtime > last_mtime: 209 | last_mtime = mtime 210 | print(f"[host] change detected -> reload {mod_path}", flush=True) 211 | # teardown 212 | if hasattr(mod, "on_stop") and callable(mod.on_stop): # type: ignore[attr-defined] 213 | try: 214 | mod.on_stop(ctx) # type: ignore[attr-defined] 215 | except Exception: 216 | pass 217 | # reload fresh state 218 | mod = _load_module(mod_path) 219 | ctx._state.clear() 220 | ctx._pos = 0.0 221 | if hasattr(mod, "init") and callable(mod.init): # type: ignore[attr-defined] 222 | mod.init(ctx) # type: ignore[attr-defined] 223 | except Exception: 224 | pass 225 | finally: 226 | try: 227 | if hasattr(mod, "on_stop") and callable(mod.on_stop): # type: ignore[attr-defined] 228 | mod.on_stop(ctx) # type: ignore[attr-defined] 229 | except Exception: 230 | pass 231 | try: 232 | sub_ticks.close(0) 233 | sub_fills.close(0) 234 | except Exception: 235 | pass 236 | ctx.close() 237 | -------------------------------------------------------------------------------- /strategies/sma_crossover/run_sma.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import argparse 4 | import signal 5 | import sys 6 | import time 7 | from collections import deque 8 | from typing import Deque, Optional 9 | 10 | import zmq 11 | 12 | try: 13 | from rt_sim.utils import load_config 14 | from rt_sim.transport import Transport 15 | except ModuleNotFoundError: 16 | # Allow running directly from repo root 17 | import pathlib 18 | 19 | sys.path.append(str(pathlib.Path(__file__).resolve().parents[2])) 20 | from rt_sim.utils import load_config 21 | from rt_sim.transport import Transport 22 | 23 | 24 | def run( 25 | strategy_id: str, 26 | config_path: str, 27 | window: int, 28 | qty: float, 29 | conflate: bool = False, 30 | topic: str = "", 31 | threshold_bps: float = 10.0, 32 | min_interval_s: float = 5.0, 33 | print_each: bool = False, 34 | report_sec: float = 1.0, 35 | ) -> None: 36 | cfg = load_config(config_path) 37 | ep = cfg["transport"]["endpoints"] 38 | ticks_addr = ep["ticks_pub"] 39 | fills_addr = ep["fills_pub"] 40 | orders_addr = ep["orders_push"] 41 | 42 | t = Transport( 43 | hwm_ticks=int(cfg["transport"]["hwm"]["ticks_pub"]), 44 | hwm_orders=int(cfg["transport"]["hwm"]["orders"]), 45 | hwm_fills=int(cfg["transport"]["hwm"]["fills_pub"]), 46 | ) 47 | 48 | # Sockets 49 | # Subscribe to all topics by default to avoid topic mismatch issues 50 | sub_ticks = t.connect_sub(ticks_addr, topic=topic, conflate=conflate) 51 | sub_fills = t.connect_sub(fills_addr, topic=strategy_id, conflate=False) 52 | push_orders = t.connect_push(orders_addr) 53 | push_orders.setsockopt(zmq.LINGER, 500) 54 | 55 | print( 56 | f"[sma] strategy_id={strategy_id} window={window} qty={qty} | ticks={ticks_addr} topic='{topic or '*'}' conflate={conflate} | fills={fills_addr} | orders={orders_addr}", 57 | flush=True, 58 | ) 59 | 60 | # SMA state 61 | buf: Deque[float] = deque(maxlen=window) 62 | ssum: float = 0.0 63 | last_side: Optional[str] = None # 'LONG' or 'SHORT' 64 | pos: float = 0.0 65 | last_trade_ts: float = 0.0 66 | 67 | # Allow brief time for SUB subscriptions to propagate 68 | time.sleep(0.2) 69 | poller = zmq.Poller() 70 | poller.register(sub_ticks, zmq.POLLIN) 71 | poller.register(sub_fills, zmq.POLLIN) 72 | 73 | running = True 74 | tick_count_total = 0 75 | tick_count_window = 0 76 | t_last = time.time() 77 | 78 | def _sigint(_sig, _frm): 79 | nonlocal running 80 | running = False 81 | 82 | signal.signal(signal.SIGINT, _sigint) 83 | 84 | try: 85 | # Initial wait for first tick (up to ~3s) 86 | t0 = time.time() 87 | while True: 88 | socks = dict(poller.poll(timeout=200)) 89 | if sub_ticks in socks and socks[sub_ticks] == zmq.POLLIN: 90 | _, first_tick = t.recv_json(sub_ticks) 91 | px0 = float(first_tick.get("price")) 92 | buf.append(px0); ssum += px0 93 | tick_count_total += 1; tick_count_window += 1 94 | print(f"[sma] first tick px={px0:.5f}", flush=True) 95 | break 96 | if time.time() - t0 > 3.0: 97 | print("[sma] waiting for ticks... (no data yet)", flush=True) 98 | t0 = time.time() 99 | # continue waiting 100 | # Main loop 101 | while running: 102 | socks = dict(poller.poll(timeout=200)) 103 | # Handle fills to track position 104 | if sub_fills in socks and socks[sub_fills] == zmq.POLLIN: 105 | _, fill = t.recv_json(sub_fills) 106 | side = str(fill.get("side", "")).upper() 107 | fqty = float(fill.get("qty", 0.0)) 108 | pos += fqty if side == "BUY" else -fqty 109 | print( 110 | f"[sma] fill side={side} qty={fqty} px={float(fill.get('fill_price', 0.0)):.5f} pos={pos}", 111 | flush=True, 112 | ) 113 | 114 | # Handle ticks 115 | if sub_ticks in socks and socks[sub_ticks] == zmq.POLLIN: 116 | _, tick = t.recv_json(sub_ticks) 117 | px = float(tick.get("price")) 118 | tick_count_total += 1 119 | tick_count_window += 1 120 | # Update SMA 121 | if len(buf) == window: 122 | ssum -= buf[0] 123 | buf.append(px) 124 | ssum += px 125 | sma = ssum / len(buf) 126 | 127 | if print_each: 128 | if len(buf) < window: 129 | print( 130 | f"[sma] tick #{tick_count_total} px={px:.5f} (warm-up {len(buf)}/{window})", 131 | flush=True, 132 | ) 133 | else: 134 | print( 135 | f"[sma] tick #{tick_count_total} px={px:.5f} sma={sma:.5f}", 136 | flush=True, 137 | ) 138 | 139 | if len(buf) >= window: 140 | # Crossover with hysteresis and min interval 141 | diff_bps = (px - sma) / max(1e-12, sma) * 10000.0 142 | if abs(diff_bps) < threshold_bps: 143 | # Inside no-trade band; do nothing 144 | pass 145 | else: 146 | want_side = "LONG" if diff_bps > 0 else "SHORT" 147 | if last_side is None: 148 | last_side = want_side 149 | elif want_side != last_side and (time.time() - last_trade_ts) >= min_interval_s: 150 | if want_side == "LONG" and pos <= 0: 151 | order_qty = abs(pos) + qty 152 | payload = {"strategy_id": strategy_id, "side": "BUY", "qty": float(order_qty), "tag": f"sma_up_{threshold_bps:.1f}bps"} 153 | Transport.send_json_push(push_orders, payload) 154 | print(f"[sma] BUY {order_qty} (px={px:.5f}, sma={sma:.5f}, diff={diff_bps:.2f}bps)", flush=True) 155 | last_trade_ts = time.time() 156 | elif want_side == "SHORT" and pos >= 0 and qty > 0: 157 | order_qty = abs(pos) + qty 158 | payload = {"strategy_id": strategy_id, "side": "SELL", "qty": float(order_qty), "tag": f"sma_dn_{threshold_bps:.1f}bps"} 159 | Transport.send_json_push(push_orders, payload) 160 | print(f"[sma] SELL {order_qty} (px={px:.5f}, sma={sma:.5f}, diff={diff_bps:.2f}bps)", flush=True) 161 | last_trade_ts = time.time() 162 | last_side = want_side 163 | # periodic tick count report 164 | now = time.time() 165 | if not print_each and (now - t_last) >= report_sec: 166 | rate = tick_count_window / max(1e-6, (now - t_last)) 167 | print(f"[sma] ticks last {report_sec:.0f}s: {tick_count_window} (rate={rate:.1f}/s) total={tick_count_total}", flush=True) 168 | tick_count_window = 0 169 | t_last = now 170 | finally: 171 | try: 172 | sub_ticks.close(0) 173 | sub_fills.close(0) 174 | push_orders.close(0) 175 | except Exception: 176 | pass 177 | 178 | 179 | def main() -> None: 180 | p = argparse.ArgumentParser(description="Run a simple SMA crossover strategy over ZMQ") 181 | p.add_argument("--config", default="configs/default.yaml") 182 | p.add_argument("--strategy-id", default="sma1") 183 | p.add_argument("--window", type=int, default=50) 184 | p.add_argument("--qty", type=float, default=100.0) 185 | p.add_argument("--no-conflate", action="store_true") 186 | p.add_argument("--topic", default="", help="Tick topic to subscribe to (empty = all)") 187 | p.add_argument("--print-each", action="store_true", help="Print a line for every tick") 188 | p.add_argument("--report-sec", type=float, default=1.0, help="When not printing each tick, report tick counts every N seconds") 189 | p.add_argument("--threshold-bps", type=float, default=10.0, help="No-trade band around SMA in basis points (default 10 bps)") 190 | p.add_argument("--min-interval-s", type=float, default=5.0, help="Minimum seconds between trades (default 5s)") 191 | args = p.parse_args() 192 | 193 | run( 194 | strategy_id=args.strategy_id, 195 | config_path=args.config, 196 | window=args.window, 197 | qty=args.qty, 198 | conflate=not args.no_conflate, 199 | topic=args.topic, 200 | print_each=args.print_each, 201 | report_sec=args.report_sec, 202 | threshold_bps=args.threshold_bps, 203 | min_interval_s=args.min_interval_s, 204 | ) 205 | 206 | 207 | if __name__ == "__main__": 208 | main() 209 | -------------------------------------------------------------------------------- /rt_sim/cli.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import multiprocessing as mp 4 | from pathlib import Path 5 | from typing import Optional 6 | 7 | import click 8 | import json 9 | import time 10 | import yaml 11 | 12 | from .simulator import run as run_simulator 13 | from .transport import Transport 14 | from .broker import run as run_broker 15 | from .strategy_host import run as run_strategy_host 16 | from .utils import load_config, new_run_id, seed_everything 17 | from .recorder import prepare_run_directory, load_run_summary 18 | 19 | 20 | def _sim_entry(cfg: dict, run_id: str, export_dir: str | None) -> None: 21 | """Top-level simulator process entry (must be picklable for multiprocessing).""" 22 | t = Transport( 23 | hwm_ticks=int(cfg["transport"]["hwm"]["ticks_pub"]), 24 | hwm_orders=int(cfg["transport"]["hwm"]["orders"]), 25 | hwm_fills=int(cfg["transport"]["hwm"]["fills_pub"]), 26 | ) 27 | run_simulator(cfg, t, run_id, export_dir=export_dir) 28 | 29 | 30 | def _broker_entry(cfg: dict, run_id: str, export_dir: str | None) -> None: 31 | t = Transport( 32 | hwm_ticks=int(cfg["transport"]["hwm"]["ticks_pub"]), 33 | hwm_orders=int(cfg["transport"]["hwm"]["orders"]), 34 | hwm_fills=int(cfg["transport"]["hwm"]["fills_pub"]), 35 | ) 36 | run_broker(cfg, t, run_id, export_dir=export_dir) 37 | 38 | 39 | @click.group(name="sim") 40 | def cli() -> None: 41 | """algosim command-line tools""" 42 | 43 | 44 | @cli.command("run") 45 | @click.option("--config", "config_path", type=click.Path(exists=True), default="configs/default.yaml") 46 | @click.option("--headless/--no-headless", default=True, help="Run without UI") 47 | @click.option("--inline/--no-inline", default=False, help="Run in current process for debugging") 48 | def cmd_run(config_path: str, headless: bool, inline: bool) -> None: 49 | """Run the simulator (and optionally headless only). UI is launched separately via streamlit.""" 50 | cfg = load_config(config_path) 51 | seed_everything(int(cfg["run"]["seed"])) 52 | run_id = new_run_id() 53 | export_base = Path(cfg.get("run", {}).get("export_dir", "runs/last")) 54 | run_dir = prepare_run_directory(export_base, run_id, cfg) 55 | 56 | if inline: 57 | click.echo(f"Simulator+Broker starting inline (run_id={run_id}). Press Ctrl-C to stop.") 58 | click.echo(f"Recording artifacts to {run_dir}") 59 | try: 60 | # Run both in current process: simulator on a child process, broker here 61 | sp = mp.Process(target=_sim_entry, args=(cfg, run_id, str(run_dir)), daemon=True) 62 | sp.start() 63 | _broker_entry(cfg, run_id, str(run_dir)) 64 | except KeyboardInterrupt: 65 | click.echo("Stopping...") 66 | else: 67 | ps = mp.Process(target=_sim_entry, args=(cfg, run_id, str(run_dir)), daemon=True) 68 | pb = mp.Process(target=_broker_entry, args=(cfg, run_id, str(run_dir)), daemon=True) 69 | ps.start(); pb.start() 70 | click.echo(f"Simulator started (pid={ps.pid}); Broker started (pid={pb.pid}). Ctrl-C to stop.") 71 | click.echo(f"Recording artifacts to {run_dir}") 72 | try: 73 | ps.join(); pb.join() 74 | except KeyboardInterrupt: 75 | click.echo("Stopping...") 76 | finally: 77 | for proc in (ps, pb): 78 | if proc.is_alive(): 79 | proc.terminate(); proc.join(timeout=1) 80 | 81 | 82 | @cli.command("new-strategy") 83 | @click.argument("name") 84 | def cmd_new_strategy(name: str) -> None: 85 | """Scaffold a new strategy folder with template.""" 86 | base = Path("strategies") / name 87 | base.mkdir(parents=True, exist_ok=True) 88 | strat = base / "strategy.py" 89 | if strat.exists(): 90 | click.echo(f"Strategy already exists at {strat}") 91 | return 92 | strat.write_text( 93 | ( 94 | "NAME = \"SMA Crossover\"\n" 95 | "PARAMS = {\"fast\": 20, \"slow\": 50, \"qty\": 1}\n\n" 96 | "def init(ctx):\n ctx.fast = ctx.indicator.SMA(ctx.get_param('fast', 20))\n ctx.slow = ctx.indicator.SMA(ctx.get_param('slow', 50))\n ctx.set_state('qty', ctx.get_param('qty', 1))\n\n" 97 | "def on_tick(ctx, tick):\n p = tick['price']\n f = ctx.fast.update(p)\n s = ctx.slow.update(p)\n if f is None or s is None:\n return\n pos = ctx.position()\n if f > s and pos <= 0:\n ctx.place_market_order('BUY', abs(pos) + ctx.get_state('qty', 1), tag='bullish')\n elif f < s and pos >= 0:\n ctx.place_market_order('SELL', abs(pos) + ctx.get_state('qty', 1), tag='bearish')\n\n" 98 | "def on_stop(ctx):\n pass\n" 99 | ) 100 | ) 101 | click.echo(f"Created {strat}") 102 | 103 | 104 | @cli.command("run-strategy") 105 | @click.option("--config", "config_path", type=click.Path(exists=True), default="configs/default.yaml") 106 | @click.option("--path", "strategy_path", type=click.Path(exists=True), required=True, help="Path to strategy.py") 107 | @click.option("--id", "strategy_id", default=None, help="Override strategy_id/topic for fills") 108 | @click.option("--params", "params_json", default=None, help="JSON object to override strategy PARAMS") 109 | @click.option("--topic", default="", help="Tick topic to subscribe (empty=all)") 110 | @click.option("--conflate/--no-conflate", default=False) 111 | def cmd_run_strategy(config_path: str, strategy_path: str, strategy_id: str | None, params_json: str | None, topic: str, conflate: bool) -> None: 112 | """Run a strategy module via the built-in host with ctx API.""" 113 | cfg = load_config(config_path) 114 | run_id = new_run_id() 115 | try: 116 | params = json.loads(params_json) if params_json else None 117 | except Exception as e: 118 | raise click.ClickException(f"Invalid --params JSON: {e}") 119 | 120 | t = Transport( 121 | hwm_ticks=int(cfg["transport"]["hwm"]["ticks_pub"]), 122 | hwm_orders=int(cfg["transport"]["hwm"]["orders"]), 123 | hwm_fills=int(cfg["transport"]["hwm"]["fills_pub"]), 124 | ) 125 | click.echo( 126 | f"Starting strategy host (id={strategy_id or 'auto'}) using {strategy_path} | topic='{topic or '*'}' conflate={conflate}" 127 | ) 128 | try: 129 | run_strategy_host(cfg, t, run_id, strategy_path, strategy_id=strategy_id, params_override=params, topic=topic, conflate=conflate) 130 | except KeyboardInterrupt: 131 | click.echo("Stopping strategy host...") 132 | 133 | 134 | @cli.command("report") 135 | @click.argument("run_dir", type=click.Path(exists=True, file_okay=False)) 136 | def cmd_report(run_dir: str) -> None: 137 | """Summarize a recorded run (counts, metadata).""" 138 | summary = load_run_summary(run_dir) 139 | click.echo(f"Run directory: {summary.get('run_dir', run_dir)}") 140 | if summary.get("run_id"): 141 | click.echo(f"Run ID: {summary['run_id']}") 142 | if summary.get("created_utc"): 143 | click.echo(f"Created (UTC): {summary['created_utc']}") 144 | if summary.get("seed") is not None: 145 | click.echo(f"Seed: {summary['seed']}") 146 | if summary.get("meta_error"): 147 | click.echo(f"Meta warning: {summary['meta_error']}") 148 | click.echo( 149 | f"Events → ticks: {summary.get('ticks_count', 0)}, orders: {summary.get('orders_count', 0)}, fills: {summary.get('fills_count', 0)}" 150 | ) 151 | 152 | 153 | @cli.command("replay") 154 | @click.argument("run_dir", type=click.Path(exists=True, file_okay=False)) 155 | @click.option("--speed", default=1.0, show_default=True, help="Playback speed multiplier (1.0 = real-time)") 156 | @click.option("--publish/--no-publish", default=True, show_default=True, help="Publish ticks over ZMQ during replay") 157 | @click.option("--echo/--no-echo", default=False, show_default=True, help="Print each tick to stdout during replay") 158 | def cmd_replay(run_dir: str, speed: float, publish: bool, echo: bool) -> None: 159 | """Replay recorded ticks (optionally rebroadcast over ZMQ).""" 160 | run_path = Path(run_dir) 161 | ticks_path = run_path / "ticks.jsonl" 162 | if not ticks_path.exists(): 163 | raise click.ClickException(f"No ticks.jsonl found in {run_dir}") 164 | 165 | cfg_path = run_path / "config_used.yaml" 166 | cfg = yaml.safe_load(cfg_path.read_text()) if cfg_path.exists() else {} 167 | transport_cfg = cfg.get("transport", {}) 168 | endpoints = transport_cfg.get("endpoints", {}) 169 | ticks_ep = endpoints.get("ticks_pub", "tcp://127.0.0.1:5555") 170 | 171 | pub = None 172 | if publish: 173 | t = Transport( 174 | hwm_ticks=int(transport_cfg.get("hwm", {}).get("ticks_pub", 20000) or 20000), 175 | hwm_orders=int(transport_cfg.get("hwm", {}).get("orders", 20000) or 20000), 176 | hwm_fills=int(transport_cfg.get("hwm", {}).get("fills_pub", 20000) or 20000), 177 | ) 178 | pub = t.bind_pub(ticks_ep, kind="ticks") 179 | click.echo(f"Publishing replayed ticks to {ticks_ep}") 180 | 181 | prev_ts: Optional[float] = None 182 | count = 0 183 | with ticks_path.open("r", encoding="utf-8") as fh: 184 | for line in fh: 185 | if not line.strip(): 186 | continue 187 | tick = json.loads(line) 188 | ts_sim = float(tick.get("ts_sim", 0.0)) 189 | if prev_ts is not None and speed > 0: 190 | delay = max(0.0, (ts_sim - prev_ts) / max(speed, 1e-6)) 191 | if delay: 192 | time.sleep(delay) 193 | if publish and pub is not None: 194 | topic = tick.get("asset_id", "") 195 | Transport.send_json(pub, topic or "", tick) 196 | if echo: 197 | price = tick.get("price") 198 | click.echo(f"[{tick.get('seq')}] ts_sim={ts_sim:.3f} price={price}") 199 | prev_ts = ts_sim 200 | count += 1 201 | 202 | click.echo(f"Replayed {count} ticks from {ticks_path}") 203 | 204 | 205 | def main() -> None: 206 | cli() 207 | 208 | 209 | if __name__ == "__main__": 210 | main() 211 | -------------------------------------------------------------------------------- /outline.md: -------------------------------------------------------------------------------- 1 | # Real-Time Algorithmic Trading Simulator — MVP Specification (ZMQ) 2 | 3 | > **Purpose**: A teaching/illustration tool that simulates real-time pricing, lets users plug in tiny strategy scripts, executes orders, and visualizes P/L and live metrics in a web UI (Streamlit + Plotly). Runs locally first; designed to be CLI-and-LLM-friendly (Codex/GPT‑5). **This edition adopts ZeroMQ (pyzmq) for inter-process communication.** 4 | 5 | The project/the app shall be called "algosim". 6 | 7 | --- 8 | 9 | ## 1) Goals & Non‑Goals 10 | 11 | **Goals (MVP)** 12 | 13 | * Real-time price stream from a configurable stochastic model (Vasicek/OU), with fixed or random inter-arrival times and a speed control. 14 | * Streamlit web app for control + live visualization (price, indicators, trades, P/L, key metrics). 15 | * Strategy plug-in via a tiny Python script following a clear, minimal API (hot-reload capability). 16 | * Simple broker/execution with MARKET orders, configurable latency, slippage, and costs. 17 | * Live portfolio & P/L tracking; rolling stats (drawdown, Sharpe, win rate, etc.). 18 | * Deterministic replays via seeds; export run artifacts (events JSONL, trades CSV). 19 | * CLI for headless runs and quick scaffolding. 20 | * **ZeroMQ transport** for clean message flows (ticks, orders, fills), teachable and extensible. 21 | 22 | **Non‑Goals (MVP)** 23 | 24 | * Multi-asset universe, order book depth, L2 microstructure realism. 25 | * Limit/stop orders, partial fills, margin models, corporate actions. 26 | * Distributed deployments; external data feeds (can be added later thanks to ZMQ). 27 | 28 | --- 29 | 30 | ## 2) Primary User Stories 31 | 32 | * **Educator**: “Start/stop simulations at different speeds, tweak model/fees, and show live how a strategy reacts.” 33 | * **Learner**: “Write a 20–50 line strategy (e.g., SMA cross), hot-reload it, and watch P/L + stats update.” 34 | * **Researcher**: “Run deterministic headless replays, export trades/metrics, and compare settings quickly.” 35 | 36 | --- 37 | 38 | ## 3) System Architecture (MVP, ZeroMQ) 39 | 40 | **Process model**: Single OS process supervising separate **simulator**, **broker**, **UI**, and **strategy sandbox** components communicating over **ZeroMQ** sockets (pyzmq). Default endpoints use `tcp://127.0.0.1` for local development/teaching. The strategy runs in a **separate process** for isolation. A minimal in‑proc fallback may exist for unit tests, but the MVP runs on ZMQ. 41 | 42 | **Core components** 43 | 44 | 1. **Market Simulator**: Generates price ticks using Vasicek/OU; schedules next tick on fixed Δt or Poisson arrivals. 45 | 2. **Transport (ZeroMQ)**: PUB/SUB and PUSH/PULL sockets for ticks, orders, and fills using `tcp://127.0.0.1` endpoints. 46 | 3. **Strategy Sandbox**: Separate Python process executing `strategy.py` (API below), consuming `Tick` events, emitting `Order` objects via ZMQ. 47 | 4. **Broker/Execution**: Applies latency/slippage/costs; returns fills; updates portfolio & cash; pushes PnL/metrics via ZMQ. 48 | 5. **Portfolio & Risk Engine**: Mark‑to‑market, realized/unrealized P/L, exposure, drawdown, rolling Sharpe, trade stats. 49 | 6. **UI (Streamlit)**: Controls + Plotly charts; status panels; live logs; file export buttons. 50 | 7. **Recorder**: Writes JSONL event log + CSVs for prices and trades; captures config/seed for reproducibility. 51 | 52 | **Inter‑component IPC (MVP, ZeroMQ)** 53 | 54 | * **Ticks**: `PUB` (simulator) → `SUB` (strategies, UI) at `tcp://127.0.0.1:5555` with **topic = asset\_id** (e.g., `"X"`). UI subscriber uses **`CONFLATE=1`** to render only the most recent tick. 55 | * **Orders**: `PUSH` (strategy) → `PULL` (broker) at `tcp://127.0.0.1:5556`. 56 | * **Fills/Positions**: `PUB` (broker) → `SUB` (strategies, UI) at `tcp://127.0.0.1:5557` with **topic = strategy\_id** (UI subscribes to all). 57 | * **Serialization**: **JSON**. Each payload includes `seq`, `ts_sim`, `ts_wall`, `run_id`, and relevant IDs. 58 | * **Admin/control**: **in‑proc** for MVP (no ZMQ REQ/REP). Health via optional PUB heartbeats later. 59 | 60 | --- 61 | 62 | ## 3a) Transport (ZeroMQ) — Defaults & Liveness 63 | 64 | * **Endpoints (TCP local)** 65 | 66 | * `ticks_pub`: `tcp://127.0.0.1:5555` 67 | * `orders_push`: `tcp://127.0.0.1:5556` 68 | * `fills_pub`: `tcp://127.0.0.1:5557` 69 | * **HWM**: `SNDHWM/RCVHWM = 20000` (teaching default). UI tick subscriber sets **`CONFLATE=1`** to always show the freshest price and avoid backpressure. 70 | * **Determinism**: rely on included `seq` and `ts_sim` in every message to detect gaps and enable exact replays. 71 | * **Reconnects**: ZMQ handles transient disconnects; the UI displays gap counts if `seq` jumps. 72 | 73 | --- 74 | 75 | ## 4) Data & Event Schemas (Pydantic models) 76 | 77 | **Tick** 78 | 79 | ```json 80 | { 81 | "ts": "ISO-8601 or ns epoch", 82 | "seq": 12345, 83 | "price": 101.2345, 84 | "model_state": {"x": 0.12, "dt": 0.2, "kappa": 0.8, "theta": 0.0, "sigma": 0.1}, 85 | "asset_id": "X", 86 | "run_id": "2025-09-09T10:00:00Z" 87 | } 88 | ``` 89 | 90 | **Order (MARKET only in MVP)** 91 | 92 | ```json 93 | { 94 | "ts": "…", 95 | "strategy_id": "sma1", 96 | "side": "BUY" | "SELL", 97 | "qty": 1.0, 98 | "tag": "optional string from strategy", 99 | "run_id": "…" 100 | } 101 | ``` 102 | 103 | **Fill** 104 | 105 | ```json 106 | { 107 | "ts": "…", 108 | "strategy_id": "sma1", 109 | "side": "BUY" | "SELL", 110 | "qty": 1.0, 111 | "fill_price": 101.25, 112 | "slippage_bps": 1.0, 113 | "commission": 0.50, 114 | "latency_ms": 50, 115 | "order_tag": "…", 116 | "run_id": "…" 117 | } 118 | ``` 119 | 120 | **Position/P\&L snapshot** 121 | 122 | ```json 123 | { 124 | "ts": "…", 125 | "pos": 3.0, 126 | "cash": 9993.25, 127 | "last_price": 101.25, 128 | "unrealized": -12.50, 129 | "realized": 8.75, 130 | "equity": 9989.50, 131 | "run_id": "…" 132 | } 133 | ``` 134 | 135 | --- 136 | 137 | ## 5) Market Simulator (Vasicek/OU) 138 | 139 | **Model**: $dX_t = \kappa(\theta - X_t)\,dt + \sigma\,dW_t$ 140 | 141 | **Discretization** (exact OU or Euler; MVP uses exact): 142 | 143 | * Exact step for Δt: $X_{t+Δ} = \theta + (X_t - \theta) e^{-\kappa Δ} + \sigma \sqrt{\tfrac{1 - e^{-2\kappa Δ}}{2\kappa}}\, \varepsilon$, $\varepsilon\sim\mathcal{N}(0,1)$. 144 | 145 | **Mapping to price**: 146 | 147 | * Option A (default): $P_t = P_0 \cdot e^{X_t}$ for positivity. 148 | * Option B: $P_t = P_0 + X_t$ with floor at $>0$ (teaching: mean reversion around $P_0$). 149 | 150 | **Inter‑arrival times**: 151 | 152 | * **Fixed**: constant Δt (e.g., 200 ms). 153 | * **Random**: exponential with mean μ (Poisson arrivals). UI shows a **speed** slider that rescales μ. 154 | 155 | **Controls (UI + YAML)**: `kappa, theta, sigma, P0, x0, dt_mode={fixed,poisson}, dt_fixed_ms, dt_mean_ms, seed`. 156 | 157 | --- 158 | 159 | ## 6) Strategy Plug‑in API (tiny, file‑based) 160 | 161 | **File**: `strategies//strategy.py` 162 | 163 | **Minimal API** 164 | 165 | ```python 166 | # strategy.py (MVP API) 167 | 168 | NAME = "SMA Crossover" 169 | PARAMS = {"fast": 20, "slow": 50, "qty": 1} 170 | 171 | def init(ctx): 172 | """Called once on strategy start. `ctx` exposes: 173 | - ctx.get_param(name, default) 174 | - ctx.set_state(key, value) / ctx.get_state(key, default) 175 | - ctx.indicator.SMA(window) 176 | - ctx.place_market_order(side: str, qty: float, tag: str | None) 177 | """ 178 | 179 | def on_tick(ctx, tick): 180 | """Called on each incoming Tick. Compute signals and place orders. 181 | `tick` is a dict-like (`tick["price"]`, `tick["ts"]`).""" 182 | 183 | def on_stop(ctx): 184 | """Called on strategy shutdown (optional).""" 185 | ``` 186 | 187 | **Hot‑reload**: Strategy process restarts upon file change (debounced). State may be optionally rehydrated from last snapshot (MVP: fresh start). 188 | 189 | **Safety**: Separate process; limited APIs via `ctx`; no network/file access in MVP beyond strategy folder (best‑effort; trust model is local teaching). 190 | 191 | --- 192 | 193 | ## 7) Broker/Execution (MVP) 194 | 195 | * **Order type**: MARKET only. 196 | * **Latency**: constant `latency_ms` before fill. 197 | * **Slippage**: `slippage_bps` applied to last price toward worse side. 198 | * **Costs**: `commission_fixed` + `commission_bps * notional`. 199 | * **Fill rule**: All‑or‑nothing, immediate after latency. 200 | * **Positioning**: Net quantity; short allowed; no margin model (equity can go negative in MVP). 201 | 202 | --- 203 | 204 | ## 8) Portfolio, P/L & Metrics 205 | 206 | * **P/L**: Mark‑to‑market each tick; realized on fills; equity = cash + pos × price. 207 | * **Rolling metrics** (configurable window): 208 | 209 | * Max drawdown & duration (from equity curve) 210 | * Sharpe (annualized from per‑tick/second returns with user‑set conversion) 211 | * Win rate (# profitable trades / total) 212 | * Avg trade P/L, avg hold time 213 | * Exposure (% time pos ≠ 0) 214 | * **Snapshot frequency**: every N ticks and on events. 215 | 216 | --- 217 | 218 | ## 9) Streamlit UI (MVP) 219 | 220 | **Layout** 221 | 222 | * **Sidebar**: simulation controls (Start/Pause/Reset), seed, speed, Δt mode, Vasicek params, slippage/costs/latency, strategy picker, hot‑reload toggle, export. 223 | * **Top KPIs**: Equity, P/L (realized/unrealized), Drawdown, Sharpe, Exposure, Win rate, Trades. 224 | * **Main charts (Plotly)**: 225 | 226 | 1. Price + indicators + trade markers (arrows for BUY/SELL) 227 | 2. Equity/P\&L curve 228 | 3. Histogram of returns (last N) 229 | * **Tables**: Recent trades (time, side, qty, price, cost, tag); Config snapshot. 230 | * **Status**: Event rate (ticks/sec), **ZMQ** socket status (connected, HWM), strategy health. 231 | 232 | **Update cadence**: UI throttled to \~10 fps; **ticks SUB uses `CONFLATE=1`** so it renders the latest tick without back‑pressuring the simulator. Background loops process all fills/PNL updates at full speed. 233 | 234 | --- 235 | 236 | ## 10) CLI Tools (for local + LLM demos) 237 | 238 | * `sim run --config config.yaml [--headless]` → run simulation; headless streams logs/metrics to stdout. 239 | * `sim new-strategy my_sma` → scaffold `strategies/my_sma/` with template. 240 | * `sim eval --config config.yaml --seed 42 --duration 60s --out out/` → deterministic replay; writes CSV/JSONL. 241 | * `sim doctor strategies/my_sma/strategy.py` → static checks for API compliance. 242 | 243 | --- 244 | 245 | ## 11) Configuration (YAML) 246 | 247 | ```yaml 248 | transport: 249 | type: zmq 250 | endpoints: 251 | ticks_pub: tcp://127.0.0.1:5555 252 | orders_push: tcp://127.0.0.1:5556 253 | fills_pub: tcp://127.0.0.1:5557 254 | hwm: 255 | ticks_pub: 20000 256 | orders: 20000 257 | fills_pub: 20000 258 | conflate: 259 | ui_ticks_sub: true 260 | 261 | model: 262 | type: vasicek 263 | kappa: 0.8 264 | theta: 0.0 265 | sigma: 0.2 266 | P0: 100.0 267 | x0: 0.0 268 | schedule: 269 | mode: poisson # fixed | poisson 270 | dt_fixed_ms: 200 271 | dt_mean_ms: 150 272 | speed: 1.0 # multiplier 273 | execution: 274 | latency_ms: 50 275 | slippage_bps: 1.0 276 | commission_bps: 0.5 277 | commission_fixed: 0.0 278 | strategy: 279 | path: strategies/sma_crossover/strategy.py 280 | params: {fast: 20, slow: 50, qty: 1} 281 | run: 282 | seed: 42 283 | duration_s: 0 # 0 means unlimited until Stop 284 | export_dir: runs/last 285 | ui: 286 | throttle_fps: 10 287 | ``` 288 | 289 | --- 290 | 291 | ## 12) Logging & Persistence 292 | 293 | * **Events**: JSONL (`runs//events.jsonl`) with all Tick/Order/Fill/PnLUpdate. 294 | * **Trades**: CSV (`runs//trades.csv`). 295 | * **Config**: Store effective YAML + git commit (if repo) for provenance. 296 | * **Repro**: `sim eval` replays with same seed and config. 297 | 298 | --- 299 | 300 | ## 13) Testing Strategy (PyTest) 301 | 302 | * **Unit**: OU step exactness (mean/variance vs theory), SMA signals, P/L arithmetic, slippage/costs application. 303 | * **Property-based**: No NaN equity; equity continuity; idempotent replay with fixed seed. 304 | * **Integration**: End‑to‑end 5‑second run asserting trade count and metric bounds. 305 | * **Transport smoke**: ZMQ sockets bind/connect; SUB receives ticks; PULL receives orders; PUB/PUB topics filter correctly. 306 | 307 | --- 308 | 309 | ## 14) Performance & Reliability 310 | 311 | * Async event loop; **ZeroMQ** sockets sized with HWM; UI ticks use conflation to avoid rendering backlog. 312 | * Bounded order queues (ZMQ PUSH/PULL) with metrics; warn on slow strategy (> latency budget). 313 | * Clean shutdown on Ctrl‑C; flush logs; write final snapshot; ZMQ context terminated cleanly. 314 | 315 | --- 316 | 317 | ## 15) Security & Sandboxing (local teaching) 318 | 319 | * Strategy executed in a separate process with a constrained API. 320 | * MVP allows local imports; later: optional restricted mode (no network/files) via audit hooks or subprocess policy. 321 | * **Local-only transport** via `127.0.0.1`; no external exposure by default. 322 | 323 | --- 324 | 325 | ## 16) Milestones & Acceptance Criteria 326 | 327 | **M0 – Skeleton (Day 1)** 328 | 329 | * Repo scaffold, env, CLI bootstrap; black/ruff/pytest configured; ZMQ context and sockets wired. 330 | 331 | **M1 – Simulator + UI (Day 2–3)** 332 | 333 | * OU price stream; fixed/poisson intervals; Streamlit chart updates; start/pause/reset; ticks published via ZMQ. 334 | 335 | **M2 – Strategy Plug‑in (Day 4)** 336 | 337 | * Strategy process, API (`init/on_tick/on_stop`), SMA example; hot‑reload; orders via PUSH/PULL. 338 | 339 | **M3 – Broker & P/L (Day 5)** 340 | 341 | * MARKET fills with latency/slippage/costs; live P/L; trades table; fills PUB to UI/strategy. 342 | 343 | **M4 – Metrics & Export (Day 6)** 344 | 345 | * Rolling stats; JSONL/CSV export; repro via seed; `sim eval` CLI. 346 | 347 | **M5 – Polish (Day 7)** 348 | 349 | * Error handling, UI KPIs (socket health/HWM), `sim doctor`, docs. 350 | 351 | **Acceptance (MVP)** 352 | 353 | * Run locally: start sim, load SMA strategy, observe trades & P/L updating in UI, export artifacts, replay deterministically via CLI and obtain identical trades. 354 | 355 | --- 356 | 357 | ## 17) Project Layout 358 | 359 | ``` 360 | rt-sim/ 361 | README.md 362 | requirements.txt 363 | pyproject.toml 364 | rt_sim/ 365 | __init__.py 366 | app_streamlit.py 367 | cli.py 368 | transport.py # ZMQ transport abstraction 369 | models.py # pydantic schemas 370 | simulator.py 371 | strategy_host.py # separate process stub 372 | broker.py 373 | portfolio.py 374 | metrics.py 375 | recorder.py 376 | utils.py 377 | strategies/ 378 | sma_crossover/ 379 | strategy.py 380 | configs/ 381 | default.yaml 382 | runs/ 383 | … 384 | tests/ 385 | test_ou.py 386 | test_broker.py 387 | test_strategy_api.py 388 | test_pnl.py 389 | test_transport.py 390 | ``` 391 | 392 | --- 393 | 394 | ## 18) Dependencies & Environment 395 | 396 | * Python 3.11+ 397 | * `streamlit`, `plotly`, `numpy`, `pandas`, `pydantic`, `scipy`, `pyyaml`, `click`, `pytest`, `ruff`, `black`, **`pyzmq`**. 398 | * Optional: `uvloop` (non‑Windows). 399 | 400 | **Setup** 401 | 402 | ```bash 403 | python -m venv .venv && source .venv/bin/activate 404 | pip install -r requirements.txt 405 | streamlit run rt_sim/app_streamlit.py 406 | ``` 407 | 408 | --- 409 | 410 | ## 19) Example Strategy Template (SMA Crossover) 411 | 412 | ```python 413 | NAME = "SMA Crossover" 414 | PARAMS = {"fast": 20, "slow": 50, "qty": 1} 415 | 416 | def init(ctx): 417 | f, s = ctx.get_param("fast", 20), ctx.get_param("slow", 50) 418 | ctx.set_state("fast", f) 419 | ctx.set_state("slow", s) 420 | ctx.set_state("qty", ctx.get_param("qty", 1)) 421 | ctx.fast = ctx.indicator.SMA(f) 422 | ctx.slow = ctx.indicator.SMA(s) 423 | 424 | def on_tick(ctx, tick): 425 | p = tick["price"] 426 | f, s = ctx.fast.update(p), ctx.slow.update(p) 427 | if f is None or s is None: 428 | return 429 | pos = ctx.position() # current signed qty 430 | if f > s and pos <= 0: 431 | ctx.place_market_order("BUY", abs(pos) + ctx.get_state("qty", 1), tag="bullish") 432 | elif f < s and pos >= 0: 433 | ctx.place_market_order("SELL", abs(pos) + ctx.get_state("qty", 1), tag="bearish") 434 | 435 | def on_stop(ctx): 436 | pass 437 | ``` 438 | 439 | *(Template provided for API illustration; actual code generated in implementation stage.)* 440 | 441 | --- 442 | 443 | ## 20) Future Extensions (Post‑MVP) 444 | 445 | * Alternative models (GBM, Heston-lite, jumps), regime switching. 446 | * Multi‑asset, correlations; portfolio constraints; risk overlays. 447 | * Limit/stop orders, partial fills, VWAP/TWAP execution. 448 | * Latency distributions; network jitter; dropped ticks. 449 | * Strategy timeouts, resource quotas; richer sandboxing. 450 | * WebSocket event stream; Electron/desktop packaging. 451 | 452 | --- 453 | 454 | ## 21) LLM Demo Notes (Codex/GPT‑5 CLI) 455 | 456 | * Prompts to generate: new strategies, metric modules, or model variants. 457 | * Use `sim new-strategy` then ask LLM to implement `on_tick` logic per the API. 458 | * Use `sim eval` with fixed seeds to compare LLM‑generated variants reproducibly. 459 | * Showcase **ZMQ patterns** (PUB/SUB, PUSH/PULL) in prompts so codegen targets the correct transport calls. 460 | 461 | --- 462 | 463 | ## 22) Open Design Questions (tracked in repo issues) 464 | 465 | 1. Prefer exact OU step (scipy) or Euler for pedagogical transparency? 466 | 2. Default price mapping: exponential vs additive? 467 | 3. Hot-reload state rehydration: reset vs checkpoint restore? 468 | 4. Commission/slippage defaults for teaching (bps values)? 469 | 5. Rolling metrics windows & annualization factor (ticks→time conversion)? 470 | 6. Strategy sandbox restrictions (file/network) in MVP vs later? 471 | 7. Deterministic wall‑clock vs simulated clock semantics for latency? 472 | 8. First multi-asset extension (2 correlated OU) and multi-topic ZMQ layout? 473 | -------------------------------------------------------------------------------- /rt_sim/app_streamlit.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import json 4 | import sys 5 | import threading 6 | import time 7 | from collections import deque 8 | 9 | import plotly.graph_objs as go 10 | import streamlit as st 11 | import zmq 12 | from queue import Queue, Full, Empty 13 | from pathlib import Path 14 | import os 15 | import json as _json 16 | import signal as _signal 17 | import subprocess 18 | import multiprocessing as mp 19 | 20 | # Background listener writes into a Queue object passed at thread start. 21 | # Keep the reference in session_state so reruns use the same queue. 22 | _LAST_THREAD_ERROR: str | None = None 23 | # Global log buffer for strategy host (avoid touching st.session_state from threads) 24 | STRAT_LOGS: deque[str] = deque(maxlen=2000) 25 | STRAT_LOG_LOCK = threading.Lock() 26 | PID_REG_PATH: Path = Path(__file__).resolve().parents[1] / "runs/strategy_hosts.json" 27 | 28 | 29 | def read_pid_registry() -> list[int]: 30 | try: 31 | data = _json.loads(PID_REG_PATH.read_text()) 32 | if isinstance(data, list): 33 | return [int(x) for x in data] 34 | except Exception: 35 | pass 36 | return [] 37 | 38 | 39 | def write_pid_registry(pids: list[int]) -> None: 40 | try: 41 | PID_REG_PATH.parent.mkdir(parents=True, exist_ok=True) 42 | PID_REG_PATH.write_text(_json.dumps([int(x) for x in pids])) 43 | except Exception: 44 | pass 45 | 46 | 47 | def register_pid(pid: int) -> None: 48 | pids = read_pid_registry() 49 | if pid not in pids: 50 | pids.append(pid) 51 | write_pid_registry(pids) 52 | 53 | 54 | def unregister_pid(pid: int) -> None: 55 | pids = read_pid_registry() 56 | if pid in pids: 57 | pids.remove(pid) 58 | write_pid_registry(pids) 59 | 60 | try: 61 | from rt_sim.transport import Transport 62 | from rt_sim.utils import load_config, new_run_id 63 | except ModuleNotFoundError: 64 | # Fallback: add project root to sys.path when running via `streamlit run` 65 | import sys 66 | from pathlib import Path 67 | 68 | sys.path.append(str(Path(__file__).resolve().parents[1])) 69 | from rt_sim.transport import Transport 70 | from rt_sim.utils import load_config, new_run_id 71 | 72 | # Top-level process entry helpers for local demo controls 73 | def _proc_broker_entry(cfg: dict, run_id: str) -> None: 74 | from rt_sim.transport import Transport 75 | from rt_sim.broker import run as run_broker 76 | 77 | t = Transport( 78 | hwm_ticks=int(cfg["transport"]["hwm"]["ticks_pub"]), 79 | hwm_orders=int(cfg["transport"]["hwm"]["orders"]), 80 | hwm_fills=int(cfg["transport"]["hwm"]["fills_pub"]), 81 | ) 82 | run_broker(cfg, t, run_id) 83 | 84 | 85 | def _proc_sim_entry(cfg: dict, run_id: str) -> None: 86 | from rt_sim.transport import Transport 87 | from rt_sim.simulator import run as run_sim 88 | 89 | t = Transport( 90 | hwm_ticks=int(cfg["transport"]["hwm"]["ticks_pub"]), 91 | hwm_orders=int(cfg["transport"]["hwm"]["orders"]), 92 | hwm_fills=int(cfg["transport"]["hwm"]["fills_pub"]), 93 | ) 94 | run_sim(cfg, t, run_id) 95 | 96 | 97 | def _proc_strategy_entry( 98 | cfg: dict, 99 | run_id: str, 100 | strategy_path: str, 101 | strategy_id: str | None, 102 | params_json: str | None, 103 | topic: str, 104 | conflate: bool, 105 | ) -> None: 106 | from rt_sim.transport import Transport 107 | from rt_sim.strategy_host import run as run_host 108 | import json as _json 109 | 110 | t = Transport( 111 | hwm_ticks=int(cfg["transport"]["hwm"]["ticks_pub"]), 112 | hwm_orders=int(cfg["transport"]["hwm"]["orders"]), 113 | hwm_fills=int(cfg["transport"]["hwm"]["fills_pub"]), 114 | ) 115 | params = None 116 | if params_json: 117 | try: 118 | params = _json.loads(params_json) 119 | except Exception: 120 | params = None 121 | run_host(cfg, t, run_id, strategy_path, strategy_id=strategy_id, params_override=params, topic=topic, conflate=conflate) 122 | 123 | 124 | st.set_page_config(page_title="algosim — Ticks", layout="wide") 125 | 126 | 127 | def ensure_state(): 128 | if "ticks" not in st.session_state: 129 | st.session_state.ticks = deque(maxlen=2000) 130 | if "fills" not in st.session_state: 131 | st.session_state.fills = deque(maxlen=500) 132 | if "pnl" not in st.session_state: 133 | st.session_state.pnl = deque(maxlen=2000) # (ts_wall, equity) 134 | if "pos_series" not in st.session_state: 135 | st.session_state.pos_series = deque(maxlen=2000) # (ts_wall, pos) 136 | if "initial_cash" not in st.session_state: 137 | cfg0 = st.session_state.get("cfg", load_config(None)) 138 | try: 139 | st.session_state.initial_cash = float(cfg0.get("portfolio", {}).get("initial_cash", 100000.0)) 140 | except Exception: 141 | st.session_state.initial_cash = 100000.0 142 | if "listener_thread" not in st.session_state: 143 | st.session_state.listener_thread = None 144 | if "listener_event" not in st.session_state: 145 | st.session_state.listener_event = threading.Event() 146 | if "fills_thread" not in st.session_state: 147 | st.session_state.fills_thread = None 148 | if "fills_event" not in st.session_state: 149 | st.session_state.fills_event = threading.Event() 150 | if "conflate" not in st.session_state: 151 | st.session_state.conflate = False 152 | if "queue" not in st.session_state: 153 | # size 1 when conflating, large otherwise 154 | st.session_state.queue = Queue(maxsize=(1 if st.session_state.conflate else 10000)) 155 | if "test_recv_stats" not in st.session_state: 156 | st.session_state.test_recv_stats = None 157 | if "test_fills_stats" not in st.session_state: 158 | st.session_state.test_fills_stats = None 159 | if "fills_queue" not in st.session_state: 160 | st.session_state.fills_queue = Queue(maxsize=10000) 161 | if "auto_refresh" not in st.session_state: 162 | st.session_state.auto_refresh = True 163 | if "refresh_hz" not in st.session_state: 164 | st.session_state.refresh_hz = 1 165 | if "pos" not in st.session_state: 166 | st.session_state.pos = 0.0 167 | if "cash" not in st.session_state: 168 | # Initialize cash from config portfolio.initial_cash if available 169 | cfg0 = st.session_state.get("cfg", load_config(None)) 170 | try: 171 | st.session_state.cash = float(cfg0.get("portfolio", {}).get("initial_cash", 100000.0)) 172 | except Exception: 173 | st.session_state.cash = 100000.0 174 | if "last_price" not in st.session_state: 175 | st.session_state.last_price = None 176 | if "last_portfolio" not in st.session_state: 177 | st.session_state.last_portfolio = None 178 | if "last_run_id" not in st.session_state: 179 | st.session_state.last_run_id = None 180 | if "proc_broker" not in st.session_state: 181 | st.session_state.proc_broker = None 182 | if "proc_sim" not in st.session_state: 183 | st.session_state.proc_sim = None 184 | if "proc_strategy" not in st.session_state: 185 | st.session_state.proc_strategy = None 186 | if "strategy_path" not in st.session_state: 187 | st.session_state.strategy_path = "strategies/mean_reversion/strategy.py" 188 | if "strategy_code" not in st.session_state: 189 | try: 190 | st.session_state.strategy_code = Path(st.session_state.strategy_path).read_text() 191 | except Exception: 192 | st.session_state.strategy_code = "" 193 | if "strategy_log_thread" not in st.session_state: 194 | st.session_state.strategy_log_thread = None 195 | if "strategy_log_event" not in st.session_state: 196 | st.session_state.strategy_log_event = threading.Event() 197 | if "tick_chart" not in st.session_state: 198 | base_fig = go.Figure( 199 | data=[ 200 | go.Scattergl(name="Price", mode="lines", x=[], y=[], line=dict(color="#3498db", width=2)), 201 | go.Scattergl( 202 | name="Buys", 203 | mode="markers", 204 | x=[], 205 | y=[], 206 | marker=dict(symbol="triangle-up", color="#2ecc71", size=12, line=dict(color="white", width=1)), 207 | hoverinfo="text", 208 | ), 209 | go.Scattergl( 210 | name="Sells", 211 | mode="markers", 212 | x=[], 213 | y=[], 214 | marker=dict(symbol="triangle-down", color="#e74c3c", size=12, line=dict(color="white", width=1)), 215 | hoverinfo="text", 216 | ), 217 | ], 218 | layout=go.Layout(uirevision="price_stream", height=500, margin=dict(l=10, r=10, t=10, b=10)), 219 | ) 220 | st.session_state.tick_chart = {"fig": base_fig, "datarevision": 0} 221 | # PID registry helpers are provided at module scope 222 | 223 | 224 | def start_listener(cfg): 225 | event: threading.Event = st.session_state.listener_event 226 | # If an old listener is running, stop it to recreate with current conflate setting/queue size. 227 | if event.is_set(): 228 | stop_listener() 229 | time.sleep(0.05) 230 | 231 | # Recreate queue respecting current conflate setting 232 | st.session_state.queue = Queue(maxsize=(1 if st.session_state.conflate else 10000)) 233 | q: Queue = st.session_state.queue 234 | event.set() 235 | 236 | def _loop(ev: threading.Event, conflate: bool, q_out: Queue): 237 | global _LAST_THREAD_ERROR 238 | try: 239 | t = Transport(hwm_ticks=int(cfg["transport"]["hwm"]["ticks_pub"])) 240 | sub = t.connect_sub( 241 | cfg["transport"]["endpoints"]["ticks_pub"], topic="X", conflate=conflate 242 | ) 243 | poller = zmq.Poller() 244 | poller.register(sub, zmq.POLLIN) 245 | while ev.is_set(): 246 | socks = dict(poller.poll(timeout=100)) 247 | if sub in socks and socks[sub] == zmq.POLLIN: 248 | _, payload = t.recv_json(sub) 249 | # Push to module queue; main thread will drain 250 | try: 251 | q_out.put_nowait((payload.get("ts_wall", time.time()), payload.get("price"), payload.get("seq"))) 252 | except Full: 253 | # Keep only the latest: drop one and insert 254 | try: 255 | q_out.get_nowait() 256 | except Empty: 257 | pass 258 | try: 259 | q_out.put_nowait((payload.get("ts_wall", time.time()), payload.get("price"), payload.get("seq"))) 260 | except Full: 261 | pass 262 | sub.close(0) 263 | except Exception as e: 264 | _LAST_THREAD_ERROR = f"Listener thread error: {e}" 265 | 266 | th = threading.Thread(target=_loop, args=(event, st.session_state.conflate, q), daemon=True) 267 | th.start() 268 | st.session_state.listener_thread = th 269 | 270 | # Start fills listener (subscribe to all topics) 271 | fev: threading.Event = st.session_state.fills_event 272 | if fev.is_set(): 273 | stop_fills() 274 | time.sleep(0.05) 275 | st.session_state.fills_queue = Queue(maxsize=10000) 276 | fq: Queue = st.session_state.fills_queue 277 | fev.set() 278 | 279 | def _fills_loop(ev: threading.Event, fq_out: Queue): 280 | global _LAST_THREAD_ERROR 281 | try: 282 | t2 = Transport(hwm_ticks=int(cfg["transport"]["hwm"]["fills_pub"])) 283 | sub2 = t2.connect_sub(cfg["transport"]["endpoints"]["fills_pub"], topic="", conflate=False) 284 | poller2 = zmq.Poller(); poller2.register(sub2, zmq.POLLIN) 285 | while ev.is_set(): 286 | socks2 = dict(poller2.poll(timeout=100)) 287 | if sub2 in socks2 and socks2[sub2] == zmq.POLLIN: 288 | topic, payload = t2.recv_json(sub2) 289 | try: 290 | fq_out.put_nowait((topic, payload)) 291 | except Full: 292 | try: 293 | fq_out.get_nowait() 294 | fq_out.put_nowait((topic, payload)) 295 | except Exception: 296 | pass 297 | sub2.close(0) 298 | except Exception as e: 299 | _LAST_THREAD_ERROR = f"Fills listener error: {e}" 300 | 301 | fth = threading.Thread(target=_fills_loop, args=(fev, fq), daemon=True) 302 | fth.start() 303 | st.session_state.fills_thread = fth 304 | 305 | 306 | def stop_listener(): 307 | event: threading.Event = st.session_state.listener_event 308 | event.clear() 309 | th = st.session_state.listener_thread 310 | if th and th.is_alive(): 311 | th.join(timeout=0.5) 312 | st.session_state.listener_thread = None 313 | 314 | stop_fills() 315 | 316 | 317 | def stop_fills(): 318 | fev: threading.Event = st.session_state.fills_event 319 | fev.clear() 320 | fth = st.session_state.fills_thread 321 | if fth and fth.is_alive(): 322 | fth.join(timeout=0.5) 323 | st.session_state.fills_thread = None 324 | 325 | 326 | def main(): 327 | ensure_state() 328 | st.title("algosim — Real-Time Ticks (MVP)") 329 | 330 | cfg = st.session_state.get("cfg", load_config(None)) 331 | 332 | with st.sidebar: 333 | st.header("Connection") 334 | cols = st.columns(2) 335 | if cols[0].button("Start SUB"): 336 | start_listener(cfg) 337 | if cols[1].button("Stop SUB"): 338 | stop_listener() 339 | st.caption("Use the Admin tab for diagnostics and advanced controls.") 340 | 341 | tab_ticks, tab_fills, tab_pnl, tab_strategy, tab_admin = st.tabs( 342 | ["Ticks", "Fills / Orders", "P&L", "Strategy", "Admin"] 343 | ) 344 | 345 | with tab_ticks: 346 | # Render mode selector on top, default to Chart 347 | render_mode = st.radio("Render mode", ["Chart", "Text"], index=0, horizontal=True) 348 | st.caption("Subscribe to ticks; choose text or chart rendering below.") 349 | placeholder = st.empty() 350 | 351 | # Simple autorefresh loop 352 | chart_refresh_ms = int(1000 / max(1, int(st.session_state.get("refresh_hz", 5)))) 353 | # Drain incoming queue into ticks before drawing 354 | drained = 0 355 | try: 356 | while True: 357 | ts, price, seq = st.session_state.queue.get_nowait() 358 | st.session_state.ticks.append((ts, price)) 359 | st.session_state.last_price = price 360 | # Update arrival times and gap metrics 361 | now = time.time() 362 | if "arrival_times" not in st.session_state: 363 | st.session_state.arrival_times = deque(maxlen=500) 364 | st.session_state.arrival_times.append(now) 365 | last_seq = st.session_state.get("last_seq") 366 | if seq is not None: 367 | if last_seq is not None and seq != last_seq + 1: 368 | st.session_state["gap_count"] = st.session_state.get("gap_count", 0) + 1 369 | st.session_state["last_seq"] = seq 370 | drained += 1 371 | except Empty: 372 | pass 373 | if drained: 374 | import datetime as _dt 375 | 376 | st.session_state.last_recv_ts = _dt.datetime.now().isoformat(timespec="seconds") 377 | 378 | # Drain fills queue 379 | fdrained = 0 380 | try: 381 | while True: 382 | topic, payload = st.session_state.fills_queue.get_nowait() 383 | if topic == "portfolio" or payload.get("type") == "portfolio": 384 | st.session_state.last_portfolio = payload 385 | st.session_state.pos = float(payload.get("pos", st.session_state.pos)) 386 | st.session_state.cash = float(payload.get("cash", st.session_state.cash)) 387 | last_px = payload.get("last_price", st.session_state.last_price) 388 | if last_px is not None: 389 | st.session_state.last_price = float(last_px) 390 | tsw = float(payload.get("ts_wall", time.time())) 391 | eq = float(payload.get("equity", st.session_state.cash + st.session_state.pos * float(st.session_state.last_price or 0.0))) 392 | st.session_state.pnl.append((tsw, eq)) 393 | st.session_state.pos_series.append((tsw, st.session_state.pos)) 394 | run_id = payload.get("run_id") 395 | if run_id: 396 | st.session_state.last_run_id = run_id 397 | else: 398 | tsf = float(payload.get("ts_wall", time.time())) 399 | st.session_state.fills.append((tsf, payload)) 400 | pos_after = payload.get("pos_after") 401 | cash_after = payload.get("cash_after") 402 | equity_after = payload.get("equity_after") 403 | if pos_after is not None: 404 | st.session_state.pos = float(pos_after) 405 | else: 406 | side = str(payload.get("side", "")).upper() 407 | qty = float(payload.get("qty", 0.0)) 408 | if side == "BUY": 409 | st.session_state.pos += qty 410 | elif side == "SELL": 411 | st.session_state.pos -= qty 412 | if cash_after is not None: 413 | st.session_state.cash = float(cash_after) 414 | else: 415 | qty = float(payload.get("qty", 0.0)) 416 | price = float(payload.get("fill_price", 0.0)) 417 | commission = float(payload.get("commission", 0.0)) 418 | side = str(payload.get("side", "")).upper() 419 | if side == "BUY": 420 | st.session_state.cash -= price * qty + commission 421 | elif side == "SELL": 422 | st.session_state.cash += price * qty - commission 423 | last_px = payload.get("fill_price", st.session_state.last_price) 424 | if last_px is not None: 425 | st.session_state.last_price = float(last_px) 426 | if equity_after is not None: 427 | eq_val = float(equity_after) 428 | else: 429 | px = st.session_state.last_price if st.session_state.last_price is not None else 0.0 430 | eq_val = st.session_state.cash + st.session_state.pos * float(px) 431 | st.session_state.pnl.append((tsf, eq_val)) 432 | st.session_state.pos_series.append((tsf, st.session_state.pos)) 433 | fdrained += 1 434 | run_id = payload.get("run_id") 435 | if run_id: 436 | st.session_state.last_run_id = run_id 437 | except Empty: 438 | pass 439 | 440 | with placeholder.container(): 441 | data = list(st.session_state.ticks) 442 | if not data: 443 | st.info("No ticks yet. Start the simulator and then Start SUB.") 444 | else: 445 | if render_mode == "Text": 446 | # Render as plain text lines (ts_wall ISO-ish, price) 447 | import datetime as _dt 448 | 449 | def _fmt(ts: float, px: float) -> str: 450 | ts_str = _dt.datetime.fromtimestamp(ts).isoformat(timespec="milliseconds") 451 | return f"{ts_str} price={px:.5f}" 452 | 453 | lines = [_fmt(ts, px) for ts, px in data[-500:]] # show last 500 lines 454 | st.text("\n".join(lines)) 455 | st.caption(f"Tick count: {len(data)} (showing last {min(len(data), 500)})") 456 | else: 457 | # Convert epoch seconds to ISO strings for display on x-axis 458 | import datetime as _dt 459 | 460 | x_raw, y_vals = zip(*data) 461 | x = [_dt.datetime.fromtimestamp(ts) for ts in x_raw] 462 | chart_state = st.session_state.tick_chart 463 | fig = chart_state["fig"] 464 | fig.data[0].x = list(x) 465 | fig.data[0].y = list(y_vals) 466 | 467 | buys = [pt for pt in st.session_state.fills if str(pt[1].get("side", "")).upper() == "BUY"] 468 | sells = [pt for pt in st.session_state.fills if str(pt[1].get("side", "")).upper() == "SELL"] 469 | 470 | def _prep_points(points): 471 | xs, ys, texts = [], [], [] 472 | for ts_fill, fill_payload in points: 473 | px_fill = fill_payload.get("fill_price") 474 | if px_fill is None: 475 | continue 476 | xs.append(_dt.datetime.fromtimestamp(ts_fill)) 477 | ys.append(px_fill) 478 | qty_fill = float(fill_payload.get("qty", 0.0)) 479 | pos_after = float(fill_payload.get("pos_after", st.session_state.pos)) 480 | side_txt = str(fill_payload.get("side", "")).upper() 481 | texts.append( 482 | f"{side_txt} {qty_fill:g} @ {float(px_fill):.5f}
Position: {pos_after:,.2f}" 483 | ) 484 | return xs, ys, texts 485 | 486 | buy_x, buy_y, buy_text = _prep_points(buys) 487 | sell_x, sell_y, sell_text = _prep_points(sells) 488 | fig.data[1].x = list(buy_x) 489 | fig.data[1].y = list(buy_y) 490 | fig.data[1].hovertext = list(buy_text) 491 | fig.data[2].x = list(sell_x) 492 | fig.data[2].y = list(sell_y) 493 | fig.data[2].hovertext = list(sell_text) 494 | 495 | chart_state["datarevision"] += 1 496 | fig.layout.datarevision = chart_state["datarevision"] 497 | 498 | st.plotly_chart(fig, use_container_width=True, config={"displayModeBar": True}) 499 | st.caption(f"Tick count: {len(data)}") 500 | 501 | with tab_fills: 502 | # Manual orders on top 503 | st.subheader("Manual Orders") 504 | col1, col2, col3 = st.columns([1,1,2]) 505 | qty = col1.number_input("Qty", min_value=0.0, value=1.0, step=1.0, format="%f") 506 | tag = col2.text_input("Tag", value="manual") 507 | waiting_for_price = st.session_state.last_price is None 508 | if waiting_for_price: 509 | st.info("Waiting for first tick price — start simulator and ensure ticks flow before sending orders.") 510 | def _send_order(side: str): 511 | try: 512 | cfg_loc = st.session_state.get("cfg", load_config(None)) 513 | t3 = Transport( 514 | hwm_ticks=int(cfg_loc["transport"]["hwm"]["ticks_pub"]), 515 | hwm_orders=int(cfg_loc["transport"]["hwm"]["orders"]), 516 | hwm_fills=int(cfg_loc["transport"]["hwm"]["fills_pub"]), 517 | ) 518 | push = t3.connect_push(cfg_loc["transport"]["endpoints"]["orders_push"]) 519 | # Ensure message flushes before close 520 | import zmq as _zmq 521 | push.setsockopt(_zmq.LINGER, 500) 522 | payload = {"strategy_id": "ui", "side": side, "qty": float(qty), "tag": tag} 523 | Transport.send_json_push(push, payload) 524 | # tiny delay helps handshake 525 | time.sleep(0.01) 526 | push.close() 527 | st.success(f"Sent {side} {qty:g}") 528 | except Exception as e: 529 | st.error(f"Failed to send order: {e}") 530 | cbu, cse = st.columns(2) 531 | if cbu.button("BUY", disabled=waiting_for_price): 532 | _send_order("BUY") 533 | if cse.button("SELL", disabled=waiting_for_price): 534 | _send_order("SELL") 535 | 536 | # Fills list below 537 | st.subheader("Fills (latest)") 538 | if st.session_state.fills: 539 | lines = [] 540 | pos = 0.0 541 | for _, f in list(st.session_state.fills)[-200:]: 542 | side = f.get("side"); qty = float(f.get("qty", 0)) 543 | pos += qty if side == "BUY" else -qty 544 | ts_str = __import__("datetime").datetime.fromtimestamp(f.get("ts_wall", time.time())).isoformat(timespec="seconds") 545 | lines.append(f"{ts_str} {side} {qty:g} @ {float(f.get('fill_price', 0.0)):.5f} pos≈{pos:,.2f}") 546 | st.text_area( 547 | "Recent fills", 548 | value="\n".join(reversed(lines)), 549 | height=220, 550 | key="fills_latest_view", 551 | help="Latest fills with running position.", 552 | disabled=True, 553 | ) 554 | else: 555 | st.caption("No fills yet.") 556 | 557 | with tab_pnl: 558 | st.subheader("Live Position & P&L") 559 | pos = st.session_state.pos 560 | cash = st.session_state.cash 561 | last_px = st.session_state.last_price 562 | pos_value = (pos * float(last_px)) if last_px is not None else 0.0 563 | eq = cash + pos_value 564 | # Chart equity over time 565 | if st.session_state.pnl: 566 | import datetime as _dt 567 | from rt_sim.metrics import ( 568 | compute_drawdown, 569 | compute_sharpe_from_equity, 570 | compute_time_weighted_exposure, 571 | compute_trade_stats, 572 | compute_time_weighted_dollar_exposure, 573 | ) 574 | t_raw, eq_vals = zip(*list(st.session_state.pnl)) 575 | # Defer plotting until after KPIs so metrics appear above the chart 576 | 577 | # Compute rolling metrics on the visible equity curve 578 | dd, _, _ = compute_drawdown(list(eq_vals)) 579 | # Approx annualization: assume 1 Hz samples => 31,536,000 seconds/year 580 | # Scale per-step Sharpe with factor chosen conservatively (3600*24*252 ~ trading seconds) 581 | ann = float(st.session_state.get("ann_factor", 3600.0 * 24.0 * 252.0)) 582 | sharpe = compute_sharpe_from_equity(list(eq_vals), annualization_factor=ann) 583 | # Exposure based on time-weighted pos_series 584 | if st.session_state.pos_series: 585 | pt, pv = zip(*list(st.session_state.pos_series)) 586 | exposure = compute_time_weighted_exposure(list(pt), list(pv)) 587 | else: 588 | exposure = 0.0 589 | # Trade stats from fills 590 | fills_payloads = [f for _, f in list(st.session_state.fills)] 591 | tstats = compute_trade_stats(fills_payloads) 592 | win_rate = tstats.get("win_rate", 0.0) 593 | avg_pl = tstats.get("avg_trade_pl", 0.0) 594 | avg_hold = tstats.get("avg_hold_s", 0.0) 595 | 596 | metric_items = [ 597 | ("Position (qty)", f"{pos:,.2f}"), 598 | ("Position Value", f"${pos_value:,.2f}"), 599 | ("Cash", f"${cash:,.2f}"), 600 | ("Equity", f"${eq:,.2f}"), 601 | ("Max Drawdown", f"{dd*100:,.2f}%"), 602 | ("Sharpe (approx)", f"{sharpe:,.2f}"), 603 | ("Exposure", f"{exposure*100:,.1f}%"), 604 | ("Win Rate", f"{win_rate*100:,.1f}%"), 605 | ("Avg Trade P/L", f"${avg_pl:,.2f}"), 606 | ("Avg Hold (s)", f"{avg_hold:,.1f}"), 607 | ] 608 | # Dollar exposure (avg as % of initial cash) 609 | rel_dexp = None 610 | if st.session_state.ticks and st.session_state.pos_series: 611 | tt, tp = zip(*list(st.session_state.ticks)) 612 | pt, pv = zip(*list(st.session_state.pos_series)) 613 | try: 614 | rel_dexp = compute_time_weighted_dollar_exposure( 615 | list(tt), list(tp), list(zip(pt, pv)), float(st.session_state.initial_cash) 616 | ) 617 | except Exception: 618 | rel_dexp = 0.0 619 | if rel_dexp is not None: 620 | metric_items.append(("Dollar Exposure (avg)", f"{rel_dexp*100:,.1f}%")) 621 | 622 | per_row = 6 623 | card_style = "font-size:0.8rem; color:#a0a0a0; margin-bottom:0.15rem;" 624 | value_style = "font-size:1.05rem; font-weight:600; margin:0;" 625 | for i in range(0, len(metric_items), per_row): 626 | cols = st.columns(per_row) 627 | for col, (label, value) in zip(cols, metric_items[i : i + per_row]): 628 | col.markdown( 629 | f"
{label}
{value}
", 630 | unsafe_allow_html=True, 631 | ) 632 | 633 | # Now render equity chart below the KPIs 634 | x = [_dt.datetime.fromtimestamp(ts).isoformat(timespec="seconds") for ts in t_raw] 635 | figp = go.Figure(data=[go.Scatter(x=x, y=list(eq_vals), mode="lines", name="Equity")]) 636 | figp.update_layout(height=400, margin=dict(l=10, r=10, t=10, b=10)) 637 | st.plotly_chart(figp, use_container_width=True) 638 | else: 639 | st.caption("No P&L data yet. Send an order to create fills or wait for ticks.") 640 | 641 | with tab_strategy: 642 | st.subheader("Strategy Host") 643 | spath_in = st.text_input("Strategy path", value=st.session_state.strategy_path) 644 | base_dir = Path(__file__).resolve().parents[1] 645 | spath = Path(spath_in) 646 | if not spath.is_absolute(): 647 | spath = base_dir / spath 648 | st.session_state.strategy_path = str(spath) 649 | colsS = st.columns(2) 650 | if colsS[0].button("Load file"): 651 | try: 652 | st.session_state.strategy_code = Path(st.session_state.strategy_path).read_text() 653 | st.success("Loaded strategy file.") 654 | except Exception as e: 655 | st.error(f"Failed to load: {e}") 656 | if colsS[1].button("Save file"): 657 | try: 658 | Path(st.session_state.strategy_path).write_text(st.session_state.strategy_code) 659 | st.success("Saved strategy file.") 660 | except Exception as e: 661 | st.error(f"Failed to save: {e}") 662 | 663 | st.text_area("strategy.py", height=260, key="strategy_code") 664 | st.divider() 665 | st.subheader("Run Controls") 666 | sid = st.text_input("Strategy ID", value="sma1") 667 | topic_str = st.text_input("Tick topic (empty=all)", value="X") 668 | conflate = st.checkbox("Conflate latest only (ticks)", value=False, key="strategy_conflate") 669 | params_json = st.text_area("PARAMS override (JSON)", value="", placeholder='{"fast": 20, "slow": 50, "qty": 1, "threshold_bps": 15, "min_interval_s": 10}') 670 | stop_mode = st.radio( 671 | "Stop action", 672 | ("Flatten position", "Terminate only"), 673 | index=0, 674 | key="strategy_stop_mode", 675 | horizontal=True, 676 | help="Choose whether to send a flattening market order before shutting down the host.", 677 | ) 678 | flatten_on_stop = stop_mode == "Flatten position" 679 | sbtn = st.columns(2) 680 | if sbtn[0].button("Start strategy host"): 681 | try: 682 | # if already running 683 | proc = st.session_state.proc_strategy 684 | if proc and getattr(proc, "poll", lambda: None)() is None: 685 | st.warning("Strategy host already running") 686 | else: 687 | # Build subprocess command to capture stdout 688 | cfg_path = st.session_state.get("cfg_path", "configs/default.yaml") 689 | # Resolve cfg path relative to project root if needed 690 | cfg_p = Path(cfg_path) 691 | if not cfg_p.is_absolute(): 692 | cfg_p = base_dir / cfg_p 693 | conflate_flag = "--conflate" if conflate else "--no-conflate" 694 | cmd = [ 695 | sys.executable, 696 | "-u", 697 | "-m", 698 | "rt_sim.cli", 699 | "run-strategy", 700 | "--config", 701 | str(cfg_p), 702 | "--path", 703 | st.session_state.strategy_path, 704 | "--id", 705 | sid, 706 | "--topic", 707 | topic_str, 708 | conflate_flag, 709 | ] 710 | if params_json.strip(): 711 | cmd += ["--params", params_json] 712 | env = dict(os.environ) 713 | env["PYTHONUNBUFFERED"] = "1" 714 | # set log file for host to write into 715 | log_path = Path(base_dir / f"runs/strategy_host_{int(time.time())}.log") 716 | env["STRAT_LOG_FILE"] = str(log_path) 717 | p = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, text=True, bufsize=1, env=env) 718 | st.session_state.strategy_log_path = str(log_path) 719 | st.session_state.proc_strategy = p 720 | register_pid(p.pid) 721 | st.success(f"Strategy host started (pid={p.pid})") 722 | except Exception as e: 723 | st.error(f"Failed to start strategy host: {e}") 724 | if sbtn[1].button("Stop strategy host"): 725 | p = st.session_state.proc_strategy 726 | if p and getattr(p, "poll", lambda: 1)() is None: 727 | try: 728 | st.session_state.strategy_log_event.clear() 729 | p.terminate() 730 | try: 731 | p.wait(timeout=1) 732 | except Exception: 733 | p.kill() 734 | unregister_pid(p.pid) 735 | st.success("Strategy host stopped") 736 | if flatten_on_stop and sid: 737 | def _strategy_net_position(strategy_id: str) -> float: 738 | total = 0.0 739 | for _, f in st.session_state.fills: 740 | if f.get("strategy_id") != strategy_id: 741 | continue 742 | qty = float(f.get("qty", 0.0)) 743 | side_txt = str(f.get("side", "")).upper() 744 | total += qty if side_txt == "BUY" else -qty 745 | return total 746 | 747 | net_pos = _strategy_net_position(sid) 748 | if abs(net_pos) > 1e-9: 749 | close_side = "SELL" if net_pos > 0 else "BUY" 750 | close_qty = abs(net_pos) 751 | 752 | def _send_flatten_order(side: str, qty: float) -> None: 753 | cfg_loc = st.session_state.get("cfg", load_config(None)) 754 | t_local = Transport( 755 | hwm_ticks=int(cfg_loc["transport"]["hwm"]["ticks_pub"]), 756 | hwm_orders=int(cfg_loc["transport"]["hwm"]["orders"]), 757 | hwm_fills=int(cfg_loc["transport"]["hwm"]["fills_pub"]), 758 | ) 759 | push_socket = t_local.connect_push(cfg_loc["transport"]["endpoints"]["orders_push"]) 760 | import zmq as _zmq 761 | 762 | push_socket.setsockopt(_zmq.LINGER, 500) 763 | payload = { 764 | "strategy_id": sid, 765 | "side": side, 766 | "qty": float(qty), 767 | "tag": "auto_flatten", 768 | } 769 | Transport.send_json_push(push_socket, payload) 770 | time.sleep(0.01) 771 | push_socket.close() 772 | 773 | try: 774 | _send_flatten_order(close_side, close_qty) 775 | st.info( 776 | f"Requested {close_side} {close_qty:g} to flatten strategy '{sid}'. Check fills to confirm." 777 | ) 778 | except Exception as err: 779 | st.error(f"Failed to send flatten order: {err}") 780 | else: 781 | st.info("Strategy position already flat.") 782 | except Exception as e: 783 | st.error(f"Failed to stop strategy host: {e}") 784 | else: 785 | st.info("Strategy host not running") 786 | alive = bool(st.session_state.proc_strategy and getattr(st.session_state.proc_strategy, "poll", lambda: 1)() is None) 787 | st.caption(f"Strategy host alive: {alive}") 788 | 789 | st.subheader("Live Logs") 790 | log_path = Path(st.session_state.get("strategy_log_path", "")) 791 | if log_path.exists(): 792 | try: 793 | content = log_path.read_text() 794 | lines = content.strip().splitlines() 795 | except Exception: 796 | lines = [] 797 | else: 798 | lines = [] 799 | if not lines: 800 | st.caption("No logs yet.") 801 | st.text_area( 802 | "Log output", 803 | value="\n".join(reversed(lines[-400:])) if lines else "", 804 | height=240, 805 | key="strategy_logs_view", 806 | help="Most recent strategy host log lines.", 807 | disabled=True, 808 | ) 809 | 810 | st.divider() 811 | st.subheader("Manage Strategy Hosts") 812 | pid_list = read_pid_registry() 813 | st.caption(f"Tracked strategy host PIDs: {pid_list if pid_list else '[]'}") 814 | if st.button("Stop ALL strategy hosts"): 815 | stopped = [] 816 | still = [] 817 | for pid in pid_list: 818 | try: 819 | os.kill(pid, _signal.SIGTERM) 820 | stopped.append(pid) 821 | except Exception: 822 | still.append(pid) 823 | # Quick cleanup of registry 824 | # Rebuild registry based on processes that are still alive 825 | remaining = [] 826 | for pid in pid_list: 827 | try: 828 | os.kill(pid, 0) 829 | except Exception: 830 | continue 831 | else: 832 | remaining.append(pid) 833 | write_pid_registry(remaining) 834 | st.success(f"Sent SIGTERM to: {stopped}. Remaining tracked: {remaining}") 835 | 836 | with tab_admin: 837 | st.subheader("Listener Settings") 838 | listener_cols = st.columns(3) 839 | with listener_cols[0]: 840 | st.checkbox("Conflate latest only", key="conflate") 841 | with listener_cols[1]: 842 | st.checkbox("Auto-refresh", key="auto_refresh") 843 | with listener_cols[2]: 844 | st.button("Refresh now", on_click=lambda: None) 845 | st.slider("Refresh rate (Hz)", 1, 20, key="refresh_hz") 846 | 847 | run_id = st.session_state.get("last_run_id") 848 | if run_id: 849 | export_base = Path(cfg.get("run", {}).get("export_dir", "runs/last")) 850 | st.caption(f"Latest run artifacts: {export_base / run_id}") 851 | 852 | st.divider() 853 | st.subheader("Diagnostics") 854 | diag_col1, diag_col2 = st.columns(2) 855 | tick_feedback = diag_col1.empty() 856 | fill_feedback = diag_col2.empty() 857 | test_window_s = 3.0 858 | if diag_col1.button(f"Test receive ({int(test_window_s)}s)"): 859 | try: 860 | t = Transport(hwm_ticks=int(cfg["transport"]["hwm"]["ticks_pub"])) 861 | sub = t.connect_sub(cfg["transport"]["endpoints"]["ticks_pub"], topic="X", conflate=False) 862 | time.sleep(0.1) # allow subscription handshake 863 | poller = zmq.Poller(); poller.register(sub, zmq.POLLIN) 864 | import time as _time 865 | 866 | start = _time.time(); cnt = 0; last = None 867 | while _time.time() - start < test_window_s: 868 | socks = dict(poller.poll(timeout=100)) 869 | if sub in socks and socks[sub] == zmq.POLLIN: 870 | _, payload = t.recv_json(sub) 871 | cnt += 1; last = payload 872 | sub.close(0) 873 | st.session_state.test_recv_stats = { 874 | "count": cnt, 875 | "last_price": (last or {}).get("price"), 876 | "last_seq": (last or {}).get("seq"), 877 | "window_s": test_window_s, 878 | "ts": _time.strftime("%Y-%m-%d %H:%M:%S"), 879 | } 880 | if cnt > 0: 881 | price_val = st.session_state.test_recv_stats.get("last_price") 882 | price_str = f"{float(price_val):.5f}" if price_val not in (None, "") else "n/a" 883 | tick_feedback.success( 884 | f"{cnt} messages in ~{test_window_s:.0f}s | last price={price_str} seq={st.session_state.test_recv_stats.get('last_seq', '-') }" 885 | ) 886 | else: 887 | tick_feedback.info(f"No ticks observed in ~{test_window_s:.0f}s window.") 888 | except Exception as e: 889 | tick_feedback.error(f"Test receive failed: {e}") 890 | if diag_col2.button(f"Test fills ({int(test_window_s)}s)"): 891 | try: 892 | t = Transport(hwm_ticks=int(cfg["transport"]["hwm"]["fills_pub"])) 893 | sub = t.connect_sub(cfg["transport"]["endpoints"]["fills_pub"], topic="", conflate=False) 894 | time.sleep(0.1) # allow subscription handshake 895 | poller = zmq.Poller(); poller.register(sub, zmq.POLLIN) 896 | import time as _time 897 | 898 | start = _time.time(); cnt = 0; last = None 899 | while _time.time() - start < test_window_s: 900 | socks = dict(poller.poll(timeout=100)) 901 | if sub in socks and socks[sub] == zmq.POLLIN: 902 | _, payload = t.recv_json(sub) 903 | cnt += 1; last = payload 904 | sub.close(0) 905 | st.session_state.test_fills_stats = { 906 | "count": cnt, 907 | "last_price": (last or {}).get("fill_price"), 908 | "last_qty": (last or {}).get("qty"), 909 | "last_side": (last or {}).get("side"), 910 | "window_s": test_window_s, 911 | "ts": _time.strftime("%Y-%m-%d %H:%M:%S"), 912 | } 913 | if cnt > 0: 914 | price_val = st.session_state.test_fills_stats["last_price"] 915 | price_str = f"{price_val:.5f}" if price_val is not None else "n/a" 916 | fill_feedback.success( 917 | f"{cnt} fills in ~{test_window_s:.0f}s | last {st.session_state.test_fills_stats['last_side']} {st.session_state.test_fills_stats['last_qty']} @ {price_str}" 918 | ) 919 | else: 920 | fill_feedback.info(f"No fills observed in ~{test_window_s:.0f}s window.") 921 | except Exception as e: 922 | fill_feedback.error(f"Test fills failed: {e}") 923 | tr = st.session_state.get("test_recv_stats") 924 | if tr: 925 | price_val = tr.get("last_price") 926 | price_str = f"{float(price_val):.5f}" if price_val not in (None, "") else "n/a" 927 | st.caption( 928 | f"Last receive test @ {tr['ts']}: count={tr['count']} over ~{tr['window_s']:.0f}s | last_price={price_str} | last_seq={tr.get('last_seq', '-') }" 929 | ) 930 | trf = st.session_state.get("test_fills_stats") 931 | if trf: 932 | price_val = trf.get("last_price") 933 | price_str = f"{float(price_val):.5f}" if price_val not in (None, "") else "n/a" 934 | side = trf.get("last_side", "-") 935 | qty = trf.get("last_qty", "-") 936 | st.caption( 937 | f"Last fills test @ {trf['ts']}: count={trf['count']} over ~{trf['window_s']:.0f}s | last {side} {qty} @ {price_str}" 938 | ) 939 | 940 | st.divider() 941 | st.subheader("Local Processes") 942 | broker_cols = st.columns(2) 943 | if broker_cols[0].button("Start local broker"): 944 | try: 945 | if st.session_state.proc_broker and st.session_state.proc_broker.is_alive(): 946 | broker_cols[0].warning("Broker already running") 947 | else: 948 | run_id = new_run_id() 949 | p = mp.Process(target=_proc_broker_entry, args=(cfg, run_id), daemon=True) 950 | p.start(); st.session_state.proc_broker = p 951 | broker_cols[0].success(f"Broker started (pid={p.pid})") 952 | except Exception as e: 953 | broker_cols[0].error(f"Failed to start broker: {e}") 954 | if broker_cols[1].button("Stop local broker"): 955 | p = st.session_state.proc_broker 956 | if p and p.is_alive(): 957 | p.terminate(); p.join(timeout=1) 958 | broker_cols[1].success("Broker stopped") 959 | else: 960 | broker_cols[1].info("Broker not running") 961 | sim_cols = st.columns(2) 962 | if sim_cols[0].button("Start local simulator"): 963 | try: 964 | if st.session_state.proc_sim and st.session_state.proc_sim.is_alive(): 965 | sim_cols[0].warning("Simulator already running") 966 | else: 967 | run_id = new_run_id() 968 | p = mp.Process(target=_proc_sim_entry, args=(cfg, run_id), daemon=True) 969 | p.start(); st.session_state.proc_sim = p 970 | sim_cols[0].success(f"Simulator started (pid={p.pid})") 971 | except Exception as e: 972 | sim_cols[0].error(f"Failed to start simulator: {e}") 973 | if sim_cols[1].button("Stop local simulator"): 974 | p = st.session_state.proc_sim 975 | if p and p.is_alive(): 976 | p.terminate(); p.join(timeout=1) 977 | sim_cols[1].success("Simulator stopped") 978 | else: 979 | sim_cols[1].info("Simulator not running") 980 | 981 | st.divider() 982 | st.subheader("Metrics Settings") 983 | default_ann = int(3600 * 24 * 252) 984 | st.number_input( 985 | "Sharpe annualization factor", 986 | min_value=1, 987 | value=st.session_state.get("ann_factor", default_ann), 988 | key="ann_factor", 989 | help="Scale per-step Sharpe to annualized (e.g., trading-seconds-per-year)", 990 | ) 991 | 992 | st.divider() 993 | st.subheader("Status") 994 | ep = cfg["transport"]["endpoints"]["ticks_pub"] 995 | ep_f = cfg["transport"]["endpoints"]["fills_pub"] 996 | status_cols = st.columns(2) 997 | with status_cols[0]: 998 | st.write(f"Ticks endpoint: `{ep}`") 999 | st.write(f"Fills endpoint: `{ep_f}`") 1000 | st.write(f"Queue size: {st.session_state.queue.qsize()}") 1001 | with status_cols[1]: 1002 | th = st.session_state.listener_thread 1003 | fth = st.session_state.fills_thread 1004 | st.write(f"Listener alive: {bool(th and th.is_alive())}") 1005 | st.write(f"Fills listener alive: {bool(fth and fth.is_alive())}") 1006 | last = st.session_state.get("last_recv_ts") 1007 | st.write(f"Last received: {last if last else 'none yet'}") 1008 | if _LAST_THREAD_ERROR: 1009 | st.error(_LAST_THREAD_ERROR) 1010 | atimes = st.session_state.get("arrival_times", deque()) 1011 | rate = 0.0 1012 | if atimes: 1013 | cutoff = time.time() - 5.0 1014 | recent = [t for t in atimes if t >= cutoff] 1015 | if len(recent) >= 2: 1016 | dur = max(1e-6, (recent[-1] - recent[0])) 1017 | rate = len(recent) / dur 1018 | metric_cols = st.columns(3) 1019 | metric_cols[0].metric("Approx ticks/sec", f"{rate:.1f}") 1020 | metric_cols[1].metric("Seq gaps", st.session_state.get("gap_count", 0)) 1021 | metric_cols[2].metric("UI fills captured", len(st.session_state.fills)) 1022 | if not (st.session_state.listener_thread and st.session_state.listener_thread.is_alive()): 1023 | st.caption("UI fills update only while the main listener (Start SUB) is running.") 1024 | 1025 | st.divider() 1026 | st.subheader("Config") 1027 | cfg_path = st.text_input("Config path", value="configs/default.yaml") 1028 | base_dir = Path(__file__).resolve().parents[1] 1029 | resolved = Path(cfg_path) 1030 | if not resolved.is_absolute(): 1031 | resolved = base_dir / resolved 1032 | if st.button("Load config"): 1033 | try: 1034 | st.session_state.cfg = load_config(str(resolved)) 1035 | st.session_state.cfg_path = str(resolved) 1036 | cfg = st.session_state.cfg 1037 | try: 1038 | init_cash = float(cfg.get("portfolio", {}).get("initial_cash", 100000.0)) 1039 | except Exception: 1040 | init_cash = 100000.0 1041 | st.session_state.initial_cash = init_cash 1042 | st.session_state.pos = 0.0 1043 | st.session_state.cash = init_cash 1044 | st.session_state.last_price = None 1045 | st.session_state.pnl = deque(maxlen=2000) 1046 | st.session_state.fills = deque(maxlen=500) 1047 | st.session_state.pos_series = deque(maxlen=2000) 1048 | st.success(f"Loaded config: {resolved} (initial cash set to ${init_cash:,.2f})") 1049 | except Exception as e: 1050 | st.error(f"Failed to load config: {e}") 1051 | st.caption(f"Using config: {resolved}") 1052 | st.code(json.dumps(cfg["transport"], indent=2)) 1053 | 1054 | if st.session_state.get("auto_refresh", True): 1055 | st_autorefresh = st.empty() 1056 | st_autorefresh.caption("Auto-refresh active") 1057 | time.sleep(chart_refresh_ms / 1000) 1058 | try: 1059 | st.rerun() 1060 | except Exception: 1061 | if hasattr(st, "experimental_rerun"): 1062 | st.experimental_rerun() # type: ignore[attr-defined] 1063 | 1064 | 1065 | if __name__ == "__main__": 1066 | main() 1067 | --------------------------------------------------------------------------------