├── .gitignore ├── pics ├── overview.png ├── gas-growth.png ├── simulation.png ├── quantisation.png ├── pivot-selection.png ├── gas-usage-usdc-dai.png ├── gas-usage-usdc-weth.png ├── gas-usage-wbtc-weth.png ├── uniswap-ring-buffer-sizes.png ├── gas-usage-usdc-weth-10min-window.png ├── gas-usage-usdc-weth-normal-period.png ├── ring-buffer.svg ├── weighted-median.svg ├── linear-search.svg └── binary-search.svg ├── package.json ├── sim ├── crawl-uniswap │ ├── import-bigquery.pl │ ├── schema.sql │ └── crawl-uniswap.js ├── README.md └── plot.gnu ├── hardhat.config.js ├── generate-toc.pl ├── contracts ├── uniswap │ ├── StubUniswapV3Pool.sol │ └── Oracle.sol └── MedianOracle.sol ├── test └── basic.js ├── tasks └── sim.js └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /artifacts/ 2 | /cache/ 3 | /node_modules/ 4 | /package-lock.json 5 | -------------------------------------------------------------------------------- /pics/overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/euler-xyz/median-oracle/HEAD/pics/overview.png -------------------------------------------------------------------------------- /pics/gas-growth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/euler-xyz/median-oracle/HEAD/pics/gas-growth.png -------------------------------------------------------------------------------- /pics/simulation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/euler-xyz/median-oracle/HEAD/pics/simulation.png -------------------------------------------------------------------------------- /pics/quantisation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/euler-xyz/median-oracle/HEAD/pics/quantisation.png -------------------------------------------------------------------------------- /pics/pivot-selection.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/euler-xyz/median-oracle/HEAD/pics/pivot-selection.png -------------------------------------------------------------------------------- /pics/gas-usage-usdc-dai.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/euler-xyz/median-oracle/HEAD/pics/gas-usage-usdc-dai.png -------------------------------------------------------------------------------- /pics/gas-usage-usdc-weth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/euler-xyz/median-oracle/HEAD/pics/gas-usage-usdc-weth.png -------------------------------------------------------------------------------- /pics/gas-usage-wbtc-weth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/euler-xyz/median-oracle/HEAD/pics/gas-usage-wbtc-weth.png -------------------------------------------------------------------------------- /pics/uniswap-ring-buffer-sizes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/euler-xyz/median-oracle/HEAD/pics/uniswap-ring-buffer-sizes.png -------------------------------------------------------------------------------- /pics/gas-usage-usdc-weth-10min-window.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/euler-xyz/median-oracle/HEAD/pics/gas-usage-usdc-weth-10min-window.png -------------------------------------------------------------------------------- /pics/gas-usage-usdc-weth-normal-period.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/euler-xyz/median-oracle/HEAD/pics/gas-usage-usdc-weth-normal-period.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "@nomiclabs/hardhat-ethers": "^2.0.6", 4 | "@nomiclabs/hardhat-waffle": "^2.0.3", 5 | "better-sqlite3": "^7.5.3", 6 | "chai": "^4.3.6", 7 | "cross-fetch": "^3.1.5", 8 | "ganache-cli": "^6.12.2", 9 | "hardhat": "^2.9.6", 10 | "seedrandom": "^3.0.5" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /sim/crawl-uniswap/import-bigquery.pl: -------------------------------------------------------------------------------- 1 | use strict; 2 | 3 | use Date::Parse; 4 | use DBI; 5 | 6 | my $dbh = DBI->connect("dbi:SQLite:dbname=results.db","",""); 7 | 8 | my $sth = $dbh->prepare("INSERT INTO Block (blockNumber, timestamp) VALUES (?,?)"); 9 | 10 | $dbh->{AutoCommit} = 0; 11 | 12 | while() { 13 | /^(\d+),(.*)/ || next; 14 | my $blockNumber = $1; 15 | my $ts = str2time($2); 16 | $sth->execute($blockNumber, $ts); 17 | if ($blockNumber % 10000 == 0) { 18 | print "$blockNumber\n"; 19 | } 20 | } 21 | 22 | $dbh->commit; 23 | -------------------------------------------------------------------------------- /sim/crawl-uniswap/schema.sql: -------------------------------------------------------------------------------- 1 | PRAGMA encoding = "UTF-8"; 2 | PRAGMA foreign_keys = ON; 3 | 4 | 5 | CREATE TABLE Pair ( 6 | name TEXT PRIMARY KEY, 7 | uniswapPoolAddr TEXT 8 | ); 9 | 10 | 11 | CREATE TABLE Block ( 12 | blockNumber INTEGER PRIMARY KEY, 13 | timestamp INTEGER NOT NULL 14 | ); 15 | 16 | 17 | CREATE TABLE Swap ( 18 | pairName TEXT NOT NULL, 19 | blockNumber INTEGER NOT NULL, 20 | logIndex INTEGER NOT NULL, 21 | tick INTEGER NOT NULL, 22 | sqrtPriceX96 TEXT NOT NULL, 23 | 24 | PRIMARY KEY (pairName, blockNumber, logIndex), 25 | FOREIGN KEY (pairName) REFERENCES Pair(name) 26 | ); 27 | -------------------------------------------------------------------------------- /sim/README.md: -------------------------------------------------------------------------------- 1 | ## Configuration 2 | 3 | * Uniswap pairs to crawl: `sim/crawl-uniswap/crawl-uniswap.js` 4 | * Simulation parameters: `tasks/sim.js` 5 | * Plot parameters: `sim/plot.gnu` 6 | 7 | ## Crawl Uniswap logs data 8 | 9 | In `sim/crawl-uniswap/` directory: 10 | 11 | sqlite3 results.db < schema.sql 12 | RPC_URL=https://REPLACE_ME node crawl-uniswap.js 13 | 14 | ## Import BigQuery logs 15 | 16 | On google BigQuery, run the following query to get block timestamps, and export results as CSV: 17 | 18 | SELECT number,timestamp FROM `bigquery-public-data.crypto_ethereum.blocks` WHERE number >= 12369621 19 | 20 | Import block timestamps into DB: 21 | 22 | perl import-bigquery.pl < bq-results-20220526-103739-1653561636441.csv 23 | 24 | ## Run simulation 25 | 26 | In top-level directory: 27 | 28 | npx hardhat sim | grep ^csv | > median.csv 29 | MODE=uniswap npx hardhat sim | grep ^csv | > uniswap.csv 30 | 31 | ## Plot results 32 | 33 | gnuplot sim/plot.gnu 34 | -------------------------------------------------------------------------------- /hardhat.config.js: -------------------------------------------------------------------------------- 1 | require("@nomiclabs/hardhat-waffle"); 2 | const fs = require("fs"); 3 | 4 | // Load tasks 5 | 6 | const files = fs.readdirSync('./tasks'); 7 | 8 | for (let file of files) { 9 | if (!file.endsWith('.js')) continue; 10 | require(`./tasks/${file}`); 11 | } 12 | 13 | // Config 14 | 15 | module.exports = { 16 | networks: { 17 | hardhat: { 18 | hardfork: 'berlin', 19 | blockGasLimit: 100_000_000, 20 | }, 21 | }, 22 | 23 | solidity: { 24 | compilers: [ 25 | { 26 | version: "0.8.13", 27 | settings: { 28 | optimizer: { 29 | enabled: true, 30 | runs: 1000000, 31 | }, 32 | }, 33 | }, 34 | { 35 | version: "0.7.6", 36 | settings: { 37 | optimizer: { 38 | enabled: true, 39 | runs: 1000000, 40 | }, 41 | }, 42 | }, 43 | ], 44 | }, 45 | }; 46 | -------------------------------------------------------------------------------- /generate-toc.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use common::sense; 4 | 5 | my $lines = ''; 6 | my $headers = []; 7 | my $seenHeaders = {}; 8 | 9 | { 10 | open(my $fh, '<', 'README.md') || die "unable to open README.md: $!"; 11 | 12 | while (<$fh>) { 13 | next if /^/ .. /^/; 14 | 15 | $lines .= $_; 16 | 17 | if (/^[#]+ (.*)/) { 18 | my $whole = $&; 19 | my $title = $1; 20 | 21 | my $link = title2link($1); 22 | die "duplicate header: $link" if $seenHeaders->{$link}; 23 | $seenHeaders->{$link}++; 24 | push @$headers, $whole; 25 | } 26 | } 27 | } 28 | 29 | my $toc = ''; 30 | 31 | for my $header (@$headers) { 32 | $header =~ /^(#+) (.*)/; 33 | my $prefix = $1; 34 | my $title = $2; 35 | 36 | next if $prefix eq '#'; 37 | 38 | $prefix =~ s/^##//; 39 | $prefix =~ s/^\s+//; 40 | $prefix =~ s/#/ /g; 41 | $prefix = "$prefix*"; 42 | 43 | my $link = title2link($title); 44 | $toc .= "$prefix [$title](#$link)\n"; 45 | } 46 | 47 | { 48 | open(my $ofh, '>', 'README.md.tmp') || die "unable to open README.md: $!"; 49 | 50 | $lines =~ s{}{\n\n$toc}; 51 | 52 | print $ofh $lines; 53 | } 54 | 55 | 56 | while ($lines =~ m{\[.*?\][(]#(.*?)[)]}g) { 57 | my $link = $1; 58 | if (!$seenHeaders->{$link}) { 59 | print STDERR "WARNING: Unresolved link: $link\n"; 60 | } 61 | } 62 | 63 | system("mv -f README.md.tmp README.md"); 64 | 65 | 66 | 67 | sub title2link { 68 | my $title = shift; 69 | my $link = lc $title; 70 | $link =~ s/\s+/-/g; 71 | $link =~ s/[+]//g; 72 | return $link; 73 | } 74 | -------------------------------------------------------------------------------- /sim/plot.gnu: -------------------------------------------------------------------------------- 1 | reset 2 | 3 | #set terminal wxt size 800,600 enhanced font 'Verdana,9' persist 4 | 5 | set terminal pngcairo size 800,600 enhanced font 'Verdana,9' 6 | set output 'output.png' 7 | 8 | 9 | 10 | set datafile separator comma 11 | 12 | 13 | # define axis 14 | # remove border on top and right and set color to gray 15 | set style line 11 lc rgb '#808080' lt 1 16 | set border 3 back ls 11 17 | set tics nomirror 18 | # define grid 19 | set style line 12 lc rgb '#808080' lt 0 lw 1 20 | set grid back ls 12 21 | 22 | # color definitions 23 | #set style line 1 lc rgb '#8b1a0e' pt 1 ps 1 lt 1 lw 2 # --- red 24 | #set style line 2 lc rgb '#5e9c36' pt 6 ps 1 lt 1 lw 2 # --- green 25 | 26 | set style line 1 linecolor rgb '#0060ad' linetype 1 linewidth 3 27 | set style line 2 linecolor rgb '#dd3033' linetype 1 linewidth 3 28 | set style line 3 linecolor rgb '#0df023' linetype 1 linewidth 3 29 | set style line 4 linecolor rgb '#813033' linetype 1 linewidth 2 30 | set style line 5 linecolor rgb '#d130d3' linetype 1 linewidth 2 31 | set style line 6 linecolor rgb '#57ccc7' linetype 1 linewidth 2 32 | 33 | 34 | 35 | set xdata time 36 | set timefmt "%s" 37 | set format x "%Y-%m-%d\n%H:%M:%S" 38 | set key Left center top reverse box samplen 2 width 2 39 | 40 | 41 | ## Quantisation plots 42 | 43 | #set title "Quantisation: USDC/WETH .3%" 44 | #set ylabel 'Price' 45 | #plot \ 46 | # 'median.csv' using 2:7 with points pt 2 lt rgb '#ff0000' title 'Trades', \ 47 | # 'median.csv' using 2:4 with lines linestyle 5 title 'Uniswap3 Tick', \ 48 | # 'median.csv' using 2:8 with lines linestyle 6 title 'Quantised Tick', \ 49 | 50 | ## Price comparisons 51 | 52 | #set title "Oracle simulation, USDC/WETH .3%, 30 min window" 53 | #plot 'median.csv' using 2:5 with lines linestyle 1 title '30m Median', \ 54 | # 'median.csv' using 2:6 with lines linestyle 3 title '30m TWAP (ours)', \ 55 | # 'uniswap.csv' using 2:6 with lines linestyle 2 title '30m TWAP (uniswap3)', \ 56 | # 'median.csv' using 2:7 with points title 'Trade', \ 57 | # #'median.csv' using 2:8 with lines linestyle 5 title 'Current (quantised)' 58 | 59 | ## Gas scatter 60 | 61 | #set title "Oracle read gas usage: USDC/WETH .3%, 30 min window" 62 | #set ylabel 'Gas' 63 | #plot 'uniswap.csv' using 2:9 with points pt 7 ps 1 lt rgb "#5e9c36" title 'Uniswap3', \ 64 | # 'median.csv' using 2:9 with points pt 7 ps 1 lt rgb "#8b1a0e" title 'Our Oracle', \ 65 | 66 | ## Binary search overhead 67 | 68 | set title "Uniswap3 gas usage at different ring buffer sizes: USDC/WETH .3%, 30 min window" 69 | set ylabel 'Gas' 70 | plot 'uniswap144.csv' using 2:9 with points pt 7 ps 1 lt rgb "blue" title 'Ring-Buffer Size 144', \ 71 | 'uniswap1440.csv' using 2:9 with points pt 7 ps 1 lt rgb "red" title 'Ring-Buffer Size 1440', \ 72 | 73 | ## Growth/Pivot Selection 74 | 75 | #set title "Gas Usage as Ring Buffer Accesses Increase" 76 | #set ylabel 'Gas' 77 | #set xlabel 'Ring Buffer Accesses' 78 | #plot \ 79 | # 'growth-best.csv' using 2:3 with lines linestyle 1 title 'Average+Median, Best Case', \ 80 | # 'growth-rand.csv' using 2:3 with lines linestyle 3 title 'Average+Median, Random', \ 81 | # 'growth-avg-only.csv' using 2:3 with lines linestyle 4 title 'Average only', \ 82 | # #'growth-worst.csv' using 2:3 with lines linestyle 2 title 'Worse case: O(N^2)', \ 83 | -------------------------------------------------------------------------------- /contracts/uniswap/StubUniswapV3Pool.sol: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BUSL-1.1 2 | pragma solidity =0.7.6; 3 | 4 | import './Oracle.sol'; 5 | 6 | contract StubUniswapV3Pool { 7 | using Oracle for Oracle.Observation[65535]; 8 | 9 | struct Slot0 { 10 | // the current price 11 | uint160 sqrtPriceX96; 12 | // the current tick 13 | int24 tick; 14 | // the most-recently updated index of the observations array 15 | uint16 observationIndex; 16 | // the current maximum number of observations that are being stored 17 | uint16 observationCardinality; 18 | // the next maximum number of observations to store, triggered in observations.write 19 | uint16 observationCardinalityNext; 20 | // the current protocol fee as a percentage of the swap fee taken on withdrawal 21 | // represented as an integer denominator (1/x)% 22 | uint8 feeProtocol; 23 | // whether the pool is locked 24 | bool unlocked; 25 | } 26 | 27 | Slot0 public slot0; 28 | 29 | Oracle.Observation[65535] public observations; 30 | 31 | constructor(uint16 _ringSize) { 32 | int24 tick = 0; 33 | 34 | (uint16 cardinality, uint16 cardinalityNext) = observations.initialize(_blockTimestamp()); 35 | 36 | slot0 = Slot0({ 37 | sqrtPriceX96: 0, 38 | tick: tick, 39 | observationIndex: 0, 40 | observationCardinality: cardinality, 41 | observationCardinalityNext: cardinalityNext, 42 | feeProtocol: 0, 43 | unlocked: true 44 | }); 45 | 46 | // Grow to 144 47 | 48 | uint16 observationCardinalityNext = _ringSize; 49 | 50 | uint16 observationCardinalityNextOld = slot0.observationCardinalityNext; // for the event 51 | uint16 observationCardinalityNextNew = 52 | observations.grow(observationCardinalityNextOld, observationCardinalityNext); 53 | slot0.observationCardinalityNext = observationCardinalityNextNew; 54 | } 55 | 56 | function _blockTimestamp() internal view virtual returns (uint32) { 57 | return uint32(block.timestamp); // truncation is desired 58 | } 59 | 60 | function updateOracle(int24 newTick) external { 61 | Slot0 memory slot0Start = slot0; 62 | 63 | (uint16 observationIndex, uint16 observationCardinality) = 64 | observations.write( 65 | slot0Start.observationIndex, 66 | _blockTimestamp(), 67 | slot0Start.tick, 68 | 0, 69 | slot0Start.observationCardinality, 70 | slot0Start.observationCardinalityNext 71 | ); 72 | (slot0.sqrtPriceX96, slot0.tick, slot0.observationIndex, slot0.observationCardinality) = ( 73 | 0, 74 | newTick, 75 | observationIndex, 76 | observationCardinality 77 | ); 78 | } 79 | 80 | function readOracle(uint16 desiredAge) external view returns (uint16, int24, int24) { // returns (actualAge, median=0, average) 81 | uint32[] memory secondsAgos = new uint32[](2); 82 | secondsAgos[0] = desiredAge; 83 | secondsAgos[1] = 0; 84 | 85 | (int56[] memory tickCumulatives,) = 86 | observations.observe( 87 | _blockTimestamp(), 88 | secondsAgos, 89 | slot0.tick, 90 | slot0.observationIndex, 91 | 0, 92 | slot0.observationCardinality 93 | ); 94 | 95 | return ( 96 | desiredAge, 97 | uint16(0), 98 | int24((tickCumulatives[1] - tickCumulatives[0]) / int56(int(desiredAge))) 99 | ); 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /test/basic.js: -------------------------------------------------------------------------------- 1 | const { expect } = require("chai"); 2 | const seedrandom = require("seedrandom"); 3 | 4 | describe("median oracle tests", function () { 5 | it("quantisation", async function () { 6 | const [owner] = await ethers.getSigners(); 7 | 8 | let MedianOracleFactory = await ethers.getContractFactory("MedianOracle"); 9 | 10 | let oracle = await MedianOracleFactory.deploy(144); 11 | 12 | let ts = (await ethers.provider.getBlock()).timestamp; 13 | let origTs = ts; 14 | 15 | let TICK_MAX = 887272; 16 | let TICK_MIN = -887272; 17 | 18 | let checkTick = async (tickIn, tickOut) => { 19 | let tx = await oracle.updateOracle(tickIn); 20 | await tx.wait(); 21 | 22 | ts += 2000; 23 | await ethers.provider.send("evm_setNextBlockTimestamp", [ts]); 24 | await ethers.provider.send("evm_mine"); 25 | 26 | let res = await oracle.readOracle(1800); 27 | 28 | expect(res[1]).to.equal(tickOut); 29 | expect(res[1]).to.be.gte(TICK_MIN); 30 | expect(res[1]).to.be.lte(TICK_MAX); 31 | }; 32 | 33 | await checkTick(0, 15); 34 | await checkTick(1, 15); 35 | await checkTick(15, 15); 36 | await checkTick(29, 15); 37 | await checkTick(30, 45); 38 | await checkTick(59, 45); 39 | await checkTick(60, 75); 40 | 41 | await checkTick(-1, -15); 42 | await checkTick(-2, -15); 43 | await checkTick(-15, -15); 44 | await checkTick(-29, -15); 45 | await checkTick(-30, -15); 46 | await checkTick(-31, -45); 47 | await checkTick(-60, -45); 48 | await checkTick(-61, -75); 49 | 50 | await checkTick(TICK_MAX, 887265); 51 | await checkTick(TICK_MIN, -887265); 52 | }); 53 | 54 | it("fuzz", async function () { 55 | const [owner] = await ethers.getSigners(); 56 | 57 | let MedianOracleFactory = await ethers.getContractFactory("MedianOracle"); 58 | 59 | let oracle = await MedianOracleFactory.deploy(144); 60 | 61 | let ts = (await ethers.provider.getBlock()).timestamp; 62 | let origTs = ts; 63 | 64 | let rng = seedrandom(''); 65 | 66 | let updates = []; 67 | 68 | let seen = 0; 69 | 70 | while (seen < 100) { 71 | let duration = Math.floor(rng() * 30) + 1; 72 | ts += duration; 73 | await ethers.provider.send("evm_setNextBlockTimestamp", [ts]); 74 | 75 | if (updates.length > 0) updates[updates.length - 1].duration += duration; 76 | 77 | { 78 | let price = Math.floor((rng() * 10_000) - 5_000); 79 | 80 | updates.push({ 81 | price: Math.floor(price / 30) * 30 + 15, 82 | duration: 0, 83 | }); 84 | 85 | let tx = await oracle.updateOracle(price); 86 | await tx.wait(); 87 | } 88 | 89 | const windowLen = 1800; 90 | 91 | let res = await oracle.readOracle(windowLen); 92 | 93 | // Check result 94 | 95 | let arr = []; 96 | 97 | for (let i = updates.length - 1; i >= 0; i--) { 98 | //if (updates[i].duration) console.log("ZZ",[Math.floor(updates[i].price/30), Math.min(updates[i].duration, windowLen - arr.length)]); 99 | for (let j = 0; j < updates[i].duration; j++) { 100 | arr.push(updates[i].price); 101 | if (arr.length === windowLen) break; 102 | } 103 | if (arr.length === windowLen) break; 104 | } 105 | 106 | arr.sort((a,b) => Math.sign(a-b)); 107 | 108 | let median = arr[Math.ceil(windowLen/2) - 1]; 109 | median = Math.floor(median / 30) * 30 + 15; 110 | 111 | if (process.env.VERBOSE) { 112 | console.log("------------"); 113 | console.log("SEEN", seen); 114 | console.log("RES", res); 115 | console.log("MEDIAN=",median); 116 | console.log("AL=",arr.length); 117 | } 118 | 119 | if (arr.length === windowLen) { 120 | seen++; 121 | 122 | if (res[1] !== median) { 123 | console.log("DIFFERENT",res[1],median); 124 | throw("DIFFERENT"); 125 | } 126 | } 127 | } 128 | }); 129 | }); 130 | -------------------------------------------------------------------------------- /tasks/sim.js: -------------------------------------------------------------------------------- 1 | const betterSqlite3 = require('better-sqlite3'); 2 | 3 | let pool = 'USDC/WETH/3000'; 4 | let ringSize = 144; 5 | let windowSize = 1800; 6 | 7 | // Around the crash 8 | let blockFrom = 14735000; // May-08-2022 08:27:41 AM +UTC 9 | let blockTo = 14775000; // May-14-2022 05:49:46 PM +UTC 10 | 11 | // "Normal" period 12 | //let blockFrom = 14595000; 13 | //let blockTo = 14635000; 14 | 15 | //let blockFrom = 14760000; // May-12-2022 08:16:38 AM +UTC 16 | //let blockTo = 14765000; // May-13-2022 03:24:18 AM +UTC 17 | 18 | let priceInvert = true; 19 | let decimalScaler = 1e12; // USDC 20 | //let decimalScaler = 1e0; // normal 21 | let minTimeStep = 60*15; 22 | 23 | 24 | task("sim") 25 | .setAction(async ({ args, }) => { 26 | 27 | await hre.run("compile"); 28 | 29 | const db = new betterSqlite3('./sim/crawl-uniswap/results.db'); 30 | 31 | db.pragma('encoding = "UTF-8"'); 32 | db.pragma('foreign_keys = ON'); 33 | db.pragma('defer_foreign_keys = ON'); 34 | 35 | let blocks = db.prepare(`SELECT s.tick, s.blockNumber, s.sqrtPriceX96, b.timestamp 36 | FROM Swap s, Block b 37 | WHERE s.pairName = ? 38 | AND s.blockNumber = b.blockNumber 39 | AND s.blockNumber>? AND s.blockNumber minTimeStep) { 53 | newBlocks.push({ 54 | tick: newBlocks[newBlocks.length - 1].tick, 55 | sqrtPriceX96: newBlocks[newBlocks.length - 1].sqrtPriceX96, 56 | timestamp: newBlocks[newBlocks.length - 1].timestamp + minTimeStep, 57 | virtualBlock: true, 58 | }); 59 | } 60 | 61 | newBlocks.push(blocks[i]); 62 | } 63 | 64 | blocks = newBlocks; 65 | } 66 | 67 | // Only apply last in block, since we are increasing timestamp by 1 every time 68 | 69 | { 70 | let newBlocks = []; 71 | 72 | for (let i = 1; i < blocks.length - 1; i++) { 73 | if (blocks[i].blockNumber === blocks[i+1].blockNumber) continue; 74 | newBlocks.push(blocks[i]); 75 | } 76 | 77 | blocks = newBlocks; 78 | } 79 | 80 | console.log("POST-PROC BLOCKS = ", blocks.length); 81 | 82 | let oracle; 83 | 84 | let mode = process.env.MODE || 'median'; 85 | 86 | if (mode === "uniswap") { 87 | const factory = await ethers.getContractFactory("StubUniswapV3Pool"); 88 | oracle = await factory.deploy(ringSize); 89 | await oracle.deployed(); 90 | } else if (mode === 'median') { 91 | const factory = await ethers.getContractFactory("MedianOracle"); 92 | oracle = await factory.deploy(ringSize); 93 | await oracle.deployed(); 94 | } else { 95 | throw("unrecognized mode: ", mode); 96 | } 97 | 98 | await (await oracle.updateOracle(blocks[0].tick)).wait(); 99 | 100 | let ts = (await ethers.provider.getBlock()).timestamp + 86400; 101 | await ethers.provider.send("evm_setNextBlockTimestamp", [ts]); 102 | await ethers.provider.send("evm_mine"); 103 | 104 | let gases = []; 105 | 106 | for (let i = 1; i < blocks.length; i++) { 107 | if ((i % 100) === 1) console.error(`${i}/${blocks.length} (${100*i/blocks.length}%)`); 108 | 109 | let delay = blocks[i].timestamp - blocks[i-1].timestamp; 110 | ts += delay; 111 | if (delay > 0) await ethers.provider.send("evm_setNextBlockTimestamp", [ts]); 112 | 113 | console.log(`DELAY ${delay} / PRICE ${blocks[i].tick}`); 114 | if (blocks[i].tick) { 115 | await (await oracle.updateOracle(blocks[i].tick)).wait(); 116 | } else { 117 | await ethers.provider.send("evm_mine"); 118 | } 119 | 120 | let res; 121 | let gas; 122 | 123 | try { 124 | gas = (await oracle.estimateGas.readOracle(windowSize)).toNumber() - 21_000; 125 | console.log(`${i} GAS`,gas); 126 | gases.push(gas); 127 | 128 | res = await oracle.readOracle(windowSize); 129 | console.log(`${i} RES`,res); 130 | } catch(e) { 131 | console.log(`SKIPPING ERR: ${e}`); 132 | continue; 133 | } 134 | 135 | let tickToPrice = (p) => { 136 | let o = Math.pow(1.0001, p) / decimalScaler; 137 | if (priceInvert) o = 1/o; 138 | return o; 139 | }; 140 | 141 | let sqrtPriceX96ToPrice = (p) => { 142 | p = parseInt(p); 143 | let o = p*p/(2**(96*2)) / decimalScaler; 144 | if (priceInvert) o = 1/o; 145 | return o; 146 | }; 147 | 148 | let requantisedTick = tickToPrice(Math.floor(blocks[i].tick / 30) * 30 + 15); 149 | 150 | console.log(`csv,${blocks[i].timestamp},${res[1]},${tickToPrice(blocks[i].tick)},${tickToPrice(res[1])},${tickToPrice(res[2])},${sqrtPriceX96ToPrice(blocks[i].sqrtPriceX96)},${requantisedTick},${gas}`); 151 | } 152 | 153 | console.error(`MIN GAS: ${Math.min.apply(null, gases)}`); 154 | console.error(`MAX GAS: ${Math.max.apply(null, gases)}`); 155 | console.error(`AVG GAS: ${avg(gases)}`); 156 | }); 157 | 158 | function avg(arr) { 159 | let sum = 0; 160 | for (let n of arr) sum += n; 161 | return sum / arr.length; 162 | } 163 | -------------------------------------------------------------------------------- /sim/crawl-uniswap/crawl-uniswap.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Error.stackTraceLimit = 10000; 3 | 4 | const ethers = require('ethers'); 5 | const betterSqlite3 = require('better-sqlite3'); 6 | const fetch = require('cross-fetch'); 7 | 8 | 9 | // Config 10 | 11 | let rpcUrl = process.env.RPC_URL; 12 | let batchSize = 2000; 13 | let crawlDelaySeconds = 1; 14 | 15 | let startBlock = 12369621; // Uniswap 3 factory deployed: May-04-2021 07:27:00 PM +UTC 16 | let endBlock = 14843012; // Arbitrary end-point: May-25-2022 04:58:50 PM +UTC 17 | 18 | let uniswapFactoryAddr = '0x1F98431c8aD98523631AE4a59f267346ea31F984'; 19 | let quoteToken = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'; // WETH 20 | 21 | let toks = { 22 | WETH: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', 23 | USDC: '0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48', 24 | DAI: '0x6b175474e89094c44da98b954eedeac495271d0f', 25 | UNI: '0x1f9840a85d5af5bf1d1762f925bdaddc4201f984', 26 | MKR: '0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2', 27 | WBTC: '0x2260fac5e5542a773aa44fbcfedf7c193bc2c599', 28 | FRAX: '0x853d955acef822db058eb8505911ed77f175b99e', 29 | }; 30 | 31 | let pairsToCrawl = [ 32 | "USDC/WETH/3000", 33 | "DAI/WETH/3000", 34 | "DAI/USDC/100", 35 | "UNI/WETH/3000", 36 | "MKR/WETH/3000", 37 | "FRAX/USDC/500", 38 | "WBTC/WETH/3000", 39 | ]; 40 | 41 | // End of config 42 | 43 | 44 | 45 | let factoryIface = new ethers.utils.Interface([ 46 | 'function getPool(address tokenA, address tokenB, uint24 fee) external view returns (address pool)', 47 | ]); 48 | 49 | 50 | let pairIface = new ethers.utils.Interface([ 51 | 'event Swap(address indexed sender, address indexed recipient, int256 amount0, int256 amount1, uint160 sqrtPriceX96, uint128 liquidity, int24 tick)', 52 | ]); 53 | 54 | 55 | const db = new betterSqlite3('./results.db'); 56 | 57 | db.pragma('encoding = "UTF-8"'); 58 | db.pragma('foreign_keys = ON'); 59 | db.pragma('defer_foreign_keys = ON'); 60 | 61 | 62 | let uniswapPoolAddrToPair = {}; 63 | 64 | 65 | 66 | let provider = new ethers.providers.JsonRpcProvider(rpcUrl); 67 | 68 | 69 | 70 | main(); 71 | 72 | async function main() { 73 | await populateTokenAddrs(); 74 | await crawl(); 75 | } 76 | 77 | 78 | 79 | 80 | async function populateTokenAddrs() { 81 | console.log(`Looking up ${pairsToCrawl.length} pairs`); 82 | 83 | let uniswapFactory = new ethers.Contract(uniswapFactoryAddr, factoryIface, provider); 84 | 85 | let skipped = 0; 86 | let lookedUp = 0; 87 | 88 | for (let pairName of pairsToCrawl) { 89 | let row = db.prepare(`SELECT name, uniswapPoolAddr FROM Pair WHERE name = ?`) 90 | .get(pairName); 91 | 92 | let pair = unpackPairToCrawl(pairName); 93 | 94 | if (row) { 95 | uniswapPoolAddrToPair[row.uniswapPoolAddr] = pair; 96 | console.log(`${pairName} already in DB`); 97 | skipped++; 98 | continue; 99 | } 100 | 101 | lookedUp++; 102 | 103 | let uniswapPoolAddr = await uniswapFactory.getPool(pair.baseAddr, pair.quoteAddr, pair.fee); 104 | uniswapPoolAddr = uniswapPoolAddr.toLowerCase(); 105 | uniswapPoolAddrToPair[uniswapPoolAddr] = pair; 106 | 107 | console.log(`${pairName}: Uniswap pair=${uniswapPoolAddr}`); 108 | 109 | db.prepare('INSERT OR REPLACE INTO Pair (name, uniswapPoolAddr) VALUES (?,?)') 110 | .run(pairName, uniswapPoolAddr); 111 | } 112 | 113 | 114 | console.log(`Looked up ${lookedUp}/${skipped+lookedUp} tokens`); 115 | } 116 | 117 | 118 | 119 | async function crawl() { 120 | { 121 | let highestBlock = db.prepare(`SELECT MAX(blockNumber) FROM Swap`).pluck().get(); 122 | if (highestBlock && highestBlock > startBlock) { 123 | startBlock = highestBlock + 1; 124 | } 125 | 126 | console.log(`startBlock = ${startBlock}`); 127 | } 128 | 129 | 130 | let currBlock = startBlock; 131 | 132 | while (currBlock <= endBlock) { 133 | let rangeEnd = currBlock + batchSize - 1; 134 | if (rangeEnd > endBlock) rangeEnd = endBlock; 135 | 136 | console.log(`Fetching blocks: ${currBlock} - ${rangeEnd}`); 137 | 138 | let params = [{ 139 | fromBlock: '0x' + currBlock.toString(16), 140 | toBlock: '0x' + rangeEnd.toString(16), 141 | 142 | address: Object.keys(uniswapPoolAddrToPair), 143 | topics: [pairIface.getEventTopic('Swap')], 144 | }]; 145 | 146 | let query = { 147 | jsonrpc: "2.0", 148 | id: 1, 149 | method: "eth_getLogs", 150 | params, 151 | }; 152 | 153 | let res = await fetch(rpcUrl, { 154 | method: 'post', 155 | body: JSON.stringify(query), 156 | headers: { 'Content-Type': 'application/json' }, 157 | }); 158 | 159 | let resJson = await res.json(); 160 | if (resJson.error) { 161 | console.error(`ERROR from eth_getLogs. Response:`); 162 | console.error(resJson); 163 | console.error(`Request params:`); 164 | console.error(params); 165 | process.exit(1); 166 | } 167 | 168 | // Add Swap records 169 | //console.log("RES" + JSON.stringify(resJson)); 170 | 171 | db.transaction(() => { 172 | for (let log of resJson.result) { 173 | let pair = uniswapPoolAddrToPair[log.address.toLowerCase()]; 174 | 175 | //console.log(log); 176 | 177 | let parsedLog = pairIface.parseLog(log); 178 | 179 | //console.log(parsedLog); 180 | 181 | db.prepare(`INSERT OR REPLACE INTO Swap (pairName, blockNumber, logIndex, tick, sqrtPriceX96) 182 | VALUES (?, ?, ?, ?, ?)`) 183 | .run(pair.name, 184 | parseInt(log.blockNumber, 16), 185 | parseInt(log.logIndex, 16), 186 | parsedLog.args.tick, 187 | parsedLog.args.sqrtPriceX96.toString()); 188 | } 189 | })(); 190 | 191 | 192 | currBlock += batchSize; 193 | await delay(1000 * crawlDelaySeconds); 194 | } 195 | } 196 | 197 | 198 | 199 | ////// 200 | 201 | async function delay(ms) { 202 | return new Promise(resolve => setTimeout(resolve, ms)); 203 | } 204 | 205 | function getCurrTimeMilliseconds() { 206 | return (new Date()).getTime(); 207 | } 208 | 209 | function unpackPairToCrawl(pair) { 210 | let p = pair.split("/"); 211 | 212 | let o = { 213 | name: pair, 214 | base: p[0], 215 | quote: p[1], 216 | baseAddr: toks[p[0]], 217 | quoteAddr: toks[p[1]], 218 | fee: parseInt(p[2]), 219 | }; 220 | 221 | if (!o.baseAddr) throw(`Couldn't lookup ${p[0]}`); 222 | if (!o.quoteAddr) throw(`Couldn't lookup ${p[0]}`); 223 | 224 | return o; 225 | } 226 | -------------------------------------------------------------------------------- /contracts/MedianOracle.sol: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: UNLICENSED 2 | pragma solidity ^0.8.0; 3 | 4 | contract MedianOracle { 5 | int constant TICK_TRUNCATION = 30; 6 | uint[8192] ringBuffer; 7 | 8 | int16 public currTick; 9 | uint16 public ringCurr; 10 | uint16 public ringSize; 11 | uint64 public lastUpdate; 12 | 13 | constructor(uint16 _ringSize) { 14 | ringCurr = 0; 15 | ringSize = _ringSize; 16 | lastUpdate = uint64(block.timestamp); 17 | } 18 | 19 | function updateOracle(int newTick) external { 20 | require(newTick >= -887272 && newTick <= 887272, "newTick out of range"); 21 | 22 | unchecked { 23 | int _currTick = currTick; 24 | uint _ringCurr = ringCurr; 25 | uint _ringSize = ringSize; 26 | uint _lastUpdate = lastUpdate; 27 | 28 | newTick = quantiseTick(newTick); 29 | 30 | if (newTick == _currTick) return; 31 | 32 | uint elapsed = block.timestamp - _lastUpdate; 33 | 34 | if (elapsed != 0) { 35 | _ringCurr = (_ringCurr + 1) % _ringSize; 36 | writeRing(_ringCurr, _currTick, clampTime(elapsed)); 37 | } 38 | 39 | currTick = int16(newTick); 40 | ringCurr = uint16(_ringCurr); 41 | ringSize = uint16(_ringSize); 42 | lastUpdate = uint64(block.timestamp); 43 | } 44 | } 45 | 46 | function readOracle(uint desiredAge) external view returns (uint16, int24, int24) { // returns (actualAge, median, average) 47 | require(desiredAge <= type(uint16).max, "desiredAge out of range"); 48 | 49 | unchecked { 50 | int _currTick = currTick; 51 | uint _ringCurr = ringCurr; 52 | uint _ringSize = ringSize; 53 | uint cache = lastUpdate; // stores lastUpdate for first part of function, but then overwritten and used for something else 54 | 55 | uint[] memory arr; 56 | uint actualAge = 0; 57 | 58 | // Load ring buffer entries into memory 59 | 60 | { 61 | uint arrSize = 0; 62 | uint256 freeMemoryPointer; 63 | assembly { 64 | arr := mload(0x40) 65 | freeMemoryPointer := add(arr, 0x20) 66 | } 67 | 68 | // Populate first element in arr with current tick, if any time has elapsed since current tick was set 69 | 70 | { 71 | uint duration = clampTime(block.timestamp - cache); 72 | 73 | if (duration != 0) { 74 | if (duration > desiredAge) duration = desiredAge; 75 | actualAge += duration; 76 | 77 | uint packed = memoryPackTick(_currTick, duration); 78 | 79 | assembly { 80 | mstore(freeMemoryPointer, packed) 81 | freeMemoryPointer := add(freeMemoryPointer, 0x20) 82 | } 83 | arrSize++; 84 | } 85 | 86 | _currTick = unQuantiseTick(_currTick) * int(duration); // _currTick now becomes the average accumulator 87 | } 88 | 89 | // Continue populating elements until we have satisfied desiredAge 90 | 91 | { 92 | uint i = _ringCurr; 93 | cache = type(uint).max; // overwrite lastUpdate, use to cache storage reads 94 | 95 | while (actualAge != desiredAge) { 96 | int tick; 97 | uint duration; 98 | 99 | { 100 | if (cache == type(uint).max) cache = ringBuffer[i / 8]; 101 | uint entry = cache >> (32 * (i % 8)); 102 | tick = int(int16(uint16((entry >> 16) & 0xFFFF))); 103 | duration = entry & 0xFFFF; 104 | } 105 | 106 | if (duration == 0) break; // uninitialised 107 | 108 | if (actualAge + duration > desiredAge) duration = desiredAge - actualAge; 109 | actualAge += duration; 110 | 111 | uint packed = memoryPackTick(tick, duration); 112 | 113 | assembly { 114 | mstore(freeMemoryPointer, packed) 115 | freeMemoryPointer := add(freeMemoryPointer, 0x20) 116 | } 117 | arrSize++; 118 | 119 | _currTick += unQuantiseTick(tick) * int(duration); 120 | 121 | if (i & 7 == 0) cache = type(uint).max; 122 | 123 | i = (i + _ringSize - 1) % _ringSize; 124 | if (i == _ringCurr) break; // wrapped back around 125 | } 126 | 127 | assembly { 128 | mstore(arr, arrSize) 129 | mstore(0x40, freeMemoryPointer) 130 | } 131 | } 132 | } 133 | 134 | return ( 135 | uint16(actualAge), 136 | int24(unQuantiseTick(unMemoryPackTick(weightedMedian(arr, actualAge / 2)))), 137 | int24(_currTick / int(actualAge)) 138 | ); 139 | } 140 | } 141 | 142 | // QuickSelect, modified to account for item weights 143 | 144 | function weightedMedian(uint[] memory arr, uint targetWeight) private pure returns (uint) { 145 | unchecked { 146 | uint weightAccum = 0; 147 | uint left = 0; 148 | uint right = (arr.length - 1) * 32; 149 | uint arrp; 150 | 151 | assembly { 152 | arrp := add(arr, 32) 153 | } 154 | 155 | while (true) { 156 | if (left == right) return memload(arrp, left); 157 | 158 | uint pivot = memload(arrp, (left + right) >> 6 << 5); 159 | uint i = left - 32; 160 | uint j = right + 32; 161 | uint leftWeight = 0; 162 | 163 | while (true) { 164 | i += 32; 165 | while (true) { 166 | uint w = memload(arrp, i); 167 | if (w >= pivot) break; 168 | leftWeight += w & 0xFFFF; 169 | i += 32; 170 | } 171 | 172 | do j -= 32; while (memload(arrp, j) > pivot); 173 | 174 | if (i >= j) { 175 | if (i == j) leftWeight += memload(arrp, j) & 0xFFFF; 176 | break; 177 | } 178 | 179 | leftWeight += memswap(arrp, i, j) & 0xFFFF; 180 | } 181 | 182 | if (weightAccum + leftWeight >= targetWeight) { 183 | right = j; 184 | } else { 185 | weightAccum += leftWeight; 186 | left = j + 32; 187 | } 188 | } 189 | } 190 | 191 | assert(false); 192 | return 0; 193 | } 194 | 195 | // Array access without bounds checking 196 | 197 | function memload(uint arrp, uint i) private pure returns (uint ret) { 198 | assembly { 199 | ret := mload(add(arrp, i)) 200 | } 201 | } 202 | 203 | // Swap two items in array without bounds checking, returns new element in i 204 | 205 | function memswap(uint arrp, uint i, uint j) private pure returns (uint output) { 206 | assembly { 207 | let iOffset := add(arrp, i) 208 | let jOffset := add(arrp, j) 209 | output := mload(jOffset) 210 | mstore(jOffset, mload(iOffset)) 211 | mstore(iOffset, output) 212 | } 213 | } 214 | 215 | function writeRing(uint index, int tick, uint duration) private { 216 | unchecked { 217 | uint packed = (uint(uint16(int16(tick))) << 16) | duration; 218 | 219 | uint shift = 32 * (index % 8); 220 | ringBuffer[index / 8] = (ringBuffer[index / 8] & ~(0xFFFFFFFF << shift)) 221 | | (packed << shift); 222 | } 223 | } 224 | 225 | function clampTime(uint t) private pure returns (uint) { 226 | unchecked { 227 | return t > type(uint16).max ? uint(type(uint16).max) : t; 228 | } 229 | } 230 | 231 | function quantiseTick(int tick) private pure returns (int) { 232 | unchecked { 233 | return (tick + (tick < 0 ? -(TICK_TRUNCATION-1) : int(0))) / TICK_TRUNCATION; 234 | } 235 | } 236 | 237 | function unQuantiseTick(int tick) private pure returns (int) { 238 | unchecked { 239 | return tick * TICK_TRUNCATION + (TICK_TRUNCATION/2); 240 | } 241 | } 242 | 243 | function memoryPackTick(int tick, uint duration) private pure returns (uint) { 244 | unchecked { 245 | return (uint(tick + 32768) << 16) | duration; 246 | } 247 | } 248 | 249 | function unMemoryPackTick(uint rec) private pure returns (int) { 250 | unchecked { 251 | return int(rec >> 16) - 32768; 252 | } 253 | } 254 | } 255 | -------------------------------------------------------------------------------- /pics/ring-buffer.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 19 | 21 | 29 | 35 | 36 | 44 | 50 | 51 | 52 | 74 | 76 | 77 | 79 | image/svg+xml 80 | 82 | 83 | 84 | 85 | 86 | 91 | 97 | 102 | 107 | 112 | 117 | 122 | 127 | 133 | 138 | latest 149 | 12 160 | 11 171 | 10 182 | 9 193 | 8 204 | 7 215 | 6 226 | 5 237 | 4 248 | 3 259 | 2 270 | 1 281 | 287 | next to be overwritten 298 | 299 | 300 | -------------------------------------------------------------------------------- /pics/weighted-median.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 19 | 21 | 29 | 35 | 36 | 37 | 59 | 61 | 62 | 64 | image/svg+xml 65 | 67 | 68 | 69 | 70 | 71 | 76 | 83 | 92 | 101 | 108 | 117 | 1.4 129 | 1.3 141 | 1.9 153 | 1.2 165 | 1.4 177 | 184 | 193 | 202 | 209 | 218 | 1.4 230 | 1.3 242 | 1.9 254 | 1.2 266 | 1.4 278 | 284 | 290 | 295 | mid-point 312 | 317 | 322 | 327 | 332 | Price segments 344 | Sorted by price 356 | weighted median = 1.4 368 | 369 | 370 | -------------------------------------------------------------------------------- /pics/linear-search.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 19 | 21 | 29 | 35 | 36 | 44 | 50 | 51 | 59 | 64 | 65 | 73 | 79 | 80 | 81 | 103 | 105 | 106 | 108 | image/svg+xml 109 | 111 | 112 | 113 | 114 | 115 | 120 | 126 | 131 | 136 | 141 | 146 | 151 | 156 | 162 | 167 | latest 178 | 10 189 | 8 200 | 7 211 | 5 222 | 4 233 | 3 244 | 2 255 | 1 266 | 272 | 278 | 12 289 | 11 300 | 9 311 | 6 322 | 328 | Linear Search 339 | 340 | 341 | -------------------------------------------------------------------------------- /contracts/uniswap/Oracle.sol: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BUSL-1.1 2 | pragma solidity >=0.5.0 <0.8.0; 3 | 4 | /// @title Oracle 5 | /// @notice Provides price and liquidity data useful for a wide variety of system designs 6 | /// @dev Instances of stored oracle data, "observations", are collected in the oracle array 7 | /// Every pool is initialized with an oracle array length of 1. Anyone can pay the SSTOREs to increase the 8 | /// maximum length of the oracle array. New slots will be added when the array is fully populated. 9 | /// Observations are overwritten when the full length of the oracle array is populated. 10 | /// The most recent observation is available, independent of the length of the oracle array, by passing 0 to observe() 11 | library Oracle { 12 | struct Observation { 13 | // the block timestamp of the observation 14 | uint32 blockTimestamp; 15 | // the tick accumulator, i.e. tick * time elapsed since the pool was first initialized 16 | int56 tickCumulative; 17 | // the seconds per liquidity, i.e. seconds elapsed / max(1, liquidity) since the pool was first initialized 18 | uint160 secondsPerLiquidityCumulativeX128; 19 | // whether or not the observation is initialized 20 | bool initialized; 21 | } 22 | 23 | /// @notice Transforms a previous observation into a new observation, given the passage of time and the current tick and liquidity values 24 | /// @dev blockTimestamp _must_ be chronologically equal to or greater than last.blockTimestamp, safe for 0 or 1 overflows 25 | /// @param last The specified observation to be transformed 26 | /// @param blockTimestamp The timestamp of the new observation 27 | /// @param tick The active tick at the time of the new observation 28 | /// @param liquidity The total in-range liquidity at the time of the new observation 29 | /// @return Observation The newly populated observation 30 | function transform( 31 | Observation memory last, 32 | uint32 blockTimestamp, 33 | int24 tick, 34 | uint128 liquidity 35 | ) private pure returns (Observation memory) { 36 | uint32 delta = blockTimestamp - last.blockTimestamp; 37 | return 38 | Observation({ 39 | blockTimestamp: blockTimestamp, 40 | tickCumulative: last.tickCumulative + int56(tick) * delta, 41 | secondsPerLiquidityCumulativeX128: last.secondsPerLiquidityCumulativeX128 + 42 | ((uint160(delta) << 128) / (liquidity > 0 ? liquidity : 1)), 43 | initialized: true 44 | }); 45 | } 46 | 47 | /// @notice Initialize the oracle array by writing the first slot. Called once for the lifecycle of the observations array 48 | /// @param self The stored oracle array 49 | /// @param time The time of the oracle initialization, via block.timestamp truncated to uint32 50 | /// @return cardinality The number of populated elements in the oracle array 51 | /// @return cardinalityNext The new length of the oracle array, independent of population 52 | function initialize(Observation[65535] storage self, uint32 time) 53 | internal 54 | returns (uint16 cardinality, uint16 cardinalityNext) 55 | { 56 | self[0] = Observation({ 57 | blockTimestamp: time, 58 | tickCumulative: 0, 59 | secondsPerLiquidityCumulativeX128: 0, 60 | initialized: true 61 | }); 62 | return (1, 1); 63 | } 64 | 65 | /// @notice Writes an oracle observation to the array 66 | /// @dev Writable at most once per block. Index represents the most recently written element. cardinality and index must be tracked externally. 67 | /// If the index is at the end of the allowable array length (according to cardinality), and the next cardinality 68 | /// is greater than the current one, cardinality may be increased. This restriction is created to preserve ordering. 69 | /// @param self The stored oracle array 70 | /// @param index The index of the observation that was most recently written to the observations array 71 | /// @param blockTimestamp The timestamp of the new observation 72 | /// @param tick The active tick at the time of the new observation 73 | /// @param liquidity The total in-range liquidity at the time of the new observation 74 | /// @param cardinality The number of populated elements in the oracle array 75 | /// @param cardinalityNext The new length of the oracle array, independent of population 76 | /// @return indexUpdated The new index of the most recently written element in the oracle array 77 | /// @return cardinalityUpdated The new cardinality of the oracle array 78 | function write( 79 | Observation[65535] storage self, 80 | uint16 index, 81 | uint32 blockTimestamp, 82 | int24 tick, 83 | uint128 liquidity, 84 | uint16 cardinality, 85 | uint16 cardinalityNext 86 | ) internal returns (uint16 indexUpdated, uint16 cardinalityUpdated) { 87 | Observation memory last = self[index]; 88 | 89 | // early return if we've already written an observation this block 90 | if (last.blockTimestamp == blockTimestamp) return (index, cardinality); 91 | 92 | // if the conditions are right, we can bump the cardinality 93 | if (cardinalityNext > cardinality && index == (cardinality - 1)) { 94 | cardinalityUpdated = cardinalityNext; 95 | } else { 96 | cardinalityUpdated = cardinality; 97 | } 98 | 99 | indexUpdated = (index + 1) % cardinalityUpdated; 100 | self[indexUpdated] = transform(last, blockTimestamp, tick, liquidity); 101 | } 102 | 103 | /// @notice Prepares the oracle array to store up to `next` observations 104 | /// @param self The stored oracle array 105 | /// @param current The current next cardinality of the oracle array 106 | /// @param next The proposed next cardinality which will be populated in the oracle array 107 | /// @return next The next cardinality which will be populated in the oracle array 108 | function grow( 109 | Observation[65535] storage self, 110 | uint16 current, 111 | uint16 next 112 | ) internal returns (uint16) { 113 | require(current > 0, 'I'); 114 | // no-op if the passed next value isn't greater than the current next value 115 | if (next <= current) return current; 116 | // store in each slot to prevent fresh SSTOREs in swaps 117 | // this data will not be used because the initialized boolean is still false 118 | for (uint16 i = current; i < next; i++) self[i].blockTimestamp = 1; 119 | return next; 120 | } 121 | 122 | /// @notice comparator for 32-bit timestamps 123 | /// @dev safe for 0 or 1 overflows, a and b _must_ be chronologically before or equal to time 124 | /// @param time A timestamp truncated to 32 bits 125 | /// @param a A comparison timestamp from which to determine the relative position of `time` 126 | /// @param b From which to determine the relative position of `time` 127 | /// @return bool Whether `a` is chronologically <= `b` 128 | function lte( 129 | uint32 time, 130 | uint32 a, 131 | uint32 b 132 | ) private pure returns (bool) { 133 | // if there hasn't been overflow, no need to adjust 134 | if (a <= time && b <= time) return a <= b; 135 | 136 | uint256 aAdjusted = a > time ? a : a + 2**32; 137 | uint256 bAdjusted = b > time ? b : b + 2**32; 138 | 139 | return aAdjusted <= bAdjusted; 140 | } 141 | 142 | /// @notice Fetches the observations beforeOrAt and atOrAfter a target, i.e. where [beforeOrAt, atOrAfter] is satisfied. 143 | /// The result may be the same observation, or adjacent observations. 144 | /// @dev The answer must be contained in the array, used when the target is located within the stored observation 145 | /// boundaries: older than the most recent observation and younger, or the same age as, the oldest observation 146 | /// @param self The stored oracle array 147 | /// @param time The current block.timestamp 148 | /// @param target The timestamp at which the reserved observation should be for 149 | /// @param index The index of the observation that was most recently written to the observations array 150 | /// @param cardinality The number of populated elements in the oracle array 151 | /// @return beforeOrAt The observation recorded before, or at, the target 152 | /// @return atOrAfter The observation recorded at, or after, the target 153 | function binarySearch( 154 | Observation[65535] storage self, 155 | uint32 time, 156 | uint32 target, 157 | uint16 index, 158 | uint16 cardinality 159 | ) private view returns (Observation memory beforeOrAt, Observation memory atOrAfter) { 160 | uint256 l = (index + 1) % cardinality; // oldest observation 161 | uint256 r = l + cardinality - 1; // newest observation 162 | uint256 i; 163 | while (true) { 164 | i = (l + r) / 2; 165 | 166 | beforeOrAt = self[i % cardinality]; 167 | 168 | // we've landed on an uninitialized tick, keep searching higher (more recently) 169 | if (!beforeOrAt.initialized) { 170 | l = i + 1; 171 | continue; 172 | } 173 | 174 | atOrAfter = self[(i + 1) % cardinality]; 175 | 176 | bool targetAtOrAfter = lte(time, beforeOrAt.blockTimestamp, target); 177 | 178 | // check if we've found the answer! 179 | if (targetAtOrAfter && lte(time, target, atOrAfter.blockTimestamp)) break; 180 | 181 | if (!targetAtOrAfter) r = i - 1; 182 | else l = i + 1; 183 | } 184 | } 185 | 186 | /// @notice Fetches the observations beforeOrAt and atOrAfter a given target, i.e. where [beforeOrAt, atOrAfter] is satisfied 187 | /// @dev Assumes there is at least 1 initialized observation. 188 | /// Used by observeSingle() to compute the counterfactual accumulator values as of a given block timestamp. 189 | /// @param self The stored oracle array 190 | /// @param time The current block.timestamp 191 | /// @param target The timestamp at which the reserved observation should be for 192 | /// @param tick The active tick at the time of the returned or simulated observation 193 | /// @param index The index of the observation that was most recently written to the observations array 194 | /// @param liquidity The total pool liquidity at the time of the call 195 | /// @param cardinality The number of populated elements in the oracle array 196 | /// @return beforeOrAt The observation which occurred at, or before, the given timestamp 197 | /// @return atOrAfter The observation which occurred at, or after, the given timestamp 198 | function getSurroundingObservations( 199 | Observation[65535] storage self, 200 | uint32 time, 201 | uint32 target, 202 | int24 tick, 203 | uint16 index, 204 | uint128 liquidity, 205 | uint16 cardinality 206 | ) private view returns (Observation memory beforeOrAt, Observation memory atOrAfter) { 207 | // optimistically set before to the newest observation 208 | beforeOrAt = self[index]; 209 | 210 | // if the target is chronologically at or after the newest observation, we can early return 211 | if (lte(time, beforeOrAt.blockTimestamp, target)) { 212 | if (beforeOrAt.blockTimestamp == target) { 213 | // if newest observation equals target, we're in the same block, so we can ignore atOrAfter 214 | return (beforeOrAt, atOrAfter); 215 | } else { 216 | // otherwise, we need to transform 217 | return (beforeOrAt, transform(beforeOrAt, target, tick, liquidity)); 218 | } 219 | } 220 | 221 | // now, set before to the oldest observation 222 | beforeOrAt = self[(index + 1) % cardinality]; 223 | if (!beforeOrAt.initialized) beforeOrAt = self[0]; 224 | 225 | // ensure that the target is chronologically at or after the oldest observation 226 | require(lte(time, beforeOrAt.blockTimestamp, target), 'OLD'); 227 | 228 | // if we've reached this point, we have to binary search 229 | return binarySearch(self, time, target, index, cardinality); 230 | } 231 | 232 | /// @dev Reverts if an observation at or before the desired observation timestamp does not exist. 233 | /// 0 may be passed as `secondsAgo' to return the current cumulative values. 234 | /// If called with a timestamp falling between two observations, returns the counterfactual accumulator values 235 | /// at exactly the timestamp between the two observations. 236 | /// @param self The stored oracle array 237 | /// @param time The current block timestamp 238 | /// @param secondsAgo The amount of time to look back, in seconds, at which point to return an observation 239 | /// @param tick The current tick 240 | /// @param index The index of the observation that was most recently written to the observations array 241 | /// @param liquidity The current in-range pool liquidity 242 | /// @param cardinality The number of populated elements in the oracle array 243 | /// @return tickCumulative The tick * time elapsed since the pool was first initialized, as of `secondsAgo` 244 | /// @return secondsPerLiquidityCumulativeX128 The time elapsed / max(1, liquidity) since the pool was first initialized, as of `secondsAgo` 245 | function observeSingle( 246 | Observation[65535] storage self, 247 | uint32 time, 248 | uint32 secondsAgo, 249 | int24 tick, 250 | uint16 index, 251 | uint128 liquidity, 252 | uint16 cardinality 253 | ) internal view returns (int56 tickCumulative, uint160 secondsPerLiquidityCumulativeX128) { 254 | if (secondsAgo == 0) { 255 | Observation memory last = self[index]; 256 | if (last.blockTimestamp != time) last = transform(last, time, tick, liquidity); 257 | return (last.tickCumulative, last.secondsPerLiquidityCumulativeX128); 258 | } 259 | 260 | uint32 target = time - secondsAgo; 261 | 262 | (Observation memory beforeOrAt, Observation memory atOrAfter) = 263 | getSurroundingObservations(self, time, target, tick, index, liquidity, cardinality); 264 | 265 | if (target == beforeOrAt.blockTimestamp) { 266 | // we're at the left boundary 267 | return (beforeOrAt.tickCumulative, beforeOrAt.secondsPerLiquidityCumulativeX128); 268 | } else if (target == atOrAfter.blockTimestamp) { 269 | // we're at the right boundary 270 | return (atOrAfter.tickCumulative, atOrAfter.secondsPerLiquidityCumulativeX128); 271 | } else { 272 | // we're in the middle 273 | uint32 observationTimeDelta = atOrAfter.blockTimestamp - beforeOrAt.blockTimestamp; 274 | uint32 targetDelta = target - beforeOrAt.blockTimestamp; 275 | return ( 276 | beforeOrAt.tickCumulative + 277 | ((atOrAfter.tickCumulative - beforeOrAt.tickCumulative) / observationTimeDelta) * 278 | targetDelta, 279 | beforeOrAt.secondsPerLiquidityCumulativeX128 + 280 | uint160( 281 | (uint256( 282 | atOrAfter.secondsPerLiquidityCumulativeX128 - beforeOrAt.secondsPerLiquidityCumulativeX128 283 | ) * targetDelta) / observationTimeDelta 284 | ) 285 | ); 286 | } 287 | } 288 | 289 | /// @notice Returns the accumulator values as of each time seconds ago from the given time in the array of `secondsAgos` 290 | /// @dev Reverts if `secondsAgos` > oldest observation 291 | /// @param self The stored oracle array 292 | /// @param time The current block.timestamp 293 | /// @param secondsAgos Each amount of time to look back, in seconds, at which point to return an observation 294 | /// @param tick The current tick 295 | /// @param index The index of the observation that was most recently written to the observations array 296 | /// @param liquidity The current in-range pool liquidity 297 | /// @param cardinality The number of populated elements in the oracle array 298 | /// @return tickCumulatives The tick * time elapsed since the pool was first initialized, as of each `secondsAgo` 299 | /// @return secondsPerLiquidityCumulativeX128s The cumulative seconds / max(1, liquidity) since the pool was first initialized, as of each `secondsAgo` 300 | function observe( 301 | Observation[65535] storage self, 302 | uint32 time, 303 | uint32[] memory secondsAgos, 304 | int24 tick, 305 | uint16 index, 306 | uint128 liquidity, 307 | uint16 cardinality 308 | ) internal view returns (int56[] memory tickCumulatives, uint160[] memory secondsPerLiquidityCumulativeX128s) { 309 | require(cardinality > 0, 'I'); 310 | 311 | tickCumulatives = new int56[](secondsAgos.length); 312 | secondsPerLiquidityCumulativeX128s = new uint160[](secondsAgos.length); 313 | for (uint256 i = 0; i < secondsAgos.length; i++) { 314 | (tickCumulatives[i], secondsPerLiquidityCumulativeX128s[i]) = observeSingle( 315 | self, 316 | time, 317 | secondsAgos[i], 318 | tick, 319 | index, 320 | liquidity, 321 | cardinality 322 | ); 323 | } 324 | } 325 | } 326 | -------------------------------------------------------------------------------- /pics/binary-search.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 19 | 21 | 29 | 35 | 36 | 44 | 50 | 51 | 59 | 64 | 65 | 74 | 80 | 81 | 89 | 95 | 96 | 97 | 119 | 121 | 122 | 124 | image/svg+xml 125 | 127 | 128 | 129 | 130 | 131 | 136 | 142 | 147 | 152 | 157 | 162 | 167 | 172 | 178 | 183 | latest 194 | 10 205 | 8 216 | 7 227 | 5 238 | 4 249 | 3 260 | 2 271 | 277 | 283 | 289 | 295 | 12 306 | 11 317 | 9 328 | 6 339 | 344 | 352 | 358 | Binary Search 369 | 375 | 1 386 | 387 | 388 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Median Oracle 2 | 3 | This is a proof-of-concept smart contract that proposes a new on-chain price oracle design. It is intended to perform the same function as Uniswap TWAP (Time-Weighted Average Price) oracles: digest the information resulting from trades on an AMM and allow for efficient querying of past prices in a way that mitigates the [snapshot problem](https://blog.euler.finance/prices-and-oracles-2da0126a138) (for example manipulation with flash loans). 4 | 5 | While Uniswap oracles return arithmetic (Uniswap2) or geometric (Uniswap3) price averages within an arbitrary window of time, this oracle returns the median price *and* the geometric TWAP within the window. 6 | 7 | The following plot is from our simulation, using real Swap events from the Uniswap3 USDC/WETH 0.3% pool: 8 | 9 | ![](pics/simulation.png) 10 | 11 | The 30 minute median price tracks the 30 minute TWAPs fairly closely. Both are lagging indicators that effectively smooth out the rough spikes caused by trading activity, although the TWAP glides over smooth curves whereas the median jumps between levels. The TWAPs computed by our oracle don't precisely match those computed by Uniswap3 due to price [quantisation](#quantisation). 12 | 13 | The contract code is available in this repo in the file `contracts/MedianOracle.sol` . 14 | 15 | 16 | 17 | * [Motivation](#motivation) 18 | * [TWAP](#twap) 19 | * [Median Filtering](#median-filtering) 20 | * [Pros and Cons of Median Prices](#pros-and-cons-of-median-prices) 21 | * [Gas Usage](#gas-usage) 22 | * [Design](#design) 23 | * [Limitations](#limitations) 24 | * [Quantisation](#quantisation) 25 | * [Ring Buffer](#ring-buffer) 26 | * [Updates](#updates) 27 | * [Binary Search](#binary-search) 28 | * [Linear Search](#linear-search) 29 | * [Availability](#availability) 30 | * [Simulation](#simulation) 31 | * [Window Sizing](#window-sizing) 32 | * [Binary Search Overhead](#binary-search-overhead) 33 | * [Worst-Case Growth](#worst-case-growth) 34 | * [Future Work](#future-work) 35 | * [Author and Copyright](#author-and-copyright) 36 | 37 | 38 | ## Motivation 39 | 40 | ### TWAP 41 | 42 | Various attacks against TWAPs have been [extensively analysed](https://github.com/euler-xyz/uni-v3-twap-manipulation/blob/master/cost-of-attack.pdf). In most cases TWAPs provide adequate security. TWAPs are *not* vulnerable to "zero-block" attacks where an attacker manipulates a price but then moves it back in the same block. This is because 0 time passes within a block's execution, so the time-weight component is 0. 43 | 44 | However, there are some concerns with TWAP's long-term security properties. 45 | 46 | So called "one-block" attacks are when an attacker manipulates a price and then arbitrages it back in the next block to recover the funds at risk before anybody else has a chance to. This is possible (but risky) when using flashbots or selfish mining (where a miner tries to mine two blocks in a row but withholds the first one until the second is found). However, in order for a one-block attack to significantly modify the TWAP, the price must be moved to a very high (or very low) level, because it needs to outweigh all the other price periods in the window. One of the advantages of geometric over arithmetic averaging is that price security is symmetric in up and down directions (or, equivalently, when the pair is inverted). 47 | 48 | "Multi-block" attacks are when an attacker manipulates a price and then attempts to hold it at this manipulated price for multiple blocks, either sacrificing funds to arbitrageurs or censoring transactions in some manner. These attacks don't require moving the price as far as in a one-block attack since the manipulated price will have a larger time-weight. 49 | 50 | In both of these cases, a TWAP will start to move immediately, and the speed at which it moves is related to how large the price manipulation is. As soon as the averaged price reaches a target level, an attack can begin by, for example, borrowing from a lending protocol using price-inflated collateral. 51 | 52 | The exact attacker threat model for these attacks is still being determined. Certainly it includes attackers who can control or influence multiple blocks in a row. Miners have theoretically always had this ability, and systems like Flashbots have made it more accessible to everyone else. 53 | 54 | Furthermore, with Ethereum's move to Proof of Stake, the mechanism of who creates blocks will change. Rather than being chosen effectively at random each block as with Proof of Work, validators will know ahead of time exactly when they will be able to create blocks and can choose to time attacks based on that. What's more, sometimes by chance validators (or colluding groups of validators) will find they are permitted to create multiple blocks in a row. There are also potentially relevant changes coming to flashbots such as "MEV Boost" which could allow attackers to control entire block bodies and more. 55 | 56 | All of this is to say that we think the time is now to being investigating alternate oracle designs that will improve security in the presence of price-manipulators who are also participants in blockchain consensus. 57 | 58 | ### Median Filtering 59 | 60 | Median filtering is often used in image processing because it is very effective at removing [salt-and-pepper](https://medium.com/analytics-vidhya/remove-salt-and-pepper-noise-with-median-filtering-b739614fe9db) noise. This type of noise is caused by impulses, such as a cosmic ray, or a burned out camera pixel. By throwing away outlying pixels from the image and replacing them with their more reasonable-valued neighbours, images can be cleaned up with minimal distortion. Because the impulses are ignored, their presence has no impact on the output image. This is in contrast with other types of filters that incorporate the impulses into some kind of average. 61 | 62 | For the same reason, our proposed price oracle uses median instead of mean as its foundation. Imagine that every second we pushed the current price onto a big array. To query a window of N seconds, we would take the last N prices, sort them, and return the middle (median) price. If the price has been stable for some period of time, this means that there is no value you can move the price to that will impact the median in the following block. What's more, even if you manipulate the price and are able to hold it at the manipulated level, you will need to sustain this manipulated price for N/2 seconds for it to have an impact. 63 | 64 | Of course, recording a price every second is impractical, so it makes more sense to think of each time interval as a stick, where the length corresponds to the amount of time the price was in effect. The sticks are sorted according to price (either ascending or descending, it doesn't matter) and arranged end-to-end in a big line. The stick that overlaps the mid-point of this big line provides the oracle's output price. In technical terms, this is called the "weighted median", where weight corresponds to stick length. 65 | 66 | 67 | 68 | ### Pros and Cons of Median Prices 69 | 70 | Median prices cannot be significantly influenced unless an attacker can control the block-ending prices of around N/2 blocks in the window. If we want a 30 minute window then an attacker would need to control 72 blocks within a 144 block period (assuming 12.5 second block times, and that the price has been otherwise stable within the period). This is in contrast to TWAP, where a few block-ending prices (or even one) can manipulate the oracle output to a dangerous level, if the manipulations are large enough. 71 | 72 | However, once the median switches over to the new manipulated price after N/2 blocks, the output oracle will jump immediately to that price level. By contrast, the TWAP would still be between the original price and the new manipulated level. 73 | 74 | Additionally, when there is a "legitimate" movement in price, it will take longer to be reflected in the median, while the TWAP will immediately begin to transition to the new level. TWAP price movements are also smooth over time (with per-second granularity) whereas median prices jump between levels at seemingly arbitrary points in time. This smoothness is sometimes a desirable feature, for example when implementing Dutch auction mechanisms (as in Euler's Liquidation module). 75 | 76 | Because neither TWAPs nor median prices are better in all circumstances, our oracle computes both of them at the same time, and allows the caller to choose which one (or both) to use. 77 | 78 | ### Gas Usage 79 | 80 | For many applications, the gas required to read the oracle is a major expense. For example, in lending markets like Compound/AAVE/Euler, a user's net assets and liabilities may need to be computed by a contract to determine if a requested action is permitted, and each of these may need an up-to-date price. 81 | 82 | Although Uniswap3's gas usage is often acceptable, systems that use centralised and [non-objective](https://blog.euler.finance/prices-and-oracles-2da0126a138) oracles like Chainlink currently have a competitive advantage. Ideally gas efficiency of our oracle would match or exceed Chainlink's. 83 | 84 | There are various implementation challenges with computing the median on-chain. First of all, the gas overhead to update the price oracle (ie during a swap) must be minimised so as not to force swappers to maintain the oracle (which they likely don't care about). In this design, we have assumed that update gas costs must remain constant with respect to the number of observations mantained by the oracle, and should involve no more than 1 cold storage slot access (as with Uniswap3). This rules out maintaining the set of observations in any kind of sorted data-structure. Therefore the weighted median must be entirely computed during oracle read time. 85 | 86 | 87 | ## Design 88 | 89 | ### Limitations 90 | 91 | For reasons that will become clear in the following sections, our oracle has a few fundamental API limitations compared to Uniswap: 92 | 93 | * The minimum price resolution available is 0.3%, giving a maximum error of 0.15% (see the [Quantisation](#quantisation) section) 94 | * The maximum time window that can be requested is 65535 seconds (about 18 hours 12 minutes) 95 | * The time window must always be aligned to the present -- you cannot query historical windows 96 | * Only price can be queried, not pool liquidity 97 | 98 | Additionally the proof of concept has a few implementation limitations that will be addressed in the future: 99 | 100 | * The ring buffer is not resizable 101 | * The storage of ring-buffer meta-data is not packed optimally, and will need to be integrated with the application contract 102 | 103 | ### Quantisation 104 | 105 | Uniswap3 has a clever innovation that involves encoding prices into "ticks". These are essentially logarithms of the prices scaled and truncated in such a way that they can be efficiently stored. In Uniswap3's scheme, any tick can be represented by a number between -887272 and 887272 inclusive. This is 20.759 bits of information so it can be stored in an `int24` data-type which occupies only 3 bytes. Since there are many more possible prices than there are ticks, multiple prices map to the same tick, and therefore converting a price into a tick loses information. 106 | 107 | This operation -- lossily compressing a large input domain down to a smaller output range -- is called quantisation. Our oracle design builds upon Uniswap3's tick specification (because we hope that in the future it could be adapted to work with... Uniswap4?), but adds another level of quantisation. 108 | 109 | Given a tick value, we divide it by 30 and take the floor, giving a number between -29576 and 29575 inclusive. This is 15.8521 bits of information and can therefore pack into an `int16` (2 bytes). In order to recover an approximation of the original tick, we multiply by 30 and add 15. In signal processing jargon, this is called a mid-riser quantisation, since if you imagine the plot of inputs to outputs being a staircase, the 0 input is right at the edge of a stair (the "riser"). In our case, an input tick of 0 will remap to a tick of 15, and a tick of -1 to -15. 110 | 111 | Although packing a tick into 2 bytes has significant gas benefits (as we'll describe later), it reduces the precision of prices that can be tracked. While a Uniswap3 tick is always guaranteed to be no more than 0.005% away from the original price, our quantised ticks can be up to 0.15% away. 112 | 113 | So, our oracle trades off price precision for gas efficiency. In real-world AMM usage, price movements are often relatively small. This may be due to small swaps or perhaps larger swaps that are arbitraged back in the same block to nearly the original price. With our oracle, price movements that don't change the quantised tick do not result in an oracle update. This reduces the work needed during reads because there are fewer slots to scan (see the [simulation](#simulation) of USDC/DAI for a stark example of this). Note that Uniswap3 does this as well, but at 0.01% granularity instead of 0.3%. Most Chainlink oracles effectively have a price granularity of 1%. 114 | 115 | The price error introduced by quantisation can be observed by zooming in on a section of our simulation's graphs. At this zoom, we can see quantised levels as flat plateaus that only roughly approximate the trade prices. The Uniswap3 ticks are much more precise, but differences can be seen with them also: 116 | 117 | ![](pics/quantisation.png) 118 | 119 | One possible worst-case trading pattern for our oracle would be a pool that oscillates across a tick boundary frequently. Even though the price movements would be small, each would involve a new entry added to the ring buffer. See the [Future Work](#future-work) section for an idea on how to mitigate this. 120 | 121 | 122 | ### Ring Buffer 123 | 124 | #### Updates 125 | 126 | Like Uniswap3, our proposed oracle uses a ring buffer. Updates in both systems work the same in that they will overwrite older values, meaning that once somebody has paid the gas to populate the entries, no storage writes will involve writing to a storage location containing 0 (which is particularly expensive). 127 | 128 | 129 | 130 | Unlike Uniswap3, however, we pack multiple observations into a single slot. Each 32-byte slot is divided into 8 sub-slots of 4 bytes each. The first two bytes of a sub-slot contain the [quantised](#quantisation) tick and the second two bytes contain the number of seconds since the previous update. 131 | 132 | Because of how time is encoded, this means that an observation cannot be longer than 65535 seconds (a value of 0 seconds is invalid, and is reserved to indicate an uninitialised sub-slot). This results in a limitation of the oracle, in that windows of larger than 65535 seconds (about 18 hours 12 minutes) cannot be queried. Price durations over 65535 seconds [saturate](https://en.wikipedia.org/wiki/Saturation_arithmetic) to 65535, which is acceptable since a window can never be longer than that, meaning processing is guaranteed to stop when it encounters an observation with this age. 133 | 134 | One consequence of this encoding is that pre-populating the ring buffer ("increasing the cardinality" in Uniswap terms) requires 1/8th the gas compared to Uniswap3, per observation. 135 | 136 | #### Binary Search 137 | 138 | Uniswap3's oracle has the advantage of a lower worst-case bound on gas usage. In addition, its gas usage is more predictable. The way that it works is by loading the first and last slots in the ring buffer to check boundary conditions, and then binary searching the ring buffer to find the newest accumulator record older than the requested window, and then interpolating the needed value: 139 | 140 | 141 | 142 | Because binary search halves the search space on every load, this method will make approximately `log2(N)` storage loads, where `N` is the size of the ring buffer. See [Binary Search Overhead](#binary-search-overhead) for an example of how increasing the size of the ring buffer will increase the cost of oracle reads. 143 | 144 | #### Linear Search 145 | 146 | Our oracle's read mechanism is entirely different from Uniswap3's. Because we need to process each element in our window, we have decided to not use accumulators at all, and instead aggregate all the needed data as we scan, with a linear search "backwards" (most recent observation first) in the ring buffer: 147 | 148 | 149 | 150 | First we check if the time since the last update is older than the window. If so, it simply returns a cached copy of the current tick. Both of these values will be packed into a shared storage slot of the containing smart contract, so in this case no ring buffer access is needed whatsoever. 151 | 152 | Otherwise, a memory array is created that will store observations from the ring buffer. If the current price has been in effect for a non-zero amount of time, then a "virtual" observation is pushed onto the memory array representing the elapsed time at the current price. 153 | 154 | The oracle then proceeds to read backwards in the ring buffer until it finds an observation older than or equal to the requested window. Because we keep a cached value of the current ring buffer entry on the stack, 8 elements are fetched with each storage load. Each element read is pushed onto the memory array. If adding the last element onto the array pushes the total observation time over the window length, it is artificially shortened. If there are not enough observations to satisfy the requested window, then the requested window parameter *itself* is shortened. This means that after loading from the ring buffer, the sum of the durations of all elements in the memory array is exactly equal to the (possibly shortened) window length. 155 | 156 | At this point we have an unordered pile of sticks in our memory array. Our original description of weighted median called for sorting them, however that would involve some unnecessary work. We just want to find the element that overlaps the middle of the total length of the stick: we don't care about the orderings of the sticks before or after that point. 157 | 158 | There are various solutions to this problem, but our proof of concept uses the standard textbook solution (literally -- see exercise 9-2 in [CLRS Algorithms](https://www.amazon.com/Introduction-Algorithms-fourth-Thomas-Cormen/dp/026204630X/)). It implements a variation of QuickSelect, which is itself a variation of QuickSort. While QuickSort partitions its input into two segments and then recurses into each one, QuickSelect only recurses into the segment where the position of the element it is seeking resides (which it knows because it has determined the index of the pivot element). This allows a position to be selected in O(N) time, rather than O(N log(N)) as with QuickSort. The variation required for weighted median simply chooses which direction to recurse based on the accumulated weights on one side (compared with half of the total weight), rather than an absolute position (which is unknown). 159 | 160 | As always, there are a few tricks involved getting this to work efficiently on-chain: 161 | 162 | * Solidity doesn't actually support dynamic memory arrays, so unfortunately to do this in one pass we need to use a bit of assembly. This works by saving the free memory pointer ahead of time and then storing each element into unused space. At the end we increase the free memory pointer and store the original value (and length) into a memory array. Of course we need to ensure that no other memory allocations occur during the construction. 163 | * Each element of the memory array is encoded specially. The observation's tick is converted into a non-negative integer by adding a large value to it, and then this value is cast to a `uint` and then shifted left, leaving lower order bits free to contain the duration of the observation. This way, sorting is possible by simply using the regular `>` and `<` operations on the `uint` datatype, and moving the `uint`s automatically carries along the durations. Empirically, unlike storage, packing elements in memory beyond the word size is usually not worthwhile. 164 | * In the weighted median implementation, to reduce overhead as much as possible we use unchecked "pointer arithmetic" instead of array indices. This makes the code a bit harder to follow, for example because we need to do `i += 32` instead of `i++`, but reduces gas usage by over 30%. 165 | 166 | Once we have found the element that overlaps the weight mid-point, it is simply a matter of extracting its tick from the memory encoding, unquantising it, and returning the result. 167 | 168 | ### Availability 169 | 170 | Another difference between Uniswap3 and our proposed oracle is how requests for window lengths that cannot be satisfied are handled. This comes up when a ring buffer is too short and the required data has already been overwritten. 171 | 172 | In this case, Uniswap3 simply fails with the error message `OLD`. This can be problematic for some systems that try to maintain system availability. For example, liquidating a user on a borrowing platform may involve querying the prices for all the user's assets and liabilities. If users can exhaust the ring buffers for one of their tokens, then they could cause liquidations to fail and therefore prevent themselves from being liquidated. 173 | 174 | Sometimes systems would actually be willing to use a shorter window if available. The best way I have found to do this on Uniswap3 is to optimistically query for the ideal window with `observe()`. If that fails with `OLD`, then call `slot0()` on the pool to get the current `index` and `cardinality` and use these parameters to lookup the oldest available observation with `observations()` (and handle the case where that observation is uninitialised), then finally call `observe()` again with the timestamp of this oldest available observation. 175 | 176 | To simplify this, and avoid the extra gas costs, our oracle returns the size of the longest available window, along with the median and TWAP for that (shortened) window. This leaves the decision of what to do up to the calling code. If the window is adequate then the results can be used, otherwise the transaction can be reverted. 177 | 178 | 179 | ## Simulation 180 | 181 | In order to analyse our proposed oracle, we have constructed a simulation. We downloaded all the Uniswap3 `Swap` logs for various common pairs on the Ethereum mainnet, and replayed them in a test environment. The test environment performs the implied pricing oracle updates against our oracle and a stripped down version of Uniswap3 which does nothing except for update the price oracle. This allows us to compare the current price to the median and TWAP, as well as examine gas usage between the two systems. 182 | 183 | The following plots in this section show the cost to read the oracles around the time of the Great Crypto Crash of May 12th, 2022. This period of time was chosen because there was an extreme amount of price activity and trading volume, which are the worst-case conditions for our oracle. Both Uniswap3 and our oracle have their ring buffer sizes set to 144. Our oracle is computing TWAP and median price, where Uniswap3 is computing only TWAP. 184 | 185 | ![](pics/gas-usage-wbtc-weth.png) 186 | 187 | ![](pics/gas-usage-usdc-dai.png) 188 | 189 | ![](pics/gas-usage-usdc-weth.png) 190 | 191 | * The USDC/DAI pair illustrates the best-case scenario for the median price oracle. In this case, the price almost entirely remained within a 0.3% quantisation, so reads could be serviced with either 1 or 2 SLOADs. 192 | * USDC/WETH during the crash is the worst behaviour we have simulated with our oracle. Note that the Uniswap3 gas costs are understated: This pair actually has a ring buffer of size 1440 on mainnet, so in real life its costs are [higher](#binary-search-overhead). 193 | 194 | ### Window Sizing 195 | 196 | One commonly cited advantage of centralised oracle systems like Chainlink is that they can respond to price movements faster than can TWAPs. This is true, but in practice isn't as much of an issue as is implied, for reasons outside the scope of this analysis. 197 | 198 | As described above, the issue with TWAPs is they need to be sufficiently long in order to have many "good" samples out-weigh the presumably few "bad" samples. However, with median oracles, "bad" samples, or to be more neutral, "outliers", do not have significant impact on the output price until their time-in-effect approaches half of the window size. 199 | 200 | Because of this, we have reason to believe that median oracles can support shorter windows than TWAPs while maintaining equivalent security. If true, this would have two benefits: 201 | 202 | * Legitimate price movements are reflected in the oracle output faster, leaving shorter opportunities to attack protocols with stale prices 203 | * Shorter windows reduce worst-case gas consumption 204 | 205 | To demonstrate the second point, we re-ran the USDC/WETH example above with a 10 minute window instead of a 30 minute window. There was a significant improvement, and even at the peak of the crash the highest gas usage remained well below typical Uniswap3 costs: 206 | 207 | ![](pics/gas-usage-usdc-weth-10min-window.png) 208 | 209 | How much shorter the windows can be still needs to be researched. It will be especially important to create a detailed attacker threat model considering the changes resulting from Proof of Stake, MEV Boost, etc. 210 | 211 | ### Binary Search Overhead 212 | 213 | As described above, the larger the ring buffer grows in Uniswap3, the higher the gas costs for price reads will be. For example, here is the simulation of Uniswap3's oracle given two different ring buffer sizes: 144 and 1440 (the current cardinality for USDC/WETH .3% at the time of writing). Both simulations are querying a 30 minute window: 214 | 215 | ![](pics/uniswap-ring-buffer-sizes.png) 216 | 217 | The initial data is skewed since the buffer is being populated. Once the horizontal bands start we are at the steady state. We haven't investigated this distribution in detail, but believe the bands have to do with how many iterations the binary search requires. By contrast, our proposed oracle is not affected by larger ring buffer sizes (assuming that the desired window is satisfiable). 218 | 219 | ### Worst-Case Growth 220 | 221 | The major disadvantage of our oracle, aside from the API limitations described above, is that its worst-case gas usage is higher that Uniswap3's. Even though in nearly every real-world situation we have simulated our oracle uses less gas than Uniswap3, we should carefully consider the worst-case, which could arise in the following circumstances: 222 | 223 | * Extreme market volatility, during which very high gas costs could be especially harmful (users trying to close positions, etc) 224 | * Adversarial price activity designed to cause expense/inconvenience to oracle consumers (ie, users trying to prevent liquidation of their accounts) 225 | 226 | There are two primary cost centres in our oracle design: 227 | 228 | 1. Loading the required ring-buffer entries from storage into memory 229 | 1. Finding the weighted median with QuickSelect 230 | 231 | The following diagram shows how the cost of querying the oracle grows versus the number of ring buffer elements it needs to access. Each element represents a 0.3% or greater price movement from the previous block. 232 | 233 | The "Average only" line was created with a modified version of the oracle that doesn't compute the median at all. The "Best Case" line uses pre-sorted price movements so the least possible amount of work needs to be performed by QuickSelect. Finally, the "Random" line uses a random shuffling of prices, to better represent typical costs: 234 | 235 | 236 | 237 | In the above graph a distinct "stair-case" pattern can be seen in the Average and Best Case lines. Each step represents an additional SLOAD from the ring buffer (since 8 elements are packed per storage slot). 238 | 239 | Another concern with QuickSelect (and the QuickSort family in general) is that certain specially crafted inputs can cause the algorithm to degrade to `O(N^2)` performance, depending on how the "pivot" element is selected: 240 | 241 | 242 | 243 | This behaviour will almost never arise naturally, so it would need to be an adversarially selected input. Our plan to deal with this is to have the pivot elements selected at random (a very common implementation technique with QuickSort). The randomness would be sourced from a PRNG seeded with the hash of the following: 244 | 245 | * Previous block hash: So that an attack is only applicable for a single block 246 | * `msg.sender`: So that an attacker can't target multiple people at once 247 | 248 | One approach that oracle consumers can take is to limit the gas available for an oracle read. If it exceeds a gas allowance then the transaction could be aborted, or other action taken. This is only useful for contracts that can accept some [unavailability](#availability) from their price feeds. 249 | 250 | Alternatively, we could add a ring buffer entry `limit` parameter to the API. If this limit is hit, then the backwards scan will stop and the results for a shorter window will be returned instead. 251 | 252 | 253 | 254 | ## Future Work 255 | 256 | * The ring buffer is not currently resizeable, however this is a relatively simple change and can be done in the same way as Uniswap3. 257 | * In order to prevent the issue of prices oscillating around a quantised tick boundary and causing frequent appends to the ring buffer, a possible solution may be to look back one additional entry when applying a price update where the price has only moved one tick (and perhaps only if we're *not* on a storage slot boundary). If the previous entry is the same tick as the new tick, the most recent entry in the price oracle could be updated to account for the passed time, and then swapped with the previous entry. This would very slightly skew the calculated median (by at most one tick), but probably would not be noticeable in practice. 258 | * Our initial design had a "threshold" parameter that could be specified when reading the oracle. This would further quantise the data after reading from the ring buffer in order to coalesce nearby ticks together, further trading off price accuracy for gas efficiency. This might benefit from future study. 259 | 260 | 261 | 262 | ## Author and Copyright 263 | 264 | Doug Hoyte 265 | 266 | (C) Euler XYZ Ltd. 267 | 268 | The code is currently unlicensed, but will be released under an open source license in the near future. 269 | --------------------------------------------------------------------------------