├── .gitattributes ├── .gitignore ├── DB.cs ├── EthInclude.csproj ├── EthInclude.sln ├── Flashbots.cs ├── FlashbotsModel.cs ├── LICENSE ├── MySql ├── create_db_tx_time │ ├── tx_time_eth_tx.sql │ ├── tx_time_fb_block.sql │ ├── tx_time_fb_bundle.sql │ ├── tx_time_fb_bundle_groupby.sql │ └── tx_time_fb_tx.sql └── reports │ ├── delay_distribution.sql │ ├── delay_stats.sql │ ├── mev_report.sql │ ├── sandwich_backruns_browser.sql │ └── update_bundles_nblock.sql ├── Program.cs ├── README.md ├── TxTime.cs └── appsettings.json /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # Settings file 7 | appsettings.json 8 | 9 | # User-specific files 10 | *.rsuser 11 | *.suo 12 | *.user 13 | *.userosscache 14 | *.sln.docstates 15 | 16 | # User-specific files (MonoDevelop/Xamarin Studio) 17 | *.userprefs 18 | 19 | # Mono auto generated files 20 | mono_crash.* 21 | 22 | # Build results 23 | [Dd]ebug/ 24 | [Dd]ebugPublic/ 25 | [Rr]elease/ 26 | [Rr]eleases/ 27 | x64/ 28 | x86/ 29 | [Ww][Ii][Nn]32/ 30 | [Aa][Rr][Mm]/ 31 | [Aa][Rr][Mm]64/ 32 | bld/ 33 | [Bb]in/ 34 | [Oo]bj/ 35 | [Oo]ut/ 36 | [Ll]og/ 37 | [Ll]ogs/ 38 | 39 | # Visual Studio 2015/2017 cache/options directory 40 | .vs/ 41 | # Uncomment if you have tasks that create the project's static files in wwwroot 42 | #wwwroot/ 43 | 44 | # Visual Studio 2017 auto generated files 45 | Generated\ Files/ 46 | 47 | # MSTest test Results 48 | [Tt]est[Rr]esult*/ 49 | [Bb]uild[Ll]og.* 50 | 51 | # NUnit 52 | *.VisualState.xml 53 | TestResult.xml 54 | nunit-*.xml 55 | 56 | # Build Results of an ATL Project 57 | [Dd]ebugPS/ 58 | [Rr]eleasePS/ 59 | dlldata.c 60 | 61 | # Benchmark Results 62 | BenchmarkDotNet.Artifacts/ 63 | 64 | # .NET Core 65 | project.lock.json 66 | project.fragment.lock.json 67 | artifacts/ 68 | 69 | # ASP.NET Scaffolding 70 | ScaffoldingReadMe.txt 71 | 72 | # StyleCop 73 | StyleCopReport.xml 74 | 75 | # Files built by Visual Studio 76 | *_i.c 77 | *_p.c 78 | *_h.h 79 | *.ilk 80 | *.meta 81 | *.obj 82 | *.iobj 83 | *.pch 84 | *.pdb 85 | *.ipdb 86 | *.pgc 87 | *.pgd 88 | *.rsp 89 | *.sbr 90 | *.tlb 91 | *.tli 92 | *.tlh 93 | *.tmp 94 | *.tmp_proj 95 | *_wpftmp.csproj 96 | *.log 97 | *.vspscc 98 | *.vssscc 99 | .builds 100 | *.pidb 101 | *.svclog 102 | *.scc 103 | 104 | # Chutzpah Test files 105 | _Chutzpah* 106 | 107 | # Visual C++ cache files 108 | ipch/ 109 | *.aps 110 | *.ncb 111 | *.opendb 112 | *.opensdf 113 | *.sdf 114 | *.cachefile 115 | *.VC.db 116 | *.VC.VC.opendb 117 | 118 | # Visual Studio profiler 119 | *.psess 120 | *.vsp 121 | *.vspx 122 | *.sap 123 | 124 | # Visual Studio Trace Files 125 | *.e2e 126 | 127 | # TFS 2012 Local Workspace 128 | $tf/ 129 | 130 | # Guidance Automation Toolkit 131 | *.gpState 132 | 133 | # ReSharper is a .NET coding add-in 134 | _ReSharper*/ 135 | *.[Rr]e[Ss]harper 136 | *.DotSettings.user 137 | 138 | # TeamCity is a build add-in 139 | _TeamCity* 140 | 141 | # DotCover is a Code Coverage Tool 142 | *.dotCover 143 | 144 | # AxoCover is a Code Coverage Tool 145 | .axoCover/* 146 | !.axoCover/settings.json 147 | 148 | # Coverlet is a free, cross platform Code Coverage Tool 149 | coverage*.json 150 | coverage*.xml 151 | coverage*.info 152 | 153 | # Visual Studio code coverage results 154 | *.coverage 155 | *.coveragexml 156 | 157 | # NCrunch 158 | _NCrunch_* 159 | .*crunch*.local.xml 160 | nCrunchTemp_* 161 | 162 | # MightyMoose 163 | *.mm.* 164 | AutoTest.Net/ 165 | 166 | # Web workbench (sass) 167 | .sass-cache/ 168 | 169 | # Installshield output folder 170 | [Ee]xpress/ 171 | 172 | # DocProject is a documentation generator add-in 173 | DocProject/buildhelp/ 174 | DocProject/Help/*.HxT 175 | DocProject/Help/*.HxC 176 | DocProject/Help/*.hhc 177 | DocProject/Help/*.hhk 178 | DocProject/Help/*.hhp 179 | DocProject/Help/Html2 180 | DocProject/Help/html 181 | 182 | # Click-Once directory 183 | publish/ 184 | 185 | # Publish Web Output 186 | *.[Pp]ublish.xml 187 | *.azurePubxml 188 | # Note: Comment the next line if you want to checkin your web deploy settings, 189 | # but database connection strings (with potential passwords) will be unencrypted 190 | *.pubxml 191 | *.publishproj 192 | 193 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 194 | # checkin your Azure Web App publish settings, but sensitive information contained 195 | # in these scripts will be unencrypted 196 | PublishScripts/ 197 | 198 | # NuGet Packages 199 | *.nupkg 200 | # NuGet Symbol Packages 201 | *.snupkg 202 | # The packages folder can be ignored because of Package Restore 203 | **/[Pp]ackages/* 204 | # except build/, which is used as an MSBuild target. 205 | !**/[Pp]ackages/build/ 206 | # Uncomment if necessary however generally it will be regenerated when needed 207 | #!**/[Pp]ackages/repositories.config 208 | # NuGet v3's project.json files produces more ignorable files 209 | *.nuget.props 210 | *.nuget.targets 211 | 212 | # Microsoft Azure Build Output 213 | csx/ 214 | *.build.csdef 215 | 216 | # Microsoft Azure Emulator 217 | ecf/ 218 | rcf/ 219 | 220 | # Windows Store app package directories and files 221 | AppPackages/ 222 | BundleArtifacts/ 223 | Package.StoreAssociation.xml 224 | _pkginfo.txt 225 | *.appx 226 | *.appxbundle 227 | *.appxupload 228 | 229 | # Visual Studio cache files 230 | # files ending in .cache can be ignored 231 | *.[Cc]ache 232 | # but keep track of directories ending in .cache 233 | !?*.[Cc]ache/ 234 | 235 | # Others 236 | ClientBin/ 237 | ~$* 238 | *~ 239 | *.dbmdl 240 | *.dbproj.schemaview 241 | *.jfm 242 | *.pfx 243 | *.publishsettings 244 | orleans.codegen.cs 245 | 246 | # Including strong name files can present a security risk 247 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 248 | #*.snk 249 | 250 | # Since there are multiple workflows, uncomment next line to ignore bower_components 251 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 252 | #bower_components/ 253 | 254 | # RIA/Silverlight projects 255 | Generated_Code/ 256 | 257 | # Backup & report files from converting an old project file 258 | # to a newer Visual Studio version. Backup files are not needed, 259 | # because we have git ;-) 260 | _UpgradeReport_Files/ 261 | Backup*/ 262 | UpgradeLog*.XML 263 | UpgradeLog*.htm 264 | ServiceFabricBackup/ 265 | *.rptproj.bak 266 | 267 | # SQL Server files 268 | *.mdf 269 | *.ldf 270 | *.ndf 271 | 272 | # Business Intelligence projects 273 | *.rdl.data 274 | *.bim.layout 275 | *.bim_*.settings 276 | *.rptproj.rsuser 277 | *- [Bb]ackup.rdl 278 | *- [Bb]ackup ([0-9]).rdl 279 | *- [Bb]ackup ([0-9][0-9]).rdl 280 | 281 | # Microsoft Fakes 282 | FakesAssemblies/ 283 | 284 | # GhostDoc plugin setting file 285 | *.GhostDoc.xml 286 | 287 | # Node.js Tools for Visual Studio 288 | .ntvs_analysis.dat 289 | node_modules/ 290 | 291 | # Visual Studio 6 build log 292 | *.plg 293 | 294 | # Visual Studio 6 workspace options file 295 | *.opt 296 | 297 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 298 | *.vbw 299 | 300 | # Visual Studio LightSwitch build output 301 | **/*.HTMLClient/GeneratedArtifacts 302 | **/*.DesktopClient/GeneratedArtifacts 303 | **/*.DesktopClient/ModelManifest.xml 304 | **/*.Server/GeneratedArtifacts 305 | **/*.Server/ModelManifest.xml 306 | _Pvt_Extensions 307 | 308 | # Paket dependency manager 309 | .paket/paket.exe 310 | paket-files/ 311 | 312 | # FAKE - F# Make 313 | .fake/ 314 | 315 | # CodeRush personal settings 316 | .cr/personal 317 | 318 | # Python Tools for Visual Studio (PTVS) 319 | __pycache__/ 320 | *.pyc 321 | 322 | # Cake - Uncomment if you are using it 323 | # tools/** 324 | # !tools/packages.config 325 | 326 | # Tabs Studio 327 | *.tss 328 | 329 | # Telerik's JustMock configuration file 330 | *.jmconfig 331 | 332 | # BizTalk build output 333 | *.btp.cs 334 | *.btm.cs 335 | *.odx.cs 336 | *.xsd.cs 337 | 338 | # OpenCover UI analysis results 339 | OpenCover/ 340 | 341 | # Azure Stream Analytics local run output 342 | ASALocalRun/ 343 | 344 | # MSBuild Binary and Structured Log 345 | *.binlog 346 | 347 | # NVidia Nsight GPU debugger configuration file 348 | *.nvuser 349 | 350 | # MFractors (Xamarin productivity tool) working folder 351 | .mfractor/ 352 | 353 | # Local History for Visual Studio 354 | .localhistory/ 355 | 356 | # BeatPulse healthcheck temp database 357 | healthchecksdb 358 | 359 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 360 | MigrationBackup/ 361 | 362 | # Ionide (cross platform F# VS Code tools) working folder 363 | .ionide/ 364 | 365 | # Fody - auto-generated XML schema 366 | FodyWeavers.xsd 367 | -------------------------------------------------------------------------------- /DB.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | using System.Diagnostics; 7 | using Nethereum.JsonRpc.WebSocketStreamingClient; 8 | using Nethereum.RPC.Eth.DTOs; 9 | using Nethereum.RPC.Reactive.Eth.Subscriptions; 10 | using Nethereum.Web3; 11 | using MySql.Data; 12 | using MySql.Data.MySqlClient; 13 | using Microsoft.Extensions.Configuration; 14 | 15 | namespace EthInclude 16 | { 17 | public class DB 18 | { 19 | static IConfiguration _config; 20 | public static ulong PendingTxsToImport; 21 | 22 | public static string GetConfig(string key) 23 | { 24 | if (_config == null) 25 | _config = new ConfigurationBuilder() 26 | .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) 27 | .Build(); 28 | 29 | return _config[key]; 30 | } 31 | 32 | public static void WriteFlashbots(FB.Root fb) 33 | { 34 | if (fb == null) 35 | return; 36 | 37 | Stopwatch sw = new Stopwatch(); 38 | sw.Start(); 39 | 40 | using (MySqlConnection conn = new MySqlConnection(_config["MySqlConnection"])) 41 | { 42 | conn.Open(); 43 | using (var cmd = new MySqlCommand()) 44 | { 45 | cmd.Connection = conn; 46 | 47 | foreach (FB.Block b in fb.blocks) 48 | { 49 | cmd.CommandText = @$"INSERT IGNORE INTO `tx_time`.`fb_block` 50 | (`block_number`, 51 | `miner_reward`, 52 | `miner`, 53 | `coinbase_transfer`, 54 | `gas_used`, 55 | `gas_price`) 56 | VALUES 57 | ({b.block_number}, 58 | {b.miner_reward}, 59 | '{b.miner}', 60 | {b.coinbase_transfers}, 61 | {b.gas_used}, 62 | {b.gas_price})"; 63 | cmd.ExecuteNonQuery(); 64 | 65 | foreach (FB.Transaction t in b.transactions) 66 | { 67 | cmd.CommandText = @$"INSERT IGNORE INTO `tx_time`.`fb_tx` 68 | (`hash`, 69 | `index`, 70 | `bundle`, 71 | `block_number`, 72 | `eoa_address`, 73 | `to_address`, 74 | `gas_used`, 75 | `gas_price`, 76 | `coinbase_transfer`, 77 | `total_miner_reward`) 78 | VALUES 79 | ('{t.transaction_hash}', 80 | {t.tx_index}, 81 | {t.bundle_index}, 82 | {t.block_number}, 83 | '{t.eoa_address}', 84 | '{t.to_address}', 85 | {t.gas_used}, 86 | {t.gas_price}, 87 | '{t.coinbase_transfer}', 88 | {t.total_miner_reward})"; 89 | try 90 | { 91 | cmd.ExecuteNonQuery(); 92 | } 93 | catch (Exception e) 94 | { 95 | Console.WriteLine("error WriteFlashbots " + e.ToString()); 96 | } 97 | } 98 | } 99 | } 100 | conn.Close(); 101 | 102 | sw.Stop(); 103 | Console.WriteLine("fb block {2} update {0} blocks in {1} ms", fb.blocks.Count, sw.ElapsedMilliseconds, fb.latest_block_number); 104 | } 105 | } 106 | 107 | public static async Task WriteTxTimeAsync(List tts) 108 | { 109 | await Task.Run(() => 110 | { 111 | Stopwatch sw = new Stopwatch(); 112 | sw.Start(); 113 | 114 | using (MySqlConnection conn = new MySqlConnection(DB.GetConfig("MySqlConnection"))) 115 | { 116 | conn.Open(); 117 | using (var cmd = new MySqlCommand()) 118 | { 119 | cmd.Connection = conn; 120 | 121 | foreach (TxTime tt in tts) 122 | { 123 | // this happens when we haven't yet recieved a block 124 | if (tt.ArrivalBlockNumber == null) 125 | continue; 126 | 127 | string sql = @$"INSERT INTO `tx_time`.`eth_tx` 128 | (`hash`, 129 | `block_number`, 130 | `index`, 131 | `gas`, 132 | `gas_price`, 133 | `arrival_time`, 134 | `inclusion_time`, 135 | `is_warm`, 136 | `is_dark`, 137 | `delay_ms`, 138 | `arrival_index`, 139 | `inclusion_index`, 140 | `arrival_block`) 141 | VALUES 142 | ('{tt.Hash}', 143 | {tt.BlockNumber}, 144 | {tt.TxIndex}, 145 | {tt.Gas}, 146 | {tt.GasPrice}, 147 | '{tt.ArrivalTime.ToString("yyyy-MM-dd HH:mm:ss.fff")}', 148 | '{tt.InclusionTime.ToString("yyyy-MM-dd HH:mm:ss.fff")}', 149 | {(tt.IsWarm ? "1" : "0")}, 150 | {(tt.IsDark ? "1" : "0")}, 151 | {tt.Delay.TotalMilliseconds}, 152 | {tt.ArrivalIndex}, 153 | {tt.InclusionIndex}, 154 | {tt.ArrivalBlockNumber});"; 155 | cmd.CommandText = sql; 156 | try 157 | { 158 | cmd.ExecuteNonQuery(); 159 | } 160 | catch (Exception e) 161 | { 162 | Console.WriteLine("error WriteTxTimeAsync " + e.ToString()); 163 | } 164 | } 165 | } 166 | conn.Close(); 167 | } 168 | 169 | sw.Stop(); 170 | Console.WriteLine("tx_time update {0} rows in {1} ms", tts.Count, sw.ElapsedMilliseconds); 171 | }); 172 | } 173 | } 174 | } -------------------------------------------------------------------------------- /EthInclude.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Exe 5 | netcoreapp3.1 6 | eth_tx_order 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | PreserveNewest 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /EthInclude.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.31205.134 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EthInclude", "EthInclude.csproj", "{0A410433-F5D6-4FBD-90F8-86212384F08E}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {0A410433-F5D6-4FBD-90F8-86212384F08E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {0A410433-F5D6-4FBD-90F8-86212384F08E}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {0A410433-F5D6-4FBD-90F8-86212384F08E}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {0A410433-F5D6-4FBD-90F8-86212384F08E}.Release|Any CPU.Build.0 = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | GlobalSection(ExtensibilityGlobals) = postSolution 23 | SolutionGuid = {D6913E30-E512-49EC-B883-94E39E0F967C} 24 | EndGlobalSection 25 | EndGlobal 26 | -------------------------------------------------------------------------------- /Flashbots.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using System.IO; 5 | using System.Net; 6 | using System.Threading.Tasks; 7 | using Newtonsoft.Json; 8 | 9 | namespace EthInclude 10 | { 11 | public class Flashbots 12 | { 13 | const string FlashbotsUri = @"https://blocks.flashbots.net/v1/blocks?limit=3"; // return the latest 3 blocks to make up for any we missed 14 | 15 | public static async Task Collect(int preDelayMs) 16 | { 17 | await Task.Run(() => 18 | { 19 | System.Threading.Thread.Sleep(preDelayMs); 20 | FB.Root fb = Flashbots.Get(); 21 | DB.WriteFlashbots(fb); 22 | return; 23 | }); 24 | } 25 | 26 | public static FB.Root Get() 27 | { 28 | FB.Root root = null; 29 | try 30 | { 31 | HttpWebRequest req = (HttpWebRequest)WebRequest.Create(FlashbotsUri); 32 | using (WebResponse resp = req.GetResponse()) 33 | { 34 | StreamReader sr = new StreamReader(resp.GetResponseStream()); 35 | string json = sr.ReadToEnd(); 36 | root = JsonConvert.DeserializeObject(json); 37 | resp.Close(); // try everything to close the connection so source ip works 38 | } 39 | } 40 | catch (Exception e) 41 | { 42 | Console.WriteLine("error Flashbots.Get() " + e.ToString()); 43 | } 44 | return root; 45 | } 46 | } 47 | } -------------------------------------------------------------------------------- /FlashbotsModel.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics.CodeAnalysis; 4 | using System.Text; 5 | using System.Text.Json.Serialization; 6 | using Nethereum.Hex.HexTypes; 7 | 8 | namespace EthInclude.FB 9 | { 10 | // Root myDeserializedClass = JsonConvert.DeserializeObject(myJsonResponse); 11 | public class Transaction : IComparable 12 | { 13 | public string transaction_hash { get; set; } 14 | public int tx_index { get; set; } 15 | public int bundle_index { get; set; } 16 | public ulong block_number { get; set; } 17 | public string eoa_address { get; set; } 18 | public string to_address { get; set; } 19 | public ulong gas_used { get; set; } 20 | public ulong gas_price { get; set; } 21 | public ulong coinbase_transfer { get; set; } 22 | public ulong total_miner_reward { get; set; } 23 | 24 | public int CompareTo(Transaction other) 25 | { 26 | if (this.bundle_index != other.bundle_index) 27 | return this.bundle_index.CompareTo(other.bundle_index); 28 | return this.tx_index.CompareTo(other.tx_index); 29 | } 30 | } 31 | 32 | public class Block 33 | { 34 | public ulong block_number { get; set; } 35 | public ulong miner_reward { get; set; } 36 | public string miner { get; set; } 37 | public ulong coinbase_transfers { get; set; } 38 | public ulong gas_used { get; set; } 39 | public ulong gas_price { get; set; } 40 | public List transactions { get; set; } 41 | } 42 | 43 | public class Root 44 | { 45 | public List blocks { get; set; } 46 | public ulong latest_block_number { get; set; } 47 | } 48 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 pmcgoohan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MySql/create_db_tx_time/tx_time_eth_tx.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.13 Distrib 5.6.24, for Win64 (x86_64) 2 | -- 3 | -- Host: localhost Database: tx_time 4 | -- ------------------------------------------------------ 5 | -- Server version 8.0.23 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Table structure for table `eth_tx` 20 | -- 21 | 22 | DROP TABLE IF EXISTS `eth_tx`; 23 | /*!40101 SET @saved_cs_client = @@character_set_client */; 24 | /*!40101 SET character_set_client = utf8 */; 25 | CREATE TABLE `eth_tx` ( 26 | `hash` char(66) NOT NULL, 27 | `block_number` bigint unsigned DEFAULT NULL, 28 | `index` int DEFAULT NULL, 29 | `gas` bigint unsigned DEFAULT NULL, 30 | `gas_price` bigint unsigned DEFAULT NULL, 31 | `arrival_time` datetime(3) DEFAULT NULL, 32 | `inclusion_time` datetime(3) DEFAULT NULL, 33 | `is_warm` bit(1) DEFAULT NULL, 34 | `is_dark` bit(1) DEFAULT NULL, 35 | `delay_ms` int DEFAULT NULL, 36 | `arrival_index` bigint unsigned DEFAULT NULL, 37 | `inclusion_index` bigint unsigned DEFAULT NULL, 38 | `arrival_block` bigint DEFAULT NULL, 39 | PRIMARY KEY (`hash`) 40 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; 41 | /*!40101 SET character_set_client = @saved_cs_client */; 42 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 43 | 44 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 45 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 46 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 47 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 48 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 49 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 50 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 51 | 52 | -- Dump completed on 2021-07-07 10:00:23 53 | -------------------------------------------------------------------------------- /MySql/create_db_tx_time/tx_time_fb_block.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.13 Distrib 5.6.24, for Win64 (x86_64) 2 | -- 3 | -- Host: localhost Database: tx_time 4 | -- ------------------------------------------------------ 5 | -- Server version 8.0.23 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Table structure for table `fb_block` 20 | -- 21 | 22 | DROP TABLE IF EXISTS `fb_block`; 23 | /*!40101 SET @saved_cs_client = @@character_set_client */; 24 | /*!40101 SET character_set_client = utf8 */; 25 | CREATE TABLE `fb_block` ( 26 | `block_number` bigint unsigned NOT NULL, 27 | `miner_reward` bigint unsigned DEFAULT NULL, 28 | `miner` varchar(42) DEFAULT NULL, 29 | `coinbase_transfer` bigint unsigned DEFAULT NULL, 30 | `gas_used` bigint unsigned DEFAULT NULL, 31 | `gas_price` bigint unsigned DEFAULT NULL, 32 | `bundle_count` int DEFAULT NULL, 33 | PRIMARY KEY (`block_number`) 34 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; 35 | /*!40101 SET character_set_client = @saved_cs_client */; 36 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 37 | 38 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 39 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 40 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 41 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 42 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 43 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 44 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 45 | 46 | -- Dump completed on 2021-07-07 10:00:23 47 | -------------------------------------------------------------------------------- /MySql/create_db_tx_time/tx_time_fb_bundle.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.13 Distrib 5.6.24, for Win64 (x86_64) 2 | -- 3 | -- Host: localhost Database: tx_time 4 | -- ------------------------------------------------------ 5 | -- Server version 8.0.23 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Table structure for table `fb_bundle` 20 | -- 21 | 22 | DROP TABLE IF EXISTS `fb_bundle`; 23 | /*!40101 SET @saved_cs_client = @@character_set_client */; 24 | /*!40101 SET character_set_client = utf8 */; 25 | CREATE TABLE `fb_bundle` ( 26 | `block_number` bigint unsigned NOT NULL, 27 | `bundle` int NOT NULL, 28 | `count` int DEFAULT NULL, 29 | `is_first_dark` bit(1) DEFAULT NULL, 30 | `is_last_dark` bit(1) DEFAULT NULL, 31 | `not_dark_count` int DEFAULT NULL, 32 | `min_delay_ms` int DEFAULT NULL, 33 | `max_delay_ms` int DEFAULT NULL, 34 | `sum_delay_ms` int DEFAULT NULL, 35 | `is_within_n` bit(1) DEFAULT NULL, 36 | PRIMARY KEY (`block_number`,`bundle`), 37 | KEY `idx_fb_bundle_bundle` (`bundle`) 38 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; 39 | /*!40101 SET character_set_client = @saved_cs_client */; 40 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 41 | 42 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 43 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 44 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 45 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 46 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 47 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 48 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 49 | 50 | -- Dump completed on 2021-07-07 10:00:23 51 | -------------------------------------------------------------------------------- /MySql/create_db_tx_time/tx_time_fb_bundle_groupby.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.13 Distrib 5.6.24, for Win64 (x86_64) 2 | -- 3 | -- Host: localhost Database: tx_time 4 | -- ------------------------------------------------------ 5 | -- Server version 8.0.23 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Table structure for table `fb_bundle_groupby` 20 | -- 21 | 22 | DROP TABLE IF EXISTS `fb_bundle_groupby`; 23 | /*!40101 SET @saved_cs_client = @@character_set_client */; 24 | /*!40101 SET character_set_client = utf8 */; 25 | CREATE TABLE `fb_bundle_groupby` ( 26 | `block_number` bigint unsigned DEFAULT NULL, 27 | `bundle` int DEFAULT NULL, 28 | `not_dark_count` bigint NOT NULL DEFAULT '0', 29 | `min_delay_ms` int DEFAULT NULL, 30 | `max_delay_ms` int DEFAULT NULL, 31 | `sum_delay_ms` decimal(32,0) DEFAULT NULL 32 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; 33 | /*!40101 SET character_set_client = @saved_cs_client */; 34 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 35 | 36 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 37 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 38 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 39 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 40 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 41 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 42 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 43 | 44 | -- Dump completed on 2021-07-07 10:00:23 45 | -------------------------------------------------------------------------------- /MySql/create_db_tx_time/tx_time_fb_tx.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.13 Distrib 5.6.24, for Win64 (x86_64) 2 | -- 3 | -- Host: localhost Database: tx_time 4 | -- ------------------------------------------------------ 5 | -- Server version 8.0.23 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Table structure for table `fb_tx` 20 | -- 21 | 22 | DROP TABLE IF EXISTS `fb_tx`; 23 | /*!40101 SET @saved_cs_client = @@character_set_client */; 24 | /*!40101 SET character_set_client = utf8 */; 25 | CREATE TABLE `fb_tx` ( 26 | `hash` varchar(66) NOT NULL, 27 | `index` int DEFAULT NULL, 28 | `bundle` int DEFAULT NULL, 29 | `block_number` bigint unsigned DEFAULT NULL, 30 | `eoa_address` varchar(42) DEFAULT NULL, 31 | `to_address` varchar(42) DEFAULT NULL, 32 | `gas_used` bigint unsigned DEFAULT NULL, 33 | `gas_price` bigint unsigned DEFAULT NULL, 34 | `coinbase_transfer` varchar(42) DEFAULT NULL, 35 | `total_miner_reward` bigint unsigned DEFAULT NULL, 36 | PRIMARY KEY (`hash`) 37 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; 38 | /*!40101 SET character_set_client = @saved_cs_client */; 39 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 40 | 41 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 42 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 43 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 44 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 45 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 46 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 47 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 48 | 49 | -- Dump completed on 2021-07-07 10:00:23 50 | -------------------------------------------------------------------------------- /MySql/reports/delay_distribution.sql: -------------------------------------------------------------------------------- 1 | -- produce a distribution of the number of txs at each block inclusion delay 2 | select eth_tx.block_number - eth_tx.arrival_block as block_delay,count(*) as count 3 | from tx_time.eth_tx 4 | where is_warm = 1 and eth_tx.arrival_block > 0 5 | group by eth_tx.block_number - eth_tx.arrival_block 6 | order by eth_tx.block_number - eth_tx.arrival_block 7 | -------------------------------------------------------------------------------- /MySql/reports/delay_stats.sql: -------------------------------------------------------------------------------- 1 | -- statistical analysis of inclusion delay times by block count and time (avg and stdev) 2 | select min(eth_tx.arrival_time) as from_time, 3 | max(eth_tx.arrival_time) as to_time, 4 | count(*) as tx_count, 5 | avg(eth_tx.block_number - eth_tx.arrival_block) as avg_inclusion_delay_blocks, 6 | stddev(eth_tx.block_number - eth_tx.arrival_block) as stdev_inclusion_delay_blocks, 7 | avg(eth_tx.delay_ms) as avg_inclusion_delay_ms, 8 | stddev(eth_tx.delay_ms) as stdev_inclusion_delay_ms 9 | from tx_time.eth_tx 10 | where is_warm = 1 and eth_tx.arrival_block > 0 and eth_tx.block_number > 0; 11 | 12 | -- statistical analysis of inclusion delay times for mempool only txs by block count and time (avg and stdev) 13 | select min(eth_tx.arrival_time) as from_time, 14 | max(eth_tx.arrival_time) as to_time, 15 | count(*) as tx_count, 16 | avg(eth_tx.block_number - eth_tx.arrival_block) as avg_inclusion_delay_blocks, 17 | stddev(eth_tx.block_number - eth_tx.arrival_block) as stdev_inclusion_delay_blocks, 18 | avg(eth_tx.delay_ms) as avg_inclusion_delay_ms, 19 | stddev(eth_tx.delay_ms) as stdev_inclusion_delay_ms 20 | from tx_time.eth_tx 21 | where is_warm = 1 and is_dark = 0 and delay_ms > 0 and eth_tx.arrival_block > 0 and eth_tx.block_number > 0; 22 | -------------------------------------------------------------------------------- /MySql/reports/mev_report.sql: -------------------------------------------------------------------------------- 1 | -- sandwich, frontrun, backrun reports 2 | -- run update_bundles_nblock first 3 | 4 | SELECT 5 | (SELECT count(*) FROM tx_time.fb_bundle) as total, 6 | (SELECT count(*) FROM tx_time.fb_bundle where not_dark_count > 0 and is_first_dark = 1 and is_last_dark = 1 and min_delay_ms > 0) as sandwich, 7 | (SELECT count(*) FROM tx_time.fb_bundle where not_dark_count > 0 and is_first_dark = 1 and is_last_dark = 0 and min_delay_ms > 0) as frontrun, 8 | (SELECT count(*) FROM tx_time.fb_bundle where not_dark_count > 0 and is_first_dark = 0 and is_last_dark = 1 and min_delay_ms > 0) as backrun, 9 | (SELECT count(*) FROM tx_time.fb_bundle where is_within_n = 1) as total_cl, 10 | (SELECT count(*) FROM tx_time.fb_bundle where is_within_n = 1 and not_dark_count > 0 and is_first_dark = 1 and is_last_dark = 1 and min_delay_ms > 0) as sandwich_cl, 11 | (SELECT count(*) FROM tx_time.fb_bundle where is_within_n = 1 and not_dark_count > 0 and is_first_dark = 1 and is_last_dark = 0 and min_delay_ms > 0) as frontrun_cl, 12 | (SELECT count(*) FROM tx_time.fb_bundle where is_within_n = 1 and not_dark_count > 0 and is_first_dark = 0 and is_last_dark = 1 and min_delay_ms > 0) as backrun_cl -------------------------------------------------------------------------------- /MySql/reports/sandwich_backruns_browser.sql: -------------------------------------------------------------------------------- 1 | -- sandwich 2 | SELECT eth_tx.block_number,eth_tx.arrival_block,eth_tx.delay_ms,fb_bundle.*,fb_tx.* FROM fb_bundle,fb_tx,eth_tx 3 | where not_dark_count > 0 and is_first_dark = 1 and is_last_dark = 1 and min_delay_ms > 0 4 | and fb_bundle.bundle = fb_tx.bundle and fb_bundle.block_number = fb_tx.block_number 5 | and eth_tx.hash = fb_tx.hash 6 | order by fb_bundle.block_number desc, fb_bundle.bundle asc; 7 | 8 | -- backrun 9 | SELECT eth_tx.block_number,eth_tx.arrival_block,eth_tx.delay_ms,fb_bundle.*,fb_tx.* FROM fb_bundle,fb_tx,eth_tx 10 | where not_dark_count > 0 and is_first_dark = 0 and is_last_dark = 1 and min_delay_ms > 0 11 | and fb_bundle.bundle = fb_tx.bundle and fb_bundle.block_number = fb_tx.block_number 12 | and eth_tx.hash = fb_tx.hash 13 | order by fb_bundle.block_number desc, fb_bundle.bundle asc; 14 | -------------------------------------------------------------------------------- /MySql/reports/update_bundles_nblock.sql: -------------------------------------------------------------------------------- 1 | -- run this on offline data before perfoming analysis 2 | 3 | drop table fb_bundle_groupby; 4 | truncate table fb_bundle; 5 | 6 | -- create bundle rows from fb_tx 7 | insert into fb_bundle 8 | select block_number,bundle,count(*) as count,0,0,0,0,0,0,0 9 | from fb_tx 10 | group by block_number,bundle 11 | order by block_number asc,bundle asc; 12 | 13 | -- update first 14 | update eth_tx,fb_tx,fb_bundle set fb_bundle.is_first_dark = eth_tx.is_dark 15 | where fb_tx.index = 0 and 16 | eth_tx.is_warm = 1 and 17 | eth_tx.hash = fb_tx.hash and 18 | fb_bundle.block_number = fb_tx.block_number and 19 | fb_bundle.bundle = fb_tx.bundle; 20 | 21 | -- update last 22 | update eth_tx,fb_tx,fb_bundle set fb_bundle.is_last_dark = eth_tx.is_dark 23 | where fb_tx.index = fb_bundle.count - 1 and 24 | eth_tx.is_warm = 1 and 25 | eth_tx.hash = fb_tx.hash and 26 | fb_bundle.block_number = fb_tx.block_number and 27 | fb_bundle.bundle = fb_tx.bundle; 28 | 29 | -- aggregate bundle data 30 | create table fb_bundle_groupby 31 | select fb_tx.block_number, 32 | fb_tx.bundle, 33 | count(*) as not_dark_count, 34 | min(delay_ms) as min_delay_ms, 35 | max(delay_ms) as max_delay_ms, 36 | sum(delay_ms) as sum_delay_ms 37 | from fb_tx, eth_tx 38 | where is_warm = 1 and is_dark = 0 and delay_ms > 0 and eth_tx.hash = fb_tx.hash 39 | group by fb_tx.block_number,fb_tx.bundle; 40 | 41 | -- update bundle from aggregate 42 | update fb_bundle, fb_bundle_groupby 43 | set fb_bundle.not_dark_count = fb_bundle_groupby.not_dark_count, 44 | fb_bundle.min_delay_ms = fb_bundle_groupby.min_delay_ms, 45 | fb_bundle.max_delay_ms = fb_bundle_groupby.max_delay_ms, 46 | fb_bundle.sum_delay_ms = fb_bundle_groupby.sum_delay_ms 47 | where fb_bundle.block_number = fb_bundle_groupby.block_number and 48 | fb_bundle.bundle = fb_bundle_groupby.bundle; 49 | 50 | -- if any txs in the bundle arrived >n blocks before they are included, set is_within_n false 51 | update fb_bundle set is_within_n = 1; 52 | update fb_bundle, eth_tx, fb_tx set is_within_n = 0 53 | where eth_tx.hash = fb_tx.hash and 54 | fb_bundle.block_number = fb_tx.block_number and 55 | fb_bundle.bundle = fb_tx.bundle and (eth_tx.block_number - eth_tx.arrival_block) > 1; -------------------------------------------------------------------------------- /Program.cs: -------------------------------------------------------------------------------- 1 | /// MIT License 2 | /// Copyright © 2021 pmcgoohan 3 | /// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | /// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 5 | /// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 6 | 7 | using Nethereum.JsonRpc.WebSocketStreamingClient; 8 | using Nethereum.RPC.Eth.DTOs; 9 | using Nethereum.RPC.Reactive.Eth.Subscriptions; 10 | using Nethereum.Web3; 11 | using System; 12 | using System.Reactive.Linq; 13 | using System.Collections.Generic; 14 | using System.Diagnostics; 15 | using System.IO; 16 | using System.Text; 17 | using System.Drawing; 18 | 19 | namespace EthInclude 20 | { 21 | class Program 22 | { 23 | static Nethereum.Web3.Web3 _web3; 24 | static Dictionary _txh = new Dictionary(); 25 | static ulong _arrivalIndex = 0; 26 | static ulong _inclusionIndex = 0; 27 | static DateTime _lastTimestamp = DateTime.MinValue; 28 | static Nethereum.Hex.HexTypes.HexBigInteger _lastFlashbotsBlock; 29 | static Nethereum.Hex.HexTypes.HexBigInteger _lastBlock; 30 | 31 | static void Main(string[] args) 32 | { 33 | // collect outstanding pending txs to estimate tx isWarm flag 34 | Console.WriteLine("how many pending txs currently at https://etherscan.io/txsPending?"); 35 | DB.PendingTxsToImport = ulong.Parse(Console.ReadLine()); 36 | DB.PendingTxsToImport += 10000; // add a safety amount 37 | 38 | // connect to eth node 39 | var client = new StreamingWebSocketClient(DB.GetConfig("WssNodePath")); 40 | _web3 = new Nethereum.Web3.Web3(DB.GetConfig("HttpsNodePath")); 41 | 42 | // subscribe blocks 43 | var blockHeaderSubscription = new EthNewBlockHeadersObservableSubscription(client); 44 | blockHeaderSubscription.GetSubscriptionDataResponsesAsObservable().Subscribe(NewBlock); 45 | 46 | // subscribe pending txs 47 | var pendingTransactionsSubscription = new EthNewPendingTransactionObservableSubscription(client); 48 | pendingTransactionsSubscription.GetSubscriptionDataResponsesAsObservable().Subscribe(NewPendingTx); 49 | 50 | // wait for data 51 | client.StartAsync().Wait(); 52 | blockHeaderSubscription.SubscribeAsync().Wait(); 53 | pendingTransactionsSubscription.SubscribeAsync().Wait(); 54 | 55 | // wait for exit 56 | Console.ReadLine(); 57 | } 58 | 59 | public static void NewPendingTx(string txh) 60 | { 61 | // record new pending txs as they arrive 62 | DateTime timestamp = DateTime.Now; 63 | var tt = new TxTime(); 64 | tt.Hash = txh; 65 | tt.ArrivalTime = timestamp; 66 | tt.ArrivalIndex = _arrivalIndex; 67 | tt.ArrivalBlockNumber = _lastBlock; 68 | tt.IsWarm = _arrivalIndex > DB.PendingTxsToImport; 69 | if (_txh.TryAdd(txh, tt)) 70 | _arrivalIndex++; 71 | } 72 | 73 | public static void NewBlock(Block block) 74 | { 75 | _lastBlock = block.Number; 76 | DateTime timestamp = DateTime.Now; 77 | 78 | // get block txs 79 | var blockTxsReq = _web3.Eth.Blocks.GetBlockWithTransactionsByNumber.SendRequestAsync(block.Number); 80 | blockTxsReq.Wait(); 81 | 82 | // iterate txs 83 | var btxs = blockTxsReq.Result; 84 | if (btxs == null || btxs.Transactions == null) return; 85 | List tts = new List(btxs.Transactions.Length); 86 | 87 | foreach (var t in btxs.Transactions) 88 | { 89 | TxTime tt; 90 | if (!_txh.TryGetValue(t.TransactionHash, out tt)) 91 | { 92 | // if we haven't seen this tx before, it has likely been sent direct to the miner (Flashbots, MistX, Mining pool, etc) 93 | // until all pending txs have been downloaded, the first of these will be set incorrectly (isWarm set false) 94 | tt = new TxTime(); 95 | tt.IsDark = true; 96 | tt.Hash = t.TransactionHash; 97 | tt.ArrivalTime = timestamp; 98 | tt.ArrivalIndex = _arrivalIndex++; 99 | tt.ArrivalBlockNumber = block.Number; 100 | tt.IsWarm = _arrivalIndex > DB.PendingTxsToImport; 101 | } 102 | else 103 | { 104 | // if we have already assigned this tx to a block, don't overwrite 105 | if (tt.BlockNumber != null) 106 | continue; 107 | 108 | tt.IsDark = false; 109 | } 110 | 111 | // updating block inclusion details 112 | tt.BlockNumber = block.Number; 113 | tt.TxIndex = t.TransactionIndex; 114 | tt.Gas = t.Gas; 115 | tt.GasPrice = t.GasPrice; 116 | tt.InclusionTime = timestamp; 117 | tt.InclusionIndex = _inclusionIndex++; 118 | if (tt.IsDark) 119 | tt.Delay = new TimeSpan(0); 120 | else 121 | tt.Delay = tt.InclusionTime - tt.ArrivalTime; 122 | 123 | tts.Add(tt); 124 | } 125 | 126 | // non blocking write to the db 127 | _ = DB.WriteTxTimeAsync(tts); 128 | 129 | // non blocking collection and write of flashbots data (if this is a new block) 130 | if (block.Number != _lastFlashbotsBlock) 131 | { 132 | _ = Flashbots.Collect(4000); // delay by a few seconds to give them a chance to update the api with the new block 133 | _lastFlashbotsBlock = block.Number; 134 | } 135 | 136 | Console.WriteLine("block " + block.Number); 137 | } 138 | 139 | static void VisualizeBitmap() 140 | { 141 | // get pixel at arrival index 142 | // write to inclusion index 143 | 144 | // load timed tx data and sort by arrival index 145 | List tts = TxTime.Load(DB.GetConfig("ReadFile")); // note: first version using csv files rather than MySql - not yet updated for db 146 | tts.Sort(); 147 | 148 | Bitmap bmpIn = new Bitmap(DB.GetConfig("ReadBmp"), true); 149 | Bitmap bmpOut = new Bitmap(DB.GetConfig("ReadBmp"), true); 150 | 151 | int x, y; 152 | for (x = 0; x < bmpOut.Width; x++) 153 | for (y = 0; y < bmpOut.Height; y++) 154 | bmpOut.SetPixel(x, y, Color.Transparent); 155 | 156 | for (int i = 0; i < tts.Count; i++) 157 | { 158 | // allow wrap around if we run out of image data 159 | int k = 0; 160 | do 161 | { 162 | int ai = (int)tts[i].ArrivalIndex + k; 163 | x = ai % bmpIn.Width; 164 | y = ai / bmpIn.Width; 165 | k -= bmpIn.Width * bmpIn.Height; 166 | } while (y >= bmpIn.Height); 167 | 168 | if (y < bmpIn.Height) 169 | { 170 | // get the pixel at the arrival index (having wrapped around if required) 171 | Color pixelColor = bmpIn.GetPixel(x, y); 172 | 173 | int ii = (int)tts[i].InclusionIndex; 174 | x = ii % bmpOut.Width; 175 | y = ii / bmpOut.Width; 176 | 177 | // write it where it was included 178 | if (y < bmpOut.Height) 179 | bmpOut.SetPixel((int)x, (int)y, pixelColor); 180 | } 181 | } 182 | 183 | bmpOut.Save(DB.GetConfig("WriteBmp")); 184 | } 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Eth Include 2 | 3 | Extraction of Ethereum transaction arrival and inclusion times allowing for limited MEV type classification. 4 | 5 | ## Description 6 | 7 | This project was created to provide supporting data for my [Plain Alex](https://github.com/pmcgoohan/targeting-zero-mev/blob/main/README.md) content layer solution and [my presentation](https://www.youtube.com/watch?v=zf2l3veT9EI&t=114s) at the EthGlobal / HackMoney / MEV.wtf summit. 8 | 9 | It allows for the collection of tx metadata such as arrival time in the mempool and subsequent inclusion time in the Ethereum blockchain. 10 | 11 | In doing so, it allows for the broad categorization of [Flashbots MEVA bundles](https://flashbots-explorer.marto.lol/) into the MEV types: frontrunning, backrunning, sandwiching and other. 12 | 13 | ## MEV Categorization Methodology 14 | 15 | MEV type categorization can be performed by examining the relative position of dark txs (those added at block creation by the miner) and lit txs (those that arrive through the mempool) in individual MEVA bundles. 16 | 17 | |MEV type|Identifying features| 18 | |---|---| 19 | |sandwich attack|dark first and last txs, lit middle txs| 20 | |frontrun attack|dark first tx, lit final txs| 21 | |backrun attack|lit first txs, dark final tx| 22 | |other|none of the above, including single tx bundles| 23 | 24 | ### Limitations 25 | 26 | Because categorization relies on arrival/inclusion time metadata and not the deep parsing of tx contents, some bundles may be misclassified. 27 | 28 | Informal spot checks suggest it does a pretty good job, but the technique would benefit from greater auditing. 29 | 30 | ## Getting Started 31 | 32 | * Build the MySql tx_time database using the MySql/create_db_tx_time scripts. 33 | * Set appsettings.json MySqlConnection to point to this db 34 | * Set appsettings.json WssNodePath to point to an Ethereum Node web socket (eg: Infura) 35 | * Compile and run the EthInclude.csproj Console app 36 | * Enter the number of pending txs in the system (typically around 150000. For accurate arrival times, ignore these early txs when reporting) 37 | * Collect data for as long as you want (note: disconnections are not gracefully handled at the moment) 38 | * Run MySql/reports/update_bundles_nblock.sql against the db to update reporting fields 39 | * Run sql reports against it (see MySql/reports for examples) 40 | 41 | ### Dependencies 42 | 43 | * .Net Core and IDE (Visual Studio 2019 / Visual Studio Code) 44 | * MySql 45 | * Ethereum node (eg: [Infura](https://infura.io/)) 46 | * [Flashbots API](https://blocks.flashbots.net/) 47 | 48 | ## Sample Data 49 | 50 | This is the [sample data set](https://drive.google.com/file/d/1WPknOb-Y3jIGaNc-2wA3VuWkUjXxuS8O) used in MEV type categorization in [my talk](https://www.youtube.com/watch?v=zf2l3veT9EI&t=114s). 51 | 52 | ## Authors 53 | 54 | P mcgoohan 55 | 56 | Find me on 57 | [Twitter](https://twitter.com/pmcgoohanCrypto), 58 | [Medium](https://pmcgoohan.medium.com), 59 | [Github](https://github.com/pmcgoohan), 60 | [Eth.Research](https://ethresear.ch/u/pmcgoohan), 61 | [Reddit (predicting MEV in 2014)](https://www.reddit.com/r/ethereum/comments/2d84yv/miners_frontrunning) 62 | 63 | ## License 64 | 65 | This project is licensed under the MIT License - see the LICENSE.md file for details 66 | 67 | ## Acknowledgments 68 | 69 | * [Infura](https://infura.io/) 70 | * [Flashbots API](https://blocks.flashbots.net/) 71 | -------------------------------------------------------------------------------- /TxTime.cs: -------------------------------------------------------------------------------- 1 | using Nethereum.Contracts; 2 | using Nethereum.JsonRpc.WebSocketStreamingClient; 3 | using Nethereum.RPC.Eth.DTOs; 4 | using Nethereum.ABI.FunctionEncoding.Attributes; 5 | using Nethereum.JsonRpc.Client.Streaming; 6 | using Nethereum.RPC.Eth.Subscriptions; 7 | using Nethereum.RPC.Reactive.Eth; 8 | using Nethereum.RPC.Reactive.Eth.Subscriptions; 9 | using Nethereum.RPC.Reactive.Extensions; 10 | using Nethereum.JsonRpc.WebSocketClient; 11 | using Nethereum.Hex.HexTypes; 12 | using Nethereum.Web3; 13 | using Nethereum.Web3.Accounts; 14 | using System; 15 | using System.Reactive.Linq; 16 | using System.Collections.Generic; 17 | using System.Diagnostics; 18 | using System.IO; 19 | using System.Text; 20 | using System.Drawing; 21 | using System.Diagnostics.CodeAnalysis; 22 | 23 | namespace EthInclude 24 | { 25 | public class TxTime : IComparable 26 | { 27 | public string Hash; 28 | public HexBigInteger BlockNumber; 29 | public HexBigInteger TxIndex; 30 | public HexBigInteger Gas; 31 | public HexBigInteger GasPrice; 32 | public DateTime ArrivalTime; 33 | public DateTime InclusionTime; 34 | public bool IsWarm; // set if the initial glut of pending txs was processed by the time this was created 35 | public bool IsDark; // set if the tx was not seen in the mempool, and was therefore likely added by the miner at block creation time 36 | public TimeSpan Delay; // between arrival and inclusion 37 | public ulong ArrivalIndex; 38 | public ulong InclusionIndex; 39 | public HexBigInteger ArrivalBlockNumber; 40 | 41 | public static List Load(string file) 42 | { 43 | List r = new List(); 44 | 45 | using (var sr = new StreamReader(file)) 46 | { 47 | // skip header 48 | sr.ReadLine(); 49 | 50 | while (!sr.EndOfStream) 51 | { 52 | string[] f = sr.ReadLine().Split(","); 53 | if (f.Length != 8) continue; 54 | 55 | // only load the fields we are interested in 56 | TxTime tt = new TxTime(); 57 | tt.ArrivalIndex = ulong.Parse(f[6]); 58 | tt.InclusionIndex = ulong.Parse(f[7]); 59 | r.Add(tt); 60 | } 61 | } 62 | 63 | return r; 64 | } 65 | 66 | public int CompareTo(TxTime other) 67 | { 68 | return this.ArrivalIndex.CompareTo(other.ArrivalIndex); 69 | } 70 | } 71 | } -------------------------------------------------------------------------------- /appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "MySqlConnection": "server=localhost;user=[enter];database=tx_time;port=3306;password=[enter]", 3 | "WssNodePath": "wss://mainnet.infura.io/ws/v3/[enter]", 4 | "HttpsNodePath": "https://mainnet.infura.io/v3/[enter]", 5 | "WriteFile": "C:\\txtime.csv", 6 | "ReadFile": "E:\\txtimeOrange.csv", 7 | "WriteBmp": "E:\\target.bmp", 8 | "ReadBmp": "E:\\source.bmp" 9 | } 10 | --------------------------------------------------------------------------------