├── .gitattributes ├── .gitignore ├── FiddlerImportNetlog ├── FiddlerImportNetlog.csproj ├── FiddlerImportNetlog.sln ├── FiddlerInterface.cs ├── Importer.cs ├── Interesting Snippets.txt ├── JSON.cs └── Properties │ └── AssemblyInfo.cs ├── LICENSE ├── README.md └── installer ├── Addon.ver ├── FiddlerImportNetLog.nsi ├── addon.ico └── go.bat /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ################# 2 | ## Visual Studio 3 | ################# 4 | 5 | *.vs/ 6 | 7 | *.exe 8 | 9 | # User-specific files 10 | *.suo 11 | *.user 12 | *.sln.docstates 13 | *.[Cc]ache 14 | 15 | # Build results 16 | 17 | [Dd]ebug/ 18 | [Dd]ebugPublic/ 19 | [Rr]elease/ 20 | [Rr]eleases/ 21 | x64/ 22 | x86/ 23 | build/ 24 | bld/ 25 | [Bb]in/ 26 | [Oo]bj/ 27 | TestResults/ 28 | *.vshost.exe 29 | 30 | # MSTest test Results 31 | [Tt]est[Rr]esult*/ 32 | [Bb]uild[Ll]og.* 33 | 34 | *_i.c 35 | *_p.c 36 | *.ilk 37 | *.meta 38 | *.obj 39 | *.pch 40 | *.pdb 41 | *.pgc 42 | *.pgd 43 | *.rsp 44 | *.sbr 45 | *.tlb 46 | *.tli 47 | *.tlh 48 | *.tmp 49 | *.tmp_proj 50 | *.log 51 | *.vspscc 52 | *.vssscc 53 | .builds 54 | *.pidb 55 | *.log 56 | *.scc 57 | 58 | # Visual C++ cache files 59 | ipch/ 60 | *.aps 61 | *.ncb 62 | *.opensdf 63 | *.sdf 64 | *.cachefile 65 | 66 | # Visual Studio profiler 67 | *.psess 68 | *.vsp 69 | *.vspx 70 | 71 | # Guidance Automation Toolkit 72 | *.gpState 73 | 74 | # ReSharper is a .NET coding add-in 75 | _ReSharper*/ 76 | *.[Rr]e[Ss]harper 77 | 78 | # TeamCity is a build add-in 79 | _TeamCity* 80 | 81 | # DotCover is a Code Coverage Tool 82 | *.dotCover 83 | 84 | # NCrunch 85 | *.ncrunch* 86 | .*crunch*.local.xml 87 | 88 | 89 | ############# 90 | ## Windows detritus 91 | ############# 92 | 93 | # Windows image file caches 94 | Thumbs.db 95 | ehthumbs.db 96 | 97 | # Folder config file 98 | Desktop.ini 99 | 100 | # Recycle Bin used on file shares 101 | $RECYCLE.BIN/ 102 | 103 | # Mac crap 104 | .DS_Store 105 | 106 | # NuGet Packages 107 | *.nupkg 108 | # The packages folder can be ignored because of Package Restore 109 | **/packages/* 110 | # except build/, which is used as an MSBuild target. 111 | !**/packages/build/ 112 | # If using the old MSBuild-Integrated Package Restore, uncomment this: 113 | #!**/packages/repositories.config -------------------------------------------------------------------------------- /FiddlerImportNetlog/FiddlerImportNetlog.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {7B641C3F-89B9-44BD-BC81-42A3A9E11161} 8 | Library 9 | Properties 10 | FiddlerImportNetlog 11 | FiddlerImportNetlog 12 | v4.7.2 13 | 512 14 | true 15 | 16 | 17 | 18 | true 19 | full 20 | false 21 | bin\Debug\ 22 | DEBUG;TRACE 23 | prompt 24 | 4 25 | 26 | 27 | pdbonly 28 | true 29 | bin\Release\ 30 | TRACE 31 | prompt 32 | 4 33 | 34 | 35 | 36 | ..\..\..\Users\ericlaw\AppData\Local\Programs\Fiddler\Fiddler.exe 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | copy $(TargetPath) "%25userprofile%25\OneDrive - Microsoft\Documents\Fiddler2\ImportExport\$(TargetFilename)" 55 | 56 | -------------------------------------------------------------------------------- /FiddlerImportNetlog/FiddlerImportNetlog.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.28307.421 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FiddlerImportNetlog", "FiddlerImportNetlog.csproj", "{7B641C3F-89B9-44BD-BC81-42A3A9E11161}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {7B641C3F-89B9-44BD-BC81-42A3A9E11161}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {7B641C3F-89B9-44BD-BC81-42A3A9E11161}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {7B641C3F-89B9-44BD-BC81-42A3A9E11161}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {7B641C3F-89B9-44BD-BC81-42A3A9E11161}.Release|Any CPU.Build.0 = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | GlobalSection(ExtensibilityGlobals) = postSolution 23 | SolutionGuid = {6BA34491-CF1B-4BB2-80F7-789CCB5E2901} 24 | EndGlobalSection 25 | EndGlobal 26 | -------------------------------------------------------------------------------- /FiddlerImportNetlog/FiddlerInterface.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.IO; 5 | using System.Text; 6 | using Fiddler; 7 | using System.IO.Compression; 8 | 9 | namespace FiddlerImportNetlog 10 | { 11 | [ProfferFormat("NetLog JSON", 12 | "Chromium's JSON-based event log format (v1.3.7). See https://textslashplain.com/2020/01/17/capture-network-logs-from-edge-and-chrome/ for more info.", 13 | // We handle import of JSON files, whether uncompressed, or compressed with ZIP or GZ. I'm not completely sure I remember the implications 14 | // of declaring .gz here, nor why .zip isn't mentioned. Is this about the drag/drop import feature? 15 | ".json;.gz" 16 | )] 17 | public class NetLogFormatImport : ISessionImporter 18 | { 19 | public Session[] ImportSessions(string sFormat, Dictionary dictOptions, EventHandler evtProgressNotifications) 20 | { 21 | if ((sFormat != "NetLog JSON")) { Debug.Assert(false); return null; } 22 | 23 | MemoryStream strmContent = null; 24 | string sFilename = null; 25 | if (null != dictOptions) 26 | { 27 | if (dictOptions.ContainsKey("Filename")) 28 | { 29 | sFilename = dictOptions["Filename"] as string; 30 | } 31 | else if (dictOptions.ContainsKey("Content")) 32 | { 33 | strmContent = new MemoryStream(Encoding.UTF8.GetBytes(dictOptions["Content"] as string)); 34 | } 35 | } 36 | 37 | if ((null == strmContent) && string.IsNullOrEmpty(sFilename)) 38 | { 39 | sFilename = Fiddler.Utilities.ObtainOpenFilename("Import " + sFormat, "NetLog JSON (*.json[.gz], *.zip)|*.json;*.json.gz;*.zip"); 40 | } 41 | 42 | if ((null != strmContent) || !String.IsNullOrEmpty(sFilename)) 43 | { 44 | try 45 | { 46 | List listSessions = new List(); 47 | StreamReader oSR; 48 | 49 | if (null != strmContent) 50 | { 51 | oSR = new StreamReader(strmContent); 52 | } 53 | else 54 | { 55 | Stream oFS = File.OpenRead(sFilename); 56 | FiddlerApplication.Log.LogFormat("!NetLog Importer is loading {0}", sFilename); 57 | // Check to see if this file data was GZIP'd or PKZIP'd. 58 | bool bWasGZIP = false; 59 | bool bWasPKZIP = false; 60 | int bFirst = oFS.ReadByte(); 61 | if (bFirst == 0x1f && oFS.ReadByte() == 0x8b) 62 | { 63 | bWasGZIP = true; 64 | evtProgressNotifications?.Invoke(null, new ProgressCallbackEventArgs(0, "Import file was compressed using gzip/DEFLATE.")); 65 | } 66 | else if (bFirst == 0x50 && oFS.ReadByte() == 0x4b) { 67 | bWasPKZIP = true; 68 | evtProgressNotifications?.Invoke(null, new ProgressCallbackEventArgs(0, "Import file was a ZIP archive.")); 69 | } 70 | 71 | oFS.Position = 0; 72 | if (bWasGZIP) 73 | { 74 | oFS = GetUnzippedBytes(oFS); 75 | } 76 | else if (bWasPKZIP) 77 | { 78 | // Open the first JSON file. 79 | ZipArchive oZA = new ZipArchive(oFS, ZipArchiveMode.Read, false, Encoding.UTF8); 80 | foreach (ZipArchiveEntry oZE in oZA.Entries) 81 | { 82 | if (oZE.FullName.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) 83 | { 84 | oFS = oZE.Open(); 85 | break; 86 | } 87 | } 88 | } 89 | 90 | oSR = new StreamReader(oFS, Encoding.UTF8); 91 | } 92 | 93 | using (oSR) 94 | { 95 | new NetlogImporter(oSR, listSessions, evtProgressNotifications); 96 | } 97 | return listSessions.ToArray(); 98 | } 99 | catch (Exception eX) 100 | { 101 | FiddlerApplication.ReportException(eX, "Failed to import NetLog"); 102 | return null; 103 | } 104 | } 105 | return null; 106 | } 107 | 108 | /// 109 | /// Read the all bytes of the supplied DEFLATE-compressed file and return a memorystream containing the expanded bytes. 110 | /// 111 | private MemoryStream GetUnzippedBytes(Stream oFS) 112 | { 113 | long fileLength = oFS.Length; 114 | if (fileLength > Int32.MaxValue) 115 | throw new IOException("file over 2gb"); 116 | 117 | int index = 0; 118 | int count = (int)fileLength; 119 | byte[] bytes = new byte[count]; 120 | 121 | while (count > 0) 122 | { 123 | int n = oFS.Read(bytes, index, count); 124 | index += n; 125 | count -= n; 126 | } 127 | 128 | return new MemoryStream(Utilities.GzipExpand(bytes)); 129 | } 130 | 131 | public void Dispose() { } 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /FiddlerImportNetlog/Importer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Generic; 4 | using System.Diagnostics; 5 | using System.IO; 6 | using System.Linq; 7 | using System.Security.Cryptography; 8 | using System.Security.Cryptography.X509Certificates; 9 | using System.Text; 10 | using Fiddler; 11 | using FiddlerImportNetlog.WebFormats; 12 | 13 | namespace FiddlerImportNetlog 14 | { 15 | class NetlogImporter 16 | { 17 | /// 18 | /// The NetLog file itself contains the mapping between string constants and the magic numbers used in the event entries. 19 | /// 20 | struct Magics 21 | { 22 | // Sources 23 | public int SRC_NONE; 24 | public int SRC_URL_REQUEST; 25 | public int SRC_SOCKET; 26 | public int SRC_HOST_RESOLVER_IMPL_JOB; 27 | 28 | // Events 29 | public int REQUEST_ALIVE; 30 | public int URL_REQUEST_START_JOB; 31 | public int SEND_HEADERS; 32 | public int SEND_QUIC_HEADERS; 33 | public int SEND_HTTP2_HEADERS; 34 | public int READ_HEADERS; 35 | public int FAKE_RESPONSE_HEADERS_CREATED; 36 | public int READ_EARLY_HINTS_RESPONSE_HEADERS; 37 | public int COOKIE_INCLUSION_STATUS; 38 | public int FILTERED_BYTES_READ; 39 | public int SEND_BODY; 40 | public int SEND_REQUEST; 41 | public int SSL_CERTIFICATES_RECEIVED; 42 | public int SSL_HANDSHAKE_MESSAGE_SENT; 43 | public int SSL_HANDSHAKE_MESSAGE_RECEIVED; 44 | public int TCP_CONNECT; 45 | public int SOCKET_BYTES_SENT; 46 | public int HOST_RESOLVER_IMPL_REQUEST; 47 | public int HOST_RESOLVER_IMPL_JOB; 48 | public int HOST_RESOLVER_IMPL_PROC_TASK; 49 | } 50 | 51 | internal static string DescribeExceptionWithStack(Exception eX) 52 | { 53 | StringBuilder oSB = new StringBuilder(512); 54 | oSB.AppendLine(eX.Message); 55 | oSB.AppendLine(eX.StackTrace); 56 | if (null != eX.InnerException) 57 | { 58 | oSB.AppendFormat(" < {0}", eX.InnerException.Message); 59 | } 60 | return oSB.ToString(); 61 | } 62 | 63 | /// 64 | /// Remove disabled extensions, and any name/value pairs where the value is empty. 65 | /// 66 | private static ArrayList FilterExtensions(ArrayList al) 67 | { 68 | ArrayList alOut = new ArrayList(); 69 | if (null != al) 70 | { 71 | foreach (Hashtable htItem in al) 72 | { 73 | if ((bool)htItem["enabled"]) 74 | { 75 | List keysToDrop = new List { "enabled", "kioskOnly", "kioskEnabled", "offlineEnabled" }; 76 | foreach (DictionaryEntry kvp in htItem) 77 | { 78 | if ((kvp.Value is string) && String.IsNullOrWhiteSpace(kvp.Value as string)) keysToDrop.Add(kvp.Key as string); 79 | } 80 | foreach (string key in keysToDrop) htItem.Remove(key); 81 | alOut.Add(htItem); 82 | } 83 | } 84 | } 85 | return alOut; 86 | } 87 | 88 | private static DateTime GetTimeStamp(object o, long baseTime) 89 | { 90 | // TODO: Something reasonable if o is null? 91 | long t = baseTime; 92 | if (null != o) 93 | { 94 | if (o is string) 95 | { 96 | t += Int64.Parse(o as string); 97 | } 98 | else 99 | { 100 | t += (long)(double)o; 101 | } 102 | } 103 | return DateTimeOffset.FromUnixTimeMilliseconds(t).DateTime.ToLocalTime(); 104 | } 105 | 106 | #region Fields 107 | List _listSessions; 108 | readonly EventHandler _evtProgressNotifications; 109 | Magics NetLogMagics; 110 | 111 | string _sClient; 112 | long _baseTime; 113 | DateTimeOffset _dtBaseTime; 114 | Dictionary dictEventTypes; 115 | Dictionary dictNetErrors; 116 | #endregion Fields 117 | 118 | internal NetlogImporter(StreamReader oSR, List listSessions, EventHandler evtProgressNotifications) 119 | { 120 | _listSessions = listSessions; 121 | _evtProgressNotifications = evtProgressNotifications; 122 | Stopwatch oSW = Stopwatch.StartNew(); 123 | string sJSONData = oSR.ReadToEnd(); 124 | Hashtable htFile = JSON.JsonDecode(sJSONData, out _) as Hashtable; 125 | 126 | // If JSON-parsing failed, it's possible that the file was truncated either during capture or transfer. 127 | // Try repairing the end of file by replacing the last (incomplete) line. 128 | // This strategy is borrowed from the online "Catapult" netlog viewer app. 129 | if (null == htFile) 130 | { 131 | int iEnd = Math.Max(sJSONData.LastIndexOf(",\n"), sJSONData.LastIndexOf(",\r")); 132 | if (iEnd > 0) { 133 | sJSONData = sJSONData.Substring(0, iEnd) + "]}"; 134 | htFile = JSON.JsonDecode(sJSONData, out _) as Hashtable; 135 | } 136 | if (null == htFile) { 137 | NotifyProgress(1.00f, "Aborting; file is not properly-formatted NetLog JSON."); 138 | FiddlerApplication.DoNotifyUser("This file is not properly-formatted NetLog JSON.", "Import aborted"); 139 | return; 140 | } 141 | else { FiddlerApplication.DoNotifyUser("This file was truncated and may be missing data.\nParsing any readable data.", "Warning"); } 142 | } 143 | 144 | NotifyProgress(0.25f, "Finished parsing JSON file; took " + oSW.ElapsedMilliseconds + "ms."); 145 | if (!ExtractSessionsFromJSON(htFile)) 146 | { 147 | if (!(htFile["traceEvents"] is ArrayList alTraceEvents)) 148 | { 149 | FiddlerApplication.DoNotifyUser("This JSON file does not seem to contain NetLog data.", "Unexpected Data"); 150 | Session sessFile = Session.BuildFromData(false, 151 | new HTTPRequestHeaders( 152 | String.Format("/file.json"), 153 | new[] { "Host: IMPORTED", "Date: " + DateTime.UtcNow.ToString() }), 154 | Utilities.emptyByteArray, 155 | new HTTPResponseHeaders(200, "File Data", new[] { "Content-Type: application/json; charset=utf-8" }), 156 | Encoding.UTF8.GetBytes(JSON.JsonEncode(htFile)), 157 | SessionFlags.ImportedFromOtherTool | SessionFlags.RequestGeneratedByFiddler | SessionFlags.ResponseGeneratedByFiddler | SessionFlags.ServedFromCache); 158 | listSessions.Insert(0, sessFile); 159 | } 160 | else 161 | { 162 | ExtractSessionsFromTraceJSON(alTraceEvents); 163 | } 164 | } 165 | } 166 | 167 | private void ExtractSessionsFromTraceJSON(ArrayList alTraceEvents) 168 | { 169 | // Sources 170 | NetLogMagics.SRC_NONE = 0; 171 | NetLogMagics.SRC_URL_REQUEST = 1; 172 | NetLogMagics.SRC_SOCKET = 2; 173 | NetLogMagics.SRC_HOST_RESOLVER_IMPL_JOB = 3; 174 | 175 | // Events 176 | NetLogMagics.REQUEST_ALIVE = 10; 177 | NetLogMagics.URL_REQUEST_START_JOB = 11; 178 | NetLogMagics.SEND_HEADERS = 12; 179 | NetLogMagics.SEND_QUIC_HEADERS = 13; 180 | NetLogMagics.SEND_HTTP2_HEADERS = 14; 181 | NetLogMagics.READ_HEADERS = 15; 182 | NetLogMagics.FAKE_RESPONSE_HEADERS_CREATED = 16; 183 | NetLogMagics.READ_EARLY_HINTS_RESPONSE_HEADERS = 17; 184 | NetLogMagics.COOKIE_INCLUSION_STATUS = 18; 185 | NetLogMagics.FILTERED_BYTES_READ = 19; 186 | NetLogMagics.SEND_BODY = 20; 187 | NetLogMagics.SEND_REQUEST = 21; 188 | NetLogMagics.SSL_CERTIFICATES_RECEIVED = 22; 189 | NetLogMagics.SSL_HANDSHAKE_MESSAGE_SENT = 23; 190 | NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED = 24; 191 | NetLogMagics.TCP_CONNECT = 25; 192 | NetLogMagics.SOCKET_BYTES_SENT = 26; 193 | 194 | NetLogMagics.HOST_RESOLVER_IMPL_REQUEST = 30; 195 | NetLogMagics.HOST_RESOLVER_IMPL_JOB = 31; 196 | NetLogMagics.HOST_RESOLVER_IMPL_PROC_TASK = 32; 197 | 198 | List listEvents = new List(); 199 | foreach (Hashtable htItem in alTraceEvents) 200 | { 201 | if ((htItem["scope"] as string) =="netlog") 202 | { 203 | listEvents.Add(htItem); 204 | } 205 | } 206 | int iEvent = 0; 207 | int iLastPct = 25; 208 | var dictURLRequests = new Dictionary>(); 209 | int cEvents = listEvents.Count; 210 | foreach (Hashtable htEvent in listEvents) 211 | { 212 | ++iEvent; 213 | var htArgs = htEvent["args"] as Hashtable; 214 | if (null == htArgs) continue; 215 | //var htParams = htArgs["params"] as Hashtable; 216 | //if (null == htParams) continue; 217 | 218 | 219 | #region ParseCertificateRequestMessagesAndDumpToLog 220 | /* 221 | if (iSourceType == NetLogMagics.SOCKET) 222 | { 223 | try 224 | { 225 | // All events we care about should have parameters. 226 | if (!(htEvent["params"] is Hashtable htParams)) continue; 227 | int iType = getIntValue(htEvent["type"], -1); 228 | 229 | List events; 230 | int iSocketID = getIntValue(htSource["id"], -1); 231 | 232 | if (iType != NetLogMagics.SSL_CERTIFICATES_RECEIVED && 233 | iType != NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED) continue; 234 | 235 | // Get (or create) the List of entries for this SOCKET. 236 | if (!dictSockets.ContainsKey(iSocketID)) 237 | { 238 | events = new List(); 239 | dictSockets.Add(iSocketID, events); 240 | } 241 | else 242 | { 243 | events = dictSockets[iSocketID]; 244 | } 245 | // Add this event to the SOCKET's list. 246 | events.Add(htEvent); 247 | } 248 | catch { } 249 | 250 | continue; 251 | } 252 | */ 253 | #endregion ParseCertificateRequestMessagesAndDumpToLog 254 | 255 | var sSourceType = htArgs["source_type"] as string; 256 | if (null == sSourceType) continue; 257 | 258 | // Collect only events related to URL_REQUESTS. 259 | if (sSourceType != "URL_REQUEST") continue; 260 | 261 | var sName = htEvent["name"] as string; 262 | if (null == sName) continue; 263 | 264 | int iURLRequestID = getHexValue(htEvent["id"], -1); 265 | { 266 | List events; 267 | 268 | // Get (or create) the List of entries for this URLRequest. 269 | if (!dictURLRequests.ContainsKey(iURLRequestID)) 270 | { 271 | events = new List(); 272 | dictURLRequests.Add(iURLRequestID, events); 273 | } 274 | else 275 | { 276 | events = dictURLRequests[iURLRequestID]; 277 | } 278 | 279 | // Add this event to the URLRequest's list. 280 | events.Add(htEvent); 281 | } 282 | int iPct = (int)(100 * (0.25f + 0.50f * (iEvent / (float)cEvents))); 283 | if (iPct != iLastPct) 284 | { 285 | NotifyProgress(iPct / 100f, "Parsed an event for a URLRequest"); 286 | iLastPct = iPct; 287 | } 288 | } 289 | 290 | int cURLRequests = dictURLRequests.Count; 291 | 292 | NotifyProgress(0.75f, "Finished reading event entries, saw " + cURLRequests.ToString() + " URLRequests"); 293 | 294 | GenerateSessionsFromURLRequests(dictURLRequests); 295 | 296 | //GenerateDebugTreeSession(dictURLRequests); 297 | //GenerateSocketListSession(dictSockets); 298 | 299 | NotifyProgress(1, "Import Completed."); 300 | } 301 | 302 | private void NotifyProgress(float fPct, string sMessage) 303 | { 304 | _evtProgressNotifications?.Invoke(null, new ProgressCallbackEventArgs(fPct, sMessage)); 305 | } 306 | 307 | private int getIntValue(object oValue, int iDefault) 308 | { 309 | if (null == oValue) return iDefault; 310 | if (!(oValue is Double)) return iDefault; 311 | return (int)(double)oValue; 312 | } 313 | private int getHexValue(object oValue, int iDefault) 314 | { 315 | if (null == oValue) return iDefault; 316 | string sHexValue = oValue as String; 317 | if (String.IsNullOrEmpty(sHexValue)) return iDefault; 318 | try 319 | { 320 | int result = Convert.ToInt32(sHexValue, 16); 321 | return result; 322 | } 323 | catch 324 | { 325 | return iDefault; 326 | } 327 | } 328 | 329 | public bool ExtractSessionsFromJSON(Hashtable htFile) 330 | { 331 | if (!(htFile["constants"] is Hashtable htConstants)) return false; 332 | if (!(htConstants["clientInfo"] is Hashtable htClientInfo)) return false; 333 | this._sClient = htClientInfo["name"] as string; 334 | 335 | #region LookupConstants 336 | Hashtable htEventTypes = htConstants["logEventTypes"] as Hashtable; 337 | Hashtable htNetErrors = htConstants["netError"] as Hashtable; 338 | Hashtable htSourceTypes = htConstants["logSourceType"] as Hashtable; 339 | string sDetailLevel = htConstants["logCaptureMode"] as string; 340 | 341 | // TODO: These should probably use a convenient wrapper for GetHashtableInt 342 | 343 | // Sources 344 | NetLogMagics.SRC_NONE = getIntValue(htSourceTypes["NONE"], 0); 345 | NetLogMagics.SRC_URL_REQUEST = getIntValue(htSourceTypes["URL_REQUEST"], -9999); 346 | NetLogMagics.SRC_SOCKET = getIntValue(htSourceTypes["SOCKET"], -9998); 347 | NetLogMagics.SRC_HOST_RESOLVER_IMPL_JOB = getIntValue(htSourceTypes["HOST_RESOLVER_IMPL_JOB"], -9997); 348 | 349 | #region GetEventTypes 350 | // HTTP-level Events 351 | NetLogMagics.REQUEST_ALIVE = getIntValue(htEventTypes["REQUEST_ALIVE"], -999); 352 | NetLogMagics.URL_REQUEST_START_JOB = getIntValue(htEventTypes["URL_REQUEST_START_JOB"], -998); 353 | NetLogMagics.SEND_HEADERS = getIntValue(htEventTypes["HTTP_TRANSACTION_SEND_REQUEST_HEADERS"], -997); 354 | NetLogMagics.SEND_QUIC_HEADERS = getIntValue(htEventTypes["HTTP_TRANSACTION_QUIC_SEND_REQUEST_HEADERS"], -996); 355 | NetLogMagics.SEND_HTTP2_HEADERS = getIntValue(htEventTypes["HTTP_TRANSACTION_HTTP2_SEND_REQUEST_HEADERS"], -995); 356 | NetLogMagics.READ_HEADERS = getIntValue(htEventTypes["HTTP_TRANSACTION_READ_RESPONSE_HEADERS"], -994); 357 | NetLogMagics.READ_EARLY_HINTS_RESPONSE_HEADERS = getIntValue(htEventTypes["HTTP_TRANSACTION_READ_EARLY_HINTS_RESPONSE_HEADERS"], -993); 358 | NetLogMagics.FAKE_RESPONSE_HEADERS_CREATED = getIntValue(htEventTypes["URL_REQUEST_FAKE_RESPONSE_HEADERS_CREATED"], -992); 359 | NetLogMagics.FILTERED_BYTES_READ = getIntValue(htEventTypes["URL_REQUEST_JOB_FILTERED_BYTES_READ"], -991); 360 | NetLogMagics.COOKIE_INCLUSION_STATUS = getIntValue(htEventTypes["COOKIE_INCLUSION_STATUS"], -990); 361 | NetLogMagics.SEND_BODY = getIntValue(htEventTypes["HTTP_TRANSACTION_SEND_REQUEST_BODY"], -989); 362 | NetLogMagics.SEND_REQUEST = getIntValue(htEventTypes["HTTP_TRANSACTION_SEND_REQUEST"], -988); 363 | 364 | // Socket-level Events 365 | NetLogMagics.SSL_CERTIFICATES_RECEIVED = getIntValue(htEventTypes["SSL_CERTIFICATES_RECEIVED"], -899); 366 | NetLogMagics.SSL_HANDSHAKE_MESSAGE_SENT = getIntValue(htEventTypes["SSL_HANDSHAKE_MESSAGE_SENT"], -898); 367 | NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED = getIntValue(htEventTypes["SSL_HANDSHAKE_MESSAGE_RECEIVED"], -897); 368 | NetLogMagics.TCP_CONNECT = getIntValue(htEventTypes["TCP_CONNECT"], -896); 369 | NetLogMagics.SOCKET_BYTES_SENT = getIntValue(htEventTypes["SOCKET_BYTES_SENT"], -895); 370 | 371 | // DNS 372 | NetLogMagics.HOST_RESOLVER_IMPL_REQUEST = getIntValue(htEventTypes["HOST_RESOLVER_IMPL_REQUEST"], -799); 373 | NetLogMagics.HOST_RESOLVER_IMPL_JOB = getIntValue(htEventTypes["HOST_RESOLVER_IMPL_JOB"], -798); 374 | NetLogMagics.HOST_RESOLVER_IMPL_PROC_TASK = getIntValue(htEventTypes["HOST_RESOLVER_IMPL_PROC_TASK"], -797); 375 | 376 | // Get ALL event type names as strings for pretty print view 377 | dictEventTypes = new Dictionary(); 378 | foreach (DictionaryEntry de in htEventTypes) 379 | { 380 | dictEventTypes.Add((int)(double)de.Value, de.Key as String); 381 | } 382 | #endregion 383 | 384 | #region GetNetErrors 385 | dictNetErrors = new Dictionary(); 386 | foreach (DictionaryEntry de in htNetErrors) 387 | { 388 | dictNetErrors.Add((int)(double)de.Value, de.Key as String); 389 | } 390 | #endregion 391 | 392 | int iLogVersion = getIntValue(htConstants["logFormatVersion"], 0); 393 | NotifyProgress(0, "Found NetLog v" + iLogVersion + "."); 394 | #endregion LookupConstants 395 | 396 | #region GetBaseTime 397 | // Base time for all events' relative timestamps. 398 | object o = htConstants["timeTickOffset"]; 399 | if (o is string) 400 | { 401 | _baseTime = Int64.Parse(o as string); 402 | } 403 | else 404 | { 405 | _baseTime = (long)(double)o; 406 | } 407 | _dtBaseTime = TimeZoneInfo.ConvertTime(DateTimeOffset.FromUnixTimeMilliseconds(_baseTime), TimeZoneInfo.Local); 408 | FiddlerApplication.Log.LogFormat("Base capture time is {0} aka {1}", _baseTime, _dtBaseTime); 409 | #endregion 410 | 411 | // Create a Summary Session, the response body of which we'll fill in later. 412 | Session sessSummary = Session.BuildFromData(false, 413 | new HTTPRequestHeaders( 414 | String.Format("/CAPTURE_INFO"), 415 | new[] { "Host: NETLOG" /* TODO: Put something useful here */, "Date: " + _dtBaseTime.ToString("r") }), 416 | Utilities.emptyByteArray, 417 | new HTTPResponseHeaders(200, "Analyzed Data", new[] { "Content-Type: text/plain; charset=utf-8" }), 418 | Utilities.emptyByteArray, 419 | SessionFlags.ImportedFromOtherTool | SessionFlags.RequestGeneratedByFiddler | SessionFlags.ResponseGeneratedByFiddler | SessionFlags.ServedFromCache); 420 | setAllTimers(sessSummary, _baseTime); 421 | _listSessions.Add(sessSummary); 422 | 423 | { // Create a RAW data session with all of the JSON text for debugging purposes. 424 | Session sessRaw = Session.BuildFromData(false, 425 | new HTTPRequestHeaders( 426 | String.Format("/RAW_JSON"), 427 | new[] { "Host: NETLOG" }), 428 | Utilities.emptyByteArray, 429 | new HTTPResponseHeaders(200, "Analyzed Data", new[] { "Content-Type: application/json; charset=utf-8" }), 430 | Encoding.UTF8.GetBytes(JSON.JsonEncode(htFile)), 431 | SessionFlags.ImportedFromOtherTool | SessionFlags.RequestGeneratedByFiddler | SessionFlags.ResponseGeneratedByFiddler | SessionFlags.ServedFromCache); 432 | setAllTimers(sessRaw, _baseTime); 433 | _listSessions.Add(sessRaw); 434 | } 435 | 436 | Hashtable htPolledData = htFile["polledData"] as Hashtable; 437 | if (null != htPolledData) 438 | { 439 | ArrayList alExtensions = FilterExtensions(htPolledData["extensionInfo"] as ArrayList); 440 | 441 | Session sessExtensions = Session.BuildFromData(false, 442 | new HTTPRequestHeaders( 443 | String.Format("/ENABLED_EXTENSIONS"), 444 | new[] { "Host: NETLOG" }), 445 | Utilities.emptyByteArray, 446 | new HTTPResponseHeaders(200, "Analyzed Data", new[] { "Content-Type: application/json; charset=utf-8" }), 447 | Encoding.UTF8.GetBytes(JSON.JsonEncode(alExtensions)), 448 | SessionFlags.ImportedFromOtherTool | SessionFlags.RequestGeneratedByFiddler | SessionFlags.ResponseGeneratedByFiddler | SessionFlags.ServedFromCache); 449 | setAllTimers(sessExtensions, _baseTime); 450 | _listSessions.Add(sessExtensions); 451 | } 452 | 453 | int iEvent = -1; 454 | int iLastPct = 25; 455 | var dictURLRequests = new Dictionary>(); 456 | var dictSockets = new Dictionary>(); 457 | var dictDNSResolutions = new Dictionary>(); 458 | 459 | // Loop over events; bucket those associated to URLRequests by the source request's ID. 460 | ArrayList alEvents = htFile["events"] as ArrayList; 461 | int cEvents = alEvents.Count; 462 | foreach (Hashtable htEvent in alEvents) 463 | { 464 | ++iEvent; 465 | var htSource = htEvent["source"] as Hashtable; 466 | if (null == htSource) continue; 467 | int iSourceType = getIntValue(htSource["type"], -1); 468 | 469 | #region ParseCertificateRequestMessagesAndDumpToLog 470 | if (NetLogMagics.SRC_SOCKET == iSourceType) 471 | { 472 | try 473 | { 474 | // All events we care about should have parameters. 475 | if (!(htEvent["params"] is Hashtable htParams)) continue; 476 | int iType = getIntValue(htEvent["type"], -1); 477 | 478 | List events; 479 | int iSocketID = getIntValue(htSource["id"], -1); 480 | 481 | /*if (iType == NetLogMagics.SOCKET_BYTES_SENT) 482 | { 483 | FiddlerApplication.Log.LogFormat("!!!! IT WORKED!!!!"); 484 | //htParams["bytes"] 485 | }*/ 486 | 487 | if (iType != NetLogMagics.SSL_CERTIFICATES_RECEIVED && 488 | iType != NetLogMagics.SSL_HANDSHAKE_MESSAGE_SENT && 489 | iType != NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED && 490 | iType != NetLogMagics.TCP_CONNECT) continue; 491 | 492 | // Get (or create) the List of entries for this SOCKET. 493 | if (!dictSockets.ContainsKey(iSocketID)) 494 | { 495 | events = new List(); 496 | dictSockets.Add(iSocketID, events); 497 | } 498 | else 499 | { 500 | events = dictSockets[iSocketID]; 501 | } 502 | // Add this event to the SOCKET's list. 503 | events.Add(htEvent); 504 | } 505 | catch { } 506 | 507 | continue; 508 | } 509 | #endregion ParseCertificateRequestMessagesAndDumpToLog 510 | 511 | // DNS lookup 512 | if (NetLogMagics.SRC_NONE == iSourceType) 513 | { 514 | int iType = getIntValue(htEvent["type"], -1); 515 | if (iType == NetLogMagics.HOST_RESOLVER_IMPL_REQUEST) 516 | { 517 | // TODO: Do we actually care about any of the flags here? 518 | // FiddlerApplication.Log.LogString("!!0" + JSON.JsonEncode(htEvent)); 519 | /* 520 | "source":{"type":0, "start_time":"817048", "id":3246}, 521 | "params":{"network_isolation_key":"null null", "dns_query_type":0, "allow_cached_response":true, "is_speculative":false, 522 | "host":"ragnsells.crm4.dynamics.com:443"}, "time":"817048", 523 | "type":3, "phase":1} 524 | */ 525 | } 526 | } 527 | 528 | if (NetLogMagics.SRC_HOST_RESOLVER_IMPL_JOB == iSourceType) 529 | { 530 | // All events we care about should have parameters. 531 | if (!(htEvent["params"] is Hashtable htParams)) continue; 532 | int iType = getIntValue(htEvent["type"], -1); 533 | 534 | // Two DNS-related events have all the data we care about. 535 | if ((iType == NetLogMagics.HOST_RESOLVER_IMPL_JOB) || (iType == NetLogMagics.HOST_RESOLVER_IMPL_PROC_TASK)) 536 | { 537 | int iResolutionID = getIntValue(htSource["id"], -1); 538 | List events; 539 | // Get (or create) the List of entries for this sDNSHost. 540 | if (!dictDNSResolutions.ContainsKey(iResolutionID)) 541 | { 542 | events = new List(); 543 | dictDNSResolutions.Add(iResolutionID, events); 544 | } 545 | else 546 | { 547 | events = dictDNSResolutions[iResolutionID]; 548 | } 549 | // Add this event to the sDNSHost's list. 550 | events.Add(htEvent); 551 | continue; 552 | } 553 | } 554 | 555 | // Collect only events related to URL_REQUESTS. 556 | if (NetLogMagics.SRC_URL_REQUEST != iSourceType) continue; 557 | 558 | int iURLRequestID = getIntValue(htSource["id"], -1); 559 | 560 | { 561 | List events; 562 | 563 | // Get (or create) the List of entries for this URLRequest. 564 | if (!dictURLRequests.ContainsKey(iURLRequestID)) 565 | { 566 | events = new List(); 567 | dictURLRequests.Add(iURLRequestID, events); 568 | } 569 | else 570 | { 571 | events = dictURLRequests[iURLRequestID]; 572 | } 573 | 574 | // Add this event to the URLRequest's list. 575 | events.Add(htEvent); 576 | } 577 | int iPct = (int)(100 * (0.25f + 0.50f * (iEvent / (float)cEvents))); 578 | if (iPct != iLastPct) 579 | { 580 | NotifyProgress(iPct / 100f, "Parsed an event for a URLRequest"); 581 | iLastPct = iPct; 582 | } 583 | } 584 | 585 | int cURLRequests = dictURLRequests.Count; 586 | 587 | NotifyProgress(0.75f, "Finished reading event entries, saw " + cURLRequests.ToString() + " URLRequests"); 588 | 589 | GenerateSessionsFromURLRequests(dictURLRequests); 590 | 591 | StringBuilder sbClientInfo = new StringBuilder(); 592 | sbClientInfo.AppendFormat("Sensitivity:\t{0}\n", sDetailLevel); 593 | sbClientInfo.AppendFormat("Client:\t\t{0} v{1}\n", _sClient, htClientInfo["version"]); 594 | sbClientInfo.AppendFormat("Channel:\t\t{0}\n", htClientInfo["version_mod"]); 595 | sbClientInfo.AppendFormat("Commit Hash:\t{0}\n", htClientInfo["cl"]); 596 | sbClientInfo.AppendFormat("OS:\t\t{0}\n", htClientInfo["os_type"]); 597 | 598 | sbClientInfo.AppendFormat("\nCommandLine:\t{0}\n\n", htClientInfo["command_line"]); 599 | sbClientInfo.AppendFormat("Capture started:\t{0}\n", _dtBaseTime); 600 | sbClientInfo.AppendFormat("URLRequests:\t\t{0} found.\n", cURLRequests); 601 | 602 | sessSummary.utilSetResponseBody(sbClientInfo.ToString()); 603 | 604 | GenerateDebugTreeSession(dictURLRequests); 605 | GenerateSocketListSession(dictSockets); 606 | GenerateDNSResolutionListSession(dictDNSResolutions); 607 | 608 | NotifyProgress(1, "Import Completed."); 609 | return true; 610 | } 611 | 612 | /// 613 | /// Add a JSON session of the URL_REQUEST buckets for diagnostic purposes. 614 | /// WARNING: LOSSY. MANGLES TREE. DO THIS LAST. 615 | /// 616 | /// 617 | /// 618 | /// 619 | private void GenerateDebugTreeSession(Dictionary> dictURLRequests) 620 | { 621 | try 622 | { 623 | Hashtable htDebug = new Hashtable(); 624 | 625 | foreach (KeyValuePair> kvpURLRequest in dictURLRequests) 626 | { 627 | // Store off the likely initial URL for this URL Request 628 | string sUrl = String.Empty; 629 | 630 | // Remove data we're unlikely to need, and replace magics with constant strings. 631 | foreach (Hashtable ht in kvpURLRequest.Value) 632 | { 633 | ht.Remove("source"); 634 | ht.Remove("time"); 635 | 636 | try 637 | { 638 | // Replace Event type integers with names. 639 | int iType = getIntValue(ht["type"], -1); 640 | ht["type"] = dictEventTypes[iType]; 641 | 642 | if (iType == NetLogMagics.URL_REQUEST_START_JOB) { 643 | sUrl = ((string)(ht["params"] as Hashtable)?["url"] ?? sUrl); 644 | } 645 | 646 | // Replace Event phase integers with names. 647 | int iPhase = getIntValue(ht["phase"], -1); 648 | ht["phase"] = (iPhase == 1) ? "BEGIN" : (iPhase == 2) ? "END" : "NONE"; 649 | 650 | // Replace NetError integers with names. 651 | Hashtable htParams = ht["params"] as Hashtable; 652 | if (null != htParams && htParams["net_error"] is Double d) 653 | { 654 | int iErr = (int)d; 655 | if (iErr != 0) // 0 isn't valid; NetErrors are usually negative. 656 | { 657 | htParams["net_error"] = dictNetErrors[iErr]; 658 | } 659 | } 660 | } 661 | catch (Exception e) { FiddlerApplication.Log.LogFormat(DescribeExceptionWithStack(e)); } 662 | } 663 | 664 | // Copy List to ArrayList, which is the only type the serializer understands. 665 | ArrayList alE = new ArrayList(kvpURLRequest.Value); 666 | 667 | htDebug.Add(String.Format("{0} - {1}", kvpURLRequest.Key, sUrl), alE); 668 | } 669 | 670 | if (htDebug.Count > 0) 671 | { 672 | Session sessURLRequests = Session.BuildFromData(false, 673 | new HTTPRequestHeaders( 674 | String.Format("/URL_REQUESTS"), 675 | new[] { "Host: NETLOG" }), 676 | Utilities.emptyByteArray, 677 | new HTTPResponseHeaders(200, "Analyzed Data", new[] { "Content-Type: application/json; charset=utf-8" }), 678 | Encoding.UTF8.GetBytes(JSON.JsonEncode(htDebug)), 679 | SessionFlags.ImportedFromOtherTool | SessionFlags.RequestGeneratedByFiddler | SessionFlags.ResponseGeneratedByFiddler | SessionFlags.ServedFromCache); 680 | setAllTimers(sessURLRequests, _baseTime); 681 | _listSessions.Add(sessURLRequests); 682 | } 683 | } 684 | catch (Exception e) { FiddlerApplication.Log.LogFormat("GenerateDebugTreeSession failed: "+ DescribeExceptionWithStack(e)); } 685 | } 686 | 687 | private void GenerateSocketListSession(Dictionary> dictSockets) 688 | { 689 | try 690 | { 691 | Hashtable htAllSockets = new Hashtable(); 692 | foreach (KeyValuePair> kvpSocket in dictSockets) 693 | { 694 | string sSubjectCNinFirstCert = String.Empty; 695 | Hashtable htThisSocket = new Hashtable(); 696 | 697 | foreach (Hashtable htEvent in kvpSocket.Value) 698 | { 699 | int iType = getIntValue(htEvent["type"], -1); 700 | var htParams = (Hashtable) htEvent["params"]; 701 | 702 | if (iType == NetLogMagics.TCP_CONNECT) 703 | { 704 | if (htParams.ContainsKey("local_address")) 705 | { 706 | htThisSocket.Add("local_address", htParams["local_address"]); 707 | } 708 | //"remote_address", "local_address", "address_list" 709 | if (htParams.ContainsKey("remote_address")) 710 | { 711 | htThisSocket.Add("remote_address", htParams["remote_address"]); 712 | } 713 | if (htParams.ContainsKey("address_list")) 714 | { 715 | htThisSocket.Add("address_list", htParams["address_list"]); 716 | } 717 | continue; 718 | } 719 | 720 | if (iType == NetLogMagics.SSL_CERTIFICATES_RECEIVED) 721 | { 722 | StringBuilder sbCertsReceived = new StringBuilder(); 723 | ArrayList alCerts = htParams["certificates"] as ArrayList; 724 | if (alCerts.Count < 1) continue; 725 | 726 | Hashtable htParsedCerts = new Hashtable(alCerts.Count); 727 | try 728 | { 729 | for (int i = 0; i < alCerts.Count; i++) 730 | { 731 | var htThisCert = new Hashtable(); 732 | htParsedCerts.Add(i.ToString(), htThisCert); 733 | var certItem = new X509Certificate2(); 734 | 735 | certItem.Import(Encoding.ASCII.GetBytes(alCerts[i] as string)); 736 | 737 | // Try to promote the SubjectCN to the title of this Socket. 738 | if (String.IsNullOrEmpty(sSubjectCNinFirstCert)) 739 | { 740 | sSubjectCNinFirstCert = (" - " + certItem.GetNameInfo(X509NameType.SimpleName, false)).ToLower(); 741 | } 742 | 743 | htThisCert.Add("Parsed", new ArrayList 744 | { 745 | "Subject: " + certItem.GetNameInfo(X509NameType.SimpleName, false), 746 | "Issuer: " + certItem.Issuer, 747 | "Expires: " + certItem.NotAfter.ToString("yyyy-MM-dd") 748 | }); 749 | 750 | htThisCert.Add("RAW", new ArrayList 751 | { 752 | alCerts[i] 753 | }); 754 | } 755 | htThisSocket.Add("Server Certificates", htParsedCerts); 756 | } 757 | catch (Exception ex) 758 | { 759 | FiddlerApplication.Log.LogString(ex.Message); 760 | htThisSocket.Add("Server Certificates", alCerts); 761 | } 762 | 763 | continue; 764 | } 765 | 766 | if (iType == NetLogMagics.SSL_HANDSHAKE_MESSAGE_SENT) 767 | { 768 | // https://source.chromium.org/chromium/chromium/src/+/main:third_party/boringssl/src/include/openssl/ssl3.h;l=306;drc=5539ecff898c79b0771340051d62bf81649e448d 769 | int iHandshakeMessageType = getIntValue(htParams["type"], -1); 770 | 771 | if ((iHandshakeMessageType != 1/*ClientHello*/)) continue; 772 | 773 | // Okay, it's a ClientHello. Log it. 774 | string sBase64Bytes = htParams["bytes"] as string; 775 | if (String.IsNullOrEmpty(sBase64Bytes)) continue; 776 | // FiddlerApplication.Log.LogFormat("Saw Handshake Message Sent of type={0}", iHandshakeMessageType); 777 | 778 | if (iHandshakeMessageType == 1 /*ClientHello*/) 779 | { 780 | try 781 | { 782 | var htClientHello = new Hashtable(); 783 | htThisSocket.Add("ClientHello", htClientHello); // TODO: Figure out why we're often hitting this twice. 784 | 785 | byte[] arr = Convert.FromBase64String(sBase64Bytes); 786 | 787 | MemoryStream oMS = new MemoryStream(); 788 | // BUG BUG BUG: HACKERY; we have to construct a fake header here. 789 | oMS.WriteByte(0x16); // TLS handshake protocol 790 | oMS.WriteByte(0x3); 791 | oMS.WriteByte(0x3); // TODO: We should at least fill the version info correctly. 792 | oMS.WriteByte(0); 793 | oMS.WriteByte(0x9b); 794 | oMS.Write(arr, 0, arr.Length); 795 | 796 | oMS.Position = 0; 797 | string sDesc = Utilities.UNSTABLE_DescribeClientHello(oMS); 798 | //FiddlerApplication.Log.LogFormat("Got ClientHello:\n{0}\n{1}", Utilities.ByteArrayToHexView(arr, 16), sDesc); 799 | 800 | htClientHello.Add("RAW", sBase64Bytes); 801 | ArrayList arrDesc = new ArrayList(sDesc.Split('\n').Select(s => s.Trim().Replace('\t', ' ')).Where(s => !string.IsNullOrEmpty(s)).Skip(2).ToArray()); 802 | htClientHello.Add("Parsed", arrDesc); 803 | } 804 | catch { } 805 | 806 | continue; 807 | } 808 | } 809 | 810 | // {"params":{"certificates":["-----BEGIN CERTIFICATE-----\nMIINqg==\n-----END CERTIFICATE-----\n","-----BEGIN CERTIFICATE-----\u4\n-----END CERTIFICATE-----\n"]},"phase":0,"source":{"id":789,"type":8},"time":"464074729","type":69}, 811 | // Parse out client certificate requests (Type 13==CertificateRequest) 812 | // {"params":{"bytes":"DQA...","type":13},"phase":0,"source":{"id":10850,"type":8},"time":"160915359","type":60 (SSL_HANDSHAKE_MESSAGE_RECEIVED)}) 813 | if (iType == NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED) 814 | { 815 | // https://source.chromium.org/chromium/chromium/src/+/main:third_party/boringssl/src/include/openssl/ssl3.h;l=306;drc=5539ecff898c79b0771340051d62bf81649e448d 816 | int iHandshakeMessageType = getIntValue(htParams["type"], -1); 817 | 818 | if ((iHandshakeMessageType != 2/*ServerHello*/) && 819 | (iHandshakeMessageType != 13/*CertificateRequest*/)) continue; 820 | 821 | // Okay, it's a ServerHello or CertificateRequest. Log it. 822 | string sBase64Bytes = htParams["bytes"] as string; 823 | if (String.IsNullOrEmpty(sBase64Bytes)) continue; 824 | // FiddlerApplication.Log.LogFormat("Saw Handshake Message Received of type={0}", iHandshakeMessageType); 825 | 826 | if (iHandshakeMessageType == 2 /*ServerHello*/) 827 | { 828 | try { 829 | var htServerHello = new Hashtable(); 830 | htThisSocket.Add("ServerHello", htServerHello); // TODO: Figure out why we're often reaching this twice. 831 | 832 | byte[] arr = Convert.FromBase64String(sBase64Bytes); 833 | 834 | MemoryStream oMS = new MemoryStream(); 835 | // BUG BUG BUG: HACKERY; we have to construct a fake header here to feed it into the Utilities function which was meant for reading socket data not NetLog messages. 836 | oMS.WriteByte(0x16); 837 | oMS.WriteByte(0x3); 838 | oMS.WriteByte(0x3); // TODO: We probably should at least fill the version info properly! 839 | oMS.WriteByte(0); 840 | oMS.WriteByte(0x9b); 841 | oMS.Write(arr, 0, arr.Length); 842 | 843 | oMS.Position = 0; 844 | string sDesc = Utilities.UNSTABLE_DescribeServerHello(oMS); 845 | // FiddlerApplication.Log.LogFormat("Got ServerHello:\n{0}\n{1}", Utilities.ByteArrayToHexView(arr, 16), sDesc); 846 | 847 | htServerHello.Add("RAW", sBase64Bytes); 848 | ArrayList arrDesc = new ArrayList(sDesc.Split('\n').Select(s => s.Trim().Replace('\t', ' ')).Where(s => !string.IsNullOrEmpty(s)).Skip(2).ToArray()); 849 | htServerHello.Add("Parsed", arrDesc); 850 | 851 | // We learn if the server is using TLS/1.3 by checking if ServerHello's supported_versions specifies TLS/1.3 852 | // https://datatracker.ietf.org/doc/html/rfc8446#section-4.2.1 853 | // Note: This is Super Hacky and depends on Fiddler not changing the format of this string. 854 | if (sDesc.Contains("supported_versions\tTls1.3")) 855 | htThisSocket.Add("Negotiated TLS Version", "1.3"); 856 | } 857 | catch {} 858 | 859 | continue; 860 | } 861 | 862 | Debug.Assert(iHandshakeMessageType == 13 /*CertificateRequest*/); 863 | 864 | // BORING SSL verion of the parsing logic: 865 | // https://cs.chromium.org/chromium/src/third_party/boringssl/src/ssl/handshake_client.cc?l=1102&rcl=5ce7022394055e183c12368778d361461fe90a6e 866 | 867 | var htCertFilter = new Hashtable(); 868 | htThisSocket.Add("Request for Client Certificate", htCertFilter); 869 | htThisSocket.Add("RAW", sBase64Bytes); 870 | 871 | byte[] arrCertRequest = Convert.FromBase64String(sBase64Bytes); 872 | Debug.Assert(13 == arrCertRequest[0]); 873 | 874 | /* Each version of TLS redefined the format of the CertificateRequest message. 875 | * TLS 1.0/TLS/1.1: https://www.rfc-editor.org/rfc/rfc4346#section-7.4.4 876 | struct { 877 | ClientCertificateType certificate_types<1..2^8-1>; 878 | DistinguishedName certificate_authorities<3..2^16-1>; 879 | } CertificateRequest; 880 | 881 | * TLS 1.2: https://www.rfc-editor.org/rfc/rfc5246#section-7.4.4 882 | struct { 883 | ClientCertificateType certificate_types<1..2^8-1>; 884 | SignatureAndHashAlgorithm supported_signature_algorithms<2^16-1>; 885 | DistinguishedName certificate_authorities<0..2^16-1>; 886 | } CertificateRequest; 887 | 888 | * TLS 1.3: https://datatracker.ietf.org/doc/html/rfc8446#section-4.3.2 889 | struct { 890 | opaque certificate_request_context<0..2^8-1>; 891 | Extension extensions<2..2^16-1>; 892 | } CertificateRequest 893 | */ 894 | 895 | if ((htThisSocket["Negotiated TLS Version"] as string) == "1.3") 896 | { 897 | ParseTLS1dot3CertificateRequest(htCertFilter, arrCertRequest); 898 | continue; 899 | } 900 | 901 | // TLS/1.2 path 902 | 903 | byte cCertTypes = arrCertRequest[4]; 904 | var alCertTypes = new ArrayList(); 905 | for (int ixCertType = 0; ixCertType 0) 962 | { 963 | int cbThisDN = (arrCertRequest[iPtr++] << 8) + arrCertRequest[iPtr++]; 964 | Debug.Assert(cbThisDN < cbCADistinguishedNames); 965 | try 966 | { 967 | byte[] bytesDER = new byte[cbThisDN]; 968 | Buffer.BlockCopy(arrCertRequest, iPtr, bytesDER, 0, cbThisDN); 969 | AsnEncodedData asndata = new AsnEncodedData(bytesDER); 970 | alCADNs.Add(new X500DistinguishedName(asndata).Name); 971 | } 972 | catch { Debug.Assert(false); } 973 | iPtr += cbThisDN; 974 | cbCADistinguishedNames -= (2 + cbThisDN); 975 | } 976 | htCertFilter.Add("Accepted Authorities", alCADNs); 977 | } 978 | catch { } 979 | 980 | continue; 981 | } 982 | } 983 | 984 | if (htThisSocket.Count > 0) 985 | { 986 | htAllSockets.Add(kvpSocket.Key + sSubjectCNinFirstCert, htThisSocket); 987 | } 988 | } 989 | 990 | // Don't add a node if there were no sockets. 991 | if (htAllSockets.Count > 0) 992 | { 993 | Session sessAllSockets = Session.BuildFromData(false, 994 | new HTTPRequestHeaders( 995 | String.Format("/SOCKETS"), 996 | new[] { "Host: NETLOG" }), 997 | Utilities.emptyByteArray, 998 | new HTTPResponseHeaders(200, "Analyzed Data", new[] { "Content-Type: application/json; charset=utf-8" }), 999 | Encoding.UTF8.GetBytes(JSON.JsonEncode(htAllSockets)), 1000 | SessionFlags.ImportedFromOtherTool | SessionFlags.RequestGeneratedByFiddler | SessionFlags.ResponseGeneratedByFiddler | SessionFlags.ServedFromCache); 1001 | setAllTimers(sessAllSockets, _baseTime); 1002 | _listSessions.Add(sessAllSockets); 1003 | } 1004 | } 1005 | catch (Exception e) { FiddlerApplication.Log.LogFormat("GenerateSocketListSession failed: " + DescribeExceptionWithStack(e)); } 1006 | } 1007 | 1008 | private static void setAllTimers(Session oS, long dt) 1009 | { 1010 | var oTimers = oS.Timers; 1011 | oTimers.ClientConnected = oTimers.ClientBeginRequest = oTimers.FiddlerGotRequestHeaders = oTimers.FiddlerBeginRequest = 1012 | oTimers.ClientBeginResponse = oTimers.FiddlerGotResponseHeaders = oTimers.ServerBeginResponse = 1013 | oTimers.ServerDoneResponse = oTimers.ClientDoneResponse = GetTimeStamp(0.0, dt); 1014 | } 1015 | 1016 | /* TLS 1.3: https://datatracker.ietf.org/doc/html/rfc8446#section-4.3.2 1017 | struct { 1018 | opaque certificate_request_context<0..2^8-1>; 1019 | Extension extensions<2..2^16-1>; 1020 | } CertificateRequest 1021 | 1022 | In TLS/1.3, fields for the certificate request are carried by "extensions": 1023 | https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml 1024 | Extension 0x2F (decimal 47) => certificate_authorities 1025 | Extension 0x0D (decimal 13) => signature_algorithms 1026 | Extension 0x32 (decimal 50) => signature_algorithms_cert 1027 | */ 1028 | private void ParseTLS1dot3CertificateRequest(Hashtable htCertFilter, byte[] arrCertRequest) 1029 | { 1030 | int iPayloadSize = (arrCertRequest[1] << 16) + 1031 | (arrCertRequest[2] << 8) + 1032 | arrCertRequest[3]; 1033 | 1034 | Debug.Assert(iPayloadSize == arrCertRequest.Length - 4); 1035 | 1036 | int iPtr = 4; 1037 | 1038 | // The first field of the request is a length-prefixed 0-255 byte opaque array named certificate_request_context 1039 | iPtr += 1+arrCertRequest[iPtr]; 1040 | 1041 | int cbExtensionList = (arrCertRequest[iPtr++] << 8) + 1042 | (arrCertRequest[iPtr++]); 1043 | Debug.Assert(iPtr + cbExtensionList == arrCertRequest.Length); 1044 | while (iPtr < arrCertRequest.Length) 1045 | { 1046 | int iExtensionType = (arrCertRequest[iPtr++] << 8) + arrCertRequest[iPtr++]; 1047 | int iExtDataLen = (arrCertRequest[iPtr++] << 8) + arrCertRequest[iPtr++]; 1048 | 1049 | byte[] arrExtData = new byte[iExtDataLen]; 1050 | Buffer.BlockCopy(arrCertRequest, iPtr, arrExtData, 0, arrExtData.Length); 1051 | 1052 | switch (iExtensionType) 1053 | { 1054 | case 0x2f: // certificate_authorities 1055 | try { 1056 | var alCADNs = new ArrayList(); 1057 | int iX = 0; 1058 | int cbCADistinguishedNames = (arrExtData[iX++] << 8) + arrExtData[iX++]; 1059 | while (cbCADistinguishedNames > 0) 1060 | { 1061 | int cbThisDN = (arrExtData[iX++] << 8) + arrExtData[iX++]; 1062 | try 1063 | { 1064 | byte[] bytesDER = new byte[cbThisDN]; 1065 | Buffer.BlockCopy(arrExtData, iX, bytesDER, 0, cbThisDN); 1066 | AsnEncodedData asndata = new AsnEncodedData(bytesDER); 1067 | alCADNs.Add(new X500DistinguishedName(asndata).Name); 1068 | } 1069 | catch { Debug.Assert(false); } 1070 | cbCADistinguishedNames -= (2 + cbThisDN); 1071 | iX += cbThisDN; 1072 | } 1073 | htCertFilter.Add("Accepted Authorities", alCADNs); 1074 | } 1075 | catch { htCertFilter.Add("Accepted Authorities", "Parse failure"); } 1076 | break; 1077 | case 0x0d: // signature_algorithms 1078 | try { 1079 | int iX = 0; 1080 | int cbSigHashAlgs = (arrExtData[iX++] << 8) + 1081 | arrExtData[iX++]; 1082 | Debug.Assert((cbSigHashAlgs % 2) == 0); 1083 | 1084 | var alSigSchemes = new ArrayList(); 1085 | 1086 | for (int ixSigHashPair = 0; ixSigHashPair < cbSigHashAlgs / 2; ++ixSigHashPair) 1087 | { 1088 | alSigSchemes.Add(GetTLS13SigSchemeString((arrExtData[iX + (2 * ixSigHashPair)] << 8) + arrExtData[1+ iX + (2 * ixSigHashPair)])); 1089 | } 1090 | htCertFilter.Add("Accepted SignatureSchemes", alSigSchemes); 1091 | } 1092 | catch { htCertFilter.Add("Accepted SignatureSchemes", "Parse failure"); } 1093 | break; 1094 | default: 1095 | htCertFilter.Add("FilterExt #" + iExtensionType.ToString(), "Length" + iExtDataLen.ToString()); 1096 | break; 1097 | } 1098 | 1099 | iPtr += (iExtDataLen); // Skip the data*/ 1100 | } 1101 | } 1102 | 1103 | private void GenerateDNSResolutionListSession(Dictionary> dictDNSResolutions) 1104 | { 1105 | if (dictDNSResolutions.Count < 1) return; 1106 | try 1107 | { 1108 | Hashtable htAllResolutions = new Hashtable(); 1109 | foreach (KeyValuePair> kvpResolution in dictDNSResolutions) 1110 | { 1111 | string sHost = String.Empty; 1112 | Hashtable htData = new Hashtable(); 1113 | foreach (Hashtable htEvent in kvpResolution.Value) 1114 | { 1115 | int iType = getIntValue(htEvent["type"], -1); 1116 | var htParams = (Hashtable)htEvent["params"]; 1117 | 1118 | // TODO: HOST_RESOLVER_IMPL_JOB_REQUEST_ATTACH has a list of all of the sslconnectjobs 1119 | // that attached to this resolution looking for an address to use. 1120 | 1121 | if (iType == NetLogMagics.HOST_RESOLVER_IMPL_JOB) 1122 | { 1123 | sHost = (htParams["host"] as String) ?? "(missing)"; 1124 | continue; 1125 | } 1126 | if (iType == NetLogMagics.HOST_RESOLVER_IMPL_PROC_TASK) 1127 | { 1128 | // TODO: What if there's more than one? 1129 | if (htParams.ContainsKey("canonical_name") && ((htParams["canonical_name"] as String) == String.Empty)) 1130 | { 1131 | htParams.Remove("canonical_name"); 1132 | } 1133 | htData = htParams; 1134 | continue; 1135 | } 1136 | 1137 | } 1138 | htAllResolutions.Add(sHost, htData); 1139 | } 1140 | 1141 | Session sessDNS = Session.BuildFromData(false, 1142 | new HTTPRequestHeaders( 1143 | String.Format("/DNS_LOOKUPS"), 1144 | new[] { "Host: NETLOG" }), 1145 | Utilities.emptyByteArray, 1146 | new HTTPResponseHeaders(200, "Analyzed Data", new[] { "Content-Type: application/json; charset=utf-8" }), 1147 | Encoding.UTF8.GetBytes(JSON.JsonEncode(htAllResolutions)), 1148 | SessionFlags.ImportedFromOtherTool | SessionFlags.RequestGeneratedByFiddler | SessionFlags.ResponseGeneratedByFiddler | SessionFlags.ServedFromCache); 1149 | setAllTimers(sessDNS, _baseTime); 1150 | _listSessions.Add(sessDNS); 1151 | } 1152 | catch (Exception e) { FiddlerApplication.Log.LogFormat("GenerateDNSResolutionListSession failed: " + DescribeExceptionWithStack(e)); } 1153 | } 1154 | 1155 | // https://www.rfc-editor.org/rfc/rfc8446#section-4.3.2:~:text=extensions%20contains%20a-,SignatureSchemeList,-value%3A%0A%0A%20%20%20%20%20%20enum%20%7B%0A%20%20%20%20%20%20%20%20%20%20/*%20RSASSA 1156 | private static string GetTLS13SigSchemeString(int iValue) 1157 | { 1158 | switch (iValue) 1159 | { 1160 | case 0x0401: return "rsa_pkcs1_sha256"; 1161 | case 0x0501: return "rsa_pkcs1_sha384"; 1162 | case 0x0601: return "rsa_pkcs1_sha512"; 1163 | 1164 | /* ECDSA algorithms */ 1165 | case 0x0403: return "ecdsa_secp256r1_sha256"; 1166 | case 0x0503: return "ecdsa_secp384r1_sha384"; 1167 | case 0x0603: return "ecdsa_secp521r1_sha512"; 1168 | 1169 | /* RSASSA-PSS algorithms with public key OID rsaEncryption */ 1170 | case 0x0804: return "rsa_pss_rsae_sha256"; 1171 | case 0x0805: return "rsa_pss_rsae_sha384"; 1172 | case 0x0806: return "rsa_pss_rsae_sha512"; 1173 | 1174 | /* EdDSA algorithms */ 1175 | case 0x0807: return "ed25519"; 1176 | case 0x0808: return "ed448"; 1177 | 1178 | /* RSASSA-PSS algorithms with public key OID RSASSA-PSS */ 1179 | case 0x0809: return "rsa_pss_pss_sha256"; 1180 | case 0x080a: return "rsa_pss_pss_sha384"; 1181 | case 0x080b: return "rsa_pss_pss_sha512"; 1182 | 1183 | case 0x0201: return "rsa_pkcs1_sha1"; 1184 | case 0x0202: return "dsa_sha1"; 1185 | case 0x0203: return "ecdsa_sha1"; 1186 | 1187 | default: return String.Format("unknown(0x{0:x})", iValue); 1188 | } 1189 | } 1190 | 1191 | private static string GetHashSigString(int iHash, int iSig) 1192 | { 1193 | string sHash; 1194 | string sSig; 1195 | switch (iHash) 1196 | { 1197 | // Hash https://www.iana.org/assignments/tls-parameters/tls-parameters.xml#tls-parameters-18 1198 | case 0: sHash = "none"; break; 1199 | case 1: sHash = "md5"; break; 1200 | case 2: sHash = "sha1"; break; 1201 | case 3: sHash = "sha224"; break; 1202 | case 4: sHash = "sha256"; break; 1203 | case 5: sHash = "sha384"; break; 1204 | case 6: sHash = "sha512"; break; 1205 | case 8: sHash = "intrinsic"; break; 1206 | default: sHash = String.Format("unknown(0x{0:x})", iHash); break; 1207 | } 1208 | switch (iSig) 1209 | { 1210 | // Sigs https://www.iana.org/assignments/tls-parameters/tls-parameters.xml#tls-parameters-16 1211 | case 0: sSig = "anonymous"; break; 1212 | case 1: sSig = "rsa"; break; 1213 | case 2: sSig = "dsa"; break; 1214 | case 3: sSig = "ecdsa"; break; 1215 | case 4: sSig = "(reserved-4)"; break; 1216 | case 5: sSig = "(reserved-5)"; break; 1217 | case 6: sSig = "(reserved-6)"; break; 1218 | case 7: sSig = "ed25519"; break; 1219 | case 8: sSig = "ed448"; break; 1220 | case 64: sSig = "gostr34102012_256"; break; 1221 | case 65: sSig = "gostr34102012_512"; break; 1222 | default: sSig = String.Format("unknown(0x{0:x})", iSig); break; 1223 | } 1224 | return String.Format("{0}_{1}", sHash, sSig); 1225 | } 1226 | 1227 | private int GenerateSessionsFromURLRequests(Dictionary> dictURLRequests) 1228 | { 1229 | int cURLRequests = dictURLRequests.Count; 1230 | int iLastPct; 1231 | int iRequest = 0; 1232 | iLastPct = 75; 1233 | 1234 | // Iterate over each URLRequest's events bucket and parse one or more Sessions out of it. 1235 | foreach (KeyValuePair> kvpUR in dictURLRequests) 1236 | { 1237 | ++iRequest; 1238 | ParseSessionsFromBucket(kvpUR); 1239 | int iPct = (int)(100 * (0.75f + 0.25f * (iRequest / (float)cURLRequests))); 1240 | if (iPct != iLastPct) 1241 | { 1242 | NotifyProgress(iPct / 100f, "Completed analysis of URLRequest"); 1243 | iLastPct = iPct; 1244 | } 1245 | } 1246 | 1247 | return iLastPct; 1248 | } 1249 | 1250 | // Each bucket contains all of the events associated with a URL_REQUEST, and each URL_REQUEST may contain 1251 | // one or more (Auth, Redirects) Web Sessions. 1252 | private void ParseSessionsFromBucket(KeyValuePair> kvpUR) 1253 | { 1254 | List listEvents = kvpUR.Value; 1255 | 1256 | SessionFlags oSF = SessionFlags.ImportedFromOtherTool | SessionFlags.ResponseStreamed; // IsHTTPS? 1257 | HTTPRequestHeaders oRQH = null; 1258 | HTTPResponseHeaders oRPH = null; 1259 | ArrayList alEarlyHints = null; 1260 | MemoryStream msResponseBody = new MemoryStream(); 1261 | Dictionary dictSessionFlags = new Dictionary(); 1262 | List listCookieSendExclusions = new List(); 1263 | List listCookieSetExclusions = new List(); 1264 | 1265 | dictSessionFlags["X-Netlog-URLRequest-ID"] = kvpUR.Key.ToString(); 1266 | dictSessionFlags["X-ProcessInfo"] = String.Format("{0}:0", _sClient); 1267 | 1268 | string sURL = String.Empty; 1269 | string sMethod = "GET"; 1270 | SessionTimers oTimers = new SessionTimers(); 1271 | 1272 | int cbDroppedResponseBody = 0; 1273 | bool bHasStartJob = false; 1274 | bool bHasSendRequest = false; 1275 | 1276 | foreach (Hashtable htEvent in listEvents) 1277 | { 1278 | try 1279 | { 1280 | int iType = getIntValue(htEvent["type"], -1); 1281 | 1282 | if (iType == -1) 1283 | { 1284 | string sType = (htEvent["name"] as String); 1285 | switch (sType) { 1286 | case "REQUEST_ALIVE": iType = NetLogMagics.REQUEST_ALIVE; break; 1287 | case "URL_REQUEST_START_JOB": iType = NetLogMagics.URL_REQUEST_START_JOB; break; 1288 | case "HTTP_TRANSACTION_SEND_REQUEST_HEADERS": iType = NetLogMagics.SEND_HEADERS; break; 1289 | case "HTTP_TRANSACTION_QUIC_SEND_REQUEST_HEADERS": iType = NetLogMagics.SEND_QUIC_HEADERS; break; 1290 | case "HTTP_TRANSACTION_HTTP2_SEND_REQUEST_HEADERS": iType = NetLogMagics.SEND_HTTP2_HEADERS; break; 1291 | case "HTTP_TRANSACTION_READ_RESPONSE_HEADERS": iType = NetLogMagics.READ_HEADERS; break; 1292 | case "URL_REQUEST_JOB_FILTERED_BYTES_READ": iType = NetLogMagics.FILTERED_BYTES_READ; break; 1293 | case "COOKIE_INCLUSION_STATUS": iType = NetLogMagics.COOKIE_INCLUSION_STATUS; break; 1294 | case "HTTP_TRANSACTION_SEND_REQUEST_BODY": iType = NetLogMagics.SEND_BODY; break; 1295 | case "HTTP_TRANSACTION_SEND_REQUEST": iType = NetLogMagics.SEND_REQUEST; break; 1296 | case "SSL_CERTIFICATES_RECEIVED": iType = NetLogMagics.SSL_CERTIFICATES_RECEIVED; break; 1297 | case "SSL_HANDSHAKE_MESSAGE_SENT": iType = NetLogMagics.SSL_HANDSHAKE_MESSAGE_SENT; break; 1298 | case "SSL_HANDSHAKE_MESSAGE_RECEIVED": iType = NetLogMagics.SSL_HANDSHAKE_MESSAGE_RECEIVED; break; 1299 | case "SOCKET_BYTES_SENT": iType = NetLogMagics.SOCKET_BYTES_SENT; break; 1300 | } 1301 | } 1302 | 1303 | var htParams = htEvent["params"] as Hashtable; 1304 | if (null == htParams) 1305 | { 1306 | // The "Trace" format nests the params object inside an args object. 1307 | var htArgs = htEvent["args"] as Hashtable; 1308 | if (null != htArgs) htParams = htArgs["params"] as Hashtable; 1309 | } 1310 | 1311 | // Most events we care about should have parameters. LANDMINE_MEME HERE 1312 | if (iType != NetLogMagics.SEND_REQUEST && null == htParams) continue; 1313 | 1314 | // FiddlerApplication.Log.LogFormat("URLRequest#{0} - Event type: {1} - {2}", kvpUR.Key, iType, sURL); 1315 | 1316 | #region ParseImportantEvents 1317 | // C# cannot |switch()| on non-constant case values. Hrmph. 1318 | if (iType == NetLogMagics.REQUEST_ALIVE) 1319 | { 1320 | int iTrafficAnnotation = getIntValue(htParams["traffic_annotation"], 0); 1321 | if (iTrafficAnnotation > 0) 1322 | { 1323 | string sAnnotation = iTrafficAnnotation.ToString(); 1324 | switch (iTrafficAnnotation) 1325 | { 1326 | // TODO (Bug #3): Lookup a friendly string from https://source.chromium.org/chromium/chromium/src/+/master:tools/traffic_annotation/summary/annotations.xml;l=27?q=101845102&ss=chromium 1327 | case 63171670: sAnnotation += " (navigation_url_loader)"; break; 1328 | case 101845102: sAnnotation += " (blink_resource_loader)"; break; 1329 | case 110815970: sAnnotation += " (resource prefetch)"; break; 1330 | case 112189210: sAnnotation += " (favicon_loader)"; break; 1331 | case 16469669: sAnnotation += " (background_fetch)"; break; 1332 | case 35266994: sAnnotation += " (early_hints_preload)"; break; 1333 | case 113711087: sAnnotation += " (edge_replace_update_client)"; break; 1334 | case 107267424: sAnnotation += " (open_search)"; break; 1335 | case 21498113: sAnnotation += " (service_worker_script_load)"; break; 1336 | case 88863520: sAnnotation += " (autofill_query)"; break; 1337 | case 30454590: sAnnotation += " (smartscreen)"; break; 1338 | } 1339 | dictSessionFlags["X-Netlog-Traffic_Annotation"] = sAnnotation; 1340 | } 1341 | continue; 1342 | } 1343 | 1344 | if (iType == NetLogMagics.URL_REQUEST_START_JOB) 1345 | { 1346 | // If we already had a URL_REQUEST_START_JOB on this URL_REQUEST, we are probably chasing a redirect. 1347 | // "finish" off the existing Session and start a new one at this point. 1348 | // TODO: This is really hacky right now. 1349 | if (bHasStartJob) 1350 | { 1351 | FiddlerApplication.Log.LogFormat("Got more than one START_JOB on URLRequest #{0} for {1}", kvpUR.Key, sURL); 1352 | AnnotateHeadersWithUnstoredCookies(oRPH, listCookieSetExclusions); 1353 | BuildAndAddSession(ref oSF, oRQH, oRPH, msResponseBody, dictSessionFlags, sURL, sMethod, oTimers, cbDroppedResponseBody); 1354 | alEarlyHints = null; oRQH = null; oRPH = null; msResponseBody = new MemoryStream(); sURL = String.Empty; sMethod = "GET"; oTimers = new SessionTimers(); 1355 | // We are effectively on a new request, don't act like we've seen headers for it before. 1356 | bHasSendRequest = false; 1357 | listCookieSetExclusions.Clear(); 1358 | listCookieSendExclusions.Clear(); 1359 | // ISSUE: There are probably some dictSessionFlags values that should be cleared here. 1360 | dictSessionFlags.Remove("ui-comment"); 1361 | dictSessionFlags.Remove("ui-backcolor"); 1362 | } 1363 | 1364 | bHasStartJob = true; 1365 | sURL = (string)htParams["url"]; 1366 | sMethod = (string)htParams["method"]; 1367 | 1368 | // In case we don't get these later. 1369 | oTimers.ClientBeginRequest = oTimers.FiddlerGotRequestHeaders = oTimers.FiddlerBeginRequest = GetTimeStamp(htEvent["time"], _baseTime); 1370 | continue; 1371 | } 1372 | 1373 | if (iType == NetLogMagics.SEND_REQUEST) 1374 | { 1375 | // Only look for "BEGIN" events. 1376 | if ((getIntValue(htEvent["phase"], -1) != 1) && 1377 | (htEvent["ph"] as string != "b")) continue; 1378 | 1379 | // If we already had a SEND_REQUEST on this URL_REQUEST, we are probably in a HTTP Auth transaction. 1380 | // "finish" off the existing Session and start a new one at this point. 1381 | // TODO: This is really hacky right now. 1382 | if (bHasSendRequest) 1383 | { 1384 | FiddlerApplication.Log.LogFormat("Got more than one SendRequest on the URLRequest for {0}", sURL); 1385 | AnnotateHeadersWithUnstoredCookies(oRPH, listCookieSetExclusions); 1386 | BuildAndAddSession(ref oSF, oRQH, oRPH, msResponseBody, dictSessionFlags, sURL, sMethod, oTimers, cbDroppedResponseBody); 1387 | // Keep sURL and sMethod, they shouldn't be changing. 1388 | alEarlyHints = null; oRQH = null; oRPH = null; msResponseBody = new MemoryStream(); oTimers = new SessionTimers(); 1389 | 1390 | listCookieSetExclusions.Clear(); 1391 | listCookieSendExclusions.Clear(); 1392 | // ISSUE: There are probably some dictSessionFlags values that should be cleared here. 1393 | } 1394 | 1395 | bHasSendRequest = true; 1396 | continue; 1397 | } 1398 | 1399 | if (iType == NetLogMagics.SEND_HEADERS) 1400 | { 1401 | oTimers.ClientBeginRequest = oTimers.FiddlerGotRequestHeaders = oTimers.FiddlerBeginRequest = GetTimeStamp(htEvent["time"], _baseTime); 1402 | ArrayList alHeaderLines = htParams["headers"] as ArrayList; 1403 | if (null != alHeaderLines && alHeaderLines.Count > 0) 1404 | { 1405 | string sRequest = sMethod + " " + sURL + " HTTP/1.1\r\n" + String.Join("\r\n", alHeaderLines.Cast().ToArray()); 1406 | oRQH = Fiddler.Parser.ParseRequest(sRequest); 1407 | AnnotateHeadersWithUnsentCookies(oRQH, listCookieSendExclusions); 1408 | } 1409 | continue; 1410 | } 1411 | 1412 | if (iType == NetLogMagics.SEND_QUIC_HEADERS) 1413 | { 1414 | dictSessionFlags["X-Transport"] = "QUIC"; 1415 | oTimers.ClientBeginRequest = oTimers.FiddlerGotRequestHeaders = oTimers.FiddlerBeginRequest = GetTimeStamp(htEvent["time"], _baseTime); 1416 | string sRequest = HeadersToString(htParams["headers"]); 1417 | if (!String.IsNullOrEmpty(sRequest)) 1418 | { 1419 | oRQH = Fiddler.Parser.ParseRequest(sRequest); 1420 | AnnotateHeadersWithUnsentCookies(oRQH, listCookieSendExclusions); 1421 | } 1422 | continue; 1423 | } 1424 | 1425 | if (iType == NetLogMagics.SEND_HTTP2_HEADERS) 1426 | { 1427 | dictSessionFlags["X-Transport"] = "HTTP2"; 1428 | oTimers.ClientBeginRequest = oTimers.FiddlerGotRequestHeaders = oTimers.FiddlerBeginRequest = GetTimeStamp(htEvent["time"], _baseTime); 1429 | string sRequest = HeadersToString(htParams["headers"]); 1430 | if (!String.IsNullOrEmpty(sRequest)) 1431 | { 1432 | oRQH = Fiddler.Parser.ParseRequest(sRequest); 1433 | AnnotateHeadersWithUnsentCookies(oRQH, listCookieSendExclusions); 1434 | } 1435 | continue; 1436 | } 1437 | 1438 | if (iType == NetLogMagics.COOKIE_INCLUSION_STATUS) 1439 | { 1440 | string sOperation = (htParams["operation"] as string) ?? String.Empty; 1441 | string sCookieName = (htParams["name"] as string) ?? "(name-unavailable)"; 1442 | 1443 | // Edge-specific fields 1444 | // bool bIsLegacyCookie = (htParams["msft_browser_legacy_cookie"] as Boolean) ?? false; 1445 | // string bBrowserProvenance = (htParams["browser_provenance"] as string) ?? String.Empty /*Native*/; 1446 | 1447 | // TODO: As of Chrome 81, CookieInclusionStatusNetLogParams also adds |domain| and |path| attributes available if "sensitive" data is included. 1448 | 1449 | // In Chrome 81.3993, the |exclusion_reason| field was renamed to |status| because the |cookie_inclusion_status| entries are 1450 | // now also emitted for included cookies. 1451 | string sExclusionReasons = (htParams["exclusion_reason"] as string); 1452 | if (String.IsNullOrEmpty(sExclusionReasons)) sExclusionReasons = (htParams["status"] as string) ?? String.Empty; 1453 | 1454 | // If the log indicates that the cookie was included, just skip it for now. 1455 | // https://source.chromium.org/chromium/chromium/src/+/master:net/cookies/canonical_cookie.cc;l=899?q=GetDebugString%20cookie&ss=chromium&originalUrl=https:%2F%2Fcs.chromium.org%2F 1456 | if (sExclusionReasons.OICContains("include")) 1457 | { 1458 | if ("expire" == sOperation) 1459 | { 1460 | // EXCLUDE_INVALID_DOMAIN,EXCLUDE_OVERWRITE_HTTP_ONLY,EXCLUDE_OVERWRITE_SECURE, 1461 | // EXCLUDE_FAILURE_TO_STORE (e.g. Set-Cookie header > 4096 characters), 1462 | // EXCLUDE_NONCOOKIEABLE_SCHEME,EXCLUDE_INVALID_PREFIX 1463 | listCookieSetExclusions.Add(String.Format("The cookie '{0}' was sent already expired.", sCookieName)); 1464 | } 1465 | 1466 | // TODO: Offer a richer cookie-debugging story that exposes the domain/path/inclusion status. 1467 | continue; 1468 | } 1469 | 1470 | // See |ExclusionReason| list in https://cs.chromium.org/chromium/src/net/cookies/canonical_cookie.h?type=cs&q=EXCLUDE_SAMESITE_LAX&sq=package:chromium&g=0&l=304 1471 | // EXCLUDE_HTTP_ONLY, EXCLUDE_SECURE_ONLY,EXCLUDE_DOMAIN_MISMATCH,EXCLUDE_NOT_ON_PATH,EXCLUDE_INVALID_PREFIX 1472 | // EXCLUDE_SAMESITE_STRICT,EXCLUDE_SAMESITE_LAX,EXCLUDE_SAMESITE_EXTENDED, 1473 | // EXCLUDE_SAMESITE_UNSPECIFIED_TREATED_AS_LAX,EXCLUDE_SAMESITE_NONE_INSECURE, 1474 | // EXCLUDE_USER_PREFERENCES, 1475 | 1476 | if ("store" == sOperation) 1477 | { 1478 | // EXCLUDE_INVALID_DOMAIN,EXCLUDE_OVERWRITE_HTTP_ONLY,EXCLUDE_OVERWRITE_SECURE, 1479 | // EXCLUDE_FAILURE_TO_STORE (e.g. Set-Cookie header > 4096 characters), 1480 | // EXCLUDE_NONCOOKIEABLE_SCHEME,EXCLUDE_INVALID_PREFIX 1481 | listCookieSetExclusions.Add(String.Format("Blocked set of '{0}' due to '{1}'", sCookieName, sExclusionReasons)); 1482 | } 1483 | else if ("expire" == sOperation) 1484 | { 1485 | listCookieSetExclusions.Add(String.Format("Blocked expire (set) of '{0}' due to '{1}'", sCookieName, sExclusionReasons)); 1486 | } 1487 | else if ("send" == sOperation) 1488 | { 1489 | // Don't warn about cookies which are obviously inapplicable 1490 | if (!new string[] { "EXCLUDE_DOMAIN_MISMATCH", "EXCLUDE_NOT_ON_PATH" }.Any(s => sExclusionReasons.Contains(s))) 1491 | { 1492 | listCookieSendExclusions.Add(String.Format("Blocked send of '{0}' due to '{1}'", sCookieName, sExclusionReasons)); 1493 | } 1494 | } 1495 | else { Debug.Assert(false, "Unknown operation"); } 1496 | 1497 | continue; 1498 | } 1499 | 1500 | if (iType == NetLogMagics.SEND_BODY) 1501 | { 1502 | int iBodyLength = getIntValue(htParams["length"], 0); 1503 | if (iBodyLength > 0) 1504 | { 1505 | oSF |= SessionFlags.RequestBodyDropped; 1506 | dictSessionFlags["X-RequestBodyLength"] = iBodyLength.ToString("N0"); 1507 | } 1508 | continue; 1509 | } 1510 | 1511 | if (iType == NetLogMagics.READ_EARLY_HINTS_RESPONSE_HEADERS) 1512 | { 1513 | ArrayList alHeaderLines = htParams["headers"] as ArrayList; 1514 | if (null != alHeaderLines && alHeaderLines.Count > 0) 1515 | { 1516 | if (null == alEarlyHints) alEarlyHints = new ArrayList(); 1517 | alEarlyHints.AddRange(alHeaderLines); 1518 | } 1519 | 1520 | } 1521 | if ((iType == NetLogMagics.READ_HEADERS) || 1522 | (iType == NetLogMagics.FAKE_RESPONSE_HEADERS_CREATED)) 1523 | { 1524 | ArrayList alHeaderLines = htParams["headers"] as ArrayList; 1525 | if (null != alHeaderLines && alHeaderLines.Count > 0) 1526 | { 1527 | string sResponse = string.Join("\r\n", alHeaderLines.Cast().ToArray()); 1528 | oRPH = Fiddler.Parser.ParseResponse(sResponse); 1529 | if (null != alEarlyHints && alEarlyHints.Count > 0) 1530 | { 1531 | foreach (string s in alEarlyHints) 1532 | { 1533 | oRPH.Add("_X-NetLog-Found-Early-Hint", s); 1534 | } 1535 | } 1536 | } 1537 | 1538 | oTimers.ClientBeginResponse = oTimers.FiddlerGotResponseHeaders = oTimers.ServerBeginResponse = GetTimeStamp(htEvent["time"], _baseTime); 1539 | continue; 1540 | } 1541 | 1542 | // ISSUE: WHAT ABOUT "URL_REQUEST_JOB_BYTES_READ" BYTES? DONT WANT DUPLICATES. 1543 | 1544 | if (iType == NetLogMagics.FILTERED_BYTES_READ) 1545 | { 1546 | string sBase64Bytes = htParams["bytes"] as string; 1547 | if (!String.IsNullOrEmpty(sBase64Bytes)) 1548 | { 1549 | byte[] arrThisRead = Convert.FromBase64String(sBase64Bytes); 1550 | msResponseBody.Write(arrThisRead, 0, arrThisRead.Length); // WTF, why so verbose? 1551 | } 1552 | else 1553 | { 1554 | cbDroppedResponseBody += getIntValue(htParams["byte_count"], 0); 1555 | } 1556 | oTimers.ServerDoneResponse = oTimers.ClientDoneResponse = GetTimeStamp(htEvent["time"], _baseTime); 1557 | continue; 1558 | } 1559 | } 1560 | catch (Exception eX) 1561 | { 1562 | FiddlerApplication.Log.LogFormat("Parsing event failed:\n{0}", DescribeExceptionWithStack(eX)); 1563 | } 1564 | #endregion ParseImportantEvents 1565 | } 1566 | 1567 | bool bCookieSetFailed = listCookieSetExclusions.Count > 0; 1568 | if (bCookieSetFailed) { 1569 | dictSessionFlags["ui-backcolor"] = "#FF8080"; 1570 | dictSessionFlags["ui-comments"] = "A cookie set by Set-Cookie was not stored."; 1571 | AnnotateHeadersWithUnstoredCookies(oRPH, listCookieSetExclusions); 1572 | } 1573 | BuildAndAddSession(ref oSF, oRQH, oRPH, msResponseBody, dictSessionFlags, sURL, sMethod, oTimers, cbDroppedResponseBody); 1574 | } 1575 | 1576 | private static void AnnotateHeadersWithUnsentCookies(HTTPRequestHeaders oRQH, List listExclusions) 1577 | { 1578 | if (null == oRQH) return; 1579 | foreach (string sExclusion in listExclusions) { 1580 | oRQH.Add("$NETLOG-CookieNotSent", sExclusion); 1581 | } 1582 | 1583 | listExclusions.Clear(); 1584 | } 1585 | 1586 | private static void AnnotateHeadersWithUnstoredCookies(HTTPResponseHeaders oRPH, List listExclusions) 1587 | { 1588 | if (null == oRPH) return; 1589 | foreach (string sExclusion in listExclusions) 1590 | { 1591 | oRPH.Add("$NETLOG-CookieNotStored", sExclusion); 1592 | } 1593 | 1594 | listExclusions.Clear(); 1595 | } 1596 | 1597 | private void BuildAndAddSession(ref SessionFlags oSF, HTTPRequestHeaders oRQH, HTTPResponseHeaders oRPH, MemoryStream msResponseBody, 1598 | Dictionary dictSessionFlags, string sURL, string sMethod, SessionTimers oTimers, int cbDroppedResponseBody) 1599 | { 1600 | // TODO: Sanity-check missing headers. 1601 | if (null == oRQH && !String.IsNullOrWhiteSpace(sURL)) 1602 | { 1603 | oRQH = Fiddler.Parser.ParseRequest(sMethod + " " + sURL + " HTTP/1.1\r\nMissing-Data: Request Headers not captured in NetLog\r\n\r\n"); 1604 | } 1605 | 1606 | if (msResponseBody.Length < 1 && cbDroppedResponseBody > 0) 1607 | { 1608 | dictSessionFlags["X-RESPONSEBODYTRANSFERLENGTH"] = cbDroppedResponseBody.ToString("N0"); 1609 | oSF |= SessionFlags.ResponseBodyDropped; 1610 | } 1611 | 1612 | if ((null != oRPH) && msResponseBody.Length > 0) 1613 | { 1614 | // Body bytes stored in the file were already unchunked and decompressed, so rename these 1615 | // headers so we can use this session for AutoResponder playback, etc. 1616 | oRPH.RenameHeaderItems("Content-Encoding", "X-Netlog-Removed-Content-Encoding"); 1617 | oRPH.RenameHeaderItems("Transfer-Encoding", "X-Netlog-Removed-Transfer-Encoding"); 1618 | string sOriginalCL = oRPH["Content-Length"]; 1619 | oRPH["Content-Length"] = msResponseBody.Length.ToString(); 1620 | if (!String.IsNullOrEmpty(sOriginalCL) && oRPH["Content-Length"] != sOriginalCL) 1621 | { 1622 | oRPH["X-Netlog-Original-Content-Length"] = sOriginalCL; 1623 | } 1624 | } 1625 | 1626 | Session oS = Session.BuildFromData(false, 1627 | oRQH, 1628 | Utilities.emptyByteArray, 1629 | oRPH, 1630 | msResponseBody.ToArray(), 1631 | oSF); 1632 | 1633 | // Store the URL from the URLRequest here, because it might have a URL Fragment in it, and the URL built 1634 | // out of the headers definitely should not. 1635 | if (oS.fullUrl != sURL) { 1636 | oS["X-Netlog-URLRequest-URL"] = sURL; 1637 | } 1638 | 1639 | // Attach the SessionFlags to the new Session. 1640 | foreach (KeyValuePair sFlag in dictSessionFlags) 1641 | { 1642 | oS[sFlag.Key] = sFlag.Value; 1643 | } 1644 | 1645 | // Assign the timestamps we read from the events. 1646 | oS.Timers = oTimers; 1647 | 1648 | _listSessions.Add(oS); 1649 | // FiddlerApplication.Log.LogFormat("Added Session #{0}", oS.id); 1650 | } 1651 | 1652 | // Chrome annoyingly uses both Hashtables (JS Object) and Arraylists (JS Array) to represent headers 1653 | // depending on which event is in use. 1654 | public static string HeadersToString(object o) 1655 | { 1656 | if (o is ArrayList) return HeaderArrayToHeaderString((ArrayList)o); 1657 | if (o is Hashtable) return HeaderHashtableToHeaderString((Hashtable)o); 1658 | if (null != o) Debug.Assert(false, "Unexpected header format"); 1659 | return String.Empty; 1660 | } 1661 | 1662 | private static string HeaderHashtableToHeaderString(Hashtable ht) 1663 | { 1664 | if (null == ht || ht.Count < 1) return String.Empty; 1665 | 1666 | string sMethod = "MISSING"; 1667 | string sScheme = "MISSING"; 1668 | string sAuthority = "MISSING"; 1669 | string sPath = "/MISSING"; 1670 | List slHeaders = new List(); 1671 | 1672 | foreach (DictionaryEntry de in ht) 1673 | { 1674 | KeyValuePair kvp = new KeyValuePair((string)de.Key, (string)de.Value); 1675 | if (kvp.Key.StartsWith(":")) 1676 | { 1677 | if (kvp.Key.Equals(":method")) { sMethod = kvp.Value; continue; } 1678 | if (kvp.Key.Equals(":scheme")) { sScheme = kvp.Value; continue; } 1679 | if (kvp.Key.Equals(":authority")) { sAuthority = kvp.Value; continue; } 1680 | if (kvp.Key.Equals(":path")) { sPath = kvp.Value; continue; } 1681 | } 1682 | slHeaders.Add(kvp.Key + ": " + kvp.Value); 1683 | } 1684 | 1685 | return String.Format("{0} {1}://{2}{3} {4}\r\n{5}", 1686 | sMethod, sScheme, sAuthority, sPath, "HTTP/1.1", 1687 | String.Join("\r\n", slHeaders.ToArray())); 1688 | } 1689 | 1690 | private static string HeaderArrayToHeaderString(ArrayList alIn) 1691 | { 1692 | if (null == alIn || alIn.Count < 1) return String.Empty; 1693 | 1694 | string sMethod = "MISSING"; 1695 | string sScheme = "MISSING"; 1696 | string sAuthority = "MISSING"; 1697 | string sPath = "/MISSING"; 1698 | List slHeaders = new List(); 1699 | 1700 | foreach (string s in alIn) 1701 | { 1702 | if (s.StartsWith(":")) 1703 | { 1704 | if (s.StartsWith(":method: ")) { sMethod = s.Substring(9); continue; } 1705 | if (s.StartsWith(":scheme: ")) { sScheme = s.Substring(9); continue; } 1706 | if (s.StartsWith(":authority: ")) { sAuthority = s.Substring(12); continue; } 1707 | if (s.StartsWith(":path: ")) { sPath = s.Substring(7); continue; } 1708 | } 1709 | slHeaders.Add(s); 1710 | } 1711 | 1712 | return String.Format("{0} {1}://{2}{3} {4}\r\n{5}", 1713 | sMethod, sScheme, sAuthority, sPath, "HTTP/1.1", 1714 | String.Join("\r\n", slHeaders.ToArray())); 1715 | } 1716 | 1717 | } 1718 | } 1719 | -------------------------------------------------------------------------------- /FiddlerImportNetlog/Interesting Snippets.txt: -------------------------------------------------------------------------------- 1 | Life of a URLRequest 2 | https://chromium.googlesource.com/chromium/src/+/HEAD/net/docs/life-of-a-url-request.md 3 | 4 | NetLog Design Doc 5 | https://www.chromium.org/developers/design-documents/network-stack/netlog 6 | 7 | Source ref: 8 | https://cs.chromium.org/chromium/src/components/net_log/chrome_net_log.cc?type=cs&g=0&l=58 9 | 10 | 11 | Code references below are (c)The Chromium Authors. 12 | 13 | void ChromeNetLog::StartWritingToFile( 14 | const base::FilePath& path, 15 | net::NetLogCaptureMode capture_mode, 16 | const base::CommandLine::StringType& command_line_string, 17 | const std::string& channel_string) { 18 | DCHECK(!path.empty()); 19 | 20 | // TODO(739485): The log file does not contain about:flags data. 21 | file_net_log_observer_ = net::FileNetLogObserver::CreateUnbounded( 22 | path, GetConstants(command_line_string, channel_string)); 23 | 24 | file_net_log_observer_->StartObserving(this, capture_mode); 25 | } 26 | 27 | 28 | 29 | network_context->CreateNetLogExporter(mojo::MakeRequest(&net_log_exporter_)); 30 | base::Value custom_constants = base::Value::FromUniquePtrValue( 31 | ChromeNetLog::GetPlatformConstants(command_line_string, channel_string)); 32 | 33 | 34 | 35 | // NetLogCaptureMode specifies the granularity of events that should be emitted 36 | // to the log. It is a simple wrapper around an integer, so it should be passed 37 | // to functions by value rather than by reference. 38 | class NET_EXPORT NetLogCaptureMode { 39 | public: 40 | // NOTE: Default assignment and copy constructor are OK. 41 | 42 | // The default constructor creates a capture mode equivalent to 43 | // Default(). 44 | NetLogCaptureMode(); 45 | 46 | // Constructs a capture mode which logs basic events and event parameters. 47 | // include_cookies_and_credentials() --> false 48 | // include_socket_bytes() --> false 49 | static NetLogCaptureMode Default(); 50 | 51 | // Constructs a capture mode which logs basic events, and additionally makes 52 | // no effort to strip cookies and credentials. 53 | // include_cookies_and_credentials() --> true 54 | // include_socket_bytes() --> false 55 | // TODO(bnc): Consider renaming to IncludePrivacyInfo(). 56 | static NetLogCaptureMode IncludeCookiesAndCredentials(); 57 | 58 | // Constructs a capture mode which logs the data sent/received from sockets. 59 | // include_cookies_and_credentials() --> true 60 | // include_socket_bytes() --> true 61 | static NetLogCaptureMode IncludeSocketBytes(); 62 | 63 | // If include_cookies_and_credentials() is true , then it is OK to log 64 | // events which contain cookies, credentials or other privacy sensitive data. 65 | // TODO(bnc): Consider renaming to include_privacy_info(). 66 | bool include_cookies_and_credentials() const; 67 | 68 | // If include_socket_bytes() is true, then it is OK to output the actual 69 | // bytes read/written from the network, even if it contains private data. 70 | bool include_socket_bytes() const; 71 | 72 | 73 | std::unique_ptr HttpRequestHeaders::NetLogCallback( 74 | const std::string* request_line, 75 | NetLogCaptureMode capture_mode) const { 76 | auto dict = std::make_unique(); 77 | dict->SetKey("line", NetLogStringValue(*request_line)); 78 | auto headers = std::make_unique(); 79 | for (auto it = headers_.begin(); it != headers_.end(); ++it) { 80 | std::string log_value = 81 | ElideHeaderValueForNetLog(capture_mode, it->key, it->value); 82 | headers->GetList().push_back( 83 | NetLogStringValue(base::StrCat({it->key, ": ", log_value}))); 84 | } 85 | dict->Set("headers", std::move(headers)); 86 | return std::move(dict); 87 | } 88 | 89 | 90 | 91 | std::string ElideHeaderValueForNetLog(NetLogCaptureMode capture_mode, 92 | const std::string& header, 93 | const std::string& value) { 94 | std::string::const_iterator redact_begin = value.begin(); 95 | std::string::const_iterator redact_end = value.begin(); 96 | 97 | if (redact_begin == redact_end && 98 | !capture_mode.include_cookies_and_credentials()) { 99 | if (base::EqualsCaseInsensitiveASCII(header, "set-cookie") || 100 | base::EqualsCaseInsensitiveASCII(header, "set-cookie2") || 101 | base::EqualsCaseInsensitiveASCII(header, "cookie") || 102 | base::EqualsCaseInsensitiveASCII(header, "authorization") || 103 | base::EqualsCaseInsensitiveASCII(header, "proxy-authorization")) { 104 | redact_begin = value.begin(); 105 | redact_end = value.end(); 106 | } else if (base::EqualsCaseInsensitiveASCII(header, "www-authenticate") || 107 | base::EqualsCaseInsensitiveASCII(header, "proxy-authenticate")) { 108 | // Look for authentication information from data received from the server 109 | // in multi-round Negotiate authentication. 110 | HttpAuthChallengeTokenizer challenge(value.begin(), value.end()); 111 | if (ShouldRedactChallenge(&challenge)) { 112 | redact_begin = challenge.params_begin(); 113 | redact_end = challenge.params_end(); 114 | } 115 | } 116 | } 117 | 118 | if (redact_begin == redact_end) 119 | return value; 120 | 121 | return std::string(value.begin(), redact_begin) + 122 | base::StringPrintf("[%ld bytes were stripped]", 123 | static_cast(redact_end - redact_begin)) + 124 | std::string(redact_end, value.end()); 125 | } 126 | -------------------------------------------------------------------------------- /FiddlerImportNetlog/JSON.cs: -------------------------------------------------------------------------------- 1 | /// 2 | /// Originally based on http://techblog.procurios.nl/k/618/news/view/14605/14863/How-do-I-write-my-own-parser-for-JSON.html 3 | /// Licensed under the http://www.opensource.org/licenses/mit-license.php by Patrick van Bergen. 4 | /// Modified based on real-world experience. 5 | /// 6 | using Fiddler; 7 | using System; 8 | using System.Collections; 9 | using System.Diagnostics; 10 | using System.Globalization; 11 | using System.IO; 12 | using System.Text; 13 | 14 | namespace FiddlerImportNetlog.WebFormats 15 | { 16 | /// 17 | /// This class encodes and decodes JSON text http://www.json.org/ 18 | /// JSON uses Arrays and Objects, similar to DotNet datatypes ArrayList and Hashtable. 19 | /// All numbers are parsed to doubles. 20 | /// 21 | public class JSON 22 | { 23 | /// 24 | /// Wrapper around JSON Parse output; needed because JScript.NET does not support OUT Parameters 25 | /// 26 | public class JSONParseResult 27 | { 28 | public object JSONObject 29 | { 30 | get; 31 | set; 32 | } 33 | public JSONParseErrors JSONErrors 34 | { 35 | get; 36 | set; 37 | } 38 | } 39 | 40 | public class JSONParseErrors 41 | { 42 | /// 43 | /// Index of the last error in the stream 44 | /// 45 | public int iErrorIndex { get; set; } 46 | 47 | /// 48 | /// Warnings found while parsing JSON 49 | /// 50 | public string sWarningText { get; set; } 51 | } 52 | 53 | internal enum JSONTokens : byte 54 | { 55 | NONE = 0, 56 | CURLY_OPEN = 1, 57 | CURLY_CLOSE = 2, 58 | SQUARED_OPEN = 3, 59 | SQUARED_CLOSE = 4, 60 | COLON = 5, 61 | COMMA = 6, 62 | STRING = 7, 63 | NUMBER = 8, 64 | TRUE = 9, 65 | FALSE = 10, 66 | NULL = 11, 67 | /// 68 | /// This token is a JavaScript Identifier that was not properly quoted as it should have been. 69 | /// 70 | IMPLIED_IDENTIFIER_NAME = 12 71 | // Note: We don't allow Regular Expressions in JSON as they are not allowed per-spec 72 | } 73 | 74 | private const int BUILDER_DEFAULT_CAPACITY = 2048; 75 | 76 | // This result type should have been JSONParseResult. Probably can't change it now. Sigh. 77 | public static object JsonDecode(string sJSON) 78 | { 79 | JSONParseResult oResult = new JSONParseResult(); 80 | JSONParseErrors oErrors; 81 | oResult.JSONObject = JsonDecode(sJSON, out oErrors); 82 | oResult.JSONErrors = oErrors; 83 | return oResult; 84 | } 85 | 86 | /// 87 | /// Parses the JSON string into an object 88 | /// 89 | /// A JSON string. 90 | /// An ArrayList, a Hashtable, a double, a string, null, true, or false 91 | public static object JsonDecode(string sJSON, out JSONParseErrors oErrors) 92 | { 93 | oErrors = new JSONParseErrors() { iErrorIndex = -1, sWarningText = String.Empty }; 94 | if (!String.IsNullOrEmpty(sJSON)) 95 | { 96 | char[] charArray = sJSON.ToCharArray(); 97 | int index = 0; 98 | bool success = true; 99 | object value = ParseValue(charArray, ref index, ref success, ref oErrors); 100 | return value; 101 | } 102 | else 103 | { 104 | return null; 105 | } 106 | } 107 | 108 | private static Hashtable ParseObject(char[] json, ref int index, ref JSONParseErrors oErrors) 109 | { 110 | Hashtable htThisObject = new Hashtable(); 111 | JSONTokens jtToken; 112 | 113 | NextToken(json, ref index); 114 | 115 | bool done = false; 116 | while (!done) 117 | { 118 | jtToken = LookAhead(json, index); 119 | if (jtToken == JSONTokens.NONE) 120 | { 121 | return null; 122 | } 123 | else if (jtToken == JSONTokens.COMMA) 124 | { 125 | NextToken(json, ref index); 126 | } 127 | else if (jtToken == JSONTokens.CURLY_CLOSE) 128 | { 129 | NextToken(json, ref index); 130 | return htThisObject; 131 | } 132 | else 133 | { 134 | // We *should* be looking at a quoted identifier name here. Some non-compliant JSON authors omit 135 | // the wrapping quote marks, so we accommodate that case first. 136 | string sName = null; 137 | 138 | if (jtToken == JSONTokens.IMPLIED_IDENTIFIER_NAME) 139 | { 140 | sName = ParseUnquotedIdentifier(json, ref index, ref oErrors); 141 | if (null == sName) 142 | { 143 | if (oErrors.iErrorIndex < 0) oErrors.iErrorIndex = index; 144 | return null; 145 | } 146 | } 147 | else 148 | { 149 | Debug.Assert(jtToken == JSONTokens.STRING, "Unexpected Token type; expecting String/Identifier"); 150 | sName = ParseString(json, ref index); 151 | if (null == sName) 152 | { 153 | if (oErrors.iErrorIndex < 0) oErrors.iErrorIndex = index; 154 | return null; 155 | } 156 | } 157 | 158 | // Colon delimits the name from its value 159 | jtToken = NextToken(json, ref index); 160 | if (jtToken != JSONTokens.COLON) 161 | { 162 | if (oErrors.iErrorIndex < 0) oErrors.iErrorIndex = index; 163 | return null; 164 | } 165 | 166 | // value 167 | bool success = true; 168 | object value = ParseValue(json, ref index, ref success, ref oErrors); 169 | if (!success) 170 | { 171 | Debug.Assert(false); 172 | oErrors.iErrorIndex = index; 173 | return null; 174 | } 175 | 176 | htThisObject[sName] = value; 177 | } 178 | } 179 | 180 | return htThisObject; 181 | } 182 | 183 | /// 184 | /// 185 | /// 186 | /// 187 | /// 188 | /// 189 | private static ArrayList ParseArray(char[] json, ref int index, ref JSONParseErrors oErrors) 190 | { 191 | ArrayList alThisArray = new ArrayList(); 192 | 193 | // [ 194 | NextToken(json, ref index); 195 | 196 | bool done = false; 197 | while (!done) 198 | { 199 | JSONTokens jtToken = LookAhead(json, index); 200 | if (jtToken == JSONTokens.NONE) 201 | { 202 | if (oErrors.iErrorIndex < 0) oErrors.iErrorIndex = index; 203 | return null; 204 | } 205 | else if (jtToken == JSONTokens.COMMA) 206 | { 207 | NextToken(json, ref index); 208 | } 209 | else if (jtToken == JSONTokens.SQUARED_CLOSE) 210 | { 211 | NextToken(json, ref index); 212 | break; 213 | } 214 | else 215 | { 216 | bool success = true; 217 | object value = ParseValue(json, ref index, ref success, ref oErrors); 218 | if (!success) 219 | { 220 | if (oErrors.iErrorIndex < 0) oErrors.iErrorIndex = index; 221 | return null; 222 | } 223 | 224 | alThisArray.Add(value); 225 | } 226 | } 227 | 228 | return alThisArray; 229 | } 230 | 231 | private static object ParseValue(char[] json, ref int index, ref bool success, ref JSONParseErrors oErrors) 232 | { 233 | switch (LookAhead(json, index)) 234 | { 235 | case JSONTokens.IMPLIED_IDENTIFIER_NAME: 236 | return ParseUnquotedIdentifier(json, ref index, ref oErrors); 237 | 238 | case JSONTokens.STRING: 239 | return ParseString(json, ref index); 240 | 241 | case JSONTokens.NUMBER: 242 | return ParseNumber(json, ref index); 243 | 244 | case JSONTokens.CURLY_OPEN: 245 | return ParseObject(json, ref index, ref oErrors); 246 | 247 | case JSONTokens.SQUARED_OPEN: 248 | return ParseArray(json, ref index, ref oErrors); 249 | 250 | case JSONTokens.TRUE: 251 | NextToken(json, ref index); 252 | return true; 253 | 254 | case JSONTokens.FALSE: 255 | NextToken(json, ref index); 256 | return false; 257 | 258 | case JSONTokens.NULL: 259 | NextToken(json, ref index); 260 | return null; 261 | 262 | case JSONTokens.NONE: 263 | break; 264 | } 265 | 266 | success = false; 267 | return null; 268 | } 269 | 270 | /// 271 | /// Similar to ParseString, but exits at the first whitespace or non-identifier character 272 | /// 273 | /// 274 | /// 275 | /// 276 | private static string ParseUnquotedIdentifier(char[] json, ref int index, ref JSONParseErrors oErrors) 277 | { 278 | EatWhitespace(json, ref index); 279 | 280 | int ixStart = index; 281 | StringBuilder s = new StringBuilder(BUILDER_DEFAULT_CAPACITY); 282 | char c; 283 | 284 | bool complete = false; 285 | while (!complete) 286 | { 287 | if (index == json.Length) 288 | { 289 | break; 290 | } 291 | 292 | c = json[index]; 293 | if (!isValidIdentifierChar(c)) 294 | { 295 | if (s.Length < 1) return null; 296 | complete = true; 297 | break; 298 | } 299 | else 300 | { 301 | s.Append(c); 302 | } 303 | 304 | ++index; 305 | } 306 | 307 | if (!complete) 308 | { 309 | return null; 310 | } 311 | 312 | oErrors.sWarningText = string.Format("{0}Illegal/Unquoted identifier '{1}' at position {2}.\n", oErrors.sWarningText, 313 | s.ToString(), 314 | ixStart); 315 | 316 | return s.ToString(); 317 | } 318 | 319 | private static string ParseString(char[] json, ref int index) 320 | { 321 | StringBuilder s = new StringBuilder(BUILDER_DEFAULT_CAPACITY); 322 | char c; 323 | 324 | EatWhitespace(json, ref index); 325 | 326 | // Eat the opening quote 327 | index++; 328 | 329 | bool complete = false; 330 | while (!complete) { 331 | 332 | if (index == json.Length) { 333 | break; 334 | } 335 | 336 | c = json[index++]; 337 | if (c == '"') { 338 | complete = true; 339 | break; 340 | } else if (c == '\\') { 341 | 342 | if (index == json.Length) { 343 | break; 344 | } 345 | c = json[index++]; 346 | if (c == '"') { 347 | s.Append('"'); 348 | } else if (c == '\\') { 349 | s.Append('\\'); 350 | } else if (c == '/') { 351 | s.Append('/'); 352 | } else if (c == 'b') { 353 | s.Append('\b'); 354 | } else if (c == 'f') { 355 | s.Append('\f'); 356 | } else if (c == 'n') { 357 | s.Append('\n'); 358 | } else if (c == 'r') { 359 | s.Append('\r'); 360 | } else if (c == 't') { 361 | s.Append('\t'); 362 | } else if (c == 'u') { 363 | int remainingLength = json.Length - index; 364 | if (remainingLength >= 4) { 365 | uint codePoint = UInt32.Parse(new string(json, index, 4), NumberStyles.HexNumber); 366 | // convert the integer codepoint to a unicode char and add to string 367 | 368 | string sChar; 369 | try 370 | { 371 | sChar = Char.ConvertFromUtf32((int)codePoint); 372 | } 373 | catch (Exception eX) 374 | { 375 | Debug.WriteLine("JSONConvert failed: " + eX.Message); 376 | // If character conversion fails, use the Unicode Replacement character. 377 | sChar = "\uFFFD"; 378 | } 379 | 380 | s.Append(sChar); 381 | // skip 4 chars 382 | index += 4; 383 | } else { 384 | break; 385 | } 386 | } 387 | } else { 388 | s.Append(c); 389 | } 390 | 391 | } 392 | 393 | if (!complete) 394 | { 395 | return null; 396 | } 397 | 398 | return s.ToString(); 399 | } 400 | 401 | private static double ParseNumber(char[] json, ref int index) 402 | { 403 | EatWhitespace(json, ref index); 404 | 405 | int lastIndex = GetLastIndexOfNumber(json, index); 406 | int charLength = (lastIndex - index) + 1; 407 | 408 | string sNumber = new String(json, index, charLength); 409 | 410 | index = lastIndex + 1; 411 | return Double.Parse(sNumber, CultureInfo.InvariantCulture); 412 | } 413 | 414 | /// 415 | /// Gets the character index of the end of the JavaScript number we're currently parsing. 416 | /// Numbers are tricky because of exponential notation, etc. 417 | /// 418 | /// 419 | /// 420 | /// 421 | private static int GetLastIndexOfNumber(char[] json, int index) 422 | { 423 | int lastIndex; 424 | for (lastIndex = index; lastIndex < json.Length; lastIndex++) 425 | { 426 | if ("0123456789+-.eE".IndexOf(json[lastIndex]) == -1) 427 | { 428 | break; 429 | } 430 | } 431 | return lastIndex - 1; 432 | } 433 | 434 | private static void EatWhitespace(char[] json, ref int index) 435 | { 436 | for (; index < json.Length; index++) 437 | { 438 | if (" \t\n\r".IndexOf(json[index]) == -1) 439 | { 440 | break; 441 | } 442 | } 443 | } 444 | 445 | private static JSONTokens LookAhead(char[] json, int index) 446 | { 447 | int saveIndex = index; 448 | return NextToken(json, ref saveIndex); 449 | } 450 | 451 | private static JSONTokens NextToken(char[] json, ref int index) 452 | { 453 | EatWhitespace(json, ref index); 454 | 455 | if (index == json.Length) 456 | { 457 | return JSONTokens.NONE; 458 | } 459 | 460 | char c = json[index]; 461 | ++index; 462 | switch (c) 463 | { 464 | case '{': 465 | return JSONTokens.CURLY_OPEN; 466 | case '}': 467 | return JSONTokens.CURLY_CLOSE; 468 | case '[': 469 | return JSONTokens.SQUARED_OPEN; 470 | case ']': 471 | return JSONTokens.SQUARED_CLOSE; 472 | case ',': 473 | return JSONTokens.COMMA; 474 | case '"': 475 | return JSONTokens.STRING; 476 | case '0': case '1': case '2': case '3': case '4': 477 | case '5': case '6': case '7': case '8': case '9': 478 | case '-': 479 | return JSONTokens.NUMBER; 480 | case ':': 481 | return JSONTokens.COLON; 482 | } 483 | --index; 484 | 485 | int remainingLength = json.Length - index; 486 | 487 | if (remainingLength >= 5) { 488 | if (json[index] == 'f' && 489 | json[index + 1] == 'a' && 490 | json[index + 2] == 'l' && 491 | json[index + 3] == 's' && 492 | json[index + 4] == 'e') 493 | { 494 | index += 5; 495 | return JSONTokens.FALSE; 496 | } 497 | } 498 | 499 | if (remainingLength >= 4) { 500 | if (json[index] == 't' && 501 | json[index + 1] == 'r' && 502 | json[index + 2] == 'u' && 503 | json[index + 3] == 'e') 504 | { 505 | index += 4; 506 | return JSONTokens.TRUE; 507 | } 508 | } 509 | 510 | if (remainingLength >= 4) 511 | { 512 | if (json[index] == 'n' && 513 | json[index + 1] == 'u' && 514 | json[index + 2] == 'l' && 515 | json[index + 3] == 'l') 516 | { 517 | index += 4; 518 | return JSONTokens.NULL; 519 | } 520 | } 521 | 522 | // 523 | // At this point, we can check to see if this is a simple ASCII text string and 524 | // if so, return JSON.TOKEN_IMPLIED_IDENTIFIER which basically behaves like JSON.TOKEN_STRING 525 | // 526 | if (isValidIdentifierStart(json[index])) 527 | { 528 | return JSONTokens.IMPLIED_IDENTIFIER_NAME; 529 | } 530 | 531 | return JSONTokens.NONE; 532 | } 533 | 534 | /// 535 | /// Returns TRUE if this character might be the valid start of a JavaScript variable identifier 536 | /// 537 | /// 538 | /// TODO: See http://stackoverflow.com/questions/1661197/valid-characters-for-javascript-variable-names to see how 539 | /// this method doesn't account for all valid variable names, just the most reasonable ones 540 | /// The character to test 541 | /// TRUE if the character may start a JavaScript identifier 542 | private static bool isValidIdentifierStart(char c) 543 | { 544 | if ((c == '_') || (c == '$')) return true; 545 | if ((c == '\'')) return true; // Single-quoting on variable names is illegal but we WARN 546 | if (char.IsLetter(c)) return true; 547 | return false; 548 | } 549 | 550 | /// 551 | /// Returns TRUE if this character may validly appear within a JavaScript variable identifier. 552 | /// TODO: See comment on isValidIdentifierStart() for details about how this is incomplete 553 | /// 554 | /// The character to test 555 | /// TRUE if the character may appear within a JavaScript identifier 556 | private static bool isValidIdentifierChar(char c) 557 | { 558 | if ((c == '-') || (c == '_') || (c == '$')) return true; 559 | if (char.IsLetterOrDigit(c)) return true; 560 | if ((c == '\'')) return true; // Single-quoting on variable names is illegal but we WARN 561 | return false; 562 | } 563 | 564 | /// 565 | /// Converts an IDictionary or IList object graph into a JSON string 566 | /// 567 | /// A Hashtable or ArrayList 568 | /// A JSON encoded string, or null if object 'json' is not serializable 569 | public static string JsonEncode(object json) 570 | { 571 | StringBuilder builder = new StringBuilder(BUILDER_DEFAULT_CAPACITY); 572 | bool success = JSON.SerializeValue(json, builder); 573 | return (success ? builder.ToString() : null); 574 | } 575 | 576 | private static bool SerializeObject(IDictionary anObject, StringBuilder builder) 577 | { 578 | builder.Append("{"); 579 | 580 | IDictionaryEnumerator e = anObject.GetEnumerator(); 581 | bool first = true; 582 | while (e.MoveNext()) 583 | { 584 | string key = e.Key.ToString(); 585 | object value = e.Value; 586 | 587 | if (!first) { 588 | builder.Append(", "); 589 | } 590 | 591 | SerializeString(key, builder); 592 | builder.Append(":"); 593 | if (!SerializeValue(value, builder)) 594 | { 595 | return false; 596 | } 597 | 598 | first = false; 599 | } 600 | 601 | builder.Append("}"); 602 | return true; 603 | } 604 | 605 | private static bool SerializeArray(IList anArray, StringBuilder builder) 606 | { 607 | builder.Append("["); 608 | 609 | bool first = true; 610 | for (int i = 0; i < anArray.Count; i++) 611 | { 612 | object value = anArray[i]; 613 | 614 | if (!first) 615 | { 616 | builder.Append(", "); 617 | } 618 | 619 | if (!SerializeValue(value, builder)) 620 | { 621 | return false; 622 | } 623 | 624 | first = false; 625 | } 626 | 627 | builder.Append("]"); 628 | return true; 629 | } 630 | 631 | private static bool SerializeValue(object value, StringBuilder builder) 632 | { 633 | if (null == value) 634 | { 635 | builder.Append("null"); 636 | } else if (value is string) { 637 | SerializeString((string)value, builder); 638 | } else if (value is Hashtable) { 639 | SerializeObject((Hashtable)value, builder); 640 | } else if (value is ArrayList) { 641 | SerializeArray((ArrayList)value, builder); 642 | } else if (IsNumeric(value)) { 643 | SerializeNumber(Convert.ToDouble(value), builder); 644 | } else if ((value is Boolean) && ((Boolean)value == true)) { 645 | builder.Append("true"); 646 | } else if ((value is Boolean) && ((Boolean)value == false)) { 647 | builder.Append("false"); 648 | } 649 | else 650 | { 651 | return false; 652 | } 653 | return true; 654 | } 655 | 656 | private static void SerializeString(string aString, StringBuilder builder) 657 | { 658 | builder.Append("\""); 659 | 660 | char[] charArray = aString.ToCharArray(); 661 | for (int i = 0; i < charArray.Length; i++) 662 | { 663 | switch(charArray[i]) 664 | { 665 | case '"': 666 | builder.Append("\\\""); 667 | break; 668 | case '\\': 669 | builder.Append(@"\\"); 670 | break; 671 | 672 | case '\b': 673 | builder.Append(@"\b"); 674 | break; 675 | 676 | case '\f': 677 | builder.Append(@"\f"); 678 | break; 679 | 680 | case '\n': 681 | builder.Append(@"\n"); 682 | break; 683 | 684 | case '\r': 685 | builder.Append(@"\r"); 686 | break; 687 | 688 | case '\t': 689 | builder.Append(@"\t"); 690 | break; 691 | 692 | default: 693 | char c = charArray[i]; 694 | int codepoint = Convert.ToInt32(c); 695 | if ((codepoint >= 32) && (codepoint <= 126)) 696 | { 697 | builder.Append(c); 698 | } 699 | else 700 | { 701 | builder.Append("\\u" + codepoint.ToString("x").PadLeft(4, '0')); 702 | } 703 | break; 704 | } 705 | } 706 | builder.Append("\""); 707 | } 708 | 709 | private static void SerializeNumber(double number, StringBuilder builder) 710 | { 711 | builder.Append(Convert.ToString(number, CultureInfo.InvariantCulture)); 712 | } 713 | 714 | /// 715 | /// Determines if a given object is numeric in any way 716 | /// 717 | private static bool IsNumeric(object o) 718 | { 719 | try 720 | { 721 | Double.Parse(o.ToString()); 722 | } 723 | catch (Exception) 724 | { 725 | return false; 726 | } 727 | return true; 728 | } 729 | } 730 | } -------------------------------------------------------------------------------- /FiddlerImportNetlog/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.InteropServices; 3 | 4 | [assembly: AssemblyTitle("FiddlerImportNetlog")] 5 | [assembly: AssemblyDescription("Import Chromium NetLog events to Fiddler")] 6 | [assembly: AssemblyCopyright("Copyright ©2025 Eric Lawrence")] 7 | [assembly: System.Resources.NeutralResourcesLanguage("en-US")] 8 | [assembly: ComVisible(false)] 9 | [assembly: AssemblyVersion("1.3.7.0")] // ALWAYS UPDATE THE VERSION in the [ProfferFormat] attribute in FiddlerInterface.cs to match! 10 | [assembly: Fiddler.RequiredVersion("4.6.0.0")] 11 | 12 | 13 | /* 14 | TODO: 15 | HTTP_STREAM_JOB has a binding between the request and the socket. Hook them up so we can propagate the connection info to the URL_REQUEST-generated Sessions. 16 | 17 | t=3262 [st=0] SOCKET_POOL_BOUND_TO_SOCKET 18 | --> source_dependency = 1250 (SOCKET) 19 | t=3262 [st=0] HTTP_STREAM_JOB_BOUND_TO_REQUEST 20 | --> source_dependency = 1701 (URL_REQUEST) 21 | */ 22 | 23 | // v1.3.7 24 | // Workaround Telerik breaking Fiddler.WebFormats 25 | 26 | // v1.3.6.0 27 | // Support "expire" for cookies (added in Chrome 134) 28 | // Update copyright to 2025 29 | 30 | // v1.3.5.1 31 | // Add support for truncated file recovery. 32 | 33 | // v1.3.4.5 34 | // Set timers for pseudo sessions to try to fix timeline. TODO: Not sure I'm happy with how this works right now!!!!!!!!! 35 | 36 | // v1.3.4.4 37 | // Add lightweight breakout of server certinfo 38 | 39 | // v1.3.4.3 40 | // Set oTimers' values for ClientBeginResponse, ClientDoneResponse, and ServerDoneResponse so Timeline view works better. 41 | 42 | // v1.3.4.2 43 | // When renaming Transfer-Encoding/Chunk-Encoding headers, set Content-Length to enhance AutoResponder playback 44 | // Update copyright to 2023 45 | 46 | // v1.3.4.1 47 | // Fix parsing of TLS/1.3 sigscheme list 48 | 49 | // v1.3.4.0 50 | // Parse certificaterequest message properly on TLS/1.3 connections 51 | // Add smartscreen to net annotations 52 | 53 | // v1.3.3.0 54 | // Add ClientHello and ServerHello to SecureSocket list 55 | 56 | // v1.3.2.3 57 | // Add more traffic_annotation values 58 | 59 | // v1.3.2.2 60 | // Add ".json;.gz" hint to ProfferFormat registration 61 | 62 | // v1.3.2.1 63 | // Add DNS entries to log 64 | // Add READ_EARLY_HINTS_RESPONSE_HEADERS - _X-NetLog-Found-Early-Hint - https://www.fastly.com/blog/beyond-server-push-experimenting-with-the-103-early-hints-status-code 65 | 66 | // v1.3.1.0 67 | // Support for FAKE_RESPONSE_HEADERS_CREATED for HSTS and Automatic HTTPS upgrades 68 | // Add socket address info to generated SOCKETS list's session 69 | 70 | // v1.3.0.1 71 | // Less Log spam 72 | // Write imported filename to log 73 | 74 | // v1.3 75 | // Support importing NetLog events from a Chromium trace json file 76 | 77 | // v1.2.7 78 | // Flag failed Set-Cookies in Web Sessions list 79 | 80 | // v1.2.6 81 | // Record X-Netlog-URLRequest-ID and X-ProcessInfo, even when we don't receive a StartRequest 82 | // Add mappings for most common traffic_annotation values when writing X-Netlog-Traffic_Annotation 83 | 84 | // v1.2.5 85 | // Record sensitivity level 86 | 87 | // v1.2.4 88 | // Better parse HTTP Auth where there are multiple SendRequests 89 | 90 | // v1.2.3 91 | // Add |traffic_annotation| to session properties 92 | 93 | // v1.2.2 94 | // Update Cookie Inclusion reasons to match latest CL 81.0.3993 https://chromium-review.googlesource.com/c/chromium/src/+/1960865 95 | 96 | // v1.2.1 97 | // Add Cookie Exclusion warnings 98 | 99 | // v1.2 100 | // Parse CertificateRequest TLS Handshake message and SSL_HANDSHAKE_MESSAGE_RECEIVED. 101 | 102 | // TODO: Surface the CN for each certificate from the server 103 | /* TODO: Parse out messages indicating whether the client sent a cert, and what that cert was. 104 | 105 | t= 7906 [st= 580] SSL_CLIENT_CERT_PROVIDED 106 | --> cert_count = 2 107 | t= 7906 [st= 580] SSL_HANDSHAKE_MESSAGE_SENT 108 | --> bytes = 109 | 0B 00 0D C0 00 0D BD 00 06 B4 30 82 06 B0 30 82 . .. .. ..0...0. 110 | 04 98 A0 03 02 01 02 02 13 1C 00 4A 4D 7F 50 4B .......... JM.PK 111 | 6F 8E 33 AB 1B 04 00 01 00 4A 4D 7F 30 0D 06 09 o.3... . JM.0... 112 | 2A 86 48 86 F7 0D 01 01 0B 05 00 30 15 31 13 30 *.H....... 0.1.0 113 | ... 114 | 4D 86 43 E1 23 A0 F9 B7 4F AF 84 AF 48 EC D5 F8 M.C.#...O...H... 115 | DE 4A BD 6B A7 FB 3E 5E 3E E7 8E 11 64 96 2D EB .J.k..>^>...d.-. 116 | 69 0A C8 2C i.., 117 | --> type = 11 118 | 119 | */ 120 | 121 | // v1.1.2 122 | // Support ZIP compressed JSON logs 123 | 124 | // v1.1.1.2 125 | // Support .gz compressed JSON logs 126 | 127 | // v1.1.1.1 128 | // Correct rename of Transfer-Encoding and Content-Encoding response headers 129 | 130 | // v1.1.1 131 | // Better exception handling around debugtree creation 132 | // Publish as open source on GitHub. 133 | 134 | // v1.0.0.1 135 | // Change installer to not require Admin 136 | // Handle logs where htConstants["timeTickOffset"] is a string rather than a long 137 | 138 | // v1.0.1.0 139 | // Improve exception handling 140 | // Handle cases where headers are missing 141 | // Handle cases where headers are encoded as a JSObject instead of a JSArray 142 | // Rename Content-Encoding header to avoid confusion 143 | 144 | // v1.0.2.0 145 | // Basic support for timers 146 | 147 | // v1.0.3.0 148 | // Handle captures that are missing polledData (e.g. extensions) because capture was created at Browser Startup 149 | 150 | // v1.0.4.0 151 | // Reduce progress notification spew 152 | 153 | // v1.1 154 | // Cleanup code 155 | // Support multiple sessions per URL_REQUEST entry (e.g. on redirection) 156 | 157 | 158 | 159 | 160 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c)2019 Eric Lawrence 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | 3. Neither the name of the copyright holder nor the names of its 16 | contributors may be used to endorse or promote products derived from 17 | this software without specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 26 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # FiddlerImportNetlog 2 | 3 | I wrote a [Blog Post](https://www.telerik.com/blogs/building-fiddler-importers) that explains this extension. 4 | 5 | It allows you to import [Chromium NetLog](https://www.chromium.org/developers/design-documents/network-stack/netlog) traffic captures into Fiddler. 6 | 7 | Note that the NetLog format is somewhat lossy (e.g. request body bytes are never included, and credentials and response bodies *may* be excluded) so full-fidelity import is not generally possible. 8 | 9 | If you'd just like to add this importer without building it yourself, you can [Download it from the Releases page](https://github.com/ericlaw1979/FiddlerImportNetlog/releases/) 10 | -------------------------------------------------------------------------------- /installer/Addon.ver: -------------------------------------------------------------------------------- 1 | 1.3.7.0 2 | -------------------------------------------------------------------------------- /installer/FiddlerImportNetLog.nsi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ericlaw1979/FiddlerImportNetlog/1a927f4fe1519d681b112c5b02c6a1cc20142fb9/installer/FiddlerImportNetLog.nsi -------------------------------------------------------------------------------- /installer/addon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ericlaw1979/FiddlerImportNetlog/1a927f4fe1519d681b112c5b02c6a1cc20142fb9/installer/addon.ico -------------------------------------------------------------------------------- /installer/go.bat: -------------------------------------------------------------------------------- 1 | @title FiddlerImportNetLog Builder 2 | @cd C:\src\FiddlerImportNetLog\installer 3 | @C:\tools\signtool sign /as /d "Fiddler NetLog Importer" /du "https://textslashplain.com/" /n "Eric Lawrence" /tr http://timestamp.digicert.com /td SHA256 /fd SHA256 C:\src\FiddlerImportNetLog\FiddlerImportNetLog\bin\release\FiddlerImportNetLog.dll 4 | @filever C:\src\FiddlerImportNetLog\FiddlerImportNetLog\bin\release\FiddlerImportNetLog.dll > Addon.ver 5 | @C:\src\NSIS\MakeNSIS.EXE /V3 FiddlerImportNetLog.nsi 6 | @CHOICE /M "Would you like to sign?" 7 | @if %ERRORLEVEL%==2 goto done 8 | :sign 9 | @C:\tools\signtool sign /as /d "Fiddler NetLog Importer" /du "https://textslashplain.com/" /n "Eric Lawrence" /tr http://timestamp.digicert.com /td SHA256 /fd SHA256 FiddlerImportNetLog.exe 10 | @if %ERRORLEVEL%==-1 goto sign 11 | @:upload 12 | @:done 13 | @title Command Prompt --------------------------------------------------------------------------------