├── .gitattributes ├── .gitignore ├── .idea └── .idea.iisGeolocate │ └── .idea │ ├── .gitignore │ ├── encodings.xml │ ├── indexLayout.xml │ └── vcs.xml ├── LICENSE ├── README.md ├── iisGeolocate.saproj ├── iisGeolocate.sln └── iisGeolocate ├── FodyWeavers.xml ├── FodyWeavers.xsd ├── GlobeInfo.ico ├── Program.cs ├── Properties └── AssemblyInfo.cs ├── SampleIISLog.log └── iisGeolocate.csproj /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.userosscache 8 | *.sln.docstates 9 | 10 | # User-specific files (MonoDevelop/Xamarin Studio) 11 | *.userprefs 12 | 13 | # Build results 14 | [Dd]ebug/ 15 | [Dd]ebugPublic/ 16 | [Rr]elease/ 17 | [Rr]eleases/ 18 | [Xx]64/ 19 | [Xx]86/ 20 | [Bb]uild/ 21 | bld/ 22 | [Bb]in/ 23 | [Oo]bj/ 24 | 25 | # Visual Studio 2015 cache/options directory 26 | .vs/ 27 | # Uncomment if you have tasks that create the project's static files in wwwroot 28 | #wwwroot/ 29 | 30 | # MSTest test Results 31 | [Tt]est[Rr]esult*/ 32 | [Bb]uild[Ll]og.* 33 | 34 | # NUNIT 35 | *.VisualState.xml 36 | TestResult.xml 37 | 38 | # Build Results of an ATL Project 39 | [Dd]ebugPS/ 40 | [Rr]eleasePS/ 41 | dlldata.c 42 | 43 | # DNX 44 | project.lock.json 45 | artifacts/ 46 | 47 | *_i.c 48 | *_p.c 49 | *_i.h 50 | *.ilk 51 | *.meta 52 | *.obj 53 | *.pch 54 | *.pdb 55 | *.pgc 56 | *.pgd 57 | *.rsp 58 | *.sbr 59 | *.tlb 60 | *.tli 61 | *.tlh 62 | *.tmp 63 | *.tmp_proj 64 | *.log 65 | *.vspscc 66 | *.vssscc 67 | .builds 68 | *.pidb 69 | *.svclog 70 | *.scc 71 | 72 | # Chutzpah Test files 73 | _Chutzpah* 74 | 75 | # Visual C++ cache files 76 | ipch/ 77 | *.aps 78 | *.ncb 79 | *.opendb 80 | *.opensdf 81 | *.sdf 82 | *.cachefile 83 | *.VC.db 84 | 85 | # Visual Studio profiler 86 | *.psess 87 | *.vsp 88 | *.vspx 89 | *.sap 90 | 91 | # TFS 2012 Local Workspace 92 | $tf/ 93 | 94 | # Guidance Automation Toolkit 95 | *.gpState 96 | 97 | # ReSharper is a .NET coding add-in 98 | _ReSharper*/ 99 | *.[Rr]e[Ss]harper 100 | *.DotSettings.user 101 | 102 | # JustCode is a .NET coding add-in 103 | .JustCode 104 | 105 | # TeamCity is a build add-in 106 | _TeamCity* 107 | 108 | # DotCover is a Code Coverage Tool 109 | *.dotCover 110 | 111 | # NCrunch 112 | _NCrunch_* 113 | .*crunch*.local.xml 114 | nCrunchTemp_* 115 | 116 | # MightyMoose 117 | *.mm.* 118 | AutoTest.Net/ 119 | 120 | # Web workbench (sass) 121 | .sass-cache/ 122 | 123 | # Installshield output folder 124 | [Ee]xpress/ 125 | 126 | # DocProject is a documentation generator add-in 127 | DocProject/buildhelp/ 128 | DocProject/Help/*.HxT 129 | DocProject/Help/*.HxC 130 | DocProject/Help/*.hhc 131 | DocProject/Help/*.hhk 132 | DocProject/Help/*.hhp 133 | DocProject/Help/Html2 134 | DocProject/Help/html 135 | 136 | # Click-Once directory 137 | publish/ 138 | 139 | # Publish Web Output 140 | *.[Pp]ublish.xml 141 | *.azurePubxml 142 | 143 | # TODO: Un-comment the next line if you do not want to checkin 144 | # your web deploy settings because they may include unencrypted 145 | # passwords 146 | #*.pubxml 147 | *.publishproj 148 | 149 | # NuGet Packages 150 | *.nupkg 151 | # The packages folder can be ignored because of Package Restore 152 | **/packages/* 153 | # except build/, which is used as an MSBuild target. 154 | !**/packages/build/ 155 | # Uncomment if necessary however generally it will be regenerated when needed 156 | #!**/packages/repositories.config 157 | # NuGet v3's project.json files produces more ignoreable files 158 | *.nuget.props 159 | *.nuget.targets 160 | 161 | # Microsoft Azure Build Output 162 | csx/ 163 | *.build.csdef 164 | 165 | # Microsoft Azure Emulator 166 | ecf/ 167 | rcf/ 168 | 169 | # Microsoft Azure ApplicationInsights config file 170 | ApplicationInsights.config 171 | 172 | # Windows Store app package directory 173 | AppPackages/ 174 | BundleArtifacts/ 175 | 176 | # Visual Studio cache files 177 | # files ending in .cache can be ignored 178 | *.[Cc]ache 179 | # but keep track of directories ending in .cache 180 | !*.[Cc]ache/ 181 | 182 | # Others 183 | ClientBin/ 184 | [Ss]tyle[Cc]op.* 185 | ~$* 186 | *~ 187 | *.dbmdl 188 | *.dbproj.schemaview 189 | *.pfx 190 | *.publishsettings 191 | node_modules/ 192 | orleans.codegen.cs 193 | 194 | # RIA/Silverlight projects 195 | Generated_Code/ 196 | 197 | # Backup & report files from converting an old project file 198 | # to a newer Visual Studio version. Backup files are not needed, 199 | # because we have git ;-) 200 | _UpgradeReport_Files/ 201 | Backup*/ 202 | UpgradeLog*.XML 203 | UpgradeLog*.htm 204 | 205 | # SQL Server files 206 | *.mdf 207 | *.ldf 208 | 209 | # Business Intelligence projects 210 | *.rdl.data 211 | *.bim.layout 212 | *.bim_*.settings 213 | 214 | # Microsoft Fakes 215 | FakesAssemblies/ 216 | 217 | # GhostDoc plugin setting file 218 | *.GhostDoc.xml 219 | 220 | # Node.js Tools for Visual Studio 221 | .ntvs_analysis.dat 222 | 223 | # Visual Studio 6 build log 224 | *.plg 225 | 226 | # Visual Studio 6 workspace options file 227 | *.opt 228 | 229 | # Visual Studio LightSwitch build output 230 | **/*.HTMLClient/GeneratedArtifacts 231 | **/*.DesktopClient/GeneratedArtifacts 232 | **/*.DesktopClient/ModelManifest.xml 233 | **/*.Server/GeneratedArtifacts 234 | **/*.Server/ModelManifest.xml 235 | _Pvt_Extensions 236 | 237 | # LightSwitch generated files 238 | GeneratedArtifacts/ 239 | ModelManifest.xml 240 | 241 | # Paket dependency manager 242 | .paket/paket.exe 243 | 244 | # FAKE - F# Make 245 | .fake/ -------------------------------------------------------------------------------- /.idea/.idea.iisGeolocate/.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Rider ignored files 5 | /contentModel.xml 6 | /modules.xml 7 | /projectSettingsUpdater.xml 8 | /.idea.iisGeolocate.iml 9 | # Editor-based HTTP Client requests 10 | /httpRequests/ 11 | # Datasource local storage ignored files 12 | /dataSources/ 13 | /dataSources.local.xml 14 | -------------------------------------------------------------------------------- /.idea/.idea.iisGeolocate/.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.idea/.idea.iisGeolocate/.idea/indexLayout.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/.idea.iisGeolocate/.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Eric 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # iisGeolocate 2 | geolocate ip addresses in IIS logs 3 | 4 | When the tool is started (it is a command line tool, so open a PowerShell window then run iisgeolocate.exe from there vs double clicking) 5 | 6 | Additionally, every unique, geolocated IP will be written out to a file in the '--csv' directory called !UniqueIPs.tsv. This is a comma separated file you can load into Timeline Explorer and go nuts on. 7 | 8 | Extract the program, then: 9 | 10 | 1. run iisgeolocate.exe and see usage 11 | 2. run iisgeolocate.exe -d --csv 12 | 3. wait 13 | 4. look in the out directory for processed logs, a file containing all unique IPs, and a file with any bad data in it (i.e. not valid csv data). REVIEW IT 14 | 15 | The Geolocation data will be added to the end of each log entry 16 | 17 | let me know if you have any issues or if you want other features added 18 | -------------------------------------------------------------------------------- /iisGeolocate.saproj: -------------------------------------------------------------------------------- 1 | 2 | .\iisGeolocate\bin\Release\iisGeolocate.exe 3 | iisGeolocate 4 | Eric Zimmerman 501-313-3778 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | iisGeolocate 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | -------------------------------------------------------------------------------- /iisGeolocate.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 14 4 | VisualStudioVersion = 14.0.25420.1 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "iisGeolocate", "iisGeolocate\iisGeolocate.csproj", "{FCDF899C-D8A8-43C4-8540-B4CD9E7FE791}" 7 | EndProject 8 | Global 9 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 10 | Debug|Any CPU = Debug|Any CPU 11 | Release|Any CPU = Release|Any CPU 12 | EndGlobalSection 13 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 14 | {FCDF899C-D8A8-43C4-8540-B4CD9E7FE791}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 15 | {FCDF899C-D8A8-43C4-8540-B4CD9E7FE791}.Debug|Any CPU.Build.0 = Debug|Any CPU 16 | {FCDF899C-D8A8-43C4-8540-B4CD9E7FE791}.Release|Any CPU.ActiveCfg = Release|Any CPU 17 | {FCDF899C-D8A8-43C4-8540-B4CD9E7FE791}.Release|Any CPU.Build.0 = Release|Any CPU 18 | EndGlobalSection 19 | GlobalSection(SolutionProperties) = preSolution 20 | HideSolutionNode = FALSE 21 | EndGlobalSection 22 | EndGlobal 23 | -------------------------------------------------------------------------------- /iisGeolocate/FodyWeavers.xml: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | -------------------------------------------------------------------------------- /iisGeolocate/FodyWeavers.xsd: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | A list of assembly names to exclude from the default action of "embed all Copy Local references", delimited with line breaks 13 | 14 | 15 | 16 | 17 | A list of assembly names to include from the default action of "embed all Copy Local references", delimited with line breaks. 18 | 19 | 20 | 21 | 22 | A list of runtime assembly names to exclude from the default action of "embed all Copy Local references", delimited with line breaks 23 | 24 | 25 | 26 | 27 | A list of runtime assembly names to include from the default action of "embed all Copy Local references", delimited with line breaks. 28 | 29 | 30 | 31 | 32 | Obsolete, use UnmanagedWinX86Assemblies instead 33 | 34 | 35 | 36 | 37 | A list of unmanaged X86 (32 bit) assembly names to include, delimited with line breaks. 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | The order of preloaded assemblies, delimited with line breaks. 47 | 48 | 49 | 50 | 51 | 52 | This will copy embedded files to disk before loading them into memory. This is helpful for some scenarios that expected an assembly to be loaded from a physical file. 53 | 54 | 55 | 56 | 57 | Controls if .pdbs for reference assemblies are also embedded. 58 | 59 | 60 | 61 | 62 | Controls if runtime assemblies are also embedded. 63 | 64 | 65 | 66 | 67 | Controls whether the runtime assemblies are embedded with their full path or only with their assembly name. 68 | 69 | 70 | 71 | 72 | Embedded assemblies are compressed by default, and uncompressed when they are loaded. You can turn compression off with this option. 73 | 74 | 75 | 76 | 77 | As part of Costura, embedded assemblies are no longer included as part of the build. This cleanup can be turned off. 78 | 79 | 80 | 81 | 82 | The attach method no longer subscribes to the `AppDomain.AssemblyResolve` (.NET 4.x) and `AssemblyLoadContext.Resolving` (.NET 6.0+) events. 83 | 84 | 85 | 86 | 87 | Costura by default will load as part of the module initialization. This flag disables that behavior. Make sure you call CosturaUtility.Initialize() somewhere in your code. 88 | 89 | 90 | 91 | 92 | Costura will by default use assemblies with a name like 'resources.dll' as a satellite resource and prepend the output path. This flag disables that behavior. 93 | 94 | 95 | 96 | 97 | A list of assembly names to exclude from the default action of "embed all Copy Local references", delimited with | 98 | 99 | 100 | 101 | 102 | A list of assembly names to include from the default action of "embed all Copy Local references", delimited with |. 103 | 104 | 105 | 106 | 107 | A list of runtime assembly names to exclude from the default action of "embed all Copy Local references", delimited with | 108 | 109 | 110 | 111 | 112 | A list of runtime assembly names to include from the default action of "embed all Copy Local references", delimited with |. 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | The order of preloaded assemblies, delimited with |. 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 'true' to run assembly verification (PEVerify) on the target assembly after all weavers have been executed. 134 | 135 | 136 | 137 | 138 | A comma-separated list of error codes that can be safely ignored in assembly verification. 139 | 140 | 141 | 142 | 143 | 'false' to turn off automatic generation of the XML Schema file. 144 | 145 | 146 | 147 | 148 | -------------------------------------------------------------------------------- /iisGeolocate/GlobeInfo.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EricZimmerman/iisGeolocate/1c8dbe9decd1bd23c4f9400a967cdb8390d0176f/iisGeolocate/GlobeInfo.ico -------------------------------------------------------------------------------- /iisGeolocate/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.CommandLine; 4 | using System.CommandLine.Help; 5 | using System.CommandLine.NamingConventionBinder; 6 | using System.Globalization; 7 | using System.IO; 8 | using System.Linq; 9 | using System.Reflection; 10 | using System.Threading.Tasks; 11 | using CsvHelper; 12 | using CsvHelper.Configuration; 13 | using Exceptionless; 14 | using MaxMind.GeoIP2; 15 | using MaxMind.GeoIP2.Exceptions; 16 | using Serilog; 17 | 18 | 19 | namespace iisGeolocate; 20 | 21 | internal class Program 22 | { 23 | private static Dictionary _uniqueIps; 24 | 25 | private static readonly string Header = 26 | $"iisgeolocate version {Assembly.GetExecutingAssembly().GetName().Version}" + 27 | "\r\n\r\nAuthor: Eric Zimmerman (saericzimmerman@gmail.com)" + 28 | "\r\nhttps://github.com/EricZimmerman/iisGeolocate"; 29 | 30 | private static RootCommand _rootCommand; 31 | 32 | private static async Task Main(string[] args) 33 | { 34 | ExceptionlessClient.Default.Startup("ujUuuNlhz7ZQKoDxBohBMKmPxErDgbFmNdYvPRHM"); 35 | 36 | _rootCommand = new RootCommand 37 | { 38 | new Option( 39 | "-d", 40 | "The directory that contains IIS logs. This will be recursively searched for *.log files"), 41 | 42 | new Option( 43 | "--csv", 44 | "The directory to write results to"), 45 | 46 | new Option( 47 | "--sbl", 48 | () => false, 49 | "When true, do NOT show bad lines to console (they are still logged to a file)"), 50 | 51 | new Option( 52 | "--nul", 53 | () => false, 54 | "When true, do NOT create updated CSV files in --csv directory") 55 | }; 56 | 57 | _rootCommand.Options.Single(t=>t.Name == "d").IsRequired = true; 58 | _rootCommand.Options.Single(t=>t.Name == "csv").IsRequired = true; 59 | 60 | _rootCommand.Description = Header; 61 | 62 | _rootCommand.Handler = CommandHandler.Create(DoWork); 63 | 64 | await _rootCommand.InvokeAsync(args); 65 | 66 | Log.CloseAndFlush(); 67 | } 68 | 69 | private static void DoWork(string d, string csv, bool sbl, bool nul) 70 | { 71 | 72 | var template = "{Message:lj}{NewLine}{Exception}"; 73 | 74 | Log.Logger = new LoggerConfiguration() 75 | .WriteTo.Console(outputTemplate: template) 76 | .CreateLogger(); 77 | 78 | 79 | var baseDir = AppDomain.CurrentDomain.BaseDirectory; 80 | 81 | if (string.IsNullOrEmpty(d) || string.IsNullOrEmpty(csv)) 82 | { 83 | var helpBld = new HelpBuilder(LocalizationResources.Instance, Console.WindowWidth); 84 | var hc = new HelpContext(helpBld, _rootCommand, Console.Out); 85 | 86 | helpBld.Write(hc); 87 | 88 | Log.Warning("Both -d and --csv are required. Exiting"); 89 | Console.WriteLine(); 90 | return; 91 | } 92 | 93 | _uniqueIps = new Dictionary(); 94 | 95 | Log.Information("{Header}",Header); 96 | Console.WriteLine(); 97 | 98 | d = Path.GetFullPath(d); 99 | csv = Path.GetFullPath(csv); 100 | 101 | if (Directory.Exists(d) == false) 102 | { 103 | Log.Warning("{D} does not exist. Exiting",d); 104 | Console.WriteLine(); 105 | return; 106 | } 107 | 108 | var litePath = Path.Combine(baseDir, "GeoLite2-City.mmdb"); 109 | var cityPath = Path.Combine(baseDir, "GeoIP2-City.mmdb"); 110 | 111 | if (File.Exists(litePath) == false && File.Exists(cityPath) == false) 112 | { 113 | Log.Fatal("{CityLite} or {CityIp} missing! Cannot continue. Exiting","GeoLite2-City.mmdb","GeoIP2-City.mmdb"); 114 | Console.WriteLine(); 115 | return; 116 | } 117 | 118 | var dbName = litePath; 119 | 120 | if (File.Exists(cityPath)) 121 | { 122 | Log.Information("Found {Db}, so using that vs lite...","GeoIP2-City.mmdb"); 123 | dbName = cityPath; 124 | } 125 | 126 | var logFiles = Directory.GetFiles(d, "*.log", SearchOption.AllDirectories); 127 | 128 | if (logFiles.Length > 0) 129 | { 130 | Log.Information("Found {Count:N0} log files",logFiles.Length); 131 | } 132 | else 133 | { 134 | Log.Fatal("No files ending in {Log} found. Exiting...",".log"); 135 | Console.WriteLine(); 136 | return; 137 | } 138 | 139 | if (Directory.Exists(csv) == false) 140 | { 141 | Directory.CreateDirectory(csv); 142 | } 143 | 144 | Log.Information("NOTE: multicast, private, or reserved addresses will be SKIPPED (including IPv6 that starts with {Mask}","fe80"); 145 | 146 | var badDataFile = Path.Combine(csv, "BadDataRows_REVIEW_ME.txt"); 147 | var badStream = new StreamWriter(badDataFile); 148 | 149 | Console.WriteLine(); 150 | Log.Information("All malformed data rows will be IGNORED but written to {BadDataFile}. REVIEW THIS!",badDataFile); 151 | Console.WriteLine(); 152 | 153 | var ipinfo = new Dictionary(); 154 | 155 | using (var reader = new DatabaseReader(dbName)) 156 | { 157 | foreach (var file in logFiles) 158 | { 159 | Log.Information("Opening {File}",file); 160 | 161 | var fileChunks = new Dictionary>(); 162 | 163 | using var inStream = File.OpenText(file); 164 | if (inStream.BaseStream.Length == 0) 165 | { 166 | Log.Information("\t{File} is empty. Skipping...",file); 167 | inStream.Close(); 168 | continue; 169 | } 170 | 171 | var line = inStream.ReadLine(); 172 | 173 | if (line.StartsWith("#") == false) 174 | { 175 | Log.Information("\tThe first line in {File} does not start with a #! Is this an IIS log? Skipping...",file); 176 | inStream.Close(); 177 | continue; 178 | } 179 | 180 | if (line.StartsWith("#Software: Microsoft Exchange")) 181 | { 182 | Log.Information("\tSkipping {File}! Does not appear to be an IIS related file. Skipping...",file); 183 | inStream.Close(); 184 | continue; 185 | } 186 | 187 | string lastHeaderRow = null; 188 | 189 | while (line != null) 190 | { 191 | if (line.StartsWith("#")) 192 | { 193 | if (line.StartsWith("#Fields:")) 194 | { 195 | var headerRow = line.Substring(9); 196 | 197 | //need to change to underscore so the dynamic object knows how to get data out vs trying to subtract c - ip. stupid microsoft and these names 198 | headerRow = headerRow.Replace("-", "_"); 199 | 200 | if (headerRow == lastHeaderRow) 201 | { 202 | //the second header is the same, so keep appending 203 | line = inStream.ReadLine(); 204 | continue; 205 | } 206 | 207 | //new data based on header 208 | 209 | lastHeaderRow = headerRow; 210 | 211 | fileChunks.Add(headerRow, new List()); 212 | 213 | headerRow = $"{headerRow} GeoCity GeoCountry"; 214 | 215 | fileChunks[lastHeaderRow].Add(headerRow); 216 | 217 | line = inStream.ReadLine(); 218 | continue; 219 | } 220 | 221 | line = inStream.ReadLine(); 222 | continue; 223 | } 224 | 225 | //this is where data needs to be persisted for later 226 | fileChunks[lastHeaderRow].Add(line); 227 | 228 | line = inStream.ReadLine(); 229 | } 230 | 231 | //at this point, iterate all fileChunks and make it a csv, do lookup, update extra fields, write it out 232 | 233 | var ts = DateTimeOffset.UtcNow; 234 | var counter = 0; 235 | 236 | Log.Information("\tLog chunks found in {File}: {Count:N0}. Processing chunks...",file,fileChunks.Count); 237 | 238 | foreach (var fileChunk in fileChunks) 239 | { 240 | counter += 1; 241 | 242 | Log.Information("\tFound {Count:N0} rows in chunk {Counter:N0}",fileChunk.Value.Count,counter); 243 | 244 | //outcsv stuff 245 | 246 | var logBaseName = Path.GetFileNameWithoutExtension(file); 247 | 248 | var fout = Path.Combine(csv, $"{ts:yyyyMMddHHmmss}_{logBaseName}_Chunk{counter}.csv"); 249 | 250 | CsvWriter csvOut = null; 251 | 252 | if (nul == false) 253 | { 254 | csvOut = new CsvWriter(new StreamWriter(fout), CultureInfo.CurrentCulture); 255 | } 256 | 257 | //outcsv stuff end 258 | 259 | var conf = new CsvConfiguration(CultureInfo.CurrentCulture); 260 | //hack so the idiotic iis logs can be processed 261 | conf.WhiteSpaceChars[0] = '|'; 262 | conf.Delimiter = " "; 263 | 264 | conf.BadDataFound = rawData => 265 | { 266 | badStream.Write(rawData.RawRecord); 267 | if (sbl) 268 | { 269 | return; 270 | } 271 | 272 | Log.Warning("Bad data found! Ignoring!!! Row: '{Bad}'",rawData.RawRecord.Trim()); 273 | }; 274 | 275 | //write out lines to temp file to avoid out of memory error 276 | var tmp = Path.Combine(baseDir, "tmp.txt"); 277 | File.WriteAllLines(tmp, fileChunk.Value); 278 | 279 | using (var sw = new StreamReader(tmp)) 280 | { 281 | var csvReader = new CsvReader(sw, conf); 282 | 283 | csvReader.Read(); 284 | csvReader.ReadHeader(); 285 | 286 | while (csvReader.Read()) 287 | { 288 | var record = csvReader.GetRecord(); 289 | 290 | string ip = record.c_ip; 291 | 292 | if (ip == "127.0.0.1" || ip == "::1" || ip.StartsWith("10.") || ip.StartsWith("192.168")) 293 | { 294 | record.GeoCity = "NA"; 295 | record.GeoCountry = "NA"; 296 | } 297 | else 298 | { 299 | if (ipinfo.ContainsKey(ip) == false) 300 | { 301 | var gr = GetIpInfo(ip, reader); 302 | ipinfo.Add(ip, gr); 303 | } 304 | 305 | record.GeoCity = ipinfo[ip].City; 306 | record.GeoCountry = ipinfo[ip].Country; 307 | } 308 | 309 | csvOut?.WriteRecord(record); 310 | csvOut?.NextRecord(); 311 | 312 | if (csvOut?.Row % 10_000 == 0) 313 | { 314 | csvOut.Flush(); 315 | } 316 | } 317 | 318 | csvOut?.Flush(); 319 | csvOut?.Dispose(); 320 | 321 | sw.Close(); 322 | } 323 | 324 | File.Delete(tmp); 325 | } 326 | 327 | badStream.Flush(); 328 | } 329 | 330 | badStream.Flush(); 331 | badStream.Close(); 332 | } 333 | 334 | Console.WriteLine(); 335 | 336 | if (_uniqueIps.Count <= 0) 337 | { 338 | Log.Information("No unique, geolocated IPs found!"); 339 | Console.WriteLine(); 340 | return; 341 | } 342 | 343 | Log.Information("Saving unique IPs to {File}","!UniqueIPs.csv"); 344 | 345 | using (var uniqOut = new StreamWriter(File.OpenWrite(Path.Combine(csv, "!UniqueIPs.csv")))) 346 | { 347 | var csw = new CsvWriter(uniqOut, CultureInfo.CurrentCulture); 348 | csw.WriteHeader(); 349 | csw.NextRecord(); 350 | csw.WriteRecords(_uniqueIps.Values); 351 | uniqOut.Flush(); 352 | } 353 | 354 | Console.WriteLine(); 355 | } 356 | 357 | private static GeoResults GetIpInfo(string ip, DatabaseReader reader) 358 | { 359 | var gr = new GeoResults(); 360 | gr.City = "NA"; 361 | gr.Country = "NA"; 362 | 363 | try 364 | { 365 | var city = reader.City(ip); 366 | gr.City = city.City.Name?.Replace(' ', '_'); 367 | gr.Country = city.Country.Name?.Replace(' ', '_'); 368 | 369 | 370 | if (_uniqueIps.ContainsKey(ip) == false) 371 | { 372 | var ui = new UniqueIp { City = city.City.Name }; 373 | ui.Country = city.Country.Name; 374 | ui.IpAddress = ip; 375 | 376 | _uniqueIps.Add(ip, ui); 377 | } 378 | } 379 | 380 | catch (AddressNotFoundException) 381 | { 382 | //eat it 383 | } 384 | catch (Exception ex) 385 | { 386 | Log.Error(ex,"Error {Message} for ip: {Ip}",ex.Message,ip); 387 | } 388 | 389 | return gr; 390 | } 391 | 392 | internal class GeoResults 393 | { 394 | public string City { get; set; } 395 | public string Country { get; set; } 396 | } 397 | 398 | internal class UniqueIp 399 | { 400 | public string IpAddress { get; set; } 401 | public string City { get; set; } 402 | public string Country { get; set; } 403 | } 404 | 405 | internal class ApplicationArguments 406 | { 407 | public string LogDirectory { get; set; } 408 | public bool SuppressBadLines { get; set; } 409 | public bool NoUpdatedLogs { get; set; } 410 | public string CsvDirectory { get; set; } 411 | } 412 | } -------------------------------------------------------------------------------- /iisGeolocate/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.InteropServices; 2 | 3 | // Setting ComVisible to false makes the types in this assembly not visible 4 | // to COM components. If you need to access a type in this assembly from 5 | // COM, set the ComVisible attribute to true on that type. 6 | [assembly: ComVisible(false)] 7 | 8 | // The following GUID is for the ID of the typelib if this project is exposed to COM 9 | [assembly: Guid("fcdf899c-d8a8-43c4-8540-b4cd9e7fe791")] -------------------------------------------------------------------------------- /iisGeolocate/SampleIISLog.log: -------------------------------------------------------------------------------- 1 | #Software: Microsoft Internet Information Services 6.0 2 | #Version: 1.0 3 | #Date: 2002-05-24 20:18:01 4 | #Fields: date time c-ip cs-username s-ip s-port cs-method cs-uri-stem cs-uri-query sc-status sc-bytes cs-bytes time-taken cs(User-Agent) cs(Referrer) 5 | 2002-05-24 20:18:01 172.224.24.114 - 206.73.118.24 80 GET /Default.htm - 200 7930 248 31 Mozilla/4.0+(compatible;+MSIE+5.01;+Windows+2000+Server) http://64.224.24.114/ -------------------------------------------------------------------------------- /iisGeolocate/iisGeolocate.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | Exe 4 | net462;net6.0;net9.0 5 | true 6 | false 7 | iisGeolocate 8 | Eric R. Zimmerman 501-313-3778 9 | iisGeolocate 10 | Add geolocation information to IIS logs 11 | Eric Zimmerman 12 | 10 13 | 2.2.0 14 | 15 | 16 | GlobeInfo.ico 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | all 31 | 32 | 33 | 34 | 35 | all 36 | 37 | 38 | 39 | 40 | 41 | 42 | --------------------------------------------------------------------------------