├── .gitattributes ├── .gitignore ├── .nuget ├── NuGet.Config ├── NuGet.exe ├── NuGet.targets └── packages.config ├── LICENSE.txt ├── Package.nuspec ├── Packages.dgml ├── Performance1.psess ├── README.md ├── TimberWinR.Builds └── Readme.txt ├── TimberWinR.ExtractID ├── Program.cs ├── Properties │ └── AssemblyInfo.cs └── TimberWinR.ExtractID.csproj ├── TimberWinR.ServiceHost ├── App.config ├── Installer.Designer.cs ├── Installer.cs ├── Program.cs ├── Properties │ └── AssemblyInfo.cs ├── TimberWinR.ServiceHost.1.0.0.0.nupkg ├── TimberWinR.ServiceHost.csproj ├── config.json ├── default.json ├── loopback.json ├── packages.config └── timberwinr.ico ├── TimberWinR.TestGenerator ├── CommandLineOptions.cs ├── Dynamic.cs ├── JsonLogFileGenerator.cs ├── LogFileGenerator.cs ├── Program.cs ├── Properties │ └── AssemblyInfo.cs ├── RedisTestGenerator.cs ├── TcpTestGenerator.cs ├── TimberWinR.TestGenerator.csproj ├── UdpTestGenerator.cs ├── default.json ├── packages.config ├── results1.json ├── results2.json ├── results3.json ├── results4.json ├── results5.json ├── results7.json ├── sample-apache.log ├── test1-twconfig.json ├── test1.json ├── test2-tw.json ├── test2.json ├── test3-tw.json ├── test3.json ├── test4-tw.json ├── test4.json ├── test5-twconfig.json ├── test5.json ├── test7-tw.json └── test7.json ├── TimberWinR.UnitTests ├── Configuration.cs ├── DateFilterTests.cs ├── FakeRediServer.cs ├── GeoIPFilterTests.cs ├── GrokFilterTests.cs ├── JsonFilterTests.cs ├── Multiline1.txt ├── Multiline2.txt ├── MultilineTests.cs ├── Parser │ └── ElasticsearchOutputTests.cs ├── Properties │ └── AssemblyInfo.cs ├── TailFileTests.cs ├── TestDynamicBatchCount.cs ├── TimberWinR.UnitTests.csproj ├── app.config └── packages.config ├── TimberWinR.sln ├── TimberWinR ├── Codecs │ ├── JsonCodec.cs │ ├── Multiline.cs │ └── PlainCodec.cs ├── Configuration.cs ├── ConfigurationErrors.cs ├── Diagnostics │ └── Diagnostics.cs ├── Filters │ ├── DateFilter.cs │ ├── FilterBase.cs │ ├── GeoIPFilter.cs │ ├── GrokFilter.cs │ ├── JsonFilter.cs │ └── MutateFilter.cs ├── GeoLite2City.mmdb ├── ICodec.cs ├── Inputs │ ├── FieldDefinitions.cs │ ├── GeneratorInput.cs │ ├── IISLog.cs │ ├── IISW3CInputListener.cs │ ├── IISW3CLog.cs │ ├── IISW3CRowReader.cs │ ├── InputBase.cs │ ├── InputListener.cs │ ├── LogsFileDatabase.cs │ ├── LogsListener.cs │ ├── ParameterDefinitions.cs │ ├── StdinListener.cs │ ├── TailFileInput.cs │ ├── TailFileListener.cs │ ├── TcpInputListener.cs │ ├── UdpInputListener.cs │ ├── W3CInputListener.cs │ ├── WindowsEvent.cs │ └── WindowsEvtInputListener.cs ├── LogErrors.cs ├── Manager.cs ├── Outputs │ ├── Elasticsearch.cs │ ├── File.cs │ ├── OutputSender.cs │ ├── Redis.cs │ ├── StatsD.cs │ └── Stdout.cs ├── Parser.cs ├── Parsers.cs ├── Properties │ ├── AssemblyInfo.cs │ ├── Resources.Designer.cs │ └── Resources.resx ├── ReleaseNotes.md ├── TimberWinR.csproj ├── configSchema.xsd ├── lib │ ├── com-logparser │ │ └── Interop.MSUtil.dll │ └── net40 │ │ ├── Interop.MSUtil.dll │ │ └── TimberWinR.dll ├── mdocs │ ├── Codec.md │ ├── DateFilter.md │ ├── ElasticsearchOutput.md │ ├── FileOutput.md │ ├── Filters.md │ ├── Generator.md │ ├── GeoIPFilter.md │ ├── GrokFilter.md │ ├── IISW3CInput.md │ ├── JsonFilter.md │ ├── Logs.md │ ├── MutateFilter.md │ ├── RedisOutput.md │ ├── StatsD.md │ ├── StdinInput.md │ ├── StdoutOutput.md │ ├── TailFiles.md │ ├── TcpInput.md │ ├── UdpInput.md │ ├── W3CInput.md │ └── WindowsEvents.md ├── packages.config └── testconf.xml ├── TimberWix ├── Product.wxs └── TimberWinR.Wix.wixproj ├── appveyor.yml ├── chocolatey.png ├── chocolateyInstall.ps1.template ├── chocolateyUninstall.ps1.guid ├── chocolateyUninstall.ps1.template ├── chocolateyUninstall.ps1.template.orig ├── mdocs ├── DateFilter.md ├── GrokFilter.md └── MutateFilter.md ├── packages └── repositories.config ├── timberwinr.jpg └── timberwinr.nuspec.template /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.sln.docstates 8 | 9 | # Build results 10 | 11 | [Dd]ebug/ 12 | [Rr]elease/ 13 | x64/ 14 | build/ 15 | [Bb]in/ 16 | [Oo]bj/ 17 | 18 | # Enable "build/" folder in the NuGet Packages folder since NuGet packages use it for MSBuild targets 19 | !packages/*/build/ 20 | 21 | # MSTest test Results 22 | [Tt]est[Rr]esult*/ 23 | [Bb]uild[Ll]og.* 24 | 25 | *_i.c 26 | *_p.c 27 | *.ilk 28 | *.meta 29 | *.obj 30 | *.pch 31 | *.pdb 32 | *.pgc 33 | *.pgd 34 | *.rsp 35 | *.sbr 36 | *.tlb 37 | *.tli 38 | *.tlh 39 | *.tmp 40 | *.tmp_proj 41 | *.log 42 | *.vspscc 43 | *.vssscc 44 | .builds 45 | *.pidb 46 | *.log 47 | *.scc 48 | 49 | # Visual C++ cache files 50 | ipch/ 51 | *.aps 52 | *.ncb 53 | *.opensdf 54 | *.sdf 55 | *.cachefile 56 | 57 | # Visual Studio profiler 58 | *.psess 59 | *.vsp 60 | *.vspx 61 | 62 | # Guidance Automation Toolkit 63 | *.gpState 64 | 65 | # ReSharper is a .NET coding add-in 66 | _ReSharper*/ 67 | *.[Rr]e[Ss]harper 68 | 69 | # TeamCity is a build add-in 70 | _TeamCity* 71 | 72 | # DotCover is a Code Coverage Tool 73 | *.dotCover 74 | 75 | # NCrunch 76 | *.ncrunch* 77 | .*crunch*.local.xml 78 | 79 | # Installshield output folder 80 | [Ee]xpress/ 81 | 82 | # DocProject is a documentation generator add-in 83 | DocProject/buildhelp/ 84 | DocProject/Help/*.HxT 85 | DocProject/Help/*.HxC 86 | DocProject/Help/*.hhc 87 | DocProject/Help/*.hhk 88 | DocProject/Help/*.hhp 89 | DocProject/Help/Html2 90 | DocProject/Help/html 91 | 92 | # Click-Once directory 93 | publish/ 94 | 95 | # Publish Web Output 96 | *.Publish.xml 97 | 98 | # NuGet Packages Directory 99 | ## TODO: If you have NuGet Package Restore enabled, uncomment the next line 100 | #packages/ 101 | 102 | # Windows Azure Build Output 103 | csx 104 | *.build.csdef 105 | 106 | # Windows Store app package directory 107 | AppPackages/ 108 | 109 | # Others 110 | sql/ 111 | *.Cache 112 | ClientBin/ 113 | [Ss]tyle[Cc]op.* 114 | ~$* 115 | *~ 116 | *.dbmdl 117 | *.[Pp]ublish.xml 118 | *.pfx 119 | *.publishsettings 120 | 121 | # RIA/Silverlight projects 122 | Generated_Code/ 123 | 124 | # Backup & report files from converting an old project file to a newer 125 | # Visual Studio version. Backup files are not needed, because we have git ;-) 126 | _UpgradeReport_Files/ 127 | Backup*/ 128 | UpgradeLog*.XML 129 | UpgradeLog*.htm 130 | 131 | # SQL Server files 132 | App_Data/*.mdf 133 | App_Data/*.ldf 134 | 135 | 136 | #LightSwitch generated files 137 | GeneratedArtifacts/ 138 | _Pvt_Extensions/ 139 | ModelManifest.xml 140 | 141 | # ========================= 142 | # Windows detritus 143 | # ========================= 144 | 145 | # Windows image file caches 146 | Thumbs.db 147 | ehthumbs.db 148 | 149 | # Folder config file 150 | Desktop.ini 151 | 152 | # Recycle Bin used on file shares 153 | $RECYCLE.BIN/ 154 | 155 | # Mac desktop service store files 156 | .DS_Store 157 | packages 158 | *.msi 159 | -------------------------------------------------------------------------------- /.nuget/NuGet.Config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.nuget/NuGet.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TerribleDev/TimberWinR/5165224c65c81d7e3f26df9f2bc5e6415cc8ceb6/.nuget/NuGet.exe -------------------------------------------------------------------------------- /.nuget/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2014 Cimpress 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /Package.nuspec: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | TimberWinR 5 | 1.0.1 6 | efontana 7 | efontana 8 | https://github.com/efontana/TimberWinR/blob/master/LICENSE.txt 9 | https://github.com/efontana/TimberWinR 10 | http://ICON_URL_HERE_OR_DELETE_THIS_LINE 11 | false 12 | Native .NET Windows Logging to Redis agent 13 | Summary of changes made in this release of the package. 14 | Copyright 2014 15 | Tag1 Tag2 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /Packages.dgml: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 33 | 34 | -------------------------------------------------------------------------------- /Performance1.psess: -------------------------------------------------------------------------------- 1 | 2 | 3 | Sampling 4 | None 5 | true 6 | Timestamp 7 | Cycles 8 | 10000000 9 | 10 10 | 10 11 | 12 | false 13 | 14 | 15 | 16 | false 17 | 18 | 19 | false 20 | 21 | 22 | -------------------------------------------------------------------------------- /TimberWinR.Builds/Readme.txt: -------------------------------------------------------------------------------- 1 | .MSI Packages Here 2 | ------------------ 3 | 4 | -------------------------------------------------------------------------------- /TimberWinR.ExtractID/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Text; 6 | using Microsoft.Win32.SafeHandles; 7 | using System.Runtime.InteropServices; 8 | 9 | namespace TimberWinR.ExtractID 10 | { 11 | class MsiHandle : SafeHandleMinusOneIsInvalid 12 | { 13 | public MsiHandle() 14 | : base(true) 15 | { } 16 | 17 | protected override bool ReleaseHandle() 18 | { 19 | return NativeMethods.MsiCloseHandle(handle) == 0; 20 | } 21 | } 22 | 23 | class NativeMethods 24 | { 25 | const string MsiDll = "Msi.dll"; 26 | 27 | [DllImport(MsiDll, CharSet = CharSet.Unicode, ExactSpelling = true)] 28 | public extern static uint MsiOpenPackageW(string szPackagePath, out MsiHandle product); 29 | 30 | [DllImport(MsiDll, ExactSpelling = true)] 31 | public extern static uint MsiCloseHandle(IntPtr hAny); 32 | 33 | [DllImport(MsiDll, CharSet = CharSet.Unicode, ExactSpelling = true)] 34 | static extern uint MsiGetProductPropertyW(MsiHandle hProduct, string szProperty, StringBuilder value, ref int length); 35 | 36 | 37 | [DllImport(MsiDll, ExactSpelling = true)] 38 | public static extern int MsiSetInternalUI(int value, IntPtr hwnd); 39 | 40 | public static uint MsiGetProductProperty(MsiHandle hProduct, string szProperty, out string value) 41 | { 42 | StringBuilder sb = new StringBuilder(1024); 43 | int length = sb.Capacity; 44 | uint err; 45 | value = null; 46 | if (0 == (err = MsiGetProductPropertyW(hProduct, szProperty, sb, ref length))) 47 | { 48 | sb.Length = length; 49 | value = sb.ToString(); 50 | return 0; 51 | } 52 | 53 | return err; 54 | } 55 | } 56 | 57 | 58 | 59 | class Program 60 | { 61 | [STAThread] 62 | static int Main(string[] args) 63 | { 64 | if (args.Length < 2) 65 | { 66 | Console.Error.WriteLine("Expecting MSI and Tempolate file arguments"); 67 | return 1; 68 | } 69 | 70 | string msiDirectory = args[0]; 71 | string updateFile = args[1]; 72 | string newFile = args[2]; 73 | 74 | string msiFile = Directory.GetFiles(msiDirectory, "TimberWinR*.msi").FirstOrDefault(); 75 | 76 | NativeMethods.MsiSetInternalUI(2, IntPtr.Zero); // Hide all UI. Without this you get a MSI dialog 77 | 78 | MsiHandle msi; 79 | uint err; 80 | if (0 != (err = NativeMethods.MsiOpenPackageW(msiFile, out msi))) 81 | { 82 | Console.Error.WriteLine("Can't open MSI, error {0}", err); 83 | return 1; 84 | } 85 | 86 | // Strings available in all MSIs 87 | string productCode; 88 | using (msi) 89 | { 90 | if (0 != NativeMethods.MsiGetProductProperty(msi, "ProductCode", out productCode)) 91 | throw new InvalidOperationException("Can't obtain product code"); 92 | 93 | string contents = File.ReadAllText(args[1]); 94 | 95 | contents = contents.Replace("${PROJECTGUID}", productCode); 96 | 97 | File.WriteAllText(args[2], contents); 98 | 99 | Console.WriteLine("Updated {0} ProductID: {1}", args[2], productCode); 100 | 101 | return 0; 102 | } 103 | } 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /TimberWinR.ExtractID/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("TimberWinR.ExtractID")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("TimberWinR.ExtractID")] 13 | [assembly: AssemblyCopyright("Copyright © 2014")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("cfb670ee-743d-49c7-b2bf-456bac3a88ef")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /TimberWinR.ExtractID/TimberWinR.ExtractID.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {99096939-E9DD-4499-883D-4726745A5843} 8 | Exe 9 | Properties 10 | TimberWinR.ExtractID 11 | TimberWinR.ExtractID 12 | v4.0 13 | 512 14 | 15 | 16 | AnyCPU 17 | true 18 | full 19 | false 20 | bin\Debug\ 21 | DEBUG;TRACE 22 | prompt 23 | 4 24 | 25 | 26 | AnyCPU 27 | pdbonly 28 | true 29 | bin\Release\ 30 | TRACE 31 | prompt 32 | 4 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 55 | -------------------------------------------------------------------------------- /TimberWinR.ServiceHost/App.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /TimberWinR.ServiceHost/Installer.Designer.cs: -------------------------------------------------------------------------------- 1 | namespace TimberWinR.ServiceHost 2 | { 3 | partial class Installer 4 | { 5 | /// 6 | /// Required designer variable. 7 | /// 8 | private System.ComponentModel.IContainer components = null; 9 | 10 | /// 11 | /// Clean up any resources being used. 12 | /// 13 | /// true if managed resources should be disposed; otherwise, false. 14 | protected override void Dispose(bool disposing) 15 | { 16 | if (disposing && (components != null)) 17 | { 18 | components.Dispose(); 19 | } 20 | base.Dispose(disposing); 21 | } 22 | 23 | #region Component Designer generated code 24 | 25 | /// 26 | /// Required method for Designer support - do not modify 27 | /// the contents of this method with the code editor. 28 | /// 29 | private void InitializeComponent() 30 | { 31 | components = new System.ComponentModel.Container(); 32 | } 33 | 34 | #endregion 35 | } 36 | } -------------------------------------------------------------------------------- /TimberWinR.ServiceHost/Installer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Generic; 4 | using System.ComponentModel; 5 | using System.Configuration.Install; 6 | using System.IO; 7 | using System.Linq; 8 | using System.Threading.Tasks; 9 | using Microsoft.Win32; 10 | 11 | namespace TimberWinR.ServiceHost 12 | { 13 | [RunInstaller(true)] 14 | public partial class Installer : System.Configuration.Install.Installer 15 | { 16 | public Installer() 17 | { 18 | InitializeComponent(); 19 | } 20 | 21 | public override void Install(IDictionary stateSaver) 22 | { 23 | base.Install(stateSaver); 24 | 25 | string keyPath = @"HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\services\TimberWinR"; 26 | string keyName = "ImagePath"; 27 | 28 | string currentValue = Registry.GetValue(keyPath, keyName, "").ToString(); 29 | if (!string.IsNullOrEmpty(currentValue)) 30 | { 31 | string configFile = Context.Parameters["configfile"]; 32 | if (!string.IsNullOrEmpty(configFile) && !currentValue.Contains("-configFile ")) 33 | { 34 | currentValue += string.Format(" -configFile \"{0}\"", configFile.Replace("\\\\", "\\")); 35 | Registry.SetValue(keyPath, keyName, currentValue); 36 | } 37 | 38 | currentValue = Registry.GetValue(keyPath, keyName, "").ToString(); 39 | 40 | string logDir = Context.Parameters["logdir"]; 41 | if (!string.IsNullOrEmpty(logDir) && !currentValue.Contains("-logDir ")) 42 | { 43 | currentValue += string.Format(" -logDir \"{0}\"", logDir.Replace("\\\\", "\\")); 44 | Registry.SetValue(keyPath, keyName, currentValue); 45 | } 46 | } 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /TimberWinR.ServiceHost/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("TimberWinR.ServiceHost")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("TimberWinR.ServiceHost")] 13 | [assembly: AssemblyCopyright("Copyright © 2014-2015")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("901ea25a-faf6-4870-9a43-e20b964ee3d2")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.3.26.0")] 36 | [assembly: AssemblyFileVersion("1.3.26.0")] 37 | -------------------------------------------------------------------------------- /TimberWinR.ServiceHost/TimberWinR.ServiceHost.1.0.0.0.nupkg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TerribleDev/TimberWinR/5165224c65c81d7e3f26df9f2bc5e6415cc8ceb6/TimberWinR.ServiceHost/TimberWinR.ServiceHost.1.0.0.0.nupkg -------------------------------------------------------------------------------- /TimberWinR.ServiceHost/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "TimberWinR": { 3 | "Inputs": { 4 | "Stdin": [ 5 | { 6 | "codec": "json" 7 | } 8 | ], 9 | "WindowsEvents": [ 10 | { 11 | "source": "System,Application", 12 | "binaryFormat": "PRINT", 13 | "resolveSIDS": true 14 | } 15 | ], 16 | "Tcp": [ 17 | { 18 | "_comment": "Output from NLog", 19 | "port": 5140 20 | } 21 | ] 22 | }, 23 | "Filters": [ 24 | { 25 | "grok": { 26 | "condition": "\"[type]\" == \"Win32-Eventlog\"", 27 | "match": [ 28 | "Message", 29 | "" 30 | ], 31 | "remove_field": [ 32 | "ComputerName" 33 | ] 34 | } 35 | }, 36 | { 37 | "grok": { 38 | "match": [ 39 | "message", 40 | "%{SYSLOGLINE}" 41 | ], 42 | "add_tag": [ 43 | "rn_%{Index}", 44 | "bar" 45 | ], 46 | "add_field": [ 47 | "foo_%{logsource}", 48 | "Hello dude from %{ComputerName}" 49 | ] 50 | } 51 | }, 52 | { 53 | "grok": { 54 | "match": [ 55 | "Text", 56 | "%{SYSLOGLINE}" 57 | ], 58 | "add_tag": [ 59 | "rn_%{RecordNumber}", 60 | "bar" 61 | ] 62 | } 63 | }, 64 | { 65 | "date": { 66 | "condition": "\"[type]\" == \"Win32-FileLog\"", 67 | "match": [ 68 | "timestamp", 69 | "MMM d HH:mm:sss", 70 | "MMM dd HH:mm:ss" 71 | ], 72 | "target": "UtcTimestamp", 73 | "convertToUTC": true 74 | } 75 | }, 76 | { 77 | "mutate": { 78 | "_comment": "Orion Rules", 79 | "rename": [ 80 | "ComputerName", "Host", 81 | "SID", "Username" 82 | ] 83 | } 84 | } 85 | ], 86 | "Outputs": { 87 | "Redis": [ 88 | { 89 | "threads": 1, 90 | "interval": 5000, 91 | "batch_count": 500, 92 | "host": [ 93 | "tstlexiceapp006.mycompany.svc" 94 | ] 95 | } 96 | ], 97 | "Elasticsearch": [ 98 | { 99 | "threads": 1, 100 | "interval": 5000, 101 | "host": [ 102 | "tstlexiceapp003.mycompany.svc" 103 | ] 104 | } 105 | ] 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /TimberWinR.ServiceHost/default.json: -------------------------------------------------------------------------------- 1 | { 2 | "TimberWinR": { 3 | "Inputs": { 4 | "WindowsEvents": [ 5 | { 6 | "source": "Application,System", 7 | "binaryFormat": "PRINT", 8 | "resolveSIDS": true 9 | } 10 | ], 11 | "Tcp": [ 12 | { 13 | "_comment": "Output from NLog", 14 | "port": 5140 15 | } 16 | ] 17 | }, 18 | "Filters": [ 19 | { 20 | "grok": { 21 | "condition": "\"[EventTypeName]\" == \"Information Event\"", 22 | "match": [ 23 | "Text", 24 | "" 25 | ], 26 | "drop": "true" 27 | } 28 | } 29 | ], 30 | "Outputs": { 31 | "Redis": [ 32 | { 33 | "_comment": "Change the host to your Redis instance", 34 | "port": 6379, 35 | "host": [ 36 | "logaggregator.mycompany.svc" 37 | ] 38 | } 39 | ] 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /TimberWinR.ServiceHost/loopback.json: -------------------------------------------------------------------------------- 1 | { 2 | "TimberWinR": { 3 | "Inputs": { 4 | "Stdin": [ 5 | { 6 | } 7 | ] 8 | }, 9 | "Outputs": { 10 | "Stdout": [ 11 | { 12 | } 13 | ] 14 | }, 15 | "Filters": [ 16 | { 17 | "grok": { 18 | "match": [ 19 | "message", 20 | "" 21 | ], 22 | "add_field": [ 23 | "ComputerName", "test" 24 | ] 25 | } 26 | } 27 | ] 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /TimberWinR.ServiceHost/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /TimberWinR.ServiceHost/timberwinr.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TerribleDev/TimberWinR/5165224c65c81d7e3f26df9f2bc5e6415cc8ceb6/TimberWinR.ServiceHost/timberwinr.ico -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/CommandLineOptions.cs: -------------------------------------------------------------------------------- 1 | using System.ComponentModel; 2 | using CommandLine; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Linq; 6 | using System.Text; 7 | using CommandLine.Text; 8 | 9 | namespace TimberWinR.TestGenerator 10 | { 11 | class CommandLineOptions 12 | { 13 | // [Option('r', "read", Required = true, HelpText = "Input file to be processed.")] 14 | // public string InputFile { get; set; } 15 | [Option("timberWinRConfig", DefaultValue = "default.json", HelpText = "Config file/directory to use")] 16 | public string TimberWinRConfigFile { get; set; } 17 | 18 | [Option("start", HelpText = "Start an instance of TimberWinR")] 19 | public bool StartTimberWinR { get; set; } 20 | 21 | [Option("testDir", DefaultValue = ".", HelpText = "Test directory to use (created if necessary)")] 22 | public string TestDir { get; set; } 23 | 24 | [Option("testFile", DefaultValue = "", HelpText = "Config file/directory to use")] 25 | public string TestFile { get; set; } 26 | 27 | [Option("resultsFile", HelpText = "Expected results Results json file")] 28 | public string ExpectedResultsFile { get; set; } 29 | 30 | [Option("totalMessages", DefaultValue = 0, HelpText = "The total number of messages to send to the output(s)")] 31 | public int TotalMessages { get; set; } 32 | 33 | [Option('n', "numMessages", DefaultValue = 1000, HelpText = "The number of messages to send to the output(s)")] 34 | public int NumMessages { get; set; } 35 | 36 | [Option('l', "logLevel", DefaultValue = "debug", HelpText = "Logging Level Debug|Error|Fatal|Info|Off|Trace|Warn")] 37 | public string LogLevel { get; set; } 38 | 39 | [Option('v', "verbose", DefaultValue = true, HelpText = "Prints all messages to standard output.")] 40 | public bool Verbose { get; set; } 41 | 42 | [Option("jsonLogDir", DefaultValue = ".", HelpText = "Json LogGenerator Log directory")] 43 | public string JsonLogDir { get; set; } 44 | 45 | [OptionArray('j', "json", DefaultValue = new string[] {})] 46 | public string[] JsonLogFiles { get; set; } 47 | 48 | [OptionArray("jroll", DefaultValue = new string[] { })] 49 | public string[] JsonRollingLogFiles { get; set; } 50 | 51 | [Option("jsonRate", DefaultValue = 30, HelpText = "Json Rate in Milliseconds between generation of log lines")] 52 | public int JsonRate { get; set; } 53 | 54 | [Option('u', "udp", DefaultValue = 0, HelpText = "Enable UDP generator on this Port")] 55 | public int Udp { get; set; } 56 | 57 | [Option("udp-host", DefaultValue = "localhost", HelpText = "Host to send Udp data to")] 58 | public string UdpHost { get; set; } 59 | 60 | [Option("udp-rate", DefaultValue = 10, HelpText = "Udp Rate in Milliseconds between generation of log lines")] 61 | public int UdpRate { get; set; } 62 | 63 | [Option('t', "tcp", DefaultValue = 0, HelpText = "Enable Tcp generator on this Port")] 64 | public int Tcp { get; set; } 65 | 66 | [Option("tcp-host", DefaultValue = "localhost", HelpText = "Host to send Tcp data to")] 67 | public string TcpHost { get; set; } 68 | 69 | [Option("tcp-rate", DefaultValue = 10, HelpText = "Tcp Rate in Milliseconds between generation of log lines")] 70 | public int TcpRate { get; set; } 71 | 72 | [Option('r', "redis", DefaultValue = 0, HelpText = "Enable Redis generator on this Port")] 73 | public int Redis { get; set; } 74 | 75 | [Option("redis-host", DefaultValue = "", HelpText = "Host to send Redis data to")] 76 | public string RedisHost { get; set; } 77 | 78 | [ParserState] 79 | public IParserState LastParserState { get; set; } 80 | 81 | [HelpOption] 82 | public string GetUsage() 83 | { 84 | return HelpText.AutoBuild(this, 85 | (HelpText current) => HelpText.DefaultParsingErrorsHandler(this, current)); 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/LogFileGenerator.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics; 2 | using System.Runtime.InteropServices; 3 | using System.Runtime.Remoting.Messaging; 4 | using System.Threading; 5 | using Newtonsoft.Json; 6 | using Newtonsoft.Json.Linq; 7 | using NLog; 8 | using NLog.Config; 9 | using NLog.Targets; 10 | 11 | using System; 12 | using System.Collections.Generic; 13 | using System.Linq; 14 | using System.Net; 15 | using System.Net.Sockets; 16 | using System.Text; 17 | using System.IO; 18 | 19 | 20 | namespace TimberWinR.TestGenerator 21 | { 22 | class LogFileTestParameters 23 | { 24 | public int NumMessages { get; set; } 25 | public string LogFileDir { get; set; } 26 | public string LogFileName { get; set; } 27 | public int SleepTimeMilliseconds { get; set; } 28 | public LogFileTestParameters() 29 | { 30 | SleepTimeMilliseconds = 30; 31 | LogFileDir = "."; 32 | NumMessages = 10; 33 | } 34 | } 35 | 36 | class LogFileGenerator 37 | { 38 | public static int Generate(JsonLogFileTestParameters parms) 39 | { 40 | LogManager.GetCurrentClassLogger().Info("Start LogFile Generation for: {0} on Thread: {1}", Path.GetFullPath(parms.LogFileName), Thread.CurrentThread.ManagedThreadId); 41 | 42 | var logFilePath = Path.Combine(parms.LogFileDir, parms.LogFileName); 43 | 44 | try 45 | { 46 | if (File.Exists(logFilePath)) 47 | { 48 | LogManager.GetCurrentClassLogger().Info("Deleting file: {0}", logFilePath); 49 | File.Delete(logFilePath); 50 | } 51 | } 52 | catch (Exception ex) 53 | { 54 | LogManager.GetCurrentClassLogger().Error(ex); 55 | } 56 | 57 | 58 | var hostName = System.Environment.MachineName + "." + 59 | Microsoft.Win32.Registry.LocalMachine.OpenSubKey( 60 | "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString(); 61 | 62 | var watch = Stopwatch.StartNew(); 63 | 64 | // This text is always added, making the file longer over time 65 | // if it is not deleted. 66 | using (StreamWriter sw = File.AppendText(logFilePath)) 67 | { 68 | for (int i = 0; i < parms.NumMessages; i++) 69 | { 70 | JObject o = new JObject 71 | { 72 | {"LineNumber", i+1}, 73 | {"Application", "logfile-generator"}, 74 | {"Host", hostName}, 75 | {"UtcTimestamp", DateTime.UtcNow.ToString("o")}, 76 | {"Type", "log"}, 77 | {"Message", string.Format("{0}: Testgenerator logfile message {1}", i+1, DateTime.UtcNow.ToString("o"))}, 78 | {"Index", "logstash"} 79 | }; 80 | sw.WriteLine(o.ToString(Formatting.None)); 81 | 82 | Thread.Sleep(parms.SleepTimeMilliseconds); 83 | } 84 | LogManager.GetCurrentClassLogger().Info("Elapsed Time for {0} was {1} seconds", Path.GetFullPath(parms.LogFileName), watch.Elapsed); 85 | watch.Reset(); 86 | } 87 | 88 | LogManager.GetCurrentClassLogger().Info("Finished LogFile Generation for: {0} elapsed: {1}", Path.GetFullPath(parms.LogFileName), watch.Elapsed); 89 | 90 | return parms.NumMessages; 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("TimberWinR.TestGenerator")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("TimberWinR.TestGenerator")] 13 | [assembly: AssemblyCopyright("Copyright © 2015")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("a56bf91c-c5f8-4771-8ef8-ab9ad28179c4")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/RedisTestGenerator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using Newtonsoft.Json.Linq; 6 | using ServiceStack.Redis; 7 | 8 | namespace TimberWinR.TestGenerator 9 | { 10 | class RedisTestParameters 11 | { 12 | public int Port { get; set; } 13 | public string Host { get; set; } 14 | public int NumMessages { get; set; } 15 | public RedisTestParameters() 16 | { 17 | NumMessages = 100; 18 | Port = 6379; 19 | Host = "localhost"; 20 | } 21 | } 22 | 23 | class RedisTestGenerator 24 | { 25 | public static void Generate(RedisTestParameters parms) 26 | { 27 | var hostName = System.Environment.MachineName + "." + 28 | Microsoft.Win32.Registry.LocalMachine.OpenSubKey( 29 | "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString(); 30 | 31 | var rc = new RedisClient(parms.Host, parms.Port); 32 | 33 | for (int i = 0; i < parms.NumMessages; i++) 34 | { 35 | JObject o = new JObject 36 | { 37 | {"Application", "redis-generator"}, 38 | {"Host", hostName}, 39 | {"UtcTimestamp", DateTime.UtcNow.ToString("o")}, 40 | {"Type", "redis"}, 41 | {"Message", "redis message " + DateTime.UtcNow.ToString("o")}, 42 | {"Index", "logstash"} 43 | }; 44 | byte[] bytes = System.Text.Encoding.UTF8.GetBytes(o.ToString()); 45 | var restult = rc.RPush("logstash", bytes); 46 | } 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/TcpTestGenerator.cs: -------------------------------------------------------------------------------- 1 | using System.Threading; 2 | using Newtonsoft.Json.Linq; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Linq; 6 | using System.Net; 7 | using System.Net.Sockets; 8 | using System.Text; 9 | using ServiceStack.Text; 10 | 11 | namespace TimberWinR.TestGenerator 12 | { 13 | class TcpTestParameters 14 | { 15 | public int Port { get; set; } 16 | public string Host { get; set; } 17 | public int NumMessages { get; set; } 18 | public int SleepTimeMilliseconds { get; set; } 19 | public TcpTestParameters() 20 | { 21 | NumMessages = 100; 22 | Port = 5140; 23 | Host = "localhost"; 24 | SleepTimeMilliseconds = 10; 25 | } 26 | } 27 | 28 | class TcpTestGenerator 29 | { 30 | public static int Generate(TcpTestParameters parms) 31 | { 32 | TcpClient server = new TcpClient(parms.Host, parms.Port); 33 | 34 | var hostName = System.Environment.MachineName + "." + 35 | Microsoft.Win32.Registry.LocalMachine.OpenSubKey( 36 | "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString(); 37 | 38 | 39 | using (NetworkStream stream = server.GetStream()) 40 | { 41 | for (int i = 0; i < parms.NumMessages; i++) 42 | { 43 | JObject o = new JObject 44 | { 45 | {"Application", "tcp-generator"}, 46 | {"Host", hostName}, 47 | {"UtcTimestamp", DateTime.UtcNow.ToString("o")}, 48 | {"Type", "tcp"}, 49 | {"Message", "tcp message " + DateTime.UtcNow.ToString("o")}, 50 | {"Index", "logstash"} 51 | }; 52 | byte[] data = Encoding.UTF8.GetBytes(string.Format("{0}\n", o.ToString())); 53 | stream.Write(data, 0, data.Length); 54 | Thread.Sleep(parms.SleepTimeMilliseconds); 55 | } 56 | } 57 | 58 | return parms.NumMessages; 59 | } 60 | 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/UdpTestGenerator.cs: -------------------------------------------------------------------------------- 1 | using System.Security.Cryptography; 2 | using System.Threading; 3 | using Newtonsoft.Json; 4 | using Newtonsoft.Json.Linq; 5 | using NLog; 6 | using NLog.Config; 7 | using NLog.Targets; 8 | 9 | using System; 10 | using System.Collections.Generic; 11 | using System.Linq; 12 | using System.Net; 13 | using System.Net.Sockets; 14 | using System.Text; 15 | 16 | namespace TimberWinR.TestGenerator 17 | { 18 | class UdpTestParameters 19 | { 20 | public int Port { get; set; } 21 | public string Host { get; set; } 22 | public int NumMessages { get; set; } 23 | public int SleepTimeMilliseconds { get; set; } 24 | public UdpTestParameters() 25 | { 26 | NumMessages = 100; 27 | Port = 6379; 28 | Host = "localhost"; 29 | SleepTimeMilliseconds = 1; 30 | } 31 | } 32 | 33 | class UdpTestGenerator 34 | { 35 | public static int Generate(UdpTestParameters parms) 36 | { 37 | var hostName = System.Environment.MachineName + "." + 38 | Microsoft.Win32.Registry.LocalMachine.OpenSubKey( 39 | "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString(); 40 | 41 | IPAddress broadcast; 42 | if (!IPAddress.TryParse(parms.Host, out broadcast)) 43 | broadcast = Dns.GetHostEntry(parms.Host).AddressList[0]; 44 | 45 | Socket s = new Socket(broadcast.AddressFamily, SocketType.Dgram, ProtocolType.Udp); 46 | 47 | LogManager.GetCurrentClassLogger().Info("Start UDP Generation"); 48 | 49 | for (int i = 0; i < parms.NumMessages; i++) 50 | { 51 | JObject o = new JObject 52 | { 53 | {"Application", "udp-generator"}, 54 | {"Executable", "VP.Common.SvcFrm.Services.Host, Version=29.7.0.0, Culture=neutral, PublicKeyToken=null"}, 55 | {"RenderedMessage", "Responding to RequestSchedule message from 10.1.230.36 with Ack because: PRJ byte array is null."}, 56 | {"Team", "Manufacturing Software"}, 57 | {"RecordNumber", i}, 58 | {"Host", hostName}, 59 | {"UtcTimestamp", DateTime.UtcNow.ToString("o")}, 60 | {"Type", "VP.Fulfillment.Direct.Initialization.LogWrapper"}, 61 | {"Message", "Testgenerator udp message " + DateTime.UtcNow.ToString("o")}, 62 | {"Index", "logstash"} 63 | }; 64 | 65 | string hashedString = ""; 66 | foreach(var key in o) 67 | { 68 | hashedString += key.ToString(); 69 | } 70 | 71 | var source = ASCIIEncoding.ASCII.GetBytes(hashedString); 72 | var md5 = new MD5CryptoServiceProvider().ComputeHash(source); 73 | var hash = string.Concat(md5.Select(x => x.ToString("X2"))); 74 | 75 | o["md5"] = hash; 76 | 77 | byte[] sendbuf = Encoding.UTF8.GetBytes(o.ToString(Formatting.None)); 78 | IPEndPoint ep = new IPEndPoint(broadcast, parms.Port); 79 | s.SendTo(sendbuf, ep); 80 | 81 | if (i % 1000 == 0) 82 | LogManager.GetCurrentClassLogger().Info("Sent {0} of {1} messages", i, parms.NumMessages); 83 | 84 | Thread.Sleep(parms.SleepTimeMilliseconds); 85 | } 86 | 87 | LogManager.GetCurrentClassLogger().Info("Finished UDP Generation"); 88 | 89 | return parms.NumMessages; 90 | } 91 | 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/default.json: -------------------------------------------------------------------------------- 1 | { 2 | "TimberWinR": { 3 | "Inputs": { 4 | "Udp": [ 5 | { 6 | "_comment": "Output from NLog", 7 | "port": 5140 8 | } 9 | ], 10 | "TailFiles": [ 11 | { 12 | "interval": 5, 13 | "logSource": "log files", 14 | "location": "*.jlog", 15 | "recurse": -1 16 | } 17 | ] 18 | }, 19 | "Filters": [ 20 | { 21 | "grok": { 22 | "condition": "\"[EventTypeName]\" == \"Information Event\"", 23 | "match": [ 24 | "Text", 25 | "" 26 | ], 27 | "drop": "true" 28 | } 29 | } 30 | ], 31 | "Outputs": { 32 | "Redis": [ 33 | { 34 | "_comment": "Change the host to your Redis instance", 35 | "port": 6379, 36 | "batch_count": 500, 37 | "threads": 2, 38 | "host": [ 39 | "tstlexiceapp006.mycompany.svc" 40 | ] 41 | } 42 | ] 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/results1.json: -------------------------------------------------------------------------------- 1 | { 2 | "Results": { 3 | "Inputs": [ 4 | { 5 | "taillog": { 6 | "test1: message sent count": "[messages] == 7404", 7 | "test2: average cpu": "[avgCpuUsage] <= 30", 8 | "test3: maximum memory": "[maxMemUsage] <= 30" 9 | } 10 | }, 11 | { 12 | "udp": { 13 | "test1: message sent count": "[messages] == 1234", 14 | "test2: average cpu": "[avgCpuUsage] <= 30", 15 | "test3: maximum memory": "[maxMemUsage] <= 30" 16 | } 17 | } 18 | ] 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/results2.json: -------------------------------------------------------------------------------- 1 | { 2 | "Results": { 3 | "Inputs": [ 4 | { 5 | "taillog": { 6 | "test1: message sent count": "[messages] == 7404", 7 | "test2: average cpu": "[avgCpuUsage] <= 30", 8 | "test3: maximum memory": "[maxMemUsage] <= 20" 9 | } 10 | }, 11 | { 12 | "udp": { 13 | "test1: message sent count": "[messages] == 1234", 14 | "test2: average cpu": "[avgCpuUsage] <= 30", 15 | "test3: maximum memory": "[maxMemUsage] <= 20" 16 | } 17 | } 18 | ] 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/results3.json: -------------------------------------------------------------------------------- 1 | { 2 | "Results": { 3 | "Inputs": [ 4 | { 5 | "taillog": { 6 | "test1: message sent count": "[messages] == 7404", 7 | "test2: average cpu": "[avgCpuUsage] <= 30", 8 | "test3: maximum memory": "[maxMemUsage] <= 15" 9 | } 10 | }, 11 | { 12 | "tcp": { 13 | "test4: message sent count": "[messages] == 1234", 14 | "test5: average cpu": "[avgCpuUsage] <= 30", 15 | "test6: maximum memory": "[maxMemUsage] <= 15" 16 | } 17 | } 18 | ] 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/results4.json: -------------------------------------------------------------------------------- 1 | { 2 | "Results": { 3 | "Inputs": [ 4 | { 5 | "taillog": { 6 | "test1: message sent count": "[messages] == 7404", 7 | "test2: average cpu": "[avgCpuUsage] <= 30", 8 | "test3: maximum memory": "[maxMemUsage] <= 15" 9 | } 10 | }, 11 | { 12 | "tcp": { 13 | "test4: message sent count": "[messages] == 1234", 14 | "test5: average cpu": "[avgCpuUsage] <= 30", 15 | "test6: maximum memory": "[maxMemUsage] <= 15" 16 | } 17 | } 18 | ] 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/results5.json: -------------------------------------------------------------------------------- 1 | { 2 | "Results": { 3 | "Inputs": [ 4 | { 5 | "udp": { 6 | "test1: message sent count": "[messages] == 80000", 7 | "test2: average cpu": "[avgCpuUsage] <= 30", 8 | "test3: maximum memory": "[maxMemUsage] <= 30" 9 | } 10 | } 11 | ] 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/results7.json: -------------------------------------------------------------------------------- 1 | { 2 | "Results": { 3 | "Inputs": [ 4 | { 5 | "udp": { 6 | "test1: message sent count": "[messages] == 80000", 7 | "test2: average cpu": "[avgCpuUsage] <= 30", 8 | "test3: maximum memory": "[maxMemUsage] <= 30" 9 | } 10 | } 11 | ] 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/test1-twconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "TimberWinR": { 3 | "Inputs": { 4 | "Udp": [ 5 | { 6 | "_comment": "Output from NLog", 7 | "port": 5140 8 | } 9 | ], 10 | "TailFiles": [ 11 | { 12 | "interval": 5, 13 | "logSource": "log files", 14 | "location": "*.jlog", 15 | "recurse": -1 16 | } 17 | ] 18 | }, 19 | "Filters": [ 20 | { 21 | "grok": { 22 | "condition": "\"[EventTypeName]\" == \"Information Event\"", 23 | "match": [ 24 | "Text", 25 | "" 26 | ], 27 | "drop": "true" 28 | }, 29 | "json": { 30 | "type": "Win32-TailFile", 31 | "source": "Text", 32 | "promote": "Text" 33 | } 34 | } 35 | ], 36 | "Outputs": { 37 | "Redis": [ 38 | { 39 | "_comment": "Change the host to your Redis instance", 40 | "port": 6379, 41 | "batch_count": 500, 42 | "threads": 1, 43 | "host": [ 44 | "tstlexiceapp006.mycompany.svc" 45 | ] 46 | } 47 | ] 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/test1.json: -------------------------------------------------------------------------------- 1 | { 2 | "test": "Test 1", 3 | "arguments": { 4 | "--start": "", 5 | "--testFile": "test1.json", 6 | "--testDir": "test1", 7 | "--timberWinRConfig": "test1-twconfig.json", 8 | "--numMessages": 1234, 9 | "--logLevel": "debug", 10 | "--udp-host": "localhost", 11 | "--udp": "5140", 12 | "--jroll": ["r1.jlog", "r2.jlog"], 13 | "--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"], 14 | "--resultsFile": "results1.json" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/test2-tw.json: -------------------------------------------------------------------------------- 1 | { 2 | "TimberWinR": { 3 | "Inputs": { 4 | "Udp": [ 5 | { 6 | "_comment": "Output from NLog", 7 | "port": 5140 8 | } 9 | ], 10 | "Logs": [ 11 | { 12 | "interval": 5, 13 | "logSource": "log files", 14 | "location": "*.jlog", 15 | "recurse": -1 16 | } 17 | ] 18 | }, 19 | "Filters": [ 20 | { 21 | "grok": { 22 | "condition": "\"[EventTypeName]\" == \"Information Event\"", 23 | "match": [ 24 | "Text", 25 | "" 26 | ], 27 | "drop": "true" 28 | } 29 | } 30 | ], 31 | "Outputs": { 32 | "Redis": [ 33 | { 34 | "_comment": "Change the host to your Redis instance", 35 | "port": 6379, 36 | "batch_count": 500, 37 | "threads": 2, 38 | "host": [ 39 | "tstlexiceapp006.mycompany.svc" 40 | ] 41 | } 42 | ] 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/test2.json: -------------------------------------------------------------------------------- 1 | { 2 | "test": "Test 2", 3 | "arguments": { 4 | "--start": "", 5 | "--testFile": "test2.json", 6 | "--testDir": "test2", 7 | "--timberWinRConfig": "test2-tw.json", 8 | "--numMessages": 1234, 9 | "--logLevel": "debug", 10 | "--udp": "5140", 11 | "--jroll": ["r1.jlog", "r2.jlog"], 12 | "--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"], 13 | "--resultsFile": "results2.json" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/test3-tw.json: -------------------------------------------------------------------------------- 1 | { 2 | "TimberWinR": { 3 | "Inputs": { 4 | "Udp": [ 5 | { 6 | "_comment": "Output from NLog", 7 | "port": 5140 8 | } 9 | ], 10 | "TailFiles": [ 11 | { 12 | "interval": 5, 13 | "logSource": "log files", 14 | "location": "d:\\logs\\sta\\sta.log", 15 | "recurse": -1 16 | } 17 | ] 18 | }, 19 | "Filters": [ 20 | { 21 | "grok": { 22 | "condition": "\"[EventTypeName]\" == \"Information Event\"", 23 | "match": [ 24 | "Text", 25 | "" 26 | ], 27 | "drop": "true" 28 | }, 29 | "json": { 30 | "type": "Win32-TailFile", 31 | "source": "Text", 32 | "promote": "Text" 33 | } 34 | } 35 | ], 36 | "Outputs": { 37 | "Redis": [ 38 | { 39 | "_comment": "Change the host to your Redis instance", 40 | "port": 6379, 41 | "batch_count": 500, 42 | "threads": 2, 43 | "host": [ 44 | "tstlexiceapp006.mycompany.svc" 45 | ] 46 | } 47 | ] 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/test3.json: -------------------------------------------------------------------------------- 1 | { 2 | "test": "Test 3", 3 | "arguments": { 4 | "--start": "", 5 | "--testFile": "test3.json", 6 | "--testDir": "test3", 7 | "--timberWinRConfig": "test3-tw.json", 8 | "--numMessages": 1234, 9 | "--logLevel": "debug", 10 | "--resultsFile": "results3.json" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/test4-tw.json: -------------------------------------------------------------------------------- 1 | { 2 | "TimberWinR": { 3 | "Inputs": { 4 | "Udp": [ 5 | { 6 | "_comment": "Output from NLog", 7 | "port": 5140 8 | } 9 | ], 10 | "TailFiles": [ 11 | { 12 | "interval": 5, 13 | "logSource": "log files", 14 | "location": "d:\\logs\\sta\\sta.log", 15 | "recurse": -1 16 | } 17 | ] 18 | }, 19 | "Filters": [ 20 | { 21 | "grok": { 22 | "condition": "\"[EventTypeName]\" == \"Information Event\"", 23 | "match": [ 24 | "Text", 25 | "" 26 | ], 27 | "drop": "true" 28 | }, 29 | "json": { 30 | "type": "Win32-TailFile", 31 | "source": "Text", 32 | "promote": "Text" 33 | } 34 | } 35 | ], 36 | "Outputs": { 37 | "Redis": [ 38 | { 39 | "_comment": "Change the host to your Redis instance", 40 | "port": 6379, 41 | "batch_count": 500, 42 | "threads": 2, 43 | "host": [ 44 | "tstlexiceapp006.mycompany.svc" 45 | ] 46 | } 47 | ] 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/test4.json: -------------------------------------------------------------------------------- 1 | { 2 | "test": "Test 4", 3 | "arguments": { 4 | "--testFile": "test4.json", 5 | "--testDir": "test4", 6 | "--timberWinRConfig": "test4-tw.json", 7 | "--numMessages": 1234, 8 | "--logLevel": "debug", 9 | "--resultsFile": "results4.json" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/test5-twconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "TimberWinR": { 3 | "Inputs": { 4 | "Udp": [ 5 | { 6 | "_comment": "Output from NLog", 7 | "port": 5140, 8 | "add_field": [ 9 | "Environment", 10 | "PLANT_TST_TIMBERWINR" 11 | ], 12 | "rename": [ 13 | "Type", 14 | "type" 15 | ] 16 | } 17 | ], 18 | "TailFiles": [ 19 | { 20 | "interval": 5, 21 | "logSource": "log files", 22 | "location": "*.jlog", 23 | "recurse": -1 24 | } 25 | ] 26 | }, 27 | "Outputs": { 28 | "File": [ 29 | { 30 | "format": "indented", 31 | "file_name": "test5_output.txt" 32 | } 33 | ] 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/test5.json: -------------------------------------------------------------------------------- 1 | { 2 | "test": "Test 5", 3 | "arguments": { 4 | "--start": "", 5 | "--testFile": "test5.json", 6 | "--testDir": "test5", 7 | "--timberWinRConfig": "test5-twconfig.json", 8 | "--numMessages": 80000, 9 | "--logLevel": "debug", 10 | "--udp-host": "localhost", 11 | "--udp": "5140", 12 | "--udp-rate": 1, 13 | "--resultsFile": "results5.json" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/test7-tw.json: -------------------------------------------------------------------------------- 1 | { 2 | "TimberWinR": { 3 | "Inputs": { 4 | "TailFiles": [ 5 | { 6 | "interval": 5, 7 | "logSource": "apache log files", 8 | "location": "..\\sample-apache.log", 9 | "recurse": -1 10 | } 11 | ] 12 | }, 13 | "Filters": [ 14 | { 15 | "grok": { 16 | "type": "Win32-TailLog", 17 | "match": [ 18 | "Text", 19 | "%{COMBINEDAPACHELOG}" 20 | ] 21 | } 22 | } 23 | ], 24 | "Outputs": { 25 | "StatsD": [ 26 | { 27 | "type": "Win32-TailLog", 28 | "namespace": "timberwinrtest", 29 | "port": 8125, 30 | "host": "devlexicesnu003.mycompany.svc", 31 | "increment": ["apache.response.%{response}"], 32 | "count": ["apache.bytes", "%{bytes}"] 33 | } 34 | ] 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /TimberWinR.TestGenerator/test7.json: -------------------------------------------------------------------------------- 1 | { 2 | "test": "Test 7", 3 | "arguments": { 4 | "--totalMessages": 2223, 5 | "--start": "", 6 | "--testFile": "test7.json", 7 | "--testDir": "test7", 8 | "--timberWinRConfig": "test7-tw.json", 9 | "--numMessages": 1234, 10 | "--logLevel": "debug", 11 | "--resultsFile": "results7.json" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /TimberWinR.UnitTests/Configuration.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | using TimberWinR; 8 | using TimberWinR.Inputs; 9 | using TimberWinR.Filters; 10 | using Newtonsoft.Json.Linq; 11 | using TimberWinR.Parser; 12 | 13 | namespace TimberWinR.UnitTests 14 | { 15 | [TestFixture] 16 | public class ConfigurationTest 17 | { 18 | [Test] 19 | public void TestInvalidMatchConfig() 20 | { 21 | string grokJson = @"{ 22 | ""TimberWinR"":{ 23 | ""Filters"":[ 24 | { 25 | ""grok"":{ 26 | ""condition"": ""[type] == \""Win32-FileLog\"""", 27 | ""match"":[ 28 | ""Text"" 29 | ] 30 | } 31 | }] 32 | } 33 | }"; 34 | 35 | try 36 | { 37 | Configuration c = Configuration.FromString(grokJson); 38 | Assert.IsTrue(false, "Should have thrown an exception"); 39 | } 40 | catch (TimberWinR.Parser.Grok.GrokFilterException) 41 | { 42 | } 43 | } 44 | 45 | [Test] 46 | public void TestInvalidAddTagConfig() 47 | { 48 | string grokJson = @"{ 49 | ""TimberWinR"":{ 50 | ""Filters"":[ 51 | { 52 | ""grok"":{ 53 | ""condition"": ""[type] == \""Win32-FileLog\"""", 54 | ""match"":[ 55 | ""Text"", """" 56 | ], 57 | ""add_tag"": [ 58 | ""rn_%{Index}"", 59 | ], 60 | } 61 | }] 62 | } 63 | }"; 64 | 65 | try 66 | { 67 | Configuration c = Configuration.FromString(grokJson); 68 | Assert.IsTrue(false, "Should have thrown an exception"); 69 | } 70 | catch (TimberWinR.Parser.Grok.GrokAddTagException ex) 71 | { 72 | Assert.AreEqual(ex.Message, "Grok filter add_tag requires tuples"); 73 | } 74 | } 75 | 76 | [Test] 77 | public void TestRedisHostCount() 78 | { 79 | string redisJson = @"{ 80 | ""TimberWinR"": 81 | { 82 | ""Outputs"": 83 | { 84 | ""Redis"": 85 | [{ 86 | ""host"": 87 | [ 88 | ""logaggregator.mycompany.svc"" 89 | ] 90 | }] 91 | } 92 | } 93 | }"; 94 | 95 | 96 | Configuration c = Configuration.FromString(redisJson); 97 | RedisOutputParameters redis = c.RedisOutputs.First() as RedisOutputParameters; 98 | Assert.IsTrue(redis.Host.Length >= 1); 99 | } 100 | 101 | [Test] 102 | public void TestDateFilterMatchCount() 103 | { 104 | string dateJson = @"{ 105 | ""TimberWinR"":{ 106 | ""Filters"":[ 107 | { 108 | ""date"":{ 109 | ""target"": ""timestamp"", 110 | ""match"":[ 111 | ""timestamp"", 112 | ""MMM d HH:mm:sss"", 113 | ""MMM dd HH:mm:ss"" 114 | ], 115 | } 116 | }] 117 | } 118 | }"; 119 | 120 | 121 | Configuration c = Configuration.FromString(dateJson); 122 | DateFilter date = c.Filters.First() as DateFilter; 123 | Assert.IsTrue(date.Match.Length >= 2); 124 | } 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /TimberWinR.UnitTests/DateFilterTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using NUnit.Framework; 7 | using TimberWinR.Parser; 8 | using Newtonsoft.Json.Linq; 9 | 10 | namespace TimberWinR.UnitTests 11 | { 12 | [TestFixture] 13 | public class DateFilterTests 14 | { 15 | [Test] 16 | public void TestDate1() 17 | { 18 | JObject json = new JObject 19 | { 20 | {"message", "2014-01-31 08:23:47,123"} 21 | }; 22 | 23 | string grokJson = @"{ 24 | ""TimberWinR"":{ 25 | ""Filters"":[ 26 | { 27 | ""date"":{ 28 | ""match"":[ 29 | ""message"", 30 | ""yyyy-MM-dd HH:mm:ss,fff"" 31 | ] 32 | } 33 | }] 34 | } 35 | }"; 36 | 37 | Configuration c = Configuration.FromString(grokJson); 38 | 39 | DateFilter date = c.Filters.First() as DateFilter; 40 | 41 | Assert.IsTrue(date.Apply(json)); 42 | 43 | var ts = json["@timestamp"].ToString(); 44 | 45 | Assert.AreEqual(ts, "1/31/2014 8:23:47 AM"); 46 | 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /TimberWinR.UnitTests/FakeRediServer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Threading; 4 | using System.Net; 5 | using System.Net.Sockets; 6 | using Newtonsoft.Json; 7 | using Newtonsoft.Json.Linq; 8 | using NUnit.Framework; 9 | using TimberWinR.Parser; 10 | using System.Text; 11 | using System.Collections.Generic; 12 | 13 | namespace TimberWinR.UnitTests 14 | { 15 | // Class which implements a Fake redis server for test purposes. 16 | class FakeRediServer 17 | { 18 | private readonly System.Net.Sockets.TcpListener _tcpListenerV4; 19 | private readonly System.Net.Sockets.TcpListener _tcpListenerV6; 20 | private Thread _listenThreadV4; 21 | private Thread _listenThreadV6; 22 | private readonly int _port; 23 | private CancellationToken _cancelToken; 24 | private bool _shutdown; 25 | 26 | public FakeRediServer(CancellationToken cancelToken, int port = 6379) 27 | { 28 | _port = port; 29 | _cancelToken = cancelToken; 30 | _shutdown = false; 31 | 32 | _tcpListenerV6 = new System.Net.Sockets.TcpListener(IPAddress.IPv6Any, port); 33 | _tcpListenerV4 = new System.Net.Sockets.TcpListener(IPAddress.Any, port); 34 | 35 | _listenThreadV4 = new Thread(new ParameterizedThreadStart(ListenForClients)); 36 | _listenThreadV4.Start(_tcpListenerV4); 37 | 38 | _listenThreadV6 = new Thread(new ParameterizedThreadStart(ListenForClients)); 39 | _listenThreadV6.Start(_tcpListenerV6); 40 | } 41 | 42 | public void Shutdown() 43 | { 44 | _shutdown = true; 45 | this._tcpListenerV4.Stop(); 46 | this._tcpListenerV6.Stop(); 47 | } 48 | 49 | 50 | private void ListenForClients(object olistener) 51 | { 52 | System.Net.Sockets.TcpListener listener = olistener as System.Net.Sockets.TcpListener; 53 | 54 | listener.Start(); 55 | 56 | 57 | while (!_cancelToken.IsCancellationRequested && !_shutdown) 58 | { 59 | try 60 | { 61 | //blocks until a client has connected to the server 62 | TcpClient client = listener.AcceptTcpClient(); 63 | 64 | // Wait for a client, spin up a thread. 65 | var clientThread = new Thread(new ParameterizedThreadStart(HandleNewClient)); 66 | clientThread.Start(client); 67 | } 68 | catch (SocketException ex) 69 | { 70 | if (ex.SocketErrorCode == SocketError.Interrupted) 71 | break; 72 | } 73 | } 74 | } 75 | 76 | private void HandleNewClient(object client) 77 | { 78 | var tcpClient = (TcpClient)client; 79 | 80 | try 81 | { 82 | NetworkStream clientStream = tcpClient.GetStream(); 83 | int i; 84 | Byte[] bytes = new Byte[16535]; 85 | String data = null; 86 | 87 | do 88 | { 89 | try 90 | { 91 | // Loop to receive all the data sent by the client. 92 | while ((i = clientStream.Read(bytes, 0, bytes.Length)) != 0) 93 | { 94 | // Translate data bytes to a ASCII string. 95 | data = System.Text.Encoding.ASCII.GetString(bytes, 0, i); 96 | //System.Diagnostics.Debug.WriteLine(String.Format("Received: {0}", data)); 97 | 98 | // Process the data sent by the client. 99 | data = ":1000\r\n"; 100 | 101 | byte[] msg = System.Text.Encoding.ASCII.GetBytes(data); 102 | 103 | // Send back a response. 104 | clientStream.Write(msg, 0, msg.Length); 105 | // System.Diagnostics.Debug.WriteLine(String.Format("Sent: {0}", data)); 106 | } 107 | } 108 | catch (IOException) 109 | { 110 | } 111 | } while (true); 112 | } 113 | catch (Exception ex) 114 | { 115 | System.Diagnostics.Debug.WriteLine(ex.ToString()); 116 | } 117 | tcpClient.Close(); 118 | } 119 | 120 | private void ProcessJson(JObject json) 121 | { 122 | Console.WriteLine(json.ToString()); 123 | } 124 | 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /TimberWinR.UnitTests/GeoIPFilterTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | using Newtonsoft.Json; 8 | using NUnit.Framework; 9 | using TimberWinR.Parser; 10 | using Newtonsoft.Json.Linq; 11 | 12 | namespace TimberWinR.UnitTests 13 | { 14 | [TestFixture] 15 | public class GeoIPFilterTests 16 | { 17 | [Test] 18 | public void TestDropConditions() 19 | { 20 | JObject jsonInputLine1 = new JObject 21 | { 22 | {"type", "Win32-FileLog"}, 23 | {"IP", "8.8.8.8"} 24 | }; 25 | 26 | 27 | string jsonFilter = @"{ 28 | ""TimberWinR"":{ 29 | ""Filters"":[ 30 | { 31 | ""geoip"":{ 32 | ""type"": ""Win32-FileLog"", 33 | ""target"": ""mygeoip"", 34 | ""source"": ""IP"" 35 | } 36 | }] 37 | } 38 | }"; 39 | 40 | // Positive Tests 41 | Configuration c = Configuration.FromString(jsonFilter); 42 | GeoIP jf = c.Filters.First() as GeoIP; 43 | Assert.IsTrue(jf.Apply(jsonInputLine1)); 44 | 45 | JObject stuff = jsonInputLine1["mygeoip"] as JObject; 46 | Assert.IsNotNull(stuff); 47 | 48 | Assert.AreEqual("8.8.8.8", stuff["ip"].ToString()); 49 | Assert.AreEqual("US", stuff["country_code2"].ToString()); 50 | Assert.AreEqual("United States", stuff["country_name"].ToString()); 51 | Assert.AreEqual("CA", stuff["region_name"].ToString()); 52 | Assert.AreEqual("Mountain View", stuff["city_name"].ToString()); 53 | Assert.AreEqual("California", stuff["real_region_name"].ToString()); 54 | Assert.AreEqual(37.386f, (float)stuff["latitude"]); 55 | Assert.AreEqual(-122.0838f, (float) stuff["longitude"]); 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /TimberWinR.UnitTests/Multiline1.txt: -------------------------------------------------------------------------------- 1 | multiline1 \ 2 | ml1_1 \ 3 | ml1_2 \ 4 | ml1_2 5 | singleline1 6 | singleline2 7 | multiline2 \ 8 | ml2_1 \ 9 | ml2_2 10 | multiline3 \ 11 | ml3_1 \ 12 | ml3_2 13 | singleline3 -------------------------------------------------------------------------------- /TimberWinR.UnitTests/Multiline2.txt: -------------------------------------------------------------------------------- 1 | 2015-01-07 13:14:26,572 TEST DEBUG [THREAD : 25] - Sending message to TServer - tcp://10.1111.11.111:1111 2 | 'RequestAttachUserData' ('30') 3 | message attributes: 4 | AttributeConnID [long] = 00890 5 | AttributeReferenceID [int] = 88 6 | AttributeThisDN [str] = "2214" 7 | AttributeUserData [bstr] = KVList: 8 | 'ActivityID' [str] = "1-XXXXXX" 9 | 2015-01-07 13:14:26,574 TEST DEBUG [THREAD : 25] - Writing message RequestAttachUserData in 'proxy1' via '.StatePrimary proxy: proxy1' 10 | 2015-01-07 13:14:26,575 TEST DEBUG [THREAD : 25] - sending RequestAttachUserData to Test.Platform.Commons.Connection.CommonConnection 11 | 2015-01-07 13:20:31,665 TEST DEBUG [THREAD : SelectorThread] - Proxy got message 'EventOnHook' ('87') 12 | message attributes: 13 | AttributeEventSequenceNumber [long] = 4899493 14 | Time = ComplexClass(TimeStamp): 15 | AttributeTimeinuSecs [int] = 573000 16 | AttributeTimeinSecs [int] = 1420644031 17 | AttributeThisDN [str] = "2214" 18 | . Processing with state .StatePrimary proxy: proxy1 19 | 2015-01-07 14:14:26,666 TEST DEBUG [THREAD : 25] - sending RequestAttachUserData to Test.Platform.Commons.Connection.CommonConnection 20 | -------------------------------------------------------------------------------- /TimberWinR.UnitTests/Parser/ElasticsearchOutputTests.cs: -------------------------------------------------------------------------------- 1 | using TimberWinR.Outputs; 2 | 3 | namespace TimberWinR.UnitTests.Parser 4 | { 5 | using System; 6 | 7 | using Newtonsoft.Json.Linq; 8 | 9 | using NUnit.Framework; 10 | 11 | using TimberWinR.Parser; 12 | 13 | public class ElasticsearchOutputTests 14 | { 15 | private ElasticsearchOutputParameters parser; 16 | 17 | [SetUp] 18 | public void Setup() 19 | { 20 | this.parser = new ElasticsearchOutputParameters(); 21 | } 22 | 23 | [Test] 24 | public void Given_no_index_returns_default_index_name() 25 | { 26 | this.parser.Index = "someindex"; 27 | var json = new JObject(); 28 | 29 | var result = this.parser.GetIndexName(json); 30 | 31 | Assert.AreEqual("someindex", result); 32 | } 33 | 34 | [Test] 35 | public void Given_index_with_date_format_and_timestamp_returns_name_by_timestamp() 36 | { 37 | this.parser.Index = "someindex-%{yyyy.MM.dd}"; 38 | var json = new JObject(); 39 | json.Add(new JProperty("@timestamp", "2011-11-30T18:45:32.450Z")); 40 | 41 | var result = this.parser.GetIndexName(json); 42 | 43 | Assert.AreEqual("someindex-2011.11.30", result); 44 | } 45 | 46 | [Test] 47 | public void Given_index_with_date_format_and_no_timestamp_returns_name_by_current_date() 48 | { 49 | this.parser.Index = "someindex-%{yyyy.MM.dd}"; 50 | var json = new JObject(); 51 | 52 | var result = this.parser.GetIndexName(json); 53 | 54 | Assert.AreEqual("someindex-" + DateTime.UtcNow.ToString("yyyy.MM.dd"), result); 55 | } 56 | 57 | [Test] 58 | public void Given_no_ssl_then_validate_does_not_throw() 59 | { 60 | parser.Ssl = false; 61 | Assert.That(() => parser.Validate(), Throws.Nothing); 62 | } 63 | 64 | [Test] 65 | public void Given_ssl_and_no_username_then_validate_throws() 66 | { 67 | parser.Ssl = true; 68 | parser.Password = "pass"; 69 | 70 | Assert.That(() => parser.Validate(), Throws.Exception.InstanceOf()); 71 | } 72 | 73 | [Test] 74 | public void Given_ssl_and_no_password_then_validate_throws() 75 | { 76 | parser.Ssl = true; 77 | parser.Username = "user"; 78 | 79 | Assert.That(() => parser.Validate(), Throws.Exception.InstanceOf()); 80 | } 81 | 82 | [Test] 83 | public void Given_ssl_and_username_and_password_then_validate_does_not_throw() 84 | { 85 | parser.Ssl = true; 86 | parser.Username = "user"; 87 | parser.Password = "pass"; 88 | 89 | Assert.That(() => parser.Validate(), Throws.Nothing); 90 | } 91 | 92 | [Test] 93 | [TestCase("host", 1234, false, null, null, "http://host:1234/")] 94 | [TestCase("host", 1234, true, "user", "pass", "https://user:pass@host:1234/")] 95 | [TestCase("host", 1234, true, "user:", "pass@", "https://user%3A:pass%40@host:1234/")] 96 | public void ComposeUri_Matches_Expected(string host, int port, bool ssl, string username, string password, string expectedUri) 97 | { 98 | var uri = ElasticsearchOutput.ComposeUri(host, port, ssl, username, password); 99 | 100 | Assert.That(uri.ToString(), Is.EqualTo(expectedUri)); 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /TimberWinR.UnitTests/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("TimberWinR.UnitTests")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("TimberWinR.UnitTests")] 13 | [assembly: AssemblyCopyright("Copyright © 2014")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("873f589f-b5f4-407b-a88b-9f051579737a")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | -------------------------------------------------------------------------------- /TimberWinR.UnitTests/TailFileTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | using Newtonsoft.Json; 9 | using NUnit.Framework; 10 | using TimberWinR.Inputs; 11 | using TimberWinR.Parser; 12 | using Newtonsoft.Json.Linq; 13 | using System.Diagnostics; 14 | 15 | namespace TimberWinR.UnitTests 16 | { 17 | [TestFixture] 18 | public class TailFileTests 19 | { 20 | [Test] 21 | public void TestTailFile() 22 | { 23 | List events = new List(); 24 | 25 | if (File.Exists(".timberwinrdb")) 26 | File.Delete(".timberwinrdb"); 27 | 28 | var mgr = new Manager(); 29 | mgr.LogfileDir = "."; 30 | 31 | var tf = new TailFileArguments(); 32 | var cancelTokenSource = new CancellationTokenSource(); 33 | tf.Location = "TestTailFile1.log"; 34 | 35 | 36 | if (File.Exists(tf.Location)) 37 | File.Delete(tf.Location); 38 | 39 | try 40 | { 41 | var listener = new TailFileListener(tf, cancelTokenSource.Token); 42 | 43 | listener.OnMessageRecieved += o => 44 | { 45 | events.Add(o); 46 | if (events.Count >= 100) 47 | cancelTokenSource.Cancel(); 48 | }; 49 | 50 | GenerateLogFile(tf.Location); 51 | 52 | bool createdFile = false; 53 | while (!listener.Stop && !cancelTokenSource.IsCancellationRequested) 54 | { 55 | Thread.Sleep(100); 56 | if (!createdFile) 57 | { 58 | GenerateLogFile(tf.Location); 59 | createdFile = true; 60 | } 61 | } 62 | } 63 | catch (OperationCanceledException) 64 | { 65 | Console.WriteLine("Done!"); 66 | } 67 | catch (Exception ex) 68 | { 69 | Console.WriteLine(ex.ToString()); 70 | 71 | } 72 | finally 73 | { 74 | Assert.AreEqual(100, events.Count); 75 | } 76 | } 77 | 78 | private static void GenerateLogFile(string fileName) 79 | { 80 | using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName)) 81 | { 82 | for (int i = 0; i < 100; i++) 83 | { 84 | file.WriteLine("Log Line Number {0}", i); 85 | } 86 | } 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /TimberWinR.UnitTests/TestDynamicBatchCount.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading.Tasks; 7 | using Newtonsoft.Json; 8 | using NUnit.Framework; 9 | using TimberWinR.Parser; 10 | using Newtonsoft.Json.Linq; 11 | using System.Threading; 12 | 13 | 14 | namespace TimberWinR.UnitTests 15 | { 16 | [TestFixture] 17 | public class TestDynamicBatchCount 18 | { 19 | // [Test] 20 | public void TestDynamicBatch() 21 | { 22 | var mgr = new Manager(); 23 | mgr.LogfileDir = "."; 24 | 25 | mgr.Config = new Configuration(); 26 | 27 | CancellationTokenSource cancelTokenSource = new CancellationTokenSource(); 28 | 29 | var cancelToken = cancelTokenSource.Token; 30 | 31 | FakeRediServer fr = new FakeRediServer(cancelToken); 32 | 33 | var redisParams = new RedisOutputParameters(); 34 | redisParams.BatchCount = 10; 35 | redisParams.MaxBatchCount = 40; 36 | redisParams.Interval = 100; 37 | 38 | var redisOutput = new Outputs.RedisOutput(mgr, redisParams, cancelToken); 39 | 40 | 41 | // Message is irrelavant 42 | JObject jsonMessage = new JObject 43 | { 44 | {"type", "Win32-FileLog"}, 45 | {"ComputerName", "dev.vistaprint.net"}, 46 | {"Text", "{\"Email\":\"james@example.com\",\"Active\":true,\"CreatedDate\":\"2013-01-20T00:00:00Z\",\"Roles\":[\"User\",\"Admin\"]}"} 47 | }; 48 | 49 | 50 | // Send 1000 messages at max throttle 51 | for (int i = 0; i < 1000; i++) 52 | { 53 | Thread.Sleep(10); 54 | redisOutput.Startup(jsonMessage); 55 | } 56 | 57 | while (redisOutput.SentMessages < 1000) 58 | { 59 | System.Diagnostics.Debug.WriteLine(redisOutput.SentMessages); 60 | Thread.Sleep(1000); 61 | } 62 | 63 | fr.Shutdown(); 64 | 65 | cancelTokenSource.Cancel(); 66 | 67 | System.Diagnostics.Debug.WriteLine(redisOutput.ToJson()); 68 | System.Diagnostics.Debug.WriteLine(redisOutput.QueueDepth); 69 | 70 | JObject json = redisOutput.ToJson(); 71 | var mbc = json["redis"]["reachedMaxBatchCountTimes"].Value(); 72 | var sm = json["redis"]["sentMessageCount"].Value(); 73 | var errs = json["redis"]["errors"].Value(); 74 | var cbc = json["redis"]["currentBatchCount"].Value(); 75 | 76 | // No errors 77 | Assert.AreEqual(0, errs); 78 | 79 | // Should have reached max at least 1 time 80 | Assert.GreaterOrEqual(mbc, 1); 81 | 82 | // Should have sent 1000 messages 83 | Assert.AreEqual(1000, sm); 84 | 85 | // Should reset back down to original 86 | Assert.AreEqual(cbc, 10); 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /TimberWinR.UnitTests/app.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /TimberWinR.UnitTests/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /TimberWinR/Codecs/JsonCodec.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using Newtonsoft.Json.Linq; 6 | using TimberWinR.Parser; 7 | 8 | namespace TimberWinR.Codecs 9 | { 10 | class JsonCodec : ICodec 11 | { 12 | private CodecArguments _codecArguments; 13 | 14 | public void Apply(string msg, Inputs.InputListener listener) 15 | { 16 | JObject jobject = JObject.Parse(msg); 17 | listener.AddDefaultFields(jobject); 18 | listener.ProcessJson(jobject); 19 | } 20 | 21 | public JsonCodec(CodecArguments args) 22 | { 23 | _codecArguments = args; 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /TimberWinR/Codecs/Multiline.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Text.RegularExpressions; 6 | using Newtonsoft.Json.Linq; 7 | using TimberWinR.Inputs; 8 | using TimberWinR.Parser; 9 | 10 | namespace TimberWinR.Codecs 11 | { 12 | public class Multiline : ICodec 13 | { 14 | private CodecArguments _codecArguments; 15 | private List _multiline { get; set; } 16 | 17 | // return true to cancel codec 18 | public Multiline(CodecArguments args) 19 | { 20 | _codecArguments = args; 21 | } 22 | 23 | public void Apply(string msg, InputListener listener) 24 | { 25 | if (_codecArguments.Re == null) 26 | _codecArguments.Re = new Regex(_codecArguments.Pattern); 27 | 28 | Match match = _codecArguments.Re.Match(msg); 29 | 30 | bool isMatch = (match.Success && !_codecArguments.Negate) || (!match.Success && _codecArguments.Negate); 31 | 32 | switch (_codecArguments.What) 33 | { 34 | case CodecArguments.WhatType.previous: 35 | if (isMatch) 36 | { 37 | if (_multiline == null) 38 | _multiline = new List(); 39 | 40 | _multiline.Add(msg); 41 | } 42 | else // No Match 43 | { 44 | if (_multiline != null) 45 | { 46 | string single = string.Join("\n", _multiline.ToArray()); 47 | _multiline = null; 48 | JObject jo = new JObject(); 49 | jo["message"] = single; 50 | jo.Add("tags", new JArray(_codecArguments.MultilineTag)); 51 | listener.AddDefaultFields(jo); 52 | listener.ProcessJson(jo); 53 | } 54 | _multiline = new List(); 55 | _multiline.Add(msg); 56 | } 57 | break; 58 | case CodecArguments.WhatType.next: 59 | if (isMatch) 60 | { 61 | if (_multiline == null) 62 | _multiline = new List(); 63 | _multiline.Add(msg); 64 | } 65 | else // No match 66 | { 67 | if (_multiline != null) 68 | { 69 | _multiline.Add(msg); 70 | string single = string.Join("\n", _multiline.ToArray()); 71 | _multiline = null; 72 | JObject jo = new JObject(); 73 | jo["message"] = single; 74 | jo.Add("tags", new JArray(_codecArguments.MultilineTag)); 75 | listener.AddDefaultFields(jo); 76 | listener.ProcessJson(jo); 77 | } 78 | else 79 | { 80 | JObject jo = new JObject(); 81 | jo["message"] = msg; 82 | listener.AddDefaultFields(jo); 83 | listener.ProcessJson(jo); 84 | } 85 | } 86 | break; 87 | } 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /TimberWinR/Codecs/PlainCodec.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using Newtonsoft.Json.Linq; 6 | using TimberWinR.Parser; 7 | 8 | namespace TimberWinR.Codecs 9 | { 10 | public class PlainCodec : ICodec 11 | { 12 | private CodecArguments _codecArguments; 13 | 14 | public void Apply(string msg, Inputs.InputListener listener) 15 | { 16 | JObject json = new JObject(); 17 | listener.AddDefaultFields(json); 18 | json["message"] = ExpandField(msg, json); 19 | listener.ProcessJson(json); 20 | } 21 | 22 | protected string ExpandField(string fieldName, JObject json) 23 | { 24 | foreach (var token in json.Children()) 25 | { 26 | string replaceString = "%{" + token.Path + "}"; 27 | fieldName = fieldName.Replace(replaceString, json[token.Path].ToString()); 28 | } 29 | return fieldName; 30 | } 31 | 32 | 33 | public PlainCodec(CodecArguments args) 34 | { 35 | _codecArguments = args; 36 | } 37 | 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /TimberWinR/ConfigurationErrors.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Xml; 3 | using System.Xml.Linq; 4 | 5 | namespace TimberWinR 6 | { 7 | public class ConfigurationErrors 8 | { 9 | public class MissingRequiredTagException : Exception 10 | { 11 | public MissingRequiredTagException(string tagName) 12 | : base( 13 | string.Format("Missing required tag \"{0}\"", tagName)) 14 | { 15 | } 16 | } 17 | 18 | public class MissingRequiredAttributeException : Exception 19 | { 20 | public MissingRequiredAttributeException(XElement e, string attributeName) 21 | : base( 22 | string.Format("{0}:{1} Missing required attribute \"{2}\" for element <{3}>", e.Document.BaseUri, 23 | ((IXmlLineInfo)e).LineNumber, attributeName, e.Name.ToString())) 24 | { 25 | } 26 | } 27 | 28 | public class InvalidAttributeNameException : Exception 29 | { 30 | public InvalidAttributeNameException(XAttribute a) 31 | : base( 32 | string.Format("{0}:{1} Invalid Attribute Name <{2} {3}>", a.Document.BaseUri, 33 | ((IXmlLineInfo)a).LineNumber, a.Parent.Name, a.Name.ToString())) 34 | { 35 | } 36 | } 37 | 38 | public class InvalidAttributeDateValueException : Exception 39 | { 40 | public InvalidAttributeDateValueException(XAttribute a) 41 | : base( 42 | string.Format( 43 | "{0}:{1} Invalid date format given for attribute. Format must be \"yyyy-MM-dd hh:mm:ss\". <{2} {3}>", 44 | a.Document.BaseUri, 45 | ((IXmlLineInfo)a).LineNumber, a.Parent.Name, a.ToString())) 46 | { 47 | } 48 | } 49 | 50 | public class InvalidAttributeIntegerValueException : Exception 51 | { 52 | public InvalidAttributeIntegerValueException(XAttribute a) 53 | : base( 54 | string.Format("{0}:{1} Integer value not given for attribute. <{2} {3}>", a.Document.BaseUri, 55 | ((IXmlLineInfo)a).LineNumber, a.Parent.Name, a.ToString())) 56 | { 57 | } 58 | } 59 | 60 | public class InvalidAttributeValueException : Exception 61 | { 62 | public InvalidAttributeValueException(XAttribute a) 63 | : base( 64 | string.Format("{0}:{1} Invalid Attribute Value <{2} {3}>", a.Document.BaseUri, 65 | ((IXmlLineInfo)a).LineNumber, a.Parent.Name, a.ToString())) 66 | { 67 | } 68 | } 69 | 70 | public class InvalidElementNameException : Exception 71 | { 72 | public InvalidElementNameException(XElement e) 73 | : base( 74 | string.Format("{0}:{1} Invalid Element Name <{2}> <{3}>", e.Document.BaseUri, 75 | ((IXmlLineInfo)e).LineNumber, e.Parent.Name, e.ToString())) 76 | { 77 | } 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /TimberWinR/Filters/DateFilter.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Globalization; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Xml; 7 | using Newtonsoft.Json.Linq; 8 | using System.Xml.Linq; 9 | using TimberWinR.Parser; 10 | using RapidRegex.Core; 11 | using System.Text.RegularExpressions; 12 | 13 | namespace TimberWinR.Parser 14 | { 15 | public partial class DateFilter : LogstashFilter 16 | { 17 | public override bool Apply(JObject json) 18 | { 19 | if (!string.IsNullOrEmpty(Type)) 20 | { 21 | JToken json_type = json["type"]; 22 | if (json_type != null && json_type.ToString() != Type) 23 | return true; // Filter does not apply. 24 | } 25 | 26 | if (Condition != null && !EvaluateCondition(json, Condition)) 27 | return true; 28 | 29 | if (Matches(json)) 30 | { 31 | AddFields(json); 32 | } 33 | 34 | return true; 35 | } 36 | 37 | public override JObject ToJson() 38 | { 39 | JObject json = new JObject( 40 | new JProperty("date", 41 | new JObject( 42 | new JProperty("condition", Condition), 43 | new JProperty("type", Type), 44 | new JProperty("addfields", AddField) 45 | ))); 46 | return json; 47 | } 48 | 49 | // copy_field "field1" -> "field2" 50 | private void AddFields(Newtonsoft.Json.Linq.JObject json) 51 | { 52 | string srcField = Match[0]; 53 | 54 | if (AddField != null && AddField.Length > 0) 55 | { 56 | for (int i = 0; i < AddField.Length; i++) 57 | { 58 | string dstField = ExpandField(AddField[i], json); 59 | if (json[srcField] != null) 60 | AddOrModify(json, dstField, json[srcField]); 61 | } 62 | } 63 | } 64 | 65 | 66 | private bool Matches(Newtonsoft.Json.Linq.JObject json) 67 | { 68 | string field = Match[0]; 69 | 70 | CultureInfo ci = CultureInfo.CreateSpecificCulture(Locale); 71 | 72 | JToken token = null; 73 | if (json.TryGetValue(field, out token)) 74 | { 75 | string text = token.ToString(); 76 | if (!string.IsNullOrEmpty(text)) 77 | { 78 | DateTime ts; 79 | var exprArray = Match.Skip(1).ToArray(); 80 | var resolver = new RegexGrokResolver(); 81 | for (int i=0; i 0) 55 | { 56 | for (int i = 0; i < Remove.Length; i += 1) 57 | { 58 | string name = ExpandField(Remove[i], json); 59 | RemoveProperty(json, name); 60 | } 61 | } 62 | } 63 | 64 | private void ApplyRenames(JObject json) 65 | { 66 | if (Rename != null && Rename.Length > 0) 67 | { 68 | for (int i = 0; i < Rename.Length; i += 2) 69 | { 70 | string oldName = ExpandField(Rename[i], json); 71 | string newName = ExpandField(Rename[i + 1], json); 72 | RenameProperty(json, oldName, newName); 73 | } 74 | } 75 | } 76 | 77 | private void ApplySplits(JObject json) 78 | { 79 | if (Split != null && Split.Length > 0) 80 | { 81 | for (int i = 0; i < Split.Length; i += 2) 82 | { 83 | string fieldName = Split[i]; 84 | string splitChar = Split[i + 1]; 85 | 86 | JArray array = null; 87 | if (json[fieldName] != null) 88 | { 89 | string valueToSplit = json[fieldName].ToString(); 90 | string[] values = valueToSplit.Split(new string[] {splitChar}, StringSplitOptions.None); 91 | foreach (string value in values) 92 | { 93 | if (array == null) 94 | array = new JArray(value); 95 | else 96 | array.Add(value); 97 | } 98 | 99 | } 100 | } 101 | } 102 | } 103 | 104 | 105 | private void ApplyReplace(JObject json) 106 | { 107 | if (Replace != null && Replace.Length > 0) 108 | { 109 | for (int i = 0; i < Replace.Length; i += 2) 110 | { 111 | string fieldName = Replace[0]; 112 | string replaceValue = ExpandField(Replace[i + 1], json); 113 | ReplaceProperty(json, fieldName, replaceValue); 114 | } 115 | } 116 | } 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /TimberWinR/GeoLite2City.mmdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TerribleDev/TimberWinR/5165224c65c81d7e3f26df9f2bc5e6415cc8ceb6/TimberWinR/GeoLite2City.mmdb -------------------------------------------------------------------------------- /TimberWinR/ICodec.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using TimberWinR.Inputs; 6 | 7 | namespace TimberWinR 8 | { 9 | public interface ICodec 10 | { 11 | void Apply(string msg, InputListener listener); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /TimberWinR/Inputs/FieldDefinitions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | 6 | namespace TimberWinR.Inputs 7 | { 8 | /// 9 | /// Field Definition 10 | /// 11 | public class FieldDefinition 12 | { 13 | public string Name { get; set; } 14 | public Type FieldType { get; set; } 15 | 16 | public DateTime ToDateTime(object o) 17 | { 18 | return (DateTime) o; 19 | } 20 | 21 | public int ToInt(object o) 22 | { 23 | return (int) o; 24 | } 25 | 26 | public string ToString(object o) 27 | { 28 | return (string) o; 29 | } 30 | 31 | public float ToFloat(object o) 32 | { 33 | return (float)o; 34 | } 35 | public FieldDefinition(string fieldName, Type fieldType) 36 | { 37 | Name = fieldName; 38 | FieldType = fieldType; 39 | } 40 | } 41 | 42 | public class FieldDefinitions : IEnumerable 43 | { 44 | private List _fields; 45 | 46 | public FieldDefinition this[int index] 47 | { 48 | get { return _fields[index]; } 49 | set { _fields.Insert(index, value); } 50 | } 51 | 52 | public void Add(string fieldName, Type fieldType) 53 | { 54 | _fields.Add(new FieldDefinition(fieldName, fieldType)); 55 | } 56 | 57 | public FieldDefinitions() 58 | { 59 | _fields = new List(); 60 | } 61 | 62 | public IEnumerator GetEnumerator() 63 | { 64 | return _fields.GetEnumerator(); 65 | } 66 | 67 | System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() 68 | { 69 | return this.GetEnumerator(); 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /TimberWinR/Inputs/GeneratorInput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading; 6 | 7 | using Newtonsoft.Json; 8 | using Newtonsoft.Json.Linq; 9 | using NLog; 10 | using RestSharp.Extensions; 11 | using TimberWinR.Codecs; 12 | using TimberWinR.Parser; 13 | 14 | 15 | namespace TimberWinR.Inputs 16 | { 17 | public class GeneratorInput : InputListener 18 | { 19 | public override JObject ToJson() 20 | { 21 | JObject json = new JObject( 22 | new JProperty("message", _params.Message), 23 | new JProperty("messages", _sentMessages), 24 | new JProperty("generator", "enabled")); 25 | return json; 26 | } 27 | 28 | private TimberWinR.Parser.GeneratorParameters _params; 29 | private Thread _listenThread; 30 | private ICodec _codec; 31 | private int _sentMessages; 32 | 33 | public GeneratorInput(TimberWinR.Parser.GeneratorParameters parameters, CancellationToken cancelToken) 34 | : base(cancelToken, "Win32-InputGen") 35 | { 36 | _params = parameters; 37 | 38 | if (_params.CodecArguments != null) 39 | { 40 | switch (_params.CodecArguments.Type) 41 | { 42 | case CodecArguments.CodecType.json: 43 | _codec = new JsonCodec(_params.CodecArguments); 44 | break; 45 | case CodecArguments.CodecType.multiline: 46 | _codec = new Multiline(_params.CodecArguments); 47 | break; 48 | case CodecArguments.CodecType.plain: 49 | _codec = new PlainCodec(_params.CodecArguments); 50 | break; 51 | } 52 | } 53 | 54 | _listenThread = new Thread(new ThreadStart(GenerateData)); 55 | _listenThread.Start(); 56 | } 57 | 58 | private void GenerateData() 59 | { 60 | LogManager.GetCurrentClassLogger().Info("Generator Creating {0} Lines", _params.Count); 61 | 62 | int numMessages = _params.Count; 63 | 64 | // Infinite or until done. 65 | for (int i = 0; (_params.Count == 0 || i < numMessages); i++) 66 | { 67 | if (CancelToken.IsCancellationRequested) 68 | break; 69 | 70 | string msg = ToPrintable(_params.Message); 71 | 72 | if (_codec != null) 73 | _codec.Apply(msg, this); 74 | else 75 | { 76 | JObject jo = new JObject(); 77 | jo["Message"] = msg; 78 | AddDefaultFields(jo); 79 | ProcessJson(jo); 80 | } 81 | 82 | Thread.Sleep(_params.Rate); 83 | } 84 | 85 | Finished(); 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /TimberWinR/Inputs/IISLog.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Globalization; 4 | using System.Text; 5 | using System.Xml.Linq; 6 | 7 | namespace TimberWinR.Inputs 8 | { 9 | public class IISLog : InputBase 10 | { 11 | public const string ParentTagName = "IISLogs"; 12 | public new const string TagName = "IISLog"; 13 | 14 | public string Name { get; private set; } 15 | public string Location { get; private set; } 16 | public int ICodepage { get; private set; } 17 | public int Recurse { get; private set; } 18 | public string MinDateMod { get; private set; } 19 | public string Locale { get; private set; } 20 | public string ICheckpoint { get; private set; } 21 | public List Fields { get; private set; } 22 | 23 | public static void Parse(List iislogs, XElement iislogElement) 24 | { 25 | iislogs.Add(parseIISLog(iislogElement)); 26 | } 27 | 28 | static IISLog parseIISLog(XElement e) 29 | { 30 | return new IISLog(e); 31 | } 32 | 33 | public IISLog(XElement parent) 34 | { 35 | Name = ParseRequiredStringAttribute(parent, "name"); 36 | Location = ParseRequiredStringAttribute(parent, "location"); 37 | ICodepage = ParseIntAttribute(parent, "iCodepage", -2); 38 | Recurse = ParseIntAttribute(parent, "recurse", 0); 39 | MinDateMod = ParseDateAttribute(parent, "minDateMod"); 40 | Locale = ParseStringAttribute(parent, "locale", "DEF"); 41 | ICheckpoint = ParseStringAttribute(parent, "iCheckpoint"); 42 | parseFields(parent); 43 | } 44 | 45 | private void parseFields(XElement parent) 46 | { 47 | Dictionary allPossibleFields = new Dictionary() 48 | { 49 | { "LogFilename", typeof(string) }, 50 | { "LogRow", typeof(int) }, 51 | { "UserIP", typeof(string) }, 52 | { "UserName", typeof(string) }, 53 | { "Date", typeof(DateTime) }, 54 | { "Time", typeof(DateTime) }, 55 | { "ServiceInstance", typeof(string) }, 56 | { "HostName", typeof(string) }, 57 | { "ServerIP", typeof(string) }, 58 | { "TimeTaken", typeof(int) }, 59 | { "BytesSent", typeof(int) }, 60 | { "BytesReceived", typeof(int) }, 61 | { "StatusCode", typeof(int) }, 62 | { "Win32StatusCode", typeof(int) }, 63 | { "RequestType", typeof(string) }, 64 | { "Target", typeof(string) }, 65 | { "Parameters", typeof(string) } 66 | }; 67 | 68 | Fields = ParseFields(parent, allPossibleFields); 69 | } 70 | 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /TimberWinR/Inputs/IISW3CLog.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using System.Xml.Linq; 5 | 6 | namespace TimberWinR.Inputs 7 | { 8 | public class IISW3CLog : InputBase 9 | { 10 | public const string ParentTagName = "IISW3CLogs"; 11 | public new const string TagName = "IISW3CLog"; 12 | 13 | public string Name { get; private set; } 14 | public string Location { get; private set; } 15 | public int ICodepage { get; private set; } 16 | public int Recurse { get; private set; } 17 | public string MinDateMod { get; private set; } 18 | public bool DQuotes { get; private set; } 19 | public bool DirTime { get; private set; } 20 | public bool ConsolidateLogs { get; private set; } 21 | public string ICheckpoint { get; private set; } 22 | public List Fields { get; private set; } 23 | 24 | public static void Parse(List iisw3clogs, XElement iisw3clogElement) 25 | { 26 | iisw3clogs.Add(parseIISW3CLog(iisw3clogElement)); 27 | } 28 | 29 | static IISW3CLog parseIISW3CLog(XElement e) 30 | { 31 | return new IISW3CLog(e); 32 | } 33 | 34 | public IISW3CLog(XElement parent) 35 | { 36 | Name = ParseRequiredStringAttribute(parent, "name"); 37 | Location = ParseRequiredStringAttribute(parent, "location"); 38 | ICodepage = ParseIntAttribute(parent, "iCodepage", -2); 39 | Recurse = ParseIntAttribute(parent, "recurse", 0); 40 | DQuotes = ParseBoolAttribute(parent, "dQuotes", false); 41 | DirTime = ParseBoolAttribute(parent, "dirTime", false); 42 | ConsolidateLogs = ParseBoolAttribute(parent, "consolidateLogs", false); 43 | ICheckpoint = ParseStringAttribute(parent, "iCheckpoint"); 44 | parseFields(parent); 45 | } 46 | 47 | private void parseFields(XElement parent) 48 | { 49 | Dictionary allPossibleFields = new Dictionary() 50 | { 51 | { "LogFilename", typeof(string) }, 52 | { "LogRow", typeof(int) }, 53 | { "date", typeof(DateTime) }, 54 | { "time", typeof(DateTime) }, 55 | { "c-ip", typeof(string) }, 56 | { "cs-username", typeof(string) }, 57 | { "s-sitename", typeof(string) }, 58 | { "s-computername", typeof(int) }, 59 | { "s-ip", typeof(string) }, 60 | { "s-port", typeof(int) }, 61 | { "cs-method", typeof(string) }, 62 | { "cs-uri-stem", typeof(string) }, 63 | { "cs-uri-query", typeof(string) }, 64 | { "sc-status", typeof(int) }, 65 | { "sc-substatus", typeof(int) }, 66 | { "sc-win32-status", typeof(int) }, 67 | { "sc-bytes", typeof(int) }, 68 | { "cs-bytes", typeof(int) }, 69 | { "time-taken", typeof(int) }, 70 | { "cs-version", typeof(string) }, 71 | { "cs-host", typeof(string) }, 72 | { "cs(User-Agent)", typeof(string) }, 73 | { "cs(Cookie)", typeof(string) }, 74 | { "cs(Referer)", typeof(string) }, 75 | { "s-event", typeof(string) }, 76 | { "s-process-type", typeof(string) }, 77 | { "s-user-time", typeof(double) }, 78 | { "s-kernel-time", typeof(double) }, 79 | { "s-page-faults", typeof(int) }, 80 | { "s-total-procs", typeof(int) }, 81 | { "s-active-procs", typeof(int) }, 82 | { "s-stopped-procs", typeof(int) } 83 | }; 84 | 85 | Fields = ParseFields(parent, allPossibleFields); 86 | } 87 | 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /TimberWinR/Inputs/IISW3CRowReader.cs: -------------------------------------------------------------------------------- 1 | namespace TimberWinR.Inputs 2 | { 3 | using System; 4 | using System.Collections.Generic; 5 | 6 | using Interop.MSUtil; 7 | 8 | using Newtonsoft.Json.Linq; 9 | 10 | using TimberWinR.Parser; 11 | 12 | public class IisW3CRowReader 13 | { 14 | private readonly List fields; 15 | private IDictionary columnMap; 16 | 17 | public IisW3CRowReader(List fields) 18 | { 19 | this.fields = fields; 20 | } 21 | 22 | public JObject ReadToJson(ILogRecord row) 23 | { 24 | var json = new JObject(); 25 | foreach (var field in this.fields) 26 | { 27 | if (this.columnMap.ContainsKey(field.Name)) 28 | { 29 | object v = row.getValue(field.Name); 30 | if (field.DataType == typeof(DateTime)) 31 | { 32 | DateTime dt = DateTime.Parse(v.ToString()); 33 | json.Add(new JProperty(field.Name, dt)); 34 | } 35 | else 36 | { 37 | json.Add(new JProperty(field.Name, v)); 38 | } 39 | } 40 | } 41 | 42 | AddTimestamp(json); 43 | 44 | return json; 45 | } 46 | 47 | public void ReadColumnMap(ILogRecordset rs) 48 | { 49 | this.columnMap = new Dictionary(); 50 | for (int col = 0; col < rs.getColumnCount(); col++) 51 | { 52 | string colName = rs.getColumnName(col); 53 | this.columnMap[colName] = col; 54 | } 55 | } 56 | 57 | private static void AddTimestamp(JObject json) 58 | { 59 | if (json["date"] != null && json["time"] != null) 60 | { 61 | var date = DateTime.Parse(json["date"].ToString()); 62 | var time = DateTime.Parse(json["time"].ToString()); 63 | date = new DateTime(date.Year, date.Month, date.Day, time.Hour, time.Minute, time.Second, time.Millisecond); 64 | 65 | json.Add(new JProperty("@timestamp", date.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"))); 66 | } 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /TimberWinR/Inputs/InputBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Data.Common; 4 | using System.Globalization; 5 | using System.Linq; 6 | using System.Text; 7 | using System.Xml.Linq; 8 | 9 | namespace TimberWinR.Inputs 10 | { 11 | public abstract class InputBase 12 | { 13 | public const string TagName = "Inputs"; 14 | 15 | public override string ToString() 16 | { 17 | StringBuilder sb = new StringBuilder(); 18 | sb.Append(String.Format("{0}\n", this.GetType().ToString())); 19 | sb.Append("Parameters:\n"); 20 | foreach (var prop in this.GetType().GetProperties()) 21 | { 22 | if (prop != null) 23 | { 24 | if (prop.Name == "Fields") 25 | { 26 | sb.Append(String.Format("{0}:\n", prop.Name)); 27 | foreach (FieldDefinition f in (List)prop.GetValue(this, null)) 28 | { 29 | sb.Append(String.Format("\t{0}\n", f.Name)); 30 | } 31 | } 32 | else 33 | { 34 | sb.Append(String.Format("\t{0}: {1}\n", prop.Name, prop.GetValue(this, null))); 35 | } 36 | } 37 | } 38 | return sb.ToString(); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /TimberWinR/Inputs/ParameterDefinitions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Reflection; 5 | using System.Text; 6 | 7 | namespace TimberWinR.Inputs 8 | { 9 | public class ParameterValue 10 | { 11 | public ParameterDefinition Parameter { get; private set; } 12 | protected string CurrentValue { get; set; } 13 | 14 | protected ParameterValue(ParameterDefinition paramDef) 15 | { 16 | Parameter = paramDef; 17 | CurrentValue = Parameter.DefaultValue; 18 | } 19 | 20 | public override string ToString() 21 | { 22 | return string.Format("-{0}:{1}", Parameter.Name, CurrentValue); 23 | } 24 | } 25 | 26 | public class BooleanParameterValue : ParameterValue 27 | { 28 | public BooleanParameterValue(ParameterDefinition paramDef) 29 | : base(paramDef) 30 | { 31 | } 32 | 33 | public bool Value 34 | { 35 | get 36 | { 37 | return CurrentValue.Equals(Parameter.LegalValues[Parameter.LegalValues.Count - 1], 38 | StringComparison.OrdinalIgnoreCase); 39 | } 40 | set 41 | { 42 | string v = Parameter.LegalValues[0]; 43 | if (value) 44 | v = Parameter.LegalValues[1]; 45 | 46 | CurrentValue = v; 47 | } 48 | } 49 | 50 | } 51 | 52 | public class EnumParameterValue : ParameterValue 53 | { 54 | public EnumParameterValue(ParameterDefinition paramDef) 55 | : base(paramDef) 56 | { 57 | } 58 | 59 | public string Value 60 | { 61 | get { return CurrentValue; } 62 | set 63 | { 64 | if (Parameter.LegalValues.Contains(value)) 65 | CurrentValue = value; 66 | else 67 | throw new ArgumentException(string.Format("Illegal value: '{0}'", value), Parameter.Name); 68 | } 69 | } 70 | 71 | } 72 | 73 | public class TimestampParameterValue : ParameterValue 74 | { 75 | public TimestampParameterValue(ParameterDefinition paramDef) 76 | : base(paramDef) 77 | { 78 | } 79 | 80 | public DateTime Value 81 | { 82 | get { return DateTime.Parse(CurrentValue); } 83 | set { CurrentValue = value.ToString(); } 84 | } 85 | } 86 | 87 | public class ParameterDefinition 88 | { 89 | public string Name { get; set; } 90 | public string DefaultValue { get; set; } 91 | public List LegalValues { get; set; } 92 | public Type ParameterType { get; set; } 93 | 94 | public ParameterDefinition(string name, string defaultValue, Type parameterType) 95 | { 96 | Name = name; 97 | DefaultValue = defaultValue; 98 | LegalValues = null; 99 | ParameterType = parameterType; 100 | } 101 | 102 | public ParameterDefinition(string name, string defaultValue, Type parameterType, IEnumerable legalValues) 103 | { 104 | Name = name; 105 | DefaultValue = defaultValue; 106 | LegalValues = legalValues.ToList(); 107 | ParameterType = parameterType; 108 | } 109 | } 110 | 111 | public class ParameterDefinitions : IEnumerable 112 | { 113 | private readonly List _params; 114 | 115 | public ParameterDefinition this[int index] 116 | { 117 | get { return _params[index]; } 118 | set { _params.Insert(index, value); } 119 | } 120 | 121 | public void Add(string paramName, string defaultValue, Type parameterType) 122 | { 123 | _params.Add(new ParameterDefinition(paramName, defaultValue, parameterType)); 124 | } 125 | 126 | public void Add(string paramName, string defaultValue, Type parameterType, IEnumerable legalValues) 127 | { 128 | _params.Add(new ParameterDefinition(paramName, defaultValue, parameterType, legalValues)); 129 | } 130 | 131 | public ParameterDefinitions() 132 | { 133 | _params = new List(); 134 | } 135 | 136 | public IEnumerator GetEnumerator() 137 | { 138 | return _params.GetEnumerator(); 139 | } 140 | 141 | System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() 142 | { 143 | return this.GetEnumerator(); 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /TimberWinR/Inputs/StdinListener.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.Linq; 5 | using System.Runtime.InteropServices; 6 | using System.Text; 7 | using System.Text.RegularExpressions; 8 | using System.Threading; 9 | using Newtonsoft.Json; 10 | using Newtonsoft.Json.Linq; 11 | using NLog; 12 | using TimberWinR.Codecs; 13 | using TimberWinR.Parser; 14 | 15 | namespace TimberWinR.Inputs 16 | { 17 | public class StdinListener : InputListener 18 | { 19 | [DllImport("User32.Dll", EntryPoint = "PostMessageA")] 20 | private static extern bool PostMessage(IntPtr hWnd, uint msg, int wParam, int lParam); 21 | 22 | private Thread _listenThread; 23 | private CodecArguments _codecArguments; 24 | private ICodec _codec; 25 | const int VK_RETURN = 0x0D; 26 | const int WM_KEYDOWN = 0x100; 27 | 28 | public StdinListener(TimberWinR.Parser.Stdin arguments, CancellationToken cancelToken) 29 | : base(cancelToken, "Win32-Console") 30 | { 31 | _codecArguments = arguments.CodecArguments; 32 | if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline) 33 | _codec = new Multiline(_codecArguments); 34 | 35 | _listenThread = new Thread(new ThreadStart(ListenToStdin)); 36 | _listenThread.Start(); 37 | } 38 | 39 | public override JObject ToJson() 40 | { 41 | JObject json = new JObject( 42 | new JProperty("stdin", "enabled")); 43 | 44 | 45 | if (_codecArguments != null) 46 | { 47 | var cp = new JProperty("codec", 48 | new JArray( 49 | new JObject( 50 | new JProperty("type", _codecArguments.Type.ToString()), 51 | new JProperty("what", _codecArguments.What.ToString()), 52 | new JProperty("negate", _codecArguments.Negate), 53 | new JProperty("multilineTag", _codecArguments.MultilineTag), 54 | new JProperty("pattern", _codecArguments.Pattern)))); 55 | json.Add(cp); 56 | } 57 | 58 | return json; 59 | } 60 | 61 | public override void Shutdown() 62 | { 63 | LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType); 64 | // This must come from another thread. 65 | ThreadPool.QueueUserWorkItem((o) => 66 | { 67 | Thread.Sleep(100); 68 | var hWnd = System.Diagnostics.Process.GetCurrentProcess().MainWindowHandle; 69 | PostMessage(hWnd, WM_KEYDOWN, VK_RETURN, 0); 70 | }); 71 | base.Shutdown(); 72 | } 73 | 74 | private void ListenToStdin() 75 | { 76 | LogManager.GetCurrentClassLogger().Info("StdIn Ready"); 77 | 78 | while (!CancelToken.IsCancellationRequested) 79 | { 80 | string line = Console.ReadLine(); 81 | if (line != null) 82 | { 83 | string msg = ToPrintable(line); 84 | 85 | if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline) 86 | _codec.Apply(msg, this); 87 | else 88 | { 89 | JObject jo = new JObject(); 90 | jo["message"] = msg; 91 | AddDefaultFields(jo); 92 | ProcessJson(jo); 93 | } 94 | } 95 | } 96 | Finished(); 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /TimberWinR/Inputs/TailFileInput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using System.Xml.Linq; 5 | 6 | namespace TimberWinR.Inputs 7 | { 8 | public class TailFileInput : InputBase 9 | { 10 | public const string ParentTagName = "Logs"; 11 | public new const string TagName = "Log"; 12 | 13 | public string Name { get; private set; } 14 | public string Location { get; private set; } 15 | public int ICodepage { get; private set; } 16 | public int Recurse { get; private set; } 17 | public bool SplitLongLines { get; private set; } 18 | public List Fields { get; private set; } 19 | 20 | public static void Parse(List logs, XElement logElement) 21 | { 22 | logs.Add(parseLog(logElement)); 23 | } 24 | 25 | static TailFileInput parseLog(XElement e) 26 | { 27 | return new TailFileInput(e); 28 | } 29 | 30 | public TailFileInput(XElement parent) 31 | { 32 | Name = ParseRequiredStringAttribute(parent, "name"); 33 | Location = ParseRequiredStringAttribute(parent, "location"); 34 | ICodepage = ParseIntAttribute(parent, "iCodepage", 0); 35 | Recurse = ParseIntAttribute(parent, "recurse", 0); 36 | SplitLongLines = ParseBoolAttribute(parent, "splitLongLines", false); 37 | parseFields(parent); 38 | } 39 | 40 | private void parseFields(XElement parent) 41 | { 42 | Dictionary allPossibleFields = new Dictionary() 43 | { 44 | { "LogFilename", typeof(string) }, 45 | { "Index", typeof(int) }, 46 | { "Text", typeof(string) } 47 | }; 48 | 49 | Fields = ParseFields(parent, allPossibleFields); 50 | } 51 | 52 | } 53 | } -------------------------------------------------------------------------------- /TimberWinR/Inputs/WindowsEvent.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Text; 4 | using System.Xml.Linq; 5 | using Microsoft.SqlServer.Server; 6 | 7 | namespace TimberWinR.Inputs 8 | { 9 | public class WindowsEvent : InputBase 10 | { 11 | public const string ParentTagName = "WindowsEvents"; 12 | public new const string TagName = "Event"; 13 | 14 | public string Source { get; private set; } 15 | public bool FullText { get; private set; } 16 | public bool ResolveSIDS { get; private set; } 17 | public bool FormatMsg { get; private set; } 18 | public string MsgErrorMode { get; private set; } 19 | public bool FullEventCode { get; private set; } 20 | public string Direction { get; private set; } 21 | public string StringsSep { get; private set; } 22 | public string ICheckpoint { get; private set; } 23 | public string BinaryFormat { get; private set; } 24 | public List Fields { get; private set; } 25 | 26 | public static void Parse(List events, XElement eventElement) 27 | { 28 | events.Add(parseEvent(eventElement)); 29 | } 30 | 31 | static WindowsEvent parseEvent(XElement e) 32 | { 33 | return new WindowsEvent(e); 34 | } 35 | 36 | WindowsEvent(XElement parent) 37 | { 38 | Source = ParseRequiredStringAttribute(parent, "source"); 39 | FullText = ParseBoolAttribute(parent, "fullText", true); 40 | ResolveSIDS = ParseBoolAttribute(parent, "resolveSIDS", true); 41 | FormatMsg = ParseBoolAttribute(parent, "formatMsg", true); 42 | MsgErrorMode = ParseEnumAttribute(parent, "msgErrorMode", new string[] {"NULL", "ERROR", "MSG"}, "MSG"); 43 | FullEventCode = ParseBoolAttribute(parent, "fullEventCode", false); ; 44 | Direction = ParseEnumAttribute(parent, "direction", new string[] { "FW", "BW" }, "FW"); 45 | StringsSep = ParseStringAttribute(parent, "stringsSep", "|"); 46 | BinaryFormat = ParseEnumAttribute(parent, "binaryFormat", new string[] { "ASC", "PRINT", "HEX" }, "PRINT"); 47 | parseFields(parent); 48 | } 49 | 50 | private void parseFields(XElement parent) 51 | { 52 | Dictionary allPossibleFields = new Dictionary() 53 | { 54 | { "EventLog", typeof(string) }, 55 | { "RecordNumber", typeof(int) }, 56 | { "TimeGenerated", typeof(DateTime) }, 57 | { "TimeWritten", typeof(DateTime) }, 58 | { "EventID", typeof(int) }, 59 | { "EventType", typeof(int) }, 60 | { "EventTypeName", typeof(string) }, 61 | { "EventCategory", typeof(int) }, 62 | { "EventCategoryName", typeof(string) }, 63 | { "SourceName", typeof(string) }, 64 | { "Strings", typeof(string) }, 65 | { "ComputerName", typeof(string) }, 66 | { "SID", typeof(string) }, 67 | { "Message", typeof(string) }, 68 | { "Data", typeof(string) } 69 | }; 70 | 71 | Fields = ParseFields(parent, allPossibleFields); 72 | } 73 | 74 | } 75 | } -------------------------------------------------------------------------------- /TimberWinR/LogErrors.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using Newtonsoft.Json; 6 | using Newtonsoft.Json.Linq; 7 | using Newtonsoft.Json.Serialization; 8 | using TimberWinR.Parser; 9 | 10 | namespace TimberWinR 11 | { 12 | public class LogErrors 13 | { 14 | public static JObject LogException(Exception ex) 15 | { 16 | return LogException("Exception", ex); 17 | } 18 | 19 | public static JObject LogException(string errorMessage, Exception ex) 20 | { 21 | JObject result = new JObject(); 22 | result["type"] = "TimberWinR-Error"; 23 | result["ErrorMessage"] = errorMessage; 24 | 25 | try 26 | { 27 | JsonConvert.DefaultSettings = () => new JsonSerializerSettings 28 | { 29 | Formatting = Formatting.Indented, 30 | NullValueHandling = NullValueHandling.Ignore 31 | }; 32 | 33 | var exJson = JObject.Parse(JsonConvert.SerializeObject(ex)); 34 | 35 | result.Merge(exJson, new JsonMergeSettings 36 | { 37 | MergeArrayHandling = MergeArrayHandling.Replace 38 | }); 39 | return result; 40 | } 41 | catch (Exception ex1) 42 | { 43 | result["ErrorMessage"] = ex1.ToString(); 44 | return result; 45 | } 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /TimberWinR/Outputs/OutputSender.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | using System.Threading; 6 | using Newtonsoft.Json.Linq; 7 | using TimberWinR.Inputs; 8 | 9 | namespace TimberWinR.Outputs 10 | { 11 | public abstract class OutputSender 12 | { 13 | public CancellationToken CancelToken { get; private set; } 14 | private List _inputs; 15 | public string Name { get; set; } 16 | 17 | public OutputSender(CancellationToken cancelToken, string name) 18 | { 19 | CancelToken = cancelToken; 20 | Name = name; 21 | _inputs = new List(); 22 | } 23 | 24 | public void Connect(InputListener listener) 25 | { 26 | listener.OnMessageRecieved += MessageReceivedHandler; 27 | } 28 | 29 | public void Startup(JObject json) 30 | { 31 | MessageReceivedHandler(json); 32 | } 33 | 34 | public abstract JObject ToJson(); 35 | protected abstract void MessageReceivedHandler(JObject jsonMessage); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /TimberWinR/Outputs/Stdout.cs: -------------------------------------------------------------------------------- 1 | using Newtonsoft.Json.Linq; 2 | using NLog; 3 | using System; 4 | using System.Collections.Generic; 5 | using System.Linq; 6 | using System.Threading; 7 | using System.Threading.Tasks; 8 | 9 | namespace TimberWinR.Outputs 10 | { 11 | public class StdoutOutput : OutputSender 12 | { 13 | private TimberWinR.Manager _manager; 14 | private readonly int _interval; 15 | private readonly object _locker = new object(); 16 | private readonly List _jsonQueue; 17 | private long _sentMessages; 18 | public bool Stop { get; set; } 19 | 20 | public StdoutOutput(TimberWinR.Manager manager, Parser.StdoutOutputParameters eo, CancellationToken cancelToken) 21 | : base(cancelToken, "Stdout") 22 | { 23 | _sentMessages = 0; 24 | _manager = manager; 25 | _interval = eo.Interval; 26 | _jsonQueue = new List(); 27 | 28 | var elsThread = new Task(StdoutSender, cancelToken); 29 | elsThread.Start(); 30 | } 31 | 32 | public override JObject ToJson() 33 | { 34 | JObject json = new JObject( 35 | new JProperty("stdout", 36 | new JObject( 37 | new JProperty("queuedMessageCount", _jsonQueue.Count), 38 | new JProperty("sentMessageCount", _sentMessages)))); 39 | 40 | return json; 41 | } 42 | 43 | // 44 | // Pull off messages from the Queue, batch them up and send them all across 45 | // 46 | private void StdoutSender() 47 | { 48 | using (var syncHandle = new ManualResetEventSlim()) 49 | { 50 | // Execute the query 51 | while (!Stop) 52 | { 53 | if (!CancelToken.IsCancellationRequested) 54 | { 55 | try 56 | { 57 | JObject[] messages; 58 | lock (_locker) 59 | { 60 | messages = _jsonQueue.Take(_jsonQueue.Count).ToArray(); 61 | _jsonQueue.RemoveRange(0, messages.Length); 62 | } 63 | 64 | if (messages.Length > 0) 65 | { 66 | try 67 | { 68 | foreach (JObject obj in messages) 69 | { 70 | Console.WriteLine(obj.ToString()); 71 | Interlocked.Increment(ref _sentMessages); 72 | } 73 | } 74 | catch (Exception ex) 75 | { 76 | LogManager.GetCurrentClassLogger().Error(ex); 77 | } 78 | } 79 | if (!Stop) 80 | syncHandle.Wait(TimeSpan.FromMilliseconds(_interval), CancelToken); 81 | } 82 | catch (OperationCanceledException) 83 | { 84 | break; 85 | } 86 | catch (Exception) 87 | { 88 | } 89 | } 90 | } 91 | } 92 | } 93 | 94 | protected override void MessageReceivedHandler(Newtonsoft.Json.Linq.JObject jsonMessage) 95 | { 96 | if (_manager.Config.Filters != null) 97 | { 98 | if (ApplyFilters(jsonMessage)) 99 | return; 100 | } 101 | 102 | var message = jsonMessage.ToString(); 103 | LogManager.GetCurrentClassLogger().Debug(message); 104 | 105 | lock (_locker) 106 | { 107 | _jsonQueue.Add(jsonMessage); 108 | } 109 | } 110 | 111 | private bool ApplyFilters(JObject json) 112 | { 113 | bool drop = false; 114 | 115 | foreach (var filter in _manager.Config.Filters) 116 | { 117 | if (!filter.Apply(json)) 118 | drop = true; 119 | } 120 | 121 | return drop; 122 | } 123 | 124 | } 125 | } 126 | 127 | -------------------------------------------------------------------------------- /TimberWinR/Parsers.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Globalization; 4 | using System.Linq; 5 | using System.Xml; 6 | using System.Xml.Linq; 7 | using TimberWinR.Inputs; 8 | 9 | public abstract class Parsers 10 | { 11 | protected static string ParseRequiredStringAttribute(XElement e, string attributeName) 12 | { 13 | XAttribute a = e.Attribute(attributeName); 14 | if (a != null) 15 | return a.Value; 16 | else 17 | throw new TimberWinR.ConfigurationErrors.MissingRequiredAttributeException(e, attributeName); 18 | } 19 | 20 | protected static string ParseStringAttribute(XElement e, string attributeName, string defaultValue = "") 21 | { 22 | string retValue = defaultValue; 23 | XAttribute a = e.Attribute(attributeName); 24 | if (a != null) 25 | retValue = a.Value; 26 | return retValue; 27 | } 28 | 29 | protected static string ParseDateAttribute(XElement e, string attributeName, string defaultValue = "") 30 | { 31 | string retValue = defaultValue; 32 | XAttribute a = e.Attribute(attributeName); 33 | if (a != null) 34 | { 35 | DateTime dt; 36 | if (DateTime.TryParseExact(a.Value, 37 | "yyyy-MM-dd hh:mm:ss", 38 | CultureInfo.InvariantCulture, 39 | DateTimeStyles.None, 40 | out dt)) 41 | { 42 | retValue = a.Value; 43 | } 44 | else 45 | { 46 | throw new TimberWinR.ConfigurationErrors.InvalidAttributeDateValueException(a); 47 | } 48 | } 49 | 50 | return retValue; 51 | } 52 | 53 | protected static bool ParseRequiredBoolAttribute(XElement e, string attributeName) 54 | { 55 | XAttribute a = e.Attribute(attributeName); 56 | if (a == null) 57 | throw new TimberWinR.ConfigurationErrors.InvalidAttributeValueException(e.Attribute(attributeName)); 58 | 59 | switch (a.Value) 60 | { 61 | case "ON": 62 | case "true": 63 | return true; 64 | 65 | case "OFF": 66 | case "false": 67 | return false; 68 | 69 | default: 70 | throw new TimberWinR.ConfigurationErrors.InvalidAttributeValueException(e.Attribute(attributeName)); 71 | } 72 | } 73 | 74 | protected static string ParseEnumAttribute(XElement e, string attributeName, IEnumerable values, string defaultValue) 75 | { 76 | XAttribute a = e.Attribute(attributeName); 77 | 78 | if (a != null) 79 | { 80 | string v = a.Value; 81 | if (values.Contains(v)) 82 | return v; 83 | else 84 | throw new TimberWinR.ConfigurationErrors.InvalidAttributeValueException(e.Attribute(attributeName)); 85 | } 86 | return defaultValue; 87 | } 88 | 89 | protected static int ParseIntAttribute(XElement e, string attributeName, int defaultValue) 90 | { 91 | XAttribute a = e.Attribute(attributeName); 92 | if (a != null) 93 | { 94 | int valInt; 95 | if (int.TryParse(a.Value, out valInt)) 96 | return valInt; 97 | else 98 | throw new TimberWinR.ConfigurationErrors.InvalidAttributeIntegerValueException(a); 99 | } 100 | return defaultValue; 101 | } 102 | protected static bool ParseBoolAttribute(XElement e, string attributeName, bool defaultValue) 103 | { 104 | bool retValue = defaultValue; 105 | XAttribute a = e.Attribute(attributeName); 106 | 107 | if (a != null) 108 | { 109 | switch (a.Value) 110 | { 111 | case "ON": 112 | case "true": 113 | retValue = true; 114 | break; 115 | 116 | case "OFF": 117 | case "false": 118 | retValue = false; 119 | break; 120 | } 121 | } 122 | return retValue; 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /TimberWinR/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("TimberWinR")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("TimberWinR")] 13 | [assembly: AssemblyCopyright("Copyright © 2014")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("b27e2311-c12b-4f71-83b2-82630b43103e")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.2.0.0")] 36 | [assembly: AssemblyFileVersion("1.2.0.0")] 37 | -------------------------------------------------------------------------------- /TimberWinR/Properties/Resources.Designer.cs: -------------------------------------------------------------------------------- 1 | //------------------------------------------------------------------------------ 2 | // 3 | // This code was generated by a tool. 4 | // Runtime Version:4.0.30319.18444 5 | // 6 | // Changes to this file may cause incorrect behavior and will be lost if 7 | // the code is regenerated. 8 | // 9 | //------------------------------------------------------------------------------ 10 | 11 | namespace TimberWinR.Properties { 12 | using System; 13 | 14 | 15 | /// 16 | /// A strongly-typed resource class, for looking up localized strings, etc. 17 | /// 18 | // This class was auto-generated by the StronglyTypedResourceBuilder 19 | // class via a tool like ResGen or Visual Studio. 20 | // To add or remove a member, edit your .ResX file then rerun ResGen 21 | // with the /str option, or rebuild your VS project. 22 | [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "4.0.0.0")] 23 | [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] 24 | [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] 25 | internal class Resources { 26 | 27 | private static global::System.Resources.ResourceManager resourceMan; 28 | 29 | private static global::System.Globalization.CultureInfo resourceCulture; 30 | 31 | [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] 32 | internal Resources() { 33 | } 34 | 35 | /// 36 | /// Returns the cached ResourceManager instance used by this class. 37 | /// 38 | [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] 39 | internal static global::System.Resources.ResourceManager ResourceManager { 40 | get { 41 | if (object.ReferenceEquals(resourceMan, null)) { 42 | global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("TimberWinR.Properties.Resources", typeof(Resources).Assembly); 43 | resourceMan = temp; 44 | } 45 | return resourceMan; 46 | } 47 | } 48 | 49 | /// 50 | /// Overrides the current thread's CurrentUICulture property for all 51 | /// resource lookups using this strongly typed resource class. 52 | /// 53 | [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] 54 | internal static global::System.Globalization.CultureInfo Culture { 55 | get { 56 | return resourceCulture; 57 | } 58 | set { 59 | resourceCulture = value; 60 | } 61 | } 62 | 63 | /// 64 | /// Looks up a localized string similar to <xs:schema attributeFormDefault="unqualified" elementFormDefault="qualified" xmlns:xs="http://www.w3.org/2001/XMLSchema"> 65 | /// <xs:element name="TimberWinR"> 66 | /// <xs:complexType> 67 | /// <xs:sequence> 68 | /// <xs:element name="Inputs"> 69 | /// <xs:complexType> 70 | /// <xs:sequence> 71 | /// <xs:element name="WindowsEvents" minOccurs="0"> 72 | /// <xs:complexType> 73 | /// <xs:sequence> 74 | /// <xs:element name="Event" maxOccurs="unbounded" minOccurs="0"> 75 | /// [rest of string was truncated]";. 76 | /// 77 | internal static string configSchema { 78 | get { 79 | return ResourceManager.GetString("configSchema", resourceCulture); 80 | } 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /TimberWinR/ReleaseNotes.md: -------------------------------------------------------------------------------- 1 | TimberWinR Release Notes 2 | ================================== 3 | A Native Windows to Redis/Elasticsearch Logstash Agent which runs as a service. 4 | 5 | Version / Date 6 | ### 1.3.26.0 - 2015-05-15 7 | 1. Added StatsD outputter 8 | 2. Fixed shutdown hang if shutdown was received before service was fully started up. 9 | 3. Closed issue [#36](https://github.com/Cimpress-MCP/TimberWinR/issues/36) 10 | 11 | ### 1.3.25.0 - 2015-04-30 12 | 1. Fixed Issue [#49](https://github.com/Cimpress-MCP/TimberWinR/issues/49) 13 | 2. Fixed potential non-thread safe when renaming properties 14 | 3. Added add_field, rename support to Udp/Tcp Input Listeners 15 | 4. Fixed issue with multiple renames (was previously only renaming the first one) 16 | 5. Added File outputter for testing. 17 | 18 | ### 1.3.24.0 - 2015-04-29 19 | 1. Fixed potential bug in TailFiles when tailing log files which are partially flushed 20 | to disk, it now will not process the line until the \r\n has been seen. 21 | 2. Added Generator input. 22 | 23 | ### 1.3.23.0 - 2015-04-23 24 | 1. Fixed bug with parsing a single json config file, rather than reading 25 | JSON files from a directory. 26 | 2. Diabled elasticsearch outputter ping by default and parameterized the ping capability. 27 | 28 | ### 1.3.22.0 - 2015-04-14 29 | 1. Fixed minor bug with TailFiles and service re-starts not picking up 30 | rolled files right away. 31 | 32 | ### 1.3.21.0 - 2015-04-13 33 | 1. Rolled Udp listener support to V4 only, too many issues with dual mode sockets 34 | and hosts file. If we want to add this back, I will add a Udpv6 input. 35 | 36 | ### 1.3.20.0 - 2015-04-03 37 | 38 | 1. A re-factoring of Logs and TailLogs to be more efficient and detect log rolling correctly, 39 | this requires http://support.microsoft.com/en-us/kb/172190 which will be detected and 40 | set by TimberWinR, however, requires a reboot. 41 | 2. Fixed issue [#38](https://github.com/Cimpress-MCP/TimberWinR/issues/38) diagnostic output not showing drop flag for Grok filter. 42 | 3. Created TimberWinR.TestGenerator for complete testing of TimberWinR 43 | 4. Fixed ipv4/ipv6 thread-safe issue with UdpInputListener which might lead to corrupted input data. 44 | 45 | ### 1.3.19.1 - 2015-03-03 46 | 47 | 1. Added new Redis parameter _max\_batch\_count_ which increases the _batch\_count_ dynamically over time 48 | to handle input flooding. Default is _batch\_count_ * 10 49 | 50 | ### 1.3.19.0 - 2015-02-26 51 | 52 | 1. Added support for Multiline codecs for Stdin and Logs listeners, closes issue [#23](https://github.com/Cimpress-MCP/TimberWinR/issues/23) 53 | 2. Added new TailFiles input type which uses a native implementation (more-efficient) than using LogParser's Log 54 | 3. Updated Udp input listner to use UTF8 Encoding rather than ASCII 55 | 4. Reduced noisy complaint about missing log files for Logs listener 56 | 5. Fixed bug when tailing non-existent log files which resulted in high cpu-usage. 57 | 6. Added feature to watch the configuration directory 58 | 59 | ### 1.3.18.0 - 2014-12-22 60 | 61 | 1. Fixed bug introduced in 1.3.17.0 which changed the meaning of the delay for Elasticsearch, Redis and Stdout 62 | intervals to be interpreted as seconds instead of milliseconds. 1.3.17.0 should not be used. 63 | 2. Removed ability for installer to downgrade which was leading to leaving previous versions laying around (i.e. reverts 1.3.13.0 change) 64 | 65 | ### 1.3.17.0 - 2014-12-19 66 | 67 | 1. Continued work improving shutdown time by using syncHandle.Wait instead of Thread.Sleep 68 | 69 | ### 1.3.16.0 - 2014-12-19 70 | 71 | 1. Added logSource property to the Log input to facility the steering of log messages to different indices. 72 | 73 | ### 1.3.15.0 - 2014-12-12 74 | 75 | 1. Fixed bug whereby if the Udp or Tcp inputs receive an impropery formatted Json it caused the thread to terminate, and ignore 76 | future messages. 77 | 78 | ### 1.3.14.0 - 2014-12-11 79 | 80 | 1. Fixed bug with the Grok filter to match properly the value of the Text field against non-blank entries. 81 | 82 | ### 1.3.13.0 - 2014-12-02 83 | 84 | 1. Fixed MSI installer to allow downgrades. 85 | 86 | ### 1.3.12.0 - 2014-11-25 87 | 88 | 1. Fixed all remaining memory leaks due to the COM Weak Surrogate which requires an explicit GC.Collect 89 | 90 | ### 1.3.11.0 - 2014-11-21 91 | 92 | 1. Re-worked WindowsEvent listener to enable shutting down in a quicker fashion. 93 | 94 | ### 1.3.10.0 - 2014-11-18 95 | 96 | 1. Refactored Conditions handler to use non-leaking evaluator. 97 | 98 | ### 1.3.9.0 - 2014-11-11 99 | 100 | 1. Merged in pull request #9 101 | 2. Updated chocolately uninstall to preserve GUID 102 | 103 | ### 1.3.8.0 - 2014-11-06 104 | 105 | 1. Added interval parameter to WindowsEvent input listener 106 | 2. Increased default value for interval to 60 seconds for polling WindowsEvents 107 | 108 | ### 1.3.7.0 - 2014-10-21 109 | 110 | 1. Added additional information for diagnostics port 111 | 2. Completed minor handling of Log rolling detection 112 | 113 | ### 1.3.6.0 - 2014-10-16 114 | 115 | 1. Handle rolling of logs whereby the logfile remains the same, but the content resets back to 0 bytes. 116 | -------------------------------------------------------------------------------- /TimberWinR/lib/com-logparser/Interop.MSUtil.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TerribleDev/TimberWinR/5165224c65c81d7e3f26df9f2bc5e6415cc8ceb6/TimberWinR/lib/com-logparser/Interop.MSUtil.dll -------------------------------------------------------------------------------- /TimberWinR/lib/net40/Interop.MSUtil.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TerribleDev/TimberWinR/5165224c65c81d7e3f26df9f2bc5e6415cc8ceb6/TimberWinR/lib/net40/Interop.MSUtil.dll -------------------------------------------------------------------------------- /TimberWinR/lib/net40/TimberWinR.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TerribleDev/TimberWinR/5165224c65c81d7e3f26df9f2bc5e6415cc8ceb6/TimberWinR/lib/net40/TimberWinR.dll -------------------------------------------------------------------------------- /TimberWinR/mdocs/Codec.md: -------------------------------------------------------------------------------- 1 | # Codec 2 | 3 | ## Parameters 4 | The following parameters are allowed when configuring the Codec. 5 | 6 | | Parameter | Type | Description | Details | Default | 7 | | :---------------- |:---------------| :----------------------------------------------------------------------- | :--------------------------- | :-- | 8 | | *type* | enum |Codec type 'multiline' | Must be 'multiline' | | 9 | | *pattern* | regex |Regular expression to be matched | Must be legal .NET Regex | | 10 | | *what* | enum |Value can be previous or next | If the pattern matched, does event belong to the next or previous event? | | 11 | | *negate* | bool |Inverts the pattern sense | If true, a message not matching the pattern will constitute a match of the multiline filter and the what will be applied. (vice-versa is also true) | false | 12 | | *multiline_tag* | string |Tag to be added when multiline conversion is applied | | multiline | 13 | 14 | This codec applies to [Logs](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Logs.md) and [Stdin](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/StdinInput.md) only. 15 | 16 | Example Input: Mutliline input log file 17 | 18 | ```json 19 | { 20 | "TimberWinR": { 21 | "Inputs": { 22 | "Logs": [ 23 | { 24 | "location": "C:\\Logs1\\multiline.log", 25 | "recurse": -1, 26 | "codec": { 27 | "negate": false, 28 | "type": "multiline", 29 | "pattern": "(^.+Exception: .+)|(^\\s+at .+)|(^\\s+... \\d+ more)|(^\\s*Caused by:.+)", 30 | "what": "previous" 31 | } 32 | } 33 | } 34 | } 35 | } 36 | ``` 37 | -------------------------------------------------------------------------------- /TimberWinR/mdocs/DateFilter.md: -------------------------------------------------------------------------------- 1 | # Date Filter 2 | The date filter is used for parsing dates from fields, and then using that date or timestamp as the logstash timestamp for the event. 3 | For example, syslog events usually have timestamps like this: 4 | 5 | ``` 6 | "Apr 17 09:32:01" 7 | ``` 8 | You would use the date format "MMM dd HH:mm:ss" to parse this. 9 | 10 | The date filter is especially important for sorting events and for backfilling old data. If you don't 11 | get the date correct in your event, then searching for them later will likely sort out of order. 12 | 13 | In the absence of this filter, TimberWinR will choose a timestamp based on the first time it sees 14 | the event (at input time), if the timestamp is not already set in the event. For example, with 15 | file input, the timestamp is set to the time of each read. 16 | 17 | ## Date Parameters 18 | The following parameters and operations are allowed when using the Date filter. 19 | 20 | | Operation | Type | Description | Default | 21 | | :---------------|:----------------|:---------------|:------------| 22 | | *add_field* | array |If the filter is successful, add an arbitrary field to this event. Tag names can be dynamic and include parts of the event using the %{field} syntax. | | 23 | | *condition* | string |C# expression | | 24 | | *convertToUTC* | boolean |Converts time to UTC | false | 25 | | *match* | [string] |Required field and pattern must match before any subsequent date operations are executed. | | 26 | | *locale* | string | Specify a locale to be used for date parsing | en-US | 27 | | *target* | string | Store the matching timestamp into the given target field. If not provided, default to updating the @timestamp field of the event. | @timestamp | 28 | 29 | ## Parameter Details 30 | ### match 31 | The date formats allowed are anything allowed by [C# DateTime Format](http://msdn.microsoft.com/en-us/library/8kb3ddd4(v=vs.110).aspx). You can see the docs for this format here: 32 | Given this configuration 33 | ```json 34 | "Filters": [ 35 | { 36 | "date": { 37 | "condition": "\"[type]\" == \"Win32-FileLog\"", 38 | "match": [ 39 | "timestamp", 40 | "MMM d HH:mm:sss", 41 | "MMM dd HH:mm:ss" 42 | ], 43 | "add_field": [ 44 | "UtcTimestamp" 45 | ], 46 | "convertToUTC": true 47 | } 48 | } 49 | ] 50 | ``` 51 | 52 | ### condition "C# expression" 53 | If present, the condition must evaluate to true in order for the remaining operations to be performed. If there is no condition specified 54 | then the operation(s) will be executed in order. 55 | ```json 56 | "Filters": [ 57 | { 58 | "grok": { 59 | "condition": "\"[type]\" == \"Win32-EventLog\"" 60 | "add_field": [ 61 | "ComputerName", "%{Host}" 62 | ] 63 | } 64 | } 65 | ] 66 | ``` 67 | The above example will add a field ComputerName set to the value of Host only for Win32-EventLog types. 68 | 69 | ### add_field ["fieldName", "fieldValue", ...] 70 | The fields must be in pairs with fieldName first and value second. 71 | ```json 72 | "Filters": [ 73 | { 74 | "date": { 75 | "condition": "\"[type]\" == \"Win32-FileLog\"", 76 | "match": [ 77 | "timestamp", 78 | "MMM d HH:mm:sss", 79 | "MMM dd HH:mm:ss" 80 | ], 81 | "add_field": [ 82 | "UtcTimestamp" 83 | ] 84 | } 85 | } 86 | ] 87 | ``` 88 | 89 | ### convertToUTC "true|false" 90 | If true and the filter matches, the time parsed will be converted to UTC 91 | ```json 92 | "Filters": [ 93 | { 94 | "date": { 95 | "condition": "\"[type]\" == \"Win32-FileLog\"", 96 | "match": [ 97 | "timestamp", 98 | "MMM d HH:mm:sss", 99 | "MMM dd HH:mm:ss" 100 | ], 101 | "add_field": [ 102 | "UtcTimestamp" 103 | ], 104 | "convertToUTC": true 105 | } 106 | } 107 | ] 108 | ``` 109 | -------------------------------------------------------------------------------- /TimberWinR/mdocs/ElasticsearchOutput.md: -------------------------------------------------------------------------------- 1 | # Output: Elasticsearch 2 | 3 | The Elasticsearch output passes on data directly to Elasticsearch. 4 | 5 | ## Parameters 6 | The following parameters are allowed when configuring the Elasticsearch output. 7 | 8 | | Parameter | Type | Description | Details | Default | 9 | | :-------------|:---------|:------------------------------------------------------------| :--------------------------- | :-- | 10 | | *flush_size* | integer | Maximum number of messages before flushing | | 50000 | 11 | | *host* | [string] | Array of hostname(s) of your Elasticsearch server(s) | IP or DNS name | | 12 | | *idle_flush_time* | integer | Maximum number of seconds elapsed before triggering a flush | | 10 | 13 | | *index* | [string] | The index name to use | index used/created | logstash-yyyy.dd.mm | 14 | | *interval* | integer | Interval in milliseconds to sleep during batch sends | Interval | 5000 | 15 | | *max_queue_size* | integer | Maximum Elasticsearch queue depth | | 50000 | 16 | | *port* | integer | Elasticsearch port number | This port must be open | 9200 | 17 | | *ssl* | bool | If true, use an HTTPS connection to Elasticsearch. See [this page] (https://www.elastic.co/guide/en/found/current/elk-and-found.html#_using_logstash) for a configuration example. | *username* and *password* are also required for HTTPS connections. | false | 18 | | *username* | string | Username for Elasticsearch credentials. | Required for HTTPS connection. | | 19 | | *password* | string | Password for Elasticsearch credentials. | Required for HTTPS connection. | | 20 | | *queue_overflow_discard_oldest* | bool | If true, discard oldest messages when max_queue_size reached otherwise discard newest | | true | 21 | | *threads* | [string] | Number of Threads | Number of worker threads processing messages | 1 | 22 | | *enable_ping* | bool | If true, pings the server to test for keep alive | | false | 23 | | *ping_timeout* | integer | Default ping timeout when enable_ping is true | milliseconds | 200 | 24 | 25 | ### Index parameter 26 | If you want to output your data everyday to a new index, use following index format: "index-%{yyyy.MM.dd}". Here date format could be any forwat which you need. 27 | 28 | Example Input: 29 | ```json 30 | { 31 | "TimberWinR": { 32 | "Outputs": { 33 | "Elasticsearch": [ 34 | { 35 | "threads": 1, 36 | "interval": 5000, 37 | "host": [ 38 | "tstlexiceapp006.mycompany.svc" 39 | ] 40 | } 41 | ] 42 | } 43 | } 44 | } 45 | ``` -------------------------------------------------------------------------------- /TimberWinR/mdocs/FileOutput.md: -------------------------------------------------------------------------------- 1 | # Output: File 2 | 3 | The File output passes on data into a text file. 4 | 5 | ## Parameters 6 | The following parameters are allowed when configuring the File output. 7 | 8 | | Parameter | Type | Description | Details | Default | 9 | | :-------------|:---------|:------------------------------------------------------------| :--------------------------- | :-- | 10 | | *interval* | integer | Interval in milliseconds to sleep before appending data | Interval | 1000 | 11 | | *file_name* | string | Name of the file to be created | | timberwinr.out | 12 | 13 | Example Input: 14 | ```json 15 | { 16 | "TimberWinR": { 17 | "Outputs": { 18 | "File": [ 19 | { 20 | "file_name": "foo.out", 21 | "interval": 1000 22 | } 23 | ] 24 | } 25 | } 26 | } 27 | ``` -------------------------------------------------------------------------------- /TimberWinR/mdocs/Filters.md: -------------------------------------------------------------------------------- 1 | # Filters 2 | The following filters are provided. 3 | 4 | 5 | | Filter | Description 6 | | :---------------- |:----------------------------------------------------------------------- 7 | | *[grok][4]* |Similar to the [logstash grok][1] filter 8 | | *[date][5]* |Similar to the [logstash date][2] filter 9 | | *[mutate][6]* |Similar to the [logstash mutate][3] filter 10 | Example Input: 11 | ```json 12 | "Filters": [ 13 | { 14 | "grok": { 15 | "condition": "\"[type]\" == \"Win32-Eventlog\"", 16 | "match": [ 17 | "Message", 18 | "" 19 | ], 20 | "remove_field": [ 21 | "ComputerName" 22 | ] 23 | } 24 | }, 25 | { 26 | "grok": { 27 | "match": [ 28 | "message", 29 | "%{SYSLOGLINE}" 30 | ], 31 | "add_field": [ 32 | "Hello", "from %{logsource}" 33 | ] 34 | } 35 | }, 36 | { 37 | "date": { 38 | "condition": "\"[type]\" == \"Win32-FileLog\"", 39 | "match": [ 40 | "timestamp", 41 | "MMM d HH:mm:sss", 42 | "MMM dd HH:mm:ss" 43 | ], 44 | "add_field": [ 45 | "UtcTimestamp" 46 | ], 47 | "convertToUTC": true 48 | } 49 | }, 50 | { 51 | "mutate": { 52 | "_comment": "Custom Rules", 53 | "rename": [ 54 | "ComputerName", "Host", 55 | "host", "Host", 56 | "message","Message", 57 | "type","Type", 58 | "SID", "Username" 59 | ] 60 | } 61 | } 62 | ] 63 | ``` 64 | [1]: http://logstash.net/docs/1.4.2/filters/grok 65 | [2]: http://logstash.net/docs/1.4.2/filters/date 66 | [3]: http://logstash.net/docs/1.4.2/filters/mutate 67 | [4]: https://github.com/Cimpress-MCP/TimberWinR/blob/master/mdocs/GrokFilter.md 68 | [5]: https://github.com/Cimpress-MCP/TimberWinR/blob/master/mdocs/DateFilter.md 69 | [6]: https://github.com/Cimpress-MCP/TimberWinR/blob/master/mdocs/MutateFilter.md 70 | -------------------------------------------------------------------------------- /TimberWinR/mdocs/Generator.md: -------------------------------------------------------------------------------- 1 | # Input: Generator 2 | 3 | The Generator input can be used to Generate log files for test purposes. 4 | 5 | ## Parameters 6 | The following parameters are allowed when configuring the test log Generator. 7 | 8 | | Parameter | Type | Description | Details | Default | 9 | | :---------------- |:---------------| :----------------------------------------------------------------------- | :--------------------------- | :-- | 10 | | *type* | string |Message type | | Win32-InputGen | 11 | | *message* | string |Message format to send | | Hello, World! | 12 | | *count* | integer |Number of messages to generate | 0 - Infinite, otherwise that number | 0 | 13 | | *rate* | integer |Sleep time between generated messages | Milliseconds | 10 | 14 | | [codec](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Codec.md) | object | Codec to use | 15 | 16 | Example: Generate 100000 "Hello Win32-InputGen" messages 17 | 18 | ```json 19 | { 20 | "TimberWinR": { 21 | "Inputs": { 22 | "Generator": [ 23 | { 24 | "message": "Hello %{type}", 25 | "count": 100000 26 | } 27 | ] 28 | } 29 | } 30 | } 31 | ``` 32 | ## Fields 33 | After a successful parse of the generated line, the following fields are added: 34 | 35 | | Name | Type | Description | 36 | | ---- |:-----| :-----------| 37 | | Message | STRING | Text line content | 38 | -------------------------------------------------------------------------------- /TimberWinR/mdocs/GrokFilter.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TerribleDev/TimberWinR/5165224c65c81d7e3f26df9f2bc5e6415cc8ceb6/TimberWinR/mdocs/GrokFilter.md -------------------------------------------------------------------------------- /TimberWinR/mdocs/Logs.md: -------------------------------------------------------------------------------- 1 | # Input: Logs 2 | 3 | The Logs input will monitor a log (text) file similar to how a Linux "tail -f" command works. 4 | 5 | ## Parameters 6 | The following parameters are allowed when configuring WindowsEvents. 7 | 8 | | Parameter | Type | Description | Details | Default | 9 | | :---------------- |:---------------| :----------------------------------------------------------------------- | :--------------------------- | :-- | 10 | | *iCodepage* | integer |Codepage of the text file. | 0 is the system codepage, -1 is UNICODE. | 0 | 11 | | *location* | string |Location of file(s) to monitor | Path to text file(s) including wildcards. | | 12 | | *logSource* | string |Source name | Used for conditions | | 13 | | *recurse* | integer |Max subdirectory recursion level. | 0 disables subdirectory recursion; -1 enables unlimited recursion. | 0 | 14 | | *splitLongLines* | boolean |Behavior when event messages or event category names cannot be resolved. |When a text line is longer than 128K characters, the format truncates the line and either discards the remaining of the line (when this parameter is set to "false"), or processes the remainder of the line as a new line (when this parameter is set to "true").| false | 15 | | *type* | string |Typename for this Input | | Win32-FileLog | 16 | | [codec](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Codec.md) | object | Codec to use | 17 | 18 | Example Input: Monitors all files (recursively) located at C:\Logs1\ matching *.log as a pattern. I.e. C:\Logs1\foo.log, C:\Logs1\Subdir\Log2.log, etc. 19 | 20 | ```json 21 | { 22 | "TimberWinR": { 23 | "Inputs": { 24 | "Logs": [ 25 | { 26 | "logSource": "log files", 27 | "location": "C:\\Logs1\\*.log", 28 | "recurse": -1 29 | } 30 | ] 31 | } 32 | } 33 | } 34 | ``` 35 | ## Fields 36 | After a successful parse of an event, the following fields are added: 37 | 38 | | Name | Type | Description | 39 | | ---- |:-----| :-----------| 40 | | LogFilename | STRING |Full path of the file containing this line | 41 | | Index | INTEGER | Line number | 42 | | Text | STRING | Text line content | 43 | | type | STRING | Win32-FileLog | 44 | -------------------------------------------------------------------------------- /TimberWinR/mdocs/MutateFilter.md: -------------------------------------------------------------------------------- 1 | # Mutate Filter 2 | The mutate filter allows you to perform general mutations on fields. You can rename, remove, replace and modify fields in your events. This filter will automatically be applied to all inputs before sending to the outputs. If you want to make a 3 | filter conditional, use the ***condition*** property to specify a legal C# expression. 4 | 5 | ## Mutate Operations 6 | The following operations are allowed when mutating a field. 7 | 8 | | Operation | Type | Description 9 | | :-----------|:----------------|:-----------------------------------------------------------------------| 10 | | *condition* | property:string |C# Expression 11 | | *remove* | property:array |Remove one or more fields 12 | | *rename* | property:array |Rename one or more fields 13 | | *replace* | property:array |Replace a field with a new value. The new value can include %{foo} strings to help you build a new value from other parts of the event. 14 | | *split* | property:array |Separator between values of the "Strings" field. 15 | 16 | ## Details 17 | ### condition "C# expression" 18 | If present, the condition must evaluate to true in order for the remaining operations to be performed. If there is no condition specified 19 | then the operation(s) will be executed in order. 20 | ```json 21 | "Filters": [ 22 | { 23 | "mutate": { 24 | "condition": "\"[type]\" == \"Win32-EventLog\"", 25 | "rename": [ 26 | "ComputerName", "Host" 27 | ] 28 | } 29 | } 30 | ] 31 | ``` 32 | The above example will rename ComputerName to Host only for Win32-EventLog types. 33 | 34 | ### remove ["name", ...] 35 | Removes field. 36 | ```json 37 | "Filters": [ 38 | { 39 | "mutate": { 40 | "remove": [ 41 | "ComputerName", "Username" 42 | ] 43 | } 44 | } 45 | ] 46 | ``` 47 | ### rename ["oldname", "newname", ...] 48 | The fields must be in pairs with oldname first and newname second. 49 | ```json 50 | "Filters": [ 51 | { 52 | "mutate": { 53 | "rename": [ 54 | "ComputerName", "Host", 55 | "host", "Host", 56 | "message","Message", 57 | "type","Type", 58 | "SID", "Username" 59 | ] 60 | } 61 | } 62 | ] 63 | ``` 64 | ### replace ["field", "newvalue", ...] 65 | Replaces field with newvalue. The replacements must be described in pairs. 66 | ```json 67 | "Filters": [ 68 | { 69 | "mutate": { 70 | "replace": [ 71 | "message", "%{source_host}: My new message" 72 | ] 73 | } 74 | } 75 | ] 76 | ``` 77 | ### split 78 | Split a field into an array of values. The first arguments is the fieldName and the second is the separator. 79 | ```json 80 | "Filters": [ 81 | { 82 | "mutate": { 83 | "split": [ 84 | "InsertionStrings", "|" 85 | ] 86 | } 87 | } 88 | ] 89 | ``` 90 | 91 | 92 | -------------------------------------------------------------------------------- /TimberWinR/mdocs/RedisOutput.md: -------------------------------------------------------------------------------- 1 | # Output: Redis 2 | 3 | The Redis output passes on data to Redis to be consumed by the Logtash indexer. 4 | 5 | ## Parameters 6 | The following parameters are allowed when configuring the Redis output. 7 | 8 | | Parameter | Type | Description | Details | Default | 9 | | :-------------|:---------|:------------------------------------------------------------| :--------------------------- | :-- | 10 | | *batch_count* | integer | Sent as a single message | Number of messages to aggregate | 200 | 11 | | *host* | string | The hostname(s) of your Redis server(s) | IP or DNS name | | 12 | | *index* | string | The name of the redis list | logstash index name | logstash | 13 | | *interval* | integer | Interval in milliseconds to sleep during batch sends | Interval | 5000 | 14 | | *max_batch_count* | integer | Dynamically adjusted count maximum | Increases over time | batch_count * 10 | 15 | | *max_queue_size* | integer | Maximum redis queue depth | | 50000 | 16 | | *port* | integer | Redis port number | This port must be open | 6379 | 17 | | *queue_overflow_discard_oldest* | bool | If true, discard oldest messages when max_queue_size reached otherwise discard newest | | true | 18 | | *threads* | string | Location of log files(s) to monitor | Number of worker theads to send messages | 1 | 19 | 20 | Example Input: 21 | ```json 22 | { 23 | "TimberWinR": { 24 | "Outputs": { 25 | "Redis": [ 26 | { 27 | "threads": 1, 28 | "interval": 5000, 29 | "batch_count": 500, 30 | "host": [ 31 | "tstlexiceapp006.mycompany.svc" 32 | ] 33 | } 34 | ] 35 | } 36 | } 37 | } 38 | ``` -------------------------------------------------------------------------------- /TimberWinR/mdocs/StatsD.md: -------------------------------------------------------------------------------- 1 | # Output: StatsD 2 | 3 | The StatsD output passes on data directly to StatsD. (https://github.com/etsy/statsd) 4 | 5 | ## Parameters 6 | The following parameters are allowed when configuring the StatsD output. 7 | 8 | | Parameter | Type | Description | Details | Default | 9 | | :-------------|:---------|:------------------------------------------------------------| :--------------------------- | :-- | 10 | | *count* | string | Array of (metric_name, gauge name) pairs counted | Must come in pairs | | 11 | | *decrement* | string | Array of metrics to be decremented | | | 12 | | *flush_size* | integer | Maximum number of messages before flushing | | 50000 | 13 | | *gauge* | string | Array of (metric_name, gauge name) pairs gauged | Must come in pairs | | 14 | | *host* | string | Hostname or IP of StatsD server | localhost | | 15 | | *idle_flush_time* | integer | Maximum number of seconds elapsed before triggering a flush | | 10 | 16 | | *increment* | string | Array of metrics to be incremented | | | 17 | | *interval* | integer | Interval in milliseconds to sleep between sends | Interval | 5000 | 18 | | *max_queue_size* | integer | Maximum StatsD queue depth | | 50000 | 19 | | *namespace* | string | Namespace for stats | timberwinr | | 20 | | *port* | integer | StatsD port number | This port must be open | 8125 | 21 | | *queue_overflow_discard_oldest* | bool | If true, discard oldest messages when max_queue_size reached otherwise discard newest | | true | 22 | | *sample_rate* | integer | StatsD sample rate | | 1 | 23 | | *sender* | string | Sender name | FQDN | | 24 | | *threads* | string | Number of Threads processing messages | | 1 | 25 | | *timing* | string | Array of (metric_name, timing_name) pairs timed | Must come in pairs | | 26 | | *type* | string |Type to which this filter applies, if empty, applies to all types. 27 | 28 | ### Example Usage 29 | Example Input: Tail an apache log file, and record counts for bytes and increments for response codes. 30 | 31 | sample-apache.log (snip) 32 | ``` 33 | 180.76.5.25 - - [13/May/2015:17:02:26 -0700] "GET /frameset.htm HTTP/1.1" 404 89 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)" "www.redlug.com" 34 | 208.115.113.94 - - [13/May/2015:17:03:55 -0700] "GET /robots.txt HTTP/1.1" 200 37 "-" "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)" "redlug.com" 35 | 208.115.113.94 - - [13/May/2015:17:03:55 -0700] "GET /robots.txt HTTP/1.1" 200 37 "-" "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)" "www.redlug.com" 36 | ``` 37 | 38 | Note: [COMBINEDAPACHELOG](https://github.com/elastic/logstash/blob/v1.4.2/patterns/grok-patterns) is a standard 39 | Grok Pattern. 40 | 41 | TimberWinR configuration 42 | 43 | ```json 44 | { 45 | "TimberWinR": { 46 | "Inputs": { 47 | "TailFiles": [ 48 | { 49 | "interval": 5, 50 | "logSource": "apache log files", 51 | "location": "..\\sample-apache.log", 52 | "recurse": -1 53 | } 54 | ] 55 | }, 56 | "Filters": [ 57 | { 58 | "grok": { 59 | "type": "Win32-TailLog", 60 | "match": [ 61 | "Text", 62 | "%{COMBINEDAPACHELOG}" 63 | ] 64 | } 65 | } 66 | ], 67 | "Outputs": { 68 | "StatsD": [ 69 | { 70 | "type": "Win32-TailLog", 71 | "port": 8125, 72 | "host": "stats.mycompany.svc", 73 | "increment": ["apache.response.%{response}"], 74 | "count": ["apache.bytes", "%{bytes}"] 75 | } 76 | ] 77 | } 78 | } 79 | } 80 | 81 | ``` 82 | 83 | Assuming your FQDN is something like mymachine.mycompany.com, you should see the following in Graphite: 84 | 85 | ``` 86 | stats.counters.timberwinr.mymachine.mycompany.com.apache.bytes.count 87 | stats.counters.timberwinr.mymachine.mycompany.com.apache.bytes.rate 88 | stats.counters.timberwinr.mymachine.mycompany.com.apache.response.200.count 89 | stats.counters.timberwinr.mymachine.mycompany.com.apache.response.200.rate 90 | stats.counters.timberwinr.mymachine.mycompany.com.apache.response.404.count 91 | stats.counters.timberwinr.mymachine.mycompany.com.apache.response.404.rate 92 | ... 93 | ... 94 | ``` -------------------------------------------------------------------------------- /TimberWinR/mdocs/StdinInput.md: -------------------------------------------------------------------------------- 1 | # Input: Stdin 2 | 3 | The Stdin Input will read from the console (Console.ReadLine) and build a simple message for testing. 4 | 5 | ## Parameters 6 | The following parameters are allowed when configuring WindowsEvents. 7 | 8 | | Parameter | Type | Description | Details | Default | 9 | | :---------------- |:---------------| :----------------------------------------------------------------------- | :--------------------------- | :-- | 10 | | [codec](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Codec.md) | object | Codec to use | 11 | 12 | 13 | ```json 14 | { 15 | "TimberWinR": { 16 | "Inputs": { 17 | "Stdin": [ 18 | { 19 | "_comment": "Read from Console" 20 | } 21 | ] 22 | } 23 | } 24 | } 25 | ``` 26 | ## Fields 27 | 28 | A field: "type": "Win32-Stdin" is automatically appended, and the entire JSON is passed on vertabim 29 | 30 | | Name | Type | Description | 31 | | ---- |:-----| :-----------------------------------------------------------------------| 32 | | type | STRING |Win32-Stdin | 33 | | message | STRING | The message typed in | 34 | -------------------------------------------------------------------------------- /TimberWinR/mdocs/StdoutOutput.md: -------------------------------------------------------------------------------- 1 | # Output: Stdout 2 | 3 | The Stdout output will write to the console (Console.WriteLine) and display a json structure of the received message attributes. Useful for quickly viewing and testing the inputs and filters of a setup. 4 | 5 | ## Parameters 6 | There are no Parameters at this time. 7 | 8 | ```json 9 | { 10 | "TimberWinR": { 11 | "Outputs": { 12 | "Stdout": [ 13 | { 14 | "_comment": "Output to Console" 15 | } 16 | ] 17 | } 18 | } 19 | } 20 | ``` -------------------------------------------------------------------------------- /TimberWinR/mdocs/TailFiles.md: -------------------------------------------------------------------------------- 1 | # Input: TailFiles 2 | 3 | The TailFiles input will monitor a log (text) file similar to how a Linux "tail -f" command works. This uses 4 | a native implementation rather than uses LogParser 5 | 6 | ## Parameters 7 | The following parameters are allowed when configuring WindowsEvents. 8 | 9 | | Parameter | Type | Description | Details | Default | 10 | | :---------------- |:---------------| :----------------------------------------------------------------------- | :--------------------------- | :-- | 11 | | *type* | string |Typename for this Input | | Win32-TailLog | 12 | | *location* | string |Location of file(s) to monitor | Path to text file(s) including wildcards. | | 13 | | *logSource* | string |Source name | Used for conditions | | 14 | | *recurse* | integer |Max subdirectory recursion level. | 0 disables subdirectory recursion; -1 enables unlimited recursion. | 0 | 15 | | *interval* | integer |Polling interval in seconds | Defaults every 60 seconds | 60 | 16 | | [codec](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Codec.md) | object | Codec to use | 17 | 18 | Example Input: Monitors all files (recursively) located at C:\Logs1\ matching *.log as a pattern. I.e. C:\Logs1\foo.log, C:\Logs1\Subdir\Log2.log, etc. 19 | 20 | ```json 21 | { 22 | "TimberWinR": { 23 | "Inputs": { 24 | "TailFiles": [ 25 | { 26 | "logSource": "log files", 27 | "location": "C:\\Logs1\\*.log", 28 | "recurse": -1 29 | } 30 | ] 31 | } 32 | } 33 | } 34 | ``` 35 | ## Fields 36 | After a successful parse of an event, the following fields are added: 37 | 38 | | Name | Type | Description | 39 | | ---- |:-----| :-----------| 40 | | LogFilename | STRING |Full path of the file containing this line | 41 | | Index | INTEGER | Line number | 42 | | Text | STRING | Text line content | 43 | | type | STRING | Win32-TailLog | 44 | -------------------------------------------------------------------------------- /TimberWinR/mdocs/TcpInput.md: -------------------------------------------------------------------------------- 1 | # Input: Tcp 2 | 3 | The Tcp input will open a port and listen for properly formatted JSON and will forward on the entire JSON. 4 | 5 | ## Parameters 6 | The following parameters are allowed when configuring the Tcp input. 7 | 8 | | Parameter | Type | Description | Details | Default | 9 | | :---------------- |:-----------------| :----------------------------------------------------------------------- | :--------------------------- | :-- | 10 | | *add_field* | property:array |Add field(s) to this event. Field names can be dynamic and include parts of the event using the %{field} syntax. This property must be specified in pairs. | | 11 | | *port* | integer |Port number to open | Must be an available port | | 12 | | *rename* | property:array |Rename one or more fields | | | 13 | | *type* | string |Typename for this Input | | Win32-Tcp | 14 | 15 | Example Input: Listen on Port 5140 16 | 17 | ```json 18 | { 19 | "TimberWinR": { 20 | "Inputs": { 21 | "Tcp": [ 22 | { 23 | "port": 5140 24 | } 25 | ] 26 | } 27 | } 28 | } 29 | ``` 30 | ## Fields 31 | A field: "type": "Win32-Tcp" is automatically appended, and the entire JSON is passed on vertabim. 32 | -------------------------------------------------------------------------------- /TimberWinR/mdocs/UdpInput.md: -------------------------------------------------------------------------------- 1 | # Input: Udp 2 | 3 | The Udp input will open a port and listen for properly formatted UDP datagrams to be broadcast. 4 | 5 | ## Parameters 6 | The following parameters are allowed when configuring the Udp input. 7 | 8 | | Parameter | Type | Description | Details | Default | 9 | | :---------------- |:---------------| :----------------------------------------------------------------------- | :--------------------------- | :-- | 10 | | *add_field* | property:array |Add field(s) to this event. Field names can be dynamic and include parts of the event using the %{field} syntax. This property must be specified in pairs. | | 11 | | *port* | integer |Port number to open | Must be an available port | | 12 | | *rename* | property:array |Rename one or more fields | | | 13 | | *type* | string |Typename for this Input | | Win32-Udp | 14 | 15 | Example Input: Listen on Port 5142 16 | 17 | ```json 18 | { 19 | "TimberWinR": { 20 | "Inputs": { 21 | "Udp": [ 22 | { 23 | "port": 5142 24 | } 25 | ] 26 | } 27 | } 28 | } 29 | ``` 30 | ## Fields 31 | A field: "type": "Win32-Udp" is automatically appended, and the entire JSON is passed on vertabim. 32 | -------------------------------------------------------------------------------- /TimberWinR/mdocs/W3CInput.md: -------------------------------------------------------------------------------- 1 | # Input: W3CLogs 2 | 3 | The W3C input format parses IIS log files in the W3C Extended Log File Format, and handles custom fields unlike the IISW3C input. 4 | 5 | IIS web sites logging in the W3C Extended format can be configured to log only a specific subset of the available fields. 6 | Log files in this format begin with some informative headers ("directives"), the most important of which is the "#Fields" directive, 7 | describing which fields are logged at which position in a log row. After the directives, the log entries follow. 8 | Each log entry is a space-separated list of field values. 9 | 10 | If the logging configuration of an IIS virtual site is updated, the structure of the fields in the file that is 11 | currently logged to might change according to the new configuration. In this case, a new "#Fields" directive is 12 | logged describing the new fields structure, and the W3C input format keeps track of the structure change and 13 | parses the new log entries accordingly. 14 | 15 | 16 | ## Parameters 17 | The following parameters are allowed when configuring W3CLogs input. 18 | 19 | | Parameter | Type | Description | Details | Default | 20 | | :---------------- |:---------------| :----------------------------------------------------------------------- | :--------------------------- | :-- | 21 | | *location* | string |Location of log files(s) to monitor | Path to text file(s) including wildcards, may be separated by commas | | 22 | | *iCodepage* | integer |Codepage of the text file. | 0 is the system codepage, -1 is UNICODE. | 0 | 23 | | *dtLines* | integer |Number of lines examined to determine field types at run time. | This parameter specifies the number of initial log lines that the W3C input format examines to determine the data type of the input record fields. If the value is zero, all fields will be assumed to be of the STRING data type. | false | 24 | | *dQuotes* | boolean |Specifies that string values in the log are double-quoted. | Log processors might generate W3C logs whose string values are enclosed in double-quotes. | false | 25 | | *separator* | enum |Separator character between fields. | Different W3C log files can use different separator characters between the fields; for example, Exchange Tracking log files use tab characters, while Personal Firewall log files use space characters. The "auto" value instructs the W3C input format to detect automatically the separator character used in the input log(s). | auto/space/tab/character | 26 | 27 | Example Input: 28 | ```json 29 | { 30 | "TimberWinR": { 31 | "Inputs": { 32 | "W3CLogs": [ 33 | { 34 | "location": "C:\\inetpub\\logs\\LogFiles\\W3SVC1\\*" 35 | } 36 | ] 37 | } 38 | } 39 | } 40 | ``` 41 | 42 | 43 | ## Fields 44 | After a successful parse of an event, the following fields are added [(if configured to be logged)](http://www.iis.net/learn/extensions/advanced-logging-module/advanced-logging-readme) 45 | 46 | | Name | Type | Description | 47 | | ---- |:-----| :-----------------------------------------------------------------------| 48 | |LogFilename| STRING | Full path of the log file containing this entry | 49 | |LogRow | INTEGER | Line in the log file containing this entry | 50 | 51 | After the above fields, all other fields selected to be logged will be appended. 52 | -------------------------------------------------------------------------------- /TimberWinR/mdocs/WindowsEvents.md: -------------------------------------------------------------------------------- 1 | # Input: WindowsEvents 2 | 3 | The WindowsEvents input will collect events from the Windows Event Viewer. The source parameter indicates which event 4 | logs to collect data from. You can specify more than one log by using the comma, i.e. "Application,System" will collect 5 | logs from the Application and System event logs. The default interval for scanning for new Events is 60 seconds. 6 | 7 | ## Parameters 8 | The following parameters are allowed when configuring WindowsEvents. 9 | 10 | | Parameter | Type | Description | Legal Values | Default | 11 | | :---------------- |:---------------| :----------------------------------------------------------------------- | :--------------------------- | :-- | 12 | | *source* | string |Windows event logs | Application,System,Security | System | 13 | | *binaryFormat* | string |Format of the "Data" binary field. | ASC,HEX,PRINT | **ASC** | 14 | | *msgErrorMode* | string |Behavior when event messages or event category names cannot be resolved. |NULL,ERROR,MSG | **MSG** | 15 | | *direction* | string |Direction to scan the event logs | FW,BW | **FW** | 16 | | *stringsSep* | string |Separator between values of the "Strings" field. | any string | vertical bar | 17 | | *fullEventCode* | bool |Return the full event ID code instead of the friendly code. | true,false | **false** | 18 | | *fullText* | bool |Retrieve the full text message | true,false | **true** | 19 | | *resolveSIDS* | bool |Resolve SID values into full account names | true,false | **true** | 20 | | *formatMsg* | bool |Format the text message as a single line. | true,false | **true** | 21 | | *interval* | integer | Interval in seconds to sleep during checks | Interval | 60 | 22 | 23 | ### source format 24 | The source indicates where to collect the event(s) from, it can be of these form(s): 25 | When specifying a windows path, make sure to escape the backslash(s). 26 | ``` 27 | "source": "System, Application, Security" 28 | "source": "D:\\MyEVTLogs\\*.evt" 29 | "source": "System, D:\\MyEVTLogs\\System.evt" 30 | ``` 31 | Example Input: 32 | ```json 33 | { 34 | "TimberWinR": { 35 | "Inputs": { 36 | "WindowsEvents": [ 37 | { 38 | "source": "System,Application", 39 | "binaryFormat": "PRINT", 40 | "resolveSIDS": true 41 | } 42 | ] 43 | } 44 | } 45 | } 46 | ``` 47 | ## Fields 48 | After a successful parse of an event, the following fields are added: 49 | 50 | | Name | Type | Description | 51 | | ---- |:-----| :-----------------------------------------------------------------------| 52 | | EventLog | STRING |Name of the Event Log or Event Log backup file containing this event 53 | | RecordNumber | INTEGER | Index of this event in the Event Log or Event Log backup file containing this event | 54 | | TimeGenerated | TIMESTAMP | The date and time at which the event was generated (local time) | 55 | | TimeWritten | TIMESTAMP | The date and time at which the event was logged (local time) | 56 | | EventID | INTEGER | The ID of the event | 57 | | EventType | INTEGER | The numeric type of the event | 58 | | EventTypeName | STRING | The descriptive type of the event | 59 | | EventCategory | INTEGER | The numeric category of the event | 60 | | EventCategoryName | STRING | The descriptive category of the event | 61 | | SourceName | STRING | The source that generated the event | 62 | | Strings | STRING | The textual data associated with the event 63 | | ComputerName | STRING | The name of the computer on which the event was generated | 64 | | SID | STRING | The Security Identifier associated with the event | 65 | | Message | STRING | The full event message | 66 | | Data | STRING | The binary data associated with the event | 67 | 68 | 69 | 70 | -------------------------------------------------------------------------------- /TimberWinR/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /TimberWinR/testconf.xml: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | version: 1.2.{build} 2 | 3 | build: 4 | verbosity: minimal 5 | 6 | assembly_info: 7 | patch: true 8 | file: AssemblyInfo.* 9 | assembly_version: "1.2.{build}" 10 | assembly_file_version: "{version}" 11 | assembly_informational_version: "{version}" 12 | 13 | artifacts: 14 | - path: '**\*.msi' 15 | -------------------------------------------------------------------------------- /chocolatey.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TerribleDev/TimberWinR/5165224c65c81d7e3f26df9f2bc5e6415cc8ceb6/chocolatey.png -------------------------------------------------------------------------------- /chocolateyInstall.ps1.template: -------------------------------------------------------------------------------- 1 | $packageName = 'TimberWinR-${version}' 2 | $fileType = 'msi' 3 | $silentArgs = '/quiet' 4 | $scriptPath = $(Split-Path $MyInvocation.MyCommand.Path) 5 | $fileFullPath = Join-Path $scriptPath 'TimberWinR-${version}.0.msi' 6 | try { 7 | Install-ChocolateyInstallPackage $packageName $fileType $silentArgs $fileFullPath 8 | } catch { 9 | Write-ChocolateyFailure $packageName $($_.Exception.Message) 10 | throw 11 | } 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /chocolateyUninstall.ps1.guid: -------------------------------------------------------------------------------- 1 | $packageName = 'TimberWinR-${version}' # arbitrary name for the package, used in messages 2 | $installerType = 'msi' #only one of these: exe, msi, msu 3 | $scriptPath = $(Split-Path $MyInvocation.MyCommand.Path) 4 | $fileFullPath = Join-Path $scriptPath 'TimberWinR-${version}.0.msi' 5 | $silentArgs = '${PROJECTGUID} /quiet' 6 | $validExitCodes = @(0) #please insert other valid exit codes here, exit codes for ms http://msdn.microsoft.com/en-us/library/aa368542(VS.85).aspx 7 | UnInstall-ChocolateyPackage "$packageName" "$installerType" "$silentArgs" "fileFullPath" -validExitCodes $validExitCodes 8 | -------------------------------------------------------------------------------- /chocolateyUninstall.ps1.template: -------------------------------------------------------------------------------- 1 | $packageName = 'TimberWinR-${version}' # arbitrary name for the package, used in messages 2 | $installerType = 'msi' #only one of these: exe, msi, msu 3 | $scriptPath = $(Split-Path $MyInvocation.MyCommand.Path) 4 | $fileFullPath = Join-Path $scriptPath 'TimberWinR-${version}.0.msi' 5 | $silentArgs = '{E001D138-669B-4604-88C5-02C756461C15} /quiet' 6 | $validExitCodes = @(0) #please insert other valid exit codes here, exit codes for ms http://msdn.microsoft.com/en-us/library/aa368542(VS.85).aspx 7 | UnInstall-ChocolateyPackage "$packageName" "$installerType" "$silentArgs" "fileFullPath" -validExitCodes $validExitCodes 8 | -------------------------------------------------------------------------------- /chocolateyUninstall.ps1.template.orig: -------------------------------------------------------------------------------- 1 | $packageName = 'TimberWinR-${version}' # arbitrary name for the package, used in messages 2 | $installerType = 'msi' #only one of these: exe, msi, msu 3 | $scriptPath = $(Split-Path $MyInvocation.MyCommand.Path) 4 | $fileFullPath = Join-Path $scriptPath 'TimberWinR-${version}.0.msi' 5 | $silentArgs = '${PROJECTGUID} /quiet' 6 | $validExitCodes = @(0) #please insert other valid exit codes here, exit codes for ms http://msdn.microsoft.com/en-us/library/aa368542(VS.85).aspx 7 | UnInstall-ChocolateyPackage "$packageName" "$installerType" "$silentArgs" "fileFullPath" -validExitCodes $validExitCodes -------------------------------------------------------------------------------- /mdocs/DateFilter.md: -------------------------------------------------------------------------------- 1 | # Date Filter 2 | -------------------------------------------------------------------------------- /mdocs/GrokFilter.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TerribleDev/TimberWinR/5165224c65c81d7e3f26df9f2bc5e6415cc8ceb6/mdocs/GrokFilter.md -------------------------------------------------------------------------------- /mdocs/MutateFilter.md: -------------------------------------------------------------------------------- 1 | # Mutate Filter 2 | The mutate filter allows you to perform general mutations on fields. You can rename, remove, replace and modify fields in your events. This filter will automatically be applied to all inputs before sending to the outputs. If you want to make a 3 | filter conditional, use the ***condition*** property to specify a legal C# expression. 4 | 5 | ## Mutate Operations 6 | The following operations are allowed when mutating a field. 7 | 8 | | Operation | Type | Description 9 | | :-----------|:----------------|:-----------------------------------------------------------------------| 10 | | *condition* | property:string |C# Expression 11 | | *rename* | property:array |Rename one or more fields 12 | | *replace* | property:array |Replace a field with a new value. The new value can include %{foo} strings to help you build a new value from other parts of the event. 13 | | *split* | property:array |Separator between values of the "Strings" field. 14 | 15 | ## Details 16 | ### condition "C# expression" 17 | If present, the condition must evaluate to true in order for the remaining operations to be performed. If there is no condition specified 18 | then the operation(s) will be executed in order. 19 | ```json 20 | "Filters": [ 21 | { 22 | "mutate": { 23 | "condition": "[type] == \"Win32-EventLog\"" 24 | "rename": [ 25 | "ComputerName", "Host" 26 | ] 27 | } 28 | } 29 | ] 30 | ``` 31 | The above example will rename ComputerName to Host only for Win32-EventLog types. 32 | 33 | ### rename ["oldname", "newname", ...] 34 | The fields must be in pairs with oldname first and newname second. 35 | ```json 36 | "Filters": [ 37 | { 38 | "mutate": { 39 | "rename": [ 40 | "ComputerName", "Host", 41 | "host", "Host", 42 | "message","Message", 43 | "type","Type", 44 | "SID", "Username" 45 | ] 46 | } 47 | } 48 | ] 49 | ``` 50 | ### replace ["field", "newvalue", ...] 51 | Replaces field with newvalue. The replacements must be described in pairs. 52 | ```json 53 | "Filters": [ 54 | { 55 | "mutate": { 56 | "replace": [ 57 | "message", "%{source_host}: My new message" 58 | ] 59 | } 60 | } 61 | ] 62 | ``` 63 | ### split 64 | Split a field into an array of values. The first arguments is the fieldName and the second is the separator. 65 | ```json 66 | "Filters": [ 67 | { 68 | "mutate": { 69 | "split": [ 70 | "InsertionStrings", "|" 71 | ] 72 | } 73 | } 74 | ] 75 | ``` 76 | 77 | 78 | -------------------------------------------------------------------------------- /packages/repositories.config: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /timberwinr.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TerribleDev/TimberWinR/5165224c65c81d7e3f26df9f2bc5e6415cc8ceb6/timberwinr.jpg -------------------------------------------------------------------------------- /timberwinr.nuspec.template: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | TimberWinR 7 | TimberWinR 8 | ${version}.0 9 | efontana 10 | Eric Fontana 11 | TimberWinR Shipper 12 | TimberWinR Native .NET Logstash Shipper. Use https://groups.google.com/forum/#!forum/timberwinr for support. 13 | https://github.com/Cimpress-MCP/TimberWinR 14 | TimberWinR admin 15 | 16 | http://www.ericfontana.com/timberwinr.jpg 17 | http://www.apache.org/licenses/LICENSE-2.0.html 18 | false 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | --------------------------------------------------------------------------------