6 |

7 |

8 |

10 |

11 |

12 |

13 | 14 | 15 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /src/CollectdWinService/CollectdWinService.cs: -------------------------------------------------------------------------------- 1 | using System.ServiceProcess; 2 | using NLog; 3 | 4 | namespace BloombergFLP.CollectdWin 5 | { 6 | public class CollectdWinService : ServiceBase 7 | { 8 | private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); 9 | private MetricsCollector _metricsCollector; 10 | 11 | public CollectdWinService() 12 | { 13 | InitializeComponent(); 14 | } 15 | 16 | protected override void OnStart(string[] args) 17 | { 18 | StartService(args); 19 | } 20 | 21 | protected override void OnStop() 22 | { 23 | StopService(); 24 | } 25 | 26 | private void InitializeComponent() 27 | { 28 | ServiceName = "Bloomberg Metrics Collector Service"; 29 | } 30 | 31 | // public accessibility for running as a console application 32 | public virtual void StartService(params string[] args) 33 | { 34 | Logger.Trace("StartService() begin"); 35 | _metricsCollector = new MetricsCollector(); 36 | _metricsCollector.ConfigureAll(); 37 | _metricsCollector.StartAll(); 38 | Logger.Trace("StartService() return"); 39 | } 40 | 41 | // public accessibility for running as a console application 42 | public virtual void StopService() 43 | { 44 | Logger.Trace("StopService() begin"); 45 | _metricsCollector.StopAll(); 46 | Logger.Trace("StopService() return"); 47 | } 48 | } 49 | } 50 | 51 | // ---------------------------------------------------------------------------- 52 | // Copyright (C) 2015 Bloomberg Finance L.P. 53 | // 54 | // Licensed under the Apache License, Version 2.0 (the "License"); 55 | // you may not use this file except in compliance with the License. 56 | // You may obtain a copy of the License at 57 | // http://www.apache.org/licenses/LICENSE-2.0 58 | // Unless required by applicable law or agreed to in writing, software 59 | // distributed under the License is distributed on an "AS IS" BASIS, 60 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 61 | // See the License for the specific language governing permissions and 62 | // limitations under the License. 63 | // 64 | // ----------------------------- END-OF-FILE ---------------------------------- -------------------------------------------------------------------------------- /src/CollectdWinService/Util.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Configuration; 3 | using System.Net; 4 | using System.Net.Sockets; 5 | using NLog; 6 | 7 | namespace BloombergFLP.CollectdWin 8 | { 9 | public static class Util 10 | { 11 | private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); 12 | 13 | public static double GetNow() 14 | { 15 | TimeSpan t = DateTime.UtcNow - new DateTime(1970, 1, 1); 16 | double epoch = t.TotalMilliseconds/1000; 17 | double now = Math.Round(epoch, 3); 18 | return (now); 19 | } 20 | 21 | public static string GetHostName() 22 | { 23 | string hostname = Environment.MachineName.ToLower(); 24 | try 25 | { 26 | hostname = Dns.GetHostEntry("localhost").HostName.ToLower(); 27 | } 28 | catch (SocketException) 29 | { 30 | Logger.Warn("Unable to resolve hostname, using MachineName: {0}", hostname); 31 | } 32 | var generalConfig = ConfigurationManager.GetSection("CollectdWinConfig") as CollectdWinConfig; 33 | if (generalConfig == null) 34 | { 35 | Logger.Error("Cannot get configuration section"); 36 | return hostname; 37 | } 38 | if (! (String.IsNullOrEmpty(generalConfig.GeneralSettings.HostName))) 39 | { 40 | hostname = generalConfig.GeneralSettings.HostName; 41 | } 42 | return hostname; 43 | } 44 | } 45 | } 46 | 47 | // ---------------------------------------------------------------------------- 48 | // Copyright (C) 2015 Bloomberg Finance L.P. 49 | // 50 | // Licensed under the Apache License, Version 2.0 (the "License"); 51 | // you may not use this file except in compliance with the License. 52 | // You may obtain a copy of the License at 53 | // http://www.apache.org/licenses/LICENSE-2.0 54 | // Unless required by applicable law or agreed to in writing, software 55 | // distributed under the License is distributed on an "AS IS" BASIS, 56 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 57 | // See the License for the specific language governing permissions and 58 | // limitations under the License. 59 | // 60 | // ----------------------------- END-OF-FILE ---------------------------------- -------------------------------------------------------------------------------- /src/CollectdWinService/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Configuration; 3 | using System.ServiceProcess; 4 | using NLog; 5 | 6 | namespace BloombergFLP.CollectdWin 7 | { 8 | public static class Program 9 | { 10 | private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); 11 | 12 | ///

13 | /// The main entry point for the application. 14 | ///

15 | public static void Main(string[] args) 16 | { 17 | var config = ConfigurationManager.GetSection("CollectdWinConfig") as CollectdWinConfig; 18 | if (config == null) 19 | { 20 | Logger.Error("Main(): cannot get configuration section"); 21 | return; 22 | } 23 | 24 | var collectdWinService = new CollectdWinService(); 25 | 26 | if (Array.Find(args, s => s.Equals(@"console")) != null) 27 | { 28 | Console.WriteLine("*** Starting CollectdWin in console mode***"); 29 | // run as a console application for testing and debugging purpose 30 | collectdWinService.StartService(); 31 | Console.WriteLine("*** Enter Ctrl-C to exit: ***"); 32 | Console.ReadLine(); 33 | } 34 | else 35 | { 36 | // run as a windows service 37 | ServiceBase[] servicesToRun = {collectdWinService}; 38 | ServiceBase.Run(servicesToRun); 39 | } 40 | Logger.Error("CollectdWin: exiting ..."); 41 | } 42 | } 43 | } 44 | 45 | // ---------------------------------------------------------------------------- 46 | // Copyright (C) 2015 Bloomberg Finance L.P. 47 | // 48 | // Licensed under the Apache License, Version 2.0 (the "License"); 49 | // you may not use this file except in compliance with the License. 50 | // You may obtain a copy of the License at 51 | // http://www.apache.org/licenses/LICENSE-2.0 52 | // Unless required by applicable law or agreed to in writing, software 53 | // distributed under the License is distributed on an "AS IS" BASIS, 54 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 55 | // See the License for the specific language governing permissions and 56 | // limitations under the License. 57 | // 58 | // ----------------------------- END-OF-FILE ---------------------------------- -------------------------------------------------------------------------------- /src/CollectdWinTest/MetricValueTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.VisualStudio.TestTools.UnitTesting; 3 | using BloombergFLP.CollectdWin; 4 | 5 | namespace BloombergFLP.CollectdWinTest 6 | { 7 | [TestClass] 8 | public class MetricValueTests 9 | { 10 | private MetricValue val1, val2; 11 | private string val1Str, val2Str; 12 | 13 | 14 | public MetricValueTests() 15 | { 16 | initTest(); 17 | } 18 | 19 | private void initTest() 20 | { 21 | val1 = new MetricValue 22 | { 23 | HostName = "host-1", 24 | PluginName = "cpu", 25 | PluginInstanceName = "", 26 | TypeName = "percent", 27 | TypeInstanceName = "", 28 | Values = new double[] { 123.0 } 29 | }; 30 | val1Str = @"{""host"":""host-1"", ""plugin"":""cpu"", ""plugin_instance"":"""", ""type"":""percent"", " + 31 | @"""type_instance"":"""", ""time"":0, ""interval"":0, ""dstypes"":[""gauge""], ""dsnames"":[""value""], ""values"":[123]}"; 32 | val2 = new MetricValue 33 | { 34 | HostName = "host-2", 35 | PluginName = "cpu", 36 | PluginInstanceName = "", 37 | TypeName = "percent", 38 | TypeInstanceName = "", 39 | Values = new double[] { 123.0 }, 40 | }; 41 | val2.AddMetaData("region", "ny"); 42 | val2.AddMetaData("dc", "datacenter-1"); 43 | val2Str = @"{""host"":""host-2"", ""plugin"":""cpu"", ""plugin_instance"":"""", ""type"":""percent"", " + 44 | @"""type_instance"":"""", ""time"":0, ""interval"":0, ""dstypes"":[""gauge""], ""dsnames"":[""value""], ""values"":[123]" + 45 | @", ""meta"":{""dc"":""datacenter-1"",""region"":""ny""}}"; 46 | } 47 | 48 | [TestMethod] 49 | public void MetricWithEmptyTags() 50 | { 51 | string expected = val1.GetMetricJsonStr(); 52 | Assert.AreEqual(expected, val1Str, "MetricWithEmptyTags failed"); 53 | } 54 | 55 | [TestMethod] 56 | public void MetricWithTwoTags() 57 | { 58 | string expected = val2.GetMetricJsonStr(); 59 | Assert.AreEqual(expected, val2Str, "MetricWithTwoTags failed"); 60 | } 61 | } 62 | } -------------------------------------------------------------------------------- /src/installer/CollectdWin.wixproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Debug 5 | x86 6 | 3.9 7 | 93cc9e80-b6f5-4768-998e-889bdf5ab579 8 | 2.0 9 | CollectdWin-$(Platform) 10 | Package 11 | $(MSBuildExtensionsPath32)\Microsoft\WiX\v3.x\Wix.targets 12 | $(MSBuildExtensionsPath)\Microsoft\WiX\v3.x\Wix.targets 13 | CollectdWin 14 | 15 | 16 | bin\$(Platform)\$(Configuration)\ 17 | obj\$(Platform)\$(Configuration)\ 18 | Debug 19 | 20 | 21 | bin\$(Platform)\$(Configuration)\ 22 | obj\$(Platform)\$(Configuration)\ 23 | 24 | 25 | Debug 26 | bin\$(Platform)\$(Configuration)\ 27 | obj\$(Platform)\$(Configuration)\ 28 | 29 | 30 | bin\$(Platform)\$(Configuration)\ 31 | obj\$(Platform)\$(Configuration)\ 32 | 33 | 34 | 35 | 36 | 37 | 45 | 46 | -------------------------------------------------------------------------------- /src/CollectdWinService/PluginRegistry.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Configuration; 4 | using System.Linq; 5 | using NLog; 6 | 7 | namespace BloombergFLP.CollectdWin 8 | { 9 | internal class PluginRegistry 10 | { 11 | private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); 12 | 13 | private readonly Dictionary _registry = new Dictionary(); 14 | 15 | public PluginRegistry() 16 | { 17 | var config = ConfigurationManager.GetSection("CollectdWinConfig") as CollectdWinConfig; 18 | if (config == null) 19 | { 20 | Logger.Error("Cannot get configuration section : CollectdWinConfig"); 21 | return; 22 | } 23 | foreach ( 24 | CollectdWinConfig.PluginConfig pluginConfig in 25 | config.Plugins.Cast() 26 | .Where(pluginConfig => pluginConfig.Enable)) 27 | { 28 | _registry[pluginConfig.Name] = pluginConfig.Class; 29 | } 30 | } 31 | 32 | public IList CreatePlugins() 33 | { 34 | IList plugins = new List(); 35 | foreach (var entry in _registry) 36 | { 37 | Type classType = Type.GetType(entry.Value); 38 | if (classType == null) 39 | { 40 | Logger.Error("Cannot create plugin:{0}, class:{1}", entry.Key, entry.Value); 41 | continue; 42 | } 43 | var plugin = (IMetricsPlugin) Activator.CreateInstance(classType); 44 | plugins.Add(plugin); 45 | } 46 | return (plugins); 47 | } 48 | } 49 | } 50 | 51 | // ---------------------------------------------------------------------------- 52 | // Copyright (C) 2015 Bloomberg Finance L.P. 53 | // 54 | // Licensed under the Apache License, Version 2.0 (the "License"); 55 | // you may not use this file except in compliance with the License. 56 | // You may obtain a copy of the License at 57 | // http://www.apache.org/licenses/LICENSE-2.0 58 | // Unless required by applicable law or agreed to in writing, software 59 | // distributed under the License is distributed on an "AS IS" BASIS, 60 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 61 | // See the License for the specific language governing permissions and 62 | // limitations under the License. 63 | // 64 | // ----------------------------- END-OF-FILE ---------------------------------- -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.sln.docstates 8 | src/CollectdWinService.userprefs 9 | 10 | # Build results 11 | 12 | [Dd]ebug/ 13 | [Rr]elease/ 14 | x64/ 15 | build/ 16 | [Bb]in/ 17 | [Oo]bj/ 18 | 19 | # Enable "build/" folder in the NuGet Packages folder since NuGet packages use it for MSBuild targets 20 | !packages/*/build/ 21 | 22 | # MSTest test Results 23 | [Tt]est[Rr]esult*/ 24 | [Bb]uild[Ll]og.* 25 | 26 | *_i.c 27 | *_p.c 28 | *.ilk 29 | *.meta 30 | *.obj 31 | *.pch 32 | *.pdb 33 | *.pgc 34 | *.pgd 35 | *.rsp 36 | *.sbr 37 | *.tlb 38 | *.tli 39 | *.tlh 40 | *.tmp 41 | *.tmp_proj 42 | *.log 43 | *.vspscc 44 | *.vssscc 45 | .builds 46 | *.pidb 47 | *.log 48 | *.scc 49 | 50 | # Visual C++ cache files 51 | ipch/ 52 | *.aps 53 | *.ncb 54 | *.opensdf 55 | *.sdf 56 | *.cachefile 57 | 58 | # Visual Studio profiler 59 | *.psess 60 | *.vsp 61 | *.vspx 62 | 63 | # Guidance Automation Toolkit 64 | *.gpState 65 | 66 | # ReSharper is a .NET coding add-in 67 | _ReSharper*/ 68 | *.[Rr]e[Ss]harper 69 | 70 | # TeamCity is a build add-in 71 | _TeamCity* 72 | 73 | # DotCover is a Code Coverage Tool 74 | *.dotCover 75 | 76 | # NCrunch 77 | *.ncrunch* 78 | .*crunch*.local.xml 79 | 80 | # Installshield output folder 81 | [Ee]xpress/ 82 | 83 | # DocProject is a documentation generator add-in 84 | DocProject/buildhelp/ 85 | DocProject/Help/*.HxT 86 | DocProject/Help/*.HxC 87 | DocProject/Help/*.hhc 88 | DocProject/Help/*.hhk 89 | DocProject/Help/*.hhp 90 | DocProject/Help/Html2 91 | DocProject/Help/html 92 | 93 | # Click-Once directory 94 | publish/ 95 | 96 | # Publish Web Output 97 | *.Publish.xml 98 | 99 | # NuGet Packages Directory 100 | ## TODO: If you have NuGet Package Restore enabled, uncomment the next line 101 | #packages/ 102 | 103 | # Windows Azure Build Output 104 | csx 105 | *.build.csdef 106 | 107 | # Windows Store app package directory 108 | AppPackages/ 109 | 110 | # Others 111 | sql/ 112 | *.Cache 113 | ClientBin/ 114 | [Ss]tyle[Cc]op.* 115 | ~$* 116 | *~ 117 | *.dbmdl 118 | *.[Pp]ublish.xml 119 | *.pfx 120 | *.publishsettings 121 | 122 | # RIA/Silverlight projects 123 | Generated_Code/ 124 | 125 | # Backup & report files from converting an old project file to a newer 126 | # Visual Studio version. Backup files are not needed, because we have git ;-) 127 | _UpgradeReport_Files/ 128 | Backup*/ 129 | UpgradeLog*.XML 130 | UpgradeLog*.htm 131 | 132 | # SQL Server files 133 | App_Data/*.mdf 134 | App_Data/*.ldf 135 | 136 | 137 | #LightSwitch generated files 138 | GeneratedArtifacts/ 139 | _Pvt_Extensions/ 140 | ModelManifest.xml 141 | 142 | # ========================= 143 | # Windows detritus 144 | # ========================= 145 | 146 | # Windows image file caches 147 | Thumbs.db 148 | ehthumbs.db 149 | 150 | # Folder config file 151 | Desktop.ini 152 | 153 | # Recycle Bin used on file shares 154 | $RECYCLE.BIN/ 155 | 156 | # Mac desktop service store files 157 | .DS_Store 158 | 159 | # IntelliJ IDEA 160 | .idea 161 | -------------------------------------------------------------------------------- /tools/version.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import fileinput 3 | import re 4 | import sys 5 | 6 | # 7 | # This script gets or updates version number. Version number 8 | # is in AssemblyInfo.cs with the following format: 9 | # ... 10 | # 11 | # Example: 12 | # [assembly: AssemblyVersion("0.5.3.0")] 13 | # [assembly: AssemblyFileVersion("0.5.3.0")] 14 | # 15 | 16 | def usage(parser) : 17 | parser.print_help() 18 | sys.exit(1); 19 | 20 | def main( ) : 21 | VERSION_FILE = 'src\CollectdWinService\Properties\AssemblyInfo.cs' 22 | VERSION_FORMAT = "{0}.{1}.{2}.{3}" 23 | VERSION_PATTERN = '^\[assembly: AssemblyVersion$\"(\d+).(\d+).(\d+).(\d+)\"$\]' 24 | REPLACE_PATTERN = r"(^\[assembly: Assembly.*Version$\").*(\"$\])" 25 | REPLACE_FORMAT = r"\g<1>{0}\g<2>" 26 | 27 | parser = argparse.ArgumentParser() 28 | parser.add_argument("--command", help="get|update") 29 | parser.add_argument("--part", help="major|minor|build|revision") 30 | args = parser.parse_args() 31 | 32 | vfile = open(VERSION_FILE) 33 | for line in vfile: 34 | m = re.match(VERSION_PATTERN, line) 35 | if m: 36 | cmajor = int(m.group(1)) 37 | cminor = int(m.group(2)) 38 | cbuild = int(m.group(3)) 39 | crevision = int(m.group(4)) 40 | cversion = VERSION_FORMAT.format(cmajor, cminor, cbuild, crevision) 41 | 42 | vfile.close() 43 | if args.command == "get" : 44 | print(cversion) 45 | sys.exit(0) 46 | elif args.command != "update" : 47 | print("\nError: Missing or bad COMMAND\n") 48 | usage(parser) 49 | 50 | if args.part == "major" : 51 | nmajor = cmajor + 1 52 | nminor = 0 53 | nbuild = 0 54 | nrevision = 0 55 | elif args.part == "minor" : 56 | nmajor = cmajor 57 | nminor = cminor + 1 58 | nbuild = 0 59 | nrevision = 0 60 | elif args.part == "build" : 61 | nmajor = cmajor 62 | nminor = cminor 63 | nbuild = cbuild + 1 64 | nrevision = 0 65 | elif args.part == "revision" : 66 | nmajor = cmajor 67 | nminor = cminor 68 | nbuild = cbuild 69 | nrevision = crevision + 1 70 | else : 71 | print("\nError: Missing or bad PART\n") 72 | usage(parser) 73 | 74 | nversion = VERSION_FORMAT.format(nmajor, nminor, nbuild, nrevision) 75 | 76 | for line in fileinput.input(files=[VERSION_FILE], inplace=1, backup='.bak'): 77 | line = re.sub(REPLACE_PATTERN, REPLACE_FORMAT.format(nversion), line.rstrip()) 78 | print(line) 79 | 80 | 81 | if __name__ == "__main__": 82 | main( ) 83 | 84 | 85 | #----------------------------------------------------------------------------- 86 | # Copyright (C) 2015 Bloomberg Finance L.P. 87 | # 88 | # Licensed under the Apache License, Version 2.0 (the "License"); 89 | # you may not use this file except in compliance with the License. 90 | # You may obtain a copy of the License at 91 | # http://www.apache.org/licenses/LICENSE-2.0 92 | # Unless required by applicable law or agreed to in writing, software 93 | # distributed under the License is distributed on an "AS IS" BASIS, 94 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 95 | # See the License for the specific language governing permissions and 96 | # limitations under the License. 97 | # 98 | #------------------------------ END-OF-FILE ---------------------------------- 99 | -------------------------------------------------------------------------------- /src/CollectdWinService/config/AmqpPluginConfig.cs: -------------------------------------------------------------------------------- 1 | using System.Configuration; 2 | 3 | namespace BloombergFLP.CollectdWin 4 | { 5 | internal class AmqpPluginConfig : ConfigurationSection 6 | { 7 | [ConfigurationProperty("Publish", IsRequired = false)] 8 | public PublishConfig Publish 9 | { 10 | get { return (PublishConfig) base["Publish"]; } 11 | set { base["Publish"] = value; } 12 | } 13 | 14 | public static AmqpPluginConfig GetConfig() 15 | { 16 | return (AmqpPluginConfig) ConfigurationManager.GetSection("Amqp") ?? new AmqpPluginConfig(); 17 | } 18 | 19 | public sealed class PublishConfig : ConfigurationElement 20 | { 21 | [ConfigurationProperty("Name", IsRequired = true)] 22 | public string Name 23 | { 24 | get { return (string) base["Name"]; } 25 | set { base["Name"] = value; } 26 | } 27 | 28 | [ConfigurationProperty("Host", IsRequired = false)] 29 | public string Host 30 | { 31 | get { return (string) base["Host"]; } 32 | set { base["Host"] = value; } 33 | } 34 | 35 | [ConfigurationProperty("Port", IsRequired = true)] 36 | public int Port 37 | { 38 | get { return (int) base["Port"]; } 39 | set { base["Port"] = value; } 40 | } 41 | 42 | [ConfigurationProperty("VirtualHost", IsRequired = false)] 43 | public string VirtualHost 44 | { 45 | get { return (string) base["VirtualHost"]; } 46 | set { base["VirtualHost"] = value; } 47 | } 48 | 49 | [ConfigurationProperty("User", IsRequired = false)] 50 | public string User 51 | { 52 | get { return (string) base["User"]; } 53 | set { base["User"] = value; } 54 | } 55 | 56 | [ConfigurationProperty("Password", IsRequired = false)] 57 | public string Password 58 | { 59 | get { return (string) base["Password"]; } 60 | set { base["Password"] = value; } 61 | } 62 | 63 | [ConfigurationProperty("Exchange", IsRequired = false)] 64 | public string Exchange 65 | { 66 | get { return (string) base["Exchange"]; } 67 | set { base["Exchange"] = value; } 68 | } 69 | 70 | [ConfigurationProperty("RoutingKeyPrefix", IsRequired = false)] 71 | public string RoutingKeyPrefix 72 | { 73 | get { return (string) base["RoutingKeyPrefix"]; } 74 | set { base["RoutingKeyPrefix"] = value; } 75 | } 76 | } 77 | } 78 | } 79 | 80 | // ---------------------------------------------------------------------------- 81 | // Copyright (C) 2015 Bloomberg Finance L.P. 82 | // 83 | // Licensed under the Apache License, Version 2.0 (the "License"); 84 | // you may not use this file except in compliance with the License. 85 | // You may obtain a copy of the License at 86 | // http://www.apache.org/licenses/LICENSE-2.0 87 | // Unless required by applicable law or agreed to in writing, software 88 | // distributed under the License is distributed on an "AS IS" BASIS, 89 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 90 | // See the License for the specific language governing permissions and 91 | // limitations under the License. 92 | // 93 | // ----------------------------- END-OF-FILE ---------------------------------- -------------------------------------------------------------------------------- /src/CollectdWinService/StatsdListener.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Net; 3 | using System.Net.Sockets; 4 | using System.Text; 5 | using System.Threading; 6 | using NLog; 7 | 8 | namespace BloombergFLP.CollectdWin 9 | { 10 | internal class StatsdListener 11 | { 12 | public delegate void HandleMessage(string message); 13 | 14 | private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); 15 | 16 | private readonly IPEndPoint _endPoint; 17 | 18 | private readonly HandleMessage _messageHandler; 19 | private readonly Socket _socket; 20 | private bool _run; 21 | 22 | public StatsdListener(int port, HandleMessage handleMessage) 23 | { 24 | _messageHandler = handleMessage; 25 | _endPoint = new IPEndPoint(IPAddress.Any, port); 26 | _socket = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); 27 | } 28 | 29 | private void BindSocket() 30 | { 31 | while (!_socket.IsBound) 32 | { 33 | try 34 | { 35 | _socket.Bind(_endPoint); 36 | } 37 | catch (Exception exp) 38 | { 39 | Logger.Error("BindSocket failed: ", exp); 40 | } 41 | if (_socket.IsBound) 42 | break; 43 | Thread.Sleep(10*1000); 44 | } 45 | } 46 | 47 | private void CloseSocket() 48 | { 49 | try 50 | { 51 | _socket.Shutdown(SocketShutdown.Both); 52 | _socket.Close(); 53 | } 54 | catch (Exception exp) 55 | { 56 | Logger.Error("CloseSocket failed: ", exp); 57 | } 58 | } 59 | 60 | public void Start() 61 | { 62 | Logger.Trace("Start() begin"); 63 | var buffer = new byte[1024*4]; 64 | 65 | var sender = new IPEndPoint(IPAddress.Any, 0); 66 | EndPoint remote = sender; 67 | 68 | _run = true; 69 | while (_run) 70 | { 71 | if (!_socket.IsBound) 72 | { 73 | BindSocket(); 74 | } 75 | try 76 | { 77 | int recv = _socket.ReceiveFrom(buffer, ref remote); 78 | string str = Encoding.ASCII.GetString(buffer, 0, recv); 79 | str = str.TrimEnd('\r', '\n'); 80 | 81 | _messageHandler(str); 82 | } 83 | catch 84 | { 85 | CloseSocket(); 86 | } 87 | } 88 | 89 | Logger.Trace("Start() end"); 90 | } 91 | 92 | public void Stop() 93 | { 94 | Logger.Trace("Stop() begin"); 95 | _run = false; 96 | // closing socket will cause Socket.ReceiveFrom() blocked call to 97 | // throw SocketException, a work-around for shutting down a listener. 98 | CloseSocket(); 99 | Logger.Trace("Stop() end"); 100 | } 101 | } 102 | } 103 | 104 | // ---------------------------------------------------------------------------- 105 | // Copyright (C) 2015 Bloomberg Finance L.P. 106 | // 107 | // Licensed under the Apache License, Version 2.0 (the "License"); 108 | // you may not use this file except in compliance with the License. 109 | // You may obtain a copy of the License at 110 | // http://www.apache.org/licenses/LICENSE-2.0 111 | // Unless required by applicable law or agreed to in writing, software 112 | // distributed under the License is distributed on an "AS IS" BASIS, 113 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 114 | // See the License for the specific language governing permissions and 115 | // limitations under the License. 116 | // 117 | // ----------------------------- END-OF-FILE ---------------------------------- -------------------------------------------------------------------------------- /src/CollectdWinService/StatsdPlugin.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Configuration; 4 | using System.Linq; 5 | using System.Threading; 6 | using NLog; 7 | 8 | namespace BloombergFLP.CollectdWin 9 | { 10 | internal class StatsdPlugin : IMetricsReadPlugin 11 | { 12 | private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); 13 | private bool _delCounters; 14 | private bool _delGauges, _delSets; 15 | private bool _delTimers; 16 | private float[] _percentiles; 17 | private int _port; 18 | private bool _running; 19 | private StatsdAggregator _statsdAggregator; 20 | private StatsdListener _statsdListener; 21 | private Thread _statsdThread; 22 | private bool _timerCount; 23 | private bool _timerLower; 24 | private bool _timerSum; 25 | private bool _timerUpper; 26 | 27 | public StatsdPlugin() 28 | { 29 | _running = false; 30 | } 31 | 32 | public void Configure() 33 | { 34 | var config = ConfigurationManager.GetSection("Statsd") as StatsdPluginConfig; 35 | if (config == null) 36 | { 37 | throw new Exception("Cannot get configuration section : Statsd"); 38 | } 39 | 40 | _port = config.Server.Port; 41 | 42 | _delCounters = config.DeleteCache.Counters; 43 | _delTimers = config.DeleteCache.Timers; 44 | _delGauges = config.DeleteCache.Gauges; 45 | _delSets = config.DeleteCache.Sets; 46 | 47 | _timerLower = config.Timer.Lower; 48 | _timerUpper = config.Timer.Upper; 49 | _timerSum = config.Timer.Sum; 50 | _timerCount = config.Timer.Count; 51 | _percentiles = 52 | (from StatsdPluginConfig.PercentileConfig percentileConfig in 53 | config.Timer.Percentiles 54 | select percentileConfig.Value).ToArray(); 55 | 56 | _statsdAggregator = new StatsdAggregator(_delCounters, _delTimers, _delGauges, _delSets, _timerLower, 57 | _timerUpper, 58 | _timerSum, _timerCount, _percentiles); 59 | Logger.Info("Statsd plugin configured"); 60 | } 61 | 62 | public void Start() 63 | { 64 | if (_running) 65 | return; 66 | 67 | _statsdListener = new StatsdListener(_port, HandleMessage); 68 | _statsdThread = new Thread(_statsdListener.Start); 69 | _statsdThread.Start(); 70 | 71 | _running = true; 72 | Logger.Info("Statsd plugin started"); 73 | } 74 | 75 | public void Stop() 76 | { 77 | if (!_running) 78 | return; 79 | _statsdListener.Stop(); 80 | _statsdThread.Interrupt(); 81 | 82 | _running = false; 83 | Logger.Info("Statsd plugin stopped"); 84 | } 85 | 86 | public IList Read() 87 | { 88 | return (_statsdAggregator.Read()); 89 | } 90 | 91 | public void HandleMessage(string message) 92 | { 93 | IList metrics = StatsdMetricParser.Parse(message); 94 | foreach (StatsdMetric metric in metrics) 95 | { 96 | _statsdAggregator.AddMetric(metric); 97 | } 98 | } 99 | } 100 | } 101 | 102 | // ---------------------------------------------------------------------------- 103 | // Copyright (C) 2015 Bloomberg Finance L.P. 104 | // 105 | // Licensed under the Apache License, Version 2.0 (the "License"); 106 | // you may not use this file except in compliance with the License. 107 | // You may obtain a copy of the License at 108 | // http://www.apache.org/licenses/LICENSE-2.0 109 | // Unless required by applicable law or agreed to in writing, software 110 | // distributed under the License is distributed on an "AS IS" BASIS, 111 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 112 | // See the License for the specific language governing permissions and 113 | // limitations under the License. 114 | // 115 | // ----------------------------- END-OF-FILE ---------------------------------- -------------------------------------------------------------------------------- /src/CollectdWinService/Histogram.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using NLog; 3 | 4 | namespace BloombergFLP.CollectdWin 5 | { 6 | internal class Histogram 7 | { 8 | private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); 9 | private readonly int[] _bins; 10 | private readonly int _numBins; 11 | private int _binSize; 12 | private double _max; 13 | private double _min; 14 | private int _num; 15 | private double _sum; 16 | 17 | public Histogram() 18 | { 19 | _min = _max = _sum = _num = 0; 20 | _numBins = 100; 21 | _bins = new int[100]; 22 | _binSize = 8; 23 | } 24 | 25 | private int GetBin(double val) 26 | { 27 | var bin = (int) (val/_binSize); 28 | return (bin); 29 | } 30 | 31 | public void Resize(double val) 32 | { 33 | double requiredBinSize = val/_numBins; 34 | // to reduce frequent resizing, new bin size will be the the next nearest power of 2 35 | // eg: 16, 32, 64, 128, 256, 512, 1024, 2048, 5086 36 | var newBinSize = (int) Math.Pow(2, Math.Ceiling(Math.Log(requiredBinSize, 2))); 37 | int oldBinSize = _binSize; 38 | for (int i = 1; i < _numBins; i++) 39 | { 40 | val = i*oldBinSize; 41 | int newBin = (int) val/newBinSize; 42 | if (i == newBin) 43 | continue; 44 | _bins[newBin] += _bins[i]; 45 | _bins[i] = 0; 46 | } 47 | _binSize = newBinSize; 48 | Logger.Debug("Resize() - OldBinSize:{0} has been replaced with the NewBinSize:{1}", oldBinSize, newBinSize); 49 | } 50 | 51 | public void AddValue(double val) 52 | { 53 | int bin = GetBin(val); 54 | if (bin >= _numBins) 55 | { 56 | Resize(val); 57 | bin = GetBin(val); 58 | Logger.Debug("Got new bin:{0}", bin); 59 | } 60 | _bins[bin]++; 61 | 62 | _min = (_min > val) ? val : _min; 63 | _max = (_max < val) ? val : _max; 64 | _sum += val; 65 | _num++; 66 | } 67 | 68 | public double GetPercentile(float percent) 69 | { 70 | double percentUpper = 0; 71 | double percentLower = 0; 72 | double sum = 0; 73 | 74 | if (percent < 0 || percent > 100 || _num <= 0) 75 | return (0); 76 | 77 | int i; 78 | for (i = 0; i < _numBins; i++) 79 | { 80 | percentLower = percentUpper; 81 | sum += _bins[i]; 82 | percentUpper = 100*(sum/_num); 83 | if (percentUpper >= percent) 84 | break; 85 | } 86 | if (Math.Abs(percentUpper) < 0.01 || i >= _numBins) 87 | return (0); 88 | double valLower = i*_binSize; 89 | double valUpper = (i + 1)*_binSize; 90 | 91 | double val = (((percentUpper - percent)*valLower) + ((percent - percentLower)*valUpper))/ 92 | (percentUpper - percentLower); 93 | 94 | return (val); 95 | } 96 | 97 | public void Reset() 98 | { 99 | _min = _max = _sum = _num = 0; 100 | for (int i = 0; i < _numBins; i++) 101 | _bins[i] = 0; 102 | } 103 | 104 | public override string ToString() 105 | { 106 | String logstr = String.Format("Min:{0} Max:{1} Sum{2} Num{3}", _min, _max, _sum, _num); 107 | for (int i = 0; i < _numBins; i++) 108 | logstr += ", " + _bins[i]; 109 | return (logstr); 110 | } 111 | } 112 | } 113 | 114 | // ---------------------------------------------------------------------------- 115 | // Copyright (C) 2015 Bloomberg Finance L.P. 116 | // 117 | // Licensed under the Apache License, Version 2.0 (the "License"); 118 | // you may not use this file except in compliance with the License. 119 | // You may obtain a copy of the License at 120 | // http://www.apache.org/licenses/LICENSE-2.0 121 | // Unless required by applicable law or agreed to in writing, software 122 | // distributed under the License is distributed on an "AS IS" BASIS, 123 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 124 | // See the License for the specific language governing permissions and 125 | // limitations under the License. 126 | // 127 | // ----------------------------- END-OF-FILE ---------------------------------- -------------------------------------------------------------------------------- /src/CollectdWinService/config/WindowsPerformanceCounter.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 8 | 10 | 12 | 14 | 15 | 17 | 20 | 22 | 24 | 25 | 27 | 28 | 31 | 32 | 34 | 36 | 37 | 39 | 41 | 43 | 44 | 46 | 47 | 50 | 51 | 54 | 55 | 58 | 59 | 61 | 62 | 64 | 66 | 68 | 69 | -------------------------------------------------------------------------------- /src/CollectdWinService.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 14 4 | VisualStudioVersion = 14.0.23107.0 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CollectdWinService", "CollectdWinService\CollectdWinService.csproj", "{D4244E6B-84EC-41A0-96A7-4F489F6098BC}" 7 | EndProject 8 | Project("{930C7802-8A8C-48F9-8165-68863BCCD9DD}") = "CollectdWin", "installer\CollectdWin.wixproj", "{93CC9E80-B6F5-4768-998E-889BDF5AB579}" 9 | ProjectSection(ProjectDependencies) = postProject 10 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC} = {D4244E6B-84EC-41A0-96A7-4F489F6098BC} 11 | EndProjectSection 12 | EndProject 13 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CollectdWinTest", "CollectdWinTest\CollectdWinTest.csproj", "{6F05D77A-F180-4779-95FE-75A5223417DF}" 14 | EndProject 15 | Global 16 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 17 | Debug|Any CPU = Debug|Any CPU 18 | Debug|Mixed Platforms = Debug|Mixed Platforms 19 | Debug|x64 = Debug|x64 20 | Debug|x86 = Debug|x86 21 | Release|Any CPU = Release|Any CPU 22 | Release|Mixed Platforms = Release|Mixed Platforms 23 | Release|x64 = Release|x64 24 | Release|x86 = Release|x86 25 | EndGlobalSection 26 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 27 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 28 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC}.Debug|Any CPU.Build.0 = Debug|Any CPU 29 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC}.Debug|Mixed Platforms.ActiveCfg = Debug|x86 30 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC}.Debug|Mixed Platforms.Build.0 = Debug|x86 31 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC}.Debug|Mixed Platforms.Deploy.0 = Debug|x86 32 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC}.Debug|x64.ActiveCfg = Debug|x64 33 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC}.Debug|x64.Build.0 = Debug|x64 34 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC}.Debug|x64.Deploy.0 = Debug|x64 35 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC}.Debug|x86.ActiveCfg = Debug|x86 36 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC}.Debug|x86.Build.0 = Debug|x86 37 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC}.Debug|x86.Deploy.0 = Debug|x86 38 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC}.Release|Any CPU.ActiveCfg = Release|Any CPU 39 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC}.Release|Any CPU.Build.0 = Release|Any CPU 40 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC}.Release|Mixed Platforms.ActiveCfg = Release|x86 41 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC}.Release|Mixed Platforms.Build.0 = Release|x86 42 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC}.Release|x64.ActiveCfg = Release|x64 43 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC}.Release|x64.Build.0 = Release|x64 44 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC}.Release|x86.ActiveCfg = Release|x86 45 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC}.Release|x86.Build.0 = Release|x86 46 | {93CC9E80-B6F5-4768-998E-889BDF5AB579}.Debug|Any CPU.ActiveCfg = Debug|x86 47 | {93CC9E80-B6F5-4768-998E-889BDF5AB579}.Debug|Mixed Platforms.ActiveCfg = Debug|x86 48 | {93CC9E80-B6F5-4768-998E-889BDF5AB579}.Debug|Mixed Platforms.Build.0 = Debug|x86 49 | {93CC9E80-B6F5-4768-998E-889BDF5AB579}.Debug|x64.ActiveCfg = Debug|x64 50 | {93CC9E80-B6F5-4768-998E-889BDF5AB579}.Debug|x64.Build.0 = Debug|x64 51 | {93CC9E80-B6F5-4768-998E-889BDF5AB579}.Debug|x86.ActiveCfg = Debug|x86 52 | {93CC9E80-B6F5-4768-998E-889BDF5AB579}.Debug|x86.Build.0 = Debug|x86 53 | {93CC9E80-B6F5-4768-998E-889BDF5AB579}.Release|Any CPU.ActiveCfg = Release|x86 54 | {93CC9E80-B6F5-4768-998E-889BDF5AB579}.Release|Mixed Platforms.ActiveCfg = Release|x86 55 | {93CC9E80-B6F5-4768-998E-889BDF5AB579}.Release|Mixed Platforms.Build.0 = Release|x86 56 | {93CC9E80-B6F5-4768-998E-889BDF5AB579}.Release|x64.ActiveCfg = Release|x64 57 | {93CC9E80-B6F5-4768-998E-889BDF5AB579}.Release|x64.Build.0 = Release|x64 58 | {93CC9E80-B6F5-4768-998E-889BDF5AB579}.Release|x86.ActiveCfg = Release|x86 59 | {93CC9E80-B6F5-4768-998E-889BDF5AB579}.Release|x86.Build.0 = Release|x86 60 | {6F05D77A-F180-4779-95FE-75A5223417DF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 61 | {6F05D77A-F180-4779-95FE-75A5223417DF}.Debug|Any CPU.Build.0 = Debug|Any CPU 62 | {6F05D77A-F180-4779-95FE-75A5223417DF}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU 63 | {6F05D77A-F180-4779-95FE-75A5223417DF}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU 64 | {6F05D77A-F180-4779-95FE-75A5223417DF}.Debug|x64.ActiveCfg = Debug|x64 65 | {6F05D77A-F180-4779-95FE-75A5223417DF}.Debug|x64.Build.0 = Debug|x64 66 | {6F05D77A-F180-4779-95FE-75A5223417DF}.Debug|x86.ActiveCfg = Debug|Any CPU 67 | {6F05D77A-F180-4779-95FE-75A5223417DF}.Debug|x86.Build.0 = Debug|Any CPU 68 | {6F05D77A-F180-4779-95FE-75A5223417DF}.Release|Any CPU.ActiveCfg = Release|Any CPU 69 | {6F05D77A-F180-4779-95FE-75A5223417DF}.Release|Any CPU.Build.0 = Release|Any CPU 70 | {6F05D77A-F180-4779-95FE-75A5223417DF}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU 71 | {6F05D77A-F180-4779-95FE-75A5223417DF}.Release|Mixed Platforms.Build.0 = Release|Any CPU 72 | {6F05D77A-F180-4779-95FE-75A5223417DF}.Release|x64.ActiveCfg = Release|Any CPU 73 | {6F05D77A-F180-4779-95FE-75A5223417DF}.Release|x64.Build.0 = Release|Any CPU 74 | {6F05D77A-F180-4779-95FE-75A5223417DF}.Release|x86.ActiveCfg = Release|Any CPU 75 | {6F05D77A-F180-4779-95FE-75A5223417DF}.Release|x86.Build.0 = Release|Any CPU 76 | EndGlobalSection 77 | GlobalSection(SolutionProperties) = preSolution 78 | HideSolutionNode = FALSE 79 | EndGlobalSection 80 | EndGlobal 81 | -------------------------------------------------------------------------------- /src/CollectdWinTest/CollectdWinTest.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Debug 5 | AnyCPU 6 | {6F05D77A-F180-4779-95FE-75A5223417DF} 7 | Library 8 | Properties 9 | CollectdWinTest 10 | CollectdWinTest 11 | v3.5 12 | 512 13 | {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} 14 | 10.0 15 | $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) 16 | $(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages 17 | False 18 | UnitTest 19 | 20 | 21 | true 22 | full 23 | false 24 | bin\Debug\ 25 | DEBUG;TRACE 26 | prompt 27 | 4 28 | x64 29 | 30 | 31 | pdbonly 32 | true 33 | bin\Release\ 34 | TRACE 35 | prompt 36 | 4 37 | 38 | 39 | x64 40 | bin\x64\Debug\ 41 | 42 | 43 | x64 44 | bin\x64\Release\ 45 | 46 | 47 | 48 | 49 | 3.5 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | {d4244e6b-84ec-41a0-96a7-4f489f6098bc} 71 | CollectdWinService 72 | 73 | 74 | 75 | 76 | 77 | 78 | False 79 | 80 | 81 | False 82 | 83 | 84 | False 85 | 86 | 87 | False 88 | 89 | 90 | 91 | 92 | 93 | 94 | 101 | -------------------------------------------------------------------------------- /src/CollectdWinService/config/WriteHttpPluginConfig.cs: -------------------------------------------------------------------------------- 1 | using System.Configuration; 2 | 3 | namespace BloombergFLP.CollectdWin 4 | { 5 | internal class WriteHttpPluginConfig : ConfigurationSection 6 | { 7 | [ConfigurationProperty("Nodes", IsRequired = false)] 8 | [ConfigurationCollection(typeof (WriteHttpNodeConfigCollection), AddItemName = "Node")] 9 | public WriteHttpNodeConfigCollection Nodes 10 | { 11 | get { return (WriteHttpNodeConfigCollection) base["Nodes"]; } 12 | set { base["Nodes"] = value; } 13 | } 14 | 15 | public static WriteHttpPluginConfig GetConfig() 16 | { 17 | return (WriteHttpPluginConfig) ConfigurationManager.GetSection("WriteHttp") ?? new WriteHttpPluginConfig(); 18 | } 19 | 20 | public sealed class WriteHttpNodeConfig : ConfigurationElement 21 | { 22 | [ConfigurationProperty("Name", IsRequired = true)] 23 | public string Name 24 | { 25 | get { return (string) base["Name"]; } 26 | set { base["Name"] = value; } 27 | } 28 | 29 | [ConfigurationProperty("Url", IsRequired = true)] 30 | public string Url 31 | { 32 | get { return (string) base["Url"]; } 33 | set { base["Url"] = value; } 34 | } 35 | 36 | [ConfigurationProperty("Timeout", IsRequired = true)] 37 | public int Timeout 38 | { 39 | get { return (int) base["Timeout"]; } 40 | set { base["Timeout"] = value; } 41 | } 42 | 43 | [ConfigurationProperty("BatchSize", IsRequired = true)] 44 | public int BatchSize 45 | { 46 | get { return (int) base["BatchSize"]; } 47 | set { base["BatchSize"] = value; } 48 | } 49 | 50 | [ConfigurationProperty("MaxIdleTime", IsRequired = false)] 51 | public int MaxIdleTime 52 | { 53 | get { return (int) base["MaxIdleTime"]; } 54 | set { base["MaxIdleTime"] = value; } 55 | } 56 | 57 | [ConfigurationProperty("UserName", IsRequired = false)] 58 | public string UserName 59 | { 60 | get { return (string)base["UserName"]; } 61 | set { base["UserName"] = value; } 62 | } 63 | 64 | [ConfigurationProperty("Password", IsRequired = false)] 65 | public string Password 66 | { 67 | get { return (string)base["Password"]; } 68 | set { base["Password"] = value; } 69 | } 70 | 71 | [ConfigurationProperty("SafeCharsRegex", IsRequired = false)] 72 | public string SafeCharsRegex 73 | { 74 | get { return (string)base["SafeCharsRegex"]; } 75 | set { base["SafeCharsRegex"] = value; } 76 | } 77 | 78 | [ConfigurationProperty("ReplaceWith", IsRequired = false)] 79 | public string ReplaceWith 80 | { 81 | get { return (string)base["ReplaceWith"]; } 82 | set { base["ReplaceWith"] = value; } 83 | } 84 | 85 | [ConfigurationProperty("Proxy", IsRequired = true)] 86 | public ProxyConfig Proxy 87 | { 88 | get { return (ProxyConfig) base["Proxy"]; } 89 | set { base["Proxy"] = value; } 90 | } 91 | 92 | public sealed class ProxyConfig : ConfigurationElement 93 | { 94 | [ConfigurationProperty("Enable", IsRequired = true)] 95 | public bool Enable 96 | { 97 | get { return (bool) base["Enable"]; } 98 | set { base["Enable"] = value; } 99 | } 100 | 101 | [ConfigurationProperty("Url", IsRequired = true)] 102 | public string Url 103 | { 104 | get { return (string) base["Url"]; } 105 | set { base["Url"] = value; } 106 | } 107 | } 108 | } 109 | 110 | public sealed class WriteHttpNodeConfigCollection : ConfigurationElementCollection 111 | { 112 | protected override ConfigurationElement CreateNewElement() 113 | { 114 | return new WriteHttpNodeConfig(); 115 | } 116 | 117 | protected override object GetElementKey(ConfigurationElement element) 118 | { 119 | var nodeConfig = (WriteHttpNodeConfig) element; 120 | return (nodeConfig.Name); 121 | } 122 | } 123 | } 124 | } 125 | 126 | // ---------------------------------------------------------------------------- 127 | // Copyright (C) 2015 Bloomberg Finance L.P. 128 | // 129 | // Licensed under the Apache License, Version 2.0 (the "License"); 130 | // you may not use this file except in compliance with the License. 131 | // You may obtain a copy of the License at 132 | // http://www.apache.org/licenses/LICENSE-2.0 133 | // Unless required by applicable law or agreed to in writing, software 134 | // distributed under the License is distributed on an "AS IS" BASIS, 135 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 136 | // See the License for the specific language governing permissions and 137 | // limitations under the License. 138 | // 139 | // ----------------------------- END-OF-FILE ---------------------------------- 140 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | ## [v0.5.19](https://github.com/bloomberg/collectdwin/tree/v0.5.19) (2017-09-29) 4 | [Full Changelog](https://github.com/bloomberg/collectdwin/compare/v0.5.14...v0.5.19) 5 | 6 | **Implemented enhancements:** 7 | 8 | - Add to Chocolatey [\#1](https://github.com/bloomberg/collectdwin/issues/1) 9 | 10 | **Closed issues:** 11 | 12 | - InvalidOperationException thrown when a performance category does not exist [\#35](https://github.com/bloomberg/collectdwin/issues/35) 13 | - Lot of System.FormatException in eventlog [\#33](https://github.com/bloomberg/collectdwin/issues/33) 14 | 15 | **Merged pull requests:** 16 | 17 | - Add a new configuration option : HostName. Use it in WindowsPerfo… [\#41](https://github.com/bloomberg/collectdwin/pull/41) ([tpoindessous](https://github.com/tpoindessous)) 18 | - Fix broken headings in Markdown files [\#38](https://github.com/bloomberg/collectdwin/pull/38) ([bryant1410](https://github.com/bryant1410)) 19 | - Catch exception due to invalid performance category and log error [\#36](https://github.com/bloomberg/collectdwin/pull/36) ([yogeswaran](https://github.com/yogeswaran)) 20 | - fixed string format to match the number of arguments [\#34](https://github.com/bloomberg/collectdwin/pull/34) ([yogeswaran](https://github.com/yogeswaran)) 21 | 22 | ## [v0.5.14](https://github.com/bloomberg/collectdwin/tree/v0.5.14) (2016-03-22) 23 | [Full Changelog](https://github.com/bloomberg/collectdwin/compare/v0.5.8...v0.5.14) 24 | 25 | **Fixed bugs:** 26 | 27 | - Install Fails on 2008, 2012 64 bit [\#2](https://github.com/bloomberg/collectdwin/issues/2) 28 | 29 | **Closed issues:** 30 | 31 | - Console Output [\#29](https://github.com/bloomberg/collectdwin/issues/29) 32 | 33 | **Merged pull requests:** 34 | 35 | - Add support for metadata and unit tests [\#28](https://github.com/bloomberg/collectdwin/pull/28) ([yogeswaran](https://github.com/yogeswaran)) 36 | - fix a bug with AveragesGenerator and add more debug log info. [\#27](https://github.com/bloomberg/collectdwin/pull/27) ([budaqing](https://github.com/budaqing)) 37 | - fix a bug with creating genrators [\#26](https://github.com/bloomberg/collectdwin/pull/26) ([budaqing](https://github.com/budaqing)) 38 | - add more metrics for Windows [\#25](https://github.com/bloomberg/collectdwin/pull/25) ([budaqing](https://github.com/budaqing)) 39 | - Only log error on sequential failures [\#24](https://github.com/bloomberg/collectdwin/pull/24) ([shanson7](https://github.com/shanson7)) 40 | - Configurations changes to make metrics in synch with Unix Collectd [\#23](https://github.com/bloomberg/collectdwin/pull/23) ([yogeswaran](https://github.com/yogeswaran)) 41 | - make collectdwin be able to both upgrade and downgrade. [\#22](https://github.com/bloomberg/collectdwin/pull/22) ([budaqing](https://github.com/budaqing)) 42 | 43 | ## [v0.5.8](https://github.com/bloomberg/collectdwin/tree/v0.5.8) (2015-11-11) 44 | [Full Changelog](https://github.com/bloomberg/collectdwin/compare/v0.5.3...v0.5.8) 45 | 46 | **Closed issues:** 47 | 48 | - Hostname/Machinename [\#15](https://github.com/bloomberg/collectdwin/issues/15) 49 | - Monitoring Instance="\*" issues [\#12](https://github.com/bloomberg/collectdwin/issues/12) 50 | - Event timestamp in metric JSON can have comma as the decimal separator [\#10](https://github.com/bloomberg/collectdwin/issues/10) 51 | 52 | **Merged pull requests:** 53 | 54 | - Fix delay in publishing last batch [\#21](https://github.com/bloomberg/collectdwin/pull/21) ([yogeswaran](https://github.com/yogeswaran)) 55 | - Adding basic auth to the WriteHttp plugin and minor functionality extentions [\#20](https://github.com/bloomberg/collectdwin/pull/20) ([FerventGeek](https://github.com/FerventGeek)) 56 | - Added tool for versioning [\#19](https://github.com/bloomberg/collectdwin/pull/19) ([yogeswaran](https://github.com/yogeswaran)) 57 | - Try to find FQDN of host [\#18](https://github.com/bloomberg/collectdwin/pull/18) ([pall-valmundsson](https://github.com/pall-valmundsson)) 58 | - Add WindowsPerformanceCounter configuration reloads on intervals [\#17](https://github.com/bloomberg/collectdwin/pull/17) ([pall-valmundsson](https://github.com/pall-valmundsson)) 59 | - Remove manifest properties to support msbuild on build machines [\#16](https://github.com/bloomberg/collectdwin/pull/16) ([yogeswaran](https://github.com/yogeswaran)) 60 | - Handle missing instances [\#14](https://github.com/bloomberg/collectdwin/pull/14) ([pall-valmundsson](https://github.com/pall-valmundsson)) 61 | - Fix JSON timestamp [\#13](https://github.com/bloomberg/collectdwin/pull/13) ([pall-valmundsson](https://github.com/pall-valmundsson)) 62 | - Split app.config into per-plugin config file [\#11](https://github.com/bloomberg/collectdwin/pull/11) ([yogeswaran](https://github.com/yogeswaran)) 63 | 64 | ## [v0.5.3](https://github.com/bloomberg/collectdwin/tree/v0.5.3) (2015-09-09) 65 | [Full Changelog](https://github.com/bloomberg/collectdwin/compare/v0.5.2...v0.5.3) 66 | 67 | **Implemented enhancements:** 68 | 69 | - Collectd http write plugin [\#5](https://github.com/bloomberg/collectdwin/issues/5) 70 | 71 | **Merged pull requests:** 72 | 73 | - Too many log archive files [\#9](https://github.com/bloomberg/collectdwin/pull/9) ([yogeswaran](https://github.com/yogeswaran)) 74 | - Added support for WriteHttp plugin [\#8](https://github.com/bloomberg/collectdwin/pull/8) ([yogeswaran](https://github.com/yogeswaran)) 75 | - Added support for running in console mode for troubleshooting [\#7](https://github.com/bloomberg/collectdwin/pull/7) ([yogeswaran](https://github.com/yogeswaran)) 76 | 77 | ## [v0.5.2](https://github.com/bloomberg/collectdwin/tree/v0.5.2) (2015-06-22) 78 | [Full Changelog](https://github.com/bloomberg/collectdwin/compare/v0.5.1...v0.5.2) 79 | 80 | **Merged pull requests:** 81 | 82 | - Installer should not be starting the service [\#6](https://github.com/bloomberg/collectdwin/pull/6) ([yogeswaran](https://github.com/yogeswaran)) 83 | 84 | ## [v0.5.1](https://github.com/bloomberg/collectdwin/tree/v0.5.1) (2015-06-08) 85 | 86 | 87 | \* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)* -------------------------------------------------------------------------------- /src/CollectdWinService/AmqpPlugin.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Configuration; 3 | using System.Text; 4 | using NLog; 5 | using RabbitMQ.Client; 6 | 7 | namespace BloombergFLP.CollectdWin 8 | { 9 | internal class AmqpPlugin : IMetricsWritePlugin 10 | { 11 | private const int ConnectionRetryDelay = 60; // 1 minute 12 | private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); 13 | private readonly Object _connectionLock; 14 | private IModel _channel; 15 | private bool _connected; 16 | private IConnection _connection; 17 | private string _exchange; 18 | private double _lastConnectTime; 19 | private string _routingKeyPrefix; 20 | private string _url; 21 | 22 | public AmqpPlugin() 23 | { 24 | _connected = false; 25 | _lastConnectTime = 0; 26 | _connectionLock = new object(); 27 | } 28 | 29 | public void Configure() 30 | { 31 | var config = ConfigurationManager.GetSection("Amqp") as AmqpPluginConfig; 32 | if (config == null) 33 | { 34 | throw new Exception("Cannot get configuration section : Amqp"); 35 | } 36 | 37 | string user = config.Publish.User; 38 | string password = config.Publish.Password; 39 | string host = config.Publish.Host; 40 | int port = config.Publish.Port; 41 | string vhost = config.Publish.VirtualHost; 42 | 43 | _url = "amqp://" + user + ":" + password + "@" + host + ":" + port + "/" + vhost; 44 | _exchange = config.Publish.Exchange; 45 | _routingKeyPrefix = config.Publish.RoutingKeyPrefix; 46 | Logger.Info("Amqp plugin configured"); 47 | } 48 | 49 | public void Start() 50 | { 51 | Logger.Trace("Start() begin."); 52 | StartConnection(); 53 | Logger.Info("Amqp plugin started"); 54 | } 55 | 56 | public void Stop() 57 | { 58 | Logger.Trace("CloseConnection() begin"); 59 | CloseConnection(); 60 | Logger.Info("Amqp plugin stopped"); 61 | } 62 | 63 | public void Write(MetricValue metric) 64 | { 65 | if (metric == null) 66 | { 67 | Logger.Debug("write() - Invalid null metric"); 68 | return; 69 | } 70 | if (!_connected) 71 | StartConnection(); 72 | if (_connected && _channel != null) 73 | { 74 | string routingKey = GetAmqpRoutingKey(metric); 75 | string message = metric.GetMetricJsonStr(); 76 | try 77 | { 78 | _channel.BasicPublish(_exchange, routingKey, null, Encoding.UTF8.GetBytes(message)); 79 | } 80 | catch 81 | { 82 | CloseConnection(); 83 | } 84 | } 85 | } 86 | 87 | public void Flush() 88 | { 89 | Logger.Trace("Amqp plugin flushing"); 90 | } 91 | 92 | public void StartConnection() 93 | { 94 | double now = Util.GetNow(); 95 | if (now < (_lastConnectTime + ConnectionRetryDelay)) 96 | { 97 | return; 98 | } 99 | lock (_connectionLock) 100 | { 101 | try 102 | { 103 | var cf = new ConnectionFactory {Uri = _url}; 104 | _connection = cf.CreateConnection(); 105 | _channel = _connection.CreateModel(); 106 | 107 | _connected = true; 108 | _lastConnectTime = Util.GetNow(); 109 | Logger.Debug("Connection started."); 110 | } 111 | catch (Exception exp) 112 | { 113 | Logger.Error("Got exception when connecting to AMQP broker : ", exp); 114 | } 115 | } 116 | } 117 | 118 | public void CloseConnection() 119 | { 120 | lock (_connectionLock) 121 | { 122 | try 123 | { 124 | _channel.Close(); 125 | _connection.Close(); 126 | Logger.Debug("Connection closed."); 127 | } 128 | catch (Exception exp) 129 | { 130 | Logger.Error("Got exception when closing AMQP connection : ", exp); 131 | } 132 | _connected = false; 133 | } 134 | } 135 | 136 | private string GetAmqpRoutingKey(MetricValue metric) 137 | { 138 | string routingKey = _routingKeyPrefix + "." + metric.HostName + "." + metric.PluginName + "." + 139 | metric.PluginInstanceName + "." + metric.TypeName + "." + metric.TypeInstanceName; 140 | 141 | return (routingKey); 142 | } 143 | } 144 | } 145 | 146 | // ---------------------------------------------------------------------------- 147 | // Copyright (C) 2015 Bloomberg Finance L.P. 148 | // 149 | // Licensed under the Apache License, Version 2.0 (the "License"); 150 | // you may not use this file except in compliance with the License. 151 | // You may obtain a copy of the License at 152 | // http://www.apache.org/licenses/LICENSE-2.0 153 | // Unless required by applicable law or agreed to in writing, software 154 | // distributed under the License is distributed on an "AS IS" BASIS, 155 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 156 | // See the License for the specific language governing permissions and 157 | // limitations under the License. 158 | // 159 | // ----------------------------- END-OF-FILE ---------------------------------- -------------------------------------------------------------------------------- /src/CollectdWinService/MetricsPlugin.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Globalization; 4 | using NLog; 5 | using System.Web.Script.Serialization; 6 | 7 | namespace BloombergFLP.CollectdWin 8 | { 9 | internal interface IMetricsPlugin 10 | { 11 | void Configure(); 12 | void Start(); 13 | void Stop(); 14 | } 15 | 16 | internal interface IMetricsReadPlugin : IMetricsPlugin 17 | { 18 | IList Read(); 19 | } 20 | 21 | internal interface IMetricsWritePlugin : IMetricsPlugin 22 | { 23 | void Write(MetricValue metric); 24 | void Flush(); 25 | } 26 | 27 | public class MetricValue 28 | { 29 | private const string MetricJsonFormat = 30 | @"{{""host"":""{0}"", ""plugin"":""{1}"", ""plugin_instance"":""{2}""," + 31 | @" ""type"":""{3}"", ""type_instance"":""{4}"", ""time"":{5}, ""interval"":{6}," + 32 | @" ""dstypes"":[{7}], ""dsnames"":[{8}], ""values"":[{9}]{10}}}"; 33 | private const string MetaDataJsonFormat = @", ""meta"":{0}"; 34 | 35 | private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); 36 | 37 | private readonly IDictionary Meta = new SortedDictionary(); 38 | 39 | public string HostName { get; set; } 40 | public string PluginName { get; set; } 41 | public string PluginInstanceName { get; set; } 42 | public string TypeName { get; set; } 43 | public string TypeInstanceName { get; set; } 44 | 45 | public int Interval { get; set; } 46 | public double Epoch { get; set; } 47 | public double[] Values { get; set; } 48 | 49 | public IDictionary MetaData 50 | { 51 | get 52 | { 53 | return Meta; 54 | } 55 | } 56 | 57 | public void AddMetaData(string tagName, string tagValue) 58 | { 59 | Meta[tagName] = tagValue; 60 | } 61 | 62 | public void AddMetaData(IDictionary meta) 63 | { 64 | if (meta == null) 65 | { 66 | return; 67 | } 68 | foreach(var tag in meta) 69 | { 70 | Meta[tag.Key] = tag.Value; 71 | } 72 | } 73 | 74 | public string Key() 75 | { 76 | return (HostName + "." + PluginName + "." + PluginInstanceName + "." + TypeName + "." + TypeInstanceName); 77 | } 78 | 79 | public MetricValue DeepCopy() 80 | { 81 | var other = (MetricValue) MemberwiseClone(); 82 | other.HostName = String.Copy(HostName); 83 | other.PluginName = String.Copy(PluginName); 84 | other.PluginInstanceName = String.Copy(PluginInstanceName); 85 | other.TypeName = String.Copy(TypeName); 86 | other.TypeInstanceName = String.Copy(TypeInstanceName); 87 | other.Values = (double[]) Values.Clone(); 88 | return (other); 89 | } 90 | 91 | public string EscapeString(string str) 92 | { 93 | if (string.IsNullOrEmpty(str)) 94 | { 95 | return (str); 96 | } 97 | return (str.Replace(@"\", @"\\")); 98 | } 99 | 100 | public string GetMetaDataJsonStr() 101 | { 102 | return(new JavaScriptSerializer().Serialize(MetaData)); 103 | } 104 | 105 | public string GetMetricJsonStr() 106 | { 107 | IList dsList = DataSetCollection.Instance.GetDataSource(TypeName); 108 | var dsNames = new List(); 109 | var dsTypes = new List(); 110 | if (dsList == null) 111 | { 112 | Logger.Debug("Invalid type : {0}, not found in types.db", TypeName); 113 | } 114 | else 115 | { 116 | foreach (DataSource ds in dsList) 117 | { 118 | dsNames.Add(ds.Name); 119 | dsTypes.Add(ds.Type.ToString().ToLower()); 120 | } 121 | } 122 | String epochStr = Epoch.ToString(CultureInfo.InvariantCulture); 123 | string dsTypesStr = string.Join(",", dsTypes.ConvertAll(m => string.Format("\"{0}\"", m)).ToArray()); 124 | string dsNamesStr = string.Join(",", dsNames.ConvertAll(m => string.Format("\"{0}\"", m)).ToArray()); 125 | string valStr = string.Join(",", Array.ConvertAll(Values, val => val.ToString(CultureInfo.InvariantCulture))); 126 | 127 | 128 | var metaDataStr = ""; 129 | if (MetaData.Count > 0) 130 | { 131 | metaDataStr = string.Format(MetaDataJsonFormat, GetMetaDataJsonStr()); 132 | } 133 | var res = ""; 134 | try 135 | { 136 | res = string.Format(MetricJsonFormat, HostName, PluginName, 137 | EscapeString(PluginInstanceName), TypeName, EscapeString(TypeInstanceName), epochStr, 138 | Interval, dsTypesStr, dsNamesStr, valStr, metaDataStr); 139 | } 140 | catch (Exception exp) 141 | { 142 | Logger.Error("Got exception in json conversion : {0}", exp); 143 | } 144 | return (res); 145 | } 146 | } 147 | } 148 | 149 | // ---------------------------------------------------------------------------- 150 | // Copyright (C) 2015 Bloomberg Finance L.P. 151 | // 152 | // Licensed under the Apache License, Version 2.0 (the "License"); 153 | // you may not use this file except in compliance with the License. 154 | // You may obtain a copy of the License at 155 | // http://www.apache.org/licenses/LICENSE-2.0 156 | // Unless required by applicable law or agreed to in writing, software 157 | // distributed under the License is distributed on an "AS IS" BASIS, 158 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 159 | // See the License for the specific language governing permissions and 160 | // limitations under the License. 161 | // 162 | // ----------------------------- END-OF-FILE ---------------------------------- -------------------------------------------------------------------------------- /src/CollectdWinService/config/WindowsPerformanceCounterPluginConfig.cs: -------------------------------------------------------------------------------- 1 | using System.Configuration; 2 | 3 | namespace BloombergFLP.CollectdWin 4 | { 5 | internal class WindowsPerformanceCounterPluginConfig : ConfigurationSection 6 | { 7 | [ConfigurationProperty("RefreshInstancesConfiguration", IsRequired = true)] 8 | public RefreshInstancesConfigurationConfig RefreshInstancesConfiguration 9 | { 10 | get { return (RefreshInstancesConfigurationConfig)base["RefreshInstancesConfiguration"]; } 11 | set { base["RefreshInstancesConfiguration"] = value; } 12 | } 13 | 14 | public sealed class RefreshInstancesConfigurationConfig : ConfigurationElement 15 | { 16 | [ConfigurationProperty("Enable", IsRequired = true)] 17 | public bool Enable 18 | { 19 | get { return (bool)base["Enable"]; } 20 | set { base["Enable"] = value; } 21 | } 22 | 23 | [ConfigurationProperty("Interval", IsRequired = true)] 24 | public int Interval 25 | { 26 | get { return (int)base["Interval"]; } 27 | set { base["Interval"] = value; } 28 | } 29 | } 30 | 31 | [ConfigurationProperty("Counters", IsRequired = false)] 32 | [ConfigurationCollection(typeof (CounterConfigCollection), AddItemName = "Counter")] 33 | public CounterConfigCollection Counters 34 | { 35 | get { return (CounterConfigCollection) base["Counters"]; } 36 | set { base["Counters"] = value; } 37 | } 38 | 39 | public static WindowsPerformanceCounterPluginConfig GetConfig() 40 | { 41 | return 42 | (WindowsPerformanceCounterPluginConfig) 43 | ConfigurationManager.GetSection("WindowsPerformanceCounterPlugin") ?? 44 | new WindowsPerformanceCounterPluginConfig(); 45 | } 46 | 47 | public sealed class CounterConfig : ConfigurationElement 48 | { 49 | [ConfigurationProperty("Transformer", IsRequired = false, DefaultValue = "")] 50 | public string Transformer 51 | { 52 | get { return (string)base["Transformer"]; } 53 | set { base["Transformer"] = value; } 54 | } 55 | 56 | [ConfigurationProperty("TransformerParameters", IsRequired = false)] 57 | public string TransformerParameters 58 | { 59 | get { return (string)base["TransformerParameters"]; } 60 | set { base["TransformerParameters"] = value; } 61 | } 62 | 63 | [ConfigurationProperty("Category", IsRequired = true)] 64 | public string Category 65 | { 66 | get { return (string) base["Category"]; } 67 | set { base["Category"] = value; } 68 | } 69 | 70 | [ConfigurationProperty("Name", IsRequired = true)] 71 | public string Name 72 | { 73 | get { return (string) base["Name"]; } 74 | set { base["Name"] = value; } 75 | } 76 | 77 | 78 | [ConfigurationProperty("Instance", IsRequired = false)] 79 | public string Instance 80 | { 81 | get { return (string) base["Instance"]; } 82 | set { base["Instance"] = value; } 83 | } 84 | 85 | [ConfigurationProperty("CollectdPlugin", IsRequired = true)] 86 | public string CollectdPlugin 87 | { 88 | get { return (string) base["CollectdPlugin"]; } 89 | set { base["CollectdPlugin"] = value; } 90 | } 91 | 92 | [ConfigurationProperty("CollectdPluginInstance", IsRequired = false)] 93 | public string CollectdPluginInstance 94 | { 95 | get { return (string) base["CollectdPluginInstance"]; } 96 | set { base["CollectdPluginInstance"] = value; } 97 | } 98 | 99 | [ConfigurationProperty("CollectdType", IsRequired = true)] 100 | public string CollectdType 101 | { 102 | get { return (string) base["CollectdType"]; } 103 | set { base["CollectdType"] = value; } 104 | } 105 | 106 | [ConfigurationProperty("CollectdTypeInstance", IsRequired = true)] 107 | public string CollectdTypeInstance 108 | { 109 | get { return (string) base["CollectdTypeInstance"]; } 110 | set { base["CollectdTypeInstance"] = value; } 111 | } 112 | } 113 | 114 | public sealed class CounterConfigCollection : ConfigurationElementCollection 115 | { 116 | protected override ConfigurationElement CreateNewElement() 117 | { 118 | return new CounterConfig(); 119 | } 120 | 121 | protected override object GetElementKey(ConfigurationElement element) 122 | { 123 | var counterConfig = (CounterConfig) element; 124 | return (counterConfig.CollectdPlugin + "_" + counterConfig.CollectdPluginInstance + "_" + counterConfig.CollectdType + "_" + counterConfig.CollectdTypeInstance); 125 | } 126 | } 127 | } 128 | } 129 | 130 | // ---------------------------------------------------------------------------- 131 | // Copyright (C) 2015 Bloomberg Finance L.P. 132 | // 133 | // Licensed under the Apache License, Version 2.0 (the "License"); 134 | // you may not use this file except in compliance with the License. 135 | // You may obtain a copy of the License at 136 | // http://www.apache.org/licenses/LICENSE-2.0 137 | // Unless required by applicable law or agreed to in writing, software 138 | // distributed under the License is distributed on an "AS IS" BASIS, 139 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 140 | // See the License for the specific language governing permissions and 141 | // limitations under the License. 142 | // 143 | // ----------------------------- END-OF-FILE ---------------------------------- -------------------------------------------------------------------------------- /src/CollectdWinService/CollectdWinService.resx: -------------------------------------------------------------------------------- 1 | 2 | 3 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | text/microsoft-resx 110 | 111 | 112 | 2.0 113 | 114 | 115 | System.Resources.ResXResourceReader, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 116 | 117 | 118 | System.Resources.ResXResourceWriter, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 119 | 120 | 121 | False 122 | 123 | -------------------------------------------------------------------------------- /src/CollectdWinService/config/StatsdPluginConfig.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Configuration; 3 | 4 | namespace BloombergFLP.CollectdWin 5 | { 6 | internal class StatsdPluginConfig : ConfigurationSection 7 | { 8 | [ConfigurationProperty("Server", IsRequired = true)] 9 | public ServerConfig Server 10 | { 11 | get { return (ServerConfig) base["Server"]; } 12 | set { base["Server"] = value; } 13 | } 14 | 15 | [ConfigurationProperty("DeleteCache", IsRequired = true)] 16 | public DeleteCacheConfig DeleteCache 17 | { 18 | get { return (DeleteCacheConfig) base["DeleteCache"]; } 19 | set { base["DeleteCache"] = value; } 20 | } 21 | 22 | [ConfigurationProperty("Timer", IsRequired = true)] 23 | public TimerConfig Timer 24 | { 25 | get { return (TimerConfig) base["Timer"]; } 26 | set { base["Timer"] = value; } 27 | } 28 | 29 | public static StatsdPluginConfig GetConfig() 30 | { 31 | return (StatsdPluginConfig) ConfigurationManager.GetSection("Statsd") ?? new StatsdPluginConfig(); 32 | } 33 | 34 | public sealed class DeleteCacheConfig : ConfigurationElement 35 | { 36 | [ConfigurationProperty("Counters", IsRequired = true)] 37 | public bool Counters 38 | { 39 | get { return (bool) base["Counters"]; } 40 | set { base["Counters"] = value; } 41 | } 42 | 43 | [ConfigurationProperty("Timers", IsRequired = true)] 44 | public bool Timers 45 | { 46 | get { return (bool) base["Timers"]; } 47 | set { base["Timers"] = value; } 48 | } 49 | 50 | [ConfigurationProperty("Gauges", IsRequired = true)] 51 | public bool Gauges 52 | { 53 | get { return (bool) base["Gauges"]; } 54 | set { base["Gauges"] = value; } 55 | } 56 | 57 | [ConfigurationProperty("Sets", IsRequired = true)] 58 | public bool Sets 59 | { 60 | get { return (bool) base["Sets"]; } 61 | set { base["Sets"] = value; } 62 | } 63 | } 64 | 65 | public class PercentileCollection : ConfigurationElementCollection 66 | { 67 | protected override ConfigurationElement CreateNewElement() 68 | { 69 | return new PercentileConfig(); 70 | } 71 | 72 | protected override object GetElementKey(ConfigurationElement element) 73 | { 74 | return (((PercentileConfig) element).UniqueId); 75 | } 76 | } 77 | 78 | public sealed class PercentileConfig : ConfigurationElement 79 | { 80 | public PercentileConfig() 81 | { 82 | UniqueId = Guid.NewGuid(); 83 | } 84 | 85 | internal Guid UniqueId { get; set; } 86 | 87 | [ConfigurationProperty("Value", IsRequired = true)] 88 | public float Value 89 | { 90 | get { return (float) base["Value"]; } 91 | set { base["Value"] = value; } 92 | } 93 | } 94 | 95 | public sealed class ServerConfig : ConfigurationElement 96 | { 97 | [ConfigurationProperty("Host", IsRequired = true)] 98 | public string Host 99 | { 100 | get { return (string) base["Host"]; } 101 | set { base["Host"] = value; } 102 | } 103 | 104 | [ConfigurationProperty("Port", IsRequired = true)] 105 | public int Port 106 | { 107 | get { return (int) base["Port"]; } 108 | set { base["Port"] = value; } 109 | } 110 | } 111 | 112 | public sealed class TimerConfig : ConfigurationElement 113 | { 114 | [ConfigurationProperty("Lower", IsRequired = true)] 115 | public bool Lower 116 | { 117 | get { return (bool) base["Lower"]; } 118 | set { base["Lower"] = value; } 119 | } 120 | 121 | [ConfigurationProperty("Upper", IsRequired = true)] 122 | public bool Upper 123 | { 124 | get { return (bool) base["Upper"]; } 125 | set { base["Upper"] = value; } 126 | } 127 | 128 | [ConfigurationProperty("Sum", IsRequired = true)] 129 | public bool Sum 130 | { 131 | get { return (bool) base["Sum"]; } 132 | set { base["Sum"] = value; } 133 | } 134 | 135 | [ConfigurationProperty("Count", IsRequired = true)] 136 | public bool Count 137 | { 138 | get { return (bool) base["Count"]; } 139 | set { base["Count"] = value; } 140 | } 141 | 142 | [ConfigurationProperty("Percentiles", IsRequired = false)] 143 | [ConfigurationCollection(typeof (PercentileCollection), AddItemName = "Percentile")] 144 | public PercentileCollection Percentiles 145 | { 146 | get { return (PercentileCollection) base["Percentiles"]; } 147 | set { base["Percentiles"] = value; } 148 | } 149 | } 150 | } 151 | } 152 | 153 | // ---------------------------------------------------------------------------- 154 | // Copyright (C) 2015 Bloomberg Finance L.P. 155 | // 156 | // Licensed under the Apache License, Version 2.0 (the "License"); 157 | // you may not use this file except in compliance with the License. 158 | // You may obtain a copy of the License at 159 | // http://www.apache.org/licenses/LICENSE-2.0 160 | // Unless required by applicable law or agreed to in writing, software 161 | // distributed under the License is distributed on an "AS IS" BASIS, 162 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 163 | // See the License for the specific language governing permissions and 164 | // limitations under the License. 165 | // 166 | // ----------------------------- END-OF-FILE ---------------------------------- -------------------------------------------------------------------------------- /src/installer/Product.wxs: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 24 | 28 | 31 | 34 | 36 | 37 | 38 | 40 | 42 | 48 | 54 | 60 | 66 | 72 | 82 | 87 | 88 | 89 | 90 | 96 | 102 | 108 | 114 | 120 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | -------------------------------------------------------------------------------- /src/CollectdWinService/config/CollectdWinConfig.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Configuration; 4 | 5 | namespace BloombergFLP.CollectdWin 6 | { 7 | 8 | internal class CollectdWinConfigHelper 9 | { 10 | public static IDictionary getMetaData() 11 | { 12 | IDictionary metaData = new Dictionary(); 13 | 14 | var coreConfig = ConfigurationManager.GetSection("CollectdWinConfig") as CollectdWinConfig; 15 | if (coreConfig == null) 16 | { 17 | throw new Exception("Cannot get configuration section : CollectdWinConfig"); 18 | } 19 | foreach (CollectdWinConfig.TagConfig tagConfig in coreConfig.MetaData) 20 | { 21 | metaData[tagConfig.Name] = tagConfig.Value; 22 | } 23 | return (metaData); 24 | } 25 | } 26 | 27 | internal class CollectdWinConfig : ConfigurationSection 28 | { 29 | [ConfigurationProperty("GeneralSettings", IsRequired = true)] 30 | public GeneralSettingsConfig GeneralSettings 31 | { 32 | get { return (GeneralSettingsConfig) base["GeneralSettings"]; } 33 | set { base["GeneralSettings"] = value; } 34 | } 35 | 36 | [ConfigurationProperty("Plugins", IsRequired = true)] 37 | [ConfigurationCollection(typeof (PluginCollection), AddItemName = "Plugin")] 38 | public PluginCollection Plugins 39 | { 40 | get { return (PluginCollection) base["Plugins"]; } 41 | set { base["Plugins"] = value; } 42 | } 43 | 44 | [ConfigurationProperty("MetaData", IsRequired = false)] 45 | [ConfigurationCollection(typeof(TagCollection), AddItemName = "Tag")] 46 | public TagCollection MetaData 47 | { 48 | get { return (TagCollection)base["MetaData"]; } 49 | set { base["MetaData"] = value; } 50 | } 51 | 52 | public static CollectdWinConfig GetConfig() 53 | { 54 | return (CollectdWinConfig) ConfigurationManager.GetSection("CollectdWinConfig") ?? new CollectdWinConfig(); 55 | } 56 | 57 | public sealed class GeneralSettingsConfig : ConfigurationElement 58 | { 59 | [ConfigurationProperty("Interval", IsRequired = true)] 60 | public int Interval 61 | { 62 | get { return (int) base["Interval"]; } 63 | set { base["Interval"] = value; } 64 | } 65 | 66 | [ConfigurationProperty("Timeout", IsRequired = true)] 67 | public int Timeout 68 | { 69 | get { return (int) base["Timeout"]; } 70 | set { base["Timeout"] = value; } 71 | } 72 | 73 | [ConfigurationProperty("StoreRates", IsRequired = true)] 74 | public bool StoreRates 75 | { 76 | get { return (bool) base["StoreRates"]; } 77 | set { base["StoreRates"] = value; } 78 | } 79 | 80 | [ConfigurationProperty("HostName", IsRequired = false)] 81 | public string HostName 82 | { 83 | get { return (string) base["HostName"]; } 84 | set { base["HostName"] = value; } 85 | } 86 | } 87 | 88 | public sealed class PluginCollection : ConfigurationElementCollection 89 | { 90 | protected override ConfigurationElement CreateNewElement() 91 | { 92 | return new PluginConfig(); 93 | } 94 | 95 | protected override object GetElementKey(ConfigurationElement element) 96 | { 97 | return (((PluginConfig) element).UniqueId); 98 | } 99 | } 100 | 101 | public sealed class PluginConfig : ConfigurationElement 102 | { 103 | public PluginConfig() 104 | { 105 | UniqueId = Guid.NewGuid(); 106 | } 107 | 108 | internal Guid UniqueId { get; set; } 109 | 110 | [ConfigurationProperty("Name", IsRequired = true)] 111 | public string Name 112 | { 113 | get { return (string) base["Name"]; } 114 | set { base["Name"] = value; } 115 | } 116 | 117 | [ConfigurationProperty("Class", IsRequired = true)] 118 | public string Class 119 | { 120 | get { return (string) base["Class"]; } 121 | set { base["Class"] = value; } 122 | } 123 | 124 | [ConfigurationProperty("Enable", IsRequired = true)] 125 | public bool Enable 126 | { 127 | get { return (bool) base["Enable"]; } 128 | set { base["Enable"] = value; } 129 | } 130 | } 131 | 132 | public sealed class TagCollection : ConfigurationElementCollection 133 | { 134 | protected override ConfigurationElement CreateNewElement() 135 | { 136 | return new TagConfig(); 137 | } 138 | 139 | protected override object GetElementKey(ConfigurationElement element) 140 | { 141 | return (((TagConfig)element).UniqueId); 142 | } 143 | } 144 | public sealed class TagConfig : ConfigurationElement 145 | { 146 | public TagConfig() 147 | { 148 | UniqueId = Guid.NewGuid(); 149 | } 150 | 151 | internal Guid UniqueId { get; set; } 152 | 153 | [ConfigurationProperty("Name", IsRequired = true)] 154 | public string Name 155 | { 156 | get { return (string)base["Name"]; } 157 | set { base["Name"] = value; } 158 | } 159 | 160 | [ConfigurationProperty("Value", IsRequired = true)] 161 | public string Value 162 | { 163 | get { return (string)base["Value"]; } 164 | set { base["Value"] = value; } 165 | } 166 | } 167 | } 168 | } 169 | 170 | // ---------------------------------------------------------------------------- 171 | // Copyright (C) 2015 Bloomberg Finance L.P. 172 | // 173 | // Licensed under the Apache License, Version 2.0 (the "License"); 174 | // you may not use this file except in compliance with the License. 175 | // You may obtain a copy of the License at 176 | // http://www.apache.org/licenses/LICENSE-2.0 177 | // Unless required by applicable law or agreed to in writing, software 178 | // distributed under the License is distributed on an "AS IS" BASIS, 179 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 180 | // See the License for the specific language governing permissions and 181 | // limitations under the License. 182 | // 183 | // ----------------------------- END-OF-FILE ---------------------------------- -------------------------------------------------------------------------------- /src/CollectdWinService/TypesDB.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Text; 5 | using System.Text.RegularExpressions; 6 | using NLog; 7 | 8 | namespace BloombergFLP.CollectdWin 9 | { 10 | public enum DsType 11 | { 12 | Absolute, 13 | Counter, 14 | Derive, 15 | Gauge 16 | }; 17 | 18 | public enum Status 19 | { 20 | Success, 21 | Failure 22 | }; 23 | 24 | public class DataSource 25 | { 26 | public DataSource(string name, DsType type, double min, double max) 27 | { 28 | Name = name; 29 | Type = type; 30 | Min = min; 31 | Max = max; 32 | } 33 | 34 | public string Name { get; set; } 35 | public DsType Type { get; set; } 36 | public double Min { get; set; } 37 | public double Max { get; set; } 38 | } 39 | 40 | internal class DataSetCollection 41 | { 42 | private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); 43 | private static volatile DataSetCollection _instance; 44 | private static readonly object SyncRoot = new Object(); 45 | 46 | private readonly Dictionary> _dataSetMap; 47 | // 48 | 49 | private DataSetCollection() 50 | { 51 | _dataSetMap = new Dictionary>(); 52 | } 53 | 54 | public static DataSetCollection Instance 55 | { 56 | get 57 | { 58 | if (_instance == null) 59 | { 60 | lock (SyncRoot) 61 | { 62 | if (_instance == null) 63 | { 64 | _instance = new DataSetCollection(); 65 | _instance.Load(); 66 | } 67 | } 68 | } 69 | return _instance; 70 | } 71 | } 72 | 73 | private static Status GetDouble(string dstr, out double val) 74 | { 75 | if (dstr == "u" || dstr == "U") 76 | { 77 | val = Double.NaN; 78 | return (Status.Success); 79 | } 80 | try 81 | { 82 | val = Double.Parse(dstr); 83 | } 84 | catch (Exception) 85 | { 86 | val = Double.NaN; 87 | return (Status.Failure); 88 | } 89 | return (Status.Success); 90 | } 91 | 92 | public void Print() 93 | { 94 | var sb = new StringBuilder(); 95 | foreach (var entry in _dataSetMap) 96 | { 97 | sb.Append(string.Format("\n[{0}] ==>", entry.Key)); 98 | foreach (DataSource ds in entry.Value) 99 | { 100 | sb.Append(string.Format(" [{0}:{1}:{2}:{3}]", ds.Name, ds.Type, ds.Min, ds.Max)); 101 | } 102 | } 103 | Console.WriteLine(sb.ToString()); 104 | } 105 | 106 | public void Load() 107 | { 108 | const string dataSetPattern = @"[\s\t]*(\w+)[\s\t]*(.*)$"; 109 | const string dataSourcePattern = @"(\w+):(ABSOLUTE|COUNTER|DERIVE|GAUGE):([+-]?\w+):([+-]?\w+)[,]?\s*"; 110 | 111 | var dataSetRegex = new Regex(dataSetPattern, RegexOptions.IgnoreCase); 112 | var dataSourceRegex = new Regex(dataSourcePattern, RegexOptions.IgnoreCase); 113 | 114 | string fileName = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "types.db"); 115 | string[] lines = File.ReadAllLines(fileName); 116 | 117 | foreach (string line in lines) 118 | { 119 | // skip comments and blank lines 120 | if (line.StartsWith("#") || line.Trim() == string.Empty) 121 | { 122 | continue; 123 | } 124 | Match match = dataSetRegex.Match(line); 125 | if (match.Groups.Count < 3) 126 | { 127 | Logger.Error("types.db: invalid data set [{0}]", line); 128 | continue; 129 | } 130 | string dataSetName = match.Groups[1].Value; 131 | MatchCollection matches = dataSourceRegex.Matches(line); 132 | if (matches.Count < 1) 133 | { 134 | Logger.Error("types.db: invalid data source [{0}]", line); 135 | continue; 136 | } 137 | var dataSourceList = new List(); 138 | foreach (Match m in matches) 139 | { 140 | if (m.Groups.Count != 5) 141 | { 142 | Logger.Error("types.db: cannot parse data source [{0}]", line); 143 | dataSourceList.Clear(); 144 | break; 145 | } 146 | 147 | string dsName = m.Groups[1].Value; 148 | var dstype = (DsType) Enum.Parse(typeof (DsType), m.Groups[2].Value, true); 149 | 150 | double min, max; 151 | 152 | if (GetDouble(m.Groups[3].Value, out min) != Status.Success) 153 | { 154 | Logger.Error("types.db: invalid Min value [{0}]", line); 155 | dataSourceList.Clear(); 156 | break; 157 | } 158 | 159 | if (GetDouble(m.Groups[4].Value, out max) != Status.Success) 160 | { 161 | Logger.Error("types.db: invalid Max value [{0}]", line); 162 | dataSourceList.Clear(); 163 | break; 164 | } 165 | 166 | var ds = new DataSource(dsName, dstype, min, max); 167 | dataSourceList.Add(ds); 168 | } 169 | if (dataSourceList.Count > 0) 170 | { 171 | _dataSetMap[dataSetName] = dataSourceList; 172 | } 173 | } 174 | } 175 | 176 | public IList GetDataSource(string dataSetName) 177 | { 178 | IList dataSourceList; 179 | return (_dataSetMap.TryGetValue(dataSetName, out dataSourceList) ? (dataSourceList) : (null)); 180 | } 181 | } 182 | } 183 | 184 | // ---------------------------------------------------------------------------- 185 | // Copyright (C) 2015 Bloomberg Finance L.P. 186 | // 187 | // Licensed under the Apache License, Version 2.0 (the "License"); 188 | // you may not use this file except in compliance with the License. 189 | // You may obtain a copy of the License at 190 | // http://www.apache.org/licenses/LICENSE-2.0 191 | // Unless required by applicable law or agreed to in writing, software 192 | // distributed under the License is distributed on an "AS IS" BASIS, 193 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 194 | // See the License for the specific language governing permissions and 195 | // limitations under the License. 196 | // 197 | // ----------------------------- END-OF-FILE ---------------------------------- 198 | -------------------------------------------------------------------------------- /src/CollectdWinService/Aggregator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using NLog; 5 | 6 | namespace BloombergFLP.CollectdWin 7 | { 8 | internal class CacheEntry 9 | { 10 | public MetricValue MetricRate; 11 | public double[] RawValues; 12 | 13 | public CacheEntry(MetricValue metricValue) 14 | { 15 | MetricRate = metricValue; 16 | RawValues = (double[]) metricValue.Values.Clone(); 17 | } 18 | } 19 | 20 | internal class Aggregator 21 | { 22 | private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); 23 | private readonly Dictionary _cache; 24 | private readonly Object _cacheLock; 25 | private readonly bool _storeRates; 26 | private readonly int _timeoutSeconds; 27 | 28 | public Aggregator(int timeoutSeconds, bool storeRates) 29 | { 30 | _cache = new Dictionary(); 31 | _cacheLock = new Object(); 32 | _timeoutSeconds = timeoutSeconds; 33 | _storeRates = storeRates; 34 | } 35 | 36 | public void Aggregate(ref MetricValue metricValue) 37 | { 38 | // If rates are not stored then there is nothing to aggregate 39 | if (!_storeRates) 40 | { 41 | return; 42 | } 43 | IList dsl = DataSetCollection.Instance.GetDataSource(metricValue.TypeName); 44 | if (dsl == null || metricValue.Values.Length != dsl.Count) 45 | { 46 | return; 47 | } 48 | 49 | double now = Util.GetNow(); 50 | 51 | lock (_cacheLock) 52 | { 53 | CacheEntry cEntry; 54 | string key = metricValue.Key(); 55 | 56 | if (!(_cache.TryGetValue(key, out cEntry))) 57 | { 58 | cEntry = new CacheEntry(metricValue.DeepCopy()); 59 | for (int i = 0; i < metricValue.Values.Length; i++) 60 | { 61 | if (dsl[i].Type == DsType.Derive || 62 | dsl[i].Type == DsType.Absolute || 63 | dsl[i].Type == DsType.Counter) 64 | { 65 | metricValue.Values[i] = double.NaN; 66 | cEntry.MetricRate.Values[i] = double.NaN; 67 | } 68 | } 69 | cEntry.MetricRate.Epoch = now; 70 | _cache[key] = cEntry; 71 | return; 72 | } 73 | for (int i = 0; i < metricValue.Values.Length; i++) 74 | { 75 | double rawValNew = metricValue.Values[i]; 76 | double rawValOld = cEntry.RawValues[i]; 77 | double rawValDiff = rawValNew - rawValOld; 78 | double timeDiff = cEntry.MetricRate.Epoch - now; 79 | 80 | double rateNew = rawValDiff/timeDiff; 81 | 82 | switch (dsl[i].Type) 83 | { 84 | case DsType.Gauge: 85 | // no rates calculations are done, values are stored as-is for gauge 86 | cEntry.RawValues[i] = rawValNew; 87 | cEntry.MetricRate.Values[i] = rawValNew; 88 | break; 89 | 90 | case DsType.Absolute: 91 | // similar to gauge, except value will be divided by time diff 92 | cEntry.MetricRate.Values[i] = metricValue.Values[i]/timeDiff; 93 | cEntry.RawValues[i] = rawValNew; 94 | metricValue.Values[i] = cEntry.MetricRate.Values[i]; 95 | break; 96 | 97 | case DsType.Derive: 98 | cEntry.RawValues[i] = rawValNew; 99 | cEntry.MetricRate.Values[i] = rateNew; 100 | metricValue.Values[i] = rateNew; 101 | 102 | break; 103 | 104 | case DsType.Counter: 105 | // Counters are very simlar to derive except when counter wraps around 106 | if (rawValNew < rawValOld) 107 | { 108 | // counter has wrapped around 109 | cEntry.MetricRate.Values[i] = metricValue.Values[i]/timeDiff; 110 | cEntry.RawValues[i] = rawValNew; 111 | metricValue.Values[i] = cEntry.MetricRate.Values[i]; 112 | } 113 | else 114 | { 115 | cEntry.MetricRate.Values[i] = rateNew; 116 | cEntry.RawValues[i] = rawValNew; 117 | metricValue.Values[i] = rateNew; 118 | } 119 | break; 120 | } 121 | 122 | // range checks 123 | if (metricValue.Values[i] < dsl[i].Min) 124 | { 125 | metricValue.Values[i] = dsl[i].Min; 126 | cEntry.RawValues[i] = metricValue.Values[i]; 127 | } 128 | if (metricValue.Values[i] > dsl[i].Max) 129 | { 130 | metricValue.Values[i] = dsl[i].Max; 131 | cEntry.RawValues[i] = metricValue.Values[i]; 132 | } 133 | 134 | cEntry.MetricRate.Epoch = now; 135 | _cache[key] = cEntry; 136 | } 137 | } 138 | } 139 | 140 | public void RemoveExpiredEntries() 141 | { 142 | // If rates are not stored then there is nothing to remove 143 | if (!_storeRates) 144 | { 145 | return; 146 | } 147 | double now = Util.GetNow(); 148 | double expirationTime = now - _timeoutSeconds; 149 | var removeList = new List(); 150 | 151 | lock (_cacheLock) 152 | { 153 | removeList.AddRange(from pair in _cache 154 | let cEntry = pair.Value 155 | where cEntry.MetricRate.Epoch < expirationTime 156 | select pair.Key); 157 | if (removeList.Count > 0) 158 | Logger.Debug("Removing expired entries: {0}", removeList.Count); 159 | foreach (string key in removeList) 160 | { 161 | _cache.Remove(key); 162 | } 163 | } 164 | } 165 | } 166 | } 167 | 168 | // ---------------------------------------------------------------------------- 169 | // Copyright (C) 2015 Bloomberg Finance L.P. 170 | // 171 | // Licensed under the Apache License, Version 2.0 (the "License"); 172 | // you may not use this file except in compliance with the License. 173 | // You may obtain a copy of the License at 174 | // http://www.apache.org/licenses/LICENSE-2.0 175 | // Unless required by applicable law or agreed to in writing, software 176 | // distributed under the License is distributed on an "AS IS" BASIS, 177 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 178 | // See the License for the specific language governing permissions and 179 | // limitations under the License. 180 | // 181 | // ----------------------------- END-OF-FILE ---------------------------------- -------------------------------------------------------------------------------- /src/CollectdWinService/StatsdAggregator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using NLog; 4 | 5 | namespace BloombergFLP.CollectdWin 6 | { 7 | internal class StatsdAggregator 8 | { 9 | private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); 10 | 11 | private readonly bool _delCounter; 12 | private readonly bool _delGauge; 13 | private readonly bool _delSet; 14 | private readonly bool _delTimer; 15 | private readonly string _hostName; 16 | private readonly Object _lock; 17 | private readonly Dictionary _metrics; 18 | private readonly float[] _percentiles; 19 | private readonly bool _timerCount; 20 | private readonly bool _timerLower; 21 | private readonly bool _timerSum; 22 | private readonly bool _timerUpper; 23 | 24 | public StatsdAggregator(bool delCounter, bool delTimer, bool delGauge, bool delSet, bool timerLower, 25 | bool timerUpper, bool timerSum, bool timerCount, float[] percentiles) 26 | { 27 | _lock = new Object(); 28 | _metrics = new Dictionary(); 29 | _hostName = Util.GetHostName(); 30 | 31 | _delCounter = delCounter; 32 | _delTimer = delTimer; 33 | _delGauge = delGauge; 34 | _delSet = delSet; 35 | 36 | _timerLower = timerLower; 37 | _timerUpper = timerUpper; 38 | _timerSum = timerSum; 39 | _timerCount = timerCount; 40 | 41 | _percentiles = percentiles; 42 | 43 | if (_percentiles != null && _percentiles.Length > 0) 44 | StatsdMetric.Latency.HistogramEnabled = true; 45 | else 46 | StatsdMetric.Latency.HistogramEnabled = false; 47 | 48 | Logger.Info( 49 | "Statsd config - delCounter:{0}, delTimer:{1}, delGauge:{2}, delSet:{3}, isHistogramEnabled:{4}", 50 | _delCounter, _delTimer, _delGauge, _delSet, StatsdMetric.Latency.HistogramEnabled); 51 | } 52 | 53 | public void AddMetric(StatsdMetric metric) 54 | { 55 | string key = metric.Type + "_" + metric.Name; 56 | lock (_lock) 57 | { 58 | StatsdMetric oldMetric; 59 | if (_metrics.TryGetValue(key, out oldMetric)) 60 | { 61 | //merge 62 | double val = metric.Value; 63 | oldMetric.AddValue(val); 64 | } 65 | else 66 | { 67 | _metrics[key] = metric; 68 | } 69 | } 70 | } 71 | 72 | public IList Read() 73 | { 74 | lock (_lock) 75 | { 76 | var res = new List(); 77 | var removeList = new List(); 78 | foreach (var pair in _metrics) 79 | { 80 | StatsdMetric metric = pair.Value; 81 | if (metric.NumUpdates <= 0 && 82 | ((_delCounter && metric.Type == StatsdMetric.StatsdType.StatsdCounter) || 83 | (_delTimer && metric.Type == StatsdMetric.StatsdType.StatsdTimer) || 84 | (_delGauge && metric.Type == StatsdMetric.StatsdType.StatsdGauge) || 85 | (_delSet && metric.Type == StatsdMetric.StatsdType.StatsdSet))) 86 | { 87 | removeList.Add(pair.Key); 88 | continue; 89 | } 90 | var metricVal = new MetricValue 91 | { 92 | HostName = _hostName, 93 | PluginName = "statsd", 94 | PluginInstanceName = "", 95 | TypeInstanceName = metric.Name, 96 | Values = new[] {metric.GetMetric()} 97 | }; 98 | switch (metric.Type) 99 | { 100 | case StatsdMetric.StatsdType.StatsdGauge: 101 | metricVal.TypeName = "gauge"; 102 | break; 103 | case StatsdMetric.StatsdType.StatsdTimer: 104 | metricVal.TypeName = "latency"; 105 | metricVal.TypeInstanceName += "-average"; 106 | break; 107 | case StatsdMetric.StatsdType.StatsdSet: 108 | metricVal.TypeName = "objects"; 109 | break; 110 | default: 111 | metricVal.TypeName = "derive"; 112 | break; 113 | } 114 | TimeSpan t = DateTime.UtcNow - new DateTime(1970, 1, 1); 115 | double epoch = t.TotalMilliseconds/1000; 116 | metricVal.Epoch = Math.Round(epoch, 3); 117 | 118 | res.Add(metricVal); 119 | 120 | if (metric.Type == StatsdMetric.StatsdType.StatsdTimer) 121 | { 122 | if (_timerLower) 123 | { 124 | MetricValue lowerValue = metricVal.DeepCopy(); 125 | lowerValue.TypeInstanceName = metric.Name + "-lower"; 126 | lowerValue.Values[0] = metric.Lat.Min; 127 | res.Add(lowerValue); 128 | } 129 | if (_timerUpper) 130 | { 131 | MetricValue upperValue = metricVal.DeepCopy(); 132 | upperValue.TypeInstanceName = metric.Name + "-upper"; 133 | upperValue.Values[0] = metric.Lat.Max; 134 | res.Add(upperValue); 135 | } 136 | 137 | if (_timerSum) 138 | { 139 | MetricValue upperSum = metricVal.DeepCopy(); 140 | upperSum.TypeInstanceName = metric.Name + "-Sum"; 141 | upperSum.Values[0] = metric.Lat.Sum; 142 | res.Add(upperSum); 143 | } 144 | if (_timerCount) 145 | { 146 | MetricValue upperCount = metricVal.DeepCopy(); 147 | upperCount.TypeInstanceName = metric.Name + "-count"; 148 | upperCount.Values[0] = metric.Lat.Num; 149 | res.Add(upperCount); 150 | } 151 | Histogram histogram = metric.Lat.Histogram; 152 | if (_percentiles != null && _percentiles.Length > 0 && histogram != null) 153 | { 154 | foreach (float percentile in _percentiles) 155 | { 156 | double val = histogram.GetPercentile(percentile); 157 | 158 | MetricValue mv = metricVal.DeepCopy(); 159 | mv.TypeInstanceName = metric.Name + "-percentile-" + percentile; 160 | mv.Values[0] = val; 161 | res.Add(mv); 162 | } 163 | } 164 | } 165 | metric.Reset(); 166 | } 167 | Logger.Debug("Removing entries that were not updated:{0}", removeList.Count); 168 | foreach (string key in removeList) 169 | { 170 | _metrics.Remove(key); 171 | } 172 | return (res); 173 | } 174 | } 175 | } 176 | } 177 | 178 | // ---------------------------------------------------------------------------- 179 | // Copyright (C) 2015 Bloomberg Finance L.P. 180 | // 181 | // Licensed under the Apache License, Version 2.0 (the "License"); 182 | // you may not use this file except in compliance with the License. 183 | // You may obtain a copy of the License at 184 | // http://www.apache.org/licenses/LICENSE-2.0 185 | // Unless required by applicable law or agreed to in writing, software 186 | // distributed under the License is distributed on an "AS IS" BASIS, 187 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 188 | // See the License for the specific language governing permissions and 189 | // limitations under the License. 190 | // 191 | // ----------------------------- END-OF-FILE ---------------------------------- -------------------------------------------------------------------------------- /src/CollectdWinService/CollectdWinService.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | {D4244E6B-84EC-41A0-96A7-4F489F6098BC} 8 | Exe 9 | Properties 10 | BloombergFLP.CollectdWin 11 | CollectdWinService 12 | v3.5 13 | 512 14 | 15 | false 16 | publish\ 17 | true 18 | Disk 19 | false 20 | Foreground 21 | 7 22 | Days 23 | false 24 | false 25 | true 26 | 2 27 | 1.0.0.%2a 28 | false 29 | true 30 | true 31 | 32 | 33 | AnyCPU 34 | true 35 | full 36 | false 37 | bin\Debug\ 38 | DEBUG;TRACE 39 | prompt 40 | 4 41 | 42 | 43 | AnyCPU 44 | pdbonly 45 | true 46 | bin\Release\ 47 | TRACE 48 | prompt 49 | 4 50 | 51 | 52 | true 53 | bin\x64\Debug\ 54 | DEBUG;TRACE 55 | full 56 | x64 57 | prompt 58 | MinimumRecommendedRules.ruleset 59 | 60 | 61 | bin\x64\Release\ 62 | TRACE 63 | true 64 | pdbonly 65 | x64 66 | prompt 67 | MinimumRecommendedRules.ruleset 68 | 69 | 70 | true 71 | bin\x86\Debug\ 72 | DEBUG;TRACE 73 | full 74 | x86 75 | prompt 76 | MinimumRecommendedRules.ruleset 77 | 78 | 79 | bin\x86\Release\ 80 | TRACE 81 | true 82 | pdbonly 83 | x86 84 | prompt 85 | MinimumRecommendedRules.ruleset 86 | 87 | 88 | LocalIntranet 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | ..\..\third-party\NLog.dll 98 | 99 | 100 | ..\..\third-party\RabbitMQ.Client.dll 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | Component 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | CollectdWinService.cs 141 | 142 | 143 | 144 | 145 | Designer 146 | 147 | 148 | Always 149 | 150 | 151 | Always 152 | 153 | 154 | Always 155 | 156 | 157 | Always 158 | 159 | 160 | Always 161 | Designer 162 | 163 | 164 | Always 165 | 166 | 167 | Always 168 | 169 | 170 | 171 | 172 | False 173 | .NET Framework 3.5 SP1 Client Profile 174 | false 175 | 176 | 177 | False 178 | .NET Framework 3.5 SP1 179 | true 180 | 181 | 182 | 183 | 184 | 185 | 186 | 193 | -------------------------------------------------------------------------------- /src/CollectdWinService/StatsdMetrics.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text.RegularExpressions; 5 | using NLog; 6 | 7 | namespace BloombergFLP.CollectdWin 8 | { 9 | internal class StatsdMetric 10 | { 11 | public enum StatsdType 12 | { 13 | StatsdCounter, 14 | StatsdTimer, 15 | StatsdGauge, 16 | StatsdSet 17 | }; 18 | 19 | private readonly HashSet _set; 20 | 21 | public StatsdMetric(string name, StatsdType type, double val) 22 | { 23 | Name = name; 24 | Type = type; 25 | _set = null; 26 | Lat = null; 27 | Value = val; 28 | NumUpdates = 1; 29 | 30 | switch (Type) 31 | { 32 | case StatsdType.StatsdSet: 33 | _set = new HashSet {val}; 34 | break; 35 | case StatsdType.StatsdTimer: 36 | Lat = new Latency(); 37 | Lat.Min = Lat.Max = Lat.Sum = val; 38 | Lat.Num = 1; 39 | break; 40 | } 41 | } 42 | 43 | public string Name { get; private set; } 44 | public StatsdType Type { get; private set; } 45 | public double Value { get; private set; } 46 | public Latency Lat { get; private set; } 47 | public int NumUpdates { get; private set; } 48 | 49 | public double GetMetric() 50 | { 51 | switch (Type) 52 | { 53 | case StatsdType.StatsdSet: 54 | return (_set.Count); 55 | 56 | case StatsdType.StatsdTimer: 57 | return (Lat.Sum/Lat.Num); 58 | } 59 | return (Value); 60 | } 61 | 62 | public void Reset() 63 | { 64 | NumUpdates = 0; 65 | 66 | if (Type == StatsdType.StatsdSet && _set != null) 67 | { 68 | _set.Clear(); 69 | } 70 | if (Type == StatsdType.StatsdTimer && Lat != null) 71 | { 72 | Lat.Reset(); 73 | } 74 | } 75 | 76 | public void AddValue(double value) 77 | { 78 | switch (Type) 79 | { 80 | case StatsdType.StatsdCounter: 81 | Value += value; 82 | break; 83 | case StatsdType.StatsdSet: 84 | _set.Add(value); 85 | break; 86 | case StatsdType.StatsdGauge: 87 | Value = value; 88 | break; 89 | case StatsdType.StatsdTimer: 90 | Lat.AddValue(value); 91 | break; 92 | } 93 | NumUpdates++; 94 | } 95 | 96 | public class Latency 97 | { 98 | public double Max; 99 | public double Min; 100 | public int Num; 101 | public double Sum; 102 | 103 | public Latency() 104 | { 105 | Min = Max = Sum = 0; 106 | Num = 0; 107 | Histogram = null; 108 | if (HistogramEnabled) 109 | { 110 | Histogram = new Histogram(); 111 | } 112 | } 113 | 114 | public Histogram Histogram { get; private set; } 115 | 116 | public static bool HistogramEnabled { get; set; } 117 | 118 | public void AddValue(double val) 119 | { 120 | Min = (val < Min) ? val : Min; 121 | Max = (val > Max) ? val : Max; 122 | Sum += val; 123 | Num++; 124 | if (HistogramEnabled) 125 | { 126 | Histogram.AddValue(val); 127 | } 128 | } 129 | 130 | public void Reset() 131 | { 132 | Min = Max = Sum = 0; 133 | Num = 0; 134 | if (HistogramEnabled) 135 | { 136 | Histogram.Reset(); 137 | } 138 | } 139 | }; 140 | } 141 | 142 | internal class StatsdMetricParser 143 | { 144 | private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); 145 | 146 | public static IList Parse(string str) 147 | { 148 | var metrics = new List(); 149 | string[] lines = str.Split('\n'); 150 | metrics.AddRange(lines.Select(ParseLine).Where(metric => metric != null)); 151 | return (metrics); 152 | } 153 | 154 | /* 155 | * StatsD metric collection protocol 156 | * - metrics are separated by newlines 157 | * - each line ar generally of the form: 158 | * :| 159 | * ** Gauges : :|g 160 | * ** Counters : :|c[|@] 161 | * ** Timers : :|ms 162 | * ** Sets : :|s 163 | */ 164 | 165 | public static StatsdMetric ParseLine(string line) 166 | { 167 | const string pattern = @"^(?.*):(?.*)\|(?.*)(\|\@(?.*))?$"; 168 | var regex = new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.RightToLeft); 169 | Match match = regex.Match(line); 170 | 171 | if (!match.Success) 172 | { 173 | Logger.Debug("Parser: Invalid statsd format [{0}]", line); 174 | return (null); 175 | } 176 | GroupCollection groups = match.Groups; 177 | 178 | string name = groups["name"].Value; 179 | string valstr = groups["value"].Value; 180 | string typestr = groups["type"].Value; 181 | string ratestr = groups["rate"].Value; 182 | 183 | if (String.IsNullOrEmpty(name) || String.IsNullOrEmpty(valstr) || String.IsNullOrEmpty(typestr)) 184 | { 185 | Logger.Debug("Parser: name/value/type are not optional [{0}]", line); 186 | return (null); 187 | } 188 | StatsdMetric.StatsdType type; 189 | 190 | if (String.Compare(typestr, "g", StringComparison.OrdinalIgnoreCase) == 0) 191 | type = StatsdMetric.StatsdType.StatsdGauge; 192 | else if (String.Compare(typestr, "c", StringComparison.OrdinalIgnoreCase) == 0) 193 | type = StatsdMetric.StatsdType.StatsdCounter; 194 | else if (String.Compare(typestr, "s", StringComparison.OrdinalIgnoreCase) == 0) 195 | type = StatsdMetric.StatsdType.StatsdSet; 196 | else if (String.Compare(typestr, "ms", StringComparison.OrdinalIgnoreCase) == 0) 197 | type = StatsdMetric.StatsdType.StatsdTimer; 198 | else 199 | { 200 | Logger.Debug("Parser: invalid type [{0}]", line); 201 | return (null); 202 | } 203 | double value; 204 | try 205 | { 206 | value = Convert.ToDouble(valstr); 207 | } 208 | catch (Exception) 209 | { 210 | Logger.Debug("Parser: invalid value [{0}]", line); 211 | return (null); 212 | } 213 | 214 | double rate = 0; 215 | try 216 | { 217 | if (!String.IsNullOrEmpty(ratestr)) 218 | rate = Convert.ToDouble(ratestr); 219 | } 220 | catch (Exception) 221 | { 222 | Logger.Debug("Parser: invalid rate [{0}]", line); 223 | return (null); 224 | } 225 | 226 | if (!string.IsNullOrEmpty(ratestr) && (rate <= 0 || rate > 1)) 227 | { 228 | Logger.Debug("Parser: invalid rate range [{0}]", line); 229 | return (null); 230 | } 231 | 232 | if (!string.IsNullOrEmpty(ratestr) && type != StatsdMetric.StatsdType.StatsdCounter) 233 | { 234 | Logger.Debug("Parser: rate is supported only for Counters [{0}]", line); 235 | return (null); 236 | } 237 | 238 | if (!string.IsNullOrEmpty(ratestr)) 239 | { 240 | value = value/rate; 241 | } 242 | 243 | var metric = new StatsdMetric(name, type, value); 244 | return (metric); 245 | } 246 | } 247 | } 248 | 249 | // ---------------------------------------------------------------------------- 250 | // Copyright (C) 2015 Bloomberg Finance L.P. 251 | // 252 | // Licensed under the Apache License, Version 2.0 (the "License"); 253 | // you may not use this file except in compliance with the License. 254 | // You may obtain a copy of the License at 255 | // http://www.apache.org/licenses/LICENSE-2.0 256 | // Unless required by applicable law or agreed to in writing, software 257 | // distributed under the License is distributed on an "AS IS" BASIS, 258 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 259 | // See the License for the specific language governing permissions and 260 | // limitations under the License. 261 | // 262 | // ----------------------------- END-OF-FILE ---------------------------------- -------------------------------------------------------------------------------- /src/CollectdWinService/MetricsCollector.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Configuration; 4 | using System.Linq; 5 | using System.Threading; 6 | using System.Diagnostics; 7 | using NLog; 8 | 9 | namespace BloombergFLP.CollectdWin 10 | { 11 | internal class MetricsCollector 12 | { 13 | private const int MaxQueueSize = 30000; 14 | private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); 15 | private readonly Aggregator _aggregator; 16 | private readonly int _interval; 17 | private readonly Queue _metricValueQueue; 18 | private readonly IList _plugins; 19 | private readonly Object _queueLock; 20 | private readonly int _timeout; 21 | private Thread _aggregatorThread; 22 | private Thread _readThread; 23 | private bool _runAggregatorThread; 24 | private bool _runReadThread, _runWriteThread; 25 | private Thread _writeThread; 26 | private IDictionary _metaData; 27 | 28 | public MetricsCollector() 29 | { 30 | var config = ConfigurationManager.GetSection("CollectdWinConfig") as CollectdWinConfig; 31 | if (config == null) 32 | { 33 | Logger.Error("Cannot get configuration section"); 34 | return; 35 | } 36 | 37 | _metaData = CollectdWinConfigHelper.getMetaData(); 38 | 39 | _runReadThread = false; 40 | _runWriteThread = false; 41 | 42 | var registry = new PluginRegistry(); 43 | _plugins = registry.CreatePlugins(); 44 | 45 | _interval = config.GeneralSettings.Interval; 46 | if (_interval <= 10) 47 | _interval = 10; 48 | 49 | _timeout = config.GeneralSettings.Timeout; 50 | if (_timeout <= _interval) 51 | _timeout = _interval*3; 52 | 53 | bool storeRates = config.GeneralSettings.StoreRates; 54 | 55 | _aggregator = new Aggregator(_timeout, storeRates); 56 | 57 | _metricValueQueue = new Queue(); 58 | _queueLock = new Object(); 59 | } 60 | 61 | public void ConfigureAll() 62 | { 63 | Logger.Trace("ConfigureAll() begin"); 64 | foreach (IMetricsPlugin plugin in _plugins) 65 | plugin.Configure(); 66 | Logger.Trace("ConfigureAll() return"); 67 | } 68 | 69 | public void StartAll() 70 | { 71 | Logger.Trace("StartAll() begin"); 72 | foreach (IMetricsPlugin plugin in _plugins) 73 | plugin.Start(); 74 | 75 | _runWriteThread = true; 76 | _writeThread = new Thread(WriteThreadProc); 77 | _writeThread.Start(); 78 | 79 | _runReadThread = true; 80 | _readThread = new Thread(ReadThreadProc); 81 | _readThread.Start(); 82 | 83 | _runAggregatorThread = true; 84 | _aggregatorThread = new Thread(AggregatorThreadProc); 85 | _aggregatorThread.Start(); 86 | Logger.Trace("StartAll() return"); 87 | } 88 | 89 | public void StopAll() 90 | { 91 | Logger.Trace("StopAll() begin"); 92 | _runReadThread = false; 93 | _runWriteThread = false; 94 | _runAggregatorThread = false; 95 | 96 | _readThread.Interrupt(); 97 | _writeThread.Interrupt(); 98 | _aggregatorThread.Interrupt(); 99 | 100 | foreach (IMetricsPlugin plugin in _plugins) 101 | plugin.Stop(); 102 | Logger.Trace("StopAll() end"); 103 | } 104 | 105 | private void ReadThreadProc() 106 | { 107 | Logger.Trace("ReadThreadProc() begin"); 108 | int numMetricsDropped = 0; 109 | while (_runReadThread) 110 | { 111 | try 112 | { 113 | foreach (IMetricsPlugin plugin in _plugins) 114 | { 115 | var readPlugin = plugin as IMetricsReadPlugin; 116 | if (readPlugin == null) 117 | { 118 | // skip if plugin is not a readplugin, it might be a writeplugin 119 | continue; 120 | } 121 | IList metricValues = readPlugin.Read(); 122 | if (metricValues == null || !metricValues.Any()) 123 | continue; 124 | lock (_queueLock) 125 | { 126 | foreach (MetricValue metric in metricValues) 127 | { 128 | _metricValueQueue.Enqueue(metric); 129 | while (_metricValueQueue.Count >= MaxQueueSize) 130 | { 131 | // When queue size grows above the Max limit, 132 | // old entries are removed 133 | _metricValueQueue.Dequeue(); 134 | if ((++numMetricsDropped%1000) == 0) 135 | { 136 | Logger.Error( 137 | "Number of metrics dropped : {0}", 138 | numMetricsDropped); 139 | } 140 | } 141 | } 142 | } 143 | } 144 | Thread.Sleep(_interval*1000); 145 | } 146 | catch (Exception ex) 147 | { 148 | //Get a StackTrace object for the exception 149 | StackTrace st = new StackTrace(ex, true); 150 | 151 | Logger.Error("ReadThreadProc() got exception : {0}, {1}", ex.ToString(), st.ToString()); 152 | } 153 | } 154 | Logger.Trace("ReadThreadProc() return"); 155 | } 156 | 157 | private void WriteThreadProc() 158 | { 159 | Logger.Trace("WriteThreadProc() begin"); 160 | bool needToFlush = false; 161 | while (_runWriteThread) 162 | { 163 | try 164 | { 165 | while (_metricValueQueue.Count > 0) 166 | { 167 | MetricValue metricValue = null; 168 | lock (_queueLock) 169 | { 170 | if (_metricValueQueue.Count > 0) 171 | metricValue = _metricValueQueue.Dequeue(); 172 | } 173 | if (metricValue != null) 174 | { 175 | needToFlush = true; 176 | metricValue.Interval = _interval; 177 | 178 | _aggregator.Aggregate(ref metricValue); 179 | metricValue.AddMetaData(_metaData); 180 | 181 | foreach (IMetricsPlugin plugin in _plugins) 182 | { 183 | var writePlugin = plugin as IMetricsWritePlugin; 184 | if (writePlugin == null) 185 | { 186 | // skip if plugin is not a writeplugin, it might be a readplugin 187 | continue; 188 | } 189 | writePlugin.Write(metricValue); 190 | } 191 | } 192 | } 193 | if (needToFlush) 194 | { 195 | needToFlush = false; 196 | foreach (IMetricsPlugin plugin in _plugins) 197 | { 198 | var writePlugin = plugin as IMetricsWritePlugin; 199 | if (writePlugin != null) 200 | { 201 | // flush only if it is a Write plugin 202 | writePlugin.Flush(); 203 | } 204 | } 205 | } 206 | if (_metricValueQueue.Count <= 0) 207 | { 208 | Thread.Sleep(1000); 209 | } 210 | } 211 | catch (Exception exp) 212 | { 213 | Logger.Error("WriteThreadProc() got exception : ", exp); 214 | } 215 | } 216 | Logger.Trace("WriteThreadProc() return"); 217 | } 218 | 219 | private void AggregatorThreadProc() 220 | { 221 | Logger.Trace("AggregatorThreadProc() begin"); 222 | while (_runAggregatorThread) 223 | { 224 | try 225 | { 226 | _aggregator.RemoveExpiredEntries(); 227 | Thread.Sleep(_timeout*1000); 228 | } 229 | catch (Exception exp) 230 | { 231 | Logger.Error("AggregatorThreadProc() got exception : ", exp); 232 | } 233 | } 234 | Logger.Trace("AggregatorThreadProc() return"); 235 | } 236 | } 237 | } 238 | 239 | // ---------------------------------------------------------------------------- 240 | // Copyright (C) 2015 Bloomberg Finance L.P. 241 | // 242 | // Licensed under the Apache License, Version 2.0 (the "License"); 243 | // you may not use this file except in compliance with the License. 244 | // You may obtain a copy of the License at 245 | // http://www.apache.org/licenses/LICENSE-2.0 246 | // Unless required by applicable law or agreed to in writing, software 247 | // distributed under the License is distributed on an "AS IS" BASIS, 248 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 249 | // See the License for the specific language governing permissions and 250 | // limitations under the License. 251 | // 252 | // ----------------------------- END-OF-FILE ---------------------------------- -------------------------------------------------------------------------------- /src/CollectdWinService/types.db: -------------------------------------------------------------------------------- 1 | # 2 | # types.db - Data-set specifications for the system statistics collection daemon collectd 3 | # This file is a copy from following URL: 4 | # https://raw.githubusercontent.com/collectd/collectd/master/src/types.db 5 | # 6 | # The types.db file contains one line for each data-set specification. Each 7 | # line consists of two fields delimited by spaces and/or horizontal tabs. 8 | # The first field defines the name of the data-set, while the second field 9 | # defines a list of data-source specifications, delimited by spaces and, 10 | # optionally, a comma (",") right after each list-entry. 11 | # 12 | # The format of the data-source specification has been inspired by RRDtool's 13 | # data-source specification. Each data-source is defined by a quadruple made 14 | # up of the data-source name, type, minimal and maximal values, delimited by 15 | # colons (":"): ds-name:ds-type:min:max. ds-type may be either ABSOLUTE, 16 | # COUNTER, DERIVE, or GAUGE. min and max define the range of valid values for 17 | # data stored for this data-source. If U is specified for either the min or 18 | # max value, it will be set to unknown, meaning that no range checks will 19 | # happen. See rrdcreate(1) for more details. 20 | # 21 | # 22 | absolute value:ABSOLUTE:0:U 23 | apache_bytes value:DERIVE:0:U 24 | apache_connections value:GAUGE:0:65535 25 | apache_idle_workers value:GAUGE:0:65535 26 | apache_requests value:DERIVE:0:U 27 | apache_scoreboard value:GAUGE:0:65535 28 | ath_nodes value:GAUGE:0:65535 29 | ath_stat value:DERIVE:0:U 30 | backends value:GAUGE:0:65535 31 | bitrate value:GAUGE:0:4294967295 32 | blocked_clients value:GAUGE:0:U 33 | bytes value:GAUGE:0:U 34 | cache_eviction value:DERIVE:0:U 35 | cache_operation value:DERIVE:0:U 36 | cache_ratio value:GAUGE:0:100 37 | cache_result value:DERIVE:0:U 38 | cache_size value:GAUGE:0:U 39 | ceph_bytes value:GAUGE:U:U 40 | ceph_latency value:GAUGE:U:U 41 | ceph_rate value:DERIVE:0:U 42 | changes_since_last_save value:GAUGE:0:U 43 | charge value:GAUGE:0:U 44 | compression_ratio value:GAUGE:0:2 45 | compression uncompressed:DERIVE:0:U, compressed:DERIVE:0:U 46 | connections value:DERIVE:0:U 47 | conntrack value:GAUGE:0:4294967295 48 | contextswitch value:DERIVE:0:U 49 | count value:GAUGE:0:U 50 | counter value:COUNTER:U:U 51 | cpufreq value:GAUGE:0:U 52 | cpu value:DERIVE:0:U 53 | current_connections value:GAUGE:0:U 54 | current_sessions value:GAUGE:0:U 55 | current value:GAUGE:U:U 56 | delay value:GAUGE:-1000000:1000000 57 | derive value:DERIVE:0:U 58 | df_complex value:GAUGE:0:U 59 | df_inodes value:GAUGE:0:U 60 | df used:GAUGE:0:1125899906842623, free:GAUGE:0:1125899906842623 61 | disk_latency read:GAUGE:0:U, write:GAUGE:0:U 62 | disk_merged read:DERIVE:0:U, write:DERIVE:0:U 63 | disk_octets read:DERIVE:0:U, write:DERIVE:0:U 64 | disk_ops_complex value:DERIVE:0:U 65 | disk_ops read:DERIVE:0:U, write:DERIVE:0:U 66 | disk_time read:DERIVE:0:U, write:DERIVE:0:U 67 | disk_io_time io_time:DERIVE:0:U, weighted_io_time:DERIVE:0:U 68 | dns_answer value:DERIVE:0:U 69 | dns_notify value:DERIVE:0:U 70 | dns_octets queries:DERIVE:0:U, responses:DERIVE:0:U 71 | dns_opcode value:DERIVE:0:U 72 | dns_qtype_cached value:GAUGE:0:4294967295 73 | dns_qtype value:DERIVE:0:U 74 | dns_query value:DERIVE:0:U 75 | dns_question value:DERIVE:0:U 76 | dns_rcode value:DERIVE:0:U 77 | dns_reject value:DERIVE:0:U 78 | dns_request value:DERIVE:0:U 79 | dns_resolver value:DERIVE:0:U 80 | dns_response value:DERIVE:0:U 81 | dns_transfer value:DERIVE:0:U 82 | dns_update value:DERIVE:0:U 83 | dns_zops value:DERIVE:0:U 84 | drbd_resource value:DERIVE:0:U 85 | duration seconds:GAUGE:0:U 86 | email_check value:GAUGE:0:U 87 | email_count value:GAUGE:0:U 88 | email_size value:GAUGE:0:U 89 | entropy value:GAUGE:0:4294967295 90 | expired_keys value:GAUGE:0:U 91 | fanspeed value:GAUGE:0:U 92 | file_size value:GAUGE:0:U 93 | files value:GAUGE:0:U 94 | flow value:GAUGE:0:U 95 | fork_rate value:DERIVE:0:U 96 | frequency_offset value:GAUGE:-1000000:1000000 97 | frequency value:GAUGE:0:U 98 | fscache_stat value:DERIVE:0:U 99 | gauge value:GAUGE:U:U 100 | hash_collisions value:DERIVE:0:U 101 | http_request_methods value:DERIVE:0:U 102 | http_requests value:DERIVE:0:U 103 | http_response_codes value:DERIVE:0:U 104 | humidity value:GAUGE:0:100 105 | if_collisions value:DERIVE:0:U 106 | if_dropped rx:DERIVE:0:U, tx:DERIVE:0:U 107 | if_errors rx:DERIVE:0:U, tx:DERIVE:0:U 108 | if_multicast value:DERIVE:0:U 109 | if_octets rx:DERIVE:0:U, tx:DERIVE:0:U 110 | if_packets rx:DERIVE:0:U, tx:DERIVE:0:U 111 | if_rx_errors value:DERIVE:0:U 112 | if_rx_octets value:DERIVE:0:U 113 | if_tx_errors value:DERIVE:0:U 114 | if_tx_octets value:DERIVE:0:U 115 | invocations value:DERIVE:0:U 116 | io_octets rx:DERIVE:0:U, tx:DERIVE:0:U 117 | io_packets rx:DERIVE:0:U, tx:DERIVE:0:U 118 | ipt_bytes value:DERIVE:0:U 119 | ipt_packets value:DERIVE:0:U 120 | irq value:DERIVE:0:U 121 | latency value:GAUGE:0:U 122 | links value:GAUGE:0:U 123 | load shortterm:GAUGE:0:5000, midterm:GAUGE:0:5000, longterm:GAUGE:0:5000 124 | md_disks value:GAUGE:0:U 125 | memcached_command value:DERIVE:0:U 126 | memcached_connections value:GAUGE:0:U 127 | memcached_items value:GAUGE:0:U 128 | memcached_octets rx:DERIVE:0:U, tx:DERIVE:0:U 129 | memcached_ops value:DERIVE:0:U 130 | memory value:GAUGE:0:281474976710656 131 | memory_lua value:GAUGE:0:281474976710656 132 | multimeter value:GAUGE:U:U 133 | mutex_operations value:DERIVE:0:U 134 | mysql_commands value:DERIVE:0:U 135 | mysql_handler value:DERIVE:0:U 136 | mysql_locks value:DERIVE:0:U 137 | mysql_log_position value:DERIVE:0:U 138 | mysql_octets rx:DERIVE:0:U, tx:DERIVE:0:U 139 | mysql_bpool_pages value:GAUGE:0:U 140 | mysql_bpool_bytes value:GAUGE:0:U 141 | mysql_bpool_counters value:DERIVE:0:U 142 | mysql_innodb_data value:DERIVE:0:U 143 | mysql_innodb_dblwr value:DERIVE:0:U 144 | mysql_innodb_log value:DERIVE:0:U 145 | mysql_innodb_pages value:DERIVE:0:U 146 | mysql_innodb_row_lock value:DERIVE:0:U 147 | mysql_innodb_rows value:DERIVE:0:U 148 | mysql_select value:DERIVE:0:U 149 | mysql_sort value:DERIVE:0:U 150 | nfs_procedure value:DERIVE:0:U 151 | nginx_connections value:GAUGE:0:U 152 | nginx_requests value:DERIVE:0:U 153 | node_octets rx:DERIVE:0:U, tx:DERIVE:0:U 154 | node_rssi value:GAUGE:0:255 155 | node_stat value:DERIVE:0:U 156 | node_tx_rate value:GAUGE:0:127 157 | objects value:GAUGE:0:U 158 | operations value:DERIVE:0:U 159 | packets value:DERIVE:0:U 160 | pending_operations value:GAUGE:0:U 161 | percent value:GAUGE:0:100.1 162 | percent_bytes value:GAUGE:0:100.1 163 | percent_inodes value:GAUGE:0:100.1 164 | pf_counters value:DERIVE:0:U 165 | pf_limits value:DERIVE:0:U 166 | pf_source value:DERIVE:0:U 167 | pf_states value:GAUGE:0:U 168 | pf_state value:DERIVE:0:U 169 | pg_blks value:DERIVE:0:U 170 | pg_db_size value:GAUGE:0:U 171 | pg_n_tup_c value:DERIVE:0:U 172 | pg_n_tup_g value:GAUGE:0:U 173 | pg_numbackends value:GAUGE:0:U 174 | pg_scan value:DERIVE:0:U 175 | pg_xact value:DERIVE:0:U 176 | ping_droprate value:GAUGE:0:100 177 | ping_stddev value:GAUGE:0:65535 178 | ping value:GAUGE:0:65535 179 | players value:GAUGE:0:1000000 180 | power value:GAUGE:0:U 181 | pressure value:GAUGE:0:U 182 | protocol_counter value:DERIVE:0:U 183 | ps_code value:GAUGE:0:9223372036854775807 184 | ps_count processes:GAUGE:0:1000000, threads:GAUGE:0:1000000 185 | ps_cputime user:DERIVE:0:U, syst:DERIVE:0:U 186 | ps_data value:GAUGE:0:9223372036854775807 187 | ps_disk_octets read:DERIVE:0:U, write:DERIVE:0:U 188 | ps_disk_ops read:DERIVE:0:U, write:DERIVE:0:U 189 | ps_pagefaults minflt:DERIVE:0:U, majflt:DERIVE:0:U 190 | ps_rss value:GAUGE:0:9223372036854775807 191 | ps_stacksize value:GAUGE:0:9223372036854775807 192 | ps_state value:GAUGE:0:65535 193 | ps_vm value:GAUGE:0:9223372036854775807 194 | pubsub value:GAUGE:0:U 195 | queue_length value:GAUGE:0:U 196 | records value:GAUGE:0:U 197 | requests value:GAUGE:0:U 198 | response_time value:GAUGE:0:U 199 | response_code value:GAUGE:0:U 200 | route_etx value:GAUGE:0:U 201 | route_metric value:GAUGE:0:U 202 | routes value:GAUGE:0:U 203 | segments value:GAUGE:0:65535 204 | serial_octets rx:DERIVE:0:U, tx:DERIVE:0:U 205 | signal_noise value:GAUGE:U:0 206 | signal_power value:GAUGE:U:0 207 | signal_quality value:GAUGE:0:U 208 | smart_poweron value:GAUGE:0:U 209 | smart_powercycles value:GAUGE:0:U 210 | smart_badsectors value:GAUGE:0:U 211 | smart_temperature value:GAUGE:-300:300 212 | smart_attribute current:GAUGE:0:255, worst:GAUGE:0:255, threshold:GAUGE:0:255, pretty:GAUGE:0:U 213 | snr value:GAUGE:0:U 214 | spam_check value:GAUGE:0:U 215 | spam_score value:GAUGE:U:U 216 | spl value:GAUGE:U:U 217 | swap_io value:DERIVE:0:U 218 | swap value:GAUGE:0:1099511627776 219 | tcp_connections value:GAUGE:0:4294967295 220 | temperature value:GAUGE:U:U 221 | threads value:GAUGE:0:U 222 | time_dispersion value:GAUGE:-1000000:1000000 223 | timeleft value:GAUGE:0:U 224 | time_offset value:GAUGE:-1000000:1000000 225 | total_bytes value:DERIVE:0:U 226 | total_connections value:DERIVE:0:U 227 | total_objects value:DERIVE:0:U 228 | total_operations value:DERIVE:0:U 229 | total_requests value:DERIVE:0:U 230 | total_sessions value:DERIVE:0:U 231 | total_threads value:DERIVE:0:U 232 | total_time_in_ms value:DERIVE:0:U 233 | total_values value:DERIVE:0:U 234 | uptime value:GAUGE:0:4294967295 235 | users value:GAUGE:0:65535 236 | vcl value:GAUGE:0:65535 237 | vcpu value:GAUGE:0:U 238 | virt_cpu_total value:DERIVE:0:U 239 | virt_vcpu value:DERIVE:0:U 240 | vmpage_action value:DERIVE:0:U 241 | vmpage_faults minflt:DERIVE:0:U, majflt:DERIVE:0:U 242 | vmpage_io in:DERIVE:0:U, out:DERIVE:0:U 243 | vmpage_number value:GAUGE:0:4294967295 244 | volatile_changes value:GAUGE:0:U 245 | voltage_threshold value:GAUGE:U:U, threshold:GAUGE:U:U 246 | voltage value:GAUGE:U:U 247 | vs_memory value:GAUGE:0:9223372036854775807 248 | vs_processes value:GAUGE:0:65535 249 | vs_threads value:GAUGE:0:65535 250 | 251 | # 252 | # Legacy types 253 | # (required for the v5 upgrade target) 254 | # 255 | arc_counts demand_data:COUNTER:0:U, demand_metadata:COUNTER:0:U, prefetch_data:COUNTER:0:U, prefetch_metadata:COUNTER:0:U 256 | arc_l2_bytes read:COUNTER:0:U, write:COUNTER:0:U 257 | arc_l2_size value:GAUGE:0:U 258 | arc_ratio value:GAUGE:0:U 259 | arc_size current:GAUGE:0:U, target:GAUGE:0:U, minlimit:GAUGE:0:U, maxlimit:GAUGE:0:U 260 | mysql_qcache hits:COUNTER:0:U, inserts:COUNTER:0:U, not_cached:COUNTER:0:U, lowmem_prunes:COUNTER:0:U, queries_in_cache:GAUGE:0:U 261 | mysql_threads running:GAUGE:0:U, connected:GAUGE:0:U, cached:GAUGE:0:U, created:COUNTER:0:U 262 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /src/CollectdWinService/WriteHttpPlugin.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Configuration; 4 | using System.IO; 5 | using System.Net; 6 | using System.Text; 7 | using NLog; 8 | using System.Text.RegularExpressions; 9 | 10 | namespace BloombergFLP.CollectdWin 11 | { 12 | internal class WriteHttpPlugin : IMetricsWritePlugin 13 | { 14 | private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); 15 | 16 | private readonly IList _httpWriters; 17 | 18 | public WriteHttpPlugin() 19 | { 20 | _httpWriters = new List(); 21 | } 22 | 23 | public void Configure() 24 | { 25 | var config = ConfigurationManager.GetSection("WriteHttp") as WriteHttpPluginConfig; 26 | if (config == null) 27 | { 28 | throw new Exception("Cannot get configuration section : WriteHttp"); 29 | } 30 | 31 | _httpWriters.Clear(); 32 | 33 | foreach (WriteHttpPluginConfig.WriteHttpNodeConfig node in config.Nodes) 34 | { 35 | var writer = new HttpWriter 36 | { 37 | Url = node.Url, 38 | Timeout = node.Timeout, 39 | BatchSize = node.BatchSize, 40 | MaxIdleTime = node.MaxIdleTime, 41 | EnableProxy = node.Proxy.Enable 42 | }; 43 | 44 | if (writer.EnableProxy) 45 | { 46 | writer.WebProxy = node.Proxy.Url.Length > 0 ? new WebProxy(node.Proxy.Url) : new WebProxy(); 47 | } 48 | 49 | if (!string.IsNullOrEmpty(node.UserName) && !string.IsNullOrEmpty(node.Password)) 50 | { 51 | /* Possibly misfeature- adding BasicAuthHeaderData to HttpWriter class to efficiently support basic auth, 52 | * but saves the ToBase64String string encode on each request. Better, but more expensive, would be 53 | * to put both on as secure strings and add a config param to support other auth methods while 54 | * building the HttpWebResponse on each call. @FerventGeek */ 55 | writer.BasicAuthHeaderData = System.Convert.ToBase64String( 56 | System.Text.Encoding.GetEncoding("ISO-8859-1").GetBytes(node.UserName + ":" + node.Password)); 57 | Logger.Info("Using BasicAuth for node {0}, user {1}", node.Name, node.UserName); 58 | } 59 | 60 | if (!string.IsNullOrEmpty(node.SafeCharsRegex)) 61 | { 62 | // compile for perfomace, since config is only loaded on start 63 | writer.SafeCharsRegex = new Regex("[^" + node.SafeCharsRegex + "]", RegexOptions.Compiled); 64 | Logger.Info("Using SafeChars for node {0}, regex \"{1}\" replaced with \"{2}\"", 65 | node.Name, writer.SafeCharsRegex.ToString(), node.ReplaceWith); 66 | } 67 | 68 | if (node.ReplaceWith == null) 69 | { 70 | // default, strip unsafe chars 71 | writer.ReplaceWith = ""; 72 | } 73 | else 74 | { 75 | writer.ReplaceWith = node.ReplaceWith; 76 | } 77 | 78 | _httpWriters.Add(writer); 79 | } 80 | 81 | Logger.Info("WriteHttp plugin configured"); 82 | } 83 | 84 | public void Flush() 85 | { 86 | foreach (HttpWriter writer in _httpWriters) 87 | { 88 | writer.Flush(); 89 | } 90 | } 91 | 92 | public void Start() 93 | { 94 | Logger.Info("WriteHttp - plugin started"); 95 | } 96 | 97 | public void Stop() 98 | { 99 | Logger.Info("WriteHttp - plugin stopped"); 100 | } 101 | 102 | public void Write(MetricValue metric) 103 | { 104 | if (metric == null) 105 | { 106 | Logger.Debug("write() - Invalid null metric"); 107 | return; 108 | } 109 | foreach (HttpWriter writer in _httpWriters) 110 | { 111 | writer.Write(metric); 112 | } 113 | } 114 | } 115 | 116 | internal class HttpWriter 117 | { 118 | private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); 119 | 120 | public int BatchSize = 20; 121 | public bool EnableProxy = false; 122 | public int MaxIdleTime; 123 | public int Timeout; 124 | public string Url; 125 | public WebProxy WebProxy = null; 126 | public string BasicAuthHeaderData = null; 127 | public Regex SafeCharsRegex = null; 128 | public string ReplaceWith = null; 129 | 130 | private StringBuilder _batchedMetricStr; 131 | private int _numMetrics; 132 | private int _numSequentialHttpFailures = 0; 133 | 134 | private void publish() 135 | { 136 | if (_batchedMetricStr != null) 137 | { 138 | _batchedMetricStr.Append("]"); 139 | HttpPost(_batchedMetricStr.ToString()); 140 | _batchedMetricStr = null; 141 | _numMetrics = 0; 142 | } 143 | } 144 | 145 | public void Write(MetricValue metric) 146 | { 147 | // Optional Regex replace of unsafe chars 148 | if (SafeCharsRegex != null) 149 | { 150 | metric.PluginInstanceName = SafeCharsRegex.Replace(metric.PluginInstanceName, ReplaceWith); 151 | } 152 | 153 | string message = metric.GetMetricJsonStr(); 154 | 155 | if (_batchedMetricStr == null) 156 | { 157 | _batchedMetricStr = new StringBuilder("[").Append(message); 158 | } 159 | else 160 | { 161 | _batchedMetricStr.Append(",").Append(message); 162 | } 163 | _numMetrics++; 164 | 165 | if (_numMetrics < BatchSize) return; 166 | 167 | publish(); 168 | } 169 | 170 | public void Flush() 171 | { 172 | publish(); 173 | } 174 | 175 | public void HttpPost(string metricJsonStr) 176 | { 177 | HttpWebResponse response = null; 178 | try 179 | { 180 | byte[] data = Encoding.UTF8.GetBytes(metricJsonStr); 181 | var request = (HttpWebRequest) WebRequest.Create(Url); 182 | 183 | request.Method = "POST"; 184 | request.ContentType = "application/json"; 185 | request.ContentLength = data.Length; 186 | request.UserAgent = "CollectdWin/1.0"; 187 | request.Accept = "*/*"; 188 | request.KeepAlive = true; 189 | request.Timeout = Timeout; 190 | 191 | if (EnableProxy) 192 | { 193 | request.Proxy = WebProxy; 194 | } 195 | if (MaxIdleTime > 0) 196 | { 197 | request.ServicePoint.MaxIdleTime = MaxIdleTime; 198 | } 199 | // Optional do BasicAuth for POST 200 | if (BasicAuthHeaderData != null) 201 | { 202 | request.Headers.Add("Authorization", "Basic " + BasicAuthHeaderData); 203 | } 204 | 205 | // Display service point properties. 206 | Logger.Trace("Connection properties: ServicePoint - HashCode:{0}, MaxIdleTime:{1}, IdleSince:{2}", 207 | request.ServicePoint.GetHashCode(), request.ServicePoint.MaxIdleTime, request.ServicePoint.IdleSince); 208 | 209 | using (Stream reqStream = request.GetRequestStream()) 210 | { 211 | if (Logger.IsTraceEnabled) 212 | { 213 | Logger.Trace("Adding request body : {0}", metricJsonStr); 214 | } 215 | 216 | reqStream.Write(data, 0, data.Length); 217 | } 218 | 219 | response = (HttpWebResponse) request.GetResponse(); 220 | 221 | // Skip overhead of the trace body read 222 | if (Logger.IsTraceEnabled) 223 | { 224 | Stream respStream = response.GetResponseStream(); 225 | string responseString = new StreamReader(respStream).ReadToEnd(); 226 | Logger.Trace("Got response : {0} - {1} : {2}", 227 | (int)response.StatusCode, response.StatusCode, responseString); 228 | } 229 | _numSequentialHttpFailures = 0; 230 | } 231 | catch (WebException ex) 232 | { 233 | _numSequentialHttpFailures += 1; 234 | 235 | const int errThreshold = 3; 236 | LogLevel logLevel = _numSequentialHttpFailures > errThreshold ? LogLevel.Error 237 | : LogLevel.Warn; 238 | if (ex.Status == WebExceptionStatus.ProtocolError) 239 | { 240 | HttpWebResponse exceptionResponse = (HttpWebResponse)ex.Response; 241 | 242 | Logger.Log(logLevel, "Got web exception in http post : {0} - {1}", 243 | (int)exceptionResponse.StatusCode, exceptionResponse.StatusCode); 244 | 245 | if (Logger.IsTraceEnabled) 246 | { 247 | // Skip overhead of trace body read 248 | using (var stream = exceptionResponse.GetResponseStream()) 249 | using (var reader = new StreamReader(stream)) 250 | { 251 | string errorBody = reader.ReadToEnd(); 252 | if (errorBody != null) 253 | { 254 | Logger.Trace(errorBody); 255 | } 256 | } 257 | } 258 | } 259 | else 260 | { 261 | Logger.Log(logLevel, "Got web exception in http post : {0}", ex.ToString()); 262 | } 263 | } 264 | catch (Exception exp) 265 | { 266 | Logger.Error("Got exception in http post : {0}", exp); 267 | } 268 | finally 269 | { 270 | if (response != null) 271 | { 272 | response.Close(); 273 | } 274 | } 275 | } 276 | } 277 | } 278 | 279 | // ---------------------------------------------------------------------------- 280 | // Copyright (C) 2015 Bloomberg Finance L.P. 281 | // 282 | // Licensed under the Apache License, Version 2.0 (the "License"); 283 | // you may not use this file except in compliance with the License. 284 | // You may obtain a copy of the License at 285 | // http://www.apache.org/licenses/LICENSE-2.0 286 | // Unless required by applicable law or agreed to in writing, software 287 | // distributed under the License is distributed on an "AS IS" BASIS, 288 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 289 | // See the License for the specific language governing permissions and 290 | // limitations under the License. 291 | // 292 | // ----------------------------- END-OF-FILE ---------------------------------- 293 | --------------------------------------------------------------------------------