├── .gitattributes ├── .gitignore ├── .vscode ├── launch.json └── tasks.json ├── Delete-BIN-OBJ-Folders.bat ├── Directory.Build.props ├── Directory.Build.targets ├── LICENSE ├── NuGet.config ├── NuGetPackageVerifier.json ├── README.md ├── docs ├── .env ├── docker-compose.yml ├── elasticsearch.md └── img │ ├── kibana_empty.png │ ├── kibana_index.png │ ├── kibana_initdata.png │ └── kinaba_discover.png ├── samples └── SampleApp │ ├── LoggerExtensions.cs │ ├── Program.cs │ ├── SampleApp.csproj │ └── logging.json ├── src ├── Zero.Logging.Batching │ ├── BatchLoggerConfigureOptions.cs │ ├── BatchingLogger.cs │ ├── BatchingLoggerOptions.cs │ ├── BatchingLoggerProvider.cs │ ├── LogMessage.cs │ └── Zero.Logging.Batching.csproj ├── Zero.Logging.Elasticsearch │ ├── ElasticsearchHelper.cs │ ├── EsLogger.cs │ ├── EsLoggerFactoryExtensions.cs │ ├── EsLoggerOptions.cs │ ├── EsLoggerOptionsSetup.cs │ ├── EsLoggerProvider.cs │ └── Zero.Logging.Elasticsearch.csproj └── Zero.Logging.File │ ├── FileLoggerFactoryExtensions.cs │ ├── FileLoggerOptions.cs │ ├── FileLoggerOptionsSetup.cs │ ├── FileLoggerProvider.cs │ ├── RollingIntervalEnum.cs │ ├── RollingIntervalExtensions.cs │ └── Zero.Logging.File.csproj ├── test └── Zero.Logging.Tests │ ├── BigDataTests.cs │ ├── Zero.Logging.Tests.csproj │ └── nlog.config ├── version.props └── zero-logging.sln /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.userosscache 8 | *.sln.docstates 9 | 10 | # User-specific files (MonoDevelop/Xamarin Studio) 11 | *.userprefs 12 | 13 | # Build results 14 | [Dd]ebug/ 15 | [Dd]ebugPublic/ 16 | [Rr]elease/ 17 | [Rr]eleases/ 18 | x64/ 19 | x86/ 20 | bld/ 21 | [Bb]in/ 22 | [Oo]bj/ 23 | [Ll]og/ 24 | 25 | # Visual Studio 2015 cache/options directory 26 | .vs/ 27 | # Uncomment if you have tasks that create the project's static files in wwwroot 28 | #wwwroot/ 29 | 30 | # MSTest test Results 31 | [Tt]est[Rr]esult*/ 32 | [Bb]uild[Ll]og.* 33 | 34 | # NUNIT 35 | *.VisualState.xml 36 | TestResult.xml 37 | 38 | # Build Results of an ATL Project 39 | [Dd]ebugPS/ 40 | [Rr]eleasePS/ 41 | dlldata.c 42 | 43 | # DNX 44 | project.lock.json 45 | project.fragment.lock.json 46 | artifacts/ 47 | 48 | *_i.c 49 | *_p.c 50 | *_i.h 51 | *.ilk 52 | *.meta 53 | *.obj 54 | *.pch 55 | *.pdb 56 | *.pgc 57 | *.pgd 58 | *.rsp 59 | *.sbr 60 | *.tlb 61 | *.tli 62 | *.tlh 63 | *.tmp 64 | *.tmp_proj 65 | *.log 66 | *.vspscc 67 | *.vssscc 68 | .builds 69 | *.pidb 70 | *.svclog 71 | *.scc 72 | 73 | # Chutzpah Test files 74 | _Chutzpah* 75 | 76 | # Visual C++ cache files 77 | ipch/ 78 | *.aps 79 | *.ncb 80 | *.opendb 81 | *.opensdf 82 | *.sdf 83 | *.cachefile 84 | *.VC.db 85 | *.VC.VC.opendb 86 | 87 | # Visual Studio profiler 88 | *.psess 89 | *.vsp 90 | *.vspx 91 | *.sap 92 | 93 | # TFS 2012 Local Workspace 94 | $tf/ 95 | 96 | # Guidance Automation Toolkit 97 | *.gpState 98 | 99 | # ReSharper is a .NET coding add-in 100 | _ReSharper*/ 101 | *.[Rr]e[Ss]harper 102 | *.DotSettings.user 103 | 104 | # JustCode is a .NET coding add-in 105 | .JustCode 106 | 107 | # TeamCity is a build add-in 108 | _TeamCity* 109 | 110 | # DotCover is a Code Coverage Tool 111 | *.dotCover 112 | 113 | # NCrunch 114 | _NCrunch_* 115 | .*crunch*.local.xml 116 | nCrunchTemp_* 117 | 118 | # MightyMoose 119 | *.mm.* 120 | AutoTest.Net/ 121 | 122 | # Web workbench (sass) 123 | .sass-cache/ 124 | 125 | # Installshield output folder 126 | [Ee]xpress/ 127 | 128 | # DocProject is a documentation generator add-in 129 | DocProject/buildhelp/ 130 | DocProject/Help/*.HxT 131 | DocProject/Help/*.HxC 132 | DocProject/Help/*.hhc 133 | DocProject/Help/*.hhk 134 | DocProject/Help/*.hhp 135 | DocProject/Help/Html2 136 | DocProject/Help/html 137 | 138 | # Click-Once directory 139 | publish/ 140 | 141 | # Publish Web Output 142 | *.[Pp]ublish.xml 143 | *.azurePubxml 144 | # TODO: Comment the next line if you want to checkin your web deploy settings 145 | # but database connection strings (with potential passwords) will be unencrypted 146 | #*.pubxml 147 | *.publishproj 148 | 149 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 150 | # checkin your Azure Web App publish settings, but sensitive information contained 151 | # in these scripts will be unencrypted 152 | PublishScripts/ 153 | 154 | # NuGet Packages 155 | *.nupkg 156 | # The packages folder can be ignored because of Package Restore 157 | **/packages/* 158 | # except build/, which is used as an MSBuild target. 159 | !**/packages/build/ 160 | # Uncomment if necessary however generally it will be regenerated when needed 161 | #!**/packages/repositories.config 162 | # NuGet v3's project.json files produces more ignoreable files 163 | *.nuget.props 164 | *.nuget.targets 165 | 166 | # Microsoft Azure Build Output 167 | csx/ 168 | *.build.csdef 169 | 170 | # Microsoft Azure Emulator 171 | ecf/ 172 | rcf/ 173 | 174 | # Windows Store app package directories and files 175 | AppPackages/ 176 | BundleArtifacts/ 177 | Package.StoreAssociation.xml 178 | _pkginfo.txt 179 | 180 | # Visual Studio cache files 181 | # files ending in .cache can be ignored 182 | *.[Cc]ache 183 | # but keep track of directories ending in .cache 184 | !*.[Cc]ache/ 185 | 186 | # Others 187 | ClientBin/ 188 | ~$* 189 | *~ 190 | *.dbmdl 191 | *.dbproj.schemaview 192 | *.jfm 193 | *.pfx 194 | *.publishsettings 195 | node_modules/ 196 | orleans.codegen.cs 197 | 198 | # Since there are multiple workflows, uncomment next line to ignore bower_components 199 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 200 | #bower_components/ 201 | 202 | # RIA/Silverlight projects 203 | Generated_Code/ 204 | 205 | # Backup & report files from converting an old project file 206 | # to a newer Visual Studio version. Backup files are not needed, 207 | # because we have git ;-) 208 | _UpgradeReport_Files/ 209 | Backup*/ 210 | UpgradeLog*.XML 211 | UpgradeLog*.htm 212 | 213 | # SQL Server files 214 | *.mdf 215 | *.ldf 216 | 217 | # Business Intelligence projects 218 | *.rdl.data 219 | *.bim.layout 220 | *.bim_*.settings 221 | 222 | # Microsoft Fakes 223 | FakesAssemblies/ 224 | 225 | # GhostDoc plugin setting file 226 | *.GhostDoc.xml 227 | 228 | # Node.js Tools for Visual Studio 229 | .ntvs_analysis.dat 230 | 231 | # Visual Studio 6 build log 232 | *.plg 233 | 234 | # Visual Studio 6 workspace options file 235 | *.opt 236 | 237 | # Visual Studio LightSwitch build output 238 | **/*.HTMLClient/GeneratedArtifacts 239 | **/*.DesktopClient/GeneratedArtifacts 240 | **/*.DesktopClient/ModelManifest.xml 241 | **/*.Server/GeneratedArtifacts 242 | **/*.Server/ModelManifest.xml 243 | _Pvt_Extensions 244 | 245 | # Paket dependency manager 246 | .paket/paket.exe 247 | paket-files/ 248 | 249 | # FAKE - F# Make 250 | .fake/ 251 | 252 | # JetBrains Rider 253 | .idea/ 254 | *.sln.iml 255 | 256 | # CodeRush 257 | .cr/ 258 | 259 | # Python Tools for Visual Studio (PTVS) 260 | __pycache__/ 261 | *.pyc -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to find out which attributes exist for C# debugging 3 | // Use hover for the description of the existing attributes 4 | // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": ".NET Core Launch (console)", 9 | "type": "coreclr", 10 | "request": "launch", 11 | "preLaunchTask": "build", 12 | // If you have changed target frameworks, make sure to update the program path. 13 | "program": "${workspaceRoot}/samples/SampleApp/bin/Debug/netcoreapp2.0/SampleApp.dll", 14 | "args": [], 15 | "cwd": "${workspaceRoot}/samples/SampleApp", 16 | // For more information about the 'console' field, see https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md#console-terminal-window 17 | "console": "internalConsole", 18 | "stopAtEntry": false, 19 | "internalConsoleOptions": "openOnSessionStart" 20 | }, 21 | { 22 | "name": ".NET Core Attach", 23 | "type": "coreclr", 24 | "request": "attach", 25 | "processId": "${command:pickProcess}" 26 | } 27 | ] 28 | } -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.1.0", 3 | "command": "dotnet", 4 | "isShellCommand": true, 5 | "args": [], 6 | "tasks": [ 7 | { 8 | "taskName": "build", 9 | "args": [ 10 | "${workspaceRoot}/samples/SampleApp/SampleApp.csproj" 11 | ], 12 | "isBuildCommand": true, 13 | "problemMatcher": "$msCompile" 14 | } 15 | ] 16 | } -------------------------------------------------------------------------------- /Delete-BIN-OBJ-Folders.bat: -------------------------------------------------------------------------------- 1 | @ECHO off 2 | cls 3 | 4 | ECHO Deleting all BIN and OBJ folders... 5 | ECHO. 6 | 7 | FOR /d /r . %%d in (bin,obj) DO ( 8 | IF EXIST "%%d" ( 9 | ECHO %%d | FIND /I "\node_modules\" > Nul && ( 10 | ECHO.Skipping: %%d 11 | ) || ( 12 | ECHO.Deleting: %%d 13 | rd /s/q "%%d" 14 | ) 15 | ) 16 | ) 17 | 18 | ECHO. 19 | ECHO.BIN and OBJ folders have been successfully deleted. Press any key to exit. 20 | pause > nul -------------------------------------------------------------------------------- /Directory.Build.props: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | Zero Logging 6 | RainingNight 7 | RainingNight 8 | Copyright © RainingNight 9 | https://raw.githubusercontent.com/RainingNight/zero-logging/dev/LICENSE 10 | 11 | MIT 12 | 16 | $(NoWarn);NU5125 17 | 18 | $(NoWarn);NU5105 19 | 20 | https://github.com/rainingnight/zero-logging 21 | 22 | https://github.com/rainingnight/zero-logging 23 | git 24 | 25 | 26 | 27 | 28 | true 29 | 30 | 31 | -------------------------------------------------------------------------------- /Directory.Build.targets: -------------------------------------------------------------------------------- 1 |  2 | 3 | 2.1.3 4 | 2.2.0 5 | 2.0.3 6 | 7 | 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 RainingNight 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /NuGet.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /NuGetPackageVerifier.json: -------------------------------------------------------------------------------- 1 | { 2 | "Default": { 3 | "rules": [ 4 | "DefaultCompositeRule" 5 | ] 6 | } 7 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # zero-logging 2 | 3 | Zero logger provider for [Microsoft.Extensions.Logging](https://github.com/aspnet/Logging), the logging subsystem used by ASP.NET Core. 4 | 5 | ## Logging in Elasticsearch 6 | 7 | PLEASE Read [Zero.Logging.Elasticsearch](https://github.com/RainingNight/zero-logging/blob/dev/docs/elasticsearch.md). 8 | 9 | ## Logging in File 10 | 11 | ### Install 12 | 13 | **First**, install the _Zero.Logging.File_ [NuGet package](https://www.nuget.org/packages/Zero.Logging.File) into your app: 14 | 15 | ```powershell 16 | dotnet add package Zero.Logging.File --version 1.0.0-alpha3-20180228 17 | ``` 18 | 19 | ### Configure 20 | 21 | **Next**, add file section config in appsettings.json: 22 | 23 | ```json 24 | { 25 | "Logging": { 26 | "IncludeScopes": false, 27 | "Console": { 28 | "LogLevel": { 29 | "Default": "Warning" 30 | } 31 | }, 32 | "File": { 33 | "LogLevel": { 34 | "Default": "Error" 35 | }, 36 | "RollingInterval": "Minute" 37 | } 38 | } 39 | } 40 | ``` 41 | 42 | **Finally**, in your application's _Program.cs_ file, configure _Zeor.Logging.File_ first: 43 | 44 | ```csharp 45 | public static IWebHost BuildWebHost(string[] args) => 46 | WebHost.CreateDefaultBuilder(args) 47 | .ConfigureLogging((hostingContext, logging) => 48 | { 49 | logging.AddFile(); 50 | }) 51 | .UseStartup() 52 | .Build(); 53 | ``` 54 | 55 | ### Demonstrate 56 | 57 | Call logging methods on that logger object: 58 | 59 | ```csharp 60 | public class ValuesController : Controller 61 | { 62 | private readonly ILogger _logger; 63 | 64 | public ValuesController(ILogger logger) 65 | { 66 | _logger = logger; 67 | } 68 | 69 | [HttpGet] 70 | public void Get() 71 | { 72 | _logger.LogTrace("Log Trace."); 73 | _logger.LogInformation("Log Information."); 74 | _logger.LogDebug("Log Debug."); 75 | try 76 | { 77 | throw new Exception("Boom"); 78 | } 79 | catch (Exception ex) 80 | { 81 | _logger.LogCritical(1, ex, "Unexpected critical error starting application"); 82 | _logger.LogError(1, ex, "Unexpected error"); 83 | _logger.LogWarning(1, ex, "Unexpected warning"); 84 | } 85 | } 86 | } 87 | ``` 88 | 89 | That's it! With the level bumped up a little you will see log output like: 90 | 91 | ```text 92 | # logs/log-201802271502.txt 93 | 94 | 2018-02-27 15:02:40.608 +08:00 [Critical] WebApplication1.Controllers.ValuesController: Unexpected critical error starting application 95 | System.Exception: Boom 96 | at WebApplication1.Controllers.ValuesController.Get() in C:\Users\rainging\source\repos\WebApplication1\WebApplication1\Controllers\ValuesController.cs:line 28 97 | 2018-02-27 15:02:40.631 +08:00 [Error] WebApplication1.Controllers.ValuesController: Unexpected error 98 | System.Exception: Boom 99 | at WebApplication1.Controllers.ValuesController.Get() in C:\Users\rainging\source\repos\WebApplication1\WebApplication1\Controllers\ValuesController.cs:line 28 100 | ``` 101 | 102 | -------------------------------------------------------------------------------- /docs/.env: -------------------------------------------------------------------------------- 1 | TAG=6.2.2 2 | ELASTIC_VERSION=6.2.2 3 | ELASTIC_PASSWORD=Qwer1234 -------------------------------------------------------------------------------- /docs/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | elasticsearch: 4 | image: docker.elastic.co/elasticsearch/elasticsearch:${TAG} 5 | container_name: elasticsearch 6 | environment: 7 | - http.host=0.0.0.0 8 | - transport.host=127.0.0.1 9 | - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD} 10 | ports: 11 | - 9200:9200 12 | networks: 13 | - stack 14 | 15 | kibana: 16 | image: docker.elastic.co/kibana/kibana:${TAG} 17 | container_name: kibana 18 | environment: 19 | - ELASTICSEARCH_USERNAME=kibana 20 | - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD} 21 | ports: 22 | - 5601:5601 23 | networks: 24 | - stack 25 | depends_on: 26 | - elasticsearch 27 | 28 | networks: 29 | stack: 30 | driver: bridge 31 | -------------------------------------------------------------------------------- /docs/elasticsearch.md: -------------------------------------------------------------------------------- 1 | # Logging in Elasticsearch with Kibana 2 | 3 | ## 使用Docker部署Elasticsearch和Kibana 4 | 5 | ELKstack是Elasticsearch、Logstash、Kibana三个开源软件的组合，是当今最为流行的统一日志分析平台。对于它们的介绍，网上非常之多，这里就不再多说。 6 | 7 | 在本文中只使用了`Elasticsearch`和`Kibana`，前者是分布式搜索系统，后者是一个可视化平台，使用docker来部署非常简单： 8 | 9 | ### 部署Elasticsearch 10 | 11 | 如下，绑定端口`9200`，并将容器命名为`elasticsearch`： 12 | 13 | ```bash 14 | docker run --name=elasticsearch -d -p 9200:9200 -e "http.host=0.0.0.0" -e "transport.host=127.0.0.1" docker.elastic.co/elasticsearch/elasticsearch:6.2.2 15 | ``` 16 | 17 | 然后在浏览器中打开 [http://localhost:9200/](http://localhost:9200/)，输出如下： 18 | 19 | ```json 20 | { 21 | "name": "qFQvLqr", 22 | "cluster_name": "docker-cluster", 23 | "cluster_uuid": "bdc5YhZlQHu0mCN7acNKBw", 24 | "version": { 25 | "number": "6.2.2", 26 | "build_hash": "10b1edd", 27 | "build_date": "2018-02-16T21:01:30.685723Z", 28 | "build_snapshot": false, 29 | "lucene_version": "7.2.1", 30 | "minimum_wire_compatibility_version": "5.6.0", 31 | "minimum_index_compatibility_version": "5.0.0" 32 | }, 33 | "tagline": "You Know, for Search" 34 | } 35 | ``` 36 | 37 | ### 部署Kibana 38 | 39 | Kibana的部署依赖于Elasticsearch： 40 | 41 | ```bash 42 | docker run --name=kibana --link=elasticsearch -d -p 5601:5601 docker.elastic.co/kibana/kibana:6.2.2 43 | ``` 44 | 45 | 主要注意的是，在这里使用了`--link=elasticsearch`来链接到*elasticsearch*容器，如果要使用外部的elasticsearch服务，可以使用`-e "elasticsearch.url=http://changeme:9200"`来指定。 46 | 47 | 然后在浏览器中打开 [http://localhost:5601/](http://localhost:5601/)： 48 | 49 | ![kibana_empty](./img/kibana_empty.png) 50 | 51 | 如上，部署成功，不过还没有任何数据。 52 | 53 | ### 使用docker-compose部署 54 | 55 | 当需要部署多个相关的服务时，更加推荐使用**docker-compose**来部署： 56 | 57 | 首先，我们创建一个`docker-compose.yml`文件： 58 | 59 | ```yml 60 | version: '3' 61 | services: 62 | elasticsearch: 63 | image: docker.elastic.co/elasticsearch/elasticsearch:${TAG} 64 | container_name: elasticsearch 65 | environment: 66 | - http.host=0.0.0.0 67 | - transport.host=127.0.0.1 68 | - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD} 69 | ports: 70 | - 9200:9200 71 | networks: 72 | - stack 73 | 74 | kibana: 75 | image: docker.elastic.co/kibana/kibana:${TAG} 76 | container_name: kibana 77 | environment: 78 | - ELASTICSEARCH_USERNAME=kibana 79 | - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD} 80 | ports: 81 | - 5601:5601 82 | networks: 83 | - stack 84 | depends_on: 85 | - elasticsearch 86 | 87 | networks: 88 | stack: 89 | driver: bridge 90 | ``` 91 | 92 | 如上，我们定义了`TAG`和`ELASTIC_PASSWORD`两个环境变量，方便在部署时候灵活的指定版本号和密码。 93 | 94 | 为方便测试部署，我们可以定义一个默认的环境变量文件`.env`： 95 | 96 | ```env 97 | TAG=6.2.2 98 | ELASTIC_PASSWORD=Qwer1234 99 | ``` 100 | 101 | 然后，直接运行如下命令即可： 102 | 103 | ```bash 104 | docker-compose up 105 | ``` 106 | 107 | 接下来，将日志写入到Elasticsearch。 108 | 109 | ## 记录日志到Elasticsearch 110 | 111 | 我们创建一个 ASP.NET Core WebApi 项目，添加如下Package: 112 | 113 | ```bash 114 | dotnet add package Zero.Logging.Elasticsearch --version 1.0.0-alpha3-20180228 115 | ``` 116 | 117 | ### 添加ElasticsearchProvider 118 | 119 | 然后在`Program.cs`文件中使用`AddElasticsearch`扩展方法为日志系统添加`ElasticsearchProvider`： 120 | 121 | ```csharp 122 | public static IWebHost BuildWebHost(string[] args) => 123 | WebHost.CreateDefaultBuilder(args) 124 | .ConfigureLogging((hostingContext, logging) => 125 | { 126 | logging.AddElasticsearch(); 127 | }) 128 | .UseStartup() 129 | .Build(); 130 | ``` 131 | 132 | ### 记录日志 133 | 134 | 对于日志的记录则不需要任何的修改： 135 | 136 | ```csharp 137 | public class ValuesController : Controller 138 | { 139 | private readonly ILogger _logger; 140 | 141 | public ValuesController(ILogger logger) 142 | { 143 | _logger = logger; 144 | } 145 | 146 | [HttpGet] 147 | public void Get() 148 | { 149 | _logger.LogTrace("Log Trace."); 150 | _logger.LogInformation("Log Information."); 151 | _logger.LogDebug("Log Debug."); 152 | try 153 | { 154 | throw new Exception("Boom"); 155 | } 156 | catch (Exception ex) 157 | { 158 | _logger.LogCritical(1, ex, "Unexpected critical error starting application"); 159 | _logger.LogError(1, ex, "Unexpected error"); 160 | _logger.LogWarning(1, ex, "Unexpected warning"); 161 | } 162 | } 163 | } 164 | ``` 165 | 166 | ### 在Kibana查看 167 | 168 | 刷新浏览器，显示如下： 169 | 170 | ![kibana_initdata](./img/kibana_initdata.png) 171 | 172 | 在Index pattern中输入`logstash-*`，点击下一步： 173 | 174 | ![kibana_index](./img/kibana_index.png) 175 | 176 | 如上，选择`timestamp`，创建索引，最终显示如下： 177 | 178 | ![kinaba_discover](./img/kinaba_discover.png) 179 | 180 | ### 配置 181 | 182 | 如上一行代码，零配置完成Elasticsearch的写入，默认使用的Elasticsearch地址为`http://localhost:9200`，如果我们需要额外的配置也很简单，有如下两种方式： 183 | 184 | #### 使用配置文件进行配置 185 | 186 | 在`appsettings.json`中添加如下配置： 187 | 188 | ```json 189 | { 190 | "Logging": { 191 | "IncludeScopes": false, 192 | "Console": { 193 | "LogLevel": { 194 | "Default": "Warning" 195 | } 196 | }, 197 | "Elasticsearch": { 198 | "LogLevel": { 199 | "Default": "Information" 200 | }, 201 | "ElasticsearchUrl": "http://changeme:9200", 202 | "AutoRegisterTemplate": true 203 | } 204 | } 205 | } 206 | ``` 207 | 208 | #### 使用代码进行配置 209 | 210 | ```csharp 211 | WebHost.CreateDefaultBuilder(args) 212 | .ConfigureLogging((hostingContext, logging) => 213 | { 214 | logging.AddFile().AddElasticsearch(o => 215 | { 216 | o.PipelineName = "http://changeme:9200"; 217 | o.AutoRegisterTemplate = true; 218 | }); 219 | }) 220 | ``` 221 | 222 | > 需要注意，如果使用代码的方式进行配置，则配置文件中的配置不再生效。 223 | 224 | 更多的配置信息参见 [EsLoggerOptions](https://github.com/RainingNight/zero-logging/blob/dev/src/Zero.Logging.Elasticsearch/EsLoggerOptions.cs)。 -------------------------------------------------------------------------------- /docs/img/kibana_empty.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RainingNight/zero-logging/873be46fd023ec467cfb2d6bafa5c07c784c7269/docs/img/kibana_empty.png -------------------------------------------------------------------------------- /docs/img/kibana_index.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RainingNight/zero-logging/873be46fd023ec467cfb2d6bafa5c07c784c7269/docs/img/kibana_index.png -------------------------------------------------------------------------------- /docs/img/kibana_initdata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RainingNight/zero-logging/873be46fd023ec467cfb2d6bafa5c07c784c7269/docs/img/kibana_initdata.png -------------------------------------------------------------------------------- /docs/img/kinaba_discover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RainingNight/zero-logging/873be46fd023ec467cfb2d6bafa5c07c784c7269/docs/img/kinaba_discover.png -------------------------------------------------------------------------------- /samples/SampleApp/LoggerExtensions.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using System; 5 | using Microsoft.Extensions.Logging; 6 | 7 | namespace SampleApp 8 | { 9 | internal static class LoggerExtensions 10 | { 11 | private static Func _purchaseOrderScope; 12 | private static Action _programStarting; 13 | private static Action _programStopping; 14 | 15 | static LoggerExtensions() 16 | { 17 | _purchaseOrderScope = LoggerMessage.DefineScope("PO:{PurchaseOrder}"); 18 | _programStarting = LoggerMessage.Define(LogLevel.Information, 1, "Starting at '{StartTime}' and 0x{Hello:X} is hex of 42"); 19 | _programStopping = LoggerMessage.Define(LogLevel.Information, 2, "Stopping at '{StopTime}'"); 20 | } 21 | 22 | public static IDisposable PurchaseOrderScope(this ILogger logger, string purchaseOrder) 23 | { 24 | return _purchaseOrderScope(logger, purchaseOrder); 25 | } 26 | 27 | public static void ProgramStarting(this ILogger logger, DateTimeOffset startTime, int hello, Exception exception = null) 28 | { 29 | _programStarting(logger, startTime, hello, exception); 30 | } 31 | 32 | public static void ProgramStopping(this ILogger logger, DateTimeOffset stopTime, Exception exception = null) 33 | { 34 | _programStopping(logger, stopTime, exception); 35 | } 36 | } 37 | } 38 | 39 | -------------------------------------------------------------------------------- /samples/SampleApp/Program.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) .NET Foundation. All rights reserved. 2 | // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. 3 | 4 | using System; 5 | using System.IO; 6 | using Microsoft.Extensions.Configuration; 7 | using Microsoft.Extensions.DependencyInjection; 8 | using Microsoft.Extensions.Logging; 9 | using ILogger = Microsoft.Extensions.Logging.ILogger; 10 | 11 | namespace SampleApp 12 | { 13 | public class Program 14 | { 15 | private readonly ILogger _logger; 16 | 17 | public Program() 18 | { 19 | var loggingConfiguration = new ConfigurationBuilder() 20 | .SetBasePath(Directory.GetCurrentDirectory()) 21 | .AddJsonFile("logging.json", optional: false, reloadOnChange: true) 22 | .Build(); 23 | 24 | // A Web App based program would configure logging via the WebHostBuilder. 25 | // Create a logger factory with filters that can be applied across all logger providers. 26 | var serviceCollection = new ServiceCollection() 27 | .AddLogging(builder => 28 | { 29 | builder 30 | .AddConfiguration(loggingConfiguration.GetSection("Logging")) 31 | .AddFilter("Microsoft", LogLevel.Warning) 32 | .AddFilter("System", LogLevel.Warning) 33 | .AddFilter("SampleApp.Program", LogLevel.Debug) 34 | //.AddConsole() 35 | .AddFile() 36 | .AddElasticsearch(); 37 | }); 38 | 39 | // providers may be added to a LoggerFactory before any loggers are created 40 | 41 | 42 | var serviceProvider = serviceCollection.BuildServiceProvider(); 43 | // getting the logger using the class's name is conventional 44 | _logger = serviceProvider.GetRequiredService>(); 45 | } 46 | 47 | public static void Main(string[] args) 48 | { 49 | new Program().Execute(args); 50 | } 51 | 52 | public void Execute(string[] args) 53 | { 54 | _logger.LogDebug("Begin logging...."); 55 | 56 | _logger.LogInformation("Starting"); 57 | 58 | var startTime = DateTimeOffset.Now; 59 | _logger.LogInformation(1, "Started at '{StartTime}' and 0x{Hello:X} is hex of 42", startTime, 42); 60 | // or 61 | _logger.ProgramStarting(startTime, 42); 62 | 63 | using (_logger.PurchaseOrderScope("00655321")) 64 | { 65 | try 66 | { 67 | throw new Exception("Boom"); 68 | } 69 | catch (Exception ex) 70 | { 71 | _logger.LogCritical(1, ex, "Unexpected critical error starting application"); 72 | _logger.LogError(1, ex, "Unexpected error"); 73 | _logger.LogWarning(1, ex, "Unexpected warning"); 74 | } 75 | 76 | using (_logger.BeginScope("Main")) 77 | { 78 | 79 | _logger.LogInformation("Waiting for user input"); 80 | 81 | string input; 82 | do 83 | { 84 | Console.WriteLine("Enter some test to log more, or 'quit' to exit."); 85 | input = Console.ReadLine(); 86 | 87 | _logger.LogInformation("User typed '{input}' on the command line", input); 88 | _logger.LogWarning("The time is now {Time}, it's getting late!", DateTimeOffset.Now); 89 | } 90 | while (input != "quit"); 91 | } 92 | } 93 | 94 | var endTime = DateTimeOffset.Now; 95 | _logger.LogInformation(2, "Stopping at '{StopTime}'", endTime); 96 | // or 97 | _logger.ProgramStopping(endTime); 98 | 99 | _logger.LogInformation("Stopping"); 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /samples/SampleApp/SampleApp.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp2.2 5 | Exe 6 | 7 | 8 | 9 | 10 | PreserveNewest 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /samples/SampleApp/logging.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Debug", 5 | "System": "Information", 6 | "Microsoft": "Information" 7 | }, 8 | "Console": { 9 | "IncludeScopes": "true" 10 | }, 11 | "File": { 12 | // 按分钟滚动写入 13 | "RollingInterval": "Minute", 14 | "LogLevel": { 15 | "Default": "Information" 16 | } 17 | }, 18 | "Elasticsearch": { 19 | "ElasticsearchUrl": "http://localhost:9200", 20 | "UserName": "", 21 | "Password": "" 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /src/Zero.Logging.Batching/BatchLoggerConfigureOptions.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Configuration; 2 | using Microsoft.Extensions.Options; 3 | 4 | namespace Zero.Logging.Batching 5 | { 6 | public class BatchLoggerConfigureOptions : IConfigureOptions 7 | { 8 | private readonly IConfiguration _configuration; 9 | private readonly string _isEnabledKey; 10 | 11 | public BatchLoggerConfigureOptions(IConfiguration configuration, string isEnabledKey) 12 | { 13 | _configuration = configuration; 14 | _isEnabledKey = isEnabledKey; 15 | } 16 | 17 | public void Configure(BatchingLoggerOptions options) 18 | { 19 | options.IsEnabled = TextToBoolean(_configuration.GetSection(_isEnabledKey)?.Value); 20 | } 21 | 22 | private static bool TextToBoolean(string text) 23 | { 24 | if (string.IsNullOrEmpty(text) || 25 | !bool.TryParse(text, out var result)) 26 | { 27 | result = false; 28 | } 29 | 30 | return result; 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/Zero.Logging.Batching/BatchingLogger.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Text; 3 | using Microsoft.Extensions.Logging; 4 | 5 | namespace Zero.Logging.Batching 6 | { 7 | public class BatchingLogger : ILogger 8 | { 9 | private readonly BatchingLoggerProvider _provider; 10 | private readonly string _category; 11 | 12 | public BatchingLogger(BatchingLoggerProvider loggerProvider, string categoryName) 13 | { 14 | _provider = loggerProvider; 15 | _category = categoryName; 16 | } 17 | 18 | public IDisposable BeginScope(TState state) 19 | { 20 | return null; 21 | } 22 | 23 | public bool IsEnabled(LogLevel logLevel) 24 | { 25 | return _provider.IsEnabled; 26 | } 27 | 28 | public void Log(DateTimeOffset timestamp, LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) 29 | { 30 | if (!IsEnabled(logLevel)) 31 | { 32 | return; 33 | } 34 | 35 | var builder = new StringBuilder(); 36 | builder.Append(timestamp.ToString("yyyy-MM-dd HH:mm:ss.fff zzz")); 37 | builder.Append(" ["); 38 | builder.Append(logLevel.ToString()); 39 | builder.Append("] "); 40 | builder.Append(_category); 41 | builder.Append(": "); 42 | builder.AppendLine(formatter(state, exception)); 43 | 44 | if (exception != null) 45 | { 46 | builder.AppendLine(exception.ToString()); 47 | } 48 | 49 | _provider.AddMessage(timestamp, builder.ToString()); 50 | } 51 | 52 | public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) 53 | { 54 | Log(DateTimeOffset.Now, logLevel, eventId, state, exception, formatter); 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/Zero.Logging.Batching/BatchingLoggerOptions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Zero.Logging.Batching 4 | { 5 | public class BatchingLoggerOptions 6 | { 7 | private int? _batchSize = 32; 8 | private int? _backgroundQueueSize = 1000; 9 | private TimeSpan _flushPeriod = TimeSpan.FromSeconds(1); 10 | 11 | /// 12 | /// Gets or sets the period after which logs will be flushed to the store. 13 | /// 14 | public TimeSpan FlushPeriod 15 | { 16 | get { return _flushPeriod; } 17 | set 18 | { 19 | if (value <= TimeSpan.Zero) 20 | { 21 | throw new ArgumentOutOfRangeException(nameof(value), $"{nameof(FlushPeriod)} must be positive."); 22 | } 23 | _flushPeriod = value; 24 | } 25 | } 26 | 27 | /// 28 | /// Gets or sets the maximum size of the background log message queue or null for no limit. 29 | /// After maximum queue size is reached log event sink would start blocking. 30 | /// Defaults to 1000. 31 | /// 32 | public int? BackgroundQueueSize 33 | { 34 | get { return _backgroundQueueSize; } 35 | set 36 | { 37 | if (value < 0) 38 | { 39 | throw new ArgumentOutOfRangeException(nameof(value), $"{nameof(BackgroundQueueSize)} must be non-negative."); 40 | } 41 | _backgroundQueueSize = value; 42 | } 43 | } 44 | 45 | /// 46 | /// Gets or sets a maximum number of events to include in a single batch or null for no limit. 47 | /// 48 | public int? BatchSize 49 | { 50 | get { return _batchSize; } 51 | set 52 | { 53 | if (value <= 0) 54 | { 55 | throw new ArgumentOutOfRangeException(nameof(value), $"{nameof(BatchSize)} must be positive."); 56 | } 57 | _batchSize = value; 58 | } 59 | } 60 | 61 | /// 62 | /// Gets or sets value indicating if logger accepts and queues writes. 63 | /// 64 | public bool IsEnabled { get; set; } = true; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/Zero.Logging.Batching/BatchingLoggerProvider.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | using Microsoft.Extensions.Logging; 7 | using Microsoft.Extensions.Options; 8 | 9 | namespace Zero.Logging.Batching 10 | { 11 | public abstract class BatchingLoggerProvider : ILoggerProvider 12 | { 13 | private readonly List _currentBatch = new List(); 14 | private readonly TimeSpan _interval; 15 | private readonly int? _queueSize; 16 | private readonly int? _batchSize; 17 | private readonly IDisposable _optionsChangeToken; 18 | 19 | private BlockingCollection _messageQueue; 20 | private Task _outputTask; 21 | private CancellationTokenSource _cancellationTokenSource; 22 | 23 | protected BatchingLoggerProvider(IOptionsMonitor options) 24 | { 25 | // NOTE: Only IsEnabled is monitored 26 | 27 | var loggerOptions = options.CurrentValue; 28 | if (loggerOptions.BatchSize <= 0) 29 | { 30 | throw new ArgumentOutOfRangeException(nameof(loggerOptions.BatchSize), $"{nameof(loggerOptions.BatchSize)} must be a positive number."); 31 | } 32 | if (loggerOptions.FlushPeriod <= TimeSpan.Zero) 33 | { 34 | throw new ArgumentOutOfRangeException(nameof(loggerOptions.FlushPeriod), $"{nameof(loggerOptions.FlushPeriod)} must be longer than zero."); 35 | } 36 | 37 | _interval = loggerOptions.FlushPeriod; 38 | _batchSize = loggerOptions.BatchSize; 39 | _queueSize = loggerOptions.BackgroundQueueSize; 40 | 41 | _optionsChangeToken = options.OnChange(UpdateOptions); 42 | UpdateOptions(options.CurrentValue); 43 | } 44 | 45 | public bool IsEnabled { get; private set; } 46 | 47 | private void UpdateOptions(BatchingLoggerOptions options) 48 | { 49 | var oldIsEnabled = IsEnabled; 50 | IsEnabled = options.IsEnabled; 51 | if (oldIsEnabled != IsEnabled) 52 | { 53 | if (IsEnabled) 54 | { 55 | Start(); 56 | } 57 | else 58 | { 59 | Stop(); 60 | } 61 | } 62 | 63 | } 64 | 65 | protected abstract Task WriteMessagesAsync(IEnumerable messages, CancellationToken token); 66 | 67 | private async Task ProcessLogQueue(object state) 68 | { 69 | while (!_cancellationTokenSource.IsCancellationRequested) 70 | { 71 | var limit = _batchSize ?? int.MaxValue; 72 | 73 | while (limit > 0 && _messageQueue.TryTake(out var message)) 74 | { 75 | _currentBatch.Add(message); 76 | limit--; 77 | } 78 | 79 | if (_currentBatch.Count > 0) 80 | { 81 | try 82 | { 83 | await WriteMessagesAsync(_currentBatch, _cancellationTokenSource.Token); 84 | } 85 | catch 86 | { 87 | // ignored 88 | } 89 | 90 | _currentBatch.Clear(); 91 | } 92 | 93 | await IntervalAsync(_interval, _cancellationTokenSource.Token); 94 | } 95 | } 96 | 97 | protected virtual Task IntervalAsync(TimeSpan interval, CancellationToken cancellationToken) 98 | { 99 | return Task.Delay(interval, cancellationToken); 100 | } 101 | 102 | public void AddMessage(DateTimeOffset timestamp, string message) 103 | { 104 | if (!_messageQueue.IsAddingCompleted) 105 | { 106 | try 107 | { 108 | _messageQueue.Add(new LogMessage { Message = message, Timestamp = timestamp }, _cancellationTokenSource.Token); 109 | } 110 | catch 111 | { 112 | //cancellation token canceled or CompleteAdding called 113 | } 114 | } 115 | } 116 | 117 | private void Start() 118 | { 119 | _messageQueue = _queueSize == null ? 120 | new BlockingCollection(new ConcurrentQueue()) : 121 | new BlockingCollection(new ConcurrentQueue(), _queueSize.Value); 122 | 123 | _cancellationTokenSource = new CancellationTokenSource(); 124 | _outputTask = Task.Factory.StartNew(ProcessLogQueue, null, TaskCreationOptions.LongRunning); 125 | } 126 | 127 | private void Stop() 128 | { 129 | _cancellationTokenSource.Cancel(); 130 | _messageQueue.CompleteAdding(); 131 | 132 | try 133 | { 134 | _outputTask.Wait(_interval); 135 | } 136 | catch (TaskCanceledException) 137 | { 138 | } 139 | catch (AggregateException ex) when (ex.InnerExceptions.Count == 1 && ex.InnerExceptions[0] is TaskCanceledException) 140 | { 141 | } 142 | } 143 | 144 | public void Dispose() 145 | { 146 | _optionsChangeToken?.Dispose(); 147 | if (IsEnabled) 148 | { 149 | Stop(); 150 | } 151 | } 152 | 153 | public virtual ILogger CreateLogger(string categoryName) 154 | { 155 | return new BatchingLogger(this, categoryName); 156 | } 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /src/Zero.Logging.Batching/LogMessage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Zero.Logging.Batching 4 | { 5 | public struct LogMessage 6 | { 7 | public DateTimeOffset Timestamp { get; set; } 8 | 9 | public string Message { get; set; } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/Zero.Logging.Batching/Zero.Logging.Batching.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netstandard2.0 5 | Zero file logger common lib. 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /src/Zero.Logging.Elasticsearch/ElasticsearchHelper.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text.RegularExpressions; 5 | using Elasticsearch.Net; 6 | using Zero.Logging.Batching; 7 | 8 | namespace Zero.Logging.Elasticsearch 9 | { 10 | internal class ElasticsearchHelper 11 | { 12 | private readonly ElasticLowLevelClient _client; 13 | 14 | private readonly Func _indexDecider; 15 | private readonly bool _registerTemplateOnStartup; 16 | private readonly string _templateName; 17 | private readonly string _templateMatchString; 18 | 19 | private static readonly Regex _indexFormatRegex = new Regex(@"^(.*)(?:\{0\:.+\})(.*)$"); 20 | 21 | public static ElasticsearchHelper Create(EsLoggerOptions options) 22 | { 23 | if (options == null) 24 | throw new ArgumentNullException(nameof(options)); 25 | return new ElasticsearchHelper(options); 26 | } 27 | 28 | private ElasticsearchHelper(EsLoggerOptions options) 29 | { 30 | if (string.IsNullOrWhiteSpace(options.ElasticsearchUrl)) throw new ArgumentException("options.ElasticsearchUrl"); 31 | if (string.IsNullOrWhiteSpace(options.IndexFormat)) throw new ArgumentException("options.IndexFormat"); 32 | if (string.IsNullOrWhiteSpace(options.TypeName)) throw new ArgumentException("options.TypeName"); 33 | if (string.IsNullOrWhiteSpace(options.TemplateName)) throw new ArgumentException("options.TemplateName"); 34 | 35 | _templateName = options.TemplateName; 36 | _templateMatchString = _indexFormatRegex.Replace(options.IndexFormat, @"$1*$2"); 37 | _indexDecider = options.IndexDecider ?? (logMsg => string.Format(options.IndexFormat, logMsg.Timestamp)); 38 | 39 | Options = options; 40 | 41 | IConnectionPool pool; 42 | if (options.ElasticsearchUrl.Contains(";")) 43 | { 44 | pool = new StaticConnectionPool(options.ElasticsearchUrl.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries).Select(_ => new Uri(_))); 45 | } 46 | else 47 | { 48 | pool = new SingleNodeConnectionPool(new Uri(options.ElasticsearchUrl)); 49 | } 50 | 51 | var configuration = new ConnectionConfiguration(pool, options.Connection, options.Serializer).RequestTimeout(options.ConnectionTimeout); 52 | 53 | if (!string.IsNullOrEmpty(options.UserName) && !string.IsNullOrEmpty(options.Password)) 54 | { 55 | configuration.BasicAuthentication(options.UserName, options.Password); 56 | } 57 | 58 | if (options.ModifyConnectionSettings != null) configuration = options.ModifyConnectionSettings(configuration); 59 | 60 | configuration.ThrowExceptions(); 61 | 62 | _client = new ElasticLowLevelClient(configuration); 63 | 64 | _registerTemplateOnStartup = options.AutoRegisterTemplate; 65 | TemplateRegistrationSuccess = !_registerTemplateOnStartup; 66 | } 67 | 68 | public EsLoggerOptions Options { get; } 69 | 70 | public IElasticLowLevelClient Client => _client; 71 | 72 | public bool TemplateRegistrationSuccess { get; private set; } 73 | 74 | 75 | public string Serialize(object o) 76 | { 77 | return _client.Serializer.SerializeToString(o, SerializationFormatting.None); 78 | } 79 | 80 | public string GetIndexForEvent(LogMessage e, DateTimeOffset offset) 81 | { 82 | if (!TemplateRegistrationSuccess && Options.RegisterTemplateFailure == RegisterTemplateRecovery.IndexToDeadletterIndex) 83 | { 84 | return string.Format(Options.DeadLetterIndexName, offset); 85 | } 86 | return _indexDecider(e); 87 | } 88 | 89 | public void RegisterTemplateIfNeeded() 90 | { 91 | if (!_registerTemplateOnStartup) return; 92 | 93 | try 94 | { 95 | if (!Options.OverwriteTemplate) 96 | { 97 | var templateExistsResponse = _client.IndicesExistsTemplateForAll(_templateName); 98 | if (templateExistsResponse.HttpStatusCode == 200) 99 | { 100 | TemplateRegistrationSuccess = true; 101 | 102 | return; 103 | } 104 | } 105 | 106 | var result = _client.IndicesPutTemplateForAll(_templateName, GetTempatePostData()); 107 | 108 | if (!result.Success) 109 | { 110 | ((IElasticsearchResponse)result).TryGetServerErrorReason(out var serverError); 111 | Console.WriteLine("Unable to create the template. {0}", serverError); 112 | TemplateRegistrationSuccess = false; 113 | } 114 | else 115 | TemplateRegistrationSuccess = true; 116 | 117 | } 118 | catch (Exception ex) 119 | { 120 | TemplateRegistrationSuccess = false; 121 | Console.WriteLine("Failed to create the template. {0}", ex); 122 | 123 | if (Options.RegisterTemplateFailure == RegisterTemplateRecovery.Throw) 124 | throw; 125 | } 126 | } 127 | 128 | private PostData GetTempatePostData() 129 | { 130 | //PostData no longer exposes an implict cast from object. Previously it supported that and would inspect the object Type to 131 | //determine if it it was a litteral string to write directly or if it was an object that it needed to serialse. Now the onus is 132 | //on us to tell it what type we are passing otherwise if the user specified the template as a json string it would be serialised again. 133 | var template = GetTemplateData(); 134 | if (template is string) 135 | { 136 | return PostData.String((string)template); 137 | } 138 | else 139 | { 140 | return PostData.Serializable(template); 141 | } 142 | } 143 | 144 | private object GetTemplateData() 145 | { 146 | if (Options.GetTemplateContent != null) 147 | return Options.GetTemplateContent(); 148 | 149 | var settings = new Dictionary 150 | { 151 | {"index.refresh_interval", "5s"} 152 | }; 153 | 154 | if (Options.NumberOfShards.HasValue) 155 | settings.Add("number_of_shards", Options.NumberOfShards.Value.ToString()); 156 | 157 | if (Options.NumberOfReplicas.HasValue) 158 | settings.Add("number_of_replicas", Options.NumberOfReplicas.Value.ToString()); 159 | 160 | return GetTemplateESv6(settings, _templateMatchString); 161 | 162 | } 163 | 164 | private static object GetTemplateESv6(Dictionary settings, string templateMatchString) 165 | { 166 | return new 167 | { 168 | template = templateMatchString, 169 | settings, 170 | mappings = new 171 | { 172 | _default_ = new 173 | { 174 | dynamic_templates = new List 175 | { 176 | //when you use serilog as an adaptor for third party frameworks 177 | //where you have no control over the log message they typically 178 | //contain {0} ad infinitum, we force numeric property names to 179 | //contain strings by default. 180 | { 181 | new 182 | { 183 | numerics_in_fields = new 184 | { 185 | path_match = @"fields\.[\d+]$", 186 | match_pattern = "regex", 187 | mapping = new 188 | { 189 | type = "text", 190 | index = true, 191 | norms = false 192 | } 193 | } 194 | } 195 | }, 196 | { 197 | new 198 | { 199 | string_fields = new 200 | { 201 | match = "*", 202 | match_mapping_type = "string", 203 | mapping = new 204 | { 205 | type = "text", 206 | index = true, 207 | norms = false, 208 | fields = new 209 | { 210 | raw = new 211 | { 212 | type = "keyword", 213 | index = true, 214 | ignore_above = 256 215 | } 216 | } 217 | } 218 | } 219 | } 220 | } 221 | }, 222 | properties = new Dictionary 223 | { 224 | {"message", new {type = "text", index = "true"}}, 225 | { 226 | "exceptions", new 227 | { 228 | type = "nested", 229 | properties = new Dictionary 230 | { 231 | {"Depth", new {type = "integer"}}, 232 | {"RemoteStackIndex", new {type = "integer"}}, 233 | {"HResult", new {type = "integer"}}, 234 | {"StackTraceString", new {type = "text", index = "true"}}, 235 | {"RemoteStackTraceString", new {type = "text", index = "true"}}, 236 | { 237 | "ExceptionMessage", new 238 | { 239 | type = "object", 240 | properties = new Dictionary 241 | { 242 | {"MemberType", new {type = "integer"}}, 243 | } 244 | } 245 | } 246 | } 247 | } 248 | } 249 | } 250 | } 251 | } 252 | }; 253 | } 254 | 255 | } 256 | } 257 | -------------------------------------------------------------------------------- /src/Zero.Logging.Elasticsearch/EsLogger.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using Microsoft.Extensions.Logging; 4 | using Newtonsoft.Json; 5 | using Zero.Logging.Batching; 6 | 7 | namespace Zero.Logging.Elasticsearch 8 | { 9 | public class EsLogger : ILogger 10 | { 11 | private readonly BatchingLoggerProvider _provider; 12 | private readonly string _category; 13 | 14 | public EsLogger(BatchingLoggerProvider loggerProvider, string categoryName) 15 | { 16 | _provider = loggerProvider; 17 | _category = categoryName; 18 | } 19 | 20 | public IDisposable BeginScope(TState state) 21 | { 22 | return null; 23 | } 24 | 25 | public bool IsEnabled(LogLevel logLevel) 26 | { 27 | return _provider.IsEnabled; 28 | } 29 | 30 | public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) 31 | { 32 | Log(DateTimeOffset.Now, logLevel, eventId, state, exception, formatter); 33 | } 34 | 35 | public void Log(DateTimeOffset timestamp, LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) 36 | { 37 | var jsonData = new { timestamp, level = logLevel.ToString(), category = _category, message = formatter(state, exception), exceptions = new List() }; 38 | if (exception != null) 39 | { 40 | WriteSingleException(jsonData.exceptions, exception, 0); 41 | } 42 | _provider.AddMessage(timestamp, JsonConvert.SerializeObject(jsonData)); 43 | } 44 | 45 | private void WriteException(List exceptionList, Exception exception, int depth) 46 | { 47 | WriteSingleException(exceptionList, exception, depth); 48 | if (exception.InnerException != null && depth < 20) 49 | WriteException(exceptionList, exception.InnerException, ++depth); 50 | } 51 | 52 | private void WriteSingleException(dynamic exceptionList, Exception exception, int depth) 53 | { 54 | exceptionList.Add(new ExceptionModel 55 | { 56 | depth = depth, 57 | message = exception.Message, 58 | source = exception.Source, 59 | stackTrace = exception.StackTrace, 60 | hResult = exception.HResult, 61 | helpLink = exception.HelpLink 62 | }); 63 | } 64 | 65 | internal class ExceptionModel 66 | { 67 | public int depth { get; set; } 68 | public string message { get; set; } 69 | public string source { get; set; } 70 | 71 | [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] 72 | public string stackTrace { get; set; } 73 | 74 | [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] 75 | public int hResult { get; set; } 76 | 77 | [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] 78 | public string helpLink { get; set; } 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/Zero.Logging.Elasticsearch/EsLoggerFactoryExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.Extensions.Configuration; 3 | using Microsoft.Extensions.DependencyInjection; 4 | using Microsoft.Extensions.DependencyInjection.Extensions; 5 | using Microsoft.Extensions.Logging.Configuration; 6 | using Microsoft.Extensions.Options; 7 | using Zero.Logging.Elasticsearch; 8 | 9 | namespace Microsoft.Extensions.Logging 10 | { 11 | public static class EsLoggerFactoryExtensions 12 | { 13 | /// 14 | /// Adds a file logger named 'Elasticsearch' to the factory. 15 | /// 16 | /// The to use. 17 | public static ILoggingBuilder AddElasticsearch(this ILoggingBuilder builder) 18 | { 19 | builder.AddConfiguration(); 20 | 21 | builder.Services.TryAddEnumerable(ServiceDescriptor.Singleton()); 22 | builder.Services.AddSingleton, EsLoggerOptionsSetup>(); 23 | builder.Services.AddSingleton, LoggerProviderOptionsChangeTokenSource>(); 24 | 25 | //builder.Services.TryAddEnumerable(ServiceDescriptor.Singleton, EsLoggerOptionsSetup>()); 26 | //builder.Services.TryAddEnumerable(ServiceDescriptor.Singleton, LoggerProviderOptionsChangeTokenSource>()); 27 | return builder; 28 | } 29 | 30 | 31 | /// 32 | /// Adds a file logger named 'File' to the factory. 33 | /// 34 | /// The to use. 35 | /// 36 | public static ILoggingBuilder AddElasticsearch(this ILoggingBuilder builder, Action configure) 37 | { 38 | if (configure == null) 39 | { 40 | throw new ArgumentNullException(nameof(configure)); 41 | } 42 | 43 | builder.AddElasticsearch(); 44 | builder.Services.Configure(configure); 45 | return builder; 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/Zero.Logging.Elasticsearch/EsLoggerOptions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Elasticsearch.Net; 3 | using Zero.Logging.Batching; 4 | 5 | namespace Zero.Logging.Elasticsearch 6 | { 7 | public class EsLoggerOptions : BatchingLoggerOptions 8 | { 9 | public const string DefaultNode = "http://localhost:9200"; 10 | public const string DefaultIndexFormat = "logstash-{0:yyyy.MM.dd}"; 11 | public const string DefaultDeadLetterIndexName = "deadletter-{0:yyyy.MM.dd}"; 12 | public const string DefaultTypeName = "logmessage"; 13 | public const string DefaultTemplateName = "zero-logging-template"; 14 | public static readonly TimeSpan DefaultConnectionTimeout = TimeSpan.FromSeconds(60); 15 | 16 | /// 17 | /// Configures the elasticsearch log defaults. 18 | /// 19 | public EsLoggerOptions() 20 | { 21 | IndexFormat = DefaultIndexFormat; 22 | DeadLetterIndexName = DefaultDeadLetterIndexName; 23 | TypeName = DefaultTypeName; 24 | TemplateName = DefaultTemplateName; 25 | EmitEventFailure = EmitEventFailureHandling.WriteToConsole; 26 | RegisterTemplateFailure = RegisterTemplateRecovery.IndexAnyway; 27 | 28 | ElasticsearchUrl = "http://localhost:9200"; 29 | ConnectionTimeout = DefaultConnectionTimeout; 30 | } 31 | 32 | #region Template 33 | /// 34 | /// Auto register an index template for the logs in elasticsearch. 35 | /// 36 | public bool AutoRegisterTemplate { get; set; } 37 | 38 | /// 39 | /// Specifies the option on how to handle failures when writing the template to Elasticsearch. This is only applicable when using the AutoRegisterTemplate option. 40 | /// 41 | public RegisterTemplateRecovery RegisterTemplateFailure { get; set; } 42 | 43 | /// 44 | /// When using the feature this allows you to override the default template name. 45 | /// Defaults to: zero-logging-template 46 | /// 47 | public string TemplateName { get; set; } 48 | 49 | /// 50 | /// When using the feature, this allows you to override the default template content. 51 | /// 52 | public Func GetTemplateContent { get; set; } 53 | 54 | /// 55 | /// When using the feature, this allows you to overwrite the template in Elasticsearch if it already exists. 56 | /// Defaults to: false 57 | /// 58 | public bool OverwriteTemplate { get; set; } 59 | 60 | /// 61 | /// When using the feature, this allows you to override the default number of shards. 62 | /// If not provided, this will default to the default number_of_shards configured in Elasticsearch. 63 | /// 64 | public int? NumberOfShards { get; set; } 65 | 66 | /// 67 | /// When using the feature, this allows you to override the default number of replicas. 68 | /// If not provided, this will default to the default number_of_replicas configured in Elasticsearch. 69 | /// 70 | public int? NumberOfReplicas { get; set; } 71 | #endregion 72 | 73 | /// 74 | /// The index name formatter. A string.Format using the DateTimeOffset of the event is run over this string. 75 | /// defaults to "logstash-{0:yyyy.MM.dd}". 76 | /// 77 | public string IndexFormat { get; set; } 78 | 79 | /// 80 | /// Function to decide which index to write the log. 81 | /// 82 | public Func IndexDecider { get; set; } 83 | 84 | /// 85 | /// Optionally set this value to the name of the index that should be used when the template cannot be written to ES. 86 | /// defaults to "deadletter-{0:yyyy.MM.dd}" 87 | /// 88 | public string DeadLetterIndexName { get; set; } 89 | 90 | /// 91 | /// Name the Pipeline where log are sent to es. 92 | /// 93 | public string PipelineName { get; set; } 94 | 95 | /// 96 | /// Function to decide which Pipeline to use for the LogMessage. 97 | /// 98 | public Func PipelineNameDecider { get; set; } 99 | 100 | /// 101 | /// The default elasticsearch type name to use for the log message. Defaults to: logmessage. 102 | /// 103 | public string TypeName { get; set; } 104 | 105 | /// 106 | /// Connection configuration to use for connecting to the cluster. 107 | /// 108 | public Func ModifyConnectionSettings { get; set; } 109 | 110 | /// 111 | /// Allows you to override the elasticsearch url used to communicate with elasticsearch. 112 | /// 113 | public string ElasticsearchUrl { get; set; } 114 | 115 | /// 116 | /// The user name of the elasticsearch. 117 | /// 118 | public string UserName { get; set; } 119 | 120 | /// 121 | /// The password of the elasticsearch. 122 | /// 123 | public string Password { get; set; } 124 | 125 | /// 126 | /// Allows you to override the connection used to communicate with elasticsearch. 127 | /// 128 | public IConnection Connection { get; set; } 129 | 130 | /// 131 | /// The connection timeout (in milliseconds) when sending bulk operations to elasticsearch (defaults to 5000). 132 | /// 133 | public TimeSpan ConnectionTimeout { get; set; } 134 | 135 | /// 136 | /// When passing a serializer unknown object will be serialized to object instead of relying on their ToString representation 137 | /// 138 | public IElasticsearchSerializer Serializer { get; set; } 139 | 140 | /// 141 | /// Specifies how failing emits should be handled. 142 | /// 143 | public EmitEventFailureHandling EmitEventFailure { get; set; } 144 | 145 | /// 146 | /// A callback which can be used to handle logmessage which are not submitted to Elasticsearch like when it is unable to accept the events. 147 | /// 148 | public Action FailureCallback { get; set; } 149 | } 150 | 151 | /// 152 | /// Specifies what to do when the template could not be created. This can mean that your data is not correctly indexed, so you might want to handle this failure. 153 | /// 154 | public enum RegisterTemplateRecovery 155 | { 156 | /// 157 | /// Ignore the issue and keep indexing. This is the default option. 158 | /// 159 | IndexAnyway = 1, 160 | 161 | ///// 162 | ///// Keep buffering the data until it is written. be aware you might hit a limit here. 163 | ///// 164 | //BufferUntilSuccess = 2, 165 | 166 | /// 167 | /// When the template cannot be registered, move the events to the deadletter index instead. 168 | /// 169 | IndexToDeadletterIndex = 4, 170 | 171 | /// 172 | /// When the template cannot be registered, throw an exception. 173 | /// 174 | Throw = 8 175 | } 176 | 177 | /// 178 | /// Sepecifies options for handling failures when emitting the events to Elasticsearch. Can be a combination of options. 179 | /// 180 | [Flags] 181 | public enum EmitEventFailureHandling 182 | { 183 | /// 184 | /// Send the error to the Console. 185 | /// 186 | WriteToConsole = 1, 187 | 188 | /// 189 | /// Throw the exception to the caller. 190 | /// 191 | ThrowException = 2, 192 | 193 | /// 194 | /// The failure callback function will be called when the event cannot be submitted to Elasticsearch. 195 | /// 196 | RaiseCallback = 4 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /src/Zero.Logging.Elasticsearch/EsLoggerOptionsSetup.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Logging.Configuration; 2 | using Microsoft.Extensions.Options; 3 | 4 | namespace Zero.Logging.Elasticsearch 5 | { 6 | public class EsLoggerOptionsSetup : ConfigureFromConfigurationOptions 7 | { 8 | public EsLoggerOptionsSetup(ILoggerProviderConfiguration providerConfiguration) 9 | : base(providerConfiguration.Configuration) 10 | { 11 | 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Zero.Logging.Elasticsearch/EsLoggerProvider.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | using Elasticsearch.Net; 7 | using Microsoft.Extensions.Logging; 8 | using Microsoft.Extensions.Options; 9 | using Zero.Logging.Batching; 10 | 11 | namespace Zero.Logging.Elasticsearch 12 | { 13 | [ProviderAlias("Elasticsearch")] 14 | public class EsLoggerProvider : BatchingLoggerProvider 15 | { 16 | private readonly ElasticsearchHelper _esHelper; 17 | 18 | public EsLoggerProvider(IOptionsMonitor options) : base(options) 19 | { 20 | var loggerOptions = options.CurrentValue; 21 | if (!string.IsNullOrWhiteSpace(loggerOptions.TemplateName)) 22 | { 23 | loggerOptions.AutoRegisterTemplate = true; 24 | } 25 | _esHelper = ElasticsearchHelper.Create(loggerOptions); 26 | _esHelper.RegisterTemplateIfNeeded(); 27 | } 28 | 29 | public override ILogger CreateLogger(string categoryName) 30 | { 31 | return new EsLogger(this, categoryName); 32 | } 33 | 34 | protected override async Task WriteMessagesAsync(IEnumerable messages, CancellationToken cancellationToken) 35 | { 36 | DynamicResponse result; 37 | try 38 | { 39 | result = await PostToEsAsync(messages); 40 | } 41 | catch (Exception ex) 42 | { 43 | HandleException(ex, messages); 44 | return; 45 | } 46 | 47 | // Handle the results from ES, check if there are any errors. 48 | if (result.Success && result.Body?["errors"] == true) 49 | { 50 | var indexer = 0; 51 | var items = result.Body["items"]; 52 | foreach (var item in items) 53 | { 54 | if (item.index != null && item.index.error != null) 55 | { 56 | var e = messages.ElementAt(indexer); 57 | if (_esHelper.Options.EmitEventFailure.HasFlag(EmitEventFailureHandling.WriteToConsole)) 58 | { 59 | // ES reports an error, output the error to the console. 60 | Console.WriteLine("Failed to store into Elasticsearch. Elasticsearch reports for index {0} the following: {1}", item.index._index, item.index.error); 61 | } 62 | 63 | if (_esHelper.Options.EmitEventFailure.HasFlag(EmitEventFailureHandling.RaiseCallback) && _esHelper.Options.FailureCallback != null) 64 | { 65 | // Send to a failure callback 66 | try 67 | { 68 | _esHelper.Options.FailureCallback(e); 69 | } 70 | catch (Exception ex) 71 | { 72 | // We do not let this fail too 73 | Console.WriteLine("Caught exception while emitting to callback {1}: {0}", ex, _esHelper.Options.FailureCallback); 74 | } 75 | } 76 | } 77 | indexer++; 78 | } 79 | } 80 | else if (result.Success == false && result.OriginalException != null) 81 | { 82 | HandleException(result.OriginalException, messages); 83 | } 84 | } 85 | 86 | protected virtual async Task PostToEsAsync(IEnumerable messages) where T : class, IElasticsearchResponse, new() 87 | { 88 | if (messages == null || !messages.Any()) 89 | return null; 90 | 91 | if (!_esHelper.TemplateRegistrationSuccess && _esHelper.Options.RegisterTemplateFailure == RegisterTemplateRecovery.Throw) 92 | { 93 | return null; 94 | } 95 | 96 | var payload = new List(); 97 | foreach (var e in messages) 98 | { 99 | var indexName = _esHelper.GetIndexForEvent(e, e.Timestamp.ToUniversalTime()); 100 | var action = default(object); 101 | 102 | var pipelineName = _esHelper.Options.PipelineNameDecider?.Invoke(e) ?? _esHelper.Options.PipelineName; 103 | if (string.IsNullOrWhiteSpace(pipelineName)) 104 | { 105 | action = new { index = new { _index = indexName, _type = _esHelper.Options.TypeName } }; 106 | } 107 | else 108 | { 109 | action = new { index = new { _index = indexName, _type = _esHelper.Options.TypeName, pipeline = pipelineName } }; 110 | } 111 | var actionJson = _esHelper.Serialize(action); 112 | payload.Add(actionJson); 113 | payload.Add(e.Message); 114 | } 115 | return await _esHelper.Client.BulkAsync(PostData.MultiJson(payload)); 116 | } 117 | 118 | protected virtual void HandleException(Exception ex, IEnumerable messages) 119 | { 120 | if (_esHelper.Options.EmitEventFailure.HasFlag(EmitEventFailureHandling.WriteToConsole)) 121 | { 122 | // ES reports an error, output the error to the selflog 123 | Console.WriteLine("Caught exception while preforming bulk operation to Elasticsearch: {0}", ex); 124 | } 125 | if (_esHelper.Options.EmitEventFailure.HasFlag(EmitEventFailureHandling.RaiseCallback) && _esHelper.Options.FailureCallback != null) 126 | { 127 | // Send to a failure callback 128 | try 129 | { 130 | foreach (var e in messages) 131 | { 132 | _esHelper.Options.FailureCallback(e); 133 | } 134 | } 135 | catch (Exception exCallback) 136 | { 137 | // We do not let this fail too 138 | Console.WriteLine("Caught exception while emitting to callback {1}: {0}", exCallback, _esHelper.Options.FailureCallback); 139 | } 140 | } 141 | if (_esHelper.Options.EmitEventFailure.HasFlag(EmitEventFailureHandling.ThrowException)) 142 | { 143 | throw ex; 144 | } 145 | } 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /src/Zero.Logging.Elasticsearch/Zero.Logging.Elasticsearch.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netstandard2.0 5 | $(NoWarn);CS1591 6 | false 7 | Zero elasticsearch logger provider implementation for Microsoft.Extensions.Logging. 8 | zero;logging;logging-elasticsearch; 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /src/Zero.Logging.File/FileLoggerFactoryExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.Extensions.DependencyInjection; 3 | using Microsoft.Extensions.DependencyInjection.Extensions; 4 | using Microsoft.Extensions.Logging.Configuration; 5 | using Microsoft.Extensions.Options; 6 | using Zero.Logging.File; 7 | 8 | namespace Microsoft.Extensions.Logging 9 | { 10 | public static class FileLoggerFactoryExtensions 11 | { 12 | /// 13 | /// Adds a file logger named 'File' to the factory. 14 | /// 15 | /// The to use. 16 | public static ILoggingBuilder AddFile(this ILoggingBuilder builder) 17 | { 18 | //builder.AddConfiguration(); 19 | 20 | builder.Services.TryAddEnumerable(ServiceDescriptor.Singleton()); 21 | 22 | builder.Services.AddSingleton, FileLoggerOptionsSetup>(); 23 | builder.Services.AddSingleton, LoggerProviderOptionsChangeTokenSource>(); 24 | return builder; 25 | } 26 | 27 | 28 | /// 29 | /// Adds a file logger named 'File' to the factory. 30 | /// 31 | /// The to use. 32 | /// 33 | public static ILoggingBuilder AddFile(this ILoggingBuilder builder, Action configure) 34 | { 35 | if (configure == null) 36 | { 37 | throw new ArgumentNullException(nameof(configure)); 38 | } 39 | 40 | builder.AddFile(); 41 | builder.Services.Configure(configure); 42 | return builder; 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/Zero.Logging.File/FileLoggerOptions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Zero.Logging.Batching; 3 | 4 | namespace Zero.Logging.File 5 | { 6 | public class FileLoggerOptions : BatchingLoggerOptions 7 | { 8 | private long? _fileSizeLimit = 1L * 1024 * 1024 * 1024; 9 | private int? _retainedFileCountLimit = 31; // A long month of logs 10 | private string _logDirectory = "logs"; 11 | private string _fileName = "log"; 12 | 13 | /// 14 | /// Gets or sets value indicating log write directory. 15 | /// 16 | public string LogDirectory 17 | { 18 | get { return _logDirectory; } 19 | set 20 | { 21 | if (string.IsNullOrEmpty(value)) 22 | { 23 | throw new ArgumentException(nameof(value)); 24 | } 25 | _logDirectory = value; 26 | } 27 | } 28 | 29 | /// 30 | /// Gets or sets a string representing the prefix of the file name used to store the logging information. 31 | /// The current date, in the format YYYYMMDD will be added after the given value. 32 | /// Defaults to log. 33 | /// 34 | public string FileName 35 | { 36 | get { return _fileName; } 37 | set 38 | { 39 | if (string.IsNullOrEmpty(value)) 40 | { 41 | throw new ArgumentException(nameof(value)); 42 | } 43 | _fileName = value; 44 | } 45 | } 46 | 47 | /// 48 | /// Gets or sets a strictly positive value representing the maximum log size in bytes or null for no limit. 49 | /// Once the log is full, no more messages will be appended. 50 | /// Defaults to 1GB. 51 | /// 52 | public long? FileSizeLimit 53 | { 54 | get { return _fileSizeLimit; } 55 | set 56 | { 57 | if (value <= 0) 58 | { 59 | throw new ArgumentOutOfRangeException(nameof(value), $"{nameof(FileSizeLimit)} must be positive."); 60 | } 61 | _fileSizeLimit = value; 62 | } 63 | } 64 | 65 | /// 66 | /// Gets or sets a strictly positive value representing the maximum retained file count or null for no limit. 67 | /// Defaults to 31. 68 | /// 69 | public int? RetainedFileCountLimit 70 | { 71 | get { return _retainedFileCountLimit; } 72 | set 73 | { 74 | if (value <= 0) 75 | { 76 | throw new ArgumentOutOfRangeException(nameof(value), $"{nameof(RetainedFileCountLimit)} must be positive."); 77 | } 78 | _retainedFileCountLimit = value; 79 | } 80 | } 81 | 82 | /// 83 | /// Gets or sets the frequency at which the log file should roll. 84 | /// 85 | public RollingIntervalEnum RollingInterval { get; set; } 86 | 87 | ///// 88 | ///// Gets or sets value indicating if logger accepts and queues writes. 89 | ///// 90 | //public bool IsEnabledBatching { get; set; } = true; 91 | 92 | ///// 93 | ///// Gets or sets the log filter. 94 | ///// 95 | //public Func Filter { get; set; } 96 | } 97 | } -------------------------------------------------------------------------------- /src/Zero.Logging.File/FileLoggerOptionsSetup.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Extensions.Logging.Configuration; 2 | using Microsoft.Extensions.Options; 3 | 4 | namespace Zero.Logging.File 5 | { 6 | public class FileLoggerOptionsSetup : ConfigureFromConfigurationOptions 7 | { 8 | public FileLoggerOptionsSetup(ILoggerProviderConfiguration providerConfiguration) 9 | : base(providerConfiguration.Configuration) 10 | { 11 | 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/Zero.Logging.File/FileLoggerProvider.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.IO; 3 | using System.Linq; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | using Microsoft.Extensions.Logging; 7 | using Microsoft.Extensions.Options; 8 | using Zero.Logging.Batching; 9 | 10 | namespace Zero.Logging.File 11 | { 12 | [ProviderAlias("File")] 13 | public class FileLoggerProvider : BatchingLoggerProvider 14 | { 15 | private readonly string _path; 16 | private readonly string _fileName; 17 | private readonly long? _maxFileSize; 18 | private readonly int? _maxRetainedFiles; 19 | private readonly RollingIntervalEnum _rollingInterval; 20 | 21 | public FileLoggerProvider(IOptionsMonitor options) : base(options) 22 | { 23 | var loggerOptions = options.CurrentValue; 24 | _path = loggerOptions.LogDirectory; 25 | _fileName = loggerOptions.FileName; 26 | _maxFileSize = loggerOptions.FileSizeLimit; 27 | _maxRetainedFiles = loggerOptions.RetainedFileCountLimit; 28 | _rollingInterval = loggerOptions.RollingInterval; 29 | } 30 | 31 | protected override async Task WriteMessagesAsync(IEnumerable messages, CancellationToken cancellationToken) 32 | { 33 | Directory.CreateDirectory(_path); 34 | 35 | foreach (var group in messages.GroupBy(GetGrouping)) 36 | { 37 | var fullName = Path.Combine(_path, _fileName + "-" + group.Key + ".txt"); 38 | var fileInfo = new FileInfo(fullName); 39 | if (_maxFileSize > 0 && fileInfo.Exists && fileInfo.Length > _maxFileSize) 40 | { 41 | return; 42 | } 43 | using (var streamWriter = System.IO.File.AppendText(fullName)) 44 | { 45 | foreach (var item in group) 46 | { 47 | await streamWriter.WriteAsync(item.Message); 48 | } 49 | //await streamWriter.FlushAsync(); 50 | } 51 | } 52 | 53 | RollFiles(); 54 | } 55 | 56 | protected string GetGrouping(LogMessage message) 57 | { 58 | return message.Timestamp.ToString(_rollingInterval.GetFormat()); 59 | } 60 | 61 | protected void RollFiles() 62 | { 63 | if (_maxRetainedFiles > 0) 64 | { 65 | var files = new DirectoryInfo(_path) 66 | .GetFiles(_fileName + "*") 67 | .OrderByDescending(f => f.Name) 68 | .Skip(_maxRetainedFiles.Value); 69 | 70 | foreach (var item in files) 71 | { 72 | item.Delete(); 73 | } 74 | } 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/Zero.Logging.File/RollingIntervalEnum.cs: -------------------------------------------------------------------------------- 1 | namespace Zero.Logging.File 2 | { 3 | /// 4 | /// Specifies the frequency at which the log file should roll. 5 | /// 6 | public enum RollingIntervalEnum 7 | { 8 | /// 9 | /// Roll every year. Filenames will have a four-digit year appended in the pattern yyyy. 10 | /// 11 | Year, 12 | 13 | /// 14 | /// Roll every calendar month. Filenames will have yyyyMM appended. 15 | /// 16 | Month, 17 | 18 | /// 19 | /// Roll every day. Filenames will have yyyyMMdd appended. 20 | /// 21 | Day, 22 | 23 | /// 24 | /// Roll every hour. Filenames will have yyyyMMddHH appended. 25 | /// 26 | Hour, 27 | 28 | /// 29 | /// Roll every minute. Filenames will have yyyyMMddHHmm appended. 30 | /// 31 | Minute 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/Zero.Logging.File/RollingIntervalExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Zero.Logging.File 4 | { 5 | internal static class RollingIntervalEnumExtensions 6 | { 7 | public static string GetFormat(this RollingIntervalEnum interval) 8 | { 9 | switch (interval) 10 | { 11 | case RollingIntervalEnum.Year: 12 | return "yyyy"; 13 | case RollingIntervalEnum.Month: 14 | return "yyyyMM"; 15 | case RollingIntervalEnum.Day: 16 | return "yyyyMMdd"; 17 | case RollingIntervalEnum.Hour: 18 | return "yyyyMMddHH"; 19 | case RollingIntervalEnum.Minute: 20 | return "yyyyMMddHHmm"; 21 | default: 22 | throw new ArgumentException("Invalid rolling interval"); 23 | } 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/Zero.Logging.File/Zero.Logging.File.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netstandard2.0 5 | $(NoWarn);CS1591 6 | false 7 | Zero File logger provider implementation for Microsoft.Extensions.Logging. 8 | zero;logging;logging-file; 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /test/Zero.Logging.Tests/BigDataTests.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics; 2 | using Xunit; 3 | using Xunit.Abstractions; 4 | 5 | namespace Zero.Logging.Tests 6 | { 7 | public class BigDataTest 8 | { 9 | private readonly ITestOutputHelper output; 10 | public BigDataTest(ITestOutputHelper outputHelper) 11 | { 12 | output = outputHelper; 13 | } 14 | 15 | /// 16 | /// 使用Nlog连续插入20W行字符串 17 | /// 18 | [Fact] 19 | public void NlogTest() 20 | { 21 | NLog.Logger log = NLog.LogManager.GetCurrentClassLogger(); 22 | var total = 200000; 23 | var sw = new Stopwatch(); 24 | sw.Start(); 25 | for (int i = 0; i < total; i++) 26 | { 27 | log.Info("nlog bigdata test: " + i); 28 | } 29 | sw.Stop(); 30 | log.Info($"total: {total}, Elapsed:{sw.ElapsedMilliseconds}"); 31 | output.WriteLine($"NLog测试 total: {total}, Elapsed:{sw.ElapsedMilliseconds}"); 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /test/Zero.Logging.Tests/Zero.Logging.Tests.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp2.2 5 | false 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | all 14 | runtime; build; native; contentfiles; analyzers 15 | 16 | 17 | 18 | 19 | 20 | Always 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /test/Zero.Logging.Tests/nlog.config: -------------------------------------------------------------------------------- 1 |  2 | 7 | 8 | 9 | 10 | 11 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /version.props: -------------------------------------------------------------------------------- 1 | 2 | 3 | 2.2.1 4 | rtm 5 | true 6 | t000 7 | a- 8 | 9 | $(VersionPrefix) 10 | $(VersionPrefix)-$(VersionSuffix)-final 11 | $(FeatureBranchVersionPrefix)$(VersionSuffix)-$([System.Text.RegularExpressions.Regex]::Replace('$(FeatureBranchVersionSuffix)', '[^\w-]', '-')) 12 | $(VersionSuffix)-$(BuildNumber) 13 | 14 | -------------------------------------------------------------------------------- /zero-logging.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.28315.86 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Zero.Logging.File", "src\Zero.Logging.File\Zero.Logging.File.csproj", "{C3B107A7-8601-4771-90DA-22779274C64F}" 7 | EndProject 8 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{83083C28-77B9-4692-A05A-874666B7C6BE}" 9 | EndProject 10 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "samples", "samples", "{2377EF56-906F-473D-B1E8-F956B866EA52}" 11 | EndProject 12 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SampleApp", "samples\SampleApp\SampleApp.csproj", "{047B371A-E362-4FB4-A9EF-ADD1D301E9DC}" 13 | EndProject 14 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Zero.Logging.Elasticsearch", "src\Zero.Logging.Elasticsearch\Zero.Logging.Elasticsearch.csproj", "{C6714805-F1E5-4F19-AE08-BBCF116EB6DF}" 15 | EndProject 16 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{7103152B-88DD-45E2-A79A-751033806B9F}" 17 | EndProject 18 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Zero.Logging.Tests", "test\Zero.Logging.Tests\Zero.Logging.Tests.csproj", "{E05D2BE2-3151-4F58-9136-20ABFB146D6E}" 19 | EndProject 20 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{B6696DF9-C300-42D0-A5F9-DACCEEA1AC0D}" 21 | ProjectSection(SolutionItems) = preProject 22 | Directory.Build.props = Directory.Build.props 23 | Directory.Build.targets = Directory.Build.targets 24 | version.props = version.props 25 | EndProjectSection 26 | EndProject 27 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Zero.Logging.Batching", "src\Zero.Logging.Batching\Zero.Logging.Batching.csproj", "{F6F89C45-8370-4F01-BA02-EA02E9A871C4}" 28 | EndProject 29 | Global 30 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 31 | Debug|Any CPU = Debug|Any CPU 32 | Release|Any CPU = Release|Any CPU 33 | EndGlobalSection 34 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 35 | {C3B107A7-8601-4771-90DA-22779274C64F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 36 | {C3B107A7-8601-4771-90DA-22779274C64F}.Debug|Any CPU.Build.0 = Debug|Any CPU 37 | {C3B107A7-8601-4771-90DA-22779274C64F}.Release|Any CPU.ActiveCfg = Release|Any CPU 38 | {C3B107A7-8601-4771-90DA-22779274C64F}.Release|Any CPU.Build.0 = Release|Any CPU 39 | {047B371A-E362-4FB4-A9EF-ADD1D301E9DC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 40 | {047B371A-E362-4FB4-A9EF-ADD1D301E9DC}.Debug|Any CPU.Build.0 = Debug|Any CPU 41 | {047B371A-E362-4FB4-A9EF-ADD1D301E9DC}.Release|Any CPU.ActiveCfg = Release|Any CPU 42 | {047B371A-E362-4FB4-A9EF-ADD1D301E9DC}.Release|Any CPU.Build.0 = Release|Any CPU 43 | {C6714805-F1E5-4F19-AE08-BBCF116EB6DF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 44 | {C6714805-F1E5-4F19-AE08-BBCF116EB6DF}.Debug|Any CPU.Build.0 = Debug|Any CPU 45 | {C6714805-F1E5-4F19-AE08-BBCF116EB6DF}.Release|Any CPU.ActiveCfg = Release|Any CPU 46 | {C6714805-F1E5-4F19-AE08-BBCF116EB6DF}.Release|Any CPU.Build.0 = Release|Any CPU 47 | {E05D2BE2-3151-4F58-9136-20ABFB146D6E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 48 | {E05D2BE2-3151-4F58-9136-20ABFB146D6E}.Debug|Any CPU.Build.0 = Debug|Any CPU 49 | {E05D2BE2-3151-4F58-9136-20ABFB146D6E}.Release|Any CPU.ActiveCfg = Release|Any CPU 50 | {E05D2BE2-3151-4F58-9136-20ABFB146D6E}.Release|Any CPU.Build.0 = Release|Any CPU 51 | {F6F89C45-8370-4F01-BA02-EA02E9A871C4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 52 | {F6F89C45-8370-4F01-BA02-EA02E9A871C4}.Debug|Any CPU.Build.0 = Debug|Any CPU 53 | {F6F89C45-8370-4F01-BA02-EA02E9A871C4}.Release|Any CPU.ActiveCfg = Release|Any CPU 54 | {F6F89C45-8370-4F01-BA02-EA02E9A871C4}.Release|Any CPU.Build.0 = Release|Any CPU 55 | EndGlobalSection 56 | GlobalSection(SolutionProperties) = preSolution 57 | HideSolutionNode = FALSE 58 | EndGlobalSection 59 | GlobalSection(NestedProjects) = preSolution 60 | {C3B107A7-8601-4771-90DA-22779274C64F} = {83083C28-77B9-4692-A05A-874666B7C6BE} 61 | {047B371A-E362-4FB4-A9EF-ADD1D301E9DC} = {2377EF56-906F-473D-B1E8-F956B866EA52} 62 | {C6714805-F1E5-4F19-AE08-BBCF116EB6DF} = {83083C28-77B9-4692-A05A-874666B7C6BE} 63 | {E05D2BE2-3151-4F58-9136-20ABFB146D6E} = {7103152B-88DD-45E2-A79A-751033806B9F} 64 | {F6F89C45-8370-4F01-BA02-EA02E9A871C4} = {83083C28-77B9-4692-A05A-874666B7C6BE} 65 | EndGlobalSection 66 | GlobalSection(ExtensibilityGlobals) = postSolution 67 | SolutionGuid = {A591A229-9B57-4758-993E-3F8A7A6F78A8} 68 | EndGlobalSection 69 | EndGlobal 70 | --------------------------------------------------------------------------------