├── .editorconfig ├── .gitattributes ├── .gitignore ├── Build.ps1 ├── CHANGES.md ├── LICENSE ├── README.md ├── appveyor.yml ├── assets └── Serilog.snk ├── sample └── sampleDurableLogger │ ├── Program.cs │ ├── Properties │ └── AssemblyInfo.cs │ └── sampleDurableLogger.csproj ├── serilog-sinks-loggly.sln ├── src └── Serilog.Sinks.Loggly │ ├── LoggerConfigurationLogglyExtensions.cs │ ├── Properties │ └── AssemblyInfo.cs │ ├── Serilog.Sinks.Loggly.csproj │ └── Sinks │ └── Loggly │ ├── ControlledSwitch.cs │ ├── Durable │ └── FileSetPosition.cs │ ├── DurableLogglySink.cs │ ├── ExceptionDetails.cs │ ├── ExponentialBackoffConnectionSchedule.cs │ ├── FileBasedBookmarkProvider.cs │ ├── FileBufferDataProvider.cs │ ├── FileSystemAdapter.cs │ ├── HttpLogShipper.cs │ ├── IBookmarkProvider.cs │ ├── IFileSystemAdapter.cs │ ├── InvalidPayloadLogger.cs │ ├── LogEventConverter.cs │ ├── LogIncludes.cs │ ├── LogglyConfigAdapter.cs │ ├── LogglyConfiguration.cs │ ├── LogglyFormatter.cs │ ├── LogglyPropertyFormatter.cs │ ├── LogglySink.cs │ └── PortableTimer.cs └── test └── Serilog.Sinks.Loggly.Tests ├── ExceptionSerialization.cs ├── Properties └── AssemblyInfo.cs ├── Serilog.Sinks.Loggly.Tests.csproj └── Sinks └── Loggly ├── Durable └── FileSetPositionTests.cs ├── Expectations ├── expectedInvalidPayloadFileN.json └── expectedInvalidPayloadFileRN.json ├── FileBasedBookmarkProviderTests.cs ├── FileBufferDataProviderTests.cs ├── InvalidPayloadLoggerTests.cs └── SampleBuffers ├── 20EventsN.json ├── 20EventsRN.json └── singleEvent.json /.editorconfig: -------------------------------------------------------------------------------- 1 | root=true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 4 6 | 7 | [project.json] 8 | indent_style = space 9 | indent_size = 2 10 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | 3 | * text=auto 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.userosscache 8 | *.sln.docstates 9 | .localHistory/* 10 | 11 | # User-specific files (MonoDevelop/Xamarin Studio) 12 | *.userprefs 13 | 14 | # Build results 15 | [Dd]ebug/ 16 | [Dd]ebugPublic/ 17 | [Rr]elease/ 18 | [Rr]eleases/ 19 | x64/ 20 | x86/ 21 | bld/ 22 | [Bb]in/ 23 | [Oo]bj/ 24 | 25 | # Visual Studio 2015 cache/options directory 26 | .vs/ 27 | # Uncomment if you have tasks that create the project's static files in wwwroot 28 | #wwwroot/ 29 | 30 | # MSTest test Results 31 | [Tt]est[Rr]esult*/ 32 | [Bb]uild[Ll]og.* 33 | 34 | # NUNIT 35 | *.VisualState.xml 36 | TestResult.xml 37 | 38 | # Build Results of an ATL Project 39 | [Dd]ebugPS/ 40 | [Rr]eleasePS/ 41 | dlldata.c 42 | 43 | # DNX 44 | project.lock.json 45 | artifacts/ 46 | 47 | *_i.c 48 | *_p.c 49 | *_i.h 50 | *.ilk 51 | *.meta 52 | *.obj 53 | *.pch 54 | *.pdb 55 | *.pgc 56 | *.pgd 57 | *.rsp 58 | *.sbr 59 | *.tlb 60 | *.tli 61 | *.tlh 62 | *.tmp 63 | *.tmp_proj 64 | *.log 65 | *.vspscc 66 | *.vssscc 67 | .builds 68 | *.pidb 69 | *.svclog 70 | *.scc 71 | 72 | # Chutzpah Test files 73 | _Chutzpah* 74 | 75 | # Visual C++ cache files 76 | ipch/ 77 | *.aps 78 | *.ncb 79 | *.opendb 80 | *.opensdf 81 | *.sdf 82 | *.cachefile 83 | 84 | # Visual Studio profiler 85 | *.psess 86 | *.vsp 87 | *.vspx 88 | *.sap 89 | 90 | # TFS 2012 Local Workspace 91 | $tf/ 92 | 93 | # Guidance Automation Toolkit 94 | *.gpState 95 | 96 | # ReSharper is a .NET coding add-in 97 | _ReSharper*/ 98 | *.[Rr]e[Ss]harper 99 | *.DotSettings.user 100 | 101 | # JustCode is a .NET coding add-in 102 | .JustCode 103 | 104 | # TeamCity is a build add-in 105 | _TeamCity* 106 | 107 | # DotCover is a Code Coverage Tool 108 | *.dotCover 109 | 110 | # NCrunch 111 | _NCrunch_* 112 | .*crunch*.local.xml 113 | nCrunchTemp_* 114 | 115 | # MightyMoose 116 | *.mm.* 117 | AutoTest.Net/ 118 | 119 | # Web workbench (sass) 120 | .sass-cache/ 121 | 122 | # Installshield output folder 123 | [Ee]xpress/ 124 | 125 | # DocProject is a documentation generator add-in 126 | DocProject/buildhelp/ 127 | DocProject/Help/*.HxT 128 | DocProject/Help/*.HxC 129 | DocProject/Help/*.hhc 130 | DocProject/Help/*.hhk 131 | DocProject/Help/*.hhp 132 | DocProject/Help/Html2 133 | DocProject/Help/html 134 | 135 | # Click-Once directory 136 | publish/ 137 | 138 | # Publish Web Output 139 | *.[Pp]ublish.xml 140 | *.azurePubxml 141 | # TODO: Comment the next line if you want to checkin your web deploy settings 142 | # but database connection strings (with potential passwords) will be unencrypted 143 | *.pubxml 144 | *.publishproj 145 | 146 | # NuGet Packages 147 | *.nupkg 148 | # The packages folder can be ignored because of Package Restore 149 | **/packages/* 150 | # except build/, which is used as an MSBuild target. 151 | !**/packages/build/ 152 | # Uncomment if necessary however generally it will be regenerated when needed 153 | #!**/packages/repositories.config 154 | # NuGet v3's project.json files produces more ignoreable files 155 | *.nuget.props 156 | *.nuget.targets 157 | 158 | # Microsoft Azure Build Output 159 | csx/ 160 | *.build.csdef 161 | 162 | # Microsoft Azure Emulator 163 | ecf/ 164 | rcf/ 165 | 166 | # Microsoft Azure ApplicationInsights config file 167 | ApplicationInsights.config 168 | 169 | # Windows Store app package directory 170 | AppPackages/ 171 | BundleArtifacts/ 172 | 173 | # Visual Studio cache files 174 | # files ending in .cache can be ignored 175 | *.[Cc]ache 176 | # but keep track of directories ending in .cache 177 | !*.[Cc]ache/ 178 | 179 | # Others 180 | ClientBin/ 181 | ~$* 182 | *~ 183 | *.dbmdl 184 | *.dbproj.schemaview 185 | *.pfx 186 | *.publishsettings 187 | node_modules/ 188 | orleans.codegen.cs 189 | 190 | # RIA/Silverlight projects 191 | Generated_Code/ 192 | 193 | # Backup & report files from converting an old project file 194 | # to a newer Visual Studio version. Backup files are not needed, 195 | # because we have git ;-) 196 | _UpgradeReport_Files/ 197 | Backup*/ 198 | UpgradeLog*.XML 199 | UpgradeLog*.htm 200 | 201 | # SQL Server files 202 | *.mdf 203 | *.ldf 204 | 205 | # Business Intelligence projects 206 | *.rdl.data 207 | *.bim.layout 208 | *.bim_*.settings 209 | 210 | # Microsoft Fakes 211 | FakesAssemblies/ 212 | 213 | # GhostDoc plugin setting file 214 | *.GhostDoc.xml 215 | 216 | # Node.js Tools for Visual Studio 217 | .ntvs_analysis.dat 218 | 219 | # Visual Studio 6 build log 220 | *.plg 221 | 222 | # Visual Studio 6 workspace options file 223 | *.opt 224 | 225 | # Visual Studio LightSwitch build output 226 | **/*.HTMLClient/GeneratedArtifacts 227 | **/*.DesktopClient/GeneratedArtifacts 228 | **/*.DesktopClient/ModelManifest.xml 229 | **/*.Server/GeneratedArtifacts 230 | **/*.Server/ModelManifest.xml 231 | _Pvt_Extensions 232 | 233 | # Paket dependency manager 234 | .paket/paket.exe 235 | 236 | # FAKE - F# Make 237 | .fake/ 238 | -------------------------------------------------------------------------------- /Build.ps1: -------------------------------------------------------------------------------- 1 | echo "build: Build started" 2 | 3 | Push-Location $PSScriptRoot 4 | 5 | if(Test-Path .\artifacts) { 6 | echo "build: Cleaning .\artifacts" 7 | Remove-Item .\artifacts -Force -Recurse 8 | } 9 | 10 | & dotnet restore --no-cache 11 | 12 | $branch = @{ $true = $env:APPVEYOR_REPO_BRANCH; $false = $(git symbolic-ref --short -q HEAD) }[$env:APPVEYOR_REPO_BRANCH -ne $NULL]; 13 | $revision = @{ $true = "{0:00000}" -f [convert]::ToInt32("0" + $env:APPVEYOR_BUILD_NUMBER, 10); $false = "local" }[$env:APPVEYOR_BUILD_NUMBER -ne $NULL]; 14 | #if branch includes features/, things blow up, so let's remove it 15 | $branchForPackageName = $branch -replace "feature[s]?/",''; 16 | $suffix = @{ $true = ""; $false = "$($branchForPackageName.Substring(0, [math]::Min(10,$branchForPackageName.Length)))-$revision"}[$branch -eq "master" -and $revision -ne "local"] 17 | 18 | echo "build: Version suffix is $suffix" 19 | 20 | foreach ($src in ls src/*) { 21 | Push-Location $src 22 | 23 | echo "build: Packaging project in $src" 24 | 25 | if ($suffix) { 26 | & dotnet pack -c Release -o ..\..\artifacts --version-suffix=$suffix 27 | } else { 28 | & dotnet pack -c Release -o ..\..\artifacts 29 | } 30 | if($LASTEXITCODE -ne 0) { exit 1 } 31 | 32 | Pop-Location 33 | } 34 | 35 | foreach ($test in ls test/*.PerformanceTests) { 36 | Push-Location $test 37 | 38 | echo "build: Building performance test project in $test" 39 | 40 | & dotnet build -c Release 41 | if($LASTEXITCODE -ne 0) { exit 2 } 42 | 43 | Pop-Location 44 | } 45 | 46 | foreach ($test in ls test/*.Tests) { 47 | Push-Location $test 48 | 49 | echo "build: Testing project in $test" 50 | 51 | & dotnet test -c Release 52 | if($LASTEXITCODE -ne 0) { exit 3 } 53 | 54 | Pop-Location 55 | } 56 | 57 | Pop-Location 58 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | 2.0.0 2 | * Moved the Loggly sink from its [original location](https://github.com/serilog/serilog) 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Serilog.Sinks.Loggly 2 | 3 | [![Build status](https://ci.appveyor.com/api/projects/status/x2ob36tl8brpkkjf/branch/master?svg=true)](https://ci.appveyor.com/project/serilog/serilog-sinks-loggly/branch/master) [![NuGet Version](http://img.shields.io/nuget/v/Serilog.Sinks.Loggly.svg?style=flat)](https://www.nuget.org/packages/Serilog.Sinks.Loggly/) 4 | 5 | 6 | [Loggly](http://www.loggly.com) is a cloud based log management service. Create a new input and specify that you want to use a http input with JSON enabled. Use the [loggly-csharp-configuration](https://github.com/neutmute/loggly-csharp) XML configuration syntax to configure the sink. 7 | 8 | **Package** - [Serilog.Sinks.Loggly](http://nuget.org/packages/serilog.sinks.loggly) 9 | | **Platforms** - .NET 4.5 10 | 11 | ```csharp 12 | var log = new LoggerConfiguration() 13 | .WriteTo.Loggly() 14 | .CreateLogger(); 15 | ``` 16 | 17 | Properties will be sent along to Loggly. The level is sent as a category. 18 | 19 | To use a durable logger (that will save messages locally if the connection to the server is unavailable, and resend once the connection has recovered), set the `bufferBaseFilename` argument in the `Loggly()` extension method. 20 | 21 | ```csharp 22 | var log = new LoggerConfiguration() 23 | .WriteTo.Loggly(bufferBaseFilename:@"C:\test\buffer") 24 | .CreateLogger(); 25 | ``` 26 | 27 | This will write unsent messages to a `buffer-{Date}.json` file in the specified folder (`C:\test\` in the example). 28 | 29 | The method also takes a `retainedFileCountLimit` parameter that will allow you to control how much info to store / ship when back online. By default, the value is `null` with the intent is to send all persisted data, no matter how old. If you specify a value, only the data in the last N buffer files will be shipped back, preventing stale data to be indexed (if that info is no longer usefull). 30 | 31 | The sink can also be configured from `appsettings.json` for .NET Standard / .NET Core applications that do not support XML configuration: 32 | 33 | ```json 34 | { 35 | "Serilog": { 36 | "WriteTo": [ 37 | { 38 | "Name": "Loggly", 39 | "Args": { 40 | "customerToken": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", 41 | "tags": "foo,bar" 42 | } 43 | } 44 | ], 45 | "Properties": { "Application": "SampleApp" } 46 | } 47 | } 48 | ``` 49 | 50 | The `customerToken` argument is required, if you use this form of configuration. The `tags` argument is comma-delimited. The `Application` property will also be sent to Loggly and should be set appropriately. -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | version: '{build}' 2 | skip_tags: true 3 | image: Visual Studio 2019 4 | configuration: Release 5 | init: 6 | - git config --global core.autocrlf false 7 | install: 8 | - ps: mkdir -Force ".\build\" | Out-Null 9 | build_script: 10 | - ps: ./Build.ps1 11 | test: off 12 | artifacts: 13 | - path: artifacts/Serilog.*.nupkg 14 | only_commits: 15 | files: 16 | - serilog-sinks-loggly.sln 17 | - src/Serilog.Sinks.Loggly/ 18 | - Build.ps1 19 | - assets/ 20 | - test/Serilog.Sinks.Loggly.Tests/ 21 | - appveyor.yml 22 | deploy: 23 | - provider: NuGet 24 | api_key: 25 | secure: K3/810hkTO6rd2AEHVkUQAadjGmDREus9k96QHu6hxrA1/wRTuAJemHMKtVVgIvf 26 | skip_symbols: true 27 | on: 28 | branch: /^(master|dev)$/ 29 | - provider: GitHub 30 | auth_token: 31 | secure: p4LpVhBKxGS5WqucHxFQ5c7C8cP74kbNB0Z8k9Oxx/PMaDQ1+ibmoexNqVU5ZlmX 32 | artifact: /Serilog.*\.nupkg/ 33 | tag: v$(appveyor_build_version) 34 | on: 35 | branch: master 36 | -------------------------------------------------------------------------------- /assets/Serilog.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/serilog-archive/serilog-sinks-loggly/31d5b01acff21bf1de6a4239aea2d80abaeb497a/assets/Serilog.snk -------------------------------------------------------------------------------- /sample/sampleDurableLogger/Program.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Globalization; 3 | using System.Runtime.InteropServices; 4 | using Loggly; 5 | using Loggly.Config; 6 | using Serilog; 7 | using Serilog.Context; 8 | using Serilog.Core; 9 | using Serilog.Core.Enrichers; 10 | using Serilog.Enrichers; 11 | using Serilog.Sinks.RollingFileAlternate; 12 | 13 | namespace SampleDurableLogger 14 | { 15 | public class Program 16 | { 17 | public static void Main() 18 | { 19 | SetupLogglyConfiguration(); 20 | using (var logger = CreateLogger(@"C:\test\Logs\")) 21 | { 22 | logger.Information("Test message - app started"); 23 | logger.Warning("Test message with {@Data}", new {P1 = "sample", P2 = DateTime.Now}); 24 | logger.Warning("Test2 message with {@Data}", new {P1 = "sample2", P2 = 10}); 25 | 26 | Console.WriteLine( 27 | "Disconnect to test offline. Two messages will be sent. Press enter to send and wait a minute or so before reconnecting or use breakpoints to see that send fails."); 28 | Console.ReadLine(); 29 | 30 | logger.Information("Second test message"); 31 | logger.Warning("Second test message with {@Data}", new {P1 = "sample2", P2 = DateTime.Now, P3 = DateTime.UtcNow, P4 = DateTimeOffset.Now, P5 = DateTimeOffset.UtcNow}); 32 | 33 | 34 | Console.WriteLine( 35 | "Offline messages written. Once you have confirmed that messages have been written locally, reconnect to see messages go out. Press Enter for more messages to be written."); 36 | Console.ReadLine(); 37 | 38 | logger.Information("Third test message"); 39 | logger.Warning("Third test message with {@Data}", new {P1 = "sample3", P2 = DateTime.Now}); 40 | 41 | Console.WriteLine( 42 | "Back online messages written. Check loggly and files for data. Wait a minute or so before reconnecting. Press Enter to continue"); 43 | Console.ReadLine(); 44 | 45 | using (LogContext.PushProperty("sampleProperty", "Sample Value")) 46 | { 47 | logger.Information("message to send with {@Data}", new { P1 = "sample4", P2 = DateTime.Now }); 48 | } 49 | Console.WriteLine( 50 | "Pushed property added to object. Check loggly and data. Press Enter to terminate"); 51 | Console.ReadLine(); 52 | } 53 | } 54 | 55 | static Logger CreateLogger(string logFilePath) 56 | { 57 | //write selflog to stderr 58 | Serilog.Debugging.SelfLog.Enable(Console.Error); 59 | 60 | return new LoggerConfiguration() 61 | .MinimumLevel.Debug() 62 | //Add enrichers 63 | .Enrich.FromLogContext() 64 | .Enrich.WithProcessId() 65 | .Enrich.WithThreadId() 66 | .Enrich.With(new EnvironmentUserNameEnricher()) 67 | .Enrich.With(new MachineNameEnricher()) 68 | .Enrich.With(new PropertyEnricher("Environment", "development")) 69 | //Add sinks 70 | .WriteTo.Async(s => s.Loggly( 71 | bufferBaseFilename: logFilePath + "buffer", 72 | formatProvider: CreateLoggingCulture()) 73 | .MinimumLevel.Information() 74 | ) 75 | .WriteTo.Async(s => s.RollingFileAlternate( 76 | logFilePath, 77 | outputTemplate: 78 | "[{ProcessId}] {Timestamp} [{ThreadId}] [{Level}] [{SourceContext}] [{Category}] {Message}{NewLine}{Exception}", 79 | fileSizeLimitBytes: 10 * 1024 * 1024, 80 | retainedFileCountLimit: 100, 81 | formatProvider: CreateLoggingCulture()) 82 | .MinimumLevel.Debug() 83 | ) 84 | .CreateLogger(); 85 | } 86 | 87 | 88 | static void SetupLogglyConfiguration() 89 | { 90 | //CHANGE THESE TWO TO YOUR LOGGLY ACCOUNT: DO NOT COMMIT TO Source control!!! 91 | const string appName = "AppNameHere"; 92 | const string customerToken = "yourkeyhere"; 93 | 94 | //Configure Loggly 95 | var config = LogglyConfig.Instance; 96 | config.CustomerToken = customerToken; 97 | config.ApplicationName = appName; 98 | config.Transport = new TransportConfiguration() 99 | { 100 | EndpointHostname = "logs-01.loggly.com", 101 | EndpointPort = 443, 102 | LogTransport = LogTransport.Https 103 | }; 104 | config.ThrowExceptions = true; 105 | //use the new Transport property that hides IP as of loggly-csharp 4.6.1.76 106 | config.Transport.ForwardedForIp = "0.0.0.0"; 107 | 108 | //Define Tags sent to Loggly 109 | config.TagConfig.Tags.AddRange(new ITag[]{ 110 | new ApplicationNameTag {Formatter = "application-{0}"}, 111 | new HostnameTag { Formatter = "host-{0}" } 112 | }); 113 | } 114 | 115 | static CultureInfo CreateLoggingCulture() 116 | { 117 | var loggingCulture = new CultureInfo(""); 118 | 119 | //with this DateTime and DateTimeOffset string representations will be sortable. By default, 120 | // serialization without a culture or formater will use InvariantCulture. This may or may not be 121 | // desirable, depending on the sorting needs you require or even the region your in. In this sample 122 | // the invariant culture is used as a base, but the DateTime format is changed to a specific representation. 123 | // Instead of the dd/MM/yyyy hh:mm:ss, we'll force yyyy-MM-dd HH:mm:ss.fff which is sortable and obtainable 124 | // by overriding ShortDatePattern and LongTimePattern. 125 | // 126 | //Do note that they don't include the TimeZone by default, so a datetime will not have the TZ 127 | // while a DateTimeOffset will in it's string representation. 128 | // Both use the longTimePattern for time formatting, but including the time zone in the 129 | // pattern will duplicate the TZ representation when using DateTimeOffset which serilog does 130 | // for the timestamp. 131 | // 132 | //If you do not require specific formats, this method will not be required. Just pass in null (the default) 133 | // for IFormatProvider in the Loggly() sink configuration method. 134 | loggingCulture.DateTimeFormat.ShortDatePattern = "yyyy-MM-dd"; 135 | loggingCulture.DateTimeFormat.LongTimePattern = "HH:mm:ss.fff"; 136 | 137 | return loggingCulture; 138 | } 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /sample/sampleDurableLogger/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.InteropServices; 3 | 4 | // General Information about an assembly is controlled through the following 5 | // set of attributes. Change these attribute values to modify the information 6 | // associated with an assembly. 7 | [assembly: AssemblyConfiguration("")] 8 | [assembly: AssemblyCompany("Microsoft")] 9 | [assembly: AssemblyProduct("sampleDurableLogger")] 10 | [assembly: AssemblyTrademark("")] 11 | 12 | // Setting ComVisible to false makes the types in this assembly not visible 13 | // to COM components. If you need to access a type in this assembly from 14 | // COM, set the ComVisible attribute to true on that type. 15 | [assembly: ComVisible(false)] 16 | 17 | // The following GUID is for the ID of the typelib if this project is exposed to COM 18 | [assembly: Guid("a47ed1ce-fe7c-444e-9391-10d6b60519c2")] 19 | -------------------------------------------------------------------------------- /sample/sampleDurableLogger/sampleDurableLogger.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp1.0 5 | sampleDurableLogger 6 | Exe 7 | 1.0.4 8 | false 9 | false 10 | false 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /serilog-sinks-loggly.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.26430.12 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{037440DE-440B-4129-9F7A-09B42D00397E}" 7 | EndProject 8 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "assets", "assets", "{E9D1B5E1-DEB9-4A04-8BAB-24EC7240ADAF}" 9 | ProjectSection(SolutionItems) = preProject 10 | .editorconfig = .editorconfig 11 | appveyor.yml = appveyor.yml 12 | Build.ps1 = Build.ps1 13 | README.md = README.md 14 | assets\Serilog.snk = assets\Serilog.snk 15 | EndProjectSection 16 | EndProject 17 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{2B558B69-8F95-4F82-B223-EBF60F6F31EE}" 18 | EndProject 19 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "sample", "sample", "{FD377716-21BA-45D1-9580-02C2BECA5BAB}" 20 | EndProject 21 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serilog.Sinks.Loggly", "src\Serilog.Sinks.Loggly\Serilog.Sinks.Loggly.csproj", "{94E6E098-11A0-43CF-B0CF-4BA270CE9DBD}" 22 | EndProject 23 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Serilog.Sinks.Loggly.Tests", "test\Serilog.Sinks.Loggly.Tests\Serilog.Sinks.Loggly.Tests.csproj", "{120C431E-479C-48C7-9539-CFA32399769C}" 24 | EndProject 25 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "sampleDurableLogger", "sample\sampleDurableLogger\sampleDurableLogger.csproj", "{A47ED1CE-FE7C-444E-9391-10D6B60519C2}" 26 | EndProject 27 | Global 28 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 29 | Debug|Any CPU = Debug|Any CPU 30 | Release|Any CPU = Release|Any CPU 31 | EndGlobalSection 32 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 33 | {94E6E098-11A0-43CF-B0CF-4BA270CE9DBD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 34 | {94E6E098-11A0-43CF-B0CF-4BA270CE9DBD}.Debug|Any CPU.Build.0 = Debug|Any CPU 35 | {94E6E098-11A0-43CF-B0CF-4BA270CE9DBD}.Release|Any CPU.ActiveCfg = Release|Any CPU 36 | {94E6E098-11A0-43CF-B0CF-4BA270CE9DBD}.Release|Any CPU.Build.0 = Release|Any CPU 37 | {120C431E-479C-48C7-9539-CFA32399769C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 38 | {120C431E-479C-48C7-9539-CFA32399769C}.Debug|Any CPU.Build.0 = Debug|Any CPU 39 | {120C431E-479C-48C7-9539-CFA32399769C}.Release|Any CPU.ActiveCfg = Release|Any CPU 40 | {120C431E-479C-48C7-9539-CFA32399769C}.Release|Any CPU.Build.0 = Release|Any CPU 41 | {A47ED1CE-FE7C-444E-9391-10D6B60519C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 42 | {A47ED1CE-FE7C-444E-9391-10D6B60519C2}.Debug|Any CPU.Build.0 = Debug|Any CPU 43 | {A47ED1CE-FE7C-444E-9391-10D6B60519C2}.Release|Any CPU.ActiveCfg = Release|Any CPU 44 | {A47ED1CE-FE7C-444E-9391-10D6B60519C2}.Release|Any CPU.Build.0 = Release|Any CPU 45 | EndGlobalSection 46 | GlobalSection(SolutionProperties) = preSolution 47 | HideSolutionNode = FALSE 48 | EndGlobalSection 49 | GlobalSection(NestedProjects) = preSolution 50 | {94E6E098-11A0-43CF-B0CF-4BA270CE9DBD} = {037440DE-440B-4129-9F7A-09B42D00397E} 51 | {120C431E-479C-48C7-9539-CFA32399769C} = {2B558B69-8F95-4F82-B223-EBF60F6F31EE} 52 | {A47ED1CE-FE7C-444E-9391-10D6B60519C2} = {FD377716-21BA-45D1-9580-02C2BECA5BAB} 53 | EndGlobalSection 54 | EndGlobal 55 | -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/LoggerConfigurationLogglyExtensions.cs: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Serilog Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | using System; 16 | using System.Collections.Generic; 17 | using System.Linq; 18 | using Serilog.Configuration; 19 | using Serilog.Core; 20 | using Serilog.Events; 21 | using Serilog.Sinks.Loggly; 22 | 23 | namespace Serilog 24 | { 25 | /// 26 | /// Adds the WriteTo.Loggly() extension method to . 27 | /// 28 | public static class LoggerConfigurationLogglyExtensions 29 | { 30 | /// 31 | /// Adds a sink that writes log events to the Loggly.com webservice. Properties are being send as data and the level is used as category. 32 | /// 33 | /// The logger configuration. 34 | /// The minimum log event level required in order to write an event to the sink. 35 | /// Supplies culture-specific formatting information, or null. 36 | /// The maximum number of events to post in a single batch. 37 | /// The time to wait between checking for event batches. 38 | /// Path for a set of files that will be used to buffer events until they 39 | /// can be successfully transmitted across the network. Individual files will be created using the 40 | /// pattern -{Date}.json. 41 | /// The maximum size, in bytes, to which the buffer 42 | /// log file for a specific date will be allowed to grow. By default no limit will be applied. 43 | /// The maximum size, in bytes, that the JSON representation of 44 | /// an event may take before it is dropped rather than being sent to the Loggly server. Specify null for no limit. 45 | /// The default is 1 MB. 46 | /// If provided, the switch will be updated based on the Seq server's level setting 47 | /// for the corresponding API key. Passing the same key to MinimumLevel.ControlledBy() will make the whole pipeline 48 | /// dynamically controlled. Do not specify with this setting. 49 | /// A soft limit for the number of bytes to use for storing failed requests. 50 | /// The limit is soft in that it can be exceeded by any single error payload, but in that case only that single error 51 | /// payload will be retained. 52 | /// number of files to retain for the buffer. If defined, this also controls which records 53 | /// in the buffer get sent to the remote Loggly instance 54 | /// Token used to identify the Loggly account to use 55 | /// Comma-delimited list of tags to submit to Loggly 56 | /// Hostname to send logs to. Defaults to logs-01.loggly.com. 57 | /// Used to configure underlying LogglyClient programmaticaly. Otherwise use app.Config. 58 | /// Decides if the sink should include specific properties in the log message 59 | /// Logger configuration, allowing configuration to continue. 60 | /// A required parameter is null. 61 | public static LoggerConfiguration Loggly( 62 | this LoggerSinkConfiguration loggerConfiguration, 63 | LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum, 64 | int batchPostingLimit = LogglySink.DefaultBatchPostingLimit, 65 | TimeSpan? period = null, 66 | IFormatProvider formatProvider = null, 67 | string bufferBaseFilename = null, 68 | long? bufferFileSizeLimitBytes = null, 69 | long? eventBodyLimitBytes = 1024 * 1024, 70 | LoggingLevelSwitch controlLevelSwitch = null, 71 | long? retainedInvalidPayloadsLimitBytes = null, 72 | int? retainedFileCountLimit = null, 73 | string customerToken = null, 74 | string tags = null, 75 | string endpointHostName = null, 76 | LogglyConfiguration logglyConfig = null, 77 | LogIncludes includes = null) 78 | { 79 | if (loggerConfiguration == null) throw new ArgumentNullException(nameof(loggerConfiguration)); 80 | if (bufferFileSizeLimitBytes.HasValue && bufferFileSizeLimitBytes < 0) 81 | throw new ArgumentOutOfRangeException(nameof(bufferFileSizeLimitBytes), "Negative value provided; file size limit must be non-negative."); 82 | 83 | var defaultedPeriod = period ?? LogglySink.DefaultPeriod; 84 | 85 | ILogEventSink sink; 86 | 87 | LogglyConfiguration constructedLogglyConfig = null; 88 | if (customerToken != null) 89 | { 90 | constructedLogglyConfig = new LogglyConfiguration 91 | { 92 | CustomerToken = customerToken, 93 | Tags = tags?.Split(',').ToList() ?? new List(), 94 | EndpointHostName = endpointHostName 95 | }; 96 | } 97 | 98 | var activeLogglyConfig = logglyConfig ?? constructedLogglyConfig; 99 | 100 | if (bufferBaseFilename == null) 101 | { 102 | sink = new LogglySink(formatProvider, batchPostingLimit, defaultedPeriod, activeLogglyConfig, includes); 103 | } 104 | else 105 | { 106 | sink = new DurableLogglySink( 107 | bufferBaseFilename, 108 | batchPostingLimit, 109 | defaultedPeriod, 110 | bufferFileSizeLimitBytes, 111 | eventBodyLimitBytes, 112 | controlLevelSwitch, 113 | retainedInvalidPayloadsLimitBytes, 114 | retainedFileCountLimit, 115 | formatProvider, 116 | activeLogglyConfig, 117 | includes); 118 | } 119 | 120 | return loggerConfiguration.Sink(sink, restrictedToMinimumLevel); 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Reflection; 3 | using System.Runtime.CompilerServices; 4 | 5 | [assembly: AssemblyVersion("3.1.0.0")] 6 | 7 | [assembly: CLSCompliant(true)] 8 | 9 | [assembly: InternalsVisibleTo("Serilog.Sinks.Loggly.Tests, PublicKey=00240000048000009400000006020000" + 10 | "00240000525341310004000001000100" + 11 | "FB8D13FD344A1C6FE0FE83EF33C1080B" + 12 | "F30690765BC6EB0DF26EBFDF8F21670C" + 13 | "64265B30DB09F73A0DEA5B3DB4C9D18D" + 14 | "BF6D5A25AF5CE9016F281014D79DC3B4" + 15 | "201AC646C451830FC7E61A2DFD633D34" + 16 | "C39F87B81894191652DF5AC63CC40C77" + 17 | "F3542F702BDA692E6E8A9158353DF189" + 18 | "007A49DA0F3CFD55EB250066B19485EC")] 19 | 20 | [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=00240000048000009400000006020000002" + 21 | "40000525341310004000001000100c547ca" + 22 | "c37abd99c8db225ef2f6c8a3602f3b3606c" + 23 | "c9891605d02baa56104f4cfc0734aa39b93" + 24 | "bf7852f7d9266654753cc297e7d2edfe0ba" + 25 | "c1cdcf9f717241550e0a7b191195b7667bb" + 26 | "4f64bcb8e2121380fd1d9d46ad2d92d2d15" + 27 | "605093924cceaf74c4861eff62abf69b929" + 28 | "1ed0a340e113be11e6a7d3113e92484cf70" + 29 | "45cc7")] -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Serilog.Sinks.Loggly.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Serilog sink for Loggly.com service 5 | 5.5.0 6 | Serilog Contributors;Michiel van Oudheusden 7 | net45;netstandard1.5;netstandard2.0 8 | Serilog.Sinks.Loggly 9 | ../../assets/Serilog.snk 10 | true 11 | true 12 | Serilog.Sinks.Loggly 13 | serilog;logging;Loggly;error 14 | http://serilog.net/images/serilog-sink-nuget.png 15 | http://serilog.net 16 | http://www.apache.org/licenses/LICENSE-2.0 17 | https://github.com/serilog/serilog-sinks-loggly 18 | git 19 | false 20 | Serilog 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | $(DefineConstants);HRESULTS 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/ControlledSwitch.cs: -------------------------------------------------------------------------------- 1 | // Serilog.Sinks.Seq Copyright 2016 Serilog Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | using Serilog.Core; 16 | using Serilog.Events; 17 | 18 | namespace Serilog.Sinks.Loggly 19 | { 20 | /// 21 | /// Instances of this type are single-threaded, generally only updated on a background 22 | /// timer thread. An exception is , which may be called 23 | /// concurrently but performs no synchronization. 24 | /// 25 | class ControlledLevelSwitch 26 | { 27 | // If non-null, then background level checks will be performed; set either through the constructor 28 | // or in response to a level specification from the server. Never set to null after being made non-null. 29 | LoggingLevelSwitch _controlledSwitch; 30 | LogEventLevel? _originalLevel; 31 | 32 | public ControlledLevelSwitch(LoggingLevelSwitch controlledSwitch = null) 33 | { 34 | _controlledSwitch = controlledSwitch; 35 | } 36 | 37 | public bool IsActive => _controlledSwitch != null; 38 | 39 | public bool IsIncluded(LogEvent evt) 40 | { 41 | // Concurrent, but not synchronized. 42 | var controlledSwitch = _controlledSwitch; 43 | return controlledSwitch == null || 44 | (int)controlledSwitch.MinimumLevel <= (int)evt.Level; 45 | } 46 | 47 | public void Update(LogEventLevel? minimumAcceptedLevel) 48 | { 49 | if (minimumAcceptedLevel == null) 50 | { 51 | if (_controlledSwitch != null && _originalLevel.HasValue) 52 | _controlledSwitch.MinimumLevel = _originalLevel.Value; 53 | 54 | return; 55 | } 56 | 57 | if (_controlledSwitch == null) 58 | { 59 | // The server is controlling the logging level, but not the overall logger. Hence, if the server 60 | // stops controlling the level, the switch should become transparent. 61 | _originalLevel = LevelAlias.Minimum; 62 | _controlledSwitch = new LoggingLevelSwitch(minimumAcceptedLevel.Value); 63 | return; 64 | } 65 | 66 | if (!_originalLevel.HasValue) 67 | _originalLevel = _controlledSwitch.MinimumLevel; 68 | 69 | _controlledSwitch.MinimumLevel = minimumAcceptedLevel.Value; 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/Durable/FileSetPosition.cs: -------------------------------------------------------------------------------- 1 | namespace Serilog.Sinks.Loggly.Durable 2 | { 3 | public class FileSetPosition 4 | { 5 | public FileSetPosition(long position, string fileFullPath) 6 | { 7 | NextLineStart = position; 8 | File = fileFullPath; 9 | } 10 | 11 | public long NextLineStart { get; } 12 | public string File { get; } 13 | 14 | public static readonly FileSetPosition None = default(FileSetPosition); 15 | } 16 | } -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/DurableLogglySink.cs: -------------------------------------------------------------------------------- 1 | // Serilog.Sinks.Seq Copyright 2016 Serilog Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | using System; 16 | using System.Text; 17 | using Serilog.Core; 18 | using Serilog.Events; 19 | using Serilog.Sinks.RollingFile; 20 | 21 | namespace Serilog.Sinks.Loggly 22 | { 23 | class DurableLogglySink : ILogEventSink, IDisposable 24 | { 25 | readonly HttpLogShipper _shipper; 26 | readonly RollingFileSink _sink; 27 | 28 | public DurableLogglySink( 29 | string bufferBaseFilename, 30 | int batchPostingLimit, 31 | TimeSpan period, 32 | long? bufferFileSizeLimitBytes, 33 | long? eventBodyLimitBytes, 34 | LoggingLevelSwitch levelControlSwitch, 35 | long? retainedInvalidPayloadsLimitBytes, 36 | int? retainedFileCountLimit = null, 37 | IFormatProvider formatProvider = null, 38 | LogglyConfiguration logglyConfiguration = null, 39 | LogIncludes includes = null) 40 | { 41 | if (bufferBaseFilename == null) throw new ArgumentNullException(nameof(bufferBaseFilename)); 42 | 43 | //use a consistent UTF encoding with BOM so no confusion will exist when reading / deserializing 44 | var encoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier:false); 45 | 46 | //handles sending events to Loggly's API through LogglyClient and manages the pending list 47 | _shipper = new HttpLogShipper( 48 | bufferBaseFilename, 49 | batchPostingLimit, 50 | period, 51 | eventBodyLimitBytes, 52 | levelControlSwitch, 53 | retainedInvalidPayloadsLimitBytes, 54 | encoding, 55 | retainedFileCountLimit, 56 | logglyConfiguration); 57 | 58 | //writes events to the file to support connection recovery 59 | _sink = new RollingFileSink( 60 | bufferBaseFilename + "-{Date}.json", 61 | new LogglyFormatter(formatProvider, includes), //serializes as LogglyEvent 62 | bufferFileSizeLimitBytes, 63 | retainedFileCountLimit, 64 | encoding); 65 | } 66 | 67 | public void Dispose() 68 | { 69 | _sink.Dispose(); 70 | _shipper.Dispose(); 71 | } 72 | 73 | public void Emit(LogEvent logEvent) 74 | { 75 | // This is a lagging indicator, but the network bandwidth usage benefits 76 | // are worth the ambiguity. 77 | if (_shipper.IsIncluded(logEvent)) 78 | { 79 | _sink.Emit(logEvent); 80 | } 81 | } 82 | } 83 | } -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/ExceptionDetails.cs: -------------------------------------------------------------------------------- 1 | namespace Serilog.Sinks.Loggly 2 | { 3 | class ExceptionDetails 4 | { 5 | public ExceptionDetails(string type, 6 | string message, 7 | string stackTrace, 8 | ExceptionDetails[] innerExceptions) 9 | { 10 | Type = type; 11 | Message = message; 12 | StackTrace = stackTrace; 13 | InnerExceptions = innerExceptions; 14 | } 15 | public string Type { get; } 16 | public string Message { get; } 17 | public string StackTrace { get; } 18 | public ExceptionDetails[] InnerExceptions { get; } 19 | } 20 | } -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/ExponentialBackoffConnectionSchedule.cs: -------------------------------------------------------------------------------- 1 | // Serilog.Sinks.Seq Copyright 2016 Serilog Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | using System; 16 | 17 | namespace Serilog.Sinks.Loggly 18 | { 19 | /// 20 | /// Based on the BatchedConnectionStatus class from . 21 | /// 22 | class ExponentialBackoffConnectionSchedule 23 | { 24 | static readonly TimeSpan MinimumBackoffPeriod = TimeSpan.FromSeconds(5); 25 | static readonly TimeSpan MaximumBackoffInterval = TimeSpan.FromMinutes(10); 26 | 27 | readonly TimeSpan _period; 28 | 29 | int _failuresSinceSuccessfulConnection; 30 | 31 | public ExponentialBackoffConnectionSchedule(TimeSpan period) 32 | { 33 | if (period < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(period), "The connection retry period must be a positive timespan"); 34 | 35 | _period = period; 36 | } 37 | 38 | public void MarkSuccess() 39 | { 40 | _failuresSinceSuccessfulConnection = 0; 41 | } 42 | 43 | public void MarkFailure() 44 | { 45 | ++_failuresSinceSuccessfulConnection; 46 | } 47 | 48 | public TimeSpan NextInterval 49 | { 50 | get 51 | { 52 | // Available, and first failure, just try the batch interval 53 | if (_failuresSinceSuccessfulConnection <= 1) return _period; 54 | 55 | // Second failure, start ramping up the interval - first 2x, then 4x, ... 56 | var backoffFactor = Math.Pow(2, (_failuresSinceSuccessfulConnection - 1)); 57 | 58 | // If the period is ridiculously short, give it a boost so we get some 59 | // visible backoff. 60 | var backoffPeriod = Math.Max(_period.Ticks, MinimumBackoffPeriod.Ticks); 61 | 62 | // The "ideal" interval 63 | var backedOff = (long)(backoffPeriod * backoffFactor); 64 | 65 | // Capped to the maximum interval 66 | var cappedBackoff = Math.Min(MaximumBackoffInterval.Ticks, backedOff); 67 | 68 | // Unless that's shorter than the base interval, in which case we'll just apply the period 69 | var actual = Math.Max(_period.Ticks, cappedBackoff); 70 | 71 | return TimeSpan.FromTicks(actual); 72 | } 73 | } 74 | } 75 | } 76 | 77 | -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/FileBasedBookmarkProvider.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Text; 4 | using Serilog.Debugging; 5 | using Serilog.Sinks.Loggly.Durable; 6 | 7 | namespace Serilog.Sinks.Loggly 8 | { 9 | class FileBasedBookmarkProvider : IBookmarkProvider 10 | { 11 | readonly IFileSystemAdapter _fileSystemAdapter; 12 | readonly Encoding _encoding; 13 | 14 | readonly string _bookmarkFilename; 15 | Stream _currentBookmarkFileStream; 16 | 17 | public FileBasedBookmarkProvider(string bufferBaseFilename, IFileSystemAdapter fileSystemAdapter, Encoding encoding) 18 | { 19 | _bookmarkFilename = Path.GetFullPath(bufferBaseFilename + ".bookmark"); 20 | _fileSystemAdapter = fileSystemAdapter; 21 | _encoding = encoding; 22 | } 23 | 24 | public void Dispose() 25 | { 26 | _currentBookmarkFileStream?.Dispose(); 27 | } 28 | 29 | public FileSetPosition GetCurrentBookmarkPosition() 30 | { 31 | EnsureCurrentBookmarkStreamIsOpen(); 32 | 33 | if (_currentBookmarkFileStream.Length != 0) 34 | { 35 | using (var bookmarkStreamReader = new StreamReader(_currentBookmarkFileStream, _encoding, false, 128, true)) 36 | { 37 | //set the position to 0, to begin reading the initial line 38 | bookmarkStreamReader.BaseStream.Position = 0; 39 | var bookmarkInfoLine = bookmarkStreamReader.ReadLine(); 40 | 41 | if (bookmarkInfoLine != null) 42 | { 43 | //reset position after read 44 | var parts = bookmarkInfoLine.Split(new[] {":::"}, StringSplitOptions.RemoveEmptyEntries); 45 | if (parts.Length == 2 && long.TryParse(parts[0], out long position)) 46 | { 47 | return new FileSetPosition(position, parts[1]); 48 | } 49 | 50 | SelfLog.WriteLine("Unable to read a line correctly from bookmark file"); 51 | } 52 | else 53 | { 54 | SelfLog.WriteLine( 55 | "For some unknown reason, we were unable to read the non-empty bookmark info..."); 56 | } 57 | } 58 | } 59 | 60 | //bookmark file is empty or has been misread, so return a null bookmark 61 | return null; 62 | } 63 | 64 | public void UpdateBookmark(FileSetPosition newBookmark) 65 | { 66 | EnsureCurrentBookmarkStreamIsOpen(); 67 | 68 | using (var bookmarkStreamWriter = new StreamWriter(_currentBookmarkFileStream, _encoding, 128, true)) 69 | { 70 | bookmarkStreamWriter.BaseStream.Position = 0; 71 | bookmarkStreamWriter.WriteLine("{0}:::{1}", newBookmark.NextLineStart, newBookmark.File); 72 | bookmarkStreamWriter.Flush(); 73 | } 74 | } 75 | 76 | void EnsureCurrentBookmarkStreamIsOpen() 77 | { 78 | //this will ensure a stream is available, even if it means creating a new file associated to it 79 | if (_currentBookmarkFileStream == null) 80 | _currentBookmarkFileStream = _fileSystemAdapter.Open( 81 | _bookmarkFilename, 82 | FileMode.OpenOrCreate, 83 | FileAccess.ReadWrite, 84 | FileShare.Read); 85 | } 86 | 87 | 88 | } 89 | } -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/FileBufferDataProvider.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Text; 6 | using Loggly; 7 | using Newtonsoft.Json; 8 | using Serilog.Debugging; 9 | using Serilog.Sinks.Loggly.Durable; 10 | #if HRESULTS 11 | using System.Runtime.InteropServices; 12 | #endif 13 | 14 | namespace Serilog.Sinks.Loggly 15 | { 16 | interface IBufferDataProvider 17 | { 18 | IEnumerable GetNextBatchOfEvents(); 19 | void MarkCurrentBatchAsProcessed(); 20 | void MoveBookmarkForward(); 21 | } 22 | 23 | /// 24 | /// Provides a facade to all File operations, namely bookmark management and 25 | /// buffered data readings 26 | /// 27 | internal class FileBufferDataProvider : IBufferDataProvider 28 | { 29 | #if HRESULTS 30 | //for Marshalling error checks 31 | const int ErrorSharingViolation = 32; 32 | const int ErrorLockViolation = 33; 33 | #endif 34 | readonly string _candidateSearchPath; 35 | readonly string _logFolder; 36 | 37 | readonly int _batchPostingLimit; 38 | readonly long? _eventBodyLimitBytes; 39 | readonly int? _retainedFileCountLimit; 40 | 41 | readonly IFileSystemAdapter _fileSystemAdapter; 42 | readonly IBookmarkProvider _bookmarkProvider; 43 | readonly Encoding _encoding; 44 | 45 | readonly JsonSerializer _serializer = JsonSerializer.Create(); 46 | 47 | // the following fields control the internal state and position of the queue 48 | protected FileSetPosition CurrentBookmark { get; set; } 49 | FileSetPosition _futureBookmark; 50 | IEnumerable _currentBatchOfEventsToProcess; 51 | 52 | public FileBufferDataProvider( 53 | string baseBufferFileName, 54 | IFileSystemAdapter fileSystemAdapter, 55 | IBookmarkProvider bookmarkProvider, 56 | Encoding encoding, 57 | int batchPostingLimit, 58 | long? eventBodyLimitBytes, 59 | int? retainedFileCountLimit) 60 | { 61 | //construct a valid path to a file in the log folder to get the folder path: 62 | _logFolder = Path.GetDirectoryName(Path.GetFullPath(baseBufferFileName + ".bookmark")); 63 | _candidateSearchPath = Path.GetFileName(baseBufferFileName) + "*.json"; 64 | 65 | _fileSystemAdapter = fileSystemAdapter; 66 | _bookmarkProvider = bookmarkProvider; 67 | _encoding = encoding; 68 | _batchPostingLimit = batchPostingLimit; 69 | _eventBodyLimitBytes = eventBodyLimitBytes; 70 | _retainedFileCountLimit = retainedFileCountLimit; 71 | } 72 | 73 | public IEnumerable GetNextBatchOfEvents() 74 | { 75 | //if current batch has not yet been processed, return it 76 | if (_currentBatchOfEventsToProcess != null) 77 | return _currentBatchOfEventsToProcess; 78 | 79 | //if we have a bookmark in place, it may be the next position to read from 80 | // otherwise try to get a valid one 81 | if (CurrentBookmark == null) 82 | { 83 | //read the current bookmark from file, and if invalid, try to create a valid one 84 | CurrentBookmark = TryGetValidBookmark(); 85 | 86 | if (!IsValidBookmark(CurrentBookmark)) 87 | return Enumerable.Empty(); 88 | } 89 | 90 | //bookmark is valid, so lets get the next batch from the files. 91 | RefreshCurrentListOfEvents(); 92 | 93 | //this should never return null. If there is nothing to return, please return an empty list instead. 94 | return _currentBatchOfEventsToProcess ?? Enumerable.Empty(); 95 | } 96 | 97 | public void MarkCurrentBatchAsProcessed() 98 | { 99 | //reset internal state: only write to the bookmark file if we move forward. 100 | //otherwise, there is a risk of rereading the current (first) buffer file again 101 | if(_futureBookmark != null) 102 | _bookmarkProvider.UpdateBookmark(_futureBookmark); 103 | 104 | //we can move the marker to what's in "future" (next expected position) 105 | CurrentBookmark = _futureBookmark; 106 | _currentBatchOfEventsToProcess = null; 107 | } 108 | 109 | public void MoveBookmarkForward() 110 | { 111 | //curren Batch is empty, so we should clear it out so that the enxt read cycle will refresh it correctly 112 | _currentBatchOfEventsToProcess = null; 113 | 114 | // Only advance the bookmark if no other process has the 115 | // current file locked, and its length is as we found it. 116 | // NOTE: we will typically enter this method after any buffer file is finished 117 | // (no events read from previous file). This is the oportunity to clear out files 118 | // especially the prevously read file 119 | 120 | var fileSet = GetEventBufferFileSet(); 121 | 122 | try 123 | { 124 | //if we only have two files, move to the next one imediately, unless a locking situation 125 | // impeads us from doing so 126 | if (fileSet.Length == 2 127 | && fileSet.First() == CurrentBookmark.File 128 | && IsUnlockedAtLength(CurrentBookmark.File, CurrentBookmark.NextLineStart)) 129 | { 130 | //move to next file 131 | CurrentBookmark = new FileSetPosition(0, fileSet[1]); 132 | //we can also delete the previously read file since we no longer need it 133 | _fileSystemAdapter.DeleteFile(fileSet[0]); 134 | } 135 | 136 | if (fileSet.Length > 2) 137 | { 138 | //determine where in the fileset on disk is the current bookmark (if any) 139 | //there is no garantee the the current one is the first, so we need to make 140 | //sure we start reading the next one based on the current one 141 | var currentBookMarkedFileInFileSet = CurrentBookmark != null 142 | ? Array.FindIndex(fileSet, f => f == CurrentBookmark.File) 143 | : -1; 144 | 145 | //when we have more files, we want to delete older ones, but this depends on the 146 | // limit retention policy. If no limit retention policy is in place, the intent is to 147 | // send all messages, no matter how old. In this case, we should only delete the current 148 | // file (since we are finished with it) and start the bookmark at the next one. If we do have some 149 | // retention policy in place, then delete anything older then the limit and the next 150 | // message read should be at the start of the policy limit 151 | if (_retainedFileCountLimit.HasValue) 152 | { 153 | if (fileSet.Length <= _retainedFileCountLimit.Value) 154 | { 155 | //start 156 | var fileIndexToStartAt = Math.Max(0, currentBookMarkedFileInFileSet + 1); 157 | 158 | //if index of current is not the last , use next; otherwise preserve current 159 | CurrentBookmark = fileIndexToStartAt <= fileSet.Length - 1 160 | ? new FileSetPosition(0, fileSet[fileIndexToStartAt]) 161 | : CurrentBookmark; 162 | 163 | //delete all the old files 164 | DeleteFilesInFileSetUpToIndex(fileSet, fileIndexToStartAt); 165 | } 166 | else 167 | { 168 | //start at next or first in retention count 169 | var fileIndexToStartAt = Math.Max(fileSet.Length - _retainedFileCountLimit.Value, currentBookMarkedFileInFileSet +1); 170 | 171 | CurrentBookmark =new FileSetPosition(0, fileSet[fileIndexToStartAt]); 172 | 173 | //delete all the old files 174 | DeleteFilesInFileSetUpToIndex(fileSet, fileIndexToStartAt); 175 | } 176 | } 177 | else 178 | { 179 | // not sure this can occur, but if for some reason the file is no longer in the list 180 | // we should start from the beginning, maybe; being a bit defensive here due to 181 | // https://github.com/serilog/serilog-sinks-loggly/issues/25 182 | if (currentBookMarkedFileInFileSet == -1) 183 | { 184 | //if not in file set, use first in set (or none) 185 | CurrentBookmark = fileSet.Length > 0 186 | ? new FileSetPosition(0, fileSet[0]) 187 | : null; 188 | } 189 | else 190 | { 191 | //if index of current is not the last , use next; otherwise preserve current 192 | CurrentBookmark = currentBookMarkedFileInFileSet <= fileSet.Length - 2 193 | ? new FileSetPosition(0, fileSet[currentBookMarkedFileInFileSet + 1]) 194 | : CurrentBookmark; 195 | } 196 | 197 | //also clear all the previous files in the set to avoid problems (and because 198 | //they should no longer be considered). If no previous exists (index is -1) 199 | //keep existing; also do not reomve last file as it may be written to / locked 200 | DeleteFilesInFileSetUpToIndex(fileSet, currentBookMarkedFileInFileSet + 1); 201 | } 202 | } 203 | } 204 | catch (Exception ex) 205 | { 206 | SelfLog.WriteLine("An error occured while deleteing the files...{0}", ex.Message); 207 | } 208 | finally 209 | { 210 | //even if reading / deleteing files fails, we can / should update the bookmark file 211 | //it is important that the file have the reset position, otherwise we risk failing to 212 | // move forward in the next read cycle 213 | //it's possible that no bookmark exists, especially if no valid messages have forced a 214 | // durable log file to be created. In this case, the bookmark file will be empty and 215 | // on disk 216 | if(CurrentBookmark != null) 217 | _bookmarkProvider.UpdateBookmark(CurrentBookmark); 218 | } 219 | } 220 | 221 | void DeleteFilesInFileSetUpToIndex(string[] fileSet, int fileIndexToDeleteUpTo) 222 | { 223 | for (int i = 0; i < fileIndexToDeleteUpTo && i < fileSet.Length - 1; i++) 224 | { 225 | _fileSystemAdapter.DeleteFile(fileSet[i]); 226 | } 227 | } 228 | 229 | bool IsUnlockedAtLength(string file, long maxLen) 230 | { 231 | try 232 | { 233 | using (var fileStream = _fileSystemAdapter.Open(file, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read)) 234 | { 235 | return fileStream.Length <= maxLen; 236 | } 237 | } 238 | #if HRESULTS 239 | catch (IOException ex) 240 | { 241 | //NOTE: this seems to be a way to check for file lock validations as in : 242 | // https://stackoverflow.com/questions/16642858/files-how-to-distinguish-file-lock-and-permission-denied-cases 243 | //sharing violation and LockViolation are expected, and we can follow trough if they occur 244 | 245 | var errorCode = Marshal.GetHRForException(ex) & ((1 << 16) - 1); 246 | if (errorCode != ErrorSharingViolation && errorCode != ErrorLockViolation ) 247 | { 248 | SelfLog.WriteLine("Unexpected I/O exception while testing locked status of {0}: {1}", file, ex); 249 | } 250 | } 251 | #else 252 | catch (IOException ex) 253 | { 254 | // Where no HRESULT is available, assume IOExceptions indicate a locked file 255 | SelfLog.WriteLine("Unexpected IOException while testing locked status of {0}: {1}", file, ex); 256 | } 257 | #endif 258 | catch (Exception ex) 259 | { 260 | SelfLog.WriteLine("Unexpected exception while testing locked status of {0}: {1}", file, ex); 261 | } 262 | 263 | return false; 264 | } 265 | 266 | void RefreshCurrentListOfEvents() 267 | { 268 | var events = new List(); 269 | var count = 0; 270 | var positionTracker = CurrentBookmark.NextLineStart; 271 | 272 | using (var currentBufferStream = _fileSystemAdapter.Open(CurrentBookmark.File, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) 273 | { 274 | while (count < _batchPostingLimit && TryReadLine(currentBufferStream, ref positionTracker, out string readLine)) 275 | { 276 | // Count is the indicator that work was done, so advances even in the (rare) case an 277 | // oversized event is dropped. 278 | ++count; 279 | 280 | if (_eventBodyLimitBytes.HasValue 281 | && readLine != null 282 | && _encoding.GetByteCount(readLine) > _eventBodyLimitBytes.Value) 283 | { 284 | SelfLog.WriteLine( 285 | "Event JSON representation exceeds the byte size limit of {0} and will be dropped; data: {1}", 286 | _eventBodyLimitBytes, readLine); 287 | } 288 | 289 | if (!readLine.StartsWith("{")) 290 | { 291 | //in some instances this can happen. TryReadLine no longer assumes a BOM if reading from the file start, 292 | //but there may be (unobserved yet) situations where the line read is still not complete and valid 293 | // Json. This and the try catch that follows are, therefore, attempts to preserve the 294 | //logging functionality active, though some events may be dropped in the process. 295 | SelfLog.WriteLine( 296 | "Event JSON representation does not start with the expected '{{' character. " + 297 | "This may be related to a BOM issue in the buffer file. Event will be dropped; data: {0}", 298 | readLine); 299 | } 300 | else 301 | { 302 | try 303 | { 304 | events.Add(DeserializeEvent(readLine)); 305 | } 306 | catch (Exception ex) 307 | { 308 | SelfLog.WriteLine( 309 | "Unable to deserialize the json event; Event will be dropped; exception: {0}; data: {1}", 310 | ex.Message, readLine); 311 | } 312 | } 313 | } 314 | } 315 | 316 | _futureBookmark = new FileSetPosition(positionTracker, CurrentBookmark.File); 317 | _currentBatchOfEventsToProcess = events; 318 | } 319 | 320 | // It would be ideal to chomp whitespace here, but not required. 321 | bool TryReadLine(Stream current, ref long nextStart, out string readLine) 322 | { 323 | // determine if we are reading the first line in the file. This will help with 324 | // solving the BOM marker issue ahead 325 | var firstline = nextStart == 0; 326 | 327 | if (current.Length <= nextStart) 328 | { 329 | readLine = null; 330 | return false; 331 | } 332 | 333 | // Important not to dispose this StreamReader as the stream must remain open. 334 | using (var reader = new StreamReader(current, _encoding, false, 128, true)) 335 | { 336 | // ByteOrder marker may still be a problem if we a reading the first line. We can test for it 337 | // directly from the stream. This should only affect the first readline op, anyways. Since we 338 | // If it's there, we need to move the start index by 3 bytes, so position will be correct throughout 339 | if (firstline && StreamContainsBomMarker(current)) 340 | { 341 | nextStart += 3; 342 | } 343 | 344 | //readline moves the marker forward farther then the line length, so it needs to be placed 345 | // at the right position. This makes sure we try to read a line from the right starting point 346 | current.Position = nextStart; 347 | readLine = reader.ReadLine(); 348 | 349 | if (readLine == null) 350 | return false; 351 | 352 | //If we have read the line, advance the count by the number of bytes + newline bytes to 353 | //mark the start of the next line 354 | nextStart += _encoding.GetByteCount(readLine) + _encoding.GetByteCount(Environment.NewLine); 355 | 356 | return true; 357 | } 358 | } 359 | 360 | static bool StreamContainsBomMarker(Stream current) 361 | { 362 | bool isBom = false; 363 | long currentPosition = current.Position; //save to reset after BOM check 364 | 365 | byte[] potentialBomMarker = new byte[3]; 366 | current.Position = 0; 367 | current.Read(potentialBomMarker, 0, 3); 368 | //BOM is "ef bb bf" => 239 187 191 369 | if (potentialBomMarker[0] == 239 370 | && potentialBomMarker[1] == 187 371 | && potentialBomMarker[2] == 191) 372 | { 373 | isBom = true; 374 | } 375 | 376 | current.Position = currentPosition; //put position back where it was 377 | return isBom; 378 | } 379 | 380 | LogglyEvent DeserializeEvent(string eventLine) 381 | { 382 | return _serializer.Deserialize(new JsonTextReader(new StringReader(eventLine))); 383 | } 384 | 385 | FileSetPosition TryGetValidBookmark() 386 | { 387 | //get from the bookmark file first 388 | FileSetPosition newBookmark = _bookmarkProvider.GetCurrentBookmarkPosition(); 389 | 390 | if (!IsValidBookmark(newBookmark)) 391 | { 392 | newBookmark = CreateFreshBookmarkBasedOnBufferFiles(); 393 | } 394 | 395 | return newBookmark; 396 | } 397 | 398 | FileSetPosition CreateFreshBookmarkBasedOnBufferFiles() 399 | { 400 | var fileSet = GetEventBufferFileSet(); 401 | 402 | //the new bookmark should consider file retention rules, if any 403 | // if no retention rule is in place (send all data to loggly, no matter how old) 404 | // then take the first file and make a FileSetPosition out of it, 405 | // otherwise, make the position marker relative to the oldest file as in the rule 406 | //NOTE: this only happens when the previous bookmark is invalid (that's how we 407 | // entered this method) so , if the prevous bookmark points to a valid file 408 | // that will continue to be read till the end. 409 | if (_retainedFileCountLimit.HasValue 410 | && fileSet.Length > _retainedFileCountLimit.Value) 411 | { 412 | //we have more files then our rule requires (older than needed) 413 | // so point to the oldest allowed by our rule 414 | return new FileSetPosition(0, fileSet.Skip(fileSet.Length - _retainedFileCountLimit.Value).First()); 415 | } 416 | 417 | return fileSet.Any() ? new FileSetPosition(0, fileSet.First()) : null; 418 | } 419 | 420 | bool IsValidBookmark(FileSetPosition bookmark) 421 | { 422 | return bookmark?.File != null 423 | && _fileSystemAdapter.Exists(bookmark.File); 424 | } 425 | 426 | string[] GetEventBufferFileSet() 427 | { 428 | return _fileSystemAdapter.GetFiles(_logFolder, _candidateSearchPath) 429 | .OrderBy(name => name) 430 | .ToArray(); 431 | } 432 | } 433 | } -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/FileSystemAdapter.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | 3 | namespace Serilog.Sinks.Loggly 4 | { 5 | /// 6 | /// adapter to abstract away filesystem specific / coupled calls, especially using File and Directory 7 | /// 8 | class FileSystemAdapter : IFileSystemAdapter 9 | { 10 | //TODO: can we use Physical 11 | public bool Exists(string filePath) 12 | { 13 | return System.IO.File.Exists(filePath); 14 | } 15 | 16 | public void DeleteFile(string filePath) 17 | { 18 | System.IO.File.Delete(filePath); 19 | } 20 | 21 | public Stream Open(string filePath, FileMode mode, FileAccess access, FileShare share) 22 | { 23 | return System.IO.File.Open(filePath, mode, access, share); 24 | } 25 | 26 | public void WriteAllBytes(string filePath, byte[] bytesToWrite) 27 | { 28 | System.IO.File.WriteAllBytes(filePath, bytesToWrite); 29 | } 30 | 31 | public string[] GetFiles(string path, string searchPattern) 32 | { 33 | return Directory.GetFiles(path, searchPattern); 34 | } 35 | } 36 | } -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/HttpLogShipper.cs: -------------------------------------------------------------------------------- 1 | // Serilog.Sinks.Seq Copyright 2016 Serilog Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | using System; 16 | using System.IO; 17 | using System.Linq; 18 | using System.Text; 19 | using Serilog.Core; 20 | using Serilog.Debugging; 21 | using Serilog.Events; 22 | using System.Threading.Tasks; 23 | using Loggly; 24 | 25 | #if HRESULTS 26 | using System.Runtime.InteropServices; 27 | #endif 28 | 29 | namespace Serilog.Sinks.Loggly 30 | { 31 | class HttpLogShipper : IDisposable 32 | { 33 | readonly int _batchPostingLimit; 34 | readonly ExponentialBackoffConnectionSchedule _connectionSchedule; 35 | 36 | readonly object _stateLock = new object(); 37 | readonly PortableTimer _timer; 38 | readonly ControlledLevelSwitch _controlledSwitch; 39 | volatile bool _unloading; 40 | 41 | readonly LogglyClient _logglyClient; 42 | readonly IFileSystemAdapter _fileSystemAdapter = new FileSystemAdapter(); 43 | readonly FileBufferDataProvider _bufferDataProvider; 44 | readonly InvalidPayloadLogger _invalidPayloadLogger; 45 | 46 | public HttpLogShipper( 47 | string bufferBaseFilename, 48 | int batchPostingLimit, 49 | TimeSpan period, long? 50 | eventBodyLimitBytes, 51 | LoggingLevelSwitch levelControlSwitch, 52 | long? retainedInvalidPayloadsLimitBytes, 53 | Encoding encoding, 54 | int? retainedFileCountLimit, 55 | LogglyConfiguration logglyConfiguration) 56 | { 57 | _batchPostingLimit = batchPostingLimit; 58 | 59 | _controlledSwitch = new ControlledLevelSwitch(levelControlSwitch); 60 | _connectionSchedule = new ExponentialBackoffConnectionSchedule(period); 61 | 62 | if (logglyConfiguration != null) 63 | { 64 | var adapter = new LogglyConfigAdapter(); 65 | adapter.ConfigureLogglyClient(logglyConfiguration); 66 | } 67 | 68 | _logglyClient = new LogglyClient(); //we'll use the loggly client instead of HTTP directly 69 | 70 | //create necessary path elements 71 | var candidateSearchPath = Path.GetFileName(bufferBaseFilename) + "*.json"; 72 | var logFolder = Path.GetDirectoryName(candidateSearchPath); 73 | 74 | //Filebase is currently the only option available so we will stick with it directly (for now) 75 | var encodingToUse = encoding; 76 | var bookmarkProvider = new FileBasedBookmarkProvider(bufferBaseFilename, _fileSystemAdapter, encoding); 77 | _bufferDataProvider = new FileBufferDataProvider(bufferBaseFilename, _fileSystemAdapter, bookmarkProvider, encodingToUse, batchPostingLimit, eventBodyLimitBytes, retainedFileCountLimit); 78 | _invalidPayloadLogger = new InvalidPayloadLogger(logFolder, encodingToUse, _fileSystemAdapter, retainedInvalidPayloadsLimitBytes); 79 | 80 | _timer = new PortableTimer(c => OnTick()); 81 | SetTimer(); 82 | } 83 | 84 | void CloseAndFlush() 85 | { 86 | lock (_stateLock) 87 | { 88 | if (_unloading) 89 | return; 90 | 91 | _unloading = true; 92 | } 93 | 94 | _timer.Dispose(); 95 | 96 | OnTick().GetAwaiter().GetResult(); 97 | } 98 | 99 | public bool IsIncluded(LogEvent logEvent) 100 | { 101 | return _controlledSwitch.IsIncluded(logEvent); 102 | } 103 | 104 | /// 105 | public void Dispose() 106 | { 107 | CloseAndFlush(); 108 | } 109 | 110 | void SetTimer() 111 | { 112 | // Note, called under _stateLock 113 | _timer.Start(_connectionSchedule.NextInterval); 114 | } 115 | 116 | async Task OnTick() 117 | { 118 | LogEventLevel? minimumAcceptedLevel = LogEventLevel.Debug; 119 | 120 | try 121 | { 122 | //we'll use this to control the number of events read per cycle. If the batch limit is reached, 123 | //then there is probably more events queued and we should continue to read them. Otherwise, 124 | // we can wait for the next timer tick moment to see if anything new is available. 125 | int numberOfEventsRead; 126 | do 127 | { 128 | //this should consistently return the same batch of events until 129 | //a MarkAsProcessed message is sent to the provider. Never return a null, please... 130 | var payload = _bufferDataProvider.GetNextBatchOfEvents(); 131 | numberOfEventsRead = payload.Count(); 132 | 133 | if (numberOfEventsRead > 0) 134 | { 135 | //send the loggly events through the bulk API 136 | var result = await _logglyClient.Log(payload).ConfigureAwait(false); 137 | 138 | if (result.Code == ResponseCode.Success) 139 | { 140 | _connectionSchedule.MarkSuccess(); 141 | _bufferDataProvider.MarkCurrentBatchAsProcessed(); 142 | } 143 | else if (result.Code == ResponseCode.Error) 144 | { 145 | // The connection attempt was successful - the payload we sent was the problem. 146 | _connectionSchedule.MarkSuccess(); 147 | _bufferDataProvider.MarkCurrentBatchAsProcessed(); //move foward 148 | 149 | _invalidPayloadLogger.DumpInvalidPayload(result, payload); 150 | } 151 | else 152 | { 153 | _connectionSchedule.MarkFailure(); 154 | SelfLog.WriteLine("Received failed HTTP shipping result {0}: {1}", result.Code, 155 | result.Message); 156 | break; 157 | } 158 | } 159 | else 160 | { 161 | // For whatever reason, there's nothing waiting to send. This means we should try connecting again at the 162 | // regular interval, so mark the attempt as successful. 163 | _connectionSchedule.MarkSuccess(); 164 | 165 | // not getting any batch may mean our marker is off, or at the end of the current, old file. 166 | // Try to move foward and cleanup 167 | _bufferDataProvider.MoveBookmarkForward(); 168 | } 169 | } while (numberOfEventsRead == _batchPostingLimit); 170 | //keep sending as long as we can retrieve a full batch. If not, wait for next tick 171 | } 172 | catch (Exception ex) 173 | { 174 | SelfLog.WriteLine("Exception while emitting periodic batch from {0}: {1}", this, ex); 175 | _connectionSchedule.MarkFailure(); 176 | } 177 | finally 178 | { 179 | lock (_stateLock) 180 | { 181 | _controlledSwitch.Update(minimumAcceptedLevel); 182 | 183 | if (!_unloading) 184 | SetTimer(); 185 | } 186 | } 187 | } 188 | } 189 | } 190 | 191 | 192 | 193 | -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/IBookmarkProvider.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Serilog.Sinks.Loggly.Durable; 3 | 4 | namespace Serilog.Sinks.Loggly 5 | { 6 | interface IBookmarkProvider : IDisposable 7 | { 8 | FileSetPosition GetCurrentBookmarkPosition(); 9 | 10 | void UpdateBookmark(FileSetPosition newBookmark); 11 | } 12 | } -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/IFileSystemAdapter.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | 3 | namespace Serilog.Sinks.Loggly 4 | { 5 | interface IFileSystemAdapter 6 | { 7 | //file ops 8 | bool Exists(string filePath); 9 | void DeleteFile(string filePath); 10 | Stream Open(string bookmarkFilename, FileMode openOrCreate, FileAccess readWrite, FileShare read); 11 | void WriteAllBytes(string filePath, byte[] bytesToWrite); 12 | 13 | //directory ops 14 | string[] GetFiles(string folder, string searchTerms); 15 | 16 | } 17 | } -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/InvalidPayloadLogger.cs: -------------------------------------------------------------------------------- 1 | // Serilog.Sinks.Seq Copyright 2016 Serilog Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | using System; 16 | using System.IO; 17 | using System.Linq; 18 | using System.Text; 19 | using Serilog.Debugging; 20 | using System.Collections.Generic; 21 | using Loggly; 22 | using Newtonsoft.Json; 23 | 24 | namespace Serilog.Sinks.Loggly 25 | { 26 | class InvalidPayloadLogger 27 | { 28 | const string InvalidPayloadFilePrefix = "invalid-"; 29 | readonly string _logFolder; 30 | readonly long? _retainedInvalidPayloadsLimitBytes; 31 | readonly Encoding _encoding; 32 | readonly IFileSystemAdapter _fileSystemAdapter; 33 | readonly JsonSerializer _serializer = JsonSerializer.Create(); 34 | 35 | 36 | public InvalidPayloadLogger(string logFolder, Encoding encoding, IFileSystemAdapter fileSystemAdapter, long? retainedInvalidPayloadsLimitBytes = null) 37 | { 38 | _logFolder = logFolder; 39 | _encoding = encoding; 40 | _fileSystemAdapter = fileSystemAdapter; 41 | _retainedInvalidPayloadsLimitBytes = retainedInvalidPayloadsLimitBytes; 42 | } 43 | 44 | public void DumpInvalidPayload(LogResponse result, IEnumerable payload) 45 | { 46 | var invalidPayloadFilename = $"{InvalidPayloadFilePrefix}{DateTime.UtcNow.ToString("yyyyMMddHHmmss")}-{result.Code}-{Guid.NewGuid():n}.json"; 47 | var invalidPayloadFile = Path.Combine(_logFolder, invalidPayloadFilename); 48 | SelfLog.WriteLine("HTTP shipping failed with {0}: {1}; dumping payload to {2}", result.Code, result.Message, invalidPayloadFile); 49 | 50 | byte[] bytesToWrite = SerializeLogglyEventsToBytes(payload); 51 | 52 | if (_retainedInvalidPayloadsLimitBytes.HasValue) 53 | { 54 | CleanUpInvalidPayloadFiles(_retainedInvalidPayloadsLimitBytes.Value - bytesToWrite.Length, _logFolder); 55 | } 56 | 57 | //Adding this to perist WHY the invalid payload existed 58 | // the library is not using these files to resend data, so format is not important. 59 | var errorBytes = _encoding.GetBytes(string.Format(@"Error info: HTTP shipping failed with {0}: {1}", result.Code, result.Message)); 60 | _fileSystemAdapter.WriteAllBytes(invalidPayloadFile, bytesToWrite.Concat(errorBytes).ToArray()); 61 | } 62 | 63 | byte[] SerializeLogglyEventsToBytes(IEnumerable events) 64 | { 65 | SelfLog.WriteLine("Newline to use: {0}", Environment.NewLine.Length == 2 ? "rn":"n"); 66 | using (StringWriter writer = new StringWriter() { NewLine = Environment.NewLine }) 67 | { 68 | foreach (var logglyEvent in events) 69 | { 70 | _serializer.Serialize(writer, logglyEvent); 71 | writer.Write(Environment.NewLine); 72 | } 73 | 74 | SelfLog.WriteLine("serialized events: {0}", writer.ToString()); 75 | 76 | byte[] bytes = _encoding.GetBytes(writer.ToString()); 77 | SelfLog.WriteLine("encoded events ending: {0} {1}", bytes[bytes.Length-2], bytes[bytes.Length-1]); 78 | return _encoding.GetBytes(writer.ToString()); 79 | } 80 | } 81 | 82 | static void CleanUpInvalidPayloadFiles(long maxNumberOfBytesToRetain, string logFolder) 83 | { 84 | try 85 | { 86 | var candiateFiles = Directory.EnumerateFiles(logFolder, $"{InvalidPayloadFilePrefix}*.json"); 87 | DeleteOldFiles(maxNumberOfBytesToRetain, candiateFiles); 88 | } 89 | catch (Exception ex) 90 | { 91 | SelfLog.WriteLine("Exception thrown while trying to clean up invalid payload files: {0}", ex); 92 | } 93 | } 94 | 95 | 96 | 97 | /// 98 | /// Deletes oldest files in the group of invalid-* files. 99 | /// Existing files are ordered (from most recent to oldest) and file size is acumulated. All files 100 | /// who's cumulative byte count passes the defined limit are removed. Limit is therefore bytes 101 | /// and not number of files 102 | /// 103 | /// 104 | /// 105 | static void DeleteOldFiles(long maxNumberOfBytesToRetain, IEnumerable files) 106 | { 107 | var orderedFileInfos = from candidateFile in files 108 | let candidateFileInfo = new FileInfo(candidateFile) 109 | orderby candidateFileInfo.LastAccessTimeUtc descending 110 | select candidateFileInfo; 111 | 112 | var invalidPayloadFilesToDelete = WhereCumulativeSizeGreaterThan(orderedFileInfos, maxNumberOfBytesToRetain); 113 | 114 | foreach (var fileToDelete in invalidPayloadFilesToDelete) 115 | { 116 | try 117 | { 118 | fileToDelete.Delete(); 119 | } 120 | catch (Exception ex) 121 | { 122 | SelfLog.WriteLine("Exception '{0}' thrown while trying to delete file {1}", ex.Message, fileToDelete.FullName); 123 | } 124 | } 125 | } 126 | 127 | static IEnumerable WhereCumulativeSizeGreaterThan(IEnumerable files, long maxCumulativeSize) 128 | { 129 | long cumulative = 0; 130 | foreach (var file in files) 131 | { 132 | cumulative += file.Length; 133 | if (cumulative > maxCumulativeSize) 134 | { 135 | yield return file; 136 | } 137 | } 138 | } 139 | } 140 | } 141 | 142 | -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/LogEventConverter.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using Loggly; 5 | using SyslogLevel = Loggly.Transports.Syslog.Level; 6 | using Loggly.Config; 7 | using Serilog.Debugging; 8 | using Serilog.Events; 9 | 10 | namespace Serilog.Sinks.Loggly 11 | { 12 | /// 13 | /// Converts Serilog's Log Event to loogly-csharp LogglyEvent 14 | /// method was in LogglySink originally 15 | /// 16 | public class LogEventConverter 17 | { 18 | readonly IFormatProvider _formatProvider; 19 | private readonly LogIncludes _includes; 20 | 21 | public LogEventConverter(IFormatProvider formatProvider = null, LogIncludes includes = null) 22 | { 23 | _formatProvider = formatProvider; 24 | _includes = includes ?? new LogIncludes(); 25 | } 26 | 27 | public LogglyEvent CreateLogglyEvent(LogEvent logEvent) 28 | { 29 | var logglyEvent = new LogglyEvent() { Timestamp = logEvent.Timestamp }; 30 | 31 | var isHttpTransport = LogglyConfig.Instance.Transport.LogTransport == LogTransport.Https; 32 | logglyEvent.Syslog.Level = ToSyslogLevel(logEvent); 33 | 34 | if (_includes.IncludeMessage) 35 | { 36 | logglyEvent.Data.AddIfAbsent("Message", logEvent.RenderMessage(_formatProvider)); 37 | } 38 | 39 | foreach (var key in logEvent.Properties.Keys) 40 | { 41 | var propertyValue = logEvent.Properties[key]; 42 | var simpleValue = LogglyPropertyFormatter.Simplify(propertyValue, _formatProvider); 43 | logglyEvent.Data.AddIfAbsent(key, simpleValue); 44 | } 45 | 46 | if (isHttpTransport && _includes.IncludeLevel) 47 | { 48 | // syslog will capture these via the header 49 | logglyEvent.Data.AddIfAbsent("Level", logEvent.Level.ToString()); 50 | } 51 | 52 | if (logEvent.Exception != null && _includes.IncludeExceptionWhenExists) 53 | { 54 | logglyEvent.Data.AddIfAbsent("Exception", GetExceptionInfo(logEvent.Exception)); 55 | } 56 | 57 | return logglyEvent; 58 | } 59 | 60 | static SyslogLevel ToSyslogLevel(LogEvent logEvent) 61 | { 62 | SyslogLevel syslogLevel; 63 | // map the level to a syslog level in case that transport is used. 64 | switch (logEvent.Level) 65 | { 66 | case LogEventLevel.Verbose: 67 | case LogEventLevel.Debug: 68 | syslogLevel = SyslogLevel.Notice; 69 | break; 70 | case LogEventLevel.Information: 71 | syslogLevel = SyslogLevel.Information; 72 | break; 73 | case LogEventLevel.Warning: 74 | syslogLevel = SyslogLevel.Warning; 75 | break; 76 | case LogEventLevel.Error: 77 | case LogEventLevel.Fatal: 78 | syslogLevel = SyslogLevel.Error; 79 | break; 80 | default: 81 | SelfLog.WriteLine("Unexpected logging level, writing to loggly as Information"); 82 | syslogLevel = SyslogLevel.Information; 83 | break; 84 | } 85 | return syslogLevel; 86 | } 87 | 88 | /// 89 | /// Returns a minification of the exception information. Also takes care of the InnerException(s). 90 | /// 91 | /// 92 | /// 93 | ExceptionDetails GetExceptionInfo(Exception exception) 94 | { 95 | ExceptionDetails exceptionInfo = new ExceptionDetails( 96 | exception.GetType().FullName, 97 | exception.Message, exception.StackTrace, 98 | GetInnerExceptions(exception)); 99 | 100 | return exceptionInfo; 101 | } 102 | 103 | /// 104 | /// produces the collection of the inner-exceptions if exist 105 | /// Takes care of the differentiation between AggregateException and regualr exceptions 106 | /// 107 | /// 108 | /// 109 | ExceptionDetails[] GetInnerExceptions(Exception exception) 110 | { 111 | IEnumerable exceptions = Enumerable.Empty(); 112 | var ex = exception as AggregateException; 113 | if (ex != null) 114 | { 115 | var aggregateEx = ex; 116 | exceptions = aggregateEx.Flatten().InnerExceptions; 117 | } 118 | else if (exception.InnerException != null) 119 | { 120 | exceptions = new[] { exception.InnerException }; 121 | } 122 | return exceptions.Select(GetExceptionInfo).ToArray(); 123 | } 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/LogIncludes.cs: -------------------------------------------------------------------------------- 1 | namespace Serilog.Sinks.Loggly 2 | { 3 | public class LogIncludes 4 | { 5 | /// 6 | /// Adds Serilog Level to all log events. Defaults to true. 7 | /// 8 | public bool IncludeLevel { get; set; } = true; 9 | 10 | /// 11 | /// Adds Serilog Message to all log events. Defaults to true. 12 | /// 13 | public bool IncludeMessage { get; set; } = true; 14 | 15 | /// 16 | /// Adds Serilog Exception to log events when an exception exists. Defaults to true. 17 | /// 18 | public bool IncludeExceptionWhenExists { get; set; } = true; 19 | } 20 | } -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/LogglyConfigAdapter.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Loggly.Config; 3 | 4 | namespace Serilog.Sinks.Loggly 5 | { 6 | class LogglyConfigAdapter 7 | { 8 | public void ConfigureLogglyClient(LogglyConfiguration logglyConfiguration) 9 | { 10 | var config = LogglyConfig.Instance; 11 | 12 | if (!string.IsNullOrWhiteSpace(logglyConfiguration.ApplicationName)) 13 | config.ApplicationName = logglyConfiguration.ApplicationName; 14 | 15 | if (string.IsNullOrWhiteSpace(logglyConfiguration.CustomerToken)) 16 | throw new ArgumentNullException("CustomerToken", "CustomerToken is required"); 17 | 18 | config.CustomerToken = logglyConfiguration.CustomerToken; 19 | config.IsEnabled = logglyConfiguration.IsEnabled; 20 | 21 | if (logglyConfiguration.Tags != null) 22 | { 23 | foreach (var tag in logglyConfiguration.Tags) 24 | { 25 | config.TagConfig.Tags.Add(tag); 26 | } 27 | } 28 | 29 | config.ThrowExceptions = logglyConfiguration.ThrowExceptions; 30 | 31 | if (logglyConfiguration.LogTransport != TransportProtocol.Https) 32 | config.Transport.LogTransport = (LogTransport)Enum.Parse(typeof(LogTransport), logglyConfiguration.LogTransport.ToString()); 33 | 34 | if (!string.IsNullOrWhiteSpace(logglyConfiguration.EndpointHostName)) 35 | config.Transport.EndpointHostname = logglyConfiguration.EndpointHostName; 36 | 37 | if (logglyConfiguration.EndpointPort > 0 && logglyConfiguration.EndpointPort <= ushort.MaxValue) 38 | config.Transport.EndpointPort = logglyConfiguration.EndpointPort; 39 | 40 | config.Transport.IsOmitTimestamp = logglyConfiguration.OmitTimestamp; 41 | config.Transport = config.Transport.GetCoercedToValidConfig(); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/LogglyConfiguration.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace Serilog.Sinks.Loggly 4 | { 5 | public class LogglyConfiguration 6 | { 7 | public string ApplicationName { get; set; } 8 | public string CustomerToken { get; set; } 9 | public List Tags { get; set; } 10 | public bool IsEnabled { get; set; } = true; 11 | public bool ThrowExceptions { get; set; } 12 | 13 | /// 14 | /// Defaults to Https 15 | /// 16 | public TransportProtocol LogTransport { get; set; } 17 | 18 | /// 19 | /// Defaults to logs-01.loggly.com 20 | /// 21 | public string EndpointHostName { get; set; } 22 | 23 | /// 24 | /// Defaults to default port for selected LogTransport. 25 | /// E.g. https is 443, SyslogTcp/-Udp is 514 and SyslogSecure is 6514. 26 | /// 27 | public int EndpointPort { get; set; } 28 | 29 | /// 30 | /// Defines if timestamp should automatically be added to the json body when using Https 31 | /// 32 | public bool OmitTimestamp { get; set; } 33 | } 34 | 35 | public enum TransportProtocol 36 | { 37 | /// 38 | /// Https. 39 | /// 40 | Https, 41 | 42 | /// 43 | /// SyslogSecure. 44 | /// 45 | SyslogSecure, 46 | 47 | /// 48 | /// SyslogUdp. 49 | /// 50 | SyslogUdp, 51 | 52 | /// 53 | /// SyslogTcp. 54 | /// 55 | SyslogTcp, 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/LogglyFormatter.cs: -------------------------------------------------------------------------------- 1 | // Serilog.Sinks.Seq Copyright 2016 Serilog Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | using System; 16 | using System.IO; 17 | using Newtonsoft.Json; 18 | using Serilog.Events; 19 | using Serilog.Formatting; 20 | 21 | namespace Serilog.Sinks.Loggly 22 | { 23 | /// 24 | /// Formatter for the JSON schema accepted by Loggly's /bulk endpoint. 25 | /// 26 | class LogglyFormatter : ITextFormatter 27 | { 28 | readonly JsonSerializer _serializer = JsonSerializer.Create(); 29 | readonly LogEventConverter _converter; 30 | 31 | public LogglyFormatter(IFormatProvider formatProvider, LogIncludes includes) 32 | { 33 | //the converter should receive the format provider used, in order to 34 | // handle dateTimes and dateTimeOffsets in a controlled manner 35 | _converter = new LogEventConverter(formatProvider, includes); 36 | } 37 | public void Format(LogEvent logEvent, TextWriter output) 38 | { 39 | //Serializing the LogglyEvent means we can work with it from here on out and 40 | // avoid the serialization / deserialization troubles serilog's logevent 41 | // currently poses. 42 | _serializer.Serialize(output, _converter.CreateLogglyEvent(logEvent)); 43 | output.WriteLine(); //adds the necessary linebreak 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/LogglyPropertyFormatter.cs: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Serilog Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | using System; 16 | using System.Collections.Generic; 17 | using System.Globalization; 18 | using System.Linq; 19 | using Serilog.Debugging; 20 | using Serilog.Events; 21 | 22 | namespace Serilog.Sinks.Loggly 23 | { 24 | /// 25 | /// Converts values into simple scalars, 26 | /// that render well in Loggly. 27 | /// 28 | static class LogglyPropertyFormatter 29 | { 30 | static readonly HashSet LogglyScalars = new HashSet 31 | { 32 | typeof(bool), 33 | typeof(byte), typeof(short), typeof(ushort), typeof(int), typeof(uint), 34 | typeof(long), typeof(ulong), typeof(float), typeof(double), typeof(decimal), 35 | typeof(byte[]) 36 | }; 37 | 38 | /// 39 | /// Simplify the object so as to make handling the serialized 40 | /// representation easier. 41 | /// 42 | /// The value to simplify (possibly null). 43 | /// A formatProvider to format DateTime values if a specific format is required. 44 | /// A simplified representation. 45 | public static object Simplify(LogEventPropertyValue value, IFormatProvider formatProvider) 46 | { 47 | var scalar = value as ScalarValue; 48 | if (scalar != null) 49 | return SimplifyScalar(scalar.Value, formatProvider); 50 | 51 | var dict = value as DictionaryValue; 52 | if (dict != null) 53 | { 54 | var result = new Dictionary(); 55 | foreach (var element in dict.Elements) 56 | { 57 | var key = SimplifyScalar(element.Key.Value, formatProvider); 58 | if (result.ContainsKey(key)) 59 | { 60 | SelfLog.WriteLine("The key {0} is not unique in the provided dictionary after simplification to {1}.", element.Key, key); 61 | return dict.Elements.Select(e => new Dictionary 62 | { 63 | { "Key", SimplifyScalar(e.Key.Value, formatProvider) }, 64 | { "Value", Simplify(e.Value, formatProvider) } 65 | }) 66 | .ToArray(); 67 | } 68 | result.Add(key, Simplify(element.Value, formatProvider)); 69 | } 70 | return result; 71 | } 72 | 73 | var seq = value as SequenceValue; 74 | if (seq != null) 75 | return seq.Elements.Select(propertyValue => Simplify(propertyValue, formatProvider)).ToArray(); 76 | 77 | var str = value as StructureValue; 78 | if (str != null) 79 | { 80 | var props = str.Properties.ToDictionary(p => p.Name, p => Simplify(p.Value, formatProvider)); 81 | if (str.TypeTag != null) 82 | props["$typeTag"] = str.TypeTag; 83 | return props; 84 | } 85 | 86 | return null; 87 | } 88 | 89 | static object SimplifyScalar(object value, IFormatProvider formatProvider) 90 | { 91 | if (value == null) return null; 92 | 93 | var valueType = value.GetType(); 94 | if (LogglyScalars.Contains(valueType)) return value; 95 | 96 | //handle dateTimes with the format provider 97 | if (value is DateTime) 98 | return ((DateTime)value).ToString(formatProvider ?? CultureInfo.InvariantCulture); 99 | 100 | if (value is DateTimeOffset) 101 | return ((DateTimeOffset)value).ToString(formatProvider ?? CultureInfo.InvariantCulture); 102 | 103 | return value.ToString(); 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/LogglySink.cs: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Serilog Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | using System; 16 | using System.Collections.Generic; 17 | using System.Linq; 18 | using System.Threading.Tasks; 19 | using Loggly; 20 | using Serilog.Debugging; 21 | using Serilog.Events; 22 | using Serilog.Sinks.PeriodicBatching; 23 | 24 | namespace Serilog.Sinks.Loggly 25 | { 26 | /// 27 | /// Writes log events to the Loggly.com service. 28 | /// 29 | public class LogglySink : PeriodicBatchingSink 30 | { 31 | readonly LogEventConverter _converter; 32 | readonly LogglyClient _client; 33 | readonly LogglyConfigAdapter _adapter; 34 | 35 | /// 36 | /// A reasonable default for the number of events posted in 37 | /// each batch. 38 | /// 39 | public const int DefaultBatchPostingLimit = 10; 40 | 41 | /// 42 | /// A reasonable default time to wait between checking for event batches. 43 | /// 44 | public static readonly TimeSpan DefaultPeriod = TimeSpan.FromSeconds(5); 45 | 46 | /// 47 | /// Construct a sink that saves logs to the specified storage account. Properties are being send as data and the level is used as tag. 48 | /// 49 | /// The maximum number of events to post in a single batch. 50 | /// The time to wait between checking for event batches. 51 | /// Supplies culture-specific formatting information, or null. 52 | public LogglySink(IFormatProvider formatProvider, int batchSizeLimit, TimeSpan period) : this(formatProvider, batchSizeLimit, period, null, null) 53 | { 54 | } 55 | 56 | /// 57 | /// Construct a sink that saves logs to the specified storage account. Properties are being send as data and the level is used as tag. 58 | /// 59 | /// The maximum number of events to post in a single batch. 60 | /// The time to wait between checking for event batches. 61 | /// Supplies culture-specific formatting information, or null. 62 | /// Used to configure underlying LogglyClient programmaticaly. Otherwise use app.Config. 63 | /// Decides if the sink should include specific properties in the log message 64 | public LogglySink(IFormatProvider formatProvider, int batchSizeLimit, TimeSpan period, LogglyConfiguration logglyConfig, LogIncludes includes) 65 | : base (batchSizeLimit, period) 66 | { 67 | if (logglyConfig != null) 68 | { 69 | _adapter = new LogglyConfigAdapter(); 70 | _adapter.ConfigureLogglyClient(logglyConfig); 71 | } 72 | _client = new LogglyClient(); 73 | _converter = new LogEventConverter(formatProvider, includes); 74 | } 75 | 76 | /// 77 | /// Emit a batch of log events, running asynchronously. 78 | /// 79 | /// The events to emit. 80 | /// Override either or , 81 | /// not both. 82 | protected override async Task EmitBatchAsync(IEnumerable events) 83 | { 84 | LogResponse response = await _client.Log(events.Select(_converter.CreateLogglyEvent)).ConfigureAwait(false); 85 | 86 | switch (response.Code) 87 | { 88 | case ResponseCode.Error: 89 | SelfLog.WriteLine("LogglySink received an Error response: {0}", response.Message); 90 | break; 91 | case ResponseCode.Unknown: 92 | SelfLog.WriteLine("LogglySink received an Unknown response: {0}", response.Message); 93 | break; 94 | } 95 | } 96 | 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/Serilog.Sinks.Loggly/Sinks/Loggly/PortableTimer.cs: -------------------------------------------------------------------------------- 1 | // Copyright 2013-2016 Serilog Contributors 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | using Serilog.Debugging; 16 | using System; 17 | using System.Threading; 18 | using System.Threading.Tasks; 19 | 20 | namespace Serilog.Sinks.Loggly 21 | { 22 | class PortableTimer : IDisposable 23 | { 24 | readonly object _stateLock = new object(); 25 | 26 | readonly Func _onTick; 27 | readonly CancellationTokenSource _cancel = new CancellationTokenSource(); 28 | 29 | readonly Timer _timer; 30 | 31 | bool _running; 32 | bool _disposed; 33 | 34 | public PortableTimer(Func onTick) 35 | { 36 | if (onTick == null) throw new ArgumentNullException(nameof(onTick)); 37 | 38 | _onTick = onTick; 39 | 40 | _timer = new Timer(_ => OnTick(), null, Timeout.Infinite, Timeout.Infinite); 41 | } 42 | 43 | public void Start(TimeSpan interval) 44 | { 45 | if (interval < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(interval)); 46 | 47 | lock (_stateLock) 48 | { 49 | if (_disposed) 50 | throw new ObjectDisposedException(nameof(PortableTimer)); 51 | 52 | _timer.Change(interval, Timeout.InfiniteTimeSpan); 53 | } 54 | } 55 | 56 | async void OnTick() 57 | { 58 | try 59 | { 60 | lock (_stateLock) 61 | { 62 | if (_disposed) 63 | { 64 | return; 65 | } 66 | 67 | // There's a little bit of raciness here, but it's needed to support the 68 | // current API, which allows the tick handler to reenter and set the next interval. 69 | 70 | if (_running) 71 | { 72 | Monitor.Wait(_stateLock); 73 | 74 | if (_disposed) 75 | { 76 | return; 77 | } 78 | } 79 | 80 | _running = true; 81 | } 82 | 83 | if (!_cancel.Token.IsCancellationRequested) 84 | { 85 | await _onTick(_cancel.Token); 86 | } 87 | } 88 | catch (OperationCanceledException tcx) 89 | { 90 | SelfLog.WriteLine("The timer was canceled during invocation: {0}", tcx); 91 | } 92 | finally 93 | { 94 | lock (_stateLock) 95 | { 96 | _running = false; 97 | Monitor.PulseAll(_stateLock); 98 | } 99 | } 100 | } 101 | 102 | public void Dispose() 103 | { 104 | _cancel.Cancel(); 105 | 106 | lock (_stateLock) 107 | { 108 | if (_disposed) 109 | { 110 | return; 111 | } 112 | 113 | while (_running) 114 | { 115 | Monitor.Wait(_stateLock); 116 | } 117 | 118 | _timer.Dispose(); 119 | 120 | _disposed = true; 121 | } 122 | } 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /test/Serilog.Sinks.Loggly.Tests/ExceptionSerialization.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Linq; 4 | using System.Threading; 5 | using Loggly; 6 | using Loggly.Config; 7 | using Loggly.Transports.Syslog; 8 | using Serilog.Events; 9 | using Serilog.Parsing; 10 | using Xunit; 11 | 12 | namespace Serilog.Sinks.Loggly.Tests 13 | { 14 | public class ExceptionSerialization 15 | { 16 | [Fact] 17 | public void ReturnFalseGivenValueOf1() 18 | { 19 | var writer = new StringWriter(); 20 | var formatter = new LogglyFormatter(null, null); 21 | try 22 | { 23 | ThrowException(); 24 | } 25 | catch (Exception e) 26 | { 27 | var evt = new LogEvent(DateTimeOffset.UtcNow, 28 | LogEventLevel.Error, e, 29 | new MessageTemplate("Hello", Enumerable.Empty()), 30 | Enumerable.Empty()); 31 | formatter.Format(evt, writer); 32 | } 33 | 34 | var s = writer.ToString(); 35 | Assert.NotEmpty(s); 36 | } 37 | 38 | [Fact] 39 | public void InnerExceptionsAreSerialized() 40 | { 41 | var converter = new LogEventConverter(null); 42 | Exception innerException = null; 43 | try 44 | { 45 | try 46 | { 47 | ThrowException(); 48 | 49 | } 50 | catch (Exception e) 51 | { 52 | innerException = e; 53 | throw new InvalidOperationException("Outer Exception Message",e); 54 | } 55 | } 56 | catch (InvalidOperationException e) 57 | { 58 | var evt = new LogEvent(DateTimeOffset.UtcNow, 59 | LogEventLevel.Error, e, 60 | new MessageTemplate("Hello", Enumerable.Empty()), 61 | Enumerable.Empty()); 62 | var logglyEvent = converter.CreateLogglyEvent(evt); 63 | 64 | var exceptionnDetails = logglyEvent.Data["Exception"] as ExceptionDetails; 65 | Assert.NotEmpty(exceptionnDetails.InnerExceptions); 66 | Assert.Equal(innerException.Message,exceptionnDetails.InnerExceptions[0].Message); 67 | } 68 | } 69 | 70 | [Fact] 71 | public void AggregateExceptionsAreSerialized() 72 | { 73 | var converter = new LogEventConverter(null); 74 | Exception innerException = null; 75 | try 76 | { 77 | try 78 | { 79 | ThrowException(); 80 | 81 | } 82 | catch (Exception e) 83 | { 84 | innerException = e; 85 | throw new AggregateException(e,e); 86 | } 87 | } 88 | catch (AggregateException e) 89 | { 90 | var evt = new LogEvent(DateTimeOffset.UtcNow, 91 | LogEventLevel.Error, e, 92 | new MessageTemplate("Hello", Enumerable.Empty()), 93 | Enumerable.Empty()); 94 | var logglyEvent = converter.CreateLogglyEvent(evt); 95 | 96 | var exceptionnDetails = logglyEvent.Data["Exception"] as ExceptionDetails; 97 | Assert.Equal(2,exceptionnDetails.InnerExceptions.Length); 98 | Assert.Equal(innerException.Message, exceptionnDetails.InnerExceptions[0].Message); 99 | Assert.Equal(innerException.Message, exceptionnDetails.InnerExceptions[1].Message); 100 | } 101 | 102 | } 103 | 104 | [Fact] 105 | public void LogglyClientSendException() 106 | { 107 | var config = LogglyConfig.Instance; 108 | config.CustomerToken = "83fe7674-f87d-473e-a8af-bbbbbbbbbbbb"; 109 | config.ApplicationName = $"test"; 110 | 111 | config.Transport.EndpointHostname = "logs-01.loggly.com" ; 112 | config.Transport.EndpointPort = 443; 113 | config.Transport.LogTransport = LogTransport.Https; 114 | 115 | var ct = new ApplicationNameTag { Formatter = "application-{0}" }; 116 | config.TagConfig.Tags.Add(ct); 117 | var logglyClient = new LogglyClient(); 118 | 119 | 120 | try 121 | { 122 | ThrowException(); 123 | } 124 | catch (Exception e) 125 | { 126 | LogglyEvent logglyEvent = new LogglyEvent 127 | { 128 | Timestamp = DateTimeOffset.UtcNow, 129 | Syslog = {Level = Level.Emergency} 130 | }; 131 | logglyEvent.Data.AddIfAbsent("Message", "xZx"); 132 | logglyEvent.Data.AddIfAbsent("Level", "Error"); 133 | logglyEvent.Data.AddIfAbsent("Exception", e); 134 | 135 | var res = logglyClient.Log(logglyEvent).Result; 136 | Assert.Equal(ResponseCode.Success, res.Code); 137 | } 138 | Thread.Sleep(5000); 139 | } 140 | 141 | private static void ThrowException() 142 | { 143 | throw new ArgumentOutOfRangeException("Hello", "xyz", "sdsd"); 144 | } 145 | 146 | 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /test/Serilog.Sinks.Loggly.Tests/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | // Empty file just to generate the test assembly 2 | -------------------------------------------------------------------------------- /test/Serilog.Sinks.Loggly.Tests/Serilog.Sinks.Loggly.Tests.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | net452;netcoreapp1.0;netcoreapp2.1;netcoreapp3.1 5 | Serilog.Sinks.Loggly.Tests 6 | ../../assets/Serilog.snk 7 | true 8 | true 9 | Serilog.Sinks.Loggly.Tests 10 | true 11 | true 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | all 38 | runtime; build; native; contentfiles; analyzers 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | ..\..\..\..\Program Files (x86)\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5\System.Activities.dll 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /test/Serilog.Sinks.Loggly.Tests/Sinks/Loggly/Durable/FileSetPositionTests.cs: -------------------------------------------------------------------------------- 1 | using Serilog.Sinks.Loggly.Durable; 2 | using Xunit; 3 | 4 | namespace Serilog.Sinks.Loggly.Tests.Sinks.Loggly.Durable 5 | { 6 | public class FileSetPositionTests 7 | { 8 | [Fact] 9 | public void CanCreateBookmarkInstance() 10 | { 11 | var marker = new FileSetPosition(0, @"C:\test"); 12 | Assert.NotNull(marker); 13 | } 14 | 15 | [Fact] 16 | public void CanCreateEmptyBookmark() 17 | { 18 | var marker = FileSetPosition.None; 19 | Assert.Null(marker); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /test/Serilog.Sinks.Loggly.Tests/Sinks/Loggly/Expectations/expectedInvalidPayloadFileN.json: -------------------------------------------------------------------------------- 1 | {"Timestamp":"2017-09-27T00:00:00+00:00","Syslog":{"MessageId":0,"Level":6},"Data":{},"Options":{"Tags":[]}} 2 | Error info: HTTP shipping failed with Error: 502 Bad Request -------------------------------------------------------------------------------- /test/Serilog.Sinks.Loggly.Tests/Sinks/Loggly/Expectations/expectedInvalidPayloadFileRN.json: -------------------------------------------------------------------------------- 1 | {"Timestamp":"2017-09-27T00:00:00+00:00","Syslog":{"MessageId":0,"Level":6},"Data":{},"Options":{"Tags":[]}} 2 | Error info: HTTP shipping failed with Error: 502 Bad Request -------------------------------------------------------------------------------- /test/Serilog.Sinks.Loggly.Tests/Sinks/Loggly/FileBasedBookmarkProviderTests.cs: -------------------------------------------------------------------------------- 1 | using System.IO; 2 | using System.Text; 3 | using NSubstitute; 4 | using Serilog.Sinks.Loggly.Durable; 5 | using Xunit; 6 | 7 | namespace Serilog.Sinks.Loggly.Tests.Sinks.Loggly 8 | { 9 | public class FileBasedBookmarkProviderTests 10 | { 11 | static Encoding Encoder = new UTF8Encoding(false); 12 | static string BaseBufferFileName = @"c:\test\buffer"; 13 | const string ExpectedBufferFilePath = @"C:\test\buffer-20170926.json"; 14 | const long ExpectedBytePosition = 123; 15 | 16 | public class InstanceTests 17 | { 18 | readonly IBookmarkProvider _sut = new FileBasedBookmarkProvider(BaseBufferFileName, Substitute.For(), Encoder); 19 | 20 | [Fact] 21 | public void InstanceIsValid() => Assert.NotNull(_sut); 22 | } 23 | 24 | public class ReadBookmarkTests 25 | { 26 | public class ValidBookmarkFileOnDisk 27 | { 28 | readonly FileSetPosition _sut; 29 | readonly FileSetPosition _reread; 30 | 31 | public ValidBookmarkFileOnDisk() 32 | { 33 | var fileSystemAdapter = Substitute.For(); 34 | fileSystemAdapter 35 | .Open(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) 36 | .Returns(new MemoryStream( 37 | Encoding.UTF8.GetBytes($"{ExpectedBytePosition}:::{ExpectedBufferFilePath}\r\n"))); 38 | 39 | var provider = new FileBasedBookmarkProvider(BaseBufferFileName, fileSystemAdapter, Encoder); 40 | 41 | _sut = provider.GetCurrentBookmarkPosition(); 42 | _reread = provider.GetCurrentBookmarkPosition(); 43 | } 44 | 45 | [Fact] 46 | public void ShouldHaveValidBookmark() => Assert.NotNull(_sut); 47 | 48 | [Fact] 49 | public void BookmarkPostionShouldBeCorrect() => Assert.Equal(ExpectedBytePosition, _sut.NextLineStart); 50 | 51 | [Fact] 52 | public void BookmarkBufferFilePathShouldBeCorrect() => 53 | Assert.Equal(ExpectedBufferFilePath, _sut.File); 54 | 55 | [Fact] 56 | public void RereadingtheBookmarkGivesSameValue() => Assert.Equal(_sut.NextLineStart, _reread.NextLineStart); 57 | 58 | } 59 | 60 | /// 61 | /// This case represents some observed behaviour in the bookmark file. Since writes are normally done from the start of the file, 62 | /// some garbage may remain in the file if the new written bytes are shorter than what existed. 63 | /// 64 | public class StrangeBookmarkFileOnDisk 65 | { 66 | readonly FileSetPosition _sut; 67 | 68 | public StrangeBookmarkFileOnDisk() 69 | { 70 | var fileSystemAdapter = Substitute.For(); 71 | fileSystemAdapter 72 | .Open(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) 73 | .Returns(new MemoryStream(Encoding.UTF8.GetBytes( 74 | $"{ExpectedBytePosition}:::{ExpectedBufferFilePath}\r\nsome invalid stuff in the file\n\n\n"))); 75 | 76 | var provider = new FileBasedBookmarkProvider(BaseBufferFileName, fileSystemAdapter, Encoder); 77 | 78 | _sut = provider.GetCurrentBookmarkPosition(); 79 | } 80 | 81 | [Fact] 82 | public void ShouldHaveValidBookmark() => Assert.NotNull(_sut); 83 | 84 | [Fact] 85 | public void BookmarkPostionShouldBeCorrect() => Assert.Equal(ExpectedBytePosition, _sut.NextLineStart); 86 | 87 | [Fact] 88 | public void BookmarkBufferFilePathShouldBeCorrect() => 89 | Assert.Equal(ExpectedBufferFilePath, _sut.File); 90 | 91 | } 92 | 93 | /// 94 | /// An inexistent bookmark file will create a new, empty one, and the returned stream will be empty when trying to read 95 | /// 96 | public class InexistentBookmarkFileOnDisk 97 | { 98 | FileSetPosition _sut; 99 | 100 | public InexistentBookmarkFileOnDisk() 101 | { 102 | var fileSystemAdapter = Substitute.For(); 103 | fileSystemAdapter 104 | .Open(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) 105 | .Returns(new MemoryStream(new byte[] { })); 106 | 107 | var provider = new FileBasedBookmarkProvider(BaseBufferFileName, fileSystemAdapter, Encoding.UTF8); 108 | 109 | _sut = provider.GetCurrentBookmarkPosition(); 110 | } 111 | 112 | [Fact] 113 | public void BookmarkShouldBeNull() => Assert.Null(_sut); 114 | } 115 | } 116 | 117 | public class WriteBookmarkTests 118 | { 119 | public class WriteToAnEmptyBookmarkStream 120 | { 121 | readonly MemoryStream _sut = new MemoryStream(new byte[128]); //make it big enough to take in new content, as a file stream would 122 | readonly string _expectedFileContent = $"{ExpectedBytePosition}:::{ExpectedBufferFilePath}\r\n".PadRight(128, '\0'); 123 | readonly byte[] _expectedBytes; 124 | readonly byte[] _actualBytes; 125 | 126 | public WriteToAnEmptyBookmarkStream() 127 | { 128 | var fileSystemAdapter = Substitute.For(); 129 | fileSystemAdapter 130 | .Open(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) 131 | .Returns(_sut); 132 | 133 | var provider = new FileBasedBookmarkProvider(BaseBufferFileName, fileSystemAdapter, Encoder); 134 | provider.UpdateBookmark(new FileSetPosition(ExpectedBytePosition, ExpectedBufferFilePath)); 135 | 136 | _expectedBytes = Encoder.GetBytes(_expectedFileContent); 137 | _actualBytes = _sut.ToArray(); 138 | } 139 | 140 | //compare on bytes and string - if Encoding.UTF8 is used, a BOM set of bytes may be added. 141 | //it is therefore useful to use an encoder created by the UTF8Encoding constructor as in the container class 142 | 143 | [Fact] 144 | public void StreamShouldHaveBookmarkWritten() => Assert.Equal(_expectedBytes, _actualBytes); 145 | 146 | [Fact] 147 | public void StreamContentShouldConvertToExpectedText() => Assert.Equal(_expectedFileContent, Encoder.GetString(_sut.ToArray())); 148 | 149 | } 150 | 151 | public class WriteToAnBookmarkStreamWithExistingContent 152 | { 153 | readonly MemoryStream _sut = new MemoryStream(new byte[128]); //make it big enough to take in new content, as a file stream would 154 | 155 | //contrary to the empty files, there will be sopme "garbage in the expected stream, since we are not clearing it 156 | //but just wirting on top of it. Line endings determine the truly significant info in the file 157 | //the following reflects this: 158 | readonly string _expectedFileContent = $"{ExpectedBytePosition}:::{ExpectedBufferFilePath}\r\nn{ExpectedBufferFilePath}\r\n".PadRight(128, '\0'); 159 | readonly byte[] _expectedBytes; 160 | readonly byte[] _actualBytes; 161 | 162 | public WriteToAnBookmarkStreamWithExistingContent() 163 | { 164 | //simulate a stream with existing content that may be larger or shorter than what is written. 165 | // newline detection ends up being iportant in this case 166 | var initialContent = Encoder.GetBytes($"100000:::{ExpectedBufferFilePath}{ExpectedBufferFilePath}\r\n"); 167 | _sut.Write(initialContent, 0, initialContent.Length); 168 | 169 | var fileSystemAdapter = Substitute.For(); 170 | fileSystemAdapter 171 | .Open(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) 172 | .Returns(_sut); 173 | 174 | var provider = new FileBasedBookmarkProvider(BaseBufferFileName, fileSystemAdapter, Encoder); 175 | provider.UpdateBookmark(new FileSetPosition(ExpectedBytePosition, ExpectedBufferFilePath)); 176 | 177 | _expectedBytes = Encoder.GetBytes(_expectedFileContent); 178 | _actualBytes = _sut.ToArray(); 179 | } 180 | 181 | //compare on bytes and string - if Encoding.UTF8 is used, a BOM set of bytes may be added. 182 | //it is therefore useful to use an encoder created by the UTF8Encoding constructor as in the container class 183 | 184 | [Fact] 185 | public void StreamShouldHaveBookmarkWritten() => Assert.Equal(_expectedBytes, _actualBytes); 186 | 187 | [Fact] 188 | public void StreamContentShouldConvertToExpectedText() => Assert.Equal(_expectedFileContent, Encoder.GetString(_sut.ToArray())); 189 | 190 | } 191 | } 192 | } 193 | } -------------------------------------------------------------------------------- /test/Serilog.Sinks.Loggly.Tests/Sinks/Loggly/FileBufferDataProviderTests.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.IO; 3 | using System.Linq; 4 | using System.Reflection; 5 | using System.Text; 6 | using Loggly; 7 | using Xunit; 8 | using NSubstitute; 9 | using System; 10 | using Serilog.Sinks.Loggly.Durable; 11 | 12 | namespace Serilog.Sinks.Loggly.Tests.Sinks.Loggly 13 | { 14 | /// 15 | /// This class allowws setting the initial state fo the provider in the tests, especially to 16 | /// controll the current bookmark position, as that influences the way logic happens 17 | /// 18 | internal class FileBufferDataProviderThatAllowscurrentBookmarkToBeSet : FileBufferDataProvider 19 | { 20 | public FileBufferDataProviderThatAllowscurrentBookmarkToBeSet( 21 | string baseBufferFileName, 22 | IFileSystemAdapter fileSystemAdapter, 23 | IBookmarkProvider bookmarkProvider, 24 | Encoding encoding, 25 | int batchPostingLimit, 26 | long? eventBodyLimitBytes, 27 | int? retainedFileCountLimit) 28 | : base(baseBufferFileName, 29 | fileSystemAdapter, 30 | bookmarkProvider, 31 | encoding, 32 | batchPostingLimit, 33 | eventBodyLimitBytes, 34 | retainedFileCountLimit) 35 | { 36 | } 37 | 38 | /// 39 | /// This helps set the inital bookmark in tests 40 | /// 41 | /// 42 | internal void DefineCurrentBookmark(long nextLineStartToUse, string file) 43 | { 44 | CurrentBookmark = new FileSetPosition(nextLineStartToUse, file); 45 | } 46 | } 47 | 48 | public class FileBufferDataProviderTests 49 | { 50 | static readonly string ResourceNamespace = $"Serilog.Sinks.Loggly.Tests.Sinks.Loggly"; 51 | static readonly string BaseBufferFileName = @"c:\test\buffer"; 52 | static readonly Encoding Utf8Encoder = new UTF8Encoding(true); 53 | static readonly string Bufferfile = @"C:\test\buffer001.json"; //any valid name here will suffice 54 | static readonly int BatchLimit = 10; 55 | static readonly int EventSizeLimit = 1024 * 1024; 56 | 57 | public class InstanceCreationTests 58 | { 59 | [Fact] 60 | public void CanCreateInstanceOfFileBufferDataProvider() 61 | { 62 | var mockFileSystemAdapter = Substitute.For(); 63 | var bookmarkProvider = Substitute.For(); 64 | 65 | var instance = new FileBufferDataProvider(BaseBufferFileName, mockFileSystemAdapter, bookmarkProvider, Utf8Encoder, 10, 1024*1024, null); 66 | 67 | Assert.NotNull(instance); 68 | } 69 | } 70 | 71 | /// 72 | /// In this scenario, there is neither a bufferX.json file nor a bookmark. 73 | /// 74 | public class EmptyBufferAndBookmarkScenario 75 | { 76 | readonly IEnumerable _sut; 77 | 78 | public EmptyBufferAndBookmarkScenario() 79 | { 80 | var bookmarkProvider = Substitute.For(); 81 | bookmarkProvider.GetCurrentBookmarkPosition().Returns(null as FileSetPosition); 82 | 83 | var mockFileSystem = Substitute.For(); 84 | mockFileSystem.GetFiles(Arg.Any(), Arg.Any()).Returns(new string[] { }); 85 | 86 | var provider = new FileBufferDataProvider(BaseBufferFileName, mockFileSystem, bookmarkProvider, Utf8Encoder, 10, 1024 * 1024, null); 87 | _sut = provider.GetNextBatchOfEvents(); 88 | } 89 | 90 | [Fact] 91 | public void EventListShouldBeEmpty() => Assert.Empty(_sut); 92 | } 93 | 94 | /// 95 | /// In this scenario, there is no bufferX.json file but there is a bookmark file. The bookmark, though, 96 | /// points to a file buffer file that no longer exists 97 | /// 98 | public class EmptyBufferAndOutdatedBookmarkScenario 99 | { 100 | readonly IEnumerable _sut; 101 | 102 | public EmptyBufferAndOutdatedBookmarkScenario() 103 | { 104 | var bookmarkProvider = Substitute.For(); 105 | bookmarkProvider.GetCurrentBookmarkPosition().Returns(new FileSetPosition(0, @"C:\test\existent.json")); 106 | 107 | var mockFileSystem = Substitute.For(); 108 | mockFileSystem.GetFiles(Arg.Any(), Arg.Any()).Returns(new string[] { }); 109 | 110 | var provider = new FileBufferDataProvider(BaseBufferFileName, mockFileSystem, bookmarkProvider, Utf8Encoder, 10, 1024 * 1024, null); 111 | _sut = provider.GetNextBatchOfEvents(); 112 | } 113 | 114 | [Fact] 115 | public void EventListShouldBeEmpty() => Assert.Empty(_sut); 116 | } 117 | 118 | /// 119 | /// In this scenario, there is a single Buffer.json file but no bookmark file. 120 | /// Results are the same as the SingleBufferFileAndSyncedBookmarkScenario as the 121 | /// buffer will be initialized to the first buffer file 122 | /// 123 | public class SingleBufferFileAndNoBookmarkScenario 124 | { 125 | IEnumerable _sut; 126 | IEnumerable _reRequestBatch; 127 | 128 | public SingleBufferFileAndNoBookmarkScenario() 129 | { 130 | var bookmarkProvider = Substitute.For(); 131 | bookmarkProvider.GetCurrentBookmarkPosition().Returns(null as FileSetPosition); 132 | IFileSystemAdapter fsAdapter = CreateFileSystemAdapter(Bufferfile); 133 | 134 | var provider = new FileBufferDataProvider( 135 | BaseBufferFileName, 136 | fsAdapter, 137 | bookmarkProvider, 138 | Utf8Encoder, 139 | BatchLimit, 140 | EventSizeLimit, 141 | null); 142 | _sut = provider.GetNextBatchOfEvents(); 143 | 144 | _reRequestBatch = provider.GetNextBatchOfEvents(); 145 | } 146 | 147 | IFileSystemAdapter CreateFileSystemAdapter(string bufferfile) 148 | { 149 | var fileSystemAdapter = Substitute.For(); 150 | 151 | //get files should return the single buffer file path in this scenario 152 | fileSystemAdapter.GetFiles(Arg.Any(), Arg.Any()) 153 | .Returns(new[] { bufferfile }); 154 | 155 | //when we ask for the buffer file, simulate that it exists 156 | fileSystemAdapter.Exists(bufferfile).Returns(true); 157 | 158 | //Open() should open a stream that can return two events 159 | fileSystemAdapter.Open(bufferfile, Arg.Any(), Arg.Any(), 160 | Arg.Any()) 161 | .Returns(GetSingleEventLineStreamFromResources()); 162 | 163 | return fileSystemAdapter; 164 | } 165 | 166 | [Fact] 167 | public void EventListShouldBeNotBeEmpty() => Assert.NotEmpty(_sut); 168 | 169 | [Fact] 170 | public void ShouldReadBatchOfEvents() => Assert.Single(_sut); 171 | 172 | [Fact] 173 | public void ReRequestingABatchShouldReturnSameUnprocessedEventsInQueue() => 174 | Assert.Equal(_sut, _reRequestBatch); 175 | } 176 | 177 | 178 | /// 179 | /// In this scenario, there is a single Buffer.json file but a bookmark file pointing 180 | /// to the start of the buffer. 181 | /// 182 | public class SingleBufferFileAndSyncedBookmarkScenario 183 | { 184 | readonly IEnumerable _sut; 185 | readonly IEnumerable _reRequestBatch; 186 | 187 | public SingleBufferFileAndSyncedBookmarkScenario() 188 | { 189 | var bookmarkProvider = Substitute.For(); 190 | bookmarkProvider.GetCurrentBookmarkPosition().Returns(new FileSetPosition(0, Bufferfile)); 191 | IFileSystemAdapter fsAdapter = CreateFileSystemAdapter(Bufferfile); 192 | 193 | var provider = new FileBufferDataProvider( 194 | BaseBufferFileName, 195 | fsAdapter, 196 | bookmarkProvider, 197 | Utf8Encoder, 198 | BatchLimit, 199 | EventSizeLimit, 200 | null); 201 | _sut = provider.GetNextBatchOfEvents(); 202 | 203 | _reRequestBatch = provider.GetNextBatchOfEvents(); 204 | } 205 | 206 | IFileSystemAdapter CreateFileSystemAdapter(string bufferfile) 207 | { 208 | var fileSystemAdapter = Substitute.For(); 209 | 210 | //get files should return the single buffer file path in this scenario 211 | fileSystemAdapter.GetFiles(Arg.Any(), Arg.Any()) 212 | .Returns(new string[] {bufferfile}); 213 | 214 | //when we ask for the buffer file, simulate that it exists 215 | fileSystemAdapter.Exists(bufferfile).Returns(true); 216 | 217 | //Open() should open a stream that can return two events 218 | fileSystemAdapter.Open(bufferfile, Arg.Any(), Arg.Any(), 219 | Arg.Any()) 220 | .Returns(GetSingleEventLineStreamFromResources()); 221 | 222 | return fileSystemAdapter; 223 | } 224 | 225 | 226 | 227 | [Fact] 228 | public void EventListShouldBeNotBeEmpty() => Assert.NotEmpty(_sut); 229 | 230 | [Fact] 231 | public void ShouldReadBatchOfEvents() => Assert.Single(_sut); 232 | 233 | [Fact] 234 | public void ReRequestingABatchShouldReturnSameUnprocessedEventsInQueue() => 235 | Assert.Equal(_sut, _reRequestBatch); 236 | } 237 | 238 | /// 239 | /// buffer file contains more events than a single batch. Rereading a batch should not progress without 240 | /// marking the batch as processed 241 | /// 242 | public class LongerBufferFileAndSyncedBookmarkScenario 243 | { 244 | readonly IEnumerable _sut; 245 | readonly IEnumerable _reRequestBatch; 246 | 247 | public LongerBufferFileAndSyncedBookmarkScenario() 248 | { 249 | var bookmarkProvider = Substitute.For(); 250 | bookmarkProvider.GetCurrentBookmarkPosition().Returns(new FileSetPosition(0, Bufferfile)); 251 | IFileSystemAdapter fsAdapter = CreateFileSystemAdapter(Bufferfile); 252 | 253 | var provider = new FileBufferDataProvider( 254 | BaseBufferFileName, 255 | fsAdapter, 256 | bookmarkProvider, 257 | Utf8Encoder, 258 | BatchLimit, 259 | EventSizeLimit, 260 | null); 261 | _sut = provider.GetNextBatchOfEvents(); 262 | 263 | _reRequestBatch = provider.GetNextBatchOfEvents(); 264 | } 265 | 266 | private IFileSystemAdapter CreateFileSystemAdapter(string bufferfile) 267 | { 268 | var fileSystemAdapter = Substitute.For(); 269 | 270 | //get files should return the single buffer file path in this scenario 271 | fileSystemAdapter.GetFiles(Arg.Any(), Arg.Any()) 272 | .Returns(new string[] { bufferfile }); 273 | 274 | //when we ask for the buffer file, simulate that it exists 275 | fileSystemAdapter.Exists(bufferfile).Returns(true); 276 | 277 | //Open() should open a stream that can return two events 278 | fileSystemAdapter.Open(bufferfile, Arg.Any(), Arg.Any(), 279 | Arg.Any()) 280 | .Returns(Get20LineStreamFromResources()); 281 | 282 | return fileSystemAdapter; 283 | } 284 | 285 | [Fact] 286 | public void EventListShouldBeNotBeEmpty() => Assert.NotEmpty(_sut); 287 | 288 | [Fact] 289 | public void ShouldReadBatchOfEventsLimitedToBatchCount() => Assert.Equal(10, _sut.Count()); 290 | 291 | [Fact] 292 | public void ReRequestingABatchShouldReturnSameUnprocessedEventsInQueue() => 293 | Assert.Equal(_sut, _reRequestBatch); 294 | } 295 | 296 | /// 297 | /// Gets two batches, having marked the first as processed to move through the buffer 298 | /// After reading the second batch, there should no longer be anything in the buffer, so 299 | /// the last batch should be empty 300 | /// 301 | public class AdvanceThroughBufferScenario 302 | { 303 | readonly IEnumerable _firstBatchRead; 304 | readonly IEnumerable _reRequestBatch; 305 | readonly IEnumerable _lastBatch; 306 | 307 | public AdvanceThroughBufferScenario() 308 | { 309 | var bookmarkProvider = Substitute.For(); 310 | bookmarkProvider.GetCurrentBookmarkPosition().Returns(new FileSetPosition(0, Bufferfile)); 311 | IFileSystemAdapter fsAdapter = CreateFileSystemAdapter(Bufferfile); 312 | 313 | var provider = new FileBufferDataProvider( 314 | BaseBufferFileName, 315 | fsAdapter, 316 | bookmarkProvider, 317 | Utf8Encoder, 318 | BatchLimit, 319 | EventSizeLimit, 320 | null); 321 | 322 | _firstBatchRead = provider.GetNextBatchOfEvents(); 323 | //after getting first batch, simulate moving foward 324 | provider.MarkCurrentBatchAsProcessed(); 325 | //request next batch 326 | _reRequestBatch = provider.GetNextBatchOfEvents(); 327 | //after getting second batch, simulate moving foward 328 | provider.MarkCurrentBatchAsProcessed(); 329 | //should have no events available to read 330 | _lastBatch = provider.GetNextBatchOfEvents(); 331 | 332 | } 333 | 334 | IFileSystemAdapter CreateFileSystemAdapter(string bufferfile) 335 | { 336 | var fileSystemAdapter = Substitute.For(); 337 | 338 | //get files should return the single buffer file path in this scenario 339 | fileSystemAdapter.GetFiles(Arg.Any(), Arg.Any()) 340 | .Returns(new[] { bufferfile }); 341 | 342 | //when we ask for the buffer file, simulate that it exists 343 | fileSystemAdapter.Exists(bufferfile).Returns(true); 344 | 345 | //Open() should open a stream that can return two events 346 | fileSystemAdapter.Open(bufferfile, Arg.Any(), Arg.Any(), 347 | Arg.Any()) 348 | .Returns(x => Get20LineStreamFromResources()); //use this form to reexecute the get stream for a new stream 349 | 350 | return fileSystemAdapter; 351 | } 352 | 353 | [Fact] 354 | public void EventListShouldBeNotBeEmpty() => Assert.NotEmpty(_firstBatchRead); 355 | 356 | [Fact] 357 | public void ShouldReadBatchOfEventsLimitedToBatchCount() => Assert.Equal(10, _firstBatchRead.Count()); 358 | 359 | [Fact] 360 | public void ReRequestingABatchShouldReturnSameUnprocessedEventsInQueue() => 361 | Assert.NotEqual(_firstBatchRead, _reRequestBatch); 362 | 363 | [Fact] 364 | public void LastBatchShouldBeEmpty() => Assert.Empty(_lastBatch); 365 | } 366 | 367 | /// 368 | /// In this scenario, the app may have been offline / disconnected for a few days (desktop clients, for instance) 369 | /// and multiple files may have accumulated. We may or may not want data from all the days offline, 370 | /// depending on retainedFileCountLimit's value, and should work the bookmark acordingly 371 | /// 372 | public class MultipleBufferFilesScenario 373 | { 374 | /// 375 | /// When less then the limit, bookmark should point to initial file if current bookmark is invalid 376 | /// 377 | public class LessThenLimitnumberOfBufferFiles 378 | { 379 | const int NumberOfFilesToRetain = 5; 380 | FileSetPosition _sut; 381 | 382 | public LessThenLimitnumberOfBufferFiles() 383 | { 384 | var bookmarkProvider = Substitute.For(); 385 | bookmarkProvider 386 | .GetCurrentBookmarkPosition() 387 | .Returns(new FileSetPosition(0, @"c:\unknown.json")); //should force fileset analysis 388 | bookmarkProvider 389 | .When(x => x.UpdateBookmark(Arg.Any())) 390 | .Do(x => _sut = x.ArgAt(0)); 391 | 392 | IFileSystemAdapter fsAdapter = CreateFileSystemAdapter(Bufferfile); 393 | 394 | var provider = new FileBufferDataProvider( 395 | BaseBufferFileName, 396 | fsAdapter, 397 | bookmarkProvider, 398 | Utf8Encoder, 399 | BatchLimit, 400 | EventSizeLimit, 401 | NumberOfFilesToRetain); 402 | 403 | provider.GetNextBatchOfEvents(); 404 | provider.MarkCurrentBatchAsProcessed(); 405 | } 406 | 407 | IFileSystemAdapter CreateFileSystemAdapter(string bufferfile) 408 | { 409 | var fileSystemAdapter = Substitute.For(); 410 | 411 | //get files should return the single buffer file path in this scenario 412 | fileSystemAdapter.GetFiles(Arg.Any(), Arg.Any()) 413 | .Returns(new[] {bufferfile}); 414 | 415 | //when we ask for the buffer file (and only that file), simulate that it exists; for others return false 416 | fileSystemAdapter.Exists(Arg.Any()).Returns(false); 417 | fileSystemAdapter.Exists(bufferfile).Returns(true); 418 | 419 | //Open() should open a stream that can return two events 420 | fileSystemAdapter.Open(bufferfile, Arg.Any(), Arg.Any(), 421 | Arg.Any()) 422 | .Returns(x => 423 | GetSingleEventLineStreamFromResources()); //use this form to reexecute the get stream for a new stream 424 | 425 | return fileSystemAdapter; 426 | } 427 | 428 | /// 429 | /// If we have an event, then the bookmark moved to the correct file. bookmark is private to the provider, 430 | /// and doesn't get updated in the file 431 | /// 432 | [Fact] 433 | public void ShouldReadFromFirstFileInSetOfExistingFiles() => Assert.Equal(Bufferfile, _sut.File); 434 | } 435 | 436 | public class EqualToLimitNumberOfBufferFiles 437 | { 438 | const int NumberOfFilesToRetain = 1; 439 | FileSetPosition _sut; 440 | 441 | public EqualToLimitNumberOfBufferFiles() 442 | { 443 | var bookmarkProvider = Substitute.For(); 444 | bookmarkProvider 445 | .GetCurrentBookmarkPosition() 446 | .Returns(new FileSetPosition(0, @"c:\unknown.json")); //should force fileset analysis 447 | bookmarkProvider 448 | .When(x => x.UpdateBookmark(Arg.Any())) 449 | .Do(x => _sut = x.ArgAt(0)); 450 | 451 | IFileSystemAdapter fsAdapter = CreateFileSystemAdapter(Bufferfile); 452 | 453 | var provider = new FileBufferDataProvider( 454 | BaseBufferFileName, 455 | fsAdapter, 456 | bookmarkProvider, 457 | Utf8Encoder, 458 | BatchLimit, 459 | EventSizeLimit, 460 | NumberOfFilesToRetain); 461 | 462 | provider.GetNextBatchOfEvents(); 463 | provider.MarkCurrentBatchAsProcessed(); 464 | } 465 | 466 | IFileSystemAdapter CreateFileSystemAdapter(string bufferfile) 467 | { 468 | var fileSystemAdapter = Substitute.For(); 469 | 470 | //get files should return the single buffer file path in this scenario 471 | fileSystemAdapter.GetFiles(Arg.Any(), Arg.Any()) 472 | .Returns(new[] { bufferfile }); 473 | 474 | //when we ask for the buffer file (and only that file), simulate that it exists; for others return false 475 | fileSystemAdapter.Exists(Arg.Any()).Returns(false); 476 | fileSystemAdapter.Exists(bufferfile).Returns(true); 477 | 478 | //Open() should open a stream that can return two events 479 | fileSystemAdapter.Open(bufferfile, Arg.Any(), Arg.Any(), 480 | Arg.Any()) 481 | .Returns(x => 482 | GetSingleEventLineStreamFromResources()); //use this form to reexecute the get stream for a new stream 483 | 484 | return fileSystemAdapter; 485 | } 486 | 487 | /// 488 | /// If we have an event, then the bookmark moved to the correct file. bookmark is private to the provider, 489 | /// and doesn't get updated in the file 490 | /// 491 | [Fact] 492 | public void ShouldReadFromFirstFileInSetOfExistingFiles() => Assert.Equal(Bufferfile, _sut.File); 493 | } 494 | 495 | public class MoreThenTheLimitNumberOfBufferFiles 496 | { 497 | const string UnknownJsonFileName = @"c:\a\unknown.json"; // \a\ to guarantee ordering 498 | const int NumberOfFilesToRetain = 1; 499 | FileSetPosition _sut; 500 | 501 | public MoreThenTheLimitNumberOfBufferFiles() 502 | { 503 | var bookmarkProvider = Substitute.For(); 504 | bookmarkProvider 505 | .GetCurrentBookmarkPosition() 506 | .Returns(new FileSetPosition(0, UnknownJsonFileName)); //should force fileset analysis 507 | bookmarkProvider 508 | .When(x => x.UpdateBookmark(Arg.Any())) 509 | .Do(x => _sut = x.ArgAt(0)); 510 | 511 | IFileSystemAdapter fsAdapter = CreateFileSystemAdapter(Bufferfile); 512 | 513 | var provider = new FileBufferDataProvider( 514 | BaseBufferFileName, 515 | fsAdapter, 516 | bookmarkProvider, 517 | Utf8Encoder, 518 | BatchLimit, 519 | EventSizeLimit, 520 | NumberOfFilesToRetain); 521 | 522 | provider.GetNextBatchOfEvents(); 523 | provider.MarkCurrentBatchAsProcessed(); 524 | 525 | 526 | } 527 | 528 | IFileSystemAdapter CreateFileSystemAdapter(string bufferfile) 529 | { 530 | var fileSystemAdapter = Substitute.For(); 531 | 532 | //get files should return the single buffer file path in this scenario at the end, 533 | // and equal to the number of retained files; unkowns should be ignored 534 | fileSystemAdapter.GetFiles(Arg.Any(), Arg.Any()) 535 | .Returns(new[] { UnknownJsonFileName, UnknownJsonFileName, bufferfile }); 536 | 537 | //when we ask for the buffer file (and only that file), simulate that it exists; for others return false 538 | fileSystemAdapter.Exists(Arg.Any()).Returns(false); 539 | fileSystemAdapter.Exists(bufferfile).Returns(true); 540 | 541 | //Open() should open a stream that can return two events 542 | fileSystemAdapter.Open(bufferfile, Arg.Any(), Arg.Any(), 543 | Arg.Any()) 544 | .Returns(x => 545 | GetSingleEventLineStreamFromResources()); //use this form to reexecute the get stream for a new stream 546 | 547 | return fileSystemAdapter; 548 | } 549 | 550 | /// 551 | /// If we have an event, then the bookmark moved to the correct file. bookmark is private to the provider, 552 | /// and doesn't get updated in the file 553 | /// 554 | [Fact] 555 | public void ShouldReadFromFirstFileInSetOfExistingFiles() => Assert.Equal(Bufferfile, _sut.File); 556 | } 557 | } 558 | 559 | /// 560 | /// this is typically called upon when we reach the end of a buffer file 561 | /// and need to force the marker to move forward to the next file 562 | /// 563 | public class MoveBookmarkForwardCorrectly 564 | { 565 | public class NoRetentionLimitOnBuffer 566 | { 567 | public class NoPreviousBookmark 568 | { 569 | readonly List _deletedFiles = new List(); 570 | FileSetPosition _sut; 571 | 572 | public NoPreviousBookmark() 573 | { 574 | var bookmarkProvider = Substitute.For(); 575 | bookmarkProvider 576 | .When(x => x.UpdateBookmark(Arg.Any())) 577 | .Do(x => _sut = x.ArgAt(0)); 578 | bookmarkProvider.GetCurrentBookmarkPosition().Returns(null as FileSetPosition); 579 | 580 | IFileSystemAdapter fsAdapter = CreateFileSystemAdapter(); 581 | 582 | var provider = new FileBufferDataProvider( 583 | BaseBufferFileName, 584 | fsAdapter, 585 | bookmarkProvider, 586 | Utf8Encoder, 587 | BatchLimit, 588 | EventSizeLimit, 589 | null); 590 | 591 | 592 | provider.MoveBookmarkForward(); 593 | } 594 | 595 | IFileSystemAdapter CreateFileSystemAdapter() 596 | { 597 | var fileSystemAdapter = Substitute.For(); 598 | 599 | //get files should return the single buffer file path in this scenario 600 | fileSystemAdapter.GetFiles(Arg.Any(), Arg.Any()) 601 | .Returns(new[] {@"c:\a\file001.json", @"c:\a\file002.json", @"c:\a\file003.json"}); 602 | 603 | //files exist 604 | fileSystemAdapter.Exists(Arg.Any()).Returns(true); 605 | fileSystemAdapter 606 | .When(x => x.DeleteFile(Arg.Any())) 607 | .Do(x => _deletedFiles.Add(x.ArgAt(0))); 608 | 609 | return fileSystemAdapter; 610 | } 611 | 612 | [Fact] 613 | public void BookmarkShouldBeAtStartOfNextFile() => Assert.Equal(0, _sut.NextLineStart); 614 | 615 | [Fact] 616 | public void BookmarkShouldBeAtNextFile() => Assert.Equal(@"c:\a\file001.json", _sut.File); 617 | 618 | [Fact] 619 | public void NoFileShouldHaveBeenDeleted() => 620 | Assert.Empty(_deletedFiles); 621 | } 622 | 623 | public class PreviousBookmarkStartedOnFirstFile 624 | { 625 | readonly List _deletedFiles = new List(); 626 | FileSetPosition _sut; 627 | 628 | public PreviousBookmarkStartedOnFirstFile() 629 | { 630 | var bookmarkProvider = Substitute.For(); 631 | bookmarkProvider 632 | .When(x => x.UpdateBookmark(Arg.Any())) 633 | .Do(x => _sut = x.ArgAt(0)); 634 | bookmarkProvider.GetCurrentBookmarkPosition().Returns(new FileSetPosition(0, @"c:\a\file001.json")); 635 | 636 | IFileSystemAdapter fsAdapter = CreateFileSystemAdapter(); 637 | 638 | var provider = new FileBufferDataProviderThatAllowscurrentBookmarkToBeSet( 639 | BaseBufferFileName, 640 | fsAdapter, 641 | bookmarkProvider, 642 | Utf8Encoder, 643 | BatchLimit, 644 | EventSizeLimit, 645 | null); 646 | 647 | //force the current Bookmark to be current file: 648 | provider.DefineCurrentBookmark(123, @"c:\a\file001.json"); 649 | 650 | //excercise the SUT 651 | provider.MoveBookmarkForward(); 652 | } 653 | 654 | IFileSystemAdapter CreateFileSystemAdapter() 655 | { 656 | var fileSystemAdapter = Substitute.For(); 657 | 658 | //get files should return the single buffer file path in this scenario 659 | fileSystemAdapter.GetFiles(Arg.Any(), Arg.Any()) 660 | .Returns(new[] { @"c:\a\file001.json", @"c:\a\file002.json", @"c:\a\file003.json" }); 661 | 662 | //files exist 663 | fileSystemAdapter.Exists(Arg.Any()).Returns(true); 664 | fileSystemAdapter 665 | .When(x => x.DeleteFile(Arg.Any())) 666 | .Do(x => _deletedFiles.Add(x.ArgAt(0))); 667 | 668 | return fileSystemAdapter; 669 | } 670 | 671 | [Fact] 672 | public void BookmarkShouldBeAtStartOfNextFile() => Assert.Equal(0, _sut.NextLineStart); 673 | 674 | [Fact] 675 | public void BookmarkShouldBeAtNextFile() => Assert.Equal(@"c:\a\file002.json", _sut.File); 676 | 677 | [Fact] 678 | public void PreviousFileShouldHaveBeenDeleted() => 679 | Assert.Equal(@"c:\a\file001.json", _deletedFiles.First()); 680 | 681 | [Fact] 682 | public void SingleFileShouldHaveBeenDeleted() => 683 | Assert.Single(_deletedFiles); 684 | } 685 | 686 | public class PreviousBookmarkStartedOnSecondFile 687 | { 688 | readonly List _deletedFiles = new List(); 689 | FileSetPosition _sut; 690 | 691 | public PreviousBookmarkStartedOnSecondFile() 692 | { 693 | var bookmarkProvider = Substitute.For(); 694 | bookmarkProvider 695 | .When(x => x.UpdateBookmark(Arg.Any())) 696 | .Do(x => _sut = x.ArgAt(0)); 697 | bookmarkProvider.GetCurrentBookmarkPosition().Returns(new FileSetPosition(0, @"c:\a\file001.json")); 698 | 699 | IFileSystemAdapter fsAdapter = CreateFileSystemAdapter(); 700 | 701 | var provider = new FileBufferDataProviderThatAllowscurrentBookmarkToBeSet( 702 | BaseBufferFileName, 703 | fsAdapter, 704 | bookmarkProvider, 705 | Utf8Encoder, 706 | BatchLimit, 707 | EventSizeLimit, 708 | null); 709 | 710 | //force the current Bookmark to be current file: 711 | provider.DefineCurrentBookmark(123, @"c:\a\file002.json"); 712 | 713 | //excercise the SUT 714 | provider.MoveBookmarkForward(); 715 | } 716 | 717 | IFileSystemAdapter CreateFileSystemAdapter() 718 | { 719 | var fileSystemAdapter = Substitute.For(); 720 | 721 | //get files should return the single buffer file path in this scenario 722 | fileSystemAdapter.GetFiles(Arg.Any(), Arg.Any()) 723 | .Returns(new[] { @"c:\a\file001.json", @"c:\a\file002.json", @"c:\a\file003.json" }); 724 | 725 | //files exist 726 | fileSystemAdapter.Exists(Arg.Any()).Returns(true); 727 | fileSystemAdapter 728 | .When(x => x.DeleteFile(Arg.Any())) 729 | .Do(x => _deletedFiles.Add(x.ArgAt(0))); 730 | 731 | return fileSystemAdapter; 732 | } 733 | 734 | [Fact] 735 | public void BookmarkShouldBeAtStartOfNextFile() => Assert.Equal(0, _sut.NextLineStart); 736 | 737 | [Fact] 738 | public void BookmarkShouldBeAtNextFile() => Assert.Equal(@"c:\a\file003.json", _sut.File); 739 | 740 | [Theory] 741 | [InlineData(@"c:\a\file001.json", 0)] 742 | [InlineData(@"c:\a\file002.json", 1)] 743 | public void PreviousFileShouldHaveBeenDeleted(string file, int position) => 744 | Assert.Equal(file, _deletedFiles[position]); 745 | 746 | [Fact] 747 | public void AllPreviousFilesShouldHaveBeenDeleted() => 748 | Assert.Equal(2, _deletedFiles.Count); 749 | } 750 | 751 | public class PreviousBookmarkStartedOnLastFile 752 | { 753 | readonly List _deletedFiles = new List(); 754 | FileSetPosition _sut; 755 | 756 | public PreviousBookmarkStartedOnLastFile() 757 | { 758 | var bookmarkProvider = Substitute.For(); 759 | bookmarkProvider 760 | .When(x => x.UpdateBookmark(Arg.Any())) 761 | .Do(x => _sut = x.ArgAt(0)); 762 | bookmarkProvider.GetCurrentBookmarkPosition().Returns(new FileSetPosition(0, @"c:\a\file001.json")); 763 | 764 | IFileSystemAdapter fsAdapter = CreateFileSystemAdapter(); 765 | 766 | var provider = new FileBufferDataProviderThatAllowscurrentBookmarkToBeSet( 767 | BaseBufferFileName, 768 | fsAdapter, 769 | bookmarkProvider, 770 | Utf8Encoder, 771 | BatchLimit, 772 | EventSizeLimit, 773 | null); 774 | 775 | //force the current Bookmark to be current file: 776 | provider.DefineCurrentBookmark(123, @"c:\a\file003.json"); 777 | 778 | //excercise the SUT 779 | provider.MoveBookmarkForward(); 780 | } 781 | 782 | IFileSystemAdapter CreateFileSystemAdapter() 783 | { 784 | var fileSystemAdapter = Substitute.For(); 785 | 786 | //get files should return the single buffer file path in this scenario 787 | fileSystemAdapter.GetFiles(Arg.Any(), Arg.Any()) 788 | .Returns(new[] { @"c:\a\file001.json", @"c:\a\file002.json", @"c:\a\file003.json" }); 789 | 790 | //files exist 791 | fileSystemAdapter.Exists(Arg.Any()).Returns(true); 792 | fileSystemAdapter 793 | .When(x => x.DeleteFile(Arg.Any())) 794 | .Do(x => _deletedFiles.Add(x.ArgAt(0))); 795 | 796 | return fileSystemAdapter; 797 | } 798 | 799 | [Fact] 800 | public void BookmarkShouldBeAtEndOfLastFile() => Assert.Equal(123, _sut.NextLineStart); //preserves current 801 | 802 | [Fact] 803 | public void BookmarkShouldStillPointToLastFile() => Assert.Equal(@"c:\a\file003.json", _sut.File); 804 | 805 | [Theory] 806 | [InlineData(@"c:\a\file001.json", 0)] 807 | [InlineData(@"c:\a\file002.json", 1)] 808 | public void PreviousFileShouldHaveBeenDeleted(string file, int position) => 809 | Assert.Equal(file, _deletedFiles[position]); 810 | 811 | [Fact] 812 | public void AllPreviousFilesShouldHaveBeenDeleted() => 813 | Assert.Equal(2, _deletedFiles.Count); 814 | } 815 | } 816 | 817 | public class RetentionLimitLessThenLimitNumberOfBufferFiles 818 | { 819 | const int Limit = 2; 820 | readonly List _deletedFiles = new List(); 821 | FileSetPosition _sut; 822 | 823 | public RetentionLimitLessThenLimitNumberOfBufferFiles() 824 | { 825 | var bookmarkProvider = Substitute.For(); 826 | bookmarkProvider 827 | .When(x => x.UpdateBookmark(Arg.Any())) 828 | .Do(x => _sut = x.ArgAt(0)); 829 | 830 | IFileSystemAdapter fsAdapter = CreateFileSystemAdapter(); 831 | 832 | var provider = new FileBufferDataProvider( 833 | BaseBufferFileName, 834 | fsAdapter, 835 | bookmarkProvider, 836 | Utf8Encoder, 837 | BatchLimit, 838 | EventSizeLimit, 839 | Limit); 840 | 841 | provider.MoveBookmarkForward(); 842 | } 843 | 844 | IFileSystemAdapter CreateFileSystemAdapter() 845 | { 846 | var fileSystemAdapter = Substitute.For(); 847 | 848 | //get files should return the single buffer file path in this scenario 849 | fileSystemAdapter.GetFiles(Arg.Any(), Arg.Any()) 850 | .Returns(new[] { @"c:\a\file001.json", @"c:\a\file002.json", @"c:\a\file003.json", @"c:\a\file004.json" }); 851 | 852 | //files exist 853 | fileSystemAdapter.Exists(Arg.Any()).Returns(true); 854 | fileSystemAdapter 855 | .When(x => x.DeleteFile(Arg.Any())) 856 | .Do(x => _deletedFiles.Add(x.ArgAt(0))); 857 | 858 | return fileSystemAdapter; 859 | } 860 | 861 | [Fact] 862 | public void BookmarkShouldBeAtStartOfNextFile() => Assert.Equal(0, _sut.NextLineStart); 863 | 864 | [Fact] 865 | public void BookmarkShouldBeAtNextFile() => Assert.Equal(@"c:\a\file003.json", _sut.File); 866 | 867 | [Fact] 868 | public void PreviousFilesShouldHaveBeenDeleted() 869 | { 870 | Assert.Equal(@"c:\a\file001.json", _deletedFiles[0]); 871 | Assert.Equal(@"c:\a\file002.json", _deletedFiles[1]); 872 | } 873 | 874 | [Fact] 875 | public void SingleFileShouldHaveBeenDeleted() => 876 | Assert.Equal(2, _deletedFiles.Count); 877 | } 878 | 879 | public class RetentionLimitLessThenNumberOfBufferFilesAndBookMarkOnSecondFile 880 | { 881 | const int Limit = 10; 882 | readonly List _deletedFiles = new List(); 883 | FileSetPosition _sut; 884 | 885 | public RetentionLimitLessThenNumberOfBufferFilesAndBookMarkOnSecondFile() 886 | { 887 | var bookmarkProvider = Substitute.For(); 888 | bookmarkProvider 889 | .When(x => x.UpdateBookmark(Arg.Any())) 890 | .Do(x => _sut = x.ArgAt(0)); 891 | 892 | IFileSystemAdapter fsAdapter = CreateFileSystemAdapter(); 893 | 894 | var provider = new FileBufferDataProviderThatAllowscurrentBookmarkToBeSet( 895 | BaseBufferFileName, 896 | fsAdapter, 897 | bookmarkProvider, 898 | Utf8Encoder, 899 | BatchLimit, 900 | EventSizeLimit, 901 | Limit); 902 | 903 | //force the current Bookmark to be second file: 904 | provider.DefineCurrentBookmark(123, @"c:\a\file002.json"); 905 | 906 | provider.MoveBookmarkForward(); 907 | } 908 | 909 | IFileSystemAdapter CreateFileSystemAdapter() 910 | { 911 | var fileSystemAdapter = Substitute.For(); 912 | 913 | //get files should return the single buffer file path in this scenario 914 | fileSystemAdapter.GetFiles(Arg.Any(), Arg.Any()) 915 | .Returns(new[] 916 | { 917 | @"c:\a\file001.json", 918 | @"c:\a\file002.json", 919 | @"c:\a\file003.json", 920 | @"c:\a\file004.json" 921 | }); 922 | 923 | //files exist 924 | fileSystemAdapter.Exists(Arg.Any()).Returns(true); 925 | fileSystemAdapter 926 | .When(x => x.DeleteFile(Arg.Any())) 927 | .Do(x => _deletedFiles.Add(x.ArgAt(0))); 928 | 929 | return fileSystemAdapter; 930 | } 931 | 932 | [Fact] 933 | public void BookmarkShouldBeAtStartOfNextFile() => Assert.Equal(0, _sut.NextLineStart); 934 | 935 | [Fact] 936 | public void BookmarkShouldBeAtNextFile() => Assert.Equal(@"c:\a\file003.json", _sut.File); 937 | 938 | [Theory] 939 | [InlineData(@"c:\a\file001.json", 0)] 940 | [InlineData(@"c:\a\file002.json", 1)] 941 | public void PreviousFilesShouldHaveBeenDeleted(string expectedDeletedFile, int expectedDeletedIndex) => Assert.Equal(expectedDeletedFile, _deletedFiles[expectedDeletedIndex]); 942 | } 943 | 944 | public class RetentionLimitMoreThenLimitNumberOfBufferFiles 945 | { 946 | const int Limit = 10; 947 | readonly List _deletedFiles = new List(); 948 | FileSetPosition _sut; 949 | 950 | public RetentionLimitMoreThenLimitNumberOfBufferFiles() 951 | { 952 | var bookmarkProvider = Substitute.For(); 953 | bookmarkProvider 954 | .When(x => x.UpdateBookmark(Arg.Any())) 955 | .Do(x => _sut = x.ArgAt(0)); 956 | 957 | IFileSystemAdapter fsAdapter = CreateFileSystemAdapter(); 958 | 959 | var provider = new FileBufferDataProvider( 960 | BaseBufferFileName, 961 | fsAdapter, 962 | bookmarkProvider, 963 | Utf8Encoder, 964 | BatchLimit, 965 | EventSizeLimit, 966 | Limit); 967 | 968 | provider.MoveBookmarkForward(); 969 | } 970 | 971 | IFileSystemAdapter CreateFileSystemAdapter() 972 | { 973 | var fileSystemAdapter = Substitute.For(); 974 | 975 | //get files should return the single buffer file path in this scenario 976 | fileSystemAdapter.GetFiles(Arg.Any(), Arg.Any()) 977 | .Returns(new[] { @"c:\a\file001.json", @"c:\a\file002.json", @"c:\a\file003.json", @"c:\a\file004.json" }); 978 | 979 | //files exist 980 | fileSystemAdapter.Exists(Arg.Any()).Returns(true); 981 | fileSystemAdapter 982 | .When(x => x.DeleteFile(Arg.Any())) 983 | .Do(x => _deletedFiles.Add(x.ArgAt(0))); 984 | 985 | return fileSystemAdapter; 986 | } 987 | 988 | [Fact] 989 | public void BookmarkShouldBeAtStartOfNextFile() => Assert.Equal(0, _sut.NextLineStart); 990 | 991 | [Fact] 992 | public void BookmarkShouldBeAtNextFile() => Assert.Equal(@"c:\a\file001.json", _sut.File); 993 | 994 | [Fact] 995 | public void NoFilesShouldHaveBeenDeleted() => Assert.Empty(_deletedFiles); 996 | } 997 | 998 | public class RetentionLimitMoreThenNumberOfBufferFilesAndBookMarkOnSecondFile 999 | { 1000 | const int Limit = 4; 1001 | readonly List _deletedFiles = new List(); 1002 | FileSetPosition _sut; 1003 | 1004 | public RetentionLimitMoreThenNumberOfBufferFilesAndBookMarkOnSecondFile() 1005 | { 1006 | var bookmarkProvider = Substitute.For(); 1007 | bookmarkProvider 1008 | .When(x => x.UpdateBookmark(Arg.Any())) 1009 | .Do(x => _sut = x.ArgAt(0)); 1010 | 1011 | IFileSystemAdapter fsAdapter = CreateFileSystemAdapter(); 1012 | 1013 | var provider = new FileBufferDataProviderThatAllowscurrentBookmarkToBeSet( 1014 | BaseBufferFileName, 1015 | fsAdapter, 1016 | bookmarkProvider, 1017 | Utf8Encoder, 1018 | BatchLimit, 1019 | EventSizeLimit, 1020 | Limit); 1021 | 1022 | //force the current Bookmark to be current file: 1023 | provider.DefineCurrentBookmark(123, @"c:\a\file002.json"); 1024 | 1025 | provider.MoveBookmarkForward(); 1026 | } 1027 | 1028 | IFileSystemAdapter CreateFileSystemAdapter() 1029 | { 1030 | var fileSystemAdapter = Substitute.For(); 1031 | 1032 | //get files should return the single buffer file path in this scenario 1033 | fileSystemAdapter.GetFiles(Arg.Any(), Arg.Any()) 1034 | .Returns(new[] 1035 | { 1036 | @"c:\a\file001.json", 1037 | @"c:\a\file002.json", 1038 | @"c:\a\file003.json", 1039 | @"c:\a\file004.json", 1040 | @"c:\a\file005.json", //with a limit of 4, this should be the first to be read after moving forward 1041 | @"c:\a\file006.json", 1042 | @"c:\a\file007.json", 1043 | @"c:\a\file008.json" 1044 | }); 1045 | 1046 | //files exist 1047 | fileSystemAdapter.Exists(Arg.Any()).Returns(true); 1048 | fileSystemAdapter 1049 | .When(x => x.DeleteFile(Arg.Any())) 1050 | .Do(x => _deletedFiles.Add(x.ArgAt(0))); 1051 | 1052 | return fileSystemAdapter; 1053 | } 1054 | 1055 | [Fact] 1056 | public void BookmarkShouldBeAtStartOfNextFile() => Assert.Equal(0, _sut.NextLineStart); 1057 | 1058 | [Fact] 1059 | public void BookmarkShouldBeAtNextFile() => Assert.Equal(@"c:\a\file005.json", _sut.File); 1060 | 1061 | [Theory] 1062 | [InlineData(@"c:\a\file001.json", 0)] 1063 | [InlineData(@"c:\a\file002.json", 1)] 1064 | [InlineData(@"c:\a\file003.json", 2)] 1065 | [InlineData(@"c:\a\file004.json", 3)] 1066 | public void PreviousFilesShouldHaveBeenDeleted(string expectedDeletedFile, int expectedDeletedIndex) => Assert.Equal(expectedDeletedFile, _deletedFiles[expectedDeletedIndex]); 1067 | } 1068 | 1069 | 1070 | } 1071 | 1072 | 1073 | 1074 | static Stream Get20LineStreamFromResources() 1075 | { 1076 | var resourceNameSuffix = Environment.NewLine.Length == 2 ? "RN" : "N"; 1077 | var resourceName = $"{ResourceNamespace}.SampleBuffers.20Events{resourceNameSuffix}.json"; 1078 | return GetStreamFromResources(resourceName); 1079 | } 1080 | 1081 | static Stream GetSingleEventLineStreamFromResources() 1082 | { 1083 | var resourceName = $"{ResourceNamespace}.SampleBuffers.singleEvent.json"; 1084 | return GetStreamFromResources(resourceName); 1085 | } 1086 | 1087 | static Stream GetStreamFromResources(string resourceName) 1088 | { 1089 | MemoryStream ms = new MemoryStream(); 1090 | typeof(FileBufferDataProviderTests) 1091 | .GetTypeInfo() 1092 | .Assembly 1093 | .GetManifestResourceStream(resourceName) 1094 | ?.CopyTo(ms); 1095 | return ms; 1096 | } 1097 | } 1098 | } 1099 | -------------------------------------------------------------------------------- /test/Serilog.Sinks.Loggly.Tests/Sinks/Loggly/InvalidPayloadLoggerTests.cs: -------------------------------------------------------------------------------- 1 | using Loggly; 2 | using System; 3 | using System.Collections.Generic; 4 | using System.IO; 5 | using System.Reflection; 6 | using System.Text; 7 | using System.Text.RegularExpressions; 8 | using NSubstitute; 9 | using Xunit; 10 | using Serilog.Debugging; 11 | 12 | namespace Serilog.Sinks.Loggly.Tests.Sinks.Loggly 13 | { 14 | public class InvalidPayloadLoggerTests 15 | { 16 | const string LogFolder = @"C:\tests"; //any path here will do. 17 | static readonly Encoding _utf8Encoder = new UTF8Encoding(true); 18 | 19 | [Fact] 20 | public void CanCreateInvalidShippmentLoggerInstance() 21 | { 22 | var instance = new InvalidPayloadLogger(LogFolder, _utf8Encoder, Substitute.For()); 23 | Assert.NotNull(instance); 24 | } 25 | 26 | public class InvalidPayloadPersistenceTests 27 | { 28 | string _writtenData; 29 | string _generatedFilename; 30 | 31 | public InvalidPayloadPersistenceTests() 32 | { 33 | var fsAdapter = Substitute.For(); 34 | fsAdapter.When(x => x.WriteAllBytes(Arg.Any(), Arg.Any())) 35 | .Do(x => 36 | { 37 | _generatedFilename = x.ArgAt(0); 38 | _writtenData = _utf8Encoder.GetString(x.ArgAt(1)); 39 | }); 40 | 41 | 42 | //simulate the Post to Loggly failure with an error response and fixed payload. 43 | var response = new LogResponse { Code = ResponseCode.Error, Message = "502 Bad Request" }; 44 | //just need an empty event for testing 45 | var payload = new List 46 | { 47 | new LogglyEvent 48 | { 49 | Data = new MessageData(), 50 | Options = new EventOptions(), 51 | Syslog = new SyslogHeader() {MessageId = 0}, 52 | Timestamp = DateTimeOffset.Parse("2017-09-27T00:00:00+00:00") //fixed date for comparisson 53 | } 54 | }; 55 | 56 | var instance = new InvalidPayloadLogger(LogFolder, _utf8Encoder, fsAdapter); 57 | //exercice the method 58 | instance.DumpInvalidPayload(response, payload); 59 | } 60 | 61 | [Fact] 62 | public void GeneratedFileHasEventsAndErrorInfoInContent() 63 | { 64 | using (var expectedFileTextStream = GetExpectedFileTextStream()) 65 | { 66 | #pragma warning disable SG0018 // Path traversal 67 | using (var reader = new StreamReader(expectedFileTextStream, _utf8Encoder, true)) 68 | #pragma warning restore SG0018 // Path traversal 69 | { 70 | var expectedFileTestString = reader.ReadToEnd(); 71 | Assert.Equal(expectedFileTestString, _writtenData); 72 | } 73 | } 74 | } 75 | 76 | [Fact] 77 | public void GeneratedFileHasExpectedname() 78 | { 79 | var expectedFileNameRegex = new Regex(@"invalid-\d{14}-Error-[a-fA-F0-9]{32}.json$"); 80 | Assert.Matches(expectedFileNameRegex,_generatedFilename); 81 | } 82 | } 83 | 84 | static Stream GetExpectedFileTextStream() 85 | { 86 | var resourceNameSuffix = Environment.NewLine.Length == 2 ? "RN" : "N"; 87 | var resourceName = $"Serilog.Sinks.Loggly.Tests.Sinks.Loggly.Expectations.expectedInvalidPayloadFile{resourceNameSuffix}.json"; 88 | return typeof(InvalidPayloadLoggerTests) 89 | .GetTypeInfo() 90 | .Assembly 91 | .GetManifestResourceStream(resourceName); 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /test/Serilog.Sinks.Loggly.Tests/Sinks/Loggly/SampleBuffers/20EventsN.json: -------------------------------------------------------------------------------- 1 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 2 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 3 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 4 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 5 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 6 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 7 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 8 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 9 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 10 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 11 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 12 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 13 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 14 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 15 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 16 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 17 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 18 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 19 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 20 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} -------------------------------------------------------------------------------- /test/Serilog.Sinks.Loggly.Tests/Sinks/Loggly/SampleBuffers/20EventsRN.json: -------------------------------------------------------------------------------- 1 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 2 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 3 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 4 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 5 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 6 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 7 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 8 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 9 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 10 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 11 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 12 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 13 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 14 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 15 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 16 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 17 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 18 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 19 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} 20 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} -------------------------------------------------------------------------------- /test/Serilog.Sinks.Loggly.Tests/Sinks/Loggly/SampleBuffers/singleEvent.json: -------------------------------------------------------------------------------- 1 | {"Timestamp": "2017-09-24T22:30:09.5025793+00:00","Syslog": {"MessageId": 0,"Level": 6},"Data": {"Message": "Finished handling request...","MachineName": "test-A1","Level": "Information"},"Options": {"Tags": []}} --------------------------------------------------------------------------------