├── .gitattributes ├── .gitignore ├── .nuke ├── build.schema.json └── parameters.json ├── Dockerfile ├── LICENSE.txt ├── README.md ├── build.cmd ├── build.ps1 ├── build.sh ├── build ├── .editorconfig ├── Build.cs ├── Configuration.cs ├── Directory.Build.props ├── Directory.Build.targets ├── _build.csproj └── _build.csproj.DotSettings ├── global.json ├── scripts └── reset-seq.cmd └── source ├── SeqFlatFileImport.Core ├── FileFormats │ ├── IFileFormat.cs │ ├── Iis.cs │ ├── OctopusRawTask.cs │ ├── OctopusServer.cs │ ├── OctopusTask.cs │ └── OctopusWebLog.cs ├── Importer.cs ├── Option.cs ├── Properties │ └── AssemblyInfo.cs ├── RawEvent.cs ├── Reader.cs ├── Result.cs ├── SeqEndpoint.cs └── SeqFlatFileImport.Core.csproj ├── SeqFlatFileImport.Tests ├── AutodetectByContentsTest.cs ├── AutodetectByFileNameTests.cs ├── Helpers │ ├── StubSeqEndpoint.cs │ └── TestHelper.cs ├── LogFiles │ ├── OctopusServer.txt │ ├── ServerTasks-16572.log.txt │ ├── Web-2017-03-28.log │ ├── servertasks-21_gjzwyyv2kt.txt │ └── u_ex150708.log ├── Parse │ ├── ParseTests.Iis.approved.json │ ├── ParseTests.OctopusRawTask.approved.json │ ├── ParseTests.OctopusServer.approved.json │ ├── ParseTests.OctopusTask.approved.json │ ├── ParseTests.OctopusWebLog.approved.json │ └── ParseTests.cs ├── Properties │ └── AssemblyInfo.cs └── SeqFlatFileImport.Tests.csproj ├── SeqFlatFileImport.sln └── SeqFlatFileImport ├── Options.cs ├── Program.cs ├── Properties └── AssemblyInfo.cs └── SeqFlatFileImport.csproj /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.*scc 2 | *.FileListAbsolute.txt 3 | *.aps 4 | *.bak 5 | *.[Cc]ache 6 | *.clw 7 | *.eto 8 | *.fb6lck 9 | *.fbl6 10 | *.fbpInf 11 | *.ilk 12 | *.lib 13 | *.log 14 | *.ncb 15 | *.nlb 16 | *.obj 17 | *.patch 18 | *.pch 19 | *.pdb 20 | *.plg 21 | *.[Pp]ublish.xml 22 | *.rdl.data 23 | *.sbr 24 | *.scc 25 | *.sig 26 | *.sqlsuo 27 | *.suo 28 | *.svclog 29 | *.tlb 30 | *.tlh 31 | *.tli 32 | *.tmp 33 | *.user 34 | *.vshost.* 35 | *DXCore.Solution 36 | *_i.c 37 | *_p.c 38 | Ankh.Load 39 | Backup*/ 40 | CVS/ 41 | PrecompiledWeb/ 42 | UpgradeLog*.* 43 | [Bb]in/ 44 | [Dd]ebug/ 45 | [Oo]bj/ 46 | [Rr]elease/ 47 | [Tt]humbs.db 48 | _UpgradeReport_Files 49 | _[Rr]e[Ss]harper.*/ 50 | hgignore[.-]* 51 | ignore[.-]* 52 | svnignore[.-]* 53 | lint.db 54 | tools/TestResult.xml 55 | *.ReSharper 56 | source/packages 57 | source/OctopusTools.v2.ncrunchsolution 58 | *.orig 59 | *.userprefs 60 | *.lock.json 61 | .vs 62 | .vscode 63 | /tools/ 64 | /artifacts/ 65 | /publish/ 66 | TestResult.xml 67 | *.received.* 68 | packages 69 | *.trx 70 | .idea/ 71 | !**/LogFiles/* -------------------------------------------------------------------------------- /.nuke/build.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-04/schema#", 3 | "title": "Build Schema", 4 | "$ref": "#/definitions/build", 5 | "definitions": { 6 | "build": { 7 | "type": "object", 8 | "properties": { 9 | "AutoDetectBranch": { 10 | "type": "boolean", 11 | "description": "Whether to auto-detect the branch name - this is okay for a local build, but should not be used under CI" 12 | }, 13 | "Configuration": { 14 | "type": "string", 15 | "description": "Configuration to build - Default is 'Debug' (local) or 'Release' (server)", 16 | "enum": [ 17 | "Debug", 18 | "Release" 19 | ] 20 | }, 21 | "Continue": { 22 | "type": "boolean", 23 | "description": "Indicates to continue a previously failed build attempt" 24 | }, 25 | "Help": { 26 | "type": "boolean", 27 | "description": "Shows the help text for this build assembly" 28 | }, 29 | "Host": { 30 | "type": "string", 31 | "description": "Host for execution. Default is 'automatic'", 32 | "enum": [ 33 | "AppVeyor", 34 | "AzurePipelines", 35 | "Bamboo", 36 | "Bitbucket", 37 | "Bitrise", 38 | "GitHubActions", 39 | "GitLab", 40 | "Jenkins", 41 | "Rider", 42 | "SpaceAutomation", 43 | "TeamCity", 44 | "Terminal", 45 | "TravisCI", 46 | "VisualStudio", 47 | "VSCode" 48 | ] 49 | }, 50 | "NoLogo": { 51 | "type": "boolean", 52 | "description": "Disables displaying the NUKE logo" 53 | }, 54 | "OCTOVERSION_CurrentBranch": { 55 | "type": "string", 56 | "description": "Branch name for OctoVersion to use to calculate the version number. Can be set via the environment variable OCTOVERSION_CurrentBranch" 57 | }, 58 | "Partition": { 59 | "type": "string", 60 | "description": "Partition to use on CI" 61 | }, 62 | "Plan": { 63 | "type": "boolean", 64 | "description": "Shows the execution plan (HTML)" 65 | }, 66 | "Profile": { 67 | "type": "array", 68 | "description": "Defines the profiles to load", 69 | "items": { 70 | "type": "string" 71 | } 72 | }, 73 | "Root": { 74 | "type": "string", 75 | "description": "Root directory during build execution" 76 | }, 77 | "Skip": { 78 | "type": "array", 79 | "description": "List of targets to be skipped. Empty list skips all dependencies", 80 | "items": { 81 | "type": "string", 82 | "enum": [ 83 | "BuildDockerImage", 84 | "Clean", 85 | "Compile", 86 | "Default", 87 | "Publish", 88 | "Restore", 89 | "Test" 90 | ] 91 | } 92 | }, 93 | "Solution": { 94 | "type": "string", 95 | "description": "Path to a solution file that is automatically loaded" 96 | }, 97 | "Target": { 98 | "type": "array", 99 | "description": "List of targets to be invoked. Default is '{default_target}'", 100 | "items": { 101 | "type": "string", 102 | "enum": [ 103 | "BuildDockerImage", 104 | "Clean", 105 | "Compile", 106 | "Default", 107 | "Publish", 108 | "Restore", 109 | "Test" 110 | ] 111 | } 112 | }, 113 | "Verbosity": { 114 | "type": "string", 115 | "description": "Logging verbosity during build execution. Default is 'Normal'", 116 | "enum": [ 117 | "Minimal", 118 | "Normal", 119 | "Quiet", 120 | "Verbose" 121 | ] 122 | } 123 | } 124 | } 125 | } 126 | } -------------------------------------------------------------------------------- /.nuke/parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "./build.schema.json", 3 | "Solution": "source/SeqFlatFileImport.sln" 4 | } -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/dotnet/aspnet:6.0 2 | COPY ./artifacts . 3 | ENTRYPOINT ["dotnet", "SeqFlatFileImport.dll"] 4 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) Octopus Deploy and contributors. All rights reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use 4 | these files except in compliance with the License. You may obtain a copy of the 5 | License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software distributed 10 | under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 11 | CONDITIONS OF ANY KIND, either express or implied. See the License for the 12 | specific language governing permissions and limitations under the License. 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Flat log file to Seq importer 2 | 3 | A tool for parsing flat log files and importing them into Seq. 4 | 5 | Features: 6 | - Auto-detects file type 7 | - Parses the log entries down for a somewhat structured logging experience (ie. log level, time, thread) including certain common/useful messages so that it uses a common message template. 8 | - Supports: 9 | - Octopus Server Log 10 | - Web Log 11 | - IIS Log 12 | - Octopus Deployment log exported from the web UI 13 | - It can easily be extended to other log formats or parsing extra log lines 14 | 15 | ## Example 16 | 17 | `SeqFlatFileImport.exe OctopusServer.txt OctopusServer.0.txt --batch MyDebuggingTask` 18 | 19 | `SeqFlatFileImport.exe c:\logs --batch MyDebuggingTask` 20 | 21 | ## Getting Started 22 | 1. [Install](https://nuke.build/docs/introduction/) and run `nuke` or run the `Default` target from your IDE (eg. Rider with the NUKE plugin) 23 | 2. Running directly: 24 | 1. See examples above 25 | 3. Running in Docker: 26 | 1. A docker image with the tag `seq-flat-file-import` is created from the Nuke build 27 | 2. You will need to run a Seq instance in a separate container 28 | 3. Example of importing all log files in your current working directory: 29 | 1. Windows: `docker run --rm -v ${PWD}:/logs seq-flat-file-import --server=http://host.docker.internal:5341 ./logs` 30 | 2. Linux/Mac: `docker run --rm -v $(pwd):/logs seq-flat-file-import --server=http://host.docker.internal:5341 ./logs` 31 | 32 | `.\scripts\reset-seq.cmd` can be used to reset your local Seq instance, however this does not work with Seq instances in a container. -------------------------------------------------------------------------------- /build.cmd: -------------------------------------------------------------------------------- 1 | :; set -eo pipefail 2 | :; SCRIPT_DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd) 3 | :; ${SCRIPT_DIR}/build.sh "$@" 4 | :; exit $? 5 | 6 | @ECHO OFF 7 | powershell -ExecutionPolicy ByPass -NoProfile -File "%~dp0build.ps1" %* 8 | -------------------------------------------------------------------------------- /build.ps1: -------------------------------------------------------------------------------- 1 | [CmdletBinding()] 2 | Param( 3 | [Parameter(Position=0,Mandatory=$false,ValueFromRemainingArguments=$true)] 4 | [string[]]$BuildArguments 5 | ) 6 | 7 | Write-Output "PowerShell $($PSVersionTable.PSEdition) version $($PSVersionTable.PSVersion)" 8 | 9 | Set-StrictMode -Version 2.0; $ErrorActionPreference = "Stop"; $ConfirmPreference = "None"; trap { Write-Error $_ -ErrorAction Continue; exit 1 } 10 | $PSScriptRoot = Split-Path $MyInvocation.MyCommand.Path -Parent 11 | 12 | ########################################################################### 13 | # CONFIGURATION 14 | ########################################################################### 15 | 16 | $BuildProjectFile = "$PSScriptRoot\build\_build.csproj" 17 | $TempDirectory = "$PSScriptRoot\\.nuke\temp" 18 | 19 | $DotNetGlobalFile = "$PSScriptRoot\\global.json" 20 | $DotNetInstallUrl = "https://dot.net/v1/dotnet-install.ps1" 21 | $DotNetChannel = "STS" 22 | 23 | $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = 1 24 | $env:DOTNET_CLI_TELEMETRY_OPTOUT = 1 25 | $env:DOTNET_MULTILEVEL_LOOKUP = 0 26 | 27 | ########################################################################### 28 | # EXECUTION 29 | ########################################################################### 30 | 31 | function ExecSafe([scriptblock] $cmd) { 32 | & $cmd 33 | if ($LASTEXITCODE) { exit $LASTEXITCODE } 34 | } 35 | 36 | # If dotnet CLI is installed globally and it matches requested version, use for execution 37 | if ($null -ne (Get-Command "dotnet" -ErrorAction SilentlyContinue) -and ` 38 | $(dotnet --version) -and $LASTEXITCODE -eq 0) { 39 | $env:DOTNET_EXE = (Get-Command "dotnet").Path 40 | } 41 | else { 42 | # Download install script 43 | $DotNetInstallFile = "$TempDirectory\dotnet-install.ps1" 44 | New-Item -ItemType Directory -Path $TempDirectory -Force | Out-Null 45 | [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 46 | (New-Object System.Net.WebClient).DownloadFile($DotNetInstallUrl, $DotNetInstallFile) 47 | 48 | # If global.json exists, load expected version 49 | if (Test-Path $DotNetGlobalFile) { 50 | $DotNetGlobal = $(Get-Content $DotNetGlobalFile | Out-String | ConvertFrom-Json) 51 | if ($DotNetGlobal.PSObject.Properties["sdk"] -and $DotNetGlobal.sdk.PSObject.Properties["version"]) { 52 | $DotNetVersion = $DotNetGlobal.sdk.version 53 | } 54 | } 55 | 56 | # Install by channel or version 57 | $DotNetDirectory = "$TempDirectory\dotnet-win" 58 | if (!(Test-Path variable:DotNetVersion)) { 59 | ExecSafe { & powershell $DotNetInstallFile -InstallDir $DotNetDirectory -Channel $DotNetChannel -NoPath } 60 | } else { 61 | ExecSafe { & powershell $DotNetInstallFile -InstallDir $DotNetDirectory -Version $DotNetVersion -NoPath } 62 | } 63 | $env:DOTNET_EXE = "$DotNetDirectory\dotnet.exe" 64 | } 65 | 66 | Write-Output "Microsoft (R) .NET SDK version $(& $env:DOTNET_EXE --version)" 67 | 68 | if (Test-Path env:NUKE_ENTERPRISE_TOKEN) { 69 | & $env:DOTNET_EXE nuget remove source "nuke-enterprise" > $null 70 | & $env:DOTNET_EXE nuget add source "https://f.feedz.io/nuke/enterprise/nuget" --name "nuke-enterprise" --username "PAT" --password $env:NUKE_ENTERPRISE_TOKEN > $null 71 | } 72 | 73 | ExecSafe { & $env:DOTNET_EXE build $BuildProjectFile /nodeReuse:false /p:UseSharedCompilation=false -nologo -clp:NoSummary --verbosity quiet } 74 | ExecSafe { & $env:DOTNET_EXE run --project $BuildProjectFile --no-build -- $BuildArguments } 75 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | bash --version 2>&1 | head -n 1 4 | 5 | set -eo pipefail 6 | SCRIPT_DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd) 7 | 8 | ########################################################################### 9 | # CONFIGURATION 10 | ########################################################################### 11 | 12 | BUILD_PROJECT_FILE="$SCRIPT_DIR/build/_build.csproj" 13 | TEMP_DIRECTORY="$SCRIPT_DIR//.nuke/temp" 14 | 15 | DOTNET_GLOBAL_FILE="$SCRIPT_DIR//global.json" 16 | DOTNET_INSTALL_URL="https://dot.net/v1/dotnet-install.sh" 17 | DOTNET_CHANNEL="STS" 18 | 19 | export DOTNET_CLI_TELEMETRY_OPTOUT=1 20 | export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 21 | export DOTNET_MULTILEVEL_LOOKUP=0 22 | 23 | ########################################################################### 24 | # EXECUTION 25 | ########################################################################### 26 | 27 | function FirstJsonValue { 28 | perl -nle 'print $1 if m{"'"$1"'": "([^"]+)",?}' <<< "${@:2}" 29 | } 30 | 31 | # If dotnet CLI is installed globally and it matches requested version, use for execution 32 | if [ -x "$(command -v dotnet)" ] && dotnet --version &>/dev/null; then 33 | export DOTNET_EXE="$(command -v dotnet)" 34 | else 35 | # Download install script 36 | DOTNET_INSTALL_FILE="$TEMP_DIRECTORY/dotnet-install.sh" 37 | mkdir -p "$TEMP_DIRECTORY" 38 | curl -Lsfo "$DOTNET_INSTALL_FILE" "$DOTNET_INSTALL_URL" 39 | chmod +x "$DOTNET_INSTALL_FILE" 40 | 41 | # If global.json exists, load expected version 42 | if [[ -f "$DOTNET_GLOBAL_FILE" ]]; then 43 | DOTNET_VERSION=$(FirstJsonValue "version" "$(cat "$DOTNET_GLOBAL_FILE")") 44 | if [[ "$DOTNET_VERSION" == "" ]]; then 45 | unset DOTNET_VERSION 46 | fi 47 | fi 48 | 49 | # Install by channel or version 50 | DOTNET_DIRECTORY="$TEMP_DIRECTORY/dotnet-unix" 51 | if [[ -z ${DOTNET_VERSION+x} ]]; then 52 | "$DOTNET_INSTALL_FILE" --install-dir "$DOTNET_DIRECTORY" --channel "$DOTNET_CHANNEL" --no-path 53 | else 54 | "$DOTNET_INSTALL_FILE" --install-dir "$DOTNET_DIRECTORY" --version "$DOTNET_VERSION" --no-path 55 | fi 56 | export DOTNET_EXE="$DOTNET_DIRECTORY/dotnet" 57 | fi 58 | 59 | echo "Microsoft (R) .NET SDK version $("$DOTNET_EXE" --version)" 60 | 61 | if [[ ! -z ${NUKE_ENTERPRISE_TOKEN+x} && "$NUKE_ENTERPRISE_TOKEN" != "" ]]; then 62 | "$DOTNET_EXE" nuget remove source "nuke-enterprise" &>/dev/null || true 63 | "$DOTNET_EXE" nuget add source "https://f.feedz.io/nuke/enterprise/nuget" --name "nuke-enterprise" --username "PAT" --password "$NUKE_ENTERPRISE_TOKEN" --store-password-in-clear-text &>/dev/null || true 64 | fi 65 | 66 | "$DOTNET_EXE" build "$BUILD_PROJECT_FILE" /nodeReuse:false /p:UseSharedCompilation=false -nologo -clp:NoSummary --verbosity quiet 67 | "$DOTNET_EXE" run --project "$BUILD_PROJECT_FILE" --no-build -- "$@" 68 | -------------------------------------------------------------------------------- /build/.editorconfig: -------------------------------------------------------------------------------- 1 | [*.cs] 2 | dotnet_style_qualification_for_field = false:warning 3 | dotnet_style_qualification_for_property = false:warning 4 | dotnet_style_qualification_for_method = false:warning 5 | dotnet_style_qualification_for_event = false:warning 6 | dotnet_style_require_accessibility_modifiers = never:warning 7 | 8 | csharp_style_expression_bodied_methods = true:silent 9 | csharp_style_expression_bodied_properties = true:warning 10 | csharp_style_expression_bodied_indexers = true:warning 11 | csharp_style_expression_bodied_accessors = true:warning 12 | -------------------------------------------------------------------------------- /build/Build.cs: -------------------------------------------------------------------------------- 1 | using Nuke.Common; 2 | using Nuke.Common.IO; 3 | using Nuke.Common.ProjectModel; 4 | using Nuke.Common.Tools.Docker; 5 | using Nuke.Common.Tools.DotNet; 6 | using Nuke.Common.Tools.GitVersion; 7 | using Nuke.Common.Tools.OctoVersion; 8 | using Nuke.Common.Utilities.Collections; 9 | using static Nuke.Common.Tools.DotNet.DotNetTasks; 10 | using static Nuke.Common.Tools.Docker.DockerTasks; 11 | 12 | class Build : NukeBuild 13 | { 14 | [Parameter("Configuration to build - Default is 'Debug' (local) or 'Release' (server)")] 15 | readonly Configuration Configuration = IsLocalBuild ? Configuration.Debug : Configuration.Release; 16 | 17 | [Solution(GenerateProjects = true)] 18 | readonly Solution Solution; 19 | 20 | [Parameter("Branch name for OctoVersion to use to calculate the version number. Can be set via the environment variable OCTOVERSION_CurrentBranch.", 21 | Name = "OCTOVERSION_CurrentBranch")] 22 | readonly string BranchName; 23 | 24 | [Parameter("Whether to auto-detect the branch name - this is okay for a local build, but should not be used under CI.")] 25 | readonly bool AutoDetectBranch = IsLocalBuild; 26 | 27 | [OctoVersion(UpdateBuildNumber = true, BranchMember = nameof(BranchName), AutoDetectBranchMember = nameof(AutoDetectBranch))] 28 | readonly OctoVersionInfo OctoVersionInfo; 29 | 30 | AbsolutePath SourceDirectory => RootDirectory / "source"; 31 | AbsolutePath ArtifactsDirectory => RootDirectory / "artifacts"; 32 | GitVersion GitVersionInfo; 33 | string NuGetVersion; 34 | 35 | Target Clean => _ => _ 36 | .Executes(() => 37 | { 38 | ArtifactsDirectory.CreateOrCleanDirectory(); 39 | SourceDirectory.GlobDirectories("**/bin", "**/obj").ForEach(x => x.DeleteDirectory()); 40 | }); 41 | 42 | Target Restore => _ => _ 43 | .DependsOn(Clean) 44 | .Executes(() => 45 | { 46 | DotNetRestore(_ => _ 47 | .SetProjectFile(Solution)); 48 | }); 49 | 50 | Target Compile => _ => _ 51 | .DependsOn(Restore) 52 | .Executes(() => 53 | { 54 | DotNetBuild(_ => _ 55 | .SetProjectFile(Solution) 56 | .SetConfiguration(Configuration) 57 | .SetVersion(OctoVersionInfo.FullSemVer) 58 | .SetInformationalVersion(OctoVersionInfo.InformationalVersion) 59 | .EnableNoRestore()); 60 | }); 61 | 62 | Target Test => _ => _ 63 | .DependsOn(Compile) 64 | .Executes(() => 65 | { 66 | DotNetTest(_ => _ 67 | .SetProjectFile(Solution) 68 | .SetConfiguration(Configuration) 69 | .EnableNoBuild() 70 | .EnableNoRestore()); 71 | }); 72 | 73 | Target Publish => _ => _ 74 | .DependsOn(Test) 75 | .Executes(() => 76 | { 77 | DotNetPublish(_ => _ 78 | .SetProject(Solution.SeqFlatFileImport) 79 | .SetConfiguration(Configuration) 80 | .SetOutput(ArtifactsDirectory)); 81 | }); 82 | 83 | Target BuildDockerImage => _ => _ 84 | .DependsOn(Publish) 85 | .Executes(() => 86 | { 87 | DockerBuild(x => x 88 | .SetPath(RootDirectory) 89 | .SetFile(RootDirectory / "Dockerfile") 90 | .SetTag("seq-flat-file-import")); 91 | }); 92 | 93 | Target Default => _ => _ 94 | .DependsOn(BuildDockerImage); 95 | 96 | public static int Main() => Execute(x => x.Default); 97 | } -------------------------------------------------------------------------------- /build/Configuration.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.ComponentModel; 3 | using System.Linq; 4 | using Nuke.Common.Tooling; 5 | 6 | [TypeConverter(typeof(TypeConverter))] 7 | public class Configuration : Enumeration 8 | { 9 | public static Configuration Debug = new Configuration { Value = nameof(Debug) }; 10 | public static Configuration Release = new Configuration { Value = nameof(Release) }; 11 | 12 | public static implicit operator string(Configuration configuration) 13 | { 14 | return configuration.Value; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /build/Directory.Build.props: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /build/Directory.Build.targets: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /build/_build.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | net8.0 6 | CS0649;CS0169 7 | .. 8 | .. 9 | 1 10 | 11 | true 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /build/_build.csproj.DotSettings: -------------------------------------------------------------------------------- 1 |  2 | DO_NOT_SHOW 3 | DO_NOT_SHOW 4 | DO_NOT_SHOW 5 | DO_NOT_SHOW 6 | Implicit 7 | Implicit 8 | ExpressionBody 9 | 0 10 | NEXT_LINE 11 | True 12 | False 13 | 120 14 | IF_OWNER_IS_SINGLE_LINE 15 | WRAP_IF_LONG 16 | False 17 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 18 | <Policy Inspect="True" Prefix="" Suffix="" Style="AaBb" /> 19 | True 20 | True 21 | True 22 | True 23 | True 24 | True 25 | True 26 | True 27 | True 28 | -------------------------------------------------------------------------------- /global.json: -------------------------------------------------------------------------------- 1 | { 2 | "sdk": { 3 | "version": "8.0.100", 4 | "rollForward": "latestFeature" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /scripts/reset-seq.cmd: -------------------------------------------------------------------------------- 1 | net stop Seq 2 | 3 | rmdir "C:\ProgramData\Seq" /S /Q 4 | 5 | net start Seq 6 | 7 | start "" http://localhost:5341 -------------------------------------------------------------------------------- /source/SeqFlatFileImport.Core/FileFormats/IFileFormat.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace SeqFlatFileImport.Core.FileFormats 4 | { 5 | public interface IFileFormat 6 | { 7 | string Name { get; } 8 | IReadOnlyList AutodetectFileNameRegexes { get; } 9 | int Ordinal { get; } 10 | bool AutodetectFromContents(string[] firstFewLines); 11 | IEnumerable Read(IEnumerable lines); 12 | } 13 | } -------------------------------------------------------------------------------- /source/SeqFlatFileImport.Core/FileFormats/Iis.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | 6 | namespace SeqFlatFileImport.Core.FileFormats 7 | { 8 | public class Iis : IFileFormat 9 | { 10 | public virtual string Name => "IIS"; 11 | 12 | public virtual IReadOnlyList AutodetectFileNameRegexes { get; } = new[] 13 | { 14 | @"u_ex[0-9]{6}\.log" 15 | }; 16 | 17 | public int Ordinal => 0; 18 | 19 | public bool AutodetectFromContents(string[] firstFewLines) 20 | { 21 | if (firstFewLines.Length == 0) 22 | return false; 23 | return firstFewLines[0].StartsWith("#Software: Microsoft Internet Information Services"); 24 | } 25 | 26 | public IEnumerable Read(IEnumerable lines) 27 | { 28 | var fieldNames = Array.Empty(); 29 | var template = ""; 30 | 31 | foreach (var line in lines.Where(l => l.Length > 0)) 32 | { 33 | if (line[0] == '#') 34 | { 35 | if (line.StartsWith("#Fields")) 36 | { 37 | fieldNames = line.Split(' ').Skip(1).Select(s => s.Replace("-", "_")).ToArray(); 38 | template = GetTemplate(fieldNames); 39 | } 40 | } 41 | else 42 | { 43 | var fields = line.Split(' '); 44 | var properties = new Dictionary(); 45 | for (var x = 0; x < fields.Length; x++) 46 | properties[fieldNames.Length > x ? fieldNames[x] : x.ToString()] = fields[x]; 47 | 48 | 49 | yield return new RawEvent() 50 | { 51 | Timestamp = GetTimestamp(properties), 52 | Level = "Information", 53 | MessageTemplate = template, 54 | Properties = properties 55 | }; 56 | } 57 | } 58 | } 59 | 60 | private string GetTemplate(string[] fieldNames) 61 | { 62 | var sb = new StringBuilder(); 63 | if (fieldNames.Contains("cs_method")) 64 | sb.Append("{cs_method}"); 65 | if (fieldNames.Contains("time_taken")) 66 | sb.Append(" {time_taken}"); 67 | if (fieldNames.Contains("sc_status")) 68 | { 69 | sb.Append("{sc_status}"); 70 | if (fieldNames.Contains("sc_substatus")) 71 | sb.Append(".{sc_substatus}"); 72 | } 73 | if (fieldNames.Contains("cs_uri_stem")) 74 | sb.Append(" {cs_uri_stem} "); 75 | if (fieldNames.Contains("cs_uri_query")) 76 | sb.Append(" {cs_uri_query} "); 77 | return sb.ToString().Trim(); 78 | } 79 | 80 | private DateTimeOffset GetTimestamp(Dictionary properties) 81 | { 82 | if (!properties.ContainsKey("date")) 83 | return DateTimeOffset.Now; 84 | 85 | var datetime = properties.ContainsKey("time") ? DateTime.Parse(properties["date"] + " " + properties["time"]) : DateTime.Parse(""+properties["date"]); 86 | return new DateTimeOffset(datetime, TimeSpan.Zero); 87 | } 88 | } 89 | } -------------------------------------------------------------------------------- /source/SeqFlatFileImport.Core/FileFormats/OctopusRawTask.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text.RegularExpressions; 5 | using Newtonsoft.Json; 6 | using Newtonsoft.Json.Linq; 7 | 8 | namespace SeqFlatFileImport.Core.FileFormats 9 | { 10 | public class OctopusRawTask : IFileFormat 11 | { 12 | public string Name => "OctopusRawTask"; 13 | public IReadOnlyList AutodetectFileNameRegexes { get; } = new[] { @"servertasks-\d+.*txt" }; 14 | public int Ordinal => 0; 15 | 16 | public bool AutodetectFromContents(string[] firstFewLines) 17 | { 18 | if (firstFewLines.Length == 0) 19 | return false; 20 | 21 | return firstFewLines[0].StartsWith("[\"ServerTask"); 22 | } 23 | 24 | public IEnumerable Read(IEnumerable lines) 25 | { 26 | foreach (var line in lines) 27 | { 28 | 29 | var json = JsonConvert.DeserializeObject(line) as ICollection; 30 | 31 | if (json == null) 32 | throw new Exception("Unexpected line: " + line); 33 | 34 | yield return new RawEvent 35 | { 36 | Level = ParseLevel(json.ElementAt(1).ToString()), 37 | MessageTemplate = json.ElementAt(3).ToString(), 38 | Timestamp = ParseTime(json.ElementAt(2).ToString()), 39 | Properties = ParseProperties(json.ElementAt(0).ToString(), json.ElementAt(3).ToString()) 40 | }; 41 | } 42 | } 43 | 44 | private static DateTimeOffset ParseTime(string value) 45 | { 46 | return DateTimeOffset.Parse(value).UtcDateTime; 47 | } 48 | 49 | private static string ParseLevel(string value) 50 | { 51 | return value switch 52 | { 53 | "INF" => "Information", 54 | "VBS" => "Verbose", 55 | "WRN" => "Warning", 56 | "ERR" => "Error", 57 | "FAT" => "Fatal", 58 | _ => value 59 | }; 60 | } 61 | 62 | private static readonly Regex TaskId = new(@"^(?\w+-\d+)"); 63 | private readonly IDictionary correlationProperties = new Dictionary(); 64 | 65 | private Dictionary ParseProperties(string correlationId, string message) 66 | { 67 | var properties = new Dictionary(); 68 | 69 | var tokens = correlationId.Split('/'); 70 | 71 | var match = TaskId.Match(tokens[0]); 72 | properties.Add("Task ID", match.Groups["TaskId"].Value); 73 | 74 | for (var i = 1; i < tokens.Length; i++) 75 | { 76 | var token = tokens[i]; 77 | if (correlationProperties.TryGetValue(token, out var value)) 78 | { 79 | properties[$"Property {i}"] = value; 80 | } 81 | else 82 | { 83 | properties[$"Property {i}"] = message; 84 | correlationProperties[token] = message; 85 | } 86 | } 87 | 88 | return properties; 89 | } 90 | } 91 | } -------------------------------------------------------------------------------- /source/SeqFlatFileImport.Core/FileFormats/OctopusServer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text.RegularExpressions; 5 | 6 | namespace SeqFlatFileImport.Core.FileFormats 7 | { 8 | public class OctopusServer : IFileFormat 9 | { 10 | /// 11 | /// Octopus 3.x -> 3.15.x 12 | /// 13 | private static readonly Regex LineRegex_3_0 = 14 | new( 15 | @"^(?[0-9\-]{10} [0-9\:\.]{13}) +(?[0-9]+) +(?[A-Z]+) +(?.+)$"); 16 | 17 | /// 18 | /// We added PID in 3.15.x 19 | /// 20 | private static readonly Regex LineRegex_3_15 = 21 | new( 22 | @"^(?[0-9\-]{10} [0-9\:\.]{13}) +(?[0-9]+) +(?[0-9]+) +(?[A-Z]+) +(?.+)$"); 23 | 24 | private static readonly Regex ExceptionRegex = new(@"Exception \(0x[0-9]+\):"); 25 | private const RegexOptions DefaultOptions = RegexOptions.Singleline; 26 | 27 | private static readonly TemplateRegex[] TemplateRegexes = 28 | { 29 | new( 30 | new Regex( 31 | @"^Reader took (?[0-9]+)ms \((?[0-9]+)ms until the first record\) in transaction '(?.*)': (?.*)", 32 | DefaultOptions), 33 | "Reader took {Elapsed}ms ({FirstRecord}ms until the first record) in transaction '{Transaction}': {Query}" 34 | ), 35 | new( 36 | new Regex(@"^(?[A-Z]+)\s+(?http.*) (?.*)", DefaultOptions), 37 | "{Method} {Url} {CorrelationId}" 38 | ), 39 | new( 40 | new Regex(@"^Request took (?[0-9]+)ms: (?[A-Z]+)\s+(?http.*) (?.*)", 41 | DefaultOptions), 42 | "Request took {Elapsed}ms: {Method} {Url} {CorrelationId}" 43 | ), 44 | new( 45 | new Regex(@"^(?[A-Za-z]+) took (?[0-9]+)ms: (?.*)", DefaultOptions), 46 | "{Operation} took {Elapsed}ms: {Query}" 47 | ), 48 | new( 49 | new Regex(@"^Unhandled exception from web server: (?.*)", DefaultOptions), 50 | "Unhandled exception from web server: {Message}" 51 | ), 52 | new( 53 | new Regex(@"^listen://(?.+):(?[0-9]+)/ +[0-9]+ +(?.*)", DefaultOptions), 54 | "listen://{IP}:{Port}/ {Message}" 55 | ), 56 | new( 57 | new Regex(@"^poll://(?[a-z0-9]+)/ +[0-9]+ +(?.*)", DefaultOptions), 58 | "poll://{Id}/ {Message}" 59 | ), 60 | new( 61 | new Regex(@"^https://(?[^:]+):(?[0-9]+)/ +[0-9]+ +Retry attempt (?[0-9]+)", 62 | DefaultOptions), 63 | "https://{Host}:{Port}/ Retry attempt {n}" 64 | ), 65 | new( 66 | new Regex(@"^https://(?[^:]+):(?[0-9]+)/ +[0-9]+ +Opening a new connection$", 67 | DefaultOptions), 68 | "https://{Host}:{Port}/ Opening a new connection" 69 | ), 70 | new( 71 | new Regex(@"^https://(?[^:]+):(?[0-9]+)/ +[0-9]+ + Performing TLS handshake$", 72 | DefaultOptions), 73 | "https://{Host}:{Port}/ Performing TLS handshake" 74 | ), 75 | new( 76 | new Regex( 77 | @"^https://(?[^:]+):(?[0-9]+)/ +[0-9]+ +Secure connection established. Server at (?[^ ]+) identified by thumbprint: (?[A-Z0-9]+), using protocol (?[A-Za-z0-9]+)", 78 | DefaultOptions), 79 | "https://{Host}:{Port}/ Secure connection established. Server at {Endpoint} identified by thumbprint: {Thumbprint}, using protocol {Protocol}" 80 | ), 81 | new( 82 | new Regex( 83 | @"^https://(?[^:]+):(?[0-9]+)/ +[0-9]+ +No connection could be made because the target machine actively refused it (?.+)$", 84 | DefaultOptions), 85 | "https://{Host}:{Port}/ No connection could be made because the target machine actively refused it {Endpoint}" 86 | ), 87 | new( 88 | new Regex(@"^https://(?[^:]+):(?[0-9]+)/ +[0-9]+ +(?.*)", DefaultOptions), 89 | "https://{Host}:{Port}/ {Message}" 90 | ), 91 | new( 92 | new Regex( 93 | @"""(?[A-Z]+)"" ""(?[A-Z]+)"" to ""(?.*)"" completed with (?[0-9]+) in .* \((?[0-9]+.[0-9]+)ms\)", 94 | DefaultOptions), 95 | "{Protocol} {Method} to {Url} completed with {Code} in {Elapsed}" 96 | ), 97 | new( 98 | new Regex( 99 | @"^Execute reader took (?[0-9]+)ms in transaction '(?.*)': (?.*)", 100 | DefaultOptions), 101 | "Execute reader took {Elapsed} in transaction {Transaction}: {Query}" 102 | ) 103 | }; 104 | 105 | public string Name => "OctopusServer"; 106 | public IReadOnlyList AutodetectFileNameRegexes { get; } = new[] {@"OctopusServer.*\.txt"}; 107 | public int Ordinal => 0; 108 | 109 | public bool AutodetectFromContents(string[] firstFewLines) 110 | { 111 | return false; 112 | } 113 | 114 | public IEnumerable Read(IEnumerable lines) 115 | { 116 | var lineNumber = 0; 117 | Match currentLogEntryMatch = null; 118 | var currentLogEntryStartingLineNumber = lineNumber; 119 | var buffer = new List(); 120 | 121 | var lineRegex = new[] {LineRegex_3_0, LineRegex_3_15}; 122 | 123 | foreach (var line in lines) 124 | { 125 | lineNumber++; 126 | var newLogEntryMatch = 127 | lineRegex.Select(regex => regex.Match(line)).FirstOrDefault(match => match.Success); 128 | if (newLogEntryMatch?.Success == true) 129 | { 130 | // Flush the existing entry (maybe we don't have one yet?) 131 | if (currentLogEntryMatch != null) 132 | { 133 | yield return ProcessLogMessage(currentLogEntryStartingLineNumber, currentLogEntryMatch, buffer); 134 | buffer.Clear(); 135 | } 136 | 137 | // Start accumulating the new log entry 138 | currentLogEntryMatch = newLogEntryMatch; 139 | currentLogEntryStartingLineNumber = lineNumber; 140 | } 141 | else 142 | { 143 | buffer.Add(line); 144 | } 145 | } 146 | 147 | // Flush the final entry 148 | if (currentLogEntryMatch != null) 149 | { 150 | yield return ProcessLogMessage(currentLogEntryStartingLineNumber, currentLogEntryMatch, buffer); 151 | buffer.Clear(); 152 | } 153 | } 154 | 155 | private static RawEvent ProcessLogMessage(int lineNumber, Match currentLogEntryMatch, 156 | IReadOnlyCollection lines) 157 | { 158 | var properties = new Dictionary 159 | { 160 | {"LineNumber", lineNumber}, 161 | {"PID", currentLogEntryMatch.Groups["PID"].Value}, 162 | {"Thread", currentLogEntryMatch.Groups["Thread"].Value} 163 | }; 164 | 165 | var messageLines = new[] {currentLogEntryMatch.Groups["Message"].Value} 166 | .Concat(lines.TakeWhile(line => !ExceptionRegex.Match(line).Success)).ToArray(); 167 | var message = string.Join(Environment.NewLine, messageLines); 168 | var messageTemplate = MagicUpTheMessageTemplate(message, properties); 169 | 170 | var rawEvent = new RawEvent 171 | { 172 | Timestamp = DateTimeOffset.Parse(currentLogEntryMatch.Groups["Timestamp"].Value), 173 | Level = ConvertLevel(currentLogEntryMatch.Groups["Level"].Value), 174 | Properties = properties, 175 | MessageTemplate = messageTemplate 176 | }; 177 | 178 | var exceptionLines = lines.Except(messageLines).ToArray(); 179 | if (exceptionLines.Any()) 180 | { 181 | rawEvent.Exception = string.Join(Environment.NewLine, exceptionLines); 182 | } 183 | 184 | return rawEvent; 185 | } 186 | 187 | private static string MagicUpTheMessageTemplate(string message, Dictionary properties) 188 | { 189 | foreach (var tr in TemplateRegexes) 190 | { 191 | var match = tr.Regex.Match(message); 192 | if (match.Success) 193 | { 194 | foreach (var name in tr.Regex.GetGroupNames().Where(n => n != "0")) 195 | { 196 | var strVal = match.Groups[name].Value; 197 | if (int.TryParse(strVal, out var intVal)) 198 | properties[name] = intVal; 199 | else if (float.TryParse(strVal, out var floatVal)) 200 | properties[name] = floatVal; 201 | else 202 | properties[name] = strVal; 203 | } 204 | 205 | return tr.Template; 206 | } 207 | } 208 | 209 | return message; 210 | } 211 | 212 | private static string ConvertLevel(string str) 213 | { 214 | return str switch 215 | { 216 | "FATAL" => "Fatal", 217 | "ERROR" => "Error", 218 | "WARN" => "Warning", 219 | "DEBUG" => "Debug", 220 | "TRACE" => "Verbose", 221 | _ => "Information" 222 | }; 223 | } 224 | 225 | private class TemplateRegex 226 | { 227 | public Regex Regex { get; } 228 | public string Template { get; } 229 | 230 | public TemplateRegex(Regex regex, string template) 231 | { 232 | Regex = regex; 233 | Template = template; 234 | } 235 | } 236 | } 237 | } -------------------------------------------------------------------------------- /source/SeqFlatFileImport.Core/FileFormats/OctopusTask.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text.RegularExpressions; 5 | 6 | namespace SeqFlatFileImport.Core.FileFormats 7 | { 8 | public class OctopusTask : IFileFormat 9 | { 10 | private const string HeaderStart = " |"; 11 | private static readonly Regex MessageRegex = new(@"^(?