├── DslParser
    ├── packages.config
    ├── App.config
    ├── DataRepresentation
    │   ├── DslLogicalOperator.cs
    │   ├── DateRange.cs
    │   ├── DslObject.cs
    │   ├── DslOperator.cs
    │   ├── DslQueryModel.cs
    │   └── MatchCondition.cs
    ├── Parsing
    │   ├── Tokenizers
    │   │   ├── ITokenizer.cs
    │   │   ├── SlowAndSimple
    │   │   │   ├── TokenMatch.cs
    │   │   │   ├── TokenDefinition.cs
    │   │   │   └── SimpleRegexTokenizer.cs
    │   │   └── MoreEfficient
    │   │   │   ├── TokenMatch.cs
    │   │   │   ├── TokenDefinition.cs
    │   │   │   └── MoreEfficientRegexTokenizer.cs
    │   ├── Tokens
    │   │   ├── DslToken.cs
    │   │   └── TokenType.cs
    │   └── Parser.cs
    ├── MsdnSqlLikeGrammar.txt
    ├── Exceptions
    │   └── DslParserException.cs
    ├── Entities
    │   └── ErrorCountRecord.cs
    ├── ParserGrammarRules.txt
    ├── Properties
    │   └── AssemblyInfo.cs
    ├── SqlGeneration
    │   ├── SqlExecutor.cs
    │   ├── SqlGenerator.cs
    │   └── AdoQueryPayload.cs
    ├── DslParser.csproj
    └── Program.cs
├── README.md
├── DslParser.sln
├── License.txt
├── .gitattributes
└── .gitignore


/DslParser/packages.config:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="utf-8"?>
2 | <packages>
3 |   <package id="Newtonsoft.Json" version="8.0.2" targetFramework="net45" />
4 | </packages>


--------------------------------------------------------------------------------
/DslParser/App.config:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="utf-8" ?>
2 | <configuration>
3 |     <startup> 
4 |         <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
5 |     </startup>
6 | </configuration>


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # DslParser
2 | Example of a DSL parse that takes a DSL and generates a SQL query and parameters.
3 | 
4 | Based on my blog series http://jack-vanlightly.com/blog/2016/2/3/how-to-create-a-query-language-dsl and this post http://jack-vanlightly.com/blog/2016/2/24/a-more-efficient-regex-tokenizer
5 | 


--------------------------------------------------------------------------------
/DslParser/DataRepresentation/DslLogicalOperator.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using System.Text;
 5 | using System.Threading.Tasks;
 6 | 
 7 | namespace DslParser.DataRepresentation
 8 | {
 9 |     public enum DslLogicalOperator
10 |     {
11 |         NotDefined,
12 |         Or,
13 |         And
14 |     }
15 | }
16 | 


--------------------------------------------------------------------------------
/DslParser/DataRepresentation/DateRange.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using System.Text;
 5 | using System.Threading.Tasks;
 6 | 
 7 | namespace DslParser.DataRepresentation
 8 | {
 9 |     public class DateRange
10 |     {
11 |         public DateTime From { get; set; }
12 |         public DateTime To { get; set; }
13 |     }
14 | }
15 | 


--------------------------------------------------------------------------------
/DslParser/Parsing/Tokenizers/ITokenizer.cs:
--------------------------------------------------------------------------------
 1 | using DslParser.Parsing.Tokens;
 2 | using System;
 3 | using System.Collections.Generic;
 4 | using System.Linq;
 5 | using System.Text;
 6 | using System.Threading.Tasks;
 7 | 
 8 | namespace DslParser.Parsing.Tokenizers
 9 | {
10 |     public interface ITokenizer
11 |     {
12 |         IEnumerable<DslToken> Tokenize(string queryDsl);
13 |     }
14 | }
15 | 


--------------------------------------------------------------------------------
/DslParser/DataRepresentation/DslObject.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using System.Text;
 5 | using System.Threading.Tasks;
 6 | 
 7 | namespace DslParser.DataRepresentation
 8 | {
 9 |     public enum DslObject
10 |     {
11 |         Application,
12 |         ExceptionType,
13 |         Message,
14 |         StackFrame,
15 |         Fingerprint
16 |     }
17 | }
18 | 


--------------------------------------------------------------------------------
/DslParser/MsdnSqlLikeGrammar.txt:
--------------------------------------------------------------------------------
 1 | MATCH <match-condition>
 2 | [AND|OR <match-condition>]
 3 | BETWEEN 'yyyy-MM-dd HH:mm:ss' AND 'yyyy-MM-dd HH:mm:ss'
 4 | LIMIT integer
 5 | 
 6 | <match-condition>
 7 | <object> {{=|!=|[NOT] LIKE} 'string_value'}|{[NOT] IN('string_value'[,'string_value']*)}
 8 | 
 9 | <object>
10 | {app|application}
11 | |{ex|exception}
12 | |{msg|message}
13 | |{sf|stackframe}
14 | |{fp|fingerprint}
15 | 


--------------------------------------------------------------------------------
/DslParser/Parsing/Tokenizers/SlowAndSimple/TokenMatch.cs:
--------------------------------------------------------------------------------
 1 | using DslParser.Parsing.Tokens;
 2 | 
 3 | namespace DslParser.Parsing.Tokenizers.SlowAndSimple
 4 | {
 5 |     public class TokenMatch
 6 |     {
 7 |         public bool IsMatch { get; set; }
 8 |         public TokenType TokenType { get; set; }
 9 |         public string Value { get; set; }
10 |         public string RemainingText { get; set; }
11 |     }
12 | }
13 | 


--------------------------------------------------------------------------------
/DslParser/DataRepresentation/DslOperator.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using System.Text;
 5 | using System.Threading.Tasks;
 6 | 
 7 | namespace DslParser.DataRepresentation
 8 | {
 9 |     public enum DslOperator
10 |     {
11 |         NotDefined,
12 |         Equals,
13 |         NotEquals,
14 |         Like,
15 |         NotLike,
16 |         In,
17 |         NotIn
18 |     }
19 | }
20 | 


--------------------------------------------------------------------------------
/DslParser/Parsing/Tokenizers/MoreEfficient/TokenMatch.cs:
--------------------------------------------------------------------------------
 1 | using DslParser.Parsing.Tokens;
 2 | 
 3 | namespace DslParser.Parsing.Tokenizers.MoreEfficient
 4 | {
 5 |     public class TokenMatch
 6 |     {
 7 |         public TokenType TokenType { get; set; }
 8 |         public string Value { get; set; }
 9 |         public int StartIndex { get; set; }
10 |         public int EndIndex { get; set; }
11 |         public int Precedence { get; set; }
12 |     }
13 | }
14 | 


--------------------------------------------------------------------------------
/DslParser/Exceptions/DslParserException.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using System.Text;
 5 | using System.Threading.Tasks;
 6 | 
 7 | namespace DslParser.Exceptions
 8 | {
 9 |     [Serializable]
10 |     public class DslParserException : Exception
11 |     {
12 |         public DslParserException(string message)
13 |             : base(message)
14 |         {
15 |             
16 |         }
17 |     }
18 | }
19 | 


--------------------------------------------------------------------------------
/DslParser/DataRepresentation/DslQueryModel.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using System.Text;
 5 | using System.Threading.Tasks;
 6 | 
 7 | namespace DslParser.DataRepresentation
 8 | {
 9 |     public class DslQueryModel
10 |     {
11 |         public DslQueryModel()
12 |         {
13 |             MatchConditions = new List<MatchCondition>();
14 |         }
15 | 
16 |         public DateRange DateRange { get; set; }
17 |         public int? Limit { get; set; }
18 |         public IList<MatchCondition> MatchConditions { get; set; }
19 |     }
20 | }
21 | 


--------------------------------------------------------------------------------
/DslParser/DataRepresentation/MatchCondition.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using System.Text;
 5 | using System.Threading.Tasks;
 6 | using DslParser.Parsing.Tokens;
 7 | 
 8 | namespace DslParser.DataRepresentation
 9 | {
10 |     public class MatchCondition
11 |     {
12 |         public DslObject Object { get; set; }
13 |         public DslOperator Operator { get; set; }
14 |         public string Value { get; set; }
15 |         public List<string> Values { get; set; }
16 |         
17 |         public DslLogicalOperator LogOpToNextCondition { get; set; }
18 |     }
19 | }
20 | 


--------------------------------------------------------------------------------
/DslParser/Entities/ErrorCountRecord.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using System.Text;
 5 | using System.Threading.Tasks;
 6 | 
 7 | namespace DslParser.Entities
 8 | {
 9 |     public class ErrorCountRecord
10 |     {
11 |         public string Fingerprint { get; set; }
12 |         public string ApplicationId { get; set; }
13 |         public string OriginExceptionType { get; set; }
14 |         public string OriginStackFrame { get; set; }
15 |         public string LowestAppStackFrame { get; set; }
16 |         public string HighestAppStackFrame { get; set; }
17 |         public int Count { get; set; }
18 |     }
19 | }
20 | 


--------------------------------------------------------------------------------
/DslParser/Parsing/Tokens/DslToken.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using System.Text;
 5 | using System.Threading.Tasks;
 6 | 
 7 | namespace DslParser.Parsing.Tokens
 8 | {
 9 |     public class DslToken
10 |     {
11 |         public DslToken(TokenType tokenType)
12 |         {
13 |             TokenType = tokenType;
14 |             Value = string.Empty;
15 |         }
16 | 
17 |         public DslToken(TokenType tokenType, string value)
18 |         {
19 |             TokenType = tokenType;
20 |             Value = value;
21 |         }
22 | 
23 |         public TokenType TokenType { get; set; }
24 |         public string Value { get; set; }
25 | 
26 |         public DslToken Clone()
27 |         {
28 |             return new DslToken(TokenType, Value);
29 |         }
30 |     }
31 | }
32 | 


--------------------------------------------------------------------------------
/DslParser/Parsing/Tokens/TokenType.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using System.Text;
 5 | using System.Threading.Tasks;
 6 | 
 7 | namespace DslParser.Parsing.Tokens
 8 | {
 9 |     public enum TokenType
10 |     {
11 |         NotDefined,
12 |         And,
13 |         Application,
14 |         Between,
15 |         CloseParenthesis,
16 |         Comma,
17 |         DateTimeValue,
18 |         Equals,
19 |         ExceptionType,
20 |         Fingerprint,
21 |         In,
22 |         Invalid,
23 |         Like,
24 |         Limit,
25 |         Match,
26 |         Message,
27 |         NotEquals,
28 |         NotIn,
29 |         NotLike,
30 |         Number,
31 |         Or,
32 |         OpenParenthesis,
33 |         StackFrame,
34 |         StringValue,
35 |         SequenceTerminator
36 |     }
37 | }
38 | 


--------------------------------------------------------------------------------
/DslParser/ParserGrammarRules.txt:
--------------------------------------------------------------------------------
 1 | S -> MATCH
 2 | MATCH -> match MATCH_CONDITION
 3 | 
 4 | MATCH_CONDITION -> object operator string_literal MATCH_CONDITION_NEXT
 5 | MATCH_CONDITION -> object in open_bracket STRING_LITERAL_LIST close_bracket  MATCH_CONDITION_NEXT
 6 | MATCH_CONDITION -> object not_in open_bracket STRING_LITERAL_LIST close_bracket MATCH_CONDITION_NEXT
 7 | MATCH_CONDITION_NEXT -> and MATCH_CONDITION
 8 | MATCH_CONDITION_NEXT -> or MATCH_CONDITION
 9 | MATCH_CONDITION_NEXT -> DATE_CONDITION
10 | 
11 | STRING_LITERAL_LIST -> string_literal STRING_LITERAL_LIST_NEXT
12 | STRING_LITERAL_LIST_NEXT -> comma string_literal STRING_LITERAL_LIST_NEXT
13 | STRING_LITERAL_LIST_NEXT -> empty_string
14 | 
15 | DATE_CONDITION -> between datetime_literal and datetime_literal  DATE_CONDITION_NEXT
16 | DATE_CONDITION_NEXT -> LIMIT
17 | DATE_CONDITION_NEXT -> eos
18 | 
19 | LIMIT -> limit number eos


--------------------------------------------------------------------------------
/DslParser.sln:
--------------------------------------------------------------------------------
 1 | 
 2 | Microsoft Visual Studio Solution File, Format Version 12.00
 3 | # Visual Studio 2012
 4 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DslParser", "DslParser\DslParser.csproj", "{7184C7D3-8C9C-4DBD-9792-0B93F18CB095}"
 5 | EndProject
 6 | Global
 7 | 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 8 | 		Debug|Any CPU = Debug|Any CPU
 9 | 		Release|Any CPU = Release|Any CPU
10 | 	EndGlobalSection
11 | 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
12 | 		{7184C7D3-8C9C-4DBD-9792-0B93F18CB095}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
13 | 		{7184C7D3-8C9C-4DBD-9792-0B93F18CB095}.Debug|Any CPU.Build.0 = Debug|Any CPU
14 | 		{7184C7D3-8C9C-4DBD-9792-0B93F18CB095}.Release|Any CPU.ActiveCfg = Release|Any CPU
15 | 		{7184C7D3-8C9C-4DBD-9792-0B93F18CB095}.Release|Any CPU.Build.0 = Release|Any CPU
16 | 	EndGlobalSection
17 | 	GlobalSection(SolutionProperties) = preSolution
18 | 		HideSolutionNode = FALSE
19 | 	EndGlobalSection
20 | 	GlobalSection(Performance) = preSolution
21 | 		HasPerformanceSessions = true
22 | 	EndGlobalSection
23 | EndGlobal
24 | 


--------------------------------------------------------------------------------
/License.txt:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2017 Jack Vanlightly
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/DslParser/Parsing/Tokenizers/MoreEfficient/TokenDefinition.cs:
--------------------------------------------------------------------------------
 1 | using System.Collections.Generic;
 2 | using System.Text.RegularExpressions;
 3 | using DslParser.Parsing.Tokens;
 4 | 
 5 | namespace DslParser.Parsing.Tokenizers.MoreEfficient
 6 | {
 7 |     public class TokenDefinition
 8 |     {
 9 |         private Regex _regex;
10 |         private readonly TokenType _returnsToken;
11 |         private readonly int _precedence;
12 | 
13 |         public TokenDefinition(TokenType returnsToken, string regexPattern, int precedence)
14 |         {
15 |             _regex = new Regex(regexPattern, RegexOptions.IgnoreCase|RegexOptions.Compiled);
16 |             _returnsToken = returnsToken;
17 |             _precedence = precedence;
18 |         }
19 | 
20 |         public IEnumerable<TokenMatch> FindMatches(string inputString)
21 |         {
22 |             var matches = _regex.Matches(inputString);
23 |             for(int i=0; i<matches.Count; i++)
24 |             {
25 |                 yield return new TokenMatch()
26 |                 {
27 |                     StartIndex = matches[i].Index,
28 |                     EndIndex = matches[i].Index + matches[i].Length,
29 |                     TokenType = _returnsToken,
30 |                     Value = matches[i].Value,
31 |                     Precedence = _precedence
32 |                 };
33 |             }
34 |         }
35 |     }
36 | }
37 | 


--------------------------------------------------------------------------------
/DslParser/Parsing/Tokenizers/SlowAndSimple/TokenDefinition.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using System.Text;
 5 | using System.Text.RegularExpressions;
 6 | using System.Threading.Tasks;
 7 | using DslParser.Parsing.Tokenizers.SlowAndSimple;
 8 | 
 9 | namespace DslParser.Parsing.Tokens.Tokenizers.SlowAndSimple
10 | {
11 |     public class TokenDefinition
12 |     {
13 |         private Regex _regex;
14 |         private readonly TokenType _returnsToken;
15 | 
16 |         public TokenDefinition(TokenType returnsToken, string regexPattern)
17 |         {
18 |             _regex = new Regex(regexPattern, RegexOptions.IgnoreCase);
19 |             _returnsToken = returnsToken;
20 |         }
21 | 
22 |         public TokenMatch Match(string inputString)
23 |         {
24 |             var match = _regex.Match(inputString);
25 |             if (match.Success)
26 |             {
27 |                 string remainingText = string.Empty;
28 |                 if (match.Length != inputString.Length)
29 |                     remainingText = inputString.Substring(match.Length);
30 | 
31 |                 return new TokenMatch()
32 |                 {
33 |                     IsMatch = true,
34 |                     RemainingText = remainingText,
35 |                     TokenType = _returnsToken,
36 |                     Value = match.Value
37 |                 };
38 |             }
39 |             else
40 |             {
41 |                 return new TokenMatch() { IsMatch = false};
42 |             }
43 | 
44 |         }
45 |     }
46 | }
47 | 


--------------------------------------------------------------------------------
/DslParser/Properties/AssemblyInfo.cs:
--------------------------------------------------------------------------------
 1 | using System.Reflection;
 2 | using System.Runtime.CompilerServices;
 3 | using System.Runtime.InteropServices;
 4 | 
 5 | // General Information about an assembly is controlled through the following 
 6 | // set of attributes. Change these attribute values to modify the information
 7 | // associated with an assembly.
 8 | [assembly: AssemblyTitle("DslParser")]
 9 | [assembly: AssemblyDescription("")]
10 | [assembly: AssemblyConfiguration("")]
11 | [assembly: AssemblyCompany("Hewlett-Packard Company")]
12 | [assembly: AssemblyProduct("DslParser")]
13 | [assembly: AssemblyCopyright("Copyright © Hewlett-Packard Company 2016")]
14 | [assembly: AssemblyTrademark("")]
15 | [assembly: AssemblyCulture("")]
16 | 
17 | // Setting ComVisible to false makes the types in this assembly not visible 
18 | // to COM components.  If you need to access a type in this assembly from 
19 | // COM, set the ComVisible attribute to true on that type.
20 | [assembly: ComVisible(false)]
21 | 
22 | // The following GUID is for the ID of the typelib if this project is exposed to COM
23 | [assembly: Guid("ce12d2c1-d812-468c-954e-34fb4e98f080")]
24 | 
25 | // Version information for an assembly consists of the following four values:
26 | //
27 | //      Major Version
28 | //      Minor Version 
29 | //      Build Number
30 | //      Revision
31 | //
32 | // You can specify all the values or you can default the Build and Revision Numbers 
33 | // by using the '*' as shown below:
34 | // [assembly: AssemblyVersion("1.0.*")]
35 | [assembly: AssemblyVersion("1.0.0.0")]
36 | [assembly: AssemblyFileVersion("1.0.0.0")]
37 | 


--------------------------------------------------------------------------------
/DslParser/SqlGeneration/SqlExecutor.cs:
--------------------------------------------------------------------------------
 1 | using DslParser.Entities;
 2 | using System;
 3 | using System.Collections.Generic;
 4 | using System.Data.SqlClient;
 5 | using System.Linq;
 6 | using System.Text;
 7 | using System.Threading.Tasks;
 8 | 
 9 | namespace DslParser.SqlGeneration
10 | {
11 |     public class SqlExecutor
12 |     {
13 |         public IList<ErrorCountRecord> GetTopRankingErrors(AdoQueryPayload adoQueryPayload)
14 |         {
15 |             var results = new List<ErrorCountRecord>();
16 | 
17 |             using (var connection = new SqlConnection("Database=ErrorsDb;Server=(local);Trusted_Connection=true;"))
18 |             {
19 |                 using (var command = new SqlCommand(adoQueryPayload.GetSqlText(), connection))
20 |                 {
21 |                     foreach (var parameter in adoQueryPayload.Parameters)
22 |                         command.Parameters.Add(parameter);
23 | 
24 |                     using (var reader = command.ExecuteReader())
25 |                     {
26 |                         var record = new ErrorCountRecord();
27 |                         record.ApplicationId = reader["ApplicationId"].ToString();
28 |                         record.Count = (int)reader["Count"];
29 |                         record.Fingerprint = reader["FingerprintText"].ToString();
30 |                         record.HighestAppStackFrame = reader["HighestAppStackFrame"].ToString();
31 |                         record.LowestAppStackFrame = reader["LowestAppStackFrame"].ToString();
32 |                         record.OriginExceptionType = reader["OriginExceptionType"].ToString();
33 |                         record.OriginStackFrame = reader["OriginStackFrame"].ToString();
34 | 
35 |                         results.Add(record);
36 |                     }
37 |                 }
38 |             }
39 | 
40 |             return results;
41 |         }
42 |     }
43 | }
44 | 


--------------------------------------------------------------------------------
/.gitattributes:
--------------------------------------------------------------------------------
 1 | ###############################################################################
 2 | # Set default behavior to automatically normalize line endings.
 3 | ###############################################################################
 4 | * text=auto
 5 | 
 6 | ###############################################################################
 7 | # Set default behavior for command prompt diff.
 8 | #
 9 | # This is need for earlier builds of msysgit that does not have it on by
10 | # default for csharp files.
11 | # Note: This is only used by command line
12 | ###############################################################################
13 | #*.cs     diff=csharp
14 | 
15 | ###############################################################################
16 | # Set the merge driver for project and solution files
17 | #
18 | # Merging from the command prompt will add diff markers to the files if there
19 | # are conflicts (Merging from VS is not affected by the settings below, in VS
20 | # the diff markers are never inserted). Diff markers may cause the following 
21 | # file extensions to fail to load in VS. An alternative would be to treat
22 | # these files as binary and thus will always conflict and require user
23 | # intervention with every merge. To do so, just uncomment the entries below
24 | ###############################################################################
25 | #*.sln       merge=binary
26 | #*.csproj    merge=binary
27 | #*.vbproj    merge=binary
28 | #*.vcxproj   merge=binary
29 | #*.vcproj    merge=binary
30 | #*.dbproj    merge=binary
31 | #*.fsproj    merge=binary
32 | #*.lsproj    merge=binary
33 | #*.wixproj   merge=binary
34 | #*.modelproj merge=binary
35 | #*.sqlproj   merge=binary
36 | #*.wwaproj   merge=binary
37 | 
38 | ###############################################################################
39 | # behavior for image files
40 | #
41 | # image files are treated as binary by default.
42 | ###############################################################################
43 | #*.jpg   binary
44 | #*.png   binary
45 | #*.gif   binary
46 | 
47 | ###############################################################################
48 | # diff behavior for common document formats
49 | # 
50 | # Convert binary document formats to text before diffing them. This feature
51 | # is only available from the command line. Turn it on by uncommenting the 
52 | # entries below.
53 | ###############################################################################
54 | #*.doc   diff=astextplain
55 | #*.DOC   diff=astextplain
56 | #*.docx  diff=astextplain
57 | #*.DOCX  diff=astextplain
58 | #*.dot   diff=astextplain
59 | #*.DOT   diff=astextplain
60 | #*.pdf   diff=astextplain
61 | #*.PDF   diff=astextplain
62 | #*.rtf   diff=astextplain
63 | #*.RTF   diff=astextplain
64 | 


--------------------------------------------------------------------------------
/DslParser/Parsing/Tokenizers/MoreEfficient/MoreEfficientRegexTokenizer.cs:
--------------------------------------------------------------------------------
 1 | using System;
 2 | using System.Collections.Generic;
 3 | using System.Linq;
 4 | using System.Text;
 5 | using System.Threading.Tasks;
 6 | using DslParser.Parsing.Tokens;
 7 | 
 8 | namespace DslParser.Parsing.Tokenizers.MoreEfficient
 9 | {
10 |     public class PrecedenceBasedRegexTokenizer : ITokenizer
11 |     {
12 |         private List<TokenDefinition> _tokenDefinitions;
13 | 
14 |         public PrecedenceBasedRegexTokenizer()
15 |         {
16 |             _tokenDefinitions = new List<TokenDefinition>();
17 | 
18 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.And, "and", 1));
19 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Application, "app|application", 1));
20 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Between, "between", 1));
21 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.CloseParenthesis, "\\)", 1));
22 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Comma, ",", 1));
23 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Equals, "=", 1));
24 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.ExceptionType, "ex|exception", 1));
25 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Fingerprint, "fingerprint", 1));
26 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.NotIn, "not in", 1));
27 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.In, "in", 1));
28 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Like, "like", 1));
29 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Limit, "limit", 1));
30 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Match, "match", 1));
31 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Message, "msg|message", 1));
32 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.NotEquals, "!=", 1));
33 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.NotLike, "not like", 1));
34 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.OpenParenthesis, "\\(", 1));
35 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Or, "or", 1));
36 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.StackFrame, "sf|stackframe", 1));
37 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.DateTimeValue, "\\d\\d\\d\\d-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d", 2));
38 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.StringValue, "'([^']*)'", 1));
39 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Number, "\\d+", 2));
40 |         }
41 | 
42 |         public IEnumerable<DslToken> Tokenize(string lqlText)
43 |         {
44 |             var tokenMatches = FindTokenMatches(lqlText);
45 | 
46 |             var groupedByIndex = tokenMatches.GroupBy(x => x.StartIndex)
47 |                 .OrderBy(x => x.Key)
48 |                 .ToList();
49 | 
50 |             TokenMatch lastMatch = null;
51 |             for (int i = 0; i < groupedByIndex.Count; i++)
52 |             {
53 |                 var bestMatch = groupedByIndex[i].OrderBy(x => x.Precedence).First();
54 |                 if (lastMatch != null && bestMatch.StartIndex < lastMatch.EndIndex)
55 |                     continue;
56 | 
57 |                 yield return new DslToken(bestMatch.TokenType, bestMatch.Value);
58 | 
59 |                 lastMatch = bestMatch;
60 |             }
61 | 
62 |             yield return new DslToken(TokenType.SequenceTerminator);
63 |         }
64 | 
65 |         private List<TokenMatch> FindTokenMatches(string lqlText)
66 |         {
67 |             var tokenMatches = new List<TokenMatch>();
68 | 
69 |             foreach (var tokenDefinition in _tokenDefinitions)
70 |                 tokenMatches.AddRange(tokenDefinition.FindMatches(lqlText).ToList());
71 | 
72 |             return tokenMatches;
73 |         }
74 |     }
75 | }
76 | 


--------------------------------------------------------------------------------
/DslParser/SqlGeneration/SqlGenerator.cs:
--------------------------------------------------------------------------------
  1 | using DslParser.DataRepresentation;
  2 | using System;
  3 | using System.Collections.Generic;
  4 | using System.Linq;
  5 | using System.Text;
  6 | using System.Threading.Tasks;
  7 | 
  8 | namespace DslParser.SqlGeneration
  9 | {
 10 |     public class SqlGenerator
 11 |     {
 12 |         public AdoQueryPayload GenerateQueryPayload(DslQueryModel dslQueryModel)
 13 |         {
 14 |             var adoQueryPayload = new AdoQueryPayload();
 15 | 
 16 |             if(dslQueryModel.Limit.HasValue)
 17 |                 adoQueryPayload.AppendLine("SELECT TOP " + dslQueryModel.Limit.Value);
 18 |             else
 19 |                 adoQueryPayload.AppendLine("SELECT");
 20 | 
 21 |             adoQueryPayload.AppendLine(@"        ED.FingerprintText
 22 |         ,ED.ApplicationId
 23 |         ,ED.OriginExceptionType
 24 |         ,ED.OriginStackFrame
 25 |         ,ED.LowestAppStackFrame
 26 |         ,ED.HighestAppStackFrame
 27 |         ,SUM(T.Frequency) AS TotalErrors
 28 | FROM Timeline T
 29 | JOIN ErrorDefinition AS ED ON T.Fingerprint = ED.Fingerprint");
 30 | 
 31 |             adoQueryPayload.Append("WHERE T.ErrorDateTime BETWEEN ");
 32 |             adoQueryPayload.AddFromDateParameter(dslQueryModel.DateRange.From);
 33 |             adoQueryPayload.Append(" AND ");
 34 |             adoQueryPayload.AddToDateParameter(dslQueryModel.DateRange.To);
 35 |             adoQueryPayload.AddNewLine();
 36 | 
 37 |             for(int i=0; i<dslQueryModel.MatchConditions.Count; i++)
 38 |             {
 39 |                 if (i == 0)
 40 |                     AddLogicalOperator(adoQueryPayload, DslLogicalOperator.And); 
 41 |                 else
 42 |                     AddLogicalOperator(adoQueryPayload, dslQueryModel.MatchConditions[i-1].LogOpToNextCondition);
 43 | 
 44 |                 AddClause(dslQueryModel.MatchConditions[i], adoQueryPayload);
 45 |             }
 46 | 
 47 |             adoQueryPayload.AppendLine(@"GROUP BY ED.FingerprintText
 48 |         ,ED.ApplicationId
 49 |         ,ED.OriginExceptionType
 50 |         ,ED.OriginStackFrame
 51 |         ,ED.LowestAppStackFrame
 52 |         ,ED.HighestAppStackFrame
 53 | ORDER BY TotalErrors DESC");
 54 | 
 55 |             return adoQueryPayload;
 56 |         }
 57 | 
 58 |         private void AddLogicalOperator(AdoQueryPayload queryPayload, DslLogicalOperator logicalOperator)
 59 |         {
 60 |             if (logicalOperator == DslLogicalOperator.And)
 61 |                 queryPayload.Append("AND ");
 62 |             else if (logicalOperator == DslLogicalOperator.Or)
 63 |                 queryPayload.Append("OR ");
 64 |         }
 65 | 
 66 |         private void AddClause(MatchCondition matchCondition, AdoQueryPayload queryPayload)
 67 |         {
 68 |             queryPayload.AppendColumnName(matchCondition.Object);
 69 | 
 70 |             switch (matchCondition.Operator)
 71 |             {
 72 |                 case DslOperator.Equals:
 73 |                     queryPayload.Append(" = ");
 74 |                     queryPayload.AddParameter(matchCondition);
 75 |                     break;
 76 |                 case DslOperator.NotEquals:
 77 |                     queryPayload.Append(" <> ");
 78 |                     queryPayload.AddParameter(matchCondition);
 79 |                     break;
 80 |                 case DslOperator.Like:
 81 |                     queryPayload.Append(" LIKE '%' + ");
 82 |                     queryPayload.AddParameter(matchCondition);
 83 |                     queryPayload.Append(" + '%'");
 84 |                     break;
 85 |                 case DslOperator.NotLike:
 86 |                     queryPayload.Append(" NOT LIKE '%' + ");
 87 |                     queryPayload.AddParameter(matchCondition);
 88 |                     queryPayload.Append(" + '%'");
 89 |                     break;
 90 |                 case DslOperator.In:
 91 |                     queryPayload.Append(" IN (");
 92 |                     queryPayload.AddParameter(matchCondition);
 93 |                     queryPayload.Append(")");
 94 |                     break;
 95 |                 case DslOperator.NotIn:
 96 |                     queryPayload.Append(" NOT IN (");
 97 |                     queryPayload.AddParameter(matchCondition);
 98 |                     queryPayload.Append(")");
 99 |                     break;
100 |                 default:
101 |                     throw new Exception("DSL Operator not supported for SQL query generation: " + matchCondition.Operator);
102 |             }
103 | 
104 |             queryPayload.AddNewLine();
105 |         }
106 | 
107 |         
108 |     }
109 | }
110 | 


--------------------------------------------------------------------------------
/DslParser/DslParser.csproj:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="utf-8"?>
 2 | <Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
 3 |   <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
 4 |   <PropertyGroup>
 5 |     <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
 6 |     <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
 7 |     <ProjectGuid>{7184C7D3-8C9C-4DBD-9792-0B93F18CB095}</ProjectGuid>
 8 |     <OutputType>Exe</OutputType>
 9 |     <AppDesignerFolder>Properties</AppDesignerFolder>
10 |     <RootNamespace>DslParser</RootNamespace>
11 |     <AssemblyName>DslParser</AssemblyName>
12 |     <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
13 |     <FileAlignment>512</FileAlignment>
14 |   </PropertyGroup>
15 |   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
16 |     <PlatformTarget>AnyCPU</PlatformTarget>
17 |     <DebugSymbols>true</DebugSymbols>
18 |     <DebugType>full</DebugType>
19 |     <Optimize>false</Optimize>
20 |     <OutputPath>bin\Debug\</OutputPath>
21 |     <DefineConstants>DEBUG;TRACE</DefineConstants>
22 |     <ErrorReport>prompt</ErrorReport>
23 |     <WarningLevel>4</WarningLevel>
24 |   </PropertyGroup>
25 |   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
26 |     <PlatformTarget>AnyCPU</PlatformTarget>
27 |     <DebugType>pdbonly</DebugType>
28 |     <Optimize>true</Optimize>
29 |     <OutputPath>bin\Release\</OutputPath>
30 |     <DefineConstants>TRACE</DefineConstants>
31 |     <ErrorReport>prompt</ErrorReport>
32 |     <WarningLevel>4</WarningLevel>
33 |   </PropertyGroup>
34 |   <ItemGroup>
35 |     <Reference Include="Newtonsoft.Json, Version=8.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
36 |       <HintPath>..\packages\Newtonsoft.Json.8.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
37 |       <Private>True</Private>
38 |     </Reference>
39 |     <Reference Include="System" />
40 |     <Reference Include="System.Core" />
41 |     <Reference Include="System.Xml.Linq" />
42 |     <Reference Include="System.Data.DataSetExtensions" />
43 |     <Reference Include="Microsoft.CSharp" />
44 |     <Reference Include="System.Data" />
45 |     <Reference Include="System.Xml" />
46 |   </ItemGroup>
47 |   <ItemGroup>
48 |     <Compile Include="DataRepresentation\DateRange.cs" />
49 |     <Compile Include="DataRepresentation\DslLogicalOperator.cs" />
50 |     <Compile Include="DataRepresentation\DslObject.cs" />
51 |     <Compile Include="DataRepresentation\DslOperator.cs" />
52 |     <Compile Include="DataRepresentation\DslQueryModel.cs" />
53 |     <Compile Include="DataRepresentation\MatchCondition.cs" />
54 |     <Compile Include="Entities\ErrorCountRecord.cs" />
55 |     <Compile Include="Exceptions\DslParserException.cs" />
56 |     <Compile Include="Parsing\Tokenizers\ITokenizer.cs" />
57 |     <Compile Include="Parsing\Tokenizers\MoreEfficient\MoreEfficientRegexTokenizer.cs" />
58 |     <Compile Include="Parsing\Tokenizers\MoreEfficient\TokenDefinition.cs" />
59 |     <Compile Include="Parsing\Tokenizers\MoreEfficient\TokenMatch.cs" />
60 |     <Compile Include="Parsing\Tokenizers\SlowAndSimple\SimpleRegexTokenizer.cs" />
61 |     <Compile Include="Parsing\Parser.cs" />
62 |     <Compile Include="Parsing\Tokens\DslToken.cs" />
63 |     <Compile Include="Program.cs" />
64 |     <Compile Include="Properties\AssemblyInfo.cs" />
65 |     <Compile Include="Parsing\Tokenizers\SlowAndSimple\TokenDefinition.cs" />
66 |     <Compile Include="Parsing\Tokenizers\SlowAndSimple\TokenMatch.cs" />
67 |     <Compile Include="Parsing\Tokens\TokenType.cs" />
68 |     <Compile Include="SqlGeneration\AdoQueryPayload.cs" />
69 |     <Compile Include="SqlGeneration\SqlExecutor.cs" />
70 |     <Compile Include="SqlGeneration\SqlGenerator.cs" />
71 |   </ItemGroup>
72 |   <ItemGroup>
73 |     <None Include="App.config" />
74 |     <None Include="packages.config" />
75 |   </ItemGroup>
76 |   <ItemGroup>
77 |     <Content Include="MsdnSqlLikeGrammar.txt" />
78 |     <Content Include="ParserGrammarRules.txt" />
79 |   </ItemGroup>
80 |   <ItemGroup />
81 |   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
82 |   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
83 |        Other similar extension points exist, see Microsoft.Common.targets.
84 |   <Target Name="BeforeBuild">
85 |   </Target>
86 |   <Target Name="AfterBuild">
87 |   </Target>
88 |   -->
89 | </Project>


--------------------------------------------------------------------------------
/DslParser/Parsing/Tokenizers/SlowAndSimple/SimpleRegexTokenizer.cs:
--------------------------------------------------------------------------------
  1 | using System;
  2 | using System.Collections.Generic;
  3 | using System.Linq;
  4 | using System.Text;
  5 | using System.Text.RegularExpressions;
  6 | using System.Threading.Tasks;
  7 | using DslParser.Exceptions;
  8 | using DslParser.Parsing.Tokens;
  9 | using DslParser.Parsing.Tokens.Tokenizers.SlowAndSimple;
 10 | 
 11 | namespace DslParser.Parsing.Tokenizers.SlowAndSimple
 12 | {
 13 |     public class SimpleRegexTokenizer : ITokenizer
 14 |     {
 15 |         private List<TokenDefinition> _tokenDefinitions;
 16 | 
 17 |         public SimpleRegexTokenizer()
 18 |         {
 19 |             _tokenDefinitions = new List<TokenDefinition>();
 20 | 
 21 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.And, "^and"));
 22 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Application, "^app|^application"));
 23 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Between, "^between"));
 24 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.CloseParenthesis, "^\\)"));
 25 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Comma, "^,"));
 26 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Equals, "^="));
 27 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.ExceptionType, "^ex|^exception"));
 28 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Fingerprint, "^fingerprint"));
 29 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.NotIn, "^not in"));
 30 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.In, "^in"));
 31 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Like, "^like"));
 32 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Limit, "^limit"));
 33 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Match, "^match"));
 34 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Message, "^msg|^message"));
 35 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.NotEquals, "^!="));
 36 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.NotLike, "^not like"));
 37 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.OpenParenthesis, "^\\("));
 38 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Or, "^or"));
 39 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.StackFrame, "^sf|^stackframe"));
 40 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.DateTimeValue, "^\\d\\d\\d\\d-\\d\\d-\\d\\d \\d\\d:\\d\\d:\\d\\d"));
 41 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.StringValue, "^'[^']*'"));
 42 |             _tokenDefinitions.Add(new TokenDefinition(TokenType.Number, "^\\d+"));
 43 |         }
 44 | 
 45 |             
 46 |         public IEnumerable<DslToken> Tokenize(string lqlText)
 47 |         {
 48 |             var tokens = new List<DslToken>();
 49 | 
 50 |             string remainingText = lqlText;
 51 | 
 52 |             while (!string.IsNullOrWhiteSpace(remainingText))
 53 |             {
 54 |                 var match = FindMatch(remainingText);
 55 |                 if (match.IsMatch)
 56 |                 {
 57 |                     tokens.Add(new DslToken(match.TokenType, match.Value));
 58 |                     remainingText = match.RemainingText;
 59 |                 }
 60 |                 else
 61 |                 {
 62 |                     if (IsWhitespace(remainingText))
 63 |                     {
 64 |                         remainingText = remainingText.Substring(1);
 65 |                     }
 66 |                     else
 67 |                     {
 68 |                         var invalidTokenMatch = CreateInvalidTokenMatch(remainingText);
 69 |                         tokens.Add(new DslToken(invalidTokenMatch.TokenType, invalidTokenMatch.Value));
 70 |                         remainingText = invalidTokenMatch.RemainingText;
 71 |                     }
 72 |                 }
 73 |             }
 74 | 
 75 |             tokens.Add(new DslToken(TokenType.SequenceTerminator, string.Empty));
 76 | 
 77 |             return tokens;
 78 |         }
 79 | 
 80 |         private TokenMatch FindMatch(string lqlText)
 81 |         {
 82 |             foreach (var tokenDefinition in _tokenDefinitions)
 83 |             {
 84 |                 var match = tokenDefinition.Match(lqlText);
 85 |                 if (match.IsMatch)
 86 |                     return match;
 87 |             }
 88 | 
 89 |             return new TokenMatch() {  IsMatch = false };
 90 |         }
 91 | 
 92 |         private bool IsWhitespace(string lqlText)
 93 |         {
 94 |             return Regex.IsMatch(lqlText, "^\\s+");
 95 |         }
 96 | 
 97 |         private TokenMatch CreateInvalidTokenMatch(string lqlText)
 98 |         {
 99 |             var match = Regex.Match(lqlText, "(^\\S+\\s)|^\\S+");
100 |             if (match.Success)
101 |             {
102 |                 return new TokenMatch()
103 |                 {
104 |                     IsMatch = true,
105 |                     RemainingText = lqlText.Substring(match.Length),
106 |                     TokenType = TokenType.Invalid,
107 |                     Value = match.Value.Trim()
108 |                 };
109 |             }
110 | 
111 |             throw new DslParserException("Failed to generate invalid token");
112 |         }
113 |     }
114 | }
115 | 


--------------------------------------------------------------------------------
/DslParser/SqlGeneration/AdoQueryPayload.cs:
--------------------------------------------------------------------------------
  1 | using DslParser.DataRepresentation;
  2 | using System;
  3 | using System.Collections.Generic;
  4 | using System.Data;
  5 | using System.Data.SqlClient;
  6 | using System.Linq;
  7 | using System.Text;
  8 | using System.Threading.Tasks;
  9 | 
 10 | namespace DslParser.SqlGeneration
 11 | {
 12 |     public class AdoQueryPayload
 13 |     {
 14 |         private int _paramCounter;
 15 |         private StringBuilder _sb;
 16 | 
 17 |         public AdoQueryPayload()
 18 |         {
 19 |             Parameters = new List<SqlParameter>();
 20 |             _sb = new StringBuilder();
 21 |         }
 22 | 
 23 |         public IList<SqlParameter> Parameters { get; private set; }
 24 | 
 25 |         public void Append(string queryText)
 26 |         {
 27 |             _sb.Append(queryText);
 28 |         }
 29 | 
 30 |         public void AppendLine(string queryText)
 31 |         {
 32 |             _sb.Append(queryText + Environment.NewLine);
 33 |         }
 34 | 
 35 |         public void AddNewLine()
 36 |         {
 37 |             _sb.Append(Environment.NewLine);
 38 |         }
 39 | 
 40 |         public void AppendColumnName(DslObject dslObject)
 41 |         {
 42 |             _sb.Append(GetColumnName(dslObject));
 43 |         }
 44 | 
 45 |         public void AddParameter(MatchCondition matchCondition)
 46 |         {
 47 |             if (matchCondition.Operator == DslOperator.In || matchCondition.Operator == DslOperator.NotIn)
 48 |             {
 49 |                 AddInParameters(matchCondition.Object, matchCondition.Values);
 50 |             }
 51 |             else
 52 |             {
 53 |                 AddParameter(matchCondition.Object, matchCondition.Value);
 54 |             }
 55 |         }
 56 | 
 57 |         public void AddFromDateParameter(DateTime dateValue)
 58 |         {
 59 |             AddDateParameter(dateValue, "@FromDate");
 60 |         }
 61 | 
 62 |         public void AddToDateParameter(DateTime dateValue)
 63 |         {
 64 |             AddDateParameter(dateValue, "@ToDate");
 65 |         }
 66 | 
 67 |         public string GetSqlText()
 68 |         {
 69 |             return _sb.ToString();
 70 |         }
 71 | 
 72 |         private void AddDateParameter(DateTime dateValue, string paramName)
 73 |         {
 74 |             _sb.Append(paramName);
 75 | 
 76 |             if (!Parameters.Any(x => x.ParameterName.Equals(paramName)))
 77 |             {
 78 |                 var parameter = new SqlParameter(paramName, SqlDbType.DateTime);
 79 |                 parameter.Value = dateValue;
 80 | 
 81 |                 Parameters.Add(parameter);
 82 |             }
 83 |         }
 84 | 
 85 |         private void AddInParameters(DslObject dslObject, List<string> values)
 86 |         {
 87 |             int counter = 0;
 88 |             foreach (var value in values)
 89 |             {
 90 |                 if (counter > 0)
 91 |                     _sb.Append(",");
 92 | 
 93 |                 IncrementParamCounter();
 94 |                 var paramName = GetParameterName();
 95 |                 var parameter = new SqlParameter(paramName, SqlDbType.VarChar, GetVarcharLength(dslObject));
 96 |                 parameter.Value = value;
 97 |                 Parameters.Add(parameter);
 98 |                 _sb.Append(paramName);
 99 | 
100 |                 counter++;
101 |             }
102 |         }
103 | 
104 |         private void AddParameter(DslObject dslObject, string value)
105 |         {
106 |             IncrementParamCounter();
107 |             var paramName = GetParameterName();
108 |             _sb.Append(paramName);
109 | 
110 |             var parameter = new SqlParameter(paramName, SqlDbType.VarChar, GetVarcharLength(dslObject));
111 |             parameter.Value = value;
112 |             Parameters.Add(parameter);
113 |         }
114 | 
115 |         private void IncrementParamCounter()
116 |         {
117 |             _paramCounter++;
118 |         }
119 | 
120 |         private string GetParameterName()
121 |         {
122 |             return "@Param" + _paramCounter;
123 |         }
124 | 
125 |         private string GetColumnName(DslObject dslObject)
126 |         {
127 |             switch (dslObject)
128 |             {
129 |                 case DslObject.Application:
130 |                     return "ED.ApplicationId";
131 |                 case DslObject.Fingerprint:
132 |                     return "ED.FingerprintText";
133 |                 case DslObject.StackFrame:
134 |                     return "EB.StackFrame";
135 |                 case DslObject.ExceptionType:
136 |                     return "EB.ExceptionType";
137 |                 case DslObject.Message:
138 |                     return "T.MessageDetails";
139 |                 default:
140 |                     throw new Exception("LQL object not supported for SQL generation: " + dslObject.ToString());
141 |             }
142 |         }
143 | 
144 |         private int GetVarcharLength(DslObject dslObject)
145 |         {
146 |             switch (dslObject)
147 |             {
148 |                 case DslObject.Application:
149 |                     return 200;
150 |                 case DslObject.StackFrame:
151 |                     return 1000;
152 |                 case DslObject.ExceptionType:
153 |                     return 200;
154 |                 case DslObject.Message:
155 |                     return 1000;
156 |                 case DslObject.Fingerprint:
157 |                     return 32;
158 |                 default:
159 |                     return 50;
160 |             }
161 |         }
162 |     }
163 | }
164 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | ## Ignore Visual Studio temporary files, build results, and
  2 | ## files generated by popular Visual Studio add-ons.
  3 | 
  4 | # User-specific files
  5 | *.suo
  6 | *.user
  7 | *.userosscache
  8 | *.sln.docstates
  9 | 
 10 | # User-specific files (MonoDevelop/Xamarin Studio)
 11 | *.userprefs
 12 | 
 13 | # Build results
 14 | [Dd]ebug/
 15 | [Dd]ebugPublic/
 16 | [Rr]elease/
 17 | [Rr]eleases/
 18 | x64/
 19 | x86/
 20 | bld/
 21 | [Bb]in/
 22 | [Oo]bj/
 23 | [Ll]og/
 24 | 
 25 | # Visual Studio 2015 cache/options directory
 26 | .vs/
 27 | # Uncomment if you have tasks that create the project's static files in wwwroot
 28 | #wwwroot/
 29 | 
 30 | # MSTest test Results
 31 | [Tt]est[Rr]esult*/
 32 | [Bb]uild[Ll]og.*
 33 | 
 34 | # NUNIT
 35 | *.VisualState.xml
 36 | TestResult.xml
 37 | 
 38 | # Build Results of an ATL Project
 39 | [Dd]ebugPS/
 40 | [Rr]eleasePS/
 41 | dlldata.c
 42 | 
 43 | # DNX
 44 | project.lock.json
 45 | project.fragment.lock.json
 46 | artifacts/
 47 | 
 48 | *_i.c
 49 | *_p.c
 50 | *_i.h
 51 | *.ilk
 52 | *.meta
 53 | *.obj
 54 | *.pch
 55 | *.pdb
 56 | *.pgc
 57 | *.pgd
 58 | *.rsp
 59 | *.sbr
 60 | *.tlb
 61 | *.tli
 62 | *.tlh
 63 | *.tmp
 64 | *.tmp_proj
 65 | *.log
 66 | *.vspscc
 67 | *.vssscc
 68 | .builds
 69 | *.pidb
 70 | *.svclog
 71 | *.scc
 72 | 
 73 | # Chutzpah Test files
 74 | _Chutzpah*
 75 | 
 76 | # Visual C++ cache files
 77 | ipch/
 78 | *.aps
 79 | *.ncb
 80 | *.opendb
 81 | *.opensdf
 82 | *.sdf
 83 | *.cachefile
 84 | *.VC.db
 85 | *.VC.VC.opendb
 86 | 
 87 | # Visual Studio profiler
 88 | *.psess
 89 | *.vsp
 90 | *.vspx
 91 | *.sap
 92 | 
 93 | # TFS 2012 Local Workspace
 94 | $tf/
 95 | 
 96 | # Guidance Automation Toolkit
 97 | *.gpState
 98 | 
 99 | # ReSharper is a .NET coding add-in
100 | _ReSharper*/
101 | *.[Rr]e[Ss]harper
102 | *.DotSettings.user
103 | 
104 | # JustCode is a .NET coding add-in
105 | .JustCode
106 | 
107 | # TeamCity is a build add-in
108 | _TeamCity*
109 | 
110 | # DotCover is a Code Coverage Tool
111 | *.dotCover
112 | 
113 | # NCrunch
114 | _NCrunch_*
115 | .*crunch*.local.xml
116 | nCrunchTemp_*
117 | 
118 | # MightyMoose
119 | *.mm.*
120 | AutoTest.Net/
121 | 
122 | # Web workbench (sass)
123 | .sass-cache/
124 | 
125 | # Installshield output folder
126 | [Ee]xpress/
127 | 
128 | # DocProject is a documentation generator add-in
129 | DocProject/buildhelp/
130 | DocProject/Help/*.HxT
131 | DocProject/Help/*.HxC
132 | DocProject/Help/*.hhc
133 | DocProject/Help/*.hhk
134 | DocProject/Help/*.hhp
135 | DocProject/Help/Html2
136 | DocProject/Help/html
137 | 
138 | # Click-Once directory
139 | publish/
140 | 
141 | # Publish Web Output
142 | *.[Pp]ublish.xml
143 | *.azurePubxml
144 | # TODO: Comment the next line if you want to checkin your web deploy settings
145 | # but database connection strings (with potential passwords) will be unencrypted
146 | #*.pubxml
147 | *.publishproj
148 | 
149 | # Microsoft Azure Web App publish settings. Comment the next line if you want to
150 | # checkin your Azure Web App publish settings, but sensitive information contained
151 | # in these scripts will be unencrypted
152 | PublishScripts/
153 | 
154 | # NuGet Packages
155 | *.nupkg
156 | # The packages folder can be ignored because of Package Restore
157 | **/packages/*
158 | # except build/, which is used as an MSBuild target.
159 | !**/packages/build/
160 | # Uncomment if necessary however generally it will be regenerated when needed
161 | #!**/packages/repositories.config
162 | # NuGet v3's project.json files produces more ignoreable files
163 | *.nuget.props
164 | *.nuget.targets
165 | 
166 | # Microsoft Azure Build Output
167 | csx/
168 | *.build.csdef
169 | 
170 | # Microsoft Azure Emulator
171 | ecf/
172 | rcf/
173 | 
174 | # Windows Store app package directories and files
175 | AppPackages/
176 | BundleArtifacts/
177 | Package.StoreAssociation.xml
178 | _pkginfo.txt
179 | 
180 | # Visual Studio cache files
181 | # files ending in .cache can be ignored
182 | *.[Cc]ache
183 | # but keep track of directories ending in .cache
184 | !*.[Cc]ache/
185 | 
186 | # Others
187 | ClientBin/
188 | ~$*
189 | *~
190 | *.dbmdl
191 | *.dbproj.schemaview
192 | *.jfm
193 | *.pfx
194 | *.publishsettings
195 | node_modules/
196 | orleans.codegen.cs
197 | 
198 | # Since there are multiple workflows, uncomment next line to ignore bower_components
199 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
200 | #bower_components/
201 | 
202 | # RIA/Silverlight projects
203 | Generated_Code/
204 | 
205 | # Backup & report files from converting an old project file
206 | # to a newer Visual Studio version. Backup files are not needed,
207 | # because we have git ;-)
208 | _UpgradeReport_Files/
209 | Backup*/
210 | UpgradeLog*.XML
211 | UpgradeLog*.htm
212 | 
213 | # SQL Server files
214 | *.mdf
215 | *.ldf
216 | 
217 | # Business Intelligence projects
218 | *.rdl.data
219 | *.bim.layout
220 | *.bim_*.settings
221 | 
222 | # Microsoft Fakes
223 | FakesAssemblies/
224 | 
225 | # GhostDoc plugin setting file
226 | *.GhostDoc.xml
227 | 
228 | # Node.js Tools for Visual Studio
229 | .ntvs_analysis.dat
230 | 
231 | # Visual Studio 6 build log
232 | *.plg
233 | 
234 | # Visual Studio 6 workspace options file
235 | *.opt
236 | 
237 | # Visual Studio LightSwitch build output
238 | **/*.HTMLClient/GeneratedArtifacts
239 | **/*.DesktopClient/GeneratedArtifacts
240 | **/*.DesktopClient/ModelManifest.xml
241 | **/*.Server/GeneratedArtifacts
242 | **/*.Server/ModelManifest.xml
243 | _Pvt_Extensions
244 | 
245 | # Paket dependency manager
246 | .paket/paket.exe
247 | paket-files/
248 | 
249 | # FAKE - F# Make
250 | .fake/
251 | 
252 | # JetBrains Rider
253 | .idea/
254 | *.sln.iml
255 | 
256 | # CodeRush
257 | .cr/
258 | 
259 | # Python Tools for Visual Studio (PTVS)
260 | __pycache__/
261 | *.pyc


--------------------------------------------------------------------------------
/DslParser/Parsing/Parser.cs:
--------------------------------------------------------------------------------
  1 | using System;
  2 | using System.Collections.Generic;
  3 | using System.Globalization;
  4 | using System.Linq;
  5 | using System.Text;
  6 | using System.Threading.Tasks;
  7 | using DslParser.DataRepresentation;
  8 | using DslParser.Exceptions;
  9 | using DslParser.Parsing.Tokens;
 10 | 
 11 | namespace DslParser.Parsing
 12 | {
 13 |     public class Parser
 14 |     {
 15 |         private Stack<DslToken> _tokenSequence;
 16 |         private DslToken _lookaheadFirst;
 17 |         private DslToken _lookaheadSecond;
 18 | 
 19 |         private DslQueryModel _queryModel;
 20 |         private MatchCondition _currentMatchCondition;
 21 |         
 22 |         private const string ExpectedObjectErrorText = "Expected =, !=, LIKE, NOT LIKE, IN or NOT IN but found: ";
 23 | 
 24 |         public DslQueryModel Parse(List<DslToken> tokens)
 25 |         {
 26 |             LoadSequenceStack(tokens);
 27 |             PrepareLookaheads();
 28 |             _queryModel = new DslQueryModel();
 29 | 
 30 |             Match();
 31 | 
 32 |             DiscardToken(TokenType.SequenceTerminator);
 33 | 
 34 |             return _queryModel;
 35 |         }
 36 | 
 37 |         private void LoadSequenceStack(List<DslToken> tokens)
 38 |         {
 39 |             _tokenSequence = new Stack<DslToken>();
 40 |             int count = tokens.Count;
 41 |             for (int i = count - 1; i >= 0; i--)
 42 |             {
 43 |                 _tokenSequence.Push(tokens[i]);
 44 |             }
 45 |         }
 46 | 
 47 |         private void PrepareLookaheads()
 48 |         {
 49 |             _lookaheadFirst = _tokenSequence.Pop();
 50 |             _lookaheadSecond = _tokenSequence.Pop();
 51 |         }
 52 | 
 53 |         private DslToken ReadToken(TokenType tokenType)
 54 |         {
 55 |             if (_lookaheadFirst.TokenType != tokenType)
 56 |                 throw new DslParserException(string.Format("Expected {0} but found: {1}", tokenType.ToString().ToUpper(), _lookaheadFirst.Value));
 57 | 
 58 |             return _lookaheadFirst;
 59 |         }
 60 | 
 61 |         private void DiscardToken()
 62 |         {
 63 |             _lookaheadFirst = _lookaheadSecond.Clone();
 64 | 
 65 |             if (_tokenSequence.Any())
 66 |                 _lookaheadSecond = _tokenSequence.Pop();
 67 |             else
 68 |                 _lookaheadSecond = new DslToken(TokenType.SequenceTerminator, string.Empty);
 69 |         }
 70 | 
 71 |         private void DiscardToken(TokenType tokenType)
 72 |         {
 73 |             if (_lookaheadFirst.TokenType != tokenType)
 74 |                 throw new DslParserException(string.Format("Expected {0} but found: {1}", tokenType.ToString().ToUpper(), _lookaheadFirst.Value));
 75 | 
 76 |             DiscardToken();
 77 |         }
 78 | 
 79 |         private void Match()
 80 |         {
 81 |             DiscardToken(TokenType.Match);
 82 |             MatchCondition();
 83 |         }
 84 | 
 85 |         private void MatchCondition()
 86 |         {
 87 |             CreateNewMatchCondition();
 88 | 
 89 |             if (IsObject(_lookaheadFirst))
 90 |             {
 91 |                 if (IsEqualityOperator(_lookaheadSecond))
 92 |                 {
 93 |                     EqualityMatchCondition();
 94 |                 }
 95 |                 else if (_lookaheadSecond.TokenType == TokenType.In)
 96 |                 {
 97 |                     InCondition();
 98 |                 }
 99 |                 else if (_lookaheadSecond.TokenType == TokenType.NotIn)
100 |                 {
101 |                     NotInCondition();
102 |                 }
103 |                 else
104 |                 {
105 |                     throw new DslParserException(ExpectedObjectErrorText + " " + _lookaheadSecond.Value);
106 |                 }
107 | 
108 |                 MatchConditionNext();
109 |             }
110 |             else
111 |             {
112 |                 throw new DslParserException(ExpectedObjectErrorText + _lookaheadFirst.Value);
113 |             }
114 |         }
115 | 
116 |         private void EqualityMatchCondition()
117 |         {
118 |             _currentMatchCondition.Object = GetObject(_lookaheadFirst);
119 |             DiscardToken();
120 |             _currentMatchCondition.Operator = GetOperator(_lookaheadFirst);
121 |             DiscardToken();
122 |             _currentMatchCondition.Value = _lookaheadFirst.Value;
123 |             DiscardToken();
124 |         }
125 | 
126 |         private DslObject GetObject(DslToken token)
127 |         {
128 |             switch (token.TokenType)
129 |             {
130 |                 case TokenType.Application:
131 |                     return DslObject.Application;
132 |                 case TokenType.ExceptionType:
133 |                     return DslObject.ExceptionType;
134 |                 case TokenType.Fingerprint:
135 |                     return DslObject.Fingerprint;
136 |                 case TokenType.Message:
137 |                     return DslObject.Message;
138 |                 case TokenType.StackFrame:
139 |                     return DslObject.StackFrame;
140 |                 default:
141 |                     throw new DslParserException(ExpectedObjectErrorText + token.Value);
142 |             }
143 |         }
144 | 
145 |         private DslOperator GetOperator(DslToken token)
146 |         {
147 |             switch (token.TokenType)
148 |             {
149 |                 case TokenType.Equals:
150 |                     return DslOperator.Equals;
151 |                 case TokenType.NotEquals:
152 |                     return DslOperator.NotEquals;
153 |                 case TokenType.Like:
154 |                     return DslOperator.Like;
155 |                 case TokenType.NotLike:
156 |                     return DslOperator.NotLike;
157 |                 case TokenType.In:
158 |                     return DslOperator.In;
159 |                 case TokenType.NotIn:
160 |                     return DslOperator.NotIn;
161 |                 default:
162 |                     throw new DslParserException("Expected =, !=, LIKE, NOT LIKE, IN, NOT IN but found: " + token.Value);
163 |             }
164 |         }
165 | 
166 |         private void NotInCondition()
167 |         {
168 |             ParseInCondition(DslOperator.NotIn);
169 |         }
170 | 
171 |         private void InCondition()
172 |         {
173 |             ParseInCondition(DslOperator.In);
174 |         }
175 | 
176 |         private void ParseInCondition(DslOperator inOperator)
177 |         {
178 |             _currentMatchCondition.Operator = inOperator;
179 |             _currentMatchCondition.Values = new List<string>();
180 |             _currentMatchCondition.Object = GetObject(_lookaheadFirst);
181 |             DiscardToken();
182 | 
183 |             if (inOperator == DslOperator.In)
184 |                 DiscardToken(TokenType.In);
185 |             else if (inOperator == DslOperator.NotIn)
186 |                 DiscardToken(TokenType.NotIn);
187 | 
188 |             DiscardToken(TokenType.OpenParenthesis);
189 |             StringLiteralList();
190 |             DiscardToken(TokenType.CloseParenthesis);
191 |         }
192 | 
193 |         private void StringLiteralList()
194 |         {
195 |             _currentMatchCondition.Values.Add(ReadToken(TokenType.StringValue).Value);
196 |             DiscardToken(TokenType.StringValue);
197 |             StringLiteralListNext();
198 |         }
199 | 
200 |         private void StringLiteralListNext()
201 |         {
202 |             if (_lookaheadFirst.TokenType == TokenType.Comma)
203 |             {
204 |                 DiscardToken(TokenType.Comma);
205 |                 _currentMatchCondition.Values.Add(ReadToken(TokenType.StringValue).Value);
206 |                 DiscardToken(TokenType.StringValue);
207 |                 StringLiteralListNext();
208 |             }
209 |             else
210 |             {
211 |                 // nothing
212 |             }
213 |         }
214 | 
215 |         private void MatchConditionNext()
216 |         {
217 |             if (_lookaheadFirst.TokenType == TokenType.And)
218 |             {
219 |                 AndMatchCondition();
220 |             }
221 |             else if (_lookaheadFirst.TokenType == TokenType.Or)
222 |             {
223 |                 OrMatchCondition();
224 |             }
225 |             else if (_lookaheadFirst.TokenType == TokenType.Between)
226 |             {
227 |                 DateCondition();
228 |             }
229 |             else
230 |             {
231 |                 throw new DslParserException("Expected AND, OR or BETWEEN but found: " + _lookaheadFirst.Value);
232 |             }
233 |         }
234 | 
235 |         private void AndMatchCondition()
236 |         {
237 |             _currentMatchCondition.LogOpToNextCondition = DslLogicalOperator.And;
238 |             DiscardToken(TokenType.And);
239 |             MatchCondition();
240 |         }
241 | 
242 |         private void OrMatchCondition()
243 |         {
244 |             _currentMatchCondition.LogOpToNextCondition = DslLogicalOperator.Or;
245 |             DiscardToken(TokenType.Or);
246 |             MatchCondition();
247 |         }
248 | 
249 |         private void DateCondition()
250 |         {
251 |             DiscardToken(TokenType.Between);
252 | 
253 |             _queryModel.DateRange = new DateRange();
254 |             _queryModel.DateRange.From = DateTime.ParseExact(ReadToken(TokenType.DateTimeValue).Value, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
255 |             DiscardToken(TokenType.DateTimeValue);
256 |             DiscardToken(TokenType.And);
257 |             _queryModel.DateRange.To = DateTime.ParseExact(ReadToken(TokenType.DateTimeValue).Value, "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
258 |             DiscardToken(TokenType.DateTimeValue);
259 |             DateConditionNext();
260 |         }
261 | 
262 |         private void DateConditionNext()
263 |         {
264 |             if (_lookaheadFirst.TokenType == TokenType.Limit)
265 |             {
266 |                 Limit();
267 |             }
268 |             else if (_lookaheadFirst.TokenType == TokenType.SequenceTerminator)
269 |             {
270 |                 // nothing
271 |             }
272 |             else
273 |             {
274 |                 throw new DslParserException("Expected LIMIT or the end of the query but found: " + _lookaheadFirst.Value);
275 |             }
276 | 
277 |         }
278 | 
279 |         private void Limit()
280 |         {
281 |             DiscardToken(TokenType.Limit);
282 |             int limit = 0;
283 |             bool success = int.TryParse(ReadToken(TokenType.Number).Value, out limit);
284 |             if (success)
285 |                 _queryModel.Limit = limit;
286 |             else
287 |                 throw new DslParserException("Expected an integer number but found " + ReadToken(TokenType.Number).Value);
288 | 
289 |             DiscardToken(TokenType.Number);
290 |         }
291 | 
292 |         private bool IsObject(DslToken token)
293 |         {
294 |             return token.TokenType == TokenType.Application
295 |                    || token.TokenType == TokenType.ExceptionType
296 |                    || token.TokenType == TokenType.Fingerprint
297 |                    || token.TokenType == TokenType.Message
298 |                    || token.TokenType == TokenType.StackFrame;
299 |         }
300 | 
301 |         private bool IsEqualityOperator(DslToken token)
302 |         {
303 |             return token.TokenType == TokenType.Equals
304 |                    || token.TokenType == TokenType.NotEquals
305 |                    || token.TokenType == TokenType.Like
306 |                    || token.TokenType == TokenType.NotLike;
307 |         }
308 | 
309 |         private void CreateNewMatchCondition()
310 |         {
311 |             _currentMatchCondition = new MatchCondition();
312 |             _queryModel.MatchConditions.Add(_currentMatchCondition);
313 |         }
314 | 
315 |     }
316 | }
317 | 


--------------------------------------------------------------------------------
/DslParser/Program.cs:
--------------------------------------------------------------------------------
  1 | using System;
  2 | using System.Collections.Generic;
  3 | using System.Diagnostics;
  4 | using System.IO;
  5 | using System.Linq;
  6 | using System.Text;
  7 | using System.Threading.Tasks;
  8 | using DslParser.Exceptions;
  9 | using DslParser.Parsing;
 10 | using DslParser.Parsing.Tokenizers;
 11 | using DslParser.Parsing.Tokenizers.MoreEfficient;
 12 | using DslParser.Parsing.Tokenizers.SlowAndSimple;
 13 | using Newtonsoft.Json;
 14 | using DslParser.SqlGeneration;
 15 | using DslParser.DataRepresentation;
 16 | 
 17 | namespace DslParser
 18 | {
 19 |     class Program
 20 |     {
 21 |         static void Main(string[] args)
 22 |         {
 23 |             new Program().Run();
 24 |         }
 25 | 
 26 |         public void Run()
 27 |         {
 28 |             while (true)
 29 |             {
 30 |                 Console.WriteLine("Press 1 : view inefficient version output");
 31 |                 Console.WriteLine("Press 2 : Perf test of inefficient tokenizer with a small query");
 32 |                 Console.WriteLine("Press 3 : Perf test of inefficient tokenizer with a large query");
 33 |                 Console.WriteLine("Press 4 : view more efficient version output");
 34 |                 Console.WriteLine("Press 5 : Perf test of more efficient tokenizer with a small query");
 35 |                 Console.WriteLine("Press 6 : Perf test of more efficient tokenizer with a large query");
 36 | 
 37 |                 var key = Console.ReadKey();
 38 |                 Console.WriteLine("");
 39 | 
 40 |                 switch (key.KeyChar.ToString())
 41 |                 {
 42 |                     case "1":
 43 |                         ITokenizer slowTokenizer = new SimpleRegexTokenizer();
 44 |                         RunOnceAndPrintOutput(slowTokenizer, "Run with inefficient tokenizer");
 45 |                         break;
 46 |                     case "2":
 47 |                         PerfTestWithSlowTokenizerAndSmallQuery();
 48 |                         break;
 49 |                     case "3":
 50 |                         PerfTestWithSlowTokenizerAndLargeQuery();
 51 |                         break;
 52 |                     case "4":
 53 |                         ITokenizer fastTokenizer = new PrecedenceBasedRegexTokenizer();
 54 |                         RunOnceAndPrintOutput(fastTokenizer, "Run with faster tokenizer");
 55 |                         break;
 56 |                     case "5":
 57 |                         PerfTestWithFastTokenizerAndSmallQuery();
 58 |                         break;
 59 |                     case "6":
 60 |                         PerfTestWithFastTokenizerAndLargeQuery();
 61 |                         break;
 62 |                     default:
 63 |                         Console.WriteLine("Press 1, 2, 3, 4, 5 or 6");
 64 |                         break;
 65 |                 }
 66 | 
 67 |                 Console.WriteLine("");
 68 |             }
 69 |         }
 70 | 
 71 |         public void RunOnceAndPrintOutput(ITokenizer tokenizer, string startMessage)
 72 |         {
 73 |             Console.WriteLine(startMessage);
 74 |             Console.WriteLine("");
 75 | 
 76 |             var parser = new Parser();
 77 |             var sqlGenerator = new SqlGenerator();
 78 | 
 79 |             string query = @"MATCH app = 'MyTestApp'
 80 | AND ex IN ('System.NullReferenceException', 'System.FormatException')
 81 | BETWEEN 2016-01-01 00:00:00 AND 2016-02-01 00:00:00
 82 | LIMIT 100";
 83 | 
 84 |             Console.WriteLine("");
 85 |             Console.WriteLine("The DSL query:");
 86 |             Console.WriteLine(query);
 87 |             Console.WriteLine("");
 88 |             Console.WriteLine("Tokens generated:");
 89 | 
 90 |             var tokenSequence = tokenizer.Tokenize(query).ToList();
 91 |             foreach(var token in tokenSequence)
 92 |                 Console.WriteLine(string.Format("TokenType: {0}, Value: {1}", token.TokenType, token.Value));
 93 | 
 94 |             var dataRepresentation = parser.Parse(tokenSequence);
 95 |             Console.WriteLine("");
 96 |             Console.WriteLine("Data Representation (serialized to JSON)");
 97 |             Console.WriteLine(JsonConvert.SerializeObject(dataRepresentation, Formatting.Indented));
 98 | 
 99 |             Console.WriteLine("");
100 |             Console.WriteLine("SQL Generated:");
101 |             var sql = sqlGenerator.GenerateQueryPayload(dataRepresentation);
102 |             Console.WriteLine(sql.GetSqlText());
103 | 
104 |             Console.WriteLine("");
105 |             Console.WriteLine("Process complete");
106 |         }
107 | 
108 |         public void PerfTestWithSlowTokenizerAndSmallQuery()
109 |         {
110 |             ITokenizer tokenizer = new SimpleRegexTokenizer();
111 |             string query = @"MATCH app = 'MyTestApp'
112 | AND ex IN ('System.NullReferenceException', 'System.FormatException')
113 | BETWEEN 2016-01-01 00:00:00 AND 2016-02-01 00:00:00
114 | LIMIT 100";
115 | 
116 |             PerfTest(tokenizer, query, "Slow tokenizer + small query");
117 |         }
118 | 
119 |         public void PerfTestWithSlowTokenizerAndLargeQuery()
120 |         {
121 |             ITokenizer tokenizer = new SimpleRegexTokenizer();
122 |             string query = @"MATCH app = 'MyTestApp'
123 | AND ex IN ('System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException')
124 | AND sf = 'sadsdfsdfsdfsdfssdfjhsfjhsdfjhsdfjhsdfjhsdjfhsdjhfsdjfhsdhfsdjhfsdjhfjsdhfjsdhfjhsdjfhsdjfh'
125 | AND sf = 'fggdfgdfgfdgdfgdfgggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggh'
126 | AND sf = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
127 | AND sf = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
128 | AND sf = 'ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'
129 | AND sf = 'ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd'
130 | AND sf = '1eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee'
131 | AND sf = '2sadsdfsdfsdfsdfssdfjhsfjhsdfjhsdfjhsdfjhsdjfhsdjhfsdjfhsdhfsdjhfsdjhfjsdhfjsdhfjhsdjfhsdjfh'
132 | AND sf = '3fggdfgdfgfdgdfgdfgggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggh'
133 | AND sf = '4aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
134 | AND sf = '5bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
135 | AND sf = '6ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'
136 | AND sf = '7ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd'
137 | AND sf = '8eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee'
138 | AND ex IN ('System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException')
139 | AND ex IN ('System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException')
140 | AND ex IN ('System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException')
141 | BETWEEN 2016-01-01 00:00:00 AND 2016-02-01 00:00:00
142 | LIMIT 100";
143 | 
144 |             PerfTest(tokenizer, query, "Slow tokenizer + large query");
145 |         }
146 | 
147 |         public void PerfTestWithFastTokenizerAndSmallQuery()
148 |         {
149 |             ITokenizer tokenizer = new PrecedenceBasedRegexTokenizer();
150 |             string query = @"MATCH app = 'MyTestApp'
151 | AND ex IN ('System.NullReferenceException', 'System.FormatException')
152 | BETWEEN 2016-01-01 00:00:00 AND 2016-02-01 00:00:00
153 | LIMIT 100";
154 | 
155 |             PerfTest(tokenizer, query, "Fast tokenizer + small query");
156 |         }
157 | 
158 |         public void PerfTestWithFastTokenizerAndLargeQuery()
159 |         {
160 |             ITokenizer tokenizer = new PrecedenceBasedRegexTokenizer();
161 |             string query = @"MATCH app = 'MyTestApp'
162 | AND ex IN ('System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException')
163 | AND sf = 'sadsdfsdfsdfsdfssdfjhsfjhsdfjhsdfjhsdfjhsdjfhsdjhfsdjfhsdhfsdjhfsdjhfjsdhfjsdhfjhsdjfhsdjfh'
164 | AND sf = 'fggdfgdfgfdgdfgdfgggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggh'
165 | AND sf = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
166 | AND sf = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
167 | AND sf = 'ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'
168 | AND sf = 'ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd'
169 | AND sf = '1eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee'
170 | AND sf = '2sadsdfsdfsdfsdfssdfjhsfjhsdfjhsdfjhsdfjhsdjfhsdjhfsdjfhsdhfsdjhfsdjhfjsdhfjsdhfjhsdjfhsdjfh'
171 | AND sf = '3fggdfgdfgfdgdfgdfgggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggh'
172 | AND sf = '4aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
173 | AND sf = '5bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
174 | AND sf = '6ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'
175 | AND sf = '7ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd'
176 | AND sf = '8eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee'
177 | AND ex IN ('System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException')
178 | AND ex IN ('System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException')
179 | AND ex IN ('System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException','System.NullReferenceException', 'System.FormatException')
180 | BETWEEN 2016-01-01 00:00:00 AND 2016-02-01 00:00:00
181 | LIMIT 100";
182 | 
183 |             PerfTest(tokenizer, query, "Fast tokenizer + large query");
184 |         }
185 | 
186 |         public void PerfTest(ITokenizer tokenizer, string query, string startMessage)
187 |         {
188 |             Console.WriteLine(startMessage);
189 |             Console.WriteLine("Will run the process 1000 times. Query char count: " + query.Length);
190 | 
191 |             var sw = new Stopwatch();
192 |             sw.Start();
193 | 
194 |             for(int i=0; i<1000; i++)
195 |                 RunOnceWithoutOutput(tokenizer, query);
196 | 
197 |             sw.Stop();
198 |             Console.WriteLine("Elapsed milliseconds: " + sw.ElapsedMilliseconds);
199 |             Console.WriteLine("");
200 |         }
201 | 
202 |         public void RunOnceWithoutOutput(ITokenizer tokenizer, string queryText)
203 |         {
204 |             var parser = new Parser();
205 |             var sqlGenerator = new SqlGenerator();
206 | 
207 |             var tokenSequence = tokenizer.Tokenize(queryText).ToList();
208 |             var dataRepresentation = parser.Parse(tokenSequence);
209 |             var sql = sqlGenerator.GenerateQueryPayload(dataRepresentation);
210 |         }
211 | 
212 |     }
213 | }
214 | 


--------------------------------------------------------------------------------