├── .gitignore ├── LICENSE ├── README.md └── src ├── Lexer.VS10.sln ├── Lexer.VS15.sln ├── Lexer.VS8.sln └── Lexer ├── Lexer.VS10.csproj ├── Lexer.VS15.csproj ├── Lexer.VS8.csproj ├── Lexer.cs └── Properties └── AssemblyInfo.cs /.gitignore: -------------------------------------------------------------------------------- 1 | # Build Folders (you can keep bin if you'd like, to store dlls and pdbs) 2 | [Bb]in/ 3 | [Oo]bj/ 4 | 5 | # mstest test results 6 | TestResults 7 | 8 | ## Ignore Visual Studio temporary files, build results, and 9 | ## files generated by popular Visual Studio add-ons. 10 | 11 | # User-specific files 12 | *.suo 13 | *.user 14 | *.sln.docstates 15 | 16 | # Build results 17 | [Dd]ebug/ 18 | [Rr]elease/ 19 | x64/ 20 | *_i.c 21 | *_p.c 22 | *.ilk 23 | *.meta 24 | *.obj 25 | *.pch 26 | *.pdb 27 | *.pgc 28 | *.pgd 29 | *.rsp 30 | *.sbr 31 | *.tlb 32 | *.tli 33 | *.tlh 34 | *.tmp 35 | *.log 36 | *.vspscc 37 | *.vssscc 38 | .builds 39 | 40 | # Visual C++ cache files 41 | ipch/ 42 | *.aps 43 | *.ncb 44 | *.opensdf 45 | *.sdf 46 | 47 | # Visual Studio profiler 48 | *.psess 49 | *.vsp 50 | *.vspx 51 | 52 | # Guidance Automation Toolkit 53 | *.gpState 54 | 55 | # ReSharper is a .NET coding add-in 56 | _ReSharper* 57 | 58 | # NCrunch 59 | *.ncrunch* 60 | .*crunch*.local.xml 61 | 62 | # Installshield output folder 63 | [Ee]xpress 64 | 65 | # DocProject is a documentation generator add-in 66 | DocProject/buildhelp/ 67 | DocProject/Help/*.HxT 68 | DocProject/Help/*.HxC 69 | DocProject/Help/*.hhc 70 | DocProject/Help/*.hhk 71 | DocProject/Help/*.hhp 72 | DocProject/Help/Html2 73 | DocProject/Help/html 74 | 75 | # Click-Once directory 76 | publish 77 | 78 | # Publish Web Output 79 | *.Publish.xml 80 | 81 | # NuGet Packages Directory 82 | packages 83 | 84 | # Windows Azure Build Output 85 | csx 86 | *.build.csdef 87 | 88 | # Windows Store app package directory 89 | AppPackages/ 90 | 91 | # Others 92 | [Bb]in 93 | [Oo]bj 94 | sql 95 | TestResults 96 | [Tt]est[Rr]esult* 97 | *.Cache 98 | ClientBin 99 | [Ss]tyle[Cc]op.* 100 | ~$* 101 | *.dbmdl 102 | Generated_Code #added for RIA/Silverlight projects 103 | 104 | # Backup & report files from converting an old project file to a newer 105 | # Visual Studio version. Backup files are not needed, because we have git ;-) 106 | _UpgradeReport_Files/ 107 | Backup*/ 108 | UpgradeLog*.XML 109 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 Oleg Shevchenko 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | lexer 2 | ===== 3 | 4 | .NET Lexer (StringTokenizer) class 5 | -------------------------------------------------------------------------------- /src/Lexer.VS10.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 11.00 3 | # Visual Studio 2010 4 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lexer.VS10", "Lexer\Lexer.VS10.csproj", "{1A63652C-F7C5-4FFE-8308-D92E421B46A2}" 5 | EndProject 6 | Global 7 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 8 | Debug|Any CPU = Debug|Any CPU 9 | Release|Any CPU = Release|Any CPU 10 | EndGlobalSection 11 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 12 | {1A63652C-F7C5-4FFE-8308-D92E421B46A2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 13 | {1A63652C-F7C5-4FFE-8308-D92E421B46A2}.Debug|Any CPU.Build.0 = Debug|Any CPU 14 | {1A63652C-F7C5-4FFE-8308-D92E421B46A2}.Release|Any CPU.ActiveCfg = Release|Any CPU 15 | {1A63652C-F7C5-4FFE-8308-D92E421B46A2}.Release|Any CPU.Build.0 = Release|Any CPU 16 | EndGlobalSection 17 | GlobalSection(SolutionProperties) = preSolution 18 | HideSolutionNode = FALSE 19 | EndGlobalSection 20 | EndGlobal 21 | -------------------------------------------------------------------------------- /src/Lexer.VS15.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 11.00 3 | # Visual Studio 2010 4 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lexer.VS15", "Lexer\Lexer.VS15.csproj", "{1A63652C-F7C5-4FFE-8308-D92E421B46A2}" 5 | EndProject 6 | Global 7 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 8 | Debug|Any CPU = Debug|Any CPU 9 | Release|Any CPU = Release|Any CPU 10 | EndGlobalSection 11 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 12 | {1A63652C-F7C5-4FFE-8308-D92E421B46A2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 13 | {1A63652C-F7C5-4FFE-8308-D92E421B46A2}.Debug|Any CPU.Build.0 = Debug|Any CPU 14 | {1A63652C-F7C5-4FFE-8308-D92E421B46A2}.Release|Any CPU.ActiveCfg = Release|Any CPU 15 | {1A63652C-F7C5-4FFE-8308-D92E421B46A2}.Release|Any CPU.Build.0 = Release|Any CPU 16 | EndGlobalSection 17 | GlobalSection(SolutionProperties) = preSolution 18 | HideSolutionNode = FALSE 19 | EndGlobalSection 20 | EndGlobal 21 | -------------------------------------------------------------------------------- /src/Lexer.VS8.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 10.00 3 | # Visual Studio 2008 4 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lexer", "Lexer\Lexer.VS8.csproj", "{4A63652C-F7C5-4FFE-8308-D92E421B46A2}" 5 | EndProject 6 | Global 7 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 8 | Debug|Any CPU = Debug|Any CPU 9 | Release|Any CPU = Release|Any CPU 10 | EndGlobalSection 11 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 12 | {4A63652C-F7C5-4FFE-8308-D92E421B46A2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 13 | {4A63652C-F7C5-4FFE-8308-D92E421B46A2}.Debug|Any CPU.Build.0 = Debug|Any CPU 14 | {4A63652C-F7C5-4FFE-8308-D92E421B46A2}.Release|Any CPU.ActiveCfg = Release|Any CPU 15 | {4A63652C-F7C5-4FFE-8308-D92E421B46A2}.Release|Any CPU.Build.0 = Release|Any CPU 16 | EndGlobalSection 17 | GlobalSection(SolutionProperties) = preSolution 18 | HideSolutionNode = FALSE 19 | EndGlobalSection 20 | EndGlobal 21 | -------------------------------------------------------------------------------- /src/Lexer/Lexer.VS10.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Debug 5 | AnyCPU 6 | 9.0.30729 7 | 2.0 8 | {1A63652C-F7C5-4FFE-8308-D92E421B46A2} 9 | Library 10 | Properties 11 | Lexer 12 | Lexer 13 | v4.0 14 | 512 15 | 16 | 17 | 18 | 19 | 3.5 20 | 21 | publish\ 22 | true 23 | Disk 24 | false 25 | Foreground 26 | 7 27 | Days 28 | false 29 | false 30 | true 31 | 0 32 | 1.0.0.%2a 33 | false 34 | false 35 | true 36 | 37 | 38 | true 39 | full 40 | false 41 | bin\Debug\ 42 | DEBUG;TRACE 43 | prompt 44 | 4 45 | 46 | 47 | pdbonly 48 | true 49 | bin\Release\ 50 | TRACE 51 | prompt 52 | 4 53 | 54 | 55 | 56 | 57 | 3.5 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | False 67 | .NET Framework 3.5 SP1 Client Profile 68 | false 69 | 70 | 71 | False 72 | .NET Framework 3.5 SP1 73 | true 74 | 75 | 76 | False 77 | Windows Installer 3.1 78 | true 79 | 80 | 81 | 82 | 89 | -------------------------------------------------------------------------------- /src/Lexer/Lexer.VS15.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Debug 5 | AnyCPU 6 | 9.0.30729 7 | 2.0 8 | {1A63652C-F7C5-4FFE-8308-D92E421B46A2} 9 | Library 10 | Properties 11 | Lexer 12 | Lexer 13 | v4.0 14 | 512 15 | 16 | 17 | 18 | 19 | 3.5 20 | 21 | publish\ 22 | true 23 | Disk 24 | false 25 | Foreground 26 | 7 27 | Days 28 | false 29 | false 30 | true 31 | 0 32 | 1.0.0.%2a 33 | false 34 | false 35 | true 36 | 37 | 38 | true 39 | full 40 | false 41 | bin\Debug\ 42 | DEBUG;TRACE 43 | prompt 44 | 4 45 | 46 | 47 | pdbonly 48 | true 49 | bin\Release\ 50 | TRACE 51 | prompt 52 | 4 53 | 54 | 55 | 56 | 57 | 3.5 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | False 67 | .NET Framework 3.5 SP1 Client Profile 68 | false 69 | 70 | 71 | False 72 | .NET Framework 3.5 SP1 73 | true 74 | 75 | 76 | False 77 | Windows Installer 3.1 78 | true 79 | 80 | 81 | 82 | 89 | -------------------------------------------------------------------------------- /src/Lexer/Lexer.VS8.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Debug 5 | AnyCPU 6 | 9.0.30729 7 | 2.0 8 | {4A63652C-F7C5-4FFE-8308-D92E421B46A2} 9 | Library 10 | Properties 11 | Lexer 12 | Lexer 13 | v3.5 14 | 512 15 | 16 | 17 | true 18 | full 19 | false 20 | bin\Debug\ 21 | DEBUG;TRACE 22 | prompt 23 | 4 24 | 25 | 26 | pdbonly 27 | true 28 | bin\Release\ 29 | TRACE 30 | prompt 31 | 4 32 | 33 | 34 | 35 | 36 | 3.5 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 51 | -------------------------------------------------------------------------------- /src/Lexer/Lexer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections; 3 | using System.Collections.Generic; 4 | using System.Globalization; 5 | using System.IO; 6 | using System.Text; 7 | 8 | /* 9 | The MIT License (MIT) 10 | 11 | Copyright (c) 2013 Oleg Shevchenko 12 | 13 | Permission is hereby granted, free of charge, to any person obtaining a copy of 14 | this software and associated documentation files (the "Software"), to deal in 15 | the Software without restriction, including without limitation the rights to 16 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 17 | the Software, and to permit persons to whom the Software is furnished to do so, 18 | subject to the following conditions: 19 | 20 | The above copyright notice and this permission notice shall be included in all 21 | copies or substantial portions of the Software. 22 | 23 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 24 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 25 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 26 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 27 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 28 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 29 | */ 30 | 31 | namespace System.Text 32 | { 33 | public class Lexer : IEnumerable, IEnumerator 34 | { 35 | private readonly LexerSettings _settings; 36 | private LexerBehavior _behavior; 37 | private TextReader _reader; 38 | private string _text; 39 | private int _position; 40 | private int _start; 41 | private int _textLen; 42 | private int _textPos; 43 | private int _textBeg; 44 | private int _bufBeg; 45 | private int _maxSymLen; 46 | private int _lineBegin; 47 | private int _lineNumber; 48 | private int _endLineBegin; 49 | private int _endLineNumber; 50 | private StringBuilder _buffer; 51 | private StringBuilder _tokenBuffer; 52 | private Token _current; 53 | private Token _next; 54 | 55 | private Lexer(string text, TextReader reader, LexerBehavior behavior, LexerSettings settings) 56 | { 57 | if (settings == null) 58 | { 59 | settings = LexerSettings.Default; 60 | } 61 | else 62 | { 63 | settings = settings.Clone(); 64 | } 65 | 66 | _text = text; 67 | _reader = reader; 68 | _behavior = behavior; 69 | _settings = settings; 70 | 71 | if (settings.Symbols != null) 72 | { 73 | foreach (KeyValuePair entry in settings.Symbols) 74 | { 75 | int len = entry.Key.Length; 76 | if (len > _maxSymLen) 77 | { 78 | _maxSymLen = len; 79 | } 80 | } 81 | } 82 | 83 | Reset(); 84 | } 85 | 86 | public Lexer(string text, LexerBehavior behavior, LexerSettings settings) 87 | : this(text, null, behavior, settings) 88 | { 89 | } 90 | 91 | public Lexer(string text, LexerBehavior behavior) 92 | : this(text, null, behavior, null) 93 | { 94 | } 95 | 96 | public Lexer(string text, LexerSettings settings) 97 | : this(text, null, LexerBehavior.Default, settings) 98 | { 99 | } 100 | 101 | public Lexer(string text) 102 | : this(text, null, LexerBehavior.Default, null) 103 | { 104 | } 105 | 106 | public Lexer(TextReader reader, LexerBehavior behavior, LexerSettings settings) 107 | : this(null, reader, behavior, settings) 108 | { 109 | } 110 | 111 | public Lexer(TextReader reader, LexerBehavior behavior) 112 | : this(null, reader, behavior, null) 113 | { 114 | } 115 | 116 | public Lexer(TextReader reader, LexerSettings settings) 117 | : this(null, reader, LexerBehavior.Default, settings) 118 | { 119 | } 120 | 121 | public Lexer(TextReader reader) 122 | : this(null, reader, LexerBehavior.Default, null) 123 | { 124 | } 125 | 126 | private const int BufferCapacity = 8192; 127 | 128 | private const char EndOfTextChar = unchecked((char)-1); 129 | 130 | public Token Current 131 | { 132 | get 133 | { 134 | return _current; 135 | } 136 | } 137 | 138 | public bool IsEmpty 139 | { 140 | get 141 | { 142 | return _text == null; 143 | } 144 | } 145 | 146 | public void Reset() 147 | { 148 | int readerPos = _position - _textPos; 149 | _current = new Token(TokenType.Start, null, null, CommonLexem.Start, 0, 0, 0, 0, 0, 0); 150 | _next = null; 151 | _textPos = 0; 152 | _position = 0; 153 | _textBeg = 0; 154 | _tokenBuffer = null; 155 | _buffer = null; 156 | _bufBeg = -1; 157 | 158 | if (_reader != null) 159 | { 160 | if (_text != null && readerPos > 0) 161 | { 162 | StreamReader streamReader = _reader as StreamReader; 163 | if (streamReader != null && streamReader.BaseStream.CanSeek) 164 | { 165 | streamReader.BaseStream.Seek(0, SeekOrigin.Begin); 166 | _text = null; 167 | } 168 | } 169 | 170 | if (_text == null) 171 | { 172 | _textLen = 0; 173 | ReadCharBuffer(); 174 | } 175 | } 176 | else 177 | { 178 | _textLen = (_text == null ? 0 : _text.Length); 179 | } 180 | } 181 | 182 | public Token GetNextToken(LexerBehavior behavior) 183 | { 184 | LexerBehavior saveBehavior = _behavior; 185 | _behavior = behavior; 186 | try 187 | { 188 | return GetNextToken(); 189 | } 190 | finally 191 | { 192 | _behavior = saveBehavior; 193 | } 194 | } 195 | 196 | public Token GetNextToken() 197 | { 198 | if (_next != null) 199 | { 200 | _current = _next; 201 | _next = null; 202 | } 203 | else 204 | { 205 | _current = GetToken(); 206 | } 207 | 208 | return _current; 209 | } 210 | 211 | public Token PeekNextToken(LexerBehavior behavior) 212 | { 213 | LexerBehavior saveBehavior = _behavior; 214 | _behavior = behavior; 215 | try 216 | { 217 | return PeekNextToken(); 218 | } 219 | finally 220 | { 221 | _behavior = saveBehavior; 222 | } 223 | } 224 | 225 | public Token PeekNextToken() 226 | { 227 | if (_next == null) 228 | { 229 | _next = GetToken(); 230 | } 231 | 232 | return _next; 233 | } 234 | 235 | #region Private Implementation 236 | 237 | private Token GetToken() 238 | { 239 | if (_text == null) 240 | { 241 | return new Token(TokenType.End, "", "", CommonLexem.End, 0, 0, 0, 0, 0, 0); 242 | } 243 | 244 | _lineBegin = _endLineBegin; 245 | _lineNumber = _endLineNumber; 246 | _start = _position; 247 | _textBeg = _textPos; 248 | _bufBeg = -1; 249 | _tokenBuffer = null; 250 | _buffer = null; 251 | 252 | char currentChar = PeekChar(); 253 | bool skip; 254 | do 255 | { 256 | skip = false; 257 | // end 258 | if (currentChar == EndOfTextChar && EndOfText()) 259 | { 260 | return GetEndToken(); 261 | } 262 | 263 | // separator 264 | if (currentChar <= ' ') 265 | { 266 | bool skipWhiteSpaces = (_behavior & LexerBehavior.SkipWhiteSpaces) != 0; 267 | do 268 | { 269 | ReadNext(); 270 | if (skipWhiteSpaces) 271 | { 272 | _textBeg = _textPos; 273 | } 274 | 275 | if (EndOfLine(currentChar)) 276 | { 277 | if (skipWhiteSpaces) 278 | { 279 | _textBeg = _textPos; 280 | } 281 | else if ((_settings.Options & LexerOptions.EndOfLineAsToken) != 0) 282 | { 283 | return new Token(TokenType.EndOfLine, "", GetTokenText(), 0, _start, _position, _lineBegin, _lineNumber, _endLineBegin, _endLineNumber); 284 | } 285 | } 286 | 287 | currentChar = PeekChar(); 288 | if (currentChar == EndOfTextChar && EndOfText()) 289 | { 290 | break; 291 | } 292 | 293 | } while (currentChar <= ' '); 294 | 295 | if (!skipWhiteSpaces) 296 | { 297 | return new Token(TokenType.WhiteSpace, "", GetTokenText(), 0, _start, _position, _lineBegin, _lineNumber, _endLineBegin, _endLineNumber); 298 | } 299 | 300 | _textBeg = _textPos; 301 | skip = true; 302 | _start = _position; 303 | } 304 | 305 | // inline comment 306 | string[] inlineComments = _settings.InlineComments; 307 | if (inlineComments != null) 308 | { 309 | for (int inlineCommentIndex = 0; inlineCommentIndex < inlineComments.Length; inlineCommentIndex++) 310 | { 311 | string inlineComment = inlineComments[inlineCommentIndex]; 312 | if (NextSymbolIs(inlineComment)) 313 | { 314 | bool skipComments = ((_behavior & LexerBehavior.SkipComments) != 0); 315 | skip = true; 316 | if (skipComments) 317 | { 318 | _textBeg = _textPos; 319 | } 320 | 321 | currentChar = PeekChar(); 322 | while (true) 323 | { 324 | if (currentChar == '\r' || currentChar == '\n') 325 | { 326 | break; 327 | } 328 | 329 | currentChar = NextChar(); 330 | if (currentChar == EndOfTextChar && EndOfText()) 331 | { 332 | break; 333 | } 334 | 335 | if (skipComments) 336 | { 337 | _textBeg = _textPos; 338 | } 339 | } 340 | 341 | if (skipComments) 342 | { 343 | _start = _position; 344 | } 345 | else 346 | { 347 | return new Token(TokenType.Comment, "", GetTokenText(), 0, _start, _position, _lineBegin, _lineNumber, _lineBegin, _lineNumber); 348 | } 349 | 350 | break; 351 | } 352 | } 353 | } 354 | 355 | // comment 356 | if (!string.IsNullOrEmpty(_settings.CommentBegin) && NextSymbolIs(_settings.CommentBegin)) 357 | { 358 | bool skipComments = ((_behavior & LexerBehavior.SkipComments) != 0); 359 | skip = true; 360 | if (skipComments) 361 | { 362 | _textBeg = _textPos; 363 | } 364 | 365 | while (true) 366 | { 367 | if (NextSymbolIs(_settings.CommentEnd)) 368 | { 369 | currentChar = PeekChar(); 370 | if (skipComments) 371 | { 372 | _textBeg = _textPos; 373 | } 374 | 375 | break; 376 | } 377 | 378 | currentChar = NextChar(); 379 | if (currentChar == EndOfTextChar && EndOfText()) 380 | { 381 | break; 382 | } 383 | else 384 | { 385 | EndOfLine(currentChar); 386 | } 387 | 388 | if (skipComments) 389 | { 390 | _textBeg = _textPos; 391 | } 392 | } 393 | 394 | if (skipComments) 395 | { 396 | _start = _position; 397 | } 398 | else 399 | { 400 | return new Token(TokenType.Comment, "", GetTokenText(), 0, _start, _position, _lineBegin, _lineNumber, _endLineBegin, _endLineNumber); 401 | } 402 | } 403 | 404 | _lineNumber = _endLineNumber; 405 | _lineBegin = _endLineBegin; 406 | 407 | } while (skip); 408 | 409 | // quoted string 410 | char[] stringQuotes = _settings.StringQuotes; 411 | if (stringQuotes != null) 412 | { 413 | for (int i = 0; i < stringQuotes.Length; i++) 414 | { 415 | char stringQuoteChar = stringQuotes[i]; 416 | if (currentChar == stringQuoteChar || i == 0 && currentChar == _settings.StringPrefix && PeekChar(1) == stringQuoteChar) 417 | { 418 | return GetQuotedStringToken(currentChar != stringQuoteChar, stringQuoteChar); 419 | } 420 | } 421 | } 422 | 423 | // quoted identifier 424 | bool isIdentQuote = currentChar == _settings.IdentQuote; 425 | bool quote = isIdentQuote || currentChar == _settings.IdentQuoteBegin; 426 | char nextChar; 427 | if (quote || currentChar == _settings.IdentPrefix && (isIdentQuote = (nextChar = PeekChar(1)) == _settings.IdentQuote || nextChar == _settings.IdentQuoteBegin)) 428 | { 429 | return GetQuotedIdentifierToken(!quote, isIdentQuote); 430 | } 431 | 432 | // prefix identifier 433 | if (currentChar == _settings.IdentPrefix) 434 | { 435 | return GetPrefixedIdentifierToken(); 436 | } 437 | 438 | // number 439 | if (currentChar >= '0' && currentChar <= '9') 440 | { 441 | return GetNumberToken(currentChar); 442 | } 443 | 444 | // keyword / identifier 445 | if (Char.IsLetter(currentChar) || currentChar == '_' || IsIdentChar(currentChar)) 446 | { 447 | return GetKeywordOrIdentifierToken(currentChar); 448 | } 449 | 450 | // predefined symbol 451 | if (_settings.Symbols != null) 452 | { 453 | string symbol = PeekSubstring(_maxSymLen); 454 | for (int i = symbol.Length; i > 0; i--, symbol = symbol.Substring(0, i)) 455 | { 456 | int symbolId; 457 | if (_settings.Symbols.TryGetValue(symbol, out symbolId)) 458 | { 459 | Skip(i); 460 | string symbolText = (_behavior & LexerBehavior.PersistTokenText) != 0 ? symbol : null; 461 | return new Token(TokenType.Symbol, symbol, symbolText, (int)symbolId, _start, _position, _lineBegin, _lineNumber, _lineBegin, _lineNumber); 462 | } 463 | } 464 | } 465 | 466 | // just a char 467 | currentChar = NextChar(); 468 | string charText = (_behavior & LexerBehavior.PersistTokenText) != 0 ? currentChar.ToString() : null; 469 | return new Token(TokenType.Char, currentChar, charText, 0, _start, _position, _lineBegin, _lineNumber, _lineBegin, _lineNumber); 470 | } 471 | 472 | private Token GetEndToken() 473 | { 474 | if (_reader != null) 475 | { 476 | _reader.Close(); 477 | } 478 | 479 | return new Token(TokenType.End, "", "", CommonLexem.End, _start, _start, _lineBegin, _lineNumber, _lineBegin, _lineNumber); 480 | } 481 | 482 | private Token GetQuotedIdentifierToken(bool prefix, bool isIdentQuote) 483 | { 484 | if (prefix) 485 | { 486 | ReadNext(); 487 | } 488 | 489 | char quoteEnd; 490 | bool doubleQuote; 491 | if (isIdentQuote) 492 | { 493 | quoteEnd = _settings.IdentQuote; 494 | doubleQuote = (_settings.Options & LexerOptions.IdentDoubleQuote) != 0; 495 | } 496 | else 497 | { 498 | quoteEnd = _settings.IdentQuoteEnd; 499 | doubleQuote = false; 500 | } 501 | 502 | ReadNext(); 503 | _bufBeg = _textPos; 504 | 505 | while (true) 506 | { 507 | char currentChar = NextChar(); 508 | BufferAdd(currentChar); 509 | 510 | if (currentChar == quoteEnd) 511 | { 512 | if (doubleQuote && PeekChar() == quoteEnd) 513 | { 514 | EnsureBuffer(1); 515 | currentChar = NextChar(); 516 | BufferAdd(currentChar); 517 | } 518 | else 519 | { 520 | break; 521 | } 522 | } 523 | 524 | if (currentChar == EndOfTextChar && EndOfText()) 525 | { 526 | break; 527 | } 528 | else 529 | { 530 | EndOfLine(currentChar); 531 | } 532 | } 533 | 534 | string val = GetBufferValue(-1); 535 | return new Token(TokenType.Identifier, val, GetTokenText(), 0, _start, _position, _lineBegin, _lineNumber, _endLineBegin, _endLineNumber); 536 | } 537 | 538 | private Token GetQuotedStringToken(bool prefix, char stringQuoteChar) 539 | { 540 | char escapeChar; 541 | bool escaping; 542 | bool doubleQuote; 543 | if (prefix) 544 | { 545 | escapeChar = '\0'; 546 | escaping = false; 547 | doubleQuote = true; 548 | ReadNext(); 549 | } 550 | else 551 | { 552 | escapeChar = _settings.StringEscapeChar; 553 | escaping = (_settings.Options & LexerOptions.StringEscaping) != 0; 554 | doubleQuote = (_settings.Options & LexerOptions.StringDoubleQuote) != 0; 555 | } 556 | 557 | ReadNext(); 558 | _bufBeg = _textPos; 559 | 560 | while (true) 561 | { 562 | char currentChar = NextChar(); 563 | BufferAdd(currentChar); 564 | 565 | if (currentChar == escapeChar && escaping) 566 | { 567 | EnsureBuffer(1); 568 | currentChar = NextChar(); 569 | BufferAdd(currentChar); 570 | } 571 | else if (currentChar == stringQuoteChar) 572 | { 573 | if (doubleQuote && PeekChar() == stringQuoteChar) 574 | { 575 | EnsureBuffer(1); 576 | currentChar = NextChar(); 577 | BufferAdd(currentChar); 578 | } 579 | else 580 | { 581 | break; 582 | } 583 | } 584 | else if (currentChar == EndOfTextChar && EndOfText()) 585 | { 586 | break; 587 | } 588 | else 589 | { 590 | EndOfLine(currentChar); 591 | } 592 | } 593 | 594 | string val = GetBufferValue(-1); 595 | return new Token(TokenType.QuotedString, val, GetTokenText(), 0, _start, _position, _lineBegin, _lineNumber, _endLineBegin, _endLineNumber); 596 | } 597 | 598 | private Token GetKeywordOrIdentifierToken(char currentChar) 599 | { 600 | _bufBeg = _textPos; 601 | do 602 | { 603 | ReadNext(); 604 | BufferAdd(currentChar); 605 | currentChar = PeekChar(); 606 | } while (Char.IsLetterOrDigit(currentChar) || currentChar == '_' || IsIdentChar(currentChar)); 607 | 608 | string val = GetBufferValue(0); 609 | 610 | int id = 0; 611 | TokenType tokenType = TokenType.Identifier; 612 | if ((_settings.Options & LexerOptions.IdentToUpper) != 0) 613 | { 614 | val = val.ToUpper(_settings.CultureInfo); 615 | if (_settings.Keywords != null && _settings.Keywords.TryGetValue(val, out id)) 616 | { 617 | tokenType = TokenType.Keyword; 618 | } 619 | } 620 | else 621 | { 622 | if (_settings.Keywords != null && _settings.Keywords.TryGetValue(val.ToUpper(_settings.CultureInfo), out id)) 623 | { 624 | tokenType = TokenType.Keyword; 625 | } 626 | 627 | if ((_settings.Options & LexerOptions.IdentToLower) != 0) 628 | { 629 | val = val.ToLower(); 630 | } 631 | } 632 | 633 | return new Token(tokenType, val, GetTokenText(), (int)id, _start, _position, _lineBegin, _lineNumber, _lineBegin, _lineNumber); 634 | } 635 | 636 | private Token GetNumberToken(char currentChar) 637 | { 638 | _bufBeg = _textPos; 639 | do 640 | { 641 | ReadNext(); 642 | BufferAdd(currentChar); 643 | currentChar = PeekChar(); 644 | } 645 | while (currentChar >= '0' && currentChar <= '9'); 646 | 647 | string decimalSeparator = _settings.DecimalSeparator; 648 | if (SymbolIs(decimalSeparator)) 649 | { 650 | int ln = decimalSeparator.Length; 651 | char ch = PeekChar(ln); 652 | if (ch >= '0' && ch <= '9') 653 | { 654 | Skip(ln); 655 | BufferAdd(decimalSeparator); 656 | currentChar = ch; 657 | do 658 | { 659 | ReadNext(); 660 | BufferAdd(currentChar); 661 | currentChar = PeekChar(); 662 | } while (currentChar >= '0' && currentChar <= '9'); 663 | } 664 | } 665 | 666 | if (char.IsLetter(currentChar)) 667 | { 668 | do 669 | { 670 | ReadNext(); 671 | BufferAdd(currentChar); 672 | currentChar = PeekChar(); 673 | 674 | } while ((currentChar >= '0' && currentChar <= '9') || currentChar == '-' || currentChar == '+' || Char.IsLetter(currentChar)); 675 | 676 | string val = GetBufferValue(0); 677 | return new Token(TokenType.Number, val, GetTokenText(), 0, _start, _position, _lineBegin, _lineNumber, _lineBegin, _lineNumber); 678 | } 679 | else 680 | { 681 | string val = GetBufferValue(0); 682 | decimal decimalVal; 683 | long intVal; 684 | if (long.TryParse(val, out intVal)) 685 | { 686 | return new Token(TokenType.Integer, intVal, GetTokenText(), 0, _start, _position, _lineBegin, _lineNumber, _lineBegin, _lineNumber); 687 | } 688 | else if (decimal.TryParse(val, out decimalVal)) 689 | { 690 | return new Token(TokenType.Decimal, decimalVal, GetTokenText(), 0, _start, _position, _lineBegin, _lineNumber, _lineBegin, _lineNumber); 691 | } 692 | else 693 | { 694 | return new Token(TokenType.Number, val, GetTokenText(), 0, _start, _position, _lineBegin, _lineNumber, _lineBegin, _lineNumber); 695 | } 696 | } 697 | } 698 | 699 | private Token GetPrefixedIdentifierToken() 700 | { 701 | ReadNext(); 702 | _bufBeg = _textPos; 703 | 704 | char currentChar = PeekChar(); 705 | if (Char.IsLetterOrDigit(currentChar) || currentChar == '_' || IsIdentChar(currentChar)) 706 | { 707 | do 708 | { 709 | ReadNext(); 710 | BufferAdd(currentChar); 711 | currentChar = PeekChar(); 712 | } 713 | while (Char.IsLetterOrDigit(currentChar) || currentChar == '_' || IsIdentChar(currentChar)); 714 | } 715 | 716 | string val = GetBufferValue(0); 717 | if ((_settings.Options & LexerOptions.IdentToUpper) != 0) 718 | { 719 | val = val.ToUpper(_settings.CultureInfo); 720 | } 721 | else if ((_settings.Options & LexerOptions.IdentToLower) != 0) 722 | { 723 | val = val.ToLower(_settings.CultureInfo); 724 | } 725 | 726 | return new Token(TokenType.Identifier, val, GetTokenText(), 0, _start, _position, _lineBegin, _lineNumber, _lineBegin, _lineNumber); 727 | } 728 | 729 | private bool IsIdentChar(char currentChar) 730 | { 731 | char[] identChars = _settings.IdentChars; 732 | if (identChars != null) 733 | { 734 | int len = identChars.Length; 735 | for (int i = 0; i < len; i++) 736 | { 737 | char ch = identChars[i]; 738 | if (currentChar == ch) 739 | { 740 | return true; 741 | } 742 | } 743 | } 744 | 745 | return false; 746 | } 747 | 748 | private char PeekChar() 749 | { 750 | if (_textPos < _textLen) 751 | { 752 | return _text[_textPos]; 753 | } 754 | 755 | if (_textLen == BufferCapacity) 756 | { 757 | ReadCharBuffer(); 758 | if (_textPos < _textLen) 759 | { 760 | return _text[_textPos]; 761 | } 762 | } 763 | 764 | return EndOfTextChar; 765 | } 766 | 767 | private char PeekChar(int ofs) 768 | { 769 | int i = _textPos + ofs; 770 | if (i < _textLen) 771 | { 772 | return _text[i]; 773 | } 774 | 775 | if (_textLen == BufferCapacity) 776 | { 777 | ReadCharBuffer(); 778 | ofs += _textPos; 779 | if (ofs < _textLen) 780 | { 781 | return _text[ofs]; 782 | } 783 | } 784 | 785 | return EndOfTextChar; 786 | } 787 | 788 | private string PeekSubstring(int count) 789 | { 790 | if (_textPos + count <= _textLen) 791 | { 792 | return _text.Substring(_textPos, count); 793 | } 794 | 795 | if (_textLen == BufferCapacity) 796 | { 797 | ReadCharBuffer(); 798 | } 799 | 800 | int i = _textLen - _textPos; 801 | if (count <= i) 802 | { 803 | return _text.Substring(_textPos, count); 804 | } 805 | else 806 | { 807 | return _text.Substring(_textPos, i); 808 | } 809 | } 810 | 811 | private char NextChar() 812 | { 813 | if (_textPos < _textLen) 814 | { 815 | _position++; 816 | return _text[_textPos++]; 817 | } 818 | 819 | if (_textLen == BufferCapacity) 820 | { 821 | ReadCharBuffer(); 822 | if (_textPos < _textLen) 823 | { 824 | _position++; 825 | return _text[_textPos++]; 826 | } 827 | } 828 | 829 | return EndOfTextChar; 830 | } 831 | 832 | private void ReadNext() 833 | { 834 | if (_textPos < _textLen) 835 | { 836 | _position++; 837 | _textPos++; 838 | } 839 | else 840 | { 841 | if (_textLen == BufferCapacity) 842 | { 843 | ReadCharBuffer(); 844 | _position++; 845 | _textPos++; 846 | } 847 | } 848 | } 849 | 850 | private bool NextSymbolIs(string s) 851 | { 852 | int ln = s.Length; 853 | if (_textLen - _textPos < ln && _textLen == BufferCapacity) 854 | { 855 | ReadCharBuffer(); 856 | } 857 | 858 | if (_textLen - _textPos < ln || _text[_textPos] != s[0]) 859 | { 860 | return false; 861 | } 862 | 863 | if (_settings.CompareInfo.Compare(_text, _textPos, ln, s, 0, ln, CompareOptions.None) == 0) 864 | { 865 | _position += ln; 866 | _textPos += ln; 867 | return true; 868 | } 869 | 870 | return false; 871 | } 872 | 873 | private bool SymbolIs(string s) 874 | { 875 | int ln = s.Length; 876 | if (_textLen - _textPos < ln && _textLen == BufferCapacity) 877 | { 878 | ReadCharBuffer(); 879 | } 880 | 881 | if (_textLen - _textPos < ln || _text[_textPos] != s[0]) 882 | { 883 | return false; 884 | } 885 | 886 | return (_settings.CompareInfo.Compare(_text, _textPos, ln, s, 0, ln, CompareOptions.None) == 0); 887 | } 888 | 889 | private void Skip(int ofs) 890 | { 891 | if (_textLen - _textPos < ofs && _textLen == BufferCapacity) 892 | { 893 | ReadCharBuffer(); 894 | } 895 | 896 | int i = Math.Min(_textLen - _textPos, ofs); 897 | _position += i; 898 | _textPos += i; 899 | } 900 | 901 | private bool EndOfLine(char currentChar) 902 | { 903 | if (currentChar == '\r') 904 | { 905 | _endLineNumber++; 906 | _endLineBegin = _position; 907 | currentChar = PeekChar(); 908 | if (currentChar == '\n') 909 | { 910 | ReadNext(); 911 | BufferAdd(currentChar); 912 | _endLineBegin = _position; 913 | } 914 | 915 | return true; 916 | } 917 | else if (currentChar == '\n') 918 | { 919 | _endLineNumber++; 920 | _endLineBegin = _position; 921 | 922 | return true; 923 | } 924 | 925 | return false; 926 | } 927 | 928 | private bool EndOfText() 929 | { 930 | if (_textPos < _textLen) 931 | { 932 | return false; 933 | } 934 | 935 | if (_textLen == BufferCapacity) 936 | { 937 | ReadCharBuffer(); 938 | return _textPos >= _textLen; 939 | } 940 | 941 | return true; 942 | } 943 | 944 | private void BufferAdd(char currentChar) 945 | { 946 | if (_buffer != null) 947 | { 948 | _buffer.Append(currentChar); 949 | } 950 | else if (_bufBeg >= 0 && _textPos >= _textLen) 951 | { 952 | _buffer = new StringBuilder(_text, _bufBeg, _textPos - _bufBeg, BufferCapacity); 953 | } 954 | } 955 | 956 | private void BufferAdd(string str) 957 | { 958 | if (_buffer != null) 959 | { 960 | _buffer.Append(str); 961 | } 962 | else if (_bufBeg >= 0 && _textPos >= _textLen) 963 | { 964 | _buffer = new StringBuilder(_text, _bufBeg, _textPos - _bufBeg, BufferCapacity); 965 | } 966 | } 967 | 968 | private void EnsureBuffer(int ofs) 969 | { 970 | if (_buffer == null) 971 | { 972 | _buffer = new StringBuilder(_text, _bufBeg, _textPos - _bufBeg - ofs, BufferCapacity); 973 | } 974 | else 975 | { 976 | _buffer.Remove(_buffer.Length - ofs, ofs); 977 | } 978 | } 979 | 980 | private string GetBufferValue(int ofs) 981 | { 982 | if (_buffer != null) 983 | { 984 | return _buffer.ToString(0, _buffer.Length + ofs); 985 | } 986 | else 987 | { 988 | return _text.Substring(_bufBeg, _textPos - _bufBeg + ofs); 989 | } 990 | } 991 | 992 | private void ReadCharBuffer() 993 | { 994 | if (_reader == null) 995 | { 996 | return; 997 | } 998 | 999 | if (_tokenBuffer != null) 1000 | { 1001 | _tokenBuffer.Append(_text, 0, _textPos); 1002 | } 1003 | else if (_textBeg < _textPos && (_behavior & LexerBehavior.PersistTokenText) != 0) 1004 | { 1005 | _tokenBuffer = new StringBuilder(_text, _textBeg, _textPos - _textBeg, BufferCapacity); 1006 | } 1007 | else 1008 | { 1009 | _textBeg = 0; 1010 | } 1011 | 1012 | char[] charBuffer = new char[BufferCapacity]; 1013 | if (_textPos < _textLen) 1014 | { 1015 | if (_textPos == 0) 1016 | { 1017 | throw new ArgumentException("'BufferCapacity' too small."); 1018 | } 1019 | _textLen -= _textPos; 1020 | _text.CopyTo(_textPos, charBuffer, 0, _textLen); 1021 | } 1022 | else 1023 | { 1024 | _textLen = 0; 1025 | } 1026 | 1027 | _textLen += _reader.Read(charBuffer, _textLen, BufferCapacity - _textLen); 1028 | _text = new string(charBuffer, 0, _textLen); 1029 | _textPos = 0; 1030 | } 1031 | 1032 | private string GetTokenText() 1033 | { 1034 | if (_tokenBuffer != null) 1035 | { 1036 | _tokenBuffer.Append(_text, 0, _textPos); 1037 | return _tokenBuffer.ToString(0, _tokenBuffer.Length); 1038 | } 1039 | 1040 | if ((_behavior & LexerBehavior.PersistTokenText) == 0) 1041 | { 1042 | return null; 1043 | } 1044 | else 1045 | { 1046 | return _text.Substring(_textBeg, _textPos - _textBeg); 1047 | } 1048 | } 1049 | 1050 | #endregion 1051 | 1052 | #region IEnumerable Members 1053 | 1054 | IEnumerator IEnumerable.GetEnumerator() 1055 | { 1056 | return this; 1057 | } 1058 | 1059 | #endregion 1060 | 1061 | #region IEnumerable Members 1062 | 1063 | IEnumerator IEnumerable.GetEnumerator() 1064 | { 1065 | return this; 1066 | } 1067 | 1068 | #endregion 1069 | 1070 | #region IEnumerator Members 1071 | 1072 | object IEnumerator.Current 1073 | { 1074 | get 1075 | { 1076 | return _current; 1077 | } 1078 | } 1079 | 1080 | bool IEnumerator.MoveNext() 1081 | { 1082 | return GetNextToken().Type != TokenType.End; 1083 | } 1084 | 1085 | #endregion 1086 | 1087 | #region IDisposable Members 1088 | 1089 | public void Dispose() 1090 | { 1091 | if (_reader != null) 1092 | { 1093 | _reader.Dispose(); 1094 | } 1095 | } 1096 | 1097 | #endregion 1098 | } 1099 | 1100 | public enum TokenType 1101 | { 1102 | Char, 1103 | Symbol, 1104 | Number, 1105 | Decimal, 1106 | Integer, 1107 | Identifier, 1108 | Keyword, 1109 | QuotedString, 1110 | WhiteSpace, 1111 | EndOfLine, 1112 | Comment, 1113 | Start, 1114 | End 1115 | } 1116 | 1117 | [Flags] 1118 | public enum LexerBehavior 1119 | { 1120 | SkipWhiteSpaces = 1, 1121 | SkipComments = 2, 1122 | PersistTokenText = 4, 1123 | Default = PersistTokenText 1124 | } 1125 | 1126 | [Flags] 1127 | public enum LexerOptions 1128 | { 1129 | IdentIgnoreCase = 1, 1130 | IdentToLower = 3, 1131 | IdentToUpper = 5, 1132 | IdentDoubleQuote = 8, 1133 | StringEscaping = 16, 1134 | StringDoubleQuote = 32, 1135 | EndOfLineAsToken = 64 1136 | } 1137 | 1138 | public sealed class Token 1139 | { 1140 | public readonly TokenType Type; 1141 | public readonly object Value; 1142 | public readonly string Text; 1143 | public readonly int Id; 1144 | public readonly int StartPosition; 1145 | public readonly int EndPosition; 1146 | public readonly int LineBegin; 1147 | public readonly int LineNumber; 1148 | public readonly int EndLineBegin; 1149 | public readonly int EndLineNumber; 1150 | 1151 | public Token(TokenType type, object value, string text, int id, int startPosition, int endPosition, int lineBegin, int lineNumber, int endLineBegin, int endLineNumber) 1152 | { 1153 | Type = type; 1154 | Value = value; 1155 | Text = text; 1156 | Id = id; 1157 | StartPosition = startPosition; 1158 | EndPosition = endPosition; 1159 | LineBegin = lineBegin; 1160 | LineNumber = lineNumber; 1161 | EndLineBegin = endLineBegin; 1162 | EndLineNumber = endLineNumber; 1163 | } 1164 | 1165 | public int LinePosition 1166 | { 1167 | get 1168 | { 1169 | return StartPosition - LineBegin; 1170 | } 1171 | } 1172 | 1173 | public int EndLinePosition 1174 | { 1175 | get 1176 | { 1177 | return EndPosition - EndLineBegin; 1178 | } 1179 | } 1180 | } 1181 | 1182 | public sealed class LexerSettings : ICloneable 1183 | { 1184 | public LexerOptions Options { get; set; } 1185 | public IDictionary Symbols { get; set; } 1186 | public IDictionary Keywords { get; set; } 1187 | public CultureInfo CultureInfo { get; set; } 1188 | public CompareInfo CompareInfo { get; set; } 1189 | public char[] StringQuotes { get; set; } 1190 | public char StringEscapeChar { get; set; } 1191 | public char StringPrefix { get; set; } 1192 | public char IdentQuote { get; set; } 1193 | public char IdentQuoteBegin { get; set; } 1194 | public char IdentQuoteEnd { get; set; } 1195 | public char IdentPrefix { get; set; } 1196 | public char[] IdentChars { get; set; } 1197 | public string[] InlineComments { get; set; } 1198 | public string CommentBegin { get; set; } 1199 | public string CommentEnd { get; set; } 1200 | public string DecimalSeparator { get; set; } 1201 | 1202 | public static LexerSettings Default 1203 | { 1204 | get 1205 | { 1206 | LexerSettings settings = new LexerSettings(); 1207 | settings.CultureInfo = CultureInfo.InvariantCulture; 1208 | settings.CompareInfo = CultureInfo.InvariantCulture.CompareInfo; 1209 | settings.DecimalSeparator = "."; 1210 | settings.Options = LexerOptions.IdentIgnoreCase | LexerOptions.StringDoubleQuote; 1211 | settings.StringQuotes = new char[] { '\"', '\'' }; 1212 | settings.InlineComments = new string[] { "//" }; 1213 | settings.CommentBegin = "/*"; 1214 | settings.CommentEnd = "*/"; 1215 | settings.StringEscapeChar = '\\'; 1216 | settings.StringPrefix = '@'; 1217 | settings.IdentQuote = '\0'; 1218 | settings.IdentQuoteBegin = '\0'; 1219 | settings.IdentQuoteEnd = '\0'; 1220 | 1221 | return settings; 1222 | } 1223 | } 1224 | 1225 | #region ICloneable Members 1226 | 1227 | object ICloneable.Clone() 1228 | { 1229 | return Clone(); 1230 | } 1231 | 1232 | public LexerSettings Clone() 1233 | { 1234 | LexerSettings settings = (LexerSettings)MemberwiseClone(); 1235 | 1236 | if (settings.CultureInfo == null) 1237 | { 1238 | settings.CultureInfo = CultureInfo.InvariantCulture; 1239 | } 1240 | 1241 | if (settings.CompareInfo == null) 1242 | { 1243 | settings.CompareInfo = settings.CultureInfo.CompareInfo; 1244 | } 1245 | 1246 | if (string.IsNullOrEmpty(settings.DecimalSeparator)) 1247 | { 1248 | settings.DecimalSeparator = settings.CultureInfo.NumberFormat.NumberDecimalSeparator; 1249 | } 1250 | 1251 | if (settings.Symbols != null && settings.Symbols.Count > 0) 1252 | { 1253 | settings.Symbols = new Dictionary(settings.Symbols); 1254 | } 1255 | else 1256 | { 1257 | settings.Symbols = null; 1258 | } 1259 | 1260 | if (settings.Keywords != null && settings.Keywords.Count > 0) 1261 | { 1262 | bool ignoreCase = (settings.Options & LexerOptions.IdentIgnoreCase) != 0; 1263 | settings.Keywords = new Dictionary(settings.Keywords, StringComparer.Create(settings.CultureInfo, ignoreCase)); 1264 | } 1265 | else 1266 | { 1267 | settings.Keywords = null; 1268 | } 1269 | 1270 | if (settings.StringQuotes != null) 1271 | { 1272 | settings.StringQuotes = (char[])settings.StringQuotes.Clone(); 1273 | } 1274 | 1275 | if (settings.IdentChars != null) 1276 | { 1277 | settings.IdentChars = (char[])settings.IdentChars.Clone(); 1278 | } 1279 | 1280 | string[] inlineComments = settings.InlineComments; 1281 | if (inlineComments != null) 1282 | { 1283 | int length = inlineComments.Length; 1284 | int count = 0; 1285 | for (int i = 0; i < length; i++) 1286 | { 1287 | string inlineComment = inlineComments[i]; 1288 | if (inlineComment == null) 1289 | { 1290 | continue; 1291 | } 1292 | 1293 | if (i != count) 1294 | { 1295 | inlineComments[count] = inlineComment; 1296 | } 1297 | 1298 | count++; 1299 | } 1300 | 1301 | if (count == 0) 1302 | { 1303 | settings.InlineComments = null; 1304 | } 1305 | else 1306 | { 1307 | string[] arr = new string[count]; 1308 | Array.Copy(inlineComments, 0, arr, 0, count); 1309 | } 1310 | } 1311 | 1312 | if (!string.IsNullOrEmpty(settings.CommentBegin) && string.IsNullOrEmpty(settings.CommentEnd)) 1313 | { 1314 | settings.CommentEnd = settings.CommentBegin; 1315 | } 1316 | 1317 | return settings; 1318 | } 1319 | 1320 | #endregion 1321 | } 1322 | 1323 | internal static class CommonLexem 1324 | { 1325 | public const int Start = 1; 1326 | public const int End = 2; 1327 | } 1328 | } -------------------------------------------------------------------------------- /src/Lexer/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | [assembly: AssemblyTitle("Lexer")] 9 | [assembly: AssemblyDescription("")] 10 | [assembly: AssemblyConfiguration("")] 11 | [assembly: AssemblyCompany("")] 12 | [assembly: AssemblyProduct("Lexer")] 13 | [assembly: AssemblyCopyright("Copyright © Lexer 2013")] 14 | [assembly: AssemblyTrademark("")] 15 | [assembly: AssemblyCulture("")] 16 | 17 | // Setting ComVisible to false makes the types in this assembly not visible 18 | // to COM components. If you need to access a type in this assembly from 19 | // COM, set the ComVisible attribute to true on that type. 20 | [assembly: ComVisible(false)] 21 | 22 | // The following GUID is for the ID of the typelib if this project is exposed to COM 23 | [assembly: Guid("656c4fc3-a92f-4cd4-95a5-a5819845e271")] 24 | 25 | // Version information for an assembly consists of the following four values: 26 | // 27 | // Major Version 28 | // Minor Version 29 | // Build Number 30 | // Revision 31 | // 32 | // You can specify all the values or you can default the Build and Revision Numbers 33 | // by using the '*' as shown below: 34 | // [assembly: AssemblyVersion("1.0.*")] 35 | [assembly: AssemblyVersion("1.0.0.0")] 36 | [assembly: AssemblyFileVersion("1.0.0.0")] 37 | --------------------------------------------------------------------------------