├── .github ├── dependabot.yml └── workflows │ └── codeql.yml ├── .gitignore ├── README.md ├── Tokenizer.Tests ├── AssertWriter.cs ├── CandidateTokenListTests.cs ├── ConcatenationTests.cs ├── Extensions │ ├── ObjectExtensionsTests.cs │ └── StringExtensionsTest.cs ├── HintTests.cs ├── ListTests.cs ├── MultilineTests.cs ├── Parsers │ ├── PreTokenParserTests.cs │ └── TokenParserTests.cs ├── Properties │ └── AssemblyInfo.cs ├── SampleTests.cs ├── Samples │ ├── Data │ │ ├── 08.pl.txt │ │ ├── abogado.txt │ │ ├── aloespa.com.ve.txt │ │ ├── amazon.co.jp.txt │ │ ├── available.co.ca.txt │ │ ├── bbc.co.uk.txt │ │ ├── com.txt │ │ ├── facebook.com.txt │ │ ├── google.biz.txt │ │ ├── google.cc.txt │ │ ├── google.co.za.txt │ │ ├── google.eu.org.txt │ │ ├── google.fi.txt │ │ ├── google.tr.txt │ │ ├── google.vg.txt │ │ ├── moscowfood.coop.txt │ │ ├── not.found.vg.txt │ │ └── sil.org.txt │ └── Patterns │ │ ├── whois.cc.txt │ │ ├── whois.co.ca.txt │ │ ├── whois.co.za.txt │ │ ├── whois.coop.txt │ │ ├── whois.eu.org.txt │ │ ├── whois.generic.txt │ │ ├── whois.iana.txt │ │ ├── whois.jprs.jp.txt │ │ ├── whois.nic.br.txt │ │ ├── whois.tr.txt │ │ ├── whois.uk.txt │ │ ├── whois.ve.txt │ │ ├── whois.verisign-grs.com.txt │ │ ├── whois.vg.not.found.txt │ │ └── whois.vg.txt ├── SerilogConfig.cs ├── SplitTests.cs ├── TemplateCollectionTests.cs ├── TemplateTests.cs ├── TokenMatcherTests.cs ├── TokenTests.cs ├── Tokenizer.Tests.csproj ├── TokenizerOptionsTests.cs ├── TokenizerTests.cs ├── Transformers │ ├── BlowsUpTransformer.cs │ ├── RemoveEndTransformerTest.cs │ ├── RemoveEndTransformerTests.cs │ ├── RemoveStartTransformerTest.cs │ ├── RemoveStartTransformerTests.cs │ ├── RemoveTransformerTest.cs │ ├── RemoveTransformerTests.cs │ ├── ReplaceTransformerTests.cs │ ├── SetTransformerTests.cs │ ├── SplitTransformerTest.cs │ ├── SplitTransformerTests.cs │ ├── SubstringAfterLastTransformerTests.cs │ ├── SubstringAfterTransformerTests.cs │ ├── SubstringBeforeLastTransformerTests.cs │ ├── SubstringBeforeTransformerTests.cs │ ├── ToDateTimeTransformerTests.cs │ ├── ToDateTimeUtcTransformerTests.cs │ ├── ToLowerTransformerTests.cs │ ├── ToUpperTransformerTests.cs │ └── TrimTransformerTests.cs ├── Types │ ├── BoolTests.cs │ └── EnumTests.cs ├── Validators │ ├── ContainsValidatorTest.cs │ ├── ContainsValidatorTests.cs │ ├── EndsWithValidatorTest.cs │ ├── EndsWithValidatorTests.cs │ ├── IsDateTimeValidatorTests.cs │ ├── IsDomainNameValidatorTests.cs │ ├── IsEmailValidatorTests.cs │ ├── IsLooseAbsoluteUrlValidatorTests.cs │ ├── IsLooseUrlValidatorTests.cs │ ├── IsNotEmptyValidatorTests.cs │ ├── IsNotValidatorTests.cs │ ├── IsNumericValidatorTests.cs │ ├── IsPhoneNumberValidatorTests.cs │ ├── IsUrlValidatorTests.cs │ ├── MaxLengthValidatorTests.cs │ ├── MinLengthValidatorTests.cs │ └── StartsWithValidatorTests.cs └── WindowsClipboard.cs ├── Tokenizer.sln ├── Tokenizer ├── CandidateTokenList.cs ├── Enumerators │ ├── FileLocation.cs │ ├── PreTokenEnumerator.cs │ └── TokenEnumerator.cs ├── Exceptions │ ├── ParsingException.cs │ ├── TokenAssignmentException.cs │ ├── TokenMatcherException.cs │ ├── TokenizerException.cs │ ├── TypeConversionException.cs │ └── ValidationException.cs ├── Extensions │ ├── ObjectExtensions.cs │ └── StringExtensions.cs ├── Hint.cs ├── HintMatch.cs ├── HintResult.cs ├── ITokenDecorator.cs ├── LICENSE.txt ├── Logging │ ├── LogExtensions.cs │ └── LogIndentation.cs ├── Match.cs ├── Parsers │ ├── PreTokenParser.cs │ └── TokenParser.cs ├── PreTemplate.cs ├── PreToken.cs ├── PreTokenDecorator.cs ├── Properties │ └── AssemblyInfo.cs ├── Template.cs ├── TemplateCollection.cs ├── Token.cs ├── TokenDecoratorContext.cs ├── TokenMatcher.cs ├── TokenMatcherResult.cs ├── TokenResult.cs ├── TokenizeResult.cs ├── TokenizeResultBase.cs ├── Tokenizer.cs ├── Tokenizer.csproj ├── TokenizerOptions.cs ├── Transformers │ ├── ITokenTransformer.cs │ ├── RemoveEndTransformer.cs │ ├── RemoveStartTransformer.cs │ ├── RemoveTransformer.cs │ ├── ReplaceTransformer.cs │ ├── SetTransformer.cs │ ├── SplitTransformer.cs │ ├── SubstringAfterLastTransformer.cs │ ├── SubstringAfterTransformer.cs │ ├── SubstringBeforeLastTransformer.cs │ ├── SubstringBeforeTransformer.cs │ ├── ToDateTimeTransformer.cs │ ├── ToDateTimeUtcTransformer.cs │ ├── ToLowerTransformer.cs │ ├── ToUpperTransformer.cs │ └── TrimTransformer.cs └── Validators │ ├── ContainsValidator.cs │ ├── EndsWithValidator.cs │ ├── ITokenValidator.cs │ ├── IsDateTimeValidator.cs │ ├── IsDomainNameValidator.cs │ ├── IsEmailValidator.cs │ ├── IsLooseAbsoluteUrlValidator.cs │ ├── IsLooseUrlValidator.cs │ ├── IsNotEmptyValidator.cs │ ├── IsNotValidator.cs │ ├── IsNumericValidator.cs │ ├── IsPhoneNumberValidator.cs │ ├── IsUrlValidator.cs │ ├── MaxLengthValidator.cs │ ├── MinLengthValidator.cs │ └── StartsWithValidator.cs └── appveyor.yml /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: nuget 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "04:00" 8 | open-pull-requests-limit: 10 9 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: [ "master" ] 6 | pull_request: 7 | branches: [ "master" ] 8 | schedule: 9 | - cron: "34 21 * * 5" 10 | 11 | jobs: 12 | analyze: 13 | name: Analyze 14 | runs-on: ubuntu-latest 15 | permissions: 16 | actions: read 17 | contents: read 18 | security-events: write 19 | 20 | strategy: 21 | fail-fast: false 22 | matrix: 23 | language: [ csharp ] 24 | 25 | steps: 26 | - name: Checkout 27 | uses: actions/checkout@v3 28 | 29 | - name: Initialize CodeQL 30 | uses: github/codeql-action/init@v2 31 | with: 32 | languages: ${{ matrix.language }} 33 | queries: +security-and-quality 34 | 35 | - name: Autobuild 36 | uses: github/codeql-action/autobuild@v2 37 | 38 | - name: Perform CodeQL Analysis 39 | uses: github/codeql-action/analyze@v2 40 | with: 41 | category: "/language:${{ matrix.language }}" 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | #ignore thumbnails created by windows 3 | Thumbs.db 4 | #Ignore files build by Visual Studio 5 | *.obj 6 | *.exe 7 | *.pdb 8 | *.user 9 | *.aps 10 | *.pch 11 | *.vspscc 12 | *_i.c 13 | *_p.c 14 | *.ncb 15 | *.suo 16 | *.tlb 17 | *.tlh 18 | *.bak 19 | *.cache 20 | *.ilk 21 | *.log 22 | [Bb]in 23 | [Dd]ebug*/ 24 | *.lib 25 | *.sbr 26 | obj/ 27 | [Rr]elease*/ 28 | _ReSharper*/ 29 | [Tt]est[Rr]esult* 30 | Working 31 | Build/build.txt 32 | artifacts/ 33 | packages 34 | !packages/repositories.config 35 | .vs/* -------------------------------------------------------------------------------- /Tokenizer.Tests/AssertWriter.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | 6 | namespace Tokens 7 | { 8 | /// 9 | /// Helper class to write out the expected assertions for a unit test 10 | /// 11 | internal class AssertWriter 12 | { 13 | private static StringBuilder sb = new StringBuilder(); 14 | 15 | public static void Write(TokenizeResult result) 16 | { 17 | sb.Clear(); 18 | 19 | var listNames = new List(); 20 | 21 | foreach (var match in result.Matches) 22 | { 23 | var name = match.Token.Name; 24 | 25 | if (result.Matches.Count(m => m.Token.Name == name) > 1) 26 | { 27 | if (listNames.Contains(name)) continue; 28 | 29 | var listMatches = result.All(name); 30 | var listCount = 0; 31 | 32 | sb.AppendLine(); 33 | WriteValue($@"result.All(""{name}"").Count", listMatches.Count); 34 | foreach (var listMatch in listMatches) 35 | { 36 | WriteValue($@"result.All(""{name}"")[{listCount}]", listMatch); 37 | 38 | listCount++; 39 | } 40 | sb.AppendLine(); 41 | 42 | listNames.Add(name); 43 | } 44 | else 45 | { 46 | WriteValue($@"result.First(""{name}"")", match.Value); 47 | } 48 | } 49 | 50 | Console.Write(sb.ToString()); 51 | WindowsClipboard.SetText(sb.ToString()); 52 | } 53 | 54 | private static void WriteValue(string name, object value) 55 | { 56 | 57 | if (value is string) 58 | { 59 | sb.AppendLine($@" Assert.AreEqual(""{value}"", {name});"); 60 | } 61 | 62 | if (value is int) 63 | { 64 | sb.AppendLine($@" Assert.AreEqual({value}, {name});"); 65 | } 66 | 67 | if (value is DateTime dateTime) 68 | { 69 | sb.AppendLine($@" Assert.AreEqual(new DateTime({dateTime.Year}, {dateTime.Month:00}, {dateTime.Day:00}, {dateTime.Hour:00}, {dateTime.Minute:00}, {dateTime.Second:00}, {dateTime.Millisecond:000}, DateTimeKind.Utc), {name});"); 70 | 71 | } 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /Tokenizer.Tests/CandidateTokenListTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | 3 | namespace Tokens 4 | { 5 | [TestFixture] 6 | public class CandidateTokenListTests 7 | { 8 | [Test] 9 | public void TestAddToken() 10 | { 11 | var token = new Token("foo") { Preamble = "bar" }; 12 | 13 | var list = new CandidateTokenList(); 14 | 15 | list.Add(token); 16 | 17 | Assert.AreEqual(1, list.Count); 18 | Assert.AreEqual("bar", list.Preamble); 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /Tokenizer.Tests/ListTests.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using NUnit.Framework; 3 | 4 | namespace Tokens 5 | { 6 | [TestFixture] 7 | public class ListTests 8 | { 9 | private Tokenizer tokenizer; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | SerilogConfig.Init(); 15 | 16 | tokenizer = new Tokenizer(new TokenizerOptions{ EnableLogging = true }); 17 | } 18 | 19 | [Test] 20 | public void TestExtractSingleValue() 21 | { 22 | const string pattern = @"Domains: 23 | { DomainName : Repeating, IsDomainName } 24 | 25 | { SecondaryDomain }"; 26 | const string input = @"Domains: 27 | one.com 28 | two.com 29 | three.com 30 | 31 | secondary.com"; 32 | 33 | var results = tokenizer.Tokenize(pattern, input); 34 | 35 | var domains = results.Matches.Where(m => m.Token.Name == "DomainName").ToList(); 36 | 37 | Assert.AreEqual(3, domains.Count); 38 | Assert.AreEqual("one.com", domains[0].Value); 39 | Assert.AreEqual("two.com", domains[1].Value); 40 | Assert.AreEqual("three.com", domains[2].Value); 41 | 42 | Assert.AreEqual("secondary.com", results.First("SecondaryDomain")); 43 | } 44 | } 45 | } 46 | 47 | -------------------------------------------------------------------------------- /Tokenizer.Tests/MultilineTests.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using NUnit.Framework; 3 | 4 | namespace Tokens 5 | { 6 | [TestFixture] 7 | public class MultilineTests 8 | { 9 | private Tokenizer tokenizer; 10 | 11 | private class Student 12 | { 13 | public string FirstName { get; set; } 14 | 15 | public string LastName { get; set; } 16 | 17 | public List Classes { get; set; } 18 | } 19 | 20 | [SetUp] 21 | public void SetUp() 22 | { 23 | SerilogConfig.Init(); 24 | 25 | tokenizer = new Tokenizer(new TokenizerOptions{ EnableLogging = true }); 26 | } 27 | 28 | [Test] 29 | public void TestMultilineRepeating() 30 | { 31 | const string pattern = @"First Name: 32 | {FirstName $ } 33 | 34 | Classes: 35 | {Classes $ * } 36 | 37 | Last Name: 38 | {LastName $ } 39 | "; 40 | const string input = @"First Name: 41 | Alice 42 | 43 | Classes: 44 | French 45 | History 46 | Maths 47 | 48 | Last Name: 49 | Smith 50 | "; 51 | 52 | var result = tokenizer.Tokenize(pattern, input); 53 | 54 | Assert.AreEqual("Alice", result.Value.FirstName); 55 | Assert.AreEqual(3, result.Value.Classes.Count); 56 | Assert.AreEqual("French", result.Value.Classes[0]); 57 | Assert.AreEqual("History", result.Value.Classes[1]); 58 | Assert.AreEqual("Maths", result.Value.Classes[2]); 59 | Assert.AreEqual("Smith", result.Value.LastName); 60 | } 61 | 62 | [Test] 63 | public void TestMultilineRepeatingIndented() 64 | { 65 | const string pattern = @" Relevant dates: 66 | Registered on: {FirstName} 67 | Expiry date: {LastName} 68 | 69 | Registration status: 70 | Registered until expiry date. 71 | 72 | Name servers: 73 | { Classes $ *} 74 | "; 75 | const string input = @" Relevant dates: 76 | Registered on: Alice 77 | Expiry date: Smith 78 | 79 | Registration status: 80 | Registered until expiry date. 81 | 82 | Name servers: 83 | ns1.rbsov.bbc.co.uk 212.58.241.67 84 | ns1.tcams.bbc.co.uk 212.72.49.3 85 | ns1.thdow.bbc.co.uk 212.58.240.163"; 86 | 87 | var result = tokenizer.Tokenize(pattern, input); 88 | 89 | Assert.AreEqual("Alice", result.Value.FirstName); 90 | Assert.AreEqual(3, result.Value.Classes.Count); 91 | Assert.AreEqual("ns1.rbsov.bbc.co.uk 212.58.241.67", result.Value.Classes[0]); 92 | Assert.AreEqual("ns1.tcams.bbc.co.uk 212.72.49.3", result.Value.Classes[1]); 93 | Assert.AreEqual("ns1.thdow.bbc.co.uk 212.58.240.163", result.Value.Classes[2]); 94 | Assert.AreEqual("Smith", result.Value.LastName); 95 | } 96 | } 97 | } 98 | 99 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Reflection; 2 | using System.Runtime.CompilerServices; 3 | using System.Runtime.InteropServices; 4 | 5 | // General Information about an assembly is controlled through the following 6 | // set of attributes. Change these attribute values to modify the information 7 | // associated with an assembly. 8 | // to COM components. If you need to access a type in this assembly from 9 | // COM, set the ComVisible attribute to true on that type. 10 | [assembly: ComVisible(false)] 11 | 12 | // The following GUID is for the ID of the typelib if this project is exposed to COM 13 | [assembly: Guid("9b90da10-1eae-4cde-a1a0-660985d70686")] 14 | 15 | // Version information for an assembly consists of the following four values: 16 | // 17 | // Major Version 18 | // Minor Version 19 | // Build Number 20 | // Revision 21 | // 22 | // You can specify all the values or you can default the Build and Revision Numbers 23 | // by using the '*' as shown below: 24 | // [assembly: AssemblyVersion("1.0.*")] 25 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Data/08.pl.txt: -------------------------------------------------------------------------------- 1 | DOMAIN NAME: 08.pl 2 | 3 | registrant type: organization 4 | 5 | nameservers: dns111.ovh.net. 6 | 7 | ns111.ovh.net. 8 | 9 | created: 2004.02.07 06:45:12 10 | 11 | last modified: 2019.02.01 18:05:52 12 | 13 | renewal date: 2020.02.06 13:00:00 14 | 15 | option created: 2017.01.20 04:34:55 16 | 17 | option expiration date: 2020.01.20 04:34:55 18 | 19 | dnssec: Signed 20 | 21 | DS: 46726 8 2 22 | F2A6D8AE119F40330E2C562813320D1AC008A14B65A9D8D1DB40D4AE214977FF 23 | 24 | REGISTRAR: 25 | 26 | OVH SAS 27 | 28 | 2 Rue Kellermann 29 | 30 | 59100 Roubaix 31 | 32 | Francja/France 33 | 34 | +48.717500200 35 | 36 | https://www.ovh.pl/abuse/ 37 | 38 | WHOIS database responses and Registrant data available at: https://dns.pl/en/whois 39 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Data/abogado.txt: -------------------------------------------------------------------------------- 1 | % IANA WHOIS server 2 | % for more information on IANA, visit http://www.iana.org 3 | % This query returned 1 object 4 | 5 | domain: ABOGADO 6 | 7 | organisation: Minds + Machines Group Limited 8 | address: Craigmuir Chambers, Road Town Tortola VG 1110 9 | address: Virgin Islands, British 10 | 11 | contact: administrative 12 | name: Admin Contact 13 | organisation: Minds + Machines Ltd 14 | address: 32 Nassau St, Dublin 2 15 | address: Ireland 16 | phone: +1-877-734-4783 17 | e-mail: ops@mmx.co 18 | 19 | contact: technical 20 | name: TLD Registry Services Technical 21 | organisation: Nominet 22 | address: Minerva House, 23 | address: Edmund Halley Road, 24 | address: Oxford Science Park, 25 | address: Oxford, 26 | address: OX4 4DQ 27 | address: United Kingdom 28 | phone: +44.1865332211 29 | e-mail: registrytechnical@nominet.uk 30 | 31 | nserver: DNS1.NIC.ABOGADO 213.248.217.13 2a01:618:401:0:0:0:0:13 32 | nserver: DNS2.NIC.ABOGADO 103.49.81.13 2401:fd80:401:0:0:0:0:13 33 | nserver: DNS3.NIC.ABOGADO 213.248.221.13 2a01:618:405:0:0:0:0:13 34 | nserver: DNS4.NIC.ABOGADO 2401:fd80:405:0:0:0:0:13 43.230.49.13 35 | nserver: DNSA.NIC.ABOGADO 156.154.100.3 2001:502:ad09:0:0:0:0:3 36 | nserver: DNSB.NIC.ABOGADO 156.154.101.3 37 | nserver: DNSC.NIC.ABOGADO 156.154.102.3 38 | nserver: DNSD.NIC.ABOGADO 156.154.103.3 39 | ds-rdata: 55367 8 2 A3065C94F7700D7CAB948BBD4A39845E3881F70E61FFB1D9E71DE1AF566919C6 40 | 41 | whois: whois.nic.abogado 42 | 43 | status: ACTIVE 44 | remarks: Registration information: http://mm-registry.com 45 | 46 | created: 2014-07-10 47 | changed: 2018-06-29 48 | source: IANA 49 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Data/aloespa.com.ve.txt: -------------------------------------------------------------------------------- 1 | Servidor Whois de NIC-Venezuela (.VE) 2 | 3 | Este servidor contiene informacion autoritativa exclusivamente de dominios .VE 4 | Cualquier consulta sobre este servicio, puede hacerla al correo electronico whois@nic.ve 5 | 6 | Titular: 7 | Rafael Perez (aloespa.com.ve-dom) registro@tepuynet.com 8 | Rafael Perez 9 | Caracas 10 | Caracas, D. Federal VE 11 | 12 | 13 | Nombre de Dominio: aloespa.com.ve 14 | 15 | Contacto Administrativo: 16 | Tepuynet (aloespa.com.ve-adm) registro@tepuynet.com 17 | Tepuynet C.A. 18 | Av. Bolivar Norte Torre Banaven, Piso 9 Ofic. 9-9 19 | Valencia, Carabobo VE 20 | 2418246437 (FAX) 2418246437 21 | 22 | Contacto Tecnico: 23 | Tepuynet (aloespa.com.ve-tec) registro@tepuynet.com 24 | Tepuynet C.A. 25 | Av. Bolivar Norte Torre Banaven, Piso 9 Ofic. 9-9 26 | Valencia, Carabobo VE 27 | 2418246437 (FAX) 2418246437 28 | 29 | Contacto de Cobranza: 30 | Tepuynet (aloespa.com.ve-bil) registro@tepuynet.com 31 | Tepuynet C.A. 32 | Av. Bolivar Norte Torre Banaven, Piso 9 Ofic. 9-9 33 | Valencia, Carabobo VE 34 | 2418246437 (FAX) 2418246437 35 | 36 | Fecha de Vencimiento: 2010-11-21 15:21:32 37 | Ultima Actualizacion: 2006-06-08 21:54:41 38 | Fecha de Creacion: 2005-11-21 15:21:32 39 | 40 | Estatus del dominio: SUSPENDIDO 41 | 42 | Servidor(es) de Nombres de Dominio: 43 | 44 | - ns10.tepuyserver.net 45 | - ns9.tepuyserver.net 46 | 47 | NIC-Venezuela - CONATEL 48 | http://www.nic.ve 49 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Data/amazon.co.jp.txt: -------------------------------------------------------------------------------- 1 | [ JPRS database provides information on network administration. Its use is ] 2 | [ restricted to network administration purposes. For further information, ] 3 | [ use 'whois -h whois.jprs.jp help'. To suppress Japanese output, add'/e' ] 4 | [ at the end of command, e.g. 'whois -h whois.jprs.jp xxx/e'. ] 5 | 6 | Domain Information: 7 | a. [Domain Name] AMAZON.CO.JP 8 | g. [Organization] Amazon, Inc. 9 | l. [Organization Type] Foreign Company 10 | m. [Administrative Contact] JC076JP 11 | n. [Technical Contact] IK4644JP 12 | p. [Name Server] ns1.p31.dynect.net 13 | p. [Name Server] ns2.p31.dynect.net 14 | p. [Name Server] pdns1.ultradns.net 15 | p. [Name Server] pdns6.ultradns.co.uk 16 | s. [Signing Key] 17 | [State] Connected (2019/11/30) 18 | [Registered Date] 2002/11/21 19 | [Connected Date] 2002/11/21 20 | [Last Update] 2018/12/01 01:01:57 (JST) 21 | 22 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Data/available.co.ca.txt: -------------------------------------------------------------------------------- 1 | U34JEDZCQ.CO.CA is available. 2 | 3 | This information is collected for information purposes only, and is intended 4 | only for determining the owner of .CO.CA domains. No guarantees will be made 5 | regarding the accuracy of this data. 6 | TERMS OF USE: By submitting this query you are agreeing that under no 7 | circumstances will you use this any of this data: 8 | (1) In any fashion relating to, supporting, or allowing the tranmission of any 9 | form of mass unsolicited contact of any kind. 10 | (2) Any form of high-volume automated or bulk lookups of any kind. 11 | Storage or distribution of any of this information in any volume fashion 12 | We reserve the right to terminate your access at any time. 13 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Data/bbc.co.uk.txt: -------------------------------------------------------------------------------- 1 | 2 | Domain name: 3 | bbc.co.uk 4 | 5 | Registrant: 6 | British Broadcasting Corporation 7 | 8 | Registrant type: 9 | UK Corporation by Royal Charter 10 | 11 | Registrant's address: 12 | British Broadcasting Corporation 13 | Broadcasting House 14 | Portland Place 15 | London 16 | W1A 1AA 17 | United Kingdom 18 | 19 | Data validation: 20 | Registrant contact details validated by Nominet on 12-Jun-2014 21 | 22 | Registrar: 23 | British Broadcasting Corporation [Tag = BBC] 24 | URL: http://www.bbc.co.uk 25 | 26 | Relevant dates: 27 | Registered on: before Aug-1996 28 | Expiry date: 13-Dec-2014 29 | Last updated: 12-Jun-2014 30 | 31 | Registration status: 32 | Registered until expiry date. 33 | 34 | Name servers: 35 | ns1.rbsov.bbc.co.uk 212.58.241.67 36 | ns1.tcams.bbc.co.uk 212.72.49.3 37 | ns1.thdow.bbc.co.uk 212.58.240.163 38 | 39 | WHOIS lookup made at 10:35:59 22-Oct-2014 40 | 41 | -- 42 | This WHOIS information is provided for free by Nominet UK the central registry 43 | for .uk domain names. This information and the .uk WHOIS are: 44 | 45 | Copyright Nominet UK 1996 - 2014. 46 | 47 | You may not access the .uk WHOIS or use any data from it except as permitted 48 | by the terms of use available in full at http://www.nominet.org.uk/whoisterms, 49 | which includes restrictions on: (A) use of the data for advertising, or its 50 | repackaging, recompilation, redistribution or reuse (B) obscuring, removing 51 | or hiding any or all of this notice and (C) exceeding query rate or volume 52 | limits. The data is provided on an 'as-is' basis and may lag behind the 53 | register. Access may be withdrawn or restricted at any time. 54 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Data/com.txt: -------------------------------------------------------------------------------- 1 | % IANA WHOIS server 2 | % for more information on IANA, visit http://www.iana.org 3 | % This query returned 1 object 4 | 5 | domain: com 6 | 7 | organisation: VeriSign Global Registry Services 8 | address: 12061 Bluemont Way 9 | address: Reston Virginia 20190 10 | address: United States 11 | 12 | contact: administrative 13 | name: Registry Customer Service 14 | organisation: VeriSign Global Registry Services 15 | address: 12061 Bluemont Way 16 | address: Reston Virginia 20190 17 | address: United States 18 | phone: +1 703 925-6999 19 | fax-no: +1 703 948 3978 20 | e-mail: info@verisign-grs.com 21 | 22 | contact: technical 23 | name: Registry Customer Service 24 | organisation: VeriSign Global Registry Services 25 | address: 12061 Bluemont Way 26 | address: Reston Virginia 20190 27 | address: United States 28 | phone: +1 703 925-6999 29 | fax-no: +1 703 948 3978 30 | e-mail: info@verisign-grs.com 31 | 32 | nserver: A.GTLD-SERVERS.NET 192.5.6.30 2001:503:a83e:0:0:0:2:30 33 | nserver: B.GTLD-SERVERS.NET 192.33.14.30 2001:503:231d:0:0:0:2:30 34 | nserver: C.GTLD-SERVERS.NET 192.26.92.30 35 | nserver: D.GTLD-SERVERS.NET 192.31.80.30 36 | nserver: E.GTLD-SERVERS.NET 192.12.94.30 37 | nserver: F.GTLD-SERVERS.NET 192.35.51.30 38 | nserver: G.GTLD-SERVERS.NET 192.42.93.30 39 | nserver: H.GTLD-SERVERS.NET 192.54.112.30 40 | nserver: I.GTLD-SERVERS.NET 192.43.172.30 41 | nserver: J.GTLD-SERVERS.NET 192.48.79.30 42 | nserver: K.GTLD-SERVERS.NET 192.52.178.30 43 | nserver: L.GTLD-SERVERS.NET 192.41.162.30 44 | nserver: M.GTLD-SERVERS.NET 192.55.83.30 45 | ds-rdata: 30909 8 2 E2D3C916F6DEEAC73294E8268FB5885044A833FC5459588F4A9184CFC41A5766 46 | 47 | whois: whois.verisign-grs.com 48 | 49 | status: ACTIVE 50 | remarks: Registration information: http://www.verisign-grs.com 51 | 52 | created: 1985-01-01 53 | changed: 2012-02-15 54 | source: IANA 55 | 56 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Data/facebook.com.txt: -------------------------------------------------------------------------------- 1 |  Domain Name: FACEBOOK.COM 2 | Registry Domain ID: 2320948_DOMAIN_COM-VRSN 3 | Registrar WHOIS Server: whois.registrarsafe.com 4 | Registrar URL: http://www.registrarsafe.com 5 | Updated Date: 2018-07-23T18:17:13Z 6 | Creation Date: 1997-03-29T05:00:00Z 7 | Registry Expiry Date: 2028-03-30T04:00:00Z 8 | Registrar: RegistrarSafe, LLC 9 | Registrar IANA ID: 3237 10 | Registrar Abuse Contact Email: abusecomplaints@registrarsafe.com 11 | Registrar Abuse Contact Phone: +1-650-308-7004 12 | Domain Status: clientDeleteProhibited https://icann.org/epp#clientDeleteProhibited 13 | Domain Status: clientTransferProhibited https://icann.org/epp#clientTransferProhibited 14 | Domain Status: clientUpdateProhibited https://icann.org/epp#clientUpdateProhibited 15 | Domain Status: serverDeleteProhibited https://icann.org/epp#serverDeleteProhibited 16 | Domain Status: serverTransferProhibited https://icann.org/epp#serverTransferProhibited 17 | Domain Status: serverUpdateProhibited https://icann.org/epp#serverUpdateProhibited 18 | Name Server: A.NS.FACEBOOK.COM 19 | Name Server: B.NS.FACEBOOK.COM 20 | DNSSEC: unsigned 21 | URL of the ICANN Whois Inaccuracy Complaint Form: https://www.icann.org/wicf/ 22 | >>> Last update of whois database: 2018-10-31T15:28:46Z <<< 23 | 24 | For more information on Whois status codes, please visit https://icann.org/epp 25 | 26 | NOTICE: The expiration date displayed in this record is the date the 27 | registrar's sponsorship of the domain name registration in the registry is 28 | currently set to expire. This date does not necessarily reflect the expiration 29 | date of the domain name registrant's agreement with the sponsoring 30 | registrar. Users may consult the sponsoring registrar's Whois database to 31 | view the registrar's reported date of expiration for this registration. 32 | 33 | TERMS OF USE: You are not authorized to access or query our Whois 34 | database through the use of electronic processes that are high-volume and 35 | automated except as reasonably necessary to register domain names or 36 | modify existing registrations; the Data in VeriSign Global Registry 37 | Services' ("VeriSign") Whois database is provided by VeriSign for 38 | information purposes only, and to assist persons in obtaining information 39 | about or related to a domain name registration record. VeriSign does not 40 | guarantee its accuracy. By submitting a Whois query, you agree to abide 41 | by the following terms of use: You agree that you may use this Data only 42 | for lawful purposes and that under no circumstances will you use this Data 43 | to: (1) allow, enable, or otherwise support the transmission of mass 44 | unsolicited, commercial advertising or solicitations via e-mail, telephone, 45 | or facsimile; or (2) enable high volume, automated, electronic processes 46 | that apply to VeriSign (or its computer systems). The compilation, 47 | repackaging, dissemination or other use of this Data is expressly 48 | prohibited without the prior written consent of VeriSign. You agree not to 49 | use electronic processes that are automated and high-volume to access or 50 | query the Whois database except as reasonably necessary to register 51 | domain names or modify existing registrations. VeriSign reserves the right 52 | to restrict your access to the Whois database in its sole discretion to ensure 53 | operational stability. VeriSign may restrict or terminate your access to the 54 | Whois database for failure to abide by these terms of use. VeriSign 55 | reserves the right to modify these terms at any time. 56 | 57 | The Registry database contains ONLY .COM, .NET, .EDU domains and 58 | Registrars. 59 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Data/google.biz.txt: -------------------------------------------------------------------------------- 1 | Domain Name: google.biz 2 | Registry Domain ID: D2835288-BIZ 3 | Registrar WHOIS Server: 4 | Registrar URL: www.markmonitor.com 5 | Updated Date: 2017-02-22T10:27:42Z 6 | Creation Date: 2002-03-27T16:03:44Z 7 | Registry Expiry Date: 2018-03-26T23:59:59Z 8 | Registrar: MarkMonitor, Inc. 9 | Registrar IANA ID: 292 10 | Registrar Abuse Contact Email: abusecomplaints@markmonitor.com 11 | Registrar Abuse Contact Phone: +1.2083895740 12 | Domain Status: clientTransferProhibited https://icann.org/epp#clientTransferProhibited 13 | Domain Status: clientDeleteProhibited https://icann.org/epp#clientDeleteProhibited 14 | Domain Status: clientUpdateProhibited https://icann.org/epp#clientUpdateProhibited 15 | Registry Registrant ID: C42709140-BIZ 16 | Registrant Name: DNS Admin 17 | Registrant Organization: Google Inc. 18 | Registrant Street: 1600 Amphitheatre Parkway 19 | Registrant Street: 20 | Registrant Street: 21 | Registrant City: Mountain View 22 | Registrant State/Province: CA 23 | Registrant Postal Code: 94043 24 | Registrant Country: US 25 | Registrant Phone: +1.6502530000 26 | Registrant Phone Ext: 27 | Registrant Fax: +1.6502530001 28 | Registrant Fax Ext: 29 | Registrant Email: dns-admin@google.com 30 | Registry Admin ID: C42709140-BIZ 31 | Admin Name: DNS Admin 32 | Admin Organization: Google Inc. 33 | Admin Street: 1600 Amphitheatre Parkway 34 | Admin Street: 35 | Admin Street: 36 | Admin City: Mountain View 37 | Admin State/Province: CA 38 | Admin Postal Code: 94043 39 | Admin Country: US 40 | Admin Phone: +1.6502530000 41 | Admin Phone Ext: 42 | Admin Fax: +1.6502530001 43 | Admin Fax Ext: 44 | Admin Email: dns-admin@google.com 45 | Registry Tech ID: C42709140-BIZ 46 | Tech Name: DNS Admin 47 | Tech Organization: Google Inc. 48 | Tech Street: 1600 Amphitheatre Parkway 49 | Tech Street: 50 | Tech Street: 51 | Tech City: Mountain View 52 | Tech State/Province: CA 53 | Tech Postal Code: 94043 54 | Tech Country: US 55 | Tech Phone: +1.6502530000 56 | Tech Phone Ext: 57 | Tech Fax: +1.6502530001 58 | Tech Fax Ext: 59 | Tech Email: dns-admin@google.com 60 | Name Server: ns1.google.com 61 | Name Server: ns2.google.com 62 | Name Server: ns4.google.com 63 | Name Server: ns3.google.com 64 | DNSSEC: unsigned 65 | URL of the ICANN Whois Inaccuracy Complaint Form: https://www.icann.org/wicf/ 66 | >>> Last update of WHOIS database: 2018-02-15T13:59:36Z <<< 67 | 68 | For more information on Whois status codes, please visit https://icann.org/epp 69 | 70 | NeuStar, Inc., the Registry Operator for .BIZ, has collected this information for the WHOIS database through an ICANN-Accredited Registrar. This information is provided to you for informational purposes only and is designed to assist persons in determining contents of a domain name registration record in the NeuStar registry database. NeuStar makes this information available to you "as is" and does not guarantee its accuracy. By submitting a WHOIS query, you agree that you will use this data only for lawful purposes and that, under no circumstances will you use this data: (1) to allow, enable, or otherwise support the transmission of mass unsolicited, commercial advertising or solicitations via direct mail, electronic mail, or by telephone; (2) in contravention of any applicable data and privacy protection acts; or (3) to enable high volume, automated, electronic processes that apply to the registry (or its systems). Compilation, repackaging, dissemination, or other use of the WHOIS database in its entirety, or of a substantial portion thereof, is not allowed without NeuStar's prior written permission. NeuStar reserves the right to modify or change these conditions at any time without prior or subsequent notification of any kind. By executing this query, in any manner whatsoever, you agree to abide by these terms. 71 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Data/google.cc.txt: -------------------------------------------------------------------------------- 1 |  Domain Name: GOOGLE.CC 2 | Registry Domain ID: 86420657_DOMAIN_CC-VRSN 3 | Registrar WHOIS Server: whois.markmonitor.com 4 | Registrar URL: http://www.markmonitor.com 5 | Updated Date: 2017-05-06T09:28:40Z 6 | Creation Date: 1999-06-07T04:00:00Z 7 | Registry Expiry Date: 2018-06-07T04:00:00Z 8 | Registrar: MARKMONITOR INC. 9 | Registrar IANA ID: 292 10 | Registrar Abuse Contact Email: abusecomplaints@markmonitor.com 11 | Registrar Abuse Contact Phone: +1.2083895740 12 | Domain Status: clientDeleteProhibited https://icann.org/epp#clientDeleteProhibited 13 | Domain Status: clientTransferProhibited https://icann.org/epp#clientTransferProhibited 14 | Domain Status: clientUpdateProhibited https://icann.org/epp#clientUpdateProhibited 15 | Domain Status: serverDeleteProhibited https://icann.org/epp#serverDeleteProhibited 16 | Domain Status: serverTransferProhibited https://icann.org/epp#serverTransferProhibited 17 | Domain Status: serverUpdateProhibited https://icann.org/epp#serverUpdateProhibited 18 | Name Server: NS1.GOOGLE.COM 19 | Name Server: NS2.GOOGLE.COM 20 | Name Server: NS3.GOOGLE.COM 21 | Name Server: NS4.GOOGLE.COM 22 | DNSSEC: unsigned 23 | URL of the ICANN Whois Inaccuracy Complaint Form: https://www.icann.org/wicf/ 24 | >>> Last update of WHOIS database: 2017-08-01T00:30:52Z <<< 25 | 26 | For more information on Whois status codes, please visit https://icann.org/epp 27 | 28 | NOTICE: The expiration date displayed in this record is the date the 29 | registrar's sponsorship of the domain name registration in the registry is 30 | currently set to expire. This date does not necessarily reflect the 31 | expiration date of the domain name registrant's agreement with the 32 | sponsoring registrar. Users may consult the sponsoring registrar's 33 | Whois database to view the registrar's reported date of expiration 34 | for this registration. 35 | 36 | TERMS OF USE: You are not authorized to access or query our Whois 37 | database through the use of electronic processes that are high-volume and 38 | automated except as reasonably necessary to register domain names or 39 | modify existing registrations; the Data in VeriSign's ("VeriSign") Whois 40 | database is provided by VeriSign for information purposes only, and to 41 | assist persons in obtaining information about or related to a domain name 42 | registration record. VeriSign does not guarantee its accuracy. 43 | By submitting a Whois query, you agree to abide by the following terms of 44 | use: You agree that you may use this Data only for lawful purposes and that 45 | under no circumstances will you use this Data to: (1) allow, enable, or 46 | otherwise support the transmission of mass unsolicited, commercial 47 | advertising or solicitations via e-mail, telephone, or facsimile; or 48 | (2) enable high volume, automated, electronic processes that apply to 49 | VeriSign (or its computer systems). The compilation, repackaging, 50 | dissemination or other use of this Data is expressly prohibited without 51 | the prior written consent of VeriSign. You agree not to use electronic 52 | processes that are automated and high-volume to access or query the 53 | Whois database except as reasonably necessary to register domain names 54 | or modify existing registrations. VeriSign reserves the right to restrict 55 | your access to the Whois database in its sole discretion to ensure 56 | operational stability. VeriSign may restrict or terminate your access to the 57 | Whois database for failure to abide by these terms of use. VeriSign 58 | reserves the right to modify these terms at any time. 59 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Data/google.co.za.txt: -------------------------------------------------------------------------------- 1 | Domain Name: google.co.za 2 | Domain ID: dom_1SZMF--1 3 | WHOIS Server: coza-whois12.dns.net.za 4 | 5 | Updated Date: 2016-09-24T16:20:09Z 6 | Creation Date: 2001-06-25T20:37:59Z 7 | Registry Expiry Date: 2017-06-25T20:37:59Z 8 | Sponsoring Registrar: MarkMonitor 9 | Domain Status: ok https://icann.org/epp#ok 10 | 11 | Registrant ID: mmr-2383 12 | Registrant Name: Google Inc. 13 | Registrant Organization: 14 | Registrant Street: 1600 Amphitheatre Parkway 15 | Registrant City: Mountain View 16 | Registrant State/Province: CA 17 | Registrant Postal Code: 94043 18 | Registrant Country: US 19 | Registrant Phone: +1.6502530000 20 | Registrant Phone Ext: 21 | Registrant Fax: +1.6506188571 22 | Registrant Fax Ext: 23 | Registrant Email: dns-admin@google.com 24 | 25 | Admin ID: mmr-2383 26 | Admin Name: Google Inc. 27 | Admin Organization: 28 | Admin Street: 1600 Amphitheatre Parkway 29 | Admin City: Mountain View 30 | Admin State/Province: CA 31 | Admin Postal Code: 94043 32 | Admin Country: US 33 | Admin Phone: +1.6502530000 34 | Admin Phone Ext: 35 | Admin Fax: +1.6506188571 36 | Admin Fax Ext: 37 | Admin Email: dns-admin@google.com 38 | 39 | Billing ID: mmr-2383 40 | Billing Name: Google Inc. 41 | Billing Organization: 42 | Billing Street: 1600 Amphitheatre Parkway 43 | Billing City: Mountain View 44 | Billing State/Province: CA 45 | Billing Postal Code: 94043 46 | Billing Country: US 47 | Billing Phone: +1.6502530000 48 | Billing Phone Ext: 49 | Billing Fax: +1.6506188571 50 | Billing Fax Ext: 51 | Billing Email: dns-admin@google.com 52 | 53 | Tech ID: mmr-2383 54 | Tech Name: Google Inc. 55 | Tech Organization: 56 | Tech Street: 1600 Amphitheatre Parkway 57 | Tech City: Mountain View 58 | Tech State/Province: CA 59 | Tech Postal Code: 94043 60 | Tech Country: US 61 | Tech Phone: +1.6502530000 62 | Tech Phone Ext: 63 | Tech Fax: +1.6506188571 64 | Tech Fax Ext: 65 | Tech Email: dns-admin@google.com 66 | 67 | 68 | Name Server: ns1.google.com 69 | Name Server: ns2.google.com 70 | Name Server: ns3.google.com 71 | Name Server: ns4.google.com 72 | DNSSEC: unsigned 73 | >>> Last update of WHOIS database: 2016-12-07T08:03:59Z <<< 74 | 75 | # WHOIS lookup made at 2016-12-07T08:03:59Z 76 | # -- 77 | # For more information on Whois status codes, please visit https://icann.org/epp 78 | # 79 | # The use of this Whois facility is subject to the following terms and 80 | # conditions. https://registry.net.za/whois_terms 81 | # Copyright (c) ZACR 1995-2016 82 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Data/google.eu.org.txt: -------------------------------------------------------------------------------- 1 | domain: GOOGLE.EU.ORG 2 | address: Mueller Michael 3 | address: Wilhelm-Busch-Str. 35 4 | address: 32108 Bad Salzuflen 5 | address: Germany 6 | changed: 2003-03-28 00:00:00+01:00 7 | tech-c: MM114-FREE 8 | admin-c: MM114-FREE 9 | 10 | person: Mueller Michael 11 | nic-hdl: MM114-FREE 12 | address: Wilhelm-Busch-Str. 35 13 | address: 32108 Bad Salzuflen 14 | address: Germany 15 | phone: +49 005222 94569 16 | e-mail: mm114-d7ea0ef920c90b777acb325103212a7c@handles.eu.org 17 | changed: 2003-09-14 00:00:00+02:00 18 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Data/google.fi.txt: -------------------------------------------------------------------------------- 1 |  2 | domain: google.fi 3 | descr: Google Finland Oy 4 | descr: 09073468 5 | address: Domain Administrator 6 | address: Mannerheimintie 12 B 7 | address: 00100 8 | address: HELSINKI 9 | phone: 35896966890 10 | status: Granted 11 | created: 30.6.2006 12 | modified: 7.6.2013 13 | expires: 4.7.2014 14 | nserver: ns1.google.com [Ok] 15 | nserver: ns2.google.com [Ok] 16 | nserver: ns3.google.com [Ok] 17 | nserver: ns4.google.com [Ok] 18 | dnssec: no 19 | 20 | More information is available at https://domain.fi/ 21 | Copyright (c) Finnish Communications Regulatory Authority 22 | 23 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Data/google.tr.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flipbit/tokenizer/3daf535163793ff7a91d5148f98b259eb546ce7a/Tokenizer.Tests/Samples/Data/google.tr.txt -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Data/google.vg.txt: -------------------------------------------------------------------------------- 1 | ; The data in the WHOIS database of KSregistry GmbH is provided by 2 | ; KSregistry GmbH for information purposes, and to assist persons in 3 | ; obtaining information about or related to domain name registration 4 | ; records. KSregistry GmbH does not guarantee its accuracy. 5 | ; By submitting a WHOIS query, you agree that you will use this data 6 | ; only for lawful purposes and that, under no circumstances, you will 7 | ; use this data to 8 | ; 1) allow, enable, or otherwise support the transmission of mass 9 | ; unsolicited, commercial advertising or solicitations via E-mail 10 | ; (spam); or 11 | ; 2) enable high volume, automated, electronic processes that apply 12 | ; to KSregistry GmbH or its systems. 13 | ; KSregistry GmbH reserves the right to modify these terms. 14 | ; By submitting this query, you agree to abide by this policy. 15 | 16 | domain name: GOOGLE.VG 17 | registrar: MarkMonitor Inc. 18 | url: www.markmonitor.com 19 | status: clientupdateprohibited 20 | status: clienttransferprohibited 21 | created date: 1999-06-05 00:00:00 22 | updated date: 2013-03-01 00:02:14 23 | expiration date: 2013-06-05 00:00:00 24 | 25 | owner-contact: P-GFI26 26 | owner-organization: Google, Inc. 27 | owner-name: Google, Inc. 28 | owner-street: 1600 Amphitheatre Parkway 29 | owner-city: Mountain View 30 | owner-zip: 94043 31 | owner-country: US 32 | owner-phone: +1.6503300100 33 | owner-fax: +1.6506181499 34 | owner-email: dns-admin@google.com 35 | 36 | admin-contact: P-GFI26 37 | admin-organization: Google, Inc. 38 | admin-name: Google, Inc. 39 | admin-street: 1600 Amphitheatre Parkway 40 | admin-city: Mountain View 41 | admin-zip: 94043 42 | admin-country: US 43 | admin-phone: +1.6503300100 44 | admin-fax: +1.6506181499 45 | admin-email: dns-admin@google.com 46 | 47 | tech-contact: P-GFI26 48 | tech-organization: Google, Inc. 49 | tech-name: Google, Inc. 50 | tech-street: 1600 Amphitheatre Parkway 51 | tech-city: Mountain View 52 | tech-zip: 94043 53 | tech-country: US 54 | tech-phone: +1.6503300100 55 | tech-fax: +1.6506181499 56 | tech-email: dns-admin@google.com 57 | 58 | billing-contact: P-UDM24 59 | billing-organization: MarkMonitor 60 | billing-name: UNKNOWN MarkMonitor 61 | billing-street: 391 North Ancestor Place 62 | billing-city: ID 63 | billing-zip: 83704 64 | billing-country: US 65 | billing-phone: +1.2083895740 66 | billing-fax: +1.2083895799 67 | billing-email: ccops@markmonitor.com 68 | 69 | nameserver: ns1.google.com 70 | nameserver: ns2.google.com 71 | nameserver: ns3.google.com 72 | nameserver: ns4.google.com 73 | 74 | ; Please register your domains at; www.markmonitor.com 75 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Data/not.found.vg.txt: -------------------------------------------------------------------------------- 1 | not found... 2 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Data/sil.org.txt: -------------------------------------------------------------------------------- 1 | Domain Name: SIL.ORG 2 | Registry Domain ID: D154969-LROR 3 | Registrar WHOIS Server: whois.enom.com 4 | Registrar URL: http://www.enom.com 5 | Updated Date: 2018-03-06T00:17:46Z 6 | Creation Date: 1991-04-15T04:00:00Z 7 | Registry Expiry Date: 2020-04-16T04:00:00Z 8 | Registrar Registration Expiration Date: 9 | Registrar: eNom, Inc. 10 | Registrar IANA ID: 48 11 | Registrar Abuse Contact Email: abuse@enom.com 12 | Registrar Abuse Contact Phone: +1.4252982646 13 | Reseller: 14 | Domain Status: clientTransferProhibited https://icann.org/epp#clientTransferProhibited 15 | Registrant Organization: Summer Institute of Linguistics 16 | Registrant State/Province: TX 17 | Registrant Country: US 18 | Name Server: NSJ1.WSFO.ORG 19 | Name Server: NSC1.WSFO.ORG 20 | Name Server: NSD1.WSFO.ORG 21 | DNSSEC: unsigned 22 | URL of the ICANN Whois Inaccuracy Complaint Form https://www.icann.org/wicf/) 23 | >>> Last update of WHOIS database: 2019-05-23T10:51:13Z <<< 24 | 25 | For more information on Whois status codes, please visit https://icann.org/epp 26 | 27 | Access to Public Interest Registry WHOIS information is provided to assist persons in determining the contents of a domain name registration record in the Public Interest Registry registry database. The data in this record is provided by Public Interest Registry for informational purposes only, and Public Interest Registry does not guarantee its accuracy. This service is intended only for query-based access. You agree that you will use this data only for lawful purposes and that, under no circumstances will you use this data to (a) allow, enable, or otherwise support the transmission by e-mail, telephone, or facsimile of mass unsolicited, commercial advertising or solicitations to entities other than the data recipient's own existing customers; or (b) enable high volume, automated, electronic processes that send queries or data to the systems of Registry Operator, a Registrar, or Afilias except as reasonably necessary to register domain names or modify existing registrations. All rights reserved. Public Interest Registry reserves the right to modify these terms at any time. By submitting this query, you agree to abide by this policy. 28 | 29 | The Registrar of Record identified in this output may have an RDDS service that can be queried for additional information on how to contact the Registrant, Admin, or Tech contact of the queried domain name. 30 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Patterns/whois.cc.txt: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # .cc Parsing Template 4 | # 5 | name: ccwhois.verisign-grs.com/cc/FoundRegistered 6 | 7 | # Use this template for queries to ccwhois.verisign-grs.com: 8 | tag: ccwhois.verisign-grs.com 9 | tag: cc 10 | 11 | # Set query response type: 12 | set: Status = Found 13 | 14 | # Response must contain: 15 | hint: Domain Name 16 | --- 17 | Domain Name: { DomainName : IsDomainName, ToLower, EOL } 18 | Registry Domain ID: { RegistryDomainId ? : EOL } 19 | Registrar WHOIS Server: { Registrar.WhoisServerUrl ? : IsDomainName, ToLower, EOL } 20 | Registrar URL: { Registrar.Url ? : ToLower, EOL } 21 | Updated Date: { Updated ? : ToDateTime("yyyy-MM-ddTHH:mm:ssZ"), EOL } 22 | Creation Date: { Registered ? : ToDateTime("yyyy-MM-ddTHH:mm:ssZ"), EOL } 23 | Registry Expiry Date: { Expiration ? : ToDateTime("yyyy-MM-ddTHH:mm:ssZ"), EOL } 24 | Registrar: { Registrar.Name ? : EOL }MARKMONITOR INC. 25 | Registrar IANA ID: { Registrar.IanaId ? : EOL } 26 | Registrar Abuse Contact Email: { Registrar.AbuseEmail ? : IsEmail, EOL } 27 | Registrar Abuse Contact Phone: { Registrar.AbuseTelephoneNumber ? : IsPhoneNumber, EOL } 28 | Domain Status: { DomainStatus ? : Repeating, SubstringBefore(' '), EOL } 29 | Name Server: { NameServers ? : Repeating, EOL, ToLower, IsDomainName } 30 | DNSSEC: { DnsSecStatus ? : EOL } 31 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Patterns/whois.co.ca.txt: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # .co.ca Parsing Template 4 | # 5 | name: whois.co.ca/co.ca/NotFound 6 | 7 | # Use this template for queries to whois.co.ca: 8 | tag: whois.co.ca 9 | tag: co.ca 10 | 11 | # Set query response type: 12 | set: Status = NotFound 13 | 14 | hint: is available 15 | --- 16 | { DomainName ? : IsDomainName, ToLower } is available.{ Null } 17 | 18 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Patterns/whois.coop.txt: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # .coop Parsing Template 4 | # 5 | name: whois.nic.coop/coop/Found 6 | 7 | # Use this template for queries to whois.nic.coop: 8 | tag: whois.nic.coop 9 | tag: coop 10 | 11 | # End tokens on new lines 12 | terminateOnNewLine: true 13 | 14 | # Use a FixUp to assign fields 15 | IgnoreMissingProperties: true 16 | 17 | # Extract contact information 18 | tag: fixup-contact 19 | 20 | # Set query response type: 21 | set: Status = Found 22 | --- 23 | Domain ID: { RegistryDomainId ? } 24 | Domain Name: { DomainName : IsDomainName, ToLower } 25 | Expiry Date: { Expiration ? : ToDateTimeUtc("dd MMM yyyy HH:mm:ss") } 26 | Domain Status: { DomainStatus ? : Repeating} 27 | Sponsoring registrar: { Registrar.Name ? } 28 | Sponsoring registrar ID: { Registrar.IanaId ? : IsNumeric } 29 | Created: { Registered ? : ToDateTimeUtc("dd MMM yyyy HH:mm:ss") } 30 | Last updated: { Updated ? : ToDateTimeUtc("dd MMMyyyy HH:mm:ss") } 31 | 32 | Contact Type: { Type ? : Repeating } 33 | Contact ID: { Contact.Id ? : Repeating } 34 | Name: { Contact.Name ? : Repeating } 35 | Organisation: { Contact.Organization ? : Repeating } 36 | Street 1: { Address ? : IsNotEmpty, Repeating } 37 | Street 2: { Address ? : IsNotEmpty, Repeating } 38 | Street 3: { Address ? : IsNotEmpty, Repeating } 39 | City: { Address ? : IsNotEmpty, Repeating } 40 | State/Province: { Address ? : IsNotEmpty, Repeating } 41 | Postal code: { Address ? : IsNotEmpty, Repeating } 42 | Country: { Address ? : IsNotEmpty, Repeating } 43 | Voice: { Phone ? : IsPhoneNumber, Repeating } 44 | Voice extn: { PhoneExt ? : IsNumeric, Repeating } 45 | Fax: { Fax ? : IsPhoneNumber, Repeating } 46 | Fax extn: { FaxExt ? : IsNumeric, Repeating } 47 | Email: { Email ? : IsEmail, Repeating } 48 | 49 | Host Name: { NameServers ? : IsDomainName, ToLower, Repeating } 50 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Patterns/whois.eu.org.txt: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # .eu.org Parsing Template 4 | # 5 | name: generic/tld/Found03 6 | 7 | # Use this template for catch-all parsing:: 8 | tag: catch-all 9 | 10 | # Lines can appear out of order 11 | outOfOrder: true 12 | 13 | # End tokens on new lines 14 | terminateOnNewLine: true 15 | 16 | # Use token preamble on current line only 17 | trimPreambleBeforeNewLine: true 18 | 19 | IgnoreMissingProperties: true 20 | 21 | # Set query response type: 22 | set: Status = Found 23 | --- 24 | domain: { DomainName : IsDomainName, ToLower } 25 | address: { Address : IsNotEmpty, Repeating } 26 | changed: { Changed ? : ToDateTimeUtc("yyyy-MM-dd HH:mm:ssZ") } 27 | tech-c: { TechnicalContact.RegistryId ? } 28 | admin-c: { AdminContact.RegistryId ? } 29 | 30 | person: { Contact.Name } 31 | nic-hdl: { Contact.Id } 32 | phone: { Phone : IsPhoneNumber } 33 | e-mail: { Email : IsEmail } 34 | 35 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Patterns/whois.iana.txt: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # IANA TLD Parsing Template 4 | # 5 | name: whois.iana.org/tld/Found 6 | 7 | # Use this template for queries to whois.iana.org: 8 | tag: whois.iana.org 9 | 10 | # Set query response type: 11 | set: Status = Found 12 | --- 13 | % IANA WHOIS server 14 | % for more information on IANA, visit http://www.iana.org 15 | % This query returned 1 object 16 | 17 | domain: {Tld:ToLower} 18 | 19 | organisation: {Organization.Name?} 20 | address: {Organization.Address*} 21 | 22 | contact: administrative 23 | name: {AdminContact.Name?} 24 | organisation: {AdminContact.Organization?} 25 | address: {AdminContact.Address*} 26 | phone: {AdminContact.TelephoneNumber?} 27 | fax-no: {AdminContact.FaxNumber?} 28 | e-mail: {AdminContact.Email?} 29 | 30 | contact: technical 31 | name: {TechContact.Name?} 32 | organisation: {TechContact.Organization?} 33 | address: {TechContact.Address*} 34 | phone: {TechContact.TelephoneNumber?} 35 | fax-no: {TechContact.FaxNumber?} 36 | e-mail: {TechContact.Email?} 37 | 38 | nserver: {NameServers$*} 39 | nserver: {NameServers$*} 40 | 41 | whois: {Url$?} 42 | 43 | status: ACTIVE 44 | remarks: {Remarks?} 45 | 46 | created: {Created$?} 47 | changed: {Changed$?} 48 | source: IANA 49 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Patterns/whois.jprs.jp.txt: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # JPRS Template for .jp domains 4 | # 5 | hint: JPRS database provides information on network administration. 6 | --- 7 | [ JPRS database provides information on network administration. Its use is ] 8 | [ restricted to network administration purposes. For further information, ] 9 | [ use 'whois -h whois.jprs.jp help'. To suppress Japanese output, add'/e' ] 10 | [ at the end of command, e.g. 'whois -h whois.jprs.jp xxx/e'. ] 11 | 12 | Domain Information: 13 | a. [Domain Name] {DomainName!$:Trim, ToLower()} 14 | g. [Organization] {Registrar.Name?$} 15 | m. [Administrative Contact] {AdminContact.Name?$} 16 | n. [Technical Contact] {TechnicalContact.Name?$} 17 | p. [Name Server] {NameServers$*} 18 | [Registered Date] {Registered?$:ToDateTime("yyyy/MM/dd")} 19 | [Connected Date] {Registered?$:ToDateTime("yyyy/MM/dd")} 20 | [Last Update] {Updated?$:SubstringBefore(' '), ToDateTime("yyyy/MM/dd")} 21 | 22 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Patterns/whois.nic.br.txt: -------------------------------------------------------------------------------- 1 | --- 2 | OutOfOrder: true 3 | --- 4 | domain: {DomainName$:ToLower()} 5 | owner: {Registrar.Name?$} 6 | ownerid: {RegistryDomainId$} 7 | owner-c: {Registrant.Name?$} 8 | admin-c: {AdminContact.Name?$} 9 | tech-c: {TechnicalContact.Name?$} 10 | nserver: {NameServers$*} 11 | created: {Registered?$:SubstringBefore(" "),ToDateTimeUtc("yyyyMMdd")} 12 | changed: {Updated?$:ToDateTimeUtc("yyyyMMdd")} 13 | expires: {Expiration?$:ToDateTimeUtc("yyyyMMdd")} -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Patterns/whois.tr.txt: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # .tr Parsing Template 4 | # 5 | name: whois.nic.tr/tr/Found 6 | 7 | # Use this template for queries to whois.nic.tr: 8 | tag: whois.nic.tr 9 | tag: tr 10 | 11 | # end tokens on new lines 12 | terminateOnNewline: true 13 | 14 | # Set query response type: 15 | set: Status = Found 16 | --- 17 | ** Registrant: 18 | { Registrant.Name ? } 19 | { Registrant.Address ? : IsNotEmpty } 20 | { Registrant.Address ? : IsNotEmpty } 21 | { Registrant.Address ? : Replace('Out of Turkey,', ''), IsNotEmpty } 22 | { Registrant.Address ? : IsNotEmpty, Trim } 23 | { Registrant.Email ? : IsEmail } 24 | { Registrant.TelephoneNumber ? : IsPhoneNumber } 25 | { Registrant.FaxNumber ? : IsPhoneNumber } 26 | 27 | 28 | ** Administrative Contact: 29 | NIC Handle : { AdminContact.RegistryId } 30 | Organization Name : { AdminContact.Organization ? } 31 | Address : { AdminContact.Address ? : Replace('Hidden upon user request', ''), IsNotEmpty } 32 | { AdminContact.Address ? : Repeating, IsNotEmpty } 33 | Phone : { AdminContact.TelephoneNumber ? : IsPhoneNumber } 34 | Fax : { AdminContact.FaxNumber ? : IsPhoneNumber } 35 | 36 | 37 | ** Technical Contact: 38 | NIC Handle : { TechnicalContact.RegistryId } 39 | Organization Name : { TechnicalContact.Organization ? } 40 | Address : { TechnicalContact.Address ? : Replace('Hidden upon user request', ''), IsNotEmpty } 41 | { TechnicalContact.Address ? : Repeating, IsNotEmpty } 42 | Phone : { TechnicalContact.TelephoneNumber ? : IsPhoneNumber } 43 | Fax : { TechnicalContact.FaxNumber ? : IsPhoneNumber } 44 | 45 | 46 | ** Billing Contact: 47 | NIC Handle : { BillingContact.RegistryId } 48 | Organization Name : { BillingContact.Organization ? } 49 | Address : { BillingContact.Address ? : Replace('Hidden upon user request', ''), IsNotEmpty } 50 | { BillingContact.Address ? : Repeating, IsNotEmpty } 51 | Phone : { BillingContact.TelephoneNumber ? : IsPhoneNumber } 52 | Fax : { BillingContact.FaxNumber ? : IsPhoneNumber } 53 | 54 | 55 | ** Domain Servers: 56 | { NameServers ? : SubstringBefore(' '), IsDomainName, ToLower, Repeating } 57 | 58 | ** Additional Info: 59 | Created on..............: { Registered ? : ToDateTimeUtc("yyyy-MMM-dd.") } 60 | Expires on..............: { Expiration ? : ToDateTimeUtc("yyyy-MMM-dd.") } 61 | 62 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Patterns/whois.uk.txt: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # .uk Parsing Template 4 | # 5 | name: whois.nic.uk/uk/Found 6 | 7 | # Use this template for queries to whois.nic.uk: 8 | tag: whois.nic.uk 9 | tag: uk 10 | 11 | # End tokens on new lines 12 | terminateOnNewLine: true 13 | 14 | # Set query response type: 15 | set: Status = Found 16 | --- 17 | 18 | Domain name: 19 | { DomainName : IsDomainName, ToLower } 20 | 21 | Registrant: 22 | { Registrant.Name ? } 23 | 24 | Registrant's address: 25 | { Registrant.Address ? : IsNotEmpty, Repeating } 26 | 27 | Registrar: 28 | { Registrar.Name ? } 29 | URL: { Registrar.Url ? : IsUrl, ToLower } 30 | 31 | Relevant dates: 32 | Registered on: { Registered ? : Trim, Replace('before ', '01-'), ToDateTimeUtc("dd-MMM-yyyy") } 33 | Registered on: { Registered ? : Trim, ToDateTimeUtc("dd-MMM-yyyy") } 34 | Expiry date: { Expiration ? : Trim, ToDateTimeUtc("dd-MMM-yyyy") } 35 | Last updated: { Updated ? : Trim, ToDateTimeUtc("dd-MMM-yyyy") } 36 | 37 | Registration status: 38 | { DomainStatus ? : Repeating } 39 | 40 | Name servers: 41 | { NameServers ? : SubstringBefore(' '), IsDomainName, ToLower, Repeating } 42 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Patterns/whois.ve.txt: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # .ve Parsing Template 4 | # 5 | name: whois.nic.ve/ve/Found 6 | 7 | # Use this template for queries to whois.nic.ve: 8 | tag: whois.nic.ve 9 | tag: ve 10 | 11 | # Set query response type: 12 | set: Status = Found 13 | --- 14 | Titular: 15 | { Registrant.Name } ({ Registrant.RegistryId }) { Registrant.Email : IsEmail, EOL } 16 | { Registrant.Address ? : IsNotEmpty, EOL } 17 | { Registrant.Address ? : IsNotEmpty, EOL } 18 | { Registrant.Address ? : IsNotEmpty, EOL } 19 | { Registrant.TelephoneNumber ? : IsPhoneNumber, Once } (FAX) { Registrant.FaxNumber ? : IsPhoneNumber, EOL } 20 | 21 | Nombre de Dominio: { DomainName : IsDomainName, ToLower, EOL } 22 | 23 | Contacto Administrativo: 24 | { AdminContact.Name } ({ AdminContact.RegistryId }) { AdminContact.Email ? : IsEmail, EOL } 25 | { AdminContact.Address ? : IsNotEmpty, EOL } 26 | { AdminContact.Address ? : IsNotEmpty, EOL } 27 | { AdminContact.Address ? : IsNotEmpty, EOL } 28 | { AdminContact.TelephoneNumber ? : SubstringBefore(' x'), IsPhoneNumber } (FAX) { AdminContact.FaxNumber ? : IsPhoneNumber, EOL } 29 | 30 | Contacto Tecnico: 31 | { TechnicalContact.Name } ({ TechnicalContact.RegistryId }) { TechnicalContact.Email ? : IsEmail, EOL } 32 | { TechnicalContact.Address ? : IsNotEmpty, EOL } 33 | { TechnicalContact.Address ? : IsNotEmpty, EOL } 34 | { TechnicalContact.Address ? : IsNotEmpty, EOL } 35 | { TechnicalContact.TelephoneNumber ? : SubstringBefore(' x'), IsPhoneNumber } (FAX) { TechnicalContact.FaxNumber ? : IsPhoneNumber, EOL } 36 | 37 | Contacto de Cobranza: 38 | { BillingContact.Name } ({ BillingContact.RegistryId }) { BillingContact.Email ? : IsEmail, EOL } 39 | { BillingContact.Address ? : IsNotEmpty, EOL } 40 | { BillingContact.Address ? : IsNotEmpty, EOL } 41 | { BillingContact.Address ? : IsNotEmpty, EOL } 42 | { BillingContact.TelephoneNumber ? : SubstringBefore(' x'), IsPhoneNumber } (FAX) { BillingContact.FaxNumber ? : IsPhoneNumber, EOL } 43 | 44 | Fecha de Vencimiento: { Expiration ? : ToDateTimeUtc("yyyy-MM-dd HH:mm:ss"), EOL } 45 | Ultima Actualizacion: { Updated ? : ToDateTimeUtc("yyyy-MM-dd HH:mm:ss"), EOL } 46 | Fecha de Creacion: { Registered ? : ToDateTimeUtc("yyyy-MM-dd HH:mm:ss"), EOL } 47 | 48 | Estatus del dominio: { DomainStatus ? : Repeating, EOL } 49 | 50 | Servidor(es) de Nombres de Dominio: 51 | 52 | { NameServers ? : Remove('- '), IsDomainName, ToLower, Repeating, EOL } 53 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Patterns/whois.verisign-grs.com.txt: -------------------------------------------------------------------------------- 1 |  Domain Name: {WhoisRedirect.Domain$:ToLower} 2 | Registrar WHOIS Server: {WhoisRedirect.Url$} 3 | Registrar URL: {WhoisRedirect.ReferralUrl?} 4 | Updated Date: {WhoisRedirect.ModifiedDate:ToDateTimeUtc('yyyy-MM-ddTHH:mm:ssZ')} 5 | Creation Date: {WhoisRedirect.CreatedDate:ToDateTimeUtc('yyyy-MM-ddTHH:mm:ssZ')} 6 | Registry Expiry Date: {WhoisRedirect.ExpirationDate:ToDateTimeUtc('yyyy-MM-ddTHH:mm:ssZ')} 7 | Registrar: {WhoisRedirect.Registrar$?} 8 | Name Server: {WhoisRedirect.NameServers*$?} 9 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Patterns/whois.vg.not.found.txt: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # .vg Parsing Template 4 | # 5 | name: ccwhois.ksregistry.net/vg/NotFound 6 | 7 | # Use this template for queries to ccwhois.ksregistry.net: 8 | tag: ccwhois.ksregistry.net 9 | tag: vg 10 | 11 | # Set query response type: 12 | set: Status = NotFound 13 | 14 | # Must contain: 15 | hint: not found 16 | --- 17 | not found 18 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Samples/Patterns/whois.vg.txt: -------------------------------------------------------------------------------- 1 | --- 2 | # 3 | # .vg Parsing Template 4 | # 5 | name: ccwhois.ksregistry.net/vg/Found 6 | 7 | # Use this template for queries to ccwhois.ksregistry.net: 8 | tag: ccwhois.ksregistry.net 9 | tag: vg 10 | 11 | # Set query response type: 12 | set: Status = Found 13 | 14 | # Must contain: 15 | hint: The data in the WHOIS database of KSregistry GmbH 16 | --- 17 | domain name: { DomainName : IsDomainName, ToLower, EOL } 18 | registrar: { Registrar.Name ? : EOL } 19 | url: { Registrar.Url ? : IsDomainName, ToLower, EOL } 20 | status: { DomainStatus ? : Repeating, EOL } 21 | created date: { Registered ? : ToDateTime("yyyy-MM-dd HH:mm:ss"), EOL } 22 | updated date: { Updated ? : ToDateTime("yyyy-MM-dd HH:mm:ss"), EOL } 23 | expiration date: { Expiration ? : ToDateTime("yyyy-MM-dd HH:mm:ss"), EOL } 24 | 25 | owner-contact: { Registrant.RegistryId ? : EOL } 26 | owner-organization: { Registrant.Organization ? : EOL } 27 | owner-name: { Registrant.Name ? : EOL } 28 | owner-street: { Registrant.Address ? : EOL } 29 | owner-city: { Registrant.Address ? : EOL } 30 | owner-zip: { Registrant.Address ? : EOL } 31 | owner-country: { Registrant.Address ? : EOL } 32 | owner-phone: { Registrant.TelephoneNumber ? : IsPhoneNumber, EOL } 33 | owner-fax: { Registrant.FaxNumber ? : IsPhoneNumber, EOL } 34 | owner-email: { Registrant.Email ? : IsEmail, EOL } 35 | 36 | admin-contact: { AdminContact.RegistryId ? : EOL } 37 | admin-organization: { AdminContact.Organization ? : EOL } 38 | admin-name: { AdminContact.Name ? : EOL } 39 | admin-street: { AdminContact.Address ? : EOL } 40 | admin-city: { AdminContact.Address ? : EOL } 41 | admin-zip: { AdminContact.Address ? : EOL } 42 | admin-country: { AdminContact.Address ? : EOL } 43 | admin-phone: { AdminContact.TelephoneNumber ? : IsPhoneNumber, EOL } 44 | admin-fax: { AdminContact.FaxNumber ? : IsPhoneNumber, EOL } 45 | admin-email: { AdminContact.Email ? : IsEmail, EOL } 46 | 47 | tech-contact: { TechnicalContact.RegistryId ? : EOL } 48 | tech-organization: { TechnicalContact.Organization ? : EOL } 49 | tech-name: { TechnicalContact.Name ? : EOL } 50 | tech-street: { TechnicalContact.Address ? : EOL } 51 | tech-city: { TechnicalContact.Address ? : EOL } 52 | tech-zip: { TechnicalContact.Address ? : EOL } 53 | tech-country: { TechnicalContact.Address ? : EOL } 54 | tech-phone: { TechnicalContact.TelephoneNumber ? : IsPhoneNumber, EOL } 55 | tech-fax: { TechnicalContact.FaxNumber ? : IsPhoneNumber, EOL } 56 | tech-email: { TechnicalContact.Email ? : IsEmail, EOL } 57 | 58 | billing-contact: { BillingContact.RegistryId ? : EOL } 59 | billing-organization: { BillingContact.Organization ? : EOL } 60 | billing-name: { BillingContact.Name ? : EOL } 61 | billing-street: { BillingContact.Address ? : EOL } 62 | billing-city: { BillingContact.Address ? : EOL } 63 | billing-zip: { BillingContact.Address ? : EOL } 64 | billing-country: { BillingContact.Address ? : EOL } 65 | billing-phone: { BillingContact.TelephoneNumber ? : IsPhoneNumber, EOL } 66 | billing-fax: { BillingContact.FaxNumber ? : IsPhoneNumber, EOL } 67 | billing-email: { BillingContact.Email ? : IsEmail, EOL } 68 | 69 | nameserver: { NameServers * : IsDomainName, Repeating, EOL } 70 | nameserver: { NameServers * : IsDomainName, Repeating, EOL } 71 | -------------------------------------------------------------------------------- /Tokenizer.Tests/SerilogConfig.cs: -------------------------------------------------------------------------------- 1 | using Serilog; 2 | 3 | namespace Tokens 4 | { 5 | class SerilogConfig 6 | { 7 | public static void Init() 8 | { 9 | Log.Logger = new LoggerConfiguration() 10 | .MinimumLevel 11 | .Verbose() 12 | .WriteTo 13 | .Console() 14 | .CreateLogger(); 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Tokenizer.Tests/SplitTests.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using NUnit.Framework; 3 | 4 | namespace Tokens 5 | { 6 | [TestFixture] 7 | public class SplitTests 8 | { 9 | private Tokenizer tokenizer; 10 | 11 | private class Foo 12 | { 13 | public List Names { get; set; } 14 | } 15 | 16 | [SetUp] 17 | public void SetUp() 18 | { 19 | SerilogConfig.Init(); 20 | 21 | tokenizer = new Tokenizer(new TokenizerOptions{ EnableLogging = true }); 22 | } 23 | 24 | [Test] 25 | public void TestSplitValue() 26 | { 27 | const string pattern = @"Names: { Names : Split(',') }"; 28 | const string input = @"Names: Alice,Bob,Charles"; 29 | 30 | var results = tokenizer.Tokenize(pattern, input); 31 | 32 | var foo = results.Value; 33 | 34 | Assert.AreEqual(3, foo.Names.Count); 35 | Assert.AreEqual("Alice", foo.Names[0]); 36 | Assert.AreEqual("Bob", foo.Names[1]); 37 | Assert.AreEqual("Charles", foo.Names[2]); 38 | } 39 | } 40 | } 41 | 42 | -------------------------------------------------------------------------------- /Tokenizer.Tests/TemplateCollectionTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | 3 | namespace Tokens 4 | { 5 | [TestFixture] 6 | public class TemplateCollectionTests 7 | { 8 | private TemplateCollection collection; 9 | 10 | [SetUp] 11 | public void SetUp() 12 | { 13 | collection = new TemplateCollection(); 14 | } 15 | 16 | [Test] 17 | public void TestCollectionContainsTagWhenTrue() 18 | { 19 | var template = new Template(); 20 | template.Tags.Add("One"); 21 | 22 | collection.Add(template); 23 | 24 | Assert.IsTrue(collection.ContainsTag("One")); 25 | } 26 | 27 | [Test] 28 | public void TestCollectionContainsTagWhenTrueAndDifferentCase() 29 | { 30 | var template = new Template(); 31 | template.Tags.Add("One"); 32 | 33 | collection.Add(template); 34 | 35 | Assert.IsTrue(collection.ContainsTag("one")); 36 | } 37 | 38 | [Test] 39 | public void TestCollectionContainsTagWhenFalse() 40 | { 41 | var template = new Template(); 42 | template.Tags.Add("One"); 43 | 44 | collection.Add(template); 45 | 46 | Assert.IsFalse(collection.ContainsTag("two")); 47 | } 48 | 49 | [Test] 50 | public void TestCollectionContainsAllTagsWhenTrue() 51 | { 52 | var template = new Template(); 53 | template.Tags.Add("One"); 54 | template.Tags.Add("Two"); 55 | 56 | collection.Add(template); 57 | 58 | Assert.IsTrue(collection.ContainsAllTags("One", "Two")); 59 | } 60 | 61 | [Test] 62 | public void TestCollectionContainsAllTagsWhenFalse() 63 | { 64 | var template = new Template(); 65 | template.Tags.Add("One"); 66 | template.Tags.Add("Two"); 67 | 68 | collection.Add(template); 69 | 70 | Assert.IsFalse(collection.ContainsAllTags("One", "Two", "Three")); 71 | } 72 | 73 | [Test] 74 | public void TestCollectionCount() 75 | { 76 | collection.Add(new Template("One", string.Empty)); 77 | collection.Add(new Template("Two", string.Empty)); 78 | collection.Add(new Template("Three", string.Empty)); 79 | 80 | Assert.AreEqual(3, collection.Count); 81 | 82 | collection.Clear(); 83 | 84 | Assert.AreEqual(0, collection.Count); 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /Tokenizer.Tests/TokenTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using NUnit.Framework; 3 | using Tokens.Enumerators; 4 | using Tokens.Validators; 5 | 6 | namespace Tokens 7 | { 8 | [TestFixture] 9 | public class TokenTests 10 | { 11 | private Token token; 12 | 13 | public class Person 14 | { 15 | public string Name { get; set; } 16 | 17 | public int Age { get; set; } 18 | 19 | public DateTime Birthday { get; set; } 20 | } 21 | 22 | [SetUp] 23 | public void SetUp() 24 | { 25 | token = new Token("Test"); 26 | } 27 | 28 | [Test] 29 | public void TestSetTokenValue() 30 | { 31 | var person = new Person(); 32 | 33 | token.Name = "Person.Name"; 34 | 35 | var assigned = token.Assign(person, "Sue", TokenizerOptions.Defaults, new FileLocation(), out var value); 36 | 37 | Assert.AreEqual(true, assigned); 38 | Assert.AreEqual("Sue", person.Name); 39 | } 40 | 41 | [Test] 42 | public void TestSetTokenValueWithValidator() 43 | { 44 | var person = new Person(); 45 | 46 | token.Name = "Person.Age"; 47 | token.Decorators.Add(new TokenDecoratorContext(typeof(IsNumericValidator))); 48 | 49 | var assigned = token.Assign(person, "20", TokenizerOptions.Defaults, new FileLocation(), out var value); 50 | 51 | Assert.AreEqual(true, assigned); 52 | Assert.AreEqual(20, person.Age); 53 | } 54 | 55 | [Test] 56 | public void TestSetTokenValueWithValidatorWhenInvalid() 57 | { 58 | var person = new Person(); 59 | 60 | token.Name = "Person.Age"; 61 | token.Decorators.Add(new TokenDecoratorContext(typeof(IsNumericValidator))); 62 | 63 | var assigned = token.Assign(person, "Twenty", TokenizerOptions.Defaults, new FileLocation(), out var value); 64 | 65 | Assert.AreEqual(false, assigned); 66 | Assert.AreEqual(0, person.Age); 67 | } 68 | 69 | [Test] 70 | public void TestSetTokenValueWhenNull() 71 | { 72 | var person = new Person(); 73 | 74 | token.Name = "Person.Name"; 75 | 76 | var assigned = token.Assign(person, "Sue", TokenizerOptions.Defaults, new FileLocation(), out var value); 77 | 78 | Assert.AreEqual(true, assigned); 79 | Assert.AreEqual("Sue", person.Name); 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Tokenizer.Tests.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | netcoreapp2.1 4 | Tokens 5 | 6 | Library 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /Tokenizer.Tests/TokenizerOptionsTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using System.Linq; 3 | using Tokens.Parsers; 4 | 5 | namespace Tokens 6 | { 7 | [TestFixture] 8 | public class TokenizerOptionsTests 9 | { 10 | [Test] 11 | public void TestTrimBeforePreambleWhenTrue() 12 | { 13 | const string content = "Should be trimmed\r\nPreamble: { First } Second: { Second }"; 14 | 15 | var parser = new TokenParser(); 16 | 17 | parser.Options.TrimPreambleBeforeNewLine = true; 18 | 19 | var template = parser.Parse(content); 20 | 21 | Assert.AreEqual(2, template.Tokens.Count); 22 | Assert.AreEqual("Preamble: ", template.Tokens.ElementAt(0).Preamble); 23 | Assert.AreEqual("Second: ", template.Tokens.ElementAt(1).Preamble); 24 | } 25 | 26 | [Test] 27 | public void TestTrimBeforePreambleWhenFalse() 28 | { 29 | const string content = "Should not be trimmed\r\nPreamble: { First } Second: { Second }"; 30 | 31 | var parser = new TokenParser(); 32 | 33 | parser.Options.TrimPreambleBeforeNewLine = false; 34 | 35 | var template = parser.Parse(content); 36 | 37 | Assert.AreEqual(2, template.Tokens.Count); 38 | Assert.AreEqual("Should not be trimmed\nPreamble: ", template.Tokens.ElementAt(0).Preamble); 39 | Assert.AreEqual("Second: ", template.Tokens.ElementAt(1).Preamble); 40 | } 41 | 42 | [Test] 43 | public void TestTrimBeforePreambleWhenSetFromFrontMatter() 44 | { 45 | const string content = "---\nTrimPreambleBeforeNewLine: true\n---\nShould be trimmed\r\nPreamble: { First } Second: { Second }"; 46 | 47 | var parser = new TokenParser(); 48 | 49 | parser.Options.TrimPreambleBeforeNewLine = false; 50 | 51 | var template = parser.Parse(content); 52 | 53 | Assert.AreEqual(2, template.Tokens.Count); 54 | Assert.AreEqual("Preamble: ", template.Tokens.ElementAt(0).Preamble); 55 | Assert.AreEqual("Second: ", template.Tokens.ElementAt(1).Preamble); 56 | Assert.IsTrue(template.Options.TrimPreambleBeforeNewLine); 57 | } 58 | 59 | [Test] 60 | public void TestTerminateOnNewLineWhenSetFromFrontMatter() 61 | { 62 | const string content = "---\nTerminateOnNewLine: true\n---\nPreamble: { First }\n Trimmed"; 63 | 64 | var parser = new TokenParser(); 65 | 66 | parser.Options.TrimPreambleBeforeNewLine = false; 67 | 68 | var template = parser.Parse(content); 69 | 70 | Assert.AreEqual(2, template.Tokens.Count); 71 | Assert.AreEqual("Preamble: ", template.Tokens.ElementAt(0).Preamble); 72 | Assert.IsTrue(template.Options.TerminateOnNewline); 73 | } 74 | 75 | [Test] 76 | public void TestTerminateOnNewLineWhenNotSetFromFrontMatter() 77 | { 78 | const string content = "Preamble: { First }\n Trimmed"; 79 | 80 | var parser = new TokenParser(); 81 | 82 | parser.Options.TrimPreambleBeforeNewLine = false; 83 | 84 | var template = parser.Parse(content); 85 | 86 | Assert.AreEqual(2, template.Tokens.Count); 87 | Assert.AreEqual("Preamble: ", template.Tokens.ElementAt(0).Preamble); 88 | Assert.IsFalse(template.Options.TerminateOnNewline); 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Transformers/BlowsUpTransformer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Tokens.Transformers 4 | { 5 | public class BlowsUpTransformer : ITokenTransformer 6 | { 7 | public bool CanTransform(object value, string[] args, out object transformed) 8 | { 9 | throw new NotImplementedException(); 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Transformers/RemoveEndTransformerTest.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | [TestFixture] 7 | public class RemoveEndTransformerTest 8 | { 9 | private RemoveEndTransformer transformer; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | transformer = new RemoveEndTransformer(); 15 | } 16 | 17 | [Test] 18 | public void TestRemoveEnd() 19 | { 20 | var result = transformer.CanTransform("one two three", new [] { "three" }, out var transformed); 21 | 22 | Assert.IsTrue(result); 23 | Assert.AreEqual("one two ", transformed); 24 | } 25 | 26 | [Test] 27 | public void TestRemoveEndWhenNotPresent() 28 | { 29 | var result = transformer.CanTransform("one two three", new [] { "two" }, out var transformed); 30 | 31 | Assert.IsTrue(result); 32 | Assert.AreEqual("one two three", transformed); 33 | } 34 | 35 | [Test] 36 | public void TestSubstringAfterWhenMissingArgument() 37 | { 38 | Assert.Throws(() => transformer.CanTransform("one two three", null, out var t)); 39 | } 40 | 41 | [Test] 42 | public void TestSubstringAfterWhenEmpty() 43 | { 44 | var result = transformer.CanTransform(string.Empty, null, out var transformed); 45 | 46 | Assert.IsTrue(result); 47 | Assert.AreEqual(string.Empty, transformed); 48 | } 49 | 50 | [Test] 51 | public void TestSubstringAfterWhenNull() 52 | { 53 | var result = transformer.CanTransform(null, null, out var transformed); 54 | 55 | Assert.IsTrue(result); 56 | Assert.AreEqual(string.Empty, transformed); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Transformers/RemoveEndTransformerTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | [TestFixture] 7 | public class RemoveEndTransformerTests 8 | { 9 | private RemoveEndTransformer transformer; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | transformer = new RemoveEndTransformer(); 15 | } 16 | 17 | [Test] 18 | public void TestRemoveEnd() 19 | { 20 | var result = transformer.CanTransform("one two three", new [] { "three" }, out var transformed); 21 | 22 | Assert.IsTrue(result); 23 | Assert.AreEqual("one two ", transformed); 24 | } 25 | 26 | [Test] 27 | public void TestRemoveEndWhenNotPresent() 28 | { 29 | var result = transformer.CanTransform("one two three", new [] { "two" }, out var transformed); 30 | 31 | Assert.IsTrue(result); 32 | Assert.AreEqual("one two three", transformed); 33 | } 34 | 35 | [Test] 36 | public void TestSubstringAfterWhenMissingArgument() 37 | { 38 | Assert.Throws(() => transformer.CanTransform("one two three", null, out var t)); 39 | } 40 | 41 | [Test] 42 | public void TestSubstringAfterWhenEmpty() 43 | { 44 | var result = transformer.CanTransform(string.Empty, null, out var transformed); 45 | 46 | Assert.IsTrue(result); 47 | Assert.AreEqual(string.Empty, transformed); 48 | } 49 | 50 | [Test] 51 | public void TestSubstringAfterWhenNull() 52 | { 53 | var result = transformer.CanTransform(null, null, out var transformed); 54 | 55 | Assert.IsTrue(result); 56 | Assert.AreEqual(string.Empty, transformed); 57 | } 58 | 59 | [Test] 60 | public void TestForDocumentation() 61 | { 62 | var template = "Domain Name: { DomainName : RemoveEnd('.') }"; 63 | var input = "Domain Name: domain.com."; 64 | 65 | var result = new Tokenizer().Tokenize(template, input); 66 | 67 | Assert.AreEqual("domain.com", result.First("DomainName")); 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Transformers/RemoveStartTransformerTest.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | [TestFixture] 7 | public class RemoveStartTransformerTest 8 | { 9 | private RemoveStartTransformer transformer; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | transformer = new RemoveStartTransformer(); 15 | } 16 | 17 | [Test] 18 | public void TestRemoveStart() 19 | { 20 | var result = transformer.CanTransform("one two three", new [] { "one" }, out var transformed); 21 | 22 | Assert.IsTrue(result); 23 | Assert.AreEqual(" two three", transformed); 24 | } 25 | 26 | [Test] 27 | public void TestRemoveStartWhenNotPresent() 28 | { 29 | var result = transformer.CanTransform("one two three", new [] { "two" }, out var transformed); 30 | 31 | Assert.IsTrue(result); 32 | Assert.AreEqual("one two three", transformed); 33 | } 34 | 35 | [Test] 36 | public void TestSubstringAfterWhenMissingArgument() 37 | { 38 | Assert.Throws(() => transformer.CanTransform("one two three", null, out var t)); 39 | } 40 | 41 | [Test] 42 | public void TestSubstringAfterWhenEmpty() 43 | { 44 | var result = transformer.CanTransform(string.Empty, null, out var transformed); 45 | 46 | Assert.IsTrue(result); 47 | Assert.AreEqual(string.Empty, transformed); 48 | } 49 | 50 | [Test] 51 | public void TestSubstringAfterWhenNull() 52 | { 53 | var result = transformer.CanTransform(null, null, out var transformed); 54 | 55 | Assert.IsTrue(result); 56 | Assert.AreEqual(string.Empty, transformed); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Transformers/RemoveStartTransformerTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | [TestFixture] 7 | public class RemoveStartTransformerTests 8 | { 9 | private RemoveStartTransformer transformer; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | transformer = new RemoveStartTransformer(); 15 | } 16 | 17 | [Test] 18 | public void TestRemoveStart() 19 | { 20 | var result = transformer.CanTransform("one two three", new [] { "one" }, out var transformed); 21 | 22 | Assert.IsTrue(result); 23 | Assert.AreEqual(" two three", transformed); 24 | } 25 | 26 | [Test] 27 | public void TestRemoveStartWhenNotPresent() 28 | { 29 | var result = transformer.CanTransform("one two three", new [] { "two" }, out var transformed); 30 | 31 | Assert.IsTrue(result); 32 | Assert.AreEqual("one two three", transformed); 33 | } 34 | 35 | [Test] 36 | public void TestSubstringAfterWhenMissingArgument() 37 | { 38 | Assert.Throws(() => transformer.CanTransform("one two three", null, out var t)); 39 | } 40 | 41 | [Test] 42 | public void TestSubstringAfterWhenEmpty() 43 | { 44 | var result = transformer.CanTransform(string.Empty, null, out var transformed); 45 | 46 | Assert.IsTrue(result); 47 | Assert.AreEqual(string.Empty, transformed); 48 | } 49 | 50 | [Test] 51 | public void TestSubstringAfterWhenNull() 52 | { 53 | var result = transformer.CanTransform(null, null, out var transformed); 54 | 55 | Assert.IsTrue(result); 56 | Assert.AreEqual(string.Empty, transformed); 57 | } 58 | 59 | [Test] 60 | public void TestForDocumentation() 61 | { 62 | var template = ": { DomainName : RemoveEnd('.') }"; 63 | var input = "Domain Name: domain.com."; 64 | 65 | var result = new Tokenizer().Tokenize(template, input); 66 | 67 | Assert.AreEqual("domain.com", result.First("DomainName")); 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Transformers/RemoveTransformerTest.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | [TestFixture] 7 | public class RemoveTransformerTest 8 | { 9 | private RemoveTransformer transformer; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | transformer = new RemoveTransformer(); 15 | } 16 | 17 | [Test] 18 | public void TestSubstringAfter() 19 | { 20 | var result = transformer.CanTransform("one two three", new [] { "two" }, out var transformed); 21 | 22 | Assert.IsTrue(result); 23 | Assert.AreEqual("one three", transformed); 24 | } 25 | 26 | [Test] 27 | public void TestSubstringAfterWhenMissingArgument() 28 | { 29 | Assert.Throws(() => transformer.CanTransform("one two three", null, out var t)); 30 | } 31 | 32 | [Test] 33 | public void TestSubstringAfterWhenEmpty() 34 | { 35 | var result = transformer.CanTransform(string.Empty, null, out var transformed); 36 | 37 | Assert.IsTrue(result); 38 | Assert.AreEqual(string.Empty, transformed); 39 | } 40 | 41 | [Test] 42 | public void TestSubstringAfterWhenNull() 43 | { 44 | var result = transformer.CanTransform(null, null, out var transformed); 45 | 46 | Assert.IsTrue(result); 47 | Assert.AreEqual(string.Empty, transformed); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Transformers/RemoveTransformerTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | [TestFixture] 7 | public class RemoveTransformerTests 8 | { 9 | private RemoveTransformer transformer; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | transformer = new RemoveTransformer(); 15 | } 16 | 17 | [Test] 18 | public void TestSubstringAfter() 19 | { 20 | var result = transformer.CanTransform("one two three", new [] { "two" }, out var transformed); 21 | 22 | Assert.IsTrue(result); 23 | Assert.AreEqual("one three", transformed); 24 | } 25 | 26 | [Test] 27 | public void TestSubstringAfterWhenMissingArgument() 28 | { 29 | Assert.Throws(() => transformer.CanTransform("one two three", null, out var t)); 30 | } 31 | 32 | [Test] 33 | public void TestSubstringAfterWhenEmpty() 34 | { 35 | var result = transformer.CanTransform(string.Empty, null, out var transformed); 36 | 37 | Assert.IsTrue(result); 38 | Assert.AreEqual(string.Empty, transformed); 39 | } 40 | 41 | [Test] 42 | public void TestSubstringAfterWhenNull() 43 | { 44 | var result = transformer.CanTransform(null, null, out var transformed); 45 | 46 | Assert.IsTrue(result); 47 | Assert.AreEqual(string.Empty, transformed); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Transformers/ReplaceTransformerTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | [TestFixture] 7 | public class ReplaceTransformerTests 8 | { 9 | private ReplaceTransformer transformer; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | transformer = new ReplaceTransformer(); 15 | } 16 | 17 | [Test] 18 | public void TestSubstringAfter() 19 | { 20 | var result = transformer.CanTransform("one two three", new [] { "two", "four" }, out var transformed); 21 | 22 | Assert.IsTrue(result); 23 | Assert.AreEqual("one four three", transformed); 24 | } 25 | 26 | [Test] 27 | public void TestSubstringAfterWhenMissingArgument() 28 | { 29 | Assert.Throws(() => transformer.CanTransform("one two three", null, out var t)); 30 | } 31 | 32 | [Test] 33 | public void TestSubstringAfterWhenEmpty() 34 | { 35 | var result = transformer.CanTransform(string.Empty, null, out var transformed); 36 | 37 | Assert.IsTrue(result); 38 | Assert.AreEqual(string.Empty, transformed); 39 | } 40 | 41 | [Test] 42 | public void TestSubstringAfterWhenNull() 43 | { 44 | var result = transformer.CanTransform(null, null, out var transformed); 45 | 46 | Assert.IsTrue(result); 47 | Assert.AreEqual(string.Empty, transformed); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Transformers/SetTransformerTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using NUnit.Framework; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | [TestFixture] 7 | public class SetTransformerTests 8 | { 9 | private SetTransformer transformer; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | transformer = new SetTransformer(); 15 | } 16 | 17 | [Test] 18 | public void TestSet() 19 | { 20 | var result = transformer.CanTransform("input", new [] { "output" }, out var transformed); 21 | 22 | Assert.IsTrue(result); 23 | Assert.AreEqual("output", transformed); 24 | } 25 | 26 | [Test] 27 | public void TestSetWhenEmpty() 28 | { 29 | Assert.Throws(() => transformer.CanTransform(string.Empty, null, out var t));; 30 | } 31 | 32 | [Test] 33 | public void TestSetWhenTooManyArguments() 34 | { 35 | Assert.Throws(() => transformer.CanTransform("input", new [] { "1", "2" }, out var t)); 36 | } 37 | 38 | [Test] 39 | public void TestInTemplate() 40 | { 41 | var pattern = @"Name: { Name : Set('Alice') }"; 42 | var input = "Name: Bob"; 43 | 44 | var result = new Tokenizer().Tokenize(pattern, input); 45 | 46 | Assert.AreEqual("Alice", result.First("Name")); 47 | } 48 | 49 | [Test] 50 | public void TestInTemplateWithShortHand() 51 | { 52 | var pattern = @"Name: { Name = 'Alice' : ToUpper }"; 53 | var input = "Name: Bob"; 54 | 55 | var result = new Tokenizer().Tokenize(pattern, input); 56 | 57 | Assert.AreEqual("ALICE", result.First("Name")); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Transformers/SplitTransformerTest.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | [TestFixture] 7 | public class SplitTransformerTest 8 | { 9 | private SplitTransformer transformer; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | transformer = new SplitTransformer(); 15 | } 16 | 17 | [Test] 18 | public void TestSplitInput() 19 | { 20 | var result = transformer.CanTransform("1,2,3,4", new [] { "," }, out var transformed); 21 | 22 | Assert.IsTrue(result); 23 | 24 | var list = transformed as string[]; 25 | 26 | Assert.AreEqual(4, list.Length); 27 | Assert.AreEqual("1", list[0]); 28 | Assert.AreEqual("2", list[1]); 29 | Assert.AreEqual("3", list[2]); 30 | Assert.AreEqual("4", list[3]); 31 | } 32 | 33 | [Test] 34 | public void TestSplitInputWhenNoSeparator() 35 | { 36 | var result = transformer.CanTransform("1-2-3-4", new [] { "," }, out var transformed); 37 | 38 | Assert.IsTrue(result); 39 | Assert.AreEqual("1-2-3-4", transformed); 40 | } 41 | 42 | [Test] 43 | public void TestSplitWhenMissingArgument() 44 | { 45 | Assert.Throws(() => transformer.CanTransform("1,2,3,4", null, out var t)); 46 | } 47 | 48 | [Test] 49 | public void TestSplitWhenEmptyInput() 50 | { 51 | var result = transformer.CanTransform(string.Empty, null, out var transformed); 52 | 53 | Assert.IsTrue(result); 54 | Assert.AreEqual(string.Empty, transformed); 55 | } 56 | 57 | [Test] 58 | public void TestSplitWhenNullInput() 59 | { 60 | var result = transformer.CanTransform(null, null, out var transformed); 61 | 62 | Assert.IsTrue(result); 63 | Assert.AreEqual(string.Empty, transformed); 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Transformers/SplitTransformerTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | [TestFixture] 7 | public class SplitTransformerTests 8 | { 9 | private SplitTransformer transformer; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | transformer = new SplitTransformer(); 15 | } 16 | 17 | [Test] 18 | public void TestSplitInput() 19 | { 20 | var result = transformer.CanTransform("1,2,3,4", new [] { "," }, out var transformed); 21 | 22 | Assert.IsTrue(result); 23 | 24 | var list = transformed as string[]; 25 | 26 | Assert.AreEqual(4, list.Length); 27 | Assert.AreEqual("1", list[0]); 28 | Assert.AreEqual("2", list[1]); 29 | Assert.AreEqual("3", list[2]); 30 | Assert.AreEqual("4", list[3]); 31 | } 32 | 33 | [Test] 34 | public void TestSplitInputWhenNoSeparator() 35 | { 36 | var result = transformer.CanTransform("1-2-3-4", new [] { "," }, out var transformed); 37 | 38 | Assert.IsTrue(result); 39 | Assert.AreEqual("1-2-3-4", transformed); 40 | } 41 | 42 | [Test] 43 | public void TestSplitWhenMissingArgument() 44 | { 45 | Assert.Throws(() => transformer.CanTransform("1,2,3,4", null, out var t)); 46 | } 47 | 48 | [Test] 49 | public void TestSplitWhenEmptyInput() 50 | { 51 | var result = transformer.CanTransform(string.Empty, null, out var transformed); 52 | 53 | Assert.IsTrue(result); 54 | Assert.AreEqual(string.Empty, transformed); 55 | } 56 | 57 | [Test] 58 | public void TestSplitWhenNullInput() 59 | { 60 | var result = transformer.CanTransform(null, null, out var transformed); 61 | 62 | Assert.IsTrue(result); 63 | Assert.AreEqual(string.Empty, transformed); 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Transformers/SubstringAfterLastTransformerTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | [TestFixture] 7 | public class SubstringAfterLastTransformerTests 8 | { 9 | private SubstringAfterLastTransformer transformer; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | transformer = new SubstringAfterLastTransformer(); 15 | } 16 | 17 | [Test] 18 | public void TestSubstringAfter() 19 | { 20 | var result = transformer.CanTransform("one two two three", new [] { "two" }, out var transformed); 21 | 22 | Assert.IsTrue(result); 23 | Assert.AreEqual(" three", transformed); 24 | } 25 | 26 | [Test] 27 | public void TestSubstringAfterWhenMissingArgument() 28 | { 29 | Assert.Throws(() => transformer.CanTransform("one two three", null, out var t)); 30 | } 31 | 32 | [Test] 33 | public void TestSubstringAfterWhenEmpty() 34 | { 35 | var result = transformer.CanTransform(string.Empty, null, out var transformed); 36 | 37 | Assert.IsTrue(result); 38 | Assert.AreEqual(string.Empty, transformed); 39 | } 40 | 41 | [Test] 42 | public void TestSubstringAfterWhenNull() 43 | { 44 | var result = transformer.CanTransform(null, null, out var transformed); 45 | 46 | Assert.IsTrue(result); 47 | Assert.AreEqual(string.Empty, transformed); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Transformers/SubstringAfterTransformerTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | [TestFixture] 7 | public class SubstringAfterTransformerTests 8 | { 9 | private SubstringAfterTransformer transformer; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | transformer = new SubstringAfterTransformer(); 15 | } 16 | 17 | [Test] 18 | public void TestSubstringAfter() 19 | { 20 | var result = transformer.CanTransform("one two three", new [] { "two" }, out var transformed); 21 | 22 | Assert.IsTrue(result); 23 | Assert.AreEqual(" three", transformed); 24 | } 25 | 26 | [Test] 27 | public void TestSubstringAfterWhenMissingArgument() 28 | { 29 | Assert.Throws(() => transformer.CanTransform("one two three", null, out var t)); 30 | } 31 | 32 | [Test] 33 | public void TestSubstringAfterWhenEmpty() 34 | { 35 | var result = transformer.CanTransform(string.Empty, null, out var transformed); 36 | 37 | Assert.IsTrue(result); 38 | Assert.AreEqual(string.Empty, transformed); 39 | } 40 | 41 | [Test] 42 | public void TestSubstringAfterWhenNull() 43 | { 44 | var result = transformer.CanTransform(null, null, out var transformed); 45 | 46 | Assert.IsTrue(result); 47 | Assert.AreEqual(string.Empty, transformed); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Transformers/SubstringBeforeLastTransformerTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | [TestFixture] 7 | public class SubstringBeforeLastTransformerTests 8 | { 9 | private SubstringBeforeLastTransformer transformer; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | transformer = new SubstringBeforeLastTransformer(); 15 | } 16 | 17 | [Test] 18 | public void TestSubstringAfter() 19 | { 20 | var result = transformer.CanTransform("one two two three", new [] { "two" }, out var transformed); 21 | 22 | Assert.IsTrue(result); 23 | Assert.AreEqual("one two ", transformed); 24 | } 25 | 26 | [Test] 27 | public void TestSubstringAfterWhenMissingArgument() 28 | { 29 | Assert.Throws(() => transformer.CanTransform("one two three", null, out var t)); 30 | } 31 | 32 | [Test] 33 | public void TestSubstringAfterWhenEmpty() 34 | { 35 | var result = transformer.CanTransform(string.Empty, null, out var transformed); 36 | 37 | Assert.IsTrue(result); 38 | Assert.AreEqual(string.Empty, transformed); 39 | } 40 | 41 | [Test] 42 | public void TestSubstringAfterWhenNull() 43 | { 44 | var result = transformer.CanTransform(null, null, out var transformed); 45 | 46 | Assert.IsTrue(result); 47 | Assert.AreEqual(string.Empty, transformed); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Transformers/SubstringBeforeTransformerTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | [TestFixture] 7 | public class SubstringBeforeTransformerTests 8 | { 9 | private SubstringBeforeTransformer transformer; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | transformer = new SubstringBeforeTransformer(); 15 | } 16 | 17 | [Test] 18 | public void TestSubstringAfter() 19 | { 20 | var result = transformer.CanTransform("one two three", new [] { "two" }, out var transformed); 21 | 22 | Assert.IsTrue(result); 23 | Assert.AreEqual("one ", transformed); 24 | } 25 | 26 | [Test] 27 | public void TestSubstringAfterWhenMissingArgument() 28 | { 29 | Assert.Throws(() => transformer.CanTransform("one two three", null, out var t)); 30 | } 31 | 32 | [Test] 33 | public void TestSubstringAfterWhenEmpty() 34 | { 35 | var result = transformer.CanTransform(string.Empty, null, out var transformed); 36 | 37 | Assert.IsTrue(result); 38 | Assert.AreEqual(string.Empty, transformed); 39 | } 40 | 41 | [Test] 42 | public void TestSubstringAfterWhenNull() 43 | { 44 | var result = transformer.CanTransform(null, null, out var transformed); 45 | 46 | Assert.IsTrue(result); 47 | Assert.AreEqual(string.Empty, transformed); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Transformers/ToLowerTransformerTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | 3 | namespace Tokens.Transformers 4 | { 5 | [TestFixture] 6 | public class ToLowerTransformerTests 7 | { 8 | private ToLowerTransformer @operator; 9 | 10 | [SetUp] 11 | public void SetUp() 12 | { 13 | @operator = new ToLowerTransformer(); 14 | } 15 | 16 | [Test] 17 | public void TestToLower() 18 | { 19 | var result = @operator.CanTransform("TEST", null, out var t); 20 | 21 | Assert.IsTrue(result); 22 | Assert.AreEqual("test", t); 23 | } 24 | 25 | [Test] 26 | public void TestToLowerWhenEmpty() 27 | { 28 | var result = @operator.CanTransform(string.Empty, null, out var t); 29 | 30 | Assert.IsTrue(result); 31 | Assert.AreEqual(string.Empty, t); 32 | } 33 | 34 | [Test] 35 | public void TestToLowerWhenNull() 36 | { 37 | var result = @operator.CanTransform(null, null, out var t); 38 | 39 | Assert.IsTrue(result); 40 | Assert.AreEqual(string.Empty, t); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Transformers/ToUpperTransformerTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | 3 | namespace Tokens.Transformers 4 | { 5 | [TestFixture] 6 | public class ToUpperTransformerTests 7 | { 8 | private ToUpperTransformer @operator; 9 | 10 | [SetUp] 11 | public void SetUp() 12 | { 13 | @operator = new ToUpperTransformer(); 14 | } 15 | 16 | [Test] 17 | public void TestToUpper() 18 | { 19 | @operator.CanTransform("test", null, out var t); 20 | 21 | Assert.AreEqual("TEST", t); 22 | } 23 | 24 | [Test] 25 | public void TestToUpperWhenEmpty() 26 | { 27 | @operator.CanTransform(string.Empty, null, out var t); 28 | 29 | Assert.AreEqual(string.Empty, t); 30 | } 31 | 32 | [Test] 33 | public void TestToUpperWhenNull() 34 | { 35 | @operator.CanTransform(null, null, out var t); 36 | 37 | Assert.AreEqual(string.Empty, t); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Transformers/TrimTransformerTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | 3 | namespace Tokens.Transformers 4 | { 5 | [TestFixture] 6 | public class TrimTransformerTests 7 | { 8 | private TrimTransformer transformer; 9 | 10 | [SetUp] 11 | public void SetUp() 12 | { 13 | transformer = new TrimTransformer(); 14 | } 15 | 16 | [Test] 17 | public void TestTrim() 18 | { 19 | var result = transformer.CanTransform(" TEST ", null, out var t); 20 | 21 | Assert.AreEqual("TEST", t); 22 | } 23 | 24 | [Test] 25 | public void TestTrimWhenEmpty() 26 | { 27 | var result = transformer.CanTransform(string.Empty, null, out var t); 28 | 29 | Assert.AreEqual(string.Empty, t); 30 | } 31 | 32 | [Test] 33 | public void TestTrimWhenNull() 34 | { 35 | var result = transformer.CanTransform(null, null, out var t); 36 | 37 | Assert.AreEqual(string.Empty, t); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Types/BoolTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | 3 | namespace Tokens.Types 4 | { 5 | [TestFixture] 6 | public class BoolTests 7 | { 8 | private Tokenizer tokenizer; 9 | 10 | private class Student 11 | { 12 | public string Name { get; set; } 13 | 14 | public bool Enrolled { get; set; } 15 | } 16 | 17 | [SetUp] 18 | public void SetUp() 19 | { 20 | SerilogConfig.Init(); 21 | 22 | tokenizer = new Tokenizer(new TokenizerOptions{ EnableLogging = true }); 23 | } 24 | 25 | [Test] 26 | public void TestSetBoolValueWhenTrue() 27 | { 28 | const string pattern = @"Name: {Name}, Enrolled: {Enrolled}"; 29 | const string input = @"Name: Alice, Enrolled: true"; 30 | 31 | var result = tokenizer.Tokenize(pattern, input); 32 | 33 | Assert.AreEqual("Alice", result.Value.Name); 34 | Assert.AreEqual(true, result.Value.Enrolled); 35 | } 36 | 37 | [Test] 38 | public void TestSetBoolValueWhenTrueAndUpperCase() 39 | { 40 | const string pattern = @"Name: {Name}, Enrolled: {Enrolled}"; 41 | const string input = @"Name: Alice, Enrolled: TRUE"; 42 | 43 | var result = tokenizer.Tokenize(pattern, input); 44 | 45 | Assert.AreEqual("Alice", result.Value.Name); 46 | Assert.AreEqual(true, result.Value.Enrolled); 47 | } 48 | 49 | [Test] 50 | public void TestSetBoolValueWhenFalse() 51 | { 52 | const string pattern = @"Name: {Name}, Enrolled: {Enrolled}"; 53 | const string input = @"Name: Alice, Enrolled: False"; 54 | 55 | var result = tokenizer.Tokenize(pattern, input); 56 | 57 | Assert.AreEqual("Alice", result.Value.Name); 58 | Assert.AreEqual(false, result.Value.Enrolled); 59 | } 60 | } 61 | } 62 | 63 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Types/EnumTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | 3 | namespace Tokens.Types 4 | { 5 | [TestFixture] 6 | public class EnumTests 7 | { 8 | private Tokenizer tokenizer; 9 | 10 | private class Student 11 | { 12 | public string Name { get; set; } 13 | 14 | public Grade Grade { get; set; } 15 | } 16 | 17 | private enum Grade 18 | { 19 | GradeA, 20 | GradeB, 21 | GradeC, 22 | } 23 | 24 | [SetUp] 25 | public void SetUp() 26 | { 27 | SerilogConfig.Init(); 28 | 29 | tokenizer = new Tokenizer(new TokenizerOptions{ EnableLogging = true }); 30 | } 31 | 32 | [Test] 33 | public void TestSetEnumValue() 34 | { 35 | const string pattern = @"Name: {Name}, Grade: {Grade}"; 36 | const string input = @"Name: Alice, Grade: GradeB"; 37 | 38 | var result = tokenizer.Tokenize(pattern, input); 39 | 40 | Assert.AreEqual("Alice", result.Value.Name); 41 | Assert.AreEqual(Grade.GradeB, result.Value.Grade); 42 | } 43 | 44 | [Test] 45 | public void TestSetEnumValueWhenWrongCase() 46 | { 47 | const string pattern = @"Name: {Name}, Grade: {Grade}"; 48 | const string input = @"Name: Alice, Grade: Gradec"; 49 | 50 | var result = tokenizer.Tokenize(pattern, input); 51 | 52 | Assert.AreEqual("Alice", result.Value.Name); 53 | Assert.AreEqual(Grade.GradeC, result.Value.Grade); 54 | } 55 | 56 | [Test] 57 | public void TestSetEnumValueWhenIncorrectValue() 58 | { 59 | const string pattern = @"Name: {Name}, Grade: {Grade}"; 60 | const string input = @"Name: Alice, Grade: GradeE"; 61 | 62 | var result = tokenizer.Tokenize(pattern, input); 63 | 64 | Assert.AreEqual("Alice", result.Value.Name); 65 | Assert.AreEqual(Grade.GradeA, result.Value.Grade); 66 | Assert.AreEqual(1, result.Exceptions.Count); 67 | } 68 | } 69 | } 70 | 71 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Validators/ContainsValidatorTest.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Validators 5 | { 6 | [TestFixture] 7 | public class ContainsValidatorTest 8 | { 9 | private ContainsValidator validator; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | validator = new ContainsValidator(); 15 | } 16 | 17 | [Test] 18 | public void TestValidateValueWhenTrue() 19 | { 20 | var result = validator.IsValid("hello world", "o wor"); 21 | 22 | Assert.IsTrue(result); 23 | } 24 | 25 | [Test] 26 | public void TestValidateValueWhenFalse() 27 | { 28 | var result = validator.IsValid("hello world", "spoon"); 29 | 30 | Assert.IsFalse(result); 31 | } 32 | 33 | [Test] 34 | public void TestValidateValueWhenMissingArgument() 35 | { 36 | Assert.Throws(() => validator.IsValid("hello world")); 37 | } 38 | 39 | [Test] 40 | public void TestValidateValueWhenNull() 41 | { 42 | var result = validator.IsValid(null); 43 | 44 | Assert.IsFalse(result); 45 | } 46 | 47 | [Test] 48 | public void TestValidateValueWhenEmpty() 49 | { 50 | var result = validator.IsValid(string.Empty); 51 | 52 | Assert.IsFalse(result); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Validators/ContainsValidatorTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Validators 5 | { 6 | [TestFixture] 7 | public class ContainsValidatorTests 8 | { 9 | private ContainsValidator validator; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | validator = new ContainsValidator(); 15 | } 16 | 17 | [Test] 18 | public void TestValidateValueWhenTrue() 19 | { 20 | var result = validator.IsValid("hello world", "o wor"); 21 | 22 | Assert.IsTrue(result); 23 | } 24 | 25 | [Test] 26 | public void TestValidateValueWhenFalse() 27 | { 28 | var result = validator.IsValid("hello world", "spoon"); 29 | 30 | Assert.IsFalse(result); 31 | } 32 | 33 | [Test] 34 | public void TestValidateValueWhenMissingArgument() 35 | { 36 | Assert.Throws(() => validator.IsValid("hello world")); 37 | } 38 | 39 | [Test] 40 | public void TestValidateValueWhenNull() 41 | { 42 | var result = validator.IsValid(null); 43 | 44 | Assert.IsFalse(result); 45 | } 46 | 47 | [Test] 48 | public void TestValidateValueWhenEmpty() 49 | { 50 | var result = validator.IsValid(string.Empty); 51 | 52 | Assert.IsFalse(result); 53 | } 54 | 55 | [Test] 56 | public void TestForDocumentation() 57 | { 58 | var template = "Name: { Name : Contains('B') }"; 59 | var input = "Name: Alice Name: Bob"; 60 | 61 | var result = new Tokenizer().Tokenize(template, input); 62 | 63 | Assert.AreEqual("Bob", result.First("Name")); 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Validators/EndsWithValidatorTest.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Validators 5 | { 6 | [TestFixture] 7 | public class EndsWithValidatorTest 8 | { 9 | private EndsWithValidator validator; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | validator = new EndsWithValidator(); 15 | } 16 | 17 | [Test] 18 | public void TestValidateValueWhenTrue() 19 | { 20 | var result = validator.IsValid("hello world", "world"); 21 | 22 | Assert.IsTrue(result); 23 | } 24 | 25 | [Test] 26 | public void TestValidateValueWhenFalse() 27 | { 28 | var result = validator.IsValid("hello world", "hello"); 29 | 30 | Assert.IsFalse(result); 31 | } 32 | 33 | [Test] 34 | public void TestValidateValueWhenMissingArgument() 35 | { 36 | Assert.Throws(() => validator.IsValid("hello world")); 37 | } 38 | 39 | [Test] 40 | public void TestValidateValueWhenNull() 41 | { 42 | var result = validator.IsValid(null); 43 | 44 | Assert.IsFalse(result); 45 | } 46 | 47 | [Test] 48 | public void TestValidateValueWhenEmpty() 49 | { 50 | var result = validator.IsValid(string.Empty); 51 | 52 | Assert.IsFalse(result); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Validators/EndsWithValidatorTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Validators 5 | { 6 | [TestFixture] 7 | public class EndsWithValidatorTests 8 | { 9 | private EndsWithValidator validator; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | validator = new EndsWithValidator(); 15 | } 16 | 17 | [Test] 18 | public void TestValidateValueWhenTrue() 19 | { 20 | var result = validator.IsValid("hello world", "world"); 21 | 22 | Assert.IsTrue(result); 23 | } 24 | 25 | [Test] 26 | public void TestValidateValueWhenFalse() 27 | { 28 | var result = validator.IsValid("hello world", "hello"); 29 | 30 | Assert.IsFalse(result); 31 | } 32 | 33 | [Test] 34 | public void TestValidateValueWhenMissingArgument() 35 | { 36 | Assert.Throws(() => validator.IsValid("hello world")); 37 | } 38 | 39 | [Test] 40 | public void TestValidateValueWhenNull() 41 | { 42 | var result = validator.IsValid(null); 43 | 44 | Assert.IsFalse(result); 45 | } 46 | 47 | [Test] 48 | public void TestValidateValueWhenEmpty() 49 | { 50 | var result = validator.IsValid(string.Empty); 51 | 52 | Assert.IsFalse(result); 53 | } 54 | 55 | [Test] 56 | public void TestForDocumentation() 57 | { 58 | var template = "Email: { AdminEmail : EndsWith('admin.com') }"; 59 | var input = "Email: alice@customer.com Email: bob@admin.com"; 60 | 61 | var result = new Tokenizer().Tokenize(template, input); 62 | 63 | Assert.AreEqual("bob@admin.com", result.First("AdminEmail")); 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Validators/IsDateTimeValidatorTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | 3 | namespace Tokens.Validators 4 | { 5 | [TestFixture] 6 | public class IsDateTimeValidatorTests 7 | { 8 | private IsDateTimeValidator validator; 9 | 10 | [SetUp] 11 | public void SetUp() 12 | { 13 | validator = new IsDateTimeValidator(); 14 | } 15 | 16 | [Test] 17 | public void TestValidateValueWhenValid() 18 | { 19 | var result = validator.IsValid("1 May 2019"); 20 | 21 | Assert.IsTrue(result); 22 | } 23 | 24 | [Test] 25 | public void TestValidateValueWhenNewIsoDate() 26 | { 27 | var result = validator.IsValid("2019-05-01"); 28 | 29 | Assert.IsTrue(result); 30 | } 31 | 32 | [Test] 33 | public void TestValidateValueWhenHasTime() 34 | { 35 | var result = validator.IsValid("2019-05-01 14:00:00"); 36 | 37 | Assert.IsTrue(result); 38 | } 39 | 40 | [Test] 41 | public void TestValidateValueWhenInvalid() 42 | { 43 | var result = validator.IsValid("hello world"); 44 | 45 | Assert.IsFalse(result); 46 | } 47 | 48 | [Test] 49 | public void TestValidateValueWhenNull() 50 | { 51 | var result = validator.IsValid(null); 52 | 53 | Assert.IsFalse(result); 54 | } 55 | 56 | [Test] 57 | public void TestValidateValueWhenEmpty() 58 | { 59 | var result = validator.IsValid(string.Empty); 60 | 61 | Assert.IsFalse(result); 62 | } 63 | 64 | [Test] 65 | public void TestForDocumentation() 66 | { 67 | var template = "Date: { Date : IsDateTime('yyyy-MM-dd') }"; 68 | var input = "Date: 3rd Oct 2019 Date: 2019-10-04"; 69 | 70 | var result = new Tokenizer().Tokenize(template, input); 71 | 72 | Assert.AreEqual("2019-10-04", result.First("Date")); 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Validators/IsDomainNameValidatorTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | 3 | namespace Tokens.Validators 4 | { 5 | [TestFixture] 6 | public class IsDomainNameValidatorTests 7 | { 8 | private IsDomainNameValidator validator; 9 | 10 | [SetUp] 11 | public void SetUp() 12 | { 13 | validator = new IsDomainNameValidator(); 14 | } 15 | 16 | [Test] 17 | public void TestValidateValueWhenValid() 18 | { 19 | var result = validator.IsValid("github.com"); 20 | 21 | Assert.IsTrue(result); 22 | } 23 | 24 | [Test] 25 | public void TestValidateValueWhenNewTld() 26 | { 27 | var result = validator.IsValid("hello.ninja"); 28 | 29 | Assert.IsTrue(result); 30 | } 31 | 32 | [Test] 33 | public void TestValidateValueWhenHasSubdomain() 34 | { 35 | var result = validator.IsValid("www.hello.ninja"); 36 | 37 | Assert.IsTrue(result); 38 | } 39 | 40 | [Test] 41 | public void TestValidateValueWhenInvalidDomain() 42 | { 43 | var result = validator.IsValid("hello world"); 44 | 45 | Assert.IsFalse(result); 46 | } 47 | 48 | [Test] 49 | public void TestValidateValueWhenNull() 50 | { 51 | var result = validator.IsValid(null); 52 | 53 | Assert.IsFalse(result); 54 | } 55 | 56 | [Test] 57 | public void TestValidateValueWhenEmpty() 58 | { 59 | var result = validator.IsValid(string.Empty); 60 | 61 | Assert.IsFalse(result); 62 | } 63 | 64 | [Test] 65 | public void TestForDocumentation() 66 | { 67 | var template = "Web: { Domain : IsDomainName }"; 68 | var input = "Web: n/a Web: www.flipbit.co.uk"; 69 | 70 | var result = new Tokenizer().Tokenize(template, input); 71 | 72 | Assert.AreEqual("www.flipbit.co.uk", result.First("Domain")); 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Validators/IsEmailValidatorTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | 3 | namespace Tokens.Validators 4 | { 5 | [TestFixture] 6 | public class IsEmailValidatorTests 7 | { 8 | private IsEmailValidator validator; 9 | 10 | [SetUp] 11 | public void SetUp() 12 | { 13 | validator = new IsEmailValidator(); 14 | } 15 | 16 | [Test] 17 | public void TestValidateValueWhenValid() 18 | { 19 | var result = validator.IsValid("hello@example.com"); 20 | 21 | Assert.IsTrue(result); 22 | } 23 | 24 | [Test] 25 | public void TestValidateValueWhenInvalid() 26 | { 27 | var result = validator.IsValid("hello world"); 28 | 29 | Assert.IsFalse(result); 30 | } 31 | 32 | [Test] 33 | public void TestValidateValueWhenNull() 34 | { 35 | var result = validator.IsValid(null); 36 | 37 | Assert.IsFalse(result); 38 | } 39 | 40 | [Test] 41 | public void TestValidateValueWhenEmpty() 42 | { 43 | var result = validator.IsValid(string.Empty); 44 | 45 | Assert.IsFalse(result); 46 | } 47 | 48 | [Test] 49 | public void TestForDocumentation() 50 | { 51 | var template = "Email: { Email : IsEmail }"; 52 | var input = "Email: webmaster at host.com Email: hello@domain.com"; 53 | 54 | var result = new Tokenizer().Tokenize(template, input); 55 | 56 | Assert.AreEqual("hello@domain.com", result.First("Email")); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Validators/IsLooseAbsoluteUrlValidatorTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | 3 | namespace Tokens.Validators 4 | { 5 | [TestFixture] 6 | public class IsLooseAbsoluteUrlValidatorTests 7 | { 8 | private IsLooseAbsoluteUrlValidator validator; 9 | 10 | [SetUp] 11 | public void SetUp() 12 | { 13 | validator = new IsLooseAbsoluteUrlValidator(); 14 | } 15 | 16 | [Test] 17 | public void TestValidateValueWhenHttp() 18 | { 19 | var result = validator.IsValid("http://github.com"); 20 | 21 | Assert.IsTrue(result); 22 | } 23 | 24 | [Test] 25 | public void TestValidateValueWhenHttps() 26 | { 27 | var result = validator.IsValid("https://github.com"); 28 | 29 | Assert.IsTrue(result); 30 | } 31 | 32 | [Test] 33 | public void TestValidateValueWhenNoProtocol() 34 | { 35 | var result = validator.IsValid("github.com"); 36 | 37 | Assert.IsTrue(result); 38 | } 39 | 40 | [Test] 41 | public void TestValidateValueWhenInvalidUrl() 42 | { 43 | var result = validator.IsValid("hello world"); 44 | 45 | Assert.IsFalse(result); 46 | } 47 | 48 | [Test] 49 | public void TestValidateValueWhenNull() 50 | { 51 | var result = validator.IsValid(null); 52 | 53 | Assert.IsFalse(result); 54 | } 55 | 56 | [Test] 57 | public void TestValidateValueWhenEmpty() 58 | { 59 | var result = validator.IsValid(string.Empty); 60 | 61 | Assert.IsFalse(result); 62 | } 63 | 64 | [Test] 65 | public void TestForDocumentation() 66 | { 67 | var template = "Server: { ServerUrl : IsLooseAbsoluteUrl, EOL }"; 68 | var input = "Server: Not Specified\nServer: www.server.com"; 69 | 70 | var result = new Tokenizer().Tokenize(template, input); 71 | 72 | Assert.AreEqual("www.server.com", result.First("ServerUrl")); 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Validators/IsLooseUrlValidatorTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | 3 | namespace Tokens.Validators 4 | { 5 | [TestFixture] 6 | public class IsLooseUrlValidatorTests 7 | { 8 | private IsLooseUrlValidator validator; 9 | 10 | [SetUp] 11 | public void SetUp() 12 | { 13 | validator = new IsLooseUrlValidator(); 14 | } 15 | 16 | [Test] 17 | public void TestValidateValueWhenHttp() 18 | { 19 | var result = validator.IsValid("http://github.com"); 20 | 21 | Assert.IsTrue(result); 22 | } 23 | 24 | [Test] 25 | public void TestValidateValueWhenHttps() 26 | { 27 | var result = validator.IsValid("https://github.com"); 28 | 29 | Assert.IsTrue(result); 30 | } 31 | 32 | [Test] 33 | public void TestValidateValueWhenNoProtocol() 34 | { 35 | var result = validator.IsValid("github.com"); 36 | 37 | Assert.IsTrue(result); 38 | } 39 | 40 | [Test] 41 | public void TestValidateValueWhenRelativeUrl() 42 | { 43 | var result = validator.IsValid("/foo/bar.html"); 44 | 45 | Assert.IsTrue(result); 46 | } 47 | 48 | [Test] 49 | public void TestValidateValueWhenInvalidUrl() 50 | { 51 | var result = validator.IsValid("hello world"); 52 | 53 | Assert.IsFalse(result); 54 | } 55 | 56 | [Test] 57 | public void TestValidateValueWhenNull() 58 | { 59 | var result = validator.IsValid(null); 60 | 61 | Assert.IsFalse(result); 62 | } 63 | 64 | [Test] 65 | public void TestValidateValueWhenEmpty() 66 | { 67 | var result = validator.IsValid(string.Empty); 68 | 69 | Assert.IsFalse(result); 70 | } 71 | 72 | [Test] 73 | public void TestForDocumentation() 74 | { 75 | var template = "Server: { ServerUrl : IsLooseUrl, EOL }"; 76 | var input = "Server: Not specified\nServer: www.server.com"; 77 | 78 | var result = new Tokenizer().Tokenize(template, input); 79 | 80 | Assert.AreEqual("www.server.com", result.First("ServerUrl")); 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Validators/IsNotEmptyValidatorTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | 3 | namespace Tokens.Validators 4 | { 5 | [TestFixture] 6 | public class IsNotEmptyValidatorTests 7 | { 8 | private IsNotEmptyValidator validator; 9 | 10 | [SetUp] 11 | public void SetUp() 12 | { 13 | validator = new IsNotEmptyValidator(); 14 | } 15 | 16 | [Test] 17 | public void TestValidateValueWhenValid() 18 | { 19 | var result = validator.IsValid("hello world"); 20 | 21 | Assert.IsTrue(result); 22 | } 23 | 24 | [Test] 25 | public void TestValidateValueWhenNull() 26 | { 27 | var result = validator.IsValid(null); 28 | 29 | Assert.IsFalse(result); 30 | } 31 | 32 | [Test] 33 | public void TestValidateValueWhenEmpty() 34 | { 35 | var result = validator.IsValid(string.Empty); 36 | 37 | Assert.IsFalse(result); 38 | } 39 | 40 | [Test] 41 | public void TestForDocumentation() 42 | { 43 | var template = "Middle Name: { MiddleName : IsNotEmpty, EOL }"; 44 | var input = "Middle Name:\nMiddle Name: Charles"; 45 | 46 | var result = new Tokenizer().Tokenize(template, input); 47 | 48 | Assert.AreEqual("Charles", result.First("MiddleName")); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Validators/IsNotValidatorTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | 3 | namespace Tokens.Validators 4 | { 5 | [TestFixture] 6 | public class IsNotValidatorTests 7 | { 8 | private IsNotValidator validator; 9 | 10 | [SetUp] 11 | public void SetUp() 12 | { 13 | validator = new IsNotValidator(); 14 | } 15 | 16 | [Test] 17 | public void TestValidateValueWhenInvalid() 18 | { 19 | var result = validator.IsValid("hello world", "hello world"); 20 | 21 | Assert.IsFalse(result); 22 | } 23 | 24 | [Test] 25 | public void TestValidateValueWhenValid() 26 | { 27 | var result = validator.IsValid("hello world", "hello"); 28 | 29 | Assert.IsTrue(result); 30 | } 31 | 32 | [Test] 33 | public void TestValidateValueWhenNull() 34 | { 35 | var result = validator.IsValid(null, "hello"); 36 | 37 | Assert.IsTrue(result); 38 | } 39 | 40 | [Test] 41 | public void TestValidateValueWhenEmpty() 42 | { 43 | var result = validator.IsValid(string.Empty, "hello"); 44 | 45 | Assert.IsTrue(result); 46 | } 47 | 48 | [Test] 49 | public void TestForDocumentation() 50 | { 51 | var template = "Address: { Address : IsNot('N/A'), EOL }"; 52 | var input = "Address: N/A\nAddress: 10 Acacia Avenue"; 53 | 54 | var result = new Tokenizer().Tokenize(template, input); 55 | 56 | Assert.AreEqual("10 Acacia Avenue", result.First("Address")); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Validators/IsNumericValidatorTests.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using NUnit.Framework; 3 | 4 | namespace Tokens.Validators 5 | { 6 | [TestFixture] 7 | public class IsNumericValidatorTests 8 | { 9 | private IsNumericValidator validator; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | SerilogConfig.Init(); 15 | 16 | validator = new IsNumericValidator(); 17 | } 18 | 19 | [Test] 20 | public void TestValidateValueWhenNumericInteger() 21 | { 22 | var result = validator.IsValid("100"); 23 | 24 | Assert.IsTrue(result); 25 | } 26 | 27 | [Test] 28 | public void TestValidateValueWhenNumericFloat() 29 | { 30 | var result = validator.IsValid("10.0"); 31 | 32 | Assert.IsTrue(result); 33 | } 34 | 35 | [Test] 36 | public void TestValidateValueWhenNotNumeric() 37 | { 38 | var result = validator.IsValid("hello world"); 39 | 40 | Assert.IsFalse(result); 41 | } 42 | 43 | [Test] 44 | public void TestValidateValueWhenNull() 45 | { 46 | var result = validator.IsValid(null); 47 | 48 | Assert.IsFalse(result); 49 | } 50 | 51 | [Test] 52 | public void TestNotValidator() 53 | { 54 | var pattern = @"Age: { Age : !IsNumeric }"; 55 | var input = "Age: ten"; 56 | 57 | var result = new Tokenizer().Tokenize(pattern, input); 58 | 59 | Assert.AreEqual("ten", result.First("Age")); 60 | } 61 | 62 | [Test] 63 | public void TestForDocumentation() 64 | { 65 | var template = "Age: { Age : IsNumeric }"; 66 | var input = "Age: Ten Age: 10"; 67 | 68 | var result = new Tokenizer().Tokenize(template, input); 69 | 70 | Assert.AreEqual("10", result.First("Age")); 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Validators/IsPhoneNumberValidatorTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | 3 | namespace Tokens.Validators 4 | { 5 | [TestFixture] 6 | public class IsPhoneNumberValidatorTests 7 | { 8 | private IsPhoneNumberValidator validator; 9 | 10 | [SetUp] 11 | public void SetUp() 12 | { 13 | validator = new IsPhoneNumberValidator(); 14 | } 15 | 16 | [Test] 17 | public void TestValidateValueWhenValidUk() 18 | { 19 | var result = validator.IsValid("01603 123123"); 20 | 21 | Assert.IsTrue(result); 22 | } 23 | 24 | [Test] 25 | public void TestValidateValueWhenValidUkWithCountryCode() 26 | { 27 | var result = validator.IsValid("+44 (0) 1603 123123"); 28 | 29 | Assert.IsTrue(result); 30 | } 31 | 32 | [Test] 33 | public void TestValidateValueWhenValidWithDots() 34 | { 35 | var result = validator.IsValid("+44.1603.123123"); 36 | 37 | Assert.IsTrue(result); 38 | } 39 | 40 | [Test] 41 | public void TestValidateValueWhenValidWithDashes() 42 | { 43 | var result = validator.IsValid("+44-1603-123123"); 44 | 45 | Assert.IsTrue(result); 46 | } 47 | [Test] 48 | public void TestValidateValueWhenValidUkWithNoAreaCode() 49 | { 50 | var result = validator.IsValid("123123"); 51 | 52 | Assert.IsTrue(result); 53 | } 54 | 55 | [Test] 56 | public void TestValidateValueWhenInvalid() 57 | { 58 | var result = validator.IsValid("hello world"); 59 | 60 | Assert.IsFalse(result); 61 | } 62 | 63 | [Test] 64 | public void TestValidateValueWhenInvalidWithNumber() 65 | { 66 | var result = validator.IsValid("hello world 0123456789"); 67 | 68 | Assert.IsFalse(result); 69 | } 70 | 71 | [Test] 72 | public void TestValidateValueWhenTooShort() 73 | { 74 | var result = validator.IsValid("12345"); 75 | 76 | Assert.IsFalse(result); 77 | } 78 | 79 | [Test] 80 | public void TestValidateValueWhenNull() 81 | { 82 | var result = validator.IsValid(null); 83 | 84 | Assert.IsFalse(result); 85 | } 86 | 87 | [Test] 88 | public void TestValidateValueWhenEmpty() 89 | { 90 | var result = validator.IsValid(string.Empty); 91 | 92 | Assert.IsFalse(result); 93 | } 94 | 95 | [Test] 96 | public void TestForDocumentation() 97 | { 98 | var template = "Phone: { Phone : IsPhoneNumber }"; 99 | var input = "Phone: Disconnected Phone: +44 (0) 1603 555-1234"; 100 | 101 | var result = new Tokenizer().Tokenize(template, input); 102 | 103 | Assert.AreEqual("+44 (0) 1603 555-1234", result.First("Phone")); 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Validators/IsUrlValidatorTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | 3 | namespace Tokens.Validators 4 | { 5 | [TestFixture] 6 | public class IsUrlValidatorTests 7 | { 8 | private IsUrlValidator validator; 9 | 10 | [SetUp] 11 | public void SetUp() 12 | { 13 | validator = new IsUrlValidator(); 14 | } 15 | 16 | [Test] 17 | public void TestValidateValueWhenHttp() 18 | { 19 | var result = validator.IsValid("http://github.com"); 20 | 21 | Assert.IsTrue(result); 22 | } 23 | 24 | [Test] 25 | public void TestValidateValueWhenHttps() 26 | { 27 | var result = validator.IsValid("https://github.com"); 28 | 29 | Assert.IsTrue(result); 30 | } 31 | 32 | [Test] 33 | public void TestValidateValueWhenInvalidUrl() 34 | { 35 | var result = validator.IsValid("hello world"); 36 | 37 | Assert.IsFalse(result); 38 | } 39 | 40 | [Test] 41 | public void TestValidateValueWhenNull() 42 | { 43 | var result = validator.IsValid(null); 44 | 45 | Assert.IsFalse(result); 46 | } 47 | 48 | [Test] 49 | public void TestValidateValueWhenEmpty() 50 | { 51 | var result = validator.IsValid(string.Empty); 52 | 53 | Assert.IsFalse(result); 54 | } 55 | 56 | [Test] 57 | public void TestForDocumentation() 58 | { 59 | var template = "Server: { ServerUrl : IsUrl, EOL }"; 60 | var input = "Server: 192.168.1.1\nServer: http://www.server.com"; 61 | 62 | var result = new Tokenizer().Tokenize(template, input); 63 | 64 | Assert.AreEqual("http://www.server.com", result.First("ServerUrl")); 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Validators/MaxLengthValidatorTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Validators 5 | { 6 | [TestFixture] 7 | public class MaxLengthValidatorTests 8 | { 9 | private MaxLengthValidator validator; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | validator = new MaxLengthValidator(); 15 | } 16 | 17 | [Test] 18 | public void TestValidMaximumLengthWhenValid() 19 | { 20 | var result = validator.IsValid("hello", "100"); 21 | 22 | Assert.IsTrue(result); 23 | } 24 | 25 | [Test] 26 | public void TestValidMaximumLengthWhenInvalid() 27 | { 28 | var result = validator.IsValid("hello world", "5"); 29 | 30 | Assert.IsFalse(result); 31 | } 32 | 33 | [Test] 34 | public void TestValidMaximumLengthWhenNoParameters() 35 | { 36 | Assert.Throws(() => validator.IsValid("hello world")); 37 | } 38 | 39 | [Test] 40 | public void TestValidMaximumLengthWhenParametersNotAnInteger() 41 | { 42 | Assert.Throws(() => validator.IsValid("hello world", "hello")); 43 | } 44 | 45 | [Test] 46 | public void TestForDocumentation() 47 | { 48 | var template = "Zip: { ZipCode : MaxLength(5) }"; 49 | var input = "Zip: 123456 Zip: 78912"; 50 | 51 | var result = new Tokenizer().Tokenize(template, input); 52 | 53 | Assert.AreEqual("78912", result.First("ZipCode")); 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Validators/MinLengthValidatorTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Validators 5 | { 6 | [TestFixture] 7 | public class MinLengthValidatorTests 8 | { 9 | private MinLengthValidator validator; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | validator = new MinLengthValidator(); 15 | } 16 | 17 | [Test] 18 | public void TestValidMinimumLengthWhenValid() 19 | { 20 | var result = validator.IsValid("hello", "3"); 21 | 22 | Assert.IsTrue(result); 23 | } 24 | 25 | [Test] 26 | public void TestValidMinimumLengthWhenInvalid() 27 | { 28 | var result = validator.IsValid("hello world", "255"); 29 | 30 | Assert.IsFalse(result); 31 | } 32 | 33 | [Test] 34 | public void TestValidMinimumLengthWhenNoParameters() 35 | { 36 | Assert.Throws(() => validator.IsValid("hello world")); 37 | } 38 | 39 | [Test] 40 | public void TestValidMinimumLengthWhenParametersNotAnInteger() 41 | { 42 | Assert.Throws(() => validator.IsValid("hello world", "hello")); 43 | } 44 | 45 | [Test] 46 | public void TestForDocumentation() 47 | { 48 | var template = "Zip: { ZipCode : MinLength(5), EOL }"; 49 | var input = "Zip: 123\nZip: 45678"; 50 | 51 | var result = new Tokenizer().Tokenize(template, input); 52 | 53 | Assert.AreEqual("45678", result.First("ZipCode")); 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /Tokenizer.Tests/Validators/StartsWithValidatorTests.cs: -------------------------------------------------------------------------------- 1 | using NUnit.Framework; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Validators 5 | { 6 | [TestFixture] 7 | public class StartsWithValidatorTests 8 | { 9 | private StartsWithValidator validator; 10 | 11 | [SetUp] 12 | public void SetUp() 13 | { 14 | validator = new StartsWithValidator(); 15 | } 16 | 17 | [Test] 18 | public void TestValidateValueWhenTrue() 19 | { 20 | var result = validator.IsValid("hello world", "hello"); 21 | 22 | Assert.IsTrue(result); 23 | } 24 | 25 | [Test] 26 | public void TestValidateValueWhenFalse() 27 | { 28 | var result = validator.IsValid("hello world", "world"); 29 | 30 | Assert.IsFalse(result); 31 | } 32 | 33 | [Test] 34 | public void TestValidateValueWhenMissingArgument() 35 | { 36 | Assert.Throws(() => validator.IsValid("hello world")); 37 | } 38 | 39 | [Test] 40 | public void TestValidateValueWhenNull() 41 | { 42 | var result = validator.IsValid(null); 43 | 44 | Assert.IsFalse(result); 45 | } 46 | 47 | [Test] 48 | public void TestValidateValueWhenEmpty() 49 | { 50 | var result = validator.IsValid(string.Empty); 51 | 52 | Assert.IsFalse(result); 53 | } 54 | 55 | [Test] 56 | public void TestForDocumentation() 57 | { 58 | var template = "Ip: { InternalIpAddress : StartsWith('192') }"; 59 | var input = "Ip: 80.34.123.45 Ip: 192.168.1.1"; 60 | 61 | var result = new Tokenizer().Tokenize(template, input); 62 | 63 | Assert.AreEqual("192.168.1.1", result.First("InternalIpAddress")); 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /Tokenizer.Tests/WindowsClipboard.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.ComponentModel; 3 | using System.Runtime.InteropServices; 4 | using System.Threading; 5 | 6 | namespace Tokens 7 | { 8 | static class WindowsClipboard 9 | { 10 | public static void SetText(string text) 11 | { 12 | OpenClipboard(); 13 | 14 | EmptyClipboard(); 15 | IntPtr hGlobal = default; 16 | try 17 | { 18 | var bytes = (text.Length + 1) * 2; 19 | hGlobal = Marshal.AllocHGlobal(bytes); 20 | 21 | if (hGlobal == default) 22 | { 23 | ThrowWin32(); 24 | } 25 | 26 | var target = GlobalLock(hGlobal); 27 | 28 | if (target == default) 29 | { 30 | ThrowWin32(); 31 | } 32 | 33 | try 34 | { 35 | Marshal.Copy(text.ToCharArray(), 0, target, text.Length); 36 | } 37 | finally 38 | { 39 | GlobalUnlock(target); 40 | } 41 | 42 | if (SetClipboardData(cfUnicodeText, hGlobal) == default) 43 | { 44 | ThrowWin32(); 45 | } 46 | 47 | hGlobal = default; 48 | } 49 | finally 50 | { 51 | if (hGlobal != default) 52 | { 53 | Marshal.FreeHGlobal(hGlobal); 54 | } 55 | 56 | CloseClipboard(); 57 | } 58 | } 59 | 60 | public static void OpenClipboard() 61 | { 62 | var num = 10; 63 | while (true) 64 | { 65 | if (OpenClipboard(default)) 66 | { 67 | break; 68 | } 69 | 70 | if (--num == 0) 71 | { 72 | ThrowWin32(); 73 | } 74 | 75 | Thread.Sleep(100); 76 | } 77 | } 78 | 79 | const uint cfUnicodeText = 13; 80 | 81 | static void ThrowWin32() 82 | { 83 | throw new Win32Exception(Marshal.GetLastWin32Error()); 84 | } 85 | 86 | [DllImport("kernel32.dll", SetLastError = true)] 87 | static extern IntPtr GlobalLock(IntPtr hMem); 88 | 89 | [DllImport("kernel32.dll", SetLastError = true)] 90 | [return: MarshalAs(UnmanagedType.Bool)] 91 | static extern bool GlobalUnlock(IntPtr hMem); 92 | 93 | [DllImport("user32.dll", SetLastError = true)] 94 | [return: MarshalAs(UnmanagedType.Bool)] 95 | static extern bool OpenClipboard(IntPtr hWndNewOwner); 96 | 97 | [DllImport("user32.dll", SetLastError = true)] 98 | [return: MarshalAs(UnmanagedType.Bool)] 99 | static extern bool CloseClipboard(); 100 | 101 | [DllImport("user32.dll", SetLastError = true)] 102 | static extern IntPtr SetClipboardData(uint uFormat, IntPtr data); 103 | 104 | [DllImport("user32.dll")] 105 | static extern bool EmptyClipboard(); 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /Tokenizer.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 15 4 | VisualStudioVersion = 15.0.28010.2036 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tokenizer", "Tokenizer\Tokenizer.csproj", "{84646AE1-9D18-4E43-95A6-0BDD204A5D67}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tokenizer.Tests", "Tokenizer.Tests\Tokenizer.Tests.csproj", "{FDE2BD45-D224-47C7-A04D-37C0D6D882C4}" 9 | EndProject 10 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{03008BA1-74FF-4DD9-BE0A-71AD03C4F969}" 11 | ProjectSection(SolutionItems) = preProject 12 | Appveyor.yml = Appveyor.yml 13 | EndProjectSection 14 | EndProject 15 | Global 16 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 17 | Debug|Any CPU = Debug|Any CPU 18 | Release|Any CPU = Release|Any CPU 19 | EndGlobalSection 20 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 21 | {84646AE1-9D18-4E43-95A6-0BDD204A5D67}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 22 | {84646AE1-9D18-4E43-95A6-0BDD204A5D67}.Debug|Any CPU.Build.0 = Debug|Any CPU 23 | {84646AE1-9D18-4E43-95A6-0BDD204A5D67}.Release|Any CPU.ActiveCfg = Release|Any CPU 24 | {84646AE1-9D18-4E43-95A6-0BDD204A5D67}.Release|Any CPU.Build.0 = Release|Any CPU 25 | {FDE2BD45-D224-47C7-A04D-37C0D6D882C4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 26 | {FDE2BD45-D224-47C7-A04D-37C0D6D882C4}.Debug|Any CPU.Build.0 = Debug|Any CPU 27 | {FDE2BD45-D224-47C7-A04D-37C0D6D882C4}.Release|Any CPU.ActiveCfg = Release|Any CPU 28 | {FDE2BD45-D224-47C7-A04D-37C0D6D882C4}.Release|Any CPU.Build.0 = Release|Any CPU 29 | EndGlobalSection 30 | GlobalSection(SolutionProperties) = preSolution 31 | HideSolutionNode = FALSE 32 | EndGlobalSection 33 | GlobalSection(ExtensibilityGlobals) = postSolution 34 | SolutionGuid = {C47F1313-4F0E-4747-B60F-C6465C1CA415} 35 | EndGlobalSection 36 | EndGlobal 37 | -------------------------------------------------------------------------------- /Tokenizer/CandidateTokenList.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Text; 3 | using Tokens.Enumerators; 4 | 5 | namespace Tokens 6 | { 7 | /// 8 | /// Holds a list of candidate tokens to match during a Tokenize operation. 9 | /// 10 | internal class CandidateTokenList 11 | { 12 | private readonly List tokens = new List(); 13 | 14 | public void Add(Token token) 15 | { 16 | if (tokens.Count == 0) 17 | { 18 | Preamble = token.Preamble; 19 | TerminateOnNewLine = token.TerminateOnNewLine; 20 | IsNullToken = string.IsNullOrWhiteSpace(token.Name); 21 | tokens.Add(token); 22 | } 23 | else 24 | { 25 | tokens.Add(token); 26 | } 27 | } 28 | 29 | public void AddRange(IEnumerable tokens) 30 | { 31 | foreach (var token in tokens) 32 | { 33 | Add(token); 34 | } 35 | } 36 | 37 | public void Clear() 38 | { 39 | Preamble = null; 40 | tokens.Clear(); 41 | } 42 | 43 | public bool TryAssign(object target, StringBuilder value, TokenizerOptions options, FileLocation location, out Token assigned, out object assignedValue) 44 | { 45 | assigned = null; 46 | assignedValue = null; 47 | 48 | var valueString = value.ToString(); 49 | 50 | foreach (var token in tokens) 51 | { 52 | if (token.Assign(target, valueString, options, location, out assignedValue)) 53 | { 54 | assigned = token; 55 | 56 | return true; 57 | } 58 | } 59 | 60 | return false; 61 | } 62 | 63 | public bool CanAnyAssign(string value) 64 | { 65 | foreach (var token in tokens) 66 | { 67 | if (token.CanAssign(value)) 68 | { 69 | return true; 70 | } 71 | } 72 | 73 | return false; 74 | } 75 | 76 | public bool Any => Count > 0; 77 | 78 | public int Count => tokens.Count; 79 | 80 | public string Preamble { get; private set; } 81 | 82 | public bool TerminateOnNewLine { get; private set; } 83 | 84 | public bool IsNullToken { get; private set; } 85 | 86 | public IList Tokens => tokens; 87 | 88 | public void Remove(Token token) 89 | { 90 | tokens.Remove(token); 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /Tokenizer/Enumerators/FileLocation.cs: -------------------------------------------------------------------------------- 1 | namespace Tokens.Enumerators 2 | { 3 | /// 4 | /// Represents a location in a text file 5 | /// 6 | public class FileLocation 7 | { 8 | private int newLineCounter = 0; 9 | 10 | /// 11 | /// The column number 12 | /// 13 | public int Column { get; private set; } 14 | 15 | /// 16 | /// The line number 17 | /// 18 | public int Line { get; private set; } 19 | 20 | /// 21 | /// The paragraph number 22 | /// 23 | public int Paragraph { get; private set; } 24 | 25 | /// 26 | /// Creates a new instance of this class 27 | /// 28 | public FileLocation() 29 | { 30 | Column = 0; 31 | Line = 1; 32 | Paragraph = 1; 33 | } 34 | 35 | /// 36 | /// Increments the column count 37 | /// 38 | internal void Increment(string value) 39 | { 40 | if (value == "\r") return; 41 | if (value == "\n") return; 42 | 43 | if (string.IsNullOrWhiteSpace(value) == false) 44 | { 45 | newLineCounter = 0; 46 | } 47 | 48 | Column++; 49 | } 50 | 51 | /// 52 | /// Increments the line and resets the column counts 53 | /// 54 | internal void NewLine() 55 | { 56 | if (Column == 1) 57 | { 58 | if (newLineCounter == 1) 59 | { 60 | Paragraph++; 61 | } 62 | } 63 | 64 | Column = 1; 65 | Line++; 66 | newLineCounter++; 67 | } 68 | 69 | /// 70 | /// Resets the counts 71 | /// 72 | internal void Reset() 73 | { 74 | Column = 0; 75 | Line = 1; 76 | Paragraph = 1; 77 | } 78 | 79 | /// 80 | /// Clones this instance 81 | /// 82 | /// 83 | public FileLocation Clone() 84 | { 85 | return new FileLocation 86 | { 87 | Column = Column, 88 | Line = Line, 89 | Paragraph = Paragraph 90 | }; 91 | } 92 | 93 | /// 94 | /// Returns a string representation of this instance 95 | /// 96 | /// 97 | public override string ToString() 98 | { 99 | return $"Ln: {Line} Col: {Column} Para: {Paragraph}"; 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /Tokenizer/Enumerators/PreTokenEnumerator.cs: -------------------------------------------------------------------------------- 1 | using System.Text; 2 | 3 | namespace Tokens.Enumerators 4 | { 5 | internal class PreTokenEnumerator 6 | { 7 | private readonly string pattern; 8 | private readonly int patternLength; 9 | 10 | private int currentLocation; 11 | private bool resetNextLine; 12 | 13 | public PreTokenEnumerator(string pattern) 14 | { 15 | this.pattern = pattern; 16 | 17 | if (string.IsNullOrEmpty(pattern)) 18 | { 19 | patternLength = 0; 20 | } 21 | else 22 | { 23 | patternLength = pattern.Length; 24 | } 25 | 26 | currentLocation = 0; 27 | Location = new FileLocation(); 28 | } 29 | 30 | public bool IsEmpty => currentLocation >= patternLength; 31 | 32 | public FileLocation Location { get; } 33 | 34 | public string Next() 35 | { 36 | if (IsEmpty) return string.Empty; 37 | 38 | var next = pattern.Substring(currentLocation, 1); 39 | currentLocation++; 40 | 41 | if (resetNextLine) 42 | { 43 | Location.NewLine(); 44 | resetNextLine = false; 45 | } 46 | else 47 | { 48 | Location.Increment(next); 49 | } 50 | 51 | if (next == "\n") 52 | { 53 | resetNextLine = true; 54 | } 55 | 56 | return next; 57 | } 58 | 59 | public string Next(int length) 60 | { 61 | var sb = new StringBuilder(); 62 | 63 | for (var i = 0; i < length; i++) 64 | { 65 | sb.Append(Next()); 66 | } 67 | 68 | return sb.ToString(); 69 | } 70 | 71 | public string Peek() 72 | { 73 | if (IsEmpty) return string.Empty; 74 | 75 | return pattern.Substring(currentLocation, 1); 76 | } 77 | 78 | public string Peek(int length) 79 | { 80 | if (IsEmpty) return string.Empty; 81 | 82 | var different = (currentLocation + length) - patternLength; 83 | if (different > 0) length -= different; 84 | 85 | if (length < 1) return string.Empty; 86 | 87 | return pattern.Substring(currentLocation, length); 88 | } 89 | } 90 | } -------------------------------------------------------------------------------- /Tokenizer/Exceptions/ParsingException.cs: -------------------------------------------------------------------------------- 1 | using System.Text; 2 | using Tokens.Enumerators; 3 | 4 | namespace Tokens.Exceptions 5 | { 6 | public class ParsingException : TokenizerException 7 | { 8 | internal ParsingException(string message, PreTokenEnumerator enumerator) : this(message, enumerator.Location) 9 | { 10 | } 11 | 12 | public ParsingException(string message, FileLocation location) : base(message) 13 | { 14 | Column = location.Column; 15 | Line = location.Line; 16 | } 17 | 18 | public int Line { get; set; } 19 | 20 | public int Column { get; set; } 21 | 22 | public override string Message 23 | { 24 | get 25 | { 26 | var sb = new StringBuilder(); 27 | 28 | sb.AppendLine(base.Message); 29 | sb.AppendLine(); 30 | sb.AppendLine($"Column: {Column}"); 31 | sb.AppendLine($"Line: {Line}"); 32 | 33 | return sb.ToString(); 34 | } 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /Tokenizer/Exceptions/TokenAssignmentException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Tokens.Exceptions 4 | { 5 | public class TokenAssignmentException : TokenizerException 6 | { 7 | public Token Token { get; } 8 | 9 | public TokenAssignmentException(Token token, string message) : base(message) 10 | { 11 | Token = token; 12 | } 13 | 14 | public TokenAssignmentException(Token token, Exception innerException) : base($"Unable to assign: {token.Name}", innerException) 15 | { 16 | Token = token; 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /Tokenizer/Exceptions/TokenMatcherException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Tokens.Exceptions 4 | { 5 | public class TokenMatcherException : TokenizerException 6 | { 7 | public TokenMatcherException(string message) : base(message) 8 | { 9 | } 10 | 11 | public TokenMatcherException(string message, Exception innerException) : base(message, innerException) 12 | { 13 | } 14 | 15 | public Template Template { get; set; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Tokenizer/Exceptions/TokenizerException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Tokens.Exceptions 4 | { 5 | /// 6 | /// Thrown by the when an exception occurs 7 | /// 8 | public class TokenizerException : Exception 9 | { 10 | /// 11 | /// Initializes a new instance of the class. 12 | /// 13 | /// The message that describes the error. 14 | public TokenizerException(string message) : base(message) 15 | { 16 | } 17 | 18 | /// 19 | /// Initializes a new instance of the class. 20 | /// 21 | public TokenizerException(string message, Exception innerException) : base(message, innerException) 22 | { 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /Tokenizer/Exceptions/TypeConversionException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Tokens.Exceptions 4 | { 5 | /// 6 | /// Thrown when a value can't be converted into it's destination type 7 | /// 8 | public class TypeConversionException : TokenizerException 9 | { 10 | public TypeConversionException(string message) : base(message) 11 | { 12 | } 13 | 14 | public TypeConversionException(string message, Exception innerException) : base(message, innerException) 15 | { 16 | } 17 | 18 | /// 19 | /// The target type to be converted to 20 | /// 21 | public Type TargetType { get; set; } 22 | 23 | /// 24 | /// The value being converted 25 | /// 26 | public object Value { get; set; } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /Tokenizer/Exceptions/ValidationException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Tokens.Exceptions 4 | { 5 | /// 6 | /// Thrown when an validation can't be mapped from a pattern 7 | /// 8 | public class ValidationException : Exception 9 | { 10 | /// 11 | /// Initializes a new instance of the class. 12 | /// 13 | /// The message that describes the error. 14 | public ValidationException(string message) : base(message) 15 | { 16 | } 17 | 18 | /// 19 | /// Initializes a new instance of the class. 20 | /// 21 | /// The message. 22 | /// The exception. 23 | public ValidationException(string message, Exception exception) : base(message, exception) 24 | { 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /Tokenizer/Hint.cs: -------------------------------------------------------------------------------- 1 | namespace Tokens 2 | { 3 | /// 4 | /// Defines a string of text that can occur in a template's input. 5 | /// A hint can optionally be required to be present. 6 | /// Hints are used when determining whether the input is valid, and to determine 7 | /// tne best matched template for a given input. 8 | /// 9 | public class Hint 10 | { 11 | /// 12 | /// The text to appear in the input 13 | /// 14 | public string Text { get; set; } 15 | 16 | /// 17 | /// If true then this hint must appear in the input in order for the 18 | /// to be considered successfully matched. 19 | /// 20 | public bool Optional { get; set; } 21 | 22 | /// 23 | /// Clones this instance 24 | /// 25 | public Hint Clone() 26 | { 27 | return new Hint 28 | { 29 | Text = Text, 30 | Optional = Optional 31 | }; 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Tokenizer/HintMatch.cs: -------------------------------------------------------------------------------- 1 | using Tokens.Enumerators; 2 | 3 | namespace Tokens 4 | { 5 | public class HintMatch 6 | { 7 | public string Text { get; set; } 8 | 9 | public bool Optional { get; set; } 10 | 11 | public FileLocation Location { get; set; } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /Tokenizer/HintResult.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using Tokens.Enumerators; 4 | 5 | namespace Tokens 6 | { 7 | /// 8 | /// Contains the results of processing a for 9 | /// strings. 10 | /// 11 | public class HintResult 12 | { 13 | public HintResult() 14 | { 15 | Matches = new List(); 16 | Misses = new List(); 17 | } 18 | 19 | /// 20 | /// Gets the hint matches 21 | /// 22 | public IList Matches { get; } 23 | 24 | /// 25 | /// Gets the hint misses 26 | /// 27 | public IList Misses { get; } 28 | 29 | internal bool AddMatch(Hint hint, TokenEnumerator enumerator) 30 | { 31 | if (Matches.Any(m => m.Text == hint.Text)) return false; 32 | 33 | Matches.Add(new HintMatch 34 | { 35 | Text = hint.Text, 36 | Optional = hint.Optional, 37 | Location = enumerator.Location.Clone() 38 | }); 39 | 40 | return true; 41 | } 42 | 43 | internal bool AddMiss(Hint hint) 44 | { 45 | if (Misses.Any(m => m.Text == hint.Text) || 46 | Matches.Any(m => m.Text == hint.Text)) return false; 47 | 48 | Misses.Add(hint.Clone()); 49 | 50 | return true; 51 | } 52 | 53 | public bool HasMissingRequiredHints => Misses.Any(m => m.Optional == false); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /Tokenizer/ITokenDecorator.cs: -------------------------------------------------------------------------------- 1 | namespace Tokens 2 | { 3 | /// 4 | /// Defines a class that can modify the matching or transformation of text 5 | /// on a . 6 | /// 7 | public interface ITokenDecorator 8 | { 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Tokenizer/LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 Chris Wood 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /Tokenizer/Logging/LogExtensions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.ComponentModel; 4 | using System.Text; 5 | 6 | namespace Tokens.Logging 7 | { 8 | static class VerboseLogExtensions 9 | { 10 | public static void Verbose(this ILog log, string message, params object[] args) 11 | { 12 | log.TraceFormat(string.Concat(LogIndentation.Indentation, message), args); 13 | } 14 | 15 | public static void Verbose(this ILog log, Exception exception, string message, params object[] args) 16 | { 17 | log.TraceException(string.Concat(LogIndentation.Indentation, message), exception, args); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /Tokenizer/Logging/LogIndentation.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Threading; 3 | 4 | namespace Tokens.Logging 5 | { 6 | internal class LogIndentation : IDisposable 7 | { 8 | private static int width; 9 | 10 | public LogIndentation() 11 | { 12 | Interlocked.Add(ref width, 2); 13 | } 14 | 15 | 16 | public void Dispose() 17 | { 18 | Interlocked.Add(ref width, -2); 19 | } 20 | 21 | public static string Indentation => new string(' ', Math.Abs(width)); 22 | } 23 | } -------------------------------------------------------------------------------- /Tokenizer/Match.cs: -------------------------------------------------------------------------------- 1 | using Tokens.Enumerators; 2 | 3 | namespace Tokens 4 | { 5 | /// 6 | /// Represent a match in a 7 | /// 8 | public class Match 9 | { 10 | /// 11 | /// Gets or sets the matched 12 | /// 13 | public Token Token { get; set; } 14 | 15 | /// 16 | /// Gets or sets the value of the match 17 | /// 18 | public object Value { get; set; } 19 | 20 | /// 21 | /// Gets the location in the file where this match was made 22 | /// 23 | public FileLocation Location { get; set; } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /Tokenizer/PreTemplate.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace Tokens 4 | { 5 | /// 6 | /// Holds parsed template input before being transformed into a 7 | /// . 8 | /// 9 | internal class PreTemplate 10 | { 11 | /// 12 | /// Creates a new instance of the class. 13 | /// 14 | public PreTemplate() 15 | { 16 | Tokens = new List(); 17 | Hints = new List(); 18 | Tags = new List(); 19 | } 20 | 21 | /// 22 | /// Holds the that this instance was created with. 23 | /// 24 | public TokenizerOptions Options { get; set; } 25 | 26 | /// 27 | /// Contains a list of objects that were found in the input string 28 | /// 29 | public IList Tokens { get; } 30 | 31 | /// 32 | /// Contains a list of objects that were found in the input string 33 | /// 34 | public IList Hints { get; } 35 | 36 | /// 37 | /// Contains a list of tags that were found in the input string 38 | /// 39 | public IList Tags { get; } 40 | 41 | /// 42 | /// Specifies the name of the template. 43 | /// 44 | public string Name { get; set; } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /Tokenizer/PreToken.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Text; 3 | using Tokens.Enumerators; 4 | 5 | namespace Tokens 6 | { 7 | /// 8 | /// Intermediate data structure that holds the syntactically verified 9 | /// template token data. 10 | /// 11 | internal class PreToken 12 | { 13 | private readonly StringBuilder preamble; 14 | private readonly StringBuilder name; 15 | private readonly StringBuilder value; 16 | 17 | public PreToken() 18 | { 19 | Decorators = new List(); 20 | preamble = new StringBuilder(); 21 | name = new StringBuilder(); 22 | value = new StringBuilder(); 23 | } 24 | 25 | public int Id { get; set; } 26 | 27 | public int DependsOnId { get; set; } 28 | 29 | public string Preamble => preamble.ToString(); 30 | 31 | public string Name => name.ToString(); 32 | 33 | public string Value => value.ToString(); 34 | 35 | public bool Optional { get; set; } 36 | 37 | public bool TerminateOnNewline { get; set; } 38 | 39 | public bool Repeating { get; set; } 40 | 41 | public bool Required { get; set; } 42 | 43 | public bool IsNull { get; set; } 44 | 45 | public bool ConsiderOnce { get; set; } 46 | 47 | public string Content { get; set; } 48 | 49 | public FileLocation Location { get; set; } 50 | 51 | public IList Decorators { get; } 52 | 53 | public void AppendPreamble(string value) 54 | { 55 | if (value == "\r") return; 56 | 57 | preamble.Append(value); 58 | } 59 | 60 | public void AppendName(string value) 61 | { 62 | name.Append(value); 63 | } 64 | 65 | public void AppendValue(string value) 66 | { 67 | this.value.Append(value); 68 | } 69 | 70 | public void AppendDecorators(IEnumerable decorators) 71 | { 72 | if (decorators == null) return; 73 | 74 | foreach (var decorator in decorators) 75 | { 76 | Decorators.Add(decorator); 77 | } 78 | } 79 | 80 | public bool HasValue => value.Length > 0; 81 | 82 | public bool IsFrontMatterToken { get; set; } 83 | 84 | public override string ToString() 85 | { 86 | return Content; 87 | } 88 | 89 | internal void TrimPreambleBeforeNewLine() 90 | { 91 | var preambleContent = preamble.ToString(); 92 | 93 | if (preambleContent.Contains("\n")) 94 | { 95 | var trimmed = preambleContent.Substring(preambleContent.LastIndexOf("\n") + 1); 96 | 97 | preamble.Clear(); 98 | preamble.Append(trimmed); 99 | } 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /Tokenizer/PreTokenDecorator.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Text; 3 | 4 | namespace Tokens 5 | { 6 | internal class PreTokenDecorator 7 | { 8 | private readonly StringBuilder name; 9 | 10 | public PreTokenDecorator() 11 | { 12 | name = new StringBuilder(); 13 | Args = new List(); 14 | } 15 | 16 | public string Name => name.ToString(); 17 | 18 | public IList Args { get; } 19 | 20 | public bool IsNotDecorator { get; set; } 21 | 22 | public void AppendName(string value) 23 | { 24 | name.Append(value); 25 | } 26 | } 27 | } -------------------------------------------------------------------------------- /Tokenizer/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.CompilerServices; 2 | using System.Runtime.InteropServices; 3 | 4 | [assembly: ComVisible(false)] 5 | 6 | // The following GUID is for the ID of the typelib if this project is exposed to COM 7 | [assembly: Guid("4df0ad4e-4904-4f7c-ba79-586b3ba1384b")] 8 | [assembly: InternalsVisibleTo("Tokenizer.Tests")] -------------------------------------------------------------------------------- /Tokenizer/TemplateCollection.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Concurrent; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | 5 | namespace Tokens 6 | { 7 | /// 8 | /// Collection of objects. 9 | /// 10 | public class TemplateCollection 11 | { 12 | private readonly ConcurrentDictionary templates; 13 | 14 | /// 15 | /// Returns the names of the templates in this collection 16 | /// 17 | public IList Names => templates.Keys.ToList(); 18 | 19 | /// 20 | /// Returns the number of templates in this collection 21 | /// 22 | public int Count => templates.Count; 23 | 24 | /// 25 | /// Creates a new instance of the class. 26 | /// 27 | public TemplateCollection() 28 | { 29 | templates = new ConcurrentDictionary(); 30 | } 31 | 32 | /// 33 | /// Adds a template to the collection. 34 | /// If a template with the same name already exists, it will be replaced. 35 | /// 36 | /// 37 | public void Add(Template template) 38 | { 39 | templates.AddOrUpdate(template.Name, template, (key, existing) => template); 40 | } 41 | 42 | /// 43 | /// Tries to get the template with the given name. If the template exists, 44 | /// will return true with the template set as template. 45 | /// If the template doesn't exist, will return false. 46 | /// 47 | public bool TryGet(string name, out Template template) 48 | { 49 | return templates.TryGetValue(name, out template); 50 | } 51 | 52 | /// 53 | /// Gets the template with the given name. If the template doesn't exist, 54 | /// will return null. 55 | /// 56 | public Template Get(string name) 57 | { 58 | return TryGet(name, out var template) ? template : null; 59 | } 60 | 61 | /// 62 | /// Clears all templates from this collection 63 | /// 64 | public void Clear() 65 | { 66 | templates.Clear(); 67 | } 68 | 69 | /// 70 | /// Determines if any templates are in this collection that contain the given 71 | /// tag. 72 | /// 73 | public bool ContainsTag(string tag) 74 | { 75 | foreach (var name in Names) 76 | { 77 | if (!TryGet(name, out var template)) continue; 78 | 79 | if (template.HasTag(tag)) 80 | { 81 | return true; 82 | } 83 | } 84 | 85 | return false; 86 | } 87 | 88 | /// 89 | /// Determines if any templates in this collection contain all the given tags. 90 | /// 91 | public bool ContainsAllTags(params string[] tags) 92 | { 93 | foreach (var name in Names) 94 | { 95 | if (!TryGet(name, out var template)) continue; 96 | 97 | if (template.HasTags(tags)) 98 | { 99 | return true; 100 | } 101 | } 102 | 103 | return false; 104 | } 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /Tokenizer/TokenDecoratorContext.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using Tokens.Transformers; 5 | using Tokens.Validators; 6 | 7 | namespace Tokens 8 | { 9 | /// 10 | /// Contains an instance of a that can perform 11 | /// an operation on a . 12 | /// 13 | public class TokenDecoratorContext 14 | { 15 | public TokenDecoratorContext(Type tokenDecorator) 16 | { 17 | DecoratorType = tokenDecorator; 18 | Parameters = new List(); 19 | } 20 | 21 | /// 22 | /// Specifies the decorator type 23 | /// 24 | public Type DecoratorType { get; } 25 | 26 | /// 27 | /// Creates an instance of the decorator 28 | /// 29 | /// 30 | public ITokenDecorator CreateDecorator() 31 | { 32 | return (ITokenDecorator) Activator.CreateInstance(DecoratorType); 33 | } 34 | 35 | /// 36 | /// Contains the parameters to pass the decorator 37 | /// 38 | public IList Parameters { get; } 39 | 40 | /// 41 | /// Returns true if the decorator is a used to transform 42 | /// the token value. 43 | /// 44 | public bool IsTransformer => typeof(ITokenTransformer).IsAssignableFrom(DecoratorType); 45 | 46 | /// 47 | /// Returns true if the decorator is a used to validate 48 | /// the token value. 49 | /// 50 | public bool IsValidator => typeof(ITokenValidator).IsAssignableFrom(DecoratorType); 51 | 52 | /// 53 | /// Determines if this validator should reverse it's output 54 | /// 55 | public bool IsNotValidator { get; set; } 56 | 57 | /// 58 | /// Transforms the token value. 59 | /// 60 | public bool CanTransform(object value, out object transformed) 61 | { 62 | var instance = (ITokenTransformer) CreateDecorator(); 63 | 64 | return instance.CanTransform(value, Parameters.ToArray(), out transformed); 65 | } 66 | 67 | /// 68 | /// Validates the token value. 69 | /// 70 | public bool Validate(object value) 71 | { 72 | var instance = (ITokenValidator) CreateDecorator(); 73 | 74 | if (IsNotValidator) 75 | { 76 | return !instance.IsValid(value, Parameters.ToArray()); 77 | } 78 | 79 | return instance.IsValid(value, Parameters.ToArray()); 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /Tokenizer/TokenMatcherResult.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | 4 | namespace Tokens 5 | { 6 | /// 7 | /// Contains the result of running a match against multiple 8 | /// objects against an input string with the . 9 | /// 10 | public class TokenMatcherResult 11 | { 12 | public TokenMatcherResult() 13 | { 14 | Results = new List(); 15 | } 16 | 17 | /// 18 | /// Contains the result of processing each against the input text. 19 | /// 20 | public IList Results { get; private set; } 21 | 22 | /// 23 | /// Returns the best matching result 24 | /// 25 | public TokenizeResult BestMatch { get; internal set; } 26 | 27 | public bool Success => BestMatch != null; 28 | 29 | internal TokenizeResult GetBestMatch() => Results 30 | .Where(r => r.Success) 31 | .OrderByDescending(r => r.Hints.Matches.Count) 32 | .ThenByDescending(r => r.Tokens.Matches.Count) 33 | .ThenBy(r => r.Template.Tokens.Count) 34 | .ThenBy(r => r.Template.Name) 35 | .FirstOrDefault(); 36 | } 37 | 38 | public class TokenMatcherResult where T : class, new() 39 | { 40 | public TokenMatcherResult() 41 | { 42 | Results = new List>(); 43 | } 44 | 45 | /// 46 | /// Contains the result of processing each against the input text. 47 | /// 48 | public IList> Results { get; private set; } 49 | 50 | /// 51 | /// Returns the best matching result 52 | /// 53 | public TokenizeResult BestMatch { get; internal set; } 54 | 55 | public bool Success => BestMatch != null; 56 | 57 | internal TokenizeResult GetBestMatch() => Results 58 | .Where(r => r.Success) 59 | .OrderByDescending(r => r.Hints.Matches.Count) 60 | .ThenByDescending(r => r.Tokens.Matches.Count) 61 | .ThenBy(r => r.Template.Tokens.Count) 62 | .ThenBy(r => r.Template.Name) 63 | .FirstOrDefault(); 64 | } 65 | } -------------------------------------------------------------------------------- /Tokenizer/TokenResult.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using Tokens.Enumerators; 5 | 6 | namespace Tokens 7 | { 8 | public class TokenResult 9 | { 10 | public TokenResult() 11 | { 12 | Matches = new List(); 13 | Misses = new List(); 14 | } 15 | 16 | public IList Matches { get; } 17 | 18 | public IList Misses { get; } 19 | 20 | internal void AddMatch(Token token, object value, FileLocation location) 21 | { 22 | if (TryConcatMatch(token, value, location)) return; 23 | 24 | Matches.Add(new Match 25 | { 26 | Token = token, 27 | Value = value, 28 | Location = location.Clone() 29 | }); 30 | } 31 | 32 | private bool TryConcatMatch(Token token, object value, FileLocation location) 33 | { 34 | if (token.Concatenate == false) return false; 35 | 36 | if (Matches.Any(m => m.Token.Name == token.Name) == false) return false; 37 | 38 | var match = Matches.First(m => m.Token.Name == token.Name); 39 | 40 | if (token.CanConcatenate(match.Value, value) == false) return false; 41 | 42 | match.Value = token.ConcatenateValues(match.Value, value, token.ConcatenationString); 43 | 44 | return true; 45 | } 46 | 47 | internal void AddMiss(Token token) 48 | { 49 | Misses.Add(token); 50 | } 51 | 52 | public bool HasMissingRequiredTokens => Misses.Any(m => m.Required); 53 | 54 | public bool HasMatches => Matches.Any(); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /Tokenizer/TokenizeResult.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | using Tokens.Exceptions; 4 | 5 | namespace Tokens 6 | { 7 | /// 8 | /// Holds the result of attempting to parse an input string against a 9 | /// . 10 | /// 11 | public class TokenizeResult : TokenizeResultBase 12 | { 13 | /// 14 | /// Creates a new instance of the class. 15 | /// 16 | public TokenizeResult(Template template) : base(template) 17 | { 18 | } 19 | 20 | /// 21 | /// A dictionary of values extracted from the input string. 22 | /// 23 | public IList Matches => Tokens.Matches; 24 | 25 | public object First(string key) 26 | { 27 | if (Matches.Any(m => m.Token.Name == key) == false) 28 | { 29 | throw new TokenizerException($"Token '{key}' was not found in the input text."); 30 | } 31 | 32 | return Matches.First(m => m.Token.Name == key).Value; 33 | } 34 | 35 | public T First(string key) 36 | { 37 | if (Matches.Any(m => m.Token.Name == key) == false) 38 | { 39 | throw new TokenizerException($"Token '{key}' was not found in the input text."); 40 | } 41 | 42 | return (T) Matches.First(m => m.Token.Name == key).Value; 43 | } 44 | 45 | public object FirstOrDefault(string key) 46 | { 47 | if (Matches.Any(m => m.Token?.Name == key)) 48 | { 49 | return Matches.First(m => m.Token.Name == key).Value; 50 | } 51 | 52 | return null; 53 | } 54 | 55 | public T FirstOrDefault(string key) 56 | { 57 | if (Matches.Any(m => m.Token?.Name == key)) 58 | { 59 | return (T) Matches.First(m => m.Token.Name == key).Value; 60 | } 61 | 62 | return default(T); 63 | } 64 | 65 | public IList All(string key) 66 | { 67 | return Matches 68 | .Where(m => m.Token.Name == key) 69 | .Select(m => m.Value) 70 | .ToList(); 71 | } 72 | 73 | public bool Contains(string key) 74 | { 75 | return Matches.Any(m => m.Token.Name == key); 76 | } 77 | } 78 | 79 | /// 80 | /// Holds the result of attempting to parse an input string against a 81 | /// to generate an object of type . 82 | /// 83 | public class TokenizeResult : TokenizeResultBase where T : class, new() 84 | { 85 | /// 86 | /// Creates a new instance of the class. 87 | /// 88 | public TokenizeResult(Template template) : base(template) 89 | { 90 | Value = new T(); 91 | } 92 | 93 | /// 94 | /// An instance of populated with data from the input string. 95 | /// 96 | public T Value { get; set; } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /Tokenizer/TokenizeResultBase.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | 4 | namespace Tokens 5 | { 6 | /// 7 | /// Base class that holds the result of attempting to parse an input string against a 8 | /// . 9 | /// 10 | public class TokenizeResultBase 11 | { 12 | /// 13 | /// Creates a new instance of the class. 14 | /// 15 | public TokenizeResultBase(Template template) 16 | { 17 | Exceptions = new List(); 18 | 19 | Hints = new HintResult(); 20 | Tokens = new TokenResult(); 21 | 22 | Template = template; 23 | } 24 | 25 | /// 26 | /// The containing the mapping between tokens in the 27 | /// and properties on the object . 28 | /// 29 | public Template Template { get; set; } 30 | 31 | /// 32 | /// A list of any exceptions that occurred during the matching process 33 | /// 34 | public IList Exceptions { get; } 35 | 36 | /// 37 | /// The matches that where made during the tokenization process 38 | /// 39 | public TokenResult Tokens { get; set; } 40 | 41 | /// 42 | /// Gets the hints found in the input 43 | /// 44 | public HintResult Hints { get; set; } 45 | 46 | /// 47 | /// Determines whether the matching process was successful 48 | /// 49 | public bool Success => Tokens.HasMatches && 50 | Tokens.HasMissingRequiredTokens == false && 51 | Hints.HasMissingRequiredHints == false; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /Tokenizer/Tokenizer.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | netstandard2.0;net452 4 | true 5 | 2.2.2.0 6 | 2.2.2 7 | Chris Wood 8 | flipbit.co.uk 9 | http://github.com/flipbit/tokenizer 10 | tokenizer tokens pattern matching 11 | Tokenizer extracts structured information from blocks of text and reflects them onto .NET objects 12 | Chris Wood 2021 13 | Tokens 14 | 15 | 16 | 17 | all 18 | runtime; build; native; contentfiles; analyzers 19 | 20 | 21 | 22 | 23 | LICENSE.txt 24 | true 25 | true 26 | snupkg 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /Tokenizer/TokenizerOptions.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Tokens 4 | { 5 | /// 6 | /// Options for the . 7 | /// 8 | public class TokenizerOptions 9 | { 10 | public static TokenizerOptions Defaults => new TokenizerOptions(); 11 | 12 | public TokenizerOptions() 13 | { 14 | ResetDefaults(); 15 | } 16 | 17 | public bool IgnoreMissingProperties { get; set; } 18 | 19 | public bool TrimLeadingWhitespaceInTokenPreamble { get; set; } 20 | 21 | public bool TrimPreambleBeforeNewLine { get; set; } 22 | 23 | public bool TrimTrailingWhiteSpace { get; set; } 24 | 25 | public bool OutOfOrderTokens { get; set; } 26 | 27 | /// 28 | /// Determines the type to use when matching Token names to object properties 29 | /// 30 | public StringComparison TokenStringComparison { get; set; } 31 | 32 | public bool EnableLogging { get; set; } 33 | 34 | /// 35 | /// If set, token values will be extracted up till the first new line character. 36 | /// 37 | public bool TerminateOnNewline { get; set; } 38 | 39 | /// 40 | /// Resets the options to their default values 41 | /// 42 | public void ResetDefaults() 43 | { 44 | TrimLeadingWhitespaceInTokenPreamble = true; 45 | TrimPreambleBeforeNewLine = false; 46 | TrimTrailingWhiteSpace = true; 47 | TokenStringComparison = StringComparison.InvariantCulture; 48 | OutOfOrderTokens = false; 49 | EnableLogging = false; 50 | TerminateOnNewline = false; 51 | IgnoreMissingProperties = false; 52 | } 53 | 54 | public TokenizerOptions Clone() 55 | { 56 | return new TokenizerOptions 57 | { 58 | TrimTrailingWhiteSpace = TrimTrailingWhiteSpace, 59 | TrimLeadingWhitespaceInTokenPreamble = TrimLeadingWhitespaceInTokenPreamble, 60 | TokenStringComparison = TokenStringComparison, 61 | OutOfOrderTokens = OutOfOrderTokens, 62 | EnableLogging = EnableLogging, 63 | TrimPreambleBeforeNewLine = TrimPreambleBeforeNewLine, 64 | TerminateOnNewline = TerminateOnNewline, 65 | IgnoreMissingProperties = IgnoreMissingProperties 66 | }; 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /Tokenizer/Transformers/ITokenTransformer.cs: -------------------------------------------------------------------------------- 1 | namespace Tokens.Transformers 2 | { 3 | /// 4 | /// Defines an operation that can be performed on a token 5 | /// 6 | public interface ITokenTransformer : ITokenDecorator 7 | { 8 | /// 9 | /// Attempts to transform the given input 10 | /// 11 | bool CanTransform(object value, string[] args, out object transformed); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /Tokenizer/Transformers/RemoveEndTransformer.cs: -------------------------------------------------------------------------------- 1 | using Tokens.Exceptions; 2 | using Tokens.Extensions; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | /// 7 | /// Removes occurrences of a string from the end of a token value 8 | /// 9 | public class RemoveEndTransformer : ITokenTransformer 10 | { 11 | public bool CanTransform(object value, string[] args, out object transformed) 12 | { 13 | if (value == null || string.IsNullOrEmpty(value.ToString())) 14 | { 15 | transformed = string.Empty; 16 | return true; 17 | } 18 | 19 | if (args == null || args.Length != 1) throw new TokenizerException($"RemoveEnd(value): missing arguments processing: {value}"); 20 | 21 | var valueString = value.ToString(); 22 | if (valueString.EndsWith(args[0])) 23 | { 24 | transformed = valueString.SubstringBeforeLastString(args[0]); 25 | } 26 | else 27 | { 28 | transformed = value; 29 | } 30 | 31 | return true; 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Tokenizer/Transformers/RemoveStartTransformer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Tokens.Exceptions; 3 | using Tokens.Extensions; 4 | 5 | namespace Tokens.Transformers 6 | { 7 | /// 8 | /// Removes occurrences of a string from the start of a token value 9 | /// 10 | public class RemoveStartTransformer : ITokenTransformer 11 | { 12 | public bool CanTransform(object value, string[] args, out object transformed) 13 | { 14 | if (value == null || string.IsNullOrEmpty(value.ToString())) 15 | { 16 | transformed = string.Empty; 17 | return true; 18 | } 19 | 20 | if (args == null || args.Length != 1) throw new TokenizerException($"RemoveStart(value): missing arguments processing: {value}"); 21 | 22 | var valueString = value.ToString(); 23 | if (valueString.StartsWith(args[0])) 24 | { 25 | transformed = valueString.SubstringAfterString(args[0]); 26 | } 27 | else 28 | { 29 | transformed = value; 30 | } 31 | 32 | return true; 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /Tokenizer/Transformers/RemoveTransformer.cs: -------------------------------------------------------------------------------- 1 | using Tokens.Exceptions; 2 | 3 | namespace Tokens.Transformers 4 | { 5 | /// 6 | /// Removes occurrences of a string 7 | /// 8 | public class RemoveTransformer : ITokenTransformer 9 | { 10 | public bool CanTransform(object value, string[] args, out object transformed) 11 | { 12 | if (value == null || string.IsNullOrEmpty(value.ToString())) 13 | { 14 | transformed = string.Empty; 15 | return true; 16 | } 17 | 18 | if (args == null || args.Length != 1) throw new TokenizerException($"Remove(value): missing arguments processing: {value}"); 19 | 20 | transformed = value.ToString().Replace(args[0], string.Empty); 21 | 22 | return true; 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /Tokenizer/Transformers/ReplaceTransformer.cs: -------------------------------------------------------------------------------- 1 | using Tokens.Exceptions; 2 | 3 | namespace Tokens.Transformers 4 | { 5 | /// 6 | /// Replaces occurrences of a string with another 7 | /// 8 | public class ReplaceTransformer : ITokenTransformer 9 | { 10 | public bool CanTransform(object value, string[] args, out object transformed) 11 | { 12 | if (value == null || string.IsNullOrEmpty(value.ToString())) 13 | { 14 | transformed = string.Empty; 15 | return true; 16 | } 17 | 18 | if (args == null || args.Length != 2) throw new TokenizerException($"Replace(from, to): missing arguments processing: {value}"); 19 | 20 | transformed = value.ToString().Replace(args[0], args[1]); 21 | 22 | return true; 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /Tokenizer/Transformers/SetTransformer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Tokens.Transformers 4 | { 5 | /// 6 | /// Sets the token value 7 | /// 8 | public class SetTransformer : ITokenTransformer 9 | { 10 | public bool CanTransform(object value, string[] args, out object transformed) 11 | { 12 | if (args == null || args.Length != 1) 13 | { 14 | throw new ArgumentException("Set() must specified one argument to set - Set( value)"); 15 | } 16 | 17 | transformed = args[0].Trim(); 18 | 19 | return true; 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /Tokenizer/Transformers/SplitTransformer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | /// 7 | /// Removes occurrences of a string from then end of a token value 8 | /// 9 | public class SplitTransformer : ITokenTransformer 10 | { 11 | public bool CanTransform(object value, string[] args, out object transformed) 12 | { 13 | if (value == null || string.IsNullOrEmpty(value.ToString())) 14 | { 15 | transformed = string.Empty; 16 | return true; 17 | } 18 | 19 | if (args == null || args.Length != 1) throw new TokenizerException($"Split(value): missing arguments processing: {value}"); 20 | 21 | var valueArray = value.ToString().Split(new[] { args[0] }, StringSplitOptions.RemoveEmptyEntries); 22 | if (valueArray.Length > 1) 23 | { 24 | transformed = valueArray; 25 | } 26 | else 27 | { 28 | transformed = value; 29 | } 30 | 31 | return true; 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Tokenizer/Transformers/SubstringAfterLastTransformer.cs: -------------------------------------------------------------------------------- 1 | using Tokens.Exceptions; 2 | using Tokens.Extensions; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | /// 7 | /// Trims the token value after the first occurence of the given string 8 | /// 9 | public class SubstringAfterLastTransformer : ITokenTransformer 10 | { 11 | public bool CanTransform(object value, string[] args, out object transformed) 12 | { 13 | if (value == null || string.IsNullOrEmpty(value.ToString())) 14 | { 15 | transformed = string.Empty; 16 | return true; 17 | } 18 | 19 | if (args == null || args.Length == 0) throw new TokenizerException($"SubstringAfterLast(): missing argument processing: {value}"); 20 | 21 | transformed = value.ToString().SubstringAfterLastString(args[0]); 22 | 23 | return true; 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Tokenizer/Transformers/SubstringAfterTransformer.cs: -------------------------------------------------------------------------------- 1 | using Tokens.Exceptions; 2 | using Tokens.Extensions; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | /// 7 | /// Trims the token value after the first occurence of the given string 8 | /// 9 | public class SubstringAfterTransformer : ITokenTransformer 10 | { 11 | public bool CanTransform(object value, string[] args, out object transformed) 12 | { 13 | if (value == null || string.IsNullOrEmpty(value.ToString())) 14 | { 15 | transformed = string.Empty; 16 | return true; 17 | } 18 | 19 | if (args == null || args.Length == 0) throw new TokenizerException($"SubstringAfter(): missing argument processing: {value}"); 20 | 21 | transformed = value.ToString().SubstringAfterString(args[0]); 22 | 23 | return true; 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Tokenizer/Transformers/SubstringBeforeLastTransformer.cs: -------------------------------------------------------------------------------- 1 | using Tokens.Exceptions; 2 | using Tokens.Extensions; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | /// 7 | /// Trims the token value after the first occurence of the given string 8 | /// 9 | public class SubstringBeforeLastTransformer : ITokenTransformer 10 | { 11 | public bool CanTransform(object value, string[] args, out object transformed) 12 | { 13 | if (value == null || string.IsNullOrEmpty(value.ToString())) 14 | { 15 | transformed = string.Empty; 16 | return true; 17 | } 18 | 19 | if (args == null || args.Length == 0) throw new TokenizerException($"SubstringBeforeLast(): missing argument processing: {value}"); 20 | 21 | transformed = value.ToString().SubstringBeforeLastString(args[0]); 22 | 23 | return true; 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Tokenizer/Transformers/SubstringBeforeTransformer.cs: -------------------------------------------------------------------------------- 1 | using Tokens.Exceptions; 2 | using Tokens.Extensions; 3 | 4 | namespace Tokens.Transformers 5 | { 6 | /// 7 | /// Trims the token value after the first occurence of the given string 8 | /// 9 | public class SubstringBeforeTransformer : ITokenTransformer 10 | { 11 | public bool CanTransform(object value, string[] args, out object transformed) 12 | { 13 | if (value == null || string.IsNullOrEmpty(value.ToString())) 14 | { 15 | transformed = string.Empty; 16 | return true; 17 | } 18 | 19 | if (args == null || args.Length == 0) throw new TokenizerException($"SubstringBefore(): missing argument processing: {value}"); 20 | 21 | transformed = value.ToString().SubstringBeforeString(args[0]); 22 | 23 | return true; 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Tokenizer/Transformers/ToDateTimeUtcTransformer.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Globalization; 3 | using Tokens.Extensions; 4 | 5 | namespace Tokens.Transformers 6 | { 7 | /// 8 | /// Converts the token value to a in UTC format 9 | /// 10 | public class ToDateTimeUtcTransformer : ITokenTransformer 11 | { 12 | public bool CanTransform(object value, string[] args, out object transformed) 13 | { 14 | var valueString = value as string; 15 | 16 | if (string.IsNullOrWhiteSpace(valueString) == false) 17 | { 18 | if (valueString.Contains("(UTC)")) 19 | { 20 | valueString = valueString.SubstringBeforeString("(UTC)"); 21 | } 22 | 23 | if (valueString.Contains("UTC")) 24 | { 25 | valueString = valueString.SubstringBeforeString("UTC"); 26 | } 27 | 28 | value = valueString.Trim(); 29 | } 30 | 31 | if (ToDateTimeTransformer.TryParseDateTime(value, args, DateTimeStyles.AssumeUniversal, out var result)) 32 | { 33 | transformed = DateTime.SpecifyKind(result.ToUniversalTime(), DateTimeKind.Utc); 34 | return true; 35 | }; 36 | 37 | transformed = value; 38 | 39 | return false; 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /Tokenizer/Transformers/ToLowerTransformer.cs: -------------------------------------------------------------------------------- 1 | namespace Tokens.Transformers 2 | { 3 | /// 4 | /// Converts the token value to lower case 5 | /// 6 | public class ToLowerTransformer : ITokenTransformer 7 | { 8 | public bool CanTransform(object value, string[] args, out object transformed) 9 | { 10 | if (value == null) 11 | { 12 | transformed = string.Empty; 13 | return true; 14 | } 15 | 16 | transformed = value.ToString().ToLower(); 17 | 18 | return true; 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /Tokenizer/Transformers/ToUpperTransformer.cs: -------------------------------------------------------------------------------- 1 | namespace Tokens.Transformers 2 | { 3 | /// 4 | /// Converts the token value to upper case 5 | /// 6 | public class ToUpperTransformer : ITokenTransformer 7 | { 8 | public bool CanTransform(object value, string[] args, out object transformed) 9 | { 10 | if (value == null) 11 | { 12 | transformed = string.Empty; 13 | } 14 | else 15 | { 16 | transformed = value.ToString().ToUpper(); 17 | } 18 | 19 | return true; 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /Tokenizer/Transformers/TrimTransformer.cs: -------------------------------------------------------------------------------- 1 | namespace Tokens.Transformers 2 | { 3 | /// 4 | /// Trims the token value 5 | /// 6 | public class TrimTransformer : ITokenTransformer 7 | { 8 | public bool CanTransform(object value, string[] args, out object transformed) 9 | { 10 | if (value == null) 11 | { 12 | transformed = string.Empty; 13 | return true; 14 | } 15 | 16 | transformed = value.ToString().Trim(); 17 | 18 | return true; 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /Tokenizer/Validators/ContainsValidator.cs: -------------------------------------------------------------------------------- 1 | using Tokens.Exceptions; 2 | 3 | namespace Tokens.Validators 4 | { 5 | /// 6 | /// Validator to determine if a token value ends with a given string 7 | /// 8 | public class ContainsValidator : ITokenValidator 9 | { 10 | /// 11 | /// Determines whether the specified token is valid. 12 | /// 13 | public bool IsValid(object value, params string[] args) 14 | { 15 | if (value == null) return false; 16 | 17 | var valueString = value.ToString(); 18 | 19 | if (string.IsNullOrEmpty(valueString)) return false; 20 | 21 | if (args == null || args.Length == 0) throw new TokenizerException($"Contains(): missing argument processing: {value}"); 22 | 23 | return valueString.Contains(args[0]); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Tokenizer/Validators/EndsWithValidator.cs: -------------------------------------------------------------------------------- 1 | using Tokens.Exceptions; 2 | 3 | namespace Tokens.Validators 4 | { 5 | /// 6 | /// Validator to determine if a token value ends with a given string 7 | /// 8 | public class EndsWithValidator : ITokenValidator 9 | { 10 | /// 11 | /// Determines whether the specified token is valid. 12 | /// 13 | public bool IsValid(object value, params string[] args) 14 | { 15 | if (value == null) return false; 16 | 17 | var valueString = value.ToString(); 18 | 19 | if (string.IsNullOrEmpty(valueString)) return false; 20 | 21 | if (args == null || args.Length == 0) throw new TokenizerException($"EndsWith(): missing argument processing: {value}"); 22 | 23 | return valueString.EndsWith(args[0]); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Tokenizer/Validators/ITokenValidator.cs: -------------------------------------------------------------------------------- 1 | namespace Tokens.Validators 2 | { 3 | /// 4 | /// Interface to define the validation of tokens discovered in input text 5 | /// 6 | public interface ITokenValidator : ITokenDecorator 7 | { 8 | /// 9 | /// Determines whether the specified token is valid. 10 | /// 11 | bool IsValid(object value, params string[] args); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /Tokenizer/Validators/IsDateTimeValidator.cs: -------------------------------------------------------------------------------- 1 | using Tokens.Transformers; 2 | 3 | namespace Tokens.Validators 4 | { 5 | /// 6 | /// Validator to determine if a token value is a date time string 7 | /// 8 | public class IsDateTimeValidator : ITokenValidator 9 | { 10 | /// 11 | /// Determines whether the specified token is valid. 12 | /// 13 | public bool IsValid(object value, params string[] args) 14 | { 15 | if (value == null) return false; 16 | 17 | var valueString = value.ToString(); 18 | 19 | if (string.IsNullOrEmpty(valueString)) return false; 20 | 21 | return ToDateTimeTransformer.TryParseDateTime(valueString, args, out _); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Tokenizer/Validators/IsDomainNameValidator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Tokens.Validators 4 | { 5 | /// 6 | /// Validator to determine if a token value is a domain name 7 | /// 8 | public class IsDomainNameValidator : ITokenValidator 9 | { 10 | /// 11 | /// Determines whether the specified token is valid. 12 | /// 13 | public bool IsValid(object value, params string[] args) 14 | { 15 | if (value == null) return false; 16 | 17 | var valueString = value.ToString(); 18 | 19 | if (string.IsNullOrEmpty(valueString)) return false; 20 | 21 | return Uri.CheckHostName(valueString) == UriHostNameType.Dns; 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Tokenizer/Validators/IsEmailValidator.cs: -------------------------------------------------------------------------------- 1 | using System.Net.Mail; 2 | 3 | namespace Tokens.Validators 4 | { 5 | /// 6 | /// Validator to determine if a token value is an email address 7 | /// 8 | public class IsEmailValidator : ITokenValidator 9 | { 10 | /// 11 | /// Determines whether the specified token is valid. 12 | /// 13 | public bool IsValid(object value, params string[] args) 14 | { 15 | if (value == null) return false; 16 | 17 | var valueString = value.ToString(); 18 | 19 | if (string.IsNullOrEmpty(valueString)) return false; 20 | if (valueString.Contains("@-")) return false; 21 | if (valueString.Trim().Contains(" ")) return false; 22 | 23 | try 24 | { 25 | var address = new MailAddress(valueString); 26 | 27 | return address.Address == valueString; 28 | } 29 | catch 30 | { 31 | return false; 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /Tokenizer/Validators/IsLooseAbsoluteUrlValidator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Tokens.Validators 4 | { 5 | /// 6 | /// Validator to determine if a token value is a URL 7 | /// 8 | public class IsLooseAbsoluteUrlValidator : ITokenValidator 9 | { 10 | /// 11 | /// Determines whether the specified token is valid. 12 | /// 13 | public bool IsValid(object value, params string[] args) 14 | { 15 | if (value == null) return false; 16 | 17 | var valueString = value.ToString(); 18 | 19 | if (string.IsNullOrEmpty(valueString)) return false; 20 | 21 | var result = Uri.IsWellFormedUriString(valueString, UriKind.Absolute); 22 | 23 | if (!result) 24 | { 25 | result = Uri.IsWellFormedUriString($"http://{valueString}", UriKind.Absolute); 26 | } 27 | 28 | return result; 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /Tokenizer/Validators/IsLooseUrlValidator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Tokens.Validators 4 | { 5 | /// 6 | /// Validator to determine if a token value is a URL 7 | /// 8 | public class IsLooseUrlValidator : ITokenValidator 9 | { 10 | /// 11 | /// Determines whether the specified token is valid. 12 | /// 13 | public bool IsValid(object value, params string[] args) 14 | { 15 | if (value == null) return false; 16 | 17 | var valueString = value.ToString(); 18 | 19 | if (string.IsNullOrEmpty(valueString)) return false; 20 | 21 | var result = Uri.IsWellFormedUriString(valueString, UriKind.RelativeOrAbsolute); 22 | 23 | if (!result) 24 | { 25 | result = Uri.IsWellFormedUriString($"http://{valueString}", UriKind.Absolute); 26 | } 27 | 28 | return result; 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /Tokenizer/Validators/IsNotEmptyValidator.cs: -------------------------------------------------------------------------------- 1 | namespace Tokens.Validators 2 | { 3 | /// 4 | /// Validator to determine if a token value is not empty 5 | /// 6 | public class IsNotEmptyValidator : ITokenValidator 7 | { 8 | /// 9 | /// Determines whether the specified token is valid. 10 | /// 11 | public bool IsValid(object value, params string[] args) 12 | { 13 | if (value == null) return false; 14 | 15 | var valueString = value.ToString(); 16 | 17 | return !string.IsNullOrEmpty(valueString); 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /Tokenizer/Validators/IsNotValidator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Tokens.Validators 4 | { 5 | /// 6 | /// Validator to determine if a token value is not the specified value 7 | /// 8 | public class IsNotValidator : ITokenValidator 9 | { 10 | /// 11 | /// Determines whether the specified token is valid. 12 | /// 13 | public bool IsValid(object value, params string[] args) 14 | { 15 | if (value == null) return true; 16 | 17 | var valueString = value.ToString(); 18 | 19 | if (string.IsNullOrEmpty(valueString)) return true; 20 | 21 | if (args.Length != 1) throw new ArgumentException("IsNot() - Must supply a string value"); 22 | 23 | return valueString != args[0]; 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Tokenizer/Validators/IsNumericValidator.cs: -------------------------------------------------------------------------------- 1 | namespace Tokens.Validators 2 | { 3 | /// 4 | /// Validator to determine if a token value is numeric 5 | /// 6 | public class IsNumericValidator : ITokenValidator 7 | { 8 | /// 9 | /// Determines whether the specified token is valid. 10 | /// 11 | public bool IsValid(object value, params string[] args) 12 | { 13 | if (value == null) return false; 14 | 15 | var valueString = value.ToString(); 16 | 17 | if (string.IsNullOrEmpty(valueString)) return false; 18 | 19 | return float.TryParse(valueString, out var test); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /Tokenizer/Validators/IsPhoneNumberValidator.cs: -------------------------------------------------------------------------------- 1 | using Tokens.Extensions; 2 | 3 | namespace Tokens.Validators 4 | { 5 | /// 6 | /// Validator to determine if a token value is a phone number 7 | /// 8 | public class IsPhoneNumberValidator : ITokenValidator 9 | { 10 | /// 11 | /// Determines whether the specified token is valid. 12 | /// 13 | public bool IsValid(object value, params string[] args) 14 | { 15 | if (value == null) return false; 16 | 17 | var valueString = value.ToString(); 18 | 19 | if (string.IsNullOrEmpty(valueString)) return false; 20 | 21 | var trimmed = valueString.Keep(" ()+.-0123456789"); 22 | 23 | if (trimmed != valueString) return false; 24 | 25 | if (string.IsNullOrWhiteSpace(trimmed)) return false; 26 | 27 | var numbers = trimmed.Keep("0123456789"); 28 | 29 | return numbers.Length >= 6; 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /Tokenizer/Validators/IsUrlValidator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | 3 | namespace Tokens.Validators 4 | { 5 | /// 6 | /// Validator to determine if a token value is a URL 7 | /// 8 | public class IsUrlValidator : ITokenValidator 9 | { 10 | /// 11 | /// Determines whether the specified token is valid. 12 | /// 13 | public bool IsValid(object value, params string[] args) 14 | { 15 | if (value == null) return false; 16 | 17 | var valueString = value.ToString(); 18 | 19 | if (string.IsNullOrEmpty(valueString)) return false; 20 | 21 | return Uri.IsWellFormedUriString(valueString, UriKind.Absolute); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Tokenizer/Validators/MaxLengthValidator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Validators 5 | { 6 | /// 7 | /// Validator to determine if a token value meets a maximum length requirement 8 | /// 9 | public class MaxLengthValidator : ITokenValidator 10 | { 11 | /// 12 | /// Determines whether the specified token is valid. 13 | /// 14 | public bool IsValid(object value, params string[] args) 15 | { 16 | if (args.Length == 0) 17 | { 18 | throw new ValidationException("You must specified a MaxLength value, e.g. 'MaxLength(255)'"); 19 | } 20 | 21 | try 22 | { 23 | var maxLength = Convert.ToInt32(args[0]); 24 | 25 | return value.ToString().Length <= maxLength; 26 | } 27 | catch (FormatException ex) 28 | { 29 | throw new ValidationException("MaxLength parameter must be an integer", ex); 30 | } 31 | 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Tokenizer/Validators/MinLengthValidator.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Tokens.Exceptions; 3 | 4 | namespace Tokens.Validators 5 | { 6 | /// 7 | /// Validator to determine if a token value meets a maximum length requirement 8 | /// 9 | public class MinLengthValidator : ITokenValidator 10 | { 11 | /// 12 | /// Determines whether the specified token is valid. 13 | /// 14 | public bool IsValid(object value, params string[] args) 15 | { 16 | if (args.Length == 0) 17 | { 18 | throw new ValidationException("You must specified a MinLength value, e.g. 'MinLength(50)'"); 19 | } 20 | 21 | try 22 | { 23 | var minLength = Convert.ToInt32(args[0]); 24 | 25 | return value.ToString().Length >= minLength; 26 | } 27 | catch (FormatException ex) 28 | { 29 | throw new ValidationException("MinLength parameter must be an integer", ex); 30 | } 31 | 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Tokenizer/Validators/StartsWithValidator.cs: -------------------------------------------------------------------------------- 1 | using Tokens.Exceptions; 2 | 3 | namespace Tokens.Validators 4 | { 5 | /// 6 | /// Validator to determine if a token value starts with a given string 7 | /// 8 | public class StartsWithValidator : ITokenValidator 9 | { 10 | /// 11 | /// Determines whether the specified token is valid. 12 | /// 13 | public bool IsValid(object value, params string[] args) 14 | { 15 | if (value == null) return false; 16 | 17 | var valueString = value.ToString(); 18 | 19 | if (string.IsNullOrEmpty(valueString)) return false; 20 | 21 | if (args == null || args.Length == 0) throw new TokenizerException($"StartsWith(): missing argument processing: {value}"); 22 | 23 | return valueString.StartsWith(args[0]); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | version: '2.1.{build}' 2 | pull_requests: 3 | do_not_increment_build_number: true 4 | branches: 5 | only: 6 | - master 7 | nuget: 8 | disable_publish_on_pr: true 9 | image: Visual Studio 2017 10 | dotnet_csproj: 11 | patch: true 12 | file: '**\*.csproj' 13 | version: '{version}' 14 | package_version: '{version}' 15 | assembly_version: '{version}' 16 | file_version: '{version}' 17 | informational_version: '{version}' 18 | init: 19 | # Good practise, because Windows line endings are different from Unix/Linux ones 20 | - cmd: git config --global core.autocrlf true 21 | install: 22 | # Install repo specific stuff here 23 | before_build: 24 | # Display .NET Core version 25 | - cmd: dotnet --version 26 | # Display minimal restore text 27 | - cmd: dotnet restore ./Tokenizer/Tokenizer.csproj --verbosity m 28 | build_script: 29 | # output will be in ./Tokenizer/Tokenizer/bin/Release/{framework} 30 | - cmd: dotnet build ./Tokenizer/Tokenizer.csproj -c Release -f netstandard2.0 31 | - cmd: dotnet build ./Tokenizer/Tokenizer.csproj -c Release -f net452 32 | - cmd: dotnet pack ./Tokenizer/Tokenizer.csproj -c Release 33 | after_build: 34 | # For once the build has completed 35 | artifacts: 36 | - path: .\**\*.nupkg 37 | name: Package 38 | type: NuGetPackage 39 | clone_depth: 1 40 | test_script: 41 | # restore packages for our unit tests 42 | - cmd: dotnet restore ./Tokenizer.Tests/Tokenizer.Tests.csproj --verbosity m 43 | # run the unit tests (requires changing into the test directory) 44 | - cmd: cd Tokenizer.Tests 45 | - cmd: dotnet test 46 | on_finish : 47 | --------------------------------------------------------------------------------