├── .github
└── workflows
│ └── testSuite.yml
├── .gitignore
├── LICENSE
├── Package.swift
├── README.md
├── Sources
└── SwiLex
│ ├── Error.swift
│ ├── Regex.swift
│ ├── Scanner.swift
│ ├── SwiLex.swift
│ ├── SwiLexable.swift
│ └── Token.swift
├── SwiLex.png
└── Tests
├── LinuxMain.swift
└── SwiLexTests
├── CalculatorTokensTests.swift
├── Tools
└── TokensAssert.swift
├── WordNumberTokensTests.swift
└── XCTestManifests.swift
/.github/workflows/testSuite.yml:
--------------------------------------------------------------------------------
1 | name: TestSuite
2 |
3 | on:
4 | push:
5 | branches: [ main ]
6 | pull_request:
7 | branches: [ main ]
8 |
9 | jobs:
10 | build:
11 |
12 | runs-on: macos-latest
13 |
14 | steps:
15 | - uses: actions/checkout@v2
16 | - name: Build
17 | run: swift build -v
18 | - name: Run tests
19 | run: swift test -v
20 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 | /build
3 | /.build
4 | /.swiftpm
5 | /*.xcodeproj
6 | /Packages
7 | xcuserdata/
8 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2020 yassir RAMDANI
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/Package.swift:
--------------------------------------------------------------------------------
1 | // swift-tools-version:5.2
2 |
3 | import PackageDescription
4 |
5 | let package = Package(
6 | name: "SwiLex",
7 | products: [
8 | .library(name: "SwiLex", targets: ["SwiLex"]),
9 | ],
10 | targets: [
11 | .target(name: "SwiLex"),
12 | .testTarget(name: "SwiLexTests", dependencies: ["SwiLex"]),
13 | ]
14 | )
15 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 | 
3 |
4 | 
5 | 
6 | 
7 |
8 | 
9 |
10 |
11 | 
12 |
13 |
14 | A universal lexer library in Swift.
15 |
16 |
17 | - [Introduction](#introduction)
18 | - [Installation](#installation)
19 | - [Usage](#usage)
20 | - [Simple Lexer example](#simple-lexer-example)
21 | - [Conditianal tokens Lexer example](#conditianal-tokens-lexer-example)
22 | - [Documentation](#documentation)
23 | - [Features](#features)
24 | - [Contributing](#contributing)
25 | - [Author](#author)
26 | - [License](#license)
27 |
28 |
29 | ## Introduction
30 | In computer science a Lexer or a tokenizer is a program that converts a sequence of characters (raw string) into a sequence of tokens.
31 |
32 | SwiLex is a universal lexer which means that you can use it to build *any* lexer only by defining your tokens in a Swift enum (in few lines of code).
33 |
34 | *Combined with **SwiParse** it allows to build a full Parser. ([learn more](https://github.com/yassram/SwiParse/))*
35 |
36 | ## Installation
37 | SwiLex is distributed as a Swift Package using [SPM](https://swift.org/package-manager/).
38 |
39 | To install SwiLex, add the following line to the `dependencies` list in your `Package.swift`:
40 |
41 | ```swift
42 | .package(url: "https://github.com/yassram/SwiLex.git", .upToNextMinor(from: "1.1.0")),
43 | ```
44 |
45 | *SwiLex will support other dependency managers soon.*
46 |
47 | ## Usage
48 |
49 | Import SwiLex to use it:
50 |
51 | ```swift
52 | import SwiLex
53 | ```
54 |
55 | ### Simple Lexer example
56 |
57 | Define a SwiLaxable enum to list your possible tokens with their corresponding regular expressions:
58 |
59 | ```swift
60 | enum WordNumberTokens: String, SwiLexable {
61 | static var separators: Set = [" "]
62 |
63 | case text = "[A-Za-z]*"
64 | case number = "[0-9]*"
65 |
66 | case eof
67 | case none
68 | }
69 | ```
70 |
71 | Then create an instance of SwiLex and call the lex function:
72 |
73 | ```swift
74 | var lexer = SwiLex()
75 | let tokenList = try lexer.lex(input: " H3Ll0 W0r 1d 1234 aBcD ")
76 |
77 | // returns [text[H], number[3], text[Ll], number[0], text[W], number[0],
78 | // text[r], number[1], text[d], number[1234], text[aBcD]]
79 | ```
80 |
81 | This will return a list of tokens with the type of the token and its value (the matched string).
82 |
83 | ### Conditianal tokens Lexer example
84 |
85 | SwiLex provides a mechanism for conditionally activating tokens. This tokens are only available for specific `Mode`.
86 |
87 | In the following example we want to make the `.quotedString` token available only when the `.quote` mode is active.
88 |
89 | Start by defining the possible `Mode`s.
90 |
91 | ```swift
92 | enum Modes {
93 | case normal
94 | case quote
95 | }
96 | ```
97 |
98 | Then define a SwiLaxable enum to list the possible tokens with their corresponding regular expressions (like for a simple Lexer):
99 |
100 | ```swift
101 | enum QuotedStringTextNumberTokens: String, SwiLexable {
102 | static var separators: Set = [" "]
103 |
104 | case text = "[A-Za-z]*"
105 | case number = "[0-9]*"
106 | case doubleQuote = "\""
107 | case quotedString = #"[^"]*"#
108 |
109 | case eof
110 | case none
111 | }
112 | ```
113 |
114 | Next, define the tokens' availability for the possible modes by implementing the `activeForModes` property:
115 |
116 | ```swift
117 | extension QuotedStringTextNumberTokens {
118 | var activeForModes: Set {
119 | switch self {
120 | case .doubleQuote:
121 | return [.quote, .normal]
122 | case .quotedString:
123 | return [.quote]
124 | default:
125 | return [.normal]
126 | }
127 | }
128 | }
129 | ```
130 | > - `.doubleQuote` is available for both `.normal` and `.quote` modes allowing to switch between them (by defining the end and the begining of a quote).
131 | > - `.quotedString` is only available for the `.quote` mode.
132 | > - All the other tokens are only availbale for the `.normal` mode.
133 |
134 | The last step is to tell when to change the curent mode.
135 |
136 | For that implement the function `changeMode(current mode: Modes?) -> Modes?`:
137 |
138 | ```swift
139 | extension QuotedStringTextNumberTokens {
140 | func changeMode(current mode: Modes?) -> Modes? {
141 | switch self {
142 | case .doubleQuote:
143 | return mode == .normal ? .quote : .normal
144 | default:
145 | return mode
146 | }
147 | }
148 | }
149 | ```
150 |
151 | Finally create an instance of SwiLex and call the lex function with the initial mode:
152 |
153 | ```swift
154 | let input = #"4lb3rt Einstein said "If you can't explain it to a 6 year old, you don't understand it yourself.""#
155 |
156 | var lexer = SwiLex()
157 | let tokenList = try lexer.lex(input: input, initialMode: .normal)
158 |
159 | // [number[4], text[lb], number[3], text[rt], text[Einstein], text[said], doubleQuote["],
160 | // quotedString[If you can't explain it to a 6 year old, you don't understand it yourself.],
161 | // doubleQuote["]]
162 |
163 | ```
164 |
165 | This will return a list of tokens with the type of the token and its value (the matched string).
166 |
167 | ### Documentation
168 | A documentation with more examples is available [here](https://github.com/yassram/SwiLex/wiki)
169 |
170 |
171 | ## Features
172 | - [x] Tokens defined using only a simple `SwiLexable` enum.
173 | - [x] Support conditional tokens with custom modes.
174 | - [x] Support actions on each match by implementing the `onLex(raw: Substring)` function.
175 | - [x] Errors with line number and the issue's substring.
176 | - [ ] Add detailed documentation with more examples.
177 | - [ ] Support Cocoapods and Carthage.
178 | - [x] 🔥 **SwiParse**, a general-purpose parser generator that can be linked to **SwiLex** to generate a full parser. *([released here](https://github.com/yassram/SwiParse))* 🔥
179 |
180 |
181 | ## Contributing
182 | This is an open source project, so feel free to contribute. How?
183 | - Open an issue.
184 | - Send feedback via email.
185 | - Propose your own fixes, suggestions and open a pull request with the changes.
186 |
187 | ## Author
188 | Yassir Ramdani
189 |
190 | ## License
191 |
192 | ```
193 | MIT License
194 |
195 | Copyright (c) 2020 yassir RAMDANI
196 |
197 | Permission is hereby granted, free of charge, to any person obtaining a copy
198 | of this software and associated documentation files (the "Software"), to deal
199 | in the Software without restriction, including without limitation the rights
200 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
201 | copies of the Software, and to permit persons to whom the Software is
202 | furnished to do so, subject to the following conditions:
203 |
204 | The above copyright notice and this permission notice shall be included in all
205 | copies or substantial portions of the Software.
206 |
207 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
208 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
209 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
210 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
211 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
212 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
213 | SOFTWARE.
214 | ```
215 |
--------------------------------------------------------------------------------
/Sources/SwiLex/Error.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Error.swift
3 | // SwiLex
4 | //
5 | // Created by Yassir Ramdani on 12/09/2020.
6 | // Copyright © 2020 Yassir Ramdani. All rights reserved.
7 | //
8 |
9 | import Foundation
10 |
11 | public enum SwiLexError: Error {
12 | case noTokenMatched(at: Substring, line: Int)
13 | }
14 |
15 | extension SwiLexError : LocalizedError {
16 | public var errorDescription: String? {
17 | switch self {
18 | case let .noTokenMatched(at: str, line: line):
19 | return "No token matched \(str), line: \(line)"
20 | }
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/Sources/SwiLex/Regex.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Regex.swift
3 | // SwiLex
4 | //
5 | // Created by Yassir Ramdani on 05/09/2020.
6 | // Copyright © 2020 Yassir Ramdani. All rights reserved.
7 | //
8 |
9 | import Foundation
10 |
11 | extension Substring {
12 | func matches(pattern: String) -> String.Index {
13 | guard let matchRange = range(of: pattern, options: .regularExpression),
14 | matchRange.lowerBound == startIndex
15 | else { return startIndex }
16 | return matchRange.upperBound
17 | }
18 | }
19 |
--------------------------------------------------------------------------------
/Sources/SwiLex/Scanner.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Scanner.swift
3 | // SwiLex
4 | //
5 | // Created by Yassir Ramdani on 06/09/2020.
6 | // Copyright © 2020 Yassir Ramdani. All rights reserved.
7 | //
8 |
9 | import Foundation
10 |
11 | extension SwiLex {
12 | struct Scanner {
13 | private let input: String
14 | private(set) var index: String.Index
15 | private(set) var currentLineNumber: Int
16 |
17 | init(input: String) {
18 | self.input = input
19 | index = self.input.startIndex
20 | currentLineNumber = 0
21 | }
22 | }
23 | }
24 |
25 | extension SwiLex.Scanner {
26 | private var end: String.Index { input.endIndex }
27 | var reachedEnd: Bool { index == end }
28 | var remainingString: Substring { input[index...] }
29 |
30 | private mutating func next() {
31 | index = input.index(after: index)
32 | }
33 |
34 | func buffer(to index: String.Index) -> Substring {
35 | input[self.index ..< index]
36 | }
37 |
38 | mutating func move(to index: String.Index) {
39 | currentLineNumber += buffer(to: index).filter { $0 == "\n" }.count
40 | self.index = index
41 | }
42 |
43 | mutating func removeSeparators() {
44 | while !reachedEnd {
45 | if Tokens.separators.contains(input[index]) {
46 | currentLineNumber += input[index] == "\n" ? 1 : 0
47 | next()
48 | } else { break }
49 | }
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/Sources/SwiLex/SwiLex.swift:
--------------------------------------------------------------------------------
1 | //
2 | // SwiLex.swift
3 | // SwiLex
4 | //
5 | // Created by Yassir Ramdani on 05/09/2020.
6 | // Copyright © 2020 Yassir Ramdani. All rights reserved.
7 | //
8 |
9 | import Foundation
10 |
11 | public struct SwiLex {
12 | /// The active mode *if conditional tokens are used*.
13 | private var currentMode: Tokens.Mode?
14 |
15 | // MARK: Public interface
16 |
17 | public init() {}
18 |
19 | /// The lexing function.
20 | ///
21 | /// Lexes an input string using a **SwiLexable enum** that defines the available tokens and their **regular expressions**.
22 | ///
23 | /// - Returns: A list of Tokens from a **raw string** and a **SwiLexable enum** defining the available tokens and their **regular expressions**.
24 | ///
25 | /// - Parameters:
26 | /// - input: The input string to be lexed.
27 | /// - initialMode: The initial active mode if conditional tokens are used.
28 | ///
29 | ///
30 | /// # Example
31 | ///
32 | /// ```
33 | /// enum WordNumberTokens: String, SwiLexable {
34 | /// static var separators: Set = [" "]
35 | /// case txt = "[A-Za-z]*"
36 | /// case number = "[0-9]*"
37 | /// }
38 | /// var lexer = SwiLex()
39 | /// let tokens = try lexer.lex(input: " H3Ll0 W0r1d")
40 | /// // returns [txt[H], number[3], txt[Ll], number[0], txt[W], number[0], txt[r], number[1], txt[d]]
41 | /// ```
42 | /// - Note: More examples with complex cases and conditional tokens are available in the project documentation.
43 | @discardableResult
44 | public mutating func lex(input: String, initialMode: Tokens.Mode? = nil) throws -> [Token] {
45 | currentMode = initialMode
46 |
47 | var scanner = Scanner(input: input)
48 | scanner.removeSeparators()
49 |
50 | var stream = [Token]()
51 |
52 | while !scanner.reachedEnd {
53 | var longestMatch: Tokens?
54 | var longestMatchIndex = scanner.index
55 | for token in Tokens.allCases {
56 | if let currentMode = currentMode, !token.activeForModes.contains(currentMode) { continue }
57 | let currentMatchIndex = scanner.remainingString.matches(pattern: token.pattern)
58 | if currentMatchIndex > longestMatchIndex {
59 | longestMatch = token
60 | longestMatchIndex = currentMatchIndex
61 | }
62 | }
63 |
64 | guard let matchedToken = longestMatch else {
65 | throw SwiLexError.noTokenMatched(at: scanner.remainingString.prefix(20),
66 | line: scanner.currentLineNumber)
67 | }
68 |
69 | let rawString = scanner.buffer(to: longestMatchIndex)
70 | let newToken = Token(type: matchedToken, value: rawString)
71 | stream.append(newToken)
72 | matchedToken.onLex(raw: rawString)
73 | currentMode = matchedToken.changeMode(current: currentMode)
74 | scanner.move(to: longestMatchIndex)
75 | scanner.removeSeparators()
76 | }
77 | let end = Token(type: Tokens.eof, value: "")
78 | stream.append(end)
79 | return stream
80 | }
81 | }
82 |
--------------------------------------------------------------------------------
/Sources/SwiLex/SwiLexable.swift:
--------------------------------------------------------------------------------
1 | //
2 | // SwiLexable.swift
3 | // SwiLex
4 | //
5 | // Created by Yassir Ramdani on 05/09/2020.
6 | // Copyright © 2020 Yassir Ramdani. All rights reserved.
7 | //
8 |
9 | import Foundation
10 |
11 | // MARK: Public interface
12 |
13 | /// A type of tokens that can be used to **lex** an input string using **SwiLex** (terminal tokens).
14 | public protocol SwiLexable: Hashable, RawRepresentable, CaseIterable where Self.RawValue == String {
15 | /// Possible modes while Lexing.
16 | associatedtype Mode: Hashable = AnyHashable
17 |
18 | static var eof: Self { get }
19 | static var none: Self { get }
20 |
21 | /// Characters used as sparators.
22 | ///
23 | /// Separators are ignored by the lexer and won't appear in the result token list.
24 | static var separators: Set { get }
25 |
26 | /// The pattern (*regular expression*) of the token.
27 | var pattern: String { get }
28 | /// The set of modes for which the token is active.
29 | ///
30 | /// SwiLex supports conditional tokens.
31 | ///
32 | /// Conditional tokens are tokens that are only availble if some specific modes are active.
33 | ///
34 | /// - note: If not set, all tokens are available by default.
35 | ///
36 | /// # Example
37 | /// In the bellow example, `.dubQuote` can be matched in both `.normal` and `.quote` modes
38 | /// (`.normal` mode to start a quote and `.quote` mode to end it).
39 | ///
40 | /// `.quotedStr` can only be matched in `.quote` mode.
41 | ///
42 | /// ```
43 | /// var activeForModes: Set {
44 | /// switch self {
45 | /// case .dubQuote:
46 | /// return [.quote, .normal]
47 | /// case .quotedStr:
48 | /// return [.quote]
49 | /// default:
50 | /// return [.normal]
51 | /// }
52 | /// }
53 | /// ```
54 | ///
55 | var activeForModes: Set { get }
56 |
57 | /// This function is called whenever a substring matches a token to perform actions on the matched string.
58 | ///
59 | /// - Parameters:
60 | /// - raw: The matched substring.
61 | func onLex(raw: Substring)
62 |
63 | /// This function is called whenever a substring matches a token to update the **current mode** if multiple modes are used.
64 | ///
65 | /// This function is used to support conditional tokens.
66 | ///
67 | /// # Example
68 | /// In the bellow example, we use changeMode to switch between the `.normal` and the `.quote` mode each time a `.dubQuote` is matched.
69 | ///
70 | /// By changing the **current mode** we can enable / disable a set of tokens.
71 | ///
72 | /// ```
73 | /// func changeMode(current mode: Mode?) -> Mode? {
74 | /// switch self {
75 | /// case .dubQuote:
76 | /// return mode == .normal ? .quote : .normal
77 | /// default:
78 | /// return mode
79 | /// }
80 | /// }
81 | /// ```
82 | ///
83 | /// - note: By changing the current mode, only tokens for witch the attribute **activeForModes** contains the new mode are **active**.
84 | ///
85 | /// - Parameters:
86 | /// - mode: The current active mode.
87 | func changeMode(current mode: Mode?) -> Mode?
88 | }
89 |
90 | // MARK: Default implementation
91 |
92 | public extension SwiLexable {
93 | /// The pattern (*regular expression*) of the token.
94 | /// By default the token's pattern is its rawValue.
95 | var pattern: String { rawValue }
96 |
97 | /// The set of modes for which the token is active.
98 | /// By default it is an empty set.
99 | var activeForModes: Set { [] }
100 |
101 | /// This function is called whenever a substring matches a token to perform actions on the matched string.
102 | /// By default this function does nothing,
103 | ///
104 | /// - Parameters:
105 | /// - raw: The matched substring.
106 | func onLex(raw _: Substring) {}
107 |
108 | /// This function is called whenever a substring matches a token to update the **current mode** if multiple modes are used.
109 | /// By default the current mode does not change.
110 | /// - Parameters:
111 | /// - mode: The current active mode.
112 | func changeMode(current mode: Mode?) -> Mode? { return mode }
113 | }
114 |
--------------------------------------------------------------------------------
/Sources/SwiLex/Token.swift:
--------------------------------------------------------------------------------
1 | //
2 | // Token.swift
3 | // SwiLex
4 | //
5 | // Created by Yassir Ramdani on 05/09/2020.
6 | // Copyright © 2020 Yassir Ramdani. All rights reserved.
7 | //
8 |
9 | import Foundation
10 |
11 | // MARK: Public interface
12 |
13 | /// An extracted token.
14 | public struct Token: CustomStringConvertible {
15 | // MARK: Properties
16 |
17 | /// The type of the token.
18 | public var type: Tokens
19 | /// The raw substring value of the matched token.
20 | public var value: Substring
21 |
22 | /// A default description for *pretty* prints.
23 | public var description: String {
24 | return "\(type)[\(value)]"
25 | }
26 | }
27 |
28 | extension Token: Equatable {
29 | public static func == (lhs: Token, rhs: Token) -> Bool {
30 | return lhs.type == rhs.type && lhs.value == rhs.value
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/SwiLex.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/yassram/SwiLex/b9d886e7114ed4166a963cb560e61da98013b625/SwiLex.png
--------------------------------------------------------------------------------
/Tests/LinuxMain.swift:
--------------------------------------------------------------------------------
1 | import XCTest
2 |
3 | import SwiLexTests
4 |
5 | var tests = [XCTestCaseEntry]()
6 | tests += WordNumberTokensTests.allTests()
7 | tests += CalculatorTokensTests.allTests()
8 | XCTMain(tests)
9 |
--------------------------------------------------------------------------------
/Tests/SwiLexTests/CalculatorTokensTests.swift:
--------------------------------------------------------------------------------
1 | @testable import SwiLex
2 | import XCTest
3 |
4 | final class CalculatorTokensTests: XCTestCase {
5 | enum CalculatorTokens: String, SwiLexable {
6 | static var separators: Set = [" ", "\n"]
7 | case op = "[+-/*]"
8 | case float = "[0-9]+.[0-9]+"
9 | case int = "[0-9]+"
10 | case lParenthesis = #"\("#
11 | case rParenthesis = #"\)"#
12 |
13 | case eof
14 | case none
15 | }
16 |
17 | func testIntSum() throws {
18 | var lexer = SwiLex()
19 | let tokens = try lexer.lex(input: "1 + 1")
20 | AssertEqualTokens(tokens1: tokens,
21 | tokens2: [
22 | Token(type: .int, value: "1"),
23 | Token(type: .op, value: "+"),
24 | Token(type: .int, value: "1"),
25 | Token(type: .eof, value: ""),
26 | ])
27 | }
28 |
29 | func testFloatSum() throws {
30 | var lexer = SwiLex()
31 | let tokens = try lexer.lex(input: "1.2 + 1.004")
32 | AssertEqualTokens(tokens1: tokens,
33 | tokens2: [
34 | Token(type: .float, value: "1.2"),
35 | Token(type: .op, value: "+"),
36 | Token(type: .float, value: "1.004"),
37 | Token(type: .eof, value: ""),
38 | ])
39 | }
40 |
41 | func testFloatIntSum() throws {
42 | var lexer = SwiLex()
43 | let tokens = try lexer.lex(input: " 1332.4322 + 1 ")
44 | AssertEqualTokens(tokens1: tokens,
45 | tokens2: [
46 | Token(type: .float, value: "1332.4322"),
47 | Token(type: .op, value: "+"),
48 | Token(type: .int, value: "1"),
49 | Token(type: .eof, value: ""),
50 | ])
51 | }
52 |
53 | func testComplexEprssion() throws {
54 | var lexer = SwiLex()
55 | let tokens = try lexer.lex(input: "( 1332.4322 + 1 ) *2 / 44.44 + ((2.3- 2) * 4 ) / 0.3 ")
56 | print(tokens)
57 | AssertEqualTokens(tokens1: tokens,
58 | tokens2: [
59 | Token(type: .lParenthesis, value: "("),
60 | Token(type: .float, value: "1332.4322"),
61 | Token(type: .op, value: "+"),
62 | Token(type: .int, value: "1"),
63 | Token(type: .rParenthesis, value: ")"),
64 | Token(type: .op, value: "*"),
65 | Token(type: .int, value: "2"),
66 | Token(type: .op, value: "/"),
67 | Token(type: .float, value: "44.44"),
68 | Token(type: .op, value: "+"),
69 | Token(type: .lParenthesis, value: "("),
70 | Token(type: .lParenthesis, value: "("),
71 | Token(type: .float, value: "2.3"),
72 | Token(type: .op, value: "-"),
73 | Token(type: .int, value: "2"),
74 | Token(type: .rParenthesis, value: ")"),
75 | Token(type: .op, value: "*"),
76 | Token(type: .int, value: "4"),
77 | Token(type: .rParenthesis, value: ")"),
78 | Token(type: .op, value: "/"),
79 | Token(type: .float, value: "0.3"),
80 | Token(type: .eof, value: ""),
81 | ])
82 | }
83 |
84 | static var allTests = [
85 | ("testIntSum", testIntSum),
86 | ("testFloatSum", testFloatSum),
87 | ("testFloatIntSum", testFloatIntSum),
88 | ("testComplexEprssion", testComplexEprssion),
89 | ]
90 |
91 | override func setUp() {
92 | super.setUp()
93 | continueAfterFailure = false
94 | }
95 | }
96 |
--------------------------------------------------------------------------------
/Tests/SwiLexTests/Tools/TokensAssert.swift:
--------------------------------------------------------------------------------
1 | //
2 | // TokensAssert.swift
3 | // SwiLex
4 | //
5 | // Created by Yassir Ramdani on 12/09/2020.
6 | // Copyright © 2020 Yassir Ramdani. All rights reserved.
7 | //
8 |
9 | import SwiLex
10 | import XCTest
11 |
12 | func AssertEqualTokens(tokens1: [Token], tokens2: [Token]) {
13 | XCTAssertEqual(tokens1.count, tokens2.count, "Wrong tokens' number")
14 | for i in 0 ..< tokens1.count {
15 | XCTAssertEqual(tokens1[i].type, tokens2[i].type, "Wrong token's type at index \(i)")
16 | XCTAssertEqual(tokens1[i].value, tokens2[i].value, "Wrong token's value at index \(i)")
17 | }
18 | }
19 |
--------------------------------------------------------------------------------
/Tests/SwiLexTests/WordNumberTokensTests.swift:
--------------------------------------------------------------------------------
1 | @testable import SwiLex
2 | import XCTest
3 |
4 | final class WordNumberTokensTests: XCTestCase {
5 | enum WordNumberTokens: String, SwiLexable {
6 | static var separators: Set = [" ", "\n"]
7 | case txt = "[A-Za-z]*"
8 | case number = "[0-9]*"
9 |
10 | case eof
11 | case none
12 | }
13 |
14 | func testWords() throws {
15 | var lexer = SwiLex()
16 | let tokens = try lexer.lex(input: " HELLO h i abc ABC Hi hello boy XD")
17 | AssertEqualTokens(tokens1: tokens,
18 | tokens2: [
19 | Token(type: .txt, value: "HELLO"),
20 | Token(type: .txt, value: "h"),
21 | Token(type: .txt, value: "i"),
22 | Token(type: .txt, value: "abc"),
23 | Token(type: .txt, value: "ABC"),
24 | Token(type: .txt, value: "Hi"),
25 | Token(type: .txt, value: "hello"),
26 | Token(type: .txt, value: "boy"),
27 | Token(type: .txt, value: "XD"),
28 | Token(type: .eof, value: ""),
29 | ])
30 | }
31 |
32 | func testNumbers() throws {
33 | var lexer = SwiLex()
34 | let tokens = try lexer.lex(input: " 233 2 1 23123 3333333 324 33 2")
35 | AssertEqualTokens(tokens1: tokens,
36 | tokens2: [
37 | Token(type: .number, value: "233"),
38 | Token(type: .number, value: "2"),
39 | Token(type: .number, value: "1"),
40 | Token(type: .number, value: "23123"),
41 | Token(type: .number, value: "3333333"),
42 | Token(type: .number, value: "324"),
43 | Token(type: .number, value: "33"),
44 | Token(type: .number, value: "2"),
45 | Token(type: .eof, value: ""),
46 | ])
47 | }
48 |
49 | func testWordsAndNumbers() throws {
50 | var lexer = SwiLex()
51 | let tokens = try lexer.lex(input: " 233 2 1 23123abc ABC Hi3333333 324 33 2")
52 | AssertEqualTokens(tokens1: tokens,
53 | tokens2: [
54 | Token(type: .number, value: "233"),
55 | Token(type: .number, value: "2"),
56 | Token(type: .number, value: "1"),
57 | Token(type: .number, value: "23123"),
58 | Token(type: .txt, value: "abc"),
59 | Token(type: .txt, value: "ABC"),
60 | Token(type: .txt, value: "Hi"),
61 | Token(type: .number, value: "3333333"),
62 | Token(type: .number, value: "324"),
63 | Token(type: .number, value: "33"),
64 | Token(type: .number, value: "2"),
65 | Token(type: .eof, value: ""),
66 | ])
67 | }
68 |
69 | static var allTests = [
70 | ("testWords", testWords),
71 | ("testNumbers", testNumbers),
72 | ("testWordsAndNumbers", testWordsAndNumbers),
73 | ]
74 | }
75 |
--------------------------------------------------------------------------------
/Tests/SwiLexTests/XCTestManifests.swift:
--------------------------------------------------------------------------------
1 | import XCTest
2 |
3 | #if !canImport(ObjectiveC)
4 | public func allTests() -> [XCTestCaseEntry] {
5 | return [
6 | testCase(WordNumberTokensTests.allTests),
7 | testCase(CalculatorTokensTests.allTests),
8 | ]
9 | }
10 | #endif
11 |
--------------------------------------------------------------------------------