├── .github └── workflows │ └── integrate.yaml ├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── source └── tcppLibrary.hpp └── tests ├── CMakeLists.txt ├── coreTests.cpp ├── lexerTests.cpp ├── main.cpp ├── stringInputStreamTests.cpp └── tokensOutputStreamTests.cpp /.github/workflows/integrate.yaml: -------------------------------------------------------------------------------- 1 | name: Integration 2 | 3 | on: 4 | push: 5 | branches: [ main, master ] 6 | pull_request: 7 | branches: [ '**', '!gh-pages', '!coverage' ] 8 | types: [ opened, reopened, ready_for_review, synchronize ] 9 | workflow_call: 10 | inputs: 11 | ref: 12 | description: Reference to use for checking out 13 | default: ${{ github.sha }} 14 | type: string 15 | 16 | defaults: 17 | run: 18 | shell: bash 19 | 20 | jobs: 21 | build: 22 | name: Build 23 | strategy: 24 | matrix: 25 | os: [ ubuntu-latest, macos-latest, windows-latest ] 26 | runs-on: ${{ matrix.os }} 27 | steps: 28 | - name: Clone recurse submodules 29 | uses: actions/checkout@v3 30 | with: 31 | submodules: recursive 32 | - uses: seanmiddleditch/gha-setup-ninja@master 33 | - name: Configure 34 | run: | 35 | mkdir build 36 | cd build 37 | cmake -G Ninja .. 38 | - name: Make 39 | working-directory: build 40 | run: ninja 41 | - name: Test 42 | working-directory: build/tests 43 | run: ctest 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vs/ 2 | [Oo]bj/ 3 | [Bb]in/ 4 | tmp/ 5 | Release/ 6 | Debug/ 7 | build/ 8 | 9 | TODO 10 | 11 | *.opensdf 12 | *.sdf 13 | *.log 14 | 15 | *.sln 16 | *.user 17 | *.filters 18 | *.vcxproj 19 | 20 | CMakeCache.txt 21 | CMakeFiles 22 | CMakeScripts 23 | Testing 24 | Makefile 25 | cmake_install.cmake 26 | install_manifest.txt 27 | compile_commands.json 28 | CTestTestfile.cmake 29 | 30 | CMakeSettings.json -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "tests/ThirdParty/Catch2"] 2 | path = tests/ThirdParty/Catch2 3 | url = https://github.com/catchorg/Catch2.git 4 | branch = v2.x 5 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.8) 2 | 3 | project (tcpp) 4 | 5 | # Global variables are declared here 6 | set(TCPP_TESTS_NAME "tests") 7 | 8 | # Global options are declared here 9 | option(IS_PLUGIN_BUILDING_ENABLED "The option shows whether plugins should be built or not" ON) 10 | option(IS_SAMPLES_BUILDING_ENABLED "The option shows whether sample projects should be built or not" ON) 11 | option(IS_TESTING_ENABLED "The option turns on/off tests" ON) 12 | 13 | if (IS_TESTING_ENABLED) 14 | enable_testing() 15 | endif () 16 | 17 | if (IS_TESTING_ENABLED) 18 | add_subdirectory(tests) 19 | endif () 20 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Foreword 2 | 3 | First of all, thank you for you patience and attention for this project. We glad to see any new developer here. Hope that you've found out the project is useful for you. So if you want to contribute and support it, please, read this guide first before your hands get dirty. 4 | 5 | As I wrote above we glad to work with any developer with different skills level. If you have no enough experience in C# development you can work on documentation (e.g. write tutorials), write bug reports, etc. **good first issue** label in Issues tab will be the best entry point for all newcomers. 6 | 7 | ## Typical workflow 8 | 9 | *** 10 | 11 | * Design a feature. 12 | 13 | * Decompose it into a list of tasks. 14 | 15 | * Write code. 16 | 17 | * Cover written code with bunch of unit tests 18 | 19 | * Commit the result to the repository of the project. 20 | 21 | When we decide that the feature is ready for integration into the main build we firstly merge it into test-build. After successfull build of the branch later it can be merged into master. Usually master branch contains only final releases of the project. 22 | 23 | ## Styleguides 24 | 25 | *** 26 | 27 | ### Git commit messages styleguide 28 | 29 | In the project we try to stick to notation which was described at [this](https://chris.beams.io/posts/git-commit/) great article. 30 | 31 | We also use some kind of notation for names of branches. All branches that introduce a new feature should start from _**feature/branch-name**_. The following rules are used for other types of branches: 32 | 33 | * _**fix/branch-name**_ - fix some issue. 34 | 35 | * _**refactoring[/branch-name]**_ - the code in this branch refactors, optimize some part of the project. 36 | 37 | * _**test-build**_ - before the changes all the developers have introduced will appear in **master** branch they should be tested. We use this branch for this purpose. 38 | 39 | ### C\+\+ Styleguide 40 | 41 | * Be sure you use single tab for indentation. 42 | 43 | * All interfaces names should start from **I** prefix (e.g. **IManager**). Classes don't need any prefix (e.g. **Manager**). Structures identifiers starts from **T** prefix (e.g. **TVector2**). Names of classes, interfaces and structures should stick to camel case (e.g. **ComplexNameOfSomeClass**). 44 | 45 | * Enumerations' names start from **E_** prefix and stick to snake case. For instance, **E_MESSAGE_TYPE**. 46 | 47 | * Names of global variables and constants should start from Upper case, for instance **SingletonInstance**. 48 | 49 | * All local variables should start from lower case (**someLocalVariable**). Be sure choose some proper and clear names for them. But remember there is no special notation for names except described above. 50 | 51 | * Members of classes should start from *m* prefix (e.g. "mIsEnabled") including static variables. All public methods names should start from Upper case and stick to camel case. Protected and private methods start from **_** prefix (e.g. **_somePrivateMethod**). The same rules are applicable for properties both public, private (protected). Names of public events start from **On** prefix. **Is** prefix are appreciated for methods that are logical predicates that tell to a user whether some variable is true or false. 52 | 53 | * Stick to Allman style of indentation. For instance 54 | ```csharp 55 | //... 56 | while (true) 57 | { 58 | DoSomething1(); 59 | //... 60 | DoSomething2(); 61 | } 62 | //... 63 | ``` 64 | * Single-statement block should be wrapped in braces too. Also add extra space after operators like **if**, **for**, **switch** and etc. The extra space isn't used in case of method's invocation. 65 | ```csharp 66 | // Wrong (DoSomething() call should be wrapped with braces) 67 | while (true) 68 | DoSomething(); 69 | 70 | // Wrong (needs extra space after while) 71 | while(true) 72 | { 73 | DoSomething(); 74 | } 75 | 76 | // Wrong (function call doesn't need extra space) 77 | while (true) 78 | { 79 | DoSomething (); 80 | } 81 | 82 | // Right! 83 | while (true) 84 | { 85 | DoSomething(); 86 | } 87 | ``` 88 | 89 | These are the main rules of the notation that's used within the project. Hope that nothing important wasn't missed) If you have some questions that aren't mentioned here either write [e-mail](mailto:ildar2571@yandex.ru) or send message into our [gitter](https://gitter.im/bnoazx005/TinyECS). -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright (c) 2018 Ildar Kasimov ildar.kasimov94@gmail.com 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TCPP (Tiny C PreProcessor) 2 | 3 | [![Actions Status](https://github.com/bnoazx005/tcpp/actions/workflows/integrate.yaml/badge.svg)](https://github.com/bnoazx005/tcpp/actions) 4 | 5 | TCPP is small single-header library which provides implementation of C preprocessor (almost). Most part of the library is based upon official specifications of the preprocessor https://docs.freebsd.org/info/cpp/cpp.pdf, https://gcc.gnu.org/onlinedocs/cpp/. 6 | 7 | This project was started with the only one need which was to implement preprocessor for GLSL and HLSL languages and use it within custom game engine. That's it. So I've implemented the simplest working tool as I guess. And I really hope that this library will help for someone who shares my ideas and beliefs. I've not chosen Boost.Wave library because of its dependencies. Yeah, Boost looks so modular until you try to integrate the only module into some project. So, that's why this library is designed in single-header way. 8 | 9 | ## Table of contents 10 | 11 | 1. ### [Current Features](#current-features) 12 | 2. ### [How to Use](#how-to-use) 13 | 14 | *** 15 | 16 | ### Current Features: 17 | 18 | * Conditional preprocessing 19 | 20 | * Support of object-like and function-like macros, except variadic ones 21 | 22 | * Error handling via the only callback 23 | 24 | * Simple API: all the library based on a few classes with simplistic interfaces 25 | 26 | * Exceptions free code: we believe that's explicit error handling is much more robust and ease way 27 | 28 | *** 29 | 30 | ### How to Use 31 | 32 | Just copy **tcppLibrary.hpp** into your working directory and export it within some of your source file like the following 33 | ```cpp 34 | #define TCPP_IMPLEMENTATION 35 | #include "tcppLibrary.hpp" 36 | ``` 37 | Be sure, that's **TCPP_IMPLEMENTATION** flag is defined only once in the project. 38 | -------------------------------------------------------------------------------- /source/tcppLibrary.hpp: -------------------------------------------------------------------------------- 1 | /*! 2 | \file tcppLibrary.hpp 3 | \date 29.12.2019 4 | \author Ildar Kasimov 5 | 6 | This file is a single-header library which was written in C++14 standard. 7 | The main purpose of the library is to implement simple yet flexible C/C++ preprocessor. 8 | We've hardly tried to stick to C98 preprocessor's specifications, but if you've found 9 | some mistakes or inaccuracies, please, make us to know about them. The project has started 10 | as minimalistic implementation of preprocessor for GLSL and HLSL languages. 11 | 12 | The usage of the library is pretty simple, just copy this file into your enviornment and 13 | predefine TCPP_IMPLEMENTATION macro before its inclusion like the following 14 | 15 | \code 16 | #define TCPP_IMPLEMENTATION 17 | #include "tcppLibrary.hpp" 18 | \endcode 19 | 20 | Because of the initial goal of the project wasn't full featured C preprocessor but its subset 21 | that's useful for GLSL and HLSL, some points from the specification of the C preprocessor could 22 | not be applied to this library, at least for now. There is a list of unimplemented features placed 23 | below 24 | 25 | ##Configuration 26 | 27 | There are list of available options presented below. To enable some option just use #define directive before inclusion library's header file. 28 | Do not forget to provide this definitions in header files to provide access to specific types in case you need to use them. 29 | 30 | \code 31 | #define TCPP_SOME_EXAMPLE_CONFIG 32 | //... other configs 33 | #define TCPP_IMPLEMENTATION 34 | #include "tcppLibrary.hpp 35 | \endcode 36 | 37 | * TCPP_OUTPUT_TOKENS_EXTENSION_ENABLED: The config determines which approach is used to provide processed output for a user. 38 | Originally Preprocessor::Process returns output data as a std::string's instance. **TokensOutputStream** is returned with Preprocessor::Process 39 | in case of TCPP_OUTPUT_TOKENS_EXTENSION_ENABLED is defined 40 | 41 | \todo Implement support of char literals 42 | \todo Improve existing performance for massive input files 43 | \todo Add support of integral literals like L, u, etc 44 | \todo Implement built-in directives like #pragma, #error and others 45 | */ 46 | 47 | #pragma once 48 | 49 | #include 50 | #include 51 | #include 52 | #include 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | #include 59 | #include 60 | #include 61 | 62 | 63 | ///< Library's configs 64 | #define TCPP_DISABLE_EXCEPTIONS 1 65 | 66 | 67 | #if TCPP_DISABLE_EXCEPTIONS 68 | #define TCPP_NOEXCEPT noexcept 69 | #else 70 | #define TCPP_NOEXCEPT 71 | #endif 72 | 73 | #include 74 | #define TCPP_ASSERT(assertion) assert(assertion) 75 | 76 | 77 | namespace tcpp 78 | { 79 | /*! 80 | interface IInputStream 81 | 82 | \brief The interface describes the functionality that all input streams should 83 | provide 84 | */ 85 | 86 | class IInputStream 87 | { 88 | public: 89 | IInputStream() TCPP_NOEXCEPT = default; 90 | virtual ~IInputStream() TCPP_NOEXCEPT = default; 91 | 92 | virtual std::string ReadLine() TCPP_NOEXCEPT = 0; 93 | virtual bool HasNextLine() const TCPP_NOEXCEPT = 0; 94 | }; 95 | 96 | 97 | using TInputStreamUniquePtr = std::unique_ptr; 98 | 99 | 100 | /*! 101 | class StringInputStream 102 | 103 | \brief The class is the simplest implementation of the input stream, which 104 | is a simple string 105 | */ 106 | 107 | class StringInputStream : public IInputStream 108 | { 109 | public: 110 | StringInputStream() TCPP_NOEXCEPT = delete; 111 | explicit StringInputStream(const std::string& source) TCPP_NOEXCEPT; 112 | StringInputStream(const StringInputStream& inputStream) TCPP_NOEXCEPT; 113 | StringInputStream(StringInputStream&& inputStream) TCPP_NOEXCEPT; 114 | virtual ~StringInputStream() TCPP_NOEXCEPT = default; 115 | 116 | std::string ReadLine() TCPP_NOEXCEPT override; 117 | bool HasNextLine() const TCPP_NOEXCEPT override; 118 | 119 | StringInputStream& operator= (const StringInputStream&) TCPP_NOEXCEPT; 120 | StringInputStream& operator= (StringInputStream&&) TCPP_NOEXCEPT; 121 | private: 122 | std::string mSourceStr; 123 | }; 124 | 125 | 126 | enum class E_TOKEN_TYPE : unsigned int 127 | { 128 | IDENTIFIER, 129 | DEFINE, 130 | IF, 131 | ELSE, 132 | ELIF, 133 | UNDEF, 134 | ENDIF, 135 | INCLUDE, 136 | DEFINED, 137 | IFNDEF, 138 | IFDEF, 139 | SPACE, 140 | BLOB, 141 | OPEN_BRACKET, 142 | CLOSE_BRACKET, 143 | OPEN_SQUARE_BRACKET, 144 | CLOSE_SQUARE_BRACKET, 145 | OPEN_BRACE, 146 | CLOSE_BRACE, 147 | COMMA, 148 | NEWLINE, 149 | LESS, 150 | GREATER, 151 | QUOTES, 152 | KEYWORD, 153 | END, 154 | REJECT_MACRO, ///< Special type of a token to provide meta information 155 | STRINGIZE_OP, 156 | CONCAT_OP, 157 | NUMBER, 158 | PLUS, 159 | MINUS, 160 | SLASH, 161 | STAR, 162 | OR, 163 | AND, 164 | AMPERSAND, 165 | VLINE, 166 | LSHIFT, 167 | RSHIFT, 168 | NOT, 169 | GE, 170 | LE, 171 | EQ, 172 | NE, 173 | SEMICOLON, 174 | CUSTOM_DIRECTIVE, 175 | COMMENTARY, 176 | UNKNOWN, 177 | ELLIPSIS, 178 | }; 179 | 180 | 181 | /*! 182 | struct TToken 183 | 184 | \brief The structure is a type to contain all information about single token 185 | */ 186 | 187 | typedef struct TToken 188 | { 189 | E_TOKEN_TYPE mType; 190 | 191 | std::string mRawView; 192 | 193 | size_t mLineId; 194 | size_t mPos; 195 | } TToken, *TTokenPtr; 196 | 197 | 198 | using TTokensSequence = std::vector; 199 | using TTokensSequenceIter = TTokensSequence::iterator; 200 | using TTokensSequenceConstIter = TTokensSequence::const_iterator; 201 | 202 | 203 | #ifdef TCPP_OUTPUT_TOKENS_EXTENSION_ENABLED 204 | 205 | class TokensOutputStream 206 | { 207 | public: 208 | explicit TokensOutputStream(TTokensSequence tokens) TCPP_NOEXCEPT; 209 | 210 | // Methods to provide support of iterating over the sequence using for loop 211 | TTokensSequenceIter begin() TCPP_NOEXCEPT; 212 | TTokensSequenceIter end() TCPP_NOEXCEPT; 213 | TTokensSequenceConstIter begin() const TCPP_NOEXCEPT; 214 | TTokensSequenceConstIter end() const TCPP_NOEXCEPT; 215 | 216 | const TToken& GetNextToken() TCPP_NOEXCEPT; 217 | const TToken& PeekNextToken(size_t offset = 1) TCPP_NOEXCEPT; 218 | bool HasNextToken() const TCPP_NOEXCEPT; 219 | 220 | const TTokensSequence& GetSequence() const TCPP_NOEXCEPT; 221 | private: 222 | TTokensSequence mTokens{}; 223 | private: 224 | TTokensSequenceIter mCurrIt{}; 225 | }; 226 | 227 | #endif 228 | 229 | /*! 230 | class Lexer 231 | 232 | \brief The class implements lexer's functionality 233 | */ 234 | 235 | class Lexer 236 | { 237 | private: 238 | using TTokensQueue = std::list; 239 | using TStreamStack = std::stack; 240 | using TDirectivesMap = std::vector>; 241 | using TDirectiveHandlersArray = std::unordered_set; 242 | public: 243 | Lexer() TCPP_NOEXCEPT = delete; 244 | explicit Lexer(TInputStreamUniquePtr pInputStream) TCPP_NOEXCEPT; 245 | ~Lexer() TCPP_NOEXCEPT = default; 246 | 247 | bool AddCustomDirective(const std::string& directive) TCPP_NOEXCEPT; 248 | 249 | TToken GetNextToken() TCPP_NOEXCEPT; 250 | 251 | /*! 252 | \brief The method allows to peek token without changing current pointer 253 | 254 | \param[in] offset Distance of overlooking, passing 0 is equivalent to invoking GetNextToken() 255 | */ 256 | 257 | TToken PeekNextToken(size_t offset = 1); 258 | 259 | bool HasNextToken() const TCPP_NOEXCEPT; 260 | 261 | void AppendFront(const TTokensSequence& tokens) TCPP_NOEXCEPT; 262 | 263 | void PushStream(TInputStreamUniquePtr stream) TCPP_NOEXCEPT; 264 | void PopStream() TCPP_NOEXCEPT; 265 | 266 | size_t GetCurrLineIndex() const TCPP_NOEXCEPT; 267 | size_t GetCurrPos() const TCPP_NOEXCEPT; 268 | private: 269 | TToken _getNextTokenInternal(bool ignoreQueue) TCPP_NOEXCEPT; 270 | 271 | TToken _scanTokens(std::string& inputLine) TCPP_NOEXCEPT; 272 | 273 | std::string _requestSourceLine() TCPP_NOEXCEPT; 274 | 275 | TToken _scanSeparatorTokens(char ch, std::string& inputLine) TCPP_NOEXCEPT; 276 | 277 | IInputStream* _getActiveStream() const TCPP_NOEXCEPT; 278 | private: 279 | static const TToken mEOFToken; 280 | 281 | TDirectivesMap mDirectivesTable; 282 | 283 | TTokensQueue mTokensQueue; 284 | 285 | std::string mCurrLine; 286 | 287 | size_t mCurrLineIndex = 1; 288 | size_t mCurrPos = 0; 289 | 290 | TStreamStack mStreamsContext; 291 | 292 | TDirectiveHandlersArray mCustomDirectivesMap; 293 | }; 294 | 295 | 296 | /*! 297 | struct TMacroDesc 298 | 299 | \brief The type describes a single macro definition's description 300 | */ 301 | 302 | typedef struct TMacroDesc 303 | { 304 | std::string mName; 305 | std::vector mArgsNames; 306 | TTokensSequence mValue; 307 | bool mVariadic = false; 308 | } TMacroDesc, *TMacroDescPtr; 309 | 310 | 311 | enum class E_ERROR_TYPE : unsigned int 312 | { 313 | UNEXPECTED_TOKEN, 314 | UNBALANCED_ENDIF, 315 | INVALID_MACRO_DEFINITION, 316 | MACRO_ALREADY_DEFINED, 317 | INCONSISTENT_MACRO_ARITY, 318 | UNDEFINED_MACRO, 319 | INVALID_INCLUDE_DIRECTIVE, 320 | UNEXPECTED_END_OF_INCLUDE_PATH, 321 | ANOTHER_ELSE_BLOCK_FOUND, 322 | ELIF_BLOCK_AFTER_ELSE_FOUND, 323 | UNDEFINED_DIRECTIVE, 324 | INCORRECT_OPERATION_USAGE, 325 | INCORRECT_STRINGIFY_OPERATOR_USAGE, 326 | }; 327 | 328 | 329 | std::string ErrorTypeToString(const E_ERROR_TYPE& errorType) TCPP_NOEXCEPT; 330 | 331 | 332 | /*! 333 | struct TErrorInfo 334 | 335 | \brief The type contains all information about happened error 336 | */ 337 | 338 | typedef struct TErrorInfo 339 | { 340 | E_ERROR_TYPE mType; 341 | size_t mLine; 342 | } TErrorInfo, *TErrorInfoPtr; 343 | 344 | 345 | /*! 346 | class Preprocessor 347 | 348 | \brief The class implements main functionality of C preprocessor. To preprocess 349 | some source code string, create an instance of this class and call Process method. 350 | The usage of the class is the same as iterators. 351 | */ 352 | 353 | class Preprocessor 354 | { 355 | public: 356 | using TOnErrorCallback = std::function; 357 | using TOnIncludeCallback = std::function; 358 | using TSymTable = std::vector; 359 | using TContextStack = std::list; 360 | using TDirectiveHandler = std::function; 361 | using TDirectivesMap = std::unordered_map; 362 | 363 | typedef struct TPreprocessorConfigInfo 364 | { 365 | TOnErrorCallback mOnErrorCallback = {}; 366 | TOnIncludeCallback mOnIncludeCallback = {}; 367 | 368 | bool mSkipComments = false; ///< When it's true all tokens which are E_TOKEN_TYPE::COMMENTARY will be thrown away from preprocessor's output 369 | } TPreprocessorConfigInfo, *TPreprocessorConfigInfoPtr; 370 | 371 | typedef struct TIfStackEntry 372 | { 373 | bool mShouldBeSkipped = true; 374 | bool mHasElseBeenFound = false; 375 | bool mHasIfBlockBeenEntered = false; 376 | bool mIsParentBlockActive = true; 377 | 378 | TIfStackEntry(bool shouldBeSkipped, bool isParentBlockActive) : 379 | mShouldBeSkipped(shouldBeSkipped), 380 | mHasElseBeenFound(false), 381 | mHasIfBlockBeenEntered(!shouldBeSkipped), 382 | mIsParentBlockActive(isParentBlockActive) {} 383 | } TIfStackEntry, *TIfStackEntryPtr; 384 | 385 | using TIfStack = std::stack; 386 | 387 | using TPreprocessResult = 388 | #ifdef TCPP_OUTPUT_TOKENS_EXTENSION_ENABLED 389 | TokensOutputStream; 390 | #else 391 | TTokensSequence; 392 | #endif 393 | 394 | public: 395 | Preprocessor() TCPP_NOEXCEPT = delete; 396 | Preprocessor(const Preprocessor&) TCPP_NOEXCEPT = delete; 397 | Preprocessor(Lexer& lexer, const TPreprocessorConfigInfo& config) TCPP_NOEXCEPT; 398 | ~Preprocessor() TCPP_NOEXCEPT = default; 399 | 400 | bool AddCustomDirectiveHandler(const std::string& directive, const TDirectiveHandler& handler) TCPP_NOEXCEPT; 401 | 402 | TPreprocessResult Process() TCPP_NOEXCEPT; 403 | static std::string ToString(const TPreprocessResult& tokens) TCPP_NOEXCEPT; 404 | 405 | Preprocessor& operator= (const Preprocessor&) TCPP_NOEXCEPT = delete; 406 | 407 | TSymTable GetSymbolsTable() const TCPP_NOEXCEPT; 408 | private: 409 | void _createMacroDefinition() TCPP_NOEXCEPT; 410 | void _removeMacroDefinition(const std::string& macroName) TCPP_NOEXCEPT; 411 | 412 | TTokensSequence _expandMacroDefinition(const TMacroDesc& macroDesc, const TToken& idToken, const std::function& getNextTokenCallback) const TCPP_NOEXCEPT; 413 | TTokensSequence _expandArg(const std::function& getNextTokenCallback) const TCPP_NOEXCEPT; 414 | 415 | void _expect(const E_TOKEN_TYPE& expectedType, const E_TOKEN_TYPE& actualType) const TCPP_NOEXCEPT; 416 | 417 | void _processInclusion() TCPP_NOEXCEPT; 418 | 419 | TIfStackEntry _processIfConditional() TCPP_NOEXCEPT; 420 | TIfStackEntry _processIfdefConditional() TCPP_NOEXCEPT; 421 | TIfStackEntry _processIfndefConditional() TCPP_NOEXCEPT; 422 | void _processElseConditional(TIfStackEntry& currStackEntry) TCPP_NOEXCEPT; 423 | void _processElifConditional(TIfStackEntry& currStackEntry) TCPP_NOEXCEPT; 424 | 425 | int _evaluateExpression(const TTokensSequence& exprTokens) const TCPP_NOEXCEPT; 426 | 427 | bool _shouldTokenBeSkipped() const TCPP_NOEXCEPT; 428 | private: 429 | Lexer* mpLexer; 430 | 431 | TOnErrorCallback mOnErrorCallback; 432 | TOnIncludeCallback mOnIncludeCallback; 433 | 434 | TSymTable mSymTable; 435 | mutable TContextStack mContextStack; 436 | TIfStack mConditionalBlocksStack; 437 | TDirectivesMap mCustomDirectivesHandlersMap; 438 | 439 | bool mSkipCommentsTokens; 440 | }; 441 | 442 | 443 | ///< implementation of the library is placed below 444 | #if defined(TCPP_IMPLEMENTATION) 445 | 446 | static const std::string EMPTY_STR_VALUE = ""; 447 | 448 | 449 | std::string ErrorTypeToString(const E_ERROR_TYPE& errorType) TCPP_NOEXCEPT 450 | { 451 | switch (errorType) 452 | { 453 | case E_ERROR_TYPE::UNEXPECTED_TOKEN: 454 | return "Unexpected token"; 455 | case E_ERROR_TYPE::UNBALANCED_ENDIF: 456 | return "Unbalanced endif"; 457 | case E_ERROR_TYPE::INVALID_MACRO_DEFINITION: 458 | return "Invalid macro definition"; 459 | case E_ERROR_TYPE::MACRO_ALREADY_DEFINED: 460 | return "The macro is already defined"; 461 | case E_ERROR_TYPE::INCONSISTENT_MACRO_ARITY: 462 | return "Inconsistent number of arguments between definition and invocation of the macro"; 463 | case E_ERROR_TYPE::UNDEFINED_MACRO: 464 | return "Undefined macro"; 465 | case E_ERROR_TYPE::INVALID_INCLUDE_DIRECTIVE: 466 | return "Invalid #include directive"; 467 | case E_ERROR_TYPE::UNEXPECTED_END_OF_INCLUDE_PATH: 468 | return "Unexpected end of include path"; 469 | case E_ERROR_TYPE::ANOTHER_ELSE_BLOCK_FOUND: 470 | return "#else directive should be last one"; 471 | case E_ERROR_TYPE::ELIF_BLOCK_AFTER_ELSE_FOUND: 472 | return "#elif found after #else block"; 473 | case E_ERROR_TYPE::UNDEFINED_DIRECTIVE: 474 | return "Undefined directive"; 475 | case E_ERROR_TYPE::INCORRECT_OPERATION_USAGE: 476 | return "Incorrect operation usage"; 477 | case E_ERROR_TYPE::INCORRECT_STRINGIFY_OPERATOR_USAGE: 478 | return "Incorrect usage of stringification operation"; 479 | } 480 | 481 | return EMPTY_STR_VALUE; 482 | } 483 | 484 | 485 | StringInputStream::StringInputStream(const std::string& source) TCPP_NOEXCEPT: 486 | IInputStream(), mSourceStr(source) 487 | { 488 | } 489 | 490 | StringInputStream::StringInputStream(const StringInputStream& inputStream) TCPP_NOEXCEPT: 491 | mSourceStr(inputStream.mSourceStr) 492 | { 493 | } 494 | 495 | StringInputStream::StringInputStream(StringInputStream&& inputStream) TCPP_NOEXCEPT: 496 | mSourceStr(std::move(inputStream.mSourceStr)) 497 | { 498 | } 499 | 500 | std::string StringInputStream::ReadLine() TCPP_NOEXCEPT 501 | { 502 | std::string::size_type pos = mSourceStr.find_first_of('\n'); 503 | pos = (pos == std::string::npos) ? pos : (pos + 1); 504 | 505 | std::string currLine = mSourceStr.substr(0, pos); 506 | mSourceStr.erase(0, pos); 507 | 508 | return currLine; 509 | } 510 | 511 | bool StringInputStream::HasNextLine() const TCPP_NOEXCEPT 512 | { 513 | return !mSourceStr.empty(); 514 | } 515 | 516 | StringInputStream& StringInputStream::operator= (const StringInputStream& stream) TCPP_NOEXCEPT 517 | { 518 | mSourceStr = stream.mSourceStr; 519 | return *this; 520 | } 521 | 522 | StringInputStream& StringInputStream::operator= (StringInputStream&& stream) TCPP_NOEXCEPT 523 | { 524 | mSourceStr = std::move(stream.mSourceStr); 525 | return *this; 526 | } 527 | 528 | 529 | const TToken Lexer::mEOFToken = { E_TOKEN_TYPE::END }; 530 | 531 | Lexer::Lexer(TInputStreamUniquePtr pInputStream) TCPP_NOEXCEPT: 532 | mDirectivesTable 533 | { 534 | { "define", E_TOKEN_TYPE::DEFINE }, 535 | { "ifdef", E_TOKEN_TYPE::IFDEF }, 536 | { "ifndef", E_TOKEN_TYPE::IFNDEF }, 537 | { "if", E_TOKEN_TYPE::IF }, 538 | { "else", E_TOKEN_TYPE::ELSE }, 539 | { "elif", E_TOKEN_TYPE::ELIF }, 540 | { "undef", E_TOKEN_TYPE::UNDEF }, 541 | { "endif", E_TOKEN_TYPE::ENDIF }, 542 | { "include", E_TOKEN_TYPE::INCLUDE }, 543 | { "defined", E_TOKEN_TYPE::DEFINED }, 544 | }, mCurrLine(), mCurrLineIndex(0) 545 | { 546 | PushStream(std::move(pInputStream)); 547 | } 548 | 549 | bool Lexer::AddCustomDirective(const std::string& directive) TCPP_NOEXCEPT 550 | { 551 | if (mCustomDirectivesMap.find(directive) != mCustomDirectivesMap.cend()) 552 | { 553 | return false; 554 | } 555 | 556 | mCustomDirectivesMap.insert(directive); 557 | return true; 558 | } 559 | 560 | TToken Lexer::GetNextToken() TCPP_NOEXCEPT 561 | { 562 | return _getNextTokenInternal(false); 563 | } 564 | 565 | TToken Lexer::PeekNextToken(size_t offset) 566 | { 567 | if (!mTokensQueue.empty() && offset < mTokensQueue.size()) 568 | { 569 | return *std::next(mTokensQueue.begin(), offset); 570 | } 571 | 572 | const size_t count = offset - mTokensQueue.size(); 573 | for (size_t i = 0; i < count; i++) 574 | { 575 | mTokensQueue.push_back(_getNextTokenInternal(true)); 576 | } 577 | 578 | if (mTokensQueue.empty()) 579 | { 580 | return GetNextToken(); 581 | } 582 | 583 | return mTokensQueue.back(); 584 | } 585 | 586 | bool Lexer::HasNextToken() const TCPP_NOEXCEPT 587 | { 588 | IInputStream* pCurrInputStream = _getActiveStream(); 589 | return (pCurrInputStream ? pCurrInputStream->HasNextLine() : false) || !mCurrLine.empty() || !mTokensQueue.empty(); 590 | } 591 | 592 | void Lexer::AppendFront(const TTokensSequence& tokens) TCPP_NOEXCEPT 593 | { 594 | mTokensQueue.insert(mTokensQueue.begin(), tokens.begin(), tokens.end()); 595 | } 596 | 597 | void Lexer::PushStream(TInputStreamUniquePtr stream) TCPP_NOEXCEPT 598 | { 599 | TCPP_ASSERT(stream); 600 | 601 | if (!stream) 602 | { 603 | return; 604 | } 605 | 606 | mStreamsContext.push(std::move(stream)); 607 | } 608 | 609 | void Lexer::PopStream() TCPP_NOEXCEPT 610 | { 611 | if (mStreamsContext.empty()) 612 | { 613 | return; 614 | } 615 | 616 | mStreamsContext.pop(); 617 | } 618 | 619 | size_t Lexer::GetCurrLineIndex() const TCPP_NOEXCEPT 620 | { 621 | return mCurrLineIndex; 622 | } 623 | 624 | size_t Lexer::GetCurrPos() const TCPP_NOEXCEPT 625 | { 626 | return mCurrPos; 627 | } 628 | 629 | 630 | static std::tuple EatNextChar(std::string& str, size_t pos, size_t count = 1) 631 | { 632 | str.erase(0, count); 633 | return { pos + count, str.empty() ? static_cast(EOF) : str.front() }; 634 | } 635 | 636 | 637 | static char PeekNextChar(const std::string& str, size_t step = 1) 638 | { 639 | return (step < str.length()) ? str[step] : static_cast(EOF); 640 | } 641 | 642 | 643 | static std::string ExtractSingleLineComment(const std::string& currInput) TCPP_NOEXCEPT 644 | { 645 | std::stringstream ss(currInput); 646 | std::string commentStr; 647 | 648 | std::getline(ss, commentStr, '\n'); 649 | return commentStr; 650 | } 651 | 652 | 653 | static std::string ExtractMultiLineComments(std::string& currInput, const std::function& requestNextLineFunctor) TCPP_NOEXCEPT 654 | { 655 | std::string input = currInput; 656 | std::string commentStr; 657 | 658 | // \note here below all states of DFA are placed 659 | std::function enterCommentBlock = [&enterCommentBlock, &commentStr, requestNextLineFunctor](std::string& input) 660 | { 661 | commentStr.append(input.substr(0, 2)); 662 | input.erase(0, 2); // \note remove /* 663 | 664 | while (input.rfind("*/", 0) != 0 && !input.empty()) 665 | { 666 | commentStr.push_back(input.front()); 667 | input.erase(0, 1); 668 | 669 | if (input.rfind("//", 0) == 0) 670 | { 671 | commentStr.append(input.substr(0, 2)); 672 | input.erase(0, 2); 673 | } 674 | 675 | if (input.rfind("/*", 0) == 0) 676 | { 677 | input = enterCommentBlock(input); 678 | } 679 | 680 | if (input.empty() && requestNextLineFunctor) 681 | { 682 | input = requestNextLineFunctor(); 683 | } 684 | } 685 | 686 | commentStr.append(input.substr(0, 2)); 687 | input.erase(0, 2); // \note remove */ 688 | 689 | return input; 690 | }; 691 | 692 | std::string::size_type pos = input.find("/*"); 693 | if (pos != std::string::npos) 694 | { 695 | std::string restStr = input.substr(pos, std::string::npos); 696 | enterCommentBlock(restStr); 697 | 698 | currInput = commentStr + restStr; 699 | 700 | return commentStr; 701 | } 702 | 703 | return commentStr; 704 | } 705 | 706 | 707 | TToken Lexer::_getNextTokenInternal(bool ignoreQueue) TCPP_NOEXCEPT 708 | { 709 | if (!ignoreQueue && !mTokensQueue.empty()) 710 | { 711 | auto currToken = mTokensQueue.front(); 712 | mTokensQueue.pop_front(); 713 | 714 | return currToken; 715 | } 716 | 717 | if (mCurrLine.empty()) 718 | { 719 | // \note if it's still empty then we've reached the end of the source 720 | if ((mCurrLine = _requestSourceLine()).empty()) 721 | { 722 | return mEOFToken; 723 | } 724 | } 725 | 726 | return _scanTokens(mCurrLine); 727 | } 728 | 729 | TToken Lexer::_scanTokens(std::string& inputLine) TCPP_NOEXCEPT 730 | { 731 | char ch = '\0'; 732 | 733 | static const std::unordered_set keywordsMap 734 | { 735 | "auto", "double", "int", "struct", 736 | "break", "else", "long", "switch", 737 | "case", "enum", "register", "typedef", 738 | "char", "extern", "return", "union", 739 | "const", "float", "short", "unsigned", 740 | "continue", "for", "signed", "void", 741 | "default", "goto", "sizeof", "volatile", 742 | "do", "if", "static", "while" 743 | }; 744 | 745 | static const std::string separators = ",()[]<>\"+-*/&|!=;{}"; 746 | 747 | std::string currStr = EMPTY_STR_VALUE; 748 | 749 | while (!inputLine.empty()) 750 | { 751 | ch = inputLine.front(); 752 | 753 | if (ch == '.' && PeekNextChar(inputLine, 1) == '.' && PeekNextChar(inputLine, 2) == '.') 754 | { 755 | // flush current blob 756 | if (!currStr.empty()) 757 | { 758 | return { E_TOKEN_TYPE::BLOB, currStr, mCurrLineIndex, mCurrPos }; 759 | } 760 | 761 | mCurrPos = std::get(EatNextChar(inputLine, mCurrPos, 3)); 762 | return { E_TOKEN_TYPE::ELLIPSIS, "...", mCurrLineIndex, mCurrPos }; 763 | } 764 | 765 | if (ch == '/') 766 | { 767 | std::string commentStr; 768 | 769 | if (PeekNextChar(inputLine, 1) == '/') // \note Found a single line C++ style comment 770 | { 771 | commentStr = ExtractSingleLineComment(inputLine); 772 | } 773 | else if (PeekNextChar(inputLine, 1) == '*') /// \note multi-line commentary 774 | { 775 | commentStr = ExtractMultiLineComments(inputLine, std::bind(&Lexer::_requestSourceLine, this)); 776 | } 777 | 778 | if (!commentStr.empty()) 779 | { 780 | mCurrPos = std::get(EatNextChar(inputLine, mCurrPos, commentStr.length())); 781 | return { E_TOKEN_TYPE::COMMENTARY, commentStr, mCurrLineIndex, mCurrPos }; 782 | } 783 | } 784 | 785 | if (ch == '\n' || ch == '\r') 786 | { 787 | // flush current blob 788 | if (!currStr.empty()) 789 | { 790 | return { E_TOKEN_TYPE::BLOB, currStr, mCurrLineIndex }; 791 | } 792 | 793 | std::string separatorStr; 794 | separatorStr.push_back(ch); 795 | 796 | const char nextCh = PeekNextChar(inputLine, 1); 797 | if (ch == '\r' && nextCh == '\n') 798 | { 799 | separatorStr.push_back(nextCh); 800 | } 801 | 802 | mCurrPos = std::get(EatNextChar(inputLine, mCurrPos, separatorStr.length())); 803 | return { E_TOKEN_TYPE::NEWLINE, separatorStr, mCurrLineIndex, mCurrPos}; 804 | } 805 | 806 | if (std::isspace(ch)) 807 | { 808 | // flush current blob 809 | if (!currStr.empty()) 810 | { 811 | return { E_TOKEN_TYPE::BLOB, currStr, mCurrLineIndex, mCurrPos }; 812 | } 813 | 814 | std::string separatorStr; 815 | separatorStr.push_back(inputLine.front()); 816 | 817 | mCurrPos = std::get(EatNextChar(inputLine, mCurrPos)); 818 | return { E_TOKEN_TYPE::SPACE, std::move(separatorStr), mCurrLineIndex, mCurrPos }; 819 | } 820 | 821 | if (ch == '#') // \note it could be # operator or a directive 822 | { 823 | // flush current blob 824 | if (!currStr.empty()) 825 | { 826 | return { E_TOKEN_TYPE::BLOB, currStr, mCurrLineIndex, mCurrPos }; 827 | } 828 | 829 | /// \note Skip whitespaces if there're exist 830 | do 831 | { 832 | mCurrPos = std::get(EatNextChar(inputLine, mCurrPos)); 833 | } 834 | while (std::isspace(PeekNextChar(inputLine, 0))); 835 | 836 | for (const auto& currDirective : mDirectivesTable) 837 | { 838 | auto&& currDirectiveStr = std::get(currDirective); 839 | 840 | if (inputLine.rfind(currDirectiveStr, 0) == 0) 841 | { 842 | inputLine.erase(0, currDirectiveStr.length()); 843 | mCurrPos += currDirectiveStr.length(); 844 | 845 | return { std::get(currDirective), EMPTY_STR_VALUE, mCurrLineIndex, mCurrPos }; 846 | } 847 | } 848 | 849 | // \note custom directives 850 | for (const auto& currDirectiveStr : mCustomDirectivesMap) 851 | { 852 | if (inputLine.rfind(currDirectiveStr, 0) == 0) 853 | { 854 | inputLine.erase(0, currDirectiveStr.length()); 855 | mCurrPos += currDirectiveStr.length(); 856 | 857 | return { E_TOKEN_TYPE::CUSTOM_DIRECTIVE, currDirectiveStr, mCurrLineIndex, mCurrPos }; 858 | } 859 | } 860 | 861 | // \note if we've reached this line it's # operator not directive 862 | if (!inputLine.empty()) 863 | { 864 | char nextCh = inputLine.front(); 865 | switch (nextCh) 866 | { 867 | case '#': // \note concatenation operator 868 | inputLine.erase(0, 1); 869 | ++mCurrPos; 870 | return { E_TOKEN_TYPE::CONCAT_OP, EMPTY_STR_VALUE, mCurrLineIndex, mCurrPos }; 871 | default: 872 | if (nextCh != ' ') // \note stringification operator 873 | { 874 | return { E_TOKEN_TYPE::STRINGIZE_OP, EMPTY_STR_VALUE, mCurrLineIndex, mCurrPos }; 875 | } 876 | 877 | return { E_TOKEN_TYPE::BLOB, "#", mCurrLineIndex, mCurrPos }; 878 | } 879 | } 880 | } 881 | 882 | if (std::isdigit(ch)) 883 | { 884 | // flush current blob 885 | if (!currStr.empty()) 886 | { 887 | return { E_TOKEN_TYPE::BLOB, currStr, mCurrLineIndex, mCurrPos }; 888 | } 889 | 890 | std::string number; 891 | std::string::size_type i = 0; 892 | 893 | if (ch == '0' && !inputLine.empty()) 894 | { 895 | mCurrPos = std::get(EatNextChar(inputLine, mCurrPos)); 896 | 897 | number.push_back(ch); 898 | 899 | char nextCh = inputLine.front(); 900 | if (nextCh == 'x' || std::isdigit(nextCh)) 901 | { 902 | inputLine.erase(0, 1); 903 | ++mCurrPos; 904 | 905 | number.push_back(nextCh); 906 | } 907 | else 908 | { 909 | return { E_TOKEN_TYPE::NUMBER, number, mCurrLineIndex, mCurrPos }; 910 | } 911 | } 912 | 913 | uint8_t charsToRemove = 0; 914 | 915 | while ((i < inputLine.length()) && std::isdigit(ch = inputLine[i++])) 916 | { 917 | number.push_back(ch); 918 | ++charsToRemove; 919 | } 920 | 921 | inputLine.erase(0, charsToRemove); 922 | mCurrPos += charsToRemove; 923 | 924 | return { E_TOKEN_TYPE::NUMBER, number, mCurrLineIndex, mCurrPos }; 925 | } 926 | 927 | if (ch == '_' || std::isalpha(ch)) ///< \note parse identifier 928 | { 929 | // flush current blob 930 | if (!currStr.empty()) 931 | { 932 | return { E_TOKEN_TYPE::BLOB, currStr, mCurrLineIndex }; 933 | } 934 | 935 | std::string identifier; 936 | 937 | do 938 | { 939 | identifier.push_back(ch); 940 | mCurrPos = std::get(EatNextChar(inputLine, mCurrPos)); 941 | } while (!inputLine.empty() && (std::isalnum(ch = inputLine.front()) || (ch == '_'))); 942 | 943 | return { (keywordsMap.find(identifier) != keywordsMap.cend()) ? E_TOKEN_TYPE::KEYWORD : E_TOKEN_TYPE::IDENTIFIER, identifier, mCurrLineIndex, mCurrPos }; 944 | } 945 | 946 | mCurrPos = std::get(EatNextChar(inputLine, mCurrPos)); 947 | 948 | if ((separators.find_first_of(ch) != std::string::npos)) 949 | { 950 | if (!currStr.empty()) 951 | { 952 | auto separatingToken = _scanSeparatorTokens(ch, inputLine); 953 | if (separatingToken.mType != mEOFToken.mType) 954 | { 955 | mTokensQueue.push_front(separatingToken); 956 | } 957 | 958 | return { E_TOKEN_TYPE::BLOB, currStr, mCurrLineIndex, mCurrPos }; // flush current blob 959 | } 960 | 961 | auto separatingToken = _scanSeparatorTokens(ch, inputLine); 962 | if (separatingToken.mType != mEOFToken.mType) 963 | { 964 | return separatingToken; 965 | } 966 | } 967 | 968 | currStr.push_back(ch); 969 | } 970 | 971 | // flush current blob 972 | if (!currStr.empty()) 973 | { 974 | return { E_TOKEN_TYPE::BLOB, currStr, mCurrLineIndex, mCurrPos }; 975 | } 976 | 977 | PopStream(); 978 | 979 | //\note try to continue preprocessing if there is at least one input stream 980 | if (!mStreamsContext.empty()) 981 | { 982 | return GetNextToken(); 983 | } 984 | 985 | return mEOFToken; 986 | } 987 | 988 | 989 | static bool IsEscapeSequenceAtPos(const std::string& str, std::string::size_type pos) 990 | { 991 | if (pos + 1 >= str.length() || pos >= str.length() || str[pos] != '\\') 992 | { 993 | return false; 994 | } 995 | 996 | static constexpr char escapeSymbols[] { '\'', '\\', 'n', '\"', 'a', 'b', 'f', 'r', 't', 'v' }; 997 | 998 | char testedSymbol = str[pos + 1]; 999 | 1000 | for (char ch : escapeSymbols) 1001 | { 1002 | if (ch == testedSymbol) 1003 | { 1004 | return true; 1005 | } 1006 | } 1007 | 1008 | return false; 1009 | } 1010 | 1011 | 1012 | /*! 1013 | \return The method eats all whitespace tokens (TT_SPACE, TT_COMMENTARY) and returns current token as a result 1014 | */ 1015 | 1016 | template 1017 | static TToken TrySkipWhitespaceTokensSequence(TAction getNextToken, const TToken& initialToken) 1018 | { 1019 | TToken currToken = initialToken; 1020 | 1021 | while (currToken.mType == E_TOKEN_TYPE::SPACE) 1022 | { 1023 | currToken = getNextToken(); 1024 | } 1025 | 1026 | return currToken; 1027 | } 1028 | 1029 | 1030 | std::string Lexer::_requestSourceLine() TCPP_NOEXCEPT 1031 | { 1032 | IInputStream* pCurrInputStream = _getActiveStream(); 1033 | 1034 | if (!pCurrInputStream->HasNextLine()) 1035 | { 1036 | return EMPTY_STR_VALUE; 1037 | } 1038 | 1039 | std::string sourceLine = pCurrInputStream->ReadLine(); 1040 | ++mCurrLineIndex; 1041 | 1042 | /// \note join lines that were splitted with backslash sign 1043 | std::string::size_type pos = 0; 1044 | while (((pos = sourceLine.find_first_of('\\')) != std::string::npos) 1045 | && (std::isspace(PeekNextChar(sourceLine, pos + 1)) || PeekNextChar(sourceLine, pos + 1) == EOF) && !IsEscapeSequenceAtPos(sourceLine, pos)) 1046 | { 1047 | if (pCurrInputStream->HasNextLine()) 1048 | { 1049 | sourceLine.replace(pos ? (pos - 1) : 0, std::string::npos, pCurrInputStream->ReadLine()); 1050 | ++mCurrLineIndex; 1051 | 1052 | continue; 1053 | } 1054 | 1055 | sourceLine.erase(sourceLine.begin() + pos, sourceLine.end()); 1056 | } 1057 | 1058 | return sourceLine; 1059 | } 1060 | 1061 | TToken Lexer::_scanSeparatorTokens(char ch, std::string& inputLine) TCPP_NOEXCEPT 1062 | { 1063 | switch (ch) 1064 | { 1065 | case ',': 1066 | return { E_TOKEN_TYPE::COMMA, ",", mCurrLineIndex, mCurrPos }; 1067 | case '(': 1068 | return { E_TOKEN_TYPE::OPEN_BRACKET, "(", mCurrLineIndex, mCurrPos }; 1069 | case ')': 1070 | return { E_TOKEN_TYPE::CLOSE_BRACKET, ")", mCurrLineIndex, mCurrPos }; 1071 | case '[': 1072 | return { E_TOKEN_TYPE::OPEN_SQUARE_BRACKET, "[", mCurrLineIndex, mCurrPos }; 1073 | case ']': 1074 | return { E_TOKEN_TYPE::CLOSE_SQUARE_BRACKET, "]", mCurrLineIndex, mCurrPos }; 1075 | case '{': 1076 | return { E_TOKEN_TYPE::OPEN_BRACE, "{", mCurrLineIndex, mCurrPos }; 1077 | case '}': 1078 | return { E_TOKEN_TYPE::CLOSE_BRACE, "}", mCurrLineIndex, mCurrPos }; 1079 | case '<': 1080 | if (!inputLine.empty()) 1081 | { 1082 | char nextCh = inputLine.front(); 1083 | switch (nextCh) 1084 | { 1085 | case '<': 1086 | inputLine.erase(0, 1); 1087 | ++mCurrPos; 1088 | return { E_TOKEN_TYPE::LSHIFT, "<<", mCurrLineIndex, mCurrPos }; 1089 | case '=': 1090 | inputLine.erase(0, 1); 1091 | ++mCurrPos; 1092 | return { E_TOKEN_TYPE::LE, "<=", mCurrLineIndex, mCurrPos }; 1093 | } 1094 | } 1095 | 1096 | return { E_TOKEN_TYPE::LESS, "<", mCurrLineIndex, mCurrPos }; 1097 | case '>': 1098 | if (!inputLine.empty()) 1099 | { 1100 | char nextCh = inputLine.front(); 1101 | switch (nextCh) 1102 | { 1103 | case '>': 1104 | inputLine.erase(0, 1); 1105 | ++mCurrPos; 1106 | return { E_TOKEN_TYPE::RSHIFT, ">>", mCurrLineIndex, mCurrPos }; 1107 | case '=': 1108 | inputLine.erase(0, 1); 1109 | ++mCurrPos; 1110 | return { E_TOKEN_TYPE::GE, ">=", mCurrLineIndex, mCurrPos }; 1111 | } 1112 | } 1113 | 1114 | return { E_TOKEN_TYPE::GREATER, ">", mCurrLineIndex, mCurrPos }; 1115 | case '\"': 1116 | return { E_TOKEN_TYPE::QUOTES, "\"", mCurrLineIndex, mCurrPos }; 1117 | case '+': 1118 | return { E_TOKEN_TYPE::PLUS, "+", mCurrLineIndex, mCurrPos }; 1119 | case '-': 1120 | return { E_TOKEN_TYPE::MINUS, "-", mCurrLineIndex, mCurrPos }; 1121 | case '*': 1122 | return { E_TOKEN_TYPE::STAR, "*", mCurrLineIndex, mCurrPos }; 1123 | case '/': 1124 | return { E_TOKEN_TYPE::SLASH, "/", mCurrLineIndex, mCurrPos }; 1125 | case '&': 1126 | if (!inputLine.empty() && inputLine.front() == '&') 1127 | { 1128 | inputLine.erase(0, 1); 1129 | ++mCurrPos; 1130 | return { E_TOKEN_TYPE::AND, "&&", mCurrLineIndex, mCurrPos }; 1131 | } 1132 | 1133 | return { E_TOKEN_TYPE::AMPERSAND, "&", mCurrLineIndex, mCurrPos }; 1134 | case '|': 1135 | if (!inputLine.empty() && inputLine.front() == '|') 1136 | { 1137 | inputLine.erase(0, 1); 1138 | ++mCurrPos; 1139 | return { E_TOKEN_TYPE::OR, "||", mCurrLineIndex, mCurrPos }; 1140 | } 1141 | 1142 | return { E_TOKEN_TYPE::VLINE, "|", mCurrLineIndex, mCurrPos }; 1143 | case '!': 1144 | if (!inputLine.empty() && inputLine.front() == '=') 1145 | { 1146 | inputLine.erase(0, 1); 1147 | ++mCurrPos; 1148 | return { E_TOKEN_TYPE::NE, "!=", mCurrLineIndex, mCurrPos }; 1149 | } 1150 | 1151 | return { E_TOKEN_TYPE::NOT, "!", mCurrLineIndex, mCurrPos }; 1152 | case '=': 1153 | if (!inputLine.empty() && inputLine.front() == '=') 1154 | { 1155 | inputLine.erase(0, 1); 1156 | ++mCurrPos; 1157 | return { E_TOKEN_TYPE::EQ, "==", mCurrLineIndex, mCurrPos }; 1158 | } 1159 | 1160 | return { E_TOKEN_TYPE::BLOB, "=", mCurrLineIndex, mCurrPos }; 1161 | 1162 | case ';': 1163 | return { E_TOKEN_TYPE::SEMICOLON, ";", mCurrLineIndex, mCurrPos }; 1164 | } 1165 | 1166 | return mEOFToken; 1167 | } 1168 | 1169 | IInputStream* Lexer::_getActiveStream() const TCPP_NOEXCEPT 1170 | { 1171 | return mStreamsContext.empty() ? nullptr : mStreamsContext.top().get(); 1172 | } 1173 | 1174 | 1175 | static const std::vector BuiltInDefines 1176 | { 1177 | "__LINE__", 1178 | "__VA_ARGS__", 1179 | }; 1180 | 1181 | 1182 | Preprocessor::Preprocessor(Lexer& lexer, const TPreprocessorConfigInfo& config) TCPP_NOEXCEPT: 1183 | mpLexer(&lexer), mOnErrorCallback(config.mOnErrorCallback), mOnIncludeCallback(config.mOnIncludeCallback), mSkipCommentsTokens(config.mSkipComments) 1184 | { 1185 | for (auto&& currSystemDefine : BuiltInDefines) 1186 | { 1187 | mSymTable.push_back({ currSystemDefine }); 1188 | } 1189 | } 1190 | 1191 | bool Preprocessor::AddCustomDirectiveHandler(const std::string& directive, const TDirectiveHandler& handler) TCPP_NOEXCEPT 1192 | { 1193 | if ((mCustomDirectivesHandlersMap.find(directive) != mCustomDirectivesHandlersMap.cend()) || 1194 | !mpLexer->AddCustomDirective(directive)) 1195 | { 1196 | return false; 1197 | } 1198 | 1199 | mCustomDirectivesHandlersMap.insert({ directive, handler }); 1200 | 1201 | return true; 1202 | } 1203 | 1204 | 1205 | static bool inline IsParentBlockActive(const Preprocessor::TIfStack& conditionalsContext) TCPP_NOEXCEPT 1206 | { 1207 | return conditionalsContext.empty() ? true : (conditionalsContext.top().mIsParentBlockActive && !conditionalsContext.top().mShouldBeSkipped); 1208 | } 1209 | 1210 | 1211 | Preprocessor::TPreprocessResult Preprocessor::Process() TCPP_NOEXCEPT 1212 | { 1213 | TCPP_ASSERT(mpLexer); 1214 | 1215 | TTokensSequence processedTokens{}; 1216 | 1217 | auto appendToken = [&processedTokens, this](const TToken& token) 1218 | { 1219 | if (_shouldTokenBeSkipped()) 1220 | { 1221 | return; 1222 | } 1223 | 1224 | processedTokens.emplace_back(token); 1225 | }; 1226 | 1227 | // \note first stage of preprocessing, expand macros and include directives 1228 | while (mpLexer->HasNextToken()) 1229 | { 1230 | auto currToken = mpLexer->GetNextToken(); 1231 | 1232 | switch (currToken.mType) 1233 | { 1234 | case E_TOKEN_TYPE::DEFINE: 1235 | _createMacroDefinition(); 1236 | break; 1237 | case E_TOKEN_TYPE::UNDEF: 1238 | currToken = mpLexer->GetNextToken(); 1239 | _expect(E_TOKEN_TYPE::SPACE, currToken.mType); 1240 | 1241 | currToken = mpLexer->GetNextToken(); 1242 | _expect(E_TOKEN_TYPE::IDENTIFIER, currToken.mType); 1243 | 1244 | _removeMacroDefinition(currToken.mRawView); 1245 | break; 1246 | case E_TOKEN_TYPE::IF: 1247 | mConditionalBlocksStack.push(_processIfConditional()); 1248 | break; 1249 | case E_TOKEN_TYPE::IFNDEF: 1250 | mConditionalBlocksStack.push(_processIfndefConditional()); 1251 | break; 1252 | case E_TOKEN_TYPE::IFDEF: 1253 | mConditionalBlocksStack.push(_processIfdefConditional()); 1254 | break; 1255 | case E_TOKEN_TYPE::ELIF: 1256 | _processElifConditional(mConditionalBlocksStack.top()); 1257 | break; 1258 | case E_TOKEN_TYPE::ELSE: 1259 | _processElseConditional(mConditionalBlocksStack.top()); 1260 | break; 1261 | case E_TOKEN_TYPE::ENDIF: 1262 | if (mConditionalBlocksStack.empty()) 1263 | { 1264 | mOnErrorCallback({ E_ERROR_TYPE::UNBALANCED_ENDIF, mpLexer->GetCurrLineIndex() }); 1265 | } 1266 | else 1267 | { 1268 | mConditionalBlocksStack.pop(); 1269 | } 1270 | break; 1271 | case E_TOKEN_TYPE::INCLUDE: 1272 | _processInclusion(); 1273 | break; 1274 | case E_TOKEN_TYPE::IDENTIFIER: // \note try to expand some macro here 1275 | { 1276 | auto iter = std::find_if(mSymTable.cbegin(), mSymTable.cend(), [&currToken](auto&& item) 1277 | { 1278 | return item.mName == currToken.mRawView; 1279 | }); 1280 | 1281 | auto contextIter = std::find_if(mContextStack.cbegin(), mContextStack.cend(), [&currToken](auto&& item) 1282 | { 1283 | return item == currToken.mRawView; 1284 | }); 1285 | 1286 | if (iter != mSymTable.cend() && contextIter == mContextStack.cend()) 1287 | { 1288 | mpLexer->AppendFront(_expandMacroDefinition(*iter, currToken, [this] { return mpLexer->GetNextToken(); })); 1289 | } 1290 | else 1291 | { 1292 | appendToken(currToken); 1293 | } 1294 | } 1295 | break; 1296 | case E_TOKEN_TYPE::REJECT_MACRO: 1297 | mContextStack.erase(std::remove_if(mContextStack.begin(), mContextStack.end(), [&currToken](auto&& item) 1298 | { 1299 | return item == currToken.mRawView; 1300 | }), mContextStack.end()); 1301 | break; 1302 | case E_TOKEN_TYPE::CONCAT_OP: 1303 | while (processedTokens.back().mType == E_TOKEN_TYPE::SPACE) 1304 | { 1305 | processedTokens.erase(processedTokens.end() - 1); 1306 | } 1307 | 1308 | currToken = TrySkipWhitespaceTokensSequence([this] { return mpLexer->GetNextToken(); }, mpLexer->GetNextToken()); 1309 | 1310 | if (!_shouldTokenBeSkipped()) 1311 | { 1312 | processedTokens.back().mRawView += currToken.mRawView; 1313 | } 1314 | 1315 | break; 1316 | case E_TOKEN_TYPE::STRINGIZE_OP: 1317 | { 1318 | if (mContextStack.empty()) 1319 | { 1320 | mOnErrorCallback({ E_ERROR_TYPE::INCORRECT_OPERATION_USAGE, mpLexer->GetCurrLineIndex() }); 1321 | continue; 1322 | } 1323 | 1324 | appendToken(TToken{ E_TOKEN_TYPE::QUOTES, "\"" }); 1325 | appendToken((currToken = mpLexer->GetNextToken())); 1326 | appendToken(TToken{ E_TOKEN_TYPE::QUOTES, "\"" }); 1327 | } 1328 | break; 1329 | case E_TOKEN_TYPE::CUSTOM_DIRECTIVE: 1330 | { 1331 | auto customDirectiveIter = mCustomDirectivesHandlersMap.find(currToken.mRawView); 1332 | if (customDirectiveIter != mCustomDirectivesHandlersMap.cend()) 1333 | { 1334 | appendToken(customDirectiveIter->second(*this, *mpLexer)); 1335 | } 1336 | else 1337 | { 1338 | mOnErrorCallback({ E_ERROR_TYPE::UNDEFINED_DIRECTIVE, mpLexer->GetCurrLineIndex() }); 1339 | } 1340 | } 1341 | break; 1342 | default: 1343 | if (E_TOKEN_TYPE::COMMENTARY == currToken.mType && mSkipCommentsTokens) 1344 | { 1345 | break; 1346 | } 1347 | 1348 | appendToken(currToken); 1349 | break; 1350 | } 1351 | 1352 | if (!mpLexer->HasNextToken()) 1353 | { 1354 | mpLexer->PopStream(); 1355 | } 1356 | } 1357 | 1358 | #ifdef TCPP_OUTPUT_TOKENS_EXTENSION_ENABLED 1359 | return TokensOutputStream{ processedTokens }; 1360 | #else 1361 | return processedTokens; 1362 | #endif 1363 | } 1364 | 1365 | std::string Preprocessor::ToString(const TPreprocessResult& tokens) TCPP_NOEXCEPT 1366 | { 1367 | std::string output = EMPTY_STR_VALUE; 1368 | 1369 | for (const TToken& currToken : tokens) 1370 | { 1371 | output.append(currToken.mRawView); 1372 | } 1373 | 1374 | return output; 1375 | } 1376 | 1377 | Preprocessor::TSymTable Preprocessor::GetSymbolsTable() const TCPP_NOEXCEPT 1378 | { 1379 | return mSymTable; 1380 | } 1381 | 1382 | void Preprocessor::_createMacroDefinition() TCPP_NOEXCEPT 1383 | { 1384 | TMacroDesc macroDesc; 1385 | 1386 | auto currToken = mpLexer->GetNextToken(); 1387 | _expect(E_TOKEN_TYPE::SPACE, currToken.mType); 1388 | 1389 | currToken = mpLexer->GetNextToken(); 1390 | _expect(E_TOKEN_TYPE::IDENTIFIER, currToken.mType); 1391 | 1392 | macroDesc.mName = currToken.mRawView; 1393 | 1394 | auto extractValue = [this](TMacroDesc& desc, Lexer& lexer) 1395 | { 1396 | TToken currToken = TrySkipWhitespaceTokensSequence([this] { return mpLexer->GetNextToken(); }, mpLexer->GetNextToken()); 1397 | 1398 | if (currToken.mType != E_TOKEN_TYPE::NEWLINE) 1399 | { 1400 | desc.mValue.push_back(currToken); 1401 | 1402 | const bool isStringificationOperator = currToken.mType == E_TOKEN_TYPE::STRINGIZE_OP; 1403 | 1404 | while ((currToken = lexer.GetNextToken()).mType != E_TOKEN_TYPE::NEWLINE) 1405 | { 1406 | if (E_TOKEN_TYPE::IDENTIFIER == currToken.mType && currToken.mRawView == desc.mName) 1407 | { 1408 | desc.mValue.emplace_back(TToken{ E_TOKEN_TYPE::BLOB, currToken.mRawView }); // \note Prevent self recursion 1409 | continue; 1410 | } 1411 | 1412 | if (E_TOKEN_TYPE::IDENTIFIER == currToken.mType && isStringificationOperator) // \note Validate argument of the operator 1413 | { 1414 | auto&& macroArgs = desc.mArgsNames; 1415 | 1416 | if (std::find(macroArgs.cbegin(), macroArgs.cend(), currToken.mRawView) == macroArgs.cend()) 1417 | { 1418 | mOnErrorCallback({ E_ERROR_TYPE::INCORRECT_STRINGIFY_OPERATOR_USAGE, mpLexer->GetCurrLineIndex() }); 1419 | } 1420 | } 1421 | 1422 | desc.mValue.push_back(currToken); 1423 | } 1424 | } 1425 | 1426 | if (desc.mValue.empty()) 1427 | { 1428 | desc.mValue.push_back({ E_TOKEN_TYPE::NUMBER, "1", mpLexer->GetCurrLineIndex() }); 1429 | } 1430 | 1431 | _expect(E_TOKEN_TYPE::NEWLINE, currToken.mType); 1432 | }; 1433 | 1434 | currToken = mpLexer->GetNextToken(); 1435 | 1436 | switch (currToken.mType) 1437 | { 1438 | case E_TOKEN_TYPE::SPACE: // object like macro 1439 | extractValue(macroDesc, *mpLexer); 1440 | break; 1441 | case E_TOKEN_TYPE::NEWLINE: 1442 | case E_TOKEN_TYPE::END: 1443 | macroDesc.mValue.push_back({ E_TOKEN_TYPE::NUMBER, "1", mpLexer->GetCurrLineIndex() }); 1444 | break; 1445 | case E_TOKEN_TYPE::OPEN_BRACKET: // function line macro 1446 | { 1447 | // \note parse arguments 1448 | while (true) 1449 | { 1450 | currToken = TrySkipWhitespaceTokensSequence([this] { return mpLexer->GetNextToken(); }, mpLexer->GetNextToken()); 1451 | 1452 | switch (currToken.mType) 1453 | { 1454 | case E_TOKEN_TYPE::IDENTIFIER: 1455 | macroDesc.mArgsNames.push_back(currToken.mRawView); 1456 | break; 1457 | case E_TOKEN_TYPE::ELLIPSIS: 1458 | macroDesc.mArgsNames.push_back("__VA_ARGS__"); 1459 | macroDesc.mVariadic = true; 1460 | break; 1461 | default: 1462 | mOnErrorCallback({ E_ERROR_TYPE::UNEXPECTED_TOKEN, mpLexer->GetCurrLineIndex() }); 1463 | } 1464 | 1465 | currToken = TrySkipWhitespaceTokensSequence([this] { return mpLexer->GetNextToken(); }, mpLexer->GetNextToken()); 1466 | if (macroDesc.mVariadic) 1467 | { 1468 | _expect(E_TOKEN_TYPE::CLOSE_BRACKET, currToken.mType); 1469 | break; 1470 | } 1471 | if (currToken.mType == E_TOKEN_TYPE::CLOSE_BRACKET) 1472 | { 1473 | break; 1474 | } 1475 | 1476 | _expect(E_TOKEN_TYPE::COMMA, currToken.mType); 1477 | } 1478 | 1479 | // \note parse macro's value 1480 | extractValue(macroDesc, *mpLexer); 1481 | } 1482 | break; 1483 | default: 1484 | mOnErrorCallback({ E_ERROR_TYPE::INVALID_MACRO_DEFINITION, mpLexer->GetCurrLineIndex() }); 1485 | break; 1486 | } 1487 | 1488 | if (_shouldTokenBeSkipped()) 1489 | { 1490 | return; 1491 | } 1492 | 1493 | if (std::find_if(mSymTable.cbegin(), mSymTable.cend(), [¯oDesc](auto&& item) { return item.mName == macroDesc.mName; }) != mSymTable.cend()) 1494 | { 1495 | mOnErrorCallback({ E_ERROR_TYPE::MACRO_ALREADY_DEFINED, mpLexer->GetCurrLineIndex() }); 1496 | return; 1497 | } 1498 | 1499 | mSymTable.push_back(macroDesc); 1500 | } 1501 | 1502 | void Preprocessor::_removeMacroDefinition(const std::string& macroName) TCPP_NOEXCEPT 1503 | { 1504 | if (_shouldTokenBeSkipped()) 1505 | { 1506 | return; 1507 | } 1508 | 1509 | auto iter = std::find_if(mSymTable.cbegin(), mSymTable.cend(), [¯oName](auto&& item) { return item.mName == macroName; }); 1510 | if (iter == mSymTable.cend()) 1511 | { 1512 | mOnErrorCallback({ E_ERROR_TYPE::UNDEFINED_MACRO, mpLexer->GetCurrLineIndex() }); 1513 | return; 1514 | } 1515 | 1516 | mSymTable.erase(iter); 1517 | 1518 | auto currToken = mpLexer->GetNextToken(); 1519 | _expect(E_TOKEN_TYPE::NEWLINE, currToken.mType); 1520 | } 1521 | 1522 | TTokensSequence Preprocessor::_expandMacroDefinition(const TMacroDesc& macroDesc, const TToken& idToken, const std::function& getNextTokenCallback) const TCPP_NOEXCEPT 1523 | { 1524 | // \note expand object like macro with simple replacement 1525 | if (macroDesc.mArgsNames.empty()) 1526 | { 1527 | static const std::unordered_map> systemMacrosTable 1528 | { 1529 | { BuiltInDefines[0], [&idToken]() { return TToken {E_TOKEN_TYPE::BLOB, std::to_string(idToken.mLineId)}; }}, // __LINE__ 1530 | { BuiltInDefines[1], [&idToken]() { return TToken { E_TOKEN_TYPE::BLOB, idToken.mRawView }; } }, // __VA_ARGS__ 1531 | }; 1532 | 1533 | if (E_TOKEN_TYPE::CONCAT_OP == mpLexer->PeekNextToken().mType) // If an argument is stringized or concatenated, the prescan does not occur. 1534 | { 1535 | return { TToken { E_TOKEN_TYPE::BLOB, macroDesc.mName } }; // BLOB type is used instead of IDENTIFIER to prevent infinite loop 1536 | } 1537 | 1538 | auto iter = systemMacrosTable.find(macroDesc.mName); 1539 | if (iter != systemMacrosTable.cend()) 1540 | { 1541 | return { iter->second() }; 1542 | } 1543 | 1544 | return macroDesc.mValue; 1545 | } 1546 | 1547 | mContextStack.push_back(macroDesc.mName); 1548 | 1549 | // \note function like macro's case 1550 | auto currToken = TrySkipWhitespaceTokensSequence(getNextTokenCallback, getNextTokenCallback()); 1551 | 1552 | if (E_TOKEN_TYPE::OPEN_BRACKET != currToken.mType) 1553 | { 1554 | return { TToken { E_TOKEN_TYPE::BLOB, macroDesc.mName }, currToken }; // \note Function like macro without brackets is not expanded 1555 | } 1556 | 1557 | _expect(E_TOKEN_TYPE::OPEN_BRACKET, currToken.mType); 1558 | 1559 | std::vector processingTokens; 1560 | TTokensSequence currArgTokens; 1561 | 1562 | uint8_t currNestingLevel = 0; 1563 | 1564 | // \note read all arguments values 1565 | while (true) 1566 | { 1567 | currArgTokens.clear(); 1568 | 1569 | bool hasAnySpace = false; 1570 | while ((currToken = getNextTokenCallback()).mType == E_TOKEN_TYPE::SPACE) // \note skip space tokens 1571 | { 1572 | hasAnySpace = true; 1573 | } 1574 | 1575 | if (E_TOKEN_TYPE::CLOSE_BRACKET == currToken.mType || E_TOKEN_TYPE::COMMA == currToken.mType) 1576 | { 1577 | if (hasAnySpace) 1578 | { 1579 | currArgTokens.push_back({ E_TOKEN_TYPE::SPACE, " " }); 1580 | } 1581 | else 1582 | { 1583 | break; // \note There should be at least one space for empty argument 1584 | } 1585 | } 1586 | else 1587 | { 1588 | currArgTokens.push_back(currToken); 1589 | } 1590 | 1591 | if (E_TOKEN_TYPE::CLOSE_BRACKET != currToken.mType) 1592 | { 1593 | if (E_TOKEN_TYPE::OPEN_BRACKET == currToken.mType) 1594 | { 1595 | ++currNestingLevel; 1596 | } 1597 | 1598 | currToken = TrySkipWhitespaceTokensSequence(getNextTokenCallback, getNextTokenCallback()); 1599 | 1600 | while ((currToken.mType != E_TOKEN_TYPE::COMMA && 1601 | currToken.mType != E_TOKEN_TYPE::NEWLINE && 1602 | currToken.mType != E_TOKEN_TYPE::CLOSE_BRACKET) || currNestingLevel) 1603 | { 1604 | switch (currToken.mType) 1605 | { 1606 | case E_TOKEN_TYPE::OPEN_BRACKET: 1607 | ++currNestingLevel; 1608 | break; 1609 | case E_TOKEN_TYPE::CLOSE_BRACKET: 1610 | --currNestingLevel; 1611 | break; 1612 | default: 1613 | break; 1614 | } 1615 | 1616 | currArgTokens.push_back(currToken); 1617 | currToken = getNextTokenCallback(); 1618 | } 1619 | 1620 | if (currToken.mType != E_TOKEN_TYPE::COMMA && currToken.mType != E_TOKEN_TYPE::CLOSE_BRACKET) 1621 | { 1622 | _expect(E_TOKEN_TYPE::COMMA, currToken.mType); 1623 | } 1624 | } 1625 | 1626 | processingTokens.push_back(currArgTokens); 1627 | 1628 | if (currToken.mType == E_TOKEN_TYPE::CLOSE_BRACKET) 1629 | { 1630 | break; 1631 | } 1632 | } 1633 | 1634 | if(macroDesc.mVariadic ? 1635 | (processingTokens.size() + 1 < macroDesc.mArgsNames.size()) : 1636 | (processingTokens.size() != macroDesc.mArgsNames.size())) 1637 | { 1638 | mOnErrorCallback({ E_ERROR_TYPE::INCONSISTENT_MACRO_ARITY, mpLexer->GetCurrLineIndex() }); 1639 | } 1640 | 1641 | // \note execute macro's expansion 1642 | TTokensSequence replacementList{ macroDesc.mValue.cbegin(), macroDesc.mValue.cend() }; 1643 | const auto& argsList = macroDesc.mArgsNames; 1644 | 1645 | std::string replacementValue; 1646 | 1647 | std::unordered_map cachedArgsSequences 1648 | { 1649 | { EMPTY_STR_VALUE, {} } 1650 | }; 1651 | 1652 | for (size_t currArgIndex = 0; currArgIndex < processingTokens.size(); ++currArgIndex) 1653 | { 1654 | const bool variadics = macroDesc.mVariadic && currArgIndex == macroDesc.mArgsNames.size() - 1; 1655 | const std::string& currArgName = variadics ? argsList.back() : argsList[currArgIndex]; 1656 | 1657 | replacementValue.clear(); 1658 | 1659 | for (auto&& currArgToken : processingTokens[currArgIndex]) 1660 | { 1661 | replacementValue.append(currArgToken.mRawView); 1662 | } 1663 | 1664 | if (variadics) 1665 | { 1666 | for (size_t i = currArgIndex + 1; i < processingTokens.size(); ++i) 1667 | { 1668 | replacementValue.append(","); 1669 | for (auto&& currArgToken : processingTokens[i]) 1670 | { 1671 | replacementValue.append(currArgToken.mRawView); 1672 | } 1673 | } 1674 | } 1675 | 1676 | for (size_t currTokenIndex = 0; currTokenIndex < replacementList.size(); ++currTokenIndex) 1677 | { 1678 | TToken& currToken = replacementList[currTokenIndex]; 1679 | 1680 | if ((currToken.mType != E_TOKEN_TYPE::IDENTIFIER) || (currToken.mRawView != currArgName)) 1681 | { 1682 | continue; 1683 | } 1684 | 1685 | // \note Check whether argument is used as an operand of token-pasting or stringification operators 1686 | auto nextNonSpaceToken = std::find_if(replacementList.cbegin() + currTokenIndex + 1, replacementList.cend(), [](const TToken& token) { return token.mType != E_TOKEN_TYPE::SPACE; }); 1687 | auto prevNonSpaceToken = std::find_if(replacementList.rbegin() + (replacementList.size() - currTokenIndex), replacementList.rend(), [](const TToken& token) { return token.mType != E_TOKEN_TYPE::SPACE; }); 1688 | 1689 | const bool isTokenPastingOperand = 1690 | (nextNonSpaceToken != replacementList.cend() && nextNonSpaceToken->mType == E_TOKEN_TYPE::CONCAT_OP) || 1691 | (prevNonSpaceToken != replacementList.rend() && prevNonSpaceToken->mType == E_TOKEN_TYPE::CONCAT_OP); 1692 | const bool isStringifyOperand = currTokenIndex > 0 && replacementList[currTokenIndex - 1].mType == E_TOKEN_TYPE::STRINGIZE_OP; 1693 | 1694 | if (isTokenPastingOperand || isStringifyOperand) 1695 | { 1696 | currToken.mRawView = replacementValue; 1697 | continue; 1698 | } 1699 | 1700 | TTokensSequence& currExpandedArgTokens = cachedArgsSequences[EMPTY_STR_VALUE]; 1701 | 1702 | auto prevComputationIt = cachedArgsSequences.find(replacementValue); 1703 | if (prevComputationIt == cachedArgsSequences.cend()) 1704 | { 1705 | auto it = processingTokens[currArgIndex].begin(); 1706 | currExpandedArgTokens = _expandArg([&processingTokens, &it, currArgIndex] { if (it == processingTokens[currArgIndex].end()) return TToken{ E_TOKEN_TYPE::END }; return *it++; }); 1707 | } 1708 | else 1709 | { 1710 | currExpandedArgTokens = prevComputationIt->second; 1711 | } 1712 | 1713 | currToken = *currExpandedArgTokens.cbegin(); 1714 | 1715 | if (currExpandedArgTokens.size() > 1) 1716 | { 1717 | replacementList.insert(replacementList.cbegin() + currTokenIndex + 1, currExpandedArgTokens.cbegin() + 1, currExpandedArgTokens.cend()); 1718 | currTokenIndex += currExpandedArgTokens.size() - 1; 1719 | } 1720 | } 1721 | 1722 | if (variadics) 1723 | { 1724 | break; 1725 | } 1726 | } 1727 | 1728 | replacementList.push_back({ E_TOKEN_TYPE::REJECT_MACRO, macroDesc.mName }); 1729 | return replacementList; 1730 | } 1731 | 1732 | TTokensSequence Preprocessor::_expandArg(const std::function& getNextTokenCallback) const TCPP_NOEXCEPT 1733 | { 1734 | TTokensSequence outputSequence{}; 1735 | 1736 | TToken currToken; 1737 | 1738 | while ((currToken = getNextTokenCallback()).mType != E_TOKEN_TYPE::END) 1739 | { 1740 | switch (currToken.mType) 1741 | { 1742 | case E_TOKEN_TYPE::IDENTIFIER: // \note try to expand some macro here 1743 | { 1744 | auto iter = std::find_if(mSymTable.cbegin(), mSymTable.cend(), [&currToken](auto&& item) 1745 | { 1746 | return item.mName == currToken.mRawView; 1747 | }); 1748 | 1749 | auto contextIter = std::find_if(mContextStack.cbegin(), mContextStack.cend(), [&currToken](auto&& item) 1750 | { 1751 | return item == currToken.mRawView; 1752 | }); 1753 | 1754 | if (iter != mSymTable.cend() && contextIter == mContextStack.cend()) 1755 | { 1756 | auto expandedMacroTokens = _expandMacroDefinition(*iter, currToken, getNextTokenCallback); 1757 | outputSequence.insert(outputSequence.cbegin(), expandedMacroTokens.cbegin(), expandedMacroTokens.cend()); 1758 | } 1759 | else 1760 | { 1761 | outputSequence.emplace_back(currToken); 1762 | } 1763 | } 1764 | break; 1765 | case E_TOKEN_TYPE::REJECT_MACRO: 1766 | mContextStack.erase(std::remove_if(mContextStack.begin(), mContextStack.end(), [&currToken](auto&& item) 1767 | { 1768 | return item == currToken.mRawView; 1769 | }), mContextStack.end()); 1770 | break; 1771 | case E_TOKEN_TYPE::CONCAT_OP: 1772 | #if 0 1773 | while (processedStr.back() == ' ') // \note Remove last character in the processed source if it was a whitespace 1774 | { 1775 | processedStr.erase(processedStr.length() - 1); 1776 | } 1777 | 1778 | currToken = TrySkipWhitespaceTokensSequence(getNextTokenCallback(), getNextTokenCallback()); 1779 | appendToken(currToken.mRawView); 1780 | #endif 1781 | break; 1782 | case E_TOKEN_TYPE::STRINGIZE_OP: 1783 | { 1784 | if (mContextStack.empty()) 1785 | { 1786 | mOnErrorCallback({ E_ERROR_TYPE::INCORRECT_OPERATION_USAGE, mpLexer->GetCurrLineIndex() }); 1787 | continue; 1788 | } 1789 | 1790 | TToken stringLiteralToken = currToken; 1791 | stringLiteralToken.mType = E_TOKEN_TYPE::BLOB; 1792 | stringLiteralToken.mRawView = "\"" + (currToken = mpLexer->GetNextToken()).mRawView + "\""; 1793 | 1794 | outputSequence.emplace_back(stringLiteralToken); 1795 | } 1796 | break; 1797 | default: 1798 | if (E_TOKEN_TYPE::COMMENTARY == currToken.mType && mSkipCommentsTokens) 1799 | { 1800 | break; 1801 | } 1802 | 1803 | outputSequence.emplace_back(currToken); 1804 | break; 1805 | } 1806 | } 1807 | 1808 | TCPP_ASSERT(!outputSequence.empty()); 1809 | 1810 | return outputSequence; 1811 | } 1812 | 1813 | void Preprocessor::_expect(const E_TOKEN_TYPE& expectedType, const E_TOKEN_TYPE& actualType) const TCPP_NOEXCEPT 1814 | { 1815 | if (expectedType == actualType) 1816 | { 1817 | return; 1818 | } 1819 | 1820 | mOnErrorCallback({ E_ERROR_TYPE::UNEXPECTED_TOKEN, mpLexer->GetCurrLineIndex() }); 1821 | } 1822 | 1823 | void Preprocessor::_processInclusion() TCPP_NOEXCEPT 1824 | { 1825 | if (_shouldTokenBeSkipped()) 1826 | { 1827 | return; 1828 | } 1829 | 1830 | TToken currToken = TrySkipWhitespaceTokensSequence([this] { return mpLexer->GetNextToken(); }, mpLexer->GetNextToken()); 1831 | 1832 | if (currToken.mType != E_TOKEN_TYPE::LESS && currToken.mType != E_TOKEN_TYPE::QUOTES) 1833 | { 1834 | while ((currToken = mpLexer->GetNextToken()).mType == E_TOKEN_TYPE::NEWLINE); // \note skip to end of current line 1835 | 1836 | mOnErrorCallback({ E_ERROR_TYPE::INVALID_INCLUDE_DIRECTIVE, mpLexer->GetCurrLineIndex() }); 1837 | return; 1838 | } 1839 | 1840 | bool isSystemPathInclusion = currToken.mType == E_TOKEN_TYPE::LESS; 1841 | 1842 | std::string path; 1843 | 1844 | while (true) 1845 | { 1846 | if ((currToken = mpLexer->GetNextToken()).mType == E_TOKEN_TYPE::QUOTES || 1847 | currToken.mType == E_TOKEN_TYPE::GREATER) 1848 | { 1849 | break; 1850 | } 1851 | 1852 | if (currToken.mType == E_TOKEN_TYPE::NEWLINE) 1853 | { 1854 | mOnErrorCallback({ E_ERROR_TYPE::UNEXPECTED_END_OF_INCLUDE_PATH, mpLexer->GetCurrLineIndex() }); 1855 | break; 1856 | } 1857 | 1858 | path.append(currToken.mRawView); 1859 | } 1860 | 1861 | currToken = TrySkipWhitespaceTokensSequence([this] { return mpLexer->GetNextToken(); }, mpLexer->GetNextToken()); 1862 | 1863 | if (E_TOKEN_TYPE::NEWLINE != currToken.mType && E_TOKEN_TYPE::END != currToken.mType) 1864 | { 1865 | mOnErrorCallback({ E_ERROR_TYPE::UNEXPECTED_TOKEN, mpLexer->GetCurrLineIndex() }); 1866 | } 1867 | 1868 | if (mOnIncludeCallback) 1869 | { 1870 | mpLexer->PushStream(std::move(mOnIncludeCallback(path, isSystemPathInclusion))); 1871 | } 1872 | } 1873 | 1874 | Preprocessor::TIfStackEntry Preprocessor::_processIfConditional() TCPP_NOEXCEPT 1875 | { 1876 | auto currToken = mpLexer->GetNextToken(); 1877 | _expect(E_TOKEN_TYPE::SPACE, currToken.mType); 1878 | 1879 | TTokensSequence expressionTokens; 1880 | 1881 | while ((currToken = mpLexer->GetNextToken()).mType != E_TOKEN_TYPE::NEWLINE) 1882 | { 1883 | if (E_TOKEN_TYPE::SPACE == currToken.mType) 1884 | { 1885 | continue; 1886 | } 1887 | 1888 | expressionTokens.push_back(currToken); 1889 | } 1890 | 1891 | _expect(E_TOKEN_TYPE::NEWLINE, currToken.mType); 1892 | 1893 | // \note IsDisabledBlockProcessed is used to inherit disabled state for nested blocks 1894 | return TIfStackEntry(static_cast(!_evaluateExpression(expressionTokens)), IsParentBlockActive(mConditionalBlocksStack)); 1895 | } 1896 | 1897 | 1898 | Preprocessor::TIfStackEntry Preprocessor::_processIfdefConditional() TCPP_NOEXCEPT 1899 | { 1900 | auto currToken = mpLexer->GetNextToken(); 1901 | _expect(E_TOKEN_TYPE::SPACE, currToken.mType); 1902 | 1903 | currToken = mpLexer->GetNextToken(); 1904 | _expect(E_TOKEN_TYPE::IDENTIFIER, currToken.mType); 1905 | 1906 | std::string macroIdentifier = currToken.mRawView; 1907 | 1908 | do { 1909 | currToken = mpLexer->GetNextToken(); 1910 | } while (currToken.mType == E_TOKEN_TYPE::SPACE); 1911 | 1912 | _expect(E_TOKEN_TYPE::NEWLINE, currToken.mType); 1913 | 1914 | bool skip = std::find_if(mSymTable.cbegin(), mSymTable.cend(), [¯oIdentifier](auto&& item) 1915 | { 1916 | return item.mName == macroIdentifier; 1917 | }) == mSymTable.cend(); 1918 | 1919 | // \note IsParentBlockActive is used to inherit disabled state for nested blocks 1920 | return TIfStackEntry(skip, IsParentBlockActive(mConditionalBlocksStack)); 1921 | } 1922 | 1923 | Preprocessor::TIfStackEntry Preprocessor::_processIfndefConditional() TCPP_NOEXCEPT 1924 | { 1925 | auto currToken = mpLexer->GetNextToken(); 1926 | _expect(E_TOKEN_TYPE::SPACE, currToken.mType); 1927 | 1928 | currToken = mpLexer->GetNextToken(); 1929 | _expect(E_TOKEN_TYPE::IDENTIFIER, currToken.mType); 1930 | 1931 | std::string macroIdentifier = currToken.mRawView; 1932 | 1933 | do { 1934 | currToken = mpLexer->GetNextToken(); 1935 | } while (currToken.mType == E_TOKEN_TYPE::SPACE); 1936 | 1937 | _expect(E_TOKEN_TYPE::NEWLINE, currToken.mType); 1938 | 1939 | bool skip = std::find_if(mSymTable.cbegin(), mSymTable.cend(), [¯oIdentifier](auto&& item) 1940 | { 1941 | return item.mName == macroIdentifier; 1942 | }) != mSymTable.cend(); 1943 | 1944 | // \note IsParentBlockActive is used to inherit disabled state for nested blocks 1945 | return TIfStackEntry(skip, IsParentBlockActive(mConditionalBlocksStack)); 1946 | } 1947 | 1948 | void Preprocessor::_processElseConditional(TIfStackEntry& currStackEntry) TCPP_NOEXCEPT 1949 | { 1950 | if (currStackEntry.mHasElseBeenFound) 1951 | { 1952 | mOnErrorCallback({ E_ERROR_TYPE::ANOTHER_ELSE_BLOCK_FOUND, mpLexer->GetCurrLineIndex() }); 1953 | return; 1954 | } 1955 | 1956 | currStackEntry.mShouldBeSkipped = 1957 | currStackEntry.mHasIfBlockBeenEntered || 1958 | !currStackEntry.mShouldBeSkipped; 1959 | 1960 | currStackEntry.mHasElseBeenFound = true; 1961 | } 1962 | 1963 | void Preprocessor::_processElifConditional(TIfStackEntry& currStackEntry) TCPP_NOEXCEPT 1964 | { 1965 | if (currStackEntry.mHasElseBeenFound) 1966 | { 1967 | mOnErrorCallback({ E_ERROR_TYPE::ELIF_BLOCK_AFTER_ELSE_FOUND, mpLexer->GetCurrLineIndex() }); 1968 | return; 1969 | } 1970 | 1971 | auto currToken = mpLexer->GetNextToken(); 1972 | _expect(E_TOKEN_TYPE::SPACE, currToken.mType); 1973 | 1974 | TTokensSequence expressionTokens; 1975 | 1976 | while ((currToken = mpLexer->GetNextToken()).mType != E_TOKEN_TYPE::NEWLINE) 1977 | { 1978 | expressionTokens.push_back(currToken); 1979 | } 1980 | 1981 | _expect(E_TOKEN_TYPE::NEWLINE, currToken.mType); 1982 | 1983 | currStackEntry.mShouldBeSkipped = 1984 | currStackEntry.mHasIfBlockBeenEntered || 1985 | static_cast(!_evaluateExpression(expressionTokens)); 1986 | 1987 | if (!currStackEntry.mShouldBeSkipped) currStackEntry.mHasIfBlockBeenEntered = true; 1988 | } 1989 | 1990 | int Preprocessor::_evaluateExpression(const TTokensSequence& exprTokens) const TCPP_NOEXCEPT 1991 | { 1992 | TTokensSequence tokens{ exprTokens.begin(), exprTokens.end() }; 1993 | tokens.push_back({ E_TOKEN_TYPE::END }); 1994 | 1995 | auto evalPrimary = [this, &tokens]() 1996 | { 1997 | while (E_TOKEN_TYPE::SPACE == tokens.front().mType) /// \note Skip whitespaces 1998 | { 1999 | tokens.erase(tokens.cbegin()); 2000 | } 2001 | 2002 | auto currToken = tokens.front(); 2003 | 2004 | switch (currToken.mType) 2005 | { 2006 | case E_TOKEN_TYPE::IDENTIFIER: 2007 | { 2008 | // \note macro call 2009 | TToken identifierToken; 2010 | if (currToken.mRawView == "defined") 2011 | { 2012 | // defined ( X ) 2013 | do { 2014 | tokens.erase(tokens.cbegin()); 2015 | } while (tokens.front().mType == E_TOKEN_TYPE::SPACE); 2016 | 2017 | _expect(E_TOKEN_TYPE::OPEN_BRACKET, tokens.front().mType); 2018 | 2019 | do { 2020 | tokens.erase(tokens.cbegin()); 2021 | } while (tokens.front().mType == E_TOKEN_TYPE::SPACE); 2022 | 2023 | _expect(E_TOKEN_TYPE::IDENTIFIER, tokens.front().mType); 2024 | 2025 | identifierToken = tokens.front(); 2026 | 2027 | do { 2028 | tokens.erase(tokens.cbegin()); 2029 | } while (tokens.front().mType == E_TOKEN_TYPE::SPACE); 2030 | 2031 | _expect(E_TOKEN_TYPE::CLOSE_BRACKET, tokens.front().mType); 2032 | 2033 | // \note simple identifier 2034 | return static_cast(std::find_if(mSymTable.cbegin(), mSymTable.cend(), [&identifierToken](auto&& item) 2035 | { 2036 | return item.mName == identifierToken.mRawView; 2037 | }) != mSymTable.cend()); 2038 | } 2039 | else 2040 | { 2041 | tokens.erase(tokens.cbegin()); 2042 | identifierToken = currToken; 2043 | } 2044 | 2045 | /// \note Try to expand macro's value 2046 | auto it = std::find_if(mSymTable.cbegin(), mSymTable.cend(), [&identifierToken](auto&& item) 2047 | { 2048 | return item.mName == identifierToken.mRawView; 2049 | }); 2050 | 2051 | if (it == mSymTable.cend()) 2052 | { 2053 | /// \note Lexer for now doesn't support numbers recognition so numbers are recognized as identifiers too 2054 | return atoi(identifierToken.mRawView.c_str()); 2055 | } 2056 | else 2057 | { 2058 | if (it->mArgsNames.empty()) 2059 | { 2060 | return _evaluateExpression(it->mValue); /// simple macro replacement 2061 | } 2062 | 2063 | /// \note Macro function call so we should firstly expand that one 2064 | auto currTokenIt = tokens.cbegin(); 2065 | return _evaluateExpression(_expandMacroDefinition(*it, identifierToken, [&currTokenIt] { return *currTokenIt++; })); 2066 | } 2067 | 2068 | return 0; /// \note Something went wrong so return 0 2069 | } 2070 | 2071 | case E_TOKEN_TYPE::NUMBER: 2072 | tokens.erase(tokens.cbegin()); 2073 | return std::stoi(currToken.mRawView); 2074 | 2075 | case E_TOKEN_TYPE::OPEN_BRACKET: 2076 | tokens.erase(tokens.cbegin()); 2077 | return _evaluateExpression(tokens); 2078 | 2079 | default: 2080 | break; 2081 | } 2082 | 2083 | return 0; 2084 | }; 2085 | 2086 | auto evalUnary = [&tokens, &evalPrimary]() 2087 | { 2088 | while (E_TOKEN_TYPE::SPACE == tokens.front().mType) /// \note Skip whitespaces 2089 | { 2090 | tokens.erase(tokens.cbegin()); 2091 | } 2092 | 2093 | bool resultApply = false; 2094 | TToken currToken; 2095 | while ((currToken = tokens.front()).mType == E_TOKEN_TYPE::NOT || currToken.mType == E_TOKEN_TYPE::MINUS) 2096 | { 2097 | switch (currToken.mType) 2098 | { 2099 | case E_TOKEN_TYPE::MINUS: 2100 | // TODO fix this 2101 | break; 2102 | case E_TOKEN_TYPE::NOT: 2103 | tokens.erase(tokens.cbegin()); 2104 | resultApply = !resultApply; 2105 | break; 2106 | default: 2107 | break; 2108 | } 2109 | } 2110 | 2111 | // even number of NOTs: false ^ false = false, false ^ true = true 2112 | // odd number of NOTs: true ^ false = true (!false), true ^ true = false (!true) 2113 | return static_cast(resultApply) ^ evalPrimary(); 2114 | }; 2115 | 2116 | auto evalMultiplication = [&tokens, &evalUnary]() 2117 | { 2118 | int result = evalUnary(); 2119 | int secondOperand = 0; 2120 | 2121 | TToken currToken; 2122 | while ((currToken = tokens.front()).mType == E_TOKEN_TYPE::STAR || currToken.mType == E_TOKEN_TYPE::SLASH) 2123 | { 2124 | switch (currToken.mType) 2125 | { 2126 | case E_TOKEN_TYPE::STAR: 2127 | tokens.erase(tokens.cbegin()); 2128 | result = result * evalUnary(); 2129 | break; 2130 | case E_TOKEN_TYPE::SLASH: 2131 | tokens.erase(tokens.cbegin()); 2132 | 2133 | secondOperand = evalUnary(); 2134 | result = secondOperand ? (result / secondOperand) : 0 /* division by zero is considered as false in the implementation */; 2135 | break; 2136 | default: 2137 | break; 2138 | } 2139 | } 2140 | 2141 | return result; 2142 | }; 2143 | 2144 | auto evalAddition = [&tokens, &evalMultiplication]() 2145 | { 2146 | int result = evalMultiplication(); 2147 | 2148 | TToken currToken; 2149 | while ((currToken = tokens.front()).mType == E_TOKEN_TYPE::PLUS || currToken.mType == E_TOKEN_TYPE::MINUS) 2150 | { 2151 | switch (currToken.mType) 2152 | { 2153 | case E_TOKEN_TYPE::PLUS: 2154 | tokens.erase(tokens.cbegin()); 2155 | result = result + evalMultiplication(); 2156 | break; 2157 | case E_TOKEN_TYPE::MINUS: 2158 | tokens.erase(tokens.cbegin()); 2159 | result = result - evalMultiplication(); 2160 | break; 2161 | default: 2162 | break; 2163 | } 2164 | } 2165 | 2166 | return result; 2167 | }; 2168 | 2169 | auto evalComparison = [&tokens, &evalAddition]() 2170 | { 2171 | int result = evalAddition(); 2172 | 2173 | TToken currToken; 2174 | while ((currToken = tokens.front()).mType == E_TOKEN_TYPE::LESS || 2175 | currToken.mType == E_TOKEN_TYPE::GREATER || 2176 | currToken.mType == E_TOKEN_TYPE::LE || 2177 | currToken.mType == E_TOKEN_TYPE::GE) 2178 | { 2179 | switch (currToken.mType) 2180 | { 2181 | case E_TOKEN_TYPE::LESS: 2182 | tokens.erase(tokens.cbegin()); 2183 | result = result < evalAddition(); 2184 | break; 2185 | case E_TOKEN_TYPE::GREATER: 2186 | tokens.erase(tokens.cbegin()); 2187 | result = result > evalAddition(); 2188 | break; 2189 | case E_TOKEN_TYPE::LE: 2190 | tokens.erase(tokens.cbegin()); 2191 | result = result <= evalAddition(); 2192 | break; 2193 | case E_TOKEN_TYPE::GE: 2194 | tokens.erase(tokens.cbegin()); 2195 | result = result >= evalAddition(); 2196 | break; 2197 | default: 2198 | break; 2199 | } 2200 | } 2201 | 2202 | return result; 2203 | }; 2204 | 2205 | auto evalEquality = [&tokens, &evalComparison]() 2206 | { 2207 | int result = evalComparison(); 2208 | 2209 | TToken currToken; 2210 | while ((currToken = tokens.front()).mType == E_TOKEN_TYPE::EQ || currToken.mType == E_TOKEN_TYPE::NE) 2211 | { 2212 | switch (currToken.mType) 2213 | { 2214 | case E_TOKEN_TYPE::EQ: 2215 | tokens.erase(tokens.cbegin()); 2216 | result = result == evalComparison(); 2217 | break; 2218 | case E_TOKEN_TYPE::NE: 2219 | tokens.erase(tokens.cbegin()); 2220 | result = result != evalComparison(); 2221 | break; 2222 | default: 2223 | break; 2224 | } 2225 | } 2226 | 2227 | return result; 2228 | }; 2229 | 2230 | auto evalAndExpr = [&tokens, &evalEquality]() 2231 | { 2232 | int result = evalEquality(); 2233 | 2234 | while (E_TOKEN_TYPE::SPACE == tokens.front().mType) 2235 | { 2236 | tokens.erase(tokens.cbegin()); 2237 | } 2238 | 2239 | while (tokens.front().mType == E_TOKEN_TYPE::AND) 2240 | { 2241 | tokens.erase(tokens.cbegin()); 2242 | result = result && evalEquality(); 2243 | } 2244 | 2245 | return result; 2246 | }; 2247 | 2248 | auto evalOrExpr = [&tokens, &evalAndExpr]() 2249 | { 2250 | int result = evalAndExpr(); 2251 | 2252 | while (tokens.front().mType == E_TOKEN_TYPE::OR) 2253 | { 2254 | tokens.erase(tokens.cbegin()); 2255 | result = result || evalAndExpr(); 2256 | } 2257 | 2258 | return result; 2259 | }; 2260 | 2261 | return evalOrExpr(); 2262 | } 2263 | 2264 | bool Preprocessor::_shouldTokenBeSkipped() const TCPP_NOEXCEPT 2265 | { 2266 | return !mConditionalBlocksStack.empty() && (mConditionalBlocksStack.top().mShouldBeSkipped || !mConditionalBlocksStack.top().mIsParentBlockActive); 2267 | } 2268 | 2269 | 2270 | #ifdef TCPP_OUTPUT_TOKENS_EXTENSION_ENABLED 2271 | 2272 | TokensOutputStream::TokensOutputStream(TTokensSequence tokens) TCPP_NOEXCEPT 2273 | : mTokens(std::move(tokens)), mCurrIt(mTokens.begin()) 2274 | { 2275 | } 2276 | 2277 | TTokensSequenceIter TokensOutputStream::begin() TCPP_NOEXCEPT 2278 | { 2279 | return mTokens.begin(); 2280 | } 2281 | 2282 | TTokensSequenceIter TokensOutputStream::end() TCPP_NOEXCEPT 2283 | { 2284 | return mTokens.end(); 2285 | } 2286 | 2287 | TTokensSequenceConstIter TokensOutputStream::begin() const TCPP_NOEXCEPT 2288 | { 2289 | return mTokens.cbegin(); 2290 | } 2291 | 2292 | TTokensSequenceConstIter TokensOutputStream::end() const TCPP_NOEXCEPT 2293 | { 2294 | return mTokens.cend(); 2295 | } 2296 | 2297 | const TToken& TokensOutputStream::GetNextToken() TCPP_NOEXCEPT 2298 | { 2299 | if (mCurrIt == mTokens.end()) 2300 | { 2301 | return mTokens.back(); 2302 | } 2303 | 2304 | return *mCurrIt++; 2305 | } 2306 | 2307 | const TToken& TokensOutputStream::PeekNextToken(size_t offset) TCPP_NOEXCEPT 2308 | { 2309 | TTokensSequenceIter peekIt = mCurrIt + offset; 2310 | if (peekIt == mTokens.end()) 2311 | { 2312 | return mTokens.back(); 2313 | } 2314 | 2315 | return *peekIt; 2316 | } 2317 | 2318 | bool TokensOutputStream::HasNextToken() const TCPP_NOEXCEPT 2319 | { 2320 | return mCurrIt != mTokens.end(); 2321 | } 2322 | 2323 | const TTokensSequence& TokensOutputStream::GetSequence() const TCPP_NOEXCEPT 2324 | { 2325 | return mTokens; 2326 | } 2327 | 2328 | #endif 2329 | 2330 | #endif 2331 | } -------------------------------------------------------------------------------- /tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.8) 2 | 3 | project(tcpp-tests LANGUAGES CXX) 4 | 5 | option(IS_TESTING_ENABLED "The option turns on/off tests" ON) 6 | 7 | if (NOT DEFINED ${TCPP_TESTS_NAME}) 8 | set(TCPP_TESTS_NAME ${PROJECT_NAME}) 9 | endif () 10 | 11 | set_property(GLOBAL PROPERTY USE_FOLDERS ON) 12 | 13 | if (IS_TESTING_ENABLED) 14 | enable_testing() 15 | endif () 16 | 17 | # attach tcpp 18 | include_directories("${CMAKE_CURRENT_SOURCE_DIR}/../source") 19 | 20 | # include Catch2 21 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/ThirdParty/Catch2/contrib/") 22 | add_subdirectory(ThirdParty/Catch2) 23 | 24 | set(HEADERS ) 25 | 26 | set(SOURCES 27 | "${CMAKE_CURRENT_SOURCE_DIR}/coreTests.cpp" 28 | "${CMAKE_CURRENT_SOURCE_DIR}/lexerTests.cpp" 29 | "${CMAKE_CURRENT_SOURCE_DIR}/stringInputStreamTests.cpp" 30 | "${CMAKE_CURRENT_SOURCE_DIR}/tokensOutputStreamTests.cpp" 31 | "${CMAKE_CURRENT_SOURCE_DIR}/main.cpp") 32 | 33 | source_group("includes" FILES ${HEADERS}) 34 | source_group("sources" FILES ${SOURCES}) 35 | 36 | add_definitions(-DTCPP_OUTPUT_TOKENS_EXTENSION_ENABLED) 37 | 38 | if (MSVC) #cl.exe compiler's options 39 | 40 | #Debug compiler's options 41 | set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /WX /std:c++14 /MDd /W3 /GS /Zc:inline /Od /Zi /Zc:wchar_t") 42 | set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /RTC1 /Gd /Oy- /EHsc /nologo /diagnostics:classic /errorReport:prompt /sdl /permissive- /analyze-") 43 | set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /D _DEBUG") 44 | 45 | #Release compiler's options 46 | set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /std:c++14 /permissive- /GS /GL /analyze- /W3 /Gy /Zc:wchar_t /Zi /O2 /sdl /Zc:inline") 47 | set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /fp:precise /D _WINDLL /D _MBCS /errorReport:prompt /WX- /Zc:forScope /Gd /Oy- /Oi /MD /EHsc /nologo /diagnostics:classic") 48 | endif(MSVC) 49 | 50 | 51 | if (UNIX) 52 | message(STATUS "UNIX system has detected...") 53 | 54 | include(CheckCXXCompilerFlag) 55 | 56 | CHECK_CXX_COMPILER_FLAG("-std=c++1y" COMPILER_SUPPORTS_CXX14) 57 | 58 | if(COMPILER_SUPPORTS_CXX14) 59 | message(STATUS "C++14 is enabled") 60 | 61 | set(CMAKE_CXX_STANDARD 14) 62 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 63 | else() 64 | message(ERROR "The compiler ${CMAKE_CXX_COMPILER} has no C++14 support. Please use a different C++ compiler.") 65 | endif() 66 | endif (UNIX) 67 | 68 | add_executable(${TCPP_TESTS_NAME} ${SOURCES} ${HEADERS}) 69 | target_link_libraries(${TCPP_TESTS_NAME} Catch2::Catch2) 70 | 71 | include(CTest) 72 | include(Catch) 73 | 74 | catch_discover_tests(${TCPP_TESTS_NAME}) 75 | -------------------------------------------------------------------------------- /tests/coreTests.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "tcppLibrary.hpp" 3 | #include 4 | 5 | 6 | using namespace tcpp; 7 | 8 | 9 | static bool ContainsMacro(const Preprocessor& preprocessor, const std::string& macroIdentifier) 10 | { 11 | const auto& symTable = preprocessor.GetSymbolsTable(); 12 | 13 | auto it = std::find_if(symTable.cbegin(), symTable.cend(), [¯oIdentifier](auto&& symTableEntry) 14 | { 15 | return symTableEntry.mName == macroIdentifier; 16 | }); 17 | 18 | return it != symTable.cend(); 19 | } 20 | 21 | 22 | TEST_CASE("Preprocessor Tests") 23 | { 24 | auto errorCallback = [](const TErrorInfo&) 25 | { 26 | REQUIRE(false); 27 | }; 28 | 29 | SECTION("TestProcess_PassSourceWithoutMacros_ReturnsEquaivalentSource") 30 | { 31 | std::string inputSource = "void main/* this is a comment*/(/*void*/)\n{\n\treturn/* */ 42;\n}"; 32 | Lexer lexer(std::make_unique(inputSource)); 33 | 34 | Preprocessor preprocessor(lexer, { errorCallback }); 35 | REQUIRE(!Preprocessor::ToString(preprocessor.Process()).empty()); 36 | } 37 | 38 | SECTION("TestProcess_PassSourceWithSimpleMacro_ReturnsSourceWithExpandedMacro") 39 | { 40 | std::string inputSource = "#define VALUE 42\n void main()\n{\n\treturn VALUE;\n}"; 41 | Lexer lexer(std::make_unique(inputSource)); 42 | 43 | Preprocessor preprocessor(lexer, { errorCallback }); 44 | preprocessor.Process(); 45 | } 46 | 47 | SECTION("TestProcess_PassSourceWithSimpleMacroWithoutValue_ReturnsSourceWithExpandedMacro") 48 | { 49 | std::string inputSource = "#define VALUE\nVALUE"; 50 | Lexer lexer(std::make_unique(inputSource)); 51 | 52 | Preprocessor preprocessor(lexer, { errorCallback }); 53 | REQUIRE(Preprocessor::ToString(preprocessor.Process()) == "1"); 54 | } 55 | 56 | SECTION("TestProcess_PassSourceWithCorrectFuncMacro_ReturnsSourceWithExpandedMacro") 57 | { 58 | std::string inputSource = "#define ADD(X, Y) X + Y\n void main()\n{\n\treturn ADD(2, 3);\n}"; 59 | Lexer lexer(std::make_unique(inputSource)); 60 | 61 | Preprocessor preprocessor(lexer, { errorCallback }); 62 | preprocessor.Process(); 63 | } 64 | 65 | SECTION("TestProcess_PassSourceWithIncludeDirective_ReturnsSourceStringWithIncludeDirective") 66 | { 67 | std::string inputSource = "#include \n#include \"non_system_path\"\n void main()\n{\n\treturn ADD(2, 3);\n}"; 68 | Lexer lexer(std::make_unique(inputSource)); 69 | 70 | const std::tuple expectedPaths[]{ { "system", true }, { "non_system_path", false } }; 71 | short currExpectedPathIndex = 0; 72 | 73 | Preprocessor preprocessor(lexer, { errorCallback, [&inputSource, &currExpectedPathIndex, &expectedPaths](const std::string& path, bool isSystem) 74 | { 75 | auto expectedResultPair = expectedPaths[currExpectedPathIndex++]; 76 | 77 | REQUIRE(path == std::get(expectedResultPair)); 78 | REQUIRE(isSystem == std::get(expectedResultPair)); 79 | 80 | return std::make_unique(""); 81 | } }); 82 | preprocessor.Process(); 83 | } 84 | 85 | SECTION("TestProcess_PassSourceWithIncludeDirective_ReturnsSourceStringWithIncludeDirective") 86 | { 87 | std::string inputSource = "__LINE__\n__LINE__\n__LINE__"; 88 | Lexer lexer(std::make_unique(inputSource)); 89 | 90 | Preprocessor preprocessor(lexer, { errorCallback }); 91 | REQUIRE(Preprocessor::ToString(preprocessor.Process()) == "1\n2\n3"); 92 | } 93 | 94 | SECTION("TestProcess_PassSourceWithStringizeOperator_ReturnsSourceWithStringifiedToken") 95 | { 96 | std::string inputSource = "#define FOO(Name) #Name\n FOO(Text)"; 97 | Lexer lexer(std::make_unique(inputSource)); 98 | 99 | Preprocessor preprocessor(lexer, { errorCallback }); 100 | REQUIRE(Preprocessor::ToString(preprocessor.Process()) == " \"Text\""); 101 | } 102 | 103 | /*SECTION("TestProcess_PassSourceWithConcatenationOperator_ReturnsSourceWithConcatenatedTokens") 104 | { 105 | std::string inputSource = "#define CAT(X, Y) X ## Y\n CAT(4, 2)"; 106 | StringInputStream input(inputSource); 107 | Lexer lexer(input); 108 | 109 | Preprocessor preprocessor(lexer, errorCallback); 110 | REQUIRE(preprocessor.Process() == " 42"); 111 | }*/ 112 | 113 | SECTION("TestProcess_PassSourceWithConditionalBlocks_ReturnsSourceWithoutThisBlock") 114 | { 115 | std::string inputSource = "#if FOO\none#endif\n two three"; 116 | Lexer lexer(std::make_unique(inputSource)); 117 | 118 | Preprocessor preprocessor(lexer, { errorCallback }); 119 | REQUIRE(Preprocessor::ToString(preprocessor.Process()) == "\n two three"); 120 | } 121 | 122 | SECTION("TestProcess_PassSourceWithConditionalBlocks_ReturnsSourceWithoutIfBlock") 123 | { 124 | std::string inputSource = "#if FOO\n // this block will be skiped\n if block\n#else\n else block #endif"; 125 | Lexer lexer(std::make_unique(inputSource)); 126 | 127 | Preprocessor preprocessor(lexer, { errorCallback }); 128 | REQUIRE(Preprocessor::ToString(preprocessor.Process()) == "\n else block "); 129 | } 130 | 131 | SECTION("TestProcess_PassSourceWithConditionalBlocks_ReturnsSourceWithoutElseBlock") 132 | { 133 | std::string inputSource = "#if 1\n if block\n#else\n else block #endif"; 134 | Lexer lexer(std::make_unique(inputSource)); 135 | 136 | Preprocessor preprocessor(lexer, { errorCallback }); 137 | REQUIRE(Preprocessor::ToString(preprocessor.Process()) == " if block\n"); 138 | } 139 | 140 | SECTION("TestProcess_PassSourceWithElifBlocks_ReturnsSourceWithElabledElifBlock") 141 | { 142 | std::string inputSource = "#if 0\none\n#elif 1\ntwo\n#else\nthree\n#endif"; 143 | Lexer lexer(std::make_unique(inputSource)); 144 | 145 | Preprocessor preprocessor(lexer, { errorCallback }); 146 | REQUIRE(Preprocessor::ToString(preprocessor.Process()) == "two\n"); 147 | } 148 | 149 | SECTION("TestProcess_PassSourceWithFewElifBlocks_ReturnsSourceWithElabledElifBlock") 150 | { 151 | std::string inputSource = "#if 0\none\n#elif 0\ntwo\n#elif 1\nthree\n#else\nfour\n#endif"; 152 | Lexer lexer(std::make_unique(inputSource)); 153 | 154 | Preprocessor preprocessor(lexer, { errorCallback }); 155 | REQUIRE(Preprocessor::ToString(preprocessor.Process()) == "three\n"); 156 | } 157 | 158 | SECTION("TestProcess_PassSourceWithInvalidElseBlock_ReturnsError") 159 | { 160 | std::string inputSource = "#if 0\none\n#elif 0\ntwo\n#else\nfour\n#elif 1\nthree\n#endif"; 161 | Lexer lexer(std::make_unique(inputSource)); 162 | 163 | bool result = false; 164 | 165 | Preprocessor preprocessor(lexer, { [&result](auto&&) 166 | { 167 | result = true; 168 | } }); 169 | 170 | preprocessor.Process(); 171 | REQUIRE(result); 172 | } 173 | 174 | SECTION("TestProcess_PassSourceWithNestedConditionalBlocks_CorrectlyProcessedNestedBlocks") 175 | { 176 | std::string inputSource = "#if 1\none\n#if 0\ntwo\n#endif\nfour\n#elif 0\nthree\n#endif"; 177 | Lexer lexer(std::make_unique(inputSource)); 178 | 179 | Preprocessor preprocessor(lexer, { errorCallback }); 180 | REQUIRE(Preprocessor::ToString(preprocessor.Process()) == "one\n\nfour\n"); 181 | } 182 | 183 | SECTION("TestProcess_PassSourceWithIfdefBlock_CorrectlyProcessesIfdefBlock") 184 | { 185 | std::string inputSource = "#ifdef FOO\none\n#endif\ntwo"; 186 | Lexer lexer(std::make_unique(inputSource)); 187 | 188 | Preprocessor preprocessor(lexer, { errorCallback }); 189 | REQUIRE(Preprocessor::ToString(preprocessor.Process()) == "\ntwo"); 190 | } 191 | 192 | SECTION("TestProcess_PassSourceWithIfndefBlock_CorrectlyProcessesIfndefBlock") 193 | { 194 | std::string inputSource = "#ifndef FOO\none\n#endif\ntwo"; 195 | Lexer lexer(std::make_unique(inputSource)); 196 | 197 | Preprocessor preprocessor(lexer, { errorCallback }); 198 | REQUIRE(Preprocessor::ToString(preprocessor.Process()) == "one\n\ntwo"); 199 | } 200 | 201 | SECTION("TestProcess_PassNestedActiveIfdefBlockInsideOfAnotherInactiveIfdefBlock_TopBlockShouldBeRejected") 202 | { 203 | std::string inputSource = R"( 204 | #define CONDITION_1 205 | 206 | #ifdef CONDITION_0 207 | condition_0, 208 | #ifdef CONDITION_1 209 | condition_1 210 | #endif 211 | #endif 212 | )"; 213 | Lexer lexer(std::make_unique(inputSource)); 214 | 215 | Preprocessor preprocessor(lexer, { errorCallback }); 216 | std::string output = Preprocessor::ToString(preprocessor.Process()); 217 | REQUIRE((output.find("condition_1") == std::string::npos && output.find("condition_0") == std::string::npos)); 218 | } 219 | 220 | SECTION("TestProcess_PassNestedActiveElseBlockInsideOfAnotherInactiveIfdefBlock_TopBlockShouldBeRejected") 221 | { 222 | std::string inputSource = R"( 223 | #ifdef CONDITION_0 224 | condition_0, 225 | #ifdef CONDITION_1 226 | condition_1 227 | #else 228 | condition_1_else 229 | #endif 230 | #endif 231 | )"; 232 | Lexer lexer(std::make_unique(inputSource)); 233 | 234 | Preprocessor preprocessor(lexer, { errorCallback }); 235 | std::string output = Preprocessor::ToString(preprocessor.Process()); 236 | REQUIRE(( 237 | output.find("condition_1") == std::string::npos && 238 | output.find("condition_0") == std::string::npos && 239 | output.find("condition_1_else") == std::string::npos)); 240 | } 241 | 242 | SECTION("TestProcess_PassNestedActiveElifBlockInsideOfAnotherInactiveIfdefBlock_TopBlockShouldBeRejected") 243 | { 244 | std::string inputSource = R"( 245 | #define CONDITION_2 246 | 247 | #ifdef CONDITION_0 248 | condition_0, 249 | #ifdef CONDITION_1 250 | condition_1 251 | #elif CONDITION_2 252 | condition_1_else 253 | #endif 254 | #endif 255 | )"; 256 | Lexer lexer(std::make_unique(inputSource)); 257 | 258 | Preprocessor preprocessor(lexer, { errorCallback }); 259 | std::string output = Preprocessor::ToString(preprocessor.Process()); 260 | REQUIRE(( 261 | output.find("condition_1") == std::string::npos && 262 | output.find("condition_0") == std::string::npos && 263 | output.find("condition_1_else") == std::string::npos)); 264 | } 265 | 266 | SECTION("TestProcess_PassSource_ReturnsProcessedSource") 267 | { 268 | std::string inputSource = "#define FOO\n#ifdef FOO\none\n#endif"; 269 | Lexer lexer(std::make_unique(inputSource)); 270 | 271 | Preprocessor preprocessor(lexer, { errorCallback }); 272 | REQUIRE(Preprocessor::ToString(preprocessor.Process()) == "one\n"); 273 | } 274 | 275 | SECTION("TestProcess_PassSourceWithIncludeDirective_ReturnsProcessedSource") 276 | { 277 | const std::string input("#include \ntwo"); 278 | const std::string systemInput("one\n"); 279 | Lexer lexer(std::make_unique(input)); 280 | 281 | bool result = true; 282 | 283 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 284 | { 285 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 286 | result = false; 287 | }, [&systemInput](auto&&, auto&&) 288 | { 289 | return std::make_unique(systemInput); 290 | } }); 291 | 292 | REQUIRE((result && (Preprocessor::ToString(preprocessor.Process()) == "one\ntwo"))); 293 | } 294 | 295 | SECTION("TestProcess_PassSourceWithIncludeGuards_ReturnsProcessedSource") 296 | { 297 | std::string inputSource = R"( 298 | #define FOO 299 | 300 | #include 301 | 302 | #ifndef FILE_H 303 | #define FILE_H 304 | 305 | #ifdef FOO 306 | #define BAR(x) x 307 | #endif 308 | 309 | #ifdef FOO2 310 | #define BAR(x) x,x 311 | #endif 312 | 313 | #endif 314 | )"; 315 | 316 | std::string systemSource = R"( 317 | #ifndef SYSTEM_H 318 | #define SYSTEM_H 319 | 320 | #define FOO3 321 | int x = 42; 322 | 323 | #endif 324 | )"; 325 | 326 | Lexer lexer(std::make_unique(inputSource)); 327 | 328 | bool result = true; 329 | 330 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 331 | { 332 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 333 | result = false; 334 | }, [&systemSource](auto&&, auto&&) 335 | { 336 | return std::make_unique(systemSource); 337 | } }); 338 | 339 | std::string output = Preprocessor::ToString(preprocessor.Process()); 340 | REQUIRE(result); 341 | } 342 | 343 | SECTION("TestProcess_PassSourceWithFunctionMacro_ReturnsProcessedSource") 344 | { 345 | std::string inputSource = "#define FOO(X, Y) Foo.getValue(X, Y)\nFOO(42, input.value)"; 346 | Lexer lexer(std::make_unique(inputSource)); 347 | 348 | bool result = true; 349 | 350 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 351 | { 352 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 353 | result = false; 354 | } }); 355 | 356 | preprocessor.Process(); 357 | REQUIRE(result); 358 | } 359 | 360 | SECTION("TestProcess_PassFloatingPointValue_ReturnsThisValue") 361 | { 362 | std::string inputSource = "1.0001 1.00001f vec4(1.0f, 0.2, 0.223, 1.0001f);"; 363 | Lexer lexer(std::make_unique(inputSource)); 364 | 365 | bool result = true; 366 | 367 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 368 | { 369 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 370 | result = false; 371 | } }); 372 | 373 | auto&& output = Preprocessor::ToString(preprocessor.Process()); 374 | REQUIRE(output == inputSource); 375 | } 376 | 377 | SECTION("TestProcess_PassFloatingPointValue_ReturnsThisValue2") 378 | { 379 | std::string inputSource = "float c = nebula(layer2_coord * 3.0) * 0.35 - 0.05"; 380 | Lexer lexer(std::make_unique(inputSource)); 381 | 382 | bool result = true; 383 | 384 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 385 | { 386 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 387 | result = false; 388 | } }); 389 | 390 | auto&& output = Preprocessor::ToString(preprocessor.Process()); 391 | REQUIRE(output == inputSource); 392 | } 393 | 394 | SECTION("TestProcess_PassTwoStringsWithConcatOperation_ReturnsSingleString") 395 | { 396 | std::string inputSource = "AAA ## BB"; 397 | Lexer lexer(std::make_unique(inputSource)); 398 | 399 | bool result = true; 400 | 401 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 402 | { 403 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 404 | result = false; 405 | } }); 406 | 407 | std::string str = Preprocessor::ToString(preprocessor.Process()); 408 | REQUIRE((result && (str == "AAABB"))); 409 | } 410 | 411 | SECTION("TestProcess_PassSourceWithFunctionMacro_ReturnsProcessedSource") 412 | { 413 | std::string inputSource = "#define FOO(X) \\\nint X; \\\nint X ## _Additional;\nFOO(Test)"; 414 | std::string expectedResult = "int Test;int Test_Additional;"; 415 | 416 | Lexer lexer(std::make_unique(inputSource)); 417 | 418 | bool result = true; 419 | 420 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 421 | { 422 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 423 | result = false; 424 | } }); 425 | 426 | REQUIRE((result && (Preprocessor::ToString(preprocessor.Process()) == expectedResult))); 427 | } 428 | 429 | SECTION("TestProcess_PassNestedFunctionMacroIntoAnotherFunctionMacro_ReturnsProcessedSource") 430 | { 431 | std::string inputSource = "#define FOO(X, Y) X(Y)\nFOO(Foo, Test(0, 0))"; 432 | std::string expectedResult = "Foo(Test(0, 0))"; 433 | 434 | Lexer lexer(std::make_unique(inputSource)); 435 | 436 | bool result = true; 437 | 438 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 439 | { 440 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 441 | result = false; 442 | } }); 443 | 444 | std::string actualResult = Preprocessor::ToString(preprocessor.Process()); 445 | REQUIRE((result && (actualResult == expectedResult))); 446 | } 447 | 448 | SECTION("TestProcess_PassEscapeSequenceInsideLiteralString_CorrectlyPreprocessIt") 449 | { 450 | std::string inputSource = R"( 451 | void main() { 452 | printf("test \n"); 453 | })"; 454 | 455 | std::string expectedResult = R"( 456 | void main() { 457 | printf("test \n"); 458 | })"; 459 | 460 | Lexer lexer(std::make_unique(inputSource)); 461 | 462 | bool result = true; 463 | 464 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 465 | { 466 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 467 | result = false; 468 | } }); 469 | 470 | std::string actualResult = Preprocessor::ToString(preprocessor.Process()); 471 | REQUIRE((result && (actualResult == expectedResult))); 472 | } 473 | 474 | SECTION("TestProcess_PassTextWithEscapeSequenceWithinCommentary_CommentsAreBypassedWithoutAnyChanges") 475 | { 476 | std::string inputSource = R"( 477 | Line above 478 | 479 | // "\p" 480 | Line below 481 | float getNumber() { 482 | return 1.0; 483 | })"; 484 | 485 | std::string expectedResult = R"( 486 | Line above 487 | 488 | // "\p" 489 | Line below 490 | float getNumber() { 491 | return 1.0; 492 | })"; 493 | 494 | Lexer lexer(std::make_unique(inputSource)); 495 | 496 | bool result = true; 497 | 498 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 499 | { 500 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 501 | result = false; 502 | } }); 503 | 504 | std::string actualResult = Preprocessor::ToString(preprocessor.Process()); 505 | REQUIRE((result && (actualResult == expectedResult))); 506 | } 507 | 508 | SECTION("TestProcess_VA_ARGS") 509 | { 510 | std::string inputSource = "#define DO_STUFF(x, ...) x.do_stuf(); __VA_ARGS__\n int main() { DO_STUFF(2, 3+5, 4); }"; 511 | 512 | Lexer lexer(std::make_unique(inputSource)); 513 | 514 | Preprocessor preprocessor(lexer, { [](const TErrorInfo& err) 515 | { 516 | std::cout << "Error" << ErrorTypeToString(err.mType) << "\n"; 517 | } }); 518 | 519 | preprocessor.Process(); 520 | } 521 | 522 | SECTION("TestProcess_VA_ARGS_MultiExpand") 523 | { 524 | std::string inputSource = R"( 525 | #define HEAD(x, ...) x 526 | #define TAIL(x, ...) __VA_ARGS__ 527 | 528 | int main() 529 | { 530 | return TAIL(1, 2, 3); 531 | } 532 | )"; 533 | 534 | Lexer lexer(std::make_unique(inputSource)); 535 | 536 | Preprocessor preprocessor(lexer, { [](const TErrorInfo& err) 537 | { 538 | std::cout << "Error" << ErrorTypeToString(err.mType) << "\n"; 539 | } }); 540 | 541 | preprocessor.Process(); 542 | } 543 | 544 | SECTION("TestProcess_PassDefineThatSeparatedWithSpaces_ReturnsCorrectProcessedSource") 545 | { 546 | std::string inputSource = "# define Foo"; 547 | 548 | Lexer lexer(std::make_unique(inputSource)); 549 | 550 | bool result = true; 551 | 552 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 553 | { 554 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 555 | result = false; 556 | } }); 557 | 558 | std::string str = Preprocessor::ToString(preprocessor.Process()); 559 | 560 | /// \note symbol table should contain Foo macro 561 | REQUIRE(ContainsMacro(preprocessor, "Foo")); 562 | REQUIRE((result && str.empty())); 563 | } 564 | 565 | SECTION("TestProcess_PassCodeWithCommentary_ReturnsCorrectProcessedSource") 566 | { 567 | std::string inputSource = "A;// Commentary"; 568 | std::string expectedResult = "A;// Commentary"; 569 | 570 | Lexer lexer(std::make_unique(inputSource)); 571 | 572 | bool result = true; 573 | 574 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 575 | { 576 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 577 | result = false; 578 | } }); 579 | 580 | std::string str = Preprocessor::ToString(preprocessor.Process()); 581 | 582 | REQUIRE((result && str == expectedResult)); 583 | } 584 | 585 | SECTION("TestProcess_EvaluateExpressionsInDefines_AllExpressionsShouldBeComputedCorrectly") 586 | { 587 | std::string inputSource = R"( 588 | #define A 1 589 | #define C 0 590 | #define FOO(X, Y) (X && Y) 591 | 592 | #if A && B 593 | #define PASSED_0 594 | #else 595 | #define FAILED_0 596 | #endif 597 | 598 | #if A || B 599 | #define PASSED_1 600 | #else 601 | #define FAILED_1 602 | #endif 603 | 604 | #if !A 605 | #define PASSED_2 606 | #else 607 | #define FAILED_2 608 | #endif 609 | 610 | #if A + B 611 | #define PASSED_3 612 | #else 613 | #define FAILED_3 614 | #endif 615 | 616 | #if A - B 617 | #define PASSED_4 618 | #else 619 | #define FAILED_4 620 | #endif 621 | 622 | #if A * B 623 | #define PASSED_5 624 | #else 625 | #define FAILED_5 626 | #endif 627 | 628 | #if A / B 629 | #define PASSED_6 630 | #else 631 | #define FAILED_6 632 | #endif 633 | 634 | #if C 635 | #define PASSED_7 636 | #else 637 | #define FAILED_7 638 | #endif 639 | )"; 640 | 641 | Lexer lexer(std::make_unique(inputSource)); 642 | 643 | bool result = true; 644 | 645 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 646 | { 647 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 648 | result = false; 649 | } }); 650 | 651 | std::string str = Preprocessor::ToString(preprocessor.Process()); 652 | 653 | REQUIRE(!ContainsMacro(preprocessor, "PASSED_0")); 654 | REQUIRE(ContainsMacro(preprocessor, "FAILED_0")); 655 | 656 | REQUIRE(ContainsMacro(preprocessor, "PASSED_1")); 657 | REQUIRE(!ContainsMacro(preprocessor, "FAILED_1")); 658 | 659 | REQUIRE(!ContainsMacro(preprocessor, "PASSED_2")); 660 | REQUIRE(ContainsMacro(preprocessor, "FAILED_2")); 661 | 662 | REQUIRE(ContainsMacro(preprocessor, "PASSED_3")); 663 | REQUIRE(!ContainsMacro(preprocessor, "FAILED_3")); 664 | 665 | REQUIRE(ContainsMacro(preprocessor, "PASSED_4")); 666 | REQUIRE(!ContainsMacro(preprocessor, "FAILED_4")); 667 | 668 | REQUIRE(!ContainsMacro(preprocessor, "PASSED_5")); 669 | REQUIRE(ContainsMacro(preprocessor, "FAILED_5")); 670 | 671 | REQUIRE(!ContainsMacro(preprocessor, "PASSED_6")); 672 | REQUIRE(ContainsMacro(preprocessor, "FAILED_6")); 673 | 674 | REQUIRE(!ContainsMacro(preprocessor, "PASSED_7")); 675 | REQUIRE(ContainsMacro(preprocessor, "FAILED_7")); 676 | 677 | REQUIRE(result); 678 | } 679 | 680 | SECTION("TestProcess_EvaluateMacroFunctionExpressions_MacroFunctionShouldBeExpandedBeforeEvaluation") 681 | { 682 | std::string inputSource = R"( 683 | #define A 1 684 | #define AND(X, Y) (X && Y) 685 | 686 | #if AND(A, 0) 687 | #define PASSED 688 | #else 689 | #define FAILED 690 | #endif 691 | 692 | #if AND(A, 1) 693 | #define PASSED_1 694 | #else 695 | #define FAILED_1 696 | #endif 697 | )"; 698 | 699 | Lexer lexer(std::make_unique(inputSource)); 700 | 701 | bool result = true; 702 | 703 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 704 | { 705 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 706 | result = false; 707 | } }); 708 | 709 | std::string str = Preprocessor::ToString(preprocessor.Process()); 710 | 711 | REQUIRE(!ContainsMacro(preprocessor, "PASSED")); 712 | REQUIRE(ContainsMacro(preprocessor, "FAILED")); 713 | 714 | REQUIRE(ContainsMacro(preprocessor, "PASSED_1")); 715 | REQUIRE(!ContainsMacro(preprocessor, "FAILED_1")); 716 | 717 | REQUIRE(result); 718 | } 719 | 720 | SECTION("TestProcess_PassIncludeDirectiveWithoutNewlineEscapeSequence_DirectiveShouldBeProcessedCorrectly") 721 | { 722 | std::string inputSource = "#include "; 723 | 724 | Lexer lexer(std::make_unique(inputSource)); 725 | 726 | bool result = true; 727 | 728 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 729 | { 730 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 731 | result = false; 732 | } }); 733 | 734 | std::string str = Preprocessor::ToString(preprocessor.Process()); 735 | } 736 | 737 | SECTION("TestProcess_PassSourceDangerousCommentary_CorrectlyProcessThatCommentary") 738 | { 739 | std::string inputSource = R"( 740 | #ifndef FOO_H 741 | #define FOO_H 742 | 743 | /*int foo() { 744 | return 0 ;//* 42; // this //* sequence can be considered as commentary's beginning 745 | } 746 | */ 747 | 748 | #endif 749 | )"; 750 | 751 | Lexer lexer(std::make_unique(inputSource)); 752 | 753 | bool result = true; 754 | 755 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 756 | { 757 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 758 | result = false; 759 | }, [](auto&&, auto&&) 760 | { 761 | return nullptr; 762 | } }); 763 | 764 | std::string output = Preprocessor::ToString(preprocessor.Process()); 765 | 766 | REQUIRE((!output.empty() && output.find("#endif") == std::string::npos)); 767 | } 768 | 769 | SECTION("TestProcess_PassSourceWithCommentPreprocessorSkipsThem_TheOutputDoesntContainComments") 770 | { 771 | std::string inputSource = R"( 772 | int main(int argc, char** argv) { 773 | // TEST COMMENT 774 | return -1; 775 | } 776 | )"; 777 | 778 | Lexer lexer(std::make_unique(inputSource)); 779 | 780 | bool result = true; 781 | 782 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 783 | { 784 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 785 | result = false; 786 | }, [](auto&&, auto&&) 787 | { 788 | return nullptr; 789 | }, 790 | true }); 791 | 792 | std::string output = Preprocessor::ToString(preprocessor.Process()); 793 | 794 | REQUIRE((!output.empty() && output.find("COMMENT") == std::string::npos)); 795 | } 796 | 797 | SECTION("TestProcess_PassMacroIntoFuncMacroWithConcatenation_MacroExpansionIsOmitted") 798 | { 799 | std::string inputSource = R"( 800 | #define STRCAT(a, b) a ## b 801 | STRCAT(__LINE__, b) 802 | STRCAT(a, __LINE__) 803 | )"; 804 | 805 | Lexer lexer(std::make_unique(inputSource)); 806 | 807 | bool result = true; 808 | 809 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 810 | { 811 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 812 | result = false; 813 | }, [](auto&&, auto&&) 814 | { 815 | return nullptr; 816 | }, 817 | true }); 818 | 819 | std::string output = Preprocessor::ToString(preprocessor.Process()); 820 | 821 | REQUIRE(output == "\n__LINE__b\na__LINE__\n"); // If an argument is stringized or concatenated, the prescan does not occur and macro is not expanded 822 | } 823 | #if 0 824 | SECTION("TestProcess_PassMacroIntoFuncMacroWithinAnotherFuncMacro_MacrosExpanded") 825 | { 826 | std::string inputSource = R"( 827 | #define STRCAT(a, b) a##b 828 | #define STRCAT1(a, b) STRCAT(a, b) 829 | STRCAT1(__LINE__, b) 830 | )"; 831 | 832 | Lexer lexer(std::make_unique(inputSource)); 833 | 834 | bool result = true; 835 | 836 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 837 | { 838 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 839 | result = false; 840 | }, [](auto&&, auto&&) 841 | { 842 | return nullptr; 843 | }, 844 | true }); 845 | 846 | std::string output = Preprocessor::ToString(preprocessor.Process())(); 847 | 848 | REQUIRE(output == "\n__LINE__3\n"); // If an argument is stringized or concatenated, the prescan does not occur and macro is not expanded 849 | } 850 | #endif 851 | SECTION("TestProcess_DefineSelfReferencedMacro_MacroIsExpandedOnlyOnce") 852 | { 853 | std::string inputSource = R"( 854 | #define FOO 1 + FOO 855 | FOO 856 | )"; 857 | 858 | Lexer lexer(std::make_unique(inputSource)); 859 | 860 | bool result = true; 861 | 862 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 863 | { 864 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 865 | result = false; 866 | }, [](auto&&, auto&&) 867 | { 868 | return nullptr; 869 | }, 870 | true }); 871 | 872 | std::string output = Preprocessor::ToString(preprocessor.Process()); 873 | 874 | REQUIRE(output == "\n1 + FOO\n"); 875 | } 876 | 877 | SECTION("TestProcess_FunctionMacroWithoutInvokation_MacroIsNotExpanded") 878 | { 879 | std::string inputSource = R"( 880 | #define FOO(X) X 881 | auto foo = FOO; 882 | )"; 883 | 884 | Lexer lexer(std::make_unique(inputSource)); 885 | 886 | bool result = true; 887 | 888 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 889 | { 890 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 891 | result = false; 892 | }, [](auto&&, auto&&) 893 | { 894 | return nullptr; 895 | }, 896 | true }); 897 | 898 | std::string output = Preprocessor::ToString(preprocessor.Process()); 899 | 900 | REQUIRE(output == "\nauto foo = FOO;\n"); 901 | } 902 | 903 | SECTION("TestProcess_PassCommaInBracketsAsFirstArgumentInMacro_WholeBracketsBlockAssumedAsFirstArgument") 904 | { 905 | std::string inputSource = R"( 906 | #define FIRST(X, Y) X 907 | FIRST((1, 2) c, 3) 908 | )"; 909 | 910 | Lexer lexer(std::make_unique(inputSource)); 911 | 912 | bool result = true; 913 | 914 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 915 | { 916 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 917 | result = false; 918 | }, [](auto&&, auto&&) 919 | { 920 | return nullptr; 921 | }, 922 | true }); 923 | 924 | std::string output = Preprocessor::ToString(preprocessor.Process()); 925 | REQUIRE((result && output == "\n(1, 2) c\n")); 926 | } 927 | 928 | SECTION("TestProcess_StringifyOperatorInvokedOnNonParameterToken_ProcessingErrorOccurs") 929 | { 930 | std::string inputSource = R"( 931 | #define TEST(X) #value 932 | TEST(3) 933 | )"; 934 | 935 | Lexer lexer(std::make_unique(inputSource)); 936 | 937 | bool result = true; 938 | 939 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 940 | { 941 | REQUIRE(arg.mType == E_ERROR_TYPE::INCORRECT_STRINGIFY_OPERATOR_USAGE); 942 | result = false; 943 | }, [](auto&&, auto&&) 944 | { 945 | return nullptr; 946 | }, 947 | true }); 948 | 949 | const std::string& output = Preprocessor::ToString(preprocessor.Process()); 950 | REQUIRE(!result); 951 | } 952 | 953 | SECTION("TestProcess_PassEmptyArg_MacroExpanded") 954 | { 955 | std::string inputSource = R"( 956 | #define TEST(X) X 957 | TEST( ) 958 | )"; 959 | 960 | Lexer lexer(std::make_unique(inputSource)); 961 | 962 | bool result = true; 963 | 964 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 965 | { 966 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 967 | result = false; 968 | }, [](auto&&, auto&&) 969 | { 970 | return nullptr; 971 | }, 972 | true }); 973 | 974 | preprocessor.Process(); 975 | REQUIRE(result); 976 | } 977 | 978 | SECTION("TestProcess_PassEmptyArgWithoutSpace_ProcessingErrorOccurs") 979 | { 980 | std::string inputSource = R"( 981 | #define TEST(X) X 982 | TEST() 983 | )"; 984 | 985 | Lexer lexer(std::make_unique(inputSource)); 986 | 987 | bool result = true; 988 | 989 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 990 | { 991 | REQUIRE(arg.mType == E_ERROR_TYPE::INCONSISTENT_MACRO_ARITY); 992 | result = false; 993 | }, [](auto&&, auto&&) 994 | { 995 | return nullptr; 996 | }, 997 | true }); 998 | 999 | preprocessor.Process(); 1000 | REQUIRE(!result); 1001 | } 1002 | 1003 | SECTION("TestProcess_PassDefineExpansionInBrackets_MacroCorrectlyExpanded") 1004 | { 1005 | std::string inputSource = "#define COUNT 4\nint array[COUNT];\n"; 1006 | 1007 | Lexer lexer(std::make_unique(inputSource)); 1008 | 1009 | bool result = true; 1010 | 1011 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 1012 | { 1013 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 1014 | result = false; 1015 | }, [](auto&&, auto&&) 1016 | { 1017 | return nullptr; 1018 | }, 1019 | true }); 1020 | 1021 | std::string output = Preprocessor::ToString(preprocessor.Process()); 1022 | REQUIRE((result && output == "int array[4];\n")); 1023 | } 1024 | 1025 | SECTION("TestProcess_ArgumentsAreExpandedFirstInFunctionLikeMacroes_Returns4aThen__LINE__a") 1026 | { 1027 | std::string inputSource = R"( 1028 | #define foo(a, b) a ## b 1029 | #define bar(a, b) foo(a,b) 1030 | #define baz bar(__LINE__, a) 1031 | baz 1032 | 1033 | #define moo(a, b) a ## b 1034 | #define meow moo(__LINE__, a) 1035 | meow)"; 1036 | 1037 | Lexer lexer(std::make_unique(inputSource)); 1038 | 1039 | bool result = true; 1040 | 1041 | Preprocessor preprocessor(lexer, { [&result](auto&& arg) 1042 | { 1043 | std::cerr << "Error: " << ErrorTypeToString(arg.mType) << std::endl; 1044 | result = false; 1045 | }, [](auto&&, auto&&) 1046 | { 1047 | return nullptr; 1048 | }, 1049 | true }); 1050 | 1051 | std::string output = Preprocessor::ToString(preprocessor.Process()); 1052 | REQUIRE((result && output == "\n4a\n\n__LINE__a")); 1053 | } 1054 | } -------------------------------------------------------------------------------- /tests/lexerTests.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "tcppLibrary.hpp" 3 | #include 4 | #include 5 | 6 | using namespace tcpp; 7 | 8 | 9 | class MockInputStream final : public IInputStream 10 | { 11 | public: 12 | MockInputStream(std::vector&& lines) TCPP_NOEXCEPT: 13 | mLines(std::move(lines)), mCurrLine(0) 14 | { 15 | } 16 | 17 | std::string ReadLine() TCPP_NOEXCEPT 18 | { 19 | return mLines[mCurrLine++]; 20 | } 21 | 22 | bool HasNextLine() const TCPP_NOEXCEPT 23 | { 24 | return mCurrLine + 1 <= mLines.size(); 25 | } 26 | private: 27 | std::vector mLines; 28 | size_t mCurrLine; 29 | }; 30 | 31 | 32 | TEST_CASE("Lexer Tests") 33 | { 34 | SECTION("TestGetNextToken_PassEmptyStream_ReturnsEndToken") 35 | { 36 | Lexer lexer(std::make_unique(std::vector { "" })); 37 | 38 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 39 | } 40 | 41 | SECTION("TestGetNextToken_PassStreamWithSplittedLines_ReturnsConcatenatedBlobToken") 42 | { 43 | Lexer lexer(std::make_unique(std::vector { "\\ ", " \\" })); 44 | 45 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::SPACE); 46 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 47 | } 48 | 49 | SECTION("TestGetNextToken_PassStreamWithWhitespacesLines_ReturnsAllSPACEandENDTokens") 50 | { 51 | Lexer lexer(std::make_unique(std::vector { " ", " \t " })); 52 | 53 | for (size_t i = 0; i < 8; i++) 54 | { 55 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::SPACE); 56 | } 57 | 58 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 59 | } 60 | 61 | SECTION("TestGetNextToken_PassStreamWithDirectives_ReturnsCorrespondingTokens") 62 | { 63 | Lexer lexer(std::make_unique(std::vector { "#define", "#if", "#else", "#elif", "#include", "#endif" })); 64 | 65 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::DEFINE); 66 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IF); 67 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::ELSE); 68 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::ELIF); 69 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::INCLUDE); 70 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::ENDIF); 71 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 72 | } 73 | 74 | SECTION("TestGetNextToken_PassStreamWithIdentifiers_ReturnsIdentifierToken") 75 | { 76 | Lexer lexer(std::make_unique(std::vector { "line", "_macro", "lucky_42" })); 77 | 78 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 79 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 80 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 81 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 82 | } 83 | 84 | SECTION("TestGetNextToken_PassStreamWithSeparators_ReturnsTheirTokens") 85 | { 86 | Lexer lexer(std::make_unique(std::vector { ",()<>\"&|+-*/&&||<<>>!<=>===!={}" })); 87 | 88 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::COMMA); 89 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::OPEN_BRACKET); 90 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::CLOSE_BRACKET); 91 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::LESS); 92 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::GREATER); 93 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::QUOTES); 94 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::AMPERSAND); 95 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::VLINE); 96 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::PLUS); 97 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::MINUS); 98 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::STAR); 99 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::SLASH); 100 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::AND); 101 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::OR); 102 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::LSHIFT); 103 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::RSHIFT); 104 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NOT); 105 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::LE); 106 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::GE); 107 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::EQ); 108 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NE); 109 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::OPEN_BRACE); 110 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::CLOSE_BRACE); 111 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 112 | } 113 | 114 | SECTION("TestGetNextToken_PassStreamWithLineFeeds_ReturnsNewlineToken") 115 | { 116 | Lexer lexer(std::make_unique(std::vector { "line\n", "_macro\n", "lucky_42" })); 117 | 118 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 119 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NEWLINE); 120 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 121 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NEWLINE); 122 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 123 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 124 | } 125 | 126 | SECTION("TestGetNextToken_PassStreamWithKeywords_ReturnsKeywordTokens") 127 | { 128 | std::vector keywords 129 | { 130 | "auto", "double", "int", "struct", 131 | "break", "else", "long", "switch", 132 | "case", "enum", "register", "typedef", 133 | "char", "extern", "return", "union", 134 | "const", "float", "short", "unsigned", 135 | "continue", "for", "signed", "void", 136 | "default", "goto", "sizeof", "volatile", 137 | "do", "if", "static", "while" 138 | }; 139 | 140 | size_t keywordsCount = keywords.size(); 141 | 142 | Lexer lexer(std::make_unique(std::move(keywords))); 143 | 144 | for (size_t i = 0; i < keywordsCount; ++i) 145 | { 146 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::KEYWORD); 147 | } 148 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 149 | } 150 | 151 | SECTION("TestGetNextToken_PassStreamWithSimpleMultilineComments_ReturnsSPACEAndENDTokens") 152 | { 153 | Lexer lexer(std::make_unique(std::vector { "/*test\n", " this thing skip */ " })); 154 | 155 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::COMMENTARY); 156 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::SPACE); 157 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 158 | } 159 | 160 | SECTION("TestGetNextToken_PassStreamWithNestedMultilineComments_ReturnsSPACEAndENDTokens") 161 | { 162 | Lexer lexer(std::make_unique(std::vector { "/*test\n", " /*\n", " */ /*test*/ this thing skip */ " })); 163 | 164 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::COMMENTARY); 165 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::SPACE); 166 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 167 | } 168 | 169 | SECTION("TestGetNextToken_PassStreamWithNestedMultilineComments_ReturnsSPACEAndENDTokens") 170 | { 171 | // \note without comments the string looks like that "id id2 " 172 | Lexer lexer(std::make_unique(std::vector { "id /*test\n", "\n", "*/ id2", "/*test this thing skip */ " })); 173 | 174 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 175 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::SPACE); 176 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::COMMENTARY); 177 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::SPACE); 178 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 179 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::COMMENTARY); 180 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::SPACE); 181 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 182 | } 183 | 184 | SECTION("TestAppendFront_PassFewTokensToExistingOnes_ReturnsAppendedFirstlyThenRest") 185 | { 186 | Lexer lexer(std::make_unique(std::vector { "line", "_macro", "lucky_42" })); 187 | 188 | lexer.AppendFront({ { E_TOKEN_TYPE::BLOB }, { E_TOKEN_TYPE::ELIF } }); 189 | 190 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::BLOB); 191 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::ELIF); 192 | 193 | for (short i = 0; i < 3; ++i) 194 | { 195 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 196 | } 197 | 198 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 199 | } 200 | 201 | SECTION("TestAppendFront_PassFewTokens_ReturnsAllOfThem") 202 | { 203 | Lexer lexer(std::make_unique(std::vector { "(2, 3)" })); 204 | 205 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::OPEN_BRACKET); 206 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NUMBER); 207 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::COMMA); 208 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::SPACE); 209 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NUMBER); 210 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::CLOSE_BRACKET); 211 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 212 | } 213 | 214 | SECTION("TestAppendFront_PassFewTokensToExistingOnes_ReturnsAppendedFirstlyThenRest") 215 | { 216 | Lexer lexer(std::make_unique(std::vector { "line\n", "another line\n" })); 217 | 218 | { 219 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 220 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NEWLINE); 221 | 222 | { 223 | lexer.PushStream(std::make_unique(std::vector { "(\n", ")\n" })); 224 | 225 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::OPEN_BRACKET); 226 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NEWLINE); 227 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::CLOSE_BRACKET); 228 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NEWLINE); 229 | 230 | lexer.PopStream(); 231 | } 232 | 233 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 234 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::SPACE); 235 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 236 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NEWLINE); 237 | 238 | { 239 | lexer.PushStream(std::make_unique(std::vector { "+\n", "#define\n" })); 240 | 241 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::PLUS); 242 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NEWLINE); 243 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::DEFINE); 244 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NEWLINE); 245 | 246 | lexer.PopStream(); 247 | } 248 | } 249 | 250 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 251 | } 252 | 253 | SECTION("TestGetNextToken_PassStreamWithStringificationOperators_ReturnsCorrespondingTokens") 254 | { 255 | Lexer lexer(std::make_unique(std::vector { "# ID", "#ID", "##" })); 256 | 257 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::STRINGIZE_OP); 258 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 259 | 260 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::STRINGIZE_OP); 261 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 262 | 263 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::CONCAT_OP); 264 | 265 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 266 | } 267 | 268 | SECTION("TestGetNextToken_PassNumbersInDifferentRadixes_ReturnsCorrectTokens") 269 | { 270 | Lexer lexer(std::make_unique(std::vector { "42", "0x42", "042" })); 271 | 272 | for (short i = 0; i < 3; ++i) 273 | { 274 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NUMBER); 275 | } 276 | 277 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 278 | } 279 | 280 | SECTION("TestGetNextToken_PassStreamWithKeywordLikeIdentifier_ReturnsIdentifierToken") 281 | { 282 | Lexer lexer(std::make_unique(std::vector { "float4x4" })); 283 | 284 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 285 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 286 | } 287 | 288 | SECTION("TestGetNextToken_PassStreamWithFloatingPointNumbers_ReturnsCorrectTokensSequence") 289 | { 290 | Lexer lexer(std::make_unique(std::vector { "1.0001 1.00001f" })); 291 | 292 | // \note For now we don't actually recognize floating-point numbers just process them as blobs 293 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NUMBER); 294 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::BLOB); 295 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NUMBER); 296 | 297 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::SPACE); 298 | 299 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NUMBER); 300 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::BLOB); 301 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NUMBER); 302 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 303 | 304 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 305 | } 306 | 307 | SECTION("TestGetNextToken_PassTwoStringsWithConcapOp_ReturnsCorrectTokensSequence") 308 | { 309 | Lexer lexer(std::make_unique(std::vector { "AAA ## BB" })); 310 | 311 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 312 | 313 | for (short i = 0; i < 3; ++i) 314 | { 315 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::SPACE); 316 | } 317 | 318 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::CONCAT_OP); 319 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::SPACE); 320 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 321 | 322 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 323 | } 324 | 325 | SECTION("TestGetNextToken_PassSomeCodeThatEndsWithCommentary_ReturnsCorrectTokensSequence") 326 | { 327 | Lexer lexer(std::make_unique(std::vector { "A;// comment" })); 328 | 329 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 330 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::SEMICOLON); 331 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::COMMENTARY); 332 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 333 | } 334 | 335 | SECTION("TestPeekNextToken_IterateOverSequenceUsingOffset_CorrectlyProcessStreamAndReturnsTokens") 336 | { 337 | Lexer lexer(std::make_unique(std::vector { "(2, 3)" })); 338 | 339 | REQUIRE(lexer.PeekNextToken(0).mType == E_TOKEN_TYPE::OPEN_BRACKET); // PeekNextToken(0) equals to GetNextToken() 340 | REQUIRE(lexer.PeekNextToken(1).mType == E_TOKEN_TYPE::NUMBER); 341 | REQUIRE(lexer.PeekNextToken(2).mType == E_TOKEN_TYPE::COMMA); 342 | REQUIRE(lexer.PeekNextToken(3).mType == E_TOKEN_TYPE::SPACE); 343 | REQUIRE(lexer.PeekNextToken(4).mType == E_TOKEN_TYPE::NUMBER); 344 | REQUIRE(lexer.PeekNextToken(5).mType == E_TOKEN_TYPE::CLOSE_BRACKET); 345 | REQUIRE(lexer.PeekNextToken(6).mType == E_TOKEN_TYPE::END); 346 | 347 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NUMBER); 348 | } 349 | 350 | SECTION("TestGetNextToken_StringWithDifferentStylesOfCarriageReturn_CorrectlyRecognizesNewlineToken") 351 | { 352 | Lexer lexer(std::make_unique(std::vector { "#define WIN_STYLE\r\n#define UNIX_STYLE\n" })); 353 | 354 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::DEFINE); 355 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::SPACE); 356 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 357 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NEWLINE); 358 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::DEFINE); 359 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::SPACE); 360 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::IDENTIFIER); 361 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::NEWLINE); 362 | REQUIRE(lexer.GetNextToken().mType == E_TOKEN_TYPE::END); 363 | } 364 | } -------------------------------------------------------------------------------- /tests/main.cpp: -------------------------------------------------------------------------------- 1 | #define CATCH_CONFIG_MAIN 2 | #include 3 | #define TCPP_IMPLEMENTATION 4 | #include "tcppLibrary.hpp" -------------------------------------------------------------------------------- /tests/stringInputStreamTests.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "tcppLibrary.hpp" 3 | #include 4 | 5 | using namespace tcpp; 6 | 7 | 8 | TEST_CASE("StringInputStream Tests") 9 | { 10 | SECTION("TestReadLine_PassEmptyString_ReturnsEmptyString") 11 | { 12 | StringInputStream stringInputStream(""); 13 | REQUIRE(!stringInputStream.HasNextLine()); 14 | } 15 | 16 | SECTION("TestReadLine_PassTwoLines_ReturnsEachOfThem") 17 | { 18 | std::string lines[] 19 | { 20 | "line1\n", 21 | "line2\r\n", 22 | "line3" 23 | }; 24 | 25 | std::string concatenatedString; 26 | 27 | for (const auto& currLine : lines) 28 | { 29 | concatenatedString.append(currLine); 30 | } 31 | 32 | StringInputStream stringInputStream(concatenatedString); 33 | 34 | short linesCount = 0; 35 | 36 | while (stringInputStream.HasNextLine()) 37 | { 38 | REQUIRE(stringInputStream.ReadLine() == lines[linesCount++]); 39 | } 40 | } 41 | 42 | SECTION("TestReadLine_PassStringWithoutLines_ReturnsThisLine") 43 | { 44 | const std::string& expectedLine = "line without string"; 45 | StringInputStream stringInputStream(expectedLine); 46 | REQUIRE(stringInputStream.ReadLine() == expectedLine); 47 | } 48 | 49 | SECTION("TestReadLine_PassComplexString_ReturnsAllItsLines") 50 | { 51 | std::string lines[] 52 | { 53 | "\n", 54 | "#define FOO\n", 55 | "\n", 56 | "#ifndef FILE_H\n", 57 | "#define FILE_H\n", 58 | "\n", 59 | "#ifdef FOO\n", 60 | " #define BAR(x) x\n", 61 | "#endif\n", 62 | "\n", 63 | "#ifdef FOO2\n", 64 | " #define BAR(x) x,x\n", 65 | "#endif\n", 66 | "\n", 67 | "#endif\n", 68 | }; 69 | 70 | std::string inputSource; 71 | 72 | for (const auto& currLine : lines) 73 | { 74 | inputSource.append(currLine); 75 | } 76 | 77 | StringInputStream stringInputStream(inputSource); 78 | 79 | for (auto& currLine : lines) 80 | { 81 | auto readLine = stringInputStream.ReadLine(); 82 | REQUIRE(readLine == currLine); 83 | } 84 | } 85 | } -------------------------------------------------------------------------------- /tests/tokensOutputStreamTests.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "tcppLibrary.hpp" 3 | #include 4 | 5 | using namespace tcpp; 6 | 7 | 8 | TEST_CASE("TokensOutputStream Tests") 9 | { 10 | const TTokensSequence tokens 11 | { 12 | TToken { E_TOKEN_TYPE::COMMENTARY }, 13 | TToken { E_TOKEN_TYPE::OPEN_BRACKET }, 14 | TToken { E_TOKEN_TYPE::CLOSE_BRACKET }, 15 | TToken { E_TOKEN_TYPE::END }, 16 | }; 17 | 18 | TokensOutputStream stream{ tokens }; 19 | 20 | SECTION("TestBeginEnd_IterateThroughUsingRangeBasedFor_AllElementsVisited") 21 | { 22 | int i = 0; 23 | 24 | for (const TToken& currToken : stream) 25 | { 26 | REQUIRE(currToken.mType == tokens[i++].mType); 27 | } 28 | } 29 | 30 | SECTION("TestGetNextToken_IterateThroughSequence_AllElementsVisited") 31 | { 32 | int i = 0; 33 | 34 | while (stream.HasNextToken()) 35 | { 36 | REQUIRE(stream.GetNextToken().mType == tokens[i++].mType); 37 | } 38 | } 39 | 40 | SECTION("TestGetNextToken_TryToGetNextTokenWhenNoItemsRemain_ReturnsLastElement") 41 | { 42 | int i = 0; 43 | 44 | while (stream.HasNextToken()) 45 | { 46 | REQUIRE(stream.GetNextToken().mType == tokens[i++].mType); 47 | } 48 | 49 | REQUIRE(!stream.HasNextToken()); 50 | 51 | const TToken& outboundsToken = stream.GetNextToken(); 52 | REQUIRE(outboundsToken.mType == E_TOKEN_TYPE::END); 53 | } 54 | 55 | SECTION("TestPeekNextToken_TryToIterateThroughAllElements_AllElementsVisited") 56 | { 57 | size_t i = 0; 58 | 59 | for (const TToken& expectedToken : tokens) 60 | { 61 | REQUIRE(expectedToken.mType == stream.PeekNextToken(i++).mType); 62 | } 63 | } 64 | } --------------------------------------------------------------------------------