├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── CMakeSettings.json ├── LICENSE ├── README.md ├── sqfc.grammar ├── sqfc.lua ├── src ├── commandList.cpp ├── commandList.hpp ├── commandListUnary.cpp ├── compiledCode.hpp ├── logger.cpp ├── logger.hpp ├── luaHandler.cpp ├── luaHandler.hpp ├── lzokay_stream.cpp ├── main.cpp ├── optimizer │ ├── optimizer.cpp │ ├── optimizer.h │ ├── optimizerModuleBase.cpp │ ├── optimizerModuleBase.hpp │ ├── optimizerModuleConstantFold.cpp │ ├── optimizerModuleConstantFold.hpp │ ├── optimizerModuleLua.cpp │ └── optimizerModuleLua.hpp ├── scriptCompiler.cpp ├── scriptCompiler.hpp ├── scriptSerializer.cpp └── scriptSerializer.hpp └── test ├── ace3.json └── cba_a3.json /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test-configs: 7 | name: Archive test configs 8 | runs-on: ubuntu-20.04 9 | steps: 10 | - name: Git checkout 11 | uses: actions/checkout@v2 12 | 13 | - name: Archive test configs 14 | uses: actions/upload-artifact@v2 15 | with: 16 | name: test configs 17 | path: test/*.json 18 | if-no-files-found: error 19 | 20 | # linux-build: 21 | # name: Linux build 22 | # runs-on: ubuntu-22.04 23 | # steps: 24 | # - name: Git checkout 25 | # uses: actions/checkout@v2 26 | # with: 27 | # submodules: true 28 | # 29 | # - name: Install dependencies 30 | # run: | 31 | # sudo add-apt-repository ppa:ubuntu-toolchain-r/test 32 | # sudo apt-get update 33 | # sudo apt-get install -y gcc-13 g++-13 34 | # sudo apt-get remove -y libtbb-dev 35 | # 36 | # - name: Build Linux 64 bit 37 | # run: mkdir build-linux64 && cd build-linux64 && cmake .. && cmake --build . --config Release --parallel 2 38 | # env: 39 | # CC: gcc-13 40 | # CXX: g++-13 41 | # 42 | # - name: Upload Linux 64 bit 43 | # uses: actions/upload-artifact@v2 44 | # with: 45 | # name: Linux x64 46 | # path: release 47 | # if-no-files-found: error 48 | 49 | windows-build: 50 | name: Windows build 51 | runs-on: windows-2019 52 | steps: 53 | - name: Git checkout 54 | uses: actions/checkout@v2 55 | with: 56 | submodules: true 57 | 58 | - name: Build Windows 64 bit 59 | run: mkdir build-win64 && cd build-win64 && cmake -DCMAKE_BUILD_TYPE=Release -G "Visual Studio 16 2019" -A x64 .. && cmake --build . --config Release 60 | 61 | - name: Upload Windows 64 bit 62 | uses: actions/upload-artifact@v2 63 | with: 64 | name: Windows x64 65 | path: Release 66 | if-no-files-found: error 67 | 68 | windows-test-ace3: 69 | name: Windows test ACE3 70 | runs-on: windows-2019 71 | needs: 72 | - test-configs 73 | - windows-build 74 | steps: 75 | - name: Git checkout ACE3 76 | uses: actions/checkout@v2 77 | with: 78 | repository: acemod/ACE3 79 | path: z/ace 80 | 81 | - name: Git checkout CBA A3 82 | uses: actions/checkout@v2 83 | with: 84 | repository: CBATeam/CBA_A3 85 | path: x/cba 86 | 87 | - name: Download test configs 88 | uses: actions/download-artifact@v2 89 | with: 90 | name: test configs 91 | 92 | - name: Download ArmaScriptCompiler artifact 93 | uses: actions/download-artifact@v2 94 | with: 95 | name: Windows x64 96 | 97 | - name: Setup build folder 98 | run: | 99 | xcopy z\ace\include\a3 a3 /s /e /h /i 100 | copy ace3.json sqfc.json 101 | 102 | - name: Compile ACE3 103 | run: | 104 | subst P: . 105 | Release\ArmaScriptCompiler.exe 106 | 107 | windows-test-cba-a3: 108 | name: Windows test CBA A3 109 | runs-on: windows-2019 110 | needs: 111 | - test-configs 112 | - windows-build 113 | steps: 114 | - name: Git checkout CBA A3 115 | uses: actions/checkout@v2 116 | with: 117 | repository: CBATeam/CBA_A3 118 | path: x/cba 119 | 120 | - name: Download test configs 121 | uses: actions/download-artifact@v2 122 | with: 123 | name: test configs 124 | 125 | - name: Download ArmaScriptCompiler artifact 126 | uses: actions/download-artifact@v2 127 | with: 128 | name: Windows x64 129 | 130 | - name: Setup build folder 131 | run: | 132 | xcopy x\cba\include\a3 a3 /s /e /h /i 133 | copy cba_a3.json sqfc.json 134 | 135 | - name: Compile CBA A3 136 | run: | 137 | subst P: . 138 | Release\ArmaScriptCompiler.exe 139 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | 34 | # Debug files 35 | *.dSYM/ 36 | *.su 37 | *.idb 38 | *.pdb 39 | 40 | # Build files 41 | build/ 42 | 43 | [Bb][Uu][Ii][Ll][Dd]/ 44 | 45 | #CMake 46 | CMakeLists.txt.user 47 | CMakeCache.txt 48 | CMakeFiles 49 | CMakeScripts 50 | Testing 51 | Makefile 52 | cmake_install.cmake 53 | install_manifest.txt 54 | compile_commands.json 55 | CTestTestfile.cmake 56 | _deps 57 | 58 | 59 | #VS code files 60 | \.vs/ 61 | *.vscode/ 62 | !.vscode/settings.json 63 | !.vscode/tasks.json 64 | !.vscode/launch.json 65 | !.vscode/extensions.json 66 | !.vscode/*.code-snippets 67 | 68 | # Local History for Visual Studio Code 69 | .history/ 70 | 71 | sqfc.json 72 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "lib/zstd"] 2 | path = lib/zstd 3 | url = https://github.com/facebook/zstd.git 4 | [submodule "lib/sqfvm"] 5 | path = lib/sqfvm 6 | url = https://github.com/SQFvm/vm.git 7 | [submodule "lib/cpp-base64"] 8 | path = lib/cpp-base64 9 | url = https://github.com/dedmen/cpp-base64.git 10 | [submodule "lib/lzokay"] 11 | path = lib/lzokay 12 | url = https://github.com/jackoalan/lzokay.git 13 | [submodule "lib/json"] 14 | path = lib/json 15 | url = https://github.com/azadkuh/nlohmann_json_release.git 16 | [submodule "lib/sol2"] 17 | path = lib/sol2 18 | url = https://github.com/ThePhD/sol2.git 19 | [submodule "lib/lua"] 20 | path = lib/lua 21 | url = https://github.com/walterschell/Lua.git 22 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.13) 2 | if (${CMAKE_HOST_SYSTEM_NAME} STREQUAL "Windows") 3 | set (CMAKE_SYSTEM_VERSION 8.1 CACHE TYPE INTERNAL FORCE) #Force 8.1 SDK, to keep it compatible with win7 4 | endif() 5 | project (ArmaScriptCompiler CXX) 6 | find_package (Threads) 7 | 8 | 9 | if(MSVC) 10 | set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /std:c++latest" ) 11 | #elseif(MSVC) 12 | # message(FATAL_ERROR "ERROR: You need a C++17 compatible compiler") 13 | endif() 14 | 15 | message("GENERATOR USED: '${CMAKE_GENERATOR}'") 16 | message("COMPILER USED: '${CMAKE_CXX_COMPILER_ID}'") 17 | 18 | set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${PROJECT_SOURCE_DIR}/release") 19 | 20 | set(CMAKE_CXX_STANDARD 20) 21 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 22 | set(CMAKE_CXX_EXTENSIONS OFF) 23 | 24 | SET(CMAKE_INCLUDE_CURRENT_DIR ON) 25 | set_property(GLOBAL PROPERTY USE_FOLDERS ON) 26 | 27 | set(CMAKE_SUPPRESS_REGENERATION true) 28 | set(CMAKE_CONFIGURATION_TYPES "Debug;Release" CACHE STRING "" FORCE) 29 | 30 | 31 | set(LIBRARY_PATH_ZSTD "${CMAKE_SOURCE_DIR}/lib/zstd") 32 | set(LIBRARY_PATH_SQFVM "${CMAKE_SOURCE_DIR}/lib/sqfvm") 33 | set(LIBRARY_PATH_B64 "${CMAKE_SOURCE_DIR}/lib/cpp-base64") 34 | set(LIBRARY_PATH_LZO "${CMAKE_SOURCE_DIR}/lib/lzokay") 35 | set(LIBRARY_PATH_JSON "${CMAKE_SOURCE_DIR}/lib/json") 36 | set(LIBRARY_PATH_SOL2 "${CMAKE_SOURCE_DIR}/lib/sol2") 37 | 38 | add_definitions(/DNOMINMAX) 39 | add_definitions(/D_SILENCE_ALL_CXX17_DEPRECATION_WARNINGS) #No I don't cheat! 40 | 41 | #don't ask me 42 | 43 | if(NOT CMAKE_C_COMPILE_OBJECT) 44 | set(CMAKE_C_COMPILE_OBJECT 45 | " -o -c ") 46 | endif() 47 | 48 | 49 | 50 | 51 | 52 | 53 | file(GLOB_RECURSE SOURCES_ASC "src/*.hpp" "src/*.cpp" "${LIBRARY_PATH_B64}/base64.cpp") 54 | SOURCE_GROUP("src" FILES ${SOURCES_ASC}) 55 | 56 | file(GLOB_RECURSE SOURCES_ASC_OPT "src/optimizer/*.hpp" "src/optimizer/*.cpp") 57 | SOURCE_GROUP("src" FILES ${SOURCES_ASC_OPT}) 58 | 59 | #zstd 60 | SET(ZSTD_BUILD_PROGRAMS OFF CACHE BOOL "no programs" FORCE) 61 | set(ZSTD_USE_STATIC_RUNTIME ON CACHE BOOL "yes" FORCE) 62 | set(ZSTD_BUILD_SHARED OFF CACHE BOOL "no" FORCE) 63 | 64 | add_subdirectory("${LIBRARY_PATH_ZSTD}/build/cmake") 65 | 66 | set(INCLUDE_PATH_ZSTD "${LIBRARY_PATH_ZSTD}/lib") 67 | 68 | #sqfvm 69 | #Adding manually because I don't want commands 70 | 71 | #include_directories("${LIBRARY_PATH_SQFVM}/include/json/include") 72 | #include_directories("${LIBRARY_PATH_SQFVM}/include/tclap-1.2.2/include") 73 | # 74 | #file(GLOB SOURCES_SQFVM 75 | # "${LIBRARY_PATH_SQFVM}/src/*.h" "${LIBRARY_PATH_SQFVM}/src/*.cpp" "${LIBRARY_PATH_SQFVM}/src/*.c" 76 | #) 77 | #SOURCE_GROUP("sqf_vm" FILES ${SOURCES_SQFVM}) 78 | # 79 | #set(INCLUDE_PATH_SQFVM "${LIBRARY_PATH_SQFVM}/src") 80 | # 81 | # 82 | # 83 | #list(REMOVE_ITEM SOURCES_SQFVM "${LIBRARY_PATH_SQFVM}/src/Entry.cpp") 84 | #list(REMOVE_ITEM SOURCES_SQFVM "${LIBRARY_PATH_SQFVM}/src/dllexports.cpp") 85 | 86 | 87 | set(SQFVM_BUILD_EXECUTABLE OFF CACHE BOOL "no" FORCE) 88 | set(SQFVM_BUILD_EXECUTABLE_ARMA2_LOCALKEYWORD OFF CACHE BOOL "no" FORCE) 89 | set(SQFVM_BUILD_LIBRARY OFF CACHE BOOL "no" FORCE) 90 | set(SQFVM_BUILD_LIBRARY_SQC_SUPPORT OFF CACHE BOOL "no" FORCE) 91 | set(SQFVM_BUILD_STATIC_LIBRARY ON CACHE BOOL "yes" FORCE) 92 | set(SQFVM_BUILD_STATIC_LIBRARY_SQC_SUPPORT OFF CACHE BOOL "no" FORCE) 93 | set(SQFVM_BUILD_EXECUTABLE_SQC_SUPPORT OFF CACHE BOOL "no" FORCE) 94 | set(SQFVM_BUILD_EXECUTABLE_FULL_DIAGNOSE OFF CACHE BOOL "no" FORCE) 95 | set(SQFVM_BUILD_EXECUTABLE_ARMA2_LOCALKEYWORD_FULL_DIAGNOSE OFF CACHE BOOL "no" FORCE) 96 | 97 | set(INCLUDE_PATH_SQFVM "${LIBRARY_PATH_SQFVM}/src") 98 | add_subdirectory(${LIBRARY_PATH_SQFVM}) 99 | add_subdirectory(${LIBRARY_PATH_SOL2}) 100 | 101 | set(LUA_BUILD_AS_CXX ON CACHE BOOL "yes" FORCE) 102 | add_subdirectory("lib/lua") 103 | add_definitions(/DLOADFILE_CACHE) #SQF VM fileio.cpp cache 104 | 105 | 106 | 107 | add_executable("ArmaScriptCompiler" ${SOURCES_ASC} ${SOURCES_SQFVM} ${SOURCES_ASC_OPT} "${LIBRARY_PATH_LZO}/lzokay.cpp") 108 | 109 | include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${INCLUDE_PATH_ZSTD} ${LIBRARY_PATH_B64} ${INCLUDE_PATH_SQFVM} ${LIBRARY_PATH_LZO} ${LIBRARY_PATH_JSON}) # ${INCLUDE_PATH_ZSTD} ${INCLUDE_PATH_SQFVM} 110 | 111 | #target_link_libraries("ArmaScriptCompiler" ${CMAKE_THREAD_LIBS_INIT}) 112 | 113 | target_link_libraries("ArmaScriptCompiler" slibsqfvm libzstd_static sol2 lua_static) 114 | 115 | set_target_properties("ArmaScriptCompiler" PROPERTIES PREFIX "") 116 | set_target_properties("ArmaScriptCompiler" PROPERTIES FOLDER ArmaScriptCompiler) 117 | 118 | 119 | 120 | target_compile_definitions("ArmaScriptCompiler" PUBLIC SOL_ALL_SAFETIES_ON=1 SOL_PRINT_ERRORS=1 SOL_USING_CXX_LUA=1) 121 | 122 | if(CMAKE_COMPILER_IS_GNUCXX) 123 | add_compile_definitions(__linux__) 124 | 125 | target_compile_options( 126 | "ArmaScriptCompiler" PRIVATE 127 | "-std=c++1z" 128 | "$<$:-O3>" 129 | "-s" 130 | "-fPIC" 131 | "-fpermissive" 132 | "-static-libgcc" 133 | "-static-libstdc++" 134 | "-Wno-ignored-attributes" 135 | "-static" 136 | "$<$:-ffunction-sections>" 137 | "$<$:-fdata-sections>" 138 | 139 | ) 140 | target_link_options("ArmaScriptCompiler" PRIVATE "-fPIC" "-static" "-static-libgcc" "-static-libstdc++" 141 | "$<$:-Wl,--gc-sections>" 142 | "$<$:-Wl,--strip-all>" 143 | ) 144 | set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") 145 | else() 146 | target_compile_options(lua_static PRIVATE "$<$:/MT>") 147 | target_compile_options( 148 | "ArmaScriptCompiler" PRIVATE 149 | "/MP" "/Zi" 150 | "$<$:/MT>" 151 | "$<$:/Ox>" 152 | "$<$:/Ob2>" 153 | "$<$:/Oi>" 154 | "$<$:/Ot>" 155 | "$<$:/GL>" 156 | "/F4194304" #Stack size 157 | "/FS" 158 | "/bigobj" 159 | ) 160 | 161 | target_compile_options( 162 | slibsqfvm PRIVATE 163 | "/MP" "/Zi" 164 | "$<$:/MT>" 165 | "$<$:/Ox>" 166 | "$<$:/Ob2>" 167 | "$<$:/Oi>" 168 | "$<$:/Ot>" 169 | "$<$:/GL>" 170 | "/DLOADFILE_CACHE" 171 | ) 172 | target_link_options("ArmaScriptCompiler" PRIVATE "/OPT:REF" "/OPT:ICF" "/DEBUG:FULL" "/LTCG") 173 | target_link_options(slibsqfvm PRIVATE "/OPT:REF" "/OPT:ICF" "/DEBUG:FULL" "/LTCG") 174 | endif() 175 | 176 | #Binary signing 177 | #if(EXISTS "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/sign.bat" AND MSVC) 178 | # add_custom_command(TARGET ${INTERCEPT_PLUGIN_NAME} 179 | # POST_BUILD 180 | # COMMAND ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/sign.bat 181 | # WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY} 182 | # ) 183 | #endif() 184 | -------------------------------------------------------------------------------- /CMakeSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": [ 3 | { 4 | "name": "x64-Release", 5 | "generator": "Ninja", 6 | "configurationType": "RelWithDebInfo", 7 | "buildRoot": "${env.USERPROFILE}\\CMakeBuilds\\${workspaceHash}\\build\\${name}", 8 | "installRoot": "${env.USERPROFILE}\\CMakeBuilds\\${workspaceHash}\\install\\${name}", 9 | "cmakeCommandArgs": "", 10 | "buildCommandArgs": "-v", 11 | "ctestCommandArgs": "", 12 | "inheritEnvironments": [ "msvc_x64_x64" ], 13 | "variables": [] 14 | }, 15 | { 16 | "name": "x64-Debug", 17 | "generator": "Ninja", 18 | "configurationType": "Debug", 19 | "buildRoot": "${env.USERPROFILE}\\CMakeBuilds\\${workspaceHash}\\build\\${name}", 20 | "installRoot": "${env.USERPROFILE}\\CMakeBuilds\\${workspaceHash}\\install\\${name}", 21 | "cmakeCommandArgs": "", 22 | "buildCommandArgs": "-v", 23 | "ctestCommandArgs": "", 24 | "inheritEnvironments": [ "msvc_x64_x64" ], 25 | "variables": [] 26 | } 27 | ] 28 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ArmaScriptCompiler 2 | 3 | ## Config 4 | 5 | Create `sqfc.json` in current working directory 6 | 7 | ```json 8 | { 9 | "inputDirs": [ 10 | "T:/x/", 11 | "T:/z/ace/", 12 | "T:/z/acex/", 13 | "T:/a3/" 14 | ], 15 | "includePaths": [ 16 | "T:/" 17 | ], 18 | "excludeList": [ 19 | "missions_f_contact", 20 | "missions_f_epa", 21 | "missions_f_oldman", 22 | "missions_f_tank", 23 | "missions_f_beta", 24 | "showcases\\showcase", 25 | "\\unitplay\\", 26 | "\\backups\\" 27 | ], 28 | "outputDir": "P:/", 29 | "workerThreads": 8, 30 | "rootPathMappings": [ 31 | ["T:/", "\\"] 32 | ], 33 | "logging": { 34 | "verbose": false 35 | } 36 | } 37 | ``` 38 | -------------------------------------------------------------------------------- /sqfc.grammar: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | -------------------------------------------------------------------------------- /sqfc.lua: -------------------------------------------------------------------------------- 1 | ---@diagnostic disable: lowercase-global 2 | local inspect = require 'inspect' -- https://github.com/kikito/inspect.lua 3 | print("Config loading") 4 | 5 | 6 | function ScanDirectory(directory) 7 | local dirIterator = DirectoryIterator.new(directory); 8 | 9 | local currentFile = dirIterator:begin(); 10 | local sqfExt = path.new(".sqf") 11 | local gitExt = path.new(".git") 12 | local svnExt = path.new(".svn") 13 | 14 | while not currentFile:is_end() do 15 | local file = currentFile:get(); 16 | 17 | if file:is_directory() and ( 18 | file:path():filename() == gitExt 19 | or file:path():filename() == svnExt 20 | or string.sub(tostring(file:path():filename()), 0,3) == "map" -- skip all the map layer files on full pdrive 21 | ) then 22 | print("skip dir", file) 23 | currentFile:disable_recursion_pending(); -- Don't recurse into that directory 24 | end 25 | 26 | if file:is_regular_file() and file:path():filename():extension() == sqfExt then 27 | 28 | -- Here filter our SQF files any way we want 29 | 30 | ASC:AddCompileTask(file:path()) 31 | end 32 | 33 | currentFile:next(); 34 | end 35 | end 36 | 37 | -- Called when a compiler is created, usually done once per worker thread 38 | function SetupCompiler(scriptCompiler) 39 | print("Setup compiler") 40 | 41 | 42 | -- init include file paths 43 | scriptCompiler:InitIncludePaths({path.new("T:/")}); 44 | 45 | -- example of how to define a simple macro 46 | -- #define MY_MACRO_EMPTY 47 | scriptCompiler:AddMacro(Macro.new("MY_MACRO_EMPTY")); 48 | 49 | -- #define MY_MACRO blabla 50 | scriptCompiler:AddMacro(Macro.new("MY_MACRO", blabla)); 51 | -- #define MY_MACRO_ARGS(arg1, arg2) arg1##_##arg2 52 | scriptCompiler:AddMacro(Macro.new("MY_MACRO_ARGS", {"arg1", "arg2"}, "arg1##_##arg2")); 53 | -- custom handler function, returns the final macro replacement text 54 | scriptCompiler:AddMacro(Macro.new("MY_MACRO_CUSTOM", 55 | function(diagInfo) 56 | return "testy" 57 | end 58 | )); 59 | -- custom handler function, returns the final macro replacement text, with parameters 60 | -- In this example equivalent to 61 | -- #define MY_MACRO_CUSTOM_ARGS(arg1,arg2) arg1 is at __LINE__ with arg2 62 | scriptCompiler:AddMacro(Macro.new("MY_MACRO_CUSTOM_ARGS", {"arg1", "arg2"}, 63 | function(parameters, diagInfo) 64 | return parameters[1] .. " is at " .. diagInfo.line .. " with " .. parameters[2] 65 | end 66 | )); 67 | -- custom pragma handler 68 | -- #pragma myPragma ... 69 | scriptCompiler:AddPragma(Pragma.new("myPragma", 70 | function(data, diagInfo) 71 | return "" 72 | end 73 | )); 74 | 75 | 76 | end 77 | 78 | --print("ASC meta:", inspect(getmetatable(ASC))) 79 | --ScanDirectory("P:/test"); 80 | --ASC:SetOutputDir(path.new("P:/")) 81 | --ASC:RunCompileTasks(1) 82 | print("Config loaded") 83 | 84 | 85 | local compiler = ScriptCompiler.new() 86 | 87 | -- custom handler function, returns the final macro replacement text, with parameters 88 | -- In this example equivalent to 89 | -- #define MY_MACRO_CUSTOM_ARGS(arg1,arg2) arg1 is at __LINE__ with arg2 90 | compiler:AddMacro(Macro.new("MY_MACRO_CUSTOM_ARGS", {"arg1", "arg2"}, 91 | function(parameters, diagInfo) 92 | --print("parameters:", inspect(parameters)) 93 | --print("parameters:", inspect(getmetatable(parameters))) 94 | --print("diagInfo:", inspect(getmetatable(diagInfo))) 95 | return parameters[1] .. " is at " .. diagInfo.line .. " with " .. parameters[2] 96 | end 97 | )); 98 | 99 | -- print("Preprocessed Script: \n", compiler:PreprocessFile(path.new("P:\\test.sqf"))); 100 | 101 | 102 | function optimizerNodeHandler(node) 103 | 104 | 105 | print("node meta:", inspect(getmetatable(node))) 106 | print("node:", inspect(node)) 107 | print("nodef:", node.file) 108 | print("nodel:", node.line) 109 | print("nodet:", node.type) 110 | print("nodec:", node.constant) 111 | print("nodev:", node.value) 112 | 113 | if node.type == InstructionType.callUnary and node:areChildrenConstant() and node.value=="params" then 114 | print("Params!") 115 | -- We are calling params script command and the array argument consists of only constants, we can safely optimize the array to a push instruction 116 | 117 | function resolveMakeArray(node) 118 | print("resolve", node.type) 119 | if (node.type ~= InstructionType.makeArray) then return end 120 | 121 | for i,v in ipairs(node.children) do 122 | resolveMakeArray(v) 123 | end 124 | node.value = ScriptConstantArray.new(); --dummy. Children are the contents 125 | node.type = InstructionType.push; 126 | end 127 | 128 | 129 | print("preresolve", #node.children, resolveMakeArray) 130 | resolveMakeArray(node.children[1]) 131 | 132 | --#TODO verify again that they are all push or makeArray instructions 133 | 134 | node.children[1].value = ScriptConstantArray.new() --dummy. Children are the contents 135 | node.children[1].type = InstructionType.push 136 | print("Params optimized!") 137 | end 138 | 139 | 140 | -- We need to figure out what things we consider as constants 141 | -- Push and end statement are always constants 142 | if node.type == InstructionType.push or node.type == InstructionType.endStatement then 143 | node.constant = true 144 | end 145 | 146 | 147 | function isArrayConst(node) 148 | for i,v in ipairs(node.children) do 149 | if not v.constant or v.type ~= InstructionType.push then 150 | return false 151 | end 152 | end 153 | return true 154 | end 155 | 156 | -- makeArray is only const if all children are makeArray or push 157 | if node.type == InstructionType.makeArray then 158 | node.constant = isArrayConst(node); 159 | end 160 | 161 | end 162 | 163 | local optimizer = OptimizerModuleLua.new(optimizerNodeHandler) 164 | 165 | 166 | --compiler:CompileScriptToFile(path.new("P:\\test.sqf"), path.new("P:/test.asm"), optimizer) 167 | compiler:CompileScriptToFile(path.new("T:\\z\\ace\\addons\\common\\functions\\fnc_cbaSettings_loadFromConfig.sqf"), path.new("P:/test.asm"), optimizer) 168 | 169 | -------------------------------------------------------------------------------- /src/commandList.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "runtime/runtime.h" 3 | 4 | class CommandList { 5 | public: 6 | static void initNular(::sqf::runtime::runtime& runtime); 7 | static void initUnary(::sqf::runtime::runtime& runtime); 8 | static void initBinary(::sqf::runtime::runtime& runtime); 9 | 10 | 11 | static void init(::sqf::runtime::runtime& runtime) { 12 | initNular(runtime); 13 | initUnary(runtime); 14 | initBinary(runtime); 15 | } 16 | }; 17 | -------------------------------------------------------------------------------- /src/compiledCode.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | using namespace std::string_view_literals; 8 | 9 | /* 10 | endStatement 11 | push 12 | callUnary 13 | callBinary 14 | assignTo 15 | assignToLocal 16 | callNular 17 | getVariable 18 | makeArray 19 | */ 20 | 21 | #ifndef ASC_INTERCEPT 22 | #define STRINGTYPE std::string 23 | #else 24 | #define STRINGTYPE intercept::types::r_string 25 | #include 26 | #endif 27 | 28 | #if not defined(_MSC_VER) 29 | #define __forceinline __attribute__((always_inline)) 30 | #include 31 | #define __debugbreak() raise(SIGTRAP) 32 | #endif 33 | 34 | enum class InstructionType { 35 | endStatement, 36 | push, 37 | callUnary, 38 | callBinary, 39 | callNular, 40 | assignTo, 41 | assignToLocal, 42 | getVariable, 43 | makeArray 44 | }; 45 | 46 | 47 | static std::string_view instructionTypeToString(InstructionType type) { 48 | switch (type) { 49 | case InstructionType::endStatement: return "endStatement"sv; 50 | case InstructionType::push: return "push"sv; 51 | case InstructionType::callUnary: return "callUnary"sv; 52 | case InstructionType::callBinary: return "callBinary"sv; 53 | case InstructionType::callNular: return "callNular"sv; 54 | case InstructionType::assignTo: return "assignTo"sv; 55 | case InstructionType::assignToLocal: return "assignToLocal"sv; 56 | case InstructionType::getVariable: return "getVariable"sv; 57 | case InstructionType::makeArray: return "makeArray"sv; 58 | default: __debugbreak(); 59 | } 60 | } 61 | 62 | 63 | struct ScriptInstruction { 64 | InstructionType type; 65 | size_t offset; 66 | uint8_t fileIndex; 67 | size_t line; 68 | //content string, or constant index 69 | std::variant content; 70 | }; 71 | 72 | enum class ConstantType { 73 | code, 74 | string, 75 | scalar, 76 | boolean, 77 | array, 78 | nularCommand 79 | }; 80 | 81 | struct ScriptCodePiece { 82 | std::vector code; 83 | union { 84 | struct { 85 | unsigned offset : 32; 86 | unsigned length : 31; 87 | unsigned isOffset : 1; 88 | } contentSplit; 89 | uint64_t contentString; //pointer to constants 90 | }; 91 | ScriptCodePiece(std::vector&& c, uint32_t length, uint32_t offset) : code(c) { 92 | contentSplit.isOffset = 1; 93 | if (length > 0x60'00'00'00) 94 | __debugbreak(); 95 | contentSplit.length = length; 96 | contentSplit.offset = offset; 97 | } 98 | ScriptCodePiece(std::vector&& c, uint64_t content) : code(c), contentString(content) {} 99 | //ScriptCodePiece(ScriptCodePiece&& o) noexcept : code(std::move(o.code)), contentString(o.contentString) {} 100 | //ScriptCodePiece(const ScriptCodePiece& o) noexcept : code(o.code), contentString(o.contentString) {} 101 | // 102 | //ScriptCodePiece& operator=(ScriptCodePiece&& o) noexcept { 103 | // code = std::move(o.code); 104 | // contentString = o.contentString; 105 | // return *this; 106 | //} 107 | //ScriptCodePiece& operator=(const ScriptCodePiece& o) noexcept { 108 | // code = o.code; 109 | // contentString = o.contentString; 110 | // return *this; 111 | //} 112 | 113 | ScriptCodePiece(): contentString(0) {} 114 | bool operator==(const ScriptCodePiece& other) const { 115 | 116 | // the content string/offset will be different for multiple empty pieces of code, but we don't care because empty code piece won't throw errors so doesn't matter that its wrong 117 | if (code.empty() && other.code.empty()) return true; 118 | //#TODO actually compare code contents? 119 | 120 | return false; 121 | } 122 | 123 | }; 124 | 125 | struct ScriptConstantNularCommand { 126 | STRINGTYPE commandName; 127 | ScriptConstantNularCommand(STRINGTYPE command) : commandName(command) { 128 | std::transform(commandName.begin(), commandName.end(), commandName.begin(), ::tolower); 129 | } 130 | }; 131 | 132 | 133 | struct ScriptConstantArray; 134 | 135 | using ScriptConstant = std::variant; 136 | 137 | struct ScriptConstantArray { 138 | std::vector content; 139 | bool operator==(const ScriptConstantArray& other) const; 140 | }; 141 | 142 | constexpr ConstantType getConstantType(const ScriptConstant& c) { 143 | switch (c.index()) { 144 | case 0: return ConstantType::code; 145 | case 1: return ConstantType::string; 146 | case 2: return ConstantType::scalar; 147 | case 3: return ConstantType::boolean; 148 | case 4: return ConstantType::array; 149 | case 5: return ConstantType::nularCommand; 150 | } 151 | __debugbreak(); 152 | } 153 | 154 | 155 | inline bool operator==(const ScriptConstant& left, const ScriptConstant& right) { 156 | if (left.index() != right.index()) return false; 157 | switch (getConstantType(left)) { 158 | case ConstantType::code: return std::get(left) == std::get(right); break; 159 | case ConstantType::string: return std::get(left) == std::get(right); 160 | case ConstantType::scalar: return std::get(left) == std::get(right); 161 | case ConstantType::boolean: return std::get(left) == std::get(right); 162 | case ConstantType::array:return std::get(left) == std::get(right); 163 | case ConstantType::nularCommand:return std::get(left).commandName == std::get(right).commandName; 164 | } 165 | 166 | return false; 167 | } 168 | 169 | inline bool ScriptConstantArray::operator==(const ScriptConstantArray& other) const { 170 | if (content.size() != other.content.size()) return false; 171 | 172 | return std::equal(content.begin(), content.end(), other.content.begin(), other.content.end(), 173 | [](const ScriptConstant& left, const ScriptConstant& right) 174 | { 175 | return left == right; 176 | }); 177 | } 178 | 179 | struct CompiledCodeData { 180 | uint32_t version{1}; 181 | uint64_t codeIndex; //index to main code in constants 182 | std::vector constants; 183 | std::vector fileNames; 184 | 185 | #ifdef ASC_INTERCEPT 186 | std::vector builtConstants; 187 | #endif 188 | 189 | // temporary for serialization 190 | mutable std::vector commandNameDirectory; 191 | 192 | uint16_t getIndexFromCommandNameDirectory(std::string_view text) const { 193 | 194 | auto found = std::lower_bound(commandNameDirectory.begin(), commandNameDirectory.end(), text); 195 | if (found == commandNameDirectory.end()) 196 | { 197 | __debugbreak(); 198 | } 199 | 200 | return std::distance(commandNameDirectory.begin(), found); 201 | } 202 | 203 | 204 | uint64_t AddConstant(ScriptConstant&& constant) { 205 | auto found = std::find_if(std::execution::par_unseq, constants.begin(), constants.end(), [&constant](const ScriptConstant& cnst) { 206 | return cnst == constant; 207 | }); 208 | if (found == constants.end()) { 209 | constants.emplace_back(std::move(constant)); 210 | return constants.size() - 1; 211 | } 212 | 213 | return std::distance(constants.begin(), found); 214 | } 215 | 216 | uint64_t AddConstant(const ScriptConstant& constant) { 217 | auto found = std::find_if(std::execution::par_unseq, constants.begin(), constants.end(), [&constant](const ScriptConstant& cnst) { 218 | return cnst == constant; 219 | }); 220 | if (found == constants.end()) { 221 | constants.emplace_back(constant); 222 | return constants.size() - 1; 223 | } 224 | 225 | return std::distance(constants.begin(), found); 226 | } 227 | 228 | //#TODO compress constants, don't have duplicates for a number or string 229 | }; 230 | 231 | template 232 | class Singleton { 233 | Singleton(const Singleton&) = delete; 234 | Singleton(Singleton&&) = delete; 235 | Singleton& operator=(const Singleton&) = delete; 236 | Singleton& operator=(Singleton&&) = delete; 237 | public: 238 | static __forceinline T& get() noexcept { 239 | return _singletonInstance; 240 | } 241 | static void release() { 242 | } 243 | protected: 244 | Singleton() noexcept {} 245 | static T _singletonInstance; 246 | static bool _initialized; 247 | }; 248 | template 249 | T Singleton::_singletonInstance; 250 | template 251 | bool Singleton::_initialized = false; -------------------------------------------------------------------------------- /src/logger.cpp: -------------------------------------------------------------------------------- 1 | #include "logger.hpp" 2 | #include 3 | #include 4 | 5 | 6 | std::mutex loggerMutex; 7 | 8 | void LoggerBogger::Log(LogLevel level, const std::string& msg) 9 | { 10 | if (level == LogLevel::Verbose && !m_EnableVerbose) 11 | return; 12 | 13 | std::lock_guard m(loggerMutex); 14 | 15 | std::cout << msg.c_str() << "\n"; 16 | } 17 | -------------------------------------------------------------------------------- /src/logger.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | enum class LogLevel 5 | { 6 | Normal, 7 | Verbose 8 | }; 9 | 10 | class LoggerBogger // Weird name because SQF-VM also has class Logger, without namespace (most of its stuff is in namespace, just not that) 11 | { 12 | public: 13 | //#TODO in-place formatting, vsnprintf 14 | //#TODO multithreading queue instead of locking 15 | 16 | //! This function might be hit by multiple threads 17 | void Log(LogLevel level, const std::string& msg); 18 | 19 | bool m_EnableVerbose = false; 20 | }; 21 | 22 | inline LoggerBogger GLogger; -------------------------------------------------------------------------------- /src/luaHandler.cpp: -------------------------------------------------------------------------------- 1 | #include "luaHandler.hpp" 2 | 3 | #include 4 | #include 5 | 6 | 7 | 8 | #include "scriptCompiler.hpp" 9 | 10 | #include "sol/sol.hpp" 11 | extern std::queue tasks; 12 | extern void processFile(ScriptCompiler& comp, std::filesystem::path path); 13 | extern std::filesystem::path outputDir; 14 | 15 | 16 | 17 | 18 | 19 | class DirectoryIterator { 20 | 21 | std::filesystem::path directory; 22 | public: 23 | 24 | typedef std::filesystem::recursive_directory_iterator iterator; 25 | typedef typename std::filesystem::recursive_directory_iterator::value_type value_type; 26 | 27 | 28 | DirectoryIterator(std::string path) { 29 | directory = path; 30 | } 31 | std::filesystem::recursive_directory_iterator begin() const { 32 | return std::filesystem::recursive_directory_iterator(directory, std::filesystem::directory_options::follow_directory_symlink); 33 | } 34 | 35 | std::filesystem::recursive_directory_iterator end() const { 36 | return std::filesystem::recursive_directory_iterator(); 37 | } 38 | }; 39 | 40 | 41 | namespace sol { 42 | template <> 43 | struct is_container : std::true_type {}; 44 | template <> 45 | struct is_container : std::false_type {}; 46 | 47 | template <> 48 | struct is_automagical : std::false_type {}; 49 | 50 | template <> 51 | struct is_automagical : std::false_type {}; 52 | 53 | template <> 54 | struct is_automagical : std::false_type {}; 55 | } 56 | 57 | 58 | struct LuaASC { 59 | 60 | void AddCompileTask(std::filesystem::path path) { 61 | tasks.emplace(path); 62 | } 63 | 64 | void SetOutputDir(std::filesystem::path path) { 65 | outputDir = path; 66 | } 67 | 68 | auto IterateDirectory(std::string path) { 69 | return sol::as_container(DirectoryIterator(path)); 70 | } 71 | 72 | void RunCompileTasks(int numberOfWorkerThreads) { 73 | std::mutex taskMutex; 74 | bool threadsShouldRun = true; 75 | 76 | auto workerFunc = [&]() { 77 | ScriptCompiler compiler; 78 | GLuaHandler.SetupCompiler(compiler); 79 | 80 | while (threadsShouldRun) { 81 | std::unique_lock lock(taskMutex); 82 | if (tasks.empty()) return; 83 | const auto task(std::move(tasks.front())); 84 | tasks.pop(); 85 | if (tasks.empty()) 86 | threadsShouldRun = false; 87 | lock.unlock(); 88 | 89 | 90 | //auto foundExclude = std::find_if(excludeList.begin(), excludeList.end(), [&task](const std::string& excludeItem) 91 | // { 92 | // auto taskString = task.string(); 93 | // std::transform(taskString.begin(), taskString.end(), taskString.begin(), ::tolower); 94 | // return taskString.find(excludeItem) != std::string::npos; 95 | // }); 96 | // 97 | //if (foundExclude == excludeList.end()) 98 | processFile(compiler, task); 99 | } 100 | 101 | }; 102 | std::vector workerThreads; 103 | for (int i = 0; i < numberOfWorkerThreads-1; i++) { 104 | workerThreads.push_back(std::thread(workerFunc)); 105 | } 106 | 107 | workerFunc(); 108 | 109 | for (std::thread& thread : workerThreads) { 110 | thread.join(); 111 | } 112 | } 113 | }; 114 | 115 | 116 | 117 | LuaHandler::LuaHandler() { 118 | 119 | lua.open_libraries( 120 | // print, assert, and other base functions 121 | sol::lib::base, 122 | // require and other package functions 123 | sol::lib::package, 124 | // coroutine functions and utilities 125 | sol::lib::coroutine, 126 | // string library 127 | sol::lib::string, 128 | // functionality from the OS 129 | sol::lib::os, 130 | // all things math 131 | sol::lib::math, 132 | // the table manipulator and observer functions 133 | sol::lib::table, 134 | // the debug library 135 | sol::lib::debug, 136 | // the bit library: different based on which you're using 137 | sol::lib::bit32, 138 | // input/output library 139 | sol::lib::io, 140 | // library for handling utf8: new to Lua 141 | sol::lib::utf8); 142 | 143 | 144 | auto DirIterType = lua.new_usertype( 145 | "DirectoryIterator", sol::constructors(), 146 | "begin", &DirectoryIterator::begin 147 | ); 148 | 149 | lua.new_usertype( 150 | "recursive_directory_iterator", sol::no_constructor, 151 | "disable_recursion_pending", &std::filesystem::recursive_directory_iterator::disable_recursion_pending, 152 | "next", [](std::filesystem::recursive_directory_iterator& iter) 153 | { 154 | ++iter; 155 | }, 156 | "is_end", [](std::filesystem::recursive_directory_iterator& iter) 157 | { 158 | return iter == std::filesystem::recursive_directory_iterator(); 159 | }, 160 | "get", &std::filesystem::recursive_directory_iterator::operator*, 161 | "depth", &std::filesystem::recursive_directory_iterator::depth, 162 | "options", &std::filesystem::recursive_directory_iterator::options 163 | 164 | 165 | ); 166 | 167 | lua.new_usertype( 168 | "directory_entry", sol::no_constructor, 169 | "is_block_file", static_cast(&std::filesystem::directory_entry::is_block_file), 170 | "is_directory", static_cast(&std::filesystem::directory_entry::is_directory), 171 | "is_regular_file", static_cast(&std::filesystem::directory_entry::is_regular_file), 172 | "is_character_file", static_cast(&std::filesystem::directory_entry::is_character_file), 173 | "is_symlink", static_cast(&std::filesystem::directory_entry::is_symlink), 174 | "last_write_time", static_cast(&std::filesystem::directory_entry::last_write_time), 175 | "path", &std::filesystem::directory_entry::path 176 | ); 177 | 178 | sol::automagic_enrollments enrollments; 179 | 180 | enrollments.pairs_operator = false; 181 | //{ 182 | // false, false, false, true, true, true, true, true, true 183 | //}; 184 | 185 | auto pathType = lua.new_usertype( 186 | "path", sol::constructors(), 187 | "generic_string", [](const std::filesystem::path& path) -> std::string 188 | { 189 | return path.generic_string(); 190 | }, 191 | "__tostring", static_cast(&std::filesystem::path::generic_string), 192 | "__eq", &sol::detail::comparsion_operator_wrap>, 193 | "__le", &sol::detail::comparsion_operator_wrap>, 194 | "__lt", &sol::detail::comparsion_operator_wrap>, 195 | 196 | //"__eq", sol::overload([](const std::filesystem::path& l, const std::filesystem::path& r) 197 | //{ 198 | // return l == r; 199 | //}, [](const std::filesystem::path& l, const std::string& r) 200 | //{ 201 | // return l == r; 202 | //}), 203 | "filename", &std::filesystem::path::filename, 204 | "extension", &std::filesystem::path::extension, 205 | "lexically_normal", &std::filesystem::path::lexically_normal, 206 | "parent_path", &std::filesystem::path::parent_path, 207 | "root_directory", &std::filesystem::path::root_directory, 208 | "root_name", &std::filesystem::path::root_name, 209 | "root_path", &std::filesystem::path::root_path 210 | ); 211 | 212 | lua.new_usertype( 213 | "ASC", sol::default_constructor, 214 | 215 | "AddCompileTask", &LuaASC::AddCompileTask, 216 | "IterateDirectory", &LuaASC::IterateDirectory, 217 | "RunCompileTasks", &LuaASC::RunCompileTasks, 218 | "SetOutputDir", &LuaASC::SetOutputDir 219 | ); 220 | 221 | 222 | lua.new_usertype( 223 | "ScriptCompiler", sol::no_constructor, 224 | "new", []() 225 | { 226 | ScriptCompiler compiler; 227 | GLuaHandler.SetupCompiler(compiler); 228 | return compiler; 229 | }, 230 | 231 | "InitIncludePaths", [](ScriptCompiler& comp, sol::table paths) 232 | { 233 | std::vector paths2; 234 | 235 | for (const auto& it : paths) { 236 | auto t3 = it.second.as(); 237 | paths2.emplace_back(t3); 238 | } 239 | 240 | comp.initIncludePaths(paths2); 241 | }, 242 | "AddMacro", & ScriptCompiler::addMacro, 243 | "AddPragma", & ScriptCompiler::addPragma, 244 | // #TODO ability to register SQF commands 245 | 246 | "PreprocessFile", [](ScriptCompiler& comp, const std::filesystem::path& path) 247 | { 248 | auto rootDir = path.root_path(); 249 | auto pathRelative = path.lexically_relative(rootDir); 250 | 251 | auto result = comp.preprocessScript(path.generic_string(), ("\\" / pathRelative).generic_string()); 252 | return result; 253 | }, 254 | 255 | "CompileScriptToFile", [](ScriptCompiler& comp, const std::filesystem::path& path, const std::filesystem::path& outputFile, OptimizerModuleLua& luaOptimizer) { 256 | auto rootDir = path.root_path(); 257 | auto pathRelative = path.lexically_relative(rootDir); 258 | 259 | comp.compileScriptLua(path.generic_string(), ("\\" / pathRelative).generic_string(), luaOptimizer, outputFile); 260 | } 261 | ); 262 | 263 | lua.new_usertype( 264 | "OptimizerModuleLua", sol::no_constructor, 265 | "new", [](sol::protected_function func) { 266 | OptimizerModuleLua opt; 267 | opt.nodeHandler = func; 268 | return opt; 269 | } 270 | ); 271 | 272 | 273 | lua.new_usertype( 274 | "Macro", sol::no_constructor, 275 | "new", 276 | sol::overload( 277 | [](std::string name) 278 | { 279 | return sqf::runtime::parser::macro(std::move(name)); 280 | }, 281 | [](std::string name, std::string content) 282 | { 283 | return sqf::runtime::parser::macro(std::move(name), std::move(content)); 284 | }, 285 | [](std::string name, sol::table args, std::string content) 286 | { 287 | std::vector args2; 288 | for (const auto& it : args) { 289 | auto t3 = it.second.as(); 290 | args2.emplace_back(t3); 291 | } 292 | 293 | return sqf::runtime::parser::macro(std::move(name), std::move(args2), std::move(content)); 294 | }, 295 | [](std::string name, sol::protected_function handler) 296 | { 297 | return sqf::runtime::parser::macro(std::move(name), [handler]( 298 | const sqf::runtime::parser::macro& m, 299 | const ::sqf::runtime::diagnostics::diag_info dinf, 300 | const ::sqf::runtime::fileio::pathinfo location, 301 | const std::vector& params, 302 | ::sqf::runtime::runtime& runtime) -> std::string 303 | { 304 | return handler(dinf); 305 | }); 306 | }, 307 | [](std::string name, sol::table args, sol::protected_function handler) 308 | { 309 | std::vector args2; 310 | for (const auto& it : args) { 311 | auto t3 = it.second.as(); 312 | args2.emplace_back(t3); 313 | } 314 | 315 | return sqf::runtime::parser::macro(std::move(name), std::move(args2), [handler]( 316 | const sqf::runtime::parser::macro& m, 317 | const ::sqf::runtime::diagnostics::diag_info dinf, 318 | const ::sqf::runtime::fileio::pathinfo location, 319 | const std::vector& params, 320 | ::sqf::runtime::runtime& runtime) -> std::string 321 | { 322 | return handler(sol::as_table(params), dinf); 323 | }); 324 | } 325 | ) 326 | ); 327 | 328 | lua.new_usertype( 329 | "Pragma", sol::no_constructor, 330 | "new", 331 | [](std::string name, sol::protected_function handler) 332 | { 333 | return sqf::runtime::parser::pragma(std::move(name), [handler]( 334 | const sqf::runtime::parser::pragma& m, 335 | ::sqf::runtime::runtime& runtime, 336 | const ::sqf::runtime::diagnostics::diag_info dinf, 337 | const ::sqf::runtime::fileio::pathinfo location, 338 | const std::string& data) -> std::string 339 | { 340 | return handler(data, dinf); 341 | }); 342 | } 343 | 344 | ); 345 | 346 | 347 | lua.new_usertype( 348 | "PathInfo", sol::no_constructor, 349 | "additional", sol::readonly(&sqf::runtime::fileio::pathinfo::additional), 350 | "physical", sol::readonly(&sqf::runtime::fileio::pathinfo::physical), 351 | "virtual", sol::readonly(&sqf::runtime::fileio::pathinfo::virtual_) 352 | ); 353 | 354 | lua.new_usertype( 355 | "DiagInfo", sol::no_constructor, 356 | "path", sol::readonly(&sqf::runtime::diagnostics::diag_info::path), 357 | "length", sol::readonly(&sqf::runtime::diagnostics::diag_info::length), 358 | "adjusted_offset", sol::readonly(&sqf::runtime::diagnostics::diag_info::adjusted_offset), 359 | "code_segment", sol::readonly(&sqf::runtime::diagnostics::diag_info::code_segment), 360 | "line", sol::readonly(&sqf::runtime::diagnostics::diag_info::line), 361 | "column", sol::readonly(&sqf::runtime::diagnostics::diag_info::column), 362 | "file_offset", sol::readonly(&sqf::runtime::diagnostics::diag_info::file_offset) 363 | ); 364 | 365 | lua.new_usertype( 366 | "OptimizerNode", sol::no_constructor, 367 | "type", &OptimizerModuleBase::Node::type, 368 | "file", &OptimizerModuleBase::Node::file, 369 | "line", &OptimizerModuleBase::Node::line, 370 | "offset", &OptimizerModuleBase::Node::offset, 371 | "children", &OptimizerModuleBase::Node::children, 372 | "constant", &OptimizerModuleBase::Node::constant, 373 | "value", &OptimizerModuleBase::Node::value, 374 | "areChildrenConstant", &OptimizerModuleBase::Node::areChildrenConstant 375 | ); 376 | 377 | lua.new_enum("InstructionType", 378 | "endStatement", InstructionType::endStatement, 379 | "push", InstructionType::push, 380 | "callUnary", InstructionType::callUnary, 381 | "callBinary", InstructionType::callBinary, 382 | "callNular", InstructionType::callNular, 383 | "assignTo", InstructionType::assignTo, 384 | "assignToLocal", InstructionType::assignToLocal, 385 | "getVariable", InstructionType::getVariable, 386 | "makeArray" , InstructionType::makeArray 387 | ); 388 | 389 | lua.new_usertype( 390 | "ScriptConstantArray", sol::default_constructor 391 | ); 392 | 393 | 394 | lua["ASC"] = LuaASC{}; 395 | } 396 | 397 | void LuaHandler::LoadFromFile(std::filesystem::path filePath) { 398 | isActive = true; 399 | 400 | lua.safe_script_file(filePath.string()); 401 | } 402 | 403 | 404 | void LuaHandler::SetupCompiler(ScriptCompiler& compiler) { 405 | if (!isActive) 406 | return; 407 | std::lock_guard accessGuard(luaAccess); 408 | if (!lua.get("SetupCompiler")) 409 | return; 410 | 411 | 412 | lua.get("SetupCompiler")(compiler); 413 | 414 | } 415 | -------------------------------------------------------------------------------- /src/luaHandler.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | #include "sol/sol.hpp" 6 | 7 | class ScriptCompiler; 8 | 9 | class LuaHandler { 10 | sol::state lua; 11 | // Was a lua script loaded from file? 12 | bool isActive = false; 13 | std::mutex luaAccess; 14 | public: 15 | 16 | LuaHandler(); 17 | void LoadFromFile(std::filesystem::path filePath); 18 | 19 | void SetupCompiler(ScriptCompiler& compiler); 20 | 21 | }; 22 | 23 | inline LuaHandler GLuaHandler; 24 | -------------------------------------------------------------------------------- /src/lzokay_stream.cpp: -------------------------------------------------------------------------------- 1 | // I hate this, but I need lzokay reading from a stream 2 | // this is 99% code from lzokay library from lib folder, just with std::istream used as input 3 | 4 | #include 5 | 6 | #include "lzokay.hpp" 7 | using namespace lzokay; 8 | 9 | 10 | #define NEEDS_IN(count) { \ 11 | auto curTell = (std::streamoff)inp.tellg(); \ 12 | if (curTell + (count) > inp_end) { \ 13 | dst_size = outp - dst; \ 14 | return EResult::InputOverrun; \ 15 | }} 16 | 17 | #define NEEDS_OUT(count) \ 18 | if (outp + (count) > outp_end) { \ 19 | dst_size = outp - dst; \ 20 | return EResult::OutputOverrun; \ 21 | } 22 | 23 | #define CONSUME_ZERO_BYTE_LENGTH \ 24 | std::size_t offset; \ 25 | { \ 26 | auto old_inp = inp.tellg(); \ 27 | while (inp.peek() == 0) inp.get(); \ 28 | offset = inp.tellg() - old_inp; \ 29 | if (offset > Max255Count) { \ 30 | dst_size = outp - dst; \ 31 | return EResult::Error; \ 32 | } \ 33 | } 34 | 35 | constexpr uint32_t M1Marker = 0x0; 36 | constexpr uint32_t M2Marker = 0x40; 37 | constexpr uint32_t M3Marker = 0x20; 38 | constexpr uint32_t M4Marker = 0x10; 39 | 40 | constexpr std::size_t Max255Count = std::size_t(~0) / 255 - 2; 41 | 42 | static uint16_t get_le16(std::istream& p) { 43 | uint16_t result; 44 | p.read((char*) &result, 2); 45 | return result; 46 | } 47 | 48 | 49 | EResult decompressStream(std::istream& src, std::size_t src_size, 50 | uint8_t* dst, std::size_t init_dst_size, 51 | std::size_t& dst_size) { 52 | dst_size = init_dst_size; 53 | 54 | if (src_size < 3) { 55 | dst_size = 0; 56 | return EResult::InputOverrun; 57 | } 58 | 59 | std::istream& inp = src; 60 | size_t inp_end = src_size; 61 | uint8_t* outp = dst; 62 | uint8_t* outp_end = dst + dst_size; 63 | uint8_t* lbcur; 64 | std::size_t lblen; 65 | std::size_t state = 0; 66 | std::size_t nstate = 0; 67 | 68 | /* First byte encoding */ 69 | if (inp.peek() >= 22) { 70 | /* 22..255 : copy literal string 71 | * length = (byte - 17) = 4..238 72 | * state = 4 [ don't copy extra literals ] 73 | * skip byte 74 | */ 75 | std::size_t len = inp.get() - uint8_t(17); 76 | NEEDS_IN(len) 77 | NEEDS_OUT(len) 78 | for (std::size_t i = 0; i < len; ++i) 79 | *outp++ = inp.get(); 80 | state = 4; 81 | } else if (inp.peek() >= 18) { 82 | /* 18..21 : copy 0..3 literals 83 | * state = (byte - 17) = 0..3 [ copy literals ] 84 | * skip byte 85 | */ 86 | nstate = inp.get() - uint8_t(17); 87 | state = nstate; 88 | NEEDS_IN(nstate) 89 | NEEDS_OUT(nstate) 90 | for (std::size_t i = 0; i < nstate; ++i) 91 | *outp++ = inp.get(); 92 | } 93 | /* 0..17 : follow regular instruction encoding, see below. It is worth 94 | * noting that codes 16 and 17 will represent a block copy from 95 | * the dictionary which is empty, and that they will always be 96 | * invalid at this place. 97 | */ 98 | 99 | while (true) { 100 | NEEDS_IN(1) 101 | uint8_t inst = inp.get(); 102 | if (inst & 0xC0) { 103 | /* [M2] 104 | * 1 L L D D D S S (128..255) 105 | * Copy 5-8 bytes from block within 2kB distance 106 | * state = S (copy S literals after this block) 107 | * length = 5 + L 108 | * Always followed by exactly one byte : H H H H H H H H 109 | * distance = (H << 3) + D + 1 110 | * 111 | * 0 1 L D D D S S (64..127) 112 | * Copy 3-4 bytes from block within 2kB distance 113 | * state = S (copy S literals after this block) 114 | * length = 3 + L 115 | * Always followed by exactly one byte : H H H H H H H H 116 | * distance = (H << 3) + D + 1 117 | */ 118 | NEEDS_IN(1) 119 | lbcur = outp - ((inp.get() << 3) + ((inst >> 2) & 0x7) + 1); 120 | lblen = std::size_t(inst >> 5) + 1; 121 | nstate = inst & uint8_t(0x3); 122 | } else if (inst & M3Marker) { 123 | /* [M3] 124 | * 0 0 1 L L L L L (32..63) 125 | * Copy of small block within 16kB distance (preferably less than 34B) 126 | * length = 2 + (L ?: 31 + (zero_bytes * 255) + non_zero_byte) 127 | * Always followed by exactly one LE16 : D D D D D D D D : D D D D D D S S 128 | * distance = D + 1 129 | * state = S (copy S literals after this block) 130 | */ 131 | lblen = std::size_t(inst & uint8_t(0x1f)) + 2; 132 | if (lblen == 2) { 133 | CONSUME_ZERO_BYTE_LENGTH 134 | NEEDS_IN(1) 135 | lblen += offset * 255 + 31 + inp.get(); 136 | } 137 | NEEDS_IN(2) 138 | nstate = get_le16(inp); 139 | //inp += 2; 140 | lbcur = outp - ((nstate >> 2) + 1); 141 | nstate &= 0x3; 142 | } else if (inst & M4Marker) { 143 | /* [M4] 144 | * 0 0 0 1 H L L L (16..31) 145 | * Copy of a block within 16..48kB distance (preferably less than 10B) 146 | * length = 2 + (L ?: 7 + (zero_bytes * 255) + non_zero_byte) 147 | * Always followed by exactly one LE16 : D D D D D D D D : D D D D D D S S 148 | * distance = 16384 + (H << 14) + D 149 | * state = S (copy S literals after this block) 150 | * End of stream is reached if distance == 16384 151 | */ 152 | lblen = std::size_t(inst & uint8_t(0x7)) + 2; 153 | if (lblen == 2) { 154 | CONSUME_ZERO_BYTE_LENGTH 155 | NEEDS_IN(1) 156 | lblen += offset * 255 + 7 + inp.get(); 157 | } 158 | NEEDS_IN(2) 159 | nstate = get_le16(inp); 160 | //inp += 2; 161 | lbcur = outp - (((inst & 0x8) << 11) + (nstate >> 2)); 162 | nstate &= 0x3; 163 | if (lbcur == outp) 164 | break; /* Stream finished */ 165 | lbcur -= 16384; 166 | } else { 167 | /* [M1] Depends on the number of literals copied by the last instruction. */ 168 | if (state == 0) { 169 | /* If last instruction did not copy any literal (state == 0), this 170 | * encoding will be a copy of 4 or more literal, and must be interpreted 171 | * like this : 172 | * 173 | * 0 0 0 0 L L L L (0..15) : copy long literal string 174 | * length = 3 + (L ?: 15 + (zero_bytes * 255) + non_zero_byte) 175 | * state = 4 (no extra literals are copied) 176 | */ 177 | std::size_t len = inst + 3; 178 | if (len == 3) { 179 | CONSUME_ZERO_BYTE_LENGTH 180 | NEEDS_IN(1) 181 | len += offset * 255 + 15 + inp.get(); 182 | } 183 | /* copy_literal_run */ 184 | NEEDS_IN(len) 185 | NEEDS_OUT(len) 186 | for (std::size_t i = 0; i < len; ++i) 187 | *outp++ = inp.get(); 188 | state = 4; 189 | continue; 190 | } else if (state != 4) { 191 | /* If last instruction used to copy between 1 to 3 literals (encoded in 192 | * the instruction's opcode or distance), the instruction is a copy of a 193 | * 2-byte block from the dictionary within a 1kB distance. It is worth 194 | * noting that this instruction provides little savings since it uses 2 195 | * bytes to encode a copy of 2 other bytes but it encodes the number of 196 | * following literals for free. It must be interpreted like this : 197 | * 198 | * 0 0 0 0 D D S S (0..15) : copy 2 bytes from <= 1kB distance 199 | * length = 2 200 | * state = S (copy S literals after this block) 201 | * Always followed by exactly one byte : H H H H H H H H 202 | * distance = (H << 2) + D + 1 203 | */ 204 | NEEDS_IN(1) 205 | nstate = inst & uint8_t(0x3); 206 | lbcur = outp - ((inst >> 2) + (inp.get() << 2) + 1); 207 | lblen = 2; 208 | } else { 209 | /* If last instruction used to copy 4 or more literals (as detected by 210 | * state == 4), the instruction becomes a copy of a 3-byte block from the 211 | * dictionary from a 2..3kB distance, and must be interpreted like this : 212 | * 213 | * 0 0 0 0 D D S S (0..15) : copy 3 bytes from 2..3 kB distance 214 | * length = 3 215 | * state = S (copy S literals after this block) 216 | * Always followed by exactly one byte : H H H H H H H H 217 | * distance = (H << 2) + D + 2049 218 | */ 219 | NEEDS_IN(1) 220 | nstate = inst & uint8_t(0x3); 221 | lbcur = outp - ((inst >> 2) + (inp.get() << 2) + 2049); 222 | lblen = 3; 223 | } 224 | } 225 | if (lbcur < dst) { 226 | dst_size = outp - dst; 227 | return EResult::LookbehindOverrun; 228 | } 229 | NEEDS_IN(nstate) 230 | NEEDS_OUT(lblen + nstate) 231 | /* Copy lookbehind */ 232 | for (std::size_t i = 0; i < lblen; ++i) 233 | *outp++ = *lbcur++; 234 | state = nstate; 235 | /* Copy literal */ 236 | for (std::size_t i = 0; i < nstate; ++i) 237 | *outp++ = inp.get(); 238 | } 239 | 240 | dst_size = outp - dst; 241 | if (lblen != 3) /* Ensure terminating M4 was encountered */ 242 | return EResult::Error; 243 | if (inp.tellg() == inp_end) 244 | return EResult::Success; 245 | else if (inp.tellg() < inp_end) 246 | return EResult::InputNotConsumed; 247 | else 248 | return EResult::InputOverrun; 249 | } -------------------------------------------------------------------------------- /src/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | #include 10 | 11 | #include 12 | #include "compiledCode.hpp" 13 | #include "scriptCompiler.hpp" 14 | #include "scriptSerializer.hpp" 15 | #include 16 | #include 17 | #include 18 | 19 | #include 20 | #include "luaHandler.hpp" 21 | #include "logger.hpp" 22 | 23 | std::queue tasks; 24 | std::mutex taskMutex; 25 | bool threadsShouldRun = true; 26 | std::filesystem::path outputDir; 27 | 28 | 29 | // Paths that should be considered the root of a P-drive, even if its not directly a drive letter 30 | // Basically they just get stripped off the front 31 | 32 | // rootPathMappings = {}; 33 | // inputPath = "P:/test/temp/thing.sqf" 34 | // results in virtualPath = "\\test\\temp\\thing.sqf" 35 | 36 | // rootPathMappings = { {"P:/test", "/"} } 37 | // inputPath = "P:/test/temp/thing.sqf" 38 | // results in virtualPath = "\\temp\\thing.sqf" 39 | 40 | std::vector> rootPathMappings; 41 | 42 | std::filesystem::path ApplyRootPathMapping(const std::filesystem::path& inputPath) 43 | { 44 | // Performance of this is meh, but I don't care rn 45 | 46 | for (auto& it : rootPathMappings) 47 | { 48 | auto deltaPos = std::mismatch(it.first.begin(), it.first.end(), inputPath.begin(), inputPath.end()); 49 | 50 | // If the first delta, is the end of our mapping, that means the whole mapping matched 51 | if (deltaPos.first != it.first.end()) 52 | continue; 53 | 54 | auto resultPath = it.second / inputPath.lexically_relative(it.first); 55 | return resultPath; 56 | } 57 | 58 | 59 | auto rootDir = inputPath.root_path(); 60 | auto pathRelative = inputPath.lexically_relative(rootDir); 61 | 62 | return pathRelative; 63 | } 64 | 65 | 66 | void compileRecursive(std::filesystem::path inputDir) { 67 | 68 | const std::filesystem::path ignoreGit(".git"); 69 | const std::filesystem::path ignoreSvn(".svn"); 70 | //recursively search for pboprefix 71 | for (auto i = std::filesystem::recursive_directory_iterator(inputDir, std::filesystem::directory_options::follow_directory_symlink); 72 | i != std::filesystem::recursive_directory_iterator(); 73 | ++i) { 74 | if (i->is_directory() && (i->path().filename() == ignoreGit || i->path().filename() == ignoreSvn)) { 75 | i.disable_recursion_pending(); //Don't recurse into that directory 76 | continue; 77 | } 78 | if (!i->is_regular_file()) continue; 79 | 80 | if (i->path().extension() == ".sqf"sv) { 81 | if (i->path().filename() == "fnc_zeusAttributes.sqf") continue; //Hard ignore for missing include file 82 | //if (i->path().filename() != "fnc_viewdir.sqf") continue; 83 | //if (i->path().filename() != "test.sqf") continue; //Hard ignore for missing include file 84 | //if (i->path().string().find("keybinding") == std::string::npos) continue; //CBA trying to format a code piece 85 | //if (i->path().filename().string().find("XEH_preStart") == std::string::npos) continue; //Hard ignore unit tests 86 | tasks.emplace(i->path()); 87 | } 88 | } 89 | } 90 | 91 | void processFile(ScriptCompiler& comp, std::filesystem::path path) { 92 | try { 93 | auto pathRelative = ApplyRootPathMapping(path); 94 | 95 | auto outputPath = outputDir / pathRelative.parent_path() / (path.stem().string() + ".sqfc"); 96 | 97 | //if sqfc exists, check if the sqf file has been updated (is newer). if not, skip this sqf file 98 | if (std::filesystem::exists(outputPath)) { 99 | auto sqfcWriteTime = std::filesystem::last_write_time(outputPath); 100 | auto sqfWriteTime = std::filesystem::last_write_time(path); 101 | if (sqfWriteTime <= sqfcWriteTime) //sqf file is older than sqfc 102 | return; 103 | } 104 | 105 | std::error_code ec; 106 | std::filesystem::create_directories(outputPath.parent_path(), ec); 107 | std::cout << "compile " << outputPath.generic_string() << "\n"; 108 | 109 | auto compiledData = comp.compileScript(path.generic_string(), ("\\" / pathRelative).generic_string()); 110 | 111 | if (compiledData.constants.empty()) return; // no code or failed to compile 112 | std::stringstream output(std::stringstream::binary | std::stringstream::out); 113 | //ScriptSerializer::compiledToBinaryCompressed(compiledData, output); 114 | ScriptSerializer::compiledToBinary(compiledData, output); 115 | 116 | auto data = output.str(); 117 | auto encoded = data; //base64_encode(data); 118 | 119 | std::ofstream outputFile(outputPath, std::ofstream::binary); 120 | 121 | outputFile.write(encoded.data(), encoded.length()); 122 | 123 | //ScriptSerializer::compiledToBinary(compiledData, output); 124 | outputFile.flush(); 125 | //std::istringstream data2(data, std::istringstream::binary); 126 | //auto res = ScriptSerializer::binaryToCompiledCompressed(data2); 127 | 128 | 129 | 130 | //auto outputPath2 = path.parent_path() / (path.stem().string() + ".sqfa"); 131 | //std::ofstream output2(outputPath2, std::ofstream::binary); 132 | //ScriptSerializer::compiledToHumanReadable(compiledData, output2); 133 | //output2.flush(); 134 | } catch (std::domain_error& err) { 135 | 136 | } 137 | catch (std::runtime_error& err) { 138 | 139 | } 140 | } 141 | 142 | void DecompressSQFC(std::filesystem::path inputPath, std::filesystem::path outputPath) 143 | { 144 | // To use this, also need to edit ScriptSerializer::compiledToBinary and disable compressed serialization 145 | 146 | std::ifstream inputFile(inputPath, std::ifstream::binary); 147 | auto compiledData = ScriptSerializer::binaryToCompiled(inputFile); 148 | 149 | 150 | 151 | std::stringstream output(std::stringstream::binary | std::stringstream::out); 152 | ScriptSerializer::compiledToBinary(compiledData, output); 153 | 154 | auto data = output.str(); 155 | auto encoded = data; //base64_encode(data); 156 | std::ofstream outputFile(outputPath, std::ofstream::binary); 157 | 158 | outputFile.write(encoded.data(), encoded.length()); 159 | outputFile.flush(); 160 | } 161 | 162 | 163 | int main(int argc, char* argv[]) { 164 | 165 | if (std::filesystem::exists("sqfc.lua")) { 166 | std::cout << "Using LUA for config" << "\n"; 167 | 168 | GLuaHandler.LoadFromFile("sqfc.lua"); 169 | return 0; //#TODO return real error state if any script failed 170 | } 171 | 172 | if (!std::filesystem::exists("sqfc.json")) { 173 | std::cout << "Missing sqfc.json in current working directory" << "\n"; 174 | return 1; 175 | } 176 | 177 | std::ifstream inputFile("sqfc.json"); 178 | auto json = nlohmann::json::parse(inputFile); 179 | 180 | std::vector checkConfigKeys = { "excludeList", "inputDirs", "includePaths", "outputDir", "workerThreads"}; 181 | for (const std::string& key : checkConfigKeys) { 182 | if (!json.contains(key)) { 183 | std::cout << "Missing \"" << key << "\" in sqfc.json" << "\n"; 184 | return 1; 185 | } 186 | } 187 | 188 | std::vector excludeList = json["excludeList"].get>(); 189 | 190 | std::transform(excludeList.begin(), excludeList.end(), excludeList.begin(), [](std::string inp) { 191 | std::transform(inp.begin(), inp.end(), inp.begin(), ::tolower); 192 | return inp; 193 | }); 194 | 195 | 196 | std::vector inputDirs; 197 | for (const std::string& inputDir : json["inputDirs"].get>()) { 198 | inputDirs.push_back(std::filesystem::path(inputDir)); 199 | } 200 | 201 | std::vector includePaths; 202 | for (const std::string& includePath : json["includePaths"].get>()) { 203 | includePaths.push_back(std::filesystem::path(includePath)); 204 | } 205 | 206 | outputDir = std::filesystem::path(json["outputDir"].get()); 207 | int numberOfWorkerThreads = json["workerThreads"].get(); 208 | 209 | if (!json["rootPathMappings"].is_null() && !json["rootPathMappings"].is_array()) 210 | { 211 | std::cout << "sqfc.json error, rootPathMappings has to be array of arrays"; 212 | } 213 | else 214 | { 215 | for (const auto& includePath : json["rootPathMappings"]) { 216 | auto x = includePath.get>(); 217 | auto virtualPath = x[1]; 218 | // Strip leading slash if there is one, we add that back later 219 | if (virtualPath.front() == '/' || virtualPath.front() == '\\') 220 | virtualPath.erase(0, 1); 221 | 222 | rootPathMappings.push_back({std::filesystem::path(x[0]).lexically_normal(), std::filesystem::path(virtualPath).lexically_normal()}); 223 | } 224 | } 225 | 226 | // Logging 227 | 228 | 229 | if (auto loggingConfig = json["logging"]; !loggingConfig.is_null()) 230 | { 231 | if (auto verboseCfg = loggingConfig["verbose"]; verboseCfg.is_boolean()) 232 | GLogger.m_EnableVerbose = verboseCfg; 233 | } 234 | 235 | 236 | 237 | // Setup workers 238 | 239 | std::mutex workWait; 240 | workWait.lock(); 241 | auto workerFunc = [&]() { 242 | ScriptCompiler compiler(includePaths); 243 | workWait.lock(); 244 | workWait.unlock(); 245 | 246 | while (threadsShouldRun) { 247 | std::unique_lock lock(taskMutex); 248 | if (tasks.empty()) return; 249 | const auto task(std::move(tasks.front())); 250 | tasks.pop(); 251 | if (tasks.empty()) 252 | threadsShouldRun = false; 253 | lock.unlock(); 254 | 255 | 256 | auto foundExclude = std::find_if(excludeList.begin(), excludeList.end(), [&task](const std::string& excludeItem) 257 | { 258 | auto taskString = task.string(); 259 | std::transform(taskString.begin(), taskString.end(), taskString.begin(), ::tolower); 260 | return taskString.find(excludeItem) != std::string::npos; 261 | }); 262 | 263 | if (foundExclude == excludeList.end()) 264 | processFile(compiler, task); 265 | } 266 | 267 | }; 268 | 269 | //compileRecursive("I:/ACE3/addons"); 270 | //compileRecursive("I:/CBA_A3/addons"); 271 | //compileRecursive("T:/x/"); 272 | //compileRecursive("T:/z/ace/"); 273 | //compileRecursive("T:/z/acex/"); 274 | //compileRecursive("T:/a3"); 275 | 276 | //compileRecursive("P:/test/"); 277 | for (std::filesystem::path &inputDir : inputDirs) { 278 | compileRecursive(inputDir); 279 | } 280 | 281 | workWait.unlock(); 282 | 283 | std::vector workerThreads; 284 | for (int i = 0; i < numberOfWorkerThreads; i++) { 285 | workerThreads.push_back(std::thread(workerFunc)); 286 | } 287 | 288 | workerFunc(); 289 | 290 | for (std::thread &thread : workerThreads) { 291 | thread.join(); 292 | } 293 | 294 | /* 295 | auto compiledScript = compiler.compileScript("I:/ACE3/addons/advanced_ballistics/functions/fnc_readWeaponDataFromConfig.sqf"); 296 | 297 | std::ofstream hr("P:\\human.sqfa"); 298 | ScriptSerializer::compiledToHumanReadable(compiledScript, hr); 299 | hr.close(); 300 | 301 | std::ofstream bin("P:\\binary.sqfc", std::ofstream::binary); 302 | ScriptSerializer::compiledToBinary(compiledScript, bin); 303 | bin.close(); 304 | 305 | std::ifstream bini("P:\\binary.sqfc", std::ifstream::binary); 306 | auto compData = ScriptSerializer::binaryToCompiled(bini); 307 | std::ofstream hr2("P:\\humanpostbin.sqfa"); 308 | ScriptSerializer::compiledToHumanReadable(compData, hr2); 309 | hr2.close(); 310 | 311 | 312 | 313 | std::ofstream biCn("P:\\binaryCompressed.sqfc", std::ofstream::binary); 314 | ScriptSerializer::compiledToBinaryCompressed(compiledScript, biCn); 315 | biCn.close(); 316 | 317 | std::ifstream biniC("P:\\binaryCompressed.sqfc", std::ifstream::binary); 318 | auto compDataC = ScriptSerializer::binaryToCompiledCompressed(biniC); 319 | std::ofstream hr2C("P:\\humanpostbincompressed.sqfa"); 320 | ScriptSerializer::compiledToHumanReadable(compDataC, hr2C); 321 | hr2C.close(); 322 | */ 323 | 324 | return 0; //#TODO return real error state if any script failed 325 | } 326 | -------------------------------------------------------------------------------- /src/optimizer/optimizer.cpp: -------------------------------------------------------------------------------- 1 | #include "optimizer.h" 2 | 3 | void Optimizer::optimize(Node& node) { 4 | optimizeConstantFold(node); 5 | } 6 | -------------------------------------------------------------------------------- /src/optimizer/optimizer.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "optimizerModuleConstantFold.hpp" 3 | 4 | 5 | class Optimizer : public OptimizerModuleConstantFold { 6 | public: 7 | 8 | void optimize(Node& node); 9 | 10 | 11 | }; -------------------------------------------------------------------------------- /src/optimizer/optimizerModuleBase.cpp: -------------------------------------------------------------------------------- 1 | #include "optimizerModuleBase.hpp" 2 | #include 3 | #include 4 | #include 5 | #include "runtime/d_string.h" 6 | #include 7 | 8 | #include "parser/sqf/parser.tab.hh" 9 | 10 | 11 | void OptimizerModuleBase::Node::dumpTree(std::ostream& output, size_t indent) const { 12 | for (int i = 0; i < indent; ++i) { 13 | output << " "; 14 | } 15 | output << instructionTypeToString(type) << ":" << line << "\n"; 16 | 17 | for (auto& it : children) { 18 | it.dumpTree(output, indent + 1); 19 | } 20 | } 21 | 22 | bool OptimizerModuleBase::Node::buildConstState() { 23 | return false; 24 | //switch (type) { 25 | // 26 | // case InstructionType::push: return constant = true; 27 | // case InstructionType::assignTo: return false; 28 | // case InstructionType::assignToLocal: return false; 29 | // case InstructionType::getVariable: return false; 30 | // case InstructionType::makeArray: return constant = true; 31 | //} 32 | // 33 | //if (type == InstructionType::push) { 34 | // return constant = true; 35 | //} 36 | // 37 | // 38 | // 39 | //bool childrenConstant = std::all_of(children.begin(), children.end(), [](Node& it) { 40 | // return it.buildConstState(); 41 | // }); 42 | // 43 | //bool meIsConst = false; 44 | // 45 | //switch (type) { 46 | // 47 | // case InstructionType::endStatement: return constant = childrenConstant; 48 | // case InstructionType::callUnary: return constant = canUnaryBeConst.anyOf(*this, childrenConstant); 49 | // case InstructionType::callBinary: return constant = canBinaryBeConst.anyOf(*this, childrenConstant); 50 | // case InstructionType::callNular: return constant = canNularBeConst.anyOf(*this, childrenConstant); 51 | //} 52 | 53 | 54 | 55 | 56 | } 57 | 58 | bool OptimizerModuleBase::Node::areChildrenConstant() const { 59 | return std::all_of(children.begin(), children.end(), [](const Node& it) { 60 | return it.constant; 61 | }); 62 | } 63 | 64 | std::vector OptimizerModuleBase::Node::bottomUpFlatten() { 65 | std::vector> result; 66 | std::queue myQueue; 67 | myQueue.push(this); 68 | size_t totalNodeCount = 0; 69 | int currentLevelNodeNum = 1;//used to record num of nodes in current level 70 | int nextLevelNodeNum = 0;//used to record num of nodes in next level 71 | std::vector level; 72 | while (!myQueue.empty()) { 73 | 74 | OptimizerModuleBase::Node* temp = myQueue.front(); 75 | myQueue.pop(); 76 | level.push_back(temp); 77 | totalNodeCount++; 78 | currentLevelNodeNum--; 79 | for (auto& it : temp->children) { 80 | myQueue.push(&it); 81 | nextLevelNodeNum++; 82 | } 83 | if (currentLevelNodeNum == 0) {//if we have traversed current level, turn to next level 84 | result.emplace_back(std::move(level));//push the current level into result 85 | currentLevelNodeNum = nextLevelNodeNum;//assign next level node num to current 86 | nextLevelNodeNum = 0;//set next level num to 0 87 | } 88 | } 89 | std::reverse(result.begin(), result.end()); 90 | std::vector endResult; 91 | endResult.reserve(totalNodeCount); 92 | for (auto& it : result) 93 | endResult.insert(endResult.end(), it.begin(), it.end()); 94 | 95 | return endResult; 96 | } 97 | 98 | OptimizerModuleBase::Node OptimizerModuleBase::nodeFromAST(const astnode& input) { 99 | auto nodeType = input.kind; 100 | switch (nodeType) { 101 | 102 | case sqf::parser::sqf::bison::astkind::ASSIGNMENT: { 103 | Node newNode; 104 | 105 | newNode.type = nodeType == sqf::parser::sqf::bison::astkind::ASSIGNMENT ? InstructionType::assignTo : InstructionType::assignToLocal; 106 | newNode.file = *input.token.path; 107 | newNode.line = input.token.line; 108 | newNode.offset = input.token.offset; 109 | auto varname = std::string(input.children[0].token.contents); 110 | std::transform(varname.begin(), varname.end(), varname.begin(), ::tolower); 111 | newNode.value = std::string(varname); 112 | 113 | newNode.children.emplace_back(nodeFromAST(input.children[1])); 114 | return newNode; 115 | } 116 | case sqf::parser::sqf::bison::astkind::ASSIGNMENT_LOCAL: { 117 | Node newNode; 118 | 119 | newNode.type = nodeType == sqf::parser::sqf::bison::astkind::ASSIGNMENT ? InstructionType::assignTo : InstructionType::assignToLocal; 120 | newNode.file = *input.token.path; 121 | newNode.line = input.token.line; 122 | newNode.offset = input.token.offset; 123 | auto varname = std::string(input.token.contents); 124 | std::transform(varname.begin(), varname.end(), varname.begin(), ::tolower); 125 | newNode.value = std::string(varname); 126 | 127 | newNode.children.emplace_back(nodeFromAST(input.children[0])); 128 | return newNode; 129 | } 130 | case sqf::parser::sqf::bison::astkind::EXP0: 131 | case sqf::parser::sqf::bison::astkind::EXP1: 132 | case sqf::parser::sqf::bison::astkind::EXP2: 133 | case sqf::parser::sqf::bison::astkind::EXP3: 134 | case sqf::parser::sqf::bison::astkind::EXP4: 135 | case sqf::parser::sqf::bison::astkind::EXP5: 136 | case sqf::parser::sqf::bison::astkind::EXP6: 137 | case sqf::parser::sqf::bison::astkind::EXP7: 138 | case sqf::parser::sqf::bison::astkind::EXP8: 139 | case sqf::parser::sqf::bison::astkind::EXP9: { 140 | Node newNode; 141 | 142 | newNode.type = InstructionType::callBinary; 143 | newNode.file = *input.token.path; 144 | newNode.line = input.token.line; 145 | newNode.offset = input.token.offset; 146 | auto varname = std::string(input.token.contents); 147 | std::transform(varname.begin(), varname.end(), varname.begin(), ::tolower); 148 | newNode.value = std::string(varname); 149 | 150 | newNode.children.emplace_back(nodeFromAST(input.children[0])); 151 | newNode.children.emplace_back(nodeFromAST(input.children[1])); 152 | 153 | return newNode; 154 | } 155 | case sqf::parser::sqf::bison::astkind::EXPN: { 156 | Node newNode; 157 | 158 | newNode.type = InstructionType::callNular; 159 | newNode.file = *input.token.path; 160 | newNode.line = input.token.line; 161 | newNode.offset = input.token.offset; 162 | auto varname = std::string(input.token.contents); 163 | std::transform(varname.begin(), varname.end(), varname.begin(), ::tolower); 164 | newNode.value = std::string(varname); 165 | return newNode; 166 | } 167 | case sqf::parser::sqf::bison::astkind::EXPU: { 168 | Node newNode; 169 | 170 | newNode.type = InstructionType::callUnary; 171 | newNode.file = *input.token.path; 172 | newNode.line = input.token.line; 173 | newNode.offset = input.token.offset; 174 | auto varname = std::string(input.token.contents); 175 | std::transform(varname.begin(), varname.end(), varname.begin(), ::tolower); 176 | newNode.value = std::string(varname); 177 | auto subEl = nodeFromAST(input.children[0]); 178 | 179 | newNode.children.emplace_back(std::move(subEl)); 180 | return newNode; 181 | } 182 | case sqf::parser::sqf::bison::astkind::NUMBER: 183 | case sqf::parser::sqf::bison::astkind::HEXNUMBER: { 184 | float val; 185 | auto res = 186 | (nodeType == sqf::parser::sqf::bison::astkind::HEXNUMBER) ? 187 | std::from_chars(input.token.contents.data() + 2, input.token.contents.data() + input.token.contents.size(), val, std::chars_format::hex) 188 | : 189 | std::from_chars(input.token.contents.data(), input.token.contents.data() + input.token.contents.size(), val); 190 | if (res.ec == std::errc::invalid_argument) { 191 | throw std::runtime_error("invalid scalar at: " + *input.token.path + ":" + std::to_string(input.token.line)); 192 | } 193 | else if (res.ec == std::errc::result_out_of_range) { 194 | throw std::runtime_error("scalar out of range at: " + *input.token.path + ":" + std::to_string(input.token.line)); 195 | } 196 | 197 | 198 | Node newNode; 199 | 200 | newNode.type = InstructionType::push; 201 | newNode.file = *input.token.path; 202 | newNode.line = input.token.line; 203 | newNode.offset = input.token.offset; 204 | newNode.value = val; 205 | return newNode; 206 | } 207 | case sqf::parser::sqf::bison::astkind::IDENT: { 208 | Node newNode; 209 | 210 | newNode.type = InstructionType::getVariable; 211 | newNode.file = *input.token.path; 212 | newNode.line = input.token.line; 213 | newNode.offset = input.token.offset; 214 | auto varname = std::string(input.token.contents); 215 | std::transform(varname.begin(), varname.end(), varname.begin(), ::tolower); 216 | newNode.value = std::string(varname); 217 | return newNode; 218 | } 219 | case sqf::parser::sqf::bison::astkind::STRING: { 220 | Node newNode; 221 | 222 | newNode.type = InstructionType::push; 223 | newNode.file = *input.token.path; 224 | newNode.line = input.token.line; 225 | newNode.offset = input.token.offset; 226 | newNode.value = ::sqf::types::d_string::from_sqf(input.token.contents); 227 | return newNode; 228 | } 229 | 230 | case sqf::parser::sqf::bison::astkind::BOOLEAN_TRUE: { 231 | Node newNode; 232 | 233 | newNode.type = InstructionType::push; 234 | newNode.file = *input.token.path; 235 | newNode.line = input.token.line; 236 | newNode.offset = input.token.offset; 237 | newNode.value = true; 238 | return newNode; 239 | } 240 | case sqf::parser::sqf::bison::astkind::BOOLEAN_FALSE: { 241 | Node newNode; 242 | 243 | newNode.type = InstructionType::push; 244 | newNode.file = *input.token.path; 245 | newNode.line = input.token.line; 246 | newNode.offset = input.token.offset; 247 | newNode.value = false; 248 | return newNode; 249 | } 250 | 251 | 252 | 253 | 254 | case sqf::parser::sqf::bison::astkind::CODE: { 255 | Node newNode; 256 | 257 | newNode.type = InstructionType::push; 258 | newNode.file = *input.token.path; 259 | newNode.line = input.token.line; 260 | newNode.offset = input.token.offset; 261 | 262 | auto codeEnd = input.token.offset; 263 | 264 | if (!input.children.empty()) { 265 | 266 | auto* statements = &input.children[0]; 267 | if (statements->kind != sqf::parser::sqf::bison::astkind::STATEMENTS) 268 | __debugbreak(); 269 | 270 | size_t lastToken = 0; 271 | 272 | std::vector::const_iterator> lastChildren; 273 | 274 | while (!statements->children.empty()) { 275 | const auto& lastChild = (statements->children.end() - 1); 276 | 277 | lastToken = lastChild->token.offset + lastChild->token.contents.size(); 278 | statements = &(*lastChild); 279 | lastChildren.emplace_back(lastChild); 280 | } 281 | 282 | // we also need to travel the full way back. Just finding next } is not sufficient, we may be multiple code levels deep 283 | 284 | 285 | auto endData = input.token.contents.data() + (lastToken - input.token.offset); 286 | 287 | std::reverse(lastChildren.begin(), lastChildren.end()); 288 | 289 | for (auto& it : lastChildren) { 290 | switch (it->kind) { 291 | case sqf::parser::sqf::bison::astkind::CODE: 292 | // find ending } 293 | while (*endData && *endData != '}') { 294 | ++endData; 295 | ++lastToken; 296 | } 297 | // after ending } 298 | ++endData; ++lastToken; 299 | break; 300 | case sqf::parser::sqf::bison::astkind::ARRAY: 301 | // find ending ] 302 | while (*endData && *endData != ']') { 303 | ++endData;++lastToken; 304 | } 305 | // after ending ] 306 | ++endData; ++lastToken; 307 | break; 308 | default: ; 309 | } 310 | } 311 | 312 | 313 | while (*endData && *endData != '}') { 314 | ++endData; 315 | ++lastToken; 316 | } 317 | // we stop BEFORE ending }, we only want the inner code. 318 | 319 | codeEnd = lastToken; 320 | newNode.value = ScriptCodePiece({}, codeEnd - input.token.offset - 1, input.token.offset + 1);//instructions are empty as they are in node children 321 | } else { 322 | newNode.value = ScriptCodePiece({}, 0, input.token.offset + 1);//instructions are empty as they are in node children 323 | 324 | } 325 | 326 | 327 | // empty code {} 328 | if (input.children.empty()) return newNode; 329 | 330 | if (input.children[0].kind != sqf::parser::sqf::bison::astkind::STATEMENTS) 331 | __debugbreak(); 332 | 333 | for (auto& it : input.children[0].children) { 334 | newNode.children.emplace_back(nodeFromAST(it)); 335 | } 336 | 337 | return newNode; 338 | } 339 | 340 | case sqf::parser::sqf::bison::astkind::STATEMENTS: { // this is probably wrong 341 | Node newNode; 342 | 343 | newNode.type = InstructionType::endStatement; //just a dummy 344 | //newNode.file = *input.token.path; 345 | newNode.line = input.token.line; 346 | newNode.offset = input.token.offset; 347 | 348 | for (auto& it : input.children) { 349 | newNode.children.emplace_back(nodeFromAST(it)); 350 | } 351 | 352 | return newNode; 353 | } 354 | 355 | 356 | case sqf::parser::sqf::bison::astkind::ARRAY: { 357 | 358 | Node newNode; 359 | 360 | newNode.type = InstructionType::makeArray; 361 | newNode.file = *input.token.path; 362 | newNode.line = input.token.line; 363 | newNode.offset = input.token.offset; 364 | for (auto& it : input.children) { 365 | newNode.children.emplace_back(nodeFromAST(it)); 366 | } 367 | 368 | return newNode; 369 | } 370 | //case sqf::parser::sqf::bison::astkind::NA: 371 | //case sqf::parser::sqf::bison::astkind::SQF: 372 | //case sqf::parser::sqf::bison::astkind::STATEMENT: 373 | //case sqf::parser::sqf::bison::astkind::BRACKETS: 374 | // for (auto& it : node.children) 375 | // stuffAST(output, instructions, it); 376 | default: { 377 | 378 | Node newNode; // old stuff, now handled by astkind::STATEMENTS 379 | 380 | newNode.type = InstructionType::endStatement; //just a dummy 381 | newNode.file = *input.token.path; 382 | newNode.line = input.token.line; 383 | newNode.offset = input.token.offset; 384 | for (auto& it : input.children) { 385 | newNode.children.emplace_back(nodeFromAST(it)); 386 | } 387 | return newNode; 388 | } 389 | } 390 | __debugbreak(); //should never reach 391 | } 392 | -------------------------------------------------------------------------------- /src/optimizer/optimizerModuleBase.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "../compiledCode.hpp" 3 | #include 4 | 5 | #include 6 | using astnode = sqf::parser::sqf::bison::astnode; 7 | 8 | template 9 | class Signal; 10 | 11 | template 12 | class Signal { 13 | private: 14 | typedef std::function Slot; 15 | 16 | public: 17 | void connect(Slot slot) { 18 | slots.push_back(slot); 19 | } 20 | 21 | std::vector operator() (Args&&... args) const { 22 | return emit(std::forward(args)...); 23 | } 24 | std::vector emit(Args&&... args) const { 25 | std::vector returnData; 26 | if (slots.empty()) 27 | return; 28 | for (auto& slot : slots) { 29 | returnData.push_back(slot(std::forward(args)...)); 30 | } 31 | return returnData; 32 | } 33 | bool anyOf(Args&&... args) const { 34 | for (auto& slot : slots) { 35 | if (slot(std::forward(args)...)) return true; 36 | } 37 | return false; 38 | } 39 | void removeAllSlots() { 40 | slots.clear(); 41 | } 42 | private: 43 | std::vector slots{}; 44 | }; 45 | 46 | class OptimizerModuleBase { 47 | public: 48 | class Node { 49 | public: 50 | Node() {} 51 | //Node(Node&& mv) noexcept { 52 | // type = mv.type; 53 | // file = std::move(mv.file); 54 | // line = mv.line; 55 | // offset = mv.offset; 56 | // value = std::move(mv.value); 57 | // children = std::move(mv.children); 58 | //} 59 | 60 | InstructionType type; 61 | 62 | std::string file; 63 | size_t line; 64 | size_t offset; 65 | 66 | ScriptConstant value; 67 | std::vector children; 68 | 69 | void dumpTree(std::ostream& output, size_t indent) const; 70 | bool buildConstState();//#TODO remove 71 | bool areChildrenConstant() const; 72 | 73 | 74 | 75 | std::vector bottomUpFlatten(); 76 | 77 | 78 | bool constant = false; 79 | }; 80 | 81 | static Node nodeFromAST(const astnode& input); //#TODO move to optimizer 82 | 83 | 84 | Signal canBinaryBeConst;//#TODO remove 85 | Signal canUnaryBeConst; 86 | Signal canNularBeConst; 87 | 88 | 89 | }; 90 | -------------------------------------------------------------------------------- /src/optimizer/optimizerModuleConstantFold.cpp: -------------------------------------------------------------------------------- 1 | #include "optimizerModuleConstantFold.hpp" 2 | #include 3 | #if defined(_MSC_VER) 4 | #include 5 | #else 6 | #include 7 | #endif 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | constexpr auto NularPushConstNularCommand(std::string_view name) { 14 | return [name](OptimizerModuleBase::Node& node) -> void { 15 | node.type = InstructionType::push; 16 | if (!node.children.empty()) 17 | __debugbreak(); 18 | node.children.clear(); 19 | node.constant = true; 20 | node.value = ScriptConstantNularCommand(std::string(name)); 21 | }; 22 | } 23 | 24 | #define ONLY_PUSH_BINARY if (node.children[0].type != InstructionType::push || node.children[1].type != InstructionType::push) return 25 | 26 | class OptimizerConstantFoldActionMap : public Singleton { 27 | public: 28 | 29 | OptimizerConstantFoldActionMap() { 30 | setupBinary(); 31 | setupUnary(); 32 | setupNulary(); 33 | } 34 | 35 | void processBinary(OptimizerModuleBase::Node& node) { 36 | auto& cmdName = std::get(node.value); 37 | auto found = binaryActions.find(cmdName); 38 | if (found != binaryActions.end()) 39 | found->second(node); 40 | } 41 | 42 | void processUnary(OptimizerModuleBase::Node& node) { 43 | auto& cmdName = std::get(node.value); 44 | auto found = unaryActions.find(cmdName); 45 | if (found != unaryActions.end()) 46 | found->second(node); 47 | } 48 | 49 | void processNulary(OptimizerModuleBase::Node& node) { 50 | auto& cmdName = std::get(node.value); 51 | auto found = nularyActions.find(cmdName); 52 | if (found != nularyActions.end()) 53 | found->second(node); 54 | } 55 | 56 | 57 | private: 58 | 59 | void setupBinary() { 60 | binaryActions["else"] = [](OptimizerModuleBase::Node & node) -> void { 61 | node.type = InstructionType::push; 62 | node.constant = true; 63 | 64 | node.value = ScriptConstantArray(); //dummy. Children are the contents 65 | }; 66 | 67 | //math 68 | 69 | 70 | //binaryActions["+"] = [](OptimizerModuleBase::Node & node) -> void { //#TODO array 71 | // ONLY_PUSH_BINARY; 72 | // if (node.children[0].value.index() == 1) { //string 73 | // auto leftArg = std::get(node.children[0].value); 74 | // auto rightArg = std::get(node.children[1].value); 75 | // node.value = leftArg + rightArg; 76 | // } else {//float 77 | // float leftArg = std::get(node.children[0].value); 78 | // float rightArg = std::get(node.children[1].value); 79 | // node.value = leftArg + rightArg; 80 | // } 81 | // 82 | // 83 | // 84 | // node.type = InstructionType::push; 85 | // node.children.clear(); 86 | // node.constant = true; 87 | // 88 | //}; 89 | 90 | binaryActions["-"] = [](OptimizerModuleBase::Node & node) -> void { 91 | ONLY_PUSH_BINARY; 92 | 93 | auto type = getConstantType(node.children[0].value); 94 | 95 | //#TODO fix array, this is safe if both are const, see params need to convert arrays to constants and merge 96 | if (type == ConstantType::array) { //array 97 | //std::unordered_set vals; 98 | // 99 | //for (auto& i : node.children[1].children) { //#TODO number support 100 | // if (i.value.index() != 1) 101 | // return; //not string, don't optimize 102 | // vals.emplace(std::get(i.value)); 103 | //} 104 | //std::vector newNodes; 105 | //for (auto& it : node.children[0].children) { 106 | // if (it.value.index() != 1) 107 | // return; //not string, don't optimize 108 | // auto & sval = std::get(it.value); 109 | // 110 | // auto found = vals.find(sval); 111 | // if (found == vals.end()) 112 | // newNodes.emplace_back(std::move(it)); 113 | //} 114 | //node.children[0].children = std::move(newNodes); 115 | return; 116 | } else if (type == ConstantType::scalar) {//float 117 | float leftArg = std::get(node.children[0].value); 118 | float rightArg = std::get(node.children[1].value); 119 | node.value = leftArg - rightArg; 120 | } 121 | else 122 | { 123 | std::cout << "Something is very wrong, tried to optimize operator '-' but argument type is neither array nor number?! Breaking into debugger now." << std::endl; 124 | __debugbreak(); 125 | 126 | } 127 | node.type = InstructionType::push; 128 | node.children.clear(); 129 | node.constant = true; 130 | }; 131 | 132 | binaryActions["/"] = [](OptimizerModuleBase::Node & node) -> void { 133 | ONLY_PUSH_BINARY; 134 | auto type = getConstantType(node.children[0].value); 135 | if (type != ConstantType::scalar) 136 | return; 137 | 138 | float leftArg = std::get(node.children[0].value); 139 | float rightArg = std::get(node.children[1].value); 140 | 141 | node.type = InstructionType::push; 142 | node.children.clear(); 143 | node.constant = true; 144 | node.value = leftArg / rightArg; 145 | }; 146 | binaryActions["*"] = [](OptimizerModuleBase::Node & node) -> void { 147 | ONLY_PUSH_BINARY; 148 | auto type = getConstantType(node.children[0].value); 149 | if (type != ConstantType::scalar) 150 | return; 151 | 152 | float leftArg = std::get(node.children[0].value); 153 | float rightArg = std::get(node.children[1].value); 154 | 155 | node.type = InstructionType::push; 156 | node.children.clear(); 157 | node.constant = true; 158 | node.value = leftArg * rightArg; 159 | }; 160 | 161 | binaryActions["mod"] = [](OptimizerModuleBase::Node & node) -> void { 162 | ONLY_PUSH_BINARY; 163 | auto type = getConstantType(node.children[0].value); 164 | if (type != ConstantType::scalar) 165 | return; 166 | 167 | float leftArg = std::get(node.children[0].value); 168 | float rightArg = std::get(node.children[1].value); 169 | 170 | node.type = InstructionType::push; 171 | node.children.clear(); 172 | node.constant = true; 173 | node.value = fmodf(leftArg, rightArg); 174 | }; 175 | 176 | } 177 | 178 | void setupUnary() { 179 | unaryActions["sqrt"] = [](OptimizerModuleBase::Node & node) -> void { 180 | float rightArg = std::get(node.children[0].value); 181 | 182 | node.type = InstructionType::push; 183 | node.children.clear(); 184 | node.constant = true; 185 | node.value = sqrt(rightArg); 186 | }; 187 | 188 | unaryActions["!"] = [](OptimizerModuleBase::Node & node) -> void { 189 | bool rightArg = std::get(node.children[0].value); 190 | 191 | node.type = InstructionType::push; 192 | node.children.clear(); 193 | node.constant = true; 194 | node.value = !rightArg; 195 | }; 196 | 197 | 198 | // Params is special, it takes an array but never returns part of that array, so we can safely make it constant 199 | unaryActions["params"] = [](OptimizerModuleBase::Node& node) -> void { 200 | 201 | // convert child makeArray node into a constant push 202 | if (node.children[0].type == InstructionType::makeArray) { 203 | 204 | //convert nested arrays into constants 205 | 206 | std::function resolveMakeArray = [&](OptimizerModuleBase::Node& node) 207 | { 208 | if (node.type != InstructionType::makeArray) 209 | return; 210 | 211 | for (auto& it : node.children) 212 | resolveMakeArray(it); 213 | 214 | node.value = ScriptConstantArray(); //dummy. Children are the contents 215 | node.type = InstructionType::push; 216 | }; 217 | 218 | 219 | resolveMakeArray(node.children[0]); 220 | 221 | bool allPush = std::all_of(node.children[0].children.begin(), node.children[0].children.end(), [](const OptimizerModuleBase::Node& it) 222 | { 223 | return it.type == InstructionType::push; 224 | }); 225 | if (!allPush) { 226 | std::stringstream buf; 227 | node.dumpTree(buf, 0); 228 | auto str = buf.str(); 229 | __debugbreak(); 230 | } 231 | 232 | 233 | 234 | node.children[0].value = ScriptConstantArray(); //dummy. Children are the contents 235 | node.children[0].type = InstructionType::push; 236 | 237 | 238 | // check if params array contains a value that can be modified by reference, to prevent it modifying the value in compiled code 239 | 240 | const auto& paramsList = node.children[0].children; 241 | 242 | for (auto& it : paramsList) 243 | { 244 | //#TODO throw warning on empty array passed to params? 245 | if (!it.children.empty() && std::holds_alternative(it.value)) // is array, and is not empty 246 | { 247 | // it is a [name, default, allowedTypes] array 248 | auto& paramArguments = it.children; 249 | 250 | if (paramArguments.size() < 2) // only [name], no defaults, don't care 251 | continue; 252 | 253 | if (std::holds_alternative(paramArguments[1].value)) 254 | { 255 | // !!! [name, []] the array will be passed down by ref if parameter is not provided, and cause changes to propagate/persist to compiled code if modified by reference 256 | // We know that this will be a problem, lets insert a array copy 257 | 258 | // replace ourselves with a + unary 259 | 260 | // move our array out so we can move it over instead of copying 261 | auto myArgumentsArray = std::move(node.children[0]); 262 | // our only child is now default initialized, we have no children 263 | node.children.clear(); 264 | 265 | // convert ourselves to a unary + and add the array as argument 266 | OptimizerModuleBase::Node copyNode; 267 | copyNode.type = InstructionType::callUnary; 268 | copyNode.constant = false; // Lets prevent optimizing this copy out, shouldn't be done anyway but better be safe 269 | copyNode.value = STRINGTYPE("+"); // value is name of command 270 | copyNode.file = node.file; 271 | copyNode.line = myArgumentsArray.line; 272 | copyNode.offset = myArgumentsArray.offset; 273 | copyNode.children.emplace_back(std::move(myArgumentsArray)); // Add the arguments array back 274 | 275 | node.children.emplace_back(std::move(copyNode)); 276 | 277 | // now changed params [...] to params +[...] 278 | break; // paramsList has been invalidated, cannot keep iterating 279 | } 280 | } 281 | } 282 | 283 | } 284 | }; 285 | 286 | //#TODO optimize param too, if non-array constant default value, see params array checking 287 | 288 | 289 | } 290 | 291 | void setupNulary() { 292 | nularyActions["true"] = [](OptimizerModuleBase::Node & node) -> void { 293 | node.type = InstructionType::push; 294 | if (!node.children.empty()) 295 | __debugbreak(); 296 | node.children.clear(); 297 | node.constant = true; 298 | node.value = true; 299 | }; 300 | 301 | nularyActions["false"] = [](OptimizerModuleBase::Node & node) -> void { 302 | node.type = InstructionType::push; 303 | if (!node.children.empty()) 304 | __debugbreak(); 305 | node.children.clear(); 306 | node.constant = true; 307 | node.value = false; 308 | }; 309 | 310 | #define NULAR_CONST_COMMAND(name) nularyActions[#name] = NularPushConstNularCommand(#name) 311 | 312 | // These commands will be evaluated once at compilation, and then used as a constant with a push instruction 313 | // ! Important, Arma's bytecode savegame serialization also needs to support these, else they will be serialized as a "false" boolean constant 314 | 315 | NULAR_CONST_COMMAND(nil); 316 | 317 | //#TODO test this 318 | // NULAR_CONST_COMMAND(missionnamespace); 319 | NULAR_CONST_COMMAND(uinamespace); 320 | //#TODO test this, its not initialized at preStart 321 | //NULAR_CONST_COMMAND(profilenamespace); 322 | 323 | // constant null types 324 | NULAR_CONST_COMMAND(objnull); 325 | NULAR_CONST_COMMAND(controlnull); 326 | NULAR_CONST_COMMAND(displaynull); 327 | NULAR_CONST_COMMAND(grpnull); 328 | NULAR_CONST_COMMAND(locationnull); 329 | NULAR_CONST_COMMAND(scriptnull); 330 | NULAR_CONST_COMMAND(confignull); 331 | 332 | //#TODO test, not sure if reliable at preStart? 333 | //NULAR_CONST_COMMAND(hasinterface); 334 | NULAR_CONST_COMMAND(linebreak); 335 | NULAR_CONST_COMMAND(configfile); 336 | } 337 | 338 | std::unordered_map> binaryActions; 339 | std::unordered_map> unaryActions; 340 | std::unordered_map> nularyActions; 341 | }; 342 | 343 | 344 | 345 | 346 | 347 | 348 | void OptimizerModuleConstantFold::optimizeConstantFold(Node& node) { 349 | auto worklist = node.bottomUpFlatten(); 350 | 351 | for (auto& it : worklist) { 352 | processNode(*it); 353 | } 354 | } 355 | 356 | void OptimizerModuleConstantFold::processNode(Node& node) { 357 | 358 | switch (node.type) { 359 | case InstructionType::endStatement: break; 360 | case InstructionType::push: { 361 | node.constant = true; 362 | }break; 363 | case InstructionType::callUnary: { 364 | if (!node.areChildrenConstant()) break; 365 | OptimizerConstantFoldActionMap::get().processUnary(node); 366 | 367 | } break; 368 | case InstructionType::callBinary: { 369 | if (!node.areChildrenConstant()) break; 370 | OptimizerConstantFoldActionMap::get().processBinary(node); 371 | 372 | } break; 373 | case InstructionType::callNular: { 374 | if (!node.areChildrenConstant()) break; 375 | OptimizerConstantFoldActionMap::get().processNulary(node); 376 | } break; 377 | case InstructionType::assignTo: break; 378 | case InstructionType::assignToLocal: break; 379 | case InstructionType::getVariable: break; 380 | case InstructionType::makeArray: { 381 | 382 | if (node.areChildrenConstant()) {//#TODO when converting to ASM check again if all elements are push 383 | 384 | //#TODO they could also be nested arrays, so makeArray with constants in it 385 | bool allPush = std::all_of(node.children.begin(), node.children.end(), [](const Node & it) 386 | { 387 | return it.type == InstructionType::push; 388 | }); 389 | 390 | 391 | 392 | //if (!allPush) { 393 | // std::stringstream buf; 394 | // node.dumpTree(buf, 0); 395 | // auto str = buf.str(); 396 | // __debugbreak(); 397 | //} 398 | // 399 | //node.value = ScriptConstantArray(); //dummy. Children are the contents 400 | //node.type = InstructionType::push; 401 | node.constant = true; 402 | } 403 | } break; 404 | default: ; 405 | } 406 | 407 | 408 | } 409 | -------------------------------------------------------------------------------- /src/optimizer/optimizerModuleConstantFold.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "optimizerModuleBase.hpp" 3 | 4 | class OptimizerModuleConstantFold : public virtual OptimizerModuleBase { 5 | public: 6 | void optimizeConstantFold(Node& node); 7 | 8 | private: 9 | void processNode(Node& node); 10 | }; -------------------------------------------------------------------------------- /src/optimizer/optimizerModuleLua.cpp: -------------------------------------------------------------------------------- 1 | #include "optimizerModuleLua.hpp" 2 | #include 3 | #if defined(_MSC_VER) 4 | #include 5 | #else 6 | #include 7 | #endif 8 | #include 9 | #include 10 | 11 | constexpr auto NularPushConstNularCommand(std::string_view name) { 12 | return [name](OptimizerModuleBase::Node& node) -> void { 13 | node.type = InstructionType::push; 14 | if (!node.children.empty()) 15 | __debugbreak(); 16 | node.children.clear(); 17 | node.constant = true; 18 | node.value = ScriptConstantNularCommand(std::string(name)); 19 | }; 20 | } 21 | 22 | 23 | 24 | void OptimizerModuleLua::optimizeLua(Node& node) { 25 | auto worklist = node.bottomUpFlatten(); 26 | 27 | for (auto& it : worklist) { 28 | processNode(*it); 29 | } 30 | } 31 | 32 | void OptimizerModuleLua::processNode(Node& node) { 33 | nodeHandler(node); 34 | } 35 | -------------------------------------------------------------------------------- /src/optimizer/optimizerModuleLua.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "optimizerModuleBase.hpp" 3 | 4 | #include "sol/sol.hpp" 5 | 6 | 7 | 8 | class OptimizerModuleLua : public virtual OptimizerModuleBase { 9 | public: 10 | void optimizeLua(Node& node); 11 | 12 | sol::protected_function nodeHandler; 13 | 14 | 15 | private: 16 | void processNode(Node& node); 17 | }; -------------------------------------------------------------------------------- /src/scriptCompiler.cpp: -------------------------------------------------------------------------------- 1 | #include "scriptCompiler.hpp" 2 | #include "commandList.hpp" 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | #include "logger.hpp" 10 | #include "optimizer/optimizerModuleBase.hpp" 11 | #include "optimizer/optimizer.h" 12 | #include "scriptSerializer.hpp" 13 | 14 | #include "fileio/default.h" 15 | #include "operators/ops.h" 16 | #include "parser/config/config_parser.hpp" 17 | #include "parser/preprocessor/default.h" 18 | #include "parser/sqf/parser.tab.hh" 19 | #include "runtime/d_string.h" 20 | #include "runtime/logging.h" 21 | 22 | std::once_flag commandMapInitFlag; 23 | 24 | class MyLogger : public StdOutLogger { 25 | std::ofstream logOut; 26 | std::mutex lock; 27 | public: 28 | MyLogger() : logOut("P:\\log.txt") {} 29 | void log(const LogMessageBase& message) override { 30 | std::lock_guard l(lock); 31 | StdOutLogger::log(message); 32 | 33 | logOut << Logger::loglevelstring(message.getLevel()) << ' ' << message.formatMessage() << std::endl; 34 | } 35 | }; 36 | 37 | 38 | static MyLogger vmlogger; 39 | 40 | void ScriptCompiler::init() 41 | { 42 | vmlogger.setEnabled(loglevel::trace, false); 43 | vm = std::make_unique(vmlogger, sqf::runtime::runtime::runtime_conf{}); 44 | 45 | vm->fileio(std::make_unique(vmlogger)); 46 | vm->parser_config(std::make_unique(vmlogger)); 47 | vm->parser_preprocessor(std::make_unique(vmlogger)); 48 | vm->parser_sqf(std::make_unique(vmlogger)); 49 | 50 | //sqf::operators::ops_dummy_binary(*vm); 51 | //sqf::operators::ops_dummy_unary(*vm); 52 | //sqf::operators::ops_dummy_nular(*vm); 53 | 54 | //sqf::operators::ops(*vm); 55 | CommandList::init(*vm); 56 | 57 | //std::call_once(commandMapInitFlag, []() { 58 | // CommandList::init(*vm); 59 | // //sqf::commandmap::get().init(); 60 | //}); 61 | 62 | addPragma({ "ASC_ignoreFile", [this](const sqf::runtime::parser::pragma& m, 63 | ::sqf::runtime::runtime& runtime, 64 | const ::sqf::runtime::diagnostics::diag_info dinf, 65 | const ::sqf::runtime::fileio::pathinfo location, 66 | const std::string& data) -> std::string 67 | { 68 | if (ignoreCurrentFile) 69 | return {}; 70 | 71 | GLogger.Log(LogLevel::Verbose, std::format("File '{}' skipped because it uses pragma ASC_ignoreFile", location.virtual_)); 72 | 73 | ignoreCurrentFile = true; 74 | return {}; // string return type is wrong, isn't used for anything 75 | } }); 76 | 77 | // Handle macros that must be resolved at game start time and cannot be precompiled 78 | auto runtimeMacroCallback = [this](const sqf::runtime::parser::macro& m, 79 | const ::sqf::runtime::diagnostics::diag_info dinf, 80 | const ::sqf::runtime::fileio::pathinfo location, 81 | const std::vector& params, 82 | ::sqf::runtime::runtime& runtime) -> std::string 83 | { 84 | if (ignoreCurrentFile) 85 | return {}; 86 | 87 | GLogger.Log(LogLevel::Verbose, std::format("File '{}' skipped because it uses runtime-only macro '{}'", location.virtual_, m.name())); 88 | 89 | ignoreCurrentFile = true; 90 | return {}; 91 | }; 92 | 93 | // https://community.bistudio.com/wiki/PreProcessor_Commands#has_include 94 | for (auto& it : { 95 | "__has_include", 96 | "__DATE_ARR__", 97 | "__DATE_STR__", 98 | "__DATE_STR_ISO8601__", 99 | "__TIME__", 100 | "__TIME_UTC__", 101 | "__DAY__", 102 | "__MONTH__", 103 | "__YEAR__", 104 | "__TIMESTAMP_UTC__", 105 | 106 | // We could support these, shouldn't make a different, random number is random 107 | "__RAND_INT8__", 108 | "__RAND_INT16__", 109 | "__RAND_INT32__", 110 | "__RAND_INT64__", 111 | "__RAND_UINT8__", 112 | "__RAND_UINT16__", 113 | "__RAND_UINT32__", 114 | "__RAND_UINT64__" 115 | // SQF-VM supports these, but it sets wrong values. We want runtime ones. 116 | "__GAME_VER__", 117 | "__GAME_VER_MAJ__", 118 | "__GAME_VER_MIN__", 119 | "__GAME_BUILD__", 120 | 121 | "__A3_DIAG__", 122 | "__A3_DEBUG__", 123 | "__A3_EXPERIMENTAL__", 124 | "__A3_PROFILING__" 125 | }) 126 | addMacro({ it, runtimeMacroCallback }); 127 | 128 | addMacro({ "__ARMA__", "1" }); // Only A3 has bytecode 129 | addMacro({ "__ARMA3__", "1" }); // Only A3 has bytecode 130 | 131 | } 132 | 133 | ScriptCompiler::ScriptCompiler(const std::vector& includePaths) { 134 | init(); 135 | initIncludePaths(includePaths); 136 | } 137 | 138 | ScriptCompiler::ScriptCompiler() { 139 | init(); 140 | } 141 | 142 | std::string PathToWindowsString(const std::filesystem::path& path) 143 | { 144 | std::string result = path.generic_string(); 145 | std::replace(result.begin(), result.end(), '/', '\\'); 146 | return result; 147 | } 148 | 149 | CompiledCodeData ScriptCompiler::compileScript(std::filesystem::path physicalPath, std::filesystem::path virtualPath) { 150 | std::ifstream inputFile(physicalPath); 151 | 152 | auto filesize = std::filesystem::file_size(physicalPath); 153 | if (filesize == 0) // uh. oki 154 | return CompiledCodeData(); 155 | 156 | std::string scriptCode; 157 | scriptCode.resize(filesize); 158 | inputFile.read(scriptCode.data(), filesize); 159 | 160 | // Strip UTF-8 BOM 161 | if ( 162 | static_cast(scriptCode[0]) == 0xef && 163 | static_cast(scriptCode[1]) == 0xbb && 164 | static_cast(scriptCode[2]) == 0xbf 165 | ) { 166 | scriptCode.erase(0, 3); 167 | } 168 | 169 | // #OPTION force all script files to contain a include 170 | //if (scriptCode.find("script_component.hpp\"") == std::string::npos) { 171 | // throw std::domain_error("no include"); 172 | //} 173 | 174 | auto preprocessedScript = vm->parser_preprocessor().preprocess(*vm, scriptCode, sqf::runtime::fileio::pathinfo(physicalPath.string(), PathToWindowsString(virtualPath)) ); 175 | if (!preprocessedScript) { 176 | //__debugbreak(); 177 | return CompiledCodeData(); 178 | } 179 | 180 | if (ignoreCurrentFile) 181 | { 182 | ignoreCurrentFile = false; 183 | return CompiledCodeData(); 184 | } 185 | 186 | bool errorflag = false; 187 | 188 | 189 | sqf::parser::sqf::parser sqfParser(vmlogger); 190 | sqf::parser::sqf::bison::astnode ast; 191 | sqf::parser::sqf::tokenizer tokenizer(preprocessedScript->begin(), preprocessedScript->end(), virtualPath.string()); 192 | auto errflag = !sqfParser.get_tree(*vm, tokenizer, &ast); 193 | if (errflag || ast.children.empty()) { 194 | //__debugbreak(); 195 | return CompiledCodeData(); 196 | } 197 | 198 | //print_navigate_ast(&std::cout, ast, sqf::parse::sqf::astkindname); 199 | 200 | CompiledCodeData stuff; 201 | CompileTempData temp; 202 | ScriptCodePiece mainCode; 203 | 204 | 205 | if (true) { 206 | auto& statementsNode = ast.children[0]; 207 | if (statementsNode.kind != sqf::parser::sqf::bison::astkind::STATEMENTS) 208 | __debugbreak(); 209 | 210 | ///statementsNode.kind = sqf::parser::sqf::bison::astkind::CODE; 211 | 212 | auto node = OptimizerModuleBase::nodeFromAST(statementsNode); 213 | 214 | //std::ofstream nodeo("P:\\node.txt"); 215 | //node.dumpTree(nodeo, 0); 216 | //nodeo.close(); 217 | 218 | //auto res = node.bottomUpFlatten(); 219 | 220 | Optimizer opt; 221 | 222 | opt.optimize(node); 223 | 224 | //std::ofstream nodeop("P:\\nodeOpt.txt"); 225 | //node.dumpTree(nodeop, 0); 226 | //nodeop.close(); 227 | 228 | ASTToInstructions(stuff, temp, mainCode.code, node); 229 | mainCode.contentString = stuff.AddConstant(std::move(*preprocessedScript)); 230 | stuff.codeIndex = stuff.AddConstant(std::move(mainCode)); 231 | 232 | //std::ofstream output2("P:\\outOpt.sqfa", std::ofstream::binary); 233 | //ScriptSerializer::compiledToHumanReadable(stuff, output2); 234 | //output2.flush(); 235 | } else { 236 | ASTToInstructions(stuff, temp, mainCode.code, ast); 237 | mainCode.contentString = stuff.AddConstant(std::move(*preprocessedScript)); 238 | stuff.codeIndex = stuff.AddConstant(std::move(mainCode)); 239 | } 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | //auto outputPath2 = file.parent_path() / (file.stem().string() + ".sqfa"); 249 | //std::ofstream output2(outputPath2, std::ofstream::binary); 250 | //std::ofstream output2("P:\\outOrig.sqfa", std::ofstream::binary); 251 | //ScriptSerializer::compiledToHumanReadable(stuff, output2); 252 | //output2.flush(); 253 | return stuff; 254 | } 255 | 256 | void ScriptCompiler::ASTToInstructions(CompiledCodeData& output, CompileTempData& temp, std::vector& instructions, const astnode& node) const { 257 | auto getFileIndex = [&](const std::string& filename) -> uint8_t 258 | { 259 | auto found = temp.fileLoc.find(filename); 260 | if (found != temp.fileLoc.end()) 261 | return found->second; 262 | auto index = static_cast(output.fileNames.size()); 263 | output.fileNames.emplace_back(filename); 264 | temp.fileLoc.insert({ filename, index }); 265 | return index; 266 | }; 267 | //#TODO get constant index and keep sets of bool/float/string constants 268 | 269 | auto nodeType = node.kind; 270 | switch (nodeType) { 271 | 272 | case sqf::parser::sqf::bison::astkind::ASSIGNMENT: { 273 | auto varname = std::string(node.children[0].token.contents); 274 | //need value on stack first 275 | ASTToInstructions(output, temp, instructions, node.children[1]); 276 | std::transform(varname.begin(), varname.end(), varname.begin(), ::tolower); 277 | instructions.emplace_back(ScriptInstruction{ 278 | nodeType == sqf::parser::sqf::bison::astkind::ASSIGNMENT ? 279 | InstructionType::assignTo 280 | : 281 | InstructionType::assignToLocal 282 | , node.token.offset, getFileIndex(*node.token.path), node.token.line, varname }); 283 | } break; 284 | case sqf::parser::sqf::bison::astkind::ASSIGNMENT_LOCAL: { 285 | auto varname = std::string(node.token.contents); 286 | //need value on stack first 287 | ASTToInstructions(output, temp, instructions, node.children[0]); 288 | std::transform(varname.begin(), varname.end(), varname.begin(), ::tolower); 289 | instructions.emplace_back(ScriptInstruction{ 290 | nodeType == sqf::parser::sqf::bison::astkind::ASSIGNMENT ? 291 | InstructionType::assignTo 292 | : 293 | InstructionType::assignToLocal 294 | , node.token.offset, getFileIndex(*node.token.path), node.token.line, varname }); 295 | } break; 296 | case sqf::parser::sqf::bison::astkind::EXP0: 297 | case sqf::parser::sqf::bison::astkind::EXP1: 298 | case sqf::parser::sqf::bison::astkind::EXP2: 299 | case sqf::parser::sqf::bison::astkind::EXP3: 300 | //number 301 | //binaryop 302 | //number 303 | case sqf::parser::sqf::bison::astkind::EXP4: 304 | //unary left arg 305 | //binary command 306 | //code on right 307 | case sqf::parser::sqf::bison::astkind::EXP5: 308 | case sqf::parser::sqf::bison::astkind::EXP6: 309 | //constant 310 | //binaryop 311 | //unaryop 312 | case sqf::parser::sqf::bison::astkind::EXP7: 313 | case sqf::parser::sqf::bison::astkind::EXP8: 314 | case sqf::parser::sqf::bison::astkind::EXP9: { 315 | 316 | //get left arg on stack 317 | ASTToInstructions(output, temp, instructions, node.children[0]); 318 | //get right arg on stack 319 | ASTToInstructions(output, temp, instructions, node.children[1]); 320 | //push binary op 321 | auto name = std::string(node.token.contents); 322 | std::transform(name.begin(), name.end(), name.begin(), ::tolower); 323 | instructions.emplace_back(ScriptInstruction{ InstructionType::callBinary, node.token.offset, getFileIndex(*node.token.path), node.token.line, name }); 324 | 325 | break; 326 | } 327 | case sqf::parser::sqf::bison::astkind::EXPN: { 328 | //push nular op 329 | auto name = std::string(node.token.contents); 330 | std::transform(name.begin(), name.end(), name.begin(), ::tolower); 331 | instructions.emplace_back(ScriptInstruction{ InstructionType::callNular, node.token.offset, getFileIndex(*node.token.path), node.token.line, name }); 332 | break; 333 | } 334 | case sqf::parser::sqf::bison::astkind::EXPU: { 335 | //unary operator 336 | //right arg 337 | 338 | //get right arg on stack 339 | ASTToInstructions(output, temp, instructions, node.children[0]); 340 | //push unary op 341 | auto name = std::string(node.token.contents); 342 | std::transform(name.begin(), name.end(), name.begin(), ::tolower); 343 | instructions.emplace_back(ScriptInstruction{ InstructionType::callUnary, node.token.offset, getFileIndex(*node.token.path), node.token.line, name }); 344 | break; 345 | } 346 | case sqf::parser::sqf::bison::astkind::NUMBER: 347 | case sqf::parser::sqf::bison::astkind::HEXNUMBER: { 348 | ScriptConstant newConst; 349 | 350 | float val; 351 | auto res = 352 | (nodeType == sqf::parser::sqf::bison::astkind::HEXNUMBER) ? 353 | std::from_chars(node.token.contents.data()+2, node.token.contents.data() + node.token.contents.size(), val, std::chars_format::hex) 354 | : 355 | std::from_chars(node.token.contents.data(), node.token.contents.data() + node.token.contents.size(), val); 356 | if (res.ec == std::errc::invalid_argument) { 357 | throw std::runtime_error("invalid scalar at: " + *node.token.path + ":" + std::to_string(node.token.line)); 358 | } 359 | else if (res.ec == std::errc::result_out_of_range) { 360 | throw std::runtime_error("scalar out of range at: " + *node.token.path + ":" + std::to_string(node.token.line)); 361 | } 362 | newConst = val; 363 | auto index = output.AddConstant(std::move(newConst)); 364 | 365 | instructions.emplace_back(ScriptInstruction{ InstructionType::push, node.token.offset, getFileIndex(*node.token.path), node.token.line, index }); 366 | break; 367 | } 368 | case sqf::parser::sqf::bison::astkind::IDENT: { 369 | //getvariable 370 | auto varname = std::string(node.token.contents); 371 | std::transform(varname.begin(), varname.end(), varname.begin(), ::tolower); 372 | instructions.emplace_back(ScriptInstruction{ InstructionType::getVariable, node.token.offset, getFileIndex(*node.token.path), node.token.line, varname }); 373 | break; 374 | } 375 | case sqf::parser::sqf::bison::astkind::STRING: { 376 | ScriptConstant newConst; 377 | newConst = ::sqf::types::d_string::from_sqf(std::string(node.token.contents)); 378 | auto index = output.AddConstant(std::move(newConst)); 379 | 380 | instructions.emplace_back(ScriptInstruction{ InstructionType::push, node.token.offset, getFileIndex(*node.token.path), node.token.line, index }); 381 | break; 382 | } 383 | 384 | case sqf::parser::sqf::bison::astkind::BOOLEAN_TRUE: { 385 | ScriptConstant newConst; 386 | newConst = true; 387 | auto index = output.AddConstant(std::move(newConst)); 388 | 389 | instructions.emplace_back(ScriptInstruction{ InstructionType::push, node.token.offset, getFileIndex(*node.token.path), node.token.line, index }); 390 | break; 391 | } 392 | 393 | case sqf::parser::sqf::bison::astkind::BOOLEAN_FALSE: { 394 | ScriptConstant newConst; 395 | //::sqf::types::d_string::from_sqf(std::string(node.token.contents)) 396 | newConst = false; 397 | auto index = output.AddConstant(std::move(newConst)); 398 | 399 | instructions.emplace_back(ScriptInstruction{ InstructionType::push, node.token.offset, getFileIndex(*node.token.path), node.token.line, index }); 400 | break; 401 | } 402 | case sqf::parser::sqf::bison::astkind::CODE: { 403 | ScriptConstant newConst; 404 | std::vector instr; 405 | for (auto& it : node.children) { 406 | instr.emplace_back(ScriptInstruction{ InstructionType::endStatement, node.token.offset, 0, 0 }); 407 | ASTToInstructions(output, temp, instr, it); 408 | } 409 | 410 | auto codeEnd = node.token.offset; 411 | 412 | if (!node.children.empty()) { 413 | 414 | auto* statements = &node.children[0]; 415 | if (statements->kind != sqf::parser::sqf::bison::astkind::STATEMENTS) 416 | __debugbreak(); 417 | 418 | size_t lastToken = 0; 419 | std::vector::const_iterator> lastChildren; 420 | 421 | while (!statements->children.empty()) { 422 | const auto& lastChild = (statements->children.end() - 1); 423 | 424 | lastToken = lastChild->token.offset + lastChild->token.contents.size(); 425 | statements = &(*lastChild); 426 | lastChildren.emplace_back(lastChild); 427 | } 428 | 429 | // we also need to travel the full way back. Just finding next } is not sufficient, we may be multiple code levels deep 430 | 431 | 432 | auto endData = node.token.contents.data() + (lastToken - node.token.offset); 433 | 434 | std::reverse(lastChildren.begin(), lastChildren.end()); 435 | 436 | for (auto& it : lastChildren) { 437 | switch (it->kind) { 438 | case sqf::parser::sqf::bison::astkind::CODE: 439 | // find ending } 440 | while (*endData && *endData != '}') { 441 | ++endData; 442 | ++lastToken; 443 | } 444 | // after ending } 445 | ++endData; ++lastToken; 446 | break; 447 | case sqf::parser::sqf::bison::astkind::ARRAY: 448 | // find ending ] 449 | while (*endData && *endData != ']') { 450 | ++endData; ++lastToken; 451 | } 452 | // after ending ] 453 | ++endData; ++lastToken; 454 | break; 455 | default:; 456 | } 457 | } 458 | 459 | 460 | while (*endData && *endData != '}') { 461 | ++endData; 462 | ++lastToken; 463 | } 464 | // we stop BEFORE ending }, we only want the inner code. 465 | 466 | codeEnd = lastToken; 467 | newConst = ScriptCodePiece(std::move(instr), codeEnd - node.token.offset - 1, node.token.offset + 1); 468 | } else { 469 | newConst = ScriptCodePiece(std::move(instr), 0, node.token.offset + 1); 470 | } 471 | 472 | auto index = output.AddConstant(std::move(newConst)); 473 | 474 | instructions.emplace_back(ScriptInstruction{ InstructionType::push, node.token.offset, getFileIndex(*node.token.path), node.token.line, index }); 475 | break; 476 | } 477 | case sqf::parser::sqf::bison::astkind::ARRAY: { 478 | //push elements first 479 | for (auto& it : node.children) 480 | ASTToInstructions(output, temp, instructions, it); 481 | 482 | //#TODO can already check here if all arguments are const and push a const 483 | //AST is const function. That just checks whether the whole tree only contains constants 484 | 485 | 486 | //make array instruction 487 | //array instruction has size as argument 488 | instructions.emplace_back(ScriptInstruction{ InstructionType::makeArray, node.token.offset, getFileIndex(*node.token.path), node.token.line, (uint16_t)node.children.size() }); 489 | 490 | break; 491 | } 492 | //case sqf::parser::sqf::bison::astkind::NA: 493 | //case sqf::parser::sqf::bison::astkind::SQF: 494 | //case sqf::parser::sqf::bison::astkind::STATEMENT: 495 | //case sqf::parser::sqf::bison::astkind::BRACKETS: 496 | // for (auto& it : node.children) 497 | // stuffAST(output, instructions, it); 498 | default: 499 | for (size_t i = 0; i < node.children.size(); i++) { 500 | if (i != 0 || instructions.empty()) //end statement 501 | instructions.emplace_back(ScriptInstruction{ InstructionType::endStatement, node.token.offset, 0, 0 }); 502 | ASTToInstructions(output, temp, instructions, node.children[i]); 503 | } 504 | } 505 | } 506 | 507 | ScriptConstantArray ScriptCompiler::ASTParseArray(CompiledCodeData& output, CompileTempData& temp, const OptimizerModuleBase::Node& node) const { 508 | ScriptConstantArray newConst; 509 | for (auto& it : node.children) { 510 | if (it.value.index() == 0) {//is code 511 | ScriptCodePiece codeConst; 512 | std::vector instr; 513 | for (auto& codeIt : it.children) { 514 | 515 | //std::stringstream dbg; 516 | //codeIt.dumpTree(dbg, 0); 517 | //auto res = dbg.str(); 518 | 519 | instr.emplace_back(ScriptInstruction{ InstructionType::endStatement, node.offset, 0, 0 }); 520 | ASTToInstructions(output, temp, instr, codeIt); 521 | } 522 | codeConst.contentString = std::get(it.value).contentString; 523 | codeConst.code = std::move(instr); 524 | newConst.content.emplace_back(std::move(codeConst)); 525 | } else if (it.value.index() == 4) {//array 526 | newConst.content.emplace_back(ASTParseArray(output, temp, it)); 527 | } else { 528 | newConst.content.emplace_back(it.value); 529 | } 530 | } 531 | return newConst; 532 | } 533 | 534 | void ScriptCompiler::ASTToInstructions(CompiledCodeData& output, CompileTempData& temp, 535 | std::vector& instructions, const OptimizerModuleBase::Node& node) const { 536 | 537 | auto getFileIndex = [&](const std::string & filename) -> uint8_t 538 | { 539 | auto found = temp.fileLoc.find(filename); 540 | if (found != temp.fileLoc.end()) 541 | return found->second; 542 | auto index = static_cast(output.fileNames.size()); 543 | output.fileNames.emplace_back(filename); 544 | temp.fileLoc.insert({ filename, index }); 545 | return index; 546 | }; 547 | 548 | 549 | switch (node.type) { 550 | case InstructionType::push: { 551 | switch (getConstantType(node.value)) { 552 | case ConstantType::code: {//Code 553 | ScriptConstant newConst; 554 | std::vector instr; 555 | for (auto& it : node.children) { 556 | instr.emplace_back(ScriptInstruction{ InstructionType::endStatement, node.offset, 0, 0 }); 557 | ASTToInstructions(output, temp, instr, it); 558 | } 559 | 560 | newConst = node.value; 561 | std::get(newConst).code = std::move(instr); 562 | auto index = output.AddConstant(std::move(newConst)); 563 | 564 | instructions.emplace_back(ScriptInstruction{ InstructionType::push, node.offset, getFileIndex(node.file), node.line, index }); 565 | } break; 566 | case ConstantType::string: 567 | case ConstantType::scalar: 568 | case ConstantType::boolean: 569 | case ConstantType::nularCommand: 570 | { 571 | auto index = output.AddConstant(node.value); 572 | instructions.emplace_back(ScriptInstruction{ InstructionType::push, node.offset, getFileIndex(node.file), node.line, index }); 573 | } break; 574 | case ConstantType::array: {//Array 575 | auto index = output.AddConstant(ASTParseArray(output, temp, node)); 576 | 577 | instructions.emplace_back(ScriptInstruction{ InstructionType::push, node.offset, getFileIndex(node.file), node.line, index }); 578 | } break; 579 | default: __debugbreak(); 580 | } 581 | 582 | }break; 583 | case InstructionType::callUnary: { 584 | 585 | ASTToInstructions(output, temp, instructions, node.children[0]); 586 | //push unary op 587 | auto name = std::get(node.value); 588 | std::transform(name.begin(), name.end(), name.begin(), ::tolower); 589 | instructions.emplace_back(ScriptInstruction{ InstructionType::callUnary, node.offset, getFileIndex(node.file), node.line, name }); 590 | 591 | } break; 592 | case InstructionType::callBinary: { 593 | 594 | 595 | //get left arg on stack 596 | ASTToInstructions(output, temp, instructions, node.children[0]); 597 | //get right arg on stack 598 | ASTToInstructions(output, temp, instructions, node.children[1]); 599 | //push binary op 600 | auto name = std::get(node.value); 601 | std::transform(name.begin(), name.end(), name.begin(), ::tolower); 602 | instructions.emplace_back(ScriptInstruction{ InstructionType::callBinary, node.offset, getFileIndex(node.file), node.line, name }); 603 | 604 | 605 | 606 | } break; 607 | case InstructionType::callNular: { 608 | auto name = std::get(node.value); 609 | std::transform(name.begin(), name.end(), name.begin(), ::tolower); 610 | instructions.emplace_back(ScriptInstruction{ InstructionType::callNular, node.offset, getFileIndex(node.file), node.line, name }); 611 | 612 | } break; 613 | case InstructionType::assignTo: 614 | case InstructionType::assignToLocal: { 615 | 616 | auto varname = std::get(node.value); 617 | //need value on stack first 618 | ASTToInstructions(output, temp, instructions, node.children[0]); 619 | std::transform(varname.begin(), varname.end(), varname.begin(), ::tolower); 620 | instructions.emplace_back(ScriptInstruction{ node.type, node.offset, getFileIndex(node.file), node.line, varname }); 621 | 622 | } break; 623 | case InstructionType::getVariable: { 624 | auto varname = std::get(node.value); 625 | std::transform(varname.begin(), varname.end(), varname.begin(), ::tolower); 626 | instructions.emplace_back(ScriptInstruction{ InstructionType::getVariable, node.offset, getFileIndex(node.file), node.line, varname }); 627 | } break; 628 | case InstructionType::makeArray: { 629 | for (auto& it : node.children) 630 | ASTToInstructions(output, temp, instructions, it); 631 | 632 | //#TODO can already check here if all arguments are const and push a const 633 | 634 | instructions.emplace_back(ScriptInstruction{ InstructionType::makeArray, node.offset, getFileIndex(node.file), node.line, (uint16_t)node.children.size() }); 635 | } break; 636 | 637 | 638 | case InstructionType::endStatement: { 639 | for (size_t i = 0; i < node.children.size(); i++) { 640 | if (i != 0 || instructions.empty()) //end statement 641 | instructions.emplace_back(ScriptInstruction{ InstructionType::endStatement, node.offset, 0, 0 }); 642 | ASTToInstructions(output, temp, instructions, node.children[i]); 643 | } 644 | 645 | } break; 646 | } 647 | } 648 | 649 | CompiledCodeData ScriptCompiler::compileScriptLua(std::filesystem::path physicalPath, std::filesystem::path virtualPath, OptimizerModuleLua& optimizer, const std::filesystem::path& outputFile) 650 | { 651 | std::ifstream inputFile(physicalPath); 652 | 653 | auto filesize = std::filesystem::file_size(physicalPath); 654 | if (filesize == 0) // uh. oki 655 | return CompiledCodeData(); 656 | 657 | std::string scriptCode; 658 | scriptCode.resize(filesize); 659 | inputFile.read(scriptCode.data(), filesize); 660 | 661 | if ( 662 | static_cast(scriptCode[0]) == 0xef && 663 | static_cast(scriptCode[1]) == 0xbb && 664 | static_cast(scriptCode[2]) == 0xbf 665 | ) { 666 | scriptCode.erase(0, 3); 667 | } 668 | 669 | auto preprocessedScript = vm->parser_preprocessor().preprocess(*vm, scriptCode, sqf::runtime::fileio::pathinfo(physicalPath.string(), virtualPath.string())); 670 | if (!preprocessedScript) { 671 | //__debugbreak(); 672 | return CompiledCodeData(); 673 | } 674 | bool errorflag = false; 675 | 676 | 677 | sqf::parser::sqf::parser sqfParser(vmlogger); 678 | sqf::parser::sqf::bison::astnode ast; 679 | sqf::parser::sqf::tokenizer tokenizer(preprocessedScript->begin(), preprocessedScript->end(), virtualPath.string()); 680 | auto errflag = !sqfParser.get_tree(*vm, tokenizer, &ast); 681 | if (errflag || ast.children.empty()) { 682 | //__debugbreak(); 683 | return CompiledCodeData(); 684 | } 685 | 686 | //print_navigate_ast(&std::cout, ast, sqf::parse::sqf::astkindname); 687 | 688 | CompiledCodeData stuff; 689 | CompileTempData temp; 690 | ScriptCodePiece mainCode; 691 | 692 | 693 | auto& statementsNode = ast.children[0]; 694 | if (statementsNode.kind != sqf::parser::sqf::bison::astkind::STATEMENTS) 695 | __debugbreak(); 696 | 697 | auto node = OptimizerModuleBase::nodeFromAST(statementsNode); 698 | 699 | Optimizer opt; 700 | optimizer.optimizeLua(node); 701 | 702 | ASTToInstructions(stuff, temp, mainCode.code, node); 703 | mainCode.contentString = stuff.AddConstant(std::move(*preprocessedScript)); 704 | stuff.codeIndex = stuff.AddConstant(std::move(mainCode)); 705 | 706 | std::ofstream output2(outputFile, std::ofstream::binary); 707 | ScriptSerializer::compiledToHumanReadable(stuff, output2); 708 | output2.flush(); 709 | 710 | return stuff; 711 | } 712 | 713 | void ScriptCompiler::initIncludePaths(const std::vector& paths) { 714 | for (auto& includefolder : paths) { 715 | 716 | if (includefolder.string().length() == 3 && includefolder.generic_string().substr(1) == ":/") { 717 | // pdrive 718 | 719 | //const std::filesystem::path ignoreGit(".git"); 720 | //const std::filesystem::path ignoreSvn(".svn"); 721 | // 722 | ////recursively search for pboprefix 723 | //for (auto i = std::filesystem::directory_iterator(includefolder); 724 | // i != std::filesystem::directory_iterator(); 725 | // ++i) 726 | //{ 727 | // if (!i->is_directory()) continue; 728 | // 729 | // if ((i->path().filename() == ignoreGit || i->path().filename() == ignoreSvn)) 730 | // { 731 | // continue; 732 | // } 733 | // 734 | // 735 | // vm->fileio().add_mapping(i->path().string(), i->path().filename().string()); 736 | //} 737 | auto str = includefolder.lexically_normal().string(); 738 | if (str.back() == std::filesystem::path::preferred_separator) 739 | str.pop_back(); 740 | vm->fileio().add_mapping(str, "\\"); 741 | } else { 742 | vm->fileio().add_mapping_auto(includefolder.string()); 743 | } 744 | 745 | 746 | } 747 | } 748 | 749 | void ScriptCompiler::addMacro(sqf::runtime::parser::macro macro) { 750 | vm->parser_preprocessor().push_back(macro); 751 | } 752 | 753 | void ScriptCompiler::addPragma(sqf::runtime::parser::pragma pragma) { 754 | vm->parser_preprocessor().push_back(pragma); 755 | } 756 | 757 | std::string ScriptCompiler::preprocessScript(std::filesystem::path physicalPath, std::filesystem::path virtualPath) { 758 | std::ifstream inputFile(physicalPath); 759 | 760 | auto filesize = std::filesystem::file_size(physicalPath); 761 | if (filesize == 0) // uh. oki 762 | return ""; 763 | 764 | std::string scriptCode; 765 | scriptCode.resize(filesize); 766 | inputFile.read(scriptCode.data(), filesize); 767 | 768 | if ( 769 | static_cast(scriptCode[0]) == 0xef && 770 | static_cast(scriptCode[1]) == 0xbb && 771 | static_cast(scriptCode[2]) == 0xbf 772 | ) { 773 | scriptCode.erase(0, 3); 774 | } 775 | 776 | // #OPTION force all script files to contain a include 777 | //if (scriptCode.find("script_component.hpp\"") == std::string::npos) { 778 | // throw std::domain_error("no include"); 779 | //} 780 | try { 781 | auto preprocessedScript = vm->parser_preprocessor().preprocess(*vm, scriptCode, sqf::runtime::fileio::pathinfo(physicalPath.string(), virtualPath.string())); 782 | if (!preprocessedScript) { 783 | //__debugbreak(); 784 | return ""; 785 | } 786 | 787 | return std::string(preprocessedScript->begin(), preprocessedScript->end()); 788 | } catch (std::exception ex) { 789 | __debugbreak(); 790 | } 791 | 792 | return ""; 793 | } 794 | -------------------------------------------------------------------------------- /src/scriptCompiler.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "compiledCode.hpp" 3 | #include 4 | #include 5 | #include 6 | #include "optimizer/optimizerModuleBase.hpp" 7 | 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "optimizer/optimizerModuleLua.hpp" 14 | using astnode = sqf::parser::sqf::bison::astnode; 15 | 16 | class ScriptCompiler { 17 | void init(); 18 | public: 19 | ScriptCompiler(const std::vector& includePaths); 20 | ScriptCompiler(); 21 | 22 | 23 | CompiledCodeData compileScript(std::filesystem::path physicalPath, std::filesystem::path virtualPath); 24 | 25 | CompiledCodeData compileScriptLua(std::filesystem::path physicalPath, std::filesystem::path virtualPath, OptimizerModuleLua& optimizer, const std::filesystem::path& outputFile); 26 | void initIncludePaths(const std::vector&); 27 | void addMacro(sqf::runtime::parser::macro macro); 28 | void addPragma(sqf::runtime::parser::pragma pragma); 29 | std::string preprocessScript(std::filesystem::path physicalPath, std::filesystem::path virtualPath); 30 | 31 | private: 32 | struct CompileTempData { 33 | std::unordered_map fileLoc; 34 | }; 35 | 36 | void ASTToInstructions(CompiledCodeData& output, CompileTempData& temp, std::vector& instructions, const astnode& node) const; 37 | 38 | 39 | ScriptConstantArray ASTParseArray(CompiledCodeData& output, CompileTempData& temp, const OptimizerModuleBase::Node& node) const; 40 | void ASTToInstructions(CompiledCodeData& output, CompileTempData& temp, std::vector& instructions, const OptimizerModuleBase::Node& node) const; 41 | std::unique_ptr vm; 42 | bool ignoreCurrentFile = false; 43 | }; 44 | -------------------------------------------------------------------------------- /src/scriptSerializer.cpp: -------------------------------------------------------------------------------- 1 | #include "scriptSerializer.hpp" 2 | #include 3 | #define ZSTD_STATIC_LINKING_ONLY // ZSTD_findDecompressedSize 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | static constexpr const int compressionLevel = 22; 10 | 11 | void blaBla(const CompiledCodeData& code, const std::vector& inst, std::ostream& output); 12 | 13 | void blaBLaConstant(const CompiledCodeData& code, const ScriptConstant& constant, std::ostream& output, bool inArray = false) { 14 | 15 | switch (getConstantType(constant)) { 16 | case ConstantType::code: 17 | output << "push CODE {\n"; 18 | blaBla(code, std::get<0>(constant).code, output); 19 | output << "}\n"; 20 | break; 21 | case ConstantType::string: 22 | if (!inArray) 23 | output << "push STRING " << std::get(constant) << "\n"; 24 | else 25 | output << std::get(constant) << ", "; 26 | break; 27 | case ConstantType::scalar: 28 | if (!inArray) 29 | output << "push SCALAR " << std::get(constant) << "\n"; 30 | else 31 | output << std::get(constant) << ", "; 32 | break; 33 | case ConstantType::boolean: 34 | if (!inArray) 35 | output << "push BOOL " << std::get(constant) << "\n"; 36 | else 37 | output << std::get(constant) << ", "; 38 | break; 39 | case ConstantType::array: 40 | if (!inArray) 41 | output << "push ARRAY ["; 42 | else 43 | output << "[\n"; 44 | for (auto& it : std::get<4>(constant).content) 45 | blaBLaConstant(code, it, output, true); 46 | output.seekp(-2, SEEK_CUR); 47 | output << "]\n"; 48 | break; 49 | default:; 50 | } 51 | 52 | } 53 | 54 | void blaBla(const CompiledCodeData& code, const std::vector& inst , std::ostream& output) {//#TODO move into proper func 55 | for (auto& it : inst) { 56 | switch (it.type) { 57 | case InstructionType::endStatement: 58 | output << "endStatement\n"; 59 | break; 60 | case InstructionType::push: { 61 | auto index = std::get(it.content); 62 | auto constant = code.constants[index]; 63 | blaBLaConstant(code, constant, output); 64 | } break; 65 | case InstructionType::callUnary: 66 | output << "callUnary " << std::get(it.content) << "\n"; 67 | break; 68 | case InstructionType::callBinary: 69 | output << "callBinary " << std::get(it.content) << "\n"; 70 | break; 71 | case InstructionType::callNular: 72 | output << "callNular " << std::get(it.content) << "\n"; 73 | break; 74 | case InstructionType::assignTo: 75 | output << "assignTo " << std::get(it.content) << "\n"; 76 | break; 77 | case InstructionType::assignToLocal: 78 | output << "assignToLocal " << std::get(it.content) << "\n"; 79 | break; 80 | case InstructionType::getVariable: 81 | output << "getVariable " << std::get(it.content) << "\n"; 82 | break; 83 | case InstructionType::makeArray: 84 | output << "makeArray " << std::get(it.content) << "\n"; 85 | break; 86 | default:; 87 | } 88 | } 89 | } 90 | 91 | 92 | void ScriptSerializer::compiledToHumanReadable(const CompiledCodeData& code, std::ostream& output) { 93 | 94 | //#TODO fix this 95 | //Check output of 96 | // params ["_unit", "_pos", ["_target", objNull], ["_buildings", []]]; 97 | // + [1, 2, 3]; 98 | // _buildings pushBack[1, 2, 3]; 99 | 100 | // array fuckup, no quotes on string constants inside the array 101 | 102 | blaBla(code, std::get<0>(code.constants[code.codeIndex]).code, output); 103 | } 104 | 105 | /* 106 | block type 107 | content 108 | 109 | 110 | */ 111 | 112 | enum class SerializedBlockType { 113 | constant, 114 | constantCompressed, 115 | locationInfo, 116 | code, 117 | codeDebug, 118 | commandNameDirectory // lookup table for script command/variable names. Optional 119 | }; 120 | 121 | 122 | template 123 | Type readT(std::istream& stream) { 124 | Type ret; 125 | stream.read(reinterpret_cast(&ret), sizeof(Type)); 126 | return ret; 127 | } 128 | 129 | 130 | template 131 | void writeT(Type data, std::ostream& stream) { 132 | stream.write(reinterpret_cast(&data), sizeof(Type)); 133 | } 134 | 135 | 136 | #include 137 | 138 | void ScriptSerializer::compiledToBinary(const CompiledCodeData& code, std::ostream& output) { 139 | writeT(code.version, output); //version 140 | output.flush(); 141 | 142 | // lookup table for script command/variable names. Optional, if code.commandNameDirectory is empty, script commands will be serialized as string names 143 | { 144 | 145 | std::set commandNameDirectory; 146 | collectCommandNames(code, commandNameDirectory); 147 | 148 | code.commandNameDirectory.clear(); 149 | std::copy(commandNameDirectory.begin(), commandNameDirectory.end(), std::back_inserter(code.commandNameDirectory)); 150 | std::sort(code.commandNameDirectory.begin(), code.commandNameDirectory.end()); 151 | 152 | if (code.commandNameDirectory.size() != static_cast(code.commandNameDirectory.size())) { 153 | __debugbreak(); 154 | // too big. 155 | code.commandNameDirectory.clear(); 156 | } 157 | 158 | 159 | 160 | if (!code.commandNameDirectory.empty()) { 161 | writeT(static_cast(SerializedBlockType::commandNameDirectory), output); 162 | 163 | std::ostringstream buffer(std::ostringstream::binary); 164 | writeT(static_cast(code.commandNameDirectory.size()), buffer); 165 | for (auto& it : code.commandNameDirectory) 166 | writeString(buffer, it); 167 | 168 | 169 | 170 | 171 | auto bufferContent = buffer.str(); 172 | 173 | lzokay::Dict<> dict; 174 | std::size_t estimated_size = lzokay::compress_worst_size(bufferContent.size()); 175 | std::unique_ptr compressed(new uint8_t[estimated_size]); 176 | std::size_t compressed_size; 177 | auto error = lzokay::compress((const uint8_t*)bufferContent.data(), bufferContent.size(), compressed.get(), estimated_size, 178 | compressed_size, dict); 179 | if (error < lzokay::EResult::Success) 180 | __debugbreak(); 181 | 182 | writeT(static_cast(bufferContent.size()), output); // uncompressed size 183 | writeT(static_cast(2), output); // compression method, always 2 184 | output.write((const char*)compressed.get(), compressed_size); 185 | 186 | output.flush(); 187 | } 188 | 189 | 190 | } 191 | 192 | if (true) { 193 | writeT(static_cast(SerializedBlockType::constantCompressed), output); 194 | 195 | std::ostringstream buffer(std::ostringstream::binary); 196 | writeConstants(code, buffer); 197 | auto bufferContent = buffer.str(); 198 | 199 | lzokay::Dict<> dict; 200 | std::size_t estimated_size = lzokay::compress_worst_size(bufferContent.size()); 201 | std::unique_ptr compressed(new uint8_t[estimated_size]); 202 | std::size_t compressed_size; 203 | auto error = lzokay::compress((const uint8_t*)bufferContent.data(), bufferContent.size(), compressed.get(), estimated_size, 204 | compressed_size, dict); 205 | if (error < lzokay::EResult::Success) 206 | __debugbreak(); 207 | 208 | writeT(static_cast(bufferContent.size()), output); // uncompressed size 209 | writeT(static_cast(2), output); // compression method, always 2 (That is RV engine stuff) 210 | output.write((const char*)compressed.get(), compressed_size); 211 | } else { 212 | writeT(static_cast(SerializedBlockType::constant), output); 213 | writeConstants(code, output); 214 | } 215 | 216 | output.flush(); 217 | 218 | auto pos = output.tellp(); 219 | 220 | writeT(static_cast(SerializedBlockType::locationInfo), output); 221 | output.flush(); 222 | writeT(static_cast(code.fileNames.size()), output); 223 | for (auto& it : code.fileNames) 224 | writeString(output, it); 225 | output.flush(); 226 | writeT(static_cast(SerializedBlockType::code), output); 227 | writeT(code.codeIndex, output); 228 | output.flush(); 229 | } 230 | 231 | // lzokay_stream.cpp 232 | extern lzokay::EResult decompressStream(std::istream& src, std::size_t src_size, 233 | uint8_t* dst, std::size_t init_dst_size, 234 | std::size_t& dst_size); 235 | 236 | struct DecompressedData 237 | { 238 | DecompressedData(std::vector&& data): 239 | uncompressedData(std::move(data)), 240 | buf(uncompressedData.data(), uncompressedData.size()), 241 | stream(&buf) 242 | {} 243 | 244 | std::vector uncompressedData; 245 | std::strstreambuf buf; 246 | std::istream stream; 247 | }; 248 | 249 | //#TODO make the compress write also a method 250 | 251 | // This is shit code, it depends on RVO being applied otherwise the streambuf pointer would change 252 | DecompressedData DecompressFromStream(std::istream& input) 253 | { 254 | size_t uncompressedSize = readT(input); 255 | readT(input); // compression method, always 2 256 | 257 | // compressed buffer 258 | 259 | std::vector uncompressedData; 260 | uncompressedData.resize(uncompressedSize); 261 | 262 | const auto pos = input.tellg(); 263 | input.seekg(0, SEEK_END); 264 | const std::streamoff size = input.tellg(); // -pos; 265 | input.seekg(pos, SEEK_SET); 266 | 267 | //if (pos == 0x2bc && size == 3413) 268 | //{ 269 | // std::vector compData; 270 | // compData.resize(size); 271 | // input.read((char*)compData.data(), size); 272 | // auto res = lzokay::decompress(compData.data(), size - pos, uncompressedData.data(), uncompressedSize, uncompressedSize); 273 | //} 274 | 275 | const auto error = decompressStream(input, size, uncompressedData.data(), uncompressedSize, uncompressedSize); 276 | if (error < lzokay::EResult::Success) 277 | __debugbreak(); 278 | 279 | return { std::move(uncompressedData) }; 280 | } 281 | 282 | 283 | CompiledCodeData ScriptSerializer::binaryToCompiled(std::istream& input) { 284 | CompiledCodeData output; 285 | output.version = readT(input); 286 | while (!input.eof()) { 287 | auto elementType = readT(input); 288 | auto type = static_cast(elementType); 289 | 290 | uint32_t pos = input.tellg(); 291 | 292 | switch (type) { 293 | 294 | case SerializedBlockType::constant: { 295 | readConstants(output, input); 296 | } break; 297 | case SerializedBlockType::constantCompressed: { 298 | auto decompressedData = DecompressFromStream(input); 299 | readConstants(output, decompressedData.stream); 300 | } break; 301 | case SerializedBlockType::locationInfo: { 302 | auto locCount = readT(input); 303 | 304 | for (uint16_t i = 0; i < locCount; ++i) { 305 | output.fileNames.emplace_back(readString(input)); 306 | } 307 | } break; 308 | case SerializedBlockType::code: { 309 | output.codeIndex = readT(input); 310 | } break; 311 | case SerializedBlockType::codeDebug: { 312 | __debugbreak(); // not implemented 313 | } break; 314 | case SerializedBlockType::commandNameDirectory: { 315 | // Compressed buffer 316 | 317 | auto decompressedData = DecompressFromStream(input); 318 | 319 | auto numCommandNames = readT(decompressedData.stream); 320 | output.commandNameDirectory.reserve(numCommandNames); 321 | for (int i = 0; i < numCommandNames; ++i) 322 | output.commandNameDirectory.emplace_back(readString(decompressedData.stream)); 323 | } break; 324 | } 325 | } 326 | return output; 327 | } 328 | 329 | void ScriptSerializer::compiledToBinaryCompressed(const CompiledCodeData& code, std::ostream& output) { 330 | std::stringstream uncompressedData(std::stringstream::binary | std::stringstream::out | std::stringstream::in); 331 | compiledToBinary(code, uncompressedData); 332 | 333 | std::vector uncompressedVec; 334 | 335 | std::streampos end = uncompressedData.tellg(); 336 | uncompressedData.seekg(0, std::ios_base::beg); 337 | std::streampos beg = uncompressedData.tellg(); 338 | 339 | uncompressedVec.reserve(end - beg); 340 | 341 | uncompressedVec.assign(std::istreambuf_iterator(uncompressedData), std::istreambuf_iterator()); 342 | 343 | auto compressedData = compressData(uncompressedVec); 344 | uncompressedVec.clear(); 345 | output.write(compressedData.data(), compressedData.size()); 346 | } 347 | 348 | 349 | template > 350 | class vectorwrapbuf : public std::basic_streambuf { 351 | using base = std::basic_streambuf; 352 | public: 353 | vectorwrapbuf(std::vector& vec) { 354 | base::setg(vec.data(), vec.data(), vec.data() + vec.size()); 355 | } 356 | }; 357 | 358 | 359 | CompiledCodeData ScriptSerializer::binaryToCompiledCompressed(std::istream& input) { 360 | std::vector compressedVec; 361 | 362 | compressedVec.assign(std::istreambuf_iterator(input), std::istreambuf_iterator()); 363 | 364 | auto decompressedData = decompressData({ compressedVec.data(), compressedVec.size() }); 365 | compressedVec.clear(); 366 | 367 | vectorwrapbuf databuf(decompressedData); 368 | std::istream is(&databuf); 369 | 370 | return binaryToCompiled(is); 371 | } 372 | 373 | CompiledCodeData ScriptSerializer::binaryToCompiledCompressed(std::string_view input) { 374 | auto decompressedData = decompressData(input); 375 | 376 | vectorwrapbuf databuf(decompressedData); 377 | std::istream is(&databuf); 378 | 379 | return binaryToCompiled(is); 380 | } 381 | 382 | void ScriptSerializer::instructionToBinary(const CompiledCodeData& code, const ScriptInstruction& instruction, std::ostream& output) { 383 | 384 | writeT(static_cast(instruction.type), output); 385 | 386 | if (instruction.type != InstructionType::endStatement && instruction.type != InstructionType::push) { 387 | //these can't fail, so we don't need source info 388 | writeT(static_cast(instruction.offset), output); 389 | writeT(static_cast(instruction.fileIndex), output); 390 | writeT(static_cast(instruction.line), output); 391 | } 392 | 393 | output.flush(); 394 | switch (instruction.type) { 395 | 396 | case InstructionType::endStatement: break; 397 | case InstructionType::push: { 398 | auto constantIndex = std::get(instruction.content); 399 | if (constantIndex != static_cast(constantIndex)) 400 | __debugbreak(); 401 | writeT(static_cast(constantIndex), output); 402 | } break; 403 | case InstructionType::callUnary: 404 | case InstructionType::callBinary: 405 | case InstructionType::callNular: 406 | case InstructionType::assignTo: 407 | case InstructionType::assignToLocal: 408 | case InstructionType::getVariable: 409 | if (code.commandNameDirectory.empty()) 410 | writeString(output, std::get(instruction.content)); 411 | else { 412 | auto index = code.getIndexFromCommandNameDirectory(std::get(instruction.content)); 413 | writeT(index, output); 414 | } 415 | break; 416 | case InstructionType::makeArray: { 417 | auto constantIndex = std::get(instruction.content); 418 | if (constantIndex != static_cast(constantIndex)) 419 | __debugbreak(); 420 | writeT(static_cast(constantIndex), output); 421 | } break; 422 | 423 | default: ; 424 | } 425 | } 426 | 427 | void ScriptSerializer::instructionsToBinary(const CompiledCodeData& code, const std::vector& instructions, std::ostream& output) { 428 | writeT(static_cast(instructions.size()), output); 429 | for (auto& it : instructions) 430 | instructionToBinary(code, it, output); 431 | } 432 | 433 | ScriptInstruction ScriptSerializer::binaryToInstruction(const CompiledCodeData& code, std::istream& input) { 434 | 435 | uint32_t position = input.tellg(); 436 | 437 | auto instructionType = readT(input); 438 | auto type = static_cast(instructionType); 439 | 440 | uint32_t offset = 0; 441 | uint8_t fileIndex = 0; 442 | uint16_t fileLine = 0; 443 | 444 | if (type != InstructionType::endStatement && type != InstructionType::push) { 445 | //these can't fail, so we don't need source info 446 | offset = readT(input); 447 | fileIndex = readT(input); 448 | fileLine = readT(input); 449 | } 450 | 451 | switch (type) { 452 | case InstructionType::endStatement: 453 | return ScriptInstruction{ type, offset, fileIndex, fileLine, {} }; 454 | case InstructionType::push: { 455 | uint64_t constIndex = readT(input); 456 | 457 | return ScriptInstruction{ type, offset, fileIndex, fileLine, constIndex }; 458 | } 459 | case InstructionType::callUnary: 460 | case InstructionType::callBinary: 461 | case InstructionType::callNular: 462 | case InstructionType::assignTo: 463 | case InstructionType::assignToLocal: 464 | case InstructionType::getVariable: { 465 | STRINGTYPE commandName; 466 | if (code.commandNameDirectory.empty()) 467 | commandName = readString(input); 468 | else 469 | commandName = code.commandNameDirectory[readT(input)]; 470 | 471 | return ScriptInstruction{ type, offset, fileIndex, fileLine, commandName }; 472 | } 473 | case InstructionType::makeArray: { 474 | auto arraySize = readT(input); 475 | return ScriptInstruction{ type, offset, fileIndex, fileLine, static_cast(arraySize) }; 476 | } 477 | } 478 | __debugbreak(); 479 | } 480 | 481 | std::vector ScriptSerializer::binaryToInstructions(const CompiledCodeData& code, std::istream& input) { 482 | auto pos = input.tellg(); 483 | auto count = readT(input); 484 | std::vector result; 485 | result.reserve(count); 486 | for (uint32_t i = 0; i < count; ++i) 487 | result.emplace_back(binaryToInstruction(code, input)); 488 | return result; 489 | } 490 | 491 | void ScriptSerializer::writeConstant(const CompiledCodeData& code, const ScriptConstant& constant, std::ostream& output) { 492 | auto type = getConstantType(constant); 493 | writeT(static_cast(type), output); 494 | 495 | switch (type) { 496 | case ConstantType::code: { 497 | auto& instructions = std::get(constant); 498 | writeT(instructions.contentString, output); 499 | instructionsToBinary(code, instructions.code, output); 500 | } break; 501 | case ConstantType::string: 502 | writeString(output, std::get(constant)); 503 | break; 504 | case ConstantType::scalar: 505 | writeT(std::get(constant), output); 506 | break; 507 | case ConstantType::boolean: 508 | writeT(std::get(constant), output); 509 | break; 510 | case ConstantType::array: { 511 | auto& array = std::get(constant); 512 | if (static_cast(array.content.size()) != array.content.size()) // truncation 513 | __debugbreak(); 514 | writeT(static_cast(array.content.size()), output); 515 | 516 | for (auto& cnst : array.content) 517 | writeConstant(code, cnst, output); 518 | } break; 519 | case ConstantType::nularCommand: 520 | writeString(output, std::get(constant).commandName); 521 | break; 522 | default: __debugbreak(); 523 | } 524 | } 525 | 526 | ScriptConstant ScriptSerializer::readConstant(CompiledCodeData& code, std::istream& input) { 527 | 528 | auto typeRaw = readT(input); 529 | 530 | auto type = static_cast(typeRaw); 531 | 532 | switch (type) { 533 | case ConstantType::code: { 534 | ScriptCodePiece piece; 535 | piece.contentString = readT(input); 536 | piece.code = binaryToInstructions(code, input); 537 | 538 | return piece; 539 | } break; 540 | case ConstantType::string: 541 | return readString(input); 542 | case ConstantType::scalar: 543 | return readT(input); 544 | case ConstantType::boolean: 545 | return readT(input); 546 | case ConstantType::array: { 547 | auto size = readT(input); 548 | ScriptConstantArray arr; 549 | arr.content.reserve(size); 550 | for (int i = 0; i < size; ++i) { 551 | arr.content.emplace_back(readConstant(code, input)); 552 | } 553 | return arr; 554 | } break; 555 | case ConstantType::nularCommand: 556 | return ScriptConstantNularCommand(readString(input)); 557 | default: __debugbreak(); 558 | } 559 | 560 | } 561 | 562 | void ScriptSerializer::writeConstants(const CompiledCodeData& code, std::ostream& output) { 563 | if (static_cast(code.constants.size()) != code.constants.size()) { 564 | throw std::runtime_error("too many constants"); 565 | } 566 | writeT(static_cast(code.constants.size()), output); 567 | int index = 0; 568 | for (auto& constant : code.constants) { 569 | writeConstant(code, constant, output); 570 | } 571 | } 572 | 573 | void ScriptSerializer::readConstants(CompiledCodeData& code, std::istream& input) { 574 | auto constantCount = readT(input); 575 | 576 | code.constants.reserve(constantCount); 577 | for (int i = 0; i < constantCount; ++i) { 578 | code.constants.emplace_back(readConstant(code, input)); 579 | } 580 | 581 | } 582 | 583 | void ScriptSerializer::collectCommandNames(const CompiledCodeData& code, std::set& directory) { 584 | 585 | std::function&)> collectArray = [&directory, &collectArray](const std::vector& content) 586 | { 587 | for (const auto& it : content) { 588 | auto type = getConstantType(it); 589 | if (type == ConstantType::code) { 590 | const auto& instructions = std::get(it); 591 | collectCommandNames(instructions.code, directory); 592 | } 593 | if (type == ConstantType::array) { 594 | auto& array = std::get(it); 595 | collectArray(array.content); 596 | } 597 | } 598 | }; 599 | 600 | for (const auto& it : code.constants) { 601 | auto type = getConstantType(it); 602 | if (type == ConstantType::code) { 603 | const auto& instructions = std::get(it); 604 | collectCommandNames(instructions.code, directory); 605 | } 606 | if (type == ConstantType::array) { 607 | auto& array = std::get(it); 608 | collectArray(array.content); 609 | } 610 | } 611 | } 612 | 613 | void ScriptSerializer::collectCommandNames(const std::vector& instructions, std::set& directory) { 614 | for (const auto& it : instructions) 615 | switch (it.type) { 616 | case InstructionType::endStatement: break; 617 | case InstructionType::push: break; 618 | case InstructionType::makeArray: break; 619 | case InstructionType::callUnary: 620 | case InstructionType::callBinary: 621 | case InstructionType::callNular: 622 | case InstructionType::assignTo: 623 | case InstructionType::assignToLocal: 624 | case InstructionType::getVariable: 625 | directory.emplace(std::get(it.content)); 626 | break; 627 | } 628 | } 629 | 630 | 631 | std::vector ScriptSerializer::compressData(const std::vector& data) { 632 | const size_t cBuffSize = ZSTD_compressBound(data.size()); 633 | std::vector outputBuffer; 634 | outputBuffer.resize(cBuffSize); 635 | 636 | size_t const cSize = ZSTD_compress(outputBuffer.data(), outputBuffer.size(), data.data(), data.size(), compressionLevel); 637 | if (ZSTD_isError(cSize)) { 638 | __debugbreak(); 639 | } 640 | outputBuffer.resize(cSize); 641 | return outputBuffer; 642 | } 643 | 644 | std::vector ScriptSerializer::compressDataDictionary(const std::vector& data, const std::vector& dictionary) { 645 | ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.data(), dictionary.size(), compressionLevel); 646 | 647 | const size_t cBuffSize = ZSTD_compressBound(data.size()); 648 | std::vector outputBuffer; 649 | outputBuffer.resize(cBuffSize); 650 | 651 | 652 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); 653 | if (cctx == NULL) { fprintf(stderr, "ZSTD_createCCtx() error \n"); exit(10); } 654 | size_t const cSize = ZSTD_compress_usingCDict(cctx, outputBuffer.data(), outputBuffer.size(), data.data(), data.size(), cdict); 655 | if (ZSTD_isError(cSize)) { 656 | __debugbreak(); 657 | } 658 | outputBuffer.resize(cSize); 659 | ZSTD_freeCCtx(cctx); 660 | ZSTD_freeCDict(cdict); //#TODO keep dict 661 | return outputBuffer; 662 | } 663 | 664 | std::vector ScriptSerializer::decompressData(std::string_view data) { 665 | 666 | const size_t rSize = ZSTD_findDecompressedSize(data.data(), data.size()); 667 | 668 | if (rSize == ZSTD_CONTENTSIZE_ERROR) { 669 | //fprintf(stderr, "%s : it was not compressed by zstd.\n", fname); 670 | __debugbreak(); 671 | } 672 | else if (rSize == ZSTD_CONTENTSIZE_UNKNOWN) { 673 | // fprintf(stderr, "%s : original size unknown. Use streaming decompression instead.\n", fname); 674 | __debugbreak(); 675 | } 676 | 677 | 678 | std::vector outputBuffer; 679 | outputBuffer.resize(rSize); 680 | 681 | size_t const dSize = ZSTD_decompress(outputBuffer.data(), outputBuffer.size(), data.data(), data.size()); 682 | if (dSize != rSize) { 683 | //fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize)); 684 | __debugbreak(); 685 | } 686 | 687 | return outputBuffer; 688 | } 689 | 690 | std::vector ScriptSerializer::decompressDataDictionary(const std::vector& data, const std::vector& dictionary) { 691 | ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.data(), dictionary.size()); 692 | 693 | const size_t cBuffSize = ZSTD_compressBound(data.size()); 694 | std::vector outputBuffer; 695 | outputBuffer.resize(cBuffSize); 696 | 697 | 698 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); 699 | if (dctx == NULL) { fprintf(stderr, "ZSTD_createDCtx() error \n"); exit(10); } 700 | size_t const cSize = ZSTD_decompress_usingDDict(dctx, outputBuffer.data(), outputBuffer.size(), data.data(), data.size(), ddict); 701 | if (ZSTD_isError(cSize)) { 702 | __debugbreak(); 703 | } 704 | outputBuffer.resize(cSize); 705 | ZSTD_freeDCtx(dctx); 706 | ZSTD_freeDDict(ddict); //#TODO keep dict 707 | 708 | return outputBuffer; 709 | } 710 | 711 | STRINGTYPE ScriptSerializer::readString(std::istream& input) { 712 | uint32_t length{}; // length is actually uint24 713 | input.read(reinterpret_cast(&length), 3); 714 | 715 | 716 | if constexpr (std::is_same_v) { 717 | STRINGTYPE result; 718 | result.resize(length); 719 | 720 | input.read(result.data(), length); 721 | 722 | return result; 723 | } else { // RString 724 | STRINGTYPE result; 725 | result.resize(length); 726 | 727 | input.read(result.data(), length); 728 | 729 | return result; 730 | } 731 | 732 | 733 | } 734 | 735 | void ScriptSerializer::writeString(std::ostream& output, std::string_view string) { 736 | uint32_t length = string.length(); 737 | // special stuff to write a 24bit number 738 | //Assert(length & 0x00FFFFFF == length); // check that 24bit is not truncating 739 | 740 | if ((length & 0x00FFFFFF) != length) 741 | __debugbreak(); 742 | 743 | output.write(reinterpret_cast(&length), 3); 744 | output.write(string.data(), length); 745 | } 746 | -------------------------------------------------------------------------------- /src/scriptSerializer.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | #include "compiledCode.hpp" 5 | 6 | class ScriptSerializer { 7 | public: 8 | static void compiledToHumanReadable(const CompiledCodeData& code, std::ostream& output); 9 | static void compiledToBinary(const CompiledCodeData& code, std::ostream& output); 10 | static CompiledCodeData binaryToCompiled(std::istream& input); 11 | 12 | static void compiledToBinaryCompressed(const CompiledCodeData& code, std::ostream& output); 13 | static CompiledCodeData binaryToCompiledCompressed(std::istream& input); 14 | static CompiledCodeData binaryToCompiledCompressed(std::string_view input); 15 | 16 | 17 | 18 | 19 | private: 20 | static void instructionToBinary(const CompiledCodeData& code, const ScriptInstruction& instruction, std::ostream& output); 21 | static void instructionsToBinary(const CompiledCodeData& code, const std::vector& instructions, 22 | std::ostream& output); 23 | 24 | static ScriptInstruction binaryToInstruction(const CompiledCodeData& code, std::istream& input); 25 | static std::vector binaryToInstructions(const CompiledCodeData& code, std::istream& input);; 26 | 27 | static void writeConstant(const CompiledCodeData& code, const ScriptConstant& constant, std::ostream& output); 28 | static ScriptConstant readConstant(CompiledCodeData& code, std::istream& input); 29 | 30 | static void writeConstants(const CompiledCodeData& code, std::ostream& output); 31 | static void readConstants(CompiledCodeData& code, std::istream& input); 32 | 33 | static void collectCommandNames(const CompiledCodeData& code, std::set& directory); 34 | static void collectCommandNames(const std::vector& instructions, std::set& directory); 35 | 36 | static std::vector compressData(const std::vector& data); 37 | static std::vector compressDataDictionary(const std::vector& data, const std::vector& dictionary); 38 | 39 | 40 | static std::vector decompressData(std::string_view data); 41 | static std::vector decompressDataDictionary(const std::vector& data, const std::vector& dictionary); 42 | 43 | 44 | static STRINGTYPE readString(std::istream& input); 45 | static void writeString(std::ostream& output, std::string_view string); 46 | 47 | }; 48 | -------------------------------------------------------------------------------- /test/ace3.json: -------------------------------------------------------------------------------- 1 | { 2 | "inputDirs": [ 3 | "P:/z/ace/addons/" 4 | ], 5 | "includePaths": [ 6 | "P:/" 7 | ], 8 | "excludeList": [ 9 | ], 10 | "outputDir": "P:/", 11 | "workerThreads": 6 12 | } 13 | -------------------------------------------------------------------------------- /test/cba_a3.json: -------------------------------------------------------------------------------- 1 | { 2 | "inputDirs": [ 3 | "P:/x/cba/addons/" 4 | ], 5 | "includePaths": [ 6 | "P:/" 7 | ], 8 | "excludeList": [ 9 | "\\initsettings.sqf", 10 | "\\initkeybinds.sqf", 11 | "\\xeh_prep.sqf", 12 | "\\backwards_comp.sqf", 13 | "settings\\gui_createcategory.sqf", 14 | "diagnostic\\fnc_initextendeddebugconsole.sqf", 15 | "xeh\\fnc_initdisplay.sqf", 16 | "xeh\\fnc_startloadingscreen.sqf", 17 | "xeh\\fnc_endloadingscreen.sqf" 18 | ], 19 | "outputDir": "P:/", 20 | "workerThreads": 6 21 | } 22 | --------------------------------------------------------------------------------