├── .ci └── setup.sh ├── .github ├── ISSUE_TEMPLATE │ └── bug_report.md └── workflows │ └── python_package.yml ├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── CODE_OF_CONDUCT.md ├── LICENSE ├── README.md ├── SECURITY.md ├── SUPPORT.md ├── VERSION.txt ├── azureml ├── README.md ├── components │ ├── evaluate │ │ ├── evaluate.py │ │ └── evaluate.yaml │ ├── infer │ │ ├── infer.py │ │ └── infer.yaml │ └── train │ │ ├── train.py │ │ └── train.yaml ├── data │ ├── parser_config_file.json │ ├── test.tsv │ └── train.tsv ├── requirements.txt └── run.py ├── docs ├── Customized-Parser-Tutorial.rst ├── Development-Setup-Guide.rst ├── FreeForm2-Language.rst ├── Installation-Guide.rst └── Transform-Tutorial.rst ├── examples └── freeform2_parser.cpp ├── scripts ├── MANIFEST.in ├── README.rst ├── generate_parser_config.py ├── load_precompiled_libs.py ├── publish_python_package.sh └── setup.py ├── src ├── CMakeLists.txt ├── DynamicRank.FreeForm.Library │ ├── CMakeLists.txt │ ├── inc │ │ ├── FreeForm2.h │ │ ├── FreeForm2Assert.h │ │ ├── FreeForm2Compiler.h │ │ ├── FreeForm2CompilerFactory.h │ │ ├── FreeForm2Executable.h │ │ ├── FreeForm2ExternalData.h │ │ ├── FreeForm2Features.h │ │ ├── FreeForm2Program.h │ │ ├── FreeForm2Result.h │ │ ├── FreeForm2Type.h │ │ └── NeuralInputFreeForm2.h │ ├── libs │ │ ├── Backend │ │ │ └── llvm │ │ │ │ ├── ArrayCodeGen.cpp │ │ │ │ ├── ArrayCodeGen.h │ │ │ │ ├── CMakeLists.txt │ │ │ │ ├── CompilationState.cpp │ │ │ │ ├── CompilationState.h │ │ │ │ ├── Extend │ │ │ │ ├── FreeForm2Support.cpp │ │ │ │ ├── FreeForm2Support.h │ │ │ │ ├── JITEmitter.cpp │ │ │ │ ├── JITExtend.cpp │ │ │ │ └── JITExtend.h │ │ │ │ ├── LlvmCodeGenUtils.cpp │ │ │ │ ├── LlvmCodeGenUtils.h │ │ │ │ ├── LlvmCodeGenerator.cpp │ │ │ │ ├── LlvmCodeGenerator.h │ │ │ │ ├── LlvmCompiler.cpp │ │ │ │ ├── LlvmCompiler.h │ │ │ │ ├── LlvmRuntimeLibrary.cpp │ │ │ │ └── LlvmRuntimeLibrary.h │ │ ├── Expression │ │ │ ├── Allocation.cpp │ │ │ ├── Allocation.h │ │ │ ├── ArrayDereferenceExpression.cpp │ │ │ ├── ArrayDereferenceExpression.h │ │ │ ├── ArrayLength.cpp │ │ │ ├── ArrayLength.h │ │ │ ├── ArrayLiteralExpression.cpp │ │ │ ├── ArrayLiteralExpression.h │ │ │ ├── BinaryOperator.cpp │ │ │ ├── BinaryOperator.h │ │ │ ├── BlockExpression.cpp │ │ │ ├── BlockExpression.h │ │ │ ├── CMakeLists.txt │ │ │ ├── Conditional.cpp │ │ │ ├── Conditional.h │ │ │ ├── ConvertExpression.cpp │ │ │ ├── ConvertExpression.h │ │ │ ├── DebugExpression.cpp │ │ │ ├── DebugExpression.h │ │ │ ├── Declaration.cpp │ │ │ ├── Declaration.h │ │ │ ├── Expression.cpp │ │ │ ├── Expression.h │ │ │ ├── Extern.cpp │ │ │ ├── Extern.h │ │ │ ├── FeatureSpec.cpp │ │ │ ├── FeatureSpec.h │ │ │ ├── Function.cpp │ │ │ ├── Function.h │ │ │ ├── LetExpression.cpp │ │ │ ├── LetExpression.h │ │ │ ├── LiteralExpression.cpp │ │ │ ├── LiteralExpression.h │ │ │ ├── Match.cpp │ │ │ ├── Match.h │ │ │ ├── MatchSub.h │ │ │ ├── MemberAccessExpression.cpp │ │ │ ├── MemberAccessExpression.h │ │ │ ├── Mutation.cpp │ │ │ ├── Mutation.h │ │ │ ├── OperatorExpression.cpp │ │ │ ├── OperatorExpression.h │ │ │ ├── PhiNode.cpp │ │ │ ├── PhiNode.h │ │ │ ├── Publish.cpp │ │ │ ├── Publish.h │ │ │ ├── RandExpression.cpp │ │ │ ├── RandExpression.h │ │ │ ├── RangeReduceExpression.cpp │ │ │ ├── RangeReduceExpression.h │ │ │ ├── RefExpression.cpp │ │ │ ├── RefExpression.h │ │ │ ├── SelectNth.cpp │ │ │ ├── SelectNth.h │ │ │ ├── SimpleExpressionOwner.h │ │ │ ├── StateMachine.cpp │ │ │ ├── StateMachine.h │ │ │ ├── StreamData.cpp │ │ │ ├── StreamData.h │ │ │ ├── SymbolTable.cpp │ │ │ ├── SymbolTable.h │ │ │ ├── TypeUtil.cpp │ │ │ ├── TypeUtil.h │ │ │ ├── UnaryOperator.cpp │ │ │ ├── UnaryOperator.h │ │ │ └── Visitor.h │ │ ├── External │ │ │ ├── ArrayResult.h │ │ │ ├── CMakeLists.txt │ │ │ ├── Compiler.cpp │ │ │ ├── Compiler.h │ │ │ ├── Executable.cpp │ │ │ ├── Executable.h │ │ │ ├── FreeForm2ExternalData.cpp │ │ │ ├── FreeForm2Result.cpp │ │ │ ├── FreeForm2Type.cpp │ │ │ ├── NeuralInputFreeForm2.cpp │ │ │ ├── Program.cpp │ │ │ ├── Program.h │ │ │ ├── ResultIteratorImpl.h │ │ │ ├── ValueResult.cpp │ │ │ └── ValueResult.h │ │ ├── Parse │ │ │ └── SExpression │ │ │ │ ├── inc │ │ │ │ ├── FreeForm2Tokenizer.h │ │ │ │ └── SExpressionParse.h │ │ │ │ └── libs │ │ │ │ ├── Arithmetic.cpp │ │ │ │ ├── Arithmetic.h │ │ │ │ ├── Bitwise.cpp │ │ │ │ ├── Bitwise.h │ │ │ │ ├── CMakeLists.txt │ │ │ │ ├── ExpressionFactory.cpp │ │ │ │ ├── ExpressionFactory.h │ │ │ │ ├── FreeForm2Tokenizer.cpp │ │ │ │ ├── Logic.cpp │ │ │ │ ├── Logic.h │ │ │ │ ├── MiscFactory.cpp │ │ │ │ ├── MiscFactory.h │ │ │ │ ├── OperatorExpressionFactory.h │ │ │ │ ├── ProgramParseState.cpp │ │ │ │ ├── ProgramParseState.h │ │ │ │ └── SExpressionParse.cpp │ │ ├── Shared │ │ │ ├── ArrayType.cpp │ │ │ ├── ArrayType.h │ │ │ ├── Attributes.h │ │ │ ├── BFBFile.h │ │ │ ├── CMakeLists.txt │ │ │ ├── CompoundType.cpp │ │ │ ├── CompoundType.h │ │ │ ├── FreeForm2Assert.cpp │ │ │ ├── FreeForm2Utils.cpp │ │ │ ├── FreeForm2Utils.h │ │ │ ├── FunctionType.cpp │ │ │ ├── FunctionType.h │ │ │ ├── ObjectType.cpp │ │ │ ├── ObjectType.h │ │ │ ├── QuietMetaStreams.h │ │ │ ├── StateMachineType.cpp │ │ │ ├── StateMachineType.h │ │ │ ├── StructType.cpp │ │ │ ├── StructType.h │ │ │ ├── TypeImpl.cpp │ │ │ ├── TypeImpl.h │ │ │ ├── TypeManager.cpp │ │ │ └── TypeManager.h │ │ └── Transform │ │ │ ├── AllocationVisitor.cpp │ │ │ ├── AllocationVisitor.h │ │ │ ├── CMakeLists.txt │ │ │ ├── CopyingVisitor.cpp │ │ │ ├── CopyingVisitor.h │ │ │ ├── FunctionInlineVisitor.cpp │ │ │ ├── FunctionInlineVisitor.h │ │ │ ├── NoOpVisitor.h │ │ │ ├── ObjectResolutionVisitor.cpp │ │ │ ├── ObjectResolutionVisitor.h │ │ │ ├── OperandPromotionVisitor.cpp │ │ │ ├── OperandPromotionVisitor.h │ │ │ ├── ProcessFeaturesUsed.cpp │ │ │ ├── ProcessFeaturesUsed.h │ │ │ ├── TypeCheckingVisitor.cpp │ │ │ ├── TypeCheckingVisitor.h │ │ │ ├── UniformExpressionVisitor.cpp │ │ │ └── UniformExpressionVisitor.h │ └── test │ │ ├── CMakeLists.txt │ │ ├── FreeFormLibTest.cpp │ │ ├── FreeFormLibTestSet.h │ │ ├── SimpleFeatureMap.cpp │ │ └── SimpleFeatureMap.h ├── NeuralTree.Library │ ├── CMakeLists.txt │ ├── inc │ │ ├── CsHash.h │ │ ├── FeaSpecConfig.h │ │ ├── IFeatureMap.h │ │ ├── INeuralNetFeatures.h │ │ ├── MigratedApi.h │ │ ├── NeuralInput.h │ │ ├── NeuralInputFactory.h │ │ ├── NeuralInputFreeForm2_types.h │ │ ├── NeuralInput_types.h │ │ ├── UnionBondInput_types.h │ │ ├── basic_types.h │ │ ├── common.h │ │ └── log.h │ └── src │ │ ├── CMakeLists.txt │ │ ├── CsHash.cpp │ │ ├── FeaSpecConfig.cpp │ │ ├── NeuralInput.cpp │ │ └── NeuralInputFactory.cpp ├── NeuralTreeEvaluator.Library │ ├── CMakeLists.txt │ ├── inc │ │ ├── FeatureMap.h │ │ ├── InputComputation.h │ │ └── InputExtraction.h │ ├── src │ │ ├── CMakeLists.txt │ │ ├── FeatureMap.cpp │ │ ├── InputComputation.cpp │ │ ├── InputExtraction.cpp │ │ ├── InputExtractor.cpp │ │ ├── InputExtractor.h │ │ ├── LocalFactoryHolder.cpp │ │ ├── LocalFactoryHolder.h │ │ ├── MinimalFeatureMap.cpp │ │ └── MinimalFeatureMap.h │ └── test │ │ ├── CMakeLists.txt │ │ ├── NeuralTreeEvaluatorLibTest.cpp │ │ └── data │ │ └── TrainInputIni └── TransformProcessor │ ├── CMakeLists.txt │ ├── src │ ├── CMakeLists.txt │ ├── FeatureEvaluator.cpp │ ├── FeatureEvaluator.h │ ├── FeatureEvaluatorExtendedInfo.h │ ├── IniFileParserInterface.cpp │ ├── IniFileParserInterface.h │ ├── Parser.h │ ├── TransformProcessor.cpp │ ├── TransformProcessor.h │ ├── TransformProcessorFeatureMap.cpp │ └── TransformProcessorFeatureMap.h │ └── test │ ├── CMakeLists.txt │ ├── Tests.cpp │ ├── Tests.h │ └── data │ └── integration │ ├── ExpectedOutput.txt │ └── Input.tsv └── tests ├── data ├── transform_rank_data │ ├── freeform_config.json │ └── input.tsv └── transform_simple_data │ ├── freeform_config.json │ └── input.tsv └── python_package_test └── test_transform.py /.ci/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | lgb_python_pkg_dir="./external_libs/LightGBM/python-package" 4 | # compile transformation, lightgbm, and customized parser libs. 5 | mkdir build 6 | # precompile mpi option. 7 | cd build && cmake ../ -DUSE_MPI=ON && make -j4 && cd ../ || exit -1 8 | # copy all shared libs to lightgbm python package directory. 9 | mkdir ${lgb_python_pkg_dir}/lightgbm/bin && \ 10 | cp ./lib_custom_parser.so ${lgb_python_pkg_dir}/lightgbm/bin && \ 11 | cp ./src/lib_transform.so ${lgb_python_pkg_dir}/lightgbm/bin && \ 12 | cp ./external_libs/LightGBM/lib_lightgbm.so ${lgb_python_pkg_dir}/lightgbm/bin || exit -1 13 | # modify `basic.py` to load all libs first, or cannot find them when calling python interfaces. 14 | cp ${lgb_python_pkg_dir}/lightgbm/basic.py raw && cat ./scripts/load_precompiled_libs.py ${lgb_python_pkg_dir}/lightgbm/basic.py > tmp && cp tmp ${lgb_python_pkg_dir}/lightgbm/basic.py || exit -1 15 | # install python package 16 | # install scikit-learn to avoid error "ImportError: cannot import name '_LGBMBaseCrossValidator' from 'lightgbm.compat'" 17 | pip install pytest numpy scipy pandas scikit-learn 18 | pip list 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Description 11 | 12 | 13 | ## Reproducible example 14 | 15 | 16 | ## Environment info 17 | 18 | lightgbm-transform version or commit hash: 19 | 20 | Command(s) you used to install lightgbm-transform 21 | 22 | ```shell 23 | 24 | ``` 25 | 26 | 27 | 28 | 29 | ## Additional Comments 30 | 31 | -------------------------------------------------------------------------------- /.github/workflows/python_package.yml: -------------------------------------------------------------------------------- 1 | name: Python-package 2 | 3 | on: 4 | # Triggers the workflow on push or pull request events but only for the main branch 5 | push: 6 | branches: [ main ] 7 | pull_request: 8 | branches: [ main ] 9 | 10 | # Allows you to run this workflow manually from the Actions tab 11 | workflow_dispatch: 12 | 13 | env: 14 | CONDA_ENV: test-env 15 | GITHUB_ACTIONS: 'true' 16 | 17 | jobs: 18 | test: 19 | # The type of runner that the job will run on 20 | runs-on: ubuntu-latest 21 | container: 22 | # Below image is generated by following `Installation-Guide.rst` 23 | image: krystal1130/lightgbm_transform_dependencies:20211109.v1 24 | timeout-minutes: 60 25 | strategy: 26 | fail-fast: false 27 | 28 | steps: 29 | - name: Check for dockerenv file 30 | run: (ls /.dockerenv && echo Found dockerenv) || (echo No dockerenv) 31 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 32 | - name: Checkout repository 33 | uses: actions/checkout@v2.4.0 34 | with: 35 | fetch-depth: 5 36 | # Should clone recursively, or cannot find external libs of LightGBM 37 | submodules: recursive 38 | 39 | - name: Setup and run tests 40 | shell: bash 41 | run: | 42 | sh ./.ci/setup.sh || exit -1 43 | export PYTHONPATH=${GITHUB_WORKSPACE}/external_libs/LightGBM/python-package 44 | echo ${PYTHONPATH} 45 | pytest ./tests 46 | 47 | all-successful: 48 | # https://github.community/t/is-it-possible-to-require-all-github-actions-tasks-to-pass-without-enumerating-them/117957/4?u=graingert 49 | runs-on: ubuntu-latest 50 | needs: [test] 51 | steps: 52 | - name: Note that all tests succeeded 53 | run: echo "🎉" 54 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | compile 3 | *egg-info 4 | dist 5 | *.so -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "external_libs/LightGBM"] 2 | path = external_libs/LightGBM 3 | url = https://github.com/microsoft/LightGBM.git 4 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.15.0 FATAL_ERROR) 2 | 3 | project(custom_transform) 4 | set(LIB_NAME "_custom_parser") 5 | 6 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/external_libs/LightGBM) 7 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/) 8 | 9 | set(LIBRARY_OUTPUT_PATH ${PROJECT_SOURCE_DIR}) 10 | 11 | add_library(${LIB_NAME} SHARED ${CMAKE_CURRENT_SOURCE_DIR}/examples/freeform2_parser.cpp ${CMAKE_CURRENT_SOURCE_DIR}/external_libs/LightGBM/include) 12 | 13 | target_include_directories(${LIB_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/external_libs/LightGBM/include) 14 | 15 | target_link_libraries(${LIB_NAME} 16 | -Wl,--no-as-needed 17 | -Wl,--start-group 18 | _lightgbm 19 | _transform 20 | -Wl,--end-group 21 | ) 22 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LightGBM Transformation Library 2 | 3 | The LightGBM transformation library aims at providing a flexible and automatic way to do feature transformation when using [`LightGBM`](https://github.com/microsoft/LightGBM). Compared to separate transformation. this way has several pros: 4 | 5 | - More efficient. Data preprocessing can go with parsing each line, and take advantage of multi-processing designed by lightgbm naturally. No need to store whole transformed data in file/memory. 6 | - More convenient for development and iteration. Built-in transformation could keep offline and online consistent well by saved/loaded along with model. 7 | 8 | In the repo, user could learn about: 9 | 10 | - How to customize favorite parser by playing with LightGBM Parser interface. 11 | - How to use [`FreeForm2Parser`](./examples/freeform2_parser.cpp), a powerful and efficient tool built in the repo, in model training. Instead of handling data yourself, user just need to prepare a feature spec which contains feature name, transform type and expression, and make slight changes in experiment iterations. 12 | 13 | ## Get Started and Documentation 14 | 15 | - [Installation guide](./docs/Installation-Guide.rst) 16 | - [Customized parser tutorial](./docs/Customized-Parser-Tutorial.rst) 17 | - [Transform tutorial](./docs/Transform-Tutorial.rst) 18 | - [FreeForm2 language spec](./docs/FreeForm2-Language.rst) 19 | 20 | ## Contributing 21 | 22 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 23 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 24 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 25 | 26 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 27 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 28 | provided by the bot. You will only need to do this once across all repos using our CLA. 29 | 30 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 31 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 32 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 33 | 34 | ## Trademarks 35 | 36 | This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft 37 | trademarks or logos is subject to and must follow 38 | [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). 39 | Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. 40 | Any use of third-party trademarks or logos are subject to those third-party's policies. 41 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd). 40 | 41 | -------------------------------------------------------------------------------- /SUPPORT.md: -------------------------------------------------------------------------------- 1 | # TODO: The maintainer of this repo has not yet edited this file 2 | 3 | **REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project? 4 | 5 | - **No CSS support:** Fill out this template with information about how to file issues and get help. 6 | - **Yes CSS support:** Fill out an intake form at [aka.ms/spot](https://aka.ms/spot). CSS will work with/help you to determine next steps. More details also available at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). 7 | - **Not sure?** Fill out a SPOT intake as though the answer were "Yes". CSS will help you decide. 8 | 9 | *Then remove this first heading from this SUPPORT.MD file before publishing your repo.* 10 | 11 | # Support 12 | 13 | ## How to file issues and get help 14 | 15 | This project uses GitHub Issues to track bugs and feature requests. Please search the existing 16 | issues before filing new issues to avoid duplicates. For new issues, file your bug or 17 | feature request as a new Issue. 18 | 19 | For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE 20 | FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER 21 | CHANNEL. WHERE WILL YOU HELP PEOPLE?**. 22 | 23 | ## Microsoft Support Policy 24 | 25 | Support for this **PROJECT or PRODUCT** is limited to the resources listed above. 26 | -------------------------------------------------------------------------------- /VERSION.txt: -------------------------------------------------------------------------------- 1 | 4.1.0 -------------------------------------------------------------------------------- /azureml/README.md: -------------------------------------------------------------------------------- 1 | # AzureML example 2 | 3 | In the doc, user could learn how to use lightgbm transformation in azure machine learning. 4 | The example describes a pipeline containing training for lambdarank objective, inference and evaluation using [Azure ML Component SDK](https://componentsdk.azurewebsites.net/overview.html). 5 | 6 | ## Steps 7 | - Run command `pip install -r requirements.txt` and install the dependencies of submission. 8 | - Download `config.json` of your azure ml workspace and put it under `azureml` folder. 9 | - Run command `python run.py`. 10 | - Click the shown experiment link and check in your workspace. 11 | -------------------------------------------------------------------------------- /azureml/components/evaluate/evaluate.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import logging 4 | import os 5 | from itertools import islice 6 | 7 | import numpy as np 8 | from sklearn.metrics import ndcg_score 9 | 10 | 11 | if __name__ == '__main__': 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument( 14 | '--predict-result', 15 | help='The predict result for the evaluating data.', 16 | ) 17 | parser.add_argument( 18 | '--evaluation-result', 19 | help='The output evaluation result.', 20 | ) 21 | args, _ = parser.parse_known_args() 22 | predict_result = args.predict_result 23 | evaluation_result = args.evaluation_result 24 | logger = logging.getLogger() 25 | logging.basicConfig(level=logging.DEBUG, format='%(message)s') 26 | 27 | logger.info('>>>>>Evaluation Module>>>>>') 28 | if os.path.isdir(predict_result): 29 | logger.info('[Evaluation] Directory "{}" is provided, use default file name "pred.result"'.format(predict_result)) 30 | predict_result = os.path.join(predict_result, 'pred.result') 31 | 32 | if os.path.isdir(evaluation_result): 33 | logger.info('[Evaluation] Directory "{}" is provided, use default file name "evaluation_result"'.format(evaluation_result)) 34 | evaluation_result = os.path.join(evaluation_result, 'evaluation_result') 35 | 36 | with open(predict_result) as fin: 37 | predict_result_dict = json.load(fin) 38 | 39 | output_list = [] 40 | pred_prob = np.array(predict_result_dict['pred_prob']) 41 | query = np.array(predict_result_dict["query"]).astype(int) 42 | true_label = np.array(predict_result_dict["true_label"]).astype(int) 43 | pred_prob_iter = iter(pred_prob) 44 | pred_prob_res = [list(islice(pred_prob_iter, 0, q)) for q in query] 45 | true_label_iter = iter(true_label) 46 | true_label_res = [list(islice(true_label_iter, 0, q)) for q in query] 47 | ndcg_scores = [] 48 | for true_relevance, scores in zip(true_label_res, pred_prob_res): 49 | true_relevance = np.asarray([list(true_relevance)]) 50 | scores = np.asarray([list(scores)]) 51 | ndcg_scores.append(ndcg_score(true_relevance, scores)) 52 | 53 | ndcg_scores = np.array(ndcg_scores) 54 | output_list.append(f'[Evaluation] NDCG on evaluation data: {np.array2string(ndcg_scores)}') 55 | output_list.append(f'[Evaluation] Average NDCG on evaluation data: {np.average(ndcg_scores)}') 56 | output_str = '\n'.join(output_list) 57 | logger.info(output_str) 58 | with open(evaluation_result, 'w') as fout: 59 | fout.write(output_str) 60 | 61 | logger.info('<<<<- 20 | python evaluate.py --predict-result {inputs.predict_result} --evaluation-result {outputs.evaluation_result} 21 | environment: 22 | conda: 23 | conda_dependencies: 24 | name: project_environment 25 | channels: 26 | - defaults 27 | dependencies: 28 | - python=3.8 29 | - pip=20.0 30 | - pip: 31 | - scikit-learn==0.22.2 32 | os: Linux 33 | -------------------------------------------------------------------------------- /azureml/components/infer/infer.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | $schema: http://azureml/sdk-2-0/CommandComponent.json 5 | name: microsoft.com.azureml.test.integrated_freeform2_lightGBM_infer 6 | version: 0.0.1 7 | display_name: Integrated FreeForm2 lightGBM Infer 8 | type: CommandComponent 9 | description: Infer with lightGBM feature transformation binding. 10 | tags: {} 11 | inputs: 12 | infer_data_path: 13 | type: path 14 | description: The file path for the inference data. 15 | model_path: 16 | type: path 17 | description: The file path for the trained LightGBM model. 18 | outputs: 19 | infer_result: 20 | type: path 21 | description: The file path for the inference result. 22 | command: >- 23 | python infer.py --infer-data-path {inputs.infer_data_path} --model-path {inputs.model_path} --infer-result {outputs.infer_result} 24 | environment: 25 | docker: 26 | image: docker.io/krystal1130/lightgbm_transform_dependencies:20211109.v1 27 | conda: 28 | conda_dependencies: 29 | name: project_environment 30 | channels: 31 | - defaults 32 | dependencies: 33 | - python=3.8 34 | - pip=20.0 35 | - pip: 36 | - lightgbm-transform==3.3.1.post1 37 | os: Linux -------------------------------------------------------------------------------- /azureml/components/train/train.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT License. 3 | 4 | $schema: http://azureml/sdk-2-0/CommandComponent.json 5 | name: microsoft.com.azureml.test.integrated_freeform2_lightGBM_train 6 | version: 0.0.1 7 | display_name: Integrated FreeForm2 lightGBM Train 8 | type: CommandComponent 9 | description: Train with lightGBM feature transformation binding. 10 | tags: {} 11 | inputs: 12 | train_data_path: 13 | type: path 14 | description: The file path for the training data. 15 | parser_config_path: 16 | type: path 17 | description: The json path for parser config. 18 | label: 19 | type: string 20 | description: Label column name. 21 | lr: 22 | type: float 23 | description: Learning rate. 24 | default: 0.22 25 | optional: true 26 | metric_freq: 27 | type: integer 28 | description: Metric frequency. 29 | default: 1 30 | optional: true 31 | label_gain: 32 | type: string 33 | description: Label gain. 34 | num_trees: 35 | type: integer 36 | description: Number of trees. 37 | default: 10 38 | optional: true 39 | num_leaves: 40 | type: integer 41 | description: Number of leaves. 42 | default: 30 43 | optional: true 44 | min_data_in_leaf: 45 | type: integer 46 | description: Minimum data in a leaf. 47 | default: 50 48 | optional: true 49 | max_bin: 50 | type: integer 51 | description: Maximum bin. 52 | default: 16 53 | optional: true 54 | query: 55 | type: integer 56 | description: Group column index. Must be the index of transformed feature in parser config. 57 | default: 0 58 | optional: true 59 | feature_fraction: 60 | type: float 61 | description: Feature fraction. 62 | default: 0.15 63 | optional: true 64 | outputs: 65 | model_path: 66 | type: path 67 | description: The output file path for the trained LightGBM model. 68 | command: >- 69 | python train.py --train-data-path {inputs.train_data_path} --parser-config-path {inputs.parser_config_path} --label {inputs.label} 70 | [--lr {inputs.lr}] [--metric-freq {inputs.metric_freq}] --label-gain {inputs.label_gain} [--num-trees {inputs.num_trees}] 71 | [--num-leaves {inputs.num_leaves}] [--min-data-in-leaf {inputs.min_data_in_leaf}] [--max-bin {inputs.max_bin}] 72 | [--query {inputs.query}] [--feature-fraction {inputs.feature_fraction}] --model-path {outputs.model_path} 73 | environment: 74 | docker: 75 | image: docker.io/krystal1130/lightgbm_transform_dependencies:20211109.v1 76 | conda: 77 | conda_dependencies: 78 | name: project_environment 79 | channels: 80 | - defaults 81 | dependencies: 82 | - python=3.8 83 | - pip=20.0 84 | - pip: 85 | - lightgbm-transform==3.3.1.post1 86 | os: Linux 87 | -------------------------------------------------------------------------------- /azureml/requirements.txt: -------------------------------------------------------------------------------- 1 | azure-ml-component 2 | azureml-defaults -------------------------------------------------------------------------------- /docs/Customized-Parser-Tutorial.rst: -------------------------------------------------------------------------------- 1 | LightGBM Customized Parser Tutorial 2 | =================================== 3 | 4 | The purpose of this document is to give you a tutorial on how to customize your own parser and integrate it in LightGBM to do training. 5 | The basic methodology is to register custom parser to parser factory via reflector. 6 | 7 | - Follow `LightGBM Parser `__ interface and design your parser by implementing constructor, ``ParseOneLine``, ``NumFeatures`` functions. Then register the custom parser. See `FreeForm2Parser <../examples/freeform2_parser.cpp>`__ as an example. 8 | 9 | - Append source file of your parser to library in `CMakeLists.txt <../CMakeLists.txt>`__. You could find that LightGBM and transform library are used as dependencies. 10 | 11 | - See `Installation Guide <./Installation-Guide.rst>`__ for dependencies and your parser installation. 12 | 13 | - See `Transform tutorial <./Transform-Tutorial.rst>`__ for how to use LightGBM integrated with your parser version. 14 | -------------------------------------------------------------------------------- /docs/Development-Setup-Guide.rst: -------------------------------------------------------------------------------- 1 | Development Setup Guide for LightGBM Transform 2 | ========================================== 3 | 4 | This guide provides instructions for setting up the development environment for LightGBM Transform on a Linux machine. 5 | 6 | Prerequisites 7 | ------------- 8 | 9 | * Linux operating system 10 | * Internet connection for downloading necessary files 11 | 12 | Setup Steps 13 | ----------- 14 | 15 | 1. **Install Docker** 16 | 17 | - Install Docker on your Linux machine by following the `instructions `_. Ensure Docker is properly installed and running. 18 | 19 | 2. **Pull Docker Image** 20 | 21 | - Download the required Docker image using the command: 22 | :: 23 | # Below image is generated by following `Installation-Guide.rst` 24 | docker pull krystal1130/lightgbm_transform_dependencies:20211109.v1 25 | 26 | 3. **Clone the Repository** 27 | 28 | - Clone the LightGBM Transform repository from GitHub: 29 | :: 30 | 31 | git clone --recursive https://github.com/microsoft/lightgbm-transform.git 32 | 33 | 4. **Mount the Repository to Docker Container** 34 | 35 | - Mount the cloned repository to the Docker container. Replace ```` with your local repository path: 36 | :: 37 | 38 | docker run -it -v :/lightgbm-transform krystal1130/lightgbm_transform_dependencies:20211109.v1 39 | 40 | 5. **Access the Docker Container** 41 | 42 | - Navigate to the LightGBM Transform directory inside the Docker container: 43 | :: 44 | 45 | cd /lightgbm-transform 46 | 47 | 6. **Package the LightGBM Transform Python Package** 48 | 49 | - Compile the package and place it in the ``./compile/dist`` directory using: 50 | :: 51 | 52 | sh ./scripts/publish_python_package.sh 53 | 54 | 7. **Test the Package** 55 | 56 | - Install and test the package. Replace ```` with the actual version number: 57 | :: 58 | 59 | pip install --force-reinstall ./compile/dist/lightgbm_transform--py3-none-any.whl 60 | pip install pytest numpy scipy pandas scikit-learn 61 | cd ./tests 62 | pytest 63 | 64 | 8. **PyPI Package Publishing** 65 | 66 | - To request the publishing of the package to PyPI, please create an issue on `GitHub `_. Ensure that you have completed and verified all the local setup steps as outlined above before proceeding with the request. 67 | 68 | .. Note:: 69 | This guide assumes a basic understanding of Linux, Docker, and Git. For any issues, refer to the respective tool's documentation or community support. 70 | -------------------------------------------------------------------------------- /docs/Installation-Guide.rst: -------------------------------------------------------------------------------- 1 | LightGBM Transformation Installation Guide 2 | ========================================== 3 | 4 | Currently, the transform supported LightGBM can be built with gcc/g++ on linux platform. 5 | 6 | The following dependencies should be installed before compilation: 7 | 8 | - `CMake`_ 3.15 or later. 9 | 10 | - gcc/g++ 8 or later. 11 | 12 | - libboost 1.58 or later. 13 | 14 | - `llvm`_ 3.5.1 (exact match) with cmake parameter ``-DLLVM_REQUIRES_RTTI=1``. 15 | 16 | - `bond`_ with cmake parameter ``-DBOND_ENABLE_GRPC=FALSE``. 17 | 18 | See below docker file as an example for required dependencies installation. 19 | 20 | .. code:: 21 | 22 | FROM ubuntu:20.04 23 | # Install CMake, gcc, g++, boost. 24 | RUN apt-get update && apt-get -y upgrade && DEBIAN_FRONTEND="noninteractive" apt-get install -y libboost-all-dev gcc g++ wget cmake git curl libtinfo5 25 | # Install LLVM with RTTI feature. 26 | WORKDIR /root 27 | RUN wget https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-3.5.1.tar.gz && tar zxvf llvmorg-3.5.1.tar.gz 28 | WORKDIR /root/llvm-project-llvmorg-3.5.1/llvm 29 | RUN mkdir build && cd build && cmake -DLLVM_REQUIRES_RTTI=1 .. && make -j4 && make install 30 | # Install bond. 31 | WORKDIR /root 32 | RUN git clone --recursive https://github.com/microsoft/bond.git 33 | RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y clang zlib1g-dev 34 | RUN curl -sSL https://get.haskellstack.org/ | sh 35 | WORKDIR /root/bond 36 | RUN mkdir build && cd build && cmake -DBOND_ENABLE_GRPC=FALSE .. && make -j4 && make install 37 | ... 38 | 39 | To install LightGBM transform version, you could simply run following command: 40 | 41 | .. code:: 42 | 43 | pip install lightgbm-transform 44 | 45 | Alternatively, you could publish your own package with following commands: 46 | 47 | .. code:: 48 | 49 | git clone --recursive https://github.com/microsoft/lightgbm-transform/ 50 | cd lightgbm-transform 51 | # install twine first 52 | sh ./scripts/publish_python_package.sh 53 | 54 | 55 | .. _CMake: https://cmake.org/ 56 | 57 | .. _llvm: https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-3.5.1.tar.gz 58 | 59 | .. _bond: https://github.com/microsoft/bond.git 60 | -------------------------------------------------------------------------------- /examples/freeform2_parser.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "TransformProcessor.h" 9 | using namespace std; 10 | 11 | 12 | namespace LightGBM { 13 | class FreeForm2Parser: public Parser { 14 | public: 15 | FreeForm2Parser(std::string config_str) { 16 | // config should follow json format. 17 | std::stringstream config_ss(config_str); 18 | boost::property_tree::ptree ptree; 19 | boost::property_tree::read_json(config_ss, ptree); 20 | int label_idx = -1; 21 | auto label_node = ptree.get_child_optional("labelId"); 22 | if (label_node) { 23 | label_idx = label_node->get_value(); 24 | } 25 | std::string transform_str = ""; 26 | auto transform_node = ptree.get_child_optional("transform"); 27 | if (transform_node) { 28 | transform_str = transform_node->get_value(); 29 | } 30 | std::string header_str = ""; 31 | auto header_node = ptree.get_child_optional("header"); 32 | if (header_node) { 33 | header_str = header_node->get_value(); 34 | } 35 | Log::Info("Initializing transform processor."); 36 | transform_.reset(new TransformProcessor(transform_str, header_str, label_idx)); 37 | } 38 | inline void ParseOneLine(const char* str, std::vector>* out_features, double* out_label) const override { 39 | vector out_feature_strs; 40 | out_feature_strs.clear(); 41 | transform_->Parse(str, &out_feature_strs, out_label, "\t"); 42 | transform_->Apply(&out_feature_strs, out_features); 43 | } 44 | inline int NumFeatures() const override {return transform_->GetFeatureCount();} 45 | private: 46 | std::unique_ptr transform_; 47 | }; 48 | 49 | Parser* CreateObject(std::string config_str) { return new FreeForm2Parser(config_str);} 50 | ParserReflector reflector("FreeForm2Parser", CreateObject); 51 | } 52 | -------------------------------------------------------------------------------- /scripts/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include lightgbm/bin/*.so -------------------------------------------------------------------------------- /scripts/README.rst: -------------------------------------------------------------------------------- 1 | LightGBM Feature Transformation Binding Python-package 2 | ====================================================== 3 | 4 | |License| |Python Versions| |PyPI Version| |API Docs| 5 | 6 | Installation 7 | ------------ 8 | 9 | Preparation 10 | ''''''''''' 11 | 12 | 32-bit Python is not supported. Please install 64-bit version. If you have a strong need to install with 32-bit Python, refer to `Build 32-bit Version with 32-bit Python section <#build-32-bit-version-with-32-bit-python>`__. 13 | 14 | `setuptools `_ is needed. 15 | 16 | Install from `PyPI `_ 17 | '''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' 18 | 19 | .. code:: sh 20 | 21 | pip install lightgbm-transform 22 | 23 | You may need to install `wheel `_ via ``pip install wheel`` first. 24 | 25 | Compiled library that is included in the wheel file supports **LightGBM MPI** versions out of the box. 26 | 27 | Troubleshooting 28 | --------------- 29 | 30 | In case you are facing any errors during the installation process, you can examine ``$HOME/LightGBM_compilation.log`` file, in which all operations are logged, to get more details about occurred problem. Also, please attach this file to the issue on GitHub to help faster indicate the cause of the error. 31 | 32 | Refer to `FAQ `_. 33 | 34 | Examples 35 | -------- 36 | 37 | Refer to the walk through examples in `Transform tutorial `_. 38 | 39 | Development Guide 40 | ----------------- 41 | 42 | The code style of Python-package follows `PEP 8 `_. If you would like to make a contribution and not familiar with PEP 8, please check the PEP 8 style guide first. Otherwise, the check won't pass. Only E501 (line too long) and W503 (line break occurred before a binary operator) can be ignored. 43 | 44 | Documentation strings (docstrings) are written in the NumPy style. 45 | 46 | .. |License| image:: https://img.shields.io/github/license/microsoft/lightgbm.svg 47 | :target: https://github.com/microsoft/lightgbm-transform/blob/main/LICENSE 48 | .. |Python Versions| image:: https://img.shields.io/pypi/pyversions/lightgbm.svg?logo=python&logoColor=white 49 | :target: https://pypi.org/project/lightgbm-transform 50 | .. |PyPI Version| image:: https://img.shields.io/pypi/v/lightgbm.svg?logo=pypi&logoColor=white 51 | :target: https://pypi.org/project/lightgbm-transform 52 | .. |API Docs| image:: https://readthedocs.org/projects/lightgbm/badge/?version=latest 53 | :target: https://lightgbm.readthedocs.io/en/latest/Python-API.html 54 | -------------------------------------------------------------------------------- /scripts/generate_parser_config.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import textwrap 4 | 5 | if __name__ == '__main__': 6 | parser = argparse.ArgumentParser() 7 | parser.add_argument( 8 | '--class_name', 9 | help='The custom parser class name', 10 | ) 11 | parser.add_argument( 12 | '--transform_file', 13 | help='The file path for the feature spec.', 14 | ) 15 | parser.add_argument( 16 | '--header_file', 17 | help='The file path for the header file.', 18 | default="" 19 | ) 20 | parser.add_argument( 21 | '--parser_config_file', 22 | help='The output file path for the trained LightGBM model.', 23 | ) 24 | parser.add_argument( 25 | '--query_idx', 26 | type=int, 27 | help='The index of the query column in the origin input data.', 28 | default=-1 29 | ) 30 | args, _ = parser.parse_known_args() 31 | class_name = args.class_name 32 | transform_file = args.transform_file 33 | header_file = args.header_file 34 | parser_config_file = args.parser_config_file 35 | query_idx = args.query_idx 36 | # generate json config. 37 | with open(transform_file) as fin: 38 | transform_str = ''.join(fin.readlines()) 39 | 40 | parser_config = { 41 | "className": class_name, 42 | } 43 | if (len(header_file) > 0): 44 | with open(header_file) as fin: 45 | header_str = ''.join(fin.readlines()) 46 | if (query_idx > -1): 47 | header_list = header_str.split('\t') 48 | if(query_idx >= len(header_list)): 49 | raise ValueError("Invalid query_idx!") 50 | query_name = header_list[query_idx] 51 | feature_cnt = transform_str.count("Input") 52 | query_transform = textwrap.dedent(""" 53 | [Input:%d] 54 | Transform=Linear 55 | Name=%s 56 | Intercept=0 57 | Slope=1 58 | """) % (feature_cnt, query_name) 59 | transform_str = transform_str + query_transform 60 | print("Query_idx in transformed data is: %d" % feature_cnt) 61 | parser_config["transform"] = transform_str 62 | parser_config["header"] = header_str 63 | else : 64 | if (query_idx > -1): 65 | raise ValueError("No header file exists, can't auto generate query column for transformed data!") 66 | parser_config["transform"] = transform_str 67 | with open(parser_config_file, 'w') as fout: 68 | json.dump(parser_config, fout) 69 | -------------------------------------------------------------------------------- /scripts/load_precompiled_libs.py: -------------------------------------------------------------------------------- 1 | import ctypes 2 | from pathlib import Path 3 | 4 | CUSTOM_PARSER_LIB_NAME = 'lib_custom_parser.so' 5 | for p in ['lib_transform.so', 'lib_lightgbm.so', CUSTOM_PARSER_LIB_NAME]: 6 | # due to changes in LightGBM libpath.py https://github.com/microsoft/LightGBM/pull/6192 7 | ctypes.cdll.LoadLibrary(str(Path(__file__).resolve().parent / 'bin' / p)) 8 | -------------------------------------------------------------------------------- /scripts/publish_python_package.sh: -------------------------------------------------------------------------------- 1 | # run command `sh ./scripts/publish_python_package.sh` in repo root dir. 2 | lgb_python_pkg_dir="./external_libs/LightGBM/python-package" 3 | # compile transformation, lightgbm, and customized parser libs. 4 | mkdir build 5 | # precompile mpi option. 6 | cd build && cmake ../ -DUSE_MPI=ON && make -j4 && cd ../ || exit -1 7 | # copy all shared libs to lightgbm python package directory. 8 | # modify `basic.py` to load all libs first, or cannot find them when calling python interfaces. 9 | mkdir compile 10 | cp -r ${lgb_python_pkg_dir}/lightgbm ./compile/ || exit -1 11 | mkdir compile/lightgbm/bin 12 | cp ./lib_custom_parser.so ${lgb_python_pkg_dir}/lightgbm/bin && \ 13 | cp ./src/lib_transform.so ${lgb_python_pkg_dir}/lightgbm/bin && \ 14 | cp ./external_libs/LightGBM/lib_lightgbm.so ${lgb_python_pkg_dir}/lightgbm/bin || exit -1 15 | cat ./scripts/load_precompiled_libs.py ./compile/lightgbm/basic.py > tmp && cp tmp ./compile/lightgbm/basic.py || exit -1 16 | rm -rf tmp 17 | cp ./scripts/setup.py ./compile/ && cp ./scripts/README.rst ./compile && cp VERSION.txt ./compile && cp ./scripts/MANIFEST.in ./compile || exit -1 18 | # pack wheel package. 19 | cd compile && python setup.py bdist_wheel --precompile && cd ../ || exit -1 20 | # upload package to your pypi, use testpypi as an example. 21 | # twine upload --repository pypi compile/dist/* || exit -1 22 | # revert changes. 23 | # rm -rf ${lgb_python_pkg_dir}/lightgbm/*.so build compile || exit -1 24 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.15.0 FATAL_ERROR) 2 | 3 | project(transform) 4 | 5 | # following lib paths are required by linking directories not compiling. 6 | set(LLVM_LIB 7 | LLVMAnalysis 8 | LLVMAsmParser 9 | LLVMAsmPrinter 10 | LLVMBitReader 11 | LLVMBitWriter 12 | LLVMCodeGen 13 | LLVMCore 14 | LLVMDebugInfo 15 | LLVMExecutionEngine 16 | LLVMInstCombine 17 | LLVMInstrumentation 18 | LLVMInterpreter 19 | LLVMipa 20 | LLVMipo 21 | LLVMIRReader 22 | LLVMJIT 23 | LLVMLineEditor 24 | LLVMLinker 25 | LLVMLTO 26 | LLVMMC 27 | LLVMMCAnalysis 28 | LLVMMCDisassembler 29 | LLVMMCJIT 30 | LLVMMCParser 31 | LLVMObjCARCOpts 32 | LLVMObject 33 | LLVMOption 34 | LLVMProfileData 35 | LLVMRuntimeDyld 36 | LLVMScalarOpts 37 | LLVMSelectionDAG 38 | LLVMSupport 39 | LLVMTarget 40 | LLVMTransformUtils 41 | LLVMVectorize 42 | LLVMX86AsmParser 43 | LLVMX86AsmPrinter 44 | LLVMX86CodeGen 45 | LLVMX86Desc 46 | LLVMX86Disassembler 47 | LLVMX86Info 48 | LLVMX86Utils 49 | ) 50 | 51 | set(CMAKE_POSITION_INDEPENDENT_CODE ON) 52 | if(USE_DEBUG) 53 | SET(CMAKE_BUILD_TYPE "Debug") 54 | SET(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -Wall -g -ggdb") 55 | SET(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O3 -Wall") 56 | endif() 57 | 58 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/NeuralTree.Library) 59 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/DynamicRank.FreeForm.Library) 60 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/NeuralTreeEvaluator.Library) 61 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/TransformProcessor) 62 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.15.0 FATAL_ERROR) 2 | 3 | project(DynamicRank.FreeForm.Library) 4 | 5 | set(CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS} -fpermissive") 6 | 7 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libs/Shared) 8 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libs/Expression) 9 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libs/Parse/SExpression/libs) 10 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libs/Transform) 11 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libs/Backend/llvm) 12 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libs/External) 13 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/test) 14 | 15 | install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/inc/ 16 | DESTINATION include 17 | FILES_MATCHING PATTERN "*.h" 18 | ) 19 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/inc/FreeForm2.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for license information. 4 | */ 5 | // FreeForm2.h declares the main interface to the FreeForm2 expression 6 | // evaluator, used for feature evaluation in the neural net/tree ensemble 7 | // evaluator. 8 | 9 | #pragma once 10 | 11 | #ifndef FREEFORM2_FREEFORM2_H 12 | #define FREEFORM2_FREEFORM2_H 13 | 14 | #include "FreeForm2Executable.h" 15 | #include "FreeForm2Compiler.h" 16 | #include "FreeForm2Program.h" 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/inc/FreeForm2Assert.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_ASSERT_H 9 | #define FREEFORM2_ASSERT_H 10 | 11 | namespace FreeForm2 { 12 | // Assert a condition, providing file and line of the assertion call. 13 | // Note that this function throws on failure, rather than aborting (like 14 | // standard c assert). 15 | void ThrowAssert(bool p_condition, const char *p_file, unsigned int p_line); 16 | 17 | // Assert a condition, providing the expression, file and line of the 18 | // assertion call. 19 | // Note that this function throws on failure, rather than aborting (like 20 | // standard c assert). 21 | void ThrowAssert(bool p_condition, const char *p_expression, const char *p_file, 22 | unsigned int p_line); 23 | 24 | // Assert that this function call should not be reached during normal 25 | // program execution. Note that this function throws on failure, rather 26 | // than aborting. 27 | // __declspec(noreturn) 28 | void Unreachable(const char *p_file, unsigned int p_line); 29 | }; // namespace FreeForm2 30 | 31 | // Macros for asserting. 32 | // Call the regular FreeForm2::ThrowAssert function with macro-generated 33 | // parameters 34 | #define FF2_ASSERT(cond) \ 35 | FreeForm2::ThrowAssert((cond), #cond, __FILE__, __LINE__) 36 | 37 | // Call the regular FreeForm2::Unreachable function with macro-generated 38 | // parameters 39 | #define FF2_UNREACHABLE() FreeForm2::Unreachable(__FILE__, __LINE__) 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/inc/FreeForm2Compiler.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_INC_COMPILER_H 9 | #define FREEFORM2_INC_COMPILER_H 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | namespace DynamicRank { 17 | class IFeatureMap; 18 | } 19 | 20 | namespace FreeForm2 { 21 | class CompilerImpl; 22 | class ExternalDataManager; 23 | class Program; 24 | 25 | // This class contains the results of the compilation process. Since the 26 | // result of the compilation process is backend-dependent, this class 27 | // should be downcast to the backend result type according to the compiler 28 | // used. 29 | class CompilerResults : boost::noncopyable { 30 | public: 31 | virtual ~CompilerResults(); 32 | }; 33 | 34 | // A compiler compiles multiple programs using the same set of compiler 35 | // resources, which amortises costs across different programs. 36 | class Compiler : boost::noncopyable { 37 | public: 38 | explicit Compiler(std::auto_ptr p_impl); 39 | 40 | ~Compiler(); 41 | 42 | // Compile the program producing a backend-dependent results obejct. 43 | // This method, optionally producing debug output on stderr. 44 | std::unique_ptr Compile(const Program &p_program, 45 | bool p_debugOutput); 46 | 47 | // Default optimization level, used whenever the level is not explicitly 48 | // specified. 49 | static const unsigned int c_defaultOptimizationLevel = 0; 50 | 51 | private: 52 | // Pointer to implementation class (pimpl idiom). 53 | boost::scoped_ptr m_impl; 54 | }; 55 | } // namespace FreeForm2 56 | 57 | // This is used to prevent a breaking API change (moving compiler factory into 58 | // a separate header). 59 | #include "FreeForm2CompilerFactory.h" 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/inc/FreeForm2CompilerFactory.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_COMPILER_FACTORY_H 9 | #define FREEFORM2_COMPILER_FACTORY_H 10 | 11 | #include 12 | #include 13 | 14 | namespace FreeForm2 { 15 | namespace Cpp { 16 | class ExternResolver; 17 | } 18 | 19 | class Compiler; 20 | 21 | // This class contains methods to instantiate the various types of 22 | // compilers. 23 | class CompilerFactory { 24 | public: 25 | enum DestinationFunctionType { 26 | // typedef float (*DirectEvalFun)(StreamFeatureInput*, const FeatureType[], 27 | // OutputType[]). 28 | SingleDocumentEvaluation, 29 | 30 | // typedef float (*AggregatedEvalFun)(StreamFeatureInput*, const 31 | // FeatureType[][], UInt32, UInt32, OutputType[]). 32 | DocumentSetEvaluation, 33 | }; 34 | 35 | // Create a compiler which takes a program and compiles it into an 36 | // ExecutableCompilerResults object. p_optimizationLevel describes the 37 | // degree of optimization to perform on the code with an integer, 38 | // analagous to 'gcc -O p_optimizationLevel'. 39 | static std::unique_ptr CreateExecutableCompiler( 40 | unsigned int p_optimizationLevel, 41 | DestinationFunctionType p_destinationFunctionType = 42 | SingleDocumentEvaluation); 43 | 44 | // This method creates a Compiler object to compile a program to a 45 | // C++/IFM target. The results of this compiler are defined in 46 | // FreeForm2CppCompiler.h. 47 | static std::unique_ptr CreateCppIFMCompiler( 48 | const Cpp::ExternResolver &p_resolver); 49 | 50 | // This method creates a Compiler object to compile a program to a 51 | // C++/Barramundi target. The results of this compiler are defined 52 | // in FreeForm2CppCompiler.h. 53 | static std::unique_ptr CreateCppBarramundiCompiler( 54 | const Cpp::ExternResolver &p_resolver, const std::string &p_metadataPath); 55 | 56 | // This method creates a Compiler object to compile a program to a 57 | // C++/Barramundi target with debugging instrumentation present in the 58 | // program. The printf command is the name of a printf-style function 59 | // for use with debugging statements. The results of this compiler are 60 | // defined in FreeForm2CppCompiler.h. 61 | static std::unique_ptr CreateDebuggingCppBarramundiCompiler( 62 | const Cpp::ExternResolver &p_resolver, const std::string &p_metadataPath, 63 | const std::string &p_printfCommand); 64 | 65 | // This method creates a Compiler object to compile a program to a 66 | // C++/FPGA target. The results of this compiler are defined 67 | // in FreeForm2CppCompiler.h. 68 | static std::unique_ptr CreateCppFpgaCompiler( 69 | const Cpp::ExternResolver &p_resolver, 70 | const std::string &p_outputMappingPath, const std::string &p_msdlPath); 71 | }; 72 | } // namespace FreeForm2 73 | 74 | #endif 75 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/inc/FreeForm2Features.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_INC_FEATURES_H 9 | #define FREEFORM2_INC_FEATURES_H 10 | 11 | #include 12 | 13 | namespace FreeForm2 { 14 | class TypeImpl; 15 | 16 | // The feature information struct provides a namespace for feature-related 17 | // data declarations. 18 | struct FeatureInformation { 19 | // This enum lists the type of features supported by the feature 20 | // compiler. 21 | enum FeatureType { 22 | MetaStreamFeature, 23 | DerivedFeature, 24 | AggregatedDerivedFeature, 25 | AbInitioFeature 26 | }; 27 | }; 28 | 29 | // This namespace declares the names of required external data members for 30 | // metastream features. 31 | namespace RequiredMetaStreamData { 32 | // The number of query paths in the current query. 33 | struct NumQueryPaths { 34 | static const std::string &GetName(); 35 | static const TypeImpl &GetType(); 36 | }; 37 | 38 | // The number of words in the query. 39 | struct QueryLength { 40 | static const std::string &GetName(); 41 | static const TypeImpl &GetType(); 42 | }; 43 | 44 | // The index of word candidates per term in a specific query path. 45 | struct QueryPathCandidates { 46 | static const std::string &GetName(); 47 | }; 48 | 49 | // The stream data over which a metastream feature operates. 50 | struct Stream { 51 | static const std::string &GetName(); 52 | static const TypeImpl &GetType(); 53 | }; 54 | 55 | // The number of tuples of interest per type. 56 | struct TupleOfInterestCount { 57 | static const std::string &GetName(); 58 | }; 59 | 60 | // The tuples of interest. 61 | struct TuplesOfInterest { 62 | static const std::string &GetName(); 63 | }; 64 | 65 | // Duplicate term information. 66 | struct UnsafeDuplicateTermInformation { 67 | static const std::string &GetName(); 68 | }; 69 | } // namespace RequiredMetaStreamData 70 | 71 | // This namespace declares the external data members required for 72 | // compilation of derived features. 73 | namespace RequiredDerivedFeatureData { 74 | // The data member representing the stream ID. 75 | struct StreamID { 76 | static const std::string &GetName(); 77 | static const TypeImpl &GetType(); 78 | }; 79 | } // namespace RequiredDerivedFeatureData 80 | } // namespace FreeForm2 81 | 82 | #endif 83 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Backend/llvm/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.15) 2 | 3 | set(PROJECT_NAME DRFreeFormLlvmBackendLibrary) 4 | 5 | project(${PROJECT_NAME}) 6 | 7 | set(CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS} -fpermissive") 8 | 9 | add_library(${PROJECT_NAME} STATIC 10 | ${CMAKE_CURRENT_SOURCE_DIR}/Extend/FreeForm2Support.cpp 11 | ${CMAKE_CURRENT_SOURCE_DIR}/Extend/JITEmitter.cpp 12 | ${CMAKE_CURRENT_SOURCE_DIR}/Extend/JITExtend.cpp 13 | ${CMAKE_CURRENT_SOURCE_DIR}/CompilationState.cpp 14 | ${CMAKE_CURRENT_SOURCE_DIR}/ArrayCodeGen.cpp 15 | ${CMAKE_CURRENT_SOURCE_DIR}/LlvmCodeGenerator.cpp 16 | ${CMAKE_CURRENT_SOURCE_DIR}/LlvmCodeGenUtils.cpp 17 | ${CMAKE_CURRENT_SOURCE_DIR}/LlvmCompiler.cpp 18 | ${CMAKE_CURRENT_SOURCE_DIR}/LlvmRuntimeLibrary.cpp 19 | ) 20 | 21 | target_include_directories(${PROJECT_NAME} PRIVATE 22 | ${CMAKE_CURRENT_SOURCE_DIR} 23 | ${CMAKE_CURRENT_SOURCE_DIR}/Extend 24 | ${CMAKE_CURRENT_SOURCE_DIR}/../../../inc 25 | ${CMAKE_CURRENT_SOURCE_DIR}/../../../../NeuralTree.Library/inc 26 | ${CMAKE_CURRENT_SOURCE_DIR}/../../Shared 27 | ${CMAKE_CURRENT_SOURCE_DIR}/../../Expression 28 | ${CMAKE_CURRENT_SOURCE_DIR}/../../External 29 | ${CMAKE_CURRENT_SOURCE_DIR}/../../Transform 30 | ${CMAKE_CURRENT_SOURCE_DIR}/../../Parse/SExpression/inc 31 | ) 32 | 33 | install(TARGETS ${PROJECT_NAME} 34 | LIBRARY DESTINATION lib 35 | ARCHIVE DESTINATION lib 36 | ) -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Backend/llvm/Extend/FreeForm2Support.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "FreeForm2Support.h" 7 | 8 | #include 9 | 10 | #include "JITExtend.h" 11 | 12 | namespace llvm { 13 | struct FakeJITEmitter : public llvm::JITCodeEmitter { 14 | void *MemMgr; 15 | 16 | // When outputting a function stub in the context of some other function, we 17 | // save BufferBegin/BufferEnd/CurBufferPtr here. 18 | uint8_t *SavedBufferBegin, *SavedBufferEnd, *SavedCurBufferPtr; 19 | 20 | // When reattempting to JIT a function after running out of space, we store 21 | // the estimated size of the function we're trying to JIT here, so we can 22 | // ask the memory manager for at least this much space. When we 23 | // successfully emit the function, we reset this back to zero. 24 | uintptr_t SizeEstimate; 25 | 26 | /// Relocations - These are the relocations that the function needs, as 27 | /// emitted. 28 | std::vector m_relocations; 29 | }; 30 | 31 | const std::vector &GetMachineRelocations( 32 | const ExecutionEngine *p_engine) { 33 | const JIT *jit = static_cast(p_engine); 34 | return GetJitMachineRelocations(jit); 35 | } 36 | } // namespace llvm 37 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Backend/llvm/Extend/FreeForm2Support.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #ifndef LLVM_FREEFORM2_SUPPORT_H 7 | #define LLVM_FREEFORM2_SUPPORT_H 8 | 9 | #include 10 | 11 | #include 12 | 13 | namespace llvm { 14 | class ExecutionEngine; 15 | 16 | const std::vector &GetMachineRelocations( 17 | const ExecutionEngine *p_engine); 18 | } // namespace llvm 19 | #endif 20 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Backend/llvm/LlvmRuntimeLibrary.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #include 9 | 10 | #include 11 | #include 12 | 13 | namespace llvm { 14 | class LLVMContext; 15 | class ExecutionEngine; 16 | class Module; 17 | class GlobalValue; 18 | class Function; 19 | } // namespace llvm 20 | 21 | namespace FreeForm2 { 22 | class LlvmRuntimeLibrary : boost::noncopyable { 23 | public: 24 | // Create a LLVM Runtime Library using the specified LLVMContext. 25 | LlvmRuntimeLibrary(llvm::LLVMContext &p_context); 26 | 27 | // Destroy the implementation. 28 | ~LlvmRuntimeLibrary(); 29 | 30 | // Add all runtime symbols to the specified module. This is similar to 31 | // adding forward declarations to a .cpp file. 32 | void AddLibraryToModule(llvm::Module &p_module) const; 33 | 34 | // Add global value mappings to an exeuction engine, which is necessary 35 | // when linking a module which uses the runtime library. 36 | void AddExecutionMappings(llvm::ExecutionEngine &p_engine) const; 37 | 38 | // Look up a GlobalValue by name. GlobalValues generally include 39 | // external variables and functions; see the LLVM documentation for 40 | // more information. If the value is not found, this method returns 41 | // null. 42 | llvm::GlobalValue *FindValue(SIZED_STRING p_name) const; 43 | 44 | // Find a runtime function with the specified name. This is a 45 | // specialization of FindValue for Functions. If the function is not 46 | // found, or if the GlobalValue is not a function, this method returns 47 | // null. 48 | llvm::Function *FindFunction(SIZED_STRING p_name) const; 49 | 50 | private: 51 | // The implementation of this class is hidden. 52 | class Impl; 53 | 54 | // Pointer to the implementation. 55 | std::unique_ptr m_impl; 56 | }; 57 | } // namespace FreeForm2 58 | 59 | extern "C" double FreeForm2GetRandomValue(); 60 | 61 | inline unsigned long GetTickCount() { 62 | struct timespec ts; 63 | 64 | clock_gettime(CLOCK_MONOTONIC, &ts); 65 | 66 | return (ts.tv_sec * 1000 + ts.tv_nsec / 1000000); 67 | } 68 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/Allocation.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "Allocation.h" 7 | 8 | #include "ArrayType.h" 9 | #include "FreeForm2Assert.h" 10 | 11 | using namespace FreeForm2; 12 | 13 | FreeForm2::Allocation::Allocation(AllocationType p_allocType, VariableID p_id, 14 | const TypeImpl &p_type) 15 | : m_allocType(p_allocType), m_id(p_id), m_type(p_type), m_children(0) { 16 | if (p_allocType == ArrayLiteral) { 17 | FF2_ASSERT(m_type.Primitive() == Type::Array); 18 | m_children = 1; 19 | 20 | const ArrayType &arrayType = static_cast(p_type); 21 | for (UInt32 i = 0; i < arrayType.GetDimensionCount(); i++) { 22 | m_children *= arrayType.GetDimensions()[i]; 23 | } 24 | } 25 | } 26 | 27 | FreeForm2::Allocation::Allocation(AllocationType p_allocType, VariableID p_id, 28 | const TypeImpl &p_type, size_t p_children) 29 | : m_allocType(p_allocType), 30 | m_id(p_id), 31 | m_type(p_type), 32 | m_children(p_children) {} 33 | 34 | FreeForm2::Allocation::AllocationType FreeForm2::Allocation::GetAllocationType() 35 | const { 36 | return m_allocType; 37 | } 38 | 39 | const TypeImpl &FreeForm2::Allocation::GetType() const { return m_type; } 40 | 41 | VariableID FreeForm2::Allocation::GetAllocationId() const { return m_id; } 42 | 43 | size_t FreeForm2::Allocation::GetNumChildren() const { return m_children; } 44 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/Allocation.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for license information. 4 | */ 5 | #pragma once 6 | #ifndef FREEFORM2_ARRAYALLOCATIONWRAPPER_H 7 | #define FREEFORM2_ARRAYALLOCATIONWRAPPER_H 8 | 9 | #include "Expression.h" 10 | #include "Visitor.h" 11 | #include 12 | 13 | namespace FreeForm2 14 | { 15 | class AllocationExpression; 16 | 17 | // Wraps the original expression and decouples array allocation from 18 | // initialization. 19 | class Allocation : public boost::noncopyable 20 | { 21 | public: 22 | enum AllocationType 23 | { 24 | ArrayLiteral, 25 | FeatureArray, 26 | ExternArray, 27 | LiteralStream, 28 | LiteralWord, 29 | Declaration 30 | }; 31 | 32 | Allocation(AllocationType p_allocType, 33 | VariableID p_id, 34 | const TypeImpl &p_type); 35 | 36 | Allocation(AllocationType p_allocType, 37 | VariableID p_id, 38 | const TypeImpl &p_type, 39 | size_t p_children); 40 | 41 | // Gets the type of the element to be allocated. 42 | const TypeImpl &GetType() const; 43 | 44 | // Gets the number of children of the allocation. 45 | size_t GetNumChildren() const; 46 | 47 | // Gets the type of the allocation. 48 | AllocationType GetAllocationType() const; 49 | 50 | // Gets the identifier of the array. 51 | VariableID GetAllocationId() const; 52 | 53 | private: 54 | // The type of the allocation. 55 | const AllocationType m_allocType; 56 | 57 | // The number of children. 58 | size_t m_children; 59 | 60 | // The array identificator. 61 | const VariableID m_id; 62 | 63 | // The type of the element to be allocated. 64 | const TypeImpl &m_type; 65 | }; 66 | } 67 | 68 | #endif 69 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/ArrayDereferenceExpression.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_ARRAY_DEREFERENCE_EXPRESSION_H 9 | #define FREEFORM2_ARRAY_DEREFERENCE_EXPRESSION_H 10 | 11 | #include "Expression.h" 12 | 13 | namespace FreeForm2 { 14 | class ArrayType; 15 | class ProgramParseState; 16 | 17 | // An array-dereference expression removes a dimension from an array. 18 | class ArrayDereferenceExpression : public Expression { 19 | public: 20 | ArrayDereferenceExpression(const Annotations &p_annotations, 21 | const Expression &p_array, 22 | const Expression &p_index, size_t p_version); 23 | 24 | // Methods inherited from Expression. 25 | virtual size_t GetNumChildren() const override; 26 | virtual void Accept(Visitor &p_visitor) const override; 27 | virtual void AcceptReference(Visitor &p_visitor) const override; 28 | virtual const TypeImpl &GetType() const override; 29 | 30 | const Expression &GetArray() const; 31 | const Expression &GetIndex() const; 32 | size_t GetVersion() const; 33 | 34 | // Gets the VariableID of the array object, regardless of the number 35 | // of dereferences. 36 | VariableID GetBaseArrayId() const; 37 | 38 | private: 39 | // Dereferenced type of this expression. 40 | const TypeImpl &m_type; 41 | 42 | // Array expression being dereferenced. 43 | const Expression &m_array; 44 | 45 | // Index supplied. 46 | const Expression &m_index; 47 | 48 | // A unique version number associated with a particular 49 | // value for this variable. 50 | const size_t m_version; 51 | }; 52 | }; // namespace FreeForm2 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/ArrayLength.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "ArrayLength.h" 7 | 8 | #include 9 | 10 | #include "Expression.h" 11 | #include "FreeForm2Assert.h" 12 | #include "SimpleExpressionOwner.h" 13 | #include "Visitor.h" 14 | 15 | FreeForm2::ArrayLengthExpression::ArrayLengthExpression( 16 | const Annotations &p_annotations, const Expression &p_array) 17 | : Expression(p_annotations), m_array(p_array) {} 18 | 19 | void FreeForm2::ArrayLengthExpression::Accept(Visitor &p_visitor) const { 20 | size_t stackSize = p_visitor.StackSize(); 21 | 22 | if (!p_visitor.AlternativeVisit(*this)) { 23 | m_array.Accept(p_visitor); 24 | 25 | p_visitor.Visit(*this); 26 | } 27 | 28 | FF2_ASSERT(p_visitor.StackSize() == stackSize + p_visitor.StackIncrement()); 29 | } 30 | 31 | const FreeForm2::TypeImpl &FreeForm2::ArrayLengthExpression::GetType() const { 32 | if (m_array.GetType().Primitive() != Type::Array) { 33 | std::ostringstream err; 34 | err << "Argument to array-length expression must be " 35 | << "an array (got type '" << m_array.GetType() << "')"; 36 | throw ParseError(err.str(), GetSourceLocation()); 37 | } 38 | 39 | return TypeImpl::GetUInt32Instance(true); 40 | } 41 | 42 | size_t FreeForm2::ArrayLengthExpression::GetNumChildren() const { return 1; } 43 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/ArrayLength.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_ARRAY_LENGTH_H 9 | #define FREEFORM2_ARRAY_LENGTH_H 10 | 11 | #include "Expression.h" 12 | 13 | namespace FreeForm2 { 14 | class ArrayLengthExpression : public Expression { 15 | public: 16 | // Construct an array length expression given the array in question. 17 | ArrayLengthExpression(const Annotations &p_annotations, 18 | const Expression &p_array); 19 | 20 | virtual void Accept(Visitor &p_visitor) const override; 21 | 22 | // Return the type of an array-length expression (int). 23 | virtual const TypeImpl &GetType() const override; 24 | 25 | // Return the number of child nodes for this expression. 26 | virtual size_t GetNumChildren() const override; 27 | 28 | // Get the array 29 | const Expression &GetArray() const { return m_array; } 30 | 31 | private: 32 | // Array that we're calculating the length of. 33 | const Expression &m_array; 34 | }; 35 | }; // namespace FreeForm2 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/BinaryOperator.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_BINARY_OPERATOR_H 9 | #define FREEFORM2_BINARY_OPERATOR_H 10 | 11 | #include "Expression.h" 12 | #include "FreeForm2Type.h" 13 | 14 | namespace FreeForm2 { 15 | class BinaryOperator { 16 | public: 17 | enum Operation { 18 | plus, 19 | minus, 20 | multiply, 21 | divides, 22 | mod, 23 | max, 24 | min, 25 | pow, 26 | log, 27 | 28 | eq, 29 | neq, 30 | lt, 31 | lte, 32 | gt, 33 | gte, 34 | 35 | _and, 36 | _or, 37 | 38 | _bitand, 39 | _bitor, 40 | bitshiftleft, 41 | bitshiftright, 42 | 43 | invalid 44 | }; 45 | 46 | // Select the best operand type for an operator. Best is defined in 47 | // terms of TypeUtil::SelectBestType. If no valid type is found, an 48 | // invalid TypeImpl is returned. 49 | static const TypeImpl &GetBestOperandType(Operation p_operator, 50 | const TypeImpl &p_operandType); 51 | 52 | // Return the type of a binary operator result given an operator and 53 | // an operand type. If the operand type is not a valid operand type for 54 | // the operator, the return type is undefined. 55 | static const TypeImpl &GetResultType(Operation p_operator, 56 | const TypeImpl &p_operandType); 57 | }; 58 | } // namespace FreeForm2 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/BlockExpression.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_BLOCK_EXPRESSION_H 9 | #define FREEFORM2_BLOCK_EXPRESSION_H 10 | 11 | #include 12 | 13 | #include "Expression.h" 14 | 15 | namespace FreeForm2 { 16 | // A block expression is a series of expressions. 17 | class BlockExpression : public Expression { 18 | public: 19 | // Allocate a block expression for the given array of child expressions, 20 | // with p_numBound indicating the number of symbols bound by (and not 21 | // scoped within) the immediate children of the block expression. 22 | static boost::shared_ptr Alloc( 23 | const Annotations &p_annotations, const Expression **p_children, 24 | unsigned int p_numChildren, unsigned int p_numBound); 25 | 26 | // Return the number of symbols bound by immediate children of this 27 | // block expression, and left open. 28 | unsigned int GetNumBound() const; 29 | 30 | // Methods inherited from Expression. 31 | virtual size_t GetNumChildren() const override; 32 | virtual void Accept(Visitor &p_visitor) const override; 33 | virtual const TypeImpl &GetType() const override; 34 | 35 | // Gets the p_index-th child of this expression. 36 | const Expression &GetChild(unsigned int p_index) const; 37 | 38 | private: 39 | // Private ctor: use Alloc to create block expressions. 40 | BlockExpression(const Annotations &p_annotations, 41 | const Expression **p_children, unsigned int p_numChildren, 42 | unsigned int p_numBound); 43 | 44 | // Custom deallocation method. 45 | static void DeleteAlloc(BlockExpression *p_allocated); 46 | 47 | // The return type of the block. 48 | const TypeImpl *m_returnType; 49 | 50 | // Number of children of this block. 51 | unsigned int m_numChildren; 52 | 53 | // Number of symbols left bound by children of the block expression. 54 | unsigned int m_numBound; 55 | 56 | // Children, allocated via struct hack. 57 | const Expression *m_children[1]; 58 | }; 59 | }; // namespace FreeForm2 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.15) 2 | 3 | set(PROJECT_NAME DRFreeFormExpressionLibrary) 4 | 5 | project(${PROJECT_NAME}) 6 | 7 | set(CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS} -fpermissive") 8 | 9 | add_library(${PROJECT_NAME} STATIC 10 | ${CMAKE_CURRENT_SOURCE_DIR}/Allocation.cpp 11 | ${CMAKE_CURRENT_SOURCE_DIR}/ArrayDereferenceExpression.cpp 12 | ${CMAKE_CURRENT_SOURCE_DIR}/ArrayLength.cpp 13 | ${CMAKE_CURRENT_SOURCE_DIR}/ArrayLiteralExpression.cpp 14 | ${CMAKE_CURRENT_SOURCE_DIR}/BinaryOperator.cpp 15 | ${CMAKE_CURRENT_SOURCE_DIR}/BlockExpression.cpp 16 | ${CMAKE_CURRENT_SOURCE_DIR}/Conditional.cpp 17 | ${CMAKE_CURRENT_SOURCE_DIR}/ConvertExpression.cpp 18 | ${CMAKE_CURRENT_SOURCE_DIR}/DebugExpression.cpp 19 | ${CMAKE_CURRENT_SOURCE_DIR}/Declaration.cpp 20 | ${CMAKE_CURRENT_SOURCE_DIR}/Expression.cpp 21 | ${CMAKE_CURRENT_SOURCE_DIR}/Extern.cpp 22 | ${CMAKE_CURRENT_SOURCE_DIR}/FeatureSpec.cpp 23 | ${CMAKE_CURRENT_SOURCE_DIR}/Function.cpp 24 | ${CMAKE_CURRENT_SOURCE_DIR}/LetExpression.cpp 25 | ${CMAKE_CURRENT_SOURCE_DIR}/LiteralExpression.cpp 26 | ${CMAKE_CURRENT_SOURCE_DIR}/Match.cpp 27 | ${CMAKE_CURRENT_SOURCE_DIR}/MemberAccessExpression.cpp 28 | ${CMAKE_CURRENT_SOURCE_DIR}/Mutation.cpp 29 | ${CMAKE_CURRENT_SOURCE_DIR}/OperatorExpression.cpp 30 | ${CMAKE_CURRENT_SOURCE_DIR}/PhiNode.cpp 31 | ${CMAKE_CURRENT_SOURCE_DIR}/Publish.cpp 32 | ${CMAKE_CURRENT_SOURCE_DIR}/RandExpression.cpp 33 | ${CMAKE_CURRENT_SOURCE_DIR}/RangeReduceExpression.cpp 34 | ${CMAKE_CURRENT_SOURCE_DIR}/RefExpression.cpp 35 | ${CMAKE_CURRENT_SOURCE_DIR}/SelectNth.cpp 36 | ${CMAKE_CURRENT_SOURCE_DIR}/StateMachine.cpp 37 | ${CMAKE_CURRENT_SOURCE_DIR}/StreamData.cpp 38 | ${CMAKE_CURRENT_SOURCE_DIR}/SymbolTable.cpp 39 | ${CMAKE_CURRENT_SOURCE_DIR}/TypeUtil.cpp 40 | ${CMAKE_CURRENT_SOURCE_DIR}/UnaryOperator.cpp 41 | ) 42 | 43 | target_include_directories(${PROJECT_NAME} PRIVATE 44 | ${CMAKE_CURRENT_SOURCE_DIR} 45 | ${CMAKE_CURRENT_SOURCE_DIR}/../../inc 46 | ${CMAKE_CURRENT_SOURCE_DIR}/../../../NeuralTree.Library/inc 47 | ${CMAKE_CURRENT_SOURCE_DIR}/../Shared 48 | ) 49 | 50 | install(TARGETS ${PROJECT_NAME} 51 | LIBRARY DESTINATION lib 52 | ARCHIVE DESTINATION lib 53 | ) -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/Conditional.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "Conditional.h" 7 | 8 | #include 9 | 10 | #include "Expression.h" 11 | #include "FreeForm2.h" 12 | #include "FreeForm2Assert.h" 13 | #include "SimpleExpressionOwner.h" 14 | #include "Visitor.h" 15 | 16 | FreeForm2::ConditionalExpression::ConditionalExpression( 17 | const Annotations &p_annotations, const Expression &p_condition, 18 | const Expression &p_then, const Expression &p_else) 19 | : Expression(p_annotations), 20 | m_condition(p_condition), 21 | m_then(p_then), 22 | m_else(p_else) {} 23 | 24 | const FreeForm2::Expression &FreeForm2::ConditionalExpression::GetCondition() 25 | const { 26 | return m_condition; 27 | } 28 | 29 | const FreeForm2::Expression &FreeForm2::ConditionalExpression::GetThen() const { 30 | return m_then; 31 | } 32 | 33 | const FreeForm2::Expression &FreeForm2::ConditionalExpression::GetElse() const { 34 | return m_else; 35 | } 36 | 37 | void FreeForm2::ConditionalExpression::Accept(Visitor &p_visitor) const { 38 | size_t stackSize = p_visitor.StackSize(); 39 | 40 | if (!p_visitor.AlternativeVisit(*this)) { 41 | m_else.Accept(p_visitor); 42 | m_then.Accept(p_visitor); 43 | m_condition.Accept(p_visitor); 44 | 45 | p_visitor.Visit(*this); 46 | } 47 | 48 | FF2_ASSERT(p_visitor.StackSize() == stackSize + p_visitor.StackIncrement()); 49 | } 50 | 51 | const FreeForm2::TypeImpl &FreeForm2::ConditionalExpression::GetType() const { 52 | if (m_condition.GetType().Primitive() != Type::Bool) { 53 | std::ostringstream err; 54 | err << "Condition of type '" << m_condition.GetType() 55 | << "' supplied to if expression as condition " 56 | << "(expected boolean)"; 57 | throw ParseError(err.str(), GetSourceLocation()); 58 | } 59 | 60 | if (!m_then.GetType().IsSameAs(m_else.GetType(), true)) { 61 | if (m_then.GetType().IsIntegerType() && m_else.GetType().IsIntegerType()) { 62 | return TypeImpl::GetIntInstance(true); 63 | } else { 64 | std::ostringstream err; 65 | err << "'then' (supplied '" << m_then.GetType() 66 | << "' and 'else' (supplied '" << m_else.GetType() 67 | << "' clauses of condition must have matching types."; 68 | throw ParseError(err.str(), GetSourceLocation()); 69 | } 70 | } 71 | 72 | return m_then.GetType().AsConstType(); 73 | } 74 | 75 | size_t FreeForm2::ConditionalExpression::GetNumChildren() const { return 3; } 76 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/Conditional.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_CONDITIONAL_H 9 | #define FREEFORM2_CONDITIONAL_H 10 | 11 | #include "Expression.h" 12 | 13 | namespace FreeForm2 { 14 | class ConditionalExpression : public Expression { 15 | public: 16 | ConditionalExpression(const Annotations &p_annotations, 17 | const Expression &p_condition, const Expression &p_then, 18 | const Expression &p_else); 19 | 20 | virtual void Accept(Visitor &p_visitor) const override; 21 | virtual size_t GetNumChildren() const override; 22 | virtual const TypeImpl &GetType() const override; 23 | 24 | // Gets the condition expression for this conditional. 25 | const Expression &GetCondition() const; 26 | 27 | // Gets the then expression for this conditional. 28 | const Expression &GetThen() const; 29 | 30 | // Gets the else expression for this conditional. 31 | const Expression &GetElse() const; 32 | 33 | private: 34 | // Condition used to choose between then/else. 35 | const Expression &m_condition; 36 | 37 | // Value if condition is true. 38 | const Expression &m_then; 39 | 40 | // Value if condition is false. 41 | const Expression &m_else; 42 | }; 43 | }; // namespace FreeForm2 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/DebugExpression.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "DebugExpression.h" 7 | 8 | #include 9 | 10 | #include "ArrayType.h" 11 | #include "FreeForm2Assert.h" 12 | #include "TypeImpl.h" 13 | #include "Visitor.h" 14 | 15 | using namespace FreeForm2; 16 | 17 | DebugExpression::DebugExpression(const Annotations &p_annotations, 18 | const Expression &p_child, 19 | const std::string &p_childText) 20 | : Expression(p_annotations), m_child(p_child), m_childText(p_childText) {} 21 | 22 | void DebugExpression::Accept(Visitor &p_visitor) const { 23 | const size_t stackSize = p_visitor.StackSize(); 24 | 25 | if (!p_visitor.AlternativeVisit(*this)) { 26 | m_child.Accept(p_visitor); 27 | p_visitor.Visit(*this); 28 | } 29 | 30 | FF2_ASSERT(p_visitor.StackSize() == stackSize + p_visitor.StackIncrement()); 31 | } 32 | 33 | const TypeImpl &DebugExpression::GetType() const { 34 | const TypeImpl &childType = m_child.GetType(); 35 | const TypeImpl *checkType = &childType; 36 | if (childType.Primitive() == Type::Array) { 37 | const ArrayType &type = static_cast(childType); 38 | checkType = &type.GetChildType(); 39 | } 40 | 41 | if (!checkType->IsLeafType()) { 42 | std::ostringstream err; 43 | err << "Cannot debug the expression " << m_childText << " of type " 44 | << *checkType << ". Only arrays and primitive types are supported."; 45 | throw std::runtime_error(err.str()); 46 | } 47 | 48 | return TypeImpl::GetVoidInstance(); 49 | } 50 | 51 | size_t DebugExpression::GetNumChildren() const { return 1; } 52 | 53 | const Expression &DebugExpression::GetChild() const { return m_child; } 54 | 55 | const std::string &DebugExpression::GetChildText() const { return m_childText; } 56 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/DebugExpression.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for license information. 4 | */ 5 | #pragma once 6 | 7 | #ifndef FREEFORM2_DEBUG_EXPRESSION_H 8 | #define FREEFORM2_DEBUG_EXPRESSION_H 9 | 10 | #include "Expression.h" 11 | #include 12 | 13 | namespace FreeForm2 14 | { 15 | // A DebugExpression assists developers by allowing expressions to be 16 | // debugged. The exact method of debugging depends on other compiler 17 | // settings, but generally debug instrumentation should provide the 18 | // original text of the expression being debugged, and the value of that 19 | // expression. 20 | class DebugExpression : public Expression 21 | { 22 | public: 23 | // Construct a DebugExpression which will provide debugging 24 | // capabilities for an expression. The child text refers to the 25 | // original text of the expression to debug. 26 | DebugExpression(const Annotations &p_annotations, 27 | const Expression &p_child, 28 | const std::string &p_childText); 29 | 30 | // Methods inherited from Expression. 31 | virtual void Accept(Visitor &) const override; 32 | virtual const TypeImpl &GetType() const override; 33 | virtual size_t GetNumChildren() const override; 34 | 35 | // Get the child expression. 36 | const Expression &GetChild() const; 37 | 38 | // Get the text of the child expression to be printed for debugging 39 | // purposes. 40 | const std::string &GetChildText() const; 41 | 42 | private: 43 | // The child expression. 44 | const Expression &m_child; 45 | 46 | // The original text of the child expression. 47 | std::string m_childText; 48 | }; 49 | } 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/Declaration.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "Declaration.h" 7 | 8 | #include 9 | 10 | #include "ArrayType.h" 11 | #include "FreeForm2Assert.h" 12 | //#include 13 | #include "TypeUtil.h" 14 | #include "Visitor.h" 15 | 16 | FreeForm2::DeclarationExpression::DeclarationExpression( 17 | const Annotations &p_annotations, const TypeImpl &p_type, 18 | const Expression &p_init, bool p_voidValue, VariableID p_id, 19 | size_t p_version) 20 | : Expression(p_annotations), 21 | m_declType(p_type), 22 | m_init(p_init), 23 | m_voidValue(p_voidValue), 24 | m_id(p_id), 25 | m_version(p_version) {} 26 | 27 | void FreeForm2::DeclarationExpression::Accept(Visitor &p_visitor) const { 28 | size_t stackSize = p_visitor.StackSize(); 29 | 30 | if (!p_visitor.AlternativeVisit(*this)) { 31 | m_init.Accept(p_visitor); 32 | p_visitor.Visit(*this); 33 | } 34 | 35 | FF2_ASSERT(p_visitor.StackSize() == stackSize + p_visitor.StackIncrement()); 36 | } 37 | 38 | const FreeForm2::TypeImpl &FreeForm2::DeclarationExpression::GetType() const { 39 | if (m_declType.Primitive() != Type::Unknown && 40 | !TypeUtil::IsAssignable(m_declType, m_init.GetType())) { 41 | std::ostringstream err; 42 | err << "Declaration initializer (of type " << m_init.GetType() 43 | << ") does not match declared type of variable (" << m_declType << ")"; 44 | throw ParseError(err.str(), GetSourceLocation()); 45 | } 46 | 47 | if (m_voidValue) { 48 | return TypeImpl::GetVoidInstance(); 49 | } else { 50 | FF2_ASSERT(m_init.GetType().Primitive() != Type::Unknown); 51 | FF2_ASSERT(m_init.GetType().Primitive() != Type::Void); 52 | return m_init.GetType().AsConstType(); 53 | } 54 | } 55 | 56 | size_t FreeForm2::DeclarationExpression::GetNumChildren() const { return 1; } 57 | 58 | const FreeForm2::Expression &FreeForm2::DeclarationExpression::GetInit() const { 59 | return m_init; 60 | } 61 | 62 | bool FreeForm2::DeclarationExpression::HasVoidValue() const { 63 | return m_voidValue; 64 | } 65 | 66 | const FreeForm2::TypeImpl &FreeForm2::DeclarationExpression::GetDeclaredType() 67 | const { 68 | return m_declType; 69 | } 70 | 71 | FreeForm2::VariableID FreeForm2::DeclarationExpression::GetId() const { 72 | return m_id; 73 | } 74 | 75 | size_t FreeForm2::DeclarationExpression::GetVersion() const { 76 | return m_version; 77 | } 78 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/Declaration.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_DECLARATION_H 9 | #define FREEFORM2_DECLARATION_H 10 | 11 | #include "Expression.h" 12 | 13 | namespace FreeForm2 { 14 | class DeclarationExpression : public Expression { 15 | public: 16 | // Create a declaration expression from a type (which may be 17 | // Type::Unknown) and an initialiser. p_voidValue controls whether 18 | // the DeclarationExpression evaluates to a void value (imperatively), 19 | // or to the p_init expression (functionally). 20 | DeclarationExpression(const Annotations &p_annotations, 21 | const TypeImpl &p_type, const Expression &p_init, 22 | bool p_voidValue, VariableID p_id, size_t p_version); 23 | 24 | // Virtual methods inherited from Expression. 25 | virtual void Accept(Visitor &p_visitor) const override; 26 | virtual size_t GetNumChildren() const override; 27 | virtual const TypeImpl &GetType() const override; 28 | 29 | // Whether this expression evaluates to void, or the value m_init; 30 | bool HasVoidValue() const; 31 | 32 | // Return the type of the variable declared (not the type of this 33 | // declaration expression, which is significantly different). 34 | const TypeImpl &GetDeclaredType() const; 35 | 36 | // Return the initialization expression. 37 | const Expression &GetInit() const; 38 | 39 | // Gets this declaration's unique identifier and value version. 40 | VariableID GetId() const; 41 | size_t GetVersion() const; 42 | 43 | private: 44 | // Type of the variable declared by this expression. (Note: this is 45 | // *not* the type of the declaration expression); 46 | const TypeImpl &m_declType; 47 | 48 | // Initialisation expression. 49 | const Expression &m_init; 50 | 51 | // Whether this expression evaluates to void, or the value m_init; 52 | bool m_voidValue; 53 | 54 | // A unique identificator to allow separation of allocation and usage. 55 | const VariableID m_id; 56 | 57 | // A unique version number associated with a particular 58 | // value for this variable. 59 | const size_t m_version; 60 | }; 61 | }; // namespace FreeForm2 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/LetExpression.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_LETEXPRESSION_H 9 | #define FREEFORM2_LETEXPRESSION_H 10 | 11 | #include 12 | #include 13 | 14 | #include "Expression.h" 15 | 16 | namespace FreeForm2 { 17 | class LetExpression : public Expression { 18 | public: 19 | typedef std::pair IdExpressionPair; 20 | 21 | // Methods inherited from Expression. 22 | virtual size_t GetNumChildren() const override; 23 | virtual void Accept(Visitor &p_visitor) const override; 24 | virtual const TypeImpl &GetType() const override; 25 | 26 | // Methods needed for the alternative visitation method. 27 | const Expression &GetValue() const; 28 | const IdExpressionPair *GetBound() const; 29 | 30 | // Custom allocation method. 31 | static boost::shared_ptr Alloc( 32 | const Annotations &p_annotations, 33 | const std::vector &p_children, 34 | const Expression *p_value); 35 | 36 | private: 37 | // Private constructor for struct hack allocation. 38 | LetExpression(const Annotations &p_annotations, 39 | const std::vector &p_children, 40 | const Expression *p_value); 41 | 42 | // Custom deallocation method. 43 | static void DeleteAlloc(LetExpression *p_allocated); 44 | 45 | // Sub-expression that dictates the value of the let expression. 46 | const Expression *m_value; 47 | 48 | // Number of quantities (variables) bound by this let. 49 | unsigned int m_numBound; 50 | 51 | // Array of quantites, allocated via struct hack, bound by this let. 52 | IdExpressionPair m_bound[1]; 53 | }; 54 | } // namespace FreeForm2 55 | 56 | #endif 57 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/MatchSub.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_MATCH_SUB_H 9 | #define FREEFORM2_MATCH_SUB_H 10 | 11 | #include "Expression.h" 12 | #include "FreeForm2Assert.h" 13 | 14 | namespace FreeForm2 { 15 | // A match sub- expression is a base class representing match 16 | // operators (that is, repetition, concatenation, etc). 17 | class MatchSubExpression : public Expression { 18 | public: 19 | struct Info { 20 | Info(unsigned int p_minLength, unsigned int p_maxLength) 21 | : m_minLength(p_minLength), m_maxLength(p_maxLength) { 22 | FF2_ASSERT(m_minLength <= m_maxLength); 23 | FF2_ASSERT(m_minLength != c_indeterminate); 24 | } 25 | 26 | // Calculated minimum limit on the length of matches from this FSM. 27 | // Must be less than or equal to m_maxLength, and not c_indeterminate. 28 | unsigned int m_minLength; 29 | 30 | // Calculated maximum limit on the length of matches from this FSM. 31 | // Will be c_indeterminate if there's no easily calculable limit on the 32 | // length of match using this FSM. 33 | unsigned int m_maxLength; 34 | 35 | // Constant indicating that a pattern matches arbitrarily long input. 36 | static const unsigned int c_indeterminate = UINT_MAX; 37 | }; 38 | 39 | MatchSubExpression(const Annotations &p_annotations) 40 | : Expression(p_annotations) {} 41 | 42 | virtual ~MatchSubExpression() {} 43 | 44 | // Calculate information for this sub-expression. 45 | virtual Info GetInfo() const = 0; 46 | }; 47 | }; // namespace FreeForm2 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/MemberAccessExpression.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for license information. 4 | */ 5 | #pragma once 6 | 7 | #ifndef FREEFORM2_MEMBER_ACCESS_EXPRESSION_H 8 | #define FREEFORM2_MEMBER_ACCESS_EXPRESSION_H 9 | 10 | #include "Expression.h" 11 | #include "CompoundType.h" 12 | 13 | namespace FreeForm2 14 | { 15 | // An array-dereference expression removes a dimension from an array. 16 | class MemberAccessExpression : public Expression 17 | { 18 | public: 19 | MemberAccessExpression(const Annotations &p_annotations, 20 | const Expression &p_struct, 21 | const CompoundType::Member &p_memberInfo, 22 | size_t p_version); 23 | 24 | // Methods inherited from Expression. 25 | virtual size_t GetNumChildren() const override; 26 | virtual void Accept(Visitor &p_visitor) const override; 27 | virtual void AcceptReference(Visitor &p_visitor) const override; 28 | virtual const TypeImpl &GetType() const override; 29 | virtual bool IsConstant() const override; 30 | virtual ConstantValue GetConstantValue() const override; 31 | 32 | const Expression &GetStruct() const; 33 | const CompoundType::Member &GetMemberInfo() const; 34 | size_t GetVersion() const; 35 | 36 | private: 37 | // Struct expression whose member will be accessed. 38 | const Expression &m_struct; 39 | 40 | // Member information. 41 | const CompoundType::Member &m_memberInfo; 42 | 43 | // A unique version number associated with a particular 44 | // value for this variable. 45 | const size_t m_version; 46 | }; 47 | 48 | // An unresolved member to be accessed. 49 | class UnresolvedAccessExpression : public Expression 50 | { 51 | public: 52 | UnresolvedAccessExpression(const Annotations &p_annotations, 53 | const Expression &p_object, 54 | const std::string &p_memberName, 55 | const TypeImpl &p_expectedType); 56 | 57 | // Methods inherited from Expression. 58 | virtual size_t GetNumChildren() const override; 59 | virtual void Accept(Visitor &p_visitor) const override; 60 | virtual void AcceptReference(Visitor &p_visitor) const override; 61 | virtual const TypeImpl &GetType() const override; 62 | 63 | const Expression &GetObject() const; 64 | const std::string &GetMemberName() const; 65 | 66 | private: 67 | // Struct expression whose member will be accessed. 68 | const Expression &m_object; 69 | 70 | // Member information. 71 | std::string m_memberName; 72 | 73 | // The expected type of the member access expression. 74 | const TypeImpl &m_expectedType; 75 | }; 76 | }; 77 | 78 | #endif 79 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/PhiNode.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "PhiNode.h" 7 | 8 | #include "FreeForm2Assert.h" 9 | #include "Visitor.h" 10 | 11 | boost::shared_ptr 12 | FreeForm2::PhiNodeExpression::Alloc(const Annotations &p_annotations, 13 | size_t p_version, 14 | size_t p_incomingVersionsCount, 15 | const size_t *p_incomingVersions) { 16 | FF2_ASSERT(p_incomingVersionsCount > 0); 17 | 18 | size_t bytes = sizeof(PhiNodeExpression) + 19 | (p_incomingVersionsCount - 1) * sizeof(unsigned long long); 20 | 21 | // Allocate a shared_ptr that deletes an BlockExpression 22 | // allocated in a char[]. 23 | boost::shared_ptr exp( 24 | new (new char[bytes]) 25 | PhiNodeExpression(p_annotations, p_version, p_incomingVersionsCount, 26 | p_incomingVersions), 27 | DeleteAlloc); 28 | return exp; 29 | } 30 | 31 | const FreeForm2::TypeImpl &FreeForm2::PhiNodeExpression::GetType() const { 32 | return FreeForm2::TypeImpl::GetVoidInstance(); 33 | } 34 | 35 | size_t FreeForm2::PhiNodeExpression::GetNumChildren() const { return 0; } 36 | 37 | void FreeForm2::PhiNodeExpression::Accept(Visitor &p_visitor) const { 38 | size_t stackSize = p_visitor.StackSize(); 39 | 40 | if (!p_visitor.AlternativeVisit(*this)) { 41 | p_visitor.Visit(*this); 42 | } 43 | 44 | FF2_ASSERT(p_visitor.StackSize() == stackSize + p_visitor.StackIncrement()); 45 | } 46 | 47 | size_t FreeForm2::PhiNodeExpression::GetVersion() const { return m_version; } 48 | 49 | size_t FreeForm2::PhiNodeExpression::GetIncomingVersionsCount() const { 50 | return m_incomingVersionsCount; 51 | } 52 | 53 | const size_t *FreeForm2::PhiNodeExpression::GetIncomingVersions() const { 54 | return m_incomingVersions; 55 | } 56 | 57 | FreeForm2::PhiNodeExpression::PhiNodeExpression( 58 | const Annotations &p_annotations, size_t p_version, 59 | size_t p_incomingVersionsCount, const size_t *p_incomingVersions) 60 | : Expression(p_annotations), 61 | m_version(p_version), 62 | m_incomingVersionsCount(p_incomingVersionsCount) { 63 | // We rely on the custom allocator Alloc to provide enough space 64 | // for all of the incomings. 65 | for (unsigned int i = 0; i < m_incomingVersionsCount; i++) { 66 | m_incomingVersions[i] = p_incomingVersions[i]; 67 | } 68 | } 69 | 70 | void FreeForm2::PhiNodeExpression::DeleteAlloc(PhiNodeExpression *p_allocated) { 71 | // Manually call dtor for phi node expression. 72 | p_allocated->~PhiNodeExpression(); 73 | 74 | // Dispose of memory, which we allocated in a char[]. 75 | char *mem = reinterpret_cast(p_allocated); 76 | delete[] mem; 77 | } 78 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/PhiNode.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for license information. 4 | */ 5 | #pragma once 6 | 7 | #include "Expression.h" 8 | 9 | namespace FreeForm2 10 | { 11 | // The Phi node marks places in the code where the value of a variable can 12 | // change in different branches of the code. This node should be ignored 13 | // by the backends since it doesn't affect the compiled output. 14 | // 15 | // The incoming array refers to the list of variable versions that can reach 16 | // this point of the code. 17 | class PhiNodeExpression : public Expression 18 | { 19 | public: 20 | static boost::shared_ptr 21 | Alloc(const Annotations &p_annotations, 22 | size_t p_version, 23 | size_t p_incomingVersionsCount, 24 | const size_t *p_incomingVersions); 25 | 26 | // Methods inherited from Expression 27 | virtual void Accept(Visitor &) const override; 28 | virtual size_t GetNumChildren() const override; 29 | virtual const TypeImpl &GetType() const override; 30 | 31 | // Getter methods. 32 | size_t GetVersion() const; 33 | size_t GetIncomingVersionsCount() const; 34 | const size_t *GetIncomingVersions() const; 35 | 36 | private: 37 | // Create a PhiNode expression. 38 | PhiNodeExpression(const Annotations &p_annotations, 39 | size_t p_version, 40 | size_t p_incomingVersionsCount, 41 | const size_t *p_incomingVersions); 42 | 43 | static void DeleteAlloc(PhiNodeExpression *p_allocated); 44 | 45 | size_t m_version; 46 | 47 | size_t m_incomingVersionsCount; 48 | 49 | // Allocated using the struct hack. 50 | size_t m_incomingVersions[1]; 51 | }; 52 | } 53 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/RandExpression.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "RandExpression.h" 7 | 8 | #include "FreeForm2Assert.h" 9 | #include "Visitor.h" 10 | 11 | FreeForm2::RandFloatExpression::RandFloatExpression( 12 | const Annotations &p_annotations) 13 | : Expression(p_annotations) {} 14 | 15 | void FreeForm2::RandFloatExpression::Accept(Visitor &p_visitor) const { 16 | const size_t stackSize = p_visitor.StackSize(); 17 | 18 | if (!p_visitor.AlternativeVisit(*this)) { 19 | p_visitor.Visit(*this); 20 | } 21 | 22 | FF2_ASSERT(p_visitor.StackSize() == stackSize + p_visitor.StackIncrement()); 23 | } 24 | 25 | const FreeForm2::TypeImpl &FreeForm2::RandFloatExpression::GetType() const { 26 | return TypeImpl::GetFloatInstance(true); 27 | } 28 | 29 | size_t FreeForm2::RandFloatExpression::GetNumChildren() const { return 0; } 30 | 31 | const FreeForm2::RandFloatExpression & 32 | FreeForm2::RandFloatExpression::GetInstance() { 33 | static const Annotations s_annotations; 34 | static const RandFloatExpression s_instance(s_annotations); 35 | return s_instance; 36 | } 37 | 38 | FreeForm2::RandIntExpression::RandIntExpression( 39 | const Annotations &p_annotations, const Expression &p_lowerBoundExpression, 40 | const Expression &p_upperBoundExpression) 41 | : Expression(p_annotations), 42 | m_lowerBoundExpression(p_lowerBoundExpression), 43 | m_upperBoundExpression(p_upperBoundExpression) {} 44 | 45 | void FreeForm2::RandIntExpression::Accept(Visitor &p_visitor) const { 46 | const size_t stackSize = p_visitor.StackSize(); 47 | 48 | if (!p_visitor.AlternativeVisit(*this)) { 49 | m_lowerBoundExpression.Accept(p_visitor); 50 | m_upperBoundExpression.Accept(p_visitor); 51 | p_visitor.Visit(*this); 52 | } 53 | 54 | FF2_ASSERT(p_visitor.StackSize() == stackSize + p_visitor.StackIncrement()); 55 | } 56 | 57 | const FreeForm2::TypeImpl &FreeForm2::RandIntExpression::GetType() const { 58 | return TypeImpl::GetIntInstance(true); 59 | } 60 | 61 | size_t FreeForm2::RandIntExpression::GetNumChildren() const { return 2; } 62 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/RandExpression.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for license information. 4 | */ 5 | #pragma once 6 | 7 | #ifndef FREEFORM2_RAND_EXPRESSION_H 8 | #define FREEFORM2_RAND_EXPRESSION_H 9 | 10 | #include "Expression.h" 11 | 12 | namespace FreeForm2 13 | { 14 | // The RandFloatExpression generates a random float 15 | // in the range of 0 to 1 inclusive. 16 | class RandFloatExpression : public Expression 17 | { 18 | public: 19 | // Methods inherited from Expression. 20 | void Accept(Visitor &p_visitor) const override; 21 | virtual const TypeImpl &GetType() const override; 22 | virtual size_t GetNumChildren() const override; 23 | 24 | // Get a reference to the singleton for RandFloatExpression. 25 | static const RandFloatExpression &GetInstance(); 26 | 27 | private: 28 | // Private constructor for singleton class. 29 | RandFloatExpression(const Annotations &p_annotations); 30 | }; 31 | 32 | // The RandIntExpression generates a random integer 33 | // in the range specified inclusive of the lower bound 34 | // and upper bound exclusive. 35 | class RandIntExpression : public Expression 36 | { 37 | public: 38 | // Constructor for RandIntExpression. 39 | RandIntExpression(const Annotations &p_annotations, 40 | const Expression &p_lowerBoundExpression, 41 | const Expression &p_upperBoundExpression); 42 | 43 | // Methods inherited from Expression. 44 | void Accept(Visitor &p_visitor) const override; 45 | virtual const TypeImpl &GetType() const override; 46 | virtual size_t GetNumChildren() const override; 47 | 48 | private: 49 | // Lower bound expression to generate a random number. 50 | const Expression &m_lowerBoundExpression; 51 | 52 | // Upper bound expression to generate a random number. 53 | const Expression &m_upperBoundExpression; 54 | }; 55 | }; 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/RefExpression.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_REFEXPRESSION_H 9 | #define FREEFORM2_REFEXPRESSION_H 10 | 11 | #include "Expression.h" 12 | 13 | namespace FreeForm2 { 14 | // Class representing a reference to a feature. 15 | class FeatureRefExpression : public Expression { 16 | public: 17 | FeatureRefExpression(const Annotations &p_annotations, UInt32 p_index); 18 | 19 | virtual const TypeImpl &GetType() const override; 20 | 21 | virtual size_t GetNumChildren() const override; 22 | 23 | virtual void Accept(Visitor &p_visitor) const override; 24 | 25 | UInt32 m_index; 26 | }; 27 | 28 | // Class representing a reference to a stack location. We keep track of only 29 | // a stack slot, so that the value of the expression can be generated 30 | // during compilation, rather than parsing, and looked up using the stack 31 | // slot as identifier. 32 | class VariableRefExpression : public Expression { 33 | public: 34 | // Construct a stack expression from a stack slot. 35 | VariableRefExpression(const Annotations &p_annotations, VariableID p_id, 36 | size_t p_version, const TypeImpl &p_type); 37 | virtual ~VariableRefExpression(); 38 | 39 | // Methods inherited from Expression (note that VariableRefExpression 40 | // can generate references). 41 | virtual size_t GetNumChildren() const override; 42 | virtual void Accept(Visitor &p_visitor) const override; 43 | virtual void AcceptReference(Visitor &) const override; 44 | virtual const TypeImpl &GetType() const override; 45 | 46 | VariableID GetId() const; 47 | size_t GetVersion() const; 48 | 49 | private: 50 | // ID assigned to the value. 51 | VariableID m_id; 52 | 53 | // A unique version number associated with a particular 54 | // value for this variable. 55 | const size_t m_version; 56 | 57 | // Type of the value. 58 | const TypeImpl &m_type; 59 | }; 60 | 61 | // This class represents an expression which refers to the not-yet- 62 | // instantiated object for the scope being compiled (for instance, a state 63 | // machine will use a ThisExpression to refer to a member inside the state 64 | // machine, as the machine is not instantiated at this point). 65 | class ThisExpression : public Expression { 66 | public: 67 | // Construct a ThisExpression from a compound type. 68 | ThisExpression(const Annotations &p_annotations, const TypeImpl &p_type); 69 | 70 | // Methods inherited from Visitor. 71 | virtual size_t GetNumChildren() const override; 72 | virtual void Accept(Visitor &p_visitor) const override; 73 | virtual void AcceptReference(Visitor &) const override; 74 | virtual const TypeImpl &GetType() const override; 75 | 76 | private: 77 | // The type of the object. 78 | const TypeImpl &m_type; 79 | }; 80 | } // namespace FreeForm2 81 | 82 | #endif 83 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/SimpleExpressionOwner.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_SIMPLEEXPRESSIONOWNER 9 | #define FREEFORM2_SIMPLEEXPRESSIONOWNER 10 | 11 | #include 12 | 13 | #include "Expression.h" 14 | 15 | namespace FreeForm2 { 16 | // Straight-forward expression owner, that keeps shared pointers 17 | // to given expressions. 18 | class SimpleExpressionOwner : public ExpressionOwner { 19 | public: 20 | // Transfer ownership of the given expression to the expression owner. 21 | const Expression *AddExpression( 22 | const boost::shared_ptr &p_expr) { 23 | m_exp.push_back(p_expr); 24 | return m_exp.back().get(); 25 | } 26 | 27 | private: 28 | // Vector of managed expressions. 29 | std::vector > m_exp; 30 | }; 31 | } // namespace FreeForm2 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/StreamData.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "StreamData.h" 7 | 8 | #include "FreeForm2Assert.h" 9 | #include "Visitor.h" 10 | 11 | FreeForm2::StreamDataExpression::StreamDataExpression( 12 | const FreeForm2::Annotations &p_annotations, bool p_requestsLength) 13 | : Expression(p_annotations), m_requestsLength(p_requestsLength) {} 14 | 15 | void FreeForm2::StreamDataExpression::Accept(Visitor &p_visitor) const { 16 | size_t stackSize = p_visitor.StackSize(); 17 | 18 | if (!p_visitor.AlternativeVisit(*this)) { 19 | p_visitor.Visit(*this); 20 | } 21 | 22 | FF2_ASSERT(p_visitor.StackSize() == stackSize + p_visitor.StackIncrement()); 23 | } 24 | 25 | const FreeForm2::TypeImpl &FreeForm2::StreamDataExpression::GetType() const { 26 | return TypeImpl::GetIntInstance(true); 27 | } 28 | 29 | size_t FreeForm2::StreamDataExpression::GetNumChildren() const { return 0; } 30 | 31 | const FreeForm2::UpdateStreamDataExpression & 32 | FreeForm2::UpdateStreamDataExpression::GetInstance() { 33 | static const Annotations emptyAnnotations; 34 | static const UpdateStreamDataExpression exp(emptyAnnotations); 35 | return exp; 36 | } 37 | 38 | void FreeForm2::UpdateStreamDataExpression::Accept(Visitor &p_visitor) const { 39 | size_t stackSize = p_visitor.StackSize(); 40 | 41 | if (!p_visitor.AlternativeVisit(*this)) { 42 | p_visitor.Visit(*this); 43 | } 44 | 45 | FF2_ASSERT(p_visitor.StackSize() == stackSize + p_visitor.StackIncrement()); 46 | } 47 | 48 | const FreeForm2::TypeImpl &FreeForm2::UpdateStreamDataExpression::GetType() 49 | const { 50 | return TypeImpl::GetVoidInstance(); 51 | } 52 | 53 | size_t FreeForm2::UpdateStreamDataExpression::GetNumChildren() const { 54 | return 0; 55 | } 56 | 57 | FreeForm2::UpdateStreamDataExpression::UpdateStreamDataExpression( 58 | const FreeForm2::Annotations &p_annotations) 59 | : Expression(p_annotations) {} 60 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/StreamData.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_STREAM_DATA_H 9 | #define FREEFORM2_STREAM_DATA_H 10 | 11 | #include "Expression.h" 12 | 13 | namespace FreeForm2 { 14 | // StreamDataExpression pulls either the stream count (opaque integer data 15 | // field) or the length (calculated length of the stream instance) from 16 | // currently matched stream instance. These have traditionally been called 17 | // PhraseCount/Length, or 'click phrase' data. However, i've avoided those 18 | // names because they're inaccurate and incomplete (as metastreams are a 19 | // broader abstraction than click phrases, and the data in the metastream 20 | // instance has no connection to phrases outside of click metastreams). 21 | class StreamDataExpression : public Expression { 22 | public: 23 | StreamDataExpression(const Annotations &p_annotations, bool p_requestsLength); 24 | 25 | // Virtual methods inherited from Expression. 26 | virtual void Accept(Visitor &p_visitor) const override; 27 | virtual const TypeImpl &GetType() const override; 28 | virtual size_t GetNumChildren() const override; 29 | 30 | // Whether this expression requests the stream length, with the 31 | // alternative being the stream count. 32 | bool m_requestsLength; 33 | }; 34 | 35 | // Singleton expression class to issue side-effect code to update the 36 | // stream data (length, count) during matching. This requires an 37 | // expression to leverage the FSM architecture to decide when this 38 | // must occur. If we accumulate other similar expressions, they 39 | // can be aggregated into a SystemEffectExpression, or similar. 40 | class UpdateStreamDataExpression : public Expression { 41 | public: 42 | static const UpdateStreamDataExpression &GetInstance(); 43 | 44 | // Virtual methods inherited from Expression. 45 | virtual void Accept(Visitor &p_visitor) const override; 46 | virtual const TypeImpl &GetType() const override; 47 | virtual size_t GetNumChildren() const override; 48 | 49 | private: 50 | UpdateStreamDataExpression(const Annotations &p_annotations); 51 | }; 52 | }; // namespace FreeForm2 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/TypeUtil.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_TYPE_UTIL_H 9 | #define FREEFORM2_TYPE_UTIL_H 10 | 11 | #include 12 | 13 | #include "Expression.h" 14 | #include "TypeImpl.h" 15 | 16 | namespace FreeForm2 { 17 | class ArrayType; 18 | class ConversionExpression; 19 | class TypeManager; 20 | 21 | class TypeUtil { 22 | public: 23 | // Check if one type is convertible to another type. Returns true if 24 | // a ConversionExpression exists; false otherwise. 25 | static bool IsConvertible(const TypeImpl &p_from, const TypeImpl &p_to); 26 | 27 | // Create a new Expression of the specified type for a child 28 | // expression. 29 | static boost::shared_ptr Convert(const Expression &p_expr, 30 | Type::TypePrimitive p_type); 31 | 32 | // Create a new literal Expression of the specified type for a constant 33 | // value. 34 | static boost::shared_ptr ConvertConstant( 35 | const Annotations &p_annotations, ConstantValue p_value, 36 | Type::TypePrimitive p_from, Type::TypePrimitive p_to); 37 | 38 | // Return a type that's compatible with two types, or return 39 | // Type::Invalid. If the types differ in const-ness, the resulting 40 | // type will be constant. p_allowArray specifies whether we are 41 | // allowed to unify array types. 42 | static const TypeImpl &Unify(const TypeImpl &p_type1, const TypeImpl &p_type2, 43 | TypeManager &p_typeManager, bool p_allowArray, 44 | bool p_allowPromotion); 45 | 46 | // Determine if a source type is assignable to a destination type. 47 | static bool IsAssignable(const TypeImpl &p_dest, const TypeImpl &p_source); 48 | 49 | // Return a type which matches the type argument, but with const-ness 50 | // specified by a flag. 51 | static const TypeImpl &SetConstness(const TypeImpl &p_type, bool p_isConst); 52 | }; 53 | } // namespace FreeForm2 54 | 55 | #endif 56 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Expression/UnaryOperator.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_UNARY_OPERATOR_H 9 | #define FREEFORM2_UNARY_OPERATOR_H 10 | 11 | namespace FreeForm2 { 12 | class TypeImpl; 13 | 14 | class UnaryOperator { 15 | public: 16 | enum Operation { 17 | minus, 18 | log, 19 | log1, 20 | abs, 21 | round, 22 | trunc, 23 | _not, 24 | bitnot, 25 | tanh, 26 | 27 | invalid 28 | }; 29 | 30 | // Select the best operand type for an operator. Best is defined in 31 | // terms of TypeUtil::SelectBestType. If no valid type is found, an 32 | // invalid TypeImpl is returned. 33 | static const TypeImpl &GetBestOperandType(Operation p_operator, 34 | const TypeImpl &p_operand); 35 | 36 | // Return the type of a unary operator result given an operator and 37 | // an operand type. If the operand type is not a valid operand type for 38 | // the operator, the return type is undefined. 39 | static const TypeImpl &GetReturnType(Operation p_operator, 40 | const TypeImpl &p_operand); 41 | }; 42 | } // namespace FreeForm2 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/External/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.15) 2 | 3 | set(PROJECT_NAME DRFreeFormLibrary) 4 | 5 | project(${PROJECT_NAME}) 6 | 7 | set(CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS} -fpermissive") 8 | 9 | add_library(${PROJECT_NAME} STATIC 10 | ${CMAKE_CURRENT_SOURCE_DIR}/Compiler.cpp 11 | ${CMAKE_CURRENT_SOURCE_DIR}/Executable.cpp 12 | ${CMAKE_CURRENT_SOURCE_DIR}/FreeForm2ExternalData.cpp 13 | ${CMAKE_CURRENT_SOURCE_DIR}/FreeForm2Result.cpp 14 | ${CMAKE_CURRENT_SOURCE_DIR}/FreeForm2Type.cpp 15 | ${CMAKE_CURRENT_SOURCE_DIR}/NeuralInputFreeForm2.cpp 16 | ${CMAKE_CURRENT_SOURCE_DIR}/Program.cpp 17 | ${CMAKE_CURRENT_SOURCE_DIR}/ValueResult.cpp 18 | ) 19 | 20 | target_include_directories(${PROJECT_NAME} PRIVATE 21 | ${CMAKE_CURRENT_SOURCE_DIR} 22 | ${CMAKE_CURRENT_SOURCE_DIR}/../../inc 23 | ${CMAKE_CURRENT_SOURCE_DIR}/../../../NeuralTree.Library/inc 24 | ${CMAKE_CURRENT_SOURCE_DIR}/../Shared 25 | ${CMAKE_CURRENT_SOURCE_DIR}/../Expression 26 | ${CMAKE_CURRENT_SOURCE_DIR}/../Backend/llvm 27 | ${CMAKE_CURRENT_SOURCE_DIR}/../Transform 28 | ${CMAKE_CURRENT_SOURCE_DIR}/../Parse/SExpression/inc 29 | ) 30 | 31 | install(TARGETS ${PROJECT_NAME} 32 | LIBRARY DESTINATION lib 33 | ARCHIVE DESTINATION lib 34 | ) -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/External/Compiler.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "Compiler.h" 7 | 8 | #include 9 | 10 | #include "FreeForm2Compiler.h" 11 | #include "FreeForm2Program.h" 12 | 13 | FreeForm2::CompilerImpl::~CompilerImpl() {} 14 | 15 | FreeForm2::CompilerResults::~CompilerResults() {} 16 | 17 | FreeForm2::Compiler::Compiler(std::auto_ptr p_impl) 18 | : m_impl(p_impl.release()) {} 19 | 20 | FreeForm2::Compiler::~Compiler() {} 21 | 22 | std::unique_ptr FreeForm2::Compiler::Compile( 23 | const Program &p_program, bool p_debugOutput) { 24 | return m_impl->Compile(p_program.GetImplementation(), p_debugOutput); 25 | } 26 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/External/Compiler.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for license information. 4 | */ 5 | #pragma once 6 | 7 | #ifndef FREEFORM2_COMPILER 8 | #define FREEFORM2_COMPILER 9 | 10 | #include 11 | #include 12 | 13 | namespace FreeForm2 14 | { 15 | class CompilerResults; 16 | class ProgramImpl; 17 | 18 | class CompilerImpl : boost::noncopyable 19 | { 20 | public: 21 | virtual ~CompilerImpl(); 22 | 23 | // Compile the given program. 24 | virtual std::unique_ptr Compile(const ProgramImpl &p_program, 25 | bool p_debugOutput) = 0; 26 | }; 27 | } 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/External/Executable.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "Executable.h" 7 | 8 | #include 9 | 10 | #include "Compiler.h" 11 | #include "FreeForm2Assert.h" 12 | #include "FreeForm2Compiler.h" 13 | #include "FreeForm2CompilerFactory.h" 14 | #include "FreeForm2Executable.h" 15 | #include "FreeForm2Result.h" 16 | #include "LlvmCompiler.h" 17 | 18 | FreeForm2::Executable::Executable(std::auto_ptr p_impl) 19 | : m_impl(p_impl.release()) {} 20 | 21 | boost::shared_ptr FreeForm2::Executable::Evaluate( 22 | StreamFeatureInput *p_input, const FeatureType p_features[]) const { 23 | return m_impl->Evaluate(p_input, p_features); 24 | } 25 | 26 | boost::shared_ptr FreeForm2::Executable::Evaluate( 27 | const Executable::FeatureType *const *p_features, UInt32 p_currentDocument, 28 | UInt32 p_documentCount, Int64 *p_cache) const { 29 | return m_impl->Evaluate(p_features, p_currentDocument, p_documentCount, 30 | p_cache); 31 | } 32 | 33 | FreeForm2::Executable::DirectEvalFun FreeForm2::Executable::EvaluationFunction() 34 | const { 35 | return m_impl->EvaluationFunction(); 36 | } 37 | 38 | FreeForm2::Executable::AggregatedEvalFun 39 | FreeForm2::Executable::AggregatedEvaluationFunction() const { 40 | return m_impl->AggregatedEvaluationFunction(); 41 | } 42 | 43 | const FreeForm2::Type &FreeForm2::Executable::GetType() const { 44 | return m_impl->GetType(); 45 | } 46 | 47 | size_t FreeForm2::Executable::GetExternalSize() const { 48 | return sizeof(FreeForm2::ExecutableImpl) + m_impl->GetExternalSize(); 49 | } 50 | 51 | const FreeForm2::ExecutableImpl &FreeForm2::Executable::GetImplementation() 52 | const { 53 | return *m_impl; 54 | } 55 | 56 | FreeForm2::ExecutableCompilerResults::ExecutableCompilerResults( 57 | const boost::shared_ptr &p_executable) 58 | : m_executable(p_executable) {} 59 | 60 | const boost::shared_ptr 61 | &FreeForm2::ExecutableCompilerResults::GetExecutable() const { 62 | return m_executable; 63 | } 64 | 65 | FreeForm2::ExecutableImpl::~ExecutableImpl() {} 66 | 67 | std::unique_ptr 68 | FreeForm2::CompilerFactory::CreateExecutableCompiler( 69 | unsigned int p_optimizationLevel, 70 | FreeForm2::CompilerFactory::DestinationFunctionType 71 | p_destinationFunctionType) { 72 | std::auto_ptr impl; 73 | impl.reset( 74 | new LlvmCompilerImpl(p_optimizationLevel, p_destinationFunctionType)); 75 | return std::unique_ptr(new Compiler(impl)); 76 | } 77 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/External/Executable.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_EXECUTABLE_H 9 | #define FREEFORM2_EXECUTABLE_H 10 | 11 | #include 12 | #include 13 | 14 | #include "FreeForm2CompilerFactory.h" 15 | #include "FreeForm2Executable.h" 16 | 17 | class StreamFeatureInput; 18 | 19 | namespace FreeForm2 { 20 | class Result; 21 | class Type; 22 | 23 | // An ExectuableImpl is the implementation class for exectuables, which 24 | // currently compiles and runs via one of the backends. 25 | class ExecutableImpl : boost::noncopyable { 26 | public: 27 | virtual ~ExecutableImpl(); 28 | 29 | virtual boost::shared_ptr Evaluate( 30 | StreamFeatureInput *p_input, 31 | const Executable::FeatureType p_features[]) const = 0; 32 | 33 | // List based evaluation. 34 | virtual boost::shared_ptr Evaluate( 35 | const Executable::FeatureType *const *p_features, 36 | UInt32 p_currentDocument, UInt32 p_documentCount, 37 | Int64 *p_cache) const = 0; 38 | 39 | virtual Executable::DirectEvalFun EvaluationFunction() const = 0; 40 | 41 | // Get list based evaluation function. 42 | virtual Executable::AggregatedEvalFun AggregatedEvaluationFunction() 43 | const = 0; 44 | 45 | virtual const Type &GetType() const = 0; 46 | 47 | // Get the size of external memory. 48 | virtual size_t GetExternalSize() const = 0; 49 | }; 50 | } // namespace FreeForm2 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/External/FreeForm2ExternalData.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "FreeForm2ExternalData.h" 7 | 8 | #include 9 | 10 | #include "FreeForm2Type.h" 11 | #include "TypeImpl.h" 12 | #include "TypeManager.h" 13 | 14 | FreeForm2::ExternalData::ExternalData(const std::string &p_name, 15 | const FreeForm2::TypeImpl &p_typeImpl) 16 | : m_name(p_name), m_type(&p_typeImpl), m_isCompileTimeConst(false) {} 17 | 18 | FreeForm2::ExternalData::ExternalData(const std::string &p_name, 19 | const TypeImpl &p_typeImpl, 20 | ConstantValue p_value) 21 | : m_name(p_name), 22 | m_type(&p_typeImpl), 23 | m_isCompileTimeConst(true), 24 | m_constantValue(p_value) {} 25 | 26 | FreeForm2::ExternalData::~ExternalData() {} 27 | 28 | const std::string &FreeForm2::ExternalData::GetName() const { return m_name; } 29 | 30 | const FreeForm2::TypeImpl &FreeForm2::ExternalData::GetType() const { 31 | return *m_type; 32 | } 33 | 34 | bool FreeForm2::ExternalData::IsCompileTimeConstant() const { 35 | return m_isCompileTimeConst; 36 | } 37 | 38 | FreeForm2::ConstantValue FreeForm2::ExternalData::GetCompileTimeValue() const { 39 | return m_constantValue; 40 | } 41 | 42 | FreeForm2::ExternalDataManager::ExternalDataManager() 43 | : m_typeFactory(new TypeFactory(TypeManager::CreateTypeManager())) {} 44 | 45 | FreeForm2::ExternalDataManager::~ExternalDataManager() {} 46 | 47 | FreeForm2::TypeFactory &FreeForm2::ExternalDataManager::GetTypeFactory() { 48 | return *m_typeFactory; 49 | } 50 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/External/Program.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_PROGRAM_H 9 | #define FREEFORM2_PROGRAM_H 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #include "AllocationVisitor.h" 16 | #include "FreeForm2Type.h" 17 | 18 | namespace DynamicRank { 19 | class IFeatureMap; 20 | class INeuralNetFeatures; 21 | } // namespace DynamicRank 22 | 23 | namespace FreeForm2 { 24 | class Allocation; 25 | class Expression; 26 | class ExpressionOwner; 27 | class TypeManager; 28 | 29 | // A ProgramImpl is essentially just the concrete instantiation of the 30 | // Program class, into which we parse expressions. 31 | class ProgramImpl : boost::noncopyable { 32 | public: 33 | ProgramImpl(const Expression &p_exp, 34 | boost::shared_ptr p_owner, 35 | boost::shared_ptr p_typeManager, 36 | DynamicRank::IFeatureMap &p_map); 37 | 38 | const Type &GetType() const; 39 | 40 | void ProcessFeaturesUsed(DynamicRank::INeuralNetFeatures &p_features) const; 41 | 42 | const Expression &GetExpression() const; 43 | 44 | DynamicRank::IFeatureMap &GetFeatureMap() const; 45 | 46 | const std::vector > &GetAllocations() const; 47 | 48 | private: 49 | // Top-level type implementation of this program. 50 | const TypeImpl &m_typeImpl; 51 | 52 | // Top-level type of this program. 53 | Type m_type; 54 | 55 | // Pointer to root of parsed expression tree. 56 | const Expression *m_exp; 57 | 58 | // Expression owner. 59 | boost::shared_ptr m_owner; 60 | 61 | // Type manager. 62 | boost::shared_ptr m_typeManager; 63 | 64 | // Feature map used to compile expression, used to print program 65 | // information if exceptions occur. 66 | DynamicRank::IFeatureMap &m_map; 67 | 68 | // A visitor to extract all the allocations from the program. 69 | const AllocationVisitor m_allocationVisitor; 70 | }; 71 | } // namespace FreeForm2 72 | 73 | #endif -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/External/ResultIteratorImpl.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for license information. 4 | */ 5 | #pragma once 6 | 7 | #ifndef FREEFORM2_RESULTITERATORIMPL_H 8 | #define FREEFORM2_RESULTITERATORIMPL_H 9 | 10 | #include "FreeForm2Result.h" 11 | 12 | namespace FreeForm2 13 | { 14 | class ResultIteratorImpl 15 | { 16 | public: 17 | virtual ~ResultIteratorImpl() 18 | { 19 | } 20 | 21 | // Delegated iterator_facade function to increment the iterator. 22 | virtual void increment() = 0; 23 | 24 | // Delegated iterator_facade function to decrement the iterator. 25 | virtual void decrement() = 0; 26 | 27 | // Delegated iterator_facade function to get the current element. 28 | virtual const Result &dereference() const = 0; 29 | 30 | // Delegated iterator_facade function to get the current element. 31 | virtual void advance(std::ptrdiff_t p_distance) = 0; 32 | 33 | // Virtual copy constructor. 34 | virtual std::auto_ptr Clone() const = 0; 35 | 36 | // Having an abstract iterator puts us in a tricky position, because 37 | // some of the iterator methods (like equal) accept another iterator 38 | // as arg. Since we may have any number of subclasses, equal must 39 | // be able to compare iterators that have nothing to do with each 40 | // other. As such, we use a couple of (somewhat hacky) methods 41 | // below to return enough information from each iterator to compare 42 | // and calculate the difference between them without further 43 | // knowledge. 44 | 45 | // Returns a pointer indicating current position, plus the element index. 46 | virtual std::pair Position() const = 0; 47 | 48 | // Returns number of bytes per element. 49 | virtual unsigned int ElementSize() const = 0; 50 | }; 51 | } 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Parse/SExpression/inc/SExpressionParse.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_SEXPRESSIONPARSE_H 9 | #define FREEFORM2_SEXPRESSIONPARSE_H 10 | 11 | #include 12 | #include 13 | 14 | #include "FreeForm2.h" 15 | #include "FreeForm2Tokenizer.h" 16 | 17 | namespace FreeForm2 { 18 | class ProgramParseState; 19 | class Expression; 20 | class ExpressionOwner; 21 | class ArrayLiteralExpression; 22 | class TypeManager; 23 | 24 | class SExpressionParse { 25 | public: 26 | // Main driver function for parsing. p_expressionLimit can be used to limit 27 | // the number of expressions parsed into the current expression, with zero 28 | // reserved to indicate no limit. 29 | static Token ParseTokens(ProgramParseState &p_state, 30 | unsigned int p_expressionLimit); 31 | 32 | typedef boost::tuples::tuple, 34 | boost::shared_ptr > 35 | ParserResults; 36 | 37 | // Parse an expression from a string. 38 | static ParserResults Parse(SIZED_STRING p_input, 39 | DynamicRank::IFeatureMap &p_map, 40 | bool p_mustProduceFloat, 41 | bool p_parsingAggregatedExpression); 42 | 43 | // Parse an array dereference expression. 44 | static Token ParseArrayDereference(ProgramParseState &p_state); 45 | 46 | // Function to parse a let expression. 47 | static Token ParseLet(ProgramParseState &p_state); 48 | 49 | // Function to parse a macro-let expression. 50 | static Token ParseMacroLet(ProgramParseState &p_state); 51 | 52 | // Parse a RangeReduceExpression. 53 | static Token ParseRangeReduce(ProgramParseState &p_state); 54 | 55 | // Parse an ArrayLiteralExpression. 56 | static Token ParseArrayLiteral(ProgramParseState &p_state); 57 | 58 | // Parse a lambda expression. 59 | static Token ParseLambda(ProgramParseState &p_state); 60 | 61 | // Parse an invoke expression. 62 | static Token ParseInvoke(ProgramParseState &p_state); 63 | 64 | private: 65 | // Recursively parse array literal expressions. 66 | static const ArrayLiteralExpression &ParseArrayLiteralRecurse( 67 | ProgramParseState &p_state); 68 | }; 69 | } // namespace FreeForm2 70 | 71 | #endif 72 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Parse/SExpression/libs/Arithmetic.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_ARITHMETIC_H 9 | #define FREEFORM2_ARITHMETIC_H 10 | 11 | namespace FreeForm2 { 12 | class ExpressionFactory; 13 | 14 | namespace Arithmetic { 15 | const ExpressionFactory &GetPlusInstance(); 16 | const ExpressionFactory &GetMinusInstance(); 17 | const ExpressionFactory &GetMultiplyInstance(); 18 | const ExpressionFactory &GetDividesInstance(); 19 | const ExpressionFactory &GetIntegerDivInstance(); 20 | const ExpressionFactory &GetIntegerModInstance(); 21 | const ExpressionFactory &GetModInstance(); 22 | const ExpressionFactory &GetMaxInstance(); 23 | const ExpressionFactory &GetMinInstance(); 24 | const ExpressionFactory &GetPowInstance(); 25 | const ExpressionFactory &GetUnaryLogInstance(); 26 | const ExpressionFactory &GetBinaryLogInstance(); 27 | const ExpressionFactory &GetLog1Instance(); 28 | const ExpressionFactory &GetAbsInstance(); 29 | const ExpressionFactory &GetRoundInstance(); 30 | const ExpressionFactory &GetTruncInstance(); 31 | } // namespace Arithmetic 32 | }; // namespace FreeForm2 33 | 34 | #endif 35 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Parse/SExpression/libs/Bitwise.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "Bitwise.h" 7 | 8 | #include "BinaryOperator.h" 9 | #include "OperatorExpressionFactory.h" 10 | #include "UnaryOperator.h" 11 | 12 | using namespace FreeForm2; 13 | 14 | namespace { 15 | typedef OperatorExpressionFactory OperatorExpression; 16 | static const OperatorExpression c_and(UnaryOperator::invalid, 17 | BinaryOperator::_bitand, false); 18 | static const OperatorExpression c_or(UnaryOperator::invalid, 19 | BinaryOperator::_bitor, false); 20 | static const OperatorExpression c_not(UnaryOperator::bitnot, 21 | BinaryOperator::invalid, false); 22 | } // namespace 23 | 24 | const FreeForm2::ExpressionFactory& FreeForm2::Bitwise::GetAndInstance() { 25 | return c_and; 26 | } 27 | 28 | const FreeForm2::ExpressionFactory& FreeForm2::Bitwise::GetOrInstance() { 29 | return c_or; 30 | } 31 | 32 | const FreeForm2::ExpressionFactory& FreeForm2::Bitwise::GetNotInstance() { 33 | return c_not; 34 | } 35 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Parse/SExpression/libs/Bitwise.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_BITWISE_H 9 | #define FREEFORM2_BITWISE_H 10 | 11 | namespace FreeForm2 { 12 | class ExpressionFactory; 13 | 14 | namespace Bitwise { 15 | const ExpressionFactory &GetAndInstance(); 16 | const ExpressionFactory &GetOrInstance(); 17 | const ExpressionFactory &GetNotInstance(); 18 | } // namespace Bitwise 19 | }; // namespace FreeForm2 20 | 21 | #endif 22 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Parse/SExpression/libs/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.15) 2 | 3 | set(PROJECT_NAME DRFreeFormSExpressionLibrary) 4 | 5 | 6 | Project(${PROJECT_NAME}) 7 | 8 | SET(CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS} -fpermissive") 9 | 10 | 11 | 12 | add_library(${PROJECT_NAME} STATIC 13 | ${CMAKE_CURRENT_SOURCE_DIR}/Arithmetic.cpp 14 | ${CMAKE_CURRENT_SOURCE_DIR}/Bitwise.cpp 15 | ${CMAKE_CURRENT_SOURCE_DIR}/ExpressionFactory.cpp 16 | ${CMAKE_CURRENT_SOURCE_DIR}/FreeForm2Tokenizer.cpp 17 | ${CMAKE_CURRENT_SOURCE_DIR}/Logic.cpp 18 | ${CMAKE_CURRENT_SOURCE_DIR}/MiscFactory.cpp 19 | ${CMAKE_CURRENT_SOURCE_DIR}/ProgramParseState.cpp 20 | ${CMAKE_CURRENT_SOURCE_DIR}/SExpressionParse.cpp 21 | ) 22 | 23 | target_include_directories(${PROJECT_NAME} PRIVATE 24 | ${CMAKE_CURRENT_SOURCE_DIR} 25 | ${CMAKE_CURRENT_SOURCE_DIR}/../inc 26 | ${CMAKE_CURRENT_SOURCE_DIR}/../../../../inc 27 | ${CMAKE_CURRENT_SOURCE_DIR}/../../../../../NeuralTree.Library/inc 28 | ${CMAKE_CURRENT_SOURCE_DIR}/../../../Transform 29 | ${CMAKE_CURRENT_SOURCE_DIR}/../../../Shared 30 | ${CMAKE_CURRENT_SOURCE_DIR}/../../../Expression 31 | ) 32 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Parse/SExpression/libs/ExpressionFactory.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for license information. 4 | */ 5 | #include "ExpressionFactory.h" 6 | 7 | #include 8 | 9 | const FreeForm2::Expression & 10 | FreeForm2::ExpressionFactory::Create(const ProgramParseState::ExpressionParseState &p_state, 11 | SimpleExpressionOwner &p_owner, 12 | TypeManager &p_typeManager) const 13 | { 14 | std::pair arity = Arity(); 15 | if (p_state.m_children.size() >= arity.first && p_state.m_children.size() <= arity.second) 16 | { 17 | return CreateExpression(p_state, p_owner, p_typeManager); 18 | } 19 | else 20 | { 21 | // Incorrect arity, throw exception. 22 | std::ostringstream err; 23 | err << "Arity of " << std::string(SIZED_STR(p_state.m_atom)) << " was " 24 | << p_state.m_children.size() << " but was expected to be in range [" 25 | << Arity().first << ", " 26 | << Arity().second << "]"; 27 | throw std::runtime_error(err.str()); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Parse/SExpression/libs/ExpressionFactory.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_EXPRESSIONFACTORY_H 9 | #define FREEFORM2_EXPRESSIONFACTORY_H 10 | 11 | #include 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include "ProgramParseState.h" 18 | #include "TypeImpl.h" 19 | 20 | namespace FreeForm2 { 21 | class Expression; 22 | class SimpleExpressionOwner; 23 | class TypeManager; 24 | 25 | // Base class that assists parsing by creating an expression from a given 26 | // set of children. 27 | class ExpressionFactory : boost::noncopyable { 28 | public: 29 | typedef boost::shared_ptr Ptr; 30 | 31 | typedef std::vector ChildVec; 32 | 33 | // Creates an expression from the given children, with the returned 34 | // expression being owned by the given owner. p_atom specifies the atom 35 | // with which this expression factory was identified during parsing, 36 | // which allows us to provide decent error messages. 37 | const Expression &Create( 38 | const ProgramParseState::ExpressionParseState &p_state, 39 | SimpleExpressionOwner &p_owner, TypeManager &p_typeManager) const; 40 | 41 | private: 42 | // Creates an expression from the given parse state. 43 | virtual const Expression &CreateExpression( 44 | const ProgramParseState::ExpressionParseState &p_state, 45 | SimpleExpressionOwner &p_owner, TypeManager &p_typeManager) const = 0; 46 | 47 | // Indicates the allowed arity of expressions produced from 48 | // this factory, in a [min, max] pair (both ends inclusive). 49 | virtual std::pair Arity() const = 0; 50 | }; 51 | } // namespace FreeForm2 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Parse/SExpression/libs/Logic.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "Logic.h" 7 | 8 | #include "BinaryOperator.h" 9 | #include "OperatorExpressionFactory.h" 10 | #include "UnaryOperator.h" 11 | 12 | using namespace FreeForm2; 13 | 14 | namespace { 15 | typedef OperatorExpressionFactory OperatorFactory; 16 | static const OperatorFactory c_eqFactory(UnaryOperator::invalid, 17 | BinaryOperator::eq, false); 18 | static const OperatorFactory c_notEqFactory(UnaryOperator::invalid, 19 | BinaryOperator::neq, false); 20 | static const OperatorFactory c_ltFactory(UnaryOperator::invalid, 21 | BinaryOperator::lt, false); 22 | static const OperatorFactory c_lteFactory(UnaryOperator::invalid, 23 | BinaryOperator::lte, false); 24 | static const OperatorFactory c_gtFactory(UnaryOperator::invalid, 25 | BinaryOperator::gt, false); 26 | static const OperatorFactory c_gteFactory(UnaryOperator::invalid, 27 | BinaryOperator::gte, false); 28 | static const OperatorFactory c_andFactory(UnaryOperator::invalid, 29 | BinaryOperator::_and, true); 30 | static const OperatorFactory c_orFactory(UnaryOperator::invalid, 31 | BinaryOperator::_or, true); 32 | static const OperatorFactory c_notFactory(UnaryOperator::_not, 33 | BinaryOperator::invalid, false); 34 | } // namespace 35 | 36 | const FreeForm2::ExpressionFactory& FreeForm2::Logic::GetCmpEqInstance() { 37 | return c_eqFactory; 38 | } 39 | 40 | const FreeForm2::ExpressionFactory& FreeForm2::Logic::GetCmpNotEqInstance() { 41 | return c_notEqFactory; 42 | } 43 | 44 | const FreeForm2::ExpressionFactory& FreeForm2::Logic::GetCmpLTInstance() { 45 | return c_ltFactory; 46 | } 47 | 48 | const FreeForm2::ExpressionFactory& FreeForm2::Logic::GetCmpLTEInstance() { 49 | return c_lteFactory; 50 | } 51 | 52 | const FreeForm2::ExpressionFactory& FreeForm2::Logic::GetCmpGTInstance() { 53 | return c_gtFactory; 54 | } 55 | 56 | const FreeForm2::ExpressionFactory& FreeForm2::Logic::GetCmpGTEInstance() { 57 | return c_gteFactory; 58 | } 59 | 60 | const FreeForm2::ExpressionFactory& FreeForm2::Logic::GetAndInstance() { 61 | return c_andFactory; 62 | } 63 | 64 | const FreeForm2::ExpressionFactory& FreeForm2::Logic::GetOrInstance() { 65 | return c_orFactory; 66 | } 67 | 68 | const FreeForm2::ExpressionFactory& FreeForm2::Logic::GetNotInstance() { 69 | return c_notFactory; 70 | } 71 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Parse/SExpression/libs/Logic.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_LOGIC_H 9 | #define FREEFORM2_LOGIC_H 10 | 11 | namespace FreeForm2 { 12 | class ExpressionFactory; 13 | 14 | namespace Logic { 15 | const ExpressionFactory &GetCmpEqInstance(); 16 | const ExpressionFactory &GetCmpNotEqInstance(); 17 | const ExpressionFactory &GetCmpLTInstance(); 18 | const ExpressionFactory &GetCmpLTEInstance(); 19 | const ExpressionFactory &GetCmpGTInstance(); 20 | const ExpressionFactory &GetCmpGTEInstance(); 21 | 22 | const ExpressionFactory &GetAndInstance(); 23 | const ExpressionFactory &GetOrInstance(); 24 | const ExpressionFactory &GetNotInstance(); 25 | } // namespace Logic 26 | }; // namespace FreeForm2 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Parse/SExpression/libs/MiscFactory.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for license information. 4 | */ 5 | #pragma once 6 | 7 | #ifndef FREEFORM2_MISC_FACTORY_H 8 | #define FREEFORM2_MISC_FACTORY_H 9 | 10 | namespace FreeForm2 11 | { 12 | class ExpressionFactory; 13 | 14 | namespace Conditional 15 | { 16 | const ExpressionFactory &GetIfInstance(); 17 | } 18 | 19 | namespace Random 20 | { 21 | const ExpressionFactory &GetRandomFloatInstance(); 22 | const ExpressionFactory &GetRandomIntInstance(); 23 | } 24 | 25 | // Returns an expression factory for the array-length primitive. 26 | const ExpressionFactory &GetArrayLengthInstance(); 27 | 28 | namespace Convert 29 | { 30 | const ExpressionFactory &GetFloatConvertFactory(); 31 | const ExpressionFactory &GetIntConvertFactory(); 32 | const ExpressionFactory &GetBoolConversionFactory(); 33 | const ExpressionFactory &GetIdentityFactory(); 34 | } 35 | 36 | namespace Select 37 | { 38 | const ExpressionFactory &GetSelectNthInstance(); 39 | const ExpressionFactory &GetSelectRangeInstance(); 40 | } 41 | 42 | const ExpressionFactory &GetFeatureSpecInstance(bool p_mustConvertToFloat); 43 | }; 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Shared/Attributes.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for license information. 4 | */ 5 | #pragma once 6 | 7 | // A simple means for extracting the attribute bits and word offset 8 | // fields from a "location" as recorded in the index. 9 | 10 | enum WordAttribute 11 | { 12 | ZeroAttributeState = 0, 13 | 14 | // UrlWord attribute values 15 | 16 | ServiceUrlWord = 0, 17 | SubDomainUrlWord = 1, 18 | BaseDomainUrlWord = 2, 19 | TopDomainUrlWord = 3, 20 | PortUrlWord = 4, 21 | PathUrlWord = 5, 22 | QueryUrlWord = 6, 23 | AnyUrlWord = 7, 24 | 25 | NumberOfUrlAttributeStates = 8, 26 | 27 | // BodyText attribute values 28 | 29 | NormalBodyText = 0, 30 | NavigationBodyText = 1, 31 | Reserved1BodyText = 2, 32 | Reserved2BodyText = 3, 33 | AnyBodyText = 4, 34 | 35 | NumberOfTextAttributeStates = 5, 36 | 37 | MaximumNumberOfAttributeStates = 8 38 | }; 39 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Shared/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.15) 2 | 3 | set(PROJECT_NAME DRFreeFormSharedLibrary) 4 | 5 | 6 | Project(${PROJECT_NAME}) 7 | 8 | SET(CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS} -fpermissive") 9 | 10 | 11 | 12 | add_library(${PROJECT_NAME} STATIC 13 | ${CMAKE_CURRENT_SOURCE_DIR}/ArrayType.cpp 14 | ${CMAKE_CURRENT_SOURCE_DIR}/CompoundType.cpp 15 | ${CMAKE_CURRENT_SOURCE_DIR}/FreeForm2Assert.cpp 16 | ${CMAKE_CURRENT_SOURCE_DIR}/FreeForm2Utils.cpp 17 | ${CMAKE_CURRENT_SOURCE_DIR}/FunctionType.cpp 18 | ${CMAKE_CURRENT_SOURCE_DIR}/ObjectType.cpp 19 | ${CMAKE_CURRENT_SOURCE_DIR}/StateMachineType.cpp 20 | ${CMAKE_CURRENT_SOURCE_DIR}/StructType.cpp 21 | ${CMAKE_CURRENT_SOURCE_DIR}/TypeImpl.cpp 22 | ${CMAKE_CURRENT_SOURCE_DIR}/TypeManager.cpp 23 | ) 24 | 25 | target_include_directories(${PROJECT_NAME} PRIVATE 26 | ${CMAKE_CURRENT_SOURCE_DIR} 27 | ${CMAKE_CURRENT_SOURCE_DIR}/../../inc 28 | ${CMAKE_CURRENT_SOURCE_DIR}/../../../NeuralTree.Library/inc 29 | ) 30 | 31 | install(TARGETS ${PROJECT_NAME} 32 | LIBRARY DESTINATION lib 33 | ARCHIVE DESTINATION lib 34 | ) -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Shared/CompoundType.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "CompoundType.h" 7 | 8 | FreeForm2::CompoundType::Member::Member(const std::string &p_name, 9 | const TypeImpl &p_type) 10 | : m_name(p_name), m_type(&p_type) {} 11 | 12 | FreeForm2::CompoundType::Member::Member() : m_type(NULL) {} 13 | 14 | FreeForm2::CompoundType::CompoundType(Type::TypePrimitive p_prim, 15 | bool p_isConst, 16 | TypeManager *p_typeManager) 17 | : TypeImpl(p_prim, p_isConst, p_typeManager) {} 18 | 19 | bool FreeForm2::CompoundType::IsCompoundType(const TypeImpl &p_type) { 20 | return p_type.Primitive() == Type::Struct || 21 | p_type.Primitive() == Type::StateMachine || 22 | p_type.Primitive() == Type::Object; 23 | } 24 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Shared/CompoundType.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_COMPOUND_TYPE_H 9 | #define FREEFORM2_COMPOUND_TYPE_H 10 | 11 | #include 12 | 13 | #include "FreeForm2Type.h" 14 | #include "TypeImpl.h" 15 | 16 | namespace FreeForm2 { 17 | class TypeManager; 18 | 19 | // A compound type is any type which contains named instantiations of other 20 | // types. This class provides a mechanism for looking up member information 21 | // by name. 22 | class CompoundType : public TypeImpl { 23 | public: 24 | // A member represents a single named entity contained within a 25 | // compound type. 26 | struct Member { 27 | public: 28 | // Construct a member with a name and type. 29 | Member(const std::string &p_name, const TypeImpl &p_type); 30 | 31 | // Default constructor to initialize members to empty values. 32 | Member(); 33 | 34 | // The name of this member. 35 | std::string m_name; 36 | 37 | // The type of this member. 38 | const TypeImpl *m_type; 39 | }; 40 | 41 | // Create a compound type of the give type, constness, and type 42 | // manager. 43 | CompoundType(Type::TypePrimitive p_prim, bool p_isConst, 44 | TypeManager *p_typeManager); 45 | 46 | // Find a member by name. Returns a pointer to the member object 47 | // associated with a name; if this compound type does not contain a 48 | // member of the specified name, this function returns NULL. 49 | virtual const Member *FindMember(const std::string &p_name) const = 0; 50 | 51 | // Determine whether a TypePrimitive is a compound type. 52 | static bool IsCompoundType(const TypeImpl &p_type); 53 | }; 54 | } // namespace FreeForm2 55 | 56 | #endif 57 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Shared/FreeForm2Assert.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for license information. 4 | */ 5 | #include "FreeForm2Assert.h" 6 | 7 | #include 8 | #include 9 | 10 | void FreeForm2::ThrowAssert(bool p_condition, const char *p_file, unsigned int p_line) 11 | { 12 | if (!p_condition) 13 | { 14 | std::ostringstream err; 15 | err << "Assertion error at " << p_file << ":" << p_line; 16 | throw std::runtime_error(err.str()); 17 | } 18 | } 19 | 20 | void FreeForm2::ThrowAssert(bool p_condition, const char *p_expression, const char *p_file, unsigned int p_line) 21 | { 22 | if (!p_condition) 23 | { 24 | std::ostringstream err; 25 | err << "Assertion error: \"" << p_expression << "\" failed at " << p_file << ":" << p_line; 26 | throw std::runtime_error(err.str()); 27 | } 28 | } 29 | 30 | void FreeForm2::Unreachable(const char *p_file, unsigned int p_line) 31 | { 32 | std::ostringstream err; 33 | err << "Unreachable code reached at " << p_file << ":" << p_line; 34 | throw std::runtime_error(err.str()); 35 | } 36 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Shared/FunctionType.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_FUNCTION_TYPE_H 9 | #define FREEFORM2_FUNCTION_TYPE_H 10 | 11 | #include 12 | 13 | #include "FreeForm2Type.h" 14 | #include "TypeImpl.h" 15 | 16 | namespace FreeForm2 { 17 | class TypeManager; 18 | 19 | // A function type is any type which can be called with certain parameters and 20 | // returns a value of another type. 21 | class FunctionType : public TypeImpl { 22 | public: 23 | virtual ~FunctionType() {} 24 | 25 | // Gets the return type of the function. 26 | const TypeImpl &GetReturnType() const; 27 | 28 | // Iterate over parameters. 29 | typedef const TypeImpl **ParameterIterator; 30 | ParameterIterator BeginParameters() const; 31 | ParameterIterator EndParameters() const; 32 | 33 | // Get the number of parameters. 34 | size_t GetParameterCount() const; 35 | 36 | // Get a string representation of the type. 37 | virtual const std::string &GetName() const override; 38 | static std::string GetName(const TypeImpl &p_returnType, 39 | const TypeImpl *const *p_parameterTypes, 40 | size_t p_numParams); 41 | 42 | // Create derived types based on this type. 43 | virtual const TypeImpl &AsConstType() const override; 44 | virtual const TypeImpl &AsMutableType() const override; 45 | 46 | private: 47 | // Create a compound type of the give type, constness, and type 48 | // manager. 49 | FunctionType(TypeManager &p_typeManager, const TypeImpl &p_returnType, 50 | const TypeImpl *const *p_parameterTypes, size_t p_numParams); 51 | 52 | // Compare subclass type data. 53 | virtual bool IsSameSubType(const TypeImpl &p_type, 54 | bool p_ignoreConst) const override; 55 | 56 | friend class TypeManager; 57 | 58 | // The return type of the function. 59 | const TypeImpl *m_returnType; 60 | 61 | // The string representation of this type. 62 | mutable std::string m_name; 63 | 64 | // The number of parameters. 65 | size_t m_numParameters; 66 | 67 | // The types of the parameters. This is allocated using the struct hack. 68 | const TypeImpl *m_parameterTypes[1]; 69 | }; 70 | } // namespace FreeForm2 71 | 72 | #endif 73 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Shared/ObjectType.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_OBJECT_TYPE_H 9 | #define FREEFORM2_OBJECT_TYPE_H 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #include "CompoundType.h" 16 | 17 | namespace FreeForm2 { 18 | // Object types are used to store information about external objects. 19 | class ObjectType : public CompoundType { 20 | public: 21 | // A struct that holds the information about members of an object. 22 | struct ObjectMember : public CompoundType::Member { 23 | // Constructor to initialize all members of the class. 24 | ObjectMember(const std::string &p_name, const TypeImpl &p_type, 25 | const std::string &p_externName); 26 | 27 | // Constructor used when frontend name matches the external name. 28 | ObjectMember(const std::string &p_name, const TypeImpl &p_type); 29 | 30 | // The C++ name of the member. 31 | std::string m_externName; 32 | }; 33 | 34 | // Get the name of this object type. 35 | virtual const std::string &GetName() const override; 36 | 37 | // Get the name of this object type. 38 | const std::string &GetExternName() const; 39 | 40 | // Find a member by name. Return value is NULL if the object 41 | // does not contain the specified member. 42 | virtual const ObjectType::ObjectMember *FindMember( 43 | const std::string &p_name) const override; 44 | 45 | // Create derived types based on this type. 46 | virtual const TypeImpl &AsConstType() const override; 47 | virtual const TypeImpl &AsMutableType() const override; 48 | 49 | private: 50 | // Create an ObjectType of the given function information. 51 | ObjectType(const std::string &p_name, const std::string &p_externName, 52 | const std::vector &p_members, bool p_isConst, 53 | TypeManager &p_typeManager); 54 | 55 | friend class TypeManager; 56 | 57 | // Compare subclass type data. 58 | virtual bool IsSameSubType(const TypeImpl &p_type, 59 | bool p_ignoreConst) const override; 60 | 61 | // Functions associated with this type. 62 | std::map m_members; 63 | 64 | // Name of this object in the frontend. 65 | std::string m_name; 66 | 67 | // Name of this object in the backend. 68 | std::string m_externName; 69 | }; 70 | } // namespace FreeForm2 71 | 72 | #endif -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Transform/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.15) 2 | 3 | set(PROJECT_NAME DRFreeFormTransformLibrary) 4 | 5 | 6 | Project(${PROJECT_NAME}) 7 | 8 | SET(CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS} -fpermissive") 9 | 10 | 11 | 12 | add_library(${PROJECT_NAME} STATIC 13 | ${CMAKE_CURRENT_SOURCE_DIR}/AllocationVisitor.cpp 14 | ${CMAKE_CURRENT_SOURCE_DIR}/CopyingVisitor.cpp 15 | ${CMAKE_CURRENT_SOURCE_DIR}/FunctionInlineVisitor.cpp 16 | ${CMAKE_CURRENT_SOURCE_DIR}/ObjectResolutionVisitor.cpp 17 | ${CMAKE_CURRENT_SOURCE_DIR}/OperandPromotionVisitor.cpp 18 | ${CMAKE_CURRENT_SOURCE_DIR}/ProcessFeaturesUsed.cpp 19 | ${CMAKE_CURRENT_SOURCE_DIR}/TypeCheckingVisitor.cpp 20 | ${CMAKE_CURRENT_SOURCE_DIR}/UniformExpressionVisitor.cpp 21 | ) 22 | 23 | target_include_directories(${PROJECT_NAME} PRIVATE 24 | ${CMAKE_CURRENT_SOURCE_DIR} 25 | ${CMAKE_CURRENT_SOURCE_DIR}/../../inc 26 | ${CMAKE_CURRENT_SOURCE_DIR}/../../../NeuralTree.Library/inc 27 | ${CMAKE_CURRENT_SOURCE_DIR}/../Shared 28 | ${CMAKE_CURRENT_SOURCE_DIR}/../Expression 29 | ) 30 | 31 | install(TARGETS ${PROJECT_NAME} 32 | LIBRARY DESTINATION lib 33 | ARCHIVE DESTINATION lib 34 | ) -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Transform/FunctionInlineVisitor.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #include 9 | 10 | #include "CopyingVisitor.h" 11 | 12 | namespace FreeForm2 { 13 | // This class inlines function calls, inserting the function body directly 14 | // into the resultant expression tree. This class only works with 15 | // S-expression trees (or trees which have an implicit function return). 16 | class FunctionInlineVisitor : public CopyingVisitor { 17 | public: 18 | FunctionInlineVisitor(const boost::shared_ptr &p_owner, 19 | const boost::shared_ptr &p_typeManager, 20 | VariableID p_variableId); 21 | 22 | virtual bool AlternativeVisit(const FunctionCallExpression &p_expr) override; 23 | virtual void Visit(const ReturnExpression &p_expr) override; 24 | virtual void Visit(const VariableRefExpression &p_expr) override; 25 | 26 | // Returns the variable id counter. 27 | VariableID GetVariableId(); 28 | 29 | private: 30 | // A map containing type translations for function parameters. 31 | std::map m_parameterTypeTranslation; 32 | 33 | // A map containing mappings from old variable ids to new variable ids. 34 | // This is necessary for all variables within a lambda. 35 | std::map m_newVariableIdMapping; 36 | 37 | // Counter to keep track of the next variable ID. This allows us 38 | // to ensure that the variables assigned for each function call have 39 | // unique ids. 40 | VariableID m_variableId; 41 | }; 42 | } // namespace FreeForm2 -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Transform/ObjectResolutionVisitor.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | #ifndef FREEFORM2_OBJECT_RESOLUTION_VISITOR_H 8 | #define FREEFORM2_OBJECT_RESOLUTION_VISITOR_H 9 | 10 | #include 11 | 12 | #include "CopyingVisitor.h" 13 | 14 | namespace FreeForm2 { 15 | class CompoundType; 16 | 17 | // The ObjectResolutionVisitor is responsible for two-pass object type 18 | // annotating. For example with StateMachineExpressions, when parsed all 19 | // ThisExpressions that are children of the StateMachineExpression are of 20 | // type Unknown, and all unknown variables turn into 21 | // UnresolveAccessExpressions. Since the type can only be created after 22 | // parsing is complete, this information must be added in a second pass. 23 | class ObjectResolutionVisitor : public CopyingVisitor { 24 | public: 25 | // Create an ObjectResolutionVisitor using a new ExpressionOwner 26 | // and TypeManager. 27 | ObjectResolutionVisitor(); 28 | 29 | virtual bool AlternativeVisit(const StateMachineExpression &p_expr); 30 | virtual void Visit(const ThisExpression &p_expr); 31 | virtual void Visit(const UnresolvedAccessExpression &p_expr); 32 | virtual bool AlternativeVisit(const TypeInitializerExpression &p_expr); 33 | 34 | private: 35 | // The current type with which a ThisExpression should be annotated. 36 | std::stack m_thisTypeStack; 37 | }; 38 | } // namespace FreeForm2 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Transform/OperandPromotionVisitor.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_OPERAND_PROMOTION_VISITOR_H 9 | #define FREEFORM2_OPERAND_PROMOTION_VISITOR_H 10 | 11 | #include 12 | #include 13 | 14 | #include "CopyingVisitor.h" 15 | 16 | namespace FreeForm2 { 17 | class SimpleExpressionOwner; 18 | class ExpressionOwner; 19 | class Expression; 20 | 21 | class OperandPromotionVisitor : public CopyingVisitor { 22 | public: 23 | // Allow promotion in binary operator. 24 | virtual bool AlternativeVisit(const BinaryOperatorExpression &p_expr); 25 | virtual void Visit(const BinaryOperatorExpression &p_expr); 26 | 27 | // Allow promotion in the function call expression. 28 | virtual void Visit(const FunctionCallExpression &p_expr); 29 | 30 | // Allow promotion in conditional. 31 | virtual void Visit(const ConditionalExpression &p_expr); 32 | 33 | // Allow promotion in loop structures. 34 | virtual void Visit(const RangeReduceExpression &p_expr); 35 | virtual void Visit(const ForEachLoopExpression &p_expr); 36 | virtual void Visit(const ComplexRangeLoopExpression &p_expr); 37 | 38 | // Promote void type object method expressions to statements. 39 | // virtual void Visit(const ObjectMethodExpression& p_expr); 40 | }; 41 | } // namespace FreeForm2 42 | 43 | #endif 44 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Transform/ProcessFeaturesUsed.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "ProcessFeaturesUsed.h" 7 | 8 | #include 9 | 10 | #include "RefExpression.h" 11 | 12 | FreeForm2::ProcessFeaturesUsedVisitor::ProcessFeaturesUsedVisitor( 13 | DynamicRank::INeuralNetFeatures &p_features) 14 | : m_features(p_features) {} 15 | 16 | void FreeForm2::ProcessFeaturesUsedVisitor::Visit( 17 | const FeatureRefExpression &p_expr) { 18 | m_features.ProcessFeature(p_expr.m_index); 19 | } 20 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/libs/Transform/ProcessFeaturesUsed.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #ifndef FREEFORM2_PROCESS_FEATURES_USED_H 9 | #define FREEFORM2_PROCESS_FEATURES_USED_H 10 | 11 | #include "NoOpVisitor.h" 12 | 13 | namespace DynamicRank { 14 | class IFeatureMap; 15 | class INeuralNetFeatures; 16 | }; // namespace DynamicRank 17 | 18 | namespace FreeForm2 { 19 | // Class to collect the set of features used by a program. 20 | class ProcessFeaturesUsedVisitor : public NoOpVisitor { 21 | public: 22 | ProcessFeaturesUsedVisitor(DynamicRank::INeuralNetFeatures &p_features); 23 | 24 | // Methods inherited from Visitor. 25 | virtual void Visit(const FeatureRefExpression &p_expr); 26 | 27 | private: 28 | DynamicRank::INeuralNetFeatures &m_features; 29 | }; 30 | } // namespace FreeForm2 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.15) 2 | 3 | set(PROJECT_NAME FreeForm2Test) 4 | 5 | project(${PROJECT_NAME}) 6 | 7 | link_directories( 8 | ${CMAKE_CURRENT_SOURCE_DIR}/../libs/Shared 9 | ${CMAKE_CURRENT_SOURCE_DIR}/../libs/Expression 10 | ${CMAKE_CURRENT_SOURCE_DIR}/../libs/Backend/llvm 11 | ${CMAKE_CURRENT_SOURCE_DIR}/../libs/Transform 12 | ${CMAKE_CURRENT_SOURCE_DIR}/../libs/Parse/SExpression/lib 13 | ${CMAKE_CURRENT_SOURCE_DIR}/../libs/External 14 | ${CMAKE_CURRENT_SOURCE_DIR}/../../NeuralTree.Library/src 15 | ) 16 | 17 | add_executable(${PROJECT_NAME} 18 | ${CMAKE_CURRENT_SOURCE_DIR}/FreeFormLibTest.cpp 19 | ${CMAKE_CURRENT_SOURCE_DIR}/SimpleFeatureMap.cpp 20 | ) 21 | 22 | target_include_directories(${PROJECT_NAME} PRIVATE 23 | ${CMAKE_CURRENT_SOURCE_DIR} 24 | ${CMAKE_CURRENT_SOURCE_DIR}/../inc 25 | ${CMAKE_CURRENT_SOURCE_DIR}/../../NeuralTree.Library/inc 26 | ${CMAKE_CURRENT_SOURCE_DIR}/../libs/Shared 27 | ${CMAKE_CURRENT_SOURCE_DIR}/../libs/Expression 28 | ${CMAKE_CURRENT_SOURCE_DIR}/../libs/Backend/llvm 29 | ${CMAKE_CURRENT_SOURCE_DIR}/../libs/Transform 30 | ${CMAKE_CURRENT_SOURCE_DIR}/../libs/Parse/SExpression/inc 31 | ${CMAKE_CURRENT_SOURCE_DIR}/../libs/External 32 | ) 33 | 34 | target_link_libraries(${PROJECT_NAME} 35 | -Wl,--no-as-needed 36 | -Wl,--start-group 37 | -lpthread 38 | -lz 39 | -ldl 40 | -ltinfo 41 | DRNeuralTreeLibrary 42 | DRFreeFormLlvmBackendLibrary 43 | DRFreeFormSharedLibrary 44 | DRFreeFormExpressionLibrary 45 | DRFreeFormTransformLibrary 46 | DRFreeFormSExpressionLibrary 47 | DRFreeFormLibrary 48 | ${Boost_LIBRARIES} 49 | ${LLVM_LIB} 50 | -Wl,--end-group 51 | ) 52 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/test/FreeFormLibTest.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include 7 | 8 | #include "FreeFormLibTestSet.h" 9 | #include "log.h" 10 | 11 | using namespace DynamicRank; 12 | 13 | int main() { 14 | Log::Info("-------- FreeForm Library Test Starts --------"); 15 | FreeFormLibTestSet::TestParser("(if (== Foo Bar) 1 0)"); 16 | FreeFormLibTestSet::TestParser( 17 | "(* (ln1 NumberOfCompleteMatches_IETBSatModel-IM-Prod) " 18 | "OriginalQueryMaxNumberOfPerfectMatches_BingClicks-Prod)"); 19 | FreeFormLibTestSet::TestNeuralInputLoadSave(); 20 | Log::Info("-------- FreeForm Library Test Finished --------"); 21 | return 0; 22 | } 23 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/test/SimpleFeatureMap.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "SimpleFeatureMap.h" 7 | 8 | #include 9 | 10 | SimpleFeatureMap::SimpleFeatureMap() : m_numberOfFeatures(0) {} 11 | 12 | SimpleFeatureMap::~SimpleFeatureMap() {} 13 | 14 | bool SimpleFeatureMap::GetExistingFeatureIndex(const char *featureName, 15 | UInt32 &featureIndex) const { 16 | std::string featureNameStr(featureName); 17 | std::map::const_iterator it = 18 | m_featureMap.find(featureNameStr); 19 | 20 | if (it == m_featureMap.end()) { 21 | return false; 22 | } 23 | featureIndex = it->second; 24 | return true; 25 | } 26 | 27 | bool SimpleFeatureMap::ObtainFeatureIndex(const char *featureName, 28 | UInt32 &featureIndex) { 29 | const std::string featureNameStr(featureName); 30 | std::map::iterator it = 31 | m_featureMap.find(featureNameStr); 32 | if (it != m_featureMap.end()) { 33 | featureIndex = it->second; 34 | } else { 35 | UInt32 iFeature = m_numberOfFeatures; 36 | std::pair pair(featureNameStr, iFeature); 37 | m_featureMap.insert(pair); 38 | m_reverseFeatureMap.push_back(std::string(featureName)); 39 | m_numberOfFeatures++; 40 | featureIndex = iFeature; 41 | } 42 | return true; 43 | } 44 | 45 | bool SimpleFeatureMap::ObtainFeatureIndex(const SIZED_STRING &featureName, 46 | UInt32 &featureIndex) { 47 | std::string localFeatureName((const char *)featureName.pbData, 48 | featureName.cbData); 49 | return ObtainFeatureIndex(localFeatureName.c_str(), featureIndex); 50 | } 51 | 52 | bool SimpleFeatureMap::GetFeatureName(UInt32 featureIndex, char *featureName, 53 | UInt32 maxNameLength) const { 54 | if (featureIndex < m_numberOfFeatures) { 55 | _snprintf_s(featureName, maxNameLength, _TRUNCATE, "%s", 56 | m_reverseFeatureMap[featureIndex].c_str()); 57 | return true; 58 | } 59 | return false; 60 | } 61 | 62 | const std::string &SimpleFeatureMap::GetFeatureName(UInt32 featureIndex) const { 63 | return m_reverseFeatureMap[featureIndex]; 64 | } 65 | 66 | UInt32 SimpleFeatureMap::GetNumberOfFeatures() const { 67 | return m_numberOfFeatures; 68 | } 69 | -------------------------------------------------------------------------------- /src/DynamicRank.FreeForm.Library/test/SimpleFeatureMap.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #include "IFeatureMap.h" 13 | #include "basic_types.h" 14 | 15 | #define _snprintf_s(a, b, c, ...) snprintf(a, b, __VA_ARGS__) 16 | 17 | class SimpleFeatureMap : public DynamicRank::IFeatureMap { 18 | private: 19 | std::map m_featureMap; 20 | std::vector m_reverseFeatureMap; 21 | UInt32 m_numberOfFeatures; 22 | 23 | public: 24 | SimpleFeatureMap(); 25 | ~SimpleFeatureMap(); 26 | 27 | // Get the index of a feature, or return false if there is no such feature. 28 | bool GetExistingFeatureIndex(const char *featureName, 29 | UInt32 &featureIndex) const; 30 | 31 | // Convert the FeatureName to the FeatureIndex. 32 | bool ObtainFeatureIndex(const char *featureName, UInt32 &featureIndex); 33 | 34 | // Convert the FeatureName to the FeatureIndex. 35 | bool ObtainFeatureIndex(const SIZED_STRING &featureName, 36 | UInt32 &featureIndex); 37 | 38 | // Convert the FeatureIndex to the FeatureName. 39 | bool GetFeatureName(UInt32 featureIndex, char *featureName, 40 | UInt32 maxNameLength) const; 41 | const std::string &GetFeatureName(UInt32 featureIndex) const; 42 | 43 | // The number of features within the map. 44 | UInt32 GetNumberOfFeatures() const; 45 | }; 46 | -------------------------------------------------------------------------------- /src/NeuralTree.Library/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.15.0 FATAL_ERROR) 2 | 3 | project(NeuralTree.Library CXX) 4 | 5 | set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -std=c++11") 6 | 7 | add_subdirectory(src) 8 | -------------------------------------------------------------------------------- /src/NeuralTree.Library/inc/FeaSpecConfig.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | using namespace std; 13 | 14 | namespace DynamicRank { 15 | 16 | class Config { 17 | private: 18 | string _path; 19 | map > _config; 20 | Config(map > &config) { _config = config; }; 21 | 22 | public: 23 | static Config *GetRawConfiguration(string str); 24 | bool DoesSectionExist(char *section); 25 | bool DoesParameterExist(const char *section, const char *parameterName); 26 | bool GetStringParameter(const char *section, const char *parameterName, 27 | string &value); 28 | bool GetStringParameter(const char *section, const char *parameterName, 29 | char *value, size_t valueSize); 30 | bool GetDoubleParameter(const char *section, const char *parameterName, 31 | double *value); 32 | double GetDoubleParameter(const char *section, const char *parameterName, 33 | double defaultValue); 34 | bool GetBoolParameter(const char *section, const char *parameterName, 35 | bool *value); 36 | bool GetBoolParameter(const char *section, const char *parameterName, 37 | bool defaultValue); 38 | }; 39 | } // namespace DynamicRank -------------------------------------------------------------------------------- /src/NeuralTree.Library/inc/IFeatureMap.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #include "basic_types.h" 9 | 10 | namespace DynamicRank { 11 | 12 | // A Feature is defined by any of the following three parameters 13 | // FeatureName : The Name of the Feature 14 | // FeatureIndex : The Index of the Feature according to the FeatureMap 15 | // FeatureName <-> FeatureIndex conversion is a combination of the above two 16 | class IFeatureMap { 17 | public: 18 | // Destructor 19 | virtual ~IFeatureMap() {} 20 | 21 | // Convert the FeatureName to the FeatureIndex. 22 | virtual bool ObtainFeatureIndex(const char *p_featureName, 23 | UInt32 &p_featureIndex) = 0; 24 | 25 | // Convert the FeatureName to the FeatureIndex. 26 | virtual bool ObtainFeatureIndex(const SIZED_STRING &p_featureName, 27 | UInt32 &p_featureIndex) = 0; 28 | 29 | // Convert the FeatureIndex to the FeatureName. 30 | virtual bool GetFeatureName(UInt32 p_featureIndex, char *p_featureName, 31 | UInt32 p_maxNameLength) const = 0; 32 | 33 | // The number of features within the map. 34 | virtual UInt32 GetNumberOfFeatures() const = 0; 35 | }; 36 | } // namespace DynamicRank 37 | -------------------------------------------------------------------------------- /src/NeuralTree.Library/inc/INeuralNetFeatures.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | #include 8 | #include 9 | 10 | #include "IFeatureMap.h" 11 | 12 | namespace DynamicRank { 13 | 14 | // Interface to apply custom process on all features input. This is especially 15 | // useful for on-demand feature extraction. 16 | class INeuralNetFeatures { 17 | public: 18 | // Process a given feature index. The feature index is the value returned from 19 | // a FeatureMap. 20 | virtual void ProcessFeature(UInt32 p_featureIndex) = 0; 21 | 22 | // Same as above but will also pass in a vector of strings for all the 23 | // segments. An empty vector means that the feature is part of the main L2 24 | // ranker. 25 | virtual void ProcessFeature(UInt32 p_featureIndex, 26 | const std::vector &p_segments) = 0; 27 | }; 28 | 29 | } // namespace DynamicRank 30 | -------------------------------------------------------------------------------- /src/NeuralTree.Library/inc/MigratedApi.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | #include 8 | 9 | template 10 | constexpr std::size_t countof(T const (&)[N]) noexcept { 11 | return N; 12 | } 13 | 14 | static inline unsigned int rotl(const unsigned int x, int bits) { 15 | const unsigned int n = ((bits % 32) + 32) % 32; 16 | return (x << n) | (x >> (32 - n)); 17 | } 18 | 19 | #ifdef _snprintf_s 20 | #undef _snprintf_s 21 | #endif 22 | #define _snprintf_s(a, b, c, ...) snprintf(a, b, __VA_ARGS__) 23 | -------------------------------------------------------------------------------- /src/NeuralTree.Library/inc/basic_types.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #ifdef _In_z_ 13 | #undef _In_z_ 14 | #endif 15 | #define _In_z_ 16 | 17 | typedef uint32_t UInt32; 18 | typedef uint8_t UInt8; 19 | typedef uint64_t UInt64; 20 | 21 | typedef int64_t Int64; 22 | typedef int32_t Int32; 23 | typedef unsigned long DWORD; 24 | 25 | typedef size_t Size_t; 26 | 27 | #define MAX_UINT32 ((UInt32)-1) 28 | 29 | struct SIZED_STRING { 30 | union { 31 | const UInt8 *pbData; 32 | const char *pcData; 33 | }; 34 | size_t cbData; 35 | }; 36 | 37 | #define _TRUNCATE ((size_t)-1) 38 | 39 | // A utility class for creating temporary SIZED_STRINGs 40 | class CStackSizedString : public SIZED_STRING { 41 | public: 42 | // Null-terminated input 43 | CStackSizedString(_In_z_ const char *szValue) { 44 | #if defined(_MSC_VER) && _MSC_VER >= 1910 45 | // Suppression: error C26490: Don't use reinterpret_cast. 46 | // reinterpreting the bits of the char* to the UInt* (both 8 bits per item) 47 | // is necessary here 48 | [[gsl::suppress(type .1)]] 49 | #endif 50 | pbData = reinterpret_cast(szValue); 51 | cbData = strlen(szValue); 52 | } 53 | 54 | // Name/size pair 55 | CStackSizedString(const UInt8 *pbValue, size_t cbValue) { 56 | pbData = pbValue; 57 | cbData = cbValue; 58 | } 59 | 60 | // Name/size pair 61 | CStackSizedString(const char *pcValue, size_t cbValue) { 62 | pcData = pcValue; 63 | cbData = cbValue; 64 | } 65 | 66 | private: 67 | // prevent heap allocation 68 | void *operator new(size_t); 69 | }; 70 | 71 | #define UINT_MAX 0xffffffffu 72 | 73 | // convenience macros to make SIZED_STRINGs more usable, particularly with the 74 | // FEX Document class and FexSprintf expand a SIZED_STRING: 75 | #define SIZED_STR(sizedstr) (char *)sizedstr.pbData, sizedstr.cbData 76 | // reverse, for printing to FexSprintf: 77 | #define SIZED_STR_REV(sizedstr) sizedstr.cbData, sizedstr.pcData 78 | // References, for loading with document->GetField( SIZED_STR_REF(str) ); 79 | #define SIZED_STR_REF(sizedstr) &sizedstr.pcData, &sizedstr.cbData 80 | // Convert a SIZED_STRING to std::basic_string 81 | #define SIZED_STR_STL(sizedstr) \ 82 | (sizedstr.cbData > 0 ? std::string(SIZED_STR(sizedstr)) : std::string()) 83 | 84 | typedef int BOOL; 85 | 86 | #ifndef FALSE 87 | #define FALSE 0 88 | #endif 89 | 90 | #ifndef TRUE 91 | #define TRUE 1 92 | #endif 93 | -------------------------------------------------------------------------------- /src/NeuralTree.Library/inc/common.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | namespace DynamicRank { 8 | namespace Common { 9 | inline static std::string Trim(std::string str) { 10 | if (str.empty()) { 11 | return str; 12 | } 13 | str.erase(str.find_last_not_of(" \f\n\r\t\v") + 1); 14 | str.erase(0, str.find_first_not_of(" \f\n\r\t\v")); 15 | return str; 16 | } 17 | 18 | inline static std::string LoadStringFromFile(const char* filename, 19 | int row_num = INT_MAX) { 20 | if (filename == NULL || *filename == '\0') { 21 | return ""; 22 | } 23 | std::stringstream ss; 24 | std::ifstream fin(filename); 25 | std::string line = ""; 26 | int i = 0; 27 | while (std::getline(fin, line) && i++ < row_num) { 28 | ss << line << "\n"; 29 | } 30 | return ss.str(); 31 | } 32 | 33 | } // namespace Common 34 | } // namespace DynamicRank -------------------------------------------------------------------------------- /src/NeuralTree.Library/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(PROJECT_NAME DRNeuralTreeLibrary) 2 | 3 | set(Headers 4 | ../inc/basic_types.h 5 | ../inc/CsHash.h 6 | ../inc/FeaSpecConfig.h 7 | ../inc/IFeatureMap.h 8 | ../inc/INeuralNetFeatures.h 9 | ../inc/MigratedApi.h 10 | ../inc/NeuralInput_types.h 11 | ../inc/NeuralInput.h 12 | ../inc/NeuralInputFactory.h 13 | ../inc/NeuralInputFreeForm2_types.h 14 | ../inc/UnionBondInput_types.h 15 | ../inc/log.h 16 | ../inc/common.h 17 | ) 18 | source_group("Headers" FILES ${Headers}) 19 | 20 | set(Sources 21 | "CsHash.cpp" 22 | "FeaSpecConfig.cpp" 23 | "NeuralInput.cpp" 24 | "NeuralInputFactory.cpp" 25 | ) 26 | source_group("Sources" FILES ${Sources}) 27 | 28 | set(ALL_FILES 29 | ${Headers} 30 | ${Sources} 31 | ) 32 | 33 | add_library(${PROJECT_NAME} STATIC ${ALL_FILES}) 34 | 35 | target_include_directories(${PROJECT_NAME} PRIVATE 36 | "${CMAKE_CURRENT_SOURCE_DIR}/../inc" 37 | ) 38 | -------------------------------------------------------------------------------- /src/NeuralTreeEvaluator.Library/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.15.0 FATAL_ERROR) 2 | 3 | project(NeuralTreeEvaluator.Library) 4 | 5 | set(CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS} -fpermissive") 6 | 7 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src) 8 | add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/test) 9 | 10 | install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/inc/ 11 | DESTINATION include 12 | FILES_MATCHING PATTERN "*.h" 13 | ) 14 | -------------------------------------------------------------------------------- /src/NeuralTreeEvaluator.Library/inc/FeatureMap.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for license information. 4 | */ 5 | #pragma once 6 | 7 | #include "basic_types.h" 8 | 9 | class MinimalFeatureMap; 10 | 11 | UInt32 FeatureMapGetFeatureCount(MinimalFeatureMap *featureMap); 12 | UInt32 FeatureMapGetFeatureNameMaxLength(MinimalFeatureMap *featureMap); 13 | bool FeatureMapGetFeatureIndex( 14 | MinimalFeatureMap *featureMap, 15 | const char *featureName, 16 | UInt32 *featureIndex); 17 | bool FeatureMapGetFeatureName( 18 | MinimalFeatureMap *featureMap, 19 | UInt32 featureIndex, 20 | char *featureNameBuffer, 21 | UInt32 sizeOfBuffer, 22 | UInt32 *featureNameLength); -------------------------------------------------------------------------------- /src/NeuralTreeEvaluator.Library/inc/InputComputation.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #include 9 | 10 | void InputGetFeatures(const DynamicRank::NeuralInput *input, UInt32 *features, 11 | UInt32 sizeOfFeatures, UInt32 *featureCount); 12 | bool InputIsCopy(const DynamicRank::NeuralInput *input); 13 | double InputEvaluate(const DynamicRank::NeuralInput *input, 14 | UInt32 *featureValues); 15 | void InputEvaluateMany(const DynamicRank::NeuralInput *input, 16 | UInt32 **featureValues, double *outputs, UInt32 count); -------------------------------------------------------------------------------- /src/NeuralTreeEvaluator.Library/inc/InputExtraction.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #include 9 | #include 10 | 11 | class InputExtractor; 12 | class MinimalFeatureMap; 13 | 14 | InputExtractor *InputExtractorCreateFromInputStr(const std::string &str); 15 | InputExtractor *InputExtractorCreateFromFreeformV2(const char *freeform); 16 | void InputExtractorDispose(InputExtractor *extractor); 17 | MinimalFeatureMap *InputExtractorGetFeatureMap(InputExtractor *extractor); 18 | 19 | bool InputExtractorGetInputName(InputExtractor *extractor, UInt32 inputIndex, 20 | char *buffer, UInt32 sizeOfBuffer, 21 | UInt32 *length); 22 | 23 | UInt32 InputExtractorGetInputCount(InputExtractor *extractor); 24 | const DynamicRank::NeuralInput *InputExtractorGetInput( 25 | InputExtractor *extractor, UInt32 index); 26 | 27 | bool InputExtractorGetSectionContent(InputExtractor *extractor, 28 | char *sectionName, 29 | char *sectionContentBuffer, 30 | UInt32 sizeOfBuffer, 31 | UInt32 *sectionContentLength); -------------------------------------------------------------------------------- /src/NeuralTreeEvaluator.Library/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(PROJECT_NAME NeuralTreeEvaluatorLibrary) 2 | 3 | set(Headers 4 | "InputExtractor.h" 5 | "LocalFactoryHolder.h" 6 | "MinimalFeatureMap.h" 7 | ) 8 | source_group("Headers" FILES ${Headers}) 9 | 10 | set(Sources 11 | "FeatureMap.cpp" 12 | "InputComputation.cpp" 13 | "InputExtraction.cpp" 14 | "InputExtractor.cpp" 15 | "LocalFactoryHolder.cpp" 16 | "MinimalFeatureMap.cpp" 17 | ) 18 | source_group("Sources" FILES ${Sources}) 19 | 20 | set(inc 21 | "../inc/FeatureMap.h" 22 | "../inc/InputComputation.h" 23 | "../inc/InputExtraction.h" 24 | ) 25 | source_group("inc" FILES ${inc}) 26 | 27 | set(ALL_FILES 28 | ${Headers} 29 | ${Sources} 30 | ${inc} 31 | ) 32 | 33 | add_library(${PROJECT_NAME} STATIC ${ALL_FILES}) 34 | 35 | target_include_directories(${PROJECT_NAME} PRIVATE 36 | ${CMAKE_CURRENT_SOURCE_DIR} 37 | ${CMAKE_CURRENT_SOURCE_DIR}/../inc 38 | ${CMAKE_CURRENT_SOURCE_DIR}/../../NeuralTree.Library/inc 39 | ${CMAKE_CURRENT_SOURCE_DIR}/../../DynamicRank.FreeForm.Library/inc 40 | ${CMAKE_CURRENT_SOURCE_DIR}/../../DynamicRank.FreeForm.Library/lib/Shared 41 | ${CMAKE_CURRENT_SOURCE_DIR}/../../DynamicRank.FreeForm.Library/lib/Expression 42 | ${CMAKE_CURRENT_SOURCE_DIR}/../../DynamicRank.FreeForm.Library/lib/Backend/llvm 43 | ${CMAKE_CURRENT_SOURCE_DIR}/../../DynamicRank.FreeForm.Library/lib/Transform 44 | ${CMAKE_CURRENT_SOURCE_DIR}/../../DynamicRank.FreeForm.Library/lib/Parse/SExpression/inc 45 | ${CMAKE_CURRENT_SOURCE_DIR}/../../DynamicRank.FreeForm.Library/lib/External 46 | ) 47 | -------------------------------------------------------------------------------- /src/NeuralTreeEvaluator.Library/src/FeatureMap.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "FeatureMap.h" 7 | 8 | #include "MinimalFeatureMap.h" 9 | 10 | UInt32 FeatureMapGetFeatureCount(MinimalFeatureMap *featureMap) { 11 | if (featureMap) { 12 | return featureMap->GetNumberOfFeatures(); 13 | } 14 | return 0; 15 | } 16 | 17 | UInt32 FeatureMapGetFeatureNameMaxLength(MinimalFeatureMap *featureMap) { 18 | if (featureMap) { 19 | UInt32 max = 0; 20 | UInt32 num = featureMap->GetNumberOfFeatures(); 21 | for (UInt32 i = 0; i < num; ++i) { 22 | UInt32 len = (UInt32)featureMap->GetFeatureName(i).length(); 23 | if (len > max) { 24 | max = len; 25 | } 26 | } 27 | return max; 28 | } 29 | return 0; 30 | } 31 | 32 | bool FeatureMapGetFeatureIndex(MinimalFeatureMap *featureMap, 33 | const char *featureName, UInt32 *featureIndex) { 34 | if (featureMap) { 35 | return featureMap->GetExistingFeatureIndex(featureName, *featureIndex); 36 | } 37 | return false; 38 | } 39 | 40 | bool FeatureMapGetFeatureName(MinimalFeatureMap *featureMap, 41 | UInt32 featureIndex, char *featureNameBuffer, 42 | UInt32 sizeOfBuffer, UInt32 *featureNameLength) { 43 | memset(featureNameBuffer, 0x00, sizeOfBuffer); 44 | if (featureMap) { 45 | std::string name = featureMap->GetFeatureName(featureIndex); 46 | *featureNameLength = (UInt32)name.length(); 47 | if (name.length() > sizeOfBuffer) { 48 | return false; 49 | } 50 | memcpy(featureNameBuffer, name.c_str(), name.length()); 51 | return true; 52 | } 53 | return false; 54 | } -------------------------------------------------------------------------------- /src/NeuralTreeEvaluator.Library/src/InputComputation.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for license information. 4 | */ 5 | #include "InputComputation.h" 6 | #include 7 | 8 | void InputGetFeatures(const DynamicRank::NeuralInput *input, UInt32 *features, UInt32 sizeOfFeatures, UInt32 *featureCount) 9 | { 10 | if (input == NULL) 11 | { 12 | *featureCount = 0; 13 | return; 14 | } 15 | std::vector featureList; 16 | input->GetAllAssociatedFeatures(featureList); 17 | *featureCount = (UInt32)featureList.size(); 18 | for (UInt32 i = 0; i < *featureCount && i < sizeOfFeatures; ++i) 19 | { 20 | features[i] = featureList[i]; 21 | } 22 | } 23 | 24 | bool InputIsCopy(const DynamicRank::NeuralInput *input) 25 | { 26 | if (input == NULL) 27 | { 28 | return false; 29 | } 30 | const DynamicRank::NeuralInput *baseInput = input; 31 | const DynamicRank::NeuralInputCached *cachedInput = dynamic_cast(input); 32 | if (cachedInput != NULL) 33 | { 34 | baseInput = cachedInput->GetBaseInput(); 35 | } 36 | 37 | const DynamicRank::NeuralInputLinear *linearInput = dynamic_cast(baseInput); 38 | 39 | // Do a quick check for the identity function once we establish that this is a linear input. 40 | return linearInput != NULL && linearInput->EvaluateInput(0) == 0.0 && linearInput->EvaluateInput(1) == 1.0; 41 | } 42 | 43 | double InputEvaluate(const DynamicRank::NeuralInput *input, UInt32 *featureValues) 44 | { 45 | return input->Evaluate(featureValues); 46 | } 47 | 48 | void InputEvaluateMany(const DynamicRank::NeuralInput *input, UInt32 **featureValues, double *outputs, UInt32 count) 49 | { 50 | for (UInt32 i = 0; i < count; ++i) 51 | { 52 | outputs[i] = input->Evaluate(featureValues[i]); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/NeuralTreeEvaluator.Library/src/InputExtraction.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "InputExtraction.h" 7 | 8 | #include 9 | #include 10 | 11 | #include "InputExtractor.h" 12 | #include "LocalFactoryHolder.h" 13 | #include "MinimalFeatureMap.h" 14 | 15 | InputExtractor *InputExtractorCreateFromInputStr(const string &str) { 16 | return InputExtractor::CreateFromInputStr(str); 17 | } 18 | 19 | InputExtractor *InputExtractorCreateFromFreeformV2(const char *freeform) { 20 | return InputExtractor::CreateFromFreeform2(freeform); 21 | } 22 | 23 | void InputExtractorDispose(InputExtractor *extractor) { 24 | if (extractor != NULL) { 25 | delete extractor; 26 | } 27 | } 28 | 29 | MinimalFeatureMap *InputExtractorGetFeatureMap(InputExtractor *extractor) { 30 | if (extractor != NULL) { 31 | return extractor->GetFeatureMap(); 32 | } 33 | 34 | return 0; 35 | } 36 | 37 | bool InputExtractorGetInputName(InputExtractor *extractor, UInt32 inputIndex, 38 | char *buffer, UInt32 sizeOfBuffer, 39 | UInt32 *length) { 40 | memset(buffer, 0x00, sizeOfBuffer); 41 | if (extractor != NULL) { 42 | std::string content = extractor->GetInputName(inputIndex); 43 | *length = (UInt32)content.length(); 44 | if (content.length() > sizeOfBuffer) return false; 45 | memcpy(buffer, content.c_str(), content.length()); 46 | return true; 47 | } 48 | return false; 49 | } 50 | 51 | UInt32 InputExtractorGetInputCount(InputExtractor *extractor) { 52 | if (extractor != NULL) { 53 | return extractor->GetInputCount(); 54 | } 55 | return 0; 56 | } 57 | 58 | const DynamicRank::NeuralInput *InputExtractorGetInput( 59 | InputExtractor *extractor, UInt32 index) { 60 | if (extractor != NULL) { 61 | return extractor->GetInput(index); 62 | } 63 | return NULL; 64 | } 65 | -------------------------------------------------------------------------------- /src/NeuralTreeEvaluator.Library/src/InputExtractor.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for license information. 4 | */ 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include "MinimalFeatureMap.h" 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | using namespace DynamicRank; 22 | 23 | class InputExtractor 24 | { 25 | private: 26 | // The configuration file from which we update these. 27 | DynamicRank::Config *m_config; 28 | // The list of inputs corresponding to evaluators. 29 | std::vector > m_inputs; 30 | // The feature map. 31 | boost::shared_ptr m_featureMap; 32 | InputExtractor( 33 | DynamicRank::Config *config, 34 | boost::shared_ptr featureMap, 35 | std::vector > &inputs); 36 | 37 | public: 38 | ~InputExtractor(void); 39 | 40 | static InputExtractor *CreateFromConfig(DynamicRank::Config *config); 41 | static InputExtractor *CreateFromInputStr(const string &str); 42 | static InputExtractor *CreateFromFreeform2(const char *freeform); 43 | 44 | MinimalFeatureMap *GetFeatureMap() const; 45 | UInt32 GetInputCount() const; 46 | const DynamicRank::NeuralInput *GetInput(UInt32 index) const; 47 | const std::string GetInputName(UInt32 index) const; 48 | }; -------------------------------------------------------------------------------- /src/NeuralTreeEvaluator.Library/src/LocalFactoryHolder.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "LocalFactoryHolder.h" 7 | 8 | using namespace DynamicRank; 9 | 10 | LocalFactoryHolder::LocalFactoryHolder() { 11 | m_ffv2loader = boost::shared_ptr< 12 | FreeForm2::CompiledNeuralInputLoader >( 13 | new FreeForm2::CompiledNeuralInputLoader( 14 | "freeform2")); 15 | 16 | m_inputFactory.AddTransform("freeform2", m_ffv2loader); 17 | } 18 | 19 | NeuralInputFactory &LocalFactoryHolder::GetInputFactory() { 20 | return m_inputFactory; 21 | } 22 | 23 | void LocalFactoryHolder::PostLoad(bool useAggregatedCompiler) { 24 | std::unique_ptr compiler( 25 | FreeForm2::CompilerFactory::CreateExecutableCompiler(0)); 26 | m_ffv2loader->Compile(*compiler); 27 | } -------------------------------------------------------------------------------- /src/NeuralTreeEvaluator.Library/src/LocalFactoryHolder.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for license information. 4 | */ 5 | #pragma once 6 | 7 | #include "FreeForm2.h" 8 | #include "NeuralInputFreeForm2.h" 9 | 10 | // Several different areas may require access to an input and ensemble factory. 11 | class LocalFactoryHolder 12 | { 13 | private: 14 | DynamicRank::NeuralInputFactory m_inputFactory; 15 | boost::shared_ptr > m_ffv2loader; 16 | 17 | public: 18 | LocalFactoryHolder(); 19 | DynamicRank::NeuralInputFactory &GetInputFactory(); 20 | // Call this function after loading has finished, but 21 | // evaluation has not yet begun. 22 | void PostLoad(bool useAggregatedCompiler); 23 | }; 24 | -------------------------------------------------------------------------------- /src/NeuralTreeEvaluator.Library/src/MinimalFeatureMap.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for license information. 4 | */ 5 | #include "MinimalFeatureMap.h" 6 | #include 7 | #include "MigratedApi.h" 8 | 9 | MinimalFeatureMap::MinimalFeatureMap() 10 | : m_numberOfFeatures(0) 11 | { 12 | } 13 | 14 | MinimalFeatureMap::~MinimalFeatureMap() 15 | { 16 | } 17 | 18 | bool MinimalFeatureMap::GetExistingFeatureIndex( 19 | const char *featureName, UInt32 &featureIndex) const 20 | { 21 | std::string featureNameStr(featureName); 22 | std::map::const_iterator it = m_featureMap.find(featureNameStr); 23 | 24 | if (it == m_featureMap.end()) 25 | { 26 | return false; 27 | } 28 | featureIndex = it->second; 29 | return true; 30 | } 31 | 32 | bool MinimalFeatureMap::ObtainFeatureIndex( 33 | const char *featureName, UInt32 &featureIndex) 34 | { 35 | const std::string featureNameStr(featureName); 36 | std::map::iterator it = m_featureMap.find(featureNameStr); 37 | if (it != m_featureMap.end()) 38 | { 39 | featureIndex = it->second; 40 | } 41 | else 42 | { 43 | UInt32 iFeature = m_numberOfFeatures; 44 | std::pair pair(featureNameStr, iFeature); 45 | m_featureMap.insert(pair); 46 | m_reverseFeatureMap.push_back(std::string(featureName)); 47 | m_numberOfFeatures++; 48 | featureIndex = iFeature; 49 | } 50 | return true; 51 | } 52 | 53 | bool MinimalFeatureMap::ObtainFeatureIndex( 54 | const SIZED_STRING &featureName, UInt32 &featureIndex) 55 | { 56 | std::string localFeatureName((const char *)featureName.pbData, featureName.cbData); 57 | return ObtainFeatureIndex(localFeatureName.c_str(), featureIndex); 58 | } 59 | 60 | bool MinimalFeatureMap::GetFeatureName( 61 | UInt32 featureIndex, char *featureName, UInt32 maxNameLength) const 62 | { 63 | if (featureIndex < m_numberOfFeatures) 64 | { 65 | _snprintf_s( 66 | featureName, 67 | maxNameLength, 68 | _TRUNCATE, 69 | "%s", 70 | m_reverseFeatureMap[featureIndex].c_str()); 71 | return true; 72 | } 73 | return false; 74 | } 75 | 76 | const std::string &MinimalFeatureMap::GetFeatureName(UInt32 featureIndex) const 77 | { 78 | return m_reverseFeatureMap[featureIndex]; 79 | } 80 | 81 | UInt32 MinimalFeatureMap::GetNumberOfFeatures() const 82 | { 83 | return m_numberOfFeatures; 84 | } 85 | -------------------------------------------------------------------------------- /src/NeuralTreeEvaluator.Library/src/MinimalFeatureMap.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | 8 | #include 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | class MinimalFeatureMap : public DynamicRank::IFeatureMap { 15 | private: 16 | std::map m_featureMap; 17 | std::vector m_reverseFeatureMap; 18 | UInt32 m_numberOfFeatures; 19 | 20 | public: 21 | MinimalFeatureMap(); 22 | ~MinimalFeatureMap(); 23 | 24 | // Get the index of a feature, or return false if there is no such feature. 25 | bool GetExistingFeatureIndex(const char *featureName, 26 | UInt32 &featureIndex) const; 27 | 28 | // Convert the FeatureName to the FeatureIndex. 29 | bool ObtainFeatureIndex(const char *featureName, UInt32 &featureIndex); 30 | 31 | // Convert the FeatureName to the FeatureIndex. 32 | bool ObtainFeatureIndex(const SIZED_STRING &featureName, 33 | UInt32 &featureIndex); 34 | 35 | // Convert the FeatureIndex to the FeatureName. 36 | bool GetFeatureName(UInt32 featureIndex, char *featureName, 37 | UInt32 maxNameLength) const; 38 | const std::string &GetFeatureName(UInt32 featureIndex) const; 39 | 40 | // The number of features within the map. 41 | UInt32 GetNumberOfFeatures() const; 42 | }; 43 | -------------------------------------------------------------------------------- /src/NeuralTreeEvaluator.Library/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.15) 2 | 3 | set(PROJECT_NAME NeuralTreeEvaluatorTest) 4 | 5 | Project(${PROJECT_NAME}) 6 | 7 | link_directories( 8 | ${CMAKE_CURRENT_SOURCE_DIR}/../../DynamicRank.FreeForm.Library/lib/Shared 9 | ${CMAKE_CURRENT_SOURCE_DIR}/../../DynamicRank.FreeForm.Library/lib/Expression 10 | ${CMAKE_CURRENT_SOURCE_DIR}/../../DynamicRank.FreeForm.Library/lib/Backend/llvm 11 | ${CMAKE_CURRENT_SOURCE_DIR}/../../DynamicRank.FreeForm.Library/lib/Transform 12 | ${CMAKE_CURRENT_SOURCE_DIR}/../../DynamicRank.FreeForm.Library/lib/Parse/SExpression/lib 13 | ${CMAKE_CURRENT_SOURCE_DIR}/../../DynamicRank.FreeForm.Library/lib/External 14 | ${CMAKE_CURRENT_SOURCE_DIR}/../../NeuralTree.Library/src 15 | ${CMAKE_CURRENT_SOURCE_DIR}/../src 16 | ) 17 | 18 | add_executable(${PROJECT_NAME} 19 | ${CMAKE_CURRENT_SOURCE_DIR}/NeuralTreeEvaluatorLibTest.cpp 20 | ) 21 | 22 | target_include_directories(${PROJECT_NAME} PRIVATE 23 | ${CMAKE_CURRENT_SOURCE_DIR} 24 | ${CMAKE_CURRENT_SOURCE_DIR}/../inc 25 | ${CMAKE_CURRENT_SOURCE_DIR}/../../NeuralTree.Library/inc 26 | ${CMAKE_CURRENT_SOURCE_DIR}/../../DynamicRank.FreeForm.Library/lib/Shared 27 | ${CMAKE_CURRENT_SOURCE_DIR}/../../DynamicRank.FreeForm.Library/lib/Expression 28 | ${CMAKE_CURRENT_SOURCE_DIR}/../../DynamicRank.FreeForm.Library/lib/Backend/llvm 29 | ${CMAKE_CURRENT_SOURCE_DIR}/../../DynamicRank.FreeForm.Library/lib/Transform 30 | ${CMAKE_CURRENT_SOURCE_DIR}/../../DynamicRank.FreeForm.Library/lib/Parse/SExpression/inc 31 | ${CMAKE_CURRENT_SOURCE_DIR}/../../DynamicRank.FreeForm.Library/lib/External 32 | ) 33 | 34 | target_link_libraries(${PROJECT_NAME} 35 | -Wl,--no-as-needed 36 | -Wl,--start-group 37 | -lpthread 38 | -lz 39 | -ldl 40 | -ltinfo 41 | DRNeuralTreeLibrary 42 | DRFreeFormLlvmBackendLibrary 43 | DRFreeFormSharedLibrary 44 | DRFreeFormExpressionLibrary 45 | DRFreeFormTransformLibrary 46 | DRFreeFormSExpressionLibrary 47 | DRFreeFormLibrary 48 | NeuralTreeEvaluatorLibrary 49 | ${Boost_LIBRARIES} 50 | ${LLVM_LIB} 51 | -Wl,--end-group 52 | ) 53 | -------------------------------------------------------------------------------- /src/NeuralTreeEvaluator.Library/test/NeuralTreeEvaluatorLibTest.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | #include 8 | 9 | #include "InputComputation.h" 10 | #include "InputExtraction.h" 11 | #include "common.h" 12 | #include "log.h" 13 | using namespace DynamicRank; 14 | 15 | void EvaluateBasicInputs(string fea_spec_path) { 16 | DynamicRank::Config *config = DynamicRank::Config::GetRawConfiguration( 17 | Common::LoadStringFromFile(fea_spec_path.c_str())); 18 | assert(config->DoesSectionExist("Input:3")); 19 | InputExtractor *extractor = InputExtractorCreateFromInputStr( 20 | Common::LoadStringFromFile(fea_spec_path.c_str())); 21 | assert(extractor != NULL); 22 | } 23 | 24 | void EvaluateFreeFormHelper(InputExtractor *extractor) { 25 | MinimalFeatureMap *featureMap = InputExtractorGetFeatureMap(extractor); 26 | int inputCount = InputExtractorGetInputCount(extractor); 27 | Log::Info("Input count: %s", to_string(inputCount).c_str()); 28 | assert(inputCount == 1); 29 | 30 | const DynamicRank::NeuralInput *input = InputExtractorGetInput(extractor, 0); 31 | UInt32 features[2]; 32 | features[0] = 0; 33 | features[1] = 0; 34 | double value = InputEvaluate(input, features); 35 | Log::Info("Evaluated value: %s", to_string(value).c_str()); 36 | assert(value == 1); 37 | 38 | features[0] = 1; 39 | value = InputEvaluate(input, features); 40 | Log::Info("Evaluated value: %s", to_string(value).c_str()); 41 | assert(value == 0); 42 | } 43 | 44 | void EvaluateStandaloneFreeFormV2() { 45 | InputExtractor *extractor = 46 | InputExtractorCreateFromFreeformV2("(if (== Foo Bar) 1 0)"); 47 | EvaluateFreeFormHelper(extractor); 48 | } 49 | 50 | int main() { 51 | EvaluateBasicInputs("./data/TrainInputIni"); 52 | EvaluateStandaloneFreeFormV2(); 53 | return 0; 54 | } 55 | -------------------------------------------------------------------------------- /src/NeuralTreeEvaluator.Library/test/data/TrainInputIni: -------------------------------------------------------------------------------- 1 | 2 | [Input:1] 3 | Line1=(* f0 f1) 4 | Transform=FreeForm2 5 | Slope=1 6 | Intercept=0 7 | 8 | [Input:2] 9 | Line1=(* f1 f2) 10 | Transform=FreeForm2 11 | Slope=1 12 | Intercept=0 13 | 14 | [Input:3] 15 | Line1=(* f3 f4) 16 | Transform=FreeForm2 17 | Slope=1 18 | Intercept=0 19 | -------------------------------------------------------------------------------- /src/TransformProcessor/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.15.0 FATAL_ERROR) 2 | 3 | project ("TransformProcessor") 4 | 5 | set(CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS} -fpermissive") 6 | 7 | add_subdirectory (${CMAKE_CURRENT_SOURCE_DIR}/src) 8 | add_subdirectory (${CMAKE_CURRENT_SOURCE_DIR}/test) -------------------------------------------------------------------------------- /src/TransformProcessor/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.15) 2 | 3 | set(PROJECT_NAME TransformProcessor) 4 | 5 | project(${PROJECT_NAME}) 6 | 7 | set(LIBRARY_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/../../) 8 | 9 | add_library(_transform SHARED 10 | IniFileParserInterface.h 11 | FeatureEvaluator.h 12 | TransformProcessorFeatureMap.h 13 | IniFileParserInterface.cpp 14 | FeatureEvaluator.cpp 15 | TransformProcessorFeatureMap.cpp 16 | TransformProcessor.h 17 | TransformProcessor.cpp 18 | ) 19 | 20 | target_include_directories(_transform PUBLIC 21 | ${CMAKE_CURRENT_SOURCE_DIR} 22 | ${CMAKE_CURRENT_SOURCE_DIR}/../../NeuralTreeEvaluator.Library/inc 23 | ${CMAKE_CURRENT_SOURCE_DIR}/../../NeuralTree.Library/inc 24 | ) 25 | 26 | target_link_libraries(_transform 27 | -Wl,--no-as-needed 28 | -Wl,--start-group 29 | -lpthread 30 | -lz 31 | -ldl 32 | -ltinfo 33 | DRNeuralTreeLibrary 34 | DRFreeFormLlvmBackendLibrary 35 | DRFreeFormSharedLibrary 36 | DRFreeFormExpressionLibrary 37 | DRFreeFormTransformLibrary 38 | DRFreeFormSExpressionLibrary 39 | DRFreeFormLibrary 40 | NeuralTreeEvaluatorLibrary 41 | ${Boost_LIBRARIES} 42 | ${LLVM_LIB} 43 | -Wl,--end-group 44 | ) 45 | -------------------------------------------------------------------------------- /src/TransformProcessor/src/FeatureEvaluator.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "FeatureEvaluator.h" 7 | 8 | #include 9 | 10 | FeatureEvaluator::FeatureEvaluator(Parser *parser_interface, uint32_t id) { 11 | parser_interface_ = parser_interface; 12 | input_id_ = id; 13 | inputptr_ = parser_interface_->GetInput((uint32_t)this->input_id_); 14 | } 15 | 16 | vector FeatureEvaluator::GetRequiredRawFeatureIndices() { 17 | vector features(parser_interface_->GetFeatureCount()); 18 | uint32_t feature_count; 19 | this->parser_interface_->GetInputFeatures( 20 | this->inputptr_, reinterpret_cast(features.data()), 21 | (uint32_t)features.size(), &feature_count); 22 | std::vector subvector(features.begin(), 23 | features.begin() + feature_count); 24 | sort(subvector.begin(), subvector.end()); 25 | subvector.erase(unique(subvector.begin(), subvector.end()), subvector.end()); 26 | return subvector; 27 | } 28 | 29 | bool FeatureEvaluator::IsRawFeatureEvaluator() { 30 | return this->parser_interface_->IsCopyInput(this->inputptr_); 31 | } 32 | 33 | double FeatureEvaluator::Evaluate(uint32_t *input) { 34 | uint32_t *input1 = input; 35 | return this->parser_interface_->EvaluateInput(this->inputptr_, input1); 36 | } -------------------------------------------------------------------------------- /src/TransformProcessor/src/FeatureEvaluator.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | #include 8 | 9 | #include "Parser.h" 10 | 11 | using namespace std; 12 | 13 | class FeatureEvaluator { 14 | public: 15 | FeatureEvaluator(Parser *parser_interface, uint32_t input_id); 16 | vector GetRequiredRawFeatureIndices(); 17 | double Evaluate(uint32_t *input); 18 | bool IsRawFeatureEvaluator(); 19 | 20 | private: 21 | uint32_t input_id_; 22 | Parser *parser_interface_; 23 | void *inputptr_; 24 | }; -------------------------------------------------------------------------------- /src/TransformProcessor/src/FeatureEvaluatorExtendedInfo.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | #include 8 | 9 | #include "FeatureEvaluator.h" 10 | #include "IniFileParserInterface.h" 11 | #include "TransformProcessorFeatureMap.h" 12 | 13 | using namespace std; 14 | 15 | class FeatureEvaluatorExtendedInfo { 16 | public: 17 | FeatureEvaluator *feature_evaluator_; 18 | FeatureMap *featuremap_; 19 | vector required_feature_mapped_indexes_; 20 | int index_ = -1; 21 | bool israw_ = false; 22 | }; 23 | -------------------------------------------------------------------------------- /src/TransformProcessor/src/IniFileParserInterface.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for license information. 4 | */ 5 | #pragma once 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include "Parser.h" 16 | #include "FeatureEvaluator.h" 17 | #include "TransformProcessorFeatureMap.h" 18 | #include 19 | #include 20 | #include 21 | 22 | using namespace std; 23 | 24 | class IniFileParserInterface : public Parser 25 | { 26 | public: 27 | static IniFileParserInterface *CreateFromInputStr(const string &str); 28 | static IniFileParserInterface *Createfromfreeform2(string freeform); 29 | static void C_GetInputFeatures(void *input_ptr, uint32_t *features, uint32_t size_of_features, uint32_t *feature_count); 30 | static double C_EvaluateInput(void *input_ptr, uint32_t *input1); 31 | static bool C_IsCopyInput(void *p_input); 32 | static void *C_GetInput(void *input_extractor_ptr, uint32_t index); 33 | static uint32_t C_GetInputCount(void *input_extractor_ptr); 34 | static bool C_GetInputName(void *input_extractor_ptr, uint32_t feature_index, char *buffer, uint32_t size_of_buffer, uint32_t *result_length); 35 | static bool C_GetFeatureName(void *input_extractor_ptr, uint32_t feature_index, char *buffer, uint32_t size_of_buffer, uint32_t *result_length); 36 | static uint32_t C_GetFeatureCount(void *input_extractor_ptr); 37 | static void *GetFeatureMap(void *feature_map_ptr); 38 | void *GetInput(uint32_t index); 39 | string StringFetch(bool (*fp)(void *input_extractor_ptr, uint32_t feature_index, char *buffer, uint32_t size_of_buffer, uint32_t *result_length), uint32_t id, void *object_ptr); 40 | string GetInputName(uint32_t input_id); 41 | string GetFeatureName(uint32_t input_id); 42 | void GetInputFeatures(void *input_ptr, uint32_t *features, uint32_t size_of_features, uint32_t *feature_count); 43 | uint32_t GetInputCount(); 44 | double EvaluateInput(void *input_ptr, uint32_t *input1); 45 | bool IsCopyInput(void *p_input); 46 | uint32_t GetFeatureCount(); 47 | ~IniFileParserInterface(void); 48 | 49 | private: 50 | void *input_extractor_ptr_; 51 | void *feature_map_ptr_; 52 | IniFileParserInterface(void *p_input_extractor); 53 | }; 54 | -------------------------------------------------------------------------------- /src/TransformProcessor/src/Parser.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | using namespace std; 17 | 18 | class Parser { 19 | public: 20 | virtual void *GetInput(uint32_t index) = 0; 21 | virtual string GetInputName(uint32_t input_id) = 0; 22 | virtual string GetFeatureName(uint32_t input_id) = 0; 23 | virtual uint32_t GetInputCount() = 0; 24 | virtual uint32_t GetFeatureCount() = 0; 25 | virtual void GetInputFeatures(void *input_ptr, uint32_t *features, 26 | uint32_t size_of_features, 27 | uint32_t *feature_count) = 0; 28 | virtual bool IsCopyInput(void *p_input) = 0; 29 | virtual double EvaluateInput(void *input_ptr, uint32_t *input1) = 0; 30 | }; 31 | -------------------------------------------------------------------------------- /src/TransformProcessor/src/TransformProcessor.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | #include 8 | #include 9 | #include 10 | 11 | #include "FeatureEvaluator.h" 12 | #include "FeatureEvaluatorExtendedInfo.h" 13 | #include "IniFileParserInterface.h" 14 | #include "TransformProcessorFeatureMap.h" 15 | 16 | using namespace std; 17 | using std::string; 18 | 19 | // Object-oriented packaging of TransformProcessor 20 | class TransformedData { 21 | private: 22 | double _label; 23 | vector > _sparse_features; 24 | 25 | public: 26 | TransformedData(double label, vector > sparse_features) 27 | : _label(label), _sparse_features(sparse_features) {} 28 | double Label() { return this->_label; }; 29 | vector > Features() { return this->_sparse_features; }; 30 | string ToString(int precision = 5); 31 | static vector ToString(vector dataset, 32 | int precision = 5); 33 | }; 34 | 35 | class TransformProcessor { 36 | private: 37 | int _label_id; 38 | std::map _column_index_mapping; 39 | vector _feature_evaluator_list; 40 | unsigned int _feature_index_start; 41 | IniFileParserInterface *_from_input_str = nullptr; 42 | static bool CloseToZero(double val); 43 | static double ConvertStrToDouble(string val); 44 | static const double EPS; 45 | 46 | public: 47 | TransformProcessor(const string &transform_str, const std::string &header_str, 48 | int label_id); 49 | void Parse(const char *str, vector *out_feature_strs, 50 | double *out_label, string delimiter = "\t"); 51 | void Apply(vector *input_row, 52 | vector > *out_features); 53 | vector ApplyForFile(string data_path); 54 | int GetFeatureCount(); 55 | }; 56 | -------------------------------------------------------------------------------- /src/TransformProcessor/src/TransformProcessorFeatureMap.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "TransformProcessorFeatureMap.h" 7 | 8 | FeatureMap::FeatureMap(Parser *parser_interface) { 9 | parser_interface_ = parser_interface; 10 | } 11 | 12 | string FeatureMap::GetRawFeatureName(int index) { 13 | return this->parser_interface_->GetFeatureName(index); 14 | } -------------------------------------------------------------------------------- /src/TransformProcessor/src/TransformProcessorFeatureMap.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #pragma once 7 | #include 8 | #include 9 | 10 | #include "Parser.h" 11 | 12 | using namespace std; 13 | 14 | class FeatureMap { 15 | public: 16 | FeatureMap(Parser *parser_interface); 17 | string GetRawFeatureName(int index); 18 | ~FeatureMap(void); 19 | 20 | private: 21 | Parser *parser_interface_; 22 | }; -------------------------------------------------------------------------------- /src/TransformProcessor/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.15) 2 | 3 | set(PROJECT_NAME Tests) 4 | 5 | project(${PROJECT_NAME}) 6 | 7 | link_directories( 8 | ${CMAKE_CURRENT_SOURCE_DIR}/../../NeuralTree.Library/src 9 | ${CMAKE_CURRENT_SOURCE_DIR}/../../NeuralTreeEvaluator.Library/src 10 | ${CMAKE_CURRENT_SOURCE_DIR}/../../TransformProcessor/TransformProcessor 11 | ) 12 | 13 | add_executable(${PROJECT_NAME} 14 | Tests.h 15 | Tests.cpp 16 | ) 17 | 18 | target_include_directories(${PROJECT_NAME} PRIVATE 19 | ${CMAKE_CURRENT_SOURCE_DIR} 20 | ${CMAKE_CURRENT_SOURCE_DIR}/../../NeuralTreeEvaluator.Library/inc 21 | ${CMAKE_CURRENT_SOURCE_DIR}/../../NeuralTree.Library/inc 22 | ${CMAKE_CURRENT_SOURCE_DIR}/../../TransformProcessor/TransformProcessor 23 | ) 24 | 25 | 26 | target_link_libraries(${PROJECT_NAME} 27 | -Wl,--no-as-needed 28 | -Wl,--start-group 29 | -lpthread 30 | -lz 31 | -ldl 32 | -ltinfo 33 | DRNeuralTreeLibrary 34 | DRFreeFormLlvmBackendLibrary 35 | DRFreeFormSharedLibrary 36 | DRFreeFormExpressionLibrary 37 | DRFreeFormTransformLibrary 38 | DRFreeFormSExpressionLibrary 39 | DRFreeFormLibrary 40 | NeuralTreeEvaluatorLibrary 41 | _transform 42 | ${Boost_LIBRARIES} 43 | ${LLVM_LIB} 44 | -Wl,--end-group 45 | ) 46 | 47 | add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD 48 | COMMAND ${CMAKE_COMMAND} -E copy_directory 49 | ${CMAKE_CURRENT_SOURCE_DIR}/data/ $/TestData) 50 | -------------------------------------------------------------------------------- /src/TransformProcessor/test/Tests.cpp: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for 4 | * license information. 5 | */ 6 | #include "Tests.h" 7 | 8 | #include 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "common.h" 17 | #include "log.h" 18 | 19 | using namespace std; 20 | using namespace DynamicRank; 21 | 22 | void TestGetFeatureCount() { 23 | std::unique_ptr transform; 24 | transform.reset(new TransformProcessor( 25 | Common::LoadStringFromFile( 26 | "./TestData/integration/SmoothedTrainInputIni"), 27 | Common::LoadStringFromFile("./TestData/integration/Header.tsv", 1), 7)); 28 | assert(transform->GetFeatureCount() == 4865); 29 | } 30 | 31 | void TestTransformE2E() { 32 | string test_data_path = "./TestData/integration/"; 33 | string input_data_path = test_data_path + "Input.tsv"; 34 | string input_ini_path = test_data_path + "SmoothedTrainInputIni"; 35 | string input_head_path = test_data_path + "Header.tsv"; 36 | string expected_output_path = test_data_path + "ExpectedOutput.txt"; 37 | string actual_output_path = test_data_path + "Actual.txt"; 38 | int label_id = 7; 39 | 40 | vector transformed_data; 41 | std::unique_ptr transform; 42 | transform.reset(new TransformProcessor( 43 | Common::LoadStringFromFile(input_ini_path.c_str()), 44 | Common::LoadStringFromFile(input_head_path.c_str(), 1), label_id)); 45 | transformed_data = 46 | TransformedData::ToString(transform->ApplyForFile(input_data_path)); 47 | ofstream output_fout(actual_output_path.c_str()); 48 | for (auto str : transformed_data) output_fout << str << endl; 49 | output_fout.close(); 50 | 51 | ifstream expectin(expected_output_path.c_str()); 52 | ifstream actualin(actual_output_path.c_str()); 53 | string expect_line; 54 | string actual_line; 55 | std::getline(expectin, expect_line); 56 | std::getline(actualin, actual_line); 57 | if (expect_line.compare(actual_line) != 0) { 58 | Log::Fatal("Failed to compare expect output and actual output!"); 59 | } else { 60 | Log::Info("All passed"); 61 | } 62 | } 63 | 64 | int main() { 65 | TestGetFeatureCount(); 66 | TestTransformE2E(); 67 | return 0; 68 | } -------------------------------------------------------------------------------- /src/TransformProcessor/test/Tests.h: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright (c) 2021 Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. See LICENSE file in the project root for license information. 4 | */ 5 | #pragma once 6 | #include 7 | #include 8 | #include "TransformProcessor.h" 9 | 10 | using namespace std; 11 | using std::string; 12 | -------------------------------------------------------------------------------- /tests/data/transform_rank_data/freeform_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/lightgbm-transform/49fa0d1e35e4a8543f4baec9a7c23cb9bb312e71/tests/data/transform_rank_data/freeform_config.json -------------------------------------------------------------------------------- /tests/data/transform_simple_data/freeform_config.json: -------------------------------------------------------------------------------- 1 | {"className": "FreeForm2Parser", "transform": "[Input:1]\nLine1=(+ feature_1 feature_2)\nTransform=FreeForm2\nSlope=1\nIntercept=0\n\n[Input:2]\nTransform=FreeForm2\nLine1=(* feature_1 feature_3)\n\n[Input:3]\nTransform=FreeForm2\nLine1=(max feature_6 feature_7)\n\n[Input:4]\nTransform=Linear\nName=feature_8\nIntercept=0\nSlope=1\n\n[Input:5]\nTransform=Linear\nName=feature_9\nIntercept=0\nSlope=1\n", "header": "feature_0\tfeature_1\tfeature_2\tfeature_3\tfeature_4\tfeature_5\tfeature_6\tfeature_7\tfeature_8\tfeature_9\tlabels"} --------------------------------------------------------------------------------