├── .github └── workflows │ ├── release.yml │ └── test.yml ├── .gitignore ├── CONTRIBUTING.md ├── KNOWN_ISSUES.md ├── LICENSE.txt ├── NOTICE.txt ├── README.md ├── build.gradle ├── config └── checkstyle │ ├── checkstyle.xml │ └── suppressions.xml ├── demo └── xgboost-demo │ ├── README.md │ ├── featmap.txt │ ├── xgb-model.json │ ├── xgb.py │ └── xgboost.txt ├── docs ├── Makefile ├── advanced-functionality.rst ├── building-features.rst ├── conf.py ├── core-concepts.rst ├── faq.rst ├── feature-engineering.rst ├── fits-in.rst ├── index.rst ├── logging-features.rst ├── make.bat ├── searching-with-your-model.rst ├── training-models.rst └── x-pack.rst ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── licenses ├── antlr4-runtime-4.5.1-1.jar.sha1 ├── antlr4-runtime-LICENSE.txt ├── antlr4-runtime-NOTICE.txt ├── asm-5.0.4.jar.sha1 ├── asm-LICENSE.txt ├── asm-NOTICE.txt ├── asm-commons-5.0.4.jar.sha1 ├── asm-tree-5.0.4.jar.sha1 ├── lucene-LICENSE.txt ├── lucene-NOTICE.txt └── lucene-expressions-6.6.0.jar.sha1 ├── publish └── build.gradle ├── sample_models ├── coord_ascent.txt ├── lambdaMART.txt ├── linRegression.txt ├── mart.txt └── randomForest.txt ├── scripts ├── xgboost_model_fixr.py └── xgboost_model_fixr_test.py ├── settings.gradle └── src ├── javaRestTest └── java │ └── com │ └── o19s │ └── es │ └── ltr │ ├── NodeSettingsIT.java │ ├── action │ ├── AddFeaturesToSetActionIT.java │ ├── BaseIntegrationTest.java │ ├── LTRStatsActionIT.java │ ├── ListStoresActionIT.java │ └── ValidatingFeatureStoreActionIT.java │ ├── logging │ └── LoggingIT.java │ └── query │ └── StoredLtrQueryIT.java ├── main ├── java │ └── com │ │ └── o19s │ │ └── es │ │ ├── explore │ │ ├── ExplorerQuery.java │ │ ├── ExplorerQueryBuilder.java │ │ ├── ExplorerScorer.java │ │ ├── PostingsExplorerQuery.java │ │ └── StatisticsHelper.java │ │ ├── ltr │ │ ├── LtrQueryContext.java │ │ ├── LtrQueryParserPlugin.java │ │ ├── action │ │ │ ├── AddFeaturesToSetAction.java │ │ │ ├── CachesStatsAction.java │ │ │ ├── ClearCachesAction.java │ │ │ ├── CreateModelFromSetAction.java │ │ │ ├── FeatureStoreAction.java │ │ │ ├── LTRStatsAction.java │ │ │ ├── ListStoresAction.java │ │ │ ├── TransportAddFeatureToSetAction.java │ │ │ ├── TransportCacheStatsAction.java │ │ │ ├── TransportClearCachesAction.java │ │ │ ├── TransportCreateModelFromSetAction.java │ │ │ ├── TransportFeatureStoreAction.java │ │ │ ├── TransportLTRStatsAction.java │ │ │ └── TransportListStoresAction.java │ │ ├── feature │ │ │ ├── Feature.java │ │ │ ├── FeatureSet.java │ │ │ ├── FeatureValidation.java │ │ │ ├── LtrModel.java │ │ │ ├── PrebuiltFeature.java │ │ │ ├── PrebuiltFeatureSet.java │ │ │ ├── PrebuiltLtrModel.java │ │ │ └── store │ │ │ │ ├── CompiledLtrModel.java │ │ │ │ ├── ExtraLoggingSupplier.java │ │ │ │ ├── FeatureNormDefinition.java │ │ │ │ ├── FeatureStore.java │ │ │ │ ├── FeatureSupplier.java │ │ │ │ ├── MinMaxFeatureNormDefinition.java │ │ │ │ ├── OptimizedFeatureSet.java │ │ │ │ ├── PrecompiledExpressionFeature.java │ │ │ │ ├── PrecompiledTemplateFeature.java │ │ │ │ ├── ScriptFeature.java │ │ │ │ ├── StandardFeatureNormDefinition.java │ │ │ │ ├── StorableElement.java │ │ │ │ ├── StoredFeature.java │ │ │ │ ├── StoredFeatureNormalizers.java │ │ │ │ ├── StoredFeatureSet.java │ │ │ │ ├── StoredLtrModel.java │ │ │ │ └── index │ │ │ │ ├── CachedFeatureStore.java │ │ │ │ ├── Caches.java │ │ │ │ └── IndexFeatureStore.java │ │ ├── logging │ │ │ ├── LoggingFetchSubPhase.java │ │ │ └── LoggingSearchExtBuilder.java │ │ ├── query │ │ │ ├── DerivedExpressionQuery.java │ │ │ ├── LtrQueryBuilder.java │ │ │ ├── LtrRewritableQuery.java │ │ │ ├── LtrRewriteContext.java │ │ │ ├── NoopScorer.java │ │ │ ├── RankerQuery.java │ │ │ ├── StoredLtrQueryBuilder.java │ │ │ └── ValidatingLtrQueryBuilder.java │ │ ├── ranker │ │ │ ├── ArrayFeatureVector.java │ │ │ ├── DenseFeatureVector.java │ │ │ ├── DenseLtrRanker.java │ │ │ ├── LogLtrRanker.java │ │ │ ├── LtrRanker.java │ │ │ ├── NullRanker.java │ │ │ ├── SparseFeatureVector.java │ │ │ ├── SparseLtrRanker.java │ │ │ ├── dectree │ │ │ │ └── NaiveAdditiveDecisionTree.java │ │ │ ├── linear │ │ │ │ └── LinearRanker.java │ │ │ ├── normalizer │ │ │ │ ├── FeatureNormalizingRanker.java │ │ │ │ ├── MinMaxFeatureNormalizer.java │ │ │ │ ├── Normalizer.java │ │ │ │ ├── Normalizers.java │ │ │ │ └── StandardFeatureNormalizer.java │ │ │ ├── parser │ │ │ │ ├── LinearRankerParser.java │ │ │ │ ├── LtrRankerParser.java │ │ │ │ ├── LtrRankerParserFactory.java │ │ │ │ ├── XGBoostJsonParser.java │ │ │ │ └── XGBoostRawJsonParser.java │ │ │ └── ranklib │ │ │ │ ├── DenseProgramaticDataPoint.java │ │ │ │ ├── RankLibScriptEngine.java │ │ │ │ ├── RanklibModelParser.java │ │ │ │ └── RanklibRanker.java │ │ ├── rest │ │ │ ├── AutoDetectParser.java │ │ │ ├── FeatureStoreBaseRestHandler.java │ │ │ ├── RestAddFeatureToSet.java │ │ │ ├── RestCreateModelFromSet.java │ │ │ ├── RestFeatureManager.java │ │ │ ├── RestFeatureStoreCaches.java │ │ │ ├── RestLTRStats.java │ │ │ ├── RestSearchStoreElements.java │ │ │ └── RestStoreManager.java │ │ ├── stats │ │ │ ├── LTRStat.java │ │ │ ├── LTRStats.java │ │ │ ├── StatName.java │ │ │ └── suppliers │ │ │ │ ├── CacheStatsOnNodeSupplier.java │ │ │ │ ├── PluginHealthStatusSupplier.java │ │ │ │ └── StoreStatsSupplier.java │ │ └── utils │ │ │ ├── AbstractQueryBuilderUtils.java │ │ │ ├── CheckedBiFunction.java │ │ │ ├── FeatureStoreLoader.java │ │ │ ├── Scripting.java │ │ │ └── Suppliers.java │ │ ├── template │ │ └── mustache │ │ │ ├── CustomMustacheFactory.java │ │ │ ├── CustomReflectionObjectHandler.java │ │ │ └── MustacheUtils.java │ │ └── termstat │ │ ├── TermStatQuery.java │ │ ├── TermStatQueryBuilder.java │ │ ├── TermStatScorer.java │ │ └── TermStatSupplier.java ├── plugin-metadata │ └── plugin-security.policy └── resources │ └── com │ └── o19s │ └── es │ └── ltr │ └── feature │ └── store │ └── index │ ├── fstore-index-analysis.json │ └── fstore-index-mapping.json ├── test ├── java │ └── com │ │ └── o19s │ │ └── es │ │ ├── TestExpressionsPlugin.java │ │ ├── explore │ │ ├── ExplorerQueryBuilderTests.java │ │ ├── ExplorerQueryTests.java │ │ └── StatisticsHelperTests.java │ │ ├── ltr │ │ ├── LtrQueryContextTests.java │ │ ├── LtrTestUtils.java │ │ ├── ShardStatsIT.java │ │ ├── action │ │ │ └── TransportLTRStatsActionTests.java │ │ ├── feature │ │ │ └── store │ │ │ │ ├── ExtraLoggingSupplierTests.java │ │ │ │ ├── FeatureSupplierTests.java │ │ │ │ ├── MemStore.java │ │ │ │ ├── StoredFeatureParserTests.java │ │ │ │ ├── StoredFeatureSetParserTests.java │ │ │ │ ├── StoredFeatureSetTests.java │ │ │ │ ├── StoredLtrModelParserTests.java │ │ │ │ └── index │ │ │ │ ├── CachedFeatureStoreTests.java │ │ │ │ └── IndexFeatureStoreTests.java │ │ ├── logging │ │ │ ├── LoggingFetchSubPhaseTests.java │ │ │ └── LoggingSearchExtBuilderTests.java │ │ ├── query │ │ │ ├── LtrQueryBuilderTests.java │ │ │ ├── LtrQueryTests.java │ │ │ ├── StoredLtrQueryBuilderTests.java │ │ │ └── ValidatingLtrQueryBuilderTests.java │ │ ├── ranker │ │ │ ├── DenseFeatureVectorTests.java │ │ │ ├── DenseLtrRankerTests.java │ │ │ ├── LogLtrRankerTests.java │ │ │ ├── SparseFeatureVectorTests.java │ │ │ ├── dectree │ │ │ │ └── NaiveAdditiveDecisionTreeTests.java │ │ │ ├── linear │ │ │ │ └── LinearRankerTests.java │ │ │ ├── normalizer │ │ │ │ └── NormalizersTests.java │ │ │ └── parser │ │ │ │ ├── LinearRankerParserTests.java │ │ │ │ ├── LtrRankerParserFactoryTests.java │ │ │ │ ├── XGBoostJsonParserTests.java │ │ │ │ └── XGBoostRawJsonParserTests.java │ │ ├── rest │ │ │ └── FeaturesParserTests.java │ │ └── stats │ │ │ ├── LTRStatTests.java │ │ │ ├── LTRStatsTests.java │ │ │ └── suppliers │ │ │ ├── CacheStatsOnNodeSupplierTests.java │ │ │ └── PluginHealthStatusSupplierTests.java │ │ └── termstat │ │ ├── TermStatQueryBuilderTests.java │ │ └── TermStatQueryTests.java └── resources │ ├── com │ └── o19s │ │ └── es │ │ └── ltr │ │ └── ranker │ │ └── dectree │ │ └── simple_tree.txt │ └── models │ └── xgboost-wmf.json └── yamlRestTest ├── java └── com │ └── o19s │ └── es │ └── ltr │ └── LtrQueryClientYamlTestSuiteIT.java └── resources └── rest-api-spec ├── api ├── ltr.add_features_to_set.json ├── ltr.cache_stats.json ├── ltr.clear_cache.json ├── ltr.create_feature.json ├── ltr.create_featureset.json ├── ltr.create_model.json ├── ltr.create_model_from_set.json ├── ltr.create_store.json ├── ltr.delete_feature.json ├── ltr.delete_featureset.json ├── ltr.delete_model.json ├── ltr.delete_store.json ├── ltr.get_feature.json ├── ltr.get_featureset.json ├── ltr.get_model.json ├── ltr.get_stats.json ├── ltr.get_store.json ├── ltr.list_feature.json ├── ltr.list_featureset.json ├── ltr.list_model.json ├── ltr.list_stores.json ├── ltr.update_feature.json ├── ltr.update_featureset.json └── ltr.update_model.json └── test └── fstore ├── 10_manage.yml ├── 20_features.yml ├── 30_featuresets.yml ├── 40_models.yml ├── 50_add_features_to_set.yml ├── 60_create_model_from_set.yml ├── 70_validation.yml ├── 80_search_w_partial_models.yml └── 90_get_stats.yml /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Gradle 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle 3 | 4 | name: Tag and publish a release 5 | 6 | on: 7 | push: 8 | tags: 9 | - 'v*.*.*' 10 | 11 | jobs: 12 | release: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v2 16 | - name: Set release version Name 17 | run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV 18 | - name: Set up JDK 17.0 19 | uses: actions/setup-java@v1 20 | with: 21 | java-version: 17.0 22 | - name: Grant execute permission for gradlew 23 | run: chmod +x gradlew 24 | - name: Build with Gradle 25 | run: ./gradlew build 26 | - name: Rename build assets 27 | run: mv ./build/distributions/ltr-*.zip ./ltr-plugin-${{ env.RELEASE_VERSION }}.zip 28 | - name: Create Release 29 | id: create_release 30 | uses: actions/create-release@v1 31 | env: 32 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 33 | with: 34 | tag_name: ${{ env.RELEASE_VERSION }} 35 | release_name: Release ${{ env.RELEASE_VERSION }} 36 | draft: false 37 | prerelease: false 38 | - name: Upload Release Asset 39 | id: upload-release-asset 40 | uses: actions/upload-release-asset@v1 41 | env: 42 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 43 | with: 44 | upload_url: ${{ steps.create_release.outputs.upload_url }} 45 | asset_path: ./ltr-plugin-${{ env.RELEASE_VERSION }}.zip 46 | asset_name: ltr-plugin-${{ env.RELEASE_VERSION }}.zip 47 | asset_content_type: application/zip 48 | - name: Publish to Maven Central 49 | env: 50 | SONATYPE_REPO_URL: ${{ secrets.SONATYPE_REPO_URL }} 51 | SONATYPE_USER: ${{ secrets.SONATYPE_USER }} 52 | SONATYPE_PASS: ${{ secrets.SONATYPE_PASS }} 53 | GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} 54 | GPG_SIGNING_PASS: ${{ secrets.GPG_SIGNING_PASS }} 55 | OSSRH_TOKEN: ${{ secrets.OSSRH_TOKEN }} 56 | OSSRH_TOKEN_PASSSWORD: ${{ secrets.OSSRH_TOKEN_PASSSWORD }} 57 | 58 | run: ./gradlew publish 59 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Gradle 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle 3 | 4 | name: Run CI tests with Gradle 5 | 6 | on: 7 | push: 8 | branches: [ master, main ] 9 | pull_request: 10 | branches: [ master, main ] 11 | 12 | jobs: 13 | test: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Set up JDK 17.0 18 | uses: actions/setup-java@v1 19 | with: 20 | java-version: 17.0 21 | - name: Grant execute permission for gradlew 22 | run: chmod +x gradlew 23 | - name: Build with Gradle 24 | run: ./gradlew clean check 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Build directory for read-the-docs 2 | /docs/_build/ 3 | 4 | *.patch 5 | demo/*.jar 6 | demo/utils.pyc 7 | 8 | .envrc 9 | venv 10 | sample_judgements_wfeatures.txt 11 | ml-20m/ 12 | ml-20m.zip 13 | classes/ 14 | __pycache__/ 15 | RankLib.jar 16 | tmdb.json 17 | model.txt 18 | 19 | 20 | .idea/ 21 | .gradle/ 22 | build/ 23 | out/ 24 | *.swp 25 | *-execution-hints.log 26 | *-execution-times.log 27 | 28 | // ignore the downloaded model files in git 29 | src/test/resources/models/ 30 | 31 | // intellij 32 | *.iml 33 | *.ipr 34 | *.iws 35 | 36 | // eclipse 37 | .project 38 | .classpath 39 | eclipse-build 40 | */.project 41 | */.classpath 42 | */eclipse-build 43 | .settings 44 | !/.settings/org.eclipse.core.resources.prefs 45 | !/.settings/org.eclipse.jdt.core.prefs 46 | !/.settings/org.eclipse.jdt.ui.prefs 47 | 48 | bin/* 49 | -------------------------------------------------------------------------------- /KNOWN_ISSUES.md: -------------------------------------------------------------------------------- 1 | # Known issues 2 | 3 | ## Cache deadlock 4 | 5 | All elasticsearch versions between 5.5.3 (included) and 6.3.0 (excluded) are affected by a [bug](https://github.com/elastic/elasticsearch/pull/30461) in the internal cache that may lead to a serious deadlock that could in theory leak all the search threads. 6 | A possible workaround is to configure the cache to avoid as much as possible its eviction mechanism by setting these options in your `elasticsearch.yml` file: 7 | 8 | ``` 9 | ltr.caches.expire_after_write: 0 10 | ltr.caches.expire_after_read: 0 11 | ltr.caches.max_mem: 100mb 12 | ``` 13 | 14 | Elasticsearch versions 5.6.10, 6.3.0 and onwards should include the bugfix for this bug. 15 | 16 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | Elasticsearch Learning to Rank 2 | Copyright 2016-2017 OpenSource Connections and Wikimedia Foundation 3 | 4 | 5 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | buildscript { 2 | repositories { 3 | gradlePluginPortal() 4 | } 5 | 6 | dependencies { 7 | classpath "org.elasticsearch.gradle:build-tools:${elasticsearchVersion}" 8 | } 9 | } 10 | 11 | allprojects { 12 | group = 'com.o19s' 13 | version = "${ltrVersion}-es${elasticsearchVersion}" 14 | 15 | apply plugin: 'java' 16 | 17 | repositories { 18 | mavenCentral() 19 | jcenter() 20 | mavenLocal() 21 | gradlePluginPortal() 22 | } 23 | 24 | dependencies { 25 | implementation "org.apache.lucene:lucene-expressions:${luceneVersion}" 26 | implementation "org.antlr:antlr4-runtime:${antlrVersion}" 27 | implementation "org.ow2.asm:asm:${ow2Version}" 28 | implementation "org.ow2.asm:asm-commons:${ow2Version}" 29 | implementation "org.ow2.asm:asm-tree:${ow2Version}" 30 | implementation 'com.o19s:RankyMcRankFace:0.1.1' 31 | implementation "com.github.spullara.mustache.java:compiler:0.9.3" 32 | 33 | runtimeOnly 'org.locationtech.spatial4j:spatial4j:0.7' 34 | runtimeOnly 'org.locationtech.jts:jts-core:1.15.0' 35 | } 36 | } 37 | 38 | apply plugin: 'checkstyle' 39 | apply plugin: 'idea' 40 | apply plugin: 'elasticsearch.esplugin' 41 | apply plugin: 'elasticsearch.java-rest-test' 42 | apply plugin: 'elasticsearch.yaml-rest-test' 43 | 44 | checkstyle { 45 | configFile = rootProject.file('config/checkstyle/checkstyle.xml') 46 | } 47 | 48 | sourceSets { 49 | javaRestTest { 50 | compileClasspath += sourceSets["main"].output + sourceSets["test"].output + configurations["testRuntimeClasspath"] 51 | runtimeClasspath += output + compileClasspath 52 | } 53 | yamlRestTest { 54 | compileClasspath += sourceSets["main"].output + sourceSets["test"].output + configurations["testRuntimeClasspath"] 55 | runtimeClasspath += output + compileClasspath 56 | } 57 | } 58 | 59 | esplugin { 60 | name 'ltr' 61 | description 'Learning to Rank Query w/ RankLib Models' 62 | classname 'com.o19s.es.ltr.LtrQueryParserPlugin' 63 | // license of the plugin, may be different than the above license 64 | licenseFile = rootProject.file('LICENSE.txt') 65 | // copyright notices, may be different than the above notice 66 | noticeFile = rootProject.file('NOTICE.txt') 67 | } 68 | 69 | // In this section you declare the dependencies for your production and test code 70 | // Elasticsearch dependency is included due to the build-tools, test-framework as well 71 | repositories { 72 | mavenCentral() 73 | mavenLocal() 74 | jcenter { 75 | url "https://jcenter.bintray.com/" 76 | metadataSources{ 77 | artifact() 78 | } 79 | } 80 | } 81 | 82 | java { 83 | withJavadocJar() 84 | withSourcesJar() 85 | } 86 | 87 | // Set to false to not use elasticsearch checkstyle rules 88 | checkstyleMain.enabled = true 89 | checkstyleTest.enabled = true 90 | -------------------------------------------------------------------------------- /demo/xgboost-demo/README.md: -------------------------------------------------------------------------------- 1 | This folder contains files demonstrating how to work with xgboost. 2 | 3 | ## xgboost.txt 4 | The sample training data utilized in the main demo folder but in LibSVM format which is used by xgboost. 5 | 6 | ## featmap.txt 7 | A configuration file telling xgboost the feature names that feature ordinals refer to. 8 | 9 | ## xgb.py 10 | A bare-bones script for reading `xgboost.txt` and training a model for usage with the plugin. 11 | 12 | ## xgb-model.json 13 | Included for convenience, this is the output of xgb.py, a JSON formatted model compatible with the LTR plugin. 14 | 15 | 16 | # Notes 17 | Although there are PyPI packages for xgboost, some users will find that installation fails in their local environments. If this occurs the best workaround is to clone the xgboost repository and do a local build with the latest. 18 | 19 | ``` 20 | git clone --recursive https://github.com/dmlc/xgboost 21 | cd xgboost 22 | make -j4 23 | cd python-package 24 | python setup.py install 25 | ``` 26 | 27 | Note: If doing a global install you may need to run with root privileges, the above should work in a virtual environment. 28 | -------------------------------------------------------------------------------- /demo/xgboost-demo/featmap.txt: -------------------------------------------------------------------------------- 1 | 0 na q 2 | 1 tmdb_title q 3 | 2 tmdb_multi q 4 | -------------------------------------------------------------------------------- /demo/xgboost-demo/xgb-model.json: -------------------------------------------------------------------------------- 1 | [ { "nodeid": 0, "depth": 0, "split": "tmdb_multi", "split_condition": 11.2009, "yes": 1, "no": 2, "missing": 1, "children": [ 2 | { "nodeid": 1, "depth": 1, "split": "tmdb_title", "split_condition": 2.20631, "yes": 3, "no": 4, "missing": 3, "children": [ 3 | { "nodeid": 3, "leaf": -0.03125 }, 4 | { "nodeid": 4, "leaf": -0.25 } 5 | ]}, 6 | { "nodeid": 2, "leaf": 2.45 } 7 | ]}, { "nodeid": 0, "depth": 0, "split": "tmdb_title", "split_condition": 7.56362, "yes": 1, "no": 2, "missing": 1, "children": [ 8 | { "nodeid": 1, "depth": 1, "split": "tmdb_multi", "split_condition": 11.2009, "yes": 3, "no": 4, "missing": 3, "children": [ 9 | { "nodeid": 3, "leaf": -0.0165441 }, 10 | { "nodeid": 4, "leaf": 0.04375 } 11 | ]}, 12 | { "nodeid": 2, "leaf": 0.7 } 13 | ]}] -------------------------------------------------------------------------------- /demo/xgboost-demo/xgb.py: -------------------------------------------------------------------------------- 1 | import xgboost as xgb 2 | 3 | # Read the LibSVM labels/features 4 | dtrain = xgb.DMatrix('xgboost.txt') 5 | param = {'max_depth':2, 'eta':1, 'silent':1, 'objective':'reg:linear'} 6 | num_round = 2 7 | 8 | bst = xgb.train(param, dtrain, num_round) 9 | 10 | model = bst.get_dump(fmap='featmap.txt', dump_format='json') 11 | 12 | with open('xgb-model.json', 'w') as output: 13 | output.write('[' + ','.join(list(model)) + ']') 14 | output.close() 15 | 16 | -------------------------------------------------------------------------------- /demo/xgboost-demo/xgboost.txt: -------------------------------------------------------------------------------- 1 | 4 1:9.169009 2:22.169247 2 | 3 1:6.9903145 2:20.8937 3 | 3 1:4.972313 2:19.01041 4 | 3 1:0.0 2:14.370026 5 | 0 1:0.0 2:0.0 6 | 0 1:0.0 2:0.0 7 | 0 1:0.0 2:0.0 8 | 0 1:0.0 2:0.0 9 | 0 1:0.0 2:0.0 10 | 0 1:0.0 2:0.0 11 | 4 1:8.136918 2:18.368336 12 | 3 1:6.203464 2:17.696943 13 | 3 1:0.0 2:0.0 14 | 3 1:6.203464 2:14.961248 15 | 3 1:6.203464 2:16.599255 16 | 3 1:6.203464 2:17.696943 17 | 0 1:0.0 2:0.0 18 | 0 1:0.0 2:0.0 19 | 0 1:4.4126143 2:8.031729 20 | 4 1:0.0 2:0.0 21 | 0 1:0.0 2:0.0 22 | 0 1:0.0 2:0.0 23 | 0 1:0.0 2:0.0 24 | 0 1:0.0 2:0.0 25 | 0 1:0.0 2:0.0 26 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = python -msphinx 7 | SPHINXPROJ = ElasticsearchLearningtoRank 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/faq.rst: -------------------------------------------------------------------------------- 1 | FAQ 2 | ************************** 3 | 4 | This section contains answers to common issues that may trip up users. 5 | 6 | 7 | ============================= 8 | Negative Scores 9 | ============================= 10 | 11 | Lucene does not allow queries to have negative scores. This can be problematic if you have a raw feature that has a negative value. Unfortunately there is no easy quick fix for this. If you are working with such features, you need to make them non-negative *BEFORE* you train your model. This can be accomplished by creating normalized fields with values shifted by the mininum value or you can run the score thru a function that produces a value >= 0. 12 | 13 | ============================= 14 | I found a bug 15 | ============================= 16 | 17 | If you've been fighting with the plugin it's entirely possible you've encountered a bug. Please open an issue on the Github project and we will do our best to get it sorted. If you need general support, please see the section below as we will typically close issues that are only looking for support. 18 | 19 | 20 | ============================= 21 | I'm still stuck! 22 | ============================= 23 | 24 | We'd love to hear from you! Consider joining the `Relevance Slack Community `_ and join the #es-learn-to-rank channel. 25 | 26 | -------------------------------------------------------------------------------- /docs/fits-in.rst: -------------------------------------------------------------------------------- 1 | How does the plugin fit in? 2 | ****************************** 3 | 4 | In :doc:`core-concepts` we mentioned a couple of activities you undertake when implementing learning to rank: 5 | 6 | 1. Judgment List Development 7 | 2. Feature Engineering 8 | 3. Logging features into the judgment list to create a training set 9 | 4. Training and testing models 10 | 5. Deploying and using models when searching 11 | 12 | How does Elasticsearch LTR fit into this process? 13 | 14 | ======================= 15 | What the plugin does 16 | ======================= 17 | 18 | This plugin gives you building blocks to develop and use learning to rank models. It lets you develop query-dependent features and store them in Elasticsearch. After storing a set of features, you can log them for documents returned in search results to aid in offline model development. 19 | 20 | Then other tools take over. With a logged set of features for documents, you join data with your judgment lists you've developed on your own. You've now got a training set you can use to test/train ranking models. Using of a tool like Ranklib or XGboost, you'll hopefully arrive at a satisfactory model. 21 | 22 | With a ranking model, you turn back to the plugin. You upload the model and give it a name. The model is associated with the set of features used to generate the training data. You can then search with the model, using a custom Elasticsearch Query DSL primitive that executes the model. Hopefully this lets you deliver better search to users! 23 | 24 | ======================= 25 | What the plugin is NOT 26 | ======================= 27 | 28 | The plugin does not help with judgment list creation. This is work you must do and can be very domain specific. Wikimedia Foundation wrote a `great article `_ on how they arrive at judgment lists for people searching articles. Other domains such as e-commerce might be more conversion focused. Yet others might involve human relevance judges -- either experts at your company or mechanical turk. 29 | 30 | The plugin does not train or test models. This also happens offline in tools appropriate to the task. Instead the plugin uses models generated by XGboost and Ranklib libraries. Training and testing models is CPU intensive task that, involving data scientist supervision and offline testing. Most organizations want some data science supervision on model development. And you would not want this running in your production Elasticsearch cluster! 31 | 32 | The rest of this guide is dedicated to walking you through how the plugin works to get you there. Continue on to :doc:`building-features`. 33 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. Elasticsearch Learning to Rank documentation master file, created by 2 | sphinx-quickstart on Thu Sep 28 14:00:10 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Elasticsearch Learning to Rank: the documentation 7 | ========================================================== 8 | 9 | `Learning to Rank `_ applies machine learning to relevance ranking. The `Elasticsearch Learning to Rank plugin `_ (Elasticsearch LTR) gives you tools to train and use ranking models in Elasticsearch. This plugin powers search at places like Wikimedia Foundation and Snagajob. 10 | 11 | Get started 12 | ------------------------------- 13 | 14 | - Want a quickstart? Check out the demo in `hello-ltr `_. 15 | - Brand new to learning to rank? head to :doc:`core-concepts`. 16 | - Otherwise, start with :doc:`fits-in` 17 | 18 | Installing 19 | ----------- 20 | 21 | Pre-built versions can be found `here `_. Want a build for an ES version? Follow the instructions in the `README for building `_ or `create an issue `_. Once you've found a version compatible with your Elasticsearch, you'd run a command such as:: 22 | 23 | ./bin/elasticsearch-plugin install \ 24 | https://github.com/o19s/elasticsearch-learning-to-rank/releases/download/v1.5.4-es7.11.2/ltr-plugin-v1.5.4-es7.11.2.zip 25 | 26 | (It's expected you'll confirm some security exceptions, you can pass -b to elasticsearch-plugin to automatically install) 27 | 28 | Are you using `x-pack security `_ in your cluster? we got you covered, check :doc:`x-pack` for specific configuration details. 29 | 30 | HEEELP! 31 | ------------------------------ 32 | 33 | The plugin and guide was built by the search relevance consultants at `OpenSource Connections `_ in partnership with the Wikimedia Foundation and Snagajob Engineering. Please `contact OpenSource Connections `_ or `create an issue `_ if you have any questions or feedback. 34 | 35 | 36 | Contents 37 | ------------------------------- 38 | 39 | .. toctree:: 40 | :maxdepth: 2 41 | 42 | core-concepts 43 | fits-in 44 | building-features 45 | feature-engineering 46 | logging-features 47 | training-models 48 | searching-with-your-model 49 | x-pack 50 | advanced-functionality 51 | faq 52 | :caption: Contents: 53 | 54 | 55 | Indices and tables 56 | ================== 57 | 58 | * :ref:`genindex` 59 | * :ref:`search` 60 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=python -msphinx 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | set SPHINXPROJ=ElasticsearchLearningtoRank 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The Sphinx module was not found. Make sure you have Sphinx installed, 20 | echo.then set the SPHINXBUILD environment variable to point to the full 21 | echo.path of the 'sphinx-build' executable. Alternatively you may add the 22 | echo.Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /docs/x-pack.rst: -------------------------------------------------------------------------------- 1 | On XPack Support (Security) 2 | ***************************** 3 | 4 | X-Pack is the collection of extensions provided by elastic to enhance the capabilities of the Elastic Stack with things 5 | such as reporting, monitoring and also security. If you installed x-pack your cluster will now be protected with the 6 | security module, this will also be like this if you are using Elasticsearch through the Elastic Cloud solution. 7 | 8 | ===================================================== 9 | Setup roles and users. 10 | ===================================================== 11 | 12 | After installing the plugin, the first thing you will have to do is configure the necessary users and roles to access let 13 | the LTR plugin operate. 14 | 15 | We recommend you to create two separate roles, one for administrative task such as creating the models, updating the feature sets, etc .. 16 | and one to run the queries. 17 | 18 | For this configuration, we supose you already have identified, and created two users, one for running queries and one for doing administrative tasks. If you 19 | need help to create the users, we recommend you to check the `x-pack api documentation for user management `_. 20 | 21 | To create two roles, you can do it with these commands:: 22 | 23 | POST /_xpack/security/role/ltr_admin 24 | { 25 | "cluster": [ "ltr" ], 26 | "indices": [ 27 | { 28 | "names": [ ".ltrstore*" ], 29 | "privileges": [ "all" ], 30 | } 31 | ] 32 | } 33 | 34 | POST /_xpack/security/role/ltr_query 35 | { 36 | "cluster": [ "ltr" ], 37 | "indices": [ 38 | { 39 | "names": [ ".ltrstore*" ], 40 | "privileges": [ "read" ], 41 | } 42 | ] 43 | } 44 | 45 | the first one will allow the users to perform all the operations while the last one will only allow read operations. 46 | 47 | Once the roles are defined, the last step will be to attach this roles to existing users, for this documentation we will suppose two users, ltr_admin and ltr_user. The commands to set the roles are:: 48 | 49 | POST /_xpack/security/role_mapping/ltr_admins 50 | { 51 | "roles": [ "ltr_admin" ], 52 | "rules": { 53 | "field" : { "username" : [ "ltr_admin01", "ltr_admin02" ] } 54 | }, 55 | "metadata" : { 56 | "version" : 1 57 | } 58 | } 59 | 60 | POST /_xpack/security/role_mapping/ltr_users 61 | { 62 | "roles": [ "ltr_query" ], 63 | "rules": { 64 | "field" : { "username" : [ "ltr_user01", "ltr_user02" ] } 65 | }, 66 | "metadata" : { 67 | "version" : 1 68 | } 69 | } 70 | 71 | After this two steps, your plugin will be fully functional in your x-pack protected cluster. 72 | 73 | For more in deep information on how to define roles, we recommend you to check the elastic `x-pack api documentation `_. 74 | 75 | 76 | ===================================================== 77 | Considerations 78 | ===================================================== 79 | 80 | The read access to models via the sltr query is not strictly gated by x-pack. The access will only be checked if the model needs 81 | to be loaded, however If the model is already in the cache for that node no checks will be performed. This will generally not have 82 | a major security impact, however is important to take into account in case is important for your use case. 83 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | ltrVersion = 1.5.10 2 | elasticsearchVersion = 8.17.3 3 | luceneVersion = 9.12.0 4 | ow2Version = 8.0.1 5 | antlrVersion = 4.5.1-1 6 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/o19s/elasticsearch-learning-to-rank/9cf37fe60fc8a30789e6becd852f18b91bc4e4d0/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-all.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%"=="" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%"=="" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if %ERRORLEVEL% equ 0 goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if %ERRORLEVEL% equ 0 goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | set EXIT_CODE=%ERRORLEVEL% 84 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 85 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 86 | exit /b %EXIT_CODE% 87 | 88 | :mainEnd 89 | if "%OS%"=="Windows_NT" endlocal 90 | 91 | :omega 92 | -------------------------------------------------------------------------------- /licenses/antlr4-runtime-4.5.1-1.jar.sha1: -------------------------------------------------------------------------------- 1 | 66144204f9d6d7d3f3f775622c2dd7e9bd511d97 2 | -------------------------------------------------------------------------------- /licenses/antlr4-runtime-LICENSE.txt: -------------------------------------------------------------------------------- 1 | [The "BSD license"] 2 | Copyright (c) 2015 Terence Parr, Sam Harwell 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions 7 | are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright 10 | notice, this list of conditions and the following disclaimer. 11 | 2. Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the distribution. 14 | 3. The name of the author may not be used to endorse or promote products 15 | derived from this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 | IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 | OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 | IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 | NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 | THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /licenses/antlr4-runtime-NOTICE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/o19s/elasticsearch-learning-to-rank/9cf37fe60fc8a30789e6becd852f18b91bc4e4d0/licenses/antlr4-runtime-NOTICE.txt -------------------------------------------------------------------------------- /licenses/asm-5.0.4.jar.sha1: -------------------------------------------------------------------------------- 1 | 0da08b8cce7bbf903602a25a3a163ae252435795 2 | -------------------------------------------------------------------------------- /licenses/asm-LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 France Télécom 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions 6 | are met: 7 | 1. Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | 2. Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in the 11 | documentation and/or other materials provided with the distribution. 12 | 3. Neither the name of the copyright holders nor the names of its 13 | contributors may be used to endorse or promote products derived from 14 | this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 26 | THE POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /licenses/asm-NOTICE.txt: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /licenses/asm-commons-5.0.4.jar.sha1: -------------------------------------------------------------------------------- 1 | 5a556786086c23cd689a0328f8519db93821c04c 2 | -------------------------------------------------------------------------------- /licenses/asm-tree-5.0.4.jar.sha1: -------------------------------------------------------------------------------- 1 | 396ce0c07ba2b481f25a70195c7c94922f0d1b0b -------------------------------------------------------------------------------- /licenses/lucene-expressions-6.6.0.jar.sha1: -------------------------------------------------------------------------------- 1 | 4edcafb90cef9428a6887b92ccb85ff410f1436b -------------------------------------------------------------------------------- /publish/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'maven-publish' 2 | apply plugin: 'signing' 3 | 4 | publishing { 5 | publications { 6 | ltr(MavenPublication) { 7 | from project(':').components.java 8 | 9 | pom { 10 | name = 'Elasticsearch Learning to Rank plugin' 11 | artifactId = 'elasticsearch-learning-to-rank' 12 | description = 'Learning to Rank Query w/ RankLib Models' 13 | url = 'https://elasticsearch-learning-to-rank.readthedocs.io/' 14 | 15 | licenses { 16 | license { 17 | name = 'The Apache Software License, Version 2.0' 18 | url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' 19 | } 20 | } 21 | 22 | developers { 23 | developer { 24 | id = 'osc' 25 | name = 'Oscar' 26 | email = 'oscar@o19s.com' 27 | } 28 | } 29 | 30 | scm { 31 | url = 'https://github.com/o19s/elasticsearch-learning-to-rank' 32 | connection = 'scm:git:git://github.com/o19s/elasticsearch-learning-to-rank.git' 33 | developerConnection = 'scm:git:https://github.com/o19s/elasticsearch-learning-to-rank.git' 34 | } 35 | } 36 | } 37 | } 38 | 39 | repositories { 40 | maven { 41 | name = "MavenCentral" 42 | url = System.getenv('SONATYPE_REPO_URL') 43 | 44 | credentials { 45 | username = System.getenv('OSSRH_TOKEN') 46 | password = System.getenv('OSSRH_TOKEN_PASSSWORD') 47 | } 48 | } 49 | } 50 | } 51 | 52 | signing { 53 | required { gradle.taskGraph.hasTask('publish') } 54 | def signingKey = System.getenv('GPG_SIGNING_KEY') 55 | def signingPass = System.getenv('GPG_SIGNING_PASS') 56 | 57 | useInMemoryPgpKeys(signingKey, signingPass) 58 | sign publishing.publications.ltr 59 | } 60 | -------------------------------------------------------------------------------- /sample_models/coord_ascent.txt: -------------------------------------------------------------------------------- 1 | ## Coordinate Ascent 2 | ## Restart = 5 3 | ## MaxIteration = 25 4 | ## StepBase = 0.05 5 | ## StepScale = 2.0 6 | ## Tolerance = 0.001 7 | ## Regularized = false 8 | ## Slack = 0.001 9 | 1:0.5 2:0.5 -------------------------------------------------------------------------------- /sample_models/linRegression.txt: -------------------------------------------------------------------------------- 1 | ## Linear Regression 2 | ## Lambda = 1.0E-10 3 | 0:0.30545402144173783 1:0.30545402144173783 2:0.13565247758049973 -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'ltr' 2 | include ':rest-api-spec' 3 | include 'publish' 4 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/explore/ExplorerScorer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | package com.o19s.es.explore; 17 | 18 | import org.apache.lucene.search.DocIdSetIterator; 19 | import org.apache.lucene.search.Scorer; 20 | import org.apache.lucene.search.Weight; 21 | 22 | import java.io.IOException; 23 | 24 | public class ExplorerScorer extends Scorer { 25 | private final Scorer subScorer; 26 | private final String type; 27 | 28 | protected ExplorerScorer(Weight weight, String type, Scorer subScorer) { 29 | super(weight); 30 | this.type = type; 31 | this.subScorer = subScorer; 32 | } 33 | 34 | @Override 35 | public float score() throws IOException { 36 | StatisticsHelper tf_stats = new StatisticsHelper(); 37 | 38 | // Grab freq from subscorer, or the children if available 39 | if(subScorer.getChildren().size() > 0) { 40 | for(ChildScorable child : subScorer.getChildren()) { 41 | assert child.child instanceof PostingsExplorerQuery.PostingsExplorerScorer; 42 | if(child.child.docID() == docID()) { 43 | ((PostingsExplorerQuery.PostingsExplorerScorer) child.child).setType(type); 44 | tf_stats.add(child.child.score()); 45 | } 46 | } 47 | } else { 48 | assert subScorer instanceof PostingsExplorerQuery.PostingsExplorerScorer; 49 | assert subScorer.docID() == docID(); 50 | ((PostingsExplorerQuery.PostingsExplorerScorer) subScorer).setType(type); 51 | tf_stats.add(subScorer.score()); 52 | } 53 | 54 | float retval; 55 | switch(type) { 56 | case("sum_raw_tf"): 57 | retval = tf_stats.getSum(); 58 | break; 59 | case("mean_raw_tf"): 60 | retval = tf_stats.getMean(); 61 | break; 62 | case("max_raw_tf"): 63 | case("max_raw_tp"): 64 | retval = tf_stats.getMax(); 65 | break; 66 | case("min_raw_tf"): 67 | case("min_raw_tp"): 68 | retval = tf_stats.getMin(); 69 | break; 70 | case("stddev_raw_tf"): 71 | retval = tf_stats.getStdDev(); 72 | break; 73 | case("avg_raw_tp"): 74 | retval = tf_stats.getMean(); 75 | break; 76 | default: 77 | throw new RuntimeException("Invalid stat type specified."); 78 | } 79 | 80 | return retval; 81 | } 82 | 83 | @Override 84 | public int docID() { 85 | return subScorer.docID(); 86 | } 87 | 88 | 89 | @Override 90 | public DocIdSetIterator iterator() { 91 | return subScorer.iterator(); 92 | } 93 | 94 | /** 95 | * Return the maximum score that documents between the last {@code target} 96 | * that this iterator was {@link #advanceShallow(int) shallow-advanced} to 97 | * included and {@code upTo} included. 98 | */ 99 | @Override 100 | public float getMaxScore(int upTo) throws IOException { 101 | return Float.POSITIVE_INFINITY; 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/explore/StatisticsHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | package com.o19s.es.explore; 17 | 18 | import java.util.ArrayList; 19 | 20 | public class StatisticsHelper { 21 | private final ArrayList data = new ArrayList<>(10); 22 | 23 | private float min = Float.MAX_VALUE; 24 | private float max = 0.0f; 25 | 26 | public enum AggrType { 27 | AVG("avg"), 28 | MAX("max"), 29 | MIN("min"), 30 | SUM("sum"), 31 | STDDEV("stddev"), 32 | MATCHES("matches"); 33 | 34 | private String type; 35 | 36 | AggrType(String type){ 37 | this.type = type; 38 | } 39 | 40 | public String getType() { 41 | return type; 42 | } 43 | } 44 | 45 | public StatisticsHelper() { 46 | 47 | } 48 | 49 | public void add(float val) { 50 | data.add(val); 51 | 52 | if(val < this.min) { 53 | this.min = val; 54 | } 55 | 56 | if(val > this.max) { 57 | this.max = val; 58 | } 59 | } 60 | 61 | public ArrayList getData() { 62 | return data; 63 | } 64 | 65 | public int getSize(){ 66 | return data.size(); 67 | } 68 | 69 | public float getMax() { 70 | assert !data.isEmpty(); 71 | 72 | return max; 73 | } 74 | 75 | public float getMin() { 76 | assert !data.isEmpty(); 77 | 78 | return min; 79 | } 80 | 81 | public float getMean() { 82 | assert !data.isEmpty(); 83 | 84 | return getSum() / data.size(); 85 | } 86 | 87 | public float getSum() { 88 | assert !data.isEmpty(); 89 | 90 | float sum = 0.0f; 91 | 92 | for(float a : data) { 93 | sum += a; 94 | } 95 | 96 | return sum; 97 | } 98 | 99 | public float getVariance() { 100 | assert !data.isEmpty(); 101 | 102 | float mean = getMean(); 103 | float temp = 0.0f; 104 | for(float a : data) 105 | temp += (a-mean)*(a-mean); 106 | return temp/data.size(); 107 | } 108 | 109 | public float getStdDev() { 110 | assert !data.isEmpty(); 111 | 112 | return (float) Math.sqrt(getVariance()); 113 | } 114 | 115 | public float getAggr(AggrType type) { 116 | switch(type) { 117 | case AVG: 118 | return getMean(); 119 | case MAX: 120 | return getMax(); 121 | case MIN: 122 | return getMin(); 123 | case SUM: 124 | return getSum(); 125 | case STDDEV: 126 | return getStdDev(); 127 | default: 128 | return 0.0f; 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/LtrQueryContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr; 18 | 19 | import org.elasticsearch.index.query.SearchExecutionContext; 20 | 21 | import java.util.Collections; 22 | import java.util.Set; 23 | 24 | /** 25 | * LTR queryShardContext used to track information needed for building lucene queries 26 | */ 27 | public class LtrQueryContext { 28 | private final SearchExecutionContext queryShardContext; 29 | private final Set activeFeatures; 30 | 31 | public LtrQueryContext(SearchExecutionContext context) { 32 | this(context, Collections.emptySet()); 33 | } 34 | 35 | public LtrQueryContext(SearchExecutionContext context, Set activeFeatures) { 36 | this.queryShardContext = context; 37 | this.activeFeatures = activeFeatures; 38 | } 39 | 40 | public SearchExecutionContext getSearchExecutionContext() { 41 | return queryShardContext; 42 | } 43 | 44 | public boolean isFeatureActive(String featureName) { 45 | return activeFeatures == null || activeFeatures.isEmpty() || activeFeatures.contains(featureName); 46 | } 47 | 48 | public Set getActiveFeatures() { 49 | return activeFeatures==null? Collections.emptySet(): Collections.unmodifiableSet(activeFeatures); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/action/TransportLTRStatsAction.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.action; 2 | 3 | import com.o19s.es.ltr.action.LTRStatsAction.LTRStatsNodeRequest; 4 | import com.o19s.es.ltr.action.LTRStatsAction.LTRStatsNodeResponse; 5 | import com.o19s.es.ltr.action.LTRStatsAction.LTRStatsNodesRequest; 6 | import com.o19s.es.ltr.action.LTRStatsAction.LTRStatsNodesResponse; 7 | import com.o19s.es.ltr.stats.LTRStats; 8 | import org.elasticsearch.action.FailedNodeException; 9 | import org.elasticsearch.action.support.ActionFilters; 10 | import org.elasticsearch.action.support.nodes.TransportNodesAction; 11 | import org.elasticsearch.cluster.node.DiscoveryNode; 12 | import org.elasticsearch.cluster.service.ClusterService; 13 | import org.elasticsearch.injection.guice.Inject; 14 | import org.elasticsearch.common.io.stream.StreamInput; 15 | import org.elasticsearch.tasks.Task; 16 | import org.elasticsearch.threadpool.ThreadPool; 17 | import org.elasticsearch.transport.TransportService; 18 | 19 | import java.io.IOException; 20 | import java.util.List; 21 | import java.util.Map; 22 | import java.util.Set; 23 | import java.util.stream.Collectors; 24 | 25 | public class TransportLTRStatsAction extends 26 | TransportNodesAction { 27 | 28 | private final LTRStats ltrStats; 29 | 30 | @Inject 31 | public TransportLTRStatsAction(ThreadPool threadPool, 32 | ClusterService clusterService, 33 | TransportService transportService, 34 | ActionFilters actionFilters, 35 | LTRStats ltrStats) { 36 | super(LTRStatsAction.NAME, clusterService, transportService, 37 | actionFilters, LTRStatsNodeRequest::new, 38 | threadPool.executor(ThreadPool.Names.MANAGEMENT)); 39 | this.ltrStats = ltrStats; 40 | } 41 | 42 | @Override 43 | protected LTRStatsNodesResponse newResponse(LTRStatsNodesRequest request, 44 | List nodeResponses, 45 | List failures) { 46 | Set statsToBeRetrieved = request.getStatsToBeRetrieved(); 47 | Map clusterStats = 48 | ltrStats.getClusterStats() 49 | .entrySet() 50 | .stream() 51 | .filter(e -> statsToBeRetrieved.contains(e.getKey())) 52 | .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getStatValue())); 53 | 54 | return new LTRStatsNodesResponse(clusterService.getClusterName(), nodeResponses, failures, clusterStats); 55 | } 56 | 57 | @Override 58 | protected LTRStatsNodeRequest newNodeRequest(LTRStatsNodesRequest request) { 59 | return new LTRStatsNodeRequest(request); 60 | } 61 | 62 | @Override 63 | protected LTRStatsNodeResponse newNodeResponse(StreamInput in, DiscoveryNode node) throws IOException { 64 | return new LTRStatsNodeResponse(in); 65 | } 66 | 67 | @Override 68 | protected LTRStatsNodeResponse nodeOperation(LTRStatsNodeRequest request, Task task) { 69 | LTRStatsNodesRequest nodesRequest = request.getLTRStatsNodesRequest(); 70 | Set statsToBeRetrieved = nodesRequest.getStatsToBeRetrieved(); 71 | 72 | Map statValues = 73 | ltrStats.getNodeStats() 74 | .entrySet() 75 | .stream() 76 | .filter(e -> statsToBeRetrieved.contains(e.getKey())) 77 | .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getStatValue())); 78 | return new LTRStatsNodeResponse(clusterService.localNode(), statValues); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/feature/Feature.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.feature; 18 | 19 | import com.o19s.es.ltr.LtrQueryContext; 20 | import org.apache.lucene.search.Query; 21 | 22 | import java.util.Map; 23 | 24 | /** 25 | * A feature that can be transformed into a lucene query 26 | */ 27 | public interface Feature { 28 | /** 29 | * The feature name 30 | * 31 | * @return the feature name 32 | */ 33 | String name(); 34 | 35 | /** 36 | * Transform this feature into a lucene query 37 | * 38 | * @param context the LtRQuery context on which the lucene query is going to be build on 39 | * @param set the features to be used to the build the lucene query 40 | * @param params additional parameters to be used in the building of the lucene query 41 | * @return the Lucene query build for the feature and the given parameter 42 | */ 43 | Query doToQuery(LtrQueryContext context, FeatureSet set, Map params); 44 | 45 | /** 46 | * Optional optimization step 47 | * 48 | * @return an optimized version of the feature if applicable 49 | */ 50 | default Feature optimize() { 51 | return this; 52 | } 53 | 54 | /** 55 | * Validate this feature against a featureset 56 | * Some feature may depend on other feature 57 | * 58 | * @param set the feature-set to validate the current feature against 59 | */ 60 | default void validate(FeatureSet set) { 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/feature/FeatureSet.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.feature; 18 | 19 | import com.o19s.es.ltr.LtrQueryContext; 20 | import org.apache.lucene.search.Query; 21 | 22 | import java.util.List; 23 | import java.util.Map; 24 | 25 | /** 26 | * A set of features. 27 | * Features can be identified by their name or ordinal. 28 | * A FeatureSet is dense and ordinals start at 0. 29 | */ 30 | public interface FeatureSet { 31 | 32 | /** 33 | * @return the name of the feature set 34 | */ 35 | String name(); 36 | 37 | /** 38 | * Parse and build lucene queries from the features in this set 39 | * 40 | * @param context the LtRQuery context on which the lucene queries are going to be build on 41 | * @param params additional parameters to be used in the building of the lucene queries 42 | * @return the list of queries build for the current feature-set and the given parameters 43 | */ 44 | List toQueries(LtrQueryContext context, Map params); 45 | 46 | /** 47 | * Retrieve feature ordinal by its name. 48 | * If the feature does not exist the behavior of this method is 49 | * undefined. 50 | * 51 | * @param featureName the name of the feature to retrieve its ordinal position 52 | * @return the ordinal of the given feature in the current set 53 | */ 54 | int featureOrdinal(String featureName); 55 | 56 | /** 57 | * Retrieve feature by its ordinal. 58 | * May produce unexpected results if called with unknown ordinal 59 | * 60 | * @param ord the ordinal place of the feature to retrieve 61 | * @return the feature at the given ordinal position 62 | */ 63 | Feature feature(int ord); 64 | 65 | /** 66 | * Access a feature by its name. 67 | * May produce unexpected results if called with unknown feature 68 | * 69 | * @param featureName the name of the feature to retrieve 70 | * @return the feature found with the given name 71 | */ 72 | Feature feature(String featureName); 73 | 74 | /** 75 | * Check if this set supports this feature 76 | * 77 | * @param featureName the name of the feature check 78 | * @return true if supported false otherwise 79 | */ 80 | boolean hasFeature(String featureName); 81 | 82 | /** 83 | * @return the number of features in the set 84 | */ 85 | int size(); 86 | 87 | default FeatureSet optimize() { 88 | return this; 89 | } 90 | 91 | default void validate() { 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/feature/LtrModel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.feature; 18 | 19 | import com.o19s.es.ltr.ranker.LtrRanker; 20 | 21 | /** 22 | * Represents a self contained LTR model 23 | */ 24 | public interface LtrModel { 25 | /** 26 | * @return the name of the model 27 | */ 28 | String name(); 29 | 30 | /** 31 | * @return the {@link LtrRanker} implementation used by this model 32 | */ 33 | LtrRanker ranker(); 34 | 35 | /** 36 | * @return the set of features used by this model 37 | */ 38 | FeatureSet featureSet(); 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/feature/PrebuiltFeature.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.feature; 18 | 19 | import com.o19s.es.ltr.LtrQueryContext; 20 | import org.apache.lucene.index.IndexReader; 21 | import org.apache.lucene.search.BooleanClause; 22 | import org.apache.lucene.search.Query; 23 | import org.apache.lucene.search.QueryVisitor; 24 | import org.apache.lucene.search.Weight; 25 | import org.apache.lucene.search.IndexSearcher; 26 | import org.apache.lucene.search.ScoreMode; 27 | import org.elasticsearch.core.Nullable; 28 | 29 | import java.io.IOException; 30 | import java.util.Map; 31 | import java.util.Objects; 32 | 33 | /** 34 | * A prebuilt featured query, needed by query builders 35 | * that provides the query itself. 36 | */ 37 | public class PrebuiltFeature extends Query implements Feature { 38 | private final String name; 39 | private final Query query; 40 | 41 | public PrebuiltFeature(@Nullable String name, Query query) { 42 | this.name = name; 43 | this.query = Objects.requireNonNull(query); 44 | } 45 | 46 | @Override @Nullable 47 | public String name() { 48 | return name; 49 | } 50 | 51 | @Override 52 | public Query doToQuery(LtrQueryContext context, FeatureSet set, Map params) { 53 | return query; 54 | } 55 | 56 | public Query getPrebuiltQuery() { 57 | return query; 58 | } 59 | 60 | @Override 61 | public int hashCode() { 62 | return Objects.hash(name, query); 63 | } 64 | 65 | @Override 66 | public boolean equals(Object o) { 67 | if (!(o instanceof PrebuiltFeature)) { 68 | return false; 69 | } 70 | PrebuiltFeature other = (PrebuiltFeature) o; 71 | return Objects.equals(name, other.name) 72 | && Objects.equals(query, other.query); 73 | } 74 | 75 | @Override 76 | public String toString(String field) { 77 | return query.toString(field); 78 | } 79 | 80 | @Override 81 | public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { 82 | return query.createWeight(searcher, scoreMode, boost); 83 | } 84 | 85 | @Override 86 | public Query rewrite(IndexReader reader) throws IOException { 87 | return query.rewrite(reader); 88 | } 89 | 90 | @Override 91 | public void visit(QueryVisitor visitor) { 92 | this.query.visit(visitor.getSubVisitor(BooleanClause.Occur.MUST, this)); 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/feature/PrebuiltFeatureSet.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.feature; 18 | 19 | import com.o19s.es.ltr.LtrQueryContext; 20 | import org.apache.lucene.search.Query; 21 | import org.elasticsearch.core.Nullable; 22 | 23 | import java.util.ArrayList; 24 | import java.util.List; 25 | import java.util.Map; 26 | import java.util.Objects; 27 | import java.util.stream.IntStream; 28 | 29 | public class PrebuiltFeatureSet implements FeatureSet { 30 | private final List features; 31 | private final String name; 32 | 33 | public PrebuiltFeatureSet(@Nullable String name, List features) { 34 | this.name = name; 35 | this.features = new ArrayList<>(Objects.requireNonNull(features)); 36 | } 37 | 38 | @Override 39 | public String name() { 40 | return name; 41 | } 42 | 43 | /** 44 | * Parse and build lucene queries 45 | */ 46 | @Override 47 | public List toQueries(LtrQueryContext context, Map params) { 48 | return features; 49 | } 50 | 51 | @Override 52 | public int featureOrdinal(String featureName) { 53 | int ord = findFeatureIndexByName(featureName); 54 | if (ord < 0) { 55 | throw new IllegalArgumentException("Unknown feature [" + featureName + "]"); 56 | } 57 | return ord; 58 | } 59 | 60 | @Override 61 | public Feature feature(int ord) { 62 | return (PrebuiltFeature) features.get(ord); 63 | } 64 | 65 | @Override 66 | public PrebuiltFeature feature(String name) { 67 | return (PrebuiltFeature) features.get(featureOrdinal(name)); 68 | } 69 | 70 | @Override 71 | public boolean hasFeature(String name) { 72 | return findFeatureIndexByName(name) >= 0; 73 | } 74 | 75 | @Override 76 | public int size() { 77 | return features.size(); 78 | } 79 | 80 | private int findFeatureIndexByName(String featureName) { 81 | // slow, not meant for runtime usage, mostly needed for tests 82 | // would make sense to implement a Map to do this once 83 | // feature names are mandatory and unique. 84 | return IntStream.range(0, features.size()) 85 | .filter(i -> Objects.equals(((PrebuiltFeature)features.get(i)).name(), featureName)) 86 | .findFirst() 87 | .orElse(-1); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/feature/PrebuiltLtrModel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.feature; 18 | 19 | import com.o19s.es.ltr.ranker.LtrRanker; 20 | 21 | import java.util.Objects; 22 | 23 | /** 24 | * Prebuilt model 25 | */ 26 | public class PrebuiltLtrModel implements LtrModel { 27 | private final String name; 28 | private final LtrRanker ranker; 29 | private final PrebuiltFeatureSet featureSet; 30 | 31 | public PrebuiltLtrModel(String name, LtrRanker ranker, PrebuiltFeatureSet featureSet) { 32 | this.name = Objects.requireNonNull(name); 33 | this.ranker = Objects.requireNonNull(ranker); 34 | this.featureSet = Objects.requireNonNull(featureSet); 35 | } 36 | 37 | @Override 38 | public String name() { 39 | return name; 40 | } 41 | 42 | @Override 43 | public LtrRanker ranker() { 44 | return ranker; 45 | } 46 | 47 | @Override 48 | public FeatureSet featureSet() { 49 | return featureSet; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/feature/store/CompiledLtrModel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.feature.store; 18 | 19 | import com.o19s.es.ltr.feature.FeatureSet; 20 | import com.o19s.es.ltr.feature.LtrModel; 21 | import com.o19s.es.ltr.ranker.LtrRanker; 22 | import org.apache.lucene.util.Accountable; 23 | import org.apache.lucene.util.RamUsageEstimator; 24 | 25 | import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; 26 | import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_HEADER; 27 | 28 | public class CompiledLtrModel implements LtrModel, Accountable { 29 | private static final long BASE_RAM_USED = RamUsageEstimator.shallowSizeOfInstance(StoredLtrModel.class); 30 | 31 | private final String name; 32 | private final FeatureSet set; 33 | private final LtrRanker ranker; 34 | 35 | public CompiledLtrModel(String name, FeatureSet set, LtrRanker ranker) { 36 | this.name = name; 37 | this.set = set.optimize(); 38 | this.ranker = ranker; 39 | } 40 | 41 | /** 42 | * Name of the model 43 | */ 44 | @Override 45 | public String name() { 46 | return name; 47 | } 48 | 49 | /** 50 | * Return the {@link LtrRanker} implementation used by this model 51 | */ 52 | @Override 53 | public LtrRanker ranker() { 54 | return ranker; 55 | } 56 | 57 | /** 58 | * The set of features used by this model 59 | */ 60 | @Override 61 | public FeatureSet featureSet() { 62 | return set; 63 | } 64 | 65 | /** 66 | * Return the memory usage of this object in bytes. Negative values are illegal. 67 | */ 68 | @Override 69 | public long ramBytesUsed() { 70 | return BASE_RAM_USED + name.length() * Character.BYTES + NUM_BYTES_ARRAY_HEADER 71 | + (set instanceof Accountable ? ((Accountable)set).ramBytesUsed() : set.size() * NUM_BYTES_OBJECT_HEADER) 72 | + (ranker instanceof Accountable ? 73 | ((Accountable)ranker).ramBytesUsed() : set.size() * NUM_BYTES_OBJECT_HEADER); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/feature/store/ExtraLoggingSupplier.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.feature.store; 2 | 3 | import java.util.Map; 4 | import java.util.function.Supplier; 5 | 6 | 7 | public class ExtraLoggingSupplier implements Supplier> { 8 | protected Supplier> supplier; 9 | 10 | public void setSupplier(Supplier> supplier) { 11 | this.supplier = supplier; 12 | } 13 | 14 | /** 15 | * Return a Map to add additional information to be returned when logging feature values. 16 | * 17 | * This Map will only be non-null during the LoggingFetchSubPhase. 18 | */ 19 | @Override 20 | public Map get() { 21 | if (supplier != null) { 22 | return supplier.get(); 23 | } 24 | return null; 25 | } 26 | } -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/feature/store/FeatureNormDefinition.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.feature.store; 2 | 3 | import com.o19s.es.ltr.ranker.normalizer.Normalizer; 4 | import org.elasticsearch.common.io.stream.Writeable; 5 | import org.elasticsearch.xcontent.ToXContent; 6 | 7 | /** 8 | * Parsed feature norm from model definition 9 | */ 10 | public interface FeatureNormDefinition extends ToXContent, Writeable { 11 | 12 | /** 13 | * @return 14 | * Construct the feature norm associated with this definitino 15 | */ 16 | Normalizer createFeatureNorm(); 17 | 18 | /** 19 | * @return 20 | * The feature name associated with this normalizer to 21 | * later associate with an ord 22 | */ 23 | String featureName(); 24 | 25 | /** 26 | * @return 27 | * A type of normalizer 28 | */ 29 | StoredFeatureNormalizers.Type normType(); 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/feature/store/FeatureStore.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.feature.store; 18 | 19 | import com.o19s.es.ltr.feature.Feature; 20 | import com.o19s.es.ltr.feature.FeatureSet; 21 | 22 | import java.io.IOException; 23 | 24 | /** 25 | * A feature store 26 | */ 27 | public interface FeatureStore { 28 | /** 29 | * @return the store name 30 | */ 31 | String getStoreName(); 32 | 33 | /** 34 | * @param name the name of the feature to load 35 | * @return the loaded feature 36 | * 37 | * @throws IOException if the given feature store can not be loaded 38 | */ 39 | Feature load(String name) throws IOException; 40 | 41 | /** 42 | * @param name the feature-set name to load 43 | * @return the loaded feature-set 44 | * 45 | * @throws IOException if the given feature-set can not be loaded 46 | */ 47 | FeatureSet loadSet(String name) throws IOException; 48 | 49 | /** 50 | * @param name the model name to be compiled 51 | * @return the compiled model 52 | * 53 | * @throws IOException if the model can not be loaded and compiled 54 | */ 55 | CompiledLtrModel loadModel(String name) throws IOException; 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/feature/store/StorableElement.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.feature.store; 18 | 19 | import org.elasticsearch.common.ParsingException; 20 | import org.elasticsearch.common.io.stream.NamedWriteable; 21 | import org.elasticsearch.xcontent.ToXContent; 22 | import org.elasticsearch.xcontent.XContentParser; 23 | 24 | public interface StorableElement extends ToXContent, NamedWriteable { 25 | /** 26 | * @return the element name 27 | */ 28 | String name(); 29 | 30 | /** 31 | * @return the element type 32 | */ 33 | String type(); 34 | 35 | /** 36 | * @return if the current element can be updated 37 | */ 38 | default boolean updatable() { 39 | return true; 40 | } 41 | 42 | default String id() { 43 | return generateId(type(), name()); 44 | } 45 | 46 | static String generateId(String type, String name) { 47 | return type + "-" + name; 48 | } 49 | 50 | @Override 51 | default String getWriteableName() { 52 | return type(); 53 | } 54 | 55 | abstract class StorableElementParserState { 56 | private String name; 57 | 58 | public void setName(String name) { 59 | this.name = name; 60 | } 61 | 62 | public String getName() { 63 | return this.name; 64 | } 65 | 66 | void resolveName(XContentParser parser, String name) { 67 | if (this.name == null && name != null) { 68 | this.name = name; 69 | } else if ( this.name == null /* && name == null */) { 70 | throw new ParsingException(parser.getTokenLocation(), "Field [name] is mandatory"); 71 | } else if ( /* this.name != null && */ name != null && !this.name.equals(name)) { 72 | throw new ParsingException(parser.getTokenLocation(), "Invalid [name], expected ["+name+"] but got [" + this.name+"]"); 73 | } 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/feature/store/index/CachedFeatureStore.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.feature.store.index; 18 | 19 | import com.o19s.es.ltr.feature.Feature; 20 | import com.o19s.es.ltr.feature.FeatureSet; 21 | import com.o19s.es.ltr.feature.store.CompiledLtrModel; 22 | import com.o19s.es.ltr.feature.store.FeatureStore; 23 | import org.elasticsearch.common.cache.Cache; 24 | 25 | import java.io.IOException; 26 | 27 | /** 28 | * Cache layer on top of an {@link IndexFeatureStore} 29 | */ 30 | public class CachedFeatureStore implements FeatureStore { 31 | private final FeatureStore inner; 32 | private final Caches caches; 33 | 34 | public CachedFeatureStore(FeatureStore inner, Caches caches) { 35 | this.inner = inner; 36 | this.caches = caches; 37 | } 38 | 39 | @Override 40 | public String getStoreName() { 41 | return inner.getStoreName(); 42 | } 43 | 44 | @Override 45 | public Feature load(String id) throws IOException { 46 | return caches.loadFeature(key(id), inner::load); 47 | } 48 | 49 | @Override 50 | public FeatureSet loadSet(String id) throws IOException { 51 | return caches.loadFeatureSet(key(id), inner::loadSet); 52 | } 53 | 54 | @Override 55 | public CompiledLtrModel loadModel(String id) throws IOException { 56 | return caches.loadModel(key(id), inner::loadModel); 57 | } 58 | 59 | Feature getCachedFeature(String id) { 60 | return innerGet(id, caches.featureCache()); 61 | } 62 | 63 | FeatureSet getCachedFeatureSet(String id) { 64 | return innerGet(id, caches.featureSetCache()); 65 | } 66 | 67 | CompiledLtrModel getCachedModel(String id) { 68 | return innerGet(id, caches.modelCache()); 69 | } 70 | 71 | public long totalWeight() { 72 | return featuresWeight() + featureSetWeight() + modelWeight(); 73 | } 74 | 75 | public long featuresWeight() { 76 | return caches.featureCache().weight(); 77 | } 78 | 79 | public long featureSetWeight() { 80 | return caches.featureSetCache().weight(); 81 | } 82 | 83 | public long modelWeight() { 84 | return caches.modelCache().weight(); 85 | } 86 | 87 | private T innerGet(String id, Cache cache) { 88 | return cache.get(key(id)); 89 | } 90 | 91 | private Caches.CacheKey key(String id) { 92 | return new Caches.CacheKey(inner.getStoreName(), id); 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/query/LtrRewritableQuery.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.query; 2 | 3 | import org.apache.lucene.search.Query; 4 | 5 | import java.io.IOException; 6 | 7 | public interface LtrRewritableQuery { 8 | /** 9 | * Rewrite the query so that it holds the vectorSupplier and provide extra logging support 10 | * 11 | * @param context the {@link LtrRewriteContext} to rewrite the current query 12 | * @return the rewritten query 13 | * @throws IOException in case of errors 14 | */ 15 | Query ltrRewrite(LtrRewriteContext context) throws IOException; 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/query/LtrRewriteContext.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.query; 2 | 3 | import com.o19s.es.ltr.ranker.LogLtrRanker; 4 | import com.o19s.es.ltr.ranker.LtrRanker; 5 | 6 | import java.util.function.Supplier; 7 | 8 | /** 9 | * Contains context needed to rewrite queries to holds the vectorSupplier and provide extra logging support 10 | */ 11 | public class LtrRewriteContext { 12 | private final Supplier vectorSupplier; 13 | private final LtrRanker ranker; 14 | 15 | public LtrRewriteContext(LtrRanker ranker, Supplier vectorSupplier) { 16 | this.ranker = ranker; 17 | this.vectorSupplier = vectorSupplier; 18 | } 19 | 20 | public Supplier getFeatureVectorSupplier() { 21 | return vectorSupplier; 22 | } 23 | 24 | /** 25 | * Get LogConsumer used during the LoggingFetchSubPhase 26 | * 27 | * The returned consumer will only be non-null during the logging fetch phase 28 | * 29 | * @return the LogConsumer used during the fetch-subphase, null otherwise 30 | */ 31 | public LogLtrRanker.LogConsumer getLogConsumer() { 32 | if (ranker instanceof LogLtrRanker) { 33 | return ((LogLtrRanker)ranker).getLogConsumer(); 34 | } 35 | return null; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/query/NoopScorer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2016] Doug Turnbull 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.o19s.es.ltr.query; 17 | 18 | import org.apache.lucene.search.DocIdSetIterator; 19 | import org.apache.lucene.search.Scorer; 20 | import org.apache.lucene.search.Weight; 21 | 22 | import java.io.IOException; 23 | 24 | /** 25 | * Created by doug on 2/3/17. 26 | */ 27 | public class NoopScorer extends Scorer { 28 | private final DocIdSetIterator _noopIter; 29 | /** 30 | * Constructs a Scorer 31 | * 32 | * @param weight The scorers weight 33 | * @param maxDocs maximum number of documents to score 34 | */ 35 | public NoopScorer(Weight weight, int maxDocs) { 36 | super(weight); 37 | _noopIter = DocIdSetIterator.all(maxDocs); 38 | 39 | } 40 | 41 | public NoopScorer(Weight weight, DocIdSetIterator iterator) { 42 | super(weight); 43 | _noopIter = iterator; 44 | } 45 | 46 | @Override 47 | public int docID() { 48 | return _noopIter.docID(); 49 | } 50 | 51 | @Override 52 | public float score() throws IOException { 53 | return 0; 54 | } 55 | 56 | @Override 57 | public DocIdSetIterator iterator() { 58 | return _noopIter; 59 | } 60 | 61 | /** 62 | * Return the maximum score that documents between the last {@code target} 63 | * that this iterator was {@link #advanceShallow(int) shallow-advanced} to 64 | * included and {@code upTo} included. 65 | */ 66 | @Override 67 | public float getMaxScore(int upTo) throws IOException { 68 | return Float.POSITIVE_INFINITY; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/ArrayFeatureVector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.ranker; 18 | 19 | import java.util.Arrays; 20 | 21 | public class ArrayFeatureVector implements LtrRanker.FeatureVector { 22 | public final float[] scores; 23 | public final float defaultScore; 24 | 25 | public ArrayFeatureVector(int size, float value) { 26 | scores = new float[size]; 27 | defaultScore = value; 28 | } 29 | 30 | @Override 31 | public void setFeatureScore(int featureIdx, float score) { 32 | scores[featureIdx] = score; 33 | } 34 | 35 | @Override 36 | public float getFeatureScore(int featureIdx) { 37 | return scores[featureIdx]; 38 | } 39 | 40 | public void reset() { 41 | Arrays.fill(scores, defaultScore); 42 | } 43 | 44 | @Override 45 | public float getDefaultScore() { 46 | return defaultScore; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/DenseFeatureVector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.ranker; 18 | 19 | public class DenseFeatureVector extends ArrayFeatureVector { 20 | 21 | public DenseFeatureVector(int size) { 22 | super(size, 0F); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/DenseLtrRanker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.ranker; 18 | 19 | /** 20 | * A dense ranker base class to work with {@link DenseFeatureVector} 21 | * where missing feature scores are set to 0. 22 | */ 23 | public abstract class DenseLtrRanker implements LtrRanker { 24 | @Override 25 | public DenseFeatureVector newFeatureVector(FeatureVector reuse) { 26 | if (reuse != null) { 27 | assert reuse instanceof DenseFeatureVector; 28 | DenseFeatureVector vector = (DenseFeatureVector) reuse; 29 | vector.reset(); 30 | return vector; 31 | } 32 | return new DenseFeatureVector(size()); 33 | } 34 | 35 | @Override 36 | public float score(FeatureVector vector) { 37 | assert vector instanceof DenseFeatureVector; 38 | return this.score((DenseFeatureVector) vector); 39 | } 40 | 41 | protected abstract float score(DenseFeatureVector vector); 42 | 43 | /** 44 | * @return the number of features supported by this ranker 45 | */ 46 | protected abstract int size(); 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/LogLtrRanker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.ranker; 18 | 19 | import java.util.Map; 20 | 21 | public class LogLtrRanker implements LtrRanker { 22 | private final LogConsumer logger; 23 | private final NullRanker ranker; 24 | 25 | public LogLtrRanker(NullRanker ranker, LogConsumer consumer) { 26 | this.ranker = ranker; 27 | assert(this.ranker.getClass() == NullRanker.class); 28 | this.logger = consumer; 29 | } 30 | 31 | public LogLtrRanker(LogConsumer consumer, int modelSize) { 32 | this.ranker = new NullRanker(modelSize); 33 | this.logger = consumer; 34 | } 35 | 36 | @Override 37 | public String name() { 38 | return "log(" + ranker.name() + ")"; 39 | } 40 | 41 | @Override 42 | public FeatureVector newFeatureVector(FeatureVector reuse) { 43 | final VectorWrapper wrapper; 44 | if (reuse == null) { 45 | wrapper = new VectorWrapper(logger); 46 | } else { 47 | assert reuse instanceof VectorWrapper; 48 | wrapper = (VectorWrapper) reuse; 49 | } 50 | wrapper.reset(ranker); 51 | return wrapper; 52 | } 53 | 54 | @Override 55 | public float score(FeatureVector point) { 56 | assert point instanceof VectorWrapper; 57 | return ranker.score(((VectorWrapper) point).inner); 58 | } 59 | 60 | private static class VectorWrapper implements FeatureVector { 61 | private FeatureVector inner; 62 | private final LogConsumer logger; 63 | 64 | VectorWrapper(LogConsumer consumer) { 65 | this.logger = consumer; 66 | } 67 | 68 | @Override 69 | public void setFeatureScore(int featureId, float score) { 70 | inner.setFeatureScore(featureId, score); 71 | logger.accept(featureId, score); 72 | } 73 | 74 | @Override 75 | public float getFeatureScore(int featureId) { 76 | return inner.getFeatureScore(featureId); 77 | } 78 | 79 | void reset(LtrRanker ranker) { 80 | this.inner = ranker.newFeatureVector(inner); 81 | logger.reset(); 82 | } 83 | 84 | public float getDefaultScore() { 85 | return inner.getDefaultScore(); 86 | } 87 | } 88 | 89 | public LogConsumer getLogConsumer() { 90 | return logger; 91 | } 92 | 93 | @FunctionalInterface 94 | public interface LogConsumer { 95 | void accept(int featureOrdinal, float score); 96 | default Map getExtraLoggingMap() {return null;} 97 | default void reset() {} 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/LtrRanker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.ranker; 18 | 19 | import org.elasticsearch.core.Nullable; 20 | 21 | /** 22 | * A ranker used by {@link com.o19s.es.ltr.query.RankerQuery} 23 | * to compute a score based on a set of feature scores. 24 | */ 25 | public interface LtrRanker { 26 | 27 | /** 28 | * LtrRanker name 29 | * 30 | * @return the ranker name 31 | */ 32 | String name(); 33 | 34 | /** 35 | * Data point implementation used by this ranker 36 | * A data point is used to store feature scores. 37 | * A single instance will be created for every Scorer. 38 | * The implementation does not have to be thread-safe and meant to be reused. 39 | * The ranker must always provide a new instance of the data point when this method is called with null. 40 | * 41 | * @param reuse allows Ranker to reuse a feature vector instance 42 | * @return the new feature vector to be used by this ranker 43 | */ 44 | FeatureVector newFeatureVector(@Nullable FeatureVector reuse); 45 | 46 | /** 47 | * Score the data point. 48 | * At this point all feature scores are set. 49 | * features that did not match are set with a default score 50 | * 51 | * @param point the feature vector point to compute the score for 52 | * @return the score computed for the given point 53 | */ 54 | float score(FeatureVector point); 55 | 56 | /** 57 | * A FeatureVector used to store individual feature scores 58 | */ 59 | interface FeatureVector { 60 | /** 61 | * Set the score for the given featureId 62 | * 63 | * @param featureId the feature-id to set the score for 64 | * @param score the feature's score 65 | */ 66 | void setFeatureScore(int featureId, float score); 67 | 68 | /** 69 | * Retrieve the score for the given feature-id 70 | * 71 | * @param featureId the feature-id to retrieve the score for 72 | * @return the score computed for the given feature 73 | */ 74 | float getFeatureScore(int featureId); 75 | 76 | /** 77 | * Retrieve the default score 78 | * @return the score computed for the given feature 79 | */ 80 | float getDefaultScore(); 81 | 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/NullRanker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.ranker; 18 | 19 | public class NullRanker extends DenseLtrRanker { 20 | private final int modelSize; 21 | 22 | public NullRanker(int modelSize) { 23 | this.modelSize = modelSize; 24 | } 25 | 26 | @Override 27 | public String name() { 28 | return "null_ranker"; 29 | } 30 | 31 | @Override 32 | public float score(FeatureVector point) { 33 | return 0F; 34 | } 35 | 36 | @Override 37 | protected float score(DenseFeatureVector vector) { 38 | return 0F; 39 | } 40 | 41 | @Override 42 | protected int size() { 43 | return modelSize; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/SparseFeatureVector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.ranker; 18 | 19 | public class SparseFeatureVector extends ArrayFeatureVector { 20 | 21 | public SparseFeatureVector(int size) { 22 | super(size, Float.NaN); 23 | reset(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/SparseLtrRanker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.ranker; 18 | 19 | /** 20 | * A dense ranker base class to work with {@link SparseFeatureVector} 21 | * where missing feature scores are set to 0. 22 | */ 23 | public abstract class SparseLtrRanker implements LtrRanker { 24 | @Override 25 | public SparseFeatureVector newFeatureVector(FeatureVector reuse) { 26 | if (reuse != null) { 27 | assert reuse instanceof SparseFeatureVector; 28 | SparseFeatureVector vector = (SparseFeatureVector) reuse; 29 | vector.reset(); 30 | return vector; 31 | } 32 | return new SparseFeatureVector(size()); 33 | } 34 | 35 | @Override 36 | public float score(FeatureVector vector) { 37 | assert vector instanceof SparseFeatureVector; 38 | return this.score((SparseFeatureVector) vector); 39 | } 40 | 41 | protected abstract float score(SparseFeatureVector vector); 42 | 43 | /** 44 | * @return the number of features supported by this ranker 45 | */ 46 | protected abstract int size(); 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/linear/LinearRanker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.ranker.linear; 18 | 19 | import com.o19s.es.ltr.ranker.DenseFeatureVector; 20 | import com.o19s.es.ltr.ranker.DenseLtrRanker; 21 | import org.apache.lucene.util.Accountable; 22 | import org.apache.lucene.util.RamUsageEstimator; 23 | 24 | import java.util.Arrays; 25 | import java.util.Objects; 26 | 27 | /** 28 | * Simple linear ranker that applies a dot product based 29 | * on the provided weights array. 30 | */ 31 | public class LinearRanker extends DenseLtrRanker implements Accountable { 32 | private final float[] weights; 33 | 34 | public LinearRanker(float[] weights) { 35 | this.weights = Objects.requireNonNull(weights); 36 | } 37 | 38 | @Override 39 | public String name() { 40 | return "linear"; 41 | } 42 | 43 | @Override 44 | protected float score(DenseFeatureVector point) { 45 | float[] scores = point.scores; 46 | float score = 0; 47 | for (int i = 0; i < weights.length; i++) { 48 | score += weights[i]*scores[i]; 49 | } 50 | return score; 51 | } 52 | 53 | @Override 54 | protected int size() { 55 | return weights.length; 56 | } 57 | 58 | @Override 59 | public boolean equals(Object o) { 60 | if (this == o) return true; 61 | if (o == null || getClass() != o.getClass()) return false; 62 | 63 | LinearRanker ranker = (LinearRanker) o; 64 | 65 | return Arrays.equals(weights, ranker.weights); 66 | } 67 | 68 | @Override 69 | public int hashCode() { 70 | return Arrays.hashCode(weights); 71 | } 72 | 73 | /** 74 | * Return the memory usage of this object in bytes. Negative values are illegal. 75 | */ 76 | @Override 77 | public long ramBytesUsed() { 78 | return RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.sizeOf(weights); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/normalizer/FeatureNormalizingRanker.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.ranker.normalizer; 2 | 3 | import com.o19s.es.ltr.ranker.LtrRanker; 4 | import org.apache.lucene.util.Accountable; 5 | import org.apache.lucene.util.RamUsageEstimator; 6 | 7 | import java.util.Map; 8 | import java.util.Objects; 9 | 10 | public class FeatureNormalizingRanker implements LtrRanker, Accountable { 11 | 12 | private final LtrRanker wrapped; 13 | private final Map ftrNorms; 14 | private static final long BASE_RAM_USED; 15 | 16 | private static final long PER_FTR_NORM_RAM_USED = 8; 17 | static { 18 | BASE_RAM_USED = RamUsageEstimator.shallowSizeOfInstance(FeatureNormalizingRanker.class); 19 | } 20 | 21 | public FeatureNormalizingRanker(LtrRanker wrapped, Map ftrNorms) { 22 | this.wrapped = Objects.requireNonNull(wrapped); 23 | this.ftrNorms = Objects.requireNonNull(ftrNorms); 24 | } 25 | 26 | public Map getFtrNorms() { 27 | return this.ftrNorms; 28 | } 29 | 30 | @Override 31 | public String name() { 32 | return wrapped.name(); 33 | } 34 | 35 | @Override 36 | public FeatureVector newFeatureVector(FeatureVector reuse) { 37 | return wrapped.newFeatureVector(reuse); 38 | } 39 | 40 | @Override 41 | public float score(FeatureVector point) { 42 | for (Map.Entry ordToNorm: this.ftrNorms.entrySet()) { 43 | int ord = ordToNorm.getKey(); 44 | float origFtrScore = point.getFeatureScore(ord); 45 | float normed = ordToNorm.getValue().normalize(origFtrScore); 46 | point.setFeatureScore(ord, normed); 47 | } 48 | return wrapped.score(point); 49 | } 50 | 51 | @Override 52 | public boolean equals(Object other) { 53 | if (other == null) return false; 54 | if (!(other instanceof FeatureNormalizingRanker)) { 55 | return false; 56 | } 57 | final FeatureNormalizingRanker that = (FeatureNormalizingRanker)(other); 58 | if (that == null) return false; 59 | 60 | if (!that.ftrNorms.equals(this.ftrNorms)) return false; 61 | if (!that.wrapped.equals(this.wrapped)) return false; 62 | 63 | return true; 64 | } 65 | 66 | @Override 67 | public int hashCode() { 68 | return this.wrapped.hashCode() + 69 | (31 * this.ftrNorms.hashCode()); 70 | } 71 | 72 | @Override 73 | public long ramBytesUsed() { 74 | 75 | long ftrNormSize = ftrNorms.size() * (PER_FTR_NORM_RAM_USED); 76 | 77 | if (this.wrapped instanceof Accountable) { 78 | Accountable accountable = (Accountable)this.wrapped; 79 | return BASE_RAM_USED + accountable.ramBytesUsed() + ftrNormSize; 80 | } else { 81 | return BASE_RAM_USED + ftrNormSize; 82 | } 83 | } 84 | } -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/normalizer/MinMaxFeatureNormalizer.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.ranker.normalizer; 2 | 3 | /** 4 | * MinMax Feature Normalization 5 | * Generally following the standard laid out by sklearn: 6 | * (value / (max - min)) + min to give a normalized 0-1 feature value 7 | * 8 | * See 9 | * https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.MinMaxScaler.html 10 | */ 11 | public class MinMaxFeatureNormalizer implements Normalizer { 12 | float maximum; 13 | float minimum; 14 | 15 | public MinMaxFeatureNormalizer(float minimum, float maximum) { 16 | if (minimum >= maximum) { 17 | throw new IllegalArgumentException("Minimum " + Double.toString(minimum) + 18 | " must be smaller than than maximum: " + 19 | Double.toString(maximum)); 20 | } 21 | this.minimum = minimum; 22 | this.maximum = maximum; 23 | } 24 | 25 | @Override 26 | public float normalize(float value) { 27 | return (value - minimum) / (maximum - minimum); 28 | } 29 | 30 | @Override 31 | public boolean equals(Object other) { 32 | if (this == other) return true; 33 | if (!(other instanceof MinMaxFeatureNormalizer)) return false; 34 | MinMaxFeatureNormalizer that = (MinMaxFeatureNormalizer) other; 35 | 36 | if (this.minimum != that.minimum) return false; 37 | if (this.maximum != that.maximum) return false; 38 | 39 | return true; 40 | 41 | } 42 | 43 | @Override 44 | public int hashCode() { 45 | int hashCode = Float.hashCode(this.minimum); 46 | hashCode += 31 * Float.hashCode(this.maximum); 47 | return hashCode; 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/normalizer/Normalizer.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.ranker.normalizer; 2 | 3 | /** 4 | * Interface to normalize the resulting score of a model 5 | */ 6 | public interface Normalizer { 7 | float normalize(float val); 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/normalizer/Normalizers.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.ranker.normalizer; 2 | 3 | import java.util.Collections; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | /** 8 | * Class that manages Normalizer implementations 9 | */ 10 | public class Normalizers { 11 | private static final Map NORMALIZERS = Collections.unmodifiableMap(new HashMap() {{ 12 | put(NOOP_NORMALIZER_NAME, new NoopNormalizer()); 13 | put(SIGMOID_NORMALIZER_NAME, new SigmoidNormalizer()); 14 | }}); 15 | public static final String NOOP_NORMALIZER_NAME = "noop"; 16 | public static final String SIGMOID_NORMALIZER_NAME = "sigmoid"; 17 | 18 | public static Normalizer get(String name) { 19 | Normalizer normalizer = NORMALIZERS.get(name); 20 | if (normalizer == null) { 21 | throw new IllegalArgumentException(name + " is not a valid Normalizer"); 22 | } 23 | return normalizer; 24 | } 25 | 26 | public static boolean exists(String name) { 27 | return NORMALIZERS.containsKey(name); 28 | } 29 | 30 | static class NoopNormalizer implements Normalizer { 31 | @Override 32 | public float normalize(float val) { 33 | return val; 34 | } 35 | } 36 | 37 | static class SigmoidNormalizer implements Normalizer { 38 | @Override 39 | public float normalize(float val) { 40 | return sigmoid(val); 41 | } 42 | 43 | float sigmoid(float x) { 44 | return (float) (1 / (1 + Math.exp(-x))); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/normalizer/StandardFeatureNormalizer.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.ranker.normalizer; 2 | 3 | public class StandardFeatureNormalizer implements Normalizer { 4 | 5 | private float mean; 6 | private float stdDeviation; 7 | 8 | 9 | public StandardFeatureNormalizer(float mean, float stdDeviation) { 10 | this.mean = mean; 11 | this.stdDeviation = stdDeviation; 12 | } 13 | 14 | 15 | @Override 16 | public float normalize(float value) { 17 | return (value - this.mean) / this.stdDeviation; 18 | } 19 | 20 | @Override 21 | public boolean equals(Object other) { 22 | if (this == other) return true; 23 | if (!(other instanceof StandardFeatureNormalizer)) return false; 24 | StandardFeatureNormalizer that = (StandardFeatureNormalizer) other; 25 | 26 | if (this.mean != that.mean) return false; 27 | if (this.stdDeviation != that.stdDeviation) return false; 28 | 29 | return true; 30 | 31 | } 32 | 33 | @Override 34 | public int hashCode() { 35 | int hashCode = Float.hashCode(this.mean); 36 | hashCode += 31 * Float.hashCode(this.stdDeviation); 37 | return hashCode; 38 | } 39 | 40 | } -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/parser/LinearRankerParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.ranker.parser; 18 | 19 | import com.o19s.es.ltr.feature.FeatureSet; 20 | import com.o19s.es.ltr.ranker.linear.LinearRanker; 21 | import org.elasticsearch.common.ParsingException; 22 | import org.elasticsearch.xcontent.XContentParser; 23 | import org.elasticsearch.xcontent.json.JsonXContent; 24 | 25 | import java.io.IOException; 26 | 27 | import static org.elasticsearch.xcontent.XContentParserConfiguration.EMPTY; 28 | 29 | public class LinearRankerParser implements LtrRankerParser { 30 | public static final String TYPE = "model/linear"; 31 | 32 | @Override 33 | public LinearRanker parse(FeatureSet set, String model) { 34 | try (XContentParser parser = JsonXContent.jsonXContent.createParser(EMPTY, 35 | model) 36 | ) { 37 | return parse(parser, set); 38 | } catch (IOException e) { 39 | throw new IllegalArgumentException(e.getMessage(), e); 40 | } 41 | } 42 | 43 | private LinearRanker parse(XContentParser parser, FeatureSet set) throws IOException { 44 | float[] weights = new float[set.size()]; 45 | if (parser.nextToken() != XContentParser.Token.START_OBJECT) { 46 | throw new ParsingException(parser.getTokenLocation(), "Expected start object but found [" + parser.currentToken() +"]"); 47 | } 48 | while (parser.nextToken() == XContentParser.Token.FIELD_NAME) { 49 | String fname = parser.currentName(); 50 | if (!set.hasFeature(fname)) { 51 | throw new ParsingException(parser.getTokenLocation(), "Feature [" + fname + "] is unknown."); 52 | } 53 | if (parser.nextToken() != XContentParser.Token.VALUE_NUMBER) { 54 | throw new ParsingException(parser.getTokenLocation(), "Expected a float but found [" + parser.currentToken() +"]"); 55 | } 56 | weights[set.featureOrdinal(fname)] = parser.floatValue(); 57 | } 58 | assert parser.currentToken() == XContentParser.Token.END_OBJECT; 59 | return new LinearRanker(weights); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/parser/LtrRankerParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.ranker.parser; 18 | 19 | import com.o19s.es.ltr.feature.FeatureSet; 20 | import com.o19s.es.ltr.ranker.LtrRanker; 21 | 22 | /** 23 | * A model parser (don't have to be thread-safe) 24 | */ 25 | public interface LtrRankerParser { 26 | 27 | /** 28 | * Parse the model with the given {@link FeatureSet} 29 | * 30 | * @param set the feature-set to use to parse the model 31 | * @param model the model name 32 | * @return the {@link LtrRanker} parsed from the given model and feature-set 33 | */ 34 | LtrRanker parse(FeatureSet set, String model); 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/parser/LtrRankerParserFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.ranker.parser; 18 | 19 | import java.util.Collections; 20 | import java.util.HashMap; 21 | import java.util.Map; 22 | import java.util.function.Supplier; 23 | 24 | /** 25 | * LtrModel parser registry 26 | */ 27 | public class LtrRankerParserFactory { 28 | private final Map> parsers; 29 | 30 | private LtrRankerParserFactory(Map> parsers) { 31 | this.parsers = parsers; 32 | } 33 | 34 | /** 35 | * 36 | * @param type type or content-type like string defining the model format 37 | * @return a model parser 38 | * @throws IllegalArgumentException if the type is not supported 39 | */ 40 | public LtrRankerParser getParser(String type) { 41 | Supplier supplier = parsers.get(type); 42 | if (supplier == null) { 43 | throw new IllegalArgumentException("Unsupported LtrRanker format/type [" + type + "]"); 44 | } 45 | return supplier.get(); 46 | } 47 | 48 | public static class Builder { 49 | private final Map> registry = new HashMap<>(); 50 | 51 | public Builder register(String type, Supplier parser) { 52 | if (registry.put(type, parser) != null) { 53 | throw new RuntimeException("Cannot register LtrRankerParser: [" + type + "] already registered."); 54 | } 55 | return this; 56 | } 57 | 58 | public LtrRankerParserFactory build() { 59 | return new LtrRankerParserFactory(Collections.unmodifiableMap(registry)); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/ranklib/DenseProgramaticDataPoint.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2016] Doug Turnbull 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package com.o19s.es.ltr.ranker.ranklib; 18 | 19 | import ciir.umass.edu.learning.DataPoint; 20 | import ciir.umass.edu.utilities.RankLibError; 21 | import com.o19s.es.ltr.ranker.LtrRanker; 22 | 23 | import java.util.Arrays; 24 | 25 | /** 26 | * Implements FeatureVector but without needing to pass in a string 27 | * to be parsed 28 | */ 29 | public class DenseProgramaticDataPoint extends DataPoint implements LtrRanker.FeatureVector { 30 | private static final int RANKLIB_FEATURE_INDEX_OFFSET = 1; 31 | public DenseProgramaticDataPoint(int numFeatures) { 32 | this.fVals = new float[numFeatures+RANKLIB_FEATURE_INDEX_OFFSET]; // add 1 because RankLib features 1 based 33 | } 34 | 35 | public float getFeatureValue(int fid) { 36 | if(fid > 0 && fid < this.fVals.length) { 37 | return isUnknown(this.fVals[fid])?0.0F:this.fVals[fid]; 38 | } else { 39 | throw RankLibError.create("Error in DenseDataPoint::getFeatureValue(): requesting unspecified feature, fid=" + fid); 40 | } 41 | } 42 | 43 | public void setFeatureValue(int fid, float fval) { 44 | if(fid > 0 && fid < this.fVals.length) { 45 | this.fVals[fid] = fval; 46 | } else { 47 | throw RankLibError.create("Error in DenseDataPoint::setFeatureValue(): feature (id=" + fid + ") not found."); 48 | } 49 | } 50 | 51 | public void setFeatureVector(float[] dfVals) { 52 | this.fVals = dfVals; 53 | } 54 | 55 | public float[] getFeatureVector() { 56 | return this.fVals; 57 | } 58 | 59 | @Override 60 | public void setFeatureScore(int featureIdx, float score) { 61 | // add 1 because RankLib features 1 based 62 | this.setFeatureValue(featureIdx+1, score); 63 | } 64 | 65 | @Override 66 | public float getFeatureScore(int featureIdx) { 67 | // add 1 because RankLib features 1 based 68 | return this.getFeatureValue(featureIdx+1); 69 | } 70 | 71 | public void reset() { 72 | Arrays.fill(fVals, 0F); 73 | } 74 | 75 | public float getDefaultScore() { 76 | return 0.0F; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/ranklib/RankLibScriptEngine.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2016] Doug Turnbull 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | package com.o19s.es.ltr.ranker.ranklib; 18 | 19 | import com.o19s.es.ltr.ranker.LtrRanker; 20 | import com.o19s.es.ltr.ranker.parser.LtrRankerParserFactory; 21 | import org.elasticsearch.script.ScriptContext; 22 | import org.elasticsearch.script.ScriptEngine; 23 | 24 | import java.io.IOException; 25 | import java.util.Collections; 26 | import java.util.Map; 27 | import java.util.Objects; 28 | import java.util.Set; 29 | 30 | /** 31 | * Created by doug on 12/30/16. 32 | * The ranklib models are treated like scripts in that they 33 | * can be run inline, cached, or stored in files 34 | * However, we don't use the script query because we need to execute 35 | * many underlying queries. 36 | *

37 | * So this code acts as a hook for deserializing Ranklib models from ranklib XML 38 | * and as a convenient means for caching those deserialized model 39 | */ 40 | public class RankLibScriptEngine implements ScriptEngine { 41 | 42 | public static final String NAME = "ranklib"; 43 | public static final String EXTENSION = "ranklib"; 44 | public static final ScriptContext CONTEXT = 45 | new ScriptContext<>("ranklib", RankLibModelContainer.Factory.class); 46 | private final LtrRankerParserFactory factory; 47 | 48 | public RankLibScriptEngine(LtrRankerParserFactory factory) { 49 | super(); 50 | this.factory = Objects.requireNonNull(factory); 51 | } 52 | 53 | 54 | @Override 55 | public String getType() { 56 | return NAME; 57 | } 58 | 59 | @Override 60 | public T compile(String scriptName, String scriptSource, ScriptContext context, Map params) { 61 | 62 | RankLibModelContainer.Factory retFactory = () -> { 63 | LtrRanker ltrRanker = factory.getParser(RanklibModelParser.TYPE).parse(null, scriptSource); 64 | return new RankLibModelContainer(ltrRanker); 65 | }; 66 | 67 | return context.factoryClazz.cast(retFactory); 68 | } 69 | 70 | @Override 71 | public void close() throws IOException { 72 | 73 | } 74 | 75 | @Override 76 | public Set> getSupportedContexts() { 77 | return Collections.singleton(RankLibScriptEngine.CONTEXT); 78 | } 79 | 80 | public static class RankLibModelContainer { 81 | 82 | public interface Factory { 83 | RankLibModelContainer newInstance(); 84 | } 85 | 86 | LtrRanker _ranker; 87 | 88 | public RankLibModelContainer(LtrRanker ranker) { 89 | _ranker = ranker; 90 | } 91 | 92 | public void setNextVar(String name, Object value) { 93 | _ranker = (LtrRanker) (value); 94 | 95 | } 96 | 97 | public Object run() { 98 | return _ranker; 99 | } 100 | 101 | public void execute () {} 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/ranklib/RanklibModelParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Doug Turnbull, Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.ranker.ranklib; 18 | 19 | import ciir.umass.edu.learning.Ranker; 20 | import ciir.umass.edu.learning.RankerFactory; 21 | import com.o19s.es.ltr.feature.FeatureSet; 22 | import com.o19s.es.ltr.ranker.LtrRanker; 23 | import com.o19s.es.ltr.ranker.parser.LtrRankerParser; 24 | 25 | /** 26 | * Load a ranklib model from a script file, mostly a wrapper around the 27 | * existing script that complies with the {@link LtrRankerParser} interface 28 | */ 29 | public class RanklibModelParser implements LtrRankerParser { 30 | public static final String TYPE = "model/ranklib"; 31 | private final RankerFactory factory; 32 | 33 | public RanklibModelParser(RankerFactory factory) { 34 | this.factory = factory; 35 | } 36 | 37 | @Override 38 | public LtrRanker parse(FeatureSet set, String model) { 39 | Ranker ranklibRanker = factory.loadRankerFromString(model); 40 | int numFeatures = ranklibRanker.getFeatures().length; 41 | if (set != null) { 42 | numFeatures = set.size(); 43 | } 44 | return new RanklibRanker(ranklibRanker, numFeatures); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/ranker/ranklib/RanklibRanker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.ranker.ranklib; 18 | 19 | import ciir.umass.edu.learning.Ranker; 20 | import com.o19s.es.ltr.ranker.LtrRanker; 21 | 22 | public class RanklibRanker implements LtrRanker { 23 | private final Ranker ranker; 24 | private final int featureSetSize; 25 | 26 | public RanklibRanker(Ranker ranker, int featureSetSize) { 27 | this.ranker = ranker; 28 | this.featureSetSize = featureSetSize; 29 | } 30 | 31 | /** 32 | * LtrModel name 33 | * 34 | * @return the name of the model 35 | */ 36 | @Override 37 | public String name() { 38 | return ranker.name(); 39 | } 40 | 41 | /** 42 | * data point implementation used by this ranker 43 | * A data point is used to store feature scores. 44 | * A single instance will be created for every Scorer. 45 | * The implementation must not be thread-safe. 46 | * 47 | * @return LtrRanker data point implementation 48 | */ 49 | @Override 50 | public DenseProgramaticDataPoint newFeatureVector(FeatureVector reuse) { 51 | if (reuse != null) { 52 | assert reuse instanceof DenseProgramaticDataPoint; 53 | DenseProgramaticDataPoint vector = (DenseProgramaticDataPoint) reuse; 54 | vector.reset(); 55 | return vector; 56 | } 57 | // ranklib models are 1-based 58 | return new DenseProgramaticDataPoint(featureSetSize); 59 | } 60 | 61 | /** 62 | * Score the data point. 63 | * At this point all feature scores are set. 64 | * features that did not match are set with a score to 0 65 | * 66 | * @param point the populated data point 67 | * @return the score 68 | */ 69 | @Override 70 | public float score(FeatureVector point) { 71 | assert point instanceof DenseProgramaticDataPoint; 72 | return (float) ranker.eval((DenseProgramaticDataPoint) point); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/rest/AutoDetectParser.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.rest; 2 | 3 | import com.o19s.es.ltr.feature.FeatureValidation; 4 | import com.o19s.es.ltr.feature.store.StorableElement; 5 | import com.o19s.es.ltr.feature.store.StoredFeature; 6 | import com.o19s.es.ltr.feature.store.StoredFeatureSet; 7 | import com.o19s.es.ltr.feature.store.StoredLtrModel; 8 | import org.elasticsearch.xcontent.ParseField; 9 | import org.elasticsearch.common.ParsingException; 10 | import org.elasticsearch.xcontent.ObjectParser; 11 | import org.elasticsearch.xcontent.XContentParser; 12 | 13 | import java.io.IOException; 14 | 15 | import static com.o19s.es.ltr.query.ValidatingLtrQueryBuilder.SUPPORTED_TYPES; 16 | import static java.util.stream.Collectors.joining; 17 | 18 | class AutoDetectParser { 19 | private String expectedName; 20 | private StorableElement element; 21 | private FeatureValidation validation; 22 | 23 | private static final ObjectParser PARSER = new ObjectParser<>("storable_elements"); 24 | 25 | static { 26 | PARSER.declareObject(AutoDetectParser::setElement, 27 | StoredFeature::parse, 28 | new ParseField(StoredFeature.TYPE)); 29 | PARSER.declareObject(AutoDetectParser::setElement, 30 | StoredFeatureSet::parse, 31 | new ParseField(StoredFeatureSet.TYPE)); 32 | PARSER.declareObject(AutoDetectParser::setElement, 33 | StoredLtrModel::parse, 34 | new ParseField(StoredLtrModel.TYPE)); 35 | PARSER.declareObject((b, v) -> b.validation = v, 36 | (p, c) -> FeatureValidation.PARSER.apply(p, null), 37 | new ParseField("validation")); 38 | } 39 | 40 | AutoDetectParser(String name) { 41 | this.expectedName = name; 42 | } 43 | 44 | public void parse(XContentParser parser) throws IOException { 45 | PARSER.parse(parser, this, expectedName); 46 | if (element == null) { 47 | throw new ParsingException(parser.getTokenLocation(), "Element of type [" + SUPPORTED_TYPES.stream().collect(joining(",")) + 48 | "] is mandatory."); 49 | } 50 | } 51 | 52 | public StorableElement getElement() { 53 | return element; 54 | } 55 | 56 | public void setElement(StorableElement element) { 57 | if (this.element != null) { 58 | throw new IllegalArgumentException("[" + element.type() + "] already set, only one element can be set at a time (" + 59 | SUPPORTED_TYPES.stream().collect(joining(",")) + ")."); 60 | } 61 | this.element = element; 62 | } 63 | 64 | public void setValidation(FeatureValidation validation) { 65 | this.validation = validation; 66 | } 67 | 68 | public FeatureValidation getValidation() { 69 | return validation; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/rest/FeatureStoreBaseRestHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.rest; 18 | 19 | import com.o19s.es.ltr.feature.store.index.IndexFeatureStore; 20 | import org.apache.logging.log4j.LogManager; 21 | import org.apache.logging.log4j.Logger; 22 | import org.elasticsearch.rest.BaseRestHandler; 23 | import org.elasticsearch.rest.RestRequest; 24 | 25 | public abstract class FeatureStoreBaseRestHandler extends BaseRestHandler { 26 | 27 | protected String indexName(RestRequest request) { 28 | if (request.hasParam("store")) { 29 | return IndexFeatureStore.STORE_PREFIX + request.param("store"); 30 | } else { 31 | return IndexFeatureStore.DEFAULT_STORE; 32 | } 33 | } 34 | 35 | protected Logger logger = LogManager.getLogger(getClass()); 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/rest/RestSearchStoreElements.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.rest; 2 | 3 | import org.elasticsearch.client.internal.node.NodeClient; 4 | import org.elasticsearch.index.query.BoolQueryBuilder; 5 | import org.elasticsearch.rest.RestRequest; 6 | import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener; 7 | 8 | import java.util.List; 9 | 10 | import static java.util.Arrays.asList; 11 | import static java.util.Collections.unmodifiableList; 12 | import static org.elasticsearch.index.query.QueryBuilders.boolQuery; 13 | import static org.elasticsearch.index.query.QueryBuilders.matchQuery; 14 | import static org.elasticsearch.index.query.QueryBuilders.termQuery; 15 | 16 | public class RestSearchStoreElements extends FeatureStoreBaseRestHandler { 17 | private final String type; 18 | 19 | public RestSearchStoreElements(String type) { 20 | this.type = type; 21 | } 22 | 23 | @Override 24 | public String getName() { 25 | return "Search for " + type + " elements in the LTR feature store"; 26 | } 27 | 28 | @Override 29 | public List routes() { 30 | return unmodifiableList(asList( 31 | new Route(RestRequest.Method.GET, "/_ltr/{store}/_" + type), 32 | new Route(RestRequest.Method.GET, "/_ltr/_" + type) 33 | )); 34 | } 35 | 36 | @Override 37 | protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { 38 | return search(client, type, indexName(request), request); 39 | } 40 | 41 | RestChannelConsumer search(NodeClient client, String type, String indexName, RestRequest request) { 42 | String prefix = request.param("prefix"); 43 | int from = request.paramAsInt("from", 0); 44 | int size = request.paramAsInt("size", 20); 45 | BoolQueryBuilder qb = boolQuery().filter(termQuery("type", type)); 46 | if (prefix != null && !prefix.isEmpty()) { 47 | qb.must(matchQuery("name.prefix", prefix)); 48 | } 49 | return (channel) -> client.prepareSearch(indexName) 50 | // .setTypes(IndexFeatureStore.ES_TYPE) 51 | .setQuery(qb) 52 | .setSize(size) 53 | .setFrom(from) 54 | .execute(new RestRefCountedChunkedToXContentListener<>(channel)); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/stats/LTRStat.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.stats; 2 | 3 | import java.util.function.Supplier; 4 | 5 | /** 6 | * A container for a stat provided by the plugin. Each instance is associated with 7 | * an underlying supplier. The stat instance also stores a flag to indicate whether 8 | * this is a cluster level or a node level stat. 9 | */ 10 | public class LTRStat { 11 | private final boolean clusterLevel; 12 | private final Supplier supplier; 13 | 14 | public LTRStat(boolean clusterLevel, Supplier supplier) { 15 | this.clusterLevel = clusterLevel; 16 | this.supplier = supplier; 17 | } 18 | 19 | public boolean isClusterLevel() { 20 | return clusterLevel; 21 | } 22 | 23 | public Object getStatValue() { 24 | return supplier.get(); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/stats/LTRStats.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.stats; 2 | 3 | import java.util.Map; 4 | import java.util.stream.Collectors; 5 | 6 | 7 | /** 8 | * This class is the main entry-point for access to the stats that the LTR plugin keeps track of. 9 | */ 10 | public class LTRStats { 11 | private final Map stats; 12 | 13 | 14 | public LTRStats(Map stats) { 15 | this.stats = stats; 16 | } 17 | 18 | public Map getStats() { 19 | return stats; 20 | } 21 | 22 | public LTRStat getStat(String key) throws IllegalArgumentException { 23 | LTRStat stat = stats.get(key); 24 | if (stat == null) { 25 | throw new IllegalArgumentException("Stat=\"" + key + "\" does not exist"); 26 | } 27 | return stat; 28 | } 29 | 30 | public Map getNodeStats() { 31 | return getClusterOrNodeStats(false); 32 | } 33 | 34 | public Map getClusterStats() { 35 | return getClusterOrNodeStats(true); 36 | } 37 | 38 | private Map getClusterOrNodeStats(Boolean isClusterStats) { 39 | return stats.entrySet() 40 | .stream() 41 | .filter(e -> e.getValue().isClusterLevel() == isClusterStats) 42 | .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/stats/StatName.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.stats; 2 | 3 | import java.util.Collections; 4 | import java.util.HashSet; 5 | import java.util.Set; 6 | 7 | public enum StatName { 8 | PLUGIN_STATUS("status"), 9 | STORES("stores"), 10 | CACHE("cache"); 11 | 12 | private final String name; 13 | 14 | StatName(String name) { 15 | this.name = name; 16 | } 17 | 18 | public String getName() { 19 | return name; 20 | } 21 | 22 | public static Set getTopLevelStatNames() { 23 | Set statNames = new HashSet<>(); 24 | statNames.add(PLUGIN_STATUS.name); 25 | statNames.add(STORES.name); 26 | statNames.add(CACHE.name); 27 | return Collections.unmodifiableSet(statNames); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/stats/suppliers/CacheStatsOnNodeSupplier.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.stats.suppliers; 2 | 3 | import com.o19s.es.ltr.feature.store.index.Caches; 4 | import org.elasticsearch.common.cache.Cache; 5 | 6 | import java.util.Collections; 7 | import java.util.HashMap; 8 | import java.util.Map; 9 | import java.util.function.Supplier; 10 | 11 | /** 12 | * Aggregate stats on the cache used by the plugin per node. 13 | */ 14 | public class CacheStatsOnNodeSupplier implements Supplier>> { 15 | private final Caches caches; 16 | 17 | public enum Stat { 18 | CACHE_FEATURE("feature"), 19 | CACHE_FEATURE_SET("featureset"), 20 | CACHE_MODEL("model"), 21 | 22 | CACHE_HIT_COUNT("hit_count"), 23 | CACHE_MISS_COUNT("miss_count"), 24 | CACHE_EVICTION_COUNT("eviction_count"), 25 | CACHE_ENTRY_COUNT("entry_count"), 26 | CACHE_MEMORY_USAGE_IN_BYTES("memory_usage_in_bytes"); 27 | 28 | private final String name; 29 | 30 | Stat(String name) { 31 | this.name = name; 32 | } 33 | 34 | public String getName() { 35 | return name; 36 | } 37 | } 38 | 39 | public CacheStatsOnNodeSupplier(Caches caches) { 40 | this.caches = caches; 41 | } 42 | 43 | @Override 44 | public Map> get() { 45 | Map> values = new HashMap<>(); 46 | values.put(Stat.CACHE_FEATURE.getName(), getCacheStats(caches.featureCache())); 47 | values.put(Stat.CACHE_FEATURE_SET.getName(), getCacheStats(caches.featureSetCache())); 48 | values.put(Stat.CACHE_MODEL.getName(), getCacheStats(caches.modelCache())); 49 | return Collections.unmodifiableMap(values); 50 | } 51 | 52 | private Map getCacheStats(Cache cache) { 53 | Map stat = new HashMap<>(); 54 | stat.put(Stat.CACHE_HIT_COUNT.getName(), cache.stats().getHits()); 55 | stat.put(Stat.CACHE_MISS_COUNT.getName(), cache.stats().getMisses()); 56 | stat.put(Stat.CACHE_EVICTION_COUNT.getName(), cache.stats().getEvictions()); 57 | stat.put(Stat.CACHE_ENTRY_COUNT.getName(), cache.count()); 58 | stat.put(Stat.CACHE_MEMORY_USAGE_IN_BYTES.getName(), cache.weight()); 59 | return Collections.unmodifiableMap(stat); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplier.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.stats.suppliers; 2 | 3 | import com.o19s.es.ltr.feature.store.index.IndexFeatureStore; 4 | import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; 5 | import org.elasticsearch.cluster.health.ClusterIndexHealth; 6 | import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; 7 | import org.elasticsearch.cluster.service.ClusterService; 8 | import org.elasticsearch.common.util.concurrent.ThreadContext; 9 | import org.elasticsearch.core.TimeValue; 10 | import java.util.Arrays; 11 | import java.util.Locale; 12 | import java.util.function.Supplier; 13 | 14 | /** 15 | * Supplier for an overall plugin health status. 16 | */ 17 | public class PluginHealthStatusSupplier implements Supplier { 18 | private static final String STATUS_GREEN = "green"; 19 | private static final String STATUS_YELLOW = "yellow"; 20 | private static final String STATUS_RED = "red"; 21 | 22 | private final ClusterService clusterService; 23 | private final IndexNameExpressionResolver indexNameExpressionResolver; 24 | 25 | public PluginHealthStatusSupplier(ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver) { 26 | this.clusterService = clusterService; 27 | ThreadContext threadContext = new ThreadContext(clusterService.getSettings()); 28 | this.indexNameExpressionResolver = indexNameExpressionResolver; 29 | } 30 | 31 | // currently it combines the store statuses to get the overall health 32 | // this may be enhanced to monitor other aspects of the plugin, such as, 33 | // if we implement the circuit breaker and if the breaker is open. 34 | @Override 35 | public String get() { 36 | return getAggregateStoresStatus(); 37 | } 38 | 39 | private String getAggregateStoresStatus() { 40 | String[] names = indexNameExpressionResolver.concreteIndexNames(clusterService.state(), 41 | new ClusterStateRequest(TimeValue.timeValueMinutes(1)).indices( 42 | IndexFeatureStore.DEFAULT_STORE, IndexFeatureStore.STORE_PREFIX + "*")); 43 | return Arrays.stream(names) 44 | .filter(IndexFeatureStore::isIndexStore) 45 | .map(this::getLtrStoreHealthStatus) 46 | .reduce(STATUS_GREEN, this::combineStatuses); 47 | } 48 | 49 | private String combineStatuses(String s1, String s2) { 50 | if (STATUS_RED.equals(s1) || STATUS_RED.equals(s2)) { 51 | return STATUS_RED; 52 | } else if (STATUS_YELLOW.equals(s1) || STATUS_YELLOW.equals(s2)) { 53 | return STATUS_YELLOW; 54 | } else { 55 | return STATUS_GREEN; 56 | } 57 | } 58 | 59 | public String getLtrStoreHealthStatus(String storeName) { 60 | ClusterIndexHealth indexHealth = new ClusterIndexHealth( 61 | clusterService.state().metadata().index(storeName), 62 | clusterService.state().getRoutingTable().index(storeName) 63 | ); 64 | 65 | return indexHealth.getStatus().name().toLowerCase(Locale.ROOT); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/utils/AbstractQueryBuilderUtils.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.utils; 2 | 3 | import org.elasticsearch.common.io.stream.StreamInput; 4 | import org.elasticsearch.common.io.stream.StreamOutput; 5 | import org.elasticsearch.index.query.QueryBuilder; 6 | 7 | import java.io.IOException; 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | 11 | /** 12 | * Contains a few methods copied from the AbstractQueryBuilder class. These methods are not accessible from sub classes 13 | * that do not reside in the same package. 14 | */ 15 | public class AbstractQueryBuilderUtils { 16 | 17 | private AbstractQueryBuilderUtils() { 18 | // Utility class with static methods only 19 | } 20 | 21 | public static void writeQueries(StreamOutput out, List queries) throws IOException { 22 | out.writeVInt(queries.size()); 23 | for (QueryBuilder query : queries) { 24 | out.writeNamedWriteable(query); 25 | } 26 | } 27 | 28 | public static List readQueries(StreamInput in) throws IOException { 29 | int size = in.readVInt(); 30 | List queries = new ArrayList<>(size); 31 | for (int i = 0; i < size; i++) { 32 | queries.add(in.readNamedWriteable(QueryBuilder.class)); 33 | } 34 | return queries; 35 | } 36 | 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/utils/CheckedBiFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.utils; 18 | 19 | @FunctionalInterface 20 | public interface CheckedBiFunction { 21 | R apply(T var1, U var2) throws E; 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/utils/FeatureStoreLoader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.utils; 18 | 19 | import com.o19s.es.ltr.feature.store.FeatureStore; 20 | import java.util.function.Supplier; 21 | import org.elasticsearch.client.internal.Client; 22 | 23 | @FunctionalInterface 24 | public interface FeatureStoreLoader { 25 | FeatureStore load(String storeName, Supplier clientSupplier); 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/utils/Scripting.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.o19s.es.ltr.utils; 17 | 18 | import org.elasticsearch.script.DoubleValuesScript; 19 | import org.elasticsearch.script.Script; 20 | import org.elasticsearch.script.ScriptService; 21 | import org.elasticsearch.script.ScriptType; 22 | 23 | import java.io.IOException; 24 | import java.util.Collections; 25 | 26 | public class Scripting { 27 | private static ScriptService scriptService = null; 28 | 29 | private Scripting() {} 30 | 31 | public static void initScriptService(ScriptService scriptService) { 32 | Scripting.scriptService = scriptService; 33 | } 34 | 35 | public static DoubleValuesScript compile(String scriptSource) throws IOException { 36 | if (Scripting.scriptService == null) { 37 | throw new IOException("Script service not initialized."); 38 | } 39 | 40 | Script script = new Script(ScriptType.INLINE, "expression", scriptSource, Collections.EMPTY_MAP); 41 | return scriptService.compile(script, DoubleValuesScript.CONTEXT).newInstance(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/ltr/utils/Suppliers.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.utils; 18 | 19 | import org.elasticsearch.common.CheckedSupplier; 20 | 21 | import java.util.Objects; 22 | import java.util.concurrent.atomic.AtomicReference; 23 | import java.util.function.Supplier; 24 | 25 | public final class Suppliers { 26 | /** 27 | * Utility class 28 | */ 29 | private Suppliers() {} 30 | 31 | /** 32 | * @param supplier the original supplier to store 33 | * @param the supplied type 34 | * @return a supplier storing and returning the same instance 35 | */ 36 | public static Supplier memoize(Supplier supplier) { 37 | return new MemoizeSupplier<>(supplier); 38 | } 39 | 40 | private static class MemoizeSupplier implements Supplier { 41 | private volatile boolean initialized = false; 42 | private final Supplier supplier; 43 | private E value; 44 | 45 | MemoizeSupplier(Supplier supplier) { 46 | this.supplier = Objects.requireNonNull(supplier); 47 | } 48 | 49 | @Override 50 | public E get() { 51 | if (!initialized) { 52 | synchronized (this) { 53 | if (!initialized) { 54 | E t = supplier.get(); 55 | value = t; 56 | initialized = true; 57 | return t; 58 | } 59 | } 60 | } 61 | return value; 62 | } 63 | } 64 | 65 | /** 66 | * A mutable supplier 67 | */ 68 | public static class MutableSupplier implements Supplier { 69 | private final AtomicReference ref = new AtomicReference<>(); 70 | 71 | @Override 72 | public T get() { 73 | return ref.get(); 74 | } 75 | 76 | public void set(T obj) { 77 | this.ref.set(obj); 78 | } 79 | } 80 | 81 | /** 82 | * memoize the return value of the checked supplier (thread unsafe) 83 | */ 84 | public static CheckedSupplier memoizeCheckedSupplier(CheckedSupplier supplier) { 85 | return new CheckedMemoizeSupplier(supplier); 86 | } 87 | 88 | private static class CheckedMemoizeSupplier implements CheckedSupplier { 89 | private final CheckedSupplier supplier; 90 | private R value; 91 | 92 | private CheckedMemoizeSupplier(CheckedSupplier supplier) { 93 | this.supplier = supplier; 94 | } 95 | 96 | @Override 97 | public R get() throws E { 98 | if (value == null) { 99 | value = supplier.get(); 100 | } 101 | return value; 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/com/o19s/es/template/mustache/MustacheUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.template.mustache; 18 | 19 | import com.github.mustachejava.Mustache; 20 | import com.github.mustachejava.MustacheException; 21 | import org.apache.logging.log4j.Logger; 22 | import org.apache.logging.log4j.message.ParameterizedMessage; 23 | import org.apache.logging.log4j.util.Supplier; 24 | import org.elasticsearch.SpecialPermission; 25 | import org.apache.logging.log4j.LogManager; 26 | 27 | 28 | import java.io.StringReader; 29 | import java.io.StringWriter; 30 | import java.security.AccessController; 31 | import java.security.PrivilegedAction; 32 | import java.util.Map; 33 | 34 | public class MustacheUtils { 35 | public static final String TEMPLATE_LANGUAGE = "mustache"; 36 | private static final Logger logger = LogManager.getLogger(MustacheUtils.class); 37 | private static final SpecialPermission SPECIAL_PERMS = new SpecialPermission(); 38 | /** 39 | * We store templates internally always as json 40 | */ 41 | private static final CustomMustacheFactory FACTORY = new CustomMustacheFactory(); 42 | 43 | public static Mustache compile(String name, String template) { 44 | // Don't use compile(String name) to avoid caching in the factory 45 | try { 46 | return FACTORY.compile(new StringReader(template), name); 47 | } catch (MustacheException me) { 48 | throw new IllegalArgumentException(me.getMessage(), me); 49 | } 50 | } 51 | 52 | public static String execute(Mustache template, Map params) { 53 | final StringWriter writer = new StringWriter(); 54 | try { 55 | // crazy reflection here 56 | SecurityManager sm = System.getSecurityManager(); 57 | if (sm != null) { 58 | sm.checkPermission(SPECIAL_PERMS); 59 | } 60 | AccessController.doPrivileged((PrivilegedAction) () -> { 61 | template.execute(writer, params); 62 | return null; 63 | }); 64 | } catch (Exception e) { 65 | logger.error((Supplier) () -> new ParameterizedMessage("Error running {}", template), e); 66 | throw new IllegalArgumentException("Error running " + template, e); 67 | } 68 | return writer.toString(); 69 | 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/plugin-metadata/plugin-security.policy: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to Elasticsearch under one or more contributor 3 | * license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright 5 | * ownership. Elasticsearch licenses this file to you under 6 | * the Apache License, Version 2.0 (the "License"); you may 7 | * not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | grant { 21 | // needed to generate runtime classes 22 | permission java.lang.RuntimePermission "getClassLoader"; 23 | 24 | // expression runtime 25 | permission org.elasticsearch.script.ClassPermission "java.lang.String"; 26 | permission org.elasticsearch.script.ClassPermission "org.apache.lucene.expressions.Expression"; 27 | permission org.elasticsearch.script.ClassPermission "org.apache.lucene.search.DoubleValues"; 28 | // available functions 29 | permission org.elasticsearch.script.ClassPermission "java.lang.Math"; 30 | permission org.elasticsearch.script.ClassPermission "org.apache.lucene.util.MathUtil"; 31 | permission org.elasticsearch.script.ClassPermission "org.apache.lucene.util.SloppyMath"; 32 | }; 33 | -------------------------------------------------------------------------------- /src/main/resources/com/o19s/es/ltr/feature/store/index/fstore-index-analysis.json: -------------------------------------------------------------------------------- 1 | { 2 | "analysis": { 3 | "analyzer": { 4 | "name_prefix": { 5 | "tokenizer": "ltr_keyword", 6 | "filter": [ 7 | "ltr_edge_ngram" 8 | ] 9 | }, 10 | "name_prefix_search": { 11 | "tokenizer": "ltr_keyword", 12 | "filter": [ 13 | "ltr_length" 14 | ] 15 | } 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /src/main/resources/com/o19s/es/ltr/feature/store/index/fstore-index-mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "dynamic": "strict", 3 | "properties": { 4 | "name": { 5 | "type": "text", 6 | "analyzer": "keyword", 7 | "fields": { 8 | "prefix": { 9 | "type": "text", 10 | "analyzer": "name_prefix", 11 | "search_analyzer": "name_prefix_search" 12 | } 13 | } 14 | }, 15 | "type": { 16 | "type": "keyword" 17 | }, 18 | "feature": { 19 | "type": "object", 20 | "enabled": "false" 21 | }, 22 | "featureset": { 23 | "type": "object", 24 | "enabled": "false" 25 | }, 26 | "model": { 27 | "type": "object", 28 | "enabled": "false" 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /src/test/java/com/o19s/es/explore/StatisticsHelperTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | */ 16 | package com.o19s.es.explore; 17 | 18 | import org.apache.lucene.tests.util.LuceneTestCase; 19 | 20 | public class StatisticsHelperTests extends LuceneTestCase { 21 | private final float[] dataset = new float[] { 22 | 0.0f, -5.0f, 10.0f, 5.0f 23 | }; 24 | 25 | public void testStats() throws Exception { 26 | StatisticsHelper stats = new StatisticsHelper(); 27 | 28 | for(float f : dataset) { 29 | stats.add(f); 30 | } 31 | 32 | assertEquals(10.0f, stats.getMax(), 0.0f); 33 | assertEquals(-5.0f, stats.getMin(), 0.0f); 34 | assertEquals(2.5f, stats.getMean(), 0.0f); 35 | assertEquals(5.59f, stats.getStdDev(), 0.009f); 36 | assertEquals(31.25f, stats.getVariance(), 0.009f); 37 | } 38 | 39 | public void testSingleElement() throws Exception { 40 | StatisticsHelper stats = new StatisticsHelper(); 41 | 42 | stats.add(42.0f); 43 | 44 | assertEquals(42.0f, stats.getMax(), 0.0f); 45 | assertEquals(42.0f, stats.getMin(), 0.0f); 46 | assertEquals(42.0f, stats.getMean(), 0.0f); 47 | assertEquals(0.0f, stats.getStdDev(), 0.0f); 48 | assertEquals(0.0f, stats.getVariance(), 0.0f); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/test/java/com/o19s/es/ltr/LtrQueryContextTests.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr; 2 | 3 | import org.apache.lucene.tests.util.LuceneTestCase; 4 | 5 | import java.util.Arrays; 6 | import java.util.Collections; 7 | import java.util.HashSet; 8 | 9 | /* 10 | * Copyright [2017] Wikimedia Foundation 11 | * 12 | * Licensed under the Apache License, Version 2.0 (the "License"); 13 | * you may not use this file except in compliance with the License. 14 | * You may obtain a copy of the License at 15 | * 16 | * http://www.apache.org/licenses/LICENSE-2.0 17 | * 18 | * Unless required by applicable law or agreed to in writing, software 19 | * distributed under the License is distributed on an "AS IS" BASIS, 20 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 21 | * See the License for the specific language governing permissions and 22 | * limitations under the License. 23 | */ 24 | 25 | public class LtrQueryContextTests extends LuceneTestCase { 26 | 27 | public void testIsFeatureActiveForNull() { 28 | LtrQueryContext ltrContext = new LtrQueryContext(null, null); 29 | assertTrue(ltrContext.isFeatureActive("feature")); 30 | } 31 | 32 | public void testIsFeatureActiveForEmptySet() { 33 | LtrQueryContext ltrContext = new LtrQueryContext(null, Collections.emptySet()); 34 | assertTrue(ltrContext.isFeatureActive("feature")); 35 | } 36 | 37 | public void testIsFeatureActiveTrue() { 38 | LtrQueryContext ltrContext = new LtrQueryContext(null, Collections.singleton("feature")); 39 | assertTrue(ltrContext.isFeatureActive("feature")); 40 | } 41 | 42 | public void testIsFeatureActiveFalse() { 43 | LtrQueryContext ltrContext = new LtrQueryContext(null, Collections.singleton("feature1")); 44 | assertFalse(ltrContext.isFeatureActive("feature2")); 45 | } 46 | 47 | public void testGetActiveFeaturesForNull() { 48 | LtrQueryContext ltrContext = new LtrQueryContext(null, null); 49 | assertEquals(Collections.emptySet(), ltrContext.getActiveFeatures()); 50 | } 51 | 52 | public void testGetActiveFeatures() { 53 | HashSet features = new HashSet<>(Arrays.asList("feature1", "feature2")); 54 | LtrQueryContext ltrContext = new LtrQueryContext(null, features); 55 | assertEquals(features, ltrContext.getActiveFeatures()); 56 | } 57 | 58 | } 59 | 60 | -------------------------------------------------------------------------------- /src/test/java/com/o19s/es/ltr/action/TransportLTRStatsActionTests.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.action; 2 | 3 | import com.o19s.es.ltr.action.LTRStatsAction.LTRStatsNodeRequest; 4 | import com.o19s.es.ltr.action.LTRStatsAction.LTRStatsNodeResponse; 5 | import com.o19s.es.ltr.action.LTRStatsAction.LTRStatsNodesRequest; 6 | import com.o19s.es.ltr.action.LTRStatsAction.LTRStatsNodesResponse; 7 | import com.o19s.es.ltr.stats.LTRStat; 8 | import com.o19s.es.ltr.stats.LTRStats; 9 | import com.o19s.es.ltr.stats.StatName; 10 | import org.elasticsearch.action.FailedNodeException; 11 | import org.elasticsearch.action.support.ActionFilters; 12 | import org.elasticsearch.test.ESIntegTestCase; 13 | import org.elasticsearch.transport.TransportService; 14 | import org.junit.Before; 15 | 16 | import java.util.ArrayList; 17 | import java.util.HashMap; 18 | import java.util.List; 19 | import java.util.Map; 20 | 21 | import static org.mockito.Mockito.mock; 22 | 23 | public class TransportLTRStatsActionTests extends ESIntegTestCase { 24 | 25 | private TransportLTRStatsAction action; 26 | private LTRStats ltrStats; 27 | private Map statsMap; 28 | 29 | @Before 30 | public void setup() throws Exception { 31 | super.setUp(); 32 | 33 | statsMap = new HashMap<>(); 34 | statsMap.put(StatName.PLUGIN_STATUS.getName(), new LTRStat(false, () -> "cluster_stat")); 35 | statsMap.put(StatName.CACHE.getName(), new LTRStat(true, () -> "node_stat")); 36 | 37 | ltrStats = new LTRStats(statsMap); 38 | 39 | action = new TransportLTRStatsAction( 40 | client().threadPool(), 41 | clusterService(), 42 | mock(TransportService.class), 43 | mock(ActionFilters.class), 44 | ltrStats 45 | ); 46 | } 47 | 48 | public void testNewResponse() { 49 | String[] nodeIds = null; 50 | LTRStatsNodesRequest ltrStatsRequest = new LTRStatsNodesRequest(nodeIds); 51 | ltrStatsRequest.setStatsToBeRetrieved(ltrStats.getStats().keySet()); 52 | 53 | List responses = new ArrayList<>(); 54 | List failures = new ArrayList<>(); 55 | 56 | LTRStatsNodesResponse ltrStatsResponse = action.newResponse(ltrStatsRequest, responses, failures); 57 | assertEquals(1, ltrStatsResponse.getClusterStats().size()); 58 | } 59 | 60 | public void testNodeOperation() { 61 | String[] nodeIds = null; 62 | LTRStatsNodesRequest ltrStatsRequest = new LTRStatsNodesRequest(nodeIds); 63 | ltrStatsRequest.setStatsToBeRetrieved(ltrStats.getStats().keySet()); 64 | 65 | LTRStatsNodeResponse response = action.nodeOperation(new LTRStatsNodeRequest(ltrStatsRequest), null); 66 | 67 | Map stats = response.getStatsMap(); 68 | 69 | assertEquals(1, stats.size()); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/test/java/com/o19s/es/ltr/feature/store/ExtraLoggingSupplierTests.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.feature.store; 2 | 3 | import com.o19s.es.ltr.ranker.LogLtrRanker; 4 | import org.apache.lucene.tests.util.LuceneTestCase; 5 | 6 | import java.util.HashMap; 7 | import java.util.Map; 8 | 9 | public class ExtraLoggingSupplierTests extends LuceneTestCase { 10 | public void testGetWithConsumerNotSet() { 11 | ExtraLoggingSupplier supplier = new ExtraLoggingSupplier(); 12 | assertNull(supplier.get()); 13 | } 14 | 15 | public void testGetWillNullConsumerSet() { 16 | ExtraLoggingSupplier supplier = new ExtraLoggingSupplier(); 17 | supplier.setSupplier(null); 18 | assertNull(supplier.get()); 19 | } 20 | 21 | public void testGetWithSuppliedNull() { 22 | ExtraLoggingSupplier supplier = new ExtraLoggingSupplier(); 23 | supplier.setSupplier(() -> null); 24 | assertNull(supplier.get()); 25 | } 26 | 27 | public void testGetWithSuppliedMap() { 28 | Map extraLoggingMap = new HashMap<>(); 29 | 30 | LogLtrRanker.LogConsumer consumer = new LogLtrRanker.LogConsumer() { 31 | @Override 32 | public void accept(int featureOrdinal, float score) {} 33 | 34 | @Override 35 | public Map getExtraLoggingMap() { 36 | return extraLoggingMap; 37 | } 38 | }; 39 | 40 | ExtraLoggingSupplier supplier = new ExtraLoggingSupplier(); 41 | supplier.setSupplier(consumer::getExtraLoggingMap); 42 | assertTrue(supplier.get() == extraLoggingMap); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/test/java/com/o19s/es/ltr/feature/store/MemStore.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.feature.store; 18 | 19 | import com.o19s.es.ltr.feature.Feature; 20 | import com.o19s.es.ltr.feature.FeatureSet; 21 | 22 | import java.io.IOException; 23 | import java.util.HashMap; 24 | import java.util.Map; 25 | 26 | /** 27 | * in memory test store 28 | */ 29 | public class MemStore implements FeatureStore { 30 | private final Map features = new HashMap<>(); 31 | private final Map sets = new HashMap<>(); 32 | private final Map models = new HashMap<>(); 33 | 34 | private final String storeName; 35 | 36 | public MemStore(String storeName) { 37 | this.storeName = storeName; 38 | } 39 | 40 | public MemStore() { 41 | this("memstore"); 42 | } 43 | 44 | @Override 45 | public String getStoreName() { 46 | return storeName; 47 | } 48 | 49 | @Override 50 | public Feature load(String id) throws IOException { 51 | StoredFeature feature = features.get(id); 52 | if (feature == null) { 53 | throw new IllegalArgumentException("Feature [" + id + "] not found"); 54 | } 55 | return feature.optimize(); 56 | } 57 | 58 | @Override 59 | public FeatureSet loadSet(String id) throws IOException { 60 | StoredFeatureSet set = sets.get(id); 61 | if (set == null) { 62 | throw new IllegalArgumentException("Feature [" + id + "] not found"); 63 | } 64 | return set.optimize(); 65 | } 66 | 67 | @Override 68 | public CompiledLtrModel loadModel(String id) throws IOException { 69 | CompiledLtrModel model = models.get(id); 70 | if (model == null) { 71 | throw new IllegalArgumentException("Feature [" + id + "] not found"); 72 | } 73 | return model; 74 | } 75 | 76 | public void add(StoredFeature feature) { 77 | features.put(feature.name(), feature); 78 | } 79 | 80 | public void add(StoredFeatureSet set) { 81 | sets.put(set.name(), set); 82 | } 83 | 84 | public void add(CompiledLtrModel model) { 85 | models.put(model.name(), model); 86 | } 87 | 88 | public void clear() { 89 | features.clear(); 90 | sets.clear(); 91 | models.clear(); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/test/java/com/o19s/es/ltr/ranker/DenseFeatureVectorTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.ranker; 18 | 19 | import org.apache.lucene.tests.util.LuceneTestCase; 20 | 21 | public class DenseFeatureVectorTests extends LuceneTestCase { 22 | public void testConstructor() { 23 | int size = 10; 24 | DenseFeatureVector featureVector = new DenseFeatureVector(size); 25 | for (float score : featureVector.scores) { 26 | assertEquals(0F, score, Math.ulp(0F)); 27 | } 28 | } 29 | 30 | public void testSetGetReset() { 31 | int size = 10; 32 | DenseFeatureVector featureVector = new DenseFeatureVector(size); 33 | featureVector.setFeatureScore(5, 3.15F); 34 | 35 | assertEquals(3.15F, featureVector.getFeatureScore(5), Math.ulp(3.15F)); 36 | assertEquals(0F, featureVector.getFeatureScore(0), Math.ulp(0F)); 37 | 38 | featureVector.reset(); 39 | 40 | for (int featureId = 0; featureId < size; featureId++) { 41 | assertEquals(0F, featureVector.getFeatureScore(featureId), Math.ulp(0F)); 42 | } 43 | } 44 | 45 | public void testGetDefaultValue() { 46 | assertEquals(0F, new DenseFeatureVector(10).getDefaultScore(), Math.ulp(0F)); 47 | } 48 | 49 | } -------------------------------------------------------------------------------- /src/test/java/com/o19s/es/ltr/ranker/DenseLtrRankerTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.ranker; 18 | 19 | import org.apache.lucene.tests.util.LuceneTestCase; 20 | 21 | public class DenseLtrRankerTests extends LuceneTestCase { 22 | public void newFeatureVector() throws Exception { 23 | int modelSize = random().nextInt(10); 24 | DummyDenseRanker ranker = new DummyDenseRanker(modelSize); 25 | DenseFeatureVector vector = ranker.newFeatureVector(null); 26 | assertNotNull(vector); 27 | for(int i = 0; i < modelSize; i++) { 28 | assertEquals(0, vector.getFeatureScore(0), Math.ulp(0)); 29 | } 30 | float[] points = vector.scores; 31 | assertEquals(points.length, 2); 32 | 33 | for(int i = 0; i < modelSize; i++) { 34 | vector.setFeatureScore(0, random().nextFloat()); 35 | } 36 | LtrRanker.FeatureVector vector2 = ranker.newFeatureVector(vector); 37 | assertSame(vector, vector2); 38 | for(int i = 0; i < modelSize; i++) { 39 | assertEquals(0, vector.getFeatureScore(0), Math.ulp(0)); 40 | } 41 | } 42 | 43 | private static class DummyDenseRanker extends DenseLtrRanker { 44 | private final int modelSize; 45 | 46 | private DummyDenseRanker(int modelSize) { 47 | this.modelSize = modelSize; 48 | } 49 | 50 | @Override 51 | protected float score(DenseFeatureVector vector) { 52 | return 0; 53 | } 54 | 55 | @Override 56 | protected int size() { 57 | return modelSize; 58 | } 59 | 60 | @Override 61 | public String name() { 62 | return "dummy"; 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /src/test/java/com/o19s/es/ltr/ranker/LogLtrRankerTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.ranker; 18 | 19 | import com.o19s.es.ltr.ranker.linear.LinearRankerTests; 20 | import org.apache.lucene.tests.util.LuceneTestCase; 21 | import org.apache.lucene.tests.util.TestUtil; 22 | 23 | public class LogLtrRankerTests extends LuceneTestCase { 24 | public void testNewFeatureVector() throws Exception { 25 | int modelSize = TestUtil.nextInt(random(), 1, 20); 26 | 27 | final float[] expectedScores = new float[modelSize]; 28 | LinearRankerTests.fillRandomWeights(expectedScores); 29 | 30 | final float[] actualScores = new float[modelSize]; 31 | LogLtrRanker ranker = new LogLtrRanker(new NullRanker(modelSize), (i, s) -> actualScores[i] = s); 32 | LtrRanker.FeatureVector vector = ranker.newFeatureVector(null); 33 | for (int i = 0; i < expectedScores.length; i++) { 34 | vector.setFeatureScore(i, expectedScores[i]); 35 | } 36 | assertArrayEquals(expectedScores, actualScores, 0F); 37 | } 38 | 39 | public void score() throws Exception { 40 | int modelSize = TestUtil.nextInt(random(), 1, 20); 41 | LtrRanker ranker = new NullRanker(modelSize); 42 | LogLtrRanker logRanker = new LogLtrRanker(new NullRanker(modelSize), (i, s) -> {}); 43 | int nPass = TestUtil.nextInt(random(), 100, 200); 44 | float[] scores = new float[modelSize]; 45 | 46 | while (nPass-- > 0) { 47 | LinearRankerTests.fillRandomWeights(scores); 48 | LtrRanker.FeatureVector vect1 = ranker.newFeatureVector(null); 49 | LtrRanker.FeatureVector vect2 = logRanker.newFeatureVector(null); 50 | } 51 | 52 | } 53 | 54 | } -------------------------------------------------------------------------------- /src/test/java/com/o19s/es/ltr/ranker/SparseFeatureVectorTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.ranker; 18 | 19 | import org.apache.lucene.tests.util.LuceneTestCase; 20 | 21 | public class SparseFeatureVectorTests extends LuceneTestCase { 22 | public void testConstructor() { 23 | int size = 10; 24 | SparseFeatureVector featureVector = new SparseFeatureVector(size); 25 | for (float score : featureVector.scores) { 26 | assertTrue(Float.isNaN(score)); 27 | } 28 | } 29 | 30 | public void testSetGetReset() { 31 | int size = 10; 32 | SparseFeatureVector featureVector = new SparseFeatureVector(size); 33 | featureVector.setFeatureScore(5, 3.15F); 34 | 35 | assertEquals(3.15F, featureVector.getFeatureScore(5), Math.ulp(3.15F)); 36 | assertTrue(Float.isNaN(featureVector.getFeatureScore(0))); 37 | 38 | featureVector.reset(); 39 | 40 | for (int featureId = 0; featureId < size; featureId++) { 41 | assertTrue(Float.isNaN(featureVector.getFeatureScore(featureId))); 42 | } 43 | } 44 | 45 | public void testGetDefaultValue() { 46 | assertTrue(Float.isNaN(new SparseFeatureVector(10).getDefaultScore())); 47 | } 48 | 49 | } -------------------------------------------------------------------------------- /src/test/java/com/o19s/es/ltr/ranker/normalizer/NormalizersTests.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.ranker.normalizer; 2 | 3 | import org.apache.lucene.tests.util.LuceneTestCase; 4 | import org.hamcrest.CoreMatchers; 5 | 6 | public class NormalizersTests extends LuceneTestCase { 7 | 8 | public void testGet() { 9 | assertEquals(Normalizers.get(Normalizers.SIGMOID_NORMALIZER_NAME).getClass(), Normalizers.SigmoidNormalizer.class); 10 | assertEquals(Normalizers.get(Normalizers.NOOP_NORMALIZER_NAME).getClass(), Normalizers.NoopNormalizer.class); 11 | } 12 | 13 | public void testInvalidName() { 14 | assertThat(expectThrows(IllegalArgumentException.class, () -> Normalizers.get("not_normalizer")).getMessage(), 15 | CoreMatchers.containsString("is not a valid Normalizer")); 16 | } 17 | 18 | public void testExists() { 19 | assertTrue(Normalizers.exists(Normalizers.NOOP_NORMALIZER_NAME)); 20 | assertTrue(Normalizers.exists("sigmoid")); 21 | assertFalse(Normalizers.exists("not_normalizer")); 22 | } 23 | 24 | public void testNormalize() { 25 | assertEquals(Normalizers.get(Normalizers.NOOP_NORMALIZER_NAME).normalize(0.2f), 0.2f, Math.ulp(0.2f)); 26 | assertEquals(Normalizers.get(Normalizers.NOOP_NORMALIZER_NAME).normalize(-0.5f), -0.5f, Math.ulp(-0.5f)); 27 | 28 | assertEquals(Normalizers.get(Normalizers.SIGMOID_NORMALIZER_NAME).normalize(0.2f), 0.549834f, Math.ulp(0.549834f)); 29 | assertEquals(Normalizers.get(Normalizers.SIGMOID_NORMALIZER_NAME).normalize(-0.5f), 0.37754068f, Math.ulp(0.37754068f)); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/test/java/com/o19s/es/ltr/ranker/parser/LtrRankerParserFactoryTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.ranker.parser; 18 | 19 | import org.apache.lucene.tests.util.LuceneTestCase; 20 | 21 | import static org.hamcrest.CoreMatchers.containsString; 22 | 23 | public class LtrRankerParserFactoryTests extends LuceneTestCase { 24 | public void testGetParser() { 25 | LtrRankerParser parser = (set, model) -> null; 26 | LtrRankerParserFactory factory = new LtrRankerParserFactory.Builder() 27 | .register("model/test", () -> parser) 28 | .build(); 29 | assertSame(parser, factory.getParser("model/test")); 30 | assertThat(expectThrows(IllegalArgumentException.class, 31 | () -> factory.getParser("model/foobar")).getMessage(), 32 | containsString("Unsupported LtrRanker format/type [model/foobar]")); 33 | } 34 | 35 | public void testDeclareMultiple() { 36 | LtrRankerParser parser = (set, model) -> null; 37 | LtrRankerParserFactory.Builder builder = new LtrRankerParserFactory.Builder() 38 | .register("model/test", () -> parser); 39 | expectThrows(RuntimeException.class, 40 | () -> builder.register("model/test", () -> parser)); 41 | } 42 | 43 | } -------------------------------------------------------------------------------- /src/test/java/com/o19s/es/ltr/rest/FeaturesParserTests.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.o19s.es.ltr.rest; 18 | 19 | import org.apache.lucene.tests.util.LuceneTestCase; 20 | import org.elasticsearch.xcontent.XContentParser; 21 | import org.elasticsearch.xcontent.XContentParserConfiguration; 22 | 23 | import java.io.IOException; 24 | import java.util.stream.Collectors; 25 | import java.util.stream.IntStream; 26 | 27 | import static com.o19s.es.ltr.feature.store.StoredFeatureParserTests.generateTestFeature; 28 | import static org.elasticsearch.xcontent.json.JsonXContent.jsonXContent; 29 | 30 | public class FeaturesParserTests extends LuceneTestCase { 31 | public void testParseArray() throws IOException { 32 | RestAddFeatureToSet.FeaturesParserState fparser = new RestAddFeatureToSet.FeaturesParserState(); 33 | int nFeat = random().nextInt(18)+1; 34 | String featuresArray = IntStream.range(0, nFeat) 35 | .mapToObj((i) -> generateTestFeature("feat" + i)) 36 | .collect(Collectors.joining(",")); 37 | XContentParser parser = jsonXContent.createParser(XContentParserConfiguration.EMPTY, 38 | "{\"features\":[" + featuresArray + "]}"); 39 | fparser.parse(parser); 40 | assertEquals(nFeat, fparser.getFeatures().size()); 41 | assertEquals("feat0", fparser.getFeatures().get(0).name()); 42 | } 43 | } -------------------------------------------------------------------------------- /src/test/java/com/o19s/es/ltr/stats/LTRStatTests.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.stats; 2 | 3 | import org.elasticsearch.test.ESTestCase; 4 | 5 | public class LTRStatTests extends ESTestCase { 6 | public void testIsClusterLevel() { 7 | LTRStat stat1 = new LTRStat(true, () -> "test"); 8 | assertTrue(stat1.isClusterLevel()); 9 | 10 | LTRStat stat2 = new LTRStat(false, () -> "test"); 11 | assertFalse(stat2.isClusterLevel()); 12 | } 13 | 14 | public void testGetValue() { 15 | LTRStat stat2 = new LTRStat(false, () -> "test"); 16 | assertEquals("test", stat2.getStatValue()); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/test/java/com/o19s/es/ltr/stats/LTRStatsTests.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.stats; 2 | 3 | import org.elasticsearch.test.ESTestCase; 4 | import org.junit.Before; 5 | 6 | import java.util.HashMap; 7 | import java.util.HashSet; 8 | import java.util.Map; 9 | import java.util.Set; 10 | 11 | public class LTRStatsTests extends ESTestCase { 12 | 13 | private Map statsMap; 14 | private LTRStats ltrStats; 15 | 16 | @Before 17 | public void setup() { 18 | statsMap = new HashMap<>(); 19 | statsMap.put(StatName.PLUGIN_STATUS.getName(), new LTRStat(true, () -> "test")); 20 | statsMap.put(StatName.CACHE.getName(), new LTRStat(false, () -> "test")); 21 | ltrStats = new LTRStats(statsMap); 22 | } 23 | 24 | public void testGetStats() { 25 | Map stats = ltrStats.getStats(); 26 | assertEquals(stats.size(), statsMap.size()); 27 | 28 | for (Map.Entry stat : stats.entrySet()) { 29 | assertStatPresence(stat.getKey(), stat.getValue()); 30 | } 31 | } 32 | 33 | public void testGetStat() { 34 | LTRStat stat = ltrStats.getStat(StatName.PLUGIN_STATUS.getName()); 35 | assertStatPresence(StatName.PLUGIN_STATUS.getName(), stat); 36 | } 37 | 38 | private void assertStatPresence(String statName, LTRStat stat) { 39 | assertTrue(ltrStats.getStats().containsKey(statName)); 40 | assertSame(ltrStats.getStats().get(statName), stat); 41 | } 42 | 43 | public void testGetNodeStats() { 44 | Map stats = ltrStats.getStats(); 45 | Set nodeStats = new HashSet<>(ltrStats.getNodeStats().values()); 46 | 47 | for (LTRStat stat : stats.values()) { 48 | assertTrue((stat.isClusterLevel() && !nodeStats.contains(stat)) || 49 | (!stat.isClusterLevel() && nodeStats.contains(stat))); 50 | } 51 | } 52 | 53 | public void testGetClusterStats() { 54 | Map stats = ltrStats.getStats(); 55 | Set clusterStats = new HashSet<>(ltrStats.getClusterStats().values()); 56 | 57 | for (LTRStat stat : stats.values()) { 58 | assertTrue((stat.isClusterLevel() && clusterStats.contains(stat)) || 59 | (!stat.isClusterLevel() && !clusterStats.contains(stat))); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/test/java/com/o19s/es/ltr/stats/suppliers/PluginHealthStatusSupplierTests.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.ltr.stats.suppliers; 2 | 3 | import com.o19s.es.ltr.feature.store.index.IndexFeatureStore; 4 | import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; 5 | import org.elasticsearch.common.settings.Settings; 6 | import org.elasticsearch.common.util.concurrent.ThreadContext; 7 | import org.elasticsearch.indices.EmptySystemIndices; 8 | import org.elasticsearch.test.ESIntegTestCase; 9 | import org.junit.Before; 10 | 11 | public class PluginHealthStatusSupplierTests extends ESIntegTestCase { 12 | private PluginHealthStatusSupplier pluginHealthStatusSupplier; 13 | 14 | @Before 15 | public void setup() { 16 | pluginHealthStatusSupplier = 17 | new PluginHealthStatusSupplier(clusterService(), new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY), 18 | EmptySystemIndices.INSTANCE)); 19 | } 20 | 21 | public void testPluginHealthStatusNoLtrStore() { 22 | assertEquals("green", pluginHealthStatusSupplier.get()); 23 | } 24 | 25 | public void testPluginHealthStatus() { 26 | createIndex(IndexFeatureStore.DEFAULT_STORE, 27 | IndexFeatureStore.DEFAULT_STORE + "_custom1", 28 | IndexFeatureStore.DEFAULT_STORE + "_custom2"); 29 | flush(); 30 | String status = pluginHealthStatusSupplier.get(); 31 | assertTrue(status.equals("green") || status.equals("yellow")); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/java/com/o19s/es/termstat/TermStatQueryBuilderTests.java: -------------------------------------------------------------------------------- 1 | package com.o19s.es.termstat; 2 | 3 | import com.o19s.es.explore.StatisticsHelper.AggrType; 4 | 5 | import com.o19s.es.ltr.LtrQueryParserPlugin; 6 | import org.apache.lucene.search.Query; 7 | import org.elasticsearch.common.ParsingException; 8 | import org.elasticsearch.index.query.SearchExecutionContext; 9 | import org.elasticsearch.plugins.Plugin; 10 | import org.elasticsearch.test.AbstractQueryTestCase; 11 | 12 | import java.io.IOException; 13 | import java.util.Collection; 14 | 15 | import static java.util.Arrays.asList; 16 | import static org.hamcrest.CoreMatchers.instanceOf; 17 | 18 | public class TermStatQueryBuilderTests extends AbstractQueryTestCase { 19 | // TODO: Remove the TestGeoShapeFieldMapperPlugin once upstream has completed the migration. 20 | protected Collection> getPlugins() { 21 | return asList(LtrQueryParserPlugin.class); 22 | } 23 | 24 | @Override 25 | protected TermStatQueryBuilder doCreateTestQueryBuilder() { 26 | TermStatQueryBuilder builder = new TermStatQueryBuilder(); 27 | 28 | builder.analyzer("standard"); 29 | builder.expr("tf"); 30 | builder.aggr(AggrType.AVG.getType()); 31 | builder.posAggr(AggrType.AVG.getType()); 32 | builder.fields(new String[]{"text"}); 33 | builder.terms(new String[]{"cow"}); 34 | 35 | return builder; 36 | } 37 | 38 | public void testParse() throws Exception { 39 | String query = " {" + 40 | " \"term_stat\": {" + 41 | " \"expr\": \"tf\"," + 42 | " \"aggr\": \"min\"," + 43 | " \"pos_aggr\": \"max\"," + 44 | " \"fields\": [\"text\"]," + 45 | " \"terms\": [\"cow\"]" + 46 | " }" + 47 | "}"; 48 | 49 | TermStatQueryBuilder builder = (TermStatQueryBuilder) parseQuery(query); 50 | 51 | assertEquals(builder.expr(), "tf"); 52 | assertEquals(builder.aggr(), "min"); 53 | assertEquals(builder.posAggr(), "max"); 54 | 55 | } 56 | 57 | public void testMissingExpr() throws Exception { 58 | String query = " {" + 59 | " \"term_stat\": {" + 60 | " \"aggr\": \"min\"," + 61 | " \"pos_aggr\": \"max\"," + 62 | " \"fields\": [\"text\"]," + 63 | " \"terms\": [\"cow\"]" + 64 | " }" + 65 | "}"; 66 | 67 | expectThrows(ParsingException.class, () -> parseQuery(query)); 68 | } 69 | 70 | @Override 71 | protected void doAssertLuceneQuery(TermStatQueryBuilder queryBuilder, Query query, SearchExecutionContext context) throws IOException { 72 | assertThat(query, instanceOf(TermStatQuery.class)); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/test/resources/com/o19s/es/ltr/ranker/dectree/simple_tree.txt: -------------------------------------------------------------------------------- 1 | # first line after split is right 2 | # on a split line, the last 3 integers correspond to leftNodeId, rightNodeId, missingNodeId 3 | # data point: feature1:1, feature2:2, feature3:3 4 | - tree:3.4 5 | - split:feature1:2.3:1:2:1 6 | - output:3.2 7 | # right wins 8 | - split:feature2:2.2:3:4:4 9 | - split:feature3:3.2:5:6:5 10 | - output:11 11 | - output:17 12 | # left wins => output 1.2*3.4 13 | - output:1.2 14 | - tree:2.8 15 | - split:feature1:0.1:1:2:1 16 | # right wins 17 | - split:feature2:1.8:3:4:4 18 | # right wins 19 | - split:feature3:3.2:5:6:6 20 | - output:10 21 | # left wins => output 3.2*2.8 22 | - output:3.2 23 | - output:15 24 | - output:23 25 | -------------------------------------------------------------------------------- /src/yamlRestTest/java/com/o19s/es/ltr/LtrQueryClientYamlTestSuiteIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright [2017] Wikimedia Foundation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.o19s.es.ltr; 17 | 18 | import com.carrotsearch.randomizedtesting.annotations.Name; 19 | import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; 20 | import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; 21 | import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; 22 | 23 | /** 24 | * Created by doug on 12/30/16. 25 | */ 26 | public class LtrQueryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { 27 | 28 | public LtrQueryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { 29 | super(testCandidate); 30 | } 31 | 32 | @ParametersFactory 33 | public static Iterable parameters() throws Exception { 34 | return ESClientYamlSuiteTestCase.createParameters(); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.add_features_to_set.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.add_features_to_set": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/_featureset/{name}/_addfeatures/{query}", 9 | "parts": { 10 | "query": { 11 | "required": false, 12 | "type": "string", 13 | "description": "The feature names query" 14 | }, 15 | "name": { 16 | "required": true, 17 | "type": "string", 18 | "description": "The featureset name" 19 | } 20 | }, 21 | "methods": [ 22 | "POST" 23 | ] 24 | }, 25 | { 26 | "path": "/_ltr/_featureset/{name}/_addfeatures", 27 | "parts": { 28 | "name": { 29 | "required": true, 30 | "type": "string", 31 | "description": "The featureset name" 32 | } 33 | }, 34 | "methods": [ 35 | "POST" 36 | ] 37 | }, 38 | { 39 | "path": "/_ltr/{store}/_featureset/{name}/_addfeatures", 40 | "parts": { 41 | "name": { 42 | "required": true, 43 | "type": "string", 44 | "description": "The featureset name" 45 | }, 46 | "store": { 47 | "required": false, 48 | "type": "string", 49 | "description": "The store name" 50 | } 51 | }, 52 | "methods": [ 53 | "POST" 54 | ] 55 | }, 56 | { 57 | "path": "/_ltr/{store}/_featureset/{name}/_addfeatures/{query}", 58 | "parts": { 59 | "query": { 60 | "required": false, 61 | "type": "string", 62 | "description": "The feature names query" 63 | }, 64 | "name": { 65 | "required": true, 66 | "type": "string", 67 | "description": "The featureset name" 68 | }, 69 | "store": { 70 | "required": false, 71 | "type": "string", 72 | "description": "The store name" 73 | } 74 | }, 75 | "methods": [ 76 | "POST" 77 | ] 78 | } 79 | ] 80 | }, 81 | "body": { 82 | "required": "false", 83 | "description": "The feature" 84 | }, 85 | "params": { 86 | "merge": { 87 | "type": "boolean", 88 | "description": "Merge the list for features otherwise append only" 89 | }, 90 | "version": { 91 | "type": "long", 92 | "description": "Extra check to ensure that the model is created with this version of the set" 93 | }, 94 | "routing": { 95 | "type": "string", 96 | "description": "Specific routing value" 97 | } 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.cache_stats.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.cache_stats": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/_cachestats", 9 | "methods": [ 10 | "GET" 11 | ] 12 | } 13 | ] 14 | }, 15 | "body": null 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.clear_cache.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.clear_cache": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/{store}/_clearcache", 9 | "parts": { 10 | "store": { 11 | "required": false, 12 | "type": "string", 13 | "description": "The store name" 14 | } 15 | }, 16 | "methods": [ 17 | "POST" 18 | ] 19 | }, 20 | { 21 | "path": "/_ltr/_clearcache", 22 | "methods": [ 23 | "POST" 24 | ] 25 | } 26 | ] 27 | }, 28 | "body": null 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.create_feature.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.create_feature": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/_feature/{name}", 9 | "parts": { 10 | "name": { 11 | "required": true, 12 | "type": "string", 13 | "description": "The feature name" 14 | } 15 | }, 16 | "methods": [ 17 | "PUT" 18 | ] 19 | }, 20 | { 21 | "path": "/_ltr/{store}/_feature/{name}", 22 | "parts": { 23 | "name": { 24 | "required": true, 25 | "type": "string", 26 | "description": "The feature name" 27 | }, 28 | "store": { 29 | "required": false, 30 | "type": "string", 31 | "description": "The store name" 32 | } 33 | }, 34 | "methods": [ 35 | "PUT" 36 | ] 37 | } 38 | ] 39 | }, 40 | "body": { 41 | "required": "true", 42 | "description": "The feature" 43 | }, 44 | "params": { 45 | "routing": { 46 | "type": "string", 47 | "description": "Specific routing value" 48 | } 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.create_featureset.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.create_featureset": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/_featureset/{name}", 9 | "parts": { 10 | "name": { 11 | "required": true, 12 | "type": "string", 13 | "description": "The featureset name" 14 | } 15 | }, 16 | "methods": [ 17 | "PUT" 18 | ] 19 | }, 20 | { 21 | "path": "/_ltr/{store}/_featureset/{name}", 22 | "parts": { 23 | "name": { 24 | "required": true, 25 | "type": "string", 26 | "description": "The featureset name" 27 | }, 28 | "store": { 29 | "required": false, 30 | "type": "string", 31 | "description": "The store name" 32 | } 33 | }, 34 | "methods": [ 35 | "PUT" 36 | ] 37 | } 38 | ] 39 | }, 40 | "body": { 41 | "required": "true", 42 | "description": "The featureset" 43 | }, 44 | "params": { 45 | "routing": { 46 | "type": "string", 47 | "description": "Specific routing value" 48 | } 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.create_model.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.create_model": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/_model/{name}", 9 | "parts": { 10 | "name": { 11 | "required": true, 12 | "type": "string", 13 | "description": "The model name" 14 | } 15 | }, 16 | "methods": [ 17 | "PUT" 18 | ] 19 | }, 20 | { 21 | "path": "/_ltr/{store}/_model/{name}", 22 | "parts": { 23 | "name": { 24 | "required": true, 25 | "type": "string", 26 | "description": "The model name" 27 | }, 28 | "store": { 29 | "required": false, 30 | "type": "string", 31 | "description": "The store name" 32 | } 33 | }, 34 | "methods": [ 35 | "PUT" 36 | ] 37 | } 38 | ] 39 | }, 40 | "body": { 41 | "required": "true", 42 | "description": "The model" 43 | }, 44 | "params": { 45 | "routing": { 46 | "type": "string", 47 | "description": "Specific routing value" 48 | } 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.create_model_from_set.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.create_model_from_set": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/_featureset/{name}/_createmodel", 9 | "parts": { 10 | "name": { 11 | "required": true, 12 | "type": "string", 13 | "description": "The featureset name" 14 | } 15 | }, 16 | "methods": [ 17 | "POST" 18 | ] 19 | }, 20 | { 21 | "path": "/_ltr/{store}/_featureset/{name}/_createmodel", 22 | "parts": { 23 | "name": { 24 | "required": true, 25 | "type": "string", 26 | "description": "The featureset name" 27 | }, 28 | "store": { 29 | "required": false, 30 | "type": "string", 31 | "description": "The store name" 32 | } 33 | }, 34 | "methods": [ 35 | "POST" 36 | ] 37 | } 38 | ] 39 | }, 40 | "body": { 41 | "required": "true", 42 | "description": "Definition of the model: name, type and model" 43 | }, 44 | "params": { 45 | "routing": { 46 | "type": "string", 47 | "description": "Specific routing value" 48 | } 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.create_store.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.create_store": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr", 9 | "methods": [ 10 | "PUT" 11 | ] 12 | }, 13 | { 14 | "path": "/_ltr/{store}", 15 | "parts": { 16 | "store": { 17 | "required": false, 18 | "type": "string", 19 | "description": "The store name" 20 | } 21 | }, 22 | "methods": [ 23 | "PUT" 24 | ] 25 | } 26 | ] 27 | }, 28 | "body": null 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.delete_feature.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.delete_feature": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/_feature/{name}", 9 | "parts": { 10 | "name": { 11 | "required": true, 12 | "type": "string", 13 | "description": "The feature name" 14 | } 15 | }, 16 | "methods": [ 17 | "DELETE" 18 | ] 19 | }, 20 | { 21 | "path": "/_ltr/{store}/_feature/{name}", 22 | "parts": { 23 | "name": { 24 | "required": true, 25 | "type": "string", 26 | "description": "The feature name" 27 | }, 28 | "store": { 29 | "required": false, 30 | "type": "string", 31 | "description": "The store name" 32 | } 33 | }, 34 | "methods": [ 35 | "DELETE" 36 | ] 37 | } 38 | ] 39 | }, 40 | "body": null, 41 | "params": { 42 | "routing": { 43 | "type": "string", 44 | "description": "Specific routing value" 45 | } 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.delete_featureset.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.delete_featureset": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/_featureset/{name}", 9 | "parts": { 10 | "name": { 11 | "required": true, 12 | "type": "string", 13 | "description": "The featureset name" 14 | } 15 | }, 16 | "methods": [ 17 | "DELETE" 18 | ] 19 | }, 20 | { 21 | "path": "/_ltr/{store}/_featureset/{name}", 22 | "parts": { 23 | "name": { 24 | "required": true, 25 | "type": "string", 26 | "description": "The featureset name" 27 | }, 28 | "store": { 29 | "required": false, 30 | "type": "string", 31 | "description": "The store name" 32 | } 33 | }, 34 | "methods": [ 35 | "DELETE" 36 | ] 37 | } 38 | ] 39 | }, 40 | "body": null, 41 | "params": { 42 | "routing": { 43 | "type": "string", 44 | "description": "Specific routing value" 45 | } 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.delete_model.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.delete_model": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/_model/{name}", 9 | "parts": { 10 | "name": { 11 | "required": true, 12 | "type": "string", 13 | "description": "The model name" 14 | } 15 | }, 16 | "methods": [ 17 | "DELETE" 18 | ] 19 | }, 20 | { 21 | "path": "/_ltr/{store}/_model/{name}", 22 | "parts": { 23 | "name": { 24 | "required": true, 25 | "type": "string", 26 | "description": "The model name" 27 | }, 28 | "store": { 29 | "required": false, 30 | "type": "string", 31 | "description": "The store name" 32 | } 33 | }, 34 | "methods": [ 35 | "DELETE" 36 | ] 37 | } 38 | ] 39 | }, 40 | "body": null, 41 | "params": { 42 | "routing": { 43 | "type": "string", 44 | "description": "Specific routing value" 45 | } 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.delete_store.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.delete_store": { 3 | "stability": "stable", 4 | "visibility": "private", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr", 9 | "methods": [ 10 | "DELETE" 11 | ] 12 | }, 13 | { 14 | "path": "/_ltr/{store}", 15 | "parts": { 16 | "store": { 17 | "required": false, 18 | "type": "string", 19 | "description": "The store name" 20 | } 21 | }, 22 | "methods": [ 23 | "DELETE" 24 | ] 25 | } 26 | ] 27 | }, 28 | "body": null 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.get_feature.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.get_feature": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/_feature/{name}", 9 | "parts": { 10 | "name": { 11 | "required": true, 12 | "type": "string", 13 | "description": "The feature name" 14 | } 15 | }, 16 | "methods": [ 17 | "GET" 18 | ] 19 | }, 20 | { 21 | "path": "/_ltr/{store}/_feature/{name}", 22 | "parts": { 23 | "name": { 24 | "required": true, 25 | "type": "string", 26 | "description": "The feature name" 27 | }, 28 | "store": { 29 | "required": false, 30 | "type": "string", 31 | "description": "The store name" 32 | } 33 | }, 34 | "methods": [ 35 | "GET" 36 | ] 37 | } 38 | ] 39 | }, 40 | "body": null 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.get_featureset.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.get_featureset": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/_featureset/{name}", 9 | "parts": { 10 | "name": { 11 | "required": true, 12 | "type": "string", 13 | "description": "The featureset name" 14 | } 15 | }, 16 | "methods": [ 17 | "GET" 18 | ] 19 | }, 20 | { 21 | "path": "/_ltr/{store}/_featureset/{name}", 22 | "parts": { 23 | "name": { 24 | "required": true, 25 | "type": "string", 26 | "description": "The featureset name" 27 | }, 28 | "store": { 29 | "required": false, 30 | "type": "string", 31 | "description": "The store name" 32 | } 33 | }, 34 | "methods": [ 35 | "GET" 36 | ] 37 | } 38 | ] 39 | }, 40 | "body": null 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.get_model.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.get_model": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/_model/{name}", 9 | "parts": { 10 | "name": { 11 | "required": true, 12 | "type": "string", 13 | "description": "The model name" 14 | } 15 | }, 16 | "methods": [ 17 | "GET" 18 | ] 19 | }, 20 | { 21 | "path": "/_ltr/{store}/_model/{name}", 22 | "parts": { 23 | "name": { 24 | "required": true, 25 | "type": "string", 26 | "description": "The model name" 27 | }, 28 | "store": { 29 | "required": false, 30 | "type": "string", 31 | "description": "The store name" 32 | } 33 | }, 34 | "methods": [ 35 | "GET" 36 | ] 37 | } 38 | ] 39 | }, 40 | "body": null 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.get_stats.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.get_stats": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/_stats", 9 | "methods": [ 10 | "GET" 11 | ] 12 | }, 13 | { 14 | "path": "/_ltr/_stats/nodes/{nodeId}", 15 | "parts": { 16 | "nodeId": { 17 | "required": false, 18 | "type": "string", 19 | "description": "id of an individual node in the cluster" 20 | } 21 | }, 22 | "methods": [ 23 | "GET" 24 | ] 25 | }, 26 | { 27 | "path": "/_ltr/_stats/{stat}", 28 | "parts": { 29 | "stat": { 30 | "required": false, 31 | "type": "string", 32 | "description": "name of the individual statistics." 33 | } 34 | }, 35 | "methods": [ 36 | "GET" 37 | ] 38 | }, 39 | { 40 | "path": "/_ltr/_stats/{stat}/nodes/{nodeId}", 41 | "parts": { 42 | "nodeId": { 43 | "required": false, 44 | "type": "string", 45 | "description": "id of an individual node in the cluster." 46 | }, 47 | "stat": { 48 | "required": false, 49 | "type": "string", 50 | "description": "name of the individual statistics." 51 | } 52 | }, 53 | "methods": [ 54 | "GET" 55 | ] 56 | } 57 | ] 58 | }, 59 | "body": null 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.get_store.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.get_store": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/{store}", 9 | "parts": { 10 | "store": { 11 | "required": true, 12 | "type": "string", 13 | "description": "The store name" 14 | } 15 | }, 16 | "methods": [ 17 | "GET" 18 | ] 19 | } 20 | ] 21 | }, 22 | "body": null 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.list_feature.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.list_feature": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/{store}/_feature", 9 | "parts": { 10 | "store": { 11 | "required": false, 12 | "type": "string", 13 | "description": "The store name" 14 | } 15 | }, 16 | "methods": [ 17 | "GET" 18 | ] 19 | }, 20 | { 21 | "path": "/_ltr/_feature", 22 | "methods": [ 23 | "GET" 24 | ] 25 | } 26 | ] 27 | }, 28 | "body": null, 29 | "params": { 30 | "prefix": { 31 | "required": false, 32 | "type": "string", 33 | "description": "Filter elements by name prefix" 34 | }, 35 | "from": { 36 | "required": false, 37 | "type": "integer", 38 | "description": "Page results" 39 | }, 40 | "size": { 41 | "required": false, 42 | "type": "integer", 43 | "description": "Page size" 44 | } 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.list_featureset.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.list_featureset": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/_featureset", 9 | "methods": [ 10 | "GET" 11 | ] 12 | }, 13 | { 14 | "path": "/_ltr/{store}/_featureset", 15 | "parts": { 16 | "store": { 17 | "required": false, 18 | "type": "string", 19 | "description": "The store name" 20 | } 21 | }, 22 | "methods": [ 23 | "GET" 24 | ] 25 | } 26 | ] 27 | }, 28 | "body": null, 29 | "params": { 30 | "prefix": { 31 | "required": false, 32 | "type": "string", 33 | "description": "Filter elements by name prefix" 34 | }, 35 | "from": { 36 | "required": false, 37 | "type": "integer", 38 | "description": "Page results" 39 | }, 40 | "size": { 41 | "required": false, 42 | "type": "integer", 43 | "description": "Page size" 44 | } 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.list_model.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.list_model": { 3 | "stability": "stable", 4 | "visibility":"public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/_model", 9 | "methods": [ 10 | "GET" 11 | ] 12 | }, 13 | { 14 | "path": "/_ltr/{store}/_model", 15 | "parts": { 16 | "store": { 17 | "required": false, 18 | "type": "string", 19 | "description": "The store name" 20 | } 21 | }, 22 | "methods": [ 23 | "GET" 24 | ] 25 | } 26 | ] 27 | }, 28 | "body": null, 29 | "params": { 30 | "prefix": { 31 | "required": false, 32 | "type": "string", 33 | "description": "Filter elements by name prefix" 34 | }, 35 | "from": { 36 | "required": false, 37 | "type": "integer", 38 | "description": "Page results" 39 | }, 40 | "size": { 41 | "required": false, 42 | "type": "integer", 43 | "description": "Page size" 44 | } 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.list_stores.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.list_stores": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr", 9 | "methods": [ 10 | "GET" 11 | ] 12 | } 13 | ] 14 | }, 15 | "body": null 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.update_feature.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.update_feature": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/_feature/{name}", 9 | "parts": { 10 | "name": { 11 | "required": true, 12 | "type": "string", 13 | "description": "The feature name" 14 | } 15 | }, 16 | "methods": [ 17 | "POST" 18 | ] 19 | }, 20 | { 21 | "path": "/_ltr/{store}/_feature/{name}", 22 | "parts": { 23 | "name": { 24 | "required": true, 25 | "type": "string", 26 | "description": "The feature name" 27 | }, 28 | "store": { 29 | "required": false, 30 | "type": "string", 31 | "description": "The store name" 32 | } 33 | }, 34 | "methods": [ 35 | "POST" 36 | ] 37 | } 38 | ] 39 | }, 40 | "body": { 41 | "required": "true", 42 | "description": "The feature" 43 | }, 44 | "params": { 45 | "routing": { 46 | "type": "string", 47 | "description": "Specific routing value" 48 | } 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.update_featureset.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.update_featureset": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/_featureset/{name}", 9 | "parts": { 10 | "name": { 11 | "required": true, 12 | "type": "string", 13 | "description": "The featureset name" 14 | } 15 | }, 16 | "methods": [ 17 | "POST" 18 | ] 19 | }, 20 | { 21 | "path": "/_ltr/{store}/_featureset/{name}", 22 | "parts": { 23 | "name": { 24 | "required": true, 25 | "type": "string", 26 | "description": "The featureset name" 27 | }, 28 | "store": { 29 | "required": false, 30 | "type": "string", 31 | "description": "The store name" 32 | } 33 | }, 34 | "methods": [ 35 | "POST" 36 | ] 37 | } 38 | ] 39 | }, 40 | "body": { 41 | "required": "true", 42 | "description": "The featureset" 43 | }, 44 | "params": { 45 | "routing": { 46 | "type": "string", 47 | "description": "Specific routing value" 48 | } 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/api/ltr.update_model.json: -------------------------------------------------------------------------------- 1 | { 2 | "ltr.update_model": { 3 | "stability": "stable", 4 | "visibility": "public", 5 | "url": { 6 | "paths": [ 7 | { 8 | "path": "/_ltr/_model/{name}", 9 | "parts": { 10 | "name": { 11 | "required": true, 12 | "type": "string", 13 | "description": "The model name" 14 | } 15 | }, 16 | "methods": [ 17 | "POST" 18 | ] 19 | }, 20 | { 21 | "path": "/_ltr/{store}/_model/{name}", 22 | "parts": { 23 | "name": { 24 | "required": true, 25 | "type": "string", 26 | "description": "The model name" 27 | }, 28 | "store": { 29 | "required": false, 30 | "type": "string", 31 | "description": "The store name" 32 | } 33 | }, 34 | "methods": [ 35 | "POST" 36 | ] 37 | } 38 | ] 39 | }, 40 | "body": { 41 | "required": "true", 42 | "description": "The model" 43 | }, 44 | "params": { 45 | "routing": { 46 | "type": "string", 47 | "description": "Specific routing value" 48 | } 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/test/fstore/60_create_model_from_set.yml: -------------------------------------------------------------------------------- 1 | --- 2 | "Create model from set on the default store": 3 | - do: 4 | ltr.create_store: {} 5 | 6 | - do: 7 | ltr.create_featureset: 8 | name: my_featureset 9 | body: 10 | featureset: 11 | name: my_featureset 12 | features: 13 | - name: feature1 14 | params: query_string 15 | template: 16 | match: 17 | field_test1: "{{query_string}}" 18 | - name: feature2 19 | params: query_string 20 | template: 21 | match: 22 | field_test2: "{{query_string}}" 23 | 24 | - do: 25 | ltr.create_model_from_set: 26 | name: my_featureset 27 | body: 28 | model: 29 | name: my_model 30 | model: 31 | type: model/linear 32 | definition: 33 | feature1: 1.3 34 | feature2: 0.3 35 | 36 | - match: { _index: .ltrstore } 37 | - match: { _id: model-my_model } 38 | - match: { _version: 1 } 39 | 40 | - do: 41 | catch: /Element of type \[model\] are not updatable, please create a new one instead./ 42 | ltr.create_model_from_set: 43 | name: my_featureset 44 | body: 45 | model: 46 | name: my_model 47 | model: 48 | type: model/linear 49 | definition: 50 | feature1: 1.3 51 | feature2: 0.3 52 | 53 | --- 54 | "Create model from set on custom store": 55 | - do: 56 | ltr.create_store: 57 | store: mystore 58 | 59 | - do: 60 | ltr.create_featureset: 61 | store: mystore 62 | name: my_featureset 63 | body: 64 | featureset: 65 | name: my_featureset 66 | features: 67 | - name: feature1 68 | params: query_string 69 | template: 70 | match: 71 | field_test1: "{{query_string}}" 72 | - name: feature2 73 | params: query_string 74 | template: 75 | match: 76 | field_test2: "{{query_string}}" 77 | 78 | - do: 79 | ltr.create_model_from_set: 80 | store: mystore 81 | name: my_featureset 82 | body: 83 | model: 84 | name: my_model 85 | model: 86 | type: model/linear 87 | definition: 88 | feature1: 1.3 89 | feature2: 0.3 90 | 91 | - match: { _index: .ltrstore_mystore } 92 | - match: { _id: model-my_model } 93 | - match: { _version: 1 } 94 | 95 | - do: 96 | catch: /Element of type \[model\] are not updatable, please create a new one instead./ 97 | ltr.create_model_from_set: 98 | store: mystore 99 | name: my_featureset 100 | body: 101 | model: 102 | name: my_model 103 | model: 104 | type: model/linear 105 | definition: 106 | feature1: 1.3 107 | feature2: 0.3 108 | 109 | --------------------------------------------------------------------------------