├── .gitignore ├── .travis.yml ├── LICENSE ├── LICENSES_THIRD_PARTY ├── common ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── msd │ │ └── gin │ │ └── halyard │ │ └── common │ │ ├── HalyardTableUtils.java │ │ └── package-info.java │ └── test │ └── java │ └── com │ └── msd │ └── gin │ └── halyard │ └── common │ ├── HBaseServerTestInstance.java │ ├── HalyardTableUtilsCalculateSplitsTest.java │ ├── HalyardTableUtilsScanTest.java │ └── HalyardTableUtilsTest.java ├── docs ├── _config.yml ├── _data │ └── docs.yml ├── _includes │ ├── footer.html │ ├── header.html │ └── nav.html ├── _layouts │ ├── doc.html │ └── page.html ├── architecture.md ├── getting-started.md ├── img │ ├── add_rdf.png │ ├── architecture.png │ ├── clear.png │ ├── new_repo.png │ ├── query.png │ ├── remove.png │ ├── result.png │ ├── update.png │ └── workbench.png ├── index.md ├── ns.md ├── tools.md └── usage.md ├── findbugs-exclude.xml ├── pom.xml ├── readme.md ├── rio ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── msd │ │ │ └── gin │ │ │ └── halyard │ │ │ └── rio │ │ │ ├── JSONParser.java │ │ │ └── NDJSONLDParser.java │ └── resources │ │ └── META-INF │ │ └── services │ │ └── org.eclipse.rdf4j.rio.RDFParserFactory │ └── test │ ├── java │ └── com │ │ └── msd │ │ └── gin │ │ └── halyard │ │ └── rio │ │ ├── JSONParserParseTest.java │ │ ├── JSONParserTest.java │ │ └── NDJSONLDParserTest.java │ └── resources │ └── com │ └── msd │ └── gin │ └── halyard │ └── rio │ ├── efo_test.ndjsonld │ ├── efo_test.ttl │ ├── empty.json │ ├── empty.ttl │ ├── emptyObj.json │ ├── emptyObj.ttl │ ├── example.json │ ├── example.ttl │ ├── nestedArrays.json │ ├── nestedArrays.ttl │ ├── primitives.json │ ├── primitives.ttl │ ├── rootArray.json │ └── rootArray.ttl ├── sail ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── msd │ │ │ └── gin │ │ │ └── halyard │ │ │ └── sail │ │ │ ├── HBaseRepositoryManager.java │ │ │ ├── HBaseSail.java │ │ │ ├── HBaseSailConfig.java │ │ │ ├── HBaseSailFactory.java │ │ │ ├── HalyardStatsBasedStatementPatternCardinalityCalculator.java │ │ │ └── package-info.java │ └── resources │ │ └── META-INF │ │ └── services │ │ └── org.eclipse.rdf4j.sail.config.SailFactory │ └── test │ └── java │ └── com │ └── msd │ └── gin │ └── halyard │ └── sail │ ├── HBaseRepositoryManagerTest.java │ ├── HBaseSailAddRemoveTest.java │ ├── HBaseSailConfigTest.java │ ├── HBaseSailFactoryTest.java │ ├── HBaseSailHashConflictTest.java │ ├── HBaseSailTest.java │ ├── LiteralSearchStatementScannerTest.java │ └── SES2154SubselectOptionalTest.java ├── sdk ├── pom.xml └── src │ └── main │ ├── assembly │ └── sdk-assembly.xml │ ├── patches │ └── rdf4j-client │ │ └── org │ │ └── eclipse │ │ └── rdf4j │ │ └── repository │ │ └── config │ │ └── hbase.ttl │ └── scripts │ ├── console │ ├── halyard │ └── halyard-defaults.xml ├── src └── main │ └── license │ └── LICENSES_THIRD_PARTY.properties ├── strategy ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── msd │ │ └── gin │ │ └── halyard │ │ ├── optimizers │ │ ├── HalyardEvaluationStatistics.java │ │ ├── HalyardFilterOptimizer.java │ │ └── HalyardQueryJoinOptimizer.java │ │ ├── strategy │ │ ├── HalyardEvaluationStrategy.java │ │ ├── HalyardQueryOptimizerPipeline.java │ │ ├── HalyardStatementPatternEvaluation.java │ │ ├── HalyardTupleExprEvaluation.java │ │ ├── HalyardValueExprEvaluation.java │ │ ├── collections │ │ │ ├── BigHashSet.java │ │ │ └── Sorter.java │ │ └── package-info.java │ │ └── vocab │ │ ├── HALYARD.java │ │ └── VOID_EXT.java │ └── test │ └── java │ └── com │ └── msd │ └── gin │ └── halyard │ ├── optimizers │ ├── HalyardEvaluationStatisticsTest.java │ ├── HalyardFilterOptimizerTest.java │ └── HalyardQueryJoinOptimizerTest.java │ ├── strategy │ ├── ArbitraryLengthPathTest.java │ ├── HalyardComplexSPARQLQueryTest.java │ ├── HalyardSPARQLQueryTest.java │ ├── HalyardSPARQLUpdateTest.java │ ├── HalyardStrategyExtendedTest.java │ ├── HalyardStrategyServiceTest.java │ ├── MemoryStoreWithHalyardStrategy.java │ ├── W3CApprovedSPARQL11QueryTest.java │ ├── W3CApprovedSPARQL11UpdateTest.java │ └── collections │ │ ├── BigHashSetTest.java │ │ └── SorterTest.java │ └── vocab │ └── VocabTest.java ├── tools ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── msd │ │ │ └── gin │ │ │ └── halyard │ │ │ └── tools │ │ │ ├── AbstractHalyardTool.java │ │ │ ├── HalyardBulkDelete.java │ │ │ ├── HalyardBulkExport.java │ │ │ ├── HalyardBulkLoad.java │ │ │ ├── HalyardBulkUpdate.java │ │ │ ├── HalyardElasticIndexer.java │ │ │ ├── HalyardEndpoint.java │ │ │ ├── HalyardExport.java │ │ │ ├── HalyardMain.java │ │ │ ├── HalyardPreSplit.java │ │ │ ├── HalyardProfile.java │ │ │ ├── HalyardStats.java │ │ │ ├── HalyardSummary.java │ │ │ ├── HalyardUpdate.java │ │ │ ├── HttpSparqlHandler.java │ │ │ ├── ParallelSplitFunction.java │ │ │ ├── QueryInputFormat.java │ │ │ ├── SimpleHttpServer.java │ │ │ └── TimeAwareHBaseSail.java │ └── resources │ │ └── logback.xml │ └── test │ ├── java │ └── com │ │ └── msd │ │ └── gin │ │ └── halyard │ │ └── tools │ │ ├── HalyardBulkDeleteTest.java │ │ ├── HalyardBulkExportTest.java │ │ ├── HalyardBulkLoadTest.java │ │ ├── HalyardBulkUpdateTest.java │ │ ├── HalyardElasticIndexerTest.java │ │ ├── HalyardEndpointTest.java │ │ ├── HalyardExportJDBCTypesTest.java │ │ ├── HalyardExportTest.java │ │ ├── HalyardMainTest.java │ │ ├── HalyardPreSplitTest.java │ │ ├── HalyardProfileTest.java │ │ ├── HalyardStatsTest.java │ │ ├── HalyardSummaryTest.java │ │ ├── HalyardUpdateTest.java │ │ ├── HttpSparqlHandlerTest.java │ │ ├── ParallelSplitFunctionTest.java │ │ └── TimeAwareHBaseSailTest.java │ └── resources │ └── com │ └── msd │ └── gin │ └── halyard │ └── tools │ ├── test.properties │ ├── testData.trig │ ├── testFail.properties │ ├── testMoreData.trig │ ├── testQuery.sparql │ ├── testScript.sh │ ├── testScript2.sh │ ├── testStatsMoreTarget.trig │ ├── testStatsTarget.trig │ └── testStatsTargetPartial.trig └── webapps ├── nbactions.xml ├── pom.xml └── src └── main ├── assembly └── webapps-assembly.xml └── patches └── rdf4j-workbench └── transformations └── create-hbase.xsl /.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /dist/ 3 | /nbproject/ 4 | /common-hbase/target/ 5 | /common/target/ 6 | /strategy/target/ 7 | /sail/target/ 8 | /tools/target/ 9 | /sdk/target/ 10 | /webapps/target/ 11 | /docs/_site 12 | /strategy/passing.log 13 | /strategy/failing.log 14 | /nbactions.xml 15 | /common/nbactions.xml 16 | /sail/nbactions.xml 17 | /rio/nbproject/ 18 | /rio/target/ 19 | /strategy/nbproject/ 20 | /tools/nbproject/ 21 | /tools/nbactions.xml 22 | /sail/nbproject/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - openjdk8 4 | sudo: required 5 | after_success: 6 | - bash <(curl -s https://codecov.io/bash) 7 | deploy: 8 | provider: releases 9 | api_key: 10 | secure: TsMForgZW7sfP3HyreeT7NGBcdBjfO96TDoOhbNL69a8IdndSoVcRjHsSNg7dyMxvzzROil37DzozPov9WnGnLab09oUKsgJcoSZ1jPnhr0/sUsR3Y13Py215AbgHxfl3ywFCcTrHl30mJDzwCHnBWApLEDzDJT1mvLaLSnLaJiawz5SBbGYM0/0yBz4PAhjahyOdpWEF4FB3+tYtPERX8LBhejVohc7yzivh3AfX6mtIywBCbvBi/j3YOR7JX4xknn5De5LIH+McEZwB4ZLk63d+Jqy4r33AGC2CThIYdaQ+uF/IIZxrHg2PA4V5sH9JxiOsnr80tLlHIDWXXzKdABDKxvpo90eMm9yWVvVK4VVWUWyAakWZUKpbARuWGlS4uonR+gaC6RI6iuzcOzjQFyC/8IrCWcNmwhF08JYncyb2JZAdQLpL7fxBuU9bzMrJO4BQeW2vQkBSAZiT6Fx0gNHd8lNiYr7tDJ7cTPLdw2nU3OP74D8hN0U11hWSqXz5ib0oJC6JfmjZ+K3BPlgXoidv9eGDBfIIo/9QeCbisnbhyd/01dTyizlC6J5dvvyFa0aEi6z6wpPaEaRCn0RgW36tP2DpqD8NiRZ4bkFLL8iUO3/VUYqe/7vm27aV5G9TUkOkaGNy05ySAQgBvoYW9jpz9KAywuwob1nYV2A58A= 11 | file: 12 | - sdk/target/halyard-sdk-*.zip 13 | - webapps/target/halyard-webapps-*.zip 14 | file_glob: true 15 | skip_cleanup: true 16 | prerelease: true 17 | on: 18 | repo: Merck/Halyard 19 | tags: true 20 | -------------------------------------------------------------------------------- /common/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | halyard-common 5 | jar 6 | 7 | com.msd.gin.halyard 8 | halyard 9 | 2.5 10 | 11 | 12 | 13 | org.eclipse.rdf4j 14 | rdf4j-runtime 15 | ${rdf4j.version} 16 | pom 17 | 18 | 19 | org.apache.hbase 20 | hbase-client 21 | ${hbase.version} 22 | provided 23 | 24 | 25 | org.apache.hbase 26 | hbase-server 27 | ${hbase.version} 28 | test 29 | 30 | 31 | org.eclipse.rdf4j 32 | rdf4j-sparql-testsuite 33 | ${rdf4j.version} 34 | test 35 | 36 | 37 | org.apache.hadoop 38 | hadoop-mapreduce-client-jobclient 39 | ${hadoop.version} 40 | test 41 | tests 42 | 43 | 44 | org.apache.hadoop 45 | hadoop-common 46 | ${hadoop.version} 47 | test 48 | tests 49 | 50 | 51 | org.apache.hadoop 52 | hadoop-mapreduce-client-hs 53 | ${hadoop.version} 54 | test 55 | 56 | 57 | org.apache.hadoop 58 | hadoop-yarn-server-resourcemanager 59 | ${hadoop.version} 60 | test 61 | 62 | 63 | org.apache.derby 64 | derby 65 | 10.12.1.1 66 | test 67 | 68 | 69 | org.apache.hadoop 70 | hadoop-yarn-server-tests 71 | ${hadoop.version} 72 | test 73 | tests 74 | 75 | 76 | 77 | 78 | 79 | org.apache.maven.plugins 80 | maven-jar-plugin 81 | 82 | 83 | 84 | test-jar 85 | 86 | 87 | 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /common/src/main/java/com/msd/gin/halyard/common/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Utilities used by multiple Halyard packages and projects. 3 | */ 4 | package com.msd.gin.halyard.common; -------------------------------------------------------------------------------- /common/src/test/java/com/msd/gin/halyard/common/HBaseServerTestInstance.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.common; 18 | 19 | import java.io.File; 20 | import java.net.InetSocketAddress; 21 | import org.apache.hadoop.conf.Configuration; 22 | import org.apache.hadoop.hbase.HBaseConfiguration; 23 | import org.apache.hadoop.hbase.HConstants; 24 | import org.apache.hadoop.hbase.LocalHBaseCluster; 25 | import org.apache.hadoop.mapreduce.MRConfig; 26 | import org.apache.hadoop.mapreduce.MRJobConfig; 27 | import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster; 28 | import org.apache.hadoop.yarn.conf.YarnConfiguration; 29 | import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; 30 | import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; 31 | import org.apache.zookeeper.server.ServerCnxnFactory; 32 | import org.apache.zookeeper.server.ZooKeeperServer; 33 | 34 | /** 35 | * 36 | * @author Adam Sotona (MSD) 37 | */ 38 | public class HBaseServerTestInstance { 39 | 40 | private static Configuration conf = null; 41 | 42 | public static synchronized Configuration getInstanceConfig() throws Exception { 43 | if (conf == null) { 44 | File zooRoot = File.createTempFile("hbase-zookeeper", ""); 45 | zooRoot.delete(); 46 | ZooKeeperServer zookeper = new ZooKeeperServer(zooRoot, zooRoot, 2000); 47 | ServerCnxnFactory factory = ServerCnxnFactory.createFactory(new InetSocketAddress("localhost", 0), 5000); 48 | factory.startup(zookeper); 49 | 50 | YarnConfiguration yconf = new YarnConfiguration(); 51 | String argLine = System.getProperty("argLine"); 52 | if (argLine != null) { 53 | yconf.set("yarn.app.mapreduce.am.command-opts", argLine.replace("jacoco.exec", "jacocoMR.exec")); 54 | } 55 | yconf.setBoolean(MRConfig.MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING, false); 56 | yconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); 57 | MiniMRYarnCluster miniCluster = new MiniMRYarnCluster("testCluster"); 58 | miniCluster.init(yconf); 59 | String resourceManagerLink = yconf.get(YarnConfiguration.RM_ADDRESS); 60 | yconf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, true); 61 | miniCluster.start(); 62 | miniCluster.waitForNodeManagersToConnect(10000); 63 | // following condition set in MiniYarnCluster:273 64 | while (resourceManagerLink.endsWith(":0")) { 65 | Thread.sleep(100); 66 | resourceManagerLink = yconf.get(YarnConfiguration.RM_ADDRESS); 67 | } 68 | 69 | File hbaseRoot = File.createTempFile("hbase-root", ""); 70 | hbaseRoot.delete(); 71 | conf = HBaseConfiguration.create(miniCluster.getConfig()); 72 | conf.set(HConstants.HBASE_DIR, hbaseRoot.toURI().toURL().toString()); 73 | conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, factory.getLocalPort()); 74 | conf.set("hbase.master.hostname", "localhost"); 75 | conf.set("hbase.regionserver.hostname", "localhost"); 76 | conf.setInt("hbase.master.info.port", -1); 77 | conf.set("hbase.fs.tmp.dir", new File(System.getProperty("java.io.tmpdir")).toURI().toURL().toString()); 78 | LocalHBaseCluster cluster = new LocalHBaseCluster(conf); 79 | cluster.startup(); 80 | } 81 | return new Configuration(conf); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /common/src/test/java/com/msd/gin/halyard/common/HalyardTableUtilsCalculateSplitsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.common; 18 | 19 | import java.util.Arrays; 20 | import java.util.Collection; 21 | import java.util.HashMap; 22 | import java.util.Map; 23 | import org.apache.commons.codec.binary.Hex; 24 | import static org.junit.Assert.*; 25 | import org.junit.Test; 26 | import org.junit.runner.RunWith; 27 | import org.junit.runners.Parameterized; 28 | 29 | /** 30 | * 31 | * @author Adam Sotona (MSD) 32 | */ 33 | @RunWith(Parameterized.class) 34 | public class HalyardTableUtilsCalculateSplitsTest { 35 | 36 | @Parameterized.Parameters 37 | public static Collection data() { 38 | return Arrays.asList(new Object[][] { 39 | {0, new String[]{"01", "02", "03", "04", "05"}}, 40 | {1, new String[]{"008000", "01", "018000", "02", "028000", "03", "04", "05"}}, 41 | {2, new String[]{"004000", "008000", "00c000", "01", "014000", "018000", "01c000", "02", "024000", "028000", "02c000", "03", "04", "05"}}, 42 | }); 43 | } 44 | 45 | private final int splits; 46 | private final String[] expected; 47 | 48 | public HalyardTableUtilsCalculateSplitsTest(int splits, String[] expected) { 49 | this.splits = splits; 50 | this.expected = expected; 51 | } 52 | 53 | @Test 54 | public void testCalculateSplits() { 55 | Map cMap = new HashMap<>(); 56 | byte bb[][] = HalyardTableUtils.calculateSplits(splits); 57 | if (expected == null) { 58 | assertNull(bb); 59 | } else { 60 | assertEquals(expected.length, bb.length); 61 | for (int i = 0; i < expected.length; i++) { 62 | assertEquals(expected[i], Hex.encodeHexString(bb[i])); 63 | } 64 | } 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /docs/_config.yml: -------------------------------------------------------------------------------- 1 | markdown: kramdown 2 | 3 | kramdown: 4 | parse_block_html: true 5 | -------------------------------------------------------------------------------- /docs/_data/docs.yml: -------------------------------------------------------------------------------- 1 | - page: getting-started 2 | name: Getting Started 3 | 4 | - page: tools 5 | name: Tools 6 | 7 | - page: usage 8 | name: Usage 9 | 10 | - page: architecture 11 | name: Architecture 12 | 13 | - page: ns 14 | name: Namespace -------------------------------------------------------------------------------- /docs/_includes/footer.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /docs/_includes/header.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Halyard | {{ page.title }} 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 22 | 23 | 24 | 25 | 39 | 40 | -------------------------------------------------------------------------------- /docs/_includes/nav.html: -------------------------------------------------------------------------------- 1 | 2 | 19 | -------------------------------------------------------------------------------- /docs/_layouts/doc.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include header.html %} 5 | 6 | 7 | 8 | {% include nav.html %} 9 | 10 |
11 | 12 |
13 | 14 |
15 | 16 | 29 | 30 |
31 | 32 |
33 | {{ content }} 34 |
35 |
36 | 37 |
38 | 39 | {% include footer.html %} 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /docs/_layouts/page.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% include header.html %} 5 | 6 | 7 | 8 | {% include nav.html %} 9 | 10 |
11 | 12 | {{ content }} 13 | 14 |
15 | 16 | {% include footer.html %} 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /docs/architecture.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Architecture 3 | layout: doc 4 | --- 5 | 6 | # Architecture 7 | 8 | [![Halyard architecture diagram](img/architecture.png){:width="100%"}](img/architecture.png) 9 | -------------------------------------------------------------------------------- /docs/getting-started.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Getting Started 3 | layout: doc 4 | --- 5 | 6 | # Overview 7 | 8 | Halyard is an extremely horizontally scalable [RDF store](https://en.wikipedia.org/wiki/Triplestore) with support for [named graphs](https://en.wikipedia.org/wiki/Named_graph), designed for storage and integration of extremely large [semantic data models](https://en.wikipedia.org/wiki/Semantic_data_model), and execution of [SPARQL 1.1](http://www.w3.org/TR/sparql11-query) queries of the whole [linked data](https://en.wikipedia.org/wiki/Linked_data) universe snapshots. The implementation of Halyard is based on [Eclipse RDF4J](http://rdf4j.org) framework and the [Apache HBase](http://hbase.apache.org) database, and it is completely written in Java. 9 | 10 | ## Build instructions 11 | 12 | Build environment prerequisites are: 13 | 14 | * Linux or Mac computer 15 | * Java SE Development Kit 8 16 | * Apache Maven software project management and comprehension tool 17 | 18 | In the Halyard project root directory execute the command: `mvn package` 19 | 20 | Optionally, you can build Halyard from NetBeans or other Java development IDE. 21 | 22 | ## Runtime requirements 23 | 24 | Halyard is expected to run on an Apache Hadoop cluster node with a configured Apache HBase client. Apache Hadoop and Apache HBase components are not bundled with Halyard. The runtime requirements are: 25 | 26 | * Apache Hadoop version 2.5.1 or higher 27 | * Apache HBase version 1.1.2 or higher 28 | * Java 8 Runtime 29 | 30 | *Note: Recommended Apache Hadoop distribution is Hortonworks Data Platform (HDP) version 2.4.2 or Amazon Elastic Map Reduce (EMR).* 31 | 32 | ### Hortonworks Data Platform sample cluster setup 33 | 34 | Hortonworks Data Platform (HDP) is a Hadoop distribution with all important parts of Hadoop included, however, it does not directly provide the hardware and core OS. 35 | 36 | The whole HDP stack installation through Amabari is described in [Hortonworks Data Platform - Apache Ambari Installation page](http://docs.hortonworks.com/HDPDocuments/Ambari-2.4.2.0/bk_ambari-installation/content/index.html). 37 | 38 | It is possible to strip down the set of Hadoop components to `HDFS`, `MapReduce2`, `YARN`, `HBase`, `ZooKeeper`, and optionally `Ambari Metrics` for cluster monitoring. 39 | 40 | Detailed documentation about Hortonworks Data Platform is accessible at . 41 | 42 | ### Amazon EMR sample cluster setup 43 | 44 | Amazon Elastic MapReduce is a service providing both hardware and software stack to run Hadoop and Halyard on top of it. 45 | 46 | Sample Amazon EMR setup is described in [Amazon EMR Management Guide - Getting Started](http://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-gs.html). 47 | 48 | For the purpose of Halyard it is important to perform the first two steps of the guide: 49 | 50 | - [Step 1: Set up prerequisites](http://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-gs-prerequisites.html) 51 | - [Step 2: Launch your sample cluster](http://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-gs-launch-sample-cluster.html) 52 | 53 | It is possible to strip down the set of provided components during `Create Cluster` by clicking on `Go to advanced options` and selecting just `Hadoop`, `ZooKeeper`, `HBase` and optionally `Ganglia` for cluster monitoring. 54 | 55 | HBase for Halyard can run in both storage modes: `HDFS` or `S3`. 56 | 57 | Instance types with redundant storage space, such as `d2.xlarge`, are highly recommended when you plan to bulk load large datasets using Halyard. 58 | 59 | Instance types with enough memory and fast disks for local caching, such as `i2.xlarge`, are recommended when the cluster would mainly serve data through Halyard. 60 | 61 | Additional EMR Task Nodes can be used to host additional Halyard SPARQL endpoints. 62 | 63 | A detailed documentation of the Amazon EMR is available at . 64 | -------------------------------------------------------------------------------- /docs/img/add_rdf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Merck/Halyard/fabb338edb14276aa7ef9f5f9b59f680335b22d3/docs/img/add_rdf.png -------------------------------------------------------------------------------- /docs/img/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Merck/Halyard/fabb338edb14276aa7ef9f5f9b59f680335b22d3/docs/img/architecture.png -------------------------------------------------------------------------------- /docs/img/clear.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Merck/Halyard/fabb338edb14276aa7ef9f5f9b59f680335b22d3/docs/img/clear.png -------------------------------------------------------------------------------- /docs/img/new_repo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Merck/Halyard/fabb338edb14276aa7ef9f5f9b59f680335b22d3/docs/img/new_repo.png -------------------------------------------------------------------------------- /docs/img/query.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Merck/Halyard/fabb338edb14276aa7ef9f5f9b59f680335b22d3/docs/img/query.png -------------------------------------------------------------------------------- /docs/img/remove.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Merck/Halyard/fabb338edb14276aa7ef9f5f9b59f680335b22d3/docs/img/remove.png -------------------------------------------------------------------------------- /docs/img/result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Merck/Halyard/fabb338edb14276aa7ef9f5f9b59f680335b22d3/docs/img/result.png -------------------------------------------------------------------------------- /docs/img/update.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Merck/Halyard/fabb338edb14276aa7ef9f5f9b59f680335b22d3/docs/img/update.png -------------------------------------------------------------------------------- /docs/img/workbench.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Merck/Halyard/fabb338edb14276aa7ef9f5f9b59f680335b22d3/docs/img/workbench.png -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Home 3 | layout: page 4 | --- 5 | 6 |
7 | 8 | # Halyard 9 | 10 | [![CI](https://api.travis-ci.org/Merck/Halyard.svg?branch=master)](https://travis-ci.org/Merck/Halyard) 11 | [![Coverage](https://codecov.io/github/Merck/Halyard/coverage.svg?branch=master)](https://codecov.io/gh/Merck/Halyard/) 12 | 13 | Halyard is an extremely horizontally scalable [RDF store](https://en.wikipedia.org/wiki/Triplestore) with support for [named graphs](https://en.wikipedia.org/wiki/Named_graph), designed for storage and integration of extremely large [semantic data models](https://en.wikipedia.org/wiki/Semantic_data_model), and execution of [SPARQL 1.1](http://www.w3.org/TR/sparql11-query) queries of the whole [linked data](https://en.wikipedia.org/wiki/Linked_data) universe snapshots. The implementation of Halyard is based on [Eclipse RDF4J](http://rdf4j.org) framework and the [Apache HBase](http://hbase.apache.org) database, and it is completely written in Java. 14 | 15 |
16 | 17 |
18 |
19 | 20 | ## Run in Amazon EMR 21 | 22 | To get started with Halyard, try deploying it on an Amazon Elastic MapReduce (EMR) cluster. 23 | 24 | See the [Amazon EMR sample cluster setup](getting-started.html#amazon-emr-sample-cluster-setup). 25 | 26 | ## Run locally 27 | 28 | Download and unzip the latest `halyard-sdk-.zip` bundle from the [releases page](https://github.com/Merck/Halyard/releases) to a Apache Hadoop cluster node with a configured Apache HBase client. 29 | 30 | Halyard is expected to run on an Apache Hadoop cluster node with configured Apache HBase client. Apache Hadoop and Apache HBase components are not bundled with Halyard. The runtime requirements are: 31 | 32 | * Apache Hadoop version 2.5.1 or higher 33 | * Apache HBase version 1.1.2 or higher 34 | * Java 8 Runtime 35 | 36 | *Note: Recommended Apache Hadoop distribution is Hortonworks Data Platform (HDP) version 2.4.2 or Amazon Elastic MapReduce (EMR).* 37 | 38 |
39 | 40 |
41 | 42 | ## Get involved 43 | 44 | * Read the [documentation](https://merck.github.io/Halyard/getting-started.html) 45 | * Download the [binaries](https://github.com/Merck/Halyard/releases) 46 | * Clone the [GitHub repository](https://github.com/Merck/Halyard) 47 | * See the [open issues](https://github.com/Merck/Halyard/issues) 48 | * Join our [discussion group](https://groups.google.com/d/forum/halyard-users) 49 | * Contact the author: [Adam Sotona](mailto:adam.sotona@merck.com) 50 | 51 |
52 | 53 |
54 | 55 | ## Articles 56 | 57 | * [Inside Halyard: 1. Triples, Keys, Columns and Values - everything upside down](https://www.linkedin.com/pulse/inside-halyard-1-triples-keys-columns-values-upside-adam-sotona) 58 | * [Inside Halyard: 2. When one working thread is not enough (PUSH versus PULL)](https://www.linkedin.com/pulse/inside-halyard-2-when-one-working-thread-enough-push-versus-sotona) 59 | * [Inside Halyard: 3. Sipping a river through drinking straws](https://www.linkedin.com/pulse/inside-halyard-3-sipping-river-through-drinking-straws-adam-sotona) 60 | * [Inside Halyard: 4. Bulk operations - shifting a mountain](https://www.linkedin.com/pulse/inside-halyard-4-bulk-operations-shifting-mountain-adam-sotona) 61 | * [Inside Halyard: 5. SPARQL Federation without border controls](https://www.linkedin.com/pulse/inside-halyard-5-sparql-federation-without-border-controls-sotona) 62 | * [Inside Halyard: 6. Statistics-based acceleration of SPARQL queries](https://www.linkedin.com/pulse/inside-halyard-6-statistics-based-acceleration-sparql-adam-sotona) 63 | * [Sailing Halyard Over Three Major Clouds](https://www.linkedin.com/pulse/sailing-halyard-over-three-major-clouds-adam-sotona/) 64 | * [Halyard Tips&Tricks - Dynamically Scaled Cloud Cluster Disk Space Issue](https://www.linkedin.com/pulse/halyard-tipstricks-dynamically-scaled-cloud-cluster-adam-sotona) 65 | * [Halyard Tips&Tricks - Heavy Load-balanced SPARQL Endpoint](https://www.linkedin.com/pulse/halyard-tipstricks-heavy-load-balanced-sparql-endpoint-adam-sotona) 66 | * [Halyard Tips&Tricks - Advanced Literal Search Techniques](https://www.linkedin.com/pulse/halyard-tipstricks-advanced-literal-search-adam-sotona) 67 | * [Halyard Tips&Tricks - Jenkins as a Job Orchestrator](https://www.linkedin.com/pulse/halyard-tipstricks-jenkins-job-orchestrator-adam-sotona) 68 | * [Halyard Tips&Tricks - Trillion Statements Challenge](https://www.linkedin.com/pulse/halyard-tipstricks-trillion-statements-challenge-adam-sotona) 69 | 70 |
71 | -------------------------------------------------------------------------------- /docs/ns.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Namespace 3 | layout: doc 4 | --- 5 | 6 | This is the home of the Halyard namespace: 7 | 8 | ``` 9 | PREFIX halyard: 10 | ``` 11 | 12 | ## In-data used identifiers 13 | 14 | ### `halyard:statsContext` 15 | 16 | A named graph context used by Halyard to store dataset statistics. 17 | 18 | ### `halyard:statsRoot` 19 | 20 | Statistics dataset root node within the [`halyard:statsContext`](#statsContext) named graph context. 21 | 22 | ### `halyard:namespacePrefix` 23 | 24 | Property used by Halyard to persist namespaces inside the datasets. 25 | 26 | ## Custom SPARQL filter functions 27 | 28 | ### `halyard:forkAndFilterBy(, ?var1 [,?varN...])` 29 | 30 | Custom SPARQL function used in [Halyard Bulk Update](tools#Halyard_Bulk_Update) and [Halyard Bulk Export](tools#Halyard_Bulk_Export). 31 | 32 | ### ~~`halyard:parallelSplitBy`~~ 33 | 34 | ~~Custom SPARQL function used in [Halyard Parallel Export](tools#Halyard_Parallel_Export).~~ 35 | 36 | ### ~~`halyard:decimateBy`~~ 37 | 38 | ## Custom data types 39 | 40 | ### `halyard:search` 41 | 42 | Custom data type passing the value as a query string to Elastic Search index (when configured). The value is replaced with all matching values retrieved from Elastic Search index during SPARQL query, during direct API repository operations, or during RDF4J Workbench exploration of the datasets. 43 | 44 | ## SAIL configuration predicates 45 | 46 | ### `halyard:tableName` 47 | 48 | ### `halyard:splitBits` 49 | 50 | ### `halyard:createTable` 51 | 52 | ### `halyard:pushStrategy` 53 | 54 | ### `halyard:evaluationTimeout` 55 | -------------------------------------------------------------------------------- /findbugs-exclude.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Halyard 2 | 3 | [![CI](https://api.travis-ci.org/Merck/Halyard.svg?branch=master)](https://travis-ci.org/Merck/Halyard) 4 | [![Coverage](https://codecov.io/github/Merck/Halyard/coverage.svg?branch=master)](https://codecov.io/gh/Merck/Halyard/) 5 | 6 | Halyard is an extremely horizontally scalable triple store with support for named graphs, designed for integration of extremely large semantic data models and for storage and SPARQL 1.1 querying of complete Linked Data universe snapshots. Halyard implementation is based on Eclipse RDF4J framework and Apache HBase database, and it is completely written in Java. 7 | 8 | **Author: [Adam Sotona](mailto:adam.sotona@merck.com)** 9 | 10 | **Discussion group: ** 11 | 12 | **Documentation: ** 13 | 14 | ## Get started 15 | 16 | [Download](https://github.com/Merck/Halyard/releases) and unzip the latest `halyard-sdk-.zip` bundle to a Apache Hadoop cluster node with configured Apache HBase client. 17 | 18 | Halyard is expected to run on an Apache Hadoop cluster node with configured Apache HBase client. Apache Hadoop and Apache HBase components are not bundled with Halyard. The runtime requirements are: 19 | 20 | * Apache Hadoop version 2.5.1 or higher 21 | * Apache HBase version 1.1.2 or higher 22 | * Java 8 Runtime 23 | 24 | *Note: Recommended Apache Hadoop distribution is the latest version of Hortonworks Data Platform (HDP) or Amazon Elastic Map Reduce (EMR).* 25 | 26 | See [Documentation](https://merck.github.io/Halyard) for usage examples, architecture information, and more. 27 | 28 | ## Repository contents 29 | 30 | * `common` - a library for direct mapping between an RDF data model and Apache HBase 31 | * `strategy` - a generic parallel asynchronous implementation of RDF4J Evaluation Strategy 32 | * `sail` - an implementation of the RDF4J Storage and Inference Layer on top of Apache HBase 33 | * `tools` - a set of command line and Apache Hadoop MapReduce tools for loading, updating, querying, and exporting the data with maximum performance 34 | * `sdk` - a distributable bundle of Eclipse RDF4J and Halyard for command line use on an Apache Hadoop cluster with configured HBase 35 | * `webapps` - a re-distribution of Eclipse RDF4J Web Applications (RDF4J-Server and RDF4J-Workbench), patched and enhanced to include Halyard as another RDF repository option 36 | -------------------------------------------------------------------------------- /rio/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | halyard-rio 5 | jar 6 | 7 | com.msd.gin.halyard 8 | halyard 9 | 2.5 10 | 11 | 12 | 13 | org.eclipse.rdf4j 14 | rdf4j-runtime 15 | ${rdf4j.version} 16 | pom 17 | 18 | 19 | junit 20 | junit 21 | 4.12 22 | test 23 | 24 | 25 | -------------------------------------------------------------------------------- /rio/src/main/java/com/msd/gin/halyard/rio/NDJSONLDParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.rio; 18 | 19 | import java.io.BufferedReader; 20 | import java.io.IOException; 21 | import java.io.InputStream; 22 | import java.io.InputStreamReader; 23 | import java.io.Reader; 24 | import java.io.StringReader; 25 | import java.nio.charset.StandardCharsets; 26 | import java.util.Arrays; 27 | import org.eclipse.rdf4j.rio.RDFFormat; 28 | import org.eclipse.rdf4j.rio.RDFHandlerException; 29 | import org.eclipse.rdf4j.rio.RDFParseException; 30 | import org.eclipse.rdf4j.rio.RDFParser; 31 | import org.eclipse.rdf4j.rio.RDFParserFactory; 32 | import org.eclipse.rdf4j.rio.jsonld.JSONLDParser; 33 | 34 | /** 35 | * 36 | * @author Adam Sotona (MSD) 37 | */ 38 | public class NDJSONLDParser extends JSONLDParser { 39 | 40 | public static class Factory implements RDFParserFactory { 41 | 42 | @Override 43 | public RDFFormat getRDFFormat() { 44 | return NDJSONLD; 45 | } 46 | 47 | @Override 48 | public RDFParser getParser() { 49 | return new NDJSONLDParser(); 50 | } 51 | } 52 | 53 | public static final RDFFormat NDJSONLD = new RDFFormat("ND-JSON-LD", Arrays.asList("application/x-nd-json-ld"), 54 | StandardCharsets.UTF_8, Arrays.asList("ndjsonld"), 55 | RDFFormat.SUPPORTS_NAMESPACES, RDFFormat.SUPPORTS_CONTEXTS); 56 | 57 | @Override 58 | public RDFFormat getRDFFormat() { 59 | return NDJSONLD; 60 | } 61 | 62 | @Override 63 | public void parse(InputStream in, String baseURI) throws IOException, RDFParseException, RDFHandlerException { 64 | parse(new InputStreamReader(in, StandardCharsets.UTF_8), baseURI); 65 | } 66 | 67 | @Override 68 | public void parse(Reader reader, String baseURI) throws IOException, RDFParseException, RDFHandlerException { 69 | BufferedReader br = new BufferedReader(reader); 70 | String line; 71 | while ((line = br.readLine()) != null) { 72 | super.parse(new StringReader(line), baseURI); 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /rio/src/main/resources/META-INF/services/org.eclipse.rdf4j.rio.RDFParserFactory: -------------------------------------------------------------------------------- 1 | com.msd.gin.halyard.rio.JSONParser$Factory 2 | com.msd.gin.halyard.rio.NDJSONLDParser$Factory 3 | -------------------------------------------------------------------------------- /rio/src/test/java/com/msd/gin/halyard/rio/JSONParserParseTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.rio; 18 | 19 | import java.io.StringReader; 20 | import java.util.Iterator; 21 | import java.util.TreeMap; 22 | import org.eclipse.rdf4j.model.Model; 23 | import org.eclipse.rdf4j.model.Statement; 24 | import org.eclipse.rdf4j.model.impl.LinkedHashModel; 25 | import org.eclipse.rdf4j.model.impl.SimpleValueFactory; 26 | import org.eclipse.rdf4j.rio.RDFFormat; 27 | import org.eclipse.rdf4j.rio.RDFParser; 28 | import org.eclipse.rdf4j.rio.Rio; 29 | import org.eclipse.rdf4j.rio.WriterConfig; 30 | import org.eclipse.rdf4j.rio.helpers.BasicWriterSettings; 31 | import org.eclipse.rdf4j.rio.helpers.ContextStatementCollector; 32 | import org.junit.Assert; 33 | import org.junit.Test; 34 | import org.junit.runner.RunWith; 35 | import org.junit.runners.Parameterized; 36 | 37 | /** 38 | * 39 | * @author Adam Sotona (MSD) 40 | */ 41 | @RunWith(Parameterized.class) 42 | public class JSONParserParseTest { 43 | 44 | @Parameterized.Parameters 45 | public static String[] data() { 46 | return new String[] {"empty", "emptyObj", "primitives", "rootArray", "nestedArrays", "example"}; 47 | } 48 | 49 | @Parameterized.Parameter 50 | public String parameter; 51 | 52 | @Test 53 | public void testParse() throws Exception { 54 | Model transformedModel = new LinkedHashModel(); 55 | RDFParser parser = new JSONParser(); 56 | parser.setValueFactory(SimpleValueFactory.getInstance()); 57 | parser.set(JSONParser.GENERATE_ONTOLOGY, true); 58 | parser.setRDFHandler(new ContextStatementCollector(transformedModel, SimpleValueFactory.getInstance())); 59 | parser.parse(JSONParserParseTest.class.getResourceAsStream(parameter + ".json"), "http://testParse/"+parameter + "/"); 60 | 61 | WriterConfig wc = new WriterConfig(); 62 | wc.set(BasicWriterSettings.PRETTY_PRINT, true); 63 | System.out.println("-------------- " + parameter + " ------------------"); 64 | Rio.write(transformedModel, System.out, RDFFormat.TURTLE, wc); 65 | 66 | Model expectedModel = Rio.parse(JSONParserParseTest.class.getResourceAsStream(parameter + ".ttl"), "http://testParse/" + parameter + "/", RDFFormat.TURTLE); 67 | 68 | JSONParserParseTest.assertEquals(expectedModel, transformedModel); 69 | } 70 | 71 | public static void assertEquals(Model expectedModel, Model transformedModel) { 72 | TreeMap fail = new TreeMap<>(); 73 | Iterator it = expectedModel.iterator(); 74 | while (it.hasNext()) { 75 | Statement st = it.next(); 76 | if (!transformedModel.contains(st)) { 77 | fail.put(st.toString(), "-" + st.toString()); 78 | } 79 | } 80 | it = transformedModel.iterator(); 81 | while (it.hasNext()) { 82 | Statement st = it.next(); 83 | if (!expectedModel.contains(st)) { 84 | fail.put(st.toString(), "+" + st.toString()); 85 | } 86 | } 87 | if (fail.size() > 0) { 88 | StringBuilder sb = new StringBuilder(); 89 | sb.append('\n'); 90 | for (String line : fail.values()) { 91 | sb.append(line).append('\n'); 92 | } 93 | Assert.fail(sb.toString()); 94 | } 95 | Assert.assertEquals(expectedModel.size(), transformedModel.size()); 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /rio/src/test/java/com/msd/gin/halyard/rio/JSONParserTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.rio; 18 | 19 | import java.io.IOException; 20 | import java.io.StringReader; 21 | import java.util.Collection; 22 | import org.eclipse.rdf4j.model.Model; 23 | import org.eclipse.rdf4j.model.impl.LinkedHashModel; 24 | import org.eclipse.rdf4j.model.impl.SimpleValueFactory; 25 | import org.eclipse.rdf4j.rio.ParserConfig; 26 | import org.eclipse.rdf4j.rio.RDFParser; 27 | import org.eclipse.rdf4j.rio.Rio; 28 | import org.eclipse.rdf4j.rio.RioSetting; 29 | import org.eclipse.rdf4j.rio.helpers.ContextStatementCollector; 30 | import org.junit.Assert; 31 | import static org.junit.Assert.assertNotNull; 32 | import org.junit.Test; 33 | 34 | /** 35 | * 36 | * @author Adam Sotona (MSD) 37 | */ 38 | public class JSONParserTest { 39 | 40 | @Test 41 | public void testJSONParserFactory() { 42 | assertNotNull(Rio.createParser(JSONParser.JSON)); 43 | } 44 | 45 | @Test 46 | public void testGetRDFFormat() { 47 | Assert.assertEquals(JSONParser.JSON, new JSONParser().getRDFFormat()); 48 | } 49 | 50 | @Test 51 | public void testGetSupportedSettings() { 52 | Collection> set = new JSONParser().getSupportedSettings(); 53 | Assert.assertTrue(set.contains(JSONParser.GENERATE_DATA)); 54 | Assert.assertTrue(set.contains(JSONParser.GENERATE_ONTOLOGY)); 55 | } 56 | 57 | @Test 58 | public void testMethodsThatDoNothing() { 59 | RDFParser p = new JSONParser(); 60 | p.setVerifyData(true); 61 | p.setPreserveBNodeIDs(true); 62 | p.setStopAtFirstError(true); 63 | p.setDatatypeHandling(RDFParser.DatatypeHandling.NORMALIZE); 64 | p.setParseErrorListener(null); 65 | p.setParseLocationListener(null); 66 | } 67 | 68 | @Test 69 | public void testParserSettings() { 70 | RDFParser p = new JSONParser(); 71 | ParserConfig pc = new ParserConfig(); 72 | Assert.assertSame(pc, p.setParserConfig(pc).getParserConfig()); 73 | } 74 | 75 | @Test (expected = IllegalArgumentException.class) 76 | public void testParseWrongJSON() throws IOException { 77 | new JSONParser().parse(new StringReader("true"), "http://test/"); 78 | } 79 | 80 | @Test (expected = IllegalArgumentException.class) 81 | public void testParseInvalidURI() throws IOException { 82 | new JSONParser().parse(new StringReader("[]"), "invalid_uri"); 83 | } 84 | 85 | @Test (expected = IllegalArgumentException.class) 86 | public void testParserMultiObj() throws IOException { 87 | Model transformedModel = new LinkedHashModel(); 88 | RDFParser parser = new JSONParser(); 89 | parser.setRDFHandler(new ContextStatementCollector(transformedModel, SimpleValueFactory.getInstance())); 90 | parser.parse(new StringReader("{}{}"), "http://test/"); 91 | } 92 | 93 | @Test 94 | public void testParse() throws Exception { 95 | JSONParser p = new JSONParser(); 96 | p.set(JSONParser.GENERATE_DATA, false); 97 | p.set(JSONParser.GENERATE_ONTOLOGY, false); 98 | p.parse(new StringReader("{\"a\": [\"b\", \"c\", \"d\"],\"e\": [{\"f\": \"g\"}, {\"h\": \"i\"}]}"), "http://test/"); 99 | } 100 | 101 | @Test (expected = RuntimeException.class) 102 | public void testInvalidMDAlgorithm() { 103 | new JSONParser("invalid"); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /rio/src/test/java/com/msd/gin/halyard/rio/NDJSONLDParserTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.rio; 18 | 19 | import org.eclipse.rdf4j.model.Model; 20 | import org.eclipse.rdf4j.model.impl.LinkedHashModel; 21 | import org.eclipse.rdf4j.model.impl.SimpleValueFactory; 22 | import org.eclipse.rdf4j.rio.RDFFormat; 23 | import org.eclipse.rdf4j.rio.RDFParser; 24 | import org.eclipse.rdf4j.rio.Rio; 25 | import org.eclipse.rdf4j.rio.helpers.ContextStatementCollector; 26 | import org.junit.Assert; 27 | import static org.junit.Assert.assertNotNull; 28 | import org.junit.Test; 29 | 30 | /** 31 | * 32 | * @author Adam Sotona (MSD) 33 | */ 34 | public class NDJSONLDParserTest { 35 | 36 | @Test 37 | public void testNDJSONLDParserFactory() { 38 | assertNotNull(Rio.createParser(NDJSONLDParser.NDJSONLD)); 39 | } 40 | 41 | @Test 42 | public void testNDJSONLDParser() throws Exception { 43 | Model transformedModel = new LinkedHashModel(); 44 | RDFParser parser = new NDJSONLDParser(); 45 | parser.setRDFHandler(new ContextStatementCollector(transformedModel, SimpleValueFactory.getInstance())); 46 | parser.parse(NDJSONLDParserTest.class.getResourceAsStream("efo_test.ndjsonld"), "http://test/"); 47 | 48 | Model expectedModel = Rio.parse(NDJSONLDParserTest.class.getResourceAsStream("efo_test.ttl"), "http://test/", RDFFormat.TURTLE); 49 | 50 | JSONParserParseTest.assertEquals(expectedModel, transformedModel); 51 | } 52 | 53 | @Test 54 | public void testGetRDFFormat() { 55 | Assert.assertEquals(NDJSONLDParser.NDJSONLD, new NDJSONLDParser().getRDFFormat()); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /rio/src/test/resources/com/msd/gin/halyard/rio/efo_test.ndjsonld: -------------------------------------------------------------------------------- 1 | {"@id":"http://www.ebi.ac.uk/efo/EFO_0007123","label":{"@language":"en","@value":"So-Eum"},"parents":["http://www.ebi.ac.uk/efo/EFO_0007119"],"@context":{"@vocab":"http://www.ebi.ac.uk/efo/","rdfs":"http://www.w3.org/2000/01/rdf-schema#","label":"rdfs:label","parents":{"@id":"rdfs:subClassOf","@type":"@id","@container":"@set"},"alternative_terms":{"@id":"alternative_term","@container":"@set"}}} 2 | {"@id":"http://www.ebi.ac.uk/efo/EFO_0006347","alternative_terms":["PAE"],"label":{"@language":"en","@value":"pulmonary artery enlargement"},"parents":["http://www.ebi.ac.uk/efo/EFO_0003765"],"@context":{"@vocab":"http://www.ebi.ac.uk/efo/","rdfs":"http://www.w3.org/2000/01/rdf-schema#","label":"rdfs:label","parents":{"@id":"rdfs:subClassOf","@type":"@id","@container":"@set"},"alternative_terms":{"@id":"alternative_term","@container":"@set"}}} 3 | {"@id":"http://www.ebi.ac.uk/efo/EFO_0004827","label":"economic and social preference","parents":["http://www.ifomis.org/bfo/1.1/snap#Quality"],"@context":{"@vocab":"http://www.ebi.ac.uk/efo/","rdfs":"http://www.w3.org/2000/01/rdf-schema#","label":"rdfs:label","parents":{"@id":"rdfs:subClassOf","@type":"@id","@container":"@set"},"alternative_terms":{"@id":"alternative_term","@container":"@set"}}} 4 | {"@id":"http://www.ebi.ac.uk/efo/EFO_0001270","alternative_terms":["alpha","alpha mating type (yeast)","mating_type_alpha"],"label":"mating type alpha","parents":["http://www.ebi.ac.uk/efo/EFO_0001268"],"@context":{"@vocab":"http://www.ebi.ac.uk/efo/","rdfs":"http://www.w3.org/2000/01/rdf-schema#","label":"rdfs:label","parents":{"@id":"rdfs:subClassOf","@type":"@id","@container":"@set"},"alternative_terms":{"@id":"alternative_term","@container":"@set"}}} 5 | {"@id":"http://www.ebi.ac.uk/efo/EFO_0007120","label":{"@language":"en","@value":"Tae-Yang"},"parents":["http://www.ebi.ac.uk/efo/EFO_0007119"],"@context":{"@vocab":"http://www.ebi.ac.uk/efo/","rdfs":"http://www.w3.org/2000/01/rdf-schema#","label":"rdfs:label","parents":{"@id":"rdfs:subClassOf","@type":"@id","@container":"@set"},"alternative_terms":{"@id":"alternative_term","@container":"@set"}}} 6 | {"@id":"http://www.ebi.ac.uk/efo/EFO_0002614","alternative_terms":["Drug resistance to insulin (disorder)"],"label":"insulin resistance","parents":["http://purl.obolibrary.org/obo/HP_0001939"],"@context":{"@vocab":"http://www.ebi.ac.uk/efo/","rdfs":"http://www.w3.org/2000/01/rdf-schema#","label":"rdfs:label","parents":{"@id":"rdfs:subClassOf","@type":"@id","@container":"@set"},"alternative_terms":{"@id":"alternative_term","@container":"@set"}}} 7 | {"@id":"http://www.ebi.ac.uk/efo/EFO_0003847","alternative_terms":["Idiocy","MENTAL DEFIC","Deficiency, Mental","Disabilities, Intellectual","Disability, Intellectual","Mental Retardations, Psychosocial","Retardations, Psychosocial Mental","Retardation, Mental","Mental Retardation, Psychosocial","Intellectual Disabilities","Psychosocial Mental Retardation","Intellectual Disability","Retardation, Psychosocial Mental","Mental Deficiency","Deficiencies, Mental","DEFIC MENTAL","Psychosocial Mental Retardations","Mental Deficiencies"],"label":"mental retardation","parents":["http://purl.obolibrary.org/obo/HP_0000707"],"@context":{"@vocab":"http://www.ebi.ac.uk/efo/","rdfs":"http://www.w3.org/2000/01/rdf-schema#","label":"rdfs:label","parents":{"@id":"rdfs:subClassOf","@type":"@id","@container":"@set"},"alternative_terms":{"@id":"alternative_term","@container":"@set"}}} 8 | {"@id":"http://www.ebi.ac.uk/efo/EFO_0006852","label":{"@language":"en","@value":"metastatic"},"parents":["http://purl.obolibrary.org/obo/HP_0012823"],"@context":{"@vocab":"http://www.ebi.ac.uk/efo/","rdfs":"http://www.w3.org/2000/01/rdf-schema#","label":"rdfs:label","parents":{"@id":"rdfs:subClassOf","@type":"@id","@container":"@set"},"alternative_terms":{"@id":"alternative_term","@container":"@set"}}} 9 | {"@id":"http://www.ebi.ac.uk/efo/EFO_0007654","label":"clinical and behavioural ideal cardiovascular health","parents":["http://www.ebi.ac.uk/efo/EFO_0007652"],"@context":{"@vocab":"http://www.ebi.ac.uk/efo/","rdfs":"http://www.w3.org/2000/01/rdf-schema#","label":"rdfs:label","parents":{"@id":"rdfs:subClassOf","@type":"@id","@container":"@set"},"alternative_terms":{"@id":"alternative_term","@container":"@set"}}} 10 | {"@id":"http://www.ebi.ac.uk/efo/EFO_0007063","label":"organism status","parents":["http://www.ifomis.org/bfo/1.1/snap#Quality"],"@context":{"@vocab":"http://www.ebi.ac.uk/efo/","rdfs":"http://www.w3.org/2000/01/rdf-schema#","label":"rdfs:label","parents":{"@id":"rdfs:subClassOf","@type":"@id","@container":"@set"},"alternative_terms":{"@id":"alternative_term","@container":"@set"}}} 11 | -------------------------------------------------------------------------------- /rio/src/test/resources/com/msd/gin/halyard/rio/efo_test.ttl: -------------------------------------------------------------------------------- 1 | @prefix rdfs: . 2 | 3 | rdfs:label "So-Eum"@en ; 4 | rdfs:subClassOf . 5 | 6 | "PAE" ; 7 | rdfs:label "pulmonary artery enlargement"@en ; 8 | rdfs:subClassOf . 9 | 10 | rdfs:label "economic and social preference" ; 11 | rdfs:subClassOf . 12 | 13 | "alpha" , "alpha mating type (yeast)" , "mating_type_alpha" ; 14 | rdfs:label "mating type alpha" ; 15 | rdfs:subClassOf . 16 | 17 | rdfs:label "Tae-Yang"@en ; 18 | rdfs:subClassOf . 19 | 20 | "Drug resistance to insulin (disorder)" ; 21 | rdfs:label "insulin resistance" ; 22 | rdfs:subClassOf . 23 | 24 | "Idiocy" , "MENTAL DEFIC" , "Deficiency, Mental" , "Disabilities, Intellectual" , "Disability, Intellectual" , "Mental Retardations, Psychosocial" , "Retardations, Psychosocial Mental" , "Retardation, Mental" , "Mental Retardation, Psychosocial" , "Intellectual Disabilities" , "Psychosocial Mental Retardation" , "Intellectual Disability" , "Retardation, Psychosocial Mental" , "Mental Deficiency" , "Deficiencies, Mental" , "DEFIC MENTAL" , "Psychosocial Mental Retardations" , "Mental Deficiencies" ; 25 | rdfs:label "mental retardation" ; 26 | rdfs:subClassOf . 27 | 28 | rdfs:label "metastatic"@en ; 29 | rdfs:subClassOf . 30 | 31 | rdfs:label "clinical and behavioural ideal cardiovascular health" ; 32 | rdfs:subClassOf . 33 | 34 | rdfs:label "organism status" ; 35 | rdfs:subClassOf . 36 | 37 | 38 | -------------------------------------------------------------------------------- /rio/src/test/resources/com/msd/gin/halyard/rio/empty.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Merck/Halyard/fabb338edb14276aa7ef9f5f9b59f680335b22d3/rio/src/test/resources/com/msd/gin/halyard/rio/empty.json -------------------------------------------------------------------------------- /rio/src/test/resources/com/msd/gin/halyard/rio/empty.ttl: -------------------------------------------------------------------------------- 1 | @prefix rdf: . 2 | @prefix rdfs: . 3 | @prefix owl: . 4 | @prefix : . 5 | 6 | a owl:Ontology . 7 | 8 | -------------------------------------------------------------------------------- /rio/src/test/resources/com/msd/gin/halyard/rio/emptyObj.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /rio/src/test/resources/com/msd/gin/halyard/rio/emptyObj.ttl: -------------------------------------------------------------------------------- 1 | @prefix rdf: . 2 | @prefix rdfs: . 3 | @prefix owl: . 4 | @prefix : . 5 | 6 | a owl:Ontology . 7 | 8 | :K1gB0IYDwNX021i3HcX_IuHj07iKxD3MRGjRy7rzxGMcBmqEiBjE7VL7Mp7Iqep3qkcNOXBA8tk40anI2m-pbA a ::Node . 9 | 10 | ::Node a rdfs:Class . 11 | 12 | -------------------------------------------------------------------------------- /rio/src/test/resources/com/msd/gin/halyard/rio/example.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": "0001", 4 | "type": "donut", 5 | "name": "Cake", 6 | "ppu": 0.55, 7 | "batters": 8 | { 9 | "batter": 10 | [ 11 | { 12 | "id": "1001", 13 | "type": "Regular" 14 | }, 15 | { 16 | "id": "1002", 17 | "type": "Chocolate" 18 | }, 19 | { 20 | "id": "1003", 21 | "type": "Blueberry" 22 | }, 23 | { 24 | "id": "1004", 25 | "type": "Devil's Food" 26 | } 27 | ] 28 | }, 29 | "topping": 30 | [ 31 | { 32 | "id": "5001", 33 | "type": "None" 34 | }, 35 | { 36 | "id": "5002", 37 | "type": "Glazed" 38 | }, 39 | { 40 | "id": "5005", 41 | "type": "Sugar" 42 | }, 43 | { 44 | "id": "5007", 45 | "type": "Powdered Sugar" 46 | }, 47 | { 48 | "id": "5006", 49 | "type": "Chocolate with Sprinkles" 50 | }, 51 | { 52 | "id": "5003", 53 | "type": "Chocolate" 54 | }, 55 | { 56 | "id": "5004", 57 | "type": "Maple" 58 | } 59 | ] 60 | }, 61 | { 62 | "id": "0002", 63 | "type": "donut", 64 | "name": "Raised", 65 | "ppu": 0.55, 66 | "batters": 67 | { 68 | "batter": 69 | [ 70 | { 71 | "id": "1001", 72 | "type": "Regular" 73 | } 74 | ] 75 | }, 76 | "topping": 77 | [ 78 | { 79 | "id": "5001", 80 | "type": "None" 81 | }, 82 | { 83 | "id": "5002", 84 | "type": "Glazed" 85 | }, 86 | { 87 | "id": "5005", 88 | "type": "Sugar" 89 | }, 90 | { 91 | "id": "5003", 92 | "type": "Chocolate" 93 | }, 94 | { 95 | "id": "5004", 96 | "type": "Maple" 97 | } 98 | ] 99 | }, 100 | { 101 | "id": "0003", 102 | "type": "donut", 103 | "name": "Old Fashioned", 104 | "ppu": 0.55, 105 | "batters": 106 | { 107 | "batter": 108 | [ 109 | { 110 | "id": "1001", 111 | "type": "Regular" 112 | }, 113 | { 114 | "id": "1002", 115 | "type": "Chocolate" 116 | } 117 | ] 118 | }, 119 | "topping": 120 | [ 121 | { 122 | "id": "5001", 123 | "type": "None" 124 | }, 125 | { 126 | "id": "5002", 127 | "type": "Glazed" 128 | }, 129 | { 130 | "id": "5003", 131 | "type": "Chocolate" 132 | }, 133 | { 134 | "id": "5004", 135 | "type": "Maple" 136 | } 137 | ] 138 | } 139 | ] -------------------------------------------------------------------------------- /rio/src/test/resources/com/msd/gin/halyard/rio/nestedArrays.json: -------------------------------------------------------------------------------- 1 | { 2 | "a": ["b", "c", "d"], 3 | "e": [{"f": "g"}, {"h": "i"}] 4 | } 5 | -------------------------------------------------------------------------------- /rio/src/test/resources/com/msd/gin/halyard/rio/nestedArrays.ttl: -------------------------------------------------------------------------------- 1 | @prefix rdf: . 2 | @prefix rdfs: . 3 | @prefix owl: . 4 | @prefix : . 5 | 6 | a owl:Ontology . 7 | 8 | :m4sUtoNKHXvHoyAbBEDOWeDENVFnt5CtRsNvsWebfVRUZKlhqOYv_oA-puVj4ZUD8qc5KbAmHx_zfUZGeQNRYA a ::Node ; 9 | :a "b" , "c" , "d" ; 10 | :e :m4sUtoNKHXvHoyAbBEDOWeDENVFnt5CtRsNvsWebfVRUZKlhqOYv_oA-puVj4ZUD8qc5KbAmHx_zfUZGeQNRYA.e:0 , :m4sUtoNKHXvHoyAbBEDOWeDENVFnt5CtRsNvsWebfVRUZKlhqOYv_oA-puVj4ZUD8qc5KbAmHx_zfUZGeQNRYA.e:1 . 11 | 12 | ::Node a rdfs:Class . 13 | 14 | :a a owl:DatatypeProperty ; 15 | rdfs:label "a" ; 16 | rdfs:domain ::Node . 17 | 18 | :e a owl:ObjectProperty ; 19 | rdfs:label "e" ; 20 | rdfs:domain ::Node ; 21 | rdfs:range :e:Node . 22 | 23 | :m4sUtoNKHXvHoyAbBEDOWeDENVFnt5CtRsNvsWebfVRUZKlhqOYv_oA-puVj4ZUD8qc5KbAmHx_zfUZGeQNRYA.e:0 a :e:Node ; 24 | :e:index 0 ; 25 | :e.f "g" . 26 | 27 | :e:Node a rdfs:Class . 28 | 29 | :e:index a owl:DatatypeProperty ; 30 | rdfs:label "index" ; 31 | rdfs:domain :e:Node . 32 | 33 | :e.f a owl:DatatypeProperty ; 34 | rdfs:label "f" ; 35 | rdfs:domain :e:Node . 36 | 37 | :m4sUtoNKHXvHoyAbBEDOWeDENVFnt5CtRsNvsWebfVRUZKlhqOYv_oA-puVj4ZUD8qc5KbAmHx_zfUZGeQNRYA.e:1 a :e:Node ; 38 | :e:index 1 ; 39 | :e.h "i" . 40 | 41 | :e.h a owl:DatatypeProperty ; 42 | rdfs:label "h" ; 43 | rdfs:domain :e:Node . 44 | 45 | -------------------------------------------------------------------------------- /rio/src/test/resources/com/msd/gin/halyard/rio/primitives.json: -------------------------------------------------------------------------------- 1 | { 2 | "string": "text", 3 | "boolean": true, 4 | "integer": 42, 5 | "double": 3.14, 6 | "boolArray": [true, false, true, false], 7 | "intArray": [1, 2, 3, 4, 5], 8 | "doubleArray": [1.1, 2.2, 3.3], 9 | "mixedArray": ["string", true, 47, 3.15, {}] 10 | } 11 | -------------------------------------------------------------------------------- /rio/src/test/resources/com/msd/gin/halyard/rio/primitives.ttl: -------------------------------------------------------------------------------- 1 | @prefix rdf: . 2 | @prefix rdfs: . 3 | @prefix owl: . 4 | @prefix : . 5 | 6 | a owl:Ontology . 7 | 8 | :aZcwCir_UAqtD78F_Y_-m_9mrm65EvIUkCh9uT3-in9ek3p8Oa2GJi4nl-3RjoieUoNEI_crpvDuNuGZ02w2BA a ::Node ; 9 | :string "text" ; 10 | :integer "42" ; 11 | :double "3.14" ; 12 | :intArray "1" , "2" , "3" , "4" , "5" ; 13 | :doubleArray "1.1" , "2.2" , "3.3" ; 14 | :mixedArray "string" , "47" , "3.15" , :aZcwCir_UAqtD78F_Y_-m_9mrm65EvIUkCh9uT3-in9ek3p8Oa2GJi4nl-3RjoieUoNEI_crpvDuNuGZ02w2BA.mixedArray:4 . 15 | 16 | ::Node a rdfs:Class . 17 | 18 | :string a owl:DatatypeProperty ; 19 | rdfs:label "string" ; 20 | rdfs:domain ::Node . 21 | 22 | :boolean a owl:DatatypeProperty ; 23 | rdfs:label "boolean" ; 24 | rdfs:domain ::Node . 25 | 26 | :integer a owl:DatatypeProperty ; 27 | rdfs:label "integer" ; 28 | rdfs:domain ::Node . 29 | 30 | :double a owl:DatatypeProperty ; 31 | rdfs:label "double" ; 32 | rdfs:domain ::Node . 33 | 34 | :boolArray a owl:DatatypeProperty ; 35 | rdfs:label "boolArray" ; 36 | rdfs:domain ::Node . 37 | 38 | :intArray a owl:DatatypeProperty ; 39 | rdfs:label "intArray" ; 40 | rdfs:domain ::Node . 41 | 42 | :doubleArray a owl:DatatypeProperty ; 43 | rdfs:label "doubleArray" ; 44 | rdfs:domain ::Node . 45 | 46 | :mixedArray a owl:DatatypeProperty , owl:ObjectProperty ; 47 | rdfs:label "mixedArray" ; 48 | rdfs:domain ::Node ; 49 | rdfs:range :mixedArray:Node . 50 | 51 | :aZcwCir_UAqtD78F_Y_-m_9mrm65EvIUkCh9uT3-in9ek3p8Oa2GJi4nl-3RjoieUoNEI_crpvDuNuGZ02w2BA.mixedArray:4 a :mixedArray:Node ; 52 | :mixedArray:index 4 . 53 | 54 | :mixedArray:Node a rdfs:Class . 55 | 56 | :mixedArray:index a owl:DatatypeProperty ; 57 | rdfs:label "index" ; 58 | rdfs:domain :mixedArray:Node . 59 | 60 | -------------------------------------------------------------------------------- /rio/src/test/resources/com/msd/gin/halyard/rio/rootArray.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "a": "b" 4 | }, 5 | { 6 | "c": "d" 7 | } 8 | ] 9 | -------------------------------------------------------------------------------- /rio/src/test/resources/com/msd/gin/halyard/rio/rootArray.ttl: -------------------------------------------------------------------------------- 1 | @prefix rdf: . 2 | @prefix rdfs: . 3 | @prefix owl: . 4 | @prefix : . 5 | 6 | a owl:Ontology . 7 | 8 | a ::Node ; 9 | :a "b" . 10 | 11 | ::Node a rdfs:Class . 12 | 13 | :a a owl:DatatypeProperty ; 14 | rdfs:label "a" ; 15 | rdfs:domain ::Node . 16 | 17 | :nYQMyuwHhEx7iPAlnw1vWUU4Ww7NDfKdZ9jBX6pkZ3htlU09GEWIe4p1Tpgn3jHPTqxoYWu2QyRJMdzuCEPniw a ::Node ; 18 | :c "d" . 19 | 20 | :c a owl:DatatypeProperty ; 21 | rdfs:label "c" ; 22 | rdfs:domain ::Node . 23 | 24 | -------------------------------------------------------------------------------- /sail/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | halyard-sail 5 | jar 6 | 7 | com.msd.gin.halyard 8 | halyard 9 | 2.5 10 | 11 | 12 | 13 | ${project.groupId} 14 | halyard-common 15 | ${project.version} 16 | 17 | 18 | ${project.groupId} 19 | halyard-strategy 20 | ${project.version} 21 | 22 | 23 | org.apache.hbase 24 | hbase-server 25 | ${hbase.version} 26 | provided 27 | 28 | 29 | org.eclipse.rdf4j 30 | rdf4j-console 31 | ${rdf4j.version} 32 | 33 | 34 | ch.qos.logback 35 | logback-classic 36 | 37 | 38 | 39 | 40 | ${project.groupId} 41 | halyard-common 42 | ${project.version} 43 | test-jar 44 | test 45 | 46 | 47 | org.apache.hadoop 48 | hadoop-mapreduce-client-jobclient 49 | ${hadoop.version} 50 | test 51 | tests 52 | 53 | 54 | org.apache.hadoop 55 | hadoop-common 56 | ${hadoop.version} 57 | test 58 | tests 59 | 60 | 61 | org.apache.hadoop 62 | hadoop-mapreduce-client-hs 63 | ${hadoop.version} 64 | test 65 | 66 | 67 | org.apache.hadoop 68 | hadoop-yarn-server-resourcemanager 69 | ${hadoop.version} 70 | test 71 | 72 | 73 | org.apache.hadoop 74 | hadoop-yarn-server-tests 75 | ${hadoop.version} 76 | test 77 | tests 78 | 79 | 80 | org.json 81 | json 82 | 20140107 83 | 84 | 85 | commons-codec 86 | commons-codec 87 | 1.9 88 | 89 | 90 | -------------------------------------------------------------------------------- /sail/src/main/java/com/msd/gin/halyard/sail/HBaseSailFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.sail; 18 | 19 | import org.apache.hadoop.hbase.HBaseConfiguration; 20 | import org.eclipse.rdf4j.sail.Sail; 21 | import org.eclipse.rdf4j.sail.config.SailConfigException; 22 | import org.eclipse.rdf4j.sail.config.SailFactory; 23 | import org.eclipse.rdf4j.sail.config.SailImplConfig; 24 | 25 | /** 26 | * Factory for constructing an HBaseSail instance. 27 | * @author Adam Sotona (MSD) 28 | */ 29 | public final class HBaseSailFactory implements SailFactory { 30 | 31 | /** 32 | * String HBaseSail type identification 33 | */ 34 | public static final String SAIL_TYPE = "openrdf:HBaseStore"; 35 | 36 | @Override 37 | public String getSailType() { 38 | return SAIL_TYPE; 39 | } 40 | 41 | /** 42 | * Factory method for instantiating an HBaseSailConfig 43 | * @return new HBaseSailConfig instance 44 | */ 45 | @Override 46 | public SailImplConfig getConfig() { 47 | return new HBaseSailConfig(); 48 | } 49 | 50 | @Override 51 | public Sail getSail(SailImplConfig config) throws SailConfigException { 52 | if (!SAIL_TYPE.equals(config.getType())) { 53 | throw new SailConfigException("Invalid Sail type: " + config.getType()); 54 | } 55 | if (config instanceof HBaseSailConfig) { 56 | HBaseSailConfig hconfig = (HBaseSailConfig) config; 57 | //instantiate the sail 58 | HBaseSail sail = new HBaseSail(HBaseConfiguration.create(), hconfig.getTablespace(), hconfig.isCreate(), hconfig.getSplitBits(), hconfig.isPush(), hconfig.getEvaluationTimeout(), hconfig.getElasticIndexURL(), null); 59 | return sail; 60 | } else { 61 | throw new SailConfigException("Invalid configuration: " + config); 62 | } 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /sail/src/main/java/com/msd/gin/halyard/sail/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Classes that support the Storage And Inference Layer (SAIL) interface between RDF4J and HBase 3 | */ 4 | package com.msd.gin.halyard.sail; -------------------------------------------------------------------------------- /sail/src/main/resources/META-INF/services/org.eclipse.rdf4j.sail.config.SailFactory: -------------------------------------------------------------------------------- 1 | com.msd.gin.halyard.sail.HBaseSailFactory -------------------------------------------------------------------------------- /sail/src/test/java/com/msd/gin/halyard/sail/HBaseRepositoryManagerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.sail; 18 | 19 | import com.msd.gin.halyard.common.HBaseServerTestInstance; 20 | import java.net.MalformedURLException; 21 | import java.util.Collection; 22 | import org.apache.http.client.HttpClient; 23 | import org.apache.http.impl.client.DefaultHttpClient; 24 | import org.eclipse.rdf4j.repository.Repository; 25 | import org.eclipse.rdf4j.repository.config.RepositoryConfig; 26 | import org.eclipse.rdf4j.repository.sail.config.SailRepositoryConfig; 27 | import org.eclipse.rdf4j.sail.memory.config.MemoryStoreConfig; 28 | import org.junit.Test; 29 | import static org.junit.Assert.*; 30 | 31 | /** 32 | * 33 | * @author Adam Sotona (MSD) 34 | */ 35 | public class HBaseRepositoryManagerTest { 36 | 37 | @Test (expected = MalformedURLException.class) 38 | public void testGetLocation() throws Exception { 39 | new HBaseRepositoryManager().getLocation(); 40 | } 41 | 42 | @Test 43 | public void testHttpClient() { 44 | HBaseRepositoryManager rm = new HBaseRepositoryManager(); 45 | assertNull(rm.getHttpClient()); 46 | @SuppressWarnings("deprecation") 47 | HttpClient cl = new DefaultHttpClient(); 48 | rm.setHttpClient(cl); 49 | assertEquals(cl, rm.getHttpClient()); 50 | } 51 | 52 | @Test 53 | public void testAddRepositoryPersists() throws Exception { 54 | HBaseRepositoryManager rm = new HBaseRepositoryManager(); 55 | rm.overrideConfiguration(HBaseServerTestInstance.getInstanceConfig()); 56 | rm.init(); 57 | rm.addRepositoryConfig(new RepositoryConfig("repoTest", new SailRepositoryConfig(new MemoryStoreConfig(false)))); 58 | assertTrue(rm.getAllRepositories().contains(rm.getRepository("repoTest"))); 59 | rm.shutDown(); 60 | //test persistence 61 | rm = new HBaseRepositoryManager(); 62 | rm.overrideConfiguration(HBaseServerTestInstance.getInstanceConfig()); 63 | rm.init(); 64 | assertTrue(rm.getAllRepositories().contains(rm.getRepository("repoTest"))); 65 | rm.shutDown(); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /sail/src/test/java/com/msd/gin/halyard/sail/HBaseSailAddRemoveTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.sail; 18 | 19 | import com.msd.gin.halyard.common.HBaseServerTestInstance; 20 | import java.util.Arrays; 21 | import java.util.Collection; 22 | import org.eclipse.rdf4j.common.iteration.CloseableIteration; 23 | import org.eclipse.rdf4j.model.IRI; 24 | import org.eclipse.rdf4j.model.Resource; 25 | import org.eclipse.rdf4j.model.Statement; 26 | import org.eclipse.rdf4j.model.Value; 27 | import org.eclipse.rdf4j.model.impl.SimpleValueFactory; 28 | import org.eclipse.rdf4j.sail.SailException; 29 | import org.junit.AfterClass; 30 | import org.junit.Test; 31 | import static org.junit.Assert.*; 32 | import org.junit.BeforeClass; 33 | import org.junit.runner.RunWith; 34 | import org.junit.runners.Parameterized; 35 | 36 | /** 37 | * 38 | * @author Adam Sotona (MSD) 39 | */ 40 | @RunWith(Parameterized.class) 41 | public class HBaseSailAddRemoveTest { 42 | private static final Resource SUBJ = SimpleValueFactory.getInstance().createIRI("http://whatever/subject/"); 43 | private static final IRI PRED = SimpleValueFactory.getInstance().createIRI("http://whatever/pred/"); 44 | private static final Value OBJ = SimpleValueFactory.getInstance().createLiteral("whatever literal"); 45 | private static final IRI CONTEXT = SimpleValueFactory.getInstance().createIRI("http://whatever/cont/"); 46 | 47 | private static HBaseSail explicitSail; 48 | 49 | @Parameterized.Parameters 50 | public static Collection data() { 51 | return Arrays.asList(new Object[][] { 52 | {null, null, null}, 53 | {SUBJ, null, null}, 54 | {null, PRED, null}, 55 | {null, null, OBJ}, 56 | {SUBJ, PRED, null}, 57 | {null, PRED, OBJ}, 58 | {SUBJ, null, OBJ}, 59 | {SUBJ, PRED, OBJ}, 60 | }); 61 | } 62 | 63 | @BeforeClass 64 | public static void setup() throws Exception { 65 | explicitSail = new HBaseSail(HBaseServerTestInstance.getInstanceConfig(), "testAddRemove", true, 0, true, 0, null, null); 66 | explicitSail.initialize(); 67 | } 68 | 69 | @AfterClass 70 | public static void teardown() throws Exception { 71 | explicitSail.shutDown(); 72 | } 73 | 74 | private final Resource subj; 75 | private final IRI pred; 76 | private final Value obj; 77 | 78 | public HBaseSailAddRemoveTest(Resource subj, IRI pred, Value obj) { 79 | this.subj = subj; 80 | this.pred = pred; 81 | this.obj = obj; 82 | } 83 | 84 | @Test 85 | public void testAddAndRemoveExplicitStatements() throws Exception { 86 | explicitSail.addStatement(null, SUBJ, PRED, OBJ); 87 | explicitSail.addStatement(null, SUBJ, PRED, OBJ, CONTEXT); 88 | explicitSail.commit(); 89 | CloseableIteration iter; 90 | iter = explicitSail.getStatements(null, null, null, true); 91 | assertTrue(iter.hasNext()); 92 | iter.next(); 93 | assertTrue(iter.hasNext()); 94 | Statement st = iter.next(); 95 | assertFalse(iter.hasNext()); 96 | iter.close(); 97 | assertEquals(SUBJ, st.getSubject()); 98 | assertEquals(PRED, st.getPredicate()); 99 | assertEquals(OBJ, st.getObject()); 100 | iter = explicitSail.getStatements(null, null, null, true, CONTEXT); 101 | assertTrue(iter.hasNext()); 102 | st = iter.next(); 103 | assertFalse(iter.hasNext()); 104 | iter.close(); 105 | assertEquals(SUBJ, st.getSubject()); 106 | assertEquals(PRED, st.getPredicate()); 107 | assertEquals(OBJ, st.getObject()); 108 | assertEquals(CONTEXT, st.getContext()); 109 | explicitSail.removeStatements(subj, pred, obj, CONTEXT); 110 | iter = explicitSail.getStatements(null, null, null, true); 111 | assertTrue(iter.hasNext()); 112 | iter.close(); 113 | explicitSail.removeStatements(subj, pred, obj); 114 | iter = explicitSail.getStatements(null, null, null, true); 115 | assertFalse(iter.hasNext()); 116 | iter.close(); 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /sail/src/test/java/com/msd/gin/halyard/sail/HBaseSailFactoryTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.sail; 18 | 19 | import org.eclipse.rdf4j.model.Model; 20 | import org.eclipse.rdf4j.model.Resource; 21 | import org.eclipse.rdf4j.sail.Sail; 22 | import org.eclipse.rdf4j.sail.config.SailConfigException; 23 | import org.eclipse.rdf4j.sail.config.SailImplConfig; 24 | import org.junit.Test; 25 | import static org.junit.Assert.*; 26 | 27 | /** 28 | * 29 | * @author Adam Sotona (MSD) 30 | */ 31 | public class HBaseSailFactoryTest { 32 | 33 | @Test 34 | public void testGetSailType() { 35 | assertEquals("openrdf:HBaseStore", new HBaseSailFactory().getSailType()); 36 | } 37 | 38 | @Test 39 | public void testGetConfig() { 40 | assertTrue(new HBaseSailFactory().getConfig() instanceof HBaseSailConfig); 41 | } 42 | 43 | @Test 44 | public void testGetSail() throws Exception { 45 | HBaseSailConfig hbsc = new HBaseSailConfig(); 46 | hbsc.setCreate(false); 47 | hbsc.setPush(false); 48 | hbsc.setSplitBits(3); 49 | hbsc.setEvaluationTimeout(480); 50 | hbsc.setTablespace("testtable"); 51 | hbsc.setElasticIndexURL("http://whatever/index"); 52 | Sail sail = new HBaseSailFactory().getSail(hbsc); 53 | assertTrue(sail instanceof HBaseSail); 54 | HBaseSail hbs = (HBaseSail)sail; 55 | assertFalse(hbs.create); 56 | assertFalse(hbs.pushStrategy); 57 | assertEquals(3, hbs.splitBits); 58 | assertEquals("testtable", hbs.tableName); 59 | assertEquals(480, hbs.evaluationTimeout); 60 | assertEquals("http://whatever/index", hbs.elasticIndexURL); 61 | } 62 | 63 | @Test(expected = SailConfigException.class) 64 | public void testGetSailFail1() throws Exception { 65 | new HBaseSailFactory().getSail(new SailImplConfig() { 66 | @Override 67 | public String getType() { 68 | return "WrongType"; 69 | } 70 | 71 | @Override 72 | public long getIterationCacheSyncThreshold() { 73 | return 0; 74 | } 75 | 76 | @Override 77 | public void validate() throws SailConfigException { 78 | } 79 | 80 | @Override 81 | public Resource export(Model graph) { 82 | return null; 83 | } 84 | 85 | @Override 86 | public void parse(Model graph, Resource implNode) throws SailConfigException { 87 | } 88 | }); 89 | } 90 | 91 | @Test(expected = SailConfigException.class) 92 | public void testGetSailFail2() throws Exception { 93 | new HBaseSailFactory().getSail(new SailImplConfig() { 94 | @Override 95 | public String getType() { 96 | return HBaseSailFactory.SAIL_TYPE; 97 | } 98 | 99 | @Override 100 | public long getIterationCacheSyncThreshold() { 101 | return 0; 102 | } 103 | 104 | @Override 105 | public void validate() throws SailConfigException { 106 | } 107 | 108 | @Override 109 | public Resource export(Model graph) { 110 | return null; 111 | } 112 | 113 | @Override 114 | public void parse(Model graph, Resource implNode) throws SailConfigException { 115 | } 116 | }); 117 | } 118 | 119 | } 120 | -------------------------------------------------------------------------------- /sail/src/test/java/com/msd/gin/halyard/sail/LiteralSearchStatementScannerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.sail; 18 | 19 | import com.msd.gin.halyard.vocab.HALYARD; 20 | import com.msd.gin.halyard.common.HBaseServerTestInstance; 21 | import com.msd.gin.halyard.common.HalyardTableUtils; 22 | import java.io.BufferedReader; 23 | import java.io.IOException; 24 | import java.io.InputStreamReader; 25 | import java.io.OutputStream; 26 | import java.net.InetAddress; 27 | import java.net.ServerSocket; 28 | import java.net.Socket; 29 | import java.nio.charset.StandardCharsets; 30 | import java.util.logging.Level; 31 | import java.util.logging.Logger; 32 | import org.apache.commons.codec.binary.Hex; 33 | import org.apache.commons.io.IOUtils; 34 | import org.eclipse.rdf4j.model.IRI; 35 | import org.eclipse.rdf4j.model.Literal; 36 | import org.eclipse.rdf4j.model.impl.SimpleValueFactory; 37 | import org.junit.AfterClass; 38 | import static org.junit.Assert.assertTrue; 39 | import org.junit.BeforeClass; 40 | import org.junit.Test; 41 | 42 | /** 43 | * 44 | * @author Adam Sotona (MSD) 45 | */ 46 | public class LiteralSearchStatementScannerTest implements Runnable { 47 | 48 | static HBaseSail hbaseSail; 49 | static ServerSocket server; 50 | static String response; 51 | 52 | 53 | @BeforeClass 54 | public static void setup() throws Exception { 55 | server = new ServerSocket(0, 50, InetAddress.getLoopbackAddress()); 56 | hbaseSail = new HBaseSail(HBaseServerTestInstance.getInstanceConfig(), "testLiteralSearch", true, 0, true, 0, "http://" + InetAddress.getLoopbackAddress().getHostAddress() + ":" + server.getLocalPort(), null); 57 | hbaseSail.initialize(); 58 | } 59 | 60 | @AfterClass 61 | public static void teardown() throws Exception { 62 | hbaseSail.shutDown(); 63 | server.close(); 64 | } 65 | 66 | @Test 67 | public void statementLiteralSearchTest() throws Exception { 68 | Literal val = SimpleValueFactory.getInstance().createLiteral("Whatever Text"); 69 | response = "HTTP/1.1 200 OK\ncontent-type: application/json; charset=UTF-8\ncontent-length: 30\n\r\n{\"hits\":{\"hits\":[{\"_id\":\"" + Hex.encodeHexString(HalyardTableUtils.hashKey(val)) + "\"}]}}"; 70 | Thread t = new Thread(this); 71 | t.setDaemon(true); 72 | t.start(); 73 | IRI whatever = SimpleValueFactory.getInstance().createIRI("http://whatever"); 74 | hbaseSail.addStatement(whatever, whatever, val); 75 | hbaseSail.commit(); 76 | assertTrue(hbaseSail.getStatements(null, null, SimpleValueFactory.getInstance().createLiteral("what", HALYARD.SEARCH_TYPE), true).hasNext()); 77 | } 78 | 79 | @Override 80 | public void run() { 81 | try (Socket s = server.accept()) { 82 | try (BufferedReader in = new BufferedReader(new InputStreamReader(s.getInputStream()))) { 83 | String line; 84 | int length = 0; 85 | while ((line = in.readLine()) != null && line.length() > 0) { 86 | if (line.startsWith("Content-Length:")) { 87 | length = Integer.parseInt(line.substring(15).trim()); 88 | } 89 | System.out.println(line); 90 | } 91 | char b[] = new char[length]; 92 | in.read(b); 93 | System.out.println(b); 94 | try (OutputStream out = s.getOutputStream()) { 95 | IOUtils.write(response, out, StandardCharsets.UTF_8); 96 | } 97 | } 98 | } catch (IOException ex) { 99 | Logger.getLogger(LiteralSearchStatementScannerTest.class.getName()).log(Level.SEVERE, null, ex); 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /sail/src/test/java/com/msd/gin/halyard/sail/SES2154SubselectOptionalTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.sail; 18 | 19 | import com.msd.gin.halyard.common.HBaseServerTestInstance; 20 | import org.eclipse.rdf4j.model.IRI; 21 | import org.eclipse.rdf4j.model.Literal; 22 | import org.eclipse.rdf4j.model.impl.SimpleValueFactory; 23 | import org.eclipse.rdf4j.model.vocabulary.RDF; 24 | import org.eclipse.rdf4j.query.QueryLanguage; 25 | import org.eclipse.rdf4j.query.TupleQueryResult; 26 | import org.eclipse.rdf4j.repository.sail.SailRepository; 27 | import static org.junit.Assert.assertEquals; 28 | import org.junit.Test; 29 | 30 | /** 31 | * 32 | * @author Adam Sotona (MSD) 33 | */ 34 | public class SES2154SubselectOptionalTest { 35 | 36 | @Test 37 | public void testSES2154SubselectOptional() throws Exception { 38 | HBaseSail sail = new HBaseSail(HBaseServerTestInstance.getInstanceConfig(), "SES2154SubselectOptionaltable", true, 0, true, 0, null, null); 39 | SailRepository rep = new SailRepository(sail); 40 | rep.initialize(); 41 | SimpleValueFactory vf = SimpleValueFactory.getInstance(); 42 | IRI person = vf.createIRI("http://schema.org/Person"); 43 | for (char c = 'a'; c < 'k'; c++) { 44 | sail.addStatement(vf.createIRI("http://example.com/" + c), RDF.TYPE, person); 45 | } 46 | sail.commit(); 47 | TupleQueryResult res = rep.getConnection().prepareTupleQuery(QueryLanguage.SPARQL, "PREFIX : \n" + "PREFIX schema: \n" + "\n" + "SELECT (COUNT(*) AS ?count)\n" + "WHERE {\n" + " {\n" + " SELECT ?person\n" + " WHERE {\n" + " ?person a schema:Person .\n" + " }\n" + " LIMIT 5\n" + " }\n" + " OPTIONAL {\n" + " [] :nonexistent [] .\n" + " }\n" + "}").evaluate(); 48 | assertEquals(5, ((Literal) res.next().getBinding("count").getValue()).intValue()); 49 | rep.shutDown(); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /sdk/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | halyard-sdk 5 | pom 6 | 7 | com.msd.gin.halyard 8 | halyard 9 | 2.5 10 | 11 | 12 | 13 | ${project.groupId} 14 | halyard-tools 15 | ${project.version} 16 | 17 | 18 | org.eclipse.rdf4j 19 | rdf4j-console 20 | ${rdf4j.version} 21 | 22 | 23 | 24 | 25 | 26 | org.apache.maven.plugins 27 | maven-dependency-plugin 28 | 2.10 29 | 30 | 31 | copy-depenndencies-for-patching 32 | process-classes 33 | 34 | copy 35 | 36 | 37 | 38 | 39 | org.eclipse.rdf4j 40 | rdf4j-repository-api 41 | ${rdf4j.version} 42 | jar 43 | 44 | 45 | ${project.build.directory} 46 | true 47 | 48 | 49 | 50 | 51 | 52 | maven-antrun-plugin 53 | 1.8 54 | 55 | 56 | patch-rdf4j-client 57 | process-classes 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | run 67 | 68 | 69 | 70 | 71 | 72 | org.apache.maven.plugins 73 | maven-assembly-plugin 74 | 2.5.3 75 | 76 | 77 | standalone-assembly 78 | package 79 | 80 | single 81 | 82 | 83 | src/main/assembly/sdk-assembly.xml 84 | false 85 | 86 | 87 | 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /sdk/src/main/assembly/sdk-assembly.xml: -------------------------------------------------------------------------------- 1 | 4 | bin 5 | 6 | zip 7 | 8 | false 9 | 10 | 11 | ${project.basedir}/.. 12 | 13 | readme.md 14 | index.html 15 | LICENSE* 16 | img/* 17 | 18 | 19 | 20 | ${project.basedir}/src/main/scripts 21 | 22 | 23 | * 24 | 25 | 0755 26 | 27 | 28 | ${project.basedir}/target 29 | lib 30 | 31 | rdf4j-repository-api*.jar 32 | 33 | 34 | 35 | 36 | 37 | lib 38 | 39 | *:jar:* 40 | 41 | 42 | *:sources 43 | *:javadoc 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /sdk/src/main/patches/rdf4j-client/org/eclipse/rdf4j/repository/config/hbase.ttl: -------------------------------------------------------------------------------- 1 | # 2 | # RDF4J configuration template for a Halyard HBase repository 3 | # 4 | @prefix rdfs: . 5 | @prefix rep: . 6 | @prefix sr: . 7 | @prefix sail: . 8 | @prefix halyard: . 9 | 10 | [] a rep:Repository ; 11 | rep:repositoryID "{%Repository ID|%}" ; 12 | rdfs:label "{%Repository title|%}" ; 13 | rep:repositoryImpl [ 14 | rep:repositoryType "openrdf:SailRepository" ; 15 | sr:sailImpl [ 16 | sail:sailType "openrdf:HBaseStore" ; 17 | halyard:tableName "{%HBase Table Name|%}" ; 18 | halyard:createTable "{%Create HBase Table if missing|true|false%}" ; 19 | halyard:splitBits "{%HBase Table presplit bits|0%}" ; 20 | halyard:pushStrategy "{%Use Halyard Push Evaluation Strategy|true|false%}" ; 21 | halyard:evaluationTimeout "{%Query Evaluation Timeout|180%}" ; 22 | halyard:elasticIndexURL "{%Optional ElasticSearch Index URL|%}" ; 23 | ] 24 | ]. 25 | -------------------------------------------------------------------------------- /sdk/src/main/scripts/console: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | lib="$(dirname "${0}")/lib" 3 | if [ -n "$JAVA_HOME" ]; then JAVA=$JAVA_HOME/bin/java; else JAVA=java; fi 4 | $JAVA -Xmx2G -cp "$lib/*:`hadoop classpath`:`hbase classpath`" org.eclipse.rdf4j.console.Console "$@" 5 | -------------------------------------------------------------------------------- /sdk/src/main/scripts/halyard: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | halyard_home="$(dirname "${0}")" 3 | libjars="$(echo "$halyard_home"/lib/*.jar | tr ' ' ',')" 4 | export HADOOP_CLASSPATH="$halyard_home/lib/*:`hadoop classpath`:`hbase classpath`" 5 | hadoop com.msd.gin.halyard.tools.HalyardMain "$1" -conf /etc/hbase/conf/hbase-site.xml -conf $halyard_home/halyard-defaults.xml -libjars $libjars "${@:2}" 6 | -------------------------------------------------------------------------------- /sdk/src/main/scripts/halyard-defaults.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | mapreduce.map.output.compress.codec 22 | org.apache.hadoop.io.compress.GzipCodec 23 | 24 | 25 | mapreduce.map.output.compress 26 | true 27 | 28 | 29 | mapreduce.job.reduce.slowstart.completedmaps 30 | 1.0 31 | 32 | 33 | mapreduce.task.timeout 34 | 3600000 35 | 36 | 37 | mapreduce.task.io.sort.factor 38 | 100 39 | 40 | 41 | mapreduce.task.io.sort.mb 42 | 1000 43 | 44 | 45 | mapreduce.input.fileinputformat.split.maxsize 46 | 200000000 47 | 48 | 49 | hbase.mapreduce.bulkload.max.hfiles.perRegion.perFamily 50 | 2048 51 | 52 | 53 | mapreduce.map.speculative 54 | false 55 | 56 | 57 | mapreduce.reduce.speculative 58 | false 59 | 60 | 61 | mapreduce.reduce.shuffle.parallelcopies 62 | 4 63 | 64 | 65 | mapreduce.reduce.shuffle.memory.limit.percent 66 | 0.2 67 | 68 | 69 | mapreduce.reduce.shuffle.input.buffer.percent 70 | 0.20 71 | 72 | 73 | mapreduce.map.memory.mb 74 | 2856 75 | 76 | 77 | mapreduce.map.java.opts 78 | -Xmx2342m 79 | 80 | 81 | mapreduce.reduce.memory.mb 82 | 5712 83 | 84 | 85 | mapreduce.reduce.java.opts 86 | -Xmx4684m 87 | 88 | 89 | fs.permissions.umask-mode 90 | 000 91 | 92 | 93 | -------------------------------------------------------------------------------- /src/main/license/LICENSES_THIRD_PARTY.properties: -------------------------------------------------------------------------------- 1 | # LICENSES_THIRD_PARTY.properties 2 | # 3 | # This file is used by the 'license-maven-plugin' to generate the 4 | # LICENSES_THIRD_PARTY file in the root source directory. 5 | # See both the "third-party-licenses" profile in the PARENT POM and 6 | # http://mojo.codehaus.org/license-maven-plugin/SNAPSHOT/examples/example-thirdparty.html 7 | # 8 | # ----------------------------------------------------------------------------- 9 | # The following Maven dependencies do NOT report their source code license 10 | # in their Maven POM. 11 | # 12 | # NOTES TO MAINTAINERS: 13 | # 1) PLEASE CHECK THE "[src]/LICENSES_THIRD_PARTY" FILE FOR SPECIFIC LICENSE NAMES! 14 | # 2) Also please add a link/URL to where you looked up the license 15 | 16 | # http://asm.ow2.org/ 17 | asm--asm--3.1=BSD License 18 | asm--asm-commons--3.1=BSD License 19 | asm--asm-tree--3.1=BSD License 20 | 21 | # http://zookeeper.apache.org/ 22 | org.apache.zookeeper--zookeeper--3.4.6=Apache Software License, Version 2.0 23 | 24 | # https://commons.apache.org/proper/commons-collections/ 25 | commons-collections--commons-collections--3.1=Apache Software License, Version 2.0 26 | 27 | # http://commons.apache.org/proper/commons-beanutils/ 28 | commons-beanutils--commons-beanutils--1.7.0=Apache Software License, Version 2.0 29 | 30 | # Apache Jakarta ORO: http://svn.apache.org/repos/asf/jakarta/oro/trunk/LICENSE 31 | oro--oro--2.0.8=Apache Software License, Version 2.0 32 | 33 | # http://jettison.codehaus.org/ 34 | org.codehaus.jettison--jettison--1.1=Apache Software License, Version 2.0 35 | 36 | # https://java.net/projects/servlet-spec/ 37 | javax.servlet.jsp--jsp-api--2.1=Common Development and Distribution License (CDDL) 38 | javax.servlet--servlet-api--2.5=Common Development and Distribution License (CDDL) 39 | 40 | # http://download.oracle.com/otndocs/jcp/jta-1.1-classes-oth-JSpec/ 41 | javax.transaction--jta--1.1=Sun Binary Code License Agreement 42 | 43 | #http://www.antlr3.org/license.html 44 | org.antlr--antlr-runtime--3.4=The BSD License 45 | -------------------------------------------------------------------------------- /strategy/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | halyard-strategy 5 | jar 6 | 7 | com.msd.gin.halyard 8 | halyard 9 | 2.5 10 | 11 | 12 | 13 | org.eclipse.rdf4j 14 | rdf4j-console 15 | ${rdf4j.version} 16 | 17 | 18 | ch.qos.logback 19 | logback-classic 20 | 21 | 22 | 23 | 24 | org.eclipse.rdf4j 25 | rdf4j-sparql-testsuite 26 | ${rdf4j.version} 27 | test 28 | 29 | 30 | -------------------------------------------------------------------------------- /strategy/src/main/java/com/msd/gin/halyard/optimizers/HalyardFilterOptimizer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.optimizers; 18 | 19 | import org.eclipse.rdf4j.query.BindingSet; 20 | import org.eclipse.rdf4j.query.Dataset; 21 | import org.eclipse.rdf4j.query.algebra.Filter; 22 | import org.eclipse.rdf4j.query.algebra.TupleExpr; 23 | import org.eclipse.rdf4j.query.algebra.evaluation.impl.FilterOptimizer; 24 | 25 | /** 26 | * 27 | * @author Adam Sotona (MSD) 28 | */ 29 | public final class HalyardFilterOptimizer extends FilterOptimizer { 30 | @Override 31 | public void optimize(TupleExpr tupleExpr, Dataset dataset, BindingSet bindings) { 32 | tupleExpr.visit(new FilterOptimizer.FilterFinder(tupleExpr) { 33 | @Override 34 | public void meet(Filter filter) { 35 | super.meet(filter); 36 | filter.visit(new FilterOptimizer.FilterRelocator(filter) { 37 | { 38 | filterVars.retainAll(filter.getArg().getBindingNames()); 39 | } 40 | }); 41 | } 42 | }); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /strategy/src/main/java/com/msd/gin/halyard/optimizers/HalyardQueryJoinOptimizer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.optimizers; 18 | 19 | import com.msd.gin.halyard.vocab.HALYARD; 20 | import java.util.HashSet; 21 | import java.util.List; 22 | import java.util.Map; 23 | import java.util.Set; 24 | import org.eclipse.rdf4j.query.BindingSet; 25 | import org.eclipse.rdf4j.query.Dataset; 26 | import org.eclipse.rdf4j.query.IncompatibleOperationException; 27 | import org.eclipse.rdf4j.query.algebra.Extension; 28 | import org.eclipse.rdf4j.query.algebra.FunctionCall; 29 | import org.eclipse.rdf4j.query.algebra.Join; 30 | import org.eclipse.rdf4j.query.algebra.TupleExpr; 31 | import org.eclipse.rdf4j.query.algebra.ValueExpr; 32 | import org.eclipse.rdf4j.query.algebra.Var; 33 | import org.eclipse.rdf4j.query.algebra.evaluation.impl.QueryJoinOptimizer; 34 | import org.eclipse.rdf4j.query.algebra.helpers.AbstractQueryModelVisitor; 35 | 36 | /** 37 | * 38 | * @author Adam Sotona (MSD) 39 | */ 40 | public final class HalyardQueryJoinOptimizer extends QueryJoinOptimizer { 41 | 42 | public HalyardQueryJoinOptimizer(HalyardEvaluationStatistics statistics) { 43 | super(statistics); 44 | } 45 | 46 | @Override 47 | public void optimize(TupleExpr tupleExpr, Dataset dataset, BindingSet bindings) { 48 | final Set parallelSplitBounds = new HashSet<>(); 49 | tupleExpr.visit(new AbstractQueryModelVisitor() { 50 | @Override 51 | public void meet(FunctionCall node) throws IncompatibleOperationException { 52 | if (HALYARD.PARALLEL_SPLIT_FUNCTION.toString().equals(node.getURI())) { 53 | for (ValueExpr arg : node.getArgs()) { 54 | if (arg instanceof Var) parallelSplitBounds.add(((Var)arg).getName()); 55 | } 56 | } 57 | super.meet(node); 58 | } 59 | }); 60 | tupleExpr.visit(new QueryJoinOptimizer.JoinVisitor() { 61 | private Set priorityBounds = new HashSet<>(); 62 | 63 | @Override 64 | protected double getTupleExprCardinality(TupleExpr tupleExpr, Map cardinalityMap, Map> varsMap, Map varFreqMap, Set boundVars) { 65 | //treat all bindings mentioned in HALYARD.PARALLEL_SPLIT_FUNCTION as bound to preffer them in optimization 66 | boundVars.addAll(parallelSplitBounds); 67 | boundVars.addAll(priorityBounds); 68 | ((HalyardEvaluationStatistics) statistics).updateCardinalityMap(tupleExpr, boundVars, parallelSplitBounds, cardinalityMap); 69 | return super.getTupleExprCardinality(tupleExpr, cardinalityMap, varsMap, varFreqMap, boundVars); //To change body of generated methods, choose Tools | Templates. 70 | } 71 | 72 | @Override 73 | public void meet(Join node) { 74 | Set origPriorityBounds = priorityBounds; 75 | try { 76 | priorityBounds = new HashSet<>(priorityBounds); 77 | super.meet(node); 78 | } finally { 79 | priorityBounds = origPriorityBounds; 80 | } 81 | } 82 | 83 | @Override 84 | protected List getExtensions(List expressions) { 85 | List exts = super.getExtensions(expressions); 86 | for (Extension ext : exts) { 87 | priorityBounds.addAll(ext.getBindingNames()); 88 | } 89 | return exts; 90 | } 91 | 92 | @Override 93 | protected List getSubSelects(List expressions) { 94 | List subs = super.getSubSelects(expressions); 95 | for (TupleExpr sub : subs) { 96 | priorityBounds.addAll(sub.getBindingNames()); 97 | } 98 | return subs; 99 | } 100 | 101 | }); 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /strategy/src/main/java/com/msd/gin/halyard/strategy/HalyardQueryOptimizerPipeline.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.strategy; 18 | 19 | import com.msd.gin.halyard.optimizers.HalyardEvaluationStatistics; 20 | import com.msd.gin.halyard.optimizers.HalyardFilterOptimizer; 21 | import com.msd.gin.halyard.optimizers.HalyardQueryJoinOptimizer; 22 | import java.util.Arrays; 23 | import org.eclipse.rdf4j.query.algebra.evaluation.EvaluationStrategy; 24 | import org.eclipse.rdf4j.query.algebra.evaluation.QueryOptimizer; 25 | import org.eclipse.rdf4j.query.algebra.evaluation.QueryOptimizerPipeline; 26 | import org.eclipse.rdf4j.query.algebra.evaluation.TripleSource; 27 | import org.eclipse.rdf4j.query.algebra.evaluation.impl.BindingAssigner; 28 | import org.eclipse.rdf4j.query.algebra.evaluation.impl.CompareOptimizer; 29 | import org.eclipse.rdf4j.query.algebra.evaluation.impl.ConjunctiveConstraintSplitter; 30 | import org.eclipse.rdf4j.query.algebra.evaluation.impl.ConstantOptimizer; 31 | import org.eclipse.rdf4j.query.algebra.evaluation.impl.DisjunctiveConstraintOptimizer; 32 | import org.eclipse.rdf4j.query.algebra.evaluation.impl.IterativeEvaluationOptimizer; 33 | import org.eclipse.rdf4j.query.algebra.evaluation.impl.OrderLimitOptimizer; 34 | import org.eclipse.rdf4j.query.algebra.evaluation.impl.QueryModelNormalizer; 35 | import org.eclipse.rdf4j.query.algebra.evaluation.impl.SameTermFilterOptimizer; 36 | 37 | /** 38 | * 39 | * @author Adam Sotona 40 | */ 41 | public class HalyardQueryOptimizerPipeline implements QueryOptimizerPipeline { 42 | 43 | private final HalyardEvaluationStatistics statistics; 44 | private final TripleSource tripleSource; 45 | private final EvaluationStrategy strategy; 46 | 47 | public HalyardQueryOptimizerPipeline(EvaluationStrategy strategy, TripleSource tripleSource, HalyardEvaluationStatistics statistics) { 48 | this.strategy = strategy; 49 | this.tripleSource = tripleSource; 50 | this.statistics = statistics; 51 | } 52 | 53 | @Override 54 | public Iterable getOptimizers() { 55 | return Arrays.asList( 56 | new BindingAssigner(), 57 | new ConstantOptimizer(strategy), 58 | new CompareOptimizer(), 59 | new ConjunctiveConstraintSplitter(), 60 | new DisjunctiveConstraintOptimizer(), 61 | new SameTermFilterOptimizer(), 62 | new QueryModelNormalizer(), 63 | new HalyardQueryJoinOptimizer(statistics), 64 | new IterativeEvaluationOptimizer(), 65 | new HalyardFilterOptimizer(), 66 | new OrderLimitOptimizer()); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /strategy/src/main/java/com/msd/gin/halyard/strategy/collections/BigHashSet.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.strategy.collections; 18 | 19 | import java.io.Closeable; 20 | import java.io.IOException; 21 | import java.io.Serializable; 22 | import java.util.Iterator; 23 | import java.util.Set; 24 | import org.mapdb.DB; 25 | import org.mapdb.DBMaker; 26 | 27 | /** 28 | * TODO 29 | * This is a MapDB implementation, however a hash set backed by HDFS is expected here 30 | * @author Adam Sotona (MSD) 31 | * @param Serializable element type 32 | */ 33 | public class BigHashSet implements Iterable, Closeable { 34 | 35 | private static final String SET_NAME = "temp"; 36 | 37 | private final DB db = DBMaker.newTempFileDB().deleteFilesAfterClose().closeOnJvmShutdown().transactionDisable().make(); 38 | private final Set set = db.getHashSet(SET_NAME); 39 | 40 | /** 41 | * Adds element to the BigHashSet 42 | * @param e Serializable element 43 | * @return boolean if the element has been already present 44 | * @throws IOException throws IOException in case of problem with underlying storage 45 | */ 46 | public boolean add(E e) throws IOException { 47 | try { 48 | return set.add(e); 49 | } catch (IllegalAccessError err) { 50 | throw new IOException(err); 51 | } 52 | } 53 | 54 | @Override 55 | public Iterator iterator() { 56 | return set.iterator(); 57 | } 58 | 59 | /** 60 | * Checks for element presence in the BigHashSet 61 | * @param e Serializable element 62 | * @return boolean if the element has been present 63 | * @throws IOException throws IOException in case of problem with underlying storage 64 | */ 65 | public boolean contains(E e) throws IOException { 66 | try { 67 | return set.contains(e); 68 | } catch (IllegalAccessError err) { 69 | throw new IOException(err); 70 | } 71 | } 72 | 73 | @Override 74 | public void close() { 75 | try { 76 | db.close(); 77 | } catch (IllegalAccessError ignore) { 78 | //silent close 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /strategy/src/main/java/com/msd/gin/halyard/strategy/collections/Sorter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.strategy.collections; 18 | 19 | import java.io.Closeable; 20 | import java.io.IOException; 21 | import java.io.Serializable; 22 | import java.util.Iterator; 23 | import java.util.Map; 24 | import java.util.NavigableMap; 25 | import org.mapdb.DB; 26 | import org.mapdb.DBMaker; 27 | 28 | /** 29 | * Sorter does not preserve unique instances. All equal instances are mapped to a single instance. 30 | * TODO 31 | * This is a MapDB implementation, however a merge-sort backed by HDFS is expected here. 32 | * @author Adam Sotona (MSD) 33 | * @param Comparable and Serializable element type 34 | */ 35 | public class Sorter & Serializable> implements Iterable>, Closeable { 36 | 37 | private static final String MAP_NAME = "temp"; 38 | 39 | private final DB db; 40 | private final NavigableMap map; 41 | private final long limit; 42 | private final boolean distinct; 43 | private long size = 0; 44 | 45 | /** 46 | * Constructs Sorter with optional limit and optional distinct filtering 47 | * @param limit long limit, where Long.MAXLONG means no limit 48 | * @param distinct optional boolean switch to do not preserve multiple equal elements 49 | */ 50 | public Sorter(long limit, boolean distinct) { 51 | this.db = DBMaker.newTempFileDB().deleteFilesAfterClose().closeOnJvmShutdown().transactionDisable().make(); 52 | this.map = db.createTreeMap(MAP_NAME).make(); 53 | this.limit = limit; 54 | this.distinct = distinct; 55 | } 56 | 57 | /** 58 | * Adds new element to the sorter 59 | * @param e element to be added to Sorter 60 | * @throws IOException throws IOException in case of problem with underlying storage 61 | */ 62 | public void add(E e) throws IOException { 63 | if (size < limit || e.compareTo(map.lastKey()) < 0) try { 64 | Long c = map.get(e); 65 | if (c == null) { 66 | map.put(e, 1l); 67 | size ++; 68 | } else if (!distinct) { 69 | map.put(e, c + 1l); 70 | size ++; 71 | } 72 | while (size > limit) { 73 | // Discard key(s) that are currently sorted last 74 | Map.Entry last = map.lastEntry(); 75 | if (last.getValue() > size - limit) { 76 | map.put(last.getKey(), last.getValue() + limit - size); 77 | size = limit; 78 | } else { 79 | map.remove(last.getKey()); 80 | size -= last.getValue(); 81 | } 82 | } 83 | } catch (IllegalAccessError err) { 84 | throw new IOException(err); 85 | } 86 | } 87 | 88 | @Override 89 | public Iterator> iterator() { 90 | return map.entrySet().iterator(); 91 | } 92 | 93 | @Override 94 | public void close() { 95 | try { 96 | db.close(); 97 | } catch (IllegalAccessError|IllegalStateException ignore) { 98 | //silent close 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /strategy/src/main/java/com/msd/gin/halyard/strategy/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Classes that define the way Halyard evaluates a query and pushes query subtasks. 3 | */ 4 | package com.msd.gin.halyard.strategy; -------------------------------------------------------------------------------- /strategy/src/main/java/com/msd/gin/halyard/vocab/HALYARD.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Merck Sharp & Dohme Corp., a subsidiary of Merck & Co., Inc. 3 | * All rights reserved. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.vocab; 18 | 19 | import org.eclipse.rdf4j.model.IRI; 20 | import org.eclipse.rdf4j.model.impl.SimpleValueFactory; 21 | 22 | /** 23 | * IRI constants used by Halyard. 24 | * @author Adam Sotona (MSD) 25 | */ 26 | public final class HALYARD { 27 | 28 | HALYARD() {} 29 | 30 | private static final SimpleValueFactory SVF = SimpleValueFactory.getInstance(); 31 | 32 | public static final String PREFIX = "halyard"; 33 | 34 | public static final String NAMESPACE = "http://merck.github.io/Halyard/ns#"; 35 | 36 | public static final IRI STATS_ROOT_NODE = SVF.createIRI(NAMESPACE, "statsRoot"); 37 | 38 | public static final IRI STATS_GRAPH_CONTEXT = SVF.createIRI(NAMESPACE, "statsContext"); 39 | 40 | public static final IRI SYSTEM_GRAPH_CONTEXT = SVF.createIRI(NAMESPACE, "system"); 41 | 42 | public static final IRI NAMESPACE_PREFIX_PROPERTY = HALYARD.SVF.createIRI(NAMESPACE, "namespacePrefix"); 43 | 44 | public final static IRI TABLE_NAME_PROPERTY = SVF.createIRI(NAMESPACE, "tableName"); 45 | 46 | public final static IRI SPLITBITS_PROPERTY = SVF.createIRI(NAMESPACE, "splitBits"); 47 | 48 | public final static IRI CREATE_TABLE_PROPERTY = SVF.createIRI(NAMESPACE, "createTable"); 49 | 50 | public final static IRI PUSH_STRATEGY_PROPERTY = SVF.createIRI(NAMESPACE, "pushStrategy"); 51 | 52 | public final static IRI EVALUATION_TIMEOUT_PROPERTY = SVF.createIRI(NAMESPACE, "evaluationTimeout"); 53 | 54 | public final static IRI ELASTIC_INDEX_URL_PROPERTY = SVF.createIRI(NAMESPACE, "elasticIndexURL"); 55 | 56 | public final static IRI SEARCH_TYPE = SVF.createIRI(NAMESPACE, "search"); 57 | 58 | public final static IRI PARALLEL_SPLIT_FUNCTION = SVF.createIRI(NAMESPACE, "forkAndFilterBy"); 59 | 60 | } 61 | -------------------------------------------------------------------------------- /strategy/src/main/java/com/msd/gin/halyard/vocab/VOID_EXT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2014 Merck Sharp & Dohme Corp., a subsidiary of Merck & Co., Inc. 3 | * All rights reserved. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.vocab; 18 | 19 | import org.eclipse.rdf4j.model.IRI; 20 | import org.eclipse.rdf4j.model.impl.SimpleValueFactory; 21 | 22 | /** 23 | * Prefix, namespace and IRIs for the parts of the VIOD ontology used in Halyard 24 | * @author Adam Sotona (MSD) 25 | */ 26 | public final class VOID_EXT { 27 | 28 | VOID_EXT(){} 29 | 30 | public static final String PREFIX = "void-ext"; 31 | 32 | public static final String NAMESPACE = "http://ldf.fi/void-ext#"; 33 | 34 | private static final SimpleValueFactory SVF = SimpleValueFactory.getInstance(); 35 | 36 | public static final IRI DISTINCT_IRI_REFERENCE_SUBJECTS = SVF.createIRI(NAMESPACE, "distinctIRIReferenceSubjects"); 37 | 38 | public static final IRI DISTINCT_IRI_REFERENCE_OBJECTS = SVF.createIRI(NAMESPACE, "distinctIRIReferenceObjects"); 39 | 40 | public static final IRI DISTINCT_BLANK_NODE_OBJECTS = SVF.createIRI(NAMESPACE, "distinctBlankNodeObjects"); 41 | 42 | public static final IRI DISTINCT_BLANK_NODE_SUBJECTS = SVF.createIRI(NAMESPACE, "distinctBlankNodeSubjects"); 43 | 44 | public static final IRI DISTINCT_LITERALS = SVF.createIRI(NAMESPACE, "distinctLiterals"); 45 | 46 | public static final IRI SUBJECT = SVF.createIRI(NAMESPACE, "subject"); 47 | 48 | public static final IRI SUBJECT_PARTITION = SVF.createIRI(NAMESPACE, "subjectPartition"); 49 | 50 | public static final IRI OBJECT = SVF.createIRI(NAMESPACE, "object"); 51 | 52 | public static final IRI OBJECT_PARTITION = SVF.createIRI(NAMESPACE, "objectPartition"); 53 | 54 | } 55 | -------------------------------------------------------------------------------- /strategy/src/test/java/com/msd/gin/halyard/optimizers/HalyardEvaluationStatisticsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.optimizers; 18 | 19 | import com.msd.gin.halyard.vocab.HALYARD; 20 | import java.util.Arrays; 21 | import java.util.Collection; 22 | import java.util.HashSet; 23 | import java.util.Set; 24 | import org.eclipse.rdf4j.query.parser.sparql.SPARQLParser; 25 | import org.junit.Assert; 26 | import org.junit.Test; 27 | import org.junit.runner.RunWith; 28 | import org.junit.runners.Parameterized; 29 | 30 | /** 31 | * 32 | * @author Adam Sotona (MSD) 33 | */ 34 | @RunWith(Parameterized.class) 35 | public class HalyardEvaluationStatisticsTest { 36 | 37 | @Parameterized.Parameters 38 | public static Collection data() { 39 | return Arrays.asList(new Object[][] { 40 | {"select * where {?s a ?o}", 1.0E2, null, null}, 41 | {"select * where {?s ?p ?o}", 1.0E3, null, null}, 42 | {"select * where {?s a \"1\"}", 1.0E1, null, null}, 43 | {"select * where {?a ?b ?c; ?d ?e}", 1.0E8, null, null}, 44 | {"select * where {?a a ?c; ?d ?e}", 1.0E6, null, null}, 45 | {"select * where {?a ?b ?c; a ?e}", 1.0E7, null, null}, 46 | {"select * where {?s a ?o}", 1.0E1, new String[]{"o"}, null}, 47 | {"select * where {?s ?p ?o}", 1.0E1, new String[]{"s", "o"}, null}, 48 | {"select * where {?s a \"1\"}", 1.0, new String[]{"s"}, null}, 49 | {"select * where {?a ?b ?c; ?d ?e}", 1.0E4, new String[]{"b", "c"}, null}, 50 | {"select * where {?a a ?c; ?d ?e}", 1.0E3, new String[]{"d", "c"}, null}, 51 | {"select * where {?a ?b ?c; a ?e}", 1.0E2, new String[]{"b", "e", "c"}, null}, 52 | {"select * where {{?a a \"1\". optional {?a a ?b}} union {?a a \"2\"}}", 1010.0, null, null}, 53 | {"select * where {?s a \"1\"^^<" + HALYARD.SEARCH_TYPE + ">}", 1.0E-4, new String[]{"s"}, null}, 54 | {"select * where {?a ?b ?c}", 1.0E-4, new String[]{"a"}, new String[]{"a"}}, 55 | {"select * where {?a ?b ?c}", 1.0E-11, new String[]{"a", "b"}, new String[]{"a", "b"}}, 56 | {"select * where {?a ?b ?c}", 1.0E-18, new String[]{"a", "b" , "c"}, new String[]{"a", "b", "c"}}, 57 | {"select * where {?a ?b ?c}", 1.0E-3, null, new String[]{"a"}}, 58 | {"select * where {?a ?b ?c}", 1.0E-9, null, new String[]{"a", "b"}}, 59 | }); 60 | } 61 | 62 | private final String query; 63 | private final double cardinality; 64 | private final Set boundVars = new HashSet<>(); 65 | private final Set priorityVars = new HashSet<>(); 66 | 67 | public HalyardEvaluationStatisticsTest(String query, double cardinality, String[] boundVars, String[] priorityVars) { 68 | this.query = query; 69 | this.cardinality = cardinality; 70 | if (boundVars != null) { 71 | this.boundVars.addAll(Arrays.asList(boundVars)); 72 | } 73 | if (priorityVars != null) { 74 | this.priorityVars.addAll(Arrays.asList(priorityVars)); 75 | } 76 | } 77 | 78 | @Test 79 | public void testCardinality() { 80 | Assert.assertEquals(query, cardinality, new HalyardEvaluationStatistics(null, null).getCardinality(new SPARQLParser().parseQuery(query, "http://baseuri/").getTupleExpr(), boundVars, priorityVars), cardinality/1000000.0); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /strategy/src/test/java/com/msd/gin/halyard/optimizers/HalyardFilterOptimizerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.optimizers; 18 | 19 | import org.eclipse.rdf4j.query.algebra.Filter; 20 | import org.eclipse.rdf4j.query.algebra.Join; 21 | import org.eclipse.rdf4j.query.algebra.TupleExpr; 22 | import org.eclipse.rdf4j.query.algebra.evaluation.impl.FilterOptimizer; 23 | import org.eclipse.rdf4j.query.algebra.helpers.AbstractQueryModelVisitor; 24 | import org.eclipse.rdf4j.query.parser.sparql.SPARQLParser; 25 | import static org.junit.Assert.assertEquals; 26 | import org.junit.Test; 27 | 28 | /** 29 | * 30 | * @author Adam Sotona (MSD) 31 | */ 32 | public class HalyardFilterOptimizerTest { 33 | 34 | @Test 35 | public void testPropagateFilterMoreAgresivelly() { 36 | final TupleExpr expr = new SPARQLParser().parseQuery("select * where {?a ?b ?c, ?d. filter (?d = ?nonexistent)}", "http://baseuri/").getTupleExpr(); 37 | new HalyardFilterOptimizer().optimize(expr, null, null); 38 | expr.visit(new AbstractQueryModelVisitor(){ 39 | @Override 40 | public void meet(Join node) throws RuntimeException { 41 | assertEquals(expr.toString(), "Filter", node.getRightArg().getSignature()); 42 | super.meet(node); 43 | } 44 | @Override 45 | public void meet(Filter node) throws RuntimeException { 46 | assertEquals(expr.toString(), "StatementPattern", node.getArg().getSignature()); 47 | } 48 | }); 49 | } 50 | 51 | @Test 52 | public void testKeepBindWithFilter() { 53 | TupleExpr expr = new SPARQLParser().parseQuery("SELECT ?x\nWHERE {BIND (\"x\" AS ?x) FILTER (?x = \"x\")}", "http://baseuri/").getTupleExpr(); 54 | TupleExpr clone = expr.clone(); 55 | new HalyardFilterOptimizer().optimize(clone, null, null); 56 | assertEquals(expr, clone); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /strategy/src/test/java/com/msd/gin/halyard/optimizers/HalyardQueryJoinOptimizerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.optimizers; 18 | 19 | import com.msd.gin.halyard.vocab.HALYARD; 20 | import org.eclipse.rdf4j.query.algebra.Join; 21 | import org.eclipse.rdf4j.query.algebra.StatementPattern; 22 | import org.eclipse.rdf4j.query.algebra.TupleExpr; 23 | import org.eclipse.rdf4j.query.algebra.helpers.AbstractQueryModelVisitor; 24 | import org.eclipse.rdf4j.query.parser.sparql.SPARQLParser; 25 | import org.junit.Test; 26 | import static org.junit.Assert.*; 27 | 28 | /** 29 | * 30 | * @author Adam Sotona (MSD) 31 | */ 32 | public class HalyardQueryJoinOptimizerTest { 33 | 34 | @Test 35 | public void testQueryJoinOptimizer() { 36 | final TupleExpr expr = new SPARQLParser().parseQuery("select * where {?a ?b ?c, \"1\".}", "http://baseuri/").getTupleExpr(); 37 | new HalyardQueryJoinOptimizer(new HalyardEvaluationStatistics(null, null)).optimize(expr, null, null); 38 | expr.visit(new AbstractQueryModelVisitor(){ 39 | @Override 40 | public void meet(Join node) throws RuntimeException { 41 | assertTrue(expr.toString(), ((StatementPattern)node.getLeftArg()).getObjectVar().hasValue()); 42 | assertEquals(expr.toString(), "c", ((StatementPattern)node.getRightArg()).getObjectVar().getName()); 43 | } 44 | }); 45 | } 46 | 47 | @Test 48 | public void testQueryJoinOptimizerWithSplitFunction() { 49 | final TupleExpr expr = new SPARQLParser().parseQuery("select * where {?a a \"1\";?b ?d. filter (<" + HALYARD.PARALLEL_SPLIT_FUNCTION + ">(10, ?d))}", "http://baseuri/").getTupleExpr(); 50 | new HalyardQueryJoinOptimizer(new HalyardEvaluationStatistics(null, null)).optimize(expr, null, null); 51 | expr.visit(new AbstractQueryModelVisitor(){ 52 | @Override 53 | public void meet(Join node) throws RuntimeException { 54 | assertEquals(expr.toString(), "d", ((StatementPattern)node.getLeftArg()).getObjectVar().getName()); 55 | assertTrue(expr.toString(), ((StatementPattern)node.getRightArg()).getObjectVar().hasValue()); 56 | } 57 | }); 58 | } 59 | 60 | @Test 61 | public void testQueryJoinOptimizarWithBind() { 62 | final TupleExpr expr = new SPARQLParser().parseQuery("SELECT * WHERE { BIND ( AS ?b) ?a ?b , \"whatever\".}", "http://baseuri/").getTupleExpr(); 63 | new HalyardQueryJoinOptimizer(new HalyardEvaluationStatistics(null, null)).optimize(expr, null, null); 64 | expr.visit(new AbstractQueryModelVisitor(){ 65 | @Override 66 | public void meet(Join node) throws RuntimeException { 67 | if (node.getLeftArg() instanceof StatementPattern) { 68 | assertEquals(expr.toString(), "b", ((StatementPattern)node.getLeftArg()).getObjectVar().getName()); 69 | } 70 | } 71 | }); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /strategy/src/test/java/com/msd/gin/halyard/strategy/ArbitraryLengthPathTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.strategy; 18 | 19 | import java.util.Arrays; 20 | import java.util.Collection; 21 | import static junit.framework.TestCase.assertTrue; 22 | import org.eclipse.rdf4j.model.ValueFactory; 23 | import org.eclipse.rdf4j.query.QueryLanguage; 24 | import org.eclipse.rdf4j.repository.Repository; 25 | import org.eclipse.rdf4j.repository.RepositoryConnection; 26 | import org.eclipse.rdf4j.repository.sail.SailRepository; 27 | 28 | import org.junit.After; 29 | import org.junit.Before; 30 | import org.junit.Test; 31 | import org.junit.runner.RunWith; 32 | import org.junit.runners.Parameterized; 33 | 34 | /** 35 | * @author Adam Sotona (MSD) 36 | */ 37 | @RunWith(Parameterized.class) 38 | public class ArbitraryLengthPathTest { 39 | @Parameterized.Parameters 40 | public static Collection data() { 41 | return Arrays.asList(10, 100, 1000, 10000, 100000); 42 | } 43 | 44 | private final int n; 45 | private Repository repo; 46 | private RepositoryConnection con; 47 | 48 | public ArbitraryLengthPathTest(int n) { 49 | this.n = n; 50 | } 51 | 52 | @Before 53 | public void setUp() throws Exception { 54 | repo = new SailRepository(new MemoryStoreWithHalyardStrategy()); 55 | repo.initialize(); 56 | con = repo.getConnection(); 57 | } 58 | 59 | @After 60 | public void tearDown() throws Exception { 61 | con.close(); 62 | repo.shutDown(); 63 | } 64 | 65 | @Test 66 | public void testN() throws Exception { 67 | ValueFactory vf = con.getValueFactory(); 68 | for (int i = 0; i < n; i++) { 69 | con.add(vf.createURI("urn:test:root"), vf.createURI("urn:test:hasChild"), vf.createURI("urn:test:node" + i)); 70 | } 71 | con.add(vf.createURI("urn:test:root"), vf.createURI("urn:test:hasChild"), vf.createURI("urn:test:node-end")); 72 | String sparql = "ASK { * }"; 73 | assertTrue(con.prepareBooleanQuery(QueryLanguage.SPARQL, sparql).evaluate()); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /strategy/src/test/java/com/msd/gin/halyard/strategy/HalyardComplexSPARQLQueryTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.strategy; 18 | 19 | import org.eclipse.rdf4j.query.parser.sparql.ComplexSPARQLQueryTest; 20 | import org.eclipse.rdf4j.repository.Repository; 21 | import org.eclipse.rdf4j.repository.sail.SailRepository; 22 | import org.junit.Ignore; 23 | import org.junit.Test; 24 | 25 | /** 26 | * 27 | * @author Adam Sotona (MSD) 28 | */ 29 | public class HalyardComplexSPARQLQueryTest extends ComplexSPARQLQueryTest { 30 | 31 | @Override 32 | protected Repository newRepository() throws Exception { 33 | return new SailRepository(new MemoryStoreWithHalyardStrategy()); 34 | } 35 | 36 | // @Rule 37 | // public SingleTestRule test = new SingleTestRule(); 38 | // 39 | // public class SingleTestRule implements MethodRule { 40 | // 41 | // @Override 42 | // public Statement apply(final Statement statement, final FrameworkMethod method, final Object target) { 43 | // return new Statement() { 44 | // @Override 45 | // public void evaluate() throws Throwable { 46 | // if ("test27NormalizeIRIFunction".equals(method.getName())) { 47 | // statement.evaluate(); 48 | // } 49 | // } 50 | // }; 51 | // } 52 | // } 53 | 54 | @Override 55 | @Test 56 | @Ignore 57 | //skip invalid test 58 | public void testEmptyUnion() throws Exception { 59 | super.testEmptyUnion(); 60 | } 61 | 62 | @Override 63 | @Test 64 | @Ignore 65 | public void testBindScope() throws Exception { 66 | super.testBindScope(); 67 | } 68 | 69 | @Override 70 | @Test 71 | @Ignore 72 | public void testBindScopeUnion() { 73 | super.testBindScopeUnion(); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /strategy/src/test/java/com/msd/gin/halyard/strategy/HalyardSPARQLQueryTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.strategy; 18 | 19 | import junit.framework.Test; 20 | import org.eclipse.rdf4j.query.Dataset; 21 | import org.eclipse.rdf4j.query.parser.sparql.manifest.SPARQL11ManifestTest; 22 | import org.eclipse.rdf4j.query.parser.sparql.manifest.SPARQLQueryTest; 23 | import org.eclipse.rdf4j.repository.Repository; 24 | import org.eclipse.rdf4j.repository.dataset.DatasetRepository; 25 | import org.eclipse.rdf4j.repository.sail.SailRepository; 26 | 27 | /** 28 | * 29 | * @author Adam Sotona (MSD) 30 | */ 31 | public class HalyardSPARQLQueryTest extends SPARQLQueryTest { 32 | 33 | public static Test suite() 34 | throws Exception { 35 | return SPARQL11ManifestTest.suite(new SPARQLQueryTest.Factory() { 36 | 37 | @Override 38 | public HalyardSPARQLQueryTest createSPARQLQueryTest(String testURI, String name, String queryFileURL, String resultFileURL, Dataset dataSet, boolean laxCardinality) { 39 | return createSPARQLQueryTest(testURI, name, queryFileURL, resultFileURL, dataSet, laxCardinality, false); 40 | } 41 | 42 | @Override 43 | public HalyardSPARQLQueryTest createSPARQLQueryTest(String testURI, String name, String queryFileURL, String resultFileURL, Dataset dataSet, boolean laxCardinality, boolean checkOrder) { 44 | String[] ignoredTests = { 45 | // test case incompatible with RDF 1.1 - see 46 | // http://lists.w3.org/Archives/Public/public-sparql-dev/2013AprJun/0006.html 47 | "STRDT TypeErrors", 48 | // test case incompatible with RDF 1.1 - see 49 | // http://lists.w3.org/Archives/Public/public-sparql-dev/2013AprJun/0006.html 50 | "STRLANG TypeErrors", 51 | // known issue: SES-937 52 | "sq03 - Subquery within graph pattern, graph variable is not bound"}; 53 | 54 | return new HalyardSPARQLQueryTest(testURI, name, queryFileURL, resultFileURL, dataSet, laxCardinality, checkOrder, ignoredTests); 55 | } 56 | }, true, false, false, "service"); 57 | } 58 | 59 | protected HalyardSPARQLQueryTest(String testURI, String name, String queryFileURL, String resultFileURL, 60 | Dataset dataSet, boolean laxCardinality, boolean checkOrder, String... ignoredTests) { 61 | super(testURI, name, queryFileURL, resultFileURL, dataSet, laxCardinality, checkOrder, ignoredTests); 62 | } 63 | 64 | @Override 65 | protected Repository newRepository() { 66 | return new DatasetRepository(new SailRepository(new MemoryStoreWithHalyardStrategy())); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /strategy/src/test/java/com/msd/gin/halyard/strategy/HalyardSPARQLUpdateTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.strategy; 18 | 19 | import org.eclipse.rdf4j.query.parser.sparql.SPARQLUpdateTest; 20 | import org.eclipse.rdf4j.repository.Repository; 21 | import org.eclipse.rdf4j.repository.sail.SailRepository; 22 | 23 | /** 24 | * 25 | * @author Adam Sotona (MSD) 26 | */ 27 | public class HalyardSPARQLUpdateTest extends SPARQLUpdateTest { 28 | 29 | @Override 30 | protected Repository newRepository() 31 | throws Exception 32 | { 33 | return new SailRepository(new MemoryStoreWithHalyardStrategy()); 34 | } 35 | 36 | } -------------------------------------------------------------------------------- /strategy/src/test/java/com/msd/gin/halyard/strategy/HalyardStrategyServiceTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.strategy; 18 | 19 | import static junit.framework.TestCase.*; 20 | import org.eclipse.rdf4j.common.iteration.CloseableIteration; 21 | import org.eclipse.rdf4j.model.IRI; 22 | import org.eclipse.rdf4j.model.Resource; 23 | import org.eclipse.rdf4j.model.Statement; 24 | import org.eclipse.rdf4j.model.Value; 25 | import org.eclipse.rdf4j.model.ValueFactory; 26 | import org.eclipse.rdf4j.model.impl.SimpleValueFactory; 27 | import org.eclipse.rdf4j.query.QueryEvaluationException; 28 | import org.eclipse.rdf4j.query.algebra.evaluation.TripleSource; 29 | import org.eclipse.rdf4j.query.algebra.evaluation.federation.FederatedService; 30 | import org.eclipse.rdf4j.query.algebra.evaluation.federation.FederatedServiceResolver; 31 | import org.junit.Test; 32 | 33 | /** 34 | * @author Adam Sotona (MSD) 35 | */ 36 | public class HalyardStrategyServiceTest { 37 | 38 | @Test 39 | public void testGetService() { 40 | assertNull(new HalyardEvaluationStrategy(getTripleSource(), null, new FederatedServiceResolver() { 41 | @Override 42 | public FederatedService getService(String serviceUrl) throws QueryEvaluationException { 43 | return null; 44 | } 45 | }, null, 0).getService(null)); 46 | } 47 | 48 | @Test(expected = UnsupportedOperationException.class) 49 | public void testServiceEvaluateFail() { 50 | new HalyardEvaluationStrategy(getTripleSource(), null, null, null, 0).evaluate(null, null, null); 51 | } 52 | 53 | private TripleSource getTripleSource() { 54 | return new TripleSource() { 55 | @Override 56 | public CloseableIteration getStatements(Resource subj, IRI pred, Value obj, Resource... contexts) throws QueryEvaluationException { 57 | throw new QueryEvaluationException(); 58 | } 59 | 60 | @Override 61 | public ValueFactory getValueFactory() { 62 | return SimpleValueFactory.getInstance(); 63 | } 64 | }; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /strategy/src/test/java/com/msd/gin/halyard/strategy/MemoryStoreWithHalyardStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.strategy; 18 | 19 | import com.msd.gin.halyard.optimizers.HalyardEvaluationStatistics; 20 | import org.eclipse.rdf4j.common.iteration.CloseableIteration; 21 | import org.eclipse.rdf4j.common.iteration.FilterIteration; 22 | import org.eclipse.rdf4j.model.IRI; 23 | import org.eclipse.rdf4j.model.Resource; 24 | import org.eclipse.rdf4j.model.Statement; 25 | import org.eclipse.rdf4j.model.Value; 26 | import org.eclipse.rdf4j.model.ValueFactory; 27 | import org.eclipse.rdf4j.query.Dataset; 28 | import org.eclipse.rdf4j.query.QueryEvaluationException; 29 | import org.eclipse.rdf4j.query.algebra.evaluation.EvaluationStrategy; 30 | import org.eclipse.rdf4j.query.algebra.evaluation.TripleSource; 31 | import org.eclipse.rdf4j.query.algebra.evaluation.impl.EvaluationStatistics; 32 | import org.eclipse.rdf4j.query.algebra.evaluation.impl.StandardQueryOptimizerPipeline; 33 | import org.eclipse.rdf4j.sail.NotifyingSailConnection; 34 | import org.eclipse.rdf4j.sail.SailException; 35 | import org.eclipse.rdf4j.sail.memory.MemoryStore; 36 | import org.eclipse.rdf4j.sail.memory.MemoryStoreConnection; 37 | 38 | /** 39 | * 40 | * @author Adam Sotona (MSD) 41 | */ 42 | class MemoryStoreWithHalyardStrategy extends MemoryStore { 43 | 44 | @Override 45 | protected NotifyingSailConnection getConnectionInternal() throws SailException { 46 | return new MemoryStoreConnection(this) { 47 | 48 | @Override 49 | protected EvaluationStrategy getEvaluationStrategy(Dataset dataset, final TripleSource tripleSource) { 50 | EvaluationStrategy es = new HalyardEvaluationStrategy(new TripleSource() { 51 | @Override 52 | public CloseableIteration getStatements(Resource subj, IRI pred, Value obj, Resource... contexts) throws QueryEvaluationException { 53 | return new FilterIteration(tripleSource.getStatements(subj, pred, obj, contexts)){ 54 | boolean first = true; 55 | 56 | @Override 57 | public boolean hasNext() throws QueryEvaluationException { 58 | if (first) try { 59 | first = false; 60 | Thread.sleep(20); 61 | } catch (InterruptedException ex) { 62 | //ignore 63 | } 64 | return super.hasNext(); //To change body of generated methods, choose Tools | Templates. 65 | } 66 | 67 | @Override 68 | protected boolean accept(Statement object) throws QueryEvaluationException { 69 | return true; 70 | } 71 | }; 72 | } 73 | 74 | @Override 75 | public ValueFactory getValueFactory() { 76 | return tripleSource.getValueFactory(); 77 | } 78 | }, dataset, null, new HalyardEvaluationStatistics(null, null), -1); 79 | es.setOptimizerPipeline(new StandardQueryOptimizerPipeline(es, tripleSource, new EvaluationStatistics())); 80 | return es; 81 | } 82 | 83 | }; 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /strategy/src/test/java/com/msd/gin/halyard/strategy/W3CApprovedSPARQL11QueryTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.strategy; 18 | 19 | import junit.framework.Test; 20 | import org.eclipse.rdf4j.query.Dataset; 21 | import org.eclipse.rdf4j.query.parser.sparql.manifest.SPARQL11ManifestTest; 22 | import org.eclipse.rdf4j.query.parser.sparql.manifest.SPARQLQueryTest; 23 | import org.eclipse.rdf4j.repository.Repository; 24 | import org.eclipse.rdf4j.repository.dataset.DatasetRepository; 25 | 26 | import org.eclipse.rdf4j.repository.sail.SailRepository; 27 | 28 | /** 29 | * 30 | * @author Adam Sotona (MSD) 31 | */ 32 | public class W3CApprovedSPARQL11QueryTest extends SPARQLQueryTest { 33 | 34 | public static Test suite() throws Exception { 35 | return SPARQL11ManifestTest.suite(new Factory() { 36 | @Override 37 | public W3CApprovedSPARQL11QueryTest createSPARQLQueryTest(String testURI, String name, String queryFileURL, String resultFileURL, Dataset dataSet, boolean laxCardinality) { 38 | return createSPARQLQueryTest(testURI, name, queryFileURL, resultFileURL, dataSet, laxCardinality, false); 39 | } 40 | @Override 41 | public W3CApprovedSPARQL11QueryTest createSPARQLQueryTest(String testURI, String name, String queryFileURL, String resultFileURL, Dataset dataSet, boolean laxCardinality, boolean checkOrder) { 42 | String[] ignoredTests = { 43 | // test case incompatible with RDF 1.1 - see 44 | // http://lists.w3.org/Archives/Public/public-sparql-dev/2013AprJun/0006.html 45 | "STRDT TypeErrors", 46 | // test case incompatible with RDF 1.1 - see 47 | // http://lists.w3.org/Archives/Public/public-sparql-dev/2013AprJun/0006.html 48 | "STRLANG TypeErrors", 49 | // known issue: SES-937 50 | "sq03 - Subquery within graph pattern, graph variable is not bound"}; 51 | 52 | return new W3CApprovedSPARQL11QueryTest(testURI, name, queryFileURL, resultFileURL, dataSet, laxCardinality, checkOrder, ignoredTests); 53 | } 54 | }, true, true, false, "service"); 55 | } 56 | 57 | protected W3CApprovedSPARQL11QueryTest(String testURI, String name, String queryFileURL, String resultFileURL, Dataset dataSet, boolean laxCardinality, boolean checkOrder, String... ignoredTests) { 58 | super(testURI, name, queryFileURL, resultFileURL, dataSet, laxCardinality, checkOrder, ignoredTests); 59 | } 60 | 61 | @Override 62 | protected Repository newRepository() { 63 | return new DatasetRepository(new SailRepository(new MemoryStoreWithHalyardStrategy())); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /strategy/src/test/java/com/msd/gin/halyard/strategy/W3CApprovedSPARQL11UpdateTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.strategy; 18 | 19 | import java.util.Map; 20 | 21 | import junit.framework.Test; 22 | import org.eclipse.rdf4j.model.IRI; 23 | import org.eclipse.rdf4j.query.parser.sparql.manifest.SPARQL11ManifestTest; 24 | import org.eclipse.rdf4j.query.parser.sparql.manifest.SPARQLUpdateConformanceTest; 25 | import org.eclipse.rdf4j.repository.contextaware.ContextAwareRepository; 26 | import org.eclipse.rdf4j.repository.sail.SailRepository; 27 | 28 | /** 29 | * 30 | * @author Adam Sotona (MSD) 31 | */ 32 | public class W3CApprovedSPARQL11UpdateTest extends SPARQLUpdateConformanceTest { 33 | 34 | public W3CApprovedSPARQL11UpdateTest(String testURI, String name, String requestFile, IRI defaultGraphURI, Map inputNamedGraphs, IRI resultDefaultGraphURI, Map resultNamedGraphs) { 35 | super(testURI, name, requestFile, defaultGraphURI, inputNamedGraphs, resultDefaultGraphURI, resultNamedGraphs); 36 | } 37 | 38 | public static Test suite() throws Exception { 39 | return SPARQL11ManifestTest.suite( 40 | (String testURI1, String name, String requestFile, IRI defaultGraphURI, Map inputNamedGraphs1, IRI resultDefaultGraphURI, Map resultNamedGraphs1) 41 | -> new W3CApprovedSPARQL11UpdateTest(testURI1, name, requestFile, defaultGraphURI, inputNamedGraphs1, resultDefaultGraphURI, resultNamedGraphs1), true, true, false); 42 | } 43 | 44 | @Override 45 | protected ContextAwareRepository newRepository() throws Exception { 46 | SailRepository repo = new SailRepository(new MemoryStoreWithHalyardStrategy()); 47 | return new ContextAwareRepository(repo); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /strategy/src/test/java/com/msd/gin/halyard/strategy/collections/BigHashSetTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.strategy.collections; 18 | 19 | import java.io.IOException; 20 | import org.junit.Test; 21 | import static org.junit.Assert.*; 22 | 23 | /** 24 | * 25 | * @author Adam Sotona (MSD) 26 | */ 27 | public class BigHashSetTest { 28 | 29 | @Test 30 | public void testBigHashSet() throws Exception { 31 | BigHashSet bhs = new BigHashSet<>(); 32 | bhs.add("hi"); 33 | assertEquals("hi", bhs.iterator().next()); 34 | assertTrue(bhs.contains("hi")); 35 | bhs.close(); 36 | bhs.close(); 37 | } 38 | 39 | @Test(expected = IOException.class) 40 | public void testFailAdd() throws Exception { 41 | BigHashSet bhs = new BigHashSet<>(); 42 | bhs.close(); 43 | bhs.add("hi"); 44 | } 45 | 46 | @Test(expected = IOException.class) 47 | public void testFailContains() throws Exception { 48 | BigHashSet bhs = new BigHashSet<>(); 49 | bhs.close(); 50 | bhs.contains("hi"); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /strategy/src/test/java/com/msd/gin/halyard/strategy/collections/SorterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.strategy.collections; 18 | 19 | import java.io.IOException; 20 | import java.util.Iterator; 21 | import java.util.Map.Entry; 22 | import org.junit.Assert; 23 | import org.junit.Test; 24 | 25 | /** 26 | * 27 | * @author Adam Sotona (MSD) 28 | */ 29 | public class SorterTest { 30 | 31 | @Test 32 | public void testNoLimitNoDistinct() throws Exception { 33 | Sorter s = new Sorter<>(Long.MAX_VALUE, false); 34 | s.add("C"); 35 | s.add("C"); 36 | s.add("A"); 37 | s.add("B"); 38 | s.add("B"); 39 | Iterator> it = s.iterator(); 40 | assertEquals("A", 1, it.next()); 41 | assertEquals("B", 2, it.next()); 42 | assertEquals("C", 2, it.next()); 43 | Assert.assertFalse(it.hasNext()); 44 | s.close(); 45 | s.close(); 46 | } 47 | 48 | @Test 49 | public void testNoLimitDistinct() throws Exception { 50 | Sorter s = new Sorter<>(Long.MAX_VALUE, true); 51 | s.add("C"); 52 | s.add("C"); 53 | s.add("A"); 54 | s.add("B"); 55 | s.add("B"); 56 | Iterator> it = s.iterator(); 57 | assertEquals("A", 1, it.next()); 58 | assertEquals("B", 1, it.next()); 59 | assertEquals("C", 1, it.next()); 60 | Assert.assertFalse(it.hasNext()); 61 | s.close(); 62 | s.close(); 63 | } 64 | 65 | @Test 66 | public void testLimitNoDistinct() throws Exception { 67 | Sorter s = new Sorter<>(3, false); 68 | s.add("C"); 69 | s.add("C"); 70 | s.add("A"); 71 | s.add("A"); 72 | s.add("B"); 73 | s.add("B"); 74 | s.add("B"); 75 | Iterator> it = s.iterator(); 76 | assertEquals("A", 2, it.next()); 77 | assertEquals("B", 1, it.next()); 78 | Assert.assertFalse(it.hasNext()); 79 | s.close(); 80 | s.close(); 81 | } 82 | 83 | @Test 84 | public void testLimitDistinct() throws Exception { 85 | Sorter s = new Sorter<>(2, true); 86 | s.add("C"); 87 | s.add("C"); 88 | s.add("A"); 89 | s.add("B"); 90 | s.add("A"); 91 | s.add("B"); 92 | s.add("B"); 93 | Iterator> it = s.iterator(); 94 | assertEquals("A", 1, it.next()); 95 | assertEquals("B", 1, it.next()); 96 | Assert.assertFalse(it.hasNext()); 97 | s.close(); 98 | s.close(); 99 | } 100 | 101 | private static void assertEquals(String s, long l, Entry e) { 102 | Assert.assertEquals(s, e.getKey()); 103 | Assert.assertEquals(s, l, (long)e.getValue()); 104 | } 105 | 106 | @Test(expected = IOException.class) 107 | public void testFailAdd() throws Exception { 108 | Sorter s = new Sorter<>(Long.MAX_VALUE, false); 109 | s.close(); 110 | s.add("hi"); 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /strategy/src/test/java/com/msd/gin/halyard/vocab/VocabTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.vocab; 18 | 19 | import org.junit.Test; 20 | 21 | /** 22 | * 23 | * @author Adam Sotona (MSD) 24 | */ 25 | public class VocabTest { 26 | 27 | @Test 28 | public void testInstantiateConstants() { 29 | new HALYARD(); 30 | new VOID_EXT(); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /tools/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | halyard-tools 5 | jar 6 | 7 | com.msd.gin.halyard 8 | halyard 9 | 2.5 10 | 11 | 12 | 13 | ${project.groupId} 14 | halyard-common 15 | ${project.version} 16 | 17 | 18 | ${project.groupId} 19 | halyard-strategy 20 | ${project.version} 21 | 22 | 23 | ${project.groupId} 24 | halyard-sail 25 | ${project.version} 26 | 27 | 28 | ${project.groupId} 29 | halyard-rio 30 | ${project.version} 31 | 32 | 33 | org.eclipse.rdf4j 34 | rdf4j-console 35 | ${rdf4j.version} 36 | 37 | 38 | ch.qos.logback 39 | logback-classic 40 | 41 | 42 | 43 | 44 | org.apache.hbase 45 | hbase-server 46 | ${hbase.version} 47 | provided 48 | 49 | 50 | ${project.groupId} 51 | halyard-common 52 | ${project.version} 53 | test-jar 54 | test 55 | 56 | 57 | org.apache.hadoop 58 | hadoop-mapreduce-client-jobclient 59 | ${hadoop.version} 60 | test 61 | tests 62 | 63 | 64 | org.apache.hadoop 65 | hadoop-common 66 | ${hadoop.version} 67 | test 68 | tests 69 | 70 | 71 | org.apache.hadoop 72 | hadoop-mapreduce-client-hs 73 | ${hadoop.version} 74 | test 75 | 76 | 77 | org.apache.hadoop 78 | hadoop-yarn-server-resourcemanager 79 | ${hadoop.version} 80 | test 81 | 82 | 83 | org.apache.hadoop 84 | hadoop-yarn-server-tests 85 | ${hadoop.version} 86 | test 87 | tests 88 | 89 | 90 | commons-codec 91 | commons-codec 92 | 1.9 93 | 94 | 95 | org.apache.derby 96 | derby 97 | 10.12.1.1 98 | test 99 | 100 | 101 | -------------------------------------------------------------------------------- /tools/src/main/java/com/msd/gin/halyard/tools/HalyardMain.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.tools; 18 | 19 | import java.io.PrintWriter; 20 | import java.util.Arrays; 21 | import org.apache.commons.cli.HelpFormatter; 22 | import org.apache.commons.cli.UnrecognizedOptionException; 23 | import org.apache.hadoop.util.ToolRunner; 24 | 25 | /** 26 | * 27 | * @author Adam Sotona (MSD) 28 | */ 29 | public final class HalyardMain { 30 | 31 | HalyardMain() { 32 | } 33 | 34 | public static void main(String args[]) throws Exception { 35 | String first = args.length > 0 ? args[0] : null; 36 | if ("-v".equals(first) || "--version".equals(first)) { 37 | System.out.println("halyard version " + AbstractHalyardTool.getVersion()); 38 | } else { 39 | AbstractHalyardTool tools[] = new AbstractHalyardTool[] { 40 | new HalyardPreSplit(), 41 | new HalyardBulkLoad(), 42 | new HalyardStats(), 43 | new HalyardElasticIndexer(), 44 | new HalyardUpdate(), 45 | new HalyardBulkUpdate(), 46 | new HalyardExport(), 47 | new HalyardBulkExport(), 48 | new HalyardBulkDelete(), 49 | new HalyardProfile(), 50 | new HalyardEndpoint(), 51 | new HalyardSummary() 52 | }; 53 | for (AbstractHalyardTool tool : tools) { 54 | if (tool.name.equalsIgnoreCase(first)) { 55 | int ret = ToolRunner.run(tool, Arrays.copyOfRange(args, 1, args.length)); 56 | if (ret != 0) throw new RuntimeException("Tool " + tool.name + " exits with code: " + ret); 57 | return; 58 | } 59 | } 60 | try { 61 | if (first != null && !"-h".equals(first) && !"--help".equals(first)) { 62 | String msg = "Unrecognized command or option: " + first; 63 | System.out.println(msg); 64 | throw new UnrecognizedOptionException(msg); 65 | } 66 | } finally { 67 | PrintWriter pw = new PrintWriter(System.out); 68 | HelpFormatter hf = new HelpFormatter(); 69 | hf.printWrapped(pw, 100, "usage: halyard [ -h | -v | [] [-h] ...]"); 70 | hf.printWrapped(pw, 100, "\ncommands are:\n----------------------------------------------------------------------------------------------------"); 71 | for (AbstractHalyardTool tool : tools) { 72 | hf.printWrapped(pw, 100, 11, tool.name + " ".substring(tool.name.length()) + tool.header); 73 | } 74 | hf.printWrapped(pw, 100, 0, "\ngenericHadoopOptions are:\n----------------------------------------------------------------------------------------------------"); 75 | hf.printWrapped(pw, 100, 45, "-conf specify an application configuration file"); 76 | hf.printWrapped(pw, 100, 45, "-D use value for given property"); 77 | hf.printWrapped(pw, 100, 45, "-fs specify a namenode"); 78 | hf.printWrapped(pw, 100, 45, "-jt specify a job tracker"); 79 | hf.printWrapped(pw, 100, 45, "-files specify comma separated files to be copied to the map reduce cluster"); 80 | hf.printWrapped(pw, 100, 45, "-archives specify comma separated archives to be unarchived on the compute machines."); 81 | hf.printWrapped(pw, 100, 0, "\nJVM options of the Halyard command and local command-line applications can be specified using HADOOP_OPTS and HADOOP_CLIENT_OPTS environment variables, however JVM options of the Halyard MapReduce applications can only be modified using specific Hadoop properties."); 82 | pw.flush(); 83 | } 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /tools/src/main/java/com/msd/gin/halyard/tools/HalyardUpdate.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.tools; 18 | 19 | import org.apache.commons.cli.CommandLine; 20 | import org.eclipse.rdf4j.query.QueryLanguage; 21 | import org.eclipse.rdf4j.query.Update; 22 | import org.eclipse.rdf4j.query.impl.MapBindingSet; 23 | import org.eclipse.rdf4j.repository.sail.SailRepository; 24 | 25 | /** 26 | * Command line tool executing SPARQL Update query on Halyard dataset directly 27 | * @author Adam Sotona (MSD) 28 | */ 29 | public final class HalyardUpdate extends AbstractHalyardTool { 30 | 31 | public HalyardUpdate() { 32 | super( 33 | "update", 34 | "Halyard Update is a command-line application designed to run SPARQL Update operations to transform data in an HBase Halyard dataset", 35 | "Example: halyard update -s my_dataset -q 'insert {?o owl:sameAs ?s} where {?s owl:sameAs ?o}'" 36 | ); 37 | addOption("s", "source-dataset", "dataset_table", "Source HBase table with Halyard RDF store", true, true); 38 | addOption("q", "update-operation", "sparql_update_operation", "SPARQL update operation to be executed", true, true); 39 | addOption("i", "elastic-index", "elastic_index_url", "Optional ElasticSearch index URL", false, true); 40 | } 41 | 42 | 43 | public int run(CommandLine cmd) throws Exception { 44 | SailRepository rep = new SailRepository(new TimeAwareHBaseSail(getConf(), cmd.getOptionValue('s'), false, 0, true, 0, cmd.getOptionValue('i'), null)); 45 | rep.initialize(); 46 | try { 47 | Update u = rep.getConnection().prepareUpdate(QueryLanguage.SPARQL, cmd.getOptionValue('q')); 48 | ((MapBindingSet)u.getBindings()).addBinding(new TimeAwareHBaseSail.TimestampCallbackBinding()); 49 | LOG.info("Update execution started"); 50 | u.execute(); 51 | LOG.info("Update finished"); 52 | } finally { 53 | rep.shutDown(); 54 | } 55 | return 0; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /tools/src/main/java/com/msd/gin/halyard/tools/SimpleHttpServer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.tools; 18 | 19 | import com.sun.net.httpserver.HttpHandler; 20 | import com.sun.net.httpserver.HttpServer; 21 | 22 | 23 | import java.io.IOException; 24 | import java.net.InetSocketAddress; 25 | import java.util.concurrent.ExecutorService; 26 | import java.util.concurrent.Executors; 27 | import java.util.logging.Level; 28 | import java.util.logging.Logger; 29 | 30 | /** 31 | * This class is used to create a simple HTTP server listening on a specific port and handling requests by a 32 | * specified handler. 33 | * 34 | * @author sykorjan 35 | */ 36 | public class SimpleHttpServer { 37 | 38 | private HttpServer httpServer; 39 | 40 | private static final Logger LOGGER = Logger.getLogger(SimpleHttpServer.class.getName()); 41 | 42 | /** 43 | * Instantiate a new HTTP server. Use port number 0 (zero) to let the system select a new port number. 44 | * 45 | * @param port number of port 46 | * @param context context path 47 | * @param handler handler for handling HTTP requests 48 | */ 49 | public SimpleHttpServer(int port, String context, HttpHandler handler) throws IOException { 50 | // Maximum number of incoming TCP connections is set to system default value 51 | int backlog = 0; 52 | // Create HTTP server 53 | httpServer = HttpServer.create(new InetSocketAddress(port), backlog); 54 | // Create HTTP context with a given handler 55 | httpServer.createContext(context, handler); 56 | // Create an executor 57 | httpServer.setExecutor(Executors.newFixedThreadPool(4 * Runtime.getRuntime().availableProcessors())); 58 | } 59 | 60 | /** 61 | * Start server 62 | */ 63 | public void start() { 64 | httpServer.start(); 65 | LOGGER.log(Level.INFO, "Server started and is listening on port " + httpServer.getAddress().getPort()); 66 | } 67 | 68 | /** 69 | * Stop server 70 | */ 71 | public void stop() { 72 | // stop immediately 73 | ((ExecutorService) httpServer.getExecutor()).shutdown(); 74 | httpServer.stop(0); 75 | LOGGER.log(Level.INFO, "Server stopped"); 76 | } 77 | 78 | public InetSocketAddress getAddress() { 79 | return httpServer.getAddress(); 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /tools/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /tools/src/test/java/com/msd/gin/halyard/tools/HalyardBulkDeleteTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.tools; 18 | 19 | import com.msd.gin.halyard.common.HBaseServerTestInstance; 20 | import com.msd.gin.halyard.sail.HBaseSail; 21 | import java.io.File; 22 | import org.apache.commons.cli.MissingOptionException; 23 | import org.apache.commons.cli.UnrecognizedOptionException; 24 | import org.apache.hadoop.conf.Configuration; 25 | import org.apache.hadoop.util.ToolRunner; 26 | import org.eclipse.rdf4j.common.iteration.CloseableIteration; 27 | import org.eclipse.rdf4j.model.Statement; 28 | import org.eclipse.rdf4j.model.ValueFactory; 29 | import org.eclipse.rdf4j.model.impl.SimpleValueFactory; 30 | import org.eclipse.rdf4j.sail.SailException; 31 | import org.junit.Assert; 32 | import static org.junit.Assert.*; 33 | import org.junit.Test; 34 | 35 | /** 36 | * 37 | * @author Adam Sotona (MSD) 38 | */ 39 | public class HalyardBulkDeleteTest { 40 | 41 | private static final String TABLE = "bulkdeletetesttable"; 42 | 43 | @Test 44 | public void testBulkDelete() throws Exception { 45 | ValueFactory vf = SimpleValueFactory.getInstance(); 46 | Configuration conf = HBaseServerTestInstance.getInstanceConfig(); 47 | HBaseSail sail = new HBaseSail(conf, TABLE, true, -1, true, 0, null, null); 48 | sail.initialize(); 49 | for (int i=0; i<5; i++) { 50 | for (int j=0; j<5; j++) { 51 | sail.addStatement(vf.createIRI("http://whatever/subj" + i), vf.createIRI("http://whatever/pred"), vf.createIRI("http://whatever/obj" + j), i == 0 ? null: vf.createIRI("http://whatever/ctx" + i)); 52 | } 53 | } 54 | sail.commit(); 55 | sail.shutDown(); 56 | File htableDir = File.createTempFile("test_htable", ""); 57 | htableDir.delete(); 58 | 59 | assertEquals(0, ToolRunner.run(conf, new HalyardBulkDelete(), new String[]{ "-t", TABLE, "-o", "", "-g", "NONE", "-g", "", "-f", htableDir.toURI().toURL().toString()})); 60 | 61 | assertCount(23); 62 | 63 | htableDir = File.createTempFile("test_htable", ""); 64 | htableDir.delete(); 65 | 66 | assertEquals(0, ToolRunner.run(conf, new HalyardBulkDelete(), new String[]{ "-t", TABLE, "-s", "", "-f", htableDir.toURI().toURL().toString()})); 67 | 68 | assertCount(18); 69 | 70 | htableDir = File.createTempFile("test_htable", ""); 71 | htableDir.delete(); 72 | 73 | assertEquals(0, ToolRunner.run(conf, new HalyardBulkDelete(), new String[]{ "-t", TABLE, "-p", "", "-f", htableDir.toURI().toURL().toString()})); 74 | 75 | assertCount(0); 76 | } 77 | 78 | private static void assertCount(int expected) throws Exception { 79 | HBaseSail sail = new HBaseSail(HBaseServerTestInstance.getInstanceConfig(), TABLE, false, 0, true, 0, null, null); 80 | sail.initialize(); 81 | try { 82 | int count; 83 | try (CloseableIteration iter = sail.getStatements(null, null, null, true)) { 84 | count = 0; 85 | while (iter.hasNext()) { 86 | iter.next(); 87 | count++; 88 | } 89 | } 90 | Assert.assertEquals(expected, count); 91 | } finally { 92 | sail.shutDown(); 93 | } 94 | } 95 | 96 | @Test 97 | public void testHelp() throws Exception { 98 | assertEquals(-1, new HalyardBulkDelete().run(new String[]{"-h"})); 99 | } 100 | 101 | @Test(expected = MissingOptionException.class) 102 | public void testRunNoArgs() throws Exception { 103 | assertEquals(-1, new HalyardBulkDelete().run(new String[]{})); 104 | } 105 | 106 | @Test 107 | public void testRunVersion() throws Exception { 108 | assertEquals(0, new HalyardBulkDelete().run(new String[]{"-v"})); 109 | } 110 | 111 | @Test(expected = UnrecognizedOptionException.class) 112 | public void testRunInvalid() throws Exception { 113 | new HalyardBulkDelete().run(new String[]{"-invalid"}); 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /tools/src/test/java/com/msd/gin/halyard/tools/HalyardMainTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.tools; 18 | 19 | import org.apache.commons.cli.MissingOptionException; 20 | import org.apache.commons.cli.UnrecognizedOptionException; 21 | import org.junit.Test; 22 | 23 | /** 24 | * 25 | * @author Adam Sotona (MSD) 26 | */ 27 | public class HalyardMainTest { 28 | 29 | @Test 30 | public void testEmpty() throws Exception { 31 | HalyardMain.main(new String[]{}); 32 | } 33 | 34 | @Test 35 | public void testVersion() throws Exception { 36 | HalyardMain.main(new String[]{"-v"}); 37 | } 38 | 39 | @Test 40 | public void testVersionLong() throws Exception { 41 | HalyardMain.main(new String[]{"--version"}); 42 | } 43 | 44 | @Test 45 | public void testHelp() throws Exception { 46 | HalyardMain.main(new String[]{"-h"}); 47 | } 48 | 49 | @Test 50 | public void testHelpLong() throws Exception { 51 | HalyardMain.main(new String[]{"--help"}); 52 | } 53 | 54 | @Test(expected = UnrecognizedOptionException.class) 55 | public void testInvalid() throws Exception { 56 | HalyardMain.main(new String[]{"invalid"}); 57 | } 58 | 59 | @Test(expected = MissingOptionException.class) 60 | public void testPresplit() throws Exception { 61 | HalyardMain.main(new String[]{"presplit"}); 62 | } 63 | 64 | @Test(expected = MissingOptionException.class) 65 | public void testBulkload() throws Exception { 66 | HalyardMain.main(new String[]{"bulkload"}); 67 | } 68 | 69 | @Test(expected = MissingOptionException.class) 70 | public void testStats() throws Exception { 71 | HalyardMain.main(new String[]{"stats"}); 72 | } 73 | 74 | @Test(expected = MissingOptionException.class) 75 | public void testEsindex() throws Exception { 76 | HalyardMain.main(new String[]{"esindex"}); 77 | } 78 | 79 | @Test(expected = MissingOptionException.class) 80 | public void testUpdate() throws Exception { 81 | HalyardMain.main(new String[]{"update"}); 82 | } 83 | 84 | @Test(expected = MissingOptionException.class) 85 | public void testBulkupdate() throws Exception { 86 | HalyardMain.main(new String[]{"bulkupdate"}); 87 | } 88 | 89 | @Test(expected = MissingOptionException.class) 90 | public void testExport() throws Exception { 91 | HalyardMain.main(new String[]{"export"}); 92 | } 93 | 94 | @Test(expected = MissingOptionException.class) 95 | public void testBulkexport() throws Exception { 96 | HalyardMain.main(new String[]{"bulkexport"}); 97 | } 98 | 99 | @Test(expected = MissingOptionException.class) 100 | public void testBulkdelete() throws Exception { 101 | HalyardMain.main(new String[]{"bulkdelete"}); 102 | } 103 | 104 | @Test(expected = MissingOptionException.class) 105 | public void testProfile() throws Exception { 106 | HalyardMain.main(new String[]{"profile"}); 107 | } 108 | 109 | @Test(expected = MissingOptionException.class) 110 | public void testEndpoint() throws Exception { 111 | HalyardMain.main(new String[]{"endpoint"}); 112 | } 113 | 114 | public void testConstructor() { 115 | new HalyardMain(); 116 | } 117 | 118 | @Test(expected = RuntimeException.class) 119 | public void testExit() throws Exception { 120 | HalyardMain.main(new String[]{"bulkload", "-h"}); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /tools/src/test/java/com/msd/gin/halyard/tools/HalyardPreSplitTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.tools; 18 | 19 | import com.msd.gin.halyard.common.HBaseServerTestInstance; 20 | import com.msd.gin.halyard.common.HalyardTableUtils; 21 | import java.io.File; 22 | import java.io.FileOutputStream; 23 | import java.io.PrintStream; 24 | import org.apache.commons.cli.MissingOptionException; 25 | import org.apache.hadoop.hbase.client.HTable; 26 | import org.apache.hadoop.util.ToolRunner; 27 | import static org.junit.Assert.*; 28 | import org.junit.Test; 29 | 30 | /** 31 | * 32 | * @author Adam Sotona (MSD) 33 | */ 34 | public class HalyardPreSplitTest { 35 | 36 | @Test 37 | public void testPreSplit() throws Exception { 38 | File file = File.createTempFile("test_triples", ".nq"); 39 | try (PrintStream ps = new PrintStream(new FileOutputStream(file))) { 40 | ps.println(" \"whatever NT value 1\" ."); 41 | ps.println(" \"whatever NT value 2\" ."); 42 | } 43 | 44 | assertEquals(0, ToolRunner.run(HBaseServerTestInstance.getInstanceConfig(), new HalyardPreSplit(), new String[]{"-d", "1", "-l", "0", "-s", file.toURI().toURL().toString(), "-t", "preSplitTable"})); 45 | 46 | try (HTable t = HalyardTableUtils.getTable(HBaseServerTestInstance.getInstanceConfig(), "preSplitTable", false, 0)) { 47 | assertEquals(17, t.getRegionLocator().getStartKeys().length); 48 | } 49 | } 50 | 51 | @Test 52 | public void testPreSplitOfExisting() throws Exception { 53 | File file = File.createTempFile("test_triples", ".nq"); 54 | try (PrintStream ps = new PrintStream(new FileOutputStream(file))) { 55 | ps.println(" \"whatever NT value 1\" ."); 56 | ps.println(" \"whatever NT value 2\" ."); 57 | } 58 | 59 | HalyardTableUtils.getTable(HBaseServerTestInstance.getInstanceConfig(), "preSplitTable2", true, -1).close(); 60 | 61 | assertEquals(-1, ToolRunner.run(HBaseServerTestInstance.getInstanceConfig(), new HalyardPreSplit(), new String[]{"-d", "1", "-l", "0", "-s", file.toURI().toURL().toString(), "-t", "preSplitTable2"})); 62 | } 63 | 64 | @Test 65 | public void testHelp() throws Exception { 66 | assertEquals(-1, new HalyardPreSplit().run(new String[]{"-h"})); 67 | } 68 | 69 | @Test(expected = MissingOptionException.class) 70 | public void testRunNoArgs() throws Exception { 71 | assertEquals(-1, new HalyardPreSplit().run(new String[]{})); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /tools/src/test/java/com/msd/gin/halyard/tools/HalyardProfileTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.tools; 18 | 19 | import com.msd.gin.halyard.common.HBaseServerTestInstance; 20 | import com.msd.gin.halyard.common.HalyardTableUtils; 21 | import org.apache.commons.cli.MissingOptionException; 22 | import org.junit.Test; 23 | import org.apache.commons.cli.ParseException; 24 | import org.apache.hadoop.util.ToolRunner; 25 | import org.eclipse.rdf4j.query.MalformedQueryException; 26 | import org.eclipse.rdf4j.repository.RepositoryException; 27 | 28 | /** 29 | * 30 | * @author Adam Sotona (MSD) 31 | */ 32 | public class HalyardProfileTest { 33 | 34 | @Test 35 | public void testHelp() throws Exception { 36 | runProfile("-h"); 37 | } 38 | 39 | @Test(expected = MissingOptionException.class) 40 | public void testRunNoArgs() throws Exception { 41 | runProfile(); 42 | } 43 | 44 | @Test 45 | public void testVersion() throws Exception { 46 | runProfile("-v"); 47 | } 48 | 49 | @Test(expected = ParseException.class) 50 | public void testMissingArgs() throws Exception { 51 | runProfile("-s", "whatever"); 52 | } 53 | 54 | @Test(expected = ParseException.class) 55 | public void testUnknownArg() throws Exception { 56 | runProfile("-y"); 57 | } 58 | 59 | @Test(expected = ParseException.class) 60 | public void testDupArgs() throws Exception { 61 | runProfile("-s", "whatever", "-q", "query", "-s", "whatever2"); 62 | } 63 | 64 | @Test(expected = MalformedQueryException.class) 65 | public void testInvalidQuery() throws Exception { 66 | HalyardTableUtils.getTable(HBaseServerTestInstance.getInstanceConfig(), "profile", true, -1); 67 | runProfile("-s", "profile", "-q", "invalid * query {}"); 68 | } 69 | 70 | @Test(expected = RepositoryException.class) 71 | public void testNonexistentRepository() throws Exception { 72 | runProfile("-s", "whatever", "-q", "construct {?s ?p ?o} where {?s ?p ?o}"); 73 | } 74 | 75 | @Test 76 | public void testProfile() throws Exception { 77 | HalyardTableUtils.getTable(HBaseServerTestInstance.getInstanceConfig(), "profile", true, -1); 78 | runProfile("-s", "profile", "-q", "select * where {{?s ?o, ?o2} union {?s ?p ?o3. optional {?s ?p2 ?o4 }}}"); 79 | } 80 | 81 | @Test 82 | public void testProfileGraph() throws Exception { 83 | HalyardTableUtils.getTable(HBaseServerTestInstance.getInstanceConfig(), "profile", true, -1); 84 | runProfile("-s", "profile", "-q", "construct {?s ?o ?o2} where {{?s ?o, ?o2} union {?s ?p ?o3. optional {?s ?p2 ?o4 }}}"); 85 | } 86 | 87 | @Test 88 | public void testProfileBoolean() throws Exception { 89 | HalyardTableUtils.getTable(HBaseServerTestInstance.getInstanceConfig(), "profile", true, -1); 90 | runProfile("-s", "profile", "-q", "ask where {{?s ?o, ?o2} union {?s ?p ?o3. optional {?s ?p2 ?o4 }}}"); 91 | } 92 | 93 | private static int runProfile(String ... args) throws Exception { 94 | return ToolRunner.run(HBaseServerTestInstance.getInstanceConfig(), new HalyardProfile(), args); 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /tools/src/test/java/com/msd/gin/halyard/tools/TimeAwareHBaseSailTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Merck Sharp & Dohme Corp. a subsidiary of Merck & Co., 3 | * Inc., Kenilworth, NJ, USA. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package com.msd.gin.halyard.tools; 18 | 19 | import com.msd.gin.halyard.common.HBaseServerTestInstance; 20 | import com.msd.gin.halyard.sail.HBaseSail; 21 | import org.eclipse.rdf4j.common.iteration.CloseableIteration; 22 | import org.eclipse.rdf4j.model.Statement; 23 | import org.eclipse.rdf4j.query.QueryLanguage; 24 | import org.eclipse.rdf4j.query.Update; 25 | import org.eclipse.rdf4j.query.impl.MapBindingSet; 26 | import org.eclipse.rdf4j.repository.sail.SailRepository; 27 | import org.eclipse.rdf4j.repository.sail.SailRepositoryConnection; 28 | import org.eclipse.rdf4j.sail.SailException; 29 | import org.junit.Test; 30 | import static org.junit.Assert.assertFalse; 31 | import static org.junit.Assert.assertTrue; 32 | 33 | /** 34 | * 35 | * @author Adam Sotona (MSD) 36 | */ 37 | public class TimeAwareHBaseSailTest { 38 | 39 | @Test 40 | public void timestampLongTest() throws Exception { 41 | CloseableIteration iter; 42 | HBaseSail sail = new TimeAwareHBaseSail(HBaseServerTestInstance.getInstanceConfig(), "timestamptable", true, 0, true, 0, null, null); 43 | SailRepository rep = new SailRepository(sail); 44 | rep.initialize(); 45 | SailRepositoryConnection con = rep.getConnection(); 46 | assertTrue(testUpdate(con, "insert { } where {bind(2 as ?HALYARD_TIMESTAMP_SPECIAL_VARIABLE)}")); 47 | assertTrue(testUpdate(con, "delete { } where {bind(1 as ?HALYARD_TIMESTAMP_SPECIAL_VARIABLE)}")); 48 | assertFalse(testUpdate(con, "delete { } where {bind(4 as ?HALYARD_TIMESTAMP_SPECIAL_VARIABLE)}")); 49 | assertFalse(testUpdate(con, "insert { } where {bind(3 as ?HALYARD_TIMESTAMP_SPECIAL_VARIABLE)}")); 50 | assertTrue(testUpdate(con, "insert { } where {bind(4 as ?HALYARD_TIMESTAMP_SPECIAL_VARIABLE)}")); 51 | rep.shutDown(); 52 | 53 | } 54 | 55 | @Test 56 | public void timestampDateTimeTest() throws Exception { 57 | CloseableIteration iter; 58 | HBaseSail sail = new TimeAwareHBaseSail(HBaseServerTestInstance.getInstanceConfig(), "timestamptable", true, 0, true, 0, null, null); 59 | SailRepository rep = new SailRepository(sail); 60 | rep.initialize(); 61 | SailRepositoryConnection con = rep.getConnection(); 62 | assertTrue(testUpdate(con, "insert { } where {bind(\"2002-05-30T09:30:10.2\"^^ as ?HALYARD_TIMESTAMP_SPECIAL_VARIABLE)}")); 63 | assertTrue(testUpdate(con, "delete { } where {bind(\"2002-05-30T09:30:10.1\"^^ as ?HALYARD_TIMESTAMP_SPECIAL_VARIABLE)}")); 64 | assertFalse(testUpdate(con, "delete { } where {bind(\"2002-05-30T09:30:10.4\"^^ as ?HALYARD_TIMESTAMP_SPECIAL_VARIABLE)}")); 65 | assertFalse(testUpdate(con, "insert { } where {bind(\"2002-05-30T09:30:10.3\"^^ as ?HALYARD_TIMESTAMP_SPECIAL_VARIABLE)}")); 66 | assertTrue(testUpdate(con, "insert { } where {bind(\"2002-05-30T09:30:10.4\"^^ as ?HALYARD_TIMESTAMP_SPECIAL_VARIABLE)}")); 67 | rep.shutDown(); 68 | } 69 | 70 | private boolean testUpdate(SailRepositoryConnection con, String update) { 71 | Update u = con.prepareUpdate(update); 72 | ((MapBindingSet)u.getBindings()).addBinding(new TimeAwareHBaseSail.TimestampCallbackBinding()); 73 | u.execute(); 74 | con.commit(); 75 | return con.prepareTupleQuery(QueryLanguage.SPARQL, "select * where { ?p ?o}").evaluate().hasNext(); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /tools/src/test/resources/com/msd/gin/halyard/tools/test.properties: -------------------------------------------------------------------------------- 1 | query1=SELECT * WHERE { ?s ?p ?o . } LIMIT 10 2 | query2=@testQuery.sparql 3 | 4 | 5 | -------------------------------------------------------------------------------- /tools/src/test/resources/com/msd/gin/halyard/tools/testFail.properties: -------------------------------------------------------------------------------- 1 | query3=@missing_file 2 | 3 | 4 | -------------------------------------------------------------------------------- /tools/src/test/resources/com/msd/gin/halyard/tools/testMoreData.trig: -------------------------------------------------------------------------------- 1 | @prefix : . 2 | { 3 | :subjA :predB "whatever value ABC". 4 | } 5 | 6 | :graph0 { 7 | :subjA :predB "whatever value ABC". 8 | } 9 | 10 | :graph1 { 11 | :subjA :predB "whatever value ABC". 12 | } 13 | 14 | :graph2 { 15 | :subjA :predB "whatever value ABC". 16 | } 17 | -------------------------------------------------------------------------------- /tools/src/test/resources/com/msd/gin/halyard/tools/testQuery.sparql: -------------------------------------------------------------------------------- 1 | SELECT * WHERE { 2 | ?s ?p ?o . 3 | } LIMIT 10 4 | 5 | -------------------------------------------------------------------------------- /tools/src/test/resources/com/msd/gin/halyard/tools/testScript.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | curl \ 4 | --data-urlencode "query=SELECT * WHERE { ?s ?p ?o . } LIMIT 10" \ 5 | $ENDPOINT > ${1} -------------------------------------------------------------------------------- /tools/src/test/resources/com/msd/gin/halyard/tools/testScript2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | curl ${ENDPOINT}query1 > ${1} 4 | curl ${ENDPOINT}query2 >> ${1} 5 | -------------------------------------------------------------------------------- /tools/src/test/resources/com/msd/gin/halyard/tools/testStatsMoreTarget.trig: -------------------------------------------------------------------------------- 1 | @prefix : . 2 | @prefix halyard: . 3 | @prefix void: . 4 | @prefix void-ext: . 5 | @prefix xsd: . 6 | @prefix sd: . 7 | @prefix rdf: . 8 | 9 | halyard:statsContext { 10 | halyard:statsRoot a void:Dataset , sd:Dataset , sd:Graph ; 11 | sd:defaultGraph halyard:statsRoot ; 12 | sd:namedGraph :graph0 , :graph1 , :graph2 ; 13 | void:classes "1000"^^xsd:long ; 14 | void:triples "2001"^^xsd:long ; 15 | void:propertyPartition halyard:statsRoot_property_v0EPmHVxqhkyM3Yh_Wfu7gMOZGU ; 16 | void:properties "14"^^xsd:long ; 17 | void-ext:distinctLiterals "857"^^xsd:long ; 18 | void:distinctObjects "1957"^^xsd:long ; 19 | void:distinctSubjects "191"^^xsd:long ; 20 | void-ext:distinctBlankNodeObjects "100"^^xsd:long ; 21 | void-ext:distinctBlankNodeSubjects "91"^^xsd:long ; 22 | void-ext:distinctIRIReferenceObjects "1000"^^xsd:long ; 23 | void-ext:distinctIRIReferenceSubjects "100"^^xsd:long . 24 | 25 | :graph0 sd:name :graph0 ; 26 | sd:graph :graph0 ; 27 | a sd:NamedGraph , sd:Graph , void:Dataset ; 28 | void:classes "450"^^xsd:long ; 29 | void:triples "900"^^xsd:long ; 30 | void:propertyPartition :graph0_property_v0EPmHVxqhkyM3Yh_Wfu7gMOZGU ; 31 | void:properties "14"^^xsd:long ; 32 | void-ext:distinctLiterals "386"^^xsd:long ; 33 | void:distinctObjects "886"^^xsd:long ; 34 | void:distinctSubjects "91"^^xsd:long ; 35 | void-ext:distinctBlankNodeObjects "50"^^xsd:long ; 36 | void-ext:distinctBlankNodeSubjects "41"^^xsd:long ; 37 | void-ext:distinctIRIReferenceObjects "450"^^xsd:long ; 38 | void-ext:distinctIRIReferenceSubjects "50"^^xsd:long . 39 | 40 | :graph0_property_v0EPmHVxqhkyM3Yh_Wfu7gMOZGU a void:Dataset ; 41 | void:property rdf:type ; 42 | void:triples "450"^^xsd:long . 43 | 44 | :graph1 sd:name :graph1 ; 45 | sd:graph :graph1 ; 46 | a sd:NamedGraph , sd:Graph , void:Dataset ; 47 | void:classes "450"^^xsd:long ; 48 | void:triples "901"^^xsd:long ; 49 | void:propertyPartition :graph1_property_v0EPmHVxqhkyM3Yh_Wfu7gMOZGU ; 50 | void:properties "15"^^xsd:long ; 51 | void-ext:distinctLiterals "387"^^xsd:long ; 52 | void:distinctObjects "887"^^xsd:long ; 53 | void:distinctSubjects "91"^^xsd:long ; 54 | void-ext:distinctBlankNodeObjects "50"^^xsd:long ; 55 | void-ext:distinctBlankNodeSubjects "40"^^xsd:long ; 56 | void-ext:distinctIRIReferenceObjects "450"^^xsd:long ; 57 | void-ext:distinctIRIReferenceSubjects "51"^^xsd:long . 58 | 59 | :graph1_property_v0EPmHVxqhkyM3Yh_Wfu7gMOZGU a void:Dataset ; 60 | void:property rdf:type ; 61 | void:triples "450"^^xsd:long . 62 | 63 | halyard:statsRoot_property_v0EPmHVxqhkyM3Yh_Wfu7gMOZGU a void:Dataset ; 64 | void:property rdf:type ; 65 | void:triples "1000"^^xsd:long . 66 | } 67 | -------------------------------------------------------------------------------- /tools/src/test/resources/com/msd/gin/halyard/tools/testStatsTarget.trig: -------------------------------------------------------------------------------- 1 | @prefix : . 2 | @prefix halyard: . 3 | @prefix void: . 4 | @prefix void-ext: . 5 | @prefix xsd: . 6 | @prefix sd: . 7 | @prefix rdf: . 8 | 9 | halyard:statsContext { 10 | halyard:statsRoot a void:Dataset , sd:Dataset , sd:Graph ; 11 | sd:defaultGraph halyard:statsRoot ; 12 | sd:namedGraph :graph0 , :graph1 , :graph2 ; 13 | void:classes "1000"^^xsd:long ; 14 | void:triples "2001"^^xsd:long ; 15 | void:propertyPartition halyard:statsRoot_property_v0EPmHVxqhkyM3Yh_Wfu7gMOZGU ; 16 | void:properties "14"^^xsd:long ; 17 | void-ext:distinctLiterals "857"^^xsd:long ; 18 | void:distinctObjects "1957"^^xsd:long ; 19 | void:distinctSubjects "191"^^xsd:long ; 20 | void-ext:distinctBlankNodeObjects "100"^^xsd:long ; 21 | void-ext:distinctBlankNodeSubjects "91"^^xsd:long ; 22 | void-ext:distinctIRIReferenceObjects "1000"^^xsd:long ; 23 | void-ext:distinctIRIReferenceSubjects "100"^^xsd:long . 24 | 25 | :graph0 sd:name :graph0 ; 26 | sd:graph :graph0 ; 27 | a sd:NamedGraph , sd:Graph , void:Dataset ; 28 | void:classes "450"^^xsd:long ; 29 | void:triples "900"^^xsd:long ; 30 | void:propertyPartition :graph0_property_v0EPmHVxqhkyM3Yh_Wfu7gMOZGU ; 31 | void:properties "14"^^xsd:long ; 32 | void-ext:distinctLiterals "386"^^xsd:long ; 33 | void:distinctObjects "886"^^xsd:long ; 34 | void:distinctSubjects "91"^^xsd:long ; 35 | void-ext:distinctBlankNodeObjects "50"^^xsd:long ; 36 | void-ext:distinctBlankNodeSubjects "41"^^xsd:long ; 37 | void-ext:distinctIRIReferenceObjects "450"^^xsd:long ; 38 | void-ext:distinctIRIReferenceSubjects "50"^^xsd:long . 39 | 40 | :graph0_property_v0EPmHVxqhkyM3Yh_Wfu7gMOZGU a void:Dataset ; 41 | void:property rdf:type ; 42 | void:triples "450"^^xsd:long . 43 | 44 | :graph1 sd:name :graph1 ; 45 | sd:graph :graph1 ; 46 | a sd:NamedGraph , sd:Graph , void:Dataset ; 47 | void:classes "450"^^xsd:long ; 48 | void:triples "900"^^xsd:long ; 49 | void:propertyPartition :graph1_property_v0EPmHVxqhkyM3Yh_Wfu7gMOZGU ; 50 | void:properties "14"^^xsd:long ; 51 | void-ext:distinctLiterals "386"^^xsd:long ; 52 | void:distinctObjects "886"^^xsd:long ; 53 | void:distinctSubjects "90"^^xsd:long ; 54 | void-ext:distinctBlankNodeObjects "50"^^xsd:long ; 55 | void-ext:distinctBlankNodeSubjects "40"^^xsd:long ; 56 | void-ext:distinctIRIReferenceObjects "450"^^xsd:long ; 57 | void-ext:distinctIRIReferenceSubjects "50"^^xsd:long . 58 | 59 | :graph1_property_v0EPmHVxqhkyM3Yh_Wfu7gMOZGU a void:Dataset ; 60 | void:property rdf:type ; 61 | void:triples "450"^^xsd:long . 62 | 63 | halyard:statsRoot_property_v0EPmHVxqhkyM3Yh_Wfu7gMOZGU a void:Dataset ; 64 | void:property rdf:type ; 65 | void:triples "1000"^^xsd:long . 66 | } 67 | -------------------------------------------------------------------------------- /tools/src/test/resources/com/msd/gin/halyard/tools/testStatsTargetPartial.trig: -------------------------------------------------------------------------------- 1 | @prefix : . 2 | @prefix halyard: . 3 | @prefix void: . 4 | @prefix void-ext: . 5 | @prefix xsd: . 6 | @prefix sd: . 7 | @prefix rdf: . 8 | 9 | halyard:statsContext { 10 | halyard:statsRoot sd:namedGraph :graph0 . 11 | 12 | :graph0 sd:name :graph0 ; 13 | sd:graph :graph0 ; 14 | a sd:NamedGraph , sd:Graph , void:Dataset ; 15 | void:classes "450"^^xsd:long ; 16 | void:triples "900"^^xsd:long ; 17 | void:propertyPartition :graph0_property_v0EPmHVxqhkyM3Yh_Wfu7gMOZGU ; 18 | void:properties "14"^^xsd:long ; 19 | void-ext:distinctLiterals "386"^^xsd:long ; 20 | void:distinctObjects "886"^^xsd:long ; 21 | void:distinctSubjects "91"^^xsd:long ; 22 | void-ext:distinctBlankNodeObjects "50"^^xsd:long ; 23 | void-ext:distinctBlankNodeSubjects "41"^^xsd:long ; 24 | void-ext:distinctIRIReferenceObjects "450"^^xsd:long ; 25 | void-ext:distinctIRIReferenceSubjects "50"^^xsd:long . 26 | 27 | :graph0_property_v0EPmHVxqhkyM3Yh_Wfu7gMOZGU a void:Dataset ; 28 | void:property rdf:type ; 29 | void:triples "450"^^xsd:long . 30 | } 31 | -------------------------------------------------------------------------------- /webapps/nbactions.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | run 5 | 6 | jar 7 | 8 | 9 | process-classes 10 | org.codehaus.mojo:exec-maven-plugin:1.2.1:exec 11 | 12 | 13 | -classpath %classpath com.msd.gin.halyard.webapps.EmbeddedTomcatMain 14 | java 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /webapps/src/main/assembly/webapps-assembly.xml: -------------------------------------------------------------------------------- 1 | 4 | bin 5 | 6 | zip 7 | 8 | false 9 | 10 | 11 | ${project.basedir}/.. 12 | 13 | readme.md 14 | index.html 15 | LICENSE* 16 | img/* 17 | 18 | 19 | 20 | ${project.basedir}/target 21 | 22 | 23 | rdf4j-server.war 24 | rdf4j-workbench.war 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /webapps/src/main/patches/rdf4j-workbench/transformations/create-hbase.xsl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 |
17 | 18 | 19 | 20 | 23 | 30 | 31 | 32 | 33 | 36 | 40 | 41 | 42 | 43 | 46 | 50 | 51 | 52 | 53 | 56 | 60 | 61 | 62 | 63 | 66 | 74 | 75 | 76 | 77 | 80 | 84 | 85 | 86 | 87 | 90 | 98 | 99 | 100 | 101 | 104 | 108 | 109 | 110 | 111 | 114 | 118 | 119 | 120 | 121 | 122 | 129 | 130 | 131 |
21 | 22 | 24 | 29 |
34 | 35 | 37 | 39 |
44 | 45 | 47 | 49 |
54 | HBase Table Name 55 | 57 | 59 |
64 | Create HBase Table if missing 65 | 67 | 69 | 70 | 72 | 73 |
78 | HBase Table presplit bits 79 | 81 | 83 |
88 | Use Halyard Push Evaluation Strategy 89 | 91 | 93 | 94 | 96 | 97 |
102 | Query Evaluation Timeout (s) 103 | 105 | 107 |
112 | Optional ElasticSearch Index URL 113 | 115 | 117 |
123 | 126 | 128 |
132 |
133 | 135 | 136 | 137 | 138 | --------------------------------------------------------------------------------