├── .gitignore ├── LICENSE ├── NOTICE ├── README.md ├── lizard-base ├── Lizard-v1.thrift ├── Lizard-v2.thrift ├── Lizard.thrift ├── gen-thrift ├── pom.xml └── src │ ├── main │ └── java │ │ ├── lizard │ │ ├── api │ │ │ ├── Async.java │ │ │ ├── NodeHandler.java │ │ │ ├── TLZ │ │ │ │ ├── NodeCtl.java │ │ │ │ ├── Ping.java │ │ │ │ ├── TLZ_Index.java │ │ │ │ ├── TLZ_IndexName.java │ │ │ │ ├── TLZ_NodeId.java │ │ │ │ ├── TLZ_NodeTable.java │ │ │ │ ├── TLZ_Patch.java │ │ │ │ ├── TLZ_PatchAction.java │ │ │ │ ├── TLZ_PatchEntry.java │ │ │ │ ├── TLZ_Ping.java │ │ │ │ ├── TLZ_ShardIndex.java │ │ │ │ ├── TLZ_SubjectPredicateList.java │ │ │ │ ├── TLZ_ThriftObjectTable.java │ │ │ │ ├── TLZ_TupleNodeId.java │ │ │ │ └── TxnCtl.java │ │ │ ├── TLZlib.java │ │ │ ├── TxnClient.java │ │ │ └── TxnHandler.java │ │ ├── comms │ │ │ ├── CommsException.java │ │ │ ├── ConnState.java │ │ │ ├── Connection.java │ │ │ ├── ConnectionMgr.java │ │ │ └── thrift │ │ │ │ ├── ThriftClient.java │ │ │ │ ├── ThriftLib.java │ │ │ │ ├── ThriftServer.java │ │ │ │ └── ThriftServerAcceptFork.java │ │ └── system │ │ │ ├── Component.java │ │ │ ├── ComponentBase.java │ │ │ ├── ComponentTxn.java │ │ │ ├── Distributor.java │ │ │ ├── LifeCycle.java │ │ │ ├── LifeCycleBase.java │ │ │ ├── LizardConst.java │ │ │ ├── LizardException.java │ │ │ ├── LzLib.java │ │ │ ├── LzLog.java │ │ │ ├── LzTxnId.java │ │ │ └── RemoteControl.java │ │ └── migrate │ │ ├── Q.java │ │ └── TupleIndexEmpty.java │ └── test │ └── java │ └── lizard │ ├── comms │ ├── CommsTestLib.java │ ├── TS_LzComms.java │ ├── TestLzClientServer.java │ └── TestTLZ.java │ └── test │ └── LzBaseTest.java ├── lizard-cluster ├── pom.xml └── src │ └── main │ └── java │ └── lizard │ ├── build │ ├── LzBuildZk.java │ ├── LzBuilderDataset.java │ ├── LzBuilderIndexServer.java │ ├── LzBuilderNodeServer.java │ └── LzDeploy.java │ ├── cluster │ ├── Cluster.java │ ├── ClusterCtl.java │ ├── ClusterMonitor.java │ └── Platform.java │ ├── conf │ ├── ConfCluster.java │ ├── ConfDataset.java │ ├── ConfDeploy.java │ ├── ConfElement.java │ ├── ConfIndex.java │ ├── ConfIndexElement.java │ ├── ConfNodeTable.java │ ├── ConfNodeTableElement.java │ ├── ConfZookeeper.java │ ├── Config.java │ ├── Endpoint.java │ ├── LzConfigurationException.java │ ├── NetAddr.java │ ├── NetHost.java │ ├── VNode.java │ ├── VNodeAddr.java │ ├── VNodeLayout.java │ ├── assembler │ │ ├── AssemblerYaml.java │ │ ├── LizardAssembler.java │ │ └── VocabLizard.java │ └── parsers │ │ ├── LzConfParserRDF.java │ │ ├── LzConfParserYAML.java │ │ ├── LzDatasetDesc.java │ │ └── YAML.java │ ├── dataset │ ├── TransactionalComponentRemote.java │ └── TransactionalComponentZkLock.java │ ├── deploy │ └── Deploy.java │ └── sys │ └── Lizard.java ├── lizard-cmds ├── README.md ├── bin │ ├── init │ ├── lizard-command │ ├── lz │ ├── lz-servers │ └── shard ├── pom.xml └── src │ └── main │ └── java │ └── lz │ ├── LZ_Deploy.java │ ├── LZ_Monitor.java │ ├── LZ_Query.java │ ├── LZ_ShardIndex.java │ ├── LZ_Zookeeper.java │ └── lz.java ├── lizard-dist └── pom.xml ├── lizard-index-server ├── pom.xml └── src │ └── main │ └── java │ └── lizard │ └── index │ ├── ClusterAccessor.java │ ├── ClusterTupleIndex.java │ ├── DistributorTupleIndex.java │ ├── DistributorTuplesBySubject.java │ ├── DistributorTuplesReplicate.java │ ├── MergeIterator.java │ ├── RemoteAccessData.java │ ├── Shard.java │ ├── TClientIndex.java │ ├── TClientIndexOps.java │ ├── THandlerTupleIndex.java │ ├── TServerIndex.java │ └── TupleIndexRemote.java ├── lizard-node-server ├── pom.xml └── src │ └── main │ └── java │ └── lizard │ └── node │ ├── ClusterNodeTable.java │ ├── DistributorNodes.java │ ├── DistributorNodesBySegment.java │ ├── DistributorNodesReplicate.java │ ├── NodeTableRemote.java │ ├── TClientNode.java │ ├── THandlerNodeTable.java │ ├── TServerNode.java │ └── ThriftObjectFile.java ├── lizard-query-server ├── pom.xml └── src │ └── main │ └── java │ └── lizard │ └── query │ ├── AccessorRemote.java │ ├── LizardQuery.java │ ├── LzDataset.java │ └── QuackLizard.java ├── lz-dev ├── .gitignore ├── D.ttl ├── D1.ttl ├── D2.ttl ├── Q.rq ├── README.md ├── configuration.txt ├── dev-run ├── fix-headers ├── fu ├── gen-thrift ├── log4j.properties ├── lz ├── machine1.ttl ├── machine2.ttl ├── pom.xml ├── pushdata ├── setup-yaml │ ├── config-dev.yaml │ ├── config-disk-1.yaml │ ├── config-disk-2.yaml │ └── config-layout.yaml ├── setup1 │ ├── README.md │ ├── conf-dataset.ttl │ ├── conf-index.ttl │ ├── conf-layout.ttl │ ├── conf-node.ttl │ ├── fuseki-server.ttl │ ├── log4j.properties │ ├── lz-fuseki │ └── lz-servers ├── setup1a │ ├── README.md │ ├── conf-dataset.ttl │ ├── conf-index.ttl │ ├── conf-node.ttl │ ├── fuseki.ttl │ ├── log4j.properties │ ├── lz-fuseki │ └── lz-servers ├── setup2 │ ├── README.md │ ├── conf-dataset.ttl │ ├── conf-index.ttl │ ├── conf-layout.ttl │ ├── conf-node.ttl │ ├── fuseki-server.ttl │ ├── log4j.properties │ ├── lz-fuseki │ └── lz-servers ├── src-header-text └── src │ └── main │ └── java │ ├── conf2 │ ├── Lizard2.java │ └── LzConfigDefault.java │ ├── lizard │ └── index │ │ └── tuple │ │ ├── TupleIndexConverter.java │ │ └── TupleIndexReorder.java │ └── lz_dev │ ├── DeployDev.java │ ├── LzDev.java │ ├── LzMain.java │ ├── PROJECT_Lizard.java │ └── perf │ └── PerfNodeTableBatches.java ├── lz-setup ├── README.md ├── gitignore └── org.eclipse.jdt.ui.prefs └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | 3 | # Package Files # 4 | *.jar 5 | *.war 6 | *.ear 7 | 8 | # Eclipse 9 | .classpath 10 | .project 11 | .settings 12 | /classes 13 | 14 | # maven 15 | target 16 | 17 | # Development 18 | # Emacs files 19 | \#* 20 | .#* 21 | *.bak 22 | hs_err_pid*.log 23 | 24 | # Cached classpath information 25 | /lizard-cmds/lizard.classpath 26 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Lizard 2 | ====== 3 | Copyright 2013, 2014, 2015 Andy Seaborne 4 | Copyright 2014, 2015 Epimoprhics Ltd. 5 | 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Project Lizard 2 | ============== 3 | 4 | _Project Lizard_ -- a clustered SPARQL database providing a fault-tolerant operation. 5 | 6 | ## Fault tolerant 7 | 8 | Lizard uses several machines to provide the SPARQL service. If any one 9 | machine fails (e.g. hardware or software problem), then Lizard is able to 10 | continue providing the SPARQL service. Lizard assumes problems are 11 | "fail-stop" -- the machine fails and stops on an error, and does not 12 | generate malicious information. No [Byzantine fault tolerance] 13 | (https://en.wikipedia.org/wiki/Byzantine_fault_tolerance). 14 | 15 | ## License 16 | 17 | License: Apache Software License 18 | 19 | See [LICENSE](LICENSE) and [NOTICE](NOTICE) for details. 20 | 21 | ## Status 22 | 23 | Prototype. 24 | 25 | Lizard is yet not suitable for production use. 26 | 27 | Do not store the only copy of data in the database. 28 | Persistent data formats will change; there will be no 29 | migration path other than to reload the database. 30 | 31 | ## Wiki 32 | 33 | [Lizard Wiki](https://github.com/afs/lizard/wiki) 34 | 35 | ## Downloads 36 | 37 | Currently, Lizard requires building from source. 38 | 39 | ## Build 40 | 41 | Lizard requires Java 8. 42 | 43 | To build Lizard: 44 | 45 | 1. Get the source: `git clone https://github.com/afs/lizard/`. 46 | 1. Build and install the related "[Mantis](https://github.com/afs/mantis)" library. 47 | 1. Get the source. 48 | 1. `mvn clean install` 49 | 1. Build and install Lizard locally: `mvn clean install` 50 | 51 | ## Run 52 | 53 | -------------------------------------------------------------------------------- /lizard-base/Lizard-v2.thrift: -------------------------------------------------------------------------------- 1 | namespace java lizard.api.TLZ 2 | 3 | include "BinaryRDF.thrift" 4 | 5 | // ---- Admin 6 | service Ping { void ping() } 7 | 8 | // Used in the standalone server only. 9 | struct TLZ_Ping { 10 | 1: required i64 marker ; 11 | } 12 | 13 | // ---- Transaction 14 | typedef i64 TxnId 15 | 16 | service NodeCtl { 17 | void ping() 18 | oneway void stop() 19 | } 20 | 21 | service TxnCtl extends NodeCtl { 22 | void txnBeginRead(1: TxnId txnId) 23 | void txnBeginWrite(1: TxnId txnId) 24 | void txnPrepare(1: TxnId txnId) 25 | void txnCommit(1: TxnId txnId) 26 | void txnAbort(1: TxnId txnId) 27 | void txnEnd(1: TxnId txnId) 28 | } 29 | 30 | // ---- Index 31 | 32 | struct TLZ_TupleNodeId { 33 | 1: required i64 S ; 34 | 9: required i64 P ; 35 | 3: required i64 O ; 36 | 4: optional i64 G ; 37 | } 38 | 39 | enum TLZ_IndexName { SPO , POS , PSO , OSP } 40 | 41 | struct TLZ_ShardIndex { 42 | 1: required TLZ_IndexName indexName ; 43 | 9: required i32 shardId ; 44 | } 45 | 46 | struct TLZ_SubjectPredicateList { 47 | 1: required i64 subject ; 48 | 9: list predicates ; 49 | } 50 | 51 | enum TLZ_PatchAction { ADD , DEL } 52 | 53 | struct TLZ_PatchEntry { 54 | 1: required TLZ_PatchAction action ; 55 | 9: required TLZ_TupleNodeId tuple ; 56 | } 57 | 58 | struct TLZ_Patch { 59 | 1: required list entities ; 60 | } 61 | 62 | // ---- Index 63 | service TLZ_Index extends TxnCtl { 64 | void idxAdd (1: i64 requestId, 9: TxnId txnId, 3: TLZ_ShardIndex shard, 4: TLZ_TupleNodeId tuple) 65 | void idxAddAll(1: i64 requestId, 9: TxnId txnId, 3: TLZ_ShardIndex shard, 4: list tuples) 66 | void idxDelete(1: i64 requestId, 9: TxnId txnId, 3: TLZ_ShardIndex shard, 4: TLZ_TupleNodeId tuple) 67 | void idxDeleteAll(1: i64 requestId, 9: TxnId txnId, 3: TLZ_ShardIndex shard, 4: list tuples) 68 | list idxFind(1: i64 requestId, 9: TxnId txnId, 3: TLZ_ShardIndex shard, 4: TLZ_TupleNodeId pattern) 69 | // TLZ_SubjectPredicateList find(X) 70 | // patch 71 | } 72 | 73 | // Local Variables: 74 | // tab-width: 2 75 | // indent-tabs-mode: nil 76 | // comment-default-style: "//" 77 | // End: 78 | 79 | 80 | struct TLZ_NodeId { 81 | 1: required i64 nodeId ; 82 | } 83 | 84 | service TLZ_NodeTable extends TxnCtl { 85 | TLZ_NodeId allocNodeId(1: i64 requestId, 9: TxnId txnId, 3: BinaryRDF.RDF_Term node) 86 | TLZ_NodeId findByNode(1: i64 requestId, 9: TxnId txnId, 3: BinaryRDF.RDF_Term node) 87 | BinaryRDF.RDF_Term findByNodeId(1: i64 requestId, 9: TxnId txnId, 3: TLZ_NodeId nodeId) 88 | list allocNodeIds(1: i64 requestId, 9: TxnId txnId, 3: list nodes) 89 | list lookupNodeIds(1: i64 requestId, 9: TxnId txnId, 3: list nodeIds) 90 | } 91 | 92 | // Local Variables: 93 | // tab-width: 2 94 | // indent-tabs-mode: nil 95 | // comment-default-style: "//" 96 | // End: 97 | -------------------------------------------------------------------------------- /lizard-base/gen-thrift: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if (( $# == 0 )) 4 | then 5 | echo "No input files" 1>&2 6 | echo "Usage: $(basename $0) FILE ..." 1>&2 7 | exit 1 8 | fi 9 | 10 | # Directory for Jena RDF term encoding for "BinaryRDF.thrift" 11 | INC=$HOME/Jena/jena-arq/Grammar/ 12 | 13 | PKG="src/main/java/lizard/api/TLZ" 14 | rm -f $PKG/*.java 15 | 16 | THRIFT=$HOME/ASF/thrift-0.10.0/compiler/cpp/thrift 17 | 18 | # No -r : do not generate code for included files. 19 | $THRIFT -I $INC -out src/main/java -gen java "$@" 20 | 21 | for f in $PKG/*.java 22 | do 23 | perl -i.bak -p -e 's/^\@SuppressWarnings.*$/\@SuppressWarnings("all")/' $f 24 | rm $f.bak 25 | done 26 | -------------------------------------------------------------------------------- /lizard-base/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 23 | 4.0.0 24 | 25 | 26 | org.seaborne.lizard 27 | lizard 28 | .. 29 | 0.0.0-SNAPSHOT 30 | 31 | 32 | lizard-base 33 | 34 | jar 35 | 36 | 37 | 38 | Apache 2.0 License 39 | http://www.apache.org/licenses/LICENSE-2.0 40 | 41 | 42 | 43 | 44 | 45 | 46 | org.apache.jena 47 | apache-jena-libs 48 | pom 49 | 50 | 51 | 52 | org.apache.jena 53 | jena-base 54 | tests 55 | test 56 | 57 | 58 | 59 | org.apache.jena 60 | jena-core 61 | tests 62 | test 63 | 64 | 65 | 66 | org.apache.jena 67 | jena-arq 68 | tests 69 | test 70 | 71 | 72 | 73 | junit 74 | junit 75 | test 76 | 77 | 78 | 79 | org.slf4j 80 | slf4j-api 81 | 82 | 83 | 84 | org.slf4j 85 | slf4j-log4j12 86 | 87 | 88 | 89 | log4j 90 | log4j 91 | 92 | 93 | 94 | org.apache.curator 95 | curator-framework 96 | 97 | 98 | 99 | org.apache.curator 100 | curator-recipes 101 | 102 | 103 | 104 | 105 | 106 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/api/NodeHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.api; 19 | import org.apache.thrift.TException ; 20 | import org.slf4j.Logger ; 21 | 22 | import lizard.api.TLZ.NodeCtl ; 23 | 24 | public abstract class NodeHandler implements NodeCtl.Iface { 25 | 26 | protected boolean active = true ; 27 | @Override 28 | public void ping() throws TException { 29 | checkActive() ; 30 | log().info("ping") ; 31 | } 32 | 33 | @Override 34 | public void stop() throws TException { 35 | active = false ; 36 | log().info("stop") ; 37 | } 38 | 39 | protected abstract Logger log() ; 40 | protected abstract Logger logtxn() ; 41 | protected abstract String getLabel() ; 42 | protected void checkActive() throws TException { 43 | if ( ! active ) 44 | throw new TException("Not active") ; 45 | } 46 | 47 | } 48 | 49 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/api/TLZ/TLZ_IndexName.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.10.0) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | package lizard.api.TLZ; 8 | 9 | public enum TLZ_IndexName implements org.apache.thrift.TEnum { 10 | SPO(0), 11 | POS(1), 12 | PSO(2), 13 | OSP(3); 14 | 15 | private final int value; 16 | 17 | private TLZ_IndexName(int value) { 18 | this.value = value; 19 | } 20 | 21 | /** 22 | * Get the integer value of this enum value, as defined in the Thrift IDL. 23 | */ 24 | @Override 25 | public int getValue() { 26 | return value; 27 | } 28 | 29 | /** 30 | * Find a the enum type by its integer value, as defined in the Thrift IDL. 31 | * @return null if the value is not found. 32 | */ 33 | public static TLZ_IndexName findByValue(int value) { 34 | switch (value) { 35 | case 0: 36 | return SPO; 37 | case 1: 38 | return POS; 39 | case 2: 40 | return PSO; 41 | case 3: 42 | return OSP; 43 | default: 44 | return null; 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/api/TLZ/TLZ_PatchAction.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.10.0) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | package lizard.api.TLZ; 8 | 9 | public enum TLZ_PatchAction implements org.apache.thrift.TEnum { 10 | ADD(0), 11 | DEL(1); 12 | 13 | private final int value; 14 | 15 | private TLZ_PatchAction(int value) { 16 | this.value = value; 17 | } 18 | 19 | /** 20 | * Get the integer value of this enum value, as defined in the Thrift IDL. 21 | */ 22 | @Override 23 | public int getValue() { 24 | return value; 25 | } 26 | 27 | /** 28 | * Find a the enum type by its integer value, as defined in the Thrift IDL. 29 | * @return null if the value is not found. 30 | */ 31 | public static TLZ_PatchAction findByValue(int value) { 32 | switch (value) { 33 | case 0: 34 | return ADD; 35 | case 1: 36 | return DEL; 37 | default: 38 | return null; 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/comms/CommsException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.comms; 19 | 20 | import lizard.system.LizardException ; 21 | 22 | public class CommsException extends LizardException 23 | { 24 | public CommsException(String msg, Throwable cause) { super(msg, cause) ; } 25 | public CommsException(String msg) { super(msg) ; } 26 | public CommsException(Throwable cause) { super(cause) ; } 27 | public CommsException() { super() ; } 28 | } 29 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/comms/ConnState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.comms; 19 | 20 | import lizard.system.Component ; 21 | 22 | /** The state of a connection - not to be confused with {@link Component}. 23 | * A LifeCycle is for a component - you can have an {@code OK} component with a 24 | * {@code DEAD} connection because the {@code DEAD} connection may be actively 25 | * restarted or rebound by the component. 26 | * 27 | * @see Component 28 | */ 29 | public enum ConnState { 30 | NOT_ACTIVATED, 31 | OK , 32 | RECONNECTING, 33 | DEAD, 34 | CLOSED 35 | } 36 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/comms/Connection.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.comms; 19 | 20 | import lizard.system.Component ; 21 | import org.apache.jena.atlas.lib.Closeable ; 22 | 23 | public interface Connection extends Closeable, Component 24 | { 25 | public ConnState getConnectionStatus() ; 26 | 27 | public String label() ; 28 | 29 | public void setConnectionStatus(ConnState status) ; 30 | 31 | @Override 32 | public void close() ; 33 | } -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/comms/ConnectionMgr.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.comms; 19 | 20 | import org.slf4j.Logger ; 21 | import static org.apache.jena.atlas.logging.FmtLog.* ; 22 | import org.slf4j.LoggerFactory ; 23 | 24 | public class ConnectionMgr 25 | { 26 | private static Logger log = LoggerFactory.getLogger(ConnectionMgr.class) ; 27 | 28 | public static Connection create(String label, String addr, int port) { 29 | return createNetwork(label, addr, port) ; 30 | } 31 | 32 | public static Connection createNetwork(String label, String addr, int port) { 33 | debug(log, "create restartable(%s,%s,%d)", label, addr, port) ; 34 | return null ; 35 | } 36 | 37 | public static Connection createNetworkDirect(String label, String addr, int port) { 38 | debug(log, "create direct(%s,%s,%d)", label, addr, port) ; 39 | return null ; 40 | } 41 | 42 | } -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/comms/thrift/ThriftClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.comms.thrift; 19 | 20 | import lizard.system.Component ; 21 | import lizard.system.ComponentBase ; 22 | import lizard.system.LizardException ; 23 | import org.apache.jena.atlas.logging.FmtLog ; 24 | import org.apache.thrift.TException ; 25 | import org.apache.thrift.protocol.TProtocol ; 26 | import org.apache.thrift.transport.TSocket ; 27 | import org.apache.thrift.transport.TTransport ; 28 | import org.slf4j.Logger ; 29 | import org.slf4j.LoggerFactory ; 30 | 31 | public class ThriftClient extends ComponentBase implements Component { 32 | 33 | private static Logger log = LoggerFactory.getLogger(ThriftClient.class) ; 34 | 35 | private final String host ; 36 | private final int port ; 37 | 38 | private TProtocol protocol ; 39 | 40 | public ThriftClient(String host, int port) { 41 | this.host = host ; 42 | this.port = port; 43 | super.setLabel(host+":"+port) ; 44 | } 45 | 46 | public String getRemoteHost() { return host ; } 47 | public int getRemotePort() { return port ; } 48 | 49 | @Override 50 | public void start() { 51 | if ( super.isRunning() ) { 52 | FmtLog.error(log, "Already started (%s)", getLabel()) ; 53 | return ; 54 | } 55 | FmtLog.debug(log, "Start: %s", getLabel()) ; 56 | startProtocol() ; 57 | super.start() ; 58 | } 59 | 60 | public void startProtocol() { 61 | FmtLog.debug(log, "Start thrift: %s", getLabel()) ; 62 | @SuppressWarnings("resource") 63 | TTransport transport = new TSocket("localhost", port) ; 64 | try { 65 | transport.open() ; 66 | protocol = ThriftLib.protocol(transport) ; 67 | } 68 | catch (TException e) { 69 | transport.close() ; 70 | throw new LizardException(e) ; 71 | } 72 | } 73 | 74 | public void close() { 75 | protocol.getTransport().close() ; 76 | } 77 | 78 | 79 | @Override 80 | public void stop() { 81 | close() ; 82 | super.stop() ; 83 | } 84 | 85 | public TProtocol protocol() { 86 | return protocol ; 87 | } 88 | 89 | @Override 90 | public String toString() { 91 | return getLabel() ; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/comms/thrift/ThriftServer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.comms.thrift; 19 | 20 | import lizard.system.Component ; 21 | import lizard.system.ComponentBase ; 22 | import lizard.system.LizardException ; 23 | import org.apache.jena.atlas.logging.Log ; 24 | import org.apache.jena.dboe.transaction.txn.TransactionalSystem ; 25 | import org.apache.thrift.TException ; 26 | import org.apache.thrift.transport.TServerSocket ; 27 | import org.apache.thrift.transport.TServerTransport ; 28 | import org.slf4j.Logger ; 29 | import org.slf4j.LoggerFactory ; 30 | 31 | /** Base of all servers */ 32 | public class ThriftServer extends ComponentBase implements Component { 33 | private static Logger log = LoggerFactory.getLogger(ThriftServer.class) ; 34 | 35 | private final int port ; 36 | private final TransactionalSystem txnSystem ; 37 | protected final TServerTransport serverTransport ; 38 | 39 | public ThriftServer(TransactionalSystem txnSystem, int port) { 40 | this.txnSystem = txnSystem ; 41 | this.port = port; 42 | try { serverTransport = new TServerSocket(port) ; } 43 | catch (TException e) { throw new LizardException(e) ; } 44 | } 45 | 46 | public int getPort() { return port ; } 47 | public TransactionalSystem getTxnSystem() { return txnSystem ; } 48 | 49 | @Override 50 | public void start() { 51 | if (super.isRunning() ) { 52 | Log.error(this, "Already started: "+getLabel()) ; 53 | return ; 54 | } 55 | super.start() ; 56 | } 57 | 58 | @Override 59 | public void stop() { 60 | super.stop() ; 61 | serverTransport.close() ; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/comms/thrift/ThriftServerAcceptFork.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.comms.thrift; 19 | 20 | import org.apache.thrift.TException ; 21 | import org.apache.thrift.transport.TServerTransport ; 22 | import org.apache.thrift.transport.TTransport ; 23 | import org.slf4j.Logger ; 24 | import org.slf4j.LoggerFactory ; 25 | 26 | /** This is a very raw ThriftServer that accepts connections and passes to a handler. 27 | * The handler is expecting to engage in a long term interation with client, 28 | * possibly stream based. 29 | */ 30 | public class ThriftServerAcceptFork extends ThriftServer { 31 | 32 | /** Handle a new Connection */ 33 | public interface Handler { public void handle(TTransport transport) ; } 34 | 35 | private static Logger log = LoggerFactory.getLogger(ThriftServerAcceptFork.class) ; 36 | 37 | private final Handler handler ; 38 | 39 | public ThriftServerAcceptFork(int port, Handler handler) { 40 | super(null, port) ; 41 | this.handler = handler ; 42 | } 43 | 44 | @Override 45 | public void start() { 46 | super.start() ; 47 | new Thread(() -> server(serverTransport)).start() ; 48 | } 49 | 50 | @Override 51 | public void stop() { 52 | serverTransport.close() ; 53 | super.stop() ; 54 | } 55 | 56 | // OLD 57 | // Better - use one the TServer implementations with custom TProcessor 58 | public void server(TServerTransport serverTransport) { 59 | for ( ;; ) 60 | try { 61 | TTransport transport = serverTransport.accept() ; 62 | log.debug("Connection: "+transport) ; 63 | new Thread(()->handler.handle(transport)).start() ; 64 | } 65 | catch (TException e) {} 66 | catch (Exception e) { e.printStackTrace(); } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/system/Component.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.system; 19 | 20 | 21 | /** Any software component that has a start-stop lifecycle */ 22 | 23 | public interface Component extends LifeCycle { 24 | public String getLabel() ; 25 | public void setLabel(String label) ; 26 | } 27 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/system/ComponentBase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.system; 19 | 20 | /** Implementation of a lifecycle. 21 | * Unfortunately, Java does nor have traits/mixin style multiple inheritance 22 | * so classes have to choose their super class. Sometimes, that means 23 | * using this class by implementation inheritance (i.e. call through). 24 | */ 25 | public class ComponentBase extends LifeCycleBase implements Component { 26 | 27 | private String label ; 28 | 29 | public ComponentBase(String label) { this.label = label ; } 30 | public ComponentBase() { this("Unset") ; } 31 | 32 | @Override 33 | public String getLabel() { 34 | return label ; 35 | } 36 | 37 | @Override 38 | public void setLabel(String label) { 39 | this.label = label ; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/system/ComponentTxn.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.system; 19 | 20 | import org.apache.jena.query.ReadWrite ; 21 | 22 | /** Client-facing component transaction interface */ 23 | public interface ComponentTxn { 24 | /** begin */ 25 | public void begin(long txnId, ReadWrite mode) ; 26 | /** prepare */ 27 | public void prepare() ; 28 | /** commit */ 29 | public void commit() ; 30 | /** abort */ 31 | public void abort() ; 32 | /** end */ 33 | public void end() ; 34 | 35 | } 36 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/system/Distributor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.system; 19 | 20 | 21 | public interface Distributor { 22 | // Currently, just a marker interface. 23 | // Quorum supprt in LzBuilderDataset 24 | } 25 | 26 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/system/LifeCycle.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.system; 19 | 20 | /** Any software component that has a start-stop lifecycle */ 21 | 22 | public interface LifeCycle { 23 | enum Status { UNSTARTED, FAILED, STOPPED, OK } 24 | 25 | /** Start service element. When this call returns the element is usable. */ 26 | public void start() ; 27 | 28 | /** Stop this service - no guarantee that the element has actually stopped 29 | * when this call returns but the eleemnt is to be considered not for use. 30 | */ 31 | public void stop() ; 32 | 33 | /** "Running" means after .start() and before .stop() */ 34 | public boolean isRunning() ; 35 | 36 | /** A service element can go bad */ 37 | public boolean hasFailed() ; 38 | 39 | public void setStatus(Status status) ; 40 | 41 | // public void addLifeCycleListener(LifeCycle.Listener listener); 42 | // 43 | // public void removeLifeCycleListener(LifeCycle.Listener listener); 44 | // 45 | // /** Listener. 46 | // * A listener for Lifecycle events. 47 | // */ 48 | // public interface Listener extends EventListener 49 | // { 50 | // public void lifeCycleStarting(LifeCycle event); 51 | // public void lifeCycleStarted(LifeCycle event); 52 | // public void lifeCycleFailure(LifeCycle event,Throwable cause); 53 | // public void lifeCycleStopping(LifeCycle event); 54 | // public void lifeCycleStopped(LifeCycle event); 55 | // } 56 | } 57 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/system/LifeCycleBase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.system; 19 | 20 | /** Implementation of a lifecycle. 21 | * Unfortunately, Java does nor have traits/mixin style multiple inheritance 22 | * so classes have to choose their super class. Sometimes, that means 23 | * using this class by implementation inheritance (i.e. call through). 24 | */ 25 | public class LifeCycleBase implements LifeCycle { 26 | 27 | private Status status = Status.UNSTARTED ; 28 | 29 | public LifeCycleBase() { } 30 | 31 | @Override 32 | public void start() { 33 | status = Status.OK ; 34 | } 35 | 36 | @Override 37 | public void stop() { 38 | status = Status.STOPPED ; 39 | } 40 | 41 | @Override 42 | public boolean isRunning() { 43 | return status == Status.OK ; 44 | } 45 | 46 | @Override 47 | public boolean hasFailed() { 48 | return status == Status.FAILED ; 49 | } 50 | 51 | @Override 52 | public void setStatus(Status status) { this.status = status ; } 53 | } 54 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/system/LizardConst.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | 19 | package lizard.system; 20 | 21 | import java.util.UUID ; 22 | 23 | import org.apache.jena.shared.uuid.JenaUUID ; 24 | 25 | public class LizardConst { 26 | public static UUID uuidNil = JenaUUID.nil().asUUID() ; 27 | } 28 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/system/LizardException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.system; 19 | 20 | import org.apache.jena.dboe.DBOpEnvException ; 21 | 22 | public class LizardException extends DBOpEnvException 23 | { 24 | public LizardException(String msg, Throwable cause) { super(msg, cause) ; } 25 | public LizardException(String msg) { super(msg) ; } 26 | public LizardException(Throwable cause) { super(cause) ; } 27 | public LizardException() { super() ; } 28 | } 29 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/system/LzLib.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.system; 19 | 20 | import java.nio.file.Files ; 21 | import java.nio.file.Path ; 22 | import java.nio.file.Paths ; 23 | import java.util.Arrays ; 24 | import java.util.List ; 25 | 26 | import org.apache.jena.riot.RDFDataMgr ; 27 | import org.apache.jena.riot.RiotNotFoundException ; 28 | 29 | import org.apache.jena.rdf.model.Model ; 30 | import org.apache.jena.rdf.model.ModelFactory ; 31 | 32 | /** General utilities for Lizard. 33 | * Not Lizard specific and might usefully migrate to Jena proper. 34 | */ 35 | 36 | public class LzLib { 37 | public static Model readAll(String ... files) { 38 | return readAll(Arrays.asList(files)) ; 39 | } 40 | 41 | public static Model readAll(List files) { 42 | return readAll(ModelFactory.createDefaultModel(), files) ; 43 | } 44 | 45 | public static Model readAll(Model model, List files) { 46 | for ( String fn : files ) { 47 | Path p = Paths.get(fn) ; 48 | if ( ! Files.exists(p) ) { 49 | throw new RiotNotFoundException("File not found: "+fn) ; 50 | } 51 | } 52 | 53 | for ( String fn : files ) { 54 | RDFDataMgr.read(model, fn); 55 | } 56 | return model ; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/system/LzLog.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.system; 19 | 20 | import org.slf4j.Logger ; 21 | import org.slf4j.LoggerFactory ; 22 | 23 | /** The system loggers */ 24 | public class LzLog { 25 | public static final Logger logConf = LoggerFactory.getLogger("lizard.lz-config") ; 26 | public static final Logger logSystem = LoggerFactory.getLogger("lizard.lz-system") ; 27 | /* Base name for transaction logging */ 28 | public static final String logTxnBase = "lizard.lz-txn" ; 29 | 30 | } 31 | 32 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/system/LzTxnId.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.system; 19 | 20 | import java.util.UUID ; 21 | import java.util.concurrent.atomic.AtomicLong ; 22 | 23 | import org.apache.jena.atlas.lib.BitsLong ; 24 | 25 | public class LzTxnId { 26 | public static final LzTxnId txnNil = new LzTxnId(LizardConst.uuidNil, 0) ; 27 | 28 | static final private UUID here = UUID.randomUUID() ; 29 | static final private AtomicLong counter = new AtomicLong(0) ; 30 | 31 | private final UUID uuid ; 32 | private final long count ; 33 | 34 | public static LzTxnId alloc() { 35 | long x = counter.incrementAndGet() ; 36 | return new LzTxnId(here, x) ; 37 | } 38 | 39 | private LzTxnId(UUID uuid, long x) { 40 | this.uuid = uuid ; 41 | this.count = x ; 42 | } 43 | 44 | public long generation() { 45 | if ( this.equals(txnNil) ) 46 | return 0L ; 47 | long x = uuid.getLeastSignificantBits() ; 48 | x = BitsLong.clear(x, 32, 64) ; 49 | int y = (int)count ; 50 | x = x | count ; 51 | return x ; 52 | } 53 | 54 | @Override 55 | public int hashCode() { 56 | final int prime = 31 ; 57 | int result = 1 ; 58 | result = prime * result + (int)(count ^ (count >>> 32)) ; 59 | result = prime * result + ((uuid == null) ? 0 : uuid.hashCode()) ; 60 | return result ; 61 | } 62 | 63 | @Override 64 | public boolean equals(Object obj) { 65 | if ( this == obj ) 66 | return true ; 67 | if ( obj == null ) 68 | return false ; 69 | if ( getClass() != obj.getClass() ) 70 | return false ; 71 | LzTxnId other = (LzTxnId)obj ; 72 | if ( count != other.count ) 73 | return false ; 74 | if ( uuid == null ) { 75 | if ( other.uuid != null ) 76 | return false ; 77 | } else if ( !uuid.equals(other.uuid) ) 78 | return false ; 79 | return true ; 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/lizard/system/RemoteControl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.system; 19 | 20 | /** Any software element that can be pinged */ 21 | public interface RemoteControl { 22 | public void ping() ; 23 | public void remoteStop() ; 24 | 25 | } 26 | -------------------------------------------------------------------------------- /lizard-base/src/main/java/migrate/TupleIndexEmpty.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package migrate; 19 | 20 | import java.util.Collection ; 21 | import java.util.Iterator ; 22 | 23 | import org.apache.jena.atlas.iterator.Iter ; 24 | import org.apache.jena.atlas.lib.tuple.Tuple ; 25 | import org.apache.jena.atlas.lib.tuple.TupleMap ; 26 | import org.apache.jena.tdb2.store.NodeId ; 27 | import org.apache.jena.tdb2.store.tupletable.TupleIndexBase ; 28 | 29 | /** An immutably empty tuple index - it is a sink (add provided but does nothing) */ 30 | public class TupleIndexEmpty extends TupleIndexBase { 31 | 32 | public TupleIndexEmpty(TupleMap mapping, String name) { 33 | super(mapping.length(), mapping, name) ; 34 | } 35 | 36 | @Override 37 | public Iterator> all() { 38 | return Iter.nullIterator() ; 39 | } 40 | 41 | @Override 42 | public long size() { 43 | return 0 ; 44 | } 45 | 46 | @Override 47 | public boolean isEmpty() { 48 | return true ; 49 | } 50 | 51 | @Override 52 | public void clear() {} 53 | 54 | @Override 55 | public void sync() {} 56 | 57 | @Override 58 | public void close() {} 59 | 60 | @Override 61 | protected void performAdd(Tuple tuple) { 62 | } 63 | 64 | @Override 65 | protected void performDelete(Tuple tuple) { 66 | } 67 | 68 | @Override 69 | public void addAll(Collection> tuples) {} 70 | 71 | @Override 72 | public void deleteAll(Collection> tuples) {} 73 | 74 | @Override 75 | protected Iterator> performFind(Tuple tuple) { 76 | return Iter.nullIterator() ; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /lizard-base/src/test/java/lizard/comms/CommsTestLib.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.comms; 19 | 20 | public class CommsTestLib { 21 | 22 | } 23 | -------------------------------------------------------------------------------- /lizard-base/src/test/java/lizard/comms/TS_LzComms.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.comms; 19 | 20 | import org.junit.runner.RunWith ; 21 | import org.junit.runners.Suite ; 22 | 23 | @RunWith(Suite.class) 24 | @Suite.SuiteClasses( { 25 | TestTLZ.class, 26 | TestLzClientServer.class 27 | }) 28 | 29 | public class TS_LzComms { } 30 | -------------------------------------------------------------------------------- /lizard-base/src/test/java/lizard/comms/TestLzClientServer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.comms; 19 | 20 | import java.util.concurrent.atomic.AtomicInteger ; 21 | import java.util.concurrent.atomic.AtomicReference ; 22 | 23 | import lizard.api.TLZ.TLZ_Ping ; 24 | import lizard.comms.thrift.ThriftClient ; 25 | import lizard.comms.thrift.ThriftLib ; 26 | import lizard.comms.thrift.ThriftServerAcceptFork ; 27 | import lizard.comms.thrift.ThriftServerAcceptFork.Handler ; 28 | import lizard.test.LzBaseTest ; 29 | import org.apache.thrift.TException ; 30 | import org.apache.thrift.protocol.TProtocol ; 31 | import org.apache.thrift.transport.TMemoryBuffer ; 32 | import org.apache.thrift.transport.TTransport ; 33 | import org.junit.Test ; 34 | 35 | /** 36 | * Test Client/Server (not using Thrift services) 37 | */ 38 | public class TestLzClientServer extends LzBaseTest { 39 | 40 | public static final int test_port = 12345 ; 41 | public static AtomicInteger handlerErrors = new AtomicInteger(0) ; 42 | public static AtomicInteger handlerWarnings = new AtomicInteger(0) ; 43 | public static AtomicReference handlerException = new AtomicReference<>(null) ; 44 | 45 | @Test public void wire_01() throws Exception { 46 | Handler handler = new Handler() { 47 | @Override 48 | public void handle(TTransport transport) { 49 | try { 50 | TProtocol proto = ThriftLib.protocol(transport) ; 51 | TLZ_Ping p = new TLZ_Ping() ; 52 | p.read(proto); 53 | p.write(proto); 54 | transport.flush(); 55 | } catch (TException ex) { 56 | handlerException.set(ex); 57 | } 58 | } 59 | }; 60 | ThriftServerAcceptFork server = new ThriftServerAcceptFork(test_port, handler) ; 61 | 62 | server.start() ; 63 | 64 | ThriftClient c = new ThriftClient("localhost", test_port) ; 65 | c.start() ; 66 | TProtocol proto = c.protocol() ; 67 | TLZ_Ping p = new TLZ_Ping(1324) ; 68 | p.write(proto); 69 | proto.getTransport().flush() ; 70 | TLZ_Ping p2 = new TLZ_Ping() ; 71 | p2.read(proto); 72 | assertEquals(p.getMarker(), p2.getMarker()) ; 73 | assertNull(handlerException.get()) ; 74 | c.stop(); 75 | server.stop() ; 76 | 77 | } 78 | 79 | private static TProtocol protocol() { 80 | TTransport transport = new TMemoryBuffer(1024) ; 81 | TProtocol proto = ThriftLib.protocol(transport) ; 82 | return proto ; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /lizard-base/src/test/java/lizard/test/LzBaseTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.test; 19 | 20 | import org.apache.jena.atlas.junit.BaseTest ; 21 | 22 | public class LzBaseTest extends BaseTest { 23 | 24 | } 25 | -------------------------------------------------------------------------------- /lizard-cluster/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 23 | 4.0.0 24 | 25 | 26 | org.seaborne.lizard 27 | lizard 28 | .. 29 | 0.0.0-SNAPSHOT 30 | 31 | 32 | lizard-cluster 33 | 34 | jar 35 | 36 | 37 | 38 | 39 | http://www.apache.org/licenses/LICENSE-2.0 40 | 41 | 42 | 43 | 44 | 45 | 46 | org.seaborne.lizard 47 | lizard-node-server 48 | 0.0.0-SNAPSHOT 49 | 50 | 51 | org.seaborne.lizard 52 | lizard-index-server 53 | 0.0.0-SNAPSHOT 54 | 55 | 56 | org.seaborne.lizard 57 | lizard-query-server 58 | 0.0.0-SNAPSHOT 59 | 60 | 61 | org.apache.jena 62 | jena-fuseki-embedded 63 | 64 | 65 | org.yaml 66 | snakeyaml 67 | 68 | 69 | org.apache.curator 70 | curator-test 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/build/LzBuildZk.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.build; 19 | 20 | import lizard.conf.Config ; 21 | import lizard.system.LizardException ; 22 | import org.apache.curator.test.TestingServer ; 23 | import org.apache.jena.atlas.logging.FmtLog ; 24 | import org.apache.jena.dboe.migrate.L ; 25 | import org.apache.zookeeper.server.ServerConfig ; 26 | import org.apache.zookeeper.server.ZooKeeperServerMain ; 27 | import org.slf4j.Logger ; 28 | 29 | public class LzBuildZk { 30 | private static Logger logConf = Config.logConf ; 31 | 32 | static TestingServer zkTestServer; 33 | static String zkConnectString ; 34 | 35 | /** Run a full Zookeepr here */ 36 | public static void zookeeper(int port, String zkConfDir) { 37 | FmtLog.info(logConf, "Start Zookeeper %s : %d", zkConfDir, port) ; 38 | ServerConfig config = new ServerConfig(); 39 | config.parse(new String[] {Integer.toString(port), zkConfDir}) ; 40 | ZooKeeperServerMain zk = new ZooKeeperServerMain(); 41 | L.async(()-> { 42 | try { zk.runFromConfig(config) ; } 43 | catch (Exception e) { FmtLog.warn(logConf, "Failed to run zookeeper: "+e.getMessage(), e); } 44 | }) ; 45 | } 46 | 47 | /** Run an ephemeral zookeeper */ 48 | public static void zookeeperSimple(int port) { 49 | FmtLog.info(logConf, "Start Zookeeper (ephemeral): %d", port) ; 50 | try { zkTestServer = new TestingServer(port) ; } 51 | catch (Exception e) { throw new LizardException(e) ; } 52 | } 53 | } 54 | 55 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/build/LzBuilderIndexServer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.build; 19 | 20 | import lizard.cluster.Platform ; 21 | import lizard.conf.ConfCluster ; 22 | import lizard.conf.ConfIndexElement ; 23 | import lizard.conf.Config ; 24 | import lizard.conf.NetHost ; 25 | import lizard.index.TServerIndex ; 26 | import org.apache.jena.atlas.logging.FmtLog ; 27 | import org.apache.jena.dboe.base.file.Location ; 28 | import org.apache.jena.dboe.transaction.txn.TransactionCoordinator ; 29 | import org.apache.jena.dboe.transaction.txn.TransactionalBase ; 30 | import org.apache.jena.dboe.transaction.txn.TransactionalSystem ; 31 | import org.apache.jena.tdb2.setup.StoreParams ; 32 | import org.apache.jena.tdb2.setup.TDBBuilder ; 33 | import org.apache.jena.tdb2.store.tupletable.TupleIndex ; 34 | import org.slf4j.Logger ; 35 | 36 | public class LzBuilderIndexServer { 37 | private static Logger logConf = Config.logConf ; 38 | 39 | public static void build(Platform platform, Location baseLocation, StoreParams params, ConfCluster confCluster, NetHost here) { 40 | confCluster.eltsIndex.stream() 41 | .filter(x -> x.addr.sameHost(here)) 42 | .forEach(x -> { 43 | buildIndexServer(platform, baseLocation, params, x) ; 44 | }) ; 45 | } 46 | 47 | public static TServerIndex buildIndexServer(Platform platform, Location baseLocation, StoreParams params, ConfIndexElement x) { 48 | Location location = baseLocation.getSubLocation(x.data) ; 49 | int port = x.addr.getPort() ; 50 | String data = x.data ; 51 | FmtLog.info(logConf, "buildIndexServer[%s]: port=%s %s", x.conf.indexOrder, port, location) ; 52 | TransactionCoordinator coord = TDBBuilder.buildTransactionCoordinator(location) ; 53 | TransactionalSystem txnSystem = new TransactionalBase(x.toString(), coord) ; 54 | String primary = TDBBuilder.choosePrimaryForIndex(params, x.conf.indexOrder) ; 55 | TDBBuilder builder = TDBBuilder.create(coord, location, params) ; 56 | TupleIndex tupleIndex = builder.buildTupleIndex(primary, x.conf.indexOrder, x.name) ; 57 | TServerIndex serverindex = TServerIndex.create(txnSystem, port, tupleIndex) ; 58 | platform.add(serverindex) ; 59 | return serverindex ; 60 | } 61 | } 62 | 63 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/build/LzBuilderNodeServer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.build; 19 | 20 | import lizard.cluster.Platform ; 21 | import lizard.conf.ConfCluster ; 22 | import lizard.conf.ConfNodeTableElement ; 23 | import lizard.conf.Config ; 24 | import lizard.conf.NetHost ; 25 | import lizard.node.TServerNode ; 26 | import org.apache.jena.atlas.logging.FmtLog ; 27 | import org.apache.jena.dboe.base.file.Location ; 28 | import org.apache.jena.dboe.transaction.txn.TransactionCoordinator ; 29 | import org.apache.jena.dboe.transaction.txn.TransactionalBase ; 30 | import org.apache.jena.dboe.transaction.txn.TransactionalSystem ; 31 | import org.apache.jena.tdb2.setup.StoreParams ; 32 | import org.apache.jena.tdb2.setup.TDBBuilder ; 33 | import org.apache.jena.tdb2.store.nodetable.NodeTable ; 34 | import org.slf4j.Logger ; 35 | 36 | public class LzBuilderNodeServer { 37 | private static Logger logConf = Config.logConf ; 38 | 39 | public static void build(Platform platform, Location baseLocation, StoreParams params, ConfCluster confCluster, NetHost here) { 40 | confCluster.eltsNodeTable.stream() 41 | .filter(x -> x.addr.sameHost(here)) 42 | .forEach(x -> { 43 | buildNodeServer(platform, baseLocation, params, x) ; 44 | }) ; 45 | } 46 | 47 | public static TServerNode buildNodeServer(Platform platform, Location baseLocation, StoreParams params, ConfNodeTableElement x) { 48 | Location location = baseLocation.getSubLocation(x.data) ; 49 | int port = x.addr.getPort() ; 50 | FmtLog.info(logConf, "buildNodeServer: port=%s %s", port, location) ; 51 | TransactionCoordinator coord = TDBBuilder.buildTransactionCoordinator(location) ; 52 | TDBBuilder builder = TDBBuilder.create(coord, location, params) ; 53 | NodeTable nt = builder.buildNodeTable(params.getNodeTableBaseName()) ; 54 | TransactionalSystem txnSystem = new TransactionalBase(x.toString(), coord) ; 55 | TServerNode serverNode = TServerNode.create(txnSystem, port, nt) ; 56 | platform.add(serverNode) ; 57 | return serverNode ; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/cluster/ClusterCtl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.cluster; 19 | 20 | import lizard.system.LizardException ; 21 | 22 | /** Operations on the cluster. 23 | * 24 | * There is one cluster configuration reprenting the current cluster 25 | * and this class provides operations on the current cluster via a global 26 | * singleton {@link Cluster}. 27 | * 28 | * There may be other "cluster" as changes are prepared but only one 29 | * is active for query and update, as defined by this class. 30 | */ 31 | public class ClusterCtl { 32 | private static Cluster current = null ; 33 | 34 | public final static String namespace = "/lizard" ; 35 | public final static String members = "/lizard/members" ; 36 | 37 | public static Cluster get() { 38 | if ( current == null ) 39 | throw new LizardException("No cluster set") ; 40 | return current ; 41 | } 42 | 43 | public static void set(Cluster cluster) { 44 | current = cluster ; 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/cluster/Platform.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.cluster; 19 | 20 | import java.util.LinkedHashSet ; 21 | import java.util.Set ; 22 | 23 | import lizard.system.Component ; 24 | import lizard.system.ComponentBase ; 25 | import lizard.system.LifeCycle ; 26 | 27 | import org.slf4j.Logger ; 28 | import org.slf4j.LoggerFactory ; 29 | 30 | /** Set of server components on one machine (not query engines) */ 31 | public class Platform implements LifeCycle { 32 | 33 | private static Logger log = LoggerFactory.getLogger(Platform.class) ; 34 | 35 | private Set components = new LinkedHashSet<>() ; 36 | private ComponentBase lifecycle = new ComponentBase() ; 37 | 38 | public Platform() { } 39 | 40 | @Override 41 | public void start() { 42 | components.forEach(s->s.start()) ; 43 | lifecycle.start() ; 44 | } 45 | 46 | @Override 47 | public void stop() { 48 | components.forEach(s->s.stop()) ; 49 | lifecycle.stop() ; 50 | } 51 | 52 | @Override 53 | public boolean isRunning() { return lifecycle.isRunning() ; } 54 | 55 | @Override 56 | public boolean hasFailed() { return lifecycle.hasFailed() ; } 57 | 58 | @Override 59 | public void setStatus(Status status) { 60 | for ( Component s : components ) 61 | s.setStatus(status); 62 | lifecycle.setStatus(status) ; 63 | } 64 | 65 | public void add(Component server) { 66 | components.add(server) ; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/ConfDataset.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf; 19 | 20 | import java.util.ArrayList; 21 | import java.util.Arrays ; 22 | import java.util.List ; 23 | 24 | public class ConfDataset { 25 | public final List indexes = new ArrayList<>(); 26 | public ConfNodeTable nodeTable ; 27 | 28 | public ConfDataset(ConfNodeTable nodeTable, ConfIndex... indexes) { 29 | this.nodeTable = nodeTable ; 30 | this.indexes.addAll(Arrays.asList(indexes)) ; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/ConfDeploy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf; 19 | 20 | import java.util.ArrayList; 21 | import java.util.List ; 22 | 23 | // Here : cf ConfCluster which is cluster wide. 24 | public class ConfDeploy { 25 | public ConfCluster confCluster = null ; 26 | public ConfZookeeper localZk = null ; 27 | public List ntReplicas = new ArrayList<>(); 28 | public List idxReplicas = new ArrayList<>(); 29 | public ConfDataset confDataset = null ; 30 | } 31 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/ConfElement.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf; 19 | 20 | public class ConfElement { 21 | public final String name ; 22 | public final String data ; 23 | public final X conf ; 24 | public final VNodeAddr addr ; 25 | public ConfElement(String name, String data, X conf, VNodeAddr addr) { 26 | this.name = name ; 27 | this.data = data ; 28 | this.conf = conf ; 29 | this.addr = addr ; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/ConfIndex.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf; 19 | 20 | import org.apache.jena.atlas.io.IndentedWriter ; 21 | import org.apache.jena.atlas.lib.tuple.TupleMap ; 22 | 23 | public class ConfIndex { 24 | public final String indexOrder ; 25 | public final TupleMap tmap ; 26 | public final int readQuorum ; 27 | public final int writeQuorum ; 28 | 29 | public ConfIndex(TupleMap tmap, String indexOrder, int readQuorum, int writeQuorum) { 30 | super() ; 31 | this.indexOrder = indexOrder ; 32 | this.readQuorum = readQuorum ; 33 | this.writeQuorum = writeQuorum ; 34 | this.tmap = tmap ; 35 | } 36 | 37 | public void print(IndentedWriter out) { 38 | out.print(indexOrder) ; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/ConfIndexElement.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf; 19 | 20 | public class ConfIndexElement extends ConfElement { 21 | public ConfIndexElement(String name, String data, ConfIndex index, VNodeAddr addr) { 22 | super(name, data, index, addr) ; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/ConfNodeTable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf; 19 | 20 | import org.apache.jena.atlas.io.IndentedWriter ; 21 | 22 | public class ConfNodeTable { 23 | public final int readQuorum ; 24 | public final int writeQuorum ; 25 | public ConfNodeTable(int readQuorum, int writeQuorum) { 26 | this.readQuorum = readQuorum ; 27 | this.writeQuorum = writeQuorum ; 28 | } 29 | public void print(IndentedWriter out) { 30 | out.print("nodetable"); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/ConfNodeTableElement.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf; 19 | 20 | public class ConfNodeTableElement extends ConfElement { 21 | public ConfNodeTableElement(String name, String data, ConfNodeTable nodetable, VNodeAddr addr) { 22 | super(name, data, nodetable, addr) ; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/ConfZookeeper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf; 19 | 20 | import org.apache.jena.atlas.io.IndentedWriter ; 21 | 22 | /** Sever reference */ 23 | public class ConfZookeeper { 24 | public final String hostname ; 25 | public final int port ; 26 | 27 | public static ConfZookeeper create(String hostname, int port) { 28 | return new ConfZookeeper(hostname, port) ; 29 | } 30 | 31 | private ConfZookeeper(String hostname, int port) { 32 | this.hostname = hostname ; 33 | this.port = port ; 34 | } 35 | 36 | public void print(IndentedWriter out) { 37 | out.print("zookeeper://"+connectString()) ; 38 | } 39 | 40 | public String connectString() { 41 | return hostname+":"+port ; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/Config.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf; 19 | 20 | import org.apache.jena.atlas.lib.StrUtils ; 21 | import org.apache.jena.riot.system.PrefixMap ; 22 | import org.apache.jena.riot.system.PrefixMapFactory ; 23 | import org.slf4j.Logger ; 24 | import org.slf4j.LoggerFactory ; 25 | 26 | public class Config { 27 | public static Logger logConf = LoggerFactory.getLogger("lizard.Config") ; 28 | 29 | public static final String prefixes = StrUtils.strjoinNL 30 | ("PREFIX : ", 31 | "PREFIX lizard: ", 32 | "PREFIX list: " , 33 | "PREFIX xsd: ", 34 | "PREFIX rdf: ", 35 | "PREFIX rdfs: ", 36 | "") ; 37 | 38 | public static PrefixMap confPrefixes = PrefixMapFactory.create() ; 39 | static { 40 | confPrefixes.add("", "urn:lizard:") ; 41 | confPrefixes.add("lizard", "urn:lizard:ns#") ; 42 | confPrefixes.add("list", "http://jena.apache.org/ARQ/list#") ; 43 | confPrefixes.add("xsd", "http://www.w3.org/2001/XMLSchema#") ; 44 | confPrefixes.add("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#") ; 45 | confPrefixes.add("rdfs", "http://www.w3.org/2000/01/rdf-schema#") ; 46 | } 47 | } 48 | 49 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/Endpoint.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf; 19 | 20 | import java.util.Objects ; 21 | 22 | /** An endpoint a name and a port. 23 | * 24 | * The name might be a virtual vNode name or a DNS address. 25 | * 26 | * @see NetAddr 27 | * @see VNodeAddr 28 | * 29 | */ 30 | public class Endpoint { 31 | 32 | public static Endpoint create(String dnsName, int port) { 33 | return new Endpoint(dnsName, port) ; 34 | } 35 | 36 | public static Endpoint createHere(int port) { 37 | return new Endpoint("localhost", port) ; 38 | } 39 | 40 | private final String name ; 41 | private final int port ; 42 | public Endpoint(String name, int port) { 43 | Objects.requireNonNull(name, "Hostname is null") ; 44 | this.name = name ; 45 | this.port = port ; 46 | } 47 | @Override 48 | public int hashCode() { 49 | final int prime = 31 ; 50 | int result = 1 ; 51 | result = prime * result + ((getName() == null) ? 0 : getName().hashCode()) ; 52 | result = prime * result + getPort() ; 53 | return result ; 54 | } 55 | @Override 56 | public boolean equals(Object obj) { 57 | if ( this == obj ) 58 | return true ; 59 | if ( obj == null ) 60 | return false ; 61 | if ( getClass() != obj.getClass() ) 62 | return false ; 63 | Endpoint other = (Endpoint)obj ; 64 | if ( getName() == null ) { 65 | if ( other.getName() != null ) 66 | return false ; 67 | } else if ( !getName().equals(other.getName()) ) 68 | return false ; 69 | if ( getPort() != other.getPort() ) 70 | return false ; 71 | return true ; 72 | } 73 | 74 | public boolean sameHost(NetHost here) { 75 | return sameHost(here.hostname) ; 76 | } 77 | 78 | public boolean sameHost(String here) { 79 | return Objects.equals(getName(), here) ; 80 | } 81 | 82 | public String getName() { 83 | return name; 84 | } 85 | 86 | public int getPort() { 87 | return port; 88 | } 89 | 90 | @Override public String toString() { 91 | String x = getName() ; 92 | if ( getPort() > 0 ) 93 | x = x+":"+getPort() ; 94 | return x ; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/LzConfigurationException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf; 19 | 20 | import lizard.system.LizardException ; 21 | 22 | public class LzConfigurationException extends LizardException { 23 | public LzConfigurationException(String msg, Throwable cause) { super(msg, cause) ; } 24 | public LzConfigurationException(String msg) { super(msg) ; } 25 | public LzConfigurationException(Throwable cause) { super(cause) ; } 26 | public LzConfigurationException() { super() ; } 27 | } 28 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/NetAddr.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf; 19 | 20 | 21 | public class NetAddr extends Endpoint { 22 | public static NetAddr create(String dnsName, int port) { 23 | return new NetAddr(dnsName, port) ; 24 | } 25 | 26 | public static NetAddr createHere(int port) { 27 | return new NetAddr("localhost", port) ; 28 | } 29 | 30 | public NetAddr(String hostname, int port) { 31 | super(hostname, port) ; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/NetHost.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf; 19 | 20 | // Typed string. 21 | public class NetHost { 22 | public final String hostname ; 23 | 24 | public static NetHost create(String hostname) { 25 | return new NetHost(hostname) ; 26 | } 27 | 28 | private NetHost(String hostname) { 29 | this.hostname = hostname ; 30 | } 31 | 32 | @Override 33 | public String toString() { 34 | return "NetHost:"+hostname ; 35 | } 36 | 37 | @Override 38 | public int hashCode() { 39 | final int prime = 31 ; 40 | int result = 1 ; 41 | result = prime * result + ((hostname == null) ? 0 : hostname.hashCode()) ; 42 | return result ; 43 | } 44 | 45 | @Override 46 | public boolean equals(Object obj) { 47 | if ( this == obj ) 48 | return true ; 49 | if ( obj == null ) 50 | return false ; 51 | if ( getClass() != obj.getClass() ) 52 | return false ; 53 | NetHost other = (NetHost)obj ; 54 | if ( hostname == null ) { 55 | if ( other.hostname != null ) 56 | return false ; 57 | } else if ( !hostname.equals(other.hostname) ) 58 | return false ; 59 | return true ; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/VNode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf; 19 | 20 | /** 21 | * Configuration representation of one vNode - that is, a JVM instance. VNodes 22 | * host index and node table servers. There is a symbolic name, address mapping 23 | * and an port for contact (e.g. admin). The contact port is not the port 24 | * for service access. 25 | *

26 | * Many vNodes can be on one machine - it is the responsibility of the 27 | * configuration to avoid clashes of ports. 28 | */ 29 | public class VNode { 30 | 31 | public final String vname ; 32 | private final NetAddr endpoint ; 33 | public final String localFileRoot ; 34 | 35 | public VNode(String vname, NetAddr endpoint, String loc) { 36 | this.vname = vname ; 37 | this.endpoint = endpoint ; 38 | this.localFileRoot = loc ; 39 | } 40 | 41 | public String getDNSname() { 42 | return endpoint.getName() ; 43 | } 44 | 45 | public NetAddr getAdminEndpoint() { 46 | return endpoint ; 47 | } 48 | 49 | @Override 50 | public String toString() { 51 | return vname+"->"+endpoint ; 52 | } 53 | } 54 | 55 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/VNodeAddr.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf; 19 | 20 | import org.apache.jena.atlas.logging.FmtLog ; 21 | 22 | /** 23 | * Address on a vNode service. This is the vNode abstract name and a port number. 24 | *

25 | * Many vNodes can be on one machine - it is the responsibility of the 26 | * configuration to avoid clashes of ports. 27 | */ 28 | public class VNodeAddr extends Endpoint { 29 | public static VNodeAddr create(String dnsName, int port) { 30 | return new VNodeAddr(dnsName, port) ; 31 | } 32 | 33 | public VNodeAddr(String name, int port) { 34 | super(name, port) ; 35 | } 36 | 37 | public NetAddr placement(VNodeLayout placements, int port) { 38 | VNode vNode = placements.get(getName()) ; 39 | if ( vNode == null ) { 40 | FmtLog.warn(this.getClass(), "No placement for %s\n", getName()) ; 41 | return null ; 42 | } 43 | return NetAddr.create(vNode.getDNSname(), port) ; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/VNodeLayout.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf; 19 | 20 | import java.util.HashMap ; 21 | 22 | public class VNodeLayout extends HashMap { 23 | } 24 | 25 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/assembler/AssemblerYaml.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf.assembler; 19 | 20 | import lizard.build.LzDeploy ; 21 | import lizard.conf.ConfCluster ; 22 | import lizard.conf.parsers.LzConfParserYAML ; 23 | import lizard.query.LizardQuery ; 24 | import lizard.sys.Lizard ; 25 | 26 | import org.apache.jena.assembler.Assembler ; 27 | import org.apache.jena.assembler.Mode ; 28 | import org.apache.jena.assembler.exceptions.AssemblerException ; 29 | import org.apache.jena.atlas.lib.FileOps ; 30 | import org.apache.jena.query.Dataset ; 31 | import org.apache.jena.rdf.model.Property ; 32 | import org.apache.jena.rdf.model.Resource ; 33 | import org.apache.jena.sparql.core.assembler.DatasetAssembler ; 34 | 35 | public class AssemblerYaml extends DatasetAssembler { 36 | @Override 37 | public Dataset createDataset(Assembler a, Resource root, Mode mode) 38 | { 39 | LizardQuery.init() ; 40 | return make(root) ; 41 | } 42 | 43 | 44 | static Dataset make(Resource root) { 45 | // YAML configuration 46 | Lizard.init(); 47 | Property pConfiguration = root.getModel().createProperty("urn:lizard:", "configuration") ; 48 | Property pLayout = root.getModel().createProperty("urn:lizard:", "configuration") ; 49 | 50 | if ( ! root.hasProperty(pConfiguration) ) 51 | throw new AssemblerException(root, "Missing the Lizard config file via "+pConfiguration) ; 52 | if ( ! root.hasProperty(pLayout) ) 53 | throw new AssemblerException(root, "Missing the Lizard config file via "+pLayout) ; 54 | 55 | String confFile = root.getProperty(pConfiguration).getString() ; 56 | String layoutFile = root.getProperty(pLayout).getString() ; 57 | 58 | if ( ! FileOps.exists(confFile) ) 59 | throw new AssemblerException(root, "No such file: "+confFile) ; 60 | if ( ! FileOps.exists(layoutFile) ) 61 | throw new AssemblerException(root, "No such file: "+confFile) ; 62 | 63 | ConfCluster conf = LzConfParserYAML.parseConfFile(confFile,layoutFile) ; 64 | String here = "here" ; 65 | Dataset ds = LzDeploy.deployDataset(conf, here); 66 | return ds ; 67 | } 68 | } 69 | 70 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/assembler/LizardAssembler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf.assembler; 19 | 20 | import lizard.build.LzBuilderDataset ; 21 | import lizard.conf.ConfCluster ; 22 | import lizard.conf.ConfDataset ; 23 | import lizard.conf.parsers.LzConfParserRDF ; 24 | import lizard.query.LizardQuery ; 25 | import lizard.query.LzDataset ; 26 | import lizard.sys.Lizard ; 27 | import org.apache.jena.assembler.Assembler ; 28 | import org.apache.jena.assembler.Mode ; 29 | import org.apache.jena.dboe.base.file.Location ; 30 | import org.apache.jena.query.Dataset ; 31 | import org.apache.jena.rdf.model.Resource ; 32 | import org.apache.jena.sparql.core.assembler.AssemblerUtils ; 33 | import org.apache.jena.sparql.core.assembler.DatasetAssembler ; 34 | import org.slf4j.Logger ; 35 | import org.slf4j.LoggerFactory ; 36 | 37 | /** Create a Lizard dataset */ 38 | public class LizardAssembler extends DatasetAssembler { 39 | // RDF version 40 | private static Logger log = LoggerFactory.getLogger(LizardAssembler.class) ; 41 | 42 | @Override 43 | public Dataset createDataset(Assembler a, Resource root, Mode mode) 44 | { 45 | LizardQuery.init() ; 46 | return make(root) ; 47 | } 48 | 49 | /* Example: 50 | rdf:type lizard:Dataset ; 51 | ... 52 | */ 53 | 54 | static Dataset make(Resource root) { 55 | Lizard.init(); 56 | ConfCluster confCluster = LzConfParserRDF.parseConfFile(root.getModel()) ; 57 | ConfDataset confDataset = confCluster.dataset ; 58 | Location baseLocation = null ; 59 | // if ( confCluster.fileroot == null ) 60 | // baseLocation = Location.mem() ; 61 | // else 62 | // baseLocation = Location.create(confCluster.fileroot) ; 63 | 64 | // XXX $$TEMP$$ 65 | baseLocation = Location.mem() ; 66 | 67 | Location locationQueryServer = baseLocation.getSubLocation("query") ; 68 | log.warn("No 'same vnode' support for datasets yet"); 69 | LzDataset lz = LzBuilderDataset.build(confCluster, locationQueryServer, "$NOWHERE$") ; 70 | Dataset ds = LzBuilderDataset.dataset(lz) ; 71 | AssemblerUtils.setContext(root, ds.getContext()); 72 | return ds ; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/assembler/VocabLizard.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf.assembler; 19 | 20 | import org.apache.jena.assembler.Assembler ; 21 | import org.apache.jena.assembler.assemblers.AssemblerGroup ; 22 | import org.apache.jena.rdf.model.Resource ; 23 | import org.apache.jena.tdb2.assembler.Vocab ; 24 | 25 | public class VocabLizard { 26 | public final static String namespace = "urn:lizard:ns#" ; 27 | private static final String NS = namespace ; 28 | 29 | public static String getURI() { return NS ; } 30 | 31 | // Types 32 | public static final Resource lzDataset = Vocab.type(NS, "Dataset") ; 33 | 34 | private static boolean initialized = false ; 35 | static synchronized public void init() 36 | { 37 | if ( initialized ) 38 | return ; 39 | registerWith(Assembler.general) ; 40 | initialized = true ; 41 | } 42 | 43 | static void registerWith(AssemblerGroup g) 44 | { 45 | // Wire in the extension assemblers (extensions relative to the Jena assembler framework) 46 | // Domain and range for properties. 47 | // Separated and use ja:imports 48 | assemblerClass(g, lzDataset, new LizardAssembler()) ; 49 | } 50 | 51 | public static void assemblerClass(AssemblerGroup group, Resource r, Assembler a) 52 | { 53 | if ( group == null ) 54 | group = Assembler.general ; 55 | group.implementWith(r, a) ; 56 | //assemblerAssertions.add(r, RDFS.subClassOf, JA.Object) ; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/conf/parsers/LzDatasetDesc.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.conf.parsers ; 19 | 20 | import java.util.List ; 21 | 22 | import org.apache.jena.rdf.model.Resource ; 23 | 24 | public class LzDatasetDesc { 25 | public final Resource resource ; 26 | public final String name ; 27 | public final List indexes ; 28 | public final List nodes ; 29 | 30 | public LzDatasetDesc(Resource r, String name, List indexes, List nodes) { 31 | this.resource = r ; 32 | this.name = name ; 33 | this.indexes = indexes ; 34 | this.nodes = nodes ; 35 | } 36 | 37 | @Override 38 | public String toString() { return name ; } 39 | } 40 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/dataset/TransactionalComponentZkLock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.dataset; 19 | 20 | import lizard.cluster.Cluster ; 21 | import org.apache.jena.dboe.transaction.txn.ComponentId ; 22 | import org.apache.jena.dboe.transaction.txn.TransactionalComponentBase ; 23 | import org.apache.jena.dboe.transaction.txn.TxnId ; 24 | import org.apache.jena.query.ReadWrite ; 25 | 26 | final public class TransactionalComponentZkLock extends TransactionalComponentBase { 27 | 28 | public TransactionalComponentZkLock(ComponentId id) { 29 | super(id) ; 30 | } 31 | 32 | @Override 33 | protected Object _begin(ReadWrite readWrite, TxnId txnId) { 34 | if ( readWrite == ReadWrite.WRITE ) 35 | Cluster.acquireWriteLock() ; 36 | return null ; 37 | } 38 | 39 | @Override 40 | protected void _complete(TxnId txnId, Object state) { 41 | if ( getReadWriteMode() == ReadWrite.WRITE ) 42 | Cluster.releaseWriteLock() ; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /lizard-cluster/src/main/java/lizard/sys/Lizard.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.sys; 19 | 20 | import lizard.conf.assembler.VocabLizard ; 21 | import lizard.query.LizardQuery ; 22 | import lizard.system.LzLog ; 23 | 24 | public class Lizard { 25 | static { init() ; } // Via assembler. 26 | public synchronized static void init() { 27 | LzLog.logSystem.info("Lizard.init") ; 28 | VocabLizard.init(); 29 | LizardQuery.init(); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /lizard-cmds/README.md: -------------------------------------------------------------------------------- 1 | ## Environent variables: 2 | 3 | bin/init sets these, using the script location to set `LIZARD_HOME`. 4 | 5 | | Name | Meaning | Default | 6 | | ----- | -------- | -------- | 7 | |`LIZARD_HOME`| Lizard install directory | `$HOME/Lizard` | 8 | |`LZ_RUN` | Lizard runtime for cmds | `$LIZARD_HOME/lizard-cmds` | 9 | |`LZ_BIN` | Lizard cmd scripts | `$LIZARD_HOME/lizard-cmds/bin` | 10 | |`LZ_CONF` | Directory for config file | `$LIZARD_HOME/lz-dev/conf` | 11 | 12 | -------------------------------------------------------------------------------- /lizard-cmds/bin/init: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ## Assumes LIZARD_HOME is set 4 | ## Sets: 5 | ## LZ_RUN 6 | ## LZ_BIN 7 | ## LZ_DEV 8 | ## LZ_CP (lizard classpath) 9 | ## Merges to form CP (classpath to use) 10 | 11 | # Maven repository 12 | REPO="$HOME/.m2/repo" 13 | CPFILE="lizard.classpath" 14 | 15 | export LZ_RUN="${LZ_RUN:-$LIZARD_HOME/lizard-cmds}" 16 | export LZ_BIN="${LZ_BIN:-$LIZARD_HOME/lizard-cmds/bin}" 17 | export LZ_DEV="${LZ_DEV:-$LIZARD_HOME/lz-dev}" 18 | 19 | if [[ $LZ_DEBUG == 1 ]] 20 | then 21 | echo "LIZARD_HOME=$LIZARD_HOME" 22 | echo "LZ_RUN=$LZ_RUN" 23 | echo "LZ_BIN=$LZ_BIN" 24 | echo "LZ_DEV=$LZ_DEV" 25 | fi 26 | 27 | ## Build classpath 28 | 29 | if [[ ! -s "$LZ_RUN/$CPFILE" ]] 30 | then 31 | echo "No classpath file: $CPFILE" 1>&2 32 | echo "Run \"(cd $LZ_RUN ; mvn dependency:build-classpath -Dmdep.outputFile=$CPFILE)\"" ; 1>&2 33 | exit 2 34 | fi 35 | 36 | 37 | ## Accumulate LZ_CP 38 | accLZCP() { 39 | local C="$1" 40 | if [[ -z $LZ_CP ]] 41 | then 42 | LZ_CP=$C 43 | else 44 | LZ_CP=$LZ_CP:$C 45 | fi 46 | } 47 | 48 | XCP="$(cat $LZ_RUN/$CPFILE)" 49 | XCP="$(echo $XCP | sed -e s/:/\\n/g)" 50 | LZ_CP="" 51 | for C in $XCP 52 | do 53 | if [[ $C =~ "/org/seaborne/lizard/" ]] ; then continue ; fi 54 | if [[ $C =~ "/org/apache/jena/jena-core/" ]] ; then continue ; fi 55 | if [[ $C =~ "/org/apache/jena/jena-arq/" ]] ; then continue ; fi 56 | if [[ $C =~ "/org/apache/jena/jena-tdb/" ]] ; then continue ; fi 57 | if [[ $C =~ "/org/apache/jena/apache-jena-libs/" ]] ; then continue ; fi 58 | accLZCP "$C" 59 | done 60 | 61 | # And a built-in zookeeper (for now). 62 | accLZCP "$REPO/org/apache/curator/curator-test/2.7.1/curator-test-2.7.1.jar" 63 | accLZCP "$REPO/org/javassist/javassist/3.18.1-GA/javassist-3.18.1-GA.jar" 64 | accLZCP "$REPO/org/apache/commons/commons-math/2.2/commons-math-2.2.jar" 65 | 66 | # Prepend any development directories here 67 | DEVDIRS="jena-core jena-tdb jena-arq" 68 | JENA_DEV="$HOME/Jena" 69 | 70 | for X in $DEVDIRS 71 | do 72 | CPX="$JENA_DEV/$X/target/classes" 73 | if [ -e "$CPX" ] 74 | then 75 | accLZCP "$CPX" 76 | fi 77 | done 78 | 79 | DEVDIRS="base node-server index-server query-server cluster cmds" 80 | for X in $DEVDIRS 81 | do 82 | accLZCP "$LIZARD_HOME/lizard-$X/target/classes" 83 | #accLZCP "$LIZARD_HOME/lizard-$X/target/test-classes" 84 | done 85 | 86 | LOGGER=${LOGGER:-} # "-D-Dlog4j.configuration=file:$LIZARD_HOME/log4j.properties" 87 | JVM_ARGS="${JVM_ARGS:--Xmx1200M}" 88 | 89 | if [[ -z $CP ]] 90 | then 91 | CP="$LZ_CP" 92 | else 93 | CP="$CP:$LZ_CP" 94 | fi 95 | -------------------------------------------------------------------------------- /lizard-cmds/bin/lizard-command: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Symbolically link to this script and 4 | # Sets up enviroment and calls "$0-cmd" 5 | 6 | ## Set SCRIPT 7 | ## Set SCRIPT_DIR 8 | SCRIPT="${BASH_SOURCE[0]}"; 9 | SCRIPT_DIR="$(dirname "$SCRIPT")" 10 | while[[ -L "${SCRIPT_DIR}" ]] ; do SCRIPT_DIR="$(readlink "${SCRIPT_DIR}")"; done 11 | SCRIPT="$(basename "$SCRIPT")" 12 | 13 | # Lizard 14 | if [[ -z "$LIZARD_HOME" ]] 15 | then 16 | # Assume command in ${LIZARD_HOME}/lizard-cmds/bin 17 | LIZARD_HOME="${SCRIPT_DIR%/bin}" 18 | LIZARD_HOME="${SCRIPT_DIR%/lizard-*}" 19 | fi 20 | 21 | LZ_CMDS="$LIZARD_HOME/lizard-cmds" 22 | CMD="$SCRIPT_DIR/$SCRIPT" 23 | 24 | source "$LZ_CMDS/bin/init" 25 | 26 | # If linking: 27 | ## exec "${CMD}-cmd" "$@" 28 | # If copy template: 29 | 30 | -------------------------------------------------------------------------------- /lizard-cmds/bin/lz: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Find where the script is and set LIZARD_HOME 4 | # If LIZARD_HOME is empty 5 | if [ -z "$LIZARD_HOME" ] 6 | then 7 | SCRIPT="$0" 8 | # Catch common issue: script has been symlinked 9 | if [ -L "$SCRIPT" ] 10 | then 11 | SCRIPT="$(readlink "$0")" 12 | # If link is relative 13 | case "$SCRIPT" in 14 | /*) ;; # fine 15 | *) SCRIPT=$( dirname "$0" )/$SCRIPT;; # fix 16 | esac 17 | fi 18 | D="$( cd "$( dirname "$SCRIPT")" && pwd )" 19 | # Strip directories from Lizard home to bin directory. 20 | D="${D%/bin}" 21 | D="${D%/lizard-*}" 22 | LIZARD_HOME="$D" 23 | fi 24 | LZ_CMDS="$LIZARD_HOME/lizard-cmds" 25 | 26 | source "$LZ_CMDS/bin/init" 27 | 28 | CMD="$(basename $0)" 29 | 30 | if [[ ! $PWD =~ $LIZARD_HOME ]] 31 | then 32 | echo "LIZARD_HOME=$LIZARD_HOME" 33 | fi 34 | 35 | usage() { 36 | echo "Usage: $(basename $0) subcmd" 2>&1 37 | echo " Subcommands: 'deploy', 'query', 'servers', 'monitor', 'shard', 'split', 'zk'" 2>&1 38 | } 39 | 40 | if [[ $# = 0 ]] 41 | then 42 | usage 43 | exit 1 44 | fi 45 | 46 | SUB=$1 47 | shift 48 | 49 | case $SUB in 50 | help) usage ; exit 0 ;; 51 | servers) exec "$LZ_CMDS/bin/lz-servers" ;; 52 | deploy) CMD="LZ_Deploy" ;; 53 | query) CMD="LZ_Query" ;; 54 | monitor) CMD="LZ_Monitor" ;; 55 | shard) CMD="LZ_ShardIndex" ;; 56 | split) CMD="LZ_SplitDB" ;; 57 | zk|zookeeper) CMD="LZ_Zookeeper" ;; 58 | *) 59 | echo "Unknown subcommand: $SUB" 60 | usage 61 | exit 1 62 | ;; 63 | esac 64 | 65 | if [[ -e log4j.properties ]] 66 | then 67 | LOGGING="-Dlog4j.configuration=file:log4j.properties" 68 | fi 69 | 70 | java -cp "$CP" $JVM_ARGS $LOGGING lz.$CMD "$@" 71 | -------------------------------------------------------------------------------- /lizard-cmds/bin/lz-servers: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export LIZARD_HOME="${LIZARD_HOME:-$HOME/Lizard}" 4 | 5 | ## FONT="6x10" 6 | ## GEO1='180x25+1000+020' 7 | ## GEO2='180x25+1000+320' 8 | 9 | # Or 7x14 10 | FONT="7x13" 11 | declare -a GEO 12 | GEO=('180x25+800+020' '180x25+800+420' '180x25+800+820') 13 | 14 | CMD="xterm -fn $FONT -geometry" 15 | 16 | cd "$LZ_DEV" 17 | 18 | LZ_CONF="${LZ_CONF:-conf-index.ttl conf-node.ttl}" 19 | LZ_DEPLOY="${LZ_DEPLOY:-machine1.ttl machine2.ttl}" 20 | 21 | 22 | echo "LIZARD_HOME=$LIZARD_HOME" 23 | if [[ $LZ_DEBUG == 1 ]] 24 | then 25 | echo "LZ_RUN = $LZ_RUN" 26 | echo "LZ_BIN = $LZ_BIN" 27 | echo "LZ_DEV = $LZ_DEV" 28 | fi 29 | 30 | echo "LZ_CONF = $LZ_CONF" 31 | echo "LZ_DEPLOY = $LZ_DEPLOY" 32 | 33 | echo "Deployment:" 34 | 35 | declare -i I=0 36 | for D in $LZ_DEPLOY 37 | do 38 | G=${GEO[$I]} 39 | echo " lz deploy --deploy $D $LZ_CONF" 40 | $CMD "$G" -e $LZ_BIN/lz deploy --deploy $D $LZ_CONF & 41 | #echo "$D $G" 42 | ((I++)) 43 | done 44 | 45 | wait 46 | -------------------------------------------------------------------------------- /lizard-cmds/bin/shard: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Setup. 4 | SCRIPT="${BASH_SOURCE[0]}"; 5 | SCRIPT_DIR="$(dirname "$SCRIPT")" 6 | while [[ -L "${SCRIPT_DIR}" ]] 7 | do 8 | SCRIPT_DIR="$(readlink "${SCRIPT_DIR}")" 9 | done 10 | SCRIPT="$(basename "$SCRIPT")" 11 | 12 | if [[ $DEBUG == 1 ]] 13 | then 14 | echo "SCRIPT = $SCRIPT" 15 | echo "SCRIPT_DIR = $SCRIPT_DIR" 16 | fi 17 | 18 | # Lizard 19 | if [[ -z "$LIZARD_HOME" ]] 20 | then 21 | # Assume command in ${LIZARD_HOME}/lizard-cmds/bin 22 | LIZARD_HOME="${SCRIPT_DIR%/bin}" 23 | LIZARD_HOME="${SCRIPT_DIR%/lizard-*}" 24 | fi 25 | 26 | LZ_CMDS="$LIZARD_HOME/lizard-cmds" 27 | CMD="$SCRIPT_DIR/$SCRIPT" 28 | 29 | ## OTT (classpath processing) 30 | source "$LZ_CMDS/bin/init" 31 | 32 | ## --------------------------------- 33 | ## if [[ -z $1 ]] 34 | ## then 35 | ## echo "No location given" 2>&1 36 | ## exit 2 37 | ## fi 38 | 39 | LOC_SRC=DB 40 | LOC_DEST=DB1 41 | 42 | rm -rf "$LOC_DEST" 43 | mkdir -p "$LOC_DEST" 44 | 45 | # Split into 46 | 47 | # SPO,POS,OSP 48 | 49 | 50 | # tdb.CmdRewriterIndex 51 | # SPO, POS, (POS->PSO), PSO 52 | 53 | for X in SPO POS 54 | do 55 | echo "Copy $X : $LOC_SRC -> $LOC_DEST" 56 | cp "$LOC_SRC/$X".* "$LOC_DEST" 57 | done 58 | 59 | CMD_IDX="org.apache.jena.tdb.store.bulkloader2.CmdIndexCopy" 60 | echo "Rewrite POS to PSO : $LOC_SRC -> $LOC_DEST" 61 | java -cp "$CP" $CMD_IDX "$LOC_SRC" POS "$LOC_DEST" PSO 62 | 63 | ## for X in SPO POS OSP 64 | ## do 65 | ## lz shard --loc $LOCATION SPO 2 66 | ## done 67 | ## 68 | ## # Nodes 69 | 70 | echo "Copy TDB configuration" 71 | echo "XXXXX" 72 | -------------------------------------------------------------------------------- /lizard-cmds/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 23 | 4.0.0 24 | 25 | 26 | org.seaborne.lizard 27 | lizard 28 | .. 29 | 0.0.0-SNAPSHOT 30 | 31 | 32 | lizard-cmds 33 | 34 | jar 35 | 36 | 37 | 38 | 39 | http://www.apache.org/licenses/LICENSE-2.0 40 | 41 | 42 | 43 | 44 | 45 | 46 | org.seaborne.lizard 47 | lizard-cluster 48 | 0.0.0-SNAPSHOT 49 | 50 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /lizard-cmds/src/main/java/lz/LZ_Monitor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lz; 19 | 20 | import jena.cmd.CmdException ; 21 | import jena.cmd.CmdGeneral ; 22 | import lizard.cluster.Cluster ; 23 | 24 | import org.apache.jena.atlas.lib.Lib ; 25 | import org.apache.jena.atlas.logging.LogCtl ; 26 | import org.slf4j.Logger ; 27 | import org.slf4j.LoggerFactory ; 28 | 29 | public class LZ_Monitor extends CmdGeneral { 30 | static { LogCtl.setCmdLogging(); } 31 | 32 | private static Logger log = LoggerFactory.getLogger(LZ_Monitor.class) ; 33 | private String connectString = "localhost:2181" ; 34 | 35 | public static void main(String ...args) { 36 | new LZ_Monitor(args).mainRun(); 37 | } 38 | 39 | protected LZ_Monitor(String[] argv) { 40 | super(argv) ; 41 | } 42 | 43 | @Override 44 | protected String getSummary() { 45 | return "monitor [Zookeeper ConnectString]"; 46 | } 47 | 48 | 49 | @Override 50 | protected void processModulesAndArgs() { 51 | if ( super.getPositional().size() > 1 ) 52 | throw new CmdException("Too many arguments") ; 53 | if ( super.getPositional().size() == 1 ) 54 | connectString = super.getArg(0) ; 55 | } 56 | 57 | @Override 58 | protected void exec() { 59 | try { 60 | Cluster.createSystem(connectString) ; 61 | Cluster.monitor() ; 62 | for ( ;; ) 63 | Lib.sleep(10000) ; 64 | // Cluster.close(); 65 | // Lib.sleep(1000) ; 66 | // System.out.println("DONE"); 67 | } catch (Exception ex) {} 68 | } 69 | 70 | @Override 71 | protected String getCommandName() { 72 | return "monitor" ; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /lizard-cmds/src/main/java/lz/LZ_Zookeeper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lz; 19 | 20 | import jena.cmd.ArgDecl ; 21 | import jena.cmd.CmdException ; 22 | import jena.cmd.CmdGeneral ; 23 | import lizard.build.LzBuildZk ; 24 | import lizard.cluster.Cluster ; 25 | 26 | import org.apache.jena.atlas.lib.FileOps ; 27 | import org.apache.jena.atlas.lib.Lib ; 28 | import org.apache.jena.atlas.logging.LogCtl ; 29 | import org.slf4j.Logger ; 30 | import org.slf4j.LoggerFactory ; 31 | 32 | /** Run a simple zookeeper */ 33 | public class LZ_Zookeeper extends CmdGeneral { 34 | static { LogCtl.setCmdLogging(); } 35 | public static Logger log = LoggerFactory.getLogger("Lizard") ; 36 | public static Logger logConf = LoggerFactory.getLogger("Conf") ; 37 | 38 | protected ArgDecl argPort = new ArgDecl(ArgDecl.HasValue, "port") ; 39 | private int port ; 40 | private final static int dftPort = 2186 ; 41 | 42 | public static void main(String ...args) { 43 | new LZ_Zookeeper(args).mainRun(); 44 | } 45 | 46 | protected LZ_Zookeeper(String[] argv) { 47 | super(argv) ; 48 | super.add(argPort, argPort.getKeyName(), "Port number for the zookeeper instance") ; 49 | } 50 | 51 | @Override 52 | protected String getSummary() { 53 | return "lz-zookeeper --port PORT"; 54 | } 55 | 56 | 57 | @Override 58 | protected void processModulesAndArgs() { 59 | if ( super.getPositional().size() > 1 ) 60 | throw new CmdException("Too many arguments") ; 61 | if ( super.contains(argPort) ) 62 | port = Integer.parseInt(getArg(argPort).getValue()) ; 63 | else 64 | port = dftPort ; 65 | } 66 | 67 | @Override 68 | protected void exec() { 69 | try { 70 | String dataDirectory = System.getProperty("java.io.tmpdir")+"/zk" ; 71 | FileOps.ensureDir(dataDirectory); 72 | FileOps.clearAll(dataDirectory); 73 | LzBuildZk.zookeeper(port, dataDirectory); 74 | // Give ZK time to start. 75 | Lib.sleep(100) ; 76 | Cluster.createSystem("localhost:"+port); 77 | Cluster.monitor() ; 78 | for ( ;; ) 79 | Lib.sleep(10000) ; 80 | } catch (Exception ex) {} 81 | } 82 | 83 | @Override 84 | protected String getCommandName() { 85 | return "lz-zookeeper" ; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /lizard-cmds/src/main/java/lz/lz.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lz; 19 | 20 | import org.apache.jena.atlas.logging.LogCtl ; 21 | 22 | // Done by script? 23 | public class lz { 24 | static { LogCtl.setLog4j(); } 25 | 26 | public static void main(String... $args) { 27 | if ( $args.length < 1) { 28 | System.err.println("Subcommands: deploy query"); 29 | CmdError("lz: no subcommand") ; 30 | } 31 | 32 | String[] args = new String[$args.length-1] ; 33 | System.arraycopy($args, 1, args, 0, $args.length-1); 34 | String subcmd = $args[0] ; 35 | 36 | switch ( subcmd ) { 37 | case "deploy": { new LZ_Deploy(args).mainRun() ; break ; } 38 | case "query": { new LZ_Query(args).mainRun(); break ; } 39 | case "monitor": { new LZ_Monitor(args).mainRun(); break ; } 40 | case "zk": case "zookeeper": 41 | { new LZ_Zookeeper(args).mainRun(); break ; } 42 | default: 43 | CmdError("lz: no such subcommand: "+subcmd) ; 44 | break ; 45 | } 46 | } 47 | 48 | private static void CmdError(String string, Object ...args) { 49 | System.err.printf(string, args) ; 50 | if ( ! string.endsWith("\n") ) 51 | System.err.println() ; 52 | System.exit(99) ; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /lizard-dist/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 23 | 4.0.0 24 | 25 | 26 | org.seaborne.lizard 27 | lizard 28 | .. 29 | 0.0.0-SNAPSHOT 30 | 31 | 32 | lizard-dist 33 | 34 | jar 35 | 36 | 37 | 38 | 39 | http://www.apache.org/licenses/LICENSE-2.0 40 | 41 | 42 | 43 | 44 | 45 | 46 | org.seaborne.lizard 47 | lizard-cmds 48 | 0.0.0-SNAPSHOT 49 | 50 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /lizard-index-server/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 23 | 4.0.0 24 | 25 | 26 | org.seaborne.lizard 27 | lizard 28 | .. 29 | 0.0.0-SNAPSHOT 30 | 31 | 32 | lizard-index-server 33 | 34 | jar 35 | 36 | 37 | 38 | 39 | http://www.apache.org/licenses/LICENSE-2.0 40 | 41 | 42 | 43 | 44 | 45 | 46 | org.seaborne.lizard 47 | lizard-base 48 | 0.0.0-SNAPSHOT 49 | 50 | 51 | 52 | org.seaborne.mantis 53 | dboe-quack 54 | 55 | 56 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /lizard-index-server/src/main/java/lizard/index/ClusterAccessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.index; 19 | 20 | import java.util.ArrayList ; 21 | import java.util.Iterator ; 22 | import java.util.List ; 23 | 24 | import org.apache.jena.atlas.lib.tuple.Tuple ; 25 | import org.apache.jena.tdb2.store.NodeId ; 26 | import org.seaborne.dboe.engine.access.AccessorBase ; 27 | 28 | /** Needed? Replace ClusterTupleIndex? */ 29 | public class ClusterAccessor extends AccessorBase // implements Accessor 30 | { 31 | // Or ClusterTupleIndex. 32 | 33 | private final DistributorTupleIndex distributor ; 34 | 35 | public ClusterAccessor(DistributorTupleIndex distributor) { 36 | this.distributor = distributor ; 37 | } 38 | 39 | @Override 40 | public Iterator> accessTuples(Tuple pattern) { 41 | 42 | List places = distributor.findAt(pattern) ; 43 | List>> segments = new ArrayList<>() ; 44 | 45 | for ( TupleIndexRemote idx : places ) { 46 | Iterator> iter = idx.find(pattern) ; 47 | segments.add(iter) ; 48 | } 49 | return new MergeIterator(segments, null, false) ; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /lizard-index-server/src/main/java/lizard/index/DistributorTupleIndex.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.index; 19 | 20 | import java.util.Collection ; 21 | import java.util.List ; 22 | 23 | import lizard.system.Distributor ; 24 | import org.apache.jena.atlas.lib.tuple.Tuple ; 25 | import org.apache.jena.tdb2.store.NodeId ; 26 | 27 | /** Policy for the placement of triples (and for finding them). */ 28 | public interface DistributorTupleIndex extends Distributor { 29 | /** Places where a triple is stored. 30 | * Each place must be updated. 31 | * @param tuple 32 | * @return List 33 | */ 34 | 35 | public List storeAt(Tuple tuple) ; 36 | 37 | /** Places to try to find a triple pattern. 38 | * Each place in the list must be tried. 39 | * @param tuple 40 | * @return List 41 | */ 42 | public List findAt(Tuple tuple) ; 43 | 44 | /** A set of places that covers the entire space, with one replicator per possible place */ 45 | public Collection allFind() ; 46 | 47 | /** A set of places that covers the entire space, all places, for storage */ 48 | public Collection allStore() ; 49 | 50 | /** All the elements managed by this distributor */ 51 | public Collection allRemotes() ; 52 | } 53 | -------------------------------------------------------------------------------- /lizard-index-server/src/main/java/lizard/index/RemoteAccessData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.index; 19 | 20 | import java.util.Iterator ; 21 | 22 | import org.apache.jena.atlas.lib.tuple.Tuple ; 23 | import org.apache.jena.tdb2.store.NodeId ; 24 | import org.seaborne.dboe.engine.access.AccessData ; 25 | import org.slf4j.Logger ; 26 | import org.slf4j.LoggerFactory ; 27 | 28 | /** Adapter from TClientIndex to AccessData */ 29 | 30 | public class RemoteAccessData implements AccessData { 31 | 32 | // XXX This extra class from TupleIndexRenote to TClientIndex makes index 33 | // more complicted than node tables. Maybe they are? 34 | static long counter = 0 ; 35 | static Logger log = LoggerFactory.getLogger(RemoteAccessData.class) ; 36 | 37 | private final TClientIndex client ; 38 | 39 | public RemoteAccessData(TClientIndex conn) { 40 | this.client = conn ; 41 | } 42 | 43 | @Override 44 | public Iterator> accessTuples(Tuple pattern) { 45 | return client.find(pattern) ; 46 | 47 | // TLZ_TupleNodeId t = TLZlib.build(pattern) ; 48 | // long id = (++counter) ; 49 | // List> results = null ; 50 | // log.debug("RemoteAccessor.find - "+results.size()) ; 51 | // return results.iterator() ; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /lizard-index-server/src/main/java/lizard/index/Shard.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.index; 19 | 20 | import com.google.common.hash.HashCode ; 21 | import com.google.common.hash.HashFunction ; 22 | import com.google.common.hash.Hashing ; 23 | 24 | import org.apache.jena.atlas.lib.tuple.Tuple ; 25 | import org.apache.jena.atlas.lib.tuple.TupleMap ; 26 | import org.apache.jena.tdb2.store.NodeId ; 27 | 28 | public class Shard { 29 | // Copied from proto-lizard:DistributorTuplesBySubject 30 | // BAD Use Fnv. 31 | 32 | public static final long NO_SHARD = -1 ; 33 | 34 | /** Calculate the chard for this tuple or tuple pattern. 35 | * Return {@link #NO_SHARD} for "any" 36 | * @param tuple 37 | * @return int 38 | */ 39 | public static long shardBySubject(TupleMap mapper, int numShard, Tuple tuple) { 40 | NodeId n = mapper.mapSlot(0, tuple) ; 41 | if ( NodeId.isAny(n) ) 42 | return NO_SHARD ; 43 | long shard = shard(n, numShard) ; 44 | return shard ; 45 | } 46 | 47 | private static HashFunction hasher = Hashing.goodFastHash(32) ; 48 | 49 | public static long shard(NodeId n, int numShard) { 50 | if ( n == null ) 51 | throw new IllegalArgumentException("Asked to hash null") ; 52 | if ( numShard == 0 ) 53 | // Zero bits - fixed. 54 | return 0L ; 55 | // Must use all bits (lower few are probably all zero) 56 | //byte[] b = new byte[NodeId.SIZE] ; 57 | 58 | System.err.println("Update for NdoieId bytes"); 59 | long v = 0L; 60 | int x = hash(v) & 0xFFFFFF ; 61 | return x % numShard ; 62 | } 63 | 64 | static final public int hash(long v) { 65 | HashCode hc = hasher.hashLong(v) ; 66 | return hasher.hashLong(v).asInt() ; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /lizard-index-server/src/main/java/lizard/index/TClientIndexOps.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.index; 19 | 20 | import java.util.Collection ; 21 | import java.util.Iterator ; 22 | 23 | import lizard.comms.ConnState ; 24 | import lizard.comms.Connection ; 25 | import lizard.system.ComponentTxn ; 26 | import lizard.system.RemoteControl ; 27 | import org.apache.jena.atlas.lib.tuple.Tuple ; 28 | import org.apache.jena.atlas.lib.tuple.TupleFactory ; 29 | import org.apache.jena.tdb2.store.NodeId ; 30 | 31 | interface TClientIndexOps extends Connection, ComponentTxn, RemoteControl 32 | { 33 | @Override 34 | public void start() ; 35 | 36 | @Override 37 | public void stop() ; 38 | 39 | /** Insert a tuple */ 40 | public void add(Tuple tuple) ; 41 | 42 | /** Insert a tuple */ 43 | public void addAll(Collection> tuples) ; 44 | 45 | /** Delete a tuple */ 46 | public void delete(Tuple tuple) ; 47 | 48 | /** Delete a tuple */ 49 | public void deleteAll(Collection> tuples) ; 50 | 51 | @Override 52 | public void ping() ; 53 | 54 | public Iterator> find(Tuple pattern) ; 55 | 56 | static Tuple tupleAny4 = TupleFactory.tuple(NodeId.NodeIdAny, NodeId.NodeIdAny, NodeId.NodeIdAny, NodeId.NodeIdAny) ; 57 | static Tuple tupleAny3 = TupleFactory.tuple(NodeId.NodeIdAny, NodeId.NodeIdAny, NodeId.NodeIdAny) ; 58 | 59 | /** return an iterator of everything */ 60 | default public Iterator> all() { return find(tupleAny3) ; } 61 | 62 | @Override 63 | public ConnState getConnectionStatus() ; 64 | 65 | @Override 66 | public String label() ; 67 | 68 | @Override 69 | public void setConnectionStatus(ConnState status) ; 70 | 71 | @Override 72 | public void close() ; 73 | } 74 | 75 | -------------------------------------------------------------------------------- /lizard-index-server/src/main/java/lizard/index/TServerIndex.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.index; 19 | 20 | import lizard.api.TLZ.TLZ_Index ; 21 | import lizard.comms.thrift.ThriftServer ; 22 | import org.apache.jena.atlas.logging.FmtLog ; 23 | import org.apache.jena.dboe.transaction.txn.TransactionalSystem ; 24 | import org.apache.jena.tdb2.store.tupletable.TupleIndex ; 25 | import org.apache.thrift.protocol.TCompactProtocol ; 26 | import org.apache.thrift.server.TServer ; 27 | import org.apache.thrift.server.TThreadPoolServer ; 28 | import org.slf4j.Logger ; 29 | import org.slf4j.LoggerFactory ; 30 | 31 | public class TServerIndex extends ThriftServer 32 | { 33 | private static Logger log = LoggerFactory.getLogger(TServerIndex.class) ; 34 | private TupleIndex index ; 35 | 36 | public static TServerIndex create(TransactionalSystem txnSystem, int port, TupleIndex index) { 37 | return new TServerIndex(txnSystem, port, index) ; 38 | } 39 | 40 | private TServerIndex(TransactionalSystem txnSystem, int port, TupleIndex index) { 41 | super(txnSystem, port) ; 42 | setLabel("IndexServer["+port+"]") ; 43 | this.index = index ; 44 | } 45 | 46 | @Override 47 | public void start() { 48 | FmtLog.info(log, "Start index server: port = %d", getPort()) ; 49 | // Semapahores to sync. 50 | //Semaphore sema = new Semaphore(0) ; 51 | 52 | new Thread(()-> { 53 | try { 54 | getTxnSystem().getTxnMgr().start(); 55 | TLZ_Index.Iface handler = new THandlerTupleIndex(getTxnSystem(), getLabel(), index) ; 56 | TLZ_Index.Processor processor = new TLZ_Index.Processor(handler); 57 | TThreadPoolServer.Args args = new TThreadPoolServer.Args(serverTransport) ; 58 | args.processor(processor) ; 59 | args.inputProtocolFactory(new TCompactProtocol.Factory()) ; 60 | args.outputProtocolFactory(new TCompactProtocol.Factory()) ; 61 | TServer server = new TThreadPoolServer(args); 62 | FmtLog.info(log, "Started index server: port = %d", getPort()) ; 63 | //sema.release(1); 64 | server.serve(); 65 | FmtLog.info(log, "Finished index server: port = %d", getPort()) ; 66 | getTxnSystem().getTxnMgr().shutdown(); 67 | } catch (Exception e) { 68 | e.printStackTrace(); 69 | } 70 | }) .start() ; 71 | //sema.acquireUninterruptibly(); 72 | super.start(); 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /lizard-node-server/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 23 | 4.0.0 24 | 25 | 26 | org.seaborne.lizard 27 | lizard 28 | .. 29 | 0.0.0-SNAPSHOT 30 | 31 | 32 | lizard-node-server 33 | 34 | jar 35 | 36 | 37 | 38 | 39 | http://www.apache.org/licenses/LICENSE-2.0 40 | 41 | 42 | 43 | 44 | 45 | 46 | org.seaborne.lizard 47 | lizard-base 48 | 0.0.0-SNAPSHOT 49 | 50 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /lizard-node-server/src/main/java/lizard/node/DistributorNodes.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.node; 19 | 20 | import java.util.Collection ; 21 | import java.util.List ; 22 | 23 | import lizard.system.Distributor ; 24 | import org.apache.jena.graph.Node ; 25 | import org.apache.jena.tdb2.store.NodeId ; 26 | 27 | public interface DistributorNodes extends Distributor { 28 | /** W quorum of {@link NodeTableRemote} for storing a node */ 29 | public List storeAt(Node node) ; 30 | /** R quorum of {@link NodeTableRemote} for finding the node for a nodeid */ 31 | public List findAt(NodeId nodeid) ; 32 | /** R quorum of {@link NodeTableRemote} for finding the nodeid for a node */ 33 | public List findAt(Node node) ; 34 | /** 35 | * Collection of all the places to go to find something - will contain R of N replicas. 36 | * Normally, R=1, W=N. 37 | */ 38 | public Collection allFind() ; 39 | /** 40 | * Collection of all the places to go to store something - will contain W of N replica. 41 | * Normally, R=1, W=N. 42 | */ 43 | public Collection allStore() ; 44 | 45 | /** All the elements managed by this distributor */ 46 | public Collection allRemotes() ; 47 | } 48 | -------------------------------------------------------------------------------- /lizard-node-server/src/main/java/lizard/node/TServerNode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.node; 19 | 20 | 21 | import lizard.api.TLZ.TLZ_NodeTable ; 22 | import lizard.comms.thrift.ThriftServer ; 23 | import org.apache.jena.atlas.logging.FmtLog ; 24 | import org.apache.jena.dboe.transaction.txn.TransactionalSystem ; 25 | import org.apache.jena.tdb2.store.nodetable.NodeTable ; 26 | import org.apache.thrift.protocol.TCompactProtocol ; 27 | import org.apache.thrift.server.TServer ; 28 | import org.apache.thrift.server.TThreadPoolServer ; 29 | import org.slf4j.Logger ; 30 | import org.slf4j.LoggerFactory ; 31 | 32 | public class TServerNode extends ThriftServer 33 | { 34 | private static Logger log = LoggerFactory.getLogger(TServerNode.class) ; 35 | private final NodeTable nodeTable ; 36 | 37 | public static TServerNode create(TransactionalSystem txnSystem, int port, NodeTable nodeTable) { 38 | return new TServerNode(txnSystem, port, nodeTable) ; 39 | } 40 | 41 | private TServerNode(TransactionalSystem txnSystem, int port, NodeTable nodeTable) { 42 | super(txnSystem, port) ; 43 | setLabel("NodeServer["+port+"]") ; 44 | this.nodeTable = nodeTable ; 45 | } 46 | 47 | @Override 48 | public void start() { 49 | //FmtLog.debug(log, "Start node server, port = %d", getPort()) ; 50 | TLZ_NodeTable.Iface handler = new THandlerNodeTable(getTxnSystem(), getLabel(), nodeTable) ; 51 | TLZ_NodeTable.Processor processor = new TLZ_NodeTable.Processor(handler); 52 | 53 | // Semapahores to sync?? 54 | new Thread(()-> { 55 | try { 56 | getTxnSystem().getTxnMgr().start(); 57 | TThreadPoolServer.Args args = new TThreadPoolServer.Args(serverTransport) ; 58 | args.processor(processor) ; 59 | args.inputProtocolFactory(new TCompactProtocol.Factory()) ; 60 | args.outputProtocolFactory(new TCompactProtocol.Factory()) ; 61 | TServer server = new TThreadPoolServer(args); 62 | FmtLog.info(log, "Started node server: port = %d", getPort()) ; 63 | server.serve(); 64 | FmtLog.info(log, "Finished node server: port = %d", getPort()) ; 65 | getTxnSystem().getTxnMgr().shutdown(); 66 | } catch (Exception e) { 67 | e.printStackTrace(); 68 | } 69 | }) .start() ; 70 | super.start() ; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /lizard-node-server/src/main/java/lizard/node/ThriftObjectFile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.node ; 19 | 20 | import org.apache.jena.dboe.base.file.BinaryDataFile ; 21 | import org.apache.jena.riot.thrift.TRDF ; 22 | import org.apache.jena.riot.thrift.wire.RDF_Term ; 23 | import org.apache.jena.tdb2.TDBException ; 24 | import org.apache.jena.tdb2.store.nodetable.TReadAppendFileTransport ; 25 | import org.apache.thrift.TException ; 26 | import org.apache.thrift.protocol.TProtocol ; 27 | 28 | /** Thrift items :store and retrieve */ 29 | 30 | public class ThriftObjectFile { 31 | // Write buffering is done in the underlying BinaryDataFile 32 | BinaryDataFile diskFile ; 33 | private TReadAppendFileTransport transport ; 34 | private final TProtocol protocol ; 35 | 36 | public ThriftObjectFile(BinaryDataFile objectFile) { 37 | try { 38 | this.diskFile = objectFile ; 39 | transport = new TReadAppendFileTransport(diskFile) ; 40 | if ( ! transport.isOpen() ) 41 | transport.open(); 42 | this.protocol = TRDF.protocol(transport) ; 43 | } 44 | catch (Exception ex) { 45 | throw new TDBException("ThriftObjectFile", ex) ; 46 | } 47 | } 48 | 49 | public long writeToTable(RDF_Term term) { 50 | try { 51 | long x = diskFile.length() ; 52 | term.write(protocol) ; 53 | return x ; 54 | } catch (Exception ex) { 55 | throw new TDBException("ThriftObjectFile/Write", ex) ; 56 | } 57 | } 58 | 59 | public RDF_Term readFromTable(long posn) { 60 | // Assumes external synchronization. 61 | try { 62 | transport.readPosition(posn) ; 63 | RDF_Term term = new RDF_Term() ; 64 | term.read(protocol) ; 65 | return term ; 66 | } catch (TException ex) { 67 | throw new TDBException("ThriftObjectFile/Read", ex) ; 68 | } 69 | } 70 | 71 | public void flush() { 72 | try { transport.flush(); } 73 | catch (Exception ex) { throw new TDBException("ThriftObjectFile", ex) ; } 74 | } 75 | 76 | public void close() { 77 | if ( transport.isOpen() ) { 78 | try { transport.close() ; } 79 | catch (Exception ex) { throw new TDBException("ThriftObjectFile", ex) ; } 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /lizard-query-server/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 23 | 4.0.0 24 | 25 | 26 | org.seaborne.lizard 27 | lizard 28 | .. 29 | 0.0.0-SNAPSHOT 30 | 31 | 32 | lizard-query-server 33 | 34 | jar 35 | 36 | 37 | 38 | 39 | http://www.apache.org/licenses/LICENSE-2.0 40 | 41 | 42 | 43 | 44 | 45 | 46 | org.seaborne.lizard 47 | lizard-node-server 48 | 0.0.0-SNAPSHOT 49 | 50 | 51 | 52 | org.seaborne.lizard 53 | lizard-index-server 54 | 0.0.0-SNAPSHOT 55 | 56 | 57 | 58 | org.seaborne.mantis 59 | dboe-quack 60 | 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /lizard-query-server/src/main/java/lizard/query/AccessorRemote.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.query; 19 | 20 | import java.util.Iterator ; 21 | 22 | import org.apache.jena.atlas.lib.tuple.Tuple ; 23 | import org.apache.jena.tdb2.store.NodeId ; 24 | import org.seaborne.dboe.engine.PredicateObjectList ; 25 | import org.seaborne.dboe.engine.Row ; 26 | import org.seaborne.dboe.engine.Slot ; 27 | import org.seaborne.dboe.engine.access.Accessor ; 28 | import org.seaborne.dboe.engine.access.AccessorBase ; 29 | import org.seaborne.dboe.engine.tdb.AccessorTDB ; 30 | import org.seaborne.dboe.engine.tdb.AccessorTDBDebug ; 31 | import org.seaborne.dboe.engine.tdb.StorageTDB ; 32 | 33 | 34 | public class AccessorRemote extends AccessorTDB implements Accessor { 35 | public static AccessorTDB create(StorageTDB db) { 36 | AccessorTDB a = new AccessorRemote(db) ; 37 | a = new AccessorTDBDebug("Lizard: ", a) ; 38 | return a ; 39 | } 40 | 41 | // Temp way 42 | private AccessorRemote(StorageTDB db) { 43 | super(db) ; 44 | } 45 | 46 | @Override 47 | public Iterator> accessTuples(Tuple pattern) { 48 | // just while we wire the system 49 | return getDB().getTupleTable(pattern).find(pattern) ; 50 | } 51 | 52 | @Override 53 | public Iterator> accessRows(Tuple> pattern) { 54 | return AccessorBase.accessRows(this,pattern) ; 55 | } 56 | 57 | @Override 58 | public Iterator> fetch(PredicateObjectList accessList) { 59 | return AccessorBase.fetch(this, accessList).iterator() ; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /lizard-query-server/src/main/java/lizard/query/LizardQuery.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.query; 19 | 20 | import org.seaborne.dboe.engine.Quack2 ; 21 | 22 | public class LizardQuery { 23 | 24 | private static boolean initialized = false ; 25 | static { initialization1() ; } 26 | 27 | private static synchronized void initialization1() 28 | { 29 | // Called at start. 30 | if ( initialized ) 31 | return ; 32 | initialized = true ; 33 | // query engine. 34 | Quack2.init() ; 35 | // This would be global and all TDB. 36 | //Quack.setOpExecutorFactory(null) ; 37 | } 38 | 39 | /** called when Fuseki integrates Lizard */ 40 | public static void init() { } 41 | } 42 | 43 | -------------------------------------------------------------------------------- /lizard-query-server/src/main/java/lizard/query/LzDataset.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.query; 19 | 20 | import java.util.List ; 21 | 22 | import lizard.system.Component ; 23 | import lizard.system.LifeCycle ; 24 | import org.apache.jena.dboe.transaction.txn.TransactionCoordinator ; 25 | import org.apache.jena.tdb2.store.DatasetGraphTDB ; 26 | 27 | /** The query platform for Lizard - a dataset and the local components (clients to index and nodes) */ 28 | public class LzDataset implements LifeCycle { 29 | private DatasetGraphTDB dsg ; 30 | private List components ; 31 | private boolean started = false ; 32 | private TransactionCoordinator txnCoord ; 33 | 34 | public LzDataset(TransactionCoordinator txnCoord, DatasetGraphTDB dsg, List components) { 35 | this.txnCoord = txnCoord ; 36 | this.dsg = dsg ; 37 | this.components = components ; 38 | } 39 | 40 | public DatasetGraphTDB getDataset() { return dsg ; } 41 | 42 | public TransactionCoordinator getTxnMgr() { 43 | return txnCoord ; 44 | } 45 | 46 | public List getStartables() { 47 | return components ; 48 | } 49 | 50 | public List getComponents() { 51 | return this.components ; 52 | } 53 | 54 | @Override 55 | public void start() { 56 | txnCoord.start(); 57 | components.stream().forEach(s -> s.start() ); 58 | started = true ; 59 | } 60 | 61 | @Override 62 | public void stop() { 63 | components.stream().forEach(s -> s.stop()); 64 | started = false ; 65 | txnCoord.shutdown(); 66 | } 67 | 68 | @Override 69 | public boolean isRunning() { 70 | return started ; 71 | } 72 | 73 | @Override 74 | public boolean hasFailed() { 75 | return false ; 76 | } 77 | 78 | @Override 79 | public void setStatus(Status status) {} 80 | } 81 | -------------------------------------------------------------------------------- /lizard-query-server/src/main/java/lizard/query/QuackLizard.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.query; 19 | 20 | import org.apache.jena.sparql.engine.ExecutionContext ; 21 | import org.apache.jena.sparql.engine.main.OpExecutor ; 22 | import org.apache.jena.sparql.engine.main.OpExecutorFactory ; 23 | import org.apache.jena.tdb2.store.DatasetGraphTDB ; 24 | import org.seaborne.dboe.engine.Quack ; 25 | import org.seaborne.dboe.engine.explain.Explain2 ; 26 | import org.seaborne.dboe.engine.tdb.* ; 27 | import org.seaborne.dboe.engine.tdb.OpExecutorQuackTDB.OpExecSetup ; 28 | 29 | public class QuackLizard { 30 | 31 | private static OpExecSetup setupLizard = new OpExecSetup() { 32 | @Override 33 | public AccessorTDB initAccessor(DatasetGraphTDB dsgtdb) { 34 | return AccessorRemote.create(new StorageTDB(dsgtdb)) ; 35 | } 36 | 37 | @Override 38 | public Planner initPlanner(DatasetGraphTDB dsgtdb, AccessorTDB accessor) { 39 | return new PlannerPredObjList(accessor) ; 40 | } 41 | } ; 42 | 43 | public static final OpExecutorFactory factoryLizard = new OpExecutorFactory() { 44 | @Override 45 | public OpExecutor create(ExecutionContext execCxt) { 46 | Explain2.explain(Quack.quackExec, "Lizard") ; 47 | return new OpExecutorQuackTDB(execCxt, setupLizard) ; 48 | } 49 | } ; 50 | } 51 | -------------------------------------------------------------------------------- /lz-dev/.gitignore: -------------------------------------------------------------------------------- 1 | # Speciifc Lizard environments (local to machine) 2 | ##setup-* 3 | ##!setup-simple 4 | # Fuseki run area 5 | run/ 6 | run-*/ 7 | lz-dev.classpath 8 | # Databases 9 | DB* 10 | zkConf/ 11 | fu-conf.ttl 12 | 13 | # During initial development, allow small data test files. 14 | ## D.ttl 15 | ## D.trig 16 | ## D.nt 17 | ## D.nq 18 | /--mem-- 19 | /classes 20 | /tmp 21 | /LZ 22 | /LZ? 23 | -------------------------------------------------------------------------------- /lz-dev/D.ttl: -------------------------------------------------------------------------------- 1 | PREFIX : 2 | 3 | :s1 :p :o1 . 4 | :s2 :p :o2 . 5 | :s1 :q :o1 . 6 | :s2 :q :o2 . 7 | -------------------------------------------------------------------------------- /lz-dev/D1.ttl: -------------------------------------------------------------------------------- 1 | PREFIX : 2 | 3 | :z1 :p :o1 . 4 | :z2 :p :o2 . 5 | -------------------------------------------------------------------------------- /lz-dev/D2.ttl: -------------------------------------------------------------------------------- 1 | PREFIX : 2 | 3 | :s2 :p :o2 . 4 | :s2 :q :o2 . 5 | -------------------------------------------------------------------------------- /lz-dev/Q.rq: -------------------------------------------------------------------------------- 1 | PREFIX : 2 | 3 | SELECT * { 4 | ?s :p ?o 5 | } -------------------------------------------------------------------------------- /lz-dev/README.md: -------------------------------------------------------------------------------- 1 | This area is not a module in the build. 2 | 3 | It is a scratch pad used for development and testing. 4 | -------------------------------------------------------------------------------- /lz-dev/configuration.txt: -------------------------------------------------------------------------------- 1 | Design notes on configuration: 2 | 3 | Two sets : 4 | static 5 | - describes the clister over all 6 | - can be deployed indifefrent ways 7 | 8 | All 9 | :name 10 | 11 | ------------ 12 | static: 13 | 14 | cluster 15 | :zookeeper configuration file. 16 | :zookeeper data directory 17 | write lock timeout 18 | 19 | dataset 20 | :indexes (I1, I2) 21 | :nodetable (N) 22 | 23 | index services 24 | :indexOrder 25 | :server (replicas) 26 | :quorumR 27 | :quorumW 28 | 29 | node 30 | :servers (replicas) 31 | :policy for update 32 | :quorumR 33 | :quorumW 34 | 35 | ------------- 36 | Host table. 37 | <#server> -> (machine,port) 38 | 39 | ------------- 40 | 41 | per machine: 42 | 43 | deployment 44 | - different on each machine. 45 | - a list of hosted units 46 | - file root name 47 | 48 | YAML , JSON++ 49 | 50 | 51 | cluster 52 | : top area for files 53 | lizard/ 54 | "test" -- in memory override. 55 | 56 | dataset 57 | :location "rel dir" for journal. Override/dft="dataset" 58 | 59 | index servers 60 | replica -> (port, location) 61 | OR autonaming. 62 | 63 | node servers 64 | 65 | -------------------------------------------------------------------------------- /lz-dev/dev-run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | LIZARD_HOME="${LIZARD_HOME:-$HOME/Lizard}" 4 | LZ_CMDS="$LIZARD_HOME/lizard-cmds" 5 | 6 | CPFILE=lz-dev.classpath 7 | if [[ -e $CPFILE ]] 8 | then 9 | CP="$(cat "$CPFILE")" 10 | else 11 | echo "No classpath file: $CPFILE" 1>&2 12 | echo "Run \"mvn dependency:build-classpath -Dmdep.outputFile=$CPFILE\"" ; 1>&2 13 | exit 2 14 | ## source "$LZ_CMDS/bin/init" 15 | ## # lz-dev special jars 16 | ## CP="$CP:${M2_REPO}/org/apache/curator/curator-test/2.7.1/curator-test-2.7.1.jar" 17 | ## CP="$CP:${M2_REPO}/org/javassist/javassist/3.18.1-GA/javassist-3.18.1-GA.jar" 18 | ## CP="$CP:${M2_REPO}/org/apache/commons/commons-math/2.2/commons-math-2.2.jar" 19 | ## CP="$CP:${M2_REPO}/org/yaml/snakeyaml/1.9/snakeyaml-1.9.jar" 20 | fi 21 | 22 | CP="$CP:target/classes" 23 | 24 | if [[ $# = 0 ]] 25 | then 26 | echo "No main given" 2>&1 27 | exit 1 28 | fi 29 | 30 | MAIN=$1 31 | shift 32 | 33 | CP="$LIZARD_HOME/lz-dev/target/classes:$CP" 34 | 35 | java -cp "$CP" $JVM_ARGS $LOGGING $MAIN "$@" 36 | -------------------------------------------------------------------------------- /lz-dev/fix-headers: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | FILES=$(find src -name \*.java) 4 | 5 | for f in $FILES 6 | do 7 | perl -i.bak fix.pl $f 8 | done 9 | -------------------------------------------------------------------------------- /lz-dev/fu: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Run Fuseki 3 | 4 | LIZARD_HOME="${LIZARD_HOME:-$HOME/Lizard}" 5 | LZ_CMDS="$LIZARD_HOME/lizard-cmds" 6 | VER_FU=2.3.1-SNAPSHOT 7 | 8 | # either this or that 9 | #source "$LZ_CMDS/bin/init" 10 | 11 | if false 12 | then 13 | source "$LZ_CMDS/bin/init" 14 | CP="$CP:${M2_REPO}/org/yaml/snakeyaml/1.9/snakeyaml-1.9.jar" 15 | else 16 | CPFILE=lz-dev.classpath 17 | if [[ -e $CPFILE ]] 18 | then 19 | CP="$(cat "$CPFILE")" 20 | else 21 | echo "No classpath file: $CPFILE" 1>&2 22 | echo "Run \"mvn dependency:build-classpath -Dmdep.outputFile=$CPFILE\"" ; 1>&2 23 | exit 2 24 | fi 25 | fi 26 | 27 | # Fuseki 28 | CP="$CP:${M2_REPO}/org/apache/jena/jena-fuseki-server/${VER_FU}/jena-fuseki-server-${VER_FU}.jar" 29 | # Local files. 30 | CP="$CP:$PWD/target/classes" 31 | 32 | # Remove slf4j (inside fuseki server) 33 | CP="$(echo $CP | perl -p -e 's!:[^:]*/org/slf4j/[^:]*!:!g')" 34 | 35 | FUSEKI_LOG=${FUSEKI_LOG:-} 36 | JVM_ARGS="${JVM_ARGS:--Xmx1200M}" 37 | 38 | export FUSEKI_HOME=$HOME/Jena/jena-fuseki2/jena-fuseki-core 39 | export FUSEKI_BASE=$PWD/run 40 | [[ -e $FUSEKI_BASE ]] || mkdir -p $FUSEKI_BASE 41 | 42 | ## X="$(echo $CP | sed -e "s/:/\n"/g)" 43 | ## echo "$X" 44 | ## exit 45 | 46 | ## Make a Fuseki config file. 47 | cat setup1/fuseki-server.ttl setup1/conf-*ttl > fu-conf.ttl 48 | 49 | exec java -cp "$CP" $JVM_ARGS $FUSEKI_LOG org.apache.jena.fuseki.cmd.FusekiCmd \ 50 | "--conf=fu-conf.ttl" 51 | -------------------------------------------------------------------------------- /lz-dev/gen-thrift: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if (( $# == 0 )) 4 | then 5 | echo "No input files" 1>&2 6 | echo "Usage: $(basename $0) FILE ..." 1>&2 7 | exit 1 8 | fi 9 | 10 | 11 | PKG="src/main/java/rpc/thrift" 12 | rm -f $PKG/*.java 13 | 14 | thrift -r -out src/main/java -gen java "$@" 15 | 16 | for f in $PKG/*.java 17 | do 18 | perl -i.bak -p -e 's/^public (class|enum)/\@SuppressWarnings("all")\npublic $1/' $f 19 | rm $f.bak 20 | done 21 | 22 | -------------------------------------------------------------------------------- /lz-dev/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, stdlog 2 | 3 | log4j.appender.stdlog=org.apache.log4j.ConsoleAppender 4 | ## log4j.appender.stdlog.target=System.err 5 | log4j.appender.stdlog.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.stdlog.layout.ConversionPattern=%d{HH:mm:ss} %-5p %-20c{1} :: %m%n 7 | 8 | # Execution logging 9 | log4j.logger.org.apache.jena.arq.info=INFO 10 | log4j.logger.org.apache.jena.arq.exec=INFO 11 | 12 | # Lizard 13 | log4j.logger.lizard=INFO 14 | ## log4j.logger.lizard.comms.server=INFO 15 | ## log4j.logger.lizard.comms.client=INFO 16 | 17 | # To DEBUG 18 | 19 | log4j.logger.lizard.index.THandlerTupleIndex=WARN 20 | log4j.logger.lizard.index.TClientIndex=WARN 21 | log4j.logger.lizard.index.ClusterTupleIndex=WARN 22 | 23 | log4j.logger.lizard.node.THandlerNodeTable=WARN 24 | log4j.logger.lizard.node.TClientNode=WARN 25 | log4j.logger.lizard.node.ClusterNodeTable=WARN 26 | 27 | log4j.logger.lizard.lz-txn=WARN 28 | log4j.logger.lizard.lz-txn.NodeTableTxn=WARN 29 | log4j.logger.lizard.lz-txn.IndexTxn=WARN 30 | 31 | 32 | # TDB loader 33 | log4j.logger.org.apache.jena.tdb.loader=INFO 34 | # TDB syslog. 35 | log4j.logger.TDB=INFO 36 | 37 | # Everything else in Jena 38 | log4j.logger.org.apache.jena=INFO 39 | log4j.logger.org.apache.jena=WARN 40 | log4j.logger.org.apache.jena=INFO 41 | 42 | # Fuseki 43 | # Server log. 44 | log4j.logger.org.apache.jena.fuseki.Server=INFO 45 | # HTTP Request log. 46 | log4j.logger.org.apache.jena.fuseki.Fuseki=INFO 47 | # NCSA Format Request log. 48 | log4j.logger.org.apache.jena.fuseki.Request=OFF 49 | # Internal logs 50 | log4j.logger.org.apache.jena.fuseki=INFO 51 | 52 | # Zookeeper and friends 53 | log4j.logger.org.apache.zookeeper=WARN 54 | log4j.logger.org.apache.curator=WARN 55 | -------------------------------------------------------------------------------- /lz-dev/lz: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # May need to add in development specific setup here. 4 | 5 | # For now: 6 | # Temp fix until egit/jgit gets symbolic link handling 7 | LIZARD_HOME=${LIZARD_HOME:-${HOME}/Lizard} 8 | 9 | ${LIZARD_HOME}/lizard-cmds/bin/lz "$@" 10 | -------------------------------------------------------------------------------- /lz-dev/machine1.ttl: -------------------------------------------------------------------------------- 1 | # The cluster deployment descriptor. 2 | # This is used to override the configuration in the config files 3 | # with locations of servers for this machine. 4 | 5 | @base . 6 | @prefix : . 7 | @prefix lizard: . 8 | 9 | @prefix fuseki: . 10 | @prefix rdf: . 11 | @prefix rdfs: . 12 | @prefix tdb: . 13 | @prefix ja: . 14 | 15 | <#node-1> a :NodeServer ; 16 | :name "Node-1" ; 17 | :hostname "localhost" ; 18 | :port 4011 ; 19 | :data "N1" ; 20 | . 21 | 22 | # Index-1 23 | <#index-11> a :IndexServer ; 24 | :name "Index-11" ; 25 | :hostname "localhost" ; 26 | :port 4012 ; 27 | :data "I11" ; 28 | . 29 | 30 | # Index-2 31 | <#index-21> a :IndexServer ; 32 | :name "Index-21" ; 33 | :hostname "localhost" ; 34 | :port 4016 ; 35 | :data "I21" ; 36 | . 37 | -------------------------------------------------------------------------------- /lz-dev/machine2.ttl: -------------------------------------------------------------------------------- 1 | # The cluster deployment descriptor. 2 | # This is used to override the configuration in the config files 3 | # with locations of servers for this machine. 4 | 5 | @base . 6 | @prefix : . 7 | @prefix lizard: . 8 | 9 | @prefix fuseki: . 10 | @prefix rdf: . 11 | @prefix rdfs: . 12 | @prefix tdb: . 13 | @prefix ja: . 14 | 15 | <#node-2> a :NodeServer ; 16 | :name "Node-2" ; 17 | :hostname "localhost" ; 18 | :port 4013 ; 19 | :data "N2" ; 20 | . 21 | 22 | # Index-1 23 | <#index-12> a :IndexServer ; 24 | :name "Index-21" ; 25 | :hostname "localhost" ; 26 | :port 2014 ; 27 | :data "I21" ; 28 | . 29 | 30 | # Index-2 31 | <#index-22> a :IndexServer ; 32 | :name "Index-22" ; 33 | :hostname "localhost" ; 34 | :port 4018 ; 35 | :data "I22" ; 36 | . 37 | -------------------------------------------------------------------------------- /lz-dev/pushdata: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DATA=/home/afs/Datasets/MusicBrainz/artists.nt 4 | 5 | curl -XPOST \ 6 | --header "Content-type: text/turtle" \ 7 | --data-binary "@$DATA" \ 8 | http://localhost:3030/lz/data 9 | -------------------------------------------------------------------------------- /lz-dev/setup-yaml/config-dev.yaml: -------------------------------------------------------------------------------- 1 | # This file is the same as the built-in development configuration. 2 | # All in one JVM, in memory storage, single test zookeeper server, 3 | # one server each for POS, PSO and nodetable. 4 | 5 | cluster: 6 | zookeeper: [ "vnode1:2281" ] 7 | fileroot: "--mem--" 8 | 9 | ## sparql: 10 | ## servers: [ qs-1 ] 11 | ## 12 | ## qs-1: 13 | ## hostname: vnode1 14 | ## port: 3030 15 | 16 | dataset: 17 | indexes: [ index-pos, index-pso] 18 | nodes: nodetable 19 | 20 | # Index 21 | index-pos: 22 | name: "POS" 23 | servers: [pos-1] 24 | 25 | index-pso: 26 | name: "PSO" 27 | servers: [pso-1] 28 | 29 | #Index shards 30 | pos-1: 31 | name: "Server-POS" 32 | hostname: vnode1 33 | port: 4012 34 | data: "data/POS-1" 35 | 36 | pso-1: 37 | name: "Server-PSO" 38 | hostname: vnode1 39 | port: 4014 40 | data: "data/PSO-1" 41 | 42 | # Node table 43 | 44 | nodetable: 45 | servers: [ node-1 ] 46 | 47 | node-1: 48 | name: "ServerNode" 49 | hostname: vnode1 50 | port: 4011 51 | data: "data/N1" 52 | -------------------------------------------------------------------------------- /lz-dev/setup-yaml/config-disk-1.yaml: -------------------------------------------------------------------------------- 1 | # This file describes the whole Lizard system. 2 | # In addition, there is a separate description 3 | # of which machine the deployment is happeneing 4 | # on. 5 | 6 | cluster: 7 | zookeeper: [ "localhost:2181" ] 8 | fileroot: LZ 9 | 10 | ## sparql: 11 | ## servers: [ qs-1 ] 12 | ## 13 | ## qs-1: 14 | ## hostname: localhost 15 | ## port: 3030 16 | 17 | dataset: 18 | indexes: [index-pos, index-pso] 19 | nodes: nodetable 20 | 21 | # Index 22 | index-pos: 23 | name: "POS" 24 | servers: [pos-1] 25 | 26 | index-pso: 27 | name: "PSO" 28 | servers: [pso-1] 29 | 30 | #Index shards 31 | pos-1: 32 | hostname: localhost 33 | port: 4012 34 | name: "ServerPOS" 35 | data: "data1/pos" 36 | 37 | pso-1: 38 | hostname: localhost 39 | port: 4016 40 | name: "ServerPSO" 41 | data: "data1/pso" 42 | 43 | # Node table 44 | 45 | nodetable: 46 | servers: [ node-1 ] 47 | 48 | node-1: 49 | name: "ServerNode1" 50 | hostname: localhost 51 | port: 4011 52 | data: "data1/Nodes" 53 | -------------------------------------------------------------------------------- /lz-dev/setup-yaml/config-disk-2.yaml: -------------------------------------------------------------------------------- 1 | # This file describes the whole Lizard system. 2 | # In addition, there is a separate description 3 | # of which machine the deployment is happeneing 4 | # on. 5 | 6 | cluster: 7 | zookeeper: [ "localhost:2181" ] 8 | fileroot: LZ 9 | 10 | ## sparql: 11 | ## servers: [ qs-1 ] 12 | ## 13 | ## qs-1: 14 | ## hostname: localhost 15 | ## port: 3030 16 | 17 | dataset: 18 | indexes: [index-pos, index-pso] 19 | nodes: nodetable 20 | 21 | # Index 22 | index-pos: 23 | name: "POS" 24 | servers: [pos-1, pos-2] 25 | 26 | index-pso: 27 | name: "PSO" 28 | servers: [pso-1, pso-2] 29 | 30 | #Index shards 31 | pos-1: 32 | hostname: localhost 33 | port: 4012 34 | 35 | pos-2: 36 | hostname: localhost 37 | port: 4014 38 | 39 | pso-1: 40 | hostname: localhost 41 | port: 4016 42 | 43 | pso-2: 44 | hostname: localhost 45 | port: 4018 46 | 47 | # Node table 48 | 49 | nodetable: 50 | servers: [ node-1, node-2 ] 51 | 52 | # 2 servers, each with one segment. 53 | node-1: 54 | name: "ServerNode1" 55 | hostname: localhost 56 | port: 4011 57 | data: "N1" 58 | 59 | node-2: 60 | name: "ServerNode2" 61 | hostname: localhost 62 | port: 4013 63 | data: "N2" 64 | ## --- 65 | ## # Deployment part. -------------------------------------------------------------------------------- /lz-dev/setup-yaml/config-layout.yaml: -------------------------------------------------------------------------------- 1 | # Placement: mapping from virtual name to real address. 2 | 3 | zookeeper: [ 4 | { hostname: "localhost" , port: 2186 } 5 | ] 6 | 7 | 8 | vnodes: [ 9 | { vname: "vnode1" , hostname: "localhost" , port: 1100 } 10 | ] 11 | -------------------------------------------------------------------------------- /lz-dev/setup1/README.md: -------------------------------------------------------------------------------- 1 | This runs Lizard with one node server and one index server on 2 | two machines, with no index sharding. 3 | 4 | It is not fault tolerant. 5 | 6 | It is just for testing and performance measurements. 7 | 8 | *machine-all*: 9 | Deployment descriptor for all in one process. 10 | 11 | *machine-nodes, machines-indexes*: 12 | Deployment descriptors for separate node and index machines. 13 | -------------------------------------------------------------------------------- /lz-dev/setup1/conf-dataset.ttl: -------------------------------------------------------------------------------- 1 | base 2 | prefix : 3 | prefix lizard: 4 | 5 | prefix fuseki: 6 | prefix rdf: 7 | prefix rdfs: 8 | prefix tdb: 9 | prefix ja: 10 | 11 | ## ---- Client/Fuseki Assembler 12 | 13 | ## ## -- Init 14 | ## # TDB 15 | 16 | [] ja:loadClass "org.apache.jena.tdb.TDB" . 17 | tdb:DatasetTDB rdfs:subClassOf ja:RDFDataset . 18 | tdb:GraphTDB rdfs:subClassOf ja:Model . 19 | 20 | ## # Lizard 21 | [] ja:loadClass "lizard.Lizard" . 22 | lizard:Dataset rdfs:subClassOf ja:RDFDataset . 23 | 24 | ## Query engine / dataset 25 | 26 | <#dataset> rdf:type lizard:Dataset ; 27 | :indexes ( <#index1> <#index2> ) ; 28 | :nodetable <#node> ; 29 | . 30 | -------------------------------------------------------------------------------- /lz-dev/setup1/conf-index.ttl: -------------------------------------------------------------------------------- 1 | @base . 2 | @prefix : . 3 | 4 | @prefix fuseki: . 5 | @prefix rdf: . 6 | @prefix rdfs: . 7 | @prefix tdb: . 8 | @prefix ja: . 9 | 10 | # 2 services, SPO and POS 11 | # 1 server each. 12 | 13 | <#index1> rdf:type :IndexService ; 14 | :name "IndexSPO" ; 15 | :indexOrder "SPO" ; 16 | :servers ( <#index-s1> ) ; 17 | . 18 | 19 | <#index2> rdf:type :IndexService ; 20 | :name "IndexPOS" ; 21 | :indexOrder "POS" ; 22 | :servers ( <#index-s2> ) ; 23 | . 24 | 25 | <#index3> rdf:type :IndexService ; 26 | :name "IndexPSO" ; 27 | :indexOrder "PSO" ; 28 | :servers ( <#index-s3> ) ; 29 | . 30 | 31 | # index servers 32 | <#index-s1> a :IndexServer ; 33 | :name "ServerIndex-s1" ; 34 | :hostname "vnode1" ; 35 | :port 4012 ; 36 | :data "Idx1" ; # relative to the base location of installation. 37 | . 38 | 39 | <#index-s2> a :IndexServer ; 40 | :name "ServerIndex-s2" ; 41 | :hostname "vnode1" ; 42 | :port 4014 ; 43 | :data "Idx2" ; 44 | . 45 | 46 | <#index-s3> a :IndexServer ; 47 | :name "ServerIndex-s3" ; 48 | :hostname "vnode1" ; 49 | :port 4016 ; 50 | :data "Idx3" ; 51 | . 52 | -------------------------------------------------------------------------------- /lz-dev/setup1/conf-layout.ttl: -------------------------------------------------------------------------------- 1 | base 2 | prefix : 3 | prefix lizard: 4 | 5 | prefix rdf: 6 | prefix rdfs: 7 | prefix tdb: 8 | prefix ja: 9 | 10 | # Zookeeper 11 | 12 | :zookeeper 13 | :server [ :hostname "localhost" ; :port 2186 ] ; 14 | . 15 | 16 | # Placement: mapping from virtual name to real address. 17 | 18 | [ :vname "vnode1" ; :hostname "localhost" ; :port 1100 ] . 19 | [ :vname "vnode2" ; :hostname "localhost" ; :port 1101 ] . 20 | -------------------------------------------------------------------------------- /lz-dev/setup1/conf-node.ttl: -------------------------------------------------------------------------------- 1 | @base . 2 | @prefix : . 3 | 4 | @prefix fuseki: . 5 | @prefix rdf: . 6 | @prefix rdfs: . 7 | @prefix tdb: . 8 | @prefix ja: . 9 | 10 | # 1 service, 1 server. 11 | <#node> a :NodeService ; 12 | :name "Node" ; 13 | :servers ( <#node-server> ) ; 14 | . 15 | 16 | <#node-server> a :NodeServer ; 17 | :name "ServerNode" ; 18 | :hostname "vnode1" ; 19 | :port 4011 ; 20 | :data "N" ; 21 | . 22 | -------------------------------------------------------------------------------- /lz-dev/setup1/fuseki-server.ttl: -------------------------------------------------------------------------------- 1 | base 2 | prefix : 3 | prefix lizard: 4 | 5 | prefix fuseki: 6 | prefix rdf: 7 | prefix rdfs: 8 | prefix tdb: 9 | prefix ja: 10 | 11 | 12 | ## Lizard 13 | [] ja:loadClass "lizard.sys.Lizard" . 14 | lizard:Dataset rdfs:subClassOf ja:RDFDataset . 15 | 16 | [] rdf:type fuseki:Server ; 17 | fuseki:services ( 18 | <#service1> 19 | ) . 20 | 21 | <#service1> rdf:type fuseki:Service ; 22 | rdfs:label "Lizard" ; 23 | fuseki:name "lz" ; 24 | fuseki:serviceQuery "query" ; 25 | fuseki:serviceQuery "sparql" ; 26 | fuseki:serviceQuery "query" ; 27 | fuseki:serviceUpdate "update" ; 28 | fuseki:serviceUpload "upload" ; 29 | fuseki:serviceReadWriteGraphStore "data" ; 30 | fuseki:serviceReadGraphStore "get" ; 31 | fuseki:dataset <#dataset> ; 32 | . 33 | ## ------------ Lizard 34 | 35 | <#dataset> rdf:type lizard:Dataset ; 36 | :indexes ( <#index1> <#index2> ) ; 37 | :nodetable <#node> ; 38 | . 39 | 40 | ## ------------ Indexes 41 | 42 | <#index1> rdf:type :IndexService ; 43 | :name "IndexSPO" ; 44 | :indexOrder "SPO" ; 45 | :servers ( <#index-s1> ) ; 46 | . 47 | 48 | <#index2> rdf:type :IndexService ; 49 | :name "IndexPOS" ; 50 | :indexOrder "POS" ; 51 | :servers ( <#index-s2> ) ; 52 | . 53 | 54 | <#index-s1> a :IndexServer ; 55 | :name "ServerIndex11" ; 56 | :hostname "localhost" ; 57 | :port 4012 ; 58 | :data "DB/Idx1" ; 59 | . 60 | 61 | <#index-s2> a :IndexServer ; 62 | :name "ServerIndex12" ; 63 | :hostname "localhost" ; 64 | :port 4014 ; 65 | :data "DB/Idx2" ; 66 | . 67 | 68 | ## ------------ Nodes 69 | 70 | <#node> a :NodeService ; 71 | :name "Node" ; 72 | :servers ( <#node-server> ) ; 73 | . 74 | 75 | <#node-server> a :NodeServer ; 76 | :name "ServerNode" ; 77 | :hostname "localhost" ; 78 | :port 4011 ; 79 | :data "DB/N" ; 80 | . 81 | -------------------------------------------------------------------------------- /lz-dev/setup1/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, stdlog 2 | 3 | log4j.appender.stdlog=org.apache.log4j.ConsoleAppender 4 | ## log4j.appender.stdlog.target=System.err 5 | log4j.appender.stdlog.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.stdlog.layout.ConversionPattern=%d{HH:mm:ss} %-5p %-20c{1} :: %m%n 7 | 8 | # Execution logging 9 | log4j.logger.org.apache.jena.arq.info=INFO 10 | log4j.logger.org.apache.jena.arq.exec=INFO 11 | 12 | # Lizard 13 | log4j.logger.lizard=INFO 14 | ## log4j.logger.lizard.comms.server=INFO 15 | ## log4j.logger.lizard.comms.client=INFO 16 | 17 | # TDB loader 18 | log4j.logger.org.apache.jena.tdb.loader=INFO 19 | # TDB syslog. 20 | log4j.logger.TDB=INFO 21 | 22 | # Everything else in Jena 23 | log4j.logger.org.apache.jena=INFO 24 | log4j.logger.org.apache.jena=WARN 25 | log4j.logger.org.apache.jena=INFO 26 | 27 | # Fuseki 28 | # Server log. 29 | log4j.logger.org.apache.jena.fuseki.Server=INFO 30 | # Request log. 31 | log4j.logger.org.apache.jena.fuseki.Fuseki=INFO 32 | # Internal logs 33 | log4j.logger.org.apache.jena.fuseki=INFO 34 | 35 | # Zookeeper and friends 36 | log4j.logger.org.apache.zookeeper=WARN 37 | log4j.logger.org.apache.curator=WARN 38 | -------------------------------------------------------------------------------- /lz-dev/setup1/lz-fuseki: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ../fu --conf fuseki.ttl 4 | 5 | -------------------------------------------------------------------------------- /lz-dev/setup1/lz-servers: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ../lz deploy --deploy deploy-jvm.ttl conf-* 4 | -------------------------------------------------------------------------------- /lz-dev/setup1a/README.md: -------------------------------------------------------------------------------- 1 | This runs Lizard with one node server and one index server on 2 | two machines, with no index sharding. 3 | 4 | It is not fault tolerant. 5 | 6 | It is just for testing and performance measurements. 7 | 8 | *machine-all*: 9 | Deployment descriptor for all in one process. 10 | 11 | *machine-nodes, machines-indexes*: 12 | Deployment descriptors for separate node and index machines. 13 | -------------------------------------------------------------------------------- /lz-dev/setup1a/conf-dataset.ttl: -------------------------------------------------------------------------------- 1 | base 2 | prefix : 3 | prefix lizard: 4 | 5 | prefix fuseki: 6 | prefix rdf: 7 | prefix rdfs: 8 | prefix tdb: 9 | prefix ja: 10 | 11 | ## ---- Client/Fuseki Assembler 12 | 13 | ## ## -- Init 14 | ## # TDB 15 | 16 | [] ja:loadClass "org.apache.jena.tdb.TDB" . 17 | tdb:DatasetTDB rdfs:subClassOf ja:RDFDataset . 18 | tdb:GraphTDB rdfs:subClassOf ja:Model . 19 | 20 | ## # Lizard 21 | [] ja:loadClass "lizard.Lizard" . 22 | lizard:Dataset rdfs:subClassOf ja:RDFDataset . 23 | 24 | ## Query engine / dataset 25 | 26 | <#dataset> rdf:type lizard:Dataset ; 27 | :indexes ( <#index1> <#index2> ) ; 28 | :nodetable <#node> ; 29 | . 30 | -------------------------------------------------------------------------------- /lz-dev/setup1a/conf-index.ttl: -------------------------------------------------------------------------------- 1 | @base . 2 | @prefix : . 3 | 4 | @prefix fuseki: . 5 | @prefix rdf: . 6 | @prefix rdfs: . 7 | @prefix tdb: . 8 | @prefix ja: . 9 | 10 | # 2 services, SPO and POS 11 | # 1 server each. 12 | 13 | <#index1> rdf:type :IndexService ; 14 | :name "IndexSPO" ; 15 | :indexOrder "SPO" ; 16 | :servers ( <#index-s1> ) ; 17 | . 18 | 19 | <#index2> rdf:type :IndexService ; 20 | :name "IndexPOS" ; 21 | :indexOrder "POS" ; 22 | :servers ( <#index-s2> ) ; 23 | . 24 | 25 | <#index3> rdf:type :IndexService ; 26 | :name "IndexPSO" ; 27 | :indexOrder "PSO" ; 28 | :servers ( <#index-s3> ) ; 29 | . 30 | 31 | # 2 servers 32 | <#index-s1> a :IndexServer ; 33 | :name "ServerIndex-s1" ; 34 | :hostname "localhost" ; 35 | :port 4012 ; 36 | :data "Idx1" ; # relative to the base location of installation. 37 | . 38 | 39 | <#index-s2> a :IndexServer ; 40 | :name "ServerIndex-s2" ; 41 | :hostname "localhost" ; 42 | :port 4014 ; 43 | :data "Idx2" ; 44 | . 45 | 46 | <#index-s3> a :IndexServer ; 47 | :name "ServerIndex-s3" ; 48 | :hostname "localhost" ; 49 | :port 4016 ; 50 | :data "Idx3" ; 51 | . 52 | 53 | -------------------------------------------------------------------------------- /lz-dev/setup1a/conf-node.ttl: -------------------------------------------------------------------------------- 1 | @base . 2 | @prefix : . 3 | 4 | @prefix fuseki: . 5 | @prefix rdf: . 6 | @prefix rdfs: . 7 | @prefix tdb: . 8 | @prefix ja: . 9 | 10 | # 1 service, 1 server. 11 | <#node> a :NodeService ; 12 | :name "Node" ; 13 | :servers ( <#node-server> ) ; 14 | . 15 | 16 | <#node-server> a :NodeServer ; 17 | :name "ServerNode" ; 18 | :hostname "localhost" ; 19 | :port 4011 ; 20 | :data "N" ; 21 | . 22 | -------------------------------------------------------------------------------- /lz-dev/setup1a/fuseki.ttl: -------------------------------------------------------------------------------- 1 | base 2 | prefix : 3 | prefix lizard: 4 | 5 | prefix fuseki: 6 | prefix rdf: 7 | prefix rdfs: 8 | prefix tdb: 9 | prefix ja: 10 | 11 | 12 | ## Lizard 13 | [] ja:loadClass "lizard.sys.Lizard" . 14 | lizard:Dataset rdfs:subClassOf ja:RDFDataset . 15 | 16 | [] rdf:type fuseki:Server ; 17 | fuseki:services ( 18 | <#service1> 19 | ) . 20 | 21 | <#service1> rdf:type fuseki:Service ; 22 | rdfs:label "Lizard" ; 23 | fuseki:name "lz" ; 24 | fuseki:serviceQuery "query" ; 25 | fuseki:serviceQuery "sparql" ; 26 | fuseki:serviceQuery "query" ; 27 | fuseki:serviceUpdate "update" ; 28 | fuseki:serviceUpload "upload" ; 29 | fuseki:serviceReadWriteGraphStore "data" ; 30 | fuseki:serviceReadGraphStore "get" ; 31 | fuseki:dataset <#dataset> ; 32 | . 33 | ## ------------ Lizard 34 | 35 | <#dataset> rdf:type lizard:Dataset ; 36 | :indexes ( <#index1> <#index2> ) ; 37 | :nodetable <#node> ; 38 | . 39 | 40 | ## ------------ Indexes 41 | 42 | <#index1> rdf:type :IndexService ; 43 | :name "IndexSPO" ; 44 | :indexOrder "SPO" ; 45 | :servers ( <#index-s1> ) ; 46 | . 47 | 48 | <#index2> rdf:type :IndexService ; 49 | :name "IndexPOS" ; 50 | :indexOrder "POS" ; 51 | :servers ( <#index-s2> ) ; 52 | . 53 | 54 | <#index-s1> a :IndexServer ; 55 | :name "ServerIndex11" ; 56 | :hostname "localhost" ; 57 | :port 4012 ; 58 | :data "DB/Idx1" ; 59 | . 60 | 61 | <#index-s2> a :IndexServer ; 62 | :name "ServerIndex12" ; 63 | :hostname "localhost" ; 64 | :port 4014 ; 65 | :data "DB/Idx2" ; 66 | . 67 | 68 | ## ------------ Nodes 69 | 70 | <#node> a :NodeService ; 71 | :name "Node" ; 72 | :servers ( <#node-server> ) ; 73 | . 74 | 75 | <#node-server> a :NodeServer ; 76 | :name "ServerNode" ; 77 | :hostname "localhost" ; 78 | :port 4011 ; 79 | :data "DB/N" ; 80 | . 81 | -------------------------------------------------------------------------------- /lz-dev/setup1a/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, stdlog 2 | 3 | log4j.appender.stdlog=org.apache.log4j.ConsoleAppender 4 | ## log4j.appender.stdlog.target=System.err 5 | log4j.appender.stdlog.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.stdlog.layout.ConversionPattern=%d{HH:mm:ss} %-5p %-20c{1} :: %m%n 7 | 8 | # Execution logging 9 | log4j.logger.org.apache.jena.arq.info=INFO 10 | log4j.logger.org.apache.jena.arq.exec=INFO 11 | 12 | # Lizard 13 | log4j.logger.lizard=INFO 14 | ## log4j.logger.lizard.comms.server=INFO 15 | ## log4j.logger.lizard.comms.client=INFO 16 | 17 | # TDB loader 18 | log4j.logger.org.apache.jena.tdb.loader=INFO 19 | # TDB syslog. 20 | log4j.logger.TDB=INFO 21 | 22 | # Everything else in Jena 23 | log4j.logger.org.apache.jena=INFO 24 | log4j.logger.org.apache.jena=WARN 25 | log4j.logger.org.apache.jena=INFO 26 | 27 | # Fuseki 28 | # Server log. 29 | log4j.logger.org.apache.jena.fuseki.Server=INFO 30 | # Request log. 31 | log4j.logger.org.apache.jena.fuseki.Fuseki=INFO 32 | # Internal logs 33 | log4j.logger.org.apache.jena.fuseki=INFO 34 | 35 | # Zookeeper and friends 36 | log4j.logger.org.apache.zookeeper=WARN 37 | log4j.logger.org.apache.curator=WARN 38 | -------------------------------------------------------------------------------- /lz-dev/setup1a/lz-fuseki: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ../fu --conf fuseki.ttl 4 | 5 | -------------------------------------------------------------------------------- /lz-dev/setup1a/lz-servers: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ../lz deploy --deploy deploy-jvm.ttl conf-* 4 | -------------------------------------------------------------------------------- /lz-dev/setup2/README.md: -------------------------------------------------------------------------------- 1 | This runs Lizard with one node server and one index server on 2 | two machines, with no index sharding. 3 | 4 | It is not fault tolerant. 5 | 6 | It is just for testing and performance measurements. 7 | 8 | *machine-all*: 9 | Deployment descriptor for all in one process. 10 | 11 | *machine-nodes, machines-indexes*: 12 | Deployment descriptors for separate node and index machines. 13 | -------------------------------------------------------------------------------- /lz-dev/setup2/conf-dataset.ttl: -------------------------------------------------------------------------------- 1 | base 2 | prefix : 3 | prefix lizard: 4 | 5 | prefix fuseki: 6 | prefix rdf: 7 | prefix rdfs: 8 | prefix tdb: 9 | prefix ja: 10 | 11 | ## ---- Client/Fuseki Assembler 12 | 13 | ## ## -- Init 14 | ## # TDB 15 | 16 | [] ja:loadClass "org.apache.jena.tdb.TDB" . 17 | tdb:DatasetTDB rdfs:subClassOf ja:RDFDataset . 18 | tdb:GraphTDB rdfs:subClassOf ja:Model . 19 | 20 | ## # Lizard 21 | [] ja:loadClass "lizard.Lizard" . 22 | lizard:Dataset rdfs:subClassOf ja:RDFDataset . 23 | 24 | ## Query engine / dataset 25 | 26 | <#dataset> rdf:type lizard:Dataset ; 27 | :indexes ( <#index1> <#index2> ) ; 28 | :nodetable <#node> ; 29 | . 30 | -------------------------------------------------------------------------------- /lz-dev/setup2/conf-index.ttl: -------------------------------------------------------------------------------- 1 | @base . 2 | @prefix : . 3 | 4 | @prefix fuseki: . 5 | @prefix rdf: . 6 | @prefix rdfs: . 7 | @prefix tdb: . 8 | @prefix ja: . 9 | 10 | # Replicated. 11 | 12 | <#index1> rdf:type :IndexService ; 13 | :name "IndexSPO" ; 14 | :indexOrder "SPO" ; 15 | :servers ( <#index-s11> <#index-s12> ) ; 16 | . 17 | 18 | <#index2> rdf:type :IndexService ; 19 | :name "IndexPOS" ; 20 | :indexOrder "POS" ; 21 | :servers ( <#index-s21> <#index-s22> ) ; 22 | . 23 | 24 | ## <#index3> rdf:type :IndexService ; 25 | ## :name "IndexPSO" ; 26 | ## :indexOrder "PSO" ; 27 | ## :servers ( <#index-s31> <#index-s32> ) ; 28 | ## . 29 | 30 | # index servers 1 31 | <#index-s11> a :IndexServer ; 32 | :name "ServerIndex-s11" ; 33 | :hostname "vnode1" ; 34 | :port 4012 ; 35 | :data "Idx11" ; 36 | . 37 | 38 | <#index-s21> a :IndexServer ; 39 | :name "ServerIndex-s21" ; 40 | :hostname "vnode1" ; 41 | :port 4013 ; 42 | :data "Idx21" ; 43 | . 44 | 45 | ## <#index-s31> a :IndexServer ; 46 | ## :name "ServerIndex-s31" ; 47 | ## :hostname "vnode1" ; 48 | ## :port 4014 ; 49 | ## :data "Idx31" ; 50 | ## . 51 | 52 | # index servers 2 53 | <#index-s12> a :IndexServer ; 54 | :name "ServerIndex-s12" ; 55 | :hostname "vnode2" ; 56 | :port 4015 ; 57 | :data "Idx12" ; 58 | . 59 | 60 | <#index-s22> a :IndexServer ; 61 | :name "ServerIndex-s22" ; 62 | :hostname "vnode2" ; 63 | :port 4016 ; 64 | :data "Idx22" ; 65 | . 66 | 67 | ## <#index-s32> a :IndexServer ; 68 | ## :name "ServerIndex-s32" ; 69 | ## :hostname "vnode2" ; 70 | ## :port 4017 ; 71 | ## :data "Idx32" ; 72 | ## . 73 | -------------------------------------------------------------------------------- /lz-dev/setup2/conf-layout.ttl: -------------------------------------------------------------------------------- 1 | base 2 | prefix : 3 | prefix lizard: 4 | 5 | prefix rdf: 6 | prefix rdfs: 7 | prefix tdb: 8 | prefix ja: 9 | 10 | # Zookeeper 11 | 12 | :zookeeper 13 | :server [ :hostname "localhost" ; :port 2186 ] ; 14 | . 15 | 16 | # Placement: mapping from virtual name to real address. 17 | 18 | [ :vname "vnode1" ; :hostname "localhost" ; :port 1100 ; :fileroot "DB1" ] . 19 | [ :vname "vnode2" ; :hostname "localhost" ; :port 1101 ; :fileroot "DB2" ] . 20 | -------------------------------------------------------------------------------- /lz-dev/setup2/conf-node.ttl: -------------------------------------------------------------------------------- 1 | @base . 2 | @prefix : . 3 | 4 | @prefix fuseki: . 5 | @prefix rdf: . 6 | @prefix rdfs: . 7 | @prefix tdb: . 8 | @prefix ja: . 9 | 10 | # replicated. 11 | 12 | <#node> a :NodeService ; 13 | :name "Node" ; 14 | :servers ( <#node-server1> <#node-server2> ) ; 15 | . 16 | 17 | <#node-server1> a :NodeServer ; 18 | :name "ServerNode" ; 19 | :hostname "vnode1" ; 20 | :port 4021 ; 21 | :data "N1" ; 22 | . 23 | 24 | <#node-server2> a :NodeServer ; 25 | :name "ServerNode" ; 26 | :hostname "vnode2" ; 27 | :port 4022 ; 28 | :data "N2" ; 29 | . 30 | -------------------------------------------------------------------------------- /lz-dev/setup2/fuseki-server.ttl: -------------------------------------------------------------------------------- 1 | base 2 | prefix : 3 | prefix lizard: 4 | 5 | prefix fuseki: 6 | prefix rdf: 7 | prefix rdfs: 8 | prefix tdb: 9 | prefix ja: 10 | 11 | 12 | ## Lizard 13 | [] ja:loadClass "lizard.sys.Lizard" . 14 | lizard:Dataset rdfs:subClassOf ja:RDFDataset . 15 | 16 | [] rdf:type fuseki:Server ; 17 | fuseki:services ( 18 | <#service1> 19 | ) . 20 | 21 | <#service1> rdf:type fuseki:Service ; 22 | rdfs:label "Lizard" ; 23 | fuseki:name "lz" ; 24 | fuseki:serviceQuery "query" ; 25 | fuseki:serviceQuery "sparql" ; 26 | fuseki:serviceQuery "query" ; 27 | fuseki:serviceUpdate "update" ; 28 | fuseki:serviceUpload "upload" ; 29 | fuseki:serviceReadWriteGraphStore "data" ; 30 | fuseki:serviceReadGraphStore "get" ; 31 | fuseki:dataset <#dataset> ; 32 | . 33 | ## ------------ Lizard 34 | 35 | <#dataset> rdf:type lizard:Dataset ; 36 | :indexes ( <#index1> <#index2> ) ; 37 | :nodetable <#node> ; 38 | . 39 | 40 | ## ------------ Indexes 41 | 42 | <#index1> rdf:type :IndexService ; 43 | :name "IndexSPO" ; 44 | :indexOrder "SPO" ; 45 | :servers ( <#index-s1> ) ; 46 | . 47 | 48 | <#index2> rdf:type :IndexService ; 49 | :name "IndexPOS" ; 50 | :indexOrder "POS" ; 51 | :servers ( <#index-s2> ) ; 52 | . 53 | 54 | <#index-s1> a :IndexServer ; 55 | :name "ServerIndex11" ; 56 | :hostname "localhost" ; 57 | :port 4012 ; 58 | :data "DB/Idx1" ; 59 | . 60 | 61 | <#index-s2> a :IndexServer ; 62 | :name "ServerIndex12" ; 63 | :hostname "localhost" ; 64 | :port 4014 ; 65 | :data "DB/Idx2" ; 66 | . 67 | 68 | ## ------------ Nodes 69 | 70 | <#node> a :NodeService ; 71 | :name "Node" ; 72 | :servers ( <#node-server> ) ; 73 | . 74 | 75 | <#node-server> a :NodeServer ; 76 | :name "ServerNode" ; 77 | :hostname "localhost" ; 78 | :port 4011 ; 79 | :data "DB/N" ; 80 | . 81 | -------------------------------------------------------------------------------- /lz-dev/setup2/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, stdlog 2 | 3 | log4j.appender.stdlog=org.apache.log4j.ConsoleAppender 4 | ## log4j.appender.stdlog.target=System.err 5 | log4j.appender.stdlog.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.stdlog.layout.ConversionPattern=%d{HH:mm:ss} %-5p %-20c{1} :: %m%n 7 | 8 | # Execution logging 9 | log4j.logger.org.apache.jena.arq.info=INFO 10 | log4j.logger.org.apache.jena.arq.exec=INFO 11 | 12 | # Lizard 13 | log4j.logger.lizard=INFO 14 | ## log4j.logger.lizard.comms.server=INFO 15 | ## log4j.logger.lizard.comms.client=INFO 16 | 17 | # TDB loader 18 | log4j.logger.org.apache.jena.tdb.loader=INFO 19 | # TDB syslog. 20 | log4j.logger.TDB=INFO 21 | 22 | # Everything else in Jena 23 | log4j.logger.org.apache.jena=INFO 24 | log4j.logger.org.apache.jena=WARN 25 | log4j.logger.org.apache.jena=INFO 26 | 27 | # Fuseki 28 | # Server log. 29 | log4j.logger.org.apache.jena.fuseki.Server=INFO 30 | # Request log. 31 | log4j.logger.org.apache.jena.fuseki.Fuseki=INFO 32 | # Internal logs 33 | log4j.logger.org.apache.jena.fuseki=INFO 34 | 35 | # Zookeeper and friends 36 | log4j.logger.org.apache.zookeeper=WARN 37 | log4j.logger.org.apache.curator=WARN 38 | -------------------------------------------------------------------------------- /lz-dev/setup2/lz-fuseki: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ../fu --conf fuseki.ttl 4 | 5 | -------------------------------------------------------------------------------- /lz-dev/setup2/lz-servers: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ../lz deploy --deploy deploy-jvm.ttl conf-* 4 | -------------------------------------------------------------------------------- /lz-dev/src-header-text: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | -------------------------------------------------------------------------------- /lz-dev/src/main/java/conf2/Lizard2.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package conf2; 19 | 20 | import lizard.conf.assembler.AssemblerYaml ; 21 | import lizard.conf.assembler.VocabLizard ; 22 | import lizard.query.LizardQuery ; 23 | import lizard.system.LzLog ; 24 | 25 | import org.apache.jena.assembler.Assembler ; 26 | 27 | public class Lizard2 { 28 | static { init() ; } // Via assembler. 29 | public synchronized static void init() { 30 | LzLog.logSystem.info("Lizard.init") ; 31 | //VocabLizard.init(); 32 | Assembler.general.implementWith(VocabLizard.lzDataset, new AssemblerYaml()) ; 33 | LizardQuery.init(); 34 | } 35 | } 36 | 37 | -------------------------------------------------------------------------------- /lz-dev/src/main/java/conf2/LzConfigDefault.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package conf2; 19 | 20 | import lizard.conf.* ; 21 | import org.apache.jena.atlas.lib.tuple.TupleMap ; 22 | 23 | /** An assortment of ConfCluster configurations */ 24 | public class LzConfigDefault { 25 | 26 | // Setup for development : one of each, all one JVM; one local zookeeper. 27 | public static ConfCluster setup_mem_local() { 28 | int port = 2180 ; 29 | int zkPort = 2188 ; 30 | 31 | // Dataset 32 | ConfNodeTable confNT = new ConfNodeTable(1, 1) ; 33 | ConfIndex posIdx = new ConfIndex(TupleMap.create("SPO", "POS"), "POS", 1, 1) ; 34 | ConfIndex psoIdx = new ConfIndex(TupleMap.create("SPO", "PSO"), "PSO", 1, 1) ; 35 | ConfDataset confDatabase = new ConfDataset(confNT, posIdx, psoIdx) ; 36 | 37 | // Shards 38 | ConfIndexElement posIdx1 = new ConfIndexElement(posIdx.indexOrder+"-1", "POS-1", posIdx, VNodeAddr.create("local", 2010)) ; 39 | ConfIndexElement psoIdx1 = new ConfIndexElement(psoIdx.indexOrder+"-1", "PSO-1", psoIdx, VNodeAddr.create("local", 2012)) ; 40 | ConfNodeTableElement nt1 = new ConfNodeTableElement("Nodes-1", "N1", confNT, VNodeAddr.create("local", 2014)) ; 41 | 42 | // The zookeeper server. 43 | ConfZookeeper confZookeeper = ConfZookeeper.create("localhost", 2186) ; 44 | 45 | // Cluster 46 | ConfCluster confCluster = new ConfCluster(confDatabase) ; 47 | VNode vNode = new VNode("local", NetAddr.create("localhost", port), null) ; 48 | confCluster.placements.put("local", vNode) ; 49 | confCluster.zkServer.add(confZookeeper) ; 50 | confCluster.addIndexElements(posIdx1, psoIdx1) ; 51 | confCluster.addNodeElements(nt1) ; 52 | return confCluster ; 53 | } 54 | 55 | } 56 | 57 | -------------------------------------------------------------------------------- /lz-dev/src/main/java/lizard/index/tuple/TupleIndexConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lizard.index.tuple; 19 | 20 | import java.util.Collection ; 21 | import java.util.Iterator ; 22 | 23 | import org.apache.jena.atlas.lib.tuple.Tuple ; 24 | import org.apache.jena.atlas.lib.tuple.TupleMap ; 25 | import org.seaborne.dboe.base.record.Record ; 26 | import org.seaborne.tdb2.store.NodeId ; 27 | import org.seaborne.tdb2.store.tupletable.TupleIndexBase ; 28 | 29 | /** Framework for general mapping of Tuples and base storage. 30 | * No recording. 31 | */ 32 | public class TupleIndexConverter extends TupleIndexBase { 33 | 34 | interface Converter { 35 | R tupleToRecord(Tuple tuple) ; 36 | Tuple recordToTuple(R record) ; 37 | } 38 | 39 | Converter converter = new Converter() { 40 | @Override 41 | public Record tupleToRecord(Tuple tuple) { 42 | return null ; 43 | } 44 | 45 | @Override 46 | public Tuple recordToTuple(Record record) { 47 | return null ; 48 | }} ; 49 | 50 | protected TupleIndexConverter(int N, TupleMap tupleMap, String name) { 51 | super(N, tupleMap, name) ; 52 | } 53 | 54 | @Override 55 | public Iterator> all() { 56 | return null ; 57 | } 58 | 59 | @Override 60 | public long size() { 61 | return 0 ; 62 | } 63 | 64 | @Override 65 | public boolean isEmpty() { 66 | return false ; 67 | } 68 | 69 | @Override 70 | public void clear() {} 71 | 72 | @Override 73 | public void sync() {} 74 | 75 | @Override 76 | public void close() {} 77 | 78 | @Override 79 | public void addAll(Collection> tuples) {} 80 | 81 | @Override 82 | public void deleteAll(Collection> tuples) {} 83 | 84 | @Override 85 | protected void performAdd(Tuple tuple) { 86 | } 87 | 88 | @Override 89 | protected void performDelete(Tuple tuple) { 90 | } 91 | 92 | @Override 93 | protected Iterator> performFind(Tuple tuple) { 94 | return null ; 95 | } 96 | } 97 | 98 | -------------------------------------------------------------------------------- /lz-dev/src/main/java/lz_dev/LzMain.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lz_dev; 19 | 20 | import java.io.IOException ; 21 | import java.nio.file.DirectoryStream ; 22 | import java.nio.file.Files ; 23 | import java.nio.file.Path ; 24 | import java.nio.file.Paths ; 25 | import java.util.ArrayList ; 26 | import java.util.List ; 27 | 28 | import lz.lz ; 29 | 30 | import org.apache.jena.atlas.logging.LogCtl ; 31 | 32 | public class LzMain { 33 | static { LogCtl.setLog4j(); } 34 | public static void main(String[] argv) throws IOException { 35 | 36 | List args = new ArrayList<>() ; 37 | args.add("deploy") ; 38 | args.add("--server=vnode1") ; 39 | args.add("--fuseki=3030") ; 40 | 41 | try ( DirectoryStream directory = Files.newDirectoryStream(Paths.get("setup2"), "conf*ttl") ) { 42 | directory.forEach(p->args.add(p.toString())); 43 | } 44 | 45 | 46 | lz.main(args.toArray(new String[0])) ; 47 | } 48 | } 49 | 50 | -------------------------------------------------------------------------------- /lz-dev/src/main/java/lz_dev/PROJECT_Lizard.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | * 14 | * See the NOTICE file distributed with this work for additional 15 | * information regarding copyright ownership. 16 | */ 17 | 18 | package lz_dev ; 19 | 20 | public class PROJECT_Lizard { 21 | // Untested since ColumnMap -> TupleMap. 22 | 23 | // Multiple deployment. 24 | // Debug killing a server. 25 | // Does not recover. 26 | // Info begine-end transaction 27 | // Second bad operation blocks? 28 | // Read quorum vs write quorum 29 | // Much better handling of failure cases of distributed transactions. 30 | 31 | // Pref "local" indexes? 32 | 33 | // TCP connections: 34 | // TCP_QUICKACK (needs reset on every use) and TCP_NODELAY 35 | 36 | // *** TxnClient is currently doing the synchonization 37 | // Sort out ThriftRunnable / Runnable. 38 | // maybe a fixed class to catch+convert Thrift to LizardExceptions 39 | 40 | // NodeTableCache and transactions. 41 | 42 | // Reduce overheads - wire to node table non-conversion. 43 | 44 | // === Tasks 45 | // Update replics in parallel. 46 | 47 | // TxnClient. 48 | // TServer(Node|Index) -- what about locking operations? 49 | 50 | // THandlerNodeTable can store TLZ forms. 51 | // Not a node table but an Index+ObjectFileStorage directly. 52 | // Don't decode/encode. 53 | // NodeTableNative : BiMap[T] 54 | 55 | // Scaling: 56 | // A - Fix locks on ThriftLib. 57 | // B - Batching updates (and flow reversal) : switch between R and W modes. 58 | // C - Block allocate txn ids but still need the lock. 59 | 60 | // Node caching 61 | // 1 - move to hash NodeId 62 | // 2 - move to 10 byte NodeIds 63 | // Special Ids for rdf: rdfs: (xsd:) URIs? 64 | 65 | // Very large scans. Batching in replies 66 | 67 | // ---- Configuration 68 | // Robust parsing of YAML/RDF configurations. 69 | // Component naming: 70 | // location/Index/... 71 | // location/Nodes/... 72 | 73 | // ---- Configuration 74 | // Check/get LZ_* commands working. 75 | // LZ_Deploy + s-put. 76 | 77 | // prepare-abort testing. 78 | // From Mantis: Component has W lifecycle and an R lifecycle 79 | 80 | // README to wiki 81 | 82 | // Thrift : Other servers - TNonblockingServer+TFramedTransport 83 | 84 | // Failures 85 | // Protocol message "go away" to simulate failures 86 | 87 | // 1. RemoteAcces improvements. Include a filter? predicate list Jump start 88 | // find(...) from S=foo ==> filter between X and Y 89 | // Stream find() 90 | 91 | // index to index copy "tdbidxcopy --loc=DB SPO POS no overwrite 2. 92 | // splitter : conf+existing TDB -> set of directories. 3. zloader 93 | } 94 | -------------------------------------------------------------------------------- /lz-setup/README.md: -------------------------------------------------------------------------------- 1 | Material to help setup development systems for Lizard. 2 | 3 | * Don't forget to set user/email in your `.git/config` if `${HOME}/.gitconfig` is different 4 | * A `.gitignore` 5 | * `org.eclipse.jdt.ui.prefs` : set the template to create the right Java headers (copyright/license) for new files. 6 | 7 | ``` 8 | cd $LIZARD_HOME 9 | for m in lizard-* lz-* 10 | do 11 | cp $LIZARD_HOME/lz-setup/org.eclipse.jdt.ui.prefs $m/.settings 12 | done 13 | ``` 14 | -------------------------------------------------------------------------------- /lz-setup/gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | 3 | # Package Files # 4 | *.jar 5 | *.war 6 | *.ear 7 | 8 | # Eclipse 9 | .classpath 10 | .project 11 | .settings 12 | /classes 13 | 14 | # maven 15 | target 16 | 17 | # Development 18 | # Emacs files 19 | \#* 20 | .#* 21 | *.bak 22 | /dev 23 | /run 24 | /webapp 25 | webapp 26 | 27 | # Development Data and Database 28 | DB* 29 | # Durecing initial development, allow small data test files. 30 | ## D.ttl 31 | ## D.trig 32 | ## D.nt 33 | ## D.nq 34 | /--mem-- 35 | /classes 36 | /tmp 37 | /LZ 38 | /LZ? 39 | /lizard-cmds/lizard.classpath 40 | --------------------------------------------------------------------------------