├── .gitignore ├── CREDITS ├── DISCLAIMER ├── HISTORY.md ├── LICENSE ├── LICENSE-binary ├── NOTICE ├── README.md ├── RETIRED.txt ├── assembly ├── pom.xml └── src │ └── main │ └── assembly │ ├── assembly.xml │ └── source-assembly.xml ├── bin ├── crail ├── start-crail.sh └── stop-crail.sh ├── checkstyle.xml ├── client ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── crail │ │ ├── CrailBlockLocation.java │ │ ├── CrailBuffer.java │ │ ├── CrailBufferCache.java │ │ ├── CrailBufferedInputStream.java │ │ ├── CrailBufferedOutputStream.java │ │ ├── CrailBufferedStatistics.java │ │ ├── CrailContainer.java │ │ ├── CrailDirectory.java │ │ ├── CrailFile.java │ │ ├── CrailInputStream.java │ │ ├── CrailKeyValue.java │ │ ├── CrailLocationClass.java │ │ ├── CrailMultiFile.java │ │ ├── CrailNode.java │ │ ├── CrailNodeType.java │ │ ├── CrailOutputStream.java │ │ ├── CrailPurgeOperation.java │ │ ├── CrailResult.java │ │ ├── CrailStatistics.java │ │ ├── CrailStorageClass.java │ │ ├── CrailStore.java │ │ ├── CrailTable.java │ │ ├── FileBufferedInputStream.java │ │ ├── MultiFileBufferedInputStream.java │ │ ├── Upcoming.java │ │ ├── conf │ │ ├── Configurable.java │ │ ├── CrailConfiguration.java │ │ └── CrailConstants.java │ │ ├── core │ │ ├── CoreBlockLocation.java │ │ ├── CoreDataOperation.java │ │ ├── CoreDataStore.java │ │ ├── CoreDirectory.java │ │ ├── CoreFile.java │ │ ├── CoreIOStatistics.java │ │ ├── CoreInputStream.java │ │ ├── CoreMetaDataOperation.java │ │ ├── CoreNode.java │ │ ├── CoreOutputStream.java │ │ ├── CoreStream.java │ │ ├── CoreStreamStatistics.java │ │ ├── CoreSubOperation.java │ │ ├── CoreSyncOperation.java │ │ ├── DirectoryInputStream.java │ │ ├── DirectoryOutputStream.java │ │ └── DirectoryRecord.java │ │ ├── memory │ │ ├── BufferCache.java │ │ ├── MappedBufferCache.java │ │ └── OffHeapBuffer.java │ │ ├── metadata │ │ ├── BlockInfo.java │ │ ├── DataNodeInfo.java │ │ ├── DataNodeStatistics.java │ │ ├── DataNodeStatus.java │ │ ├── FileInfo.java │ │ └── FileName.java │ │ ├── rpc │ │ ├── RpcClient.java │ │ ├── RpcConnection.java │ │ ├── RpcCreateFile.java │ │ ├── RpcDeleteFile.java │ │ ├── RpcDispatcher.java │ │ ├── RpcErrors.java │ │ ├── RpcFuture.java │ │ ├── RpcGetBlock.java │ │ ├── RpcGetDataNode.java │ │ ├── RpcGetFile.java │ │ ├── RpcGetLocation.java │ │ ├── RpcPing.java │ │ ├── RpcRemoveDataNode.java │ │ ├── RpcRenameFile.java │ │ ├── RpcResponse.java │ │ └── RpcVoid.java │ │ ├── storage │ │ ├── StorageClient.java │ │ ├── StorageEndpoint.java │ │ ├── StorageFuture.java │ │ └── StorageResult.java │ │ ├── tools │ │ ├── CrailBenchmark.java │ │ ├── CrailFsck.java │ │ └── RemoveDataNode.java │ │ └── utils │ │ ├── AtomicIntegerModulo.java │ │ ├── BlockCache.java │ │ ├── BufferCheckpoint.java │ │ ├── CrailImmediateOperation.java │ │ ├── CrailUtils.java │ │ ├── EndpointCache.java │ │ ├── MultiFuture.java │ │ ├── NextBlockCache.java │ │ └── RingBuffer.java │ └── test │ └── java │ └── org │ └── apache │ └── crail │ └── ClientTest.java ├── conf ├── core-site.xml.template ├── crail-env.sh.template ├── crail-site.conf.template ├── log4j.properties └── slaves.template ├── doc ├── Makefile └── source │ ├── conf.py │ ├── config.rst │ ├── contact.rst │ ├── contribute.rst │ ├── deploy.rst │ ├── develop.rst │ ├── docker.rst │ ├── fsck.rst │ ├── index.rst │ ├── introduction.rst │ ├── iobench.rst │ ├── release.rst │ ├── run.rst │ ├── shell.rst │ ├── source.rst │ └── spark.rst ├── docker ├── Dockerfile ├── RDMA │ ├── Dockerfile │ └── crail-site.conf ├── core-site.xml.env ├── crail-env.sh ├── crail-site.conf └── start-crail-docker.sh ├── hdfs ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── crail │ │ └── hdfs │ │ ├── CrailHDFS.java │ │ ├── CrailHDFSInputStream.java │ │ ├── CrailHDFSOutputStream.java │ │ ├── CrailHadoopFileSystem.java │ │ ├── GetConf.java │ │ └── tools │ │ └── HdfsIOBenchmark.java │ └── test │ └── java │ └── org │ └── apache │ └── crail │ └── hdfs │ └── TestCrailHDFSContract.java ├── libexec ├── crail-daemon.sh ├── crail-daemons.sh └── slaves.sh ├── licenses ├── LICENSE.base64.txt ├── LICENSE.bsd2c ├── LICENSE.bsd3c ├── LICENSE.cddl1 ├── LICENSE.cddl1.1 ├── LICENSE.commons-logging.txt ├── LICENSE.commons-math3-3.1.1 ├── LICENSE.epl ├── LICENSE.epl2 ├── LICENSE.hadoop-X-2.7.3 ├── LICENSE.hadoop-common-test-0.22.0 ├── LICENSE.httpclient-4.1.2 ├── LICENSE.jsr166y.txt ├── LICENSE.jzlib.txt ├── LICENSE.log4j.txt ├── LICENSE.mit ├── LICENSE.protobuf.txt ├── LICENSE.slf4j.txt └── LICENSE.webbit.txt ├── namenode ├── pom.xml └── src │ └── main │ └── java │ └── org │ └── apache │ └── crail │ └── namenode │ ├── AbstractNode.java │ ├── BlockStore.java │ ├── DataNodeBlocks.java │ ├── DirectoryBlocks.java │ ├── ElasticNameNodeService.java │ ├── FileBlocks.java │ ├── FileStore.java │ ├── FreeCapacityPolicy.java │ ├── GCServer.java │ ├── KeyValueBlocks.java │ ├── LogDispatcher.java │ ├── LogRecord.java │ ├── LogResponse.java │ ├── LogService.java │ ├── MultiFileBlocks.java │ ├── NameNode.java │ ├── NameNodeBlockInfo.java │ ├── NameNodeService.java │ ├── PolicyRunner.java │ ├── Sequencer.java │ └── TableBlocks.java ├── pom.xml ├── rpc-darpc ├── pom.xml └── src │ └── main │ └── java │ └── org │ └── apache │ └── crail │ └── namenode │ └── rpc │ └── darpc │ ├── DaRPCConstants.java │ ├── DaRPCNameNode.java │ ├── DaRPCNameNodeClient.java │ ├── DaRPCNameNodeConnection.java │ ├── DaRPCNameNodeFuture.java │ ├── DaRPCNameNodeProtocol.java │ ├── DaRPCNameNodeRequest.java │ ├── DaRPCNameNodeResponse.java │ ├── DaRPCNameNodeServer.java │ └── DaRPCServiceDispatcher.java ├── rpc-narpc ├── pom.xml └── src │ └── main │ ├── assembly │ └── assembly.xml │ └── java │ └── org │ └── apache │ └── crail │ └── namenode │ └── rpc │ └── tcp │ ├── TcpFuture.java │ ├── TcpNameNode.java │ ├── TcpNameNodeClient.java │ ├── TcpNameNodeRequest.java │ ├── TcpNameNodeResponse.java │ ├── TcpNameNodeServer.java │ ├── TcpRpcConnection.java │ ├── TcpRpcConstants.java │ └── TcpRpcDispatcher.java ├── rpc ├── pom.xml └── src │ └── main │ └── java │ └── org │ └── apache │ └── crail │ └── rpc │ ├── RpcBinding.java │ ├── RpcNameNodeService.java │ ├── RpcNameNodeState.java │ ├── RpcProtocol.java │ ├── RpcRequestMessage.java │ ├── RpcResponseMessage.java │ └── RpcServer.java ├── storage-narpc ├── pom.xml └── src │ └── main │ ├── assembly │ └── assembly.xml │ └── java │ └── org │ └── apache │ └── crail │ └── storage │ └── tcp │ ├── TcpStorageClient.java │ ├── TcpStorageConstants.java │ ├── TcpStorageEndpoint.java │ ├── TcpStorageFuture.java │ ├── TcpStorageProtocol.java │ ├── TcpStorageRequest.java │ ├── TcpStorageResponse.java │ ├── TcpStorageServer.java │ ├── TcpStorageTier.java │ └── TcpStorageUtils.java ├── storage-nvmf ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── crail │ │ └── storage │ │ └── nvmf │ │ ├── NvmfStorageClient.java │ │ ├── NvmfStorageConstants.java │ │ ├── NvmfStorageServer.java │ │ ├── NvmfStorageTier.java │ │ └── client │ │ ├── NvmfFuture.java │ │ ├── NvmfRegisteredBufferCache.java │ │ ├── NvmfStagingBufferCache.java │ │ ├── NvmfStorageEndpoint.java │ │ └── NvmfUnalignedWriteFuture.java │ └── test │ └── java │ └── org │ └── apache │ └── crail │ └── storage │ └── nvmf │ └── client │ └── NvmfStagingBufferCacheTest.java ├── storage-rdma ├── pom.xml └── src │ └── main │ └── java │ └── org │ └── apache │ └── crail │ └── storage │ └── rdma │ ├── MrCache.java │ ├── RdmaConstants.java │ ├── RdmaStorageClient.java │ ├── RdmaStorageEndpointFactory.java │ ├── RdmaStorageGroup.java │ ├── RdmaStorageServer.java │ ├── RdmaStorageServerEndpoint.java │ ├── RdmaStorageTier.java │ └── client │ ├── RdmaActiveFuture.java │ ├── RdmaLocalFuture.java │ ├── RdmaPassiveFuture.java │ ├── RdmaStorageActiveEndpoint.java │ ├── RdmaStorageActiveEndpointFactory.java │ ├── RdmaStorageActiveGroup.java │ ├── RdmaStorageLocalEndpoint.java │ ├── RdmaStoragePassiveEndpoint.java │ ├── RdmaStoragePassiveEndpointFactory.java │ └── RdmaStoragePassiveGroup.java └── storage ├── pom.xml └── src └── main └── java └── org └── apache └── crail └── storage ├── StorageResource.java ├── StorageRpcClient.java ├── StorageServer.java ├── StorageTier.java └── StorageUtils.java /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # Package Files # 8 | *.jar 9 | *.war 10 | *.nar 11 | *.ear 12 | *.zip 13 | *.tar.gz 14 | *.rar 15 | 16 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 17 | hs_err_pid* 18 | 19 | .idea/ 20 | *.iml 21 | 22 | target/ 23 | assembly/target/ 24 | client/target/ 25 | doc/build/ 26 | hdfs/target/ 27 | namenode/target/ 28 | rpc/target/ 29 | rpc-darpc/target/ 30 | rpc-narpc/target/ 31 | storage/target/ 32 | storage-nvmf/target/ 33 | storage-rdma/target/ 34 | -------------------------------------------------------------------------------- /CREDITS: -------------------------------------------------------------------------------- 1 | The Crail project code was originally created, designed, developed (under an 2 | IBM copyright dated 2015-2017) and donated by the IBM Corporation (http://www.ibm.com/) 3 | to the Apache Software Foundation (ASF). 4 | -------------------------------------------------------------------------------- /DISCLAIMER: -------------------------------------------------------------------------------- 1 | Apache Crail (incubating) is an effort undergoing incubation at The 2 | Apache Software Foundation (ASF), sponsored by the Apache Incubator PMC. 3 | 4 | Incubation is required of all newly accepted 5 | projects until a further review indicates that the 6 | infrastructure, communications, and decision making process have 7 | stabilized in a manner consistent with other successful ASF 8 | projects. 9 | 10 | While incubation status is not necessarily a reflection 11 | of the completeness or stability of the code, it does indicate 12 | that the project has yet to be fully endorsed by the ASF. 13 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Apache Crail incubating 2 | Copyright 2017-2018 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). -------------------------------------------------------------------------------- /RETIRED.txt: -------------------------------------------------------------------------------- 1 | This podling has been retired, please see: http://incubator.apache.org/projects/index.html##crail 2 | -------------------------------------------------------------------------------- /bin/crail: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | bin=`which $0` 19 | bin=`dirname ${bin}` 20 | bin=`cd "$bin"; pwd` 21 | 22 | LIBEXEC_DIR="$bin"/../libexec 23 | JAVA=$JAVA_HOME/bin/java 24 | 25 | function print_usage(){ 26 | echo "Usage: crail COMMAND" 27 | echo " where COMMAND is one of:" 28 | echo " namenode run the Crail namenode" 29 | echo " datanode run a Crail datanode" 30 | echo " removeDatanode remove a Crail datanode" 31 | echo " fsck run a Crail file check command" 32 | echo " fs run a Crail shell command" 33 | echo " iobench run a Crail benchmark/test" 34 | echo " test run a Crail unit test" 35 | } 36 | 37 | if [ $# = 0 ]; then 38 | print_usage 39 | exit 40 | fi 41 | 42 | COMMAND=$1 43 | shift 44 | 45 | case $COMMAND in 46 | # usage flags 47 | --help|-help|-h) 48 | print_usage 49 | exit 50 | ;; 51 | esac 52 | 53 | if [ "$COMMAND" = "namenode" ] ; then 54 | CLASS=org.apache.crail.namenode.NameNode 55 | elif [ "$COMMAND" = "datanode" ] ; then 56 | CLASS=org.apache.crail.storage.StorageServer 57 | elif [ "$COMMAND" = "removeDatanode" ] ; then 58 | CLASS=org.apache.crail.tools.RemoveDataNode 59 | elif [ "$COMMAND" = "fsck" ] ; then 60 | CLASS=org.apache.crail.tools.CrailFsck 61 | elif [ "$COMMAND" = "fs" ] ; then 62 | CLASS=org.apache.hadoop.fs.FsShell 63 | elif [ "$COMMAND" = "getconf" ] ; then 64 | CLASS=org.apache.crail.hdfs.GetConf 65 | elif [ "$COMMAND" = "iobench" ] ; then 66 | CLASS=org.apache.crail.tools.CrailBenchmark 67 | elif [ "$COMMAND" = "hdfsbench" ] ; then 68 | CLASS=org.apache.crail.hdfs.tools.HdfsIOBenchmark 69 | elif [ "$COMMAND" = "test" ] ; then 70 | CLASS=org.junit.runner.JUnitCore 71 | fi 72 | 73 | CONF_PATH="$bin"/../conf 74 | export CLASSPATH="$bin"/../jars/*:${CONF_PATH}:. 75 | export LD_LIBRARY_PATH="$bin/../lib:$LD_LIBRARY_PATH" 76 | 77 | if [ -f "${CONF_PATH}/crail-env.sh" ]; then 78 | # Promote all variable declarations to environment (exported) variables 79 | set -a 80 | . "${CONF_PATH}/crail-env.sh" 81 | set +a 82 | fi 83 | 84 | exec "$JAVA" -Dproc_$COMMAND -Dsun.nio.PageAlignDirectMemory=true $CRAIL_EXTRA_JAVA_OPTIONS $CLASS "$@" 85 | -------------------------------------------------------------------------------- /bin/start-crail.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | set -e 19 | 20 | usage="Usage: start-crail.sh" 21 | 22 | bin=`dirname "${BASH_SOURCE-$0}"` 23 | CRAIL_HOME=`cd "$bin/.."; pwd` 24 | bin=`cd "$bin/../bin"; pwd` 25 | 26 | LIBEXEC_DIR="$bin"/../libexec 27 | 28 | # get arguments 29 | if [ $# -ge 1 ]; then 30 | nameStartOpt="$1" 31 | shift 32 | case "$nameStartOpt" in 33 | (-upgrade) 34 | ;; 35 | (-rollback) 36 | dataStartOpt="$nameStartOpt" 37 | ;; 38 | (*) 39 | echo $usage 40 | exit 1 41 | ;; 42 | esac 43 | fi 44 | 45 | #Add other possible options 46 | nameStartOpt="$nameStartOpt $@" 47 | 48 | #--------------------------------------------------------- 49 | # namenodes 50 | 51 | NAMENODES=$($CRAIL_HOME/bin/crail getconf namenode) 52 | 53 | echo "Starting namenodes on [$NAMENODES]" 54 | 55 | "$LIBEXEC_DIR/crail-daemons.sh" \ 56 | --hostnames "$NAMENODES" \ 57 | --script "$bin/crail" start namenode $nameStartOpt 58 | 59 | #--------------------------------------------------------- 60 | # datanodes (using default slaves file) 61 | 62 | if [ -n "$HADOOP_SECURE_DN_USER" ]; then 63 | echo \ 64 | "Attempting to start secure cluster, skipping datanodes. " \ 65 | "Run start-secure-dns.sh as root to complete startup." 66 | else 67 | "$LIBEXEC_DIR/crail-daemons.sh" \ 68 | --script "$bin/crail" start datanode $dataStartOpt 69 | fi 70 | 71 | exit 72 | 73 | # eof 74 | -------------------------------------------------------------------------------- /bin/stop-crail.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | bin=`dirname "${BASH_SOURCE-$0}"` 19 | CRAIL_HOME=`cd "$bin/.."; pwd` 20 | bin=`cd "$bin"; pwd` 21 | 22 | LIBEXEC_DIR="$bin"/../libexec 23 | 24 | #--------------------------------------------------------- 25 | # namenodes 26 | 27 | NAMENODES=$($CRAIL_HOME/bin/crail getconf namenode) 28 | 29 | echo "Stopping namenodes on [$NAMENODES]" 30 | 31 | "$LIBEXEC_DIR/crail-daemons.sh" \ 32 | --hostnames "$NAMENODES" \ 33 | --script "$bin/crail" stop namenode 34 | 35 | #--------------------------------------------------------- 36 | # datanodes (using default slaves file) 37 | 38 | if [ -n "$HADOOP_SECURE_DN_USER" ]; then 39 | echo \ 40 | "Attempting to stop secure cluster, skipping datanodes. " \ 41 | "Run stop-secure-dns.sh as root to complete shutdown." 42 | else 43 | "$LIBEXEC_DIR/crail-daemons.sh" \ 44 | --script "$bin/crail" stop datanode 45 | fi 46 | 47 | exit 48 | 49 | # eof 50 | -------------------------------------------------------------------------------- /client/pom.xml: -------------------------------------------------------------------------------- 1 | 18 | 19 | 4.0.0 20 | 21 | 22 | org.apache.crail 23 | crail-parent 24 | 1.3-incubating-SNAPSHOT 25 | ../pom.xml 26 | 27 | 28 | org.apache.crail 29 | crail-client 30 | jar 31 | 1.3-incubating-SNAPSHOT 32 | Crail Client Project 33 | http://maven.apache.org 34 | 35 | 36 | 37 | junit 38 | junit 39 | 40 | 41 | log4j 42 | log4j 43 | 44 | 45 | commons-cli 46 | commons-cli 47 | 48 | 49 | org.slf4j 50 | slf4j-log4j12 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | org.apache.maven.plugins 60 | maven-jar-plugin 61 | 2.6 62 | 63 | 64 | 65 | test-jar 66 | 67 | 68 | 69 | 70 | 71 | 72 | org.apache.maven.plugins 73 | maven-compiler-plugin 74 | 3.3 75 | 76 | 1.8 77 | 1.8 78 | 79 | 80 | 81 | 82 | 83 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/CrailBlockLocation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | public interface CrailBlockLocation { 22 | public long getOffset(); 23 | public long getLength(); 24 | public String[] getNames(); 25 | public String[] getHosts(); 26 | public String[] getTopology(); 27 | public int[] getStorageTypes(); 28 | public int[] getLocationAffinities(); 29 | } 30 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/CrailBuffer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | import java.nio.ByteBuffer; 22 | 23 | public interface CrailBuffer { 24 | 25 | //ByteBuffer metadata 26 | 27 | public int capacity(); 28 | 29 | public int position(); 30 | 31 | public CrailBuffer position(int newPosition); 32 | 33 | public int limit(); 34 | 35 | public CrailBuffer limit(int newLimit); 36 | 37 | public CrailBuffer clear(); 38 | 39 | public CrailBuffer flip(); 40 | 41 | public int remaining(); 42 | 43 | public boolean hasRemaining(); 44 | 45 | public CrailBuffer slice(); 46 | 47 | //Crail metadata 48 | 49 | public long address(); 50 | 51 | public CrailBuffer getRegion(); 52 | 53 | public ByteBuffer getByteBuffer(); 54 | 55 | //ByteBuffer data 56 | 57 | public byte get(); 58 | 59 | public ByteBuffer put(byte b); 60 | 61 | public CrailBuffer get(byte[] buf, int off, int bufferRemaining); 62 | 63 | public CrailBuffer put(byte[] dataBuf, int off, int bufferRemaining); 64 | 65 | public CrailBuffer get(byte[] bytes); 66 | 67 | public CrailBuffer put(byte[] bytes); 68 | 69 | public short getShort(); 70 | 71 | public CrailBuffer putShort(short value); 72 | 73 | public int getInt(); 74 | 75 | public CrailBuffer putInt(int value); 76 | 77 | public long getLong(); 78 | 79 | public CrailBuffer putLong(long value); 80 | 81 | public float getFloat(); 82 | 83 | public CrailBuffer putFloat(float value); 84 | 85 | public double getDouble(); 86 | 87 | public CrailBuffer putDouble(double value); 88 | 89 | public CrailBuffer put(ByteBuffer buf); 90 | 91 | public CrailBuffer get(ByteBuffer buf); 92 | } 93 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/CrailBufferCache.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | public interface CrailBufferCache { 22 | CrailBuffer allocateBuffer() throws Exception; 23 | void freeBuffer(CrailBuffer buffer) throws Exception; 24 | } 25 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/CrailBufferedStatistics.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | import java.util.concurrent.atomic.AtomicLong; 22 | 23 | import org.apache.crail.CrailStatistics.StatisticsProvider; 24 | 25 | class CrailBufferedStatistics implements StatisticsProvider { 26 | private String type; 27 | private AtomicLong totalOps; 28 | private AtomicLong blockingOps; 29 | private AtomicLong nonBlockingOps; 30 | 31 | public CrailBufferedStatistics(String type){ 32 | this.type = type; 33 | this.totalOps = new AtomicLong(0); 34 | this.blockingOps = new AtomicLong(0); 35 | this.nonBlockingOps = new AtomicLong(0); 36 | } 37 | 38 | public void mergeStatistics(StatisticsProvider provider){ 39 | if (provider instanceof CrailBufferedStatistics){ 40 | CrailBufferedStatistics newProvider = (CrailBufferedStatistics) provider; 41 | this.totalOps.addAndGet(newProvider.getTotalOps()); 42 | this.blockingOps.addAndGet(newProvider.getBlockingOps()); 43 | this.nonBlockingOps.addAndGet(newProvider.getNonBlockingOps()); 44 | } 45 | } 46 | 47 | @Override 48 | public String providerName() { 49 | return type; 50 | } 51 | 52 | @Override 53 | public String printStatistics() { 54 | return "totalOps " + getTotalOps() + ", blockingOps " + getBlockingOps() + ", nonBlockingOps " + getNonBlockingOps(); 55 | } 56 | 57 | @Override 58 | public void resetStatistics() { 59 | this.totalOps.set(0); 60 | this.blockingOps.set(0); 61 | this.nonBlockingOps.set(0); 62 | } 63 | 64 | public void incTotalOps(){ 65 | this.totalOps.incrementAndGet(); 66 | } 67 | 68 | public void incBlockingOps(){ 69 | this.blockingOps.incrementAndGet(); 70 | } 71 | 72 | public void incNonBlockingOps(){ 73 | this.nonBlockingOps.incrementAndGet(); 74 | } 75 | 76 | public long getTotalOps(){ 77 | return totalOps.get(); 78 | } 79 | 80 | public long getBlockingOps(){ 81 | return blockingOps.get(); 82 | } 83 | 84 | public long getNonBlockingOps(){ 85 | return nonBlockingOps.get(); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/CrailContainer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | import java.util.Iterator; 22 | 23 | public interface CrailContainer extends CrailNode { 24 | public abstract int files(); 25 | public abstract Iterator listEntries() throws Exception; 26 | } 27 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/CrailDirectory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | import java.util.Iterator; 22 | 23 | public interface CrailDirectory extends CrailContainer { 24 | public abstract int files(); 25 | public abstract Iterator listEntries() throws Exception; 26 | } -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/CrailFile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | public interface CrailFile extends CrailNode { 22 | public abstract CrailInputStream getDirectInputStream(long readHint) throws Exception; 23 | public abstract CrailOutputStream getDirectOutputStream(long writeHint) throws Exception; 24 | public abstract long getToken(); 25 | public abstract long getFd(); 26 | 27 | default CrailBufferedInputStream getBufferedInputStream(long readHint) throws Exception { 28 | return new FileBufferedInputStream(this, readHint); 29 | } 30 | 31 | default CrailBufferedOutputStream getBufferedOutputStream(long writeHint) throws Exception { 32 | return new CrailBufferedOutputStream(this, writeHint); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/CrailInputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | import java.io.IOException; 22 | import java.util.concurrent.Future; 23 | 24 | public interface CrailInputStream { 25 | public CrailNode getFile(); 26 | public Future read(CrailBuffer dataBuf) throws Exception; 27 | public void seek(long pos) throws IOException; 28 | public int available(); 29 | public long position(); 30 | void close() throws Exception; 31 | } 32 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/CrailKeyValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | public interface CrailKeyValue extends CrailFile { 22 | 23 | } 24 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/CrailLocationClass.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | public class CrailLocationClass { 22 | public static final CrailLocationClass PARENT = CrailLocationClass.get(-1); 23 | public static final CrailLocationClass DEFAULT = CrailLocationClass.get(0); 24 | 25 | private int value; 26 | 27 | public static CrailLocationClass get(int value){ 28 | return new CrailLocationClass(value); 29 | } 30 | 31 | private CrailLocationClass(int value){ 32 | this.value = value; 33 | } 34 | 35 | public int value(){ 36 | return this.value; 37 | } 38 | 39 | @Override 40 | public String toString() { 41 | return "" + value; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/CrailMultiFile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | public interface CrailMultiFile extends CrailContainer { 22 | default CrailBufferedInputStream getMultiStream(int outstanding) throws Exception{ 23 | return new MultiFileBufferedInputStream(this.getFileSystem(), listEntries(), outstanding, files()); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/CrailNode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | 22 | public interface CrailNode { 23 | public CrailStore getFileSystem(); 24 | public String getPath(); 25 | public abstract CrailNode syncDir() throws Exception; 26 | public abstract long getModificationTime(); 27 | public abstract long getCapacity(); 28 | public abstract CrailNodeType getType(); 29 | public abstract CrailFile asFile() throws Exception; 30 | public abstract CrailContainer asContainer() throws Exception; 31 | public abstract CrailDirectory asDirectory() throws Exception; 32 | public abstract CrailMultiFile asMultiFile() throws Exception; 33 | public abstract CrailTable asTable() throws Exception; 34 | public abstract CrailKeyValue asKeyValue() throws Exception; 35 | public abstract CrailBlockLocation[] getBlockLocations(long start, long len) throws Exception; 36 | } 37 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/CrailNodeType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | public enum CrailNodeType { 22 | DATAFILE(0), DIRECTORY(1), MULTIFILE(2), STREAMFILE(3), TABLE(4), KEYVALUE(5); 23 | 24 | private int label; 25 | 26 | CrailNodeType(int label){ 27 | this.label = label; 28 | } 29 | 30 | public int getLabel(){ 31 | return this.label; 32 | } 33 | 34 | public boolean isDirectory(){ 35 | return this == DIRECTORY; 36 | } 37 | 38 | public boolean isDataFile(){ 39 | return this == DATAFILE; 40 | } 41 | 42 | public boolean isMultiFile(){ 43 | return this == MULTIFILE; 44 | } 45 | 46 | public boolean isStreamFile(){ 47 | return this == STREAMFILE; 48 | } 49 | 50 | public boolean isTable() { 51 | return this == TABLE; 52 | } 53 | 54 | public boolean isKeyValue() { 55 | return this == KEYVALUE; 56 | } 57 | 58 | public boolean isContainer(){ 59 | return this == DIRECTORY || this == MULTIFILE || this == TABLE; 60 | } 61 | 62 | public static CrailNodeType parse(int label) { 63 | for (CrailNodeType val : CrailNodeType.values()) { 64 | if (val.getLabel() == label) { 65 | return val; 66 | } 67 | } 68 | throw new IllegalArgumentException(); 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/CrailOutputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | import java.io.IOException; 22 | import java.util.concurrent.Future; 23 | 24 | public interface CrailOutputStream { 25 | CrailNode getFile(); 26 | Future write(CrailBuffer dataBuf) throws Exception; 27 | Future sync() throws IOException; 28 | long position(); 29 | long getWriteHint(); 30 | void close() throws Exception; 31 | } 32 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/CrailPurgeOperation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | import org.apache.crail.utils.MultiFuture; 22 | 23 | class CrailPurgeOperation extends MultiFuture implements CrailResult { 24 | private long completedLen; 25 | 26 | public CrailPurgeOperation() { 27 | this.completedLen = 0; 28 | } 29 | 30 | @Override 31 | public void aggregate(CrailResult obj) { 32 | this.completedLen += obj.getLen(); 33 | } 34 | 35 | @Override 36 | public CrailResult getAggregate() { 37 | return this; 38 | } 39 | 40 | @Override 41 | public long getLen() { 42 | return completedLen; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/CrailResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | public interface CrailResult { 22 | public long getLen(); 23 | } 24 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/CrailStatistics.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | import java.util.concurrent.ConcurrentHashMap; 22 | 23 | import org.apache.crail.utils.CrailUtils; 24 | import org.slf4j.Logger; 25 | 26 | public class CrailStatistics { 27 | private static final Logger LOG = CrailUtils.getLogger(); 28 | private ConcurrentHashMap statistics; 29 | 30 | public CrailStatistics(){ 31 | statistics = new ConcurrentHashMap(); 32 | } 33 | 34 | public void addProvider(StatisticsProvider provider){ 35 | StatisticsProvider existing = statistics.putIfAbsent(provider.providerName(), provider); 36 | if (existing != null){ 37 | existing.mergeStatistics(provider); 38 | } 39 | } 40 | 41 | public void print(String tag){ 42 | LOG.info("CrailStatistics, tag=" + tag); 43 | for (StatisticsProvider provider : statistics.values()){ 44 | LOG.info("provider=" + provider.providerName() + " [" + provider.printStatistics() + "]"); 45 | } 46 | } 47 | 48 | public void reset(){ 49 | for (StatisticsProvider provider : statistics.values()){ 50 | provider.resetStatistics(); 51 | } 52 | } 53 | 54 | public static interface StatisticsProvider { 55 | public String providerName(); 56 | public String printStatistics(); 57 | public void mergeStatistics(StatisticsProvider provider); 58 | public void resetStatistics(); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/CrailStorageClass.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | public class CrailStorageClass { 22 | public static final CrailStorageClass PARENT = CrailStorageClass.get(-1); 23 | public static final CrailStorageClass DEFAULT = CrailStorageClass.get(0); 24 | 25 | private int value; 26 | 27 | public static CrailStorageClass get(int value){ 28 | return new CrailStorageClass(value); 29 | } 30 | 31 | private CrailStorageClass(int value){ 32 | this.value = value; 33 | } 34 | 35 | public int value(){ 36 | return this.value; 37 | } 38 | 39 | @Override 40 | public String toString() { 41 | return "" + value; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/CrailTable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | public interface CrailTable extends CrailContainer { 22 | 23 | } 24 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/FileBufferedInputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | import java.io.IOException; 22 | 23 | import org.apache.crail.conf.CrailConstants; 24 | 25 | class FileBufferedInputStream extends CrailBufferedInputStream { 26 | private CrailInputStream inputStream; 27 | 28 | FileBufferedInputStream(CrailFile file, long readHint) throws Exception { 29 | super(file.getFileSystem(), Math.max(CrailConstants.BUFFER_SIZE, CrailConstants.SLICE_SIZE)/Math.min(CrailConstants.BUFFER_SIZE, CrailConstants.SLICE_SIZE), file.getCapacity()); 30 | this.inputStream = file.getDirectInputStream(readHint); 31 | } 32 | 33 | @Override 34 | public CrailInputStream getStream() throws Exception { 35 | return inputStream; 36 | } 37 | 38 | @Override 39 | public void putStream() throws Exception { 40 | 41 | } 42 | 43 | @Override 44 | public void close() throws IOException { 45 | super.close(); 46 | try { 47 | inputStream.close(); 48 | } catch(Exception e){ 49 | throw new IOException(e); 50 | } 51 | } 52 | 53 | } 54 | 55 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/Upcoming.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail; 20 | 21 | import java.util.concurrent.Future; 22 | 23 | public interface Upcoming extends Future { 24 | T early() throws Exception; 25 | } 26 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/conf/Configurable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.conf; 20 | 21 | import org.slf4j.Logger; 22 | 23 | public interface Configurable { 24 | public void init(CrailConfiguration conf, String[] args) throws Exception; 25 | public void printConf(Logger log); 26 | } 27 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/core/CoreSubOperation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.core; 20 | 21 | import java.io.IOException; 22 | 23 | import org.apache.crail.conf.CrailConstants; 24 | import org.apache.crail.utils.CrailUtils; 25 | 26 | public class CoreSubOperation { 27 | private long fd; 28 | private long fileOffset; 29 | private int bufferPosition; 30 | private int len; 31 | 32 | private long blockOffset; 33 | private long blockStart; 34 | private long key; 35 | 36 | public CoreSubOperation(long fd, long fileOffset, int bufferPosition, int writeLen) throws IOException { 37 | this.fd = fd; 38 | this.fileOffset = fileOffset; 39 | this.bufferPosition = bufferPosition; 40 | this.len = writeLen; 41 | 42 | this.blockOffset = fileOffset % CrailConstants.BLOCK_SIZE; 43 | this.blockStart = CrailUtils.blockStartAddress(fileOffset); 44 | // this.key = CoreSubOperation.createKey(fd, blockStart); 45 | this.key = blockStart; 46 | } 47 | 48 | public long getFd() { 49 | return fd; 50 | } 51 | 52 | public long getBlockOffset() { 53 | return blockOffset; 54 | } 55 | 56 | public long getBlockStart() { 57 | return blockStart; 58 | } 59 | 60 | public int getBufferPosition(){ 61 | return bufferPosition; 62 | } 63 | 64 | public int getLen() { 65 | return len; 66 | } 67 | 68 | @Override 69 | public String toString() { 70 | return "fd " + fd + ", fileOffset " + fileOffset + ", blockOffset " + blockOffset + ", len " + len + ", blockStart " + blockStart; 71 | } 72 | 73 | public long key(){ 74 | return this.key; 75 | } 76 | 77 | public static long createKey(long fd, long fileOffset){ 78 | long offset = CrailUtils.blockStartAddress(fileOffset); 79 | return offset; 80 | // return fd + ":" + offset; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/core/CoreSyncOperation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.core; 20 | 21 | import java.util.concurrent.Future; 22 | 23 | import org.apache.crail.CrailResult; 24 | 25 | public class CoreSyncOperation { 26 | private DirectoryOutputStream stream; 27 | private Future future; 28 | 29 | public CoreSyncOperation(DirectoryOutputStream stream, Future future) { 30 | this.stream = stream; 31 | this.future = future; 32 | } 33 | 34 | public void close() throws Exception { 35 | future.get(); 36 | stream.close(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/core/DirectoryOutputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.core; 20 | 21 | import java.io.IOException; 22 | import java.util.concurrent.Future; 23 | 24 | import org.apache.crail.CrailBuffer; 25 | import org.apache.crail.CrailResult; 26 | 27 | public class DirectoryOutputStream { 28 | private CoreOutputStream stream; 29 | private CrailBuffer internalBuf; 30 | private CoreDataStore fs; 31 | private boolean open; 32 | 33 | public DirectoryOutputStream(CoreOutputStream stream) 34 | throws Exception { 35 | this.fs = stream.getFile().getFileSystem(); 36 | this.stream = stream; 37 | this.internalBuf = fs.allocateBuffer(); 38 | this.open = true; 39 | } 40 | 41 | Future writeRecord(DirectoryRecord record, long offset) throws Exception { 42 | if (!open) { 43 | throw new IOException("stream closed"); 44 | } 45 | 46 | internalBuf.clear(); 47 | record.write(internalBuf); 48 | internalBuf.flip(); 49 | stream.seek(offset); 50 | Future future = stream.write(internalBuf); 51 | return future; 52 | } 53 | 54 | public void close() throws IOException { 55 | try { 56 | if (!open){ 57 | return; 58 | } 59 | 60 | stream.close(); 61 | fs.freeBuffer(internalBuf); 62 | internalBuf = null; 63 | open = false; 64 | } catch (Exception e) { 65 | throw new IOException(e); 66 | } 67 | } 68 | 69 | //debug 70 | 71 | public int getBufCapacity(){ 72 | return internalBuf.capacity(); 73 | } 74 | 75 | public int getBufPosition(){ 76 | return internalBuf.position(); 77 | } 78 | 79 | public int getBufLimit(){ 80 | return internalBuf.limit(); 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/core/DirectoryRecord.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.core; 20 | 21 | import org.apache.crail.CrailBuffer; 22 | import org.apache.crail.conf.CrailConstants; 23 | import org.apache.crail.utils.CrailUtils; 24 | 25 | public class DirectoryRecord { 26 | private int valid; 27 | private String parent; 28 | private String filename; 29 | 30 | public DirectoryRecord(boolean valid, String path){ 31 | this.valid = valid == true ? 1 : 0; 32 | this.parent = CrailUtils.getParent(path); 33 | this.filename = CrailUtils.getName(path); 34 | } 35 | 36 | public DirectoryRecord(String parent) { 37 | this.valid = 0; 38 | this.parent = parent; 39 | this.filename = null; 40 | 41 | } 42 | 43 | public void write(CrailBuffer buffer) throws Exception { 44 | int oldposition = buffer.position(); 45 | buffer.putInt(valid); 46 | byte barray[] = filename.getBytes(); 47 | buffer.putInt(barray.length); 48 | buffer.put(barray); 49 | buffer.position(oldposition + CrailConstants.DIRECTORY_RECORD); 50 | } 51 | 52 | public void update(CrailBuffer buffer) { 53 | int oldlimit = buffer.limit(); 54 | int tmplimit = buffer.position() + CrailConstants.DIRECTORY_RECORD; 55 | buffer.limit(tmplimit); 56 | valid = buffer.getInt(); 57 | if (valid == 1) { 58 | int length = buffer.getInt(); 59 | byte barray[] = new byte[length]; 60 | buffer.get(barray); 61 | filename = new String(barray); 62 | } else { 63 | filename = null; 64 | } 65 | buffer.position(tmplimit); 66 | buffer.limit(oldlimit); 67 | } 68 | 69 | public boolean isValid() { 70 | return valid == 1; 71 | } 72 | 73 | public String toString() { 74 | return valid + "\t\t" + filename; 75 | } 76 | 77 | public String getFile() { 78 | return filename; 79 | } 80 | 81 | public String getParent() { 82 | return parent; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/metadata/BlockInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.metadata; 20 | 21 | import java.net.UnknownHostException; 22 | import java.nio.ByteBuffer; 23 | 24 | public class BlockInfo { 25 | public static int CSIZE = DataNodeInfo.CSIZE + 24; 26 | 27 | protected DataNodeInfo dnInfo; 28 | protected long lba; 29 | protected long addr; 30 | protected int length; 31 | protected int lkey; 32 | 33 | public BlockInfo(){ 34 | this.dnInfo = new DataNodeInfo(); 35 | this.lba = 0; 36 | this.addr = 0; 37 | this.length = 0; 38 | this.lkey = 0; 39 | } 40 | 41 | public BlockInfo(DataNodeInfo dnInfo, long lba, long addr, int length, int lkey){ 42 | this.dnInfo = dnInfo; 43 | this.lba = lba; 44 | this.addr = addr; 45 | this.length = length; 46 | this.lkey = lkey; 47 | } 48 | 49 | public void setBlockInfo(BlockInfo blockInfo) { 50 | this.dnInfo.setDataNodeInfo(blockInfo.getDnInfo()); 51 | this.lba = blockInfo.getLba(); 52 | this.addr = blockInfo.getAddr(); 53 | this.length = blockInfo.getLength(); 54 | this.lkey = blockInfo.getLkey(); 55 | 56 | } 57 | 58 | public int write(ByteBuffer buffer){ 59 | this.getDnInfo().write(buffer); 60 | buffer.putLong(this.getLba()); 61 | buffer.putLong(this.getAddr()); 62 | buffer.putInt(this.getLength()); 63 | buffer.putInt(this.getLkey()); 64 | return CSIZE; 65 | } 66 | 67 | public void update(ByteBuffer buffer) throws UnknownHostException{ 68 | this.dnInfo.update(buffer); 69 | this.lba = buffer.getLong(); 70 | this.addr = buffer.getLong(); 71 | this.length = buffer.getInt(); 72 | this.lkey = buffer.getInt(); 73 | } 74 | 75 | public long getLba() { 76 | return lba; 77 | } 78 | 79 | public long getAddr() { 80 | return addr; 81 | } 82 | 83 | public int getLength() { 84 | return length; 85 | } 86 | 87 | public int getLkey() { 88 | return lkey; 89 | } 90 | 91 | public DataNodeInfo getDnInfo() { 92 | return dnInfo; 93 | } 94 | 95 | @Override 96 | public String toString() { 97 | return "BlockInfo [dnInfo=" + dnInfo + ", lba=" + lba + ", addr=" 98 | + addr + ", length=" + length + ", lkey=" + lkey + "]"; 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/metadata/DataNodeStatistics.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.metadata; 20 | 21 | import java.net.UnknownHostException; 22 | import java.nio.ByteBuffer; 23 | 24 | public class DataNodeStatistics { 25 | public static int CSIZE = DataNodeStatus.CSIZE + 12; 26 | 27 | private DataNodeStatus status; 28 | private long serviceId; 29 | private int freeBlockCount; 30 | 31 | public DataNodeStatistics(){ 32 | this.status = new DataNodeStatus(); 33 | this.serviceId = 0; 34 | this.freeBlockCount = 0; 35 | } 36 | 37 | public int write(ByteBuffer buffer){ 38 | this.status.write(buffer); 39 | buffer.putLong(serviceId); 40 | buffer.putInt(freeBlockCount); 41 | return CSIZE; 42 | } 43 | 44 | public void update(ByteBuffer buffer) throws UnknownHostException { 45 | this.status.update(buffer); 46 | this.serviceId = buffer.getLong(); 47 | this.freeBlockCount = buffer.getInt(); 48 | } 49 | 50 | public int getFreeBlockCount() { 51 | return freeBlockCount; 52 | } 53 | 54 | public void setFreeBlockCount(int blockCount) { 55 | this.freeBlockCount = blockCount; 56 | } 57 | 58 | public DataNodeStatus getStatus() { 59 | return this.status; 60 | } 61 | 62 | public void setStatus(short status) { 63 | this.status.setStatus(status); 64 | } 65 | 66 | public void setStatistics(DataNodeStatistics statistics) { 67 | this.serviceId = statistics.getServiceId(); 68 | this.freeBlockCount = statistics.getFreeBlockCount(); 69 | } 70 | 71 | public void setServiceId(long serviceId) { 72 | this.serviceId = serviceId; 73 | } 74 | 75 | public long getServiceId(){ 76 | return serviceId; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/metadata/DataNodeStatus.java: -------------------------------------------------------------------------------- 1 | package org.apache.crail.metadata; 2 | 3 | import java.nio.ByteBuffer; 4 | 5 | public class DataNodeStatus { 6 | public static final int CSIZE = 2; 7 | 8 | private short status; 9 | 10 | public static final short STATUS_DATANODE_STOP = 1; 11 | 12 | public DataNodeStatus() { 13 | this.status = 0; 14 | } 15 | 16 | public int write(ByteBuffer buffer) { 17 | buffer.putShort(status); 18 | return CSIZE; 19 | } 20 | 21 | public void update(ByteBuffer buffer) { 22 | this.status = buffer.getShort(); 23 | } 24 | 25 | public short getStatus() { 26 | return this.status; 27 | } 28 | 29 | public void setStatus(short status) { 30 | this.status = status; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/rpc/RpcClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.rpc; 20 | 21 | import java.net.InetSocketAddress; 22 | 23 | import org.apache.crail.conf.Configurable; 24 | 25 | public interface RpcClient extends Configurable { 26 | public RpcConnection connect(InetSocketAddress address) throws Exception ; 27 | public void close(); 28 | 29 | @SuppressWarnings("unchecked") 30 | public static RpcClient createInstance(String name) throws Exception { 31 | Class nodeClass = Class.forName(name); 32 | if (RpcClient.class.isAssignableFrom(nodeClass)){ 33 | Class clientClass = (Class) nodeClass; 34 | RpcClient clientInstance = clientClass.newInstance(); 35 | return clientInstance; 36 | } else { 37 | throw new Exception("Cannot instantiate rpc client of type " + name); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/rpc/RpcCreateFile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.rpc; 20 | 21 | import org.apache.crail.metadata.BlockInfo; 22 | import org.apache.crail.metadata.FileInfo; 23 | 24 | public interface RpcCreateFile extends RpcResponse { 25 | public FileInfo getFile(); 26 | public FileInfo getParent(); 27 | public BlockInfo getFileBlock(); 28 | public BlockInfo getDirBlock(); 29 | } 30 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/rpc/RpcDeleteFile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.rpc; 20 | 21 | import org.apache.crail.metadata.FileInfo; 22 | 23 | public interface RpcDeleteFile extends RpcResponse { 24 | public FileInfo getFile(); 25 | public FileInfo getParent(); 26 | } 27 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/rpc/RpcFuture.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.rpc; 20 | 21 | import java.util.concurrent.Future; 22 | 23 | public interface RpcFuture extends Future { 24 | 25 | public abstract int getTicket(); 26 | 27 | public abstract boolean isPrefetched(); 28 | 29 | public abstract void setPrefetched(boolean prefetched); 30 | 31 | } -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/rpc/RpcGetBlock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.rpc; 20 | 21 | import org.apache.crail.metadata.BlockInfo; 22 | 23 | public interface RpcGetBlock extends RpcResponse { 24 | public BlockInfo getBlockInfo(); 25 | public void setBlockInfo(BlockInfo blockInfo); 26 | } 27 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/rpc/RpcGetDataNode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.rpc; 20 | 21 | import org.apache.crail.metadata.DataNodeStatistics; 22 | 23 | public interface RpcGetDataNode extends RpcResponse { 24 | public DataNodeStatistics getStatistics(); 25 | } 26 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/rpc/RpcGetFile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.rpc; 20 | 21 | import org.apache.crail.metadata.BlockInfo; 22 | import org.apache.crail.metadata.FileInfo; 23 | 24 | public interface RpcGetFile extends RpcResponse { 25 | public FileInfo getFile(); 26 | public BlockInfo getFileBlock(); 27 | } 28 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/rpc/RpcGetLocation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.rpc; 20 | 21 | import java.io.IOException; 22 | 23 | import org.apache.crail.metadata.BlockInfo; 24 | 25 | public interface RpcGetLocation extends RpcResponse { 26 | public BlockInfo getBlockInfo() throws IOException; 27 | public long getFd(); 28 | } 29 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/rpc/RpcPing.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.rpc; 20 | 21 | public interface RpcPing extends RpcResponse { 22 | public int getData(); 23 | } 24 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/rpc/RpcRemoveDataNode.java: -------------------------------------------------------------------------------- 1 | package org.apache.crail.rpc; 2 | 3 | public interface RpcRemoveDataNode extends RpcResponse { 4 | public short getRpcStatus(); 5 | } 6 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/rpc/RpcRenameFile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.rpc; 20 | 21 | import org.apache.crail.metadata.BlockInfo; 22 | import org.apache.crail.metadata.FileInfo; 23 | 24 | public interface RpcRenameFile extends RpcResponse { 25 | public FileInfo getSrcParent(); 26 | public FileInfo getSrcFile(); 27 | public FileInfo getDstParent(); 28 | public FileInfo getDstFile(); 29 | public BlockInfo getSrcBlock(); 30 | public BlockInfo getDstBlock(); 31 | } 32 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/rpc/RpcResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.rpc; 20 | 21 | public interface RpcResponse { 22 | public short getError(); 23 | } 24 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/rpc/RpcVoid.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.rpc; 20 | 21 | public interface RpcVoid extends RpcResponse { 22 | } 23 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/storage/StorageClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage; 20 | 21 | import java.io.IOException; 22 | 23 | import org.apache.crail.CrailBufferCache; 24 | import org.apache.crail.CrailStatistics; 25 | import org.apache.crail.conf.CrailConfiguration; 26 | import org.apache.crail.metadata.DataNodeInfo; 27 | import org.slf4j.Logger; 28 | 29 | public interface StorageClient { 30 | StorageEndpoint createEndpoint(DataNodeInfo info) throws IOException; 31 | void close() throws Exception; 32 | void init(CrailStatistics statistics, CrailBufferCache bufferCache, CrailConfiguration configuration, 33 | String[] args) throws IOException; 34 | void printConf(Logger log); 35 | 36 | 37 | @SuppressWarnings("unchecked") 38 | public static StorageClient createInstance(String name) throws Exception { 39 | Class nodeClass = Class.forName(name); 40 | if (StorageClient.class.isAssignableFrom(nodeClass)){ 41 | Class storageClientClass = (Class) nodeClass; 42 | StorageClient client = storageClientClass.newInstance(); 43 | return client; 44 | } else { 45 | throw new Exception("Cannot instantiate storage client of type " + name); 46 | } 47 | 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/storage/StorageEndpoint.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage; 20 | 21 | import java.io.IOException; 22 | 23 | import org.apache.crail.CrailBuffer; 24 | import org.apache.crail.metadata.BlockInfo; 25 | 26 | public interface StorageEndpoint{ 27 | 28 | public abstract StorageFuture write(CrailBuffer buffer, BlockInfo remoteMr, long remoteOffset) throws IOException, 29 | InterruptedException; 30 | 31 | public abstract StorageFuture read(CrailBuffer buffer, BlockInfo remoteMr, long remoteOffset) throws IOException, 32 | InterruptedException; 33 | 34 | public abstract void close() throws IOException, InterruptedException; 35 | 36 | public abstract boolean isLocal(); 37 | } 38 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/storage/StorageFuture.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage; 20 | 21 | import java.util.concurrent.Future; 22 | 23 | public interface StorageFuture extends Future { 24 | public boolean isSynchronous(); 25 | } 26 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/storage/StorageResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage; 20 | 21 | public interface StorageResult { 22 | public int getLen(); 23 | } 24 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/utils/AtomicIntegerModulo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.utils; 20 | 21 | import java.util.concurrent.atomic.AtomicInteger; 22 | 23 | public class AtomicIntegerModulo { 24 | private AtomicInteger counter; 25 | 26 | public AtomicIntegerModulo(){ 27 | this.counter = new AtomicInteger(0); 28 | } 29 | 30 | public int getAndIncrement() { 31 | return counter.getAndIncrement() & Integer.MAX_VALUE; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/utils/BlockCache.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.utils; 20 | 21 | import java.util.concurrent.ConcurrentHashMap; 22 | 23 | import org.apache.crail.metadata.BlockInfo; 24 | 25 | public class BlockCache { 26 | private ConcurrentHashMap blockCache; 27 | 28 | public BlockCache(){ 29 | this.blockCache = new ConcurrentHashMap(); 30 | } 31 | 32 | public FileBlockCache getFileBlockCache(long fd){ 33 | FileBlockCache fileBlockCache = blockCache.get(fd); 34 | if (fileBlockCache == null){ 35 | fileBlockCache = new FileBlockCache(fd); 36 | FileBlockCache oldFileBlockCache = blockCache.putIfAbsent(fd, fileBlockCache); 37 | if (oldFileBlockCache != null){ 38 | fileBlockCache = oldFileBlockCache; 39 | } 40 | } 41 | return fileBlockCache; 42 | } 43 | 44 | public void remove(long fd) { 45 | blockCache.remove(fd); 46 | } 47 | 48 | public void purge() { 49 | blockCache.clear(); 50 | } 51 | 52 | public static class FileBlockCache { 53 | private long fd; 54 | private ConcurrentHashMap fileBlockCache; 55 | 56 | public FileBlockCache(long fd){ 57 | this.fd = fd; 58 | this.fileBlockCache = new ConcurrentHashMap(); 59 | } 60 | 61 | public void put(long blockstart, BlockInfo block){ 62 | this.fileBlockCache.put(blockstart, block); 63 | } 64 | 65 | public BlockInfo get(long blockstart){ 66 | return this.fileBlockCache.get(blockstart); 67 | } 68 | 69 | public boolean containsKey(long blockstart) { 70 | return this.fileBlockCache.containsKey(blockstart); 71 | } 72 | 73 | public long getFd() { 74 | return fd; 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/utils/BufferCheckpoint.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.utils; 20 | 21 | import java.util.concurrent.ConcurrentHashMap; 22 | 23 | import org.apache.crail.CrailBuffer; 24 | import org.slf4j.Logger; 25 | 26 | public class BufferCheckpoint { 27 | private static final Logger LOG = CrailUtils.getLogger(); 28 | private ConcurrentHashMap checkpoint; 29 | 30 | public BufferCheckpoint(){ 31 | this.checkpoint = new ConcurrentHashMap(); 32 | } 33 | 34 | public void checkIn(CrailBuffer buffer) throws Exception { 35 | long address = buffer.address(); 36 | CrailBuffer oldBuf = checkpoint.putIfAbsent(address, buffer); 37 | if (oldBuf != null){ 38 | LOG.info("ERROR Buffer already in use!"); 39 | StackTraceElement[] stack = Thread.currentThread().getStackTrace(); 40 | for (StackTraceElement element : stack){ 41 | LOG.info(element.toString()); 42 | } 43 | throw new Exception("ERROR Buffer already in use!"); 44 | } 45 | } 46 | 47 | public void checkOut(CrailBuffer buffer){ 48 | long address = buffer.address(); 49 | checkpoint.remove(address); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/utils/CrailImmediateOperation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.utils; 20 | 21 | import java.util.concurrent.ExecutionException; 22 | import java.util.concurrent.Future; 23 | import java.util.concurrent.TimeUnit; 24 | import java.util.concurrent.TimeoutException; 25 | 26 | import org.apache.crail.CrailResult; 27 | 28 | public class CrailImmediateOperation implements Future, CrailResult{ 29 | private long len; 30 | 31 | public CrailImmediateOperation(int len){ 32 | this.len = len; 33 | } 34 | 35 | @Override 36 | public boolean cancel(boolean mayInterruptIfRunning) { 37 | return false; 38 | } 39 | 40 | @Override 41 | public boolean isCancelled() { 42 | return false; 43 | } 44 | 45 | @Override 46 | public boolean isDone() { 47 | return true; 48 | } 49 | 50 | @Override 51 | public CrailResult get() throws InterruptedException, ExecutionException { 52 | return this; 53 | } 54 | 55 | @Override 56 | public CrailResult get(long timeout, TimeUnit unit) 57 | throws InterruptedException, ExecutionException, TimeoutException { 58 | return this; 59 | } 60 | 61 | @Override 62 | public long getLen() { 63 | return len; 64 | } 65 | } 66 | 67 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/utils/NextBlockCache.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.utils; 20 | 21 | import java.util.concurrent.ConcurrentHashMap; 22 | 23 | import org.apache.crail.rpc.RpcFuture; 24 | import org.apache.crail.rpc.RpcGetBlock; 25 | 26 | public class NextBlockCache { 27 | private ConcurrentHashMap nextBlockCache; 28 | 29 | public NextBlockCache(){ 30 | this.nextBlockCache = new ConcurrentHashMap(); 31 | } 32 | 33 | public FileNextBlockCache getFileBlockCache(long fd){ 34 | FileNextBlockCache fileBlockCache = nextBlockCache.get(fd); 35 | if (fileBlockCache == null){ 36 | fileBlockCache = new FileNextBlockCache(fd); 37 | FileNextBlockCache oldFileBlockCache = nextBlockCache.putIfAbsent(fd, fileBlockCache); 38 | if (oldFileBlockCache != null){ 39 | fileBlockCache = oldFileBlockCache; 40 | } 41 | } 42 | return fileBlockCache; 43 | } 44 | 45 | public void remove(long fd) { 46 | nextBlockCache.remove(fd); 47 | } 48 | 49 | public void purge() { 50 | nextBlockCache.clear(); 51 | } 52 | 53 | public static class FileNextBlockCache { 54 | private long fd; 55 | private ConcurrentHashMap> fileBlockCache; 56 | 57 | public FileNextBlockCache(long fd){ 58 | this.fd = fd; 59 | this.fileBlockCache = new ConcurrentHashMap>(); 60 | } 61 | 62 | public void put(long blockstart, RpcFuture block){ 63 | this.fileBlockCache.putIfAbsent(blockstart, block); 64 | } 65 | 66 | public RpcFuture get(long blockstart){ 67 | return this.fileBlockCache.get(blockstart); 68 | } 69 | 70 | public boolean containsKey(long blockstart) { 71 | return this.fileBlockCache.containsKey(blockstart); 72 | } 73 | 74 | public long getFd() { 75 | return fd; 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /client/src/main/java/org/apache/crail/utils/RingBuffer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.utils; 20 | 21 | public class RingBuffer { 22 | public T[] elements = null; 23 | 24 | private int size = 0; 25 | private int writePos = 0; 26 | private int available = 0; 27 | 28 | public RingBuffer(int capacity) { 29 | this.size = capacity; 30 | this.elements = (T[]) new Object[capacity]; 31 | } 32 | 33 | public void clear() { 34 | this.writePos = 0; 35 | this.available = 0; 36 | } 37 | 38 | public int capacity() { 39 | return this.size; 40 | } 41 | 42 | public int size() { 43 | return this.available; 44 | } 45 | 46 | public boolean isEmpty(){ 47 | return this.available == 0; 48 | } 49 | 50 | public int remaining() { 51 | return this.size - this.available; 52 | } 53 | 54 | public boolean add(T element){ 55 | if(available < size){ 56 | if(writePos >= size){ 57 | writePos = 0; 58 | } 59 | elements[writePos] = element; 60 | writePos++; 61 | available++; 62 | return true; 63 | } 64 | 65 | return false; 66 | } 67 | 68 | public T poll() { 69 | if(available == 0){ 70 | return null; 71 | } 72 | int nextSlot = writePos - available; 73 | if(nextSlot < 0){ 74 | nextSlot += size; 75 | } 76 | T nextObj = elements[nextSlot]; 77 | available--; 78 | return nextObj; 79 | } 80 | 81 | public T peek() { 82 | if(available == 0){ 83 | return null; 84 | } 85 | int nextSlot = writePos - available; 86 | if(nextSlot < 0){ 87 | nextSlot += size; 88 | } 89 | T nextObj = elements[nextSlot]; 90 | return nextObj; 91 | } 92 | } -------------------------------------------------------------------------------- /conf/core-site.xml.template: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | fs.crail.impl 22 | org.apache.crail.hdfs.CrailHadoopFileSystem 23 | 24 | 25 | fs.defaultFS 26 | crail://localhost:9060 27 | 28 | 29 | fs.AbstractFileSystem.crail.impl 30 | org.apache.crail.hdfs.CrailHDFS 31 | 32 | 33 | io.file.buffer.size 34 | 1048576 35 | 36 | 37 | -------------------------------------------------------------------------------- /conf/crail-env.sh.template: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # This env varibale allows setting additional java parameter 3 | #CRAIL_EXTRA_JAVA_OPTIONS="-Xmx24G -Xmn16G" 4 | -------------------------------------------------------------------------------- /conf/crail-site.conf.template: -------------------------------------------------------------------------------- 1 | crail.namenode.address crail://localhost:9060 2 | crail.cachepath /dev/hugepages/cache 3 | crail.cachelimit 1073741824 4 | crail.storage.tcp.interface eth0 5 | crail.storage.tcp.datapath /dev/hugepages/data 6 | crail.storage.tcp.storagelimit 1073741824 7 | 8 | -------------------------------------------------------------------------------- /conf/log4j.properties: -------------------------------------------------------------------------------- 1 | # Set everything to be logged to the console 2 | log4j.rootCategory=INFO, console 3 | log4j.logger.com.ibm.disni=OFF, console 4 | log4j.logger.com.ibm.darpc=OFF, console 5 | log4j.appender.console=org.apache.log4j.ConsoleAppender 6 | log4j.appender.console.target=System.err 7 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 9 | 10 | log4j.additivity.com.ibm.disni=false 11 | log4j.additivity.com.ibm.darpc=false 12 | 13 | # Settings to quiet third party logs that are too verbose 14 | #log4j.logger.com.ibm.jverbs=OFF 15 | -------------------------------------------------------------------------------- /conf/slaves.template: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/incubator-crail/7245094cbf7cbb4507e31a64fed7d0017da9bd34/conf/slaves.template -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # Minimal makefile for Sphinx documentation 17 | # 18 | 19 | # You can set these variables from the command line. 20 | SPHINXOPTS = 21 | SPHINXBUILD = sphinx-build 22 | SPHINXPROJ = ApacheCrailincubating 23 | SOURCEDIR = source 24 | BUILDDIR = build 25 | 26 | # Put it first so that "make" without argument is like "make help". 27 | help: 28 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 29 | 30 | .PHONY: help Makefile 31 | 32 | # Catch-all target: route all unknown targets to Sphinx using the new 33 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 34 | %: Makefile 35 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /doc/source/contact.rst: -------------------------------------------------------------------------------- 1 | .. Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | .. use this file except in compliance with the License. You may obtain a copy of 3 | .. the License at 4 | .. 5 | .. http://www.apache.org/licenses/LICENSE-2.0 6 | .. 7 | .. Unless required by applicable law or agreed to in writing, software 8 | .. distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | .. WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | .. License for the specific language governing permissions and limitations under 11 | .. the License. 12 | 13 | Contact 14 | ======= 15 | 16 | Feel free to ask questions any questions on our mailing list: dev@crail.apache.org 17 | 18 | If you find any issues please report them at https://issues.apache.org/jira/projects/CRAIL/issues 19 | -------------------------------------------------------------------------------- /doc/source/contribute.rst: -------------------------------------------------------------------------------- 1 | .. Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | .. use this file except in compliance with the License. You may obtain a copy of 3 | .. the License at 4 | .. 5 | .. http://www.apache.org/licenses/LICENSE-2.0 6 | .. 7 | .. Unless required by applicable law or agreed to in writing, software 8 | .. distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | .. WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | .. License for the specific language governing permissions and limitations under 11 | .. the License. 12 | 13 | Contribute 14 | ========== 15 | 16 | For any potential changes/proposals we recommend that you open a `JIRA ticket `_ to have a discussion. After making necessary code changes, please open a pull request at `Github `_, and update the JIRA. 17 | -------------------------------------------------------------------------------- /doc/source/deploy.rst: -------------------------------------------------------------------------------- 1 | .. Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | .. use this file except in compliance with the License. You may obtain a copy of 3 | .. the License at 4 | .. 5 | .. http://www.apache.org/licenses/LICENSE-2.0 6 | .. 7 | .. Unless required by applicable law or agreed to in writing, software 8 | .. distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | .. WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | .. License for the specific language governing permissions and limitations under 11 | .. the License. 12 | 13 | Deploy Crail 14 | ============ 15 | 16 | .. TODO: fill in binary release url 17 | 18 | Download the latest binary image from here and :ref:`configure ` it. 19 | Alternatively you can :ref:`build from source ` or use our :ref:`Docker container image `. 20 | 21 | 22 | -------------------------------------------------------------------------------- /doc/source/fsck.rst: -------------------------------------------------------------------------------- 1 | .. Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | .. use this file except in compliance with the License. You may obtain a copy of 3 | .. the License at 4 | .. 5 | .. http://www.apache.org/licenses/LICENSE-2.0 6 | .. 7 | .. Unless required by applicable law or agreed to in writing, software 8 | .. distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | .. WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | .. License for the specific language governing permissions and limitations under 11 | .. the License. 12 | 13 | fsck 14 | ==== 15 | 16 | The fsck is used to query Crail internals and perform management operations. 17 | 18 | 19 | Reference 20 | ----------------- 21 | 22 | .. list-table:: 23 | :header-rows: 1 24 | 25 | * - Argument 26 | - Default 27 | - Experiment type 28 | - Description 29 | * - :code:`-t ` 30 | - *-* 31 | - N/A 32 | - * :code:`getLocations` 33 | * :code:`directoryDump` 34 | * :code:`namenodeDump` 35 | * :code:`blockStatistics` 36 | * :code:`ping` 37 | * :code:`createDirectory` 38 | * - :code:`-f ` 39 | - /tmp.dat 40 | - * :code:`getLocations` 41 | * :code:`directoryDump` 42 | * :code:`namenodeDump` 43 | * :code:`blockStatistics` 44 | * :code:`createDirectory` 45 | - Path to perform operation with 46 | * - :code:`-y ` 47 | - 0 48 | - * :code:`getLocations` 49 | - Offset into file 50 | * - :code:`-l ` 51 | - 1 52 | - * :code:`getLocations` 53 | - Length starting from offset (-y) 54 | * - :code:`-c ` 55 | - 0 56 | - * :code:`createDirectory` 57 | - Storage class of directory 58 | * - :code:`-p ` 59 | - 0 60 | - * :code:`createDirectory` 61 | - Location class of directory 62 | -------------------------------------------------------------------------------- /doc/source/index.rst: -------------------------------------------------------------------------------- 1 | .. Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | .. use this file except in compliance with the License. You may obtain a copy of 3 | .. the License at 4 | .. 5 | .. http://www.apache.org/licenses/LICENSE-2.0 6 | .. 7 | .. Unless required by applicable law or agreed to in writing, software 8 | .. distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | .. WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | .. License for the specific language governing permissions and limitations under 11 | .. the License. 12 | 13 | Apache Crail incubating 14 | ======================= 15 | *Apache Crail (Incubating) is a high-performance distributed data store designed for fast sharing of ephemeral data in distributed data processing workloads.* 16 | 17 | .. toctree:: 18 | :maxdepth: 1 19 | :caption: Overview 20 | 21 | introduction 22 | 23 | .. toctree:: 24 | :maxdepth: 1 25 | :caption: Deployment 26 | 27 | deploy 28 | source 29 | docker 30 | config 31 | run 32 | 33 | .. toctree:: 34 | :maxdepth: 1 35 | :caption: Applications 36 | 37 | shell 38 | iobench 39 | fsck 40 | spark 41 | 42 | .. toctree:: 43 | :maxdepth: 1 44 | :caption: Development 45 | 46 | develop 47 | 48 | .. toctree:: 49 | :maxdepth: 1 50 | :caption: Help 51 | 52 | release 53 | contribute 54 | contact 55 | -------------------------------------------------------------------------------- /doc/source/introduction.rst: -------------------------------------------------------------------------------- 1 | .. Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | .. use this file except in compliance with the License. You may obtain a copy of 3 | .. the License at 4 | .. 5 | .. http://www.apache.org/licenses/LICENSE-2.0 6 | .. 7 | .. Unless required by applicable law or agreed to in writing, software 8 | .. distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | .. WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | .. License for the specific language governing permissions and limitations under 11 | .. the License. 12 | 13 | Introduction 14 | ============ 15 | 16 | 17 | Apache Crail (Incubating) is a fast multi-tiered distributed storage system designed from ground up for high-performance network and storage hardware. The unique features of Crail include: 18 | 19 | * Zero-copy network access from userspace 20 | * Integration of multiple storage tiers such DRAM, flash and disaggregated shared storage 21 | * Ultra-low latencies for both meta data and data operations. For instance: opening, reading and closing a small file residing in the distributed DRAM tier less than 10 microseconds, which is in the same ballpark as some of the fastest RDMA-based key/value stores 22 | * High-performance sequential read/write operations: For instance: read operations on large files residing in the distributed DRAM tier are typically limited only by the performance of the network 23 | * Very low CPU consumption: a single core sharing both application and file system client can drive sequential read/write operations at the speed of up to 100Gbps and more 24 | * Asynchronous API leveraging the asynchronous nature of RDMA-based networking hardware 25 | * Extensible plugin architecture: new storage tiers tailored to specific hardware can be added easily 26 | 27 | Crail is implemented in Java offering a Java API which integrates directly with the Java off-heap memory. Crail is designed for performance critical temporary data within a scope of a rack or two. 28 | -------------------------------------------------------------------------------- /doc/source/shell.rst: -------------------------------------------------------------------------------- 1 | .. Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | .. use this file except in compliance with the License. You may obtain a copy of 3 | .. the License at 4 | .. 5 | .. http://www.apache.org/licenses/LICENSE-2.0 6 | .. 7 | .. Unless required by applicable law or agreed to in writing, software 8 | .. distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | .. WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | .. License for the specific language governing permissions and limitations under 11 | .. the License. 12 | 13 | Shell 14 | ===== 15 | 16 | Crail provides an implementation of the HDFS API thus allows interaction using the HDFS shell. 17 | For the HDFS adapter to work properly the :ref:`core-site.xml` needs to be configured 18 | properly. 19 | 20 | .. code-block:: bash 21 | 22 | $CRAIL_HOME/bin/crail fs 23 | 24 | Not all shell commands are support but the following operations have been tested to work: 25 | 26 | .. code-block:: bash 27 | 28 | $CRAIL_HOME/bin/crail fs -ls 29 | $CRAIL_HOME/bin/crail fs -mkdir 30 | $CRAIL_HOME/bin/crail fs -copyFromLocal 31 | $CRAIL_HOME/bin/crail fs -copyToLocal 32 | $CRAIL_HOME/bin/crail fs -cat 33 | 34 | -------------------------------------------------------------------------------- /doc/source/source.rst: -------------------------------------------------------------------------------- 1 | .. Licensed under the Apache License, Version 2.0 (the "License"); you may not 2 | .. use this file except in compliance with the License. You may obtain a copy of 3 | .. the License at 4 | .. 5 | .. http://www.apache.org/licenses/LICENSE-2.0 6 | .. 7 | .. Unless required by applicable law or agreed to in writing, software 8 | .. distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 9 | .. WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 10 | .. License for the specific language governing permissions and limitations under 11 | .. the License. 12 | 13 | Building from source 14 | ==================== 15 | 16 | Follow the steps below to build Crail from source. 17 | 18 | Requirements 19 | ------------ 20 | 21 | * Java 8 or higher 22 | * RDMA-based network, e.g., Infiniband, iWARP, RoCE. There are two options to run Crail without RDMA networking hardware: (a) use SoftiWARP, (b) us the TCP/DRAM storage tier 23 | * Libdisni.so, available as part of `DiSNI `_ 24 | 25 | Building 26 | -------- 27 | 28 | To build Crail from source using `Apache Maven `_ execute the following steps: 29 | 30 | 1. (a) Clone from Apache: :code:`git clone http://git-wip-us.apache.org/repos/asf/incubator-crail.git` or 31 | (b) Clone from Github: :code:`git clone https://github.com/apache/incubator-crail` or 32 | (c) Download and unpack the latest source release from `here `_ 33 | 2. Run: :code:`mvn -DskipTests install` 34 | 3. Copy tarball from :code:`assembly/target` to the cluster and unpack it using :code:`tar xvfz crail-X.Y-incubating-bin.tar.gz` 35 | 36 | **Note:** *later, when deploying Crail, make sure libdisni.so is part of your LD_LIBRARY_PATH. The easiest way to make it work is to copy libdisni.so into $CRAIL_HOME/lib/* 37 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License.) 15 | 16 | FROM ubuntu:18.04 17 | MAINTAINER Apache Crail 18 | 19 | # TODO: automate update version on new release 20 | ARG GIT_COMMIT="v1.2" 21 | ENV LOG_COMMIT=$GIT_COMMIT 22 | 23 | RUN echo "Crail-$LOG_COMMIT install openjdk8, git and envsubst" && \ 24 | apt-get update && apt-get install --no-install-recommends -y \ 25 | openjdk-8-jdk-headless \ 26 | git \ 27 | gettext-base \ 28 | libxml2-utils 29 | ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 30 | ENV PATH=${PATH}:${JAVA_HOME}/bin 31 | 32 | # We need to install maven separately since it otherwise pulls in java 10 33 | RUN echo "Crail-$LOG_COMMIT install maven" && \ 34 | apt-get install --no-install-recommends -y maven 35 | 36 | 37 | RUN echo "Crail-$LOG_COMMIT clone & build Crail repo" && \ 38 | git clone https://github.com/apache/incubator-crail.git && \ 39 | cd incubator-crail && \ 40 | git checkout ${GIT_COMMIT} && \ 41 | mvn -DskipTests package 42 | 43 | RUN echo "Move crail to /crail" && \ 44 | rm -rf /crail && \ 45 | v=`xmllint --xpath "string(/*[local-name()='project']/*[local-name()='version'])" /incubator-crail/pom.xml` && \ 46 | mv /incubator-crail/assembly/target/apache-crail-${v}-bin/apache-crail-${v} /crail 47 | 48 | ENV CRAIL_HOME /crail 49 | ENV PATH=${PATH}:${CRAIL_HOME}/bin 50 | 51 | 52 | ENV NAMENODE_HOST=localhost 53 | ENV NAMENODE_PORT=9060 54 | ENV INTERFACE=eth0 55 | ENV DATAPATH=/dev/hugepages/data 56 | ENV STORAGELIMIT=1073741824 57 | ENV CACHEPATH=/dev/hugepages/cache 58 | ENV CACHELIMIT=0 59 | 60 | COPY ./start-crail-docker.sh $CRAIL_HOME/bin 61 | COPY ./crail-env.sh $CRAIL_HOME/conf 62 | COPY ./core-site.xml.env $CRAIL_HOME/conf 63 | COPY ./crail-site.conf $CRAIL_HOME/conf 64 | 65 | ENTRYPOINT ["start-crail-docker.sh"] 66 | -------------------------------------------------------------------------------- /docker/RDMA/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License.) 15 | 16 | # TODO: automate update version 17 | FROM apache/incubator-crail:1.2 18 | MAINTAINER Apache Crail 19 | 20 | RUN echo "Crail-$LOG_COMMIT install rdma libraries and autotools" && \ 21 | apt-get update && apt-get install -y --no-install-recommends \ 22 | autoconf autotools-dev automake libtool make g++ \ 23 | librdmacm-dev libibverbs-dev ibverbs-providers 24 | 25 | RUN echo "Retrieve DiSNI jar version to match native library build" && \ 26 | DISNI_COMMIT=v$(ls $CRAIL_HOME/jars/disni* | grep -oP "\d+\.\d+(?=\.jar$)") 27 | 28 | RUN echo "Crail-$LOG_COMMIT clone and build disni native library" && \ 29 | cd && git clone https://github.com/zrlio/disni.git && \ 30 | cd ~/disni/libdisni && \ 31 | git checkout $DISNI_COMMIT && \ 32 | ./autoprepare.sh && \ 33 | ./configure --with-jdk=$JAVA_HOME && \ 34 | make && make install 35 | ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib 36 | 37 | COPY ./crail-site.conf $CRAIL_HOME/conf 38 | -------------------------------------------------------------------------------- /docker/RDMA/crail-site.conf: -------------------------------------------------------------------------------- 1 | crail.cachepath $CACHEPATH 2 | crail.cachelimit $CACHELIMIT 3 | 4 | 5 | crail.namenode.address crail://$NAMENODE_HOST:$NAMENODE_PORT 6 | crail.namenode.rpctype org.apache.crail.namenode.rpc.darpc.DaRPCNameNode 7 | crail.namenode.darpc.polling true 8 | 9 | crail.storage.rdma.interface $INTERFACE 10 | crail.storage.rdma.datapath $DATAPATH 11 | crail.storage.rdma.storagelimit $STORAGELIMIT 12 | 13 | crail.storage.types org.apache.crail.storage.rdma.RdmaStorageTier 14 | -------------------------------------------------------------------------------- /docker/core-site.xml.env: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | fs.crail.impl 22 | org.apache.crail.hdfs.CrailHadoopFileSystem 23 | 24 | 25 | fs.defaultFS 26 | crail://$NAMENODE_HOST:$NAMENODE_PORT 27 | 28 | 29 | fs.AbstractFileSystem.crail.impl 30 | org.apache.crail.hdfs.CrailHDFS 31 | 32 | 33 | io.file.buffer.size 34 | 1048576 35 | 36 | 37 | -------------------------------------------------------------------------------- /docker/crail-env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | # This env varibale allows setting additional java parameter 20 | #CRAIL_EXTRA_JAVA_OPTIONS="-Xmx24G -Xmn16G" 21 | -------------------------------------------------------------------------------- /docker/crail-site.conf: -------------------------------------------------------------------------------- 1 | crail.namenode.address crail://$NAMENODE_HOST:$NAMENODE_PORT 2 | crail.cachepath $CACHEPATH 3 | crail.cachelimit $CACHELIMIT 4 | crail.storage.tcp.interface $INTERFACE 5 | crail.storage.tcp.datapath $DATAPATH 6 | crail.storage.tcp.storagelimit $STORAGELIMIT 7 | 8 | -------------------------------------------------------------------------------- /docker/start-crail-docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # substitude env variables in core-site.xml 19 | envsubst < $CRAIL_HOME/conf/core-site.xml.env > $CRAIL_HOME/conf/core-site.xml 20 | 21 | crail $@ 22 | -------------------------------------------------------------------------------- /hdfs/src/main/java/org/apache/crail/hdfs/CrailHDFSOutputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.hdfs; 20 | 21 | import java.io.IOException; 22 | 23 | import org.apache.crail.CrailBufferedOutputStream; 24 | import org.apache.hadoop.fs.FSDataOutputStream; 25 | import org.apache.hadoop.fs.FileSystem.Statistics; 26 | 27 | public class CrailHDFSOutputStream extends FSDataOutputStream { 28 | public CrailHDFSOutputStream(CrailBufferedOutputStream outputStream, Statistics stats) throws IOException { 29 | super(outputStream, stats); 30 | } 31 | } 32 | 33 | -------------------------------------------------------------------------------- /hdfs/src/main/java/org/apache/crail/hdfs/GetConf.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.hdfs; 20 | 21 | import java.net.URI; 22 | import org.apache.hadoop.conf.Configuration; 23 | 24 | public class GetConf { 25 | 26 | public static void main(String[] args) throws Exception { 27 | if (args.length < 1){ 28 | System.out.println("Needs at least one argument"); 29 | } 30 | 31 | String key = args[0]; 32 | Configuration conf = new Configuration(); 33 | if (key.equalsIgnoreCase("namenode")){ 34 | String defaultFS = conf.get("fs.defaultFS"); 35 | URI uri = new URI(defaultFS); 36 | System.out.println(uri.getHost()); 37 | } else { 38 | String value = conf.get(key); 39 | System.out.println(value); 40 | } 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /hdfs/src/test/java/org/apache/crail/hdfs/TestCrailHDFSContract.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.hdfs; 20 | 21 | import java.net.URI; 22 | 23 | import org.apache.crail.hdfs.CrailHadoopFileSystem; 24 | import org.apache.hadoop.conf.Configuration; 25 | import org.apache.hadoop.fs.FileSystem; 26 | import org.apache.hadoop.fs.FileSystemContractBaseTest; 27 | 28 | public class TestCrailHDFSContract extends FileSystemContractBaseTest { 29 | 30 | @Override 31 | protected void setUp() throws Exception { 32 | Configuration conf = new Configuration(); 33 | fs = new CrailHadoopFileSystem(); 34 | fs.initialize(URI.create(conf.get("fs.defaultFS")), conf); 35 | } 36 | 37 | // -------------------- 38 | 39 | protected final static String TEST_UMASK = "062"; 40 | protected FileSystem fs; 41 | protected byte[] data = dataset(getBlockSize() * 2, 0, 255); 42 | 43 | @Override 44 | protected void tearDown() throws Exception { 45 | fs.delete(path("/test"), true); 46 | fs.close(); 47 | } 48 | 49 | /** 50 | * Create a dataset for use in the tests; all data is in the range base to 51 | * (base+modulo-1) inclusive 52 | * 53 | * @param len 54 | * length of data 55 | * @param base 56 | * base of the data 57 | * @param modulo 58 | * the modulo 59 | * @return the newly generated dataset 60 | */ 61 | protected byte[] dataset(int len, int base, int modulo) { 62 | byte[] dataset = new byte[len]; 63 | for (int i = 0; i < len; i++) { 64 | dataset[i] = (byte) (base + (i % modulo)); 65 | } 66 | return dataset; 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /libexec/crail-daemons.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | # Run a Hadoop command on all slave hosts. 20 | 21 | usage="Usage: crail-daemons.sh [--config confdir] [--hosts hostlistfile] [start|stop] command args..." 22 | 23 | # if no args specified, show usage 24 | if [ $# -le 1 ]; then 25 | echo $usage 26 | exit 1 27 | fi 28 | 29 | bin=`dirname "${BASH_SOURCE-$0}"` 30 | bin=`cd "$bin"; pwd` 31 | 32 | LIBEXEC_DIR="$bin"/../libexec 33 | if [ "--hostnames" = "$1" ]; then 34 | shift 35 | export HADOOP_SLAVE_NAMES=$1 36 | shift 37 | fi 38 | 39 | 40 | exec "$LIBEXEC_DIR/slaves.sh" cd "$CRAIL_HOME" \; "$LIBEXEC_DIR/crail-daemon.sh" "$@" 41 | -------------------------------------------------------------------------------- /libexec/slaves.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | # Run a shell command on all slave hosts. 20 | # 21 | # Environment Variables 22 | # 23 | # HADOOP_SLAVES File naming remote hosts. 24 | # Default is ${HADOOP_CONF_DIR}/slaves. 25 | # HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_PREFIX}/conf. 26 | # HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. 27 | # HADOOP_SSH_OPTS Options passed to ssh when running remote commands. 28 | ## 29 | 30 | usage="Usage: slaves.sh [--config confdir] command..." 31 | 32 | # if no args specified, show usage 33 | if [ $# -le 0 ]; then 34 | echo $usage 35 | exit 1 36 | fi 37 | 38 | bin=`dirname "${BASH_SOURCE-$0}"` 39 | bin=`cd "$bin"; pwd` 40 | 41 | DEFAULT_LIBEXEC_DIR="$bin"/../libexec 42 | HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} 43 | #. $HADOOP_LIBEXEC_DIR/hadoop-config.sh 44 | 45 | if [ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]; then 46 | . "${HADOOP_CONF_DIR}/hadoop-env.sh" 47 | fi 48 | 49 | # Where to start the script, see hadoop-config.sh 50 | # (it set up the variables based on command line options) 51 | if [ "$HADOOP_SLAVE_NAMES" != '' ] ; then 52 | SLAVE_NAMES=$HADOOP_SLAVE_NAMES 53 | else 54 | SLAVE_FILE=${HADOOP_SLAVES:-${CRAIL_HOME}/conf/slaves} 55 | readarray SLAVE_NAMES < "$SLAVE_FILE" 56 | #SLAVE_NAMES=$(cat "$SLAVE_FILE" | sed 's/#.*$//;/^$/d') 57 | fi 58 | 59 | # start the daemons 60 | for line in "${SLAVE_NAMES[@]}" ; do 61 | slave=$(echo $line | sed -e 's/ .*$//g') 62 | args=$(echo $line | sed -e "s/$slave//g") 63 | ssh $HADOOP_SSH_OPTS $slave $"${@// /\\ }" $args 2>&1 & 64 | if [ "$HADOOP_SLAVE_SLEEP" != "" ]; then 65 | sleep $HADOOP_SLAVE_SLEEP 66 | fi 67 | done 68 | 69 | wait 70 | -------------------------------------------------------------------------------- /licenses/LICENSE.base64.txt: -------------------------------------------------------------------------------- 1 | The person or persons who have associated work with this document (the 2 | "Dedicator" or "Certifier") hereby either (a) certifies that, to the best of 3 | his knowledge, the work of authorship identified is in the public domain of 4 | the country from which the work is published, or (b) hereby dedicates whatever 5 | copyright the dedicators holds in the work of authorship identified below (the 6 | "Work") to the public domain. A certifier, moreover, dedicates any copyright 7 | interest he may have in the associated work, and for these purposes, is 8 | described as a "dedicator" below. 9 | 10 | A certifier has taken reasonable steps to verify the copyright status of this 11 | work. Certifier recognizes that his good faith efforts may not shield him from 12 | liability if in fact the work certified is not in the public domain. 13 | 14 | Dedicator makes this dedication for the benefit of the public at large and to 15 | the detriment of the Dedicator's heirs and successors. Dedicator intends this 16 | dedication to be an overt act of relinquishment in perpetuate of all present 17 | and future rights under copyright law, whether vested or contingent, in the 18 | Work. Dedicator understands that such relinquishment of all rights includes 19 | the relinquishment of all rights to enforce (by lawsuit or otherwise) those 20 | copyrights in the Work. 21 | 22 | Dedicator recognizes that, once placed in the public domain, the Work may be 23 | freely reproduced, distributed, transmitted, used, modified, built upon, or 24 | otherwise exploited by anyone for any purpose, commercial or non-commercial, 25 | and in any way, including by methods that have not yet been invented or 26 | conceived. 27 | -------------------------------------------------------------------------------- /licenses/LICENSE.bsd2c: -------------------------------------------------------------------------------- 1 | Copyright 2 | 3 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 4 | 5 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 6 | 7 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 9 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 10 | -------------------------------------------------------------------------------- /licenses/LICENSE.bsd3c: -------------------------------------------------------------------------------- 1 | Copyright 2 | 3 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 4 | 5 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 6 | 7 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 9 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 10 | 11 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 12 | -------------------------------------------------------------------------------- /licenses/LICENSE.jsr166y.txt: -------------------------------------------------------------------------------- 1 | The person or persons who have associated work with this document (the 2 | "Dedicator" or "Certifier") hereby either (a) certifies that, to the best of 3 | his knowledge, the work of authorship identified is in the public domain of 4 | the country from which the work is published, or (b) hereby dedicates whatever 5 | copyright the dedicators holds in the work of authorship identified below (the 6 | "Work") to the public domain. A certifier, moreover, dedicates any copyright 7 | interest he may have in the associated work, and for these purposes, is 8 | described as a "dedicator" below. 9 | 10 | A certifier has taken reasonable steps to verify the copyright status of this 11 | work. Certifier recognizes that his good faith efforts may not shield him from 12 | liability if in fact the work certified is not in the public domain. 13 | 14 | Dedicator makes this dedication for the benefit of the public at large and to 15 | the detriment of the Dedicator's heirs and successors. Dedicator intends this 16 | dedication to be an overt act of relinquishment in perpetuity of all present 17 | and future rights under copyright law, whether vested or contingent, in the 18 | Work. Dedicator understands that such relinquishment of all rights includes 19 | the relinquishment of all rights to enforce (by lawsuit or otherwise) those 20 | copyrights in the Work. 21 | 22 | Dedicator recognizes that, once placed in the public domain, the Work may be 23 | freely reproduced, distributed, transmitted, used, modified, built upon, or 24 | otherwise exploited by anyone for any purpose, commercial or non-commercial, 25 | and in any way, including by methods that have not yet been invented or 26 | conceived. 27 | -------------------------------------------------------------------------------- /licenses/LICENSE.jzlib.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2000,2001,2002,2003,2004 ymnk, JCraft,Inc. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, 7 | this list of conditions and the following disclaimer. 8 | 9 | 2. Redistributions in binary form must reproduce the above copyright 10 | notice, this list of conditions and the following disclaimer in 11 | the documentation and/or other materials provided with the distribution. 12 | 13 | 3. The names of the authors may not be used to endorse or promote products 14 | derived from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, 17 | INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 18 | FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, 19 | INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, 20 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, 22 | OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 23 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 24 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 25 | EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | 27 | -------------------------------------------------------------------------------- /licenses/LICENSE.mit: -------------------------------------------------------------------------------- 1 | Copyright 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /licenses/LICENSE.protobuf.txt: -------------------------------------------------------------------------------- 1 | Copyright 2008, Google Inc. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above 11 | copyright notice, this list of conditions and the following disclaimer 12 | in the documentation and/or other materials provided with the 13 | distribution. 14 | * Neither the name of Google Inc. nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | Code generated by the Protocol Buffer compiler is owned by the owner 31 | of the input file used when generating it. This code is not 32 | standalone and requires a support library to be linked with it. This 33 | support library is itself covered by the above license. 34 | -------------------------------------------------------------------------------- /licenses/LICENSE.slf4j.txt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2004-2007 QOS.ch 3 | * All rights reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person obtaining 6 | * a copy of this software and associated documentation files (the 7 | * "Software"), to deal in the Software without restriction, including 8 | * without limitation the rights to use, copy, modify, merge, publish, 9 | * distribute, sublicense, and/or sell copies of the Software, and to 10 | * permit persons to whom the Software is furnished to do so, subject to 11 | * the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be 14 | * included in all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | */ 24 | -------------------------------------------------------------------------------- /licenses/LICENSE.webbit.txt: -------------------------------------------------------------------------------- 1 | (BSD License: http://www.opensource.org/licenses/bsd-license) 2 | 3 | Copyright (c) 2011, Joe Walnes, Aslak Hellesøy and contributors 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or 7 | without modification, are permitted provided that the 8 | following conditions are met: 9 | 10 | * Redistributions of source code must retain the above 11 | copyright notice, this list of conditions and the 12 | following disclaimer. 13 | 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the 16 | following disclaimer in the documentation and/or other 17 | materials provided with the distribution. 18 | 19 | * Neither the name of the Webbit nor the names of 20 | its contributors may be used to endorse or promote products 21 | derived from this software without specific prior written 22 | permission. 23 | 24 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 25 | CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 26 | INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 27 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 28 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 29 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 30 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 31 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 32 | GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 33 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 34 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 36 | OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 37 | POSSIBILITY OF SUCH DAMAGE. 38 | 39 | -------------------------------------------------------------------------------- /namenode/src/main/java/org/apache/crail/namenode/ElasticNameNodeService.java: -------------------------------------------------------------------------------- 1 | package org.apache.crail.namenode; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.crail.conf.CrailConstants; 6 | 7 | public class ElasticNameNodeService extends NameNodeService { 8 | 9 | PolicyRunner policyRunner; 10 | 11 | public ElasticNameNodeService() throws IOException { 12 | 13 | this.policyRunner = new FreeCapacityPolicy(this, 14 | CrailConstants.ELASTICSTORE_SCALEUP, 15 | CrailConstants.ELASTICSTORE_SCALEDOWN, 16 | CrailConstants.ELASTICSTORE_MINNODES, 17 | CrailConstants.ELASTICSTORE_MAXNODES); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /namenode/src/main/java/org/apache/crail/namenode/GCServer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.namenode; 20 | 21 | import java.util.concurrent.DelayQueue; 22 | 23 | import org.apache.crail.utils.CrailUtils; 24 | import org.slf4j.Logger; 25 | 26 | public class GCServer implements Runnable { 27 | private static final Logger LOG = CrailUtils.getLogger(); 28 | 29 | private NameNodeService rpcService; 30 | private DelayQueue deleteQueue; 31 | 32 | public GCServer(NameNodeService service, DelayQueue deleteQueue){ 33 | this.rpcService = service; 34 | this.deleteQueue = deleteQueue; 35 | } 36 | 37 | @Override 38 | public void run() { 39 | while(true){ 40 | try{ 41 | AbstractNode file = deleteQueue.take(); 42 | if (file.getType().isContainer()){ 43 | file.clearChildren(deleteQueue); 44 | } 45 | rpcService.freeFile(file); 46 | } catch(Exception e){ 47 | LOG.info("Exception during GC: " + e.getMessage()); 48 | } 49 | } 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /namenode/src/main/java/org/apache/crail/namenode/KeyValueBlocks.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.namenode; 20 | 21 | import org.apache.crail.CrailNodeType; 22 | 23 | public class KeyValueBlocks extends FileBlocks { 24 | public KeyValueBlocks(long fd, int fileComponent, CrailNodeType type, 25 | int storageClass, int locationClass, boolean enumerable) { 26 | super(fd, fileComponent, type, storageClass, locationClass, enumerable); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /namenode/src/main/java/org/apache/crail/namenode/MultiFileBlocks.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.namenode; 20 | 21 | import org.apache.crail.CrailNodeType; 22 | 23 | public class MultiFileBlocks extends DirectoryBlocks { 24 | 25 | MultiFileBlocks(long fd, int fileComponent, CrailNodeType type, 26 | int storageClass, int locationClass, boolean enumerable) { 27 | super(fd, fileComponent, type, storageClass, locationClass, enumerable); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /namenode/src/main/java/org/apache/crail/namenode/NameNodeBlockInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.namenode; 20 | 21 | import org.apache.crail.metadata.BlockInfo; 22 | import org.apache.crail.metadata.DataNodeInfo; 23 | 24 | public class NameNodeBlockInfo extends BlockInfo { 25 | private BlockInfo region; 26 | private long offset; 27 | 28 | public NameNodeBlockInfo(BlockInfo region, long offset, int length){ 29 | this.region = region; 30 | this.offset = offset; 31 | this.length = length; 32 | 33 | this.dnInfo = this.getDnInfo(); 34 | this.lba = this.getLba(); 35 | this.addr = this.getAddr(); 36 | this.lkey = this.getLkey(); 37 | } 38 | 39 | @Override 40 | public long getLba() { 41 | return region.getLba() + offset; 42 | } 43 | 44 | @Override 45 | public long getAddr() { 46 | return region.getAddr() + offset; 47 | } 48 | 49 | @Override 50 | public int getLkey() { 51 | return region.getLkey(); 52 | } 53 | 54 | @Override 55 | public DataNodeInfo getDnInfo() { 56 | return region.getDnInfo(); 57 | } 58 | 59 | public BlockInfo getRegion() { 60 | return region; 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /namenode/src/main/java/org/apache/crail/namenode/PolicyRunner.java: -------------------------------------------------------------------------------- 1 | package org.apache.crail.namenode; 2 | 3 | import org.apache.crail.conf.CrailConstants; 4 | import org.apache.crail.rpc.RpcNameNodeService; 5 | import org.apache.crail.utils.CrailUtils; 6 | import org.slf4j.Logger; 7 | 8 | public abstract class PolicyRunner implements Runnable { 9 | 10 | static final Logger LOG = CrailUtils.getLogger(); 11 | NameNodeService service; 12 | int instances = 0; 13 | 14 | PolicyRunner(RpcNameNodeService service){ 15 | this.service = (NameNodeService) service; 16 | Thread runner = new Thread(this); 17 | runner.start(); 18 | } 19 | 20 | public abstract void checkPolicy(); 21 | 22 | public void run() { 23 | 24 | while (true) { 25 | checkPolicy(); 26 | 27 | try { 28 | Thread.sleep(CrailConstants.ELASTICSTORE_POLICYRUNNER_INTERVAL); 29 | } catch(Exception e) { 30 | e.printStackTrace(); 31 | } 32 | } 33 | 34 | } 35 | 36 | public void launchDatanode() { 37 | 38 | try { 39 | String port = Integer.toString(50020+this.instances); 40 | Process p = new ProcessBuilder(System.getenv("CRAIL_HOME") + "/bin/crail", "datanode", "--", "-p" + port).start(); 41 | 42 | LOG.info("Launched new datanode instance"); 43 | this.instances++; 44 | 45 | } catch(Exception e) { 46 | LOG.error("Unable to launch datanode"); 47 | e.printStackTrace(); 48 | } 49 | 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /namenode/src/main/java/org/apache/crail/namenode/Sequencer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.namenode; 20 | 21 | public interface Sequencer { 22 | long getNextId(); 23 | } 24 | -------------------------------------------------------------------------------- /namenode/src/main/java/org/apache/crail/namenode/TableBlocks.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.namenode; 20 | 21 | import org.apache.crail.CrailNodeType; 22 | import org.apache.crail.conf.CrailConstants; 23 | 24 | public class TableBlocks extends DirectoryBlocks { 25 | 26 | TableBlocks(long fd, int fileComponent, CrailNodeType type, 27 | int storageClass, int locationClass, boolean enumerable) { 28 | super(fd, fileComponent, type, storageClass, locationClass, enumerable); 29 | } 30 | 31 | @Override 32 | public AbstractNode putChild(AbstractNode child) throws Exception { 33 | if (!child.getType().isKeyValue()){ 34 | throw new Exception("Attempt to create key/value pair in container other than a table"); 35 | } 36 | 37 | AbstractNode oldNode = children.put(child.getComponent(), child); 38 | if (child.isEnumerable()) { 39 | child.setDirOffset(dirOffsetCounter.getAndAdd(CrailConstants.DIRECTORY_RECORD)); 40 | } 41 | return oldNode; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /rpc-darpc/src/main/java/org/apache/crail/namenode/rpc/darpc/DaRPCNameNode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.namenode.rpc.darpc; 20 | 21 | import org.apache.crail.rpc.RpcBinding; 22 | import org.apache.crail.rpc.RpcNameNodeService; 23 | import org.apache.crail.rpc.RpcServer; 24 | 25 | public class DaRPCNameNode extends DaRPCNameNodeClient implements RpcBinding { 26 | @Override 27 | public RpcServer launchServer(RpcNameNodeService service) { 28 | return new DaRPCNameNodeServer(service); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /rpc-darpc/src/main/java/org/apache/crail/namenode/rpc/darpc/DaRPCNameNodeFuture.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.namenode.rpc.darpc; 20 | 21 | import java.util.concurrent.ExecutionException; 22 | import java.util.concurrent.TimeUnit; 23 | import java.util.concurrent.TimeoutException; 24 | 25 | import org.apache.crail.rpc.RpcFuture; 26 | 27 | import com.ibm.darpc.DaRPCFuture; 28 | 29 | public class DaRPCNameNodeFuture implements RpcFuture { 30 | private DaRPCFuture future; 31 | private boolean prefetched; 32 | private T response; 33 | 34 | public DaRPCNameNodeFuture(DaRPCFuture future, T response) { 35 | this.future = future; 36 | this.response = response; 37 | this.prefetched = false; 38 | } 39 | 40 | @Override 41 | public T get() throws InterruptedException, ExecutionException { 42 | future.get(); 43 | return response; 44 | } 45 | 46 | @Override 47 | public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { 48 | future.get(timeout, unit); 49 | return response; 50 | } 51 | 52 | @Override 53 | public boolean isDone() { 54 | return future.isDone(); 55 | } 56 | 57 | @Override 58 | public int getTicket() { 59 | return future.getTicket(); 60 | } 61 | 62 | @Override 63 | public boolean isPrefetched() { 64 | return prefetched; 65 | } 66 | 67 | @Override 68 | public void setPrefetched(boolean prefetched) { 69 | this.prefetched = prefetched; 70 | } 71 | 72 | @Override 73 | public boolean cancel(boolean mayInterruptIfRunning) { 74 | return future.cancel(mayInterruptIfRunning); 75 | } 76 | 77 | @Override 78 | public boolean isCancelled() { 79 | return future.isCancelled(); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /rpc-darpc/src/main/java/org/apache/crail/namenode/rpc/darpc/DaRPCNameNodeProtocol.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.namenode.rpc.darpc; 20 | 21 | import com.ibm.darpc.DaRPCProtocol; 22 | 23 | public class DaRPCNameNodeProtocol implements DaRPCProtocol{ 24 | 25 | @Override 26 | public DaRPCNameNodeRequest createRequest() { 27 | return new DaRPCNameNodeRequest(); 28 | } 29 | 30 | @Override 31 | public DaRPCNameNodeResponse createResponse() { 32 | return new DaRPCNameNodeResponse(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /rpc-narpc/src/main/assembly/assembly.xml: -------------------------------------------------------------------------------- 1 | 18 | 21 | dist 22 | 23 | dir 24 | 25 | false 26 | 27 | 28 | 29 | jars 30 | true 31 | false 32 | runtime 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /rpc-narpc/src/main/java/org/apache/crail/namenode/rpc/tcp/TcpFuture.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.namenode.rpc.tcp; 20 | 21 | import java.util.concurrent.ExecutionException; 22 | import java.util.concurrent.TimeUnit; 23 | import java.util.concurrent.TimeoutException; 24 | 25 | import org.apache.crail.rpc.RpcFuture; 26 | 27 | import com.ibm.narpc.NaRPCFuture; 28 | 29 | public class TcpFuture implements RpcFuture { 30 | private NaRPCFuture future; 31 | private T response; 32 | private boolean prefetched; 33 | 34 | public TcpFuture(NaRPCFuture future, T resp) { 35 | this.future = future; 36 | this.response = resp; 37 | this.prefetched = false; 38 | } 39 | 40 | @Override 41 | public boolean cancel(boolean mayInterruptIfRunning) { 42 | return false; 43 | } 44 | 45 | @Override 46 | public boolean isCancelled() { 47 | return false; 48 | } 49 | 50 | @Override 51 | public boolean isDone() { 52 | return future.isDone(); 53 | } 54 | 55 | @Override 56 | public T get() throws InterruptedException, ExecutionException { 57 | future.get(); 58 | return response; 59 | } 60 | 61 | @Override 62 | public T get(long timeout, TimeUnit unit) throws InterruptedException, 63 | ExecutionException, TimeoutException { 64 | future.get(timeout, unit); 65 | return response; 66 | } 67 | 68 | @Override 69 | public int getTicket() { 70 | return (int) future.getTicket(); 71 | } 72 | 73 | @Override 74 | public boolean isPrefetched() { 75 | return prefetched; 76 | } 77 | 78 | @Override 79 | public void setPrefetched(boolean prefetched) { 80 | this.prefetched = prefetched; 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /rpc-narpc/src/main/java/org/apache/crail/namenode/rpc/tcp/TcpNameNode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.namenode.rpc.tcp; 20 | 21 | import org.apache.crail.rpc.RpcBinding; 22 | import org.apache.crail.rpc.RpcNameNodeService; 23 | import org.apache.crail.rpc.RpcServer; 24 | 25 | public class TcpNameNode extends TcpNameNodeClient implements RpcBinding { 26 | 27 | @Override 28 | public RpcServer launchServer(RpcNameNodeService service) { 29 | try { 30 | return new TcpNameNodeServer(service); 31 | } catch(Exception e){ 32 | return null; 33 | } 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /rpc-narpc/src/main/java/org/apache/crail/namenode/rpc/tcp/TcpNameNodeClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.namenode.rpc.tcp; 20 | 21 | import java.io.IOException; 22 | import java.net.InetSocketAddress; 23 | import java.util.LinkedList; 24 | 25 | import org.apache.crail.conf.CrailConfiguration; 26 | import org.apache.crail.rpc.RpcClient; 27 | import org.apache.crail.rpc.RpcConnection; 28 | import org.slf4j.Logger; 29 | 30 | import com.ibm.narpc.NaRPCClientGroup; 31 | import com.ibm.narpc.NaRPCEndpoint; 32 | 33 | public class TcpNameNodeClient implements RpcClient { 34 | private NaRPCClientGroup clientGroup; 35 | private LinkedList allConnections; 36 | 37 | public void init(CrailConfiguration conf, String[] strings) throws IOException { 38 | try { 39 | TcpRpcConstants.updateConstants(conf); 40 | TcpRpcConstants.verify(); 41 | this.clientGroup = new NaRPCClientGroup(TcpRpcConstants.NAMENODE_TCP_QUEUEDEPTH, TcpRpcConstants.NAMENODE_TCP_MESSAGESIZE, true); 42 | this.allConnections = new LinkedList(); 43 | } catch(Exception e){ 44 | throw new IOException(e); 45 | } 46 | } 47 | 48 | public void printConf(Logger logger) { 49 | TcpRpcConstants.printConf(logger); 50 | } 51 | 52 | /* This function comes from RPCClient interface */ 53 | public RpcConnection connect(InetSocketAddress address) throws IOException { 54 | try { 55 | NaRPCEndpoint endpoint = clientGroup.createEndpoint(); 56 | endpoint.connect(address); 57 | TcpRpcConnection connection = new TcpRpcConnection(endpoint); 58 | allConnections.add(connection); 59 | return connection; 60 | } catch(Exception e){ 61 | throw new IOException(e); 62 | } 63 | } 64 | 65 | @Override 66 | public void close() { 67 | try { 68 | for (TcpRpcConnection connection : allConnections){ 69 | connection.close(); 70 | } 71 | } catch(Exception e){ 72 | } 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /rpc-narpc/src/main/java/org/apache/crail/namenode/rpc/tcp/TcpNameNodeServer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.namenode.rpc.tcp; 20 | 21 | import java.io.IOException; 22 | import java.net.InetSocketAddress; 23 | 24 | import org.apache.crail.conf.CrailConfiguration; 25 | import org.apache.crail.rpc.RpcNameNodeService; 26 | import org.apache.crail.rpc.RpcServer; 27 | import org.apache.crail.utils.CrailUtils; 28 | import org.slf4j.Logger; 29 | 30 | import com.ibm.narpc.NaRPCServerChannel; 31 | import com.ibm.narpc.NaRPCServerEndpoint; 32 | import com.ibm.narpc.NaRPCServerGroup; 33 | 34 | public class TcpNameNodeServer extends RpcServer { 35 | private static final Logger LOG = CrailUtils.getLogger(); 36 | 37 | private TcpRpcDispatcher dispatcher; 38 | private NaRPCServerGroup serverGroup; 39 | private NaRPCServerEndpoint serverEndpoint; 40 | 41 | public TcpNameNodeServer(RpcNameNodeService service) throws IOException { 42 | this.dispatcher = new TcpRpcDispatcher(service); 43 | } 44 | 45 | @Override 46 | public void init(CrailConfiguration conf, String[] arg1) throws Exception { 47 | TcpRpcConstants.updateConstants(conf); 48 | TcpRpcConstants.verify(); 49 | this.serverGroup = new NaRPCServerGroup( 50 | dispatcher, TcpRpcConstants.NAMENODE_TCP_QUEUEDEPTH, 51 | TcpRpcConstants.NAMENODE_TCP_MESSAGESIZE, true, TcpRpcConstants.NAMENODE_TCP_CORES); 52 | this.serverEndpoint = serverGroup.createServerEndpoint(); 53 | InetSocketAddress inetSocketAddress = CrailUtils.getNameNodeAddress(); 54 | serverEndpoint.bind(inetSocketAddress); 55 | } 56 | 57 | @Override 58 | public void printConf(Logger logger) { 59 | TcpRpcConstants.printConf(logger); 60 | } 61 | 62 | public void run() { 63 | try { 64 | while (true) { 65 | NaRPCServerChannel endpoint = serverEndpoint.accept(); 66 | LOG.info("new connection from " + endpoint.address()); 67 | } 68 | } catch (Exception e) { 69 | e.printStackTrace(); 70 | } 71 | 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /rpc-narpc/src/main/java/org/apache/crail/namenode/rpc/tcp/TcpRpcConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.namenode.rpc.tcp; 20 | 21 | import java.io.IOException; 22 | 23 | import org.apache.crail.conf.CrailConfiguration; 24 | import org.apache.crail.utils.CrailUtils; 25 | import org.slf4j.Logger; 26 | 27 | public class TcpRpcConstants { 28 | private static final Logger LOG = CrailUtils.getLogger(); 29 | 30 | public static final String NAMENODE_TCP_QUEUEDEPTH_KEY = "crail.namenode.tcp.queueDepth"; 31 | public static int NAMENODE_TCP_QUEUEDEPTH = 32; 32 | 33 | public static final String NAMENODE_TCP_MESSAGESIZE_KEY = "crail.namenode.tcp.messageSize"; 34 | public static int NAMENODE_TCP_MESSAGESIZE = 512; 35 | 36 | public static final String NAMENODE_TCP_CORES_KEY = "crail.namenode.tcp.cores"; 37 | public static int NAMENODE_TCP_CORES = 1; 38 | 39 | public static void updateConstants(CrailConfiguration conf){ 40 | if (conf.get(NAMENODE_TCP_QUEUEDEPTH_KEY) != null) { 41 | NAMENODE_TCP_QUEUEDEPTH = Integer.parseInt(conf.get(NAMENODE_TCP_QUEUEDEPTH_KEY)); 42 | } 43 | if (conf.get(NAMENODE_TCP_MESSAGESIZE_KEY) != null) { 44 | NAMENODE_TCP_MESSAGESIZE = Integer.parseInt(conf.get(NAMENODE_TCP_MESSAGESIZE_KEY)); 45 | } 46 | if (conf.get(NAMENODE_TCP_CORES_KEY) != null) { 47 | NAMENODE_TCP_CORES = Integer.parseInt(conf.get(NAMENODE_TCP_CORES_KEY)); 48 | } 49 | } 50 | 51 | public static void verify() throws IOException { 52 | } 53 | 54 | public static void printConf(Logger logger) { 55 | LOG.info(NAMENODE_TCP_QUEUEDEPTH_KEY + " " + NAMENODE_TCP_QUEUEDEPTH); 56 | LOG.info(NAMENODE_TCP_MESSAGESIZE_KEY + " " + NAMENODE_TCP_MESSAGESIZE); 57 | LOG.info(NAMENODE_TCP_CORES_KEY + " " + NAMENODE_TCP_CORES); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /rpc/pom.xml: -------------------------------------------------------------------------------- 1 | 18 | 19 | 4.0.0 20 | 21 | 22 | org.apache.crail 23 | crail-parent 24 | 1.3-incubating-SNAPSHOT 25 | ../pom.xml 26 | 27 | 28 | org.apache.crail 29 | crail-rpc 30 | jar 31 | 1.3-incubating-SNAPSHOT 32 | Crail RPC Project 33 | http://maven.apache.org 34 | 35 | 36 | 37 | junit 38 | junit 39 | 40 | 41 | log4j 42 | log4j 43 | 44 | 45 | org.apache.crail 46 | crail-client 47 | ${project.parent.version} 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | org.apache.maven.plugins 57 | maven-jar-plugin 58 | 2.6 59 | 60 | 61 | 62 | test-jar 63 | 64 | 65 | 66 | 67 | 68 | 69 | org.apache.maven.plugins 70 | maven-compiler-plugin 71 | 3.3 72 | 73 | 1.8 74 | 1.8 75 | 76 | 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /rpc/src/main/java/org/apache/crail/rpc/RpcBinding.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.rpc; 20 | 21 | import org.apache.crail.rpc.RpcClient; 22 | 23 | 24 | public interface RpcBinding extends RpcClient { 25 | public RpcServer launchServer(RpcNameNodeService service); 26 | 27 | @SuppressWarnings("unchecked") 28 | public static RpcBinding createInstance(String name) throws Exception { 29 | Class nodeClass = Class.forName(name); 30 | if (RpcBinding.class.isAssignableFrom(nodeClass)){ 31 | Class bindingClass = (Class) nodeClass; 32 | RpcBinding bindingInstance = bindingClass.newInstance(); 33 | return bindingInstance; 34 | } else { 35 | throw new Exception("Cannot instantiate datanode of type " + name); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /rpc/src/main/java/org/apache/crail/rpc/RpcNameNodeState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.rpc; 20 | 21 | public interface RpcNameNodeState { 22 | public short getError(); 23 | public void setError(short error); 24 | } 25 | -------------------------------------------------------------------------------- /rpc/src/main/java/org/apache/crail/rpc/RpcServer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.rpc; 20 | 21 | import org.apache.crail.conf.Configurable; 22 | 23 | public abstract class RpcServer implements Configurable { 24 | public abstract void run(); 25 | } 26 | -------------------------------------------------------------------------------- /storage-narpc/src/main/assembly/assembly.xml: -------------------------------------------------------------------------------- 1 | 18 | 19 | 22 | dist 23 | 24 | dir 25 | 26 | false 27 | 28 | 29 | 30 | jars 31 | true 32 | false 33 | runtime 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage.tcp; 20 | 21 | import java.io.IOException; 22 | 23 | import org.apache.crail.CrailBufferCache; 24 | import org.apache.crail.CrailStatistics; 25 | import org.apache.crail.conf.CrailConfiguration; 26 | import org.apache.crail.conf.CrailConstants; 27 | import org.apache.crail.metadata.DataNodeInfo; 28 | import org.apache.crail.storage.StorageClient; 29 | import org.apache.crail.storage.StorageEndpoint; 30 | import org.apache.crail.utils.CrailUtils; 31 | import org.slf4j.Logger; 32 | 33 | import com.ibm.narpc.NaRPCClientGroup; 34 | import com.ibm.narpc.NaRPCEndpoint; 35 | 36 | public class TcpStorageClient implements StorageClient { 37 | private NaRPCClientGroup clientGroup; 38 | 39 | @Override 40 | public void init(CrailStatistics statistics, CrailBufferCache bufferCache, CrailConfiguration conf, String[] args) 41 | throws IOException { 42 | TcpStorageConstants.updateConstants(conf); 43 | 44 | this.clientGroup = new NaRPCClientGroup(TcpStorageConstants.STORAGE_TCP_QUEUE_DEPTH, (int) CrailConstants.BLOCK_SIZE*2, false); 45 | } 46 | 47 | @Override 48 | public void printConf(Logger logger) { 49 | TcpStorageConstants.printConf(logger); 50 | } 51 | 52 | @Override 53 | public void close() throws Exception { 54 | } 55 | 56 | @Override 57 | public StorageEndpoint createEndpoint(DataNodeInfo info) throws IOException { 58 | try { 59 | NaRPCEndpoint narpcEndpoint = clientGroup.createEndpoint(); 60 | TcpStorageEndpoint endpoint = new TcpStorageEndpoint(narpcEndpoint); 61 | endpoint.connect(CrailUtils.datanodeInfo2SocketAddr(info)); 62 | return endpoint; 63 | } catch(Exception e){ 64 | throw new IOException(e); 65 | } 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageFuture.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage.tcp; 20 | 21 | import java.util.concurrent.ExecutionException; 22 | import java.util.concurrent.TimeUnit; 23 | import java.util.concurrent.TimeoutException; 24 | 25 | import org.apache.crail.storage.StorageFuture; 26 | import org.apache.crail.storage.StorageResult; 27 | 28 | import com.ibm.narpc.NaRPCFuture; 29 | 30 | public class TcpStorageFuture implements StorageFuture, StorageResult { 31 | private NaRPCFuture future; 32 | private int len; 33 | 34 | public TcpStorageFuture(NaRPCFuture future, int len) { 35 | this.future = future; 36 | this.len = len; 37 | } 38 | 39 | @Override 40 | public boolean cancel(boolean mayInterruptIfRunning) { 41 | return false; 42 | } 43 | 44 | @Override 45 | public boolean isCancelled() { 46 | return false; 47 | } 48 | 49 | @Override 50 | public boolean isDone() { 51 | return future.isDone(); 52 | } 53 | 54 | @Override 55 | public StorageResult get() throws InterruptedException, ExecutionException { 56 | future.get(); 57 | return this; 58 | } 59 | 60 | @Override 61 | public StorageResult get(long timeout, TimeUnit unit) 62 | throws InterruptedException, ExecutionException, TimeoutException { 63 | future.get(timeout, unit); 64 | return this; 65 | } 66 | 67 | @Override 68 | public boolean isSynchronous() { 69 | return false; 70 | } 71 | 72 | @Override 73 | public int getLen() { 74 | return len; 75 | } 76 | 77 | } 78 | -------------------------------------------------------------------------------- /storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageProtocol.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage.tcp; 20 | 21 | public class TcpStorageProtocol { 22 | public static final int REQ_READ = 1; 23 | public static final int REQ_WRITE = 2; 24 | 25 | public static final int RET_OK = 0; 26 | public static final int RET_RPC_UNKNOWN = 1; 27 | } 28 | -------------------------------------------------------------------------------- /storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageTier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage.tcp; 20 | 21 | import org.apache.crail.storage.StorageServer; 22 | import org.apache.crail.storage.StorageTier; 23 | 24 | public class TcpStorageTier extends TcpStorageClient implements StorageTier { 25 | public StorageServer launchServer () throws Exception { 26 | TcpStorageServer datanodeServer = new TcpStorageServer(); 27 | return datanodeServer; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /storage-narpc/src/main/java/org/apache/crail/storage/tcp/TcpStorageUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage.tcp; 20 | 21 | import java.nio.ByteBuffer; 22 | 23 | import org.apache.crail.utils.CrailUtils; 24 | import org.slf4j.Logger; 25 | 26 | public class TcpStorageUtils { 27 | private static final Logger LOG = CrailUtils.getLogger(); 28 | public static void printBuffer(String name, ByteBuffer buffer){ 29 | String state = buffer.toString(); 30 | String data = ""; 31 | for (int i = 0; i < buffer.remaining(); i++){ 32 | data += buffer.get(buffer.position() + i) + ","; 33 | } 34 | LOG.info("buffer " + name + ", value " + state + ": " + data); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /storage-nvmf/src/main/java/org/apache/crail/storage/nvmf/NvmfStorageTier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage.nvmf; 20 | 21 | import org.apache.crail.storage.StorageServer; 22 | import org.apache.crail.storage.StorageTier; 23 | import org.apache.crail.utils.CrailUtils; 24 | import org.slf4j.Logger; 25 | 26 | public class NvmfStorageTier extends NvmfStorageClient implements StorageTier { 27 | private static final Logger LOG = CrailUtils.getLogger(); 28 | 29 | public StorageServer launchServer() throws Exception { 30 | LOG.info("initalizing NVMf storage tier"); 31 | NvmfStorageServer storageServer = new NvmfStorageServer(); 32 | return storageServer; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /storage-nvmf/src/main/java/org/apache/crail/storage/nvmf/client/NvmfRegisteredBufferCache.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage.nvmf.client; 20 | 21 | import com.ibm.jnvmf.Freeable; 22 | import com.ibm.jnvmf.KeyedNativeBuffer; 23 | import com.ibm.jnvmf.QueuePair; 24 | import org.apache.crail.CrailBuffer; 25 | 26 | import java.io.IOException; 27 | import java.util.Map; 28 | import java.util.concurrent.ConcurrentHashMap; 29 | 30 | class NvmfRegisteredBufferCache implements Freeable { 31 | private final QueuePair queuePair; 32 | private final Map regionMap; 33 | private boolean valid; 34 | 35 | public NvmfRegisteredBufferCache(QueuePair queuePair) { 36 | this.queuePair = queuePair; 37 | this.regionMap = new ConcurrentHashMap<>(); 38 | this.valid = true; 39 | } 40 | 41 | int getRemoteKey(CrailBuffer buffer) throws IOException { 42 | CrailBuffer regionBuffer = buffer.getRegion(); 43 | KeyedNativeBuffer keyedNativeBuffer = regionMap.get(regionBuffer.address()); 44 | if (keyedNativeBuffer == null) { 45 | /* region has not been registered yet */ 46 | keyedNativeBuffer = queuePair.registerMemory(regionBuffer.getByteBuffer()); 47 | KeyedNativeBuffer prevKeyedNativeBuffer = 48 | regionMap.putIfAbsent(keyedNativeBuffer.getAddress(), keyedNativeBuffer); 49 | if (prevKeyedNativeBuffer != null) { 50 | /* someone registered the same region in parallel */ 51 | keyedNativeBuffer.free(); 52 | keyedNativeBuffer = prevKeyedNativeBuffer; 53 | } 54 | } 55 | return keyedNativeBuffer.getRemoteKey(); 56 | } 57 | 58 | 59 | @Override 60 | public void free() throws IOException { 61 | for (KeyedNativeBuffer buffer : regionMap.values()) { 62 | buffer.free(); 63 | } 64 | valid = false; 65 | } 66 | 67 | @Override 68 | public boolean isValid() { 69 | return valid; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /storage-rdma/src/main/java/org/apache/crail/storage/rdma/RdmaStorageEndpointFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage.rdma; 20 | 21 | import java.io.IOException; 22 | 23 | import com.ibm.disni.*; 24 | import com.ibm.disni.verbs.*; 25 | 26 | public class RdmaStorageEndpointFactory implements RdmaEndpointFactory { 27 | private RdmaStorageServer closer; 28 | private RdmaActiveEndpointGroup group; 29 | 30 | public RdmaStorageEndpointFactory(RdmaActiveEndpointGroup group, RdmaStorageServer closer){ 31 | this.group = group; 32 | this.closer = closer; 33 | } 34 | 35 | @Override 36 | public RdmaStorageServerEndpoint createEndpoint(RdmaCmId id, boolean serverSide) throws IOException { 37 | return new RdmaStorageServerEndpoint(group, id, closer, serverSide); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /storage-rdma/src/main/java/org/apache/crail/storage/rdma/RdmaStorageGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage.rdma; 20 | 21 | import java.io.IOException; 22 | 23 | import org.apache.crail.metadata.DataNodeInfo; 24 | import org.apache.crail.storage.StorageEndpoint; 25 | 26 | public interface RdmaStorageGroup { 27 | 28 | public StorageEndpoint createEndpoint(DataNodeInfo info) throws IOException; 29 | 30 | public void close() throws InterruptedException, IOException; 31 | 32 | public int getType(); 33 | } -------------------------------------------------------------------------------- /storage-rdma/src/main/java/org/apache/crail/storage/rdma/RdmaStorageServerEndpoint.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage.rdma; 20 | 21 | import java.io.IOException; 22 | 23 | import com.ibm.disni.verbs.*; 24 | import com.ibm.disni.*; 25 | 26 | public class RdmaStorageServerEndpoint extends RdmaActiveEndpoint { 27 | private RdmaStorageServer closer; 28 | 29 | public RdmaStorageServerEndpoint(RdmaActiveEndpointGroup endpointGroup, RdmaCmId idPriv, RdmaStorageServer closer, boolean serverSide) throws IOException { 30 | super(endpointGroup, idPriv, serverSide); 31 | this.closer = closer; 32 | } 33 | 34 | public void dispatchCqEvent(IbvWC wc) throws IOException { 35 | 36 | } 37 | 38 | public synchronized void dispatchCmEvent(RdmaCmEvent cmEvent) 39 | throws IOException { 40 | super.dispatchCmEvent(cmEvent); 41 | int eventType = cmEvent.getEvent(); 42 | if (eventType == RdmaCmEvent.EventType.RDMA_CM_EVENT_DISCONNECTED 43 | .ordinal()) { 44 | closer.close(this); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /storage-rdma/src/main/java/org/apache/crail/storage/rdma/RdmaStorageTier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage.rdma; 20 | 21 | import org.apache.crail.storage.StorageServer; 22 | import org.apache.crail.storage.StorageTier; 23 | 24 | public class RdmaStorageTier extends RdmaStorageClient implements StorageTier { 25 | 26 | public StorageServer launchServer () throws Exception { 27 | RdmaStorageServer datanodeServer = new RdmaStorageServer(); 28 | return datanodeServer; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /storage-rdma/src/main/java/org/apache/crail/storage/rdma/client/RdmaLocalFuture.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage.rdma.client; 20 | 21 | import java.util.concurrent.ExecutionException; 22 | import java.util.concurrent.TimeUnit; 23 | import java.util.concurrent.TimeoutException; 24 | 25 | import org.apache.crail.storage.StorageFuture; 26 | import org.apache.crail.storage.StorageResult; 27 | 28 | import sun.misc.Unsafe; 29 | 30 | public class RdmaLocalFuture implements StorageFuture, StorageResult { 31 | private Unsafe unsafe; 32 | private long srcAddr; 33 | private long dstAddr; 34 | private int remaining; 35 | 36 | private int len; 37 | private boolean isDone; 38 | 39 | public RdmaLocalFuture(Unsafe unsafe, long srcAddr, long dstAddr, int remaining) { 40 | this.unsafe = unsafe; 41 | this.srcAddr = srcAddr; 42 | this.dstAddr = dstAddr; 43 | this.remaining = remaining; 44 | 45 | this.len = 0; 46 | this.isDone = false; 47 | } 48 | 49 | 50 | @Override 51 | public int getLen() { 52 | return len; 53 | } 54 | 55 | @Override 56 | public boolean cancel(boolean mayInterruptIfRunning) { 57 | return false; 58 | } 59 | 60 | @Override 61 | public boolean isCancelled() { 62 | return false; 63 | } 64 | 65 | @Override 66 | public boolean isDone() { 67 | if (!isDone){ 68 | getDone(); 69 | } 70 | return isDone; 71 | } 72 | 73 | @Override 74 | public StorageResult get() throws InterruptedException, ExecutionException { 75 | if (!isDone){ 76 | getDone(); 77 | } 78 | return this; 79 | } 80 | 81 | @Override 82 | public StorageResult get(long timeout, TimeUnit unit) 83 | throws InterruptedException, ExecutionException, TimeoutException { 84 | if (!isDone){ 85 | getDone(); 86 | } 87 | return this; 88 | } 89 | 90 | @Override 91 | public boolean isSynchronous() { 92 | return true; 93 | } 94 | 95 | void getDone(){ 96 | unsafe.copyMemory(srcAddr, dstAddr, remaining); 97 | len = remaining; 98 | isDone = true; 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /storage-rdma/src/main/java/org/apache/crail/storage/rdma/client/RdmaStorageActiveEndpointFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage.rdma.client; 20 | 21 | import java.io.IOException; 22 | 23 | import com.ibm.disni.verbs.*; 24 | import com.ibm.disni.*; 25 | 26 | public class RdmaStorageActiveEndpointFactory implements RdmaEndpointFactory { 27 | private RdmaStorageActiveGroup group; 28 | 29 | public RdmaStorageActiveEndpointFactory(RdmaStorageActiveGroup group){ 30 | this.group = group; 31 | } 32 | 33 | @Override 34 | public RdmaStorageActiveEndpoint createEndpoint(RdmaCmId id, boolean serverSide) throws IOException { 35 | return new RdmaStorageActiveEndpoint(group, id, serverSide); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /storage-rdma/src/main/java/org/apache/crail/storage/rdma/client/RdmaStoragePassiveEndpointFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage.rdma.client; 20 | 21 | import java.io.IOException; 22 | import com.ibm.disni.verbs.*; 23 | import com.ibm.disni.*; 24 | 25 | public class RdmaStoragePassiveEndpointFactory implements RdmaEndpointFactory { 26 | private RdmaStoragePassiveGroup group; 27 | 28 | public RdmaStoragePassiveEndpointFactory(RdmaStoragePassiveGroup group){ 29 | this.group = group; 30 | } 31 | 32 | @Override 33 | public RdmaStoragePassiveEndpoint createEndpoint(RdmaCmId id, boolean serverSide) throws IOException { 34 | return new RdmaStoragePassiveEndpoint(group, id, serverSide); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /storage/pom.xml: -------------------------------------------------------------------------------- 1 | 18 | 19 | 4.0.0 20 | 21 | 22 | org.apache.crail 23 | crail-parent 24 | 1.3-incubating-SNAPSHOT 25 | ../pom.xml 26 | 27 | 28 | org.apache.crail 29 | crail-storage 30 | jar 31 | 1.3-incubating-SNAPSHOT 32 | Crail Storage Project 33 | http://maven.apache.org 34 | 35 | 36 | 37 | junit 38 | junit 39 | 40 | 41 | log4j 42 | log4j 43 | 44 | 45 | commons-cli 46 | commons-cli 47 | 48 | 49 | org.apache.crail 50 | crail-client 51 | ${project.parent.version} 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | org.apache.maven.plugins 61 | maven-jar-plugin 62 | 2.6 63 | 64 | 65 | 66 | test-jar 67 | 68 | 69 | 70 | 71 | 72 | 73 | org.apache.maven.plugins 74 | maven-compiler-plugin 75 | 3.3 76 | 77 | 1.8 78 | 1.8 79 | 80 | 81 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /storage/src/main/java/org/apache/crail/storage/StorageResource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage; 20 | 21 | public class StorageResource { 22 | private long address; 23 | private int length; 24 | private int key; 25 | 26 | public static StorageResource createResource(long address, int length, int key){ 27 | return new StorageResource(address, length, key); 28 | } 29 | 30 | private StorageResource(long address, int length, int key){ 31 | this.address = address; 32 | this.length = length; 33 | this.key = key; 34 | } 35 | 36 | public long getAddress() { 37 | return address; 38 | } 39 | 40 | public int getLength() { 41 | return length; 42 | } 43 | 44 | public int getKey() { 45 | return key; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /storage/src/main/java/org/apache/crail/storage/StorageTier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage; 20 | 21 | import org.apache.crail.storage.StorageClient; 22 | 23 | public interface StorageTier extends StorageClient { 24 | 25 | public abstract StorageServer launchServer() throws Exception; 26 | 27 | @SuppressWarnings("unchecked") 28 | public static StorageTier createInstance(String name) throws Exception { 29 | Class nodeClass = Class.forName(name); 30 | if (StorageTier.class.isAssignableFrom(nodeClass)){ 31 | Class storageTier = (Class) nodeClass; 32 | StorageTier tier = storageTier.newInstance(); 33 | return tier; 34 | } else { 35 | throw new Exception("Cannot instantiate datanode of type " + name); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /storage/src/main/java/org/apache/crail/storage/StorageUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one or more 4 | * contributor license agreements. See the NOTICE file distributed with 5 | * this work for additional information regarding copyright ownership. 6 | * The ASF licenses this file to You under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.crail.storage; 20 | 21 | import java.io.File; 22 | import java.io.IOException; 23 | import java.net.InetAddress; 24 | import java.net.InetSocketAddress; 25 | import java.net.InterfaceAddress; 26 | import java.net.NetworkInterface; 27 | import java.util.List; 28 | 29 | public class StorageUtils { 30 | public static InetSocketAddress getDataNodeAddress(String ifname, int port) throws IOException { 31 | NetworkInterface netif = NetworkInterface.getByName(ifname); 32 | if (netif == null){ 33 | throw new IOException("Cannot find network interface with name " + ifname); 34 | } 35 | List addresses = netif.getInterfaceAddresses(); 36 | InetAddress addr = null; 37 | for (InterfaceAddress address: addresses){ 38 | if (address.getBroadcast() != null){ 39 | InetAddress _addr = address.getAddress(); 40 | addr = _addr; 41 | } 42 | } 43 | 44 | if (addr == null){ 45 | throw new IOException("Network interface with name " + ifname + " has no valid IP address"); 46 | } 47 | InetSocketAddress inetAddr = new InetSocketAddress(addr, port); 48 | return inetAddr; 49 | } 50 | 51 | public static void clean(String base, String path) throws IOException { 52 | try { 53 | File dataDir = new File(path); 54 | if (!dataDir.exists()){ 55 | if (!dataDir.mkdirs()) { 56 | throw new IOException("crail.datapath " + base + " either does not exist or has no write permissions"); 57 | } 58 | } 59 | for (File child : dataDir.listFiles()) { 60 | child.delete(); 61 | } 62 | } catch(SecurityException e) { 63 | throw new IOException("Error when trying to access " + base, e); 64 | } 65 | } 66 | 67 | public static String getDatanodeDirectory(String datapath, InetSocketAddress address) throws IllegalArgumentException { 68 | if (address == null) { 69 | throw new IllegalArgumentException("Address paramater cannot be null!"); 70 | } 71 | return datapath + address.getAddress() + "-" + address.getPort(); 72 | } 73 | } 74 | --------------------------------------------------------------------------------