14 |
15 |
16 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/proto/grfa.proto:
--------------------------------------------------------------------------------
1 | option java_package = "org.apache.giraffa";
2 | option java_outer_classname = "GiraffaProtos";
3 | option java_generate_equals_and_hash = true;
4 | package giraffa;
5 |
6 | import "hdfs.proto";
7 | import "Security.proto";
8 |
9 | message UnlocatedBlockProto {
10 | required hadoop.hdfs.ExtendedBlockProto b = 1;
11 | required uint64 offset = 2;
12 | required bool corrupt = 3;
13 | required hadoop.common.TokenProto blockToken = 4;
14 | }
15 |
16 | message RenameStateProto {
17 | required bool flag = 1;
18 | optional bytes src = 2;
19 | }
20 |
21 | message FileLeaseProto {
22 | required string holder = 1;
23 | required string path = 2;
24 | required uint64 lastUpdate = 3;
25 | }
26 |
--------------------------------------------------------------------------------
/giraffa-core/src/site/rules-pmd.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/giraffa-core/src/test/resources/webapps/secondary/WEB-INF/web.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
6 |
9 |
10 |
11 |
12 |
13 | org.apache.hadoop.hdfs.server.namenode.status_jsp
14 | org.apache.hadoop.hdfs.server.namenode.status_jsp
15 |
16 |
17 |
18 | org.apache.hadoop.hdfs.server.namenode.status_jsp
19 | /status.jsp
20 |
21 |
22 |
--------------------------------------------------------------------------------
/giraffa-core/src/header.txt:
--------------------------------------------------------------------------------
1 | Licensed to the Apache Software Foundation (ASF) under one
2 | or more contributor license agreements. See the NOTICE file
3 | distributed with this work for additional information
4 | regarding copyright ownership. The ASF licenses this file
5 | to you under the Apache License, Version 2.0 (the
6 | "License"); you may not use this file except in compliance
7 | with the License. You may obtain a copy of the License at
8 |
9 | http://www.apache.org/licenses/LICENSE-2.0
10 |
11 | Unless required by applicable law or agreed to in writing, software
12 | distributed under the License is distributed on an "AS IS" BASIS,
13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | See the License for the specific language governing permissions and
15 | limitations under the License.
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Licensed to the Apache Software Foundation (ASF) under one or more
2 | # contributor license agreements. See the NOTICE file distributed with
3 | # this work for additional information regarding copyright ownership.
4 | # The ASF licenses this file to You under the Apache License, Version 2.0
5 | # (the "License"); you may not use this file except in compliance with
6 | # the License. You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | .classpath
17 | .project
18 | .settings
19 | .externalToolBuilders/
20 | *.iml
21 | .idea/clover/*
22 | .idea
23 | build/
24 | .gradle
25 | .DS_STORE
26 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/resources/giraffa-env.sh:
--------------------------------------------------------------------------------
1 | # Licensed to the Apache Software Foundation (ASF) under one or more
2 | # contributor license agreements. See the NOTICE file distributed with
3 | # this work for additional information regarding copyright ownership.
4 | # The ASF licenses this file to You under the Apache License, Version 2.0
5 | # (the "License"); you may not use this file except in compliance with
6 | # the License. You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | #export HADOOP_HOME=~/hadoop/hadoop-2.5.1
17 | #export HBASE_HOME=~/hbase/hbase-1.0.1
18 | export GIRAFFA_ROOT_LOGGER=INFO,RFA
19 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/resources/mapred-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
16 |
17 |
18 |
19 | mapreduce.framework.name
20 | yarn
21 |
22 |
23 | mapreduce.terasort.simplepartitioner
24 | true
25 |
26 |
27 |
--------------------------------------------------------------------------------
/giraffa-core/src/test/java/org/apache/giraffa/RenameRecoveryState.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.giraffa;
19 |
20 | public enum RenameRecoveryState {
21 | PUT_SETFLAG, DELETE, PUT_NOFLAG
22 | }
23 |
--------------------------------------------------------------------------------
/settings.gradle:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | rootProject.name = 'giraffa'
19 |
20 | include 'giraffa-core', 'giraffa-standalone'
21 |
22 | project(":giraffa-core").name = 'giraffa-core'
23 | project(":giraffa-standalone").name = 'giraffa-standalone'
24 |
--------------------------------------------------------------------------------
/giraffa-standalone/src/main/resources/bin/yarn-giraffa:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | # Licensed to the Apache Software Foundation (ASF) under one or more
3 | # contributor license agreements. See the NOTICE file distributed with
4 | # this work for additional information regarding copyright ownership.
5 | # The ASF licenses this file to You under the Apache License, Version 2.0
6 | # (the "License"); you may not use this file except in compliance with
7 | # the License. You may obtain a copy of the License at
8 | #
9 | # http://www.apache.org/licenses/LICENSE-2.0
10 | #
11 | # Unless required by applicable law or agreed to in writing, software
12 | # distributed under the License is distributed on an "AS IS" BASIS,
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | # See the License for the specific language governing permissions and
15 | # limitations under the License.
16 |
17 | bin=`dirname "$0"`
18 | bin=`cd "$bin"; pwd`
19 |
20 | . "$bin"/giraffa-config.sh
21 |
22 | export HADOOP_CLASSPATH=$GIRAFFA_CLASSPATH
23 | $HADOOP_HOME/bin/yarn --config $GIRAFFA_CONF_DIR $*
24 |
--------------------------------------------------------------------------------
/giraffa-standalone/src/main/resources/bin/yarn-giraffa-daemon.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | # Licensed to the Apache Software Foundation (ASF) under one or more
3 | # contributor license agreements. See the NOTICE file distributed with
4 | # this work for additional information regarding copyright ownership.
5 | # The ASF licenses this file to You under the Apache License, Version 2.0
6 | # (the "License"); you may not use this file except in compliance with
7 | # the License. You may obtain a copy of the License at
8 | #
9 | # http://www.apache.org/licenses/LICENSE-2.0
10 | #
11 | # Unless required by applicable law or agreed to in writing, software
12 | # distributed under the License is distributed on an "AS IS" BASIS,
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | # See the License for the specific language governing permissions and
15 | # limitations under the License.
16 |
17 | bin=`dirname "$0"`
18 | bin=`cd "$bin"; pwd`
19 |
20 | . "$bin"/giraffa-config.sh
21 |
22 | export HADOOP_CLASSPATH=$GIRAFFA_CLASSPATH
23 | $HADOOP_HOME/sbin/yarn-daemon.sh --config $GIRAFFA_CONF_DIR $*
24 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/resources/core-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 | fs.defaultFS
9 | grfa:///
10 | The name of the default file system. A URI whose
11 | scheme and authority determine the FileSystem implementation. The
12 | uri's scheme determines the config property (fs.SCHEME.impl) naming
13 | the FileSystem implementation class. The uri's authority is used to
14 | determine the host, port, etc. for a filesystem.
15 |
16 |
17 |
18 | fs.grfa.impl
19 | org.apache.giraffa.GiraffaFileSystem
20 | The FileSystem for grfa.
21 |
22 |
23 | fs.AbstractFileSystem.grfa.impl
24 | org.apache.giraffa.GiraffaFs
25 | The AbstractFileSystem for grfa.
26 |
27 |
28 |
--------------------------------------------------------------------------------
/giraffa-core/src/test/resources/webapps/secondary/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
19 | Hadoop Administration
20 |
21 |
22 |
30 |
31 |
32 |
33 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/assembly/conf.xml:
--------------------------------------------------------------------------------
1 |
4 |
5 | conf
6 |
7 |
8 | zip
9 |
10 |
11 | false
12 |
13 |
14 |
15 | src/main/resources
16 |
17 |
18 | core-site.xml
19 | giraffa-default.xml
20 | giraffa-env.sh
21 | log4j.properties
22 | capacity-scheduler.xml
23 | configuration.xsl
24 | container-executor.cfg
25 | mapred-env.sh
26 | mapred-site.xml
27 | yarn-env.sh
28 | yarn-site.xml
29 |
30 |
31 |
32 |
33 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/giraffa/FullPathRowKeyFactory.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.giraffa;
19 |
20 | import java.io.IOException;
21 |
22 | public class FullPathRowKeyFactory extends RowKeyFactory {
23 |
24 | @Override // RowKeyFactory
25 | public RowKey getRowKey(String src, long inodeId) throws IOException {
26 | return new FullPathRowKey(src);
27 | }
28 |
29 | @Override // RowKeyFactory
30 | public RowKey getRowKey(String src, byte[] bytes) throws IOException {
31 | return new FullPathRowKey(src, bytes);
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/giraffa-core/src/test/resources/data1k:
--------------------------------------------------------------------------------
1 | 12345678901234
2 | 12345678901234
3 | 12345678901234
4 | 12345678901234
5 | 12345678901234
6 | 12345678901234
7 | 12345678901234
8 | 12345678901234
9 | 12345678901234
10 | 12345678901234
11 | 12345678901234
12 | 12345678901234
13 | 12345678901234
14 | 12345678901234
15 | 12345678901234
16 | 12345678901234
17 | 12345678901234
18 | 12345678901234
19 | 12345678901234
20 | 12345678901234
21 | 12345678901234
22 | 12345678901234
23 | 12345678901234
24 | 12345678901234
25 | 12345678901234
26 | 12345678901234
27 | 12345678901234
28 | 12345678901234
29 | 12345678901234
30 | 12345678901234
31 | 12345678901234
32 | 12345678901234
33 | 12345678901234
34 | 12345678901234
35 | 12345678901234
36 | 12345678901234
37 | 12345678901234
38 | 12345678901234
39 | 12345678901234
40 | 12345678901234
41 | 12345678901234
42 | 12345678901234
43 | 12345678901234
44 | 12345678901234
45 | 12345678901234
46 | 12345678901234
47 | 12345678901234
48 | 12345678901234
49 | 12345678901234
50 | 12345678901234
51 | 12345678901234
52 | 12345678901234
53 | 12345678901234
54 | 12345678901234
55 | 12345678901234
56 | 12345678901234
57 | 12345678901234
58 | 12345678901234
59 | 12345678901234
60 | 12345678901234
61 | 12345678901234
62 | 12345678901234
63 | 12345678901234
64 | 12345678901234
65 | 12345678901234
66 | 12345678901234
67 | 12345678901234
68 | 12345678901234
69 | 12345678901234
70 | 12345678901234
71 | 12345678901234
72 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/resources/mapred-env.sh:
--------------------------------------------------------------------------------
1 | # Licensed to the Apache Software Foundation (ASF) under one or more
2 | # contributor license agreements. See the NOTICE file distributed with
3 | # this work for additional information regarding copyright ownership.
4 | # The ASF licenses this file to You under the Apache License, Version 2.0
5 | # (the "License"); you may not use this file except in compliance with
6 | # the License. You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | # export JAVA_HOME=/home/y/libexec/jdk1.6.0/
17 |
18 | export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000
19 |
20 | export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA
21 |
22 | #export HADOOP_JOB_HISTORYSERVER_OPTS=
23 | #export HADOOP_MAPRED_LOG_DIR="" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default.
24 | #export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger.
25 | #export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default.
26 | #export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default
27 | #export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0.
28 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/hadoop/hdfs/HDFSAdapter.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.hadoop.hdfs;
19 |
20 | import org.apache.hadoop.hdfs.protocol.ClientProtocol;
21 |
22 | /**
23 | * Adapter to HDFS client side APIs.
24 | */
25 | public class HDFSAdapter {
26 | private static DFSClient getClient(DistributedFileSystem hdfs) {
27 | return hdfs.getClient();
28 | }
29 |
30 | public static String getClientName(DistributedFileSystem hdfs) {
31 | return getClient(hdfs).getClientName();
32 | }
33 |
34 | public static ClientProtocol getClientProtocol(DistributedFileSystem hdfs) {
35 | return getClient(hdfs).getNamenode();
36 | }
37 | }
38 |
39 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/resources/configuration.xsl:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
name
26 |
value
27 |
description
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/resources/hbase-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
19 |
20 |
21 |
22 |
23 |
24 | hbase.coprocessor.region.classes
25 | org.apache.giraffa.hbase.NamespaceProcessor
26 |
27 |
28 |
29 | hbase.dynamic.jars.dir
30 | ${hbase.local.dir}/jars/
31 |
32 |
33 |
34 | hbase.rpc.callerfactory.class
35 | org.apache.hadoop.hbase.client.GiraffaRpcRetryingCallerFactory
36 |
37 |
38 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/giraffa/id/IdGeneratorService.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.giraffa.id;
19 |
20 | import org.apache.hadoop.util.IdGenerator;
21 |
22 | /**
23 | * Id generator that connects to a service to store and compute values.
24 | * Implementations are expected to be thread-safe and increment atomically.
25 | */
26 | public interface IdGeneratorService extends IdGenerator {
27 |
28 | /**
29 | * Return the id stored by this generator before any increments.
30 | */
31 | long getInitialValue();
32 |
33 | /**
34 | * Initialize the connection to the service.
35 | */
36 | void initialize();
37 |
38 | /**
39 | * Close the connection to the service.
40 | */
41 | void close();
42 | }
43 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRpcUtil.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.hadoop.hbase.ipc;
19 |
20 | import org.apache.hadoop.security.UserGroupInformation;
21 |
22 | import java.io.IOException;
23 |
24 | public class HBaseRpcUtil {
25 |
26 | /** Returns the RPC remote user when invoked inside an RPC. Note this
27 | * may be different than the current user if called within another doAs
28 | * @return connection's UGI or current user if not an RPC
29 | */
30 | public static UserGroupInformation getRemoteUser() throws IOException {
31 | RpcServer.Call call = RpcServer.CurCall.get();
32 | if (call != null && call.connection != null) {
33 | return call.connection.user;
34 | } else {
35 | return UserGroupInformation.getCurrentUser();
36 | }
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/giraffa-core/src/test/resources/webapps/datanode/WEB-INF/web.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
6 |
9 |
10 |
11 |
12 |
13 | org.apache.hadoop.hdfs.server.datanode.browseBlock_jsp
14 | org.apache.hadoop.hdfs.server.datanode.browseBlock_jsp
15 |
16 |
17 |
18 | org.apache.hadoop.hdfs.server.datanode.tail_jsp
19 | org.apache.hadoop.hdfs.server.datanode.tail_jsp
20 |
21 |
22 |
23 | org.apache.hadoop.hdfs.server.datanode.browseDirectory_jsp
24 | org.apache.hadoop.hdfs.server.datanode.browseDirectory_jsp
25 |
26 |
27 |
28 | org.apache.hadoop.hdfs.server.datanode.browseBlock_jsp
29 | /browseBlock.jsp
30 |
31 |
32 |
33 | org.apache.hadoop.hdfs.server.datanode.tail_jsp
34 | /tail.jsp
35 |
36 |
37 |
38 | org.apache.hadoop.hdfs.server.datanode.browseDirectory_jsp
39 | /browseDirectory.jsp
40 |
41 |
42 |
43 |
--------------------------------------------------------------------------------
/giraffa-standalone/src/main/resources/bin/stop-giraffa.sh:
--------------------------------------------------------------------------------
1 | # Licensed to the Apache Software Foundation (ASF) under one or more
2 | # contributor license agreements. See the NOTICE file distributed with
3 | # this work for additional information regarding copyright ownership.
4 | # The ASF licenses this file to You under the Apache License, Version 2.0
5 | # (the "License"); you may not use this file except in compliance with
6 | # the License. You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 |
17 | # Stop Giraffa fs daemons.
18 |
19 | # TODO
20 | # usage="Usage: stop-grfa.sh [--config confdir] [--hadoop hadoopdir] [--hbase hbasedir]"
21 |
22 | usage="Usage: stop-grfa.sh"
23 |
24 | bin=`dirname "${BASH_SOURCE-$0}"`
25 | bin=`cd "$bin"; pwd`
26 |
27 | . "$bin"/giraffa-config.sh
28 |
29 | # get arguments
30 |
31 | # stop hdfs
32 | # "$HADOOP_HOME"/sbin/stop-dfs.sh
33 |
34 | echo "HADOOP_HOME = " $HADOOP_HOME
35 |
36 | cd "$HADOOP_HOME"
37 | . "$HADOOP_HOME"/libexec/hdfs-config.sh
38 |
39 | echo "stopping namenode"
40 | "$HADOOP_HOME"/sbin/hadoop-daemon.sh --script "$bin"/hdfs stop namenode
41 |
42 | echo "stopping datanode"
43 | "$HADOOP_HOME"/sbin/hadoop-daemon.sh --script "$bin"/hdfs stop datanode
44 |
45 | # start hbase
46 | echo "stopping hbase"
47 | HADOOP_HOME=
48 | "$HBASE_HOME"/bin/stop-hbase.sh
49 |
--------------------------------------------------------------------------------
/giraffa-core/src/site/site.xml:
--------------------------------------------------------------------------------
1 |
5 |
6 |
7 | Giraffa - A distributed highly available file system
8 |
9 |
10 |
11 | https://ssl.gstatic.com/codesite/ph/images/apacheextras.png
12 |
13 |
14 |
15 |
16 |
17 |
18 | org.apache.maven.skins
19 | maven-fluido-skin
20 | ${maven-fluido-skin.version}
21 |
22 |
23 |
24 |
25 | true
26 | true
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
--------------------------------------------------------------------------------
/giraffa-standalone/src/main/assembly/standalone.xml:
--------------------------------------------------------------------------------
1 |
4 |
5 | standalone
6 |
7 |
8 | dir
9 |
10 |
11 | false
12 |
13 |
14 |
15 | src/main/resources/bin
16 | giraffa/bin
17 | true
18 |
19 | *
20 |
21 | 0775
22 |
23 |
24 | src/main/assembly
25 | giraffa/logs
26 |
27 | *
28 |
29 |
30 |
31 |
32 |
33 |
34 | giraffa/lib
35 |
36 | org.apache.hadoop:hadoop-hdfs
37 | org.apache.hadoop:hadoop-common
38 | org.apache.hadoop:hadoop-auth
39 | org.slf4j:slf4j-api
40 | org.apache.hbase:hbase
41 |
42 | true
43 | true
44 |
45 |
46 |
47 |
48 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/giraffa/GiraffaFs.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.giraffa;
19 |
20 | import org.apache.hadoop.conf.Configuration;
21 | import org.apache.hadoop.fs.DelegateToFileSystem;
22 |
23 | import java.io.IOException;
24 | import java.net.URI;
25 | import java.net.URISyntaxException;
26 |
27 | /**
28 | * AbstractFileSystem implementation of Giraffa that delegates all calls to
29 | * GiraffaFileSystem. This makes Giraffa accessible using the FileContext API
30 | * and applications that use that API, such as Yarn. To enable, set the
31 | * fs.AbstractFileSystem.grfa.impl property to this class in core-site.xml.
32 | */
33 | public class GiraffaFs extends DelegateToFileSystem {
34 | public GiraffaFs(final URI theUri, final Configuration conf)
35 | throws IOException, URISyntaxException {
36 | super(theUri, new GiraffaFileSystem(), conf,
37 | GiraffaConfiguration.GRFA_URI_SCHEME, false);
38 | }
39 |
40 | @Override
41 | public int getUriDefaultPort() { // AbstractFileSystem
42 | return -1; // no default port for grfa:///
43 | }
44 | }
45 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/giraffa/NamespaceService.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.giraffa;
19 |
20 | import java.io.Closeable;
21 | import java.io.IOException;
22 |
23 | import org.apache.hadoop.hdfs.protocol.ClientProtocol;
24 |
25 | /**
26 | * {@link NamespaceService} is a common interface that provides access
27 | * to a reliable storage system, like HBase, which maintains
28 | * the Giraffa file system metadata.
29 | *
30 | * It plays a role of a proxy used by DFSClient to communicate with
31 | * the underlying storage system as if it is a NameNode.
32 | * It implements {@link ClientProtocol} and is a replacement of the
33 | * NameNode RPC proxy.
34 | *
35 | * Implement this interface to make Giraffa client connect to different
36 | * highly available storage systems.
37 | *
38 | * {@link org.apache.giraffa.hbase.NamespaceAgent} is the default implementation of
39 | * {@link NamespaceService} for HBase.
40 | */
41 | public interface NamespaceService extends ClientProtocol, Closeable {
42 |
43 | public void initialize(GiraffaConfiguration conf) throws IOException;
44 |
45 | public void format(GiraffaConfiguration conf) throws IOException;
46 | }
47 |
--------------------------------------------------------------------------------
/giraffa-standalone/src/main/resources/bin/start-giraffa.sh:
--------------------------------------------------------------------------------
1 | # Licensed to the Apache Software Foundation (ASF) under one or more
2 | # contributor license agreements. See the NOTICE file distributed with
3 | # this work for additional information regarding copyright ownership.
4 | # The ASF licenses this file to You under the Apache License, Version 2.0
5 | # (the "License"); you may not use this file except in compliance with
6 | # the License. You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 |
17 | # Start Giraffa fs daemons.
18 |
19 | # TODO
20 | # usage="Usage: start-grfa.sh [--config confdir] [--hadoop hadoopdir] [--hbase hbasedir] [-upgrade|-rollback]"
21 |
22 | usage="Usage: start-grfa.sh [-upgrade|-rollback]"
23 |
24 | bin=`dirname "${BASH_SOURCE-$0}"`
25 | bin=`cd "$bin"; pwd`
26 | cd $bin/..
27 | pwd
28 |
29 | . "$bin"/giraffa-config.sh
30 |
31 | # get arguments
32 | if [ $# -ge 1 ]; then
33 | nameStartOpt=$1
34 | shift
35 | case $nameStartOpt in
36 | (-upgrade)
37 | ;;
38 | (-rollback)
39 | dataStartOpt=$nameStartOpt
40 | ;;
41 | (*)
42 | echo $usage
43 | exit 1
44 | ;;
45 | esac
46 | fi
47 |
48 | # start hdfs
49 | # "$HADOOP_HOME"/sbin/start-dfs.sh $nameStartOpt
50 |
51 | echo "HADOOP_HOME = " $HADOOP_HOME
52 |
53 | cd "$HADOOP_HOME"
54 | . "$HADOOP_HOME"/libexec/hdfs-config.sh
55 |
56 | echo "starting namenode"
57 | "$HADOOP_HOME"/sbin/hadoop-daemon.sh start namenode $nameStartOpt
58 |
59 | echo "starting datanode"
60 | "$HADOOP_HOME"/sbin/hadoop-daemon.sh start datanode $dataStartOpt
61 |
62 | # start hbase
63 | echo "HBASE_HOME = " $HBASE_HOME
64 |
65 | cd "$HBASE_HOME"
66 | echo "starting hbase"
67 | HADOOP_HOME=
68 | . "$HBASE_HOME"/bin/start-hbase.sh
69 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/proto/Security.proto:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | /**
20 | * These .proto interfaces are private and stable.
21 | * Please see http://wiki.apache.org/hadoop/Compatibility
22 | * for what changes are allowed for a *stable* .proto interface.
23 | */
24 |
25 | option java_package = "org.apache.hadoop.security.proto";
26 | option java_outer_classname = "SecurityProtos";
27 | option java_generic_services = true;
28 | option java_generate_equals_and_hash = true;
29 | package hadoop.common;
30 |
31 | /**
32 | * Security token identifier
33 | */
34 | message TokenProto {
35 | required bytes identifier = 1;
36 | required bytes password = 2;
37 | required string kind = 3;
38 | required string service = 4;
39 | }
40 |
41 | message GetDelegationTokenRequestProto {
42 | required string renewer = 1;
43 | }
44 |
45 | message GetDelegationTokenResponseProto {
46 | optional hadoop.common.TokenProto token = 1;
47 | }
48 |
49 | message RenewDelegationTokenRequestProto {
50 | required hadoop.common.TokenProto token = 1;
51 | }
52 |
53 | message RenewDelegationTokenResponseProto {
54 | required uint64 newExpiryTime = 1;
55 | }
56 |
57 | message CancelDelegationTokenRequestProto {
58 | required hadoop.common.TokenProto token = 1;
59 | }
60 |
61 | message CancelDelegationTokenResponseProto { // void response
62 | }
63 |
64 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/giraffa/id/SegmentedIdGenerator.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.giraffa.id;
19 |
20 | import org.apache.hadoop.util.IdGenerator;
21 |
22 | /**
23 | * ID generator that allocates large segments of sequential ids at a time from a
24 | * given {@link IdGeneratorService} to minimize calls to the service. This class
25 | * is thread-safe as long as the underlying generator is thread-safe.
26 | */
27 | public class SegmentedIdGenerator implements IdGenerator {
28 |
29 | private static final long SEGMENT_SIZE = 1000;
30 |
31 | private final long initialValue;
32 | private final IdGeneratorService service;
33 |
34 | private long offset;
35 | private long value;
36 |
37 | public SegmentedIdGenerator(long initialValue,
38 | IdGeneratorService service) {
39 | this.initialValue = initialValue;
40 | this.service = service;
41 | }
42 |
43 | @Override // IdGenerator
44 | public synchronized long nextValue() {
45 | if (offset == 0) {
46 | newSegment();
47 | }
48 | if (++offset == SEGMENT_SIZE){
49 | offset = 0;
50 | }
51 | return ++value;
52 | }
53 |
54 | private synchronized void newSegment() {
55 | long segmentOffset = service.nextValue() - service.getInitialValue() - 1;
56 | value = segmentOffset * SEGMENT_SIZE + initialValue;
57 | }
58 | }
59 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/resources/giraffa-default.xml:
--------------------------------------------------------------------------------
1 |
2 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 | grfa.table.name
33 | Namespace
34 |
35 | The name of the HBase table that stores the GiraffaFS namespace.
36 |
37 |
38 |
39 |
40 | grfa.rowkey.factory.class
41 | org.apache.giraffa.FullPathRowKeyFactory
42 |
43 | Class that implements RowKeyFactory for the namespace table.
44 |
45 |
46 |
47 |
48 | grfa.rowkey.caching
49 | true
50 |
51 | True if row key caching is enabled.
52 |
53 |
54 |
55 |
56 |
57 |
58 | grfa.http-address
59 | 0.0.0.0:40010
60 |
61 | Giraffa UI http server address and port.
62 |
63 |
64 |
65 |
66 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/giraffa/web/GiraffaWebServer.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.giraffa.web;
19 |
20 | import org.apache.hadoop.conf.Configuration;
21 | import org.apache.hadoop.http.HttpServer2;
22 | import org.apache.hadoop.net.NetUtils;
23 |
24 | import javax.servlet.http.HttpServlet;
25 | import java.io.IOException;
26 | import java.net.InetSocketAddress;
27 | import java.net.URI;
28 |
29 | class GiraffaWebServer {
30 |
31 | private final HttpServer2 httpServer;
32 |
33 | GiraffaWebServer(String giraffa,
34 | InetSocketAddress infoSocAddr,
35 | Configuration conf)
36 | throws IOException {
37 | URI uri = URI.create("http://" + NetUtils.getHostPortString(infoSocAddr));
38 | httpServer = new HttpServer2.Builder().setName(giraffa)
39 | .hostName(infoSocAddr.getHostName()).setConf(conf)
40 | .addEndpoint(uri).build();
41 | }
42 |
43 | void addServlet(String name,
44 | String pathSpec,
45 | Class clazz) {
46 | httpServer.addServlet(name, pathSpec, clazz);
47 | }
48 |
49 | void setAttribute(String name,
50 | Object value) {
51 | httpServer.setAttribute(name, value);
52 | }
53 |
54 | void start() throws IOException {
55 | httpServer.start();
56 | }
57 |
58 | boolean isAlive() {
59 | return httpServer.isAlive();
60 | }
61 |
62 | void stop() throws Exception {
63 | httpServer.stop();
64 | }
65 | }
66 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/hadoop/hbase/client/GiraffaRpcRetryingCallerFactory.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.hadoop.hbase.client;
19 |
20 | import java.io.IOException;
21 |
22 | import org.apache.commons.logging.Log;
23 | import org.apache.commons.logging.LogFactory;
24 | import org.apache.hadoop.conf.Configuration;
25 | import org.apache.hadoop.hbase.NotServingRegionException;
26 | import org.apache.hadoop.hbase.exceptions.ConnectionClosingException;
27 |
28 | /**
29 | * Hook for loading our own FileSystemExceptionInterceptor to block filesystem
30 | * operations from being retried and properly handle exceptions. The name of
31 | * this class is specified in the property "hbase.rpc.callerfactory.class"
32 | */
33 | public class GiraffaRpcRetryingCallerFactory extends RpcRetryingCallerFactory {
34 | public static final Log LOG =
35 | LogFactory.getLog(GiraffaRpcRetryingCallerFactory.class);
36 |
37 | public GiraffaRpcRetryingCallerFactory(Configuration conf) {
38 | super(conf, new FileSystemExceptionInterceptor());
39 | }
40 |
41 | public static class FileSystemExceptionInterceptor
42 | extends NoOpRetryableCallerInterceptor {
43 |
44 | @Override // RetryingCallerInterceptor
45 | public void handleFailure(RetryingCallerInterceptorContext context,
46 | Throwable t)
47 | throws IOException {
48 | // detect and throw any filesystem-related IOExceptions
49 | if(t instanceof IOException && !(t instanceof NotServingRegionException
50 | || t instanceof ConnectionClosingException)) {
51 | LOG.debug("Got exception: ", t);
52 | throw (IOException) t;
53 | }
54 | }
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/libraries/libraries.gradle:
--------------------------------------------------------------------------------
1 | ext {
2 | libraries = [:]
3 | }
4 |
5 | libraries.hadoop = [
6 | "org.apache.hadoop:hadoop-common:${hadoop_version}",
7 | "org.apache.hadoop:hadoop-hdfs:${hadoop_version}",
8 | "org.apache.hadoop:hadoop-minicluster:${hadoop_version}",
9 | "org.apache.hadoop:hadoop-client:${hadoop_version}",
10 | "org.apache.hadoop:hadoop-auth:${hadoop_version}",
11 | "org.apache.hadoop:hadoop-mapreduce-client-core:${hadoop_version}",
12 | "org.apache.hadoop:hadoop-annotations:${hadoop_version}"
13 | ]
14 |
15 | libraries.hbase = [
16 | "org.apache.hbase:hbase-common:${hbase_version}",
17 | "org.apache.hbase:hbase-client:${hbase_version}",
18 | "org.apache.hbase:hbase-server:${hbase_version}",
19 | "org.apache.hbase:hbase-protocol:${hbase_version}",
20 | "org.apache.hbase:hbase-hadoop-compat:${hbase_version}",
21 | "org.apache.hbase:hbase-hadoop2-compat:${hbase_version}",
22 | "org.apache.hbase:hbase-it:${hbase_version}",
23 | ]
24 |
25 | libraries.hbase_tests = [
26 | "org.apache.hbase:hbase-common:${hbase_version}:tests",
27 | "org.apache.hbase:hbase-client:${hbase_version}:tests",
28 | "org.apache.hbase:hbase-server:${hbase_version}:tests",
29 | "org.apache.hbase:hbase-protocol:${hbase_version}:tests",
30 | "org.apache.hbase:hbase-hadoop-compat:${hbase_version}:tests",
31 | "org.apache.hbase:hbase-hadoop2-compat:${hbase_version}:tests",
32 | "org.apache.hbase:hbase-it:${hbase_version}:tests",
33 | ]
34 |
35 | libraries.curator = [
36 | "org.apache.curator:curator-framework:${curator_version}",
37 | "org.apache.curator:curator-recipes:${curator_version}"
38 | ]
39 |
40 | libraries.guava = [
41 | "com.google.guava:guava:${guava_version}"
42 | ]
43 |
44 | libraries.webui = [
45 | "commons-fileupload:commons-fileupload:${commons_fileupload_version}",
46 | "org.webjars:bootstrap:${twitter_bootstrap_version}",
47 | "org.webjars:jasny-bootstrap:${bootstrap_jasny_ext_version}",
48 | "org.webjars:jquery:${jquery_version}",
49 | "org.webjars:jquery-form:${jquery_form_version}",
50 | "org.webjars:datatables:${datatables_version}",
51 | "org.webjars:webjars-servlet-2.x:${webjars_servlet_version}"
52 | ]
53 |
54 | libraries.jasper = [
55 | "org.apache.tomcat:jasper:6.0.45",
56 | "javax.servlet:jstl:1.1.2",
57 | "taglibs:standard:1.1.2"
58 | ]
59 |
60 | libraries.test = [
61 | "junit:junit:${junit_version}",
62 | "org.jmockit:jmockit:${jmockit_version}",
63 | "org.hamcrest:hamcrest-all:${hamcrest_version}"
64 | ]
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/giraffa/RowKeyFactoryProvider.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.giraffa;
19 |
20 | import static org.apache.giraffa.GiraffaConfiguration.GRFA_CACHING_DEFAULT;
21 | import static org.apache.giraffa.GiraffaConfiguration.GRFA_CACHING_KEY;
22 | import static org.apache.giraffa.GiraffaConfiguration.GRFA_ROWKEY_FACTORY_DEFAULT;
23 | import static org.apache.giraffa.GiraffaConfiguration.GRFA_ROWKEY_FACTORY_KEY;
24 |
25 | import java.io.IOException;
26 |
27 | import org.apache.hadoop.conf.Configuration;
28 | import org.apache.hadoop.util.ReflectionUtils;
29 |
30 | public class RowKeyFactoryProvider {
31 |
32 | private static Class rowKeyFactoryClass;
33 |
34 | public static RowKeyFactory createFactory(Configuration conf)
35 | throws IOException {
36 | boolean caching = conf.getBoolean(GRFA_CACHING_KEY, GRFA_CACHING_DEFAULT);
37 | RowKeyFactory.setCache(caching);
38 | Class rkfClass = registerFactory(conf);
39 | RowKeyFactory rkf = ReflectionUtils.newInstance(rkfClass, conf);
40 | rkf.initialize(conf);
41 | rowKeyFactoryClass = rkfClass;
42 | return rkf;
43 | }
44 |
45 | @SuppressWarnings("unchecked")
46 | private static synchronized Class registerFactory(Configuration conf)
47 | throws IOException {
48 | Class factory;
49 | try {
50 | if(rowKeyFactoryClass != null)
51 | return rowKeyFactoryClass;
52 | factory = (Class) conf.getClass(
53 | GRFA_ROWKEY_FACTORY_KEY, GRFA_ROWKEY_FACTORY_DEFAULT);
54 | } catch(Exception e) {
55 | throw new IOException("Error retrieving RowKeyFactory class", e);
56 | }
57 | return factory;
58 | }
59 | }
60 |
--------------------------------------------------------------------------------
/giraffa-core/src/test/java/org/apache/giraffa/TestDirectoryTable.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.giraffa;
19 |
20 | import static org.junit.Assert.assertEquals;
21 | import static org.junit.Assert.assertFalse;
22 | import static org.junit.Assert.assertTrue;
23 |
24 | import java.io.IOException;
25 |
26 | import org.junit.Test;
27 |
28 | public class TestDirectoryTable {
29 |
30 | @Test
31 | public void testDirectory() throws IOException, ClassNotFoundException {
32 | DirectoryTable dirTable = new DirectoryTable();
33 | assertTrue(dirTable.isEmpty());
34 |
35 | assertTrue(dirTable.addEntry(new FullPathRowKey("/abc")));
36 | assertTrue(dirTable.addEntry(new FullPathRowKey("/def")));
37 | assertTrue(dirTable.addEntry(new FullPathRowKey("/ghi")));
38 | byte[] out = dirTable.toBytes();
39 |
40 | //dirTable serialized
41 |
42 | dirTable = new DirectoryTable(out);
43 | assertEquals(dirTable.getEntries().size(), 3);
44 | assertTrue(dirTable.contains("abc"));
45 | assertTrue(dirTable.contains("def"));
46 | assertTrue(dirTable.contains("ghi"));
47 | assertEquals("/abc", dirTable.getEntry("abc").getPath().toString());
48 | assertEquals("/def", dirTable.getEntry("def").getPath().toString());
49 | assertEquals("/ghi", dirTable.getEntry("ghi").getPath().toString());
50 |
51 | assertFalse(dirTable.addEntry(new FullPathRowKey("/abc")));
52 |
53 | //remove entries
54 | assertTrue(dirTable.removeEntry("abc"));
55 | assertEquals(dirTable.getEntries().size(), 2);
56 |
57 | assertTrue(dirTable.removeEntry("def"));
58 | assertEquals(dirTable.getEntries().size(), 1);
59 |
60 | assertTrue(dirTable.removeEntry("ghi"));
61 | assertEquals(dirTable.getEntries().size(), 0);
62 |
63 | assertTrue(dirTable.isEmpty());
64 | }
65 | }
66 |
--------------------------------------------------------------------------------
/gradlew.bat:
--------------------------------------------------------------------------------
1 | @if "%DEBUG%" == "" @echo off
2 | @rem ##########################################################################
3 | @rem
4 | @rem Gradle startup script for Windows
5 | @rem
6 | @rem ##########################################################################
7 |
8 | @rem Set local scope for the variables with windows NT shell
9 | if "%OS%"=="Windows_NT" setlocal
10 |
11 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
12 | set DEFAULT_JVM_OPTS=
13 |
14 | set DIRNAME=%~dp0
15 | if "%DIRNAME%" == "" set DIRNAME=.
16 | set APP_BASE_NAME=%~n0
17 | set APP_HOME=%DIRNAME%
18 |
19 | @rem Find java.exe
20 | if defined JAVA_HOME goto findJavaFromJavaHome
21 |
22 | set JAVA_EXE=java.exe
23 | %JAVA_EXE% -version >NUL 2>&1
24 | if "%ERRORLEVEL%" == "0" goto init
25 |
26 | echo.
27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
28 | echo.
29 | echo Please set the JAVA_HOME variable in your environment to match the
30 | echo location of your Java installation.
31 |
32 | goto fail
33 |
34 | :findJavaFromJavaHome
35 | set JAVA_HOME=%JAVA_HOME:"=%
36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe
37 |
38 | if exist "%JAVA_EXE%" goto init
39 |
40 | echo.
41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
42 | echo.
43 | echo Please set the JAVA_HOME variable in your environment to match the
44 | echo location of your Java installation.
45 |
46 | goto fail
47 |
48 | :init
49 | @rem Get command-line arguments, handling Windowz variants
50 |
51 | if not "%OS%" == "Windows_NT" goto win9xME_args
52 | if "%@eval[2+2]" == "4" goto 4NT_args
53 |
54 | :win9xME_args
55 | @rem Slurp the command line arguments.
56 | set CMD_LINE_ARGS=
57 | set _SKIP=2
58 |
59 | :win9xME_args_slurp
60 | if "x%~1" == "x" goto execute
61 |
62 | set CMD_LINE_ARGS=%*
63 | goto execute
64 |
65 | :4NT_args
66 | @rem Get arguments from the 4NT Shell from JP Software
67 | set CMD_LINE_ARGS=%$
68 |
69 | :execute
70 | @rem Setup the command line
71 |
72 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
73 |
74 | @rem Execute Gradle
75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
76 |
77 | :end
78 | @rem End local scope for the variables with windows NT shell
79 | if "%ERRORLEVEL%"=="0" goto mainEnd
80 |
81 | :fail
82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
83 | rem the _cmd.exe /c_ return code!
84 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
85 | exit /b 1
86 |
87 | :mainEnd
88 | if "%OS%"=="Windows_NT" endlocal
89 |
90 | :omega
91 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/giraffa/FileLease.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.giraffa;
19 |
20 | import static org.apache.hadoop.util.Time.now;
21 |
22 | /**
23 | * Leases provide exclusive access to files for write.
24 | */
25 | public class FileLease implements Comparable {
26 |
27 | public final String holder;
28 | public final String path;
29 | public final long lastUpdate;
30 |
31 | /**
32 | * Constructor for rebuilding from FileLeaseProto.
33 | */
34 | public FileLease(String holder, String path, long lastUpdate) {
35 | this.holder = holder;
36 | this.path = path;
37 | this.lastUpdate = lastUpdate;
38 | }
39 |
40 | public String getHolder() {
41 | return holder;
42 | }
43 |
44 | public String getPath() {
45 | return path;
46 | }
47 |
48 | public long getLastUpdate() {
49 | return lastUpdate;
50 | }
51 |
52 | public boolean expiredHardLimit(long hardLimit) {
53 | return now() - lastUpdate > hardLimit;
54 | }
55 |
56 | @Override
57 | public String toString() {
58 | return "FileLease[holder=" + holder + ", path=" + path + ", lastUpdate=" +
59 | lastUpdate + "]";
60 | }
61 |
62 | @Override
63 | public boolean equals(Object o) {
64 | if(!(o instanceof FileLease))
65 | return false;
66 | FileLease other = (FileLease) o;
67 | if(!holder.equals(other.holder))
68 | return false;
69 | if(!path.equals(other.path))
70 | return false;
71 | if(lastUpdate != other.lastUpdate)
72 | return false;
73 | return true;
74 | }
75 |
76 | @Override
77 | public int compareTo(FileLease o) {
78 | FileLease l1 = this;
79 | long lu1 = l1.lastUpdate;
80 | long lu2 = o.lastUpdate;
81 | if (lu1 < lu2) {
82 | return -1;
83 | } else if (lu1 > lu2) {
84 | return 1;
85 | } else {
86 | return l1.holder.compareTo(o.holder);
87 | }
88 | }
89 | }
90 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/giraffa/XAttrPermissionFilter.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.giraffa;
19 |
20 | import org.apache.hadoop.fs.XAttr;
21 | import org.apache.hadoop.hdfs.XAttrHelper;
22 | import org.apache.hadoop.security.AccessControlException;
23 |
24 | import java.util.ArrayList;
25 | import java.util.List;
26 |
27 | /**
28 | * Copied and refined from
29 | * ${@link org.apache.hadoop.hdfs.server.namenode.XAttrPermissionFilter}
30 | * Since that one is private
31 | *
32 | * The logic is the same as its original, only coding style is refined
33 | */
34 | public class XAttrPermissionFilter {
35 | static void checkPermissionForApi(FSPermissionChecker pc, XAttr xAttr)
36 | throws AccessControlException {
37 | assert (pc != null && xAttr != null) : "Argument is null";
38 | if (xAttr.getNameSpace() != XAttr.NameSpace.USER &&
39 | (xAttr.getNameSpace() != XAttr.NameSpace.TRUSTED || !pc.isSuperUser())) {
40 | throw new AccessControlException("User doesn\'t have permission"
41 | + " for xAttr: " + XAttrHelper.getPrefixName(xAttr));
42 | }
43 | }
44 |
45 | static void checkPermissionForApi(FSPermissionChecker pc, List xAttrs)
46 | throws AccessControlException {
47 | assert (xAttrs != null) : "Argument is null";
48 | for (XAttr xAttr : xAttrs) {
49 | checkPermissionForApi(pc, xAttr);
50 | }
51 | }
52 |
53 | static List filterXAttrsForApi(FSPermissionChecker pc,
54 | List xAttrs) {
55 | assert (xAttrs != null) : "Argument is null";
56 | List filteredXAttrs = new ArrayList();
57 | for (XAttr xAttr : xAttrs) {
58 | if (xAttr.getNameSpace() == XAttr.NameSpace.USER ||
59 | (xAttr.getNameSpace() == XAttr.NameSpace.TRUSTED && pc.isSuperUser())) {
60 | filteredXAttrs.add(xAttr);
61 | }
62 | }
63 | return filteredXAttrs;
64 | }
65 | }
66 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/giraffa/UnlocatedBlock.java:
--------------------------------------------------------------------------------
1 | package org.apache.giraffa;
2 |
3 | import java.util.ArrayList;
4 | import java.util.List;
5 |
6 | import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
7 | import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
8 | import org.apache.hadoop.hdfs.protocol.LocatedBlock;
9 | import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
10 | import org.apache.hadoop.security.token.Token;
11 |
12 | public class UnlocatedBlock {
13 |
14 | private ExtendedBlock b;
15 | private long offset; // offset of the first byte of the block in the file
16 | // corrupt flag is true if all of the replicas of a block are corrupt.
17 | // else false. If block has few corrupt replicas, they are filtered and
18 | // their locations are not part of this object
19 | private boolean corrupt;
20 | private Token blockToken = new Token();
21 |
22 | public UnlocatedBlock(LocatedBlock b) {
23 | this(b.getBlock(), b.getStartOffset(), b.isCorrupt());
24 | this.blockToken = b.getBlockToken();
25 | }
26 |
27 | public UnlocatedBlock(ExtendedBlock b, long startOffset, boolean corrupt) {
28 | this.b = b;
29 | this.offset = startOffset;
30 | this.corrupt = corrupt;
31 | }
32 |
33 | public Token getBlockToken() {
34 | return blockToken;
35 | }
36 |
37 | public ExtendedBlock getBlock() {
38 | return b;
39 | }
40 |
41 | public long getStartOffset() {
42 | return offset;
43 | }
44 |
45 | public long getBlockSize() {
46 | return b.getNumBytes();
47 | }
48 |
49 | public void setBlockToken(Token token) {
50 | this.blockToken = token;
51 | }
52 |
53 | public void setStartOffset(long value) {
54 | this.offset = value;
55 | }
56 |
57 | public void setCorrupt(boolean corrupt) {
58 | this.corrupt = corrupt;
59 | }
60 |
61 | public boolean isCorrupt() {
62 | return this.corrupt;
63 | }
64 |
65 | @Override
66 | public String toString() {
67 | return getClass().getSimpleName() + "{" + b
68 | + "; getBlockSize()=" + getBlockSize()
69 | + "; corrupt=" + corrupt
70 | + "; offset=" + offset
71 | + "}";
72 | }
73 |
74 | public LocatedBlock toLocatedBlock(DatanodeInfo[] locs) {
75 | LocatedBlock blk = new LocatedBlock(this.getBlock(), locs,
76 | this.getStartOffset(), this.isCorrupt());
77 | blk.setBlockToken(this.getBlockToken());
78 | return blk;
79 | }
80 |
81 | public static List toLocatedBlocks(List blocks,
82 | List locs) {
83 | List locatedBlocks = new ArrayList();
84 | int size = blocks.size();
85 | for(int i = 0; i < size; i++) {
86 | locatedBlocks.add(blocks.get(i).toLocatedBlock(locs.get(i)));
87 | }
88 | return locatedBlocks;
89 | }
90 | }
91 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/hadoop/hdfs/GiraffaClient.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.hadoop.hdfs;
19 |
20 | import java.io.IOException;
21 |
22 | import org.apache.giraffa.GiraffaConfiguration;
23 | import org.apache.giraffa.NamespaceService;
24 | import org.apache.hadoop.fs.ContentSummary;
25 | import org.apache.hadoop.fs.FileSystem.Statistics;
26 |
27 | /**
28 | * GiraffaClient is an extension of DFSClient
29 | * with the namenode proxy member replaced with NamespaceAgent.
30 | * This allows to reuse all the logic of DFSClient communicating
31 | * to NameNode and DataNodes for Giraffa.
32 | * NamespaceAgent seamlessly replaces client-to-NameNode communication
33 | * with client-to-HBase communication, exchanging meta-data information with
34 | * HBase instead of NameNode.
35 | * The client-to-DataNode communication remains unchanged.
36 | */
37 | public class GiraffaClient extends DFSClient {
38 |
39 | public GiraffaClient(GiraffaConfiguration conf, Statistics stats)
40 | throws IOException {
41 | super(null, conf.newNamespaceService(), conf, stats);
42 | getNamespaceService().initialize(conf);
43 | }
44 |
45 | public NamespaceService getNamespaceService() {
46 | return (NamespaceService) this.namenode;
47 | }
48 |
49 | public static void format(GiraffaConfiguration conf) throws IOException {
50 | NamespaceService namespace = conf.newNamespaceService();
51 | try {
52 | namespace.format(conf);
53 | } finally {
54 | namespace.close();
55 | }
56 | }
57 |
58 | public String getClientName() {
59 | return super.getClientName();
60 | }
61 |
62 | @Override // DFSClient
63 | public ContentSummary getContentSummary(String src)
64 | throws IOException {
65 | return super.getContentSummary(src);
66 | }
67 |
68 | @Override // DFSClient
69 | public void setQuota(String src, long namespaceQuota, long diskspaceQuota)
70 | throws IOException {
71 | super.setQuota(src, namespaceQuota, diskspaceQuota);
72 | }
73 |
74 | @Override // DFSClient
75 | public void close() throws IOException {
76 | super.close();
77 | getNamespaceService().close();
78 | }
79 | }
80 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/giraffa/RowKey.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.giraffa;
19 |
20 | import java.io.IOException;
21 |
22 | /**
23 | * Abstract class defining a row key for a file system object in the
24 | * Namespace Table in HBase.
25 | * The key is an arbitrary byte array, accessible via {@link #getKey()}.
26 | * A row key implementation defines a particular ordering of the objects in the
27 | * Namespace Table.
28 | * Based on that ordering HBase partitions tables into regions.
29 | * Hence, the row key ordering defines the locality of the file system
30 | * objects, because adjacent objects are likely to fall into the same region.
31 | *
32 | * Extend this class to define a specific row key implementation.
33 | */
34 | public abstract class RowKey {
35 |
36 | /**
37 | * Get full path of the file system object represented by the underlying row.
38 | * @return full path
39 | */
40 | public abstract String getPath();
41 |
42 | /**
43 | * Get the row key of the file system object.
44 | * The method should generate the key if it has not been generated before
45 | * or return the generated value.
46 | * @return row key as a byte array
47 | */
48 | public abstract byte[] getKey() throws IOException;
49 |
50 | /**
51 | * Generate or regenerate the row key based on the path.
52 | * Key generation can be a distributed operation for some RowKey
53 | * implementations.
54 | * @return row key as a byte array
55 | */
56 | public abstract byte[] generateKey() throws IOException;
57 |
58 | public abstract byte[] getStartListingKey(byte[] startAfter) throws IOException;
59 |
60 | public abstract byte[] getStopListingKey() throws IOException;
61 |
62 | @Override // Object
63 | public abstract boolean equals(Object o);
64 |
65 | @Override // Object
66 | public abstract int hashCode();
67 |
68 | @Override // Object
69 | public String toString() {
70 | return getClass().getSimpleName() + ": " + getKeyString()
71 | +" | " + getPath();
72 | }
73 |
74 | /**
75 | * Return a String representation of the underlying byte array for use in
76 | * {@link #toString()}.
77 | */
78 | public abstract String getKeyString();
79 | }
80 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/resources/webapps/giraffa/WEB-INF/web.xml:
--------------------------------------------------------------------------------
1 |
2 |
8 |
9 |
10 | org.apache.giraffa.web.index_jsp
11 | org.apache.giraffa.web.index_jsp
12 |
13 |
14 |
15 | org.apache.giraffa.web.stats_jsp
16 | org.apache.giraffa.web.stats_jsp
17 |
18 |
19 |
20 | org.apache.giraffa.web.filebrowser_jsp
21 | org.apache.giraffa.web.filebrowser_jsp
22 |
23 |
24 |
25 | org.apache.giraffa.web.hbase_jsp
26 | org.apache.giraffa.web.hbase_jsp
27 |
28 |
29 |
30 | org.apache.giraffa.web.htable_jsp
31 | org.apache.giraffa.web.htable_jsp
32 |
33 |
34 |
35 | org.apache.giraffa.web.namenode_jsp
36 | org.apache.giraffa.web.namenode_jsp
37 |
38 |
39 |
40 | WebjarsServlet
41 | org.webjars.servlet.WebjarsServlet
42 | 2
43 |
44 |
45 |
46 | org.apache.giraffa.web.index_jsp
47 | /index.jsp
48 |
49 |
50 |
51 | org.apache.giraffa.web.stats_jsp
52 | /stats.jsp
53 |
54 |
55 |
56 | org.apache.giraffa.web.filebrowser_jsp
57 | /filebrowser.jsp
58 |
59 |
60 |
61 | org.apache.giraffa.web.hbase_jsp
62 | /hbase.jsp
63 |
64 |
65 |
66 | org.apache.giraffa.web.htable_jsp
67 | /htable.jsp
68 |
69 |
70 |
71 | org.apache.giraffa.web.namenode_jsp
72 | /namenode.jsp
73 |
74 |
75 |
76 | WebjarsServlet
77 | /webjars/*
78 |
79 |
80 |
81 |
--------------------------------------------------------------------------------
/giraffa-core/src/test/java/org/apache/giraffa/TestGiraffaFSContract.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.giraffa;
19 |
20 | import static org.apache.hadoop.fs.CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY;
21 |
22 | import junit.extensions.TestSetup;
23 | import junit.framework.Test;
24 | import junit.framework.TestSuite;
25 |
26 | import org.apache.hadoop.fs.FileSystem;
27 | import org.apache.hadoop.fs.FileSystemContractBaseTest;
28 | import org.apache.hadoop.hbase.HBaseTestingUtility;
29 |
30 | /**
31 | * Test filesystem contract
32 | */
33 | public class TestGiraffaFSContract extends FileSystemContractBaseTest {
34 |
35 | private static final HBaseTestingUtility UTIL =
36 | GiraffaTestUtils.getHBaseTestingUtility();
37 |
38 | public TestGiraffaFSContract(String name) {
39 | setName(name);
40 | }
41 |
42 | @Override
43 | public void setUp() throws Exception {
44 | GiraffaConfiguration conf =
45 | new GiraffaConfiguration(UTIL.getConfiguration());
46 | conf.set(FS_PERMISSIONS_UMASK_KEY, TEST_UMASK);
47 | GiraffaTestUtils.setGiraffaURI(conf);
48 | GiraffaFileSystem.format(conf, false);
49 | fs = FileSystem.get(conf);
50 | super.setUp();
51 | }
52 |
53 | @Override
54 | protected void tearDown() throws Exception {
55 | super.tearDown();
56 | fs.close();
57 | }
58 |
59 | @Override
60 | public void testRenameFileAsExistingDirectory() throws Exception {
61 | // WONTFIX: behaviour is HDFS specific as described in HADOOP-6240
62 | }
63 |
64 | @Override
65 | public void testRenameDirectoryAsExistingDirectory() throws Exception {
66 | // WONTFIX: behaviour is HDFS specific as described in HADOOP-6240
67 | }
68 |
69 | public static Test suite() throws Exception {
70 | TestSuite suite = new TestSuite(TestGiraffaFSContract.class);
71 | return new TestSetup(suite) {
72 | @Override
73 | protected void setUp() throws Exception {
74 | System.setProperty(
75 | HBaseTestingUtility.BASE_TEST_DIRECTORY_KEY,
76 | GiraffaTestUtils.BASE_TEST_DIRECTORY);
77 | UTIL.startMiniCluster(1);
78 | }
79 |
80 | @Override
81 | protected void tearDown() throws Exception {
82 | UTIL.shutdownMiniCluster();
83 | }
84 | };
85 | }
86 | }
87 |
--------------------------------------------------------------------------------
/giraffa-core/src/test/resources/webapps/hdfs/WEB-INF/web.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
6 |
9 |
10 |
11 |
12 |
13 | org.apache.hadoop.hdfs.server.namenode.block_005finfo_005fxml_jsp
14 | org.apache.hadoop.hdfs.server.namenode.block_005finfo_005fxml_jsp
15 |
16 |
17 |
18 | org.apache.hadoop.hdfs.server.namenode.dfsnodelist_jsp
19 | org.apache.hadoop.hdfs.server.namenode.dfsnodelist_jsp
20 |
21 |
22 |
23 | org.apache.hadoop.hdfs.server.namenode.nn_005fbrowsedfscontent_jsp
24 | org.apache.hadoop.hdfs.server.namenode.nn_005fbrowsedfscontent_jsp
25 |
26 |
27 |
28 | org.apache.hadoop.hdfs.server.namenode.dfshealth_jsp
29 | org.apache.hadoop.hdfs.server.namenode.dfshealth_jsp
30 |
31 |
32 |
33 | org.apache.hadoop.hdfs.server.namenode.corrupt_005freplicas_005fxml_jsp
34 | org.apache.hadoop.hdfs.server.namenode.corrupt_005freplicas_005fxml_jsp
35 |
36 |
37 |
38 | org.apache.hadoop.hdfs.server.namenode.corrupt_005ffiles_jsp
39 | org.apache.hadoop.hdfs.server.namenode.corrupt_005ffiles_jsp
40 |
41 |
42 |
43 | org.apache.hadoop.hdfs.server.namenode.block_005finfo_005fxml_jsp
44 | /block_info_xml.jsp
45 |
46 |
47 |
48 | org.apache.hadoop.hdfs.server.namenode.dfsnodelist_jsp
49 | /dfsnodelist.jsp
50 |
51 |
52 |
53 | org.apache.hadoop.hdfs.server.namenode.nn_005fbrowsedfscontent_jsp
54 | /nn_browsedfscontent.jsp
55 |
56 |
57 |
58 | org.apache.hadoop.hdfs.server.namenode.dfshealth_jsp
59 | /dfshealth.jsp
60 |
61 |
62 |
63 | org.apache.hadoop.hdfs.server.namenode.corrupt_005freplicas_005fxml_jsp
64 | /corrupt_replicas_xml.jsp
65 |
66 |
67 |
68 | org.apache.hadoop.hdfs.server.namenode.corrupt_005ffiles_jsp
69 | /corrupt_files.jsp
70 |
71 |
72 |
--------------------------------------------------------------------------------
/giraffa-core/src/test/java/org/apache/giraffa/TestRestartGiraffa.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.giraffa;
19 |
20 | import org.apache.commons.logging.Log;
21 | import org.apache.commons.logging.LogFactory;
22 | import org.apache.hadoop.fs.FileSystem;
23 | import org.apache.hadoop.hbase.HBaseTestingUtility;
24 | import org.apache.hadoop.hdfs.DFSTestUtil;
25 | import org.apache.hadoop.io.IOUtils;
26 | import org.junit.After;
27 | import org.junit.AfterClass;
28 | import org.junit.Before;
29 | import org.junit.BeforeClass;
30 | import org.junit.Test;
31 |
32 | import java.io.IOException;
33 |
34 | public class TestRestartGiraffa {
35 | static final Log LOG = LogFactory.getLog(TestRestartGiraffa.class);
36 | private static final HBaseTestingUtility UTIL =
37 | GiraffaTestUtils.getHBaseTestingUtility();
38 | private GiraffaFileSystem grfs;
39 |
40 | @BeforeClass
41 | public static void beforeClass() throws Exception {
42 | System.setProperty(
43 | HBaseTestingUtility.BASE_TEST_DIRECTORY_KEY, GiraffaTestUtils.BASE_TEST_DIRECTORY);
44 | UTIL.startMiniCluster(1);
45 | }
46 |
47 | @Before
48 | public void before() throws IOException {
49 | GiraffaConfiguration conf =
50 | new GiraffaConfiguration(UTIL.getConfiguration());
51 | GiraffaTestUtils.setGiraffaURI(conf);
52 | GiraffaFileSystem.format(conf, false);
53 | grfs = (GiraffaFileSystem) FileSystem.get(conf);
54 | }
55 |
56 | @After
57 | public void after() throws IOException {
58 | IOUtils.cleanup(LOG, grfs);
59 | }
60 |
61 | @AfterClass
62 | public static void afterClass() throws Exception {
63 | UTIL.shutdownMiniCluster();
64 | }
65 |
66 | @Test
67 | public void testRestart() throws Exception {
68 | DFSTestUtil fsUtil = new DFSTestUtil("TestRestartGiraffa", 10, 3, 1024, 0);
69 | fsUtil.createFiles(grfs, "testGiraffa");
70 | fsUtil.checkFiles(grfs, "testGiraffa");
71 | grfs.close();
72 |
73 | // restart the cluster
74 | UTIL.shutdownMiniHBaseCluster();
75 | Thread.sleep(2000);
76 | UTIL.restartHBaseCluster(1);
77 | UTIL.getMiniHBaseCluster();
78 |
79 | GiraffaConfiguration conf =
80 | new GiraffaConfiguration(UTIL.getConfiguration());
81 | GiraffaTestUtils.setGiraffaURI(conf);
82 | grfs = (GiraffaFileSystem) FileSystem.get(conf);
83 | fsUtil.checkFiles(grfs, "testGiraffa");
84 | }
85 | }
86 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/giraffa/DirectoryTable.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.giraffa;
19 |
20 | import java.io.ByteArrayInputStream;
21 | import java.io.ByteArrayOutputStream;
22 | import java.io.IOException;
23 | import java.io.ObjectInputStream;
24 | import java.io.ObjectOutputStream;
25 | import java.io.Serializable;
26 | import java.util.Collection;
27 | import java.util.Collections;
28 | import java.util.HashMap;
29 | import java.util.Map;
30 |
31 | import org.apache.hadoop.fs.Path;
32 |
33 | /**
34 | * Directory in Giraffa is a row, which associates file and sub-directory names
35 | * contained in the directory with their row keys.
36 | */
37 | public class DirectoryTable implements Serializable {
38 | private static final long serialVersionUID = 987654321098765432L;
39 |
40 | private Map childrenKeys;
41 |
42 | public DirectoryTable() {
43 | childrenKeys = new HashMap();
44 | }
45 |
46 | @SuppressWarnings("unchecked")
47 | public
48 | DirectoryTable(byte[] list) throws IOException, ClassNotFoundException {
49 | ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(list));
50 | try {
51 | childrenKeys = (HashMap) in.readObject();
52 | } catch (IOException e) {
53 | throw e;
54 | } catch (ClassNotFoundException e) {
55 | throw e;
56 | } finally {
57 | in.close();
58 | }
59 | }
60 |
61 | public Collection getEntries() {
62 | return Collections.unmodifiableCollection(childrenKeys.values());
63 | }
64 |
65 | int size() {
66 | return childrenKeys.size();
67 | }
68 |
69 | public boolean isEmpty() {
70 | return size() == 0;
71 | }
72 |
73 | boolean contains(String fileName) {
74 | return childrenKeys.containsKey(fileName);
75 | }
76 |
77 | RowKey getEntry(String fileName) {
78 | return childrenKeys.get(fileName);
79 | }
80 |
81 | public boolean addEntry(RowKey child) {
82 | return childrenKeys.put(new Path(child.getPath()).getName(), child) == null;
83 | }
84 |
85 | public boolean removeEntry(String fileName) {
86 | return childrenKeys.remove(fileName) != null;
87 | }
88 |
89 | public byte[] toBytes() throws IOException {
90 | ByteArrayOutputStream bos = new ByteArrayOutputStream();
91 | ObjectOutputStream objOut = new ObjectOutputStream(bos);
92 | objOut.writeObject(childrenKeys);
93 | byte[] retVal = bos.toByteArray();
94 | objOut.close();
95 | return retVal;
96 | }
97 | }
98 |
--------------------------------------------------------------------------------
/giraffa-core/src/test/java/org/apache/giraffa/TestGiraffaPermission.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.giraffa;
19 |
20 | import mockit.Deencapsulation;
21 | import mockit.Invocation;
22 | import mockit.Mock;
23 | import mockit.MockUp;
24 |
25 | import org.apache.hadoop.fs.Path;
26 | import org.apache.hadoop.hbase.HBaseTestingUtility;
27 | import org.apache.hadoop.hdfs.TestDFSPermission;
28 | import org.apache.hadoop.security.UserGroupInformation;
29 | import org.junit.After;
30 | import org.junit.AfterClass;
31 | import org.junit.Before;
32 | import org.junit.BeforeClass;
33 |
34 | import java.io.IOException;
35 |
36 |
37 | public class TestGiraffaPermission extends TestDFSPermission {
38 |
39 | private static MockTestDFSPermission mockTest;
40 | private static HBaseTestingUtility UTIL;
41 | private static GiraffaConfiguration conf;
42 |
43 | @BeforeClass
44 | public static void beforeClass() throws Exception {
45 | System.setProperty(
46 | HBaseTestingUtility.BASE_TEST_DIRECTORY_KEY,
47 | GiraffaTestUtils.BASE_TEST_DIRECTORY);
48 | mockTest = new MockTestDFSPermission();
49 | UTIL = GiraffaTestUtils.getHBaseTestingUtility();
50 | UTIL.startMiniCluster();
51 | conf = new GiraffaConfiguration(UTIL.getConfiguration());
52 | GiraffaTestUtils.setGiraffaURI(conf);
53 | Deencapsulation.setField(TestDFSPermission.class, conf);
54 | }
55 |
56 | @Before
57 | @Override
58 | public void setUp() throws IOException {
59 | GiraffaFileSystem.format(conf, false);
60 | }
61 |
62 | @After
63 | @Override
64 | public void tearDown() throws IOException {
65 | // do nothing
66 | }
67 |
68 | @AfterClass
69 | public static void afterClass() throws Exception {
70 | UTIL.shutdownMiniCluster();
71 | mockTest.tearDown();
72 | }
73 |
74 | /**
75 | * Prevents ancestor permission checking.
76 | */
77 | private static class MockTestDFSPermission extends MockUp {
78 | @Mock
79 | public void testPermissionCheckingPerUser(Invocation inv,
80 | UserGroupInformation ugi,
81 | short[] ancestorPermission,
82 | short[] parentPermission,
83 | short[] filePermission,
84 | Path[] parentDirs,
85 | Path[] files,
86 | Path[] dirs) {
87 | for (int i = 0; i < ancestorPermission.length; i++) {
88 | ancestorPermission[i] = 0777;
89 | }
90 | inv.proceed();
91 | }
92 | }
93 | }
94 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/hadoop/hdfs/server/namenode/HLMAdapter.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.hadoop.hdfs.server.namenode;
19 |
20 | import java.util.ArrayList;
21 | import java.util.Collection;
22 | import java.util.SortedSet;
23 | import java.util.TreeSet;
24 | import org.apache.giraffa.FileLease;
25 |
26 | /**
27 | * HDFS LeaseManager adapter.
28 | * Used to implement giraffa.LeaseManager for now.
29 | */
30 | public class HLMAdapter {
31 | /** hdfs.LeaseManager */
32 | private final LeaseManager hlm;
33 |
34 | public HLMAdapter() {
35 | hlm = new LeaseManager(null);
36 | }
37 |
38 | public FileLease addLease(FileLease fileLease) {
39 | String holder = fileLease.getHolder();
40 | String path = fileLease.getPath();
41 | LeaseManager.Lease lease = hlm.addLease(holder, path);
42 | return wrapLease(lease, path);
43 | }
44 |
45 | public boolean removeLease(FileLease fileLease) {
46 | String holder = fileLease.getHolder();
47 | String path = fileLease.getPath();
48 | hlm.removeLease(holder, path);
49 | return getLeases(holder) == null;
50 | }
51 |
52 | public boolean removeLease(FileLease fileLease, String path) {
53 | String holder = fileLease.getHolder();
54 | hlm.removeLease(holder, path);
55 | return getLeases(holder) == null;
56 | }
57 |
58 | private static FileLease wrapLease(LeaseManager.Lease lease, String src) {
59 | if(lease == null)
60 | return null;
61 | else
62 | return new FileLease(lease.getHolder(), src, lease.getLastUpdate());
63 | }
64 |
65 | private Collection wrapLease(LeaseManager.Lease lease) {
66 | if(lease == null)
67 | return null;
68 | else {
69 | Collection leases = new ArrayList();
70 | for(String path : lease.getPaths()) {
71 | leases.add(wrapLease(lease, path));
72 | }
73 | return leases;
74 | }
75 | }
76 |
77 | public boolean isLeaseSoftLimitExpired(String holder) {
78 | LeaseManager.Lease lease = hlm.getLease(holder);
79 | return lease.expiredSoftLimit();
80 | }
81 |
82 | public void renewLease(String clientName) {
83 | hlm.renewLease(clientName);
84 | }
85 |
86 | public Collection getLeases(String holder) {
87 | LeaseManager.Lease lease = hlm.getLease(holder);
88 | return wrapLease(lease);
89 | }
90 |
91 | public SortedSet getSortedLeases() {
92 | SortedSet sortedLeases = hlm.getSortedLeases();
93 | SortedSet sortedFileLeases = new TreeSet();
94 | for(LeaseManager.Lease lease : sortedLeases) {
95 | sortedFileLeases.addAll(wrapLease(lease));
96 | }
97 | return sortedFileLeases;
98 | }
99 | }
100 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/giraffa/GiraffaConfiguration.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.giraffa;
19 |
20 | import org.apache.giraffa.hbase.NamespaceAgent;
21 | import org.apache.hadoop.conf.Configuration;
22 | import org.apache.hadoop.hdfs.DFSConfigKeys;
23 | import org.apache.hadoop.util.ReflectionUtils;
24 |
25 | public class GiraffaConfiguration extends Configuration {
26 | public static final String GRFA_URI_SCHEME = "grfa";
27 | public static final String GRFA_TABLE_NAME_KEY = "grfa.table.name";
28 | public static final String GRFA_TABLE_NAME_DEFAULT = "Namespace";
29 | public static final String GRFA_ROWKEY_FACTORY_KEY =
30 | "grfa.rowkey.factory.class";
31 | public static final Class
32 | GRFA_ROWKEY_FACTORY_DEFAULT = FullPathRowKeyFactory.class;
33 | public static final String GRFA_CACHING_KEY = "grfa.rowkey.caching";
34 | public static final Boolean GRFA_CACHING_DEFAULT = true;
35 | public static final String GRFA_NAMESPACE_SERVICE_KEY =
36 | "grfa.namespace.service.class";
37 | public static final Class
38 | GRFA_NAMESPACE_SERVICE_DEFAULT = NamespaceAgent.class;
39 | public static final String GRFA_HDFS_ADDRESS_KEY = "grfa.hdfs.address";
40 | public static final String GRFA_HDFS_ADDRESS_DEFAULT = "file:///";
41 | public static final String GRFA_HBASE_ADDRESS_KEY = "grfa.hbase.address";
42 | public static final String GRFA_HBASE_ADDRESS_DEFAULT = "file:///";
43 | public static final String GRFA_LIST_LIMIT_KEY =
44 | DFSConfigKeys.DFS_LIST_LIMIT;
45 | public static final int GRFA_LIST_LIMIT_DEFAULT =
46 | DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT;
47 |
48 | public static final String GRFA_WEB_ADDRESS_KEY = "grfa.http-address";
49 | public static final String GRFA_WEB_ADDRESS_DEFAULT = "0.0.0.0:40010";
50 |
51 | static {
52 | // adds the default resources
53 | addDefaultResource("giraffa-default.xml");
54 | addDefaultResource("giraffa-site.xml");
55 | addDefaultResource("hbase-default.xml");
56 | addDefaultResource("hbase-site.xml");
57 | }
58 |
59 | public GiraffaConfiguration() {
60 | super();
61 | }
62 |
63 | public GiraffaConfiguration(Configuration conf) {
64 | super(conf);
65 | }
66 |
67 | public NamespaceService newNamespaceService() {
68 | Class serviceClass =
69 | getClass(GRFA_NAMESPACE_SERVICE_KEY, GRFA_NAMESPACE_SERVICE_DEFAULT,
70 | NamespaceService.class);
71 | return ReflectionUtils.newInstance(serviceClass, null);
72 | }
73 |
74 | public static String getGiraffaTableName(Configuration conf) {
75 | return conf.get(GRFA_TABLE_NAME_KEY, GRFA_TABLE_NAME_DEFAULT);
76 | }
77 | }
78 |
--------------------------------------------------------------------------------
/giraffa-standalone/src/main/resources/bin/giraffa:
--------------------------------------------------------------------------------
1 | # Licensed to the Apache Software Foundation (ASF) under one or more
2 | # contributor license agreements. See the NOTICE file distributed with
3 | # this work for additional information regarding copyright ownership.
4 | # The ASF licenses this file to You under the Apache License, Version 2.0
5 | # (the "License"); you may not use this file except in compliance with
6 | # the License. You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | # This is giraffa shell.
17 |
18 | bin=`dirname "$0"`
19 | bin=`cd "$bin"; pwd`
20 |
21 | . "$bin"/giraffa-config.sh
22 |
23 | function print_usage(){
24 | echo "Usage: giraffa [--config confdir] COMMAND"
25 | echo " where COMMAND is one of:"
26 | echo " format format giraffa file system"
27 | echo " fs run a generic filesystem user client"
28 | echo " version print the version"
29 | echo " jar run a jar file"
30 | echo " distcp copy file or directories recursively"
31 | echo " archive -archiveName NAME -p * create a hadoop archive"
32 | echo " classpath prints the class path needed to get the"
33 | echo " Hadoop jar and the required libraries"
34 | echo " daemonlog get/set the log level for each daemon"
35 | echo " or"
36 | echo " CLASSNAME run the class named CLASSNAME"
37 | echo ""
38 | echo "Most commands print help when invoked w/o parameters."
39 | }
40 |
41 | if [ $# = 0 ]; then
42 | print_usage
43 | exit
44 | fi
45 |
46 | COMMAND=$1
47 | case $COMMAND in
48 | format)
49 | #format command
50 | CLASS=org.apache.giraffa.GiraffaFileSystem
51 |
52 | if $cygwin; then
53 | CLASSPATH=`cygpath -p -w "$CLASSPATH"`
54 | fi
55 | export CLASSPATH=$CLASSPATH
56 | echo "Executing: " $CLASS
57 | echo "CLASSPATH: " $CLASSPATH
58 | # exec "$JAVA" $JAVA_HEAP_MAX $GIRAFFA_OPTS $CLASS "$@"
59 | exec "$JAVA" $JAVA_HEAP_MAX $CLASS "$@"
60 | exit
61 | ;;
62 |
63 | #hdfs commands
64 | namenode|secondarynamenode|datanode|dfs|dfsadmin|fsck|balancer|classpath)
65 | #try to locate hdfs and if present, delegate to it.
66 | if [ -f "${HADOOP_HDFS_HOME}"/bin/hdfs ]; then
67 | exec "${HADOOP_HDFS_HOME}"/bin/hdfs $*
68 | elif [ -f "${HADOOP_HOME}"/bin/hdfs ]; then
69 | exec "${HADOOP_HOME}"/bin/hdfs $*
70 | else
71 | echo "HDFS not found."
72 | exit
73 | fi
74 | ;;
75 |
76 | #mapred commands
77 | mradmin|jobtracker|tasktracker|pipes|job|queue)
78 | #try to locate mapred and if present, delegate to it.
79 | if [ -f "${HADOOP_MAPRED_HOME}"/bin/mapred ]; then
80 | exec "${HADOOP_MAPRED_HOME}"/bin/mapred $*
81 | elif [ -f "${HADOOP_HOME}"/bin/mapred ]; then
82 | exec "${HADOOP_HOME}"/bin/mapred $*
83 | else
84 | echo "MAPRED not found."
85 | exit
86 | fi
87 | ;;
88 |
89 | #hadoop commands
90 | *)
91 | if $cygwin; then
92 | CLASSPATH=`cygpath -p -w "$CLASSPATH"`
93 | fi
94 | export HADOOP_CLASSPATH=$GIRAFFA_CLASSPATH
95 | export HADOOP_OPTS=$GIRAFFA_OPTS
96 | if [ -f "${HADOOP_HOME}"/bin/hadoop ]; then
97 | exec "${HADOOP_HOME}"/bin/hadoop --config "${GIRAFFA_CONF_DIR}" $*
98 | else
99 | echo "Hadoop not found."
100 | exit
101 | fi
102 | shift
103 | ;;
104 |
105 | esac
106 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/resources/webapps/giraffa/WEB-INF/tags/genericpage.tag:
--------------------------------------------------------------------------------
1 | <%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core" %>
2 | <%@ taglib uri="http://java.sun.com/jsp/jstl/fmt" prefix="fmt" %>
3 | <%@ attribute name="title" required="false" type="java.lang.String"%>
4 | <%@ attribute name="subtitle" required="false" type="java.lang.String"%>
5 | <%@ attribute name="isGiraffaPage" required="false" type="java.lang.Boolean"%>
6 | <%@ attribute name="isHbasePage" required="false" type="java.lang.Boolean"%>
7 | <%@ attribute name="isNamenodePage" required="false" type="java.lang.Boolean"%>
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 | <fmt:message key="webapp.title"/>
18 |
19 |
20 | ${title}
21 |
22 |
23 |
24 |
25 | <fmt:message key="webapp.title.short"/> — ${subtitle}
26 |
27 |
28 |
29 |
31 |
33 |
34 |
35 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
--------------------------------------------------------------------------------
/giraffa-core/src/site/check_style.xml:
--------------------------------------------------------------------------------
1 |
2 |
5 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
73 |
74 |
75 |
76 |
77 |
78 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/giraffa/hbase/ZKSequentialNumber.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.giraffa.hbase;
19 |
20 | import static java.util.concurrent.TimeUnit.MILLISECONDS;
21 |
22 | import org.apache.curator.RetryPolicy;
23 | import org.apache.curator.framework.CuratorFramework;
24 | import org.apache.curator.framework.CuratorFrameworkFactory;
25 | import org.apache.curator.framework.recipes.atomic.AtomicValue;
26 | import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong;
27 | import org.apache.curator.framework.recipes.atomic.PromotedToLock;
28 | import org.apache.curator.retry.RetryNTimes;
29 | import org.apache.giraffa.id.IdGeneratorService;
30 | import org.apache.hadoop.conf.Configuration;
31 | import org.apache.hadoop.hbase.zookeeper.ZKConfig;
32 |
33 | /**
34 | * Sequential ID generator that uses Zookeeper to store and compute ids.
35 | * Increments are done atomically, and this class is thread-safe.
36 | */
37 | public class ZKSequentialNumber implements IdGeneratorService {
38 |
39 | private static final RetryPolicy RETRY = new RetryNTimes(100, 10);
40 | private static final long TIMEOUT = 100;
41 |
42 | private final String name;
43 | private final long initialValue;
44 | private final CuratorFramework client;
45 | private final DistributedAtomicLong id;
46 |
47 | /**
48 | * @param name unique name identifying this generator's data on Zookeeper
49 | * @param conf configuration information for connecting to Zookeeper
50 | */
51 | public ZKSequentialNumber(String name,
52 | long initialValue,
53 | Configuration conf) {
54 | String idPath = "/ids" + name + "/id";
55 | String lockPath = "/ids" + name + "/lock";
56 | PromotedToLock lock = PromotedToLock.builder()
57 | .lockPath(lockPath)
58 | .retryPolicy(RETRY)
59 | .timeout(TIMEOUT, MILLISECONDS)
60 | .build();
61 | String quorum = ZKConfig.getZKQuorumServersString(conf);
62 | this.name = name;
63 | this.initialValue = initialValue;
64 | this.client = CuratorFrameworkFactory.newClient(quorum, RETRY);
65 | this.id = new DistributedAtomicLong(client, idPath, RETRY, lock);
66 | }
67 |
68 | @Override // IdGenerator
69 | public long nextValue() {
70 | AtomicValue value;
71 | try {
72 | value = id.increment();
73 | } catch (Exception e) {
74 | throw new RuntimeException("Failed to increment: " + name, e);
75 | }
76 | if (value.succeeded()) {
77 | return value.postValue();
78 | } else {
79 | throw new RuntimeException("Failed to increment: " + name);
80 | }
81 | }
82 |
83 | @Override // IdGeneratorService
84 | public long getInitialValue() {
85 | return initialValue;
86 | }
87 |
88 | @Override // IdGeneratorService
89 | public void initialize() {
90 | client.start();
91 | try {
92 | id.initialize(initialValue);
93 | } catch (Exception e) {
94 | throw new RuntimeException("Failed to start: " + name, e);
95 | }
96 | }
97 |
98 | @Override // IdGeneratorService
99 | public void close() {
100 | client.close();
101 | }
102 | }
103 |
--------------------------------------------------------------------------------
/giraffa-core/src/main/java/org/apache/giraffa/web/GiraffaWebUtils.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.giraffa.web;
19 |
20 | import org.apache.commons.lang.ArrayUtils;
21 | import org.apache.commons.lang.StringUtils;
22 | import org.apache.giraffa.GiraffaConfiguration;
23 | import org.apache.giraffa.GiraffaFileSystem;
24 | import org.apache.hadoop.fs.FileSystem;
25 | import org.apache.hadoop.fs.Path;
26 |
27 | import javax.servlet.ServletContext;
28 | import javax.servlet.http.HttpServletRequest;
29 | import java.io.IOException;
30 | import java.io.UnsupportedEncodingException;
31 | import java.net.URLDecoder;
32 |
33 | public class GiraffaWebUtils {
34 |
35 | public static GiraffaFileSystem getGiraffaFileSystem(
36 | ServletContext servletContext) throws IOException {
37 | GiraffaFileSystem grfs = null;
38 | if (servletContext.getAttribute("grfa") == null) {
39 | grfs = (GiraffaFileSystem) FileSystem.get(new GiraffaConfiguration());
40 | servletContext.setAttribute("grfa", grfs);
41 | } else {
42 | grfs = (GiraffaFileSystem) servletContext.getAttribute("grfa");
43 | }
44 | return grfs;
45 | }
46 |
47 | @SuppressWarnings("deprecation")
48 | public static Path extractPath(HttpServletRequest request) {
49 | String path = request.getPathInfo();
50 | if (StringUtils.isEmpty(path)) {
51 | return new Path("/");
52 | } else {
53 | try {
54 | path = URLDecoder.decode(path, "UTF-8");
55 | } catch (UnsupportedEncodingException e) {
56 | path = URLDecoder.decode(path);
57 | }
58 | return new Path(path);
59 | }
60 |
61 | }
62 |
63 | public static int safeLongToInt(long l) {
64 | if (l < Integer.MIN_VALUE || l > Integer.MAX_VALUE) {
65 | throw new IllegalArgumentException(
66 | l + " cannot be cast to int without changing its value.");
67 | }
68 | return (int) l;
69 | }
70 |
71 | /**
72 | *
Check if a String ends with any of an array of specified strings.