├── .gitignore ├── 20_glusterfs_hadoop_sudoers ├── COPYING ├── README ├── conf ├── core-site.xml └── mapred-site.xml ├── contribute.md ├── glusterfs-hadoop.spec.tmpl ├── pom.xml ├── pom.xml-1.2.1 ├── resources ├── eclipse-formatting-settings.xml └── setup.test.env.sh ├── src ├── main │ └── java │ │ └── org │ │ └── apache │ │ └── hadoop │ │ ├── fs │ │ ├── glusterfs │ │ │ ├── GlusterFSBrickClass.java │ │ │ ├── GlusterFSBrickRepl.java │ │ │ ├── GlusterFSXattr.java │ │ │ ├── GlusterFileStatus.java │ │ │ ├── GlusterFileSystem.java │ │ │ ├── GlusterFileSystemCRC.java │ │ │ ├── GlusterVolume.java │ │ │ ├── Util.java │ │ │ └── Version.java │ │ └── local │ │ │ ├── GlusterFs.java │ │ │ ├── GlusterFsCRC.java │ │ │ ├── GlusterVol.java │ │ │ └── RawLocalFsG.java │ │ └── yarn │ │ └── server │ │ └── nodemanager │ │ └── GlusterContainerExecutor.java └── test │ └── java │ └── org │ └── apache │ └── hadoop │ └── fs │ └── test │ ├── GlusterVolumeTest.java │ ├── TestBlockLocation.java │ ├── TimestampPrecision.java │ ├── connector │ ├── HcfsTestConnector.java │ ├── HcfsTestConnectorFactory.java │ └── HcfsTestConnectorInterface.java │ └── unit │ ├── GlusterMultipleVolumeTest.java │ ├── HCFSPerformanceIOTests.java │ ├── HCFSTestWorkingDir.java │ ├── HcfsFileSystemTest.java │ ├── HcfsMainOperationsBaseTest.java │ ├── HcfsTestGetFileBlockLocations.java │ └── HcfsUmaskTest.java ├── test.sh └── tools └── build-deploy-jar.py /.gitignore: -------------------------------------------------------------------------------- 1 | src/main/resources/git.properties 2 | target/ 3 | .project 4 | .classpath 5 | .settings/ 6 | -------------------------------------------------------------------------------- /20_glusterfs_hadoop_sudoers: -------------------------------------------------------------------------------- 1 | # GlusterFS sudo settings for multi-tennancy 2 | Defaults:%hadoop !requiretty 3 | %hadoop ALL=NOPASSWD:/usr/bin/getfattr -m . -n trusted.glusterfs.pathinfo * 4 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | GlusterFS Hadoop Plugin 2 | ======================= 3 | 4 | INTRODUCTION 5 | ------------ 6 | 7 | This document describes how to use GlusterFS (http://www.gluster.org/) as a backing store with Hadoop. 8 | 9 | This plugin replaces the hadoop file system (typically, the Hadoop Distributed File System) with the 10 | GlusterFileSystem, which writes to a local directory which FUSE mounts a proxy to a gluster system. 11 | 12 | REQUIREMENTS 13 | ------------ 14 | 15 | * Supported OS is GNU/Linux 16 | * GlusterFS installed on all machines in the cluster 17 | * Java Runtime Environment (JRE) 18 | * Maven 3x (needed if you are building the plugin from source) 19 | * JDK 6+ (needed if you are building the plugin from source) 20 | 21 | NOTE: Plugin relies on two *nix command line utilities to function properly. They are: 22 | 23 | * mount: Used to mount GlusterFS volumes. 24 | * getfattr: Used to fetch Extended Attributes of a file 25 | 26 | Make sure they are installed on all hosts in the cluster and their locations are in $PATH 27 | environment variable. 28 | 29 | 30 | INSTALLATION 31 | ------------ 32 | 33 | ** NOTE: Example below is for Hadoop version 0.20.2 ($GLUSTER_HOME/hdfs/0.20.2) ** 34 | 35 | * Building the plugin from source [Maven (http://maven.apache.org/) and JDK is required to build the plugin] 36 | 37 | Change to glusterfs-hadoop directory in the GlusterFS source tree and build the plugin. 38 | 39 | # cd $GLUSTER_HOME/hdfs/0.20.2 40 | # mvn package 41 | 42 | On a successful build the plugin will be present in the `target` directory. 43 | (NOTE: version number will be a part of the plugin) 44 | 45 | # ls target/ 46 | classes glusterfs-0.20.2-0.1.jar maven-archiver surefire-reports test-classes 47 | 48 | Copy the plugin to lib/ directory in your $HADOOP_HOME dir. 49 | 50 | # cp target/glusterfs-0.20.2-0.1.jar $HADOOP_HOME/lib 51 | 52 | Copy the sample configuration file that ships with this source (conf/core-site.xml) to conf 53 | directory in your $HADOOP_HOME dir. 54 | 55 | # cp conf/core-site.xml $HADOOP_HOME/conf 56 | 57 | * Installing the plugin from RPM 58 | 59 | See the plugin documentation for installing from RPM. 60 | 61 | 62 | CLUSTER INSTALLATION 63 | -------------------- 64 | 65 | In case it is tedious to do the above steps(s) on all hosts in the cluster; use the build-and-deploy.py script to 66 | build the plugin in one place and deploy it (along with the configuration file on all other hosts). 67 | 68 | This should be run on the host which is that hadoop master [Job Tracker]. 69 | 70 | * STEPS (You would have done Step 1 and 2 anyway while deploying Hadoop) 71 | 72 | 1. Edit conf/slaves file in your hadoop distribution; one line for each slave. 73 | 2. Setup password-less ssh b/w hadoop master and slave(s). 74 | 3. Edit conf/core-site.xml with all glusterfs related configurations (see CONFIGURATION) 75 | 4. Run the following 76 | # cd $GLUSTER_HOME/hdfs/0.20.2/tools 77 | # python ./build-and-deploy.py -b -d /path/to/hadoop/home -c 78 | 79 | This will build the plugin and copy it (and the config file) to all slaves (mentioned in $HADOOP_HOME/conf/slaves). 80 | 81 | Script options: 82 | -b : build the plugin 83 | -d : location of hadoop directory 84 | -c : deploy core-site.xml 85 | -m : deploy mapred-site.xml 86 | -h : deploy hadoop-env.sh 87 | 88 | 89 | CONFIGURATION 90 | ------------- 91 | 92 | All plugin configuration is done in a single XML file (core-site.xml) with tags in each 93 | block. 94 | 95 | Brief explanation of the tunables and the values they accept (change them where-ever needed) are mentioned below 96 | 97 | name: fs.glusterfs.impl 98 | value: org.apache.hadoop.fs.glusterfs.GlusterFileSystem 99 | 100 | The default FileSystem API to use (there is little reason to modify this). 101 | 102 | name: fs.default.name 103 | value: glusterfs:/// 104 | 105 | The default name that hadoop uses to represent file as a URI (typically a server:port tuple). Use any host 106 | in the cluster as the server and any port number. This option has to be in server:port format for hadoop 107 | to create file URI; but is not used by plugin. 108 | 109 | name: fs.glusterfs.volname 110 | value: volume-dist-rep 111 | 112 | The volume to mount. 113 | 114 | 115 | name: fs.glusterfs.mount 116 | value: /mnt/glusterfs 117 | 118 | This is the directory where the gluster volume is mounted 119 | 120 | name: fs.glusterfs.server 121 | value: localhost 122 | 123 | To mount a volume the plugin needs to know the hostname or the IP of a GlusterFS server in the cluster. 124 | Mention it here. 125 | 126 | USAGE 127 | ----- 128 | 129 | Once configured, start Hadoop Map/Reduce daemons 130 | 131 | # cd $HADOOP_HOME 132 | # ./bin/start-mapred.sh 133 | 134 | If the map/reduce job/task trackers are up, all I/O will be done to GlusterFS. 135 | 136 | 137 | FOR HACKERS 138 | ----------- 139 | 140 | * Source Layout (./src/) 141 | 142 | For the overall architecture, see. Currently, we use the hadoop RawLocalFileSystem as 143 | the basis - and wrap it with the GlusterVolume class. That class is then used by the 144 | Hadoop 1x (GlusterFileSystem) and Hadoop 2x (GlusterFs) adapters. 145 | 146 | https://forge.gluster.org/hadoop/pages/Architecture 147 | 148 | ./tools/build-deploy-jar.py <--- Build and Deployment Script 149 | ./conf/core-site.xml <--- Sample configuration file 150 | ./pom.xml <--- build XML file (used by maven) 151 | 152 | ./COPYING <--- License 153 | ./README <--- This file 154 | 155 | 156 | 157 | JENKINS 158 | ------- 159 | 160 | #Method 1) Modify JENKINS_USER in /etc/sysconfig/jenkins 161 | JENKINS_USER=root 162 | 163 | #Method 2) Directly modify /etc/init.d/jenkins 164 | #daemon --user "$JENKINS_USER" --pidfile "$JENKINS_PID_FILE" $JAVA_CMD $PARAMS > /dev/null 165 | echo "WARNING: RUNNING AS ROOT" 166 | daemon --user root --pidfile "$JENKINS_PID_FILE" $JAVA_CMD $PARAMS > /dev/null 167 | 168 | 169 | BUILDING 170 | -------- 171 | 172 | Building requires a working gluster mount for unit tests. 173 | The unit tests read test resources from glusterconfig.properties - a file which should be present 174 | 175 | 1) edit your .bashrc, or else at your terminal run : 176 | 177 | export GLUSTER_MOUNT=/mnt/glusterfs 178 | export HCFS_FILE_SYSTEM_CONNECTOR=org.apache.hadoop.fs.test.connector.glusterfs.GlusterFileSystemTestConnector 179 | export HCFS_CLASSNAME=org.apache.hadoop.fs.glusterfs.GlusterFileSystem 180 | 181 | (in eclipse - see below , you will add these at the "Run Configurations" menu, 182 | in VM arguments, prefixed with -D, for example, "-DGLUSTER_MOUNT=x -DHCFS_FILE_SYSTEM_CONNECTOR=y ...") 183 | 184 | 2) run: 185 | mvn clean package 186 | 187 | 3) The jar artifact will be in target/ 188 | 189 | DEVELOPING 190 | ---------- 191 | 192 | 0) Create a mock gluster mount: 193 | 194 | #Create raw disk and format it... 195 | truncate -s 1G /export/debugging_fun.brick 196 | sudo mkfs.xfs /export/debugging_fun.brick 197 | 198 | #Mount it as loopback fs 199 | mount -o loop /export/debugging_fun.brick /mnt/mybrick ; 200 | 201 | #Now make a mount point for it, and also, for gluster itself 202 | mkdir /mnt/mybrick/glusterbrick 203 | mkdir /mnt/glusterfs 204 | MNT="/mnt/glusterfs" 205 | BRICK="/mnt/mybrick/glusterbrick" 206 | 207 | #Create a gluster volume that writes to the brick 208 | sudo gluster volume create HadoopVol 10.10.61.230:$BRICK 209 | 210 | #Mount the volume on top of the newly created brick 211 | mount -t glusterfs mount -t glusterfs $(hostname):HadoopVol $MNT 212 | 213 | 1) Run "mvn eclipse:eclipse", and import into eclipse. 214 | 215 | 2) Add the exported env variables above via Run Configurations as described in the above section. 216 | 217 | 3) Develop and run unit tests as you would any other java app. -------------------------------------------------------------------------------- /conf/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | fs.glusterfs.impl 11 | org.apache.hadoop.fs.glusterfs.GlusterFileSystem 12 | 13 | 14 | 15 | 21 | 22 | 23 | fs.default.name 24 | glusterfs:/// 25 | 26 | 27 | 28 | 29 | 30 | fs.AbstractFileSystem.glusterfs.impl 31 | org.apache.hadoop.fs.local.GlusterFs 32 | 33 | 34 | 35 | 41 | 42 | 43 | fs.glusterfs.volumes 44 | gv0,gv1 45 | 46 | 47 | 48 | fs.glusterfs.volume.fuse.gv0 49 | /mnt/gv0 50 | 51 | 52 | 53 | fs.glusterfs.volume.fuse.gv1 54 | /mnt/gv1 55 | 56 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /conf/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | yarn.app.mapreduce.am.staging-dir 5 | glusterfs:///tmp/hadoop-yarn/staging/mapred/.staging 6 | 7 | 8 | mapred.healthChecker.script.path 9 | glusterfs:///mapred/jobstatus 10 | 11 | 12 | mapred.job.tracker.history.completed.location 13 | glusterfs:///mapred/history/done 14 | 15 | 16 | mapred.system.dir 17 | glusterfs:///mapred/system 18 | 19 | 20 | mapreduce.jobhistory.done-dir 21 | glusterfs:///job-history/done 22 | 23 | 24 | 25 | mapreduce.jobhistory.intermediate-done-dir 26 | glusterfs:///job-history/intermediate-done 27 | 28 | 29 | 30 | mapreduce.jobtracker.staging.root.dir 31 | glusterfs:///user 32 | 33 | -------------------------------------------------------------------------------- /contribute.md: -------------------------------------------------------------------------------- 1 | # Gluster Hadoop Interface - Contributing 2 | 3 | ## Overview 4 | The gluster-hadoop interface is an open source gluster optimized implementation of the org.apache.hadoop.fs.FileSystem class. 5 | 6 | Community participation is open to those who follow the process. 7 | 8 | 9 | ### Bugs 10 | Bugs are tracked via Red Hat's Bugzilla tracking tools under the Red Hat Storage component. To open a bug visit [Red Hat Bugzilla](https://bugzilla.redhat.com/enter_bug.cgi?product=Red%20Hat%20Storage) and select the gluster-hadoop component. 11 | 12 | ### Submitting Patches 13 | To submit a patch to the gluster source repository, use the github.com pull request feature. 14 | 15 | Start by creating a clone of the hadoop-gluster github repo. After you've cloned the repo, you can code a fix and push the changes to your personal code repository. 16 | 17 | After the changes are pushed to github open a pull request to integrate the changes into the hadoop-gluster source repository. 18 | 19 | Please include any technical information about the patch, a link to a bugzilla bug, and any other relevant information. 20 | 21 | 22 | ### Request for merge 23 | Once your push request has been submited through github please notify bdc@redhat.com. Patches are merged by vote of the PMC. 24 | 25 | -------------------------------------------------------------------------------- /glusterfs-hadoop.spec.tmpl: -------------------------------------------------------------------------------- 1 | Name: rhs-hadoop 2 | Version: $version 3 | # release number is automatically updated when source version is the same 4 | Release: $release%{?dist} 5 | #if $epoch 6 | Epoch: $epoch 7 | #end if 8 | License: Apache License 9 | URL: https://forge.gluster.org/hadoop/glusterfs-hadoop 10 | Summary: GlusterFS Hadoop Plug-in 11 | Group: Applications/File 12 | 13 | #for $i, $artifact in $enumerate($all_artifacts) 14 | Source$i: $artifact 15 | #end for 16 | Source100: 20_glusterfs_hadoop_sudoers 17 | 18 | BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) 19 | BuildArch: noarch 20 | 21 | # yes, this is /usr/lib even on 64-bit 22 | %global hadoop_libdir %{_prefix}/lib/hadoop/lib 23 | 24 | %description 25 | Hadoop FileSystem support for GlusterFS 26 | 27 | %package javadoc 28 | Summary: API documentation for %{name} 29 | 30 | %description javadoc 31 | %{summary} 32 | 33 | %prep 34 | 35 | %build 36 | 37 | %install 38 | rm -rf %{buildroot} 39 | /bin/mkdir -p %{buildroot}%{_javadir} 40 | /bin/mkdir -p %{buildroot}%{hadoop_libdir} 41 | /bin/mkdir -p %{buildroot}%{_sysconfdir}/sudoers.d 42 | 43 | #for $i, $artifact in $enumerate($all_artifacts) 44 | #if $artifact.endswith('.jar') 45 | /usr/bin/install -m 644 %{SOURCE$i} %{buildroot}%{_javadir} 46 | /bin/ln -s ../../../..%{_javadir}/$artifact %{buildroot}%{hadoop_libdir}/$artifact 47 | #end if 48 | #end for 49 | 50 | # move sudoers file to /etc/sudoers.d/ 51 | install -m 644 %{SOURCE100} %{buildroot}%{_sysconfdir}/sudoers.d/ 52 | 53 | %clean 54 | rm -rf %{buildroot} 55 | 56 | %files javadoc -f .mfiles-javadoc 57 | %doc COPYING 58 | 59 | %files 60 | %defattr(-,root,root,-) 61 | #for $i, $artifact in $enumerate($all_artifacts) 62 | #if $artifact.endswith('.jar') 63 | %{_javadir}/$artifact 64 | %{hadoop_libdir}/$artifact 65 | #end if 66 | #end for 67 | %config %{_sysconfdir}/sudoers.d/20_glusterfs_hadoop_sudoers 68 | 69 | %changelog 70 | * Thu Mar 05 2015 Shubhendu Tripathi 2.1.6-4 71 | - bz#1094881 Modified to generate javadoc 72 | 73 | * Thu Oct 13 2014 Shubhendu Tripathi 2.1.6-3 74 | - bz#1122909 Corrected the product name as GlusterFS instead of 75 | - Red Hat Storage in description 76 | 77 | * Thu Oct 09 2014 Shubhendu Tripathi 2.1.6-2 78 | - bz#1122909 corrected the description 79 | 80 | * Thu Oct 09 2014 Shubhendu Tripathi 2.1.6-1 81 | - bz#1065417 fixed rpmlint errors and warnings 82 | 83 | * Wed Feb 06 2014 Jeff Vance 2.1.6 84 | - fix name from "gluster" to "glusterfs-hadoop" 85 | - minor README updates 86 | 87 | * Wed Feb 05 2014 Jeff Vance 2.1.5-2 88 | - installs the sudoers file. BZ 1059986 89 | 90 | * Wed Jan 9 2014 Jay Vyas 2.1.4-1 91 | - rename to rhs-hadoop for release 92 | 93 | * Fri Nov 22 2013 Jay Vyas 2.1.4 94 | - Initial import into the glusterfs-hadoop plug-in as part of our deployment. 95 | 96 | * Fri May 18 2012 Anthony Towns - 0.20.2-0 97 | - Initial build 98 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | org.gluster 5 | glusterfs-hadoop 6 | jar 7 | 9 | 2.3.13 10 | glusterfs-hadoop 11 | hadoop filesystem impl for glusterfs 12 | 13 | 14 | apache 15 | 16 | 17 | 18 | git://forge.gluster.org/hadoop/glusterfs-hadoop.git 19 | 20 | http://maven.apache.org 21 | 22 | 23 | bchilds 24 | bradley childs 25 | bchilds@gmail.com 26 | red hat 27 | http://www.redhat.com 28 | 29 | Dev 30 | 31 | 32 | -6 33 | 34 | 35 | 36 | 37 | 38 | ossrh 39 | https://oss.sonatype.org/content/repositories/snapshots 40 | 41 | 42 | ossrh 43 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 44 | 45 | 46 | 47 | 48 | 49 | 50 | org.apache.hadoop 51 | hadoop-yarn-server-nodemanager 52 | 2.3.0 53 | 54 | 55 | 56 | junit 57 | junit 58 | 4.9 59 | test 60 | 61 | 62 | org.apache.hadoop 63 | hadoop-common 64 | 2.3.0 65 | 66 | 67 | org.apache.hadoop 68 | hadoop-common-test 69 | 0.22.0 70 | test 71 | 72 | 73 | org.apache.hadoop 74 | hadoop-test 75 | 1.0.0 76 | test 77 | 78 | 79 | org.slf4j 80 | slf4j-api 81 | 1.5.8 82 | compile 83 | 84 | 85 | org.slf4j 86 | slf4j-log4j12 87 | 1.7.3 88 | 89 | 90 | 91 | 92 | 93 | 94 | conf 95 | 96 | 97 | 98 | 99 | 100 | org.springframework.build 101 | aws-maven 102 | 5.0.0.RELEASE 103 | 104 | 105 | 106 | 107 | 108 | 109 | org.apache.maven.plugins 110 | maven-source-plugin 111 | 2.2.1 112 | 113 | 114 | attach-sources 115 | 116 | jar-no-fork 117 | 118 | 119 | 120 | 121 | 122 | org.apache.maven.plugins 123 | maven-javadoc-plugin 124 | 2.9.1 125 | 126 | 127 | attach-javadocs 128 | 129 | jar 130 | 131 | 132 | 133 | 134 | 135 | org.sonatype.plugins 136 | nexus-staging-maven-plugin 137 | 1.6.3 138 | true 139 | 140 | ossrh 141 | https://oss.sonatype.org/ 142 | true 143 | 144 | 145 | 146 | 147 | org.apache.maven.plugins 148 | maven-compiler-plugin 149 | 2.3.2 150 | 151 | 1.5 152 | 1.5 153 | 154 | 155 | 156 | org.apache.maven.plugins 157 | maven-gpg-plugin 158 | 1.5 159 | 160 | 161 | sign-artifacts 162 | verify 163 | 164 | sign 165 | 166 | 167 | 168 | 169 | 170 | pl.project13.maven 171 | git-commit-id-plugin 172 | 2.1.4 173 | 174 | 175 | 176 | revision 177 | 178 | 179 | 180 | 181 | 182 | 183 | git 184 | 185 | 186 | dd.MM.yyyy '@' HH:mm:ss z 187 | 188 | 190 | true 191 | 192 | 195 | ${project.basedir}/.git 196 | 197 | 198 | 201 | 202 | 206 | true 207 | 208 | 210 | true 211 | 212 | 214 | src/main/resources/git.properties 215 | 216 | 218 | 219 | false 220 | 221 | 222 | 226 | 227 | 228 | true 229 | 230 | 231 | false 232 | 234 | false 235 | 238 | 7 239 | 240 | 242 | -dirty 243 | 244 | 247 | false 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | -------------------------------------------------------------------------------- /pom.xml-1.2.1: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | org.apache.hadoop.fs.glusterfs 5 | glusterfs 6 | jar 7 | 2.0-SNAPSHOT 8 | glusterfs 9 | http://maven.apache.org 10 | 11 | 12 | junit 13 | junit 14 | 4.11 15 | 16 | 17 | org.apache.hadoop 18 | hadoop-core 19 | 1.2.1 20 | 21 | 22 | org.apache.hadoop 23 | hadoop-test 24 | 1.2.1 25 | test 26 | 27 | 28 | org.apache.hadoop 29 | hadoop-test 30 | 1.2.1 31 | test 32 | 33 | 34 | org.slf4j 35 | slf4j-api 36 | 1.5.8 37 | compile 38 | 39 | 40 | org.slf4j 41 | slf4j-log4j12 42 | 1.7.3 43 | 44 | 45 | 46 | 47 | 48 | org.apache.maven.plugins 49 | maven-surefire-plugin 50 | 2.14 51 | 52 | 57 | 58 | 59 | GLUSTER_MOUNT 60 | ${GLUSTER_MOUNT} 61 | 62 | 63 | HCFS_CLASSNAME 64 | ${HCFS_CLASSNAME} 65 | 66 | 67 | HCFS_FILE_SYSTEM_CONNECTOR 68 | ${HCFS_FILE_SYSTEM_CONNECTOR} 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | org.apache.maven.plugins 77 | maven-compiler-plugin 78 | 2.3.2 79 | 80 | 1.5 81 | 1.5 82 | 83 | 84 | 85 | 86 | pl.project13.maven 87 | git-commit-id-plugin 88 | 2.1.4 89 | 90 | 91 | 92 | revision 93 | 94 | 95 | 96 | 97 | 98 | 99 | git 100 | 101 | 102 | dd.MM.yyyy '@' HH:mm:ss z 103 | 104 | 105 | true 106 | 107 | 112 | ${project.basedir}/.git 113 | 114 | 115 | 120 | 121 | 127 | true 128 | 129 | 130 | true 131 | 132 | 133 | src/main/resources/git.properties 134 | 135 | 136 | 137 | false 138 | 139 | 140 | 145 | 146 | 147 | true 148 | 149 | 150 | false 151 | 155 | false 156 | 162 | 7 163 | 164 | 165 | -dirty 166 | 167 | 171 | false 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | -------------------------------------------------------------------------------- /resources/eclipse-formatting-settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | -------------------------------------------------------------------------------- /resources/setup.test.env.sh: -------------------------------------------------------------------------------- 1 | #* Copyright (c) 2011 Gluster, Inc. 2 | # This file is part of GlusterFS. 3 | # 4 | # Licensed under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 | # implied. See the License for the specific language governing 14 | # permissions and limitations under the License. 15 | # 16 | # This script creates a gluster volume using block files mounted on loopback as bricks. 17 | # The puprose is to allow scripted create and delete of gluster volumes without needing to alter 18 | # disk partition or volume structure. 19 | 20 | print_help() { 21 | echo "Usage: $0 [OPTION]..." 22 | echo "" 23 | echo "Options:" 24 | echo " -w, --work working directory for gluster block device bricks" 25 | echo " -v, --volume gluster volume to create" 26 | echo " -h, --help show this message" 27 | echo "" 28 | echo "Note: bricks and gluster volume will not persist on reboot. Please edit fstab manually if you wish." 29 | } 30 | 31 | # parse options 32 | while [ "${1+isset}" ]; do 33 | case "$1" in 34 | -v|--volume) 35 | GLUSTER_VOLUME=$2 36 | shift 2 37 | ;; 38 | -w|--work) 39 | WORK_DIR=$2 40 | shift 2 41 | ;; 42 | -h|--help) 43 | print_help 44 | exit 45 | ;; 46 | *) 47 | echo "Error: Unknown option: $1" >&2 48 | exit 1 49 | ;; 50 | esac 51 | done 52 | 53 | if [ -z ${GLUSTER_VOLUME} ] 54 | then 55 | echo "I am error. No volume specified." 56 | echo "" 57 | print_help 58 | exit 1; 59 | fi 60 | 61 | if [ -z ${WORK_DIR} ] 62 | then 63 | echo "I am error. No temp directory set." 64 | echo "" 65 | print_help 66 | exit 1; 67 | fi 68 | 69 | NUMBER_OF_BRICKS=3 70 | 71 | mkdir -p ${WORK_DIR}/rpm 72 | mkdir -p ${WORK_DIR}/bricks 73 | mkdir -p ${WORK_DIR}/blocks 74 | 75 | 76 | 77 | # Grap and extract the RPM 78 | #cd ${WORK_DIR}/rpm 79 | #wget ${GLUSTER_URL} 80 | #rpm2cpio *.rpm | cpio -idmv 81 | 82 | #create some loopback bricks 83 | 84 | createGlusterVolume(){ 85 | LOOP_BRICKS=( ) 86 | 87 | # TODO: Need to run a force on this command so it doesn't prompt 88 | 89 | GLUSTER_VOLUME_CMD="gluster volume create ${GLUSTER_VOLUME} " 90 | HOSTNAME=`hostname` 91 | 92 | for (( i = 0 ; i <= ${NUMBER_OF_BRICKS}; i++ )) 93 | do 94 | # Create an empty block device to use as loop back filesystem 95 | BLOCK=${WORK_DIR}/blocks/block${i} 96 | BRICK=${WORK_DIR}/bricks/brick${i} 97 | 98 | dd if=/dev/zero of=${BLOCK} bs=1024 count=30720 99 | 100 | # find a free loopback device 101 | # LOOP_BRICKS[${i}]=${WORK_DIR}/bricks/brick${i} 102 | LB_BRICK=`losetup -f` 103 | LOOP_BRICKS[${i}]="${LB_BRICK}" 104 | 105 | 106 | echo "creating loopback file system on loopback: ${LB_BRICK} block: ${BLOCK}" 107 | losetup ${LB_BRICK} ${BLOCK} 108 | echo "Making loopback block brick on ${LB_BRICK}" 109 | # mkfs.xfs -m 1 -v ${BLOCK} 110 | mkfs -t ext4 -m 1 -v ${LB_BRICK} 111 | 112 | 113 | mkdir -p ${BRICK} 114 | mount -t ext4 $LB_BRICK $BRICK 115 | GLUSTER_VOLUME_CMD="${GLUSTER_VOLUME_CMD} ${HOSTNAME}:${BRICK}" 116 | done 117 | 118 | # Run the gluster command to create the volume 119 | 120 | 121 | echo "running: ${GLUSTER_VOLUME_CMD}" 122 | $GLUSTER_VOLUME_CMD 123 | gluster volume start ${GLUSTER_VOLUME} 124 | } 125 | 126 | createCleanupScript(){ 127 | # clean up after ourselfs 128 | # create an unmount script for the bricks 129 | echo "#!/bin/sh" > ${WORK_DIR}/cleanup.sh 130 | chmod +x ${WORK_DIR}/cleanup.sh 131 | 132 | echo "gluster volume stop ${GLUSTER_VOLUME}" >> ${WORK_DIR}/cleanup.sh 133 | echo "gluster volume delete ${GLUSTER_VOLUME}" >> ${WORK_DIR}/cleanup.sh 134 | # Unmount the bricks and loopback devices 135 | 136 | for (( i = 0 ; i <= ${NUMBER_OF_BRICKS}; i++ )) 137 | do 138 | # Create an empty block device to use as loop back filesystem 139 | BLOCK=${WORK_DIR}/blocks/block${i} 140 | BRICK=${WORK_DIR}/bricks/brick${i} 141 | echo "umount $BRICK" >> ${WORK_DIR}/cleanup.sh 142 | echo "losetup -d ${LOOP_BRICKS[$i]}" >> ${WORK_DIR}/cleanup.sh 143 | echo "rm -rf $BLOCK" >> ${WORK_DIR}/cleanup.sh 144 | echo "rm -rf $BRICK" >> ${WORK_DIR}/cleanup.sh 145 | 146 | done 147 | 148 | echo "rm -rf ${WORK_DIR}" >> ${WORK_DIR}/cleanup.sh 149 | } 150 | 151 | createGlusterVolume 152 | createCleanupScript 153 | echo "Cleanup script: ${WORK_DIR}/cleanup.sh" 154 | -------------------------------------------------------------------------------- /src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSBrickClass.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Copyright (c) 2011 Gluster, Inc. 4 | * This file is part of GlusterFS. 5 | * 6 | * Licensed under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 15 | * implied. See the License for the specific language governing 16 | * permissions and limitations under the License. 17 | * 18 | */ 19 | 20 | package org.apache.hadoop.fs.glusterfs; 21 | 22 | import java.io.*; 23 | 24 | public class GlusterFSBrickClass{ 25 | String host; 26 | String exportedFile; 27 | long start; 28 | long end; 29 | boolean isChunked; 30 | int stripeSize; // Stripe size in bytes 31 | int nrStripes; // number of stripes 32 | int switchCount; // for SR, DSR - number of replicas of each stripe 33 | // -1 for others 34 | 35 | public GlusterFSBrickClass(String brick, long start, long len, boolean flag, int stripeSize, int nrStripes, int switchCount) throws IOException{ 36 | this.host=brick2host(brick); 37 | this.exportedFile=brick2file(brick); 38 | this.start=start; 39 | this.end=start+len; 40 | this.isChunked=flag; 41 | this.stripeSize=stripeSize; 42 | this.nrStripes=nrStripes; 43 | this.switchCount=switchCount; 44 | } 45 | 46 | public boolean isChunked(){ 47 | return isChunked; 48 | } 49 | 50 | public String brickIsLocal(String hostname){ 51 | String path=null; 52 | if(host.equals(hostname)) 53 | path=exportedFile; 54 | 55 | return path; 56 | } 57 | 58 | public int[] getBrickNumberInTree(long start,int len){ 59 | long end=len; 60 | int startNodeInTree=((int) (start/stripeSize))%nrStripes; 61 | int endNodeInTree=((int) ((start+len)/stripeSize))%nrStripes; 62 | 63 | if(startNodeInTree!=endNodeInTree){ 64 | end=(start-(start%stripeSize))+stripeSize; 65 | end-=start; 66 | } 67 | 68 | return new int[]{startNodeInTree,endNodeInTree,(int) end}; 69 | } 70 | 71 | public boolean brickHasFilePart(int nodeInTree,int nodeLoc){ 72 | if(switchCount==-1) 73 | return (nodeInTree==nodeLoc); 74 | 75 | nodeInTree*=switchCount; 76 | for(int i=nodeInTree;i<(nodeInTree+switchCount);i++){ 77 | if(i==nodeLoc) 78 | return true; 79 | } 80 | 81 | return false; 82 | } 83 | 84 | public String brick2host(String brick) throws IOException{ 85 | String[] hf=null; 86 | 87 | hf=brick.split(":"); 88 | if(hf.length!=2) 89 | throw new IOException("Error getting hostname from brick"); 90 | 91 | return hf[0]; 92 | } 93 | 94 | public String brick2file(String brick) throws IOException{ 95 | String[] hf=null; 96 | 97 | hf=brick.split(":"); 98 | if(hf.length!=2) 99 | throw new IOException("Error getting hostname from brick"); 100 | 101 | return hf[1]; 102 | } 103 | 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSBrickRepl.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Copyright (c) 2011 Gluster, Inc. 4 | * This file is part of GlusterFS. 5 | * 6 | * Licensed under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 15 | * implied. See the License for the specific language governing 16 | * permissions and limitations under the License. 17 | * 18 | */ 19 | 20 | package org.apache.hadoop.fs.glusterfs; 21 | 22 | public class GlusterFSBrickRepl{ 23 | private String[] replHost; 24 | private long start; 25 | private long len; 26 | private int cnt; 27 | 28 | GlusterFSBrickRepl(int replCount, long start, long len){ 29 | this.replHost=new String[replCount]; 30 | this.start=start; 31 | this.len=len; 32 | this.cnt=0; 33 | } 34 | 35 | public void addHost(String host){ 36 | this.replHost[cnt++]=host; 37 | } 38 | 39 | public String[] getReplHosts(){ 40 | return this.replHost; 41 | } 42 | 43 | public long getStartLen(){ 44 | return this.start; 45 | } 46 | 47 | public long getOffLen(){ 48 | return this.len; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFSXattr.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Copyright (c) 2011 Gluster, Inc. 4 | * This file is part of GlusterFS. 5 | * 6 | * Licensed under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 15 | * implied. See the License for the specific language governing 16 | * permissions and limitations under the License. 17 | * 18 | */ 19 | 20 | package org.apache.hadoop.fs.glusterfs; 21 | 22 | import java.io.BufferedReader; 23 | import java.io.IOException; 24 | import java.io.InputStreamReader; 25 | import java.util.ArrayList; 26 | import java.util.regex.Matcher; 27 | import java.util.regex.Pattern; 28 | 29 | import org.apache.hadoop.fs.BlockLocation; 30 | 31 | public class GlusterFSXattr{ 32 | 33 | private String getFattrCmdBase = null; 34 | private String filename = null; 35 | private String xattrValue = null; 36 | 37 | public GlusterFSXattr(String fileName) { 38 | this(fileName, "sudo getfattr -m . -n trusted.glusterfs.pathinfo"); 39 | } 40 | 41 | public GlusterFSXattr(String fileName, String getAttr){ 42 | getFattrCmdBase=getAttr; 43 | this.filename = fileName; 44 | } 45 | 46 | public void reset(){ 47 | xattrValue=null; 48 | } 49 | 50 | public static String shellToString(String shellCommand) throws IOException{ 51 | 52 | Process p=Runtime.getRuntime().exec(shellCommand); 53 | BufferedReader brInput=null; 54 | String s=null; 55 | 56 | brInput=new BufferedReader(new InputStreamReader(p.getInputStream())); 57 | 58 | String value=""; 59 | while ((s=brInput.readLine())!=null) 60 | value+=s; 61 | 62 | return value; 63 | } 64 | 65 | /* Caches the xattr value. Must call reset() to re-query */ 66 | public String execGetFattr() throws IOException{ 67 | if(xattrValue==null){ 68 | xattrValue=shellToString(this.getFattrCmdBase + " " + filename); 69 | } 70 | 71 | return xattrValue; 72 | } 73 | 74 | public BlockLocation[] getPathInfo(long start, long len) { 75 | String xattr = null; 76 | Pattern blockReg = Pattern.compile(""); 77 | try { 78 | xattr = execGetFattr(); 79 | } catch (IOException e) { 80 | // problem executing getfattr command, fail gracefully. 81 | } 82 | 83 | Matcher matcher = blockReg.matcher(xattr); 84 | ArrayList list = new ArrayList(); 85 | while(matcher.find()){ 86 | list.add(matcher.group(1)); 87 | } 88 | 89 | /* no pathinfo found*/ 90 | if(list.size() == 0) 91 | return null; 92 | 93 | String hosts[] = list.toArray(new String[list.size()]); 94 | return new BlockLocation[]{ new BlockLocation(null, hosts, start, len) }; 95 | 96 | } 97 | 98 | } -------------------------------------------------------------------------------- /src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileStatus.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Copyright (c) 2011 Red Hat, Inc. 4 | * This file is part of GlusterFS. 5 | * 6 | * Licensed under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 15 | * implied. See the License for the specific language governing 16 | * permissions and limitations under the License. 17 | * 18 | * Implements the Hadoop FileSystem Interface to allow applications to store 19 | * files on GlusterFS and run Map/Reduce jobs on the data. 20 | * 21 | * 22 | */ 23 | 24 | package org.apache.hadoop.fs.glusterfs; 25 | 26 | import java.io.DataOutput; 27 | import java.io.File; 28 | import java.io.IOException; 29 | import java.util.StringTokenizer; 30 | 31 | import org.apache.hadoop.fs.FileStatus; 32 | import org.apache.hadoop.fs.permission.FsPermission; 33 | import org.apache.hadoop.util.Shell; 34 | import org.apache.hadoop.util.StringUtils; 35 | /* 36 | * Copied from org.apache.fs.RawLocalFileSystem.RawFileStatus 37 | */ 38 | public class GlusterFileStatus extends FileStatus{ 39 | /* 40 | * We can add extra fields here. It breaks at least CopyFiles.FilePair(). We 41 | * recognize if the information is already loaded by check if 42 | * onwer.equals(""). 43 | */ 44 | protected GlusterVolume fs; 45 | 46 | private boolean isPermissionLoaded(){ 47 | return !super.getOwner().equals(""); 48 | } 49 | 50 | 51 | /* 52 | * This constructor is the only difference than the RawLocalFileStatus impl. RawLocalFileSystem converts a raw file path to path with the same prefix. ends up with a double /mnt/glusterfs. 53 | */ 54 | GlusterFileStatus(File f, long defaultBlockSize, GlusterVolume fs){ 55 | super(f.length(), f.isDirectory(), 1, defaultBlockSize, fs.trimTimestamp(f.lastModified()), fs.fileToPath(f)); 56 | this.fs=fs; 57 | } 58 | 59 | GlusterFileStatus(File f, GlusterVolume fs){ 60 | this(f,f.length(),fs); 61 | } 62 | 63 | @Override 64 | public FsPermission getPermission(){ 65 | if(!isPermissionLoaded()){ 66 | loadPermissionInfo(); 67 | } 68 | return super.getPermission(); 69 | } 70 | 71 | @Override 72 | public String getOwner(){ 73 | if(!isPermissionLoaded()){ 74 | loadPermissionInfo(); 75 | } 76 | return super.getOwner(); 77 | } 78 | 79 | @Override 80 | public String getGroup(){ 81 | if(!isPermissionLoaded()){ 82 | loadPermissionInfo(); 83 | } 84 | return super.getGroup(); 85 | } 86 | 87 | // / loads permissions, owner, and group from `ls -ld` 88 | private void loadPermissionInfo(){ 89 | IOException e=null; 90 | try{ 91 | StringTokenizer t=new StringTokenizer(Util.execCommand(fs.pathToFile(getPath()), Util.getGET_PERMISSION_COMMAND())); 92 | // expected format 93 | // -rw------- 1 username groupname ... 94 | String permission=t.nextToken(); 95 | if(permission.length()>10){ // files with ACLs might have a '+' 96 | permission=permission.substring(0, 10); 97 | } 98 | setPermission(FsPermission.valueOf(permission)); 99 | t.nextToken(); 100 | setOwner(t.nextToken()); 101 | setGroup(t.nextToken()); 102 | }catch (Shell.ExitCodeException ioe){ 103 | if(ioe.getExitCode()!=1){ 104 | e=ioe; 105 | }else{ 106 | setPermission(null); 107 | setOwner(null); 108 | setGroup(null); 109 | } 110 | }catch (IOException ioe){ 111 | e=ioe; 112 | }finally{ 113 | if(e!=null){ 114 | throw new RuntimeException("Error while running command to get "+"file permissions : "+StringUtils.stringifyException(e)); 115 | } 116 | } 117 | } 118 | 119 | @Override 120 | public void write(DataOutput out) throws IOException{ 121 | if(!isPermissionLoaded()){ 122 | loadPermissionInfo(); 123 | } 124 | super.write(out); 125 | } 126 | 127 | } 128 | -------------------------------------------------------------------------------- /src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystem.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Copyright (c) 2013 Red Hat, Inc. 4 | * This file is part of GlusterFS. 5 | * 6 | * Licensed under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 15 | * implied. See the License for the specific language governing 16 | * permissions and limitations under the License. 17 | * 18 | */ 19 | 20 | /** 21 | * Implements the Hadoop FileSystem Interface to allow applications to store 22 | * files on GlusterFS and run Map/Reduce jobs on the data. This code does NOT perform a CRC 23 | * on files. 24 | * 25 | * gluster file systems are specified with the glusterfs:// prefix. 26 | * 27 | * 28 | */ 29 | package org.apache.hadoop.fs.glusterfs; 30 | 31 | import java.io.File; 32 | import java.io.IOException; 33 | import java.net.URI; 34 | 35 | import org.apache.hadoop.conf.Configuration; 36 | import org.apache.hadoop.fs.FileSystem; 37 | import org.apache.hadoop.fs.FileUtil; 38 | import org.apache.hadoop.fs.FilterFileSystem; 39 | import org.apache.hadoop.fs.permission.FsPermission; 40 | import org.apache.hadoop.fs.Path; 41 | import org.slf4j.Logger; 42 | import org.slf4j.LoggerFactory; 43 | 44 | public class GlusterFileSystem extends FilterFileSystem{ 45 | 46 | protected static final Logger log=LoggerFactory.getLogger(GlusterFileSystem.class); 47 | 48 | public GlusterFileSystem(){ 49 | super(new GlusterVolume()); 50 | Version v=new Version(); 51 | log.info("Initializing GlusterFS, CRC disabled."); 52 | log.info("GIT INFO="+v); 53 | log.info("GIT_TAG="+v.getTag()); 54 | } 55 | 56 | /** Convert a path to a File. */ 57 | public File pathToFile(Path path){ 58 | return ((GlusterVolume) fs).pathToFile(path); 59 | } 60 | 61 | /** 62 | * Get file status. 63 | */ 64 | public boolean exists(Path f) throws IOException{ 65 | File path=pathToFile(f); 66 | if(path.exists()){ 67 | return true; 68 | }else{ 69 | return false; 70 | } 71 | } 72 | 73 | public void setConf(Configuration conf){ 74 | log.info("Configuring GlusterFS"); 75 | if(conf!=null) conf.addResource("glusterfs-site.xml"); 76 | super.setConf(conf); 77 | } 78 | 79 | /* 80 | * if GlusterFileSystem is the default filesystem, real local URLs come back 81 | * without a file:/ scheme name (BUG!). the glusterfs file system is 82 | * assumed. force a schema. 83 | */ 84 | 85 | public void copyFromLocalFile(boolean delSrc,Path src,Path dst) throws IOException{ 86 | FileSystem srcFs=new Path("file:/"+src.toString()).getFileSystem(getConf()); 87 | FileSystem dstFs=dst.getFileSystem(getConf()); 88 | FileUtil.copy(srcFs, src, dstFs, dst, delSrc, getConf()); 89 | } 90 | 91 | public void copyToLocalFile(boolean delSrc,Path src,Path dst) throws IOException{ 92 | FileSystem srcFs=src.getFileSystem(getConf()); 93 | FileSystem dstFs=new Path("file:/"+dst.toString()).getFileSystem(getConf()); 94 | FileUtil.copy(srcFs, src, dstFs, dst, delSrc, getConf()); 95 | } 96 | 97 | @Override 98 | public boolean mkdirs(Path f) throws IOException { 99 | return mkdirs(f, FsPermission.getDirDefault().applyUMask(FsPermission.getUMask(getConf()))); 100 | } 101 | 102 | public String toString(){ 103 | return "Gluster File System, no CRC."; 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/main/java/org/apache/hadoop/fs/glusterfs/GlusterFileSystemCRC.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2013 Red Hat, Inc. 3 | * This file is part of GlusterFS. 4 | * 5 | * Licensed under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 14 | * implied. See the License for the specific language governing 15 | * permissions and limitations under the License. 16 | * 17 | */ 18 | 19 | /** 20 | * Implements the Hadoop FileSystem Interface to allow applications to store 21 | * files on GlusterFS and run Map/Reduce jobs on the data. CRC are performed on files. 22 | * 23 | * gluster file systems are specified with the glusterfs:// prefix. 24 | * 25 | * 26 | */ 27 | package org.apache.hadoop.fs.glusterfs; 28 | 29 | import java.io.IOException; 30 | 31 | import org.apache.hadoop.conf.Configuration; 32 | import org.apache.hadoop.fs.FileSystem; 33 | import org.apache.hadoop.fs.FileUtil; 34 | import org.apache.hadoop.fs.LocalFileSystem; 35 | import org.apache.hadoop.fs.Path; 36 | import org.slf4j.Logger; 37 | import org.slf4j.LoggerFactory; 38 | 39 | 40 | public class GlusterFileSystemCRC extends LocalFileSystem{ 41 | 42 | protected static final Logger log = LoggerFactory.getLogger(GlusterFileSystemCRC.class); 43 | 44 | public GlusterFileSystemCRC(){ 45 | super(new GlusterVolume()); 46 | Version v = new Version(); 47 | log.info("GIT INFO="+v); 48 | log.info("GIT_TAG="+v.getTag()); 49 | } 50 | 51 | public void setConf(Configuration conf){ 52 | log.info("Initializing GlusterFS with CRC."); 53 | super.setConf(conf); 54 | } 55 | 56 | /* 57 | * if GlusterFileSystem is the default filesystem, real local URLs come back without a file:/ scheme name (BUG!). the glusterfs 58 | * file system is assumed. force a schema. 59 | */ 60 | @Override 61 | public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException { 62 | FileSystem srcFs = new Path("file:/" + src.toString()).getFileSystem(getConf()); 63 | FileSystem dstFs = dst.getFileSystem(getConf()); 64 | FileUtil.copy(srcFs, src, dstFs, dst, delSrc, getConf()); 65 | } 66 | 67 | @Override 68 | public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException { 69 | FileSystem srcFs = src.getFileSystem(getConf()); 70 | FileSystem dstFs = new Path("file:/" + dst.toString()).getFileSystem(getConf()); 71 | FileUtil.copy(srcFs, src, dstFs, dst, delSrc, getConf()); 72 | } 73 | 74 | public String toString(){ 75 | return "Gluster File System - CRC Enabled"; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/org/apache/hadoop/fs/glusterfs/GlusterVolume.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Copyright (c) 2011 Red Hat, Inc. 4 | * This file is part of GlusterFS. 5 | * 6 | * Licensed under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 15 | * implied. See the License for the specific language governing 16 | * permissions and limitations under the License. 17 | * 18 | * Extends the RawLocalFileSystem to add support for Gluster Volumes. 19 | * 20 | */ 21 | 22 | package org.apache.hadoop.fs.glusterfs; 23 | 24 | 25 | import java.io.File; 26 | import java.io.FileNotFoundException; 27 | import java.io.IOException; 28 | import java.net.URI; 29 | import java.util.Arrays; 30 | import java.util.Enumeration; 31 | import java.util.Hashtable; 32 | import java.util.Vector; 33 | 34 | import org.apache.hadoop.conf.Configuration; 35 | import org.apache.hadoop.fs.BlockLocation; 36 | import org.apache.hadoop.fs.FileAlreadyExistsException; 37 | import org.apache.hadoop.fs.FileStatus; 38 | import org.apache.hadoop.fs.FileUtil; 39 | import org.apache.hadoop.fs.Path; 40 | import org.apache.hadoop.fs.RawLocalFileSystem; 41 | import org.apache.hadoop.fs.permission.FsPermission; 42 | import org.slf4j.Logger; 43 | import org.slf4j.LoggerFactory; 44 | 45 | public class GlusterVolume extends RawLocalFileSystem{ 46 | 47 | static final Logger log = LoggerFactory.getLogger(GlusterVolume.class); 48 | 49 | /** 50 | * General reason for these constants is to help us decide 51 | * when to override the specified buffer size. See implementation 52 | * of logic below, which might change overtime. 53 | */ 54 | public static final int OVERRIDE_WRITE_BUFFER_SIZE = 1024 * 4; 55 | public static final int OPTIMAL_WRITE_BUFFER_SIZE = 1024 * 128; 56 | public static final int DEFAULT_BLOCK_SIZE = 64 * 1024 * 1024; 57 | 58 | protected URI NAME = null; 59 | 60 | protected Hashtable volumes=new Hashtable(); 61 | protected String default_volume = null; 62 | protected boolean sortDirectoryListing = false; 63 | protected int tsPrecisionChop; 64 | 65 | protected static GlusterFSXattr attr = null; 66 | 67 | public GlusterVolume(){} 68 | 69 | public GlusterVolume(Configuration conf){ 70 | this(); 71 | this.setConf(conf); 72 | } 73 | 74 | public URI getUriOrCreate(URI u) { 75 | if(NAME==null) 76 | return URI.create("glusterfs:///"); 77 | return NAME; 78 | } 79 | 80 | public URI getUri() { 81 | return getUriOrCreate(NAME); 82 | } 83 | 84 | public void initialize(URI uri, Configuration conf) throws IOException { 85 | /* we only really care about the URI up to the path, so strip other things off */ 86 | String auth = uri.getAuthority(); 87 | if(auth==null) 88 | auth = ""; 89 | this.NAME = URI.create(uri.getScheme() + "://" + auth + "/") ; 90 | super.initialize(this.NAME, conf); 91 | } 92 | 93 | /* 94 | * This expands URIs to include 'default' values. 95 | * For instance, glusterfs:///path would expand to glusterfs://defaultvolume/path 96 | */ 97 | protected URI canonicalizeUri(URI uri) { 98 | String auth = uri.getAuthority(); 99 | if(auth==null) 100 | auth = default_volume; 101 | return URI.create(uri.getScheme() + "://" + auth + "/" + uri.getPath()) ; 102 | } 103 | 104 | /* check if a path is on the same volume as this instance */ 105 | public boolean sameVolume(Path p){ 106 | URI thisUri = this.NAME; 107 | URI thatUri = p.toUri(); 108 | if(!thisUri.getScheme().equalsIgnoreCase(thatUri.getScheme())) return false; 109 | if((thisUri.getAuthority()==null && thatUri.getAuthority()==null)) return true; 110 | return (thatUri.getAuthority()!=null && thatUri.getAuthority().equalsIgnoreCase(thisUri.getAuthority())); 111 | } 112 | 113 | public void setConf(Configuration conf){ 114 | log.info("Initializing gluster volume.."); 115 | super.setConf(conf); 116 | 117 | if(conf!=null){ 118 | 119 | try{ 120 | String r=conf.get("fs.glusterfs.volumes", ""); 121 | 122 | if("".equals(r)){ 123 | log.error("fs.glusterfs.volumes not defined."); 124 | throw new RuntimeException("Error loading gluster configuration.. fs.glusterfs.volumes not defined."); 125 | } 126 | String[] v=r.split(","); 127 | 128 | default_volume = v[0]; 129 | for(int i=0;i all = volumes.keys(); 255 | String rawPath = path.getAbsolutePath(); 256 | 257 | String volume = null; 258 | String root = null; 259 | 260 | while(volume==null && all.hasMoreElements()){ 261 | String nextVolume = all.nextElement(); 262 | String nextPath = volumes.get(nextVolume); 263 | if(rawPath.startsWith(nextPath)){ 264 | volume = nextVolume; 265 | root = nextPath; 266 | } 267 | } 268 | 269 | if(volume==null){ 270 | throw new RuntimeException("No volume matching path: " + path); 271 | } 272 | 273 | if(default_volume.equalsIgnoreCase(volume)) 274 | volume = ""; 275 | return new Path("glusterfs://" + volume + "/" + rawPath.substring(root.length())); 276 | } 277 | 278 | public boolean rename(Path src, Path dst) throws IOException { 279 | File dest = pathToFile(dst); 280 | 281 | /* two HCFS semantics java.io.File doesn't honor */ 282 | if(dest.exists() && dest.isFile() || !(new File(dest.getParent()).exists())) return false; 283 | 284 | if (!dest.exists() && pathToFile(src).renameTo(dest)) { 285 | return true; 286 | } 287 | return FileUtil.copy(this, src, this, dst, true, getConf()); 288 | } 289 | /** 290 | * Delete the given path to a file or directory. 291 | * @param p the path to delete 292 | * @param recursive to delete sub-directories 293 | * @return true if the file or directory and all its contents were deleted 294 | * @throws IOException if p is non-empty and recursive is false 295 | */ 296 | @Override 297 | public boolean delete(Path p, boolean recursive) throws IOException { 298 | File f = pathToFile(p); 299 | if(!f.exists()){ 300 | /* HCFS semantics expect 'false' if attempted file deletion on non existent file */ 301 | return false; 302 | }else if (f.isFile()) { 303 | return f.delete(); 304 | } else if (!recursive && f.isDirectory() && 305 | (FileUtil.listFiles(f).length != 0)) { 306 | throw new IOException("Directory " + f.toString() + " is not empty"); 307 | } 308 | return FileUtil.fullyDelete(f); 309 | } 310 | 311 | public boolean mkdirs(Path f) throws IOException { 312 | if(f == null) { 313 | throw new IllegalArgumentException("mkdirs path arg is null"); 314 | } 315 | 316 | f = f.makeQualified(this); 317 | 318 | return super.mkdirs(f); 319 | } 320 | 321 | public FileStatus[] listStatus(Path f) throws IOException { 322 | File localf = pathToFile(f); 323 | Vector results = new Vector(); 324 | 325 | 326 | if (!localf.exists()) { 327 | throw new FileNotFoundException("File " + f + " does not exist"); 328 | } 329 | if (localf.isFile()) { 330 | return new FileStatus[] { 331 | new GlusterFileStatus(localf, getDefaultBlockSize(), this) }; 332 | } 333 | 334 | if(localf.isDirectory() && !localf.canRead()){ 335 | throw new IOException("Access denied : " + f.toString()); 336 | } 337 | 338 | File[] names = localf.listFiles(); 339 | if (names == null) { 340 | return null; 341 | } 342 | 343 | for (int i = 0; i < names.length; i++) { 344 | try { 345 | FileStatus listing = getFileStatus(fileToPath(names[i])); 346 | if(sortDirectoryListing){ 347 | int j; 348 | for(j=0;j0){ 351 | results.insertElementAt(listing,j); 352 | break; 353 | } 354 | 355 | } 356 | if(results.size()==j) 357 | results.add(listing); 358 | }else{ 359 | results.add(listing); 360 | } 361 | 362 | } catch (FileNotFoundException e) { 363 | log.info("ignoring invisible path : " + names[i]); 364 | } 365 | } 366 | 367 | return results.toArray(new FileStatus[results.size()]); 368 | 369 | } 370 | 371 | public FileStatus getFileStatus(Path f) throws IOException { 372 | 373 | File path = null; 374 | 375 | try{ 376 | path = pathToFile(f); 377 | }catch(IllegalArgumentException ex){ 378 | throw new FileNotFoundException( "File " + f + " does not exist on this volume." + ex); 379 | } 380 | 381 | if (path.exists()) { 382 | return new GlusterFileStatus(pathToFile(f), getDefaultBlockSize(), this); 383 | } else { 384 | throw new FileNotFoundException( "File " + f + " does not exist."); 385 | } 386 | } 387 | 388 | public long getBlockSize(Path path) throws IOException{ 389 | return getLength(path); 390 | } 391 | 392 | public void setOwner(Path p, String username, String groupname) 393 | throws IOException { 394 | super.setOwner(p,username,groupname); 395 | 396 | } 397 | 398 | public void setPermission(Path p, FsPermission permission) 399 | throws IOException { 400 | super.setPermission(p,permission); 401 | } 402 | 403 | public BlockLocation[] getFileBlockLocations(FileStatus file,long start,long len) throws IOException{ 404 | File f=pathToFile(file.getPath()); 405 | BlockLocation[] result = new GlusterFSXattr(f.getPath()).getPathInfo(start, len); 406 | if(result==null){ 407 | log.info("GLUSTERFS: Problem getting host/block location for file "+f.getPath()); 408 | } 409 | 410 | return result; 411 | } 412 | 413 | public String toString(){ 414 | return "Gluster volume: " + this.NAME; 415 | } 416 | 417 | } 418 | -------------------------------------------------------------------------------- /src/main/java/org/apache/hadoop/fs/glusterfs/Util.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Copyright (c) 2013 Red Hat, Inc. 4 | * This file is part of GlusterFS. 5 | * 6 | * Licensed under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 15 | * implied. See the License for the specific language governing 16 | * permissions and limitations under the License. 17 | * 18 | * Implements the Hadoop FileSystem Interface to allow applications to store 19 | * files on GlusterFS and run Map/Reduce jobs on the data. 20 | * 21 | * 22 | */ 23 | 24 | package org.apache.hadoop.fs.glusterfs; 25 | 26 | import java.io.File; 27 | import java.io.IOException; 28 | 29 | import org.apache.hadoop.fs.FileUtil; 30 | import org.apache.hadoop.util.Shell; 31 | 32 | public class Util{ 33 | 34 | public static String execCommand(File f,String...cmd) throws IOException{ 35 | String[] args=new String[cmd.length+1]; 36 | System.arraycopy(cmd, 0, args, 0, cmd.length); 37 | args[cmd.length]=FileUtil.makeShellPath(f, true); 38 | String output=Shell.execCommand(args); 39 | return output; 40 | } 41 | 42 | /* copied from unstalbe hadoop API org.apache.hadoop.Shell */ 43 | public static String[] getGET_PERMISSION_COMMAND(){ 44 | // force /bin/ls, except on windows. 45 | return new String[]{(WINDOWS ? "ls" : "/bin/ls"),"-ld"}; 46 | } 47 | 48 | /* copied from unstalbe hadoop API org.apache.hadoop.Shell */ 49 | 50 | public static final boolean WINDOWS /* borrowed from Path.WINDOWS */ 51 | =System.getProperty("os.name").startsWith("Windows"); 52 | // / loads permissions, owner, and group from `ls -ld` 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/org/apache/hadoop/fs/glusterfs/Version.java: -------------------------------------------------------------------------------- 1 | package org.apache.hadoop.fs.glusterfs; 2 | 3 | import java.io.IOException; 4 | 5 | import java.util.Properties; 6 | 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | /** 11 | * Versioning stuff for the shim. This class is not tested since there is no 12 | * deterministic behaviour (i.e. it might not work if not building from binary), 13 | * and the effects are pure side effects. 14 | */ 15 | public class Version extends Properties{ 16 | static final Logger LOG = LoggerFactory.getLogger(Version.class); 17 | public Version() { 18 | super(); 19 | try{ 20 | load(this.getClass().getClassLoader().getResourceAsStream("git.properties")); 21 | } 22 | catch(Throwable t){ 23 | LOG.warn("Couldn't find GIT properties for version info " + 24 | t +". This jar may have been built OUTSIDE a GIT repo."); 25 | } 26 | } 27 | public String getTag(){ 28 | String commit = this.getProperty("git.commit.id.describe"); 29 | final String tag = commit != null ? 30 | commit.split("-")[0]: 31 | "no version info available. check log warnings."; 32 | 33 | return tag; 34 | } 35 | 36 | /** 37 | * For use with terminal version checking. 38 | 39 | Example, run with an argument to get single property: 40 | java -cp /home/Development/hadoop-glusterfs/glusterfs-2.0-SNAPSHOT.jar \ 41 | org.apache.hadoop.fs.glusterfs.Version git.commit.id.describe | cut -d'-' -f 1 42 | 43 | Or just run (no args, prints all properties) 44 | java -cp /home/Development/hadoop-glusterfs/glusterfs-2.0-SNAPSHOT.jar \ 45 | */ 46 | public static void main(String[] args){ 47 | Version v = new Version(); 48 | //Dump the whole version info if no arg 49 | if(args.length==0){ 50 | System.out.println(v); 51 | } 52 | //if specific arg given, print just that. 53 | else{ 54 | String prop = v.get(args[0])+""; 55 | System.out.println( 56 | prop!=null? 57 | prop 58 | :"Couldnt find property "+prop); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/org/apache/hadoop/fs/local/GlusterFs.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Copyright (c) 2013 Red Hat, Inc. 4 | * This file is part of GlusterFS. 5 | * 6 | * Licensed under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 15 | * implied. See the License for the specific language governing 16 | * permissions and limitations under the License. 17 | * 18 | */ 19 | 20 | /** 21 | * Implements the Hadoop FileSystem 2.x Interface to allow applications to store 22 | * files on GlusterFS and run Map/Reduce jobs on the data. This code does not perform a CRC 23 | * on the files. 24 | * 25 | * gluster file systems are specified with the glusterfs:// prefix. 26 | * 27 | * 28 | */ 29 | 30 | package org.apache.hadoop.fs.local; 31 | 32 | import java.io.IOException; 33 | import java.net.URI; 34 | import java.net.URISyntaxException; 35 | 36 | import org.apache.hadoop.conf.Configuration; 37 | import org.apache.hadoop.fs.FilterFs; 38 | 39 | public class GlusterFs extends FilterFs{ 40 | 41 | GlusterFs(Configuration conf) throws IOException, URISyntaxException{ 42 | super(new GlusterVol(conf)); 43 | } 44 | 45 | GlusterFs(final URI theUri, final Configuration conf) throws IOException, URISyntaxException{ 46 | this(conf); 47 | } 48 | 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/org/apache/hadoop/fs/local/GlusterFsCRC.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Copyright (c) 2013 Red Hat, Inc. 4 | * This file is part of GlusterFS. 5 | * 6 | * Licensed under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 15 | * implied. See the License for the specific language governing 16 | * permissions and limitations under the License. 17 | * 18 | */ 19 | 20 | /** 21 | * Implements the Hadoop FileSystem 2.x Interface to allow applications to store 22 | * files on GlusterFS and run Map/Reduce jobs on the data. This code does perform a CRC 23 | * on the files. 24 | * 25 | * gluster file systems are specified with the glusterfs:// prefix. 26 | * 27 | * 28 | */ 29 | 30 | package org.apache.hadoop.fs.local; 31 | 32 | import java.io.IOException; 33 | import java.net.URI; 34 | import java.net.URISyntaxException; 35 | 36 | import org.apache.hadoop.conf.Configuration; 37 | import org.apache.hadoop.fs.ChecksumFs; 38 | 39 | public class GlusterFsCRC extends ChecksumFs{ 40 | 41 | GlusterFsCRC(Configuration conf) throws IOException, URISyntaxException{ 42 | super(new GlusterVol(conf)); 43 | } 44 | 45 | GlusterFsCRC(final URI theUri, final Configuration conf) throws IOException, URISyntaxException{ 46 | this(conf); 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/org/apache/hadoop/fs/local/GlusterVol.java: -------------------------------------------------------------------------------- 1 | package org.apache.hadoop.fs.local; 2 | 3 | import java.io.IOException; 4 | import java.net.URI; 5 | import java.net.URISyntaxException; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | import org.apache.hadoop.fs.FsConstants; 9 | import org.apache.hadoop.fs.Path; 10 | import org.apache.hadoop.fs.RawLocalFileSystem; 11 | import org.apache.hadoop.fs.glusterfs.GlusterFileSystemCRC; 12 | import org.apache.hadoop.fs.glusterfs.GlusterVolume; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | public class GlusterVol extends RawLocalFsG{ 17 | 18 | protected static final Logger log = LoggerFactory.getLogger(GlusterFileSystemCRC.class); 19 | 20 | public static final URI DEFAULT_URI=URI.create("glusterfs:///"); 21 | 22 | GlusterVol(final Configuration conf) throws IOException, URISyntaxException { 23 | this(DEFAULT_URI, conf); 24 | 25 | } 26 | 27 | /** 28 | * This constructor has the signature needed by 29 | * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}. 30 | * 31 | * @param theUri which must be that of localFs 32 | * @param conf 33 | * @throws IOException 34 | * @throws URISyntaxException 35 | */ 36 | GlusterVol(final URI theUri, final Configuration conf) throws IOException, URISyntaxException { 37 | super(theUri, new GlusterVolume(), conf, false); 38 | } 39 | 40 | 41 | 42 | 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/org/apache/hadoop/fs/local/RawLocalFsG.java: -------------------------------------------------------------------------------- 1 | package org.apache.hadoop.fs.local; 2 | 3 | import java.io.FileNotFoundException; 4 | import java.io.IOException; 5 | import java.net.URI; 6 | import java.net.URISyntaxException; 7 | 8 | import org.apache.hadoop.conf.Configuration; 9 | import org.apache.hadoop.fs.DelegateToFileSystem; 10 | import org.apache.hadoop.fs.FileStatus; 11 | import org.apache.hadoop.fs.FileSystem; 12 | import org.apache.hadoop.fs.FsConstants; 13 | import org.apache.hadoop.fs.FsServerDefaults; 14 | import org.apache.hadoop.fs.Path; 15 | import org.apache.hadoop.fs.RawLocalFileSystem; 16 | import org.apache.hadoop.fs.permission.FsPermission; 17 | import org.apache.hadoop.util.Shell; 18 | 19 | public class RawLocalFsG extends DelegateToFileSystem{ 20 | 21 | /* 22 | * need a constructor on apache's RawLocalFileSystem that takes FileSystem object 23 | * 24 | */ 25 | 26 | RawLocalFsG(final URI theUri, final FileSystem fs, final Configuration conf,boolean authorityRequired) throws IOException, URISyntaxException{ 27 | super(theUri, fs, conf, theUri.getScheme(), authorityRequired); 28 | } 29 | 30 | 31 | 32 | /* ------------------- below this line is c + p from RawLocalFs.. --------------------------*/ 33 | RawLocalFsG(final Configuration conf) throws IOException, URISyntaxException { 34 | this(FsConstants.LOCAL_FS_URI, conf); 35 | } 36 | 37 | /** 38 | * This constructor has the signature needed by 39 | * {@link AbstractFileSystem#createFileSystem(URI, Configuration)}. 40 | * 41 | * @param theUri which must be that of localFs 42 | * @param conf 43 | * @throws IOException 44 | * @throws URISyntaxException 45 | */ 46 | RawLocalFsG(final URI theUri, final Configuration conf) throws IOException, 47 | URISyntaxException { 48 | super(theUri, new RawLocalFileSystem(), conf, 49 | FsConstants.LOCAL_FS_URI.getScheme(), false); 50 | } 51 | 52 | 53 | @Override 54 | public int getUriDefaultPort() { 55 | return -1; // No default port for file:/// 56 | } 57 | 58 | @Override 59 | public FsServerDefaults getServerDefaults() throws IOException { 60 | return LocalConfigKeys.getServerDefaults(); 61 | } 62 | 63 | @Override 64 | public boolean supportsSymlinks() { 65 | return true; 66 | } 67 | 68 | @Override 69 | public void createSymlink(Path target, Path link, boolean createParent) 70 | throws IOException { 71 | final String targetScheme = target.toUri().getScheme(); 72 | if (targetScheme != null && !"file".equals(targetScheme)) { 73 | throw new IOException("Unable to create symlink to non-local file "+ 74 | "system: "+target.toString()); 75 | } 76 | if (createParent) { 77 | mkdir(link.getParent(), FsPermission.getDirDefault(), true); 78 | } 79 | // NB: Use createSymbolicLink in java.nio.file.Path once available 80 | try { 81 | Shell.execCommand(Shell.LINK_COMMAND, "-s", 82 | new URI(target.toString()).getPath(), 83 | new URI(link.toString()).getPath()); 84 | } catch (URISyntaxException x) { 85 | throw new IOException("Invalid symlink path: "+x.getMessage()); 86 | } catch (IOException x) { 87 | throw new IOException("Unable to create symlink: "+x.getMessage()); 88 | } 89 | } 90 | 91 | /** 92 | * Returns the target of the given symlink. Returns the empty string if 93 | * the given path does not refer to a symlink or there is an error 94 | * acessing the symlink. 95 | */ 96 | private String readLink(Path p) { 97 | /* NB: Use readSymbolicLink in java.nio.file.Path once available. Could 98 | * use getCanonicalPath in File to get the target of the symlink but that 99 | * does not indicate if the given path refers to a symlink. 100 | */ 101 | try { 102 | final String path = p.toUri().getPath(); 103 | return Shell.execCommand(Shell.READ_LINK_COMMAND, path).trim(); 104 | } catch (IOException x) { 105 | return ""; 106 | } 107 | } 108 | 109 | /** 110 | * Return a FileStatus representing the given path. If the path refers 111 | * to a symlink return a FileStatus representing the link rather than 112 | * the object the link refers to. 113 | */ 114 | @Override 115 | public FileStatus getFileLinkStatus(final Path f) throws IOException { 116 | String target = readLink(f); 117 | try { 118 | FileStatus fs = getFileStatus(f); 119 | // If f refers to a regular file or directory 120 | if ("".equals(target)) { 121 | return fs; 122 | } 123 | // Otherwise f refers to a symlink 124 | return new FileStatus(fs.getLen(), 125 | false, 126 | fs.getReplication(), 127 | fs.getBlockSize(), 128 | fs.getModificationTime(), 129 | fs.getAccessTime(), 130 | fs.getPermission(), 131 | fs.getOwner(), 132 | fs.getGroup(), 133 | new Path(target), 134 | f); 135 | } catch (FileNotFoundException e) { 136 | /* The exists method in the File class returns false for dangling 137 | * links so we can get a FileNotFoundException for links that exist. 138 | * It's also possible that we raced with a delete of the link. Use 139 | * the readBasicFileAttributes method in java.nio.file.attributes 140 | * when available. 141 | */ 142 | if (!"".equals(target)) { 143 | return new FileStatus(0, false, 0, 0, 0, 0, FsPermission.getDefault(), 144 | "", "", new Path(target), f); 145 | } 146 | // f refers to a file or directory that does not exist 147 | throw e; 148 | } 149 | } 150 | 151 | @Override 152 | public Path getLinkTarget(Path f) throws IOException { 153 | /* We should never get here. Valid local links are resolved transparently 154 | * by the underlying local file system and accessing a dangling link will 155 | * result in an IOException, not an UnresolvedLinkException, so FileContext 156 | * should never call this function. 157 | */ 158 | throw new AssertionError(); 159 | } 160 | 161 | } 162 | -------------------------------------------------------------------------------- /src/main/java/org/apache/hadoop/yarn/server/nodemanager/GlusterContainerExecutor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.hadoop.yarn.server.nodemanager; 20 | 21 | import org.apache.hadoop.fs.Path; 22 | import org.slf4j.Logger; 23 | public class GlusterContainerExecutor extends LinuxContainerExecutor { 24 | 25 | static Logger log = org.slf4j.LoggerFactory.getLogger(GlusterContainerExecutor.class); 26 | /** 27 | * We override the YARN-1235 patch here. 28 | * @param user 29 | * @return user name 30 | */ 31 | @Override 32 | String getRunAsUser(String user) { 33 | log.info("Container EXEC overrid: returning user " + user); 34 | //return UserGroupInformation.isSecurityEnabled() ? user : nonsecureLocalUser; 35 | return user; 36 | } 37 | 38 | @Override 39 | public void deleteAsUser(String arg0,Path arg1,Path...arg2){ 40 | log.info("DELETE AS USER " + arg0 + " " + arg1 +" " + arg2); 41 | super.deleteAsUser(arg0, arg1, arg2); 42 | } 43 | } 44 | 45 | -------------------------------------------------------------------------------- /src/test/java/org/apache/hadoop/fs/test/GlusterVolumeTest.java: -------------------------------------------------------------------------------- 1 | package org.apache.hadoop.fs.test; 2 | 3 | import java.io.IOException; 4 | import junit.framework.Assert; 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.junit.Test; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.apache.hadoop.fs.glusterfs.*; 10 | /** 11 | * This class is for tests which are gluster specific. 12 | */ 13 | public class GlusterVolumeTest { 14 | 15 | Logger log = LoggerFactory.getLogger(GlusterVolumeTest.class); 16 | static Configuration config = null; 17 | 18 | @org.junit.Test 19 | public void testNullURI() { 20 | Assert.assertNotNull(new org.apache.hadoop.fs.glusterfs.GlusterVolume().getUriOrCreate(null)); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/test/java/org/apache/hadoop/fs/test/TestBlockLocation.java: -------------------------------------------------------------------------------- 1 | package org.apache.hadoop.fs.test; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import java.io.IOException; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | import org.apache.hadoop.fs.BlockLocation; 9 | import org.apache.hadoop.fs.FileSystem; 10 | import org.apache.hadoop.fs.Path; 11 | import org.apache.hadoop.fs.glusterfs.GlusterFSXattr; 12 | import org.apache.hadoop.fs.test.connector.HcfsTestConnectorFactory; 13 | import org.apache.hadoop.fs.test.connector.HcfsTestConnectorInterface; 14 | import org.junit.BeforeClass; 15 | import org.junit.Test; 16 | 17 | public class TestBlockLocation { 18 | 19 | /* 20 | * Test subclass to simulate / inject xattr values instead of requiring 21 | * filesystem. 22 | * 23 | */ 24 | class TestBlockLocationHarnace extends GlusterFSXattr { 25 | String xattrValue = null; 26 | 27 | TestBlockLocationHarnace(String xattrValue) { 28 | super(null); 29 | this.xattrValue = xattrValue; 30 | } 31 | 32 | public String execGetFattr() throws IOException { 33 | return this.xattrValue; 34 | } 35 | } 36 | 37 | @Test 38 | public void testReplicateVolume() throws IOException { 39 | String xattr = "trusted.glusterfs.pathinfo=\"( ( )\""; 40 | xattr = xattr.substring(28, xattr.length() - 1); 41 | TestBlockLocationHarnace tbl = new TestBlockLocationHarnace(xattr); 42 | 43 | int start = 0; 44 | int len = 0; 45 | BlockLocation[] blocks = tbl.getPathInfo(start, len); 46 | assertTrue(blocks.length == 1); 47 | assertTrue(blocks[0].getHosts()[0].equals("vm-1")); 48 | } 49 | 50 | @Test 51 | public void testReplicateVolume2() throws IOException { 52 | String xattr = "trusted.glusterfs.pathinfo=\"( ( ))\""; 53 | xattr = xattr.substring(28, xattr.length() - 1); 54 | TestBlockLocationHarnace tbl = new TestBlockLocationHarnace(xattr); 55 | BlockLocation[] blocks = tbl.getPathInfo(0, 10000); 56 | 57 | assertTrue(blocks.length == 1); 58 | String hosts[] = blocks[0].getHosts(); 59 | 60 | assertTrue(hosts[0].equals("vm-2")); 61 | assertTrue(hosts[1].equals("vm-1")); 62 | } 63 | 64 | /* 65 | * Make sure the xattr code fails gracefull when the input is garbage. 66 | */ 67 | @Test 68 | public void testFailureVolumeGarbage() { 69 | String xattr = "dsfsffsfdsf"; 70 | TestBlockLocationHarnace tbl = new TestBlockLocationHarnace(xattr); 71 | BlockLocation[] blocks = tbl.getPathInfo(0, 10000); 72 | assertNull(blocks); 73 | } 74 | 75 | /* 76 | * Make sure the xattr code fails gracefully when the input is empty. 77 | */ 78 | 79 | @Test 80 | public void testFailureVolumeEmpty() { 81 | String xattr = ""; 82 | TestBlockLocationHarnace tbl = new TestBlockLocationHarnace(xattr); 83 | BlockLocation[] blocks = tbl.getPathInfo(0, 10000); 84 | assertNull(blocks); 85 | } 86 | 87 | /* 88 | * 89 | * stripe not yet supported.trusted.pathinfo is incorrect in gluster: 90 | * https://bugzilla.redhat.com/show_bug.cgi?id=1200914 91 | */ 92 | 93 | public void testStripeVolume() { 94 | String xattr = "trusted.glusterfs.pathinfo=\"( ( ( )( )( ))\""; 95 | ; 96 | xattr = xattr.substring(28, xattr.length() - 1); 97 | TestBlockLocationHarnace tbl = new TestBlockLocationHarnace(xattr); 98 | BlockLocation[] blocks = tbl.getPathInfo(0, 10000); 99 | 100 | } 101 | 102 | } 103 | -------------------------------------------------------------------------------- /src/test/java/org/apache/hadoop/fs/test/TimestampPrecision.java: -------------------------------------------------------------------------------- 1 | package org.apache.hadoop.fs.test; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | 5 | import java.io.IOException; 6 | import java.net.URI; 7 | import java.net.URISyntaxException; 8 | 9 | import org.apache.hadoop.conf.Configuration; 10 | import org.apache.hadoop.fs.glusterfs.GlusterVolume; 11 | import org.junit.Test; 12 | 13 | public class TimestampPrecision { 14 | 15 | @Test 16 | public void testSimpleTruncate0() throws IOException, URISyntaxException{ 17 | Configuration config = new Configuration(); 18 | config.setInt("fs.glusterfs.timestamp.trim", 0); 19 | GlusterVolume fs = new GlusterVolume(); 20 | fs.initialize(new URI("glusterfs:///"), config); 21 | 22 | long value1 = 999911111; 23 | long trim = fs.trimTimestamp(value1); 24 | assertEquals(trim,value1); 25 | 26 | } 27 | 28 | 29 | @Test 30 | public void testSimpleTruncate1() throws IOException, URISyntaxException{ 31 | Configuration config = new Configuration(); 32 | config.setInt("fs.glusterfs.timestamp.trim", 1); 33 | GlusterVolume fs = new GlusterVolume(); 34 | fs.initialize(new URI("glusterfs:///"), config); 35 | 36 | long value1 = 999911111; 37 | long trim = fs.trimTimestamp(value1); 38 | assertEquals(trim,999911110 ); 39 | 40 | } 41 | 42 | @Test 43 | public void testSimpleTruncate2() throws IOException, URISyntaxException{ 44 | Configuration config = new Configuration(); 45 | config.setInt("fs.glusterfs.timestamp.trim", 2); 46 | GlusterVolume fs = new GlusterVolume(); 47 | fs.initialize(new URI("glusterfs:///"), config); 48 | 49 | long value1 = 999911111; 50 | long trim = fs.trimTimestamp(value1); 51 | assertEquals(trim,999911100 ); 52 | 53 | } 54 | @Test 55 | public void testSimpleTruncate3() throws IOException, URISyntaxException{ 56 | Configuration config = new Configuration(); 57 | config.setInt("fs.glusterfs.timestamp.trim", 3); 58 | GlusterVolume fs = new GlusterVolume(); 59 | fs.initialize(new URI("glusterfs:///"), config); 60 | 61 | long value1 = 999911111; 62 | long trim = fs.trimTimestamp(value1); 63 | assertEquals(trim,999911000 ); 64 | 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/test/java/org/apache/hadoop/fs/test/connector/HcfsTestConnector.java: -------------------------------------------------------------------------------- 1 | package org.apache.hadoop.fs.test.connector; 2 | 3 | import java.io.IOException; 4 | import java.io.InputStream; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.fs.FileSystem; 8 | 9 | 10 | /* 11 | * Generic HCFS file system test connector. 12 | * This test connector takes a fully qualified o.a.h.f.FileSystem implementor class 13 | * as an environment variable. 14 | * 15 | */ 16 | public class HcfsTestConnector implements HcfsTestConnectorInterface { 17 | 18 | public Configuration createConfiguration(){ 19 | Configuration c = new Configuration(); 20 | InputStream config = HcfsTestConnector.class.getClassLoader().getResourceAsStream("core-site.xml"); 21 | c.addResource(config); 22 | 23 | return c; 24 | } 25 | 26 | public FileSystem create() throws IOException{ 27 | return FileSystem.get(createConfiguration()); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/test/java/org/apache/hadoop/fs/test/connector/HcfsTestConnectorFactory.java: -------------------------------------------------------------------------------- 1 | package org.apache.hadoop.fs.test.connector; 2 | 3 | 4 | public class HcfsTestConnectorFactory { 5 | 6 | /* Loads an HCFS file system adapter via environment variable */ 7 | public static HcfsTestConnectorInterface getHcfsTestConnector() throws RuntimeException{ 8 | String testConnector = System.getProperty("HCFS_FILE_SYSTEM_CONNECTOR"); 9 | if(testConnector==null || "".equals(testConnector)){ 10 | testConnector = HcfsTestConnector.class.getCanonicalName(); 11 | } 12 | 13 | return getHcfsTestConnector(testConnector); 14 | } 15 | 16 | public static HcfsTestConnectorInterface getHcfsTestConnector(String hcfsName) throws RuntimeException{ 17 | try { 18 | return (HcfsTestConnectorInterface)Class.forName(hcfsName).newInstance(); 19 | } catch (Exception e) { 20 | throw new RuntimeException("Cannont instatiate HCFS File System from HCFS_FILE_SYSTEM env variable. Error:\n " + e); 21 | } 22 | 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/test/java/org/apache/hadoop/fs/test/connector/HcfsTestConnectorInterface.java: -------------------------------------------------------------------------------- 1 | package org.apache.hadoop.fs.test.connector; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.fs.FileSystem; 7 | 8 | 9 | /* generic interface for creating HCFS file sytems for testing purposes */ 10 | 11 | public interface HcfsTestConnectorInterface { 12 | 13 | /* return a fully configured instantiated file system for testing */ 14 | public FileSystem create() throws IOException; 15 | 16 | /* returns a configuration file with properties for a given FS */ 17 | public Configuration createConfiguration(); 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/test/java/org/apache/hadoop/fs/test/unit/GlusterMultipleVolumeTest.java: -------------------------------------------------------------------------------- 1 | package org.apache.hadoop.fs.test.unit; 2 | 3 | import java.io.IOException; 4 | 5 | import junit.framework.Assert; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | import org.apache.hadoop.fs.FileSystem; 9 | import org.apache.hadoop.fs.Path; 10 | import org.apache.hadoop.fs.test.connector.HcfsTestConnectorFactory; 11 | import org.apache.hadoop.fs.test.connector.HcfsTestConnectorInterface; 12 | import org.junit.After; 13 | import org.junit.AfterClass; 14 | import org.junit.BeforeClass; 15 | import org.junit.Test; 16 | import org.slf4j.Logger; 17 | import org.slf4j.LoggerFactory; 18 | 19 | public class GlusterMultipleVolumeTest { 20 | 21 | Logger log = LoggerFactory.getLogger(HCFSPerformanceIOTests.class); 22 | static Configuration config = null; 23 | 24 | 25 | @BeforeClass 26 | public static void setup() throws Exception { 27 | HcfsTestConnectorInterface connector = HcfsTestConnectorFactory.getHcfsTestConnector(); 28 | config = connector.createConfiguration(); 29 | } 30 | 31 | public static FileSystem getFileSystem(Path p) throws IOException{ 32 | return FileSystem.get(p.toUri(), config); 33 | } 34 | 35 | @Test 36 | public void testDefaultPath() throws IOException{ 37 | Path p1 = new Path("glusterfs:///test1.txt"); 38 | Path p2 = new Path("glusterfs://gv0/test1.txt"); 39 | Path p3 = new Path("glusterfs://gv1/test1.txt"); 40 | 41 | FileSystem fs1 = getFileSystem(p1); 42 | FileSystem fs2 = getFileSystem(p2); 43 | FileSystem fs3 = getFileSystem(p3); 44 | fs1.create(p1); 45 | 46 | /* the gluster volume and default is based on configuration settings, but expected that default = gv0 */ 47 | Assert.assertTrue(fs1.exists(p1)); 48 | Assert.assertTrue(fs2.exists(p2)); 49 | Assert.assertFalse(fs3.exists(p3)); 50 | 51 | fs1.delete(p1); 52 | } 53 | 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/org/apache/hadoop/fs/test/unit/HCFSPerformanceIOTests.java: -------------------------------------------------------------------------------- 1 | package org.apache.hadoop.fs.test.unit; 2 | 3 | import static org.apache.hadoop.fs.FileSystemTestHelper.getTestRootPath; 4 | 5 | import java.io.IOException; 6 | 7 | import org.apache.hadoop.fs.FSDataOutputStream; 8 | import org.apache.hadoop.fs.FileSystem; 9 | import org.apache.hadoop.fs.Path; 10 | import org.apache.hadoop.fs.glusterfs.GlusterVolume; 11 | import org.apache.hadoop.fs.test.connector.HcfsTestConnectorFactory; 12 | import org.apache.hadoop.fs.test.connector.HcfsTestConnectorInterface; 13 | import org.junit.After; 14 | import org.junit.AfterClass; 15 | import org.junit.Assert; 16 | import org.junit.BeforeClass; 17 | import org.junit.Test; 18 | import org.mortbay.log.Log; 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | /** 23 | * A class for performance and IO related unit tests. 24 | * 25 | * - Write buffering 26 | * - Read buffering 27 | * - Object caching / File lookup caching. 28 | * - Seeking 29 | */ 30 | public class HCFSPerformanceIOTests { 31 | 32 | static FileSystem fs ; 33 | Logger log = LoggerFactory.getLogger(HCFSPerformanceIOTests.class); 34 | 35 | @BeforeClass 36 | public static void setup() throws Exception { 37 | HcfsTestConnectorInterface connector = HcfsTestConnectorFactory.getHcfsTestConnector(); 38 | fs= connector.create(); 39 | } 40 | 41 | @AfterClass 42 | public static void after() throws IOException{ 43 | fs.close(); 44 | } 45 | 46 | public Path bufferoutpath(){ 47 | return getTestRootPath(fs, "buffering_test"+HCFSPerformanceIOTests.class.getName()); 48 | } 49 | 50 | @After 51 | public void tearDown() throws Exception { 52 | fs.delete(bufferoutpath(),true); 53 | } 54 | 55 | //String to append to file we are writing. 56 | static final String CONTENT="1234"; 57 | 58 | /** 59 | * This is a complex test. It documents the expected behaviour of the 60 | * FileSystem buffering. 61 | * 62 | * It assumes that the configuration value of FS is == the {@link GlusterVolume} OVERRIDE_WRITE_BUFFER_SIZE. 63 | * Then, it starts writing to a stream. 64 | */ 65 | @Test 66 | public void testBufferSpill() throws Exception { 67 | 68 | /** 69 | * Sanity check: This test expects that an override is being performed, i.e., that 70 | * the buffering is going to be set to the optimal size, because the file system 71 | * detected that the configured original buffer size was == to the "bad default" value which 72 | * we have decide to override, for the sack of "reasonable defaults" out of the box. 73 | */ 74 | Assert.assertEquals( 75 | GlusterVolume.OPTIMAL_WRITE_BUFFER_SIZE, 76 | fs.getConf().getInt("io.file.buffer.size",-1)); 77 | 78 | FSDataOutputStream os = fs.create(bufferoutpath()); 79 | 80 | int written=0; 81 | 82 | /** 83 | * Now, we assert that no data is spilled to disk until we reach the optimal size. 84 | */ 85 | while(written < GlusterVolume.OPTIMAL_WRITE_BUFFER_SIZE){ 86 | os.write(CONTENT.getBytes()); 87 | written+=CONTENT.getBytes().length; 88 | Assert.assertTrue("asserting that file not written yet...",fs.getLength(bufferoutpath())==0); 89 | } 90 | os.flush(); 91 | 92 | Assert.assertTrue("asserting that is now written... ",fs.getLength(bufferoutpath()) >= GlusterVolume.OPTIMAL_WRITE_BUFFER_SIZE); 93 | 94 | os.close(); 95 | } 96 | } -------------------------------------------------------------------------------- /src/test/java/org/apache/hadoop/fs/test/unit/HCFSTestWorkingDir.java: -------------------------------------------------------------------------------- 1 | package org.apache.hadoop.fs.test.unit; 2 | 3 | import static org.apache.hadoop.fs.FileSystemTestHelper.getTestRootPath; 4 | 5 | import java.io.File; 6 | 7 | import junit.framework.Assert; 8 | 9 | import org.apache.hadoop.fs.FileSystem; 10 | import org.apache.hadoop.fs.FileUtil; 11 | import org.apache.hadoop.fs.Path; 12 | import org.apache.hadoop.fs.test.connector.HcfsTestConnectorFactory; 13 | import org.apache.hadoop.fs.test.connector.HcfsTestConnectorInterface; 14 | import org.junit.After; 15 | import org.junit.Before; 16 | import org.junit.Test; 17 | import org.slf4j.Logger; 18 | import org.slf4j.LoggerFactory; 19 | 20 | public class HCFSTestWorkingDir{ 21 | 22 | Logger log = LoggerFactory.getLogger(HCFSTestWorkingDir.class); 23 | FileSystem fSys; 24 | 25 | @Before 26 | public void setUp() throws Exception{ 27 | HcfsTestConnectorInterface connector= 28 | HcfsTestConnectorFactory.getHcfsTestConnector(); 29 | fSys=connector.create(); 30 | } 31 | 32 | /** 33 | * This test fails in hadoop 1.2.0, if we do not have 34 | * logic in the GlusterVolume to use "getInitialWorkingDirectory" 35 | * as the starting working directory. 36 | */ 37 | @Test 38 | public void test() throws Exception { 39 | Path outpath = new Path("to_"); 40 | if(fSys.exists(outpath)){ 41 | fSys.delete(outpath,true); 42 | } 43 | File tmpfile = new File("/tmp/test_copyfromlocal"); 44 | tmpfile.createNewFile(); 45 | log.info(tmpfile.getAbsolutePath()); 46 | Assert.assertTrue(tmpfile.exists()); 47 | 48 | fSys.copyFromLocalFile(false, false, 49 | new Path(tmpfile.getPath()), 50 | outpath); 51 | 52 | Assert.assertTrue(fSys.exists(outpath)); 53 | fSys.delete(outpath,true); 54 | } 55 | 56 | @After 57 | public void tearDown() throws Exception{ 58 | fSys.delete(getTestRootPath(fSys, "test"), true); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/test/java/org/apache/hadoop/fs/test/unit/HcfsFileSystemTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Copyright (c) 2011 Gluster, Inc. 4 | * This file is part of GlusterFS. 5 | * 6 | * Licensed under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 15 | * implied. See the License for the specific language governing 16 | * permissions and limitations under the License. 17 | * 18 | * 19 | * Base test class for GlusterFS + hadoop testing. 20 | * Requires existing/working gluster volume named "hadoop-gluster". 21 | * 22 | * The default volume name can be overridden with env variable gluster-volume 23 | * 24 | */ 25 | 26 | package org.apache.hadoop.fs.test.unit; 27 | 28 | import static org.apache.hadoop.fs.FileSystemTestHelper.getTestRootPath; 29 | import static org.junit.Assert.assertEquals; 30 | import static org.junit.Assert.assertFalse; 31 | import static org.junit.Assert.assertTrue; 32 | 33 | import java.io.IOException; 34 | import java.util.Iterator; 35 | 36 | import org.apache.hadoop.conf.Configuration; 37 | import org.apache.hadoop.fs.FSDataInputStream; 38 | import org.apache.hadoop.fs.FSDataOutputStream; 39 | import org.apache.hadoop.fs.FileStatus; 40 | import org.apache.hadoop.fs.FileSystem; 41 | import org.apache.hadoop.fs.LocatedFileStatus; 42 | import org.apache.hadoop.fs.Path; 43 | import org.apache.hadoop.fs.RemoteIterator; 44 | import org.apache.hadoop.fs.permission.FsAction; 45 | import org.apache.hadoop.fs.permission.FsPermission; 46 | import org.apache.hadoop.fs.test.connector.HcfsTestConnectorFactory; 47 | import org.apache.hadoop.fs.test.connector.HcfsTestConnectorInterface; 48 | import org.junit.After; 49 | import org.junit.AfterClass; 50 | import org.junit.Assert; 51 | import org.junit.BeforeClass; 52 | import org.junit.Test; 53 | 54 | /** 55 | * Unit test for HCFS classes. 56 | * 57 | */ 58 | public class HcfsFileSystemTest{ 59 | 60 | static FileSystem fs ; 61 | 62 | 63 | /** 64 | * See MAPREDUCE-5902 for context on why this test is critical 65 | * for ecosystem interoperability. 66 | */ 67 | @org.junit.Test 68 | public void testEncodedPaths() throws Exception { 69 | //FileSystem fs2 = FileSystem.getLocal(new Configuration()); 70 | FileSystem fs2 = fs; 71 | Path encodedFiles=new Path("/tmp/encodedTest"+System.currentTimeMillis()); 72 | fs2.mkdirs(encodedFiles); 73 | fs2.create(new Path(encodedFiles,"a")); 74 | fs2.create(new Path(encodedFiles,"a%2")); 75 | fs2.create(new Path(encodedFiles,"a%2a")); 76 | fs2.create(new Path(encodedFiles,"a%3a")); 77 | fs2.create(new Path(encodedFiles,"a%4a")); 78 | Assert.assertEquals(5, fs2.listStatus(encodedFiles).length); 79 | fs2.delete(encodedFiles); 80 | } 81 | 82 | @BeforeClass 83 | public static void setup() throws Exception { 84 | HcfsTestConnectorInterface connector = HcfsTestConnectorFactory.getHcfsTestConnector(); 85 | fs= connector.create(); 86 | } 87 | 88 | @AfterClass 89 | public static void after() throws IOException{ 90 | fs.close(); 91 | } 92 | 93 | @After 94 | public void tearDown() throws Exception { 95 | fs.delete(getTestRootPath(fs, "test"),true); 96 | } 97 | 98 | @org.junit.Test 99 | public void testTolerantMkdirs() throws Exception{ 100 | Path longPath=new Path("a/b/c/d"); 101 | assertFalse(fs.exists(longPath)); 102 | fs.mkdirs(longPath); 103 | assertTrue(fs.exists(longPath)); 104 | fs.mkdirs(new Path("a")); 105 | assertTrue(fs.exists(longPath)); 106 | assertTrue(fs.exists(new Path("a"))); 107 | fs.mkdirs(new Path("a/b")); 108 | assertTrue(fs.exists(longPath)); 109 | assertTrue(fs.exists(new Path("a/b"))); 110 | fs.mkdirs(new Path("a/b/c")); 111 | assertTrue(fs.exists(longPath)); 112 | assertTrue(fs.exists(new Path("a/b/c"))); 113 | 114 | /* delete the directories */ 115 | 116 | fs.delete(new Path("a"), true); 117 | assertFalse(fs.exists(longPath)); 118 | 119 | } 120 | 121 | /** 122 | * BZ908898 : Test that confirms that ownership is preserved in gluster 123 | * FileStatus. 124 | */ 125 | @org.junit.Test 126 | public void testOwner() throws Exception{ 127 | final String me=System.getProperties().getProperty("user.name"); 128 | Path myFile=new Path("to_owned_by_me.txt"); 129 | fs.create(myFile); 130 | Assert.assertEquals(fs.getFileStatus(myFile).getOwner(), me); 131 | fs.delete(myFile); 132 | } 133 | 134 | @org.junit.Test 135 | public void testTextWriteAndRead() throws Exception{ 136 | 137 | String testString="Is there anyone out there?"; 138 | String readChars=null; 139 | 140 | FSDataOutputStream dfsOut=null; 141 | dfsOut=fs.create(new Path("test1.txt")); 142 | dfsOut.writeUTF(testString); 143 | dfsOut.close(); 144 | 145 | FSDataInputStream dfsin=null; 146 | 147 | dfsin=fs.open(new Path("test1.txt")); 148 | readChars=dfsin.readUTF(); 149 | dfsin.close(); 150 | 151 | assertEquals(testString, readChars); 152 | 153 | fs.delete(new Path("test1.txt"), true); 154 | 155 | assertFalse(fs.exists(new Path("test1"))); 156 | } 157 | 158 | @org.junit.Test 159 | public void testPermissions() throws Exception{ 160 | 161 | Path myFile=new Path("filePerm.txt"); 162 | fs.create(myFile); 163 | short perm=0777; 164 | fs.setPermission(myFile, new FsPermission(perm)); 165 | assertEquals(fs.getFileStatus(myFile).getPermission().toShort(), perm); 166 | 167 | perm=0700; 168 | fs.setPermission(myFile, new FsPermission(perm)); 169 | assertEquals(fs.getFileStatus(myFile).getPermission().toShort(), perm); 170 | 171 | fs.delete(myFile); 172 | assertFalse(fs.exists(myFile)); 173 | 174 | /* directory permissions */ 175 | Path directory = new Path("aa/bb/cc"); 176 | perm = 0700; 177 | fs.mkdirs(directory, new FsPermission(perm)); 178 | assertEquals(fs.getFileStatus(directory).getPermission().toShort(), perm); 179 | fs.delete(new Path("aa"),true); 180 | assertFalse(fs.exists(directory)); 181 | 182 | 183 | perm = 0777; 184 | fs.mkdirs(directory, new FsPermission(perm)); 185 | assertEquals(fs.getFileStatus(directory).getPermission().toShort(), perm); 186 | fs.delete(new Path("aa"),true); 187 | assertFalse(fs.exists(directory)); 188 | } 189 | 190 | @org.junit.Test 191 | public void testZDirs() throws Exception{ 192 | final Path subDir1=new Path("td_dir.1"); 193 | final Path baseDir=new Path("td_testDirs1"); 194 | final Path test1=new Path("td_test1"); 195 | final Path test2=new Path("td_test/dir.2"); 196 | 197 | assertFalse(fs.exists(baseDir)); 198 | assertFalse(fs.isDirectory(baseDir)); 199 | 200 | // make the dir 201 | fs.mkdirs(baseDir); 202 | 203 | assertTrue(fs.isDirectory(baseDir)); 204 | // fs.setWorkingDirectory(baseDir); 205 | 206 | fs.mkdirs(subDir1); 207 | 208 | assertTrue(fs.isDirectory(subDir1)); 209 | 210 | assertFalse(fs.exists(test1)); 211 | 212 | assertFalse(fs.isDirectory(test2)); 213 | 214 | fs.create(new Path(baseDir, "dummyfile")); 215 | FileStatus[] p=fs.listStatus(baseDir); 216 | assertEquals(p.length, 1); 217 | 218 | fs.delete(baseDir, true); 219 | assertFalse(fs.exists(baseDir)); 220 | 221 | fs.delete(subDir1, true); 222 | assertFalse(fs.exists(subDir1)); 223 | 224 | 225 | fs.delete(baseDir); 226 | fs.delete(test1); 227 | fs.delete(test2); 228 | } 229 | 230 | @org.junit.Test 231 | public void testFiles() throws Exception{ 232 | 233 | Path subDir1=new Path("tf_dir.1"); 234 | Path baseDir=new Path("tf_testDirs1"); 235 | Path file1=new Path("tf_dir.1/foo.1"); 236 | Path file2=new Path("tf_dir.1/foo.2"); 237 | 238 | fs.mkdirs(baseDir); 239 | assertTrue(fs.isDirectory(baseDir)); 240 | // fs.setWorkingDirectory(baseDir); 241 | 242 | fs.mkdirs(subDir1); 243 | 244 | FSDataOutputStream s1=fs.create(file1, true, 4096, (short) 1, (long) 4096, null); 245 | FSDataOutputStream s2=fs.create(file2, true, 4096, (short) 1, (long) 4096, null); 246 | 247 | s1.close(); 248 | s2.close(); 249 | 250 | FileStatus[] p=fs.listStatus(subDir1); 251 | assertEquals(p.length, 2); 252 | 253 | fs.delete(file1, true); 254 | p=fs.listStatus(subDir1); 255 | assertEquals(p.length, 1); 256 | 257 | fs.delete(file2, true); 258 | p=fs.listStatus(subDir1); 259 | assertEquals(p.length, 0); 260 | 261 | fs.delete(baseDir, true); 262 | assertFalse(fs.exists(baseDir)); 263 | 264 | fs.delete(subDir1); 265 | fs.delete(file1); 266 | fs.delete(file2); 267 | } 268 | 269 | public void testFileIO() throws Exception{ 270 | 271 | Path subDir1=new Path("tfio_dir.1"); 272 | Path file1=new Path("tfio_dir.1/foo.1"); 273 | Path baseDir=new Path("tfio_testDirs1"); 274 | 275 | fs.mkdirs(baseDir); 276 | assertTrue(fs.isDirectory(baseDir)); 277 | // fs.setWorkingDirectory(baseDir); 278 | 279 | fs.mkdirs(subDir1); 280 | 281 | FSDataOutputStream s1=fs.create(file1, true, 4096, (short) 1, (long) 4096, null); 282 | 283 | int bufsz=4096; 284 | byte[] data=new byte[bufsz]; 285 | 286 | for(int i=0;i 4 | * This file is part of GlusterFS. 5 | * 6 | * Licensed under the Apache License, Version 2.0 7 | * (the "License"); you may not use this file except in compliance with 8 | * the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 15 | * implied. See the License for the specific language governing 16 | * permissions and limitations under the License. 17 | * 18 | */ 19 | 20 | package org.apache.hadoop.fs.test.unit; 21 | 22 | import static org.apache.hadoop.fs.FileSystemTestHelper.getTestRootPath; 23 | import static org.junit.Assert.assertEquals; 24 | import static org.junit.Assert.assertFalse; 25 | import static org.junit.Assert.assertTrue; 26 | 27 | import java.io.IOException; 28 | 29 | import org.apache.hadoop.fs.FSDataInputStream; 30 | import org.apache.hadoop.fs.FSDataOutputStream; 31 | import org.apache.hadoop.fs.FileStatus; 32 | import org.apache.hadoop.fs.FileSystem; 33 | import org.apache.hadoop.fs.Path; 34 | import org.apache.hadoop.conf.Configuration; 35 | import org.apache.hadoop.fs.permission.FsAction; 36 | import org.apache.hadoop.fs.permission.FsPermission; 37 | import org.apache.hadoop.fs.test.connector.HcfsTestConnectorFactory; 38 | import org.apache.hadoop.fs.test.connector.HcfsTestConnectorInterface; 39 | import org.junit.After; 40 | import org.junit.AfterClass; 41 | import org.junit.Assert; 42 | import org.junit.BeforeClass; 43 | import org.junit.Test; 44 | 45 | /** 46 | * Unit test for HCFS classes. 47 | * 48 | */ 49 | public class HcfsUmaskTest{ 50 | 51 | static FileSystem fs ; 52 | 53 | @BeforeClass 54 | public static void setup() throws Exception { 55 | HcfsTestConnectorInterface connector = HcfsTestConnectorFactory.getHcfsTestConnector(); 56 | fs= connector.create(); 57 | } 58 | 59 | @AfterClass 60 | public static void after() throws IOException{ 61 | fs.close(); 62 | } 63 | 64 | @After 65 | public void tearDown() throws Exception { 66 | fs.delete(getTestRootPath(fs, "test"),true); 67 | } 68 | 69 | @org.junit.Test 70 | public void testMkdirsWithUmask() throws Exception { 71 | Configuration conf = fs.getConf(); 72 | String oldUmask = conf.get("fs.permissions.umask-mode"); 73 | Path dir = new Path("dirUmask022"); 74 | conf.set("fs.permissions.umask-mode", "022"); 75 | assertTrue(fs.mkdirs(dir)); 76 | conf.set("fs.permissions.umask-mode", oldUmask); 77 | FileStatus status = fs.getFileStatus(dir); 78 | assertTrue(status.isDirectory()); 79 | assertEquals((short)0755, status.getPermission().toShort()); 80 | 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # this could be conditionally set. it may be set higher up as part of the global env variables. 4 | rm -rf target/ 5 | # runs in debug mode. 6 | mvn package -Dmaven.surefire.debug 7 | 8 | 9 | -------------------------------------------------------------------------------- /tools/build-deploy-jar.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | ## 4 | # 5 | # Copyright (c) 2011 Gluster, Inc. 6 | # This file is part of GlusterFS. 7 | # 8 | # Licensed under the Apache License, Version 2.0 9 | # (the "License"); you may not use this file except in compliance with 10 | # the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | # implied. See the License for the specific language governing 18 | # permissions and limitations under the License. 19 | # 20 | ## 21 | 22 | import getopt 23 | import glob 24 | import sys, os 25 | import shutil 26 | import subprocess, shlex 27 | 28 | def usage(): 29 | print "usage: python build-deploy-jar.py [-b/--build] -d/--dir [-c/--core] [-m/--mapred] [-h/--henv]" 30 | 31 | def addSlash(s): 32 | if not (s[-1] == '/'): 33 | s = s + '/' 34 | 35 | return s 36 | 37 | def whereis(program): 38 | abspath = None 39 | for path in (os.environ.get('PATH', '')).split(':'): 40 | abspath = os.path.join(path, program) 41 | if os.path.exists(abspath) and not os.path.isdir(abspath): 42 | return abspath 43 | 44 | return None 45 | 46 | def getLatestJar(targetdir): 47 | glusterfsJar = glob.glob(targetdir + "*.jar") 48 | if len(glusterfsJar) == 0: 49 | print "No GlusterFS jar file found in %s ... exiting" % (targetdir) 50 | return None 51 | 52 | # pick up the latest jar file - just in case ... 53 | stat = latestJar = None 54 | ctime = 0 55 | 56 | for jar in glusterfsJar: 57 | stat = os.stat(jar) 58 | if stat.st_ctime > ctime: 59 | latestJar = jar 60 | ctime = stat.st_ctime 61 | 62 | return latestJar 63 | 64 | # build the glusterfs hadoop plugin using maven 65 | def build_jar(targetdir): 66 | location = whereis('mvn') 67 | 68 | if location == None: 69 | print "Cannot find maven to build glusterfs hadoop jar" 70 | print "please install maven or if it's already installed then fix your PATH environ" 71 | return None 72 | 73 | # do a clean packaging 74 | if os.path.exists(targetdir) and os.path.isdir(targetdir): 75 | print "Cleaning up directories ... [ " + targetdir + " ]" 76 | shutil.rmtree(targetdir) 77 | 78 | print "Building glusterfs jar ..." 79 | process = subprocess.Popen(['package'], shell=True, 80 | executable=location, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 81 | 82 | process.wait() 83 | if not process.returncode == 0: 84 | print "Building glusterfs jar failed ... exiting" 85 | return None 86 | 87 | latestJar = getLatestJar(targetdir) 88 | return latestJar 89 | 90 | def rcopy(f, host, libdir): 91 | print " * doing remote copy to host %s" % (host) 92 | scpCmd = "scp %s %s:%s" % (f, host, libdir) 93 | 94 | os.system(scpCmd); 95 | 96 | def deployInSlave(f, confdir, libdir, cc, cm, he): 97 | slavefile = confdir + "slaves" 98 | 99 | ccFile = confdir + "core-site.xml" 100 | cmFile = confdir + "mapred-site.xml" 101 | heFile = confdir + "hadoop-env.sh" 102 | 103 | sf = open(slavefile, 'r') 104 | for host in sf: 105 | host = host.rstrip('\n') 106 | print " >>> Deploying %s on %s ..." % (os.path.basename(f), host) 107 | rcopy(f, host, libdir) 108 | 109 | if cc: 110 | print " >>> Deploying [%s] on %s ..." % (os.path.basename(ccFile), host) 111 | rcopy(ccFile, host, confdir) 112 | 113 | if cm: 114 | print " >>> Deploying [%s] on %s ..." % (os.path.basename(cmFile), host) 115 | rcopy(cmFile, host, confdir) 116 | 117 | if he: 118 | print " >>> Deploying [%s] on %s ..." % (os.path.basename(heFile), host) 119 | rcopy(heFile, host, confdir); 120 | 121 | print "<<< Done\n" 122 | 123 | sf.close() 124 | 125 | def deployInMaster(f, confdir, libdir): 126 | import socket 127 | masterfile = confdir + "masters" 128 | 129 | mf = open(masterfile, 'r') 130 | for host in mf: 131 | host = host.rstrip('\n') 132 | print " >>> Deploying %s on %s ..." % (os.path.basename(f), host) 133 | h = host 134 | try: 135 | socket.inet_aton(host) 136 | h = socket.getfqdn(host) 137 | except socket.error: 138 | pass 139 | 140 | if h == socket.gethostname() or h == 'localhost': 141 | # local cp 142 | print " * doing local copy" 143 | shutil.copy(f, libdir) 144 | else: 145 | # scp the file 146 | rcopy(f, h, libdir) 147 | 148 | print "<<< Done\n" 149 | 150 | mf.close() 151 | 152 | if __name__ == '__main__': 153 | opt = args = [] 154 | try: 155 | opt, args = getopt.getopt(sys.argv[1:], "bd:cmh", ["build", "dir=", "core", "mapred", "henv"]); 156 | except getopt.GetoptError, err: 157 | print str(err) 158 | usage() 159 | sys.exit(1) 160 | 161 | needbuild = hadoop_dir = copyCore = copyMapred = copyHadoopEnv = None 162 | 163 | for k, v in opt: 164 | if k in ("-b", "--build"): 165 | needbuild = True 166 | elif k in ("-d", "--dir"): 167 | hadoop_dir = v 168 | elif k in ("-c", "--core"): 169 | copyCore = True 170 | elif k in ("-m", "--mapred"): 171 | copyMapred = True 172 | elif k in ("-h", "--henv"): 173 | copyHadoopEnv = True 174 | else: 175 | pass 176 | 177 | if hadoop_dir == None: 178 | print 'hadoop directory missing' 179 | usage() 180 | sys.exit(1) 181 | 182 | os.chdir(os.path.dirname(sys.argv[0]) + '/..') 183 | targetdir = './target/' 184 | 185 | if needbuild: 186 | jar = build_jar(targetdir) 187 | if jar == None: 188 | sys.exit(1) 189 | else: 190 | jar = getLatestJar(targetdir) 191 | if jar == None: 192 | print "Maybe you want to build it ? with -b option" 193 | sys.exit(1) 194 | 195 | print "" 196 | print "*** Deploying %s *** " % (jar) 197 | 198 | # copy jar to local hadoop distribution (master) 199 | hadoop_home = addSlash(hadoop_dir) 200 | if not (os.path.exists(hadoop_home) and os.path.isdir(hadoop_home)): 201 | print "path " + hadoop_home + " does not exist or is not adiretory"; 202 | sys.exit(1); 203 | 204 | hadoop_conf = hadoop_home + "conf/" 205 | hadoop_lib = hadoop_home + "lib/" 206 | 207 | print " >>> Scanning hadoop master file for host(s) to deploy" 208 | deployInMaster(jar, hadoop_conf, hadoop_lib) 209 | 210 | print "" 211 | print " >>> Scanning hadoop slave file for host(s) to deploy" 212 | deployInSlave(jar, hadoop_conf, hadoop_lib, copyCore, copyMapred, copyHadoopEnv) 213 | --------------------------------------------------------------------------------