├── .gitignore ├── README.md ├── bin └── setup.sh ├── pom.xml └── src ├── main ├── hive │ ├── lsp.ddl │ └── nn_stats.ddl ├── java │ ├── JCECheck.java │ └── com │ │ ├── dstreev │ │ ├── hadoop │ │ │ └── util │ │ │ │ ├── HdfsLsPlus.java │ │ │ │ ├── HdfsNNStats.java │ │ │ │ ├── HdfsRepair.java │ │ │ │ ├── JmxJsonParser.java │ │ │ │ ├── NamenodeJmxBean.java │ │ │ │ └── NamenodeJmxParser.java │ │ └── hdfs │ │ │ └── shell │ │ │ └── command │ │ │ ├── Constants.java │ │ │ ├── Direction.java │ │ │ ├── LocalCat.java │ │ │ ├── LocalHead.java │ │ │ ├── LocalMkdir.java │ │ │ └── LocalRm.java │ │ └── instanceone │ │ └── hdfs │ │ └── shell │ │ ├── HdfsShell.java │ │ ├── command │ │ ├── FSUtil.java │ │ ├── HdfsAbstract.java │ │ ├── HdfsCd.java │ │ ├── HdfsCommand.java │ │ ├── HdfsConnect.java │ │ ├── HdfsPwd.java │ │ ├── LocalCd.java │ │ ├── LocalLs.java │ │ └── LocalPwd.java │ │ └── completers │ │ └── FileSystemNameCompleter.java └── resources │ └── banner.txt └── test ├── java └── com │ ├── dstreev │ └── hadoop │ │ └── util │ │ ├── DirectoryReaderTest.java │ │ └── NamenodeParserTest.java │ └── instanceone │ └── hdfs │ └── shell │ └── ShellTest.java └── resources ├── nn_2.3.5.1_active.json ├── nn_2.3.5.1_standby.json └── topUserOpsCount.json /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### OSX template 3 | .DS_Store 4 | .AppleDouble 5 | .LSOverride 6 | 7 | # Icon must end with two \r 8 | Icon 9 | 10 | # Thumbnails 11 | ._* 12 | 13 | # Files that might appear in the root of a volume 14 | .DocumentRevisions-V100 15 | .fseventsd 16 | .Spotlight-V100 17 | .TemporaryItems 18 | .Trashes 19 | .VolumeIcon.icns 20 | 21 | # Directories potentially created on remote AFP share 22 | .AppleDB 23 | .AppleDesktop 24 | Network Trash Folder 25 | Temporary Items 26 | .apdisk 27 | 28 | 29 | ### Java template 30 | *.class 31 | 32 | # Mobile Tools for Java (J2ME) 33 | .mtj.tmp/ 34 | 35 | # Package Files # 36 | *.jar 37 | *.war 38 | *.ear 39 | 40 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 41 | hs_err_pid* 42 | 43 | 44 | ### Eclipse template 45 | *.pydevproject 46 | .metadata 47 | .gradle 48 | bin/ 49 | tmp/ 50 | *.tmp 51 | *.bak 52 | *.swp 53 | *~.nib 54 | local.properties 55 | .settings/ 56 | .loadpath 57 | 58 | # Eclipse Core 59 | .project 60 | 61 | # External tool builders 62 | .externalToolBuilders/ 63 | 64 | # Locally stored "Eclipse launch configurations" 65 | *.launch 66 | 67 | # CDT-specific 68 | .cproject 69 | 70 | # JDT-specific (Eclipse Java Development Tools) 71 | .classpath 72 | 73 | # Java annotation processor (APT) 74 | .factorypath 75 | 76 | # PDT-specific 77 | .buildpath 78 | 79 | # sbteclipse plugin 80 | .target 81 | 82 | # TeXlipse plugin 83 | .texlipse 84 | 85 | 86 | ### JetBrains template 87 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion 88 | 89 | *.iml 90 | 91 | ## Directory-based project format: 92 | .idea/ 93 | # if you remove the above rule, at least ignore the following: 94 | 95 | # User-specific stuff: 96 | # .idea/workspace.xml 97 | # .idea/tasks.xml 98 | # .idea/dictionaries 99 | 100 | # Sensitive or high-churn files: 101 | # .idea/dataSources.ids 102 | # .idea/dataSources.xml 103 | # .idea/sqlDataSources.xml 104 | # .idea/dynamic.xml 105 | # .idea/uiDesigner.xml 106 | 107 | # Gradle: 108 | # .idea/gradle.xml 109 | # .idea/libraries 110 | 111 | # Mongo Explorer plugin: 112 | # .idea/mongoSettings.xml 113 | 114 | ## File-based project format: 115 | *.ipr 116 | *.iws 117 | 118 | ## Plugin-specific files: 119 | 120 | # IntelliJ 121 | /out/ 122 | 123 | # mpeltonen/sbt-idea plugin 124 | .idea_modules/ 125 | 126 | # JIRA plugin 127 | atlassian-ide-plugin.xml 128 | 129 | # Crashlytics plugin (for Android Studio and IntelliJ) 130 | com_crashlytics_export_strings.xml 131 | crashlytics.properties 132 | crashlytics-build.properties 133 | 134 | 135 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # We're moving... 2 | 3 | We've been replaced! Well, actually we're growing to more than just hdfs. A new project has been created to reflect this and is now the active replacement. Please see `hadoopcli` for all the same capabilities, plus more! 4 | 5 | [Hadoop-Cli](https://github.com/dstreev/hadoop-cli) 6 | 7 | ## HDFS-CLI 8 | 9 | HDFS-CLI is an interactive command line shell that makes interacting with the Hadoop Distribted Filesystem (HDFS) 10 | simpler and more intuitive than the standard command-line tools that come with Hadoop. If you're familiar with OS X, Linux, or even Windows terminal/console-based applications, then you are likely familiar with features such as tab completion, command history, and ANSI formatting. 11 | 12 | ### Binary Package 13 | 14 | [Pre-Built Distribution](https://github.com/dstreev/hdfs-cli/releases) 15 | 16 | Download the release files to a temp location. As a root user, chmod +x the 3 shell script files then run the 'setup.sh'. This will create and install the hdfscli application to your path. 17 | 18 | Try it out on a host with default configs: 19 | 20 | hdfscli 21 | 22 | To use an alternate HADOOP_CONF_DIR: 23 | 24 | hdfscli --config /var/hadoop/dev-cfg 25 | 26 | ### Release Notes 27 | 28 | #### 2.3.2-SNAPSHOT (in-progress) 29 | 30 | ##### Behavioural Changes 31 | - Due to the complexities and requirements of connecting to an environment, I've removed the 32 | ability to connect manually to an environment by simply using 'connect'. This option was there from the beginning, but as more and more features are added, I'm finding myself hacking 33 | away at recreating the settings and controls enabled through the configurations available in 34 | `hdfs-site.xml` and `core-site.xml`. Therefore, the options `-k for kerberos` and `-a for auto connect` are no longer available. Unless specified via the `--config` option, the `hdfs-site.xml` and `core-site.xml` files in the default location of `/etc/hadoop/conf` will be used to establish all of the environment variables needed to connect. You can have multiple 35 | directories with various `hdfs|core-site.xml` files in them and use the `--config` option to 36 | enable connectivity to alternate hadoop environments. 37 | ##### Removed Options 38 | `-a` Auto Connect. Use `--config` for alternate site files or nothing for the default `/etc/hadoop/conf`. 39 | `-k` Kerberos Option 40 | 41 | ##### Enhancements 42 | I noticed some pauses coming from inquiries into the Namenode JMX for `nnstat`. Instead of requesting the 43 | entire Namenode Jmx stack, now we target only the JmxBeans that we're interested in. This will help 44 | the observed pauses and relieve the Namenode of some unnecessary work. 45 | 46 | #### 2.3.1-SNAPSHOT (in-progress) 47 | - Added new 'nnstat' function to collect Namenode JMX stats for long term analysis. 48 | 49 | See [NN Stat Feature](https://youtu.be/CZxx_BxCX4Y) 50 | 51 | Also checkout how to auto fetch the stats via a script. Use this technique to run cron jobs to gather the stats. 52 | 53 | Where `-i stats` defines the initialization file. See [Auto nnstat](https://youtu.be/gy43_Hg2RXk) 54 | ``` 55 | hdfscli -i stats 56 | ``` 57 | 58 | #### 2.3.0-SNAPSHOT 59 | - Added new 'lsp' function. Consider it an 'ls' PLUS. 60 | 61 | #### 2.2.1-SNAPSHOT 62 | 63 | - External Config Support (See Below) 64 | - Supports NN HA (thru hdfs and core site files) 65 | - Auto Config support using a default config directory. 66 | 67 | #### 2.2.0-SNAPSHOT 68 | 69 | - Setup Script to help deploy (bin/setup.sh) 70 | - hdfscli shell script to launch (bin/hdfscli.sh) 71 | - Support for initialization Script (-i ) 72 | - Kerberos Support via default and --config option for hadoop site files. 73 | 74 | #### 2.1.0 75 | 76 | - Added support for create/delete/rename Snapshot 77 | 78 | #### 2.0.0 79 | 80 | - Initial Forked Release of P. Taylor Goetz. 81 | - Update to 2.6.0 Hadoop Libraries 82 | - Re-wrote Command Implementation to use FSShell as basis for issuing commands. 83 | - Provide Context Feedback in command window to show local and remote context. 84 | - Added several missing hdfs dfs commands that didn't exist earlier. 85 | 86 | ### Building 87 | 88 | This project requires the artifacts from https://github.com/dstreev/stemshell , which is a forked enhancement that has added support for processing command line parameters and deals with quoted variables. 89 | 90 | ### Basic Usage 91 | HDFS-CLI works much like a command-line ftp client: You first establish a connection to a remote HDFS filesystem, 92 | then manage local/remote files and transfers. 93 | 94 | To start HDFS-CLI, run the following command: 95 | 96 | java -jar hdfs-cli-full-bin.jar 97 | 98 | ### Command Documentation 99 | 100 | Help for any command can be obtained by executing the `help` command: 101 | 102 | help pwd 103 | 104 | Note that currently, documentation may be limited. 105 | 106 | #### Local vs. Remote Commands 107 | When working within a HDFS-CLI session, you manage both local (on your computer) and remote (HDFS) files. By convention, commands that apply to both local and remote filesystems are differentiated by prepending an `l` 108 | character to the name to denote "local". 109 | 110 | For example: 111 | 112 | `lls` lists local files in the local current working directory. 113 | 114 | `ls` lists remote files in the remote current working directory. 115 | 116 | Every HDFS-CLI session keeps track of both the local and remote current working directories. 117 | 118 | ### Support for External Configurations (core-site.xml,hdfs-site.xml) 119 | 120 | By default, hdfs-cli will use `/etc/hadoop/conf` as the default location to search for 121 | `core-site.xml` and `hdfs-site.xml`. If you want to use an alternate, use the `--config` 122 | option when starting up hdfs-cli. 123 | 124 | The `--config` option takes 1 parameter, a local directory. This directory should contain hdfs-site.xml and core-site.xml files. When used, you'll automatically be connected to hdfs and changed to you're hdfs home directory. 125 | 126 | Example Connection parameters. 127 | 128 | # Use the hadoop files in the input directory to configure and connect to HDFS. 129 | hdfscli --config ../mydir 130 | 131 | This can be used in conjunction with the 'Startup' Init option below to run a set of commands automatically after the connection is made. The 'connect' option should NOT be used in the initialization script. 132 | 133 | ### Startup Initialization Option 134 | 135 | Using the option '-i ' when launching the CLI, it will run all the commands in the file. 136 | 137 | The file needs to be location in the $HOME/.hdfs-cli directory. For example: 138 | 139 | # If you're using the helper shell script 140 | hdfscli -i test 141 | 142 | # If you're using the java command 143 | java -jar hdfs-cli-full-bin.jar -i test 144 | 145 | 146 | Will initialize the session with the command(s) in $HOME/.hdfs-cli/test. One command per line. 147 | 148 | The contents could be any set of valid commands that you would use in the cli. For example: 149 | 150 | cd user/dstreev 151 | 152 | ### NN Stats 153 | 154 | Collect Namenode stats from the available Namenode JMX url's. 155 | 156 | 3 Type of stats are current collected and written to hdfs (with -o option) or to screen (no option specified) 157 | 158 | The 'default' delimiter for all records is '\u0001' (Cntl-A) 159 | 160 | >> Namenode Information: (optionally written to the directory 'nn_info') 161 | Fields: Timestamp, HostAndPort, State, Version, Used, Free, Safemode, TotalBlocks, TotalFiles, NumberOfMissingBlocks, NumberOfMissingBlocksWithReplicationFactorOne 162 | 163 | >> Filesystem State: (optionally written to the directory 'fs_state') 164 | Fields: Timestamp, HostAndPort, State, CapacityUsed, CapacityRemaining, BlocksTotal, PendingReplicationBlocks, UnderReplicatedBlocks, ScheduledReplicationBlocks, PendingDeletionBlocks, FSState, NumLiveDataNodes, NumDeadDataNodes, NumDecomLiveDataNodes, NumDecomDeadDataNodes, VolumeFailuresTotal 165 | 166 | >> Top User Operations: (optionally written to the directory 'top_user_ops') 167 | Fields: Timestamp, HostAndPort, State, WindowLenMs, Operation, User, Count 168 | 169 | [Hive Table DDL for NN Stats](./src/main/hive/nn_stats.ddl) 170 | 171 | ### Enhanced Directory Listing (lsp) 172 | 173 | Like 'ls', you can fetch many details about a file. But with this, you can also add information about the file that includes: 174 | - Block Size 175 | - Access Time 176 | - Ratio of File Size to Block 177 | - Datanode information for the files blocks (Host and Block Id) 178 | 179 | Use help to get the options: 180 | 181 | help lsp 182 | 183 | ``` 184 | usage: stats [OPTION ...] [ARGS ...] 185 | Options: 186 | -d,--maxDepth Depth of Recursion (default 5), use '-1' 187 | for unlimited 188 | -f,--format Comma separated list of one or more: 189 | permissions_long,replication,user,group,siz 190 | e,block_size,ratio,mod,access,path,datanode 191 | _info (default all of the above) 192 | -o,--output Output File (HDFS) (default System.out) 193 | ``` 194 | 195 | When not argument is specified, it will use the current directory. 196 | 197 | Examples: 198 | 199 | # Using the default format, output a listing to the files in `/user/dstreev/perf` to `/tmp/test.out` 200 | lsp -o /tmp/test.out /user/dstreev/perf 201 | 202 | Output with the default format of: 203 | 204 | permissions_long,replication,user,group,size,block_size,ratio,mod,access,path,datanode_info 205 | 206 | ``` 207 | rw-------,3,dstreev,hdfs,429496700,134217728,3.200,2015-10-24 12:26:39.689,2015-10-24 12:23:27.406,/user/dstreev/perf/teragen_27/part-m-00004,10.0.0.166,d2.hdp.local,blk_1073747900 208 | rw-------,3,dstreev,hdfs,429496700,134217728,3.200,2015-10-24 12:26:39.689,2015-10-24 12:23:27.406,/user/dstreev/perf/teragen_27/part-m-00004,10.0.0.167,d3.hdp.local,blk_1073747900 209 | rw-------,3,dstreev,hdfs,33,134217728,2.459E-7,2015-10-24 12:27:09.134,2015-10-24 12:27:06.560,/user/dstreev/perf/terasort_27/_partition.lst,10.0.0.166,d2.hdp.local,blk_1073747909 210 | rw-------,3,dstreev,hdfs,33,134217728,2.459E-7,2015-10-24 12:27:09.134,2015-10-24 12:27:06.560,/user/dstreev/perf/terasort_27/_partition.lst,10.0.0.167,d3.hdp.local,blk_1073747909 211 | rw-------,1,dstreev,hdfs,543201700,134217728,4.047,2015-10-24 12:29:28.706,2015-10-24 12:29:20.882,/user/dstreev/perf/terasort_27/part-r-00002,10.0.0.167,d3.hdp.local,blk_1073747920 212 | rw-------,1,dstreev,hdfs,543201700,134217728,4.047,2015-10-24 12:29:28.706,2015-10-24 12:29:20.882,/user/dstreev/perf/terasort_27/part-r-00002,10.0.0.167,d3.hdp.local,blk_1073747921 213 | ``` 214 | 215 | With the file in HDFS, you can build a [hive table](./src/main/hive/lsp.ddl) on top of it to do some analysis. One of the reasons I created this was to be able to review a directory used by some process and get a baring on the file construction and distribution across the cluster. 216 | 217 | #### Use Cases 218 | - The ratio can be used to identify files that are below the block size (small files). 219 | - With the Datanode information, you can determine if a dataset is hot-spotted on a cluster. All you need is a full list of hosts to join the results with. 220 | 221 | ### Available Commands 222 | 223 | #### Common Commands 224 | help display help information 225 | put upload local files to the remote HDFS 226 | get (todo) retrieve remote files from HDFS to Local Filesystem 227 | 228 | #### Remote (HDFS) Commands 229 | cd change current working directory 230 | ls list directory contents 231 | rm delete files/directories 232 | pwd print working directory path 233 | cat print file contents 234 | chown change ownership 235 | chmod change permissions 236 | chgrp change group 237 | head print first few lines of a file 238 | mkdir create directories 239 | count Count the number of directories, files and bytes under the paths that match the specified file pattern. 240 | stat Print statistics about the file/directory at in the specified format. 241 | tail Displays last kilobyte of the file to stdout. 242 | text Takes a source file and outputs the file in text format. 243 | touchz Create a file of zero length. 244 | usage Return the help for an individual command. 245 | 246 | createSnapshot Create Snapshot 247 | deleteSnapshot Delete Snapshot 248 | renameSnapshot Rename Snapshot 249 | 250 | #### Local (Local File System) Commands 251 | lcd change current working directory 252 | lls list directory contents 253 | lrm delete files/directories 254 | lpwd print working directory path 255 | lcat print file contents 256 | lhead print first few lines of a file 257 | lmkdir create directories 258 | 259 | #### Tools and Utilities 260 | lsp ls plus. Includes Block information and locations. 261 | nnstat Namenode Statistics 262 | 263 | ### Known Bugs/Limitations 264 | 265 | * No support for paths containing spaces 266 | * No support for Windows XP 267 | * Path Completion for chown, chmod, chgrp, rm is broken 268 | 269 | ### Road Map 270 | 271 | - Support input variables 272 | - Expand to support Extended ACL's (get/set) 273 | - Add Support for setrep 274 | - HA Commands 275 | - NN and RM 276 | 277 | 278 | 279 | 280 | 281 | -------------------------------------------------------------------------------- /bin/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Should be run as root. 4 | 5 | cd `dirname $0` 6 | 7 | mkdir -p /usr/local/hdfs-cli/bin 8 | mkdir -p /usr/local/hdfs-cli/lib 9 | 10 | cp -f hdfscli /usr/local/hdfs-cli/bin 11 | cp -f JCECheck /usr/local/hdfs-cli/bin 12 | 13 | if [ -f ../target/hdfs-cli-full-bin.jar ]; then 14 | cp -f ../target/hdfs-cli-full-bin.jar /usr/local/hdfs-cli/lib 15 | fi 16 | 17 | if [ -f hdfs-cli-full-bin.jar ]; then 18 | cp -f hdfs-cli-full-bin.jar /usr/local/hdfs-cli/lib 19 | fi 20 | 21 | chmod -R +r /usr/local/hdfs-cli 22 | chmod +x /usr/local/hdfs-cli/bin/hdfscli 23 | chmod +x /usr/local/hdfs-cli/bin/JCECheck 24 | 25 | ln -sf /usr/local/hdfs-cli/bin/JCECheck /usr/local/bin/JCECheck 26 | ln -sf /usr/local/hdfs-cli/bin/hdfscli /usr/local/bin/hdfscli 27 | 28 | 29 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.instanceone.hdfs 5 | hdfs-cli 6 | 2.3.2-SNAPSHOT 7 | Hadoop HDFS CLI 8 | Hadoop HDFS Command Line Interface 9 | 10 | 2.6.0 11 | 4.11 12 | 2.7.3 13 | 2.4 14 | 15 | 16 | 17 | 18 | com.github.ptgoetz 19 | stemshell 20 | 0.3.2-SNAPSHOT 21 | 22 | 23 | org.apache.hadoop 24 | hadoop-client 25 | ${hadoop.version} 26 | 27 | 28 | commons-io 29 | commons-io 30 | ${commons-io.version} 31 | 32 | 33 | com.fasterxml.jackson.core 34 | jackson-core 35 | ${jackson.parser} 36 | 37 | 38 | junit 39 | junit 40 | ${junit.version} 41 | test 42 | 43 | 44 | 45 | 46 | 47 | org.apache.maven.plugins 48 | maven-shade-plugin 49 | 2.0 50 | 51 | 52 | 53 | 54 | 55 | package 56 | 57 | shade 58 | 59 | 60 | true 61 | hdfs-cli-full-bin 62 | true 63 | 64 | 65 | 67 | 69 | com.instanceone.hdfs.shell.HdfsShell 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | org.apache.maven.plugins 78 | maven-compiler-plugin 79 | 80 | 1.7 81 | 1.7 82 | 83 | 84 | 85 | 86 | 87 | -------------------------------------------------------------------------------- /src/main/hive/lsp.ddl: -------------------------------------------------------------------------------- 1 | create database if not exists ${DB}; 2 | 3 | use ${DB}; 4 | 5 | drop table hdfs_analysis; 6 | 7 | create external table if not exists hdfs_analysis ( 8 | permissions_long string, 9 | replication int, 10 | user_ string, 11 | group_ string, 12 | size bigint, 13 | block_size bigint, 14 | ratio double, 15 | mod string, 16 | access string, 17 | path_ string, 18 | ip_address string, 19 | hostname string, 20 | block_id string 21 | ) 22 | ROW FORMAT DELIMITED 23 | FIELDS TERMINATED BY ',' 24 | STORED AS TEXTFILE 25 | LOCATION '${LOCATION}'; 26 | 27 | drop table hosts; 28 | 29 | -- Build a hosts table from known blocks. Might be better to manually build this 30 | -- record set to ensure ALL hosts are accounted for!! 31 | create table hosts 32 | stored as orc 33 | as 34 | select distinct hostname from hdfs_analysis; 35 | 36 | select * from hosts; -------------------------------------------------------------------------------- /src/main/hive/nn_stats.ddl: -------------------------------------------------------------------------------- 1 | use ${DB}; 2 | 3 | drop table if exists nn_info; 4 | create external table if not exists nn_info ( 5 | date_ String, 6 | HostAndPort String, 7 | State_ String, 8 | Version String, 9 | Used DOUBLE, 10 | Free_ DOUBLE, 11 | Safemode STRING, 12 | TotalBlocks BIGINT, 13 | TotalFiles BIGINT, 14 | NumberOfMissingBlocks BIGINT, 15 | NumberOfMissingBlocksWithReplicationFactorOne BIGINT 16 | ) 17 | STORED AS TEXTFILE 18 | LOCATION '${BASE_DIR}/nn_info'; 19 | 20 | drop table if exists fs_state; 21 | create external table if not exists fs_state ( 22 | date_ String, 23 | HostAndPort String, 24 | State_ String, 25 | CapacityUsed BIGINT, 26 | CapacityRemaining BIGINT, 27 | BlocksTotal BIGINT, 28 | PendingReplicationBlocks BIGINT, 29 | UnderReplicatedBlocks BIGINT, 30 | ScheduledReplicationBlocks BIGINT, 31 | PendingDeletionBlocks BIGINT, 32 | FSState String, 33 | NumLiveDataNodes INT, 34 | NumDeadDataNodes INT, 35 | NumDecomLiveDataNodes INT, 36 | NumDecomDeadDataNodes INT, 37 | VolumeFailuresTotal INT 38 | ) 39 | STORED AS TEXTFILE 40 | LOCATION '${BASE_DIR}/fs_state'; 41 | 42 | drop table if exists top_user_ops; 43 | create external table if not exists top_user_ops ( 44 | date_ String, 45 | HostAndPort String, 46 | State_ String, 47 | WindowLenMs BIGINT, 48 | Operation String, 49 | User_ String, 50 | Count_ BIGINT) 51 | STORED AS TEXTFILE 52 | LOCATION '${BASE_DIR}/top_user_ops'; -------------------------------------------------------------------------------- /src/main/java/JCECheck.java: -------------------------------------------------------------------------------- 1 | import javax.crypto.Cipher; 2 | import java.security.NoSuchAlgorithmException; 3 | import java.security.Provider; 4 | import java.security.Security; 5 | import java.util.Iterator; 6 | 7 | /** 8 | * Created by dstreev on 2015-09-29. 9 | */ 10 | public class JCECheck { 11 | public static void main(final String[] args) { 12 | final Provider[] providers = Security.getProviders(); 13 | for (int i = 0; i < providers.length; i++) { 14 | final String name = providers[i].getName(); 15 | final double version = providers[i].getVersion(); 16 | System.out.println("Provider[" + i + "]:: " + name + " " + version); 17 | if (args.length > 0) { 18 | final Iterator it = providers[i].keySet().iterator(); 19 | while (it.hasNext()) { 20 | final String element = (String) it.next(); 21 | if (element.toLowerCase().startsWith(args[0].toLowerCase()) 22 | || args[0].equals("-all")) 23 | System.out.println("\t" + element); 24 | } 25 | } 26 | } 27 | try { 28 | int keyLength = Cipher.getMaxAllowedKeyLength("AES"); 29 | if (keyLength > 128) { 30 | System.out.println("JCE is available for Unlimited encryption. ["+keyLength+"]"); 31 | } else { 32 | System.out.println("JCE is NOT available for Unlimited encryption. ["+keyLength+"]"); 33 | } 34 | } catch (NoSuchAlgorithmException e) { 35 | e.printStackTrace(); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/com/dstreev/hadoop/util/HdfsLsPlus.java: -------------------------------------------------------------------------------- 1 | package com.dstreev.hadoop.util; 2 | 3 | import com.dstreev.hdfs.shell.command.Constants; 4 | import com.dstreev.hdfs.shell.command.Direction; 5 | import com.instanceone.hdfs.shell.command.HdfsAbstract; 6 | import com.instanceone.stemshell.Environment; 7 | import jline.console.ConsoleReader; 8 | import org.apache.commons.cli.CommandLine; 9 | import org.apache.commons.cli.Option; 10 | import org.apache.commons.cli.Options; 11 | import org.apache.hadoop.conf.Configuration; 12 | import org.apache.hadoop.fs.FSDataOutputStream; 13 | import org.apache.hadoop.fs.FileStatus; 14 | import org.apache.hadoop.fs.FileSystem; 15 | import org.apache.hadoop.fs.Path; 16 | import org.apache.hadoop.fs.shell.PathData; 17 | import org.apache.hadoop.hdfs.DFSClient; 18 | import org.apache.hadoop.hdfs.protocol.DatanodeInfo; 19 | import org.apache.hadoop.hdfs.protocol.LocatedBlock; 20 | import org.apache.hadoop.hdfs.protocol.LocatedBlocks; 21 | 22 | import java.io.IOException; 23 | import java.math.BigDecimal; 24 | import java.math.MathContext; 25 | import java.math.RoundingMode; 26 | import java.net.URI; 27 | import java.text.DateFormat; 28 | import java.text.SimpleDateFormat; 29 | import java.util.ArrayList; 30 | import java.util.Arrays; 31 | import java.util.Date; 32 | import java.util.List; 33 | 34 | import static com.dstreev.hadoop.util.HdfsLsPlus.PRINT_OPTION.*; 35 | 36 | /** 37 | * Created by dstreev on 2016-02-15. 38 | *

39 | * The intent here is to provide a means of querying the Namenode and 40 | * producing Metadata about the directory AND the files in it. 41 | */ 42 | public class HdfsLsPlus extends HdfsAbstract { 43 | 44 | private FileSystem fs = null; 45 | 46 | private static DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); 47 | 48 | // TODO: Extended ACL's 49 | private static String DEFAULT_FORMAT = "permissions_long,replication,user,group,size,block_size,ratio,mod,access,path,datanode_info"; 50 | 51 | enum PRINT_OPTION { 52 | PERMISSIONS_LONG, 53 | PERMISSIONS_SHORT, 54 | REPLICATION, 55 | USER, 56 | GROUP, 57 | SIZE, 58 | BLOCK_SIZE, 59 | RATIO, 60 | MOD, 61 | ACCESS, 62 | PATH, 63 | DATANODE_INFO 64 | } 65 | 66 | // default 67 | private PRINT_OPTION[] print_options = new PRINT_OPTION[]{PERMISSIONS_LONG, PATH, REPLICATION, 68 | USER, 69 | GROUP, 70 | SIZE, 71 | BLOCK_SIZE, 72 | RATIO, 73 | MOD, 74 | ACCESS, 75 | DATANODE_INFO}; 76 | 77 | private static int DEFAULT_DEPTH = 5; 78 | private static String DEFAULT_SEPARATOR = ","; 79 | private static String DEFAULT_NEWLINE = "\n"; 80 | private int currentDepth = 0; 81 | private int maxDepth = DEFAULT_DEPTH; 82 | // private Boolean recurse = Boolean.TRUE; 83 | private String format = DEFAULT_FORMAT; 84 | private Configuration configuration = null; 85 | private DFSClient dfsClient = null; 86 | private FSDataOutputStream outFS = null; 87 | private static MathContext mc = new MathContext(4, RoundingMode.HALF_UP); 88 | private int count = 0; 89 | 90 | public HdfsLsPlus(String name) { 91 | super(name); 92 | } 93 | 94 | public HdfsLsPlus(String name, Environment env, Direction directionContext) { 95 | super(name, env, directionContext); 96 | } 97 | 98 | public HdfsLsPlus(String name, Environment env, Direction directionContext, int directives) { 99 | super(name, env, directionContext, directives); 100 | } 101 | 102 | public HdfsLsPlus(String name, Environment env, Direction directionContext, int directives, boolean directivesBefore, boolean directivesOptional) { 103 | super(name, env, directionContext, directives, directivesBefore, directivesOptional); 104 | } 105 | 106 | public HdfsLsPlus(String name, Environment env) { 107 | super(name, env); 108 | } 109 | 110 | public void setMaxDepth(int maxDepth) { 111 | this.maxDepth = maxDepth; 112 | } 113 | 114 | // public void setRecurse(Boolean recurse) { 115 | // this.recurse = recurse; 116 | // } 117 | 118 | public void setFormat(String format) { 119 | this.format = format; 120 | String[] strOptions = this.format.split(","); 121 | List options_list = new ArrayList<>(); 122 | for (String strOption: strOptions) { 123 | PRINT_OPTION in = PRINT_OPTION.valueOf(strOption.toUpperCase()); 124 | if (in != null) { 125 | options_list.add(in); 126 | } 127 | } 128 | print_options = new PRINT_OPTION[strOptions.length]; 129 | print_options = options_list.toArray(print_options); 130 | } 131 | 132 | private void writeItem(PathData item, FileStatus itemStatus) { 133 | try { 134 | StringBuilder sb = new StringBuilder(); 135 | 136 | boolean in = false; 137 | for (PRINT_OPTION option : print_options) { 138 | if (in && option != DATANODE_INFO) 139 | sb.append(DEFAULT_SEPARATOR); 140 | in = true; 141 | switch (option) { 142 | case PERMISSIONS_SHORT: 143 | sb.append(itemStatus.getPermission().toExtendedShort()); 144 | break; 145 | case PERMISSIONS_LONG: 146 | sb.append(itemStatus.getPermission()); 147 | break; 148 | case REPLICATION: 149 | sb.append(itemStatus.getReplication()); 150 | break; 151 | case USER: 152 | sb.append(itemStatus.getOwner()); 153 | break; 154 | case GROUP: 155 | sb.append(itemStatus.getGroup()); 156 | break; 157 | case SIZE: 158 | sb.append(itemStatus.getLen()); 159 | break; 160 | case BLOCK_SIZE: 161 | sb.append(itemStatus.getBlockSize()); 162 | break; 163 | case RATIO: 164 | Double blockRatio = (double) itemStatus.getLen() / itemStatus.getBlockSize(); 165 | BigDecimal ratioBD = new BigDecimal(blockRatio, mc); 166 | sb.append(ratioBD.toString()); 167 | break; 168 | case MOD: 169 | sb.append(df.format(new Date(itemStatus.getModificationTime()))); 170 | break; 171 | case ACCESS: 172 | sb.append(df.format(new Date(itemStatus.getAccessTime()))); 173 | break; 174 | case PATH: 175 | sb.append(item.toString()); 176 | break; 177 | } 178 | } 179 | if (Arrays.asList(print_options).contains(DATANODE_INFO)) { 180 | LocatedBlocks blocks = null; 181 | blocks = dfsClient.getLocatedBlocks(item.toString(), 0, Long.MAX_VALUE); 182 | for (LocatedBlock block : blocks.getLocatedBlocks()) { 183 | DatanodeInfo[] datanodeInfo = block.getLocations(); 184 | // System.out.println("\tBlock: " + block.getBlock().getBlockName()); 185 | 186 | for (DatanodeInfo dni : datanodeInfo) { 187 | // System.out.println(dni.getIpAddr() + " - " + dni.getHostName()); 188 | StringBuilder sb1 = new StringBuilder(sb); 189 | sb1.append(DEFAULT_SEPARATOR); 190 | sb1.append(dni.getIpAddr()).append(DEFAULT_SEPARATOR); 191 | sb1.append(dni.getHostName()).append(DEFAULT_SEPARATOR); 192 | sb1.append(block.getBlock().getBlockName()); 193 | postItem(sb1.append(DEFAULT_NEWLINE).toString()); 194 | } 195 | } 196 | } else { 197 | postItem(sb.append(DEFAULT_NEWLINE).toString()); 198 | } 199 | 200 | } catch (IOException e) { 201 | e.printStackTrace(); 202 | } 203 | } 204 | 205 | private void postItem(String line) { 206 | if (outFS != null) { 207 | try { 208 | outFS.write(line.getBytes()); 209 | } catch (IOException e) { 210 | e.printStackTrace(); 211 | } 212 | count++; 213 | if (count % 10 == 0) 214 | System.out.print("."); 215 | if (count % 1000 == 0) 216 | System.out.println(); 217 | if (count % 10000 == 0) 218 | System.out.println("----------"); 219 | } else { 220 | System.out.print(line); 221 | } 222 | } 223 | 224 | private void processPath(PathData path) { 225 | // 226 | currentDepth++; 227 | if (maxDepth == -1 || currentDepth <= (maxDepth + 1)) { 228 | FileStatus fileStatus = path.stat; 229 | if (fileStatus.isDirectory()) { 230 | PathData[] pathDatas = new PathData[0]; 231 | try { 232 | pathDatas = path.getDirectoryContents(); 233 | } catch (IOException e) { 234 | e.printStackTrace(); 235 | } 236 | for (PathData intPd : pathDatas) { 237 | processPath(intPd); 238 | } 239 | } else { 240 | // Go through contents. 241 | writeItem(path, fileStatus); 242 | } 243 | } else { 244 | System.out.println("Max Depth of: " + maxDepth + " Reached. Sub-folder will not be traversed beyond this depth. Increase of set to -1 for unlimited depth"); 245 | } 246 | currentDepth--; 247 | } 248 | 249 | @Override 250 | public void execute(Environment environment, CommandLine cmd, ConsoleReader consoleReader) { 251 | // reset counter. 252 | count = 0; 253 | System.out.println("Beginning 'lsp' collection."); 254 | 255 | // Get the Filesystem 256 | configuration = (Configuration) env.getValue(Constants.CFG); 257 | 258 | String hdfs_uri = (String) env.getProperty(Constants.HDFS_URL); 259 | 260 | fs = (FileSystem) env.getValue(Constants.HDFS); 261 | 262 | if (fs == null) { 263 | System.out.println("Please connect first"); 264 | return; 265 | } 266 | 267 | URI nnURI = fs.getUri(); 268 | 269 | try { 270 | dfsClient = new DFSClient(nnURI, configuration); 271 | } catch (IOException e) { 272 | e.printStackTrace(); 273 | return; 274 | } 275 | 276 | Option[] cmdOpts = cmd.getOptions(); 277 | String[] cmdArgs = cmd.getArgs(); 278 | 279 | if (cmd.hasOption("maxDepth")) { 280 | setMaxDepth(Integer.parseInt(cmd.getOptionValue("maxDepth"))); 281 | } else { 282 | setMaxDepth(DEFAULT_DEPTH); 283 | } 284 | 285 | if (cmd.hasOption("format")) { 286 | setFormat(cmd.getOptionValue("format")); 287 | } else { 288 | setFormat(DEFAULT_FORMAT); 289 | } 290 | 291 | String outputFile = null; 292 | 293 | if (cmd.hasOption("output")) { 294 | outputFile = buildPath2(fs.getWorkingDirectory().toString().substring(((String) env.getProperty(Constants.HDFS_URL)).length()), cmd.getOptionValue("output")); 295 | Path pof = new Path(outputFile); 296 | try { 297 | if (fs.exists(pof)) 298 | fs.delete(pof, false); 299 | outFS = fs.create(pof); 300 | } catch (IOException e) { 301 | e.printStackTrace(); 302 | } 303 | 304 | } 305 | 306 | String targetPath = null; 307 | if (cmdArgs.length > 0) { 308 | String pathIn = cmdArgs[0]; 309 | targetPath = buildPath2(fs.getWorkingDirectory().toString().substring(((String) env.getProperty(Constants.HDFS_URL)).length()), pathIn); 310 | } else { 311 | targetPath = fs.getWorkingDirectory().toString().substring(((String) env.getProperty(Constants.HDFS_URL)).length()); 312 | 313 | } 314 | 315 | currentDepth = 0; 316 | 317 | PathData targetPathData = null; 318 | try { 319 | targetPathData = new PathData(targetPath, configuration); 320 | } catch (IOException e) { 321 | e.printStackTrace(); 322 | return; 323 | } 324 | 325 | processPath(targetPathData); 326 | 327 | if (outFS != null) 328 | try { 329 | outFS.close(); 330 | } catch (IOException e) { 331 | e.printStackTrace(); 332 | } finally { 333 | outFS = null; 334 | } 335 | System.out.println(); 336 | System.out.println("'lsp' complete."); 337 | 338 | } 339 | 340 | @Override 341 | public Options getOptions() { 342 | Options opts = super.getOptions(); 343 | 344 | // opts.addOption("r", "recurse", false, "recurse (default false)"); 345 | 346 | Option formatOption = Option.builder("f").required(false) 347 | .argName("output-format") 348 | .desc("Comma separated list of one or more: permissions_long,replication,user,group,size,block_size,ratio,mod,access,path,datanode_info (default all of the above)") 349 | .hasArg(true) 350 | .numberOfArgs(1) 351 | .valueSeparator(',') 352 | .longOpt("format") 353 | .build(); 354 | opts.addOption(formatOption); 355 | 356 | Option depthOption = Option.builder("d").required(false) 357 | .argName("maxDepth") 358 | .desc("Depth of Recursion (default 5), use '-1' for unlimited") 359 | .hasArg(true) 360 | .numberOfArgs(1) 361 | .longOpt("maxDepth") 362 | .build(); 363 | opts.addOption(depthOption); 364 | 365 | Option outputOption = Option.builder("o").required(false) 366 | .argName("output") 367 | .desc("Output File (HDFS) (default System.out)") 368 | .hasArg(true) 369 | .numberOfArgs(1) 370 | .longOpt("output") 371 | .build(); 372 | opts.addOption(outputOption); 373 | 374 | return opts; 375 | } 376 | 377 | } -------------------------------------------------------------------------------- /src/main/java/com/dstreev/hadoop/util/HdfsNNStats.java: -------------------------------------------------------------------------------- 1 | package com.dstreev.hadoop.util; 2 | 3 | import com.dstreev.hdfs.shell.command.Constants; 4 | import com.dstreev.hdfs.shell.command.Direction; 5 | import com.instanceone.hdfs.shell.command.HdfsAbstract; 6 | import com.instanceone.stemshell.Environment; 7 | import jline.console.ConsoleReader; 8 | import org.apache.commons.cli.CommandLine; 9 | import org.apache.commons.cli.Option; 10 | import org.apache.commons.cli.Options; 11 | import org.apache.commons.io.IOUtils; 12 | import org.apache.hadoop.conf.Configuration; 13 | import org.apache.hadoop.fs.FSDataOutputStream; 14 | import org.apache.hadoop.fs.Path; 15 | import org.apache.hadoop.hdfs.DFSClient; 16 | import org.apache.hadoop.hdfs.DistributedFileSystem; 17 | 18 | import java.io.IOException; 19 | import java.net.MalformedURLException; 20 | import java.net.URI; 21 | import java.net.URL; 22 | import java.net.URLConnection; 23 | import java.text.DateFormat; 24 | import java.text.SimpleDateFormat; 25 | import java.util.Date; 26 | import java.util.LinkedHashMap; 27 | import java.util.List; 28 | import java.util.Map; 29 | 30 | /** 31 | * Created by dstreev on 2016-02-15. 32 | *

33 | * The intent here is to provide a means of querying the Namenode and 34 | * producing Metadata about the directory AND the files in it. 35 | */ 36 | public class HdfsNNStats extends HdfsAbstract { 37 | private Configuration configuration = null; 38 | // private DFSClient dfsClient = null; 39 | private FSDataOutputStream outFS = null; 40 | private String baseOutputDir = null; 41 | 42 | private DistributedFileSystem fs = null; 43 | 44 | private static String DEFAULT_FILE_FORMAT = "yyyy-MM"; 45 | 46 | private DateFormat dfFile = null; 47 | 48 | // private static String FS_STATE_JMX_BEAN = "Hadoop:service=NameNode,name=FSNamesystemState"; 49 | // private static String NN_INFO_JMX_BEAN = "Hadoop:service=NameNode,name=NameNodeInfo"; 50 | 51 | // enum TARGET_BEAN { 52 | // FS_STATE_JMX_BEAN("Hadoop:service=NameNode,name=FSNamesystemState"), 53 | //// NN_STATUS_JMX_BEAN("Hadoop:service=NameNode,name=NameNodeStatus"), 54 | // NN_INFO_JMX_BEAN("Hadoop:service=NameNode,name=NameNodeInfo"); 55 | // 56 | // private String beanName; 57 | // 58 | // public String getBeanName() { 59 | // return beanName; 60 | // } 61 | // private TARGET_BEAN(String beanName) { 62 | // this.beanName = beanName; 63 | // } 64 | // } 65 | 66 | public HdfsNNStats(String name) { 67 | super(name); 68 | } 69 | 70 | public HdfsNNStats(String name, Environment env, Direction directionContext) { 71 | super(name, env, directionContext); 72 | } 73 | 74 | public HdfsNNStats(String name, Environment env, Direction directionContext, int directives) { 75 | super(name, env, directionContext, directives); 76 | } 77 | 78 | public HdfsNNStats(String name, Environment env, Direction directionContext, int directives, boolean directivesBefore, boolean directivesOptional) { 79 | super(name, env, directionContext, directives, directivesBefore, directivesOptional); 80 | } 81 | 82 | public HdfsNNStats(String name, Environment env) { 83 | super(name, env); 84 | } 85 | 86 | protected void getHelp() { 87 | StringBuilder sb = new StringBuilder(); 88 | sb.append("Collect Namenode stats from the available Namenode JMX url's.").append("\n"); 89 | sb.append("").append("\n"); 90 | sb.append("3 Type of stats are current collected and written to hdfs (with -o option) or to screen (no option specified)").append("\n"); 91 | sb.append("The 'default' delimiter for all records is '\u0001' (Cntl-A)").append("\n"); 92 | sb.append("").append("\n"); 93 | sb.append(">> Namenode Information: (optionally written to the directory 'nn_info')").append("\n"); 94 | sb.append("Fields: Timestamp, HostAndPort, State, Version, Used, Free, Safemode, TotalBlocks, TotalFiles, NumberOfMissingBlocks, NumberOfMissingBlocksWithReplicationFactorOne").append("\n"); 95 | sb.append("").append("\n"); 96 | sb.append(">> Filesystem State: (optionally written to the directory 'fs_state')").append("\n"); 97 | sb.append("Fields: Timestamp, HostAndPort, State, CapacityUsed, CapacityRemaining, BlocksTotal, PendingReplicationBlocks, UnderReplicatedBlocks, ScheduledReplicationBlocks, PendingDeletionBlocks, FSState, NumLiveDataNodes, NumDeadDataNodes, NumDecomLiveDataNodes, NumDecomDeadDataNodes, VolumeFailuresTotal").append("\n"); 98 | sb.append("").append("\n"); 99 | sb.append(">> Top User Operations: (optionally written to the directory 'top_user_ops')").append("\n"); 100 | sb.append("Fields: Timestamp, HostAndPort, State, WindowLenMs, Operation, User, Count").append("\n"); 101 | sb.append("").append("\n"); 102 | System.out.println(sb.toString()); 103 | } 104 | 105 | @Override 106 | public void execute(Environment environment, CommandLine cmd, ConsoleReader consoleReader) { 107 | 108 | if (cmd.hasOption("help")) { 109 | getHelp(); 110 | return; 111 | } 112 | 113 | System.out.println("Beginning 'Namenode Stat' collection."); 114 | 115 | // Get the Filesystem 116 | configuration = (Configuration) env.getValue(Constants.CFG); 117 | 118 | fs = (DistributedFileSystem) env.getValue(Constants.HDFS); 119 | 120 | if (fs == null) { 121 | System.out.println("Please connect first"); 122 | return; 123 | } 124 | 125 | URI nnURI = fs.getUri(); 126 | 127 | // Find the hdfs http urls. 128 | Map> namenodeJmxUrls = getNamenodeHTTPUrls(configuration); 129 | 130 | Option[] cmdOpts = cmd.getOptions(); 131 | String[] cmdArgs = cmd.getArgs(); 132 | 133 | if (cmd.hasOption("fileFormat")) { 134 | dfFile = new SimpleDateFormat(cmd.getOptionValue("fileFormat")); 135 | } else { 136 | dfFile = new SimpleDateFormat(DEFAULT_FILE_FORMAT); 137 | } 138 | 139 | if (cmd.hasOption("output")) { 140 | baseOutputDir = buildPath2(fs.getWorkingDirectory().toString().substring(((String) env.getProperty(Constants.HDFS_URL)).length()), cmd.getOptionValue("output")); 141 | } else { 142 | baseOutputDir = null; 143 | } 144 | 145 | // For each URL. 146 | for (Map.Entry> entry : namenodeJmxUrls.entrySet()) { 147 | try { 148 | 149 | URLConnection statusConnection = entry.getKey().openConnection(); 150 | String statusJson = IOUtils.toString(statusConnection.getInputStream()); 151 | 152 | 153 | for (Map.Entry innerEntry : entry.getValue().entrySet()) { 154 | // System.out.println(entry.getKey() + ": " + innerEntry.getValue()); 155 | 156 | URLConnection httpConnection = innerEntry.getValue().openConnection(); 157 | String beanJson = IOUtils.toString(httpConnection.getInputStream()); 158 | 159 | NamenodeJmxParser njp = null; 160 | 161 | String outputFilename = dfFile.format(new Date()) + ".txt"; 162 | njp = new NamenodeJmxParser(statusJson, beanJson); 163 | 164 | // URL Query should match key. 165 | switch (innerEntry.getKey()) { 166 | case NN_INFO_JMX_BEAN: 167 | // Get and Save NN Info. 168 | String nnInfo = njp.getNamenodeInfo(); 169 | // Open NN Info File. 170 | if (baseOutputDir != null) { 171 | String nnInfoPathStr = null; 172 | if (baseOutputDir.endsWith("/")) { 173 | nnInfoPathStr = baseOutputDir + "nn_info/" + outputFilename; 174 | } else { 175 | nnInfoPathStr = baseOutputDir + "/nn_info/" + outputFilename; 176 | } 177 | 178 | Path nnInfoPath = new Path(nnInfoPathStr); 179 | FSDataOutputStream nnInfoOut = null; 180 | if (fs.exists(nnInfoPath)) { 181 | // System.out.println("NN Info APPENDING"); 182 | nnInfoOut = fs.append(nnInfoPath); 183 | } else { 184 | // System.out.println("NN Info CREATING"); 185 | nnInfoOut = fs.create(nnInfoPath); 186 | } 187 | nnInfoOut.write(nnInfo.getBytes()); 188 | // Newline 189 | nnInfoOut.write("\n".getBytes()); 190 | nnInfoOut.close(); 191 | } else { 192 | System.out.println(">> Namenode Info: "); 193 | System.out.println(nnInfo); 194 | } 195 | break; 196 | case FS_STATE_JMX_BEAN: 197 | List topUserOps = njp.getTopUserOpRecords(); 198 | // Get and Save TopUserOps 199 | // Open TopUserOps file. 200 | if (topUserOps.size() > 0) { 201 | if (baseOutputDir != null) { 202 | String topUserOpsPathStr = null; 203 | if (baseOutputDir.endsWith("/")) { 204 | topUserOpsPathStr = baseOutputDir + "top_user_ops/" + outputFilename; 205 | } else { 206 | topUserOpsPathStr = baseOutputDir + "/top_user_ops/" + outputFilename; 207 | } 208 | 209 | Path topUserOpsPath = new Path(topUserOpsPathStr); 210 | FSDataOutputStream topUserOpsOut = null; 211 | if (fs.exists(topUserOpsPath)) { 212 | // System.out.println("Top User Ops APPENDING"); 213 | topUserOpsOut = fs.append(topUserOpsPath); 214 | } else { 215 | // System.out.println("Top User Ops CREATING"); 216 | topUserOpsOut = fs.create(topUserOpsPath); 217 | } 218 | for (String record : topUserOps) { 219 | topUserOpsOut.write(record.getBytes()); 220 | // Newline 221 | topUserOpsOut.write("\n".getBytes()); 222 | 223 | } 224 | topUserOpsOut.close(); 225 | } else { 226 | System.out.println(">> Top User Operations: "); 227 | for (String record : topUserOps) { 228 | System.out.println(record); 229 | } 230 | } 231 | } 232 | 233 | // Get and Save FS State 234 | String fsState = njp.getFSState(); 235 | // Open FS State Stat File. 236 | if (baseOutputDir != null) { 237 | String fsStatePathStr = null; 238 | if (baseOutputDir.endsWith("/")) { 239 | fsStatePathStr = baseOutputDir + "fs_state/" + outputFilename; 240 | } else { 241 | fsStatePathStr = baseOutputDir + "/fs_state/" + outputFilename; 242 | } 243 | 244 | Path fsStatePath = new Path(fsStatePathStr); 245 | FSDataOutputStream fsStateOut = null; 246 | if (fs.exists(fsStatePath)) { 247 | // System.out.println("FS State APPENDING"); 248 | fsStateOut = fs.append(fsStatePath); 249 | } else { 250 | // System.out.println("FS State CREATING"); 251 | fsStateOut = fs.create(fsStatePath); 252 | } 253 | fsStateOut.write(fsState.getBytes()); 254 | // Newline 255 | fsStateOut.write("\n".getBytes()); 256 | fsStateOut.close(); 257 | } else { 258 | System.out.println(">> Filesystem State: "); 259 | System.out.println(fsState); 260 | } 261 | break; 262 | } 263 | 264 | 265 | } 266 | } catch (Throwable e) { 267 | e.printStackTrace(); 268 | } 269 | 270 | } 271 | // Get Namenode Info 272 | // Open File for Append. 273 | if (fs instanceof DistributedFileSystem) { 274 | System.out.println("Filesystem Reference is Distributed"); 275 | } else { 276 | System.out.println("Filesystem Reference is NOT distributed"); 277 | } 278 | 279 | } 280 | 281 | private Map> getNamenodeHTTPUrls(Configuration configuration) { 282 | Map> rtn = new LinkedHashMap>(); 283 | 284 | // Determine if HA is enabled. 285 | // Look for 'dfs.nameservices', if present, then HA is enabled. 286 | String nameServices = configuration.get("dfs.nameservices"); 287 | if (nameServices != null) { 288 | // HA Enabled 289 | String[] nnRefs = configuration.get("dfs.ha.namenodes." + nameServices).split(","); 290 | 291 | // Get the http addresses. 292 | for (String nnRef : nnRefs) { 293 | String hostAndPort = configuration.get("dfs.namenode.http-address." + nameServices + "." + nnRef); 294 | if (hostAndPort != null) { 295 | try { 296 | URL statusURL = new URL("http://" + hostAndPort + "/jmx?qry=" + NamenodeJmxParser.NN_STATUS_JMX_BEAN); 297 | rtn.put(statusURL, getURLMap(hostAndPort)); 298 | } catch (MalformedURLException e) { 299 | e.printStackTrace(); 300 | } 301 | } 302 | } 303 | } else { 304 | // Standalone 305 | String hostAndPort = configuration.get("dfs.namenode.http-address"); 306 | try { 307 | URL statusURL = new URL("http://" + hostAndPort + "/jmx?qry=" + NamenodeJmxParser.NN_STATUS_JMX_BEAN); 308 | rtn.put(statusURL, getURLMap(hostAndPort)); 309 | } catch (MalformedURLException e) { 310 | e.printStackTrace(); 311 | } 312 | } 313 | 314 | return rtn; 315 | } 316 | 317 | private Map getURLMap(String hostAndPort) { 318 | Map rtn = new LinkedHashMap<>(); 319 | try { 320 | for (NamenodeJmxBean target_bean : NamenodeJmxBean.values()) { 321 | rtn.put(target_bean, new URL("http://" + hostAndPort + "/jmx?qry=" + target_bean.getBeanName())); 322 | } 323 | } catch (MalformedURLException e) { 324 | e.printStackTrace(); 325 | } 326 | return rtn; 327 | } 328 | 329 | @Override 330 | public Options getOptions() { 331 | Options opts = super.getOptions(); 332 | 333 | Option helpOption = Option.builder("h").required(false) 334 | .argName("help") 335 | .desc("Help") 336 | .hasArg(false) 337 | .longOpt("help") 338 | .build(); 339 | opts.addOption(helpOption); 340 | 341 | Option formatOption = Option.builder("ff").required(false) 342 | .argName("fileFormat") 343 | .desc("Output filename format. Value must be a pattern of 'SimpleDateFormat' format options.") 344 | .hasArg(true) 345 | .numberOfArgs(1) 346 | .longOpt("fileFormat") 347 | .build(); 348 | opts.addOption(formatOption); 349 | 350 | Option outputOption = Option.builder("o").required(false) 351 | .argName("output") 352 | .desc("Output Base Directory (HDFS) (default System.out) from which all other sub-directories are based.") 353 | .hasArg(true) 354 | .numberOfArgs(1) 355 | .longOpt("output") 356 | .build(); 357 | opts.addOption(outputOption); 358 | 359 | return opts; 360 | } 361 | 362 | } -------------------------------------------------------------------------------- /src/main/java/com/dstreev/hadoop/util/HdfsRepair.java: -------------------------------------------------------------------------------- 1 | package com.dstreev.hadoop.util; 2 | 3 | import com.dstreev.hdfs.shell.command.Direction; 4 | import com.instanceone.hdfs.shell.command.HdfsAbstract; 5 | import com.instanceone.stemshell.Environment; 6 | import jline.console.ConsoleReader; 7 | import org.apache.commons.cli.CommandLine; 8 | import org.apache.commons.cli.Options; 9 | 10 | /** 11 | * Created by dstreev on 2015-11-22. 12 | * 13 | * Syntax: 14 | * repair [-n ] [path] 15 | * repair -n 100 /user 16 | */ 17 | public class HdfsRepair extends HdfsAbstract { 18 | 19 | public HdfsRepair(String name) { 20 | super(name); 21 | } 22 | 23 | public HdfsRepair(String name, Environment env, Direction directionContext ) { 24 | super(name, env, directionContext); 25 | } 26 | 27 | public HdfsRepair(String name, Environment env, Direction directionContext, int directives ) { 28 | super(name,env,directionContext,directives); 29 | } 30 | 31 | public HdfsRepair(String name, Environment env, Direction directionContext, int directives, boolean directivesBefore, boolean directivesOptional ) { 32 | super(name,env,directionContext,directives,directivesBefore,directivesOptional); 33 | } 34 | 35 | public HdfsRepair(String name, Environment env) { 36 | super(name,env); 37 | } 38 | 39 | @Override 40 | public void execute(Environment environment, CommandLine commandLine, ConsoleReader consoleReader) { 41 | System.out.println("Not implemented yet... :( "); 42 | /* 43 | 44 | Run fsck from the current hdfs path. 45 | 46 | Issue the repairs as they come in. We don't want to get the whole list and then repair. 47 | Question: Will these be blocking? Probably not, so we need another thread to work from a queue of entries 48 | that's populated by this process. 49 | 50 | From the OutputStream, get the files listed as "Under replicated" 51 | 52 | Split the line on ":" and issue an HDFS setrep to 3 on the file. 53 | 54 | 55 | */ 56 | 57 | } 58 | 59 | @Override 60 | public Options getOptions() { 61 | Options opts = super.getOptions(); 62 | opts.addOption("n", true, "repair file count limit"); 63 | opts.addOption("r", true, "override replication value"); 64 | return opts; 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/dstreev/hadoop/util/JmxJsonParser.java: -------------------------------------------------------------------------------- 1 | package com.dstreev.hadoop.util; 2 | 3 | import org.codehaus.jackson.JsonNode; 4 | import org.codehaus.jackson.JsonParser; 5 | import org.codehaus.jackson.map.ObjectMapper; 6 | 7 | import java.io.IOException; 8 | import java.io.InputStream; 9 | import java.util.Iterator; 10 | import java.util.Map; 11 | import java.util.TreeMap; 12 | 13 | /** 14 | * Created by dstreev on 2016-03-21. 15 | */ 16 | public class JmxJsonParser { 17 | 18 | private ObjectMapper mapper = null; 19 | private JsonNode root = null; 20 | private JsonNode beanArrayNode = null; 21 | 22 | private static String NAME = "name"; 23 | 24 | private Map> beanMap = new TreeMap>(); 25 | 26 | public JmxJsonParser(String input) throws Exception { 27 | mapper = new ObjectMapper(); 28 | try { 29 | root = mapper.readValue(input, JsonNode.class); 30 | beanArrayNode = root.get("beans"); 31 | if (beanArrayNode == null) { 32 | throw new Exception("Couldn't locate Jmx 'Beans' array."); 33 | } 34 | } catch (IOException e) { 35 | e.printStackTrace(); 36 | } 37 | 38 | } 39 | 40 | public Map getJmxBeanContent(String name) { 41 | Map rtn = null; 42 | rtn = beanMap.get(name); 43 | if (rtn == null) { 44 | for (JsonNode beanNode : beanArrayNode) { 45 | if (beanNode.get(NAME).asText().equals(name)) { 46 | Map content = new TreeMap(); 47 | // Iterator iNodes = beanNode.iterator(); 48 | Iterator> iEntries = beanNode.getFields(); 49 | 50 | while (iEntries.hasNext()) { 51 | Map.Entry entry = iEntries.next(); 52 | 53 | if (entry.getValue().isNumber()) { 54 | content.put(entry.getKey(), entry.getValue().getNumberValue()); 55 | } else { 56 | content.put(entry.getKey(), entry.getValue().asText()); 57 | } 58 | } 59 | beanMap.put(name, content); 60 | rtn = content; 61 | break; 62 | } 63 | } 64 | } 65 | return rtn; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/dstreev/hadoop/util/NamenodeJmxBean.java: -------------------------------------------------------------------------------- 1 | package com.dstreev.hadoop.util; 2 | 3 | /** 4 | * Created by dstreev on 2016-03-24. 5 | */ 6 | public enum NamenodeJmxBean { 7 | FS_STATE_JMX_BEAN("Hadoop:service=NameNode,name=FSNamesystemState"), 8 | NN_INFO_JMX_BEAN("Hadoop:service=NameNode,name=NameNodeInfo"); 9 | 10 | private String beanName; 11 | 12 | public String getBeanName() { 13 | return beanName; 14 | } 15 | 16 | NamenodeJmxBean(String beanName) { 17 | this.beanName = beanName; 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/com/dstreev/hadoop/util/NamenodeJmxParser.java: -------------------------------------------------------------------------------- 1 | package com.dstreev.hadoop.util; 2 | 3 | import org.apache.commons.io.IOUtils; 4 | import org.codehaus.jackson.JsonNode; 5 | import org.codehaus.jackson.map.ObjectMapper; 6 | 7 | import java.io.IOException; 8 | import java.io.InputStream; 9 | import java.text.DateFormat; 10 | import java.text.SimpleDateFormat; 11 | import java.util.*; 12 | 13 | /** 14 | * Created by dstreev on 2016-03-21. 15 | */ 16 | public class NamenodeJmxParser { 17 | 18 | public static String NN_STATUS_JMX_BEAN = "Hadoop:service=NameNode,name=NameNodeStatus"; 19 | 20 | private static String TOP_USER_OPS_COUNT = "TopUserOpCounts"; 21 | 22 | private JmxJsonParser jjp = null; 23 | private String delimiter = "\u0001"; 24 | 25 | private Map metadata = new LinkedHashMap(); 26 | 27 | 28 | public NamenodeJmxParser(String resource) throws Exception { 29 | ClassLoader classLoader = getClass().getClassLoader(); 30 | 31 | InputStream resourceStream = classLoader.getResourceAsStream(resource); 32 | // InputStream dataIn = classLoader.getResourceAsStream(resource); 33 | String resourceJson = IOUtils.toString(resourceStream); 34 | 35 | init(resourceJson, resourceJson); 36 | } 37 | 38 | public NamenodeJmxParser(String statusIn, String dataIn) throws Exception { 39 | init(statusIn, dataIn); 40 | } 41 | 42 | private void init(String statusIn, String dataIn) throws Exception { 43 | JmxJsonParser statusjp = new JmxJsonParser(statusIn); 44 | jjp = new JmxJsonParser(dataIn); 45 | 46 | // Get Host Info 47 | Map nnStatusMap = statusjp.getJmxBeanContent(NN_STATUS_JMX_BEAN); 48 | String[] statusInclude = {"HostAndPort", "State"}; 49 | 50 | DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ssZ"); 51 | metadata.put("Timestamp", df.format(new Date())); 52 | 53 | transferTo(metadata, nnStatusMap, statusInclude); 54 | 55 | } 56 | 57 | public String getDelimiter() { 58 | return delimiter; 59 | } 60 | 61 | public void setDelimiter(String delimiter) { 62 | this.delimiter = delimiter; 63 | } 64 | 65 | private void transferTo(Map target, Map source, String[] transferItems) { 66 | if (transferItems == null) { 67 | for (Map.Entry entry : source.entrySet()) { 68 | target.put(entry.getKey(), entry.getValue()); 69 | } 70 | } else { 71 | for (String transferItem : transferItems) { 72 | if (source.containsKey(transferItem)) { 73 | target.put(transferItem, source.get(transferItem)); 74 | } else { 75 | System.out.println("Source doesn't contain key: " + transferItem); 76 | } 77 | } 78 | } 79 | } 80 | 81 | private List mapValuesToList(Map map, String[] fields) { 82 | List rtn = new LinkedList<>(); 83 | if (fields == null) { 84 | for (Map.Entry entry : map.entrySet()) { 85 | rtn.add(entry.getValue().toString()); 86 | } 87 | } else { 88 | for (String field: fields) { 89 | if (map.containsKey(field)) { 90 | rtn.add(map.get(field).toString()); 91 | } else { 92 | System.out.println("Map doesn't contain key: " + field); 93 | } 94 | } 95 | } 96 | return rtn; 97 | } 98 | 99 | public String listToString(List list) { 100 | StringBuilder sb = new StringBuilder(); 101 | Iterator iItems = list.iterator(); 102 | while (iItems.hasNext()) { 103 | sb.append(iItems.next()); 104 | if (iItems.hasNext()) { 105 | sb.append(delimiter); 106 | } 107 | } 108 | return sb.toString(); 109 | } 110 | 111 | public List getTopUserOpRecords() { 112 | List rtn = null; 113 | Map fsState = jjp.getJmxBeanContent(NamenodeJmxBean.FS_STATE_JMX_BEAN.getBeanName()); 114 | 115 | ObjectMapper mapper = new ObjectMapper(); 116 | String tuocStr = fsState.get(TOP_USER_OPS_COUNT).toString(); 117 | 118 | try { 119 | JsonNode tuocNode = mapper.readValue(tuocStr.getBytes(), JsonNode.class); 120 | rtn = buildTopUserOpsCountRecords(tuocNode); 121 | } catch (IOException e) { 122 | e.printStackTrace(); 123 | } 124 | 125 | return rtn; 126 | } 127 | 128 | public String getNamenodeInfo() { 129 | 130 | List working = mapValuesToList(metadata, null); 131 | 132 | Map nnInfo = jjp.getJmxBeanContent(NamenodeJmxBean.NN_INFO_JMX_BEAN.getBeanName()); 133 | 134 | String[] fields = {"Version", "Used", "Free", "Safemode", "TotalBlocks", "TotalFiles", "NumberOfMissingBlocks", "NumberOfMissingBlocksWithReplicationFactorOne"}; 135 | 136 | List fieldList = mapValuesToList(nnInfo, fields); 137 | 138 | working.addAll(fieldList); 139 | 140 | return listToString(working); 141 | // return working; 142 | 143 | /* 144 | "name" : "Hadoop:service=NameNode,name=NameNodeInfo", 145 | "modelerType" : "org.apache.hadoop.hdfs.server.namenode.FSNamesystem", 146 | "Total" : 1254254346240, 147 | "UpgradeFinalized" : true, 148 | "ClusterId" : "CID-b255ee79-e4f1-44a8-b134-044c25d7bfd4", 149 | "Version" : "2.7.1.2.3.5.1-68, rfe3c6b6dd1526d3c46f61a2e8fab9bb5eb649989", 150 | "Used" : 43518906368, 151 | "Free" : 1203736371256, 152 | "Safemode" : "", 153 | "NonDfsUsedSpace" : 6999068616, 154 | "PercentUsed" : 3.4697034, 155 | "BlockPoolUsedSpace" : 43518906368, 156 | "PercentBlockPoolUsed" : 3.4697034, 157 | "PercentRemaining" : 95.972275, 158 | "CacheCapacity" : 0, 159 | "CacheUsed" : 0, 160 | "TotalBlocks" : 7813, 161 | "TotalFiles" : 9555, 162 | "NumberOfMissingBlocks" : 0, 163 | "NumberOfMissingBlocksWithReplicationFactorOne" : 0, 164 | 165 | */ 166 | } 167 | 168 | public String getFSState() { 169 | 170 | List working = mapValuesToList(metadata, null); 171 | 172 | Map fsState = jjp.getJmxBeanContent(NamenodeJmxBean.FS_STATE_JMX_BEAN.getBeanName()); 173 | 174 | String[] fields = {"CapacityUsed", "CapacityRemaining", "BlocksTotal", "PendingReplicationBlocks", "UnderReplicatedBlocks", "ScheduledReplicationBlocks", "PendingDeletionBlocks", "FSState", "NumLiveDataNodes", "NumDeadDataNodes", "NumDecomLiveDataNodes", "NumDecomDeadDataNodes", "VolumeFailuresTotal"}; 175 | 176 | 177 | List fieldList = mapValuesToList(fsState, fields); 178 | 179 | working.addAll(fieldList); 180 | 181 | return listToString(working); 182 | 183 | /* 184 | "CapacityTotal" : 1254254346240, 185 | "CapacityUsed" : 43518906368, 186 | "CapacityRemaining" : 1203736371256, 187 | "TotalLoad" : 36, 188 | "SnapshotStats" : "{\"SnapshottableDirectories\":4,\"Snapshots\":8}", 189 | "FsLockQueueLength" : 0, 190 | "BlocksTotal" : 7813, 191 | "MaxObjects" : 0, 192 | "FilesTotal" : 9555, 193 | "PendingReplicationBlocks" : 0, 194 | "UnderReplicatedBlocks" : 4, 195 | "ScheduledReplicationBlocks" : 0, 196 | "PendingDeletionBlocks" : 0, 197 | "BlockDeletionStartTime" : 1458341216736, 198 | "FSState" : "Operational", 199 | "NumLiveDataNodes" : 3, 200 | "NumDeadDataNodes" : 0, 201 | "NumDecomLiveDataNodes" : 0, 202 | "NumDecomDeadDataNodes" : 0, 203 | "VolumeFailuresTotal" : 0, 204 | "EstimatedCapacityLostTotal" : 0, 205 | "NumDecommissioningDataNodes" : 0, 206 | "NumStaleDataNodes" : 0, 207 | "NumStaleStorages" : 0, 208 | 209 | */ 210 | } 211 | 212 | private List buildTopUserOpsCountRecords(JsonNode topNode) { 213 | List rtn = null; 214 | if (topNode != null) { 215 | rtn = new ArrayList(); 216 | try { 217 | StringBuilder sbHeader = new StringBuilder(); 218 | // Build the Key for the Record. 219 | for (String key : metadata.keySet()) { 220 | sbHeader.append(metadata.get(key)).append(delimiter); 221 | } 222 | 223 | // Cycle through the Windows 224 | for (JsonNode wNode : topNode.get("windows")) { 225 | StringBuilder sbWindow = new StringBuilder(sbHeader); 226 | 227 | sbWindow.append(wNode.get("windowLenMs").asText()).append(delimiter); 228 | // Cycle through the Operations. 229 | for (JsonNode opNode : wNode.get("ops")) { 230 | // Get Op Type 231 | StringBuilder sbOp = new StringBuilder(sbWindow); 232 | sbOp.append(opNode.get("opType").asText()).append(delimiter); 233 | // Cycle through the Users. 234 | for (JsonNode userNode : opNode.get("topUsers")) { 235 | StringBuilder sbUser = new StringBuilder(sbOp); 236 | sbUser.append(userNode.get("user").asText()).append(delimiter); 237 | sbUser.append(userNode.get("count").asText()); 238 | // Add to the list. 239 | rtn.add(sbUser.toString()); 240 | } 241 | } 242 | } 243 | 244 | } catch (Throwable t) { 245 | t.printStackTrace(); 246 | } 247 | } 248 | return rtn; 249 | } 250 | } 251 | -------------------------------------------------------------------------------- /src/main/java/com/dstreev/hdfs/shell/command/Constants.java: -------------------------------------------------------------------------------- 1 | package com.dstreev.hdfs.shell.command; 2 | 3 | /** 4 | * Created by dstreev on 2015-11-22. 5 | */ 6 | public interface Constants { 7 | 8 | String HDFS_URL = "hdfs.url"; 9 | String HDFS = "hdfs.fs"; 10 | String LOCAL_FS = "local.fs"; 11 | String CFG = "config"; 12 | 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/com/dstreev/hdfs/shell/command/Direction.java: -------------------------------------------------------------------------------- 1 | package com.dstreev.hdfs.shell.command; 2 | 3 | /** 4 | * Created by dstreev on 2015-11-22. 5 | */ 6 | public enum Direction { 7 | LOCAL_REMOTE, 8 | REMOTE_LOCAL, 9 | REMOTE_REMOTE, 10 | NONE 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/com/dstreev/hdfs/shell/command/LocalCat.java: -------------------------------------------------------------------------------- 1 | package com.instanceone.hdfs.shell.command; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.io.InputStreamReader; 7 | 8 | import com.dstreev.hdfs.shell.command.Constants; 9 | import jline.console.ConsoleReader; 10 | import jline.console.completer.Completer; 11 | 12 | import org.apache.commons.cli.CommandLine; 13 | import org.apache.commons.cli.Options; 14 | import org.apache.hadoop.fs.FileSystem; 15 | import org.apache.hadoop.fs.Path; 16 | 17 | import com.instanceone.hdfs.shell.completers.FileSystemNameCompleter; 18 | import com.instanceone.stemshell.Environment; 19 | 20 | /** 21 | * Created by dstreev on 2015-11-22. 22 | */ 23 | 24 | public class LocalCat extends HdfsCommand { 25 | 26 | public static final int LINE_COUNT = 10; 27 | 28 | private Environment env; 29 | private boolean local = false; 30 | 31 | public LocalCat(String name, Environment env, boolean local) { 32 | super(name, env); 33 | // this.env = env; 34 | this.local = local; 35 | } 36 | 37 | public void execute(Environment env, CommandLine cmd, ConsoleReader console) { 38 | FileSystem hdfs = this.local ? (FileSystem)env.getValue(Constants.LOCAL_FS) : (FileSystem)env.getValue(Constants.HDFS); 39 | logv(cmd, "CWD: " + hdfs.getWorkingDirectory()); 40 | 41 | if(cmd.getArgs().length == 1){ 42 | Path path = new Path(hdfs.getWorkingDirectory(), cmd.getArgs()[0]); 43 | BufferedReader reader = null; 44 | try { 45 | InputStream is = hdfs.open(path); 46 | InputStreamReader isr = new InputStreamReader(is); 47 | reader = new BufferedReader(isr); 48 | String line = null; 49 | for(int i = 0; (line = reader.readLine()) != null;i++ ){ 50 | log(cmd, line); 51 | } 52 | } 53 | catch (IOException e) { 54 | log(cmd, "Error reading file '" + cmd.getArgs()[0] + "': " + e.getMessage()); 55 | } finally{ 56 | try { 57 | if(reader != null){ 58 | reader.close(); 59 | } 60 | } 61 | catch (IOException e) { 62 | e.printStackTrace(); 63 | } 64 | } 65 | } else{ 66 | // usage(); 67 | } 68 | FSUtil.prompt(env); 69 | } 70 | 71 | @Override 72 | public Options getOptions() { 73 | Options opts = super.getOptions(); 74 | return opts; 75 | } 76 | 77 | @Override 78 | public Completer getCompleter() { 79 | return new FileSystemNameCompleter(this.env, this.local); 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/com/dstreev/hdfs/shell/command/LocalHead.java: -------------------------------------------------------------------------------- 1 | package com.instanceone.hdfs.shell.command; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.io.InputStreamReader; 7 | 8 | import com.dstreev.hdfs.shell.command.Constants; 9 | import jline.console.ConsoleReader; 10 | import jline.console.completer.Completer; 11 | 12 | import org.apache.commons.cli.CommandLine; 13 | import org.apache.commons.cli.Options; 14 | import org.apache.hadoop.fs.FileSystem; 15 | import org.apache.hadoop.fs.Path; 16 | 17 | import com.instanceone.hdfs.shell.completers.FileSystemNameCompleter; 18 | import com.instanceone.stemshell.Environment; 19 | 20 | /** 21 | * Created by dstreev on 2015-11-22. 22 | */ 23 | 24 | public class LocalHead extends HdfsCommand { 25 | 26 | public static final int LINE_COUNT = 10; 27 | 28 | private Environment env; 29 | private boolean local = false; 30 | 31 | public LocalHead(String name, Environment env, boolean local) { 32 | super(name, env); 33 | this.env = env; 34 | this.local = local; 35 | } 36 | 37 | public void execute(Environment env, CommandLine cmd, ConsoleReader console) { 38 | FileSystem hdfs = this.local ? (FileSystem) env.getValue(Constants.LOCAL_FS) 39 | : (FileSystem) env.getValue(Constants.HDFS); 40 | logv(cmd, "CWD: " + hdfs.getWorkingDirectory()); 41 | 42 | if (cmd.getArgs().length == 1) { 43 | int lineCount = Integer.parseInt(cmd.getOptionValue("n", 44 | String.valueOf(LINE_COUNT))); 45 | Path path = new Path(hdfs.getWorkingDirectory(), cmd.getArgs()[0]); 46 | BufferedReader reader = null; 47 | try { 48 | InputStream is = hdfs.open(path); 49 | InputStreamReader isr = new InputStreamReader(is); 50 | reader = new BufferedReader(isr); 51 | String line = null; 52 | for (int i = 0; ((i <= lineCount) && (line = reader.readLine()) != null); i++) { 53 | log(cmd, line); 54 | } 55 | } 56 | catch (IOException e) { 57 | log(cmd, "Error reading file '" + cmd.getArgs()[0] 58 | + "': " + e.getMessage()); 59 | } 60 | finally { 61 | try { 62 | if (reader != null) { 63 | reader.close(); 64 | } 65 | } 66 | catch (IOException e) { 67 | e.printStackTrace(); 68 | } 69 | } 70 | } 71 | else { 72 | // usage(); 73 | } 74 | FSUtil.prompt(env); 75 | } 76 | 77 | @Override 78 | public Options getOptions() { 79 | Options opts = super.getOptions(); 80 | opts.addOption("n", "linecount", true, 81 | "number of lines to display (defaults to 10)"); 82 | return opts; 83 | } 84 | 85 | @Override 86 | public Completer getCompleter() { 87 | return new FileSystemNameCompleter(this.env, this.local); 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/com/dstreev/hdfs/shell/command/LocalMkdir.java: -------------------------------------------------------------------------------- 1 | package com.dstreev.hdfs.shell.command; 2 | 3 | import java.io.IOException; 4 | 5 | import com.dstreev.hdfs.shell.command.Constants; 6 | import com.instanceone.hdfs.shell.command.FSUtil; 7 | import com.instanceone.hdfs.shell.command.HdfsCommand; 8 | 9 | import jline.console.ConsoleReader; 10 | import jline.console.completer.Completer; 11 | 12 | import org.apache.commons.cli.CommandLine; 13 | import org.apache.commons.cli.Options; 14 | import org.apache.hadoop.fs.FileSystem; 15 | import org.apache.hadoop.fs.Path; 16 | 17 | import com.instanceone.hdfs.shell.completers.FileSystemNameCompleter; 18 | import com.instanceone.stemshell.Environment; 19 | 20 | /** 21 | * Created by dstreev on 2015-11-22. 22 | */ 23 | 24 | public class LocalMkdir extends HdfsCommand { 25 | 26 | public static final int LINE_COUNT = 10; 27 | 28 | private Environment env; 29 | private boolean local = false; 30 | 31 | public LocalMkdir(String name, Environment env, boolean local) { 32 | super(name, env); 33 | // this.env = env; 34 | this.local = local; 35 | } 36 | 37 | public void execute(Environment env, CommandLine cmd, ConsoleReader console) { 38 | FileSystem hdfs = this.local ? (FileSystem) env.getValue(Constants.LOCAL_FS) 39 | : (FileSystem) env.getValue(Constants.HDFS); 40 | logv(cmd, "CWD: " + hdfs.getWorkingDirectory()); 41 | 42 | if (cmd.getArgs().length == 1) { 43 | Path path = new Path(hdfs.getWorkingDirectory(), cmd.getArgs()[0]); 44 | 45 | try { 46 | logv(cmd, "Create directory: " + path); 47 | hdfs.mkdirs(path); 48 | 49 | } 50 | catch (IOException e) { 51 | log(cmd, "Error creating directory '" + cmd.getArgs()[0] 52 | + "': " + e.getMessage()); 53 | } 54 | } 55 | else { 56 | } 57 | FSUtil.prompt(env); 58 | } 59 | 60 | @Override 61 | public Options getOptions() { 62 | Options opts = super.getOptions(); 63 | return opts; 64 | } 65 | 66 | @Override 67 | public Completer getCompleter() { 68 | return new FileSystemNameCompleter(this.env, this.local); 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/com/dstreev/hdfs/shell/command/LocalRm.java: -------------------------------------------------------------------------------- 1 | package com.dstreev.hdfs.shell.command; 2 | 3 | import com.dstreev.hdfs.shell.command.Constants; 4 | import com.instanceone.hdfs.shell.command.FSUtil; 5 | import com.instanceone.hdfs.shell.command.HdfsCommand; 6 | import jline.console.ConsoleReader; 7 | 8 | import org.apache.commons.cli.CommandLine; 9 | import org.apache.commons.cli.Options; 10 | import org.apache.hadoop.fs.FileSystem; 11 | import org.apache.hadoop.fs.Path; 12 | 13 | import com.instanceone.stemshell.Environment; 14 | 15 | /** 16 | * Created by dstreev on 2015-11-22. 17 | */ 18 | 19 | public class LocalRm extends HdfsCommand { 20 | private boolean local = false; 21 | 22 | public LocalRm(String name, boolean local) { 23 | super(name); 24 | this.local = local; 25 | } 26 | 27 | public void execute(Environment env, CommandLine cmd, ConsoleReader reader) { 28 | try { 29 | FileSystem hdfs = this.local ? (FileSystem) env.getValue(Constants.LOCAL_FS) 30 | : (FileSystem) env.getValue(Constants.HDFS); 31 | String remoteFile = cmd.getArgs()[0]; 32 | 33 | logv(cmd, "HDFS file: " + remoteFile); 34 | Path hdfsPath = new Path(hdfs.getWorkingDirectory(), remoteFile); 35 | logv(cmd, "Remote path: " + hdfsPath); 36 | 37 | boolean recursive = cmd.hasOption("r"); 38 | logv(cmd, "Deleting recursively..."); 39 | hdfs.delete(hdfsPath, recursive); 40 | 41 | FSUtil.prompt(env); 42 | 43 | } 44 | catch (Throwable e) { 45 | log(cmd, "Error: " + e.getMessage()); 46 | } 47 | 48 | } 49 | 50 | @Override 51 | public Options getOptions() { 52 | Options opts = super.getOptions(); 53 | opts.addOption("r", false, "delete recursively"); 54 | return opts; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/instanceone/hdfs/shell/HdfsShell.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 P. Taylor Goetz (ptgoetz@gmail.com) 2 | 3 | package com.instanceone.hdfs.shell; 4 | 5 | import com.dstreev.hadoop.util.HdfsLsPlus; 6 | import com.dstreev.hadoop.util.HdfsNNStats; 7 | import com.dstreev.hdfs.shell.command.Direction; 8 | import com.dstreev.hdfs.shell.command.LocalMkdir; 9 | import com.dstreev.hdfs.shell.command.LocalRm; 10 | import com.instanceone.hdfs.shell.command.*; 11 | import com.dstreev.hadoop.util.HdfsRepair; 12 | import com.instanceone.hdfs.shell.command.LocalCat; 13 | import com.instanceone.hdfs.shell.command.LocalHead; 14 | import com.instanceone.stemshell.Environment; 15 | import com.instanceone.stemshell.commands.Env; 16 | import com.instanceone.stemshell.commands.Exit; 17 | import com.instanceone.stemshell.commands.Help; 18 | import com.instanceone.stemshell.commands.HistoryCmd; 19 | import jline.console.ConsoleReader; 20 | import org.apache.commons.cli.*; 21 | 22 | import java.io.BufferedReader; 23 | import java.io.File; 24 | import java.io.FileReader; 25 | import java.io.IOException; 26 | 27 | public class HdfsShell extends com.instanceone.stemshell.Shell { 28 | 29 | public static void main(String[] args) throws Exception { 30 | new HdfsShell().run(args); 31 | } 32 | 33 | private Options getOptions() { 34 | // create Options object 35 | Options options = new Options(); 36 | 37 | // add i option 38 | Option initOption = Option.builder("i").required(false) 39 | .argName("init set").desc("Initialize with set") 40 | .longOpt("init") 41 | .hasArg(true).numberOfArgs(1) 42 | .build(); 43 | options.addOption(initOption); 44 | 45 | Option helpOption = Option.builder("?").required(false) 46 | .longOpt("help") 47 | .build(); 48 | options.addOption(helpOption); 49 | 50 | // TODO: Scripting 51 | //options.addOption("f", true, "Script file"); 52 | 53 | return options; 54 | 55 | } 56 | 57 | @Override 58 | protected void preProcessInitializationArguments(String[] arguments) { 59 | super.preProcessInitializationArguments(arguments); 60 | 61 | // create Options object 62 | Options options = getOptions(); 63 | 64 | CommandLineParser parser = new DefaultParser(); 65 | CommandLine cmd = null; 66 | 67 | try { 68 | cmd = parser.parse(options, arguments); 69 | } catch (ParseException pe) { 70 | HelpFormatter formatter = new HelpFormatter(); 71 | formatter.printHelp("hdfs-cli", options); 72 | } 73 | 74 | if (cmd.hasOption("help")) { 75 | HelpFormatter formatter = new HelpFormatter(); 76 | formatter.printHelp("hdfs-cli", options); 77 | System.exit(-1); 78 | } 79 | 80 | } 81 | 82 | @Override 83 | protected void postProcessInitializationArguments(String[] arguments, ConsoleReader reader) { 84 | super.postProcessInitializationArguments(arguments, reader); 85 | 86 | // create Options object 87 | Options options = getOptions(); 88 | 89 | CommandLineParser parser = new DefaultParser(); 90 | CommandLine cmd = null; 91 | 92 | try { 93 | cmd = parser.parse(options, arguments); 94 | } catch (ParseException pe) { 95 | HelpFormatter formatter = new HelpFormatter(); 96 | formatter.printHelp("hdfs-cli", options); 97 | } 98 | 99 | autoConnect(reader); 100 | 101 | if (cmd.hasOption("init")) { 102 | initialSet(cmd.getOptionValue("init"), reader); 103 | } 104 | 105 | } 106 | 107 | private void initialSet(String set, ConsoleReader reader) { 108 | System.out.println("-- Initializing with set: " + set); 109 | 110 | File dir = new File(System.getProperty("user.home"), "." 111 | + this.getName()); 112 | if (dir.exists() && dir.isFile()) { 113 | throw new IllegalStateException( 114 | "Default configuration file exists and is not a directory: " 115 | + dir.getAbsolutePath()); 116 | } else if (!dir.exists()) { 117 | dir.mkdir(); 118 | } 119 | // directory created, touch history file 120 | File setFile = new File(dir, set); 121 | if (!setFile.exists()) { 122 | try { 123 | if (!setFile.createNewFile()) { 124 | throw new IllegalStateException( 125 | "Unable to create set file: " 126 | + setFile.getAbsolutePath()); 127 | } else { 128 | System.out.println("New Initialization File create in: " + System.getProperty("user.home") + System.getProperty("file.separator") + this.getName() + System.getProperty("file.separator") + set + ". Add commands to this file to initialized the next session"); 129 | } 130 | } catch (IOException ioe) { 131 | ioe.printStackTrace(); 132 | } 133 | } else { 134 | try { 135 | BufferedReader br = new BufferedReader(new FileReader(setFile)); 136 | String line = null; 137 | while ((line = br.readLine()) != null) { 138 | System.out.println(line); 139 | String line2 = line.trim(); 140 | if (line2.length() > 0 && !line2.startsWith("#")) { 141 | processInput(line2, reader); 142 | } 143 | } 144 | } catch (Exception e) { 145 | e.printStackTrace(); 146 | } 147 | 148 | } 149 | 150 | } 151 | 152 | private void autoConnect(ConsoleReader reader) { 153 | try { 154 | String userHome = System.getProperty("user.name"); 155 | processInput("connect", reader); 156 | processInput("cd /user/" + userHome, reader); 157 | } catch (Exception e) { 158 | e.printStackTrace(); 159 | } 160 | 161 | } 162 | 163 | private void runScript(String file, ConsoleReader reader) { 164 | 165 | } 166 | 167 | @Override 168 | public void initialize(Environment env) throws Exception { 169 | 170 | env.addCommand(new Exit("exit")); 171 | env.addCommand(new LocalLs("lls", env)); 172 | env.addCommand(new LocalPwd("lpwd")); 173 | env.addCommand(new LocalCd("lcd", env)); 174 | env.addCommand(new HdfsCd("cd", env)); 175 | env.addCommand(new HdfsPwd("pwd")); 176 | 177 | // remote local 178 | env.addCommand(new HdfsCommand("get", env, Direction.REMOTE_LOCAL)); 179 | env.addCommand(new HdfsCommand("copyFromLocal", env, Direction.LOCAL_REMOTE)); 180 | // local remote 181 | env.addCommand(new HdfsCommand("put", env, Direction.LOCAL_REMOTE)); 182 | env.addCommand(new HdfsCommand("copyToLocal", env, Direction.REMOTE_LOCAL)); 183 | // src dest 184 | env.addCommand(new HdfsCommand("cp", env, Direction.REMOTE_REMOTE)); 185 | 186 | // amend to context path, if present 187 | env.addCommand(new HdfsCommand("chown", env, Direction.NONE, 1)); 188 | env.addCommand(new HdfsCommand("chmod", env, Direction.NONE, 1)); 189 | env.addCommand(new HdfsCommand("chgrp", env, Direction.NONE, 1)); 190 | 191 | env.addCommand(new HdfsCommand("createSnapshot", env, Direction.NONE, 1, false, true)); 192 | env.addCommand(new HdfsCommand("deleteSnapshot", env, Direction.NONE, 1, false, false)); 193 | env.addCommand(new HdfsCommand("renameSnapshot", env, Direction.NONE, 2, false, false)); 194 | 195 | env.addCommand(new HdfsCommand("du", env, Direction.NONE)); 196 | env.addCommand(new HdfsCommand("df", env, Direction.NONE)); 197 | env.addCommand(new HdfsCommand("dus", env, Direction.NONE)); 198 | env.addCommand(new HdfsCommand("ls", env, Direction.NONE)); 199 | env.addCommand(new HdfsCommand("lsr", env, Direction.NONE)); 200 | // env.addCommand(new HdfsCommand("find", env, Direction.NONE, 1, false)); 201 | 202 | 203 | env.addCommand(new HdfsCommand("mkdir", env, Direction.NONE)); 204 | 205 | env.addCommand(new HdfsCommand("count", env, Direction.NONE)); 206 | env.addCommand(new HdfsCommand("stat", env, Direction.NONE)); 207 | env.addCommand(new HdfsCommand("tail", env, Direction.NONE)); 208 | env.addCommand(new HdfsCommand("head", env, Direction.NONE)); 209 | // env.addCommand(new HdfsCommand("test", env, Direction.NONE)); 210 | env.addCommand(new HdfsCommand("touchz", env, Direction.NONE)); 211 | 212 | env.addCommand(new HdfsCommand("rm", env, Direction.NONE)); 213 | env.addCommand(new HdfsCommand("rmdir", env, Direction.NONE)); 214 | env.addCommand(new HdfsCommand("mv", env, Direction.REMOTE_REMOTE)); 215 | env.addCommand(new HdfsCommand("cat", env, Direction.NONE)); 216 | env.addCommand(new HdfsCommand("text", env, Direction.NONE)); 217 | env.addCommand(new HdfsCommand("checksum", env, Direction.NONE)); 218 | env.addCommand(new HdfsCommand("usage", env)); 219 | 220 | // Security Help 221 | // env.addCommand(new HdfsUGI("ugi")); 222 | // env.addCommand(new HdfsKrb("krb", env, Direction.NONE, 1)); 223 | 224 | // HDFS Utils 225 | //env.addCommand(new HdfsRepair("repair", env, Direction.NONE, 2, true, true)); 226 | 227 | env.addCommand(new LocalHead("lhead", env, true)); 228 | env.addCommand(new LocalCat("lcat", env, true)); 229 | env.addCommand(new LocalMkdir("lmkdir", env, true)); 230 | env.addCommand(new LocalRm("lrm", true)); 231 | env.addCommand(new Env("env")); 232 | env.addCommand(new HdfsConnect("connect")); 233 | env.addCommand(new Help("help", env)); 234 | env.addCommand(new HistoryCmd("history")); 235 | 236 | // HDFS Tools 237 | env.addCommand(new HdfsLsPlus("lsp", env, Direction.NONE)); 238 | env.addCommand(new HdfsNNStats("nnstat", env, Direction.NONE)); 239 | 240 | } 241 | 242 | @Override 243 | public String getName() { 244 | return "hdfs-cli"; 245 | } 246 | 247 | } 248 | -------------------------------------------------------------------------------- /src/main/java/com/instanceone/hdfs/shell/command/FSUtil.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 P. Taylor Goetz (ptgoetz@gmail.com) 2 | 3 | package com.instanceone.hdfs.shell.command; 4 | 5 | import java.text.DateFormat; 6 | import java.text.DecimalFormat; 7 | import java.text.SimpleDateFormat; 8 | import java.util.Date; 9 | 10 | import com.dstreev.hdfs.shell.command.Constants; 11 | import com.instanceone.stemshell.Environment; 12 | import org.apache.hadoop.fs.FileStatus; 13 | 14 | import com.instanceone.stemshell.format.ANSIStyle; 15 | import org.apache.hadoop.fs.FileSystem; 16 | 17 | public class FSUtil { 18 | 19 | private FSUtil() { 20 | } 21 | 22 | public static String longFormat(FileStatus file) { 23 | String retval = (file.isDir() ? "d" : "-") 24 | + file.getPermission() 25 | + (file.getPermission().getAclBit() ? "+":"") 26 | + (file.getPermission().getEncryptedBit() ? "#":"") 27 | + "\t" 28 | + file.getOwner() 29 | + "\t" 30 | + file.getGroup() 31 | + "\t" 32 | + formatFileSize(file.getLen()) 33 | + "\t" 34 | + formatDate(file.getModificationTime()) 35 | 36 | + "\t" 37 | + (file.isDir() ? ANSIStyle.style(file.getPath() 38 | .getName(), ANSIStyle.FG_GREEN) : file 39 | .getPath().getName()); 40 | 41 | return retval; 42 | } 43 | 44 | public static String shortFormat(FileStatus file) { 45 | String retval = (file.isDir() ? ANSIStyle.style(file.getPath() 46 | .getName(), ANSIStyle.FG_GREEN) : file.getPath() 47 | .getName()); 48 | 49 | return retval; 50 | } 51 | 52 | public static void prompt(Environment env) { 53 | try { 54 | StringBuilder sb = new StringBuilder(); 55 | 56 | FileSystem localfs = (FileSystem) env.getValue(Constants.LOCAL_FS); 57 | FileSystem hdfs = (FileSystem) env.getValue(Constants.HDFS); 58 | 59 | String hdfswd = hdfs.getWorkingDirectory().toString(); 60 | String localwd = localfs.getWorkingDirectory().toString(); 61 | 62 | String hwd = ANSIStyle.style(hdfswd, ANSIStyle.FG_GREEN) ; 63 | 64 | String lwd = ANSIStyle.style(localwd, ANSIStyle.FG_YELLOW); 65 | 66 | env.setPrompt(" " + hwd + "\n " + lwd + "\n$ "); 67 | 68 | } catch (Exception e) { 69 | e.printStackTrace(); 70 | } 71 | } 72 | 73 | public static String formatDate(long millis){ 74 | Date date = new Date(millis); 75 | return formatDate(date); 76 | } 77 | 78 | public static String formatDate(Date date){ 79 | DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss z"); 80 | return df.format(date); 81 | } 82 | 83 | public static String formatFileSize(long size) { 84 | if(size <= 0) return "0"; 85 | final String[] units = new String[] { "B", "K", "M", "G", "T" }; 86 | int digitGroups = (int) (Math.log10(size)/Math.log10(1024)); 87 | return new DecimalFormat("#,##0.#").format(size/Math.pow(1024, digitGroups)) + units[digitGroups]; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/com/instanceone/hdfs/shell/command/HdfsAbstract.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 P. Taylor Goetz (ptgoetz@gmail.com) 2 | 3 | package com.instanceone.hdfs.shell.command; 4 | 5 | import com.dstreev.hdfs.shell.command.Constants; 6 | import com.dstreev.hdfs.shell.command.Direction; 7 | import com.instanceone.hdfs.shell.completers.FileSystemNameCompleter; 8 | import com.instanceone.stemshell.Environment; 9 | import com.instanceone.stemshell.command.AbstractCommand; 10 | import jline.console.completer.Completer; 11 | import org.apache.hadoop.fs.FileSystem; 12 | 13 | public abstract class HdfsAbstract extends AbstractCommand { 14 | protected Environment env; 15 | 16 | // public static final String HDFS_URL = "hdfs.url"; 17 | // public static final String HDFS = "hdfs.fs"; 18 | // public static final String LOCAL_FS = "local.fs"; 19 | // public static final String CFG = "config"; 20 | 21 | // public enum Direction { 22 | // LOCAL_REMOTE, 23 | // REMOTE_LOCAL, 24 | // REMOTE_REMOTE, 25 | // NONE; 26 | // } 27 | 28 | enum Side { 29 | LEFT,RIGHT 30 | } 31 | 32 | protected Direction directionContext = null; 33 | 34 | protected int directives = 0; 35 | protected boolean directivesBefore = true; 36 | protected boolean directivesOptional = false; 37 | 38 | public HdfsAbstract(String name) { 39 | super(name); 40 | } 41 | 42 | public HdfsAbstract(String name, Environment env, Direction directionContext ) { 43 | super(name); 44 | this.env = env; 45 | this.directionContext = directionContext; 46 | } 47 | 48 | public HdfsAbstract(String name, Environment env, Direction directionContext, int directives ) { 49 | super(name); 50 | this.env = env; 51 | this.directionContext = directionContext; 52 | this.directives = directives; 53 | } 54 | 55 | public HdfsAbstract(String name, Environment env, Direction directionContext, int directives, boolean directivesBefore, boolean directivesOptional ) { 56 | super(name); 57 | this.env = env; 58 | this.directionContext = directionContext; 59 | this.directives = directives; 60 | this.directivesBefore = directivesBefore; 61 | this.directivesOptional = directivesOptional; 62 | } 63 | 64 | public HdfsAbstract(String name, Environment env) { 65 | super(name); 66 | this.env = env; 67 | } 68 | // 69 | // public void execute(Environment env, CommandLine cmd, ConsoleReader reader) { 70 | // FsShell shell = new FsShell(); 71 | // 72 | // Configuration conf = (Configuration)env.getValue(Constants.CFG); 73 | // 74 | // String hdfs_uri = (String)env.getProperty(Constants.HDFS_URL); 75 | // 76 | // FileSystem hdfs = (FileSystem) env.getValue(Constants.HDFS); 77 | // 78 | // if (hdfs == null) { 79 | // System.out.println("Please connect first"); 80 | // } 81 | // conf.set("fs.defaultFS", hdfs_uri); 82 | // 83 | // conf.setQuietMode(false); 84 | // shell.setConf(conf); 85 | // int res; 86 | // String[] argv = null; 87 | // 88 | // Option[] cmdOpts = cmd.getOptions(); 89 | // String[] cmdArgs = cmd.getArgs(); 90 | // 91 | // // TODO: Need to Handle context aware file operations. 92 | // // put, get, mv, copy.., chmod, chown, chgrp, count 93 | // int pathCount = 0; 94 | // 95 | // String leftPath = null; 96 | // String rightPath = null; 97 | // 98 | // switch (directionContext) { 99 | // case REMOTE_LOCAL: 100 | // pathCount += 2; // Source and Destination Path Elements. 101 | // break; 102 | // case LOCAL_REMOTE: 103 | // pathCount += 2; // Source and Destination Path Elements. 104 | // 105 | // break; 106 | // case REMOTE_REMOTE: 107 | // pathCount += 2; // Source and Destination Path Elements. 108 | // 109 | // break; 110 | // default: // NONE 111 | // pathCount += 1; 112 | // } 113 | // 114 | // leftPath = buildPath(Side.LEFT, cmdArgs, directionContext); 115 | // if (directionContext != Direction.NONE) { 116 | // rightPath = buildPath(Side.RIGHT, cmdArgs, directionContext); 117 | // } 118 | // 119 | // String[] newCmdArgs = new String[pathCount]; 120 | // if (rightPath != null) { 121 | // newCmdArgs[0] = leftPath; 122 | // newCmdArgs[1] = rightPath; 123 | // } else { 124 | // newCmdArgs[0] = leftPath; 125 | // } 126 | // 127 | // argv = new String[cmdOpts.length + newCmdArgs.length + 1 + directives]; 128 | // 129 | // int pos = 1; 130 | // 131 | // for (Option opt: cmdOpts) { 132 | // argv[pos++] = "-" + opt.getOpt(); 133 | // } 134 | // 135 | // if (directivesBefore) { 136 | // for (int i = 0; i < directives; i++) { 137 | // argv[pos++] = cmdArgs[i]; 138 | // } 139 | // } 140 | // 141 | // for (String arg: newCmdArgs) { 142 | // argv[pos++] = arg; 143 | // } 144 | // 145 | // if (!directivesBefore) { 146 | // for (int i = directives; i > 0; i--) { 147 | // try { 148 | // argv[pos++] = cmdArgs[cmdArgs.length - (i)]; 149 | // } catch (Exception e) { 150 | // // Can happen when args are optional 151 | // } 152 | // } 153 | // } 154 | // 155 | // argv[0] = "-" + getName(); 156 | // 157 | // System.out.println("HDFS Command: " + Arrays.toString(argv)); 158 | // 159 | // try { 160 | // res = ToolRunner.run(shell, argv); 161 | // } catch (Exception e) { 162 | // e.printStackTrace(); 163 | // } finally { 164 | // try { 165 | // shell.close(); 166 | // } catch (IOException e) { 167 | // e.printStackTrace(); 168 | // } 169 | // } 170 | // } 171 | 172 | protected String buildPath(Side side, String[] args, Direction context) { 173 | String rtn = null; 174 | 175 | FileSystem localfs = (FileSystem)env.getValue(Constants.LOCAL_FS); 176 | FileSystem hdfs = (FileSystem) env.getValue(Constants.HDFS); 177 | 178 | String in = null; 179 | 180 | switch (side) { 181 | case LEFT: 182 | if (args.length > 0) 183 | if (directivesBefore) { 184 | in = args[directives]; 185 | } else { 186 | if (directivesOptional) { 187 | if (args.length > directives) { 188 | in = args[args.length-(directives+1)]; 189 | } else { 190 | // in is null 191 | } 192 | } else { 193 | in = args[args.length-(directives+1)]; 194 | } 195 | } 196 | switch (context) { 197 | case REMOTE_LOCAL: 198 | case REMOTE_REMOTE: 199 | case NONE: 200 | rtn = buildPath2(hdfs.getWorkingDirectory().toString().substring(((String)env.getProperty(Constants.HDFS_URL)).length()), in); 201 | break; 202 | case LOCAL_REMOTE: 203 | rtn = buildPath2(localfs.getWorkingDirectory().toString().substring(5), in); 204 | break; 205 | } 206 | break; 207 | case RIGHT: 208 | if (args.length > 1) 209 | if (directivesBefore) 210 | in = args[directives + 1]; 211 | else 212 | in = args[args.length-(directives+1)]; 213 | switch (context) { 214 | case REMOTE_LOCAL: 215 | rtn = buildPath2(localfs.getWorkingDirectory().toString().substring(5), in); 216 | break; 217 | case LOCAL_REMOTE: 218 | case REMOTE_REMOTE: 219 | rtn = buildPath2(hdfs.getWorkingDirectory().toString().substring(((String)env.getProperty(Constants.HDFS_URL)).length()), in); 220 | break; 221 | case NONE: 222 | break; 223 | } 224 | break; 225 | } 226 | if (rtn != null && rtn.contains(" ")) { 227 | rtn = "'" + rtn + "'"; 228 | } 229 | return rtn; 230 | } 231 | 232 | protected String buildPath2(String current, String input) { 233 | if (input != null) { 234 | if (input.startsWith("/")) 235 | return input; 236 | else 237 | return current + "/" + input; 238 | } else { 239 | return current; 240 | } 241 | } 242 | 243 | // @Override 244 | // public Options getOptions() { 245 | // Options opts = super.getOptions(); 246 | // opts.addOption("l", false, "show extended file attributes"); 247 | // opts.addOption("R", false, "recurse"); 248 | // opts.addOption("f", false, "force / is file"); 249 | // opts.addOption("p", false, "preserve"); 250 | // opts.addOption("h", false, "human readable"); 251 | // opts.addOption("s", false, "summary"); 252 | // opts.addOption("q", false, "query"); 253 | // opts.addOption("d", false, "dump / path is directory"); 254 | // opts.addOption("e", false, "encoding / path exists"); 255 | // opts.addOption("t", false, "sort by Timestamp"); 256 | // opts.addOption("S", false, "sort by Size / path not empty"); 257 | // opts.addOption("r", false, "reverse"); 258 | // opts.addOption("z", false, "file length is 0"); 259 | // opts.addOption("u", false, "user access time"); 260 | // opts.addOption("skipTrash", false, "Skip Trash"); 261 | // opts.addOption("ignorecrc", false, "ignorecrc"); 262 | // opts.addOption("crc", false, "crc"); 263 | // 264 | //// opts.addOption("ignore-fail-on-non-empty", false, "ignore-fail-on-non-empty"); 265 | // return opts; 266 | // } 267 | 268 | @Override 269 | public Completer getCompleter() { 270 | return new FileSystemNameCompleter(this.env, false); 271 | } 272 | 273 | 274 | } 275 | -------------------------------------------------------------------------------- /src/main/java/com/instanceone/hdfs/shell/command/HdfsCd.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 P. Taylor Goetz (ptgoetz@gmail.com) 2 | 3 | package com.instanceone.hdfs.shell.command; 4 | 5 | import java.io.IOException; 6 | 7 | import com.dstreev.hdfs.shell.command.Constants; 8 | import jline.console.ConsoleReader; 9 | import jline.console.completer.Completer; 10 | 11 | import org.apache.commons.cli.CommandLine; 12 | import org.apache.hadoop.fs.FileSystem; 13 | import org.apache.hadoop.fs.Path; 14 | 15 | import com.instanceone.hdfs.shell.completers.FileSystemNameCompleter; 16 | import com.instanceone.stemshell.Environment; 17 | 18 | public class HdfsCd extends HdfsCommand { 19 | private Environment env; 20 | 21 | public HdfsCd(String name, Environment env) { 22 | super(name, env); 23 | this.env = env; 24 | } 25 | 26 | public void execute(Environment env, CommandLine cmd, ConsoleReader reader) { 27 | FileSystem hdfs = null; 28 | try { 29 | hdfs = (FileSystem)env.getValue(Constants.HDFS); 30 | 31 | String dir = cmd.getArgs().length == 0 ? "/" : cmd.getArgs()[0]; 32 | logv(cmd, "CWD before: " + hdfs.getWorkingDirectory()); 33 | logv(cmd, "Requested CWD: " + dir); 34 | 35 | Path newPath = null; 36 | if(dir.startsWith("/")){ 37 | newPath = new Path(env.getProperty(Constants.HDFS_URL), dir); 38 | } else{ 39 | newPath = new Path(hdfs.getWorkingDirectory(), dir); 40 | } 41 | 42 | Path qPath = newPath.makeQualified(hdfs); 43 | logv(cmd, "" + newPath); 44 | if (hdfs.getFileStatus(qPath).isDir() && hdfs.exists(qPath)) { 45 | hdfs.setWorkingDirectory(qPath); 46 | } 47 | else { 48 | log(cmd, "No such directory: " + dir); 49 | } 50 | 51 | } 52 | catch (IOException e) { 53 | System.out.println(e.getMessage()); 54 | } finally { 55 | FSUtil.prompt(env); 56 | } 57 | } 58 | 59 | @Override 60 | public Completer getCompleter() { 61 | return new FileSystemNameCompleter(this.env, false); 62 | } 63 | 64 | 65 | 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/com/instanceone/hdfs/shell/command/HdfsCommand.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 P. Taylor Goetz (ptgoetz@gmail.com) 2 | 3 | package com.instanceone.hdfs.shell.command; 4 | 5 | import com.dstreev.hdfs.shell.command.Constants; 6 | import com.dstreev.hdfs.shell.command.Direction; 7 | import com.instanceone.stemshell.Environment; 8 | import jline.console.ConsoleReader; 9 | import org.apache.commons.cli.CommandLine; 10 | import org.apache.commons.cli.Option; 11 | import org.apache.commons.cli.Options; 12 | import org.apache.hadoop.conf.Configuration; 13 | import org.apache.hadoop.fs.FileSystem; 14 | import org.apache.hadoop.fs.FsShell; 15 | import org.apache.hadoop.util.ToolRunner; 16 | 17 | import java.io.IOException; 18 | import java.util.Arrays; 19 | 20 | public class HdfsCommand extends HdfsAbstract { 21 | 22 | public HdfsCommand(String name) { 23 | super(name); 24 | } 25 | 26 | public HdfsCommand(String name, Environment env, Direction directionContext ) { 27 | super(name, env, directionContext); 28 | } 29 | 30 | public HdfsCommand(String name, Environment env, Direction directionContext, int directives ) { 31 | super(name,env,directionContext,directives); 32 | } 33 | 34 | public HdfsCommand(String name, Environment env, Direction directionContext, int directives, boolean directivesBefore, boolean directivesOptional ) { 35 | super(name,env,directionContext,directives,directivesBefore,directivesOptional); 36 | } 37 | 38 | public HdfsCommand(String name, Environment env) { 39 | super(name,env); 40 | } 41 | 42 | public void execute(Environment env, CommandLine cmd, ConsoleReader reader) { 43 | FsShell shell = new FsShell(); 44 | 45 | Configuration conf = (Configuration)env.getValue(Constants.CFG); 46 | 47 | String hdfs_uri = (String)env.getProperty(Constants.HDFS_URL); 48 | 49 | FileSystem hdfs = (FileSystem) env.getValue(Constants.HDFS); 50 | 51 | if (hdfs == null) { 52 | System.out.println("Please connect first"); 53 | } 54 | conf.set("fs.defaultFS", hdfs_uri); 55 | 56 | conf.setQuietMode(false); 57 | shell.setConf(conf); 58 | int res; 59 | String[] argv = null; 60 | 61 | Option[] cmdOpts = cmd.getOptions(); 62 | String[] cmdArgs = cmd.getArgs(); 63 | 64 | // TODO: Need to Handle context aware file operations. 65 | // put, get, mv, copy.., chmod, chown, chgrp, count 66 | int pathCount = 0; 67 | 68 | String leftPath = null; 69 | String rightPath = null; 70 | 71 | switch (directionContext) { 72 | case REMOTE_LOCAL: 73 | pathCount += 2; // Source and Destination Path Elements. 74 | break; 75 | case LOCAL_REMOTE: 76 | pathCount += 2; // Source and Destination Path Elements. 77 | 78 | break; 79 | case REMOTE_REMOTE: 80 | pathCount += 2; // Source and Destination Path Elements. 81 | 82 | break; 83 | default: // NONE 84 | pathCount += 1; 85 | } 86 | 87 | leftPath = buildPath(Side.LEFT, cmdArgs, directionContext); 88 | if (directionContext != Direction.NONE) { 89 | rightPath = buildPath(Side.RIGHT, cmdArgs, directionContext); 90 | } 91 | 92 | String[] newCmdArgs = new String[pathCount]; 93 | if (rightPath != null) { 94 | newCmdArgs[0] = leftPath; 95 | newCmdArgs[1] = rightPath; 96 | } else { 97 | newCmdArgs[0] = leftPath; 98 | } 99 | 100 | argv = new String[cmdOpts.length + newCmdArgs.length + 1 + directives]; 101 | 102 | int pos = 1; 103 | 104 | for (Option opt: cmdOpts) { 105 | argv[pos++] = "-" + opt.getOpt(); 106 | } 107 | 108 | if (directivesBefore) { 109 | for (int i = 0; i < directives; i++) { 110 | argv[pos++] = cmdArgs[i]; 111 | } 112 | } 113 | 114 | for (String arg: newCmdArgs) { 115 | argv[pos++] = arg; 116 | } 117 | 118 | if (!directivesBefore) { 119 | for (int i = directives; i > 0; i--) { 120 | try { 121 | argv[pos++] = cmdArgs[cmdArgs.length - (i)]; 122 | } catch (Exception e) { 123 | // Can happen when args are optional 124 | } 125 | } 126 | } 127 | 128 | argv[0] = "-" + getName(); 129 | 130 | System.out.println("HDFS Command: " + Arrays.toString(argv)); 131 | 132 | try { 133 | res = ToolRunner.run(shell, argv); 134 | } catch (Exception e) { 135 | e.printStackTrace(); 136 | } finally { 137 | try { 138 | shell.close(); 139 | } catch (IOException e) { 140 | e.printStackTrace(); 141 | } 142 | } 143 | } 144 | 145 | @Override 146 | public Options getOptions() { 147 | Options opts = super.getOptions(); 148 | opts.addOption("l", false, "show extended file attributes"); 149 | opts.addOption("R", false, "recurse"); 150 | opts.addOption("f", false, "force / is file"); 151 | opts.addOption("p", false, "preserve"); 152 | opts.addOption("h", false, "human readable"); 153 | opts.addOption("s", false, "summary"); 154 | opts.addOption("q", false, "query"); 155 | opts.addOption("d", false, "dump / path is directory"); 156 | opts.addOption("e", false, "encoding / path exists"); 157 | opts.addOption("t", false, "sort by Timestamp"); 158 | opts.addOption("S", false, "sort by Size / path not empty"); 159 | opts.addOption("r", false, "reverse"); 160 | opts.addOption("z", false, "file length is 0"); 161 | opts.addOption("u", false, "user access time"); 162 | opts.addOption("skipTrash", false, "Skip Trash"); 163 | opts.addOption("ignorecrc", false, "ignorecrc"); 164 | opts.addOption("crc", false, "crc"); 165 | 166 | // opts.addOption("ignore-fail-on-non-empty", false, "ignore-fail-on-non-empty"); 167 | return opts; 168 | } 169 | 170 | // @Override 171 | // public Completer getCompleter() { 172 | // return new FileSystemNameCompleter(this.env, false); 173 | // } 174 | 175 | 176 | } 177 | -------------------------------------------------------------------------------- /src/main/java/com/instanceone/hdfs/shell/command/HdfsConnect.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 P. Taylor Goetz (ptgoetz@gmail.com) 2 | 3 | package com.instanceone.hdfs.shell.command; 4 | 5 | import java.io.File; 6 | import java.io.IOException; 7 | import java.net.URI; 8 | 9 | import com.dstreev.hdfs.shell.command.Constants; 10 | import com.instanceone.stemshell.command.AbstractCommand; 11 | import jline.console.ConsoleReader; 12 | import jline.console.completer.Completer; 13 | import jline.console.completer.StringsCompleter; 14 | 15 | import org.apache.commons.cli.CommandLine; 16 | import org.apache.hadoop.conf.Configuration; 17 | import org.apache.hadoop.fs.FileSystem; 18 | import org.apache.hadoop.fs.Path; 19 | 20 | import com.instanceone.stemshell.Environment; 21 | import org.apache.hadoop.security.UserGroupInformation; 22 | 23 | public class HdfsConnect extends AbstractCommand { 24 | 25 | public static final String HADOOP_CONF_DIR = "HADOOP_CONF_DIR"; 26 | private static final String[] HADOOP_CONF_FILES = {"core-site.xml", "hdfs-site.xml"}; 27 | 28 | public HdfsConnect(String name) { 29 | super(name); 30 | Completer completer = new StringsCompleter("hdfs://localhost:8020/", "hdfs://hdfshost:8020/"); 31 | this.completer = completer; 32 | } 33 | 34 | public void execute(Environment env, CommandLine cmd, ConsoleReader reader) { 35 | try { 36 | // Get a value that over rides the default, if nothing then use default. 37 | // Requires Java 1.8... 38 | // String hadoopConfDirProp = System.getenv().getOrDefault(HADOOP_CONF_DIR, "/etc/hadoop/conf"); 39 | 40 | String hadoopConfDirProp = System.getenv().get(HADOOP_CONF_DIR); 41 | // Set a default 42 | if (hadoopConfDirProp == null) 43 | hadoopConfDirProp = "/etc/hadoop/conf"; 44 | 45 | Configuration config = new Configuration(false); 46 | 47 | File hadoopConfDir = new File(hadoopConfDirProp).getAbsoluteFile(); 48 | for (String file : HADOOP_CONF_FILES) { 49 | File f = new File(hadoopConfDir, file); 50 | if (f.exists()) { 51 | config.addResource(new Path(f.getAbsolutePath())); 52 | } 53 | } 54 | 55 | FileSystem hdfs = null; 56 | try { 57 | hdfs = FileSystem.get(config); 58 | } catch (Throwable t) { 59 | t.printStackTrace(); 60 | } 61 | 62 | env.setValue(Constants.CFG, config); 63 | env.setValue(Constants.HDFS, hdfs); 64 | // set working dir to root 65 | hdfs.setWorkingDirectory(hdfs.makeQualified(new Path("/"))); 66 | 67 | FileSystem local = FileSystem.getLocal(new Configuration()); 68 | env.setValue(Constants.LOCAL_FS, local); 69 | env.setProperty(Constants.HDFS_URL, hdfs.getUri().toString()); 70 | 71 | FSUtil.prompt(env); 72 | 73 | log(cmd, "Connected: " + hdfs.getUri()); 74 | logv(cmd, "HDFS CWD: " + hdfs.getWorkingDirectory()); 75 | logv(cmd, "Local CWD: " + local.getWorkingDirectory()); 76 | 77 | } catch (IOException e) { 78 | log(cmd, e.getMessage()); 79 | } 80 | } 81 | 82 | @Override 83 | public Completer getCompleter() { 84 | return this.completer; 85 | } 86 | 87 | 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/com/instanceone/hdfs/shell/command/HdfsPwd.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 P. Taylor Goetz (ptgoetz@gmail.com) 2 | 3 | package com.instanceone.hdfs.shell.command; 4 | 5 | import com.dstreev.hdfs.shell.command.Constants; 6 | import jline.console.ConsoleReader; 7 | 8 | import org.apache.commons.cli.CommandLine; 9 | import org.apache.commons.cli.Options; 10 | import org.apache.hadoop.fs.FileSystem; 11 | 12 | import com.instanceone.stemshell.Environment; 13 | 14 | public class HdfsPwd extends HdfsCommand { 15 | 16 | public HdfsPwd(String name) { 17 | super(name); 18 | } 19 | 20 | public void execute(Environment env, CommandLine cmd, ConsoleReader reader) { 21 | FileSystem hdfs = (FileSystem) env.getValue(Constants.HDFS); 22 | String wd = hdfs.getWorkingDirectory().toString(); 23 | if (cmd.hasOption("l")) { 24 | log(cmd, wd); 25 | } 26 | else { 27 | log(cmd, wd.substring(env.getProperty(Constants.HDFS_URL).length())); 28 | } 29 | FSUtil.prompt(env); 30 | 31 | } 32 | 33 | @Override 34 | public Options getOptions() { 35 | Options opts = super.getOptions(); 36 | opts.addOption("l", false, "show the full HDFS URL"); 37 | return opts; 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/instanceone/hdfs/shell/command/LocalCd.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 P. Taylor Goetz (ptgoetz@gmail.com) 2 | 3 | package com.instanceone.hdfs.shell.command; 4 | 5 | import java.io.IOException; 6 | 7 | import com.dstreev.hdfs.shell.command.Constants; 8 | import jline.console.ConsoleReader; 9 | import jline.console.completer.Completer; 10 | 11 | import org.apache.commons.cli.CommandLine; 12 | import org.apache.hadoop.fs.FileSystem; 13 | import org.apache.hadoop.fs.Path; 14 | 15 | import com.instanceone.hdfs.shell.completers.FileSystemNameCompleter; 16 | import com.instanceone.stemshell.Environment; 17 | 18 | public class LocalCd extends HdfsCommand { 19 | private Environment env; 20 | 21 | public LocalCd(String name, Environment env) { 22 | super(name,env); 23 | // this.env = env; 24 | } 25 | 26 | public void execute(Environment env, CommandLine cmd, ConsoleReader reader) { 27 | try { 28 | 29 | FileSystem localfs = (FileSystem) env.getValue(Constants.LOCAL_FS); 30 | String dir = cmd.getArgs().length == 0 ? System 31 | .getProperty("user.home") : cmd.getArgs()[0]; 32 | logv(cmd, "Change Dir to: " + dir); 33 | logv(cmd, "CWD: " + localfs.getWorkingDirectory()); 34 | Path newPath = null; 35 | if (dir.startsWith("~/")) { 36 | dir = System.getProperty("user.home") + dir.substring(1); 37 | } 38 | logv(cmd,"Dir: " + dir); 39 | newPath = new Path(dir); 40 | 41 | Path qPath = localfs.makeQualified(newPath); 42 | logv(cmd, "Qual Path: " + qPath); 43 | 44 | if (localfs.getFileStatus(qPath).isDir() && localfs.exists(qPath)) { 45 | localfs.setWorkingDirectory(qPath); 46 | } 47 | else { 48 | log(cmd, "No such directory: " + dir); 49 | } 50 | FSUtil.prompt(env); 51 | } 52 | catch (IOException e) { 53 | log(cmd, e.getMessage()); 54 | } 55 | 56 | } 57 | 58 | @Override 59 | public Completer getCompleter() { 60 | return new FileSystemNameCompleter(this.env, true); 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/com/instanceone/hdfs/shell/command/LocalLs.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 P. Taylor Goetz (ptgoetz@gmail.com) 2 | 3 | package com.instanceone.hdfs.shell.command; 4 | 5 | import static com.instanceone.hdfs.shell.command.FSUtil.longFormat; 6 | import static com.instanceone.hdfs.shell.command.FSUtil.shortFormat; 7 | 8 | import java.io.IOException; 9 | 10 | import com.dstreev.hdfs.shell.command.Constants; 11 | import jline.console.ConsoleReader; 12 | import jline.console.completer.Completer; 13 | 14 | import org.apache.commons.cli.CommandLine; 15 | import org.apache.commons.cli.Options; 16 | import org.apache.hadoop.fs.FileStatus; 17 | import org.apache.hadoop.fs.FileSystem; 18 | import org.apache.hadoop.fs.Path; 19 | 20 | import com.instanceone.hdfs.shell.completers.FileSystemNameCompleter; 21 | import com.instanceone.stemshell.Environment; 22 | 23 | public class LocalLs extends HdfsCommand { 24 | private Environment env; 25 | 26 | public LocalLs(String name, Environment env) { 27 | super(name, env); 28 | } 29 | 30 | public void execute(Environment env, CommandLine cmd, ConsoleReader reader) { 31 | try { 32 | FileSystem localfs = (FileSystem)env.getValue(Constants.LOCAL_FS); 33 | Path srcPath = cmd.getArgs().length == 0 ? localfs.getWorkingDirectory() : new Path(localfs.getWorkingDirectory(), cmd.getArgs()[0]); 34 | FileStatus[] files = localfs.listStatus(srcPath); 35 | for (FileStatus file : files) { 36 | if (cmd.hasOption("l")) { 37 | log(cmd, longFormat(file)); 38 | } 39 | else { 40 | log(cmd, shortFormat(file)); 41 | } 42 | } 43 | FSUtil.prompt(env); 44 | } 45 | catch (IOException e) { 46 | log(cmd, e.getMessage()); 47 | } 48 | } 49 | 50 | 51 | @Override 52 | public Options getOptions() { 53 | // TODO Auto-generated method stub 54 | Options opts = super.getOptions(); 55 | opts.addOption("l", false, "show extended file attributes"); 56 | return opts; 57 | } 58 | 59 | @Override 60 | public Completer getCompleter() { 61 | return new FileSystemNameCompleter(this.env, true); 62 | } 63 | 64 | 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/com/instanceone/hdfs/shell/command/LocalPwd.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 P. Taylor Goetz (ptgoetz@gmail.com) 2 | 3 | package com.instanceone.hdfs.shell.command; 4 | 5 | import com.dstreev.hdfs.shell.command.Constants; 6 | import jline.console.ConsoleReader; 7 | 8 | import org.apache.commons.cli.CommandLine; 9 | import org.apache.commons.cli.Options; 10 | import org.apache.hadoop.fs.FileSystem; 11 | 12 | import com.instanceone.stemshell.Environment; 13 | 14 | public class LocalPwd extends HdfsCommand { 15 | 16 | public LocalPwd(String name) { 17 | super(name); 18 | } 19 | 20 | public void execute(Environment env, CommandLine cmd, ConsoleReader reader) { 21 | FileSystem localfs = (FileSystem)env.getValue(Constants.LOCAL_FS); 22 | 23 | String wd = localfs.getWorkingDirectory().toString(); 24 | if (cmd.hasOption("l")) { 25 | log(cmd, wd); 26 | } 27 | else { 28 | // strip off prefix: "file:" 29 | log(cmd, wd.substring(5)); 30 | } 31 | FSUtil.prompt(env); 32 | } 33 | 34 | @Override 35 | public Options getOptions() { 36 | Options opts = super.getOptions(); 37 | opts.addOption("l", false, "show the full file system URL"); 38 | return opts; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/instanceone/hdfs/shell/completers/FileSystemNameCompleter.java: -------------------------------------------------------------------------------- 1 | package com.instanceone.hdfs.shell.completers; 2 | 3 | import java.io.IOException; 4 | import java.util.List; 5 | 6 | import com.dstreev.hdfs.shell.command.Constants; 7 | import jline.console.completer.Completer; 8 | 9 | import org.apache.hadoop.fs.FileStatus; 10 | import org.apache.hadoop.fs.FileSystem; 11 | import org.apache.hadoop.fs.Path; 12 | 13 | import com.instanceone.stemshell.Environment; 14 | 15 | public class FileSystemNameCompleter implements Completer { 16 | private Environment env; 17 | private boolean local = false; 18 | 19 | public FileSystemNameCompleter(Environment env, boolean local) { 20 | // this.includeFiles = includeFiles; 21 | this.env = env; 22 | this.local = local; 23 | } 24 | 25 | @SuppressWarnings("unused") 26 | private static String strip(String prefix, String target) { 27 | return target.substring(prefix.length()); 28 | } 29 | 30 | // TODO add ability to handle ~/ for local filesystems 31 | public int complete(String buffer, final int cursor, 32 | final List candidates) { 33 | 34 | FileSystem fs; 35 | 36 | String prefix; 37 | 38 | if (!this.local) { 39 | fs = (FileSystem) env.getValue(Constants.HDFS); 40 | prefix = env.getProperty(Constants.HDFS_URL); 41 | } 42 | else { 43 | fs = (FileSystem) env.getValue(Constants.LOCAL_FS); 44 | prefix = "file:" + (buffer != null && buffer.startsWith("/") ? "/" : ""); 45 | } 46 | if(fs == null){ 47 | // System.out.println("Not connected."); 48 | return 0; 49 | } 50 | // System.out.println(prefix); 51 | 52 | Path basePath = fs.getWorkingDirectory(); 53 | // String curPath = strip(prefix, basePath.toString()); 54 | // System.out.println("curPath: " + curPath); 55 | 56 | if (buffer == null) { 57 | // System.out.println("Buffer was null!"); 58 | buffer = "./"; 59 | } 60 | // 61 | // System.out.println("Prefix: " + prefix); 62 | // System.out.println("Match: '" + buffer + "'"); 63 | // System.out.println("Base Path: " + basePath); 64 | 65 | Path completionPath = buffer.startsWith("/") ? new Path(prefix, buffer) 66 | : new Path(basePath, buffer); 67 | // System.out.println("Comp. Path: " + completionPath); 68 | // System.out.println("Comp. Parent: " + completionPath.getParent()); 69 | Path completionDir = (completionPath.getParent() == null || buffer 70 | .endsWith("/")) ? completionPath : completionPath 71 | .getParent(); 72 | // System.out.println("Comp. Dir: " + completionDir); 73 | try { 74 | FileStatus[] entries = fs.listStatus(completionDir); 75 | // System.out.println("Possible matches:"); 76 | // for (FileStatus fStat : entries) { 77 | // System.out.println(fStat.getPath().getName()); 78 | // if(fStat.getPath().toString().startsWith(completionPath.toString())){ 79 | // System.out.println("^ WOOP!"); 80 | // } 81 | // } 82 | 83 | return matchFiles(buffer, completionPath.toString(), entries, 84 | candidates); 85 | 86 | } 87 | catch (IOException e) { 88 | e.printStackTrace(); 89 | return -1; 90 | } 91 | } 92 | 93 | protected String separator() { 94 | return "/"; 95 | } 96 | 97 | protected int matchFiles(final String buffer, final String translated, 98 | final FileStatus[] files, 99 | final List candidates) { 100 | if (files == null) { 101 | return -1; 102 | } 103 | 104 | int matches = 0; 105 | 106 | // first pass: just count the matches 107 | for (FileStatus file : files) { 108 | // System.out.println("Checking: " + file.getPath()); 109 | if (file.getPath().toString().startsWith(translated)) { 110 | // System.out.println("Found match: " + file.getPath()); 111 | matches++; 112 | } 113 | } 114 | for (FileStatus file : files) { 115 | if (file.getPath().toString().startsWith(translated)) { 116 | String name = file.getPath().getName() 117 | + (matches == 1 && file.isDir() ? separator() 118 | : " "); 119 | // System.out.println("Adding candidate: " + name); 120 | candidates.add(name); 121 | } 122 | } 123 | 124 | final int index = buffer.lastIndexOf(separator()); 125 | 126 | return index + separator().length(); 127 | } 128 | 129 | } 130 | -------------------------------------------------------------------------------- /src/main/resources/banner.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------- 2 | _..--""-. .-""--.._ 3 | _.-' \ __...----...__ / '-._ 4 | .' .:::...,' ',...:::. '. 5 | ( .'``'''::; ;::'''``'. ) 6 | \ '-) (-' / 7 | \ / \ / 8 | \ .'.-. .-.'. / 9 | \ | \0| |0/ | / 10 | | \ | .-==-. | / | 11 | \ `/`; ;`\` / 12 | '.._ (_ | .-==-. | _) _..' 13 | `"`"-`/ `/' '\` \`-"`"` 14 | / /`; .==. ;`\ \ 15 | .---./_/ \ .==. / \ \ 16 | / '. `-.__) | `" 17 | | =(`-. '==. ; 18 | jgs \ '. `-. / 19 | \_:_) `"--.....-' 20 | 21 | _ _ ___ ___ ___ ___ _ ___ 22 | | || | \| __/ __|___ / __| | |_ _| 23 | | __ | |) | _|\__ \___| (__| |__ | | 24 | |_||_|___/|_| |___/ \___|____|___| v2.3.1-SNAPSHOT 25 | -------------------------------------------------------------------- 26 | 27 | -------------------------------------------------------------------------------- /src/test/java/com/dstreev/hadoop/util/DirectoryReaderTest.java: -------------------------------------------------------------------------------- 1 | package com.dstreev.hadoop.util; 2 | 3 | /** 4 | * Created by dstreev on 2016-02-15. 5 | */ 6 | public class DirectoryReaderTest { 7 | } 8 | -------------------------------------------------------------------------------- /src/test/java/com/dstreev/hadoop/util/NamenodeParserTest.java: -------------------------------------------------------------------------------- 1 | package com.dstreev.hadoop.util; 2 | 3 | import org.junit.Test; 4 | 5 | import java.util.List; 6 | 7 | import static org.junit.Assert.assertFalse; 8 | import static org.junit.Assert.assertTrue; 9 | 10 | /** 11 | * Created by dstreev on 2016-03-21. 12 | */ 13 | public class NamenodeParserTest { 14 | 15 | @Test 16 | public void getTopUserOpsString() { 17 | try { 18 | NamenodeJmxParser njp = new NamenodeJmxParser("nn_2.3.5.1_active.json"); 19 | List userOpsList = njp.getTopUserOpRecords(); 20 | for (String userOps : userOpsList) { 21 | System.out.println(userOps); 22 | } 23 | // assertTrue(njp.getTopUserOpRecords() != null); 24 | } catch (Exception e) { 25 | e.printStackTrace(); 26 | assertTrue(false); 27 | } 28 | } 29 | 30 | @Test 31 | public void getFSState() { 32 | try { 33 | NamenodeJmxParser njp = new NamenodeJmxParser("nn_2.3.5.1_active.json"); 34 | String fsState = njp.getFSState(); 35 | System.out.println(fsState); 36 | } catch (Exception e) { 37 | e.printStackTrace(); 38 | assertTrue(false); 39 | } 40 | } 41 | 42 | @Test 43 | public void getNamenodeInfo() { 44 | try { 45 | NamenodeJmxParser njp = new NamenodeJmxParser("nn_2.3.5.1_active.json"); 46 | String nnInfo = njp.getNamenodeInfo(); 47 | System.out.println(nnInfo); 48 | } catch (Exception e) { 49 | e.printStackTrace(); 50 | assertTrue(false); 51 | } 52 | } 53 | 54 | @Test 55 | public void getTopUserOpsStringStandby() { 56 | try { 57 | NamenodeJmxParser njp = new NamenodeJmxParser("nn_2.3.5.1_standby.json"); 58 | List userOpsList = njp.getTopUserOpRecords(); 59 | for (String userOps : userOpsList) { 60 | System.out.println(userOps); 61 | } 62 | // assertTrue(njp.getTopUserOpRecords() != null); 63 | } catch (Exception e) { 64 | e.printStackTrace(); 65 | assertTrue(false); 66 | } 67 | } 68 | 69 | @Test 70 | public void getFSStateStandby() { 71 | try { 72 | NamenodeJmxParser njp = new NamenodeJmxParser("nn_2.3.5.1_standby.json"); 73 | String fsState = njp.getFSState(); 74 | System.out.println(fsState); 75 | } catch (Exception e) { 76 | e.printStackTrace(); 77 | assertTrue(false); 78 | } 79 | } 80 | 81 | @Test 82 | public void getNamenodeInfoStandby() { 83 | try { 84 | NamenodeJmxParser njp = new NamenodeJmxParser("nn_2.3.5.1_standby.json"); 85 | String nnInfo = njp.getNamenodeInfo(); 86 | System.out.println(nnInfo); 87 | } catch (Exception e) { 88 | e.printStackTrace(); 89 | assertTrue(false); 90 | } 91 | } 92 | 93 | @Test 94 | public void getTopUserOpsStringStandalone() { 95 | try { 96 | NamenodeJmxParser njp = new NamenodeJmxParser("nn_2.3.2.0_standalone.json"); 97 | List userOpsList = njp.getTopUserOpRecords(); 98 | for (String userOps : userOpsList) { 99 | System.out.println(userOps); 100 | } 101 | // assertTrue(njp.getTopUserOpRecords() != null); 102 | } catch (Exception e) { 103 | e.printStackTrace(); 104 | assertTrue(false); 105 | } 106 | } 107 | 108 | @Test 109 | public void getFSStateStandalone() { 110 | try { 111 | NamenodeJmxParser njp = new NamenodeJmxParser("nn_2.3.2.0_standalone.json"); 112 | String fsState = njp.getFSState(); 113 | System.out.println(fsState); 114 | } catch (Exception e) { 115 | e.printStackTrace(); 116 | assertTrue(false); 117 | } 118 | } 119 | 120 | @Test 121 | public void getNamenodeInfoStandalone() { 122 | try { 123 | NamenodeJmxParser njp = new NamenodeJmxParser("nn_2.3.2.0_standalone.json"); 124 | String nnInfo = njp.getNamenodeInfo(); 125 | System.out.println(nnInfo); 126 | } catch (Exception e) { 127 | e.printStackTrace(); 128 | assertTrue(false); 129 | } 130 | } 131 | 132 | } 133 | -------------------------------------------------------------------------------- /src/test/java/com/instanceone/hdfs/shell/ShellTest.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 P. Taylor Goetz (ptgoetz@gmail.com) 2 | 3 | package com.instanceone.hdfs.shell; 4 | 5 | import jline.console.ConsoleReader; 6 | import jline.console.completer.AggregateCompleter; 7 | import jline.console.completer.ArgumentCompleter; 8 | import jline.console.completer.FileNameCompleter; 9 | import jline.console.completer.StringsCompleter; 10 | 11 | public class ShellTest { 12 | 13 | public static void main(String[] args) throws Exception { 14 | 15 | 16 | 17 | StringsCompleter animal = new StringsCompleter("animal"); 18 | StringsCompleter animals = new StringsCompleter("dog", "cat", "pig", "horse"); 19 | ArgumentCompleter argCompleter = new ArgumentCompleter(animal, animals); 20 | 21 | StringsCompleter car = new StringsCompleter("car"); 22 | StringsCompleter forDom = new StringsCompleter("foreign", "domestic"); 23 | StringsCompleter cars = new StringsCompleter("honda", "subaru", "saab"); 24 | ArgumentCompleter argCompleter2 = new ArgumentCompleter(car, forDom, cars); 25 | 26 | StringsCompleter file = new StringsCompleter("file"); 27 | FileNameCompleter fnc = new FileNameCompleter(); 28 | ArgumentCompleter fileCompleter = new ArgumentCompleter(file, fnc); 29 | 30 | 31 | AggregateCompleter completer = new AggregateCompleter(argCompleter, argCompleter2, fileCompleter); 32 | 33 | 34 | ConsoleReader reader = new ConsoleReader(); 35 | reader.addCompleter(completer); 36 | 37 | @SuppressWarnings("unused") 38 | String line; 39 | while ((line = reader.readLine("hdfs-cli % ")) != null) { 40 | 41 | } 42 | } 43 | 44 | 45 | 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/test/resources/topUserOpsCount.json: -------------------------------------------------------------------------------- 1 | { 2 | "timestamp": "2016-03-21T15:00:57-0400", 3 | "windows": [ 4 | { 5 | "windowLenMs": 300000, 6 | "ops": [ 7 | { 8 | "opType": "delete", 9 | "topUsers": [ 10 | { 11 | "user": "hive", 12 | "count": 8 13 | }, 14 | { 15 | "user": "ambari-qa", 16 | "count": 3 17 | } 18 | ], 19 | "totalCount": 11 20 | }, 21 | { 22 | "opType": "*", 23 | "topUsers": [ 24 | { 25 | "user": "citi_2769197d", 26 | "count": 284 27 | }, 28 | { 29 | "user": "wre", 30 | "count": 152 31 | }, 32 | { 33 | "user": "ststrt", 34 | "count": 78 35 | }, 36 | { 37 | "user": "hive", 38 | "count": 63 39 | }, 40 | { 41 | "user": "ambari-qa", 42 | "count": 21 43 | }, 44 | { 45 | "user": "logiq3", 46 | "count": 6 47 | }, 48 | { 49 | "user": "oozie", 50 | "count": 5 51 | }, 52 | { 53 | "user": "mapred", 54 | "count": 4 55 | } 56 | ], 57 | "totalCount": 613 58 | }, 59 | { 60 | "opType": "mkdirs", 61 | "topUsers": [ 62 | { 63 | "user": "hive", 64 | "count": 14 65 | }, 66 | { 67 | "user": "ambari-qa", 68 | "count": 8 69 | } 70 | ], 71 | "totalCount": 22 72 | }, 73 | { 74 | "opType": "listStatus", 75 | "topUsers": [ 76 | { 77 | "user": "oozie", 78 | "count": 5 79 | }, 80 | { 81 | "user": "mapred", 82 | "count": 4 83 | } 84 | ], 85 | "totalCount": 9 86 | }, 87 | { 88 | "opType": "getfileinfo", 89 | "topUsers": [ 90 | { 91 | "user": "citi_2769197d", 92 | "count": 284 93 | }, 94 | { 95 | "user": "wre", 96 | "count": 152 97 | }, 98 | { 99 | "user": "ststrt", 100 | "count": 78 101 | }, 102 | { 103 | "user": "hive", 104 | "count": 49 105 | }, 106 | { 107 | "user": "ambari-qa", 108 | "count": 20 109 | }, 110 | { 111 | "user": "logiq3", 112 | "count": 6 113 | } 114 | ], 115 | "totalCount": 589 116 | } 117 | ] 118 | }, 119 | { 120 | "windowLenMs": 1500000, 121 | "ops": [ 122 | { 123 | "opType": "delete", 124 | "topUsers": [ 125 | { 126 | "user": "hive", 127 | "count": 30 128 | }, 129 | { 130 | "user": "ambari-qa", 131 | "count": 12 132 | } 133 | ], 134 | "totalCount": 42 135 | }, 136 | { 137 | "opType": "*", 138 | "topUsers": [ 139 | { 140 | "user": "citi_2769197d", 141 | "count": 720 142 | }, 143 | { 144 | "user": "wre", 145 | "count": 544 146 | }, 147 | { 148 | "user": "hive", 149 | "count": 242 150 | }, 151 | { 152 | "user": "ststrt", 153 | "count": 234 154 | }, 155 | { 156 | "user": "ambari-qa", 157 | "count": 134 158 | }, 159 | { 160 | "user": "mapred", 161 | "count": 18 162 | }, 163 | { 164 | "user": "oozie", 165 | "count": 10 166 | }, 167 | { 168 | "user": "logiq3", 169 | "count": 6 170 | } 171 | ], 172 | "totalCount": 1908 173 | }, 174 | { 175 | "opType": "mkdirs", 176 | "topUsers": [ 177 | { 178 | "user": "hive", 179 | "count": 44 180 | }, 181 | { 182 | "user": "ambari-qa", 183 | "count": 25 184 | } 185 | ], 186 | "totalCount": 69 187 | }, 188 | { 189 | "opType": "listStatus", 190 | "topUsers": [ 191 | { 192 | "user": "mapred", 193 | "count": 18 194 | }, 195 | { 196 | "user": "oozie", 197 | "count": 11 198 | } 199 | ], 200 | "totalCount": 29 201 | }, 202 | { 203 | "opType": "getfileinfo", 204 | "topUsers": [ 205 | { 206 | "user": "citi_2769197d", 207 | "count": 720 208 | }, 209 | { 210 | "user": "wre", 211 | "count": 492 212 | }, 213 | { 214 | "user": "ststrt", 215 | "count": 234 216 | }, 217 | { 218 | "user": "hive", 219 | "count": 182 220 | }, 221 | { 222 | "user": "ambari-qa", 223 | "count": 98 224 | }, 225 | { 226 | "user": "logiq3", 227 | "count": 6 228 | } 229 | ], 230 | "totalCount": 1732 231 | } 232 | ] 233 | }, 234 | { 235 | "windowLenMs": 60000, 236 | "ops": [ 237 | { 238 | "opType": "delete", 239 | "topUsers": [ 240 | { 241 | "user": "hive", 242 | "count": 1 243 | }, 244 | { 245 | "user": "ambari-qa", 246 | "count": 1 247 | } 248 | ], 249 | "totalCount": 2 250 | }, 251 | { 252 | "opType": "*", 253 | "topUsers": [ 254 | { 255 | "user": "ststrt", 256 | "count": 78 257 | }, 258 | { 259 | "user": "wre", 260 | "count": 26 261 | }, 262 | { 263 | "user": "ambari-qa", 264 | "count": 12 265 | }, 266 | { 267 | "user": "hive", 268 | "count": 11 269 | }, 270 | { 271 | "user": "logiq3", 272 | "count": 6 273 | }, 274 | { 275 | "user": "mapred", 276 | "count": 2 277 | }, 278 | { 279 | "user": "oozie", 280 | "count": 1 281 | } 282 | ], 283 | "totalCount": 136 284 | }, 285 | { 286 | "opType": "mkdirs", 287 | "topUsers": [ 288 | { 289 | "user": "hive", 290 | "count": 2 291 | }, 292 | { 293 | "user": "ambari-qa", 294 | "count": 2 295 | } 296 | ], 297 | "totalCount": 4 298 | }, 299 | { 300 | "opType": "listStatus", 301 | "topUsers": [ 302 | { 303 | "user": "mapred", 304 | "count": 2 305 | }, 306 | { 307 | "user": "oozie", 308 | "count": 1 309 | } 310 | ], 311 | "totalCount": 3 312 | }, 313 | { 314 | "opType": "getfileinfo", 315 | "topUsers": [ 316 | { 317 | "user": "ststrt", 318 | "count": 78 319 | }, 320 | { 321 | "user": "wre", 322 | "count": 26 323 | }, 324 | { 325 | "user": "ambari-qa", 326 | "count": 9 327 | }, 328 | { 329 | "user": "hive", 330 | "count": 7 331 | }, 332 | { 333 | "user": "logiq3", 334 | "count": 6 335 | } 336 | ], 337 | "totalCount": 126 338 | } 339 | ] 340 | } 341 | ] 342 | } 343 | --------------------------------------------------------------------------------