├── .gitignore ├── resources ├── architecture.png ├── architecture.psd ├── spash-logo.png └── spash-logo.psd ├── core ├── src │ ├── main │ │ ├── resources │ │ │ ├── hdfs-site.xml │ │ │ └── log4j.properties │ │ └── java │ │ │ └── it │ │ │ └── nerdammer │ │ │ └── spash │ │ │ └── shell │ │ │ ├── SpashShellFactory.java │ │ │ ├── command │ │ │ ├── Command.java │ │ │ ├── spi │ │ │ │ ├── UnknownCommand.java │ │ │ │ ├── NoOpCommand.java │ │ │ │ ├── ExitCommand.java │ │ │ │ ├── PwdCommand.java │ │ │ │ ├── EchoCommand.java │ │ │ │ ├── MkDirCommand.java │ │ │ │ ├── RmDirCommand.java │ │ │ │ ├── HeadCommand.java │ │ │ │ ├── CatCommand.java │ │ │ │ ├── CdCommand.java │ │ │ │ ├── WriteCommand.java │ │ │ │ ├── RmCommand.java │ │ │ │ ├── GrepCommand.java │ │ │ │ └── LsCommand.java │ │ │ ├── ExecutionContext.java │ │ │ ├── AbstractCommand.java │ │ │ ├── CommandResult.java │ │ │ ├── CommandFactory.java │ │ │ ├── ExpressionTokenizer.java │ │ │ └── CommandTokenizer.java │ │ │ ├── api │ │ │ ├── fs │ │ │ │ ├── SpashFileSystem.java │ │ │ │ ├── FileSystemFacade.java │ │ │ │ └── FileSystemFacadeImpl.java │ │ │ └── spark │ │ │ │ ├── SpashSparkSubsystem.java │ │ │ │ ├── SparkFacade.java │ │ │ │ └── SparkFacadeImpl.java │ │ │ ├── common │ │ │ ├── SerializableFunction.java │ │ │ ├── SpashCollectionEmptyAdapter.java │ │ │ ├── SpashCollectionRDDAdapter.java │ │ │ ├── SpashCollectionUnionAdapter.java │ │ │ ├── SpashCollection.java │ │ │ ├── SpashCollectionListAdapter.java │ │ │ └── TabulatedValue.java │ │ │ ├── SpashExitException.java │ │ │ ├── Spash.java │ │ │ ├── SpashSession.java │ │ │ ├── SpashCommandCompleter.java │ │ │ ├── SpashConfig.java │ │ │ ├── SshServerFactory.java │ │ │ └── SpashShell.java │ └── test │ │ └── java │ │ └── it │ │ └── nerdammer │ │ └── spash │ │ └── shell │ │ └── command │ │ ├── ExpressionTokenizerTest.java │ │ └── CommandTokenizerTest.java └── pom.xml ├── release ├── src │ ├── conf │ │ └── spash.properties │ ├── assembly │ │ ├── bin.xml │ │ └── assembly-1.1.3.xsd │ └── bin │ │ └── spash.sh └── pom.xml ├── pom.xml ├── README.md └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .idea/* 3 | *.ser 4 | target 5 | */target 6 | */*.iml -------------------------------------------------------------------------------- /resources/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nerdammer/spash/HEAD/resources/architecture.png -------------------------------------------------------------------------------- /resources/architecture.psd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nerdammer/spash/HEAD/resources/architecture.psd -------------------------------------------------------------------------------- /resources/spash-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nerdammer/spash/HEAD/resources/spash-logo.png -------------------------------------------------------------------------------- /resources/spash-logo.psd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nerdammer/spash/HEAD/resources/spash-logo.psd -------------------------------------------------------------------------------- /core/src/main/resources/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | dfs.client.use.datanode.hostname 5 | true 6 | 7 | -------------------------------------------------------------------------------- /release/src/conf/spash.properties: -------------------------------------------------------------------------------- 1 | # Configuration file for Spash 2 | 3 | # Spash Configuration 4 | spash.listen.port=2222 5 | spash.key.algorithm=RSA 6 | spash.key.length=2048 7 | 8 | # HDFS Configuration 9 | spash.hdfs.host=quickstart 10 | spash.hdfs.port=8020 11 | 12 | -------------------------------------------------------------------------------- /core/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=DEBUG, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/SpashShellFactory.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell; 2 | 3 | import org.apache.sshd.common.Factory; 4 | import org.apache.sshd.server.Command; 5 | 6 | /** 7 | * The factory of the Spash shell. 8 | * 9 | * @author Nicola Ferraro 10 | */ 11 | public class SpashShellFactory implements Factory { 12 | 13 | public Command create() 14 | { 15 | return new SpashShell(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/Command.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command; 2 | 3 | import it.nerdammer.spash.shell.SpashSession; 4 | 5 | /** 6 | * An abstract Spash command. 7 | * 8 | * @author Nicola Ferraro 9 | */ 10 | public interface Command { 11 | 12 | /** 13 | * Executes the command action. 14 | * 15 | * @param ctx the execution context 16 | * @return the associated {@code CommandResult} 17 | */ 18 | CommandResult execute(ExecutionContext ctx); 19 | 20 | } 21 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | it.nerdammer.spash 8 | spash-pom 9 | 0.1 10 | 11 | pom 12 | 13 | 14 | core 15 | release 16 | 17 | 18 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/api/fs/SpashFileSystem.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.api.fs; 2 | 3 | import it.nerdammer.spash.shell.SpashConfig; 4 | 5 | /** 6 | * Holds a reference to the default file system. 7 | * 8 | * @author Nicola Ferraro 9 | */ 10 | public final class SpashFileSystem { 11 | 12 | private static final FileSystemFacade FACADE_INSTANCE = new FileSystemFacadeImpl(SpashConfig.getInstance().hdfsHost(), SpashConfig.getInstance().hdfsPort()); 13 | 14 | private SpashFileSystem() { 15 | } 16 | 17 | public static FileSystemFacade get() { 18 | return FACADE_INSTANCE; 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/spi/UnknownCommand.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command.spi; 2 | 3 | import it.nerdammer.spash.shell.command.Command; 4 | import it.nerdammer.spash.shell.command.CommandResult; 5 | import it.nerdammer.spash.shell.SpashSession; 6 | import it.nerdammer.spash.shell.command.ExecutionContext; 7 | 8 | /** 9 | * @author Nicola Ferraro 10 | */ 11 | public class UnknownCommand implements Command { 12 | 13 | public UnknownCommand(String commandString) { 14 | } 15 | 16 | @Override 17 | public CommandResult execute(ExecutionContext ctx) { 18 | return CommandResult.error(this, "command not found"); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/spi/NoOpCommand.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command.spi; 2 | 3 | import it.nerdammer.spash.shell.command.Command; 4 | import it.nerdammer.spash.shell.command.CommandResult; 5 | import it.nerdammer.spash.shell.SpashSession; 6 | import it.nerdammer.spash.shell.command.ExecutionContext; 7 | 8 | /** 9 | * A command that does not do anything. 10 | * 11 | * @author Nicola Ferraro 12 | */ 13 | public class NoOpCommand implements Command { 14 | 15 | public NoOpCommand(String commandStr) {} 16 | 17 | @Override 18 | public CommandResult execute(ExecutionContext ctx) { 19 | return CommandResult.success(this); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/common/SerializableFunction.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.common; 2 | 3 | import ch.lambdaj.function.convert.Converter; 4 | import org.apache.spark.api.java.function.Function; 5 | 6 | import java.io.Serializable; 7 | 8 | /** 9 | * A {@code Function} that is also {@code Serializable}. 10 | * 11 | * @author Nicola Ferraro 12 | */ 13 | public abstract class SerializableFunction implements Function, Converter, Serializable { 14 | 15 | @Override 16 | public R call(T t) throws Exception { 17 | return apply(t); 18 | } 19 | 20 | @Override 21 | public R convert(T t) { 22 | return apply(t); 23 | } 24 | 25 | public abstract R apply(T v1); 26 | 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/SpashExitException.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell; 2 | 3 | /** 4 | * An exception that signal the willingness to exit the shell. 5 | * 6 | * @author Nicola Ferraro 7 | */ 8 | public class SpashExitException extends RuntimeException { 9 | 10 | public SpashExitException() { 11 | } 12 | 13 | public SpashExitException(String message) { 14 | super(message); 15 | } 16 | 17 | public SpashExitException(String message, Throwable cause) { 18 | super(message, cause); 19 | } 20 | 21 | public SpashExitException(Throwable cause) { 22 | super(cause); 23 | } 24 | 25 | public SpashExitException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { 26 | super(message, cause, enableSuppression, writableStackTrace); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/spi/ExitCommand.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command.spi; 2 | 3 | import it.nerdammer.spash.shell.SpashExitException; 4 | import it.nerdammer.spash.shell.command.AbstractCommand; 5 | import it.nerdammer.spash.shell.command.CommandResult; 6 | import it.nerdammer.spash.shell.SpashSession; 7 | import it.nerdammer.spash.shell.command.ExecutionContext; 8 | 9 | import java.util.List; 10 | 11 | /** 12 | * A command that exits the shell. 13 | * 14 | * @author Nicola Ferraro 15 | */ 16 | public class ExitCommand extends AbstractCommand { 17 | 18 | public ExitCommand(String commandString) { 19 | super(commandString); 20 | } 21 | 22 | @Override 23 | public CommandResult execute(ExecutionContext ctx) { 24 | 25 | List args = this.getArguments(); 26 | if(args.size()>0) { 27 | return CommandResult.error(this, "Unexpected arguments: " + args); 28 | } 29 | 30 | throw new SpashExitException(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/api/spark/SpashSparkSubsystem.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.api.spark; 2 | 3 | import it.nerdammer.spash.shell.api.fs.FileSystemFacade; 4 | import it.nerdammer.spash.shell.api.fs.FileSystemFacadeImpl; 5 | import org.apache.spark.SparkConf; 6 | import org.apache.spark.api.java.JavaSparkContext; 7 | 8 | /** 9 | * Holds a reference to the Spark context. 10 | * 11 | * @author Nicola Ferraro 12 | */ 13 | public class SpashSparkSubsystem { 14 | 15 | private static final SparkFacade FACADE_INSTANCE; 16 | 17 | static { 18 | String master = System.getProperty("spark.master"); 19 | if(master==null) { 20 | System.setProperty("spark.master", "local"); 21 | } 22 | 23 | SparkConf conf = new SparkConf().setAppName("Spash"); 24 | JavaSparkContext sc = new JavaSparkContext(conf); 25 | FACADE_INSTANCE = new SparkFacadeImpl(sc); 26 | } 27 | 28 | private SpashSparkSubsystem() { 29 | } 30 | 31 | public static SparkFacade get() { 32 | return FACADE_INSTANCE; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /release/src/assembly/bin.xml: -------------------------------------------------------------------------------- 1 | 4 | distribution 5 | 6 | tar.gz 7 | dir 8 | 9 | 10 | 11 | 12 | /lib 13 | 14 | 15 | 16 | 17 | 18 | src/bin 19 | bin 20 | 21 | *.sh 22 | 23 | 24 | 25 | 26 | src/conf 27 | conf 28 | 29 | * 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/ExecutionContext.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command; 2 | 3 | import it.nerdammer.spash.shell.SpashSession; 4 | 5 | /** 6 | * The context associated with the execution of a {@code Command}. 7 | * 8 | * @author Nicola Ferraro 9 | */ 10 | public class ExecutionContext { 11 | 12 | private SpashSession session; 13 | 14 | private CommandResult previousCommandResult; 15 | 16 | public ExecutionContext() { 17 | } 18 | 19 | public ExecutionContext(SpashSession session, CommandResult previousCommandResult) { 20 | this.session = session; 21 | this.previousCommandResult = previousCommandResult; 22 | } 23 | 24 | public SpashSession getSession() { 25 | return session; 26 | } 27 | 28 | public void setSession(SpashSession session) { 29 | this.session = session; 30 | } 31 | 32 | public CommandResult getPreviousCommandResult() { 33 | return previousCommandResult; 34 | } 35 | 36 | public void setPreviousCommandResult(CommandResult previousCommandResult) { 37 | this.previousCommandResult = previousCommandResult; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/Spash.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell; 2 | 3 | import it.nerdammer.spash.shell.api.fs.SpashFileSystem; 4 | import it.nerdammer.spash.shell.api.spark.SpashSparkSubsystem; 5 | import org.apache.sshd.server.SshServer; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.util.Arrays; 10 | 11 | /** 12 | * The main class of the application. 13 | * 14 | * @author Nicola Ferraro 15 | */ 16 | public class Spash { 17 | 18 | public static void main(String[] args) throws Exception { 19 | 20 | Logger logger = LoggerFactory.getLogger(Spash.class); 21 | 22 | logger.info("Initializing the file system"); 23 | SpashFileSystem.get().getFileSystem(); 24 | 25 | logger.info("Initializing Spark by running a simple job"); 26 | SpashSparkSubsystem.get().parallelize(Arrays.asList(1, 2, 3)).collect(); 27 | 28 | logger.info("Starting the Spash shell"); 29 | SshServer server = SshServerFactory.create(); 30 | server.start(); 31 | 32 | logger.info("Spash shell started"); 33 | 34 | synchronized(Spash.class) { 35 | Spash.class.wait(); 36 | } 37 | 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /release/src/bin/spash.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Compute the bin path 4 | pushd `dirname $0` > /dev/null 5 | SPASH_BIN=`pwd` 6 | popd > /dev/null 7 | 8 | # Compute the base path 9 | SPASH_BASE=`dirname $SPASH_BIN` 10 | 11 | # Compute the conf path 12 | SPASH_CONF=$SPASH_BASE/conf 13 | 14 | # Compute the lib path 15 | SPASH_LIB=$SPASH_BASE/lib 16 | 17 | # Jar list 18 | APP_JAR_LIST=`ls -dm $SPASH_LIB/* | tr -d ' \r\n'` 19 | 20 | # Main Jar 21 | APP_MAIN_JAR=`echo $APP_JAR_LIST | tr ',' '\n' | grep spash-core` 22 | 23 | # Hdfs Nio Jar 24 | APP_HDFS_NIO_JAR=`echo $APP_JAR_LIST | tr ',' '\n' | grep jsr203hadoop` 25 | 26 | # Jar Classpath 27 | APP_CLASSPATH=`echo $APP_JAR_LIST | tr ',' '\n' | grep -v spash-core | grep -v jsr203hadoop | tr '\n' ':'` 28 | 29 | # Jar csv 30 | APP_JARS_CSV=`echo $APP_CLASSPATH | tr ':' ','` 31 | 32 | echo "Using classpath: $APP_CLASSPATH" 33 | echo "Using main jar: $APP_MAIN_JAR" 34 | 35 | echo "Starting Spash" 36 | 37 | eval "spark-submit --class it.nerdammer.spash.shell.Spash \ 38 | --master yarn --deploy-mode client \ 39 | --jars $APP_JARS_CSV \ 40 | --driver-class-path $APP_HDFS_NIO_JAR \ 41 | --driver-java-options \"-Dspash.config.dir=$SPASH_CONF -Dspash.config=$SPASH_CONF/spash.properties\" \ 42 | $APP_MAIN_JAR" 43 | 44 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/spi/PwdCommand.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command.spi; 2 | 3 | import it.nerdammer.spash.shell.command.AbstractCommand; 4 | import it.nerdammer.spash.shell.command.CommandResult; 5 | import it.nerdammer.spash.shell.command.ExecutionContext; 6 | import it.nerdammer.spash.shell.common.SpashCollection; 7 | import it.nerdammer.spash.shell.common.SpashCollectionListAdapter; 8 | 9 | import java.util.Collections; 10 | import java.util.List; 11 | 12 | /** 13 | * A command that returns the current directory. 14 | * 15 | * @author Nicola Ferraro 16 | */ 17 | public class PwdCommand extends AbstractCommand { 18 | 19 | public PwdCommand(String commandString) { 20 | super(commandString); 21 | } 22 | 23 | @Override 24 | public CommandResult execute(ExecutionContext ctx) { 25 | 26 | List args = this.getArguments(); 27 | if(args.size()>0) { 28 | return CommandResult.error(this, "Unexpected arguments: " + args); 29 | } 30 | 31 | SpashCollection content = new SpashCollectionListAdapter<>(Collections.singletonList(ctx.getSession().getWorkingDir())); 32 | return CommandResult.success(this, content); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/SpashSession.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell; 2 | 3 | /** 4 | * Contains all information related to the user's session. 5 | * 6 | * @author Nicola Ferraro 7 | */ 8 | public class SpashSession { 9 | 10 | /** 11 | * The default working directory. 12 | */ 13 | private static final String DEFAULT_WORKING_DIR = "/"; 14 | 15 | /** 16 | * The user connected to this session. 17 | */ 18 | private String user; 19 | 20 | /** 21 | * The current working directory. 22 | */ 23 | private String workingDir = DEFAULT_WORKING_DIR; 24 | 25 | public SpashSession(String user) { 26 | this.user = user; 27 | } 28 | 29 | public String getUser() { 30 | return user; 31 | } 32 | 33 | public void setUser(String user) { 34 | this.user = user; 35 | } 36 | 37 | public String getWorkingDir() { 38 | return workingDir; 39 | } 40 | 41 | public void setWorkingDir(String workingDir) { 42 | this.workingDir = workingDir; 43 | } 44 | 45 | @Override 46 | public String toString() { 47 | return "SpashSession{" + 48 | "user='" + user + '\'' + 49 | '}'; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/common/SpashCollectionEmptyAdapter.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.common; 2 | 3 | import org.apache.spark.api.java.JavaRDD; 4 | import org.apache.spark.api.java.JavaSparkContext; 5 | 6 | import java.io.PrintWriter; 7 | import java.util.Collections; 8 | import java.util.List; 9 | 10 | /** 11 | * An empty SpashCollection. 12 | * 13 | * @author Nicola Ferraro 14 | */ 15 | public class SpashCollectionEmptyAdapter implements SpashCollection { 16 | 17 | public SpashCollectionEmptyAdapter() { 18 | } 19 | 20 | @Override 21 | public void mkString(PrintWriter writer) { 22 | } 23 | 24 | @Override 25 | public SpashCollection map(SerializableFunction f) { 26 | return new SpashCollectionEmptyAdapter<>(); 27 | } 28 | 29 | @Override 30 | public SpashCollection union(SpashCollection coll) { 31 | return coll; 32 | } 33 | 34 | @Override 35 | public SpashCollection filter(SerializableFunction condition) { 36 | return this; 37 | } 38 | 39 | @Override 40 | public JavaRDD toRDD(JavaSparkContext sc) { 41 | return sc.emptyRDD(); 42 | } 43 | 44 | @Override 45 | public List collect() { 46 | return Collections.emptyList(); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/spi/EchoCommand.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command.spi; 2 | 3 | import it.nerdammer.spash.shell.command.AbstractCommand; 4 | import it.nerdammer.spash.shell.command.CommandResult; 5 | import it.nerdammer.spash.shell.command.ExecutionContext; 6 | import it.nerdammer.spash.shell.common.SpashCollectionListAdapter; 7 | 8 | import java.util.Collections; 9 | import java.util.List; 10 | 11 | /** 12 | * Command to echo the user input. 13 | * 14 | * @author Nicola Ferraro 15 | */ 16 | public class EchoCommand extends AbstractCommand { 17 | 18 | public EchoCommand(String commandString) { 19 | super(commandString); 20 | } 21 | 22 | @Override 23 | public CommandResult execute(ExecutionContext ctx) { 24 | 25 | List args = this.getArguments(); 26 | if(args.size()==0) { 27 | return CommandResult.error(this, "No arguments provided"); 28 | } 29 | 30 | StringBuilder bui = new StringBuilder(); 31 | for(int i=0; i stream = Collections.singletonList(bui.toString()); 39 | return CommandResult.success(this, new SpashCollectionListAdapter<>(stream)); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/spi/MkDirCommand.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command.spi; 2 | 3 | import it.nerdammer.spash.shell.api.fs.FileSystemFacade; 4 | import it.nerdammer.spash.shell.api.fs.SpashFileSystem; 5 | import it.nerdammer.spash.shell.command.AbstractCommand; 6 | import it.nerdammer.spash.shell.command.CommandResult; 7 | import it.nerdammer.spash.shell.command.ExecutionContext; 8 | 9 | import java.nio.file.Path; 10 | import java.util.List; 11 | 12 | /** 13 | * Command to create a directory. 14 | * 15 | * @author Nicola Ferraro 16 | */ 17 | public class MkDirCommand extends AbstractCommand { 18 | 19 | public MkDirCommand(String commandString) { 20 | super(commandString); 21 | } 22 | 23 | @Override 24 | public CommandResult execute(ExecutionContext ctx) { 25 | 26 | FileSystemFacade fs = SpashFileSystem.get(); 27 | 28 | List files = this.getArguments(); 29 | if(files.size()==0) { 30 | return CommandResult.error(this, "Missing argument"); 31 | } else if(files.size()>1) { 32 | return CommandResult.error(this, "Too many arguments"); 33 | } 34 | 35 | String file = files.get(0); 36 | 37 | Path path = fs.getAbsolutePath(ctx.getSession().getWorkingDir(), file); 38 | boolean created = fs.mkdir(path.toString()); 39 | if(!created) { 40 | return CommandResult.error(this, "File exists"); 41 | } 42 | 43 | return CommandResult.success(this); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/api/spark/SparkFacade.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.api.spark; 2 | 3 | import it.nerdammer.spash.shell.common.SpashCollection; 4 | 5 | import java.nio.file.Path; 6 | import java.util.List; 7 | 8 | /** 9 | * Provide useful Spark functions and methods to access HDFS from Spark. 10 | * 11 | * @author Nicola Ferraro 12 | */ 13 | public interface SparkFacade { 14 | 15 | /** 16 | * Returns a {@code SpashCollection} from the content of the file or directory. 17 | * 18 | * @param file the file or the directory to read 19 | * @return a collection of lines contained in the file 20 | */ 21 | SpashCollection read(Path file); 22 | 23 | /** 24 | * Writes the provided content to the file. 25 | * 26 | * @param content the content 27 | * @param file the output file 28 | */ 29 | void write(SpashCollection content, Path file); 30 | 31 | /** 32 | * Returns a {@code SpashCollection} from the first lines of the file or directory. 33 | * 34 | * @param file the file or the directory to read 35 | * @param lines the number of lines to show 36 | * @return a collection of the first lines contained in the file 37 | */ 38 | SpashCollection head(Path file, int lines); 39 | 40 | /** 41 | * Parallelizes a collection as an RDD backed {@code SpashCollection}. 42 | * 43 | * @param coll the standard java collection 44 | * @param the type of the contained objects 45 | * @return the parallelized collection 46 | */ 47 | SpashCollection parallelize(List coll); 48 | 49 | } 50 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/common/SpashCollectionRDDAdapter.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.common; 2 | 3 | import org.apache.spark.api.java.JavaRDD; 4 | import org.apache.spark.api.java.JavaSparkContext; 5 | 6 | import java.io.PrintWriter; 7 | import java.util.Iterator; 8 | import java.util.List; 9 | 10 | /** 11 | * Adapts a {@code org.apache.spark.api.java.JavaRDD} to a {@code SpashCollection}. 12 | * 13 | * @author Nicola Ferraro 14 | */ 15 | public class SpashCollectionRDDAdapter implements SpashCollection { 16 | 17 | private JavaRDD target; 18 | 19 | public SpashCollectionRDDAdapter(JavaRDD target) { 20 | this.target = target; 21 | } 22 | 23 | @Override 24 | public void mkString(PrintWriter writer) { 25 | Iterator it = target.toLocalIterator(); 26 | while(it.hasNext()) { 27 | T el = it.next(); 28 | writer.println(el != null ? el.toString() : ""); 29 | } 30 | } 31 | 32 | @Override 33 | public SpashCollection map(final SerializableFunction f) { 34 | return new SpashCollectionRDDAdapter<>(target.map(f)); 35 | } 36 | 37 | @Override 38 | public SpashCollection union(SpashCollection coll) { 39 | return new SpashCollectionUnionAdapter<>(this, coll); 40 | } 41 | 42 | @Override 43 | public SpashCollection filter(SerializableFunction condition) { 44 | return new SpashCollectionRDDAdapter<>(target.filter(condition)); 45 | } 46 | 47 | @Override 48 | public JavaRDD toRDD(JavaSparkContext sc) { 49 | return this.target; 50 | } 51 | 52 | @Override 53 | public List collect() { 54 | return this.target.collect(); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/spi/RmDirCommand.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command.spi; 2 | 3 | import it.nerdammer.spash.shell.api.fs.FileSystemFacade; 4 | import it.nerdammer.spash.shell.api.fs.SpashFileSystem; 5 | import it.nerdammer.spash.shell.command.AbstractCommand; 6 | import it.nerdammer.spash.shell.command.CommandResult; 7 | import it.nerdammer.spash.shell.command.ExecutionContext; 8 | 9 | import java.nio.file.Path; 10 | import java.util.List; 11 | 12 | /** 13 | * Command to remove a directory. 14 | * 15 | * @author Nicola Ferraro 16 | */ 17 | public class RmDirCommand extends AbstractCommand { 18 | 19 | public RmDirCommand(String commandString) { 20 | super(commandString); 21 | } 22 | 23 | @Override 24 | public CommandResult execute(ExecutionContext ctx) { 25 | 26 | FileSystemFacade fs = SpashFileSystem.get(); 27 | 28 | List files = this.getArguments(); 29 | if(files.size()==0) { 30 | return CommandResult.error(this, "Missing argument"); 31 | } else if(files.size()>1) { 32 | return CommandResult.error(this, "Too many arguments"); 33 | } 34 | 35 | String file = files.get(0); 36 | 37 | Path path = fs.getAbsolutePath(ctx.getSession().getWorkingDir(), file); 38 | 39 | if(!fs.exists(path.toString())) { 40 | return CommandResult.error(this, "No such file or directory"); 41 | } 42 | 43 | if(!fs.isDirectory(path.toString())) { 44 | return CommandResult.error(this, "Not a directory"); 45 | } 46 | 47 | boolean removed = fs.rm(path.toString(), false); 48 | if(!removed) { 49 | return CommandResult.error(this, "Directory not empty"); 50 | } 51 | 52 | return CommandResult.success(this); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/spi/HeadCommand.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command.spi; 2 | 3 | import it.nerdammer.spash.shell.api.fs.FileSystemFacade; 4 | import it.nerdammer.spash.shell.api.fs.SpashFileSystem; 5 | import it.nerdammer.spash.shell.api.spark.SpashSparkSubsystem; 6 | import it.nerdammer.spash.shell.command.AbstractCommand; 7 | import it.nerdammer.spash.shell.command.CommandResult; 8 | import it.nerdammer.spash.shell.command.ExecutionContext; 9 | import it.nerdammer.spash.shell.common.SpashCollection; 10 | import it.nerdammer.spash.shell.SpashSession; 11 | 12 | import java.nio.file.Path; 13 | 14 | /** 15 | * Command to get the first lines of a file or directory. 16 | * 17 | * @author Nicola Ferraro 18 | */ 19 | public class HeadCommand extends AbstractCommand { 20 | 21 | public static final int DEFAULT_NUM_LINES = 10; 22 | 23 | public HeadCommand(String commandString) { 24 | super(commandString); 25 | } 26 | 27 | @Override 28 | public CommandResult execute(ExecutionContext ctx) { 29 | 30 | FileSystemFacade fs = SpashFileSystem.get(); 31 | 32 | if(this.getArguments().size()==0) { 33 | return CommandResult.error(this, "No file provided"); 34 | } else if(this.getArguments().size()>1) { 35 | return CommandResult.error(this, "Too many arguments"); 36 | } 37 | 38 | String file = this.getArguments().get(0); 39 | 40 | Path path = fs.getAbsolutePath(ctx.getSession().getWorkingDir(), file); 41 | boolean exists = fs.exists(path.toString()); 42 | if(!exists) { 43 | return CommandResult.error(this, "No such file or directory"); 44 | } 45 | 46 | SpashCollection content = SpashSparkSubsystem.get().head(path, DEFAULT_NUM_LINES); 47 | 48 | return CommandResult.success(this, content); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/spi/CatCommand.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command.spi; 2 | 3 | import it.nerdammer.spash.shell.api.fs.FileSystemFacade; 4 | import it.nerdammer.spash.shell.api.fs.SpashFileSystem; 5 | import it.nerdammer.spash.shell.api.spark.SpashSparkSubsystem; 6 | import it.nerdammer.spash.shell.command.AbstractCommand; 7 | import it.nerdammer.spash.shell.command.CommandResult; 8 | import it.nerdammer.spash.shell.command.ExecutionContext; 9 | import it.nerdammer.spash.shell.common.SpashCollection; 10 | import it.nerdammer.spash.shell.SpashSession; 11 | import it.nerdammer.spash.shell.common.SpashCollectionEmptyAdapter; 12 | 13 | import java.nio.file.Path; 14 | import java.util.List; 15 | 16 | /** 17 | * Command to get the content of a file or directory. 18 | * 19 | * @author Nicola Ferraro 20 | */ 21 | public class CatCommand extends AbstractCommand { 22 | 23 | public CatCommand(String commandString) { 24 | super(commandString); 25 | } 26 | 27 | @Override 28 | public CommandResult execute(ExecutionContext ctx) { 29 | 30 | FileSystemFacade fs = SpashFileSystem.get(); 31 | 32 | List files = this.getArguments(); 33 | if(files.size()==0) { 34 | return CommandResult.error(this, "No file provided"); 35 | } 36 | 37 | SpashCollection content = new SpashCollectionEmptyAdapter<>(); 38 | for(String file : files) { 39 | Path path = fs.getAbsolutePath(ctx.getSession().getWorkingDir(), file); 40 | boolean exists = fs.exists(path.toString()); 41 | if (!exists) { 42 | return CommandResult.error(this, "No such file or directory"); 43 | } 44 | 45 | content = content.union(SpashSparkSubsystem.get().read(path)); 46 | } 47 | 48 | return CommandResult.success(this, content); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/common/SpashCollectionUnionAdapter.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.common; 2 | 3 | import org.apache.spark.api.java.JavaRDD; 4 | import org.apache.spark.api.java.JavaSparkContext; 5 | 6 | import java.io.PrintWriter; 7 | import java.util.List; 8 | 9 | /** 10 | * A SpashCollection that is the union of two {@code SpashCollection}s. 11 | * 12 | * @author Nicola Ferraro 13 | */ 14 | public class SpashCollectionUnionAdapter implements SpashCollection { 15 | 16 | /** 17 | * The first collection. 18 | */ 19 | private SpashCollection one; 20 | 21 | /** 22 | * The second collection. 23 | */ 24 | private SpashCollection two; 25 | 26 | public SpashCollectionUnionAdapter(SpashCollection one, SpashCollection two) { 27 | this.one = one; 28 | this.two = two; 29 | } 30 | 31 | @Override 32 | public void mkString(PrintWriter writer) { 33 | one.mkString(writer); 34 | two.mkString(writer); 35 | } 36 | 37 | @Override 38 | public SpashCollection map(SerializableFunction f) { 39 | return new SpashCollectionUnionAdapter<>(one.map(f), two.map(f)); 40 | } 41 | 42 | @Override 43 | public SpashCollection union(SpashCollection coll) { 44 | return new SpashCollectionUnionAdapter<>(this, coll); 45 | } 46 | 47 | @Override 48 | public SpashCollection filter(SerializableFunction condition) { 49 | return new SpashCollectionUnionAdapter<>(one.filter(condition), two.filter(condition)); 50 | } 51 | 52 | @Override 53 | public JavaRDD toRDD(JavaSparkContext sc) { 54 | return this.one.toRDD(sc).union(this.two.toRDD(sc)); 55 | } 56 | 57 | @Override 58 | public List collect() { 59 | List res = one.collect(); 60 | res.addAll(two.collect()); 61 | return res; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/spi/CdCommand.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command.spi; 2 | 3 | import it.nerdammer.spash.shell.api.fs.FileSystemFacade; 4 | import it.nerdammer.spash.shell.api.fs.SpashFileSystem; 5 | import it.nerdammer.spash.shell.command.AbstractCommand; 6 | import it.nerdammer.spash.shell.command.CommandResult; 7 | import it.nerdammer.spash.shell.SpashSession; 8 | import it.nerdammer.spash.shell.command.ExecutionContext; 9 | 10 | import java.nio.file.Path; 11 | 12 | /** 13 | * @author Nicola Ferraro 14 | */ 15 | public class CdCommand extends AbstractCommand { 16 | 17 | public CdCommand(String commandString) { 18 | super(commandString); 19 | } 20 | 21 | @Override 22 | public CommandResult execute(ExecutionContext ctx) { 23 | 24 | try { 25 | FileSystemFacade fs = SpashFileSystem.get(); 26 | 27 | if(this.getArguments().size()==0) { 28 | return CommandResult.error(this, "No file provided"); 29 | } else if(this.getArguments().size()>1) { 30 | return CommandResult.error(this, "Too many arguments"); 31 | } 32 | 33 | String dir = this.getArguments().get(0); 34 | if (dir == null) { 35 | dir = "/"; // the default dir 36 | } 37 | 38 | Path dest = fs.getAbsolutePath(ctx.getSession().getWorkingDir(), dir); 39 | boolean exists = fs.exists(dest.toString()); 40 | if (!exists) { 41 | return CommandResult.error(this, "No such file or directory"); 42 | } 43 | 44 | boolean isDir = fs.isDirectory(dest.toString()); 45 | if (!isDir) { 46 | return CommandResult.error(this, "Not a directory"); 47 | } 48 | 49 | ctx.getSession().setWorkingDir(dest.toAbsolutePath().normalize().toString()); 50 | 51 | return CommandResult.success(this); 52 | } catch(IllegalArgumentException e) { 53 | return CommandResult.error(this, "Illegal input"); 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/spi/WriteCommand.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command.spi; 2 | 3 | import it.nerdammer.spash.shell.api.fs.FileSystemFacade; 4 | import it.nerdammer.spash.shell.api.fs.SpashFileSystem; 5 | import it.nerdammer.spash.shell.api.spark.SparkFacade; 6 | import it.nerdammer.spash.shell.api.spark.SpashSparkSubsystem; 7 | import it.nerdammer.spash.shell.command.AbstractCommand; 8 | import it.nerdammer.spash.shell.command.CommandResult; 9 | import it.nerdammer.spash.shell.command.ExecutionContext; 10 | import it.nerdammer.spash.shell.common.SpashCollection; 11 | import it.nerdammer.spash.shell.common.SpashCollectionEmptyAdapter; 12 | 13 | import java.nio.file.Path; 14 | import java.util.List; 15 | 16 | /** 17 | * Command to append the result of a the previous command to a Hadoop file. 18 | * 19 | * @author Nicola Ferraro 20 | */ 21 | public class WriteCommand extends AbstractCommand { 22 | 23 | public WriteCommand(String commandString) { 24 | super(commandString); 25 | } 26 | 27 | @Override 28 | public CommandResult execute(ExecutionContext ctx) { 29 | 30 | FileSystemFacade fs = SpashFileSystem.get(); 31 | SparkFacade spark = SpashSparkSubsystem.get(); 32 | 33 | List files = this.getArguments(); 34 | if(files.size()==0) { 35 | return CommandResult.error(this, "No file provided"); 36 | } else if(files.size()>1) { 37 | return CommandResult.error(this, "Too many arguments"); 38 | } 39 | 40 | String file = this.getArguments().get(0); 41 | Path path = fs.getAbsolutePath(ctx.getSession().getWorkingDir(), file); 42 | 43 | CommandResult prevRes = ctx.getPreviousCommandResult(); 44 | if(prevRes!=null && !prevRes.isSuccess()) { 45 | return CommandResult.success(this); 46 | } 47 | 48 | SpashCollection content = prevRes!=null ? prevRes.getContent() : new SpashCollectionEmptyAdapter(); 49 | spark.write(content, path); 50 | 51 | return CommandResult.success(this); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/common/SpashCollection.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.common; 2 | 3 | import org.apache.spark.api.java.JavaRDD; 4 | import org.apache.spark.api.java.JavaSparkContext; 5 | 6 | import java.io.PrintWriter; 7 | import java.io.Serializable; 8 | import java.util.List; 9 | 10 | /** 11 | * An abstract collection. It can be implemented either by a standard java collection or a distributed collection. 12 | * 13 | * @author Nicola Ferraro 14 | */ 15 | public interface SpashCollection extends Serializable { 16 | 17 | /** 18 | * Prints the whole collection to the given writer. 19 | * 20 | * @param writer the writer on which the collection will be printed on. 21 | */ 22 | void mkString(PrintWriter writer); 23 | 24 | /** 25 | * Tranforms every element of the collection into another element. 26 | * 27 | * @param f the transformation function 28 | * @param the new type 29 | * @return the transformed collection 30 | */ 31 | SpashCollection map(SerializableFunction f); 32 | 33 | /** 34 | * Merges this collection to the one provided as input. 35 | * 36 | * @param coll the collection to merge 37 | * @return the union of the two collections 38 | */ 39 | SpashCollection union(SpashCollection coll); 40 | 41 | /** 42 | * Filters a collection retaining only elements respecting the given condition. 43 | * 44 | * @param condition the condition that must evaluate to true for retained elements 45 | * @return te resulting collection 46 | */ 47 | SpashCollection filter(SerializableFunction condition); 48 | 49 | /** 50 | * Converts this collection to a Spark {@code JavaRDD}. 51 | * 52 | * @param sc the current Spark context 53 | * @return a RDD corresponding to this collection 54 | */ 55 | JavaRDD toRDD(JavaSparkContext sc); 56 | 57 | /** 58 | * Collects the whole collection into a {@code List}. 59 | * 60 | * @return the elements of the collection 61 | */ 62 | List collect(); 63 | 64 | } 65 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/api/spark/SparkFacadeImpl.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.api.spark; 2 | 3 | import it.nerdammer.spash.shell.api.fs.FileSystemFacade; 4 | import it.nerdammer.spash.shell.api.fs.SpashFileSystem; 5 | import it.nerdammer.spash.shell.common.SpashCollection; 6 | import it.nerdammer.spash.shell.common.SpashCollectionListAdapter; 7 | import it.nerdammer.spash.shell.common.SpashCollectionRDDAdapter; 8 | import org.apache.spark.api.java.JavaRDD; 9 | import org.apache.spark.api.java.JavaSparkContext; 10 | 11 | import java.net.URI; 12 | import java.nio.file.Path; 13 | import java.util.List; 14 | 15 | /** 16 | * The default implementation of the {@code SparkFacade}. 17 | * 18 | * @author Nicola Ferraro 19 | */ 20 | public class SparkFacadeImpl implements SparkFacade { 21 | 22 | private JavaSparkContext sc; 23 | 24 | public SparkFacadeImpl(JavaSparkContext sc) { 25 | this.sc = sc; 26 | } 27 | 28 | @Override 29 | public SpashCollection read(Path file) { 30 | URI uri = SpashFileSystem.get().getURI(file.normalize().toString()); 31 | JavaRDD rdd = sc.textFile(uri.toString()); 32 | 33 | return new SpashCollectionRDDAdapter<>(rdd); 34 | } 35 | 36 | @Override 37 | public void write(SpashCollection content, Path file) { 38 | FileSystemFacade fs = SpashFileSystem.get(); 39 | if(fs.exists(file.toString())) { 40 | fs.rm(file.toString(), true); 41 | } 42 | 43 | URI uri = SpashFileSystem.get().getURI(file.normalize().toString()); 44 | content.toRDD(sc).saveAsTextFile(uri.toString()); 45 | } 46 | 47 | @Override 48 | public SpashCollection head(Path file, int lines) { 49 | URI uri = SpashFileSystem.get().getURI(file.normalize().toString()); 50 | JavaRDD rdd = sc.textFile(uri.toString()); 51 | List stream = rdd.take(lines); 52 | 53 | return new SpashCollectionListAdapter<>(stream); 54 | } 55 | 56 | @Override 57 | public SpashCollection parallelize(List coll) { 58 | return new SpashCollectionRDDAdapter<>(sc.parallelize(coll)); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/spi/RmCommand.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command.spi; 2 | 3 | import com.google.common.collect.ImmutableMap; 4 | import it.nerdammer.spash.shell.api.fs.FileSystemFacade; 5 | import it.nerdammer.spash.shell.api.fs.SpashFileSystem; 6 | import it.nerdammer.spash.shell.command.AbstractCommand; 7 | import it.nerdammer.spash.shell.command.CommandResult; 8 | import it.nerdammer.spash.shell.command.ExecutionContext; 9 | 10 | import java.nio.file.Path; 11 | import java.util.LinkedList; 12 | import java.util.List; 13 | 14 | /** 15 | * Command to remove a file or directory. 16 | * 17 | * @author Nicola Ferraro 18 | */ 19 | public class RmCommand extends AbstractCommand { 20 | 21 | public RmCommand(String commandString) { 22 | super(commandString, ImmutableMap.builder() 23 | .put("r", false) 24 | .build()); 25 | } 26 | 27 | @Override 28 | public CommandResult execute(ExecutionContext ctx) { 29 | 30 | FileSystemFacade fs = SpashFileSystem.get(); 31 | 32 | List files = this.getArguments(); 33 | if(files.size()==0) { 34 | return CommandResult.error(this, "Missing argument"); 35 | } 36 | 37 | boolean recursive = this.getOptions().containsKey("r"); 38 | 39 | List paths = new LinkedList<>(); 40 | for(String file : files) { 41 | Path path = fs.getAbsolutePath(ctx.getSession().getWorkingDir(), file); 42 | paths.add(path); 43 | 44 | if(!fs.exists(path.toString())) { 45 | return CommandResult.error(this, "No such file or directory"); 46 | } 47 | 48 | boolean isDirectory = fs.isDirectory(path.toString()); 49 | if(isDirectory && !recursive) { 50 | return CommandResult.error(this, "is a directory"); 51 | } 52 | } 53 | 54 | for(Path path : paths) { 55 | boolean removed = fs.rm(path.toString(), recursive); 56 | if(!removed) { 57 | return CommandResult.error(this, "Unable to remove file: operation interrupted"); 58 | } 59 | } 60 | 61 | return CommandResult.success(this); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /release/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | 8 | it.nerdammer.spash 9 | spash-pom 10 | 0.1 11 | 12 | 13 | spash-release 14 | 15 | pom 16 | 17 | 18 | 19 | ${project.groupId} 20 | spash-core 21 | ${project.version} 22 | 23 | 24 | 25 | org.apache.hadoop 26 | hadoop-hdfs 27 | 28 | 29 | org.apache.hadoop 30 | hadoop-client 31 | 32 | 33 | org.apache.spark 34 | spark-core_2.10 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | maven-assembly-plugin 46 | 2.6 47 | 48 | src/assembly/bin.xml 49 | spash-${project.version}-bin 50 | 51 | 52 | 53 | package 54 | 55 | single 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/AbstractCommand.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command; 2 | 3 | import java.util.*; 4 | 5 | /** 6 | * Provides useful methods and constructor for concrete commands. 7 | * 8 | * @author Nicola Ferraro 9 | */ 10 | public abstract class AbstractCommand implements Command { 11 | 12 | private CommandTokenizer commandTokenizer; 13 | 14 | public AbstractCommand(String commandString) { 15 | this(commandString, Collections.emptyMap()); 16 | } 17 | 18 | public AbstractCommand(String commandString, Map parametersValueInfo) { 19 | Set parameters = parametersValueInfo.keySet(); 20 | Set valuedParameters = new TreeSet<>(); 21 | for(Map.Entry e : parametersValueInfo.entrySet()) { 22 | if(e.getValue().booleanValue()) { 23 | valuedParameters.add(e.getKey()); 24 | } 25 | } 26 | 27 | this.commandTokenizer = new CommandTokenizer(commandString, valuedParameters); 28 | 29 | if(!parameters.containsAll(this.commandTokenizer.getOptions().keySet())) { 30 | Set wrongParameters = new TreeSet<>(this.commandTokenizer.getOptions().keySet()); 31 | wrongParameters.removeAll(parameters); 32 | throw new IllegalArgumentException("Unknown options: " + wrongParameters); 33 | } 34 | } 35 | 36 | 37 | /** 38 | * Returns the command associated to the command string. 39 | * 40 | * @return the command 41 | */ 42 | public String getCommand() { 43 | return this.commandTokenizer.getCommand(); 44 | } 45 | 46 | /** 47 | * Returns the option map associated to the command string. 48 | * Options not requiring a value contain null as value of the map. 49 | * 50 | * @return the option map 51 | */ 52 | public Map getOptions() { 53 | return this.commandTokenizer.getOptions(); 54 | } 55 | 56 | /** 57 | * Returns the argument part of the command string (the one after the parameter set). 58 | * 59 | * @return the arguments 60 | */ 61 | public List getArguments() { 62 | return this.commandTokenizer.getArguments(); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/common/SpashCollectionListAdapter.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.common; 2 | 3 | import ch.lambdaj.Lambda; 4 | import ch.lambdaj.function.convert.Converter; 5 | import org.apache.spark.api.java.JavaRDD; 6 | import org.apache.spark.api.java.JavaSparkContext; 7 | import org.hamcrest.BaseMatcher; 8 | import org.hamcrest.Description; 9 | 10 | import java.io.PrintWriter; 11 | import java.util.List; 12 | 13 | /** 14 | * Adapts a Java {@code Iterable} to a SpashCollection. 15 | * 16 | * @author Nicola Ferraro 17 | */ 18 | public class SpashCollectionListAdapter implements SpashCollection { 19 | 20 | /** 21 | * The iterable object. 22 | */ 23 | private List target; 24 | 25 | public SpashCollectionListAdapter(List target) { 26 | this.target = target; 27 | } 28 | 29 | @Override 30 | public void mkString(PrintWriter writer) { 31 | for(T el : target) { 32 | writer.println(el != null ? el.toString() : ""); 33 | } 34 | } 35 | 36 | @Override 37 | public SpashCollection map(final SerializableFunction f) { 38 | return new SpashCollectionListAdapter<>(Lambda.convert(target, new Converter() { 39 | @Override 40 | public R convert(T t) { 41 | return f.apply(t); 42 | } 43 | })); 44 | } 45 | 46 | @Override 47 | public SpashCollection union(SpashCollection coll) { 48 | return new SpashCollectionUnionAdapter<>(this, coll); 49 | } 50 | 51 | @Override 52 | public SpashCollection filter(final SerializableFunction condition) { 53 | return new SpashCollectionListAdapter<>(Lambda.filter(new BaseMatcher() { 54 | @Override 55 | public boolean matches(Object o) { 56 | return condition.apply((T) o); 57 | } 58 | @Override 59 | public void describeTo(Description description) { 60 | } 61 | }, target)); 62 | } 63 | 64 | @Override 65 | public JavaRDD toRDD(JavaSparkContext sc) { 66 | return sc.parallelize(this.target); 67 | } 68 | 69 | @Override 70 | public List collect() { 71 | return this.target; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/common/TabulatedValue.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.common; 2 | 3 | import ch.lambdaj.Lambda; 4 | import ch.lambdaj.function.convert.StringLengthConverter; 5 | import org.apache.commons.lang3.StringUtils; 6 | 7 | import java.util.ArrayList; 8 | import java.util.Iterator; 9 | import java.util.List; 10 | 11 | /** 12 | * @author Nicola Ferraro 13 | */ 14 | public class TabulatedValue implements Iterable { 15 | 16 | private List values; 17 | 18 | public TabulatedValue() { 19 | this.values = new ArrayList<>(); 20 | } 21 | 22 | public void set(int pos, String value) { 23 | this.values.set(pos, value); 24 | } 25 | 26 | public String remove(int pos) { 27 | return this.values.remove(pos); 28 | } 29 | 30 | public void add(String value) { 31 | this.values.add(value); 32 | } 33 | 34 | @Override 35 | public Iterator iterator() { 36 | return values.iterator(); 37 | } 38 | 39 | public String toString(List columnSizes) { 40 | if(columnSizes.size()!=this.values.size()) { 41 | throw new IllegalArgumentException("Size list and value list must have the same length: " + this.values.size() + "/" + columnSizes.size()); 42 | } 43 | 44 | Iterator sizes = columnSizes.iterator(); 45 | Iterator strs = this.iterator(); 46 | 47 | StringBuilder bui = new StringBuilder(); 48 | 49 | while(sizes.hasNext() && strs.hasNext()) { 50 | String str = strs.next(); 51 | int size = sizes.next(); 52 | bui.append(StringUtils.rightPad(str, size)); 53 | 54 | if(strs.hasNext()) { 55 | bui.append(" "); 56 | } 57 | } 58 | 59 | return bui.toString(); 60 | } 61 | 62 | public List columnSizes() { 63 | return Lambda.convert(this.values, new StringLengthConverter()); 64 | } 65 | 66 | public static List combineColumnSizes(List s1, List s2) { 67 | Iterator i1 = s1.iterator(); 68 | Iterator i2 = s2.iterator(); 69 | List res = new ArrayList<>(s1.size()); 70 | while(i1.hasNext() && i2.hasNext()) { 71 | int v = Math.max(i1.next(), i2.next()); 72 | res.add(v); 73 | } 74 | 75 | if(i1.hasNext() || i2.hasNext()) { 76 | throw new IllegalArgumentException("Collections have different sizes"); 77 | } 78 | 79 | return res; 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/CommandResult.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command; 2 | 3 | import it.nerdammer.spash.shell.common.SpashCollection; 4 | 5 | /** 6 | * Defines the result of the execution of a command. 7 | * 8 | * @author Nicola Ferraro 9 | */ 10 | public class CommandResult { 11 | 12 | /** 13 | * The executed command. 14 | */ 15 | private Command command; 16 | 17 | /** 18 | * Indicates whether the command completed successfully. 19 | */ 20 | private boolean success; 21 | 22 | /** 23 | * An error message explaining the problem when the command failed to execute. 24 | */ 25 | private String errorMessage; 26 | 27 | /** 28 | * The result of the command. 29 | */ 30 | private SpashCollection content; 31 | 32 | private CommandResult(Command command, boolean success, String errorMessage, SpashCollection content) { 33 | this.command = command; 34 | this.success = success; 35 | this.errorMessage = errorMessage; 36 | this.content = content; 37 | } 38 | 39 | public static CommandResult success(Command command, SpashCollection content) { 40 | return new CommandResult(command, true, null, content); 41 | } 42 | 43 | public static CommandResult success(Command command) { 44 | return new CommandResult(command, true, null, null); 45 | } 46 | 47 | public static CommandResult error(Command command, String errorMessage) { 48 | return new CommandResult(command, false, errorMessage, null); 49 | } 50 | 51 | public Command getCommand() { 52 | return command; 53 | } 54 | 55 | public void setCommand(Command command) { 56 | this.command = command; 57 | } 58 | 59 | public boolean isSuccess() { 60 | return success; 61 | } 62 | 63 | public void setSuccess(boolean success) { 64 | this.success = success; 65 | } 66 | 67 | public String getErrorMessage() { 68 | return errorMessage; 69 | } 70 | 71 | public void setErrorMessage(String errorMessage) { 72 | this.errorMessage = errorMessage; 73 | } 74 | 75 | public SpashCollection getContent() { 76 | return content; 77 | } 78 | 79 | public void setContent(SpashCollection content) { 80 | this.content = content; 81 | } 82 | 83 | @Override 84 | public String toString() { 85 | return "CommandResult{" + 86 | "command=" + command + 87 | ", success=" + success + 88 | '}'; 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/SpashCommandCompleter.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell; 2 | 3 | import ch.lambdaj.Lambda; 4 | import ch.lambdaj.function.convert.DefaultStringConverter; 5 | import ch.lambdaj.function.convert.PropertyExtractor; 6 | import it.nerdammer.spash.shell.api.fs.SpashFileSystem; 7 | import it.nerdammer.spash.shell.command.CommandFactory; 8 | import jline.console.completer.Completer; 9 | import jline.internal.Preconditions; 10 | import org.hamcrest.text.StringStartsWith; 11 | 12 | import java.nio.file.Path; 13 | import java.util.List; 14 | import java.util.Set; 15 | 16 | /** 17 | * @author Nicola Ferraro 18 | */ 19 | public class SpashCommandCompleter implements Completer { 20 | 21 | private Set commands; 22 | 23 | private SpashSession session; 24 | 25 | public SpashCommandCompleter(SpashSession session) { 26 | this.commands = CommandFactory.getInstance().getAvailableCommands(); 27 | this.session = session; 28 | } 29 | 30 | @Override 31 | public int complete(String buffer, int cursor, List candidates) { 32 | Preconditions.checkNotNull(candidates); 33 | 34 | String text = contextualBuffer(buffer, cursor); 35 | 36 | List commands = Lambda.filter(StringStartsWith.startsWith(text), this.commands); 37 | if(commands.size()>0) { 38 | candidates.addAll(commands); 39 | 40 | if(candidates.size()==1) { 41 | candidates.set(0, candidates.get(0) + " "); 42 | } 43 | 44 | return candidates.isEmpty() ? -1 : 0; 45 | 46 | } else if(text.contains(" ")) { 47 | int insertion = text.lastIndexOf(" ") + 1; 48 | String tailBuffer = text.substring(insertion); 49 | 50 | List files = Lambda.convert(Lambda.convert(SpashFileSystem.get().ls(this.session.getWorkingDir()).collect(), new PropertyExtractor("fileName")), new DefaultStringConverter()); 51 | files = Lambda.filter(StringStartsWith.startsWith(tailBuffer), files); 52 | 53 | candidates.addAll(files); 54 | 55 | if(candidates.size()==1) { 56 | candidates.set(0, candidates.get(0) + " "); 57 | } 58 | 59 | return candidates.isEmpty() ? -1 : insertion; 60 | } 61 | 62 | return -1; 63 | } 64 | 65 | private String contextualBuffer(String buffer, int cursor) { 66 | if(buffer==null) { 67 | return ""; 68 | } else if(cursor> commands; 20 | 21 | private CommandFactory() { 22 | this.commands = new TreeMap<>(); 23 | commands.put(">", WriteCommand.class); 24 | commands.put("cat", CatCommand.class); 25 | commands.put("cd", CdCommand.class); 26 | commands.put("echo", EchoCommand.class); 27 | commands.put("exit", ExitCommand.class); 28 | commands.put("grep", GrepCommand.class); 29 | commands.put("head", HeadCommand.class); 30 | commands.put("ls", LsCommand.class); 31 | commands.put("mkdir", MkDirCommand.class); 32 | commands.put("pwd", PwdCommand.class); 33 | commands.put("rm", RmCommand.class); 34 | commands.put("rmdir", RmDirCommand.class); 35 | commands.put("test", NoOpCommand.class); 36 | } 37 | 38 | public static CommandFactory getInstance() { 39 | return INSTANCE; 40 | } 41 | 42 | public Set getAvailableCommands() { 43 | return this.commands.keySet(); 44 | } 45 | 46 | public Command getCommand(String commandStr) { 47 | 48 | if(commandStr.trim().equals("")) { 49 | return new NoOpCommand(commandStr); 50 | } else { 51 | String[] args = commandStr.split(" "); // TODO parse 52 | 53 | if(commands.containsKey(args[0])) { 54 | try { 55 | Class commandClass = commands.get(args[0]); 56 | Command c = commandClass.getConstructor(String.class).newInstance(commandStr); 57 | return c; 58 | } catch(InvocationTargetException e) { 59 | Throwable t = e.getCause(); 60 | if(t instanceof RuntimeException) { 61 | throw (RuntimeException)t; 62 | } else if(t instanceof Error) { 63 | throw (Error)t; 64 | } else { 65 | throw new IllegalStateException(t); 66 | } 67 | } catch(RuntimeException e) { 68 | throw e; 69 | } catch(Exception e) { 70 | throw new IllegalStateException("Unable to create command '" + commandStr + "'", e); 71 | } 72 | } 73 | } 74 | 75 | return new UnknownCommand(commandStr); 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/api/fs/FileSystemFacade.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.api.fs; 2 | 3 | import it.nerdammer.spash.shell.common.SpashCollection; 4 | 5 | import java.net.URI; 6 | import java.nio.file.FileSystem; 7 | import java.nio.file.Path; 8 | import java.nio.file.attribute.PosixFileAttributes; 9 | 10 | /** 11 | * An abstraction for a file system. 12 | * 13 | * @author Nicola Ferraro 14 | */ 15 | public interface FileSystemFacade { 16 | 17 | /** 18 | * Returns the defaul filesystem. 19 | * 20 | * @return the filesystem 21 | */ 22 | FileSystem getFileSystem(); 23 | 24 | /** 25 | * Returns a collection of paths contained in the given path. 26 | * 27 | * @param path the base path 28 | * @return the sub paths 29 | */ 30 | SpashCollection ls(String path); 31 | 32 | /** 33 | * Retrieve the absolute path of a given path provided as string. 34 | * The given path can be either absolute or relative. When it is relative, the base path is used to 35 | * get the absolute path. Absolute paths are simply transformed into {@link Path} objects. 36 | * 37 | * @param base the current base path 38 | * @param path the path that will be converted 39 | * @return the absolute path object 40 | */ 41 | Path getAbsolutePath(String base, String path); 42 | 43 | /** 44 | * Indicates wether the given path exists. 45 | * 46 | * @param path the path to check 47 | * @return true if it exists, false otherwise 48 | */ 49 | boolean exists(String path); 50 | 51 | /** 52 | * Indicates wether the given path is a directory. 53 | * 54 | * @param path the path to check 55 | * @return true if it is a directory, false otherwise 56 | */ 57 | boolean isDirectory(String path); 58 | 59 | /** 60 | * Returns the full URI of the path, to identify the resource from outside the facade. 61 | * 62 | * @param path the path of the file 63 | * @return the URI of the path 64 | */ 65 | URI getURI(String path); 66 | 67 | /** 68 | * Creates the directory corresponding to the given path. 69 | * 70 | * @param path the absolute path of the directory 71 | * @return true if the directory did not exist before 72 | */ 73 | boolean mkdir(String path); 74 | 75 | /** 76 | * Removes the directory or file corresponding to the given path. If the target is a directory, it must be empty. 77 | * 78 | * @param path the absolute path of the directory or file 79 | * @param recursive true if the content of the directory should be removed as well 80 | * @return true if the target can be removed 81 | */ 82 | boolean rm(String path, boolean recursive); 83 | 84 | /** 85 | * Gets the posix file attributes of the given file. 86 | * 87 | * @param path the file 88 | * @return the posix file attributes 89 | */ 90 | PosixFileAttributes getAttributes(String path); 91 | 92 | } 93 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/SpashConfig.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.io.FileNotFoundException; 7 | import java.io.FileReader; 8 | import java.io.IOException; 9 | import java.io.Reader; 10 | import java.util.Properties; 11 | 12 | /** 13 | * Configuration options for Spash. 14 | * 15 | * @author Nicola Ferraro 16 | */ 17 | public class SpashConfig { 18 | 19 | private static final SpashConfig INSTANCE = new SpashConfig(); 20 | 21 | private Logger logger = LoggerFactory.getLogger(getClass()); 22 | 23 | private Properties properties; 24 | 25 | private SpashConfig() { 26 | 27 | this.properties = new Properties(); 28 | 29 | String configName = "spash.config"; 30 | String configFile = System.getProperty(configName); 31 | if(configFile==null) { 32 | logger.warn("No configuration property " + configName + " found. Using defaults."); 33 | } else { 34 | try(Reader r = new FileReader(configFile)) { 35 | properties.load(r); 36 | } catch(FileNotFoundException e) { 37 | throw new IllegalStateException("Unable to find the file defined in " + configName + " property: " + configFile, e); 38 | } catch(IOException e) { 39 | throw new IllegalStateException("Unable to load the file defined in " + configName + " property: " + configFile, e); 40 | } 41 | } 42 | 43 | } 44 | 45 | public static SpashConfig getInstance() { 46 | return INSTANCE; 47 | } 48 | 49 | public String spashKeyFileName() { 50 | String configName = "spash.config.dir"; 51 | String configDir = System.getProperty(configName); 52 | return configDir!=null ? configDir + "/key.ser" : "key.ser"; 53 | } 54 | 55 | public String spashKeyAlgorithm() { 56 | return propOrElse("spash.key.algorithm", "RSA"); 57 | } 58 | 59 | public int spashKeyLength() { 60 | return propOrElseAsInt("spash.key.length", 2048); 61 | } 62 | 63 | public int spashListenPort() { 64 | return propOrElseAsInt("spash.listen.port", 2222); 65 | } 66 | 67 | public String hdfsHost() { 68 | return propOrElse("spash.hdfs.host", "hdfshost"); 69 | } 70 | 71 | public int hdfsPort() { 72 | return propOrElseAsInt("spash.hdfs.port", 8020); 73 | } 74 | 75 | public String hdfsHostPort() { 76 | return hdfsHost() + ":" + hdfsPort(); 77 | } 78 | 79 | 80 | private int propOrElseAsInt(String name, int defaulz) { 81 | String val = this.properties.getProperty(name); 82 | if(val==null) { 83 | return defaulz; 84 | } 85 | try { 86 | return Integer.parseInt(val); 87 | } catch(NumberFormatException e) { 88 | throw new IllegalStateException("Illegal configuration for parameter " + name + ". Value: " + val, e); 89 | } 90 | } 91 | 92 | private String propOrElse(String name, String defaulz) { 93 | return this.properties.getProperty(name, defaulz); 94 | } 95 | 96 | } 97 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/spi/GrepCommand.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command.spi; 2 | 3 | import ch.lambdaj.Lambda; 4 | import com.google.common.collect.ImmutableMap; 5 | import it.nerdammer.spash.shell.command.*; 6 | import it.nerdammer.spash.shell.common.SerializableFunction; 7 | import it.nerdammer.spash.shell.common.SpashCollection; 8 | import it.nerdammer.spash.shell.common.SpashCollectionEmptyAdapter; 9 | 10 | import java.util.List; 11 | 12 | /** 13 | * Command to filter collections of data. 14 | * 15 | * @author Nicola Ferraro 16 | */ 17 | public class GrepCommand extends AbstractCommand { 18 | 19 | public GrepCommand(String commandString) { 20 | super(commandString, ImmutableMap.builder() 21 | .put("v", false) 22 | .put("i", false) 23 | .build()); 24 | } 25 | 26 | @Override 27 | public CommandResult execute(ExecutionContext ctx) { 28 | 29 | List args = this.getArguments(); 30 | if(args.size()==0) { 31 | return CommandResult.error(this, "Missing argument"); 32 | } else if(args.size()==1) { 33 | // pipe mode 34 | SpashCollection prev = ctx.getPreviousCommandResult() != null ? ctx.getPreviousCommandResult().getContent() : null; 35 | if(prev==null) { 36 | prev = new SpashCollectionEmptyAdapter<>(); 37 | } 38 | return execute(ctx, prev); 39 | } else { 40 | // cat mode 41 | 42 | String catCmdString = "cat " + Lambda.join(args.subList(1, args.size()), " "); 43 | Command cat = CommandFactory.getInstance().getCommand(catCmdString); 44 | CommandResult res = cat.execute(ctx); 45 | if(!res.isSuccess()) { 46 | return CommandResult.error(this, res.getErrorMessage()); 47 | } 48 | 49 | return execute(ctx, res.getContent()); 50 | } 51 | 52 | } 53 | 54 | protected CommandResult execute(ExecutionContext ctx, SpashCollection source) { 55 | 56 | String filter = this.getArguments().get(0); 57 | boolean reverse = this.getOptions().containsKey("v"); 58 | boolean insensitive = this.getOptions().containsKey("i"); 59 | 60 | SpashCollection content = source.filter(new FilterFunction(filter, reverse, insensitive)); 61 | 62 | return CommandResult.success(this, content); 63 | } 64 | 65 | static class FilterFunction extends SerializableFunction { 66 | 67 | private String filter; 68 | 69 | private boolean reverse; 70 | 71 | private boolean insensitive; 72 | 73 | public FilterFunction(String filter, boolean reverse, boolean insensitive) { 74 | this.filter = filter; 75 | this.reverse = reverse; 76 | this.insensitive = insensitive; 77 | } 78 | 79 | @Override 80 | public Boolean apply(String s) { 81 | return reverse ^ ( 82 | insensitive ? 83 | s.toLowerCase().contains(filter.toLowerCase()) : 84 | s.contains(filter) 85 | ); 86 | } 87 | 88 | } 89 | 90 | } 91 | 92 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/SshServerFactory.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell; 2 | 3 | import it.nerdammer.spash.shell.api.fs.SpashFileSystem; 4 | import org.apache.sshd.common.NamedFactory; 5 | import org.apache.sshd.common.file.FileSystemFactory; 6 | import org.apache.sshd.common.keyprovider.AbstractKeyPairProvider; 7 | import org.apache.sshd.common.keyprovider.KeyPairProvider; 8 | import org.apache.sshd.common.session.Session; 9 | import org.apache.sshd.server.Command; 10 | import org.apache.sshd.server.SshServer; 11 | import org.apache.sshd.server.auth.UserAuth; 12 | import org.apache.sshd.server.auth.password.PasswordAuthenticator; 13 | import org.apache.sshd.server.auth.password.PasswordChangeRequiredException; 14 | import org.apache.sshd.server.auth.password.UserAuthPasswordFactory; 15 | import org.apache.sshd.server.keyprovider.AbstractGeneratorHostKeyProvider; 16 | import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider; 17 | import org.apache.sshd.server.scp.ScpCommandFactory; 18 | import org.apache.sshd.server.session.ServerSession; 19 | import org.apache.sshd.server.subsystem.sftp.SftpSubsystemFactory; 20 | 21 | import java.io.File; 22 | import java.io.IOException; 23 | import java.nio.file.FileSystem; 24 | import java.util.ArrayList; 25 | import java.util.List; 26 | 27 | /** 28 | * Allows creating {@code SshServer} instances. 29 | * 30 | * @author Nicola Ferraro 31 | */ 32 | public class SshServerFactory { 33 | 34 | 35 | public static SshServer create() { 36 | 37 | SshServer sshd = SshServer.setUpDefaultServer(); 38 | sshd.setPort(SpashConfig.getInstance().spashListenPort()); 39 | 40 | AbstractGeneratorHostKeyProvider keyProvider = new SimpleGeneratorHostKeyProvider(new File(SpashConfig.getInstance().spashKeyFileName())); 41 | keyProvider.setAlgorithm(SpashConfig.getInstance().spashKeyAlgorithm()); 42 | keyProvider.setKeySize(SpashConfig.getInstance().spashKeyLength()); 43 | 44 | sshd.setKeyPairProvider(keyProvider); 45 | 46 | List> userAuthFactories = new ArrayList>(); 47 | userAuthFactories.add(new UserAuthPasswordFactory()); 48 | sshd.setUserAuthFactories(userAuthFactories); 49 | 50 | sshd.setPasswordAuthenticator(new PasswordAuthenticator() { 51 | @Override 52 | public boolean authenticate(String username, String password, ServerSession serverSession) throws PasswordChangeRequiredException { 53 | return username!=null && username.length()>0 && username.equals(password); 54 | } 55 | }); 56 | 57 | sshd.setShellFactory(new SpashShellFactory()); 58 | 59 | List> namedFactoryList = new ArrayList<>(); 60 | namedFactoryList.add(new SftpSubsystemFactory()); 61 | sshd.setSubsystemFactories(namedFactoryList); 62 | 63 | sshd.setCommandFactory(new ScpCommandFactory()); 64 | 65 | sshd.setFileSystemFactory(new FileSystemFactory() { 66 | @Override 67 | public FileSystem createFileSystem(Session session) throws IOException { 68 | return SpashFileSystem.get().getFileSystem(); 69 | } 70 | }); 71 | 72 | return sshd; 73 | } 74 | 75 | } 76 | 77 | 78 | -------------------------------------------------------------------------------- /core/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | 8 | it.nerdammer.spash 9 | spash-pom 10 | 0.1 11 | 12 | 13 | spash-core 14 | 15 | 16 | 1.1.0 17 | 2.3.3 18 | 2.6.0 19 | 1.6.1 20 | 1.7.18 21 | 1.2.17 22 | 2.12 23 | 1.0.2 24 | 25 | 26 | 27 | 28 | 29 | org.apache.sshd 30 | sshd-core 31 | ${sshd.version} 32 | 33 | 34 | org.slf4j 35 | slf4j-jdk14 36 | 37 | 38 | 39 | 40 | 41 | com.googlecode.lambdaj 42 | lambdaj 43 | ${lambdaj.version} 44 | 45 | 46 | 47 | 48 | org.apache.hadoop 49 | hadoop-hdfs 50 | ${hadoop.version} 51 | 52 | 53 | javax.servlet 54 | servlet-api 55 | 56 | 57 | 58 | 59 | 60 | org.apache.hadoop 61 | hadoop-client 62 | ${hadoop.version} 63 | 64 | 65 | javax.servlet 66 | servlet-api 67 | 68 | 69 | 70 | 71 | 72 | org.apache.spark 73 | spark-core_2.10 74 | ${spark.version} 75 | 76 | 77 | 78 | org.slf4j 79 | slf4j-log4j12 80 | ${slf4j.version} 81 | 82 | 83 | log4j 84 | log4j 85 | ${log4j.version} 86 | 87 | 88 | 89 | jline 90 | jline 91 | ${jline.version} 92 | 93 | 94 | 95 | com.github.jsr203hadoop 96 | jsr203hadoop 97 | ${jsr203hadoop.version} 98 | 99 | 100 | junit 101 | junit 102 | 4.12 103 | test 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | org.apache.maven.plugins 112 | maven-compiler-plugin 113 | 3.5.1 114 | 115 | 1.7 116 | 1.7 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/ExpressionTokenizer.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command; 2 | 3 | import ch.lambdaj.Lambda; 4 | import ch.lambdaj.function.convert.Converter; 5 | import org.hamcrest.number.OrderingComparisons; 6 | 7 | import java.util.ArrayList; 8 | import java.util.Arrays; 9 | import java.util.LinkedList; 10 | import java.util.List; 11 | 12 | /** 13 | * Tokenizes an expression to extract all command strings contained in it. 14 | * 15 | * @author Nicola Ferraro 16 | */ 17 | public class ExpressionTokenizer { 18 | 19 | private List commandStrings; 20 | 21 | public ExpressionTokenizer(String arg) { 22 | if(arg==null) { 23 | throw new IllegalArgumentException("Null string"); 24 | } 25 | if(containsUnquoted(arg, "||")) { 26 | throw new IllegalArgumentException("Unexpected sequence: '||'"); 27 | } 28 | if(containsUnquoted(arg, ">>>")) { 29 | throw new IllegalArgumentException("Unexpected sequence: '>>>'"); 30 | } 31 | 32 | List piped = splitUnquoted(arg, "|"); 33 | LinkedList exprs = new LinkedList<>(Lambda.convert(piped, new Converter() { 34 | @Override 35 | public String convert(String s) { 36 | return s.trim(); 37 | } 38 | })); 39 | 40 | this.commandStrings = new ArrayList<>(); 41 | while(!exprs.isEmpty()) { 42 | String expr = exprs.poll(); 43 | Integer start = minPositive(indexOfUnquoted(expr, " "), indexOfUnquoted(expr, "\t"), indexOfUnquoted(expr, "\r"), indexOfUnquoted(expr, "\n")); 44 | if(start==null) { 45 | this.commandStrings.add(expr.trim()); 46 | continue; 47 | } 48 | 49 | int idx = indexOfUnquoted(expr, ">", start); 50 | if(idx>0) { 51 | String first = expr.substring(0, idx).trim(); 52 | this.commandStrings.add(first); 53 | exprs.addFirst(expr.substring(idx).trim()); 54 | } else { 55 | this.commandStrings.add(expr.trim()); 56 | } 57 | } 58 | } 59 | 60 | public List getCommandStrings() { 61 | return commandStrings; 62 | } 63 | 64 | private Integer minPositive(Integer... args) { 65 | return min(Lambda.filter(OrderingComparisons.greaterThanOrEqualTo(0), Arrays.asList(args))); 66 | } 67 | 68 | private Integer min(List args) { 69 | Integer min = null; 70 | for(int i=0; i splitUnquoted(String str, String sub) { 77 | List part = new ArrayList<>(); 78 | int pos = indexOfUnquoted(str, sub); 79 | if(pos >= 0) { 80 | String s = str.substring(0, pos); 81 | part.add(s); 82 | pos += sub.length(); 83 | if(pos part2 = splitUnquoted(str2, sub); 86 | part.addAll(part2); 87 | } 88 | } else { 89 | part.add(str); 90 | } 91 | 92 | return part; 93 | } 94 | 95 | protected boolean containsUnquoted(String str, String sub) { 96 | return indexOfUnquoted(str, sub) >= 0; 97 | } 98 | 99 | protected int indexOfUnquoted(String str, String sub) { 100 | return indexOfUnquoted(str, sub, 0); 101 | } 102 | 103 | protected int indexOfUnquoted(String str, String sub, int startPos) { 104 | int start = startPos; 105 | while(start >= 0 && start < str.length()) { 106 | start = str.indexOf(sub, start); 107 | if(!isQuoted(str, start)) { 108 | return start; 109 | } 110 | start = start + 1; 111 | } 112 | return -1; 113 | } 114 | 115 | protected boolean isQuoted(String str, int pos) { 116 | int start = str.indexOf("\""); 117 | if(start < 0 || pos < start) { 118 | return false; 119 | } 120 | int end = str.indexOf("\"", start + 1); 121 | if(end < 0) { 122 | return false; 123 | } 124 | if(pos <= end) { 125 | return true; 126 | } 127 | 128 | return isQuoted(str.substring(end + 1), pos-end-1); 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/spi/LsCommand.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command.spi; 2 | 3 | import com.google.common.collect.ImmutableMap; 4 | import it.nerdammer.spash.shell.api.fs.SpashFileSystem; 5 | import it.nerdammer.spash.shell.command.AbstractCommand; 6 | import it.nerdammer.spash.shell.command.CommandResult; 7 | import it.nerdammer.spash.shell.command.ExecutionContext; 8 | import it.nerdammer.spash.shell.common.SerializableFunction; 9 | import it.nerdammer.spash.shell.common.SpashCollection; 10 | import it.nerdammer.spash.shell.common.SpashCollectionListAdapter; 11 | import it.nerdammer.spash.shell.common.TabulatedValue; 12 | 13 | import java.nio.file.Path; 14 | import java.nio.file.attribute.PosixFileAttributes; 15 | import java.nio.file.attribute.PosixFilePermission; 16 | import java.text.SimpleDateFormat; 17 | import java.util.*; 18 | 19 | /** 20 | * A command that lists the files contained in the current directory. 21 | * 22 | * @author Nicola Ferraro 23 | */ 24 | public class LsCommand extends AbstractCommand { 25 | 26 | public LsCommand(String commandString) { 27 | super(commandString, ImmutableMap.builder() 28 | .put("l", false) 29 | .build()); 30 | } 31 | 32 | @Override 33 | public CommandResult execute(ExecutionContext ctx) { 34 | 35 | List args = this.getArguments(); 36 | if(args.size()>0) { 37 | return CommandResult.error(this, "Unexpected arguments: " + args); 38 | } 39 | 40 | SpashCollection files = SpashFileSystem.get().ls(ctx.getSession().getWorkingDir()); 41 | 42 | SpashCollection fileNames = files.map(new SerializableFunction() { 43 | @Override 44 | public String apply(Path v1) { 45 | return v1.getFileName().toString(); 46 | } 47 | }); 48 | 49 | SpashCollection res = fileNames; 50 | if(this.getOptions().containsKey("l")) { 51 | res = toDetail(ctx, fileNames); 52 | } 53 | 54 | return CommandResult.success(this, res); 55 | } 56 | 57 | private SpashCollection toDetail(ExecutionContext ctx, SpashCollection fileColl) { 58 | List files = fileColl.collect(); 59 | if(files.isEmpty()) { 60 | return fileColl; 61 | } 62 | 63 | SimpleDateFormat fmtThisYear = new SimpleDateFormat("dd MMM HH:mm"); 64 | SimpleDateFormat fmtPast = new SimpleDateFormat("dd MMM yyyy"); 65 | 66 | List fileDets = new ArrayList<>(); 67 | for(String file : files) { 68 | 69 | String fullFile = SpashFileSystem.get().getAbsolutePath(ctx.getSession().getWorkingDir(), file).normalize().toString(); 70 | 71 | TabulatedValue val = new TabulatedValue(); 72 | 73 | PosixFileAttributes attr = SpashFileSystem.get().getAttributes(fullFile); 74 | Set perms = attr.permissions(); 75 | 76 | StringBuilder permStr = new StringBuilder(); 77 | permStr.append(attr.isDirectory() ? "d" : "-"); 78 | permStr.append(perms.contains(PosixFilePermission.OWNER_READ) ? "r" : "-"); 79 | permStr.append(perms.contains(PosixFilePermission.OWNER_WRITE) ? "w" : "-"); 80 | permStr.append(perms.contains(PosixFilePermission.OWNER_EXECUTE) ? "x" : "-"); 81 | permStr.append(perms.contains(PosixFilePermission.GROUP_READ) ? "r" : "-"); 82 | permStr.append(perms.contains(PosixFilePermission.GROUP_WRITE) ? "w" : "-"); 83 | permStr.append(perms.contains(PosixFilePermission.GROUP_EXECUTE) ? "x" : "-"); 84 | permStr.append(perms.contains(PosixFilePermission.OTHERS_READ) ? "r" : "-"); 85 | permStr.append(perms.contains(PosixFilePermission.OTHERS_WRITE) ? "w" : "-"); 86 | permStr.append(perms.contains(PosixFilePermission.OTHERS_EXECUTE) ? "x" : "-"); 87 | 88 | val.add(permStr.toString()); 89 | 90 | val.add(String.valueOf(SpashFileSystem.get().ls(fullFile).collect().size() + 2)); 91 | 92 | val.add(attr.owner().getName()); 93 | val.add(attr.group().getName()); 94 | 95 | val.add(String.valueOf(attr.size())); 96 | 97 | long fileTime = attr.lastModifiedTime().toMillis(); 98 | Date fileDate = new Date(fileTime); 99 | 100 | Calendar lastYear = Calendar.getInstance(); 101 | lastYear.add(Calendar.YEAR, -1); 102 | long lastYearTime = lastYear.getTimeInMillis(); 103 | 104 | if(fileTime>lastYearTime) { 105 | val.add(fmtThisYear.format(fileDate)); 106 | } else { 107 | val.add(fmtPast.format(fileDate)); 108 | } 109 | 110 | val.add(file); 111 | fileDets.add(val); 112 | } 113 | 114 | List combinedColSizes = fileDets.get(0).columnSizes(); 115 | for(TabulatedValue val : fileDets) { 116 | List colSizes = val.columnSizes(); 117 | combinedColSizes = TabulatedValue.combineColumnSizes(combinedColSizes, colSizes); 118 | } 119 | 120 | List res = new ArrayList<>(); 121 | for(TabulatedValue val : fileDets) { 122 | String vStr = val.toString(combinedColSizes); 123 | res.add(vStr); 124 | } 125 | 126 | return new SpashCollectionListAdapter<>(res); 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/command/CommandTokenizer.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command; 2 | 3 | import java.util.*; 4 | import java.util.regex.Matcher; 5 | import java.util.regex.Pattern; 6 | 7 | /** 8 | * Tokenizes a command string to allow getting all the parts. 9 | * 10 | * @author Nicola Ferraro 11 | */ 12 | public class CommandTokenizer { 13 | 14 | private static Pattern regex = Pattern.compile("[\"][^\"]*[\"]|[^\\s]+"); 15 | 16 | private Set valuedParameters; 17 | 18 | private String command; 19 | 20 | private Map options; 21 | 22 | private List arguments; 23 | 24 | public CommandTokenizer(String arg) { 25 | this(arg, Collections.emptySet()); 26 | } 27 | 28 | public CommandTokenizer(String arg, Set valuedParameters) { 29 | if(arg==null || arg.trim().length()==0) { 30 | throw new IllegalArgumentException("Empty string: '" + arg + "'"); 31 | } 32 | if(valuedParameters==null) { 33 | throw new IllegalArgumentException("valued parameters cannot be null, use an empty set"); 34 | } 35 | 36 | this.valuedParameters = valuedParameters; 37 | 38 | 39 | List components = new ArrayList<>(); 40 | Matcher m = regex.matcher(arg); 41 | while(m.find()) { 42 | String next = m.group(); 43 | components.add(next); 44 | } 45 | 46 | if(components.size()==0) { 47 | throw new IllegalArgumentException("No command found"); 48 | } 49 | 50 | this.command = components.get(0); 51 | 52 | this.options = new TreeMap<>(); 53 | int i=1; 54 | for(; i1) { 86 | // multiparam 87 | for(char p : param.toCharArray()) { 88 | String pp = p + ""; 89 | if(this.options.containsKey(pp)) { 90 | throw new IllegalArgumentException("Parameter '" + pp + "' is defined twice"); 91 | } 92 | if(this.valuedParameters.contains(pp)) { 93 | throw new IllegalArgumentException("Missing value for parameter '" + pp + "'"); 94 | } 95 | this.options.put(pp, null); 96 | } 97 | } else { 98 | // single param 99 | if(this.options.containsKey(param)) { 100 | throw new IllegalArgumentException("Parameter '" + param + "' is defined twice"); 101 | } 102 | 103 | String value = null; 104 | 105 | boolean needsValue = this.valuedParameters.contains(param); 106 | if(needsValue && i==components.size()-1) { 107 | throw new IllegalArgumentException("Missing value for parameter '" + param + "'"); 108 | } else if(needsValue) { 109 | value = components.get(++i); 110 | } 111 | 112 | this.options.put(param, value); 113 | } 114 | 115 | } 116 | } 117 | 118 | this.arguments = new ArrayList<>(); 119 | for(; i getOptions() { 140 | return this.options; 141 | } 142 | 143 | /** 144 | * Returns the argument part of the command string (the one after the parameter set). 145 | * 146 | * @return the arguments 147 | */ 148 | public List getArguments() { 149 | return arguments; 150 | } 151 | 152 | private String unquoted(String s) { 153 | if(s.startsWith("\"") && s.endsWith("\"") && s.length()>=2) { 154 | s = s.substring(1, s.length()-1); 155 | } 156 | return s; 157 | } 158 | 159 | } 160 | -------------------------------------------------------------------------------- /core/src/test/java/it/nerdammer/spash/shell/command/ExpressionTokenizerTest.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command; 2 | 3 | import org.junit.Test; 4 | 5 | import java.util.*; 6 | 7 | import static org.junit.Assert.*; 8 | 9 | /** 10 | * @author Nicola Ferraro 11 | */ 12 | public class ExpressionTokenizerTest { 13 | 14 | @Test 15 | public void testQuoted1() { 16 | String str = "01234\"678\"0"; 17 | ExpressionTokenizer t = new ExpressionTokenizer(str); 18 | 19 | assertFalse(t.isQuoted(str, 0)); 20 | assertFalse(t.isQuoted(str, 1)); 21 | assertFalse(t.isQuoted(str, 4)); 22 | 23 | assertTrue(t.isQuoted(str, 5)); 24 | assertTrue(t.isQuoted(str, 6)); 25 | assertTrue(t.isQuoted(str, 7)); 26 | assertTrue(t.isQuoted(str, 8)); 27 | assertTrue(t.isQuoted(str, 9)); 28 | 29 | assertFalse(t.isQuoted(str, 10)); 30 | } 31 | 32 | @Test 33 | public void testQuoted2() { 34 | String str = "01234\"678\"0"; 35 | ExpressionTokenizer t = new ExpressionTokenizer(str); 36 | 37 | assertFalse(t.isQuoted(str, 11)); 38 | assertFalse(t.isQuoted(str, 15)); 39 | } 40 | 41 | @Test 42 | public void testQuoted3() { 43 | String str = "01234\"67890"; 44 | ExpressionTokenizer t = new ExpressionTokenizer(str); 45 | 46 | assertFalse(t.isQuoted(str, 11)); 47 | assertFalse(t.isQuoted(str, 15)); 48 | } 49 | 50 | @Test 51 | public void testQuoted4() { 52 | String str = "01234\"678\"01\""; 53 | ExpressionTokenizer t = new ExpressionTokenizer(str); 54 | 55 | assertTrue(t.isQuoted(str, 6)); 56 | assertTrue(t.isQuoted(str, 7)); 57 | assertTrue(t.isQuoted(str, 8)); 58 | assertTrue(t.isQuoted(str, 9)); 59 | 60 | assertFalse(t.isQuoted(str, 10)); 61 | assertFalse(t.isQuoted(str, 11)); 62 | assertFalse(t.isQuoted(str, 12)); 63 | assertFalse(t.isQuoted(str, 13)); 64 | assertFalse(t.isQuoted(str, 14)); 65 | } 66 | 67 | @Test 68 | public void testContainsUnquoted1() { 69 | String str = "ls -ltr >> file1.txt"; 70 | ExpressionTokenizer t = new ExpressionTokenizer(str); 71 | 72 | assertEquals(8, t.indexOfUnquoted(str, ">>")); 73 | } 74 | 75 | @Test 76 | public void testContainsUnquoted2() { 77 | String str = "ls -ltr \">>\" file1.txt"; 78 | ExpressionTokenizer t = new ExpressionTokenizer(str); 79 | 80 | assertEquals(-1, t.indexOfUnquoted(str, ">>")); 81 | } 82 | 83 | @Test 84 | public void testContainsUnquoted3() { 85 | String str = "ls \"-ltr >>\" file1.txt"; 86 | ExpressionTokenizer t = new ExpressionTokenizer(str); 87 | 88 | assertEquals(-1, t.indexOfUnquoted(str, ">>")); 89 | } 90 | 91 | @Test 92 | public void testContainsUnquoted4() { 93 | String str = "ls \"-ltr >>\" > file1.txt"; 94 | ExpressionTokenizer t = new ExpressionTokenizer(str); 95 | 96 | assertEquals(13, t.indexOfUnquoted(str, ">")); 97 | } 98 | 99 | @Test 100 | public void testContainsUnquoted5() { 101 | String str = "ls \"-ltr >>\" > > file1.txt"; 102 | ExpressionTokenizer t = new ExpressionTokenizer(str); 103 | 104 | assertEquals(13, t.indexOfUnquoted(str, ">")); 105 | assertEquals(15, t.indexOfUnquoted(str, ">", 14)); 106 | } 107 | 108 | @Test 109 | public void testContainsUnquoted6() { 110 | String str = "ls \"-ltr >\">\" > >> file1.txt"; 111 | ExpressionTokenizer t = new ExpressionTokenizer(str); 112 | 113 | assertEquals(11, t.indexOfUnquoted(str, ">")); 114 | assertEquals(14, t.indexOfUnquoted(str, ">", 12)); 115 | assertEquals(16, t.indexOfUnquoted(str, ">", 15)); 116 | assertEquals(17, t.indexOfUnquoted(str, ">", 17)); 117 | } 118 | 119 | @Test 120 | public void testSplitUnquoted1() { 121 | String str = "ls -ltr | grep -v hello"; 122 | ExpressionTokenizer t = new ExpressionTokenizer(str); 123 | 124 | assertEquals(Arrays.asList("ls -ltr ", " grep -v hello"), t.splitUnquoted(str, "|")); 125 | } 126 | 127 | @Test 128 | public void testSplitUnquoted2() { 129 | String str = "ls -ltr \"|\" | grep -v hello"; 130 | ExpressionTokenizer t = new ExpressionTokenizer(str); 131 | 132 | assertEquals(Arrays.asList("ls -ltr \"|\" ", " grep -v hello"), t.splitUnquoted(str, "|")); 133 | } 134 | 135 | @Test 136 | public void testTokenizer1() { 137 | String str = "ls -ltr \"|\" | grep -v hello"; 138 | ExpressionTokenizer t = new ExpressionTokenizer(str); 139 | 140 | assertEquals(Arrays.asList("ls -ltr \"|\"", "grep -v hello"), t.getCommandStrings()); 141 | } 142 | 143 | @Test 144 | public void testTokenizer2() { 145 | String str = "ls -ltr \"|\" \">>>\" > 2 | grep -v hello"; 146 | ExpressionTokenizer t = new ExpressionTokenizer(str); 147 | 148 | assertEquals(Arrays.asList("ls -ltr \"|\" \">>>\"", "> 2", "grep -v hello"), t.getCommandStrings()); 149 | } 150 | 151 | @Test 152 | public void testTokenizer3() { 153 | String str = "ls -ltr \"|\" \">>>\" >> 2 | grep -v hello | > asd"; 154 | ExpressionTokenizer t = new ExpressionTokenizer(str); 155 | 156 | assertEquals(Arrays.asList("ls -ltr \"|\" \">>>\"", ">> 2", "grep -v hello", "> asd"), t.getCommandStrings()); 157 | } 158 | 159 | @Test 160 | public void testTokenizer4() { 161 | String str = "echo hello > file"; 162 | ExpressionTokenizer t = new ExpressionTokenizer(str); 163 | 164 | assertEquals(Arrays.asList("echo hello", "> file"), t.getCommandStrings()); 165 | } 166 | 167 | @Test 168 | public void testTokenizer5() { 169 | String str = " > file"; 170 | ExpressionTokenizer t = new ExpressionTokenizer(str); 171 | 172 | assertEquals(Arrays.asList("> file"), t.getCommandStrings()); 173 | } 174 | 175 | @Test 176 | public void testTokenizer6() { 177 | String str = ""; 178 | ExpressionTokenizer t = new ExpressionTokenizer(str); 179 | 180 | assertEquals(Arrays.asList(""), t.getCommandStrings()); 181 | } 182 | 183 | } 184 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ![Spash](https://raw.githubusercontent.com/nerdammer/spash/master/resources/spash-logo.png?) 2 | 3 | Spash is a command line tool for Big Data platforms that simulates a real Unix environment, providing most of the commands of a typical bash shell on top of **YARN, HDFS and Apache Spark**. 4 | 5 | Spash uses the HDFS APIs to execute simple file operations and Apache Spark to perform parallel computations on big datasets. 6 | With Spash, managing a Big Data cluster becomes *as natural as writing bash commands*. 7 | 8 | Spash is still a *proof of concept* and **needs contributions** before going in production. 9 | 10 | ## Architecture 11 | 12 | The Spash daemon runs on an edge node of a Big Data cluster and listens for incoming *SSH* connections on port `2222`. 13 | Clients can connect using the OS native terminal application, or [Putty](http://www.putty.org/) on Windows. 14 | 15 | The Spash daemon will emulate a Unix OS and leverage the power of Spark to perform efficient computations on distributed data. 16 | 17 | ![Spash Architecture](https://raw.githubusercontent.com/nerdammer/spash/master/resources/architecture.png?) 18 | 19 | ## The world before Spash 20 | For those who don't remember the classic way of doing simple operations on HDFS, here's a reminder: 21 | 22 | ```console 23 | bash$ hdfs dfs -ls / 24 | # 25 | ## 26 | ### [wait for the JVM to load and execute] 27 | ... 28 | ########################################## 29 | bash$ hdfs dfs -copyFromLocal myFile / 30 | # 31 | ## 32 | ### [wait for the JVM to load and execute] 33 | ... 34 | ########################################## 35 | bash$ 36 | ``` 37 | 38 | ## Spash makes it easier 39 | Just run the Spash daemon on a node of your Big Data cluster, you can connect to it using `ssh user@hdfshost -p 2222` (the password is `user`) and then run all your favourite bash commands to manipulate data. 40 | 41 | ```console 42 | user@spash:/# echo "Maybe I can provide a real life example..." 43 | Maybe I can provide a real life example... 44 | 45 | user@spash:/# echo "My content" > myFile 46 | 47 | user@spash:/# ls 48 | myFile 49 | 50 | user@spash:/# cat myFile 51 | My content 52 | 53 | user@spash:/# echo text2 > myFile2 54 | 55 | user@spash:/# ls -l 56 | drwxr-xr-x 4 user supergroup 0 30 mar 18:34 myFile 57 | drwxr-xr-x 4 user supergroup 0 30 mar 18:35 myFile2 58 | 59 | user@spash:/# cat myFile myFile2 > myFile3 60 | 61 | user@spash:/# cat myFile3 62 | My content 63 | text2 64 | 65 | user@spash:/# cat myFile3 | grep -v 2 66 | My content 67 | 68 | user@spash:/# exit 69 | ``` 70 | 71 | And this is just the tip of the iceberg. From your host, you can use **scp** (`scp myfile -P 2222 user@hdfshost:/`) to copy a file into hdfs. You can even use **FileZilla or WinSCP to browse HDFS**. 72 | 73 | With some code contributions, you will be able to **transfer files using these tools** in the future (now you can just browse). 74 | 75 | ## Contributing to the Project 76 | Spash is an open source project and needs contributors. 77 | **If you like the idea, fork it and start playing**. 78 | 79 | Setting up the development environment is as easy as executing the following steps. 80 | 81 | What you need: 82 | - **You preferred IDE**: to build the software, that is 100% Java (IntelliJ or Eclipse are the preferred IDEs); 83 | - **Docker**: to run a HDFS container; 84 | - **SSH Client**: if you have Linux or Mac you already have it. If you use Windows you need (eg.) Putty to connect. 85 | 86 | Fork the project then clone it into your machine to get the source code. Configure it as a Maven project in your IDE. Contribute using *pull requests*. 87 | 88 | Start a HDFS *Docker* container using the following script: 89 | 90 | ```console 91 | docker run -d -h hdfshost --name dfs -p 8020:8020 -p 50070:50070 -p 50010:50010 -p 50020:50020 -p 50075:50075 dockmob/hadoop -t pseudodistributed 92 | ``` 93 | 94 | The script above creates a container named `dfs` and binds all the necessary ports to the docker machine. In order to connect to the HDFS container, your docker machine should have hostname `hdfshost` according to your host machine. 95 | It means: 96 | 97 | **On Windows or OS X**: 98 | Your docker machine usually binds to the address `192.168.99.100`. 99 | Make sure that it is bound there by running `docker-machine ip default` (*"default"* is the standard name of the docker machine). 100 | You need to append the row `192.168.99.100 hdfshost` to the `/etc/hosts` file (`C:\Windows\System32\Drivers\etc\hosts` in Windows) 101 | 102 | **On Linux**: 103 | Your docker machine is **your local host**. 104 | Add `hdfshost` at the end of the row containg your loopback address (eg. `127.0.0.1 localhost hdfshost`). 105 | 106 | The operations above should be executed only the first time you create the container. Once the container is created, you can use: 107 | - `docker stop dfs` to stop the container; 108 | - `docker start dfs` to start it again. 109 | 110 | You can make any change to the source code, then: 111 | - Start the `dfs` container; 112 | - Compile the `core` project and run the `it.nerdammer.spash.shell.Spash` main class; 113 | - Connect using your ssh client to `localhost` on port `2222`. Eg. `ssh user@localhost -p 2222`, using password `user` when prompted; 114 | - Enjoy. 115 | 116 | ## Installation 117 | Binary packages are available for running Spash in a real cluster environment (eg. Cloudera CDH 5). 118 | Look at the [releases section](https://github.com/nerdammer/spash/releases) to download the latest version. 119 | 120 | Packages (*tar.gz*) need only to be extracted on a Spark client node (eg. the Master Node of the cluster). 121 | 122 | You need to configure the `spash.properties` file to set up some common properties, for instance, the address of your HDFS cluster. 123 | 124 | The software can be started using the `spash.sh` command in the `bin` directory. 125 | 126 | Spash binds by default to the `2222` port on the machine where it is launched (you can change the port from `spash.properties`). 127 | 128 | ## Supported Commands 129 | Spash currently supports the following list of commands to manipulate data: 130 | - cat 131 | - cd 132 | - echo 133 | - exit 134 | - grep 135 | - head 136 | - ls 137 | - mkdir 138 | - pwd 139 | - rm 140 | - rmdir 141 | - write (>) 142 | 143 | ## Supported Environments 144 | Spash has been tested on the following environments: 145 | - Cloudera CDH 5 146 | - Dockmob Docker Containers 147 | 148 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/SpashShell.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell; 2 | 3 | import it.nerdammer.spash.shell.command.*; 4 | import jline.console.ConsoleReader; 5 | import jline.console.UserInterruptException; 6 | import org.apache.commons.lang3.exception.ExceptionUtils; 7 | import org.apache.sshd.server.Environment; 8 | import org.apache.sshd.server.ExitCallback; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.io.*; 13 | import java.util.LinkedList; 14 | 15 | /** 16 | * A java shell that emulates bash. 17 | * 18 | * @author Nicola Ferraro 19 | */ 20 | public class SpashShell implements org.apache.sshd.server.Command, Runnable { 21 | 22 | private static final Logger log = LoggerFactory.getLogger(SpashShell.class); 23 | 24 | private InputStream in; 25 | private OutputStream out; 26 | private OutputStream err; 27 | private ExitCallback callback; 28 | private Environment environment; 29 | private Thread thread; 30 | private ConsoleReader reader; 31 | private PrintWriter writer; 32 | 33 | private SpashSession session; 34 | 35 | public InputStream getIn() { 36 | return in; 37 | } 38 | 39 | public OutputStream getOut() { 40 | return out; 41 | } 42 | 43 | public OutputStream getErr() { 44 | return err; 45 | } 46 | 47 | public Environment getEnvironment() { 48 | return environment; 49 | } 50 | 51 | public void setInputStream(InputStream in) { 52 | this.in = in; 53 | } 54 | 55 | public void setOutputStream(OutputStream out) { 56 | this.out = out; 57 | } 58 | 59 | public void setErrorStream(OutputStream err) { 60 | this.err = err; 61 | } 62 | 63 | public void setExitCallback(ExitCallback callback) { 64 | this.callback = callback; 65 | } 66 | 67 | public void start(Environment env) throws IOException { 68 | this.environment = env; 69 | this.session = new SpashSession(environment.getEnv().get(Environment.ENV_USER)); 70 | 71 | this.thread = new Thread(this); 72 | this.thread.start(); 73 | } 74 | 75 | public void destroy() { 76 | if (reader != null) 77 | reader.shutdown(); 78 | thread.interrupt(); 79 | } 80 | 81 | @Override 82 | public void run() { 83 | try { 84 | 85 | reader = new ConsoleReader(in, new FilterOutputStream(out) { 86 | @Override 87 | public void write(final int i) throws IOException { 88 | super.write(i); 89 | 90 | // Workaround for all OS. Reset line after CR.. 91 | if (i == ConsoleReader.CR.toCharArray()[0]) { 92 | super.write(ConsoleReader.RESET_LINE); 93 | } 94 | } 95 | }); 96 | 97 | // enable ctrl+c 98 | reader.setHandleUserInterrupt(true); 99 | 100 | // Set the prompt 101 | reader.setPrompt(getShellPrompt()); 102 | 103 | // Set the completer 104 | reader.addCompleter(new SpashCommandCompleter(this.session)); 105 | 106 | writer = new PrintWriter(reader.getOutput()); 107 | 108 | // output welcome banner on ssh session startup 109 | writer.println("************************************************"); 110 | writer.println("* Welcome to Spash *"); 111 | writer.println("************************************************"); 112 | writer.println(); 113 | writer.flush(); 114 | 115 | String line; 116 | do { 117 | try { 118 | line = reader.readLine(); 119 | } 120 | catch(UserInterruptException e) { 121 | log.debug("ctrl+c"); 122 | line = ""; 123 | continue; 124 | } 125 | 126 | if(line!=null) { 127 | try { 128 | handleUserInput(line.trim()); 129 | } catch (InterruptedIOException | SpashExitException e) { 130 | // Ignore 131 | break; 132 | } catch (Throwable t) { 133 | writer.println("-spash: Unexpected error while executing the command."); 134 | writer.println(ExceptionUtils.getStackTrace(t)); 135 | writer.flush(); 136 | log.error("Error executing the command...", t); 137 | } 138 | 139 | // update the prompt 140 | reader.setPrompt(getShellPrompt()); 141 | } 142 | 143 | } while(line!=null); 144 | 145 | 146 | } catch (Throwable t) { 147 | writer.println("-spash: Unexpected error. The shell will be closed."); 148 | writer.println(ExceptionUtils.getStackTrace(t)); 149 | writer.flush(); 150 | log.error("Error executing InAppShell...", t); 151 | } finally { 152 | callback.onExit(0); 153 | } 154 | } 155 | 156 | 157 | private void handleUserInput(String exprStr) throws InterruptedIOException { 158 | 159 | try { 160 | ExpressionTokenizer tok = new ExpressionTokenizer(exprStr); 161 | 162 | ExecutionContext ctx = new ExecutionContext(this.session, null); 163 | 164 | LinkedList commands = new LinkedList<>(tok.getCommandStrings()); 165 | 166 | while(!commands.isEmpty()) { 167 | String commandStr = commands.poll(); 168 | 169 | Command command = CommandFactory.getInstance().getCommand(commandStr); 170 | 171 | CommandResult result = command.execute(ctx); 172 | 173 | if(result.isSuccess() && !commands.isEmpty()) { 174 | ctx.setPreviousCommandResult(result); 175 | continue; 176 | } 177 | 178 | if (result.isSuccess()) { 179 | if (result.getContent() != null) { 180 | result.getContent().mkString(writer); 181 | } 182 | } else { 183 | writer.print("-spash: "); 184 | String[] parts = commandStr.split(" "); 185 | for (String p : parts) { 186 | writer.print(p + ": "); 187 | } 188 | writer.println(result.getErrorMessage()); 189 | } 190 | } 191 | } catch(IllegalArgumentException e) { 192 | writer.print("-spash: "); 193 | writer.println(e.getMessage()); 194 | } 195 | 196 | writer.flush(); 197 | } 198 | 199 | private String getShellPrompt() { 200 | return this.session.getUser() + "@spash:" + this.session.getWorkingDir() + "# "; 201 | } 202 | 203 | } 204 | -------------------------------------------------------------------------------- /core/src/test/java/it/nerdammer/spash/shell/command/CommandTokenizerTest.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.command; 2 | 3 | import com.google.common.collect.ImmutableMap; 4 | import org.junit.Test; 5 | 6 | import java.util.*; 7 | 8 | import static org.junit.Assert.*; 9 | 10 | /** 11 | * @author Nicola Ferraro 12 | */ 13 | public class CommandTokenizerTest { 14 | 15 | @Test 16 | public void testNormalString() { 17 | CommandTokenizer t = new CommandTokenizer("ls -l"); 18 | 19 | assertEquals("ls", t.getCommand()); 20 | assertEquals(asMap("l", null), t.getOptions()); 21 | assertEquals(Collections.emptyList(), t.getArguments()); 22 | } 23 | 24 | @Test 25 | public void testQuotedString() { 26 | CommandTokenizer t = new CommandTokenizer("ls -l \"a\""); 27 | 28 | assertEquals("ls", t.getCommand()); 29 | assertEquals(asMap("l", null), t.getOptions()); 30 | assertEquals(Arrays.asList("a"), t.getArguments()); 31 | } 32 | 33 | @Test 34 | public void testQuotedEmptyString() { 35 | CommandTokenizer t = new CommandTokenizer("ls -l \"\" a"); 36 | 37 | assertEquals("ls", t.getCommand()); 38 | assertEquals(asMap("l", null), t.getOptions()); 39 | assertEquals(Arrays.asList("", "a"), t.getArguments()); 40 | } 41 | 42 | @Test 43 | public void testQuoteOnFirstSegment() { 44 | CommandTokenizer t = new CommandTokenizer("\"ls\" -l\" a"); 45 | 46 | assertEquals("\"ls\"", t.getCommand()); 47 | assertEquals(asMap("l", null, 48 | "\"", null), t.getOptions()); 49 | assertEquals(Arrays.asList("a"), t.getArguments()); 50 | } 51 | 52 | @Test 53 | public void testSpaces() { 54 | CommandTokenizer t = new CommandTokenizer("ls a"); 55 | 56 | assertEquals("ls", t.getCommand()); 57 | assertEquals(Collections.emptyMap(), t.getOptions()); 58 | assertEquals(Arrays.asList("a"), t.getArguments()); 59 | } 60 | 61 | @Test 62 | public void testSpacesInsideQuotes() { 63 | CommandTokenizer t = new CommandTokenizer("ls a \" \" \" b \""); 64 | 65 | assertEquals("ls", t.getCommand()); 66 | assertEquals(Collections.emptyMap(), t.getOptions()); 67 | assertEquals(Arrays.asList("a", " ", " b "), t.getArguments()); 68 | } 69 | 70 | @Test 71 | public void testSpecialSpaces() { 72 | CommandTokenizer t = new CommandTokenizer("cat \r \n \t a bbb \n"); 73 | 74 | assertEquals("cat", t.getCommand()); 75 | assertEquals(Collections.emptyMap(), t.getOptions()); 76 | assertEquals(Arrays.asList("a", "bbb"), t.getArguments()); 77 | } 78 | 79 | @Test 80 | public void testParameterlessValue() { 81 | CommandTokenizer t = new CommandTokenizer("cat -t hello world"); 82 | 83 | assertEquals("cat", t.getCommand()); 84 | assertEquals(asMap("t", null), t.getOptions()); 85 | assertEquals(Arrays.asList("hello", "world"), t.getArguments()); 86 | } 87 | 88 | @Test 89 | public void testParameterlessMultiparamValue() { 90 | CommandTokenizer t = new CommandTokenizer("cat -tr hello world"); 91 | 92 | assertEquals("cat", t.getCommand()); 93 | assertEquals(asMap("t", null, 94 | "r", null), t.getOptions()); 95 | assertEquals(Arrays.asList("hello", "world"), t.getArguments()); 96 | } 97 | 98 | @Test 99 | public void testParameterValue() { 100 | CommandTokenizer t = new CommandTokenizer("cat -t hello world", Collections.singleton("t")); 101 | 102 | assertEquals("cat", t.getCommand()); 103 | assertEquals(asMap("t", "hello"), t.getOptions()); 104 | assertEquals(Arrays.asList("world"), t.getArguments()); 105 | } 106 | 107 | @Test 108 | public void testMultiParameterValue() { 109 | CommandTokenizer t = new CommandTokenizer("cat -t hello -r world yes", new TreeSet<>(Arrays.asList("r", "t"))); 110 | 111 | assertEquals("cat", t.getCommand()); 112 | assertEquals(asMap("t", "hello", 113 | "r", "world"), t.getOptions()); 114 | assertEquals(Arrays.asList("yes"), t.getArguments()); 115 | } 116 | 117 | @Test 118 | public void testTextParameter() { 119 | CommandTokenizer t = new CommandTokenizer("cat --trello world yes"); 120 | 121 | assertEquals("cat", t.getCommand()); 122 | assertEquals(asMap("trello", null), t.getOptions()); 123 | assertEquals(Arrays.asList("world", "yes"), t.getArguments()); 124 | } 125 | 126 | @Test 127 | public void testWrongOptionAsParameter() { 128 | CommandTokenizer t = new CommandTokenizer("cat --trello world yes -p ciao"); 129 | 130 | assertEquals("cat", t.getCommand()); 131 | assertEquals(asMap("trello", null), t.getOptions()); 132 | assertEquals(Arrays.asList("world", "yes", "-p", "ciao"), t.getArguments()); 133 | } 134 | 135 | @Test 136 | public void testTextParameterValue() { 137 | CommandTokenizer t = new CommandTokenizer("cat --trello world yes", new TreeSet<>(Arrays.asList("trello"))); 138 | 139 | assertEquals("cat", t.getCommand()); 140 | assertEquals(asMap("trello", "world"), t.getOptions()); 141 | assertEquals(Arrays.asList("yes"), t.getArguments()); 142 | } 143 | 144 | @Test(expected = IllegalArgumentException.class) 145 | public void testInvalidCommand() { 146 | new CommandTokenizer("cat --trello", new TreeSet<>(Arrays.asList("trello"))); 147 | } 148 | 149 | @Test(expected = IllegalArgumentException.class) 150 | public void testInvalidSingleCommand() { 151 | new CommandTokenizer("cat -t", new TreeSet<>(Arrays.asList("t"))); 152 | } 153 | 154 | @Test(expected = IllegalArgumentException.class) 155 | public void testInvalidSingleCommandRequiringValue() { 156 | new CommandTokenizer("cat -te pi", new TreeSet<>(Arrays.asList("t"))); 157 | } 158 | 159 | @Test(expected = IllegalArgumentException.class) 160 | public void testSingleParamDefinedTwice() { 161 | new CommandTokenizer("cat -te -t"); 162 | } 163 | 164 | @Test(expected = IllegalArgumentException.class) 165 | public void testSingleParamDefinedTwice2() { 166 | new CommandTokenizer("cat -t -t"); 167 | } 168 | 169 | @Test(expected = IllegalArgumentException.class) 170 | public void testSingleParamDefinedTwice3() { 171 | new CommandTokenizer("cat -tt"); 172 | } 173 | 174 | @Test(expected = IllegalArgumentException.class) 175 | public void testSingleParamDefinedTwice4() { 176 | new CommandTokenizer("cat -t -e -t"); 177 | } 178 | 179 | @Test(expected = IllegalArgumentException.class) 180 | public void testTextParamDefinedTwice() { 181 | new CommandTokenizer("cat --te --te"); 182 | } 183 | 184 | @Test(expected = IllegalArgumentException.class) 185 | public void testTextParamDefinedTwice2() { 186 | new CommandTokenizer("cat --t --t"); 187 | } 188 | 189 | @Test(expected = IllegalArgumentException.class) 190 | public void testTextParamDefinedTwice3() { 191 | new CommandTokenizer("cat --help --help"); 192 | } 193 | 194 | private Map asMap(String... values) { 195 | Map map = new TreeMap<>(); 196 | for(int i=0; i+1<=values.length; i+=2) { 197 | map.put(values[i], values[i+1]); 198 | } 199 | return map; 200 | } 201 | 202 | } 203 | -------------------------------------------------------------------------------- /core/src/main/java/it/nerdammer/spash/shell/api/fs/FileSystemFacadeImpl.java: -------------------------------------------------------------------------------- 1 | package it.nerdammer.spash.shell.api.fs; 2 | 3 | import com.google.common.collect.Lists; 4 | import it.nerdammer.spash.shell.common.SpashCollection; 5 | import it.nerdammer.spash.shell.common.SpashCollectionListAdapter; 6 | 7 | import java.io.IOException; 8 | import java.net.URI; 9 | import java.nio.file.*; 10 | import java.nio.file.attribute.PosixFileAttributes; 11 | import java.util.Collections; 12 | import java.util.Comparator; 13 | import java.util.List; 14 | 15 | /** 16 | * A file system facade to access HDFS. 17 | * 18 | * @author Nicola Ferraro 19 | */ 20 | public class FileSystemFacadeImpl implements FileSystemFacade { 21 | 22 | private String host; 23 | 24 | private int port; 25 | 26 | public FileSystemFacadeImpl(String host, int port) { 27 | this.host = host; 28 | this.port = port; 29 | } 30 | 31 | @Override 32 | public FileSystem getFileSystem() { 33 | return FileSystems.getFileSystem(getURI("/")); 34 | } 35 | 36 | @Override 37 | public SpashCollection ls(String path) { 38 | if(path==null || !path.startsWith("/")) { 39 | throw new IllegalArgumentException("Paths must be absolute. Path=" + path); 40 | } 41 | 42 | try { 43 | URI uri = getURI(path); 44 | Path dir = Paths.get(uri); 45 | 46 | List children = Lists.newArrayList(Files.newDirectoryStream(dir, new DirectoryStream.Filter() { 47 | @Override 48 | public boolean accept(Path entry) throws IOException { 49 | return true; 50 | } 51 | })); 52 | 53 | return new SpashCollectionListAdapter<>(children); 54 | } catch(RuntimeException e) { 55 | throw e; 56 | } catch(Exception e) { 57 | throw new RuntimeException(e); 58 | } 59 | } 60 | 61 | @Override 62 | public Path getAbsolutePath(String base, String path) { 63 | if(path==null || base==null) { 64 | throw new IllegalArgumentException("Both arguments must be provided. Base=" + base + ", Path=" + path); 65 | } 66 | if(!base.startsWith("/")) { 67 | throw new IllegalArgumentException("Base path must be absolute. Base=" + base); 68 | } 69 | 70 | try { 71 | if (path.startsWith("/")) { 72 | URI uri = getURI(path); 73 | Path p = Paths.get(uri); 74 | return p; 75 | } 76 | 77 | Path p = Paths.get(base, path); 78 | return p; 79 | 80 | } catch(RuntimeException e) { 81 | throw e; 82 | } catch(Exception e) { 83 | throw new RuntimeException(e); 84 | } 85 | } 86 | 87 | @Override 88 | public boolean exists(String path) { 89 | if(path==null || !path.startsWith("/")) { 90 | throw new IllegalArgumentException("Paths must be absolute. Path=" + path); 91 | } 92 | 93 | try { 94 | URI uri = getURI(path); 95 | Path p = Paths.get(uri); 96 | 97 | return Files.exists(p); 98 | 99 | } catch(RuntimeException e) { 100 | throw e; 101 | } catch(Exception e) { 102 | throw new RuntimeException(e); 103 | } 104 | } 105 | 106 | @Override 107 | public boolean isDirectory(String path) { 108 | if(path==null || !path.startsWith("/")) { 109 | throw new IllegalArgumentException("Paths must be absolute. Path=" + path); 110 | } 111 | 112 | try { 113 | URI uri = getURI(path); 114 | Path p = Paths.get(uri); 115 | 116 | return Files.isDirectory(p); 117 | 118 | } catch(RuntimeException e) { 119 | throw e; 120 | } catch(Exception e) { 121 | throw new RuntimeException(e); 122 | } 123 | } 124 | 125 | @Override 126 | public URI getURI(String path) { 127 | if(path==null || !path.startsWith("/")) { 128 | throw new IllegalArgumentException("Paths must be absolute. Path=" + path); 129 | } 130 | try { 131 | return new URI("hdfs://" + host + ":" + port + path); 132 | } catch(RuntimeException e) { 133 | throw e; 134 | } catch(Exception e) { 135 | throw new RuntimeException(e); 136 | } 137 | } 138 | 139 | @Override 140 | public boolean mkdir(String path) { 141 | if(path==null || !path.startsWith("/")) { 142 | throw new IllegalArgumentException("Paths must be absolute. Path=" + path); 143 | } 144 | 145 | try { 146 | URI uri = getURI(path); 147 | Path p = Paths.get(uri); 148 | 149 | Files.createDirectory(p); 150 | return true; 151 | 152 | } catch(FileAlreadyExistsException e) { 153 | return false; 154 | } catch(RuntimeException e) { 155 | throw e; 156 | } catch(Exception e) { 157 | throw new RuntimeException(e); 158 | } 159 | } 160 | 161 | @Override 162 | public boolean rm(String path, boolean recursive) { 163 | if(path==null || !path.startsWith("/")) { 164 | throw new IllegalArgumentException("Paths must be absolute. Path=" + path); 165 | } 166 | 167 | try { 168 | URI uri = getURI(path); 169 | Path target = Paths.get(uri); 170 | 171 | List children = Lists.newArrayList(Files.newDirectoryStream(target, new DirectoryStream.Filter() { 172 | @Override 173 | public boolean accept(Path entry) throws IOException { 174 | return true; 175 | } 176 | })); 177 | 178 | if(!recursive) { 179 | long contained = children.size(); 180 | if (contained > 0) { 181 | return false; 182 | } 183 | } else { 184 | Collections.sort(children, new Comparator() { 185 | @Override 186 | public int compare(Path p1, Path p2) { 187 | return p2.normalize().toString().length() - p1.normalize().toString().length(); 188 | } 189 | }); 190 | 191 | for (Path p : children) { 192 | try { 193 | Files.delete(p); 194 | } catch (RuntimeException e) { 195 | throw e; 196 | } catch (Exception e) { 197 | throw new RuntimeException(e); 198 | } 199 | } 200 | } 201 | 202 | Files.delete(target); 203 | 204 | return true; 205 | 206 | } catch(RuntimeException e) { 207 | throw e; 208 | } catch(Exception e) { 209 | throw new RuntimeException(e); 210 | } 211 | } 212 | 213 | @Override 214 | public PosixFileAttributes getAttributes(String path) { 215 | if(path==null || !path.startsWith("/")) { 216 | throw new IllegalArgumentException("Paths must be absolute. Path=" + path); 217 | } 218 | 219 | try { 220 | URI uri = getURI(path); 221 | Path target = Paths.get(uri); 222 | 223 | PosixFileAttributes attrs = Files.readAttributes(target, PosixFileAttributes.class, LinkOption.NOFOLLOW_LINKS); 224 | return attrs; 225 | 226 | } catch(RuntimeException e) { 227 | throw e; 228 | } catch(Exception e) { 229 | throw new RuntimeException(e); 230 | } 231 | } 232 | } 233 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /release/src/assembly/assembly-1.1.3.xsd: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 1.0.0+ 10 | 11 | 12 | An assembly defines a collection of files usually distributed in an 13 | archive format such as zip, tar, or tar.gz that is generated from a 14 | project. For example, a project could produce a ZIP assembly which 15 | contains a project's JAR artifact in the root directory, the 16 | runtime dependencies in a lib/ directory, and a shell script to launch 17 | a stand-alone application. 18 | 19 | 20 | 21 | 22 | 23 | 24 | 1.0.0+ 25 | 26 | 27 | An assembly defines a collection of files usually distributed in an 28 | archive format such as zip, tar, or tar.gz that is generated from a 29 | project. For example, a project could produce a ZIP assembly which 30 | contains a project's JAR artifact in the root directory, the 31 | runtime dependencies in a lib/ directory, and a shell script to launch 32 | a stand-alone application. 33 | 34 | 35 | 36 | 37 | 38 | 39 | 1.0.0+ 40 | 41 | Sets the id of this assembly. This is a symbolic name for a 42 | particular assembly of files from this project. Also, aside from 43 | being used to distinctly name the assembled package by attaching 44 | its value to the generated archive, the id is used as your 45 | artifact's classifier when deploying. 46 | 47 | 48 | 49 | 50 | 51 | 1.0.0+ 52 | 53 | 54 | Specifies the formats of the assembly. 55 | 56 | It is often better to specify the formats via the goal parameter rather 57 | than here. For example, that allows different profiles to generate 58 | different types of archives. 59 | 60 | Multiple formats can be 61 | supplied and the Assembly Plugin will generate an archive for each 62 | of the desired formats. When deploying your project, all file formats 63 | specified will also be deployed. A format is specified by supplying 64 | one of the following values in a &lt;format&gt; subelement: 65 | <ul> 66 | <li><b>"zip"</b> - Creates a ZIP file format</li> 67 | <li><b>"tar"</b> - Creates a TAR format</li> 68 | <li><b>"tar.gz"</b> or <b>"tgz"</b> - Creates a gzip'd TAR format</li> 69 | <li><b>"tar.bz2"</b> or <b>"tbz2"</b> - Creates a bzip'd TAR format</li> 70 | <li><b>"jar"</b> - Creates a JAR format</li> 71 | <li><b>"dir"</b> - Creates an exploded directory format</li> 72 | <li><b>"war"</b> - Creates a WAR format</li> 73 | </ul> 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 0.0.0+ 86 | 87 | Includes a base directory in the final archive. For example, 88 | if you are creating an assembly named "your-app", setting 89 | includeBaseDirectory to true will create an archive that 90 | includes this base directory. If this option is set to false 91 | the archive created will unzip its content to the current 92 | directory. 93 | 94 | 95 | 96 | 97 | 98 | 1.1.0+ 99 | 100 | Sets the base directory of the resulting assembly archive. If this is not 101 | set and includeBaseDirectory == true, ${project.build.finalName} will be used instead. 102 | (Since 2.2-beta-1) 103 | 104 | 105 | 106 | 107 | 108 | 0.0.0+ 109 | 110 | Includes a site directory in the final archive. The site directory 111 | location of a project is determined by the siteDirectory parameter 112 | of the Assembly Plugin. 113 | 114 | 115 | 116 | 117 | 118 | 1.1.0+ 119 | 120 | 121 | Set of components which filter various container descriptors out of 122 | the normal archive stream, so they can be aggregated then added. 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 1.0.0+ 135 | 136 | 137 | Specifies which module files to include in the assembly. A moduleSet 138 | is specified by providing one or more of &lt;moduleSet&gt; 139 | subelements. 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 1.0.0+ 152 | 153 | 154 | Specifies which groups of files to include in the assembly. A 155 | fileSet is specified by providing one or more of &lt;fileSet&gt; 156 | subelements. 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 1.0.0+ 169 | 170 | 171 | Specifies which single files to include in the assembly. A file 172 | is specified by providing one or more of &lt;file&gt; 173 | subelements. 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 1.0.0+ 186 | 187 | 188 | Specifies which dependencies to include in the assembly. A 189 | dependencySet is specified by providing one or more of 190 | &lt;dependencySet&gt; subelements. 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 1.0.0+ 203 | 204 | 205 | Specifies which repository files to include in the assembly. A 206 | repository is specified by providing one or more of 207 | &lt;repository&gt; subelements. 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 1.0.0+ 220 | 221 | 222 | Specifies the shared components xml file locations to include in the 223 | assembly. The locations specified must be relative to the base location 224 | of the descriptor. If the descriptor was found via a &lt;descriptorRef/&gt; 225 | element in the 226 | classpath, any components it specifies will also be found on the classpath. 227 | If it is found by pathname via a &lt;descriptor/&gt; element 228 | the value here will be interpreted 229 | as a path relative to the project basedir. 230 | When multiple componentDescriptors are found, their 231 | contents are merged. Check out the <a href="component.html"> 232 | descriptor components</a> for more information. A 233 | componentDescriptor is specified by providing one or more of 234 | &lt;componentDescriptor&gt; subelements. 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 1.0.0+ 249 | 250 | A dependencySet allows inclusion and exclusion of project dependencies 251 | in the assembly. 252 | 253 | 254 | 255 | 256 | 257 | 1.0.0+ 258 | 259 | Sets the output directory relative to the root 260 | of the root directory of the assembly. For example, 261 | "log" will put the specified files in the log directory, 262 | directly beneath the root of the archive. 263 | 264 | 265 | 266 | 267 | 268 | 1.0.0+ 269 | 270 | 271 | When &lt;include&gt; subelements are present, they define a set of 272 | artifact coordinates to include. If none is present, then 273 | &lt;includes&gt; represents all valid values. 274 | 275 | Artifact coordinates may be given in simple groupId:artifactId form, 276 | or they may be fully qualified in the form groupId:artifactId:type[:classifier]:version. 277 | Additionally, wildcards can be used, as in *:maven-* 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 1.0.0+ 290 | 291 | 292 | When &lt;exclude&gt; subelements are present, they define a set of 293 | dependency artifact coordinates to exclude. If none is present, then 294 | &lt;excludes&gt; represents no exclusions. 295 | 296 | Artifact coordinates may be given in simple groupId:artifactId form, 297 | or they may be fully qualified in the form groupId:artifactId:type[:classifier]:version. 298 | Additionally, wildcards can be used, as in *:maven-* 299 | 300 | 301 | 302 | 303 | 304 | 305 | 306 | 307 | 308 | 309 | 310 | 1.0.0+ 311 | 312 | 313 | Similar to a UNIX permission, sets the file mode of the files included. 314 | THIS IS AN OCTAL VALUE. 315 | Format: (User)(Group)(Other) where each component is a sum of Read = 4, 316 | Write = 2, and Execute = 1. For example, the value 0644 317 | translates to User read-write, Group and Other read-only. The default value is 0644 318 | <a href="http://www.onlamp.com/pub/a/bsd/2000/09/06/FreeBSD_Basics.html">(more on unix-style permissions)</a> 319 | 320 | 321 | 322 | 323 | 324 | 325 | 1.0.0+ 326 | 327 | 328 | Similar to a UNIX permission, sets the directory mode of the directories 329 | included. 330 | THIS IS AN OCTAL VALUE. 331 | Format: (User)(Group)(Other) where each component is a sum of 332 | Read = 4, Write = 2, and Execute = 1. For example, the value 333 | 0755 translates to User read-write, Group and Other read-only. The default value is 0755. 334 | <a href="http://www.onlamp.com/pub/a/bsd/2000/09/06/FreeBSD_Basics.html">(more on unix-style permissions)</a> 335 | 336 | 337 | 338 | 339 | 340 | 341 | 1.1.2+ 342 | 343 | When specified as true, any include/exclude patterns which aren't used to filter an actual 344 | artifact during assembly creation will cause the build to fail with an error. This is meant 345 | to highlight obsolete inclusions or exclusions, or else signal that the assembly descriptor 346 | is incorrectly configured. (Since 2.2) 347 | 348 | 349 | 350 | 351 | 352 | 1.0.0+ 353 | 354 | Sets the mapping pattern for all dependencies included in this 355 | assembly. (Since 2.2-beta-2; 2.2-beta-1 uses ${artifactId}-${version}${dashClassifier?}.${extension} 356 | as default value). 357 | See the plugin FAQ for more details about entries usable in the outputFileNameMapping parameter. 358 | 359 | 360 | 361 | 362 | 363 | 1.0.0+ 364 | 365 | If set to true, this property will unpack all dependencies 366 | into the specified output directory. When set to false 367 | dependencies will be includes as archives (jars). Can only unpack 368 | jar, zip, tar.gz, and tar.bz archives. 369 | 370 | 371 | 372 | 373 | 374 | 1.1.0+ 375 | 376 | Allows the specification of includes and excludes, along with filtering options, for items 377 | unpacked from a dependency artifact. (Since 2.2-beta-1) 378 | 379 | 380 | 381 | 382 | 383 | 1.0.0+ 384 | 385 | Sets the dependency scope for this dependencySet. 386 | 387 | 388 | 389 | 390 | 391 | 1.1.0+ 392 | 393 | Determines whether the artifact produced during the current project's 394 | build should be included in this dependency set. (Since 2.2-beta-1) 395 | 396 | 397 | 398 | 399 | 400 | 1.1.0+ 401 | 402 | Determines whether the attached artifacts produced during the current project's 403 | build should be included in this dependency set. (Since 2.2-beta-1) 404 | 405 | 406 | 407 | 408 | 409 | 1.1.0+ 410 | 411 | Determines whether transitive dependencies will be included in the processing of 412 | the current dependency set. If true, includes/excludes/useTransitiveFiltering 413 | will apply to transitive dependency artifacts in addition to the main project 414 | dependency artifacts. If false, useTransitiveFiltering is meaningless, and 415 | includes/excludes only affect the immediate dependencies of the project. 416 | By default, this value is true. (Since 2.2-beta-1) 417 | 418 | 419 | 420 | 421 | 422 | 1.1.0+ 423 | 424 | Determines whether the include/exclude patterns in this dependency set will be applied to 425 | the transitive path of a given artifact. If true, and the current artifact is a transitive 426 | dependency brought in by another artifact which matches an inclusion or exclusion pattern, 427 | then the current artifact has the same inclusion/exclusion logic applied to it as well. By 428 | default, this value is false, in order to preserve backward compatibility with version 2.1. 429 | This means that includes/excludes only apply directly to the current artifact, and not to 430 | the transitive set of artifacts which brought it in. (Since 2.2-beta-1) 431 | 432 | 433 | 434 | 435 | 436 | 437 | 438 | 1.1.0+ 439 | 440 | Specifies options for including/excluding/filtering items extracted from an archive. (Since 2.2-beta-1) 441 | 442 | 443 | 444 | 445 | 446 | 1.1.0+ 447 | 448 | 449 | Set of file and/or directory patterns for matching items to be included from an archive as it is unpacked. 450 | Each item is specified as &lt;include&gt;some/path&lt;/include&gt; (Since 2.2-beta-1) 451 | 452 | 453 | 454 | 455 | 456 | 457 | 458 | 459 | 460 | 461 | 462 | 1.1.0+ 463 | 464 | 465 | Set of file and/or directory patterns for matching items to be excluded from an archive as it is unpacked. 466 | Each item is specified as &lt;exclude&gt;some/path&lt;/exclude&gt; (Since 2.2-beta-1) 467 | 468 | 469 | 470 | 471 | 472 | 473 | 474 | 475 | 476 | 477 | 478 | 1.1.0+ 479 | 480 | Whether to filter symbols in the files as they are unpacked from the archive, using 481 | properties from the build configuration. (Since 2.2-beta-1) 482 | 483 | 484 | 485 | 486 | 487 | 1.1.2+ 488 | 489 | 490 | Sets the line-endings of the files. (Since 2.2) 491 | Valid values: 492 | <ul> 493 | <li><b>"keep"</b> - Preserve all line endings</li> 494 | <li><b>"unix"</b> - Use Unix-style line endings</li> 495 | <li><b>"lf"</b> - Use a single line-feed line endings</li> 496 | <li><b>"dos"</b> - Use DOS-style line endings</li> 497 | <li><b>"crlf"</b> - Use Carraige-return, line-feed line endings</li> 498 | </ul> 499 | 500 | 501 | 502 | 503 | 504 | 505 | 1.1.2+ 506 | 507 | Whether standard exclusion patterns, such as those matching CVS and Subversion 508 | metadata files, should be used when calculating the files affected by this set. 509 | For backward compatibility, the default value is true. (Since 2.2) 510 | 511 | 512 | 513 | 514 | 515 | 1.1.3+ 516 | 517 | 518 | Allows to specify the encoding to use when unpacking archives, for unarchivers 519 | that support specifying encoding. If unspecified, archiver default will be used. 520 | Archiver defaults generally represent sane (modern) values. 521 | 522 | 523 | 524 | 525 | 526 | 527 | 528 | 529 | 1.0.0+ 530 | 531 | 532 | Defines a Maven repository to be included in the assembly. The artifacts 533 | available to be included in a repository are your project's dependency 534 | artifacts. The repository created contains the needed metadata entries 535 | and also contains both sha1 and md5 checksums. This is useful for creating 536 | archives which will be deployed to internal repositories. 537 | 538 | <br/><b>NOTE:</b> Currently, only artifacts from the central repository 539 | are allowed. 540 | 541 | 542 | 543 | 544 | 545 | 546 | 1.0.0+ 547 | 548 | Sets the output directory relative to the root 549 | of the root directory of the assembly. For example, 550 | "log" will put the specified files in the log directory, 551 | directly beneath the root of the archive. 552 | 553 | 554 | 555 | 556 | 557 | 1.0.0+ 558 | 559 | 560 | When &lt;include&gt; subelements are present, they define a set of 561 | artifact coordinates to include. If none is present, then 562 | &lt;includes&gt; represents all valid values. 563 | 564 | Artifact coordinates may be given in simple groupId:artifactId form, 565 | or they may be fully qualified in the form groupId:artifactId:type[:classifier]:version. 566 | Additionally, wildcards can be used, as in *:maven-* 567 | 568 | 569 | 570 | 571 | 572 | 573 | 574 | 575 | 576 | 577 | 578 | 1.0.0+ 579 | 580 | 581 | When &lt;exclude&gt; subelements are present, they define a set of 582 | dependency artifact coordinates to exclude. If none is present, then 583 | &lt;excludes&gt; represents no exclusions. 584 | 585 | Artifact coordinates may be given in simple groupId:artifactId form, 586 | or they may be fully qualified in the form groupId:artifactId:type[:classifier]:version. 587 | Additionally, wildcards can be used, as in *:maven-* 588 | 589 | 590 | 591 | 592 | 593 | 594 | 595 | 596 | 597 | 598 | 599 | 1.0.0+ 600 | 601 | 602 | Similar to a UNIX permission, sets the file mode of the files included. 603 | THIS IS AN OCTAL VALUE. 604 | Format: (User)(Group)(Other) where each component is a sum of Read = 4, 605 | Write = 2, and Execute = 1. For example, the value 0644 606 | translates to User read-write, Group and Other read-only. The default value is 0644 607 | <a href="http://www.onlamp.com/pub/a/bsd/2000/09/06/FreeBSD_Basics.html">(more on unix-style permissions)</a> 608 | 609 | 610 | 611 | 612 | 613 | 614 | 1.0.0+ 615 | 616 | 617 | Similar to a UNIX permission, sets the directory mode of the directories 618 | included. 619 | THIS IS AN OCTAL VALUE. 620 | Format: (User)(Group)(Other) where each component is a sum of 621 | Read = 4, Write = 2, and Execute = 1. For example, the value 622 | 0755 translates to User read-write, Group and Other read-only. The default value is 0755. 623 | <a href="http://www.onlamp.com/pub/a/bsd/2000/09/06/FreeBSD_Basics.html">(more on unix-style permissions)</a> 624 | 625 | 626 | 627 | 628 | 629 | 630 | 1.0.0+ 631 | 632 | If set to true, this property will trigger the creation of repository 633 | metadata which will allow the repository to be used as a functional remote 634 | repository. 635 | 636 | 637 | 638 | 639 | 640 | 1.0.0+ 641 | 642 | 643 | Specifies that you want to align a group of artifacts to a specified 644 | version. A groupVersionAlignment is specified by providing one or 645 | more of &lt;groupVersionAlignment&gt; subelements. 646 | 647 | 648 | 649 | 650 | 651 | 652 | 653 | 654 | 655 | 656 | 657 | 1.1.0+ 658 | 659 | Specifies the scope for artifacts included in this repository. (Since 2.2-beta-1) 660 | 661 | 662 | 663 | 664 | 665 | 666 | 667 | 1.0.0+ 668 | 669 | Allows a group of artifacts to be aligned to a specified version. 670 | 671 | 672 | 673 | 674 | 675 | 1.0.0+ 676 | 677 | The groupId of the artifacts for which you want to align the 678 | versions. 679 | 680 | 681 | 682 | 683 | 684 | 1.0.0+ 685 | 686 | The version you want to align this group to. 687 | 688 | 689 | 690 | 691 | 692 | 1.0.0+ 693 | 694 | 695 | When &lt;exclude&gt; subelements are present, they define the 696 | artifactIds of the artifacts to exclude. If none is present, then 697 | &lt;excludes&gt; represents no exclusions. An exclude is specified 698 | by providing one or more of &lt;exclude&gt; subelements. 699 | 700 | 701 | 702 | 703 | 704 | 705 | 706 | 707 | 708 | 709 | 710 | 711 | 712 | 1.1.0+ 713 | 714 | Configures a filter for files headed into the assembly archive, to enable 715 | aggregation of various types of descriptor fragments, such as components.xml, 716 | web.xml, etc. 717 | 718 | 719 | 720 | 721 | 722 | 1.1.0+ 723 | 724 | The handler's plexus role-hint, for lookup from the container. 725 | 726 | 727 | 728 | 729 | 730 | 1.1.1+ 731 | 732 | Configuration options for the handler. 733 | 734 | 735 | 736 | 737 | 738 | 739 | 740 | 741 | 742 | 743 | 744 | 745 | 1.0.0+ 746 | 747 | 748 | A moduleSet represent one or more project &lt;module&gt; present inside 749 | a project's pom.xml. This allows you to include sources or binaries 750 | belonging to a project's &lt;modules&gt;. 751 | 752 | <br/><b>NOTE:</b> When using &lt;moduleSets&gt; from the command-line, it 753 | is required to pass first the package phase by doing: "mvn package 754 | assembly:assembly". This bug/issue is scheduled to be addressed by Maven 2.1. 755 | 756 | 757 | 758 | 759 | 760 | 761 | 1.1.2+ 762 | 763 | If set to true, the plugin will include all projects in the current reactor for processing 764 | in this ModuleSet. These will be subject to include/exclude rules. (Since 2.2) 765 | 766 | 767 | 768 | 769 | 770 | 1.1.0+ 771 | 772 | If set to false, the plugin will exclude sub-modules from processing in this ModuleSet. 773 | Otherwise, it will process all sub-modules, each subject to include/exclude rules. (Since 2.2-beta-1) 774 | 775 | 776 | 777 | 778 | 779 | 1.0.0+ 780 | 781 | 782 | When &lt;include&gt; subelements are present, they define a set of 783 | project coordinates to include. If none is present, then 784 | &lt;includes&gt; represents all valid values. 785 | 786 | Artifact coordinates may be given in simple groupId:artifactId form, 787 | or they may be fully qualified in the form groupId:artifactId:type[:classifier]:version. 788 | Additionally, wildcards can be used, as in *:maven-* 789 | 790 | 791 | 792 | 793 | 794 | 795 | 796 | 797 | 798 | 799 | 800 | 1.0.0+ 801 | 802 | 803 | When &lt;exclude&gt; subelements are present, they define a set of 804 | project artifact coordinates to exclude. If none is present, then 805 | &lt;excludes&gt; represents no exclusions. 806 | 807 | Artifact coordinates may be given in simple groupId:artifactId form, 808 | or they may be fully qualified in the form groupId:artifactId:type[:classifier]:version. 809 | Additionally, wildcards can be used, as in *:maven-* 810 | 811 | 812 | 813 | 814 | 815 | 816 | 817 | 818 | 819 | 820 | 821 | 1.0.0+ 822 | 823 | When this is present, the plugin will include the source files of 824 | the included modules from this set in the resulting assembly. 825 | 826 | 827 | 828 | 829 | 830 | 1.0.0+ 831 | 832 | When this is present, the plugin will include the binaries of the 833 | included modules from this set in the resulting assembly. 834 | 835 | 836 | 837 | 838 | 839 | 840 | 841 | 1.0.0+ 842 | 843 | Contains configuration options for including the binary files of a 844 | project module in an assembly. 845 | 846 | 847 | 848 | 849 | 850 | 1.0.0+ 851 | 852 | Sets the output directory relative to the root 853 | of the root directory of the assembly. For example, 854 | "log" will put the specified files in the log directory, 855 | directly beneath the root of the archive. 856 | 857 | 858 | 859 | 860 | 861 | 1.0.0+ 862 | 863 | 864 | When &lt;include&gt; subelements are present, they define a set of 865 | artifact coordinates to include. If none is present, then 866 | &lt;includes&gt; represents all valid values. 867 | 868 | Artifact coordinates may be given in simple groupId:artifactId form, 869 | or they may be fully qualified in the form groupId:artifactId:type[:classifier]:version. 870 | Additionally, wildcards can be used, as in *:maven-* 871 | 872 | 873 | 874 | 875 | 876 | 877 | 878 | 879 | 880 | 881 | 882 | 1.0.0+ 883 | 884 | 885 | When &lt;exclude&gt; subelements are present, they define a set of 886 | dependency artifact coordinates to exclude. If none is present, then 887 | &lt;excludes&gt; represents no exclusions. 888 | 889 | Artifact coordinates may be given in simple groupId:artifactId form, 890 | or they may be fully qualified in the form groupId:artifactId:type[:classifier]:version. 891 | Additionally, wildcards can be used, as in *:maven-* 892 | 893 | 894 | 895 | 896 | 897 | 898 | 899 | 900 | 901 | 902 | 903 | 1.0.0+ 904 | 905 | 906 | Similar to a UNIX permission, sets the file mode of the files included. 907 | THIS IS AN OCTAL VALUE. 908 | Format: (User)(Group)(Other) where each component is a sum of Read = 4, 909 | Write = 2, and Execute = 1. For example, the value 0644 910 | translates to User read-write, Group and Other read-only. The default value is 0644 911 | <a href="http://www.onlamp.com/pub/a/bsd/2000/09/06/FreeBSD_Basics.html">(more on unix-style permissions)</a> 912 | 913 | 914 | 915 | 916 | 917 | 918 | 1.0.0+ 919 | 920 | 921 | Similar to a UNIX permission, sets the directory mode of the directories 922 | included. 923 | THIS IS AN OCTAL VALUE. 924 | Format: (User)(Group)(Other) where each component is a sum of 925 | Read = 4, Write = 2, and Execute = 1. For example, the value 926 | 0755 translates to User read-write, Group and Other read-only. The default value is 0755. 927 | <a href="http://www.onlamp.com/pub/a/bsd/2000/09/06/FreeBSD_Basics.html">(more on unix-style permissions)</a> 928 | 929 | 930 | 931 | 932 | 933 | 934 | 1.1.0+ 935 | 936 | When specified, the attachmentClassifier will cause the assembler to look at artifacts 937 | attached to the module instead of the main project artifact. If it can find an attached 938 | artifact matching the specified classifier, it will use it; otherwise, it will throw an 939 | exception. (Since 2.2-beta-1) 940 | 941 | 942 | 943 | 944 | 945 | 1.0.0+ 946 | 947 | If set to true, the plugin will include the direct and transitive dependencies of 948 | of the project modules included here. Otherwise, it will only include the module 949 | packages only. 950 | 951 | 952 | 953 | 954 | 955 | 1.1.0+ 956 | 957 | 958 | Specifies which dependencies of the module to include in the assembly. A 959 | dependencySet is specified by providing one or more of 960 | &lt;dependencySet&gt; subelements. (Since 2.2-beta-1) 961 | 962 | 963 | 964 | 965 | 966 | 967 | 968 | 969 | 970 | 971 | 972 | 1.0.0+ 973 | 974 | If set to true, this property will unpack all module packages 975 | into the specified output directory. When set to false 976 | module packages will be included as archives (jars). 977 | 978 | 979 | 980 | 981 | 982 | 1.1.0+ 983 | 984 | Allows the specification of includes and excludes, along with filtering options, for items 985 | unpacked from a module artifact. (Since 2.2-beta-1) 986 | 987 | 988 | 989 | 990 | 991 | 1.0.0+ 992 | 993 | Sets the mapping pattern for all NON-UNPACKED dependencies included 994 | in this assembly. 995 | (Since 2.2-beta-2; 2.2-beta-1 uses ${artifactId}-${version}${dashClassifier?}.${extension} as default value) 996 | NOTE: If the dependencySet specifies unpack == true, outputFileNameMapping WILL NOT BE USED; in these cases, 997 | use outputDirectory. 998 | See the plugin FAQ for more details about entries usable in the outputFileNameMapping parameter. 999 | 1000 | 1001 | 1002 | 1003 | 1004 | 1005 | 1006 | 1.0.0+ 1007 | 1008 | Contains configuration options for including the source files of a 1009 | project module in an assembly. 1010 | 1011 | 1012 | 1013 | 1014 | 1015 | 1.1.0+ 1016 | 1017 | Whether standard exclusion patterns, such as those matching CVS and Subversion 1018 | metadata files, should be used when calculating the files affected by this set. 1019 | For backward compatibility, the default value is true. (Since 2.2-beta-1) 1020 | 1021 | 1022 | 1023 | 1024 | 1025 | 1.0.0+ 1026 | 1027 | Sets the output directory relative to the root 1028 | of the root directory of the assembly. For example, 1029 | "log" will put the specified files in the log directory. 1030 | 1031 | 1032 | 1033 | 1034 | 1035 | 1.0.0+ 1036 | 1037 | 1038 | When &lt;include&gt; subelements are present, they define a set of 1039 | files and directory to include. If none is present, then 1040 | &lt;includes&gt; represents all valid values. 1041 | 1042 | 1043 | 1044 | 1045 | 1046 | 1047 | 1048 | 1049 | 1050 | 1051 | 1052 | 1.0.0+ 1053 | 1054 | 1055 | When &lt;exclude&gt; subelements are present, they define a set of 1056 | files and directory to exclude. If none is present, then 1057 | &lt;excludes&gt; represents no exclusions. 1058 | 1059 | 1060 | 1061 | 1062 | 1063 | 1064 | 1065 | 1066 | 1067 | 1068 | 1069 | 1.0.0+ 1070 | 1071 | 1072 | Similar to a UNIX permission, sets the file mode of the files included. 1073 | THIS IS AN OCTAL VALUE. 1074 | Format: (User)(Group)(Other) where each component is a sum of Read = 4, 1075 | Write = 2, and Execute = 1. For example, the value 0644 1076 | translates to User read-write, Group and Other read-only. The default value is 0644 1077 | <a href="http://www.onlamp.com/pub/a/bsd/2000/09/06/FreeBSD_Basics.html">(more on unix-style permissions)</a> 1078 | 1079 | 1080 | 1081 | 1082 | 1083 | 1084 | 1.0.0+ 1085 | 1086 | 1087 | Similar to a UNIX permission, sets the directory mode of the directories 1088 | included. 1089 | THIS IS AN OCTAL VALUE. 1090 | Format: (User)(Group)(Other) where each component is a sum of 1091 | Read = 4, Write = 2, and Execute = 1. For example, the value 1092 | 0755 translates to User read-write, Group and Other read-only. The default value is 0755. 1093 | <a href="http://www.onlamp.com/pub/a/bsd/2000/09/06/FreeBSD_Basics.html">(more on unix-style permissions)</a> 1094 | 1095 | 1096 | 1097 | 1098 | 1099 | 1100 | 1.1.0+ 1101 | 1102 | 1103 | Specifies which groups of files from each included module to include in the assembly. A 1104 | fileSet is specified by providing one or more of &lt;fileSet&gt; subelements. (Since 2.2-beta-1) 1105 | 1106 | 1107 | 1108 | 1109 | 1110 | 1111 | 1112 | 1113 | 1114 | 1115 | 1116 | 1.1.0+ 1117 | 1118 | 1119 | Specifies whether the module's finalName should be prepended to the outputDirectory 1120 | values of any fileSets applied to it. (Since 2.2-beta-1) 1121 | 1122 | 1123 | 1124 | 1125 | 1126 | 1127 | 1.1.0+ 1128 | 1129 | 1130 | Specifies whether sub-module directories below the current module should be excluded 1131 | from fileSets applied to that module. This might be useful if you only mean to copy 1132 | the sources for the exact module list matched by this ModuleSet, ignoring (or processing 1133 | separately) the modules which exist in directories below the current one. (Since 2.2-beta-1) 1134 | 1135 | 1136 | 1137 | 1138 | 1139 | 1140 | 1.1.0+ 1141 | 1142 | Sets the mapping pattern for all module base-directories included in this assembly. 1143 | NOTE: This field is only used if includeModuleDirectory == true. 1144 | Default is the module's ${artifactId} in 2.2-beta-1, and ${module.artifactId} in subsequent versions. (Since 2.2-beta-1) 1145 | 1146 | 1147 | 1148 | 1149 | 1150 | 1151 | 1152 | 1.0.0+ 1153 | 1154 | A fileSet allows the inclusion of groups of files into the assembly. 1155 | 1156 | 1157 | 1158 | 1159 | 1160 | 1.1.0+ 1161 | 1162 | Whether standard exclusion patterns, such as those matching CVS and Subversion 1163 | metadata files, should be used when calculating the files affected by this set. 1164 | For backward compatibility, the default value is true. (Since 2.2-beta-1) 1165 | 1166 | 1167 | 1168 | 1169 | 1170 | 1.0.0+ 1171 | 1172 | Sets the output directory relative to the root 1173 | of the root directory of the assembly. For example, 1174 | "log" will put the specified files in the log directory. 1175 | 1176 | 1177 | 1178 | 1179 | 1180 | 1.0.0+ 1181 | 1182 | 1183 | When &lt;include&gt; subelements are present, they define a set of 1184 | files and directory to include. If none is present, then 1185 | &lt;includes&gt; represents all valid values. 1186 | 1187 | 1188 | 1189 | 1190 | 1191 | 1192 | 1193 | 1194 | 1195 | 1196 | 1197 | 1.0.0+ 1198 | 1199 | 1200 | When &lt;exclude&gt; subelements are present, they define a set of 1201 | files and directory to exclude. If none is present, then 1202 | &lt;excludes&gt; represents no exclusions. 1203 | 1204 | 1205 | 1206 | 1207 | 1208 | 1209 | 1210 | 1211 | 1212 | 1213 | 1214 | 1.0.0+ 1215 | 1216 | 1217 | Similar to a UNIX permission, sets the file mode of the files included. 1218 | THIS IS AN OCTAL VALUE. 1219 | Format: (User)(Group)(Other) where each component is a sum of Read = 4, 1220 | Write = 2, and Execute = 1. For example, the value 0644 1221 | translates to User read-write, Group and Other read-only. The default value is 0644. 1222 | <a href="http://www.onlamp.com/pub/a/bsd/2000/09/06/FreeBSD_Basics.html">(more on unix-style permissions)</a> 1223 | 1224 | 1225 | 1226 | 1227 | 1228 | 1229 | 1.0.0+ 1230 | 1231 | 1232 | Similar to a UNIX permission, sets the directory mode of the directories 1233 | included. 1234 | THIS IS AN OCTAL VALUE. 1235 | Format: (User)(Group)(Other) where each component is a sum of 1236 | Read = 4, Write = 2, and Execute = 1. For example, the value 1237 | 0755 translates to User read-write, Group and Other read-only. The default value is 0755. 1238 | <a href="http://www.onlamp.com/pub/a/bsd/2000/09/06/FreeBSD_Basics.html">(more on unix-style permissions)</a> 1239 | 1240 | 1241 | 1242 | 1243 | 1244 | 1245 | 1.0.0+ 1246 | 1247 | Sets the absolute or relative location from the module's 1248 | directory. For example, "src/main/bin" would select this 1249 | subdirectory of the project in which this dependency is defined. 1250 | 1251 | 1252 | 1253 | 1254 | 1255 | 1.0.0+ 1256 | 1257 | 1258 | Sets the line-endings of the files in this fileSet. 1259 | Valid values: 1260 | <ul> 1261 | <li><b>"keep"</b> - Preserve all line endings</li> 1262 | <li><b>"unix"</b> - Use Unix-style line endings (i.e. "\n")</li> 1263 | <li><b>"lf"</b> - Use a single line-feed line endings (i.e. "\n")</li> 1264 | <li><b>"dos"</b> - Use DOS-/Windows-style line endings (i.e. "\r\n")</li> 1265 | <li><b>"windows"</b> - Use DOS-/Windows-style line endings (i.e. "\r\n")</li> 1266 | <li><b>"crlf"</b> - Use carriage-return, line-feed line endings (i.e. "\r\n")</li> 1267 | </ul> 1268 | 1269 | 1270 | 1271 | 1272 | 1273 | 1274 | 1.1.0+ 1275 | 1276 | Whether to filter symbols in the files as they are copied, using 1277 | properties from the build configuration. (Since 2.2-beta-1) 1278 | 1279 | 1280 | 1281 | 1282 | 1283 | 1284 | 1285 | 1.0.0+ 1286 | 1287 | A file allows individual file inclusion with the option to change 1288 | the destination filename not supported by fileSets. 1289 | 1290 | 1291 | 1292 | 1293 | 1294 | 1.0.0+ 1295 | 1296 | Sets the absolute or relative path from the module's directory 1297 | of the file to be included in the assembly. 1298 | 1299 | 1300 | 1301 | 1302 | 1303 | 1.0.0+ 1304 | 1305 | Sets the output directory relative to the root 1306 | of the root directory of the assembly. For example, 1307 | "log" will put the specified files in the log directory. 1308 | 1309 | 1310 | 1311 | 1312 | 1313 | 1.0.0+ 1314 | 1315 | Sets the destination filename in the outputDirectory. 1316 | Default is the same name as the source's file. 1317 | 1318 | 1319 | 1320 | 1321 | 1322 | 1.0.0+ 1323 | 1324 | 1325 | Similar to a UNIX permission, sets the file mode of the files included. 1326 | THIS IS AN OCTAL VALUE. 1327 | Format: (User)(Group)(Other) where each component is a sum of Read = 4, 1328 | Write = 2, and Execute = 1. For example, the value 0644 1329 | translates to User read-write, Group and Other read-only. The default value is 0644 1330 | <a href="http://www.onlamp.com/pub/a/bsd/2000/09/06/FreeBSD_Basics.html">(more on unix-style permissions)</a> 1331 | 1332 | 1333 | 1334 | 1335 | 1336 | 1337 | 1.0.0+ 1338 | 1339 | 1340 | Sets the line-endings of the files in this file. 1341 | Valid values are: 1342 | <ul> 1343 | <li><b>"keep"</b> - Preserve all line endings</li> 1344 | <li><b>"unix"</b> - Use Unix-style line endings (i.e. "\n")</li> 1345 | <li><b>"lf"</b> - Use a single line-feed line endings (i.e. "\n")</li> 1346 | <li><b>"dos"</b> - Use DOS-/Windows-style line endings (i.e. "\r\n")</li> 1347 | <li><b>"windows"</b> - Use DOS-/Windows-style line endings (i.e. "\r\n")</li> 1348 | <li><b>"crlf"</b> - Use carriage-return, line-feed line endings (i.e. "\r\n")</li> 1349 | </ul> 1350 | 1351 | 1352 | 1353 | 1354 | 1355 | 1356 | 1.0.0+ 1357 | 1358 | Sets whether to determine if the file is filtered. 1359 | 1360 | 1361 | 1362 | 1363 | 1364 | --------------------------------------------------------------------------------