├── src
    └── main
    │   ├── resources
    │       └── META-INF
    │       │   └── services
    │       │       └── org.graylog2.plugin.Plugin
    │   └── java
    │       └── org
    │           ├── apache
    │               └── hadoop
    │               │   └── fs
    │               │       └── http
    │               │           └── client
    │               │               ├── AuthenticationType.java
    │               │               ├── Response.java
    │               │               ├── Main.java
    │               │               ├── PseudoAuthenticator2.java
    │               │               ├── URLUtil.java
    │               │               ├── KerberosAuthenticator2.java
    │               │               └── WebHDFSConnection.java
    │           └── graylog
    │               └── outputs
    │                   └── hdfs
    │                       ├── WebHDFSOutputPlugin.java
    │                       ├── WebHDFSOutputModule.java
    │                       ├── WebHDFSOutputMetaData.java
    │                       └── WebHDFSOutput.java
├── webhdfs-plugin-config.png
├── .gitignore
├── LICENSE
├── README.md
├── pom.xml
├── graylog-plugin-output-hdfs.iml
└── graylog-plugin-output-webhdfs.iml


/src/main/resources/META-INF/services/org.graylog2.plugin.Plugin:
--------------------------------------------------------------------------------
1 | org.graylog.outputs.hdfs.WebHDFSOutputPlugin


--------------------------------------------------------------------------------
/webhdfs-plugin-config.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sivasamyk/graylog-plugin-output-webhdfs/HEAD/webhdfs-plugin-config.png


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | target/
 2 | pom.xml.tag
 3 | pom.xml.releaseBackup
 4 | pom.xml.versionsBackup
 5 | pom.xml.next
 6 | release.properties
 7 | dependency-reduced-pom.xml
 8 | buildNumber.properties
 9 | .mvn/timing.properties
10 | .idea/


--------------------------------------------------------------------------------
/src/main/java/org/apache/hadoop/fs/http/client/AuthenticationType.java:
--------------------------------------------------------------------------------
1 | package org.apache.hadoop.fs.http.client;
2 | 
3 | /**
4 |  * Created on 22/7/15.
5 |  */
6 | public enum AuthenticationType {
7 |     KERBEROS, PSEUDO
8 | }
9 | 


--------------------------------------------------------------------------------
/src/main/java/org/graylog/outputs/hdfs/WebHDFSOutputPlugin.java:
--------------------------------------------------------------------------------
 1 | package org.graylog.outputs.hdfs;
 2 | 
 3 | import org.graylog2.plugin.Plugin;
 4 | import org.graylog2.plugin.PluginMetaData;
 5 | import org.graylog2.plugin.PluginModule;
 6 | 
 7 | import java.util.Arrays;
 8 | import java.util.Collection;
 9 | 
10 | /**
11 |  * Implement the Plugin interface here.
12 |  */
13 | public class WebHDFSOutputPlugin implements Plugin {
14 |     @Override
15 |     public PluginMetaData metadata() {
16 |         return new WebHDFSOutputMetaData();
17 |     }
18 | 
19 |     @Override
20 |     public Collection<PluginModule> modules () {
21 |         return Arrays.<PluginModule>asList(new WebHDFSOutputModule());
22 |     }
23 | }
24 | 


--------------------------------------------------------------------------------
/src/main/java/org/apache/hadoop/fs/http/client/Response.java:
--------------------------------------------------------------------------------
 1 | package org.apache.hadoop.fs.http.client;
 2 | 
 3 | /**
 4 |  * Created on 19/7/15.
 5 |  */
 6 | public class Response {
 7 |     private int code;
 8 |     private String status;
 9 |     private String data;
10 |     private String contentType;
11 | 
12 |     public int getCode() {
13 |         return code;
14 |     }
15 | 
16 |     public void setCode(int code) {
17 |         this.code = code;
18 |     }
19 | 
20 |     public String getStatus() {
21 |         return status;
22 |     }
23 | 
24 |     public void setStatus(String status) {
25 |         this.status = status;
26 |     }
27 | 
28 |     public String getData() {
29 |         return data;
30 |     }
31 | 
32 |     public void setData(String data) {
33 |         this.data = data;
34 |     }
35 | 
36 |     public String getContentType() {
37 |         return contentType;
38 |     }
39 | 
40 |     public void setContentType(String contentType) {
41 |         this.contentType = contentType;
42 |     }
43 | }
44 | 


--------------------------------------------------------------------------------
/src/main/java/org/apache/hadoop/fs/http/client/Main.java:
--------------------------------------------------------------------------------
 1 | package org.apache.hadoop.fs.http.client;
 2 | 
 3 | 
 4 | import java.io.ByteArrayInputStream;
 5 | import java.io.ByteArrayOutputStream;
 6 | 
 7 | /**
 8 |  * Created on 5/7/15.
 9 |  */
10 | public class Main {
11 |     public static void main(String args[]) throws  Exception
12 |     {
13 |         WebHDFSConnection connection =
14 |                 new WebHDFSConnection("http://localhost:50070","hadoopuser","anything", AuthenticationType.PSEUDO);
15 |         //System.out.println(connection.listStatus("user/hadoopuser"));
16 |         System.out.print(connection.getHomeDirectory());
17 |         ByteArrayInputStream stream = new ByteArrayInputStream("India is my Country".getBytes());
18 |         System.out.println(connection.append("tmp/india2.txt", stream));
19 |         ByteArrayOutputStream os  = new ByteArrayOutputStream();
20 |         System.out.println(connection.open("tmp/india2.txt",os));
21 |         System.out.println(new String(os.toByteArray()));
22 |     }
23 | }
24 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2015 Sivasamy Kaliappan 
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/src/main/java/org/graylog/outputs/hdfs/WebHDFSOutputModule.java:
--------------------------------------------------------------------------------
 1 | package org.graylog.outputs.hdfs;
 2 | 
 3 | import com.google.inject.multibindings.MapBinder;
 4 | import org.graylog2.plugin.PluginConfigBean;
 5 | import org.graylog2.plugin.PluginModule;
 6 | import org.graylog2.plugin.outputs.MessageOutput;
 7 | 
 8 | import java.util.Collections;
 9 | import java.util.Set;
10 | 
11 | /**
12 |  * Extend the PluginModule abstract class here to add you plugin to the system.
13 |  */
14 | public class WebHDFSOutputModule extends PluginModule {
15 |     /**
16 |      * Returns all configuration beans required by this plugin.
17 |      * <p/>
18 |      * Implementing this method is optional. The default method returns an empty {@link Set}.
19 |      */
20 |     @Override
21 |     public Set<? extends PluginConfigBean> getConfigBeans() {
22 |         return Collections.emptySet();
23 |     }
24 | 
25 |     @Override
26 |     protected void configure() {
27 |         final MapBinder<String, MessageOutput.Factory<? extends MessageOutput>> outputMapBinder = outputsMapBinder();
28 |         installOutput(outputMapBinder, WebHDFSOutput.class, WebHDFSOutput.Factory.class);
29 |     }
30 | }
31 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Graylog WebHDFS Output Plugin
 2 | 
 3 | An output plugin to write messages to Hadoop HDFS over WebHDFS.
 4 | 
 5 | Tested with Hadoop 2.7.0.
 6 | 
 7 | The WebHDFS implementation uses modified code from [webhdfs-java-client](https://github.com/zxs/webhdfs-java-client).
 8 | 
 9 | Getting started
10 | ---------------
11 | 
12 | To start using this plugin place this [jar](https://github.com/sivasamyk/graylog-plugin-output-webhdfs/releases/download/1.0.1/graylog-plugin-output-webhdfs-1.0.1.jar) in the plugins directory and restart graylog server. 
13 | 
14 | Following parameters can be configured while launching the plugin
15 | 
16 | * Host - IP Address or hostname of the HDFS name node
17 | * Port - WebHDFS port configured in HDFS. 
18 | * Username - Username of pseudo authentication (currently kerberos is not supported)
19 | * File path - Path of file to store the messages. File name can be formatted with message fields or date formats. E.g ${source}_%Y_%m_%d.log for storing the messages based source and day.
20 | * Message Format - Format of message to be written. Can be formatted with message fields like ${timestamp} | ${source} | ${short_message}
21 | * Flush interval - Interval in seconds to flush the data to HDFS. Value of 0 means immediate.
22 | 
23 | ![Plugin configuration window](https://github.com/sivasamyk/graylog-plugin-output-webhdfs/raw/master/webhdfs-plugin-config.png)
24 | 
25 | 


--------------------------------------------------------------------------------
/src/main/java/org/apache/hadoop/fs/http/client/PseudoAuthenticator2.java:
--------------------------------------------------------------------------------
 1 | package org.apache.hadoop.fs.http.client;
 2 | 
 3 | /**
 4 |  * Licensed under the Apache License, Version 2.0 (the "License");
 5 |  * you may not use this file except in compliance with the License.
 6 |  * You may obtain a copy of the License at
 7 |  *
 8 |  *   http://www.apache.org/licenses/LICENSE-2.0
 9 |  *
10 |  * Unless required by applicable law or agreed to in writing, software
11 |  * distributed under the License is distributed on an "AS IS" BASIS,
12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |  * See the License for the specific language governing permissions and
14 |  * limitations under the License. See accompanying LICENSE file.
15 |  */
16 | 
17 | import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
18 | 
19 | /**
20 |  * The {@link PseudoAuthenticator} implementation provides an authentication
21 |  * equivalent to Hadoop's Simple authentication, it trusts the value of the
22 |  * 'user.name' Java System property.
23 |  * <p/>
24 |  * The 'user.name' value is propagated using an additional query string
25 |  * parameter {@link #USER_NAME} ('user.name').
26 |  */
27 | public class PseudoAuthenticator2 extends PseudoAuthenticator {
28 | 
29 | 	private String username;
30 | 
31 | 	public PseudoAuthenticator2(String username) {
32 | 		this.username = username;
33 | 	}
34 | 
35 | 	@Override
36 | 	protected String getUserName() {
37 | 		return username;
38 | 	}
39 | }
40 | 


--------------------------------------------------------------------------------
/src/main/java/org/graylog/outputs/hdfs/WebHDFSOutputMetaData.java:
--------------------------------------------------------------------------------
 1 | package org.graylog.outputs.hdfs;
 2 | 
 3 | import org.graylog2.plugin.PluginMetaData;
 4 | import org.graylog2.plugin.ServerStatus;
 5 | import org.graylog2.plugin.Version;
 6 | 
 7 | import java.net.URI;
 8 | import java.util.Collections;
 9 | import java.util.Set;
10 | 
11 | /**
12 |  * Implement the PluginMetaData interface here.
13 |  */
14 | public class WebHDFSOutputMetaData implements PluginMetaData {
15 |     @Override
16 |     public String getUniqueId() {
17 |         return "org.graylog.outputs.hdfs.WebHDFSOutputPlugin";
18 |     }
19 | 
20 |     @Override
21 |     public String getName() {
22 |         return "WebHDFSOutput";
23 |     }
24 | 
25 |     @Override
26 |     public String getAuthor() {
27 |         return "Sivasamy Kaliappan";
28 |     }
29 | 
30 |     @Override
31 |     public URI getURL() {
32 |         return URI.create("https://www.graylog.org/");
33 |     }
34 | 
35 |     @Override
36 |     public Version getVersion() {
37 |         return new Version(1, 0, 1);
38 |     }
39 | 
40 |     @Override
41 |     public String getDescription() {
42 |         return "Forwards the output to Hadoop Distributed File System (HDFS) using WebHDFS";
43 |     }
44 | 
45 |     @Override
46 |     public Version getRequiredVersion() {
47 |         return new Version(1, 0, 0);
48 |     }
49 | 
50 |     @Override
51 |     public Set<ServerStatus.Capability> getRequiredCapabilities() {
52 |         return Collections.emptySet();
53 |     }
54 | }
55 | 


--------------------------------------------------------------------------------
/src/main/java/org/apache/hadoop/fs/http/client/URLUtil.java:
--------------------------------------------------------------------------------
  1 | package org.apache.hadoop.fs.http.client;
  2 | 
  3 | import java.io.ByteArrayOutputStream;
  4 | import java.io.IOException;
  5 | import java.io.OutputStreamWriter;
  6 | import java.util.BitSet;
  7 | 
  8 | public class URLUtil {
  9 | 	/**
 10 | 	   * Array containing the safe characters set as defined by RFC 1738
 11 | 	   */
 12 | 	  private static BitSet safeCharacters;
 13 | 
 14 | 	  private static final char[] hexadecimal =
 15 | 	  {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
 16 | 	   'A', 'B', 'C', 'D', 'E', 'F'};
 17 | 
 18 | 	  static {
 19 | 	      safeCharacters = new BitSet(256);
 20 | 	      int i;
 21 | 	      // 'lowalpha' rule
 22 | 	      for (i = 'a'; i <= 'z'; i++) {
 23 | 	          safeCharacters.set(i);
 24 | 	      }
 25 | 	      // 'hialpha' rule
 26 | 	      for (i = 'A'; i <= 'Z'; i++) {
 27 | 	          safeCharacters.set(i);
 28 | 	      }
 29 | 	      // 'digit' rule
 30 | 	      for (i = '0'; i <= '9'; i++) {
 31 | 	          safeCharacters.set(i);
 32 | 	      }
 33 | 
 34 | 	      // 'safe' rule
 35 | 	      safeCharacters.set('$');
 36 | 	      safeCharacters.set('-');
 37 | 	      safeCharacters.set('_');
 38 | 	      safeCharacters.set('.');
 39 | 	      safeCharacters.set('+');
 40 | 
 41 | 	      // 'extra' rule
 42 | 	      safeCharacters.set('!');
 43 | 	      safeCharacters.set('*');
 44 | 	      safeCharacters.set('\'');
 45 | 	      safeCharacters.set('(');
 46 | 	      safeCharacters.set(')');
 47 | 	      safeCharacters.set(',');
 48 | 
 49 | 	      // special characters common to http: file: and ftp: URLs ('fsegment' and 'hsegment' rules)
 50 | 	      safeCharacters.set('/');
 51 | 	      safeCharacters.set(':');
 52 | 	      safeCharacters.set('@');
 53 | 	      safeCharacters.set('&');
 54 | 	      safeCharacters.set('=');
 55 | 	  }
 56 | 
 57 | 
 58 | 	  /**
 59 | 	   * Encode a path as required by the URL specification (<a href="http://www.ietf.org/rfc/rfc1738.txt">
 60 | 	   * RFC 1738</a>). This differs from <code>java.net.URLEncoder.encode()</code> which encodes according
 61 | 	   * to the <code>x-www-form-urlencoded</code> MIME format.
 62 | 	   *
 63 | 	   * @param path the path to encode
 64 | 	   * @return the encoded path
 65 | 	   */
 66 | 	  public static String encodePath(String path) {
 67 | 	     // stolen from org.apache.catalina.servlets.DefaultServlet ;)
 68 | 
 69 | 	      /**
 70 | 	       * Note: Here, ' ' should be encoded as "%20"
 71 | 	       * and '/' shouldn't be encoded.
 72 | 	       */
 73 | 
 74 | 	      int maxBytesPerChar = 10;
 75 | 	      StringBuffer rewrittenPath = new StringBuffer(path.length());
 76 | 	      ByteArrayOutputStream buf = new ByteArrayOutputStream(maxBytesPerChar);
 77 | 	      OutputStreamWriter writer;
 78 | 	      try {
 79 | 	          writer = new OutputStreamWriter(buf, "UTF8");
 80 | 	      } catch (Exception e) {
 81 | 	          e.printStackTrace();
 82 | 	          writer = new OutputStreamWriter(buf);
 83 | 	      }
 84 | 
 85 | 	      for (int i = 0; i < path.length(); i++) {
 86 | 	          int c = path.charAt(i);
 87 | 	          if (safeCharacters.get(c)) {
 88 | 	              rewrittenPath.append((char)c);
 89 | 	          } else {
 90 | 	              // convert to external encoding before hex conversion
 91 | 	              try {
 92 | 	                  writer.write(c);
 93 | 	                  writer.flush();
 94 | 	              } catch(IOException e) {
 95 | 	                  buf.reset();
 96 | 	                  continue;
 97 | 	              }
 98 | 	              byte[] ba = buf.toByteArray();
 99 | 	              for (int j = 0; j < ba.length; j++) {
100 | 	                  // Converting each byte in the buffer
101 | 	                  byte toEncode = ba[j];
102 | 	                  rewrittenPath.append('%');
103 | 	                  int low = (toEncode & 0x0f);
104 | 	                  int high = ((toEncode & 0xf0) >> 4);
105 | 	                  rewrittenPath.append(hexadecimal[high]);
106 | 	                  rewrittenPath.append(hexadecimal[low]);
107 | 	              }
108 | 	              buf.reset();
109 | 	          }
110 | 	      }
111 | 	      return rewrittenPath.toString();
112 | 	  }
113 | 	  
114 | 	  public static void main(String[] args) {
115 | 		  System.out.println( URLUtil.encodePath("zen/我的图片") );
116 | 	  }
117 | }
118 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <project xmlns="http://maven.apache.org/POM/4.0.0"
  3 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  4 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  5 |     <modelVersion>4.0.0</modelVersion>
  6 | 
  7 |     <groupId>org.graylog</groupId>
  8 |     <artifactId>graylog-plugin-output-webhdfs</artifactId>
  9 |     <version>1.0.1</version>
 10 |     <packaging>jar</packaging>
 11 | 
 12 |     <name>${project.artifactId}</name>
 13 |     <description>Graylog ${project.artifactId} plugin.</description>
 14 |     <url>https://www.graylog.org</url>
 15 | 
 16 |     <properties>
 17 |         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 18 |         <maven.compiler.source>1.7</maven.compiler.source>
 19 |         <maven.compiler.target>1.7</maven.compiler.target>
 20 |         <graylog2.version>1.0.0</graylog2.version>
 21 |         <graylog2.plugin-dir>/usr/share/graylog-server/plugin</graylog2.plugin-dir>
 22 |     </properties>
 23 | 
 24 |     <dependencies>
 25 |         <dependency>
 26 |             <groupId>org.graylog2</groupId>
 27 |             <artifactId>graylog2-plugin</artifactId>
 28 |             <version>${graylog2.version}</version>
 29 |             <scope>provided</scope>
 30 |         </dependency>
 31 |         <dependency>
 32 |             <groupId>commons-lang</groupId>
 33 |             <artifactId>commons-lang</artifactId>
 34 |             <version>2.6</version>
 35 |         </dependency>
 36 |         <dependency>
 37 |             <groupId>org.apache.hadoop</groupId>
 38 |             <artifactId>hadoop-auth</artifactId>
 39 |             <version>2.7.0</version>
 40 |             <exclusions>
 41 |                 <exclusion>
 42 |                     <groupId>jdk.tools</groupId>
 43 |                     <artifactId>jdk.tools</artifactId>
 44 |                 </exclusion>
 45 |             </exclusions>
 46 |         </dependency>
 47 |     </dependencies>
 48 | 
 49 |     <build>
 50 |         <plugins>
 51 |             <plugin>
 52 |                 <groupId>org.apache.maven.plugins</groupId>
 53 |                 <artifactId>maven-shade-plugin</artifactId>
 54 |                 <version>2.3</version>
 55 |                 <configuration>
 56 |                     <minimizeJar>false</minimizeJar>
 57 |                 </configuration>
 58 |                 <executions>
 59 |                     <execution>
 60 |                         <phase>package</phase>
 61 |                         <goals>
 62 |                             <goal>shade</goal>
 63 |                         </goals>
 64 |                         <configuration>
 65 |                             <transformers>
 66 |                                 <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
 67 |                             </transformers>
 68 |                         </configuration>
 69 |                     </execution>
 70 |                 </executions>
 71 |             </plugin>
 72 | 
 73 |             <plugin>
 74 |                 <artifactId>jdeb</artifactId>
 75 |                 <groupId>org.vafer</groupId>
 76 |                 <version>1.3</version>
 77 |                 <configuration>
 78 |                     <deb>${project.build.directory}/${project.artifactId}-${project.version}.deb</deb>
 79 |                     <dataSet>
 80 |                         <data>
 81 |                             <src>${project.build.directory}/${project.build.finalName}.jar</src>
 82 |                             <type>file</type>
 83 |                             <mapper>
 84 |                                 <type>perm</type>
 85 |                                 <prefix>${graylog2.plugin-dir}</prefix>
 86 |                                 <filemode>644</filemode>
 87 |                                 <user>root</user>
 88 |                                 <group>root</group>
 89 |                             </mapper>
 90 |                         </data>
 91 |                     </dataSet>
 92 |                 </configuration>
 93 |             </plugin>
 94 | 
 95 |             <plugin>
 96 |                 <groupId>org.codehaus.mojo</groupId>
 97 |                 <artifactId>rpm-maven-plugin</artifactId>
 98 |                 <version>2.1.2</version>
 99 |                 <configuration>
100 |                     <group>Application/Internet</group>
101 |                     <prefixes>
102 |                         <prefix>/usr</prefix>
103 |                     </prefixes>
104 |                     <defineStatements>
105 |                         <defineStatement>_unpackaged_files_terminate_build 0</defineStatement>
106 |                         <defineStatement>_binaries_in_noarch_packages_terminate_build 0</defineStatement>
107 |                     </defineStatements>
108 |                     <defaultFilemode>644</defaultFilemode>
109 |                     <defaultDirmode>755</defaultDirmode>
110 |                     <defaultUsername>root</defaultUsername>
111 |                     <defaultGroupname>root</defaultGroupname>
112 |                     <mappings>
113 |                         <mapping>
114 |                             <directory>${graylog2.plugin-dir}</directory>
115 |                             <sources>
116 |                                 <source>
117 |                                     <location>${project.build.directory}/</location>
118 |                                     <includes>
119 |                                         <include>${project.build.finalName}.jar</include>
120 |                                     </includes>
121 |                                 </source>
122 |                             </sources>
123 |                         </mapping>
124 |                     </mappings>
125 |                 </configuration>
126 |             </plugin>
127 |         </plugins>
128 |     </build>
129 | </project>
130 | 


--------------------------------------------------------------------------------
/graylog-plugin-output-hdfs.iml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
 3 |   <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_7" inherit-compiler-output="false">
 4 |     <output url="file://$MODULE_DIR$/target/classes" />
 5 |     <output-test url="file://$MODULE_DIR$/target/test-classes" />
 6 |     <content url="file://$MODULE_DIR$">
 7 |       <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
 8 |       <sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
 9 |       <excludeFolder url="file://$MODULE_DIR$/target" />
10 |     </content>
11 |     <orderEntry type="inheritedJdk" />
12 |     <orderEntry type="sourceFolder" forTests="false" />
13 |     <orderEntry type="module-library">
14 |       <library>
15 |         <CLASSES>
16 |           <root url="jar://$MODULE_DIR$/../graylog2-server/graylog2-plugin-interfaces/target/graylog2-plugin-1.1.2-SNAPSHOT.jar!/" />
17 |         </CLASSES>
18 |         <JAVADOC />
19 |         <SOURCES />
20 |       </library>
21 |     </orderEntry>
22 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.graylog2:graylog2-plugin:1.0.0" level="project" />
23 |     <orderEntry type="library" name="Maven: com.google.guava:guava:18.0" level="project" />
24 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.inject.extensions:guice-assistedinject:3.0" level="project" />
25 |     <orderEntry type="library" scope="PROVIDED" name="Maven: joda-time:joda-time:2.6" level="project" />
26 |     <orderEntry type="library" scope="PROVIDED" name="Maven: io.dropwizard.metrics:metrics-core:3.1.0" level="project" />
27 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.lmax:disruptor:3.3.0" level="project" />
28 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.inject:guice:3.0" level="project" />
29 |     <orderEntry type="library" scope="PROVIDED" name="Maven: javax.inject:javax.inject:1" level="project" />
30 |     <orderEntry type="library" scope="PROVIDED" name="Maven: aopalliance:aopalliance:1.0" level="project" />
31 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.inject.extensions:guice-multibindings:3.0" level="project" />
32 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.fasterxml.jackson.core:jackson-core:2.5.0" level="project" />
33 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.5.0" level="project" />
34 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.5.0" level="project" />
35 |     <orderEntry type="library" scope="PROVIDED" name="Maven: javax.ws.rs:javax.ws.rs-api:2.0.1" level="project" />
36 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.msgpack:msgpack:0.6.11" level="project" />
37 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.googlecode.json-simple:json-simple:1.1.1" level="project" />
38 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.javassist:javassist:3.18.1-GA" level="project" />
39 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.protobuf:protobuf-java:2.6.1" level="project" />
40 |     <orderEntry type="library" name="Maven: io.netty:netty:3.9.6.Final" level="project" />
41 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.glassfish.jersey.core:jersey-common:2.14" level="project" />
42 |     <orderEntry type="library" scope="PROVIDED" name="Maven: javax.annotation:javax.annotation-api:1.2" level="project" />
43 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.glassfish.jersey.bundles.repackaged:jersey-guava:2.14" level="project" />
44 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.glassfish.hk2:hk2-api:2.4.0-b06" level="project" />
45 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.glassfish.hk2:hk2-utils:2.4.0-b06" level="project" />
46 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.glassfish.hk2.external:aopalliance-repackaged:2.4.0-b06" level="project" />
47 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.glassfish.hk2.external:javax.inject:2.4.0-b06" level="project" />
48 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.glassfish.hk2:hk2-locator:2.4.0-b06" level="project" />
49 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.glassfish.hk2:osgi-resource-locator:1.0.1" level="project" />
50 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.kie:kie-api:6.1.0.Final" level="project" />
51 |     <orderEntry type="library" scope="PROVIDED" name="Maven: commons-io:commons-io:2.4" level="project" />
52 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.eaio.uuid:uuid:3.2" level="project" />
53 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.github.joschi:jadconfig:0.10.0" level="project" />
54 |     <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.9" level="project" />
55 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.code.findbugs:jsr305:3.0.0" level="project" />
56 |     <orderEntry type="library" name="Maven: commons-lang:commons-lang:2.6" level="project" />
57 |     <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-auth:2.7.0" level="project" />
58 |     <orderEntry type="library" name="Maven: commons-codec:commons-codec:1.4" level="project" />
59 |     <orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />
60 |     <orderEntry type="library" name="Maven: org.slf4j:slf4j-log4j12:1.7.10" level="project" />
61 |     <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.2.5" level="project" />
62 |     <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.2.4" level="project" />
63 |     <orderEntry type="library" name="Maven: commons-logging:commons-logging:1.1.1" level="project" />
64 |     <orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15" level="project" />
65 |     <orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-i18n:2.0.0-M15" level="project" />
66 |     <orderEntry type="library" name="Maven: org.apache.directory.api:api-asn1-api:1.0.0-M20" level="project" />
67 |     <orderEntry type="library" name="Maven: org.apache.directory.api:api-util:1.0.0-M20" level="project" />
68 |     <orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.4.6" level="project" />
69 |     <orderEntry type="library" name="Maven: jline:jline:0.9.94" level="project" />
70 |     <orderEntry type="library" name="Maven: org.apache.curator:curator-framework:2.7.1" level="project" />
71 |     <orderEntry type="library" name="Maven: org.apache.curator:curator-client:2.7.1" level="project" />
72 |   </component>
73 | </module>


--------------------------------------------------------------------------------
/graylog-plugin-output-webhdfs.iml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
 3 |   <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_7" inherit-compiler-output="false">
 4 |     <output url="file://$MODULE_DIR$/target/classes" />
 5 |     <output-test url="file://$MODULE_DIR$/target/test-classes" />
 6 |     <content url="file://$MODULE_DIR$">
 7 |       <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
 8 |       <sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
 9 |       <excludeFolder url="file://$MODULE_DIR$/target" />
10 |     </content>
11 |     <orderEntry type="inheritedJdk" />
12 |     <orderEntry type="sourceFolder" forTests="false" />
13 |     <orderEntry type="module-library">
14 |       <library>
15 |         <CLASSES>
16 |           <root url="jar://$MODULE_DIR$/../graylog2-server/graylog2-plugin-interfaces/target/graylog2-plugin-1.1.2-SNAPSHOT.jar!/" />
17 |         </CLASSES>
18 |         <JAVADOC />
19 |         <SOURCES />
20 |       </library>
21 |     </orderEntry>
22 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.graylog2:graylog2-plugin:1.0.0" level="project" />
23 |     <orderEntry type="library" name="Maven: com.google.guava:guava:18.0" level="project" />
24 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.inject.extensions:guice-assistedinject:3.0" level="project" />
25 |     <orderEntry type="library" scope="PROVIDED" name="Maven: joda-time:joda-time:2.6" level="project" />
26 |     <orderEntry type="library" scope="PROVIDED" name="Maven: io.dropwizard.metrics:metrics-core:3.1.0" level="project" />
27 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.lmax:disruptor:3.3.0" level="project" />
28 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.inject:guice:3.0" level="project" />
29 |     <orderEntry type="library" scope="PROVIDED" name="Maven: javax.inject:javax.inject:1" level="project" />
30 |     <orderEntry type="library" scope="PROVIDED" name="Maven: aopalliance:aopalliance:1.0" level="project" />
31 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.inject.extensions:guice-multibindings:3.0" level="project" />
32 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.fasterxml.jackson.core:jackson-core:2.5.0" level="project" />
33 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.5.0" level="project" />
34 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.5.0" level="project" />
35 |     <orderEntry type="library" scope="PROVIDED" name="Maven: javax.ws.rs:javax.ws.rs-api:2.0.1" level="project" />
36 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.msgpack:msgpack:0.6.11" level="project" />
37 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.googlecode.json-simple:json-simple:1.1.1" level="project" />
38 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.javassist:javassist:3.18.1-GA" level="project" />
39 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.protobuf:protobuf-java:2.6.1" level="project" />
40 |     <orderEntry type="library" name="Maven: io.netty:netty:3.9.6.Final" level="project" />
41 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.glassfish.jersey.core:jersey-common:2.14" level="project" />
42 |     <orderEntry type="library" scope="PROVIDED" name="Maven: javax.annotation:javax.annotation-api:1.2" level="project" />
43 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.glassfish.jersey.bundles.repackaged:jersey-guava:2.14" level="project" />
44 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.glassfish.hk2:hk2-api:2.4.0-b06" level="project" />
45 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.glassfish.hk2:hk2-utils:2.4.0-b06" level="project" />
46 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.glassfish.hk2.external:aopalliance-repackaged:2.4.0-b06" level="project" />
47 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.glassfish.hk2.external:javax.inject:2.4.0-b06" level="project" />
48 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.glassfish.hk2:hk2-locator:2.4.0-b06" level="project" />
49 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.glassfish.hk2:osgi-resource-locator:1.0.1" level="project" />
50 |     <orderEntry type="library" scope="PROVIDED" name="Maven: org.kie:kie-api:6.1.0.Final" level="project" />
51 |     <orderEntry type="library" scope="PROVIDED" name="Maven: commons-io:commons-io:2.4" level="project" />
52 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.eaio.uuid:uuid:3.2" level="project" />
53 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.github.joschi:jadconfig:0.10.0" level="project" />
54 |     <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.9" level="project" />
55 |     <orderEntry type="library" scope="PROVIDED" name="Maven: com.google.code.findbugs:jsr305:3.0.0" level="project" />
56 |     <orderEntry type="library" name="Maven: commons-lang:commons-lang:2.6" level="project" />
57 |     <orderEntry type="library" name="Maven: org.apache.hadoop:hadoop-auth:2.7.0" level="project" />
58 |     <orderEntry type="library" name="Maven: commons-codec:commons-codec:1.4" level="project" />
59 |     <orderEntry type="library" name="Maven: log4j:log4j:1.2.17" level="project" />
60 |     <orderEntry type="library" name="Maven: org.slf4j:slf4j-log4j12:1.7.10" level="project" />
61 |     <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.2.5" level="project" />
62 |     <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.2.4" level="project" />
63 |     <orderEntry type="library" name="Maven: commons-logging:commons-logging:1.1.1" level="project" />
64 |     <orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-kerberos-codec:2.0.0-M15" level="project" />
65 |     <orderEntry type="library" name="Maven: org.apache.directory.server:apacheds-i18n:2.0.0-M15" level="project" />
66 |     <orderEntry type="library" name="Maven: org.apache.directory.api:api-asn1-api:1.0.0-M20" level="project" />
67 |     <orderEntry type="library" name="Maven: org.apache.directory.api:api-util:1.0.0-M20" level="project" />
68 |     <orderEntry type="library" name="Maven: org.apache.zookeeper:zookeeper:3.4.6" level="project" />
69 |     <orderEntry type="library" name="Maven: jline:jline:0.9.94" level="project" />
70 |     <orderEntry type="library" name="Maven: org.apache.curator:curator-framework:2.7.1" level="project" />
71 |     <orderEntry type="library" name="Maven: org.apache.curator:curator-client:2.7.1" level="project" />
72 |   </component>
73 | </module>


--------------------------------------------------------------------------------
/src/main/java/org/graylog/outputs/hdfs/WebHDFSOutput.java:
--------------------------------------------------------------------------------
  1 | package org.graylog.outputs.hdfs;
  2 | 
  3 | import com.google.inject.assistedinject.Assisted;
  4 | import org.apache.commons.lang.text.StrSubstitutor;
  5 | import org.apache.hadoop.fs.http.client.AuthenticationType;
  6 | import org.apache.hadoop.fs.http.client.WebHDFSConnection;
  7 | import org.apache.hadoop.security.authentication.client.AuthenticationException;
  8 | import org.graylog2.plugin.Message;
  9 | import org.graylog2.plugin.configuration.Configuration;
 10 | import org.graylog2.plugin.configuration.ConfigurationRequest;
 11 | import org.graylog2.plugin.configuration.fields.ConfigurationField;
 12 | import org.graylog2.plugin.configuration.fields.NumberField;
 13 | import org.graylog2.plugin.configuration.fields.TextField;
 14 | import org.graylog2.plugin.outputs.MessageOutput;
 15 | import org.graylog2.plugin.outputs.MessageOutputConfigurationException;
 16 | import org.graylog2.plugin.streams.Stream;
 17 | import org.slf4j.Logger;
 18 | import org.slf4j.LoggerFactory;
 19 | 
 20 | import javax.inject.Inject;
 21 | import java.io.ByteArrayInputStream;
 22 | import java.io.FileNotFoundException;
 23 | import java.io.IOException;
 24 | import java.util.*;
 25 | import java.util.concurrent.atomic.AtomicBoolean;
 26 | 
 27 | public class WebHDFSOutput implements MessageOutput {
 28 | 
 29 |     private static final Logger LOG = LoggerFactory.getLogger(WebHDFSOutput.class);
 30 | 
 31 |     private static final String CK_HDFS_HOST_NAME = "HDFS_HOST_NAME";
 32 |     private static final String CK_HDFS_PORT = "HDFS_PORT";
 33 |     private static final String CK_FILE = "FILE";
 34 |     private static final String CK_MESSAGE_FORMAT = "MESSAGE_FORMAT";
 35 |     private static final String CK_FLUSH_INTERVAL = "FLUSH_INTERVAL";
 36 |     private static final String CK_CLOSE_INTERVAL = "CLOSE_INTERVAL";
 37 |     private static final String CK_APPEND = "APPEND";
 38 |     private static final String CK_REOPEN = "REOPEN";
 39 |     private static final String CK_USERNAME = "USER_NAME";
 40 | 
 41 |     private static final String FIELD_SEPARATOR = " | ";
 42 |     private Configuration configuration;
 43 |     private AtomicBoolean isRunning = new AtomicBoolean(false);
 44 |     private String fileToWrite;
 45 |     private String messageFormat;
 46 |     private long flushIntervalInMillis;
 47 |     //private boolean append;
 48 |     private Timer flushTimer;
 49 |     private TimerTask flushTask;
 50 |     private WebHDFSConnection hdfsConnection;
 51 |     private List<MessageData> messagesToWrite;
 52 | 
 53 |     @Inject
 54 |     public WebHDFSOutput(@Assisted Stream stream, @Assisted Configuration configuration)
 55 |             throws MessageOutputConfigurationException, IOException {
 56 |         this.configuration = configuration;
 57 | 
 58 |         LOG.info("WebHDFSOutput launching...");
 59 | 
 60 |         String hostname = configuration.getString(CK_HDFS_HOST_NAME);
 61 |         int port = configuration.getInt(CK_HDFS_PORT);
 62 |         String username = configuration.getString(CK_USERNAME);
 63 | 
 64 |         hdfsConnection = new WebHDFSConnection("http://" + hostname + ":" + port, username, "anything",
 65 |                 AuthenticationType.PSEUDO);
 66 | 
 67 |         messagesToWrite = new LinkedList<>();
 68 | 
 69 | 
 70 |         fileToWrite = configuration.getString(CK_FILE);
 71 |         if(fileToWrite.contains("%")) {
 72 |             fileToWrite = fileToWrite.replaceAll("%","%1\\$t");
 73 |         }
 74 |         messageFormat = configuration.getString(CK_MESSAGE_FORMAT);
 75 |         flushIntervalInMillis = configuration.getInt(CK_FLUSH_INTERVAL) * 1000;
 76 | 
 77 |         if(flushIntervalInMillis > 0) {
 78 |             flushTimer = new Timer("WebHDFS-Flush-Timer", true);
 79 |             flushTask = createFlushTask();
 80 |             flushTimer.schedule(flushTask, flushIntervalInMillis, flushIntervalInMillis);
 81 |         }
 82 | 
 83 |         //append = configuration.getBoolean(CK_APPEND);
 84 |         isRunning.set(true);
 85 |         LOG.info("WebHDFSOutput launched");
 86 |     }
 87 | 
 88 |     private TimerTask createFlushTask() {
 89 |         return new TimerTask() {
 90 |             @Override
 91 |             public void run() {
 92 |                 try {
 93 |                     writeToHdfs();
 94 |                 } catch (Exception e) {
 95 |                     LOG.warn("Exception while writing to HDFS", e);
 96 |                 }
 97 |             }
 98 |         };
 99 |     }
100 | 
101 | 
102 |     @Override
103 |     public void stop() {
104 |         LOG.info("Stopping WebHDFS output...");
105 |         if(flushTask != null) {
106 |             flushTask.cancel();
107 |         }
108 |         if(flushTimer != null) {
109 |             flushTimer.cancel();
110 |         }
111 |         isRunning.set(false);
112 |     }
113 | 
114 |     @Override
115 |     public boolean isRunning() {
116 |         return isRunning.get();
117 |     }
118 | 
119 |     public void write(Message message) throws Exception {
120 |         String path = getFormattedPath(message);
121 |         String messageToWrite = getFormattedMessage(message);
122 |         if (flushIntervalInMillis == 0) {
123 |             writeToHdfs(path, messageToWrite);
124 |         } else {
125 |             synchronized (this) {
126 |                 messagesToWrite.add(new MessageData(path, messageToWrite));
127 |             }
128 |         }
129 |     }
130 | 
131 |     private synchronized void writeToHdfs() throws IOException, AuthenticationException {
132 |         Map<String, StringBuilder> pathToDataMap = new HashMap<>();
133 |         for (MessageData message : messagesToWrite) {
134 |             StringBuilder builder = pathToDataMap.get(message.getPath());
135 | 
136 |             if (builder == null) {
137 |                 builder = new StringBuilder();
138 |                 pathToDataMap.put(message.getPath(), builder);
139 |             }
140 |             builder.append(message.getMessage());
141 |         }
142 | 
143 |         for (Map.Entry<String, StringBuilder> entry : pathToDataMap.entrySet()) {
144 |             writeToHdfs(entry.getKey(), entry.getValue().toString());
145 |         }
146 |         messagesToWrite.clear();
147 |     }
148 | 
149 |     private void writeToHdfs(String path, String data) throws IOException, AuthenticationException {
150 |         ByteArrayInputStream inputStream = new ByteArrayInputStream(
151 |                 data.getBytes());
152 |         try {
153 |             hdfsConnection.append(path, inputStream);
154 |         } catch (FileNotFoundException e) {
155 |             hdfsConnection.create(path, inputStream);
156 |         }
157 |     }
158 | 
159 | 
160 |     @Override
161 |     public void write(List<Message> list) throws Exception {
162 |         for (Message message : list) {
163 |             write(message);
164 |         }
165 |     }
166 | 
167 | 
168 |     private String getFormattedMessage(Message message) {
169 |         String formattedMessage;
170 |         if (messageFormat != null && messageFormat.length() > 0) {
171 |             formattedMessage = StrSubstitutor.replace(messageFormat, message.getFields());
172 |         } else {
173 |             formattedMessage = String.valueOf(message.getTimestamp()) + FIELD_SEPARATOR +
174 |                     message.getSource() + FIELD_SEPARATOR + message.getMessage();
175 |         }
176 | 
177 |         if (!formattedMessage.endsWith("\n")) {
178 |             formattedMessage = formattedMessage.concat("\n");
179 |         }
180 | 
181 |         return formattedMessage;
182 |     }
183 | 
184 |     private String getFormattedPath(Message message) {
185 |         String formattedPath = fileToWrite;
186 | 
187 |         if (fileToWrite.contains("${")) {
188 |             formattedPath = StrSubstitutor.replace(formattedPath, message.getFields());
189 |         }
190 | 
191 |         if(fileToWrite.contains("%")) {
192 |             formattedPath = String.format(formattedPath, message.getTimestamp().toDate());
193 |         }
194 | 
195 |         return formattedPath;
196 |     }
197 | 
198 |     public interface Factory extends MessageOutput.Factory<WebHDFSOutput> {
199 |         @Override
200 |         WebHDFSOutput create(Stream stream, Configuration configuration);
201 | 
202 |         @Override
203 |         Config getConfig();
204 | 
205 |         @Override
206 |         Descriptor getDescriptor();
207 |     }
208 | 
209 |     public static class Config extends MessageOutput.Config {
210 |         @Override
211 |         public ConfigurationRequest getRequestedConfiguration() {
212 |             final ConfigurationRequest configurationRequest = new ConfigurationRequest();
213 | 
214 |             configurationRequest.addField(new TextField(
215 |                             CK_HDFS_HOST_NAME,
216 |                             "Host",
217 |                             "",
218 |                             "IP Address or hostname of HDFS server",
219 |                             ConfigurationField.Optional.NOT_OPTIONAL)
220 |             );
221 | 
222 |             configurationRequest.addField(new NumberField(
223 |                             CK_HDFS_PORT,
224 |                             "Port",
225 |                             50070,
226 |                             "HDFS Web Port",
227 |                             ConfigurationField.Optional.NOT_OPTIONAL)
228 |             );
229 | 
230 |             configurationRequest.addField(new TextField(
231 |                             CK_USERNAME,
232 |                             "Username",
233 |                             "",
234 |                             "User name for WebHDFS connection",
235 |                             ConfigurationField.Optional.NOT_OPTIONAL)
236 |             );
237 | 
238 |             configurationRequest.addField(new TextField(
239 |                             CK_FILE,
240 |                             "File path",
241 |                             "",
242 |                             "Path of file to write messages." +
243 |                                     "Accepts message fields like ${source} or date formats like %Y_%m_%d_%H_%M",
244 |                             ConfigurationField.Optional.NOT_OPTIONAL)
245 |             );
246 | 
247 |             configurationRequest.addField(new TextField(
248 |                             CK_MESSAGE_FORMAT,
249 |                             "Message Format",
250 |                             "${timestamp} | ${source} | ${message}",
251 |                             "Format of the message to be written. Use message fields to format",
252 |                             ConfigurationField.Optional.OPTIONAL)
253 |             );
254 | 
255 |             configurationRequest.addField(new NumberField(
256 |                             CK_FLUSH_INTERVAL,
257 |                             "Flush Interval",
258 |                             0,
259 |                             "Flush interval in seconds. Recommended for high throughput outputs. 0 for immediate update",
260 |                             ConfigurationField.Optional.NOT_OPTIONAL)
261 |             );
262 | 
263 |             return configurationRequest;
264 |         }
265 |     }
266 | 
267 |     public static class Descriptor extends MessageOutput.Descriptor {
268 |         public Descriptor() {
269 |             super("WebHDFS Output", false, "", "Forwards messages to HDFS for storage");
270 |         }
271 |     }
272 | 
273 |     private static class MessageData {
274 |         private String path, message;
275 | 
276 |         public MessageData(String path, String messageToWrite) {
277 |             this.path = path;
278 |             this.message = messageToWrite;
279 |         }
280 | 
281 |         public String getPath() {
282 |             return path;
283 |         }
284 | 
285 |         public void setPath(String path) {
286 |             this.path = path;
287 |         }
288 | 
289 |         public String getMessage() {
290 |             return message;
291 |         }
292 | 
293 |         public void setMessage(String message) {
294 |             this.message = message;
295 |         }
296 |     }
297 | }
298 | 


--------------------------------------------------------------------------------
/src/main/java/org/apache/hadoop/fs/http/client/KerberosAuthenticator2.java:
--------------------------------------------------------------------------------
  1 | package org.apache.hadoop.fs.http.client;
  2 | 
  3 | /**
  4 |  * Licensed under the Apache License, Version 2.0 (the "License");
  5 |  * you may not use this file except in compliance with the License.
  6 |  * You may obtain a copy of the License at
  7 |  *
  8 |  *   http://www.apache.org/licenses/LICENSE-2.0
  9 |  *
 10 |  * Unless required by applicable law or agreed to in writing, software
 11 |  * distributed under the License is distributed on an "AS IS" BASIS,
 12 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 |  * See the License for the specific language governing permissions and
 14 |  * limitations under the License. See accompanying LICENSE file.
 15 |  */
 16 | 
 17 | import org.apache.commons.codec.binary.Base64;
 18 | import org.apache.hadoop.security.authentication.client.*;
 19 | import org.apache.hadoop.security.authentication.util.KerberosUtil;
 20 | import org.ietf.jgss.GSSContext;
 21 | import org.ietf.jgss.GSSManager;
 22 | import org.ietf.jgss.GSSName;
 23 | import org.ietf.jgss.Oid;
 24 | import org.slf4j.Logger;
 25 | import org.slf4j.LoggerFactory;
 26 | 
 27 | import javax.security.auth.Subject;
 28 | import javax.security.auth.callback.*;
 29 | import javax.security.auth.login.AppConfigurationEntry;
 30 | import javax.security.auth.login.Configuration;
 31 | import javax.security.auth.login.LoginContext;
 32 | import javax.security.auth.login.LoginException;
 33 | import java.io.IOException;
 34 | import java.net.HttpURLConnection;
 35 | import java.net.URL;
 36 | import java.security.PrivilegedActionException;
 37 | import java.security.PrivilegedExceptionAction;
 38 | import java.util.HashMap;
 39 | import java.util.Map;
 40 | 
 41 | /**
 42 |  * The {@link KerberosAuthenticator} implements the Kerberos SPNEGO
 43 |  * authentication sequence.
 44 |  * <p/>
 45 |  * It uses the default principal for the Kerberos cache (normally set via
 46 |  * kinit).
 47 |  * <p/>
 48 |  * It falls back to the {@link PseudoAuthenticator} if the HTTP endpoint does
 49 |  * not trigger an SPNEGO authentication sequence.
 50 |  */
 51 | public class KerberosAuthenticator2 implements Authenticator {
 52 | 
 53 | 	/**
 54 | 	 * HTTP header used by the SPNEGO server endpoint during an authentication
 55 | 	 * sequence.
 56 | 	 */
 57 | 	public static final String WWW_AUTHENTICATE = "WWW-Authenticate";
 58 | 
 59 | 	/**
 60 | 	 * HTTP header used by the SPNEGO client endpoint during an authentication
 61 | 	 * sequence.
 62 | 	 */
 63 | 	public static final String AUTHORIZATION = "Authorization";
 64 | 
 65 | 	/**
 66 | 	 * HTTP header prefix used by the SPNEGO client/server endpoints during an
 67 | 	 * authentication sequence.
 68 | 	 */
 69 | 	public static final String NEGOTIATE = "Negotiate";
 70 | 
 71 | 	private static final String AUTH_HTTP_METHOD = "OPTIONS";
 72 | 
 73 | 	/*
 74 | 	 * Defines the Kerberos configuration that will be used to obtain the
 75 | 	 * Kerberos principal from the Kerberos cache.
 76 | 	 */
 77 | 	private static class KerberosConfiguration extends Configuration {
 78 | 
 79 | 		private static final String OS_LOGIN_MODULE_NAME;
 80 | 		private static final boolean windows = System.getProperty("os.name")
 81 | 				.startsWith("Windows");
 82 | 
 83 | 		static {
 84 | 			if (windows) {
 85 | 				OS_LOGIN_MODULE_NAME = "com.sun.security.auth.module.NTLoginModule";
 86 | 			} else {
 87 | 				OS_LOGIN_MODULE_NAME = "com.sun.security.auth.module.UnixLoginModule";
 88 | 			}
 89 | 		}
 90 | 
 91 | 		private static final AppConfigurationEntry OS_SPECIFIC_LOGIN = new AppConfigurationEntry(
 92 | 				OS_LOGIN_MODULE_NAME,
 93 | 				AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
 94 | 				new HashMap<String, String>());
 95 | 
 96 | 		private static final Map<String, String> USER_KERBEROS_OPTIONS = new HashMap<String, String>();
 97 | 
 98 | 		static {
 99 | 			USER_KERBEROS_OPTIONS.put("doNotPrompt", "true");
100 | 			USER_KERBEROS_OPTIONS.put("useTicketCache", "true");
101 | 			USER_KERBEROS_OPTIONS.put("renewTGT", "true");
102 | 			String ticketCache = System.getenv("KRB5CCNAME");
103 | 			if (ticketCache != null) {
104 | 				USER_KERBEROS_OPTIONS.put("ticketCache", ticketCache);
105 | 			}
106 | 		}
107 | 
108 | 		private static final AppConfigurationEntry USER_KERBEROS_LOGIN = new AppConfigurationEntry(
109 | 				KerberosUtil.getKrb5LoginModuleName(),
110 | 				AppConfigurationEntry.LoginModuleControlFlag.OPTIONAL,
111 | 				USER_KERBEROS_OPTIONS);
112 | 
113 | 		private static final AppConfigurationEntry[] USER_KERBEROS_CONF = new AppConfigurationEntry[] {
114 | 				OS_SPECIFIC_LOGIN, USER_KERBEROS_LOGIN };
115 | 
116 | 		@Override
117 | 		public AppConfigurationEntry[] getAppConfigurationEntry(String appName) {
118 | 			return USER_KERBEROS_CONF;
119 | 		}
120 | 	}
121 | 	private boolean debug = true;
122 | 
123 | 	private static final Logger LOG = LoggerFactory
124 | 			.getLogger(KerberosAuthenticator2.class);
125 | 	
126 | 	
127 | 	private URL url;
128 | 	private HttpURLConnection conn;
129 | 	private Base64 base64;
130 | 
131 | 	
132 | 	private String username;
133 | 	private String password;
134 | 	
135 | 	public KerberosAuthenticator2(String username, String password) {
136 | 		super();
137 | 		this.username = username;
138 | 		this.password = password;
139 | 	}
140 | 
141 | 	@Override
142 | 	public void setConnectionConfigurator(ConnectionConfigurator connectionConfigurator) {
143 | 
144 | 	}
145 | 
146 | 	/**
147 | 	 * Performs SPNEGO authentication against the specified URL.
148 | 	 * <p/>
149 | 	 * If a token is given it does a NOP and returns the given token.
150 | 	 * <p/>
151 | 	 * If no token is given, it will perform the SPNEGO authentication sequence
152 | 	 * using an HTTP <code>OPTIONS</code> request.
153 | 	 * 
154 | 	 * @param url
155 | 	 *            the URl to authenticate against.
156 | 	 * @param token
157 | 	 *            the authentication token being used for the user.
158 | 	 * 
159 | 	 * @throws IOException
160 | 	 *             if an IO error occurred.
161 | 	 * @throws AuthenticationException
162 | 	 *             if an authentication error occurred.
163 | 	 */
164 | 	@Override
165 | 	public void authenticate(URL url, AuthenticatedURL.Token token)
166 | 			throws IOException, AuthenticationException {
167 | 		if (!token.isSet()) {
168 | 			this.url = url;
169 | 			base64 = new Base64(0);
170 | 			conn = (HttpURLConnection) url.openConnection();
171 | 			conn.setRequestMethod(AUTH_HTTP_METHOD);
172 | 			conn.connect();
173 | 			if (isNegotiate()) {
174 | 				doSpnegoSequence(token);
175 | 			} else {
176 | 				getFallBackAuthenticator().authenticate(url, token);
177 | 			}
178 | 		}
179 | 	}
180 | 
181 | 	/**
182 | 	 * If the specified URL does not support SPNEGO authentication, a fallback
183 | 	 * {@link Authenticator} will be used.
184 | 	 * <p/>
185 | 	 * This implementation returns a {@link PseudoAuthenticator}.
186 | 	 * 
187 | 	 * @return the fallback {@link Authenticator}.
188 | 	 */
189 | 	protected Authenticator getFallBackAuthenticator() {
190 | 		return new PseudoAuthenticator();
191 | 	}
192 | 
193 | 	/*
194 | 	 * Indicates if the response is starting a SPNEGO negotiation.
195 | 	 */
196 | 	private boolean isNegotiate() throws IOException {
197 | 		boolean negotiate = false;
198 | 		if (conn.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
199 | 			String authHeader = conn.getHeaderField(WWW_AUTHENTICATE);
200 | 			negotiate = authHeader != null
201 | 					&& authHeader.trim().startsWith(NEGOTIATE);
202 | 		}
203 | 		return negotiate;
204 | 	}
205 | 
206 | 	/**
207 | 	 * Implements the SPNEGO authentication sequence interaction using the
208 | 	 * current default principal in the Kerberos cache (normally set via kinit).
209 | 	 * 
210 | 	 * @param token
211 | 	 *            the authentication token being used for the user.
212 | 	 * 
213 | 	 * @throws IOException
214 | 	 *             if an IO error occurred.
215 | 	 * @throws AuthenticationException
216 | 	 *             if an authentication error occurred.
217 | 	 */
218 | 	private void doSpnegoSequence(AuthenticatedURL.Token token)
219 | 			throws IOException, AuthenticationException {
220 | 		try {
221 | 			
222 | /*		//	
223 |   			AccessControlContext context = AccessController.getContext();
224 | 			Subject subject = Subject.getSubject(context);
225 | 			if (subject == null) {
226 | 				subject = new Subject();
227 | 				LoginContext login = new LoginContext("", subject, null,
228 | 						new KerberosConfiguration());
229 | 				login.login();
230 | 			}
231 | */
232 | 			
233 | 			LoginContext loginContext = new LoginContext("", null,
234 | 					new KerberosClientCallbackHandler(username, password),
235 | 					new LoginConfig(this.debug));
236 | 			loginContext.login();
237 | 			if (LOG.isDebugEnabled()) {
238 | 				LOG.debug("Kerberos authenticated user: "
239 | 						+ loginContext.getSubject());
240 | 			}
241 | 			Subject subject = loginContext.getSubject();
242 | 			
243 | 			Subject.doAs(subject, new PrivilegedExceptionAction<Void>() {
244 | 
245 | 				@Override
246 | 				public Void run() throws Exception {
247 | 					GSSContext gssContext = null;
248 | 					try {
249 | 						GSSManager gssManager = GSSManager.getInstance();
250 | 						String servicePrincipal = "HTTP/"
251 | 								+ KerberosAuthenticator2.this.url.getHost();
252 | 						Oid oid = KerberosUtil
253 | 								.getOidInstance("NT_GSS_KRB5_PRINCIPAL");
254 | 						GSSName serviceName = gssManager.createName(
255 | 								servicePrincipal, oid);
256 | 						oid = KerberosUtil.getOidInstance("GSS_KRB5_MECH_OID");
257 | 						gssContext = gssManager.createContext(serviceName, oid,
258 | 								null, GSSContext.DEFAULT_LIFETIME);
259 | 						gssContext.requestCredDeleg(true);
260 | 						gssContext.requestMutualAuth(true);
261 | 
262 | 						byte[] inToken = new byte[0];
263 | 						byte[] outToken;
264 | 						boolean established = false;
265 | 
266 | 						// Loop while the context is still not established
267 | 						while (!established) {
268 | 							outToken = gssContext.initSecContext(inToken, 0,
269 | 									inToken.length);
270 | 							if (outToken != null) {
271 | 								sendToken(outToken);
272 | 							}
273 | 
274 | 							if (!gssContext.isEstablished()) {
275 | 								inToken = readToken();
276 | 							} else {
277 | 								established = true;
278 | 							}
279 | 						}
280 | 					} finally {
281 | 						if (gssContext != null) {
282 | 							gssContext.dispose();
283 | 							gssContext = null;
284 | 						}
285 | 					}
286 | 					return null;
287 | 				}
288 | 			});
289 | 		} catch (PrivilegedActionException ex) {
290 | 			throw new AuthenticationException(ex.getException());
291 | 		} catch (LoginException ex) {
292 | 			throw new AuthenticationException(ex);
293 | 		}
294 | 		AuthenticatedURL.extractToken(conn, token);
295 | 	}
296 | 
297 | 	/*
298 | 	 * Sends the Kerberos token to the server.
299 | 	 */
300 | 	private void sendToken(byte[] outToken) throws IOException,
301 | 			AuthenticationException {
302 | 		String token = base64.encodeToString(outToken);
303 | 		conn = (HttpURLConnection) url.openConnection();
304 | 		conn.setRequestMethod(AUTH_HTTP_METHOD);
305 | 		conn.setRequestProperty(AUTHORIZATION, NEGOTIATE + " " + token);
306 | 		conn.connect();
307 | 	}
308 | 
309 | 	/*
310 | 	 * Retrieves the Kerberos token returned by the server.
311 | 	 */
312 | 	private byte[] readToken() throws IOException, AuthenticationException {
313 | 		int status = conn.getResponseCode();
314 | 		if (status == HttpURLConnection.HTTP_OK
315 | 				|| status == HttpURLConnection.HTTP_UNAUTHORIZED) {
316 | 			String authHeader = conn.getHeaderField(WWW_AUTHENTICATE);
317 | 			if (authHeader == null || !authHeader.trim().startsWith(NEGOTIATE)) {
318 | 				throw new AuthenticationException("Invalid SPNEGO sequence, '"
319 | 						+ WWW_AUTHENTICATE + "' header incorrect: "
320 | 						+ authHeader);
321 | 			}
322 | 			String negotiation = authHeader.trim()
323 | 					.substring((NEGOTIATE + " ").length()).trim();
324 | 			return base64.decode(negotiation);
325 | 		}
326 | 		throw new AuthenticationException(
327 | 				"Invalid SPNEGO sequence, status code: " + status);
328 | 	}
329 | 
330 | 	
331 | 	
332 | 	public void setDebug(boolean debug) {
333 | 		this.debug = debug;
334 | 	}
335 | 
336 | 	private static class LoginConfig extends Configuration {
337 | 		private boolean debug;
338 | 
339 | 		public LoginConfig(boolean debug) {
340 | 			super();
341 | 			this.debug = debug;
342 | 		}
343 | 
344 | 		@Override
345 | 		public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
346 | 			HashMap<String, String> options = new HashMap<String, String>();
347 | 			options.put("storeKey", "true");
348 | 			if (debug) {
349 | 				options.put("debug", "true");
350 | 			}
351 | 
352 | 			return new AppConfigurationEntry[] { new AppConfigurationEntry(
353 | 					"com.sun.security.auth.module.Krb5LoginModule",
354 | 					AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
355 | 					options), };
356 | 		}
357 | 
358 | 	}
359 | 
360 | 	private static class KerberosClientCallbackHandler implements
361 | 			CallbackHandler {
362 | 		private String username;
363 | 		private String password;
364 | 
365 | 		public KerberosClientCallbackHandler(String username, String password) {
366 | 			this.username = username;
367 | 			this.password = password;
368 | 		}
369 | 
370 | 		@Override
371 | 		public void handle(Callback[] callbacks) throws IOException,
372 | 				UnsupportedCallbackException {
373 | 			for (Callback callback : callbacks) {
374 | 				if (callback instanceof NameCallback) {
375 | 					NameCallback ncb = (NameCallback) callback;
376 | 					ncb.setName(username);
377 | 				} else if (callback instanceof PasswordCallback) {
378 | 					PasswordCallback pwcb = (PasswordCallback) callback;
379 | 					pwcb.setPassword(password.toCharArray());
380 | 				} else {
381 | 					throw new UnsupportedCallbackException(
382 | 							callback,
383 | 							"We got a "
384 | 									+ callback.getClass().getCanonicalName()
385 | 									+ ", but only NameCallback and PasswordCallback is supported");
386 | 				}
387 | 			}
388 | 
389 | 		}
390 | 
391 | 	}
392 | }
393 | 


--------------------------------------------------------------------------------
/src/main/java/org/apache/hadoop/fs/http/client/WebHDFSConnection.java:
--------------------------------------------------------------------------------
  1 | package org.apache.hadoop.fs.http.client;
  2 | 
  3 | import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
  4 | import org.apache.hadoop.security.authentication.client.AuthenticatedURL.Token;
  5 | import org.apache.hadoop.security.authentication.client.AuthenticationException;
  6 | import org.apache.hadoop.security.authentication.client.Authenticator;
  7 | import org.slf4j.Logger;
  8 | import org.slf4j.LoggerFactory;
  9 | 
 10 | import java.io.*;
 11 | import java.net.HttpURLConnection;
 12 | import java.net.MalformedURLException;
 13 | import java.net.URL;
 14 | import java.text.MessageFormat;
 15 | import java.util.Date;
 16 | 
 17 | /**
 18 |  * ===== HTTP GET <br/>
 19 |  * <li>OPEN (see FileSystem.open)
 20 |  * <li>GETFILESTATUS (see FileSystem.getFileStatus)
 21 |  * <li>LISTSTATUS (see FileSystem.listStatus)
 22 |  * <li>GETCONTENTSUMMARY (see FileSystem.getContentSummary)
 23 |  * <li>GETFILECHECKSUM (see FileSystem.getFileChecksum)
 24 |  * <li>GETHOMEDIRECTORY (see FileSystem.getHomeDirectory)
 25 |  * <li>GETDELEGATIONTOKEN (see FileSystem.getDelegationToken)
 26 |  * <li>GETDELEGATIONTOKENS (see FileSystem.getDelegationTokens)
 27 |  * <br/>
 28 |  * ===== HTTP PUT <br/>
 29 |  * <li>CREATE (see FileSystem.create)
 30 |  * <li>MKDIRS (see FileSystem.mkdirs)
 31 |  * <li>CREATESYMLINK (see FileContext.createSymlink)
 32 |  * <li>RENAME (see FileSystem.rename)
 33 |  * <li>SETREPLICATION (see FileSystem.setReplication)
 34 |  * <li>SETOWNER (see FileSystem.setOwner)
 35 |  * <li>SETPERMISSION (see FileSystem.setPermission)
 36 |  * <li>SETTIMES (see FileSystem.setTimes)
 37 |  * <li>RENEWDELEGATIONTOKEN (see FileSystem.renewDelegationToken)
 38 |  * <li>CANCELDELEGATIONTOKEN (see FileSystem.cancelDelegationToken)
 39 |  * <br/>
 40 |  * ===== HTTP POST <br/>
 41 |  * APPEND (see FileSystem.append)
 42 |  * <br/>
 43 |  * ===== HTTP DELETE <br/>
 44 |  * DELETE (see FileSystem.delete)
 45 |  */
 46 | public class WebHDFSConnection {
 47 | 
 48 |     protected static final Logger logger = LoggerFactory.getLogger(WebHDFSConnection.class);
 49 | 
 50 |     private String httpfsUrl = null;
 51 |     private String principal;
 52 |     private String password;
 53 | 
 54 |     private Token token = new AuthenticatedURL.Token();
 55 |     private AuthenticatedURL authenticatedURL;
 56 |     private AuthenticationType authenticationType;
 57 | 
 58 |     public WebHDFSConnection(String httpfsUrl, String principal, String password,
 59 |                              AuthenticationType authenticationType) {
 60 |         this.httpfsUrl = httpfsUrl;
 61 |         this.principal = principal;
 62 |         this.password = password;
 63 |         this.authenticationType = authenticationType;
 64 |         if (this.authenticationType == AuthenticationType.PSEUDO) {
 65 |             this.authenticatedURL = new AuthenticatedURL(new PseudoAuthenticator2(principal));
 66 |         } else {
 67 |             this.authenticatedURL = new AuthenticatedURL(new KerberosAuthenticator2(principal, password));
 68 |         }
 69 |     }
 70 | 
 71 | 
 72 |     protected HttpURLConnection getURLConnection(String uri) throws AuthenticationException,IOException {
 73 |         HttpURLConnection conn;
 74 |         if (authenticationType == AuthenticationType.KERBEROS) {
 75 |             conn = authenticatedURL.openConnection(
 76 |                     new URL(new URL(httpfsUrl), uri), token);
 77 |         } else {
 78 |             String spec = uri + "&user.name=" + principal;
 79 |             conn = authenticatedURL.openConnection(new URL(new URL(httpfsUrl), spec), token);
 80 |         }
 81 |         return conn;
 82 |     }
 83 | 
 84 |     public static synchronized Token generateToken(String srvUrl, String princ, String passwd,
 85 |                                                    AuthenticationType authenticationType) {
 86 |         AuthenticatedURL.Token newToken = new AuthenticatedURL.Token();
 87 | 
 88 |         Authenticator authenticator = null;
 89 |         String spec;
 90 |         if (authenticationType == AuthenticationType.KERBEROS) {
 91 |             authenticator = new KerberosAuthenticator2(princ,passwd);
 92 |             spec = "/webhdfs/v1/?op=GETHOMEDIRECTORY";
 93 |         } else {
 94 |             authenticator = new PseudoAuthenticator2(princ);
 95 |             spec = MessageFormat.format("/webhdfs/v1/?op=GETHOMEDIRECTORY&user.name={0}", princ);
 96 |         }
 97 |         try {
 98 |             HttpURLConnection conn = new AuthenticatedURL(authenticator).openConnection(new URL(new URL(srvUrl), spec), newToken);
 99 |             conn.connect();
100 |             conn.disconnect();
101 |         } catch (Exception ex) {
102 |             logger.error(ex.getMessage());
103 |             logger.error("[" + princ + ":" + passwd + "]@" + srvUrl, ex);
104 |             // WARN
105 |             // throws  IOException,
106 |             // AuthenticationException, InterruptedException
107 |         }
108 | 
109 |         return newToken;
110 |     }
111 | 
112 |     protected static long copy(InputStream input, OutputStream result) throws IOException {
113 |         byte[] buffer = new byte[12288]; // 8K=8192 12K=12288 64K=
114 |         long count = 0L;
115 |         int n = 0;
116 |         while (-1 != (n = input.read(buffer))) {
117 |             result.write(buffer, 0, n);
118 |             count += n;
119 |             result.flush();
120 |         }
121 |         result.flush();
122 |         return count;
123 |     }
124 | 
125 |     private static Response result(HttpURLConnection conn, boolean input) throws IOException {
126 |         StringBuffer sb = new StringBuffer();
127 |         if (input) {
128 |             InputStream is = conn.getInputStream();
129 |             BufferedReader reader = new BufferedReader(new InputStreamReader(is));
130 |             String line = null;
131 | 
132 |             while ((line = reader.readLine()) != null) {
133 |                 sb.append(line);
134 |             }
135 |             reader.close();
136 |             is.close();
137 |         }
138 | 
139 |         Response response = new Response();
140 |         response.setCode(conn.getResponseCode());
141 |         response.setStatus(conn.getResponseMessage());
142 |         response.setContentType(conn.getContentType());
143 |         response.setData(sb.toString());
144 |         return response;
145 |     }
146 | 
147 |     public void ensureValidToken() {
148 |         if (!token.isSet()) { // if token is null
149 |             token = generateToken(httpfsUrl, principal, password,authenticationType);
150 |         } else {
151 |             long currentTime = new Date().getTime();
152 |             long tokenExpired = Long.parseLong(token.toString().split("&")[3].split("=")[1]);
153 |             logger.debug("[currentTime vs. tokenExpired] " + currentTime + " " + tokenExpired);
154 | 
155 |             if (currentTime > tokenExpired) { // if the token is expired
156 |                 token = generateToken(httpfsUrl, principal, password,authenticationType);
157 |             }
158 |         }
159 | 
160 |     }
161 | 
162 | 	/*
163 |      * ========================================================================
164 | 	 * GET
165 | 	 * ========================================================================
166 | 	 */
167 | 
168 |     /**
169 |      * <b>GETHOMEDIRECTORY</b>
170 |      * <p/>
171 |      * curl -i "http://<HOST>:<PORT>/webhdfs/v1/?op=GETHOMEDIRECTORY"
172 |      *
173 |      * @throws MalformedURLException
174 |      * @throws IOException
175 |      * @throws AuthenticationException
176 |      */
177 |     public String getHomeDirectory() throws IOException, AuthenticationException {
178 |         ensureValidToken();
179 |         HttpURLConnection conn = getURLConnection("/webhdfs/v1/?op=GETHOMEDIRECTORY");
180 |         conn.connect();
181 |         Response response = result(conn, true);
182 |         conn.disconnect();
183 |         return response.getData();
184 |     }
185 | 
186 |     /**
187 |      * <b>OPEN</b>
188 |      * <p/>
189 |      * curl -i -L "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=OPEN
190 |      * [&offset=<LONG>][&length=<LONG>][&buffersize=<INT>]"
191 |      *
192 |      * @param path
193 |      * @param os
194 |      * @throws AuthenticationException
195 |      * @throws IOException
196 |      * @throws MalformedURLException
197 |      */
198 |     public String open(String path, OutputStream os) throws IOException, AuthenticationException {
199 |         ensureValidToken();
200 |         HttpURLConnection conn = getURLConnection(MessageFormat.format("/webhdfs/v1/{0}?op=OPEN", URLUtil.encodePath(path)));
201 |         conn.setRequestMethod("GET");
202 |         conn.setRequestProperty("Content-Type", "application/octet-stream");
203 |         conn.connect();
204 |         InputStream is = conn.getInputStream();
205 |         copy(is, os);
206 |         is.close();
207 |         os.close();
208 |         Response resp = result(conn, false);
209 |         conn.disconnect();
210 | 
211 |         return resp.getData();
212 |     }
213 | 
214 |     /**
215 |      * <b>GETCONTENTSUMMARY</b>
216 |      * <p/>
217 |      * curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETCONTENTSUMMARY"
218 |      *
219 |      * @param path
220 |      * @throws MalformedURLException
221 |      * @throws IOException
222 |      * @throws AuthenticationException
223 |      */
224 |     public String getContentSummary(String path) throws IOException, AuthenticationException {
225 |         ensureValidToken();
226 |         HttpURLConnection conn = getURLConnection(MessageFormat.format("/webhdfs/v1/{0}?op=GETCONTENTSUMMARY", URLUtil.encodePath(path)));
227 |         conn.setRequestMethod("GET");
228 |         conn.connect();
229 |         Response resp = result(conn, true);
230 |         conn.disconnect();
231 | 
232 |         return resp.getData();
233 |     }
234 | 
235 |     /**
236 |      * <b>LISTSTATUS</b>
237 |      * <p/>
238 |      * curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=LISTSTATUS"
239 |      *
240 |      * @param path
241 |     
242 |      * @throws MalformedURLException
243 |      * @throws IOException
244 |      * @throws AuthenticationException
245 |      */
246 |     public String listStatus(String path) throws IOException, AuthenticationException {
247 |         ensureValidToken();
248 |         HttpURLConnection conn = getURLConnection(MessageFormat.format("/webhdfs/v1/{0}?op=LISTSTATUS", URLUtil.encodePath(path)));
249 |         conn.setRequestMethod("GET");
250 |         conn.connect();
251 |         Response resp = result(conn, true);
252 |         conn.disconnect();
253 | 
254 |         return resp.getData();
255 |     }
256 | 
257 |     /**
258 |      * <b>GETFILESTATUS</b>
259 |      * <p/>
260 |      * curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETFILESTATUS"
261 |      *
262 |      * @param path
263 |      * @throws MalformedURLException
264 |      * @throws IOException
265 |      * @throws AuthenticationException
266 |      */
267 |     public String getFileStatus(String path) throws IOException, AuthenticationException {
268 |         ensureValidToken();
269 |         HttpURLConnection conn = getURLConnection(MessageFormat.format("/webhdfs/v1/{0}?op=LISTSTATUS", URLUtil.encodePath(path)));
270 |         conn.setRequestMethod("GET");
271 |         conn.connect();
272 |         Response resp = result(conn, true);
273 |         conn.disconnect();
274 | 
275 |         return resp.getData();
276 |     }
277 | 
278 |     /**
279 |      * <b>GETFILECHECKSUM</b>
280 |      * <p/>
281 |      * curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=GETFILECHECKSUM"
282 |      *
283 |      * @param path
284 |      * @throws MalformedURLException
285 |      * @throws IOException
286 |      * @throws AuthenticationException
287 |      */
288 |     public String getFileCheckSum(String path) throws IOException, AuthenticationException {
289 |         ensureValidToken();
290 |         HttpURLConnection conn = getURLConnection(MessageFormat.format("/webhdfs/v1/{0}?op=GETFILECHECKSUM", URLUtil.encodePath(path)));
291 |         conn.setRequestMethod("GET");
292 |         conn.connect();
293 |         Response resp = result(conn, true);
294 |         conn.disconnect();
295 | 
296 |         return resp.getData();
297 |     }
298 | 
299 | 	/*
300 | 	 * ========================================================================
301 | 	 * PUT
302 | 	 * ========================================================================
303 | 	 */
304 | 
305 |     /**
306 |      * <b>CREATE</b>
307 |      * <p/>
308 |      * curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CREATE
309 |      * [&overwrite=<true|false>][&blocksize=<LONG>][&replication=<SHORT>]
310 |      * [&permission=<OCTAL>][&buffersize=<INT>]"
311 |      *
312 |      * @param path
313 |      * @param is
314 |     
315 |      * @throws MalformedURLException
316 |      * @throws IOException
317 |      * @throws AuthenticationException
318 |      */
319 |     public String create(String path, InputStream is) throws IOException,
320 |             AuthenticationException {
321 |         ensureValidToken();
322 |         String redirectUrl = null;
323 |         HttpURLConnection conn = getURLConnection(MessageFormat.format("/webhdfs/v1/{0}?op=CREATE", URLUtil.encodePath(path)));
324 |         conn.setRequestMethod("PUT");
325 |         conn.setInstanceFollowRedirects(false);
326 |         conn.connect();
327 |         logger.debug("Location:" + conn.getHeaderField("Location"));
328 |         Response resp = result(conn, true);
329 |         if (conn.getResponseCode() == 307)
330 |             redirectUrl = conn.getHeaderField("Location");
331 |         conn.disconnect();
332 | 
333 |         if (redirectUrl != null) {
334 |             if(authenticationType == AuthenticationType.KERBEROS) {
335 |                 conn = authenticatedURL.openConnection(new URL(redirectUrl), token);
336 |             } else {
337 |                 conn = (HttpURLConnection) new URL(redirectUrl).openConnection();
338 |             }
339 |             conn.setRequestMethod("PUT");
340 |             conn.setDoOutput(true);
341 |             conn.setDoInput(true);
342 |             conn.setUseCaches(false);
343 |             conn.setRequestProperty("Content-Type", "application/octet-stream");
344 |             // conn.setRequestProperty("Transfer-Encoding", "chunked");
345 |             final int _SIZE = is.available();
346 |             conn.setRequestProperty("Content-Length", "" + _SIZE);
347 |             conn.setFixedLengthStreamingMode(_SIZE);
348 |             conn.connect();
349 |             OutputStream os = conn.getOutputStream();
350 |             copy(is, os);
351 |             // Util.copyStream(is, os);
352 |             is.close();
353 |             os.close();
354 |             resp = result(conn, false);
355 |             conn.disconnect();
356 |         }
357 | 
358 |         return resp.getData();
359 |     }
360 | 
361 |     /**
362 |      * <b>MKDIRS</b>
363 |      * <p/>
364 |      * curl -i -X PUT
365 |      * "http://<HOST>:<PORT>/<PATH>?op=MKDIRS[&permission=<OCTAL>]"
366 |      *
367 |      * @param path
368 |     
369 |      * @throws AuthenticationException
370 |      * @throws IOException
371 |      * @throws MalformedURLException
372 |      */
373 |     public String mkdirs(String path) throws IOException, AuthenticationException {
374 |         ensureValidToken();
375 |         HttpURLConnection conn = getURLConnection(MessageFormat.format("/webhdfs/v1/{0}?op=MKDIRS", URLUtil.encodePath(path)));
376 |         conn.setRequestMethod("PUT");
377 |         conn.connect();
378 |         Response resp = result(conn, true);
379 |         conn.disconnect();
380 | 
381 |         return resp.getData();
382 |     }
383 | 
384 |     /**
385 |      * <b>CREATESYMLINK</b>
386 |      * <p/>
387 |      * curl -i -X PUT "http://<HOST>:<PORT>/<PATH>?op=CREATESYMLINK
388 |      * &destination=<PATH>[&createParent=<true|false>]"
389 |      *
390 |     
391 |      * @throws AuthenticationException
392 |      * @throws IOException
393 |      * @throws MalformedURLException
394 |      */
395 |     public String createSymLink(String srcPath, String destPath) throws IOException,
396 |             AuthenticationException {
397 |         ensureValidToken();
398 |         HttpURLConnection conn = getURLConnection(MessageFormat.format("/webhdfs/v1/{0}?op=CREATESYMLINK&destination={1}",
399 |                 URLUtil.encodePath(srcPath),URLUtil.encodePath(destPath)));
400 |         conn.setRequestMethod("PUT");
401 |         conn.connect();
402 |         Response resp = result(conn, true);
403 |         conn.disconnect();
404 | 
405 |         return resp.getData();
406 |     }
407 | 
408 |     /**
409 |      * <b>RENAME</b>
410 |      * <p/>
411 |      * curl -i -X PUT "http://<HOST>:<PORT>/<PATH>?op=RENAME
412 |      * &destination=<PATH>[&createParent=<true|false>]"
413 |      *
414 |     
415 |      * @throws AuthenticationException
416 |      * @throws IOException
417 |      * @throws MalformedURLException
418 |      */
419 |     public String rename(String srcPath, String destPath) throws IOException,
420 |             AuthenticationException {
421 |         ensureValidToken();
422 |         HttpURLConnection conn = getURLConnection(MessageFormat.format("/webhdfs/v1/{0}?op=RENAME&destination={1}",
423 |                 URLUtil.encodePath(srcPath),URLUtil.encodePath(destPath)));
424 |         conn.setRequestMethod("PUT");
425 |         conn.connect();
426 |         Response resp = result(conn, true);
427 |         conn.disconnect();
428 | 
429 |         return resp.getData();
430 |     }
431 | 
432 |     /**
433 |      * <b>SETPERMISSION</b>
434 |      * <p/>
435 |      * curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETPERMISSION
436 |      * [&permission=<OCTAL>]"
437 |      *
438 |      * @param path
439 |     
440 |      * @throws AuthenticationException
441 |      * @throws IOException
442 |      * @throws MalformedURLException
443 |      */
444 |     public String setPermission(String path) throws IOException, AuthenticationException {
445 |         ensureValidToken();
446 |         HttpURLConnection conn = getURLConnection(MessageFormat.format("/webhdfs/v1/{0}?op=SETPERMISSION", URLUtil.encodePath(path)));
447 |         conn.setRequestMethod("PUT");
448 |         conn.connect();
449 |         Response resp = result(conn, true);
450 |         conn.disconnect();
451 | 
452 |         return resp.getData();
453 |     }
454 | 
455 |     /**
456 |      * <b>SETOWNER</b>
457 |      * <p/>
458 |      * curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETOWNER
459 |      * [&owner=<USER>][&group=<GROUP>]"
460 |      *
461 |      * @param path
462 |     
463 |      * @throws AuthenticationException
464 |      * @throws IOException
465 |      * @throws MalformedURLException
466 |      */
467 |     public String setOwner(String path) throws IOException, AuthenticationException {
468 |         ensureValidToken();
469 |         HttpURLConnection conn = getURLConnection(MessageFormat.format("/webhdfs/v1/{0}?op=SETOWNER", URLUtil.encodePath(path)));
470 |         conn.setRequestMethod("PUT");
471 |         conn.connect();
472 |         Response resp = result(conn, true);
473 |         conn.disconnect();
474 | 
475 |         return resp.getData();
476 |     }
477 | 
478 |     /**
479 |      * <b>SETREPLICATION</b>
480 |      * <p/>
481 |      * curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETREPLICATION
482 |      * [&replication=<SHORT>]"
483 |      *
484 |      * @param path
485 |     
486 |      * @throws AuthenticationException
487 |      * @throws IOException
488 |      * @throws MalformedURLException
489 |      */
490 |     public String setReplication(String path) throws IOException, AuthenticationException {
491 |         ensureValidToken();
492 |         HttpURLConnection conn = getURLConnection(MessageFormat.format("/webhdfs/v1/{0}?op=SETREPLICATION", URLUtil.encodePath(path)));
493 |         conn.setRequestMethod("PUT");
494 |         conn.connect();
495 |         Response resp = result(conn, true);
496 |         conn.disconnect();
497 | 
498 |         return resp.getData();
499 |     }
500 | 
501 |     /**
502 |      * <b>SETTIMES</b>
503 |      * <p/>
504 |      * curl -i -X PUT "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=SETTIMES
505 |      * [&modificationtime=<TIME>][&accesstime=<TIME>]"
506 |      *
507 |      * @param path
508 |     
509 |      * @throws AuthenticationException
510 |      * @throws IOException
511 |      * @throws MalformedURLException
512 |      */
513 |     public String setTimes(String path) throws IOException, AuthenticationException {
514 |         ensureValidToken();
515 |         HttpURLConnection conn = getURLConnection(MessageFormat.format("/webhdfs/v1/{0}?op=SETTIMES", URLUtil.encodePath(path)));
516 |         conn.setRequestMethod("PUT");
517 |         conn.connect();
518 |         Response resp = result(conn, true);
519 |         conn.disconnect();
520 | 
521 |         return resp.getData();
522 |     }
523 | 
524 | 	/*
525 | 	 * ========================================================================
526 | 	 * POST
527 | 	 * ========================================================================
528 | 	 */
529 | 
530 |     /**
531 |      * curl -i -X POST
532 |      * "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=APPEND[&buffersize=<INT>]"
533 |      *
534 |      * @param path
535 |      * @param is
536 |     
537 |      * @throws MalformedURLException
538 |      * @throws IOException
539 |      * @throws AuthenticationException
540 |      */
541 |     public String append(String path, InputStream is) throws IOException,
542 |             AuthenticationException {
543 |         ensureValidToken();
544 |         String redirectUrl = null;
545 |         HttpURLConnection conn = getURLConnection(MessageFormat.format("/webhdfs/v1/{0}?op=APPEND", URLUtil.encodePath(path)));
546 |         conn.setRequestMethod("POST");
547 |         conn.setInstanceFollowRedirects(false);
548 |         conn.connect();
549 |         logger.debug("Location:" + conn.getHeaderField("Location"));
550 |         Response resp = result(conn, true);
551 |         if (conn.getResponseCode() == 307)
552 |             redirectUrl = conn.getHeaderField("Location");
553 |         conn.disconnect();
554 | 
555 |         if (redirectUrl != null) {
556 |             if(authenticationType == AuthenticationType.KERBEROS) {
557 |                 conn = authenticatedURL.openConnection(new URL(redirectUrl), token);
558 |             } else {
559 |                 conn = (HttpURLConnection) new URL(redirectUrl).openConnection();
560 |             }
561 |             conn.setRequestMethod("POST");
562 |             conn.setDoOutput(true);
563 |             conn.setDoInput(true);
564 |             conn.setUseCaches(false);
565 |             conn.setRequestProperty("Content-Type", "application/octet-stream");
566 |             // conn.setRequestProperty("Transfer-Encoding", "chunked");
567 |             final int _SIZE = is.available();
568 |             conn.setRequestProperty("Content-Length", "" + _SIZE);
569 |             conn.setFixedLengthStreamingMode(_SIZE);
570 |             conn.connect();
571 |             OutputStream os = conn.getOutputStream();
572 |             copy(is, os);
573 |             // Util.copyStream(is, os);
574 |             is.close();
575 |             os.close();
576 |             resp = result(conn, true);
577 |             conn.disconnect();
578 |         }
579 | 
580 |         return resp.getData();
581 |     }
582 | 
583 | 	/*
584 | 	 * ========================================================================
585 | 	 * DELETE
586 | 	 * ========================================================================
587 | 	 */
588 | 
589 |     /**
590 |      * <b>DELETE</b>
591 |      * <p/>
592 |      * curl -i -X DELETE "http://<host>:<port>/webhdfs/v1/<path>?op=DELETE
593 |      * [&recursive=<true|false>]"
594 |      *
595 |      * @param path
596 |     
597 |      * @throws AuthenticationException
598 |      * @throws IOException
599 |      * @throws MalformedURLException
600 |      */
601 |     public String delete(String path) throws IOException, AuthenticationException {
602 |         ensureValidToken();
603 |         HttpURLConnection conn = getURLConnection(MessageFormat.format("/webhdfs/v1/{0}?op=DELETE", URLUtil.encodePath(path)));
604 |         conn.setRequestMethod("DELETE");
605 |         conn.setInstanceFollowRedirects(false);
606 |         conn.connect();
607 |         Response resp = result(conn, true);
608 |         conn.disconnect();
609 | 
610 |         return resp.getData();
611 |     }
612 | 
613 |     // Begin Getter & Setter
614 |     public String getHttpfsUrl() {
615 |         return httpfsUrl;
616 |     }
617 | 
618 |     public void setHttpfsUrl(String httpfsUrl) {
619 |         this.httpfsUrl = httpfsUrl;
620 |     }
621 | 
622 |     public String getPrincipal() {
623 |         return principal;
624 |     }
625 | 
626 |     public void setPrincipal(String principal) {
627 |         this.principal = principal;
628 |     }
629 | 
630 |     public String getPassword() {
631 |         return password;
632 |     }
633 | 
634 |     public void setPassword(String password) {
635 |         this.password = password;
636 |     }
637 | 
638 |     // End Getter & Setter
639 | }
640 | 


--------------------------------------------------------------------------------