├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── LICENSE ├── .gitignore ├── gradlew.bat ├── http.rb ├── README.md └── gradlew /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/moshen/logstash-http-input/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Tue Apr 01 16:22:15 CDT 2014 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=http\://services.gradle.org/distributions/gradle-1.11-bin.zip 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2014 Colin Kennedy 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | #### Gradle 3 | 4 | 5 | .gradle 6 | build/ 7 | 8 | 9 | #### Vim 10 | 11 | 12 | [._]*.s[a-w][a-z] 13 | [._]s[a-w][a-z] 14 | *.un~ 15 | Session.vim 16 | .netrwhist 17 | *~ 18 | 19 | 20 | #### JetBrains 21 | 22 | 23 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm 24 | 25 | ## Directory-based project format 26 | .idea/ 27 | # if you remove the above rule, at least ignore user-specific stuff: 28 | # .idea/workspace.xml 29 | # .idea/tasks.xml 30 | # and these sensitive or high-churn files: 31 | # .idea/dataSources.ids 32 | # .idea/dataSources.xml 33 | # .idea/sqlDataSources.xml 34 | # .idea/dynamic.xml 35 | 36 | ## File-based project format 37 | *.ipr 38 | *.iws 39 | *.iml 40 | 41 | ## Additional for IntelliJ 42 | out/ 43 | 44 | # generated by mpeltonen/sbt-idea plugin 45 | .idea_modules/ 46 | 47 | # generated by JIRA plugin 48 | atlassian-ide-plugin.xml 49 | 50 | # generated by Crashlytics plugin (for Android Studio and Intellij) 51 | com_crashlytics_export_strings.xml 52 | 53 | 54 | #### Eclipse 55 | 56 | 57 | *.pydevproject 58 | .metadata 59 | .gradle 60 | bin/ 61 | tmp/ 62 | *.tmp 63 | *.bak 64 | *.swp 65 | *~.nib 66 | local.properties 67 | .settings/ 68 | .loadpath 69 | 70 | # External tool builders 71 | .externalToolBuilders/ 72 | 73 | # Locally stored "Eclipse launch configurations" 74 | *.launch 75 | 76 | # CDT-specific 77 | .cproject 78 | 79 | # PDT-specific 80 | .buildpath 81 | 82 | # sbteclipse plugin 83 | .target 84 | 85 | # TeXlipse plugin 86 | .texlipse 87 | 88 | 89 | #### NetBeans 90 | 91 | 92 | nbproject/private/ 93 | build/ 94 | nbbuild/ 95 | dist/ 96 | nbdist/ 97 | nbactions.xml 98 | nb-configuration.xml 99 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 12 | set DEFAULT_JVM_OPTS= 13 | 14 | set DIRNAME=%~dp0 15 | if "%DIRNAME%" == "" set DIRNAME=. 16 | set APP_BASE_NAME=%~n0 17 | set APP_HOME=%DIRNAME% 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windowz variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | if "%@eval[2+2]" == "4" goto 4NT_args 53 | 54 | :win9xME_args 55 | @rem Slurp the command line arguments. 56 | set CMD_LINE_ARGS= 57 | set _SKIP=2 58 | 59 | :win9xME_args_slurp 60 | if "x%~1" == "x" goto execute 61 | 62 | set CMD_LINE_ARGS=%* 63 | goto execute 64 | 65 | :4NT_args 66 | @rem Get arguments from the 4NT Shell from JP Software 67 | set CMD_LINE_ARGS=%$ 68 | 69 | :execute 70 | @rem Setup the command line 71 | 72 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if "%ERRORLEVEL%"=="0" goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 85 | exit /b 1 86 | 87 | :mainEnd 88 | if "%OS%"=="Windows_NT" endlocal 89 | 90 | :omega 91 | -------------------------------------------------------------------------------- /http.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require "logstash/inputs/base" 3 | require "logstash/namespace" 4 | 5 | require 'java' 6 | # Load Jetty Dependencies 7 | jarpath = File.join(File.dirname(__FILE__), 8 | "../../../vendor/jar/jetty-@version@/*.jar") 9 | Dir.glob(jarpath).each do |jar| 10 | require jar 11 | end 12 | 13 | # Read events PUT or POSTED to an http server. 14 | # 15 | # Each PUT or POST is assumed to be an event by default. 16 | class LogStash::Inputs::Http < LogStash::Inputs::Base 17 | class Interrupted < StandardError; end 18 | config_name "http" 19 | milestone 0 20 | 21 | default :codec, "plain" 22 | 23 | # The address to listen on. 24 | config :host, :validate => :string, :default => "0.0.0.0" 25 | 26 | # The port to listen on. 27 | config :port, :validate => :number, :required => true 28 | 29 | # Max form content size in bytes. Set to -1 to disable. 30 | config :maxFormSize, :validate => :number, :default => 200000 31 | 32 | # Jetty Server Connector acceptQueueSize. 0 uses implementation default. 33 | config :acceptQueueSize, :validate => :number, :default => 0 34 | 35 | def initialize(*args) 36 | super(*args) 37 | end # def initialize 38 | 39 | public 40 | def register 41 | # I don't think we need to do anything here 42 | end # def register 43 | 44 | class LogHandler < org.eclipse.jetty.server.handler.AbstractHandler 45 | def handle(target, baseRequest, httpRequest, httpResponse) 46 | case httpRequest.getMethod() 47 | when 'PUT', 'POST' 48 | setKeepAlive(httpRequest, httpResponse) 49 | httpResponse.setStatus(200) 50 | 51 | # check for Content-Encoding 52 | if httpRequest.getHeader("Content-Encoding") == "gzip" 53 | scanner = Java::java.util.Scanner.new( 54 | Java::java.util.zip.GZIPInputStream.new(httpRequest.getInputStream()), "UTF-8") 55 | .useDelimiter("\\A") 56 | else 57 | scanner = Java::java.util.Scanner.new(httpRequest.getInputStream(), "UTF-8") 58 | .useDelimiter("\\A") 59 | end 60 | 61 | if scanner.hasNext() 62 | @codec.clone.decode(scanner.next()) do |event| 63 | @parent.decorate(event) 64 | if !event.include?("host") || event["host"].empty? 65 | event["host"] = httpRequest.getRemoteAddr() 66 | end 67 | @output_queue << event 68 | end 69 | end 70 | 71 | when 'HEAD', 'GET' 72 | if setKeepAlive(httpRequest, httpResponse) 73 | httpResponse.setStatus(200) 74 | else 75 | httpResponse.setStatus(501) 76 | end 77 | 78 | else 79 | httpResponse.setStatus(501) 80 | end 81 | 82 | baseRequest.setHandled(true) 83 | end 84 | 85 | def setKeepAlive(httpRequest, httpResponse) 86 | ka = httpRequest.getHeader('Connection') 87 | if ka && ka.downcase == 'keep-alive' 88 | httpResponse.addHeader('Connection', 'Keep-Alive') 89 | return true 90 | else 91 | return false 92 | end 93 | end 94 | 95 | def setupInput(parent, output_queue) 96 | @parent = parent 97 | @codec = parent.instance_variable_get(:@codec) 98 | @output_queue = output_queue 99 | end 100 | end 101 | 102 | public 103 | def decorate(event) 104 | super(event) 105 | end 106 | 107 | public 108 | def run(output_queue) 109 | @server = Java::org.eclipse.jetty.server.Server.new( 110 | Java::java.net.InetSocketAddress.new(@host, @port) 111 | ) 112 | @server.setAttribute( 113 | 'org.eclipse.jetty.server.Request.maxFormContentSize', 114 | @maxFormSize 115 | ) 116 | 117 | @server.getConnectors()[0].setAcceptQueueSize(@acceptQueueSize) 118 | 119 | handler = LogHandler.new 120 | handler.setupInput(self, output_queue) 121 | @server.setHandler(handler) 122 | @server.start 123 | @server.join 124 | end # def run 125 | 126 | public 127 | def teardown 128 | @server.doStop 129 | end # def teardown 130 | end # class LogStash::Inputs::Http 131 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## An HTTP input for Logstash <= 1.4.x (using Jetty) 2 | 3 | ## NOTICE 4 | 5 | [Logstash 1.5.0 is introducing an official http input](https://github.com/logstash-plugins/logstash-input-http). 6 | Considering this, and the fact that I'm no longer using this plugin in my own 7 | work; I'm recommending >= 1.5.x users should switch to the official plugin. 8 | 9 | ### About 10 | 11 | Instantiates Jetty on the specified host and port, and applies the configured 12 | codec (`plain` by default) to the `PUT` or `POST` request body. 13 | 14 | ### Build / Fetch dependencies 15 | 16 | Run `./gradlew` (or gradlew.bat) to assemble. 17 | 18 | This will download the dependencies, add the right jetty version to `http.rb` 19 | and put `http.rb` in a (somewhat proper) plugin folder: 20 | `build/plugins/logstash/inputs/`. 21 | 22 | ### Installation 23 | 24 | Run `./gradlew install -PinstallDir=dirToCopyTheContentsOfBuildTo` to install. 25 | 26 | This will do everything in assemble, but also copy the files to a destination 27 | of your choosing. You probably want this to be your logstash install dir. 28 | 29 | ### Configuration 30 | 31 | The default configuration as represented in the logstash configuration file 32 | format: 33 | 34 | input { 35 | http { 36 | host => "0.0.0.0" 37 | maxFormSize => 200000 38 | acceptQueueSize => 0 39 | } 40 | } 41 | 42 | **host** - The address to listen on. Default: `"0.0.0.0"` 43 | 44 | **port** - The port to listen on. No Default, **required** 45 | 46 | **maxFormSize** - [The maximum form size for Jetty in bytes](http://www.eclipse.org/jetty/documentation/current/setting-form-size.html). 47 | Default `200000` (set to `-1` for no max size) 48 | 49 | **acceptQueueSize** - The accept queue size for the default server connector. 50 | Default `0` (Which defers to the implementation default, currently `50`) 51 | 52 | ### Example 53 | 54 | If I have the logstash tarball extracted to `../logstash-1.4.2`, I can simply 55 | run: `./gradlew install -PinstallDir=../logstash-1.4.2/` to install the http 56 | input. 57 | 58 | To test: 59 | 60 | cd ../logstash-1.4.2 61 | bin/logstash -p plugins -e 'input { http { port => 8090 } } output { stdout { codec => rubydebug } }' 62 | 63 | Which should show something like: 64 | 65 | 2014-04-01 16:53:38.805:INFO::Ruby-0-Thread-3: ../logstash-1.4.2/vendor/bundle/jruby/1.9/gems/stud-0.0.17/lib/stud/task.rb:10: Logging initialized @4347ms 66 | Using milestone 0 input plugin 'http'. This plugin isn't well supported by the community and likely has no maintainer. For more information on plugin milestones, see http://logstash.net/docs/1.4.2/plugin-milestones {:level=>:warn} 67 | 2014-04-01 16:53:43.942:INFO:oejs.Server: "Hello Logstash!", 79 | "@version" => "1", 80 | "@timestamp" => "2014-05-18T22:22:14.064Z", 81 | "host" => "0:0:0:0:0:0:0:1" 82 | } 83 | 84 | ### Tips / Performance Considerations 85 | 86 | Can quickly become overloaded if sending single log messages in each http 87 | request. Recommended usage is to use `Keep-Alive` and combine several 88 | newline-delimited log messages in each `POST` with the 89 | [`line`](http://logstash.net/docs/1.4.2/codecs/line) or 90 | [`json_lines`](http://logstash.net/docs/1.4.2/codecs/json_lines) codecs. 91 | 92 | ### Issues 93 | 94 | Doesn't override the Jetty default logging and Jetty just goes ahead and spits 95 | out logs to `STDERR`. I've found this useful for debugging, but may not be the 96 | desired behavior. 97 | 98 | Currently doesn't expose configuration of the Jetty thread pool options. This 99 | will probably change in the future. 100 | 101 | ### License 102 | 103 | Apache v2, See LICENSE file 104 | 105 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 10 | DEFAULT_JVM_OPTS="" 11 | 12 | APP_NAME="Gradle" 13 | APP_BASE_NAME=`basename "$0"` 14 | 15 | # Use the maximum available, or set MAX_FD != -1 to use that value. 16 | MAX_FD="maximum" 17 | 18 | warn ( ) { 19 | echo "$*" 20 | } 21 | 22 | die ( ) { 23 | echo 24 | echo "$*" 25 | echo 26 | exit 1 27 | } 28 | 29 | # OS specific support (must be 'true' or 'false'). 30 | cygwin=false 31 | msys=false 32 | darwin=false 33 | case "`uname`" in 34 | CYGWIN* ) 35 | cygwin=true 36 | ;; 37 | Darwin* ) 38 | darwin=true 39 | ;; 40 | MINGW* ) 41 | msys=true 42 | ;; 43 | esac 44 | 45 | # For Cygwin, ensure paths are in UNIX format before anything is touched. 46 | if $cygwin ; then 47 | [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 48 | fi 49 | 50 | # Attempt to set APP_HOME 51 | # Resolve links: $0 may be a link 52 | PRG="$0" 53 | # Need this for relative symlinks. 54 | while [ -h "$PRG" ] ; do 55 | ls=`ls -ld "$PRG"` 56 | link=`expr "$ls" : '.*-> \(.*\)$'` 57 | if expr "$link" : '/.*' > /dev/null; then 58 | PRG="$link" 59 | else 60 | PRG=`dirname "$PRG"`"/$link" 61 | fi 62 | done 63 | SAVED="`pwd`" 64 | cd "`dirname \"$PRG\"`/" >&- 65 | APP_HOME="`pwd -P`" 66 | cd "$SAVED" >&- 67 | 68 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 69 | 70 | # Determine the Java command to use to start the JVM. 71 | if [ -n "$JAVA_HOME" ] ; then 72 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 73 | # IBM's JDK on AIX uses strange locations for the executables 74 | JAVACMD="$JAVA_HOME/jre/sh/java" 75 | else 76 | JAVACMD="$JAVA_HOME/bin/java" 77 | fi 78 | if [ ! -x "$JAVACMD" ] ; then 79 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 80 | 81 | Please set the JAVA_HOME variable in your environment to match the 82 | location of your Java installation." 83 | fi 84 | else 85 | JAVACMD="java" 86 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 87 | 88 | Please set the JAVA_HOME variable in your environment to match the 89 | location of your Java installation." 90 | fi 91 | 92 | # Increase the maximum file descriptors if we can. 93 | if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then 94 | MAX_FD_LIMIT=`ulimit -H -n` 95 | if [ $? -eq 0 ] ; then 96 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 97 | MAX_FD="$MAX_FD_LIMIT" 98 | fi 99 | ulimit -n $MAX_FD 100 | if [ $? -ne 0 ] ; then 101 | warn "Could not set maximum file descriptor limit: $MAX_FD" 102 | fi 103 | else 104 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 105 | fi 106 | fi 107 | 108 | # For Darwin, add options to specify how the application appears in the dock 109 | if $darwin; then 110 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 111 | fi 112 | 113 | # For Cygwin, switch paths to Windows format before running java 114 | if $cygwin ; then 115 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 116 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules 158 | function splitJvmOpts() { 159 | JVM_OPTS=("$@") 160 | } 161 | eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS 162 | JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" 163 | 164 | exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" 165 | --------------------------------------------------------------------------------