├── log.lua ├── README.md ├── common.lua ├── logging.ini ├── example.conf ├── .gitignore ├── init.lua ├── init_worker.lua ├── index_template ├── inifile.lua ├── pump_data.pl ├── functions.lua └── LICENSE /log.lua: -------------------------------------------------------------------------------- 1 | handlers.log() -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # nginx-log-elasticsearch-lua 2 | Direct logging to Elasticsearch from Nginx using Lua 3 | -------------------------------------------------------------------------------- /common.lua: -------------------------------------------------------------------------------- 1 | module("common", package.seeall) 2 | 3 | function script_path() 4 | local str = debug.getinfo(2, "S").source:sub(2) 5 | return str:match("(.*/)") 6 | end -------------------------------------------------------------------------------- /logging.ini: -------------------------------------------------------------------------------- 1 | ; ini file, make comments with semicolons 2 | [elasticsearch] 3 | ; logging method, can be 'none', 'buffered', or 'direct' 4 | logging_method = buffered 5 | host = 172.17.8.101 6 | bulk_port = 9200 7 | bulk_uri = /_bulk 8 | bulk_scheme = http 9 | bulk_timeout = 5000 10 | ; if buffered method is used, send out logs in this many seconds 11 | send_interval = 5 -------------------------------------------------------------------------------- /example.conf: -------------------------------------------------------------------------------- 1 | lua_shared_dict log_buffer 10m; 2 | lua_shared_dict log_timer 200k; 3 | init_by_lua_file /usr/local/openresty/ngxinx/conf/lua/init.lua; 4 | init_worker_by_lua_file /usr/local/openresty/ngxinx/conf/lua/init_worker.lua; 5 | 6 | server { 7 | listen 80 default_server ; 8 | server_name .examplehost.vbox; 9 | 10 | location / { 11 | root /var/www/nginx; 12 | log_by_lua_file /usr/local/openresty/ngxinx/conf/lua/log.lua; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Lua sources 2 | luac.out 3 | 4 | # luarocks build files 5 | *.src.rock 6 | *.zip 7 | *.tar.gz 8 | 9 | # Object files 10 | *.o 11 | *.os 12 | *.ko 13 | *.obj 14 | *.elf 15 | 16 | # Precompiled Headers 17 | *.gch 18 | *.pch 19 | 20 | # Libraries 21 | *.lib 22 | *.a 23 | *.la 24 | *.lo 25 | *.def 26 | *.exp 27 | 28 | # Shared objects (inc. Windows DLLs) 29 | *.dll 30 | *.so 31 | *.so.* 32 | *.dylib 33 | 34 | # Executables 35 | *.exe 36 | *.out 37 | *.app 38 | *.i*86 39 | *.x86_64 40 | *.hex 41 | 42 | sftp-config.json 43 | 44 | -------------------------------------------------------------------------------- /init.lua: -------------------------------------------------------------------------------- 1 | -- clear the buffer on every restart 2 | ngx.shared.log_buffer:flush_all() 3 | ngx.shared.log_timer:flush_all() 4 | 5 | function script_path() 6 | local str = debug.getinfo(2, "S").source:sub(2) 7 | return str:match("(.*/)") 8 | end 9 | 10 | package.path = script_path() .. '?.lua;' .. package.path 11 | local inifile = require 'inifile' 12 | 13 | -- read the config file, save it 14 | -- inifile module from http://santos.nfshost.com/inifile.html 15 | -- https://github.com/hahawoo/love-misc-libs/blob/master/inifile/inifile.lua 16 | --local inifile = assert(loadfile(script_path() .. 'inifile.lua'), "Can't load inifile.lua") 17 | log_settings = assert(inifile.parse(script_path() .. "logging.ini"), "Can't find logging configuration") 18 | 19 | make_handler = assert(loadfile(script_path() .. 'functions.lua'), "Can't load functions.lua") 20 | -------------------------------------------------------------------------------- /init_worker.lua: -------------------------------------------------------------------------------- 1 | local log_timer = ngx.shared.log_timer 2 | 3 | -- create a timer if required 4 | local function create_timer (callback, delay, timer_name) 5 | 6 | local handler 7 | handler = function () 8 | -- get lock 9 | local my_pid = ngx.worker.pid() 10 | local success, err = log_timer:safe_add(timer_name, 1) 11 | if success then 12 | callback() 13 | log_timer:delete(timer_name) 14 | end 15 | local ok, err = ngx.timer.at(delay, handler) 16 | if not ok then 17 | ngx.log(ngx.ERR, "failed to create the timer: ", err) 18 | return 19 | end 20 | end 21 | 22 | local ok, err = ngx.timer.at(delay, handler) 23 | if not ok then 24 | ngx.log(ngx.ERR, "failed to create the timer: ", err) 25 | return 26 | end 27 | end 28 | 29 | -- MAIN 30 | handlers = {} 31 | -- create the log handler 32 | local handler = make_handler(log_settings) 33 | handlers["log"] = handler.log_request 34 | handlers["send_logs"] = handler.send_logs 35 | 36 | if log_settings.elasticsearch.logging_method == "buffered" then 37 | create_timer(handlers.send_logs, log_settings.elasticsearch.send_interval, "send_logs") 38 | end 39 | 40 | handler = nil -------------------------------------------------------------------------------- /index_template: -------------------------------------------------------------------------------- 1 | { 2 | "template" : "INDEXTEMPLATE", 3 | "settings" : { 4 | "index.refresh_interval" : "5s", 5 | "number_of_shards" : 5, 6 | "number_of_replicas" : 1, 7 | "analysis" : { 8 | "analyzer" : { 9 | "default" : { 10 | "type" : "standard", 11 | "stopwords" : "_none_" 12 | } 13 | } 14 | } 15 | }, 16 | "mappings" : { 17 | "INDEXTEMPLATE" : { 18 | "_all" : {"enabled" : false}, 19 | "dynamic_templates" : [ { 20 | "string_fields" : { 21 | "match" : "*", 22 | "match_mapping_type" : "string", 23 | "mapping" : { 24 | "type" : "multi_field", 25 | "fields" : { 26 | "{name}" : {"type": "string", "index" : "analyzed", "omit_norms" : true, "index_options" : "docs"}, 27 | "{name}.raw" : {"type": "string", "index" : "not_analyzed", "ignore_above" : 256 , "doc_values": true} 28 | } 29 | } 30 | } 31 | } ], 32 | "properties" : { 33 | "@version": { "type": "string", "index": "not_analyzed" }, 34 | 35 | "@timestamp" : { "format" : "dateOptionalTime", "type" : "date" }, 36 | "remote_addr" : { "type" : "string", "index" : "not_analyzed" , "doc_values": true}, 37 | "remote_user" : { "type": "string", "index" : "not_analyzed" , "doc_values": true}, 38 | "request" : { "type" : "string", "index" : "not_analyzed" , "doc_values": true}, 39 | "status" : { "type" : "long", "index" : "not_analyzed", "doc_values": true}, 40 | "body_bytes_sent" : { "type" : "double", "index" : "analyzed" }, 41 | "http_referer" : { "type" : "string", "index" : "not_analyzed" , "doc_values": true}, 42 | "http_user_agent" : { "type" : "string", "index" : "not_analyzed" , "doc_values": true}, 43 | "http_x_forwarded_for" : { "type" : "string", "index" : "not_analyzed" , "doc_values": true}, 44 | "request_time" : { "type" : "double", "index" : "analyzed" }, 45 | "server_name": { "type" : "string", "index" : "not_analyzed" , "doc_values": true}, 46 | "request_method" : { "type" : "string", "index" : "not_analyzed" , "doc_values": true}, 47 | "http_protocol" : { "type" : "string", "index" : "not_analyzed" , "doc_values": true}, 48 | "hostname" : { "type" : "string", "index" : "not_analyzed" , "doc_values": true}, 49 | "connection_serial_id" : { "type" : "integer" , "index" : "not_analyzed" , "doc_values": true}, 50 | "connection_requests" : { "type" : "integer" , "index" : "not_analyzed", "doc_values": true } 51 | } 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /inifile.lua: -------------------------------------------------------------------------------- 1 | -- Copyright 2011 Bart van Strien. All rights reserved. 2 | -- 3 | -- Redistribution and use in source and binary forms, with or without modification, are 4 | -- permitted provided that the following conditions are met: 5 | -- 6 | -- 1. Redistributions of source code must retain the above copyright notice, this list of 7 | -- conditions and the following disclaimer. 8 | -- 9 | -- 2. Redistributions in binary form must reproduce the above copyright notice, this list 10 | -- of conditions and the following disclaimer in the documentation and/or other materials 11 | -- provided with the distribution. 12 | -- 13 | -- THIS SOFTWARE IS PROVIDED BY BART VAN STRIEN ''AS IS'' AND ANY EXPRESS OR IMPLIED 14 | -- WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 15 | -- FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL BART VAN STRIEN OR 16 | -- CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 17 | -- CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 18 | -- SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 19 | -- ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 20 | -- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 21 | -- ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 22 | -- 23 | -- The views and conclusions contained in the software and documentation are those of the 24 | -- authors and should not be interpreted as representing official policies, either expressed 25 | -- or implied, of Bart van Strien. 26 | -- 27 | -- The above license is known as the Simplified BSD license. 28 | 29 | inifile = {} 30 | 31 | local lines 32 | local write 33 | 34 | if love then 35 | lines = love.filesystem.lines 36 | write = love.filesystem.write 37 | else 38 | lines = function(name) return assert(io.open(name)):lines() end 39 | write = function(name, contents) return assert(io.open(name, "w")):write(contents) end 40 | end 41 | 42 | function inifile.parse(name) 43 | local t = {} 44 | local section 45 | for line in lines(name) do 46 | local s = line:match("^%[([^%]]+)%]$") 47 | if s then 48 | section = s 49 | t[section] = t[section] or {} 50 | end 51 | local key, value = line:match("^%s*([^%s]+)%s*=%s*(.+)%s*$") 52 | if key and value then 53 | if tonumber(value) then value = tonumber(value) end 54 | if value == "true" then value = true end 55 | if value == "false" then value = false end 56 | t[section][key] = value 57 | end 58 | end 59 | return t 60 | end 61 | 62 | function inifile.save(name, t) 63 | local contents = "" 64 | for section, s in pairs(t) do 65 | contents = contents .. ("[%s]\n"):format(section) 66 | for key, value in pairs(s) do 67 | contents = contents .. ("%s=%s\n"):format(key, tostring(value)) 68 | end 69 | contents = contents .. "\n" 70 | end 71 | write(name, contents) 72 | end 73 | 74 | return inifile -------------------------------------------------------------------------------- /pump_data.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | 3 | # Flushing to STDOUT after each write 4 | $| = 1; 5 | 6 | use strict; 7 | use warnings; 8 | use LWP::Simple; 9 | use Getopt::Std; 10 | use vars qw/ %opt /; 11 | 12 | sub init() 13 | { 14 | my $opt_string = 'i:t:h:j:c'; 15 | getopts( "$opt_string", \%opt ) or usage(); 16 | usage() if defined $opt{h}; 17 | 18 | my $INDEX; 19 | my $url; 20 | my $TEMPLATE; 21 | 22 | if ( scalar keys(%opt) > 0 ) { 23 | 24 | if($opt{i}) { 25 | $INDEX = $opt{i}; 26 | } 27 | if($opt{t}) { 28 | $url = "http://".$opt{t}.":9200"; 29 | } 30 | if($opt{c}) { 31 | 32 | usage() if not defined $opt{i}; 33 | 34 | if($opt{j}) { 35 | $TEMPLATE = $opt{j}; 36 | } 37 | 38 | my $templatedata; 39 | open TEMPLATE , "<", $TEMPLATE or die $?; 40 | while (