├── Gemfile ├── .gitignore ├── Rakefile ├── spec └── codecs │ ├── sflow │ ├── tcp.dat │ ├── udp.dat │ ├── ipv4_tcp_header.dat │ ├── ethernet_ipv4_udp_header.dat │ ├── ethernet_vlan_ipv4_tcp_header.dat │ └── packet_header_spec.rb │ ├── sflow_flow_sample.dat │ ├── sflow_counters_sample.dat │ ├── sflow_1_counters_sample.dat │ ├── sflow_with_lag_counters.dat │ ├── sflow_flow_sample_eth_vlan.dat │ └── sflow_spec.rb ├── DEVELOPER.md ├── NOTICE.TXT ├── CHANGELOG.md ├── CONTRIBUTORS.txt ├── LICENSE ├── lib └── logstash │ └── codecs │ ├── sflow │ ├── datagram.rb │ ├── util.rb │ ├── packet_header.rb │ ├── sample.rb │ ├── flow_record.rb │ └── counter_record.rb │ ├── snmp │ └── interface_resolver.rb │ └── sflow.rb ├── logstash-codec-sflow.gemspec └── README.md /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | Gemfile.lock 3 | .bundle 4 | .idea 5 | sample 6 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | @files=[] 2 | 3 | task :default do 4 | system("rake -T") 5 | end 6 | 7 | require "logstash/devutils/rake" 8 | -------------------------------------------------------------------------------- /spec/codecs/sflow/tcp.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/path-network/logstash-codec-sflow/HEAD/spec/codecs/sflow/tcp.dat -------------------------------------------------------------------------------- /spec/codecs/sflow/udp.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/path-network/logstash-codec-sflow/HEAD/spec/codecs/sflow/udp.dat -------------------------------------------------------------------------------- /DEVELOPER.md: -------------------------------------------------------------------------------- 1 | # logstash-codec-example 2 | Example codec plugin. This should help bootstrap your effort to write your own codec plugin! -------------------------------------------------------------------------------- /spec/codecs/sflow_flow_sample.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/path-network/logstash-codec-sflow/HEAD/spec/codecs/sflow_flow_sample.dat -------------------------------------------------------------------------------- /spec/codecs/sflow/ipv4_tcp_header.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/path-network/logstash-codec-sflow/HEAD/spec/codecs/sflow/ipv4_tcp_header.dat -------------------------------------------------------------------------------- /spec/codecs/sflow_counters_sample.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/path-network/logstash-codec-sflow/HEAD/spec/codecs/sflow_counters_sample.dat -------------------------------------------------------------------------------- /spec/codecs/sflow_1_counters_sample.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/path-network/logstash-codec-sflow/HEAD/spec/codecs/sflow_1_counters_sample.dat -------------------------------------------------------------------------------- /spec/codecs/sflow_with_lag_counters.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/path-network/logstash-codec-sflow/HEAD/spec/codecs/sflow_with_lag_counters.dat -------------------------------------------------------------------------------- /spec/codecs/sflow_flow_sample_eth_vlan.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/path-network/logstash-codec-sflow/HEAD/spec/codecs/sflow_flow_sample_eth_vlan.dat -------------------------------------------------------------------------------- /spec/codecs/sflow/ethernet_ipv4_udp_header.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/path-network/logstash-codec-sflow/HEAD/spec/codecs/sflow/ethernet_ipv4_udp_header.dat -------------------------------------------------------------------------------- /spec/codecs/sflow/ethernet_vlan_ipv4_tcp_header.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/path-network/logstash-codec-sflow/HEAD/spec/codecs/sflow/ethernet_vlan_ipv4_tcp_header.dat -------------------------------------------------------------------------------- /NOTICE.TXT: -------------------------------------------------------------------------------- 1 | Elasticsearch 2 | Copyright 2012-2015 Elasticsearch 3 | 4 | This product includes software developed by The Apache Software 5 | Foundation (http://www.apache.org/). -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # logstash-codec-sflow Changelog 2 | 3 | ## Version 2.0.0 (2016-11-02) 4 | 5 | * Update for LS 5.0 6 | 7 | ## Version 1.2.1 (2016-11-02) 8 | 9 | * Add management of LAG counters: http://www.sflow.org/sflow_lag.txt -------------------------------------------------------------------------------- /CONTRIBUTORS.txt: -------------------------------------------------------------------------------- 1 | The following is a list of people who have contributed ideas, code, bug 2 | reports, or in general have helped logstash along its way. 3 | 4 | Contributors: 5 | * Nicolas Fraison (ashangit) 6 | 7 | Note: If you've sent us patches, bug reports, or otherwise contributed to 8 | Logstash, and you aren't on the list above and want to be, please let us know 9 | and we'll make sure you're here. Contributions from folks like you are what make 10 | open source awesome. 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012–2015 Elasticsearch 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /lib/logstash/codecs/sflow/datagram.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'bindata' 4 | require 'logstash/codecs/sflow/util' 5 | require 'logstash/codecs/sflow/sample' 6 | 7 | 8 | class SFlowHeader < BinData::Record 9 | endian :big 10 | uint32 :sflow_version 11 | end 12 | 13 | # noinspection RubyResolve 14 | class SFlow < BinData::Record 15 | endian :big 16 | uint32 :sflow_version 17 | uint32 :ip_version 18 | choice :agent_ip, :selection => :ip_version do 19 | sflow_ip4_addr 1 20 | sflow_ip6_addr 2 21 | end 22 | uint32 :sub_agent_id 23 | uint32 :sequence_number 24 | uint32 :uptime_in_ms 25 | uint32 :sample_count 26 | array :samples, :initial_length => :sample_count do 27 | bit20 :sample_entreprise 28 | bit12 :sample_format 29 | uint32 :sample_length 30 | choice :sample_data, :selection => lambda { "#{sample_entreprise}-#{sample_format}" } do 31 | flow_sample '0-1' 32 | counter_sample '0-2' 33 | expanded_flow_sample '0-3' 34 | expanded_counter_sample '0-4' 35 | skip :default, :length => :sample_length 36 | end 37 | end 38 | end -------------------------------------------------------------------------------- /spec/codecs/sflow_spec.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | #require "logstash/devutils/rspec/spec_helper" 4 | require "logstash/codecs/sflow" 5 | require "logstash/codecs/sflow/datagram" 6 | 7 | describe SFlow do 8 | it "should decode sflow counters" do 9 | payload = IO.read(File.join(File.dirname(__FILE__), "sflow_counters_sample.dat"), :mode => "rb") 10 | SFlow.read(payload) 11 | end 12 | 13 | it "should decode sflow 1 counters" do 14 | payload = IO.read(File.join(File.dirname(__FILE__), "sflow_1_counters_sample.dat"), :mode => "rb") 15 | SFlow.read(payload) 16 | end 17 | 18 | it "should decode sflow sample" do 19 | payload = IO.read(File.join(File.dirname(__FILE__), "sflow_flow_sample.dat"), :mode => "rb") 20 | SFlow.read(payload) 21 | end 22 | 23 | it "should decode sflow sample eth vlan" do 24 | payload = IO.read(File.join(File.dirname(__FILE__), "sflow_flow_sample_eth_vlan.dat"), :mode => "rb") 25 | SFlow.read(payload) 26 | end 27 | 28 | it "should decode sflow with lag counters" do 29 | payload = IO.read(File.join(File.dirname(__FILE__), "sflow_with_lag_counters.dat"), :mode => "rb") 30 | SFlow.read(payload) 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/logstash/codecs/snmp/interface_resolver.rb: -------------------------------------------------------------------------------- 1 | require 'snmp' 2 | require 'lru_redux' 3 | 4 | class SNMPInterfaceResolver 5 | def initialize(community, cache_size, cache_ttl, logger) 6 | @community = community 7 | @cacheSnmpInterface = LruRedux::TTL::Cache.new(cache_size, cache_ttl) 8 | @logger = logger 9 | end 10 | 11 | def get_interface(host, ifIndex) 12 | unless @cacheSnmpInterface.key?("#{host}-#{ifIndex}") 13 | begin 14 | SNMP::Manager.open(:host => host, :community => @community, :version => :SNMPv2c) do |manager| 15 | @cacheSnmpInterface["#{host}-#{ifIndex}"] = manager.get_value("ifDescr.#{ifIndex}").to_s 16 | end 17 | rescue SNMP::RequestTimeout => e 18 | # This is not the best but it avoids loosing lots of events when facing 19 | # request timeout exception with input thread restarting. 20 | # Then we can easily detect this on the log or on elasticsearch 21 | # searching for SnmpRequestTimeout descr fields 22 | @logger.error("Timeout requesting description on #{host} of index #{ifIndex}: #{e.message}") 23 | return "SnmpRequestTimeout" 24 | end 25 | end 26 | return @cacheSnmpInterface["#{host}-#{ifIndex}"] 27 | end 28 | end -------------------------------------------------------------------------------- /logstash-codec-sflow.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |s| 2 | 3 | s.name = 'logstash-codec-sflow' 4 | s.version = '2.1.3' 5 | s.licenses = ['Apache-2.0'] 6 | s.summary = 'The sflow codec is for decoding SFlow v5 flows.' 7 | s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program' 8 | s.authors = ['Konrad Zemek', 'Nicolas Fraison'] 9 | s.email = 'konrad@path.net' 10 | s.homepage = 'https://path.net' 11 | s.require_paths = ['lib'] 12 | 13 | # Files 14 | s.files = Dir['lib/**/*', 'spec/**/*', 'vendor/**/*', '*.gemspec', '*.md', 'CONTRIBUTORS', 'Gemfile', 'LICENSE', 'NOTICE.TXT'] 15 | 16 | # Tests 17 | s.test_files = s.files.grep(%r{^(test|spec|features)/}) 18 | 19 | # Special flag to let us know this is actually a logstash plugin 20 | s.metadata = {'logstash_plugin' => 'true', 'logstash_group' => 'codec'} 21 | 22 | # Gem dependencies 23 | s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99' 24 | s.add_runtime_dependency 'logstash-core', '>= 5.4.0', '<= 7.9.9' 25 | s.add_runtime_dependency 'bindata', ['~> 2.4'] 26 | s.add_runtime_dependency 'lru_redux', ['~> 1.1'] 27 | s.add_runtime_dependency 'snmp', ['~> 1.2'] 28 | s.add_development_dependency 'logstash-devutils', ['~> 1.3'] 29 | end 30 | -------------------------------------------------------------------------------- /lib/logstash/codecs/sflow/util.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'bindata' 4 | require 'ipaddr' 5 | 6 | # noinspection RubyResolve 7 | class SflowMacAddress < BinData::Primitive 8 | array :bytes, :type => :uint8, :initial_length => 6 9 | 10 | def set(val) 11 | ints = val.split(/:/).collect { |int| int.to_i(16) } 12 | self.bytes = ints 13 | end 14 | 15 | def get 16 | self.bytes.collect { |byte| byte.value.to_s(16).rjust(2, '0') }.join(':') 17 | end 18 | end 19 | 20 | # noinspection RubyResolve,RubyResolve,RubyResolve 21 | class SflowIP4Addr < BinData::Primitive 22 | endian :big 23 | uint32 :storage 24 | 25 | def set(val) 26 | ip = IPAddr.new(val) 27 | unless ip.ipv4? 28 | raise ArgumentError, "invalid IPv4 address '#{val}'" 29 | end 30 | self.storage = ip.to_i 31 | end 32 | 33 | def get 34 | IPAddr.new_ntoh([self.storage].pack('N')).to_s 35 | end 36 | end 37 | 38 | # noinspection RubyResolve 39 | class SflowIP6Addr < BinData::Primitive 40 | endian :big 41 | uint128 :storage 42 | 43 | def set(val) 44 | ip = IPAddr.new(val) 45 | unless ip.ipv6? 46 | raise ArgumentError, "invalid IPv6 address `#{val}'" 47 | end 48 | self.storage = ip.to_i 49 | end 50 | 51 | def get 52 | IPAddr.new_ntoh((0..7).map { |i| 53 | (self.storage >> (112 - 16 * i)) & 0xffff 54 | }.pack('n8')).to_s 55 | end 56 | end 57 | 58 | # noinspection RubyResolve 59 | class SflowString < BinData::Primitive 60 | endian :big 61 | uint32 :read_length 62 | string :data, :read_length => :read_length 63 | skip :length => lambda { read_length % 4 } 64 | 65 | def set(val) 66 | self.read_length = val.length 67 | self.data = val 68 | end 69 | 70 | def get 71 | self.data 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /lib/logstash/codecs/sflow/packet_header.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'bindata' 4 | require 'logstash/codecs/sflow/util' 5 | 6 | 7 | # noinspection RubyResolve 8 | class UnknownHeader < BinData::Record 9 | mandatory_parameter :size_header 10 | 11 | endian :big 12 | bit :data, :nbits => :size_header 13 | end 14 | 15 | 16 | # noinspection RubyResolve,RubyResolve 17 | class TcpHeader < BinData::Record 18 | mandatory_parameter :size_header 19 | 20 | endian :big 21 | uint16 :src_port 22 | uint16 :dst_port 23 | uint32 :tcp_seq_number 24 | uint32 :tcp_ack_number 25 | bit4 :tcp_header_length # times 4 26 | bit3 :tcp_reserved 27 | bit1 :tcp_is_nonce 28 | bit1 :tcp_is_cwr 29 | bit1 :tcp_is_ecn_echo 30 | bit1 :tcp_is_urgent 31 | bit1 :tcp_is_ack 32 | bit1 :tcp_is_push 33 | bit1 :tcp_is_reset 34 | bit1 :tcp_is_syn 35 | bit1 :tcp_is_fin 36 | uint16 :tcp_window_size 37 | uint16 :tcp_checksum 38 | uint16 :tcp_urgent_pointer 39 | array :tcp_options, :initial_length => lambda { tcp_header_length - 5 }, :onlyif => lambda { is_options?(size_header) } do 40 | string :tcp_option, :length => 4, :pad_byte => "\0" 41 | end 42 | bit :data, :nbits => lambda { size_header - data.rel_offset * 8 } 43 | 44 | def is_options?(size_header) 45 | tcp_header_length.to_i > 5 and size_header >= tcp_header_length * 4 * 8 46 | end 47 | end 48 | 49 | # noinspection RubyResolve 50 | class UdpHeader < BinData::Record 51 | mandatory_parameter :size_header 52 | 53 | endian :big 54 | uint16 :src_port 55 | uint16 :dst_port 56 | uint16 :udp_length 57 | uint16 :udp_checksum 58 | bit :data, :nbits => lambda { size_header - 64 } #skip udp data 59 | end 60 | 61 | # noinspection RubyResolve,RubyResolve 62 | class IPV4Header < BinData::Record 63 | mandatory_parameter :size_header 64 | 65 | endian :big 66 | bit4 :ip_version 67 | bit4 :ip_header_length # times 4 68 | bit6 :ip_dscp 69 | bit2 :ip_ecn 70 | uint16 :ip_total_length 71 | uint16 :ip_identification 72 | bit3 :ip_flags 73 | bit13 :ip_fragment_offset 74 | uint8 :ip_ttl 75 | uint8 :ip_protocol 76 | uint16 :ip_checksum 77 | sflow_ip4_addr :src_ip 78 | sflow_ip4_addr :dst_ip 79 | array :ip_options, :initial_length => lambda { (((ip_header_length * 4) - 20)/4).ceil }, :onlyif => :is_options? do 80 | string :ip_option, :length => 4, :pad_byte => "\0" 81 | end 82 | choice :ip_data, :selection => :ip_protocol, :onlyif => lambda { has_data?(size_header) } do 83 | tcp_header 6, :size_header => lambda { size_header - (ip_header_length * 4 * 8) } 84 | udp_header 17, :size_header => lambda { size_header - (ip_header_length * 4 * 8) } 85 | unknown_header :default, :size_header => lambda { size_header - (ip_header_length * 4 * 8) } 86 | end 87 | 88 | def has_data?(size_header) 89 | bytes_left = size_header / 8 - ip_header_length * 4 90 | case ip_protocol 91 | when 6 92 | return bytes_left >= 20 93 | when 17 94 | return bytes_left >= 8 95 | else 96 | return true 97 | end 98 | end 99 | 100 | def is_options? 101 | ip_header_length.to_i > 5 102 | end 103 | end 104 | 105 | # noinspection RubyResolve 106 | class IPV6Header < BinData::Record 107 | mandatory_parameter :size_header 108 | 109 | endian :big 110 | bit4 :ip_version 111 | bit6 :ip_dscp 112 | bit2 :ip_ecn 113 | bit20 :ipv6_flow_label 114 | uint16 :ip_payload_length 115 | uint8 :ip_protocol 116 | uint8 :ipv6_hop_limit 117 | sflow_ip6_addr :src_ip 118 | sflow_ip6_addr :dst_ip 119 | choice :ip_data, :selection => :ip_protocol do 120 | tcp_header 6, :size_header => lambda { size_header - 320 } 121 | udp_header 17, :size_header => lambda { size_header - 320 } 122 | unknown_header :default, :size_header => lambda { size_header - 320 } 123 | end 124 | end 125 | 126 | # noinspection RubyResolve 127 | class VLANHeader < BinData::Record 128 | mandatory_parameter :size_header 129 | 130 | endian :big 131 | bit3 :vlan_priority 132 | bit1 :vlan_cfi 133 | bit12 :vlan_id 134 | uint16 :vlan_type 135 | choice :vlan_data, :selection => :vlan_type do 136 | ipv4_header 2048, :size_header => lambda { size_header - (4 * 8) } 137 | ipv6_header 34525, :size_header => lambda { size_header - (4 * 8) } 138 | unknown_header :default, :size_header => lambda { size_header - (4 * 8) } 139 | end 140 | end 141 | 142 | # noinspection RubyResolve 143 | class EthernetHeader < BinData::Record 144 | mandatory_parameter :size_header 145 | 146 | endian :big 147 | sflow_mac_address :eth_dst 148 | sflow_mac_address :eth_src 149 | uint16 :eth_type 150 | choice :eth_data, :selection => :eth_type do 151 | ipv4_header 2048, :size_header => lambda { size_header - (14 * 8) } 152 | vlan_header 33024, :size_header => lambda { size_header - (14 * 8) } 153 | ipv6_header 34525, :size_header => lambda { size_header - (14 * 8) } 154 | unknown_header :default, :size_header => lambda { size_header - (14 * 8) } 155 | end 156 | end 157 | -------------------------------------------------------------------------------- /spec/codecs/sflow/packet_header_spec.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | #require "logstash/devutils/rspec/spec_helper" 4 | require "logstash/codecs/sflow/packet_header" 5 | 6 | describe UdpHeader do 7 | it "should decode udp header" do 8 | payload = IO.read(File.join(File.dirname(__FILE__), "udp.dat"), :mode => "rb") 9 | decoded = UdpHeader.new(:size_header => payload.bytesize * 8).read(payload) 10 | 11 | decoded["src_port"].to_s.should eq("20665") 12 | decoded["dst_port"].to_s.should eq("514") 13 | decoded["udp_length"].to_s.should eq("147") 14 | end 15 | end 16 | 17 | 18 | describe TcpHeader do 19 | it "should decode tcp header" do 20 | payload = IO.read(File.join(File.dirname(__FILE__), "tcp.dat"), :mode => "rb") 21 | decoded = TcpHeader.new(:size_header => payload.bytesize * 8).read(payload) 22 | 23 | decoded["src_port"].to_s.should eq("5672") 24 | decoded["dst_port"].to_s.should eq("59451") 25 | decoded["tcp_seq_number"].to_s.should eq("2671357038") 26 | decoded["tcp_ack_number"].to_s.should eq("2651945969") 27 | (decoded["tcp_header_length"].to_i*4).to_s.should eq("32") 28 | decoded["tcp_is_nonce"].to_s.should eq("0") 29 | decoded["tcp_is_cwr"].to_s.should eq("0") 30 | decoded["tcp_is_ecn_echo"].to_s.should eq("0") 31 | decoded["tcp_is_urgent"].to_s.should eq("0") 32 | decoded["tcp_is_ack"].to_s.should eq("1") 33 | decoded["tcp_is_push"].to_s.should eq("1") 34 | decoded["tcp_is_reset"].to_s.should eq("0") 35 | decoded["tcp_is_syn"].to_s.should eq("0") 36 | decoded["tcp_is_fin"].to_s.should eq("0") 37 | decoded["tcp_window_size"].to_s.should eq("147") 38 | decoded["tcp_checksum"].to_s.should eq("13042") 39 | decoded["tcp_urgent_pointer"].to_s.should eq("0") 40 | end 41 | end 42 | 43 | 44 | describe IPV4Header do 45 | it "should decode ipv4 tcp header" do 46 | payload = IO.read(File.join(File.dirname(__FILE__), "ipv4_tcp_header.dat"), :mode => "rb") 47 | decoded = IPV4Header.new(:size_header => payload.bytesize * 8).read(payload) 48 | 49 | decoded["ip_version"].to_s.should eq("4") 50 | decoded["ip_header_length"].to_s.should eq("5") 51 | decoded["ip_dscp"].to_s.should eq("0") 52 | decoded["ip_ecn"].to_s.should eq("0") 53 | decoded["ip_total_length"].to_s.should eq("476") 54 | decoded["ip_identification"].to_s.should eq("30529") 55 | decoded["ip_flags"].to_s.should eq("2") 56 | decoded["ip_fragment_offset"].to_s.should eq("0") 57 | decoded["ip_ttl"].to_s.should eq("62") 58 | decoded["ip_protocol"].to_s.should eq("6") 59 | decoded["ip_checksum"].to_s.should eq("37559") 60 | decoded["src_ip"].to_s.should eq("10.243.27.17") 61 | decoded["dst_ip"].to_s.should eq("10.243.0.45") 62 | decoded["ip_data"]["src_port"].to_s.should eq("5672") 63 | decoded["ip_data"]["dst_port"].to_s.should eq("59451") 64 | decoded["ip_data"]["tcp_seq_number"].to_s.should eq("2671357038") 65 | decoded["ip_data"]["tcp_ack_number"].to_s.should eq("2651945969") 66 | (decoded["ip_data"]["tcp_header_length"].to_i*4).to_s.should eq("32") 67 | decoded["ip_data"]["tcp_is_nonce"].to_s.should eq("0") 68 | decoded["ip_data"]["tcp_is_cwr"].to_s.should eq("0") 69 | decoded["ip_data"]["tcp_is_ecn_echo"].to_s.should eq("0") 70 | decoded["ip_data"]["tcp_is_urgent"].to_s.should eq("0") 71 | decoded["ip_data"]["tcp_is_ack"].to_s.should eq("1") 72 | decoded["ip_data"]["tcp_is_push"].to_s.should eq("1") 73 | decoded["ip_data"]["tcp_is_reset"].to_s.should eq("0") 74 | decoded["ip_data"]["tcp_is_syn"].to_s.should eq("0") 75 | decoded["ip_data"]["tcp_is_fin"].to_s.should eq("0") 76 | decoded["ip_data"]["tcp_window_size"].to_s.should eq("147") 77 | decoded["ip_data"]["tcp_checksum"].to_s.should eq("13042") 78 | decoded["ip_data"]["tcp_urgent_pointer"].to_s.should eq("0") 79 | end 80 | end 81 | 82 | 83 | describe EthernetHeader do 84 | it "should decode ethernet ipv4 udp header" do 85 | payload = IO.read(File.join(File.dirname(__FILE__), "ethernet_ipv4_udp_header.dat"), :mode => "rb") 86 | decoded = EthernetHeader.new(:size_header => payload.bytesize * 8).read(payload) 87 | 88 | decoded["eth_dst"].to_s.should eq("00:23:e9:78:16:c6") 89 | decoded["eth_src"].to_s.should eq("58:f3:9c:81:4b:81") 90 | decoded["eth_type"].to_s.should eq("2048") 91 | decoded["eth_data"]["ip_header"]["dst_ip"].to_s.should eq("10.243.27.9") 92 | decoded["eth_data"]["ip_header"]["ip_data"]["dst_port"].to_s.should eq("514") 93 | end 94 | end 95 | 96 | 97 | describe EthernetHeader do 98 | it "should decode ethernet vlan ipv4 tcp header" do 99 | payload = IO.read(File.join(File.dirname(__FILE__), "ethernet_vlan_ipv4_tcp_header.dat"), :mode => "rb") 100 | decoded = EthernetHeader.new(:size_header => payload.bytesize * 8).read(payload) 101 | 102 | decoded["eth_dst"].to_s.should eq("a0:36:9f:71:d2:e0") 103 | decoded["eth_src"].to_s.should eq("00:09:0f:09:37:1c") 104 | decoded["eth_type"].to_s.should eq("33024") 105 | decoded["eth_data"]["vlan_id"].to_s.should eq("2422") 106 | decoded["eth_data"]["vlan_type"].to_s.should eq("2048") 107 | end 108 | end -------------------------------------------------------------------------------- /lib/logstash/codecs/sflow/sample.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'bindata' 4 | require 'logstash/codecs/sflow/flow_record' 5 | require 'logstash/codecs/sflow/counter_record' 6 | 7 | class FlowSampleRecordData < BinData::Choice 8 | mandatory_parameter :record_length 9 | 10 | raw_packet_header '0-1', :record_length => :record_length 11 | ethernet_frame_data '0-2' 12 | ip4_data '0-3' 13 | ip6_data '0-4' 14 | extended_switch_data '0-1001' 15 | extended_router_data '0-1002' 16 | extended_gateway_data '0-1003' 17 | extended_user_data '0-1004' 18 | extended_url_data '0-1005' 19 | extended_mpls_data '0-1006' 20 | extended_nat_data '0-1007' 21 | extended_mpls_tunnel '0-1008' 22 | extended_mpls_vc '0-1009' 23 | extended_mpls_ftn '0-1010' 24 | extended_mpls_ldp_fec '0-1012' 25 | extended_vlan_tunnel '0-1012' 26 | extended_l2_tunnel_egress '0-1021' 27 | extended_l2_tunnel_ingress '0-1022' 28 | extended_ipv4_tunnel_egress '0-1023' 29 | extended_ipv4_tunnel_ingress '0-1024' 30 | extended_ipv6_tunnel_egress '0-1025' 31 | extended_ipv6_tunnel_ingress '0-1026' 32 | extended_decapsulate_egress '0-1027' 33 | extended_decapsulate_ingress '0-1028' 34 | extended_vni_egress '0-1029' 35 | extended_vni_ingress '0-1030' 36 | extended_socket_ipv4 '0-2100' 37 | extended_socket_ipv6 '0-2101' 38 | skip :default, :length => :record_length 39 | end 40 | 41 | class CounterSampleRecordData < BinData::Choice 42 | mandatory_parameter :record_length 43 | 44 | generic_interface '0-1' 45 | ethernet_interfaces '0-2' 46 | token_ring '0-3' 47 | hundred_base_vg '0-4' 48 | vlan '0-5' 49 | ieee80211_counters '0-6' 50 | lag_port_stats '0-7' 51 | processor_information '0-1001' 52 | radio_utilization '0-1002' 53 | of_port '0-1004' 54 | port_name '0-1005' 55 | host_descr '0-2000' 56 | host_adapters '0-2001' 57 | host_parent '0-2002' 58 | host_cpu '0-2003' 59 | host_memory '0-2004' 60 | host_disk_io '0-2005' 61 | host_net_io '0-2006' 62 | mib2_ip_group '0-2007' 63 | mib2_icmp_group '0-2008' 64 | mib2_tcp_group '0-2009' 65 | mib2_udp_group '0-2010' 66 | virt_node '0-2100' 67 | virt_cpu '0-2101' 68 | virt_memory '0-2102' 69 | virt_disk_io '0-2103' 70 | virt_net_io '0-2104' 71 | http_counters '0-2201' 72 | ovs_dp_stats '0-2207' 73 | skip :default, :length => :record_length 74 | end 75 | 76 | # noinspection RubyResolve 77 | class FlowSample < BinData::Record 78 | endian :big 79 | uint32 :flow_sequence_number 80 | uint8 :source_id_type 81 | uint24 :source_id_index 82 | uint32 :sampling_rate 83 | uint32 :sample_pool 84 | uint32 :drops 85 | uint32 :input_interface 86 | uint32 :output_interface 87 | uint32 :record_count 88 | array :records, :initial_length => :record_count do 89 | bit20 :record_entreprise 90 | bit12 :record_format 91 | uint32 :record_length 92 | flow_sample_record_data :record_data, 93 | :selection => lambda { "#{record_entreprise}-#{record_format}" }, 94 | :record_length => :record_length 95 | end 96 | end 97 | 98 | # noinspection RubyResolve 99 | class CounterSample < BinData::Record 100 | endian :big 101 | uint32 :sample_seq_number 102 | uint8 :source_id_type 103 | uint24 :source_id_index 104 | uint32 :record_count 105 | array :records, :initial_length => :record_count do 106 | bit20 :record_entreprise 107 | bit12 :record_format 108 | uint32 :record_length 109 | counter_sample_record_data :record_data, 110 | :selection => lambda { "#{record_entreprise}-#{record_format}" }, 111 | :record_length => :record_length 112 | #processor_information :record_data 113 | end 114 | end 115 | 116 | # noinspection RubyResolve 117 | class ExpandedFlowSample < BinData::Record 118 | endian :big 119 | uint32 :flow_sequence_number 120 | uint32 :source_id_type 121 | uint32 :source_id_index 122 | uint32 :sampling_rate 123 | uint32 :sample_pool 124 | uint32 :drops 125 | uint32 :input_interface_format 126 | uint32 :input_interface_value 127 | uint32 :output_interface_format 128 | uint32 :output_interface_value 129 | uint32 :record_count 130 | array :records, :initial_length => :record_count do 131 | bit20 :record_entreprise 132 | bit12 :record_format 133 | uint32 :record_length 134 | flow_sample_record_data :record_data, 135 | :selection => lambda { "#{record_entreprise}-#{record_format}" }, 136 | :record_length => :record_length 137 | end 138 | end 139 | 140 | # noinspection RubyResolve 141 | class ExpandedCounterSample < BinData::Record 142 | endian :big 143 | uint32 :sample_seq_number 144 | uint32 :source_id_type 145 | uint32 :source_id_index 146 | uint32 :record_count 147 | array :records, :initial_length => :record_count do 148 | bit20 :record_entreprise 149 | bit12 :record_format 150 | uint32 :record_length 151 | counter_sample_record_data :record_data, 152 | :selection => lambda { "#{record_entreprise}-#{record_format}" }, 153 | :record_length => :record_length 154 | #processor_information :record_data 155 | end 156 | end 157 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Logstash Codec SFlow Plugin 2 | ## Description 3 | Logstash codec plugin to decode sflow codec. 4 | 5 | This codec manage flow sample, counter flow, expanded flow sample and expanded counter flow 6 | 7 | For the (expanded) flow sample it is able to decode Ethernet, 802.1Q VLAN, IPv4, UDP and TCP header 8 | 9 | For the (expanded) counter flow it is able to decode some records of type: 10 | 11 | - Generic Interface 12 | - Ethernet Interface 13 | - VLAN 14 | - Processor Information 15 | - HTTP 16 | - LAG 17 | 18 | ## TO DO 19 | Currently this plugin does not manage all sflow counter and is not able to decode 20 | all kind of protocols. 21 | If needed you can aks for some to be added. 22 | Please provide a pcap file containing the sflow events of the counter/protocol 23 | to add in order to be able to implement it. 24 | 25 | ## Tune reported fields 26 | By default all those fields are removed from the emitted event: 27 | 28 | %w(sflow_version header_size ip_header_length ip_dscp ip_ecn ip_total_length ip_identification ip_flags 29 | ip_fragment_offset ip_ttl ip_checksum ip_options tcp_seq_number tcp_ack_number tcp_header_length tcp_reserved 30 | tcp_is_nonce tcp_is_cwr tcp_is_ecn_echo tcp_is_urgent tcp_is_ack tcp_is_push tcp_is_reset tcp_is_syn tcp_is_fin 31 | tcp_window_size tcp_checksum tcp_urgent_pointer tcp_options vlan_cfi sequence_number flow_sequence_number vlan_type 32 | udp_length udp_checksum) 33 | 34 | You can tune the list of removed fields by setting this parameter to the sflow codec *optional_removed_field* 35 | 36 | ## frame_length_times_sampling_rate output field on (expanded) flow sample 37 | 38 | This field is the length of the frame times the sampling rate. It permits to approximate the number of bits send/receive 39 | on an interface/socket. 40 | 41 | You must first ensure to have well configured the sampling rate to have an accurate output metric (See: http://blog.sflow.com/2009/06/sampling-rates.html) 42 | 43 | 44 | ## Human Readable Protocol 45 | In order to translate protocols value to a human readable protocol, you can use the 46 | logstash-filter-translate plugin 47 | ``` 48 | filter { 49 | translate { 50 | field => protocol 51 | dictionary => [ "1", "ETHERNET", 52 | "11", "IP" 53 | ] 54 | fallback => "UNKNOWN" 55 | destination => protocol 56 | override => true 57 | } 58 | translate { 59 | field => eth_type 60 | dictionary => [ "2048", "IP", 61 | "33024", "802.1Q VLAN" 62 | ] 63 | fallback => "UNKNOWN" 64 | destination => eth_type 65 | override => true 66 | } 67 | translate { 68 | field => vlan_type 69 | dictionary => [ "2048", "IP" 70 | ] 71 | fallback => "UNKNOWN" 72 | destination => vlan_type 73 | override => true 74 | } 75 | translate { 76 | field => ip_protocol 77 | dictionary => [ "6", "TCP", 78 | "17", "UDP", 79 | "50", "Encapsulating Security Payload" 80 | ] 81 | fallback => "UNKNOWN" 82 | destination => ip_protocol 83 | override => true 84 | } 85 | } 86 | ``` 87 | 88 | [![Build 89 | Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Codecs/job/logstash-plugin-codec-example-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Codecs/job/logstash-plugin-codec-example-unit/) 90 | 91 | This is a plugin for [Logstash](https://github.com/elastic/logstash). 92 | 93 | It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way. 94 | 95 | ## Documentation 96 | 97 | Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/). 98 | 99 | - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive 100 | - For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide 101 | 102 | ## Need Help? 103 | 104 | Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum. 105 | 106 | ## Developing 107 | 108 | ### 1. Plugin Developement and Testing 109 | 110 | #### Code 111 | - To get started, you'll need JRuby with the Bundler gem installed. 112 | 113 | - Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. 114 | 115 | - Install dependencies 116 | ```sh 117 | bundle install 118 | ``` 119 | 120 | #### Test 121 | 122 | ```sh 123 | bundle exec rspec 124 | ``` 125 | 126 | The Logstash code required to run the tests/specs is specified in the `Gemfile` by the line similar to: 127 | ```ruby 128 | gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5" 129 | ``` 130 | To test against another version or a local Logstash, edit the `Gemfile` to specify an alternative location, for example: 131 | ```ruby 132 | gem "logstash", :github => "elasticsearch/logstash", :ref => "master" 133 | ``` 134 | ```ruby 135 | gem "logstash", :path => "/your/local/logstash" 136 | ``` 137 | 138 | Then update your dependencies and run your tests: 139 | 140 | ```sh 141 | bundle install 142 | bundle exec rspec 143 | ``` 144 | 145 | ### 2. Running your unpublished Plugin in Logstash 146 | 147 | #### 2.1 Run in a local Logstash clone 148 | 149 | - Edit Logstash `tools/Gemfile` and add the local plugin path, for example: 150 | ```ruby 151 | gem "logstash-codec-sflow", :path => "/your/local/logstash-codec-sflow" 152 | ``` 153 | - Update Logstash dependencies 154 | ```sh 155 | rake vendor:gems 156 | ``` 157 | - Run Logstash with your plugin 158 | ```sh 159 | bin/logstash -e 'input { udp { port => 6343 codec => sflow }}' 160 | ``` 161 | At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash. 162 | 163 | #### 2.2 Run in an installed Logstash 164 | 165 | - Build your plugin gem 166 | ```sh 167 | gem build logstash-codec-sflow.gemspec 168 | ``` 169 | - Install the plugin from the Logstash home 170 | ```sh 171 | bin/plugin install /your/local/plugin/logstash-codec-sflow.gem 172 | ``` 173 | - Start Logstash and proceed to test the plugin 174 | 175 | ## Contributing 176 | 177 | All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin. 178 | 179 | Programming is not a required skill. Whatever you've seen about open source and maintainers or community members saying "send patches or die" - you will not see that here. 180 | 181 | It is more important to me that you are able to contribute. 182 | 183 | For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file. -------------------------------------------------------------------------------- /lib/logstash/codecs/sflow/flow_record.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'bindata' 4 | require 'logstash/codecs/sflow/util' 5 | require 'logstash/codecs/sflow/packet_header' 6 | 7 | # noinspection RubyResolve 8 | class RawPacketHeader < BinData::Buffer 9 | mandatory_parameter :record_length 10 | default_parameters :length => :record_length 11 | 12 | endian :big 13 | uint32 :protocol 14 | uint32 :frame_length 15 | uint32 :stripped 16 | uint32 :header_size 17 | choice :sample_header, :selection => :protocol do 18 | ethernet_header 1, :size_header => lambda { header_size * 8 } 19 | ipv4_header 11, :size_header => lambda { header_size * 8 } 20 | ipv6_header 12, :size_header => lambda { header_size * 8 } 21 | skip :default, :length => :header_size 22 | end 23 | end 24 | 25 | # noinspection RubyResolve 26 | class EthernetFrameData < BinData::Record 27 | endian :big 28 | uint32 :packet_length 29 | sflow_mac_address :src_mac 30 | skip :length => 2 31 | sflow_mac_address :dst_mac 32 | skip :length => 2 33 | uint32 :eth_type 34 | end 35 | 36 | # noinspection RubyResolve 37 | class IP4Data < BinData::Record 38 | endian :big 39 | uint32 :ip_packet_length 40 | uint32 :ip_protocol 41 | sflow_ip4_addr :src_ip 42 | sflow_ip4_addr :dst_ip 43 | uint32 :src_port 44 | uint32 :dst_port 45 | uint32 :tcp_flags 46 | uint32 :ip_type 47 | end 48 | 49 | # noinspection RubyResolve 50 | class IP6Data < BinData::Record 51 | endian :big 52 | uint32 :ip_packet_length 53 | uint32 :ip_next_header 54 | sflow_ip6_addr :src_ip 55 | sflow_ip6_addr :dst_ip 56 | uint32 :src_port 57 | uint32 :dst_port 58 | uint32 :tcp_flags 59 | uint32 :ip_priority 60 | end 61 | 62 | # noinspection RubyResolve 63 | class ExtendedSwitchData < BinData::Record 64 | endian :big 65 | uint32 :src_vlan 66 | uint32 :src_priority 67 | uint32 :dst_vlan 68 | uint32 :dst_priority 69 | end 70 | 71 | # noinspection RubyResolve 72 | class ExtendedRouterData < BinData::Record 73 | endian :big 74 | uint32 :ip_version 75 | choice :ip_address_next_hop_router, :selection => :ip_version do 76 | sflow_ip4_addr 1 77 | sflow_ip6_addr 2 78 | end 79 | uint32 :src_mask_len 80 | uint32 :dst_mask_len 81 | end 82 | 83 | # noinspection RubyResolve 84 | class ExtendedGatewayData < BinData::Record 85 | endian :big 86 | uint32 :ip_version 87 | choice :ip_address_next_hop_router, :selection => :ip_version do 88 | sflow_ip4_addr 1 89 | sflow_ip6_addr 2 90 | end 91 | uint32 :as_number_of_router 92 | uint32 :as_number_of_source 93 | uint32 :as_number_of_source_peer 94 | uint32 :dest_as_path_count 95 | array :dest_as_paths, :initial_length => :dest_as_path_count do 96 | uint32 :as_path_segment_type 97 | uint32 :as_number_count 98 | array :as_numbers, :type => :uint32, :initial_length => :as_number_count 99 | end 100 | uint32 :communities_count 101 | array :communities, :type => :uint32, :initial_length => :communities_count 102 | uint32 :local_pref 103 | end 104 | 105 | # noinspection RubyResolve 106 | class ExtendedUserData < BinData::Record 107 | endian :big 108 | uint32 :source_charset 109 | sflow_string :source_user 110 | uint32 :destination_charset 111 | sflow_string :destination_user 112 | end 113 | 114 | # noinspection RubyResolve 115 | class ExtendedUrlData < BinData::Record 116 | endian :big 117 | uint32 :direction 118 | sflow_string :url 119 | sflow_string :host 120 | end 121 | 122 | # noinspection RubyResolve 123 | class ExtendedMplsData < BinData::Record 124 | endian :big 125 | uint32 :ip_version 126 | choice :ip_address_next_hop_router, :selection => :ip_version do 127 | sflow_ip4_addr 1 128 | sflow_ip6_addr 2 129 | end 130 | uint32 :in_label_stack_count 131 | array :in_label_stack, :type => :uint32, :initial_length => :in_label_stack_count 132 | uint32 :out_label_stack_count 133 | array :out_label_stack, :type => :uint32, :initial_length => :out_label_stack_count 134 | end 135 | 136 | # noinspection RubyResolve 137 | class ExtendedNatData < BinData::Record 138 | endian :big 139 | uint32 :src_ip_version 140 | choice :src_ip_address, :selection => :src_ip_version do 141 | sflow_ip4_addr 1 142 | sflow_ip6_addr 2 143 | end 144 | uint32 :dst_ip_version 145 | choice :dst_ip_address, :selection => :dst_ip_version do 146 | sflow_ip4_addr 1 147 | sflow_ip6_addr 2 148 | end 149 | end 150 | 151 | # noinspection RubyResolve 152 | class ExtendedMplsTunnel < BinData::Record 153 | endian :big 154 | sflow_string :tunnel_name 155 | uint32 :tunnel_id 156 | uint32 :tunnel_cos_value 157 | end 158 | 159 | # noinspection RubyResolve 160 | class ExtendedMplsVc < BinData::Record 161 | endian :big 162 | sflow_string :vc_instance_name 163 | uint32 :vll_vc_id 164 | uint32 :vc_label_cos_value 165 | end 166 | 167 | # noinspection RubyResolve 168 | class ExtendedMplsFtn < BinData::Record 169 | endian :big 170 | sflow_string :mpls_ftn_descr 171 | uint32 :mpls_ftn_mask 172 | end 173 | 174 | # noinspection RubyResolve 175 | class ExtendedMplsLdpFec < BinData::Record 176 | endian :big 177 | uint32 :mpls_fec_addr_prefix_length 178 | end 179 | 180 | # noinspection RubyResolve 181 | class ExtendedVlanTunnel < BinData::Record 182 | endian :big 183 | uint32 :layers_count 184 | array :layers, :type => :uint32, :initial_length => :layers_count 185 | end 186 | 187 | # noinspection RubyResolve 188 | class ExtendedL2TunnelEgress < BinData::Record 189 | endian :big 190 | ethernet_frame_data :header 191 | end 192 | 193 | # noinspection RubyResolve 194 | class ExtendedL2TunnelIngress < BinData::Record 195 | endian :big 196 | ethernet_frame_data :header 197 | end 198 | 199 | # noinspection RubyResolve 200 | class ExtendedIpv4TunnelEgress < BinData::Record 201 | endian :big 202 | ip4_data :header 203 | end 204 | 205 | # noinspection RubyResolve 206 | class ExtendedIpv4TunnelIngress < BinData::Record 207 | endian :big 208 | ip4_data :header 209 | end 210 | 211 | # noinspection RubyResolve 212 | class ExtendedIpv6TunnelEgress < BinData::Record 213 | endian :big 214 | ip6_data :header 215 | end 216 | 217 | # noinspection RubyResolve 218 | class ExtendedIpv6TunnelIngress < BinData::Record 219 | endian :big 220 | ip6_data :header 221 | end 222 | 223 | # noinspection RubyResolve 224 | class ExtendedDecapsulateEgress < BinData::Record 225 | endian :big 226 | uint32 :inner_header_offset 227 | end 228 | 229 | # noinspection RubyResolve 230 | class ExtendedDecapsulateIngress < BinData::Record 231 | endian :big 232 | uint32 :inner_header_offset 233 | end 234 | 235 | # noinspection RubyResolve 236 | class ExtendedVniEgress < BinData::Record 237 | endian :big 238 | uint32 :vni 239 | end 240 | 241 | # noinspection RubyResolve 242 | class ExtendedVniIngress < BinData::Record 243 | endian :big 244 | uint32 :vni 245 | end 246 | 247 | # noinspection RubyResolve 248 | class ExtendedSocketIpv4 < BinData::Record 249 | endian :big 250 | uint32 :protocol 251 | sflow_ip4_addr :local_ip 252 | sflow_ip4_addr :remote_ip 253 | uint32 :local_port 254 | uint32 :remote_port 255 | end 256 | 257 | # noinspection RubyResolve 258 | class ExtendedSocketIpv6 < BinData::Record 259 | endian :big 260 | uint32 :protocol 261 | sflow_ip6_addr :local_ip 262 | sflow_ip6_addr :remote_ip 263 | uint32 :local_port 264 | uint32 :remote_port 265 | end 266 | -------------------------------------------------------------------------------- /lib/logstash/codecs/sflow.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require 'logstash/codecs/base' 3 | require 'logstash/namespace' 4 | 5 | # The "sflow" codec is for decoding sflow v5 flows. 6 | class LogStash::Codecs::Sflow < LogStash::Codecs::Base 7 | config_name 'sflow' 8 | 9 | # Specify which Sflow versions you will accept. 10 | config :versions, :validate => :array, :default => [5] 11 | 12 | # Specify which sflow fields must not be send in the event 13 | config :optional_removed_field, :validate => :array, :default => %w(sflow_version header_size 14 | ip_header_length ip_dscp ip_ecn ip_total_length ip_identification ip_flags ip_fragment_offset ip_ttl ip_checksum 15 | ip_options tcp_seq_number tcp_ack_number tcp_header_length tcp_reserved tcp_is_nonce tcp_is_cwr tcp_is_ecn_echo 16 | tcp_is_urgent tcp_is_ack tcp_is_push tcp_is_reset tcp_is_syn tcp_is_fin tcp_window_size tcp_checksum 17 | tcp_urgent_pointer tcp_options vlan_cfi sequence_number flow_sequence_number vlan_type udp_length udp_checksum) 18 | 19 | # Specify if codec must perform SNMP call so agent_ip for interface resolution. 20 | config :snmp_interface, :validate => :boolean, :default => false 21 | 22 | # Specify if codec must perform SNMP call so agent_ip for interface resolution. 23 | config :snmp_community, :validate => :string, :default => 'public' 24 | 25 | # Specify the max number of element in the interface resolution local cache (only if snmp_interface true) 26 | config :interface_cache_size, :validate => :number, :default => 1000 27 | 28 | # Specify the duration for each element in the interface resolution local cache (only if snmp_interface true) 29 | config :interface_cache_ttl, :validate => :number, :default => 3600 30 | 31 | def initialize(params = {}) 32 | super(params) 33 | @threadsafe = false 34 | end 35 | 36 | # def initialize 37 | 38 | def assign_key_value(event, bindata_kv) 39 | inspection_queue = [bindata_kv] 40 | 41 | while !inspection_queue.empty? 42 | kv = inspection_queue.shift() 43 | if kv.nil? || kv.to_s.eql?('') 44 | next 45 | end 46 | 47 | kv.each_pair do |k, v| 48 | if v.is_a?(BinData::Choice) 49 | inspection_queue.push(v) 50 | elsif !@removed_field.include?(k.to_s) && !v.is_a?(BinData::Array) 51 | event.set("#{k.to_s}", v.to_s) 52 | end 53 | end 54 | end 55 | end 56 | 57 | # @param [Object] event 58 | # @param [Object] decoded 59 | # @param [Object] sample 60 | # @param [Object] record 61 | def common_sflow(event, decoded, sample) 62 | event.set('agent_ip', decoded['agent_ip'].to_s) 63 | assign_key_value(event, decoded) 64 | assign_key_value(event, sample) 65 | end 66 | 67 | def snmp_call(event) 68 | if @snmp_interface 69 | if event.include?('source_id_type') and event.get('source_id_type').to_s == '0' 70 | if event.include?('source_id_index') 71 | event.set('source_id_index_descr', @snmp.get_interface(event.get('agent_ip'), event.get('source_id_index'))) 72 | end 73 | if event.include?('input_interface') 74 | event.set('input_interface_descr', @snmp.get_interface(event.get('agent_ip'), event.get('input_interface'))) 75 | end 76 | if event.include?('output_interface') 77 | event.set('output_interface_descr', @snmp.get_interface(event.get('agent_ip'), event.get('output_interface'))) 78 | end 79 | if event.include?('interface_index') 80 | event.set('interface_index_descr', @snmp.get_interface(event.get('agent_ip'), event.get('interface_index'))) 81 | end 82 | end 83 | end 84 | end 85 | 86 | public 87 | def register 88 | require 'logstash/codecs/sflow/datagram' 89 | require 'logstash/codecs/snmp/interface_resolver' 90 | 91 | # noinspection RubyResolve 92 | @removed_field = %w(record_length record_count record_entreprise record_format sample_entreprise sample_format 93 | sample_length sample_count sample_header data storage) | @optional_removed_field 94 | 95 | if @snmp_interface 96 | @snmp = SNMPInterfaceResolver.new(@snmp_community, @interface_cache_size, @interface_cache_ttl, @logger) 97 | end 98 | end 99 | 100 | # def register 101 | 102 | public 103 | def decode(payload) 104 | header = SFlowHeader.read(payload) 105 | unless @versions.include?(header.sflow_version) 106 | @logger.warn("Ignoring Sflow version v#{header.sflow_version}") 107 | return 108 | end 109 | 110 | decoded = SFlow.read(payload) 111 | 112 | events = [] 113 | 114 | decoded['samples'].each do |sample| 115 | @logger.debug("sample: #{sample}") 116 | #Treat case with no flow decoded (Unknown flow) 117 | if sample['sample_data'].to_s.eql? '' 118 | @logger.warn("Unknown sample entreprise #{sample['sample_entreprise'].to_s} - format #{sample['sample_format'].to_s}") 119 | next 120 | end 121 | 122 | #treat sample flow and expanded sample flow 123 | if sample['sample_entreprise'] == 0 && (sample['sample_format'] == 1 || sample['sample_format'] == 3) 124 | # Create the logstash event 125 | event = LogStash::Event.new({}) 126 | 127 | common_sflow(event, decoded, sample) 128 | 129 | sample['sample_data']['records'].each do |record| 130 | # Ensure that some data exist for the record 131 | if record['record_data'].to_s.eql? '' 132 | @logger.warn("Unknown sample_flow record: entreprise #{record['record_entreprise'].to_s}, format #{record['record_format'].to_s}") 133 | next 134 | end 135 | 136 | assign_key_value(event, record) 137 | 138 | end 139 | #compute frame_length_times_sampling_rate 140 | if event.include?('frame_length') and event.include?('sampling_rate') 141 | event.set('frame_length_times_sampling_rate', event.get('frame_length').to_i * event.get('sampling_rate').to_i) 142 | end 143 | 144 | if sample['sample_format'] == 1 145 | event.set('sflow_type', 'flow_sample') 146 | else 147 | event.set('sflow_type', 'expanded_flow_sample') 148 | end 149 | 150 | #Get interface dfescr if snmp_interface true 151 | snmp_call(event) 152 | 153 | events.push(event) 154 | 155 | #treat counter flow and expanded counter flow 156 | elsif sample['sample_entreprise'] == 0 && (sample['sample_format'] == 2 || sample['sample_format'] == 4) 157 | sample['sample_data']['records'].each do |record| 158 | # Ensure that some data exist for the record 159 | if record['record_data'].to_s.eql? '' 160 | @logger.warn("Unknown counter_flow record: entreprise #{record['record_entreprise'].to_s}, format #{record['record_format'].to_s}") 161 | next 162 | end 163 | 164 | # Create the logstash event 165 | event = LogStash::Event.new({}) 166 | common_sflow(event, decoded, sample) 167 | 168 | assign_key_value(event, record) 169 | 170 | if sample['sample_format'] == 2 171 | event.set('sflow_type', 'counter_sample') 172 | else 173 | event.set('sflow_type', 'expanded_counter_sample') 174 | end 175 | 176 | 177 | #Get interface dfescr if snmp_interface true 178 | snmp_call(event) 179 | 180 | events.push(event) 181 | end 182 | end 183 | end 184 | 185 | events.each do |event| 186 | yield event 187 | end 188 | rescue BinData::ValidityError, EOFError, IOError, RangeError => e 189 | @logger.warn("Invalid sflow packet received (#{e})") 190 | # Sflow.instance_variables.each { |ivar| puts "#{ivar}: #{Sflow.instance_variable_get(ivar)}" } 191 | end # def decode 192 | end # class LogStash::Filters::Sflow 193 | -------------------------------------------------------------------------------- /lib/logstash/codecs/sflow/counter_record.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'bindata' 4 | 5 | # noinspection RubyResolve 6 | class GenericInterface < BinData::Record 7 | endian :big 8 | uint32 :interface_index 9 | uint32 :interface_type 10 | uint64 :interface_speed 11 | uint32 :interface_direction 12 | uint32 :interface_status 13 | uint64 :input_octets 14 | uint32 :input_packets 15 | uint32 :input_multicast_packets 16 | uint32 :input_broadcast_packets 17 | uint32 :input_discarded_packets 18 | uint32 :input_errors 19 | uint32 :input_unknown_protocol_packets 20 | uint64 :output_octets 21 | uint32 :output_packets 22 | uint32 :output_multicast_packets 23 | uint32 :output_broadcast_packets 24 | uint32 :output_discarded_packets 25 | uint32 :output_errors 26 | uint32 :promiscous_mode 27 | end 28 | 29 | # noinspection RubyResolve 30 | class EthernetInterfaces < BinData::Record 31 | endian :big 32 | uint32 :dot3StatsAlignmentErrors 33 | uint32 :dot3StatsFCSErrors 34 | uint32 :dot3StatsSingleCollisionFrames 35 | uint32 :dot3StatsMultipleCollisionFrames 36 | uint32 :dot3StatsSQETestErrors 37 | uint32 :dot3StatsDeferredTransmissions 38 | uint32 :dot3StatsLateCollisions 39 | uint32 :dot3StatsExcessiveCollisions 40 | uint32 :dot3StatsInternalMacTransmitErrors 41 | uint32 :dot3StatsCarrierSenseErrors 42 | uint32 :dot3StatsFrameTooLongs 43 | uint32 :dot3StatsInternalMacReceiveErrors 44 | uint32 :dot3StatsSymbolErrors 45 | end 46 | 47 | # noinspection RubyResolve 48 | class TokenRing < BinData::Record 49 | endian :big 50 | uint32 :dot5StatsLineErrors 51 | uint32 :dot5StatsBurstErrors 52 | uint32 :dot5StatsACErrors 53 | uint32 :dot5StatsAbortTransErrors 54 | uint32 :dot5StatsInternalErrors 55 | uint32 :dot5StatsLostFrameErrors 56 | uint32 :dot5StatsReceiveCongestions 57 | uint32 :dot5StatsFrameCopiedErrors 58 | uint32 :dot5StatsTokenErrors 59 | uint32 :dot5StatsSoftErrors 60 | uint32 :dot5StatsHardErrors 61 | uint32 :dot5StatsSignalLoss 62 | uint32 :dot5StatsTransmitBeacons 63 | uint32 :dot5StatsRecoverys 64 | uint32 :dot5StatsLobeWires 65 | uint32 :dot5StatsRemoves 66 | uint32 :dot5StatsSingles 67 | uint32 :dot5StatsFreqErrors 68 | end 69 | 70 | # noinspection RubyResolve 71 | class HundredBaseVG < BinData::Record 72 | endian :big 73 | uint32 :dot12InHighPriorityFrames 74 | uint64 :dot12InHighPriorityOctets 75 | uint32 :dot12InNormPriorityFrames 76 | uint64 :dot12InNormPriorityOctets 77 | uint32 :dot12InIPMErrors 78 | uint32 :dot12InOversizeFrameErrors 79 | uint32 :dot12InDataErrors 80 | uint32 :dot12InNullAddressedFrames 81 | uint32 :dot12OutHighPriorityFrames 82 | uint64 :dot12OutHighPriorityOctets 83 | uint32 :dot12TransitionIntoTrainings 84 | uint64 :dot12HCInHighPriorityOctets 85 | uint64 :dot12HCInNormPriorityOctets 86 | uint64 :dot12HCOutHighPriorityOctets 87 | end 88 | 89 | # noinspection RubyResolve 90 | class Vlan < BinData::Record 91 | endian :big 92 | uint32 :vlan_id 93 | uint64 :octets 94 | uint32 :ucastPkts 95 | uint32 :multicastPkts 96 | uint32 :broadcastPkts 97 | uint32 :discards 98 | end 99 | 100 | # noinspection RubyResolve 101 | class Ieee80211Counters < BinData::Record 102 | endian :big 103 | uint32 :dot11_transmitted_fragments 104 | uint32 :dot11_multicast_transmitted_frames 105 | uint32 :dot11_failures 106 | uint32 :dot11_retries 107 | uint32 :dot11_multiple_retries 108 | uint32 :dot11_duplicate_frames 109 | uint32 :dot11_rts_successes 110 | uint32 :dot11_rts_failures 111 | uint32 :dot11_ack_failures 112 | uint32 :dot11_received_fragments 113 | uint32 :dot11_multicast_received_frames 114 | uint32 :dot11_fcs_errors 115 | uint32 :dot11_transmitted_frames 116 | uint32 :dot11_wep_undecryptables 117 | uint32 :dot11_qos_discarded_fragments 118 | uint32 :dot11_associated_stations 119 | uint32 :dot11_qos_cf_polls_eceived 120 | uint32 :dot11_qos_cf_polls_unused 121 | uint32 :dot11_qos_cf_polls_unusable 122 | uint32 :dot11_qos_cf_polls_lost 123 | end 124 | 125 | # noinspection RubyResolve 126 | class ProcessorInformation < BinData::Record 127 | endian :big 128 | uint32 :five_sec_cpu_percent 129 | uint32 :one_min_cpu_percent 130 | uint32 :five_min_cpu_percent 131 | uint64 :total_memory 132 | uint64 :free_memory 133 | end 134 | 135 | # noinspection RubyResolve 136 | class RadioUtilization < BinData::Record 137 | endian :big 138 | uint32 :radio_elapsed_time_ms 139 | uint32 :radio_on_channel_time_ms 140 | uint32 :radio_on_channel_busy_time_ms 141 | end 142 | 143 | # noinspection RubyResolve 144 | class OfPort < BinData::Record 145 | endian :big 146 | uint64 :datapath_id 147 | uint32 :port_no 148 | end 149 | 150 | # noinspection RubyResolve 151 | class PortName < BinData::Record 152 | endian :big 153 | sflow_string :name 154 | end 155 | 156 | # noinspection RubyResolve 157 | class HttpCounters < BinData::Record 158 | endian :big 159 | uint32 :method_option_count 160 | uint32 :method_get_count 161 | uint32 :method_head_count 162 | uint32 :method_post_count 163 | uint32 :method_put_count 164 | uint32 :method_delete_count 165 | uint32 :method_trace_count 166 | uint32 :method_connect_count 167 | uint32 :method_other_count 168 | uint32 :status_1XX_count 169 | uint32 :status_2XX_count 170 | uint32 :status_3XX_count 171 | uint32 :status_4XX_count 172 | uint32 :status_5XX_count 173 | uint32 :status_other_count 174 | end 175 | 176 | # noinspection RubyResolve 177 | class LagPortStats < BinData::Record 178 | endian :big 179 | sflow_mac_address :dot3adAggPortActorSystemID 180 | skip :length => 2 181 | sflow_mac_address :dot3adAggPortPartnerOperSystemID 182 | skip :length => 2 183 | uint32 :dot3adAggPortAttachedAggID 184 | bit8 :dot3adAggPortActorAdminState 185 | bit8 :dot3adAggPortActorOperState 186 | bit8 :dot3adAggPortPartnerAdminState 187 | bit8 :dot3adAggPortPartnerOperState 188 | uint32 :dot3adAggPortStatsLACPDUsRx 189 | uint32 :dot3adAggPortStatsMarkerPDUsRx 190 | uint32 :dot3adAggPortStatsMarkerResponsePDUsRx 191 | uint32 :dot3adAggPortStatsUnknownRx 192 | uint32 :dot3adAggPortStatsIllegalRx 193 | uint32 :dot3adAggPortStatsLACPDUsTx 194 | uint32 :dot3adAggPortStatsMarkerPDUsTx 195 | uint32 :dot3adAggPortStatsMarkerResponsePDUsTx 196 | end 197 | 198 | # noinspection RubyResolve 199 | class HostDescr < BinData::Record 200 | endian :big 201 | sflow_string :hostname 202 | array :uuid, :type => :uint8, :initial_length => 16 203 | uint32 :machine_type 204 | uint32 :os_name 205 | sflow_string :os_release 206 | end 207 | 208 | # noinspection RubyResolve 209 | class HostAdapters < BinData::Record 210 | endian :big 211 | uint32 :adapters_count 212 | array :adapters, :initial_length => :adapters_count do 213 | uint32 :if_index 214 | uint32 :mac_address_count 215 | array :mac_addresses, :initial_length => :mac_address_count do 216 | sflow_mac_address :mac_address 217 | skip :length => 2 218 | end 219 | end 220 | end 221 | 222 | # noinspection RubyResolve 223 | class HostParent < BinData::Record 224 | endian :big 225 | uint32 :container_type 226 | uint32 :container_index 227 | end 228 | 229 | # noinspection RubyResolve 230 | class HostCpu < BinData::Record 231 | endian :big 232 | float_be :load_one 233 | float_be :load_five 234 | float_be :load_fifteen 235 | uint32 :proc_run 236 | uint32 :proc_total 237 | uint32 :cpu_num 238 | uint32 :cpu_speed 239 | uint32 :uptime 240 | uint32 :cpu_user 241 | uint32 :cpu_nice 242 | uint32 :cpu_system 243 | uint32 :cpu_idle 244 | uint32 :cpu_wio 245 | uint32 :cpu_intr 246 | uint32 :cpu_sintr 247 | uint32 :interrupts 248 | uint32 :contexts 249 | uint32 :cpu_steal 250 | uint32 :cpu_guest 251 | uint32 :cpu_guest_nice 252 | end 253 | 254 | # noinspection RubyResolve 255 | class HostMemory < BinData::Record 256 | endian :big 257 | uint64 :mem_total 258 | uint64 :mem_free 259 | uint64 :mem_shared 260 | uint64 :mem_buffers 261 | uint64 :mem_cached 262 | uint64 :swap_total 263 | uint64 :swap_free 264 | uint32 :page_in 265 | uint32 :page_out 266 | uint32 :swap_in 267 | uint32 :swap_out 268 | end 269 | 270 | # noinspection RubyResolve 271 | class HostDiskIo < BinData::Record 272 | endian :big 273 | uint64 :disk_total 274 | uint64 :disk_free 275 | uint32 :part_max_used_percent 276 | uint32 :reads 277 | uint64 :bytes_read 278 | uint32 :read_time 279 | uint32 :writes 280 | uint64 :bytes_written 281 | uint32 :write_time 282 | end 283 | 284 | # noinspection RubyResolve 285 | class HostNetIo < BinData::Record 286 | endian :big 287 | uint64 :bytes_in 288 | uint32 :pkts_in 289 | uint32 :errs_in 290 | uint32 :drops_in 291 | uint64 :bytes_out 292 | uint32 :packets_out 293 | uint32 :errs_out 294 | uint32 :drops_out 295 | end 296 | 297 | # noinspection RubyResolve 298 | class Mib2IpGroup < BinData::Record 299 | endian :big 300 | uint32 :ip_forwarding 301 | uint32 :ip_default_ttl 302 | uint32 :ip_in_receives 303 | uint32 :ip_in_hdr_errors 304 | uint32 :ip_in_addr_errors 305 | uint32 :ip_forw_datagrams 306 | uint32 :ip_in_unknown_protos 307 | uint32 :ip_in_discards 308 | uint32 :ip_in_delivers 309 | uint32 :ip_out_requests 310 | uint32 :ip_out_discards 311 | uint32 :ip_out_no_routes 312 | uint32 :ip_reasm_timeout 313 | uint32 :ip_reasm_reqds 314 | uint32 :ip_reasm_oks 315 | uint32 :ip_reasm_fails 316 | uint32 :ip_frag_oks 317 | uint32 :ip_frag_fails 318 | uint32 :ip_frag_creates 319 | end 320 | 321 | # noinspection RubyResolve 322 | class Mib2IcmpGroup < BinData::Record 323 | endian :big 324 | uint32 :icmp_in_msgs 325 | uint32 :icmp_in_errors 326 | uint32 :icmp_in_dest_unreachs 327 | uint32 :icmp_in_time_excds 328 | uint32 :icmp_in_param_probs 329 | uint32 :icmp_in_src_quenchs 330 | uint32 :icmp_in_redirects 331 | uint32 :icmp_in_echos 332 | uint32 :icmp_in_echo_reps 333 | uint32 :icmp_in_timestamps 334 | uint32 :icmp_in_addr_masks 335 | uint32 :icmp_in_addr_mask_reps 336 | uint32 :icmp_out_msgs 337 | uint32 :icmp_out_errors 338 | uint32 :icmp_out_dest_unreachs 339 | uint32 :icmp_out_time_excds 340 | uint32 :icmp_out_param_probs 341 | uint32 :icmp_out_src_quenchs 342 | uint32 :icmp_out_redirects 343 | uint32 :icmp_out_echos 344 | uint32 :icmp_out_echo_reps 345 | uint32 :icmp_out_timestamps 346 | uint32 :icmp_out_timestamp_reps 347 | uint32 :icmp_out_addr_masks 348 | uint32 :icmp_out_addr_mask_reps 349 | end 350 | 351 | # noinspection RubyResolve 352 | class Mib2TcpGroup < BinData::Record 353 | endian :big 354 | uint32 :tcp_rto_algorithm 355 | uint32 :tcp_rto_min 356 | uint32 :tcp_rto_max 357 | uint32 :tcp_max_conn 358 | uint32 :tcp_active_opens 359 | uint32 :tcp_passive_opens 360 | uint32 :tcp_attempt_fails 361 | uint32 :tcp_estab_resets 362 | uint32 :tcp_curr_estab 363 | uint32 :tcp_in_segs 364 | uint32 :tcp_out_segs 365 | uint32 :tcp_retrans_segs 366 | uint32 :tcp_in_errs 367 | uint32 :tcp_out_rsts 368 | uint32 :tcp_in_csum_errs 369 | end 370 | 371 | # noinspection RubyResolve 372 | class Mib2UdpGroup < BinData::Record 373 | endian :big 374 | uint32 :udp_in_datagrams 375 | uint32 :udp_no_ports 376 | uint32 :udp_in_errors 377 | uint32 :udp_out_datagrams 378 | uint32 :udp_rcvbuf_errors 379 | uint32 :udp_sndbuf_errors 380 | uint32 :udp_in_csum_errors 381 | end 382 | 383 | # noinspection RubyResolve 384 | class VirtNode < BinData::Record 385 | endian :big 386 | uint32 :mhz 387 | uint32 :cpus 388 | uint64 :memory 389 | uint64 :memory_free 390 | uint32 :num_domains 391 | end 392 | 393 | # noinspection RubyResolve 394 | class VirtCpu < BinData::Record 395 | endian :big 396 | uint32 :state 397 | uint32 :cpu_time 398 | uint32 :nr_virt_cpu 399 | end 400 | 401 | # noinspection RubyResolve 402 | class VirtMemory < BinData::Record 403 | endian :big 404 | uint64 :memory 405 | uint64 :max_memory 406 | end 407 | 408 | # noinspection RubyResolve 409 | class VirtDiskIo < BinData::Record 410 | endian :big 411 | uint64 :capacity 412 | uint64 :allocation 413 | uint64 :physical 414 | uint32 :rd_req 415 | uint64 :rd_bytes 416 | uint32 :wr_req 417 | uint64 :wr_bytes 418 | uint32 :errs 419 | end 420 | 421 | # noinspection RubyResolve 422 | class VirtNetIo < BinData::Record 423 | endian :big 424 | uint64 :rx_bytes 425 | uint32 :rx_packets 426 | uint32 :rx_errs 427 | uint32 :rx_drop 428 | uint64 :tx_bytes 429 | uint32 :tx_packets 430 | uint32 :tx_errs 431 | uint32 :tx_drop 432 | end 433 | 434 | # noinspection RubyResolve 435 | class OvsDpStats < BinData::Record 436 | endian :big 437 | uint32 :ovs_dp_hits 438 | uint32 :ovs_dp_misses 439 | uint32 :ovs_dp_lost 440 | uint32 :ovs_dp_mask_hits 441 | uint32 :ovs_dp_flows 442 | uint32 :ovs_dp_masks 443 | end 444 | --------------------------------------------------------------------------------