├── .dialyzer-ignore-warnings ├── .gitignore ├── .travis.yml ├── AUTHORS ├── ChangeLog ├── LICENSE ├── Makefile ├── README.md ├── include └── lwes.hrl ├── lwes-dev ├── lwes.config ├── rebar.config ├── rebar.lock ├── sonar-project.properties └── src ├── lwes.app.src ├── lwes.erl ├── lwes_app.erl ├── lwes_channel.erl ├── lwes_channel_manager.erl ├── lwes_channel_sup.erl ├── lwes_emitter.erl ├── lwes_emitter_stdout.erl ├── lwes_emitter_udp.erl ├── lwes_emitter_udp_pool.erl ├── lwes_esf_lexer.xrl ├── lwes_esf_parser.yrl ├── lwes_esf_validator.erl ├── lwes_event.erl ├── lwes_file_watcher.erl ├── lwes_internal.hrl ├── lwes_journal_listener.erl ├── lwes_journaller.erl ├── lwes_mochijson2.erl ├── lwes_mochinum.erl ├── lwes_multi_emitter.erl ├── lwes_net_udp.erl ├── lwes_stats.erl ├── lwes_sup.erl └── lwes_util.erl /.dialyzer-ignore-warnings: -------------------------------------------------------------------------------- 1 | leexinc.hrl:52: The pattern can never match the type <_,_,'error' | 'skip_token',[{'{{' | '{{#' | '{{/' | '{{^' | '{{{' | '}}' | '}}}',_} | {'key',_,atom()} | {'text',_,binary()}]> 2 | leexinc.hrl:54: The pattern can never match the type <_,_,'error' | 'skip_token',[{'{{' | '{{#' | '{{/' | '{{^' | '{{{' | '}}' | '}}}',_} | {'key',_,atom()} | {'text',_,binary()}]> 3 | leexinc.hrl:59: The pattern can never match the type <_,_,'error',[{'{{' | '{{#' | '{{/' | '{{^' | '{{{' | '}}' | '}}}',_} | {'key',_,atom()} | {'text',_,binary()}]> 4 | leexinc.hrl:62: The pattern <_Rest, Line, {'error', S}, _Ts> can never match the type <_,_,'error',[{'{{' | '{{#' | '{{/' | '{{^' | '{{{' | '}}' | '}}}',_} | {'key',_,atom()} | {'text',_,binary()}]> 5 | leexinc.hrl:121: The pattern can never match the type <_,_,'error' | 'skip_token'> 6 | leexinc.hrl:123: The pattern can never match the type <_,_,'error' | 'skip_token'> 7 | leexinc.hrl:128: The pattern can never match the type <_,_,'error'> 8 | leexinc.hrl:131: The pattern can never match the type <_,_,'error'> 9 | leexinc.hrl:195: The pattern can never match the type <_,_,'error' | 'skip_token',_> 10 | leexinc.hrl:197: The pattern can never match the type <_,_,'error' | 'skip_token',_> 11 | leexinc.hrl:202: The pattern can never match the type <_,_,'error',_> 12 | leexinc.hrl:205: The pattern can never match the type <_,_,'error',_> 13 | leexinc.hrl:246: The pattern can never match the type <_,_,'error' | 'skip_token',_> 14 | leexinc.hrl:248: The pattern can never match the type <_,_,'error' | 'skip_token',_> 15 | leexinc.hrl:253: The pattern can never match the type <_,_,'error',_> 16 | leexinc.hrl:256: The pattern can never match the type <_,_,'error',_> 17 | leexinc.hrl:260: Function yyrev/2 will never be called -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | _* 2 | deps 3 | doc 4 | ebin 5 | .eunit 6 | .rebar 7 | .rebar3 8 | src/lwes_esf_lexer.erl 9 | src/lwes_esf_parser.erl 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: erlang 2 | otp_release: 3 | - 18.3 4 | - 19.3 5 | - 20.3 6 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Anthony Molinaro 2 | -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- 1 | * Thu Feb 07 2019 Anthony Molinaro (djnym) 5.0.0 2 | - Only accept {Ip,Port} or {Ip,Port,ListOfOptions} form for UDP Address 3 | configuration - BREAKS Backward compatibility if using the old 3 or 4 element 4 | forms 5 | - support pluggable emitters via the lwes_emitter behaviour 6 | 7 | * Fri Dec 14 2018 Anthony Molinaro (djnym) 4.7.1 8 | - use rebar3 in Makefile 9 | 10 | * Tue Jul 24 2018 Anthony Molinaro (djnym) 4.7.0 11 | - fix for json output of arrays 12 | - rewrite type inference and add tests for it 13 | - fix for long strings input as a list 14 | - coverage improvements 15 | 16 | * Mon May 14 2018 Anthony Molinaro (djnym) 4.6.1 17 | - make dialyzer clean under rebar3 18 | - switch to different ChangeLog format 19 | - derive version in .app.src from ChangeLog 20 | 21 | * Thu Dec 21 2017 Anthony Molinaro (djnym) 4.6.0 22 | - Use a pool of sockets for emission. 23 | 24 | * Thu Dec 21 2017 Anthony Molinaro (djnym) 4.5.2 25 | - Support ttl for opening mondemand channels 26 | 27 | * Mon Dec 18 2017 Anthony Molinaro (djnym) 4.5.1 28 | - Need to support an older form of opening channels for mondemand 29 | 30 | * Wed Dec 13 2017 Anthony Molinaro (djnym) 4.5.0 31 | - Moved stats into an ETS table 32 | - Removed queue config type (it was never used) 33 | - Fix to lwes_event:to_iolist/1 where it was converting to binary 34 | - consolidated network code into new module 35 | - lots of unit testing added 36 | - changed crypto:rand_uniform/1 to rand:uniform/1 37 | 38 | * Thu Jul 27 2017 Tim Crowder (timrc) 4.4.1 39 | - Minor validator patches 40 | --- Log message for extra, unexpected fields. 41 | --- Check for duplicate fields. 42 | --- Add unit tests. 43 | 44 | * Sat Sep 17 2016 Anthony Molinaro (djnym) 4.4.0 45 | - change to allow emitting of iolists 46 | - function added to allow easy appending of headers to event 47 | - fixes for validation 48 | 49 | * Fri Aug 05 2016 Anthony Molinaro (djnym) 4.3.0 50 | - Support added for esf based validation 51 | 52 | * Fri Jun 17 2016 Tim Whalen (twhalen) 4.2.1 53 | - Fixed lwes:emit bug with lists of events 54 | 55 | * Wed May 11 2016 Anthony Molinaro (djnym) 4.2.0 (molinaro) 56 | - attempt to use SO_REUSEPORT if available 57 | - remove use of erlang:now() from lwes_journaller 58 | 59 | * Thu Aug 20 2015 Anthony Molinaro (djnym) 4.1.0 60 | - recbuf now configurable (and defaults to the larger size) 61 | - start at a journal_listener 62 | 63 | * Tue Aug 18 2015 Anthony Molinaro (djnym) 4.0.0 64 | - improved code coverage significantly, which fixed some issues? 65 | - various json formats were structured poorly 66 | - if incoming packet already has ReceiptTime, SenderIP and SenderPort 67 | at the end of the event, don't add them (to keep proxied events honest). 68 | This breaks backward compatibility in case someone was relying on the 69 | previous behavior which was that fields showed up twice. 70 | 71 | * Mon May 11 2015 Anthony Molinaro (djnym) 3.1.0 72 | - set default TTL for multicast to 5, also allow it to be overridden 73 | 74 | * Wed May 14 2014 Anthony Molinaro (djnym) 3.0.1 75 | - fix eunit test from old refactor 76 | 77 | * Thu May 01 2014 Anthony Molinaro (djnym) 3.0.0 78 | - allow for easy re-emission of events without deserializing 79 | - prevent some malformed events (breaks backward compatibility for users of 80 | lwes_event:peek_name_from_udp/1) 81 | 82 | * Wed Mar 05 2014 Vikram Kadi (vikramkadi) 2.4.0 83 | - New type long_string added to support strings bigger than 65kb 84 | 85 | * Wed Feb 19 2014 Anthony Molinaro (djnym) 2.3.1 86 | - there was an encoding bug with string types 87 | 88 | * Tue Mar 26 2013 Anthony Molinaro (djnym) 2.3.0 89 | - added array types 90 | 91 | * Mon Oct 01 2012 Anthony Molinaro (djnym) 2.2.0 92 | - lwes_multi_emitter now supports randomly selecting groups to send to 93 | - lwes_multi_emitter now supports groups (so nesting) 94 | - send and receive stats are not collected and viewable 95 | 96 | * Thu Sep 20 2012 Anthony Molinaro (djnym) 2.1.2 97 | - hopefully last change, eep18 format should return a top-level tuple 98 | 99 | * Thu Sep 20 2012 Anthony Molinaro (djnym) 2.1.1 100 | - did the last version too fast missed some cases where json was used 101 | 102 | * Thu Sep 20 2012 Anthony Molinaro (djnym) 2.1.0 103 | - proplists was actually wrong as ejson requires eep18 and as that my 104 | primary usecase for the json output, there are now 3 json decode formats 105 | json will return mochijson2 struct format, json_proplist will return 106 | the mochijson2 proplist formant, and json_eep18 will return the mochijson2 107 | eep18 format 108 | 109 | * Mon Aug 13 2012 Anthony Molinaro (djnym) 2.0.0 110 | - switch to decoding as proplists, instead of struct's 111 | - moved some files from mochiweb into lwes, so I could skip the dependency 112 | - don't check return code of send so that it doesn't crash lwes if the network is down 113 | 114 | * Tue Jun 28 2011 Anthony Molinaro (djnym) 1.0.3 115 | - missing dependency 116 | 117 | * Wed Apr 13 2011 Anthony Molinaro (djnym) 1.0.2 118 | - need to increase recbuf size in order to journal everything 119 | 120 | * Fri Mar 25 2011 Anthony Molinaro (djnym) 1.0.1 121 | - zero length strings should be allowed 122 | - for 'json' format, use mochijson2:decode for values 123 | 124 | * Sat Mar 19 2011 Anthony Moinaro (djnym) 1.0.0 125 | - changed lwes:open to take different number of args, breaks backward compatibility 126 | - add 'dict' as a format for event attributes in deserialization 127 | - added journaller 128 | - added multi emitter 129 | - added 'json' as a format for event deserialization 130 | 131 | * Wed Dec 22 2018 Anthony Molinaro (djnym) 0.0.0 132 | - Initial version 133 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Software Copyright License Agreement (BSD License) 2 | 3 | Copyright (c) 2010-2018, OpenX Technologies, Inc. 4 | All rights reserved. 5 | 6 | Redistribution and use of this software in source and binary forms, 7 | with or without modification, are permitted provided that the following 8 | conditions are met: 9 | 10 | * Redistributions of source code must retain the above 11 | copyright notice, this list of conditions and the 12 | following disclaimer. 13 | 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the 16 | following disclaimer in the documentation and/or other 17 | materials provided with the distribution. 18 | 19 | * Neither the name of OpenX Technologies, Inc. nor the names of its 20 | contributors may be used to endorse or promote products 21 | derived from this software without specific prior 22 | written permission of OpenX Technologies, Inc. 23 | 24 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS 25 | IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 26 | TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 27 | PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 28 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 29 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 30 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 31 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 32 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 33 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 34 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | NAME=lwes 2 | 3 | REBAR3=rebar3 4 | all: 5 | $(REBAR3) compile 6 | 7 | dialyzer: 8 | $(REBAR3) dialyzer 9 | 10 | test: dialyzer 11 | $(REBAR3) as test do eunit,cover 12 | 13 | # Compile and run unit test for individual modules: 'make test-oxgw_util' 14 | # or 'make test-oxgw_util test-oxgw_request' 15 | test-%: src/%.erl 16 | $(REBAR3) as test do eunit -m $* 17 | 18 | name: 19 | @echo $(NAME) 20 | 21 | version: 22 | @echo $(shell awk 'match($$0, /[0-9]+\.[0-9]+(\.[0-9]+)+/){print substr($$0, RSTART,RLENGTH); exit}' ChangeLog) 23 | 24 | clean: 25 | if test -d _build; then $(REBAR3) clean; fi 26 | 27 | maintainer-clean: clean 28 | rm -rf _build 29 | 30 | .PHONY: all test name version clean maintainer-clean 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Light Weight Event System (LWES) 2 | ================================ 3 | Click [here](http://lwes.github.io) for more information about lwes. 4 | For more information about using lwes from erlang read on. 5 | 6 | Creating Events 7 | ------------------------- 8 | There are 2 ways of creating events, the functional way 9 | 10 | ```erlang 11 | Event0 = lwes_event:new ("MyEvent"), 12 | Event1 = lwes_event:set_uint16 (Event0, "MyUint16", 25), 13 | ``` 14 | 15 | or via records like 16 | 17 | ```erlang 18 | Event = #lwes_event { 19 | name = "MyEvent", 20 | attrs = [{uint16, "MyUint16", 25}] 21 | }, 22 | ``` 23 | 24 | Emitting to a single channel 25 | ------------------------- 26 | If you are using multicast, or only want to emit to a single channel you 27 | can open it as follows 28 | 29 | ```erlang 30 | {ok, Channel0} = lwes:open (emitter, {Ip, Port}) 31 | Channel1 = lwes:emit (Channel0, Event1). 32 | ``` 33 | 34 | Emit to several channels 35 | ------------------------- 36 | If you aren't using multicast but would like to emit to several machines, 37 | or groups of machines you can with slightly different config, 38 | 39 | ```erlang 40 | % emit to 1 of a set in a round robin fashion 41 | {ok, Channels0} = lwes:open (emitters, {1, [{Ip1,Port1},...{IpN,PortN}]}) 42 | Channels1 = lwes:emit (Channels0, Event1) 43 | Channels2 = lwes:emit (Channels1, Event2) 44 | ... 45 | lwes:close (ChannelsN) 46 | 47 | % emit to 2 of a set in an m of n fashion (ie, emit to first 2 in list, 48 | % then 2nd and 3rd, then 3rd and 4th, etc., wraps at end of list) 49 | {ok, Channels0} = lwes:open (emitters, {2, [{Ip1,Port1},...{IpN,PortN}]}) 50 | ``` 51 | 52 | Listening via callback 53 | ------------------------- 54 | ```erlang 55 | {ok, Channel} = lwes:open (listener, {Ip, Port}) 56 | lwes:listen (Channel, Fun, Type, Accum). 57 | ``` 58 | Fun is called for each event 59 | 60 | Type is one of 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 |
rawcallback is given raw udp structure, use lwes_event:from_udp to turn into event
listcallback is given an #lwes_event record where the name is a binary, and the attributes is a proplist where keys are binaries, and values are either integers (for lwes int types), binaries (for lwes strings), true|false atoms (for lwes booleans), or 4-tuples (for lwes ip addresses)
taggedcallback is given an #lwes_event record where the name is a binary, and the attributes are 3-tuples with the first element the type of data, the second the key as a binary and the third the values as in the list format
dictcallback is given an #lwes_event record where the name is a binary, and the attributes are a dictionary with a binary key and value according to the type
jsonthis returns a proplist instead of an #lwes_event record. The valuse are mostly the same as list, but ip addresses are strings (as binary). This should means you can pass the returned value to mochijson2:encode (or other json encoders), and have the event as a json document
79 | 80 | Closing channel 81 | ------------------------- 82 | ```erlang 83 | lwes:close (Channel) 84 | ``` 85 | -------------------------------------------------------------------------------- /include/lwes.hrl: -------------------------------------------------------------------------------- 1 | -ifndef(_lwes_included). 2 | -define(_lwes_included, yup). 3 | 4 | -record (lwes_event, {name, attrs}). 5 | 6 | -define (LWES_U_INT_16, uint16). 7 | -define (LWES_INT_16, int16). 8 | -define (LWES_U_INT_32, uint32). 9 | -define (LWES_INT_32, int32). 10 | -define (LWES_INT_64, int64). 11 | -define (LWES_U_INT_64, uint64). 12 | -define (LWES_STRING, string). 13 | -define (LWES_IP_ADDR, ip_addr). 14 | -define (LWES_BOOLEAN, boolean). 15 | -define (LWES_BYTE, byte). 16 | -define (LWES_FLOAT, float). 17 | -define (LWES_DOUBLE, double). 18 | -define (LWES_LONG_STRING, long_string). 19 | -define (LWES_U_INT_16_ARRAY, uint16_array). 20 | -define (LWES_INT_16_ARRAY, int16_array). 21 | -define (LWES_U_INT_32_ARRAY, uint32_array). 22 | -define (LWES_INT_32_ARRAY, int32_array). 23 | -define (LWES_INT_64_ARRAY, int64_array). 24 | -define (LWES_U_INT_64_ARRAY, uint64_array). 25 | -define (LWES_STRING_ARRAY, string_array). 26 | -define (LWES_IP_ADDR_ARRAY, ip_addr_array). 27 | -define (LWES_BOOLEAN_ARRAY, boolean_array). 28 | -define (LWES_BYTE_ARRAY, byte_array). 29 | -define (LWES_FLOAT_ARRAY, float_array). 30 | -define (LWES_DOUBLE_ARRAY, double_array). 31 | -define (LWES_N_U_INT_16_ARRAY, nullable_uint16_array). 32 | -define (LWES_N_INT_16_ARRAY, nullable_int16_array). 33 | -define (LWES_N_U_INT_32_ARRAY, nullable_uint32_array). 34 | -define (LWES_N_INT_32_ARRAY, nullable_int32_array). 35 | -define (LWES_N_INT_64_ARRAY, nullable_int64_array). 36 | -define (LWES_N_U_INT_64_ARRAY, nullable_uint64_array). 37 | -define (LWES_N_STRING_ARRAY, nullable_string_array). 38 | % TODO: this is not implemented 39 | % -define (LWES_N_IP_ADDR_ARRAY, nullable_ip_addr_array). 40 | -define (LWES_N_BOOLEAN_ARRAY, nullable_boolean_array). 41 | -define (LWES_N_BYTE_ARRAY, nullable_byte_array). 42 | -define (LWES_N_FLOAT_ARRAY, nullable_float_array). 43 | -define (LWES_N_DOUBLE_ARRAY, nullable_double_array). 44 | 45 | -endif. 46 | -------------------------------------------------------------------------------- /lwes-dev: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | exec rebar3 shell \ 4 | --setcookie lwes \ 5 | --name 'lwes@localhost.localdomain' \ 6 | --config lwes.config \ 7 | --apps lwes 8 | -------------------------------------------------------------------------------- /lwes.config: -------------------------------------------------------------------------------- 1 | [ 2 | { lwes, [ ] } 3 | ]. 4 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {clean_files, ["ebin", "doc"]}. 2 | 3 | %{eunit_opts, [verbose,inorder]}. 4 | 5 | {cover_enabled, true}. 6 | {cover_opts, [verbose]}. 7 | {cover_export_enabled, true}. 8 | {cover_print_enabled, true}. 9 | 10 | % TODO: add 'bin_opt_info' below and fix 11 | {erl_opts, [debug_info]}. 12 | 13 | %% leex options 14 | {xrl_opts, []}. 15 | 16 | %% leex files to compile first 17 | {xrl_first_files, ["src/lwes_esf_lexer.xrl"]}. 18 | 19 | %% yecc options 20 | {yrl_opts, []}. 21 | 22 | %% yecc files to compile first 23 | {yrl_first_files, ["src/lwes_esf_parser.yrl"]}. 24 | 25 | % % for my own development, I like to use reloader 26 | % {erl_opts, [debug_info, {d, 'TEST'}]}. 27 | {deps, 28 | [ 29 | % { mochiweb_reloader, 30 | % "2.3.1", 31 | % {git, "git://github.com/ostinelli/mochiweb_reloader.git", {tag, "2.3.1"}} 32 | % } 33 | ] 34 | }. 35 | -------------------------------------------------------------------------------- /rebar.lock: -------------------------------------------------------------------------------- 1 | []. 2 | -------------------------------------------------------------------------------- /sonar-project.properties: -------------------------------------------------------------------------------- 1 | # Required metadata 2 | sonar.projectKey=org.lwes:lwes-erlang 3 | sonar.projectName=lwes-erlang 4 | sonar.projectVersion=4.4.1 5 | 6 | # Comma-separated paths to directories with sources (required) 7 | sonar.sources=src 8 | 9 | # Language 10 | sonar.language=erlang 11 | 12 | # Encoding of sources files 13 | sonar.sourceEncoding=ISO-8859-1 14 | -------------------------------------------------------------------------------- /src/lwes.app.src: -------------------------------------------------------------------------------- 1 | {application, lwes, 2 | [{description, "Light Weight Event System Erlang Bindings"}, 3 | {vsn, {cmd, "/bin/bash -c 'awk \"match(\\$0, /[0-9]+\\.[0-9]+(\\.[0-9]+)+/){print substr(\\$0, RSTART,RLENGTH); exit}\" ChangeLog'"}}, 4 | {modules, []}, 5 | {registered, [lwes_channel_manager,lwes_channel_sup,lwes_sup,lwes_stats,lwes_esf_validator]}, 6 | {mod, {lwes_app, []}}, 7 | {env, []}, 8 | {applications, [kernel, stdlib]}]}. 9 | -------------------------------------------------------------------------------- /src/lwes.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Light Weight Event System (LWES) 3 | %%% 4 | %%% Creating Events 5 | %%% Event0 = lwes_event:new ("MyEvent"), 6 | %%% Event1 = lwes_event:set_uint16 (Event0, "MyUint16", 25), 7 | %%% 8 | %%% Emitting to a single channel 9 | %%% 10 | %%% {ok, Channel0} = lwes:open (emitter, {Ip, Port}) 11 | %%% Channel1 = lwes:emit (Channel0, Event1). 12 | %%% 13 | %%% Emit to several channels 14 | %%% 15 | %%% % emit to 1 of a set in a round robin fashion 16 | %%% {ok, Channels0} = lwes:open (emitters, {1, [{Ip1,Port1},...{IpN,PortN}]}) 17 | %%% Channels1 = lwes:emit (Channels0, Event1) 18 | %%% Channels2 = lwes:emit (Channels1, Event2) 19 | %%% ... 20 | %%% lwes:close (ChannelsN) 21 | %%% 22 | %%% % emit to 2 of a set in an m of n fashion (ie, emit to first 2 in list, 23 | %%% % then 2nd and 3rd, then 3rd and 4th, etc., wraps at end of list) 24 | %%% {ok, Channels0} = lwes:open (emitters, {2, [{Ip1,Port1},...{IpN,PortN}]}) 25 | %%% 26 | %%% Listening via callback 27 | %%% 28 | %%% {ok, Channel} = lwes:open (listener, {Ip, Port}) 29 | %%% lwes:listen (Channel, Fun, Type, Accum). 30 | %%% 31 | %%% Fun is called for each event 32 | %%% 33 | %%% Closing channel 34 | %%% 35 | %%% lwes:close (Channel) 36 | 37 | -module (lwes). 38 | 39 | -include_lib ("lwes.hrl"). 40 | -include ("lwes_internal.hrl"). 41 | 42 | %% API 43 | -export ([ start/0, 44 | open/2, % (Type, Config) -> {ok, Channel} 45 | emit/2, 46 | emit/3, 47 | listen/4, 48 | close/1, 49 | stats/0, 50 | stats_raw/0, 51 | enable_validation/1 ]). 52 | 53 | %%==================================================================== 54 | %% API functions 55 | %%==================================================================== 56 | 57 | start () -> 58 | application:start (lwes). 59 | 60 | % 61 | % open an lwes emitter, listener or set of emitters 62 | % 63 | % config for emitter/listener is 64 | % { Ip, Port } 65 | % 66 | % config for groups is 67 | % { NumberOfGroupsToSendTo, 68 | % group, 69 | % [ 70 | % { NumberToSendToInThisGroup, 71 | % Type, 72 | % [ 73 | % {Ip0,Port0}, 74 | % ... 75 | % {IpN,PortN} 76 | % ] 77 | % }, 78 | % ... 79 | % ] 80 | % } 81 | % an example group emission might be 82 | % { 2, 83 | % group, 84 | % [ 85 | % { 1, 86 | % random, 87 | % [ {Ip0, Port0}, 88 | % ... 89 | % {IpN, PortN} 90 | % ] 91 | % }, 92 | % { 1, 93 | % random, 94 | % [ {IpN+1, PortN+1}, 95 | % ... 96 | % {IpN+M, PortN+M} 97 | % ] 98 | % } 99 | % ] 100 | % } 101 | % which should send each event to one machine in each group 102 | 103 | open (emitter, Config) -> 104 | open (emitters, Config); 105 | open (emitters, Config) -> 106 | lwes_multi_emitter:new (Config); 107 | open (listener, Config) -> 108 | try lwes_channel:new (listener, Config) of 109 | C -> lwes_channel:open (C) 110 | catch 111 | _:_ -> { error, bad_ip_port } 112 | end; 113 | open (_, _) -> 114 | { error, bad_type }. 115 | 116 | % emit an array of events 117 | emit (ChannelsIn, []) -> 118 | ChannelsIn; 119 | 120 | emit (ChannelsIn, [ HeadEvent = #lwes_event{} | TailEvents]) -> 121 | ChannelsOut = emit (ChannelsIn, HeadEvent), 122 | emit (ChannelsOut, TailEvents); 123 | 124 | % emit an event to one or more channels 125 | emit (Channel, Event) when is_record (Channel, lwes_channel) -> 126 | lwes_channel:send_to (Channel, lwes_event:to_binary (Event)), 127 | % channel doesn't actually change for a single emitter 128 | Channel; 129 | emit (StateIn = {Module,_}, Event) when is_atom (Module) -> 130 | Module:emit (StateIn, Event); 131 | emit (Channels, Event) -> 132 | lwes_multi_emitter:emit (Channels, Event), 133 | Channels. 134 | 135 | % emit an event to one or more channels 136 | emit (Channel, Event, SpecName) -> 137 | case lwes_esf_validator:validate (SpecName, Event) of 138 | ok -> 139 | emit (Channel, Event); 140 | {error, Error} -> 141 | error_logger:error_msg("Event validation error ~p", [Error]), 142 | Channel 143 | end. 144 | % 145 | % listen for events 146 | % 147 | % Callback function - function is called with an event in given format 148 | % and the current state, it should return the next 149 | % state 150 | % 151 | % Type is one of 152 | % 153 | % raw - callback is given raw udp structure, use lwes_event:from_udp to 154 | % turn into event 155 | % list - callback is given an #lwes_event record where the name is a 156 | % binary, and the attributes is a proplist where keys are binaries, 157 | % and values are either integers (for lwes int types), binaries 158 | % (for lwes strings), true|false atoms (for lwes booleans), 159 | % or 4-tuples (for lwes ip addresses) 160 | % tagged - callback is given an #lwes_event record where the name is a 161 | % binary, and the attributes are 3-tuples with the first element 162 | % the type of data, the second the key as a binary and the 163 | % third the values as in the list format 164 | % dict - callback is given an #lwes_event record where the name is a 165 | % binary, and the attributes are a dictionary with a binary 166 | % key and value according to the type 167 | % json - this returns a proplist instead of an #lwes_event record. The 168 | % valuse are mostly the same as list, but ip addresses are strings 169 | % (as binary). This should means you can pass the returned value 170 | % to mochijson2:encode (or other json encoders), and have the event 171 | % as a json document 172 | % json_eep18 - uses the eep18 format of mochijson2 decode 173 | % json_proplist - uses the proplist format of mochijson2 decode 174 | % 175 | % Initial State is whatever you want 176 | listen (Channel, CallbackFunction, EventType, CallbackInitialState) 177 | when is_function (CallbackFunction, 2), 178 | EventType =:= raw ; EventType =:= tagged ; 179 | EventType =:= list ; EventType =:= dict ; 180 | EventType =:= json ; EventType =:= json_proplist ; 181 | EventType =:= json_eep18 -> 182 | lwes_channel:register_callback (Channel, CallbackFunction, 183 | EventType, CallbackInitialState). 184 | 185 | % close the channel or channels 186 | close (Channel) when is_record (Channel, lwes_channel) -> 187 | lwes_channel:close (Channel); 188 | close (StateIn = {Module,_}) when is_atom (Module) -> 189 | Module:close (StateIn); 190 | close (Channels) -> 191 | lwes_multi_emitter:close (Channels). 192 | 193 | stats () -> 194 | case application:get_application (lwes) of 195 | undefined -> {error, {not_started, lwes}}; 196 | _ -> 197 | lwes_stats:print(none), 198 | ok 199 | end. 200 | 201 | stats_raw () -> 202 | case application:get_application (lwes) of 203 | undefined -> {error, {not_started, lwes}}; 204 | _ -> 205 | lwes_stats:rollup(none) 206 | end. 207 | 208 | % 209 | % enable validation of the events sent via this LWES client 210 | % against the specification (ESF) 211 | % 212 | % ESFInfo - a list of tuples of the form { Name, FilePath } 213 | % 214 | % Example : [{Name1, path1}, {Name2, path2}] 215 | % 216 | % 'Name' is used to match the 'event' with a particular 217 | % ESF File 218 | 219 | % 'FilePath' is the path to the ESF File 220 | enable_validation (ESFInfo) -> 221 | lists:foreach ( 222 | fun ({ESF, File}) -> lwes_esf_validator:add_esf (ESF, File) end, 223 | ESFInfo). 224 | 225 | %%==================================================================== 226 | %% Internal functions 227 | %%==================================================================== 228 | 229 | %%==================================================================== 230 | %% Test functions 231 | %%==================================================================== 232 | -ifdef (TEST). 233 | -include_lib ("eunit/include/eunit.hrl"). 234 | 235 | -define(TEST_TABLE, lwes_test). 236 | 237 | setup() -> 238 | ok = application:start(lwes), 239 | ets:new(?TEST_TABLE,[named_table,public]), 240 | ok. 241 | 242 | teardown(ok) -> 243 | ets:delete(?TEST_TABLE), 244 | application:stop(lwes). 245 | 246 | build_one (EventsToSend, PerfectPercent, 247 | EmitterConfig, EmitterType, ListenerConfigs) -> 248 | fun() -> 249 | Listeners = 250 | [ begin 251 | {ok, L} = open(listener, LC), 252 | listen (L, 253 | fun({udp,_,_,_,E},A) -> 254 | [{E}] = ets:lookup(?TEST_TABLE, E), 255 | A 256 | end, 257 | raw, ok), 258 | L 259 | end 260 | || LC <- ListenerConfigs 261 | ], 262 | {ok, Emitter0} = open(EmitterType,EmitterConfig), 263 | 264 | InitialEvent = lwes_event:new("foo"), 265 | EmitterFinal = 266 | lists:foldl( 267 | fun (C, EmitterIn) -> 268 | EventWithCounter = lwes_event:set_uint16(InitialEvent,"bar",C), 269 | Event = lwes_event:to_binary(EventWithCounter), 270 | ets:insert(?TEST_TABLE, {Event}), 271 | emit(EmitterIn, Event) 272 | end, 273 | Emitter0, 274 | lists:seq(1,EventsToSend) 275 | ), 276 | timer:sleep(500), 277 | close(EmitterFinal), 278 | [ close(L) || L <- Listeners ], 279 | 280 | Rollup = lwes_stats:rollup(none), 281 | {Sent,Received} = 282 | lists:foldl (fun ([_,S,R,_,_,_],{AS,AR}) -> 283 | % calculated the actual percent received 284 | Percent = S/EventsToSend*100, 285 | % and then check if the difference is within 286 | % 15 percent which is about what I was seeing 287 | % while testing 288 | WithinBound = abs(PerfectPercent-Percent) < 15, 289 | case WithinBound of 290 | true -> ok; 291 | false -> 292 | ?debugFmt("WARNING: out of bounds : " 293 | "abs(~p-~p) < 15 => ~p~n", 294 | [PerfectPercent, Percent, WithinBound]), 295 | ok 296 | end, 297 | ?assert(WithinBound), 298 | {AS + S, AR + R} 299 | end, 300 | {0,0}, 301 | Rollup 302 | ), 303 | ?assertEqual(Sent, Received) 304 | end. 305 | 306 | simple_test_ () -> 307 | NumberToSendTo = 1, 308 | EventsToSend = 100, 309 | EmitterConfig = {"127.0.0.1",12321}, 310 | EmitterType = emitter, 311 | ListenerConfigs = [ EmitterConfig ], 312 | PerfectPercent = EventsToSend / length(ListenerConfigs) * NumberToSendTo, 313 | { setup, 314 | fun setup/0, 315 | fun teardown/1, 316 | [ 317 | build_one (EventsToSend, PerfectPercent, 318 | EmitterConfig, EmitterType, ListenerConfigs) 319 | ] 320 | }. 321 | 322 | mondemand_w_ttl_test_ () -> 323 | Address = {"127.0.0.1",12321, [{ttl,25}]}, 324 | NumberToSendTo = 1, 325 | EventsToSend = 100, 326 | EmitterConfig = {NumberToSendTo, [Address]}, 327 | EmitterType = emitters, 328 | ListenerConfigs = [ Address ], 329 | PerfectPercent = EventsToSend / length(ListenerConfigs) * NumberToSendTo, 330 | { setup, 331 | fun setup/0, 332 | fun teardown/1, 333 | [ 334 | build_one (EventsToSend, PerfectPercent, 335 | EmitterConfig, EmitterType, ListenerConfigs) 336 | ] 337 | }. 338 | 339 | 340 | mondemand_legacy_test_ () -> 341 | Address = {"127.0.0.1",12321}, 342 | NumberToSendTo = 1, 343 | EventsToSend = 100, 344 | % mondemand used the queue form which I got rid of, now I'm converting 345 | % those to group configs, so the following should work 346 | EmitterConfig = {NumberToSendTo, [Address]}, 347 | EmitterType = emitters, 348 | ListenerConfigs = [ Address ], 349 | PerfectPercent = EventsToSend / length(ListenerConfigs) * NumberToSendTo, 350 | { setup, 351 | fun setup/0, 352 | fun teardown/1, 353 | [ 354 | build_one (EventsToSend, PerfectPercent, 355 | EmitterConfig, EmitterType, ListenerConfigs) 356 | ] 357 | }. 358 | 359 | multi_random_test_ () -> 360 | NumberToSendTo = 2, 361 | ListenerConfigs = [ {"127.0.0.1", 30000}, 362 | {"127.0.0.1", 30001}, 363 | {"127.0.0.1", 30002} 364 | ], 365 | EmitterConfig = { NumberToSendTo, random, ListenerConfigs }, 366 | EmitterType = emitters, 367 | EventsToSend = 100, 368 | % calculate the perfect number of events per listener we should get 369 | % this is used below 370 | PerfectPercent = EventsToSend / length(ListenerConfigs) * NumberToSendTo, 371 | { setup, 372 | fun setup/0, 373 | fun teardown/1, 374 | [ 375 | build_one (EventsToSend, PerfectPercent, 376 | EmitterConfig, EmitterType, ListenerConfigs) 377 | ] 378 | }. 379 | 380 | grouped_random_test_ () -> 381 | NumberToSendTo = 3, 382 | Group1Config = [ { "127.0.0.1", 5301 }, 383 | { "127.0.0.1", 5302 } 384 | ], 385 | Group2Config = [ { "127.0.0.1", 5303 }, 386 | { "127.0.0.1", 5304 } 387 | ], 388 | Group3Config = [ { "127.0.0.1", 5305 }, 389 | { "127.0.0.1", 5306 } 390 | ], 391 | ListenerConfigs = Group1Config ++ Group2Config ++ Group3Config, 392 | EmitterConfig = { NumberToSendTo, group, 393 | [ 394 | { 1, random, Group1Config }, 395 | { 1, random, Group2Config }, 396 | { 1, random, Group3Config } 397 | ] 398 | }, 399 | EmitterType = emitters, 400 | EventsToSend = 100, 401 | PerfectPercent = EventsToSend / length(ListenerConfigs) * NumberToSendTo, 402 | { setup, 403 | fun setup/0, 404 | fun teardown/1, 405 | [ 406 | build_one (EventsToSend, PerfectPercent, 407 | EmitterConfig, EmitterType, ListenerConfigs) 408 | ] 409 | }. 410 | 411 | 412 | -endif. 413 | -------------------------------------------------------------------------------- /src/lwes_app.erl: -------------------------------------------------------------------------------- 1 | -module (lwes_app). 2 | 3 | -behaviour (application). 4 | 5 | %% API 6 | -export([start/0]). 7 | 8 | %% application callbacks 9 | -export ([start/2, stop/1]). 10 | 11 | %-=====================================================================- 12 | %- API - 13 | %-=====================================================================- 14 | start () -> 15 | application:ensure_all_started(lwes). 16 | 17 | %-=====================================================================- 18 | %- application callbacks - 19 | %-=====================================================================- 20 | start (_Type, _Args) -> 21 | lwes_sup:start_link(). 22 | 23 | stop (_State) -> 24 | ok. 25 | 26 | %-=====================================================================- 27 | %- Private - 28 | %-=====================================================================- 29 | 30 | %-=====================================================================- 31 | %- Test Functions - 32 | %-=====================================================================- 33 | -ifdef (TEST). 34 | -include_lib ("eunit/include/eunit.hrl"). 35 | 36 | lwes_app_test_ () -> 37 | [ 38 | ?_assertEqual ({ok, [lwes]},lwes_app:start()), 39 | ?_assertEqual ({ok, []},lwes_app:start()), 40 | ?_assertEqual (ok, application:stop (lwes)) 41 | ]. 42 | 43 | -endif. 44 | -------------------------------------------------------------------------------- /src/lwes_channel.erl: -------------------------------------------------------------------------------- 1 | -module (lwes_channel). 2 | 3 | -behaviour (gen_server). 4 | 5 | -include_lib ("lwes.hrl"). 6 | -include ("lwes_internal.hrl"). 7 | 8 | %% API 9 | -export ([ start_link/1, 10 | new/2, 11 | open/1, 12 | register_callback/4, 13 | send_to/2, 14 | close/1 15 | ]). 16 | 17 | %% gen_server callbacks 18 | -export ([ init/1, 19 | handle_call/3, 20 | handle_cast/2, 21 | handle_info/2, 22 | terminate/2, 23 | code_change/3 24 | ]). 25 | 26 | -record (state, {socket, channel, type, callback}). 27 | -record (callback, {function, format, state}). 28 | 29 | %%==================================================================== 30 | %% API functions 31 | %%==================================================================== 32 | start_link (Channel) -> 33 | gen_server:start_link (?MODULE, [Channel], []). 34 | 35 | new (Type, Config) -> 36 | #lwes_channel { 37 | type = Type, 38 | config = lwes_net_udp:new (Type, Config), 39 | ref = make_ref() 40 | }. 41 | 42 | open (Channel) -> 43 | { ok, _Pid } = lwes_channel_manager:open_channel (Channel), 44 | { ok, Channel}. 45 | 46 | register_callback (Channel, CallbackFunction, EventType, CallbackState) -> 47 | find_and_call ( Channel, 48 | { register, CallbackFunction, EventType, CallbackState }). 49 | 50 | send_to (Channel, Msg) -> 51 | find_and_call (Channel, { send, Msg }). 52 | 53 | close (Channel) -> 54 | find_and_cast (Channel, stop). 55 | 56 | %%==================================================================== 57 | %% gen_server callbacks 58 | %%==================================================================== 59 | init ([ Channel = #lwes_channel { type = Type, config = Config } ]) -> 60 | { ok, Socket } = lwes_net_udp:open (Type, Config), 61 | lwes_stats:initialize (lwes_net_udp:address(Config)), 62 | lwes_channel_manager:register_channel (Channel, self()), 63 | { ok, #state { socket = Socket, 64 | channel = Channel, 65 | type = Type 66 | } 67 | }. 68 | 69 | handle_call ({ register, Function, Format, Accum }, 70 | _From, 71 | State = #state { 72 | channel = #lwes_channel {type = listener } 73 | }) -> 74 | { reply, 75 | ok, 76 | State#state { callback = #callback { function = Function, 77 | format = Format, 78 | state = Accum } } }; 79 | 80 | handle_call ({ send, Packet }, 81 | _From, 82 | State = #state { 83 | socket = Socket, 84 | channel = #lwes_channel { config = Config } 85 | }) -> 86 | Reply = 87 | case lwes_net_udp:send (Socket, Config, Packet) of 88 | ok -> 89 | lwes_stats:increment_sent(lwes_net_udp:address(Config)), 90 | ok; 91 | {error, Error} -> 92 | lwes_stats:increment_errors(lwes_net_udp:address(Config)), 93 | {error, Error} 94 | end, 95 | { reply, Reply, State }; 96 | handle_call (_Request, _From, State) -> 97 | { reply, ok, State }. 98 | 99 | handle_cast (stop, State) -> 100 | {stop, normal, State}; 101 | handle_cast (_Request, State) -> 102 | { noreply, State }. 103 | 104 | % skip if we don't have a handler 105 | handle_info ( {udp, _, _, _, _}, 106 | State = #state { 107 | type = listener, 108 | channel = #lwes_channel { config = Config }, 109 | callback = undefined 110 | } ) -> 111 | lwes_stats:increment_received (lwes_net_udp:address(Config)), 112 | { noreply, State }; 113 | 114 | handle_info ( Packet = {udp, _, _, _, _}, 115 | State = #state { 116 | type = listener, 117 | channel = #lwes_channel { config = Config }, 118 | callback = #callback { function = Function, 119 | format = Format, 120 | state = CbState } 121 | } ) -> 122 | lwes_stats:increment_received (lwes_net_udp:address(Config)), 123 | Event = lwes_event:from_udp_packet (Packet, Format), 124 | NewCbState = Function (Event, CbState), 125 | { noreply, 126 | State#state { callback = #callback { function = Function, 127 | format = Format, 128 | state = NewCbState } 129 | } 130 | }; 131 | 132 | handle_info (_Request, State) -> 133 | {noreply, State}. 134 | 135 | terminate (_Reason, #state {socket = Socket, channel = Channel}) -> 136 | lwes_net_udp:close (Socket), 137 | lwes_channel_manager:unregister_channel (Channel). 138 | 139 | code_change (_OldVsn, State, _Extra) -> 140 | {ok, State}. 141 | 142 | %%==================================================================== 143 | %% Internal functions 144 | %%==================================================================== 145 | find_and_call (Channel, Msg) -> 146 | case lwes_channel_manager:find_channel (Channel) of 147 | {error, not_open} -> 148 | {error, not_open}; 149 | Pid -> 150 | gen_server:call ( Pid, Msg ) 151 | end. 152 | 153 | find_and_cast (Channel, Msg) -> 154 | case lwes_channel_manager:find_channel (Channel) of 155 | {error, not_open} -> 156 | {error, not_open}; 157 | Pid -> 158 | gen_server:cast ( Pid, Msg ) 159 | end. 160 | 161 | %%==================================================================== 162 | %% Test functions 163 | %%==================================================================== 164 | -ifdef (TEST). 165 | -include_lib ("eunit/include/eunit.hrl"). 166 | 167 | -endif. 168 | -------------------------------------------------------------------------------- /src/lwes_channel_manager.erl: -------------------------------------------------------------------------------- 1 | -module (lwes_channel_manager). 2 | 3 | -behaviour (gen_server). 4 | 5 | -include_lib ("lwes.hrl"). 6 | -include_lib ("lwes_internal.hrl"). 7 | 8 | %% API 9 | -export ([ start_link/0, 10 | open_channel/1, 11 | register_channel/2, 12 | unregister_channel/1, 13 | find_channel/1, 14 | close_channel/1 15 | ]). 16 | 17 | %% gen_server callbacks 18 | -export ([ init/1, 19 | handle_call/3, 20 | handle_cast/2, 21 | handle_info/2, 22 | terminate/2, 23 | code_change/3 24 | ]). 25 | 26 | -define (TABLE, lwes_channels). 27 | -record (state, {}). 28 | 29 | %%==================================================================== 30 | %% API 31 | %%==================================================================== 32 | start_link () -> 33 | gen_server:start_link ( { local, ?MODULE }, ?MODULE, [], []). 34 | 35 | open_channel (Channel) -> 36 | lwes_channel_sup:open_channel (Channel). 37 | 38 | register_channel (Channel, Pid) -> 39 | gen_server:call (?MODULE, {reg, Channel, Pid}). 40 | 41 | unregister_channel (Channel) -> 42 | gen_server:call (?MODULE, {unreg, Channel}). 43 | 44 | find_channel (Channel) -> 45 | case ets:lookup (?TABLE, Channel) of 46 | [] -> {error, not_open} ; 47 | [{_Channel, Pid}] -> Pid 48 | end. 49 | 50 | close_channel (Channel) -> 51 | gen_server:call (find_channel (Channel), stop). 52 | 53 | %%==================================================================== 54 | %% gen_server callbacks 55 | %%==================================================================== 56 | init ([]) -> 57 | ets:new (?TABLE, [ named_table, { read_concurrency, true } ]), 58 | { ok, #state {} }. 59 | 60 | handle_call ({reg, Key, Val}, _From, State) -> 61 | { reply, ets:insert (?TABLE, {Key, Val}), State }; 62 | handle_call ({unreg, Key}, _From, State) -> 63 | {reply, ets:delete (?TABLE, Key), State }; 64 | handle_call (Request, From, State) -> 65 | error_logger:warning_msg 66 | ("lwes_channel_manager unrecognized call ~p from ~p~n",[Request, From]), 67 | { reply, ok, State }. 68 | 69 | handle_cast (Request, State) -> 70 | error_logger:warning_msg 71 | ("lwes_channel_manager unrecognized cast ~p~n",[Request]), 72 | { noreply, State }. 73 | 74 | handle_info (Request, State) -> 75 | error_logger:warning_msg 76 | ("lwes_channel_manager unrecognized info ~p~n",[Request]), 77 | {noreply, State}. 78 | 79 | terminate (_Reason, _State) -> 80 | ets:delete (?TABLE), 81 | ok. 82 | 83 | code_change (_OldVsn, State, _Extra) -> 84 | {ok, State}. 85 | 86 | %%==================================================================== 87 | %% Internal functions 88 | %%==================================================================== 89 | 90 | %%==================================================================== 91 | %% Test functions 92 | %%==================================================================== 93 | -ifdef (TEST). 94 | -include_lib ("eunit/include/eunit.hrl"). 95 | 96 | -endif. 97 | -------------------------------------------------------------------------------- /src/lwes_channel_sup.erl: -------------------------------------------------------------------------------- 1 | -module (lwes_channel_sup). 2 | 3 | -behaviour (supervisor). 4 | 5 | %% API 6 | -export ([ start_link/0, 7 | open_channel/1 ]). 8 | 9 | %% supervisor callbacks 10 | -export ([ init/1 ]). 11 | 12 | %%==================================================================== 13 | %% API functions 14 | %%==================================================================== 15 | %% @spec start_link() -> ServerRet 16 | %% @doc API for starting the supervisor. 17 | start_link() -> 18 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 19 | 20 | open_channel (Channel) -> 21 | supervisor:start_child (?MODULE, [Channel]). 22 | 23 | %%==================================================================== 24 | %% supervisor callbacks 25 | %%==================================================================== 26 | %% @spec init([]) -> SupervisorTree 27 | %% @doc supervisor callback. 28 | init([]) -> 29 | { ok, 30 | { 31 | {simple_one_for_one, 10, 10}, 32 | [ { lwes_channel, 33 | {lwes_channel, start_link, []}, 34 | transient, 35 | 2000, 36 | worker, 37 | [lwes_channel] 38 | } 39 | ] 40 | } 41 | }. 42 | 43 | %%==================================================================== 44 | %% Test functions 45 | %%==================================================================== 46 | -ifdef (TEST). 47 | -include_lib ("eunit/include/eunit.hrl"). 48 | 49 | -endif. 50 | -------------------------------------------------------------------------------- /src/lwes_emitter.erl: -------------------------------------------------------------------------------- 1 | -module(lwes_emitter). 2 | 3 | % create any resources and return a fixed structure which will be sent in 4 | % subsequent calls, configuration is expected to be a list of some sort, 5 | % most of the time this will be something like [{key, value}]. 6 | -callback new(Config :: list()) -> State :: term(). 7 | 8 | % return an id to be used for stats gathering. In most cases this should 9 | % be an atom so that it will show up under the label column for lwes:stats() 10 | % calls 11 | -callback id(State :: term()) -> 12 | {Ip :: tuple(), Port :: integer() } 13 | | {Label :: atom(), {Ip :: tuple(), Port :: integer() }} 14 | | atom(). 15 | 16 | % prepare the event, in most cases this will use one of the to_* methods 17 | % in lwes_event (like to_binary, to_iolist, to_json, etc), but also allows 18 | % one to implement any serialization format as a plugin 19 | -callback prep(Event :: term()) -> term(). 20 | 21 | % emit the event, this will get the State from new/1 as well as the event 22 | % from prep/1 23 | -callback emit(State :: term(), Event :: term()) -> ok | {error, atom()}. 24 | 25 | % close the emitter, called on shutdown or if configuration is updated 26 | % this could be called, then immediately new/1 called again. 27 | -callback close(State :: term()) -> ok. 28 | -------------------------------------------------------------------------------- /src/lwes_emitter_stdout.erl: -------------------------------------------------------------------------------- 1 | -module (lwes_emitter_stdout). 2 | 3 | -behaviour (lwes_emitter). 4 | 5 | -export ([ new/1, 6 | id/1, 7 | prep/1, 8 | emit/2, 9 | close/1 10 | ]). 11 | 12 | -record (lwes_emitter_stdout, {config}). 13 | 14 | new (Config) -> 15 | #lwes_emitter_stdout { config = Config }. 16 | 17 | id(#lwes_emitter_stdout {config = [{label,B}]}) -> 18 | B. 19 | 20 | prep (Event) -> 21 | lwes_event:to_binary(Event). 22 | 23 | emit (L, Event) when is_list(L) -> 24 | [ emit(E, Event) || E <- L ]; 25 | emit (#lwes_emitter_stdout {}, E) -> 26 | io:format("Emit event ~p for ~p~n",[E,?MODULE]), 27 | ok. 28 | 29 | close (#lwes_emitter_stdout {}) -> 30 | ok. 31 | -------------------------------------------------------------------------------- /src/lwes_emitter_udp.erl: -------------------------------------------------------------------------------- 1 | -module (lwes_emitter_udp). 2 | 3 | -behaviour (lwes_emitter). 4 | 5 | -export ([ new/1, 6 | id/1, 7 | prep/1, 8 | emit/2, 9 | close/1 10 | ]). 11 | 12 | new (Config) -> 13 | lwes_net_udp:new (emitter, Config). 14 | 15 | id (Config) -> 16 | lwes_net_udp:address(Config). 17 | 18 | prep (Event) -> 19 | lwes_event:to_iolist(Event). 20 | 21 | emit (Config, Event) -> 22 | Address = lwes_net_udp:address(Config), 23 | lwes_emitter_udp_pool:send (lwes_emitters, Address, Event). 24 | 25 | close (_Config) -> 26 | ok. 27 | -------------------------------------------------------------------------------- /src/lwes_emitter_udp_pool.erl: -------------------------------------------------------------------------------- 1 | -module (lwes_emitter_udp_pool). 2 | 3 | -behaviour (gen_server). 4 | 5 | %% API 6 | -export ([start_link/1, 7 | send/3 8 | ]). 9 | 10 | -export ([init/1, 11 | handle_call/3, 12 | handle_cast/2, 13 | handle_info/2, 14 | terminate/2, 15 | code_change/3]). 16 | 17 | -record (state, {id, queue = queue:new() }). 18 | 19 | % there are 2 type of records stored in the ets table, we'll use the 20 | % first element as the keypos, so either the port for a socket, or 21 | % the id for config 22 | -record (connection, {socket, 23 | create_time_epoch_seconds, 24 | access_time_epoch_seconds, 25 | max_age_seconds}). 26 | -define (CONNECTION_CREATE_TIME_INDEX, #connection.create_time_epoch_seconds). 27 | -define (CONNECTION_LAST_ACCESS_INDEX, #connection.access_time_epoch_seconds). 28 | -define (CONNECTION_MAX_AGE_INDEX, #connection.max_age_seconds). 29 | 30 | -record (config, {id, 31 | max_age_seconds = 60, 32 | max_connections = 65535, % set unreasonably high? 33 | active = 0, 34 | busy = 0 35 | }). 36 | -define (CONFIG_MAX_INDEX, #config.max_connections). 37 | -define (CONFIG_ACTIVE_INDEX, #config.active). 38 | -define (CONFIG_BUSY_INDEX, #config.busy). 39 | -define (CONFIG_MAX_AGE_INDEX, #config.max_age_seconds). 40 | 41 | -define (TABLE, lwes_emitters). 42 | 43 | %%==================================================================== 44 | %% API 45 | %%==================================================================== 46 | start_link (Config = #config {id = Id}) -> 47 | gen_server:start_link({local, Id}, ?MODULE, [Config], []); 48 | start_link (ConfigList) when is_list (ConfigList) -> 49 | case parse_config (ConfigList, #config {}) of 50 | {error, E} -> {stop, {error, E}}; 51 | {ok, Config}-> start_link (Config) 52 | end. 53 | 54 | send (Id, Address, Packet) -> 55 | case checkout (Id) of 56 | {error, Error} -> 57 | {error, {checkout, Error}}; % Error is {error, busy} or {error, down} 58 | Socket -> 59 | case lwes_net_udp:send (Socket, Address, Packet) of 60 | {error, Error} -> 61 | checkin (Id, Socket, error), 62 | {error, {call, Error}}; 63 | Answer -> 64 | checkin (Id, Socket), 65 | Answer 66 | end 67 | end. 68 | 69 | out (Id) -> gen_server:call (Id, {out}). 70 | in (Id, Socket) -> gen_server:cast (Id, {in, Socket}). 71 | 72 | checkout (Id) -> 73 | case out (Id) of 74 | empty -> open (Id); 75 | {value, Socket} -> Socket 76 | end. 77 | 78 | checkin (Id, Socket, error) -> 79 | close (Id, Socket). 80 | 81 | checkin (Id, Socket) -> 82 | % at checkin, we'll do all our housekeeping 83 | 84 | % get the time now 85 | Now = seconds_since_epoch(), 86 | 87 | % fetch out the start time and set the last access 88 | [Start, MaxAge] = 89 | ets:update_counter (Id, Socket, 90 | [ {?CONNECTION_CREATE_TIME_INDEX, 0}, 91 | {?CONNECTION_MAX_AGE_INDEX, 0} ] ), 92 | 93 | % check to see if we've been alive for more than the alloted alive time 94 | case Now - Start > MaxAge of 95 | true -> 96 | % alive too long, so close 97 | close (Id, Socket); 98 | false -> 99 | ets:update_element (Id, Socket, 100 | {?CONNECTION_LAST_ACCESS_INDEX, Now}), 101 | in (Id, Socket) 102 | end. 103 | 104 | open (Id) -> 105 | % optimistically update the active connections, so we don't go above max 106 | [Max, Active] = 107 | ets:update_counter (Id, Id, 108 | [ {?CONFIG_MAX_INDEX,0}, 109 | {?CONFIG_ACTIVE_INDEX,1} 110 | ]), 111 | 112 | % then check if the active connections are more than the max 113 | case Active > Max of 114 | true -> 115 | % we decrement active connections because we optimistically incremented 116 | % above, also increment busy index. 117 | ets:update_counter (Id, Id, [{?CONFIG_BUSY_INDEX,1}, 118 | {?CONFIG_ACTIVE_INDEX,-1}]), 119 | {error, busy}; 120 | false -> 121 | case ets:lookup (Id, Id) of 122 | [#config { max_age_seconds = MaxAge } ] -> 123 | % address doesn't matter for emitter's it's added at send time 124 | Dummy = lwes_net_udp:new (emitter, {"127.0.0.1",9191}), 125 | case lwes_net_udp:open (emitter, Dummy) of 126 | {ok, Socket} -> 127 | % Let this gen_server be the controlling process, not the 128 | % process opening the connection, this allows the process 129 | % opening the connection to die without killing the connection 130 | ok = gen_udp:controlling_process (Socket, whereis (Id)), 131 | 132 | Now = seconds_since_epoch(), 133 | ets:insert_new (Id, 134 | #connection { socket = Socket, 135 | create_time_epoch_seconds = Now, 136 | access_time_epoch_seconds = Now, 137 | max_age_seconds = MaxAge 138 | }), 139 | Socket; 140 | {error, E} -> 141 | % we decrement active connections because we optimistically 142 | % incremented above, also increment busy index. 143 | ets:update_counter (Id, Id, [{?CONFIG_BUSY_INDEX,1}, 144 | {?CONFIG_ACTIVE_INDEX,-1}]), 145 | 146 | error_logger:error_msg ("Unexpected connect error 1 : ~p", [E]), 147 | {error, down} 148 | end; 149 | E2 -> 150 | % we decrement active connections because we optimistically 151 | % incremented above, also increment busy index. 152 | ets:update_counter (Id, Id, [{?CONFIG_BUSY_INDEX,1}, 153 | {?CONFIG_ACTIVE_INDEX,-1}]), 154 | 155 | error_logger:error_msg ("Unexpected connect error 2 : ~p", [E2]), 156 | {error, down} 157 | end 158 | end. 159 | 160 | close (Id, Socket) -> 161 | ets:update_counter (Id, Id, [{?CONFIG_ACTIVE_INDEX,-1}]), 162 | ets:delete (Id, Socket), 163 | lwes_net_udp:close (Socket). 164 | 165 | %%==================================================================== 166 | %% gen_server callbacks 167 | %%==================================================================== 168 | init([Config = #config { id = Id }]) -> 169 | ets:new (Id, 170 | [ named_table, 171 | public, 172 | set, 173 | {keypos, 2}, 174 | {write_concurrency, true} 175 | ]), 176 | ets:insert_new (Id, Config), 177 | {ok, #state { id = Id }}. 178 | 179 | handle_call ({size}, _From, State = #state { queue = QueueIn }) -> 180 | {reply, queue:len (QueueIn), State}; 181 | handle_call ({out}, _From, State = #state { queue = QueueIn }) -> 182 | {Value, QueueOut} = queue:out (QueueIn), 183 | {reply, Value, State#state { queue = QueueOut }}; 184 | handle_call (Request, From, State) -> 185 | io:format ("~p:handle_call ~p from ~p~n",[?MODULE, Request, From]), 186 | {reply, ok, State}. 187 | 188 | handle_cast ({in, Socket}, State = #state { queue = QueueIn }) -> 189 | {noreply, State#state { queue = queue:in (Socket, QueueIn) }}; 190 | handle_cast (Request, State) -> 191 | io:format ("~p:handle_cast ~p~n",[?MODULE, Request]), 192 | {noreply, State}. 193 | 194 | handle_info (Info, State) -> 195 | io:format ("~p:handle_info ~p~n",[?MODULE,Info]), 196 | {noreply, State}. 197 | 198 | terminate (_Reason, _State) -> 199 | ok. 200 | 201 | code_change (_OldVsn, State, _Extra) -> 202 | {ok, State}. 203 | 204 | %%-------------------------------------------------------------------- 205 | %%% Helper functions 206 | %%-------------------------------------------------------------------- 207 | parse_config ([], Config) -> 208 | validate_config (Config); 209 | parse_config ([{id, Id}|Rest], Config = #config{}) -> 210 | parse_config (Rest, Config#config {id = Id}); 211 | parse_config ([{max_age_seconds, MaxAge}|Rest], Config = #config{}) -> 212 | parse_config (Rest, Config#config {max_age_seconds = MaxAge}); 213 | parse_config ([{max_connections, MaxConnections}|Rest], Config = #config{}) -> 214 | parse_config (Rest, Config#config {max_connections = MaxConnections }); 215 | parse_config ([_|Rest], Config = #config{}) -> 216 | parse_config (Rest, Config). 217 | 218 | validate_config ( Config = #config { id = Id }) -> 219 | % double check non-defaulted options have values 220 | case Id =/= undefined of 221 | true -> {ok, Config}; 222 | false -> {error, bad_config} 223 | end. 224 | 225 | seconds_since_epoch () -> 226 | {Mega, Secs, _ } = os:timestamp(), 227 | Mega * 1000000 + Secs. 228 | 229 | %%-------------------------------------------------------------------- 230 | %%% Test functions 231 | %%-------------------------------------------------------------------- 232 | -ifdef (TEST). 233 | -include_lib ("eunit/include/eunit.hrl"). 234 | 235 | 236 | -endif. 237 | -------------------------------------------------------------------------------- /src/lwes_esf_lexer.xrl: -------------------------------------------------------------------------------- 1 | % Tokenizer definitions for ESF (refer : http://www.lwes.org/docs/) 2 | 3 | Definitions. 4 | 5 | % white space 6 | WS = [\000-\s] 7 | 8 | % literals 9 | INTEGER_LITERAL = [-+]?[1-9]([0-9])* 10 | BIG_INTEGER_LITERAL = [-+]?[1-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]([0-9])* 11 | DOUBLE_LITERAL = [-+]?([0-9])*\.([0-9])+ 12 | STRING_LITERAL = \"(\\.|[^"])*\" 13 | IPADDR_LITERAL = ([0-9])+\.([0-9])+\.([0-9])+\.([0-9])+ 14 | MACROS = \$\{[-A-Za-z0-9_.]+\} 15 | 16 | WORD = [a-zA-Z_]([-_:.a-zA-Z0-9])* 17 | COMMENT = #[^\r\n]*(\r\n|\r|\n)? 18 | 19 | Rules. 20 | 21 | {WORD} : { token, { token_type(list_to_atom(TokenChars)) , TokenLine, TokenChars } }. 22 | {INTEGER_LITERAL}|{BIG_INTEGER_LITERAL}|{DOUBLE_LITERAL}|{STRING_LITERAL}|{IPADDR_LITERAL}|{MACROS} : 23 | { token, { value , TokenLine, TokenChars } }. 24 | [\{\};=\[\]] : { token, { list_to_atom(TokenChars), TokenLine } }. 25 | 26 | % ignore comment 27 | {COMMENT} : skip_token. 28 | 29 | % ignore whitespace 30 | {WS}+ : skip_token. 31 | 32 | Erlang code. 33 | 34 | -export([token_type/1]). 35 | 36 | token_type (true) -> value; 37 | token_type (false) -> value; 38 | 39 | token_type (required) -> qualifier; 40 | token_type (nullable) -> qualifier; 41 | token_type (optional) -> qualifier; 42 | 43 | token_type (byte) -> type; 44 | token_type (uint16) -> type; 45 | token_type (int16) -> type; 46 | token_type (uint32) -> type; 47 | token_type (int32) -> type; 48 | token_type (uint64) -> type; 49 | token_type (int64) -> type; 50 | token_type (float) -> type; 51 | token_type (double) -> type; 52 | token_type (boolean) -> type; 53 | token_type (string) -> type; 54 | token_type (ip_addr) -> type; 55 | 56 | token_type (_) -> identifier. 57 | -------------------------------------------------------------------------------- /src/lwes_esf_parser.yrl: -------------------------------------------------------------------------------- 1 | Nonterminals 2 | eventlist event attributelist attribute. 3 | 4 | Terminals '{' '}' '[' ']' '=' ';' type identifier value qualifier. 5 | 6 | Rootsymbol eventlist. 7 | 8 | eventlist -> event : [ '$1' ]. 9 | eventlist -> event eventlist : [ '$1' | '$2' ]. 10 | 11 | event -> identifier '{' attributelist '}' : { event, unwrap ('$1'), '$3' }. 12 | 13 | attributelist -> attribute : [ '$1' ]. 14 | attributelist -> attribute attributelist : [ '$1' | '$2' ]. 15 | 16 | attribute -> type identifier ';' : 17 | { attribute, { unwrap_to_atom ('$1'), unwrap_to_binary ('$2'), undefined, undefined } }. 18 | attribute -> type identifier '[' value ']' ';' : 19 | { attribute, { list_to_atom(unwrap ('$1') ++ "_array"), unwrap_to_binary ('$2'), undefined, undefined } }. 20 | attribute -> type identifier '=' value ';' : 21 | { attribute, { unwrap_to_atom ('$1'), unwrap_to_binary ('$2'), unwrap ('$4'), undefined } }. 22 | attribute -> qualifier type identifier ';' : 23 | { attribute, { unwrap_to_atom ('$2'), unwrap_to_binary ('$3'), undefined, unwrap_to_atom ('$1') } }. 24 | attribute -> qualifier type identifier '[' value ']' ';' : 25 | { attribute, { list_to_atom (array_prefix (unwrap_to_atom ('$1')) ++ unwrap ('$2') ++ "_array"), unwrap_to_binary ('$3'), undefined, unwrap_to_atom('$1') } }. 26 | attribute -> qualifier type identifier '=' value ';' : 27 | { attribute, { unwrap_to_atom ('$2'), unwrap_to_binary ('$3'), unwrap ('$5') , unwrap_to_atom ('$1') } }. 28 | 29 | Erlang code. 30 | 31 | unwrap ({_, _, V}) -> V. 32 | 33 | unwrap_to_atom ({_, _, V}) -> list_to_atom(V). 34 | 35 | unwrap_to_binary({_, _, V}) -> list_to_binary(V). 36 | 37 | array_prefix (nullable) -> "nullable_"; 38 | array_prefix (_) -> []. 39 | -------------------------------------------------------------------------------- /src/lwes_esf_validator.erl: -------------------------------------------------------------------------------- 1 | -module(lwes_esf_validator). 2 | 3 | -behaviour(gen_server). 4 | 5 | %% API 6 | -export([start_link/0, 7 | add_esf/2, 8 | validate/2, 9 | stats/0]). 10 | 11 | %% gen_server callbacks 12 | -export([init/1, 13 | handle_call/3, 14 | handle_cast/2, 15 | handle_info/2, 16 | terminate/2, 17 | code_change/3]). 18 | 19 | -include_lib ("lwes.hrl"). 20 | 21 | -define (SPEC_TAB, lwes_esf_specs). 22 | 23 | -define(LEXER, lwes_esf_lexer). 24 | -define(PARSER, lwes_esf_parser). 25 | 26 | -define(META_EVENT, "MetaEventInfo"). 27 | -define(STATS_KEY, stats). 28 | 29 | %%==================================================================== 30 | %% API 31 | %%==================================================================== 32 | start_link () -> 33 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). 34 | 35 | add_esf (ESFName, ESFFile) -> 36 | Events = parse (file, ESFFile), 37 | add_esf_events(ESFName, Events). 38 | 39 | add_esf_events(ESFName, Events) -> 40 | % if 'MetaEventInfo' is defined we need to merge the attribute 41 | %specification from the 'Meta Event' to the specification of 42 | % all other 'real' events 43 | Events1 = case lists:keyfind (?META_EVENT, 2, Events) of 44 | false -> Events; 45 | MetaEvent -> 46 | RealEvents = lists:delete (MetaEvent, Events), 47 | {event, _, MetaAttrs} = MetaEvent, 48 | [{event, Name, Attrs ++ MetaAttrs } || {event, Name, Attrs} <- RealEvents] 49 | end, 50 | lists:foreach (fun (E) -> add_event (ESFName, E) end, Events1). 51 | 52 | validate (ESFName, #lwes_event {name = EventName, attrs = Attrs} = _Event) -> 53 | ets:update_counter (?SPEC_TAB, ?STATS_KEY, {2,1}), 54 | 55 | EventName1 = lwes_util:any_to_list (EventName), 56 | Key = { ESFName, EventName1 }, 57 | 58 | case ets:lookup (?SPEC_TAB, Key) of 59 | [{_, EventSpec}] -> 60 | validate_event (EventName1, EventSpec, Attrs); 61 | _ -> {error, {event_undefined, EventName}} 62 | end. 63 | 64 | stats () -> 65 | [Stats] = ets:lookup (?SPEC_TAB, ?STATS_KEY), 66 | Stats. 67 | 68 | %%==================================================================== 69 | %% gen_server callbacks 70 | %%==================================================================== 71 | 72 | init ([]) -> 73 | % make sure terminate is called 74 | process_flag (trap_exit, true), 75 | 76 | ets:new (?SPEC_TAB, [set, public, named_table, {keypos, 1}]), 77 | ets:insert (?SPEC_TAB,{?STATS_KEY, 0, 0}), 78 | { ok, {} }. 79 | 80 | handle_call ({state}, _From, State) -> 81 | {reply, State, State }. 82 | 83 | handle_cast (_Msg, State) -> 84 | { noreply, State }. 85 | 86 | handle_info (_Info, State) -> 87 | { noreply, State }. 88 | 89 | terminate (_Reason, _State) -> 90 | ets:delete (?SPEC_TAB), 91 | ok. 92 | 93 | code_change (_OldVsn, State, _Extra) -> 94 | { ok, State }. 95 | 96 | %%-------------------------------------------------------------------- 97 | %%% Internal functions 98 | %%-------------------------------------------------------------------- 99 | 100 | add_event (ESFName, Event) -> 101 | { event, EventName, Attributes } = Event, 102 | Attributes1 = [ A || {attribute, A} <- Attributes ], 103 | { RequiredAttrs, OptionalAttrs } = 104 | lists:partition( 105 | fun ({_, _, _, Qualifier}) -> Qualifier == required end, 106 | Attributes1), 107 | ets:insert (?SPEC_TAB, 108 | {{ ESFName, EventName }, { RequiredAttrs, OptionalAttrs }}). 109 | 110 | validate_event (EventName, { RequiredSpec, OptionalSpec }, EventAttrs) -> 111 | case validate_unique (EventName, EventAttrs, []) of 112 | ok -> 113 | case validate_required (EventName, RequiredSpec, EventAttrs) of 114 | {ok, OptionalAttrs} -> 115 | case validate_optional (EventName, OptionalSpec, OptionalAttrs) of 116 | ok -> ets:update_counter (?SPEC_TAB, ?STATS_KEY, {3,1}), 117 | ok; 118 | ErrorOpt -> ErrorOpt 119 | end; 120 | ErrorReq -> ErrorReq 121 | end; 122 | ErrorUnique -> ErrorUnique 123 | end. 124 | 125 | 126 | validate_unique (_, [], _) -> ok; 127 | validate_unique (EventName, [{_, AttrName, _} | T], AttrsFound) -> 128 | case lists:keyfind (AttrName, 1, AttrsFound) of 129 | false -> validate_unique(EventName, T, [{AttrName} | AttrsFound]); 130 | _ -> 131 | {error, {field_duplicate, AttrName, EventName}} 132 | end. 133 | 134 | validate_required (_, [], EventAttrs) -> {ok, EventAttrs}; 135 | 136 | validate_required (EventName, [{_Type_S, 137 | AttributeName_S, _, _} = H| T], EventAttrs) -> 138 | case lists:keyfind (AttributeName_S, 2, EventAttrs) of 139 | false -> 140 | {error, {field_missing, AttributeName_S, EventName}}; 141 | Attr -> 142 | case validate_attribute (EventName, H, Attr) of 143 | ok -> 144 | EventAttrs1 = lists:delete (Attr, EventAttrs), 145 | validate_required (EventName, T, EventAttrs1); 146 | Error -> Error 147 | end 148 | end. 149 | 150 | validate_optional (_, _OptionalSpec, []) -> ok; 151 | 152 | validate_optional (EventName, OptionalSpec, [{_Type, AttrName, _} = H | T]) -> 153 | case lists:keyfind (AttrName, 2, OptionalSpec) of 154 | false -> 155 | {error, {field_undefined, AttrName, EventName}}; 156 | Spec -> 157 | case validate_attribute (EventName, Spec, H) of 158 | ok -> validate_optional (EventName, OptionalSpec, T); 159 | Error -> Error 160 | end 161 | end. 162 | 163 | validate_attribute (EventName, {Type_S, AttrName_S, _, _}, {Type, AttrName, _}) -> 164 | case AttrName_S == AttrName andalso Type_S == Type of 165 | false -> 166 | {error, {field_type_mismatch, AttrName, EventName, Type, Type_S}}; 167 | _ -> ok 168 | end. 169 | 170 | parse (file, FileName) -> 171 | { ok, InFile } = file:open(FileName, [read]), 172 | Acc = loop (InFile, []), 173 | file:close (InFile), 174 | {ok, ParseTree} = ?PARSER:parse (Acc), 175 | ParseTree. 176 | 177 | loop (InFile, Acc) -> 178 | case io:request(InFile, { get_until, prompt, ?LEXER, token, [1] } ) of 179 | { ok, Toks, _EndLine } -> 180 | loop (InFile,Acc ++ [Toks]); 181 | { error, token } -> 182 | error_logger:error_msg ("failed to read ESF file"), 183 | Acc; 184 | { eof, _ } -> 185 | Acc 186 | end. 187 | 188 | %%-------------------------------------------------------------------- 189 | %%% Test functions 190 | %%-------------------------------------------------------------------- 191 | 192 | -ifdef(TEST). 193 | -include_lib("eunit/include/eunit.hrl"). 194 | 195 | 196 | parse_string (String) -> 197 | {ok, Tokens, _EndLine} = ?LEXER:string (String), 198 | {ok, ParseTree} = ?PARSER:parse ( Tokens ), 199 | ParseTree. 200 | 201 | validate_test_() -> 202 | EventDefs = parse_string( 203 | "TestEvent { 204 | required int32 i = 1; 205 | required boolean j; 206 | int32 k = 1; 207 | string l; 208 | }" 209 | ), 210 | Validate = fun (Attrs) -> 211 | try 212 | ets:delete (?SPEC_TAB) 213 | catch _:_ -> 0 end, 214 | ets:new (?SPEC_TAB, [set, public, named_table, {keypos, 1}]), 215 | ets:insert (?SPEC_TAB,{?STATS_KEY, 0, 0}), 216 | add_esf_events(test_esf, EventDefs), 217 | Valid = validate(test_esf, #lwes_event {name = <<"TestEvent">>, attrs = Attrs}), 218 | ets:delete (?SPEC_TAB), 219 | Valid 220 | end, 221 | [ 222 | ?_assertEqual(ok, % all fields 223 | Validate([{int32,<<"i">>,314159}, 224 | {boolean,<<"j">>,false}, 225 | {int32,<<"k">>,654321}, 226 | {string,<<"l">>,<<"foo">>}]) 227 | ), 228 | % NOTE: Default values aren't patched in anywhere currently. 229 | % So, let this fail validation for now. 230 | %?assertEqual(true, % missing required field with default 231 | % Validate([ {boolean,<<"j">>,false}, 232 | % {int32,<<"k">>,654321}, 233 | % {string,<<"l">>,<<"foo">>}]) 234 | %), 235 | ?_assertEqual({error, {field_missing, <<"j">>, "TestEvent"}}, 236 | % missing required field (no default) 237 | Validate([{int32,<<"i">>,314159}, 238 | {int32,<<"k">>,654321}, 239 | {string,<<"l">>,<<"foo">>}]) 240 | ), 241 | ?_assertEqual({error, {field_duplicate, <<"j">>, "TestEvent"}}, 242 | % duplicate field 243 | Validate([{int32,<<"i">>,314159}, 244 | {boolean,<<"j">>,false}, 245 | {boolean,<<"j">>,false}, 246 | {int32,<<"k">>,654321}, 247 | {string,<<"l">>,<<"foo">>}]) 248 | ), 249 | ?_assertEqual({error, {field_type_mismatch, <<"i">>, "TestEvent", boolean, int32}}, 250 | % type error, required field 251 | Validate([{boolean,<<"i">>,true}, 252 | {boolean,<<"j">>,false}, 253 | {int32,<<"k">>,654321}, 254 | {string,<<"l">>,<<"foo">>}]) 255 | ), 256 | ?_assertEqual({error, {field_type_mismatch, <<"k">>, "TestEvent", string, int32}}, 257 | % type error, optional field 258 | Validate([{int32,<<"i">>,314159}, 259 | {boolean,<<"j">>,false}, 260 | {string,<<"k">>,<<"blah">>}, 261 | {string,<<"l">>,<<"foo">>}]) 262 | ), 263 | ?_assertEqual({error, {field_undefined, <<"extra">>, "TestEvent"}}, 264 | % extra undefined field 265 | Validate([{int32,<<"i">>,314159}, 266 | {boolean,<<"j">>,false}, 267 | {int32,<<"k">>,654321}, 268 | {int32,<<"extra">>,12345}, 269 | {string,<<"l">>,<<"foo">>}]) 270 | ) 271 | ]. 272 | 273 | parse_test_() -> 274 | TestCases = [ 275 | { 276 | [{event,"A", 277 | [{attribute, {int32, <<"i">>, undefined, undefined}}]}], 278 | "A 279 | { 280 | int32 i; 281 | } 282 | " 283 | }, 284 | { 285 | [{event,"A", 286 | [{attribute, {int32, <<"i">>, undefined, undefined}}, 287 | {attribute, {uint32, <<"j">>, undefined, undefined}}]}], 288 | "A 289 | { 290 | int32 i; 291 | uint32 j; 292 | } 293 | " 294 | }, 295 | { 296 | [{event,"A", 297 | [{attribute, {int32, <<"i">>, undefined, undefined}}, 298 | {attribute, {uint32, <<"j">>, undefined, undefined}}]}], 299 | "A 300 | { 301 | int32 i; # comments should be ignored 302 | uint32 j; # comments should be ignored 303 | } 304 | " 305 | }, 306 | { 307 | [{event,"A", 308 | [{attribute, {int32, <<"i">>, undefined, undefined}}, 309 | {attribute, {uint32, <<"j">>, undefined, undefined}}]}], 310 | " 311 | # comments should be ignored 312 | # like this one 313 | A 314 | { 315 | int32 i; 316 | uint32 j; 317 | } 318 | " 319 | }, 320 | { 321 | [{event,"A", 322 | [{attribute, {int32, <<"i">>, "1", undefined}}, 323 | {attribute, {uint32, <<"j">>, undefined, undefined}}]}], 324 | " 325 | # comments should be ignored 326 | # like this one 327 | A 328 | { 329 | int32 i = 1; # we can set defaults 330 | uint32 j; 331 | } 332 | " 333 | }, 334 | { 335 | [{event,"A", 336 | [{attribute, {int32, <<"i">>, "-1", undefined}}, 337 | {attribute, {uint32, <<"j">>, undefined, undefined}}]}], 338 | " 339 | A 340 | { 341 | int32 i = -1; # we can set negative defaults 342 | uint32 j; 343 | } 344 | " 345 | }, 346 | { 347 | [{event,"A", 348 | [{attribute, {float, <<"i">>, "-0.1234", undefined}}, 349 | {attribute, {float, <<"j">>, "12.8999", undefined}}]}], 350 | " 351 | A 352 | { 353 | float i = -0.1234; # we can set negative defaults for float 354 | float j = 12.8999; 355 | } 356 | " 357 | }, 358 | { 359 | [{event,"A", 360 | [{attribute, {double, <<"i">>, "-0.1234", undefined}}, 361 | {attribute, {double, <<"j">>, "12.8999", undefined}}]}], 362 | " 363 | A 364 | { 365 | double i = -0.1234; # we can set negative defaults for double 366 | double j = 12.8999; 367 | } 368 | " 369 | }, 370 | { 371 | [{event,"A", 372 | [{attribute, {boolean, <<"i">>, "true", undefined}}, 373 | {attribute, {boolean, <<"j">>, "false", undefined}}]}], 374 | " 375 | A 376 | { 377 | boolean i = true; # we can set defaults for boolean 378 | boolean j = false; 379 | } 380 | " 381 | }, 382 | { 383 | [{event,"A", 384 | [{attribute, {string, <<"i">>, "\"hello\"", undefined}}, 385 | {attribute, {string, <<"j">>, "\"hello\nworld\"", undefined}}]}], 386 | " 387 | A 388 | { 389 | string i = \"hello\"; # we can set defaults for string 390 | string j = \"hello\nworld\"; 391 | } 392 | " 393 | }, 394 | { 395 | [{event,"A", 396 | [{attribute, {uint32, <<"i">>, "1", optional}}, 397 | {attribute, {int32, <<"j">>, "-1", nullable}}, 398 | {attribute, {uint32, <<"k">>, "2", required}}]}], 399 | " 400 | A 401 | { 402 | optional uint32 i = 1; # make this optional 403 | nullable int32 j = -1; # nullable 404 | required uint32 k = 2; # required 405 | } 406 | " 407 | }, 408 | { 409 | [{event,"A", 410 | [{attribute, {byte, <<"i">>, "1",undefined}}]}], 411 | " 412 | A 413 | { 414 | byte i = 1; # define 'byte' attribute 415 | } 416 | " 417 | }, 418 | { 419 | [{event,"A", 420 | [{attribute, {nullable_int32_array, <<"i">>, undefined, nullable}}]}], 421 | " 422 | A 423 | { 424 | nullable int32 i[16]; # define array attribute 425 | } 426 | " 427 | }, 428 | { 429 | [{event,"A", 430 | [{attribute, {int32_array, <<"i">>, undefined, undefined}}]}], 431 | " 432 | A 433 | { 434 | int32 i[16]; # define array attribute 435 | } 436 | " 437 | }, 438 | { 439 | [{event,"A", 440 | [{attribute, {int32_array, <<"i">>, undefined, required}}]}], 441 | " 442 | A 443 | { 444 | required int32 i[16]; # define array attribute 445 | } 446 | " 447 | } 448 | ], 449 | [ ?_assertEqual (E, parse_string(TC)) || {E, TC} <- TestCases ]. 450 | 451 | -endif. 452 | -------------------------------------------------------------------------------- /src/lwes_file_watcher.erl: -------------------------------------------------------------------------------- 1 | -module (lwes_file_watcher). 2 | 3 | %% API 4 | -export ([ scan/1, 5 | changes/1, 6 | changes/2 7 | ]). 8 | 9 | % The goal of this modules is to watch directories and files, and alert 10 | % interested parties about when things change. 11 | % 12 | % The usage is as follows 13 | % 14 | % 1. get a scan right now 15 | % 16 | % Scan = scan (["/tmp"]), 17 | % 18 | % 2. see what changed from an empty scan 19 | % 20 | % changes (Scan) 21 | % 22 | % 3. save a scan then see what changed the next time 23 | % 24 | % changes (OldScan, NewScan) 25 | % 26 | % Changes will return a list of the form 27 | % 28 | % [ {dir, added, "dir"}, 29 | % {dir, removed, "dir"}, 30 | % {file, added, "file"}, 31 | % {file, removed, "file"}, 32 | % {file, changed, "file"} 33 | % ] 34 | 35 | %%==================================================================== 36 | %% API 37 | %%==================================================================== 38 | scan (FilePaths) -> 39 | scan_filepaths (FilePaths). 40 | 41 | changes (Scan) -> 42 | file_changes ([], Scan). 43 | 44 | changes (OldScan, NewScan) -> 45 | file_changes (OldScan, NewScan). 46 | 47 | %%==================================================================== 48 | %% Internal 49 | %%==================================================================== 50 | scan_filepaths (L) -> 51 | ordsets:from_list (lists:flatten (scan_filepaths (L, []))). 52 | 53 | scan_filepaths ([First = [_|_]|Rest], Accum) -> 54 | scan_filepaths (Rest, [ scan_filepath (First) | Accum ]); 55 | scan_filepaths ([First], Accum) -> 56 | [ scan_filepath (First) | Accum ]; 57 | scan_filepaths (First, Accum) -> 58 | [ scan_filepath (First) | Accum ]. 59 | 60 | % non-tail recursively scan directory/file and return whether it is a 61 | % file, a directory or missing 62 | scan_filepath ([]) -> 63 | []; 64 | scan_filepath (FilePath) -> 65 | case filelib:is_dir (FilePath) of 66 | true -> 67 | case file:list_dir (FilePath) of 68 | {ok, Files} -> 69 | [ { dir, FilePath } | 70 | lists:foldl ( 71 | fun (F, A) -> 72 | FP = filename:join ([FilePath, F]), 73 | [ scan_filepath (FP) | A ] 74 | end, 75 | [], 76 | Files 77 | ) 78 | ]; 79 | { error, _ } -> 80 | [] 81 | end; 82 | false -> 83 | case filelib:is_regular (FilePath) of 84 | true -> 85 | [ { file, FilePath, filelib:last_modified (FilePath) } ]; 86 | false -> 87 | [ ] 88 | end 89 | end. 90 | 91 | file_changes (OldScan, NewScan) -> 92 | file_changes (OldScan, NewScan, []). 93 | 94 | file_changes ([], [], Accum) -> 95 | lists:reverse (Accum); 96 | file_changes (OL = [], [{dir, D} | NR], Accum) -> 97 | file_changes (OL, NR, [ {dir, added, D} | Accum ]); 98 | file_changes (OL = [], [{file, F, _} | NR], Accum) -> 99 | file_changes (OL, NR, [ {file, added, F} | Accum ]); 100 | file_changes ([{dir, D} | OR], NL = [], Accum) -> 101 | file_changes (OR, NL, [ {dir, removed, D} | Accum ]); 102 | file_changes ([{file, F, _} | OR], NL = [], Accum) -> 103 | file_changes (OR, NL, [ {file, removed, F} | Accum ]); 104 | file_changes ([O|OR], [N|NR], Accum) when O =:= N -> 105 | file_changes (OR, NR, Accum); 106 | file_changes ([{file, FP, OLM}|OR], [{file, FP, NLM}|NR], Accum) 107 | when OLM =/= NLM -> 108 | file_changes (OR, NR, [ {file, changed, FP} | Accum ]); 109 | file_changes ([O = {file, F, _}|OR], NF = [N|_], Accum) when O < N -> 110 | file_changes (OR, NF, [ {file, removed, F} | Accum ]); 111 | file_changes ([O = {dir, D}|OR], NF = [N|_], Accum) when O < N -> 112 | file_changes (OR, NF, [ {dir, removed, D} | Accum ]); 113 | file_changes (OF = [O|_], [N = {file, F, _}|NR], Accum) when O > N -> 114 | file_changes (OF, NR, [ {file, added, F} | Accum ]); 115 | file_changes (OF = [O|_], [N = {dir, D}|NR], Accum) when O > N -> 116 | file_changes (OF, NR, [ {dir, added, D} | Accum ]). 117 | 118 | %%==================================================================== 119 | %% Test functions 120 | %%==================================================================== 121 | -ifdef(TEST). 122 | -include_lib ("eunit/include/eunit.hrl"). 123 | 124 | % write tests here :) 125 | 126 | -endif. 127 | -------------------------------------------------------------------------------- /src/lwes_internal.hrl: -------------------------------------------------------------------------------- 1 | -ifndef(_lwes_internal_included). 2 | -define(_lwes_internal_included, yup). 3 | 4 | -define (is_ttl (V), is_integer(V), V >= 0, V =< 32). 5 | -define (is_int16 (V), is_integer(V), V >= -32768, V =< 32767). 6 | -define (is_uint16 (V), is_integer(V), V >= 0, V =< 65535). 7 | -define (is_int32 (V), is_integer(V), V >= -2147483648, V =< 2147483647). 8 | -define (is_uint32 (V), is_integer(V), V >= 0, V =< 4294967295). 9 | -define (is_int64 (V), is_integer(V), V >= -9223372036854775808, V =< 9223372036854775807). 10 | -define (is_uint64 (V), is_integer(V), V >= 0, V =< 18446744073709551615). 11 | -define (is_byte (V), is_integer(V), V >= 0, V =< 255). 12 | -define (is_string (V), is_list (V); is_binary (V); is_atom (V)). 13 | -define (is_ip_addr (V), 14 | (is_tuple (V) andalso 15 | tuple_size (V) =:= 4 andalso 16 | is_integer (element (1,V)) andalso 17 | element (1,V) >= 0 andalso 18 | element (1,V) =< 255 andalso 19 | is_integer (element (2,V)) andalso 20 | element (2,V) >= 0 andalso 21 | element (2,V) =< 255 andalso 22 | is_integer (element (3,V)) andalso 23 | element (3,V) >= 0 andalso 24 | element (3,V) =< 255 andalso 25 | is_integer (element (4,V)) andalso 26 | element (4,V) >= 0 andalso 27 | element (4,V) =< 255)). 28 | 29 | -record (lwes_channel, { type, 30 | config, 31 | ref 32 | }). 33 | -record (lwes_multi_emitter, {type, max, num, configs}). 34 | 35 | -define (LWES_TYPE_U_INT_16, 1). 36 | -define (LWES_TYPE_INT_16, 2). 37 | -define (LWES_TYPE_U_INT_32, 3). 38 | -define (LWES_TYPE_INT_32, 4). 39 | -define (LWES_TYPE_STRING, 5). 40 | -define (LWES_TYPE_IP_ADDR, 6). 41 | -define (LWES_TYPE_INT_64, 7). 42 | -define (LWES_TYPE_U_INT_64, 8). 43 | -define (LWES_TYPE_BOOLEAN, 9). 44 | -define (LWES_TYPE_BYTE, 10). 45 | -define (LWES_TYPE_FLOAT, 11). 46 | -define (LWES_TYPE_DOUBLE, 12). 47 | -define (LWES_TYPE_LONG_STRING, 13). 48 | -define (LWES_TYPE_U_INT_16_ARRAY, 129). 49 | -define (LWES_TYPE_INT_16_ARRAY, 130). 50 | -define (LWES_TYPE_U_INT_32_ARRAY, 131). 51 | -define (LWES_TYPE_INT_32_ARRAY, 132). 52 | -define (LWES_TYPE_STRING_ARRAY, 133). 53 | -define (LWES_TYPE_IP_ADDR_ARRAY, 134). 54 | -define (LWES_TYPE_INT_64_ARRAY, 135). 55 | -define (LWES_TYPE_U_INT_64_ARRAY, 136). 56 | -define (LWES_TYPE_BOOLEAN_ARRAY, 137). 57 | -define (LWES_TYPE_BYTE_ARRAY, 138). 58 | -define (LWES_TYPE_FLOAT_ARRAY, 139). 59 | -define (LWES_TYPE_DOUBLE_ARRAY, 140). 60 | -define (LWES_TYPE_N_U_INT_16_ARRAY, 141). 61 | -define (LWES_TYPE_N_INT_16_ARRAY, 142). 62 | -define (LWES_TYPE_N_U_INT_32_ARRAY, 143). 63 | -define (LWES_TYPE_N_INT_32_ARRAY, 144). 64 | -define (LWES_TYPE_N_STRING_ARRAY, 145). 65 | % TODO: this is not implemented 66 | % -define (LWES_TYPE_N_IP_ADDR_ARRAY, 146). 67 | -define (LWES_TYPE_N_INT_64_ARRAY, 147). 68 | -define (LWES_TYPE_N_U_INT_64_ARRAY, 148). 69 | -define (LWES_TYPE_N_BOOLEAN_ARRAY, 149). 70 | -define (LWES_TYPE_N_BYTE_ARRAY, 150). 71 | -define (LWES_TYPE_N_FLOAT_ARRAY, 151). 72 | -define (LWES_TYPE_N_DOUBLE_ARRAY, 152). 73 | 74 | -endif. 75 | -------------------------------------------------------------------------------- /src/lwes_journal_listener.erl: -------------------------------------------------------------------------------- 1 | -module (lwes_journal_listener). 2 | 3 | -behaviour (gen_server). 4 | 5 | %% API 6 | -export ([ start_link/1, 7 | rescan/1 8 | ]). 9 | 10 | %% gen_server callbacks 11 | -export ([ init/1, 12 | handle_call/3, 13 | handle_cast/2, 14 | handle_info/2, 15 | terminate/2, 16 | code_change/3 17 | ]). 18 | 19 | -record (state, { dirs, 20 | timer, 21 | interval, 22 | delay, 23 | last_scan, 24 | callback 25 | } ). 26 | 27 | %%==================================================================== 28 | %% API 29 | %%==================================================================== 30 | start_link (Config) -> 31 | gen_server:start_link (?MODULE, [Config], []). 32 | 33 | rescan (Pid) -> 34 | gen_server:cast (Pid, {rescan}). 35 | 36 | %%==================================================================== 37 | %% gen_server callbacks 38 | %%==================================================================== 39 | init ([Config]) -> 40 | % get appication variables 41 | Interval = proplists:get_value (interval, Config, 60), 42 | Dirs = proplists:get_value (dirs, Config, ["."]), 43 | CallbackFunction = proplists:get_value (callback, Config, undefined), 44 | Delay = Interval * 1000, 45 | 46 | % I want terminate to be called 47 | process_flag (trap_exit, true), 48 | 49 | % setup checking for journals 50 | { ok, TRef } = timer:apply_interval (Delay, ?MODULE, rescan, [self()]), 51 | 52 | % Scan = rescan ([],Dirs), 53 | { ok, 54 | #state { 55 | dirs = Dirs, 56 | timer = TRef, 57 | interval = Interval, 58 | delay = Delay, 59 | last_scan = [], 60 | callback = CallbackFunction 61 | } 62 | }. 63 | 64 | handle_call (Request, From, State) -> 65 | error_logger:warning_msg ("Unrecognized call ~p from ~p~n",[Request, From]), 66 | { reply, ok, State }. 67 | 68 | handle_cast ({rescan}, State = #state { dirs = Dirs, 69 | last_scan = LastScan, 70 | callback = CB 71 | }) -> 72 | NewScan = rescan (LastScan, Dirs, CB), 73 | { noreply, State#state { last_scan = NewScan }}; 74 | handle_cast (Request, State) -> 75 | error_logger:warning_msg ("Unrecognized cast ~p~n",[Request]), 76 | { noreply, State }. 77 | 78 | handle_info (Request, State) -> 79 | error_logger:warning_msg ("Unrecognized info ~p~n",[Request]), 80 | {noreply, State}. 81 | 82 | terminate (_Reason, #state {}) -> 83 | ok. 84 | 85 | code_change (_OldVsn, State, _Extra) -> 86 | {ok, State}. 87 | 88 | rescan (LastScan, Dirs, CB) -> 89 | NewScan = lwes_file_watcher:scan (Dirs), 90 | case lwes_file_watcher:changes (LastScan, NewScan) of 91 | [] -> 92 | io:format ("No Changes!~n",[]), 93 | ok; 94 | Changes -> 95 | [ 96 | case E of 97 | {file, added, F} -> 98 | case re:run (F, "[^\\d]\.(\\d+)\.(\\d+)\.(\\d+).gz", 99 | [{capture, all_but_first, list}]) of 100 | {match, [_, _, _]} -> 101 | Start = os:timestamp (), 102 | process_journal (F, CB), 103 | io:format ("took ~p millis to process ~p~n", 104 | [millis_diff (os:timestamp(), Start), F]), 105 | file:delete (F); 106 | nomatch -> 107 | ok 108 | end; 109 | _ -> 110 | ok 111 | end 112 | || E 113 | <- Changes 114 | ], 115 | ok 116 | end, 117 | NewScan. 118 | 119 | -define(KILO, 1000). 120 | -define(MEGA, 1000000). 121 | -define(GIGA, 1000000000). 122 | -define(TERA, 1000000000000). 123 | 124 | millis_diff ({M,S,U}, {M,S1,U1}) -> 125 | ((S-S1) * ?KILO) + ((U-U1) div ?KILO); 126 | millis_diff ({M,S,U}, {M1,S1,U1}) -> 127 | ((M-M1)*?MEGA + (S-S1))*?KILO + ((U-U1) div ?KILO). 128 | 129 | process_journal (File, CB) -> 130 | {ok, Dev} = file:open (File, [read, raw, compressed, binary]), 131 | case read_next (Dev, CB) of 132 | eof -> ok; 133 | E -> E 134 | end. 135 | 136 | read_next (Dev, CB) -> 137 | case file:read (Dev, 22) of 138 | {ok, <> } -> 148 | case file:read (Dev, S) of 149 | {ok, B} -> 150 | CB ({udp, M, {V1, V2, V3, V4}, P, B}, ok), 151 | read_next (Dev, CB); 152 | E -> E 153 | end; 154 | E -> E 155 | end. 156 | 157 | %%-------------------------------------------------------------------- 158 | %%% Test functions 159 | %%-------------------------------------------------------------------- 160 | -ifdef (TEST). 161 | -include_lib ("eunit/include/eunit.hrl"). 162 | 163 | 164 | -endif. 165 | -------------------------------------------------------------------------------- /src/lwes_journaller.erl: -------------------------------------------------------------------------------- 1 | % 2 | % This module implements an lwes journaller. 3 | % 4 | % configuration is as follows 5 | % 6 | % [ { root, "." }, % journal root 7 | % { name, "all_events.log" }, % journal name 8 | % { interval, }, % interval for jouirnal file rotation 9 | % ] 10 | 11 | -module (lwes_journaller). 12 | 13 | -behaviour (gen_server). 14 | 15 | -include_lib ("lwes_internal.hrl"). 16 | 17 | %% API 18 | -export ([ start_link/1, 19 | process_event/2, 20 | rotate/1, 21 | format_header/5 ]). 22 | 23 | %% gen_server callbacks 24 | -export ([ init/1, 25 | handle_call/3, 26 | handle_cast/2, 27 | handle_info/2, 28 | terminate/2, 29 | code_change/3 30 | ]). 31 | 32 | -record (state, { 33 | journal_root, 34 | journal_file_name, 35 | journal_file_ext, 36 | journal_current, 37 | journal_last_rotate, 38 | timer 39 | }). 40 | 41 | %%==================================================================== 42 | %% API 43 | %%==================================================================== 44 | start_link (Config) -> 45 | gen_server:start_link ( ?MODULE, [Config], []). 46 | 47 | process_event (Event, Pid) -> 48 | gen_server:cast (Pid, {process, Event}), 49 | Pid. 50 | 51 | rotate (Pid) -> 52 | gen_server:cast (Pid, {rotate}). 53 | 54 | %%==================================================================== 55 | %% gen_server callbacks 56 | %%==================================================================== 57 | init ([Config]) -> 58 | % get appication variables 59 | Root = proplists:get_value (root, Config, "."), 60 | Name = proplists:get_value (name, Config, "all_events.log"), 61 | Interval = proplists:get_value (interval, Config, 60), 62 | Ext = "gz", 63 | 64 | % I want terminate to be called 65 | process_flag (trap_exit, true), 66 | 67 | % open journal file 68 | { ok, File } = open (Root, Name, Ext), 69 | 70 | % setup rotation of journal 71 | { ok, TRef } = 72 | timer:apply_interval (Interval * 1000, ?MODULE, rotate, [self()]), 73 | 74 | { ok, #state { 75 | journal_root = Root, 76 | journal_file_name = Name, 77 | journal_file_ext = Ext, 78 | journal_current = File, 79 | journal_last_rotate = seconds_since_epoch (), 80 | timer = TRef 81 | } 82 | }. 83 | 84 | handle_call (Request, From, State) -> 85 | error_logger:warning_msg ("Unrecognized call ~p from ~p~n",[Request, From]), 86 | { reply, ok, State }. 87 | 88 | format_header (EventSize, MillisTimestamp, Ip = {Ip1,Ip2,Ip3,Ip4}, Port, SiteId) 89 | when ?is_uint16(EventSize), ?is_uint64(MillisTimestamp), ?is_ip_addr (Ip), 90 | ?is_uint16(Port), ?is_uint16(SiteId) -> 91 | <>. % 22 bytes total 101 | 102 | handle_cast ( {process, {udp, _, Ip, Port, B}}, 103 | State = #state { journal_current = Journal }) -> 104 | S = byte_size (B), 105 | M = milliseconds_since_epoch (), 106 | SiteId = 1, 107 | ok = file:write ( Journal, 108 | [ format_header(S, M, Ip, Port, SiteId), 109 | B 110 | ]), 111 | { noreply, State }; 112 | handle_cast ( {rotate}, State = #state { 113 | journal_root = Root, 114 | journal_file_name = Name, 115 | journal_file_ext = Ext, 116 | journal_current = File, 117 | journal_last_rotate = LastRotate 118 | }) -> 119 | file:close (File), 120 | rename (Root, Name, Ext, LastRotate), 121 | {ok, NewFile} = open (Root, Name, Ext), 122 | { noreply, State#state { journal_current = NewFile, 123 | journal_last_rotate = seconds_since_epoch () }}; 124 | handle_cast (Request, State) -> 125 | error_logger:warning_msg ("Unrecognized cast ~p~n",[Request]), 126 | { noreply, State }. 127 | 128 | handle_info (Request, State) -> 129 | error_logger:warning_msg ("Unrecognized info ~p~n",[Request]), 130 | {noreply, State}. 131 | 132 | terminate (_Reason, #state { 133 | journal_root = Root, 134 | journal_file_name = Name, 135 | journal_file_ext = Ext, 136 | journal_current = File, 137 | journal_last_rotate = LastRotate 138 | }) -> 139 | file:close (File), 140 | rename (Root, Name, Ext, LastRotate), 141 | ok. 142 | 143 | code_change (_OldVsn, State, _Extra) -> 144 | {ok, State}. 145 | 146 | %%==================================================================== 147 | %% Internal 148 | %%==================================================================== 149 | open (Root, Name, Ext) -> 150 | JournalFile = filename:join ([Root, string:join ([Name, Ext],".")]), 151 | file:open (JournalFile, [ write, raw, compressed ]). 152 | 153 | rename (Root, Name, Ext, LastRotate) -> 154 | {{Year,Month,Day},{Hour,Minute,Second}} = 155 | calendar:now_to_universal_time(os:timestamp()), 156 | NewFile = 157 | filename:join 158 | ([Root, 159 | io_lib:format 160 | ("~s.~4.10.0B~2.10.0B~2.10.0B~2.10.0B~2.10.0B~2.10.0B.~b.~b.~s", 161 | [ Name, Year, Month, Day, Hour, Minute, Second, LastRotate, 162 | seconds_since_epoch(), Ext])]), 163 | CurrentFile = filename:join ([Root, string:join ([Name, Ext],".")]), 164 | error_logger:info_msg("rename ~p -> ~p",[CurrentFile, NewFile]), 165 | ok = file:rename (CurrentFile, NewFile). 166 | 167 | milliseconds_since_epoch () -> 168 | {Meg, Sec, Mic} = os:timestamp(), 169 | trunc (Meg * 1000000000 + Sec * 1000 + Mic / 1000). 170 | 171 | seconds_since_epoch () -> 172 | {M, S, _ } = os:timestamp(), 173 | M*1000000+S. 174 | 175 | %%==================================================================== 176 | %% Test functions 177 | %%==================================================================== 178 | -ifdef (TEST). 179 | -include_lib ("eunit/include/eunit.hrl"). 180 | 181 | -endif. 182 | -------------------------------------------------------------------------------- /src/lwes_mochijson2.erl: -------------------------------------------------------------------------------- 1 | % This is the MIT license. 2 | % 3 | % Copyright (c) 2007 Mochi Media, Inc. 4 | % 5 | % Permission is hereby granted, free of charge, to any person obtaining a 6 | % copy of this software and associated documentation files (the "Software"), 7 | % to deal in the Software without restriction, including without limitation 8 | % the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 | % and/or sell copies of the Software, and to permit persons to whom the 10 | % Software is furnished to do so, subject to the following conditions: 11 | % 12 | % The above copyright notice and this permission notice shall be included 13 | % in all copies or substantial portions of the Software. 14 | % 15 | % THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 16 | % OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | % FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 | % THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 | % OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 | % ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 21 | % OR OTHER DEALINGS IN THE SOFTWARE. 22 | % 23 | 24 | %% @author Bob Ippolito 25 | %% @copyright 2007 Mochi Media, Inc. 26 | 27 | %% @doc Yet another JSON (RFC 4627) library for Erlang. mochijson2 works 28 | %% with binaries as strings, arrays as lists (without an {array, _}) 29 | %% wrapper and it only knows how to decode UTF-8 (and ASCII). 30 | %% 31 | %% JSON terms are decoded as follows (javascript -> erlang): 32 | %%
    33 | %%
  • {"key": "value"} -> 34 | %% {struct, [{<<"key">>, <<"value">>}]}
    • 35 | %%
  • ["array", 123, 12.34, true, false, null] -> 36 | %% [<<"array">>, 123, 12.34, true, false, null] 37 | %%
    • 38 | %%
39 | %%
    40 | %%
  • Strings in JSON decode to UTF-8 binaries in Erlang
    • 41 | %%
  • Objects decode to {struct, PropList}
    • 42 | %%
  • Numbers decode to integer or float
    • 43 | %%
  • true, false, null decode to their respective terms.
    • 44 | %%
45 | %% The encoder will accept the same format that the decoder will produce, 46 | %% but will also allow additional cases for leniency: 47 | %%
    48 | %%
  • atoms other than true, false, null will be considered UTF-8 49 | %% strings (even as a proplist key) 50 | %%
    • 51 | %%
  • {json, IoList} will insert IoList directly into the output 52 | %% with no validation 53 | %%
    • 54 | %%
  • {array, Array} will be encoded as Array 55 | %% (legacy mochijson style) 56 | %%
    • 57 | %%
  • A non-empty raw proplist will be encoded as an object as long 58 | %% as the first pair does not have an atom key of json, struct, 59 | %% or array 60 | %%
    • 61 | %%
62 | 63 | -module(lwes_mochijson2). 64 | -author('bob@mochimedia.com'). 65 | -export([encoder/1, encode/1]). 66 | -export([decoder/1, decode/1, decode/2]). 67 | 68 | %% This is a macro to placate syntax highlighters.. 69 | -define(Q, $\"). 70 | -define(ADV_COL(S, N), S#decoder{offset=N+S#decoder.offset, 71 | column=N+S#decoder.column}). 72 | -define(INC_COL(S), S#decoder{offset=1+S#decoder.offset, 73 | column=1+S#decoder.column}). 74 | -define(INC_LINE(S), S#decoder{offset=1+S#decoder.offset, 75 | column=1, 76 | line=1+S#decoder.line}). 77 | -define(INC_CHAR(S, C), 78 | case C of 79 | $\n -> 80 | S#decoder{column=1, 81 | line=1+S#decoder.line, 82 | offset=1+S#decoder.offset}; 83 | _ -> 84 | S#decoder{column=1+S#decoder.column, 85 | offset=1+S#decoder.offset} 86 | end). 87 | -define(IS_WHITESPACE(C), 88 | (C =:= $\s orelse C =:= $\t orelse C =:= $\r orelse C =:= $\n)). 89 | 90 | %% @type json_string() = atom | binary() 91 | %% @type json_number() = integer() | float() 92 | %% @type json_array() = [json_term()] 93 | %% @type json_object() = {struct, [{json_string(), json_term()}]} 94 | %% @type json_eep18_object() = {[{json_string(), json_term()}]} 95 | %% @type json_iolist() = {json, iolist()} 96 | %% @type json_term() = json_string() | json_number() | json_array() | 97 | %% json_object() | json_eep18_object() | json_iolist() 98 | 99 | -record(encoder, {handler=null, 100 | utf8=false}). 101 | 102 | -record(decoder, {object_hook=null, 103 | offset=0, 104 | line=1, 105 | column=1, 106 | state=null}). 107 | 108 | %% @spec encoder([encoder_option()]) -> function() 109 | %% @doc Create an encoder/1 with the given options. 110 | %% @type encoder_option() = handler_option() | utf8_option() 111 | %% @type utf8_option() = boolean(). Emit unicode as utf8 (default - false) 112 | encoder(Options) -> 113 | State = parse_encoder_options(Options, #encoder{}), 114 | fun (O) -> json_encode(O, State) end. 115 | 116 | %% @spec encode(json_term()) -> iolist() 117 | %% @doc Encode the given as JSON to an iolist. 118 | encode(Any) -> 119 | json_encode(Any, #encoder{}). 120 | 121 | %% @spec decoder([decoder_option()]) -> function() 122 | %% @doc Create a decoder/1 with the given options. 123 | decoder(Options) -> 124 | State = parse_decoder_options(Options, #decoder{}), 125 | fun (O) -> json_decode(O, State) end. 126 | 127 | %% @spec decode(iolist(), [{format, proplist | eep18 | struct}]) -> json_term() 128 | %% @doc Decode the given iolist to Erlang terms using the given object format 129 | %% for decoding, where proplist returns JSON objects as [{binary(), json_term()}] 130 | %% proplists, eep18 returns JSON objects as {[binary(), json_term()]}, and struct 131 | %% returns them as-is. 132 | decode(S, Options) -> 133 | json_decode(S, parse_decoder_options(Options, #decoder{})). 134 | 135 | %% @spec decode(iolist()) -> json_term() 136 | %% @doc Decode the given iolist to Erlang terms. 137 | decode(S) -> 138 | json_decode(S, #decoder{}). 139 | 140 | %% Internal API 141 | 142 | parse_encoder_options([], State) -> 143 | State; 144 | parse_encoder_options([{handler, Handler} | Rest], State) -> 145 | parse_encoder_options(Rest, State#encoder{handler=Handler}); 146 | parse_encoder_options([{utf8, Switch} | Rest], State) -> 147 | parse_encoder_options(Rest, State#encoder{utf8=Switch}). 148 | 149 | parse_decoder_options([], State) -> 150 | State; 151 | parse_decoder_options([{object_hook, Hook} | Rest], State) -> 152 | parse_decoder_options(Rest, State#decoder{object_hook=Hook}); 153 | parse_decoder_options([{format, Format} | Rest], State) 154 | when Format =:= struct orelse Format =:= eep18 orelse Format =:= proplist -> 155 | parse_decoder_options(Rest, State#decoder{object_hook=Format}). 156 | 157 | json_encode(true, _State) -> 158 | <<"true">>; 159 | json_encode(false, _State) -> 160 | <<"false">>; 161 | json_encode(null, _State) -> 162 | <<"null">>; 163 | json_encode(I, _State) when is_integer(I) -> 164 | integer_to_list(I); 165 | json_encode(F, _State) when is_float(F) -> 166 | lwes_mochinum:digits(F); 167 | json_encode(S, State) when is_binary(S); is_atom(S) -> 168 | json_encode_string(S, State); 169 | json_encode([{K, _}|_] = Props, State) when (K =/= struct andalso 170 | K =/= array andalso 171 | K =/= json) -> 172 | json_encode_proplist(Props, State); 173 | json_encode({struct, Props}, State) when is_list(Props) -> 174 | json_encode_proplist(Props, State); 175 | json_encode({Props}, State) when is_list(Props) -> 176 | json_encode_proplist(Props, State); 177 | json_encode({}, State) -> 178 | json_encode_proplist([], State); 179 | json_encode(Array, State) when is_list(Array) -> 180 | json_encode_array(Array, State); 181 | json_encode({array, Array}, State) when is_list(Array) -> 182 | json_encode_array(Array, State); 183 | json_encode({json, IoList}, _State) -> 184 | IoList; 185 | json_encode(Bad, #encoder{handler=null}) -> 186 | exit({json_encode, {bad_term, Bad}}); 187 | json_encode(Bad, State=#encoder{handler=Handler}) -> 188 | json_encode(Handler(Bad), State). 189 | 190 | json_encode_array([], _State) -> 191 | <<"[]">>; 192 | json_encode_array(L, State) -> 193 | F = fun (O, Acc) -> 194 | [$,, json_encode(O, State) | Acc] 195 | end, 196 | [$, | Acc1] = lists:foldl(F, "[", L), 197 | lists:reverse([$\] | Acc1]). 198 | 199 | json_encode_proplist([], _State) -> 200 | <<"{}">>; 201 | json_encode_proplist(Props, State) -> 202 | F = fun ({K, V}, Acc) -> 203 | KS = json_encode_string(K, State), 204 | VS = json_encode(V, State), 205 | [$,, VS, $:, KS | Acc] 206 | end, 207 | [$, | Acc1] = lists:foldl(F, "{", Props), 208 | lists:reverse([$\} | Acc1]). 209 | 210 | json_encode_string(A, State) when is_atom(A) -> 211 | L = atom_to_list(A), 212 | case json_string_is_safe(L) of 213 | true -> 214 | [?Q, L, ?Q]; 215 | false -> 216 | json_encode_string_unicode(xmerl_ucs:from_utf8(L), State, [?Q]) 217 | end; 218 | json_encode_string(B, State) when is_binary(B) -> 219 | case json_bin_is_safe(B) of 220 | true -> 221 | [?Q, B, ?Q]; 222 | false -> 223 | json_encode_string_unicode(xmerl_ucs:from_utf8(B), State, [?Q]) 224 | end; 225 | json_encode_string(I, _State) when is_integer(I) -> 226 | [?Q, integer_to_list(I), ?Q]; 227 | json_encode_string(L, State) when is_list(L) -> 228 | case json_string_is_safe(L) of 229 | true -> 230 | [?Q, L, ?Q]; 231 | false -> 232 | json_encode_string_unicode(L, State, [?Q]) 233 | end. 234 | 235 | json_string_is_safe([]) -> 236 | true; 237 | json_string_is_safe([C | Rest]) -> 238 | case C of 239 | ?Q -> 240 | false; 241 | $\\ -> 242 | false; 243 | $\b -> 244 | false; 245 | $\f -> 246 | false; 247 | $\n -> 248 | false; 249 | $\r -> 250 | false; 251 | $\t -> 252 | false; 253 | C when C >= 0, C < $\s; C >= 16#7f, C =< 16#10FFFF -> 254 | false; 255 | C when C < 16#7f -> 256 | json_string_is_safe(Rest); 257 | _ -> 258 | false 259 | end. 260 | 261 | json_bin_is_safe(<<>>) -> 262 | true; 263 | json_bin_is_safe(<>) -> 264 | case C of 265 | ?Q -> 266 | false; 267 | $\\ -> 268 | false; 269 | $\b -> 270 | false; 271 | $\f -> 272 | false; 273 | $\n -> 274 | false; 275 | $\r -> 276 | false; 277 | $\t -> 278 | false; 279 | C when C >= 0, C < $\s; C >= 16#7f -> 280 | false; 281 | C when C < 16#7f -> 282 | json_bin_is_safe(Rest) 283 | end. 284 | 285 | json_encode_string_unicode([], _State, Acc) -> 286 | lists:reverse([$\" | Acc]); 287 | json_encode_string_unicode([C | Cs], State, Acc) -> 288 | Acc1 = case C of 289 | ?Q -> 290 | [?Q, $\\ | Acc]; 291 | %% Escaping solidus is only useful when trying to protect 292 | %% against "" injection attacks which are only 293 | %% possible when JSON is inserted into a HTML document 294 | %% in-line. mochijson2 does not protect you from this, so 295 | %% if you do insert directly into HTML then you need to 296 | %% uncomment the following case or escape the output of encode. 297 | %% 298 | %% $/ -> 299 | %% [$/, $\\ | Acc]; 300 | %% 301 | $\\ -> 302 | [$\\, $\\ | Acc]; 303 | $\b -> 304 | [$b, $\\ | Acc]; 305 | $\f -> 306 | [$f, $\\ | Acc]; 307 | $\n -> 308 | [$n, $\\ | Acc]; 309 | $\r -> 310 | [$r, $\\ | Acc]; 311 | $\t -> 312 | [$t, $\\ | Acc]; 313 | C when C >= 0, C < $\s -> 314 | [unihex(C) | Acc]; 315 | C when C >= 16#7f, C =< 16#10FFFF, State#encoder.utf8 -> 316 | [xmerl_ucs:to_utf8(C) | Acc]; 317 | C when C >= 16#7f, C =< 16#10FFFF, not State#encoder.utf8 -> 318 | [unihex(C) | Acc]; 319 | C when C < 16#7f -> 320 | [C | Acc]; 321 | _ -> 322 | exit({json_encode, {bad_char, C}}) 323 | end, 324 | json_encode_string_unicode(Cs, State, Acc1). 325 | 326 | hexdigit(C) when C >= 0, C =< 9 -> 327 | C + $0; 328 | hexdigit(C) when C =< 15 -> 329 | C + $a - 10. 330 | 331 | unihex(C) when C < 16#10000 -> 332 | <> = <>, 333 | Digits = [hexdigit(D) || D <- [D3, D2, D1, D0]], 334 | [$\\, $u | Digits]; 335 | unihex(C) when C =< 16#10FFFF -> 336 | N = C - 16#10000, 337 | S1 = 16#d800 bor ((N bsr 10) band 16#3ff), 338 | S2 = 16#dc00 bor (N band 16#3ff), 339 | [unihex(S1), unihex(S2)]. 340 | 341 | json_decode(L, S) when is_list(L) -> 342 | json_decode(iolist_to_binary(L), S); 343 | json_decode(B, S) -> 344 | {Res, S1} = decode1(B, S), 345 | {eof, _} = tokenize(B, S1#decoder{state=trim}), 346 | Res. 347 | 348 | decode1(B, S=#decoder{state=null}) -> 349 | case tokenize(B, S#decoder{state=any}) of 350 | {{const, C}, S1} -> 351 | {C, S1}; 352 | {start_array, S1} -> 353 | decode_array(B, S1); 354 | {start_object, S1} -> 355 | decode_object(B, S1) 356 | end. 357 | 358 | make_object(V, #decoder{object_hook=N}) when N =:= null orelse N =:= struct -> 359 | V; 360 | make_object({struct, P}, #decoder{object_hook=eep18}) -> 361 | {P}; 362 | make_object({struct, P}, #decoder{object_hook=proplist}) -> 363 | P; 364 | make_object(V, #decoder{object_hook=Hook}) -> 365 | Hook(V). 366 | 367 | decode_object(B, S) -> 368 | decode_object(B, S#decoder{state=key}, []). 369 | 370 | decode_object(B, S=#decoder{state=key}, Acc) -> 371 | case tokenize(B, S) of 372 | {end_object, S1} -> 373 | V = make_object({struct, lists:reverse(Acc)}, S1), 374 | {V, S1#decoder{state=null}}; 375 | {{const, K}, S1} -> 376 | {colon, S2} = tokenize(B, S1), 377 | {V, S3} = decode1(B, S2#decoder{state=null}), 378 | decode_object(B, S3#decoder{state=comma}, [{K, V} | Acc]) 379 | end; 380 | decode_object(B, S=#decoder{state=comma}, Acc) -> 381 | case tokenize(B, S) of 382 | {end_object, S1} -> 383 | V = make_object({struct, lists:reverse(Acc)}, S1), 384 | {V, S1#decoder{state=null}}; 385 | {comma, S1} -> 386 | decode_object(B, S1#decoder{state=key}, Acc) 387 | end. 388 | 389 | decode_array(B, S) -> 390 | decode_array(B, S#decoder{state=any}, []). 391 | 392 | decode_array(B, S=#decoder{state=any}, Acc) -> 393 | case tokenize(B, S) of 394 | {end_array, S1} -> 395 | {lists:reverse(Acc), S1#decoder{state=null}}; 396 | {start_array, S1} -> 397 | {Array, S2} = decode_array(B, S1), 398 | decode_array(B, S2#decoder{state=comma}, [Array | Acc]); 399 | {start_object, S1} -> 400 | {Array, S2} = decode_object(B, S1), 401 | decode_array(B, S2#decoder{state=comma}, [Array | Acc]); 402 | {{const, Const}, S1} -> 403 | decode_array(B, S1#decoder{state=comma}, [Const | Acc]) 404 | end; 405 | decode_array(B, S=#decoder{state=comma}, Acc) -> 406 | case tokenize(B, S) of 407 | {end_array, S1} -> 408 | {lists:reverse(Acc), S1#decoder{state=null}}; 409 | {comma, S1} -> 410 | decode_array(B, S1#decoder{state=any}, Acc) 411 | end. 412 | 413 | tokenize_string(B, S=#decoder{offset=O}) -> 414 | case tokenize_string_fast(B, O) of 415 | {escape, O1} -> 416 | Length = O1 - O, 417 | S1 = ?ADV_COL(S, Length), 418 | <<_:O/binary, Head:Length/binary, _/binary>> = B, 419 | tokenize_string(B, S1, lists:reverse(binary_to_list(Head))); 420 | O1 -> 421 | Length = O1 - O, 422 | <<_:O/binary, String:Length/binary, ?Q, _/binary>> = B, 423 | {{const, String}, ?ADV_COL(S, Length + 1)} 424 | end. 425 | 426 | tokenize_string_fast(B, O) -> 427 | case B of 428 | <<_:O/binary, ?Q, _/binary>> -> 429 | O; 430 | <<_:O/binary, $\\, _/binary>> -> 431 | {escape, O}; 432 | <<_:O/binary, C1, _/binary>> when C1 < 128 -> 433 | tokenize_string_fast(B, 1 + O); 434 | <<_:O/binary, C1, C2, _/binary>> when C1 >= 194, C1 =< 223, 435 | C2 >= 128, C2 =< 191 -> 436 | tokenize_string_fast(B, 2 + O); 437 | <<_:O/binary, C1, C2, C3, _/binary>> when C1 >= 224, C1 =< 239, 438 | C2 >= 128, C2 =< 191, 439 | C3 >= 128, C3 =< 191 -> 440 | tokenize_string_fast(B, 3 + O); 441 | <<_:O/binary, C1, C2, C3, C4, _/binary>> when C1 >= 240, C1 =< 244, 442 | C2 >= 128, C2 =< 191, 443 | C3 >= 128, C3 =< 191, 444 | C4 >= 128, C4 =< 191 -> 445 | tokenize_string_fast(B, 4 + O); 446 | _ -> 447 | throw(invalid_utf8) 448 | end. 449 | 450 | tokenize_string(B, S=#decoder{offset=O}, Acc) -> 451 | case B of 452 | <<_:O/binary, ?Q, _/binary>> -> 453 | {{const, iolist_to_binary(lists:reverse(Acc))}, ?INC_COL(S)}; 454 | <<_:O/binary, "\\\"", _/binary>> -> 455 | tokenize_string(B, ?ADV_COL(S, 2), [$\" | Acc]); 456 | <<_:O/binary, "\\\\", _/binary>> -> 457 | tokenize_string(B, ?ADV_COL(S, 2), [$\\ | Acc]); 458 | <<_:O/binary, "\\/", _/binary>> -> 459 | tokenize_string(B, ?ADV_COL(S, 2), [$/ | Acc]); 460 | <<_:O/binary, "\\b", _/binary>> -> 461 | tokenize_string(B, ?ADV_COL(S, 2), [$\b | Acc]); 462 | <<_:O/binary, "\\f", _/binary>> -> 463 | tokenize_string(B, ?ADV_COL(S, 2), [$\f | Acc]); 464 | <<_:O/binary, "\\n", _/binary>> -> 465 | tokenize_string(B, ?ADV_COL(S, 2), [$\n | Acc]); 466 | <<_:O/binary, "\\r", _/binary>> -> 467 | tokenize_string(B, ?ADV_COL(S, 2), [$\r | Acc]); 468 | <<_:O/binary, "\\t", _/binary>> -> 469 | tokenize_string(B, ?ADV_COL(S, 2), [$\t | Acc]); 470 | <<_:O/binary, "\\u", C3, C2, C1, C0, Rest/binary>> -> 471 | C = erlang:list_to_integer([C3, C2, C1, C0], 16), 472 | if C > 16#D7FF, C < 16#DC00 -> 473 | %% coalesce UTF-16 surrogate pair 474 | <<"\\u", D3, D2, D1, D0, _/binary>> = Rest, 475 | D = erlang:list_to_integer([D3,D2,D1,D0], 16), 476 | [CodePoint] = xmerl_ucs:from_utf16be(<>), 478 | Acc1 = lists:reverse(xmerl_ucs:to_utf8(CodePoint), Acc), 479 | tokenize_string(B, ?ADV_COL(S, 12), Acc1); 480 | true -> 481 | Acc1 = lists:reverse(xmerl_ucs:to_utf8(C), Acc), 482 | tokenize_string(B, ?ADV_COL(S, 6), Acc1) 483 | end; 484 | <<_:O/binary, C1, _/binary>> when C1 < 128 -> 485 | tokenize_string(B, ?INC_CHAR(S, C1), [C1 | Acc]); 486 | <<_:O/binary, C1, C2, _/binary>> when C1 >= 194, C1 =< 223, 487 | C2 >= 128, C2 =< 191 -> 488 | tokenize_string(B, ?ADV_COL(S, 2), [C2, C1 | Acc]); 489 | <<_:O/binary, C1, C2, C3, _/binary>> when C1 >= 224, C1 =< 239, 490 | C2 >= 128, C2 =< 191, 491 | C3 >= 128, C3 =< 191 -> 492 | tokenize_string(B, ?ADV_COL(S, 3), [C3, C2, C1 | Acc]); 493 | <<_:O/binary, C1, C2, C3, C4, _/binary>> when C1 >= 240, C1 =< 244, 494 | C2 >= 128, C2 =< 191, 495 | C3 >= 128, C3 =< 191, 496 | C4 >= 128, C4 =< 191 -> 497 | tokenize_string(B, ?ADV_COL(S, 4), [C4, C3, C2, C1 | Acc]); 498 | _ -> 499 | throw(invalid_utf8) 500 | end. 501 | 502 | tokenize_number(B, S) -> 503 | case tokenize_number(B, sign, S, []) of 504 | {{int, Int}, S1} -> 505 | {{const, list_to_integer(Int)}, S1}; 506 | {{float, Float}, S1} -> 507 | {{const, list_to_float(Float)}, S1} 508 | end. 509 | 510 | tokenize_number(B, sign, S=#decoder{offset=O}, []) -> 511 | case B of 512 | <<_:O/binary, $-, _/binary>> -> 513 | tokenize_number(B, int, ?INC_COL(S), [$-]); 514 | _ -> 515 | tokenize_number(B, int, S, []) 516 | end; 517 | tokenize_number(B, int, S=#decoder{offset=O}, Acc) -> 518 | case B of 519 | <<_:O/binary, $0, _/binary>> -> 520 | tokenize_number(B, frac, ?INC_COL(S), [$0 | Acc]); 521 | <<_:O/binary, C, _/binary>> when C >= $1 andalso C =< $9 -> 522 | tokenize_number(B, int1, ?INC_COL(S), [C | Acc]) 523 | end; 524 | tokenize_number(B, int1, S=#decoder{offset=O}, Acc) -> 525 | case B of 526 | <<_:O/binary, C, _/binary>> when C >= $0 andalso C =< $9 -> 527 | tokenize_number(B, int1, ?INC_COL(S), [C | Acc]); 528 | _ -> 529 | tokenize_number(B, frac, S, Acc) 530 | end; 531 | tokenize_number(B, frac, S=#decoder{offset=O}, Acc) -> 532 | case B of 533 | <<_:O/binary, $., C, _/binary>> when C >= $0, C =< $9 -> 534 | tokenize_number(B, frac1, ?ADV_COL(S, 2), [C, $. | Acc]); 535 | <<_:O/binary, E, _/binary>> when E =:= $e orelse E =:= $E -> 536 | tokenize_number(B, esign, ?INC_COL(S), [$e, $0, $. | Acc]); 537 | _ -> 538 | {{int, lists:reverse(Acc)}, S} 539 | end; 540 | tokenize_number(B, frac1, S=#decoder{offset=O}, Acc) -> 541 | case B of 542 | <<_:O/binary, C, _/binary>> when C >= $0 andalso C =< $9 -> 543 | tokenize_number(B, frac1, ?INC_COL(S), [C | Acc]); 544 | <<_:O/binary, E, _/binary>> when E =:= $e orelse E =:= $E -> 545 | tokenize_number(B, esign, ?INC_COL(S), [$e | Acc]); 546 | _ -> 547 | {{float, lists:reverse(Acc)}, S} 548 | end; 549 | tokenize_number(B, esign, S=#decoder{offset=O}, Acc) -> 550 | case B of 551 | <<_:O/binary, C, _/binary>> when C =:= $- orelse C=:= $+ -> 552 | tokenize_number(B, eint, ?INC_COL(S), [C | Acc]); 553 | _ -> 554 | tokenize_number(B, eint, S, Acc) 555 | end; 556 | tokenize_number(B, eint, S=#decoder{offset=O}, Acc) -> 557 | case B of 558 | <<_:O/binary, C, _/binary>> when C >= $0 andalso C =< $9 -> 559 | tokenize_number(B, eint1, ?INC_COL(S), [C | Acc]) 560 | end; 561 | tokenize_number(B, eint1, S=#decoder{offset=O}, Acc) -> 562 | case B of 563 | <<_:O/binary, C, _/binary>> when C >= $0 andalso C =< $9 -> 564 | tokenize_number(B, eint1, ?INC_COL(S), [C | Acc]); 565 | _ -> 566 | {{float, lists:reverse(Acc)}, S} 567 | end. 568 | 569 | tokenize(B, S=#decoder{offset=O}) -> 570 | case B of 571 | <<_:O/binary, C, _/binary>> when ?IS_WHITESPACE(C) -> 572 | tokenize(B, ?INC_CHAR(S, C)); 573 | <<_:O/binary, "{", _/binary>> -> 574 | {start_object, ?INC_COL(S)}; 575 | <<_:O/binary, "}", _/binary>> -> 576 | {end_object, ?INC_COL(S)}; 577 | <<_:O/binary, "[", _/binary>> -> 578 | {start_array, ?INC_COL(S)}; 579 | <<_:O/binary, "]", _/binary>> -> 580 | {end_array, ?INC_COL(S)}; 581 | <<_:O/binary, ",", _/binary>> -> 582 | {comma, ?INC_COL(S)}; 583 | <<_:O/binary, ":", _/binary>> -> 584 | {colon, ?INC_COL(S)}; 585 | <<_:O/binary, "null", _/binary>> -> 586 | {{const, null}, ?ADV_COL(S, 4)}; 587 | <<_:O/binary, "true", _/binary>> -> 588 | {{const, true}, ?ADV_COL(S, 4)}; 589 | <<_:O/binary, "false", _/binary>> -> 590 | {{const, false}, ?ADV_COL(S, 5)}; 591 | <<_:O/binary, "\"", _/binary>> -> 592 | tokenize_string(B, ?INC_COL(S)); 593 | <<_:O/binary, C, _/binary>> when (C >= $0 andalso C =< $9) 594 | orelse C =:= $- -> 595 | tokenize_number(B, S); 596 | <<_:O/binary>> -> 597 | trim = S#decoder.state, 598 | {eof, S} 599 | end. 600 | %% 601 | %% Tests 602 | %% 603 | -ifdef(TEST). 604 | -include_lib("eunit/include/eunit.hrl"). 605 | 606 | 607 | %% testing constructs borrowed from the Yaws JSON implementation. 608 | 609 | %% Create an object from a list of Key/Value pairs. 610 | 611 | obj_new() -> 612 | {struct, []}. 613 | 614 | is_obj({struct, Props}) -> 615 | F = fun ({K, _}) when is_binary(K) -> true end, 616 | lists:all(F, Props). 617 | 618 | obj_from_list(Props) -> 619 | Obj = {struct, Props}, 620 | ?assert(is_obj(Obj)), 621 | Obj. 622 | 623 | %% Test for equivalence of Erlang terms. 624 | %% Due to arbitrary order of construction, equivalent objects might 625 | %% compare unequal as erlang terms, so we need to carefully recurse 626 | %% through aggregates (tuples and objects). 627 | 628 | equiv({struct, Props1}, {struct, Props2}) -> 629 | equiv_object(Props1, Props2); 630 | equiv(L1, L2) when is_list(L1), is_list(L2) -> 631 | equiv_list(L1, L2); 632 | equiv(N1, N2) when is_number(N1), is_number(N2) -> N1 == N2; 633 | equiv(B1, B2) when is_binary(B1), is_binary(B2) -> B1 == B2; 634 | equiv(A, A) when A =:= true orelse A =:= false orelse A =:= null -> true. 635 | 636 | %% Object representation and traversal order is unknown. 637 | %% Use the sledgehammer and sort property lists. 638 | 639 | equiv_object(Props1, Props2) -> 640 | L1 = lists:keysort(1, Props1), 641 | L2 = lists:keysort(1, Props2), 642 | Pairs = lists:zip(L1, L2), 643 | true = lists:all(fun({{K1, V1}, {K2, V2}}) -> 644 | equiv(K1, K2) and equiv(V1, V2) 645 | end, Pairs). 646 | 647 | %% Recursively compare tuple elements for equivalence. 648 | 649 | equiv_list([], []) -> 650 | true; 651 | equiv_list([V1 | L1], [V2 | L2]) -> 652 | equiv(V1, V2) andalso equiv_list(L1, L2). 653 | 654 | decode_test() -> 655 | [1199344435545.0, 1] = decode(<<"[1199344435545.0,1]">>), 656 | <<16#F0,16#9D,16#9C,16#95>> = decode([34,"\\ud835","\\udf15",34]). 657 | 658 | e2j_vec_test() -> 659 | test_one(e2j_test_vec(utf8), 1). 660 | 661 | test_one([], _N) -> 662 | %% io:format("~p tests passed~n", [N-1]), 663 | ok; 664 | test_one([{E, J} | Rest], N) -> 665 | %% io:format("[~p] ~p ~p~n", [N, E, J]), 666 | true = equiv(E, decode(J)), 667 | true = equiv(E, decode(encode(E))), 668 | test_one(Rest, 1+N). 669 | 670 | e2j_test_vec(utf8) -> 671 | [ 672 | {1, "1"}, 673 | {3.1416, "3.14160"}, %% text representation may truncate, trail zeroes 674 | {-1, "-1"}, 675 | {-3.1416, "-3.14160"}, 676 | {12.0e10, "1.20000e+11"}, 677 | {1.234E+10, "1.23400e+10"}, 678 | {-1.234E-10, "-1.23400e-10"}, 679 | {10.0, "1.0e+01"}, 680 | {123.456, "1.23456E+2"}, 681 | {10.0, "1e1"}, 682 | {<<"foo">>, "\"foo\""}, 683 | {<<"foo", 5, "bar">>, "\"foo\\u0005bar\""}, 684 | {<<"">>, "\"\""}, 685 | {<<"\n\n\n">>, "\"\\n\\n\\n\""}, 686 | {<<"\" \b\f\r\n\t\"">>, "\"\\\" \\b\\f\\r\\n\\t\\\"\""}, 687 | {obj_new(), "{}"}, 688 | {obj_from_list([{<<"foo">>, <<"bar">>}]), "{\"foo\":\"bar\"}"}, 689 | {obj_from_list([{<<"foo">>, <<"bar">>}, {<<"baz">>, 123}]), 690 | "{\"foo\":\"bar\",\"baz\":123}"}, 691 | {[], "[]"}, 692 | {[[]], "[[]]"}, 693 | {[1, <<"foo">>], "[1,\"foo\"]"}, 694 | 695 | %% json array in a json object 696 | {obj_from_list([{<<"foo">>, [123]}]), 697 | "{\"foo\":[123]}"}, 698 | 699 | %% json object in a json object 700 | {obj_from_list([{<<"foo">>, obj_from_list([{<<"bar">>, true}])}]), 701 | "{\"foo\":{\"bar\":true}}"}, 702 | 703 | %% fold evaluation order 704 | {obj_from_list([{<<"foo">>, []}, 705 | {<<"bar">>, obj_from_list([{<<"baz">>, true}])}, 706 | {<<"alice">>, <<"bob">>}]), 707 | "{\"foo\":[],\"bar\":{\"baz\":true},\"alice\":\"bob\"}"}, 708 | 709 | %% json object in a json array 710 | {[-123, <<"foo">>, obj_from_list([{<<"bar">>, []}]), null], 711 | "[-123,\"foo\",{\"bar\":[]},null]"} 712 | ]. 713 | 714 | %% test utf8 encoding 715 | encoder_utf8_test() -> 716 | %% safe conversion case (default) 717 | [34,"\\u0001","\\u0442","\\u0435","\\u0441","\\u0442",34] = 718 | encode(<<1,"\321\202\320\265\321\201\321\202">>), 719 | 720 | %% raw utf8 output (optional) 721 | Enc = encoder([{utf8, true}]), 722 | [34,"\\u0001",[209,130],[208,181],[209,129],[209,130],34] = 723 | Enc(<<1,"\321\202\320\265\321\201\321\202">>). 724 | 725 | input_validation_test() -> 726 | Good = [ 727 | {16#00A3, <>}, %% pound 728 | {16#20AC, <>}, %% euro 729 | {16#10196, <>} %% denarius 730 | ], 731 | lists:foreach(fun({CodePoint, UTF8}) -> 732 | Expect = list_to_binary(xmerl_ucs:to_utf8(CodePoint)), 733 | Expect = decode(UTF8) 734 | end, Good), 735 | 736 | Bad = [ 737 | %% 2nd, 3rd, or 4th byte of a multi-byte sequence w/o leading byte 738 | <>, 739 | %% missing continuations, last byte in each should be 80-BF 740 | <>, 741 | <>, 742 | <>, 743 | %% we don't support code points > 10FFFF per RFC 3629 744 | <>, 745 | %% escape characters trigger a different code path 746 | <> 747 | ], 748 | lists:foreach( 749 | fun(X) -> 750 | ok = try decode(X) catch invalid_utf8 -> ok end, 751 | %% could be {ucs,{bad_utf8_character_code}} or 752 | %% {json_encode,{bad_char,_}} 753 | {'EXIT', _} = (catch encode(X)) 754 | end, Bad). 755 | 756 | inline_json_test() -> 757 | ?assertEqual(<<"\"iodata iodata\"">>, 758 | iolist_to_binary( 759 | encode({json, [<<"\"iodata">>, " iodata\""]}))), 760 | ?assertEqual({struct, [{<<"key">>, <<"iodata iodata">>}]}, 761 | decode( 762 | encode({struct, 763 | [{key, {json, [<<"\"iodata">>, " iodata\""]}}]}))), 764 | ok. 765 | 766 | big_unicode_test() -> 767 | UTF8Seq = list_to_binary(xmerl_ucs:to_utf8(16#0001d120)), 768 | ?assertEqual( 769 | <<"\"\\ud834\\udd20\"">>, 770 | iolist_to_binary(encode(UTF8Seq))), 771 | ?assertEqual( 772 | UTF8Seq, 773 | decode(iolist_to_binary(encode(UTF8Seq)))), 774 | ok. 775 | 776 | custom_decoder_test() -> 777 | ?assertEqual( 778 | {struct, [{<<"key">>, <<"value">>}]}, 779 | (decoder([]))("{\"key\": \"value\"}")), 780 | F = fun ({struct, [{<<"key">>, <<"value">>}]}) -> win end, 781 | ?assertEqual( 782 | win, 783 | (decoder([{object_hook, F}]))("{\"key\": \"value\"}")), 784 | ok. 785 | 786 | atom_test() -> 787 | %% JSON native atoms 788 | [begin 789 | ?assertEqual(A, decode(atom_to_list(A))), 790 | ?assertEqual(iolist_to_binary(atom_to_list(A)), 791 | iolist_to_binary(encode(A))) 792 | end || A <- [true, false, null]], 793 | %% Atom to string 794 | ?assertEqual( 795 | <<"\"foo\"">>, 796 | iolist_to_binary(encode(foo))), 797 | ?assertEqual( 798 | <<"\"\\ud834\\udd20\"">>, 799 | iolist_to_binary(encode(list_to_atom(xmerl_ucs:to_utf8(16#0001d120))))), 800 | ok. 801 | 802 | key_encode_test() -> 803 | %% Some forms are accepted as keys that would not be strings in other 804 | %% cases 805 | ?assertEqual( 806 | <<"{\"foo\":1}">>, 807 | iolist_to_binary(encode({struct, [{foo, 1}]}))), 808 | ?assertEqual( 809 | <<"{\"foo\":1}">>, 810 | iolist_to_binary(encode({struct, [{<<"foo">>, 1}]}))), 811 | ?assertEqual( 812 | <<"{\"foo\":1}">>, 813 | iolist_to_binary(encode({struct, [{"foo", 1}]}))), 814 | ?assertEqual( 815 | <<"{\"foo\":1}">>, 816 | iolist_to_binary(encode([{foo, 1}]))), 817 | ?assertEqual( 818 | <<"{\"foo\":1}">>, 819 | iolist_to_binary(encode([{<<"foo">>, 1}]))), 820 | ?assertEqual( 821 | <<"{\"foo\":1}">>, 822 | iolist_to_binary(encode([{"foo", 1}]))), 823 | ?assertEqual( 824 | <<"{\"\\ud834\\udd20\":1}">>, 825 | iolist_to_binary( 826 | encode({struct, [{[16#0001d120], 1}]}))), 827 | ?assertEqual( 828 | <<"{\"1\":1}">>, 829 | iolist_to_binary(encode({struct, [{1, 1}]}))), 830 | ok. 831 | 832 | unsafe_chars_test() -> 833 | Chars = "\"\\\b\f\n\r\t", 834 | [begin 835 | ?assertEqual(false, json_string_is_safe([C])), 836 | ?assertEqual(false, json_bin_is_safe(<>)), 837 | ?assertEqual(<>, decode(encode(<>))) 838 | end || C <- Chars], 839 | ?assertEqual( 840 | false, 841 | json_string_is_safe([16#0001d120])), 842 | ?assertEqual( 843 | false, 844 | json_bin_is_safe(list_to_binary(xmerl_ucs:to_utf8(16#0001d120)))), 845 | ?assertEqual( 846 | [16#0001d120], 847 | xmerl_ucs:from_utf8( 848 | binary_to_list( 849 | decode(encode(list_to_atom(xmerl_ucs:to_utf8(16#0001d120))))))), 850 | ?assertEqual( 851 | false, 852 | json_string_is_safe([16#110000])), 853 | ?assertEqual( 854 | false, 855 | json_bin_is_safe(list_to_binary(xmerl_ucs:to_utf8([16#110000])))), 856 | %% solidus can be escaped but isn't unsafe by default 857 | ?assertEqual( 858 | <<"/">>, 859 | decode(<<"\"\\/\"">>)), 860 | ok. 861 | 862 | int_test() -> 863 | ?assertEqual(0, decode("0")), 864 | ?assertEqual(1, decode("1")), 865 | ?assertEqual(11, decode("11")), 866 | ok. 867 | 868 | large_int_test() -> 869 | ?assertEqual(<<"-2147483649214748364921474836492147483649">>, 870 | iolist_to_binary(encode(-2147483649214748364921474836492147483649))), 871 | ?assertEqual(<<"2147483649214748364921474836492147483649">>, 872 | iolist_to_binary(encode(2147483649214748364921474836492147483649))), 873 | ok. 874 | 875 | float_test() -> 876 | ?assertEqual(<<"-2147483649.0">>, iolist_to_binary(encode(-2147483649.0))), 877 | ?assertEqual(<<"2147483648.0">>, iolist_to_binary(encode(2147483648.0))), 878 | ok. 879 | 880 | handler_test() -> 881 | ?assertEqual( 882 | {'EXIT',{json_encode,{bad_term,{x,y}}}}, 883 | catch encode({x,y})), 884 | F = fun ({x,y}) -> [] end, 885 | ?assertEqual( 886 | <<"[]">>, 887 | iolist_to_binary((encoder([{handler, F}]))({x, y}))), 888 | ok. 889 | 890 | encode_empty_test_() -> 891 | [{A, ?_assertEqual(<<"{}">>, iolist_to_binary(encode(B)))} 892 | || {A, B} <- [{"eep18 {}", {}}, 893 | {"eep18 {[]}", {[]}}, 894 | {"{struct, []}", {struct, []}}]]. 895 | 896 | encode_test_() -> 897 | P = [{<<"k">>, <<"v">>}], 898 | JSON = iolist_to_binary(encode({struct, P})), 899 | [{atom_to_list(F), 900 | ?_assertEqual(JSON, iolist_to_binary(encode(decode(JSON, [{format, F}]))))} 901 | || F <- [struct, eep18, proplist]]. 902 | 903 | format_test_() -> 904 | P = [{<<"k">>, <<"v">>}], 905 | JSON = iolist_to_binary(encode({struct, P})), 906 | [{atom_to_list(F), 907 | ?_assertEqual(A, decode(JSON, [{format, F}]))} 908 | || {F, A} <- [{struct, {struct, P}}, 909 | {eep18, {P}}, 910 | {proplist, P}]]. 911 | 912 | -endif. 913 | -------------------------------------------------------------------------------- /src/lwes_mochinum.erl: -------------------------------------------------------------------------------- 1 | % This is the MIT license. 2 | % 3 | % Copyright (c) 2007 Mochi Media, Inc. 4 | % 5 | % Permission is hereby granted, free of charge, to any person obtaining a 6 | % copy of this software and associated documentation files (the "Software"), 7 | % to deal in the Software without restriction, including without limitation 8 | % the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 | % and/or sell copies of the Software, and to permit persons to whom the 10 | % Software is furnished to do so, subject to the following conditions: 11 | % 12 | % The above copyright notice and this permission notice shall be included 13 | % in all copies or substantial portions of the Software. 14 | % 15 | % THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 16 | % OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | % FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 | % THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 | % OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 | % ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE 21 | % OR OTHER DEALINGS IN THE SOFTWARE. 22 | % 23 | 24 | %% @copyright 2007 Mochi Media, Inc. 25 | %% @author Bob Ippolito 26 | 27 | %% @doc Useful numeric algorithms for floats that cover some deficiencies 28 | %% in the math module. More interesting is digits/1, which implements 29 | %% the algorithm from: 30 | %% http://www.cs.indiana.edu/~burger/fp/index.html 31 | %% See also "Printing Floating-Point Numbers Quickly and Accurately" 32 | %% in Proceedings of the SIGPLAN '96 Conference on Programming Language 33 | %% Design and Implementation. 34 | 35 | -module(lwes_mochinum). 36 | -author("Bob Ippolito "). 37 | -export([digits/1, frexp/1, int_pow/2, int_ceil/1]). 38 | 39 | %% IEEE 754 Float exponent bias 40 | -define(FLOAT_BIAS, 1022). 41 | -define(MIN_EXP, -1074). 42 | -define(BIG_POW, 4503599627370496). 43 | 44 | %% External API 45 | 46 | %% @spec digits(number()) -> string() 47 | %% @doc Returns a string that accurately represents the given integer or float 48 | %% using a conservative amount of digits. Great for generating 49 | %% human-readable output, or compact ASCII serializations for floats. 50 | digits(N) when is_integer(N) -> 51 | integer_to_list(N); 52 | digits(0.0) -> 53 | "0.0"; 54 | digits(Float) -> 55 | {Frac1, Exp1} = frexp_int(Float), 56 | [Place0 | Digits0] = digits1(Float, Exp1, Frac1), 57 | {Place, Digits} = transform_digits(Place0, Digits0), 58 | R = insert_decimal(Place, Digits), 59 | case Float < 0 of 60 | true -> 61 | [$- | R]; 62 | _ -> 63 | R 64 | end. 65 | 66 | %% @spec frexp(F::float()) -> {Frac::float(), Exp::float()} 67 | %% @doc Return the fractional and exponent part of an IEEE 754 double, 68 | %% equivalent to the libc function of the same name. 69 | %% F = Frac * pow(2, Exp). 70 | frexp(F) -> 71 | frexp1(unpack(F)). 72 | 73 | %% @spec int_pow(X::integer(), N::integer()) -> Y::integer() 74 | %% @doc Moderately efficient way to exponentiate integers. 75 | %% int_pow(10, 2) = 100. 76 | int_pow(_X, 0) -> 77 | 1; 78 | int_pow(X, N) when N > 0 -> 79 | int_pow(X, N, 1). 80 | 81 | %% @spec int_ceil(F::float()) -> integer() 82 | %% @doc Return the ceiling of F as an integer. The ceiling is defined as 83 | %% F when F == trunc(F); 84 | %% trunc(F) when F < 0; 85 | %% trunc(F) + 1 when F > 0. 86 | int_ceil(X) -> 87 | T = trunc(X), 88 | case (X - T) of 89 | Pos when Pos > 0 -> T + 1; 90 | _ -> T 91 | end. 92 | 93 | 94 | %% Internal API 95 | 96 | int_pow(X, N, R) when N < 2 -> 97 | R * X; 98 | int_pow(X, N, R) -> 99 | int_pow(X * X, N bsr 1, case N band 1 of 1 -> R * X; 0 -> R end). 100 | 101 | insert_decimal(0, S) -> 102 | "0." ++ S; 103 | insert_decimal(Place, S) when Place > 0 -> 104 | L = length(S), 105 | case Place - L of 106 | 0 -> 107 | S ++ ".0"; 108 | N when N < 0 -> 109 | {S0, S1} = lists:split(L + N, S), 110 | S0 ++ "." ++ S1; 111 | N when N < 6 -> 112 | %% More places than digits 113 | S ++ lists:duplicate(N, $0) ++ ".0"; 114 | _ -> 115 | insert_decimal_exp(Place, S) 116 | end; 117 | insert_decimal(Place, S) when Place > -6 -> 118 | "0." ++ lists:duplicate(abs(Place), $0) ++ S; 119 | insert_decimal(Place, S) -> 120 | insert_decimal_exp(Place, S). 121 | 122 | insert_decimal_exp(Place, S) -> 123 | [C | S0] = S, 124 | S1 = case S0 of 125 | [] -> 126 | "0"; 127 | _ -> 128 | S0 129 | end, 130 | Exp = case Place < 0 of 131 | true -> 132 | "e-"; 133 | false -> 134 | "e+" 135 | end, 136 | [C] ++ "." ++ S1 ++ Exp ++ integer_to_list(abs(Place - 1)). 137 | 138 | 139 | digits1(Float, Exp, Frac) -> 140 | Round = ((Frac band 1) =:= 0), 141 | case Exp >= 0 of 142 | true -> 143 | BExp = 1 bsl Exp, 144 | case (Frac =/= ?BIG_POW) of 145 | true -> 146 | scale((Frac * BExp * 2), 2, BExp, BExp, 147 | Round, Round, Float); 148 | false -> 149 | scale((Frac * BExp * 4), 4, (BExp * 2), BExp, 150 | Round, Round, Float) 151 | end; 152 | false -> 153 | case (Exp =:= ?MIN_EXP) orelse (Frac =/= ?BIG_POW) of 154 | true -> 155 | scale((Frac * 2), 1 bsl (1 - Exp), 1, 1, 156 | Round, Round, Float); 157 | false -> 158 | scale((Frac * 4), 1 bsl (2 - Exp), 2, 1, 159 | Round, Round, Float) 160 | end 161 | end. 162 | 163 | scale(R, S, MPlus, MMinus, LowOk, HighOk, Float) -> 164 | Est = int_ceil(math:log10(abs(Float)) - 1.0e-10), 165 | %% Note that the scheme implementation uses a 326 element look-up table 166 | %% for int_pow(10, N) where we do not. 167 | case Est >= 0 of 168 | true -> 169 | fixup(R, S * int_pow(10, Est), MPlus, MMinus, Est, 170 | LowOk, HighOk); 171 | false -> 172 | Scale = int_pow(10, -Est), 173 | fixup(R * Scale, S, MPlus * Scale, MMinus * Scale, Est, 174 | LowOk, HighOk) 175 | end. 176 | 177 | fixup(R, S, MPlus, MMinus, K, LowOk, HighOk) -> 178 | TooLow = case HighOk of 179 | true -> 180 | (R + MPlus) >= S; 181 | false -> 182 | (R + MPlus) > S 183 | end, 184 | case TooLow of 185 | true -> 186 | [(K + 1) | generate(R, S, MPlus, MMinus, LowOk, HighOk)]; 187 | false -> 188 | [K | generate(R * 10, S, MPlus * 10, MMinus * 10, LowOk, HighOk)] 189 | end. 190 | 191 | generate(R0, S, MPlus, MMinus, LowOk, HighOk) -> 192 | D = R0 div S, 193 | R = R0 rem S, 194 | TC1 = case LowOk of 195 | true -> 196 | R =< MMinus; 197 | false -> 198 | R < MMinus 199 | end, 200 | TC2 = case HighOk of 201 | true -> 202 | (R + MPlus) >= S; 203 | false -> 204 | (R + MPlus) > S 205 | end, 206 | case TC1 of 207 | false -> 208 | case TC2 of 209 | false -> 210 | [D | generate(R * 10, S, MPlus * 10, MMinus * 10, 211 | LowOk, HighOk)]; 212 | true -> 213 | [D + 1] 214 | end; 215 | true -> 216 | case TC2 of 217 | false -> 218 | [D]; 219 | true -> 220 | case R * 2 < S of 221 | true -> 222 | [D]; 223 | false -> 224 | [D + 1] 225 | end 226 | end 227 | end. 228 | 229 | unpack(Float) -> 230 | <> = <>, 231 | {Sign, Exp, Frac}. 232 | 233 | frexp1({_Sign, 0, 0}) -> 234 | {0.0, 0}; 235 | frexp1({Sign, 0, Frac}) -> 236 | Exp = log2floor(Frac), 237 | <> = <>, 238 | {Frac1, -(?FLOAT_BIAS) - 52 + Exp}; 239 | frexp1({Sign, Exp, Frac}) -> 240 | <> = <>, 241 | {Frac1, Exp - ?FLOAT_BIAS}. 242 | 243 | log2floor(Int) -> 244 | log2floor(Int, 0). 245 | 246 | log2floor(0, N) -> 247 | N; 248 | log2floor(Int, N) -> 249 | log2floor(Int bsr 1, 1 + N). 250 | 251 | 252 | transform_digits(Place, [0 | Rest]) -> 253 | transform_digits(Place, Rest); 254 | transform_digits(Place, Digits) -> 255 | {Place, [$0 + D || D <- Digits]}. 256 | 257 | 258 | frexp_int(F) -> 259 | case unpack(F) of 260 | {_Sign, 0, Frac} -> 261 | {Frac, ?MIN_EXP}; 262 | {_Sign, Exp, Frac} -> 263 | {Frac + (1 bsl 52), Exp - 53 - ?FLOAT_BIAS} 264 | end. 265 | 266 | %% 267 | %% Tests 268 | %% 269 | -ifdef(TEST). 270 | -include_lib("eunit/include/eunit.hrl"). 271 | 272 | int_ceil_test() -> 273 | ?assertEqual(1, int_ceil(0.0001)), 274 | ?assertEqual(0, int_ceil(0.0)), 275 | ?assertEqual(1, int_ceil(0.99)), 276 | ?assertEqual(1, int_ceil(1.0)), 277 | ?assertEqual(-1, int_ceil(-1.5)), 278 | ?assertEqual(-2, int_ceil(-2.0)), 279 | ok. 280 | 281 | int_pow_test() -> 282 | ?assertEqual(1, int_pow(1, 1)), 283 | ?assertEqual(1, int_pow(1, 0)), 284 | ?assertEqual(1, int_pow(10, 0)), 285 | ?assertEqual(10, int_pow(10, 1)), 286 | ?assertEqual(100, int_pow(10, 2)), 287 | ?assertEqual(1000, int_pow(10, 3)), 288 | ok. 289 | 290 | digits_test() -> 291 | ?assertEqual("0", 292 | digits(0)), 293 | ?assertEqual("0.0", 294 | digits(0.0)), 295 | ?assertEqual("1.0", 296 | digits(1.0)), 297 | ?assertEqual("-1.0", 298 | digits(-1.0)), 299 | ?assertEqual("0.1", 300 | digits(0.1)), 301 | ?assertEqual("0.01", 302 | digits(0.01)), 303 | ?assertEqual("0.001", 304 | digits(0.001)), 305 | ?assertEqual("1.0e+6", 306 | digits(1000000.0)), 307 | ?assertEqual("0.5", 308 | digits(0.5)), 309 | ?assertEqual("4503599627370496.0", 310 | digits(4503599627370496.0)), 311 | %% small denormalized number 312 | %% 4.94065645841246544177e-324 =:= 5.0e-324 313 | <> = <<0,0,0,0,0,0,0,1>>, 314 | ?assertEqual("5.0e-324", 315 | digits(SmallDenorm)), 316 | ?assertEqual(SmallDenorm, 317 | list_to_float(digits(SmallDenorm))), 318 | %% large denormalized number 319 | %% 2.22507385850720088902e-308 320 | <> = <<0,15,255,255,255,255,255,255>>, 321 | ?assertEqual("2.225073858507201e-308", 322 | digits(BigDenorm)), 323 | ?assertEqual(BigDenorm, 324 | list_to_float(digits(BigDenorm))), 325 | %% small normalized number 326 | %% 2.22507385850720138309e-308 327 | <> = <<0,16,0,0,0,0,0,0>>, 328 | ?assertEqual("2.2250738585072014e-308", 329 | digits(SmallNorm)), 330 | ?assertEqual(SmallNorm, 331 | list_to_float(digits(SmallNorm))), 332 | %% large normalized number 333 | %% 1.79769313486231570815e+308 334 | <> = <<127,239,255,255,255,255,255,255>>, 335 | ?assertEqual("1.7976931348623157e+308", 336 | digits(LargeNorm)), 337 | ?assertEqual(LargeNorm, 338 | list_to_float(digits(LargeNorm))), 339 | %% issue #10 - mochinum:frexp(math:pow(2, -1074)). 340 | ?assertEqual("5.0e-324", 341 | digits(math:pow(2, -1074))), 342 | ok. 343 | 344 | frexp_test() -> 345 | %% zero 346 | ?assertEqual({0.0, 0}, frexp(0.0)), 347 | %% one 348 | ?assertEqual({0.5, 1}, frexp(1.0)), 349 | %% negative one 350 | ?assertEqual({-0.5, 1}, frexp(-1.0)), 351 | %% small denormalized number 352 | %% 4.94065645841246544177e-324 353 | <> = <<0,0,0,0,0,0,0,1>>, 354 | ?assertEqual({0.5, -1073}, frexp(SmallDenorm)), 355 | %% large denormalized number 356 | %% 2.22507385850720088902e-308 357 | <> = <<0,15,255,255,255,255,255,255>>, 358 | ?assertEqual( 359 | {0.99999999999999978, -1022}, 360 | frexp(BigDenorm)), 361 | %% small normalized number 362 | %% 2.22507385850720138309e-308 363 | <> = <<0,16,0,0,0,0,0,0>>, 364 | ?assertEqual({0.5, -1021}, frexp(SmallNorm)), 365 | %% large normalized number 366 | %% 1.79769313486231570815e+308 367 | <> = <<127,239,255,255,255,255,255,255>>, 368 | ?assertEqual( 369 | {0.99999999999999989, 1024}, 370 | frexp(LargeNorm)), 371 | %% issue #10 - mochinum:frexp(math:pow(2, -1074)). 372 | ?assertEqual( 373 | {0.5, -1073}, 374 | frexp(math:pow(2, -1074))), 375 | ok. 376 | 377 | -endif. 378 | -------------------------------------------------------------------------------- /src/lwes_multi_emitter.erl: -------------------------------------------------------------------------------- 1 | % 2 | % This module implements an M of N emitter. Given N different channels, 3 | % when emit is called, the event is emitted over M of them. You can use 4 | % this in situations where you want to mimic multicast by emitting the 5 | % same event to multiple places, in addition to load balancing the emission 6 | % across a set of machines 7 | % 8 | -module (lwes_multi_emitter). 9 | 10 | -include ("lwes_internal.hrl"). 11 | 12 | %% API 13 | -export ([ new/1, 14 | select/1, 15 | emit/2, 16 | close/1 17 | ]). 18 | 19 | %%==================================================================== 20 | %% API 21 | %%==================================================================== 22 | new (Config) -> 23 | new0 (normalize_emitters_config(Config)). 24 | 25 | new0 ({NumToSelect, Type, ListOfSubConfigs}) 26 | when is_integer(NumToSelect), 27 | (is_atom(Type) andalso (Type =:= group orelse Type =:= random)), 28 | is_list(ListOfSubConfigs) -> 29 | 30 | Max = length(ListOfSubConfigs), 31 | case NumToSelect of 32 | _ when NumToSelect >= 1; NumToSelect > Max -> 33 | { ok, 34 | #lwes_multi_emitter { 35 | type = Type, 36 | max = Max, 37 | num = NumToSelect, 38 | configs = [ begin 39 | Cout = 40 | case new0(Config) of 41 | {ok, C} -> C; 42 | C -> C 43 | end, 44 | Cout 45 | end 46 | || Config <- ListOfSubConfigs ] 47 | } 48 | }; 49 | _ -> 50 | { error, bad_m_value } 51 | end; 52 | new0 ({Module,Config}) when is_atom(Module), is_list(Config) -> 53 | Emitter = Module:new (Config), 54 | lwes_stats:initialize(Module:id(Emitter)), 55 | {Module, Emitter}; 56 | new0 (Config) -> 57 | Emitter = lwes_emitter_udp:new(Config), 58 | lwes_stats:initialize(lwes_emitter_udp:id(Emitter)), 59 | {lwes_emitter_udp, Emitter}. 60 | 61 | % Normalize all emitter configuration to the form used by lwes_multi_emitter. 62 | % 63 | % This clause is for config of pluggable emission for a single emitter 64 | % {module_implementing_emitter_behaviour, Config} 65 | normalize_emitters_config ({M, Config}) when is_atom(M) -> 66 | {1, group, [{M, Config}]}; 67 | % This clause is for config which has a number and a list of SubConfigs 68 | % {NumberToEmitTo, ListOfSubConfigs} 69 | normalize_emitters_config ({N, L}) when is_integer(N), is_list(L) -> 70 | {N, group, L}; 71 | % This clause is for the simple case of a single Ip/Port so 72 | % {"127.0.0.1",9191} 73 | normalize_emitters_config (C = {Ip, Port}) when is_list(Ip), ?is_uint16(Port) -> 74 | {1, group, [C]}; 75 | % This clause is for the simple case of a single Ip/Port so 76 | % {"127.0.0.1",9191, [other,config]} 77 | normalize_emitters_config (C = {Ip, Port,Config}) 78 | when is_list(Ip), ?is_uint16(Port), is_list(Config) -> 79 | {1, group, [C]}; 80 | % This is for the case where the config is actually already in the form 81 | % the lwes_mulit_emitter recognizes 82 | normalize_emitters_config ({NumToSelect, Type, ListOfSubConfigs}) 83 | when is_integer(NumToSelect), 84 | (is_atom(Type) andalso (Type =:= group orelse Type =:= random)), 85 | is_list(ListOfSubConfigs) -> 86 | case ListOfSubConfigs of 87 | L = [{_,_}] -> 88 | {NumToSelect, Type, L}; 89 | L = [{_,Port,Config}] when ?is_uint16(Port), is_list(Config) -> 90 | {NumToSelect, Type, L}; 91 | L when is_list(L) -> 92 | {NumToSelect, Type,[ sub_normalize(S) || S <- ListOfSubConfigs ]} 93 | end. 94 | 95 | sub_normalize (C = {Ip, Port}) when is_list(Ip), ?is_uint16(Port) -> 96 | C; 97 | sub_normalize (C = {Ip, Port, Config}) 98 | when is_list(Ip), ?is_uint16(Port), is_list(Config) -> 99 | C; 100 | sub_normalize (C = {M, Config}) when is_atom(M), is_list(Config) -> 101 | C; 102 | sub_normalize (C) -> 103 | C. 104 | 105 | % returns a sorted list of places to emit to in the form 106 | % [ {EmitterModule, [State0, ...]}, ... ] 107 | select (C) -> 108 | collate(lists:sort(lists:flatten(select0(C)))). 109 | 110 | select0 (#lwes_multi_emitter {type = _, max = N, 111 | num = N, configs = Configs}) -> 112 | select0 (Configs); 113 | select0 (#lwes_multi_emitter {type = _, max = M, 114 | num = N, configs = Configs}) -> 115 | Start = rand:uniform(M), 116 | Config = list_to_tuple (Configs), 117 | Indices = wrapped_range (Start, N, M), 118 | case Indices of 119 | [I] -> select0(element(I, Config)); 120 | _ -> [ select0(element(I, Config)) || I <- Indices ] 121 | end; 122 | select0 (A = {_,P}) when is_integer(P) -> 123 | A; 124 | select0 (L) when is_list(L) -> 125 | [ select0(E) || E <- L ]; 126 | select0 (A) -> 127 | A. 128 | 129 | wrapped_range (Start, Number, Max) when Start > Max -> 130 | wrapped_range (case Start rem Max of 0 -> Max; V -> V end, Number, Max); 131 | wrapped_range (Start, Number, Max) -> 132 | wrapped_range (Start, Number, Max, []). 133 | 134 | % determine a range of integers which wrap 135 | wrapped_range (_, 0, _, Accumulated) -> 136 | lists:reverse (Accumulated); 137 | wrapped_range (Max, Number, Max, Accumulated) -> 138 | wrapped_range (1, Number - 1, Max, [Max | Accumulated]); 139 | wrapped_range (Current, Number, Max, Accumulated) -> 140 | wrapped_range (Current + 1, Number - 1, Max, [Current | Accumulated]). 141 | 142 | collate (L) -> 143 | collate0(L,[]). 144 | 145 | collate0 ([], Accum) -> 146 | Accum; 147 | collate0 ([{M,State}|RestIn],[]) -> 148 | collate0 (RestIn, [{M,[State]}]); 149 | collate0 ([{M,State} | RestIn], [{M,StateList} | RestOut]) -> 150 | collate0 (RestIn, [{M,[State | StateList]} | RestOut]); 151 | collate0 ([{M,State} | RestIn], AccumIn = [{N,_} | _]) when M =/= N -> 152 | collate0 (RestIn, [{M,[State]} | AccumIn]). 153 | 154 | emit (AllEmitters, Event) -> 155 | SelectedEmitters = select(AllEmitters), 156 | % select/1 will return a list of 2-tuples of the module to use and the 157 | % emitter configs for that module 158 | lists:foreach ( 159 | fun ({Module, Emitters}) -> 160 | % one callback to prep the event for the Emitters 161 | Packet = Module:prep (Event), 162 | 163 | % then loop over emitters 164 | lists:foreach (fun (Emitter) -> 165 | Id = Module:id (Emitter), 166 | case Module:emit (Emitter, Packet) of 167 | ok -> lwes_stats:increment_sent (Id); 168 | {error, _} -> lwes_stats:increment_errors (Id) 169 | end 170 | end, 171 | Emitters) 172 | end, 173 | SelectedEmitters), 174 | ok. 175 | 176 | close (#lwes_multi_emitter { configs = Configs }) -> 177 | [ close(C) || C <- Configs ]; 178 | close ({Module,Config}) -> 179 | lwes_stats:delete (Module:id(Config)), 180 | Module:close(Config). 181 | 182 | %%==================================================================== 183 | %% Internal functions 184 | %%==================================================================== 185 | 186 | %%==================================================================== 187 | %% Test functions 188 | %%==================================================================== 189 | -ifdef (TEST). 190 | -include_lib ("eunit/include/eunit.hrl"). 191 | 192 | config (basic) -> 193 | { "127.0.0.1", 9191 }; 194 | config (random) -> 195 | { 2, random, 196 | [ {"127.0.0.1",30000}, {"127.0.0.1",30001}, {"127.0.0.1",30002} ] }; 197 | config (group) -> 198 | { 3, group, 199 | [ {1, random, [ { "127.0.0.1",5390 }, { "127.0.0.1",5391 } ] }, 200 | {1, random, [ { "127.0.0.1",5392 }, { "127.0.0.1",5393 } ] }, 201 | {1, random, [ { "127.0.0.1",5394 }, { "127.0.0.1",5395 } ] } 202 | ] 203 | }. 204 | 205 | possible_answers (basic) -> 206 | [ 207 | [{{127,0,0,1},9191}] 208 | ]; 209 | possible_answers (random) -> 210 | [ 211 | [{{127,0,0,1},30000}, {{127,0,0,1},30001}], 212 | [{{127,0,0,1},30001}, {{127,0,0,1},30002}], 213 | [{{127,0,0,1},30000}, {{127,0,0,1},30002}] 214 | ]; 215 | possible_answers (group) -> 216 | [ 217 | [{{127,0,0,1},5390}, {{127,0,0,1},5392}, {{127,0,0,1},5394}], 218 | [{{127,0,0,1},5390}, {{127,0,0,1},5392}, {{127,0,0,1},5395}], 219 | [{{127,0,0,1},5390}, {{127,0,0,1},5393}, {{127,0,0,1},5394}], 220 | [{{127,0,0,1},5390}, {{127,0,0,1},5393}, {{127,0,0,1},5395}], 221 | [{{127,0,0,1},5391}, {{127,0,0,1},5392}, {{127,0,0,1},5394}], 222 | [{{127,0,0,1},5391}, {{127,0,0,1},5392}, {{127,0,0,1},5395}], 223 | [{{127,0,0,1},5391}, {{127,0,0,1},5393}, {{127,0,0,1},5394}], 224 | [{{127,0,0,1},5391}, {{127,0,0,1},5393}, {{127,0,0,1},5395}] 225 | ]. 226 | 227 | test_one(T) -> 228 | Config = config(T), 229 | {ok, C} = new(Config), 230 | Results = 231 | lists:foldl ( 232 | fun (_, A) -> 233 | % jumping through a few hoops here as I changed the way select 234 | % worked to not return the results of getting the identifier as 235 | % it worked before. So in this case we unbox the selection 236 | [{lwes_emitter_udp, Selected}] = select(C), 237 | % then sort the list of Ip/Address pairs 238 | Ids = lists:sort([ lwes_emitter_udp:id (S) || S <- Selected ]), 239 | Answers = possible_answers(T), 240 | % finally we check to see if we have an answer 241 | lists:member (Ids, Answers) and A 242 | end, 243 | true, 244 | lists:seq(1,100)), 245 | close(C), 246 | Results. 247 | 248 | check_selection_test_ () -> 249 | { setup, 250 | fun() -> 251 | case lwes_stats:start_link() of 252 | {error,{already_started,_}} -> exists; 253 | {ok, Pid} -> Pid 254 | end 255 | end, 256 | fun (exists) -> ok; 257 | (Pid) -> unlink(Pid), gen_server:stop(Pid) 258 | end, 259 | [ 260 | ?_assert(test_one(T)) 261 | || T <- [basic, random, group] 262 | ] 263 | }. 264 | 265 | normalize_test_ () -> 266 | [ 267 | ?_assertEqual (Expected, normalize_emitters_config(Given)) 268 | || {Given, Expected} 269 | <- [ 270 | { {"127.0.0.1",9191}, 271 | {1,group,[{"127.0.0.1",9191}]} }, 272 | { {"127.0.0.1",9191,[{ttl,25}]}, 273 | {1,group,[{"127.0.0.1",9191,[{ttl,25}]}]} }, 274 | { {2, [{"127.0.0.1",9191},{"127.0.0.1",9292}]}, 275 | {2,group,[{"127.0.0.1",9191},{"127.0.0.1",9292}]} }, 276 | { {1, [{"127.0.0.1",9191},{"127.0.0.1",9292}]}, 277 | {1,group,[{"127.0.0.1",9191},{"127.0.0.1",9292}]} }, 278 | { {2, random, [{"127.0.0.1",30000},{"127.0.0.1",30001},{"127.0.0.1",30002}]}, 279 | {2, random, [{"127.0.0.1",30000}, {"127.0.0.1",30001}, {"127.0.0.1",30002}]} }, 280 | { {3, group, [{1,random,[{"127.0.0.1",5301},{"127.0.0.1",5302}]},{1,random,[{"127.0.0.1",5311},{"127.0.0.1",5312}]},{1,random,[{"127.0.0.1",5321},{"127.0.0.1",5322}]}]}, 281 | {3, group, [{1,random,[{"127.0.0.1",5301},{"127.0.0.1",5302}]},{1,random,[{"127.0.0.1",5311},{"127.0.0.1",5312}]},{1,random,[{"127.0.0.1",5321},{"127.0.0.1",5322}]}]} }, 282 | { {2, group, [{2, random, [{"127.0.0.1",30000},{"127.0.0.1",30001},{"127.0.0.1",30002}]},{lwes_emitter_stdout,[{label,stdout}]}]}, 283 | {2, group, [{2, random, [{"127.0.0.1",30000},{"127.0.0.1",30001},{"127.0.0.1",30002}]},{lwes_emitter_stdout,[{label,stdout}]}]} 284 | } 285 | % TODO: support labels and sub-labels 286 | % {"127.0.0.1",9191} -> 287 | % {"127.0.0.1",9191,[{label, abc}]} -> 288 | % {1, group, [{"127.0.0.1",9191}], abc} 289 | % 290 | % {2,group,[{"127.0.0.1",9191},{"127.0.0.1",9292}]} -> 291 | % {2, group, [{"127.0.0.1",9191},{"127.0.0.1",9292}], abc} 292 | % 293 | % {2, group, [{2, random, [{"127.0.0.1",30000}, 294 | % {"127.0.0.1",30001}, 295 | % {"127.0.0.1",30002}]}, 296 | % {lwes_emitter_stdout,[{label,stdout}]} 297 | % ] 298 | % } -> 299 | % {2, group, [{2, random, [{"127.0.0.1",30000}, 300 | % {"127.0.0.1",30001}, 301 | % {"127.0.0.1",30002}], [{label,udp}]}, 302 | % {lwes_emitter_stdout,[{label,stdout}]} 303 | % ], [{label, grid}] 304 | % } 305 | 306 | ] 307 | ]. 308 | 309 | -endif. 310 | 311 | -------------------------------------------------------------------------------- /src/lwes_net_udp.erl: -------------------------------------------------------------------------------- 1 | -module (lwes_net_udp). 2 | 3 | -include_lib ("lwes_internal.hrl"). 4 | 5 | % This modules attempts to separate out the low level network details from the 6 | % rest of the system. It is by no means necessary to use this as a transport 7 | % but the base system assumes UDP or UDP/Multicast, so those forms are 8 | % attempted to be encapsulated here. A certain set of options is set by 9 | % default. 10 | -export([new/2, 11 | open/2, 12 | send/3, 13 | close/1, 14 | address/1 15 | ]). 16 | 17 | -define (DEFAULT_TTL, 25). 18 | -define (DEFAULT_RECBUF, 16777216). 19 | 20 | -record(lwes_net_udp, { ip :: inet:ip_address(), 21 | port :: inet:port_number(), 22 | is_multicast = false :: boolean(), 23 | ttl = ?DEFAULT_TTL :: non_neg_integer(), 24 | recbuf = ?DEFAULT_RECBUF :: non_neg_integer(), 25 | options = [] 26 | }). 27 | new (listener, Config) -> 28 | InitialStruct = check_config (Config), 29 | case InitialStruct of 30 | #lwes_net_udp { ip = Ip, is_multicast = true, ttl = TTL, 31 | recbuf = Recbuf, options = ExtraOptions} -> 32 | InitialStruct#lwes_net_udp { 33 | options = merge_options ([ { ip, Ip }, 34 | { multicast_ttl, TTL }, 35 | { multicast_loop, false }, 36 | { add_membership, {Ip, {0,0,0,0}}}, 37 | { reuseaddr, true }, 38 | { recbuf, Recbuf }, 39 | { mode, binary } 40 | ], 41 | ExtraOptions) 42 | }; 43 | #lwes_net_udp { is_multicast = false, 44 | recbuf = Recbuf, options = ExtraOptions} -> 45 | InitialStruct#lwes_net_udp { 46 | options = merge_options ([ { reuseaddr, true }, 47 | { recbuf, Recbuf }, 48 | { mode, binary } 49 | ], 50 | ExtraOptions) 51 | } 52 | end; 53 | new (emitter, Config) -> 54 | InitialStruct = check_config (Config), 55 | InitialOptions = InitialStruct#lwes_net_udp.options, 56 | TTL = InitialStruct#lwes_net_udp.ttl, 57 | 58 | % emitters don't care about most options, but including ttl 59 | % means we can use the same emitter for multicast as well as 60 | % unicast traffic 61 | InitialStruct#lwes_net_udp { 62 | options = merge_options ([ { multicast_ttl, TTL }, 63 | { mode, binary } 64 | ], 65 | InitialOptions) 66 | }. 67 | 68 | open (listener, #lwes_net_udp { port = Port, options = Options}) -> 69 | gen_udp:open (Port, Options); 70 | open (emitter, #lwes_net_udp { options = Options}) -> 71 | gen_udp:open (0, Options). 72 | 73 | % allow some of the other forms of config, so the config structure can 74 | % just be reused when sending in most cases 75 | send (Socket, {Ip, Port, _}, Packet) -> 76 | gen_udp:send (Socket, Ip, Port, Packet); 77 | send (Socket, {Ip, Port}, Packet) -> 78 | gen_udp:send (Socket, Ip, Port, Packet); 79 | send (Socket, #lwes_net_udp { ip = Ip, port = Port}, Packet) -> 80 | gen_udp:send (Socket, Ip, Port, Packet). 81 | 82 | close (Socket) -> 83 | gen_udp:close (Socket). 84 | 85 | address (#lwes_net_udp { ip = Ip, port = Port }) -> 86 | {Ip, Port}. 87 | 88 | %%==================================================================== 89 | %% Internal functions 90 | %%==================================================================== 91 | 92 | % this will merge in any passed in Overrides options on top of any defaults 93 | % listed. it's currently assumed there are no raw defaults 94 | merge_options (Defaults, Overrides) -> 95 | % first split into options and raw options 96 | {OverrideOpts, OverrideRaw} = partition (Overrides), 97 | % then merge, preferring overrides and add back the raw at the end 98 | lists:ukeymerge(1, lists:sort(OverrideOpts), lists:sort(Defaults)) 99 | ++ OverrideRaw. 100 | 101 | % gen_udp has a long list of mostly consistent options, the few outliers are 102 | % binary | list - these can also be specified as {mode, binary | list} 103 | % {raw,_,_,_} - these do not have a 2-tuple form 104 | % this function will normalize on {mode, binary | list} and separate out 105 | % {raw,_,_,_} options, resulting in two lists 106 | partition (Options) -> 107 | lists:foldl( fun (list, {A,O}) -> {[{mode,list} | A],O}; 108 | (binary, {A,O}) -> {[{mode,binary} | A],O}; 109 | (R = {raw,_,_,_},{A,O}) -> {A,[R|O]}; 110 | (Other, {A,O}) -> {[Other|A],O} 111 | end, 112 | {[],[]}, 113 | Options). 114 | 115 | parse_options ([], AccumlatedOptions) -> 116 | AccumlatedOptions; 117 | % some options are simply passed through 118 | parse_options ([{recbuf, Recbuf} | Rest], N) -> 119 | parse_options (Rest, N#lwes_net_udp{ recbuf = Recbuf }); 120 | % others have slightly different naming, so are changed 121 | parse_options ([{ttl, TTL} | Rest], N ) -> 122 | parse_options (Rest, N#lwes_net_udp { ttl = TTL }); 123 | % check_config options which get converted to raw options 124 | parse_options ([reuseport | Rest], 125 | N = #lwes_net_udp { options = OptionsIn}) -> 126 | parse_options (Rest, N#lwes_net_udp { options = reuseport() ++ OptionsIn}); 127 | % finally options which we assume are correct for now 128 | parse_options ([O | Rest], 129 | N = #lwes_net_udp { options = OptionsIn}) -> 130 | parse_options (Rest, N#lwes_net_udp { options = [ O | OptionsIn]}). 131 | 132 | is_multicast ({N1, _, _, _}) when N1 >= 224, N1 =< 239 -> 133 | true; 134 | is_multicast (_) -> 135 | false. 136 | 137 | reuseport() -> 138 | case os:type() of 139 | {unix, linux} -> 140 | [ {raw, 1, 15, <<1:32/native>>} ]; 141 | {unix, OS} when OS =:= darwin; 142 | OS =:= freebsd; 143 | OS =:= openbsd; 144 | OS =:= netbsd -> 145 | [ {raw, 16#ffff, 16#0200, <<1:32/native>>} ]; 146 | _ -> [] 147 | end. 148 | 149 | check_ip (Ip) when ?is_ip_addr (Ip) -> 150 | Ip; 151 | check_ip (IpList) when is_list (IpList) -> 152 | case inet_parse:address (IpList) of 153 | {ok, Ip} -> Ip; 154 | _ -> 155 | erlang:error(badarg) 156 | end; 157 | check_ip (_) -> 158 | % essentially turns function_clause error into badarg 159 | erlang:error (badarg). 160 | 161 | check_port(Port) when ?is_uint16 (Port) -> 162 | Port; 163 | check_port(_) -> 164 | % essentially turns function_clause error into badarg 165 | erlang:error (badarg). 166 | 167 | check_config ({Ip, Port}) -> 168 | CheckedIp = check_ip (Ip), 169 | #lwes_net_udp { ip = CheckedIp, 170 | port = check_port(Port), 171 | is_multicast = is_multicast(CheckedIp) }; 172 | check_config ({Ip, Port, Options}) when is_list (Options) -> 173 | parse_options (Options, check_config ({Ip,Port})); 174 | check_config (_) -> 175 | % essentially turns function_clause error into badarg 176 | erlang:error (badarg). 177 | 178 | 179 | %%-------------------------------------------------------------------- 180 | %%% Test functions 181 | %%-------------------------------------------------------------------- 182 | -ifdef (TEST). 183 | -include_lib ("eunit/include/eunit.hrl"). 184 | 185 | test_one (Config) -> 186 | EmitterConfig = new (emitter, Config), 187 | ListenerConfig = new (listener, Config), 188 | {ok, Emitter} = lwes_net_udp:open(emitter, EmitterConfig), 189 | {ok, Listener}= lwes_net_udp:open(listener, ListenerConfig), 190 | Input = <<"hello">>, 191 | lwes_net_udp:send(Emitter, EmitterConfig, Input), 192 | Output = receive {udp,_,_,_,P} -> P end, 193 | lwes_net_udp:close(Listener), 194 | lwes_net_udp:close(Emitter), 195 | ?assertEqual (Input, Output). 196 | 197 | % test emission and receipt with various options 198 | lwes_net_udp_test_ () -> 199 | { inorder, 200 | [ 201 | fun () -> test_one (C) end 202 | || C 203 | <- [ 204 | {"127.0.0.1",12321}, 205 | {"127.0.0.1",12321,[{ttl,3}]}, 206 | {"127.0.0.1",12321,[{ttl,12},{recbuf, 65535}]}, 207 | {"224.1.1.111",12321,[{multicast_loop, true}]} 208 | ] 209 | ] 210 | }. 211 | 212 | % test address functionality 213 | lwes_net_udp_address_test_ () -> 214 | [ 215 | ?_assertEqual (Expected, address(Given)) 216 | || {Expected, Given} 217 | <- [ 218 | {{{127,0,0,1},9191}, new(emitter, {"127.0.0.1",9191})} 219 | ] 220 | ]. 221 | 222 | % test various error cases 223 | lwes_net_udp_error_test_ () -> 224 | [ 225 | ?_assertError (badarg, new(emitter,{foo,bar})), 226 | ?_assertError (badarg, new(emitter,{"256.0.0.0",99})), 227 | ?_assertError (badarg, new(emitter,{"127.0.0.1",99999})) 228 | ]. 229 | 230 | % test options merging 231 | lwes_net_udp_options_test_ () -> 232 | [ 233 | ?_assertEqual(Expected, merge_options (Defaults, Overrides)) 234 | || {Expected, Defaults, Overrides} 235 | <- [ 236 | { [{mode,list}], [{mode,binary}], [list] }, 237 | { [{mode,binary}], [{mode,binary}], [binary] }, 238 | { [{mode,list},{raw,a,b,c}], [{mode,binary}], [list,{raw,a,b,c}] }, 239 | { [{add_membership,{{127,0,0,1},{0,0,0,0}}}, 240 | {ip,{127,0,0,1}}, 241 | {mode,binary}, 242 | {multicast_loop,true}, 243 | {multicast_ttl,12}, 244 | {recbuf,65535}, 245 | {reuseaddr,true}], 246 | [{ip, {127,0,0,1}}, 247 | {multicast_ttl, 12}, 248 | {multicast_loop, false}, 249 | {add_membership, {{127,0,0,1}, {0,0,0,0}}}, 250 | {reuseaddr, true}, 251 | {recbuf, 65535}, 252 | {mode, binary} 253 | ], 254 | [{multicast_loop, true}] 255 | } 256 | 257 | ] 258 | ]. 259 | 260 | check_config_test_ () -> 261 | [ 262 | ?_assertEqual (#lwes_net_udp {ip = {127,0,0,1}, port = 9191, 263 | is_multicast = false, 264 | ttl = 3, recbuf = 65535}, 265 | check_config ({{127,0,0,1},9191,[{ttl,3},{recbuf,65535}]})), 266 | ?_assertEqual (#lwes_net_udp {ip = {127,0,0,1}, port = 9191, 267 | is_multicast = false, 268 | ttl = 3, recbuf = 65535}, 269 | check_config ({"127.0.0.1",9191,[{ttl,3},{recbuf,65535}]})), 270 | ?_assertEqual (#lwes_net_udp {ip = {127,0,0,1}, port = 9191, 271 | is_multicast = false, 272 | ttl = 3, recbuf = 65535, 273 | options = reuseport()}, 274 | check_config ({"127.0.0.1",9191, 275 | [{ttl,3}, {recbuf,65535}, reuseport]})), 276 | ?_assertEqual (#lwes_net_udp {ip = {127,0,0,1}, port = 9191, 277 | is_multicast = false, 278 | ttl = 3, recbuf = ?DEFAULT_RECBUF}, 279 | check_config ({"127.0.0.1",9191,[{ttl,3}]})), 280 | ?_assertEqual (#lwes_net_udp {ip = {127,0,0,1}, port = 9191, 281 | is_multicast = false, 282 | ttl = 3, recbuf = ?DEFAULT_RECBUF}, 283 | check_config ({{127,0,0,1},9191,[{ttl,3}]})), 284 | ?_assertEqual (#lwes_net_udp {ip = {127,0,0,1}, port = 9191, 285 | is_multicast = false, 286 | ttl = ?DEFAULT_TTL, recbuf = ?DEFAULT_RECBUF}, 287 | check_config ({"127.0.0.1",9191})), 288 | ?_assertEqual (#lwes_net_udp {ip = {127,0,0,1}, port = 9191, 289 | is_multicast = false, 290 | ttl = ?DEFAULT_TTL, recbuf = ?DEFAULT_RECBUF}, 291 | check_config ({{127,0,0,1},9191})), 292 | ?_assertError (badarg, check_config ({"655.0.0.1",9191,3})), 293 | ?_assertError (badarg, check_config ({"655.0.0.1",9191,65})), 294 | ?_assertError (badarg, check_config ({"655.0.0.1",9191})), 295 | ?_assertError (badarg, check_config ({{655,0,0,1},9191})), 296 | ?_assertError (badarg, check_config ({{127,0,0,1},91919})) 297 | ]. 298 | 299 | -endif. 300 | -------------------------------------------------------------------------------- /src/lwes_stats.erl: -------------------------------------------------------------------------------- 1 | -module (lwes_stats). 2 | 3 | % the stats module keeps some stats about what lwes is doing, and provides 4 | % a few ways to roll those stats up. 5 | % 6 | % the key for most methods is one of two forms 7 | % 8 | % - {Label, {Ip, Port}} 9 | % - {Ip, Port} 10 | % 11 | % individual stats are incremented via function calls and the current 12 | % stats can be fetched as a proplist via fetch/1, or as a list of lists 13 | % via rollup/1 14 | 15 | %% API 16 | -export([ start_link/0, 17 | initialize/1, % (Id) 18 | increment_sent/1, % (Id) 19 | increment_received/1, % (Id) 20 | increment_errors/1, % (Id) 21 | % these 2 are only used if validating 22 | increment_considered/1, % (Id) 23 | increment_validated/1, % (Id) 24 | delete/1, % (Id) 25 | fetch/1, % (Id) 26 | rollup/1, % 'id' | 'port' | 'none' 27 | print/1 28 | ]). 29 | 30 | %% gen_server callbacks 31 | -export ([init/1, 32 | handle_call/3, 33 | handle_cast/2, 34 | handle_info/2, 35 | terminate/2, 36 | code_change/3]). 37 | 38 | % gen_server state 39 | -record (state, {}). 40 | -define (TABLE, lwes_stats). 41 | 42 | % stats record for db 43 | -record (stats, {id, % Id 44 | sent = 0, 45 | received = 0, 46 | errors = 0, 47 | considered = 0, 48 | validated = 0 49 | }). 50 | 51 | -define (STATS_SENT_INDEX, #stats.sent). 52 | -define (STATS_RECEIVED_INDEX, #stats.received). 53 | -define (STATS_ERRORS_INDEX, #stats.errors). 54 | -define (STATS_CONSIDERED_INDEX, #stats.considered). 55 | -define (STATS_VALIDATED_INDEX, #stats.validated). 56 | 57 | start_link () -> 58 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). 59 | 60 | initialize (Id = {_, {_,_}}) -> 61 | init0 (Id); 62 | initialize (Id = {_, _}) -> 63 | init0 (Id); 64 | initialize (Id) when is_atom(Id) -> 65 | init0 (Id); 66 | initialize (_) -> 67 | erlang:error(badarg). 68 | 69 | init0 (Id) -> 70 | ets:insert_new (?TABLE, #stats {id = Id}). 71 | 72 | increment_sent (Id) -> 73 | [V] = try_update_counter (Id, ?STATS_SENT_INDEX, 1), 74 | V. 75 | 76 | increment_received (Id) -> 77 | [V] = try_update_counter (Id, ?STATS_RECEIVED_INDEX, 1), 78 | V. 79 | 80 | increment_errors (Id) -> 81 | [V] = try_update_counter (Id, ?STATS_ERRORS_INDEX, 1), 82 | V. 83 | 84 | increment_considered (Id) -> 85 | [V] = try_update_counter (Id, ?STATS_CONSIDERED_INDEX, 1), 86 | V. 87 | 88 | increment_validated (Id) -> 89 | [V] = try_update_counter (Id, ?STATS_VALIDATED_INDEX, 1), 90 | V. 91 | 92 | update_counter (Id, Index, Amount) -> 93 | ets:update_counter (?TABLE, Id, [{Index,Amount}]). 94 | 95 | try_update_counter (Id, Index, Amount) -> 96 | try update_counter (Id, Index, Amount) of 97 | V -> V 98 | catch 99 | error:badarg -> 100 | initialize (Id), 101 | update_counter (Id, Index, Amount) 102 | end. 103 | 104 | 105 | delete (Id) -> 106 | ets:delete (?TABLE, Id). 107 | 108 | fetch (Id) -> 109 | case ets:lookup (?TABLE, Id) of 110 | [] -> undefined; 111 | [#stats { sent = Sent, 112 | received = Received, 113 | errors = Errors, 114 | considered = Considered, 115 | validated = Validated }] -> 116 | [ {sent, Sent}, 117 | {received, Received}, 118 | {errors, Errors}, 119 | {considered, Considered}, 120 | {validated, Validated} 121 | ] 122 | end. 123 | 124 | pivot_by_key ({Data, Keys}) -> 125 | lists:foldl( fun (Key, A) -> 126 | [ [ Key, dict:fetch({Key,sent}, Data), 127 | dict:fetch({Key,received}, Data), 128 | dict:fetch({Key,errors}, Data), 129 | dict:fetch({Key,considered}, Data), 130 | dict:fetch({Key,validated}, Data) ] | A ] 131 | end, 132 | [], 133 | dict:fetch_keys(Keys)). 134 | 135 | add_by_key (Key, S, R, E, C, V, {DataIn, KeysIn}) -> 136 | D1 = dict:update_counter({Key,sent},S,DataIn), 137 | D2 = dict:update_counter({Key,received},R,D1), 138 | D3 = dict:update_counter({Key,errors},E,D2), 139 | D4 = dict:update_counter({Key,considered},C,D3), 140 | D5 = dict:update_counter({Key,validated},V,D4), 141 | KeysOut = dict:update_counter(Key,1,KeysIn), 142 | {D5, KeysOut}. 143 | 144 | % outputs the table in a two dimensional form which should allow for easy 145 | % construction of output. 146 | % The form is 147 | % 148 | % [ [ id, sent, received, errors, considered, validated], 149 | % ... 150 | % ] 151 | rollup(label) -> 152 | pivot_by_key( 153 | lists:foldl ( 154 | fun (#stats {id = {Label,{_,_}}, 155 | sent = S, received = R, errors = E, 156 | considered = C, validated = V}, Accum) -> 157 | add_by_key ({Label,{'*','*'}}, S, R, E, C, V, Accum); 158 | (#stats {id = M, 159 | sent = S, received = R, errors = E, 160 | considered = C, validated = V}, Accum) when is_atom(M) -> 161 | add_by_key (M, S, R, E, C, V, Accum); 162 | (#stats { sent = S, received = R, errors = E, 163 | considered = C, validated = V}, Accum) -> 164 | add_by_key ({'_',{'*','*'}}, S, R, E, C, V, Accum) 165 | end, 166 | { dict:new(), dict:new() }, 167 | ets:tab2list(?TABLE) 168 | ) 169 | ); 170 | rollup(port) -> 171 | pivot_by_key( 172 | lists:foldl ( 173 | fun (#stats {id = {_,{_,Port}}, 174 | sent = S, received = R, errors = E, 175 | considered = C, validated = V}, Accum) -> 176 | add_by_key ({'*',{'*',Port}}, S, R, E, C, V, Accum); 177 | (#stats {id = {_, Port}, 178 | sent = S, received = R, errors = E, 179 | considered = C, validated = V}, Accum) -> 180 | add_by_key ({'*',{'*',Port}}, S, R, E, C, V, Accum); 181 | (#stats {id = M, 182 | sent = S, received = R, errors = E, 183 | considered = C, validated = V}, Accum) when is_atom(M) -> 184 | add_by_key (M, S, R, E, C, V, Accum) 185 | end, 186 | { dict:new(), dict:new() }, 187 | ets:tab2list(?TABLE) 188 | ) 189 | ); 190 | rollup(none) -> 191 | [ 192 | [ case I of 193 | M when is_atom(M) -> M ; 194 | {_,{_,_}} -> I ; 195 | _ -> {'*',I} 196 | end, S, R, E, C, V ] 197 | || #stats { id = I, sent = S, received = R, 198 | errors = E, considered = C, validated = V} 199 | <- ets:tab2list(?TABLE) 200 | ]. 201 | 202 | format_ip_port({'*','*'}) -> io_lib:format("*:~-5s",["*"]); 203 | format_ip_port({'*', Port}) -> io_lib:format("*:~-5b",[Port]); 204 | format_ip_port({Ip, Port}) -> io_lib:format ("~s:~-5b",[lwes_util:ip2bin (Ip), Port]). 205 | 206 | print (Type) -> 207 | io:format ("~-21s ~-21s ~10s ~10s ~10s ~10s ~10s~n", 208 | ["label","channel","sent","received", 209 | "errors","considered","validated"]), 210 | io:format ("~-21s ~-21s ~10s ~10s ~10s ~10s ~10s~n", 211 | ["---------------------", 212 | "---------------------", 213 | "----------","----------", 214 | "----------","----------","----------"]), 215 | [ 216 | begin 217 | {Label, IpPort} = 218 | case Key of 219 | {L, IP= {_,_}} -> {L, IP}; 220 | M when is_atom(M) -> {M, {'*','*'}} 221 | end, 222 | io:format ("~-21w ~-21s ~10b ~10b ~10b ~10b ~10b~n", 223 | [Label, format_ip_port (IpPort), 224 | Sent, Recv, Err, Cons, Valid] 225 | ) 226 | end 227 | || [ Key, Sent, Recv, Err, Cons, Valid ] 228 | <- rollup(Type) 229 | ], 230 | ok. 231 | 232 | %%==================================================================== 233 | %% gen_server callbacks 234 | %%==================================================================== 235 | init([]) -> 236 | ets:new (?TABLE, 237 | [ named_table, 238 | public, 239 | set, 240 | {keypos, 2}, 241 | {write_concurrency, true} 242 | ]), 243 | {ok, #state { }}. 244 | 245 | handle_call (_Request, _From, State) -> 246 | {reply, ok, State}. 247 | 248 | handle_cast (_Request, State) -> 249 | {noreply, State}. 250 | 251 | handle_info (_Info, State) -> 252 | {noreply, State}. 253 | 254 | terminate (_Reason, _State) -> 255 | ets:delete (?TABLE), 256 | ok. 257 | 258 | code_change (_OldVsn, State, _Extra) -> 259 | {ok, State}. 260 | 261 | %%-------------------------------------------------------------------- 262 | %%% Test functions 263 | %%-------------------------------------------------------------------- 264 | -ifdef (TEST). 265 | -include_lib ("eunit/include/eunit.hrl"). 266 | 267 | setup () -> 268 | case start_link() of 269 | {ok, Pid} -> Pid; 270 | {error, {already_started, _}} -> already_started 271 | end. 272 | 273 | cleanup (already_started) -> ok; 274 | cleanup (Pid) -> gen_server:stop (Pid). 275 | 276 | fetch_list (Sent, Received, Errors, Considered, Validated) -> 277 | [ {sent, Sent}, 278 | {received, Received}, 279 | {errors, Errors}, 280 | {considered, Considered}, 281 | {validated, Validated} 282 | ]. 283 | 284 | check_entry ([[_,Sent,Recv,Errors,Considered,Validated]], 285 | [[_,Sent,Recv,Errors,Considered,Validated]]) -> 286 | true. 287 | 288 | tests_with_id (Id) -> 289 | [ 290 | ?_assertEqual(true, initialize(Id)), 291 | % initialization only works once 292 | ?_assertEqual(false, initialize(Id)), 293 | % rollups should show all zeros 294 | ?_assert(check_entry([[dummy,0,0,0,0,0]], rollup(port))), 295 | ?_assert(check_entry([[dummy,0,0,0,0,0]], rollup(label))), 296 | ?_assert(check_entry([[dummy,0,0,0,0,0]], rollup(none))), 297 | % increment all stats and check counts 298 | ?_assertEqual(fetch_list(0,0,0,0,0), fetch(Id)), 299 | ?_assertEqual(1, increment_sent(Id)), 300 | ?_assertEqual(fetch_list(1,0,0,0,0), fetch(Id)), 301 | ?_assertEqual(1, increment_received(Id)), 302 | ?_assertEqual(fetch_list(1,1,0,0,0), fetch(Id)), 303 | ?_assertEqual(2, increment_received(Id)), 304 | ?_assertEqual(fetch_list(1,2,0,0,0), fetch(Id)), 305 | ?_assertEqual(1, increment_errors(Id)), 306 | ?_assertEqual(fetch_list(1,2,1,0,0), fetch(Id)), 307 | ?_assertEqual(1, increment_considered(Id)), 308 | ?_assertEqual(fetch_list(1,2,1,1,0), fetch(Id)), 309 | ?_assertEqual(1, increment_validated(Id)), 310 | ?_assertEqual(fetch_list(1,2,1,1,1), fetch(Id)), 311 | % see that the rollups reflect the same values 312 | ?_assert(check_entry([[dummy,1,2,1,1,1]], rollup(port))), 313 | ?_assert(check_entry([[dummy,1,2,1,1,1]], rollup(label))), 314 | ?_assert(check_entry([[dummy,1,2,1,1,1]], rollup(none))), 315 | % delete the test ids 316 | ?_assertEqual(true, delete(Id)), 317 | ?_assertEqual(true, delete(Id)) 318 | ]. 319 | 320 | 321 | lwes_stats_test_ () -> 322 | IdWithLabel = {foo,{{127,0,0,1},9191}}, 323 | IdNoLabel = {{127,0,0,1},9191}, 324 | BadId = undefined, 325 | NonExistentId = {foo, bar,{{127,0,0,1},9292}}, 326 | { setup, 327 | fun setup/0, 328 | fun cleanup/1, 329 | [ 330 | % tests with different types of ids 331 | tests_with_id (IdWithLabel), 332 | tests_with_id (IdNoLabel), 333 | % test a few cases with bad ids 334 | ?_assertEqual(undefined, fetch(BadId)), 335 | ?_assertEqual(undefined, fetch(NonExistentId)), 336 | ?_assertException(error, badarg, increment_sent(NonExistentId)), 337 | % these are for additional coverage on the gen_server calls which 338 | % are not currently used for anything 339 | ?_assertEqual({reply, ok, #state{}}, handle_call(foo, bar, #state{})), 340 | ?_assertEqual({noreply, #state{}}, handle_cast(foo, #state{})), 341 | ?_assertEqual({noreply, #state{}}, handle_info(foo, #state{})), 342 | ?_assertEqual({ok, #state{}}, code_change(foo, #state{}, bar)), 343 | % this just tests the case where we might be running inside of a 344 | % larger process, so mostly just here for coverage 345 | ?_assertEqual(ok, cleanup(setup())) 346 | ] 347 | }. 348 | 349 | lwes_stats_rollups_test_ () -> 350 | Id1 = {{127,0,0,1},9191}, 351 | Id2 = {{127,0,0,1},9192}, 352 | Id3 = {foo, Id1}, 353 | Id4 = {foo, Id2}, 354 | Id5 = bar, 355 | LabelRollupId1 = {foo,{'*','*'}}, 356 | LabelRollupId2 = {'_',{'*','*'}}, 357 | PortRollupId1 = {'*',{'*',9191}}, 358 | PortRollupId2 = {'*',{'*',9192}}, 359 | { setup, 360 | fun setup/0, 361 | fun cleanup/1, 362 | { inorder, 363 | [ 364 | [ ?_assertEqual(true, initialize(I)) 365 | || I <- [ Id1, Id2, Id3, Id4, Id5 ] ], 366 | ?_assertEqual(1, increment_sent(Id1)), 367 | ?_assertEqual(1, increment_sent(Id4)), 368 | ?_assertEqual(1, increment_received(Id2)), 369 | ?_assertEqual(1, increment_received(Id3)), 370 | ?_assertEqual(1, increment_errors(Id3)), 371 | ?_assertEqual(1, increment_errors(Id4)), 372 | ?_assertEqual(1, increment_sent(Id5)), 373 | ?_assertEqual(1, increment_errors(Id5)), 374 | % check no rollups 375 | ?_assertEqual(lists:sort([[{'*',Id1},1,0,0,0,0], 376 | [{'*',Id2},0,1,0,0,0], 377 | [Id3,0,1,1,0,0], 378 | [Id4,1,0,1,0,0], 379 | [Id5,1,0,1,0,0]]), 380 | lists:sort(rollup(none))), 381 | % check rollup by label 382 | ?_assertEqual(lists:sort([[LabelRollupId1,1,1,2,0,0], 383 | [LabelRollupId2,1,1,0,0,0], 384 | [Id5,1,0,1,0,0]]), 385 | lists:sort(rollup(label))), 386 | % check rollup by port 387 | ?_assertEqual(lists:sort([[PortRollupId1,1,1,1,0,0], 388 | [PortRollupId2,1,1,1,0,0], 389 | [Id5,1,0,1,0,0]]), 390 | lists:sort(rollup(port))) 391 | ] 392 | } 393 | }. 394 | 395 | -endif. 396 | -------------------------------------------------------------------------------- /src/lwes_sup.erl: -------------------------------------------------------------------------------- 1 | -module (lwes_sup). 2 | 3 | -behaviour (supervisor). 4 | 5 | %% API 6 | -export([start_link/0]). 7 | 8 | %% supervisor callbacks 9 | -export([init/1]). 10 | 11 | %-=====================================================================- 12 | %- API - 13 | %-=====================================================================- 14 | start_link() -> 15 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 16 | 17 | %-=====================================================================- 18 | %- supervisor callbacks - 19 | %-=====================================================================- 20 | init([]) -> 21 | { ok, 22 | { 23 | { one_for_one, 10, 10 }, 24 | [ 25 | { lwes_stats, 26 | { lwes_stats, start_link, [] }, 27 | permanent, 28 | 2000, 29 | worker, 30 | [ lwes_stats ] 31 | }, 32 | { lwes_emitter_udp_pool, 33 | { lwes_emitter_udp_pool, start_link, [[{id, lwes_emitters}]] }, 34 | permanent, 35 | 2000, 36 | worker, 37 | [ lwes_emitter_udp_pool ] 38 | }, 39 | { lwes_channel_manager, % child spec id 40 | { lwes_channel_manager, start_link, [] },% child function to call 41 | permanent, % always restart 42 | 2000, % time to wait for child stop 43 | worker, % type of child 44 | [ lwes_channel_manager ] % modules used by child 45 | }, 46 | { 47 | lwes_channel_sup, % child spec id 48 | { lwes_channel_sup, start_link, []}, % child function to call 49 | permanent, % always restart 50 | 2000, % time to wait for child stop 51 | supervisor, % type of child 52 | [ lwes_channel_sup ] % modules used by child 53 | }, 54 | { lwes_esf_validator, % child spec id 55 | { lwes_esf_validator, start_link, [] }, % child function to call 56 | permanent, % always restart 57 | 2000, % time to wait for child stop 58 | worker, % type of child 59 | [ lwes_esf_validator ] % modules used by child 60 | } 61 | ] 62 | } 63 | }. 64 | 65 | %-=====================================================================- 66 | %- Private - 67 | %-=====================================================================- 68 | 69 | 70 | %-=====================================================================- 71 | %- Test Functions - 72 | %-=====================================================================- 73 | -ifdef (TEST). 74 | -include_lib ("eunit/include/eunit.hrl"). 75 | 76 | lwes_sup_test_ () -> 77 | { setup, 78 | fun () -> 79 | {ok, Pid} = start_link(), 80 | Pid 81 | end, 82 | fun (Pid) -> 83 | exit (Pid, normal) 84 | end, 85 | { 86 | inorder, 87 | [ 88 | ?_assertEqual (true, true) 89 | ] 90 | } 91 | }. 92 | 93 | -endif. 94 | -------------------------------------------------------------------------------- /src/lwes_util.erl: -------------------------------------------------------------------------------- 1 | -module (lwes_util). 2 | 3 | -include_lib ("lwes.hrl"). 4 | -include_lib ("lwes_internal.hrl"). 5 | 6 | % API 7 | -export ([normalize_ip/1, 8 | ip2bin/1, 9 | ceiling/1, 10 | count_ones/1, 11 | any_to_list/1, 12 | any_to_binary/1, 13 | arr_to_binary/1, 14 | arr_to_binary/2, 15 | binary_to_any/2, 16 | binary_to_arr/2]). 17 | 18 | %%==================================================================== 19 | %% API functions 20 | %%==================================================================== 21 | normalize_ip (Ip) when ?is_ip_addr (Ip) -> 22 | Ip; 23 | normalize_ip (Ip) when is_binary (Ip) -> 24 | normalize_ip (binary_to_list (Ip)); 25 | normalize_ip (Ip) when is_list (Ip) -> 26 | case inet_parse:address (Ip) of 27 | {ok, {N1, N2, N3, N4}} -> {N1, N2, N3, N4}; 28 | _ -> erlang:error(badarg) 29 | end; 30 | normalize_ip (_) -> 31 | % essentially turns function_clause error into badarg 32 | erlang:error (badarg). 33 | 34 | ip2bin (Ip) when is_binary (Ip) -> 35 | Ip; 36 | ip2bin (Ip) -> 37 | list_to_binary (inet_parse:ntoa (Ip)). 38 | 39 | 40 | ceiling(X) when X < 0 -> 41 | trunc(X); 42 | ceiling(X) -> 43 | T = trunc(X), 44 | case X - T == 0 of 45 | true -> T; 46 | false -> T + 1 47 | end. 48 | 49 | count_ones(Bin) -> count_ones(Bin, 0). 50 | count_ones(<<>>, Counter) -> Counter; 51 | count_ones(<>, Counter) -> 52 | count_ones(Rest, Counter + X). 53 | 54 | any_to_list (B) when is_binary (B) -> 55 | binary_to_list (B); 56 | any_to_list (L) when is_list (L) -> 57 | L; 58 | any_to_list (I) when is_integer (I) -> 59 | integer_to_list (I); 60 | any_to_list (F) when is_float (F) -> 61 | float_to_list (F); 62 | any_to_list (A) when is_atom (A) -> 63 | atom_to_list (A). 64 | 65 | arr_to_binary (L, ipaddr) -> 66 | [ip2bin(E) || E <- L ]. 67 | arr_to_binary (L) -> 68 | [any_to_binary (E) || E <- L ]. 69 | 70 | any_to_binary (Ip) when ?is_ip_addr (Ip) -> 71 | ip2bin (Ip); 72 | any_to_binary (T) when is_tuple (T) -> 73 | T; 74 | any_to_binary (I) when is_integer (I) -> 75 | list_to_binary (integer_to_list (I)); 76 | any_to_binary (F) when is_float (F) -> 77 | list_to_binary (float_to_list (F)); 78 | any_to_binary (L = [H|_]) when is_tuple (H) -> 79 | L; 80 | any_to_binary (L = [[_|_]|_]) when is_list (L) -> 81 | % support list of lists, being turned into list of binaries 82 | [ any_to_binary (E) || E <- L ]; 83 | any_to_binary (L) when is_list (L) -> 84 | list_to_binary (L); 85 | any_to_binary (A) when is_atom (A) -> 86 | list_to_binary (atom_to_list (A)); 87 | any_to_binary (B) when is_binary (B) -> 88 | B. 89 | 90 | binary_to_arr (List, Type) -> 91 | [ case E of 92 | null -> undefined; 93 | _ -> binary_to_any(E, Type) 94 | end 95 | || E 96 | <- List 97 | ]. 98 | 99 | binary_to_any (Bin, binary) when is_binary (Bin) -> 100 | Bin; 101 | binary_to_any (L = [B|_], binary) when is_binary (B) -> 102 | L; 103 | binary_to_any (Bin, Type) when is_binary (Bin) -> 104 | binary_to_any (binary_to_list (Bin), Type); 105 | binary_to_any (List, integer) -> 106 | {I, _} = string:to_integer (List), 107 | I; 108 | binary_to_any (List, float) -> 109 | {F, _} = string:to_float (List), 110 | F; 111 | binary_to_any (List, ipaddr) -> normalize_ip (List); 112 | binary_to_any (L = [H|_], list) when is_binary (H) -> 113 | [ binary_to_list (E) || E <- L ]; 114 | binary_to_any (List, list) -> 115 | List; 116 | binary_to_any (List, atom) -> 117 | list_to_atom (List). 118 | 119 | 120 | %%==================================================================== 121 | %% Test functions 122 | %%==================================================================== 123 | -ifdef (TEST). 124 | -include_lib ("eunit/include/eunit.hrl"). 125 | 126 | normalize_ip_test_ () -> 127 | [ 128 | ?_assertEqual ({127,0,0,1}, normalize_ip (<<"127.0.0.1">>)), 129 | ?_assertEqual ({127,0,0,1}, normalize_ip ("127.0.0.1")), 130 | ?_assertEqual ({127,0,0,1}, normalize_ip ({127,0,0,1})), 131 | ?_assertError (badarg, normalize_ip (<<"655.0.0.1">>)), 132 | ?_assertError (badarg, normalize_ip ("655.0.0.1")), 133 | ?_assertError (badarg, normalize_ip ({655,0,0,1})) 134 | ]. 135 | 136 | 137 | ceil_test_ () -> 138 | [ 139 | ?_assertEqual(8, ceiling(7.5)), 140 | ?_assertEqual(-10, ceiling(-10.9)), 141 | ?_assertEqual(0, ceiling(0)) 142 | ]. 143 | 144 | count_ones_test_ () -> 145 | [ 146 | ?_assertEqual(0, count_ones (<<2#00000000>>)), 147 | ?_assertEqual(1, count_ones (<<2#00000001>>)), 148 | ?_assertEqual(1, count_ones (<<2#00000010>>)), 149 | ?_assertEqual(2, count_ones (<<2#00000011>>)), 150 | ?_assertEqual(2, count_ones (<<2#11000000>>)), 151 | ?_assertEqual(7, count_ones (<<2#01111111>>)), 152 | ?_assertEqual(14, count_ones (<<2#01111111,2#01111111>>)) 153 | ]. 154 | 155 | binary_test_ () -> 156 | [ 157 | ?_assertEqual (U, binary_to_any (any_to_binary (U), T)) 158 | || { U, T } 159 | <- 160 | [ 161 | { 1.35, float }, 162 | { 1, integer }, 163 | { 1234567890123, integer }, 164 | { <<"b">>, binary }, 165 | { "b", list }, 166 | { true, atom }, 167 | { ["1"], list }, 168 | { ["1","2"], list } 169 | ] 170 | ]. 171 | 172 | -endif. 173 | --------------------------------------------------------------------------------