├── LICENSE ├── README.md ├── bin ├── mkjson.py └── splunklib │ ├── __init__.py │ ├── binding.py │ ├── client.py │ ├── data.py │ ├── modularinput │ ├── __init__.py │ ├── argument.py │ ├── event.py │ ├── event_writer.py │ ├── input_definition.py │ ├── scheme.py │ ├── script.py │ ├── utils.py │ └── validation_definition.py │ ├── ordereddict.py │ ├── results.py │ ├── searchcommands │ ├── __init__.py │ ├── decorators.py │ ├── environment.py │ ├── eventing_command.py │ ├── external_search_command.py │ ├── generating_command.py │ ├── internals.py │ ├── reporting_command.py │ ├── search_command.py │ ├── streaming_command.py │ └── validators.py │ └── six.py ├── default ├── app.conf ├── commands.conf ├── logging.conf └── searchbnf.conf ├── metadata └── default.meta └── static ├── appIcon.png └── appIcon_2x.png /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Douglas Graeme Brown 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TA-jsontools 2 | 3 | JSON Tools Technology Add-On for Splunk 4 | 5 | Please see documentation here: https://github.com/doksu/TA-jsontools/wiki 6 | -------------------------------------------------------------------------------- /bin/mkjson.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from splunklib.searchcommands import dispatch, StreamingCommand, Configuration, Option, validators 4 | import sys 5 | import json 6 | import re 7 | 8 | @Configuration() 9 | class MkJSONCommand(StreamingCommand): 10 | """ 11 | 12 | ##Syntax 13 | 14 | 15 | ##Description 16 | 17 | 18 | ##Example 19 | 20 | 21 | """ 22 | includehidden = Option(require=False, validate=validators.Boolean()) 23 | outputfield = Option(require=False, validate=validators.Fieldname()) 24 | sortkeys = Option(require=False, validate=validators.Boolean()) 25 | 26 | def stream(self, events): 27 | 28 | if not self.outputfield: 29 | 30 | outputfield = "_raw" 31 | 32 | else: 33 | 34 | outputfield = self.outputfield 35 | 36 | if not self.includehidden: 37 | 38 | self.includehidden = False 39 | 40 | if not self.sortkeys: 41 | 42 | self.sortkeys = False 43 | 44 | for event in events: 45 | 46 | includedfields = set() 47 | 48 | if len(self.fieldnames) > 0: 49 | 50 | for fieldname in self.fieldnames: 51 | 52 | if fieldname in event: 53 | 54 | includedfields.add(fieldname) 55 | 56 | outputdict = {} 57 | 58 | for field in includedfields: 59 | 60 | if len(event[field]) > 0: 61 | 62 | outputdict[field] = event[field] 63 | 64 | event[outputfield] = json.dumps(outputdict, sort_keys=self.sortkeys) 65 | 66 | else: 67 | 68 | outputdict = {} 69 | 70 | for field in event: 71 | 72 | if self.includehidden or not re.match('^\_[^\_]',field): 73 | 74 | if len(event[field]) > 0: 75 | 76 | outputdict[field] = event[field] 77 | 78 | event[outputfield] = json.dumps(outputdict, sort_keys=self.sortkeys) 79 | 80 | yield event 81 | 82 | dispatch(MkJSONCommand, sys.argv, sys.stdin, sys.stdout, __name__) 83 | -------------------------------------------------------------------------------- /bin/splunklib/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011-2015 Splunk, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | """Python library for Splunk.""" 16 | 17 | from __future__ import absolute_import 18 | from splunklib.six.moves import map 19 | __version_info__ = (1, 6, 15) 20 | __version__ = ".".join(map(str, __version_info__)) 21 | -------------------------------------------------------------------------------- /bin/splunklib/data.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011-2015 Splunk, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | """The **splunklib.data** module reads the responses from splunkd in Atom Feed 16 | format, which is the format used by most of the REST API. 17 | """ 18 | 19 | from __future__ import absolute_import 20 | import sys 21 | from xml.etree.ElementTree import XML 22 | from splunklib import six 23 | 24 | __all__ = ["load"] 25 | 26 | # LNAME refers to element names without namespaces; XNAME is the same 27 | # name, but with an XML namespace. 28 | LNAME_DICT = "dict" 29 | LNAME_ITEM = "item" 30 | LNAME_KEY = "key" 31 | LNAME_LIST = "list" 32 | 33 | XNAMEF_REST = "{http://dev.splunk.com/ns/rest}%s" 34 | XNAME_DICT = XNAMEF_REST % LNAME_DICT 35 | XNAME_ITEM = XNAMEF_REST % LNAME_ITEM 36 | XNAME_KEY = XNAMEF_REST % LNAME_KEY 37 | XNAME_LIST = XNAMEF_REST % LNAME_LIST 38 | 39 | # Some responses don't use namespaces (eg: search/parse) so we look for 40 | # both the extended and local versions of the following names. 41 | 42 | def isdict(name): 43 | return name == XNAME_DICT or name == LNAME_DICT 44 | 45 | def isitem(name): 46 | return name == XNAME_ITEM or name == LNAME_ITEM 47 | 48 | def iskey(name): 49 | return name == XNAME_KEY or name == LNAME_KEY 50 | 51 | def islist(name): 52 | return name == XNAME_LIST or name == LNAME_LIST 53 | 54 | def hasattrs(element): 55 | return len(element.attrib) > 0 56 | 57 | def localname(xname): 58 | rcurly = xname.find('}') 59 | return xname if rcurly == -1 else xname[rcurly+1:] 60 | 61 | def load(text, match=None): 62 | """This function reads a string that contains the XML of an Atom Feed, then 63 | returns the 64 | data in a native Python structure (a ``dict`` or ``list``). If you also 65 | provide a tag name or path to match, only the matching sub-elements are 66 | loaded. 67 | 68 | :param text: The XML text to load. 69 | :type text: ``string`` 70 | :param match: A tag name or path to match (optional). 71 | :type match: ``string`` 72 | """ 73 | if text is None: return None 74 | text = text.strip() 75 | if len(text) == 0: return None 76 | nametable = { 77 | 'namespaces': [], 78 | 'names': {} 79 | } 80 | 81 | # Convert to unicode encoding in only python 2 for xml parser 82 | if(sys.version_info < (3, 0, 0) and isinstance(text, unicode)): 83 | text = text.encode('utf-8') 84 | 85 | root = XML(text) 86 | items = [root] if match is None else root.findall(match) 87 | count = len(items) 88 | if count == 0: 89 | return None 90 | elif count == 1: 91 | return load_root(items[0], nametable) 92 | else: 93 | return [load_root(item, nametable) for item in items] 94 | 95 | # Load the attributes of the given element. 96 | def load_attrs(element): 97 | if not hasattrs(element): return None 98 | attrs = record() 99 | for key, value in six.iteritems(element.attrib): 100 | attrs[key] = value 101 | return attrs 102 | 103 | # Parse a element and return a Python dict 104 | def load_dict(element, nametable = None): 105 | value = record() 106 | children = list(element) 107 | for child in children: 108 | assert iskey(child.tag) 109 | name = child.attrib["name"] 110 | value[name] = load_value(child, nametable) 111 | return value 112 | 113 | # Loads the given elements attrs & value into single merged dict. 114 | def load_elem(element, nametable=None): 115 | name = localname(element.tag) 116 | attrs = load_attrs(element) 117 | value = load_value(element, nametable) 118 | if attrs is None: return name, value 119 | if value is None: return name, attrs 120 | # If value is simple, merge into attrs dict using special key 121 | if isinstance(value, six.string_types): 122 | attrs["$text"] = value 123 | return name, attrs 124 | # Both attrs & value are complex, so merge the two dicts, resolving collisions. 125 | collision_keys = [] 126 | for key, val in six.iteritems(attrs): 127 | if key in value and key in collision_keys: 128 | value[key].append(val) 129 | elif key in value and key not in collision_keys: 130 | value[key] = [value[key], val] 131 | collision_keys.append(key) 132 | else: 133 | value[key] = val 134 | return name, value 135 | 136 | # Parse a element and return a Python list 137 | def load_list(element, nametable=None): 138 | assert islist(element.tag) 139 | value = [] 140 | children = list(element) 141 | for child in children: 142 | assert isitem(child.tag) 143 | value.append(load_value(child, nametable)) 144 | return value 145 | 146 | # Load the given root element. 147 | def load_root(element, nametable=None): 148 | tag = element.tag 149 | if isdict(tag): return load_dict(element, nametable) 150 | if islist(tag): return load_list(element, nametable) 151 | k, v = load_elem(element, nametable) 152 | return Record.fromkv(k, v) 153 | 154 | # Load the children of the given element. 155 | def load_value(element, nametable=None): 156 | children = list(element) 157 | count = len(children) 158 | 159 | # No children, assume a simple text value 160 | if count == 0: 161 | text = element.text 162 | if text is None: 163 | return None 164 | text = text.strip() 165 | if len(text) == 0: 166 | return None 167 | return text 168 | 169 | # Look for the special case of a single well-known structure 170 | if count == 1: 171 | child = children[0] 172 | tag = child.tag 173 | if isdict(tag): return load_dict(child, nametable) 174 | if islist(tag): return load_list(child, nametable) 175 | 176 | value = record() 177 | for child in children: 178 | name, item = load_elem(child, nametable) 179 | # If we have seen this name before, promote the value to a list 180 | if name in value: 181 | current = value[name] 182 | if not isinstance(current, list): 183 | value[name] = [current] 184 | value[name].append(item) 185 | else: 186 | value[name] = item 187 | 188 | return value 189 | 190 | # A generic utility that enables "dot" access to dicts 191 | class Record(dict): 192 | """This generic utility class enables dot access to members of a Python 193 | dictionary. 194 | 195 | Any key that is also a valid Python identifier can be retrieved as a field. 196 | So, for an instance of ``Record`` called ``r``, ``r.key`` is equivalent to 197 | ``r['key']``. A key such as ``invalid-key`` or ``invalid.key`` cannot be 198 | retrieved as a field, because ``-`` and ``.`` are not allowed in 199 | identifiers. 200 | 201 | Keys of the form ``a.b.c`` are very natural to write in Python as fields. If 202 | a group of keys shares a prefix ending in ``.``, you can retrieve keys as a 203 | nested dictionary by calling only the prefix. For example, if ``r`` contains 204 | keys ``'foo'``, ``'bar.baz'``, and ``'bar.qux'``, ``r.bar`` returns a record 205 | with the keys ``baz`` and ``qux``. If a key contains multiple ``.``, each 206 | one is placed into a nested dictionary, so you can write ``r.bar.qux`` or 207 | ``r['bar.qux']`` interchangeably. 208 | """ 209 | sep = '.' 210 | 211 | def __call__(self, *args): 212 | if len(args) == 0: return self 213 | return Record((key, self[key]) for key in args) 214 | 215 | def __getattr__(self, name): 216 | try: 217 | return self[name] 218 | except KeyError: 219 | raise AttributeError(name) 220 | 221 | def __delattr__(self, name): 222 | del self[name] 223 | 224 | def __setattr__(self, name, value): 225 | self[name] = value 226 | 227 | @staticmethod 228 | def fromkv(k, v): 229 | result = record() 230 | result[k] = v 231 | return result 232 | 233 | def __getitem__(self, key): 234 | if key in self: 235 | return dict.__getitem__(self, key) 236 | key += self.sep 237 | result = record() 238 | for k,v in six.iteritems(self): 239 | if not k.startswith(key): 240 | continue 241 | suffix = k[len(key):] 242 | if '.' in suffix: 243 | ks = suffix.split(self.sep) 244 | z = result 245 | for x in ks[:-1]: 246 | if x not in z: 247 | z[x] = record() 248 | z = z[x] 249 | z[ks[-1]] = v 250 | else: 251 | result[suffix] = v 252 | if len(result) == 0: 253 | raise KeyError("No key or prefix: %s" % key) 254 | return result 255 | 256 | 257 | def record(value=None): 258 | """This function returns a :class:`Record` instance constructed with an 259 | initial value that you provide. 260 | 261 | :param `value`: An initial record value. 262 | :type `value`: ``dict`` 263 | """ 264 | if value is None: value = {} 265 | return Record(value) 266 | 267 | -------------------------------------------------------------------------------- /bin/splunklib/modularinput/__init__.py: -------------------------------------------------------------------------------- 1 | """The following imports allow these classes to be imported via 2 | the splunklib.modularinput package like so: 3 | 4 | from splunklib.modularinput import * 5 | """ 6 | from .argument import Argument 7 | from .event import Event 8 | from .event_writer import EventWriter 9 | from .input_definition import InputDefinition 10 | from .scheme import Scheme 11 | from .script import Script 12 | from .validation_definition import ValidationDefinition 13 | -------------------------------------------------------------------------------- /bin/splunklib/modularinput/argument.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011-2015 Splunk, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | from __future__ import absolute_import 16 | try: 17 | import xml.etree.ElementTree as ET 18 | except ImportError: 19 | import xml.etree.cElementTree as ET 20 | 21 | class Argument(object): 22 | """Class representing an argument to a modular input kind. 23 | 24 | ``Argument`` is meant to be used with ``Scheme`` to generate an XML 25 | definition of the modular input kind that Splunk understands. 26 | 27 | ``name`` is the only required parameter for the constructor. 28 | 29 | **Example with least parameters**:: 30 | 31 | arg1 = Argument(name="arg1") 32 | 33 | **Example with all parameters**:: 34 | 35 | arg2 = Argument( 36 | name="arg2", 37 | description="This is an argument with lots of parameters", 38 | validation="is_pos_int('some_name')", 39 | data_type=Argument.data_type_number, 40 | required_on_edit=True, 41 | required_on_create=True 42 | ) 43 | """ 44 | 45 | # Constant values, do not change. 46 | # These should be used for setting the value of an Argument object's data_type field. 47 | data_type_boolean = "BOOLEAN" 48 | data_type_number = "NUMBER" 49 | data_type_string = "STRING" 50 | 51 | def __init__(self, name, description=None, validation=None, 52 | data_type=data_type_string, required_on_edit=False, required_on_create=False, title=None): 53 | """ 54 | :param name: ``string``, identifier for this argument in Splunk. 55 | :param description: ``string``, human-readable description of the argument. 56 | :param validation: ``string`` specifying how the argument should be validated, if using internal validation. 57 | If using external validation, this will be ignored. 58 | :param data_type: ``string``, data type of this field; use the class constants. 59 | "data_type_boolean", "data_type_number", or "data_type_string". 60 | :param required_on_edit: ``Boolean``, whether this arg is required when editing an existing modular input of this kind. 61 | :param required_on_create: ``Boolean``, whether this arg is required when creating a modular input of this kind. 62 | :param title: ``String``, a human-readable title for the argument. 63 | """ 64 | self.name = name 65 | self.description = description 66 | self.validation = validation 67 | self.data_type = data_type 68 | self.required_on_edit = required_on_edit 69 | self.required_on_create = required_on_create 70 | self.title = title 71 | 72 | def add_to_document(self, parent): 73 | """Adds an ``Argument`` object to this ElementTree document. 74 | 75 | Adds an subelement to the parent element, typically 76 | and sets up its subelements with their respective text. 77 | 78 | :param parent: An ``ET.Element`` to be the parent of a new subelement 79 | :returns: An ``ET.Element`` object representing this argument. 80 | """ 81 | arg = ET.SubElement(parent, "arg") 82 | arg.set("name", self.name) 83 | 84 | if self.title is not None: 85 | ET.SubElement(arg, "title").text = self.title 86 | 87 | if self.description is not None: 88 | ET.SubElement(arg, "description").text = self.description 89 | 90 | if self.validation is not None: 91 | ET.SubElement(arg, "validation").text = self.validation 92 | 93 | # add all other subelements to this Argument, represented by (tag, text) 94 | subelements = [ 95 | ("data_type", self.data_type), 96 | ("required_on_edit", self.required_on_edit), 97 | ("required_on_create", self.required_on_create) 98 | ] 99 | 100 | for name, value in subelements: 101 | ET.SubElement(arg, name).text = str(value).lower() 102 | 103 | return arg -------------------------------------------------------------------------------- /bin/splunklib/modularinput/event.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011-2015 Splunk, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | from __future__ import absolute_import 16 | from io import TextIOBase 17 | from splunklib.six import ensure_text 18 | 19 | try: 20 | import xml.etree.cElementTree as ET 21 | except ImportError as ie: 22 | import xml.etree.ElementTree as ET 23 | 24 | class Event(object): 25 | """Represents an event or fragment of an event to be written by this modular input to Splunk. 26 | 27 | To write an input to a stream, call the ``write_to`` function, passing in a stream. 28 | """ 29 | def __init__(self, data=None, stanza=None, time=None, host=None, index=None, source=None, 30 | sourcetype=None, done=True, unbroken=True): 31 | """There are no required parameters for constructing an Event 32 | 33 | **Example with minimal configuration**:: 34 | 35 | my_event = Event( 36 | data="This is a test of my new event.", 37 | stanza="myStanzaName", 38 | time="%.3f" % 1372187084.000 39 | ) 40 | 41 | **Example with full configuration**:: 42 | 43 | excellent_event = Event( 44 | data="This is a test of my excellent event.", 45 | stanza="excellenceOnly", 46 | time="%.3f" % 1372274622.493, 47 | host="localhost", 48 | index="main", 49 | source="Splunk", 50 | sourcetype="misc", 51 | done=True, 52 | unbroken=True 53 | ) 54 | 55 | :param data: ``string``, the event's text. 56 | :param stanza: ``string``, name of the input this event should be sent to. 57 | :param time: ``float``, time in seconds, including up to 3 decimal places to represent milliseconds. 58 | :param host: ``string``, the event's host, ex: localhost. 59 | :param index: ``string``, the index this event is specified to write to, or None if default index. 60 | :param source: ``string``, the source of this event, or None to have Splunk guess. 61 | :param sourcetype: ``string``, source type currently set on this event, or None to have Splunk guess. 62 | :param done: ``boolean``, is this a complete ``Event``? False if an ``Event`` fragment. 63 | :param unbroken: ``boolean``, Is this event completely encapsulated in this ``Event`` object? 64 | """ 65 | self.data = data 66 | self.done = done 67 | self.host = host 68 | self.index = index 69 | self.source = source 70 | self.sourceType = sourcetype 71 | self.stanza = stanza 72 | self.time = time 73 | self.unbroken = unbroken 74 | 75 | def write_to(self, stream): 76 | """Write an XML representation of self, an ``Event`` object, to the given stream. 77 | 78 | The ``Event`` object will only be written if its data field is defined, 79 | otherwise a ``ValueError`` is raised. 80 | 81 | :param stream: stream to write XML to. 82 | """ 83 | if self.data is None: 84 | raise ValueError("Events must have at least the data field set to be written to XML.") 85 | 86 | event = ET.Element("event") 87 | if self.stanza is not None: 88 | event.set("stanza", self.stanza) 89 | event.set("unbroken", str(int(self.unbroken))) 90 | 91 | # if a time isn't set, let Splunk guess by not creating a