├── .gitignore ├── LICENSE ├── LICENSE_elaticsearch-py.txt ├── LICENSE_splunk-sdk-python.txt ├── LICENSE_urllib3.txt ├── README.md ├── bin ├── elasticsearch │ ├── __init__.py │ ├── client │ │ ├── __init__.py │ │ ├── cat.py │ │ ├── cluster.py │ │ ├── indices.py │ │ ├── ingest.py │ │ ├── nodes.py │ │ ├── remote.py │ │ ├── snapshot.py │ │ ├── tasks.py │ │ └── utils.py │ ├── compat.py │ ├── connection │ │ ├── __init__.py │ │ ├── base.py │ │ ├── http_requests.py │ │ ├── http_urllib3.py │ │ └── pooling.py │ ├── connection_pool.py │ ├── exceptions.py │ ├── helpers │ │ ├── __init__.py │ │ └── test.py │ ├── serializer.py │ └── transport.py ├── elasticsplunk.py ├── splunklib │ ├── __init__.py │ ├── binding.py │ ├── client.py │ ├── data.py │ ├── modularinput │ │ ├── __init__.py │ │ ├── argument.py │ │ ├── event.py │ │ ├── event_writer.py │ │ ├── input_definition.py │ │ ├── scheme.py │ │ ├── script.py │ │ ├── utils.py │ │ └── validation_definition.py │ ├── ordereddict.py │ ├── results.py │ └── searchcommands │ │ ├── __init__.py │ │ ├── decorators.py │ │ ├── environment.py │ │ ├── eventing_command.py │ │ ├── external_search_command.py │ │ ├── generating_command.py │ │ ├── internals.py │ │ ├── reporting_command.py │ │ ├── search_command.py │ │ ├── streaming_command.py │ │ └── validators.py └── urllib3 │ ├── __init__.py │ ├── _collections.py │ ├── connection.py │ ├── connectionpool.py │ ├── contrib │ ├── __init__.py │ ├── _securetransport │ │ ├── __init__.py │ │ ├── bindings.py │ │ └── low_level.py │ ├── appengine.py │ ├── ntlmpool.py │ ├── pyopenssl.py │ ├── securetransport.py │ └── socks.py │ ├── exceptions.py │ ├── fields.py │ ├── filepost.py │ ├── packages │ ├── __init__.py │ ├── backports │ │ ├── __init__.py │ │ └── makefile.py │ ├── ordered_dict.py │ ├── six.py │ └── ssl_match_hostname │ │ ├── __init__.py │ │ └── _implementation.py │ ├── poolmanager.py │ ├── request.py │ ├── response.py │ └── util │ ├── __init__.py │ ├── connection.py │ ├── request.py │ ├── response.py │ ├── retry.py │ ├── selectors.py │ ├── ssl_.py │ ├── timeout.py │ ├── url.py │ └── wait.py ├── default ├── app.conf ├── commands.conf ├── elasticsplunk.json ├── logging.conf └── searchbnf.conf ├── metadata └── default.meta └── static ├── appIcon.png └── appIcon_2x.png /.gitignore: -------------------------------------------------------------------------------- 1 | local/* 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Bruno Moura 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /LICENSE_urllib3.txt: -------------------------------------------------------------------------------- 1 | This is the MIT license: http://www.opensource.org/licenses/mit-license.php 2 | 3 | Copyright 2008-2016 Andrey Petrov and contributors (see CONTRIBUTORS.txt) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this 6 | software and associated documentation files (the "Software"), to deal in the Software 7 | without restriction, including without limitation the rights to use, copy, modify, merge, 8 | publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons 9 | to whom the Software is furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all copies or 12 | substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 15 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR 16 | PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE 17 | FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 | OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 19 | DEALINGS IN THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ElasticSplunk Search Command 2 | ==================================================== 3 | 4 | A Search command to explore Elasticsearch data within Splunk. 5 | 6 | # Currently supported 7 | - Multiple node search 8 | - Index Specification 9 | - SSL connections 10 | - Scroll searches 11 | - Fields to include 12 | - Splunk timepicker values 13 | - Relative time values 14 | - Timestamp field specification 15 | - Index listing "action=indices-list" 16 | - Cluster health "action=cluster-health" 17 | 18 | # Included libraries 19 | - elasticsearch-py 20 | - urllib3 21 | - splunklib from the splunk-sdk-python 22 | 23 | # Examples 24 | 25 | ## Search: 26 | 27 | When searching with the ess command, it uses by default the Splunk timepicker provided time range unless the earliest and latest parameters are specified.
28 | When earliest and latest parameters are specified this will be the effective range for the search, even though the range below the search bar shows the one from the timepicker. 29 | 30 | ### Using the Splunk timepicker provided time range 31 | ``` 32 | |ess eaddr="https://node1:9200,https://node2:9200" index=indexname tsfield="@timestamp" query="field:value AND host:host*" 33 | ``` 34 | 35 | ### Using the earliest and latest parameters 36 | ``` 37 | |ess eaddr="https://node1:9200,https://node2:9200" index=indexname tsfield="@timestamp" latest=now earliest="now-24h" query="field:value AND host:host*" 38 | ``` 39 | 40 | ## List indices 41 | ``` 42 | |ess eaddr="https://node1:9200,https://node2:9200" action=indices-list" 43 | ``` 44 | 45 | ## Cluster health 46 | ``` 47 | |ess eaddr="https://node1:9200,https://node2:9200" action=cluster-health" 48 | ``` 49 | 50 | Written by Bruno Moura 51 | 52 | -------------------------------------------------------------------------------- /bin/elasticsearch/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | VERSION = (6, 0, 0) 4 | __version__ = VERSION 5 | __versionstr__ = '.'.join(map(str, VERSION)) 6 | 7 | import sys 8 | 9 | if (2, 7) <= sys.version_info < (3, 2): 10 | # On Python 2.7 and Python3 < 3.2, install no-op handler to silence 11 | # `No handlers could be found for logger "elasticsearch"` message per 12 | # 13 | import logging 14 | logger = logging.getLogger('elasticsearch') 15 | logger.addHandler(logging.NullHandler()) 16 | 17 | from .client import Elasticsearch 18 | from .transport import Transport 19 | from .connection_pool import ConnectionPool, ConnectionSelector, \ 20 | RoundRobinSelector 21 | from .serializer import JSONSerializer 22 | from .connection import Connection, RequestsHttpConnection, \ 23 | Urllib3HttpConnection 24 | from .exceptions import * 25 | 26 | -------------------------------------------------------------------------------- /bin/elasticsearch/client/cluster.py: -------------------------------------------------------------------------------- 1 | from .utils import NamespacedClient, query_params, _make_path 2 | 3 | class ClusterClient(NamespacedClient): 4 | @query_params('level', 'local', 'master_timeout', 'timeout', 5 | 'wait_for_active_shards', 'wait_for_events', 6 | 'wait_for_no_relocating_shards', 'wait_for_nodes', 'wait_for_status') 7 | def health(self, index=None, params=None): 8 | """ 9 | Get a very simple status on the health of the cluster. 10 | ``_ 11 | 12 | :arg index: Limit the information returned to a specific index 13 | :arg level: Specify the level of detail for returned information, 14 | default 'cluster', valid choices are: 'cluster', 'indices', 'shards' 15 | :arg local: Return local information, do not retrieve the state from 16 | master node (default: false) 17 | :arg master_timeout: Explicit operation timeout for connection to master 18 | node 19 | :arg timeout: Explicit operation timeout 20 | :arg wait_for_active_shards: Wait until the specified number of shards 21 | is active 22 | :arg wait_for_events: Wait until all currently queued events with the 23 | given priority are processed, valid choices are: 'immediate', 24 | 'urgent', 'high', 'normal', 'low', 'languid' 25 | :arg wait_for_no_relocating_shards: Whether to wait until there are no 26 | relocating shards in the cluster 27 | :arg wait_for_nodes: Wait until the specified number of nodes is 28 | available 29 | :arg wait_for_status: Wait until cluster is in a specific state, default 30 | None, valid choices are: 'green', 'yellow', 'red' 31 | """ 32 | return self.transport.perform_request('GET', _make_path('_cluster', 33 | 'health', index), params=params) 34 | 35 | @query_params('local', 'master_timeout') 36 | def pending_tasks(self, params=None): 37 | """ 38 | The pending cluster tasks API returns a list of any cluster-level 39 | changes (e.g. create index, update mapping, allocate or fail shard) 40 | which have not yet been executed. 41 | ``_ 42 | 43 | :arg local: Return local information, do not retrieve the state from 44 | master node (default: false) 45 | :arg master_timeout: Specify timeout for connection to master 46 | """ 47 | return self.transport.perform_request('GET', 48 | '/_cluster/pending_tasks', params=params) 49 | 50 | @query_params('allow_no_indices', 'expand_wildcards', 'flat_settings', 51 | 'ignore_unavailable', 'local', 'master_timeout') 52 | def state(self, metric=None, index=None, params=None): 53 | """ 54 | Get a comprehensive state information of the whole cluster. 55 | ``_ 56 | 57 | :arg metric: Limit the information returned to the specified metrics 58 | :arg index: A comma-separated list of index names; use `_all` or empty 59 | string to perform the operation on all indices 60 | :arg allow_no_indices: Whether to ignore if a wildcard indices 61 | expression resolves into no concrete indices. (This includes `_all` 62 | string or when no indices have been specified) 63 | :arg expand_wildcards: Whether to expand wildcard expression to concrete 64 | indices that are open, closed or both., default 'open', valid 65 | choices are: 'open', 'closed', 'none', 'all' 66 | :arg flat_settings: Return settings in flat format (default: false) 67 | :arg ignore_unavailable: Whether specified concrete indices should be 68 | ignored when unavailable (missing or closed) 69 | :arg local: Return local information, do not retrieve the state from 70 | master node (default: false) 71 | :arg master_timeout: Specify timeout for connection to master 72 | """ 73 | if index and not metric: 74 | metric = '_all' 75 | return self.transport.perform_request('GET', _make_path('_cluster', 76 | 'state', metric, index), params=params) 77 | 78 | @query_params('flat_settings', 'timeout') 79 | def stats(self, node_id=None, params=None): 80 | """ 81 | The Cluster Stats API allows to retrieve statistics from a cluster wide 82 | perspective. The API returns basic index metrics and information about 83 | the current nodes that form the cluster. 84 | ``_ 85 | 86 | :arg node_id: A comma-separated list of node IDs or names to limit the 87 | returned information; use `_local` to return information from the 88 | node you're connecting to, leave empty to get information from all 89 | nodes 90 | :arg flat_settings: Return settings in flat format (default: false) 91 | :arg timeout: Explicit operation timeout 92 | """ 93 | url = '/_cluster/stats' 94 | if node_id: 95 | url = _make_path('_cluster/stats/nodes', node_id) 96 | return self.transport.perform_request('GET', url, params=params) 97 | 98 | @query_params('dry_run', 'explain', 'master_timeout', 'metric', 99 | 'retry_failed', 'timeout') 100 | def reroute(self, body=None, params=None): 101 | """ 102 | Explicitly execute a cluster reroute allocation command including specific commands. 103 | ``_ 104 | 105 | :arg body: The definition of `commands` to perform (`move`, `cancel`, 106 | `allocate`) 107 | :arg dry_run: Simulate the operation only and return the resulting state 108 | :arg explain: Return an explanation of why the commands can or cannot be 109 | executed 110 | :arg master_timeout: Explicit operation timeout for connection to master 111 | node 112 | :arg metric: Limit the information returned to the specified metrics. 113 | Defaults to all but metadata, valid choices are: '_all', 'blocks', 114 | 'metadata', 'nodes', 'routing_table', 'master_node', 'version' 115 | :arg retry_failed: Retries allocation of shards that are blocked due to 116 | too many subsequent allocation failures 117 | :arg timeout: Explicit operation timeout 118 | """ 119 | return self.transport.perform_request('POST', '/_cluster/reroute', 120 | params=params, body=body) 121 | 122 | @query_params('flat_settings', 'include_defaults', 'master_timeout', 123 | 'timeout') 124 | def get_settings(self, params=None): 125 | """ 126 | Get cluster settings. 127 | ``_ 128 | 129 | :arg flat_settings: Return settings in flat format (default: false) 130 | :arg include_defaults: Whether to return all default clusters setting., 131 | default False 132 | :arg master_timeout: Explicit operation timeout for connection to master 133 | node 134 | :arg timeout: Explicit operation timeout 135 | """ 136 | return self.transport.perform_request('GET', '/_cluster/settings', 137 | params=params) 138 | 139 | @query_params('flat_settings', 'master_timeout', 'timeout') 140 | def put_settings(self, body=None, params=None): 141 | """ 142 | Update cluster wide specific settings. 143 | ``_ 144 | 145 | :arg body: The settings to be updated. Can be either `transient` or 146 | `persistent` (survives cluster restart). 147 | :arg flat_settings: Return settings in flat format (default: false) 148 | :arg master_timeout: Explicit operation timeout for connection to master 149 | node 150 | :arg timeout: Explicit operation timeout 151 | """ 152 | return self.transport.perform_request('PUT', '/_cluster/settings', 153 | params=params, body=body) 154 | 155 | @query_params('include_disk_info', 'include_yes_decisions') 156 | def allocation_explain(self, body=None, params=None): 157 | """ 158 | ``_ 159 | 160 | :arg body: The index, shard, and primary flag to explain. Empty means 161 | 'explain the first unassigned shard' 162 | :arg include_disk_info: Return information about disk usage and shard 163 | sizes (default: false) 164 | :arg include_yes_decisions: Return 'YES' decisions in explanation 165 | (default: false) 166 | """ 167 | return self.transport.perform_request('GET', 168 | '/_cluster/allocation/explain', params=params, body=body) 169 | 170 | -------------------------------------------------------------------------------- /bin/elasticsearch/client/ingest.py: -------------------------------------------------------------------------------- 1 | from .utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH 2 | 3 | class IngestClient(NamespacedClient): 4 | @query_params('master_timeout') 5 | def get_pipeline(self, id=None, params=None): 6 | """ 7 | ``_ 8 | 9 | :arg id: Comma separated list of pipeline ids. Wildcards supported 10 | :arg master_timeout: Explicit operation timeout for connection to master 11 | node 12 | """ 13 | return self.transport.perform_request('GET', _make_path('_ingest', 14 | 'pipeline', id), params=params) 15 | 16 | @query_params('master_timeout', 'timeout') 17 | def put_pipeline(self, id, body, params=None): 18 | """ 19 | ``_ 20 | 21 | :arg id: Pipeline ID 22 | :arg body: The ingest definition 23 | :arg master_timeout: Explicit operation timeout for connection to master 24 | node 25 | :arg timeout: Explicit operation timeout 26 | """ 27 | for param in (id, body): 28 | if param in SKIP_IN_PATH: 29 | raise ValueError("Empty value passed for a required argument.") 30 | return self.transport.perform_request('PUT', _make_path('_ingest', 31 | 'pipeline', id), params=params, body=body) 32 | 33 | @query_params('master_timeout', 'timeout') 34 | def delete_pipeline(self, id, params=None): 35 | """ 36 | ``_ 37 | 38 | :arg id: Pipeline ID 39 | :arg master_timeout: Explicit operation timeout for connection to master 40 | node 41 | :arg timeout: Explicit operation timeout 42 | """ 43 | if id in SKIP_IN_PATH: 44 | raise ValueError("Empty value passed for a required argument 'id'.") 45 | return self.transport.perform_request('DELETE', _make_path('_ingest', 46 | 'pipeline', id), params=params) 47 | 48 | @query_params('verbose') 49 | def simulate(self, body, id=None, params=None): 50 | """ 51 | ``_ 52 | 53 | :arg body: The simulate definition 54 | :arg id: Pipeline ID 55 | :arg verbose: Verbose mode. Display data output for each processor in 56 | executed pipeline, default False 57 | """ 58 | if body in SKIP_IN_PATH: 59 | raise ValueError("Empty value passed for a required argument 'body'.") 60 | return self.transport.perform_request('GET', _make_path('_ingest', 61 | 'pipeline', id, '_simulate'), params=params, body=body) 62 | -------------------------------------------------------------------------------- /bin/elasticsearch/client/nodes.py: -------------------------------------------------------------------------------- 1 | from .utils import NamespacedClient, query_params, _make_path 2 | 3 | class NodesClient(NamespacedClient): 4 | @query_params('flat_settings', 'timeout') 5 | def info(self, node_id=None, metric=None, params=None): 6 | """ 7 | The cluster nodes info API allows to retrieve one or more (or all) of 8 | the cluster nodes information. 9 | ``_ 10 | 11 | :arg node_id: A comma-separated list of node IDs or names to limit the 12 | returned information; use `_local` to return information from the 13 | node you're connecting to, leave empty to get information from all 14 | nodes 15 | :arg metric: A comma-separated list of metrics you wish returned. Leave 16 | empty to return all. 17 | :arg flat_settings: Return settings in flat format (default: false) 18 | :arg timeout: Explicit operation timeout 19 | """ 20 | return self.transport.perform_request('GET', _make_path('_nodes', 21 | node_id, metric), params=params) 22 | 23 | @query_params('completion_fields', 'fielddata_fields', 'fields', 'groups', 24 | 'include_segment_file_sizes', 'level', 'timeout', 'types') 25 | def stats(self, node_id=None, metric=None, index_metric=None, params=None): 26 | """ 27 | The cluster nodes stats API allows to retrieve one or more (or all) of 28 | the cluster nodes statistics. 29 | ``_ 30 | 31 | :arg node_id: A comma-separated list of node IDs or names to limit the 32 | returned information; use `_local` to return information from the 33 | node you're connecting to, leave empty to get information from all 34 | nodes 35 | :arg metric: Limit the information returned to the specified metrics 36 | :arg index_metric: Limit the information returned for `indices` metric 37 | to the specific index metrics. Isn't used if `indices` (or `all`) 38 | metric isn't specified. 39 | :arg completion_fields: A comma-separated list of fields for `fielddata` 40 | and `suggest` index metric (supports wildcards) 41 | :arg fielddata_fields: A comma-separated list of fields for `fielddata` 42 | index metric (supports wildcards) 43 | :arg fields: A comma-separated list of fields for `fielddata` and 44 | `completion` index metric (supports wildcards) 45 | :arg groups: A comma-separated list of search groups for `search` index 46 | metric 47 | :arg include_segment_file_sizes: Whether to report the aggregated disk 48 | usage of each one of the Lucene index files (only applies if segment 49 | stats are requested), default False 50 | :arg level: Return indices stats aggregated at index, node or shard 51 | level, default 'node', valid choices are: 'indices', 'node', 52 | 'shards' 53 | :arg timeout: Explicit operation timeout 54 | :arg types: A comma-separated list of document types for the `indexing` 55 | index metric 56 | """ 57 | return self.transport.perform_request('GET', _make_path('_nodes', 58 | node_id, 'stats', metric, index_metric), params=params) 59 | 60 | @query_params('type', 'ignore_idle_threads', 'interval', 'snapshots', 61 | 'threads', 'timeout') 62 | def hot_threads(self, node_id=None, params=None): 63 | """ 64 | An API allowing to get the current hot threads on each node in the cluster. 65 | ``_ 66 | 67 | :arg node_id: A comma-separated list of node IDs or names to limit the 68 | returned information; use `_local` to return information from the 69 | node you're connecting to, leave empty to get information from all 70 | nodes 71 | :arg type: The type to sample (default: cpu), valid choices are: 72 | 'cpu', 'wait', 'block' 73 | :arg ignore_idle_threads: Don't show threads that are in known-idle 74 | places, such as waiting on a socket select or pulling from an empty 75 | task queue (default: true) 76 | :arg interval: The interval for the second sampling of threads 77 | :arg snapshots: Number of samples of thread stacktrace (default: 10) 78 | :arg threads: Specify the number of threads to provide information for 79 | (default: 3) 80 | :arg timeout: Explicit operation timeout 81 | """ 82 | # avoid python reserved words 83 | if params and 'type_' in params: 84 | params['type'] = params.pop('type_') 85 | return self.transport.perform_request('GET', _make_path('_cluster', 86 | 'nodes', node_id, 'hotthreads'), params=params) 87 | 88 | @query_params('human', 'timeout') 89 | def usage(self, node_id=None, metric=None, params=None): 90 | """ 91 | The cluster nodes usage API allows to retrieve information on the usage 92 | of features for each node. 93 | ``_ 94 | 95 | :arg node_id: A comma-separated list of node IDs or names to limit the 96 | returned information; use `_local` to return information from the 97 | node you're connecting to, leave empty to get information from all 98 | nodes 99 | :arg metric: Limit the information returned to the specified metrics 100 | :arg human: Whether to return time and byte values in human-readable 101 | format., default False 102 | :arg timeout: Explicit operation timeout 103 | """ 104 | return self.transport.perform_request('GET', _make_path('_nodes', 105 | node_id, 'usage', metric), params=params) 106 | -------------------------------------------------------------------------------- /bin/elasticsearch/client/remote.py: -------------------------------------------------------------------------------- 1 | from .utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH 2 | 3 | class RemoteClient(NamespacedClient): 4 | @query_params() 5 | def info(self, params=None): 6 | """ 7 | ``_ 8 | """ 9 | return self.transport.perform_request('GET', '/_remote/info', 10 | params=params) 11 | 12 | -------------------------------------------------------------------------------- /bin/elasticsearch/client/snapshot.py: -------------------------------------------------------------------------------- 1 | from .utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH 2 | 3 | class SnapshotClient(NamespacedClient): 4 | @query_params('master_timeout', 'wait_for_completion') 5 | def create(self, repository, snapshot, body=None, params=None): 6 | """ 7 | Create a snapshot in repository 8 | ``_ 9 | 10 | :arg repository: A repository name 11 | :arg snapshot: A snapshot name 12 | :arg body: The snapshot definition 13 | :arg master_timeout: Explicit operation timeout for connection to master 14 | node 15 | :arg wait_for_completion: Should this request wait until the operation 16 | has completed before returning, default False 17 | """ 18 | for param in (repository, snapshot): 19 | if param in SKIP_IN_PATH: 20 | raise ValueError("Empty value passed for a required argument.") 21 | return self.transport.perform_request('PUT', _make_path('_snapshot', 22 | repository, snapshot), params=params, body=body) 23 | 24 | @query_params('master_timeout') 25 | def delete(self, repository, snapshot, params=None): 26 | """ 27 | Deletes a snapshot from a repository. 28 | ``_ 29 | 30 | :arg repository: A repository name 31 | :arg snapshot: A snapshot name 32 | :arg master_timeout: Explicit operation timeout for connection to master 33 | node 34 | """ 35 | for param in (repository, snapshot): 36 | if param in SKIP_IN_PATH: 37 | raise ValueError("Empty value passed for a required argument.") 38 | return self.transport.perform_request('DELETE', 39 | _make_path('_snapshot', repository, snapshot), params=params) 40 | 41 | @query_params('ignore_unavailable', 'master_timeout', 'verbose') 42 | def get(self, repository, snapshot, params=None): 43 | """ 44 | Retrieve information about a snapshot. 45 | ``_ 46 | 47 | :arg repository: A repository name 48 | :arg snapshot: A comma-separated list of snapshot names 49 | :arg ignore_unavailable: Whether to ignore unavailable snapshots, 50 | defaults to false which means a SnapshotMissingException is thrown 51 | :arg master_timeout: Explicit operation timeout for connection to master 52 | node 53 | :arg verbose: Whether to show verbose snapshot info or only show the 54 | basic info found in the repository index blob 55 | """ 56 | for param in (repository, snapshot): 57 | if param in SKIP_IN_PATH: 58 | raise ValueError("Empty value passed for a required argument.") 59 | return self.transport.perform_request('GET', _make_path('_snapshot', 60 | repository, snapshot), params=params) 61 | 62 | @query_params('master_timeout', 'timeout') 63 | def delete_repository(self, repository, params=None): 64 | """ 65 | Removes a shared file system repository. 66 | ``_ 67 | 68 | :arg repository: A comma-separated list of repository names 69 | :arg master_timeout: Explicit operation timeout for connection to master 70 | node 71 | :arg timeout: Explicit operation timeout 72 | """ 73 | if repository in SKIP_IN_PATH: 74 | raise ValueError("Empty value passed for a required argument 'repository'.") 75 | return self.transport.perform_request('DELETE', 76 | _make_path('_snapshot', repository), params=params) 77 | 78 | @query_params('local', 'master_timeout') 79 | def get_repository(self, repository=None, params=None): 80 | """ 81 | Return information about registered repositories. 82 | ``_ 83 | 84 | :arg repository: A comma-separated list of repository names 85 | :arg local: Return local information, do not retrieve the state from 86 | master node (default: false) 87 | :arg master_timeout: Explicit operation timeout for connection to master 88 | node 89 | """ 90 | return self.transport.perform_request('GET', _make_path('_snapshot', 91 | repository), params=params) 92 | 93 | @query_params('master_timeout', 'timeout', 'verify') 94 | def create_repository(self, repository, body, params=None): 95 | """ 96 | Registers a shared file system repository. 97 | ``_ 98 | 99 | :arg repository: A repository name 100 | :arg body: The repository definition 101 | :arg master_timeout: Explicit operation timeout for connection to master 102 | node 103 | :arg timeout: Explicit operation timeout 104 | :arg verify: Whether to verify the repository after creation 105 | """ 106 | for param in (repository, body): 107 | if param in SKIP_IN_PATH: 108 | raise ValueError("Empty value passed for a required argument.") 109 | return self.transport.perform_request('PUT', _make_path('_snapshot', 110 | repository), params=params, body=body) 111 | 112 | @query_params('master_timeout', 'wait_for_completion') 113 | def restore(self, repository, snapshot, body=None, params=None): 114 | """ 115 | Restore a snapshot. 116 | ``_ 117 | 118 | :arg repository: A repository name 119 | :arg snapshot: A snapshot name 120 | :arg body: Details of what to restore 121 | :arg master_timeout: Explicit operation timeout for connection to master 122 | node 123 | :arg wait_for_completion: Should this request wait until the operation 124 | has completed before returning, default False 125 | """ 126 | for param in (repository, snapshot): 127 | if param in SKIP_IN_PATH: 128 | raise ValueError("Empty value passed for a required argument.") 129 | return self.transport.perform_request('POST', _make_path('_snapshot', 130 | repository, snapshot, '_restore'), params=params, body=body) 131 | 132 | @query_params('ignore_unavailable', 'master_timeout') 133 | def status(self, repository=None, snapshot=None, params=None): 134 | """ 135 | Return information about all currently running snapshots. By specifying 136 | a repository name, it's possible to limit the results to a particular 137 | repository. 138 | ``_ 139 | 140 | :arg repository: A repository name 141 | :arg snapshot: A comma-separated list of snapshot names 142 | :arg ignore_unavailable: Whether to ignore unavailable snapshots, 143 | defaults to false which means a SnapshotMissingException is thrown 144 | :arg master_timeout: Explicit operation timeout for connection to master 145 | node 146 | """ 147 | return self.transport.perform_request('GET', _make_path('_snapshot', 148 | repository, snapshot, '_status'), params=params) 149 | 150 | @query_params('master_timeout', 'timeout') 151 | def verify_repository(self, repository, params=None): 152 | """ 153 | Returns a list of nodes where repository was successfully verified or 154 | an error message if verification process failed. 155 | ``_ 156 | 157 | :arg repository: A repository name 158 | :arg master_timeout: Explicit operation timeout for connection to master 159 | node 160 | :arg timeout: Explicit operation timeout 161 | """ 162 | if repository in SKIP_IN_PATH: 163 | raise ValueError("Empty value passed for a required argument 'repository'.") 164 | return self.transport.perform_request('POST', _make_path('_snapshot', 165 | repository, '_verify'), params=params) 166 | -------------------------------------------------------------------------------- /bin/elasticsearch/client/tasks.py: -------------------------------------------------------------------------------- 1 | from .utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH 2 | 3 | class TasksClient(NamespacedClient): 4 | @query_params('actions', 'detailed', 'group_by', 'nodes', 5 | 'parent_task_id', 'wait_for_completion') 6 | def list(self, params=None): 7 | """ 8 | ``_ 9 | 10 | :arg actions: A comma-separated list of actions that should be returned. 11 | Leave empty to return all. 12 | :arg detailed: Return detailed task information (default: false) 13 | :arg group_by: Group tasks by nodes or parent/child relationships, 14 | default 'nodes', valid choices are: 'nodes', 'parents' 15 | :arg nodes: A comma-separated list of node IDs or names to limit the 16 | returned information; use `_local` to return information from the 17 | node you're connecting to, leave empty to get information from all 18 | nodes 19 | :arg parent_task_id: Return tasks with specified parent task id 20 | (node_id:task_number). Set to -1 to return all. 21 | :arg wait_for_completion: Wait for the matching tasks to complete 22 | (default: false) 23 | """ 24 | return self.transport.perform_request('GET', '/_tasks', params=params) 25 | 26 | @query_params('actions', 'nodes', 'parent_task_id') 27 | def cancel(self, task_id=None, params=None): 28 | """ 29 | 30 | ``_ 31 | 32 | :arg task_id: Cancel the task with specified task id 33 | (node_id:task_number) 34 | :arg actions: A comma-separated list of actions that should be 35 | cancelled. Leave empty to cancel all. 36 | :arg nodes: A comma-separated list of node IDs or names to limit the 37 | returned information; use `_local` to return information from the 38 | node you're connecting to, leave empty to get information from all 39 | nodes 40 | :arg parent_task_id: Cancel tasks with specified parent task id 41 | (node_id:task_number). Set to -1 to cancel all. 42 | """ 43 | return self.transport.perform_request('POST', _make_path('_tasks', 44 | task_id, '_cancel'), params=params) 45 | 46 | @query_params('wait_for_completion') 47 | def get(self, task_id=None, params=None): 48 | """ 49 | Retrieve information for a particular task. 50 | ``_ 51 | 52 | :arg task_id: Return the task with specified id (node_id:task_number) 53 | :arg wait_for_completion: Wait for the matching tasks to complete 54 | (default: false) 55 | """ 56 | return self.transport.perform_request('GET', _make_path('_tasks', 57 | task_id), params=params) 58 | -------------------------------------------------------------------------------- /bin/elasticsearch/client/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import weakref 4 | from datetime import date, datetime 5 | from functools import wraps 6 | from ..compat import string_types, quote_plus, PY2 7 | 8 | # parts of URL to be omitted 9 | SKIP_IN_PATH = (None, '', b'', [], ()) 10 | 11 | def _escape(value): 12 | """ 13 | Escape a single value of a URL string or a query parameter. If it is a list 14 | or tuple, turn it into a comma-separated string first. 15 | """ 16 | 17 | # make sequences into comma-separated stings 18 | if isinstance(value, (list, tuple)): 19 | value = ','.join(value) 20 | 21 | # dates and datetimes into isoformat 22 | elif isinstance(value, (date, datetime)): 23 | value = value.isoformat() 24 | 25 | # make bools into true/false strings 26 | elif isinstance(value, bool): 27 | value = str(value).lower() 28 | 29 | # don't decode bytestrings 30 | elif isinstance(value, bytes): 31 | return value 32 | 33 | # encode strings to utf-8 34 | if isinstance(value, string_types): 35 | if PY2 and isinstance(value, unicode): 36 | return value.encode('utf-8') 37 | if not PY2 and isinstance(value, str): 38 | return value.encode('utf-8') 39 | 40 | return str(value) 41 | 42 | def _make_path(*parts): 43 | """ 44 | Create a URL string from parts, omit all `None` values and empty strings. 45 | Convert lists nad tuples to comma separated values. 46 | """ 47 | #TODO: maybe only allow some parts to be lists/tuples ? 48 | return '/' + '/'.join( 49 | # preserve ',' and '*' in url for nicer URLs in logs 50 | quote_plus(_escape(p), b',*') for p in parts if p not in SKIP_IN_PATH) 51 | 52 | # parameters that apply to all methods 53 | GLOBAL_PARAMS = ('pretty', 'human', 'error_trace', 'format', 'filter_path') 54 | 55 | def query_params(*es_query_params): 56 | """ 57 | Decorator that pops all accepted parameters from method's kwargs and puts 58 | them in the params argument. 59 | """ 60 | def _wrapper(func): 61 | @wraps(func) 62 | def _wrapped(*args, **kwargs): 63 | params = {} 64 | if 'params' in kwargs: 65 | params = kwargs.pop('params').copy() 66 | for p in es_query_params + GLOBAL_PARAMS: 67 | if p in kwargs: 68 | v = kwargs.pop(p) 69 | if v is not None: 70 | params[p] = _escape(v) 71 | 72 | # don't treat ignore and request_timeout as other params to avoid escaping 73 | for p in ('ignore', 'request_timeout'): 74 | if p in kwargs: 75 | params[p] = kwargs.pop(p) 76 | return func(*args, params=params, **kwargs) 77 | return _wrapped 78 | return _wrapper 79 | 80 | 81 | class NamespacedClient(object): 82 | def __init__(self, client): 83 | self.client = client 84 | 85 | @property 86 | def transport(self): 87 | return self.client.transport 88 | 89 | class AddonClient(NamespacedClient): 90 | @classmethod 91 | def infect_client(cls, client): 92 | addon = cls(weakref.proxy(client)) 93 | setattr(client, cls.namespace, addon) 94 | return client 95 | -------------------------------------------------------------------------------- /bin/elasticsearch/compat.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | PY2 = sys.version_info[0] == 2 4 | 5 | if PY2: 6 | string_types = basestring, 7 | from urllib import quote_plus, urlencode, unquote 8 | from urlparse import urlparse 9 | from itertools import imap as map 10 | from Queue import Queue 11 | else: 12 | string_types = str, bytes 13 | from urllib.parse import quote_plus, urlencode, urlparse, unquote 14 | map = map 15 | from queue import Queue 16 | -------------------------------------------------------------------------------- /bin/elasticsearch/connection/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import Connection 2 | from .http_requests import RequestsHttpConnection 3 | from .http_urllib3 import Urllib3HttpConnection 4 | -------------------------------------------------------------------------------- /bin/elasticsearch/connection/base.py: -------------------------------------------------------------------------------- 1 | import logging 2 | try: 3 | import simplejson as json 4 | except ImportError: 5 | import json 6 | 7 | from ..exceptions import TransportError, HTTP_EXCEPTIONS 8 | 9 | logger = logging.getLogger('elasticsearch') 10 | 11 | # create the elasticsearch.trace logger, but only set propagate to False if the 12 | # logger hasn't already been configured 13 | _tracer_already_configured = 'elasticsearch.trace' in logging.Logger.manager.loggerDict 14 | tracer = logging.getLogger('elasticsearch.trace') 15 | if not _tracer_already_configured: 16 | tracer.propagate = False 17 | 18 | 19 | class Connection(object): 20 | """ 21 | Class responsible for maintaining a connection to an Elasticsearch node. It 22 | holds persistent connection pool to it and it's main interface 23 | (`perform_request`) is thread-safe. 24 | 25 | Also responsible for logging. 26 | """ 27 | def __init__(self, host='localhost', port=9200, use_ssl=False, url_prefix='', timeout=10, **kwargs): 28 | """ 29 | :arg host: hostname of the node (default: localhost) 30 | :arg port: port to use (integer, default: 9200) 31 | :arg url_prefix: optional url prefix for elasticsearch 32 | :arg timeout: default timeout in seconds (float, default: 10) 33 | """ 34 | scheme = kwargs.get('scheme', 'http') 35 | if use_ssl or scheme == 'https': 36 | scheme = 'https' 37 | use_ssl = True 38 | self.use_ssl = use_ssl 39 | 40 | self.host = '%s://%s:%s' % (scheme, host, port) 41 | if url_prefix: 42 | url_prefix = '/' + url_prefix.strip('/') 43 | self.url_prefix = url_prefix 44 | self.timeout = timeout 45 | 46 | def __repr__(self): 47 | return '<%s: %s>' % (self.__class__.__name__, self.host) 48 | 49 | def _pretty_json(self, data): 50 | # pretty JSON in tracer curl logs 51 | try: 52 | return json.dumps(json.loads(data), sort_keys=True, indent=2, separators=(',', ': ')).replace("'", r'\u0027') 53 | except (ValueError, TypeError): 54 | # non-json data or a bulk request 55 | return data 56 | 57 | def _log_trace(self, method, path, body, status_code, response, duration): 58 | if not tracer.isEnabledFor(logging.INFO) or not tracer.handlers: 59 | return 60 | 61 | # include pretty in trace curls 62 | path = path.replace('?', '?pretty&', 1) if '?' in path else path + '?pretty' 63 | if self.url_prefix: 64 | path = path.replace(self.url_prefix, '', 1) 65 | tracer.info("curl %s-X%s 'http://localhost:9200%s' -d '%s'", 66 | "-H 'Content-Type: application/json' " if body else '', 67 | method, path, self._pretty_json(body) if body else '') 68 | 69 | if tracer.isEnabledFor(logging.DEBUG): 70 | tracer.debug('#[%s] (%.3fs)\n#%s', status_code, duration, self._pretty_json(response).replace('\n', '\n#') if response else '') 71 | 72 | def log_request_success(self, method, full_url, path, body, status_code, response, duration): 73 | """ Log a successful API call. """ 74 | # TODO: optionally pass in params instead of full_url and do urlencode only when needed 75 | 76 | # body has already been serialized to utf-8, deserialize it for logging 77 | # TODO: find a better way to avoid (de)encoding the body back and forth 78 | if body: 79 | body = body.decode('utf-8', 'ignore') 80 | 81 | logger.info( 82 | '%s %s [status:%s request:%.3fs]', method, full_url, 83 | status_code, duration 84 | ) 85 | logger.debug('> %s', body) 86 | logger.debug('< %s', response) 87 | 88 | self._log_trace(method, path, body, status_code, response, duration) 89 | 90 | def log_request_fail(self, method, full_url, path, body, duration, status_code=None, response=None, exception=None): 91 | """ Log an unsuccessful API call. """ 92 | # do not log 404s on HEAD requests 93 | if method == 'HEAD' and status_code == 404: 94 | return 95 | logger.warning( 96 | '%s %s [status:%s request:%.3fs]', method, full_url, 97 | status_code or 'N/A', duration, exc_info=exception is not None 98 | ) 99 | 100 | # body has already been serialized to utf-8, deserialize it for logging 101 | # TODO: find a better way to avoid (de)encoding the body back and forth 102 | if body: 103 | body = body.decode('utf-8', 'ignore') 104 | 105 | logger.debug('> %s', body) 106 | 107 | self._log_trace(method, path, body, status_code, response, duration) 108 | 109 | if response is not None: 110 | logger.debug('< %s', response) 111 | 112 | def _raise_error(self, status_code, raw_data): 113 | """ Locate appropriate exception and raise it. """ 114 | error_message = raw_data 115 | additional_info = None 116 | try: 117 | if raw_data: 118 | additional_info = json.loads(raw_data) 119 | error_message = additional_info.get('error', error_message) 120 | if isinstance(error_message, dict) and 'type' in error_message: 121 | error_message = error_message['type'] 122 | except (ValueError, TypeError) as err: 123 | logger.warning('Undecodable raw error response from server: %s', err) 124 | 125 | raise HTTP_EXCEPTIONS.get(status_code, TransportError)(status_code, error_message, additional_info) 126 | 127 | 128 | -------------------------------------------------------------------------------- /bin/elasticsearch/connection/http_requests.py: -------------------------------------------------------------------------------- 1 | import time 2 | import warnings 3 | try: 4 | import requests 5 | REQUESTS_AVAILABLE = True 6 | except ImportError: 7 | REQUESTS_AVAILABLE = False 8 | 9 | from .base import Connection 10 | from ..exceptions import ConnectionError, ImproperlyConfigured, ConnectionTimeout, SSLError 11 | from ..compat import urlencode, string_types 12 | 13 | class RequestsHttpConnection(Connection): 14 | """ 15 | Connection using the `requests` library. 16 | 17 | :arg http_auth: optional http auth information as either ':' separated 18 | string or a tuple. Any value will be passed into requests as `auth`. 19 | :arg use_ssl: use ssl for the connection if `True` 20 | :arg verify_certs: whether to verify SSL certificates 21 | :arg ca_certs: optional path to CA bundle. By default standard requests' 22 | bundle will be used. 23 | :arg client_cert: path to the file containing the private key and the 24 | certificate, or cert only if using client_key 25 | :arg client_key: path to the file containing the private key if using 26 | separate cert and key files (client_cert will contain only the cert) 27 | :arg headers: any custom http headers to be add to requests 28 | """ 29 | def __init__(self, host='localhost', port=9200, http_auth=None, 30 | use_ssl=False, verify_certs=True, ca_certs=None, client_cert=None, 31 | client_key=None, headers=None, **kwargs): 32 | if not REQUESTS_AVAILABLE: 33 | raise ImproperlyConfigured("Please install requests to use RequestsHttpConnection.") 34 | 35 | super(RequestsHttpConnection, self).__init__(host=host, port=port, use_ssl=use_ssl, **kwargs) 36 | self.session = requests.Session() 37 | self.session.headers = headers or {} 38 | self.session.headers.setdefault('content-type', 'application/json') 39 | if http_auth is not None: 40 | if isinstance(http_auth, (tuple, list)): 41 | http_auth = tuple(http_auth) 42 | elif isinstance(http_auth, string_types): 43 | http_auth = tuple(http_auth.split(':', 1)) 44 | self.session.auth = http_auth 45 | self.base_url = 'http%s://%s:%d%s' % ( 46 | 's' if self.use_ssl else '', 47 | host, port, self.url_prefix 48 | ) 49 | self.session.verify = verify_certs 50 | if not client_key: 51 | self.session.cert = client_cert 52 | elif client_cert: 53 | # cert is a tuple of (certfile, keyfile) 54 | self.session.cert = (client_cert, client_key) 55 | if ca_certs: 56 | if not verify_certs: 57 | raise ImproperlyConfigured("You cannot pass CA certificates when verify SSL is off.") 58 | self.session.verify = ca_certs 59 | 60 | if self.use_ssl and not verify_certs: 61 | warnings.warn( 62 | 'Connecting to %s using SSL with verify_certs=False is insecure.' % self.base_url) 63 | 64 | def perform_request(self, method, url, params=None, body=None, timeout=None, ignore=(), headers=None): 65 | url = self.base_url + url 66 | if params: 67 | url = '%s?%s' % (url, urlencode(params or {})) 68 | 69 | start = time.time() 70 | request = requests.Request(method=method, headers=headers, url=url, data=body) 71 | prepared_request = self.session.prepare_request(request) 72 | settings = self.session.merge_environment_settings(prepared_request.url, {}, None, None, None) 73 | send_kwargs = {'timeout': timeout or self.timeout} 74 | send_kwargs.update(settings) 75 | try: 76 | response = self.session.send(prepared_request, **send_kwargs) 77 | duration = time.time() - start 78 | raw_data = response.text 79 | except Exception as e: 80 | self.log_request_fail(method, url, prepared_request.path_url, body, time.time() - start, exception=e) 81 | if isinstance(e, requests.exceptions.SSLError): 82 | raise SSLError('N/A', str(e), e) 83 | if isinstance(e, requests.Timeout): 84 | raise ConnectionTimeout('TIMEOUT', str(e), e) 85 | raise ConnectionError('N/A', str(e), e) 86 | 87 | # raise errors based on http status codes, let the client handle those if needed 88 | if not (200 <= response.status_code < 300) and response.status_code not in ignore: 89 | self.log_request_fail(method, url, response.request.path_url, body, duration, response.status_code, raw_data) 90 | self._raise_error(response.status_code, raw_data) 91 | 92 | self.log_request_success(method, url, response.request.path_url, body, response.status_code, raw_data, duration) 93 | 94 | return response.status_code, response.headers, raw_data 95 | 96 | def close(self): 97 | """ 98 | Explicitly closes connections 99 | """ 100 | self.session.close() 101 | -------------------------------------------------------------------------------- /bin/elasticsearch/connection/http_urllib3.py: -------------------------------------------------------------------------------- 1 | import time 2 | import ssl 3 | import urllib3 4 | from urllib3.exceptions import ReadTimeoutError, SSLError as UrllibSSLError 5 | import warnings 6 | 7 | CA_CERTS = None 8 | 9 | try: 10 | import certifi 11 | CA_CERTS = certifi.where() 12 | except ImportError: 13 | pass 14 | 15 | from .base import Connection 16 | from ..exceptions import ConnectionError, ImproperlyConfigured, ConnectionTimeout, SSLError 17 | from ..compat import urlencode 18 | 19 | 20 | def create_ssl_context(**kwargs): 21 | """ 22 | A helper function around creating an SSL context 23 | 24 | https://docs.python.org/3/library/ssl.html#context-creation 25 | 26 | Accepts kwargs in the same manner as `create_default_context`. 27 | """ 28 | ctx = ssl.create_default_context(**kwargs) 29 | return ctx 30 | 31 | 32 | class Urllib3HttpConnection(Connection): 33 | """ 34 | Default connection class using the `urllib3` library and the http protocol. 35 | 36 | :arg host: hostname of the node (default: localhost) 37 | :arg port: port to use (integer, default: 9200) 38 | :arg url_prefix: optional url prefix for elasticsearch 39 | :arg timeout: default timeout in seconds (float, default: 10) 40 | :arg http_auth: optional http auth information as either ':' separated 41 | string or a tuple 42 | :arg use_ssl: use ssl for the connection if `True` 43 | :arg verify_certs: whether to verify SSL certificates 44 | :arg ca_certs: optional path to CA bundle. See 45 | https://urllib3.readthedocs.io/en/latest/security.html#using-certifi-with-urllib3 46 | for instructions how to get default set 47 | :arg client_cert: path to the file containing the private key and the 48 | certificate, or cert only if using client_key 49 | :arg client_key: path to the file containing the private key if using 50 | separate cert and key files (client_cert will contain only the cert) 51 | :arg ssl_version: version of the SSL protocol to use. Choices are: 52 | SSLv23 (default) SSLv2 SSLv3 TLSv1 (see ``PROTOCOL_*`` constants in the 53 | ``ssl`` module for exact options for your environment). 54 | :arg ssl_assert_hostname: use hostname verification if not `False` 55 | :arg ssl_assert_fingerprint: verify the supplied certificate fingerprint if not `None` 56 | :arg maxsize: the number of connections which will be kept open to this 57 | host. See https://urllib3.readthedocs.io/en/1.4/pools.html#api for more 58 | information. 59 | :arg headers: any custom http headers to be add to requests 60 | """ 61 | def __init__(self, host='localhost', port=9200, http_auth=None, 62 | use_ssl=False, verify_certs=True, ca_certs=None, client_cert=None, 63 | client_key=None, ssl_version=None, ssl_assert_hostname=None, 64 | ssl_assert_fingerprint=None, maxsize=10, headers=None, ssl_context=None, **kwargs): 65 | 66 | super(Urllib3HttpConnection, self).__init__(host=host, port=port, use_ssl=use_ssl, **kwargs) 67 | self.headers = urllib3.make_headers(keep_alive=True) 68 | if http_auth is not None: 69 | if isinstance(http_auth, (tuple, list)): 70 | http_auth = ':'.join(http_auth) 71 | self.headers.update(urllib3.make_headers(basic_auth=http_auth)) 72 | 73 | # update headers in lowercase to allow overriding of auth headers 74 | if headers: 75 | for k in headers: 76 | self.headers[k.lower()] = headers[k] 77 | 78 | self.headers.setdefault('content-type', 'application/json') 79 | pool_class = urllib3.HTTPConnectionPool 80 | kw = {} 81 | 82 | # if providing an SSL context, raise error if any other SSL related flag is used 83 | if ssl_context and (verify_certs or ca_certs or ssl_version): 84 | raise ImproperlyConfigured("When using `ssl_context`, `use_ssl`, `verify_certs`, `ca_certs` and `ssl_version` are not permitted") 85 | 86 | # if ssl_context provided use SSL by default 87 | if use_ssl or ssl_context: 88 | if not ca_certs and not ssl_context and verify_certs: 89 | # If no ca_certs and no sslcontext passed and asking to verify certs 90 | # raise error 91 | raise ImproperlyConfigured("Root certificates are missing for certificate " 92 | "validation. Either pass them in using the ca_certs parameter or " 93 | "install certifi to use it automatically.") 94 | if verify_certs or ca_certs or ssl_version: 95 | warnings.warn('Use of `verify_certs`, `ca_certs`, `ssl_version` have been deprecated in favor of using SSLContext`', DeprecationWarning) 96 | pool_class = urllib3.HTTPSConnectionPool 97 | 98 | if not ssl_context: 99 | # if SSLContext hasn't been passed in, create one. 100 | cafile = CA_CERTS if ca_certs is None else ca_certs 101 | # need to skip if sslContext isn't avail 102 | try: 103 | ssl_context = create_ssl_context(cafile=cafile) 104 | except AttributeError: 105 | ssl_context = None 106 | 107 | if not verify_certs and ssl_context is not None: 108 | ssl_context.check_hostname = False 109 | ssl_context.verify_mode = ssl.CERT_NONE 110 | warnings.warn( 111 | 'Connecting to %s using SSL with verify_certs=False is insecure.' % host) 112 | 113 | kw.update({ 114 | 'ssl_version': ssl_version, 115 | 'assert_hostname': ssl_assert_hostname, 116 | 'assert_fingerprint': ssl_assert_fingerprint, 117 | 'ssl_context': ssl_context, 118 | 'cert_file': client_cert, 119 | 'ca_certs': ca_certs, 120 | 'key_file': client_key, 121 | }) 122 | self.pool = pool_class(host, port=port, timeout=self.timeout, maxsize=maxsize, **kw) 123 | 124 | def perform_request(self, method, url, params=None, body=None, timeout=None, ignore=(), headers=None): 125 | url = self.url_prefix + url 126 | if params: 127 | url = '%s?%s' % (url, urlencode(params)) 128 | full_url = self.host + url 129 | 130 | start = time.time() 131 | try: 132 | kw = {} 133 | if timeout: 134 | kw['timeout'] = timeout 135 | 136 | # in python2 we need to make sure the url and method are not 137 | # unicode. Otherwise the body will be decoded into unicode too and 138 | # that will fail (#133, #201). 139 | if not isinstance(url, str): 140 | url = url.encode('utf-8') 141 | if not isinstance(method, str): 142 | method = method.encode('utf-8') 143 | 144 | if headers: 145 | request_headers = dict(self.headers) 146 | request_headers.update(headers or {}) 147 | response = self.pool.urlopen(method, url, body, retries=False, headers=self.headers, **kw) 148 | duration = time.time() - start 149 | raw_data = response.data.decode('utf-8') 150 | except Exception as e: 151 | self.log_request_fail(method, full_url, url, body, time.time() - start, exception=e) 152 | if isinstance(e, UrllibSSLError): 153 | raise SSLError('N/A', str(e), e) 154 | if isinstance(e, ReadTimeoutError): 155 | raise ConnectionTimeout('TIMEOUT', str(e), e) 156 | raise ConnectionError('N/A', str(e), e) 157 | 158 | # raise errors based on http status codes, let the client handle those if needed 159 | if not (200 <= response.status < 300) and response.status not in ignore: 160 | self.log_request_fail(method, full_url, url, body, duration, response.status, raw_data) 161 | self._raise_error(response.status, raw_data) 162 | 163 | self.log_request_success(method, full_url, url, body, response.status, 164 | raw_data, duration) 165 | 166 | return response.status, response.getheaders(), raw_data 167 | 168 | def close(self): 169 | """ 170 | Explicitly closes connection 171 | """ 172 | self.pool.close() 173 | -------------------------------------------------------------------------------- /bin/elasticsearch/connection/pooling.py: -------------------------------------------------------------------------------- 1 | try: 2 | import queue 3 | except ImportError: 4 | import Queue as queue 5 | from .base import Connection 6 | 7 | 8 | class PoolingConnection(Connection): 9 | """ 10 | Base connection class for connections that use libraries without thread 11 | safety and no capacity for connection pooling. To use this just implement a 12 | ``_make_connection`` method that constructs a new connection and returns 13 | it. 14 | """ 15 | def __init__(self, *args, **kwargs): 16 | self._free_connections = queue.Queue() 17 | super(PoolingConnection, self).__init__(*args, **kwargs) 18 | 19 | def _get_connection(self): 20 | try: 21 | return self._free_connections.get_nowait() 22 | except queue.Empty: 23 | return self._make_connection() 24 | 25 | def _release_connection(self, con): 26 | self._free_connections.put(con) 27 | 28 | def close(self): 29 | """ 30 | Explicitly close connection 31 | """ 32 | pass 33 | 34 | -------------------------------------------------------------------------------- /bin/elasticsearch/exceptions.py: -------------------------------------------------------------------------------- 1 | __all__ = [ 2 | 'ImproperlyConfigured', 'ElasticsearchException', 'SerializationError', 3 | 'TransportError', 'NotFoundError', 'ConflictError', 'RequestError', 'ConnectionError', 4 | 'SSLError', 'ConnectionTimeout', 'AuthenticationException', 'AuthorizationException' 5 | ] 6 | 7 | class ImproperlyConfigured(Exception): 8 | """ 9 | Exception raised when the config passed to the client is inconsistent or invalid. 10 | """ 11 | 12 | 13 | class ElasticsearchException(Exception): 14 | """ 15 | Base class for all exceptions raised by this package's operations (doesn't 16 | apply to :class:`~elasticsearch.ImproperlyConfigured`). 17 | """ 18 | 19 | 20 | class SerializationError(ElasticsearchException): 21 | """ 22 | Data passed in failed to serialize properly in the ``Serializer`` being 23 | used. 24 | """ 25 | 26 | 27 | class TransportError(ElasticsearchException): 28 | """ 29 | Exception raised when ES returns a non-OK (>=400) HTTP status code. Or when 30 | an actual connection error happens; in that case the ``status_code`` will 31 | be set to ``'N/A'``. 32 | """ 33 | @property 34 | def status_code(self): 35 | """ 36 | The HTTP status code of the response that precipitated the error or 37 | ``'N/A'`` if not applicable. 38 | """ 39 | return self.args[0] 40 | 41 | @property 42 | def error(self): 43 | """ A string error message. """ 44 | return self.args[1] 45 | 46 | @property 47 | def info(self): 48 | """ Dict of returned error info from ES, where available. """ 49 | return self.args[2] 50 | 51 | def __str__(self): 52 | cause = '' 53 | try: 54 | if self.info: 55 | cause = ', %r' % self.info['error']['root_cause'][0]['reason'] 56 | except LookupError: 57 | pass 58 | return 'TransportError(%s, %r%s)' % (self.status_code, self.error, cause) 59 | 60 | 61 | class ConnectionError(TransportError): 62 | """ 63 | Error raised when there was an exception while talking to ES. Original 64 | exception from the underlying :class:`~elasticsearch.Connection` 65 | implementation is available as ``.info.`` 66 | """ 67 | def __str__(self): 68 | return 'ConnectionError(%s) caused by: %s(%s)' % ( 69 | self.error, self.info.__class__.__name__, self.info) 70 | 71 | 72 | class SSLError(ConnectionError): 73 | """ Error raised when encountering SSL errors. """ 74 | 75 | 76 | class ConnectionTimeout(ConnectionError): 77 | """ A network timeout. Doesn't cause a node retry by default. """ 78 | def __str__(self): 79 | return 'ConnectionTimeout caused by - %s(%s)' % ( 80 | self.info.__class__.__name__, self.info) 81 | 82 | 83 | class NotFoundError(TransportError): 84 | """ Exception representing a 404 status code. """ 85 | 86 | 87 | class ConflictError(TransportError): 88 | """ Exception representing a 409 status code. """ 89 | 90 | 91 | class RequestError(TransportError): 92 | """ Exception representing a 400 status code. """ 93 | 94 | 95 | class AuthenticationException(TransportError): 96 | """ Exception representing a 401 status code. """ 97 | 98 | 99 | class AuthorizationException(TransportError): 100 | """ Exception representing a 403 status code. """ 101 | 102 | # more generic mappings from status_code to python exceptions 103 | HTTP_EXCEPTIONS = { 104 | 400: RequestError, 105 | 401: AuthenticationException, 106 | 403: AuthorizationException, 107 | 404: NotFoundError, 108 | 409: ConflictError, 109 | } 110 | -------------------------------------------------------------------------------- /bin/elasticsearch/helpers/test.py: -------------------------------------------------------------------------------- 1 | import time 2 | import os 3 | try: 4 | # python 2.6 5 | from unittest2 import TestCase, SkipTest 6 | except ImportError: 7 | from unittest import TestCase, SkipTest 8 | 9 | from elasticsearch import Elasticsearch 10 | from elasticsearch.exceptions import ConnectionError 11 | 12 | def get_test_client(nowait=False, **kwargs): 13 | # construct kwargs from the environment 14 | kw = {'timeout': 30} 15 | if 'TEST_ES_CONNECTION' in os.environ: 16 | from elasticsearch import connection 17 | kw['connection_class'] = getattr(connection, os.environ['TEST_ES_CONNECTION']) 18 | 19 | kw.update(kwargs) 20 | client = Elasticsearch([os.environ.get('TEST_ES_SERVER', {})], **kw) 21 | 22 | # wait for yellow status 23 | for _ in range(1 if nowait else 100): 24 | try: 25 | client.cluster.health(wait_for_status='yellow') 26 | return client 27 | except ConnectionError: 28 | time.sleep(.1) 29 | else: 30 | # timeout 31 | raise SkipTest("Elasticsearch failed to start.") 32 | 33 | def _get_version(version_string): 34 | if '.' not in version_string: 35 | return () 36 | version = version_string.strip().split('.') 37 | return tuple(int(v) if v.isdigit() else 999 for v in version) 38 | 39 | class ElasticsearchTestCase(TestCase): 40 | @staticmethod 41 | def _get_client(): 42 | return get_test_client() 43 | 44 | @classmethod 45 | def setUpClass(cls): 46 | super(ElasticsearchTestCase, cls).setUpClass() 47 | cls.client = cls._get_client() 48 | 49 | def tearDown(self): 50 | super(ElasticsearchTestCase, self).tearDown() 51 | self.client.indices.delete(index='*', ignore=404) 52 | self.client.indices.delete_template(name='*', ignore=404) 53 | 54 | @property 55 | def es_version(self): 56 | if not hasattr(self, '_es_version'): 57 | version_string = self.client.info()['version']['number'] 58 | self._es_version = _get_version(version_string) 59 | return self._es_version 60 | 61 | -------------------------------------------------------------------------------- /bin/elasticsearch/serializer.py: -------------------------------------------------------------------------------- 1 | try: 2 | import simplejson as json 3 | except ImportError: 4 | import json 5 | import uuid 6 | from datetime import date, datetime 7 | from decimal import Decimal 8 | 9 | from .exceptions import SerializationError, ImproperlyConfigured 10 | from .compat import string_types 11 | 12 | class TextSerializer(object): 13 | mimetype = 'text/plain' 14 | 15 | def loads(self, s): 16 | return s 17 | 18 | def dumps(self, data): 19 | if isinstance(data, string_types): 20 | return data 21 | 22 | raise SerializationError('Cannot serialize %r into text.' % data) 23 | 24 | class JSONSerializer(object): 25 | mimetype = 'application/json' 26 | 27 | def default(self, data): 28 | if isinstance(data, (date, datetime)): 29 | return data.isoformat() 30 | elif isinstance(data, Decimal): 31 | return float(data) 32 | elif isinstance(data, uuid.UUID): 33 | return str(data) 34 | raise TypeError("Unable to serialize %r (type: %s)" % (data, type(data))) 35 | 36 | def loads(self, s): 37 | try: 38 | return json.loads(s) 39 | except (ValueError, TypeError) as e: 40 | raise SerializationError(s, e) 41 | 42 | def dumps(self, data): 43 | # don't serialize strings 44 | if isinstance(data, string_types): 45 | return data 46 | 47 | try: 48 | return json.dumps(data, default=self.default, ensure_ascii=False) 49 | except (ValueError, TypeError) as e: 50 | raise SerializationError(data, e) 51 | 52 | DEFAULT_SERIALIZERS = { 53 | JSONSerializer.mimetype: JSONSerializer(), 54 | TextSerializer.mimetype: TextSerializer(), 55 | } 56 | 57 | class Deserializer(object): 58 | def __init__(self, serializers, default_mimetype='application/json'): 59 | try: 60 | self.default = serializers[default_mimetype] 61 | except KeyError: 62 | raise ImproperlyConfigured('Cannot find default serializer (%s)' % default_mimetype) 63 | self.serializers = serializers 64 | 65 | def loads(self, s, mimetype=None): 66 | if not mimetype: 67 | deserializer = self.default 68 | else: 69 | # split out charset 70 | mimetype = mimetype.split(';', 1)[0] 71 | try: 72 | deserializer = self.serializers[mimetype] 73 | except KeyError: 74 | raise SerializationError('Unknown mimetype, unable to deserialize: %s' % mimetype) 75 | 76 | return deserializer.loads(s) 77 | 78 | -------------------------------------------------------------------------------- /bin/splunklib/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011-2015 Splunk, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | """Python library for Splunk.""" 16 | 17 | __version_info__ = (1, 6, 2) 18 | __version__ = ".".join(map(str, __version_info__)) 19 | 20 | -------------------------------------------------------------------------------- /bin/splunklib/data.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011-2015 Splunk, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | """The **splunklib.data** module reads the responses from splunkd in Atom Feed 16 | format, which is the format used by most of the REST API. 17 | """ 18 | 19 | from xml.etree.ElementTree import XML 20 | 21 | __all__ = ["load"] 22 | 23 | # LNAME refers to element names without namespaces; XNAME is the same 24 | # name, but with an XML namespace. 25 | LNAME_DICT = "dict" 26 | LNAME_ITEM = "item" 27 | LNAME_KEY = "key" 28 | LNAME_LIST = "list" 29 | 30 | XNAMEF_REST = "{http://dev.splunk.com/ns/rest}%s" 31 | XNAME_DICT = XNAMEF_REST % LNAME_DICT 32 | XNAME_ITEM = XNAMEF_REST % LNAME_ITEM 33 | XNAME_KEY = XNAMEF_REST % LNAME_KEY 34 | XNAME_LIST = XNAMEF_REST % LNAME_LIST 35 | 36 | # Some responses don't use namespaces (eg: search/parse) so we look for 37 | # both the extended and local versions of the following names. 38 | 39 | def isdict(name): 40 | return name == XNAME_DICT or name == LNAME_DICT 41 | 42 | def isitem(name): 43 | return name == XNAME_ITEM or name == LNAME_ITEM 44 | 45 | def iskey(name): 46 | return name == XNAME_KEY or name == LNAME_KEY 47 | 48 | def islist(name): 49 | return name == XNAME_LIST or name == LNAME_LIST 50 | 51 | def hasattrs(element): 52 | return len(element.attrib) > 0 53 | 54 | def localname(xname): 55 | rcurly = xname.find('}') 56 | return xname if rcurly == -1 else xname[rcurly+1:] 57 | 58 | def load(text, match=None): 59 | """This function reads a string that contains the XML of an Atom Feed, then 60 | returns the 61 | data in a native Python structure (a ``dict`` or ``list``). If you also 62 | provide a tag name or path to match, only the matching sub-elements are 63 | loaded. 64 | 65 | :param text: The XML text to load. 66 | :type text: ``string`` 67 | :param match: A tag name or path to match (optional). 68 | :type match: ``string`` 69 | """ 70 | if text is None: return None 71 | text = text.strip() 72 | if len(text) == 0: return None 73 | nametable = { 74 | 'namespaces': [], 75 | 'names': {} 76 | } 77 | root = XML(text) 78 | items = [root] if match is None else root.findall(match) 79 | count = len(items) 80 | if count == 0: 81 | return None 82 | elif count == 1: 83 | return load_root(items[0], nametable) 84 | else: 85 | return [load_root(item, nametable) for item in items] 86 | 87 | # Load the attributes of the given element. 88 | def load_attrs(element): 89 | if not hasattrs(element): return None 90 | attrs = record() 91 | for key, value in element.attrib.iteritems(): 92 | attrs[key] = value 93 | return attrs 94 | 95 | # Parse a element and return a Python dict 96 | def load_dict(element, nametable = None): 97 | value = record() 98 | children = list(element) 99 | for child in children: 100 | assert iskey(child.tag) 101 | name = child.attrib["name"] 102 | value[name] = load_value(child, nametable) 103 | return value 104 | 105 | # Loads the given elements attrs & value into single merged dict. 106 | def load_elem(element, nametable=None): 107 | name = localname(element.tag) 108 | attrs = load_attrs(element) 109 | value = load_value(element, nametable) 110 | if attrs is None: return name, value 111 | if value is None: return name, attrs 112 | # If value is simple, merge into attrs dict using special key 113 | if isinstance(value, str): 114 | attrs["$text"] = value 115 | return name, attrs 116 | # Both attrs & value are complex, so merge the two dicts, resolving collisions. 117 | collision_keys = [] 118 | for key, val in attrs.iteritems(): 119 | if key in value and key in collision_keys: 120 | value[key].append(val) 121 | elif key in value and key not in collision_keys: 122 | value[key] = [value[key], val] 123 | collision_keys.append(key) 124 | else: 125 | value[key] = val 126 | return name, value 127 | 128 | # Parse a element and return a Python list 129 | def load_list(element, nametable=None): 130 | assert islist(element.tag) 131 | value = [] 132 | children = list(element) 133 | for child in children: 134 | assert isitem(child.tag) 135 | value.append(load_value(child, nametable)) 136 | return value 137 | 138 | # Load the given root element. 139 | def load_root(element, nametable=None): 140 | tag = element.tag 141 | if isdict(tag): return load_dict(element, nametable) 142 | if islist(tag): return load_list(element, nametable) 143 | k, v = load_elem(element, nametable) 144 | return Record.fromkv(k, v) 145 | 146 | # Load the children of the given element. 147 | def load_value(element, nametable=None): 148 | children = list(element) 149 | count = len(children) 150 | 151 | # No children, assume a simple text value 152 | if count == 0: 153 | text = element.text 154 | if text is None: 155 | return None 156 | text = text.strip() 157 | if len(text) == 0: 158 | return None 159 | return text 160 | 161 | # Look for the special case of a single well-known structure 162 | if count == 1: 163 | child = children[0] 164 | tag = child.tag 165 | if isdict(tag): return load_dict(child, nametable) 166 | if islist(tag): return load_list(child, nametable) 167 | 168 | value = record() 169 | for child in children: 170 | name, item = load_elem(child, nametable) 171 | # If we have seen this name before, promote the value to a list 172 | if value.has_key(name): 173 | current = value[name] 174 | if not isinstance(current, list): 175 | value[name] = [current] 176 | value[name].append(item) 177 | else: 178 | value[name] = item 179 | 180 | return value 181 | 182 | # A generic utility that enables "dot" access to dicts 183 | class Record(dict): 184 | """This generic utility class enables dot access to members of a Python 185 | dictionary. 186 | 187 | Any key that is also a valid Python identifier can be retrieved as a field. 188 | So, for an instance of ``Record`` called ``r``, ``r.key`` is equivalent to 189 | ``r['key']``. A key such as ``invalid-key`` or ``invalid.key`` cannot be 190 | retrieved as a field, because ``-`` and ``.`` are not allowed in 191 | identifiers. 192 | 193 | Keys of the form ``a.b.c`` are very natural to write in Python as fields. If 194 | a group of keys shares a prefix ending in ``.``, you can retrieve keys as a 195 | nested dictionary by calling only the prefix. For example, if ``r`` contains 196 | keys ``'foo'``, ``'bar.baz'``, and ``'bar.qux'``, ``r.bar`` returns a record 197 | with the keys ``baz`` and ``qux``. If a key contains multiple ``.``, each 198 | one is placed into a nested dictionary, so you can write ``r.bar.qux`` or 199 | ``r['bar.qux']`` interchangeably. 200 | """ 201 | sep = '.' 202 | 203 | def __call__(self, *args): 204 | if len(args) == 0: return self 205 | return Record((key, self[key]) for key in args) 206 | 207 | def __getattr__(self, name): 208 | try: 209 | return self[name] 210 | except KeyError: 211 | raise AttributeError(name) 212 | 213 | def __delattr__(self, name): 214 | del self[name] 215 | 216 | def __setattr__(self, name, value): 217 | self[name] = value 218 | 219 | @staticmethod 220 | def fromkv(k, v): 221 | result = record() 222 | result[k] = v 223 | return result 224 | 225 | def __getitem__(self, key): 226 | if key in self: 227 | return dict.__getitem__(self, key) 228 | key += self.sep 229 | result = record() 230 | for k,v in self.iteritems(): 231 | if not k.startswith(key): 232 | continue 233 | suffix = k[len(key):] 234 | if '.' in suffix: 235 | ks = suffix.split(self.sep) 236 | z = result 237 | for x in ks[:-1]: 238 | if x not in z: 239 | z[x] = record() 240 | z = z[x] 241 | z[ks[-1]] = v 242 | else: 243 | result[suffix] = v 244 | if len(result) == 0: 245 | raise KeyError("No key or prefix: %s" % key) 246 | return result 247 | 248 | 249 | def record(value=None): 250 | """This function returns a :class:`Record` instance constructed with an 251 | initial value that you provide. 252 | 253 | :param `value`: An initial record value. 254 | :type `value`: ``dict`` 255 | """ 256 | if value is None: value = {} 257 | return Record(value) 258 | 259 | -------------------------------------------------------------------------------- /bin/splunklib/modularinput/__init__.py: -------------------------------------------------------------------------------- 1 | """The following imports allow these classes to be imported via 2 | the splunklib.modularinput package like so: 3 | 4 | from splunklib.modularinput import * 5 | """ 6 | from .argument import Argument 7 | from .event import Event 8 | from .event_writer import EventWriter 9 | from .input_definition import InputDefinition 10 | from .scheme import Scheme 11 | from .script import Script 12 | from .validation_definition import ValidationDefinition 13 | -------------------------------------------------------------------------------- /bin/splunklib/modularinput/argument.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011-2015 Splunk, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | try: 16 | import xml.etree.ElementTree as ET 17 | except ImportError: 18 | import xml.etree.cElementTree as ET 19 | 20 | class Argument(object): 21 | """Class representing an argument to a modular input kind. 22 | 23 | ``Argument`` is meant to be used with ``Scheme`` to generate an XML 24 | definition of the modular input kind that Splunk understands. 25 | 26 | ``name`` is the only required parameter for the constructor. 27 | 28 | **Example with least parameters**:: 29 | 30 | arg1 = Argument(name="arg1") 31 | 32 | **Example with all parameters**:: 33 | 34 | arg2 = Argument( 35 | name="arg2", 36 | description="This is an argument with lots of parameters", 37 | validation="is_pos_int('some_name')", 38 | data_type=Argument.data_type_number, 39 | required_on_edit=True, 40 | required_on_create=True 41 | ) 42 | """ 43 | 44 | # Constant values, do not change. 45 | # These should be used for setting the value of an Argument object's data_type field. 46 | data_type_boolean = "BOOLEAN" 47 | data_type_number = "NUMBER" 48 | data_type_string = "STRING" 49 | 50 | def __init__(self, name, description=None, validation=None, 51 | data_type=data_type_string, required_on_edit=False, required_on_create=False, title=None): 52 | """ 53 | :param name: ``string``, identifier for this argument in Splunk. 54 | :param description: ``string``, human-readable description of the argument. 55 | :param validation: ``string`` specifying how the argument should be validated, if using internal validation. 56 | If using external validation, this will be ignored. 57 | :param data_type: ``string``, data type of this field; use the class constants. 58 | "data_type_boolean", "data_type_number", or "data_type_string". 59 | :param required_on_edit: ``Boolean``, whether this arg is required when editing an existing modular input of this kind. 60 | :param required_on_create: ``Boolean``, whether this arg is required when creating a modular input of this kind. 61 | :param title: ``String``, a human-readable title for the argument. 62 | """ 63 | self.name = name 64 | self.description = description 65 | self.validation = validation 66 | self.data_type = data_type 67 | self.required_on_edit = required_on_edit 68 | self.required_on_create = required_on_create 69 | self.title = title 70 | 71 | def add_to_document(self, parent): 72 | """Adds an ``Argument`` object to this ElementTree document. 73 | 74 | Adds an subelement to the parent element, typically 75 | and sets up its subelements with their respective text. 76 | 77 | :param parent: An ``ET.Element`` to be the parent of a new subelement 78 | :returns: An ``ET.Element`` object representing this argument. 79 | """ 80 | arg = ET.SubElement(parent, "arg") 81 | arg.set("name", self.name) 82 | 83 | if self.title is not None: 84 | ET.SubElement(arg, "title").text = self.title 85 | 86 | if self.description is not None: 87 | ET.SubElement(arg, "description").text = self.description 88 | 89 | if self.validation is not None: 90 | ET.SubElement(arg, "validation").text = self.validation 91 | 92 | # add all other subelements to this Argument, represented by (tag, text) 93 | subelements = [ 94 | ("data_type", self.data_type), 95 | ("required_on_edit", self.required_on_edit), 96 | ("required_on_create", self.required_on_create) 97 | ] 98 | 99 | for name, value in subelements: 100 | ET.SubElement(arg, name).text = str(value).lower() 101 | 102 | return arg -------------------------------------------------------------------------------- /bin/splunklib/modularinput/event.py: -------------------------------------------------------------------------------- 1 | # Copyright 2011-2015 Splunk, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"): you may 4 | # not use this file except in compliance with the License. You may obtain 5 | # a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations 13 | # under the License. 14 | 15 | try: 16 | import xml.etree.cElementTree as ET 17 | except ImportError as ie: 18 | import xml.etree.ElementTree as ET 19 | 20 | class Event(object): 21 | """Represents an event or fragment of an event to be written by this modular input to Splunk. 22 | 23 | To write an input to a stream, call the ``write_to`` function, passing in a stream. 24 | """ 25 | def __init__(self, data=None, stanza=None, time=None, host=None, index=None, source=None, 26 | sourcetype=None, done=True, unbroken=True): 27 | """There are no required parameters for constructing an Event 28 | 29 | **Example with minimal configuration**:: 30 | 31 | my_event = Event( 32 | data="This is a test of my new event.", 33 | stanza="myStanzaName", 34 | time="%.3f" % 1372187084.000 35 | ) 36 | 37 | **Example with full configuration**:: 38 | 39 | excellent_event = Event( 40 | data="This is a test of my excellent event.", 41 | stanza="excellenceOnly", 42 | time="%.3f" % 1372274622.493, 43 | host="localhost", 44 | index="main", 45 | source="Splunk", 46 | sourcetype="misc", 47 | done=True, 48 | unbroken=True 49 | ) 50 | 51 | :param data: ``string``, the event's text. 52 | :param stanza: ``string``, name of the input this event should be sent to. 53 | :param time: ``float``, time in seconds, including up to 3 decimal places to represent milliseconds. 54 | :param host: ``string``, the event's host, ex: localhost. 55 | :param index: ``string``, the index this event is specified to write to, or None if default index. 56 | :param source: ``string``, the source of this event, or None to have Splunk guess. 57 | :param sourcetype: ``string``, source type currently set on this event, or None to have Splunk guess. 58 | :param done: ``boolean``, is this a complete ``Event``? False if an ``Event`` fragment. 59 | :param unbroken: ``boolean``, Is this event completely encapsulated in this ``Event`` object? 60 | """ 61 | self.data = data 62 | self.done = done 63 | self.host = host 64 | self.index = index 65 | self.source = source 66 | self.sourceType = sourcetype 67 | self.stanza = stanza 68 | self.time = time 69 | self.unbroken = unbroken 70 | 71 | def write_to(self, stream): 72 | """Write an XML representation of self, an ``Event`` object, to the given stream. 73 | 74 | The ``Event`` object will only be written if its data field is defined, 75 | otherwise a ``ValueError`` is raised. 76 | 77 | :param stream: stream to write XML to. 78 | """ 79 | if self.data is None: 80 | raise ValueError("Events must have at least the data field set to be written to XML.") 81 | 82 | event = ET.Element("event") 83 | if self.stanza is not None: 84 | event.set("stanza", self.stanza) 85 | event.set("unbroken", str(int(self.unbroken))) 86 | 87 | # if a time isn't set, let Splunk guess by not creating a