├── .gitignore
├── LICENSE
├── README.md
└── search-elasticsearch
├── bin
├── config.template
├── elasticsearch
│ ├── __init__.py
│ ├── client
│ │ ├── __init__.py
│ │ ├── cat.py
│ │ ├── cluster.py
│ │ ├── indices.py
│ │ ├── nodes.py
│ │ ├── snapshot.py
│ │ └── utils.py
│ ├── compat.py
│ ├── connection
│ │ ├── __init__.py
│ │ ├── base.py
│ │ ├── esthrift
│ │ │ ├── Rest.py
│ │ │ ├── __init__.py
│ │ │ ├── constants.py
│ │ │ └── ttypes.py
│ │ ├── http_requests.py
│ │ ├── http_urllib3.py
│ │ ├── memcached.py
│ │ ├── pooling.py
│ │ └── thrift.py
│ ├── connection_pool.py
│ ├── exceptions.py
│ ├── helpers
│ │ ├── __init__.py
│ │ └── test.py
│ ├── serializer.py
│ └── transport.py
├── essearch.py
├── search.py
├── splunklib
│ ├── __init__.py
│ ├── binding.py
│ ├── client.py
│ ├── data.py
│ ├── modularinput
│ │ ├── __init__.py
│ │ ├── argument.py
│ │ ├── event.py
│ │ ├── event_writer.py
│ │ ├── input_definition.py
│ │ ├── scheme.py
│ │ ├── script.py
│ │ ├── utils.py
│ │ └── validation_definition.py
│ ├── ordereddict.py
│ ├── results.py
│ └── searchcommands
│ │ ├── __init__.py
│ │ ├── decorators.py
│ │ ├── generating_command.py
│ │ ├── logging.py
│ │ ├── reporting_command.py
│ │ ├── search_command.py
│ │ ├── search_command_internals.py
│ │ ├── splunk_csv
│ │ ├── __init__.py
│ │ ├── dialect.py
│ │ ├── dict_reader.py
│ │ └── dict_writer.py
│ │ ├── streaming_command.py
│ │ └── validators.py
└── urllib3
│ ├── __init__.py
│ ├── _collections.py
│ ├── connection.py
│ ├── connectionpool.py
│ ├── contrib
│ ├── __init__.py
│ ├── ntlmpool.py
│ └── pyopenssl.py
│ ├── exceptions.py
│ ├── fields.py
│ ├── filepost.py
│ ├── packages
│ ├── __init__.py
│ ├── ordered_dict.py
│ ├── six.py
│ └── ssl_match_hostname
│ │ ├── __init__.py
│ │ └── _implementation.py
│ ├── poolmanager.py
│ ├── request.py
│ ├── response.py
│ └── util
│ ├── __init__.py
│ ├── connection.py
│ ├── request.py
│ ├── response.py
│ ├── retry.py
│ ├── ssl_.py
│ ├── timeout.py
│ └── url.py
├── default
├── app.conf
├── commands.conf
├── logging.conf
└── searchbnf.conf
├── metadata
└── default.meta
└── other
└── README.md
/.gitignore:
--------------------------------------------------------------------------------
1 | config.json
2 | *.pyc
3 | *.swp
4 | *.idea
5 | *.DS_Store*
6 | *coverage_html_report*
7 | .coverage
8 | .coverage.*
9 | __stdout__
10 | docs/_build
11 | build/
12 | proxypid
13 | proxy.log
14 | MANIFEST
15 | coverage_report
16 | test.log
17 | examples/*/local
18 | examples/*/metadata
19 | tests/searchcommands_data/log/
20 | tests/searchcommands_data/output/
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | splunk-elasticsearch
2 | ====================
3 |
4 | I have created a search command for Splunk which will allow you to search Elastic Search and display the results in the Splunk GUI
5 |
6 | This project is now a valid splunk application and installs as you would any other splunk applications
7 | Steps
8 | Install python if it is not installed
9 |
10 | Install ElasticSearch https://github.com/elasticsearch/elasticsearch-py
11 | "pip install elasticsearch "
12 |
13 | This project is now a Splunk Application so just copy the splunk-elasticsearch directory to your splunk $SPLUNK_HOME/etc/apps directory and should work
14 |
15 |
16 | ======================================================
17 | git clone "This Project"
18 | rsync -av splunk-elasticsearch $SPLUNK_HOME/etc/apps
19 |
20 | Now you should be able to do a simple search like
21 | | esearch | top message
22 |
23 | or
24 | | esearch oldest=now-100d earliest=now query="some text" index=nagios* limit=1000 field=message
25 |
26 | ================================================
27 |
28 | command reference:
29 | esearch
30 | oldest = default (now-1d) uses elasticsearch timedate value or function
31 | earliest = default (now) uses elasticsearch timedate value or function
32 | index = default (*) sepecify the elasticsearch index to search
33 | limit = default (50) number of records to return
34 | field = default ("message") which elasticsearch field to query and return the value
35 | query = default ("*" | might change this to match_all) the elasticsearch query_string
36 |
37 |
38 |
39 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/config.template:
--------------------------------------------------------------------------------
1 | {
2 | "consumer-key": "PUT CONSUMER KEY HERE",
3 | "consumer-secret": "PUT CONSUMER SECRET HERE",
4 | "token": "PUT TOKEN HERE",
5 | "token-secret": "PUT TOKEN_SECRET HERE"
6 | }
--------------------------------------------------------------------------------
/search-elasticsearch/bin/elasticsearch/__init__.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 |
3 | VERSION = (1, 4, 0)
4 | __version__ = VERSION
5 | __versionstr__ = '.'.join(map(str, VERSION))
6 |
7 | import sys
8 |
9 | if (2, 7) <= sys.version_info < (3, 2):
10 | # On Python 2.7 and Python3 < 3.2, install no-op handler to silence
11 | # `No handlers could be found for logger "elasticsearch"` message per
12 | #
13 | import logging
14 | logger = logging.getLogger('elasticsearch')
15 | logger.addHandler(logging.NullHandler())
16 |
17 | from .client import Elasticsearch
18 | from .transport import Transport
19 | from .connection_pool import ConnectionPool, ConnectionSelector, \
20 | RoundRobinSelector
21 | from .serializer import JSONSerializer
22 | from .connection import Connection, RequestsHttpConnection, \
23 | Urllib3HttpConnection, MemcachedConnection, ThriftConnection
24 | from .exceptions import *
25 |
26 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/elasticsearch/client/cluster.py:
--------------------------------------------------------------------------------
1 | from .utils import NamespacedClient, query_params, _make_path
2 |
3 | class ClusterClient(NamespacedClient):
4 | @query_params('level', 'local', 'master_timeout', 'timeout',
5 | 'wait_for_active_shards', 'wait_for_nodes', 'wait_for_relocating_shards',
6 | 'wait_for_status')
7 | def health(self, index=None, params=None):
8 | """
9 | Get a very simple status on the health of the cluster.
10 | ``_
11 |
12 | :arg index: Limit the information returned to a specific index
13 | :arg level: Specify the level of detail for returned information, default u'cluster'
14 | :arg local: Return local information, do not retrieve the state from master node (default: false)
15 | :arg master_timeout: Explicit operation timeout for connection to master node
16 | :arg timeout: Explicit operation timeout
17 | :arg wait_for_active_shards: Wait until the specified number of shards is active
18 | :arg wait_for_nodes: Wait until the specified number of nodes is available
19 | :arg wait_for_relocating_shards: Wait until the specified number of relocating shards is finished
20 | :arg wait_for_status: Wait until cluster is in a specific state, default None
21 | """
22 | _, data = self.transport.perform_request('GET', _make_path('_cluster', 'health', index),
23 | params=params)
24 | return data
25 |
26 | @query_params('local', 'master_timeout')
27 | def pending_tasks(self, params=None):
28 | """
29 | The pending cluster tasks API returns a list of any cluster-level
30 | changes (e.g. create index, update mapping, allocate or fail shard)
31 | which have not yet been executed.
32 | ``_
33 |
34 | :arg local: Return local information, do not retrieve the state from master node (default: false)
35 | :arg master_timeout: Specify timeout for connection to master
36 | """
37 | _, data = self.transport.perform_request('GET', '/_cluster/pending_tasks',
38 | params=params)
39 | return data
40 |
41 | @query_params('allow_no_indices', 'expand_wildcards', 'flat_settings',
42 | 'ignore_unavailable', 'local', 'master_timeout')
43 | def state(self, metric=None, index=None, params=None):
44 | """
45 | Get a comprehensive state information of the whole cluster.
46 | ``_
47 |
48 | :arg metric: Limit the information returned to the specified metrics.
49 | Possible values: "_all", "blocks", "index_templates", "metadata",
50 | "nodes", "routing_table", "master_node", "version"
51 | :arg index: A comma-separated list of index names; use `_all` or empty
52 | string to perform the operation on all indices
53 | :arg allow_no_indices: Whether to ignore if a wildcard indices
54 | expression resolves into no concrete indices. (This includes `_all`
55 | string or when no indices have been specified)
56 | :arg expand_wildcards: Whether wildcard expressions should get expanded
57 | to open or closed indices (default: open)
58 | :arg flat_settings: Return settings in flat format (default: false)
59 | :arg ignore_unavailable: Whether specified concrete indices should be
60 | ignored when unavailable (missing or closed)
61 | :arg local: Return local information, do not retrieve the state from
62 | master node (default: false)
63 | :arg master_timeout: Specify timeout for connection to master
64 | """
65 | if index and not metric:
66 | metric = '_all'
67 | _, data = self.transport.perform_request('GET', _make_path('_cluster', 'state', metric, index), params=params)
68 | return data
69 |
70 | @query_params('flat_settings', 'human')
71 | def stats(self, node_id=None, params=None):
72 | """
73 | The Cluster Stats API allows to retrieve statistics from a cluster wide
74 | perspective. The API returns basic index metrics and information about
75 | the current nodes that form the cluster.
76 | ``_
77 |
78 | :arg node_id: A comma-separated list of node IDs or names to limit the
79 | returned information; use `_local` to return information from the node
80 | you're connecting to, leave empty to get information from all nodes
81 | :arg flat_settings: Return settings in flat format (default: false)
82 | :arg human: Whether to return time and byte values in human-readable format.
83 |
84 | """
85 | url = '/_cluster/stats'
86 | if node_id:
87 | url = _make_path('_cluster/stats/nodes', node_id)
88 | _, data = self.transport.perform_request('GET', url, params=params)
89 | return data
90 |
91 | @query_params('dry_run', 'explain', 'master_timeout', 'metric', 'timeout')
92 | def reroute(self, body=None, params=None):
93 | """
94 | Explicitly execute a cluster reroute allocation command including specific commands.
95 | ``_
96 |
97 | :arg body: The definition of `commands` to perform (`move`, `cancel`, `allocate`)
98 | :arg dry_run: Simulate the operation only and return the resulting state
99 | :arg explain: Return an explanation of why the commands can or cannot be executed
100 | :arg filter_metadata: Don't return cluster state metadata (default: false)
101 | :arg master_timeout: Explicit operation timeout for connection to master node
102 | :arg metric: Limit the information returned to the specified metrics.
103 | Defaults to all but metadata
104 | :arg timeout: Explicit operation timeout
105 | """
106 | _, data = self.transport.perform_request('POST', '/_cluster/reroute', params=params, body=body)
107 | return data
108 |
109 | @query_params('flat_settings', 'master_timeout', 'timeout')
110 | def get_settings(self, params=None):
111 | """
112 | Get cluster settings.
113 | ``_
114 |
115 | :arg flat_settings: Return settings in flat format (default: false)
116 | :arg master_timeout: Explicit operation timeout for connection to master node
117 | :arg timeout: Explicit operation timeout
118 | """
119 | _, data = self.transport.perform_request('GET', '/_cluster/settings', params=params)
120 | return data
121 |
122 | @query_params('flat_settings', 'master_timeout', 'timeout')
123 | def put_settings(self, body, params=None):
124 | """
125 | Update cluster wide specific settings.
126 | ``_
127 |
128 | :arg body: The settings to be updated. Can be either `transient` or
129 | `persistent` (survives cluster restart).
130 | :arg flat_settings: Return settings in flat format (default: false)
131 | :arg master_timeout: Explicit operation timeout for connection to master
132 | node
133 | :arg timeout: Explicit operation timeout
134 | """
135 | _, data = self.transport.perform_request('PUT', '/_cluster/settings', params=params, body=body)
136 | return data
137 |
138 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/elasticsearch/client/nodes.py:
--------------------------------------------------------------------------------
1 | from .utils import NamespacedClient, query_params, _make_path
2 |
3 | class NodesClient(NamespacedClient):
4 | @query_params('flat_settings', 'human')
5 | def info(self, node_id=None, metric=None, params=None):
6 | """
7 | The cluster nodes info API allows to retrieve one or more (or all) of
8 | the cluster nodes information.
9 | ``_
10 |
11 | :arg node_id: A comma-separated list of node IDs or names to limit the
12 | returned information; use `_local` to return information from the
13 | node you're connecting to, leave empty to get information from all
14 | nodes
15 | :arg metric: A comma-separated list of metrics you wish returned. Leave
16 | empty to return all. Choices are "settings", "os", "process",
17 | "jvm", "thread_pool", "network", "transport", "http", "plugin"
18 | :arg flat_settings: Return settings in flat format (default: false)
19 | :arg human: Whether to return time and byte values in human-readable
20 | format., default False
21 | """
22 | _, data = self.transport.perform_request('GET', _make_path('_nodes',
23 | node_id, metric), params=params)
24 | return data
25 |
26 | @query_params('delay', 'exit')
27 | def shutdown(self, node_id=None, params=None):
28 | """
29 | The nodes shutdown API allows to shutdown one or more (or all) nodes in
30 | the cluster.
31 | ``_
32 |
33 | :arg node_id: A comma-separated list of node IDs or names to perform the
34 | operation on; use `_local` to perform the operation on the node
35 | you're connected to, leave empty to perform the operation on all
36 | nodes
37 | :arg delay: Set the delay for the operation (default: 1s)
38 | :arg exit: Exit the JVM as well (default: true)
39 | """
40 | _, data = self.transport.perform_request('POST', _make_path('_cluster',
41 | 'nodes', node_id, '_shutdown'), params=params)
42 | return data
43 |
44 | @query_params('completion_fields', 'fielddata_fields', 'fields', 'groups', 'human', 'level', 'types')
45 | def stats(self, node_id=None, metric=None, index_metric=None, params=None):
46 | """
47 | The cluster nodes stats API allows to retrieve one or more (or all) of
48 | the cluster nodes statistics.
49 | ``_
50 |
51 | :arg node_id: A comma-separated list of node IDs or names to limit the
52 | returned information; use `_local` to return information from the
53 | node you're connecting to, leave empty to get information from all
54 | nodes
55 | :arg metric: Limit the information returned to the specified metrics.
56 | Possible options are: "_all", "breaker", "fs", "http", "indices",
57 | "jvm", "network", "os", "process", "thread_pool", "transport"
58 | :arg index_metric: Limit the information returned for `indices` metric
59 | to the specific index metrics. Isn't used if `indices` (or `all`)
60 | metric isn't specified. Possible options are: "_all", "completion",
61 | "docs", "fielddata", "filter_cache", "flush", "get", "id_cache",
62 | "indexing", "merge", "percolate", "refresh", "search", "segments",
63 | "store", "warmer"
64 | :arg completion_fields: A comma-separated list of fields for `fielddata`
65 | and `suggest` index metric (supports wildcards)
66 | :arg fielddata_fields: A comma-separated list of fields for `fielddata`
67 | index metric (supports wildcards)
68 | :arg fields: A comma-separated list of fields for `fielddata` and
69 | `completion` index metric (supports wildcards)
70 | :arg groups: A comma-separated list of search groups for `search` index
71 | metric
72 | :arg human: Whether to return time and byte values in human-readable
73 | format., default False
74 | :arg level: Return indices stats aggregated at node, index or shard
75 | level, default 'node'
76 | :arg types: A comma-separated list of document types for the `indexing`
77 | index metric
78 | """
79 | _, data = self.transport.perform_request('GET', _make_path('_nodes',
80 | node_id, 'stats', metric, index_metric), params=params)
81 | return data
82 |
83 | @query_params('type_', 'interval', 'snapshots', 'threads')
84 | def hot_threads(self, node_id=None, params=None):
85 | """
86 | An API allowing to get the current hot threads on each node in the cluster.
87 | ``_
88 |
89 | :arg node_id: A comma-separated list of node IDs or names to limit the
90 | returned information; use `_local` to return information from the
91 | node you're connecting to, leave empty to get information from all
92 | nodes
93 | :arg type_: The type to sample (default: cpu)
94 | :arg interval: The interval for the second sampling of threads
95 | :arg snapshots: Number of samples of thread stacktrace (default: 10)
96 | :arg threads: Specify the number of threads to provide information for
97 | (default: 3)
98 | """
99 | # avoid python reserved words
100 | if params and 'type_' in params:
101 | params['type'] = params.pop('type_')
102 | _, data = self.transport.perform_request('GET', _make_path('_nodes',
103 | node_id, 'hot_threads'), params=params)
104 | return data
105 |
106 |
107 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/elasticsearch/client/snapshot.py:
--------------------------------------------------------------------------------
1 | from .utils import NamespacedClient, query_params, _make_path, SKIP_IN_PATH
2 |
3 | class SnapshotClient(NamespacedClient):
4 | @query_params('master_timeout', 'wait_for_completion')
5 | def create(self, repository, snapshot, body=None, params=None):
6 | """
7 | Create a snapshot in repository
8 | ``_
9 |
10 | :arg repository: A repository name
11 | :arg snapshot: A snapshot name
12 | :arg body: The snapshot definition
13 | :arg master_timeout: Explicit operation timeout for connection to master
14 | node
15 | :arg wait_for_completion: Should this request wait until the operation
16 | has completed before returning, default False
17 | """
18 | for param in (repository, snapshot):
19 | if param in SKIP_IN_PATH:
20 | raise ValueError("Empty value passed for a required argument.")
21 | _, data = self.transport.perform_request('PUT', _make_path('_snapshot',
22 | repository, snapshot), params=params, body=body)
23 | return data
24 |
25 | @query_params('master_timeout')
26 | def delete(self, repository, snapshot, params=None):
27 | """
28 | Deletes a snapshot from a repository.
29 | ``_
30 |
31 | :arg repository: A repository name
32 | :arg snapshot: A snapshot name
33 | :arg master_timeout: Explicit operation timeout for connection to master
34 | node
35 | """
36 | for param in (repository, snapshot):
37 | if param in SKIP_IN_PATH:
38 | raise ValueError("Empty value passed for a required argument.")
39 | _, data = self.transport.perform_request('DELETE',
40 | _make_path('_snapshot', repository, snapshot), params=params)
41 | return data
42 |
43 | @query_params('master_timeout')
44 | def get(self, repository, snapshot, params=None):
45 | """
46 | Retrieve information about a snapshot.
47 | ``_
48 |
49 | :arg repository: A comma-separated list of repository names
50 | :arg snapshot: A comma-separated list of snapshot names
51 | :arg master_timeout: Explicit operation timeout for connection to master
52 | node
53 | """
54 | for param in (repository, snapshot):
55 | if param in SKIP_IN_PATH:
56 | raise ValueError("Empty value passed for a required argument.")
57 | _, data = self.transport.perform_request('GET', _make_path('_snapshot',
58 | repository, snapshot), params=params)
59 | return data
60 |
61 | @query_params('master_timeout', 'timeout')
62 | def delete_repository(self, repository, params=None):
63 | """
64 | Removes a shared file system repository.
65 | ``_
66 |
67 | :arg repository: A comma-separated list of repository names
68 | :arg master_timeout: Explicit operation timeout for connection to master
69 | node
70 | :arg timeout: Explicit operation timeout
71 | """
72 | if repository in SKIP_IN_PATH:
73 | raise ValueError("Empty value passed for a required argument 'repository'.")
74 | _, data = self.transport.perform_request('DELETE',
75 | _make_path('_snapshot', repository), params=params)
76 | return data
77 |
78 | @query_params('local', 'master_timeout')
79 | def get_repository(self, repository=None, params=None):
80 | """
81 | Return information about registered repositories.
82 | ``_
83 |
84 | :arg repository: A comma-separated list of repository names
85 | :arg master_timeout: Explicit operation timeout for connection to master
86 | node
87 | :arg local: Return local information, do not retrieve the state from
88 | master node (default: false)
89 | """
90 | _, data = self.transport.perform_request('GET', _make_path('_snapshot',
91 | repository), params=params)
92 | return data
93 |
94 | @query_params('master_timeout', 'timeout')
95 | def create_repository(self, repository, body, params=None):
96 | """
97 | Registers a shared file system repository.
98 | ``_
99 |
100 | :arg repository: A repository name
101 | :arg body: The repository definition
102 | :arg master_timeout: Explicit operation timeout for connection to master
103 | node
104 | :arg timeout: Explicit operation timeout
105 | """
106 | for param in (repository, body):
107 | if param in SKIP_IN_PATH:
108 | raise ValueError("Empty value passed for a required argument.")
109 | _, data = self.transport.perform_request('PUT', _make_path('_snapshot',
110 | repository), params=params, body=body)
111 | return data
112 |
113 | @query_params('master_timeout', 'wait_for_completion')
114 | def restore(self, repository, snapshot, body=None, params=None):
115 | """
116 | Restore a snapshot.
117 | ``_
118 |
119 | :arg repository: A repository name
120 | :arg snapshot: A snapshot name
121 | :arg body: Details of what to restore
122 | :arg master_timeout: Explicit operation timeout for connection to master
123 | node
124 | :arg wait_for_completion: Should this request wait until the operation
125 | has completed before returning, default False
126 | """
127 | for param in (repository, snapshot):
128 | if param in SKIP_IN_PATH:
129 | raise ValueError("Empty value passed for a required argument.")
130 | _, data = self.transport.perform_request('POST', _make_path('_snapshot',
131 | repository, snapshot, '_restore'), params=params, body=body)
132 | return data
133 |
134 | @query_params('master_timeout')
135 | def status(self, repository=None, snapshot=None, params=None):
136 | """
137 | Return information about all currently running snapshots. By specifying
138 | a repository name, it's possible to limit the results to a particular
139 | repository.
140 | ``_
141 |
142 | :arg repository: A repository name
143 | :arg snapshot: A comma-separated list of snapshot names
144 | :arg master_timeout: Explicit operation timeout for connection to master
145 | node
146 | """
147 | _, data = self.transport.perform_request('GET', _make_path('_snapshot',
148 | repository, snapshot, '_status'), params=params)
149 | return data
150 |
151 | @query_params('master_timeout', 'timeout')
152 | def verify_repository(self, repository, params=None):
153 | """
154 | Returns a list of nodes where repository was successfully verified or
155 | an error message if verification process failed.
156 | ``_
157 |
158 | :arg repository: A repository name
159 | :arg master_timeout: Explicit operation timeout for connection to master
160 | node
161 | :arg timeout: Explicit operation timeout
162 | """
163 | if repository in SKIP_IN_PATH:
164 | raise ValueError("Empty value passed for a required argument 'repository'.")
165 | _, data = self.transport.perform_request('POST', _make_path('_snapshot',
166 | repository, '_verify'), params=params)
167 | return data
168 |
169 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/elasticsearch/client/utils.py:
--------------------------------------------------------------------------------
1 | from __future__ import unicode_literals
2 |
3 | from datetime import date, datetime
4 | from functools import wraps
5 | from ..compat import string_types, quote_plus
6 |
7 | # parts of URL to be omitted
8 | SKIP_IN_PATH = (None, '', b'', [], ())
9 |
10 | def _escape(value):
11 | """
12 | Escape a single value of a URL string or a query parameter. If it is a list
13 | or tuple, turn it into a comma-separated string first.
14 | """
15 |
16 | # make sequences into comma-separated stings
17 | if isinstance(value, (list, tuple)):
18 | value = ','.join(value)
19 |
20 | # dates and datetimes into isoformat
21 | elif isinstance(value, (date, datetime)):
22 | value = value.isoformat()
23 |
24 | # make bools into true/false strings
25 | elif isinstance(value, bool):
26 | value = str(value).lower()
27 |
28 | # encode strings to utf-8
29 | if isinstance(value, string_types):
30 | try:
31 | return value.encode('utf-8')
32 | except UnicodeDecodeError:
33 | # Python 2 and str, no need to re-encode
34 | pass
35 |
36 | return str(value)
37 |
38 | def _make_path(*parts):
39 | """
40 | Create a URL string from parts, omit all `None` values and empty strings.
41 | Convert lists nad tuples to comma separated values.
42 | """
43 | #TODO: maybe only allow some parts to be lists/tuples ?
44 | return '/' + '/'.join(
45 | # preserve ',' and '*' in url for nicer URLs in logs
46 | quote_plus(_escape(p), b',*') for p in parts if p not in SKIP_IN_PATH)
47 |
48 | # parameters that apply to all methods
49 | GLOBAL_PARAMS = ('pretty', 'format')
50 |
51 | def query_params(*es_query_params):
52 | """
53 | Decorator that pops all accepted parameters from method's kwargs and puts
54 | them in the params argument.
55 | """
56 | def _wrapper(func):
57 | @wraps(func)
58 | def _wrapped(*args, **kwargs):
59 | params = kwargs.pop('params', {})
60 | for p in es_query_params + GLOBAL_PARAMS:
61 | if p in kwargs:
62 | params[p] = _escape(kwargs.pop(p))
63 |
64 | # don't treat ignore and request_timeout as other params to avoid escaping
65 | for p in ('ignore', 'request_timeout'):
66 | if p in kwargs:
67 | params[p] = kwargs.pop(p)
68 | return func(*args, params=params, **kwargs)
69 | return _wrapped
70 | return _wrapper
71 |
72 |
73 | class NamespacedClient(object):
74 | def __init__(self, client):
75 | self.client = client
76 |
77 | @property
78 | def transport(self):
79 | return self.client.transport
80 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/elasticsearch/compat.py:
--------------------------------------------------------------------------------
1 | import sys
2 |
3 | PY2 = sys.version_info[0] == 2
4 |
5 | if PY2:
6 | string_types = basestring,
7 | from urllib import quote_plus, urlencode
8 | from urlparse import urlparse
9 | from itertools import imap as map
10 | else:
11 | string_types = str, bytes
12 | from urllib.parse import quote_plus, urlencode, urlparse
13 | map = map
14 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/elasticsearch/connection/__init__.py:
--------------------------------------------------------------------------------
1 | from .base import Connection
2 | from .http_requests import RequestsHttpConnection
3 | from .http_urllib3 import Urllib3HttpConnection
4 | from .memcached import MemcachedConnection
5 | from .thrift import ThriftConnection, THRIFT_AVAILABLE
6 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/elasticsearch/connection/base.py:
--------------------------------------------------------------------------------
1 | import logging
2 | try:
3 | import simplejson as json
4 | except ImportError:
5 | import json
6 |
7 | from ..exceptions import TransportError, HTTP_EXCEPTIONS
8 |
9 | logger = logging.getLogger('elasticsearch')
10 |
11 | # create the elasticsearch.trace logger, but only set propagate to False if the
12 | # logger hasn't already been configured
13 | _tracer_already_configured = 'elasticsearch.trace' in logging.Logger.manager.loggerDict
14 | tracer = logging.getLogger('elasticsearch.trace')
15 | tracer.propagate = not _tracer_already_configured
16 |
17 |
18 | class Connection(object):
19 | """
20 | Class responsible for maintaining a connection to an Elasticsearch node. It
21 | holds persistent connection pool to it and it's main interface
22 | (`perform_request`) is thread-safe.
23 |
24 | Also responsible for logging.
25 | """
26 | transport_schema = 'http'
27 |
28 | def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, **kwargs):
29 | """
30 | :arg host: hostname of the node (default: localhost)
31 | :arg port: port to use (integer, default: 9200)
32 | :arg url_prefix: optional url prefix for elasticsearch
33 | :arg timeout: default timeout in seconds (float, default: 10)
34 | """
35 | self.host = '%s://%s:%s' % (self.transport_schema, host, port)
36 | if url_prefix:
37 | url_prefix = '/' + url_prefix.strip('/')
38 | self.url_prefix = url_prefix
39 | self.timeout = timeout
40 |
41 | def __repr__(self):
42 | return '<%s: %s>' % (self.__class__.__name__, self.host)
43 |
44 | def log_request_success(self, method, full_url, path, body, status_code, response, duration):
45 | """ Log a successful API call. """
46 | # TODO: optionally pass in params instead of full_url and do urlencode only when needed
47 | def _pretty_json(data):
48 | # pretty JSON in tracer curl logs
49 | try:
50 | return json.dumps(json.loads(data), sort_keys=True, indent=2, separators=(',', ': ')).replace("'", r'\u0027')
51 | except (ValueError, TypeError):
52 | # non-json data or a bulk request
53 | return data
54 |
55 | # body has already been serialized to utf-8, deserialize it for logging
56 | # TODO: find a better way to avoid (de)encoding the body back and forth
57 | if body:
58 | body = body.decode('utf-8')
59 |
60 | logger.info(
61 | '%s %s [status:%s request:%.3fs]', method, full_url,
62 | status_code, duration
63 | )
64 | logger.debug('> %s', body)
65 | logger.debug('< %s', response)
66 |
67 | if tracer.isEnabledFor(logging.INFO):
68 | # include pretty in trace curls
69 | path = path.replace('?', '?pretty&', 1) if '?' in path else path + '?pretty'
70 | if self.url_prefix:
71 | path = path.replace(self.url_prefix, '', 1)
72 | tracer.info("curl -X%s 'http://localhost:9200%s' -d '%s'", method, path, _pretty_json(body) if body else '')
73 |
74 | if tracer.isEnabledFor(logging.DEBUG):
75 | tracer.debug('#[%s] (%.3fs)\n#%s', status_code, duration, _pretty_json(response).replace('\n', '\n#') if response else '')
76 |
77 | def log_request_fail(self, method, full_url, body, duration, status_code=None, exception=None):
78 | """ Log an unsuccessful API call. """
79 | logger.warning(
80 | '%s %s [status:%s request:%.3fs]', method, full_url,
81 | status_code or 'N/A', duration, exc_info=exception is not None
82 | )
83 |
84 | # body has already been serialized to utf-8, deserialize it for logging
85 | # TODO: find a better way to avoid (de)encoding the body back and forth
86 | if body:
87 | body = body.decode('utf-8')
88 |
89 | logger.debug('> %s', body)
90 |
91 | def _raise_error(self, status_code, raw_data):
92 | """ Locate appropriate exception and raise it. """
93 | error_message = raw_data
94 | additional_info = None
95 | try:
96 | additional_info = json.loads(raw_data)
97 | error_message = additional_info.get('error', error_message)
98 | except:
99 | # we don't care what went wrong
100 | pass
101 |
102 | raise HTTP_EXCEPTIONS.get(status_code, TransportError)(status_code, error_message, additional_info)
103 |
104 |
105 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/elasticsearch/connection/esthrift/Rest.py:
--------------------------------------------------------------------------------
1 | #
2 | # Autogenerated by Thrift Compiler (0.9.0)
3 | #
4 | # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
5 | #
6 | # options string: py:new_style=true,utf8strings=true
7 | #
8 |
9 | from thrift.Thrift import TType, TMessageType, TException, TApplicationException
10 | from ttypes import *
11 | from thrift.Thrift import TProcessor
12 | from thrift.transport import TTransport
13 | from thrift.protocol import TBinaryProtocol, TProtocol
14 | try:
15 | from thrift.protocol import fastbinary
16 | except:
17 | fastbinary = None
18 |
19 |
20 | class Iface(object):
21 | def execute(self, request):
22 | """
23 | Parameters:
24 | - request
25 | """
26 | pass
27 |
28 |
29 | class Client(Iface):
30 | def __init__(self, iprot, oprot=None):
31 | self._iprot = self._oprot = iprot
32 | if oprot is not None:
33 | self._oprot = oprot
34 | self._seqid = 0
35 |
36 | def execute(self, request):
37 | """
38 | Parameters:
39 | - request
40 | """
41 | self.send_execute(request)
42 | return self.recv_execute()
43 |
44 | def send_execute(self, request):
45 | self._oprot.writeMessageBegin('execute', TMessageType.CALL, self._seqid)
46 | args = execute_args()
47 | args.request = request
48 | args.write(self._oprot)
49 | self._oprot.writeMessageEnd()
50 | self._oprot.trans.flush()
51 |
52 | def recv_execute(self, ):
53 | (fname, mtype, rseqid) = self._iprot.readMessageBegin()
54 | if mtype == TMessageType.EXCEPTION:
55 | x = TApplicationException()
56 | x.read(self._iprot)
57 | self._iprot.readMessageEnd()
58 | raise x
59 | result = execute_result()
60 | result.read(self._iprot)
61 | self._iprot.readMessageEnd()
62 | if result.success is not None:
63 | return result.success
64 | raise TApplicationException(TApplicationException.MISSING_RESULT, "execute failed: unknown result");
65 |
66 |
67 | class Processor(Iface, TProcessor):
68 | def __init__(self, handler):
69 | self._handler = handler
70 | self._processMap = {}
71 | self._processMap["execute"] = Processor.process_execute
72 |
73 | def process(self, iprot, oprot):
74 | (name, type, seqid) = iprot.readMessageBegin()
75 | if name not in self._processMap:
76 | iprot.skip(TType.STRUCT)
77 | iprot.readMessageEnd()
78 | x = TApplicationException(TApplicationException.UNKNOWN_METHOD, 'Unknown function %s' % (name))
79 | oprot.writeMessageBegin(name, TMessageType.EXCEPTION, seqid)
80 | x.write(oprot)
81 | oprot.writeMessageEnd()
82 | oprot.trans.flush()
83 | return
84 | else:
85 | self._processMap[name](self, seqid, iprot, oprot)
86 | return True
87 |
88 | def process_execute(self, seqid, iprot, oprot):
89 | args = execute_args()
90 | args.read(iprot)
91 | iprot.readMessageEnd()
92 | result = execute_result()
93 | result.success = self._handler.execute(args.request)
94 | oprot.writeMessageBegin("execute", TMessageType.REPLY, seqid)
95 | result.write(oprot)
96 | oprot.writeMessageEnd()
97 | oprot.trans.flush()
98 |
99 |
100 | # HELPER FUNCTIONS AND STRUCTURES
101 |
102 | class execute_args(object):
103 | """
104 | Attributes:
105 | - request
106 | """
107 |
108 | thrift_spec = (
109 | None, # 0
110 | (1, TType.STRUCT, 'request', (RestRequest, RestRequest.thrift_spec), None, ), # 1
111 | )
112 |
113 | def __init__(self, request=None,):
114 | self.request = request
115 |
116 | def read(self, iprot):
117 | if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
118 | fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
119 | return
120 | iprot.readStructBegin()
121 | while True:
122 | (fname, ftype, fid) = iprot.readFieldBegin()
123 | if ftype == TType.STOP:
124 | break
125 | if fid == 1:
126 | if ftype == TType.STRUCT:
127 | self.request = RestRequest()
128 | self.request.read(iprot)
129 | else:
130 | iprot.skip(ftype)
131 | else:
132 | iprot.skip(ftype)
133 | iprot.readFieldEnd()
134 | iprot.readStructEnd()
135 |
136 | def write(self, oprot):
137 | if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
138 | oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
139 | return
140 | oprot.writeStructBegin('execute_args')
141 | if self.request is not None:
142 | oprot.writeFieldBegin('request', TType.STRUCT, 1)
143 | self.request.write(oprot)
144 | oprot.writeFieldEnd()
145 | oprot.writeFieldStop()
146 | oprot.writeStructEnd()
147 |
148 | def validate(self):
149 | if self.request is None:
150 | raise TProtocol.TProtocolException(message='Required field request is unset!')
151 | return
152 |
153 |
154 | def __repr__(self):
155 | L = ['%s=%r' % (key, value)
156 | for key, value in self.__dict__.iteritems()]
157 | return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
158 |
159 | def __eq__(self, other):
160 | return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
161 |
162 | def __ne__(self, other):
163 | return not (self == other)
164 |
165 | class execute_result(object):
166 | """
167 | Attributes:
168 | - success
169 | """
170 |
171 | thrift_spec = (
172 | (0, TType.STRUCT, 'success', (RestResponse, RestResponse.thrift_spec), None, ), # 0
173 | )
174 |
175 | def __init__(self, success=None,):
176 | self.success = success
177 |
178 | def read(self, iprot):
179 | if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None:
180 | fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec))
181 | return
182 | iprot.readStructBegin()
183 | while True:
184 | (fname, ftype, fid) = iprot.readFieldBegin()
185 | if ftype == TType.STOP:
186 | break
187 | if fid == 0:
188 | if ftype == TType.STRUCT:
189 | self.success = RestResponse()
190 | self.success.read(iprot)
191 | else:
192 | iprot.skip(ftype)
193 | else:
194 | iprot.skip(ftype)
195 | iprot.readFieldEnd()
196 | iprot.readStructEnd()
197 |
198 | def write(self, oprot):
199 | if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None:
200 | oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec)))
201 | return
202 | oprot.writeStructBegin('execute_result')
203 | if self.success is not None:
204 | oprot.writeFieldBegin('success', TType.STRUCT, 0)
205 | self.success.write(oprot)
206 | oprot.writeFieldEnd()
207 | oprot.writeFieldStop()
208 | oprot.writeStructEnd()
209 |
210 | def validate(self):
211 | return
212 |
213 |
214 | def __repr__(self):
215 | L = ['%s=%r' % (key, value)
216 | for key, value in self.__dict__.iteritems()]
217 | return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
218 |
219 | def __eq__(self, other):
220 | return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
221 |
222 | def __ne__(self, other):
223 | return not (self == other)
224 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/elasticsearch/connection/esthrift/__init__.py:
--------------------------------------------------------------------------------
1 | __all__ = ['ttypes', 'constants', 'Rest']
2 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/elasticsearch/connection/esthrift/constants.py:
--------------------------------------------------------------------------------
1 | #
2 | # Autogenerated by Thrift Compiler (0.9.0)
3 | #
4 | # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
5 | #
6 | # options string: py:new_style=true,utf8strings=true
7 | #
8 |
9 | from thrift.Thrift import TType, TMessageType, TException, TApplicationException
10 | from ttypes import *
11 |
12 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/elasticsearch/connection/http_requests.py:
--------------------------------------------------------------------------------
1 | import time
2 | import warnings
3 | try:
4 | import requests
5 | REQUESTS_AVAILABLE = True
6 | except ImportError:
7 | REQUESTS_AVAILABLE = False
8 |
9 | from .base import Connection
10 | from ..exceptions import ConnectionError, ImproperlyConfigured, ConnectionTimeout, SSLError
11 | from ..compat import urlencode
12 |
13 | class RequestsHttpConnection(Connection):
14 | """
15 | Connection using the `requests` library.
16 |
17 | :arg http_auth: optional http auth information as either ':' separated
18 | string or a tuple
19 | :arg use_ssl: use ssl for the connection if `True`
20 | :arg verify_certs: whether to verify SSL certificates
21 | :arg ca_certs: optional path to CA bundle. By default standard requests'
22 | bundle will be used.
23 | :arg client_cert: path to the file containing the private key and the
24 | certificate
25 | """
26 | def __init__(self, host='localhost', port=9200, http_auth=None,
27 | use_ssl=False, verify_certs=False, ca_certs=None, client_cert=None,
28 | **kwargs):
29 | if not REQUESTS_AVAILABLE:
30 | raise ImproperlyConfigured("Please install requests to use RequestsHttpConnection.")
31 |
32 | super(RequestsHttpConnection, self).__init__(host= host, port=port, **kwargs)
33 | self.session = requests.session()
34 | if http_auth is not None:
35 | if not isinstance(http_auth, (tuple, list)):
36 | http_auth = http_auth.split(':', 1)
37 | http_auth = tuple(http_auth)
38 | self.session.auth = http_auth
39 | self.base_url = 'http%s://%s:%d%s' % (
40 | 's' if use_ssl else '',
41 | host, port, self.url_prefix
42 | )
43 | self.session.verify = verify_certs
44 | self.session.cert = client_cert
45 | if ca_certs:
46 | if not verify_certs:
47 | raise ImproperlyConfigured("You cannot pass CA certificates when verify SSL is off.")
48 | self.session.verify = ca_certs
49 |
50 | if use_ssl and not verify_certs:
51 | warnings.warn(
52 | 'Connecting to %s using SSL with verify_certs=False is insecure.' % self.base_url)
53 |
54 | def perform_request(self, method, url, params=None, body=None, timeout=None, ignore=()):
55 | url = self.base_url + url
56 | if params:
57 | url = '%s?%s' % (url, urlencode(params or {}))
58 |
59 | start = time.time()
60 | try:
61 | response = self.session.request(method, url, data=body, timeout=timeout or self.timeout)
62 | duration = time.time() - start
63 | raw_data = response.text
64 | except requests.exceptions.SSLError as e:
65 | self.log_request_fail(method, url, body, time.time() - start, exception=e)
66 | raise SSLError('N/A', str(e), e)
67 | except requests.Timeout as e:
68 | self.log_request_fail(method, url, body, time.time() - start, exception=e)
69 | raise ConnectionTimeout('TIMEOUT', str(e), e)
70 | except requests.ConnectionError as e:
71 | self.log_request_fail(method, url, body, time.time() - start, exception=e)
72 | raise ConnectionError('N/A', str(e), e)
73 |
74 | # raise errors based on http status codes, let the client handle those if needed
75 | if not (200 <= response.status_code < 300) and response.status_code not in ignore:
76 | self.log_request_fail(method, url, body, duration, response.status_code)
77 | self._raise_error(response.status_code, raw_data)
78 |
79 | self.log_request_success(method, url, response.request.path_url, body, response.status_code, raw_data, duration)
80 |
81 | return response.status_code, response.headers, raw_data
82 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/elasticsearch/connection/http_urllib3.py:
--------------------------------------------------------------------------------
1 | import time
2 | import urllib3
3 | from urllib3.exceptions import ReadTimeoutError, SSLError as UrllibSSLError
4 | import warnings
5 |
6 | from .base import Connection
7 | from ..exceptions import ConnectionError, ImproperlyConfigured, ConnectionTimeout, SSLError
8 | from ..compat import urlencode
9 |
10 | class Urllib3HttpConnection(Connection):
11 | """
12 | Default connection class using the `urllib3` library and the http protocol.
13 |
14 | :arg http_auth: optional http auth information as either ':' separated
15 | string or a tuple
16 | :arg use_ssl: use ssl for the connection if `True`
17 | :arg verify_certs: whether to verify SSL certificates
18 | :arg ca_certs: optional path to CA bundle. See
19 | http://urllib3.readthedocs.org/en/latest/security.html#using-certifi-with-urllib3
20 | for instructions how to get default set
21 | :arg client_cert: path to the file containing the private key and the
22 | certificate
23 | :arg maxsize: the maximum number of connections which will be kept open to
24 | this host.
25 | """
26 | def __init__(self, host='localhost', port=9200, http_auth=None,
27 | use_ssl=False, verify_certs=False, ca_certs=None, client_cert=None,
28 | maxsize=10, **kwargs):
29 |
30 | super(Urllib3HttpConnection, self).__init__(host=host, port=port, **kwargs)
31 | self.headers = {}
32 | if http_auth is not None:
33 | if isinstance(http_auth, (tuple, list)):
34 | http_auth = ':'.join(http_auth)
35 | self.headers = urllib3.make_headers(basic_auth=http_auth)
36 |
37 | pool_class = urllib3.HTTPConnectionPool
38 | kw = {}
39 | if use_ssl:
40 | pool_class = urllib3.HTTPSConnectionPool
41 |
42 | if verify_certs:
43 | kw['cert_reqs'] = 'CERT_REQUIRED'
44 | kw['ca_certs'] = ca_certs
45 | kw['cert_file'] = client_cert
46 | elif ca_certs:
47 | raise ImproperlyConfigured("You cannot pass CA certificates when verify SSL is off.")
48 | else:
49 | warnings.warn(
50 | 'Connecting to %s using SSL with verify_certs=False is insecure.' % host)
51 |
52 | self.pool = pool_class(host, port=port, timeout=self.timeout, maxsize=maxsize, **kw)
53 |
54 | def perform_request(self, method, url, params=None, body=None, timeout=None, ignore=()):
55 | url = self.url_prefix + url
56 | if params:
57 | url = '%s?%s' % (url, urlencode(params))
58 | full_url = self.host + url
59 |
60 | start = time.time()
61 | try:
62 | kw = {}
63 | if timeout:
64 | kw['timeout'] = timeout
65 |
66 | # in python2 we need to make sure the url is not unicode. Otherwise
67 | # the body will be decoded into unicode too and that will fail (#133).
68 | if not isinstance(url, str):
69 | url = url.encode('utf-8')
70 |
71 | response = self.pool.urlopen(method, url, body, retries=False, headers=self.headers, **kw)
72 | duration = time.time() - start
73 | raw_data = response.data.decode('utf-8')
74 | except UrllibSSLError as e:
75 | self.log_request_fail(method, full_url, body, time.time() - start, exception=e)
76 | raise SSLError('N/A', str(e), e)
77 | except ReadTimeoutError as e:
78 | self.log_request_fail(method, full_url, body, time.time() - start, exception=e)
79 | raise ConnectionTimeout('TIMEOUT', str(e), e)
80 | except Exception as e:
81 | self.log_request_fail(method, full_url, body, time.time() - start, exception=e)
82 | raise ConnectionError('N/A', str(e), e)
83 |
84 | if not (200 <= response.status < 300) and response.status not in ignore:
85 | self.log_request_fail(method, url, body, duration, response.status)
86 | self._raise_error(response.status, raw_data)
87 |
88 | self.log_request_success(method, full_url, url, body, response.status,
89 | raw_data, duration)
90 |
91 | return response.status, response.getheaders(), raw_data
92 |
93 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/elasticsearch/connection/memcached.py:
--------------------------------------------------------------------------------
1 | import time
2 | try:
3 | import simplejson as json
4 | except ImportError:
5 | import json
6 |
7 | from ..exceptions import TransportError, ConnectionError, ImproperlyConfigured
8 | from ..compat import urlencode
9 | from .pooling import PoolingConnection
10 |
11 | class MemcachedConnection(PoolingConnection):
12 | """
13 | Client using the `pylibmc` python library to communicate with elasticsearch
14 | using the memcached protocol. Requires plugin in the cluster.
15 |
16 | See https://github.com/elasticsearch/elasticsearch-transport-memcached for more details.
17 | """
18 | transport_schema = 'memcached'
19 |
20 | method_map = {
21 | 'PUT': 'set',
22 | 'POST': 'set',
23 | 'DELETE': 'delete',
24 | 'HEAD': 'get',
25 | 'GET': 'get',
26 | }
27 |
28 | def __init__(self, host='localhost', port=11211, **kwargs):
29 | try:
30 | import pylibmc
31 | except ImportError:
32 | raise ImproperlyConfigured("You need to install pylibmc to use the MemcachedConnection class.")
33 | super(MemcachedConnection, self).__init__(host=host, port=port, **kwargs)
34 | self._make_connection = lambda: pylibmc.Client(['%s:%s' % (host, port)], behaviors={"tcp_nodelay": True})
35 |
36 | def perform_request(self, method, url, params=None, body=None, timeout=None, ignore=()):
37 | mc = self._get_connection()
38 | url = self.url_prefix + url
39 | if params:
40 | url = '%s?%s' % (url, urlencode(params or {}))
41 | full_url = self.host + url
42 |
43 | mc_method = self.method_map.get(method, 'get')
44 |
45 | start = time.time()
46 | try:
47 | status = 200
48 | if mc_method == 'set':
49 | # no response from set commands
50 | response = ''
51 | if not json.dumps(mc.set(url, body)):
52 | status = 500
53 | else:
54 | response = mc.get(url)
55 |
56 | duration = time.time() - start
57 | if response:
58 | response = response.decode('utf-8')
59 | except Exception as e:
60 | self.log_request_fail(method, full_url, body, time.time() - start, exception=e)
61 | raise ConnectionError('N/A', str(e), e)
62 | finally:
63 | self._release_connection(mc)
64 |
65 | # try not to load the json every time
66 | if response and response[0] == '{' and ('"status"' in response or '"error"' in response):
67 | data = json.loads(response)
68 | if 'status' in data and isinstance(data['status'], int):
69 | status = data['status']
70 | elif 'error' in data:
71 | raise TransportError('N/A', data['error'])
72 |
73 | if not (200 <= status < 300) and status not in ignore:
74 | self.log_request_fail(method, url, body, duration, status)
75 | self._raise_error(status, response)
76 |
77 | self.log_request_success(method, full_url, url, body, status,
78 | response, duration)
79 |
80 | return status, {}, response
81 |
82 |
83 |
84 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/elasticsearch/connection/pooling.py:
--------------------------------------------------------------------------------
1 | try:
2 | import queue
3 | except ImportError:
4 | import Queue as queue
5 | from .base import Connection
6 |
7 |
8 | class PoolingConnection(Connection):
9 | def __init__(self, *args, **kwargs):
10 | self._free_connections = queue.Queue()
11 | super(PoolingConnection, self).__init__(*args, **kwargs)
12 |
13 | def _get_connection(self):
14 | try:
15 | return self._free_connections.get_nowait()
16 | except queue.Empty:
17 | return self._make_connection()
18 |
19 | def _release_connection(self, con):
20 | self._free_connections.put(con)
--------------------------------------------------------------------------------
/search-elasticsearch/bin/elasticsearch/connection/thrift.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 | from socket import timeout as SocketTimeout
3 | import time
4 | import logging
5 |
6 | try:
7 | from .esthrift import Rest
8 | from .esthrift.ttypes import Method, RestRequest
9 |
10 | from thrift.transport import TTransport, TSocket, TSSLSocket
11 | from thrift.protocol import TBinaryProtocol
12 | from thrift.Thrift import TException
13 | THRIFT_AVAILABLE = True
14 | except ImportError:
15 | THRIFT_AVAILABLE = False
16 |
17 | from ..exceptions import ConnectionError, ImproperlyConfigured, ConnectionTimeout
18 | from .pooling import PoolingConnection
19 |
20 | logger = logging.getLogger('elasticsearch')
21 |
22 | class ThriftConnection(PoolingConnection):
23 | """
24 | Connection using the `thrift` protocol to communicate with elasticsearch.
25 |
26 | See https://github.com/elasticsearch/elasticsearch-transport-thrift for additional info.
27 | """
28 | transport_schema = 'thrift'
29 |
30 | def __init__(self, host='localhost', port=9500, framed_transport=False, use_ssl=False, **kwargs):
31 | """
32 | :arg framed_transport: use `TTransport.TFramedTransport` instead of
33 | `TTransport.TBufferedTransport`
34 | """
35 | if not THRIFT_AVAILABLE:
36 | raise ImproperlyConfigured("Thrift is not available.")
37 |
38 | super(ThriftConnection, self).__init__(host=host, port=port, **kwargs)
39 | self._framed_transport = framed_transport
40 | self._tsocket_class = TSocket.TSocket
41 | if use_ssl:
42 | self._tsocket_class = TSSLSocket.TSSLSocket
43 | self._tsocket_args = (host, port)
44 |
45 | def _make_connection(self):
46 | socket = self._tsocket_class(*self._tsocket_args)
47 | socket.setTimeout(self.timeout * 1000.0)
48 | if self._framed_transport:
49 | transport = TTransport.TFramedTransport(socket)
50 | else:
51 | transport = TTransport.TBufferedTransport(socket)
52 |
53 | protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
54 | client = Rest.Client(protocol)
55 | client.transport = transport
56 | transport.open()
57 | return client
58 |
59 | def perform_request(self, method, url, params=None, body=None, timeout=None, ignore=()):
60 | request = RestRequest(method=Method._NAMES_TO_VALUES[method.upper()], uri=url,
61 | parameters=params, body=body)
62 |
63 | start = time.time()
64 | tclient = None
65 | try:
66 | tclient = self._get_connection()
67 | response = tclient.execute(request)
68 | duration = time.time() - start
69 | except SocketTimeout as e:
70 | self.log_request_fail(method, url, body, time.time() - start, exception=e)
71 | raise ConnectionTimeout('TIMEOUT', str(e), e)
72 | except (TException, SocketTimeout) as e:
73 | self.log_request_fail(method, url, body, time.time() - start, exception=e)
74 | if tclient:
75 | try:
76 | # try closing transport socket
77 | tclient.transport.close()
78 | except Exception as e:
79 | logger.warning(
80 | 'Exception %s occured when closing a failed thrift connection.',
81 | e, exc_info=True
82 | )
83 | raise ConnectionError('N/A', str(e), e)
84 |
85 | self._release_connection(tclient)
86 |
87 | if not (200 <= response.status < 300) and response.status not in ignore:
88 | self.log_request_fail(method, url, body, duration, response.status)
89 | self._raise_error(response.status, response.body)
90 |
91 | self.log_request_success(method, url, url, body, response.status,
92 | response.body, duration)
93 |
94 | headers = {}
95 | if response.headers:
96 | headers = dict((k.lower(), v) for k, v in response.headers.items())
97 | return response.status, headers, response.body or ''
98 |
99 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/elasticsearch/exceptions.py:
--------------------------------------------------------------------------------
1 | __all__ = [
2 | 'ImproperlyConfigured', 'ElasticsearchException', 'SerializationError',
3 | 'TransportError', 'NotFoundError', 'ConflictError', 'RequestError', 'ConnectionError',
4 | 'SSLError', 'ConnectionTimeout'
5 | ]
6 |
7 | class ImproperlyConfigured(Exception):
8 | """
9 | Exception raised when the config passed to the client is inconsistent or invalid.
10 | """
11 |
12 |
13 | class ElasticsearchException(Exception):
14 | """
15 | Base class for all exceptions raised by this package's operations (doesn't
16 | apply to :class:`~elasticsearch.ImproperlyConfigured`).
17 | """
18 |
19 |
20 | class SerializationError(ElasticsearchException):
21 | """
22 | Data passed in failed to serialize properly in the ``Serializer`` being
23 | used.
24 | """
25 |
26 |
27 | class TransportError(ElasticsearchException):
28 | """
29 | Exception raised when ES returns a non-OK (>=400) HTTP status code. Or when
30 | an actual connection error happens; in that case the ``status_code`` will
31 | be set to ``'N/A'``.
32 | """
33 | @property
34 | def status_code(self):
35 | """
36 | The HTTP status code of the response that precipitated the error or
37 | ``'N/A'`` if not applicable.
38 | """
39 | return self.args[0]
40 |
41 | @property
42 | def error(self):
43 | """ A string error message. """
44 | return self.args[1]
45 |
46 | @property
47 | def info(self):
48 | """ Dict of returned error info from ES, where available. """
49 | return self.args[2]
50 |
51 | def __str__(self):
52 | return 'TransportError(%s, %r)' % (self.status_code, self.error)
53 |
54 |
55 | class ConnectionError(TransportError):
56 | """
57 | Error raised when there was an exception while talking to ES. Original
58 | exception from the underlying :class:`~elasticsearch.Connection`
59 | implementation is available as ``.info.``
60 | """
61 | def __str__(self):
62 | return 'ConnectionError(%s) caused by: %s(%s)' % (
63 | self.error, self.info.__class__.__name__, self.info)
64 |
65 |
66 | class SSLError(ConnectionError):
67 | """ Error raised when encountering SSL errors. """
68 |
69 |
70 | class ConnectionTimeout(ConnectionError):
71 | """ A network timeout. Doesn't cause a node retry by default. """
72 | def __str__(self):
73 | return 'ConnectionTimeout caused by - %s(%s)' % (
74 | self.info.__class__.__name__, self.info)
75 |
76 |
77 | class NotFoundError(TransportError):
78 | """ Exception representing a 404 status code. """
79 |
80 |
81 | class ConflictError(TransportError):
82 | """ Exception representing a 409 status code. """
83 |
84 |
85 | class RequestError(TransportError):
86 | """ Exception representing a 400 status code. """
87 |
88 |
89 | class AuthenticationException(TransportError):
90 | """ Exception representing a 401 status code. """
91 |
92 |
93 | class AuthorizationException(TransportError):
94 | """ Exception representing a 403 status code. """
95 |
96 | # more generic mappings from status_code to python exceptions
97 | HTTP_EXCEPTIONS = {
98 | 400: RequestError,
99 | 401: AuthenticationException,
100 | 403: AuthorizationException,
101 | 404: NotFoundError,
102 | 409: ConflictError,
103 | }
104 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/elasticsearch/helpers/test.py:
--------------------------------------------------------------------------------
1 | import time
2 | import os
3 | try:
4 | # python 2.6
5 | from unittest2 import TestCase, SkipTest
6 | except ImportError:
7 | from unittest import TestCase, SkipTest
8 |
9 | from elasticsearch import Elasticsearch
10 | from elasticsearch.exceptions import ConnectionError
11 |
12 | def get_test_client(nowait=False):
13 | # construct kwargs from the environment
14 | kw = {}
15 | if 'TEST_ES_CONNECTION' in os.environ:
16 | from elasticsearch import connection
17 | kw['connection_class'] = getattr(connection, os.environ['TEST_ES_CONNECTION'])
18 |
19 | client = Elasticsearch([os.environ.get('TEST_ES_SERVER', {})], **kw)
20 |
21 | # wait for yellow status
22 | for _ in range(1 if nowait else 100):
23 | try:
24 | client.cluster.health(wait_for_status='yellow')
25 | return client
26 | except ConnectionError:
27 | time.sleep(.1)
28 | else:
29 | # timeout
30 | raise SkipTest("Elasticsearch failed to start.")
31 |
32 | def _get_version(version_string):
33 | version = version_string.strip().split('.')
34 | return tuple(int(v) if v.isdigit() else 999 for v in version)
35 |
36 | class ElasticsearchTestCase(TestCase):
37 | @staticmethod
38 | def _get_client():
39 | return get_test_client()
40 |
41 | @classmethod
42 | def setUpClass(cls):
43 | super(ElasticsearchTestCase, cls).setUpClass()
44 | cls.client = cls._get_client()
45 |
46 | def tearDown(self):
47 | super(ElasticsearchTestCase, self).tearDown()
48 | self.client.indices.delete(index='*')
49 | self.client.indices.delete_template(name='*', ignore=404)
50 |
51 | @property
52 | def es_version(self):
53 | if not hasattr(self, '_es_version'):
54 | version_string = self.client.info()['version']['number']
55 | self._es_version = _get_version(version_string)
56 | return self._es_version
57 |
58 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/elasticsearch/serializer.py:
--------------------------------------------------------------------------------
1 | try:
2 | import simplejson as json
3 | except ImportError:
4 | import json
5 | from datetime import date, datetime
6 | from decimal import Decimal
7 |
8 | from .exceptions import SerializationError, ImproperlyConfigured
9 | from .compat import string_types
10 |
11 | class TextSerializer(object):
12 | mimetype = 'text/plain'
13 |
14 | def loads(self, s):
15 | return s
16 |
17 | def dumps(self, data):
18 | if isinstance(data, string_types):
19 | return data
20 |
21 | raise SerializationError('Cannot serialize %r into text.' % data)
22 |
23 | class JSONSerializer(object):
24 | mimetype = 'application/json'
25 |
26 | def default(self, data):
27 | if isinstance(data, (date, datetime)):
28 | return data.isoformat()
29 | elif isinstance(data, Decimal):
30 | return float(data)
31 | raise TypeError("Unable to serialize %r (type: %s)" % (data, type(data)))
32 |
33 | def loads(self, s):
34 | try:
35 | return json.loads(s)
36 | except (ValueError, TypeError) as e:
37 | raise SerializationError(s, e)
38 |
39 | def dumps(self, data):
40 | # don't serialize strings
41 | if isinstance(data, string_types):
42 | return data
43 |
44 | try:
45 | return json.dumps(data, default=self.default)
46 | except (ValueError, TypeError) as e:
47 | raise SerializationError(data, e)
48 |
49 | DEFAULT_SERIALIZERS = {
50 | JSONSerializer.mimetype: JSONSerializer(),
51 | TextSerializer.mimetype: TextSerializer(),
52 | }
53 |
54 | class Deserializer(object):
55 | def __init__(self, serializers, default_mimetype='application/json'):
56 | try:
57 | self.default = serializers[default_mimetype]
58 | except KeyError:
59 | raise ImproperlyConfigured('Cannot find default serializer (%s)' % default_mimetype)
60 | self.serializers = serializers
61 |
62 | def loads(self, s, mimetype=None):
63 | if not mimetype:
64 | deserializer = self.default
65 | else:
66 | # split out charset
67 | mimetype = mimetype.split(';', 1)[0]
68 | try:
69 | deserializer = self.serializers[mimetype]
70 | except KeyError:
71 | raise SerializationError('Unknown mimetype, unable to deserialize: %s' % mimetype)
72 |
73 | return deserializer.loads(s)
74 |
75 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/essearch.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | #
3 | # Copyright 2011-2014 Splunk, Inc.
4 | #
5 | # Licensed under the Apache License, Version 2.0 (the "License"): you may
6 | # not use this file except in compliance with the License. You may obtain
7 | # a copy of the License at
8 | #
9 | # http://www.apache.org/licenses/LICENSE-2.0
10 | #
11 | # Unless required by applicable law or agreed to in writing, software
12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14 | # License for the specific language governing permissions and limitations
15 | # under the License.
16 |
17 |
18 | # python essearch.py __EXECUTE__ 'q="New York"'
19 |
20 | from datetime import datetime
21 | from elasticsearch import Elasticsearch
22 | import os, sys, time, requests, oauth2, json, urllib
23 |
24 | (isgetinfo, sys.argv) = splunk.Intersplunk.isGetInfo(sys.argv)
25 |
26 | from splunklib.searchcommands import \
27 | dispatch, GeneratingCommand, Configuration, Option, validators
28 |
29 | @Configuration()
30 | class EsCommand(GeneratingCommand):
31 | """ Generates events that are the result of a query against Elasticsearch
32 |
33 | ##Syntax
34 |
35 | .. code-block::
36 | es index= | q= | fields= | oldest= | earl= | limit=
37 |
38 | ##Description
39 |
40 | The :code:`es` issue a query to ElasticSearch, where the
41 | query is specified in :code:`q`.
42 |
43 | ##Example
44 |
45 | .. code-block::
46 | | es oldest=now-100d earliest=now query="some text" index=nagios* limit=1000 field=message
47 |
48 | This example generates events drawn from the result of the query
49 |
50 | """
51 |
52 | index = Option(doc='', require=False, default="*")
53 |
54 | q = Option(doc='', require=True)
55 |
56 | fields = Option(doc='', require=False, default="message")
57 |
58 | oldest = Option(doc='', require=False, default="now")
59 |
60 | earl = Option(doc='', require=False, default="now-1d")
61 |
62 | limit = Option(doc='', require=False, validate=validators.Integer(), default=100)
63 |
64 | def generate(self):
65 |
66 | #self.logger.debug('SimulateCommand: %s' % self) # log command line
67 |
68 | config = self.get_configuration()
69 |
70 | #pp = pprint.PrettyPrinter(indent=4)
71 | self.logger.debug('Setup ES')
72 | es = Elasticsearch()
73 | body = {
74 | "size": limit,
75 | "query": {
76 | "filtered" : {
77 | "query": {
78 | "query_string" : {
79 | "query" : q
80 | }
81 | }
82 | }
83 | }
84 | }
85 | #pp.pprint(body);
86 | res = es.search(size=50, index=index, body=body);
87 |
88 | # if response.status_code != 200:
89 | # yield {'ERROR': results['error']['text']}
90 | # return
91 |
92 |
93 | # date_time = '2014-12-21T16:11:18.419Z'
94 | # pattern = '%Y-%m-%dT%H:%M:%S.%fZ'
95 |
96 | for hit in res['hits']['hits']:
97 | yield self.getEvent(hit)
98 |
99 | def getEvent(self, result):
100 |
101 | # hit["_source"][defaultField] = hit["_source"][defaultField].replace('"',' ');
102 | # epochTimestamp = hit['_source']['@timestamp'];
103 | # hit['_source']['_epoch'] = int(time.mktime(time.strptime(epochTimestamp, pattern)))
104 | # hit['_source']["_raw"]=hit['_source'][defaultField]
105 |
106 | event = {'_time': time.time(),
107 | '_index': result['_index'],
108 | '_type': result['_type'],
109 | '_id': result['_id'],
110 | '_score': result['_score']
111 | }
112 |
113 | event["_raw"] = json.dumps(result)
114 |
115 | return event
116 |
117 | def get_configuration(self):
118 | sourcePath = os.path.dirname(os.path.abspath(__file__))
119 | config_file = open(sourcePath + '/config.json')
120 | return json.load(config_file)
121 |
122 | def __init__(self):
123 | super(GeneratingCommand, self).__init__()
124 |
125 | dispatch(EsCommand, sys.argv, sys.stdin, sys.stdout, __name__)
--------------------------------------------------------------------------------
/search-elasticsearch/bin/search.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | from datetime import datetime
3 | from elasticsearch import Elasticsearch
4 | import pprint
5 | import time
6 | import sys
7 | import json
8 |
9 | total = len(sys.argv)
10 | cmdargs = str(sys.argv)
11 | index="*"
12 | limit=100
13 | oldestDate="now-1d"
14 | earliestDate="now"
15 | defaultField="message"
16 | query="*"
17 | for i in xrange(total):
18 | opt = str (sys.argv[i])
19 | if ( opt.startswith("index") ):
20 | index= str (sys.argv[i]).split("=")[1]
21 | elif ( opt.startswith("query") ):
22 | query= str (sys.argv[i]).split("=")[1]
23 | elif ( opt.startswith("field") ):
24 | defaultField= str (sys.argv[i]).split("=")[1]
25 | elif ( opt.startswith("oldest") ):
26 | oldestDate= str (sys.argv[i]).split("=")[1]
27 | elif ( opt.startswith("earl") ):
28 | earliestDate= str (sys.argv[i]).split("=")[1]
29 | elif ( opt.startswith("limit") ):
30 | limit= str (sys.argv[i]).split("=")[1]
31 |
32 | pp = pprint.PrettyPrinter(indent=4)
33 | # to query an external server: Elasticsearch(['http://1.2.3.4:9200'])
34 | es = Elasticsearch()
35 | body = {
36 | "size": limit,
37 | "from": 0,
38 | "query": {
39 | "bool" : {
40 | "must": [{
41 | "query_string" : {
42 | "default_field" : defaultField,
43 | "query" : query
44 | }},
45 | {"range" : {
46 | "@timestamp": {
47 | "lt" : earliestDate,
48 | "gt" : oldestDate
49 | }
50 | }
51 | }],
52 | "must_not":[],
53 | "should":[]
54 | }
55 | }
56 | }
57 | #pp.pprint(body)
58 | res = es.search(size=limit, index=index, body=body)
59 | print("\"_time\",\"_raw\",\"index\",\"type\"")
60 | #pp.pprint(res)
61 | #date_time = '2014-12-21T16:11:18.419Z'
62 | pattern = '%Y-%m-%dT%H:%M:%S.%fZ'
63 |
64 | for hit in res['hits']['hits']:
65 | epochTimestamp = hit['_source']['@timestamp']
66 | hit['_source']['_epoch'] = int(time.mktime(time.strptime(epochTimestamp, pattern)))
67 | hit['_source']["_raw"]=json.dumps(hit['_source'][defaultField]).replace('"',' ')
68 | print("%(_epoch)s,\"%(_raw)s\"," % hit["_source"] +
69 | "\"%(_index)s\",\"%(_type)s\"" % hit
70 | )
71 | #test development branch
72 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/splunklib/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright 2011-2014 Splunk, Inc.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License"): you may
4 | # not use this file except in compliance with the License. You may obtain
5 | # a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 | # License for the specific language governing permissions and limitations
13 | # under the License.
14 |
15 | """Python library for Splunk."""
16 |
17 | __version_info__ = (1, 3, 1)
18 | __version__ = ".".join(map(str, __version_info__))
19 |
20 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/splunklib/data.py:
--------------------------------------------------------------------------------
1 | # Copyright 2011-2014 Splunk, Inc.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License"): you may
4 | # not use this file except in compliance with the License. You may obtain
5 | # a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 | # License for the specific language governing permissions and limitations
13 | # under the License.
14 |
15 | """The **splunklib.data** module reads the responses from splunkd in Atom Feed
16 | format, which is the format used by most of the REST API.
17 | """
18 |
19 | from xml.etree.ElementTree import XML
20 |
21 | __all__ = ["load"]
22 |
23 | # LNAME refers to element names without namespaces; XNAME is the same
24 | # name, but with an XML namespace.
25 | LNAME_DICT = "dict"
26 | LNAME_ITEM = "item"
27 | LNAME_KEY = "key"
28 | LNAME_LIST = "list"
29 |
30 | XNAMEF_REST = "{http://dev.splunk.com/ns/rest}%s"
31 | XNAME_DICT = XNAMEF_REST % LNAME_DICT
32 | XNAME_ITEM = XNAMEF_REST % LNAME_ITEM
33 | XNAME_KEY = XNAMEF_REST % LNAME_KEY
34 | XNAME_LIST = XNAMEF_REST % LNAME_LIST
35 |
36 | # Some responses don't use namespaces (eg: search/parse) so we look for
37 | # both the extended and local versions of the following names.
38 |
39 | def isdict(name):
40 | return name == XNAME_DICT or name == LNAME_DICT
41 |
42 | def isitem(name):
43 | return name == XNAME_ITEM or name == LNAME_ITEM
44 |
45 | def iskey(name):
46 | return name == XNAME_KEY or name == LNAME_KEY
47 |
48 | def islist(name):
49 | return name == XNAME_LIST or name == LNAME_LIST
50 |
51 | def hasattrs(element):
52 | return len(element.attrib) > 0
53 |
54 | def localname(xname):
55 | rcurly = xname.find('}')
56 | return xname if rcurly == -1 else xname[rcurly+1:]
57 |
58 | def load(text, match=None):
59 | """This function reads a string that contains the XML of an Atom Feed, then
60 | returns the
61 | data in a native Python structure (a ``dict`` or ``list``). If you also
62 | provide a tag name or path to match, only the matching sub-elements are
63 | loaded.
64 |
65 | :param text: The XML text to load.
66 | :type text: ``string``
67 | :param match: A tag name or path to match (optional).
68 | :type match: ``string``
69 | """
70 | if text is None: return None
71 | text = text.strip()
72 | if len(text) == 0: return None
73 | nametable = {
74 | 'namespaces': [],
75 | 'names': {}
76 | }
77 | root = XML(text)
78 | items = [root] if match is None else root.findall(match)
79 | count = len(items)
80 | if count == 0:
81 | return None
82 | elif count == 1:
83 | return load_root(items[0], nametable)
84 | else:
85 | return [load_root(item, nametable) for item in items]
86 |
87 | # Load the attributes of the given element.
88 | def load_attrs(element):
89 | if not hasattrs(element): return None
90 | attrs = record()
91 | for key, value in element.attrib.iteritems():
92 | attrs[key] = value
93 | return attrs
94 |
95 | # Parse a element and return a Python dict
96 | def load_dict(element, nametable = None):
97 | value = record()
98 | children = list(element)
99 | for child in children:
100 | assert iskey(child.tag)
101 | name = child.attrib["name"]
102 | value[name] = load_value(child, nametable)
103 | return value
104 |
105 | # Loads the given elements attrs & value into single merged dict.
106 | def load_elem(element, nametable=None):
107 | name = localname(element.tag)
108 | attrs = load_attrs(element)
109 | value = load_value(element, nametable)
110 | if attrs is None: return name, value
111 | if value is None: return name, attrs
112 | # If value is simple, merge into attrs dict using special key
113 | if isinstance(value, str):
114 | attrs["$text"] = value
115 | return name, attrs
116 | # Both attrs & value are complex, so merge the two dicts, resolving collisions.
117 | collision_keys = []
118 | for key, val in attrs.iteritems():
119 | if key in value and key in collision_keys:
120 | value[key].append(val)
121 | elif key in value and key not in collision_keys:
122 | value[key] = [value[key], val]
123 | collision_keys.append(key)
124 | else:
125 | value[key] = val
126 | return name, value
127 |
128 | # Parse a element and return a Python list
129 | def load_list(element, nametable=None):
130 | assert islist(element.tag)
131 | value = []
132 | children = list(element)
133 | for child in children:
134 | assert isitem(child.tag)
135 | value.append(load_value(child, nametable))
136 | return value
137 |
138 | # Load the given root element.
139 | def load_root(element, nametable=None):
140 | tag = element.tag
141 | if isdict(tag): return load_dict(element, nametable)
142 | if islist(tag): return load_list(element, nametable)
143 | k, v = load_elem(element, nametable)
144 | return Record.fromkv(k, v)
145 |
146 | # Load the children of the given element.
147 | def load_value(element, nametable=None):
148 | children = list(element)
149 | count = len(children)
150 |
151 | # No children, assume a simple text value
152 | if count == 0:
153 | text = element.text
154 | if text is None:
155 | return None
156 | text = text.strip()
157 | if len(text) == 0:
158 | return None
159 | return text
160 |
161 | # Look for the special case of a single well-known structure
162 | if count == 1:
163 | child = children[0]
164 | tag = child.tag
165 | if isdict(tag): return load_dict(child, nametable)
166 | if islist(tag): return load_list(child, nametable)
167 |
168 | value = record()
169 | for child in children:
170 | name, item = load_elem(child, nametable)
171 | # If we have seen this name before, promote the value to a list
172 | if value.has_key(name):
173 | current = value[name]
174 | if not isinstance(current, list):
175 | value[name] = [current]
176 | value[name].append(item)
177 | else:
178 | value[name] = item
179 |
180 | return value
181 |
182 | # A generic utility that enables "dot" access to dicts
183 | class Record(dict):
184 | """This generic utility class enables dot access to members of a Python
185 | dictionary.
186 |
187 | Any key that is also a valid Python identifier can be retrieved as a field.
188 | So, for an instance of ``Record`` called ``r``, ``r.key`` is equivalent to
189 | ``r['key']``. A key such as ``invalid-key`` or ``invalid.key`` cannot be
190 | retrieved as a field, because ``-`` and ``.`` are not allowed in
191 | identifiers.
192 |
193 | Keys of the form ``a.b.c`` are very natural to write in Python as fields. If
194 | a group of keys shares a prefix ending in ``.``, you can retrieve keys as a
195 | nested dictionary by calling only the prefix. For example, if ``r`` contains
196 | keys ``'foo'``, ``'bar.baz'``, and ``'bar.qux'``, ``r.bar`` returns a record
197 | with the keys ``baz`` and ``qux``. If a key contains multiple ``.``, each
198 | one is placed into a nested dictionary, so you can write ``r.bar.qux`` or
199 | ``r['bar.qux']`` interchangeably.
200 | """
201 | sep = '.'
202 |
203 | def __call__(self, *args):
204 | if len(args) == 0: return self
205 | return Record((key, self[key]) for key in args)
206 |
207 | def __getattr__(self, name):
208 | try:
209 | return self[name]
210 | except KeyError:
211 | raise AttributeError(name)
212 |
213 | def __delattr__(self, name):
214 | del self[name]
215 |
216 | def __setattr__(self, name, value):
217 | self[name] = value
218 |
219 | @staticmethod
220 | def fromkv(k, v):
221 | result = record()
222 | result[k] = v
223 | return result
224 |
225 | def __getitem__(self, key):
226 | if key in self:
227 | return dict.__getitem__(self, key)
228 | key += self.sep
229 | result = record()
230 | for k,v in self.iteritems():
231 | if not k.startswith(key):
232 | continue
233 | suffix = k[len(key):]
234 | if '.' in suffix:
235 | ks = suffix.split(self.sep)
236 | z = result
237 | for x in ks[:-1]:
238 | if x not in z:
239 | z[x] = record()
240 | z = z[x]
241 | z[ks[-1]] = v
242 | else:
243 | result[suffix] = v
244 | if len(result) == 0:
245 | raise KeyError("No key or prefix: %s" % key)
246 | return result
247 |
248 |
249 | def record(value=None):
250 | """This function returns a :class:`Record` instance constructed with an
251 | initial value that you provide.
252 |
253 | :param `value`: An initial record value.
254 | :type `value`: ``dict``
255 | """
256 | if value is None: value = {}
257 | return Record(value)
258 |
259 |
--------------------------------------------------------------------------------
/search-elasticsearch/bin/splunklib/modularinput/__init__.py:
--------------------------------------------------------------------------------
1 | """The following imports allow these classes to be imported via
2 | the splunklib.modularinput package like so:
3 |
4 | from splunklib.modularinput import *
5 | """
6 | from argument import Argument
7 | from event import Event
8 | from event_writer import EventWriter
9 | from input_definition import InputDefinition
10 | from scheme import Scheme
11 | from script import Script
12 | from validation_definition import ValidationDefinition
--------------------------------------------------------------------------------
/search-elasticsearch/bin/splunklib/modularinput/argument.py:
--------------------------------------------------------------------------------
1 | # Copyright 2011-2014 Splunk, Inc.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License"): you may
4 | # not use this file except in compliance with the License. You may obtain
5 | # a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 | # License for the specific language governing permissions and limitations
13 | # under the License.
14 |
15 | try:
16 | import xml.etree.ElementTree as ET
17 | except ImportError:
18 | import xml.etree.cElementTree as ET
19 |
20 | class Argument(object):
21 | """Class representing an argument to a modular input kind.
22 |
23 | ``Argument`` is meant to be used with ``Scheme`` to generate an XML
24 | definition of the modular input kind that Splunk understands.
25 |
26 | ``name`` is the only required parameter for the constructor.
27 |
28 | **Example with least parameters**::
29 |
30 | arg1 = Argument(name="arg1")
31 |
32 | **Example with all parameters**::
33 |
34 | arg2 = Argument(
35 | name="arg2",
36 | description="This is an argument with lots of parameters",
37 | validation="is_pos_int('some_name')",
38 | data_type=Argument.data_type_number,
39 | required_on_edit=True,
40 | required_on_create=True
41 | )
42 | """
43 |
44 | # Constant values, do not change.
45 | # These should be used for setting the value of an Argument object's data_type field.
46 | data_type_boolean = "BOOLEAN"
47 | data_type_number = "NUMBER"
48 | data_type_string = "STRING"
49 |
50 | def __init__(self, name, description=None, validation=None,
51 | data_type=data_type_string, required_on_edit=False, required_on_create=False, title=None):
52 | """
53 | :param name: ``string``, identifier for this argument in Splunk.
54 | :param description: ``string``, human-readable description of the argument.
55 | :param validation: ``string`` specifying how the argument should be validated, if using internal validation.
56 | If using external validation, this will be ignored.
57 | :param data_type: ``string``, data type of this field; use the class constants.
58 | "data_type_boolean", "data_type_number", or "data_type_string".
59 | :param required_on_edit: ``Boolean``, whether this arg is required when editing an existing modular input of this kind.
60 | :param required_on_create: ``Boolean``, whether this arg is required when creating a modular input of this kind.
61 | :param title: ``String``, a human-readable title for the argument.
62 | """
63 | self.name = name
64 | self.description = description
65 | self.validation = validation
66 | self.data_type = data_type
67 | self.required_on_edit = required_on_edit
68 | self.required_on_create = required_on_create
69 | self.title = title
70 |
71 | def add_to_document(self, parent):
72 | """Adds an ``Argument`` object to this ElementTree document.
73 |
74 | Adds an subelement to the parent element, typically
75 | and sets up its subelements with their respective text.
76 |
77 | :param parent: An ``ET.Element`` to be the parent of a new subelement
78 | :returns: An ``ET.Element`` object representing this argument.
79 | """
80 | arg = ET.SubElement(parent, "arg")
81 | arg.set("name", self.name)
82 |
83 | if self.description is not None:
84 | ET.SubElement(arg, "description").text = self.description
85 |
86 | if self.validation is not None:
87 | ET.SubElement(arg, "validation").text = self.validation
88 |
89 | # add all other subelements to this Argument, represented by (tag, text)
90 | subelements = [
91 | ("data_type", self.data_type),
92 | ("required_on_edit", self.required_on_edit),
93 | ("required_on_create", self.required_on_create)
94 | ]
95 |
96 | for name, value in subelements:
97 | ET.SubElement(arg, name).text = str(value).lower()
98 |
99 | return arg
--------------------------------------------------------------------------------
/search-elasticsearch/bin/splunklib/modularinput/event.py:
--------------------------------------------------------------------------------
1 | # Copyright 2011-2014 Splunk, Inc.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License"): you may
4 | # not use this file except in compliance with the License. You may obtain
5 | # a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 | # License for the specific language governing permissions and limitations
13 | # under the License.
14 |
15 | try:
16 | import xml.etree.cElementTree as ET
17 | except ImportError as ie:
18 | import xml.etree.ElementTree as ET
19 |
20 | class Event(object):
21 | """Represents an event or fragment of an event to be written by this modular input to Splunk.
22 |
23 | To write an input to a stream, call the ``write_to`` function, passing in a stream.
24 | """
25 | def __init__(self, data=None, stanza=None, time=None, host=None, index=None, source=None,
26 | sourcetype=None, done=True, unbroken=True):
27 | """There are no required parameters for constructing an Event
28 |
29 | **Example with minimal configuration**::
30 |
31 | my_event = Event(
32 | data="This is a test of my new event.",
33 | stanza="myStanzaName",
34 | time="%.3f" % 1372187084.000
35 | )
36 |
37 | **Example with full configuration**::
38 |
39 | excellent_event = Event(
40 | data="This is a test of my excellent event.",
41 | stanza="excellenceOnly",
42 | time="%.3f" % 1372274622.493,
43 | host="localhost",
44 | index="main",
45 | source="Splunk",
46 | sourcetype="misc",
47 | done=True,
48 | unbroken=True
49 | )
50 |
51 | :param data: ``string``, the event's text.
52 | :param stanza: ``string``, name of the input this event should be sent to.
53 | :param time: ``float``, time in seconds, including up to 3 decimal places to represent milliseconds.
54 | :param host: ``string``, the event's host, ex: localhost.
55 | :param index: ``string``, the index this event is specified to write to, or None if default index.
56 | :param source: ``string``, the source of this event, or None to have Splunk guess.
57 | :param sourcetype: ``string``, source type currently set on this event, or None to have Splunk guess.
58 | :param done: ``boolean``, is this a complete ``Event``? False if an ``Event`` fragment.
59 | :param unbroken: ``boolean``, Is this event completely encapsulated in this ``Event`` object?
60 | """
61 | self.data = data
62 | self.done = done
63 | self.host = host
64 | self.index = index
65 | self.source = source
66 | self.sourceType = sourcetype
67 | self.stanza = stanza
68 | self.time = time
69 | self.unbroken = unbroken
70 |
71 | def write_to(self, stream):
72 | """Write an XML representation of self, an ``Event`` object, to the given stream.
73 |
74 | The ``Event`` object will only be written if its data field is defined,
75 | otherwise a ``ValueError`` is raised.
76 |
77 | :param stream: stream to write XML to.
78 | """
79 | if self.data is None:
80 | raise ValueError("Events must have at least the data field set to be written to XML.")
81 |
82 | event = ET.Element("event")
83 | if self.stanza is not None:
84 | event.set("stanza", self.stanza)
85 | event.set("unbroken", str(int(self.unbroken)))
86 |
87 | # if a time isn't set, let Splunk guess by not creating a