├── test ├── __init__.py ├── run_tests.py ├── unit │ ├── test_action_create_index.py │ ├── __init__.py │ ├── test_action_rollover.py │ ├── test_action_clusterrouting.py │ ├── test_action_open.py │ ├── test_cli_methods.py │ ├── test_action_delete_snapshots.py │ ├── test_action_replicas.py │ ├── test_action_close.py │ ├── test_action_delete_indices.py │ ├── test_action_forcemerge.py │ └── test_action_allocation.py └── integration │ ├── test_clusterrouting.py │ ├── test_forcemerge.py │ ├── test_replicas.py │ ├── test_envvars.py │ ├── test_integrations.py │ ├── test_count_pattern.py │ ├── test_delete_snapshots.py │ ├── test_open.py │ ├── test_create_index.py │ ├── test_snapshot.py │ ├── test_es_repo_mgr.py │ ├── test_close.py │ └── __init__.py ├── curator ├── defaults │ ├── __init__.py │ ├── client_defaults.py │ ├── settings.py │ └── filtertypes.py ├── _version.py ├── validators │ ├── __init__.py │ ├── config_file.py │ ├── filters.py │ ├── actions.py │ ├── schemacheck.py │ └── options.py ├── curator_cli.py ├── __main__.py ├── __init__.py ├── exceptions.py ├── config_utils.py ├── logtools.py └── repomgrcli.py ├── setup.cfg ├── .github ├── PULL_REQUEST_TEMPLATE.md └── issue_template.md ├── unix_packages ├── cx_freeze-5.0.1.dev.tar.gz └── build_package_from_source.sh ├── requirements.txt ├── docs ├── utilities.rst ├── objectclasses.rst ├── asciidoc │ ├── inc_unit_table.asciidoc │ ├── inc_filepath.asciidoc │ ├── inc_strftime_table.asciidoc │ ├── index.asciidoc │ ├── inc_filter_by_aliases.asciidoc │ ├── inc_filter_chaining.asciidoc │ ├── inc_sources.asciidoc │ ├── inc_timestring_regex.asciidoc │ ├── inc_kinds.asciidoc │ ├── versions.asciidoc │ ├── security.asciidoc │ └── about.asciidoc ├── filters.rst ├── actionclasses.rst ├── examples.rst └── index.rst ├── Dockerfile ├── .gitignore ├── MANIFEST.in ├── examples ├── curator.yml └── actions │ ├── create_index.yml │ ├── close.yml │ ├── delete_snapshots.yml │ ├── replicas.yml │ ├── allocation.yml │ ├── delete_indices.yml │ ├── open.yml │ ├── forcemerge.yml │ ├── snapshot.yml │ ├── restore.yml │ ├── alias.yml │ └── shrink.yml ├── LICENSE.txt ├── .travis.yml ├── Vagrant ├── centos │ ├── 6 │ │ └── Vagrantfile │ └── 7 │ │ └── Vagrantfile └── ubuntu │ └── 14.04 │ └── Vagrantfile ├── run_curator.py ├── run_singleton.py ├── run_es_repo_mgr.py ├── CONTRIBUTORS ├── travis-run.sh ├── CONTRIBUTING.md ├── binary_release.py ├── NOTICE └── setup.py /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /curator/defaults/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /curator/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = '5.3.0b1' 2 | 3 | -------------------------------------------------------------------------------- /curator/validators/__init__.py: -------------------------------------------------------------------------------- 1 | from .schemacheck import SchemaCheck 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | 4 | [bdist_wheel] 5 | universal=1 6 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Fixes # 2 | 3 | ## Proposed Changes 4 | 5 | - 6 | - 7 | - 8 | -------------------------------------------------------------------------------- /curator/curator_cli.py: -------------------------------------------------------------------------------- 1 | import click 2 | from .singletons import cli 3 | 4 | def main(): 5 | cli(obj={}) 6 | -------------------------------------------------------------------------------- /unix_packages/cx_freeze-5.0.1.dev.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tes/curator/master/unix_packages/cx_freeze-5.0.1.dev.tar.gz -------------------------------------------------------------------------------- /curator/__main__.py: -------------------------------------------------------------------------------- 1 | 2 | """Executed when package directory is called as a script""" 3 | 4 | from .curator import main 5 | main() 6 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | urllib3>=1.20 2 | elasticsearch>=5.4.0,<6.0.0 3 | click>=6.7 4 | pyyaml>=3.10 5 | voluptuous>=0.9.3 6 | certifi>=2017.7.27.1 -------------------------------------------------------------------------------- /docs/utilities.rst: -------------------------------------------------------------------------------- 1 | .. _utilities: 2 | 3 | Utility & Helper Methods 4 | ======================== 5 | 6 | .. automodule:: curator.utils 7 | :members: 8 | 9 | .. autoclass:: curator.SchemaCheck 10 | :members: 11 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Docker Definition for ElasticSearch Curator 2 | 3 | FROM python:2.7.8-slim 4 | MAINTAINER Christian R. Vozar 5 | 6 | RUN pip install --quiet elasticsearch-curator 7 | 8 | ENTRYPOINT [ "/usr/local/bin/curator" ] 9 | -------------------------------------------------------------------------------- /curator/validators/config_file.py: -------------------------------------------------------------------------------- 1 | from voluptuous import * 2 | from ..defaults import client_defaults 3 | 4 | def client(): 5 | return Schema( 6 | { 7 | Optional('client'): client_defaults.config_client(), 8 | Optional('logging'): client_defaults.config_logging(), 9 | } 10 | ) 11 | -------------------------------------------------------------------------------- /curator/__init__.py: -------------------------------------------------------------------------------- 1 | from ._version import __version__ 2 | from .exceptions import * 3 | from .defaults import * 4 | from .validators import * 5 | from .logtools import * 6 | from .utils import * 7 | from .indexlist import IndexList 8 | from .snapshotlist import SnapshotList 9 | from .actions import * 10 | from .cli import * 11 | from .repomgrcli import * 12 | -------------------------------------------------------------------------------- /docs/objectclasses.rst: -------------------------------------------------------------------------------- 1 | .. _objectclasses: 2 | 3 | Object Classes 4 | ============== 5 | 6 | * `IndexList`_ 7 | * `SnapshotList`_ 8 | 9 | 10 | IndexList 11 | --------- 12 | 13 | .. autoclass:: curator.indexlist.IndexList 14 | :members: 15 | 16 | SnapshotList 17 | ------------ 18 | 19 | .. autoclass:: curator.snapshotlist.SnapshotList 20 | :members: 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | *~ 3 | *.py[co] 4 | build 5 | dist 6 | scratch 7 | docs/_build 8 | *.egg 9 | .eggs 10 | elasticsearch_curator.egg-info 11 | elasticsearch_curator_dev.egg-info 12 | .coverage 13 | .idea 14 | coverage.xml 15 | nosetests.xml 16 | index.html 17 | docs/asciidoc/html_docs 18 | wheelhouse 19 | Elastic.ico 20 | vcruntime140.dll 21 | msi_guid.txt 22 | Vagrant/centos/6/.vagrant 23 | Vagrant/centos/7/.vagrant 24 | Vagrant/ubuntu/14.04/.vagrant 25 | -------------------------------------------------------------------------------- /docs/asciidoc/inc_unit_table.asciidoc: -------------------------------------------------------------------------------- 1 | <> are calculated as follows: 2 | 3 | [width="50%", cols=" and contributors. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | python: 4 | - "2.7" 5 | - "3.4" 6 | - "3.5" 7 | - "3.6" 8 | 9 | env: 10 | - ES_VERSION=5.0.2 11 | - ES_VERSION=5.1.2 12 | - ES_VERSION=5.2.2 13 | - ES_VERSION=5.3.3 14 | - ES_VERSION=5.4.3 15 | - ES_VERSION=5.5.2 16 | - ES_VERSION=5.6.3 17 | 18 | os: linux 19 | 20 | cache: 21 | pip: true 22 | 23 | jdk: 24 | - oraclejdk8 25 | 26 | install: 27 | - pip install -r requirements.txt 28 | - pip install . 29 | 30 | script: 31 | - sudo apt-get update && sudo apt-get install oracle-java8-installer 32 | - java -version 33 | - sudo update-alternatives --set java /usr/lib/jvm/java-8-oracle/jre/bin/java 34 | - java -version 35 | - ./travis-run.sh 36 | -------------------------------------------------------------------------------- /docs/asciidoc/index.asciidoc: -------------------------------------------------------------------------------- 1 | :curator_version: 5.3.0b1 2 | :curator_major: 5 3 | :curator_doc_tree: 5.3 4 | :es_py_version: 5.4.0 5 | :es_doc_tree: 5.6 6 | :pybuild_ver: 3.6.3 7 | :ref: http://www.elastic.co/guide/en/elasticsearch/reference/{es_doc_tree} 8 | 9 | [[curator-reference]] 10 | = Curator Reference 11 | 12 | include::about.asciidoc[] 13 | include::versions.asciidoc[] 14 | include::installation.asciidoc[] 15 | include::command-line.asciidoc[] 16 | include::configuration.asciidoc[] 17 | include::actions.asciidoc[] 18 | include::options.asciidoc[] 19 | include::filters.asciidoc[] 20 | include::filter_elements.asciidoc[] 21 | include::examples.asciidoc[] 22 | include::security.asciidoc[] 23 | include::faq.asciidoc[] 24 | -------------------------------------------------------------------------------- /examples/actions/create_index.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: create_index 11 | description: Create the index as named, with the specified extra settings. 12 | options: 13 | name: myindex 14 | extra_settings: 15 | settings: 16 | number_of_shards: 2 17 | number_of_replicas: 1 18 | timeout_override: 19 | continue_if_exception: False 20 | disable_action: True 21 | -------------------------------------------------------------------------------- /docs/asciidoc/inc_filter_by_aliases.asciidoc: -------------------------------------------------------------------------------- 1 | [IMPORTANT] 2 | .API Change in Elasticsearch 5.5.0 3 | ============================ 4 | https://www.elastic.co/guide/en/elasticsearch/reference/5.5/breaking-changes-5.5.html#breaking_55_rest_changes[An update to Elasticsearch 5.5.0 changes the behavior of this filter, differing from previous 5.x versions]. 5 | 6 | If a list of <> is provided (instead of only one), indices 7 | must appear in _all_ listed <> or a 404 error will result, 8 | leading to no indices being matched. In older versions, if the index was 9 | associated with even one of the aliases in <>, it would 10 | result in a match. 11 | ============================ 12 | -------------------------------------------------------------------------------- /Vagrant/centos/6/Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure(2) do |config| 5 | config.vm.box = "elastic/centos-6-x86_64" 6 | 7 | config.vm.provision "shell", inline: <<-SHELL 8 | sudo yum -y groupinstall "Development Tools" 9 | sudo yum -y install python-devel zlib-devel bzip2-devel sqlite sqlite-devel openssl-devel 10 | SHELL 11 | 12 | config.vm.synced_folder "/curator_packages", "/curator_packages", create: true, owner: "vagrant", group: "vagrant" 13 | config.vm.synced_folder "/curator_source", "/curator_source", create: true, owner: "vagrant", group: "vagrant" 14 | 15 | config.vm.provider "virtualbox" do |v| 16 | v.customize ["modifyvm", :id, "--nictype1", "virtio"] 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /Vagrant/centos/7/Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure(2) do |config| 5 | config.vm.box = "elastic/centos-7-x86_64" 6 | 7 | config.vm.provision "shell", inline: <<-SHELL 8 | sudo yum -y groupinstall "Development Tools" 9 | sudo yum -y install python-devel zlib-devel bzip2-devel sqlite sqlite-devel openssl-devel 10 | SHELL 11 | 12 | config.vm.synced_folder "/curator_packages", "/curator_packages", create: true, owner: "vagrant", group: "vagrant" 13 | config.vm.synced_folder "/curator_source", "/curator_source", create: true, owner: "vagrant", group: "vagrant" 14 | 15 | config.vm.provider "virtualbox" do |v| 16 | v.customize ["modifyvm", :id, "--nictype1", "virtio"] 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /Vagrant/ubuntu/14.04/Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure(2) do |config| 5 | config.vm.box = "ubuntu/trusty64" 6 | 7 | config.vm.provision "shell", inline: <<-SHELL 8 | sudo apt-get -y autoremove 9 | sudo apt-get update 10 | sudo apt-get install -y libxml2-dev zlib1g-dev pkg-config python-dev make build-essential libssl-dev libbz2-dev libsqlite3-dev 11 | SHELL 12 | 13 | config.vm.synced_folder "/curator_packages", "/curator_packages", create: true, owner: "vagrant", group: "vagrant" 14 | config.vm.synced_folder "/curator_source", "/curator_source", create: true, owner: "vagrant", group: "vagrant" 15 | 16 | config.vm.provider "virtualbox" do |v| 17 | v.customize ["modifyvm", :id, "--nictype1", "virtio"] 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /test/run_tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function 3 | 4 | import sys 5 | from os.path import dirname, abspath 6 | 7 | import nose 8 | 9 | def run_all(argv=None): 10 | sys.exitfunc = lambda: sys.stderr.write('Shutting down....\n') 11 | 12 | # always insert coverage when running tests through setup.py 13 | if argv is None: 14 | argv = [ 15 | 'nosetests', '--with-xunit', 16 | '--logging-format=%(levelname)s %(name)22s %(funcName)22s:%(lineno)-4d %(message)s', 17 | '--with-xcoverage', '--cover-package=curator', '--cover-erase', 18 | '--verbose', 19 | ] 20 | 21 | nose.run_exit( 22 | argv=argv, 23 | defaultTest=abspath(dirname(__file__)) 24 | ) 25 | 26 | if __name__ == '__main__': 27 | run_all(sys.argv) 28 | -------------------------------------------------------------------------------- /examples/actions/close.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: close 11 | description: >- 12 | Close indices older than 30 days (based on index name), for logstash- 13 | prefixed indices. 14 | options: 15 | delete_aliases: False 16 | timeout_override: 17 | continue_if_exception: False 18 | disable_action: True 19 | filters: 20 | - filtertype: pattern 21 | kind: prefix 22 | value: logstash- 23 | exclude: 24 | - filtertype: age 25 | source: name 26 | direction: older 27 | timestring: '%Y.%m.%d' 28 | unit: days 29 | unit_count: 30 30 | exclude: 31 | -------------------------------------------------------------------------------- /examples/actions/delete_snapshots.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: delete_snapshots 11 | description: >- 12 | Delete snapshots from the selected repository older than 45 days 13 | (based on creation_date), for 'curator-' prefixed snapshots. 14 | options: 15 | repository: 16 | timeout_override: 17 | continue_if_exception: False 18 | disable_action: True 19 | filters: 20 | - filtertype: pattern 21 | kind: prefix 22 | value: curator- 23 | exclude: 24 | - filtertype: age 25 | source: creation_date 26 | direction: older 27 | unit: days 28 | unit_count: 45 29 | exclude: 30 | -------------------------------------------------------------------------------- /examples/actions/replicas.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: replicas 11 | description: >- 12 | Reduce the replica count to 0 for logstash- prefixed indices older than 13 | 10 days (based on index creation_date) 14 | options: 15 | count: 0 16 | wait_for_completion: False 17 | timeout_override: 18 | continue_if_exception: False 19 | disable_action: True 20 | filters: 21 | - filtertype: pattern 22 | kind: prefix 23 | value: logstash- 24 | exclude: 25 | - filtertype: age 26 | source: creation_date 27 | direction: older 28 | unit: days 29 | unit_count: 10 30 | exclude: 31 | -------------------------------------------------------------------------------- /examples/actions/allocation.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: allocation 11 | description: >- 12 | Apply shard allocation routing to 'require' 'tag=cold' for hot/cold node 13 | setup for logstash- indices older than 3 days, based on index_creation 14 | date 15 | options: 16 | key: tag 17 | value: cold 18 | allocation_type: require 19 | wait_for_completion: False 20 | continue_if_exception: False 21 | disable_action: True 22 | filters: 23 | - filtertype: pattern 24 | kind: prefix 25 | value: logstash- 26 | exclude: 27 | - filtertype: age 28 | source: creation_date 29 | direction: older 30 | unit: days 31 | unit_count: 3 32 | exclude: 33 | -------------------------------------------------------------------------------- /run_curator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Wrapper for running curator from source.""" 4 | 5 | from curator.cli import cli 6 | 7 | if __name__ == '__main__': 8 | try: 9 | cli() 10 | except Exception as e: 11 | if type(e) == type(RuntimeError()): 12 | if 'ASCII' in str(e): 13 | print('{0}'.format(e)) 14 | print( 15 | ''' 16 | 17 | When used with Python 3 (and the DEB and RPM packages of Curator are compiled 18 | and bundled with Python 3), Curator requires the locale to be unicode. Any of 19 | the above unicode definitions are acceptable. 20 | 21 | To set the locale to be unicode, try: 22 | 23 | $ export LC_ALL=en_US.utf8 24 | $ curator [ARGS] 25 | 26 | Alternately, you should be able to specify the locale on the command-line: 27 | 28 | $ LC_ALL=en_US.utf8 curator [ARGS] 29 | 30 | Be sure to substitute your unicode variant for en_US.utf8 31 | 32 | ''' 33 | ) 34 | else: 35 | import sys 36 | print('{0}'.format(e)) 37 | sys.exit(1) 38 | -------------------------------------------------------------------------------- /examples/actions/delete_indices.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: delete_indices 11 | description: >- 12 | Delete indices older than 45 days (based on index name), for logstash- 13 | prefixed indices. Ignore the error if the filter does not result in an 14 | actionable list of indices (ignore_empty_list) and exit cleanly. 15 | options: 16 | ignore_empty_list: True 17 | timeout_override: 18 | continue_if_exception: False 19 | disable_action: True 20 | filters: 21 | - filtertype: pattern 22 | kind: prefix 23 | value: logstash- 24 | exclude: 25 | - filtertype: age 26 | source: name 27 | direction: older 28 | timestring: '%Y.%m.%d' 29 | unit: days 30 | unit_count: 45 31 | exclude: 32 | -------------------------------------------------------------------------------- /docs/asciidoc/inc_filter_chaining.asciidoc: -------------------------------------------------------------------------------- 1 | [NOTE] 2 | .Filter chaining 3 | ===================================================================== 4 | It is important to note that while filters can be chained, each is linked by an 5 | implied logical *AND* operation. If you want to match from one of several 6 | different patterns, as with a logical *OR* operation, you can do so with the 7 | <> filtertype using _regex_ as the <>. 8 | 9 | This example shows how to select multiple indices based on them beginning with 10 | either `alpha-`, `bravo-`, or `charlie-`: 11 | 12 | [source,yaml] 13 | ------------- 14 | filters: 15 | - filtertype: pattern 16 | kind: regex 17 | value: '^(alpha-|bravo-|charlie-).*$' 18 | ------------- 19 | 20 | Explaining all of the different ways in which regular expressions can be used 21 | is outside the scope of this document, but hopefully this gives you some idea 22 | of how a regular expression pattern can be used when a logical *OR* is desired. 23 | ===================================================================== 24 | -------------------------------------------------------------------------------- /run_singleton.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Wrapper for running singletons from source.""" 4 | import click 5 | from curator.singletons import cli 6 | 7 | if __name__ == '__main__': 8 | try: 9 | cli(obj={}) 10 | except Exception as e: 11 | if type(e) == type(RuntimeError()): 12 | if 'ASCII' in str(e): 13 | print('{0}'.format(e)) 14 | print( 15 | ''' 16 | 17 | When used with Python 3 (and the DEB and RPM packages of Curator are compiled 18 | and bundled with Python 3), Curator requires the locale to be unicode. Any of 19 | the above unicode definitions are acceptable. 20 | 21 | To set the locale to be unicode, try: 22 | 23 | $ export LC_ALL=en_US.utf8 24 | $ curator_cli [ARGS] 25 | 26 | Alternately, you should be able to specify the locale on the command-line: 27 | 28 | $ LC_ALL=en_US.utf8 curator_cli [ARGS] 29 | 30 | Be sure to substitute your unicode variant for en_US.utf8 31 | 32 | ''' 33 | ) 34 | else: 35 | import sys 36 | print('{0}'.format(e)) 37 | sys.exit(1) 38 | -------------------------------------------------------------------------------- /run_es_repo_mgr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Wrapper for running es_repo_mgr from source.""" 4 | 5 | from curator.repomgrcli import repo_mgr_cli 6 | 7 | if __name__ == '__main__': 8 | try: 9 | repo_mgr_cli() 10 | except Exception as e: 11 | if type(e) == type(RuntimeError()): 12 | if 'ASCII' in str(e): 13 | print('{0}'.format(e)) 14 | print( 15 | ''' 16 | 17 | When used with Python 3 (and the DEB and RPM packages of Curator are compiled 18 | and bundled with Python 3), Curator requires the locale to be unicode. Any of 19 | the above unicode definitions are acceptable. 20 | 21 | To set the locale to be unicode, try: 22 | 23 | $ export LC_ALL=en_US.utf8 24 | $ es_repo_mgr [ARGS] 25 | 26 | Alternately, you should be able to specify the locale on the command-line: 27 | 28 | $ LC_ALL=en_US.utf8 es_repo_mgr [ARGS] 29 | 30 | Be sure to substitute your unicode variant for en_US.utf8 31 | 32 | ''' 33 | ) 34 | else: 35 | import sys 36 | print('{0}'.format(e)) 37 | sys.exit(1) 38 | -------------------------------------------------------------------------------- /examples/actions/open.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: open 11 | description: >- 12 | Open indices older than 30 days but younger than 60 days (based on index 13 | name), for logstash- prefixed indices. 14 | options: 15 | timeout_override: 16 | continue_if_exception: False 17 | disable_action: True 18 | filters: 19 | - filtertype: pattern 20 | kind: prefix 21 | value: logstash- 22 | exclude: 23 | - filtertype: age 24 | source: name 25 | direction: older 26 | timestring: '%Y.%m.%d' 27 | unit: days 28 | unit_count: 30 29 | exclude: 30 | - filtertype: age 31 | source: name 32 | direction: younger 33 | timestring: '%Y.%m.%d' 34 | unit: days 35 | unit_count: 60 36 | exclude: 37 | -------------------------------------------------------------------------------- /.github/issue_template.md: -------------------------------------------------------------------------------- 1 | ## For usage questions and help 2 | Please create a topic at https://discuss.elastic.co/c/elasticsearch 3 | 4 | Perhaps a topic there already has an answer for you! 5 | 6 | ## To submit a bug or report an issue 7 | 8 | 9 | ### Expected Behavior 10 | 11 | 12 | ### Actual Behavior 13 | 14 | 15 | ### Steps to Reproduce the Problem 16 | 17 | 18 | 1. 19 | 1. 20 | 1. 21 | 22 | ### Specifications 23 | 24 | - Version: 25 | - Platform: 26 | - Subsystem: 27 | 28 | 29 | ## Context (Environment) 30 | 31 | 32 | 33 | 34 | 35 | ## Detailed Description 36 | 37 | -------------------------------------------------------------------------------- /examples/actions/forcemerge.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: forcemerge 11 | description: >- 12 | forceMerge logstash- prefixed indices older than 2 days (based on index 13 | creation_date) to 2 segments per shard. Delay 120 seconds between each 14 | forceMerge operation to allow the cluster to quiesce. 15 | This action will ignore indices already forceMerged to the same or fewer 16 | number of segments per shard, so the 'forcemerged' filter is unneeded. 17 | options: 18 | max_num_segments: 2 19 | delay: 120 20 | timeout_override: 21 | continue_if_exception: False 22 | disable_action: True 23 | filters: 24 | - filtertype: pattern 25 | kind: prefix 26 | value: logstash- 27 | exclude: 28 | - filtertype: age 29 | source: creation_date 30 | direction: older 31 | unit: days 32 | unit_count: 2 33 | exclude: 34 | -------------------------------------------------------------------------------- /curator/exceptions.py: -------------------------------------------------------------------------------- 1 | class CuratorException(Exception): 2 | """ 3 | Base class for all exceptions raised by Curator which are not Elasticsearch 4 | exceptions. 5 | """ 6 | 7 | class ConfigurationError(CuratorException): 8 | """ 9 | Exception raised when a misconfiguration is detected 10 | """ 11 | 12 | class MissingArgument(CuratorException): 13 | """ 14 | Exception raised when a needed argument is not passed. 15 | """ 16 | 17 | class NoIndices(CuratorException): 18 | """ 19 | Exception raised when an operation is attempted against an empty index_list 20 | """ 21 | 22 | class NoSnapshots(CuratorException): 23 | """ 24 | Exception raised when an operation is attempted against an empty snapshot_list 25 | """ 26 | 27 | class ActionError(CuratorException): 28 | """ 29 | Exception raised when an action (against an index_list or snapshot_list) cannot be taken. 30 | """ 31 | 32 | class FailedExecution(CuratorException): 33 | """ 34 | Exception raised when an action fails to execute for some reason. 35 | """ 36 | 37 | class SnapshotInProgress(ActionError): 38 | """ 39 | Exception raised when a snapshot is already in progress 40 | """ 41 | 42 | class ActionTimeout(CuratorException): 43 | """ 44 | Exception raised when an action fails to complete in the allotted time 45 | """ -------------------------------------------------------------------------------- /examples/actions/snapshot.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: snapshot 11 | description: >- 12 | Snapshot logstash- prefixed indices older than 1 day (based on index 13 | creation_date) with the default snapshot name pattern of 14 | 'curator-%Y%m%d%H%M%S'. Wait for the snapshot to complete. Do not skip 15 | the repository filesystem access check. Use the other options to create 16 | the snapshot. 17 | options: 18 | repository: 19 | # Leaving name blank will result in the default 'curator-%Y%m%d%H%M%S' 20 | name: 21 | ignore_unavailable: False 22 | include_global_state: True 23 | partial: False 24 | wait_for_completion: True 25 | skip_repo_fs_check: False 26 | timeout_override: 27 | continue_if_exception: False 28 | disable_action: False 29 | filters: 30 | - filtertype: pattern 31 | kind: prefix 32 | value: logstash- 33 | exclude: 34 | - filtertype: age 35 | source: creation_date 36 | direction: older 37 | unit: days 38 | unit_count: 1 39 | exclude: 40 | -------------------------------------------------------------------------------- /test/unit/test_action_create_index.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import Mock, patch 3 | import elasticsearch 4 | import curator 5 | # Get test variables and constants from a single source 6 | from . import testvars as testvars 7 | 8 | class TestActionCreate_index(TestCase): 9 | def test_init_raise(self): 10 | self.assertRaises(TypeError, curator.CreateIndex, 'invalid') 11 | def test_init_raise_no_name(self): 12 | client = Mock() 13 | self.assertRaises(curator.ConfigurationError, 14 | curator.CreateIndex, client, None) 15 | def test_init(self): 16 | client = Mock() 17 | co = curator.CreateIndex(client, 'name') 18 | self.assertEqual('name', co.name) 19 | self.assertEqual(client, co.client) 20 | def test_do_dry_run(self): 21 | client = Mock() 22 | co = curator.CreateIndex(client, 'name') 23 | self.assertIsNone(co.do_dry_run()) 24 | def test_do_action(self): 25 | client = Mock() 26 | client.indices.create.return_value = None 27 | co = curator.CreateIndex(client, 'name') 28 | self.assertIsNone(co.do_action()) 29 | def test_do_action_raises_exception(self): 30 | client = Mock() 31 | client.indices.create.return_value = None 32 | client.indices.create.side_effect = testvars.fake_fail 33 | co = curator.CreateIndex(client, 'name') 34 | self.assertRaises(curator.FailedExecution, co.do_action) 35 | -------------------------------------------------------------------------------- /docs/asciidoc/inc_sources.asciidoc: -------------------------------------------------------------------------------- 1 | === `name`-based ages 2 | 3 | Using `name` as the `source` tells Curator to look for a 4 | <> within the index or snapshot name, and convert 5 | that into an epoch timestamp (epoch implies UTC). 6 | 7 | [source,yaml] 8 | ------------- 9 | - filtertype: age 10 | source: name 11 | direction: older 12 | timestring: '%Y.%m.%d' 13 | unit: days 14 | unit_count: 3 15 | ------------- 16 | 17 | include::inc_timestring_regex.asciidoc[] 18 | 19 | === `creation_date`-based ages 20 | 21 | `creation_date` extracts the epoch time of index or snapshot creation. 22 | 23 | [source,yaml] 24 | ------------- 25 | - filtertype: age 26 | source: creation_date 27 | direction: older 28 | unit: days 29 | unit_count: 3 30 | ------------- 31 | 32 | === `field_stats`-based ages 33 | 34 | NOTE: `source` can only be `field_stats` when filtering indices. 35 | 36 | `field_stats` uses the {ref}/search-field-stats.html[Field Stats API] to 37 | calculate either the `min_value` or the `max_value` of the <> 38 | as the <>, and then use that value for age 39 | comparisons. 40 | 41 | <> must be of type `date` in Elasticsearch. 42 | 43 | [source,yaml] 44 | ------------- 45 | - filtertype: age 46 | source: field_stats 47 | direction: older 48 | unit: days 49 | unit_count: 3 50 | field: '@timestamp' 51 | stats_result: min_value 52 | ------------- 53 | -------------------------------------------------------------------------------- /examples/actions/restore.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: restore 11 | description: >- 12 | Restore all indices in the most recent curator-* snapshot with state 13 | SUCCESS. Wait for the restore to complete before continuing. Do not skip 14 | the repository filesystem access check. Use the other options to define 15 | the index/shard settings for the restore. 16 | options: 17 | repository: 18 | # Leaving name blank will result in restoring the most recent snapshot by age 19 | name: 20 | # Leaving indices blank will result in restoring all indices in the snapshot 21 | indices: 22 | include_aliases: False 23 | ignore_unavailable: False 24 | include_global_state: True 25 | partial: False 26 | rename_pattern: 27 | rename_replacement: 28 | extra_settings: 29 | wait_for_completion: True 30 | skip_repo_fs_check: False 31 | timeout_override: 32 | continue_if_exception: False 33 | disable_action: False 34 | filters: 35 | - filtertype: pattern 36 | kind: prefix 37 | value: curator- 38 | exclude: 39 | - filtertype: state 40 | state: SUCCESS 41 | exclude: 42 | -------------------------------------------------------------------------------- /docs/asciidoc/inc_timestring_regex.asciidoc: -------------------------------------------------------------------------------- 1 | [WARNING] 2 | .A word about regular expression matching with timestrings 3 | ========================================================== 4 | Timestrings are parsed from strftime patterns, like `%Y.%m.%d`, into regular 5 | expressions. For example, `%Y` is 4 digits, so the regular expression for that 6 | looks like `\d{4}`, and `%m` is 2 digits, so the regular expression is `\d{2}`. 7 | 8 | What this means is that a simple timestring to match year and month, `%Y.%m` 9 | will result in a regular expression like this: `^.*\d{4}\.\d{2}.*$`. This 10 | pattern will match any 4 digits, followed by a period `.`, followed by 2 digits, 11 | occurring anywhere in the index name. This means it _will_ match monthly 12 | indices, like `index-2016.12`, as well as daily indices, like 13 | `index-2017.04.01`, which may not be the intended behavior. 14 | 15 | To compensate for this, when selecting indices matching a subset of another 16 | pattern, use a second filter with `exclude` set to `True` 17 | 18 | [source,yaml] 19 | ------------- 20 | - filtertype: pattern 21 | kind: timestring 22 | value: '%Y.%m' 23 | - filtertype: pattern 24 | kind: timestring 25 | value: '%Y.%m.%d' 26 | exclude: True 27 | ------------- 28 | 29 | This will prevent the `%Y.%m` pattern from matching the `%Y.%m` part of the 30 | daily indices. 31 | 32 | *This applies whether using `timestring` as a mere pattern match, or as part of 33 | date calculations.* 34 | ========================================================== 35 | -------------------------------------------------------------------------------- /examples/actions/alias.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: alias 11 | description: >- 12 | Alias indices older than 7 days but newer than 14 days, with a prefix of 13 | logstash- to 'last_week', remove indices older than 14 days. 14 | options: 15 | name: last_week 16 | extra_settings: 17 | timeout_override: 18 | continue_if_exception: False 19 | disable_action: True 20 | add: 21 | filters: 22 | - filtertype: pattern 23 | kind: prefix 24 | value: logstash- 25 | exclude: 26 | - filtertype: age 27 | source: name 28 | direction: older 29 | timestring: '%Y.%m.%d' 30 | unit: days 31 | unit_count: 7 32 | exclude: 33 | - filtertype: age 34 | direction: younger 35 | timestring: '%Y.%m.%d' 36 | unit: days 37 | unit_count: 14 38 | exclude: 39 | remove: 40 | filters: 41 | - filtertype: pattern 42 | kind: prefix 43 | value: logstash- 44 | exclude: 45 | - filtertype: age 46 | source: name 47 | direction: older 48 | timestring: '%Y.%m.%d' 49 | unit: days 50 | unit_count: 14 51 | exclude: 52 | -------------------------------------------------------------------------------- /examples/actions/shrink.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: shrink 11 | description: >- 12 | Shrink logstash indices older than 2 days on the node with the most 13 | available space, excluding the node named 'not_this_node'. 14 | Delete each source index after successful shrink, then reroute the shrunk 15 | index with the provided parameters. 16 | options: 17 | disable_action: False 18 | ignore_empty_list: True 19 | shrink_node: DETERMINISTIC 20 | node_filters: 21 | permit_masters: False 22 | exclude_nodes: ['not_this_node'] 23 | number_of_shards: 1 24 | number_of_replicas: 1 25 | shrink_prefix: 26 | shrink_suffix: '-shrink' 27 | copy_aliases: True 28 | delete_after: True 29 | post_allocation: 30 | allocation_type: include 31 | key: node_tag 32 | value: cold 33 | wait_for_active_shards: 1 34 | extra_settings: 35 | settings: 36 | index.codec: best_compression 37 | wait_for_completion: True 38 | wait_interval: 9 39 | max_wait: -1 40 | filters: 41 | - filtertype: pattern 42 | kind: prefix 43 | value: test_shrink- 44 | - filtertype: age 45 | source: creation_date 46 | direction: older 47 | unit: days 48 | unit_count: 2 -------------------------------------------------------------------------------- /docs/filters.rst: -------------------------------------------------------------------------------- 1 | .. _filters: 2 | 3 | Filter Methods 4 | ============== 5 | 6 | * `IndexList`_ 7 | * `SnapshotList`_ 8 | 9 | IndexList 10 | --------- 11 | 12 | .. automethod:: curator.indexlist.IndexList.filter_allocated 13 | :noindex: 14 | 15 | .. automethod:: curator.indexlist.IndexList.filter_by_age 16 | :noindex: 17 | 18 | .. automethod:: curator.indexlist.IndexList.filter_by_regex 19 | :noindex: 20 | 21 | .. automethod:: curator.indexlist.IndexList.filter_by_space 22 | :noindex: 23 | 24 | .. automethod:: curator.indexlist.IndexList.filter_closed 25 | :noindex: 26 | 27 | .. automethod:: curator.indexlist.IndexList.filter_forceMerged 28 | :noindex: 29 | 30 | .. automethod:: curator.indexlist.IndexList.filter_kibana 31 | :noindex: 32 | 33 | .. automethod:: curator.indexlist.IndexList.filter_opened 34 | :noindex: 35 | 36 | .. automethod:: curator.indexlist.IndexList.filter_none 37 | :noindex: 38 | 39 | .. automethod:: curator.indexlist.IndexList.filter_by_alias 40 | :noindex: 41 | 42 | .. automethod:: curator.indexlist.IndexList.filter_by_count 43 | :noindex: 44 | 45 | .. automethod:: curator.indexlist.IndexList.filter_period 46 | :noindex: 47 | 48 | 49 | SnapshotList 50 | ------------ 51 | 52 | .. automethod:: curator.snapshotlist.SnapshotList.filter_by_age 53 | :noindex: 54 | 55 | .. automethod:: curator.snapshotlist.SnapshotList.filter_by_regex 56 | :noindex: 57 | 58 | .. automethod:: curator.snapshotlist.SnapshotList.filter_by_state 59 | :noindex: 60 | 61 | .. automethod:: curator.snapshotlist.SnapshotList.filter_none 62 | :noindex: 63 | 64 | .. automethod:: curator.snapshotlist.SnapshotList.filter_by_count 65 | :noindex: 66 | 67 | .. automethod:: curator.snapshotlist.SnapshotList.filter_period 68 | :noindex: -------------------------------------------------------------------------------- /test/unit/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import tempfile 4 | import random 5 | import string 6 | from unittest import SkipTest, TestCase 7 | from mock import Mock 8 | from .testvars import * 9 | 10 | class CLITestCase(TestCase): 11 | def setUp(self): 12 | super(CLITestCase, self).setUp() 13 | self.args = {} 14 | dirname = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) 15 | ymlname = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) 16 | badyaml = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) 17 | # This will create a psuedo-random temporary directory on the machine 18 | # which runs the unit tests, but NOT on the machine where elasticsearch 19 | # is running. This means tests may fail if run against remote instances 20 | # unless you explicitly set `self.args['location']` to a proper spot 21 | # on the target machine. 22 | self.args['tmpdir'] = tempfile.mkdtemp(suffix=dirname) 23 | if not os.path.exists(self.args['tmpdir']): 24 | os.makedirs(self.args['tmpdir']) 25 | self.args['yamlfile'] = os.path.join(self.args['tmpdir'], ymlname) 26 | self.args['invalid_yaml'] = os.path.join(self.args['tmpdir'], badyaml) 27 | self.args['no_file_here'] = os.path.join(self.args['tmpdir'], 'not_created') 28 | with open(self.args['yamlfile'], 'w') as f: 29 | f.write(testvars.yamlconfig) 30 | with open(self.args['invalid_yaml'], 'w') as f: 31 | f.write('gobbledeygook: @failhere\n') 32 | 33 | def tearDown(self): 34 | if os.path.exists(self.args['tmpdir']): 35 | shutil.rmtree(self.args['tmpdir']) 36 | -------------------------------------------------------------------------------- /test/unit/test_action_rollover.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import Mock, patch 3 | import elasticsearch 4 | import curator 5 | # Get test variables and constants from a single source 6 | from . import testvars as testvars 7 | 8 | class TestActionRollover(TestCase): 9 | def test_init_raise_bad_client(self): 10 | self.assertRaises( 11 | TypeError, curator.Rollover, 'invalid', 'name', {}) 12 | def test_init_raise_bad_conditions(self): 13 | client = Mock() 14 | client.info.return_value = {'version': {'number': '5.0.0'} } 15 | self.assertRaises( 16 | curator.ConfigurationError, curator.Rollover, client, 'name', 'string') 17 | def test_init_raise_bad_extra_settings(self): 18 | client = Mock() 19 | client.info.return_value = {'version': {'number': '5.0.0'} } 20 | self.assertRaises( 21 | curator.ConfigurationError, curator.Rollover, client, 'name', 22 | {'a':'b'}, None, 'string') 23 | def test_init_raise_non_rollable_index(self): 24 | client = Mock() 25 | client.info.return_value = {'version': {'number': '5.0.0'} } 26 | client.indices.get_alias.return_value = testvars.alias_retval 27 | self.assertRaises( 28 | ValueError, curator.Rollover, client, testvars.named_alias, 29 | {'a':'b'}) 30 | def test_do_dry_run(self): 31 | client = Mock() 32 | client.info.return_value = {'version': {'number': '5.0.0'} } 33 | client.indices.get_alias.return_value = testvars.rollable_alias 34 | client.indices.rollover.return_value = testvars.dry_run_rollover 35 | ro = curator.Rollover( 36 | client, testvars.named_alias, testvars.rollover_conditions) 37 | self.assertIsNone(ro.do_dry_run()) 38 | -------------------------------------------------------------------------------- /curator/validators/filters.py: -------------------------------------------------------------------------------- 1 | from voluptuous import * 2 | from ..defaults import settings, filtertypes 3 | from ..exceptions import ConfigurationError 4 | from . import SchemaCheck 5 | import logging 6 | logger = logging.getLogger(__name__) 7 | 8 | def filtertype(): 9 | return { 10 | Required('filtertype'): Any( 11 | In(settings.all_filtertypes()), 12 | msg='filtertype must be one of {0}'.format( 13 | settings.all_filtertypes() 14 | ) 15 | ) 16 | } 17 | 18 | def structure(): 19 | # This is to first ensure that only the possible keys/filter elements are 20 | # there, and get a dictionary back to work with. 21 | retval = settings.structural_filter_elements() 22 | retval.update(filtertype()) 23 | return Schema(retval) 24 | 25 | def single(action, data): 26 | try: 27 | ft = data['filtertype'] 28 | except KeyError: 29 | raise ConfigurationError('Missing key "filtertype"') 30 | f = filtertype() 31 | for each in getattr(filtertypes, ft)(action, data): 32 | f.update(each) 33 | return Schema(f) 34 | 35 | def Filters(action, location=None): 36 | def f(v): 37 | def prune_nones(mydict): 38 | return dict([(k,v) for k, v in mydict.items() if v != None and v != 'None']) 39 | # This validator method simply validates all filters in the list. 40 | for idx in range(0, len(v)): 41 | pruned = prune_nones(v[idx]) 42 | filter_dict = SchemaCheck( 43 | pruned, 44 | single(action, pruned), 45 | 'filter', 46 | '{0}, filter #{1}: {2}'.format(location, idx, pruned) 47 | ).result() 48 | logger.debug('Filter #{0}: {1}'.format(idx, filter_dict)) 49 | v[idx] = filter_dict 50 | # If we've made it here without raising an Exception, it's valid 51 | return v 52 | return f 53 | -------------------------------------------------------------------------------- /curator/validators/actions.py: -------------------------------------------------------------------------------- 1 | from voluptuous import * 2 | from ..defaults import settings 3 | from . import SchemaCheck 4 | 5 | ### Schema information ### 6 | # Actions: root level 7 | def root(): 8 | return Schema({ Required('actions'): dict }) 9 | 10 | def valid_action(): 11 | return { 12 | Required('action'): Any( 13 | In(settings.all_actions()), 14 | msg='action must be one of {0}'.format( 15 | settings.all_actions() 16 | ) 17 | ) 18 | } 19 | 20 | # Basic action structure 21 | def structure(data, location): 22 | # Validate the action type first, so we can use it for other tests 23 | valid_action_type = SchemaCheck( 24 | data, 25 | Schema(valid_action(), extra=True), 26 | 'action type', 27 | location, 28 | ).result() 29 | # Build a valid schema knowing that the action has already been validated 30 | retval = valid_action() 31 | retval.update( 32 | { 33 | Optional('description', default='No description given'): Any( 34 | str, unicode 35 | ) 36 | } 37 | ) 38 | retval.update( 39 | { Optional('options', default=settings.default_options()): dict } ) 40 | action = data['action'] 41 | if action in [ 'cluster_routing', 'create_index', 'rollover']: 42 | # The cluster_routing, create_index, and rollover actions should not 43 | # have a 'filters' block 44 | pass 45 | elif action == 'alias': 46 | # The alias action should not have a filters block, but should have 47 | # an add and/or remove block. 48 | retval.update( 49 | { 50 | Optional('add'): dict, 51 | Optional('remove'): dict, 52 | } 53 | ) 54 | else: 55 | retval.update( 56 | { Optional('filters', default=settings.default_filters()): list } 57 | ) 58 | return Schema(retval) 59 | -------------------------------------------------------------------------------- /curator/config_utils.py: -------------------------------------------------------------------------------- 1 | from voluptuous import Schema 2 | from .validators import SchemaCheck, config_file 3 | from .utils import * 4 | from .logtools import LogInfo, Whitelist, Blacklist 5 | 6 | def test_config(config): 7 | # Get config from yaml file 8 | yaml_config = get_yaml(config) 9 | # if the file is empty, which is still valid yaml, set as an empty dict 10 | yaml_config = {} if not yaml_config else prune_nones(yaml_config) 11 | # Voluptuous can't verify the schema of a dict if it doesn't have keys, 12 | # so make sure the keys are at least there and are dict() 13 | for k in ['client', 'logging']: 14 | if k not in yaml_config: 15 | yaml_config[k] = {} 16 | else: 17 | yaml_config[k] = prune_nones(yaml_config[k]) 18 | return SchemaCheck(yaml_config, config_file.client(), 19 | 'Client Configuration', 'full configuration dictionary').result() 20 | 21 | def set_logging(log_opts): 22 | try: 23 | from logging import NullHandler 24 | except ImportError: 25 | from logging import Handler 26 | 27 | class NullHandler(Handler): 28 | def emit(self, record): 29 | pass 30 | 31 | # Set up logging 32 | loginfo = LogInfo(log_opts) 33 | logging.root.addHandler(loginfo.handler) 34 | logging.root.setLevel(loginfo.numeric_log_level) 35 | logger = logging.getLogger('curator.cli') 36 | # Set up NullHandler() to handle nested elasticsearch.trace Logger 37 | # instance in elasticsearch python client 38 | logging.getLogger('elasticsearch.trace').addHandler(NullHandler()) 39 | if log_opts['blacklist']: 40 | for bl_entry in ensure_list(log_opts['blacklist']): 41 | for handler in logging.root.handlers: 42 | handler.addFilter(Blacklist(bl_entry)) 43 | 44 | def process_config(yaml_file): 45 | config = test_config(yaml_file) 46 | set_logging(config['logging']) 47 | test_client_options(config['client']) 48 | return config['client'] 49 | -------------------------------------------------------------------------------- /curator/defaults/client_defaults.py: -------------------------------------------------------------------------------- 1 | from voluptuous import * 2 | 3 | # Configuration file: client 4 | def config_client(): 5 | return { 6 | Optional('hosts', default='127.0.0.1'): Any(None, str, unicode, list), 7 | Optional('port', default=9200): Any( 8 | None, All(Coerce(int), Range(min=1, max=65535)) 9 | ), 10 | Optional('url_prefix', default=''): Any(None, str, unicode), 11 | Optional('use_ssl', default=False): Boolean(), 12 | Optional('certificate', default=None): Any(None, str, unicode), 13 | Optional('client_cert', default=None): Any(None, str, unicode), 14 | Optional('client_key', default=None): Any(None, str, unicode), 15 | Optional('aws_key', default=None): Any(None, str, unicode), 16 | Optional('aws_secret_key', default=None): Any(None, str, unicode), 17 | Optional('aws_token', default=None): Any(None, str, unicode), 18 | Optional('aws_sign_request', default=False): Boolean(), 19 | Optional('aws_region'): Any(None, str, unicode), 20 | Optional('ssl_no_validate', default=False): Boolean(), 21 | Optional('http_auth', default=None): Any(None, str, unicode), 22 | Optional('timeout', default=30): All( 23 | Coerce(int), Range(min=1, max=86400)), 24 | Optional('master_only', default=False): Boolean(), 25 | } 26 | 27 | # Configuration file: logging 28 | def config_logging(): 29 | return { 30 | Optional( 31 | 'loglevel', default='INFO'): Any(None, 32 | 'NOTSET', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL', 33 | All(Coerce(int), Any(0, 10, 20, 30, 40, 50)) 34 | ), 35 | Optional('logfile', default=None): Any(None, str, unicode), 36 | Optional( 37 | 'logformat', default='default'): Any(None, All( 38 | Any(str, unicode), 39 | Any('default', 'json', 'logstash') 40 | ) 41 | ), 42 | Optional( 43 | 'blacklist', default=['elasticsearch', 'urllib3']): Any(None, list), 44 | } -------------------------------------------------------------------------------- /docs/asciidoc/inc_kinds.asciidoc: -------------------------------------------------------------------------------- 1 | The different <> are described as follows: 2 | 3 | === prefix 4 | 5 | To match all indices starting with `logstash-`: 6 | 7 | [source,yaml] 8 | ------------- 9 | - filtertype: pattern 10 | kind: prefix 11 | value: logstash- 12 | ------------- 13 | 14 | To match all indices _except_ those starting with `logstash-`: 15 | 16 | [source,yaml] 17 | ------------- 18 | - filtertype: pattern 19 | kind: prefix 20 | value: logstash- 21 | exclude: True 22 | ------------- 23 | 24 | === suffix 25 | 26 | To match all indices ending with `-prod`: 27 | 28 | [source,yaml] 29 | ------------- 30 | - filtertype: pattern 31 | kind: suffix 32 | value: -prod 33 | ------------- 34 | 35 | To match all indices _except_ those ending with `-prod`: 36 | 37 | [source,yaml] 38 | ------------- 39 | - filtertype: pattern 40 | kind: suffix 41 | value: -prod 42 | exclude: True 43 | ------------- 44 | 45 | === timestring 46 | 47 | IMPORTANT: No age calculation takes place here. It is strictly a pattern match. 48 | 49 | To match all indices with a Year.month.day pattern, like `index-2017.04.01`: 50 | 51 | [source,yaml] 52 | ------------- 53 | - filtertype: pattern 54 | kind: timestring 55 | value: '%Y.%m.%d' 56 | ------------- 57 | 58 | To match all indices _except_ those with a Year.month.day pattern, like 59 | `index-2017.04.01`: 60 | 61 | [source,yaml] 62 | ------------- 63 | - filtertype: pattern 64 | kind: timestring 65 | value: '%Y.%m.%d' 66 | exclude: True 67 | ------------- 68 | 69 | include::inc_timestring_regex.asciidoc[] 70 | 71 | === regex 72 | 73 | This <> allows you to design a regular-expression to match 74 | indices or snapshots: 75 | 76 | To match all indices starting with `a-`, `b-`, or `c-`: 77 | 78 | [source,yaml] 79 | ------------- 80 | - filtertype: pattern 81 | kind: regex 82 | value: '^a-|^b-|^c-' 83 | ------------- 84 | 85 | To match all indices _except_ those ending with `a-`, `b-`, or `c-`: 86 | 87 | [source,yaml] 88 | ------------- 89 | - filtertype: pattern 90 | kind: regex 91 | value: '^a-|^b-|^c-' 92 | exclude: True 93 | ------------- 94 | -------------------------------------------------------------------------------- /test/integration/test_clusterrouting.py: -------------------------------------------------------------------------------- 1 | import elasticsearch 2 | import curator 3 | import os 4 | import json 5 | import string, random, tempfile 6 | import click 7 | from click import testing as clicktest 8 | from mock import patch, Mock 9 | 10 | from . import CuratorTestCase 11 | from . import testvars as testvars 12 | 13 | import logging 14 | logger = logging.getLogger(__name__) 15 | 16 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 17 | port = int(port) if port else 9200 18 | 19 | 20 | class TestCLIClusterRouting(CuratorTestCase): 21 | def test_allocation_all(self): 22 | routing_type = 'allocation' 23 | value = 'all' 24 | self.write_config( 25 | self.args['configfile'], testvars.client_config.format(host, port)) 26 | self.write_config(self.args['actionfile'], 27 | testvars.cluster_routing_test.format(routing_type, value)) 28 | self.create_index('my_index') 29 | self.create_index('not_my_index') 30 | test = clicktest.CliRunner() 31 | result = test.invoke( 32 | curator.cli, 33 | [ 34 | '--config', self.args['configfile'], 35 | self.args['actionfile'] 36 | ], 37 | ) 38 | 39 | self.assertEquals(testvars.CRA_all, 40 | self.client.cluster.get_settings()) 41 | def test_extra_option(self): 42 | self.write_config( 43 | self.args['configfile'], testvars.client_config.format(host, port)) 44 | self.write_config(self.args['actionfile'], 45 | testvars.bad_option_proto_test.format('cluster_routing')) 46 | self.create_index('my_index') 47 | self.create_index('not_my_index') 48 | test = clicktest.CliRunner() 49 | result = test.invoke( 50 | curator.cli, 51 | [ 52 | '--config', self.args['configfile'], 53 | self.args['actionfile'] 54 | ], 55 | ) 56 | self.assertEqual(-1, result.exit_code) 57 | -------------------------------------------------------------------------------- /test/integration/test_forcemerge.py: -------------------------------------------------------------------------------- 1 | import elasticsearch 2 | import curator 3 | import os 4 | import json 5 | import string, random, tempfile 6 | import click 7 | from click import testing as clicktest 8 | from mock import patch, Mock 9 | 10 | from . import CuratorTestCase 11 | from . import testvars as testvars 12 | 13 | import logging 14 | logger = logging.getLogger(__name__) 15 | 16 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 17 | port = int(port) if port else 9200 18 | 19 | class TestCLIforceMerge(CuratorTestCase): 20 | def test_merge(self): 21 | count = 1 22 | idx = 'my_index' 23 | self.create_index(idx) 24 | self.add_docs(idx) 25 | ilo1 = curator.IndexList(self.client) 26 | ilo1._get_segmentcounts() 27 | self.assertEqual(3, ilo1.index_info[idx]['segments']) 28 | self.write_config( 29 | self.args['configfile'], testvars.client_config.format(host, port)) 30 | self.write_config(self.args['actionfile'], 31 | testvars.forcemerge_test.format(count, 0.20)) 32 | test = clicktest.CliRunner() 33 | result = test.invoke( 34 | curator.cli, 35 | [ 36 | '--config', self.args['configfile'], 37 | self.args['actionfile'] 38 | ], 39 | ) 40 | ilo2 = curator.IndexList(self.client) 41 | ilo2._get_segmentcounts() 42 | self.assertEqual(count, ilo2.index_info[idx]['segments']) 43 | def test_extra_option(self): 44 | self.write_config( 45 | self.args['configfile'], testvars.client_config.format(host, port)) 46 | self.write_config(self.args['actionfile'], 47 | testvars.bad_option_proto_test.format('forcemerge')) 48 | test = clicktest.CliRunner() 49 | result = test.invoke( 50 | curator.cli, 51 | [ 52 | '--config', self.args['configfile'], 53 | self.args['actionfile'] 54 | ], 55 | ) 56 | self.assertEqual(-1, result.exit_code) 57 | -------------------------------------------------------------------------------- /docs/asciidoc/versions.asciidoc: -------------------------------------------------------------------------------- 1 | [[versions]] 2 | = Versions 3 | 4 | [partintro] 5 | -- 6 | Elasticsearch Curator has been around for many different versions of 7 | Elasticsearch. The following document helps clarify which versions of Curator 8 | work with which versions of Elasticsearch. 9 | 10 | The current version of Curator is {curator_version} 11 | 12 | * <> 13 | -- 14 | 15 | [[version-compatibility]] 16 | == Version Compatibility 17 |   18 | 19 | IMPORTANT: Each listed version of Elasticsearch Curator has been fully tested 20 | against unmodified release versions of Elasticsearch. **Modified versions of Elasticsearch may not be fully supported.** 21 | 22 | The current version of Curator is {curator_version} 23 | 24 | [cols="<,<,<,<",options="header",grid="cols"] 25 | |=== 26 | |Curator Version 27 | |ES 1.x 28 | |ES 2.x 29 | |ES 5.x 30 | 31 | |          3 32 | |  ✅ footnoteref:[aws_ss,Curator is unable to make snapshots for modified versions of ES which do not allow access to the snapshot status API endpoint. As a result, Curator is unable to make snapshots in AWS ES.] 33 | |  ✅ footnoteref:[aws_ss] 34 | |  ❌ 35 | 36 | |          4 37 | |  ❌ 38 | |  ✅ footnote:[AWS ES (which is different from installing Elasticsearch on your own EC2 instances) version 5.3 officially supports Curator. If using an older version of AWS ES, please see the FAQ question, <>] 39 | |  ✅ footnote:[Not all of the APIs available in Elasticsearch 5 are available in Curator 4.] 40 | 41 | |          5 42 | |  ❌ 43 | |  ❌ 44 | |  ✅footnoteref:[aws_ss] 45 | |=== 46 | 47 | Learn more about the different versions at: 48 | 49 | * https://www.elastic.co/guide/en/elasticsearch/client/curator/3.5/index.html[Curator 3 Documentation] 50 | * https://www.elastic.co/guide/en/elasticsearch/client/curator/4.2/index.html[Curator 4 Documentation] 51 | * https://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html[Curator 5 Documentation] 52 | -------------------------------------------------------------------------------- /docs/actionclasses.rst: -------------------------------------------------------------------------------- 1 | .. _actionclasses: 2 | 3 | Action Classes 4 | ============== 5 | 6 | .. seealso:: It is important to note that each action has a `do_action()` 7 | method, which accepts no arguments. This is the means by which all 8 | actions are executed. 9 | 10 | * `Alias`_ 11 | * `Allocation`_ 12 | * `Close`_ 13 | * `ClusterRouting`_ 14 | * `CreateIndex`_ 15 | * `DeleteIndices`_ 16 | * `DeleteSnapshots`_ 17 | * `ForceMerge`_ 18 | * `IndexSettings`_ 19 | * `Open`_ 20 | * `Reindex`_ 21 | * `Replicas`_ 22 | * `Restore`_ 23 | * `Rollover`_ 24 | * `Shrink`_ 25 | * `Snapshot`_ 26 | 27 | 28 | Alias 29 | ----- 30 | .. autoclass:: curator.actions.Alias 31 | :members: 32 | 33 | Allocation 34 | ---------- 35 | .. autoclass:: curator.actions.Allocation 36 | :members: 37 | 38 | Close 39 | ----- 40 | .. autoclass:: curator.actions.Close 41 | :members: 42 | 43 | ClusterRouting 44 | -------------- 45 | .. autoclass:: curator.actions.ClusterRouting 46 | :members: 47 | 48 | CreateIndex 49 | -------------- 50 | .. autoclass:: curator.actions.CreateIndex 51 | :members: 52 | 53 | DeleteIndices 54 | ------------- 55 | .. autoclass:: curator.actions.DeleteIndices 56 | :members: 57 | 58 | DeleteSnapshots 59 | --------------- 60 | .. autoclass:: curator.actions.DeleteSnapshots 61 | :members: 62 | 63 | ForceMerge 64 | ---------- 65 | .. autoclass:: curator.actions.ForceMerge 66 | :members: 67 | 68 | IndexSettings 69 | -------------- 70 | .. autoclass:: curator.actions.IndexSettings 71 | :members: 72 | 73 | Open 74 | ---- 75 | .. autoclass:: curator.actions.Open 76 | :members: 77 | 78 | Reindex 79 | -------- 80 | .. autoclass:: curator.actions.Reindex 81 | :members: 82 | 83 | Replicas 84 | -------- 85 | .. autoclass:: curator.actions.Replicas 86 | :members: 87 | 88 | Restore 89 | -------- 90 | .. autoclass:: curator.actions.Restore 91 | :members: 92 | 93 | Rollover 94 | -------- 95 | .. autoclass:: curator.actions.Rollover 96 | :members: 97 | 98 | Shrink 99 | -------- 100 | .. autoclass:: curator.actions.Shrink 101 | :members: 102 | 103 | Snapshot 104 | -------- 105 | .. autoclass:: curator.actions.Snapshot 106 | :members: 107 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | The following is a list of people who have contributed ideas, code, bug 2 | reports, or in general have helped curator along its way. 3 | 4 | Contributors: 5 | * Jordan Sissel (jordansissel) (For Logstash, first and foremost) 6 | * Shay Banon (kimchy) (For Elasticsearch, of course!) 7 | * Aaron Mildenstein (untergeek) 8 | * Njal Karevoll 9 | * François Deppierraz 10 | * Honza Kral (HonzaKral) 11 | * Benjamin Smith (benjaminws) 12 | * Colin Moller (LeftyBC) 13 | * Elliot (edgeofnite) 14 | * Ram Viswanadha (ramv) 15 | * Chris Meisinger (cmeisinger) 16 | * Stuart Warren (stuart-warren) 17 | * (gitshaw) 18 | * (sfritz) 19 | * (sjoelsam) 20 | * Jose Diaz-Gonzalez (josegonzalez) 21 | * Arie Bro (arieb) 22 | * David Harrigan (dharrigan) 23 | * Mathieu Geli (gelim) 24 | * Nick Ethier (nickethier) 25 | * Mohab Usama (mohabusama) 26 | * (gitshaw) 27 | * Stuart Warren (stuart-warren) 28 | * Xavier Calland (xavier-calland) 29 | * Chad Schellenger (cschellenger) 30 | * Kamil Essekkat (ekamil) 31 | * (gbutt) 32 | * Ben Buchacher (bbuchacher) 33 | * Ehtesh Choudhury (shurane) 34 | * Markus Fischer (mfn) 35 | * Fabien Wernli (faxm0dem) 36 | * Michael Weiser (michaelweiser) 37 | * (digital-wonderland) 38 | * cassiano (cassianoleal) 39 | * Matt Dainty (bodgit) 40 | * Alex Philipp (alex-sf) 41 | * (krzaczek) 42 | * Justin Lintz (jlintz) 43 | * Jeremy Falling (jjfalling) 44 | * Ian Babrou (bobrik) 45 | * Ferenc Erki (ferki) 46 | * George Heppner (gheppner) 47 | * Matt Hughes (matthughes) 48 | * Brian Lalor (blalor) 49 | * Paweł Krzaczkowski (krzaczek) 50 | * Ben Tse (bt5e) 51 | * Tom Hendrikx (whyscream) 52 | * Christian Vozar (christianvozar) 53 | * Magnus Baeck (magnusbaeck) 54 | * Robin Kearney (rk295) 55 | * (cfeio) 56 | * (malagoli) 57 | * Dan Sheridan (djs52) 58 | * Michael-Keith Bernard (SegFaultAX) 59 | * Simon Lundström (simmel) 60 | * (pkr1234) 61 | * Mark Feltner (feltnerm) 62 | * William Jimenez (wjimenez5271) 63 | * Jeremy Canady (jrmycanady) 64 | * Steven Ottenhoff (steffo) 65 | * Ole Rößner (Basster) 66 | * Jack (univerio) 67 | * Tomáš Mózes (hydrapolic) 68 | * Gary Gao (garyelephant) 69 | * Panagiotis Moustafellos (pmoust) 70 | * (pbamba) 71 | * Pavel Strashkin (xaka) 72 | * Wadim Kruse (wkruse) 73 | * Richard Megginson (richm) 74 | * Thibaut Ackermann (thib-ack) 75 | * (zzugg) 76 | * Julien Mancuso (petitout) 77 | * Spencer Herzberg (sherzberg) 78 | * Luke Waite (lukewaite) 79 | * (dtrv) 80 | * Christopher "Chief" Najewicz (chiefy) 81 | * Filipe Gonçalves (basex) 82 | * Sönke Liebau (soenkeliebau) 83 | * Timothy Schroder (tschroeder-zendesk) 84 | * Jared Carey (jpcarey) 85 | * Juraj Seffer (jurajseffer) -------------------------------------------------------------------------------- /docs/examples.rst: -------------------------------------------------------------------------------- 1 | .. _examples: 2 | 3 | Examples 4 | ======== 5 | 6 | Each of these examples presupposes that the requisite modules have been imported 7 | and an instance of the Elasticsearch client object has been created: 8 | 9 | :: 10 | 11 | import elasticsearch 12 | import curator 13 | 14 | client = elasticsearch.Elasticsearch() 15 | 16 | Filter indices by prefix 17 | ++++++++++++++++++++++++ 18 | 19 | :: 20 | 21 | ilo = curator.IndexList(client) 22 | ilo.filter_by_regex(kind='prefix', value='logstash-') 23 | 24 | The contents of `ilo.indices` would then only be indices matching the `prefix`. 25 | 26 | 27 | Filter indices by suffix 28 | ++++++++++++++++++++++++ 29 | 30 | :: 31 | 32 | ilo = curator.IndexList(client) 33 | ilo.filter_by_regex(kind='suffix', value='-prod') 34 | 35 | The contents of `ilo.indices` would then only be indices matching the `suffix`. 36 | 37 | 38 | Filter indices by age (name) 39 | ++++++++++++++++++++++++++++ 40 | 41 | This example will match indices with the following criteria: 42 | 43 | * Have a date string of ``%Y.%m.%d`` 44 | * Use `days` as the unit of time measurement 45 | * Filter indices `older` than 5 `days` 46 | 47 | :: 48 | 49 | ilo = curator.IndexList(client) 50 | ilo.filter_by_age(source='name', direction='older', timestring='%Y.%m.%d', 51 | unit='days', unit_count=5 52 | ) 53 | 54 | The contents of `ilo.indices` would then only be indices matching these 55 | criteria. 56 | 57 | 58 | Filter indices by age (creation_date) 59 | +++++++++++++++++++++++++++++++++++++ 60 | 61 | This example will match indices with the following criteria: 62 | 63 | * Use `months` as the unit of time measurement 64 | * Filter indices where the index creation date is `older` than 2 `months` from 65 | this moment. 66 | 67 | :: 68 | 69 | ilo = curator.IndexList(client) 70 | ilo.filter_by_age(source='creation_date', direction='older', 71 | unit='months', unit_count=2 72 | ) 73 | 74 | The contents of `ilo.indices` would then only be indices matching these 75 | criteria. 76 | 77 | Filter indices by age (field_stats) 78 | +++++++++++++++++++++++++++++++++++ 79 | 80 | This example will match indices with the following criteria: 81 | 82 | * Use `days` as the unit of time measurement 83 | * Filter indices where the `timestamp` field's `min_value` is a date `older` 84 | than 3 `weeks` from this moment. 85 | 86 | 87 | :: 88 | 89 | ilo = curator.IndexList(client) 90 | ilo.filter_by_age(source='field_stats', direction='older', 91 | unit='weeks', unit_count=3, field='timestamp', stats_result='min_value' 92 | ) 93 | 94 | The contents of `ilo.indices` would then only be indices matching these 95 | criteria. 96 | -------------------------------------------------------------------------------- /test/integration/test_replicas.py: -------------------------------------------------------------------------------- 1 | import elasticsearch 2 | import curator 3 | import os 4 | import json 5 | import string, random, tempfile 6 | import click 7 | from click import testing as clicktest 8 | from mock import patch, Mock 9 | 10 | from . import CuratorTestCase 11 | from . import testvars as testvars 12 | 13 | import logging 14 | logger = logging.getLogger(__name__) 15 | 16 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 17 | port = int(port) if port else 9200 18 | 19 | class TestCLIReplicas(CuratorTestCase): 20 | def test_increase_count(self): 21 | count = 2 22 | idx = 'my_index' 23 | self.write_config( 24 | self.args['configfile'], testvars.client_config.format(host, port)) 25 | self.write_config(self.args['actionfile'], 26 | testvars.replicas_test.format(count)) 27 | self.create_index(idx) 28 | test = clicktest.CliRunner() 29 | result = test.invoke( 30 | curator.cli, 31 | [ 32 | '--config', self.args['configfile'], 33 | self.args['actionfile'] 34 | ], 35 | ) 36 | self.assertEqual( 37 | count, 38 | int(self.client.indices.get_settings( 39 | index=idx)[idx]['settings']['index']['number_of_replicas']) 40 | ) 41 | def test_no_count(self): 42 | self.create_index('foo') 43 | self.write_config( 44 | self.args['configfile'], testvars.client_config.format(host, port)) 45 | self.write_config(self.args['actionfile'], 46 | testvars.replicas_test.format(' ')) 47 | test = clicktest.CliRunner() 48 | result = test.invoke( 49 | curator.cli, 50 | [ 51 | '--config', self.args['configfile'], 52 | self.args['actionfile'] 53 | ], 54 | ) 55 | self.assertEqual(-1, result.exit_code) 56 | def test_extra_option(self): 57 | self.create_index('foo') 58 | self.write_config( 59 | self.args['configfile'], testvars.client_config.format(host, port)) 60 | self.write_config(self.args['actionfile'], 61 | testvars.bad_option_proto_test.format('replicas')) 62 | test = clicktest.CliRunner() 63 | result = test.invoke( 64 | curator.cli, 65 | [ 66 | '--config', self.args['configfile'], 67 | self.args['actionfile'] 68 | ], 69 | ) 70 | self.assertEqual(-1, result.exit_code) 71 | -------------------------------------------------------------------------------- /curator/logtools.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import json 3 | import logging 4 | import time 5 | 6 | class LogstashFormatter(logging.Formatter): 7 | # The LogRecord attributes we want to carry over to the Logstash message, 8 | # mapped to the corresponding output key. 9 | WANTED_ATTRS = {'levelname': 'loglevel', 10 | 'funcName': 'function', 11 | 'lineno': 'linenum', 12 | 'message': 'message', 13 | 'name': 'name'} 14 | 15 | # def converter(self, timevalue): 16 | # return time.gmtime(timevalue) 17 | 18 | def format(self, record): 19 | self.converter = time.gmtime 20 | timestamp = '%s.%03dZ' % ( 21 | self.formatTime(record, datefmt='%Y-%m-%dT%H:%M:%S'), record.msecs) 22 | result = {'message': record.getMessage(), 23 | '@timestamp': timestamp} 24 | for attribute in set(self.WANTED_ATTRS).intersection(record.__dict__): 25 | result[self.WANTED_ATTRS[attribute]] = getattr(record, attribute) 26 | return json.dumps(result, sort_keys=True) 27 | 28 | class Whitelist(logging.Filter): 29 | def __init__(self, *whitelist): 30 | self.whitelist = [logging.Filter(name) for name in whitelist] 31 | 32 | def filter(self, record): 33 | return any(f.filter(record) for f in self.whitelist) 34 | 35 | class Blacklist(Whitelist): 36 | def filter(self, record): 37 | return not Whitelist.filter(self, record) 38 | 39 | class LogInfo(object): 40 | def __init__(self, cfg): 41 | cfg['loglevel'] = 'INFO' if not 'loglevel' in cfg else cfg['loglevel'] 42 | cfg['logfile'] = None if not 'logfile' in cfg else cfg['logfile'] 43 | cfg['logformat'] = 'default' if not 'logformat' in cfg else cfg['logformat'] 44 | self.numeric_log_level = getattr(logging, cfg['loglevel'].upper(), None) 45 | self.format_string = '%(asctime)s %(levelname)-9s %(message)s' 46 | if not isinstance(self.numeric_log_level, int): 47 | raise ValueError('Invalid log level: {0}'.format(cfg['loglevel'])) 48 | 49 | self.handler = logging.StreamHandler( 50 | open(cfg['logfile'], 'a') if cfg['logfile'] else sys.stdout 51 | ) 52 | 53 | if self.numeric_log_level == 10: # DEBUG 54 | self.format_string = ( 55 | '%(asctime)s %(levelname)-9s %(name)22s ' 56 | '%(funcName)22s:%(lineno)-4d %(message)s' 57 | ) 58 | 59 | if cfg['logformat'] == 'json' or cfg['logformat'] == 'logstash': 60 | self.handler.setFormatter(LogstashFormatter()) 61 | else: 62 | self.handler.setFormatter(logging.Formatter(self.format_string)) 63 | -------------------------------------------------------------------------------- /test/unit/test_action_clusterrouting.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import Mock, patch 3 | import elasticsearch 4 | import curator 5 | # Get test variables and constants from a single source 6 | from . import testvars as testvars 7 | 8 | class TestActionAllocation(TestCase): 9 | def test_bad_client(self): 10 | self.assertRaises(TypeError, curator.ClusterRouting, 'invalid') 11 | def test_bad_setting(self): 12 | client = Mock() 13 | self.assertRaises( 14 | ValueError, curator.ClusterRouting, client, setting='invalid' 15 | ) 16 | def test_bad_routing_type(self): 17 | client = Mock() 18 | self.assertRaises( 19 | ValueError, 20 | curator.ClusterRouting, 21 | client, 22 | routing_type='invalid', 23 | setting='enable' 24 | ) 25 | def test_bad_value_with_allocation(self): 26 | client = Mock() 27 | self.assertRaises( 28 | ValueError, 29 | curator.ClusterRouting, 30 | client, 31 | routing_type='allocation', 32 | setting='enable', 33 | value='invalid' 34 | ) 35 | def test_bad_value_with_rebalance(self): 36 | client = Mock() 37 | self.assertRaises( 38 | ValueError, 39 | curator.ClusterRouting, 40 | client, 41 | routing_type='rebalance', 42 | setting='enable', 43 | value='invalid' 44 | ) 45 | def test_do_dry_run(self): 46 | client = Mock() 47 | cro = curator.ClusterRouting( 48 | client, 49 | routing_type='allocation', 50 | setting='enable', 51 | value='all' 52 | ) 53 | self.assertIsNone(cro.do_dry_run()) 54 | def test_do_action_raise_on_put_settings(self): 55 | client = Mock() 56 | client.cluster.put_settings.return_value = None 57 | client.cluster.put_settings.side_effect = testvars.fake_fail 58 | cro = curator.ClusterRouting( 59 | client, 60 | routing_type='allocation', 61 | setting='enable', 62 | value='all' 63 | ) 64 | self.assertRaises(Exception, cro.do_action) 65 | def test_do_action_wait(self): 66 | client = Mock() 67 | client.cluster.put_settings.return_value = None 68 | client.cluster.health.return_value = {'relocating_shards':0} 69 | cro = curator.ClusterRouting( 70 | client, 71 | routing_type='allocation', 72 | setting='enable', 73 | value='all', 74 | wait_for_completion=True 75 | ) 76 | self.assertIsNone(cro.do_action()) 77 | -------------------------------------------------------------------------------- /travis-run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | # There's at least 1 expected, skipped test, only with 5.0.0-alpha4 right now 5 | expected_skips=1 6 | 7 | setup_es() { 8 | download_url=$1 9 | curl -sL $download_url > elasticsearch.tar.gz 10 | mkdir elasticsearch 11 | tar -xzf elasticsearch.tar.gz --strip-components=1 -C ./elasticsearch/. 12 | } 13 | 14 | start_es() { 15 | jhome=$1 16 | es_args=$2 17 | es_port=$3 18 | es_cluster=$4 19 | export JAVA_HOME=$jhome 20 | elasticsearch/bin/elasticsearch $es_args > /tmp/$es_cluster.log & 21 | sleep 20 22 | curl http://127.0.0.1:$es_port && echo "$es_cluster Elasticsearch is up!" || cat /tmp/$es_cluster.log ./elasticsearch/logs/$es_cluster.log 23 | # curl http://127.0.0.1:$es_port && echo "ES is up!" || cat /tmp/$es_cluster.log ./elasticsearch/logs/$es_cluster.log 24 | } 25 | 26 | setup_es https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ES_VERSION.tar.gz 27 | 28 | java_home='/usr/lib/jvm/java-8-oracle' 29 | 30 | ### Build local cluster config (since 5.4 removed most flags) 31 | LC=elasticsearch/localcluster 32 | mkdir -p $LC 33 | cp elasticsearch/config/log4j2.properties $LC 34 | echo 'network.host: 127.0.0.1' > $LC/elasticsearch.yml 35 | echo 'http.port: 9200' >> $LC/elasticsearch.yml 36 | echo 'cluster.name: local' >> $LC/elasticsearch.yml 37 | echo 'node.max_local_storage_nodes: 2' >> $LC/elasticsearch.yml 38 | echo 'discovery.zen.ping.unicast.hosts: ["127.0.0.1:9200"]' >> $LC/elasticsearch.yml 39 | echo 'path.repo: /' >> $LC/elasticsearch.yml 40 | echo 'reindex.remote.whitelist: localhost:9201' >> $LC/elasticsearch.yml 41 | 42 | ### Build remote cluster config (since 5.4 removed most flags) 43 | RC=elasticsearch/remotecluster 44 | mkdir -p $RC 45 | cp elasticsearch/config/log4j2.properties $RC 46 | echo 'network.host: 127.0.0.1' > $RC/elasticsearch.yml 47 | echo 'http.port: 9201' >> $RC/elasticsearch.yml 48 | echo 'cluster.name: remote' >> $RC/elasticsearch.yml 49 | echo 'node.max_local_storage_nodes: 2' >> $RC/elasticsearch.yml 50 | echo 'discovery.zen.ping.unicast.hosts: ["127.0.0.1:9201"]' >> $RC/elasticsearch.yml 51 | 52 | 53 | start_es $java_home "-d -Epath.conf=$LC" 9200 "local" 54 | start_es $java_home "-d -Epath.conf=$RC" 9201 "remote" 55 | 56 | python setup.py test 57 | result=$(head -1 nosetests.xml | awk '{print $6 " " $7 " " $8}' | awk -F\> '{print $1}' | tr -d '"') 58 | echo "Result = $result" 59 | errors=$(echo $result | awk '{print $1}' | awk -F\= '{print $2}') 60 | failures=$(echo $result | awk '{print $2}' | awk -F\= '{print $2}') 61 | skips=$(echo $result | awk '{print $3}' | awk -F\= '{print $2}') 62 | if [[ $errors -gt 0 ]]; then 63 | exit 1 64 | elif [[ $failures -gt 0 ]]; then 65 | exit 1 66 | elif [[ $skips -gt $expected_skips ]]; then 67 | exit 1 68 | fi 69 | -------------------------------------------------------------------------------- /test/unit/test_action_open.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import Mock, patch 3 | import elasticsearch 4 | import curator 5 | # Get test variables and constants from a single source 6 | from . import testvars as testvars 7 | 8 | class TestActionOpen(TestCase): 9 | def test_init_raise(self): 10 | self.assertRaises(TypeError, curator.Open, 'invalid') 11 | def test_init(self): 12 | client = Mock() 13 | client.info.return_value = {'version': {'number': '5.0.0'} } 14 | client.indices.get_settings.return_value = testvars.settings_one 15 | client.cluster.state.return_value = testvars.clu_state_one 16 | client.indices.stats.return_value = testvars.stats_one 17 | ilo = curator.IndexList(client) 18 | oo = curator.Open(ilo) 19 | self.assertEqual(ilo, oo.index_list) 20 | self.assertEqual(client, oo.client) 21 | def test_do_dry_run(self): 22 | client = Mock() 23 | client.info.return_value = {'version': {'number': '5.0.0'} } 24 | client.indices.get_settings.return_value = testvars.settings_four 25 | client.cluster.state.return_value = testvars.clu_state_four 26 | client.indices.stats.return_value = testvars.stats_four 27 | client.indices.open.return_value = None 28 | ilo = curator.IndexList(client) 29 | ilo.filter_opened() 30 | oo = curator.Open(ilo) 31 | self.assertEqual([u'c-2016.03.05'], oo.index_list.indices) 32 | self.assertIsNone(oo.do_dry_run()) 33 | def test_do_action(self): 34 | client = Mock() 35 | client.info.return_value = {'version': {'number': '5.0.0'} } 36 | client.indices.get_settings.return_value = testvars.settings_four 37 | client.cluster.state.return_value = testvars.clu_state_four 38 | client.indices.stats.return_value = testvars.stats_four 39 | client.indices.open.return_value = None 40 | ilo = curator.IndexList(client) 41 | ilo.filter_opened() 42 | oo = curator.Open(ilo) 43 | self.assertEqual([u'c-2016.03.05'], oo.index_list.indices) 44 | self.assertIsNone(oo.do_action()) 45 | def test_do_action_raises_exception(self): 46 | client = Mock() 47 | client.info.return_value = {'version': {'number': '5.0.0'} } 48 | client.indices.get_settings.return_value = testvars.settings_four 49 | client.cluster.state.return_value = testvars.clu_state_four 50 | client.indices.stats.return_value = testvars.stats_four 51 | client.indices.open.return_value = None 52 | client.indices.open.side_effect = testvars.fake_fail 53 | ilo = curator.IndexList(client) 54 | oo = curator.Open(ilo) 55 | self.assertRaises(curator.FailedExecution, oo.do_action) 56 | -------------------------------------------------------------------------------- /test/unit/test_cli_methods.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import logging 3 | from unittest import TestCase 4 | from mock import Mock, patch, mock_open 5 | import elasticsearch 6 | import curator 7 | from curator import _version as __version__ 8 | from . import CLITestCase 9 | # Get test variables and constants from a single source 10 | from . import testvars as testvars 11 | 12 | class TestCLI_A(TestCase): 13 | def test_read_file_no_file(self): 14 | self.assertRaises(TypeError, curator.read_file) 15 | def test_loginfo_defaults(self): 16 | loginfo = curator.LogInfo({}) 17 | self.assertEqual(20, loginfo.numeric_log_level) 18 | self.assertEqual(testvars.default_format, loginfo.format_string) 19 | def test_loginfo_debug(self): 20 | loginfo = curator.LogInfo({"loglevel": "DEBUG"}) 21 | self.assertEqual(10, loginfo.numeric_log_level) 22 | self.assertEqual(testvars.debug_format, loginfo.format_string) 23 | def test_loginfo_bad_level_raises(self): 24 | self.assertRaises( 25 | ValueError, 26 | curator.LogInfo, {"loglevel": "NOTALOGLEVEL"} 27 | ) 28 | def test_loginfo_logstash_formatter(self): 29 | loginfo = curator.LogInfo({"logformat": "logstash"}) 30 | logging.root.addHandler(loginfo.handler) 31 | logging.root.setLevel(loginfo.numeric_log_level) 32 | logger = logging.getLogger('testing') 33 | logger.info('testing') 34 | self.assertEqual(20, loginfo.numeric_log_level) 35 | def test_client_options_certificate(self): 36 | a = {'use_ssl':True, 'certificate':'invalid_path'} 37 | self.assertRaises( 38 | curator.FailedExecution, 39 | curator.test_client_options, a 40 | ) 41 | def test_client_options_client_cert(self): 42 | a = {'use_ssl':True, 'client_cert':'invalid_path'} 43 | self.assertRaises( 44 | curator.FailedExecution, 45 | curator.test_client_options, a 46 | ) 47 | def test_client_options_client_key(self): 48 | a = {'use_ssl':True, 'client_key':'invalid_path'} 49 | self.assertRaises( 50 | curator.FailedExecution, 51 | curator.test_client_options, a 52 | ) 53 | 54 | class TestCLI_B(CLITestCase): 55 | def test_read_file_pass(self): 56 | cfg = curator.get_yaml(self.args['yamlfile']) 57 | self.assertEqual('localhost', cfg['client']['hosts']) 58 | self.assertEqual(9200, cfg['client']['port']) 59 | def test_read_file_corrupt_fail(self): 60 | self.assertRaises(curator.ConfigurationError, 61 | curator.get_yaml, self.args['invalid_yaml']) 62 | def test_read_file_missing_fail(self): 63 | self.assertRaises( 64 | curator.FailedExecution, 65 | curator.read_file, self.args['no_file_here'] 66 | ) 67 | -------------------------------------------------------------------------------- /curator/validators/schemacheck.py: -------------------------------------------------------------------------------- 1 | from voluptuous import * 2 | from ..exceptions import ConfigurationError 3 | import re 4 | import logging 5 | 6 | class SchemaCheck(object): 7 | def __init__(self, config, schema, test_what, location): 8 | """ 9 | Validate ``config`` with the provided voluptuous ``schema``. 10 | ``test_what`` and ``location`` are for reporting the results, in case of 11 | failure. If validation is successful, the method returns ``config`` as 12 | valid. 13 | 14 | :arg config: A configuration dictionary. 15 | :type config: dict 16 | :arg schema: A voluptuous schema definition 17 | :type schema: :class:`voluptuous.Schema` 18 | :arg test_what: which configuration block is being validated 19 | :type test_what: str 20 | :arg location: An string to report which configuration sub-block is 21 | being tested. 22 | :type location: str 23 | """ 24 | self.loggit = logging.getLogger('curator.validators.SchemaCheck') 25 | # Set the Schema for validation... 26 | self.loggit.debug('Schema: {0}'.format(schema)) 27 | self.loggit.debug('"{0}" config: {1}'.format(test_what, config)) 28 | self.config = config 29 | self.schema = schema 30 | self.test_what = test_what 31 | self.location = location 32 | 33 | def __parse_error(self): 34 | """ 35 | Report the error, and try to report the bad key or value as well. 36 | """ 37 | def get_badvalue(data_string, data): 38 | elements = re.sub('[\'\]]', '', data_string).split('[') 39 | elements.pop(0) # Get rid of data as the first element 40 | value = None 41 | for k in elements: 42 | try: 43 | key = int(k) 44 | except ValueError: 45 | key = k 46 | if value == None: 47 | value = data[key] 48 | # if this fails, it's caught below 49 | return value 50 | try: 51 | self.badvalue = get_badvalue(str(self.error).split()[-1], self.config) 52 | except: 53 | self.badvalue = '(could not determine)' 54 | 55 | def result(self): 56 | try: 57 | return self.schema(self.config) 58 | except Exception as e: 59 | try: 60 | self.error = e.errors[0] 61 | except: 62 | self.error = '{0}'.format(e) 63 | self.__parse_error() 64 | self.loggit.error('Schema error: {0}'.format(self.error)) 65 | raise ConfigurationError( 66 | 'Configuration: {0}: Location: {1}: Bad Value: "{2}", {3}. ' 67 | 'Check configuration file.'.format( 68 | self.test_what, self.location, self.badvalue, self.error) 69 | ) 70 | -------------------------------------------------------------------------------- /test/unit/test_action_delete_snapshots.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import Mock, patch 3 | import elasticsearch 4 | import curator 5 | # Get test variables and constants from a single source 6 | from . import testvars as testvars 7 | 8 | class TestActionDeleteSnapshots(TestCase): 9 | def test_init_raise(self): 10 | self.assertRaises(TypeError, curator.DeleteSnapshots, 'invalid') 11 | def test_init(self): 12 | client = Mock() 13 | client.snapshot.get.return_value = testvars.snapshots 14 | client.snapshot.get_repository.return_value = testvars.test_repo 15 | slo = curator.SnapshotList(client, repository=testvars.repo_name) 16 | do = curator.DeleteSnapshots(slo) 17 | self.assertEqual(slo, do.snapshot_list) 18 | self.assertEqual(client, do.client) 19 | def test_do_dry_run(self): 20 | client = Mock() 21 | client.snapshot.get.return_value = testvars.snapshots 22 | client.snapshot.get_repository.return_value = testvars.test_repo 23 | client.tasks.get.return_value = testvars.no_snap_tasks 24 | client.snapshot.delete.return_value = None 25 | slo = curator.SnapshotList(client, repository=testvars.repo_name) 26 | do = curator.DeleteSnapshots(slo) 27 | self.assertIsNone(do.do_dry_run()) 28 | def test_do_action(self): 29 | client = Mock() 30 | client.snapshot.get.return_value = testvars.snapshots 31 | client.snapshot.get_repository.return_value = testvars.test_repo 32 | client.tasks.get.return_value = testvars.no_snap_tasks 33 | client.snapshot.delete.return_value = None 34 | slo = curator.SnapshotList(client, repository=testvars.repo_name) 35 | do = curator.DeleteSnapshots(slo) 36 | self.assertIsNone(do.do_action()) 37 | def test_do_action_raises_exception(self): 38 | client = Mock() 39 | client.snapshot.get.return_value = testvars.snapshots 40 | client.snapshot.get_repository.return_value = testvars.test_repo 41 | client.snapshot.delete.return_value = None 42 | client.tasks.get.return_value = testvars.no_snap_tasks 43 | client.snapshot.delete.side_effect = testvars.fake_fail 44 | slo = curator.SnapshotList(client, repository=testvars.repo_name) 45 | do = curator.DeleteSnapshots(slo) 46 | self.assertRaises(curator.FailedExecution, do.do_action) 47 | def test_not_safe_to_snap_raises_exception(self): 48 | client = Mock() 49 | client.snapshot.get.return_value = testvars.inprogress 50 | client.snapshot.get_repository.return_value = testvars.test_repo 51 | client.tasks.get.return_value = testvars.no_snap_tasks 52 | slo = curator.SnapshotList(client, repository=testvars.repo_name) 53 | do = curator.DeleteSnapshots(slo, retry_interval=0, retry_count=1) 54 | self.assertRaises(curator.FailedExecution, do.do_action) 55 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Elasticsearch Curator Python API 2 | ================================ 3 | 4 | The Elasticsearch Curator Python API helps you manage your indices and 5 | snapshots. 6 | 7 | .. note:: 8 | 9 | This documentation is for the Elasticsearch Curator Python API. Documentation 10 | for the Elasticsearch Curator *CLI* -- which uses this API and is installed 11 | as an entry_point as part of the package -- is available in the 12 | `Elastic guide`_. 13 | 14 | .. _Elastic guide: http://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html 15 | 16 | Compatibility 17 | ------------- 18 | 19 | The Elasticsearch Curator Python API is compatible with the 5.x Elasticsearch versions, 20 | and supports Python versions 2.7 and later. 21 | 22 | Example Usage 23 | ------------- 24 | 25 | :: 26 | 27 | import elasticsearch 28 | import curator 29 | 30 | client = elasticsearch.Elasticsearch() 31 | 32 | ilo = curator.IndexList(client) 33 | ilo.filter_by_regex(kind='prefix', value='logstash-') 34 | ilo.filter_by_age(source='name', direction='older', timestring='%Y.%m.%d', unit='days', unit_count=30) 35 | delete_indices = curator.DeleteIndices(ilo) 36 | delete_indices.do_action() 37 | 38 | .. TIP:: 39 | See more examples in the :doc:`Examples ` page. 40 | 41 | Features 42 | -------- 43 | 44 | The API methods fall into the following categories: 45 | 46 | * :doc:`Object Classes ` build and filter index list or snapshot list objects. 47 | * :doc:`Action Classes ` act on object classes. 48 | * :doc:`Utilities ` are helper methods. 49 | 50 | Logging 51 | ~~~~~~~ 52 | 53 | The Elasticsearch Curator Python API uses the standard `logging library`_ from Python. 54 | It inherits two loggers from ``elasticsearch-py``: ``elasticsearch`` and 55 | ``elasticsearch.trace``. Clients use the ``elasticsearch`` logger to log 56 | standard activity, depending on the log level. The ``elasticsearch.trace`` 57 | logger logs requests to the server in JSON format as pretty-printed ``curl`` 58 | commands that you can execute from the command line. The ``elasticsearch.trace`` 59 | logger is not inherited from the base logger and must be activated separately. 60 | 61 | .. _logging library: http://docs.python.org/3.6/library/logging.html 62 | 63 | Contents 64 | -------- 65 | 66 | .. toctree:: 67 | :maxdepth: 2 68 | 69 | objectclasses 70 | actionclasses 71 | filters 72 | utilities 73 | examples 74 | Changelog 75 | 76 | License 77 | ------- 78 | 79 | Copyright (c) 2012–2017 Elasticsearch 80 | 81 | Licensed under the Apache License, Version 2.0 (the "License"); 82 | you may not use this file except in compliance with the License. 83 | You may obtain a copy of the License at 84 | 85 | http://www.apache.org/licenses/LICENSE-2.0 86 | 87 | Unless required by applicable law or agreed to in writing, software 88 | distributed under the License is distributed on an "AS IS" BASIS, 89 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 90 | See the License for the specific language governing permissions and 91 | limitations under the License. 92 | 93 | 94 | Indices and tables 95 | ------------------ 96 | 97 | * :ref:`genindex` 98 | * :ref:`search` 99 | -------------------------------------------------------------------------------- /test/integration/test_envvars.py: -------------------------------------------------------------------------------- 1 | import elasticsearch 2 | import curator 3 | import os 4 | import json 5 | import string, random, tempfile 6 | import click 7 | from click import testing as clicktest 8 | from mock import patch, Mock 9 | 10 | from . import CuratorTestCase 11 | from . import testvars as testvars 12 | 13 | import logging 14 | logger = logging.getLogger(__name__) 15 | 16 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 17 | port = int(port) if port else 9200 18 | 19 | def random_envvar(size): 20 | return ''.join( 21 | random.SystemRandom().choice( 22 | string.ascii_uppercase + string.digits 23 | ) for _ in range(size) 24 | ) 25 | 26 | class TestEnvVars(CuratorTestCase): 27 | def test_present(self): 28 | evar = random_envvar(8) 29 | os.environ[evar] = "1234" 30 | dollar = '${' + evar + '}' 31 | self.write_config( 32 | self.args['configfile'], 33 | testvars.client_config_envvars.format(dollar, port, 30) 34 | ) 35 | cfg = curator.get_yaml(self.args['configfile']) 36 | self.assertEqual( 37 | cfg['client']['hosts'], 38 | os.environ.get(evar) 39 | ) 40 | del os.environ[evar] 41 | def test_not_present(self): 42 | evar = random_envvar(8) 43 | dollar = '${' + evar + '}' 44 | self.write_config( 45 | self.args['configfile'], 46 | testvars.client_config_envvars.format(dollar, port, 30) 47 | ) 48 | cfg = curator.get_yaml(self.args['configfile']) 49 | self.assertIsNone(cfg['client']['hosts']) 50 | def test_not_present_with_default(self): 51 | evar = random_envvar(8) 52 | default = random_envvar(8) 53 | dollar = '${' + evar + ':' + default + '}' 54 | self.write_config( 55 | self.args['configfile'], 56 | testvars.client_config_envvars.format(dollar, port, 30) 57 | ) 58 | cfg = curator.get_yaml(self.args['configfile']) 59 | self.assertEqual( 60 | cfg['client']['hosts'], 61 | default 62 | ) 63 | def test_do_something_with_int_value(self): 64 | self.create_indices(10) 65 | evar = random_envvar(8) 66 | os.environ[evar] = "1234" 67 | dollar = '${' + evar + '}' 68 | self.write_config( 69 | self.args['configfile'], 70 | testvars.client_config_envvars.format(host, port, dollar) 71 | ) 72 | cfg = curator.get_yaml(self.args['configfile']) 73 | self.assertEqual( 74 | cfg['client']['timeout'], 75 | os.environ.get(evar) 76 | ) 77 | self.write_config(self.args['actionfile'], 78 | testvars.delete_proto.format( 79 | 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ' 80 | ) 81 | ) 82 | test = clicktest.CliRunner() 83 | result = test.invoke( 84 | curator.cli, 85 | [ 86 | '--config', self.args['configfile'], 87 | self.args['actionfile'] 88 | ], 89 | ) 90 | self.assertEquals(5, len(curator.get_indices(self.client))) 91 | del os.environ[evar] 92 | -------------------------------------------------------------------------------- /test/integration/test_integrations.py: -------------------------------------------------------------------------------- 1 | import elasticsearch 2 | import curator 3 | import os 4 | import json 5 | import string, random, tempfile 6 | import click 7 | from click import testing as clicktest 8 | from mock import patch, Mock 9 | 10 | from . import CuratorTestCase 11 | from . import testvars as testvars 12 | 13 | import logging 14 | logger = logging.getLogger(__name__) 15 | 16 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 17 | port = int(port) if port else 9200 18 | 19 | class TestFilters(CuratorTestCase): 20 | def test_filter_by_alias(self): 21 | alias = 'testalias' 22 | self.write_config( 23 | self.args['configfile'], testvars.client_config.format(host, port)) 24 | self.write_config(self.args['actionfile'], 25 | testvars.filter_by_alias.format('testalias', False)) 26 | self.create_index('my_index') 27 | self.create_index('dummy') 28 | self.client.indices.put_alias(index='dummy', name=alias) 29 | test = clicktest.CliRunner() 30 | result = test.invoke( 31 | curator.cli, 32 | [ 33 | '--config', self.args['configfile'], 34 | self.args['actionfile'] 35 | ], 36 | ) 37 | self.assertEquals(1, len(curator.get_indices(self.client))) 38 | def test_filter_by_array_of_aliases(self): 39 | alias = 'testalias' 40 | self.write_config( 41 | self.args['configfile'], testvars.client_config.format(host, port)) 42 | self.write_config(self.args['actionfile'], 43 | testvars.filter_by_alias.format(' [ testalias, foo ]', False)) 44 | self.create_index('my_index') 45 | self.create_index('dummy') 46 | self.client.indices.put_alias(index='dummy', name=alias) 47 | test = clicktest.CliRunner() 48 | result = test.invoke( 49 | curator.cli, 50 | [ 51 | '--config', self.args['configfile'], 52 | self.args['actionfile'] 53 | ], 54 | ) 55 | ver = curator.get_version(self.client) 56 | if ver >= (5,5,0): 57 | self.assertEquals(2, len(curator.get_indices(self.client))) 58 | else: 59 | self.assertEquals(1, len(curator.get_indices(self.client))) 60 | def test_filter_by_alias_bad_aliases(self): 61 | alias = 'testalias' 62 | self.write_config( 63 | self.args['configfile'], testvars.client_config.format(host, port)) 64 | self.write_config(self.args['actionfile'], 65 | testvars.filter_by_alias.format('{"this":"isadict"}', False)) 66 | self.create_index('my_index') 67 | self.create_index('dummy') 68 | self.client.indices.put_alias(index='dummy', name=alias) 69 | test = clicktest.CliRunner() 70 | result = test.invoke( 71 | curator.cli, 72 | [ 73 | '--config', self.args['configfile'], 74 | self.args['actionfile'] 75 | ], 76 | ) 77 | self.assertEquals( 78 | type(curator.ConfigurationError()), type(result.exception)) 79 | self.assertEquals(2, len(curator.get_indices(self.client))) 80 | -------------------------------------------------------------------------------- /test/integration/test_count_pattern.py: -------------------------------------------------------------------------------- 1 | import elasticsearch 2 | import curator 3 | import os 4 | import json 5 | import string, random, tempfile 6 | import time 7 | from click import testing as clicktest 8 | from mock import patch, Mock 9 | import unittest 10 | from . import CuratorTestCase 11 | from . import testvars as testvars 12 | 13 | import logging 14 | logger = logging.getLogger(__name__) 15 | 16 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 17 | port = int(port) if port else 9200 18 | # ' - filtertype: {0}\n' 19 | # ' source: {1}\n' 20 | # ' direction: {2}\n' 21 | # ' timestring: {3}\n' 22 | # ' unit: {4}\n' 23 | # ' unit_count: {5}\n' 24 | # ' field: {6}\n' 25 | # ' stats_result: {7}\n' 26 | # ' epoch: {8}\n') 27 | 28 | global_client = elasticsearch.Elasticsearch(host=host, port=port) 29 | 30 | delete_count_pattern = ('---\n' 31 | 'actions:\n' 32 | ' 1:\n' 33 | ' description: "Delete indices as filtered"\n' 34 | ' action: delete_indices\n' 35 | ' options:\n' 36 | ' continue_if_exception: False\n' 37 | ' disable_action: False\n' 38 | ' filters:\n' 39 | ' - filtertype: count\n' 40 | ' pattern: {0}\n' 41 | ' use_age: {1}\n' 42 | ' source: {2}\n' 43 | ' timestring: {3}\n' 44 | ' reverse: {4}\n' 45 | ' count: {5}\n') 46 | 47 | class TestCLICountPattern(CuratorTestCase): 48 | def test_match_proper_indices(self): 49 | for i in range(1, 4): 50 | self.create_index('a-{0}'.format(i)) 51 | for i in range(4, 7): 52 | self.create_index('b-{0}'.format(i)) 53 | for i in range(5, 9): 54 | self.create_index('c-{0}'.format(i)) 55 | self.create_index('not_a_match') 56 | self.write_config( 57 | self.args['configfile'], testvars.client_config.format(host, port)) 58 | self.write_config( 59 | self.args['actionfile'], 60 | delete_count_pattern.format( 61 | '\'^(a|b|c)-\d$\'', 'false', 'name', '\'%Y.%m.%d\'', 'true', 1 62 | ) 63 | ) 64 | test = clicktest.CliRunner() 65 | result = test.invoke( 66 | curator.cli, 67 | [ 68 | '--config', self.args['configfile'], 69 | self.args['actionfile'] 70 | ], 71 | ) 72 | indices = sorted(list(self.client.indices.get('_all'))) 73 | self.assertEquals(['a-3', 'b-6', 'c-8', 'not_a_match'], indices) 74 | def test_match_proper_indices_by_age(self): 75 | self.create_index('a-2017.10.01') 76 | self.create_index('a-2017.10.02') 77 | self.create_index('a-2017.10.03') 78 | self.create_index('b-2017.09.01') 79 | self.create_index('b-2017.09.02') 80 | self.create_index('b-2017.09.03') 81 | self.create_index('not_a_match') 82 | self.write_config( 83 | self.args['configfile'], testvars.client_config.format(host, port)) 84 | self.write_config( 85 | self.args['actionfile'], 86 | delete_count_pattern.format( 87 | '\'^(a|b)-\d{4}\.\d{2}\.\d{2}$\'', 'true', 'name', '\'%Y.%m.%d\'', 'true', 1 88 | ) 89 | ) 90 | test = clicktest.CliRunner() 91 | result = test.invoke( 92 | curator.cli, 93 | [ 94 | '--config', self.args['configfile'], 95 | self.args['actionfile'] 96 | ], 97 | ) 98 | indices = sorted(list(self.client.indices.get('_all'))) 99 | self.assertEquals(['a-2017.10.03', 'b-2017.09.03', 'not_a_match'], indices) -------------------------------------------------------------------------------- /test/unit/test_action_replicas.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import Mock, patch 3 | import elasticsearch 4 | import curator 5 | # Get test variables and constants from a single source 6 | from . import testvars as testvars 7 | 8 | class TestActionReplicas(TestCase): 9 | def test_init_raise_bad_client(self): 10 | self.assertRaises( 11 | TypeError, curator.Replicas, 'invalid', count=2) 12 | def test_init_raise_no_count(self): 13 | client = Mock() 14 | client.info.return_value = {'version': {'number': '5.0.0'} } 15 | client.indices.get_settings.return_value = testvars.settings_one 16 | client.cluster.state.return_value = testvars.clu_state_one 17 | client.indices.stats.return_value = testvars.stats_one 18 | ilo = curator.IndexList(client) 19 | self.assertRaises( 20 | curator.MissingArgument, curator.Replicas, ilo) 21 | def test_init(self): 22 | client = Mock() 23 | client.info.return_value = {'version': {'number': '5.0.0'} } 24 | client.indices.get_settings.return_value = testvars.settings_one 25 | client.cluster.state.return_value = testvars.clu_state_one 26 | client.indices.stats.return_value = testvars.stats_one 27 | client.indices.put_settings.return_value = None 28 | ilo = curator.IndexList(client) 29 | ro = curator.Replicas(ilo, count=2) 30 | self.assertEqual(ilo, ro.index_list) 31 | self.assertEqual(client, ro.client) 32 | def test_do_dry_run(self): 33 | client = Mock() 34 | client.info.return_value = {'version': {'number': '5.0.0'} } 35 | client.indices.get_settings.return_value = testvars.settings_one 36 | client.cluster.state.return_value = testvars.clu_state_one 37 | client.indices.stats.return_value = testvars.stats_one 38 | client.indices.put_settings.return_value = None 39 | ilo = curator.IndexList(client) 40 | ro = curator.Replicas(ilo, count=0) 41 | self.assertIsNone(ro.do_dry_run()) 42 | def test_do_action(self): 43 | client = Mock() 44 | client.info.return_value = {'version': {'number': '5.0.0'} } 45 | client.indices.get_settings.return_value = testvars.settings_one 46 | client.cluster.state.return_value = testvars.clu_state_one 47 | client.indices.stats.return_value = testvars.stats_one 48 | client.indices.put_settings.return_value = None 49 | ilo = curator.IndexList(client) 50 | ro = curator.Replicas(ilo, count=0) 51 | self.assertIsNone(ro.do_action()) 52 | def test_do_action_wait(self): 53 | client = Mock() 54 | client.info.return_value = {'version': {'number': '5.0.0'} } 55 | client.indices.get_settings.return_value = testvars.settings_one 56 | client.cluster.state.return_value = testvars.clu_state_one 57 | client.indices.stats.return_value = testvars.stats_one 58 | client.indices.put_settings.return_value = None 59 | client.cluster.health.return_value = {'status':'green'} 60 | ilo = curator.IndexList(client) 61 | ro = curator.Replicas(ilo, count=1, wait_for_completion=True) 62 | self.assertIsNone(ro.do_action()) 63 | def test_do_action_raises_exception(self): 64 | client = Mock() 65 | client.info.return_value = {'version': {'number': '5.0.0'} } 66 | client.indices.get_settings.return_value = testvars.settings_one 67 | client.cluster.state.return_value = testvars.clu_state_one 68 | client.indices.stats.return_value = testvars.stats_one 69 | client.indices.segments.return_value = testvars.shards 70 | client.indices.put_settings.return_value = None 71 | client.indices.put_settings.side_effect = testvars.fake_fail 72 | ilo = curator.IndexList(client) 73 | ro = curator.Replicas(ilo, count=2) 74 | self.assertRaises(curator.FailedExecution, ro.do_action) 75 | -------------------------------------------------------------------------------- /test/integration/test_delete_snapshots.py: -------------------------------------------------------------------------------- 1 | import elasticsearch 2 | import curator 3 | import os 4 | import json 5 | import string, random, tempfile 6 | import time 7 | from click import testing as clicktest 8 | from mock import patch, Mock 9 | 10 | from . import CuratorTestCase 11 | from . import testvars as testvars 12 | 13 | import logging 14 | logger = logging.getLogger(__name__) 15 | 16 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 17 | port = int(port) if port else 9200 18 | # ' repository: {0}\n' 19 | # ' - filtertype: {1}\n' 20 | # ' source: {2}\n' 21 | # ' direction: {3}\n' 22 | # ' timestring: {4}\n' 23 | # ' unit: {5}\n' 24 | # ' unit_count: {6}\n' 25 | # ' epoch: {7}\n') 26 | 27 | class TestCLIDeleteSnapshots(CuratorTestCase): 28 | def test_deletesnapshot(self): 29 | ### Create snapshots to delete and verify them 30 | self.create_repository() 31 | timestamps = [] 32 | for i in range(1,4): 33 | self.add_docs('my_index{0}'.format(i)) 34 | ilo = curator.IndexList(self.client) 35 | snap = curator.Snapshot(ilo, repository=self.args['repository'], 36 | name='curator-%Y%m%d%H%M%S', wait_interval=0.5 37 | ) 38 | snap.do_action() 39 | snapshot = curator.get_snapshot( 40 | self.client, self.args['repository'], '_all' 41 | ) 42 | self.assertEqual(i, len(snapshot['snapshots'])) 43 | time.sleep(1.0) 44 | timestamps.append(int(time.time())) 45 | time.sleep(1.0) 46 | ### Setup the actual delete 47 | self.write_config( 48 | self.args['configfile'], testvars.client_config.format(host, port)) 49 | self.write_config(self.args['actionfile'], 50 | testvars.delete_snap_proto.format( 51 | self.args['repository'], 'age', 'creation_date', 'older', ' ', 52 | 'seconds', '0', timestamps[0] 53 | ) 54 | ) 55 | test = clicktest.CliRunner() 56 | result = test.invoke( 57 | curator.cli, 58 | [ 59 | '--config', self.args['configfile'], 60 | self.args['actionfile'] 61 | ], 62 | ) 63 | snapshot = curator.get_snapshot( 64 | self.client, self.args['repository'], '_all' 65 | ) 66 | self.assertEqual(2, len(snapshot['snapshots'])) 67 | def test_no_repository(self): 68 | self.write_config( 69 | self.args['configfile'], testvars.client_config.format(host, port)) 70 | self.write_config(self.args['actionfile'], 71 | testvars.delete_snap_proto.format( 72 | ' ', 'age', 'creation_date', 'older', ' ', 73 | 'seconds', '0', ' ' 74 | ) 75 | ) 76 | test = clicktest.CliRunner() 77 | result = test.invoke( 78 | curator.cli, 79 | [ 80 | '--config', self.args['configfile'], 81 | self.args['actionfile'] 82 | ], 83 | ) 84 | self.assertEqual(-1, result.exit_code) 85 | def test_extra_options(self): 86 | self.write_config( 87 | self.args['configfile'], testvars.client_config.format(host, port)) 88 | self.write_config(self.args['actionfile'], 89 | testvars.bad_option_proto_test.format('delete_snapshots')) 90 | test = clicktest.CliRunner() 91 | result = test.invoke( 92 | curator.cli, 93 | [ 94 | '--config', self.args['configfile'], 95 | self.args['actionfile'] 96 | ], 97 | ) 98 | self.assertEqual(-1, result.exit_code) 99 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Curator 2 | 3 | All contributions are welcome: ideas, patches, documentation, bug reports, 4 | complaints, etc! 5 | 6 | Programming is not a required skill, and there are many ways to help out! 7 | It is more important to us that you are able to contribute. 8 | 9 | That said, some basic guidelines, which you are free to ignore :) 10 | 11 | ## Want to learn? 12 | 13 | Want to write your own code to do something Curator doesn't do out of the box? 14 | 15 | * [Curator API Documentation](http://curator.readthedocs.io/) Since version 2.0, 16 | Curator ships with both an API and wrapper scripts (which are actually defined 17 | as entry points). This allows you to write your own scripts to accomplish 18 | similar goals, or even new and different things with the 19 | [Curator API](http://curator.readthedocs.io/), and the 20 | [Elasticsearch Python API](http://elasticsearch-py.readthedocs.io/). 21 | 22 | Want to know how to use the command-line interface (CLI)? 23 | 24 | * [Curator CLI Documentation](http://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html) 25 | The Curator CLI Documentation is now a part of the document repository at 26 | http://elastic.co/guide at 27 | http://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html 28 | 29 | Want to lurk about and see what others are doing with Curator? 30 | 31 | * The irc channels (#logstash and #elasticsearch on irc.freenode.org) are good 32 | places for this 33 | 34 | ## Got Questions? 35 | 36 | Have a problem you want Curator to solve for you? 37 | 38 | * You are welcome to join the IRC channel #logstash (or #elasticsearch) on 39 | irc.freenode.org and ask for help there! 40 | 41 | ## Have an Idea or Feature Request? 42 | 43 | * File a ticket on [github](https://github.com/elastic/curator/issues) 44 | 45 | ## Something Not Working? Found a Bug? 46 | 47 | If you think you found a bug, it probably is a bug. 48 | 49 | * File it on [github](https://github.com/elastic/curator/issues) 50 | 51 | # Contributing Documentation and Code Changes 52 | 53 | If you have a bugfix or new feature that you would like to contribute to 54 | Curator, and you think it will take more than a few minutes to produce the fix 55 | (ie; write code), it is worth discussing the change with the Curator users and 56 | developers first! You can reach us via 57 | [github](https://github.com/elastic/curator/issues), or via IRC (#logstash or 58 | #elasticsearch on freenode irc) 59 | 60 | Documentation is in two parts: API and CLI documentation. 61 | 62 | API documentation is generated from comments inside the classes and methods 63 | within the code. This documentation is rendered and hosted at 64 | http://curator.readthedocs.io 65 | 66 | CLI documentation is in Asciidoc format in the GitHub repository at 67 | https://github.com/elastic/curator/tree/master/docs/asciidoc. 68 | This documentation can be changed via a pull request as with any other code 69 | change. 70 | 71 | ## Contribution Steps 72 | 73 | 1. Test your changes! Run the test suite ('python setup.py test'). Please note 74 | that this requires an Elasticsearch instance. The tests will try to connect 75 | to your local elasticsearch instance and run integration tests against it. 76 | **This will delete all the data stored there!** You can use the env variable 77 | `TEST_ES_SERVER` to point to a different instance (for example 78 | 'otherhost:9203'). 79 | 2. Please make sure you have signed our [Contributor License 80 | Agreement](http://www.elastic.co/contributor-agreement/). We are not 81 | asking you to assign copyright to us, but to give us the right to distribute 82 | your code without restriction. We ask this of all contributors in order to 83 | assure our users of the origin and continuing existence of the code. You 84 | only need to sign the CLA once. 85 | 3. Send a pull request! Push your changes to your fork of the repository and 86 | [submit a pull 87 | request](https://help.github.com/articles/using-pull-requests). In the pull 88 | request, describe what your changes do and mention any bugs/issues related 89 | to the pull request. 90 | -------------------------------------------------------------------------------- /test/unit/test_action_close.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import Mock, patch 3 | import elasticsearch 4 | import curator 5 | # Get test variables and constants from a single source 6 | from . import testvars as testvars 7 | 8 | class TestActionClose(TestCase): 9 | def test_init_raise(self): 10 | self.assertRaises(TypeError, curator.Close, 'invalid') 11 | def test_init(self): 12 | client = Mock() 13 | client.info.return_value = {'version': {'number': '5.0.0'} } 14 | client.indices.get_settings.return_value = testvars.settings_one 15 | client.cluster.state.return_value = testvars.clu_state_one 16 | client.indices.stats.return_value = testvars.stats_one 17 | ilo = curator.IndexList(client) 18 | co = curator.Close(ilo) 19 | self.assertEqual(ilo, co.index_list) 20 | self.assertEqual(client, co.client) 21 | def test_do_dry_run(self): 22 | client = Mock() 23 | client.info.return_value = {'version': {'number': '5.0.0'} } 24 | client.indices.get_settings.return_value = testvars.settings_one 25 | client.cluster.state.return_value = testvars.clu_state_one 26 | client.indices.stats.return_value = testvars.stats_one 27 | client.indices.flush_synced.return_value = testvars.synced_pass 28 | client.indices.close.return_value = None 29 | ilo = curator.IndexList(client) 30 | co = curator.Close(ilo) 31 | self.assertIsNone(co.do_dry_run()) 32 | def test_do_action(self): 33 | client = Mock() 34 | client.info.return_value = {'version': {'number': '5.0.0'} } 35 | client.indices.get_settings.return_value = testvars.settings_one 36 | client.cluster.state.return_value = testvars.clu_state_one 37 | client.indices.stats.return_value = testvars.stats_one 38 | client.indices.flush_synced.return_value = testvars.synced_pass 39 | client.indices.close.return_value = None 40 | ilo = curator.IndexList(client) 41 | co = curator.Close(ilo) 42 | self.assertIsNone(co.do_action()) 43 | def test_do_action_with_delete_aliases(self): 44 | client = Mock() 45 | client.info.return_value = {'version': {'number': '5.0.0'} } 46 | client.indices.get_settings.return_value = testvars.settings_one 47 | client.cluster.state.return_value = testvars.clu_state_one 48 | client.indices.stats.return_value = testvars.stats_one 49 | client.indices.flush_synced.return_value = testvars.synced_pass 50 | client.indices.close.return_value = None 51 | ilo = curator.IndexList(client) 52 | co = curator.Close(ilo, delete_aliases=True) 53 | self.assertIsNone(co.do_action()) 54 | def test_do_action_raises_exception(self): 55 | client = Mock() 56 | client.info.return_value = {'version': {'number': '5.0.0'} } 57 | client.indices.get_settings.return_value = testvars.settings_one 58 | client.cluster.state.return_value = testvars.clu_state_one 59 | client.indices.stats.return_value = testvars.stats_one 60 | client.indices.flush_synced.return_value = testvars.synced_pass 61 | client.indices.close.return_value = None 62 | client.indices.close.side_effect = testvars.fake_fail 63 | ilo = curator.IndexList(client) 64 | co = curator.Close(ilo) 65 | self.assertRaises(curator.FailedExecution, co.do_action) 66 | def test_do_action_delete_aliases_with_exception(self): 67 | client = Mock() 68 | client.info.return_value = {'version': {'number': '5.0.0'} } 69 | client.indices.get_settings.return_value = testvars.settings_one 70 | client.cluster.state.return_value = testvars.clu_state_one 71 | client.indices.stats.return_value = testvars.stats_one 72 | client.indices.flush_synced.return_value = testvars.synced_pass 73 | client.indices.close.return_value = None 74 | ilo = curator.IndexList(client) 75 | client.indices.delete_alias.side_effect = testvars.fake_fail 76 | co = curator.Close(ilo, delete_aliases=True) 77 | self.assertIsNone(co.do_action()) 78 | -------------------------------------------------------------------------------- /binary_release.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import sys 4 | import shutil 5 | import hashlib 6 | 7 | # This script simply takes the output of `python setup.py build_exe` and makes 8 | # a compressed archive (zip for windows, tar.gz for Linux) for distribution. 9 | 10 | # Utility function to read from file. 11 | def fread(fname): 12 | return open(os.path.join(os.path.dirname(__file__), fname)).read() 13 | 14 | def get_version(): 15 | VERSIONFILE="curator/_version.py" 16 | verstrline = fread(VERSIONFILE).strip() 17 | vsre = r"^__version__ = ['\"]([^'\"]*)['\"]" 18 | mo = re.search(vsre, verstrline, re.M) 19 | if mo: 20 | VERSION = mo.group(1) 21 | else: 22 | raise RuntimeError("Unable to find version string in %s." % (VERSIONFILE,)) 23 | build_number = os.environ.get('CURATOR_BUILD_NUMBER', None) 24 | if build_number: 25 | return VERSION + "b{}".format(build_number) 26 | return VERSION 27 | 28 | archive_format = 'gztar' 29 | enviro = dict(os.environ) 30 | platform = sys.platform 31 | pyver = str(sys.version_info[0]) + '.' + str(sys.version_info[1]) 32 | if platform == 'win32': 33 | # Win32 stuff 34 | archive_format = 'zip' 35 | build_name = 'exe.win-' + enviro['PROCESSOR_ARCHITECTURE'].lower() + '-' + pyver 36 | target_name = "curator-" + str(get_version()) + "-amd64" 37 | elif platform == 'linux' or platform == 'linux2': 38 | sys_string = enviro['_system_type'].lower() + '-' + enviro['_system_arch'].lower() 39 | build_name = 'exe.' + sys_string + '-' + pyver 40 | target_name = "curator-" + str(get_version()) + "-" + sys_string 41 | else: 42 | # Unsupported platform? 43 | print('Your platform ({0}) is not yet supported for binary build/distribution.'.format(platform)) 44 | sys.exit(1) 45 | 46 | #sys_string = sys_type + '-' + sys_arch 47 | #build_name = 'exe.' + sys_string + '-' + pyver 48 | #print('Expected build directory: {0}'.format(build_name)) 49 | build_path = os.path.join('build', build_name) 50 | 51 | if os.path.exists(build_path): 52 | #print("I found the path: {0}".format(build_path)) 53 | 54 | target_path = os.path.join('.', target_name) 55 | 56 | # Check to see if an older directory exists... 57 | if os.path.exists(target_path): 58 | print('An older build exists at {0}. Please delete this before continuing.'.format(target_path)) 59 | sys.exit(1) 60 | else: 61 | shutil.copytree(build_path, target_path) 62 | 63 | # Ensure the rename went smoothly, then continue 64 | if os.path.exists(target_path): 65 | #print("Build successfully renamed") 66 | if float(pyver) >= 2.7: 67 | shutil.make_archive('elasticsearch-' + target_name, archive_format, '.', target_path) 68 | if platform == 'win32': 69 | fname = 'elasticsearch-' + target_name + '.zip' 70 | else: 71 | fname = 'elasticsearch-' + target_name + '.tar.gz' 72 | # Clean up directory if we made a viable archive. 73 | if os.path.exists(fname): 74 | shutil.rmtree(target_path) 75 | else: 76 | print('Something went wrong creating the archive {0}'.format(fname)) 77 | sys.exit(1) 78 | md5sum = hashlib.md5(open(fname, 'rb').read()).hexdigest() 79 | sha1sum = hashlib.sha1(open(fname, 'rb').read()).hexdigest() 80 | with open(fname + ".md5.txt", "w") as md5_file: 81 | md5_file.write("{0}".format(md5sum)) 82 | with open(fname + ".sha1.txt", "w") as sha1_file: 83 | sha1_file.write("{0}".format(sha1sum)) 84 | print('Archive: {0}'.format(fname)) 85 | print('{0} = {1}'.format(fname + ".md5.txt", md5sum)) 86 | print('{0} = {1}'.format(fname + ".sha1.txt", sha1sum)) 87 | else: 88 | print('Your python version ({0}) is too old to use with shutil.make_archive.'.format(pyver)) 89 | print('You can manually compress the {0} directory to achieve the same result.'.format(target_name)) 90 | else: 91 | # We couldn't find a build_path 92 | print("Build not found. Please run 'python setup.py build_exe' to create the build directory.") 93 | sys.exit(1) 94 | -------------------------------------------------------------------------------- /test/unit/test_action_delete_indices.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import Mock, patch 3 | import elasticsearch 4 | import curator 5 | # Get test variables and constants from a single source 6 | from . import testvars as testvars 7 | 8 | class TestActionDeleteIndices(TestCase): 9 | def test_init_raise(self): 10 | self.assertRaises(TypeError, curator.DeleteIndices, 'invalid') 11 | def test_init_raise_bad_master_timeout(self): 12 | client = Mock() 13 | client.info.return_value = {'version': {'number': '5.0.0'} } 14 | client.indices.get_settings.return_value = testvars.settings_one 15 | client.cluster.state.return_value = testvars.clu_state_one 16 | client.indices.stats.return_value = testvars.stats_one 17 | ilo = curator.IndexList(client) 18 | self.assertRaises(TypeError, curator.DeleteIndices, ilo, 'invalid') 19 | def test_init(self): 20 | client = Mock() 21 | client.info.return_value = {'version': {'number': '5.0.0'} } 22 | client.indices.get_settings.return_value = testvars.settings_one 23 | client.cluster.state.return_value = testvars.clu_state_one 24 | client.indices.stats.return_value = testvars.stats_one 25 | ilo = curator.IndexList(client) 26 | do = curator.DeleteIndices(ilo) 27 | self.assertEqual(ilo, do.index_list) 28 | self.assertEqual(client, do.client) 29 | def test_do_dry_run(self): 30 | client = Mock() 31 | client.info.return_value = {'version': {'number': '5.0.0'} } 32 | client.indices.get_settings.return_value = testvars.settings_four 33 | client.cluster.state.return_value = testvars.clu_state_four 34 | client.indices.stats.return_value = testvars.stats_four 35 | client.indices.delete.return_value = None 36 | ilo = curator.IndexList(client) 37 | do = curator.DeleteIndices(ilo) 38 | self.assertIsNone(do.do_dry_run()) 39 | def test_do_action(self): 40 | client = Mock() 41 | client.info.return_value = {'version': {'number': '5.0.0'} } 42 | client.indices.get_settings.return_value = testvars.settings_four 43 | client.cluster.state.return_value = testvars.clu_state_four 44 | client.indices.stats.return_value = testvars.stats_four 45 | client.indices.delete.return_value = None 46 | ilo = curator.IndexList(client) 47 | do = curator.DeleteIndices(ilo) 48 | self.assertIsNone(do.do_action()) 49 | def test_do_action_not_successful(self): 50 | client = Mock() 51 | client.info.return_value = {'version': {'number': '5.0.0'} } 52 | client.indices.get_settings.return_value = testvars.settings_four 53 | client.cluster.state.return_value = testvars.clu_state_four 54 | client.indices.stats.return_value = testvars.stats_four 55 | client.indices.delete.return_value = None 56 | ilo = curator.IndexList(client) 57 | do = curator.DeleteIndices(ilo) 58 | self.assertIsNone(do.do_action()) 59 | def test_do_action_raises_exception(self): 60 | client = Mock() 61 | client.info.return_value = {'version': {'number': '5.0.0'} } 62 | client.indices.get_settings.return_value = testvars.settings_four 63 | client.cluster.state.return_value = testvars.clu_state_four 64 | client.indices.stats.return_value = testvars.stats_four 65 | client.indices.delete.return_value = None 66 | client.indices.delete.side_effect = testvars.fake_fail 67 | ilo = curator.IndexList(client) 68 | do = curator.DeleteIndices(ilo) 69 | self.assertRaises(curator.FailedExecution, do.do_action) 70 | def test_verify_result_positive(self): 71 | client = Mock() 72 | client.info.return_value = {'version': {'number': '5.0.0'} } 73 | client.indices.get_settings.return_value = testvars.settings_four 74 | client.cluster.state.return_value = testvars.clu_state_four 75 | client.indices.stats.return_value = testvars.stats_four 76 | client.indices.delete.return_value = None 77 | ilo = curator.IndexList(client) 78 | do = curator.DeleteIndices(ilo) 79 | self.assertTrue(do._verify_result([],2)) 80 | -------------------------------------------------------------------------------- /test/integration/test_open.py: -------------------------------------------------------------------------------- 1 | import elasticsearch 2 | import curator 3 | import os 4 | import json 5 | import string, random, tempfile 6 | import click 7 | from click import testing as clicktest 8 | from mock import patch, Mock 9 | 10 | from . import CuratorTestCase 11 | from . import testvars as testvars 12 | 13 | import logging 14 | logger = logging.getLogger(__name__) 15 | 16 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 17 | port = int(port) if port else 9200 18 | 19 | class TestCLIOpenClosed(CuratorTestCase): 20 | def test_open_closed(self): 21 | self.write_config( 22 | self.args['configfile'], testvars.client_config.format(host, port)) 23 | self.write_config(self.args['actionfile'], 24 | testvars.optionless_proto.format('open')) 25 | self.create_index('my_index') 26 | self.client.indices.close( 27 | index='my_index', ignore_unavailable=True) 28 | self.create_index('dummy') 29 | test = clicktest.CliRunner() 30 | result = test.invoke( 31 | curator.cli, 32 | [ 33 | '--config', self.args['configfile'], 34 | self.args['actionfile'] 35 | ], 36 | ) 37 | self.assertNotEqual( 38 | 'close', 39 | self.client.cluster.state( 40 | index='my_index', 41 | metric='metadata', 42 | )['metadata']['indices']['my_index']['state'] 43 | ) 44 | self.assertNotEqual( 45 | 'close', 46 | self.client.cluster.state( 47 | index='dummy', 48 | metric='metadata', 49 | )['metadata']['indices']['dummy']['state'] 50 | ) 51 | def test_open_opened(self): 52 | self.write_config( 53 | self.args['configfile'], testvars.client_config.format(host, port)) 54 | self.write_config(self.args['actionfile'], 55 | testvars.optionless_proto.format('open')) 56 | self.create_index('my_index') 57 | self.create_index('dummy') 58 | test = clicktest.CliRunner() 59 | result = test.invoke( 60 | curator.cli, 61 | [ 62 | '--config', self.args['configfile'], 63 | self.args['actionfile'] 64 | ], 65 | ) 66 | self.assertNotEqual( 67 | 'close', 68 | self.client.cluster.state( 69 | index='my_index', 70 | metric='metadata', 71 | )['metadata']['indices']['my_index']['state'] 72 | ) 73 | self.assertNotEqual( 74 | 'close', 75 | self.client.cluster.state( 76 | index='dummy', 77 | metric='metadata', 78 | )['metadata']['indices']['dummy']['state'] 79 | ) 80 | def test_extra_option(self): 81 | self.write_config( 82 | self.args['configfile'], testvars.client_config.format(host, port)) 83 | self.write_config(self.args['actionfile'], 84 | testvars.bad_option_proto_test.format('open')) 85 | self.create_index('my_index') 86 | self.client.indices.close( 87 | index='my_index', ignore_unavailable=True) 88 | self.create_index('dummy') 89 | test = clicktest.CliRunner() 90 | result = test.invoke( 91 | curator.cli, 92 | [ 93 | '--config', self.args['configfile'], 94 | self.args['actionfile'] 95 | ], 96 | ) 97 | self.assertEquals( 98 | 'close', 99 | self.client.cluster.state( 100 | index='my_index', 101 | metric='metadata', 102 | )['metadata']['indices']['my_index']['state'] 103 | ) 104 | self.assertNotEqual( 105 | 'close', 106 | self.client.cluster.state( 107 | index='dummy', 108 | metric='metadata', 109 | )['metadata']['indices']['dummy']['state'] 110 | ) 111 | self.assertEqual(-1, result.exit_code) 112 | -------------------------------------------------------------------------------- /curator/defaults/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | from voluptuous import * 3 | 4 | # Elasticsearch versions supported 5 | def version_max(): 6 | return (5, 99, 99) 7 | def version_min(): 8 | return (5, 0, 0) 9 | 10 | # Default Config file location 11 | def config_file(): 12 | return os.path.join(os.path.expanduser('~'), '.curator', 'curator.yml') 13 | 14 | # Default filter patterns (regular expressions) 15 | def regex_map(): 16 | return { 17 | 'timestring': r'^.*{0}.*$', 18 | 'regex': r'{0}', 19 | 'prefix': r'^{0}.*$', 20 | 'suffix': r'^.*{0}$', 21 | } 22 | 23 | def date_regex(): 24 | return { 25 | 'Y' : '4', 26 | 'G' : '4', 27 | 'y' : '2', 28 | 'm' : '2', 29 | 'W' : '2', 30 | 'V' : '2', 31 | 'U' : '2', 32 | 'd' : '2', 33 | 'H' : '2', 34 | 'M' : '2', 35 | 'S' : '2', 36 | 'j' : '3', 37 | } 38 | 39 | # Actions 40 | 41 | def cluster_actions(): 42 | return [ 'cluster_routing' ] 43 | 44 | def index_actions(): 45 | return [ 46 | 'alias', 47 | 'allocation', 48 | 'close', 49 | 'create_index', 50 | 'delete_indices', 51 | 'forcemerge', 52 | 'index_settings', 53 | 'open', 54 | 'reindex', 55 | 'replicas', 56 | 'rollover', 57 | 'shrink', 58 | 'snapshot', 59 | ] 60 | 61 | def snapshot_actions(): 62 | return [ 'delete_snapshots', 'restore' ] 63 | 64 | def all_actions(): 65 | return sorted(cluster_actions() + index_actions() + snapshot_actions()) 66 | 67 | def index_filtertypes(): 68 | return [ 69 | 'alias', 70 | 'allocated', 71 | 'age', 72 | 'closed', 73 | 'count', 74 | 'forcemerged', 75 | 'kibana', 76 | 'none', 77 | 'opened', 78 | 'pattern', 79 | 'period', 80 | 'space', 81 | ] 82 | 83 | def snapshot_filtertypes(): 84 | return ['age', 'count', 'none', 'pattern', 'period', 'state'] 85 | 86 | def all_filtertypes(): 87 | return sorted(list(set(index_filtertypes() + snapshot_filtertypes()))) 88 | 89 | def default_options(): 90 | return { 91 | 'continue_if_exception': False, 92 | 'disable_action': False, 93 | 'ignore_empty_list': False, 94 | 'timeout_override': None, 95 | } 96 | 97 | def default_filters(): 98 | return { 'filters' : [{ 'filtertype' : 'none' }] } 99 | 100 | def structural_filter_elements(): 101 | return { 102 | Optional('aliases'): Any(str, [str], unicode, [unicode]), 103 | Optional('allocation_type'): Any(str, unicode), 104 | Optional('count'): Coerce(int), 105 | Optional('date_from'): Any(str, unicode, None), 106 | Optional('date_from_format'): Any(str, unicode, None), 107 | Optional('date_to'): Any(str, unicode, None), 108 | Optional('date_to_format'): Any(str, unicode, None), 109 | Optional('direction'): Any(str, unicode), 110 | Optional('disk_space'): float, 111 | Optional('epoch'): Any(Coerce(int), None), 112 | Optional('exclude'): Any(int, str, unicode, bool, None), 113 | Optional('field'): Any(str, unicode, None), 114 | Optional('intersect'): Any(int, str, unicode, bool, None), 115 | Optional('key'): Any(str, unicode), 116 | Optional('kind'): Any(str, unicode), 117 | Optional('max_num_segments'): Coerce(int), 118 | Optional('pattern'): Any(str, unicode), 119 | Optional('period_type'): Any(str, unicode), 120 | Optional('reverse'): Any(int, str, unicode, bool, None), 121 | Optional('range_from'): Coerce(int), 122 | Optional('range_to'): Coerce(int), 123 | Optional('source'): Any(str, unicode), 124 | Optional('state'): Any(str, unicode), 125 | Optional('stats_result'): Any(str, unicode, None), 126 | Optional('timestring'): Any(str, unicode, None), 127 | Optional('threshold_behavior'): Any(str, unicode), 128 | Optional('unit'): Any(str, unicode), 129 | Optional('unit_count'): Coerce(int), 130 | Optional('unit_count_pattern'): Any(str, unicode), 131 | Optional('use_age'): Boolean(), 132 | Optional('value'): Any(int, float, str, unicode, bool), 133 | Optional('week_starts_on'): Any(str, unicode, None), 134 | } -------------------------------------------------------------------------------- /test/integration/test_create_index.py: -------------------------------------------------------------------------------- 1 | import elasticsearch 2 | import curator 3 | import os 4 | import json 5 | import string, random, tempfile 6 | import click 7 | from click import testing as clicktest 8 | from mock import patch, Mock 9 | 10 | from . import CuratorTestCase 11 | from . import testvars as testvars 12 | 13 | import logging 14 | logger = logging.getLogger(__name__) 15 | 16 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 17 | port = int(port) if port else 9200 18 | 19 | class TestCLICreateIndex(CuratorTestCase): 20 | def test_plain(self): 21 | self.write_config( 22 | self.args['configfile'], testvars.client_config.format(host, port)) 23 | self.write_config(self.args['actionfile'], 24 | testvars.create_index.format('testing')) 25 | self.assertEqual([], curator.get_indices(self.client)) 26 | test = clicktest.CliRunner() 27 | result = test.invoke( 28 | curator.cli, 29 | [ 30 | '--config', self.args['configfile'], 31 | self.args['actionfile'] 32 | ], 33 | ) 34 | self.assertEqual(['testing'], curator.get_indices(self.client)) 35 | def test_with_extra_settings(self): 36 | self.write_config( 37 | self.args['configfile'], testvars.client_config.format(host, port)) 38 | self.write_config(self.args['actionfile'], 39 | testvars.create_index_with_extra_settings.format('testing')) 40 | self.assertEqual([], curator.get_indices(self.client)) 41 | test = clicktest.CliRunner() 42 | result = test.invoke( 43 | curator.cli, 44 | [ 45 | '--config', self.args['configfile'], 46 | self.args['actionfile'] 47 | ], 48 | ) 49 | ilo = curator.IndexList(self.client) 50 | self.assertEqual(['testing'], ilo.indices) 51 | self.assertEqual(ilo.index_info['testing']['number_of_shards'], '1') 52 | self.assertEqual(ilo.index_info['testing']['number_of_replicas'], '0') 53 | def test_with_strftime(self): 54 | self.write_config( 55 | self.args['configfile'], testvars.client_config.format(host, port)) 56 | self.write_config(self.args['actionfile'], 57 | testvars.create_index.format('testing-%Y.%m.%d')) 58 | self.assertEqual([], curator.get_indices(self.client)) 59 | name = curator.parse_date_pattern('testing-%Y.%m.%d') 60 | test = clicktest.CliRunner() 61 | result = test.invoke( 62 | curator.cli, 63 | [ 64 | '--config', self.args['configfile'], 65 | self.args['actionfile'] 66 | ], 67 | ) 68 | self.assertEqual([name], curator.get_indices(self.client)) 69 | def test_with_date_math(self): 70 | self.write_config( 71 | self.args['configfile'], testvars.client_config.format(host, port)) 72 | self.write_config(self.args['actionfile'], 73 | testvars.create_index.format('')) 74 | self.assertEqual([], curator.get_indices(self.client)) 75 | name = curator.parse_date_pattern('testing-%Y.%m.%d') 76 | test = clicktest.CliRunner() 77 | result = test.invoke( 78 | curator.cli, 79 | [ 80 | '--config', self.args['configfile'], 81 | self.args['actionfile'] 82 | ], 83 | ) 84 | self.assertEqual([name], curator.get_indices(self.client)) 85 | def test_extra_option(self): 86 | self.write_config( 87 | self.args['configfile'], testvars.client_config.format(host, port)) 88 | self.write_config(self.args['actionfile'], 89 | testvars.bad_option_proto_test.format('create_index')) 90 | test = clicktest.CliRunner() 91 | result = test.invoke( 92 | curator.cli, 93 | [ 94 | '--config', self.args['configfile'], 95 | self.args['actionfile'] 96 | ], 97 | ) 98 | self.assertEqual([], curator.get_indices(self.client)) 99 | self.assertEqual(-1, result.exit_code) 100 | -------------------------------------------------------------------------------- /curator/defaults/filtertypes.py: -------------------------------------------------------------------------------- 1 | from voluptuous import * 2 | from . import settings 3 | from . import filter_elements 4 | import logging 5 | logger = logging.getLogger(__name__) 6 | 7 | ## Helpers ## 8 | 9 | def _age_elements(action, config): 10 | retval = [] 11 | is_req = True 12 | if config['filtertype'] in ['count', 'space']: 13 | is_req = True if 'use_age' in config and config['use_age'] else False 14 | retval.append(filter_elements.source(action=action, required=is_req)) 15 | if action in settings.index_actions(): 16 | retval.append(filter_elements.stats_result()) 17 | # This is a silly thing here, because the absence of 'source' will 18 | # show up in the actual schema check, but it keeps code from breaking here 19 | ts_req = False 20 | if 'source' in config: 21 | if config['source'] == 'name': 22 | ts_req = True 23 | elif action in settings.index_actions(): 24 | # field_stats must _only_ exist for Index actions (not Snapshot) 25 | if config['source'] == 'field_stats': 26 | retval.append(filter_elements.field(required=True)) 27 | else: 28 | retval.append(filter_elements.field(required=False)) 29 | retval.append(filter_elements.timestring(required=ts_req)) 30 | else: 31 | # If source isn't in the config, then the other elements are not 32 | # required, but should be Optional to prevent false positives 33 | retval.append(filter_elements.field(required=False)) 34 | retval.append(filter_elements.timestring(required=ts_req)) 35 | return retval 36 | 37 | ### Schema information ### 38 | 39 | def alias(action, config): 40 | return [ 41 | filter_elements.aliases(), 42 | filter_elements.exclude(), 43 | ] 44 | 45 | def age(action, config): 46 | # Required & Optional 47 | retval = [ 48 | filter_elements.direction(), 49 | filter_elements.unit(), 50 | filter_elements.unit_count(), 51 | filter_elements.unit_count_pattern(), 52 | filter_elements.epoch(), 53 | filter_elements.exclude(), 54 | ] 55 | retval += _age_elements(action, config) 56 | logger.debug('AGE FILTER = {0}'.format(retval)) 57 | return retval 58 | 59 | def allocated(action, config): 60 | return [ 61 | filter_elements.key(), 62 | filter_elements.value(), 63 | filter_elements.allocation_type(), 64 | filter_elements.exclude(exclude=True), 65 | ] 66 | 67 | def closed(action, config): 68 | return [ filter_elements.exclude(exclude=True) ] 69 | 70 | def count(action, config): 71 | retval = [ 72 | filter_elements.count(), 73 | filter_elements.use_age(), 74 | filter_elements.pattern(), 75 | filter_elements.reverse(), 76 | filter_elements.exclude(exclude=True), 77 | ] 78 | retval += _age_elements(action, config) 79 | return retval 80 | 81 | def forcemerged(action, config): 82 | return [ 83 | filter_elements.max_num_segments(), 84 | filter_elements.exclude(exclude=True), 85 | ] 86 | 87 | def kibana(action, config): 88 | return [ filter_elements.exclude(exclude=True) ] 89 | 90 | def none(action, config): 91 | return [ ] 92 | 93 | def opened(action, config): 94 | return [ filter_elements.exclude(exclude=True) ] 95 | 96 | def pattern(action, config): 97 | return [ 98 | filter_elements.kind(), 99 | filter_elements.value(), 100 | filter_elements.exclude(), 101 | ] 102 | 103 | def period(action, config): 104 | retval = [ 105 | filter_elements.unit(period=True), 106 | filter_elements.range_from(), 107 | filter_elements.range_to(), 108 | filter_elements.week_starts_on(), 109 | filter_elements.epoch(), 110 | filter_elements.exclude(), 111 | filter_elements.intersect(), 112 | filter_elements.period_type(), 113 | filter_elements.date_from(), 114 | filter_elements.date_from_format(), 115 | filter_elements.date_to(), 116 | filter_elements.date_to_format(), 117 | ] 118 | retval += _age_elements(action, config) 119 | return retval 120 | 121 | def space(action, config): 122 | retval = [ 123 | filter_elements.disk_space(), 124 | filter_elements.reverse(), 125 | filter_elements.use_age(), 126 | filter_elements.exclude(), 127 | filter_elements.threshold_behavior(), 128 | ] 129 | retval += _age_elements(action, config) 130 | return retval 131 | 132 | def state(action, config): 133 | return [ 134 | filter_elements.state(), 135 | filter_elements.exclude(), 136 | ] 137 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | In accordance with section 4d of the Apache 2.0 license (http://www.apache.org/licenses/LICENSE-2.0), 2 | this NOTICE file is included. 3 | 4 | All users mentioned in the CONTRIBUTORS file at https://github.com/elastic/curator/blob/master/CONTRIBUTORS 5 | must be included in any derivative work. 6 | 7 | All conditions of section 4 of the Apache 2.0 license will be enforced: 8 | 9 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in 10 | any medium, with or without modifications, and in Source or Object form, provided that You meet the 11 | following conditions: 12 | 13 | a. You must give any other recipients of the Work or Derivative Works a copy of this License; and 14 | b. You must cause any modified files to carry prominent notices stating that You changed the files; and 15 | c. You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, 16 | trademark, and attribution notices from the Source form of the Work, excluding those notices that do 17 | not pertain to any part of the Derivative Works; and 18 | d. If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that 19 | You distribute must include a readable copy of the attribution notices contained within such NOTICE 20 | file, excluding those notices that do not pertain to any part of the Derivative Works, in at least 21 | one of the following places: within a NOTICE text file distributed as part of the Derivative Works; 22 | within the Source form or documentation, if provided along with the Derivative Works; or, within a 23 | display generated by the Derivative Works, if and wherever such third-party notices normally appear. 24 | The contents of the NOTICE file are for informational purposes only and do not modify the License. 25 | You may add Your own attribution notices within Derivative Works that You distribute, alongside or as 26 | an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot 27 | be construed as modifying the License. 28 | 29 | You may add Your own copyright statement to Your modifications and may provide additional or different 30 | license terms and conditions for use, reproduction, or distribution of Your modifications, or for any 31 | such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise 32 | complies with the conditions stated in this License. 33 | 34 | Contributors: 35 | * Jordan Sissel (jordansissel) (For Logstash, first and foremost) 36 | * Shay Banon (kimchy) (For Elasticsearch, of course!) 37 | * Aaron Mildenstein (untergeek) 38 | * Njal Karevoll 39 | * François Deppierraz 40 | * Honza Kral (HonzaKral) 41 | * Benjamin Smith (benjaminws) 42 | * Colin Moller (LeftyBC) 43 | * Elliot (edgeofnite) 44 | * Ram Viswanadha (ramv) 45 | * Chris Meisinger (cmeisinger) 46 | * Stuart Warren (stuart-warren) 47 | * (gitshaw) 48 | * (sfritz) 49 | * (sjoelsam) 50 | * Jose Diaz-Gonzalez (josegonzalez) 51 | * Arie Bro (arieb) 52 | * David Harrigan (dharrigan) 53 | * Mathieu Geli (gelim) 54 | * Nick Ethier (nickethier) 55 | * Mohab Usama (mohabusama) 56 | * (gitshaw) 57 | * Stuart Warren (stuart-warren) 58 | * Xavier Calland (xavier-calland) 59 | * Chad Schellenger (cschellenger) 60 | * Kamil Essekkat (ekamil) 61 | * (gbutt) 62 | * Ben Buchacher (bbuchacher) 63 | * Ehtesh Choudhury (shurane) 64 | * Markus Fischer (mfn) 65 | * Fabien Wernli (faxm0dem) 66 | * Michael Weiser (michaelweiser) 67 | * (digital-wonderland) 68 | * cassiano (cassianoleal) 69 | * Matt Dainty (bodgit) 70 | * Alex Philipp (alex-sf) 71 | * (krzaczek) 72 | * Justin Lintz (jlintz) 73 | * Jeremy Falling (jjfalling) 74 | * Ian Babrou (bobrik) 75 | * Ferenc Erki (ferki) 76 | * George Heppner (gheppner) 77 | * Matt Hughes (matthughes) 78 | * Brian Lalor (blalor) 79 | * Paweł Krzaczkowski (krzaczek) 80 | * Ben Tse (bt5e) 81 | * Tom Hendrikx (whyscream) 82 | * Christian Vozar (christianvozar) 83 | * Magnus Baeck (magnusbaeck) 84 | * Robin Kearney (rk295) 85 | * (cfeio) 86 | * (malagoli) 87 | * Dan Sheridan (djs52) 88 | * Michael-Keith Bernard (SegFaultAX) 89 | * Simon Lundström (simmel) 90 | * (pkr1234) 91 | * Mark Feltner (feltnerm) 92 | * William Jimenez (wjimenez5271) 93 | * Jeremy Canady (jrmycanady) 94 | * Steven Ottenhoff (steffo) 95 | * Ole Rößner (Basster) 96 | * Jack (univerio) 97 | * Tomáš Mózes (hydrapolic) 98 | * Gary Gao (garyelephant) 99 | * Panagiotis Moustafellos (pmoust) 100 | * (pbamba) 101 | * Pavel Strashkin (xaka) 102 | * Wadim Kruse (wkruse) 103 | * Richard Megginson (richm) 104 | * Thibaut Ackermann (thib-ack) 105 | * (zzugg) 106 | * Julien Mancuso (petitout) 107 | -------------------------------------------------------------------------------- /test/integration/test_snapshot.py: -------------------------------------------------------------------------------- 1 | import elasticsearch 2 | import curator 3 | import os 4 | import json 5 | import string, random, tempfile 6 | from click import testing as clicktest 7 | from mock import patch, Mock 8 | 9 | from . import CuratorTestCase 10 | from . import testvars as testvars 11 | 12 | import logging 13 | logger = logging.getLogger(__name__) 14 | 15 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 16 | port = int(port) if port else 9200 17 | 18 | class TestCLISnapshot(CuratorTestCase): 19 | def test_snapshot(self): 20 | self.create_indices(5) 21 | self.create_repository() 22 | snap_name = 'snapshot1' 23 | self.write_config( 24 | self.args['configfile'], testvars.client_config.format(host, port)) 25 | self.write_config(self.args['actionfile'], 26 | testvars.snapshot_test.format(self.args['repository'], snap_name, 1, 30)) 27 | test = clicktest.CliRunner() 28 | result = test.invoke( 29 | curator.cli, 30 | [ 31 | '--config', self.args['configfile'], 32 | self.args['actionfile'] 33 | ], 34 | ) 35 | snapshot = curator.get_snapshot( 36 | self.client, self.args['repository'], '_all' 37 | ) 38 | self.assertEqual(1, len(snapshot['snapshots'])) 39 | self.assertEqual(snap_name, snapshot['snapshots'][0]['snapshot']) 40 | def test_snapshot_ignore_empty_list(self): 41 | self.create_indices(5) 42 | self.create_repository() 43 | snap_name = 'snapshot1' 44 | self.write_config( 45 | self.args['configfile'], testvars.client_config.format(host, port)) 46 | self.write_config(self.args['actionfile'], 47 | testvars.test_682.format(self.args['repository'], snap_name, True, 1, 30)) 48 | test = clicktest.CliRunner() 49 | result = test.invoke( 50 | curator.cli, 51 | [ 52 | '--config', self.args['configfile'], 53 | self.args['actionfile'] 54 | ], 55 | ) 56 | snapshot = curator.get_snapshot( 57 | self.client, self.args['repository'], '_all' 58 | ) 59 | self.assertEqual(0, len(snapshot['snapshots'])) 60 | self.assertEquals(0, len(curator.get_indices(self.client))) 61 | def test_snapshot_do_not_ignore_empty_list(self): 62 | self.create_indices(5) 63 | self.create_repository() 64 | snap_name = 'snapshot1' 65 | self.write_config( 66 | self.args['configfile'], testvars.client_config.format(host, port)) 67 | self.write_config(self.args['actionfile'], 68 | testvars.test_682.format(self.args['repository'], snap_name, False, 1, 30)) 69 | test = clicktest.CliRunner() 70 | result = test.invoke( 71 | curator.cli, 72 | [ 73 | '--config', self.args['configfile'], 74 | self.args['actionfile'] 75 | ], 76 | ) 77 | snapshot = curator.get_snapshot( 78 | self.client, self.args['repository'], '_all' 79 | ) 80 | self.assertEqual(0, len(snapshot['snapshots'])) 81 | self.assertEquals(5, len(curator.get_indices(self.client))) 82 | def test_no_repository(self): 83 | self.create_indices(5) 84 | self.write_config( 85 | self.args['configfile'], testvars.client_config.format(host, port)) 86 | self.write_config(self.args['actionfile'], 87 | testvars.snapshot_test.format(' ', 'snap_name', 1, 30)) 88 | test = clicktest.CliRunner() 89 | result = test.invoke( 90 | curator.cli, 91 | [ 92 | '--config', self.args['configfile'], 93 | self.args['actionfile'] 94 | ], 95 | ) 96 | self.assertEqual(-1, result.exit_code) 97 | def test_extra_option(self): 98 | self.create_indices(5) 99 | self.write_config( 100 | self.args['configfile'], testvars.client_config.format(host, port)) 101 | self.write_config(self.args['actionfile'], 102 | testvars.bad_option_proto_test.format('snapshot')) 103 | test = clicktest.CliRunner() 104 | result = test.invoke( 105 | curator.cli, 106 | [ 107 | '--config', self.args['configfile'], 108 | self.args['actionfile'] 109 | ], 110 | ) 111 | self.assertEqual(-1, result.exit_code) 112 | -------------------------------------------------------------------------------- /test/unit/test_action_forcemerge.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import Mock, patch 3 | import elasticsearch 4 | import curator 5 | # Get test variables and constants from a single source 6 | from . import testvars as testvars 7 | 8 | class TestActionForceMerge(TestCase): 9 | def test_init_raise_bad_client(self): 10 | self.assertRaises( 11 | TypeError, curator.ForceMerge, 'invalid', max_num_segments=2) 12 | def test_init_raise_no_segment_count(self): 13 | client = Mock() 14 | client.info.return_value = {'version': {'number': '5.0.0'} } 15 | client.indices.get_settings.return_value = testvars.settings_one 16 | client.cluster.state.return_value = testvars.clu_state_one 17 | client.indices.stats.return_value = testvars.stats_one 18 | client.indices.segments.return_value = testvars.shards 19 | ilo = curator.IndexList(client) 20 | self.assertRaises( 21 | curator.MissingArgument, curator.ForceMerge, ilo) 22 | def test_init(self): 23 | client = Mock() 24 | client.info.return_value = {'version': {'number': '5.0.0'} } 25 | client.indices.get_settings.return_value = testvars.settings_one 26 | client.cluster.state.return_value = testvars.clu_state_one 27 | client.indices.stats.return_value = testvars.stats_one 28 | client.indices.segments.return_value = testvars.shards 29 | ilo = curator.IndexList(client) 30 | fmo = curator.ForceMerge(ilo, max_num_segments=2) 31 | self.assertEqual(ilo, fmo.index_list) 32 | self.assertEqual(client, fmo.client) 33 | def test_do_dry_run(self): 34 | client = Mock() 35 | client.info.return_value = {'version': {'number': '5.0.0'} } 36 | client.indices.get_settings.return_value = testvars.settings_one 37 | client.cluster.state.return_value = testvars.clu_state_one 38 | client.indices.stats.return_value = testvars.stats_one 39 | client.indices.segments.return_value = testvars.shards 40 | client.indices.forcemerge.return_value = None 41 | client.indices.optimize.return_value = None 42 | ilo = curator.IndexList(client) 43 | fmo = curator.ForceMerge(ilo, max_num_segments=2) 44 | self.assertIsNone(fmo.do_dry_run()) 45 | def test_do_action_pre5(self): 46 | client = Mock() 47 | client.info.return_value = {'version': {'number': '5.0.0'} } 48 | client.indices.get_settings.return_value = testvars.settings_one 49 | client.cluster.state.return_value = testvars.clu_state_one 50 | client.indices.stats.return_value = testvars.stats_one 51 | client.indices.segments.return_value = testvars.shards 52 | client.info.return_value = {'version': {'number': '2.3.2'} } 53 | client.indices.optimize.return_value = None 54 | ilo = curator.IndexList(client) 55 | fmo = curator.ForceMerge(ilo, max_num_segments=2) 56 | self.assertIsNone(fmo.do_action()) 57 | def test_do_action(self): 58 | client = Mock() 59 | client.info.return_value = {'version': {'number': '5.0.0'} } 60 | client.indices.get_settings.return_value = testvars.settings_one 61 | client.cluster.state.return_value = testvars.clu_state_one 62 | client.indices.stats.return_value = testvars.stats_one 63 | client.indices.segments.return_value = testvars.shards 64 | client.info.return_value = {'version': {'number': '5.0.0'} } 65 | client.indices.forcemerge.return_value = None 66 | ilo = curator.IndexList(client) 67 | fmo = curator.ForceMerge(ilo, max_num_segments=2) 68 | self.assertIsNone(fmo.do_action()) 69 | def test_do_action_with_delay(self): 70 | client = Mock() 71 | client.info.return_value = {'version': {'number': '5.0.0'} } 72 | client.indices.get_settings.return_value = testvars.settings_one 73 | client.cluster.state.return_value = testvars.clu_state_one 74 | client.indices.stats.return_value = testvars.stats_one 75 | client.indices.segments.return_value = testvars.shards 76 | client.info.return_value = {'version': {'number': '5.0.0'} } 77 | client.indices.forcemerge.return_value = None 78 | ilo = curator.IndexList(client) 79 | fmo = curator.ForceMerge(ilo, max_num_segments=2, delay=0.050) 80 | self.assertIsNone(fmo.do_action()) 81 | def test_do_action_raises_exception(self): 82 | client = Mock() 83 | client.info.return_value = {'version': {'number': '5.0.0'} } 84 | client.indices.get_settings.return_value = testvars.settings_one 85 | client.cluster.state.return_value = testvars.clu_state_one 86 | client.indices.stats.return_value = testvars.stats_one 87 | client.indices.segments.return_value = testvars.shards 88 | client.indices.forcemerge.return_value = None 89 | client.indices.optimize.return_value = None 90 | client.indices.forcemerge.side_effect = testvars.fake_fail 91 | client.indices.optimize.side_effect = testvars.fake_fail 92 | ilo = curator.IndexList(client) 93 | fmo = curator.ForceMerge(ilo, max_num_segments=2) 94 | self.assertRaises(curator.FailedExecution, fmo.do_action) 95 | -------------------------------------------------------------------------------- /docs/asciidoc/security.asciidoc: -------------------------------------------------------------------------------- 1 | [[security]] 2 | = Security 3 | 4 | [partintro] 5 | -- 6 | Please read the following sections for help with securing the connection between 7 | Curator and Elasticsearch. 8 | 9 | * <> 10 | * <> 11 | -- 12 | 13 | [[python-security]] 14 | == Python and Secure Connectivity 15 | 16 | Curator was written in Python, which allows it to be distributed as code which 17 | can run across a wide variety of systems, including Linux, Windows, Mac OS, and 18 | any other system or architecture for which a Python interpreter has been 19 | written. Curator was also written to be usable by the 4 most recent major 20 | release branches of Python: 2.7, 3.4, 3.5, and 3.6. It may even run on other 21 | versions, but those versions are not tested. 22 | 23 | Unfortunately, this broad support comes at a cost. While Curator happily ran 24 | on Python version 2.6, this version had its last update more than 3 years ago. 25 | There have been many improvements to security, SSL/TLS and the libraries that 26 | support them since then. Not all of these have been back-ported, which results 27 | in Curator not being able to communicate securely via SSL/TLS, or in some cases 28 | even connect securely. 29 | 30 | Because it is impossible to know if a given system has the correct Python 31 | version, leave alone the most recent libraries and modules, it becomes nearly 32 | impossible to guarantee that Curator will be able to make a secure and 33 | error-free connection to a secured Elasticsearch instance for any `pip` or 34 | RPM/DEB installed modules. This has lead to an increased amount of 35 | troubleshooting and support work for Curator. The precompiled binary packages 36 | were created to address this. 37 | 38 | The precompiled binary packages (APT/YUM, Windows) have been compiled with 39 | Python {pybuild_ver}, which has all of the up-to-date libraries needed for secure 40 | transactions. These packages have been tested connecting to Security (5.x 41 | X-Pack) with self-signed PKI certificates. Connectivity via SSL or TLS to other 42 | open-source plugins may work, but is not guaranteed. 43 | 44 | If you are encountering SSL/TLS errors in Curator, please see the list of 45 | <>. 46 | 47 | [[security-errors]] 48 | == Common Security Error Messages 49 | 50 | === Elasticsearch ConnectionError 51 | 52 | [source,sh] 53 | ----------- 54 | Unable to create client connection to Elasticsearch. Error:ConnectionError(error return without exception set) caused by: SystemError(error return without exception set) 55 | ----------- 56 | 57 | This error can happen on non-secured connections as well. If it happens with a 58 | secured instance, it will usually be accompanied by one or more of the following 59 | messages 60 | 61 | === SNIMissingWarning 62 | 63 | [source,sh] 64 | ----------- 65 | SNIMissingWarning: An HTTPS request has been made, but the SNI (Subject Name Indication) extension to TLS is not available on this platform. This may cause the server to present an incorrect TLS certificate, which can cause validation failures. You can upgrade to a newer version of Python to solve this. For more information, see https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings 66 | ----------- 67 | 68 | This happens on Python 2 versions older than 2.7.9. These older versions lack 69 | https://en.wikipedia.org/wiki/Server_Name_Indication[SNI] support. This can 70 | cause servers to present a certificate that the client thinks is invalid. Follow 71 | the https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2[pyOpenSSL] 72 | guide to resolve this warning. 73 | 74 | === InsecurePlatformWarning 75 | 76 | [source,sh] 77 | ----------- 78 | InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. You can upgrade to a newer version of Python to solve this. For more information, see https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings 79 | ----------- 80 | 81 | This happens on Python 2 platforms that have an outdated **ssl** module. These 82 | older **ssl** modules can cause some insecure requests to succeed where they 83 | should fail and secure requests to fail where they should succeed. Follow the 84 | https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2[pyOpenSSL] 85 | guide to resolve this warning. 86 | 87 | === InsecureRequestWarning 88 | 89 | [source,sh] 90 | ----------- 91 | InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.org/en/latest/security.html 92 | ----------- 93 | 94 | This happens when an request is made to an HTTPS URL without certificate 95 | verification enabled. Follow the 96 | https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl[certificate verification] 97 | guide to resolve this warning. 98 | 99 | Related: 100 | 101 | [source,sh] 102 | ----------- 103 | SSLError: [Errno 1] _ssl.c:510: error:14090086:SSL routines:SSL3_GET_SERVER_CERTIFICATE:certificate verify failed 104 | ----------- 105 | -------------------------------------------------------------------------------- /test/unit/test_action_allocation.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import Mock, patch 3 | import elasticsearch 4 | import curator 5 | # Get test variables and constants from a single source 6 | from . import testvars as testvars 7 | 8 | class TestActionAllocation(TestCase): 9 | def test_init_raise(self): 10 | self.assertRaises(TypeError, curator.Allocation, 'invalid') 11 | def test_init(self): 12 | client = Mock() 13 | client.info.return_value = {'version': {'number': '5.0.0'} } 14 | client.indices.get_settings.return_value = testvars.settings_one 15 | client.cluster.state.return_value = testvars.clu_state_one 16 | client.indices.stats.return_value = testvars.stats_one 17 | ilo = curator.IndexList(client) 18 | ao = curator.Allocation(ilo, key='key', value='value') 19 | self.assertEqual(ilo, ao.index_list) 20 | self.assertEqual(client, ao.client) 21 | def test_create_body_no_key(self): 22 | client = Mock() 23 | client.info.return_value = {'version': {'number': '5.0.0'} } 24 | client.indices.get_settings.return_value = testvars.settings_one 25 | client.cluster.state.return_value = testvars.clu_state_one 26 | client.indices.stats.return_value = testvars.stats_one 27 | ilo = curator.IndexList(client) 28 | self.assertRaises(curator.MissingArgument, curator.Allocation, ilo) 29 | def test_create_body_invalid_allocation_type(self): 30 | client = Mock() 31 | client.info.return_value = {'version': {'number': '5.0.0'} } 32 | client.indices.get_settings.return_value = testvars.settings_one 33 | client.cluster.state.return_value = testvars.clu_state_one 34 | client.indices.stats.return_value = testvars.stats_one 35 | ilo = curator.IndexList(client) 36 | self.assertRaises( 37 | ValueError, 38 | curator.Allocation, ilo, 39 | key='key', value='value', allocation_type='invalid' 40 | ) 41 | def test_create_body_valid(self): 42 | client = Mock() 43 | client.info.return_value = {'version': {'number': '5.0.0'} } 44 | client.indices.get_settings.return_value = testvars.settings_one 45 | client.cluster.state.return_value = testvars.clu_state_one 46 | client.indices.stats.return_value = testvars.stats_one 47 | ilo = curator.IndexList(client) 48 | ao = curator.Allocation(ilo, key='key', value='value') 49 | self.assertEqual({'index.routing.allocation.require.key': 'value'}, ao.body) 50 | def test_do_action_raise_on_put_settings(self): 51 | client = Mock() 52 | client.info.return_value = {'version': {'number': '5.0.0'} } 53 | client.indices.get_settings.return_value = testvars.settings_one 54 | client.cluster.state.return_value = testvars.clu_state_one 55 | client.indices.stats.return_value = testvars.stats_one 56 | client.indices.put_settings.return_value = None 57 | client.indices.put_settings.side_effect = testvars.fake_fail 58 | ilo = curator.IndexList(client) 59 | ao = curator.Allocation(ilo, key='key', value='value') 60 | self.assertRaises(Exception, ao.do_action) 61 | def test_do_dry_run(self): 62 | client = Mock() 63 | client.info.return_value = {'version': {'number': '5.0.0'} } 64 | client.indices.get_settings.return_value = testvars.settings_one 65 | client.cluster.state.return_value = testvars.clu_state_one 66 | client.indices.stats.return_value = testvars.stats_one 67 | client.indices.put_settings.return_value = None 68 | ilo = curator.IndexList(client) 69 | ao = curator.Allocation(ilo, key='key', value='value') 70 | self.assertIsNone(ao.do_dry_run()) 71 | def test_do_action(self): 72 | client = Mock() 73 | client.info.return_value = {'version': {'number': '5.0.0'} } 74 | client.indices.get_settings.return_value = testvars.settings_one 75 | client.cluster.state.return_value = testvars.clu_state_one 76 | client.indices.stats.return_value = testvars.stats_one 77 | client.indices.put_settings.return_value = None 78 | ilo = curator.IndexList(client) 79 | ao = curator.Allocation(ilo, key='key', value='value') 80 | self.assertIsNone(ao.do_action()) 81 | def test_do_action_wait_v50(self): 82 | client = Mock() 83 | client.info.return_value = {'version': {'number': '5.0.0'} } 84 | client.indices.get_settings.return_value = testvars.settings_one 85 | client.cluster.state.return_value = testvars.clu_state_one 86 | client.indices.stats.return_value = testvars.stats_one 87 | client.indices.put_settings.return_value = None 88 | client.cluster.health.return_value = {'relocating_shards':0} 89 | ilo = curator.IndexList(client) 90 | ao = curator.Allocation( 91 | ilo, key='key', value='value', wait_for_completion=True) 92 | self.assertIsNone(ao.do_action()) 93 | def test_do_action_wait_v51(self): 94 | client = Mock() 95 | client.info.return_value = {'version': {'number': '5.1.1'} } 96 | client.indices.get_settings.return_value = testvars.settings_one 97 | client.cluster.state.return_value = testvars.clu_state_one 98 | client.indices.stats.return_value = testvars.stats_one 99 | client.indices.put_settings.return_value = None 100 | client.cluster.health.return_value = {'relocating_shards':0} 101 | ilo = curator.IndexList(client) 102 | ao = curator.Allocation( 103 | ilo, key='key', value='value', wait_for_completion=True) 104 | self.assertIsNone(ao.do_action()) 105 | -------------------------------------------------------------------------------- /test/integration/test_es_repo_mgr.py: -------------------------------------------------------------------------------- 1 | import elasticsearch 2 | import curator 3 | import os 4 | import json 5 | import click 6 | import string, random, tempfile 7 | from click import testing as clicktest 8 | from mock import patch, Mock, MagicMock 9 | 10 | from . import CuratorTestCase 11 | from . import testvars as testvars 12 | 13 | import logging 14 | logger = logging.getLogger(__name__) 15 | 16 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 17 | port = int(port) if port else 9200 18 | 19 | class TestLoggingModules(CuratorTestCase): 20 | def test_logger_without_null_handler(self): 21 | mock = Mock() 22 | modules = {'logger': mock, 'logger.NullHandler': mock.module} 23 | self.write_config( 24 | self.args['configfile'], 25 | testvars.client_conf_logfile.format(host, port, os.devnull) 26 | ) 27 | with patch.dict('sys.modules', modules): 28 | self.create_repository() 29 | test = clicktest.CliRunner() 30 | result = test.invoke( 31 | curator.repo_mgr_cli, 32 | [ 33 | '--config', self.args['configfile'], 34 | 'show' 35 | ] 36 | ) 37 | self.assertEqual(self.args['repository'], result.output.rstrip()) 38 | 39 | 40 | class TestCLIRepositoryCreate(CuratorTestCase): 41 | def test_create_fs_repository_success(self): 42 | self.write_config( 43 | self.args['configfile'], 44 | testvars.client_conf_logfile.format(host, port, os.devnull) 45 | ) 46 | test = clicktest.CliRunner() 47 | result = test.invoke( 48 | curator.repo_mgr_cli, 49 | [ 50 | '--config', self.args['configfile'], 51 | 'create', 52 | 'fs', 53 | '--repository', self.args['repository'], 54 | '--location', self.args['location'] 55 | ] 56 | ) 57 | self.assertTrue(1, len(self.client.snapshot.get_repository(repository=self.args['repository']))) 58 | self.assertEqual(0, result.exit_code) 59 | 60 | def test_create_fs_repository_fail(self): 61 | self.write_config( 62 | self.args['configfile'], 63 | testvars.client_conf_logfile.format(host, port, os.devnull) 64 | ) 65 | test = clicktest.CliRunner() 66 | result = test.invoke( 67 | curator.repo_mgr_cli, 68 | [ 69 | '--config', self.args['configfile'], 70 | 'create', 71 | 'fs', 72 | '--repository', self.args['repository'], 73 | '--location', os.devnull 74 | ] 75 | ) 76 | self.assertEqual(1, result.exit_code) 77 | 78 | def test_create_s3_repository_fail(self): 79 | self.write_config( 80 | self.args['configfile'], 81 | testvars.client_conf_logfile.format(host, port, os.devnull) 82 | ) 83 | test = clicktest.CliRunner() 84 | result = test.invoke( 85 | curator.repo_mgr_cli, 86 | [ 87 | '--config', self.args['configfile'], 88 | 'create', 89 | 's3', 90 | '--bucket', 'mybucket', 91 | '--repository', self.args['repository'], 92 | ] 93 | ) 94 | self.assertEqual(1, result.exit_code) 95 | 96 | 97 | class TestCLIDeleteRepository(CuratorTestCase): 98 | def test_delete_repository_success(self): 99 | self.create_repository() 100 | self.write_config( 101 | self.args['configfile'], 102 | testvars.client_conf_logfile.format(host, port, os.devnull) 103 | ) 104 | test = clicktest.CliRunner() 105 | result = test.invoke( 106 | curator.repo_mgr_cli, 107 | [ 108 | '--config', self.args['configfile'], 109 | 'delete', 110 | '--yes', # This ensures no prompting will happen 111 | '--repository', self.args['repository'] 112 | ] 113 | ) 114 | self.assertFalse( 115 | curator.repository_exists(self.client, self.args['repository']) 116 | ) 117 | def test_delete_repository_notfound(self): 118 | self.write_config( 119 | self.args['configfile'], 120 | testvars.client_conf_logfile.format(host, port, os.devnull) 121 | ) 122 | test = clicktest.CliRunner() 123 | result = test.invoke( 124 | curator.repo_mgr_cli, 125 | [ 126 | '--config', self.args['configfile'], 127 | 'delete', 128 | '--yes', # This ensures no prompting will happen 129 | '--repository', self.args['repository'] 130 | ] 131 | ) 132 | self.assertEqual(1, result.exit_code) 133 | 134 | class TestCLIShowRepositories(CuratorTestCase): 135 | def test_show_repository(self): 136 | self.create_repository() 137 | self.write_config( 138 | self.args['configfile'], 139 | testvars.client_conf_logfile.format(host, port, os.devnull) 140 | ) 141 | test = clicktest.CliRunner() 142 | result = test.invoke( 143 | curator.repo_mgr_cli, 144 | [ 145 | '--config', self.args['configfile'], 146 | 'show' 147 | ] 148 | ) 149 | self.assertEqual(self.args['repository'], result.output.rstrip()) 150 | -------------------------------------------------------------------------------- /curator/repomgrcli.py: -------------------------------------------------------------------------------- 1 | import elasticsearch 2 | import click 3 | import re 4 | import sys 5 | import logging 6 | from .defaults import settings 7 | from .exceptions import * 8 | from .config_utils import process_config 9 | from .utils import * 10 | from ._version import __version__ 11 | 12 | logger = logging.getLogger('curator.repomgrcli') 13 | 14 | def delete_callback(ctx, param, value): 15 | if not value: 16 | ctx.abort() 17 | 18 | def show_repos(client): 19 | for repository in sorted(get_repository(client, '_all').keys()): 20 | print('{0}'.format(repository)) 21 | sys.exit(0) 22 | 23 | @click.command(short_help='Filesystem Repository') 24 | @click.option('--repository', required=True, type=str, help='Repository name') 25 | @click.option( 26 | '--location', 27 | required=True, 28 | type=str, 29 | help=( 30 | 'Shared file-system location. ' 31 | 'Must match remote path, & be accessible to all master & data nodes' 32 | ) 33 | ) 34 | @click.option('--compression', type=bool, default=True, show_default=True, 35 | help='Enable/Disable metadata compression.') 36 | @click.option('--chunk_size', type=str, 37 | help='Chunk size, e.g. 1g, 10m, 5k. [unbounded]') 38 | @click.option('--max_restore_bytes_per_sec', type=str, default='20mb', 39 | show_default=True, 40 | help='Throttles per node restore rate (per second).') 41 | @click.option('--max_snapshot_bytes_per_sec', type=str, default='20mb', 42 | show_default=True, 43 | help='Throttles per node snapshot rate (per second).') 44 | @click.option('--skip_repo_fs_check', type=bool, default=False, show_default=True, 45 | help='Skip repository verification after creation') 46 | @click.pass_context 47 | def fs( 48 | ctx, repository, location, compression, chunk_size, 49 | max_restore_bytes_per_sec, max_snapshot_bytes_per_sec, 50 | skip_repo_fs_check): 51 | """ 52 | Create a filesystem repository. 53 | """ 54 | logger = logging.getLogger('curator.repomgrcli.fs') 55 | client = get_client(**ctx.obj['client_args']) 56 | try: 57 | create_repository(client, repo_type='fs', **ctx.params) 58 | except FailedExecution as e: 59 | logger.critical(e) 60 | sys.exit(1) 61 | 62 | 63 | @click.command(short_help='S3 Repository') 64 | @click.option('--repository', required=True, type=str, help='Repository name') 65 | @click.option('--bucket', required=True, type=str, help='S3 bucket name') 66 | @click.option('--region', type=str, help='S3 region. [US Standard]') 67 | @click.option('--base_path', type=str, help='S3 base path. [root]') 68 | @click.option('--access_key', type=str, 69 | help='S3 access key. [value of cloud.aws.access_key]') 70 | @click.option('--secret_key', type=str, 71 | help='S3 secret key. [value of cloud.aws.secret_key]') 72 | @click.option('--compression', type=bool, default=True, show_default=True, 73 | help='Enable/Disable metadata compression.') 74 | @click.option('--chunk_size', type=str, 75 | help='Chunk size, e.g. 1g, 10m, 5k. [unbounded]') 76 | @click.option('--max_restore_bytes_per_sec', type=str, default='20mb', 77 | show_default=True, 78 | help='Throttles per node restore rate (per second).') 79 | @click.option('--max_snapshot_bytes_per_sec', type=str, default='20mb', 80 | show_default=True, 81 | help='Throttles per node snapshot rate (per second).') 82 | @click.option('--skip_repo_fs_check', type=bool, default=False, show_default=True, 83 | help='Skip repository verification after creation') 84 | @click.pass_context 85 | def s3( 86 | ctx, repository, bucket, region, base_path, access_key, secret_key, 87 | compression, chunk_size, max_restore_bytes_per_sec, 88 | max_snapshot_bytes_per_sec, skip_repo_fs_check): 89 | """ 90 | Create an S3 repository. 91 | """ 92 | logger = logging.getLogger('curator.repomgrcli.s3') 93 | client = get_client(**ctx.obj['client_args']) 94 | try: 95 | create_repository(client, repo_type='s3', **ctx.params) 96 | except FailedExecution as e: 97 | logger.critical(e) 98 | sys.exit(1) 99 | 100 | 101 | @click.group() 102 | @click.option( 103 | '--config', 104 | help="Path to configuration file. Default: ~/.curator/curator.yml", 105 | type=click.Path(exists=True), default=settings.config_file() 106 | ) 107 | @click.pass_context 108 | def repo_mgr_cli(ctx, config): 109 | """ 110 | Repository manager for Elasticsearch Curator. 111 | """ 112 | ctx.obj = {} 113 | ctx.obj['client_args'] = process_config(config) 114 | logger = logging.getLogger(__name__) 115 | logger.debug('Client and logging options validated.') 116 | 117 | @repo_mgr_cli.group('create') 118 | @click.pass_context 119 | def _create(ctx): 120 | """Create an Elasticsearch repository""" 121 | _create.add_command(fs) 122 | _create.add_command(s3) 123 | 124 | @repo_mgr_cli.command('show') 125 | @click.pass_context 126 | def show(ctx): 127 | """ 128 | Show all repositories 129 | """ 130 | client = get_client(**ctx.obj['client_args']) 131 | show_repos(client) 132 | 133 | @repo_mgr_cli.command('delete') 134 | @click.option('--repository', required=True, help='Repository name', type=str) 135 | @click.option('--yes', is_flag=True, callback=delete_callback, 136 | expose_value=False, 137 | prompt='Are you sure you want to delete the repository?') 138 | @click.pass_context 139 | def _delete(ctx, repository): 140 | """Delete an Elasticsearch repository""" 141 | client = get_client(**ctx.obj['client_args']) 142 | try: 143 | logger.info('Deleting repository {0}...'.format(repository)) 144 | client.snapshot.delete_repository(repository=repository) 145 | except elasticsearch.NotFoundError: 146 | logger.error( 147 | 'Unable to delete repository: {0} Not Found.'.format(repository)) 148 | sys.exit(1) 149 | -------------------------------------------------------------------------------- /docs/asciidoc/about.asciidoc: -------------------------------------------------------------------------------- 1 | [[about]] 2 | = About 3 | 4 | [partintro] 5 | -- 6 | 7 | Elasticsearch Curator helps you curate, or manage, your Elasticsearch indices 8 | and snapshots by: 9 | 10 | 1. Obtaining the full list of indices (or snapshots) from the cluster, as the 11 | _actionable list_ 12 | 2. Iterate through a list of user-defined <> to progressively 13 | remove indices (or snapshots) from this _actionable list_ as needed. 14 | 3. Perform various <> on the items which remain in the 15 | _actionable list._ 16 | 17 | Learn More: 18 | 19 | * <> 20 | * <> 21 | * <> 22 | * <> 23 | * <> 24 | * <> 25 | * <> 26 | -- 27 | 28 | [[about-origin]] 29 | == Origin 30 | 31 | Curator was first called 32 | https://logstash.jira.com/browse/LOGSTASH-211[`clearESindices.py`]. Its sole 33 | function was to delete indices. It was almost immediately renamed to 34 | https://logstash.jira.com/browse/LOGSTASH-211[`logstash_index_cleaner.py`]. 35 | After a time it was briefly relocated under the 36 | https://github.com/elastic/logstash[logstash] repository as 37 | `expire_logs`, at which point it began to gain new functionality. Soon 38 | thereafter, Jordan Sissel was hired by Elastic (then still Elasticsearch), as 39 | was the original author of Curator. Not long after that it became Elasticsearch 40 | Curator and is now hosted at https://github.com/elastic/curator 41 | 42 | Curator now performs many operations on your Elasticsearch indices, from delete 43 | to snapshot to shard allocation routing. 44 | 45 | [[about-features]] 46 | == Features 47 | 48 | Curator allows for many different operations to be performed to both indices and 49 | snapshots, including: 50 | 51 | * Add or remove indices (or both!) from an <> 52 | * Change shard routing <> 53 | * <> indices 54 | * <> 55 | * <> 56 | * <> 57 | * <> closed indices 58 | * <> indices 59 | * <> indices, including from remote clusters 60 | * Change the number of <> per shard for indices 61 | * <> indices 62 | * Take a <> (backup) of indices 63 | * <> snapshots 64 | 65 | [[about-cli]] 66 | == Command-Line Interface (CLI) 67 | 68 | Curator has always been a command-line tool. This site provides the 69 | documentation for how to use Curator on the command-line. 70 | 71 | TIP: Learn more about <>. 72 | 73 | [[about-api]] 74 | == Application Program Interface (API) 75 | 76 | Curator ships with both an API and CLI tool. The API, or Application Program 77 | Interface, allows you to write your own scripts to accomplish similar goals--or 78 | even new and different things--with the same code that Curator uses. 79 | 80 | The API documentation is not on this site, but is available at 81 | http://curator.readthedocs.io/. The Curator API is built using the 82 | http://www.elastic.co/guide/en/elasticsearch/client/python-api/current/index.html[Elasticsearch 83 | Python API]. 84 | 85 | [[license]] 86 | == License 87 | 88 | Copyright (c) 2011–2017 Elasticsearch 89 | 90 | Licensed under the Apache License, Version 2.0 (the "License"); 91 | you may not use this file except in compliance with the License. 92 | You may obtain a copy of the License at 93 | 94 | http://www.apache.org/licenses/LICENSE-2.0 95 | 96 | Unless required by applicable law or agreed to in writing, software 97 | distributed under the License is distributed on an "AS IS" BASIS, 98 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 99 | See the License for the specific language governing permissions and 100 | limitations under the License. 101 | 102 | 103 | [[site-corrections]] 104 | == Site Corrections 105 | 106 | All documentation on this site allows for quick correction submission. 107 | 108 | To do this, use the "Edit" link to the right on any page. 109 | 110 | * You will need to log in with your GitHub account. 111 | * Make your corrections or additions to the documentation. 112 | * Please use the option to "Create a new branch for this commit and start a pull 113 | request." 114 | * Please make sure you have signed our 115 | http://www.elastic.co/contributor-agreement/[Contributor License Agreement]. We 116 | are not asking you to assign copyright to us, but to give us the right to 117 | distribute your code (even documentation corrections) without restriction. We 118 | ask this of all contributors in order to assure our users of the origin and 119 | continuing existence of the code. You only need to sign the CLA once. If you are 120 | uncomfortable with this, feel free to submit a 121 | https://github.com/elastic/curator/issues[GitHub Issue] with your suggested 122 | correction instead. 123 | * Changes will be reviewed and merged, if acceptable. 124 | 125 | [[about-contributing]] 126 | == Contributing 127 | 128 | We welcome contributions and bug fixes to Curator's API and CLI. 129 | 130 | We are grateful for the many 131 | https://github.com/elastic/curator/blob/master/CONTRIBUTORS[contributors] who 132 | have helped Curator become what it is today. 133 | 134 | Please read through our 135 | https://github.com/elastic/curator/blob/master/CONTRIBUTING.md[contribution] 136 | guide, and the Curator 137 | https://github.com/elastic/curator/blob/master/README.rst[readme] document. 138 | 139 | A brief overview of the steps 140 | 141 | * fork the repo 142 | * make changes in your fork 143 | * add tests to cover your changes (if necessary) 144 | * run tests 145 | * sign the http://elastic.co/contributor-agreement/[CLA] 146 | * send a pull request! 147 | 148 | TIP: To submit documentation fixes for this site, see 149 | <> 150 | -------------------------------------------------------------------------------- /test/integration/test_close.py: -------------------------------------------------------------------------------- 1 | import elasticsearch 2 | import curator 3 | import os 4 | import json 5 | import string, random, tempfile 6 | import click 7 | from click import testing as clicktest 8 | from mock import patch, Mock 9 | 10 | from . import CuratorTestCase 11 | from . import testvars as testvars 12 | 13 | import logging 14 | logger = logging.getLogger(__name__) 15 | 16 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 17 | port = int(port) if port else 9200 18 | 19 | class TestCLIClose(CuratorTestCase): 20 | def test_close_opened(self): 21 | self.write_config( 22 | self.args['configfile'], testvars.client_config.format(host, port)) 23 | self.write_config(self.args['actionfile'], 24 | testvars.optionless_proto.format('close')) 25 | self.create_index('my_index') 26 | self.create_index('dummy') 27 | test = clicktest.CliRunner() 28 | result = test.invoke( 29 | curator.cli, 30 | [ 31 | '--config', self.args['configfile'], 32 | self.args['actionfile'] 33 | ], 34 | ) 35 | self.assertEquals( 36 | 'close', 37 | self.client.cluster.state( 38 | index='my_index', 39 | metric='metadata', 40 | )['metadata']['indices']['my_index']['state'] 41 | ) 42 | self.assertNotEqual( 43 | 'close', 44 | self.client.cluster.state( 45 | index='dummy', 46 | metric='metadata', 47 | )['metadata']['indices']['dummy']['state'] 48 | ) 49 | def test_close_closed(self): 50 | self.write_config( 51 | self.args['configfile'], testvars.client_config.format(host, port)) 52 | self.write_config(self.args['actionfile'], 53 | testvars.optionless_proto.format('close')) 54 | self.create_index('my_index') 55 | self.client.indices.close( 56 | index='my_index', ignore_unavailable=True) 57 | self.create_index('dummy') 58 | test = clicktest.CliRunner() 59 | result = test.invoke( 60 | curator.cli, 61 | [ 62 | '--config', self.args['configfile'], 63 | self.args['actionfile'] 64 | ], 65 | ) 66 | self.assertEquals( 67 | 'close', 68 | self.client.cluster.state( 69 | index='my_index', 70 | metric='metadata', 71 | )['metadata']['indices']['my_index']['state'] 72 | ) 73 | self.assertNotEqual( 74 | 'close', 75 | self.client.cluster.state( 76 | index='dummy', 77 | metric='metadata', 78 | )['metadata']['indices']['dummy']['state'] 79 | ) 80 | def test_close_delete_aliases(self): 81 | # Create aliases first 82 | alias = 'testalias' 83 | index = 'my_index' 84 | self.create_index(index) 85 | self.create_index('dummy') 86 | self.create_index('my_other') 87 | self.client.indices.put_alias(index='my_index,dummy', name=alias) 88 | self.assertEquals( 89 | { 90 | "dummy":{"aliases":{"testalias":{}}}, 91 | "my_index":{"aliases":{"testalias":{}}} 92 | }, 93 | self.client.indices.get_alias(name=alias) 94 | ) 95 | # Now close `index` with delete_aliases=True (dummy stays open) 96 | self.write_config( 97 | self.args['configfile'], testvars.client_config.format(host, port)) 98 | self.write_config(self.args['actionfile'], 99 | testvars.close_delete_aliases) 100 | test = clicktest.CliRunner() 101 | result = test.invoke( 102 | curator.cli, 103 | [ 104 | '--config', self.args['configfile'], 105 | self.args['actionfile'] 106 | ], 107 | ) 108 | self.assertEquals( 109 | 'close', 110 | self.client.cluster.state( 111 | index=index, 112 | metric='metadata', 113 | )['metadata']['indices'][index]['state'] 114 | ) 115 | self.assertEquals( 116 | 'close', 117 | self.client.cluster.state( 118 | index='my_other', 119 | metric='metadata', 120 | )['metadata']['indices']['my_other']['state'] 121 | ) 122 | # Now open the indices and verify that the alias is still gone. 123 | self.client.indices.open(index=index) 124 | self.assertEquals( 125 | {"dummy":{"aliases":{"testalias":{}}}}, 126 | self.client.indices.get_alias(name=alias) 127 | ) 128 | def test_extra_option(self): 129 | self.write_config( 130 | self.args['configfile'], testvars.client_config.format(host, port)) 131 | self.write_config(self.args['actionfile'], 132 | testvars.bad_option_proto_test.format('close')) 133 | self.create_index('my_index') 134 | self.create_index('dummy') 135 | test = clicktest.CliRunner() 136 | result = test.invoke( 137 | curator.cli, 138 | [ 139 | '--config', self.args['configfile'], 140 | self.args['actionfile'] 141 | ], 142 | ) 143 | self.assertNotEqual( 144 | 'close', 145 | self.client.cluster.state( 146 | index='my_index', 147 | metric='metadata', 148 | )['metadata']['indices']['my_index']['state'] 149 | ) 150 | self.assertNotEqual( 151 | 'close', 152 | self.client.cluster.state( 153 | index='dummy', 154 | metric='metadata', 155 | )['metadata']['indices']['dummy']['state'] 156 | ) 157 | self.assertEqual(-1, result.exit_code) 158 | -------------------------------------------------------------------------------- /curator/validators/options.py: -------------------------------------------------------------------------------- 1 | from voluptuous import * 2 | from ..defaults import option_defaults 3 | 4 | ## Methods for building the schema 5 | def action_specific(action): 6 | options = { 7 | 'alias' : [ 8 | option_defaults.name(action), 9 | option_defaults.warn_if_no_indices(), 10 | option_defaults.extra_settings(), 11 | ], 12 | 'allocation' : [ 13 | option_defaults.key(), 14 | option_defaults.value(), 15 | option_defaults.allocation_type(), 16 | option_defaults.wait_for_completion(action), 17 | option_defaults.wait_interval(action), 18 | option_defaults.max_wait(action), 19 | ], 20 | 'close' : [ option_defaults.delete_aliases() ], 21 | 'cluster_routing' : [ 22 | option_defaults.routing_type(), 23 | option_defaults.cluster_routing_setting(), 24 | option_defaults.cluster_routing_value(), 25 | option_defaults.wait_for_completion(action), 26 | option_defaults.wait_interval(action), 27 | option_defaults.max_wait(action), 28 | ], 29 | 'create_index' : [ 30 | option_defaults.name(action), 31 | option_defaults.extra_settings(), 32 | ], 33 | 'delete_indices' : [], 34 | 'delete_snapshots' : [ 35 | option_defaults.repository(), 36 | option_defaults.retry_interval(), 37 | option_defaults.retry_count(), 38 | ], 39 | 'forcemerge' : [ 40 | option_defaults.delay(), 41 | option_defaults.max_num_segments(), 42 | ], 43 | 'index_settings' : [ 44 | option_defaults.index_settings(), 45 | option_defaults.ignore_unavailable(), 46 | option_defaults.preserve_existing(), 47 | ], 48 | 'open' : [], 49 | 'reindex' : [ 50 | option_defaults.request_body(), 51 | option_defaults.refresh(), 52 | option_defaults.requests_per_second(), 53 | option_defaults.slices(), 54 | option_defaults.timeout(action), 55 | option_defaults.wait_for_active_shards(action), 56 | option_defaults.wait_for_completion(action), 57 | option_defaults.wait_interval(action), 58 | option_defaults.max_wait(action), 59 | option_defaults.remote_certificate(), 60 | option_defaults.remote_client_cert(), 61 | option_defaults.remote_client_key(), 62 | option_defaults.remote_aws_key(), 63 | option_defaults.remote_aws_secret_key(), 64 | option_defaults.remote_aws_region(), 65 | option_defaults.remote_filters(), 66 | option_defaults.remote_url_prefix(), 67 | option_defaults.remote_ssl_no_validate(), 68 | option_defaults.migration_prefix(), 69 | option_defaults.migration_suffix(), 70 | ], 71 | 'replicas' : [ 72 | option_defaults.count(), 73 | option_defaults.wait_for_completion(action), 74 | option_defaults.wait_interval(action), 75 | option_defaults.max_wait(action), 76 | ], 77 | 'rollover' : [ 78 | option_defaults.name(action), 79 | option_defaults.new_index(), 80 | option_defaults.conditions(), 81 | option_defaults.extra_settings(), 82 | option_defaults.wait_for_active_shards(action), 83 | ], 84 | 'restore' : [ 85 | option_defaults.repository(), 86 | option_defaults.name(action), 87 | option_defaults.indices(), 88 | option_defaults.ignore_unavailable(), 89 | option_defaults.include_aliases(), 90 | option_defaults.include_global_state(action), 91 | option_defaults.partial(), 92 | option_defaults.rename_pattern(), 93 | option_defaults.rename_replacement(), 94 | option_defaults.extra_settings(), 95 | option_defaults.wait_for_completion(action), 96 | option_defaults.wait_interval(action), 97 | option_defaults.max_wait(action), 98 | option_defaults.skip_repo_fs_check(), 99 | ], 100 | 'snapshot' : [ 101 | option_defaults.repository(), 102 | option_defaults.name(action), 103 | option_defaults.ignore_unavailable(), 104 | option_defaults.include_global_state(action), 105 | option_defaults.partial(), 106 | option_defaults.wait_for_completion(action), 107 | option_defaults.wait_interval(action), 108 | option_defaults.max_wait(action), 109 | option_defaults.skip_repo_fs_check(), 110 | ], 111 | 'shrink' : [ 112 | option_defaults.shrink_node(), 113 | option_defaults.node_filters(), 114 | option_defaults.number_of_shards(), 115 | option_defaults.number_of_replicas(), 116 | option_defaults.shrink_prefix(), 117 | option_defaults.shrink_suffix(), 118 | option_defaults.copy_aliases(), 119 | option_defaults.delete_after(), 120 | option_defaults.post_allocation(), 121 | option_defaults.wait_for_active_shards(action), 122 | option_defaults.extra_settings(), 123 | option_defaults.wait_for_completion(action), 124 | option_defaults.wait_interval(action), 125 | option_defaults.max_wait(action), 126 | ], 127 | } 128 | return options[action] 129 | 130 | def get_schema(action): 131 | # Appending the options dictionary seems to be the best way, since the 132 | # "Required" and "Optional" elements are hashes themselves. 133 | options = {} 134 | defaults = [ 135 | option_defaults.continue_if_exception(), 136 | option_defaults.disable_action(), 137 | option_defaults.ignore_empty_list(), 138 | option_defaults.timeout_override(action), 139 | ] 140 | for each in defaults: 141 | options.update(each) 142 | for each in action_specific(action): 143 | options.update(each) 144 | return Schema(options) 145 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import sys 4 | from setuptools import setup 5 | 6 | # Utility function to read from file. 7 | def fread(fname): 8 | return open(os.path.join(os.path.dirname(__file__), fname)).read() 9 | 10 | def get_version(): 11 | VERSIONFILE="curator/_version.py" 12 | verstrline = fread(VERSIONFILE).strip() 13 | vsre = r"^__version__ = ['\"]([^'\"]*)['\"]" 14 | mo = re.search(vsre, verstrline, re.M) 15 | if mo: 16 | VERSION = mo.group(1) 17 | else: 18 | raise RuntimeError("Unable to find version string in %s." % (VERSIONFILE,)) 19 | build_number = os.environ.get('CURATOR_BUILD_NUMBER', None) 20 | if build_number: 21 | return VERSION + "b{}".format(build_number) 22 | return VERSION 23 | 24 | def get_install_requires(): 25 | res = ['elasticsearch>=5.4.0,<6.0.0' ] 26 | res.append('click>=6.7') 27 | res.append('pyyaml>=3.10') 28 | res.append('voluptuous>=0.9.3') 29 | res.append('certifi>=2017.4.17') 30 | return res 31 | 32 | try: 33 | ### cx_Freeze ### 34 | from cx_Freeze import setup, Executable 35 | try: 36 | import certifi 37 | cert_file = certifi.where() 38 | except ImportError: 39 | cert_file = '' 40 | # Dependencies are automatically detected, but it might need 41 | # fine tuning. 42 | 43 | 44 | base = 'Console' 45 | msvcrt = '' 46 | 47 | icon = None 48 | if os.path.exists('Elastic.ico'): 49 | icon = 'Elastic.ico' 50 | 51 | curator_exe = Executable( 52 | "run_curator.py", 53 | base=base, 54 | targetName = "curator", 55 | ) 56 | curator_cli_exe = Executable( 57 | "run_singleton.py", 58 | base=base, 59 | targetName = "curator_cli", 60 | ) 61 | repomgr_exe = Executable( 62 | "run_es_repo_mgr.py", 63 | base=base, 64 | targetName = "es_repo_mgr", 65 | ) 66 | 67 | if sys.platform == "win32": 68 | curator_exe = Executable( 69 | "run_curator.py", 70 | base=base, 71 | targetName = "curator.exe", 72 | icon = icon 73 | ) 74 | curator_cli_exe = Executable( 75 | "run_singleton.py", 76 | base=base, 77 | targetName = "curator_cli.exe", 78 | icon = icon 79 | ) 80 | repomgr_exe = Executable( 81 | "run_es_repo_mgr.py", 82 | base=base, 83 | targetName = "es_repo_mgr.exe", 84 | icon = icon 85 | ) 86 | msvcrt = 'vcruntime140.dll' 87 | 88 | buildOptions = dict( 89 | packages = [], 90 | excludes = [], 91 | include_files = [cert_file, msvcrt], 92 | include_msvcr = True, 93 | ) 94 | setup( 95 | name = "elasticsearch-curator", 96 | version = get_version(), 97 | author = "Elastic", 98 | author_email = "info@elastic.co", 99 | description = "Tending your Elasticsearch indices", 100 | long_description=fread('README.rst'), 101 | url = "http://github.com/elastic/curator", 102 | download_url = "https://github.com/elastic/curator/tarball/v" + get_version(), 103 | license = "Apache License, Version 2.0", 104 | install_requires = get_install_requires(), 105 | keywords = "elasticsearch time-series indexed index-expiry", 106 | packages = ["curator"], 107 | include_package_data=True, 108 | entry_points = { 109 | "console_scripts" : [ 110 | "curator = curator.cli:cli", 111 | "curator_cli = curator.curator_cli:main", 112 | "es_repo_mgr = curator.repomgrcli:repo_mgr_cli", 113 | ] 114 | }, 115 | classifiers=[ 116 | "Intended Audience :: Developers", 117 | "Intended Audience :: System Administrators", 118 | "License :: OSI Approved :: Apache Software License", 119 | "Operating System :: OS Independent", 120 | "Programming Language :: Python", 121 | "Programming Language :: Python :: 2.7", 122 | "Programming Language :: Python :: 3.4", 123 | "Programming Language :: Python :: 3.5", 124 | "Programming Language :: Python :: 3.6", 125 | ], 126 | test_suite = "test.run_tests.run_all", 127 | tests_require = ["mock", "nose", "coverage", "nosexcover"], 128 | options = {"build_exe" : buildOptions}, 129 | executables = [curator_exe,curator_cli_exe,repomgr_exe] 130 | ) 131 | ### end cx_Freeze ### 132 | except ImportError: 133 | setup( 134 | name = "elasticsearch-curator", 135 | version = get_version(), 136 | author = "Elastic", 137 | author_email = "info@elastic.co", 138 | description = "Tending your Elasticsearch indices", 139 | long_description=fread('README.rst'), 140 | url = "http://github.com/elastic/curator", 141 | download_url = "https://github.com/elastic/curator/tarball/v" + get_version(), 142 | license = "Apache License, Version 2.0", 143 | install_requires = get_install_requires(), 144 | keywords = "elasticsearch time-series indexed index-expiry", 145 | packages = ["curator"], 146 | include_package_data=True, 147 | entry_points = { 148 | "console_scripts" : [ 149 | "curator = curator.cli:cli", 150 | "curator_cli = curator.curator_cli:main", 151 | "es_repo_mgr = curator.repomgrcli:repo_mgr_cli", 152 | ] 153 | }, 154 | classifiers=[ 155 | "Intended Audience :: Developers", 156 | "Intended Audience :: System Administrators", 157 | "License :: OSI Approved :: Apache Software License", 158 | "Operating System :: OS Independent", 159 | "Programming Language :: Python", 160 | "Programming Language :: Python :: 2.7", 161 | "Programming Language :: Python :: 3.4", 162 | "Programming Language :: Python :: 3.5", 163 | "Programming Language :: Python :: 3.6", 164 | ], 165 | test_suite = "test.run_tests.run_all", 166 | tests_require = ["mock", "nose", "coverage", "nosexcover"] 167 | ) 168 | -------------------------------------------------------------------------------- /test/integration/__init__.py: -------------------------------------------------------------------------------- 1 | import time 2 | import os 3 | import shutil 4 | import tempfile 5 | import random 6 | import string 7 | from datetime import timedelta, datetime, date 8 | 9 | from elasticsearch import Elasticsearch 10 | from elasticsearch.exceptions import ConnectionError 11 | 12 | from unittest import SkipTest, TestCase 13 | from mock import Mock 14 | 15 | client = None 16 | 17 | DATEMAP = { 18 | 'months': '%Y.%m', 19 | 'weeks': '%Y.%W', 20 | 'days': '%Y.%m.%d', 21 | 'hours': '%Y.%m.%d.%H', 22 | } 23 | 24 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 25 | port = int(port) if port else 9200 26 | 27 | def random_directory(): 28 | dirname = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) 29 | directory = tempfile.mkdtemp(suffix=dirname) 30 | if not os.path.exists(directory): 31 | os.makedirs(directory) 32 | return directory 33 | 34 | def get_client(): 35 | global client 36 | if client is not None: 37 | return client 38 | 39 | client = Elasticsearch([os.environ.get('TEST_ES_SERVER', {})], timeout=300) 40 | 41 | # wait for yellow status 42 | for _ in range(100): 43 | time.sleep(.1) 44 | try: 45 | client.cluster.health(wait_for_status='yellow') 46 | return client 47 | except ConnectionError: 48 | continue 49 | else: 50 | # timeout 51 | raise SkipTest("Elasticsearch failed to start.") 52 | 53 | def setup(): 54 | get_client() 55 | 56 | class Args(dict): 57 | def __getattr__(self, att_name): 58 | return self.get(att_name, None) 59 | 60 | class CuratorTestCase(TestCase): 61 | def setUp(self): 62 | super(CuratorTestCase, self).setUp() 63 | self.client = get_client() 64 | 65 | args = {} 66 | args['host'], args['port'] = host, port 67 | args['time_unit'] = 'days' 68 | args['prefix'] = 'logstash-' 69 | self.args = args 70 | # dirname = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) 71 | # This will create a psuedo-random temporary directory on the machine 72 | # which runs the unit tests, but NOT on the machine where elasticsearch 73 | # is running. This means tests may fail if run against remote instances 74 | # unless you explicitly set `self.args['location']` to a proper spot 75 | # on the target machine. 76 | self.args['location'] = random_directory() 77 | self.args['configdir'] = random_directory() 78 | self.args['configfile'] = os.path.join(self.args['configdir'], 'curator.yml') 79 | self.args['actionfile'] = os.path.join(self.args['configdir'], 'actions.yml') 80 | self.args['repository'] = 'TEST_REPOSITORY' 81 | # if not os.path.exists(self.args['location']): 82 | # os.makedirs(self.args['location']) 83 | 84 | def tearDown(self): 85 | self.delete_repositories() 86 | self.client.indices.delete(index='*') 87 | self.client.indices.delete_template(name='*', ignore=404) 88 | for path_arg in ['location', 'configdir']: 89 | if os.path.exists(self.args[path_arg]): 90 | shutil.rmtree(self.args[path_arg]) 91 | 92 | def parse_args(self): 93 | return Args(self.args) 94 | 95 | def create_indices(self, count, unit=None): 96 | now = datetime.utcnow() 97 | unit = unit if unit else self.args['time_unit'] 98 | format = DATEMAP[unit] 99 | if not unit == 'months': 100 | step = timedelta(**{unit: 1}) 101 | for x in range(count): 102 | self.create_index(self.args['prefix'] + now.strftime(format), wait_for_yellow=False) 103 | now -= step 104 | else: # months 105 | now = date.today() 106 | d = date(now.year, now.month, 1) 107 | self.create_index(self.args['prefix'] + now.strftime(format), wait_for_yellow=False) 108 | 109 | for i in range(1, count): 110 | if d.month == 1: 111 | d = date(d.year-1, 12, 1) 112 | else: 113 | d = date(d.year, d.month-1, 1) 114 | self.create_index(self.args['prefix'] + datetime(d.year, d.month, 1).strftime(format), wait_for_yellow=False) 115 | 116 | self.client.cluster.health(wait_for_status='yellow') 117 | 118 | def create_index(self, name, shards=1, wait_for_yellow=True): 119 | self.client.indices.create( 120 | index=name, 121 | body={'settings': {'number_of_shards': shards, 'number_of_replicas': 0}} 122 | ) 123 | if wait_for_yellow: 124 | self.client.cluster.health(wait_for_status='yellow') 125 | 126 | def add_docs(self, idx): 127 | for i in ["1", "2", "3"]: 128 | self.client.create( 129 | index=idx, doc_type='log', id=i, 130 | body={"doc" + i :'TEST DOCUMENT'}, 131 | ) 132 | # This should force each doc to be in its own segment. 133 | self.client.indices.flush(index=idx, force=True) 134 | 135 | def create_snapshot(self, name, csv_indices): 136 | body = { 137 | "indices": csv_indices, 138 | "ignore_unavailable": False, 139 | "include_global_state": True, 140 | "partial": False, 141 | } 142 | self.create_repository() 143 | self.client.snapshot.create( 144 | repository=self.args['repository'], snapshot=name, body=body, 145 | wait_for_completion=True 146 | ) 147 | 148 | def create_repository(self): 149 | body = {'type':'fs', 'settings':{'location':self.args['location']}} 150 | self.client.snapshot.create_repository(repository=self.args['repository'], body=body) 151 | 152 | def delete_repositories(self): 153 | result = self.client.snapshot.get_repository(repository='_all') 154 | for repo in result: 155 | self.client.snapshot.delete_repository(repository=repo) 156 | 157 | def close_index(self, name): 158 | self.client.indices.close(index=name) 159 | 160 | def write_config(self, fname, data): 161 | with open(fname, 'w') as f: 162 | f.write(data) 163 | -------------------------------------------------------------------------------- /unix_packages/build_package_from_source.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | BASEPATH=$(pwd) 4 | PKG_TARGET=/curator_packages 5 | WORKDIR=/tmp/curator 6 | PYVER=3.6 7 | MINOR=3 8 | INPUT_TYPE=python 9 | CATEGORY=python 10 | VENDOR=Elastic 11 | MAINTAINER="'Elastic Developers '" 12 | C_POST_INSTALL=${WORKDIR}/es_curator_after_install.sh 13 | C_PRE_REMOVE=${WORKDIR}/es_curator_before_removal.sh 14 | C_POST_REMOVE=${WORKDIR}/es_curator_after_removal.sh 15 | C_PRE_UPGRADE=${WORKDIR}/es_curator_before_upgrade.sh 16 | C_POST_UPGRADE=${WORKDIR}/es_curator_after_upgrade.sh 17 | 18 | # Build our own package pre/post scripts 19 | sudo rm -rf ${WORKDIR} /opt/elasticsearch-curator 20 | mkdir -p ${WORKDIR} 21 | 22 | for file in ${C_POST_INSTALL} ${C_PRE_REMOVE} ${C_POST_REMOVE}; do 23 | echo '#!/bin/bash' > ${file} 24 | echo >> ${file} 25 | chmod +x ${file} 26 | done 27 | 28 | remove_python() { 29 | sudo rm -f /usr/local/lib/libpython${1}m.a 30 | sudo rm -f /usr/local/lib/pkgconfig/python-${1}.pc 31 | sudo rm -rf /usr/local/lib/python${1} 32 | sudo rm -f /usr/lib/libpython${1}.a 33 | sudo rm -rf /usr/local/include/python${1}m 34 | cd /usr/local/bin 35 | sudo rm -f 2to3-${1} easy_install-${1} idle${1} pip${1} pydoc${1} python${1} python${1}m python${1}m-config pyvenv-${1} 36 | cd - 37 | } 38 | 39 | build_python() { 40 | cd /tmp 41 | wget -c https://www.python.org/ftp/python/${1}/Python-${1}.tgz 42 | tar zxf Python-${1}.tgz 43 | cd Python-${1} 44 | ./configure --prefix=/usr/local 45 | sudo make altinstall 46 | sudo ln -s /usr/local/lib/libpython${1}m.a /usr/lib/libpython${1}.a 47 | cd - 48 | } 49 | 50 | echo "ln -s /opt/elasticsearch-curator/curator /usr/bin/curator" >> ${C_POST_INSTALL} 51 | echo "ln -s /opt/elasticsearch-curator/curator_cli /usr/bin/curator_cli" >> ${C_POST_INSTALL} 52 | echo "ln -s /opt/elasticsearch-curator/es_repo_mgr /usr/bin/es_repo_mgr" >> ${C_POST_INSTALL} 53 | echo "ln -s /opt/elasticsearch-curator/curator /usr/bin/curator" >> ${C_POST_UPGRADE} 54 | echo "ln -s /opt/elasticsearch-curator/curator_cli /usr/bin/curator_cli" >> ${C_POST_UPGRADE} 55 | echo "ln -s /opt/elasticsearch-curator/es_repo_mgr /usr/bin/es_repo_mgr" >> ${C_POST_UPGRADE} 56 | echo "rm -f /usr/bin/curator" >> ${C_PRE_REMOVE} 57 | echo "rm -f /usr/bin/curator_cli" >> ${C_PRE_REMOVE} 58 | echo "rm -f /usr/bin/es_repo_mgr" >> ${C_PRE_REMOVE} 59 | echo "rm -f /usr/bin/curator" >> ${C_PRE_UPGRADE} 60 | echo "rm -f /usr/bin/curator_cli" >> ${C_PRE_UPGRADE} 61 | echo "rm -f /usr/bin/es_repo_mgr" >> ${C_PRE_UPGRADE} 62 | echo 'if [ -d "/opt/elasticsearch-curator" ]; then' >> ${C_POST_REMOVE} 63 | echo ' rm -rf /opt/elasticsearch-curator' >> ${C_POST_REMOVE} 64 | echo 'fi' >> ${C_POST_REMOVE} 65 | 66 | ID=$(grep ^ID\= /etc/*release | awk -F\= '{print $2}' | tr -d \") 67 | VERSION_ID=$(grep ^VERSION_ID\= /etc/*release | awk -F\= '{print $2}' | tr -d \") 68 | if [ "${ID}x" == "x" ]; then 69 | ID=$(cat /etc/*release | grep -v LSB | uniq | awk '{print $1}' | tr "[:upper:]" "[:lower:]" ) 70 | VERSION_ID=$(cat /etc/*release | grep -v LSB | uniq | awk '{print $3}' | awk -F\. '{print $1}') 71 | fi 72 | 73 | # build 74 | if [ "${1}x" == "x" ]; then 75 | echo "Must provide version number (can be arbitrary)" 76 | exit 1 77 | else 78 | cd $(dirname $0)/.. 79 | SOURCE_DIR=$(pwd) 80 | fi 81 | 82 | case "$ID" in 83 | ubuntu|debian) 84 | PKGTYPE=deb 85 | PLATFORM=debian 86 | case "$VERSION_ID" in 87 | 1404|1604|8) PACKAGEDIR="${PKG_TARGET}/${1}/${PLATFORM}";; 88 | 9) PACKAGEDIR="${PKG_TARGET}/${1}/${PLATFORM}${VERSION_ID}";; 89 | *) PACKAGEDIR="${PKG_TARGET}/${1}/${PLATFORM}";; 90 | esac 91 | sudo apt update -y 92 | sudo apt install -y openssl zlib1g zlib1g-dev libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev dirmngr curl 93 | ;; 94 | centos|rhel) 95 | PKGTYPE=rpm 96 | PLATFORM=centos 97 | case "$VERSION_ID" in 98 | 6|7) 99 | sudo rm -f /etc/yum.repos.d/puppetlabs-pc1.repo 100 | sudo yum -y update 101 | sudo yum install -y openssl 102 | ;; 103 | *) echo "unknown system version: ${VERSION_ID}"; exit 1;; 104 | esac 105 | PACKAGEDIR="${PKG_TARGET}/${1}/${PLATFORM}/${VERSION_ID}" 106 | ;; 107 | *) echo "unknown system type: ${ID}"; exit 1;; 108 | esac 109 | 110 | HAS_PY3=$(which python${PYVER}) 111 | if [ "${HAS_PY3}x" == "x" ]; then 112 | build_python ${PYVER}.${MINOR} 113 | fi 114 | 115 | FOUNDVER=$(python${PYVER} --version | awk '{print $2}') 116 | if [ "${FOUNDVER}" != "${PYVER}.${MINOR}" ]; then 117 | remove_python $(echo ${FOUNDVER} | awk -F\. '{print $1"."$2}') 118 | build_python ${PYVER}.${MINOR} 119 | fi 120 | 121 | PIPBIN=/usr/local/bin/pip${PYVER} 122 | PYBIN=/usr/local/bin/python${PYVER} 123 | 124 | if [ -e "${HOME}/.rvm/scripts/rvm" ]; then 125 | source ${HOME}/.rvm/scripts/rvm 126 | fi 127 | HAS_FPM=$(which fpm) 128 | if [ "${HAS_FPM}x" == "x" ]; then 129 | gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 130 | curl -sSL https://get.rvm.io | bash -s stable 131 | source ${HOME}/.rvm/scripts/rvm 132 | rvm install ruby 133 | gem install fpm 134 | fi 135 | 136 | ${PIPBIN} install -U --user setuptools 137 | ${PIPBIN} install -U --user requests_aws4auth 138 | ${PIPBIN} install -U --user boto3 139 | ${PIPBIN} install -U --user cx_freeze 140 | 141 | cd $SOURCE_DIR 142 | 143 | mkdir -p ${PACKAGEDIR} 144 | ${PIPBIN} install -U --user -r requirements.txt 145 | ${PYBIN} setup.py build_exe 146 | sudo mv build/exe.linux-x86_64-${PYVER} /opt/elasticsearch-curator 147 | 148 | sudo chown -R root:root /opt/elasticsearch-curator 149 | cd $WORKDIR 150 | fpm \ 151 | -s dir \ 152 | -t ${PKGTYPE} \ 153 | -n elasticsearch-curator \ 154 | -v ${1} \ 155 | --vendor ${VENDOR} \ 156 | --maintainer "${MAINTAINER}" \ 157 | --license 'Apache-2.0' \ 158 | --category tools \ 159 | --description 'Have indices in Elasticsearch? This is the tool for you!\n\nLike a museum curator manages the exhibits and collections on display, \nElasticsearch Curator helps you curate, or manage your indices.' \ 160 | --after-install ${C_POST_INSTALL} \ 161 | --before-remove ${C_PRE_REMOVE} \ 162 | --after-remove ${C_POST_REMOVE} \ 163 | --before-upgrade ${C_PRE_UPGRADE} \ 164 | --after-upgrade ${C_POST_UPGRADE} \ 165 | --provides elasticsearch-curator \ 166 | --conflicts python-elasticsearch-curator \ 167 | --conflicts python3-elasticsearch-curator \ 168 | /opt/elasticsearch-curator 169 | 170 | mv ${WORKDIR}/*.${PKGTYPE} ${PACKAGEDIR} 171 | 172 | rm ${C_POST_INSTALL} ${C_PRE_REMOVE} ${C_POST_REMOVE} ${C_PRE_UPGRADE} ${C_POST_UPGRADE} 173 | # go back to where we started 174 | cd ${BASEPATH} 175 | --------------------------------------------------------------------------------