├── .gitignore ├── CONTRIBUTING.md ├── CONTRIBUTORS ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── NOTICE ├── README.md ├── README.rst ├── binary_release.py ├── curator ├── __init__.py ├── _version.py ├── actions.py ├── cli.py ├── cli_singletons │ ├── __init__.py │ ├── alias.py │ ├── allocation.py │ ├── close.py │ ├── delete.py │ ├── forcemerge.py │ ├── freeze.py │ ├── object_class.py │ ├── open_indices.py │ ├── replicas.py │ ├── restore.py │ ├── rollover.py │ ├── show.py │ ├── shrink.py │ ├── snapshot.py │ ├── unfreeze.py │ └── utils.py ├── config_utils.py ├── curator_cli.py ├── defaults │ ├── __init__.py │ ├── client_defaults.py │ ├── filter_elements.py │ ├── filtertypes.py │ ├── option_defaults.py │ └── settings.py ├── exceptions.py ├── indexlist.py ├── logtools.py ├── repomgrcli.py ├── singletons.py ├── snapshotlist.py ├── utils.py └── validators │ ├── __init__.py │ ├── actions.py │ ├── config_file.py │ ├── filters.py │ ├── options.py │ └── schemacheck.py ├── docs ├── Changelog.rst ├── Makefile ├── actionclasses.rst ├── asciidoc │ ├── about.asciidoc │ ├── actions.asciidoc │ ├── command-line.asciidoc │ ├── configuration.asciidoc │ ├── examples.asciidoc │ ├── faq.asciidoc │ ├── filter_elements.asciidoc │ ├── filters.asciidoc │ ├── ilm.asciidoc │ ├── inc_datemath.asciidoc │ ├── inc_filepath.asciidoc │ ├── inc_filter_by_aliases.asciidoc │ ├── inc_filter_chaining.asciidoc │ ├── inc_kinds.asciidoc │ ├── inc_sources.asciidoc │ ├── inc_strftime_table.asciidoc │ ├── inc_timestring_regex.asciidoc │ ├── inc_unit_table.asciidoc │ ├── index.asciidoc │ ├── installation.asciidoc │ ├── options.asciidoc │ ├── security.asciidoc │ └── versions.asciidoc ├── conf.py ├── examples.rst ├── filters.rst ├── index.rst ├── objectclasses.rst └── utilities.rst ├── examples ├── actions │ ├── alias.yml │ ├── allocation.yml │ ├── close.yml │ ├── create_index.yml │ ├── delete_indices.yml │ ├── delete_snapshots.yml │ ├── forcemerge.yml │ ├── freeze.yml │ ├── open.yml │ ├── reindex.yml │ ├── replicas.yml │ ├── restore.yml │ ├── shrink.yml │ ├── snapshot.yml │ └── unfreeze.yml └── curator.yml ├── requirements.txt ├── run_curator.py ├── run_es_repo_mgr.py ├── run_singleton.py ├── setup.cfg ├── setup.py ├── test ├── __init__.py ├── integration │ ├── __init__.py │ ├── test_alias.py │ ├── test_allocation.py │ ├── test_cli.py │ ├── test_close.py │ ├── test_clusterrouting.py │ ├── test_count_pattern.py │ ├── test_create_index.py │ ├── test_datemath.py │ ├── test_delete_indices.py │ ├── test_delete_snapshots.py │ ├── test_envvars.py │ ├── test_es_repo_mgr.py │ ├── test_forcemerge.py │ ├── test_freeze.py │ ├── test_integrations.py │ ├── test_open.py │ ├── test_reindex.py │ ├── test_replicas.py │ ├── test_restore.py │ ├── test_rollover.py │ ├── test_shrink.py │ ├── test_snapshot.py │ ├── test_unfreeze.py │ └── testvars.py ├── run_tests.py └── unit │ ├── __init__.py │ ├── test_action_alias.py │ ├── test_action_allocation.py │ ├── test_action_close.py │ ├── test_action_clusterrouting.py │ ├── test_action_create_index.py │ ├── test_action_delete_indices.py │ ├── test_action_delete_snapshots.py │ ├── test_action_forcemerge.py │ ├── test_action_indexsettings.py │ ├── test_action_open.py │ ├── test_action_reindex.py │ ├── test_action_replicas.py │ ├── test_action_restore.py │ ├── test_action_rollover.py │ ├── test_action_shrink.py │ ├── test_action_snapshot.py │ ├── test_class_index_list.py │ ├── test_class_snapshot_list.py │ ├── test_cli_methods.py │ ├── test_utils.py │ ├── test_validators.py │ └── testvars.py ├── travis-run.sh ├── unix_packages ├── README.md └── builder_image │ ├── CentOS-Base.repo │ ├── Dockerfile │ ├── git_build.sh │ ├── package_maker.sh │ ├── patchelf_build.sh │ ├── rpm.erb.patched │ └── ruby_build.sh └── win_release.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | .howtobuild 132 | examples/restore-s3.yml -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Curator 2 | 3 | All contributions are welcome: ideas, patches, documentation, bug reports, 4 | complaints, etc! 5 | 6 | Programming is not a required skill, and there are many ways to help out! 7 | It is more important to us that you are able to contribute. 8 | 9 | That said, some basic guidelines, which you are free to ignore :) 10 | 11 | ## Want to learn? 12 | 13 | Want to write your own code to do something Curator doesn't do out of the box? 14 | 15 | * [Curator API Documentation](http://curator.readthedocs.io/) Since version 2.0, 16 | Curator ships with both an API and wrapper scripts (which are actually defined 17 | as entry points). This allows you to write your own scripts to accomplish 18 | similar goals, or even new and different things with the 19 | [Curator API](http://curator.readthedocs.io/), and the 20 | [Elasticsearch Python API](http://elasticsearch-py.readthedocs.io/). 21 | 22 | Want to know how to use the command-line interface (CLI)? 23 | 24 | * [Curator CLI Documentation](http://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html) 25 | The Curator CLI Documentation is now a part of the document repository at 26 | http://elastic.co/guide at 27 | http://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html 28 | 29 | Want to lurk about and see what others are doing with Curator? 30 | 31 | * The irc channels (#logstash and #elasticsearch on irc.freenode.org) are good 32 | places for this 33 | 34 | ## Got Questions? 35 | 36 | Have a problem you want Curator to solve for you? 37 | 38 | * You are welcome to join the IRC channel #logstash (or #elasticsearch) on 39 | irc.freenode.org and ask for help there! 40 | 41 | ## Have an Idea or Feature Request? 42 | 43 | * File a ticket on [github](https://github.com/elastic/curator/issues) 44 | 45 | ## Something Not Working? Found a Bug? 46 | 47 | If you think you found a bug, it probably is a bug. 48 | 49 | * File it on [github](https://github.com/elastic/curator/issues) 50 | 51 | # Contributing Documentation and Code Changes 52 | 53 | If you have a bugfix or new feature that you would like to contribute to 54 | Curator, and you think it will take more than a few minutes to produce the fix 55 | (ie; write code), it is worth discussing the change with the Curator users and 56 | developers first! You can reach us via 57 | [github](https://github.com/elastic/curator/issues), or via IRC (#logstash or 58 | #elasticsearch on freenode irc) 59 | 60 | Documentation is in two parts: API and CLI documentation. 61 | 62 | API documentation is generated from comments inside the classes and methods 63 | within the code. This documentation is rendered and hosted at 64 | http://curator.readthedocs.io 65 | 66 | CLI documentation is in Asciidoc format in the GitHub repository at 67 | https://github.com/elastic/curator/tree/master/docs/asciidoc. 68 | This documentation can be changed via a pull request as with any other code 69 | change. 70 | 71 | ## Contribution Steps 72 | 73 | 1. Test your changes! Run the test suite ('python setup.py test'). Please note 74 | that this requires an Elasticsearch instance. The tests will try to connect 75 | to your local elasticsearch instance and run integration tests against it. 76 | **This will delete all the data stored there!** You can use the env variable 77 | `TEST_ES_SERVER` to point to a different instance (for example 78 | 'otherhost:9203'). 79 | 2. Please make sure you have signed our [Contributor License 80 | Agreement](http://www.elastic.co/contributor-agreement/). We are not 81 | asking you to assign copyright to us, but to give us the right to distribute 82 | your code without restriction. We ask this of all contributors in order to 83 | assure our users of the origin and continuing existence of the code. You 84 | only need to sign the CLA once. 85 | 3. Send a pull request! Push your changes to your fork of the repository and 86 | [submit a pull 87 | request](https://help.github.com/articles/using-pull-requests). In the pull 88 | request, describe what your changes do and mention any bugs/issues related 89 | to the pull request. 90 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | The following is a list of people who have contributed ideas, code, bug 2 | reports, or in general have helped curator along its way. 3 | 4 | Contributors: 5 | * Jordan Sissel (jordansissel) (For Logstash, first and foremost) 6 | * Shay Banon (kimchy) (For Elasticsearch, of course!) 7 | * Aaron Mildenstein (untergeek) 8 | * Njal Karevoll 9 | * François Deppierraz 10 | * Honza Kral (HonzaKral) 11 | * Benjamin Smith (benjaminws) 12 | * Colin Moller (LeftyBC) 13 | * Elliot (edgeofnite) 14 | * Ram Viswanadha (ramv) 15 | * Chris Meisinger (cmeisinger) 16 | * Stuart Warren (stuart-warren) 17 | * (gitshaw) 18 | * (sfritz) 19 | * (sjoelsam) 20 | * Jose Diaz-Gonzalez (josegonzalez) 21 | * Arie Bro (arieb) 22 | * David Harrigan (dharrigan) 23 | * Mathieu Geli (gelim) 24 | * Nick Ethier (nickethier) 25 | * Mohab Usama (mohabusama) 26 | * (gitshaw) 27 | * Stuart Warren (stuart-warren) 28 | * Xavier Calland (xavier-calland) 29 | * Chad Schellenger (cschellenger) 30 | * Kamil Essekkat (ekamil) 31 | * (gbutt) 32 | * Ben Buchacher (bbuchacher) 33 | * Ehtesh Choudhury (shurane) 34 | * Markus Fischer (mfn) 35 | * Fabien Wernli (faxm0dem) 36 | * Michael Weiser (michaelweiser) 37 | * (digital-wonderland) 38 | * cassiano (cassianoleal) 39 | * Matt Dainty (bodgit) 40 | * Alex Philipp (alex-sf) 41 | * (krzaczek) 42 | * Justin Lintz (jlintz) 43 | * Jeremy Falling (jjfalling) 44 | * Ian Babrou (bobrik) 45 | * Ferenc Erki (ferki) 46 | * George Heppner (gheppner) 47 | * Matt Hughes (matthughes) 48 | * Brian Lalor (blalor) 49 | * Paweł Krzaczkowski (krzaczek) 50 | * Ben Tse (bt5e) 51 | * Tom Hendrikx (whyscream) 52 | * Christian Vozar (christianvozar) 53 | * Magnus Baeck (magnusbaeck) 54 | * Robin Kearney (rk295) 55 | * (cfeio) 56 | * (malagoli) 57 | * Dan Sheridan (djs52) 58 | * Michael-Keith Bernard (SegFaultAX) 59 | * Simon Lundström (simmel) 60 | * (pkr1234) 61 | * Mark Feltner (feltnerm) 62 | * William Jimenez (wjimenez5271) 63 | * Jeremy Canady (jrmycanady) 64 | * Steven Ottenhoff (steffo) 65 | * Ole Rößner (Basster) 66 | * Jack (univerio) 67 | * Tomáš Mózes (hydrapolic) 68 | * Gary Gao (garyelephant) 69 | * Panagiotis Moustafellos (pmoust) 70 | * (pbamba) 71 | * Pavel Strashkin (xaka) 72 | * Wadim Kruse (wkruse) 73 | * Richard Megginson (richm) 74 | * Thibaut Ackermann (thib-ack) 75 | * (zzugg) 76 | * Julien Mancuso (petitout) 77 | * Spencer Herzberg (sherzberg) 78 | * Luke Waite (lukewaite) 79 | * (dtrv) 80 | * Christopher "Chief" Najewicz (chiefy) 81 | * Filipe Gonçalves (basex) 82 | * Sönke Liebau (soenkeliebau) 83 | * Timothy Schroder (tschroeder-zendesk) 84 | * Jared Carey (jpcarey) 85 | * Juraj Seffer (jurajseffer) 86 | * Roger Steneteg (rsteneteg) 87 | * Muhammad Junaid Muzammil (junmuz) 88 | * Loet Avramson (psypuff) 89 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-alpine3.17 as builder 2 | 3 | # Add the community repo for access to patchelf binary package 4 | RUN echo 'https://dl-cdn.alpinelinux.org/alpine/v3.16/community/' >> /etc/apk/repositories 5 | RUN apk --no-cache upgrade && apk --no-cache add build-base tar musl-utils openssl-dev patchelf 6 | # patchelf-wrapper is necessary now for cx_Freeze, but not for Curator itself. 7 | RUN pip3 install cx_Freeze patchelf-wrapper 8 | 9 | COPY . . 10 | RUN ln -s /lib/libc.musl-x86_64.so.1 ldd 11 | RUN ln -s /lib /lib64 12 | RUN pip3 install -r requirements.txt 13 | RUN python3 setup.py build_exe 14 | 15 | FROM alpine:3.17 16 | RUN apk --no-cache upgrade && apk --no-cache add openssl-dev expat 17 | COPY --from=builder build/exe.linux-x86_64-3.11 /curator/ 18 | RUN mkdir /.curator 19 | 20 | USER nobody:nobody 21 | ENV LD_LIBRARY_PATH /curator/lib:$LD_LIBRARY_PATH 22 | ENTRYPOINT ["/curator/curator"] 23 | 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2011–2019 Elasticsearch and contributors. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include Changelog.rst 2 | include CONTRIBUTORS CONTRIBUTING.md 3 | include LICENSE.txt 4 | include README.rst 5 | include Dockerfile 6 | recursive-exclude * __pycache__ 7 | recursive-exclude * *.py[co] 8 | recursive-include curator * 9 | recursive-include test * 10 | recursive-include docs * 11 | recursive-exclude curator *.pyc 12 | recursive-exclude curator *.pyo 13 | recursive-exclude docs *.pyc 14 | recursive-exclude docs *.pyo 15 | recursive-exclude test *.pyc 16 | recursive-exclude test *.pyo 17 | prune docs/_build 18 | prune docs/asciidoc/html_docs 19 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | In accordance with section 4d of the Apache 2.0 license (http://www.apache.org/licenses/LICENSE-2.0), 2 | this NOTICE file is included. 3 | 4 | All users mentioned in the CONTRIBUTORS file at https://github.com/elastic/curator/blob/master/CONTRIBUTORS 5 | must be included in any derivative work. 6 | 7 | All conditions of section 4 of the Apache 2.0 license will be enforced: 8 | 9 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in 10 | any medium, with or without modifications, and in Source or Object form, provided that You meet the 11 | following conditions: 12 | 13 | a. You must give any other recipients of the Work or Derivative Works a copy of this License; and 14 | b. You must cause any modified files to carry prominent notices stating that You changed the files; and 15 | c. You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, 16 | trademark, and attribution notices from the Source form of the Work, excluding those notices that do 17 | not pertain to any part of the Derivative Works; and 18 | d. If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that 19 | You distribute must include a readable copy of the attribution notices contained within such NOTICE 20 | file, excluding those notices that do not pertain to any part of the Derivative Works, in at least 21 | one of the following places: within a NOTICE text file distributed as part of the Derivative Works; 22 | within the Source form or documentation, if provided along with the Derivative Works; or, within a 23 | display generated by the Derivative Works, if and wherever such third-party notices normally appear. 24 | The contents of the NOTICE file are for informational purposes only and do not modify the License. 25 | You may add Your own attribution notices within Derivative Works that You distribute, alongside or as 26 | an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot 27 | be construed as modifying the License. 28 | 29 | You may add Your own copyright statement to Your modifications and may provide additional or different 30 | license terms and conditions for use, reproduction, or distribution of Your modifications, or for any 31 | such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise 32 | complies with the conditions stated in this License. 33 | 34 | Contributors: 35 | * Jordan Sissel (jordansissel) (For Logstash, first and foremost) 36 | * Shay Banon (kimchy) (For Elasticsearch, of course!) 37 | * Aaron Mildenstein (untergeek) 38 | * Njal Karevoll 39 | * François Deppierraz 40 | * Honza Kral (HonzaKral) 41 | * Benjamin Smith (benjaminws) 42 | * Colin Moller (LeftyBC) 43 | * Elliot (edgeofnite) 44 | * Ram Viswanadha (ramv) 45 | * Chris Meisinger (cmeisinger) 46 | * Stuart Warren (stuart-warren) 47 | * (gitshaw) 48 | * (sfritz) 49 | * (sjoelsam) 50 | * Jose Diaz-Gonzalez (josegonzalez) 51 | * Arie Bro (arieb) 52 | * David Harrigan (dharrigan) 53 | * Mathieu Geli (gelim) 54 | * Nick Ethier (nickethier) 55 | * Mohab Usama (mohabusama) 56 | * (gitshaw) 57 | * Stuart Warren (stuart-warren) 58 | * Xavier Calland (xavier-calland) 59 | * Chad Schellenger (cschellenger) 60 | * Kamil Essekkat (ekamil) 61 | * (gbutt) 62 | * Ben Buchacher (bbuchacher) 63 | * Ehtesh Choudhury (shurane) 64 | * Markus Fischer (mfn) 65 | * Fabien Wernli (faxm0dem) 66 | * Michael Weiser (michaelweiser) 67 | * (digital-wonderland) 68 | * cassiano (cassianoleal) 69 | * Matt Dainty (bodgit) 70 | * Alex Philipp (alex-sf) 71 | * (krzaczek) 72 | * Justin Lintz (jlintz) 73 | * Jeremy Falling (jjfalling) 74 | * Ian Babrou (bobrik) 75 | * Ferenc Erki (ferki) 76 | * George Heppner (gheppner) 77 | * Matt Hughes (matthughes) 78 | * Brian Lalor (blalor) 79 | * Paweł Krzaczkowski (krzaczek) 80 | * Ben Tse (bt5e) 81 | * Tom Hendrikx (whyscream) 82 | * Christian Vozar (christianvozar) 83 | * Magnus Baeck (magnusbaeck) 84 | * Robin Kearney (rk295) 85 | * (cfeio) 86 | * (malagoli) 87 | * Dan Sheridan (djs52) 88 | * Michael-Keith Bernard (SegFaultAX) 89 | * Simon Lundström (simmel) 90 | * (pkr1234) 91 | * Mark Feltner (feltnerm) 92 | * William Jimenez (wjimenez5271) 93 | * Jeremy Canady (jrmycanady) 94 | * Steven Ottenhoff (steffo) 95 | * Ole Rößner (Basster) 96 | * Jack (univerio) 97 | * Tomáš Mózes (hydrapolic) 98 | * Gary Gao (garyelephant) 99 | * Panagiotis Moustafellos (pmoust) 100 | * (pbamba) 101 | * Pavel Strashkin (xaka) 102 | * Wadim Kruse (wkruse) 103 | * Richard Megginson (richm) 104 | * Thibaut Ackermann (thib-ack) 105 | * (zzugg) 106 | * Julien Mancuso (petitout) 107 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # curator-opensearch 2 | 3 | This project is a fork of https://github.com/elastic/curator. The main motivation behind its creation was the lack of compatibility between elastic/curator and [OpenSearch](https://aws.amazon.com/blogs/opensource/introducing-opensearch/). 4 | The original curator worked well with OpenDistro. However, when trying to switch to OpenSearch, we found that elastic/curator does not support OpenSearch. 5 | 6 | To ensure compatibility with OpenSearch, we used [`opensearch-py`](https://opensearch-project.github.io/opensearch-py/) — a community-driven, Open Source fork of `elasticsearch-py`. 7 | 8 | 9 | ## Curator API Documentation 10 | 11 | Curator ships with both an API and a wrapper script (which is actually defined 12 | as an entry point). The API allows you to write your own scripts to accomplish 13 | similar goals, or even new and different things with the Curator API _and 14 | the Elasticsearch Python API_. 15 | 16 | * Curator API: http://curator.readthedocs.io/ 17 | 18 | * OpenSearch Python API: https://github.com/opensearch-project/opensearch-py/blob/main/USER_GUIDE.md 19 | 20 | 21 | ## Curator CLI Documentation 22 | 23 | The [Curator CLI Documentation](http://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html) is now a part of the document repository at http://elastic.co/guide. 24 | 25 | 26 | ## Getting Started 27 | 28 | The Getting Started guide is avaialble at https://www.elastic.co/guide/en/elasticsearch/client/curator/current/about.html 29 | 30 | curator-opensearch can be installed using the pip package manager: 31 | ```` 32 | $ pip install curator-opensearch 33 | ```` 34 | 35 | Running ``curator --help`` will show usage information. 36 | 37 | ## Community 38 | 39 | Please feel free to reach developers/maintainers and users via [GitHub Discussions](https://github.com/flant/curator-opensearch/discussions) for any questions regarding curator-opensearch. 40 | 41 | You're also welcome to follow [@flant_com](https://twitter.com/flant_com) to stay informed about all our Open Source initiatives. 42 | 43 | ## License 44 | 45 | Apache License 2.0, see [LICENSE](LICENSE). 46 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. _readme: 2 | 3 | 4 | Curator 5 | ======= 6 | 7 | Have indices in Elasticsearch? This is the tool for you! 8 | 9 | Like a museum curator manages the exhibits and collections on display, 10 | Elasticsearch Curator helps you curate, or manage your indices. 11 | 12 | 13 | `Curator API Documentation`_ 14 | ---------------------------- 15 | 16 | Curator ships with both an API and a wrapper script (which is actually defined 17 | as an entry point). The API allows you to write your own scripts to accomplish 18 | similar goals, or even new and different things with the `Curator API`_, and 19 | the `Elasticsearch Python API`_. 20 | 21 | .. _Curator API: http://curator.readthedocs.io/ 22 | 23 | .. _Curator API Documentation: `Curator API`_ 24 | 25 | .. _Elasticsearch Python API: http://elasticsearch-py.readthedocs.io/ 26 | 27 | 28 | `Curator CLI Documentation`_ 29 | ---------------------------- 30 | 31 | The `Curator CLI Documentation`_ is now a part of the document repository at 32 | http://elastic.co/guide at http://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html 33 | 34 | .. _Curator CLI Documentation: http://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html 35 | 36 | `Getting Started`_ 37 | ------------------ 38 | 39 | .. _Getting Started: https://www.elastic.co/guide/en/elasticsearch/client/curator/current/about.html 40 | 41 | See the `Installation guide `_ 42 | and the `command-line usage guide `_ 43 | 44 | Running ``curator --help`` will also show usage information. 45 | 46 | Contributing 47 | ------------ 48 | 49 | * fork the repo 50 | * make changes in your fork 51 | * add tests to cover your changes (if necessary) 52 | * run tests 53 | * sign the `CLA `_ 54 | * send a pull request! 55 | 56 | To run from source, use the ``run_curator.py`` script in the root directory of 57 | the project. 58 | 59 | Running Tests 60 | ------------- 61 | 62 | To run the test suite just run ``python setup.py test`` 63 | 64 | When changing code, contributing new code or fixing a bug please make sure you 65 | include tests in your PR (or mark it as without tests so that someone else can 66 | pick it up to add the tests). When fixing a bug please make sure the test 67 | actually tests the bug - it should fail without the code changes and pass after 68 | they're applied (it can still be one commit of course). 69 | 70 | The tests will try to connect to your local elasticsearch instance and run 71 | integration tests against it. This will delete all the data stored there! You 72 | can use the env variable ``TEST_ES_SERVER`` to point to a different instance 73 | (for example, 'otherhost:9203'). 74 | 75 | 76 | Origins 77 | ------- 78 | 79 | Curator was first called ``clearESindices.py`` [1]_ and was almost immediately 80 | renamed to ``logstash_index_cleaner.py`` [1]_. After a time it was migrated 81 | under the `logstash `_ repository as 82 | ``expire_logs``. Soon thereafter, Jordan Sissel was hired by Elasticsearch, as 83 | was the original author of this tool. It became Elasticsearch Curator after 84 | that and is now hosted at `elastic/curator `_. 85 | 86 | .. [1] `LOGSTASH-211 `_. 87 | -------------------------------------------------------------------------------- /binary_release.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import sys 4 | import shutil 5 | import hashlib 6 | 7 | # This script simply takes the output of `python setup.py build_exe` and makes 8 | # a compressed archive (zip for windows, tar.gz for Linux) for distribution. 9 | 10 | # Utility function to read from file. 11 | def fread(fname): 12 | return open(os.path.join(os.path.dirname(__file__), fname)).read() 13 | 14 | def get_version(): 15 | VERSIONFILE="curator/_version.py" 16 | verstrline = fread(VERSIONFILE).strip() 17 | vsre = r"^__version__ = ['\"]([^'\"]*)['\"]" 18 | mo = re.search(vsre, verstrline, re.M) 19 | if mo: 20 | VERSION = mo.group(1) 21 | else: 22 | raise RuntimeError("Unable to find version string in %s." % (VERSIONFILE,)) 23 | build_number = os.environ.get('CURATOR_BUILD_NUMBER', None) 24 | if build_number: 25 | return VERSION + "b{}".format(build_number) 26 | return VERSION 27 | 28 | archive_format = 'gztar' 29 | enviro = dict(os.environ) 30 | platform = sys.platform 31 | pyver = str(sys.version_info[0]) + '.' + str(sys.version_info[1]) 32 | if platform == 'win32': 33 | # Win32 stuff 34 | archive_format = 'zip' 35 | build_name = 'exe.win-' + enviro['PROCESSOR_ARCHITECTURE'].lower() + '-' + pyver 36 | target_name = "curator-" + str(get_version()) + "-amd64" 37 | elif platform == 'linux' or platform == 'linux2': 38 | sys_string = enviro['_system_type'].lower() + '-' + enviro['_system_arch'].lower() 39 | build_name = 'exe.' + sys_string + '-' + pyver 40 | target_name = "curator-" + str(get_version()) + "-" + sys_string 41 | else: 42 | # Unsupported platform? 43 | print('Your platform ({0}) is not yet supported for binary build/distribution.'.format(platform)) 44 | sys.exit(1) 45 | 46 | #sys_string = sys_type + '-' + sys_arch 47 | #build_name = 'exe.' + sys_string + '-' + pyver 48 | #print('Expected build directory: {0}'.format(build_name)) 49 | build_path = os.path.join('build', build_name) 50 | 51 | if os.path.exists(build_path): 52 | #print("I found the path: {0}".format(build_path)) 53 | 54 | target_path = os.path.join('.', target_name) 55 | 56 | # Check to see if an older directory exists... 57 | if os.path.exists(target_path): 58 | print('An older build exists at {0}. Please delete this before continuing.'.format(target_path)) 59 | sys.exit(1) 60 | else: 61 | shutil.copytree(build_path, target_path) 62 | 63 | # Ensure the rename went smoothly, then continue 64 | if os.path.exists(target_path): 65 | #print("Build successfully renamed") 66 | if float(pyver) >= 2.7: 67 | shutil.make_archive('elasticsearch-' + target_name, archive_format, '.', target_path) 68 | if platform == 'win32': 69 | fname = 'elasticsearch-' + target_name + '.zip' 70 | else: 71 | fname = 'elasticsearch-' + target_name + '.tar.gz' 72 | # Clean up directory if we made a viable archive. 73 | if os.path.exists(fname): 74 | shutil.rmtree(target_path) 75 | else: 76 | print('Something went wrong creating the archive {0}'.format(fname)) 77 | sys.exit(1) 78 | md5sum = hashlib.md5(open(fname, 'rb').read()).hexdigest() 79 | sha1sum = hashlib.sha1(open(fname, 'rb').read()).hexdigest() 80 | with open(fname + ".md5.txt", "w") as md5_file: 81 | md5_file.write("{0}".format(md5sum)) 82 | with open(fname + ".sha1.txt", "w") as sha1_file: 83 | sha1_file.write("{0}".format(sha1sum)) 84 | print('Archive: {0}'.format(fname)) 85 | print('{0} = {1}'.format(fname + ".md5.txt", md5sum)) 86 | print('{0} = {1}'.format(fname + ".sha1.txt", sha1sum)) 87 | else: 88 | print('Your python version ({0}) is too old to use with shutil.make_archive.'.format(pyver)) 89 | print('You can manually compress the {0} directory to achieve the same result.'.format(target_name)) 90 | else: 91 | # We couldn't find a build_path 92 | print("Build not found. Please run 'python setup.py build_exe' to create the build directory.") 93 | sys.exit(1) 94 | -------------------------------------------------------------------------------- /curator/__init__.py: -------------------------------------------------------------------------------- 1 | """Curator importing""" 2 | from curator._version import __version__ 3 | from curator.exceptions import * 4 | from curator.defaults import * 5 | from curator.validators import * 6 | from curator.logtools import * 7 | from curator.utils import * 8 | from curator.indexlist import IndexList 9 | from curator.snapshotlist import SnapshotList 10 | from curator.actions import * 11 | from curator.cli import * 12 | from curator.repomgrcli import * 13 | -------------------------------------------------------------------------------- /curator/_version.py: -------------------------------------------------------------------------------- 1 | """Curator Version""" 2 | __version__ = '0.0.13' 3 | -------------------------------------------------------------------------------- /curator/cli_singletons/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flant/curator-opensearch/7e50540e861fe04e2d90a5eaf3437d70f62a34bc/curator/cli_singletons/__init__.py -------------------------------------------------------------------------------- /curator/cli_singletons/alias.py: -------------------------------------------------------------------------------- 1 | """Alias Singleton""" 2 | import click 3 | from curator.cli_singletons.object_class import cli_action 4 | from curator.cli_singletons.utils import get_width, json_to_dict, validate_filter_json 5 | 6 | @click.command(context_settings=get_width()) 7 | @click.option('--name', type=str, help='Alias name', required=True) 8 | @click.option( 9 | '--add', 10 | callback=validate_filter_json, 11 | help='JSON array of filters selecting indices to ADD to alias', 12 | default=None 13 | ) 14 | @click.option( 15 | '--remove', 16 | callback=validate_filter_json, 17 | help='JSON array of filters selecting indices to REMOVE from alias', 18 | default=None 19 | ) 20 | @click.option( 21 | '--warn_if_no_indices', 22 | is_flag=True, 23 | help='Do not raise exception if there are no actionable indices in add/remove' 24 | ) 25 | @click.option( 26 | '--extra_settings', 27 | help='JSON version of extra_settings (see documentation)', 28 | callback=json_to_dict 29 | ) 30 | @click.option( 31 | '--allow_ilm_indices/--no-allow_ilm_indices', 32 | help='Allow Curator to operate on Index Lifecycle Management monitored indices.', 33 | default=False, 34 | show_default=True 35 | ) 36 | @click.pass_context 37 | def alias(ctx, name, add, remove, warn_if_no_indices, extra_settings, allow_ilm_indices): 38 | """ 39 | Add/Remove Indices to/from Alias 40 | """ 41 | manual_options = { 42 | 'name': name, 43 | 'extra_settings': extra_settings, 44 | 'allow_ilm_indices': allow_ilm_indices, 45 | } 46 | # ctx.info_name is the name of the function or name specified in @click.command decorator 47 | ignore_empty_list = warn_if_no_indices 48 | action = cli_action( 49 | ctx.info_name, 50 | ctx.obj['config']['client'], 51 | manual_options, 52 | [], # filter_list is empty in our case 53 | ignore_empty_list, 54 | add=add, remove=remove, warn_if_no_indices=warn_if_no_indices, # alias specific kwargs 55 | ) 56 | action.do_singleton_action(dry_run=ctx.obj['dry_run']) 57 | -------------------------------------------------------------------------------- /curator/cli_singletons/allocation.py: -------------------------------------------------------------------------------- 1 | """Allocation Singleton""" 2 | import click 3 | from curator.cli_singletons.object_class import cli_action 4 | from curator.cli_singletons.utils import get_width, validate_filter_json 5 | 6 | @click.command(context_settings=get_width()) 7 | @click.option('--key', type=str, required=True, help='Node identification tag') 8 | @click.option('--value', type=str, default=None, help='Value associated with --key') 9 | @click.option('--allocation_type', type=click.Choice(['require', 'include', 'exclude'])) 10 | @click.option( 11 | '--wait_for_completion/--no-wait_for_completion', 12 | default=False, 13 | help='Wait for the allocation to complete', 14 | show_default=True 15 | ) 16 | @click.option( 17 | '--max_wait', 18 | default=-1, 19 | type=int, 20 | help='Maximum number of seconds to wait_for_completion', 21 | show_default=True 22 | ) 23 | @click.option( 24 | '--wait_interval', 25 | default=9, 26 | type=int, 27 | help='Seconds to wait between completion checks.', 28 | show_default=True 29 | ) 30 | @click.option( 31 | '--ignore_empty_list', 32 | is_flag=True, 33 | help='Do not raise exception if there are no actionable indices' 34 | ) 35 | @click.option( 36 | '--allow_ilm_indices/--no-allow_ilm_indices', 37 | help='Allow Curator to operate on Index Lifecycle Management monitored indices.', 38 | default=False, 39 | show_default=True 40 | ) 41 | @click.option( 42 | '--filter_list', 43 | callback=validate_filter_json, 44 | help='JSON array of filters selecting indices to act on.', 45 | required=True 46 | ) 47 | @click.pass_context 48 | def allocation( 49 | ctx, 50 | key, 51 | value, 52 | allocation_type, 53 | wait_for_completion, 54 | max_wait, 55 | wait_interval, 56 | ignore_empty_list, 57 | allow_ilm_indices, 58 | filter_list 59 | ): 60 | """ 61 | Shard Routing Allocation 62 | """ 63 | manual_options = { 64 | 'key': key, 65 | 'value': value, 66 | 'allocation_type': allocation_type, 67 | 'wait_for_completion': wait_for_completion, 68 | 'max_wait': max_wait, 69 | 'wait_interval': wait_interval, 70 | 'allow_ilm_indices': allow_ilm_indices, 71 | } 72 | # ctx.info_name is the name of the function or name specified in @click.command decorator 73 | action = cli_action( 74 | ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list) 75 | action.do_singleton_action(dry_run=ctx.obj['dry_run']) 76 | -------------------------------------------------------------------------------- /curator/cli_singletons/close.py: -------------------------------------------------------------------------------- 1 | """Close Singleton""" 2 | import click 3 | from curator.cli_singletons.object_class import cli_action 4 | from curator.cli_singletons.utils import get_width, validate_filter_json 5 | 6 | @click.command(context_settings=get_width()) 7 | @click.option('--delete_aliases', is_flag=True, help='Delete all aliases from indices to be closed') 8 | @click.option('--skip_flush', is_flag=True, help='Skip flush phase for indices to be closed') 9 | @click.option( 10 | '--ignore_empty_list', 11 | is_flag=True, 12 | help='Do not raise exception if there are no actionable indices' 13 | ) 14 | @click.option( 15 | '--allow_ilm_indices/--no-allow_ilm_indices', 16 | help='Allow Curator to operate on Index Lifecycle Management monitored indices.', 17 | default=False, 18 | show_default=True 19 | ) 20 | @click.option( 21 | '--filter_list', 22 | callback=validate_filter_json, 23 | help='JSON array of filters selecting indices to act on.', 24 | required=True 25 | ) 26 | @click.pass_context 27 | def close(ctx, delete_aliases, skip_flush, ignore_empty_list, allow_ilm_indices, filter_list): 28 | """ 29 | Close Indices 30 | """ 31 | manual_options = { 32 | 'skip_flush': skip_flush, 33 | 'delete_aliases': delete_aliases, 34 | 'allow_ilm_indices': allow_ilm_indices, 35 | } 36 | # ctx.info_name is the name of the function or name specified in @click.command decorator 37 | action = cli_action( 38 | ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list) 39 | action.do_singleton_action(dry_run=ctx.obj['dry_run']) 40 | -------------------------------------------------------------------------------- /curator/cli_singletons/delete.py: -------------------------------------------------------------------------------- 1 | """Delete Index and Delete Snapshot Singletons""" 2 | import click 3 | from curator.cli_singletons.object_class import cli_action 4 | from curator.cli_singletons.utils import get_width, validate_filter_json 5 | 6 | #### Indices #### 7 | @click.command(context_settings=get_width()) 8 | @click.option( 9 | '--ignore_empty_list', 10 | is_flag=True, 11 | help='Do not raise exception if there are no actionable indices' 12 | ) 13 | @click.option( 14 | '--allow_ilm_indices/--no-allow_ilm_indices', 15 | help='Allow Curator to operate on Index Lifecycle Management monitored indices.', 16 | default=False, 17 | show_default=True 18 | ) 19 | @click.option( 20 | '--filter_list', 21 | callback=validate_filter_json, 22 | help='JSON array of filters selecting indices to act on.', 23 | required=True 24 | ) 25 | @click.pass_context 26 | def delete_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list): 27 | """ 28 | Delete Indices 29 | """ 30 | # ctx.info_name is the name of the function or name specified in @click.command decorator 31 | action = cli_action( 32 | 'delete_indices', 33 | ctx.obj['config']['client'], 34 | {'allow_ilm_indices':allow_ilm_indices}, 35 | filter_list, 36 | ignore_empty_list 37 | ) 38 | action.do_singleton_action(dry_run=ctx.obj['dry_run']) 39 | 40 | #### Snapshots #### 41 | @click.command(context_settings=get_width()) 42 | @click.option('--repository', type=str, required=True, help='Snapshot repository name') 43 | @click.option('--retry_count', type=int, help='Number of times to retry (max 3)') 44 | @click.option('--retry_interval', type=int, help='Time in seconds between retries') 45 | @click.option( 46 | '--ignore_empty_list', 47 | is_flag=True, 48 | help='Do not raise exception if there are no actionable snapshots' 49 | ) 50 | @click.option( 51 | '--allow_ilm_indices/--no-allow_ilm_indices', 52 | help='Allow Curator to operate on Index Lifecycle Management monitored indices.', 53 | default=False, 54 | show_default=True 55 | ) 56 | @click.option( 57 | '--filter_list', 58 | callback=validate_filter_json, 59 | help='JSON array of filters selecting snapshots to act on.', 60 | required=True 61 | ) 62 | @click.pass_context 63 | def delete_snapshots( 64 | ctx, repository, retry_count, retry_interval, 65 | ignore_empty_list, allow_ilm_indices, filter_list 66 | ): 67 | """ 68 | Delete Snapshots 69 | """ 70 | manual_options = { 71 | 'retry_count': retry_count, 72 | 'retry_interval': retry_interval, 73 | 'allow_ilm_indices': allow_ilm_indices, 74 | } 75 | # ctx.info_name is the name of the function or name specified in @click.command decorator 76 | action = cli_action( 77 | 'delete_snapshots', 78 | ctx.obj['config']['client'], 79 | manual_options, 80 | filter_list, 81 | ignore_empty_list, 82 | repository=repository 83 | ) 84 | action.do_singleton_action(dry_run=ctx.obj['dry_run']) 85 | -------------------------------------------------------------------------------- /curator/cli_singletons/forcemerge.py: -------------------------------------------------------------------------------- 1 | """ForceMerge Singleton""" 2 | import click 3 | from curator.cli_singletons.object_class import cli_action 4 | from curator.cli_singletons.utils import get_width, validate_filter_json 5 | 6 | @click.command(context_settings=get_width()) 7 | @click.option( 8 | '--max_num_segments', 9 | type=int, 10 | required=True, 11 | help='Maximum number of segments per shard (minimum of 1)' 12 | ) 13 | @click.option( 14 | '--delay', 15 | type=float, 16 | help='Time in seconds to delay between operations. Default 0. Maximum 3600' 17 | ) 18 | @click.option( 19 | '--ignore_empty_list', 20 | is_flag=True, 21 | help='Do not raise exception if there are no actionable indices' 22 | ) 23 | @click.option( 24 | '--allow_ilm_indices/--no-allow_ilm_indices', 25 | help='Allow Curator to operate on Index Lifecycle Management monitored indices.', 26 | default=False, 27 | show_default=True 28 | ) 29 | @click.option( 30 | '--filter_list', 31 | callback=validate_filter_json, 32 | help='JSON array of filters selecting indices to act on.', 33 | required=True) 34 | @click.pass_context 35 | def forcemerge(ctx, max_num_segments, delay, ignore_empty_list, allow_ilm_indices, filter_list): 36 | """ 37 | forceMerge Indices (reduce segment count) 38 | """ 39 | manual_options = { 40 | 'max_num_segments': max_num_segments, 41 | 'delay': delay, 42 | 'allow_ilm_indices': allow_ilm_indices, 43 | } 44 | # ctx.info_name is the name of the function or name specified in @click.command decorator 45 | action = cli_action( 46 | ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list) 47 | action.do_singleton_action(dry_run=ctx.obj['dry_run']) 48 | -------------------------------------------------------------------------------- /curator/cli_singletons/freeze.py: -------------------------------------------------------------------------------- 1 | """Index Freeze Singleton""" 2 | import click 3 | from curator.cli_singletons.object_class import cli_action 4 | from curator.cli_singletons.utils import get_width, validate_filter_json 5 | 6 | @click.command(context_settings=get_width()) 7 | @click.option( 8 | '--ignore_empty_list', 9 | is_flag=True, 10 | help='Do not raise exception if there are no actionable indices' 11 | ) 12 | @click.option( 13 | '--allow_ilm_indices/--no-allow_ilm_indices', 14 | help='Allow Curator to operate on Index Lifecycle Management monitored indices.', 15 | default=False, 16 | show_default=True 17 | ) 18 | @click.option( 19 | '--filter_list', 20 | callback=validate_filter_json, 21 | help='JSON array of filters selecting indices to act on.', 22 | required=True 23 | ) 24 | @click.pass_context 25 | def freeze(ctx, ignore_empty_list, allow_ilm_indices, filter_list): 26 | """ 27 | Freeze Indices 28 | """ 29 | # ctx.info_name is the name of the function or name specified in @click.command decorator 30 | action = cli_action( 31 | ctx.info_name, 32 | ctx.obj['config']['client'], 33 | {'allow_ilm_indices':allow_ilm_indices}, 34 | filter_list, 35 | ignore_empty_list 36 | ) 37 | action.do_singleton_action(dry_run=ctx.obj['dry_run']) 38 | -------------------------------------------------------------------------------- /curator/cli_singletons/open_indices.py: -------------------------------------------------------------------------------- 1 | """Open (closed) Index Singleton""" 2 | import click 3 | from curator.cli_singletons.object_class import cli_action 4 | from curator.cli_singletons.utils import get_width, validate_filter_json 5 | 6 | @click.command(name='open', context_settings=get_width()) 7 | @click.option( 8 | '--ignore_empty_list', 9 | is_flag=True, 10 | help='Do not raise exception if there are no actionable indices' 11 | ) 12 | @click.option( 13 | '--allow_ilm_indices/--no-allow_ilm_indices', 14 | help='Allow Curator to operate on Index Lifecycle Management monitored indices.', 15 | default=False, 16 | show_default=True 17 | ) 18 | @click.option( 19 | '--filter_list', 20 | callback=validate_filter_json, 21 | help='JSON array of filters selecting indices to act on.', 22 | required=True 23 | ) 24 | @click.pass_context 25 | def open_indices(ctx, ignore_empty_list, allow_ilm_indices, filter_list): 26 | """ 27 | Open Indices 28 | """ 29 | # ctx.info_name is the name of the function or name specified in @click.command decorator 30 | action = cli_action( 31 | ctx.info_name, 32 | ctx.obj['config']['client'], 33 | {'allow_ilm_indices':allow_ilm_indices}, 34 | filter_list, 35 | ignore_empty_list 36 | ) 37 | action.do_singleton_action(dry_run=ctx.obj['dry_run']) 38 | -------------------------------------------------------------------------------- /curator/cli_singletons/replicas.py: -------------------------------------------------------------------------------- 1 | """Change Replica Count Singleton""" 2 | import click 3 | from curator.cli_singletons.object_class import cli_action 4 | from curator.cli_singletons.utils import get_width, validate_filter_json 5 | 6 | @click.command(context_settings=get_width()) 7 | @click.option('--count', type=int, required=True, help='Number of replicas (max 10)') 8 | @click.option( 9 | '--wait_for_completion/--no-wait_for_completion', 10 | default=False, 11 | help='Wait for replication to complete', 12 | show_default=True 13 | ) 14 | @click.option( 15 | '--ignore_empty_list', 16 | is_flag=True, 17 | help='Do not raise exception if there are no actionable indices' 18 | ) 19 | @click.option( 20 | '--allow_ilm_indices/--no-allow_ilm_indices', 21 | help='Allow Curator to operate on Index Lifecycle Management monitored indices.', 22 | default=False, 23 | show_default=True 24 | ) 25 | @click.option( 26 | '--filter_list', 27 | callback=validate_filter_json, 28 | help='JSON array of filters selecting indices to act on.', 29 | required=True 30 | ) 31 | @click.pass_context 32 | def replicas(ctx, count, wait_for_completion, ignore_empty_list, allow_ilm_indices, filter_list): 33 | """ 34 | Change Replica Count 35 | """ 36 | manual_options = { 37 | 'count': count, 38 | 'wait_for_completion': wait_for_completion, 39 | 'allow_ilm_indices': allow_ilm_indices, 40 | } 41 | # ctx.info_name is the name of the function or name specified in @click.command decorator 42 | action = cli_action( 43 | ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list) 44 | action.do_singleton_action(dry_run=ctx.obj['dry_run']) 45 | -------------------------------------------------------------------------------- /curator/cli_singletons/restore.py: -------------------------------------------------------------------------------- 1 | """Snapshot Restore Singleton""" 2 | import click 3 | from curator.cli_singletons.object_class import cli_action 4 | from curator.cli_singletons.utils import get_width, json_to_dict, validate_filter_json 5 | 6 | @click.command(context_settings=get_width()) 7 | @click.option('--repository', type=str, required=True, help='Snapshot repository') 8 | @click.option('--name', type=str, help='Snapshot name', required=False, default=None) 9 | @click.option( 10 | '--index', 11 | multiple=True, 12 | help='Index name to restore. (Can invoke repeatedly for multiple indices)' 13 | ) 14 | @click.option( 15 | '--rename_pattern', 16 | type=str, 17 | help='Rename pattern', 18 | required=False, 19 | default=None 20 | ) 21 | @click.option( 22 | '--rename_replacement', 23 | type=str, 24 | help='Rename replacement', 25 | required=False, 26 | default=None 27 | ) 28 | @click.option( 29 | '--extra_settings', 30 | type=str, 31 | help='JSON version of extra_settings (see documentation)', 32 | callback=json_to_dict 33 | ) 34 | @click.option( 35 | '--include_aliases', 36 | is_flag=True, 37 | show_default=True, 38 | help='Include aliases with restored indices.' 39 | ) 40 | @click.option( 41 | '--ignore_unavailable', 42 | is_flag=True, 43 | show_default=True, 44 | help='Ignore unavailable shards/indices.' 45 | ) 46 | @click.option( 47 | '--include_global_state', 48 | is_flag=True, 49 | show_default=True, 50 | help='Restore cluster global state with snapshot.' 51 | ) 52 | @click.option( 53 | '--partial', 54 | is_flag=True, 55 | show_default=True, 56 | help='Restore partial data (from snapshot taken with --partial).' 57 | ) 58 | @click.option( 59 | '--wait_for_completion/--no-wait_for_completion', 60 | default=True, 61 | show_default=True, 62 | help='Wait for the snapshot to complete' 63 | ) 64 | @click.option( 65 | '--wait_interval', 66 | default=9, 67 | type=int, 68 | help='Seconds to wait between completion checks.' 69 | ) 70 | @click.option( 71 | '--max_wait', 72 | default=-1, 73 | type=int, 74 | help='Maximum number of seconds to wait_for_completion' 75 | ) 76 | @click.option( 77 | '--skip_repo_fs_check', 78 | is_flag=True, 79 | show_default=True, 80 | help='Skip repository filesystem access validation.' 81 | ) 82 | @click.option( 83 | '--ignore_empty_list', 84 | is_flag=True, 85 | help='Do not raise exception if there are no actionable indices' 86 | ) 87 | @click.option( 88 | '--allow_ilm_indices/--no-allow_ilm_indices', 89 | help='Allow Curator to operate on Index Lifecycle Management monitored indices.', 90 | default=False, 91 | show_default=True 92 | ) 93 | @click.option( 94 | '--filter_list', 95 | callback=validate_filter_json, 96 | help='JSON array of filters selecting snapshots to act on.', 97 | required=True 98 | ) 99 | @click.pass_context 100 | def restore( 101 | ctx, repository, name, index, rename_pattern, rename_replacement, extra_settings, 102 | include_aliases, ignore_unavailable, include_global_state, partial, wait_for_completion, 103 | wait_interval, max_wait, skip_repo_fs_check, ignore_empty_list, allow_ilm_indices, 104 | filter_list 105 | ): 106 | """ 107 | Restore Indices 108 | """ 109 | indices = list(index) 110 | manual_options = { 111 | 'name': name, 112 | 'indices': indices, 113 | 'rename_pattern': rename_pattern, 114 | 'rename_replacement': rename_replacement, 115 | 'ignore_unavailable': ignore_unavailable, 116 | 'include_aliases': include_aliases, 117 | 'include_global_state': include_global_state, 118 | 'partial': partial, 119 | 'skip_repo_fs_check': skip_repo_fs_check, 120 | 'wait_for_completion': wait_for_completion, 121 | 'max_wait': max_wait, 122 | 'wait_interval': wait_interval, 123 | 'allow_ilm_indices': allow_ilm_indices, 124 | } 125 | # ctx.info_name is the name of the function or name specified in @click.command decorator 126 | action = cli_action( 127 | ctx.info_name, 128 | ctx.obj['config']['client'], 129 | manual_options, 130 | filter_list, 131 | ignore_empty_list, 132 | repository=repository 133 | ) 134 | action.do_singleton_action(dry_run=ctx.obj['dry_run']) 135 | -------------------------------------------------------------------------------- /curator/cli_singletons/rollover.py: -------------------------------------------------------------------------------- 1 | """Index Rollover Singleton""" 2 | import click 3 | from curator.cli_singletons.object_class import cli_action 4 | from curator.cli_singletons.utils import get_width, json_to_dict, validate_filter_json 5 | 6 | @click.command(context_settings=get_width()) 7 | @click.option('--name', type=str, help='Alias name', required=True) 8 | @click.option('--max_age', type=str, help='max_age condition value (see documentation)') 9 | @click.option('--max_docs', type=str, help='max_docs condition value (see documentation)') 10 | @click.option('--max_size', type=str, help='max_size condition value (see documentation)') 11 | @click.option( 12 | '--extra_settings', 13 | type=str, 14 | help='JSON version of extra_settings (see documentation)', 15 | callback=json_to_dict 16 | ) 17 | @click.option( 18 | '--new_index', 19 | type=str, 20 | help='Optional new index name (see documentation)' 21 | ) 22 | @click.option( 23 | '--wait_for_active_shards', 24 | type=int, 25 | default=1, 26 | show_default=True, 27 | help='Wait for number of shards to be active before returning' 28 | ) 29 | @click.option( 30 | '--allow_ilm_indices/--no-allow_ilm_indices', 31 | help='Allow Curator to operate on Index Lifecycle Management monitored indices.', 32 | default=False, 33 | show_default=True 34 | ) 35 | @click.pass_context 36 | def rollover( 37 | ctx, name, max_age, max_docs, max_size, extra_settings, new_index, wait_for_active_shards, 38 | allow_ilm_indices 39 | ): 40 | """ 41 | Rollover Index associated with Alias 42 | """ 43 | conditions = {} 44 | for cond in ['max_age', 'max_docs', 'max_size']: 45 | if eval(cond) is not None: 46 | conditions[cond] = eval(cond) 47 | manual_options = { 48 | 'name': name, 49 | 'conditions': conditions, 50 | 'allow_ilm_indices': allow_ilm_indices, 51 | } 52 | # ctx.info_name is the name of the function or name specified in @click.command decorator 53 | action = cli_action( 54 | ctx.info_name, ctx.obj['config']['client'], manual_options, [], True, 55 | extra_settings=extra_settings, 56 | new_index=new_index, 57 | wait_for_active_shards=wait_for_active_shards 58 | ) 59 | action.do_singleton_action(dry_run=ctx.obj['dry_run']) 60 | -------------------------------------------------------------------------------- /curator/cli_singletons/show.py: -------------------------------------------------------------------------------- 1 | """Show Index/Snapshot Singletons""" 2 | from datetime import datetime 3 | import click 4 | from curator.cli_singletons.object_class import cli_action 5 | from curator.cli_singletons.utils import get_width, validate_filter_json 6 | from curator.utils import byte_size 7 | 8 | 9 | #### Indices #### 10 | @click.command(context_settings=get_width()) 11 | @click.option('--verbose', help='Show verbose output.', is_flag=True, show_default=True) 12 | @click.option('--header', help='Print header if --verbose', is_flag=True, show_default=True) 13 | @click.option('--epoch', help='Print time as epoch if --verbose', is_flag=True, show_default=True) 14 | @click.option( 15 | '--ignore_empty_list', 16 | is_flag=True, 17 | help='Do not raise exception if there are no actionable indices' 18 | ) 19 | @click.option( 20 | '--allow_ilm_indices/--no-allow_ilm_indices', 21 | help='Allow Curator to operate on Index Lifecycle Management monitored indices.', 22 | default=False, 23 | show_default=True 24 | ) 25 | @click.option( 26 | '--filter_list', 27 | callback=validate_filter_json, 28 | default='{"filtertype":"none"}', 29 | help='JSON string representing an array of filters.' 30 | ) 31 | @click.pass_context 32 | def show_indices(ctx, verbose, header, epoch, ignore_empty_list, allow_ilm_indices, filter_list): 33 | """ 34 | Show Indices 35 | """ 36 | # ctx.info_name is the name of the function or name specified in @click.command decorator 37 | action = cli_action( 38 | 'show_indices', 39 | ctx.obj['config']['client'], 40 | {'allow_ilm_indices': allow_ilm_indices}, 41 | filter_list, 42 | ignore_empty_list 43 | ) 44 | action.get_list_object() 45 | action.do_filters() 46 | indices = sorted(action.list_object.indices) 47 | # Do some calculations to figure out the proper column sizes 48 | allbytes = [] 49 | alldocs = [] 50 | for idx in indices: 51 | allbytes.append(byte_size(action.list_object.index_info[idx]['size_in_bytes'])) 52 | alldocs.append(str(action.list_object.index_info[idx]['docs'])) 53 | if epoch: 54 | timeformat = '{6:>13}' 55 | column = 'creation_date' 56 | else: 57 | timeformat = '{6:>20}' 58 | column = 'Creation Timestamp' 59 | formatting = ( 60 | '{0:' + str(len(max(indices, key=len))) + '} ' 61 | '{1:>5} ' 62 | '{2:>' + str(len(max(allbytes, key=len)) + 1) + '} ' 63 | '{3:>' + str(len(max(alldocs, key=len)) + 1) + '} ' 64 | '{4:>3} {5:>3} ' + timeformat 65 | ) 66 | # Print the header, if both verbose and header are enabled 67 | if header and verbose: 68 | click.secho( 69 | formatting.format( 70 | 'Index', 'State', 'Size', 'Docs', 'Pri', 'Rep', column 71 | ), bold=True, underline=True 72 | ) 73 | # Loop through indices and print info, if verbose 74 | for idx in indices: 75 | data = action.list_object.index_info[idx] 76 | if verbose: 77 | if epoch: 78 | datefield = data['age']['creation_date'] if 'creation_date' in data['age'] else 0 79 | else: 80 | datefield = '{0}Z'.format( 81 | datetime.utcfromtimestamp( 82 | data['age']['creation_date'] 83 | ).isoformat()) if 'creation_date' in data['age'] else 'unknown/closed' 84 | click.echo( 85 | formatting.format( 86 | idx, data['state'], byte_size(data['size_in_bytes']), 87 | data['docs'], data['number_of_shards'], data['number_of_replicas'], 88 | datefield 89 | ) 90 | ) 91 | else: 92 | click.echo('{0}'.format(idx)) 93 | 94 | #### Snapshots #### 95 | @click.command(context_settings=get_width()) 96 | @click.option('--repository', type=str, required=True, help='Snapshot repository name') 97 | @click.option( 98 | '--ignore_empty_list', 99 | is_flag=True, 100 | help='Do not raise exception if there are no actionable snapshots' 101 | ) 102 | @click.option( 103 | '--filter_list', 104 | callback=validate_filter_json, 105 | default='{"filtertype":"none"}', 106 | help='JSON string representing an array of filters.' 107 | ) 108 | @click.pass_context 109 | def show_snapshots(ctx, repository, ignore_empty_list, filter_list): 110 | """ 111 | Show Snapshots 112 | """ 113 | # ctx.info_name is the name of the function or name specified in @click.command decorator 114 | action = cli_action( 115 | 'show_snapshots', 116 | ctx.obj['config']['client'], 117 | {}, 118 | filter_list, 119 | ignore_empty_list, 120 | repository=repository 121 | ) 122 | action.get_list_object() 123 | action.do_filters() 124 | for snapshot in sorted(action.list_object.snapshots): 125 | click.secho('{0}'.format(snapshot)) 126 | -------------------------------------------------------------------------------- /curator/cli_singletons/shrink.py: -------------------------------------------------------------------------------- 1 | """Shrink Index Singleton""" 2 | import click 3 | from curator.cli_singletons.object_class import cli_action 4 | from curator.cli_singletons.utils import get_width, json_to_dict, validate_filter_json 5 | 6 | @click.command(context_settings=get_width()) 7 | @click.option( 8 | '--shrink_node', 9 | default='DETERMINISTIC', 10 | type=str, 11 | help='Named node, or DETERMINISTIC', 12 | show_default=True 13 | ) 14 | @click.option( 15 | '--node_filters', 16 | help='JSON version of node_filters (see documentation)', 17 | callback=json_to_dict 18 | ) 19 | @click.option( 20 | '--number_of_shards', 21 | default=1, 22 | type=int, 23 | help='Shrink to this many shards per index' 24 | ) 25 | @click.option( 26 | '--number_of_replicas', 27 | default=1, 28 | type=int, 29 | help='Number of replicas for the target index', 30 | show_default=True 31 | ) 32 | @click.option( 33 | '--shrink_prefix', 34 | type=str, 35 | help='Prefix for the target index name' 36 | ) 37 | @click.option( 38 | '--shrink_suffix', 39 | default='-shrink', 40 | type=str, 41 | help='Suffix for the target index name', 42 | show_default=True 43 | ) 44 | @click.option( 45 | '--copy_aliases', 46 | is_flag=True, 47 | help='Copy each source index aliases to target index' 48 | ) 49 | @click.option( 50 | '--delete_after/--no-delete_after', 51 | default=True, 52 | help='Delete source index after shrink', 53 | show_default=True 54 | ) 55 | @click.option( 56 | '--post_allocation', 57 | help='JSON version of post_allocation (see documentation)', 58 | callback=json_to_dict 59 | ) 60 | @click.option( 61 | '--extra_settings', 62 | help='JSON version of extra_settings (see documentation)', 63 | callback=json_to_dict 64 | ) 65 | @click.option( 66 | '--wait_for_active_shards', 67 | default=1, 68 | type=int, 69 | help='Wait for number of active shards before continuing' 70 | ) 71 | @click.option( 72 | '--wait_for_rebalance/--no-wait_for_rebalance', 73 | default=True, 74 | help='Wait for rebalance to complete' 75 | ) 76 | @click.option( 77 | '--wait_for_completion/--no-wait_for_completion', 78 | default=True, 79 | help='Wait for the shrink to complete' 80 | ) 81 | @click.option( 82 | '--wait_interval', 83 | default=9, 84 | type=int, 85 | help='Seconds to wait between completion checks.' 86 | ) 87 | @click.option( 88 | '--max_wait', 89 | default=-1, 90 | type=int, 91 | help='Maximum number of seconds to wait_for_completion' 92 | ) 93 | @click.option( 94 | '--ignore_empty_list', 95 | is_flag=True, 96 | help='Do not raise exception if there are no actionable indices' 97 | ) 98 | @click.option( 99 | '--allow_ilm_indices/--no-allow_ilm_indices', 100 | help='Allow Curator to operate on Index Lifecycle Management monitored indices.', 101 | default=False, 102 | show_default=True 103 | ) 104 | @click.option( 105 | '--filter_list', 106 | callback=validate_filter_json, 107 | help='JSON array of filters selecting indices to act on.', 108 | required=True 109 | ) 110 | @click.pass_context 111 | def shrink( 112 | ctx, shrink_node, node_filters, number_of_shards, number_of_replicas, shrink_prefix, 113 | shrink_suffix, copy_aliases, delete_after, post_allocation, extra_settings, 114 | wait_for_active_shards, wait_for_rebalance, wait_for_completion, wait_interval, max_wait, 115 | ignore_empty_list, allow_ilm_indices, filter_list 116 | ): 117 | """ 118 | Shrink Indices to --number_of_shards 119 | """ 120 | manual_options = { 121 | 'shrink_node': shrink_node, 122 | 'node_filters': node_filters, 123 | 'number_of_shards': number_of_shards, 124 | 'number_of_replicas': number_of_replicas, 125 | 'shrink_prefix': shrink_prefix, 126 | 'shrink_suffix': shrink_suffix, 127 | 'copy_aliases': copy_aliases, 128 | 'delete_after': delete_after, 129 | 'post_allocation': post_allocation, 130 | 'extra_settings': extra_settings, 131 | 'wait_for_active_shards': wait_for_active_shards, 132 | 'wait_for_rebalance': wait_for_rebalance, 133 | 'wait_for_completion': wait_for_completion, 134 | 'wait_interval': wait_interval, 135 | 'max_wait': max_wait, 136 | 'allow_ilm_indices': allow_ilm_indices, 137 | } 138 | # ctx.info_name is the name of the function or name specified in @click.command decorator 139 | action = cli_action( 140 | ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list) 141 | action.do_singleton_action(dry_run=ctx.obj['dry_run']) 142 | -------------------------------------------------------------------------------- /curator/cli_singletons/snapshot.py: -------------------------------------------------------------------------------- 1 | """Snapshot Singleton""" 2 | import click 3 | from curator.cli_singletons.object_class import cli_action 4 | from curator.cli_singletons.utils import get_width, validate_filter_json 5 | 6 | @click.command(context_settings=get_width()) 7 | @click.option('--repository', type=str, required=True, help='Snapshot repository') 8 | @click.option( 9 | '--name', 10 | type=str, 11 | help='Snapshot name', 12 | show_default=True, 13 | default='curator-%Y%m%d%H%M%S' 14 | ) 15 | @click.option( 16 | '--ignore_unavailable', 17 | is_flag=True, 18 | show_default=True, 19 | help='Ignore unavailable shards/indices.' 20 | ) 21 | @click.option( 22 | '--include_global_state', 23 | is_flag=True, 24 | show_default=True, 25 | help='Store cluster global state with snapshot.' 26 | ) 27 | @click.option( 28 | '--partial', 29 | is_flag=True, 30 | show_default=True, 31 | help='Do not fail if primary shard is unavailable.' 32 | ) 33 | @click.option( 34 | '--wait_for_completion/--no-wait_for_completion', 35 | default=True, 36 | show_default=True, 37 | help='Wait for the snapshot to complete' 38 | ) 39 | @click.option( 40 | '--wait_interval', 41 | default=9, 42 | type=int, 43 | help='Seconds to wait between completion checks.' 44 | ) 45 | @click.option( 46 | '--max_wait', 47 | default=-1, 48 | type=int, 49 | help='Maximum number of seconds to wait_for_completion' 50 | ) 51 | @click.option( 52 | '--skip_repo_fs_check', 53 | is_flag=True, 54 | show_default=True, 55 | help='Skip repository filesystem access validation.' 56 | ) 57 | @click.option( 58 | '--ignore_empty_list', 59 | is_flag=True, 60 | help='Do not raise exception if there are no actionable indices' 61 | ) 62 | @click.option( 63 | '--allow_ilm_indices/--no-allow_ilm_indices', 64 | help='Allow Curator to operate on Index Lifecycle Management monitored indices.', 65 | default=False, 66 | show_default=True 67 | ) 68 | @click.option( 69 | '--filter_list', 70 | callback=validate_filter_json, 71 | help='JSON array of filters selecting indices to act on.', 72 | required=True 73 | ) 74 | @click.pass_context 75 | def snapshot( 76 | ctx, repository, name, ignore_unavailable, include_global_state, partial, 77 | skip_repo_fs_check, wait_for_completion, wait_interval, max_wait, ignore_empty_list, 78 | allow_ilm_indices, filter_list 79 | ): 80 | """ 81 | Snapshot Indices 82 | """ 83 | manual_options = { 84 | 'name': name, 85 | 'repository': repository, 86 | 'ignore_unavailable': ignore_unavailable, 87 | 'include_global_state': include_global_state, 88 | 'partial': partial, 89 | 'skip_repo_fs_check': skip_repo_fs_check, 90 | 'wait_for_completion': wait_for_completion, 91 | 'max_wait': max_wait, 92 | 'wait_interval': wait_interval, 93 | 'allow_ilm_indices': allow_ilm_indices, 94 | } 95 | # ctx.info_name is the name of the function or name specified in @click.command decorator 96 | action = cli_action( 97 | ctx.info_name, ctx.obj['config']['client'], manual_options, filter_list, ignore_empty_list) 98 | action.do_singleton_action(dry_run=ctx.obj['dry_run']) 99 | -------------------------------------------------------------------------------- /curator/cli_singletons/unfreeze.py: -------------------------------------------------------------------------------- 1 | """Unfreeze Index Singleton""" 2 | import click 3 | from curator.cli_singletons.object_class import cli_action 4 | from curator.cli_singletons.utils import get_width, validate_filter_json 5 | 6 | @click.command(context_settings=get_width()) 7 | @click.option( 8 | '--ignore_empty_list', 9 | is_flag=True, 10 | help='Do not raise exception if there are no actionable indices' 11 | ) 12 | @click.option( 13 | '--allow_ilm_indices/--no-allow_ilm_indices', 14 | help='Allow Curator to operate on Index Lifecycle Management monitored indices.', 15 | default=False, 16 | show_default=True 17 | ) 18 | @click.option( 19 | '--filter_list', 20 | callback=validate_filter_json, 21 | help='JSON array of filters selecting indices to act on.', 22 | required=True 23 | ) 24 | @click.pass_context 25 | def unfreeze(ctx, ignore_empty_list, allow_ilm_indices, filter_list): 26 | """ 27 | Unfreeze Indices 28 | """ 29 | # ctx.info_name is the name of the function or name specified in @click.command decorator 30 | action = cli_action( 31 | ctx.info_name, 32 | ctx.obj['config']['client'], 33 | {'allow_ilm_indices':allow_ilm_indices}, 34 | filter_list, 35 | ignore_empty_list 36 | ) 37 | action.do_singleton_action(dry_run=ctx.obj['dry_run']) 38 | -------------------------------------------------------------------------------- /curator/config_utils.py: -------------------------------------------------------------------------------- 1 | """Configuration utilty functions""" 2 | import logging 3 | from copy import deepcopy 4 | from voluptuous import Schema 5 | from curator.validators import SchemaCheck, config_file 6 | from curator.utils import ensure_list, get_yaml, prune_nones, test_client_options 7 | from curator.logtools import LogInfo, Whitelist, Blacklist 8 | 9 | def test_config(config): 10 | """Test YAML against the schema""" 11 | # Get config from yaml file 12 | yaml_config = get_yaml(config) 13 | # if the file is empty, which is still valid yaml, set as an empty dict 14 | yaml_config = {} if not yaml_config else prune_nones(yaml_config) 15 | # Voluptuous can't verify the schema of a dict if it doesn't have keys, 16 | # so make sure the keys are at least there and are dict() 17 | for k in ['client', 'logging']: 18 | if k not in yaml_config: 19 | yaml_config[k] = {} 20 | else: 21 | yaml_config[k] = prune_nones(yaml_config[k]) 22 | return SchemaCheck( 23 | yaml_config, config_file.client(), 'Client Configuration', 'full configuration dictionary' 24 | ).result() 25 | 26 | def set_logging(log_opts): 27 | """Configure global logging options""" 28 | # Set up logging 29 | loginfo = LogInfo(log_opts) 30 | logging.root.addHandler(loginfo.handler) 31 | logging.root.setLevel(loginfo.numeric_log_level) 32 | _ = logging.getLogger('curator.cli') 33 | # Set up NullHandler() to handle nested opensearchpy.trace Logger 34 | # instance in opensearch-py python client 35 | logging.getLogger('opensearchpy.trace').addHandler(logging.NullHandler()) 36 | if log_opts['blacklist']: 37 | for bl_entry in ensure_list(log_opts['blacklist']): 38 | for handler in logging.root.handlers: 39 | handler.addFilter(Blacklist(bl_entry)) 40 | 41 | def process_config(yaml_file): 42 | """Process yaml_file and return a valid client configuration""" 43 | config = test_config(yaml_file) 44 | set_logging(config['logging']) 45 | test_client_options(config['client']) 46 | return config['client'] 47 | 48 | def password_filter(data): 49 | """ 50 | Return a deepcopy of the dictionary with any password fields hidden 51 | """ 52 | def iterdict(mydict): 53 | for key, value in mydict.items(): 54 | if isinstance(value, dict): 55 | iterdict(value) 56 | elif key == "http_auth": 57 | mydict.update({"http_auth": (value[0],"REDACTED")}) 58 | elif key == "password": 59 | mydict.update({"password": "REDACTED"}) 60 | return mydict 61 | return iterdict(deepcopy(data)) 62 | -------------------------------------------------------------------------------- /curator/curator_cli.py: -------------------------------------------------------------------------------- 1 | """CLI Wrapper used by run_curator.py""" 2 | import click 3 | from curator.singletons import cli 4 | 5 | def main(): 6 | """Main function called by run_curator.py""" 7 | # This is because click uses decorators, and pylint doesn't catch that 8 | # pylint: disable=E1120 9 | # pylint: disable=E1123 10 | cli(obj={}) 11 | -------------------------------------------------------------------------------- /curator/defaults/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flant/curator-opensearch/7e50540e861fe04e2d90a5eaf3437d70f62a34bc/curator/defaults/__init__.py -------------------------------------------------------------------------------- /curator/defaults/client_defaults.py: -------------------------------------------------------------------------------- 1 | """Define valid schemas for client configuration validation""" 2 | from six import string_types 3 | from voluptuous import All, Any, Boolean, Coerce, Optional, Range 4 | 5 | # Configuration file: client 6 | # pylint: disable=no-value-for-parameter 7 | def config_client(): 8 | """Client schema""" 9 | return { 10 | Optional('hosts', default='127.0.0.1'): Any(None, list, *string_types), 11 | Optional('port', default=9200): Any(None, All(Coerce(int), Range(min=1, max=65535))), 12 | Optional('url_prefix', default=''): Any(None, *string_types), 13 | Optional('use_ssl', default=False): Boolean(), 14 | Optional('certificate', default=None): Any(None, *string_types), 15 | Optional('client_cert', default=None): Any(None, *string_types), 16 | Optional('client_key', default=None): Any(None, *string_types), 17 | Optional('aws_key', default=None): Any(None, *string_types), 18 | Optional('aws_secret_key', default=None): Any(None, *string_types), 19 | Optional('aws_token', default=None): Any(None, *string_types), 20 | Optional('aws_sign_request', default=False): Boolean(), 21 | Optional('aws_region'): Any(None, *string_types), 22 | Optional('ssl_no_validate', default=False): Boolean(), 23 | Optional('ssl_show_warn', default=False): Boolean(), 24 | Optional('username', default=None): Any(None, *string_types), 25 | Optional('password', default=None): Any(None, *string_types), 26 | Optional('http_auth', default=None): Any(None, *string_types), 27 | Optional('timeout', default=30): All(Coerce(int), Range(min=1, max=86400)), 28 | Optional('master_only', default=False): Boolean(), 29 | Optional('api_key', default=None): Any(None, *string_types), 30 | Optional('apikey_auth', default=None): Any(None, *string_types), 31 | } 32 | 33 | # Configuration file: logging 34 | def config_logging(): 35 | """Logging schema""" 36 | return { 37 | Optional('loglevel', default='INFO'): 38 | Any(None, 'NOTSET', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL', 39 | All(Coerce(int), Any(0, 10, 20, 30, 40, 50)) 40 | ), 41 | Optional('logfile', default=None): Any(None, *string_types), 42 | Optional('logformat', default='default'): 43 | Any(None, All(Any(*string_types), Any('default', 'json', 'logstash', 'ecs'))), 44 | Optional('blacklist', default=['elasticsearch', 'urllib3']): Any(None, list), 45 | } 46 | -------------------------------------------------------------------------------- /curator/exceptions.py: -------------------------------------------------------------------------------- 1 | """Curator Exceptions""" 2 | class CuratorException(Exception): 3 | """ 4 | Base class for all exceptions raised by Curator which are not Elasticsearch 5 | exceptions. 6 | """ 7 | 8 | class ConfigurationError(CuratorException): 9 | """ 10 | Exception raised when a misconfiguration is detected 11 | """ 12 | 13 | class MissingArgument(CuratorException): 14 | """ 15 | Exception raised when a needed argument is not passed. 16 | """ 17 | 18 | class NoIndices(CuratorException): 19 | """ 20 | Exception raised when an operation is attempted against an empty index_list 21 | """ 22 | 23 | class NoSnapshots(CuratorException): 24 | """ 25 | Exception raised when an operation is attempted against an empty snapshot_list 26 | """ 27 | 28 | class ActionError(CuratorException): 29 | """ 30 | Exception raised when an action (against an index_list or snapshot_list) cannot be taken. 31 | """ 32 | 33 | class FailedExecution(CuratorException): 34 | """ 35 | Exception raised when an action fails to execute for some reason. 36 | """ 37 | 38 | class SnapshotInProgress(ActionError): 39 | """ 40 | Exception raised when a snapshot is already in progress 41 | """ 42 | 43 | class ActionTimeout(CuratorException): 44 | """ 45 | Exception raised when an action fails to complete in the allotted time 46 | """ 47 | 48 | class FailedSnapshot(CuratorException): 49 | """ 50 | Exception raised when a snapshot does not complete with state SUCCESS 51 | """ 52 | 53 | class FailedRestore(CuratorException): 54 | """ 55 | Exception raised when a Snapshot Restore does not restore all selected indices 56 | """ 57 | 58 | class FailedReindex(CuratorException): 59 | """ 60 | Exception raised when failures are found in the reindex task response 61 | """ 62 | 63 | class ClientException(CuratorException): 64 | """ 65 | Exception raised when the Elasticsearch client and/or connection is the source of the problem. 66 | """ 67 | 68 | class LoggingException(CuratorException): 69 | """ 70 | Exception raised when Curator cannot either log or configure logging 71 | """ 72 | -------------------------------------------------------------------------------- /curator/logtools.py: -------------------------------------------------------------------------------- 1 | """Logging tools""" 2 | import sys 3 | import json 4 | import logging 5 | import time 6 | from curator.exceptions import LoggingException 7 | 8 | def de_dot(dot_string, msg): 9 | """Turn message and dotted string into a nested dictionary""" 10 | arr = dot_string.split('.') 11 | arr.append(msg) 12 | retval = None 13 | for idx in range(len(arr), 1, -1): 14 | if not retval: 15 | try: 16 | retval = {arr[idx-2]: arr[idx-1]} 17 | except Exception as err: 18 | raise LoggingException(err) 19 | else: 20 | try: 21 | new_d = {arr[idx-2]: retval} 22 | retval = new_d 23 | except Exception as err: 24 | raise LoggingException(err) 25 | return retval 26 | 27 | def deepmerge(source, destination): 28 | """Merge deeply nested dictionary structures""" 29 | for key, value in source.items(): 30 | if isinstance(value, dict): 31 | node = destination.setdefault(key, {}) 32 | deepmerge(value, node) 33 | else: 34 | destination[key] = value 35 | return destination 36 | class LogstashFormatter(logging.Formatter): 37 | """Logstash formatting (JSON)""" 38 | # The LogRecord attributes we want to carry over to the Logstash message, 39 | # mapped to the corresponding output key. 40 | WANTED_ATTRS = { 41 | 'levelname': 'loglevel', 42 | 'funcName': 'function', 43 | 'lineno': 'linenum', 44 | 'message': 'message', 45 | 'name': 'name' 46 | } 47 | 48 | def format(self, record): 49 | self.converter = time.gmtime 50 | timestamp = '%s.%03dZ' % ( 51 | self.formatTime(record, datefmt='%Y-%m-%dT%H:%M:%S'), record.msecs) 52 | result = {'@timestamp': timestamp} 53 | available = record.__dict__ 54 | # This is cleverness because 'message' is NOT a member key of ``record.__dict__`` 55 | # the ``getMessage()`` method is effectively ``msg % args`` (actual keys) 56 | # By manually adding 'message' to ``available``, it simplifies the code 57 | available['message'] = record.getMessage() 58 | for attribute in set(self.WANTED_ATTRS).intersection(available): 59 | result = deepmerge( 60 | de_dot(self.WANTED_ATTRS[attribute], getattr(record, attribute)), result 61 | ) 62 | # The following is mostly for the ecs format. You can't have 2x 'message' keys in 63 | # WANTED_ATTRS, so we set the value to 'log.original' in ecs, and this code block 64 | # guarantees it still appears as 'message' too. 65 | if 'message' not in result.items(): 66 | result['message'] = available['message'] 67 | return json.dumps(result, sort_keys=True) 68 | 69 | class ECSFormatter(LogstashFormatter): 70 | """Elastic Common Schema formatting (ECS)""" 71 | # Overload LogstashFormatter attribute 72 | WANTED_ATTRS = { 73 | 'levelname': 'log.level', 74 | 'funcName': 'log.origin.function', 75 | 'lineno': 'log.origin.file.line', 76 | 'message': 'log.original', 77 | 'name': 'log.logger' 78 | } 79 | 80 | class Whitelist(logging.Filter): 81 | """How to whitelist logs""" 82 | def __init__(self, *whitelist): 83 | self.whitelist = [logging.Filter(name) for name in whitelist] 84 | 85 | def filter(self, record): 86 | return any(f.filter(record) for f in self.whitelist) 87 | 88 | class Blacklist(Whitelist): 89 | """Blacklist monkey-patch of Whitelist""" 90 | def filter(self, record): 91 | return not Whitelist.filter(self, record) 92 | 93 | class LogInfo(object): 94 | """Logging Class""" 95 | def __init__(self, cfg): 96 | cfg['loglevel'] = 'INFO' if not 'loglevel' in cfg else cfg['loglevel'] 97 | cfg['logfile'] = None if not 'logfile' in cfg else cfg['logfile'] 98 | cfg['logformat'] = 'default' if not 'logformat' in cfg else cfg['logformat'] 99 | self.numeric_log_level = getattr(logging, cfg['loglevel'].upper(), None) 100 | self.format_string = '%(asctime)s %(levelname)-9s %(message)s' 101 | if not isinstance(self.numeric_log_level, int): 102 | raise ValueError('Invalid log level: {0}'.format(cfg['loglevel'])) 103 | 104 | self.handler = logging.StreamHandler( 105 | open(cfg['logfile'], 'a') if cfg['logfile'] else sys.stdout 106 | ) 107 | 108 | if self.numeric_log_level == 10: # DEBUG 109 | self.format_string = ( 110 | '%(asctime)s %(levelname)-9s %(name)22s ' 111 | '%(funcName)22s:%(lineno)-4d %(message)s' 112 | ) 113 | 114 | if cfg['logformat'] == 'json' or cfg['logformat'] == 'logstash': 115 | self.handler.setFormatter(LogstashFormatter()) 116 | elif cfg['logformat'] == 'ecs': 117 | self.handler.setFormatter(ECSFormatter()) 118 | else: 119 | self.handler.setFormatter(logging.Formatter(self.format_string)) 120 | -------------------------------------------------------------------------------- /curator/singletons.py: -------------------------------------------------------------------------------- 1 | """CLI module for curator_cli""" 2 | import os 3 | import logging 4 | import click 5 | from curator.defaults import settings 6 | from curator.config_utils import test_config, set_logging 7 | from curator.utils import test_client_options 8 | from curator.cli_singletons.utils import config_override, false_to_none, get_width 9 | from curator.cli_singletons.alias import alias 10 | from curator.cli_singletons.allocation import allocation 11 | from curator.cli_singletons.close import close 12 | from curator.cli_singletons.delete import delete_indices, delete_snapshots 13 | from curator.cli_singletons.forcemerge import forcemerge 14 | from curator.cli_singletons.open_indices import open_indices 15 | from curator.cli_singletons.replicas import replicas 16 | from curator.cli_singletons.restore import restore 17 | from curator.cli_singletons.rollover import rollover 18 | from curator.cli_singletons.snapshot import snapshot 19 | from curator.cli_singletons.shrink import shrink 20 | from curator.cli_singletons.show import show_indices, show_snapshots 21 | from curator.cli_singletons.freeze import freeze 22 | from curator.cli_singletons.unfreeze import unfreeze 23 | from curator._version import __version__ 24 | 25 | @click.group(context_settings=get_width()) 26 | @click.option( 27 | '--config', help='Path to configuration file. Default: ~/.curator/curator.yml', 28 | type=click.Path(), default=settings.config_file() 29 | ) 30 | @click.option('--host', help='Elasticsearch host.') 31 | @click.option('--url_prefix', help='Elasticsearch http url prefix.') 32 | @click.option('--port', help='Elasticsearch port.') 33 | @click.option( 34 | '--use_ssl', is_flag=True, 35 | callback=false_to_none, help='Connect to Elasticsearch through SSL.' 36 | ) 37 | @click.option('--certificate', help='Path to certificate to use for SSL validation.') 38 | @click.option( 39 | '--client-cert', help='Path to file containing SSL certificate for client auth.', type=str) 40 | @click.option('--client-key', help='Path to file containing SSL key for client auth.', type=str) 41 | @click.option( 42 | '--ssl-no-validate', is_flag=True, 43 | callback=false_to_none, help='Do not validate SSL certificate' 44 | ) 45 | @click.option( 46 | '--ssl-show-warn', is_flag=True, 47 | callback=false_to_none, help='Show SSL warnings' 48 | ) 49 | @click.option( 50 | '--http_auth', 51 | help='Use Basic Authentication ex: user:pass -- DEPRECATED. Use username and password.' 52 | ) 53 | @click.option('--username', help='HTTP Basic Authentication username', type=str) 54 | @click.option('--password', help='HTTP Basic Authentication password', type=str) 55 | @click.option( 56 | '--apikey_auth', 57 | help='API Key Authentication encoded in base64 containing `id:api_key`', 58 | type=str 59 | ) 60 | @click.option('--timeout', help='Connection timeout in seconds.', type=int) 61 | @click.option( 62 | '--master-only', is_flag=True, callback=false_to_none, 63 | help='Only operate on elected master node.' 64 | ) 65 | @click.option('--dry-run', is_flag=True, help='Do not perform any changes.') 66 | @click.option('--loglevel', help='Log level') 67 | @click.option('--logfile', help='log file') 68 | @click.option('--logformat', help='Log output format [default|logstash|json|ecs].') 69 | @click.version_option(version=__version__) 70 | @click.pass_context 71 | def cli( 72 | ctx, config, host, url_prefix, port, use_ssl, certificate, client_cert, client_key, 73 | ssl_no_validate, ssl_show_warn, http_auth, username, password, apikey_auth, timeout, master_only, dry_run, loglevel, 74 | logfile, logformat 75 | ): 76 | """CLI input""" 77 | if os.path.isfile(config): 78 | initial_config = test_config(config) 79 | else: 80 | initial_config = None 81 | configuration = config_override(ctx, initial_config) 82 | set_logging(configuration['logging']) 83 | test_client_options(configuration['client']) 84 | ctx.obj['config'] = configuration 85 | ctx.obj['dry_run'] = dry_run 86 | # Add the subcommands 87 | cli.add_command(alias) 88 | cli.add_command(allocation) 89 | cli.add_command(close) 90 | cli.add_command(delete_indices) 91 | cli.add_command(delete_snapshots) 92 | cli.add_command(forcemerge) 93 | cli.add_command(freeze) 94 | cli.add_command(open_indices) 95 | cli.add_command(replicas) 96 | cli.add_command(snapshot) 97 | cli.add_command(restore) 98 | cli.add_command(rollover) 99 | cli.add_command(shrink) 100 | cli.add_command(show_indices) 101 | cli.add_command(show_snapshots) 102 | cli.add_command(unfreeze) 103 | -------------------------------------------------------------------------------- /curator/validators/__init__.py: -------------------------------------------------------------------------------- 1 | from curator.validators.schemacheck import SchemaCheck 2 | -------------------------------------------------------------------------------- /curator/validators/actions.py: -------------------------------------------------------------------------------- 1 | from voluptuous import All, Any, In, Schema, Optional, Required 2 | from curator.defaults import settings 3 | from curator.validators import SchemaCheck 4 | from six import string_types 5 | 6 | ### Schema information ### 7 | # Actions: root level 8 | def root(): 9 | return Schema({ Required('actions'): dict }) 10 | 11 | def valid_action(): 12 | return { 13 | Required('action'): Any( 14 | In(settings.all_actions()), 15 | msg='action must be one of {0}'.format( 16 | settings.all_actions() 17 | ) 18 | ) 19 | } 20 | 21 | # Basic action structure 22 | def structure(data, location): 23 | # Validate the action type first, so we can use it for other tests 24 | _ = SchemaCheck( 25 | data, 26 | Schema(valid_action(), extra=True), 27 | 'action type', 28 | location, 29 | ).result() 30 | # Build a valid schema knowing that the action has already been validated 31 | retval = valid_action() 32 | retval.update( 33 | { 34 | Optional('description', default='No description given'): Any( 35 | str, *string_types 36 | ) 37 | } 38 | ) 39 | retval.update( 40 | { Optional('options', default=settings.default_options()): dict } ) 41 | action = data['action'] 42 | if action in [ 'cluster_routing', 'create_index', 'rollover']: 43 | # The cluster_routing, create_index, and rollover actions should not 44 | # have a 'filters' block 45 | pass 46 | elif action == 'alias': 47 | # The alias action should not have a filters block, but should have 48 | # an add and/or remove block. 49 | retval.update( 50 | { 51 | Optional('add'): dict, 52 | Optional('remove'): dict, 53 | } 54 | ) 55 | else: 56 | retval.update( 57 | { Optional('filters', default=settings.default_filters()): list } 58 | ) 59 | return Schema(retval) 60 | -------------------------------------------------------------------------------- /curator/validators/config_file.py: -------------------------------------------------------------------------------- 1 | from voluptuous import Optional, Schema 2 | from curator.defaults import client_defaults 3 | 4 | def client(): 5 | return Schema( 6 | { 7 | Optional('client'): client_defaults.config_client(), 8 | Optional('logging'): client_defaults.config_logging(), 9 | } 10 | ) 11 | -------------------------------------------------------------------------------- /curator/validators/filters.py: -------------------------------------------------------------------------------- 1 | from voluptuous import Any, In, Required, Schema 2 | from curator.defaults import settings, filtertypes 3 | from curator.exceptions import ConfigurationError 4 | from curator.validators import SchemaCheck 5 | import logging 6 | logger = logging.getLogger(__name__) 7 | 8 | def filtertype(): 9 | return { 10 | Required('filtertype'): Any( 11 | In(settings.all_filtertypes()), 12 | msg='filtertype must be one of {0}'.format( 13 | settings.all_filtertypes() 14 | ) 15 | ) 16 | } 17 | 18 | def structure(): 19 | # This is to first ensure that only the possible keys/filter elements are 20 | # there, and get a dictionary back to work with. 21 | retval = settings.structural_filter_elements() 22 | retval.update(filtertype()) 23 | return Schema(retval) 24 | 25 | def single(action, data): 26 | try: 27 | ft = data['filtertype'] 28 | except KeyError: 29 | raise ConfigurationError('Missing key "filtertype"') 30 | f = filtertype() 31 | for each in getattr(filtertypes, ft)(action, data): 32 | f.update(each) 33 | return Schema(f) 34 | 35 | def Filters(action, location=None): 36 | def f(v): 37 | def prune_nones(mydict): 38 | return dict([(k,v) for k, v in mydict.items() if v != None and v != 'None']) 39 | # This validator method simply validates all filters in the list. 40 | for idx in range(0, len(v)): 41 | pruned = prune_nones(v[idx]) 42 | filter_dict = SchemaCheck( 43 | pruned, 44 | single(action, pruned), 45 | 'filter', 46 | '{0}, filter #{1}: {2}'.format(location, idx, pruned) 47 | ).result() 48 | logger.debug('Filter #{0}: {1}'.format(idx, filter_dict)) 49 | v[idx] = filter_dict 50 | # If we've made it here without raising an Exception, it's valid 51 | return v 52 | return f 53 | -------------------------------------------------------------------------------- /curator/validators/schemacheck.py: -------------------------------------------------------------------------------- 1 | from voluptuous import Schema 2 | from curator.exceptions import ConfigurationError 3 | import re 4 | import logging 5 | 6 | class SchemaCheck(object): 7 | def __init__(self, config, schema, test_what, location): 8 | """ 9 | Validate ``config`` with the provided voluptuous ``schema``. 10 | ``test_what`` and ``location`` are for reporting the results, in case of 11 | failure. If validation is successful, the method returns ``config`` as 12 | valid. 13 | 14 | :arg config: A configuration dictionary. 15 | :type config: dict 16 | :arg schema: A voluptuous schema definition 17 | :type schema: :class:`voluptuous.Schema` 18 | :arg test_what: which configuration block is being validated 19 | :type test_what: str 20 | :arg location: An string to report which configuration sub-block is 21 | being tested. 22 | :type location: str 23 | """ 24 | self.loggit = logging.getLogger('curator.validators.SchemaCheck') 25 | # Set the Schema for validation... 26 | self.loggit.debug('Schema: {0}'.format(schema)) 27 | self.loggit.debug('"{0}" config: {1}'.format(test_what, config)) 28 | self.config = config 29 | self.schema = schema 30 | self.test_what = test_what 31 | self.location = location 32 | 33 | def __parse_error(self): 34 | """ 35 | Report the error, and try to report the bad key or value as well. 36 | """ 37 | def get_badvalue(data_string, data): 38 | elements = re.sub(r'[\'\]]', '', data_string).split('[') 39 | elements.pop(0) # Get rid of data as the first element 40 | value = None 41 | for k in elements: 42 | try: 43 | key = int(k) 44 | except ValueError: 45 | key = k 46 | if value == None: 47 | value = data[key] 48 | # if this fails, it's caught below 49 | return value 50 | try: 51 | self.badvalue = get_badvalue(str(self.error).split()[-1], self.config) 52 | except: 53 | self.badvalue = '(could not determine)' 54 | 55 | def result(self): 56 | try: 57 | return self.schema(self.config) 58 | except Exception as e: 59 | try: 60 | # pylint: disable=E1101 61 | self.error = e.errors[0] 62 | except: 63 | self.error = '{0}'.format(e) 64 | self.__parse_error() 65 | self.loggit.error('Schema error: {0}'.format(self.error)) 66 | raise ConfigurationError( 67 | 'Configuration: {0}: Location: {1}: Bad Value: "{2}", {3}. ' 68 | 'Check configuration file.'.format( 69 | self.test_what, self.location, self.badvalue, self.error) 70 | ) 71 | -------------------------------------------------------------------------------- /docs/actionclasses.rst: -------------------------------------------------------------------------------- 1 | .. _actionclasses: 2 | 3 | Action Classes 4 | ============== 5 | 6 | .. seealso:: It is important to note that each action has a `do_action()` 7 | method, which accepts no arguments. This is the means by which all 8 | actions are executed. 9 | 10 | * `Alias`_ 11 | * `Allocation`_ 12 | * `Close`_ 13 | * `ClusterRouting`_ 14 | * `CreateIndex`_ 15 | * `DeleteIndices`_ 16 | * `DeleteSnapshots`_ 17 | * `ForceMerge`_ 18 | * `IndexSettings`_ 19 | * `Open`_ 20 | * `Reindex`_ 21 | * `Replicas`_ 22 | * `Restore`_ 23 | * `Rollover`_ 24 | * `Shrink`_ 25 | * `Snapshot`_ 26 | 27 | 28 | Alias 29 | ----- 30 | .. autoclass:: curator.actions.Alias 31 | :members: 32 | 33 | Allocation 34 | ---------- 35 | .. autoclass:: curator.actions.Allocation 36 | :members: 37 | 38 | Close 39 | ----- 40 | .. autoclass:: curator.actions.Close 41 | :members: 42 | 43 | ClusterRouting 44 | -------------- 45 | .. autoclass:: curator.actions.ClusterRouting 46 | :members: 47 | 48 | CreateIndex 49 | -------------- 50 | .. autoclass:: curator.actions.CreateIndex 51 | :members: 52 | 53 | DeleteIndices 54 | ------------- 55 | .. autoclass:: curator.actions.DeleteIndices 56 | :members: 57 | 58 | DeleteSnapshots 59 | --------------- 60 | .. autoclass:: curator.actions.DeleteSnapshots 61 | :members: 62 | 63 | ForceMerge 64 | ---------- 65 | .. autoclass:: curator.actions.ForceMerge 66 | :members: 67 | 68 | IndexSettings 69 | -------------- 70 | .. autoclass:: curator.actions.IndexSettings 71 | :members: 72 | 73 | Open 74 | ---- 75 | .. autoclass:: curator.actions.Open 76 | :members: 77 | 78 | Reindex 79 | -------- 80 | .. autoclass:: curator.actions.Reindex 81 | :members: 82 | 83 | Replicas 84 | -------- 85 | .. autoclass:: curator.actions.Replicas 86 | :members: 87 | 88 | Restore 89 | -------- 90 | .. autoclass:: curator.actions.Restore 91 | :members: 92 | 93 | Rollover 94 | -------- 95 | .. autoclass:: curator.actions.Rollover 96 | :members: 97 | 98 | Shrink 99 | -------- 100 | .. autoclass:: curator.actions.Shrink 101 | :members: 102 | 103 | Snapshot 104 | -------- 105 | .. autoclass:: curator.actions.Snapshot 106 | :members: 107 | -------------------------------------------------------------------------------- /docs/asciidoc/ilm.asciidoc: -------------------------------------------------------------------------------- 1 | [[ilm]] 2 | = Curator and Index Lifecycle Management 3 | 4 | [partintro] 5 | -- 6 | 7 | Beginning with Elasticsearch version 6.6, Elasticsearch has provided 8 | {ref}/index-lifecycle-management.html[Index Lifecycle Management] (or, ILM) to 9 | users with at least a Basic license. ILM provides users with many of the most 10 | common index management features as a matter of policy, rather than execution 11 | time analysis (which is how Curator works). 12 | 13 | -- 14 | 15 | [[ilm-actions]] 16 | == ILM Actions 17 | 18 | ILM applies policy actions as indices enter time-oriented phases: 19 | 20 | * Hot 21 | * Warm 22 | * Cold 23 | * Delete 24 | 25 | The policy actions include: 26 | 27 | * {ref}/ilm-set-priority.html[Set Priority] 28 | * {ref}/ilm-rollover.html[Rollover] 29 | * {ref}/ilm-unfollow.html[Unfollow] 30 | * {ref}/ilm-allocate.html[Allocate] 31 | * {ref}/ilm-readonly.html[Read-Only] 32 | * {ref}/ilm-forcemerge.html[Force Merge] 33 | * {ref}/ilm-shrink.html[Shrink] 34 | * {ref}/ilm-freeze.html[Freeze] 35 | * {ref}/ilm-delete.html[Delete] 36 | 37 | [[ilm-or-curator]] 38 | == ILM or Curator? 39 | 40 | If ILM provides the functionality to manage your index lifecycle, and you have 41 | at least a Basic license, consider using ILM in place of Curator. Many of the 42 | Stack components make use of ILM by default. 43 | 44 | [[ilm-beats]] 45 | === Beats 46 | 47 | NOTE: All Beats share a similar ILM configuration. Filebeats is used as a 48 | reference here. 49 | 50 | Starting with version 7.0, Filebeat uses index lifecycle management by default when it connects to a cluster that supports lifecycle management. Filebeat loads the default policy automatically and applies it to any indices created by Filebeat. 51 | 52 | You can view and edit the policy in the Index lifecycle policies UI in Kibana. For more information about working with the UI, see 53 | {kbref}/index-lifecycle-policies.html[Index lifecyle policies]. 54 | 55 | Read more about Filebeat and ILM {fbref}/ilm.html[here]. 56 | 57 | [[ilm-logstash]] 58 | === Logstash 59 | 60 | NOTE: The Index Lifecycle Management feature requires version 9.3.1 or higher of the 61 | `logstash-output-elasticsearch` plugin. 62 | 63 | Logstash can use {esref}/index-lifecycle-management.html[Index Lifecycle Management] 64 | to automate the management of indices over time. 65 | 66 | The use of Index Lifecycle Management is controlled by the `ilm_enabled` setting. By 67 | default, this will automatically detect whether the Elasticsearch instance 68 | supports ILM, and will use it if it is available. `ilm_enabled` can also be set to 69 | `true` or `false` to override the automatic detection, or disable ILM. 70 | 71 | Read more about Logstash and ILM 72 | {lsref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ilm[here]. 73 | 74 | 75 | [[ilm-and-curator]] 76 | == ILM and Curator! 77 | 78 | WARNING: Curator will not act on any index associated with an ILM policy without 79 | setting `allow_ilm_indices` to `true`. 80 | 81 | Curator and ILM _can_ coexist. However, to prevent Curator from accidentally 82 | interfering, or colliding with ILM policies, any index associated with an ILM 83 | policy name is excluded by default. This is true whether you have a Basic 84 | license or not, or whether the ILM policy is enabled or not. 85 | 86 | Curator can be configured to work with ILM-enabled indices by setting the 87 | <> option to `true` for any action. 88 | 89 | Learn more about Index Lifecycle Management 90 | {ref}/index-lifecycle-management.html[here]. 91 | -------------------------------------------------------------------------------- /docs/asciidoc/inc_datemath.asciidoc: -------------------------------------------------------------------------------- 1 | This setting may be a valid {ref}/date-math-index-names.html[Elasticsearch date math string]. 2 | 3 | A date math name takes the following form: 4 | 5 | [source,sh] 6 | ------------- 7 | 8 | ------------- 9 | 10 | [width="50%", cols="| logstash-2024.03.22 24 | || logstash-2024.03.01 25 | || logstash-2024.03 26 | || logstash-2024.02 27 | | | logstash-2024.03.23 28 | |=== -------------------------------------------------------------------------------- /docs/asciidoc/inc_filepath.asciidoc: -------------------------------------------------------------------------------- 1 | [TIP] 2 | .File paths 3 | ===================================================================== 4 | File paths can be specified as follows: 5 | 6 | *For Windows:* 7 | 8 | [source,sh] 9 | ------------- 10 | 'C:\path\to\file' 11 | ------------- 12 | 13 | *For Linux, BSD, Mac OS:* 14 | 15 | [source,sh] 16 | ------------- 17 | '/path/to/file' 18 | ------------- 19 | 20 | Using single-quotes around your file path is encouraged, especially with 21 | Windows file paths. 22 | ===================================================================== 23 | -------------------------------------------------------------------------------- /docs/asciidoc/inc_filter_by_aliases.asciidoc: -------------------------------------------------------------------------------- 1 | [IMPORTANT] 2 | .API Change in Elasticsearch 5.5.0 3 | ============================ 4 | https://www.elastic.co/guide/en/elasticsearch/reference/5.5/breaking-changes-5.5.html#breaking_55_rest_changes[An update to Elasticsearch 5.5.0 changes the behavior of this filter, differing from previous 5.x versions]. 5 | 6 | If a list of <> is provided (instead of only one), indices 7 | must appear in _all_ listed <> or a 404 error will result, 8 | leading to no indices being matched. In older versions, if the index was 9 | associated with even one of the aliases in <>, it would 10 | result in a match. 11 | ============================ 12 | -------------------------------------------------------------------------------- /docs/asciidoc/inc_filter_chaining.asciidoc: -------------------------------------------------------------------------------- 1 | [NOTE] 2 | .Filter chaining 3 | ===================================================================== 4 | It is important to note that while filters can be chained, each is linked by an 5 | implied logical *AND* operation. If you want to match from one of several 6 | different patterns, as with a logical *OR* operation, you can do so with the 7 | <> filtertype using _regex_ as the <>. 8 | 9 | This example shows how to select multiple indices based on them beginning with 10 | either `alpha-`, `bravo-`, or `charlie-`: 11 | 12 | [source,yaml] 13 | ------------- 14 | filters: 15 | - filtertype: pattern 16 | kind: regex 17 | value: '^(alpha-|bravo-|charlie-).*$' 18 | ------------- 19 | 20 | Explaining all of the different ways in which regular expressions can be used 21 | is outside the scope of this document, but hopefully this gives you some idea 22 | of how a regular expression pattern can be used when a logical *OR* is desired. 23 | ===================================================================== 24 | -------------------------------------------------------------------------------- /docs/asciidoc/inc_kinds.asciidoc: -------------------------------------------------------------------------------- 1 | The different <> are described as follows: 2 | 3 | === prefix 4 | 5 | To match all indices starting with `logstash-`: 6 | 7 | [source,yaml] 8 | ------------- 9 | - filtertype: pattern 10 | kind: prefix 11 | value: logstash- 12 | ------------- 13 | 14 | To match all indices _except_ those starting with `logstash-`: 15 | 16 | [source,yaml] 17 | ------------- 18 | - filtertype: pattern 19 | kind: prefix 20 | value: logstash- 21 | exclude: True 22 | ------------- 23 | 24 | === suffix 25 | 26 | To match all indices ending with `-prod`: 27 | 28 | [source,yaml] 29 | ------------- 30 | - filtertype: pattern 31 | kind: suffix 32 | value: -prod 33 | ------------- 34 | 35 | To match all indices _except_ those ending with `-prod`: 36 | 37 | [source,yaml] 38 | ------------- 39 | - filtertype: pattern 40 | kind: suffix 41 | value: -prod 42 | exclude: True 43 | ------------- 44 | 45 | === timestring 46 | 47 | IMPORTANT: No age calculation takes place here. It is strictly a pattern match. 48 | 49 | To match all indices with a Year.month.day pattern, like `index-2017.04.01`: 50 | 51 | [source,yaml] 52 | ------------- 53 | - filtertype: pattern 54 | kind: timestring 55 | value: '%Y.%m.%d' 56 | ------------- 57 | 58 | To match all indices _except_ those with a Year.month.day pattern, like 59 | `index-2017.04.01`: 60 | 61 | [source,yaml] 62 | ------------- 63 | - filtertype: pattern 64 | kind: timestring 65 | value: '%Y.%m.%d' 66 | exclude: True 67 | ------------- 68 | 69 | include::inc_timestring_regex.asciidoc[] 70 | 71 | === regex 72 | 73 | This <> allows you to design a regular-expression to match 74 | indices or snapshots: 75 | 76 | To match all indices starting with `a-`, `b-`, or `c-`: 77 | 78 | [source,yaml] 79 | ------------- 80 | - filtertype: pattern 81 | kind: regex 82 | value: '^a-|^b-|^c-' 83 | ------------- 84 | 85 | To match all indices _except_ those starting with `a-`, `b-`, or `c-`: 86 | 87 | [source,yaml] 88 | ------------- 89 | - filtertype: pattern 90 | kind: regex 91 | value: '^a-|^b-|^c-' 92 | exclude: True 93 | ------------- 94 | -------------------------------------------------------------------------------- /docs/asciidoc/inc_sources.asciidoc: -------------------------------------------------------------------------------- 1 | === `name`-based ages 2 | 3 | Using `name` as the `source` tells Curator to look for a 4 | <> within the index or snapshot name, and convert 5 | that into an epoch timestamp (epoch implies UTC). 6 | 7 | [source,yaml] 8 | ------------- 9 | - filtertype: age 10 | source: name 11 | direction: older 12 | timestring: '%Y.%m.%d' 13 | unit: days 14 | unit_count: 3 15 | ------------- 16 | 17 | include::inc_timestring_regex.asciidoc[] 18 | 19 | === `creation_date`-based ages 20 | 21 | `creation_date` extracts the epoch time of index or snapshot creation. 22 | 23 | [source,yaml] 24 | ------------- 25 | - filtertype: age 26 | source: creation_date 27 | direction: older 28 | unit: days 29 | unit_count: 3 30 | ------------- 31 | 32 | === `field_stats`-based ages 33 | 34 | NOTE: `source` can only be `field_stats` when filtering indices. 35 | 36 | In Curator 5.3 and older, source `field_stats` uses the 37 | http://www.elastic.co/guide/en/elasticsearch/reference/5.6/search-field-stats.html[Field Stats API] 38 | to calculate either the `min_value` or the `max_value` of the <> 39 | as the <>, and then use that value for age 40 | comparisons. In 5.4 and above, even though it is still called `field_stats`, it 41 | uses an aggregation to calculate the same values, as the `field_stats` API is 42 | no longer used in Elasticsearch 6.x and up. 43 | 44 | <> must be of type `date` in Elasticsearch. 45 | 46 | [source,yaml] 47 | ------------- 48 | - filtertype: age 49 | source: field_stats 50 | direction: older 51 | unit: days 52 | unit_count: 3 53 | field: '@timestamp' 54 | stats_result: min_value 55 | ------------- 56 | -------------------------------------------------------------------------------- /docs/asciidoc/inc_strftime_table.asciidoc: -------------------------------------------------------------------------------- 1 | The identifiers that Curator currently recognizes include: 2 | 3 | [width="50%", cols="> are calculated as follows: 2 | 3 | [width="50%", cols="> 10 | * <> 11 | -- 12 | 13 | [[python-security]] 14 | == Python and Secure Connectivity 15 | 16 | Curator was written in Python, which allows it to be distributed as code which 17 | can run across a wide variety of systems, including Linux, Windows, Mac OS, and 18 | any other system or architecture for which a Python interpreter has been 19 | written. Curator was also written to be usable by the 4 most recent major 20 | release branches of Python: 2.7, 3.4, 3.5, and 3.6. It may even run on other 21 | versions, but those versions are not tested. 22 | 23 | Unfortunately, this broad support comes at a cost. While Curator happily ran 24 | on Python version 2.6, this version had its last update more than 3 years ago. 25 | There have been many improvements to security, SSL/TLS and the libraries that 26 | support them since then. Not all of these have been back-ported, which results 27 | in Curator not being able to communicate securely via SSL/TLS, or in some cases 28 | even connect securely. 29 | 30 | Because it is impossible to know if a given system has the correct Python 31 | version, leave alone the most recent libraries and modules, it becomes nearly 32 | impossible to guarantee that Curator will be able to make a secure and 33 | error-free connection to a secured Elasticsearch instance for any `pip` or 34 | RPM/DEB installed modules. This has lead to an increased amount of 35 | troubleshooting and support work for Curator. The precompiled binary packages 36 | were created to address this. 37 | 38 | The precompiled binary packages (APT/YUM, Windows) have been compiled with 39 | Python {pybuild_ver}, which has all of the up-to-date libraries needed for secure 40 | transactions. These packages have been tested connecting to Security (5.x 41 | X-Pack) with self-signed PKI certificates. Connectivity via SSL or TLS to other 42 | open-source plugins may work, but is not guaranteed. 43 | 44 | If you are encountering SSL/TLS errors in Curator, please see the list of 45 | <>. 46 | 47 | [[security-errors]] 48 | == Common Security Error Messages 49 | 50 | === Elasticsearch ConnectionError 51 | 52 | [source,sh] 53 | ----------- 54 | Unable to create client connection to Elasticsearch. Error:ConnectionError(error return without exception set) caused by: SystemError(error return without exception set) 55 | ----------- 56 | 57 | This error can happen on non-secured connections as well. If it happens with a 58 | secured instance, it will usually be accompanied by one or more of the following 59 | messages 60 | 61 | === SNIMissingWarning 62 | 63 | [source,sh] 64 | ----------- 65 | SNIMissingWarning: An HTTPS request has been made, but the SNI (Subject Name Indication) extension to TLS is not available on this platform. This may cause the server to present an incorrect TLS certificate, which can cause validation failures. You can upgrade to a newer version of Python to solve this. For more information, see https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings 66 | ----------- 67 | 68 | This happens on Python 2 versions older than 2.7.9. These older versions lack 69 | https://en.wikipedia.org/wiki/Server_Name_Indication[SNI] support. This can 70 | cause servers to present a certificate that the client thinks is invalid. Follow 71 | the https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2[pyOpenSSL] 72 | guide to resolve this warning. 73 | 74 | === InsecurePlatformWarning 75 | 76 | [source,sh] 77 | ----------- 78 | InsecurePlatformWarning: A true SSLContext object is not available. This prevents urllib3 from configuring SSL appropriately and may cause certain SSL connections to fail. You can upgrade to a newer version of Python to solve this. For more information, see https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings 79 | ----------- 80 | 81 | This happens on Python 2 platforms that have an outdated **ssl** module. These 82 | older **ssl** modules can cause some insecure requests to succeed where they 83 | should fail and secure requests to fail where they should succeed. Follow the 84 | https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2[pyOpenSSL] 85 | guide to resolve this warning. 86 | 87 | === InsecureRequestWarning 88 | 89 | [source,sh] 90 | ----------- 91 | InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.org/en/latest/security.html 92 | ----------- 93 | 94 | This happens when an request is made to an HTTPS URL without certificate 95 | verification enabled. Follow the 96 | https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl[certificate verification] 97 | guide to resolve this warning. 98 | 99 | Related: 100 | 101 | [source,sh] 102 | ----------- 103 | SSLError: [Errno 1] _ssl.c:510: error:14090086:SSL routines:SSL3_GET_SERVER_CERTIFICATE:certificate verify failed 104 | ----------- 105 | -------------------------------------------------------------------------------- /docs/asciidoc/versions.asciidoc: -------------------------------------------------------------------------------- 1 | [[versions]] 2 | = Versions 3 | 4 | [partintro] 5 | -- 6 | Elasticsearch Curator has been around for many different versions of 7 | Elasticsearch. The following document helps clarify which versions of Curator 8 | work with which versions of Elasticsearch. 9 | 10 | The current version of Curator is {curator_version} 11 | 12 | * <> 13 | -- 14 | 15 | [[version-compatibility]] 16 | == Version Compatibility 17 |   18 | 19 | IMPORTANT: Each listed version of Elasticsearch Curator has been fully tested 20 | against unmodified release versions of Elasticsearch. **Modified versions of Elasticsearch may not be fully supported.** 21 | 22 | The current version of Curator is {curator_version} 23 | 24 | [cols="<,<,<,<,<,<",options="header",grid="cols"] 25 | |=== 26 | |Curator Version 27 | |ES 1.x 28 | |ES 2.x 29 | |ES 5.x 30 | |ES 6.x 31 | |ES 7.x 32 | 33 | |          3 34 | |  ✅ 35 | |  ✅ 36 | |  ❌ 37 | |  ❌ 38 | |  ❌ 39 | 40 | |          4 41 | |  ❌ 42 | |  ✅ 43 | |  ✅ 44 | |  ❌ 45 | |  ❌ 46 | 47 | |          5.6- 48 | |  ❌ 49 | |  ❌ 50 | |  ✅ 51 | |  ✅ 52 | |  ❌ 53 | 54 | |          5.7+ 55 | |  ❌ 56 | |  ❌ 57 | |  ✅ 58 | |  ✅ 59 | |  ✅ 60 | |=== 61 | 62 | Learn more about the different versions at: 63 | 64 | * https://www.elastic.co/guide/en/elasticsearch/client/curator/3.5/index.html[Curator 3 Documentation] 65 | * https://www.elastic.co/guide/en/elasticsearch/client/curator/4.2/index.html[Curator 4 Documentation] 66 | * https://www.elastic.co/guide/en/elasticsearch/client/curator/{curator_doc_tree}/index.html[Curator 5 Documentation] 67 | -------------------------------------------------------------------------------- /docs/examples.rst: -------------------------------------------------------------------------------- 1 | .. _examples: 2 | 3 | Examples 4 | ======== 5 | 6 | Each of these examples presupposes that the requisite modules have been imported 7 | and an instance of the Elasticsearch client object has been created: 8 | 9 | :: 10 | 11 | import elasticsearch 12 | import curator 13 | 14 | client = elasticsearch.Elasticsearch() 15 | 16 | Filter indices by prefix 17 | ++++++++++++++++++++++++ 18 | 19 | :: 20 | 21 | ilo = curator.IndexList(client) 22 | ilo.filter_by_regex(kind='prefix', value='logstash-') 23 | 24 | The contents of `ilo.indices` would then only be indices matching the `prefix`. 25 | 26 | 27 | Filter indices by suffix 28 | ++++++++++++++++++++++++ 29 | 30 | :: 31 | 32 | ilo = curator.IndexList(client) 33 | ilo.filter_by_regex(kind='suffix', value='-prod') 34 | 35 | The contents of `ilo.indices` would then only be indices matching the `suffix`. 36 | 37 | 38 | Filter indices by age (name) 39 | ++++++++++++++++++++++++++++ 40 | 41 | This example will match indices with the following criteria: 42 | 43 | * Have a date string of ``%Y.%m.%d`` 44 | * Use `days` as the unit of time measurement 45 | * Filter indices `older` than 5 `days` 46 | 47 | :: 48 | 49 | ilo = curator.IndexList(client) 50 | ilo.filter_by_age(source='name', direction='older', timestring='%Y.%m.%d', 51 | unit='days', unit_count=5 52 | ) 53 | 54 | The contents of `ilo.indices` would then only be indices matching these 55 | criteria. 56 | 57 | 58 | Filter indices by age (creation_date) 59 | +++++++++++++++++++++++++++++++++++++ 60 | 61 | This example will match indices with the following criteria: 62 | 63 | * Use `months` as the unit of time measurement 64 | * Filter indices where the index creation date is `older` than 2 `months` from 65 | this moment. 66 | 67 | :: 68 | 69 | ilo = curator.IndexList(client) 70 | ilo.filter_by_age(source='creation_date', direction='older', 71 | unit='months', unit_count=2 72 | ) 73 | 74 | The contents of `ilo.indices` would then only be indices matching these 75 | criteria. 76 | 77 | Filter indices by age (field_stats) 78 | +++++++++++++++++++++++++++++++++++ 79 | 80 | This example will match indices with the following criteria: 81 | 82 | * Use `days` as the unit of time measurement 83 | * Filter indices where the `timestamp` field's `min_value` is a date `older` 84 | than 3 `weeks` from this moment. 85 | 86 | 87 | :: 88 | 89 | ilo = curator.IndexList(client) 90 | ilo.filter_by_age(source='field_stats', direction='older', 91 | unit='weeks', unit_count=3, field='timestamp', stats_result='min_value' 92 | ) 93 | 94 | The contents of `ilo.indices` would then only be indices matching these 95 | criteria. 96 | -------------------------------------------------------------------------------- /docs/filters.rst: -------------------------------------------------------------------------------- 1 | .. _filters: 2 | 3 | Filter Methods 4 | ============== 5 | 6 | * `IndexList`_ 7 | * `SnapshotList`_ 8 | 9 | IndexList 10 | --------- 11 | 12 | .. automethod:: curator.indexlist.IndexList.filter_allocated 13 | :noindex: 14 | 15 | .. automethod:: curator.indexlist.IndexList.filter_by_age 16 | :noindex: 17 | 18 | .. automethod:: curator.indexlist.IndexList.filter_by_regex 19 | :noindex: 20 | 21 | .. automethod:: curator.indexlist.IndexList.filter_by_space 22 | :noindex: 23 | 24 | .. automethod:: curator.indexlist.IndexList.filter_closed 25 | :noindex: 26 | 27 | .. automethod:: curator.indexlist.IndexList.filter_forceMerged 28 | :noindex: 29 | 30 | .. automethod:: curator.indexlist.IndexList.filter_kibana 31 | :noindex: 32 | 33 | .. automethod:: curator.indexlist.IndexList.filter_opened 34 | :noindex: 35 | 36 | .. automethod:: curator.indexlist.IndexList.filter_none 37 | :noindex: 38 | 39 | .. automethod:: curator.indexlist.IndexList.filter_by_alias 40 | :noindex: 41 | 42 | .. automethod:: curator.indexlist.IndexList.filter_by_count 43 | :noindex: 44 | 45 | .. automethod:: curator.indexlist.IndexList.filter_period 46 | :noindex: 47 | 48 | 49 | SnapshotList 50 | ------------ 51 | 52 | .. automethod:: curator.snapshotlist.SnapshotList.filter_by_age 53 | :noindex: 54 | 55 | .. automethod:: curator.snapshotlist.SnapshotList.filter_by_regex 56 | :noindex: 57 | 58 | .. automethod:: curator.snapshotlist.SnapshotList.filter_by_state 59 | :noindex: 60 | 61 | .. automethod:: curator.snapshotlist.SnapshotList.filter_none 62 | :noindex: 63 | 64 | .. automethod:: curator.snapshotlist.SnapshotList.filter_by_count 65 | :noindex: 66 | 67 | .. automethod:: curator.snapshotlist.SnapshotList.filter_period 68 | :noindex: -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Elasticsearch Curator Python API 2 | ================================ 3 | 4 | The Elasticsearch Curator Python API helps you manage your indices and 5 | snapshots. 6 | 7 | .. note:: 8 | 9 | This documentation is for the Elasticsearch Curator Python API. Documentation 10 | for the Elasticsearch Curator *CLI* -- which uses this API and is installed 11 | as an entry_point as part of the package -- is available in the 12 | `Elastic guide`_. 13 | 14 | .. _Elastic guide: http://www.elastic.co/guide/en/elasticsearch/client/curator/current/index.html 15 | 16 | Compatibility 17 | ------------- 18 | 19 | The Elasticsearch Curator Python API is compatible with the 5.x Elasticsearch versions, 20 | and supports Python versions 2.7, and 3.5+. 21 | 22 | Installation 23 | ------------ 24 | 25 | Install the ``elasticsearch-curator`` package with `pip 26 | `_:: 27 | 28 | pip install elasticsearch-curator 29 | 30 | Example Usage 31 | ------------- 32 | 33 | :: 34 | 35 | import elasticsearch 36 | import curator 37 | 38 | client = elasticsearch.Elasticsearch() 39 | 40 | ilo = curator.IndexList(client) 41 | ilo.filter_by_regex(kind='prefix', value='logstash-') 42 | ilo.filter_by_age(source='name', direction='older', timestring='%Y.%m.%d', unit='days', unit_count=30) 43 | delete_indices = curator.DeleteIndices(ilo) 44 | delete_indices.do_action() 45 | 46 | .. TIP:: 47 | See more examples in the :doc:`Examples ` page. 48 | 49 | Features 50 | -------- 51 | 52 | The API methods fall into the following categories: 53 | 54 | * :doc:`Object Classes ` build and filter index list or snapshot list objects. 55 | * :doc:`Action Classes ` act on object classes. 56 | * :doc:`Utilities ` are helper methods. 57 | 58 | Logging 59 | ~~~~~~~ 60 | 61 | The Elasticsearch Curator Python API uses the standard `logging library`_ from Python. 62 | It inherits two loggers from ``elasticsearch-py``: ``elasticsearch`` and 63 | ``elasticsearch.trace``. Clients use the ``elasticsearch`` logger to log 64 | standard activity, depending on the log level. The ``elasticsearch.trace`` 65 | logger logs requests to the server in JSON format as pretty-printed ``curl`` 66 | commands that you can execute from the command line. The ``elasticsearch.trace`` 67 | logger is not inherited from the base logger and must be activated separately. 68 | 69 | .. _logging library: http://docs.python.org/3.6/library/logging.html 70 | 71 | Contents 72 | -------- 73 | 74 | .. toctree:: 75 | :maxdepth: 2 76 | 77 | objectclasses 78 | actionclasses 79 | filters 80 | utilities 81 | examples 82 | Changelog 83 | 84 | License 85 | ------- 86 | 87 | Copyright (c) 2012–2019 Elasticsearch 88 | 89 | Licensed under the Apache License, Version 2.0 (the "License"); 90 | you may not use this file except in compliance with the License. 91 | You may obtain a copy of the License at 92 | 93 | http://www.apache.org/licenses/LICENSE-2.0 94 | 95 | Unless required by applicable law or agreed to in writing, software 96 | distributed under the License is distributed on an "AS IS" BASIS, 97 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 98 | See the License for the specific language governing permissions and 99 | limitations under the License. 100 | 101 | 102 | Indices and tables 103 | ------------------ 104 | 105 | * :ref:`genindex` 106 | * :ref:`search` 107 | -------------------------------------------------------------------------------- /docs/objectclasses.rst: -------------------------------------------------------------------------------- 1 | .. _objectclasses: 2 | 3 | Object Classes 4 | ============== 5 | 6 | * `IndexList`_ 7 | * `SnapshotList`_ 8 | 9 | 10 | IndexList 11 | --------- 12 | 13 | .. autoclass:: curator.indexlist.IndexList 14 | :members: 15 | 16 | SnapshotList 17 | ------------ 18 | 19 | .. autoclass:: curator.snapshotlist.SnapshotList 20 | :members: 21 | -------------------------------------------------------------------------------- /docs/utilities.rst: -------------------------------------------------------------------------------- 1 | .. _utilities: 2 | 3 | Utility & Helper Methods 4 | ======================== 5 | 6 | .. automodule:: curator.utils 7 | :members: 8 | 9 | .. autoclass:: curator.SchemaCheck 10 | :members: 11 | -------------------------------------------------------------------------------- /examples/actions/alias.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: alias 11 | description: >- 12 | Alias indices older than 7 days but newer than 14 days, with a prefix of 13 | logstash- to 'last_week', remove indices older than 14 days. 14 | options: 15 | name: last_week 16 | extra_settings: 17 | timeout_override: 18 | continue_if_exception: False 19 | disable_action: True 20 | add: 21 | filters: 22 | - filtertype: pattern 23 | kind: prefix 24 | value: logstash- 25 | exclude: 26 | - filtertype: age 27 | source: name 28 | direction: older 29 | timestring: '%Y.%m.%d' 30 | unit: days 31 | unit_count: 7 32 | exclude: 33 | - filtertype: age 34 | direction: younger 35 | timestring: '%Y.%m.%d' 36 | unit: days 37 | unit_count: 14 38 | exclude: 39 | remove: 40 | filters: 41 | - filtertype: pattern 42 | kind: prefix 43 | value: logstash- 44 | exclude: 45 | - filtertype: age 46 | source: name 47 | direction: older 48 | timestring: '%Y.%m.%d' 49 | unit: days 50 | unit_count: 14 51 | exclude: 52 | -------------------------------------------------------------------------------- /examples/actions/allocation.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: allocation 11 | description: >- 12 | Apply shard allocation routing to 'require' 'tag=cold' for hot/cold node 13 | setup for logstash- indices older than 3 days, based on index_creation 14 | date 15 | options: 16 | key: tag 17 | value: cold 18 | allocation_type: require 19 | wait_for_completion: False 20 | continue_if_exception: False 21 | disable_action: True 22 | filters: 23 | - filtertype: pattern 24 | kind: prefix 25 | value: logstash- 26 | exclude: 27 | - filtertype: age 28 | source: creation_date 29 | direction: older 30 | unit: days 31 | unit_count: 3 32 | exclude: 33 | -------------------------------------------------------------------------------- /examples/actions/close.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: close 11 | description: >- 12 | Close indices older than 30 days (based on index name), for logstash- 13 | prefixed indices. 14 | options: 15 | delete_aliases: False 16 | timeout_override: 17 | continue_if_exception: False 18 | disable_action: True 19 | filters: 20 | - filtertype: pattern 21 | kind: prefix 22 | value: logstash- 23 | exclude: 24 | - filtertype: age 25 | source: name 26 | direction: older 27 | timestring: '%Y.%m.%d' 28 | unit: days 29 | unit_count: 30 30 | exclude: 31 | -------------------------------------------------------------------------------- /examples/actions/create_index.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: create_index 11 | description: Create the index as named, with the specified extra settings. 12 | options: 13 | name: myindex 14 | extra_settings: 15 | settings: 16 | number_of_shards: 1 17 | number_of_replicas: 0 18 | timeout_override: 19 | continue_if_exception: False 20 | disable_action: False 21 | -------------------------------------------------------------------------------- /examples/actions/delete_indices.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: delete_indices 11 | description: >- 12 | Delete indices older than 45 days (based on index name), for logstash- 13 | prefixed indices. Ignore the error if the filter does not result in an 14 | actionable list of indices (ignore_empty_list) and exit cleanly. 15 | options: 16 | ignore_empty_list: True 17 | timeout_override: 18 | continue_if_exception: False 19 | disable_action: True 20 | filters: 21 | - filtertype: pattern 22 | kind: prefix 23 | value: logstash- 24 | exclude: 25 | - filtertype: age 26 | source: name 27 | direction: older 28 | timestring: '%Y.%m.%d' 29 | unit: days 30 | unit_count: 45 31 | exclude: 32 | -------------------------------------------------------------------------------- /examples/actions/delete_snapshots.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: delete_snapshots 11 | description: >- 12 | Delete snapshots from the selected repository older than 45 days 13 | (based on creation_date), for 'curator-' prefixed snapshots. 14 | options: 15 | repository: 16 | timeout_override: 17 | continue_if_exception: False 18 | disable_action: True 19 | filters: 20 | - filtertype: pattern 21 | kind: prefix 22 | value: curator- 23 | exclude: 24 | - filtertype: age 25 | source: creation_date 26 | direction: older 27 | unit: days 28 | unit_count: 45 29 | exclude: 30 | -------------------------------------------------------------------------------- /examples/actions/forcemerge.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: forcemerge 11 | description: >- 12 | forceMerge logstash- prefixed indices older than 2 days (based on index 13 | creation_date) to 2 segments per shard. Delay 120 seconds between each 14 | forceMerge operation to allow the cluster to quiesce. 15 | This action will ignore indices already forceMerged to the same or fewer 16 | number of segments per shard, so the 'forcemerged' filter is unneeded. 17 | options: 18 | max_num_segments: 2 19 | delay: 120 20 | timeout_override: 21 | continue_if_exception: False 22 | disable_action: True 23 | filters: 24 | - filtertype: pattern 25 | kind: prefix 26 | value: logstash- 27 | exclude: 28 | - filtertype: age 29 | source: creation_date 30 | direction: older 31 | unit: days 32 | unit_count: 2 33 | exclude: 34 | -------------------------------------------------------------------------------- /examples/actions/freeze.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: freeze 11 | description: >- 12 | Freeze indices older than 30 days (based on index name), for logstash- 13 | prefixed indices. 14 | options: 15 | timeout_override: 16 | continue_if_exception: False 17 | disable_action: True 18 | filters: 19 | - filtertype: pattern 20 | kind: prefix 21 | value: logstash- 22 | exclude: 23 | - filtertype: age 24 | source: name 25 | direction: older 26 | timestring: '%Y.%m.%d' 27 | unit: days 28 | unit_count: 30 29 | exclude: 30 | -------------------------------------------------------------------------------- /examples/actions/open.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: open 11 | description: >- 12 | Open indices older than 30 days but younger than 60 days (based on index 13 | name), for logstash- prefixed indices. 14 | options: 15 | timeout_override: 16 | continue_if_exception: False 17 | disable_action: True 18 | filters: 19 | - filtertype: pattern 20 | kind: prefix 21 | value: logstash- 22 | exclude: 23 | - filtertype: age 24 | source: name 25 | direction: older 26 | timestring: '%Y.%m.%d' 27 | unit: days 28 | unit_count: 30 29 | exclude: 30 | - filtertype: age 31 | source: name 32 | direction: younger 33 | timestring: '%Y.%m.%d' 34 | unit: days 35 | unit_count: 60 36 | exclude: 37 | -------------------------------------------------------------------------------- /examples/actions/reindex.yml: -------------------------------------------------------------------------------- 1 | --- 2 | actions: 3 | 1: 4 | description: "Reindex from remote cluster" 5 | action: reindex 6 | options: 7 | wait_interval: 9 8 | max_wait: -1 9 | timeout: 70 10 | request_body: 11 | source: 12 | remote: 13 | host: https://es01:9200 14 | username: elastic 15 | password: elastic 16 | index: index-2024.07.30 17 | dest: 18 | index: index-2024.07.30 19 | remote_ssl_no_validate: True 20 | # remote_filters: 21 | # - filtertype: age 22 | # source: creation_date 23 | # direction: older 24 | # unit: days 25 | # unit_count: 150 26 | filters: 27 | - filtertype: none 28 | -------------------------------------------------------------------------------- /examples/actions/replicas.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: replicas 11 | description: >- 12 | Reduce the replica count to 0 for logstash- prefixed indices older than 13 | 10 days (based on index creation_date) 14 | options: 15 | count: 0 16 | wait_for_completion: False 17 | timeout_override: 18 | continue_if_exception: False 19 | disable_action: True 20 | filters: 21 | - filtertype: pattern 22 | kind: prefix 23 | value: logstash- 24 | exclude: 25 | - filtertype: age 26 | source: creation_date 27 | direction: older 28 | unit: days 29 | unit_count: 10 30 | exclude: 31 | -------------------------------------------------------------------------------- /examples/actions/restore.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: restore 11 | description: >- 12 | Restore all indices in the most recent curator-* snapshot with state 13 | SUCCESS. Wait for the restore to complete before continuing. Do not skip 14 | the repository filesystem access check. Use the other options to define 15 | the index/shard settings for the restore. 16 | options: 17 | repository: 18 | # Leaving name blank will result in restoring the most recent snapshot by age 19 | name: 20 | # Leaving indices blank will result in restoring all indices in the snapshot 21 | indices: 22 | include_aliases: False 23 | ignore_unavailable: False 24 | include_global_state: True 25 | partial: False 26 | rename_pattern: 27 | rename_replacement: 28 | extra_settings: 29 | wait_for_completion: True 30 | skip_repo_fs_check: False 31 | timeout_override: 32 | continue_if_exception: False 33 | disable_action: False 34 | filters: 35 | - filtertype: pattern 36 | kind: prefix 37 | value: curator- 38 | exclude: 39 | - filtertype: state 40 | state: SUCCESS 41 | exclude: 42 | -------------------------------------------------------------------------------- /examples/actions/shrink.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: shrink 11 | description: >- 12 | Shrink logstash indices older than 2 days on the node with the most 13 | available space, excluding the node named 'not_this_node'. 14 | Delete each source index after successful shrink, then reroute the shrunk 15 | index with the provided parameters. 16 | options: 17 | disable_action: False 18 | ignore_empty_list: True 19 | shrink_node: DETERMINISTIC 20 | node_filters: 21 | permit_masters: False 22 | exclude_nodes: ['not_this_node'] 23 | number_of_shards: 1 24 | number_of_replicas: 1 25 | shrink_prefix: 26 | shrink_suffix: '-shrink' 27 | copy_aliases: True 28 | delete_after: True 29 | post_allocation: 30 | allocation_type: include 31 | key: node_tag 32 | value: cold 33 | wait_for_active_shards: 1 34 | extra_settings: 35 | settings: 36 | index.codec: best_compression 37 | wait_for_completion: True 38 | wait_for_rebalance: True 39 | wait_interval: 9 40 | max_wait: -1 41 | filters: 42 | - filtertype: pattern 43 | kind: prefix 44 | value: test_shrink- 45 | - filtertype: age 46 | source: creation_date 47 | direction: older 48 | unit: days 49 | unit_count: 2 50 | -------------------------------------------------------------------------------- /examples/actions/snapshot.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: snapshot 11 | description: >- 12 | Snapshot logstash- prefixed indices older than 1 day (based on index 13 | creation_date) with the default snapshot name pattern of 14 | 'curator-%Y%m%d%H%M%S'. Wait for the snapshot to complete. Do not skip 15 | the repository filesystem access check. Use the other options to create 16 | the snapshot. 17 | options: 18 | repository: 19 | # Leaving name blank will result in the default 'curator-%Y%m%d%H%M%S' 20 | name: 21 | ignore_unavailable: False 22 | include_global_state: True 23 | partial: False 24 | wait_for_completion: True 25 | skip_repo_fs_check: False 26 | timeout_override: 27 | continue_if_exception: False 28 | disable_action: False 29 | filters: 30 | - filtertype: pattern 31 | kind: prefix 32 | value: logstash- 33 | exclude: 34 | - filtertype: age 35 | source: creation_date 36 | direction: older 37 | unit: days 38 | unit_count: 1 39 | exclude: 40 | -------------------------------------------------------------------------------- /examples/actions/unfreeze.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | # 5 | # Also remember that all examples have 'disable_action' set to True. If you 6 | # want to use this action as a template, be sure to set this to False after 7 | # copying it. 8 | actions: 9 | 1: 10 | action: unfreeze 11 | description: >- 12 | Unfreeze indices older than 30 days but younger than 60 days (based on index 13 | name), for logstash- prefixed indices. 14 | options: 15 | timeout_override: 16 | continue_if_exception: False 17 | disable_action: True 18 | filters: 19 | - filtertype: pattern 20 | kind: prefix 21 | value: logstash- 22 | exclude: 23 | - filtertype: age 24 | source: name 25 | direction: older 26 | timestring: '%Y.%m.%d' 27 | unit: days 28 | unit_count: 30 29 | exclude: 30 | - filtertype: age 31 | source: name 32 | direction: younger 33 | timestring: '%Y.%m.%d' 34 | unit: days 35 | unit_count: 60 36 | exclude: 37 | -------------------------------------------------------------------------------- /examples/curator.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # Remember, leave a key empty if there is no value. None will be a string, 3 | # not a Python "NoneType" 4 | client: 5 | hosts: 6 | - 127.0.0.1 7 | port: 9200 8 | url_prefix: 9 | use_ssl: True 10 | certificate: 11 | client_cert: 12 | client_key: 13 | aws_key: 14 | aws_secret_key: 15 | aws_region: 16 | ssl_no_validate: True 17 | http_auth: admin:admin 18 | username: admin 19 | password: admin 20 | timeout: 30 21 | master_only: False 22 | 23 | logging: 24 | loglevel: INFO 25 | logfile: 26 | logformat: default 27 | blacklist: ['elasticsearch', 'urllib3'] 28 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | voluptuous>=0.12.1 2 | opensearch-py>=2.0.0 3 | urllib3>=1.26.5,<2 4 | requests>=2.26.0 5 | boto3>=1.18.18 6 | requests_aws4auth>=1.1.1 7 | click>=7.0,<8.0 8 | pyyaml>=5.4.1 9 | certifi>=2021.5.30 10 | six>=1.16.0 11 | -------------------------------------------------------------------------------- /run_curator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Wrapper for running curator from source. 5 | 6 | When used with Python 3 (and the DEB and RPM packages of Curator are compiled 7 | and bundled with Python 3), Curator requires the locale to be unicode. Any of 8 | the above unicode definitions are acceptable. 9 | 10 | To set the locale to be unicode, try: 11 | 12 | $ export LC_ALL=en_US.utf8 13 | $ curator [ARGS] 14 | 15 | Alternately, you should be able to specify the locale on the command-line: 16 | 17 | $ LC_ALL=en_US.utf8 curator [ARGS] 18 | 19 | Be sure to substitute your unicode variant for en_US.utf8 20 | 21 | """ 22 | 23 | from curator.cli import cli 24 | 25 | 26 | if __name__ == '__main__': 27 | try: 28 | # This is because click uses decorators, and pylint doesn't catch that 29 | # pylint: disable=no-value-for-parameter 30 | cli() 31 | except RuntimeError as e: 32 | import sys 33 | print('{0}'.format(e)) 34 | sys.exit(1) 35 | except Exception as e: 36 | if 'ASCII' in str(e): 37 | print('{0}'.format(e)) 38 | print(__doc__) 39 | -------------------------------------------------------------------------------- /run_es_repo_mgr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Wrapper for running es_repo_mgr from source. 5 | 6 | When used with Python 3 (and the DEB and RPM packages of Curator are compiled 7 | and bundled with Python 3), Curator requires the locale to be unicode. Any of 8 | the above unicode definitions are acceptable. 9 | 10 | To set the locale to be unicode, try: 11 | 12 | $ export LC_ALL=en_US.utf8 13 | $ es_repo_mgr [ARGS] 14 | 15 | Alternately, you should be able to specify the locale on the command-line: 16 | 17 | $ LC_ALL=en_US.utf8 es_repo_mgr [ARGS] 18 | 19 | Be sure to substitute your unicode variant for en_US.utf8 20 | 21 | """ 22 | 23 | from curator.repomgrcli import repo_mgr_cli 24 | 25 | 26 | if __name__ == '__main__': 27 | try: 28 | repo_mgr_cli() 29 | except RuntimeError as e: 30 | import sys 31 | print('{0}'.format(e)) 32 | sys.exit(1) 33 | except Exception as e: 34 | if 'ASCII' in str(e): 35 | print('{0}'.format(e)) 36 | print(__doc__) 37 | -------------------------------------------------------------------------------- /run_singleton.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Wrapper for running singletons from source. 5 | 6 | When used with Python 3 (and the DEB and RPM packages of Curator are compiled 7 | and bundled with Python 3), Curator requires the locale to be unicode. Any of 8 | the above unicode definitions are acceptable. 9 | 10 | To set the locale to be unicode, try: 11 | 12 | $ export LC_ALL=en_US.utf8 13 | $ curator_cli [ARGS] 14 | 15 | Alternately, you should be able to specify the locale on the command-line: 16 | 17 | $ LC_ALL=en_US.utf8 curator_cli [ARGS] 18 | 19 | Be sure to substitute your unicode variant for en_US.utf8 20 | 21 | """ 22 | 23 | import click 24 | from curator.singletons import cli 25 | 26 | if __name__ == '__main__': 27 | try: 28 | cli(obj={}) 29 | except RuntimeError as e: 30 | import sys 31 | print('{0}'.format(e)) 32 | sys.exit(1) 33 | except Exception as e: 34 | if 'ASCII' in str(e): 35 | print('{0}'.format(e)) 36 | print(__doc__) 37 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = curator-opensearch 3 | description = "Tending your OpenSearch indices" 4 | long_description = file: README.rst 5 | version = attr: curator._version.__version__ 6 | author = Uzhinsky 7 | author_email = lspci@mail.ru 8 | url = https://github.com/flant/curator-opensearch 9 | license = Apache License, Version 2.0 10 | keywords = opensearch, time-series, indexed, index-expiry, elasticsearch 11 | classifiers = 12 | Intended Audience :: Developers 13 | Intended Audience :: System Administrators 14 | License :: OSI Approved :: Apache Software License 15 | Operating System :: OS Independent 16 | Programming Language :: Python 17 | Programming Language :: Python :: 3.7 18 | Programming Language :: Python :: 3.8 19 | Programming Language :: Python :: 3.9 20 | 21 | [options] 22 | install_requires = 23 | voluptuous>=0.12.1 24 | opensearch-py>=2.0.0 25 | urllib3>=1.26.5,<2 26 | requests>=2.26.0 27 | boto3>=1.18.18 28 | requests_aws4auth>=1.1.1 29 | click>=7.0,<8.0 30 | pyyaml>=5.4.1 31 | certifi>=2021.5.30 32 | six>=1.16.0 33 | 34 | setup_requires = 35 | voluptuous>=0.12.1 36 | opensearch-py>=2.0.0 37 | urllib3>=1.26.5,<2 38 | requests>=2.26.0 39 | boto3>=1.18.18 40 | requests_aws4auth>=1.1.1 41 | click>=7.0,<8.0 42 | pyyaml>=5.4.1 43 | certifi>=2021.5.30 44 | six>=1.16.0 45 | 46 | packages = curator 47 | include_package_data = True 48 | tests_require = 49 | mock 50 | nose 51 | coverage 52 | nosexcover 53 | 54 | [options.entry_points] 55 | console_scripts = 56 | curator = curator.cli:cli 57 | curator_cli = curator.curator_cli:main 58 | es_repo_mgr = curator.repomgrcli:repo_mgr_cli 59 | 60 | [options.packages.find] 61 | exclude = 62 | tests 63 | 64 | [bdist_wheel] 65 | universal=1 66 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flant/curator-opensearch/7e50540e861fe04e2d90a5eaf3437d70f62a34bc/test/__init__.py -------------------------------------------------------------------------------- /test/integration/test_clusterrouting.py: -------------------------------------------------------------------------------- 1 | import opensearchpy 2 | import curator 3 | import os 4 | import json 5 | import string, random, tempfile 6 | import click 7 | from click import testing as clicktest 8 | from mock import patch, Mock 9 | 10 | from . import CuratorTestCase 11 | from . import testvars as testvars 12 | 13 | import logging 14 | logger = logging.getLogger(__name__) 15 | 16 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 17 | port = int(port) if port else 9200 18 | 19 | 20 | class TestCLIClusterRouting(CuratorTestCase): 21 | def test_allocation_all(self): 22 | routing_type = 'allocation' 23 | value = 'all' 24 | self.write_config( 25 | self.args['configfile'], testvars.client_config.format(host, port)) 26 | self.write_config(self.args['actionfile'], 27 | testvars.cluster_routing_test.format(routing_type, value)) 28 | self.create_index('my_index') 29 | self.create_index('not_my_index') 30 | test = clicktest.CliRunner() 31 | result = test.invoke( 32 | curator.cli, 33 | [ 34 | '--config', self.args['configfile'], 35 | self.args['actionfile'] 36 | ], 37 | ) 38 | 39 | self.assertEquals(testvars.CRA_all, 40 | self.client.cluster.get_settings()) 41 | def test_extra_option(self): 42 | self.write_config( 43 | self.args['configfile'], testvars.client_config.format(host, port)) 44 | self.write_config(self.args['actionfile'], 45 | testvars.bad_option_proto_test.format('cluster_routing')) 46 | self.create_index('my_index') 47 | self.create_index('not_my_index') 48 | test = clicktest.CliRunner() 49 | result = test.invoke( 50 | curator.cli, 51 | [ 52 | '--config', self.args['configfile'], 53 | self.args['actionfile'] 54 | ], 55 | ) 56 | self.assertEqual(1, result.exit_code) 57 | -------------------------------------------------------------------------------- /test/integration/test_datemath.py: -------------------------------------------------------------------------------- 1 | import opensearchpy 2 | import curator 3 | import os 4 | import json 5 | import string, random, tempfile 6 | import click 7 | from click import testing as clicktest 8 | from mock import patch, Mock 9 | from datetime import timedelta, datetime, date 10 | 11 | from . import CuratorTestCase 12 | from . import testvars as testvars 13 | 14 | import logging 15 | logger = logging.getLogger(__name__) 16 | 17 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 18 | port = int(port) if port else 9200 19 | 20 | class TestParseDateMath(CuratorTestCase): 21 | def test_function_positive(self): 22 | test_string = u'<.prefix-{2001-01-01-13||+1h/h{yyyy-MM-dd-HH|-07:00}}-suffix>' 23 | expected = u'.prefix-2001-01-01-14-suffix' 24 | self.assertEqual(expected, curator.parse_datemath(self.client, test_string)) 25 | def test_assorted_datemaths(self): 26 | for test_string, expected in [ 27 | (u'', u'prefix-{0}-suffix'.format(datetime.utcnow().strftime('%Y.%m.%d'))), 28 | (u'', u'prefix-{0}'.format((datetime.utcnow()-timedelta(days=1)).strftime('%Y.%m.%d'))), 29 | (u'<{now+1d/d}>', u'{0}'.format((datetime.utcnow()+timedelta(days=1)).strftime('%Y.%m.%d'))), 30 | (u'<{now+1d/d}>', u'{0}'.format((datetime.utcnow()+timedelta(days=1)).strftime('%Y.%m.%d'))), 31 | (u'<{now+10d/d{yyyy-MM}}>', u'{0}'.format((datetime.utcnow()+timedelta(days=10)).strftime('%Y-%m'))), 32 | (u'<{now+10d/h{yyyy-MM-dd-HH|-07:00}}>', u'{0}'.format((datetime.utcnow()+timedelta(days=10)-timedelta(hours=7)).strftime('%Y-%m-%d-%H'))), 33 | ]: 34 | self.assertEqual(expected, curator.parse_datemath(self.client, test_string)) 35 | -------------------------------------------------------------------------------- /test/integration/test_envvars.py: -------------------------------------------------------------------------------- 1 | import opensearchpy 2 | import curator 3 | import os 4 | import json 5 | import string, random, tempfile 6 | import click 7 | from click import testing as clicktest 8 | from mock import patch, Mock 9 | 10 | from . import CuratorTestCase 11 | from . import testvars as testvars 12 | 13 | import logging 14 | logger = logging.getLogger(__name__) 15 | 16 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 17 | port = int(port) if port else 9200 18 | 19 | def random_envvar(size): 20 | return ''.join( 21 | random.SystemRandom().choice( 22 | string.ascii_uppercase + string.digits 23 | ) for _ in range(size) 24 | ) 25 | 26 | class TestEnvVars(CuratorTestCase): 27 | def test_present(self): 28 | evar = random_envvar(8) 29 | os.environ[evar] = "1234" 30 | dollar = '${' + evar + '}' 31 | self.write_config( 32 | self.args['configfile'], 33 | testvars.client_config_envvars.format(dollar, port, 30) 34 | ) 35 | cfg = curator.get_yaml(self.args['configfile']) 36 | self.assertEqual( 37 | cfg['client']['hosts'], 38 | os.environ.get(evar) 39 | ) 40 | del os.environ[evar] 41 | def test_not_present(self): 42 | evar = random_envvar(8) 43 | dollar = '${' + evar + '}' 44 | self.write_config( 45 | self.args['configfile'], 46 | testvars.client_config_envvars.format(dollar, port, 30) 47 | ) 48 | cfg = curator.get_yaml(self.args['configfile']) 49 | self.assertIsNone(cfg['client']['hosts']) 50 | def test_not_present_with_default(self): 51 | evar = random_envvar(8) 52 | default = random_envvar(8) 53 | dollar = '${' + evar + ':' + default + '}' 54 | self.write_config( 55 | self.args['configfile'], 56 | testvars.client_config_envvars.format(dollar, port, 30) 57 | ) 58 | cfg = curator.get_yaml(self.args['configfile']) 59 | self.assertEqual( 60 | cfg['client']['hosts'], 61 | default 62 | ) 63 | def test_do_something_with_int_value(self): 64 | self.create_indices(10) 65 | evar = random_envvar(8) 66 | os.environ[evar] = "1234" 67 | dollar = '${' + evar + '}' 68 | self.write_config( 69 | self.args['configfile'], 70 | testvars.client_config_envvars.format(host, port, dollar) 71 | ) 72 | cfg = curator.get_yaml(self.args['configfile']) 73 | self.assertEqual( 74 | cfg['client']['timeout'], 75 | os.environ.get(evar) 76 | ) 77 | self.write_config(self.args['actionfile'], 78 | testvars.delete_proto.format( 79 | 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ' 80 | ) 81 | ) 82 | test = clicktest.CliRunner() 83 | _ = test.invoke( 84 | curator.cli, 85 | [ 86 | '--config', self.args['configfile'], 87 | self.args['actionfile'] 88 | ], 89 | ) 90 | self.assertEquals(5, len(curator.get_indices(self.client))) 91 | del os.environ[evar] 92 | -------------------------------------------------------------------------------- /test/integration/test_forcemerge.py: -------------------------------------------------------------------------------- 1 | import opensearchpy 2 | import curator 3 | import os 4 | import json 5 | import string 6 | import random 7 | import tempfile 8 | from time import sleep 9 | import click 10 | from click import testing as clicktest 11 | from mock import patch, Mock 12 | 13 | from . import CuratorTestCase 14 | from . import testvars as testvars 15 | 16 | import logging 17 | logger = logging.getLogger(__name__) 18 | 19 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 20 | port = int(port) if port else 9200 21 | 22 | class TestActionFileforceMerge(CuratorTestCase): 23 | def test_merge(self): 24 | count = 1 25 | idx = 'my_index' 26 | self.create_index(idx) 27 | self.add_docs(idx) 28 | ilo1 = curator.IndexList(self.client) 29 | ilo1._get_segment_counts() 30 | self.assertEqual(3, ilo1.index_info[idx]['segments']) 31 | self.write_config( 32 | self.args['configfile'], testvars.client_config.format(host, port)) 33 | self.write_config(self.args['actionfile'], 34 | testvars.forcemerge_test.format(count, 0.9)) 35 | test = clicktest.CliRunner() 36 | _ = test.invoke( 37 | curator.cli, 38 | ['--config', self.args['configfile'], self.args['actionfile']], 39 | ) 40 | ilo2 = curator.IndexList(self.client) 41 | # This stupid block is only for the benefit of Travis CI 42 | # With Python 2.7 and ES 7.0, it apparently can finish testing before 43 | # the segments have _reported_ as fully merged. This is forcing 44 | # 3 checks before giving up and reporting the result. 45 | for _ in range(0, 3): 46 | self.client.indices.refresh(index=idx) 47 | ilo2._get_segment_counts() 48 | if ilo2.index_info[idx]['segments'] == count: 49 | break 50 | else: 51 | sleep(1) 52 | self.assertEqual(count, ilo2.index_info[idx]['segments']) 53 | def test_extra_option(self): 54 | self.write_config( 55 | self.args['configfile'], testvars.client_config.format(host, port)) 56 | self.write_config(self.args['actionfile'], 57 | testvars.bad_option_proto_test.format('forcemerge')) 58 | test = clicktest.CliRunner() 59 | result = test.invoke( 60 | curator.cli, 61 | [ 62 | '--config', self.args['configfile'], 63 | self.args['actionfile'] 64 | ], 65 | ) 66 | self.assertEqual(1, result.exit_code) 67 | 68 | class TestCLIforceMerge(CuratorTestCase): 69 | def test_merge(self): 70 | count = 1 71 | idx = 'my_index' 72 | self.create_index(idx) 73 | self.add_docs(idx) 74 | ilo1 = curator.IndexList(self.client) 75 | ilo1._get_segment_counts() 76 | self.assertEqual(3, ilo1.index_info[idx]['segments']) 77 | args = self.get_runner_args() 78 | args += [ 79 | '--config', self.args['configfile'], 80 | 'forcemerge', 81 | '--max_num_segments', str(count), 82 | '--delay', '0.9', 83 | '--filter_list', '{"filtertype":"pattern","kind":"prefix","value":"my"}', 84 | ] 85 | self.assertEqual(0, self.run_subprocess(args, logname='TestCLIforceMerge.test_merge')) 86 | ilo2 = curator.IndexList(self.client) 87 | # This stupid block is only for the benefit of Travis CI 88 | # With Python 2.7 and ES 7.0, it apparently can finish testing before 89 | # the segments have _reported_ as fully merged. This is forcing 90 | # 3 checks before giving up and reporting the result. 91 | for _ in range(0, 3): 92 | self.client.indices.refresh(index=idx) 93 | ilo2._get_segment_counts() 94 | if ilo2.index_info[idx]['segments'] == count: 95 | break 96 | else: 97 | sleep(1) 98 | self.assertEqual(count, ilo2.index_info[idx]['segments']) 99 | -------------------------------------------------------------------------------- /test/integration/test_integrations.py: -------------------------------------------------------------------------------- 1 | import opensearchpy 2 | import curator 3 | import os 4 | import json 5 | import string, random, tempfile 6 | from time import sleep 7 | import click 8 | from click import testing as clicktest 9 | from mock import patch, Mock 10 | 11 | from . import CuratorTestCase 12 | from . import testvars as testvars 13 | 14 | import logging 15 | logger = logging.getLogger(__name__) 16 | 17 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 18 | port = int(port) if port else 9200 19 | 20 | class TestFilters(CuratorTestCase): 21 | def test_filter_by_alias(self): 22 | alias = 'testalias' 23 | self.write_config( 24 | self.args['configfile'], testvars.client_config.format(host, port)) 25 | self.write_config(self.args['actionfile'], 26 | testvars.filter_by_alias.format('testalias', False)) 27 | self.create_index('my_index') 28 | self.create_index('dummy') 29 | self.client.indices.put_alias(index='dummy', name=alias) 30 | test = clicktest.CliRunner() 31 | _ = test.invoke( 32 | curator.cli, 33 | [ 34 | '--config', self.args['configfile'], 35 | self.args['actionfile'] 36 | ], 37 | ) 38 | self.assertEquals(1, len(curator.get_indices(self.client))) 39 | def test_filter_by_array_of_aliases(self): 40 | alias = 'testalias' 41 | self.write_config( 42 | self.args['configfile'], testvars.client_config.format(host, port)) 43 | self.write_config(self.args['actionfile'], 44 | testvars.filter_by_alias.format(' [ testalias, foo ]', False)) 45 | self.create_index('my_index') 46 | self.create_index('dummy') 47 | self.client.indices.put_alias(index='dummy', name=alias) 48 | test = clicktest.CliRunner() 49 | _ = test.invoke( 50 | curator.cli, 51 | [ 52 | '--config', self.args['configfile'], 53 | self.args['actionfile'] 54 | ], 55 | ) 56 | ver = curator.get_version(self.client) 57 | if ver >= (5,5,0): 58 | self.assertEquals(2, len(curator.get_indices(self.client))) 59 | else: 60 | self.assertEquals(1, len(curator.get_indices(self.client))) 61 | def test_filter_by_alias_bad_aliases(self): 62 | alias = 'testalias' 63 | self.write_config( 64 | self.args['configfile'], testvars.client_config.format(host, port)) 65 | self.write_config(self.args['actionfile'], 66 | testvars.filter_by_alias.format('{"this":"isadict"}', False)) 67 | self.create_index('my_index') 68 | self.create_index('dummy') 69 | self.client.indices.put_alias(index='dummy', name=alias) 70 | test = clicktest.CliRunner() 71 | result = test.invoke( 72 | curator.cli, 73 | [ 74 | '--config', self.args['configfile'], 75 | self.args['actionfile'] 76 | ], 77 | ) 78 | self.assertEquals( 79 | type(curator.ConfigurationError()), type(result.exception)) 80 | self.assertEquals(2, len(curator.get_indices(self.client))) 81 | def test_field_stats_skips_empty_index(self): 82 | delete_field_stats = ('---\n' 83 | 'actions:\n' 84 | ' 1:\n' 85 | ' action: delete_indices\n' 86 | ' filters:\n' 87 | ' - filtertype: age\n' 88 | ' source: field_stats\n' 89 | ' direction: older\n' 90 | ' field: "{0}"\n' 91 | ' unit: days\n' 92 | ' unit_count: 1\n' 93 | ' stats_result: min_value\n' 94 | ) 95 | idx = 'my_index' 96 | zero = 'zero' 97 | field = '@timestamp' 98 | time = '2017-12-31T23:59:59.999Z' 99 | # Create idx with a single, @timestamped doc 100 | self.client.create(index=idx, doc_type='doc', id=1, body={field: time}) 101 | # Flush to ensure it's written 102 | # Decorators make this pylint exception necessary 103 | # pylint: disable=E1123 104 | self.client.indices.flush(index=idx, force=True) 105 | self.client.indices.refresh(index=idx) 106 | # Create zero with no docs 107 | self.create_index(zero) 108 | self.write_config( 109 | self.args['configfile'], testvars.client_config.format(host, port)) 110 | self.write_config(self.args['actionfile'], delete_field_stats.format(field)) 111 | test = clicktest.CliRunner() 112 | _ = test.invoke( 113 | curator.cli, 114 | ['--config', self.args['configfile'], self.args['actionfile']], 115 | ) 116 | # It should skip deleting 'zero', as it has 0 docs 117 | self.assertEqual([zero], curator.get_indices(self.client)) 118 | -------------------------------------------------------------------------------- /test/integration/test_open.py: -------------------------------------------------------------------------------- 1 | import opensearchpy 2 | import curator 3 | import os 4 | import json 5 | import string, random, tempfile 6 | import click 7 | from click import testing as clicktest 8 | from mock import patch, Mock 9 | 10 | from . import CuratorTestCase 11 | from . import testvars as testvars 12 | 13 | import logging 14 | logger = logging.getLogger(__name__) 15 | 16 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 17 | port = int(port) if port else 9200 18 | 19 | class TestActionFileOpenClosed(CuratorTestCase): 20 | def test_open_closed(self): 21 | self.write_config( 22 | self.args['configfile'], testvars.client_config.format(host, port)) 23 | self.write_config(self.args['actionfile'], 24 | testvars.optionless_proto.format('open')) 25 | t1, t2 = ('dummy', 'my_index') 26 | self.create_index(t1) 27 | self.create_index(t2) 28 | # Decorators make this pylint exception necessary 29 | # pylint: disable=E1123 30 | self.client.indices.close(index=t2, ignore_unavailable=True) 31 | test = clicktest.CliRunner() 32 | _ = test.invoke( 33 | curator.cli, 34 | [ 35 | '--config', self.args['configfile'], 36 | self.args['actionfile'] 37 | ], 38 | ) 39 | csi = self.client.cluster.state(metric='metadata')['metadata']['indices'] 40 | self.assertNotEqual('close', csi[t2]['state']) 41 | self.assertNotEqual('close', csi[t1]['state']) 42 | def test_extra_option(self): 43 | self.write_config( 44 | self.args['configfile'], testvars.client_config.format(host, port)) 45 | self.write_config(self.args['actionfile'], 46 | testvars.bad_option_proto_test.format('open')) 47 | t1, t2 = ('dummy', 'my_index') 48 | self.create_index(t1) 49 | self.create_index(t2) 50 | # Decorators make this pylint exception necessary 51 | # pylint: disable=E1123 52 | self.client.indices.close(index=t2, ignore_unavailable=True) 53 | test = clicktest.CliRunner() 54 | result = test.invoke( 55 | curator.cli, 56 | [ 57 | '--config', self.args['configfile'], 58 | self.args['actionfile'] 59 | ], 60 | ) 61 | csi = self.client.cluster.state(metric='metadata')['metadata']['indices'] 62 | self.assertEqual('close', csi[t2]['state']) 63 | self.assertNotEqual('close', csi[t1]['state']) 64 | self.assertEqual(1, result.exit_code) 65 | 66 | class TestCLIOpenClosed(CuratorTestCase): 67 | def test_open_closed(self): 68 | t1, t2 = ('dummy', 'my_index') 69 | self.create_index(t1) 70 | self.create_index(t2) 71 | # Decorators make this pylint exception necessary 72 | # pylint: disable=E1123 73 | self.client.indices.close(index=t2, ignore_unavailable=True) 74 | args = self.get_runner_args() 75 | args += [ 76 | '--config', self.args['configfile'], 77 | 'open', 78 | '--filter_list', '{"filtertype":"pattern","kind":"prefix","value":"my"}', 79 | ] 80 | self.assertEqual(0, self.run_subprocess(args, logname='TestCLIOpenClosed.test_open_closed')) 81 | csi = self.client.cluster.state(metric='metadata')['metadata']['indices'] 82 | self.assertNotEqual('close', csi[t2]['state']) 83 | self.assertNotEqual('close', csi[t1]['state']) 84 | 85 | -------------------------------------------------------------------------------- /test/integration/test_replicas.py: -------------------------------------------------------------------------------- 1 | import opensearchpy 2 | import curator 3 | import os 4 | import json 5 | import string, random, tempfile 6 | import click 7 | from click import testing as clicktest 8 | from mock import patch, Mock 9 | 10 | from . import CuratorTestCase 11 | from . import testvars as testvars 12 | 13 | import logging 14 | logger = logging.getLogger(__name__) 15 | 16 | host, port = os.environ.get('TEST_ES_SERVER', 'localhost:9200').split(':') 17 | port = int(port) if port else 9200 18 | 19 | class TestActionFileReplicas(CuratorTestCase): 20 | def test_increase_count(self): 21 | count = 2 22 | idx = 'my_index' 23 | self.write_config( 24 | self.args['configfile'], testvars.client_config.format(host, port)) 25 | self.write_config(self.args['actionfile'], 26 | testvars.replicas_test.format(count)) 27 | self.create_index(idx) 28 | test = clicktest.CliRunner() 29 | _ = test.invoke( 30 | curator.cli, 31 | [ 32 | '--config', self.args['configfile'], 33 | self.args['actionfile'] 34 | ], 35 | ) 36 | self.assertEqual( 37 | count, 38 | int(self.client.indices.get_settings( 39 | index=idx)[idx]['settings']['index']['number_of_replicas']) 40 | ) 41 | def test_no_count(self): 42 | self.create_index('foo') 43 | self.write_config( 44 | self.args['configfile'], testvars.client_config.format(host, port)) 45 | self.write_config(self.args['actionfile'], 46 | testvars.replicas_test.format(' ')) 47 | test = clicktest.CliRunner() 48 | _ = test.invoke( 49 | curator.cli, 50 | [ 51 | '--config', self.args['configfile'], 52 | self.args['actionfile'] 53 | ], 54 | ) 55 | self.assertEqual(1, _.exit_code) 56 | def test_extra_option(self): 57 | self.create_index('foo') 58 | self.write_config( 59 | self.args['configfile'], testvars.client_config.format(host, port)) 60 | self.write_config(self.args['actionfile'], 61 | testvars.bad_option_proto_test.format('replicas')) 62 | test = clicktest.CliRunner() 63 | _ = test.invoke( 64 | curator.cli, 65 | [ 66 | '--config', self.args['configfile'], 67 | self.args['actionfile'] 68 | ], 69 | ) 70 | self.assertEqual(1, _.exit_code) 71 | 72 | class TestCLIReplicas(CuratorTestCase): 73 | def test_increase_count(self): 74 | count = 2 75 | idx = 'my_index' 76 | self.create_index(idx) 77 | args = self.get_runner_args() 78 | args += [ 79 | '--config', self.args['configfile'], 80 | 'replicas', 81 | '--count', str(count), 82 | '--filter_list', '{"filtertype":"pattern","kind":"prefix","value":"my"}', 83 | ] 84 | self.assertEqual(0, self.run_subprocess(args, logname='TestCLIOpenClosed.test_open_closed')) 85 | self.assertEqual( 86 | count, 87 | int(self.client.indices.get_settings(index=idx)[idx]['settings']['index']['number_of_replicas']) 88 | ) 89 | -------------------------------------------------------------------------------- /test/run_tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function 3 | 4 | import sys 5 | from os.path import dirname, abspath 6 | 7 | import nose 8 | 9 | def run_all(argv=None): 10 | sys.exitfunc = lambda: sys.stderr.write('Shutting down....\n') 11 | 12 | # always insert coverage when running tests through setup.py 13 | if argv is None: 14 | argv = [ 15 | 'nosetests', '--with-xunit', 16 | '--logging-format=%(levelname)s %(name)22s %(funcName)22s:%(lineno)-4d %(message)s', 17 | '--with-coverage', '--cover-package=curator', '--cover-erase', '--cover-html', 18 | '--verbose', 19 | ] 20 | 21 | nose.run_exit( 22 | argv=argv, 23 | defaultTest=abspath(dirname(__file__)) 24 | ) 25 | 26 | if __name__ == '__main__': 27 | run_all(sys.argv) 28 | -------------------------------------------------------------------------------- /test/unit/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import tempfile 4 | import random 5 | import string 6 | from unittest import SkipTest, TestCase 7 | from mock import Mock 8 | from .testvars import * 9 | 10 | class CLITestCase(TestCase): 11 | def setUp(self): 12 | super(CLITestCase, self).setUp() 13 | self.args = {} 14 | dirname = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) 15 | ymlname = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) 16 | badyaml = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) 17 | # This will create a psuedo-random temporary directory on the machine 18 | # which runs the unit tests, but NOT on the machine where elasticsearch 19 | # is running. This means tests may fail if run against remote instances 20 | # unless you explicitly set `self.args['location']` to a proper spot 21 | # on the target machine. 22 | self.args['tmpdir'] = tempfile.mkdtemp(suffix=dirname) 23 | if not os.path.exists(self.args['tmpdir']): 24 | os.makedirs(self.args['tmpdir']) 25 | self.args['yamlfile'] = os.path.join(self.args['tmpdir'], ymlname) 26 | self.args['invalid_yaml'] = os.path.join(self.args['tmpdir'], badyaml) 27 | self.args['no_file_here'] = os.path.join(self.args['tmpdir'], 'not_created') 28 | with open(self.args['yamlfile'], 'w') as f: 29 | f.write(testvars.yamlconfig) 30 | with open(self.args['invalid_yaml'], 'w') as f: 31 | f.write('gobbledeygook: @failhere\n') 32 | 33 | def tearDown(self): 34 | if os.path.exists(self.args['tmpdir']): 35 | shutil.rmtree(self.args['tmpdir']) 36 | -------------------------------------------------------------------------------- /test/unit/test_action_close.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import Mock, patch 3 | import opensearchpy 4 | import curator 5 | # Get test variables and constants from a single source 6 | from . import testvars as testvars 7 | 8 | class TestActionClose(TestCase): 9 | def test_init_raise(self): 10 | self.assertRaises(TypeError, curator.Close, 'invalid') 11 | def test_init(self): 12 | client = Mock() 13 | client.info.return_value = {'version': {'number': '5.0.0'} } 14 | client.indices.get_settings.return_value = testvars.settings_one 15 | client.cluster.state.return_value = testvars.clu_state_one 16 | client.indices.stats.return_value = testvars.stats_one 17 | ilo = curator.IndexList(client) 18 | co = curator.Close(ilo) 19 | self.assertEqual(ilo, co.index_list) 20 | self.assertEqual(client, co.client) 21 | def test_do_dry_run(self): 22 | client = Mock() 23 | client.info.return_value = {'version': {'number': '5.0.0'} } 24 | client.indices.get_settings.return_value = testvars.settings_one 25 | client.cluster.state.return_value = testvars.clu_state_one 26 | client.indices.stats.return_value = testvars.stats_one 27 | client.indices.flush_synced.return_value = testvars.synced_pass 28 | client.indices.close.return_value = None 29 | ilo = curator.IndexList(client) 30 | co = curator.Close(ilo) 31 | self.assertIsNone(co.do_dry_run()) 32 | def test_do_action(self): 33 | client = Mock() 34 | client.info.return_value = {'version': {'number': '5.0.0'} } 35 | client.indices.get_settings.return_value = testvars.settings_one 36 | client.cluster.state.return_value = testvars.clu_state_one 37 | client.indices.stats.return_value = testvars.stats_one 38 | client.indices.flush_synced.return_value = testvars.synced_pass 39 | client.indices.close.return_value = None 40 | ilo = curator.IndexList(client) 41 | co = curator.Close(ilo) 42 | self.assertIsNone(co.do_action()) 43 | def test_do_action_with_delete_aliases(self): 44 | client = Mock() 45 | client.info.return_value = {'version': {'number': '5.0.0'} } 46 | client.indices.get_settings.return_value = testvars.settings_one 47 | client.cluster.state.return_value = testvars.clu_state_one 48 | client.indices.stats.return_value = testvars.stats_one 49 | client.indices.flush_synced.return_value = testvars.synced_pass 50 | client.indices.close.return_value = None 51 | ilo = curator.IndexList(client) 52 | co = curator.Close(ilo, delete_aliases=True) 53 | self.assertIsNone(co.do_action()) 54 | def test_do_action_with_skip_flush(self): 55 | client = Mock() 56 | client.info.return_value = {'version': {'number': '5.0.0'} } 57 | client.indices.get_settings.return_value = testvars.settings_one 58 | client.cluster.state.return_value = testvars.clu_state_one 59 | client.indices.stats.return_value = testvars.stats_one 60 | client.indices.flush_synced.return_value = testvars.synced_pass 61 | client.indices.close.return_value = None 62 | ilo = curator.IndexList(client) 63 | co = curator.Close(ilo, skip_flush=True) 64 | self.assertIsNone(co.do_action()) 65 | def test_do_action_raises_exception(self): 66 | client = Mock() 67 | client.info.return_value = {'version': {'number': '5.0.0'} } 68 | client.indices.get_settings.return_value = testvars.settings_one 69 | client.cluster.state.return_value = testvars.clu_state_one 70 | client.indices.stats.return_value = testvars.stats_one 71 | client.indices.flush_synced.return_value = testvars.synced_pass 72 | client.indices.close.return_value = None 73 | client.indices.close.side_effect = testvars.fake_fail 74 | ilo = curator.IndexList(client) 75 | co = curator.Close(ilo) 76 | self.assertRaises(curator.FailedExecution, co.do_action) 77 | def test_do_action_delete_aliases_with_exception(self): 78 | client = Mock() 79 | client.info.return_value = {'version': {'number': '5.0.0'} } 80 | client.indices.get_settings.return_value = testvars.settings_one 81 | client.cluster.state.return_value = testvars.clu_state_one 82 | client.indices.stats.return_value = testvars.stats_one 83 | client.indices.flush_synced.return_value = testvars.synced_pass 84 | client.indices.close.return_value = None 85 | ilo = curator.IndexList(client) 86 | client.indices.delete_alias.side_effect = testvars.fake_fail 87 | co = curator.Close(ilo, delete_aliases=True) 88 | self.assertIsNone(co.do_action()) 89 | -------------------------------------------------------------------------------- /test/unit/test_action_clusterrouting.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import Mock, patch 3 | import opensearchpy 4 | import curator 5 | # Get test variables and constants from a single source 6 | from . import testvars as testvars 7 | 8 | class TestActionAllocation(TestCase): 9 | def test_bad_client(self): 10 | self.assertRaises(TypeError, curator.ClusterRouting, 'invalid', setting='enable') 11 | def test_bad_setting(self): 12 | client = Mock() 13 | self.assertRaises( 14 | ValueError, curator.ClusterRouting, client, setting='invalid' 15 | ) 16 | def test_bad_routing_type(self): 17 | client = Mock() 18 | self.assertRaises( 19 | ValueError, 20 | curator.ClusterRouting, 21 | client, 22 | routing_type='invalid', 23 | setting='enable' 24 | ) 25 | def test_bad_value_with_allocation(self): 26 | client = Mock() 27 | self.assertRaises( 28 | ValueError, 29 | curator.ClusterRouting, 30 | client, 31 | routing_type='allocation', 32 | setting='enable', 33 | value='invalid' 34 | ) 35 | def test_bad_value_with_rebalance(self): 36 | client = Mock() 37 | self.assertRaises( 38 | ValueError, 39 | curator.ClusterRouting, 40 | client, 41 | routing_type='rebalance', 42 | setting='enable', 43 | value='invalid' 44 | ) 45 | def test_do_dry_run(self): 46 | client = Mock() 47 | cro = curator.ClusterRouting( 48 | client, 49 | routing_type='allocation', 50 | setting='enable', 51 | value='all' 52 | ) 53 | self.assertIsNone(cro.do_dry_run()) 54 | def test_do_action_raise_on_put_settings(self): 55 | client = Mock() 56 | client.cluster.put_settings.return_value = None 57 | client.cluster.put_settings.side_effect = testvars.fake_fail 58 | cro = curator.ClusterRouting( 59 | client, 60 | routing_type='allocation', 61 | setting='enable', 62 | value='all' 63 | ) 64 | self.assertRaises(Exception, cro.do_action) 65 | def test_do_action_wait(self): 66 | client = Mock() 67 | client.cluster.put_settings.return_value = None 68 | client.cluster.health.return_value = {'relocating_shards':0} 69 | cro = curator.ClusterRouting( 70 | client, 71 | routing_type='allocation', 72 | setting='enable', 73 | value='all', 74 | wait_for_completion=True 75 | ) 76 | self.assertIsNone(cro.do_action()) 77 | -------------------------------------------------------------------------------- /test/unit/test_action_create_index.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import Mock, patch 3 | import opensearchpy 4 | import curator 5 | # Get test variables and constants from a single source 6 | from . import testvars as testvars 7 | 8 | class TestActionCreate_index(TestCase): 9 | def test_init_raise(self): 10 | self.assertRaises(TypeError, curator.CreateIndex, 'invalid') 11 | def test_init_raise_no_name(self): 12 | client = Mock() 13 | self.assertRaises(curator.ConfigurationError, 14 | curator.CreateIndex, client, None) 15 | def test_init(self): 16 | client = Mock() 17 | co = curator.CreateIndex(client, 'name') 18 | self.assertEqual('name', co.name) 19 | self.assertEqual(client, co.client) 20 | def test_do_dry_run(self): 21 | client = Mock() 22 | co = curator.CreateIndex(client, 'name') 23 | self.assertIsNone(co.do_dry_run()) 24 | def test_do_action(self): 25 | client = Mock() 26 | client.indices.create.return_value = None 27 | co = curator.CreateIndex(client, 'name') 28 | self.assertIsNone(co.do_action()) 29 | def test_do_action_raises_exception(self): 30 | client = Mock() 31 | client.indices.create.return_value = None 32 | client.indices.create.side_effect = testvars.fake_fail 33 | co = curator.CreateIndex(client, 'name') 34 | self.assertRaises(curator.FailedExecution, co.do_action) 35 | -------------------------------------------------------------------------------- /test/unit/test_action_delete_indices.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import Mock, patch 3 | import opensearchpy 4 | import curator 5 | # Get test variables and constants from a single source 6 | from . import testvars as testvars 7 | 8 | class TestActionDeleteIndices(TestCase): 9 | def test_init_raise(self): 10 | self.assertRaises(TypeError, curator.DeleteIndices, 'invalid') 11 | def test_init_raise_bad_master_timeout(self): 12 | client = Mock() 13 | client.info.return_value = {'version': {'number': '5.0.0'} } 14 | client.indices.get_settings.return_value = testvars.settings_one 15 | client.cluster.state.return_value = testvars.clu_state_one 16 | client.indices.stats.return_value = testvars.stats_one 17 | ilo = curator.IndexList(client) 18 | self.assertRaises(TypeError, curator.DeleteIndices, ilo, 'invalid') 19 | def test_init(self): 20 | client = Mock() 21 | client.info.return_value = {'version': {'number': '5.0.0'} } 22 | client.indices.get_settings.return_value = testvars.settings_one 23 | client.cluster.state.return_value = testvars.clu_state_one 24 | client.indices.stats.return_value = testvars.stats_one 25 | ilo = curator.IndexList(client) 26 | do = curator.DeleteIndices(ilo) 27 | self.assertEqual(ilo, do.index_list) 28 | self.assertEqual(client, do.client) 29 | def test_do_dry_run(self): 30 | client = Mock() 31 | client.info.return_value = {'version': {'number': '5.0.0'} } 32 | client.indices.get_settings.return_value = testvars.settings_four 33 | client.cluster.state.return_value = testvars.clu_state_four 34 | client.indices.stats.return_value = testvars.stats_four 35 | client.indices.delete.return_value = None 36 | ilo = curator.IndexList(client) 37 | do = curator.DeleteIndices(ilo) 38 | self.assertIsNone(do.do_dry_run()) 39 | def test_do_action(self): 40 | client = Mock() 41 | client.info.return_value = {'version': {'number': '5.0.0'} } 42 | client.indices.get_settings.return_value = testvars.settings_four 43 | client.cluster.state.return_value = testvars.clu_state_four 44 | client.indices.stats.return_value = testvars.stats_four 45 | client.indices.delete.return_value = None 46 | ilo = curator.IndexList(client) 47 | do = curator.DeleteIndices(ilo) 48 | self.assertIsNone(do.do_action()) 49 | def test_do_action_not_successful(self): 50 | client = Mock() 51 | client.info.return_value = {'version': {'number': '5.0.0'} } 52 | client.indices.get_settings.return_value = testvars.settings_four 53 | client.cluster.state.return_value = testvars.clu_state_four 54 | client.indices.stats.return_value = testvars.stats_four 55 | client.indices.delete.return_value = None 56 | ilo = curator.IndexList(client) 57 | do = curator.DeleteIndices(ilo) 58 | self.assertIsNone(do.do_action()) 59 | def test_do_action_raises_exception(self): 60 | client = Mock() 61 | client.info.return_value = {'version': {'number': '5.0.0'} } 62 | client.indices.get_settings.return_value = testvars.settings_four 63 | client.cluster.state.return_value = testvars.clu_state_four 64 | client.indices.stats.return_value = testvars.stats_four 65 | client.indices.delete.return_value = None 66 | client.indices.delete.side_effect = testvars.fake_fail 67 | ilo = curator.IndexList(client) 68 | do = curator.DeleteIndices(ilo) 69 | self.assertRaises(curator.FailedExecution, do.do_action) 70 | def test_verify_result_positive(self): 71 | client = Mock() 72 | client.info.return_value = {'version': {'number': '5.0.0'} } 73 | client.indices.get_settings.return_value = testvars.settings_four 74 | client.cluster.state.return_value = testvars.clu_state_four 75 | client.indices.stats.return_value = testvars.stats_four 76 | client.indices.delete.return_value = None 77 | ilo = curator.IndexList(client) 78 | do = curator.DeleteIndices(ilo) 79 | self.assertTrue(do._verify_result([],2)) 80 | -------------------------------------------------------------------------------- /test/unit/test_action_delete_snapshots.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import Mock, patch 3 | import opensearchpy 4 | import curator 5 | # Get test variables and constants from a single source 6 | from . import testvars as testvars 7 | 8 | class TestActionDeleteSnapshots(TestCase): 9 | def test_init_raise(self): 10 | self.assertRaises(TypeError, curator.DeleteSnapshots, 'invalid') 11 | def test_init(self): 12 | client = Mock() 13 | client.snapshot.get.return_value = testvars.snapshots 14 | client.snapshot.get_repository.return_value = testvars.test_repo 15 | slo = curator.SnapshotList(client, repository=testvars.repo_name) 16 | do = curator.DeleteSnapshots(slo) 17 | self.assertEqual(slo, do.snapshot_list) 18 | self.assertEqual(client, do.client) 19 | def test_do_dry_run(self): 20 | client = Mock() 21 | client.snapshot.get.return_value = testvars.snapshots 22 | client.snapshot.get_repository.return_value = testvars.test_repo 23 | client.tasks.get.return_value = testvars.no_snap_tasks 24 | client.snapshot.delete.return_value = None 25 | slo = curator.SnapshotList(client, repository=testvars.repo_name) 26 | do = curator.DeleteSnapshots(slo) 27 | self.assertIsNone(do.do_dry_run()) 28 | def test_do_action(self): 29 | client = Mock() 30 | client.snapshot.get.return_value = testvars.snapshots 31 | client.snapshot.get_repository.return_value = testvars.test_repo 32 | client.tasks.list.return_value = testvars.no_snap_tasks 33 | client.snapshot.delete.return_value = None 34 | slo = curator.SnapshotList(client, repository=testvars.repo_name) 35 | do = curator.DeleteSnapshots(slo) 36 | self.assertIsNone(do.do_action()) 37 | def test_do_action_raises_exception(self): 38 | client = Mock() 39 | client.snapshot.get.return_value = testvars.snapshots 40 | client.snapshot.get_repository.return_value = testvars.test_repo 41 | client.snapshot.delete.return_value = None 42 | client.tasks.list.return_value = testvars.no_snap_tasks 43 | client.snapshot.delete.side_effect = testvars.fake_fail 44 | slo = curator.SnapshotList(client, repository=testvars.repo_name) 45 | do = curator.DeleteSnapshots(slo) 46 | self.assertRaises(curator.FailedExecution, do.do_action) 47 | def test_not_safe_to_snap_raises_exception(self): 48 | client = Mock() 49 | client.snapshot.get.return_value = testvars.inprogress 50 | client.snapshot.get_repository.return_value = testvars.test_repo 51 | client.tasks.list.return_value = testvars.no_snap_tasks 52 | slo = curator.SnapshotList(client, repository=testvars.repo_name) 53 | do = curator.DeleteSnapshots(slo, retry_interval=0, retry_count=1) 54 | self.assertRaises(curator.FailedExecution, do.do_action) 55 | -------------------------------------------------------------------------------- /test/unit/test_action_forcemerge.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import Mock, patch 3 | import opensearchpy 4 | import curator 5 | # Get test variables and constants from a single source 6 | from . import testvars as testvars 7 | 8 | class TestActionForceMerge(TestCase): 9 | def test_init_raise_bad_client(self): 10 | self.assertRaises( 11 | TypeError, curator.ForceMerge, 'invalid', max_num_segments=2) 12 | def test_init_raise_no_segment_count(self): 13 | client = Mock() 14 | client.info.return_value = {'version': {'number': '5.0.0'} } 15 | client.indices.get_settings.return_value = testvars.settings_one 16 | client.cluster.state.return_value = testvars.clu_state_one 17 | client.indices.stats.return_value = testvars.stats_one 18 | client.indices.segments.return_value = testvars.shards 19 | ilo = curator.IndexList(client) 20 | self.assertRaises( 21 | curator.MissingArgument, curator.ForceMerge, ilo) 22 | def test_init(self): 23 | client = Mock() 24 | client.info.return_value = {'version': {'number': '5.0.0'} } 25 | client.indices.get_settings.return_value = testvars.settings_one 26 | client.cluster.state.return_value = testvars.clu_state_one 27 | client.indices.stats.return_value = testvars.stats_one 28 | client.indices.segments.return_value = testvars.shards 29 | ilo = curator.IndexList(client) 30 | fmo = curator.ForceMerge(ilo, max_num_segments=2) 31 | self.assertEqual(ilo, fmo.index_list) 32 | self.assertEqual(client, fmo.client) 33 | def test_do_dry_run(self): 34 | client = Mock() 35 | client.info.return_value = {'version': {'number': '5.0.0'} } 36 | client.indices.get_settings.return_value = testvars.settings_one 37 | client.cluster.state.return_value = testvars.clu_state_one 38 | client.indices.stats.return_value = testvars.stats_one 39 | client.indices.segments.return_value = testvars.shards 40 | client.indices.forcemerge.return_value = None 41 | client.indices.optimize.return_value = None 42 | ilo = curator.IndexList(client) 43 | fmo = curator.ForceMerge(ilo, max_num_segments=2) 44 | self.assertIsNone(fmo.do_dry_run()) 45 | def test_do_action_pre5(self): 46 | client = Mock() 47 | client.info.return_value = {'version': {'number': '5.0.0'} } 48 | client.indices.get_settings.return_value = testvars.settings_one 49 | client.cluster.state.return_value = testvars.clu_state_one 50 | client.indices.stats.return_value = testvars.stats_one 51 | client.indices.segments.return_value = testvars.shards 52 | client.info.return_value = {'version': {'number': '2.3.2'} } 53 | client.indices.optimize.return_value = None 54 | ilo = curator.IndexList(client) 55 | fmo = curator.ForceMerge(ilo, max_num_segments=2) 56 | self.assertIsNone(fmo.do_action()) 57 | def test_do_action(self): 58 | client = Mock() 59 | client.info.return_value = {'version': {'number': '5.0.0'} } 60 | client.indices.get_settings.return_value = testvars.settings_one 61 | client.cluster.state.return_value = testvars.clu_state_one 62 | client.indices.stats.return_value = testvars.stats_one 63 | client.indices.segments.return_value = testvars.shards 64 | client.info.return_value = {'version': {'number': '5.0.0'} } 65 | client.indices.forcemerge.return_value = None 66 | ilo = curator.IndexList(client) 67 | fmo = curator.ForceMerge(ilo, max_num_segments=2) 68 | self.assertIsNone(fmo.do_action()) 69 | def test_do_action_with_delay(self): 70 | client = Mock() 71 | client.info.return_value = {'version': {'number': '5.0.0'} } 72 | client.indices.get_settings.return_value = testvars.settings_one 73 | client.cluster.state.return_value = testvars.clu_state_one 74 | client.indices.stats.return_value = testvars.stats_one 75 | client.indices.segments.return_value = testvars.shards 76 | client.info.return_value = {'version': {'number': '5.0.0'} } 77 | client.indices.forcemerge.return_value = None 78 | ilo = curator.IndexList(client) 79 | fmo = curator.ForceMerge(ilo, max_num_segments=2, delay=0.050) 80 | self.assertIsNone(fmo.do_action()) 81 | def test_do_action_raises_exception(self): 82 | client = Mock() 83 | client.info.return_value = {'version': {'number': '5.0.0'} } 84 | client.indices.get_settings.return_value = testvars.settings_one 85 | client.cluster.state.return_value = testvars.clu_state_one 86 | client.indices.stats.return_value = testvars.stats_one 87 | client.indices.segments.return_value = testvars.shards 88 | client.indices.forcemerge.return_value = None 89 | client.indices.optimize.return_value = None 90 | client.indices.forcemerge.side_effect = testvars.fake_fail 91 | client.indices.optimize.side_effect = testvars.fake_fail 92 | ilo = curator.IndexList(client) 93 | fmo = curator.ForceMerge(ilo, max_num_segments=2) 94 | self.assertRaises(curator.FailedExecution, fmo.do_action) 95 | -------------------------------------------------------------------------------- /test/unit/test_action_open.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import Mock, patch 3 | import opensearchpy 4 | import curator 5 | # Get test variables and constants from a single source 6 | from . import testvars as testvars 7 | 8 | class TestActionOpen(TestCase): 9 | def test_init_raise(self): 10 | self.assertRaises(TypeError, curator.Open, 'invalid') 11 | def test_init(self): 12 | client = Mock() 13 | client.info.return_value = {'version': {'number': '5.0.0'} } 14 | client.indices.get_settings.return_value = testvars.settings_one 15 | client.cluster.state.return_value = testvars.clu_state_one 16 | client.indices.stats.return_value = testvars.stats_one 17 | ilo = curator.IndexList(client) 18 | oo = curator.Open(ilo) 19 | self.assertEqual(ilo, oo.index_list) 20 | self.assertEqual(client, oo.client) 21 | def test_do_dry_run(self): 22 | client = Mock() 23 | client.info.return_value = {'version': {'number': '5.0.0'} } 24 | client.indices.get_settings.return_value = testvars.settings_four 25 | client.cluster.state.return_value = testvars.clu_state_four 26 | client.indices.stats.return_value = testvars.stats_four 27 | client.indices.open.return_value = None 28 | ilo = curator.IndexList(client) 29 | ilo.filter_opened() 30 | oo = curator.Open(ilo) 31 | self.assertEqual([u'c-2016.03.05'], oo.index_list.indices) 32 | self.assertIsNone(oo.do_dry_run()) 33 | def test_do_action(self): 34 | client = Mock() 35 | client.info.return_value = {'version': {'number': '5.0.0'} } 36 | client.indices.get_settings.return_value = testvars.settings_four 37 | client.cluster.state.return_value = testvars.clu_state_four 38 | client.indices.stats.return_value = testvars.stats_four 39 | client.indices.open.return_value = None 40 | ilo = curator.IndexList(client) 41 | ilo.filter_opened() 42 | oo = curator.Open(ilo) 43 | self.assertEqual([u'c-2016.03.05'], oo.index_list.indices) 44 | self.assertIsNone(oo.do_action()) 45 | def test_do_action_raises_exception(self): 46 | client = Mock() 47 | client.info.return_value = {'version': {'number': '5.0.0'} } 48 | client.indices.get_settings.return_value = testvars.settings_four 49 | client.cluster.state.return_value = testvars.clu_state_four 50 | client.indices.stats.return_value = testvars.stats_four 51 | client.indices.open.return_value = None 52 | client.indices.open.side_effect = testvars.fake_fail 53 | ilo = curator.IndexList(client) 54 | oo = curator.Open(ilo) 55 | self.assertRaises(curator.FailedExecution, oo.do_action) 56 | -------------------------------------------------------------------------------- /test/unit/test_action_replicas.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import Mock, patch 3 | import opensearchpy 4 | import curator 5 | # Get test variables and constants from a single source 6 | from . import testvars as testvars 7 | 8 | class TestActionReplicas(TestCase): 9 | def test_init_raise_bad_client(self): 10 | self.assertRaises( 11 | TypeError, curator.Replicas, 'invalid', count=2) 12 | def test_init_raise_no_count(self): 13 | client = Mock() 14 | client.info.return_value = {'version': {'number': '5.0.0'} } 15 | client.indices.get_settings.return_value = testvars.settings_one 16 | client.cluster.state.return_value = testvars.clu_state_one 17 | client.indices.stats.return_value = testvars.stats_one 18 | ilo = curator.IndexList(client) 19 | self.assertRaises( 20 | curator.MissingArgument, curator.Replicas, ilo) 21 | def test_init(self): 22 | client = Mock() 23 | client.info.return_value = {'version': {'number': '5.0.0'} } 24 | client.indices.get_settings.return_value = testvars.settings_one 25 | client.cluster.state.return_value = testvars.clu_state_one 26 | client.indices.stats.return_value = testvars.stats_one 27 | client.indices.put_settings.return_value = None 28 | ilo = curator.IndexList(client) 29 | ro = curator.Replicas(ilo, count=2) 30 | self.assertEqual(ilo, ro.index_list) 31 | self.assertEqual(client, ro.client) 32 | def test_do_dry_run(self): 33 | client = Mock() 34 | client.info.return_value = {'version': {'number': '5.0.0'} } 35 | client.indices.get_settings.return_value = testvars.settings_one 36 | client.cluster.state.return_value = testvars.clu_state_one 37 | client.indices.stats.return_value = testvars.stats_one 38 | client.indices.put_settings.return_value = None 39 | ilo = curator.IndexList(client) 40 | ro = curator.Replicas(ilo, count=0) 41 | self.assertIsNone(ro.do_dry_run()) 42 | def test_do_action(self): 43 | client = Mock() 44 | client.info.return_value = {'version': {'number': '5.0.0'} } 45 | client.indices.get_settings.return_value = testvars.settings_one 46 | client.cluster.state.return_value = testvars.clu_state_one 47 | client.indices.stats.return_value = testvars.stats_one 48 | client.indices.put_settings.return_value = None 49 | ilo = curator.IndexList(client) 50 | ro = curator.Replicas(ilo, count=0) 51 | self.assertIsNone(ro.do_action()) 52 | def test_do_action_wait(self): 53 | client = Mock() 54 | client.info.return_value = {'version': {'number': '5.0.0'} } 55 | client.indices.get_settings.return_value = testvars.settings_one 56 | client.cluster.state.return_value = testvars.clu_state_one 57 | client.indices.stats.return_value = testvars.stats_one 58 | client.indices.put_settings.return_value = None 59 | client.cluster.health.return_value = {'status':'green'} 60 | ilo = curator.IndexList(client) 61 | ro = curator.Replicas(ilo, count=1, wait_for_completion=True) 62 | self.assertIsNone(ro.do_action()) 63 | def test_do_action_raises_exception(self): 64 | client = Mock() 65 | client.info.return_value = {'version': {'number': '5.0.0'} } 66 | client.indices.get_settings.return_value = testvars.settings_one 67 | client.cluster.state.return_value = testvars.clu_state_one 68 | client.indices.stats.return_value = testvars.stats_one 69 | client.indices.segments.return_value = testvars.shards 70 | client.indices.put_settings.return_value = None 71 | client.indices.put_settings.side_effect = testvars.fake_fail 72 | ilo = curator.IndexList(client) 73 | ro = curator.Replicas(ilo, count=2) 74 | self.assertRaises(curator.FailedExecution, ro.do_action) 75 | -------------------------------------------------------------------------------- /test/unit/test_action_rollover.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from mock import Mock, patch 3 | import opensearchpy 4 | import curator 5 | # Get test variables and constants from a single source 6 | from . import testvars as testvars 7 | 8 | class TestActionRollover(TestCase): 9 | def test_init_raise_bad_client(self): 10 | self.assertRaises( 11 | TypeError, curator.Rollover, 'invalid', 'name', {}) 12 | def test_init_raise_bad_conditions(self): 13 | client = Mock() 14 | client.info.return_value = {'version': {'number': '5.0.0'} } 15 | self.assertRaises( 16 | curator.ConfigurationError, curator.Rollover, client, 'name', 'string') 17 | def test_init_raise_bad_extra_settings(self): 18 | client = Mock() 19 | client.info.return_value = {'version': {'number': '5.0.0'} } 20 | self.assertRaises( 21 | curator.ConfigurationError, curator.Rollover, client, 'name', 22 | {'a':'b'}, None, 'string') 23 | def test_init_raise_non_rollable_index(self): 24 | client = Mock() 25 | client.info.return_value = {'version': {'number': '5.0.0'} } 26 | client.indices.get_alias.return_value = testvars.alias_retval 27 | self.assertRaises( 28 | ValueError, curator.Rollover, client, testvars.named_alias, 29 | {'a':'b'}) 30 | def test_do_dry_run(self): 31 | client = Mock() 32 | client.info.return_value = {'version': {'number': '5.0.0'} } 33 | client.indices.get_alias.return_value = testvars.rollable_alias 34 | client.indices.rollover.return_value = testvars.dry_run_rollover 35 | ro = curator.Rollover( 36 | client, testvars.named_alias, testvars.rollover_conditions) 37 | self.assertIsNone(ro.do_dry_run()) 38 | def test_init_raise_max_size_on_unsupported_version(self): 39 | client = Mock() 40 | client.info.return_value = {'version': {'number': '6.0.0'} } 41 | client.indices.get_alias.return_value = testvars.rollable_alias 42 | conditions = { 'max_size': '1g' } 43 | self.assertRaises( 44 | curator.ConfigurationError, curator.Rollover, client, 45 | testvars.named_alias, conditions 46 | ) 47 | def test_max_size_in_acceptable_verion(self): 48 | client = Mock() 49 | client.info.return_value = {'version': {'number': '6.1.0'} } 50 | client.indices.get_alias.return_value = testvars.rollable_alias 51 | conditions = { 'max_size': '1g' } 52 | ro = curator.Rollover(client, testvars.named_alias, conditions) 53 | self.assertEqual(conditions, ro.conditions) 54 | -------------------------------------------------------------------------------- /test/unit/test_cli_methods.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import logging 3 | from unittest import TestCase 4 | from mock import Mock, patch, mock_open 5 | import opensearchpy 6 | import curator 7 | from curator import _version as __version__ 8 | from . import CLITestCase 9 | # Get test variables and constants from a single source 10 | from . import testvars as testvars 11 | 12 | class TestCLI_A(TestCase): 13 | def test_read_file_no_file(self): 14 | self.assertRaises(TypeError, curator.read_file) 15 | def test_loginfo_defaults(self): 16 | loginfo = curator.LogInfo({}) 17 | self.assertEqual(20, loginfo.numeric_log_level) 18 | self.assertEqual(testvars.default_format, loginfo.format_string) 19 | def test_loginfo_debug(self): 20 | loginfo = curator.LogInfo({"loglevel": "DEBUG"}) 21 | self.assertEqual(10, loginfo.numeric_log_level) 22 | self.assertEqual(testvars.debug_format, loginfo.format_string) 23 | def test_loginfo_bad_level_raises(self): 24 | self.assertRaises( 25 | ValueError, 26 | curator.LogInfo, {"loglevel": "NOTALOGLEVEL"} 27 | ) 28 | def test_loginfo_logstash_formatter(self): 29 | loginfo = curator.LogInfo({"logformat": "logstash"}) 30 | logging.root.addHandler(loginfo.handler) 31 | logging.root.setLevel(loginfo.numeric_log_level) 32 | logger = logging.getLogger('testing') 33 | logger.info('testing') 34 | self.assertEqual(20, loginfo.numeric_log_level) 35 | def test_client_options_certificate(self): 36 | a = {'use_ssl':True, 'certificate':'invalid_path'} 37 | self.assertRaises( 38 | curator.FailedExecution, 39 | curator.test_client_options, a 40 | ) 41 | def test_client_options_client_cert(self): 42 | a = {'use_ssl':True, 'client_cert':'invalid_path'} 43 | self.assertRaises( 44 | curator.FailedExecution, 45 | curator.test_client_options, a 46 | ) 47 | def test_client_options_client_key(self): 48 | a = {'use_ssl':True, 'client_key':'invalid_path'} 49 | self.assertRaises( 50 | curator.FailedExecution, 51 | curator.test_client_options, a 52 | ) 53 | 54 | class TestCLI_B(CLITestCase): 55 | def test_read_file_pass(self): 56 | cfg = curator.get_yaml(self.args['yamlfile']) 57 | self.assertEqual('localhost', cfg['client']['hosts']) 58 | self.assertEqual(9200, cfg['client']['port']) 59 | def test_read_file_corrupt_fail(self): 60 | with self.assertRaises(SystemExit) as get: 61 | curator.get_yaml(self.args['invalid_yaml']) 62 | self.assertEqual(get.exception.code, 1) 63 | def test_read_file_missing_fail(self): 64 | self.assertRaises( 65 | curator.FailedExecution, 66 | curator.read_file, self.args['no_file_here'] 67 | ) 68 | -------------------------------------------------------------------------------- /travis-run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | # There's at least 1 expected, skipped test, only with 5.0.0-alpha4 right now 5 | expected_skips=1 6 | 7 | setup_es() { 8 | download_url=$1 9 | curl -sL $download_url > elasticsearch.tar.gz 10 | mkdir elasticsearch 11 | tar -xzf elasticsearch.tar.gz --strip-components=1 -C ./elasticsearch/. 12 | } 13 | 14 | start_es() { 15 | jhome=$1 16 | es_args=$2 17 | es_port=$3 18 | es_cluster=$4 19 | export JAVA_HOME=$jhome 20 | elasticsearch/bin/elasticsearch $es_args > /tmp/$es_cluster.log & 21 | sleep 20 22 | curl http://127.0.0.1:$es_port && echo "$es_cluster Elasticsearch is up!" || cat /tmp/$es_cluster.log ./elasticsearch/logs/$es_cluster.log 23 | # curl http://127.0.0.1:$es_port && echo "ES is up!" || cat /tmp/$es_cluster.log ./elasticsearch/logs/$es_cluster.log 24 | } 25 | 26 | start_es6() { 27 | jhome=$1 28 | es_args=$2 29 | path_env=$3 30 | es_port=$4 31 | es_cluster=$5 32 | export JAVA_HOME=$jhome 33 | ES_PATH_CONF=$path_env elasticsearch/bin/elasticsearch $es_args > /tmp/$es_cluster.log & 34 | sleep 20 35 | curl http://127.0.0.1:$es_port && echo "$es_cluster Elasticsearch is up!" || cat /tmp/$es_cluster.log ./elasticsearch/logs/$es_cluster.log 36 | # curl http://127.0.0.1:$es_port && echo "ES is up!" || cat /tmp/$es_cluster.log ./elasticsearch/logs/$es_cluster.log 37 | } 38 | 39 | start_es7() { 40 | es_args=$1 41 | path_env=$2 42 | es_port=$3 43 | es_cluster=$4 44 | ES_PATH_CONF=$path_env elasticsearch/bin/elasticsearch $es_args > /tmp/$es_cluster.log & 45 | sleep 20 46 | curl http://127.0.0.1:$es_port && echo "$es_cluster Elasticsearch is up!" || cat /tmp/$es_cluster.log ./elasticsearch/logs/$es_cluster.log 47 | } 48 | 49 | common_node_settings() { 50 | major=$1 51 | minor=$2 52 | port=$3 53 | clustername=$4 54 | file=$5 55 | echo 'network.host: 127.0.0.1' > $file 56 | echo "http.port: ${port}" >> $file 57 | echo "cluster.name: ${clustername}" >> $file 58 | echo "node.name: ${clustername}" >> $file 59 | echo 'node.max_local_storage_nodes: 2' >> $file 60 | if [[ $major -lt 7 ]]; then 61 | echo "discovery.zen.ping.unicast.hosts: [\"127.0.0.1:${port}\"]" >> $file 62 | else 63 | transport=$(($port+100)) 64 | echo "transport.port: ${transport}" >> $file 65 | echo "discovery.seed_hosts: [\"localhost:${transport}\"]" >> $file 66 | echo "discovery.type: single-node" >> $file 67 | fi 68 | if [[ $major -ge 6 ]] && [[ $minor -ge 3 ]]; then 69 | echo 'xpack.monitoring.enabled: false' >> $file 70 | echo 'node.ml: false' >> $file 71 | echo 'xpack.security.enabled: false' >> $file 72 | echo 'xpack.watcher.enabled: false' >> $file 73 | fi 74 | } 75 | 76 | setup_es https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ES_VERSION.tar.gz 77 | 78 | java_home='/usr/lib/jvm/java-8-openjdk-amd64/jre' 79 | 80 | ## Get major and minor version numbers 81 | MAJORVER=$(echo $ES_VERSION | awk -F\. '{print $1}') 82 | MINORVER=$(echo $ES_VERSION | awk -F\. '{print $2}') 83 | 84 | ### Build local cluster config (since 5.4 removed most flags) 85 | LC=elasticsearch/localcluster 86 | mkdir -p $LC 87 | cp elasticsearch/config/log4j2.properties $LC 88 | cp elasticsearch/config/jvm.options $LC 89 | common_node_settings $MAJORVER $MINORVER 9200 "local" "$LC/elasticsearch.yml" 90 | echo 'path.repo: /' >> $LC/elasticsearch.yml 91 | echo 'reindex.remote.whitelist: localhost:9201' >> $LC/elasticsearch.yml 92 | 93 | 94 | ### Build remote cluster config (since 5.4 removed most flags) 95 | RC=elasticsearch/remotecluster 96 | mkdir -p $RC 97 | cp elasticsearch/config/log4j2.properties $RC 98 | cp elasticsearch/config/jvm.options $RC 99 | common_node_settings $MAJORVER $MINORVER 9201 remote "$RC/elasticsearch.yml" 100 | 101 | if [[ $MAJORVER -lt 6 ]]; then 102 | start_es $java_home "-d -Epath.conf=$LC" 9200 "local" 103 | start_es $java_home "-d -Epath.conf=$RC" 9201 "remote" 104 | elif [[ $MARJORVER -eq 6 ]]; then 105 | start_es6 $java_home " " "$LC" 9200 "local" 106 | start_es6 $java_home " " "$RC" 9201 "remote" 107 | else 108 | start_es7 " " "$LC" 9200 "local" 109 | start_es7 " " "$RC" 9201 "remote" 110 | fi 111 | 112 | python setup.py test 113 | result=$(head -1 nosetests.xml | awk '{print $6 " " $7 " " $8}' | awk -F\> '{print $1}' | tr -d '"') 114 | echo "Result = $result" 115 | errors=$(echo $result | awk '{print $1}' | awk -F\= '{print $2}') 116 | failures=$(echo $result | awk '{print $2}' | awk -F\= '{print $2}') 117 | skips=$(echo $result | awk '{print $3}' | awk -F\= '{print $2}') 118 | if [[ $errors -gt 0 ]]; then 119 | exit 1 120 | elif [[ $failures -gt 0 ]]; then 121 | exit 1 122 | elif [[ $skips -gt $expected_skips ]]; then 123 | exit 1 124 | fi 125 | -------------------------------------------------------------------------------- /unix_packages/README.md: -------------------------------------------------------------------------------- 1 | # README 2 | 3 | ## Docker Package Building for Linux Systems 4 | 5 | The image is based on CentOS 6 (yes, I know it's EOL--this preserves it 6 | for the people still using it, and the resulting binary runs nearly everywhere due 7 | to it's ancient LIBC version). 8 | 9 | ### Update/edit the image 10 | 11 | If you edit the `Dockerfile`, you will see: 12 | 13 | ``` 14 | # Can change these 15 | ENV PYVER=3.9 16 | ENV PYPATCH=4 17 | ENV OPENSSL_VER=1.1.1k 18 | ``` 19 | 20 | You should be able to edit these values with whatever is available at the respective 21 | sites, python.org and openssl.org. 22 | 23 | ### Build the image 24 | To build the image, `cd` to the `builder_image` directory and run: 25 | 26 | ``` 27 | docker build -t curator_builder:latest . 28 | ``` 29 | 30 | ### Build packages 31 | 32 | The following will check out the tag identified by `tags/vX.Y.Z` from the Curator 33 | Github repository and build package which will be deposited in the present working directory. 34 | 35 | ``` 36 | docker run --rm -v $(pwd):/curator_packages curator_builder /package_maker.sh X.Y.Z 37 | ``` 38 | 39 | The result will be an RPM package named: 40 | 41 | ``` 42 | elasticsearch-curator-X.Y.Z-1.x86_64.rpm 43 | ``` 44 | 45 | and a DEB package named: 46 | 47 | ``` 48 | elasticsearch-curator_X.Y.Z_amd64.deb 49 | ``` 50 | 51 | These packages were tested in CentOS 6 & 7; Ubuntu 1404, 1604, and 1804; and Debian 8 & 9. 52 | 53 | ## Publishing of Packages 54 | 55 | This process is used to create packages which are subsequently signed with Elastic's key 56 | and published to the repositories identified in the Curator documentation. 57 | -------------------------------------------------------------------------------- /unix_packages/builder_image/CentOS-Base.repo: -------------------------------------------------------------------------------- 1 | [base] 2 | name=CentOS-$releasever - Base 3 | # mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os&infra=$infra 4 | # baseurl=http://mirror.centos.org/centos/$releasever/os/$basearch/ 5 | baseurl=https://vault.centos.org/6.10/os/$basearch/ 6 | gpgcheck=1 7 | gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6 8 | 9 | # released updates 10 | [updates] 11 | name=CentOS-$releasever - Updates 12 | # mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=updates&infra=$infra 13 | # baseurl=http://mirror.centos.org/centos/$releasever/updates/$basearch/ 14 | baseurl=https://vault.centos.org/6.10/updates/$basearch/ 15 | gpgcheck=1 16 | gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6 17 | 18 | # additional packages that may be useful 19 | [extras] 20 | name=CentOS-$releasever - Extras 21 | # mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=extras&infra=$infra 22 | # baseurl=http://mirror.centos.org/centos/$releasever/extras/$basearch/ 23 | baseurl=https://vault.centos.org/6.10/extras/$basearch/ 24 | gpgcheck=1 25 | gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6 26 | -------------------------------------------------------------------------------- /unix_packages/builder_image/Dockerfile: -------------------------------------------------------------------------------- 1 | ## If needing Debian style, uncomment this, and comment the CentOS ones 2 | #FROM ubuntu:latest as builder 3 | 4 | FROM centos:6 as builder 5 | 6 | # Can change these 7 | ENV PYVER=3.9 8 | ENV PYPATCH=4 9 | ENV OPENSSL_VER=1.1.1k 10 | 11 | # Don't change these 12 | ENV PKG_TARGET=/curator_packages 13 | ENV WORKDIR=/tmp/curator 14 | ENV VENVDIR=/opt/python 15 | ENV SRCDIR=/opt/src 16 | ENV INPUT_TYPE=python 17 | ENV CATEGORY=python 18 | ENV VENDOR=Elastic 19 | ENV MAINTAINER="'Elastic Developers '" 20 | ENV C_POST_INSTALL=${WORKDIR}/es_curator_after_install.sh 21 | ENV C_PRE_REMOVE=${WORKDIR}/es_curator_before_removal.sh 22 | ENV C_POST_REMOVE=${WORKDIR}/es_curator_after_removal.sh 23 | ENV C_PRE_UPGRADE=${WORKDIR}/es_curator_before_upgrade.sh 24 | ENV C_POST_UPGRADE=${WORKDIR}/es_curator_after_upgrade.sh 25 | 26 | ## If running Debian-style, uncomment these and comment the CentOS ones 27 | #RUN apt update 28 | #RUN apt dist-upgrade -y 29 | #RUN ln -fs /usr/share/zoneinfo/America/Denver /etc/localtime 30 | #RUN DEBIAN_FRONTEND=noninteractive apt install -y tzdata 31 | #RUN apt install -y build-essential git ca-certificates zlib1g zlib1g-dev libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev libgdbm-dev libc6-dev libbz2-dev dirmngr curl liblzma-dev libffi-dev gnupg2 rpm 32 | 33 | ## CentOS 6 covers all bases with its ANCIENT libc 34 | # This replaces the base yum repo definition since v6 is EOL now 35 | # It won't work without it 36 | COPY CentOS-Base.repo /etc/yum.repos.d 37 | RUN yum -y groupinstall "Development Tools" 38 | RUN yum -y install zlib-devel bzip2-devel curl-devel expat-devel gettext-devel sqlite-devel openssl-devel curl wget ncurses-devel readline-devel gdbm-devel xz-devel libffi-devel libuuid-devel which 39 | 40 | # Build patchelf 41 | COPY patchelf_build.sh / 42 | RUN /patchelf_build.sh 43 | 44 | # Build newer Git 45 | COPY git_build.sh / 46 | RUN /git_build.sh 47 | 48 | # Build OpenSSL 49 | RUN curl -O https://www.openssl.org/source/openssl-${OPENSSL_VER}.tar.gz 50 | RUN tar zxf openssl-${OPENSSL_VER}.tar.gz 51 | RUN cd openssl-${OPENSSL_VER}; ./config --prefix=/usr/local/openssl --openssldir=/usr/local/openssl shared zlib; make; make install 52 | RUN echo "# /etc/profile.d/openssl.sh" > /etc/profile.d/openssl.sh 53 | RUN echo "pathmunge /usr/local/openssl/bin" >> /etc/profile.d/openssl.sh 54 | RUN echo "# /etc/ld.so/conf.d/openssl-${OPENSSL_VER}.conf" > /etc/ld.so.conf.d/openssl-${OPENSSL_VER}.conf 55 | RUN echo "/usr/local/openssl/lib" >> /etc/ld.so.conf.d/openssl-${OPENSSL_VER}.conf 56 | RUN ldconfig 57 | RUN rm -rf openssl-${OPENSSL_VER} openssl-${OPENSSL_VER}.tar.gz 58 | 59 | # Build Python 60 | RUN curl -O https://www.python.org/ftp/python/${PYVER}.${PYPATCH}/Python-${PYVER}.${PYPATCH}.tgz 61 | RUN tar zxf Python-${PYVER}.${PYPATCH}.tgz 62 | RUN cd Python-${PYVER}.${PYPATCH}; ./configure --prefix=/usr/local --with-openssl=/usr/local/openssl --enable-optimizations --enable-shared; make -j3 altinstall 63 | RUN echo "# /etc/ld.so.conf.d/python${PYVER}.${PYPATCH}.conf" > /etc/ld.so.conf.d/python${PYVER}.${PYPATCH}.conf 64 | RUN echo "/usr/local/lib" >> /etc/ld.so.conf.d/python${PYVER}.${PYPATCH}.conf 65 | RUN echo "/usr/local/lib/python${PYVER}" >> /etc/ld.so.conf.d/python${PYVER}.${PYPATCH}.conf 66 | RUN ldconfig 67 | RUN /usr/local/bin/pip${PYVER} install virtualenv 68 | RUN rm -rf Python-${PYVER}.${PYPATCH}.tgz Python-${PYVER}.${PYPATCH} 69 | 70 | # Install RVM 71 | COPY ruby_build.sh / 72 | COPY rpm.erb.patched / 73 | RUN /ruby_build.sh 74 | 75 | # Cleanup after all this installation 76 | RUN yum clean all 77 | 78 | COPY package_maker.sh / 79 | RUN mkdir /curator_packages 80 | -------------------------------------------------------------------------------- /unix_packages/builder_image/git_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | yum install -y perl-ExtUtils-MakeMaker 4 | cd 5 | yum remove -y git 6 | wget https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.31.1.tar.gz 7 | tar zxf git-2.31.1.tar.gz 8 | cd git-2.31.1 9 | make prefix=/usr all 10 | make prefix=/usr install 11 | 12 | cd .. 13 | rm -rf git-2.31.* -------------------------------------------------------------------------------- /unix_packages/builder_image/package_maker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Build our own package pre/post scripts 4 | rm -rf ${WORKDIR} ${VENVDIR} ${SRCDIR} /opt/elasticsearch-curator 5 | mkdir -p ${WORKDIR} 6 | 7 | #/usr/local/bin/python?.? --version 8 | #Python 3.9.4 9 | RAWPYVER=$(/usr/local/bin/python?.? --version | awk '{print $2}') 10 | PYVER=$(echo $RAWPYVER | awk -F\. '{print $1"."$2}') 11 | MINOR=$(echo $RAWPYVER | awk -F\. '{print $3}') 12 | 13 | for file in ${C_POST_INSTALL} ${C_PRE_REMOVE} ${C_POST_REMOVE}; do 14 | echo '#!/bin/bash' > ${file} 15 | echo >> ${file} 16 | chmod +x ${file} 17 | done 18 | 19 | for binary_name in curator curator_cli es_repo_mgr; do 20 | for script_target in ${C_POST_INSTALL} ${C_POST_UPGRADE}; do 21 | echo "echo '#!/bin/bash' > /usr/bin/${binary_name}" >> ${script_target} 22 | echo "echo >> /usr/bin/${binary_name}" >> ${script_target} 23 | echo "echo -n LD_LIBRARY_PATH=/opt/elasticsearch-curator/lib /opt/elasticsearch-curator/${binary_name} >> /usr/bin/${binary_name}" >> ${script_target} 24 | echo "echo ' \"\$@\"' >> /usr/bin/${binary_name}" >> ${script_target} 25 | echo "chmod +x /usr/bin/${binary_name}" >> ${script_target} 26 | done 27 | for script_target in ${C_PRE_REMOVE} ${C_PRE_UPGRADE}; do 28 | echo "rm -f /usr/bin/${binary_name}" >> ${script_target} 29 | echo "rm -f /etc/ld.so.conf.d/elasticsearch-curator.conf" >> ${script_target} 30 | echo "ldconfig" >> ${script_target} 31 | done 32 | done 33 | 34 | echo 'if [ -d "/opt/elasticsearch-curator" ]; then' >> ${C_POST_REMOVE} 35 | echo ' rm -rf /opt/elasticsearch-curator' >> ${C_POST_REMOVE} 36 | echo 'fi' >> ${C_POST_REMOVE} 37 | 38 | # build 39 | if [ "${1}x" == "x" ]; then 40 | echo "Must provide version number (can be arbitrary)" 41 | exit 1 42 | else 43 | mkdir -p ${SRCDIR} 44 | cd ${SRCDIR} 45 | git clone https://github.com/elastic/curator.git 46 | cd curator 47 | git fetch --all --tags 48 | git checkout tags/v${1} -b mybuild_${1} 49 | RESPONSE=$? 50 | if [ $RESPONSE -ne 0 ]; then 51 | RESPONSE=0 52 | echo "tags/v${1} not found!" 53 | echo "Checking for tags/V${1}..." 54 | git checkout tags/V${1} -b mybuild_${1} 55 | RESPONSE=$? 56 | fi 57 | if [ $RESPONSE -ne 0 ]; then 58 | echo "Unable to checkout remote tag v${1}" 59 | exit 1 60 | fi 61 | GIT_PATH=$(pwd) 62 | fi 63 | 64 | if [ -e "/usr/local/rvm/scripts/rvm" ]; then 65 | source /usr/local/rvm/scripts/rvm 66 | fi 67 | 68 | 69 | # Build virtualenv 70 | mkdir -p ${VENVDIR}/${PYVER}.${MINOR} 71 | cd ${VENVDIR} 72 | /usr/local/bin/virtualenv ${PYVER}.${MINOR} 73 | source ${VENVDIR}/${PYVER}.${MINOR}/bin/activate 74 | pip install cx_freeze patchelf-wrapper 75 | 76 | # Install pre-requisites 77 | cd ${GIT_PATH} 78 | pip install -r requirements.txt 79 | python setup.py build_exe 80 | 81 | mv build/exe.linux-x86_64-${PYVER} /opt/elasticsearch-curator 82 | chown -R root:root /opt/elasticsearch-curator 83 | cd $WORKDIR 84 | 85 | for pkgtype in rpm deb; do 86 | fpm \ 87 | -s dir \ 88 | -t ${pkgtype} \ 89 | -n elasticsearch-curator \ 90 | -v ${1} \ 91 | --vendor ${VENDOR} \ 92 | --maintainer "${MAINTAINER}" \ 93 | --license 'Apache-2.0' \ 94 | --category tools \ 95 | --description 'Have indices in Elasticsearch? This is the tool for you!\n\nLike a museum curator manages the exhibits and collections on display, \nElasticsearch Curator helps you curate, or manage your indices.' \ 96 | --after-install ${C_POST_INSTALL} \ 97 | --before-remove ${C_PRE_REMOVE} \ 98 | --after-remove ${C_POST_REMOVE} \ 99 | --before-upgrade ${C_PRE_UPGRADE} \ 100 | --after-upgrade ${C_POST_UPGRADE} \ 101 | --provides elasticsearch-curator \ 102 | --conflicts python-elasticsearch-curator \ 103 | --conflicts python3-elasticsearch-curator \ 104 | /opt/elasticsearch-curator 105 | mv ${WORKDIR}/*.${pkgtype} ${PKG_TARGET} 106 | done 107 | 108 | deactivate 109 | 110 | rm ${C_POST_INSTALL} ${C_PRE_REMOVE} ${C_POST_REMOVE} ${C_PRE_UPGRADE} ${C_POST_UPGRADE} 111 | rm -rf ${SRCDIR} ${VENVDIR} 112 | -------------------------------------------------------------------------------- /unix_packages/builder_image/patchelf_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | wget http://people.centos.org/tru/devtools-2/devtools-2.repo -O /etc/yum.repos.d/devtools-2.repo 4 | yum install -y devtoolset-2-gcc devtoolset-2-binutils 5 | yum install -y devtoolset-2-gcc-c++ devtoolset-2-gcc-gfortran 6 | scl enable devtoolset-2 bash 7 | source /opt/rh/devtoolset-2/enable 8 | cd 9 | git clone https://github.com/NixOS/patchelf.git 10 | cd patchelf 11 | ./bootstrap.sh 12 | ./configure 13 | make 14 | make check 15 | make install 16 | 17 | # Cleanup after 18 | cd 19 | rm -rf patchelf 20 | exit 0 -------------------------------------------------------------------------------- /unix_packages/builder_image/ruby_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | gpg2 --keyserver hkp://pool.sks-keyservers.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB 4 | curl -sSL https://get.rvm.io | bash -s stable 5 | source /usr/local/rvm/scripts/rvm 6 | rvm install ruby 7 | GEMSSLPATH=$(gem which rubygems | sed -e 's/rubygems.rb/rubygems/') 8 | cd $GEMSSLPATH/ssl_certs/rubygems.org 9 | curl -O https://raw.githubusercontent.com/rubygems/rubygems/master/lib/rubygems/ssl_certs/rubygems.org/GlobalSignRootCA_R3.pem 10 | cd - 11 | gem install bundler 12 | gem update --system 13 | gem install rdoc 14 | gem install fpm 15 | ## MONKEY PATCH FPM https://github.com/jordansissel/fpm/issues/1777#issuecomment-810409151 16 | RUVER=$(which ruby | awk -F\/ '{print $6}') 17 | FPMVER=$(gem list | grep fpm | awk -F\( '{print $2}' | awk -F\) '{print $1}') 18 | cp /rpm.erb.patched /usr/local/rvm/gems/${RUVER}/gems/fpm-${FPMVER}/templates/rpm.erb 19 | --------------------------------------------------------------------------------