├── .github ├── FUNDING.yml └── workflows │ ├── python-codecov.yml │ ├── python-package.yml │ └── python-publish.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── MANIFEST.in ├── README.rst ├── cronicle.py ├── cronicle ├── __init__.py └── config.py ├── docs ├── _static │ └── asciicast.png ├── asciicast │ └── create_files.sh └── cronicle_screenshot.png ├── requirements-2.7.txt ├── requirements.txt ├── setup.cfg ├── setup.py ├── test └── test_cronicle.py └── tox.ini /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | ko_fi: kraymer 2 | -------------------------------------------------------------------------------- /.github/workflows/python-codecov.yml: -------------------------------------------------------------------------------- 1 | name: codecov 2 | on: 3 | push: 4 | branches: 5 | - 'master' 6 | jobs: 7 | codecov: 8 | name: Codecov Workflow 9 | runs-on: ubuntu-latest 10 | 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Set up Python 14 | uses: actions/setup-python@master 15 | with: 16 | python-version: 3.8 17 | - name: Generate coverage report 18 | run: | 19 | pip install mock pytest pytest-cov 20 | pip install -r requirements.txt 21 | python -m pytest --cov=./ --cov-report=xml 22 | - name: Upload coverage to Codecov 23 | uses: codecov/codecov-action@v1 24 | with: 25 | token: ${{ secrets.CODECOV_TOKEN }} 26 | file: ./coverage.xml 27 | flags: unittests 28 | -------------------------------------------------------------------------------- /.github/workflows/python-package.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: build 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | pull_request: 10 | branches: [ master ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | strategy: 17 | matrix: 18 | python-version: ['3.8', '3.9', '3.10', '3.11'] 19 | 20 | steps: 21 | - uses: actions/checkout@v2 22 | - name: Set up Python ${{ matrix.python-version }} 23 | uses: actions/setup-python@v2 24 | with: 25 | python-version: ${{ matrix.python-version }} 26 | - name: Install dependencies 27 | run: | 28 | python -m pip install --upgrade pip 29 | python -m pip install flake8 pytest mock 30 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 31 | if [ -f requirements-${{ matrix.python-version }}.txt ]; then pip install -r requirements-${{ matrix.python-version }}.txt; fi 32 | - name: Lint with flake8 33 | run: | 34 | # stop the build if there are Python syntax errors or undefined names 35 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 36 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 37 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 38 | - name: Test with pytest 39 | run: | 40 | python -m pytest 41 | -------------------------------------------------------------------------------- /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflows will upload a Python Package using Twine when a release is created 2 | # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries 3 | 4 | name: Upload Python Package 5 | 6 | on: 7 | release: 8 | types: [created] 9 | 10 | jobs: 11 | deploy: 12 | 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Set up Python 18 | uses: actions/setup-python@v2 19 | with: 20 | python-version: '3.x' 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | pip install setuptools wheel twine 25 | - name: Build and publish 26 | env: 27 | TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} 28 | TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} 29 | run: | 30 | python setup.py sdist bdist_wheel 31 | twine upload dist/* 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .coverage 3 | .direnv 4 | .envrc 5 | .pre-commit-config.yaml 6 | .tox/ 7 | __pycache__ 8 | bin/ 9 | dist/ 10 | cronicle.egg-info/ 11 | include/ 12 | lib/ 13 | local/ 14 | pip-selfcheck.json 15 | test/rsrc/DAILY/* 16 | test/rsrc/foobar* 17 | test/rsrc/MONTHLY/* 18 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7-alpine 2 | 3 | WORKDIR /cronicle 4 | 5 | COPY requirements.txt ./ 6 | RUN pip install --requirement requirements.txt 7 | 8 | COPY ./cronicle ./cronicle 9 | COPY *.py README.rst ./ 10 | 11 | RUN python3 setup.py install 12 | 13 | ENTRYPOINT ["cronicle"] 14 | CMD ["--help"] 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Fabrice Laporte 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # Include relevant text files. 2 | include LICENSE README.rst requirements.txt 3 | 4 | # Exclude junk. 5 | global-exclude 6 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: http://github.com/kraymer/cronicle/workflows/build/badge.svg 2 | :target: https://github.com/kraymer/cronicle/actions 3 | .. image:: https://codecov.io/gh/Kraymer/cronicle/branch/master/graph/badge.svg?token=eku8LDViVP 4 | :target: https://codecov.io/gh/Kraymer/cronicle 5 | .. image:: https://img.shields.io/github/v/release/kraymer/cronicle.svg 6 | :target: https://github.com/Kraymer/cronicle/releases 7 | .. image:: https://pepy.tech/badge/cronicle 8 | :target: https://pepy.tech/project/cronicle 9 | 10 | cronicle :hourglass\_flowing\_sand::arrows\_counterclockwise::floppy\_disk: 11 | =========================================================================== 12 | 13 | **/ˈkɹɒnɪkəl/** : 14 | 15 | 1. *n.* a factual written account of important events in the order 16 | of their occurrence 17 | 2. *n.* software to archive the *N* most recent backups of a file in 18 | a folder named after the job frequency. Recommended use is to 19 | trigger it via a cron job. 20 | 21 | Originally, ``cronicle`` has been conceived as a solution to this 22 | particular `serverfault `__ question : `How to 23 | keep: daily backups for a week, weekly for a month, monthly for a year, 24 | and yearly after 25 | that `__. 26 | 27 | |asciicast| 28 | 29 | Features 30 | -------- 31 | 32 | - **simplicity:** add one line to your ``crontab`` and you're done 33 | - **files rotation:** keep the N most recent versions of a file 34 | - **space efficient:** use symlinks in target directories to store a 35 | single occurence of each backup instead of performing copies. When 36 | removing a link, remove the underlying file if no other link point to 37 | it. 38 | 39 | Usage 40 | ----- 41 | 42 | In order to manage a file backups with cronicle, you must have a section 43 | in the ``config.yaml`` that matches the backups names. Under it you can 44 | then define values (number of archives to keep) for the five kinds of 45 | periodic archives : ``hourly``, ``daily``, ``weekly``, ``monthly``, ``yearly``. 46 | 47 | Or define a custom periodicity using the *pipe syntax* eg 48 | ``bimonthly|60: 3`` to keep archives every two months over the last six 49 | months. 50 | 51 | Example 52 | ------- 53 | 54 | If you have dumps of a database in a ``~/dumps`` directory named like 55 | ``mydb-20170101.dump``, ``mydb-20170102.dump``, and want to keep each 56 | dump for 7 days plus go back up to two months ; a working 57 | ``${HOME}/.config/cronicle/config.yaml`` content would be : 58 | 59 | :: 60 | 61 | /home/johndoe/dumps/mydb-*.dump: 62 | daily: 7 63 | monthly: 2 64 | 65 | Next ``cronicle`` call will result in the creation of folders ``DAILY`` 66 | and ``MONTHLY`` in ``/home/johndoe/dumps/``, each folder containing 67 | symlinks to the .dump files. 68 | 69 | Installation 70 | ------------ 71 | 72 | cronicle is written for `Python 73 | 2.7 `__ and `Python 74 | 3 `__, is tested on Linux and Mac OS 75 | X. 76 | 77 | Install with `pip `__ via 78 | ``pip install cronicle`` command. 79 | 80 | ``cron`` triggering 81 | ------------------- 82 | 83 | For a no-brainer use, I recommend to run cronicle via cron, just after 84 | the command in charge of performing the backup. A ``crontab`` example : 85 | 86 | :: 87 | 88 | @daily pg_dump -Fc mydb > /home/johndoe/dumps/mydb-`date +%F`.dump 89 | @daily cronicle -r /home/johndoe/dumps/mydb-`date +%F`.dump 90 | 91 | If used with the ``config.yaml`` as defined in the previous section, 92 | this daily call to cronicle guarantees that you will keep at most 9 93 | database dumps (7 latest daily + 2 monthly). 94 | 95 | 96 | .. |asciicast| image:: https://raw.githubusercontent.com/Kraymer/cronicle/master/docs/cronicle_screenshot.png 97 | :target: https://asciinema.org/a/155861 98 | 99 | -------------------------------------------------------------------------------- /cronicle.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import cronicle 4 | 5 | if __name__ == "__main__": 6 | cronicle.cronicle_cli() 7 | -------------------------------------------------------------------------------- /cronicle/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2018 Fabrice Laporte - kray.me 5 | # The MIT License http://www.opensource.org/licenses/mit-license.php 6 | 7 | """Use cron to rotate backup files! 8 | """ 9 | 10 | import copy 11 | import glob 12 | import logging 13 | import os 14 | 15 | import click 16 | import click_log 17 | 18 | import datetime as dt 19 | from collections import OrderedDict 20 | from dateutil.relativedelta import relativedelta 21 | from shutil import rmtree 22 | 23 | 24 | from .config import config 25 | 26 | 27 | __version__ = "0.4.0" 28 | 29 | logger = logging.getLogger(__name__) 30 | click_log.basic_config(logger) 31 | DEFAULT_CFG = {"daily": 0, "weekly": 0, "monthly": 0, "yearly": 0, "pattern": "*"} 32 | 33 | # Names of frequency folders that will host symlinks, and minimum delta elapsed between 2 archives 34 | FREQUENCY_FOLDER_DAYS = { 35 | "DAILY": ("days", 1), 36 | "HOURLY": ("hours", 1), 37 | "WEEKLY": ("days", 7), 38 | "MONTHLY": ("months", 1), 39 | "YEARLY": ("months", 12), 40 | } 41 | CONFIG_PATH = os.path.join(config.config_dir(), "config.yaml") 42 | 43 | 44 | def exclude_frequency_folders(lst): 45 | """Exclude folders whose name matches one of the frequency folders 46 | """ 47 | return [x for x in lst if os.path.basename(x) not in FREQUENCY_FOLDER_DAYS] 48 | 49 | 50 | def frequency_folder_days(freq_dir): 51 | """Return minimum delta between 2 archives inside given folder""" 52 | try: 53 | return FREQUENCY_FOLDER_DAYS[os.path.basename(freq_dir).upper()] 54 | except KeyError: 55 | pass 56 | try: 57 | return int(freq_dir.split("|")[-1]) 58 | except Exception: 59 | return None 60 | 61 | 62 | def file_create_date(filepath): 63 | """Return file creation date with a daily precision.""" 64 | try: 65 | filedate = os.lstat(os.path.realpath(filepath)).st_birthtime 66 | except AttributeError: 67 | filedate = os.lstat(os.path.realpath(filepath)).st_mtime 68 | return dt.date.fromtimestamp(filedate) 69 | 70 | 71 | def is_symlinked(filepath, folders): 72 | """Return True if filepath has symlinks pointing to it in given folders.""" 73 | dirname, basename = os.path.split(filepath) 74 | for folder in folders: 75 | target = os.path.abspath(os.path.join(dirname, folder, basename)) 76 | if os.path.lexists(target): 77 | return True 78 | return False 79 | 80 | 81 | def find_config(filename, cfg=None): 82 | """Return the config matched by filename""" 83 | res = copy.deepcopy(DEFAULT_CFG) 84 | dirname, basename = os.path.split(filename) 85 | 86 | if not cfg: 87 | cfg = config 88 | # Overwrite default config fields with matched config ones 89 | for pattern in cfg.keys(): 90 | abspattern = ( 91 | os.path.join(dirname, pattern) if not os.path.isabs(pattern) else pattern 92 | ) 93 | for x in glob.glob(abspattern): 94 | if not x.endswith(filename): 95 | continue 96 | pattern_cfg = cfg[pattern] if isinstance(cfg, dict) else cfg[pattern].get() 97 | res.update(pattern_cfg) 98 | for frequency in pattern_cfg: 99 | if frequency_folder_days(frequency) is None: 100 | logger.error("Invalid configuration attribute '%s'" % pattern) 101 | exit(1) 102 | res["pattern"] = pattern 103 | return res 104 | 105 | 106 | class Cronicle: 107 | def __init__(self, filenames, remove=False, dry_run=False, config=None): 108 | for filename in [os.path.abspath(x) for x in filenames]: 109 | self.dry_run = dry_run 110 | self.cfg = find_config(filename, config) 111 | 112 | if not self.cfg: 113 | logger.error( 114 | "No pattern found in %s that matches %s." % (CONFIG_PATH, filename) 115 | ) 116 | exit(1) 117 | freq_dirs = [ 118 | x.upper() 119 | for x in set(self.cfg.keys()) - set(["pattern"]) 120 | if self.cfg[x] 121 | ] 122 | for freq_dir in freq_dirs: 123 | self.timed_symlink(filename, freq_dir) 124 | for freq_dir in freq_dirs: 125 | self.rotate(filename, freq_dir, remove) 126 | 127 | def remove(self, path): 128 | if self.dry_run: 129 | logger.info("dry-run mode disabling removal of {}".format(path)) 130 | else: 131 | logger.info("Removing {}".format(path)) 132 | os.remove(path) 133 | 134 | def symlink(self, src, dst): 135 | if self.dry_run: 136 | logger.info("dry-run mode disabling symlink of {}->{}".format(dst, src)) 137 | else: 138 | logger.info("Creating symlink %s" % dst) 139 | os.symlink(src, dst) 140 | 141 | def unlink(self, path): 142 | if self.dry_run: 143 | logger.info("dry-run mode disabling unlink of {}".format(path)) 144 | else: 145 | logger.info("Unlinking %s" % path) 146 | os.unlink(path) 147 | 148 | def rmtree(self, path): 149 | if self.dry_run: 150 | logger.info("dry-run mode disabling rmtree of {}".format(path)) 151 | else: 152 | logger.info("Removing {}".format(path)) 153 | rmtree(path) 154 | 155 | def last_archive_date(self, filename, folder, pattern): 156 | """Return last archive date for given folder""" 157 | archives = self.archives_create_dates(folder, pattern) 158 | if archives: 159 | return list(archives.keys())[-1] 160 | 161 | def archives_create_dates(self, folder, pattern="*"): 162 | """Return OrderedDict of valid archives symlinks sorted by creation dates (used as keys).""" 163 | creation_dates = {} 164 | 165 | abs_pattern = os.path.join(folder, os.path.basename(pattern)) 166 | for filepath in glob.glob(abs_pattern): 167 | if os.path.islink(filepath): 168 | if os.path.exists(filepath): 169 | creation_dates[file_create_date(filepath)] = filepath 170 | else: 171 | logger.info( 172 | "No source file found at %s, deleting obsolete symlink %s." 173 | % (os.path.realpath(filepath), filepath) 174 | ) 175 | self.unlink(filepath) 176 | logger.debug("Archives dates: {}".format(creation_dates)) 177 | return OrderedDict(sorted(creation_dates.items())) 178 | 179 | def is_spaced_enough(self, filename, target_dir): 180 | """Return True if enough time elapsed between last archive 181 | and filename creation dates according to target_dir frequency. 182 | """ 183 | file_date = file_create_date(filename) 184 | _last_archive_date = self.last_archive_date( 185 | filename, target_dir, self.cfg["pattern"] 186 | ) 187 | logger.debug("File {} created at {}".format(filename, file_date)) 188 | if _last_archive_date: 189 | delta = relativedelta(file_date, _last_archive_date) 190 | logger.debug("Delta between {} and {}: {}".format( 191 | file_date, _last_archive_date, delta)) 192 | delta_unit, delta_min = frequency_folder_days(target_dir) 193 | return getattr(delta, delta_unit) >= delta_min 194 | 195 | return True 196 | 197 | def timed_symlink(self, filename, freq_dir): 198 | """Create symlink for filename in freq_dir if enough days elapsed since last archive. 199 | Return True if symlink created. 200 | """ 201 | target_dir = os.path.abspath( 202 | os.path.join(os.path.dirname(filename), freq_dir.split("|")[0]) 203 | ) 204 | 205 | if not self.is_spaced_enough(filename, target_dir): 206 | logger.warning("{}: No {} symlink created, too short delay since last archive".format(filename, freq_dir)) 207 | return 208 | target = os.path.join(target_dir, os.path.basename(filename)) 209 | if not os.path.lexists(target): 210 | if not os.path.exists(target_dir): 211 | os.makedirs(target_dir) 212 | self.symlink(os.path.relpath(filename, start=target_dir), target) 213 | else: 214 | logger.error("{}: already exists".format(target)) 215 | return 216 | return True 217 | 218 | def rotate(self, filename, freq_dir, remove): 219 | """Keep only the n last links of folder that matches same pattern than filename.""" 220 | 221 | others_freq_dirs = [ 222 | x.split("|")[0].upper() for x in set(self.cfg.keys()) - set([freq_dir]) 223 | ] 224 | target_dir = os.path.abspath( 225 | os.path.join(os.path.dirname(filename), freq_dir.split("|")[0]) 226 | ) 227 | # sort new -> old 228 | links = list(self.archives_create_dates(target_dir, self.cfg["pattern"]).values())[ 229 | ::-1 230 | ] 231 | 232 | for link in links[self.cfg[freq_dir.lower()] :]: # skip the n most recents 233 | filepath = os.path.realpath(link) 234 | self.unlink(link) 235 | if remove and not is_symlinked(filepath, others_freq_dirs): 236 | if os.path.isfile(filepath): 237 | self.remove(filepath) 238 | elif os.path.isdir(filepath): 239 | self.rmtree(filepath) 240 | 241 | 242 | @click.command( 243 | context_settings=dict(help_option_names=["-h", "--help"]), 244 | help=( 245 | "Keep rotated time-spaced archives of files. FILES names must match one of " 246 | " the patterns present in %s." % CONFIG_PATH 247 | ), 248 | epilog=( 249 | "See https://github.com/Kraymer/cronicle/#usage for " 250 | "more infos" 251 | ), 252 | ) 253 | @click.argument("filenames", type=click.Path(exists=True), metavar="FILES", nargs=-1) 254 | @click.option( 255 | "-r", 256 | "--remove", 257 | "remove", 258 | help="Remove previous file backup when no symlink points to it.", 259 | default=False, 260 | is_flag=True, 261 | ) 262 | @click.option( 263 | "-d", "--dry-run", count=True, help="Just print instead of writing on filesystem." 264 | ) 265 | @click_log.simple_verbosity_option(logger) 266 | @click.version_option(__version__) 267 | def cronicle_cli(filenames, remove, dry_run): 268 | Cronicle(exclude_frequency_folders(filenames), remove, dry_run) 269 | 270 | 271 | if __name__ == "__main__": 272 | cronicle_cli() 273 | -------------------------------------------------------------------------------- /cronicle/config.py: -------------------------------------------------------------------------------- 1 | import confuse 2 | 3 | config = confuse.Configuration("cronicle", __name__) 4 | -------------------------------------------------------------------------------- /docs/_static/asciicast.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kraymer/cronicle/1aedabd9ed285f854eaffe683a20895bb72a65f0/docs/_static/asciicast.png -------------------------------------------------------------------------------- /docs/asciicast/create_files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | touch foobar_2007-12-01.txt 4 | touch foobar_2007-12-02.txt 5 | touch foobar_2007-12-03.txt 6 | touch foobar_2007-12-04.txt 7 | touch foobar_2007-12-05.txt 8 | touch foobar_2007-12-06.txt 9 | touch foobar_2007-12-07.txt 10 | touch foobar_2007-12-08.txt 11 | touch foobar_2007-12-09.txt 12 | touch foobar_2007-12-10.txt 13 | touch foobar_2007-12-11.txt 14 | touch foobar_2007-12-12.txt 15 | touch foobar_2007-12-13.txt 16 | touch foobar_2007-12-14.txt 17 | touch foobar_2007-12-15.txt 18 | touch foobar_2007-12-16.txt 19 | touch foobar_2007-12-17.txt 20 | touch foobar_2007-12-18.txt 21 | touch foobar_2007-12-19.txt 22 | touch foobar_2007-12-20.txt 23 | touch foobar_2007-12-21.txt 24 | touch foobar_2007-12-22.txt 25 | touch foobar_2007-12-23.txt 26 | touch foobar_2007-12-24.txt 27 | touch foobar_2007-12-25.txt 28 | touch foobar_2007-12-26.txt 29 | touch foobar_2007-12-27.txt 30 | touch foobar_2007-12-28.txt 31 | touch foobar_2007-12-29.txt 32 | touch foobar_2007-12-30.txt 33 | touch foobar_2007-12-31.txt 34 | 35 | touch -h -t 200712010100 foobar_2007-12-01.txt 36 | touch -h -t 200712020100 foobar_2007-12-02.txt 37 | touch -h -t 200712030100 foobar_2007-12-03.txt 38 | touch -h -t 200712040100 foobar_2007-12-04.txt 39 | touch -h -t 200712050100 foobar_2007-12-05.txt 40 | touch -h -t 200712060100 foobar_2007-12-06.txt 41 | touch -h -t 200712070100 foobar_2007-12-07.txt 42 | touch -h -t 200712080100 foobar_2007-12-08.txt 43 | touch -h -t 200712090100 foobar_2007-12-09.txt 44 | touch -h -t 200712100100 foobar_2007-12-10.txt 45 | touch -h -t 200712110100 foobar_2007-12-11.txt 46 | touch -h -t 200712120100 foobar_2007-12-12.txt 47 | touch -h -t 200712130100 foobar_2007-12-13.txt 48 | touch -h -t 200712140100 foobar_2007-12-14.txt 49 | touch -h -t 200712150100 foobar_2007-12-15.txt 50 | touch -h -t 200712160100 foobar_2007-12-16.txt 51 | touch -h -t 200712170100 foobar_2007-12-17.txt 52 | touch -h -t 200712180100 foobar_2007-12-18.txt 53 | touch -h -t 200712190100 foobar_2007-12-19.txt 54 | touch -h -t 200712200100 foobar_2007-12-20.txt 55 | touch -h -t 200712210100 foobar_2007-12-21.txt 56 | touch -h -t 200712220100 foobar_2007-12-22.txt 57 | touch -h -t 200712230100 foobar_2007-12-23.txt 58 | touch -h -t 200712240100 foobar_2007-12-24.txt 59 | touch -h -t 200712250100 foobar_2007-12-25.txt 60 | touch -h -t 200712260100 foobar_2007-12-26.txt 61 | touch -h -t 200712270100 foobar_2007-12-27.txt 62 | touch -h -t 200712280100 foobar_2007-12-28.txt 63 | touch -h -t 200712290100 foobar_2007-12-29.txt 64 | touch -h -t 200712300100 foobar_2007-12-30.txt 65 | touch -h -t 200712310100 foobar_2007-12-31.txt 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /docs/cronicle_screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kraymer/cronicle/1aedabd9ed285f854eaffe683a20895bb72a65f0/docs/cronicle_screenshot.png -------------------------------------------------------------------------------- /requirements-2.7.txt: -------------------------------------------------------------------------------- 1 | backports.tempfile 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | click 2 | click-log 3 | confuse 4 | ordereddict 5 | python-dateutil 6 | pyyaml 7 | six 8 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [coverage:report] 2 | omit = 3 | */pyshared/* 4 | */python?.?/* 5 | 6 | [semantic_release] 7 | version_variable = cronicle/__init__.py:__version__ 8 | changelog_sections = feature,fix,documentation 9 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Copyright (c) 2018-2020 Fabrice Laporte - kray.me 5 | # The MIT License http://www.opensource.org/licenses/mit-license.php 6 | 7 | import codecs 8 | import os 9 | import re 10 | import sys 11 | import time 12 | from setuptools import setup 13 | 14 | try: 15 | from semantic_release import setup_hook 16 | 17 | setup_hook(sys.argv) 18 | except ImportError: 19 | pass 20 | 21 | 22 | PKG_NAME = "cronicle" 23 | DIRPATH = os.path.dirname(__file__) 24 | 25 | 26 | def read_rsrc(filename): 27 | with codecs.open(os.path.join(DIRPATH, filename), encoding="utf-8") as _file: 28 | return re.sub(r":(\w+\\?)+:", u"", _file.read().strip()) # no emoji 29 | 30 | 31 | with codecs.open("cronicle/__init__.py", encoding="utf-8") as fd: 32 | version = re.search( 33 | r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE 34 | ).group(1) 35 | version = version.replace("dev", str(int(time.time()))) 36 | 37 | # Deploy: python3 setup.py sdist bdist_wheel; twine upload --verbose dist/* 38 | setup( 39 | name=PKG_NAME, 40 | version=version, 41 | description="Use cron to rotate backup files!", 42 | long_description=read_rsrc("README.rst"), 43 | author="Fabrice Laporte", 44 | author_email="kraymer@gmail.com", 45 | url="https://github.com/KraYmer/cronicle", 46 | license="MIT", 47 | platforms="ALL", 48 | packages=[ 49 | "cronicle", 50 | ], 51 | entry_points={"console_scripts": ["cronicle = cronicle:cronicle_cli"]}, 52 | install_requires=read_rsrc("requirements.txt").split("\n"), 53 | extras_require={ 54 | "test": [ 55 | "coverage>=5,<6", 56 | "nose>1.3", 57 | "mock==4.0.2", 58 | "tox>=3", 59 | ] 60 | }, 61 | classifiers=[ 62 | "License :: OSI Approved :: MIT License", 63 | "Programming Language :: Python", 64 | "Environment :: Console", 65 | "Topic :: System :: Filesystems", 66 | ], 67 | keywords="cron rotate backup", 68 | ) 69 | -------------------------------------------------------------------------------- /test/test_cronicle.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import datetime as dt 4 | import glob 5 | import itertools 6 | import mock 7 | import os 8 | import unittest 9 | 10 | from dateutil import parser 11 | try: 12 | from backports import tempfile 13 | except Exception: 14 | import tempfile 15 | 16 | from cronicle import Cronicle, find_config, exclude_frequency_folders 17 | 18 | 19 | NOOP_CONFIG = {"hourly": 0, "daily": 0, "weekly": 0, "monthly": 0, "yearly": 0} 20 | 21 | 22 | def date_generator(): 23 | """Generate dates starting at 1st Dec 2019 by 1 day step increment""" 24 | from_date = dt.date(2019, 12, 1) 25 | while True: 26 | yield from_date 27 | from_date = from_date + dt.timedelta(days=1) 28 | 29 | 30 | def mock_file_create_day(filepath): 31 | """Interpret date in filename and returns it as creation date.""" 32 | return parser.parse(filepath.split("/")[-1][4:].replace("_", " ")) 33 | 34 | 35 | def create_empty_file(path): 36 | with open(path, "w"): 37 | pass 38 | 39 | 40 | class ConfigTest(unittest.TestCase): 41 | def setUp(self): 42 | self.rootdir = tempfile.TemporaryDirectory(prefix="cronicle_") 43 | self.barfile = os.path.join(self.rootdir.name, "bar.txt") 44 | create_empty_file(self.barfile) 45 | 46 | def test_find_config_ok(self): 47 | """Check loading of config when filename matches pattern""" 48 | res = find_config( 49 | self.barfile, 50 | { 51 | os.path.join(self.rootdir.name, "bar*"): {"daily": 3}, 52 | "foo*": {"weekly": 4}, 53 | }, 54 | ) 55 | self.assertEqual( 56 | res, 57 | { 58 | "daily": 3, 59 | "monthly": 0, 60 | "pattern": os.path.join(self.rootdir.name, "bar*"), 61 | "weekly": 0, 62 | "yearly": 0, 63 | }, 64 | ) 65 | 66 | def test_find_config_ko(self): 67 | """Check no config is returned when filename doesn't match any pattern""" 68 | res = find_config("foo", {"bar*": {"weekly": 3}}) 69 | self.assertEqual(res, None) 70 | 71 | def test_exclude_frequency_folders(self): 72 | """Check frequency folders are correctly excluded fromfilenames to consider to archive 73 | """ 74 | lst = ["/tmp/FOO", "/tmp/DAILY", "/tmp/BIDAILY"] 75 | self.assertEqual(exclude_frequency_folders(lst), ["/tmp/FOO", "/tmp/BIDAILY"]) 76 | 77 | 78 | class ArchiveTest(unittest.TestCase): 79 | """Create set of files to archive beforehand, call cronicle and check symlinks created.""" 80 | 81 | def setUp(self): 82 | self.rootdir = tempfile.TemporaryDirectory(prefix="cronicle_") 83 | self.config = {os.path.join(self.rootdir.name, "foo_*"): {"daily": 3}} 84 | 85 | for date in itertools.islice(date_generator(), 90): 86 | for hour in (9, 14): 87 | abspath = os.path.join( 88 | self.rootdir.name, "foo_{}_{:02d}h".format(str(date), hour) 89 | ) 90 | create_empty_file(abspath) 91 | self.last_file = abspath 92 | 93 | def test_dry_run(self): 94 | """Check filesystem is unmodified when dry-run is used""" 95 | archive = os.path.join(self.rootdir.name, "DAILY", os.path.basename(self.last_file)) 96 | Cronicle([self.last_file], dry_run=True, config=self.config) 97 | self.assertFalse(os.path.exists(archive)) 98 | Cronicle([self.last_file], dry_run=False, config=self.config) 99 | self.assertTrue(os.path.exists(archive)) 100 | 101 | def test_archives_folders(self): 102 | """Check that no empty archive folder is created.""" 103 | Cronicle([self.last_file], config=self.config) 104 | self.assertTrue(os.path.exists(os.path.join(self.rootdir.name, "DAILY"))) 105 | self.assertFalse( 106 | any( 107 | [ 108 | os.path.exists(os.path.join(self.rootdir.name, x)) 109 | for x in (u"HOURLY", u"MONTHLY", u"WEEKLY", u"YEARLY") 110 | ] 111 | ) 112 | ) 113 | 114 | @mock.patch("cronicle.file_create_date", side_effect=mock_file_create_day) 115 | def test_number_of_archives(self, mock): 116 | """Check number of archives created""" 117 | files = sorted(glob.glob(os.path.join(self.rootdir.name, "foo_*"))) 118 | Cronicle( 119 | files, 120 | config={ 121 | os.path.join(self.rootdir.name, "foo_*"): { 122 | "hourly": 3, 123 | "daily": 3, 124 | "weekly": 4, 125 | "monthly": 4, 126 | "yearly": 4, 127 | } 128 | }, 129 | ) 130 | self.assertEqual( 131 | set(os.listdir(os.path.join(self.rootdir.name, "DAILY"))), 132 | { 133 | # Archives done at 9h are kept instead of those at 14h as 134 | # symlinking the latter fail because of too short delay since 135 | # last archive 136 | "foo_2020-02-26_09h", 137 | "foo_2020-02-27_09h", 138 | "foo_2020-02-28_09h", 139 | }, 140 | ) 141 | self.assertEqual( 142 | set(os.listdir(os.path.join(self.rootdir.name, "MONTHLY"))), 143 | {"foo_2019-12-01_09h", "foo_2020-01-01_09h", "foo_2020-02-01_09h"}, 144 | ) 145 | self.assertEqual( 146 | set(os.listdir(os.path.join(self.rootdir.name, "HOURLY"))), 147 | {"foo_2020-02-27_14h", "foo_2020-02-28_09h", "foo_2020-02-28_14h"}, 148 | ) 149 | 150 | @mock.patch("cronicle.file_create_date", side_effect=mock_file_create_day) 151 | def test_rm_symlink(self, mock): 152 | """Check dangling symlinks are removed.""" 153 | files = sorted(glob.glob(os.path.join(self.rootdir.name, "foo_*"))) 154 | Cronicle(files[-2:-1], config=self.config) 155 | os.remove(files[-2]) 156 | 157 | Cronicle(files[-1:], config=self.config) 158 | self.assertEqual( 159 | set(os.listdir(os.path.join(self.rootdir.name, "DAILY"))), 160 | { 161 | "foo_2020-02-28_14h", 162 | }, 163 | ) 164 | 165 | 166 | class RotateTest(unittest.TestCase): 167 | """Call cronicle after each file creation and check files rotation.""" 168 | 169 | @mock.patch("cronicle.file_create_date", side_effect=mock_file_create_day) 170 | def test_remove(self, mock): 171 | """Check only the number of files specified in config are kept""" 172 | self.rootdir = tempfile.TemporaryDirectory(prefix="cronicle_") 173 | 174 | for date in itertools.islice(date_generator(), 30): 175 | abspath = os.path.join(self.rootdir.name, "bar_{}".format(str(date))) 176 | with open(abspath, "w"): 177 | pass 178 | Cronicle( 179 | [abspath], 180 | remove=True, 181 | config={os.path.join(self.rootdir.name, "bar_*"): {"daily": 3}}, 182 | ) 183 | 184 | self.assertEqual( 185 | set( 186 | [ 187 | x 188 | for x in os.listdir(self.rootdir.name) 189 | if os.path.isfile(os.path.join(self.rootdir.name, x)) 190 | ] 191 | ), 192 | {"bar_2019-12-28", "bar_2019-12-30", "bar_2019-12-29"}, 193 | ) 194 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27, py36, black, cov 3 | 4 | [testenv] 5 | sitepackages = false 6 | deps = 7 | pytest 8 | nose 9 | mock 10 | -r{toxinidir}/requirements.txt 11 | extras = tests 12 | whitelist_externals = 13 | python 14 | pytest 15 | commands = 16 | install: python ./setup.py install {posargs} 17 | python -m pytest -vs {posargs} 18 | 19 | [testenv:cov] 20 | deps = 21 | coverage 22 | pytest 23 | commands = 24 | python -m pytest -vs --with-coverage {posargs} 25 | 26 | [testenv:py27] 27 | deps = 28 | pytest 29 | nose 30 | mock 31 | backports.tempfile 32 | -r{toxinidir}/requirements.txt 33 | basepython = python2.7 34 | 35 | [testenv:py36] 36 | basepython = python3.6 37 | 38 | [testenv:black] 39 | deps=black 40 | basepython=python3 41 | setenv = 42 | LC_ALL=C.UTF-8 43 | LANG=C.UTF-8 44 | commands=black --check --verbose . 45 | --------------------------------------------------------------------------------