├── tests └── testset1 │ ├── subfolder │ ├── customlog.log │ └── logfileNoPostfix │ ├── log1.log │ ├── logfile emptyspace.log │ └── xmllog.xml ├── logs └── README.md ├── .gitignore ├── scripts ├── scripts.cfg ├── container.cfg ├── rmi.sh └── build.sh ├── .dockerignore ├── update-logrotate.sh ├── LICENSE ├── logrotate_ds.yaml ├── Dockerfile ├── docker-entrypoint.sh ├── .circleci └── config.yml ├── logrotateCreateConf.sh ├── logrotateConf.sh ├── logrotate.sh ├── Jenkinsfile └── README.md /tests/testset1/subfolder/customlog.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/testset1/log1.log: -------------------------------------------------------------------------------- 1 | Blabb 2 | Blabb 3 | -------------------------------------------------------------------------------- /tests/testset1/subfolder/logfileNoPostfix: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /logs/README.md: -------------------------------------------------------------------------------- 1 | Logs should be placed here. 2 | -------------------------------------------------------------------------------- /tests/testset1/logfile emptyspace.log: -------------------------------------------------------------------------------- 1 | Blibb 2 | Blibb 3 | -------------------------------------------------------------------------------- /tests/testset1/xmllog.xml: -------------------------------------------------------------------------------- 1 | Blubb 2 | Blubb 3 | Blubb 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .vagrant 3 | *.json 4 | *.tar 5 | *.tar.gz 6 | -------------------------------------------------------------------------------- /scripts/scripts.cfg: -------------------------------------------------------------------------------- 1 | readonly ROOT_DIR=$(cd ${CUR_DIR}/..; pwd) 2 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | scripts 3 | README.md 4 | .vagrant 5 | logs 6 | -------------------------------------------------------------------------------- /scripts/container.cfg: -------------------------------------------------------------------------------- 1 | #------------------ 2 | # CONTAINER VARIABLES 3 | #------------------ 4 | 5 | readonly IMAGE_NAME="blacklabelops/logrotate" 6 | -------------------------------------------------------------------------------- /update-logrotate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # A helper script for updating the /usr/bin/logrotate.d/logrotate.conf. 4 | 5 | set -e 6 | 7 | [[ ${DEBUG} == true ]] && set -x 8 | 9 | source /usr/bin/logrotate.d/logrotate.sh 10 | source /usr/bin/logrotate.d/logrotateConf.sh 11 | 12 | resetConfigurationFile 13 | 14 | #Create Logrotate Conf 15 | source /usr/bin/logrotate.d/logrotateCreateConf.sh 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Steffen Bleul 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /logrotate_ds.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | metadata: 4 | name: logrotate 5 | spec: 6 | template: 7 | metadata: 8 | labels: 9 | app: logging 10 | id: logrotate 11 | name: logrotate 12 | spec: 13 | containers: 14 | - name: logrotate-es 15 | image: blacklabelops/logrotate 16 | securityContext: 17 | privileged: true 18 | volumeMounts: 19 | - name: containers 20 | mountPath: /var/lib/docker/containers 21 | - name: varlog 22 | mountPath: /var/log/docker 23 | - name: logs 24 | mountPath: /logs 25 | env: 26 | - name: LOGS_DIRECTORIES 27 | value: "/var/lib/docker/containers /var/log/docker" 28 | - name: LOGROTATE_INTERVAL 29 | value: "hourly" 30 | - name: LOGROTATE_OLDDIR 31 | value: "/logs" 32 | volumes: 33 | - hostPath: 34 | path: /var/lib/docker/containers 35 | name: containers 36 | - hostPath: 37 | path: /var/log/docker 38 | name: varlog 39 | - hostPath: 40 | path: /var/log/containers/ 41 | name: logs 42 | -------------------------------------------------------------------------------- /scripts/rmi.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Remove Docker image 4 | # 5 | 6 | set -o errexit # abort script at first error 7 | set -o pipefail # return the exit status of the last command in the pipe 8 | set -o nounset # treat unset variables and parameters as an error 9 | 10 | # Setting environment variables 11 | readonly CUR_DIR=$(cd $(dirname ${BASH_SOURCE:-$0}); pwd) 12 | 13 | printf '%b\n' ":: Reading scrips config...." 14 | source $CUR_DIR/scripts.cfg 15 | 16 | printf '%b\n' ":: Reading container config...." 17 | source $CUR_DIR/container.cfg 18 | 19 | # Helper functions 20 | err() { 21 | printf '%b\n' "" 22 | printf '%b\n' "\033[1;31m[ERROR] $@\033[0m" 23 | printf '%b\n' "" 24 | exit 1 25 | } >&2 26 | 27 | success() { 28 | printf '%b\n' "" 29 | printf '%b\n' "\033[1;32m[SUCCESS] $@\033[0m" 30 | printf '%b\n' "" 31 | } 32 | 33 | lookForImage() { 34 | local IMAGE_LIST=$(docker images | awk '{print $1}') 35 | local IMAGE_FOUND="false" 36 | 37 | for image in $IMAGE_LIST 38 | do 39 | if [ $image = $IMAGE_NAME ]; then 40 | IMAGE_FOUND="true" 41 | fi 42 | done 43 | 44 | echo $IMAGE_FOUND 45 | } 46 | 47 | #------------------ 48 | # SCRIPT ENTRYPOINT 49 | #------------------ 50 | 51 | found=$(lookForImage) 52 | 53 | if [ $found = "false" ]; then 54 | err ""$IMAGE_NAME" not found" 55 | fi 56 | 57 | printf '%b\n' "" 58 | printf '%b\n' ":: Removing image..." 59 | 60 | docker rmi -f ${IMAGE_NAME} 61 | 62 | found=$(lookForImage) 63 | 64 | if [ $found = "true" ]; then 65 | err ""$IMAGE_NAME" still found, removing failed." 66 | fi 67 | 68 | success "Image removed successfully." 69 | -------------------------------------------------------------------------------- /scripts/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Build Docker image 4 | # 5 | 6 | set -o errexit # abort script at first error 7 | set -o pipefail # return the exit status of the last command in the pipe 8 | set -o nounset # treat unset variables and parameters as an error 9 | 10 | # Setting environment variables 11 | readonly CUR_DIR=$(cd $(dirname ${BASH_SOURCE:-$0}); pwd) 12 | 13 | printf '%b\n' ":: Reading scrips config...." 14 | source $CUR_DIR/scripts.cfg 15 | 16 | printf '%b\n' ":: Reading container config...." 17 | source $CUR_DIR/container.cfg 18 | 19 | # Helper functions 20 | err() { 21 | printf '%b\n' "" 22 | printf '%b\n' "\033[1;31m[ERROR] $@\033[0m" 23 | printf '%b\n' "" 24 | exit 1 25 | } >&2 26 | 27 | success() { 28 | printf '%b\n' "" 29 | printf '%b\n' "\033[1;32m[SUCCESS] $@\033[0m" 30 | printf '%b\n' "" 31 | } 32 | 33 | lookForImage() { 34 | local IMAGE_LIST=$(docker images | awk '{print $1}') 35 | local IMAGE_FOUND="false" 36 | 37 | for image in $IMAGE_LIST 38 | do 39 | if [ $image = $IMAGE_NAME ]; then 40 | IMAGE_FOUND="true" 41 | fi 42 | done 43 | 44 | echo $IMAGE_FOUND 45 | } 46 | 47 | #------------------ 48 | # SCRIPT ENTRYPOINT 49 | #------------------ 50 | 51 | found=$(lookForImage) 52 | if [ $found = "true" ]; then 53 | err ""$IMAGE_NAME" does exist, cannot build" 54 | fi 55 | 56 | printf '%b\n' "" 57 | printf '%b\n' ":: Building image..." 58 | 59 | docker build -t ${IMAGE_NAME} . 60 | 61 | found=$(lookForImage) 62 | if [ $found = "false" ]; then 63 | err ""$IMAGE_NAME" not found, build failed" 64 | fi 65 | 66 | success "Image build successfully." 67 | 68 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.8 2 | LABEL image="blazemeter-logrotate" 3 | LABEL MAINTAINER="Blazemeter devops team" 4 | 5 | # logrotate version (e.g. 3.9.1-r0) 6 | ARG LOGROTATE_VERSION=latest 7 | # permissions 8 | ARG CONTAINER_UID=1000 9 | ARG CONTAINER_GID=1000 10 | 11 | # install dev tools 12 | RUN export CONTAINER_USER=logrotate && \ 13 | export CONTAINER_GROUP=logrotate && \ 14 | addgroup -g $CONTAINER_GID logrotate && \ 15 | adduser -u $CONTAINER_UID -G logrotate -h /usr/bin/logrotate.d -s /bin/bash -S logrotate && \ 16 | apk add --update \ 17 | tar \ 18 | gzip \ 19 | tini \ 20 | wget \ 21 | bash \ 22 | tzdata && \ 23 | if [ "${LOGROTATE_VERSION}" = "latest" ]; \ 24 | then apk add logrotate ; \ 25 | else apk add "logrotate=${LOGROTATE_VERSION}" ; \ 26 | fi && \ 27 | mkdir -p /usr/bin/logrotate.d && \ 28 | wget --no-check-certificate -O /tmp/go-cron.tar.gz https://github.com/michaloo/go-cron/releases/download/v0.0.2/go-cron.tar.gz && \ 29 | tar xvf /tmp/go-cron.tar.gz -C /usr/bin && \ 30 | apk del \ 31 | wget && \ 32 | rm -rf /var/cache/apk/* && rm -rf /tmp/* 33 | 34 | # environment variable for this container 35 | ENV LOGROTATE_OLDDIR= \ 36 | LOGROTATE_COMPRESSION= \ 37 | LOGROTATE_INTERVAL= \ 38 | LOGROTATE_COPIES= \ 39 | LOGROTATE_SIZE= \ 40 | LOGS_DIRECTORIES= \ 41 | LOG_FILE_ENDINGS= \ 42 | LOGROTATE_LOGFILE= \ 43 | LOGROTATE_CRONSCHEDULE= \ 44 | LOGROTATE_PARAMETERS= \ 45 | LOGROTATE_STATUSFILE= \ 46 | LOG_FILE= 47 | 48 | COPY docker-entrypoint.sh /usr/bin/logrotate.d/docker-entrypoint.sh 49 | COPY update-logrotate.sh /usr/bin/logrotate.d/update-logrotate.sh 50 | COPY logrotate.sh /usr/bin/logrotate.d/logrotate.sh 51 | COPY logrotateConf.sh /usr/bin/logrotate.d/logrotateConf.sh 52 | COPY logrotateCreateConf.sh /usr/bin/logrotate.d/logrotateCreateConf.sh 53 | 54 | USER root 55 | ENTRYPOINT ["/sbin/tini","--","/usr/bin/logrotate.d/docker-entrypoint.sh"] 56 | VOLUME ["/logrotate-status"] 57 | CMD ["cron"] 58 | -------------------------------------------------------------------------------- /docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # A helper script for ENTRYPOINT. 4 | 5 | set -e 6 | 7 | [[ ${DEBUG} == true ]] && set -x 8 | 9 | source /usr/bin/logrotate.d/logrotate.sh 10 | source /usr/bin/logrotate.d/logrotateConf.sh 11 | 12 | resetConfigurationFile 13 | 14 | if [ -n "${DELAYED_START}" ]; then 15 | sleep ${DELAYED_START} 16 | fi 17 | 18 | #Create Logrotate Conf 19 | source /usr/bin/logrotate.d/logrotateCreateConf.sh 20 | 21 | cat /usr/bin/logrotate.d/logrotate.conf 22 | 23 | # ----- Crontab Generation ------ 24 | 25 | logrotate_parameters="" 26 | 27 | if [ -n "${LOGROTATE_PARAMETERS}" ]; then 28 | logrotate_parameters="-"${LOGROTATE_PARAMETERS} 29 | fi 30 | 31 | logrotate_cronlog="" 32 | 33 | if [ -n "${LOGROTATE_LOGFILE}" ] && [ -z "${SYSLOGGER}"]; then 34 | logrotate_cronlog=" 2>&1 | tee -a "${LOGROTATE_LOGFILE} 35 | else 36 | if [ -n "${SYSLOGGER}" ]; then 37 | logrotate_cronlog=" 2>&1 | "${syslogger_command} 38 | fi 39 | fi 40 | 41 | logrotate_croninterval="1 0 0 * * *" 42 | 43 | if [ -n "${LOGROTATE_INTERVAL}" ]; then 44 | case "$LOGROTATE_INTERVAL" in 45 | hourly) 46 | logrotate_croninterval='@hourly' 47 | ;; 48 | daily) 49 | logrotate_croninterval='@daily' 50 | ;; 51 | weekly) 52 | logrotate_croninterval='@weekly' 53 | ;; 54 | monthly) 55 | logrotate_croninterval='@monthly' 56 | ;; 57 | yearly) 58 | logrotate_croninterval='@yearly' 59 | ;; 60 | *) 61 | logrotate_croninterval="1 0 0 * * *" 62 | ;; 63 | esac 64 | fi 65 | 66 | if [ -n "${LOGROTATE_CRONSCHEDULE}" ]; then 67 | logrotate_croninterval=${LOGROTATE_CRONSCHEDULE} 68 | fi 69 | 70 | logrotate_cron_timetable="/usr/sbin/logrotate ${logrotate_parameters} --state=${logrotate_logstatus} /usr/bin/logrotate.d/logrotate.conf ${logrotate_cronlog}" 71 | 72 | # ----- Cron Start ------ 73 | 74 | if [ "$1" = 'cron' ]; then 75 | if [ ${logrotate_autoupdate} = "true" ]; then 76 | exec /usr/bin/go-cron "${logrotate_croninterval}" /bin/bash -c "/usr/bin/logrotate.d/update-logrotate.sh; ${logrotate_cron_timetable}" 77 | exit 78 | fi 79 | 80 | exec /usr/bin/go-cron "${logrotate_croninterval}" /bin/bash -c "${logrotate_cron_timetable}" 81 | fi 82 | 83 | #----------------------- 84 | 85 | exec "$@" 86 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build_branch: 4 | working_directory: /app 5 | docker: 6 | - image: docker:17.05.0-ce-git 7 | steps: 8 | - checkout 9 | - setup_remote_docker 10 | - run: 11 | name: Install dependencies 12 | command: | 13 | apk add --no-cache \ 14 | py-pip \ 15 | curl \ 16 | curl-dev 17 | pip install \ 18 | docker-compose 19 | - run: 20 | name: Build Branch Images 21 | command: | 22 | set +o pipefail 23 | sh buildscripts/buildSupportedAlpineImages.sh 24 | - run: 25 | name: Test Branch Images 26 | command: | 27 | sh buildscripts/testSupportedAlpineImages.sh 28 | 29 | build_releases: 30 | working_directory: /app 31 | docker: 32 | - image: docker:17.05.0-ce-git 33 | steps: 34 | - checkout 35 | - setup_remote_docker 36 | - run: 37 | name: Install dependencies 38 | command: | 39 | apk add --no-cache \ 40 | py-pip \ 41 | curl \ 42 | curl-dev 43 | pip install \ 44 | docker-compose 45 | - run: 46 | name: Build Docker Release Images 47 | command: | 48 | set +o pipefail 49 | sh buildscripts/buildSupportedAlpineImages.sh 50 | - run: 51 | name: Test Docker Release Images 52 | command: | 53 | sh buildscripts/testSupportedAlpineImages.sh 54 | - run: 55 | name: Push Dockerhub Release Images 56 | command: | 57 | docker login -u ${DOCKER_USER} -p ${DOCKER_PASS} 58 | sh buildscripts/releaseSupportedAlpineImages.sh 59 | 60 | workflows: 61 | version: 2 62 | build_pipeline: 63 | jobs: 64 | - build_releases: 65 | filters: 66 | branches: 67 | only: master 68 | - build_branch: 69 | filters: 70 | branches: 71 | ignore: 72 | - master 73 | nightly: 74 | triggers: 75 | - schedule: 76 | cron: "0 1 * * 1" 77 | filters: 78 | branches: 79 | only: 80 | - master 81 | jobs: 82 | - build_releases 83 | -------------------------------------------------------------------------------- /logrotateCreateConf.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Creation of Logfile 4 | 5 | function handleSingleFile() { 6 | local singleFile="$1" 7 | local file_owner_user=$(stat -c %U ${singleFile}) 8 | local file_owner_group=$(stat -c %G ${singleFile}) 9 | local new_logrotate_entry=$(createLogrotateConfigurationEntry "${singleFile}" "${file_owner_user}" "${file_owner_group}" "${logrotate_copies}" "${logrotate_logfile_compression}" "${logrotate_logfile_compression_delay}" "${logrotate_mode}" "${logrotate_interval}" "${logrotate_size}" "${logrotate_dateformat}" "${logrotate_maxsize}" "${logrotate_minsize}" "${logrotate_maxage}" "${logrotate_prerotate}" "${logrotate_postrotate}") 10 | echo "Inserting new ${singleFile} to /usr/bin/logrotate.d/logrotate.conf" 11 | insertConfigurationEntry "$new_logrotate_entry" "/usr/bin/logrotate.d/logrotate.conf" 12 | } 13 | 14 | # ----- Logfile Crawling ------ 15 | 16 | log_dirs="" 17 | 18 | if [ -n "${LOGS_DIRECTORIES}" ]; then 19 | log_dirs=${LOGS_DIRECTORIES} 20 | else 21 | log_dirs=${log_dir} 22 | fi 23 | 24 | logs_ending="log" 25 | LOGS_FILE_ENDINGS_INSTRUCTION="" 26 | 27 | if [ -n "${LOG_FILE_ENDINGS}" ]; then 28 | logs_ending=${LOG_FILE_ENDINGS} 29 | fi 30 | 31 | SAVEIFS=$IFS 32 | IFS=' ' 33 | COUNTER=0 34 | for ending in $logs_ending 35 | do 36 | if [ "$COUNTER" -eq "0" ]; then 37 | LOGS_FILE_ENDINGS_INSTRUCTION="$LOGS_FILE_ENDINGS_INSTRUCTION -iname "*.${ending}"" 38 | else 39 | LOGS_FILE_ENDINGS_INSTRUCTION="$LOGS_FILE_ENDINGS_INSTRUCTION -o -iname "*.${ending}"" 40 | fi 41 | let COUNTER=COUNTER+1 42 | done 43 | IFS=$SAVEIFS 44 | 45 | for d in ${log_dirs} 46 | do 47 | log_files=$(find ${d} -type f $LOGS_FILE_ENDINGS_INSTRUCTION) || continue 48 | for f in ${log_files}; 49 | do 50 | if [ -f "${f}" ]; then 51 | echo "Found new file $f, Processing..." 52 | handleSingleFile "$f" 53 | fi 54 | done 55 | done 56 | 57 | # ----- Take all Log in Subfolders ------ 58 | 59 | all_log_dirs="" 60 | 61 | if [ -n "${ALL_LOGS_DIRECTORIES}" ]; then 62 | all_log_dirs=${ALL_LOGS_DIRECTORIES} 63 | fi 64 | 65 | for d in ${all_log_dirs} 66 | do 67 | log_files=$(find ${d} -type f); 68 | for f in ${log_files}; 69 | do 70 | if [ -f "${f}" ]; then 71 | echo "Found new file $f, Processing..." 72 | handleSingleFile "$f" 73 | fi 74 | done 75 | done 76 | 77 | cat /usr/bin/logrotate.d/logrotate.conf 78 | -------------------------------------------------------------------------------- /logrotateConf.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Helper functions for manipulating logrotate configurationsfile 4 | 5 | function createLogrotateConfigurationEntry() { 6 | local file="$1" 7 | local file_user="$2" 8 | local file_owner="$3" 9 | local conf_copies="$4" 10 | local conf_logfile_compression="$5" 11 | local conf_logfile_compression_delay="$6" 12 | local conf_logrotate_mode="$7" 13 | local conf_logrotate_interval="$8" 14 | local conf_logrotate_size="$9" 15 | local conf_dateformat="${10}" 16 | local conf_maxsize="${11}" 17 | local conf_minsize="${12}" 18 | local conf_maxage="${13}" 19 | local conf_prerotate="${14}" 20 | local conf_postrotate="${15}" 21 | local new_log= 22 | new_log=${file}" {" 23 | if [ "$file_user" != "UNKNOWN" ] && [ "$file_owner" != "UNKNOWN" ]; then 24 | new_log=${new_log}"\n su ${file_user} ${file_owner}" 25 | fi 26 | new_log=${new_log}"\n rotate ${conf_copies}" 27 | new_log=${new_log}"\n missingok" 28 | if [ -n "${conf_logfile_compression}" ]; then 29 | new_log=${new_log}"\n ${conf_logfile_compression}" 30 | fi 31 | if [ -n "${conf_logfile_compression_delay}" ]; then 32 | new_log=${new_log}"\n ${conf_logfile_compression_delay}" 33 | fi 34 | if [ -n "${conf_logrotate_mode}" ]; then 35 | new_log=${new_log}"\n ${conf_logrotate_mode}" 36 | fi 37 | if [ -n "${conf_logrotate_interval}" ]; then 38 | new_log=${new_log}"\n ${conf_logrotate_interval}" 39 | fi 40 | if [ -n "${conf_logrotate_size}" ]; then 41 | new_log=${new_log}"\n ${conf_logrotate_size}" 42 | fi 43 | if [ -n "${conf_maxsize}" ]; then 44 | new_log=${new_log}"\n ${conf_maxsize}" 45 | fi 46 | if [ -n "${conf_minsize}" ]; then 47 | new_log=${new_log}"\n ${conf_minsize}" 48 | fi 49 | if [ -n "${conf_maxage}" ]; then 50 | new_log=${new_log}"\n ${conf_maxage}" 51 | fi 52 | if [ -n "${conf_dateformat}" ]; then 53 | new_log=${new_log}"\n dateext\n dateformat ${conf_dateformat}" 54 | fi 55 | if [ -n "${conf_prerotate}" ]; then 56 | new_log=${new_log}"\n prerotate" 57 | new_log=${new_log}"\n\t${conf_prerotate}" 58 | new_log=${new_log}"\n endscript" 59 | fi 60 | if [ -n "${conf_postrotate}" ]; then 61 | new_log=${new_log}"\n postrotate" 62 | new_log=${new_log}"\n\t${conf_postrotate}" 63 | new_log=${new_log}"\n endscript" 64 | fi 65 | new_log=${new_log}"\n}" 66 | echo -e $new_log 67 | } 68 | 69 | function insertConfigurationEntry() 70 | { 71 | local config=$1 72 | local config_file=$2 73 | 74 | cat >> $config_file <<_EOF_ 75 | ${config} 76 | _EOF_ 77 | } 78 | -------------------------------------------------------------------------------- /logrotate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Helper functions for configuration and running logrotate. 4 | 5 | # Resetting the default configuration file for 6 | # repeated starts. 7 | function resetConfigurationFile() { 8 | if [ -f "/usr/bin/logrotate.d/logrotate.conf" ]; then 9 | rm -f /usr/bin/logrotate.d/logrotate.conf 10 | else 11 | touch /usr/bin/logrotate.d/logrotate.conf 12 | fi 13 | 14 | cat >> /usr/bin/logrotate.d/logrotate.conf < VERSION 50 | """ 51 | } 52 | } 53 | } 54 | stage('Build Image') { 55 | steps { 56 | script { 57 | docker.build(imageTagLatest, ' -f Dockerfile .') 58 | } 59 | } 60 | } 61 | stage('Deploy to GCR') { 62 | when { expression { params.TEST_JENKINS_BUILD == false } } 63 | steps { 64 | script { 65 | pushImageToAllRegistries(appName, imageTagLatest, dockerTag) 66 | } 67 | } 68 | } 69 | stage('Perform PrismaCloud scan') { 70 | when { allOf { 71 | expression { params.PERFORM_PRISMA_SCAN == true } 72 | expression { params.TEST_JENKINS_BUILD == false } 73 | } 74 | } 75 | steps { 76 | script { 77 | runPrismaCloudScan(fullImageName, imageTag, params.FAIL_JOB_ON_SCAN_FAILURES) 78 | } 79 | } 80 | } 81 | } 82 | post { 83 | always { 84 | smartSlackNotification(alternateJobTitle: 'blazemeter logrotate package build') 85 | script { 86 | PullRequestUtils.updateBranchPullRequestsStatuses(this) 87 | } 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dockerized Logrotate 2 | 3 | [![Open Issues](https://img.shields.io/github/issues/blacklabelops/logrotate.svg)](https://github.com/blacklabelops/logrotate/issues) [![Stars on GitHub](https://img.shields.io/github/stars/blacklabelops/logrotate.svg)](https://github.com/blacklabelops/logrotate/stargazers) 4 | [![Docker Stars](https://img.shields.io/docker/stars/blacklabelops/logrotate.svg)](https://hub.docker.com/r/blacklabelops/logrotate/) [![Docker Pulls](https://img.shields.io/docker/pulls/blacklabelops/logrotate.svg)](https://hub.docker.com/r/blacklabelops/logrotate/) [![](https://badge.imagelayers.io/blacklabelops/logrotate:latest.svg)](https://imagelayers.io/?images=blacklabelops/logrotate:latest 'Get your own badge on imagelayers.io') 5 | 6 | This container can crawl for logfiles and rotate them. It is a side-car container 7 | for containers that write logfiles and need a log rotation mechanism. Just hook up some containers and define your 8 | backup volumes. 9 | 10 | ## Supported tags and respective Dockerfile links 11 | 12 | | Distribution | Version | Tag | Dockerfile | 13 | |--------------|--------------|--------------|------------| 14 | | Logrotate Alpine | latest, 1.3 | latest, 1.3 | [Dockerfile](https://github.com/blacklabelops/logrotate/blob/master/Dockerfile) | 15 | 16 | ## Make It Short 17 | 18 | In short, this container can rotate all your Docker logfiles just by typing: 19 | 20 | ~~~~ 21 | $ docker run -d \ 22 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 23 | -v /var/log/docker:/var/log/docker \ 24 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 25 | blacklabelops/logrotate 26 | ~~~~ 27 | 28 | > This will rotate all your Docker logfiles on a daily basis up to 5 times. 29 | 30 | You want to do it hourly? Just type: 31 | 32 | ~~~~ 33 | $ docker run -d \ 34 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 35 | -v /var/log/docker:/var/log/docker \ 36 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 37 | -e "LOGROTATE_INTERVAL=hourly" \ 38 | blacklabelops/logrotate 39 | ~~~~ 40 | 41 | > This will put logrotate on an hourly schedule. 42 | 43 | ## How To Attach to Logs 44 | 45 | In order to attach the side-car container to your logs you have to hook your log file folders inside volumes. Afterwards 46 | specify the folders logrotate should crawl for log files. The container attaches by default to any file ending with **.log** inside the specified folders. 47 | 48 | Environment variable for specifying log folders: `LOGS_DIRECTORIES`. Each directory must be separated by a whitespace character. 49 | 50 | Example: 51 | 52 | ~~~~ 53 | LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker 54 | ~~~~ 55 | 56 | Example Logrotating all Docker logfiles: 57 | 58 | ~~~~ 59 | $ docker run -d \ 60 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 61 | -v /var/log/docker:/var/log/docker \ 62 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 63 | blacklabelops/logrotate 64 | ~~~~ 65 | 66 | > This will logrotate any logfile(s) under /var/lib/docker/containers, /var/log/docker (or subdirectories of them). 67 | 68 | ## Customize Log File Ending 69 | 70 | You can define the file endings fluentd will attach to. The container will by default crawl for 71 | files ending with **.log**. This can be overriden and extended to any amount of file endings. 72 | 73 | Example: 74 | 75 | ~~~~ 76 | $ docker run -d \ 77 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 78 | -v /var/log/docker:/var/log/docker \ 79 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 80 | -e "LOG_FILE_ENDINGS=json xml" \ 81 | blacklabelops/logrotate 82 | ~~~~ 83 | 84 | > Crawls for file endings .json and .xml. 85 | 86 | ## Set the Log interval 87 | 88 | Logrotate can rotate logfile according to the following intervals: 89 | 90 | * `hourly` 91 | * `daily` 92 | * `weekly` 93 | * `monthly` 94 | * `yearly` 95 | 96 | You can override the default setting with the environment variable `LOGROTATE_INTERVAL`. 97 | 98 | Example: 99 | 100 | ~~~~ 101 | $ docker run -d \ 102 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 103 | -v /var/log/docker:/var/log/docker \ 104 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 105 | -e "LOGROTATE_INTERVAL=hourly" \ 106 | blacklabelops/logrotate 107 | ~~~~ 108 | 109 | > This will logrotate logfile(s) on hourly basis. 110 | 111 | ## Set the Number of Rotations 112 | 113 | The default number of rotations is five. Further rotations will delete old logfiles. You 114 | can override the default setting with the environment variable `LOGROTATE_COPIES`. 115 | 116 | Example: 117 | 118 | ~~~~ 119 | $ docker run -d \ 120 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 121 | -v /var/log/docker:/var/log/docker \ 122 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 123 | -e "LOGROTATE_COPIES=10" \ 124 | blacklabelops/logrotate 125 | ~~~~ 126 | 127 | > Will create 10 daily logs before deleting old logs. 128 | 129 | ## Set Maximum File size 130 | 131 | Logrotate can do additional rotates, when the logfile exceeds a certain file size. You can specifiy file size rotation 132 | with the environment variable `LOGROTATE_SIZE`. 133 | 134 | Valid example values: 135 | 136 | * `100k`: Will rotate when log file exceeds 100 kilobytes. 137 | * `100M`: Will rotate when log file exceeds 100 Megabytes. 138 | * `100G`: Will rotate when log file exceeds 100 Gigabytes. 139 | 140 | ~~~~ 141 | $ docker run -d \ 142 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 143 | -v /var/log/docker:/var/log/docker \ 144 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 145 | -e "LOGROTATE_SIZE=10M" \ 146 | blacklabelops/logrotate 147 | ~~~~ 148 | 149 | > This will logrotate when logfile(s) reaches 10M+. 150 | 151 | ## Set Log File compression 152 | 153 | The default logrotate setting is `nocompress`. In order to enable logfile compression 154 | you can set the environment variable `LOGROTATE_COMPRESSION` to `compress`. 155 | 156 | Example: 157 | 158 | ~~~~ 159 | $ docker run -d \ 160 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 161 | -v /var/log/docker:/var/log/docker \ 162 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 163 | -e "LOGROTATE_COMPRESSION=compress" \ 164 | blacklabelops/logrotate 165 | ~~~~ 166 | 167 | > This will compress the logrotated logs. 168 | 169 | ## Turn Off Log File delaycompress 170 | 171 | When compression is turned on, delaycompress will be set by default. To turn this off, 172 | set the environment variable `LOGROTATE_DELAYCOMPRESS` to `false`. 173 | 174 | Example: 175 | ~~~~ 176 | $ docker run -d \ 177 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 178 | -v /var/log/docker:/var/log/docker \ 179 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 180 | -e "LOGROTATE_COMPRESSION=compress" \ 181 | -e "LOGROTATE_DELAYCOMPRESS=false" \ 182 | blacklabelops/logrotate 183 | ~~~~ 184 | 185 | > This will compress all logrotated logs, including the most recent one. 186 | 187 | ## Set logrotate mode 188 | 189 | By default, logrotate will use copytruncate mode to create a new rotated file, however 190 | certain log collection applications won't work properly with this configuration. To use a different 191 | option, such as `create `, set the environment variable `LOGROTATE_MODE`. 192 | 193 | Example: 194 | ~~~~ 195 | $ docker run -d \ 196 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 197 | -v /var/log/docker:/var/log/docker \ 198 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 199 | -e "LOGROTATE_MODE=create 0644" 200 | blacklabelops/logrotate 201 | ~~~~ 202 | 203 | > This will rename the current log file, and create a new one in its place 204 | 205 | ## Set the Output directory 206 | 207 | By default, logrotate will rotate logs in their respective directories. You can 208 | specify a directory for keeping old logfiles with the environment variable `LOGROTATE_OLDDIR`. You can specify a full or relative path. 209 | 210 | Example: 211 | 212 | ~~~~ 213 | $ docker run -d \ 214 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 215 | -v /var/log/docker:/var/log/docker \ 216 | -v $(pwd)/logs:/logs/ \ 217 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 218 | -e "LOGROTATE_OLDDIR=/logs" \ 219 | blacklabelops/logrotate 220 | ~~~~ 221 | 222 | > Will move old logfiles in the local directory logs/. 223 | 224 | ## Set the Cron Schedule 225 | 226 | You can set the cron schedule independently of the logrotate interval. You can override 227 | the default schedule with the enviroment variable `LOGROTATE_CRONSCHEDULE`. 228 | 229 | Example: 230 | 231 | ~~~~ 232 | $ docker run -d \ 233 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 234 | -v /var/log/docker:/var/log/docker \ 235 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 236 | -e "LOGROTATE_CRONSCHEDULE=* * * * * *" \ 237 | blacklabelops/logrotate 238 | ~~~~ 239 | 240 | > This will logrotate on go-cron schedule \* \* \* \* \* \* (every second). 241 | 242 | ## Log and View the Logrotate Output 243 | 244 | You can specify a logfile for the periodical logrotate execution. The file 245 | is specified using the environment variable `LOGROTATE_LOGFILE`. Must be a full path! 246 | 247 | Example: 248 | 249 | ~~~~ 250 | $ docker run -d \ 251 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 252 | -v /var/log/docker:/var/log/docker \ 253 | -v $(pwd)/logs:/logs \ 254 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 255 | -e "LOGROTATE_INTERVAL=hourly" \ 256 | -e "LOGROTATE_LOGFILE=/logs/logrotatecron.log" \ 257 | blacklabelops/logrotate 258 | ~~~~ 259 | 260 | > You will be able to see logrotate output every minute in file logs/logrotatecron.log. 261 | 262 | ## Logrotate Commandline Parameters 263 | 264 | You can define the logrotate commandline parameters with the environment variable LOGROTATE_PARAMETERS. 265 | 266 | *v*: Verbose 267 | 268 | *d*: Debug, Logrotate will be emulated but never executed! 269 | 270 | *f*: Force 271 | 272 | Example for a typical testrun: 273 | 274 | ~~~~ 275 | $ docker run -d \ 276 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 277 | -v /var/log/docker:/var/log/docker \ 278 | -v $(pwd)/logs:/logs \ 279 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 280 | -e "LOGROTATE_PARAMETERS=vdf" \ 281 | -e "LOG_FILE=/logs/cron.log" \ 282 | blacklabelops/logrotate 283 | ~~~~ 284 | 285 | > Will run logrotate with: /usr/bin/logrotate -dvf 286 | 287 | ## Logrotate Status File 288 | 289 | Logrotate must remember when files have been rotated when using time intervals, e.g. 'daily'. The status file will be written by default to the container volume but you can specify a custom location with the environment variable LOGROTATE_STATUSFILE. 290 | 291 | Example: 292 | 293 | ~~~~ 294 | $ docker run -d \ 295 | -e "LOGROTATE_INTERVAL=hourly" \ 296 | -e "LOGROTATE_STATUSFILE=/logrotate-status/logrotate.status" \ 297 | -e "ALL_LOGS_DIRECTORIES=/var/log" \ 298 | -e "LOGROTATE_PARAMETERS=vf" \ 299 | blacklabelops/logrotate 300 | ~~~~ 301 | 302 | > Writes the latest status file each logrotation. Reads status files at each start. 303 | 304 | ## Log and View the Cron Output 305 | 306 | You can specify a separate logfile for cron. The file 307 | is specified using the environment variable `LOG_FILE`. Must be a full path! 308 | 309 | Example: 310 | 311 | ~~~~ 312 | $ docker run -d \ 313 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 314 | -v /var/log/docker:/var/log/docker \ 315 | -v $(pwd)/logs:/logs \ 316 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 317 | -e "LOGROTATE_INTERVAL=hourly" \ 318 | -e "LOG_FILE=/logs/cron.log" \ 319 | blacklabelops/logrotate 320 | ~~~~ 321 | 322 | > You will be able to see cron output every minute in file logs/cron.log. 323 | 324 | ## Setting a Date Extension 325 | 326 | With Logrotate it is possible to split files and name them by the date they were generated when used with `LOGROTATE_DATEFORMAT`. By setting `LOGROTATE_DATEFORMAT` you will enable the Logrotate `dateext` option. 327 | 328 | The default Logrotate format is `-%Y%m%d`, to enable the defaults `LOGROTATE_DATEFORMAT` should be set to this. 329 | 330 | Example: 331 | 332 | ~~~~ 333 | $ docker run -d \ 334 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 335 | -v /var/log/docker:/var/log/docker \ 336 | -e "LOGROTATE_INTERVAL=daily" \ 337 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 338 | -e "LOGROTATE_DATEFORMAT=-%Y%m%d" \ 339 | blacklabelops/logrotate 340 | ~~~~ 341 | 342 | > This will set logrotate to split files and name them by date format -%Y%m%d. 343 | 344 | ## Setting MaxAge and minsize 345 | 346 | Maxage , maxsize and minsize for logs can be configured with the environment variables `LOGROTATE_MAXAGE` `LOGROTATE_MAXSIZE` and `LOGROTATE_MINSIZE`. 347 | 348 | * Maxage: `Remove rotated logs older than days. The age is only checked if the logfile is to be rotated.` 349 | 350 | * Maxsize: `Log files are rotated when they grow bigger than size bytes even before the additionally specified time interval (daily, weekly, monthly, or yearly). 351 | The related size option is similar except that it is mutually exclusive with the time interval options, and it causes 352 | log files to be rotated without regard for the last rotation time. When maxsize is used, both the size and timestamp of a log file are considered.` 353 | 354 | * Minsize: `Log files are rotated when they grow bigger than size bytes, but not before the additionally specified time interval (daily, weekly, monthly, or yearly). The related size option is similar except that it is mutually exclusive with the time interval options, and it causes log files to be rotated without regard for the last rotation time. When minsize is used, both the size and timestamp of a log file are considered.` 355 | 356 | > [Source](http://manpages.ubuntu.com/manpages/yakkety/man8/logrotate.8.html) 357 | 358 | Size Parameter: `If size is followed by k, the size is assumed to be in kilo-bytes. If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G are all valid.` 359 | 360 | > [Source](http://manpages.ubuntu.com/manpages/yakkety/man8/logrotate.8.html) 361 | 362 | Example: 363 | 364 | ~~~~ 365 | $ docker run -d \ 366 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 367 | -v /var/log/docker:/var/log/docker \ 368 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 369 | -e "LOGROTATE_MAXAGE=60" \ 370 | -e "LOGROTATE_MINSIZE=100k" \ 371 | blacklabelops/logrotate 372 | ~~~~ 373 | 374 | > Maxage is sixty days and minsize is 100 kilobytes. 375 | 376 | ## Custom Skript execution 377 | 378 | Sometimes it is necessary to signal the process in order to start logrotation or stop logrotation. You 379 | can do this with the environment variables `LOGROTATE_PREROTATE_COMMAND` and `LOGROTATE_POSTROTATE_COMMAND`. 380 | 381 | Example: 382 | 383 | ~~~~ 384 | $ docker run -d \ 385 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 386 | -v /var/log/docker:/var/log/docker \ 387 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 388 | -e "LOGROTATE_PREROTATE_COMMAND=/usr/bin/yourscript.sh" \ 389 | -e "LOGROTATE_POSTROTATE_COMMAND=/usr/bin/killall -HUP httpd" \ 390 | blacklabelops/logrotate 391 | ~~~~ 392 | 393 | > Will print messages before and after rotation. 394 | 395 | ## Disable Auto Update 396 | 397 | With Logrotate by default it auto update its logrotate configuration file to ensure it only captures all the intended log file in the `LOGS_DIRECTORIES` (before it rotates the log files). It is possible to disable auto update when used with `LOGROTATE_AUTOUPDATE`. By setting `LOGROTATE_AUTOUPDATE` (to not equal true) you will disable the auto update of Logrotate. 398 | 399 | The default `LOGROTATE_AUTOUPDATE` is `true`, to disable the defaults `LOGROTATE_AUTOUPDATE` should be set not equal `true`. 400 | 401 | Example: 402 | 403 | ~~~~ 404 | $ docker run -d \ 405 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 406 | -v /var/log/docker:/var/log/docker \ 407 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 408 | -e "LOGROTATE_AUTOUPDATE=false" \ 409 | blacklabelops/logrotate 410 | ~~~~ 411 | 412 | > This will disable logrotate configuration file update (when logrotate action is triggering). 413 | 414 | # Set Time Zone 415 | 416 | With Logrotate by default it logrotate logs in `UTC` time zone. It is possible to set time zone when used with `TZ`. By setting `TZ` (to a valid time zone) it will logrotate logs in the specified time zone. 417 | 418 | The default `TZ` is `""`, to set to different time zone. E.g `Australia/Melbourne`. 419 | 420 | Example: 421 | 422 | ~~~~ 423 | $ docker run -d \ 424 | -v /var/lib/docker/containers:/var/lib/docker/containers \ 425 | -v /var/log/docker:/var/log/docker \ 426 | -e "LOGS_DIRECTORIES=/var/lib/docker/containers /var/log/docker" \ 427 | -e "TZ=Australia/Melbourne" \ 428 | blacklabelops/logrotate 429 | ~~~~ 430 | 431 | > This will logrotate in Australia/Melbourne time zone. 432 | 433 | ## Used in Kubernetes 434 | 435 | When we run container in Kubernetes, we can use the logrotate container to rotate the logs. As we create 436 | 437 | an DaemonSet in cluster ,we can deploy an logrotate container in every nodes of the cluster. 438 | 439 | ```sh 440 | # kubectl create -f logrotate_ds.yaml 441 | daemonset "logrotate" created 442 | ``` 443 | 444 | ## References 445 | 446 | * [Logrotate](http://www.linuxcommand.org/man_pages/logrotate8.html) 447 | * [Docker Homepage](https://www.docker.com/) 448 | * [Docker Userguide](https://docs.docker.com/userguide/) 449 | --------------------------------------------------------------------------------