├── LICENSE ├── __init__.py ├── changelog ├── convert.py ├── db.py ├── docs ├── CommandLine.md ├── Config-SrcDestPairs.md ├── DataStructures │ ├── ConfigFormat.json │ ├── ReportFormat.json │ └── fldDefs.txt ├── Installation.md ├── QuickStart.md ├── RcFileConfig-Apprise.md ├── RcFileConfig-EmailManagement.md ├── RcFileConfig-Main.md ├── RcFileConfig-SourceDestination.md ├── RcFileConfig.md ├── Reporting-CustomReportSpec.md ├── Reporting-ReportSection.md ├── SystemRequirements.md ├── WhatIsDupreport.md ├── images │ ├── SampleColumnSpec.jpg │ ├── TitleExample.jpg │ ├── dR_Architecture.jpg │ ├── interval_example.jpg │ ├── last_date_table.jpg │ ├── last_seen_line.jpg │ ├── report_bydate.jpg │ ├── report_bydest.jpg │ ├── report_bysource.jpg │ ├── report_noactivity.jpg │ ├── report_offline.jpg │ ├── report_offline_notitles.jpg │ ├── report_srcdest.jpg │ └── runtime_line.jpg └── readme.md ├── drdatetime.py ├── dremail.py ├── dupReport.py ├── dupReport.rc.EXAMPLE ├── dupapprise.py ├── globs.py ├── log.py ├── options.py └── report.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /changelog: -------------------------------------------------------------------------------- 1 | 3.0.10 2 | ----- 3 | - Read options file using ConfigParser() instead of SafeConfigParser(), which has been deprecated 4 | - Update regex strings to avoid syntax errors 5 | - Renamed primary Git branch from "master" to "main" 6 | 7 | 3.0.9 8 | ----- 9 | - Fix bug that reported a backup as past the nobackupwarning period even if it was still within its backup interval (Issue #180) 10 | - Fixed typo in documentation (Issue #181) 11 | - Added ignore= option to the [source-destination] sections in the .rc file See docs for more information on how this works. (Issue #178) 12 | 13 | 3.0.8 14 | ----- 15 | - Fix crashing error when subject line parsing fails (Issue #177) 16 | - Rewrote documentation to describe advanced regex options for source, destination, and delimiter 17 | - Updated GitHub home page 18 | 19 | 3.0.7 20 | ----- 21 | - Fixed parsing routines to allow for spaces in the source/destination/delimiter specification (Issue #174 - Thanks ekutner) 22 | - Updated documentaiton for source/destination/delimiter specification 23 | - Updated documentaiton to clarify the use of "enable less secure apps" on Gmail and othr email services (Issue #175) 24 | 25 | 26 | 3.0.6 27 | ----- 28 | - Added the ability to indicate job status (Success/Warning/Error/Failure) in the email subject line. See Report Section docs for more information. (Issue #172) 29 | - Updates example .rc file with new options 30 | - Fixed errors/typos in documentation 31 | 32 | 3.0.5 33 | ----- 34 | - Fixed bug that caused "smtplib.SMTPDataError: (501, b'Syntax error - line too long')" error when using GMX email servers. (Issue #166) 35 | - Fixed issue where program would crash if no valid SMTP server was connected (probably because of bad login credentials).(Issue #169) 36 | - Updated description for -v option 37 | - Fixed typos in [report] section documentation 38 | 39 | 3.0.4 40 | ----- 41 | - Fixed bug where program would crash is using JSON emails and email contained an error message. (Issue #164) 42 | 43 | 3.0.3 44 | ----- 45 | - Fixed bug where guided setup was converting inbox folder name to all uppercase (Issue #163) 46 | 47 | 3.0.2 48 | ----- 49 | - Added BytesUploaded & BytesDownload fields from JSON emails to database & reports. (Issue #157) 50 | - Added OS verison to startup log info (Issue #162) 51 | - Updated example .rc file with new options 52 | 53 | 3.0.1 54 | ------ 55 | - Made offline backup sets more visible by adding an "offline" report style, adding an [OFFLINE] indicator to the noactivity and lastseen reports, and ading a 'showoffline' option to the [report] section (Issue #156) 56 | 57 | 3.0.0 58 | ------ 59 | - Re-built reporting engine to allow for user-created reporting formats (Issue #130) 60 | - Added logic to validate custom reports before they are run 61 | - Added -o commandline option to validate the report configuration in the .rc file and exit the program. Useful when developing custom reports. 62 | - Added -y commandline option to specify which reports to run at runtime. This enables more dynamic report running 63 | - Reports can use default options (from [report] section) or customize those options per report 64 | - Created separate "no activity" report instead of including that in the standard report output. This can now be added or left off the report run at will 65 | - Created separate "lastseen" report to display the last seen backup date & time for each backupset. Replaces the [report]lastseen= option in the .rc file 66 | - Added Guided Setup for new users. If there is no .rc file when the program runs the guided Setup will take the user through the most common configurable options. -g command line option forces running of Guided Setup. -G suppresses running. (Issue #139) 67 | - Database, .rc file, and log file specification can now be either directory names or full path specifications (Issue #136) 68 | - Ability to specify multiple inbound (IMAP/POP3) and outbound (SMTP) email servers. Specified by [main]emailservers= option or -e commandline option. 69 | - Added ability to send output to syslog server or log aggregator (Issue #128) 70 | - Can now send output to JSON file (Issue #135) 71 | - Added option ([report]failedonly=) that only prints unsuccessful backup jobs (Parsed Result != 'Success') (Issue #142) 72 | - Changed displaysize option in .rc file from (mega/giga) to (none/mb/gb), deprecated the showdisplaysize option, and changed the -s option to -s none|mb|gb 73 | - Re-structured the documentation for better readibility and made it easier to find specific settings. (Issue #140) 74 | - Standardized log format for easier searching and organization (Issue #137) 75 | - Fixed long-standing bug in handling Content Transfer Encoded messages (Issue #141) 76 | - Added ability to rollback (-b and -B) to a relative time (e.g., 1w,3h) instead of an absolute date time (i.e., "04/11/2020 8:00:00") (Issue #131) 77 | - Fixed bug preventing recording of last version for a failed backup run (Issue #138) 78 | - Fixed bug where 'Limited' message/warning/error fields were not being displayed properly (Issue #147) 79 | - Fixed bug where temporary log file would sometimes not be copied to permanent log file. (Issue #148) 80 | - Created a sample .rc file (dupReport.rc.EXAMPLE) to show how various configuration options work in a real-world scenario (Issue #149) 81 | - Added [apprise] section to .rc file by default (instead of previous optional status) and made it easier for users to enable/disable it at will (Issue #150) 82 | 83 | 84 | 2.2.12 85 | ------ 86 | - Fix email parsing bug where the message uses Content Transfer Encoding (Issue #141) 87 | 88 | 2.2.11 89 | ------ 90 | - Fix SMTP connection bug for some email services (Issue #129) 91 | 92 | 2.2.10 93 | ------ 94 | - Try again to fix obscure parsing bug when job messages contain quotes (Issue #126) 95 | 96 | 2.2.9 97 | ----- 98 | - Fix obscure parsing bug when job messages contain quotes (Issue #126) 99 | 100 | 2.2.8 101 | ----- 102 | - Properly handle reports where Duplicati version is not parsed from emails (Issue #123) 103 | - Added [incoming]unreadonly option to only process unread messages. Will dramatically speed up processing. (Issue #124) 104 | 105 | 2.2.7 106 | ----- 107 | - Properly parse "Log data" field, newly introduced in Duplicati circa November 2018 (Issue #112) 108 | - Parse JSON-formatted emails from Duplicati job runs (Issue #113) 109 | - Added "Duplicati version" field option to report. Can be helpful in discovering when one of your systems is behind in Duplicati updates (Issue #114) 110 | - Added -F option. Same as -f, but also sends resulting output file as an attachment to the email. (Issue #115) 111 | - Updated database schema, now on version 1.0.3. Program will automaticaly upgrade older DB versions to the new format. 112 | - Fixed weird .rc file version numbering system. (Issue #120) 113 | - Added Duplicati version column to "Last Seen" report table (Issue #121) 114 | 115 | 2.2.6 116 | ----- 117 | - Enable setting emails to read/seen after processing - IMAP only (Issue #111) 118 | 119 | 2.2.5 120 | ----- 121 | - Fix POP3 headers to convert them to IMAP style for processing (Issue #105) 122 | - Fix several stupid spelling errors (Issue #106 & others) 123 | - Fix issue where POP3 server would fail keepalive check due to bad comparison of return from server.noop() call (Issue #107) 124 | - Updated documents to include advice for setting POP3 "Leave messages on servr" option (Issue #107) 125 | - Option to mask sensitive data from log files: [main] masksensitive (Issue #109) 126 | - Fix issue where POP3 email causes log writing to crash system (Issue #108) 127 | 128 | 2.2.4 129 | ----- 130 | - Added backup duration to reports (Issue #102) 131 | - Altered database schema to accommodate backup duration in reports. Will require (automatic) DB upgrade. BACKUP YOUR DATABSE AND .RC FILE BEFORE EXECUTING! 132 | - Added durationzeroes option in [report] section to determine if zero units should be displayed in the time duration 133 | - Updated report footer to add program version number 134 | - Fixed an issue where the colon (:) is used as part of the subject line. (Issue #104) 135 | 136 | 2.2.3 137 | ----- 138 | - Fixed bug where "Not Seen" email was being sent even if backupset was listed as offline (Issue #94) 139 | - Fixed Unhandled Exception bug on CloseEverythingAndExit() 140 | - Added feature to truncate message, warning, and error fields generated by backup jobs (Issue #97) 141 | - Added support for Apprise notifications (https://github.com/caronc/apprise) 142 | - If you specify -x on the command line you no longer need to specify valid outgong email server and account information in the .rc file. 143 | However, if you still want to send backup warning emails (i.e. you don't use -w) you still need to enter valid server and account information for the outgoing server. (Issue #99) 144 | - Fixed improper program termination method (Issue #98) 145 | 146 | 2.2.2 147 | ----- 148 | - Added option to specify backup cycle other than 1 day (Issue 91) 149 | 150 | 2.2.1 151 | ----- 152 | - Added "offline=" option in [source-destination] section to suppress "not seen" message in report (issue #88) 153 | - Addressed issue #90 154 | 155 | 2.2.0 156 | ----- 157 | - Optimize email retrieval by analyzing headers first before downloading entire email. Seeing 40%-60% reduction in running time in tests 158 | - Added options to use keepalive logic for larger inboxes to prevent timeout errors([incoming] inkeepalive= and [outgoing] outkeepalive= options in.rc file) 159 | - Added 'date' header to outgoing emails for RFC compliance (issue #77) 160 | - Added optional progress indicator to stdout to show emails are being read (issue #72) 161 | - Added option to add a "last seen date" summary table for all backup sets on report (issue #73 - [report]lastseen* options in .rc file) 162 | - Added logic to better parse incoming dates to catch simple delimiter errors (issue # 83) 163 | - [main] applyutcoffset in .rc file is now set to 'true' by default on new installations. Avoids negative date different problems (Issue #84) 164 | - [report] noactivitybg has been deprecated and is no longer used in the code. Background highlighting of unseen backups in HTML reports is now guided by the [report] lastseen* options in the .rc file 165 | 166 | 167 | 2.1.0 168 | ----- 169 | - Large-scale rewrite of code 170 | - Migrated to a python class-based structure and split code into multiple modules. Will make the 171 | code much easier to maintain going forward. 172 | - Changed from comparing date/time strings to using timestamps. Makes date comparisons and 173 | time tracking much simpler. 174 | - Simpler date/time format matching. Should eliminate (all?) local date/time format issues from 2.0.x series. 175 | - Added new reports to organize backup jobs by source, destination, and run date 176 | - Fixed and standardized internal date and time comparison processes by using timestamps everywhere in the database instead of date/time strings. 177 | - Easy modification of date/time formats for international use (dateformat= and timeformat= .rc options) 178 | - Date/time format can be specified per src/dest pair, if jobs are running in different locales. 179 | - Dates can now be displayed in 12- or 24-hour format (display24hourtime= .rc option) 180 | - Report can now use keyword substitution for subheading titles 181 | - Reports can now be sent to one or more files in HTML, txt, or csv formats (-f command line option) 182 | - Can prevent sending report email if only saving to output file (-x command line option) 183 | - Option to email job warnings/errors if in collect mode (-c command line option, warnoncollect= .rc option) 184 | - Column headings can be customized ([headings] section in .rc file) 185 | - User can select which columns to display in a report ([headings] section in .rc file) 186 | - Can specify background colors for subtitles, messages, warnings, and errors 187 | - UTC offset information from email header now applied to backup endDate and startDate fields (applyutcoffset= .rc option) 188 | - UTC timestamp now recognized in startDate & endDate fields from Duplicati emails (feature currently unused, prepping for upcoming Duplicati release feature) 189 | - Database & .rc file upgrades now handled automatically 190 | - Log information generated before log files are opened is now captured and saved 191 | - Using HTML tags in message/warning/error output to clean up the HTML report. Not supported in all browsers or email clients, so end user experience may vary. 192 | - Ability to roll back database to a specific time/date (-b and -B options). Useful for failed runs or testing. 193 | - Ability to remove a source/destination pair from the database if it is no longer in use (-m option) 194 | - Purge emails that are no longer on the server from the database. (-p option or [main] purgedb in .rc file) 195 | - Send separate warning email when a backupset has not been seen in a certain number of days ([report] nobackupwarn options) 196 | - Added "Friendly Name" support to outgoing emails 197 | - Send report and warning emails to multiple recipients 198 | - Changed bahavior of -i option (again!). After initialization, if program can continue, it will. Also eliminated undocumented -I option. 199 | 200 | 201 | 2.0.4 202 | ----- 203 | Issue #18, #19, #21 - Several iterations of fixes for these issues. Should now be able to handle most 204 | global date formats. Probably. 205 | 206 | 2.0.3 207 | ----- 208 | Issue #1 - Initialization with -i option will now always stop program after initialization. 209 | Non-critical changes/updates to the .rc file will now allow the program to continue 210 | with default option configs. 211 | Issue #7 - Fixed the way failed backup jobs are parsed & reported. 212 | Issue #8 - Flush file buffer after every log write 213 | Issue #10 - Added ability to sort the report by source or destination 214 | Issue #12 - Added level 3 logging and changed logging scheme 215 | Issue #13 - Fixed 24-hour date conversion errors 216 | Issue #14 - Deal with binary-encoded messages from mutant email senders. 217 | Non-Specific: 218 | - general code cleanup and beautification 219 | 220 | 221 | 2.0.2 222 | ----- 223 | Issue #5: 224 | - Subject found but no delimiter found. Program now abandons message with a level 2 log message. 225 | - Regular expression used as delimiter character. Delimiter character now escaped by using re.escape() around variable 226 | -------------------------------------------------------------------------------- /convert.py: -------------------------------------------------------------------------------- 1 | ##### 2 | # 3 | # Module name: convert.py 4 | # Purpose: Convert older databases and .rc files to the latest format 5 | # 6 | # Notes: 7 | # 8 | ##### 9 | 10 | # Import system modules 11 | import sqlite3 12 | import sys 13 | 14 | # Import dupReport modules 15 | import globs 16 | import options 17 | import db 18 | import drdatetime 19 | from datetime import datetime 20 | from shutil import copyfile 21 | 22 | optList210 = [ 23 | # From-section from-option to-section to-option 24 | ('main', 'dbpath', 'main', 'dbpath'), 25 | ('main', 'logpath', 'main', 'logpath'), 26 | ('main', 'verbose', 'main', 'verbose'), 27 | ('main', 'logappend', 'main', 'logappend'), 28 | ('main', 'sizereduce', 'report', 'sizedisplay'), 29 | ('main', 'subjectregex', 'main', 'subjectregex'), 30 | ('main', 'summarysubject', 'report', 'reporttitle'), 31 | ('main', 'srcregex', 'main', 'srcregex'), 32 | ('main', 'destregex', 'main', 'destregex'), 33 | ('main', 'srcdestdelimiter', 'main', 'srcdestdelimiter'), 34 | ('main', 'border', 'report', 'border'), 35 | ('main', 'padding', 'report', 'padding'), 36 | ('main', 'disperrors', 'report', 'displayerrors'), 37 | ('main', 'dispwarnings', 'report', 'displaywarnings'), 38 | ('main', 'dispmessages', 'report', 'displaymessages'), 39 | ('main', 'dateformat', 'main', 'dateformat'), 40 | ('main', 'timeformat', 'main', 'timeformat'), 41 | 42 | ('incoming', 'transport', 'incoming', 'intransport'), 43 | ('incoming', 'server', 'incoming', 'inserver'), 44 | ('incoming', 'port', 'incoming', 'inport'), 45 | ('incoming', 'encryption', 'incoming', 'inencryption'), 46 | ('incoming', 'account', 'incoming', 'inaccount'), 47 | ('incoming', 'password', 'incoming', 'inpassword'), 48 | ('incoming', 'folder', 'incoming', 'infolder'), 49 | 50 | ('outgoing', 'server', 'outgoing', 'outserver'), 51 | ('outgoing', 'port', 'outgoing', 'outport'), 52 | ('outgoing', 'encryption', 'outgoing', 'outencryption'), 53 | ('outgoing', 'account', 'outgoing', 'outaccount'), 54 | ('outgoing', 'password', 'outgoing', 'outpassword'), 55 | ('outgoing', 'sender', 'outgoing', 'outsender'), 56 | ('outgoing', 'receiver', 'outgoing', 'outreceiver') 57 | ] 58 | 59 | sizeTranslate = { 'byte': 'byte', 'none': 'byte', 'mega': 'mb', 'giga': 'gb' } 60 | v310Translate = {'source':'source','destination':'destination','date':'date','time':'time','dupversion':'dupversion','duration':'duration','files':'examinedFiles','filesplusminus':'examinedFilesDelta','size':'sizeOfExaminedFiles','sizeplusminus':'fileSizeDelta', 61 | 'errors':'errors','result':'parsedResult','joblogdata':'logdata','joberrors':'errors','added':'addedFiles','deleted':'deletedFiles','modified':'modifiedFiles','jobmessages':'messages','jobwarnings':'warnings'} 62 | 63 | def moveOption(oMgr, fromSect, fromOpt, toSect, toOpt): 64 | globs.log.write(globs.SEV_DEBUG, function='Convert', action='moveOption', msg='Moving [{}]{} to: [{}]{}'.format(fromSect, fromOpt, toSect, toOpt)) 65 | value = oMgr.getRcOption(fromSect, fromOpt) 66 | if value == None: 67 | value = '' 68 | oMgr.clearRcOption(fromSect, fromOpt) 69 | oMgr.setRcOption(toSect, toOpt, value) 70 | 71 | def convertRc(oMgr, fromVersion): 72 | # Make backup copy of rc file 73 | now = datetime.now() 74 | dateStr = now.strftime('%Y%m%d-%H%M%S') 75 | rcFileName = oMgr.options['rcfilename'] 76 | rcFileBackup = rcFileName + '.' + dateStr 77 | globs.log.write(globs.SEV_NOTICE, function='Convert', action='convertRc', msg='Backing up .rc file prior to conversion. Backup file is {}'.format(rcFileBackup)) 78 | copyfile(rcFileName, rcFileBackup) 79 | 80 | doConvertRc(oMgr, fromVersion) 81 | globs.log.write(globs.SEV_NOTICE, function='Convert', action='convertRc', msg='Updating .rc file version number to {}.{}.{}.'.format(globs.rcVersion[0],globs.rcVersion[1],globs.rcVersion[2])) 82 | oMgr.setRcOption('main', 'rcversion', '{}.{}.{}'.format(globs.rcVersion[0],globs.rcVersion[1],globs.rcVersion[2])) 83 | oMgr.updateRc() 84 | return 85 | 86 | def doConvertRc(oMgr, fromVersion): 87 | if fromVersion < 210: 88 | globs.log.write(globs.SEV_NOTICE, function='Convert', action='doConvertRc', msg='Updating .rc file from version {} to version 210.'.format(fromVersion)) 89 | # Start adding back in secitons 90 | if oMgr.parser.has_section('main') is False: 91 | globs.log.write(globs.SEV_DEBUG, function='Convert', action='convertRc', msg='Adding [main] section.') 92 | oMgr.addRcSection('main') 93 | 94 | if oMgr.parser.has_section('incoming') is False: 95 | globs.log.write(globs.SEV_DEBUG, function='Convert', action='convertRc', msg='Adding [incoming] section.') 96 | oMgr.addRcSection('incoming') 97 | 98 | if oMgr.parser.has_section('outgoing') is False: 99 | globs.log.write(globs.SEV_DEBUG, function='Convert', action='convertRc', msg='Adding [outgoing] section.') 100 | oMgr.addRcSection('outgoing') 101 | 102 | if oMgr.parser.has_section('report') is False: 103 | globs.log.write(globs.SEV_DEBUG, function='Convert', action='convertRc', msg='Adding [report] section.') 104 | oMgr.addRcSection('report') 105 | 106 | if oMgr.parser.has_section('headings') is False: 107 | globs.log.write(globs.SEV_DEBUG, function='Convert', action='convertRc', msg='Adding [headings] section.') 108 | oMgr.addRcSection('headings') 109 | 110 | for fromsection, fromoption, tosection, tooption in optList210: 111 | moveOption(oMgr, fromsection, fromoption, tosection, tooption) 112 | 113 | # Adjusted format of sizeDisplay in version 2.1 114 | szDisp = oMgr.getRcOption('report', 'sizedisplay') 115 | if szDisp == 'none': 116 | oMgr.setRcOption('report', 'sizedisplay', 'byte') 117 | oMgr.setRcOption('report', 'showsizedisplay', 'true') 118 | 119 | oMgr.updateRc() 120 | doConvertRc(oMgr, 210) 121 | elif fromVersion < 300: 122 | globs.log.write(globs.SEV_NOTICE, function='Convert', action='doConvertRc', msg='Updating .rc file from version {} to version 300.'.format(fromVersion)) 123 | # Remove deprecated options 124 | if oMgr.parser.has_option('report', 'noactivitybg') == True: # Deprecated in version 2.2.0 125 | oMgr.clearRcOption('report', 'noactivitybg') 126 | 127 | if oMgr.parser.has_option('main', 'version') == True: # Deprecated in version 2.2.7 (renamed to 'rcversion') 128 | oMgr.clearRcOption('main', 'version') 129 | 130 | oMgr.updateRc() 131 | doConvertRc(oMgr, 300) 132 | elif fromVersion < 310: 133 | globs.log.write(globs.SEV_NOTICE, function='Convert', action='doConvertRc', msg='Updating .rc file from version {} to version 310.'.format(fromVersion)) 134 | # Adjust size display option 135 | value1 = oMgr.getRcOption('report', 'sizedisplay') 136 | value2 = oMgr.getRcOption('report', 'showsizedisplay') 137 | if value2.lower() == 'false': 138 | value1 = 'none' 139 | value1 = sizeTranslate[value1] 140 | oMgr.setRcOption('report', 'sizedisplay', value1) 141 | oMgr.clearRcOption('report', 'showsizedisplay') 142 | 143 | # Change basic report options 144 | reportTitle = oMgr.getRcOption('report', 'reporttitle') 145 | oMgr.setRcOption('report', 'title', 'Duplicati Backup Summary Report') 146 | oMgr.clearRcOption('report', 'reporttitle') 147 | oMgr.setRcOption('report', 'columns', 'source:Source, destination:Destination, date: Date, time: Time, dupversion:Version, duration:Duration, examinedFiles:Files, examinedFilesDelta:+/-, sizeOfExaminedFiles:Size, fileSizeDelta:+/-, addedFiles:Added, deletedFiles:Deleted, modifiedFiles:Modified, filesWithError:Errors, parsedResult:Result, messages:Messages, warnings:Warnings, errors:Errors, logdata:Log Data') 148 | oMgr.setRcOption('report', 'weminline', 'false') 149 | moveOption(oMgr, 'report', 'subheadbg', 'report', 'groupheadingbg') 150 | 151 | # Set up report sections 152 | mainReport = oMgr.getRcOption('report', 'style') # This is the main report run 153 | oMgr.clearRcOption('report', 'style') 154 | 155 | # Add new sections using pre-defined defaults 156 | oMgr.addRcSection('srcdest') 157 | oMgr.addRcSection('bysrc') 158 | oMgr.addRcSection('bydest') 159 | oMgr.addRcSection('bydate') 160 | oMgr.addRcSection('noactivity') 161 | oMgr.addRcSection('lastseen') 162 | for section, option, default, cancontinue in options.rcParts: 163 | if section in ['srcdest','bysrc','bydest','bydate','noactivity','lastseen']: 164 | oMgr.setRcOption(section, option, default) 165 | 166 | # Now, set the default report to mimic what was in the old format 167 | oMgr.setRcOption(mainReport, 'title', reportTitle) 168 | oMgr.setRcOption('report', 'layout', mainReport + ', noactivity') 169 | 170 | # Update 'last seen' settings 171 | value1 = oMgr.getRcOption('report', 'lastseensummary') 172 | value2 = oMgr.getRcOption('report', 'lastseensummarytitle') 173 | if value1.lower() != 'none': 174 | oMgr.setRcOption('lastseen', 'title', value2) 175 | value3 = oMgr.getRcOption('report', 'layout') 176 | if value1.lower() == 'top': 177 | oMgr.setRcOption('report', 'layout', 'lastseen, ' + value3) 178 | else: 179 | oMgr.setRcOption('report', 'layout', value3 + ', lastseen') 180 | oMgr.clearRcOption('report', 'lastseensummary') 181 | oMgr.clearRcOption('report', 'lastseensummarytitle') 182 | 183 | # Adjust field background colors 184 | moveOption(oMgr, 'report', 'lastseenlow', 'report', 'normaldays') 185 | moveOption(oMgr, 'report', 'lastseenlowcolor', 'report', 'normalbg') 186 | moveOption(oMgr, 'report', 'lastseenmed', 'report', 'warningdays') 187 | moveOption(oMgr, 'report', 'lastseenmedcolor', 'report', 'warningbg') 188 | moveOption(oMgr, 'report', 'lastseenhighcolor', 'report', 'errorbg') 189 | 190 | # Convert headings to new 'columns' format 191 | headings = oMgr.getRcSection('headings') 192 | columns = '' 193 | colIndex = -1 194 | for columnName in headings: 195 | colIndex += 1 196 | if headings[columnName] != '': 197 | columns += v310Translate[columnName] + ':' + headings[columnName] 198 | if colIndex < len(headings)-1: 199 | columns += ', ' 200 | if columns[-2:] == ', ': 201 | columns = columns[:len(columns)-2:] 202 | oMgr.setRcOption(mainReport, 'columns', columns) 203 | oMgr.clearRcSection('headings') 204 | 205 | # Change to new email server format 206 | # Set [main]emailservers= option 207 | oMgr.setRcOption('main', 'emailservers', 'incoming, outgoing') 208 | 209 | # Move 'in*' to just '*' 210 | protocol = oMgr.getRcOption('incoming', 'intransport') 211 | oMgr.setRcOption('incoming', 'protocol', protocol) 212 | oMgr.clearRcOption('incoming', 'intransport') 213 | oMgr.setRcOption('outgoing', 'protocol', 'smtp') 214 | 215 | for option in ['inserver', 'inport', 'inencryption', 'inaccount', 'inpassword', 'infolder', 'inkeepalive']: 216 | optVal = oMgr.getRcOption('incoming', option) 217 | oMgr.setRcOption('incoming', option[2:], optVal) 218 | oMgr.clearRcOption('incoming', option) 219 | for option in ['outserver', 'outport', 'outencryption', 'outaccount', 'outpassword', 'outsender', 'outsendername', 'outreceiver', 'outkeepalive']: 220 | optVal = oMgr.getRcOption('outgoing', option) 221 | if optVal == None: 222 | optVal = '' 223 | oMgr.setRcOption('outgoing', option[3:], optVal) 224 | oMgr.clearRcOption('outgoing', option) 225 | 226 | # Move 'markread' to incoming 227 | markread = oMgr.getRcOption('main', 'markread') 228 | oMgr.setRcOption('incoming', 'markread', markread) 229 | oMgr.clearRcOption('main', 'markread') 230 | 231 | # Move logging levels 232 | verbose = oMgr.getRcOption('main', 'verbose') 233 | if verbose in ['1','2']: 234 | verbose = '5' 235 | elif verbose == '3': 236 | verbose = '7' 237 | oMgr.setRcOption('main', 'verbose', verbose) 238 | 239 | # Add authentication methods 240 | oMgr.setRcOption('incoming', 'authentication', 'basic') 241 | oMgr.setRcOption('outgoing', 'authentication', 'basic') 242 | 243 | # Update [apprise] section, if it exists. If it doesn't, default .rc routine will take care of it. 244 | if oMgr.hasSection('apprise'): 245 | oMgr.setRcOption('apprise', 'enabled', 'true') 246 | 247 | oMgr.updateRc() 248 | doConvertRc(oMgr, 310) 249 | else: 250 | pass 251 | 252 | return None; 253 | 254 | 255 | def convertDb(fromVersion): 256 | # Make backup copy of datavase file 257 | now = datetime.now() 258 | dateStr = now.strftime('%Y%m%d-%H%M%S') 259 | dbFileName = globs.opts['dbpath'] 260 | dbFileBackup = dbFileName + '.' + dateStr 261 | globs.log.write(globs.SEV_NOTICE, function='Convert', action='convertDb', msg='Backing up .rc file prior to conversion. Backup file is {}'.format(dbFileBackup)) 262 | copyfile(dbFileName, dbFileBackup) 263 | 264 | doConvertDb(fromVersion) 265 | globs.log.write(globs.SEV_NOTICE, function='Convert', action='convertDb', msg='Updating database file version number to {}.{}.{}.'.format(globs.dbVersion[0], globs.dbVersion[1], globs.dbVersion[2])) 266 | globs.db.execSqlStmt("UPDATE version SET major = {}, minor = {}, subminor = {} WHERE desc = 'database'".format(globs.dbVersion[0], globs.dbVersion[1], globs.dbVersion[2])) 267 | globs.db.dbCommit() 268 | 269 | def doConvertDb(fromVersion): 270 | globs.log.write(globs.SEV_NOTICE, function='Convert', action='doConvertDb', msg='Converting database from version {} to version {}.{}.{}'.format(fromVersion, globs.dbVersion[0], globs.dbVersion[1], globs.dbVersion[2])) 271 | 272 | # Database version history 273 | # 1.0.1 - Convert from character-based date/time to unix timestamp format. 274 | # 1.0.2 - Calculate & store duraction of backup 275 | # 1.0.3 - Store new logdata field and Duplicati version numbers (per backup) 276 | # 3.0.0 - changes to report table for dupReport 3.0.0 277 | # 3.0.1 - Add bytesUploaded & bytesDownloaded fields to email & reports 278 | 279 | # Update DB version number 280 | if fromVersion < 101: # Upgrade from DB version 100 (original format). 281 | globs.log.write(globs.SEV_NOTICE, function='Convert', action='doConvertDb', msg='Converting database from version {} to version 101'.format(fromVersion)) 282 | sqlStmt = "create table report (source varchar(20), destination varchar(20), timestamp real, duration real, examinedFiles int, examinedFilesDelta int, \ 283 | sizeOfExaminedFiles int, fileSizeDelta int, addedFiles int, deletedFiles int, modifiedFiles int, filesWithError int, parsedResult varchar(30), messages varchar(255), \ 284 | warnings varchar(255), errors varchar(255), failedMsg varchar(100), dupversion varchar(100), logdata varchar(255))" 285 | globs.db.execSqlStmt(sqlStmt) 286 | 287 | # Clean up bad data in emails table left from older versions. Not sure how this happened, but it really screws things up 288 | globs.db.execSqlStmt("DELETE FROM emails WHERE beginTime > '23:59:59' or endTime > '23:59:59'") 289 | 290 | # In SQLite you can't just drop and add a column (of course :-( 291 | # You need to recreate the table with the new column & copy the data 292 | globs.db.execSqlStmt("ALTER TABLE emails RENAME TO _emails_old_") 293 | globs.db.execSqlStmt("CREATE TABLE emails (messageId varchar(50), sourceComp varchar(50), destComp varchar(50), emailTimestamp real, deletedFiles int, deletedFolders int, modifiedFiles int, \ 294 | examinedFiles int, openedFiles int, addedFiles int, sizeOfModifiedFiles int, sizeOfAddedFiles int, sizeOfExaminedFiles int, sizeOfOpenedFiles int, notProcessedFiles int, addedFolders int, \ 295 | tooLargeFiles int, filesWithError int, modifiedFolders int, modifiedSymlinks int, addedSymlinks int, deletedSymlinks int, partialBackup varchar(30), dryRun varchar(30), mainOperation varchar(30), \ 296 | parsedResult varchar(30), verboseOutput varchar(30), verboseErrors varchar(30), endTimestamp real, beginTimestamp real, duration real, messages varchar(255), warnings varchar(255), errors varchar(255), \ 297 | failedMsg varchar(100), dbSeen int, dupversion varchar(100), logdata varchar(255))") 298 | globs.db.execSqlStmt("INSERT INTO emails (messageId, sourceComp, destComp, deletedFiles, deletedFolders, modifiedFiles, examinedFiles, openedFiles, addedFiles, sizeOfModifiedFiles, sizeOfAddedFiles, \ 299 | sizeOfExaminedFiles, sizeOfOpenedFiles, notProcessedFiles, addedFolders, tooLargeFiles, filesWithError, modifiedFolders, modifiedSymlinks, addedSymlinks, deletedSymlinks, partialBackup, dryRun, mainOperation, \ 300 | parsedResult, verboseOutput, verboseErrors, messages, warnings, errors, failedMsg) SELECT messageId, sourceComp, destComp, deletedFiles, deletedFolders, \ 301 | modifiedFiles, examinedFiles, openedFiles, addedFiles, sizeOfModifiedFiles, sizeOfAddedFiles, sizeOfExaminedFiles, sizeOfOpenedFiles, notProcessedFiles, addedFolders, tooLargeFiles, filesWithError, modifiedFolders, \ 302 | modifiedSymlinks, addedSymlinks, deletedSymlinks, partialBackup, dryRun, mainOperation, parsedResult, verboseOutput, verboseErrors, messages, warnings, errors, failedMsg FROM _emails_old_") 303 | 304 | # Loop through emails table to update old char-based times to timestamps 305 | dbCursor = globs.db.execSqlStmt("SELECT messageId, emailDate, emailTime, endDate, endTime, beginDate, beginTime FROM _emails_old_") 306 | emailRows = dbCursor.fetchall() 307 | for messageId, emailDate, emailTime, endDate, endTime, beginDate, beginTime in emailRows: 308 | # Create email timestamp 309 | dateStr = '{} {}'.format(emailDate,emailTime) 310 | emailTimestamp = drdatetime.toTimestamp(dateStr, 'YYYY-MM-DD', 'HH:MM:SS') 311 | 312 | # Create endTime timestamp 313 | dateStr = '{} {}'.format(endDate,endTime) 314 | endTimestamp = drdatetime.toTimestamp(dateStr, 'YYYY/MM/DD', 'HH:MM:SS') 315 | 316 | # Create beginTime timestamp 317 | dateStr = '{} {}'.format(beginDate,beginTime) 318 | beginTimestamp = drdatetime.toTimestamp(dateStr, 'YYYY/MM/DD', 'HH:MM:SS') 319 | 320 | # Update emails table with new data 321 | if endTimestamp is not None and beginTimestamp is not None: 322 | sqlStmt = "UPDATE emails SET emailTimestamp = {}, endTimestamp = {}, beginTimestamp = {}, duration = {} WHERE messageId = \'{}\'".format(emailTimestamp, endTimestamp, beginTimestamp, (endTimestamp - beginTimestamp), messageId) 323 | globs.log.write(globs.SEV_NOTICE, function='Convert', action='doConvertDb', msg=sqlStmt) 324 | globs.db.execSqlStmt(sqlStmt) 325 | 326 | globs.log.write(globs.SEV_NOTICE, function='Convert', action='doConvertDb', msg='messageId:{} emailDate={} emailTime={} emailTimestamp={} endDate={} endTime={} endTimestamp={} beginDate={} beginTime={} beginTimestamp={} duration={}'.format(messageId, emailDate, emailTime, emailTimestamp,\ 327 | endDate, endTime, endTimestamp, beginDate, beginTime, beginTimestamp, duration)) 328 | globs.db.execSqlStmt("DROP TABLE _emails_old_") 329 | 330 | # Convert date/time to timestamps in backupsets table 331 | globs.db.execSqlStmt("ALTER TABLE backupsets ADD COLUMN lastTimestamp real") 332 | dbCursor = globs.db.execSqlStmt("SELECT source, destination, lastDate, lastTime from backupsets") 333 | setRows = dbCursor.fetchall() 334 | for source, destination, lastDate, lastTime in setRows: 335 | dateStr = '{} {}'.format(lastDate,lastTime) 336 | lastTimestamp = drdatetime.toTimestamp(dateStr, 'YYYY/MM/DD', 'HH:MM:SS') 337 | 338 | sqlStmt = "UPDATE backupsets SET lastTimestamp = {} WHERE source = \'{}\' AND destination = \'{}\'".format(lastTimestamp, source, destination) 339 | globs.db.execSqlStmt(sqlStmt) 340 | globs.log.write(globs.SEV_NOTICE, function='Convert', action='doConvertDb', msg='Updating backupsets: Source={} destination={} lastDate={} lastTime={} lastTimestamp={}'.format(source, destination, lastDate, lastTime, lastTimestamp)) 341 | doConvertDb(101) 342 | elif fromVersion < 102: # Upgrade from version 101 343 | globs.log.write(globs.SEV_NOTICE, function='Convert', action='doConvertDb', msg='Converting database from version {} to version 102'.format(fromVersion)) 344 | globs.db.execSqlStmt("ALTER TABLE report ADD COLUMN duration real") 345 | globs.db.execSqlStmt("ALTER TABLE report ADD COLUMN dupversion varchar(100)") 346 | globs.db.execSqlStmt("ALTER TABLE report ADD COLUMN logdata varchar(255)") 347 | globs.db.execSqlStmt("UPDATE report SET duration = 0") 348 | globs.db.execSqlStmt("UPDATE report SET dupversion = ''") 349 | globs.db.execSqlStmt("UPDATE report SET logdata = ''") 350 | 351 | # Need to change duration column from varchar to real 352 | # In SQLite you can't just drop and add a column (of course :-( 353 | # You need to recreate the table with the new column & copy the data 354 | globs.db.execSqlStmt("ALTER TABLE emails RENAME TO _emails_old_") 355 | globs.db.execSqlStmt("CREATE TABLE emails (messageId varchar(50), sourceComp varchar(50), destComp varchar(50), emailTimestamp real, deletedFiles int, deletedFolders int, modifiedFiles int, \ 356 | examinedFiles int, openedFiles int, addedFiles int, sizeOfModifiedFiles int, sizeOfAddedFiles int, sizeOfExaminedFiles int, sizeOfOpenedFiles int, notProcessedFiles int, addedFolders int, \ 357 | tooLargeFiles int, filesWithError int, modifiedFolders int, modifiedSymlinks int, addedSymlinks int, deletedSymlinks int, partialBackup varchar(30), dryRun varchar(30), mainOperation varchar(30), \ 358 | parsedResult varchar(30), verboseOutput varchar(30), verboseErrors varchar(30), endTimestamp real, beginTimestamp real, duration real, messages varchar(255), warnings varchar(255), errors varchar(255), \ 359 | failedMsg varchar(100), dbSeen int, dupversion varchar(100), logdata varchar(255))") 360 | globs.db.execSqlStmt("INSERT INTO emails (messageId, sourceComp, destComp, emailTimestamp, deletedFiles, deletedFolders, modifiedFiles, examinedFiles, openedFiles, addedFiles, sizeOfModifiedFiles, sizeOfAddedFiles, \ 361 | sizeOfExaminedFiles, sizeOfOpenedFiles, notProcessedFiles, addedFolders, tooLargeFiles, filesWithError, modifiedFolders, modifiedSymlinks, addedSymlinks, deletedSymlinks, partialBackup, dryRun, mainOperation, \ 362 | parsedResult, verboseOutput, verboseErrors, endTimestamp, beginTimestamp, messages, warnings, errors, failedMsg, dbSeen) SELECT messageId, sourceComp, destComp, emailTimestamp, deletedFiles, deletedFolders, \ 363 | modifiedFiles, examinedFiles, openedFiles, addedFiles, sizeOfModifiedFiles, sizeOfAddedFiles, sizeOfExaminedFiles, sizeOfOpenedFiles, notProcessedFiles, addedFolders, tooLargeFiles, filesWithError, modifiedFolders, \ 364 | modifiedSymlinks, addedSymlinks, deletedSymlinks, partialBackup, dryRun, mainOperation, parsedResult, verboseOutput, verboseErrors, endTimestamp, beginTimestamp, messages, warnings, errors, failedMsg, dbSeen FROM _emails_old_") 365 | 366 | # Loop through new emails table and set duration field 367 | dbCursor = globs.db.execSqlStmt("SELECT messageId, beginTimeStamp, endTimeStamp FROM emails") 368 | emailRows = dbCursor.fetchall() 369 | for messageId, beginTimeStamp, endTimeStamp in emailRows: 370 | # Update emails table with new data 371 | if endTimeStamp is not None and beginTimeStamp is not None: 372 | sqlStmt = "UPDATE emails SET duration = {} WHERE messageId = \'{}\'".format((endTimeStamp - beginTimeStamp), messageId) 373 | globs.db.execSqlStmt(sqlStmt) 374 | globs.db.execSqlStmt("DROP TABLE _emails_old_") 375 | doConvertDb(102) 376 | elif fromVersion < 103: # Upgrade from version 102 377 | globs.log.write(globs.SEV_NOTICE, function='Convert', action='doConvertDb', msg='Converting database from version {} to version 103'.format(fromVersion)) 378 | # Add dupversion & logdata fields to emails table 379 | globs.db.execSqlStmt("ALTER TABLE emails ADD COLUMN dupversion varchar(100)") 380 | globs.db.execSqlStmt("ALTER TABLE emails ADD COLUMN logdata varchar(255)") 381 | globs.db.execSqlStmt("UPDATE emails SET dupversion = ''") 382 | globs.db.execSqlStmt("UPDATE emails SET logdata = ''") 383 | 384 | # Add dupversion & logdata fields to report table 385 | globs.db.execSqlStmt("ALTER TABLE report ADD COLUMN dupversion varchar(100)") 386 | globs.db.execSqlStmt("ALTER TABLE report ADD COLUMN logdata varchar(255)") 387 | globs.db.execSqlStmt("UPDATE report SET dupversion = ''") 388 | globs.db.execSqlStmt("UPDATE report SET logdata = ''") 389 | doConvertDb(103) 390 | elif fromVersion < 300: # Upgrade from version 103 391 | globs.log.write(globs.SEV_NOTICE, function='Convert', action='doConvertDb', msg='Converting database from version {} to version 300'.format(fromVersion)) 392 | # Add date & time fields to reports table 393 | globs.db.execSqlStmt("ALTER TABLE report ADD COLUMN date real") 394 | globs.db.execSqlStmt("ALTER TABLE report ADD COLUMN time real") 395 | globs.db.execSqlStmt("ALTER TABLE backupsets ADD COLUMN dupversion varchar(100)") 396 | 397 | # Insert last dupversion for all existing backupset rows 398 | globs.db.execSqlStmt("UPDATE backupsets SET dupversion = (SELECT emails.dupversion FROM emails WHERE backupsets.source = emails.sourceComp and backupsets.destination = emails.destComp)") 399 | doConvertDb(300) 400 | pass 401 | elif fromVersion < 301: # Upgrade from version 300 402 | globs.log.write(globs.SEV_NOTICE, function='Convert', action='doConvertDb', msg='Converting database from version {} to version 301'.format(fromVersion)) 403 | # Add BytesUplaoded & BytesDownloaded fields to reports table 404 | globs.db.execSqlStmt("ALTER TABLE emails ADD COLUMN bytesUploaded int") 405 | globs.db.execSqlStmt("ALTER TABLE emails ADD COLUMN bytesDownloaded int") 406 | globs.db.execSqlStmt("ALTER TABLE report ADD COLUMN bytesUploaded int") 407 | globs.db.execSqlStmt("ALTER TABLE report ADD COLUMN bytesDownloaded int") 408 | 409 | # Set default values for bytes Up/Downloaded 410 | globs.db.execSqlStmt("UPDATE emails SET bytesUploaded=0, bytesDownloaded=0") 411 | 412 | # Insert last dupversion for all existing backupset rows 413 | doConvertDb(301) 414 | pass 415 | else: 416 | pass 417 | 418 | return None 419 | -------------------------------------------------------------------------------- /db.py: -------------------------------------------------------------------------------- 1 | ##### 2 | # 3 | # Module name: db.py 4 | # Purpose: Sqlite3 database class defnition & functions 5 | # 6 | # Notes: 7 | # 8 | ##### 9 | 10 | # Import system modules 11 | import sqlite3 12 | import sys 13 | import os 14 | from datetime import datetime 15 | from datetime import timedelta 16 | 17 | # Import dupReport modules 18 | import globs 19 | import drdatetime 20 | 21 | class Database: 22 | dbConn = None 23 | def __init__(self, dbPath): 24 | globs.log.write(globs.SEV_NOTICE,function='Database', action='Init', msg='Initializing database manager.') 25 | 26 | # First, see if the database is there. If not, need to create it 27 | isThere = os.path.isfile(dbPath) 28 | 29 | if self.dbConn: # If not None then DB connection already exists. This is bad. 30 | globs.log.write(globs.SEV_ERROR, function='Database', action='Init', msg='SQLite3 Error: trying to reinitialize the database connection. Exiting program.') 31 | globs.closeEverythingAndExit(1) # Abort program. Can't continue with DB error 32 | 33 | try: 34 | self.dbConn = sqlite3.connect(dbPath) # Connect to database 35 | except sqlite3.Error as err: 36 | globs.log.write(globs.SEV_ERROR, function='Database', action='Init', msg='SQLite3 error connecting to database: {}. Exiting program.'.format(err.args[0])) 37 | globs.closeEverythingAndExit(1) # Abort program. Can't continue with DB error 38 | 39 | if not isThere: # Database did not exist. Need to initialize contents 40 | globs.log.write(globs.SEV_NOTICE, function='Database', action='Init', msg='New database. Needs initializing.') 41 | self.dbInitialize() 42 | return None 43 | 44 | # Close database connection 45 | def dbClose(self): 46 | globs.log.write(globs.SEV_NOTICE, function='Database', action='dbClose', msg='Closing database manager.') 47 | 48 | # Don't attempt to close a non-existant conmnection 49 | if self.dbConn: 50 | self.dbConn.close() 51 | self.dbConn = None 52 | return None 53 | 54 | # Return True if need to upgrade DB, false if DB is current. 55 | def checkDbVersion(self): 56 | needToUpgrade = False 57 | 58 | dbCursor = self.execSqlStmt('SELECT major, minor, subminor FROM version WHERE desc = \'database\'') 59 | maj, min, subm = dbCursor.fetchone() 60 | 61 | currVerNum = (maj * 100) + (min * 10) + subm 62 | newVerNum = (globs.dbVersion[0] * 100) + (globs.dbVersion[1] * 10) + globs.dbVersion[2] 63 | globs.log.write(globs.SEV_DEBUG, function='Database', action='checkDbVersion', msg='Database: current version={} new version={}'.format(currVerNum, newVerNum)) 64 | if currVerNum < newVerNum: 65 | globs.log.write(globs.SEV_NOTICE, function='Database', action='checkDbVersion', msg='Database version is out of date. Needs update to latest version.') 66 | needToUpgrade = True 67 | 68 | return needToUpgrade, currVerNum 69 | 70 | # Commit pending database transaction 71 | def dbCommit(self): 72 | globs.log.write(globs.SEV_DEBUG, function='Database', action='dbCommit', msg='Committing transaction.') 73 | if self.dbConn: # Don't try to commit to a nonexistant connection 74 | self.dbConn.commit() 75 | return None 76 | 77 | def execEmailInsertSql(self, emailParts): 78 | globs.log.write(globs.SEV_NOTICE, function='Database', action='execEmailInsertSql', msg='Inserting into emails table: messageId={} sourceComp={} destComp={}'.format(emailParts['header']['messageId'], emailParts['header']['sourceComp'], emailParts['header']['destComp'])) 79 | 80 | durVal = float(emailParts['body']['endTimestamp']) - float(emailParts['body']['beginTimestamp']) 81 | sqlStmt = "INSERT INTO emails(messageId, sourceComp, destComp, emailTimestamp, \ 82 | deletedFiles, deletedFolders, modifiedFiles, examinedFiles, \ 83 | openedFiles, addedFiles, sizeOfModifiedFiles, sizeOfAddedFiles, sizeOfExaminedFiles, \ 84 | sizeOfOpenedFiles, notProcessedFiles, addedFolders, tooLargeFiles, filesWithError, \ 85 | modifiedFolders, modifiedSymlinks, addedSymlinks, deletedSymlinks, partialBackup, \ 86 | dryRun, mainOperation, parsedResult, verboseOutput, verboseErrors, endTimestamp, \ 87 | beginTimestamp, duration, messages, warnings, errors, dbSeen, dupversion, logdata, bytesUploaded, bytesDownloaded) \ 88 | VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?)" 89 | 90 | data = (emailParts['header']['messageId'], emailParts['header']['sourceComp'], emailParts['header']['destComp'], emailParts['header']['emailTimestamp'], emailParts['body']['deletedFiles'], \ 91 | emailParts['body']['deletedFolders'], emailParts['body']['modifiedFiles'], emailParts['body']['examinedFiles'], emailParts['body']['openedFiles'], \ 92 | emailParts['body']['addedFiles'], emailParts['body']['sizeOfModifiedFiles'], emailParts['body']['sizeOfAddedFiles'], emailParts['body']['sizeOfExaminedFiles'], emailParts['body']['sizeOfOpenedFiles'], \ 93 | emailParts['body']['notProcessedFiles'], emailParts['body']['addedFolders'], emailParts['body']['tooLargeFiles'], emailParts['body']['filesWithError'], \ 94 | emailParts['body']['modifiedFolders'], emailParts['body']['modifiedSymlinks'], emailParts['body']['addedSymlinks'], emailParts['body']['deletedSymlinks'], \ 95 | emailParts['body']['partialBackup'], emailParts['body']['dryRun'], emailParts['body']['mainOperation'], emailParts['body']['parsedResult'], emailParts['body']['verboseOutput'], \ 96 | emailParts['body']['verboseErrors'], emailParts['body']['endTimestamp'], emailParts['body']['beginTimestamp'], \ 97 | durVal, emailParts['body']['messages'], emailParts['body']['warnings'], emailParts['body']['errors'], emailParts['body']['dupversion'], emailParts['body']['logdata'], emailParts['body']['bytesUploaded'], emailParts['body']['bytesDownloaded']) 98 | 99 | globs.log.write(globs.SEV_DEBUG, function='Database', action='execEmailInsertSql', msg='sqlStmt=[{}]'.format(sqlStmt)) 100 | globs.log.write(globs.SEV_DEBUG, function='Database', action='execEmailInsertSql', msg='data=[{}]'.format(data)) 101 | 102 | if not self.dbConn: 103 | return None 104 | 105 | # Set db cursor 106 | curs = self.dbConn.cursor() 107 | try: 108 | curs.execute(sqlStmt, data) 109 | except sqlite3.Error as err: 110 | globs.log.write(globs.SEV_ERROR, function='Database', action='execEmailInsertSql', msg='SQLite error: {}'.format(err.args[0])) 111 | globs.closeEverythingAndExit(1) # Abort program. Can't continue with DB error 112 | 113 | self.dbCommit() 114 | return None 115 | 116 | def execReportInsertSql(self, sqlStmt, sqlData): 117 | globs.log.write(globs.SEV_NOTICE, function='Database', action='execReportInsertSql', msg='Inserting into emails table: sqlStmt=[{}] sqlData=[{}]'.format(sqlStmt, sqlData)) 118 | 119 | if not self.dbConn: 120 | return None 121 | 122 | # Set db cursor 123 | curs = self.dbConn.cursor() 124 | try: 125 | curs.execute(sqlStmt, sqlData) 126 | except sqlite3.Error as err: 127 | globs.log.write(globs.SEV_ERROR, function='Database', action='execReportInsertSql', msg='SQLite error: {}'.format(err.args[0])) 128 | globs.closeEverythingAndExit(1) # Abort program. Can't continue with DB error 129 | 130 | self.dbCommit() 131 | return None 132 | 133 | # Execute a Sqlite command and manage exceptions 134 | # Return the cursor object to the command result 135 | def execSqlStmt(self, stmt): 136 | globs.log.write(globs.SEV_NOTICE, function='Database', action='execSqlStmt', msg='Executing SQL statement: [{}]'.format(stmt)) 137 | 138 | if not self.dbConn: 139 | return None 140 | 141 | # Set db cursor 142 | curs = self.dbConn.cursor() 143 | try: 144 | curs.execute(stmt) 145 | except sqlite3.Error as err: 146 | globs.log.write(globs.SEV_ERROR, function='Database', action='execSqlStmt', msg='SQLite error: {}'.format(err.args[0])) 147 | globs.closeEverythingAndExit(1) # Abort program. Can't continue with DB error 148 | 149 | # Return the cursor to the executed command result. 150 | return curs 151 | 152 | # Initialize database to empty, default tables 153 | def dbInitialize(self): 154 | globs.log.write(globs.SEV_NOTICE, function='Database', action='dbInitialize', msg='Initializing (resetting) database.') 155 | 156 | # Don't initialize a non-existant connection 157 | if not self.dbConn: 158 | return None 159 | 160 | # Drop any tables and indices that might already exist in the database 161 | self.execSqlStmt("drop table if exists version") 162 | self.execSqlStmt("drop table if exists emails") 163 | self.execSqlStmt("drop table if exists backupsets") 164 | self.execSqlStmt("drop table if exists report") 165 | self.execSqlStmt("drop index if exists emailindx") 166 | self.execSqlStmt("drop index if exists srcdestindx") 167 | 168 | # version table holds current database version. 169 | # Used to check for need to change database formats 170 | self.execSqlStmt("create table version (desc varchar(20), major int, minor int, subminor int)") 171 | self.execSqlStmt("insert into version(desc, major, minor, subminor) values (\'database\',{},{},{})".format(globs.dbVersion[0], globs.dbVersion[1], globs.dbVersion[2])) 172 | 173 | # emails table holds information about all emails received 174 | sqlStmt = "create table emails (messageId varchar(50), sourceComp varchar(50), destComp varchar(50), \ 175 | emailTimestamp real, deletedFiles int, deletedFolders int, modifiedFiles int, \ 176 | examinedFiles int, openedFiles int, addedFiles int, sizeOfModifiedFiles int, sizeOfAddedFiles int, sizeOfExaminedFiles int, \ 177 | sizeOfOpenedFiles int, notProcessedFiles int, addedFolders int, tooLargeFiles int, filesWithError int, \ 178 | modifiedFolders int, modifiedSymlinks int, addedSymlinks int, deletedSymlinks int, partialBackup varchar(30), \ 179 | dryRun varchar(30), mainOperation varchar(30), parsedResult varchar(30), verboseOutput varchar(30), \ 180 | verboseErrors varchar(30), endTimestamp real, \ 181 | beginTimestamp real, duration real, messages varchar(255), warnings varchar(255), errors varchar(255), failedMsg varchar(100), dbSeen int, dupversion varchar(100), logdata varchar(255), \ 182 | bytesUploaded int, bytesDownloaded int)" 183 | self.execSqlStmt(sqlStmt) 184 | self.execSqlStmt("create index emailindx on emails (messageId)") 185 | self.execSqlStmt("create index srcdestindx on emails (sourceComp, destComp)") 186 | 187 | sqlStmt = "create table report (source varchar(20), destination varchar(20), timestamp real, date real, time real, duration real, examinedFiles int, examinedFilesDelta int, \ 188 | sizeOfExaminedFiles int, fileSizeDelta int, addedFiles int, deletedFiles int, modifiedFiles int, filesWithError int, parsedResult varchar(30), messages varchar(255), \ 189 | warnings varchar(255), errors varchar(255), failedMsg varchar(100), dupversion varchar(100), logdata varchar(255), bytesUploaded int, bytesDownloaded int)" 190 | self.execSqlStmt(sqlStmt) 191 | 192 | # backup sets contains information on all source-destination pairs in the backups 193 | self.execSqlStmt("create table backupsets (source varchar(20), destination varchar(20), lastFileCount integer, lastFileSize integer, \ 194 | lastTimestamp real, dupversion varchar(100))") 195 | 196 | self.dbCommit() 197 | self.dbCompact() 198 | globs.log.write(globs.SEV_NOTICE, function='Database', action='dbInitialize', msg='Database initialization complete.') 199 | return None 200 | 201 | # See if a particular message ID is already in the database 202 | # Return True (already there) or False (not there) 203 | def searchForMessage(self, msgID): 204 | globs.log.write(globs.SEV_NOTICE, function='Database', action='searchForMessage', msg='Searching for message {} in database.'.format(msgID)) 205 | sqlStmt = "SELECT messageId FROM emails WHERE messageId=\'{}\'".format(msgID) 206 | dbCursor = self.execSqlStmt(sqlStmt) 207 | idExists = dbCursor.fetchone() 208 | if idExists: 209 | globs.log.write(globs.SEV_NOTICE, function='Database', action='searchForMessage', msg='Message [{}] already in email database'.format(msgID)) 210 | return True 211 | else: 212 | globs.log.write(globs.SEV_NOTICE, function='Database', action='searchForMessage', msg='Message [{}] not yet in email database'.format(msgID)) 213 | return False 214 | 215 | def searchSrcDestPair(self, src, dest, add2Db = True): 216 | globs.log.write(globs.SEV_NOTICE, function='Database', action='searchSrcDestPair', msg='Searching for {}{}{} in backupsets'.format(src, globs.opts['srcdestdelimiter'], dest)) 217 | sqlStmt = "SELECT source, destination FROM backupsets WHERE source=\'{}\' AND destination=\'{}\'".format(src, dest) 218 | dbCursor = self.execSqlStmt(sqlStmt) 219 | idExists = dbCursor.fetchone() 220 | if idExists: 221 | globs.log.write(globs.SEV_NOTICE, function='Database', action='searchSrcDestPair', msg='{}{}{} already in backupsets.'.format(src, globs.opts['srcdestdelimiter'], dest)) 222 | return True 223 | 224 | if add2Db is True: 225 | sqlStmt = "INSERT INTO backupsets (source, destination, lastFileCount, lastFileSize, lastTimestamp, dupversion) \ 226 | VALUES ('{}', '{}', 0, 0, 0, '')".format(src, dest) 227 | self.execSqlStmt(sqlStmt) 228 | self.dbCommit() 229 | globs.log.write(globs.SEV_NOTICE, function='Database', action='searchSrcDestPair', msg='{}{}{} added to database'.format(src, globs.opts['srcdestdelimiter'], dest)) 230 | return False 231 | 232 | # Roll back database to specific date/time 233 | # Datespec = Date & time to roll back to 234 | def rollback(self, datespec): 235 | globs.log.write(globs.SEV_NOTICE, function='Database', action='rollback', msg='Rolling back database: spec={}'.format(datespec)) 236 | 237 | # See if we're using a delta-based time spec (Issue #131) 238 | deltaParts = drdatetime.timeDeltaSpec(datespec) 239 | if deltaParts != False: 240 | today = datetime.now() 241 | globs.log.write(globs.SEV_DEBUG, function='Database', action='rollback', msg='Using delta timespec. Today={}'.format(today)) 242 | for i in range(len(deltaParts)): 243 | tval = int(deltaParts[i][:-1]) 244 | tspec = deltaParts[i][-1:] 245 | if tspec == 's': # Subtract seconds 246 | today -= timedelta(seconds=tval) 247 | elif tspec == 'm': 248 | today -= timedelta(minutes=tval) 249 | elif tspec == 'h': 250 | today -= timedelta(hours=tval) 251 | elif tspec == 'd': 252 | today -= timedelta(days=tval) 253 | elif tspec == 'w': 254 | today -= timedelta(weeks=tval) 255 | globs.log.write(globs.SEV_DEBUG, function='Database', action='rollback', msg='Rolled back {}{}. Today now={}'.format(tval,tspec, today)) 256 | newTimeStamp = today.timestamp() 257 | else: 258 | # Get timestamp for input date/time 259 | newTimeStamp = drdatetime.toTimestamp(datespec) 260 | 261 | # Delete all email records that happened after input datetime 262 | sqlStmt = 'DELETE FROM emails WHERE emailtimestamp > {}'.format(newTimeStamp) 263 | dbCursor = self.execSqlStmt(sqlStmt) 264 | 265 | # Delete all backup set records that happened after input datetime 266 | sqlStmt = 'SELECT source, destination FROM backupsets WHERE lastTimestamp > {}'.format(newTimeStamp) 267 | dbCursor = self.execSqlStmt(sqlStmt) 268 | setRows= dbCursor.fetchall() 269 | for source, destination in setRows: 270 | # Select largest timestamp from remaining data for that source/destination 271 | sqlStmt = 'select max(endTimeStamp), examinedFiles, sizeOfExaminedFiles, dupversion from emails where sourceComp = \'{}\' and destComp= \'{}\''.format(source, destination) 272 | dbCursor = self.execSqlStmt(sqlStmt) 273 | emailTimestamp, examinedFiles, sizeOfExaminedFiles, dupversion = dbCursor.fetchone() 274 | if emailTimestamp is None: 275 | # After the rollback, some srcdest pairs may have no corresponding entries in the the database, meaning they were not seen until after the rollback period 276 | # We should remove these from the database, to return it to the state it was in before the rollback. 277 | globs.log.write(globs.SEV_NOTICE, function='Database', action='rollback', msg='Deleting {}{}{} from backupsets. Not seen until after rollback.'.format(source, globs.opts['srcdestdelimiter'], destination)) 278 | sqlStmt = 'DELETE FROM backupsets WHERE source = \"{}\" AND destination = \"{}\"'.format(source, destination) 279 | dbCursor = self.execSqlStmt(sqlStmt) 280 | else: 281 | globs.log.write(globs.SEV_NOTICE, function='Database', action='rollback', msg='Resetting {}{}{} to {}'.format(source, globs.opts['srcdestdelimiter'], destination, drdatetime.fromTimestamp(emailTimestamp))) 282 | # Update backupset table to reflect rolled-back date 283 | sqlStmt = 'update backupsets set lastFileCount={}, lastFileSize={}, lastTimestamp={}, dupversion=\'{}\' where source = \'{}\' and destination = \'{}\''.format(examinedFiles, sizeOfExaminedFiles, emailTimestamp, dupversion, source, destination) 284 | dbCursor = self.execSqlStmt(sqlStmt) 285 | 286 | self.dbCommit() 287 | return None 288 | 289 | # Remove a source/destination pair from the database 290 | def removeSrcDest(self, source, destination): 291 | globs.log.write(globs.SEV_NOTICE, function='Database', action='removeSrcDest', msg='Deleting {}{}{} from database.'.format(source, globs.opts['srcdestdelimiter'], destination)) 292 | 293 | # Does the src/dest exist in the database? 294 | exists = self.searchSrcDestPair(source, destination, False) 295 | if not exists: 296 | globs.log.write(globs.SEV_NOTICE, function='Database', action='removeSrcDest', msg='Pair {}{}{} does not exist in database. Check spelling and capitalization then try again.'.format(source, globs.opts['srcdestdelimiter'], destination)) 297 | return False 298 | 299 | sqlStmt = "DELETE FROM backupsets WHERE source = \"{}\" AND destination = \"{}\"".format(source, destination) 300 | dbCursor = self.execSqlStmt(sqlStmt) 301 | 302 | sqlStmt = "DELETE FROM emails WHERE sourceComp = \"{}\" AND destComp = \"{}\"".format(source, destination) 303 | dbCursor = self.execSqlStmt(sqlStmt) 304 | 305 | self.dbCommit() 306 | 307 | globs.log.write(globs.SEV_NOTICE, function='Database', action='removeSrcDest', msg='{}{}{} removed from database.'.format(source, globs.opts['srcdestdelimiter'], destination)) 308 | globs.log.write(globs.SEV_NOTICE, function='Database', action='removeSrcDest', msg='Please remove all emails referencing the \'{}{}{}\' backup,\nor they will be added back into the database the next time dupReport is run.'.format(source, globs.opts['srcdestdelimiter'], destination)) 309 | globs.log.out('Please remove all emails referencing the \'{}{}{}\' backup,\nor they will be added back into the database the next time dupReport is run.'.format(source, globs.opts['srcdestdelimiter'], destination)) 310 | 311 | return True 312 | 313 | # Purge database of old emails 314 | def purgeOldEmails(self): 315 | globs.log.write(globs.SEV_NOTICE, function='Database', action='purgeOldEmails', msg='Purging unseen emails from database') 316 | self.execSqlStmt('DELETE FROM emails WHERE dbSeen = 0') 317 | self.dbCommit() 318 | self.dbCompact() 319 | return None 320 | 321 | # Compact database to eliminate unused space 322 | def dbCompact(self): 323 | globs.log.write(globs.SEV_NOTICE, function='Database', action='dbCompact', msg='Compacting database') 324 | 325 | # Need to reset connection isolation level in order to compress database 326 | # Why? Not sure. But see https://github.com/ghaering/pysqlite/issues/109 for details 327 | isoTmp = self.dbConn.isolation_level 328 | self.dbConn.isolation_level = None 329 | self.execSqlStmt('VACUUM') 330 | self.dbConn.isolation_level = isoTmp # Re-set isolation level back to previous value 331 | self.dbCommit() 332 | 333 | globs.log.write(globs.SEV_NOTICE, function='Database', action='dbCompact', msg='Database compaction complete.') 334 | return None 335 | 336 | 337 | 338 | 339 | -------------------------------------------------------------------------------- /docs/CommandLine.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## dupReport Command Line 4 | 5 | Once all the options have been set in the .rc file, use the following commands to run dupReport normally: 6 | 7 | | Operating System | Command Line | 8 | | ---------------- | ------------------------------------------------------------ | 9 | | Linux | **user@system:~$** /path/to/dupReport/dupReport.py \ | 10 | | Windows | **C:\users\me>** python.exe \path\to\dupreport\dupReport.py \ | 11 | 12 | Command line options alter the way dupReport operates. Many command line options have equivalent options in the dupReport.rc file. **If an option is specified on both the command line and in the .rc file, the command line option takes precedence.** 13 | 14 | dupReport has the following command line options: 15 | 16 | | Short Version | Long Version | Function | Notes | 17 | | --------------------------- | ------------------------------------------- | :----------------------------------------------------------- | ------------------------------------------------------------ | 18 | | -a | --append | Append new logs to existing log file. | Overrides [main] logappend= in dupReport.rc file. | 19 | | -b \ | --rollback \ | Roll back database to a specified date and time, then continue processing emails. | See note below for rollback command line specifications. Also See the discussion of the **dateformat**= and **timeformat=** options in ["dupReport.rc file configuration."](RcFileConfig.md) To roll back the database to th | 20 | | -B \ | --rollbackx \ | Roll back database to a specified date and time, then exit the program. | Same operation as -b, except program will exit after rolling back the database. See note below for rollback command line specifications. Also See the discussion of the **dateformat**= and **timeformat=** options in ["dupReport.rc file configuration."](RcFileConfig.md) | 21 | | -c | --collect | Collect new emails only and don't run summary report. | **-c** and **-t** options can not be used together. | 22 | | -d \ | --dbpath \ | Sets \ as the directory or full path specification where the dupReport.rc file is located. | Overrides the [main] dbpath= option in dupReport.rc file. You must have read and write access to the place where \ points. | 23 | | -f \,\ | --file \,\ | Send the report to a file in text, HTML, CSV, or JSON format. \ can be one of the following: A full path specification for a file; 'stdout', to send to the standard output device; 'stderr', to send to the standard error device. \ can be one of the following: “txt”, “html”, “csv”, or "json" | -f may be used multiple times to send the output to multiple files. **Do not** leave a space between the comma (,) and the \ specification. | 24 | | -F \,\ | --fileattach \,\ | Functions the same as the -f option, but also attaches the resulting output file to the report email. | | 25 | | -g | -guidedsetup | Forces the program to run the Guided Setup as if the program were being run for the first time. | **-g** and **-G** options can not be used together. | 26 | | -G | -noguidedsetup | Prevents running the Guided Setup, even if the program would do so under normal circumstances (e.g., if the .rc file is removed) | **-g** and **-G** options can not be used together. | 27 | | -h | --help | Display command line options. | | 28 | | -i | --initdb | Erase all information from the database and resets all the tables. | | 29 | | -k | --masksensitive | Force masking of sensitive data (such as user names, passwords, and file paths) with asterisks (*) in the log file. | Overrides the "masksensitive" option in the .rc file. The -k and -K options can not be used together. See description of "masksensitive" option for more details. | 30 | | -K | --nomasksensitive | Force display of sensitive data (such as user names, passwords, and file paths) in the log file. | Overrides the "masksensitive" option in the .rc file. The -k and -K options can not be used together. See description of "masksensitive" option for more details. | 31 | | -l \ | --logpath \ | Sets \ as the directory or full path specification where the dupReport.log file is located. | Overrides the [main] logpath= option in dupReport.rc file. You must have read and write access to the place where \ points. | 32 | | -m \ \ | --remove \ \ | Remove a source-destination pair from the database. | | 33 | | -o | --validatereport | Validate the report file for syntax and accuracy, then exit the program. | Useful for debugging new custom report specifications. | 34 | | -p | --purgedb | Purge emails that are no longer on the server from the database. | Overrides [main] purgedb in .rc file. | 35 | | -r \ | --rcpath \ | Sets \ as the directory or full path specification where the dupReport.rc file is located. | You must have read and write access to the place where \ points. | 36 | | -s {‘none’, ‘mb’, ‘gb’} | --size {‘none’, ‘mb’, ‘gb’} | Instructs the program to round file sizes to standard units: bytes ('none'), megabytes ('mb'), or gigabytes ('gb') | | 37 | | -t | --report | Run summary report only and don't collect emails. | **-c** and **-t** options can not be used together. | 38 | | -v {3, 5, 7} | --verbose {3, 5, 7} | Sets the verbosity of the information in the log file. | 3=General program execution info. 5=Program flow and status information. 7=Full debugging output. Default=5. Can be set permanently in the .rc file under [main] verbose. | 39 | | -V | --Version | Prints version information for dupReport and supporting components | | 40 | | -w | --stopbackupwarn | Suppress sending of unseen backup warning emails. Overrides all "nobackupwarn" options in the .rc file. See description of nobackwarn= option in ["[report] Section"](ReportSection.md) and ["[source-destination] Sections"](SourceDestinationSection) . | | 41 | | -x | --nomail | Do not send the report through email. This is typically used in conjunction with the -f or to save the report to a file rather than send it through email. | **NOTE 1**: If you suppress the sending of emails using '-x' you do not need to enter valid outgoing email server information in the dupReport.rc file. The [outgoing] section still needs to be present in the .rc file, but it does not need valid server or account information. **NOTE 2**: If you suppress the sending of emails using the '-x' option but still want unseen backup warning messages sent (i.e., you *don't* use the '-w' option), you must enter valid email server and account information in the [outgoing] section of the dupReport.rc file. | 42 | | -y | --layout \[,\,\] | Run specified reports during the program run. | The named reports must be specified in the .rc file or the program will exit with an error. Using this option overrides the *layout=* option in the [report] section of the .rc file. **DO NOT** leave any spaces between the report names or the commas. | 43 | 44 | 45 | 46 | ------ 47 | 48 | **Database Rollback Specifications** 49 | 50 | The rollback command line options (-b and -B) take one of three specification formats as options: 51 | 52 | ``` 53 | -b "" 54 | 55 | -b " " 56 | 57 | -b "" 58 | ``` 59 | 60 | \ and \ are in the same format specified by the “*dateformat=*” and “*timeformat=*” options specified in the [main] section of the dupReport.rc file. For example, if dateformat="MM/DD/YYYY" in the .rc file then \ for the -b option should be "04/24/2020". If timeformat="HH:MM:SS" in the .rc file then \ should be "12:37:45". 61 | 62 | Specifying a \ without a corresponding \ will imply the use of "00:00:00" (beginning of the day) as the \. 63 | 64 | \ allows you to roll back the database a set number of seconds/minutes/hours/days/weeks from "now" and uses the following format: 65 | 66 | ``` 67 | -b "Xs,Xm,Xh,Xd,Xw" 68 | ``` 69 | 70 | Replace the 'X' above with the number of seconds/minutes/hours/days/weeks you want to roll back the database. You can specify more than one time unit, but each must be separated by a comma with no spaces between them. 71 | 72 | 73 | 74 | (Return to [Main Page](readme.md)) -------------------------------------------------------------------------------- /docs/Config-SrcDestPairs.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Source-Destination Pairs 4 | 5 | The heart of dupReport's functionality is the use of "Source-Destination Pairs" to identify the source and destination systems for each backup job. The default dupReport configuration requires that Duplicati backup jobs be named in a way that indicates what is being backed up and where it is going. For instance, a Duplicati backup job named: “MediaServer-NAS" would show up in dupReport as: 6 | 7 | > **Source:** MediaServer **Destination:** NAS 8 | 9 | Source-Destination pairs are used in dupReport in the following format: 10 | 11 | ``` 12 | 13 | ``` 14 | 15 | 16 | Where: 17 | 18 | - \ is a series of alphanumeric characters 19 | - \ is a single character (typically one of the "special" characters like !,@,#,$,-,*, etc.) and **CAN NOT** be a character you use in any of your Source or Destination names 20 | - \ is a series of alphanumeric characters 21 | 22 | Spaces are allowed in the \, \, and \ if you define these specifications carefully, *though they are not recommended*, at least when you are just starting out. 23 | 24 | **Regular Expressions** 25 | 26 | dupReport uses *Regular Expressions* (also known as "regex") to define the source, destination, and delimiter specifications. Regular expressions allow you to specify patterns of characters in a shorthand way so you can match those patterns against a variety of text. If you are not familiar with regular expressions, you have two options: 27 | 28 | 1. **Accept the defaults**. In the default dupReport configuration the Source and Destination each consist of a single string of characters without spaces and the delimiter is the '-' character (for example, "MediaServer-NAS".) If you name all your backup jobs this way your Duplicati emails should be processed properly. No additional regular expression knowledge is needed on your part. 29 | 2. **Learn about regular expressions**. Building your skill set is always a good thing. You can start at [RegexOne](https://regexone.com/) (not an endorsement, just a good site), though there are lots of good regular expressions tutorials on the Internet. dupReport is written in Python, if you are looking for a language-specific tutorial. 30 | 31 | **Specifying Source, Destination, and Delimiter** 32 | 33 | The full source-destination regex specification is: 34 | 35 | ``` 36 | 37 | ``` 38 | 39 | dupReport allows you to define the Source, Destination, and Delimiter regular expressions in the [main] section of the dupReport.rc file. The defaults are: 40 | 41 | ``` 42 | [main] 43 | srcregex=\w+ 44 | destregex=\w+ 45 | srcdestdelimiter=- 46 | ``` 47 | 48 | | Option | Defintion | Meaning | 49 | | ---------------- | --------- | ------------------------------------------------------------ | 50 | | srcregex | \w+ | a single string of one or more alphanumeric characters (i.e., A-Z, a-z, 0-9, and the underscore character'_') | 51 | | destregex | \w+ | a single string of one or more alphanumeric characters (i.e., A-Z, a-z, 0-9, and the underscore character'_') | 52 | | srcdestdelimiter | - | The '-' character | 53 | 54 | Note that whitespace characters (space, tab, newlines, etc.) are **not** allowed in Source or Destination names using this default definition. If the Source or Destination systems in your Duplicati backup job names contain white space characters they will not match the expression. 55 | 56 | Using these definitions, the full Backup name dupReport will look for (\\\) is: 57 | 58 | (\w+)-(\w+) 59 | 60 | With this specification, the following list explains what will match and not match: 61 | 62 | **Will Match** 63 | 64 | - MediaServer-NAS 65 | - WebServer-GDrive 66 | - Development-B2 67 | 68 | **Will Not Match** 69 | 70 | - Media Server-NAS (space in Source name) 71 | - WebServer-G Drive (space in Destination name) 72 | - Development - B2 (space surrounding the delimiter character) 73 | - System Backup (no delimiter character to define the Destination system) 74 | 75 | 76 | 77 | If you want to get a bit more creative and allow space characters in your Source or Destination systems you can change the Source or destination to the following: 78 | 79 | ``` 80 | [main] 81 | srcregex=[^-]+ 82 | destregex=.+ 83 | srcdestdelimiter=- 84 | ``` 85 | 86 | | Option | Defintion | Meaning | 87 | | ---------------- | --------- | ------------------------------------------------------------ | 88 | | srcregex | [^-]+ | One or more characters up to (but not including) the '-' character | 89 | | destregex | .+ | All characters up to the end of the line | 90 | | srcdestdelimiter | - | The '-' character | 91 | 92 | Using these definitions, the full Backup name dupReport will look for (\\\) is: 93 | 94 | (\[^-]\+)-(.+) 95 | 96 | With this specification, the following list explains what will match and not match: 97 | 98 | **Match** 99 | 100 | - Media Server-NAS 101 | - WebServer-G Drive 102 | - Computer Under This Desk-Computer Under Other Desk 103 | 104 | **Will Not Match** 105 | 106 | - Media Server - NAS (OK, this will *technically* work, but your Source name will have an extra space at the end and your Destination system will have an extra space at the beginning. This has the potential to confuse things down the road.) 107 | 108 | 109 | 110 | If you want to allow spaces anywhere in the Source-Destination specification, you can try something like the following: 111 | 112 | ``` 113 | [main] 114 | srcregex=[^-]+ 115 | destregex=.+ 116 | srcdestdelimiter= \s*-\s* 117 | ``` 118 | 119 | | Option | Defintion | Meaning | 120 | | ---------------- | --------- | ------------------------------------------------------------ | 121 | | srcregex | [^-]* | One or more characters up to (but not including) the '-' character | 122 | | destregex | .+ | One or more characters up to the end of the line | 123 | | srcdestdelimiter | \s\*-\s\* | An arbitrary number of spaces, followed by the '-' character, followed by an arbitrary number of spaces | 124 | 125 | 126 | 127 | **A WORD OF CAUTION** 128 | 129 | Using advanced regex patterns to match your Duplicati backup job names can get extremely tricky and the results can be unexpected. The best advice is to keep your naming convention as simple as possible (i.e., "Source-Destination") and things will work much better. dupReport relies on the Source-Destination pair format for all of its operations. If you do not properly specify your Source-Destination pair formats in both the program (through the dupReport.rc file) and in Duplicati (through proper job naming) none of this will work for you. 130 | 131 | If you want to proceed with using advanced regular expressions in dupReport, you should use a good regular expression testing program to thoroughly test your regexes and understand how they work before using them in the program. [Regex101](https://regex101.com/) is a good site to use, though there are many others available on the Internet. 132 | 133 | 134 | 135 | # Identifying Emails of Interest 136 | 137 | dupReport scans the incoming mailbox looking for backup job emails. However, there may be hundreds (or thousands) of emails in the inbox, only a few of which contain information about Duplicati backup jobs. dupReport identifies "Emails of Interest" by matching the email's subject line against a pattern defined in the dupReport.rc file. If the pattern matches, the email is analyzed. If the pattern does not match, the email is ignored. 138 | 139 | You can specify the text that dupReport tries to match by adjusting the subjectregex= option in the [main] section of the dupReport.rc file. subjectregex is the regular expression definition for the desired phrase. 140 | 141 | The default for this option is: 142 | 143 | ``` 144 | [main] 145 | subjectregex=^Duplicati Backup report for 146 | ``` 147 | 148 | This instructs dupReport to look for emails whose subject line start with the phrase, "Duplicati backup report for", which is the default used in Duplicati's “send-mail-subject” advanced option. 149 | 150 | If you change the subjectregex option in the dupReport.rc file, or change the "send-mail-subject" advanced option in Duplicati, be sure that the two patterns match or you will not be able to properly match incoming emails. 151 | 152 | For example, Duplicati allows you to change the "send-mail-subject" to: 153 | 154 | ``` 155 | Duplicati %PARSEDRESULT%, %OPERATIONNAME% report for %backup-name% 156 | ``` 157 | 158 | This can result in an email with the subject line: 159 | 160 | ``` 161 | Duplicati Warning, Backup report for FileServer-B2 162 | ``` 163 | 164 | In this case, you would change the subjectregex option in the dupReport.rc file to: 165 | 166 | ``` 167 | subjectregex=^Duplicati ([\w ]*, |)Backup report for 168 | ``` 169 | 170 | 171 | 172 | (Return to [Main Page](readme.md)) -------------------------------------------------------------------------------- /docs/DataStructures/ConfigFormat.json: -------------------------------------------------------------------------------- 1 | { 2 | "defaults": { 3 | "layout": ["report1", "report2", "report3"], 4 | "title": "xxxx", 5 | "sizedisplay": "xxxxx", 6 | . 7 | . 8 | . 9 | . 10 | } 11 | "sections": [ 12 | { 13 | "name": "report1", 14 | "type": table 15 | "options": {} 16 | }, 17 | { 18 | "name": "report2", 19 | "type": table 20 | "options": {} 21 | }, 22 | { 23 | "name": "report3", 24 | "type": noactivity 25 | "options": {} 26 | }, 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /docs/DataStructures/ReportFormat.json: -------------------------------------------------------------------------------- 1 | { 2 | "sections": [ 3 | { 4 | "name": "bydest", 5 | "Title": "Table Title", 6 | "type": "report" 7 | "columnCount": 4, 8 | "columnNames": [ [colName, ColTitle, bgColor, markup], [colA,ColumnA, bgColor, markup], [colB,ColumnB, bgColor, markup], [colC, ColumnC, bgColor, markup], ... ], 9 | # Markup = 0x01 - Bold 10 | # 0x02 - Italic 11 | # 0x04 - Underline 12 | # 0x08 - Left align 13 | # 0x10 - Center align 14 | # 0x20 - Right align 15 | "inlineColumnCount": 2, 16 | "inlineColumns": [ [colName, ColTitle, bgColor, markup], [colA,ColumnA, bgColor, markup], ... ], 17 | "groups": [ 18 | { 19 | "groupHeading": ["Group 1", bgColor, markup ] 20 | "dataRows": [ 21 | [ [colVal, bgColor, markup], [colVal, bgColor, markup], [colVal, bgColor, markup], ... ], 22 | [msgVal, bgColor, markup, msgTitle], 23 | [warnVal, bgColor, markup, warnTitle], 24 | [errVal, bgColor, markup, errorTitle], 25 | [dataVal, bgColor, markup, dataTitle], 26 | [ [colVal, bgColor, markup], [colVal, bgColor, markup], [colVal, bgColor, markup], ... ], 27 | [ [colVal, bgColor, markup], [colVal, bgColor, markup], [colVal, bgColor, markup], ... ], 28 | ], 29 | }, 30 | { 31 | "groupHeading": ["Group 2", bgColor, markup ] 32 | "dataRows": [ 33 | [ [colVal, bgColor, markup], [colVal, bgColor, markup], [colVal, bgColor, markup], ... ], 34 | [msgVal, bgColor, markup, msgTitle], 35 | [warnVal, bgColor, markup, warnTitle], 36 | [errVal, bgColor, markup, errorTitle], 37 | [dataVal, bgColor, markup, dataTitle], 38 | [ [colVal, bgColor, markup], [colVal, bgColor, markup], [colVal, bgColor, markup], ... ], 39 | [ [colVal, bgColor, markup], [colVal, bgColor, markup], [colVal, bgColor, markup], ... ], 40 | [dataVal, bgColor, markup, dataTitle], 41 | ], 42 | }, 43 | ], 44 | }, 45 | "sections": [ 46 | { 47 | "name": "versionnum", 48 | "Title": "Release Versions", 49 | "type": "report" 50 | "columnCount": 4, 51 | "columnNames": [ [colName, ColTitle, bgColor, markup], [colA,ColumnA, bgColor, markup], [colB,ColumnB, bgColor, markup], [colC, ColumnC, bgColor, markup], ... ], 52 | # Markup = 0x01 - Bold 53 | # 0x02 - Italic 54 | # 0x04 - Underline 55 | # 0x08 - Left align 56 | # 0x10 - Center align 57 | # 0x20 - Right align 58 | "inlineColumnCount": 2, 59 | "inlineColumns": [ [colName, ColTitle, bgColor, markup], [colA,ColumnA, bgColor, markup], ... ], 60 | # Row Types: 61 | # 0x01 - Report title 62 | # 0x02 - Group Heading 63 | # 0x04 - Title Row 64 | # 0x08 - Data Row 65 | # 0x10 - Warn/Err/Log data 66 | datarows: [ 67 | [[RowType = 0x01, colspan = inlineColumnCount], [rptTitle, bgColor, markup]], 68 | [[RowType = 0x02, colspan = inlineColumnCount], [heading, bgColor, markup]], 69 | [[RowType = 0x04, colspan = 1], [colTitVal, bgColor, markup], [colTitVal, bgColor, markup], [colTitVal, bgColor, markup], ... ]], 70 | [[RowType = 0x08, colspan = 1], [colVal, bgColor, markup], [colVal, bgColor, markup], [colVal, bgColor, markup], ... ]], 71 | [[RowType = 0x08, colspan = 1], [colVal, bgColor, markup], [colVal, bgColor, markup], [colVal, bgColor, markup], ... ]], 72 | [[RowType = 0x10, colspan = inlineColumnCount], [msgVal, bgColor, markup, msgTitle]], 73 | [[RowType = 0x10, colspan = inlineColumnCount], [warnVal, bgColor, markup, warnTitle]], 74 | [[RowType = 0x10, colspan = inlineColumnCount], [errVal, bgColor, markup, errorTitle]], 75 | [[RowType = 0x10, colspan = inlineColumnCount], [dataVal, bgColor, markup, dataTitle]], 76 | [[RowType = 0x08, colspan = 1], [colVal, bgColor, markup], [colVal, bgColor, markup], [colVal, bgColor, markup], ... ]], 77 | [[RowType = 0x08, colspan = 1], [colVal, bgColor, markup], [colVal, bgColor, markup], [colVal, bgColor, markup], ... ]], 78 | [[RowType = 0x10, colspan = inlineColumnCount], [msgVal, bgColor, markup, msgTitle]], 79 | [[RowType = 0x10, colspan = inlineColumnCount], [warnVal, bgColor, markup, warnTitle]], 80 | [[RowType = 0x10, colspan = inlineColumnCount], [errVal, bgColor, markup, errorTitle]], 81 | [[RowType = 0x08, colspan = 1], [dataVal, bgColor, markup, dataTitle]], 82 | [[RowType = 0x08, colspan = 1], [colVal, bgColor, markup], [colVal, bgColor, markup], [colVal, bgColor, markup], ... ]], 83 | [[RowType = 0x08, colspan = 1], [colVal, bgColor, markup], [colVal, bgColor, markup], [colVal, bgColor, markup], ... ]], 84 | [[RowType = 0x02, colspan = inlineColumnCount], [heading, bgColor, markup]], 85 | [[RowType = 0x04, colspan = 1], [colTitVal, bgColor, markup], [colTitVal, bgColor, markup], [colTitVal, bgColor, markup], ... ]], 86 | [[RowType = 0x08, colspan = 1], [colVal, bgColor, markup], [colVal, bgColor, markup], [colVal, bgColor, markup], ... ]], 87 | [[RowType = 0x08, colspan = 1], [colVal, bgColor, markup], [colVal, bgColor, markup], [colVal, bgColor, markup], ... ]], 88 | ], 89 | }, 90 | { 91 | "name": "noactivity", 92 | "Title": "title", 93 | "type": "noactivity" 94 | "columnCount": 3, 95 | "columnNames": [ ['source', 'Source', bgColor, markup], ['destination', 'Destination', bgColor, markup], [lastActivity, 'Last Activity', bgColor, markup]], 96 | "inlineColumnCount": 3, 97 | "inlineColumnNames": [ ['source', 'Source', bgColor, markup], ['destination', 'Destination', bgColor, markup], [lastActivity, 'Last Activity', bgColor, markup]], 98 | "groups": [ 99 | "dataRows": [ 100 | [ [colVal, bgColor, markup], [colVal, bgColor, markup], [colVal, bgColor, markup] ], 101 | [ [colVal, bgColor, markup], [colVal, bgColor, markup], [colVal, bgColor, markup] ], 102 | [ [colVal, bgColor, markup], [colVal, bgColor, markup], [colVal, bgColor, markup] ], 103 | [ [colVal, bgColor, markup], [colVal, bgColor, markup], [colVal, bgColor, markup] ], 104 | ], 105 | ], 106 | } 107 | ], 108 | } 109 | -------------------------------------------------------------------------------- /docs/DataStructures/fldDefs.txt: -------------------------------------------------------------------------------- 1 | fldDefs = { 2 | # field [0]alignment [1]hdrDef [2]colDef 3 | 'source': ('left', '20', '20'), 4 | 'destination': ('left', '20', '20'), 5 | 'dupversion': ('left', '35', '35'), 6 | 'date': ('left', '13', '13'), 7 | 'time': ('left', '11', '11'), 8 | 'duration': ('right', '>15', '>15'), 9 | 'examinedFiles': ('right', '>12', '>12,'), 10 | 'examinedFilesDelta': ('right', '>12', '>+12,'), 11 | 'sizeOfExaminedFiles': ('right', '>20', '>20,.2f'), 12 | 'fileSizeDelta': ('right', '>20', '>20,.2f'), 13 | 'addedFiles': ('right', '>12', '>12,'), 14 | 'deletedFiles': ('right', '>12', '>12,'), 15 | 'modifiedFiles': ('right', '>12', '>12,'), 16 | 'filesWithError': ('right', '>12', '>12,'), 17 | 'parsedResult': ('left', '>13', '>13'), 18 | 'lastseen': ('left', '50', '50'), 19 | 'messages': ('center', '^50', '^50'), 20 | 'warnings': ('center', '^50', '^50'), 21 | 'errors': ('center', '^50', '^50'), 22 | 'logdata': ('center', '^50', '^50') 23 | } 24 | 25 | dataRowTypes = { 26 | 'rptTitle': 0x01, 27 | 'grpHeading': 0x02, 28 | 'rowHead': 0x04, 29 | 'data': 0x08, 30 | 'wemData': 0x10, 31 | 'singleLine': 0x20 32 | } 33 | 34 | markupDefs = { 35 | 'bold': 0x01, 36 | 'italic': 0x02, 37 | 'underline': 0x04, 38 | 'left': 0x08, 39 | 'center': 0x10, 40 | 'right': 0x20 41 | } 42 | 43 | -------------------------------------------------------------------------------- /docs/Installation.md: -------------------------------------------------------------------------------- 1 | # Installing dupReport 2 | 3 | dupReport installation is easy and quick. To begin, download the dupReport files from the GitHub repository and place them in the directory of your choice. Then, make sure Python 3 is installed on your system and operating correctly. 4 | 5 | ## First-time Installation - Guided Setup 6 | 7 | Installing dupReport is easy. To use all the default values, execute the following command: 8 | 9 | | Operating System | Command Line | 10 | | ---------------- | ----------------------------------------------------------- | 11 | | Linux | **user@system:~$** /path/to/dupReport/dupReport.py | 12 | | Windows | **C:\users\me>** python.exe \path\to\dupreport\dupReport.py | 13 | 14 | dupReport is self-initializing, so the first time you run the program the program will automatically create a database, initialize the .rc configuration file with a bunch of default values, then exit. By default, the database and .rc files will be created in the same directory where the dupReport.py script is located. If you want them created in another location use the following program options: 15 | 16 | ``` 17 | dupReport.py -r -d 18 | ``` 19 | 20 | You can specify either a directory or a full path name for the \ and \ options. dupReport will create the .rc and database files in their respective paths. If the directories do not already exist or you do not have read and write access permission to those locations the program will exit with an error message. Use of the default values for these file paths is recommended, but the option is there if you want it. 21 | 22 | The first time the program is run it will launch the Guided Setup to assist you in filling in some of the basic configuration options. The Guided Setup will ask a series of questions about how you want the program to operate, where your email servers are, and how to log into them. Defaults for each question ate provided in brackets ([]). If you hit 'Enter' without responding to a question it will take the default response as your answer. 23 | 24 | Here is an explanation for each question in the Guided Setup: 25 | 26 | ``` 27 | Welcome to the dupReport guided setup. 28 | Here we'll collect some basic information from you to help configure the program. 29 | Let's get started... 30 | 31 | Valid date formats in dupReport are: 32 | YYYY/MM/DD 33 | DD/MM/YYYY 34 | YYYY/DD/MM 35 | YYYY.MM.DD 36 | YYYY-DD-MM 37 | MM.DD.YYYY 38 | DD-MM-YYYY 39 | MM/DD/YYYY 40 | YYYY-MM-DD 41 | MM-DD-YYYY 42 | YYYY.DD.MM 43 | DD.MM.YYYY 44 | Please enter the date format for your locale [MM/DD/YYYY]: DD/MM/YYYY 45 | ``` 46 | 47 | Since most of what dupReport does involves interpreting date & time strings it's important that it knows what date format you use in your locale. There are a limited number of format options, so select the one that use use for most of your systems. If you work with multiple systems that use different date formats you can specify this on a per-system basis in the .rc file later on. 48 | 49 | ``` 50 | Do you use a syslog server or log aggregator where you want the dupReport logs sent [n]: y 51 | Enter the server name/ip and port of your syslog server [localhost:514]:syslog.localnet.com: 516 52 | ``` 53 | 54 | If you want to forward dupReport's log output to a syslog server or log aggregator you can specify the system (IP address or FQDN) and port number here. 55 | 56 | ``` 57 | Do you want file sizes displayed in bytes, megabytes, or gigabytes (enter 'byte', 'mb', or 'gb') [byte]: gb 58 | ``` 59 | 60 | Self-explanatory, hopefully. Since backup sizes can get quite large, this helps compact the report a bit by rounding the file & backup sizes to megabytes or gigabytes. 61 | 62 | ``` 63 | Now we'll need information about your 'incoming' email server. 64 | This is the email server where your Duplicati job results are sent. dupReport will collect emails from this server and analyze them. 65 | You can use more than one incoming server with dupReport, but for now we'll collect information for just one. 66 | Does the 'incoming' server use IMAP or POP3 [imap]: imap 67 | 68 | ``` 69 | 70 | ``` 71 | Please provide an IP address or DNS name for the 'incoming' server [localhost]: imap.gmail.com 72 | Please provide a port number for the 'incoming' server [993]: 73 | Does the 'incoming' server use SSL/TLS encryption [Y]: 74 | ``` 75 | 76 | These questions et the backs server & port options for the incoming server & indicates if TLS/SSL is required. 77 | 78 | ``` 79 | Please enter the user ID used to log into the 'incoming' server [youremail@emailservice.com]: sampleid@gmail.com 80 | Please enter the password used to log into the 'incoming' server [secretpassword]: 81 | ``` 82 | 83 | This is the user ID and password you use to log into the incoming server. dupReport needs these to log in and retrieve Duplicati emails from your account. Your password will not be displayed on the console. 84 | 85 | ``` 86 | Please enter the IMAP folder name to where Duplicati emails are stored on the 'incoming' server [INBOX]: INBOX 87 | ``` 88 | 89 | If your server uses IMAP you need to specify the folder where the email is stored. (Note: Gmail uses the term 'tags'). This option is only needed for IMAP systems. If you told dupReport you are using POP3 yo will not see this question. 90 | 91 | ``` 92 | Now we'll need information about your 'outgoing' email server. 93 | This is the email server that dupReport will use to send the results of its analysis. 94 | You can use more than one outgoing server with dupReport, but for now we'll collect information for just one. 95 | Please provide an IP address or DNS name for the 'outgoing' server [localhost]: smtp.gmail.com 96 | Please provide a port number for the 'outgoing' server [587]: 97 | Does the 'outgoing' server use SSL/TLS encryption [Y]: 98 | Please enter the user ID used to log into the 'outgoing' server [youremail@emailservice.com]: sampleid@gmail.com 99 | Please enter the password used to log into the 'outgoing' server [secretpassword]: 100 | 101 | ``` 102 | 103 | These options mean the same thing as they did for the 'incoming' server questions. Most likely you will use the same system for both incoming and outgoing emails, in which case your answers here will be the same as they were for the 'incoming' section. 104 | 105 | ``` 106 | What email address should be used for the sender [youremail@emailservice.com]: myid@gmail.com 107 | What is the 'friendly name' of the outgoing email sender [Your Name]: dupReport Summary 108 | ``` 109 | 110 | This tells dupReport what to put in for the "sent from" fields in the outgoing email report. 111 | 112 | ``` 113 | Who should the outgoing emails be sent to [receiver@emailservice.com]:myid@gmail.com 114 | ``` 115 | 116 | This tells dupReport where the resulting report email goes. 117 | 118 | ``` 119 | OK, all set. We've recorded your responses and (hopefully) the program will work perfectly now 120 | If it doesn't, you can change the configuration in the .rc file located at /home/user/dupReport/dupReport.rc 121 | ``` 122 | 123 | More information on the .rc file configuration can be found in the [“dupReport .rc File Configuration” section](RcFileConfig.md). 124 | 125 | ## Upgrading From a Previous Version 126 | 127 | If you have been running an earlier version of dupReport, the program will automatically update the dupReport.rc file and the dupReport.db database to the latest versions. Depending on the extent of the changes, the program may indicate that you need to edit the dupReport.rc file to set any new options correctly. 128 | 129 | Beginning with Version 3.0.0, dupReport will automatically make a backup of your database (.db) and configuration (.rc) file before the upgrade in case something goes wrong during the process. However, as a precaution **it is always recommended that you backup your .rc and .db files yourself and keep them in a safe place** before proceeding with the upgrade until you're sure everything is working properly. 130 | 131 | 132 | 133 | 134 | 135 | (Return to [Main Page](readme.md)) -------------------------------------------------------------------------------- /docs/QuickStart.md: -------------------------------------------------------------------------------- 1 | # How can I get started quickly? 2 | 3 | There's a **lot** of information in this guide about how to install, configure, and run dupReport. But you probably don't want to read all that, you want to just start running the program! Here's the quick & dirty guide to get you started. 4 | 5 | 1. Make sure Python 3.x is available and running on your system. For information on downloading and installing Python see the [Python Software Foundation web site](https://www.python.org/). 6 | 2. Make sure your Duplicati backup jobs are named properly. The best naming scheme (at least to get you started) is: 7 | 8 | ``` 9 | - 10 | ``` 11 | 12 | where \ is the name of the computer where the files are located and \ is the place where they are going to. For example, if your computer is named "MediaServer" and you are backing up to a directory on the "NAS" computer, the backup name would be: 13 | 14 | ``` 15 | MediaServer-NAS 16 | ``` 17 | 18 | For more interesting information on naming backup jobs see the section on ["Source-Destination Pairs."](Config-SrcDestPairs.md) 19 | 20 | 3. Configure a Duplicati job to send its output report to an email account. See the [Duplicati documentation for the "send-mail" advanced email options](https://duplicati.readthedocs.io/en/latest/06-advanced-options/#send-mail-to) to learn how to do this. 21 | 4. Run at least one backup with the newly-named backup job so that you have an email that dupReport can find on your email server. 22 | 5. Download the dupReport code from the [GitHub page](https://github.com/HandyGuySoftware/dupReport) by clicking the "Clone or download" button on the dupReport GitHub page, then click "Download ZIP." This will put the ZIP file on your system. Unzip the file to the directory of your choice. 23 | 6. Run dupReport.py to install the default configuration files. The first time you run the program it will launch the Guided Setup that will lead you through configuring the most common options. The instructions for the Guided Setup can be found in the ["Installation" section.](Installation.md) Read that section, do what it says, then come back here. 24 | 7. Once the Guided Setup completes it will continue to run the program and collect backup emails. If there are any errors or things that need to be adjusted in the configuration you will receive a message on what those are. 25 | 8. For subsequent runs, run the dupReport.py program using the appropriate command line as shown in the ["Basic Command LIne" section](CommandLine.md). 26 | 9. Let the program run to completion. When it is complete you should get an email with the output report. 27 | 28 | That's the quick way to do it! Now that you've seen how it works, please read the rest of this guide for LOTS more information on how to configure dupReport to get the most out of the program. 29 | 30 | 31 | 32 | 33 | 34 | (Return to [Main Page](readme.md)) 35 | 36 | -------------------------------------------------------------------------------- /docs/RcFileConfig-Apprise.md: -------------------------------------------------------------------------------- 1 | ## dupReport.rc File - [apprise] Section & Push Notifications 2 | 3 | Beginning with version 2.2.3, dupReport supports the Apprise push notification package from [@caronc](https://github.com/caronc). From the [Apprise GitHub page](https://github.com/caronc/apprise): 4 | 5 | > "Apprise allows you to take advantage of just about every notification service available to us today. Send a notification to almost all of the most popular services out there today (such as Telegram, Slack, Twitter, etc). The ones that don't exist can be adapted and supported too!" 6 | 7 | Once Apprise is enabled in dupReport, you can send configurable push notifications of backup job status to any service that Apprise supports. Apprise is not required to run dupReport. If you don't want to use Apprise notifications, you can still use all the other features of dupReport without worry. 8 | 9 | See the [Apprise GitHub page](https://github.com/caronc/apprise) for instructions on installing Apprise on your system. The installation page also includes instructions for running Apprise directly from the command line. Properly configuring the Apprise URLs for notification can be a tricky business and may involve a lot of trial-and-error before you get it right. Therefore, **we strongly suggest** you test out your Apprise URLs using the command line tool to make sure they work properly **before** trying to use them through dupReport. 10 | 11 | **Notification timing**: Because dupReport runs as a "batch" process and does not receive Duplicati backup job notifications in real time, Apprise notifications will only be sent at the time dupReport is run. For example, if the backup job completes at 1:00 AM but dupReport does not run until 6:00 PM, the Apprise notification will not be sent until 6:00 PM once dupReport has completed its processing. 12 | 13 | The default .rc file configuration includes an [apprise] section for configuring Apprise options. 14 | 15 | ``` 16 | [apprise] 17 | enabled = false 18 | ``` 19 | 20 | By default, Apprise support is disabled, as indicated by the 'enabled = false' option. To enable Apprise, simply change this option to 'true'. If dupReport sees this option set to 'true' it will load the Apprise libraries and configure the proper notifications. If dupReport does not see sees this option set to 'true' it will simply carry on without loading any Apprise support. 21 | 22 | **Important Support Note**: dupReport includes Apprise notifications because we feel it is a useful feature for our users. While we can support and address issues with dupReport's use of Apprise, we cannot provide support for Apprise issues or feature requests. Please contact the Apprise developer directly on the [Apprise GitHub page](https://github.com/caronc/apprise). 23 | 24 | ------ 25 | 26 | **Defining Notification Services** 27 | 28 | ``` 29 | services = [, , , …] 30 | ``` 31 | 32 | The services option contains the URL(s) that Apprise will use for its notifications. These are the same URLs that you used when testing Apprise from the command line. For example, if the Apprise command line is: 33 | 34 | > apprise -t 'my title' -b 'my notification body' '' 35 | 36 | The services option in the dupReport.rc file would be: 37 | 38 | ``` 39 | services = mailto://myemail:mypass@gmail.com 40 | ``` 41 | 42 | (Note that the "services=" option in the .rc file does not use quotes (')) 43 | 44 | If you want to use multiple notification services, separate the URLs for each service with a comma, for example: 45 | 46 | ``` 47 | services = , pbul://o.gn5kj6nfhv736I7jC3cj3QLRiyhgl98b 48 | ``` 49 | 50 | ------ 51 | 52 | **Adding a Message Title** 53 | 54 | ``` 55 | title = 56 | ``` 57 | 58 | This is the text that will be used for the title of the Apprise message. The default title text is: 59 | 60 | *Apprise Notification for #SRCDEST# Backup* 61 | 62 | (See the "Keyword Substitution" section below for more information on title text.) 63 | 64 | ------ 65 | 66 | **Specifying the Message Body** 67 | 68 | ``` 69 | body = <body text> 70 | ``` 71 | 72 | This is the text that will be used for the body of the Apprise message. The default body text is: 73 | 74 | *Completed at #COMPLETETIME#: #RESULT# - #ERRMSG#* 75 | 76 | (See the "Keyword Substitution" section below for more information on body text.) 77 | 78 | ------ 79 | 80 | **Keyword Substitution** 81 | 82 | You can supply keywords within the title= and body= options to customize the way it looks. Available keywords are: 83 | 84 | - \#SOURCE#: Inserts the backup job's source name 85 | - \#DESTINATION#: Inserts the backup job's destination name 86 | - \#SRCDEST#: Inserts the backup job's full \<source>-\<destination> name 87 | - \#RESULT#: Insert's the backup job's result status 88 | - \#MESSAGE#: Inserts the 'Message" field from the status email 89 | - \#WARNMSG#: Inserts the "Warning" message field from the status email 90 | - \#ERRMSG#: Inserts the "Error" message field from the status email 91 | - \#COMPLETETIME#: Inserts the backup job completion time 92 | 93 | ------ 94 | 95 | **Truncating Long Titles and Messages** 96 | 97 | ``` 98 | titletruncate = 0 99 | ``` 100 | 101 | Truncates the length of the title field. May be useful for notification services that limit available space in the title field. The default is 0 (no truncation). 102 | 103 | ``` 104 | bodytruncate = 0 105 | ``` 106 | 107 | Truncates the length of the body field. May be useful for notification services that limit available space in the body field. The default is 0 (no truncation). 108 | 109 | ------ 110 | 111 | **Specifying the Types of Messages to Send** 112 | 113 | ``` 114 | msglevel = failure 115 | ``` 116 | 117 | Indicates the types of messages that dupReport will send to Apprise. This is based on the Parsed Result field from the Duplicati status emails. The following table shows the possible values and their meaning. 118 | 119 | | Value | Types of Messages Sent | 120 | | ------------------ | ----------------------------- | 121 | | msglevel = success | success, warning, and failure | 122 | | msglevel = warning | warning and failure | 123 | | msglevel = failure | Failure only | 124 | 125 | ------ 126 | 127 | **Using Apprise for Email Notification** 128 | 129 | If you want to use email notifications through Apprise instead of direct email from dupReport for notifications, use the '-x' option on the command line to suppress sending of dupReport emails. 130 | 131 | 132 | 133 | 134 | 135 | (Return to [Main Page](readme.md)) -------------------------------------------------------------------------------- /docs/RcFileConfig-EmailManagement.md: -------------------------------------------------------------------------------- 1 | ## dupReport.rc File - Email Management 2 | 3 | dupReport needs to know where your email servers are located to properly access and parse Duplicati log emails. Email servers can be one of two types: 4 | 5 | - **Incoming**: servers that host Duplicati status emails that are read into the program. dupReport can access these systems using either IMAP or POP3 protocols. dupReport will pull mail from as many incoming email servers as you specify, but you must specify at least one incoming email server. 6 | - **Outgoing**: server(s) that you use to send out dupReport results. These servers must be running the SMTP protocol. You can specify more than one outgoing server, but you must specify at least one . dupReport will attempt to connect to the servers - in the order they are specified - until it is able to successfully connect to one. It will then use that outgoing server to send email for the remainder of the program. 7 | 8 | ------ 9 | 10 | **Specifying Email Servers** 11 | 12 | The email servers to use are specified in the [main] section of the .rc file: 13 | 14 | ``` 15 | [main] 16 | emailservers = server1, server2, server3 17 | ``` 18 | 19 | You can specify the servers in any order you like. By default, dupReport will create 2 servers, named "incoming" and "outgoing": 20 | 21 | ``` 22 | [main] 23 | emailservers = incoming, outgoing 24 | ``` 25 | 26 | You can also specify the servers you want to use on the command line using the -e option: 27 | 28 | ``` 29 | $ dupReport.py -e incoming,outgoing 30 | ``` 31 | 32 | **NOTE:** If you use the command line option there can not be any spaces between the server names and the commas (,). 33 | 34 | ------ 35 | 36 | **Email Server Specification Sections** 37 | 38 | Each email server specified in the *[main]emailservers=* option (or using the -e command line option) must have an associated specification section in the .rc file. For example, your .rc file should look something like this: 39 | 40 | ``` 41 | [main] 42 | emailservers = gmail-imap, yahoo-pop3, gmail-smtp, yahoo-smtp 43 | . 44 | . 45 | . 46 | [gmail-imap] 47 | protocol = imap 48 | . 49 | . 50 | . 51 | [yahoo-pop3] 52 | protocol = pop3 53 | . 54 | . 55 | . 56 | [gmail-smtp] 57 | protocol = smtp 58 | . 59 | . 60 | . 61 | [yahoo-smtp] 62 | protocol = smtp 63 | . 64 | . 65 | . 66 | ``` 67 | 68 | You can name your servers whatever you like, but the names used in the *emailservers=* option **<u>must exactly match</u>** the names used for each of the server specification sections. 69 | 70 | ------ 71 | 72 | **Email Server Options** 73 | 74 | Each server section uses similar options to describe the server's options. Each option described below will note whether it is needed for IMAP, POP3, and/or SMTP servers 75 | 76 | ``` 77 | protocol=<name> 78 | ``` 79 | 80 | Specify the transport protocol used to connect to the email server. Valid '\<name>' options for incoming servers are 'imap' and 'pop3'. Outgoing servers may only use 'smtp' as the '\<name>' option. dupReport will use this option to determine if this is an "incoming" or "outgoing" server. **(IMAP, POP3, SMTP)** 81 | 82 | **IMAP is highly recommended** for incoming servers. POP3 has some severe limitations when it comes to handling email. If you must use POP3 for whatever reason, make sure the "Leave messages on server" option is enabled in all your POP3 clients and/or your POP3 server. The default behavior for POP3 is to remove messages from the email server as soon as they are read, so using multiple email clients on the same server will interfere with each's ability to read email. Setting this option in your email server tells the server it to leave the messages on the server for other clients to use. Different systems configure this option differently, so check the documentation for your email system to see where this is set. 83 | 84 | ``` 85 | server=localhost 86 | ``` 87 | 88 | DNS name or IP address of email server. **(IMAP, POP3, SMTP)** 89 | 90 | ``` 91 | port=995 92 | ``` 93 | 94 | IMAP, POP3, or SMTP port for incoming email server. Typical values for these ports are: 95 | 96 | | Protocol | Port | 97 | | -------- | -------------- | 98 | | IMAP | 993 | 99 | | POP3 | 995 | 100 | | SMTP | 587 (with TLS) | 101 | 102 | Your server may use different port numbers. Please contact (or Google) your email provider to see what the values for their systems should be. **(IMAP, POP3, SMTP)** 103 | 104 | ``` 105 | encryption=tls 106 | ``` 107 | 108 | Specify encryption used by incoming email server. Can be set to 'none', 'tls' (the default), or 'ssl' **(IMAP, POP3, SMTP)** 109 | 110 | ``` 111 | account=<account_name> 112 | ``` 113 | 114 | User ID on the email system. **(IMAP, POP3, SMTP)** 115 | 116 | ***NOTE:*** If you are using Gmail as your incoming email server *and* using POP3 as your transport, put the prefix "recent:" in front of your email address, as in 117 | 118 | > inaccount=recent:user@gmail.com 119 | 120 | The Gmail default is to retrieve email starting from the oldest, with a maximum of 250 emails. If you have a large inbox this will cause you to lose the most recent emails. The "recent:" prefix tells Gmail to retrieve the most recent 30 days of email. 121 | 122 | ***NOTE:*** It is recommended that you set up a separate account to receive your Duplicati emails instead of sending them to your regular personal or business account. This will allow you to manage Duplicati messages more easily and will not interfere with the read/unread status of emails in your regular account. This will also allow you to specify different authentication requirements on your dupReport email account (see the "authentication=" specification below.) 123 | 124 | ``` 125 | password=<password> 126 | ``` 127 | 128 | Password for incoming email system. **(IMAP, POP3, SMTP)** 129 | 130 | ``` 131 | authentication=basic 132 | ``` 133 | 134 | Specifies the type of authentication to be used with the server. Available options are 'basic' and 'oauth'. If you are using Oauth please check with your service provider to identify how to obtain the proper authentication keys. **(IMAP, POP3, SMTP)** 135 | 136 | ***NOTE***: Some email systems, including GMail, require the use of multifactor authentication to access user accounts. dupReport does not support multifactor authentication. If you receive a "invalid credentials" message from Gmail (and you are sure you have specified the correct ID/password), this is most likely the reason. For these systems, you will need to specify the use of a simple ID/password combination. In GMail, this is done by enabling the "Enable less secure apps" setting in the account security management settings. Check with your email service/provider to see how this is accomplished for other services. 137 | 138 | ``` 139 | folder=INBOX 140 | ``` 141 | 142 | IMAP folder on the incoming email server where Duplicati email is stored. **(IMAP)** 143 | 144 | ``` 145 | keepalive=false 146 | ``` 147 | 148 | Large inboxes may take a long time to scan and parse, and on some systems this can lead to a server connection timeout before processing has completed. This is more likely to happen on the outgoing connection (where there may be long periods of inactivity) than on the incoming connection. However, if you are experiencing timeout errors on your incoming or outgoing server connection, set this value to 'true'. **(IMAP, POP3, SMTP)** 149 | 150 | ``` 151 | unreadonly = false 152 | ``` 153 | 154 | This option instructs the program to only read and parse messages marked as "unread" or "unseen" on the incoming email server. Setting this option to "true" This has the effect of dramatically reducing the time it takes to read your emails, as it only reads messages it hasn't seen yet. **(IMAP)** 155 | 156 | There are several things to consider when using this option: 157 | 158 | - This option is only effective on IMAP email servers. It has no effect on POP3 servers. 159 | - If any other process or user marks any of the messages on the server as read/seen this will impact dupReport's ability to properly parse all the emails. You should only use this option if dupReport is the only process accessing the IMAP server & folder where the Duplicati emails are stored. 160 | - This option should be used in conjunction with the *markread=* option set to "true" so messages will be marked as read once they are processed by dupReport. 161 | - The seen/unseen flag can be flaky and exhibit different behaviors on different IMAP servers. You should test its usage thoroughly before using it in dupReport. 162 | 163 | ``` 164 | markread = false 165 | ``` 166 | 167 | When set to "true" dupReport will instruct the email server to mark all emails as read/seen once they have been processed by the program. Allowing dupReport to mark all messages as read/seen ("true") will speed up the program by only parsing through emails it has not already seen. 168 | 169 | The default is "false," instructing dupReport to leave the mailbox in the same state as it found it. Setting this option to "false" will slow down processing because dupReport must read all messages in the mailbox looking for messages of interest. However, if you have other programs that use the mailbox or you want to control the read/seen status of your email messages manually, set this option to "false". **(IMAP)** 170 | 171 | ``` 172 | sendername = dupReport Summary 173 | ``` 174 | 175 | Defines the "friendly name" for the email account used to send dupReport results. **(SMTP)** 176 | 177 | 178 | 179 | 180 | 181 | (Return to [Main Page](readme.md)) -------------------------------------------------------------------------------- /docs/RcFileConfig-Main.md: -------------------------------------------------------------------------------- 1 | ## dupReport.rc File - [main] section 2 | 3 | The [main] section contains the high-level program options for dupReport to run properly. 4 | 5 | ------ 6 | 7 | **Program Version** 8 | 9 | ``` 10 | version=<major>.<minor>.<subminor> 11 | ``` 12 | 13 | The current version number of the program. Used to determine if the .rc file needs to be updated. **DO NOT** alter the version= option 14 | 15 | ------ 16 | 17 | **Database File Path Information** 18 | 19 | ``` 20 | dbpath=<dbpath> 21 | ``` 22 | 23 | Directory or full path name where the dupReport.db file is located. Can be overridden by the -d command line option. 24 | 25 | ------ 26 | 27 | **Log Management** 28 | 29 | ``` 30 | logpath=<logpath> 31 | ``` 32 | 33 | The directory or full path name where the dupReport.log file is located. Can be overridden by the -l command line option. 34 | 35 | ``` 36 | verbose=<severity level> 37 | ``` 38 | 39 | The settings for the *[main]verbose=* option adhere to the severity levels as defined by the syslog RFC5424: 40 | 41 | | *Verbose=* Setting | Severity Level | 42 | | ------------------ | -------------- | 43 | | 0 | Emergency | 44 | | 1 | Alert | 45 | | 2 | Critical | 46 | | 3 | Error | 47 | | 4 | Warning | 48 | | 5 | Notice | 49 | | 6 | Info | 50 | | 7 | Debug | 51 | 52 | dupReport only uses only three of these severity levels during its operation: 53 | 54 | - 0 = No log output 55 | - 5 = General program execution info. 56 | - 7 = Full debugging output 57 | 58 | The *[main]verbose=* setting in the .rc file can be overridden by the -v command line option. 59 | 60 | ``` 61 | logappend=false 62 | ``` 63 | 64 | If set to true, append new logs to the existing log file. If set to false, reset log file for each run. Can be overridden by the -a command line option. 65 | 66 | ``` 67 | masksensitive = true 68 | ``` 69 | 70 | Masks sensitive data in the log file. If set to "true" (the default), fields such as user name, password, server name, and file paths will be masked with asterisks in the log file. This is useful for maintaining privacy if the log file needs to be stored or transmitted somewhere else for debugging or analysis purposes. **NOTE**: this setting **<u>will not</u>** mask any information found in the actual emails pulled from your email server, such as sending and receiving email address, server names, etc. It only affects the data that dupReport generates as part of its operation. 71 | 72 | ------ 73 | 74 | **Syslog Output** 75 | 76 | ``` 77 | syslog = <server> 78 | ``` 79 | 80 | This option sends the log output to a syslog server or a log aggregator. \<server> can be an IP address or a fully qualified domain name. 81 | 82 | You can specify a port on the syslog server by using the format: 83 | 84 | ``` 85 | syslog = <server>:<port> 86 | ``` 87 | 88 | If you do not specify a port, port 514 will be used by default. 89 | 90 | ``` 91 | sysloglevel=5 92 | ``` 93 | 94 | This specifies the logging level used for syslog output. These levels are the same used by the *[main]verbose=* setting described above. The default setting is 5. 95 | 96 | If you are sending the syslog output to a log aggregator or SIEM, the following Grok pattern can be used to properly parse the log output: 97 | 98 | ``` 99 | "<%{INT:facil_sev:int}>\[%{TIMESTAMP_ISO8601:ts}\]\[%{DATA:severity_label}\]\[%{DATA:function}\]\[%{DATA:action}\]%{GREEDYDATA:event_message}" 100 | ``` 101 | 102 | ------ 103 | 104 | **Email Message Management** 105 | 106 | ``` 107 | subjectregex=^Duplicati Backup report for 108 | ``` 109 | 110 | A regular expression used to find backup message Emails Of Interest. This should somewhat match the text specified in the ‘send-mail-subject’ advanced option in Duplicati. 111 | 112 | **NOTE**: If you modify the subject line of your Duplicati emails by changing the ‘send-mail-subject’ advanced option in Duplicati, make sure that the subject line you construct ***does not*** use the character you specify as the Source/Destination delimiter in the srcdestdelimiter= option in dupReport.rc (see below). If the subject line uses the same character as the Source/Destination delimiter, dupReport will get confused and not parse your emails properly. 113 | 114 | ``` 115 | srcregex=\w* 116 | ``` 117 | 118 | Regular expression used to find the “source” side of the source-destination pair in the subject line of the email. 119 | 120 | ``` 121 | destregex=\w* 122 | ``` 123 | 124 | Regular expression used to find the “destination” side of the source-destination pair in the subject line of the email. 125 | 126 | ``` 127 | srcdestdelimiter=- 128 | ``` 129 | 130 | Single character used to specify the delimiter between the ‘source’ and ‘destination’ parts of the source-destination pair. 131 | 132 | ``` 133 | dateformat=MM/DD/YYYY 134 | ``` 135 | 136 | Default format for dates found in emails. Acceptable formats are: 137 | 138 | - MM/DD/YYYY 139 | - DD/MM/YYYY 140 | - MM-DD-YYYY 141 | - DD-MM-YYYY 142 | - MM.DD.YYYY 143 | - DD.MM.YYYY 144 | - YYYY/MM/DD 145 | - YYYY/DD/MM 146 | - YYYY-MM-DD 147 | - YYYY-DD-MM 148 | - YYYY.MM.DD 149 | - YYYY.DD.MM 150 | 151 | If there are problems in your report dates or if you are getting program crashes around the date/time routines, you might try checking and/or changing this value. 152 | 153 | The default format can be overridden for specific backup jobs by specifying a dateformat= line in a job-specific section of the .rc file. See [“[source-destination] Sections”](RcFileConfig-SourceDestination.md). 154 | 155 | ``` 156 | timeformat=HH:MM:SS 157 | ``` 158 | 159 | Default format for times found in emails. The only time format currently acceptable is ‘HH:MM:SS’. 160 | 161 | ``` 162 | applyutcoffset=true 163 | ``` 164 | 165 | Duplicati version 2 does not apply local time zone information when reporting start and end times for backup jobs. If set to 'true', this option takes the UTC time zone offset from the email header and applies it to the backup job start and end times. If the backup times in your report do not look right, try adjusting this value. 166 | 167 | ``` 168 | emailservers = incoming, outgoing 169 | ``` 170 | 171 | Specifies the email servers that will be used to retrieve and send emails. See the section on [Email Management](RcFileConfig-IncomingOutgoing.md) for more information on this option. 172 | 173 | ------ 174 | 175 | **Special Runtime Options** 176 | 177 | ``` 178 | warnoncollect=false 179 | ``` 180 | 181 | If true, send an email if there are warning or error messages from the Duplicati jobs in the scanned emails. This option only works if the -c ("collect only") command line option is used and allows the user to get error messages even if they are not producing an email report. 182 | 183 | ``` 184 | show24hourtime=false 185 | ``` 186 | 187 | If set to "true," times will be displayed in 24-hour (military) notation. Otherwise, dupReport will use 12-hour (AM/PM) notation. 188 | 189 | ``` 190 | purgedb=true 191 | ``` 192 | 193 | If true, emails in the database that are no longer found on the incoming email server will be purged from the database and the database will be compacted at the end of the program run. 194 | 195 | **NOTE 1:** This option only purges email messages from the dupReport system. It does not affect the status of any messages on the incoming email server. 196 | 197 | **NOTE 2:** Any source-destination pairs referenced in purged emails will remain in the database in case emails for those pairs are seen in the future. To remove obsolete source-destination pair references from the database, use the -m command line option. 198 | 199 | ``` 200 | showprogress=0 201 | ``` 202 | 203 | If this option is greater than zero, dupReport will display a dot ('.') on the system console (stdout) for every 'n' emails that are processed from the incoming server. For example, if 204 | 205 | ``` 206 | showprogress=5 207 | ``` 208 | 209 | is specified, there will be one '.' for every 5 emails that are read. 210 | 211 | 212 | 213 | 214 | 215 | (Return to [Main Page](readme.md)) -------------------------------------------------------------------------------- /docs/RcFileConfig-SourceDestination.md: -------------------------------------------------------------------------------- 1 | ## dupReport.rc File - [source-destination] sections 2 | 3 | Specific options can also be set for each Source-Destination pair in the system. For example, to specify parameters specifically for a backup job named “Shenjhou-Discovery”, there should be a section in the .rc file with the following name: 4 | 5 | ``` 6 | [Shenjhou-Discovery] 7 | ``` 8 | 9 | **IMPORTANT:** The section name must match the Source-Destination pair name ***exactly***, including capitalization and delimiter characters. If there is any difference between the Source-Destination job name and the [source-destination] section name, the program will not be able to match the proper parameters to the correct backup job. 10 | 11 | Because [source-destination] sections are optional, they are not automatically added to the .rc file by the program. You must manually add them to the .rc file if they are needed. 12 | 13 | ------ 14 | 15 | **Date and Time Format Processing** 16 | 17 | ``` 18 | dateformat= 19 | timeformat= 20 | ``` 21 | 22 | These values override any *dateformat=* and *timeformat=* values specified in the [main] section. Specifying date and time formats on a per-job basis allows the program to properly parse and sort date and time results from backups that may be running in different locales and using different date/time display formats. 23 | 24 | The allowable values for these options are the same as the *dateformat=* and *timeformat=* options in the [[main] section](RcFileConfig-Main.md). If either of these options are not specified in a [source-destination] section the equivalent option from the [main] section will be used. 25 | 26 | **NOTE:** *dateformat=* and *timeformat=* in a [source-destination] section are only applied to the parsing of *incoming* emails. Dates and times produced in the final report are always formatted according to the default *dateformat=* and *timeformat=* options in the [main] section. 27 | 28 | ------ 29 | 30 | **Warnings for Missing Backups** 31 | 32 | ``` 33 | nobackupwarn = 3 34 | nbwsubject = BACKUP WARNING: #SOURCE##DELIMITER##DESTINATION# not being backed up! 35 | receiver = person@mailserver.com 36 | ``` 37 | 38 | These options specify the parameters for sending out warning emails if a particular backup job has not been seen for a period of time. See the descriptions for these options in the [[report] section](RcFileConfig-ReportSection.md) for allowable values for these options. If any of these options are not specified in a [source-destination] section the following fallback options will be used: 39 | 40 | | Option | If not found, will use | 41 | | -------------------------------- | ----------------------- | 42 | | [source-destination] nobackwarn= | [report] nobackupwarn= | 43 | | [source-destination] nbwsubject= | [report] nbwsubject= | 44 | | [source-destination] receiver= | [outgoing] outreceiver= | 45 | 46 | ------ 47 | 48 | **Ignore Offline Systems** 49 | 50 | ``` 51 | offline = True 52 | ``` 53 | 54 | This suppresses mention of the source-destination pair in the output report. This is useful when you know a system is offline for a while and you don't want to see the "not seen in X days" warning messages in the report. 55 | 56 | ------ 57 | 58 | **Defining Backup Intervals** 59 | 60 | ``` 61 | backupinterval = 1 62 | ``` 63 | 64 | This option can be used if a back up is scheduled to run less often than dupReport is run. For example, if your "Shenjhou-Discovery" backup runs every seven days but dupReport runs daily, for 6 days in the week you will get a warning notice that "Shenjhou-Discovery" wasn't seen. Setting this option to the number of days between backups for this particular job tells dupReport that it's OK if it doesn't see a backup from that job within that time period. 65 | 66 | If a backup from a source/destination pair is not seen while scanning the emails but the number of days since the last backup is less than the *backupinterval*= value, the program will simply print a notification message rather than the standard warning message. For example: 67 | 68 | ![](images\interval_example.jpg) 69 | 70 | The first line represents a backup that missed its daily execution. The second line represents a backup that only runs every 10 days. If no *backupinterval=* value is specified in a [source-destination] section, the default is 0. 71 | 72 | 73 | 74 | 75 | 76 | (Return to [Main Page](readme.md)) -------------------------------------------------------------------------------- /docs/RcFileConfig.md: -------------------------------------------------------------------------------- 1 | # dupReport.rc Configuration 2 | 3 | The dupReport.rc file (hereafter, the ".rc file") contains configuration information for dupReport to run properly. Many options in the dupReport.rc file have equivalent command line options. **If an option is specified on both the command line and the .rc file, the command line option takes precedence.** 4 | 5 | The .rc file contains several "sections", and each section has the form: 6 | 7 | ``` 8 | [section] 9 | option1 = value1 10 | option2 = value2 11 | option3 = value3 12 | ``` 13 | 14 | "[section]" is the name of the section and the "option" lines below the section name sets various options for that section. For a complete description of how .rc files are formatted, please see this [Wikipedia Article](https://en.wikipedia.org/wiki/INI_file). The important sections in the dupReport.rc file are: 15 | 16 | | Section | Purpose | 17 | | ------------------------------------------------------------ | ------------------------------------------------------------ | 18 | | [[main]](RcFileConfig-Main.md) | Sets the main program configuration | 19 | | \[\<Email Server>] | Sets the configuration for the POP3, IMAP, and SMTP servers. See the section on [Email Management](RcFileConfig-EmailManagement.md) for more information on the configuration of these sections. | 20 | | [[\<source-destination>]](RcFileConfig-SourceDestination.md) | Sets specific operating configuration options for the backup job named "\<source-destination>" | 21 | | [[apprise]](RcFileConfig-Apprise.md) | Set configuration options for the Apprise notification service | 22 | | [[report]](Reporting-ReportSection.md) | Sets the default configuration for the reporting system | 23 | | [[\<report_name>]](Reporting-CustomReportSpec.md) | Sets specific configuration options for user-defined custom reports | 24 | 25 | Click the section name in the table above to see more information on how to configure that section. 26 | 27 | 28 | 29 | 30 | 31 | (Return to [Main Page](readme.md)) -------------------------------------------------------------------------------- /docs/Reporting-CustomReportSpec.md: -------------------------------------------------------------------------------- 1 | # **Defining Custom Report Formats** 2 | 3 | dupreport allows you to specify your own custom report formats by creating special [report] sections in the .rc file. Custom report sections have the following general structure: 4 | 5 | ``` 6 | [rptname] 7 | type = report 8 | title = Title for the report 9 | groupby = <column1>:ascending 10 | groupheading = This is the Next Group 11 | columns = <column2>:Column 2, <column3>:Column 3 12 | columnsort = <column2>:ascending 13 | ``` 14 | 15 | ------ 16 | 17 | **Custom Report Options** 18 | 19 | ``` 20 | type = report 21 | ``` 22 | 23 | The *type =* option tells dupReport how to parse the information in the report specification. All custom reports in dupReport have the type "report". If you are defining a custom report, **make sure the type field is set to "report"**. dupReport has other report types that it uses for its own reports (e.g., 'noactivity' and 'lastseen'). 24 | 25 | ``` 26 | title = Title for the Report 27 | ``` 28 | 29 | This is the title that will be included at the top of the report. The above example will produce the following report: 30 | 31 | ![](D:\Users\parents\Documents\Development\dupReport\docs\images\TitleExample.jpg) 32 | 33 | ``` 34 | groupby = <column1>:<sortorder> 35 | ``` 36 | 37 | This tells dupReport how to group the data in the report. Reports can be grouped by any of the standard columns (See ["Specifying Report Columns"](Reporting-reportSection.md) for more information) 38 | 39 | \<sortorder> tells dupReport how to sort the groups. The only allowable values are: 40 | 41 | - **ascending** - sort in alphabetical order (A-Z) 42 | - **descending** - sore in reverse alphabetical order (Z-A) 43 | 44 | The example above use the specification: *groupby = destination:ascending* 45 | 46 | If you do not want to use groupings in your report you can omit the *groupby=* option in your custom report specification. 47 | 48 | ``` 49 | groupheading = This is the Next Group 50 | ``` 51 | 52 | This specifies the title that will be used for each group. The above example used the specification: *groupheading = Destination: #DESTINATION#* 53 | 54 | **Keyword Substitution**: You can supply keywords within the *groupheading =* option to customize the way it looks. Available keywords are: 55 | 56 | | Keyword | Meaning | 57 | | -------------- | ---------------------------------------------------------- | 58 | | \#SOURCE# | Inserts the appropriate source name in the subheading | 59 | | \#DESTINATION# | Inserts the appropriate destination name in the subheading | 60 | | \#DATE# | Inserts the appropriate date in the subheading | 61 | 62 | If you do not specify a *groupby=* option in your specification you do not need to specify a *groupheading=* option. 63 | 64 | ``` 65 | columns = column2:Column 2, column3:Column 3 66 | ``` 67 | 68 | This specifies the columns used in the report. Reports can use any of the standard columns (See ["Specifying Report Columns"](Reporting-reportSection.md) for more information) 69 | 70 | ``` 71 | columnsort = column2:ascending 72 | ``` 73 | 74 | This specifies the way information in the report is sorted. The format and options are similar to the *groupby =* option above. Multiple sorting columns can be specified, each separated by a comma (',') 75 | 76 | ------ 77 | 78 | **Overriding Standard Report Options** 79 | 80 | Any custom report specification can change the default option in the [report] section simply by including that option in the custom report section. For example, the [report] section defaults the background color of the title line to white: 81 | 82 | ``` 83 | [report] 84 | titlebg = #FFFFFF 85 | ``` 86 | 87 | If you want the title background to be green, you can specify that for just your custom report: 88 | 89 | ``` 90 | [myreport] 91 | titlebg = #00CC00 92 | ``` 93 | 94 | Most of the options found in the [report] section can be changed by including them as an option in the custom report section. 95 | 96 | 97 | 98 | (Return to [Main Page](readme.md)) -------------------------------------------------------------------------------- /docs/Reporting-ReportSection.md: -------------------------------------------------------------------------------- 1 | ## dupReport Reporting 2 | 3 | dupReport contains an advanced reporting engine that allows the user to define the data included in reports and arrange the layout of report sections. This is all handled by managing the configuration of .rc file options. No programming is required. 4 | 5 | ------ 6 | 7 | ## [report] section 8 | 9 | The [report] section contains the information on how reports are laid out and the default settings for reports. The following sections describe the various options that can be set in the [report] section. 10 | 11 | ------ 12 | 13 | **Defining Report Layout** 14 | 15 | ``` 16 | layout = <section> [, <section>, <section>, ...] 17 | ``` 18 | 19 | The *layout=* option defines the sections that will be present in the final report. There can be more than one section in the report, and each section must be separated by a comma. For example, to define a report with just one section ("bydest"), set the layout option as: 20 | 21 | > layout = bydest 22 | 23 | To define a report with three sections ("bydest", "noactivity", and "lastseen") define the layout option as: 24 | 25 | > layout = bydest, noactivity, lastseen 26 | 27 | The order these sections are specified in the layout option is the order they appear in the report. So, to have the "lastseen" section shown first, define the layout option as: 28 | 29 | > layout = lastseen, bydest, noactivity 30 | 31 | **Pre-defined Reports** 32 | 33 | dupReport comes with several pre-defined report formats: 34 | 35 | **srcdest**: This prints email results grouped by Source-Destination pairs. Here is an example of the srcdest report: 36 | 37 | ![](images/report_srcdest.jpg) 38 | 39 | 40 | 41 | **bysrc**: This prints email results grouped by Source systems. Here is an example of the bysrc report: 42 | 43 | ![](images/report_bysource.jpg) 44 | 45 | 46 | 47 | **bydest**: This prints email results grouped by Destination systems. Here is an example of the bydest report: 48 | 49 | ![](images/report_bydest.jpg) 50 | 51 | 52 | 53 | **bydate**: This prints email results grouped by date. Here is an example of the bydate report: 54 | 55 | ![](images/report_bydate.jpg) 56 | 57 | 58 | 59 | **noactivity**: This prints a report of all the Source-Destination pairs that were not seen during the run of the program. This can be helpful for seeing which systems may be down or otherwise not reporting in properly. Here is an example of the noactivity report: 60 | 61 | ![](images/report_noactivity.jpg) 62 | 63 | 64 | 65 | **lastseen**: This prints a list of all the current Source-Destination pairs and the date & time they were last seen by dupReport. Here is an example of the lastseen report: 66 | 67 | ![](images/last_seen_line.jpg) 68 | 69 | 70 | 71 | **offline**: This prints a list of all the backup sets that are listed as "offline" in the .rc file (i.e., *offline=true*). Here is an example of the offline report: 72 | 73 | 74 | 75 | ![](images/report_offline.jpg) 76 | 77 | 78 | 79 | Because the offline report only has one column (the Source-Destination field) you may wish to suppress the printing of the column title by using the `suppresscolumntitles = true` option in the .rc file section for that report. The resulting report will look like this: 80 | 81 | ![](images/report_offline_notitles.jpg) 82 | 83 | 84 | 85 | Any of these pre-defined reports can be used in the *layout=* option to customize the report as you wish. Here are some examples of sample report layouts: 86 | 87 | > layout = srcdest, noactivity, lastseen (Prints the srcdest, noactivity, and lastseen reports in that order) 88 | > 89 | > layout = noactivity, offline, bydest (Prints the noactivityand bydest reports in that order) 90 | > 91 | > layout = bydate (Prints only the bydate report) 92 | > 93 | > layout = noactivity (Prints only the noactivityreport) 94 | 95 | **IMPORTANT NOTE:** Each of the reports specified in the *layout* option must have a corresponding section in the .rc file. For example, to run the bydest report there must be a corresponding [bydest] section in the .rc file: 96 | 97 | ``` 98 | [bydest] 99 | title = Report by Destination 100 | . 101 | . 102 | . 103 | ``` 104 | 105 | dupReport comes with pre-defined report sections in the .rc file for each of the pre-defined reports (srcdest, bysrc, bydest, bydate, noactivity, and lastseen). See "Creating Custom Reports" for more information in definition report sections in the .rc file. 106 | 107 | You can also specify which reports you want to run on the dupReport command line using the -y option: 108 | 109 | ``` 110 | $ dupReport -y srcdest,noactivity 111 | ``` 112 | 113 | Specification for the reports to run are the same as in the *layout=* option in the .rc file. However, **there can not be any spaces between the report names and and commas** used on the command line. 114 | 115 | ------ 116 | 117 | **Specifying Report Columns** 118 | 119 | ``` 120 | columns = <colname>:<title> [, <colname>:<title>, ...] 121 | ``` 122 | 123 | The *columns =* option specifies the columns you want in the the report and the titles you want those columns to have. Each column is defined in the format: 124 | 125 | \<colname>:\<title> 126 | 127 | Multiple column specifications are included in the option, each separated by a comma (','). There must be one \<colname>:\<title> pair specified for each column you want in the report. 128 | 129 | The "\<colname>" specification **<u>must</u>** be one of the following: 130 | 131 | | \<colname> Value | Meaning | 132 | | ------------------- | ------------------------------------------------------------ | 133 | | source | Source system for the backup job | 134 | | destination | Destination system for the backup job | 135 | | date | Date the backup job was run | 136 | | time | Time the backup job was run | 137 | | duration | The duration of the backup job (days/hours/minutes/seconds). This column can be modified to remove units that equal zero (0) by setting *durationzeroes=false* in the [report] section. | 138 | | dupversion | Duplicati version used for the backup job | 139 | | examinedFiles | Number of files examined during the backup job | 140 | | examinedFilesDelta | The increase (+) or decrease (-) in the number of files examined since the previous backup job | 141 | | sizeOfExaminedFiles | Total size of the files examined during the backup job | 142 | | fileSizeDelta | The increase (+) or decrease (-) in the total size of files examined since the previous backup job | 143 | | addedFiles | Number of files added to the backup set for this backup job | 144 | | deletedFiles | Number of files deleted from the backup set for this backup job | 145 | | modifiedFiles | Number of files modified in the backup set for this backup job | 146 | | filesWithError | Number of files that had errors during the backup job | 147 | | bytesUploaded | Number of bytes uploaded to the backup server during the backup. This field is only available if the backup job email is in JSON format. | 148 | | bytesDownloaded | Number of bytes downloaded from the backup server during the backup. This field is only available if the backup job email is in JSON format. | 149 | | parsedResult | The final result of the backup job as reported by Duplicati (e.g., Success, Failure, etc) | 150 | | messages | Messages generated by the backup job during its run. This column can also be suppressed by setting *displaymessages=false* in the [report] section. | 151 | | warnings | Warning messages generated by the backup job during its run. This column can also be suppressed by setting *displaywarnings=false* in the [report] section | 152 | | errors | Error messages generated by the backup job during its run. This column can also be suppressed by setting *displayerrors=false* in the [report] section | 153 | | logdata | Log data messages generated by the backup job during its run. This column can also be suppressed by setting *displaylogdata=false* in the [report] section | 154 | 155 | The "\<title>" specification can be anything you want to have printed as a column heading for that value. 156 | 157 | Here is an example that specifies the source, destination, date, time, and duration columns: 158 | 159 | ``` 160 | columns = source:Source, destination:Destination, date:Run Date, time:Run Time, duration:Run Duration 161 | ``` 162 | 163 | **NOTE:** The *column =* option **must all be on a single line** in the .rc file. The example line above may have wrapped due to screen size restrictions. 164 | 165 | The above specification will produce a report that looks like the following: 166 | 167 | ![](images/SampleColumnSpec.jpg) 168 | 169 | 170 | 171 | The *columns=* option in the [report] section of the .rc file is the default column set used for all reports and contains all the possible columns you can specify in a report. If you wish to create a customized report, copy this line to a new [source-destination] section of the .rc file and remove any \<colname>:\<title> pairs you do not want in the new report. 172 | 173 | ------ 174 | 175 | **Customizing the Report Appearance** 176 | 177 | ``` 178 | title=Duplicati Backup Summary Report 179 | ``` 180 | 181 | This defines the subject line for the report email that gets produced and also serves as the default title for all report sections. If you want each section to have its own title, you can (and probably should) place a *title=* option in that section to override this default. 182 | 183 | ***Title Keyword Substitution*** 184 | 185 | You use the email title to indicate whether any of the jobs in the report ended with a Success, Warning, Error, or Failure status. To do this, include any of the following keywords in the 'title=' specification: 186 | 187 | | Keyword | Meaning | 188 | | ---------- | ------------------------------------------------------------ | 189 | | #SUCCESS# | Adds '\|Success\|' to the email subject line if any of the backup jobs ended with a 'Success' Status | 190 | | #WARNING# | Adds '\|Warning\|' to the email subject line if any of the backup jobs ended with a 'Warning' Status | 191 | | #ERROR# | Adds '\|Error\|' to the email subject line if any of the backup jobs ended with a 'Error' Status | 192 | | #FAILURE# | Adds '\|Failure\|' to the email subject line if any of the backup jobs ended with a 'Failure' Status | 193 | | #ALL# | Equivalent to #SUCCESS##WARNING##ERROR##FAILURE# | 194 | | #ANYERROR# | Equivalent to #WARNING##ERROR##FAILURE# | 195 | 196 | You can use any or all of the keywords anywhere in your subject line. For example, 197 | 198 | ``` 199 | title = Duplicati Backup Summary Report #WARNING##ERROR# 200 | ``` 201 | 202 | Will produce the following subject line if any of the backup jobs ended with a Warning or Error status: 203 | 204 | 'Duplicati Backup Summary Report |Warning||Error|' 205 | 206 | The substitution will only occur if any of the jobs actually ended with the indicated status. For example, using the above specification, if some of the jobs ended with an Error stats but none ended with a Warning status, the following subject line would be produced: 207 | 208 | 'Duplicati Backup Summary Report |Error|' 209 | 210 | ***Important note***: placing these keywords at the beginning of the subject line may lead to unpredictable results on some email servers and clients. The reason for this is unknown. If you are having problems with the keywords at the beginning of your subject line, try moving them to the end of the line. 211 | 212 | ``` 213 | border=1 214 | ``` 215 | 216 | Specifies the size (in pixels) of the borders in the report table. This option only works in the HTML layout. 217 | 218 | ``` 219 | padding=5 220 | ``` 221 | 222 | Specifies the size of cell padding in the report table. This option only works in the HTML layout. 223 | 224 | ``` 225 | sizedisplay = none 226 | ``` 227 | 228 | This tells dupReport to display any file size information as bytes (sizedisplay = none), megabytes (sizedisplay = mb), or gigabytes (sizedisplay = gb) This option can be overridden by using the the -s option on the command line. 229 | 230 | ``` 231 | repeatcolumntitles = true 232 | ``` 233 | 234 | Indicates whether to repeat the column title headers for each report section (*repeatcolumntitles = true*) or only at the beginning of the report (*repeatcolumntitles= false*). The default is 'false' 235 | 236 | ``` 237 | suppresscolumntitles = false 238 | ``` 239 | 240 | If *suppresscolumntitles = true*, no column titles will be printed in the report. If *suppresscolumntitles = false* (the default) column titles will be printed in the report. 241 | 242 | ``` 243 | durationzeroes = true 244 | ``` 245 | 246 | This modifies the display of the backup job "Duration" column in the report. If *durationzeroes = true* , the time units in the job duration field will be displayed with leading zeroes, for example "0d 13h 0m 32s." If *durationzeroes = false* , any unit that equals zero (0) will not be displayed, so the previous example will be displayed as "13h 32s." 247 | 248 | ``` 249 | includeruntime = true 250 | ``` 251 | 252 | Setting this to "true" places an indication of the report's running time at the bottom of the report. The running time will look like this: 253 | 254 | ![](images/runtime_line.jpg) 255 | 256 | ------ 257 | 258 | **Specifying How to Display Backup Job Messages** 259 | 260 | Duplicati emails may include a variety of output messages in addition to the status fields. These include: 261 | 262 | - Error messages 263 | - Warning messages 264 | - General notification messages 265 | - Log data 266 | 267 | dupReport allows you to specify how you would like these messages displayed in the report. 268 | 269 | ``` 270 | displaymessages = true (Turns general messages on or off) 271 | displaywarnings = true (Turns warning messages on or off) 272 | displayerrors = true (Turns error messages on or off) 273 | displaylogdata = true (Turns logging messages on or off) 274 | ``` 275 | 276 | If any of these options are set to "false" those messages will not be shown in the report. 277 | 278 | ``` 279 | truncatemessage = 0 280 | truncatewarning = 0 281 | truncateerror = 0 282 | truncatelogdata = 0 283 | ``` 284 | 285 | These settings truncate the message, warning, error, and log data fields generated during backup job execution. Duplicati job messages can be quite lengthy and take up a lot of room in the report. These options allow you to truncate those messages to a reasonable length. A length of 0 (zero) indicates that the message should not be truncated. If the length of the message/warning/error is less than the size indicated, the entire message/warning/error will be displayed. If a message gets truncated and you'd like to view the original (full) message string, refer to the email generated for that backup job. 286 | 287 | ``` 288 | weminline = false 289 | ``` 290 | 291 | During normal operation, dupReport prints warning, error, message, and log data messages on separate lines underneath the job they are associated with to conserve horizontal space in the report (these messages can be quite lengthy!) If you want to see the messages on the same line as the backup job data, set *weminline = true*. 292 | 293 | ``` 294 | failedonly = false 295 | ``` 296 | 297 | If this is set to "true" it instructs dupReport to only print backup jobs that has a "warning" or "failed" result status. The default is "false." 298 | 299 | ``` 300 | showoffline = false 301 | ``` 302 | 303 | With this option set to "true," any backup job with it's own [source-destination] section and the *offline=true* option set within that section will be displayed in the 'noactivity' and 'lastseen' reports with an '[OFFLINE]' indicator. If this option is set to 'false,' those backup jobs will be excluded from those specific reports. The default is 'false'. 304 | 305 | ------ 306 | 307 | **Sending Email if an individual Backup Has Not Been Seen** 308 | 309 | dupReport has the ability to send email to you (or anyone) if a scheduled backup has not been seen. The following options affect how that facility works. 310 | 311 | ``` 312 | nobackupwarn = 5 313 | ``` 314 | 315 | Sets the threshold for the number of days to go without a backup from a source-destination pair before sending a separate email warning. If nobackupwarn =0, no email notices will be sent. The warning email will be sent to the email address specified by the *[outgoing] outreceiver=* option unless specifically overridden by including a *receiver=* option in a [source-destination] section. See Source-Destination Sections for more details. 316 | 317 | ``` 318 | nbwsubject = Backup Warning: #SOURCE##DELIMITER##DESTINATION# Backup Not Seen for #DAYS# Days 319 | ``` 320 | 321 | If the threshold (in days) defined by the *nobackupwarn =* option is reached, the string specified by the *nbwsubject =* option will be used as the subject of the warning email. This can be overridden on a per backupset basis by adding a *nbwsubject =* option in a [source-destination] section. 322 | 323 | **Keyword Substitution**: You can supply keywords within the nbwsubject option to customize the way it looks. Available keywords are: 324 | 325 | | Keyword | Translated to: | 326 | | -------------- | ------------------------------------------------------------ | 327 | | \#SOURCE# | The source in a source-destination pair | 328 | | \#DESTINATION# | The destination in a source-destination pair | 329 | | \#DELIMITER# | The delimiter used in a source-destination pair (as specified in [main] srcdestdelimiter) | 330 | | \#DAYS# | The number of days since the last backup | 331 | | \#DATE# | The date of the last backup | 332 | | \#TIME# | The time of the last backup | 333 | 334 | ------ 335 | 336 | **Report Colors (HTML Format Only)** 337 | 338 | When displaying the report in HTML format, there are several options that specify what colors will be used in the report. This table summarizes those options. 339 | 340 | | [report] Option | Description | 341 | | ------------------------ | ------------------------------------------------- | 342 | | titlebg = #FFFFFF | Background color for the report title. | 343 | | jobmessagebg = #FFFFFF | Background color for backup job messages | 344 | | jobwarningbg = #FFFF00 | Background color for backup job warnings | 345 | | joberrorbg = #FF0000 | Background color for backup job errors | 346 | | joblogdatabg = #FF0000 | Background color for backup job log messages | 347 | | groupheadingbg = #D3D3D3 | Background color for group headings in the report | 348 | 349 | **NOTE ON COLOR SELECTION:** All color specifications in the [report] section follow standard HTML color codes. For a sample list of colors and their HTML codes, please refer to https://www.w3schools.com/colors/colors_names.asp 350 | 351 | ------ 352 | 353 | **Actions on Past Due Backups** 354 | 355 | If a backup hasn't been seen in a number of days, colors can be used to indicate how late the backup is. For example, on-time backups can be displayed with no background, backups not seen in 5 days can be shown with a yellow background, and backups not seen in 10 days can be shown with a red background. This gives a quick visual representation on which backups may need attention. 356 | 357 | Time periods are categorized in dupReport as "normal", "warning", and "error". Using the above example, the "normal" time period for a backup would be from 0-5 days, the "warning" time period would be 6-10 days, and the error time period would be 11 days or more. 358 | 359 | To specify these time periods, use the following options in the [report] section: 360 | 361 | | [report] option | Description | Default Value | 362 | | --------------- | ------------------------------------------------------------ | ---------------- | 363 | | normaldays = | The number of days where a missed backup would be normal | 5 | 364 | | normalbg = | The background color for displaying "normal" backups | #FFFF00 (white) | 365 | | warningdays = | The number of days where a missed backup would be cause for concern | 10 | 366 | | warningbg = | The background color for displaying "warning" backups | #FF4500 (yellow) | 367 | | errorbg = | The background color for displaying "warning" backups | #FF0000 (red) | 368 | 369 | 370 | The following table clarifies how the time frames and colors are used. 371 | 372 | | Comparison | Background Color Display Option | 373 | | :------------------------------------------ | :------------------------------ | 374 | | \# days <= 'normaldays' | normalbg (Defaut: white) | 375 | | \# days > 'normaldays' and <= 'warningdays' | warningbg (Default: Yellow) | 376 | | \# days > 'warningdays' | errorbg (Default: red) | 377 | 378 | NOTE: This formula and color definition is used for all similar time-based calculations in all reports. 379 | 380 | ------ 381 | 382 | ## Special Report Sections 383 | 384 | There are some pre-defined sections in the dupReport.rc file that are used by built-in reports users can run. 385 | 386 | | Section | Purpose | 387 | | ------------ | ------------------------------------------------------------ | 388 | | [noactivity] | Specifies layout and format options for reporting on which backup jobs were not seen during the program's last run. | 389 | | [lastseen] | Specifies layout and format options for reporting on when all backup jobs were last seen during the program's last run. | 390 | | [offline] | Specifies layout and format options for reporting on which backup jobs are listed as 'offline' on the .rc file. | 391 | 392 | These reports can be added to the *layout=* option in the [report] section: 393 | 394 | ``` 395 | [report] 396 | layout = bydest,noactivity,lastseen 397 | ``` 398 | 399 | They can also be specified as part of the -y option on the command line: 400 | 401 | ``` 402 | $ dupReport.py -y bydest,noactivity,lastseen 403 | ``` 404 | 405 | 406 | 407 | (Return to [Main Page](readme.md)) -------------------------------------------------------------------------------- /docs/SystemRequirements.md: -------------------------------------------------------------------------------- 1 | # System Requirements 2 | 3 | dupReport has been formally tested on Linux (Debian 8 and 9) and Windows 10 and is officially supported on those platforms. However, users posting on the [dupReport announcement page on the Duplicati Forum](https://forum.duplicati.com/t/announcing-dupreport-a-duplicati-email-report-summary-generator/1116) have stated they’ve installed and run the program on a wide variety of operating systems. The python code uses standard Python 3 libraries, so it *should* run on any platform where Python 3 can be run. 4 | 5 | In addition to the dupReport program files, the only other software dupReport needs is Python3. Installation instructions for Python are beyond our scope here, but instructions are easily searchable on the Internet. 6 | 7 | 8 | 9 | 10 | 11 | (Return to [Main Page](readme.md)) -------------------------------------------------------------------------------- /docs/WhatIsDupreport.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # What is dupReport? 4 | 5 | dupReport is a program that will collect up status emails from Duplicati and combine them into a single email report that is sent to you. The following diagram helps explain how it works: 6 | 7 | ![dupReport Architecture](images/dR_Architecture.jpg)\ 8 | 9 | Some general points about dupReport 10 | 11 | - The program is designed for those who run backups from multiple locations and are getting multiple Duplicati emails per day. For example, we have 14 separate emails coming in per day from Duplicati backup jobs; way too many to keep track of manually. In these cases, dupReport will collect and collate all those emails and create a single report that summarizes all of them. If you only have a single instance of Duplicati running a single backup job, you may get some value from dupReport, but it may be overkill. 12 | - dupReport doesn’t interface directly with Duplicati and doesn't read the Duplicati configuration files, log files, or database. It connects to your email server, reads the backup report emails that Duplicati sends out, then parses them to create its report. It does not need to be run on the same system where Duplicati is running, it can be on a completely different system if that is more convenient for you. The only requirement is that the system where dupReport is running must be able to connect to your email server. 13 | 14 | (Return to [Main Page](readme.md)) 15 | 16 | -------------------------------------------------------------------------------- /docs/images/SampleColumnSpec.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HandyGuySoftware/dupReport/7b95411e1364633c22b321f0d04c542b32726917/docs/images/SampleColumnSpec.jpg -------------------------------------------------------------------------------- /docs/images/TitleExample.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HandyGuySoftware/dupReport/7b95411e1364633c22b321f0d04c542b32726917/docs/images/TitleExample.jpg -------------------------------------------------------------------------------- /docs/images/dR_Architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HandyGuySoftware/dupReport/7b95411e1364633c22b321f0d04c542b32726917/docs/images/dR_Architecture.jpg -------------------------------------------------------------------------------- /docs/images/interval_example.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HandyGuySoftware/dupReport/7b95411e1364633c22b321f0d04c542b32726917/docs/images/interval_example.jpg -------------------------------------------------------------------------------- /docs/images/last_date_table.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HandyGuySoftware/dupReport/7b95411e1364633c22b321f0d04c542b32726917/docs/images/last_date_table.jpg -------------------------------------------------------------------------------- /docs/images/last_seen_line.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HandyGuySoftware/dupReport/7b95411e1364633c22b321f0d04c542b32726917/docs/images/last_seen_line.jpg -------------------------------------------------------------------------------- /docs/images/report_bydate.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HandyGuySoftware/dupReport/7b95411e1364633c22b321f0d04c542b32726917/docs/images/report_bydate.jpg -------------------------------------------------------------------------------- /docs/images/report_bydest.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HandyGuySoftware/dupReport/7b95411e1364633c22b321f0d04c542b32726917/docs/images/report_bydest.jpg -------------------------------------------------------------------------------- /docs/images/report_bysource.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HandyGuySoftware/dupReport/7b95411e1364633c22b321f0d04c542b32726917/docs/images/report_bysource.jpg -------------------------------------------------------------------------------- /docs/images/report_noactivity.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HandyGuySoftware/dupReport/7b95411e1364633c22b321f0d04c542b32726917/docs/images/report_noactivity.jpg -------------------------------------------------------------------------------- /docs/images/report_offline.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HandyGuySoftware/dupReport/7b95411e1364633c22b321f0d04c542b32726917/docs/images/report_offline.jpg -------------------------------------------------------------------------------- /docs/images/report_offline_notitles.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HandyGuySoftware/dupReport/7b95411e1364633c22b321f0d04c542b32726917/docs/images/report_offline_notitles.jpg -------------------------------------------------------------------------------- /docs/images/report_srcdest.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HandyGuySoftware/dupReport/7b95411e1364633c22b321f0d04c542b32726917/docs/images/report_srcdest.jpg -------------------------------------------------------------------------------- /docs/images/runtime_line.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HandyGuySoftware/dupReport/7b95411e1364633c22b321f0d04c542b32726917/docs/images/runtime_line.jpg -------------------------------------------------------------------------------- /docs/readme.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # WELCOME TO dupReport 4 | 5 | dupReport is a Python-based email collection and reporting system for Duplicati. It will gather all your Duplicati backup status emails and produce a summary report on what Duplicati backup jobs were run and their success or failure. 6 | 7 | Here is a list of some of dupReport's most important features: 8 | 9 | - Collects all your Duplicati result emails and produces easy-to-understand status reports 10 | - Runs on multiple operating systems. dupReport has been tested on Linux (Debian 8 & 9) and Windows 10, but users have reported it working on a wide variety of operating systems 11 | - A Guided Setup for new users. If there is no configuration (.rc) file when the program runs, the Guided Setup will take the user through the most common configurable options. 12 | - Support for IMAP and POP3 email services (IMAP is recommended for better results) 13 | - Support for using multiple inbound (IMAP/POP3) and outbound (SMTP) email servers. 14 | - Supports both text and JSON formatted status emails from Duplicati 15 | - Supports SSL/TLS support for incoming/outgoing email transmissions. 16 | - Output report in HTML, Text, CSV, and JSON formats 17 | - Send results to email or local files (or both) 18 | - User-defined reporting formats with configurable column and organization options 19 | - Syslog-style logging format for easier searching and organization. 20 | 21 | - Ability to send log output to an external syslog server or log aggregator. 22 | - Support for the Apprise push notification service (<https://github.com/caronc/apprise>) 23 | 24 | Please see the (new, updated, and reorganized) documentation on these and all the dupReport features. 25 | 26 | ------ 27 | 28 | # Available Code Branches 29 | 30 | Beginning with release 2.1, the branch structure of the dupReport repository has changed. We have moved to a more organized structure based on [this article by Vincent Driessen](http://nvie.com/posts/a-successful-git-branching-model/) (with some modifications). (Thanks to @DocFraggle for suggesting this structure.) 31 | 32 | There are usually only two branches in the dupReport repository: 33 | 34 | | Branch Name | Current Version | Purpose | 35 | | ------------ | --------------- | ------------------------------------------------------------ | 36 | | **main** | 3.0.10 | This is the Release branch, which should contain <u>completely stable</u> code. If you want the latest and greatest release version, get it here. If you are looking for an earlier release, tags in this branch with the name "Release_x.x.x" will point you there. | 37 | | **pre_prod** | \<None\> | The Pre-Production branch. This is a late-stage beta branch where code should be mostly-stable, but no guarantees. Once final testing of code in this branch is complete it will be moved to master and released to the world. If you want to get a peek at what's coming up in the next release, get the code from here. **If you don't see a pre_prod branch in the repository, that means there isn't any beta code available for testing.** | 38 | 39 | If you see any additional branches in the repository, they are there for early-stage development or bug fix testing purposes. Code in such branches should be considered **<u>highly unstable</u>**. Swim here at your own risk. Void where prohibited. Batteries not included. Freshest if eaten before date on carton. For official use only. Use only in a well-ventilated area. Keep away from fire or flames. May contain peanuts. Keep away from pets and small children. 40 | 41 | Bug reports and feature requests can be made on GitHub in the [dupReport Issues Section](https://github.com/HandyGuySoftware/dupReport/issuesdupReport). <u>Please do not issue pull requests</u> before discussing any problems or suggestions as an Issue. 42 | 43 | The discussion group for dupReport is on the Duplicati Forum in [this thread](https://forum.duplicati.com/t/announcing-dupreport-a-duplicati-email-report-summary-generator/1116). 44 | 45 | The program is released under an MIT license. Please see the LICENSE file for more details. 46 | 47 | Enjoy! 48 | 49 | 50 | 51 | ------ 52 | 53 | # Documentation 54 | 55 | [What is dupReport?](WhatIsDupreport.md) 56 | 57 | [System Requirements](SystemRequirements.md) 58 | 59 | [Getting Started (Quickly)](QuickStart.md) 60 | 61 | [Understanding Source-Destination Pairs and Identifying Emails of Interest](Config-SrcDestPairs.md) 62 | 63 | Running dupReport 64 | 65 | - [Installation](Installation.md) 66 | 67 | - [Command Line Options](CommandLine.md) 68 | 69 | [dupReport.rc File Configuration](RcFileConfig.md) 70 | 71 | - Program Management: The [[main] Section](RcFileConfig-Main.md) 72 | - Handling Specific Backup Jobs: The [[\<source-destination>] Sections](RcFileConfig-SourceDestination.md) 73 | - [Email Management](RcFileConfig-EmailManagement.md) 74 | - Push Notifications Using Apprise: The [[apprise] Section](RcFileConfig-Apprise.md) 75 | 76 | Reporting 77 | 78 | - [The [report] Section](Reporting-ReportSection.md) 79 | - [Custom Report Formats](Reporting-CustomReportSpec.md) 80 | 81 | 82 | 83 | -------------------------------------------------------------------------------- /drdatetime.py: -------------------------------------------------------------------------------- 1 | ##### 2 | # 3 | # Module name: drdatetime.py 4 | # Purpose: Miscellaneous date and time management functions for dupReport 5 | # 6 | # Notes: 7 | # 8 | ##### 9 | 10 | # Import system modules 11 | import re 12 | import datetime 13 | 14 | # Import dupReport modules 15 | import globs 16 | 17 | # dtFmtDefs - definitions for date field formats 18 | # Tuples are defined as follows: 19 | # Field Purpose 20 | # 0 separator character 21 | # 1, 2, 3 Positions in format string for (year, month, day) or (hour, minute, seconds) 22 | # 23 | # Note: There's only one recognized time string format. But with all the 24 | # problems I had with date string recoznition, this makes time strings 25 | # more flexible should the need arise in the future. 26 | dtFmtDefs={ 27 | # Format Str [0]Delimiter [1]Y/H Col [2]M/Mn Col [3]D/S Col 28 | 'YYYY/MM/DD': ('/', 0, 1, 2), 29 | 'YYYY/DD/MM': ('/', 0, 2, 1), 30 | 'MM/DD/YYYY': ('/', 2, 0, 1), 31 | 'DD/MM/YYYY': ('/', 2, 1, 0), 32 | 'YYYY-MM-DD': ('-', 0, 1, 2), 33 | 'YYYY-DD-MM': ('-', 0, 2, 1), 34 | 'MM-DD-YYYY': ('-', 2, 0, 1), 35 | 'DD-MM-YYYY': ('-', 2, 1, 0), 36 | 'YYYY.MM.DD': ('.', 0, 1, 2), 37 | 'YYYY.DD.MM': ('.', 0, 2, 1), 38 | 'MM.DD.YYYY': ('.', 2, 0, 1), 39 | 'DD.MM.YYYY': ('.', 2, 1, 0), 40 | 'HH:MM:SS' : (':', 0, 1, 2) 41 | } 42 | 43 | # Issue #83. Changed regex for the date formats to allow any standard delimiter ('/', '-', or '.') 44 | # The program (via toTimestamp()) will use this regex to extract the date from the parsed emails 45 | # If the structure is correct (e.g., 'MM/DD/YYYY') but the delimiters are wrong (e.g., '04-30-2018') the program will still be able to parse it. 46 | # As a result, all the regex's for dtFmtDefs date fields are all the same now. (Change from previous versions) 47 | dateParseRegex = r'(\s)*(\d)+[/\-\.](\s)*(\d)+[/\-\.](\s)*(\d)+' # i.e., <numbers>[/-.]<numbers>[/-.]<numbers> 48 | timeParseRegex = r'(\d)+[:](\d+)[:](\d+)' # i.e., <numbers>:<numbers>:<numbers> 49 | validDateDelims = r'[/\-\.]' # Valid delimiters in a date string 50 | validTimeDelims = ':' # Valid delimiters in a time string 51 | 52 | # Print error messages to the log and stderr if there is a date or time format problem. 53 | # It happens more often than you'd think! 54 | def timeStampCrash(msg): 55 | globs.log.write(globs.SEV_NOTICE, function='DateTime', action='timeStampCrash', msg=msg) 56 | globs.log.write(globs.SEV_NOTICE, function='DateTime', action='timeStampCrash', msg='This is likely caused by an email using a different date or time format than expected,\nparticularly if you\'re collecting emails from multiple locations or time zones.') 57 | globs.log.write(globs.SEV_NOTICE, function='DateTime', action='timeStampCrash', msg='Please check the \'dateformat=\' and \'timeformat=\' value(s) in the [main] section\nand any [<source>-<destination>] sections of your .rc file for accuracy.') 58 | globs.log.err('Date/time format specification mismatch. See log file for details. Exiting program.') 59 | globs.closeEverythingAndExit(1) 60 | 61 | # Convert a date/time string to a timestamp 62 | # Input string = YYYY/MM/DD HH:MM:SS AM/PM (epochDate)." 63 | # May also be variants of the above. Must check for all cases 64 | # dtStr = date/time string 65 | # dfmt is date format - defaults to user-defined date format in .rc file 66 | # tfmt is time format - - defaults to user-defined time format in .rc file 67 | # utcOffset is UTC offset info as extracted from the incoming email message header 68 | def toTimestamp(dtStr, dfmt = None, tfmt = None, utcOffset = None): 69 | globs.log.write(globs.SEV_DEBUG, function='DateTime', action='toTimestamp', msg='Converting \'{}\' (dfmt=\'{}\', tfmt=\'{}\' offset=\'{}\')'.format(dtStr, dfmt, tfmt, utcOffset)) 70 | 71 | # Set default formats 72 | if (dfmt is None): 73 | dfmt = globs.opts['dateformat'] 74 | if (tfmt is None): 75 | tfmt = globs.opts['timeformat'] 76 | 77 | # Find proper date spec 78 | # Get column positions 79 | yrCol = dtFmtDefs[dfmt][1] # Which field holds the year? 80 | moCol = dtFmtDefs[dfmt][2] # Which field holds the month? 81 | dyCol = dtFmtDefs[dfmt][3] # Which field holds the day? 82 | 83 | # Extract the date 84 | dtPat = re.compile(dateParseRegex) # Compile regex for date/time pattern 85 | dateMatch = re.match(dtPat,dtStr) # Match regex against date/time 86 | if dateMatch: 87 | dateStr = dtStr[dateMatch.regs[0][0]:dateMatch.regs[0][1]] # Extract the date string 88 | else: 89 | timeStampCrash('Can\'t find a match for date pattern {} in date/time string {}.'.format(dfmt, dtStr)) # Write error message, close program 90 | 91 | datePart = re.split(validDateDelims, dateStr) # Split date string based on any valid delimeter 92 | year = int(datePart[yrCol]) 93 | month = int(datePart[moCol]) 94 | day = int(datePart[dyCol]) 95 | 96 | # Get column positions 97 | hrCol = dtFmtDefs[tfmt][1] # Which field holds the Hour? 98 | mnCol = dtFmtDefs[tfmt][2] # Which field holds the minute? 99 | seCol = dtFmtDefs[tfmt][3] # Which field holds the seconds? 100 | 101 | # Extract the time 102 | tmPat = re.compile(timeParseRegex) 103 | timeMatch = re.search(tmPat,dtStr) 104 | if timeMatch: 105 | timeStr = dtStr[timeMatch.regs[0][0]:timeMatch.regs[0][1]] 106 | else: 107 | timeStr = '00:00:00' 108 | globs.log.write(globs.SEV_DEBUG, function='DateTime', action='toTimestamp', msg='No time portion provided. Defaulting to \'00:00:00\'') 109 | timePart = re.split(validTimeDelims, timeStr) 110 | hour = int(timePart[hrCol]) 111 | minute = int(timePart[mnCol]) 112 | second = int(timePart[seCol]) 113 | 114 | # See if we need AM/PM adjustment 115 | pmPat = re.compile(r'AM|PM') 116 | pmMatch = re.search(pmPat,dtStr) 117 | if pmMatch: 118 | dayPart = dtStr[pmMatch.regs[0][0]:pmMatch.regs[0][1]] 119 | if ((hour == 12) and (dayPart == 'AM')): 120 | hour = 0 121 | elif ((hour != 12) and (dayPart == 'PM')): 122 | hour += 12 123 | 124 | # Convert to datetime object, then get timestamp 125 | try: 126 | ts = datetime.datetime(year, month, day, hour, minute, second).timestamp() 127 | except ValueError as err: 128 | globs.log.write(globs.SEV_ERROR, function='DateTime', action='toTimestamp', msg='Error: {}'.format(err.args[0])) 129 | timeStampCrash('Error creating timestamp: DateString={} DateFormat={} year={} month={} day={} hour={} minute={} second={}'.format(dtStr, dfmt, year, month, day, hour, minute, second)) # Write error message, close program 130 | 131 | # Apply email's UTC offset to date/time 132 | # Need to separate the two 'if' statements because the init routines crash otherwise 133 | # (Referencing globs.opts[] before they're set) 134 | if utcOffset is not None: 135 | if globs.opts['applyutcoffset']: 136 | ts += float(utcOffset) 137 | 138 | globs.log.write(globs.SEV_DEBUG, function='DateTime', action='toTimestamp', msg='Date/Time {} converted to {}'.format(dtStr, ts)) 139 | return ts 140 | 141 | # Convert an RFC 3339 format datetime string to an epoch-style timestamp 142 | # Needed because the JSON output format uses this style for datetime notation 143 | # Basically, decode the RFC3339 string elements into separate date & time strings, then send to toTimeStamp() as a normal date/time string. 144 | def toTimestampRfc3339(tsString, utcOffset = None): 145 | globs.log.write(globs.SEV_DEBUG, function='DateTime', action='toTimestampRfc3339', msg='Converting {}, offset={}'.format(tsString, utcOffset)) 146 | 147 | # Strip trailing 'Z' and last digit from milliseconds, the float number is too big to convert 148 | tsStringNew = tsString[:-2] 149 | 150 | # Convert to datetime object 151 | dt = datetime.datetime.strptime(tsStringNew, '%Y-%m-%dT%H:%M:%S.%f') 152 | 153 | # Now, use existing methods to convert to a timestamp 154 | ts = toTimestamp("{}/{}/{} {}:{}:{}".format(dt.month, dt.day, dt.year, dt.hour, dt.minute, dt.second), "MM/DD/YYYY", "HH:MM:SS", utcOffset) 155 | 156 | globs.log.write(globs.SEV_DEBUG, function='DateTime', action='toTimestampRfc3339', msg='Date/Time {} converted to {}'.format(tsString, ts)) 157 | return ts 158 | 159 | # Convert from timestamp to resulting time and date formats 160 | def fromTimestamp(ts, dfmt = None, tfmt = None): 161 | globs.log.write(globs.SEV_DEBUG, function='DateTime', action='fromTimestamp', msg='Converting {} (dfmt=\'{}\', tfmt\'{}\')'.format(ts, dfmt, tfmt)) 162 | 163 | # 'x' holds the array for yr/mo/day or hh/m/ss 164 | # Placement in the array is determined by the position values (columns 2, 3, & 4) in the dtFmtDefs[] list 165 | x = [0, 0, 0] 166 | 167 | # If date & time formats are not specified, use the global defaults as defined in the .rc file 168 | if (dfmt is None): 169 | dfmt = globs.opts['dateformat'] 170 | if (tfmt is None): 171 | tfmt = globs.opts['timeformat'] 172 | if ts is None: 173 | timeStampCrash('Timestamp conversion error.') # Write error message, close program 174 | 175 | # Get datetime object from incoming timestamp 176 | dt = datetime.datetime.fromtimestamp(float(ts)) 177 | 178 | # Get date column positions 179 | delim = dtFmtDefs[dfmt][0] # Get the Date delimeter 180 | yrCol = dtFmtDefs[dfmt][1] # Which field holds the year? 181 | moCol = dtFmtDefs[dfmt][2] # Which field holds the month? 182 | dyCol = dtFmtDefs[dfmt][3] # Which field holds the day? 183 | 184 | # Place strftime() format specs in appropriate year/month/day columns 185 | x[yrCol] = '%Y' 186 | x[moCol] = '%m' 187 | x[dyCol] = '%d' 188 | retDate = dt.strftime('{}{}{}{}{}'.format(x[0],delim,x[1],delim,x[2])) 189 | 190 | # Get time column positions 191 | delim = dtFmtDefs[tfmt][0] # Get the time delimeter 192 | hrCol = dtFmtDefs[tfmt][1] # Which field holds the Hour? 193 | mnCol = dtFmtDefs[tfmt][2] # Which field holds the minute? 194 | seCol = dtFmtDefs[tfmt][3] # Which field holds the seconds? 195 | 196 | if not globs.opts['show24hourtime']: 197 | x[hrCol] = '%I' 198 | if dt.hour < 12: 199 | ampm = ' AM' 200 | else: 201 | ampm = ' PM' 202 | else: 203 | x[hrCol] = '%H' 204 | ampm = '' 205 | 206 | x[mnCol] = '%M' 207 | x[seCol] = '%S' 208 | retTime = dt.strftime('{}{}{}{}{}{}'.format(x[0],delim,x[1],delim,x[2], ampm)) 209 | 210 | globs.log.write(globs.SEV_DEBUG, function='DateTime', action='fromTimestamp', msg='Converted {} to {} {}'.format(ts, retDate, retTime)) 211 | return retDate, retTime 212 | 213 | # Calculate # of days since some arbitrary date 214 | def daysSince(tsIn): 215 | # Get the current time (timestamp) 216 | nowTimestamp = datetime.datetime.now().timestamp() 217 | now = datetime.datetime.fromtimestamp(nowTimestamp) 218 | then = datetime.datetime.fromtimestamp(tsIn) 219 | diff = (now-then).days 220 | globs.log.write(globs.SEV_DEBUG, function='DateTime', action='daysSince', msg= 'Now={} {} Then={} {} Days Between={}'.format(nowTimestamp,fromTimestamp(nowTimestamp), tsIn, fromTimestamp(tsIn), diff)) 221 | return diff 222 | 223 | # Calculate time difference between two dates 224 | def timeDiff(td, durationZeroes = False): 225 | # Cast td as a timedelta object 226 | tDelt = datetime.timedelta(seconds = td) 227 | 228 | # Calculate unit values 229 | days = tDelt.days 230 | hours, remainder = divmod(tDelt.seconds, 3600) 231 | minutes, seconds = divmod(remainder, 60) 232 | 233 | # Set return string value based on opts['durationzeroes'] setting 234 | if durationZeroes is True: 235 | retVal = "{}d {}h {}m {}s".format(days, hours, minutes, seconds) 236 | else: # Leave out parts that == 0 237 | retVal = "" 238 | if days != 0: 239 | retVal += "{}d ".format(days) 240 | if hours != 0: 241 | retVal += "{}h ".format(hours) 242 | if minutes != 0: 243 | retVal += "{}m ".format(minutes) 244 | if seconds != 0: 245 | retVal += "{}s ".format(seconds) 246 | 247 | globs.log.write(globs.SEV_DEBUG, function='DateTime', action='timeDiff', msg='td={} duration={}'.format(td, retVal)) 248 | return retVal 249 | 250 | def checkValidDateTimeSpec(tspec, dfmt = None, tfmt = None): 251 | globs.log.write(globs.SEV_NOTICE, function='DateTime', action='checkValidDateTimeSpec', msg='Checking {} for date/time specification validity.'.format(tspec)) 252 | 253 | # Set default formats 254 | if (dfmt is None): 255 | dfmt = globs.opts['dateformat'] 256 | if (tfmt is None): 257 | tfmt = globs.opts['timeformat'] 258 | 259 | # Extract the date 260 | dtPat = re.compile(dateParseRegex) # Compile regex for date/time pattern 261 | dateMatch = re.match(dtPat,tspec) # Match regex against date/time 262 | if dateMatch == None: # Bad date, check timedelta format 263 | if timeDeltaSpec(tspec) == False: 264 | return False 265 | 266 | return True 267 | 268 | # If using a time delta rollback scheme (i.e., '1w,3h') 269 | # Check if the scheme is valid 270 | def timeDeltaSpec(spec): 271 | validSpec = True 272 | 273 | # Check if it's time delta format 274 | tsParts = spec.split(',') 275 | p = re.compile(r'\d+[smhdw]') 276 | 277 | for spec in range(len(tsParts)): 278 | m = p.match(tsParts[spec]) 279 | if m == None: 280 | validSpec = False 281 | 282 | if validSpec: 283 | return tsParts 284 | else: 285 | return False 286 | 287 | -------------------------------------------------------------------------------- /dupReport.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ###### 4 | # 5 | # Program name: dupReport.py 6 | # Purpose: Print summary reports from Duplicati backup service 7 | # Author: Stephen Fried for Handy Guy Software 8 | # Copyright: 2017, release under MIT license. See LICENSE file for details 9 | # 10 | ##### 11 | 12 | # Import system modules 13 | import time 14 | import sys 15 | import platform 16 | import os 17 | import json 18 | 19 | # Import dupReport modules 20 | import globs 21 | import db 22 | import log 23 | import report 24 | import options 25 | import dremail 26 | import drdatetime 27 | import dupapprise 28 | from datetime import datetime 29 | 30 | # Dear future programmer 31 | # When I wrote this code, only God and I knew how it worked. 32 | # Now, only God knows! 33 | # Therefore, if you are trying to optimize this program and it fails (most surely), 34 | # please increase this counter as a warning for the next person: 35 | total_hours_wasted = 1024 36 | 37 | # Print program verersion info 38 | def versionInfo(): 39 | globs.log.out('\n-----\ndupReport: A summary email report generator for Duplicati.') 40 | globs.log.out('Program Version {}.{}.{} {}'.format(globs.version[0], globs.version[1], globs.version[2], globs.status)) 41 | globs.log.out('Database Version {}.{}.{}'.format(globs.dbVersion[0], globs.dbVersion[1], globs.dbVersion[2])) 42 | globs.log.out('RC File Version {}.{}.{}'.format(globs.rcVersion[0], globs.rcVersion[1], globs.rcVersion[2])) 43 | globs.log.out('{}'.format(globs.copyright)) 44 | globs.log.out('Distributed under MIT License. See LICENSE file for details.') 45 | globs.log.out('\nFollow dupReport on Twitter @dupreport\n-----\n') 46 | return None 47 | 48 | if __name__ == "__main__": 49 | # Get program home directory 50 | globs.progPath = os.path.dirname(os.path.realpath(sys.argv[0])) 51 | 52 | # Open a LogHandler object. 53 | # We don't have a log file named yet, but we still need to capture output information 54 | # See LogHandler class description for more details 55 | globs.log = log.LogHandler() 56 | globs.log.write(globs.SEV_NOTICE, function='main', action='startup', msg='dupReport Log - Start') 57 | globs.log.write(globs.SEV_NOTICE, function='main', action='startup', msg='Program Version {}.{}.{} {}'.format(globs.version[0], globs.version[1], globs.version[2], globs.status)) 58 | globs.log.write(globs.SEV_NOTICE, function='main', action='startup', msg='Database Version {}.{}.{}'.format(globs.dbVersion[0], globs.dbVersion[1], globs.dbVersion[2])) 59 | globs.log.write(globs.SEV_NOTICE, function='main', action='startup', msg='Python version {}'.format(sys.version)) 60 | globs.log.write(globs.SEV_NOTICE, function='main', action='startup', msg='OS Platform: {}'.format(platform.platform())) 61 | globs.log.write(globs.SEV_NOTICE, function='main', action='startup', msg='Program path: {}'.format(globs.progPath)) 62 | # Check if we're running a compatible version of Python. Must be 3.0 or higher 63 | if sys.version_info.major < 3: 64 | globs.log.err('dupReport requires Python 3.0 or higher to run. Your installation is on version {}.{}.{}.\nPlease install a newer version of Python.'.format(sys.version_info.major, sys.version_info.minor, sys.version_info.micro)) 65 | globs.closeEverythingAndExit(1) 66 | 67 | # Start Program Timer 68 | startTime = time.time() 69 | 70 | # Initialize program options 71 | # This includes command line options and .rc file options 72 | canContinue = options.initOptions() 73 | if not canContinue: # Something changed in the .rc file that needs manual editing 74 | globs.closeEverythingAndExit(1) 75 | 76 | # Looking for version info on command line? (-V) 77 | if globs.opts['version']: # Print version info & exit 78 | versionInfo() 79 | globs.closeEverythingAndExit(0) 80 | 81 | # Open log file (finally!) 82 | globs.log.openLog(globs.opts['logpath'], globs.opts['logappend'], globs.opts['verbose']) 83 | 84 | # Open report object and validate report options 85 | # We may not be running reports, but the options will be needed later in the program 86 | globs.report = report.Report() 87 | if globs.report.validConfig is False: 88 | errmsg = 'Report configuration has errors. See log file {} for specific error messages.'.format(globs.logName) 89 | globs.log.err(errmsg) 90 | globs.log.write(globs.SEV_ERROR, function='main', action='ValidateConfig', msg=errmsg) 91 | globs.closeEverythingAndExit(0) 92 | if globs.opts['validatereport'] == True: # We just want to validate the report. Exit from here without doing anyting else. 93 | globs.closeEverythingAndExit(0) 94 | 95 | # See if [apprise] is enabled in .rc file. If so, initialize Apprise options 96 | if str.lower(globs.optionManager.getRcOption('apprise', 'enabled')) in ['true']: 97 | globs.appriseObj = dupapprise.dupApprise() 98 | 99 | # Open SQLITE database 100 | globs.db = db.Database(globs.opts['dbpath']) 101 | if globs.opts['initdb'] is True: 102 | # Forced initialization from command line 103 | globs.log.write(globs.SEV_NOTICE, function='main', action='InitDB', msg='Database initialization specified on command line.') 104 | globs.db.dbInitialize() 105 | else: # Check for DB version 106 | needToUpgrade, currDbVersion = globs.db.checkDbVersion() 107 | if needToUpgrade is True: 108 | import convert 109 | globs.log.write(globs.SEV_NOTICE, function='main', action='DBUpgrade', msg='Upgrading database {} from version {} to version {}{}{}'.format(globs.opts['dbpath'], currDbVersion, globs.dbVersion[0], globs.dbVersion[1], globs.dbVersion[2])) 110 | convert.convertDb(currDbVersion) 111 | globs.log.write(globs.SEV_NOTICE, function='main', action='DBUpgrade', msg='Database upgrade complete.') 112 | 113 | # Remove source/destination from database? 114 | if globs.opts['remove']: 115 | globs.db.removeSrcDest(globs.opts['remove'][0], globs.opts['remove'][1]) 116 | globs.closeEverythingAndExit(0) 117 | 118 | # Roll back the database to a specific date? 119 | if globs.opts['rollback']: # Roll back & continue 120 | globs.db.rollback(globs.opts['rollback']) 121 | elif globs.opts['rollbackx']: # Roll back and exit 122 | globs.db.rollback(globs.opts['rollbackx']) 123 | globs.closeEverythingAndExit(0) 124 | 125 | # Open email servers 126 | if globs.opts['showprogress'] > 0: 127 | globs.log.out('Connecting to email servers.') 128 | globs.emailManager = dremail.EmailManager() 129 | 130 | # Are we just collecting or not just reporting? 131 | if (globs.opts['collect'] or not globs.opts['report']): 132 | # Prep email list for potential purging (-p option or [main]purgedb=true) 133 | globs.db.execSqlStmt('UPDATE emails SET dbSeen = 0') 134 | globs.db.dbCommit() 135 | 136 | if globs.opts['showprogress'] > 0: 137 | globs.log.out('Analyzing email messages.') 138 | globs.emailManager.checkForNewMessages() 139 | 140 | # Are we just reporting or not just collecting? 141 | if (globs.opts['report'] or not globs.opts['collect']): 142 | # All email has been collected. Create the report 143 | if globs.opts['showprogress'] > 0: 144 | globs.log.out('Producing report.') 145 | 146 | globs.report.extractReportData() 147 | 148 | # Run selected report 149 | reportOutput = globs.report.createReport(globs.report.rStruct, startTime) 150 | 151 | # Do we need to send any "backup not seen" warning messages? 152 | if not globs.opts['stopbackupwarn'] or not globs.opts['nomail']: 153 | report.sendNoBackupWarnings() 154 | 155 | if globs.appriseObj is not None: 156 | globs.appriseObj.sendNotifications() 157 | 158 | # Do we need to send output to file(s)? 159 | if globs.ofileList and not globs.opts['collect']: 160 | if globs.opts['showprogress'] > 0: 161 | globs.log.out('Creating report file(s).') 162 | report.sendReportToFiles(reportOutput) 163 | 164 | # Are we forbidden from sending report to email? 165 | if not globs.opts['nomail'] and not globs.opts['collect']: 166 | if globs.opts['showprogress'] > 0: 167 | globs.log.out('Sending report emails.') 168 | 169 | # Send email to SMTP server 170 | globs.emailManager.sendEmail(msgHtml=globs.report.createFormattedOutput(reportOutput, 'html'), msgText=globs.report.createFormattedOutput(reportOutput, 'txt'), fileattach=True) 171 | 172 | # Do we need to purge the database? 173 | if globs.opts['purgedb'] == True: 174 | globs.db.purgeOldEmails() 175 | 176 | globs.log.write(globs.SEV_NOTICE, function='main', action='Complete', msg='Program completed in {:.3f} seconds. Exiting.'.format(time.time() - startTime)) 177 | 178 | if globs.opts['showprogress'] > 0: 179 | globs.log.out('Ending program.') 180 | 181 | # Bye, bye, bye, BYE, BYE! 182 | globs.closeEverythingAndExit(0) 183 | -------------------------------------------------------------------------------- /dupReport.rc.EXAMPLE: -------------------------------------------------------------------------------- 1 | [main] 2 | 3 | # Please DO NOT CHANGE the rcversion setting. dupReport uses this to determine if the file needs updating. 4 | rcversion = 3.1.0 5 | 6 | # dbpath & logpath can be either directories or full path specifications 7 | dbpath = C:\Users\yourname\Documents\dupReport 8 | logpath = C:\Users\yourname\Documents\dupReport\dupReport.log 9 | 10 | # Indicates the logging level. Valid levels are: 11 | #SEV_EMERGENCY = 0 12 | #SEV_ALERT = 1 13 | #SEV_CRITICAL = 2 14 | #SEV_ERROR = 3 15 | #SEV_WARNING = 4 16 | #SEV_NOTICE = 5 17 | #SEV_INFO = 6 18 | #SEV_DEBUG = 7 19 | # dupReport will log all messages tagged as <= the level specified in this option. 20 | # In reality, dupReport never logs anything lower than a SEV_ERROR (3), so setting this option lower than that effectively shuts off most logging. 21 | # Default level is 5 (SEV_NOTICE). Level 7 is full debug output. 22 | verbose = 7 23 | 24 | # Do you want to append each log to the last one? 25 | logappend = false 26 | 27 | # The default regular expressions for the subject, source, destination, and delimiter work for most circumstances 28 | subjectregex = ^Duplicati Backup report for 29 | srcregex = \w* 30 | destregex = \w* 31 | srcdestdelimiter = - 32 | 33 | # Date format is VERY important. PLEASE MAKE SURE you have a properly-formatted dateformat spec. 34 | # This should be set in the Guided Startup. It can be overridden on a per-backup basis in the per-back sections below if you have jobs that run in multiple locales. 35 | dateformat = MM/DD/YYYY 36 | 37 | # There is only one valid time specification. DO NOT CHANGE THIS. 38 | timeformat = HH:MM:SS 39 | 40 | # These next options are pretty straightforward. See the documentation for descriptions on how they work 41 | warnoncollect = false 42 | applyutcoffset = true 43 | show24hourtime = false 44 | purgedb = false 45 | showprogress = 0 46 | masksensitive = true 47 | markread = true 48 | 49 | # These options are useful if you want to send log output to a syslog server 50 | syslog = syslog.home.local:514 51 | sysloglevel = 7 52 | 53 | # This is a list of all the email servers, incoming and outgoing. 54 | # Each server named here must have an identically-named [server] section below somewhere else in the file 55 | emailservers = yahooimap, yahoopop3, gmailimap, gmailsmtp, yahoosmtp 56 | 57 | # These are the default incoming and outgoing email server definitions that come with dupReport 58 | # You can modify these or use custom-built definitions as shown below. 59 | # Just make sure whatever server definitions you are using are put in the "emailservers=" option in the [main] section 60 | [incoming] 61 | protocol = imap 62 | server = imap.gmail.com 63 | port = 993 64 | encryption = tls 65 | account = accountname@gmail.com 66 | password = secretpassword 67 | folder = INBOX 68 | keepalive = false 69 | unreadonly = false 70 | markread = false 71 | authentication = basic 72 | 73 | [outgoing] 74 | server = smtp.gmail.com 75 | protocol = smtp 76 | port = 587 77 | encryption = tls 78 | account = accountname@gmail.com 79 | password = secretpassword 80 | sender = accountname@gmail.com 81 | sendername = dupReport Summary 82 | receiver = accountname@gmail.com 83 | keepalive = false 84 | markread = false 85 | authentication = basic 86 | 87 | # Example settings for Yahoo's IMAP server 88 | [yahooimap] 89 | protocol = imap 90 | server = imap.mail.yahoo.com 91 | port = 993 92 | encryption = tls 93 | account = accountname@yahoo.com 94 | password = secretpassword 95 | folder = INBOX 96 | keepalive = false 97 | unreadonly = false 98 | markread = false 99 | authentication = basic 100 | 101 | # Example settings for Yahoo's POP3 server 102 | [yahoopop3] 103 | protocol = pop3 104 | server = pop.mail.yahoo.com 105 | port = 995 106 | encryption = tls 107 | account = accountname@yahoo.com 108 | password = secretpassword 109 | folder = INBOX 110 | keepalive = false 111 | unreadonly = false 112 | markread = false 113 | authentication = basic 114 | 115 | # Example settings for Gmail's IMAP server 116 | [gmailimap] 117 | protocol = imap 118 | server = imap.gmail.com 119 | port = 993 120 | encryption = tls 121 | account = accountname@gmail.com 122 | password = secretpassword 123 | folder = INBOX 124 | keepalive = false 125 | unreadonly = false 126 | markread = false 127 | authentication = basic 128 | 129 | # Example settings for Gmail's SMTP server 130 | [gmailsmtp] 131 | server = smtp.gmail.com 132 | protocol = smtp 133 | port = 587 134 | encryption = tls 135 | account = accountname@gmail.com 136 | password = secretpassword 137 | sender = accountname@gmail.com 138 | sendername = dupReport Summary 139 | receiver = accountname@gmail.com 140 | keepalive = false 141 | markread = false 142 | authentication = basic 143 | 144 | # Example settings for Yahoo's SMTP server 145 | [yahoosmtp] 146 | server = smtp.mail.yahoo.com 147 | protocol = smtp 148 | port = 587 149 | encryption = tls 150 | account = accountname@yahoo.com 151 | password = secretpassword 152 | sender = accountname@yahoo.com 153 | sendername = dupReport Summary 154 | receiver = accountname@yahoo.com 155 | keepalive = false 156 | markread = false 157 | authentication = basic 158 | 159 | # Backup-specific sections. Put information about specific backup jobs here 160 | # Each section must match the EXACT name of one of the backup jobs (i.e., <source>-<destination>) 161 | 162 | # This one says to not send email warnings if the backup hasn't been seen in x days 163 | [Server1-NAS] 164 | nobackupwarn = 0 165 | 166 | # This one sets the no-backup warning intrval to 10 days and sends the no-backup warning to another email address 167 | [Server2-NAS] 168 | nobackupwarn = 10 169 | receiver = anotheruser@gmail.com 170 | 171 | # This tells dupReport that this server is offline, so ignore any warnings about not seeing a backup 172 | [Server3-B2] 173 | offline = true 174 | 175 | # This backup runs in a different locale that uses a different date format than the one specified in the [main section] 176 | [Server4-B2] 177 | dateformat = YYYY.MM.DD 178 | 179 | # This is the main report defintion section. This sets the default options for all reports. 180 | # Most options can be overridden in report-specific sections. 181 | # See the documentation for a description of each option 182 | [report] 183 | 184 | # The layout option defines the order in which the various report sections will be shown in the final report 185 | # Each report needs a report definition section below in the .rc file 186 | layout = noactivity, lastseen, srcdest, bydestnogroups, bysrc,bydest,bydate,srcdest, noactivity, lastseen 187 | 188 | sizedisplay = mb 189 | border = 5 190 | padding = 5 191 | displayerrors = true 192 | displaywarnings = false 193 | displaymessages = false 194 | sortby = source 195 | titlebg = #FFFFFF 196 | jobmessagebg = #FFFFFF 197 | jobwarningbg = #FFFF00 198 | joberrorbg = #FF0000 199 | suppresscolumntitles = false 200 | nobackupwarn = 0 201 | nbwsubject = Backup Warning: #SOURCE##DELIMITER##DESTINATION# Backup Not Seen for #DAYS# Days 202 | truncatemessage = 0 203 | truncatewarning = 0 204 | truncateerror = 0 205 | durationzeroes = false 206 | displaylogdata = true 207 | joblogdatabg = #FFFF00 208 | truncatelogdata = 0 209 | # Email title will include an indication if any of the backup jobs ended with an error status 210 | title = Duplicati Backup Summary Report #ANYERROR# 211 | # Note that column defintions can be split into multiple lines. BUT, you MUST LEAVE SPACES BEFORE THE SECOND LINE (and the third and the fourth...) 212 | # If you try to split .rc entries up into multiple lines and you're getting strange results, chances are you did it wrong. 213 | columns = source:Source, destination:Destination, date: Date, time: Time, dupversion:Version, duration:Duration, examinedFiles:Files, 214 | examinedFilesDelta:+/-, sizeOfExaminedFiles:Size, fileSizeDelta:+/-, bytesUploaded:Uploaded, bytesDownloaded:Downloaded, addedFiles:Added, deletedFiles:Deleted, modifiedFiles:Modified, 215 | filesWithError:Errors, parsedResult:Result, messages:Messages, warnings:Warnings, errors:Errors, logdata:Log Data 216 | weminline = false 217 | groupheadingbg = #D3D3D3 218 | normaldays = 5 219 | normalbg = #FFFFFF 220 | warningdays = 10 221 | warningbg = #FFFF00 222 | errorbg = #FF0000 223 | repeatcolumntitles = true 224 | includeruntime = true 225 | failedonly = False 226 | 227 | # Sample report definitions 228 | # The field orders and column title defintions have been changed around a bit from the defaults in these examples just to demonstrate how it's done 229 | 230 | # This defines the standard "SrcDest" report. Show backup results ordered by Source-Destination pairs. 231 | [srcdest] 232 | type = report 233 | title = Duplicati Backup Summary Report - By Source/Destination 234 | groupby = source:ascending, destination:ascending 235 | groupheading = Source: #SOURCE# Destination: #DESTINATION# 236 | columns = date:Date, time:Time, dupversion:Version, parsedResult:Result, duration:Duration, examinedFilesDelta:Files +/-, fileSizeDelta: File Size (MB) +/-, 237 | addedFiles:Added, deletedFiles:Deleted, modifiedFiles:Modified, parsedResult:Result 238 | columnsort = date:ascending, time:ascending 239 | 240 | # This defines the standard "By Source" report. Show backup results ordered by Source system. 241 | [bysrc] 242 | type = report 243 | title = Duplicati Backup Summary Report - By Source 244 | groupby = source : ascending 245 | groupheading = Source: #SOURCE# 246 | columns = destination:Destiation, date:Date, time:Time, examinedFilesDelta: Files +/-, fileSizeDelta: File Size (MB) +/-, 247 | parsedResult:Result 248 | columnsort = destination:ascending, date:ascending, time:ascending 249 | 250 | # This defines the standard "By Destination" report. Show backup results ordered by Destination system. 251 | [bydest] 252 | type = report 253 | title = Duplicati Backup Summary Report - By Destination 254 | groupby = destination:ascending 255 | groupheading = Destination: #DESTINATION# 256 | columns = parsedResult:Result, source:Source, dupversion:Version, date:Date, time:Time, duration:Length, examinedFiles:# Files, examinedFilesDelta:#F +/-, sizeOfExaminedFiles:FSize (GB), fileSizeDelta:FS (GB) +/-, errors:Errors, logdata:Log Data 257 | columnsort = source:ascending, date:ascending, time:ascending 258 | suppresscolumntitles = false 259 | sizedisplay = gb 260 | 261 | # This defines the standard "By Date" report. Show backup results ordered by Date. 262 | [bydate] 263 | type = report 264 | title = Duplicati Backup Summary Report - By Date 265 | groupby = date:ascending 266 | groupheading = Date: #DATE# 267 | columns = time:Time, source:Source, destination:Destination, duration:Duration, examinedFiles:Files, examinedFilesDelta:+/-, sizeOfExaminedFiles:Size (Bytes), fileSizeDelta: Bytes +/-, parsedResult:Result, warnings:Warnings, errors:Errors, logdata:Log Data 268 | weminline = true 269 | columnsort = time:ascending 270 | sizedisplay = none 271 | 272 | # This shows how define a report that isn't split into groups 273 | [bydestnogroups] 274 | type = report 275 | title = Duplicati Backup Summary Report - By Destination (No Groups) 276 | columns = destination:Destination, source:Source, date:Date, time:Time, duration:Duration, examinedFilesDelta: Files +/-, fileSizeDelta: Size (GB) +/-, 277 | parsedResult:Result, warnings:Warnings, errors:Errors, messages:Messages 278 | columnsort = destination:ascending, source:ascending, date:ascending, time:ascending 279 | sizedisplay = gb 280 | displayerrors = true 281 | displaywarnings = true 282 | displaymessages = true 283 | 284 | # This defines a report that only shows Source-Destination pairs that showed no activity 285 | [noactivity] 286 | type = noactivity 287 | title = Non-Activity Report 288 | 289 | # This defines a report that shows when each Source-Destination pair was last seen by dupReport 290 | [lastseen] 291 | type = lastseen 292 | title = Backup Sets Last Seen 293 | 294 | # This defines a report that shos which backup sets are listed as "offline" 295 | [offline] 296 | type = offline 297 | title = Offline Backup Sets 298 | suppresscolumntitles = true 299 | 300 | # Sample apprise section for sending email notifications 301 | [apprise] 302 | status = off 303 | services = mailto://username:password@gmail.com 304 | bodytruncate = 500 305 | msglevel = warning 306 | body = Completed at #COMPLETETIME#: #RESULT# - Warning: #WARNMSG# Error: #ERRMSG# 307 | -------------------------------------------------------------------------------- /dupapprise.py: -------------------------------------------------------------------------------- 1 | ##### 2 | # 3 | # Module name: dupApprise.py 4 | # Purpose: Management class for Apprise notification service 5 | # 6 | # Notes: Uses the Apprise push notification utility from @caronc 7 | # https://github.com/caronc/apprise 8 | # For any Apprise support or feature requests, please see the Apprise GitHub site 9 | # 10 | ##### 11 | 12 | # Import system modules 13 | import db 14 | import drdatetime 15 | 16 | # Import dupReport modules 17 | import globs 18 | 19 | class dupApprise: 20 | appriseConn = None 21 | appriseOpts = None 22 | services = None 23 | 24 | def __init__(self): 25 | globs.log.write(globs.SEV_NOTICE, function='Apprise', action='Init', msg='Initializing Apprise support') 26 | 27 | import apprise 28 | 29 | # Read name/value pairs from [apprise] section 30 | self.appriseOpts = globs.optionManager.getRcSection('apprise') 31 | 32 | if 'services' not in self.appriseOpts: 33 | globs.log.write(globs.SEV_ERROR, function='Apprise', action='Init', msg='Error: No services defined for Apprise notification') 34 | globs.closeEverythingAndExit(1) # Abort program. Can't continue 35 | 36 | # Set defaults for missing values 37 | self.appriseOpts['title'] = 'Apprise Notification for #SRCDEST# Backup' if 'title' not in self.appriseOpts else self.appriseOpts['title'] 38 | self.appriseOpts['body'] = 'Completed at #COMPLETETIME#: #RESULT# - #ERRMSG#' if 'body' not in self.appriseOpts else self.appriseOpts['body'] 39 | self.appriseOpts['titletruncate'] = '0' if 'titletruncate' else self.appriseOpts['titletruncate'] 40 | self.appriseOpts['bodytruncate'] = '0' if 'bodytruncate' not in self.appriseOpts else self.appriseOpts['bodytruncate'] 41 | self.appriseOpts['msglevel'] = 'failure' if 'msglevel' not in self.appriseOpts else self.appriseOpts['msglevel'] 42 | 43 | # Normalize .rc values 44 | self.appriseOpts['titletruncate'] = int(self.appriseOpts['titletruncate']) 45 | self.appriseOpts['bodytruncate'] = int(self.appriseOpts['bodytruncate']) 46 | self.appriseOpts['msglevel'] = self.appriseOpts['msglevel'].lower() 47 | 48 | # Check for correct message level indicator 49 | if self.appriseOpts['msglevel'] not in ('success', 'warning', 'failure'): 50 | globs.log.write(globs.SEV_ERROR, function='Apprise', action='Init', msg='Error: Bad apprise message level: {}'.format(self.appriseOpts['msglevel'])) 51 | globs.closeEverythingAndExit(1) # Abort program. Can't continue. 52 | 53 | # Initialize apprise library 54 | result = self.appriseConn = apprise.Apprise() 55 | globs.log.write(globs.SEV_NOTICE, function='Apprise', action='Init', msg='Initializing Apprise library. Result={}'.format(result)) 56 | 57 | # Add individual service URLs to connection 58 | self.services = self.appriseOpts['services'].split(",") 59 | for i in self.services: 60 | result = self.appriseConn.add(i) 61 | globs.log.write(globs.SEV_NOTICE, function='Apprise', action='Init', msg='Added service {}, result={}'.format(i, result)) 62 | 63 | globs.log.write(globs.SEV_NOTICE, function='Apprise', action='Init', msg='Apprise Initialization complete.') 64 | return None 65 | 66 | def parseMessage(self, msg, source, destination, result, message, warningmessage, errormessage, completetime): 67 | globs.log.write(globs.SEV_NOTICE, function='Apprise', action='parseMessage', msg=msg) 68 | 69 | newMsg = msg 70 | newMsg = newMsg.replace('#SOURCE#',source) 71 | newMsg = newMsg.replace('#DESTINATION#',destination) 72 | newMsg = newMsg.replace('#SRCDEST#','{}{}{}'.format(source, globs.opts['srcdestdelimiter'], destination)) 73 | newMsg = newMsg.replace('#RESULT#',result) 74 | newMsg = newMsg.replace('#MESSAGE#',message) 75 | newMsg = newMsg.replace('#ERRMSG#',errormessage) 76 | newMsg = newMsg.replace('#WARNMSG#',warningmessage) 77 | newMsg = newMsg.replace('#COMPLETETIME#','{} {}'.format(completetime[0], completetime[1])) 78 | 79 | globs.log.write(globs.SEV_NOTICE, function='Apprise', action='parseMessage', msg='New message=[{}]'.format(newMsg)) 80 | return newMsg 81 | 82 | def sendNotifications(self): 83 | sqlStmt = "SELECT source, destination, parsedResult, messages, warnings, errors, timestamp FROM report ORDER BY source" 84 | dbCursor = globs.db.execSqlStmt(sqlStmt) 85 | reportRows = dbCursor.fetchall() 86 | 87 | for source, destination, parsedResult, messages, warnings, errors, timestamp in reportRows: 88 | globs.log.write(globs.SEV_NOTICE, function='Apprise', action='sendNotifications', msg='Preparing Apprise message for {}-{}, parsedResult={} msglevel={}'.format(source, destination, parsedResult, self.appriseOpts['msglevel'])) 89 | 90 | # See if we need to send a notification based on the result status 91 | if self.appriseOpts['msglevel'] == 'warning': 92 | if parsedResult.lower() not in ('warning', 'failure'): 93 | globs.log.write(globs.SEV_NOTICE, function='Apprise', action='sendNotifications', msg='Msglevel mismatch at warning level - skipping') 94 | continue 95 | elif self.appriseOpts['msglevel'] == 'failure': 96 | if parsedResult.lower() != 'failure': 97 | globs.log.write(globs.SEV_NOTICE, function='Apprise', action='sendNotifications', msg='Msglevel mismatch at failure level - skipping') 98 | continue 99 | 100 | globs.log.write(globs.SEV_DEBUG, function='Apprise', action='sendNotifications', msg='Apprise message is sendable.') 101 | 102 | newTitle = self.parseMessage(self.appriseOpts['title'], source, destination, parsedResult, messages, warnings, errors, drdatetime.fromTimestamp(timestamp)) 103 | newBody = self.parseMessage(self.appriseOpts['body'], source, destination, parsedResult, messages, warnings, errors, drdatetime.fromTimestamp(timestamp)) 104 | 105 | tLen = self.appriseOpts['titletruncate'] 106 | if tLen != 0: 107 | newTitle = (newTitle[:tLen]) if len(newTitle) > tLen else newTitle 108 | bLen = self.appriseOpts['bodytruncate'] 109 | if bLen!= 0: 110 | newBody = (newBody[:bLen]) if len(newBody) > bLen else newBody 111 | 112 | globs.log.write(globs.SEV_DEBUG, function='Apprise', action='sendNotifications', msg='Sending notification: Title=[{}] Body=[{}]'.format(newTitle, newBody)) 113 | result = self.appriseConn.notify(title=newTitle, body=newBody) 114 | globs.log.write(globs.SEV_NOTICE, function='Apprise', action='sendNotifications', msg='Apprise sent. Result={}.'.format(result)) 115 | 116 | return 117 | -------------------------------------------------------------------------------- /globs.py: -------------------------------------------------------------------------------- 1 | ##### 2 | # 3 | # Module name: globs.py 4 | # Purpose: Global object repository for use by dupReport program 5 | # 6 | ##### 7 | 8 | import os 9 | 10 | # Define version info 11 | version=[3,0,10] # Program Version 12 | status='Release' 13 | dbVersion=[3,0,1] # Required DB version 14 | rcVersion=[3,1,0] # Required RC version 15 | copyright='Copyright (c) 2017-2022 Stephen Fried for Handy Guy Software.' 16 | 17 | # Define global variables 18 | dbName='dupReport.db' # Default database name 19 | logName='dupReport.log' # Default log file name 20 | rcName='dupReport.rc' # Default configuration file name 21 | db = None # Global database object 22 | dateFormat = None # Global date format - can be overridden per backup set 23 | timeFormat = None # Global time format - can be overridden per backup set 24 | report = None # Global report object 25 | ofileList = None # List of output files 26 | optionManager = None # Global Option Manager 27 | emailManager = None # Global email server management 28 | opts = None # Global program options 29 | progPath = None # Path to script files 30 | appriseObj = None # dupApprise instance 31 | 32 | # Text & format fields for report email 33 | emailText=[] # List of email text components 34 | emailFormat=[] # Corresponding list of emial components print formats 35 | 36 | # Global variables referencing objects in other modules 37 | log = None # Log file handling 38 | inServer = None # Inbound email server 39 | outServer = None # Outbound email server 40 | 41 | # Define logging levels 42 | SEV_EMERGENCY = 0 43 | SEV_ALERT = 1 44 | SEV_CRITICAL = 2 45 | SEV_ERROR = 3 46 | SEV_WARNING = 4 47 | SEV_NOTICE = 5 48 | SEV_INFO = 6 49 | SEV_DEBUG = 7 50 | 51 | sevlevels = [ 52 | ('EMERGENCY', SEV_EMERGENCY), 53 | ('ALERT', SEV_ALERT), 54 | ('CRITICAL', SEV_CRITICAL), 55 | ('ERROR', SEV_ERROR), 56 | ('WARNING', SEV_WARNING), 57 | ('NOTICE', SEV_NOTICE), 58 | ('INFO', SEV_INFO), 59 | ('DEBUG', SEV_DEBUG) 60 | ] 61 | 62 | # Mask sensitive data in log files 63 | # Replace incoming string with string of '*' the same length of the original 64 | def maskData(inData, force = False): 65 | if inData is None: # Empty input or global options haven't been processed yet. Return unmasked input. 66 | return inData 67 | elif force: # Mask regardless of what parameter says. Useful if masking before options are processed. 68 | return "*" * len(inData) 69 | elif opts is None: 70 | return inData 71 | elif 'masksensitive' in opts and opts['masksensitive'] is True: 72 | return "*" * len(inData) # Mask data. 73 | else: 74 | return inData # Return unmasked input 75 | 76 | # Close everything and exit cleanly 77 | def closeEverythingAndExit(errcode): 78 | 79 | log.write(SEV_NOTICE, function='Globs', action='closeEverythingAndExit', msg='Closing everything...') 80 | if emailManager != None: 81 | if len(emailManager.incoming) != 0: 82 | for server in emailManager.incoming: 83 | log.write(SEV_NOTICE, function='Globs', action='closeEverythingAndExit', msg='Closing inbound email server: {}'.format(emailManager.incoming[server].name)) 84 | emailManager.incoming[server].close() 85 | if len(emailManager.incoming) != 0: 86 | for i in range(len(emailManager.outgoing)): 87 | log.write(SEV_NOTICE, function='Globs', action='closeEverythingAndExit', msg='Closing outbound email server: {}'.format(emailManager.outgoing[i].name)) 88 | emailManager.outgoing[i].close() 89 | if db is not None: 90 | log.write(SEV_NOTICE, function='Globs', action='closeEverythingAndExit', msg='Closing database file.') 91 | db.dbClose() 92 | if log is not None: 93 | log.write(SEV_NOTICE, function='Globs', action='closeEverythingAndExit', msg='Closing log file.') 94 | log.closeLog() 95 | 96 | os._exit(errcode) 97 | 98 | -------------------------------------------------------------------------------- /log.py: -------------------------------------------------------------------------------- 1 | ##### 2 | # 3 | # Module name: log.py 4 | # Purpose: Manage logging functions for dupReport 5 | # 6 | # Notes: 7 | # 8 | ##### 9 | 10 | # Import system modules 11 | import sys 12 | import globs 13 | import os 14 | import datetime 15 | import logging 16 | import socket 17 | from logging.handlers import SysLogHandler 18 | 19 | 20 | # Class to handle log management 21 | class LogHandler: 22 | def __init__(self): 23 | self.logFile = None # Handle to log file, when opened 24 | self.defLogLevel = globs.SEV_NOTICE # Default logging level. Will get updated when log is opened 25 | self.tmpFile = None # Temp file to hold log output before log file is opened. 26 | self.tmpLogPath = globs.progPath + '/' + globs.logName # Path for temp log 27 | self.hostname = socket.gethostname() 28 | self.syslog = { 29 | 'logger': None, 30 | 'host': None, 31 | 'port': 514, 32 | 'level': globs.SEV_NOTICE, 33 | 'handler': None 34 | } 35 | 36 | return None 37 | 38 | def openLog(self, path = None, append = False, level = globs.SEV_DEBUG): 39 | if self.logFile is not None: # Another log file is open. Need to close it first 40 | self.logFile.close() 41 | 42 | self.defLogLevel = level 43 | 44 | if path is not None: # Path provided. Open log file for write or append 45 | 46 | # Issue #148. We opened the default log file (self.tmpLogPath) to start collecting logs until we found the real name in the .rc file. 47 | # If it turns out that the actual log file (passwd in via the 'path' parameter') is the same as the temporary file, just keep using that. 48 | # Else, open the log file & copy the temp log contents into it. 49 | if os.path.normpath(os.path.normcase(self.tmpLogPath)) != os.path.normpath(os.path.normcase(path)): # Are the two paths different? 50 | globs.log.write(globs.SEV_DEBUG, function='Log', action='openLog', msg='Log file {} different than tmp file {}. Copying data.'.format(path, self.tmpLogPath)) 51 | try: 52 | if append is True: 53 | self.logFile = open(path,'a', encoding="utf-8") 54 | else: 55 | self.logFile = open(path,'w', encoding="utf-8") 56 | # Now,copy any existing data from the temp file 57 | if self.tmpFile is not None: 58 | self.tmpFile.close() 59 | self.tmpFile = open(self.tmpLogPath, 'r') 60 | tmpData = self.tmpFile.read() 61 | self.logFile.write(tmpData) 62 | self.logFile.flush() 63 | self.tmpFile.close() 64 | os.remove(self.tmpLogPath) 65 | self.tmpFile = None 66 | except (OSError, IOError): 67 | e = sys.exc_info()[0] 68 | globs.log.write(globs.SEV_ERROR, function='Log', action='openLog', msg='Error opening log file {}: {}'.format(path, e)) 69 | sys.stderr.write('Error opening log file {}: {}\n'.format(path, e)) 70 | 71 | if globs.opts['syslog'] != '': 72 | syslogparts = globs.opts['syslog'].split(':') 73 | self.syslog['host'] = syslogparts[0] 74 | if len(syslogparts) == 2: # Syslog port specified. 75 | self.syslog['port'] = int(syslogparts[1]) 76 | 77 | if globs.opts['sysloglevel'] not in range(8): # Severity levels are 0-7 78 | globs.log.write(globs.SEV_ERROR, function='Log', action='openLog', msg='Invalid syslog level specified: {}. Reverting to level 5 (SEV_NOTICE)'.format(globs.opts['sysloglevel'])) 79 | else: 80 | self.syslog['level'] = globs.opts['sysloglevel'] 81 | 82 | globs.log.write(globs.SEV_DEBUG, function='Log', action='openLog', msg='Opening syslog connection to {}:{}'.format(self.syslog['host'], self.syslog['port'])) 83 | try: 84 | self.syslog['logger'] = logging.getLogger() 85 | self.syslog['logger'].setLevel(self.syslog['level']) 86 | self.syslog['handler'] = SysLogHandler(address=(self.syslog['host'], self.syslog['port']), facility = 16) 87 | self.syslog['logger'].addHandler(self.syslog['handler']) 88 | self.syslog['logger'].propagate = False 89 | except : 90 | e = sys.exc_info()[0] 91 | globs.log.write(globs.SEV_ERROR, function='Log', action='connect:syslog', msg='Error connecting to syslog sever {}:{}. Most likely an incorrect server or port was specified. Msg: {}'.format(self.syslog['host'], self.syslog['port'], e)) 92 | self.syslog['handler'] = None 93 | 94 | return None 95 | 96 | def closeLog(self): 97 | if self.syslog['handler'] != None: 98 | self.write(globs.SEV_NOTICE, function='Log', action='closeLog', msg='Closing syslog connection') 99 | self.syslog['handler'].close 100 | 101 | if self.logFile is not None: # Another log file is open. Need to close it first 102 | self.logFile.close() 103 | self.logFile = None 104 | 105 | return None; 106 | 107 | # Write log info to stderr 108 | def err(self, msg): 109 | if (msg is not None) and (msg != ''): 110 | sys.stderr.write('ERROR: {}\n'.format(msg)) 111 | sys.stderr.flush() 112 | return None 113 | 114 | # Write log info to stdout 115 | def out(self, msg, newline=True): 116 | if (msg is not None) and (msg != ''): 117 | sys.stdout.write(msg) 118 | if newline: 119 | sys.stdout.write('\n') 120 | sys.stdout.flush() 121 | return None 122 | 123 | def writeSyslog(self, level, msg): 124 | 125 | newMsg = msg 126 | 127 | # Syslog messages have to be less than 1K in length. 128 | if len(msg) > 999: 129 | newMsg = msg[:999] 130 | self.write(globs.SEV_DEBUG, function='log', action='writeSyslog', msg='Truncating syslog message') 131 | 132 | if level <= self.syslog['level']: # Check that we're writing to an appropriate logging level 133 | if level <= globs.SEV_CRITICAL: 134 | self.syslog['logger'].critical(newMsg) 135 | elif level <= globs.SEV_ERROR: 136 | self.syslog['logger'].error(newMsg) 137 | elif level <= globs.SEV_WARNING: 138 | self.syslog['logger'].warning(newMsg) 139 | elif level <= globs.SEV_INFO: 140 | self.syslog['logger'].info(newMsg) 141 | elif level <= globs.SEV_DEBUG: 142 | self.syslog['logger'].debug(newMsg) 143 | 144 | return None 145 | 146 | 147 | # Write log info to log file 148 | # Log Format = [TIMESTAMP][SEVERITY][FUNCTION][ACTION]<MESSAGE> 149 | def write(self, level, function='-', action='-', msg='' ): 150 | 151 | if self.logFile is not None: 152 | logTarget = self.logFile 153 | else: 154 | # Log file hasn't been opened yet. Send output to temp file 155 | if self.tmpFile is None: 156 | # Open a temp file to hold the output 157 | self.tmpFile = open(self.tmpLogPath, 'w') 158 | logTarget = self.tmpFile 159 | 160 | if (msg is not None) and (msg != ''): # Non-empty message. Good to go... 161 | logData = '[{}][{}][{}][{}]{}'.format(datetime.datetime.now().isoformat(), globs.sevlevels[level][0], function, action, msg) 162 | 163 | # Write to program log 164 | if level <= self.defLogLevel: # Check that we're writing to an appropriate logging level 165 | logTarget.write(logData) 166 | logTarget.write('\n') 167 | logTarget.flush() # Protect against buffered log data getting lost due to program crash 168 | 169 | # Write to syslog, if necessary 170 | if self.syslog['handler'] != None: 171 | self.writeSyslog(level, logData) 172 | 173 | return None 174 | 175 | --------------------------------------------------------------------------------