├── .gitattributes ├── README.md └── S123ReportAndEmailSubmissions.py /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Automated Survey123 Reports and Emailing 2 | 3 | ## What is it? 4 | A tool to automate Survey123 Report creation & emailing survey recipients with a copy of the report. 5 | 6 | This script takes an input Survey ID (from Survey123), a report template (uploaded to the Survey Reports (Beta) tab), and will do the following: 7 | 8 | - Generate .docx reports for any new Survey submissions from the last 24 hours, 9 | - Save these reports in bulk to AGOL, download them one by one to a location, then remove them from AGOL when finished, 10 | - Read and extract an email address from each .docx file, 11 | - Send the relevant .docx file as an attachment to the relevant recipient, 12 | - Remove the .docx file once the email has sent, 13 | - Logs the daily results to a txt file in the output folder. 14 | 15 | Call this script with python "..\S123ReportAndEmailSubmissions.py" 16 | - eg. in a Unix Cron job or Windows Task Scheduler that runs once a day - since we are always looking for submissions from the last 24 hours 17 | 18 | Note: this script generates the KeyError: 'results' but still works due to the try/except/finally block... 19 | Related to this ESRI bug: 20 | *BUG-000119057 : The Python API 1.5.2 generate_report() method of the arcgis.apps.survey123 module, generates the following error: { KeyError: 'results' }* 21 | 22 | API docs: https://esri.github.io/arcgis-python-api/apidoc/html/arcgis.apps.survey123.html 23 | 24 | ## Requirements 25 | - Customise the .py file with the below variables as desired 26 | - Use with Python v3 and install the required Python libraries 27 | - Set the script to run daily with a Unix Cron job or Windows Task Scheduler 28 | 29 | ### How is this automated? 30 | To automate this process, simply add the Python script as a regular task (eg. daily) in your Task Scheduler application on a Windows device. 31 | 32 | For Linux devices, you'll need to rewrite parts of the code to point to your respective `/home` location and create a daily CRON task. 33 | 34 | 35 | ## Customisation 36 | ```python 37 | # --- AGOL information... --- 38 | org = 'https://YOUR-ORGANISATION.maps.arcgis.com' 39 | username = 'ARCGIS ONLINE USERNAME' 40 | password = 'ARCGIS ONLINE PASSWORD' 41 | 42 | 43 | # --- Survey123 variables... --- 44 | surveyID = 'ID OF SURVEY123 FORM' # ID of desired Survey123 form - a unique ID like 4c1b359c4e294c54a02b22b42413f1 45 | output_folder = r'C:\GISWORK\_tmp\Reports' # Output folder WITHOUT trailing slash. This is also where the log file is stored. 46 | 47 | # WHERE_FILTER: Use '1=1' to return all records, or something like {{"where":"=''"} - supports SQL syntax 48 | # Docs for date queries: https://www.esri.com/arcgis-blog/products/api-rest/data-management/querying-feature-services-date-time-queries/ 49 | # In our case below, we filter by records created in the last 1 day (24 hrs). This works for us as the script is run on a daily schedule. 50 | where_filter = '{"where":"CreationDate >= CURRENT_TIMESTAMP - INTERVAL \'1\' DAY"}' 51 | 52 | utc_offset = '+13:00' # UTC Offset for location (+13 is NZST) 53 | report_title = 'Daily_Export' # Title that will show in Survey123 Reports recent task list 54 | report_template = 1 # ID of the print template in Survey123 that you want to use (0 = ESRI's sample, 1 = first custom report, 2 = second custom report, etc) 55 | 56 | 57 | # --- Email SMTP settings... --- 58 | email_user = 'EMAIL ADDRESS' # Eg. user@gmail.com. Requires a valid SMTP-enabled email account (Eg. a Gmail acct with the SMTP settings below) 59 | email_password = 'EMAIL ACCOUNT PASSWORD' # Password for the email account 60 | smtp_server = 'smtp.gmail.com' 61 | smtp_port = 587 62 | ``` 63 | -------------------------------------------------------------------------------- /S123ReportAndEmailSubmissions.py: -------------------------------------------------------------------------------- 1 | import os 2 | import smtplib 3 | from email import encoders 4 | from email.mime.base import MIMEBase 5 | from email.mime.text import MIMEText 6 | from email.mime.multipart import MIMEMultipart 7 | import urllib 8 | import requests 9 | import json 10 | import sys 11 | import docx 12 | import glob 13 | import datetime 14 | from arcgis.gis import GIS 15 | from arcgis.apps.survey123._survey import SurveyManager, Survey 16 | 17 | # -------------------------------------------------------------------------------------------------------------------------- 18 | # 19 | # This script takes an input Survey ID (from Survey123), a report template (uploaded to the Survey Reports [Beta] tab), and will do the following: 20 | # 21 | # - Generate .docx reports for any new Survey submissions from the last 24 hours, 22 | # - Save these reports in bulk to AGOL, download them one by one to a location, then remove them from AGOL when finished, 23 | # - Read and extract an email address from each .docx file, 24 | # - Send the relevant .docx file as an attachment to the relevant recipient, 25 | # - Remove the .docx file once the email has sent, 26 | # - Logs the daily results to a txt file in the output folder. 27 | # 28 | # Developed by John Stowell, 2019 29 | # 30 | # Call this script with python "..\S123ReportAndEmailSubmissions.py" 31 | # - eg. in a Unix Cron job or Windows Task Scheduler that runs once a day - since we are always looking for submissions from the last 24 hours 32 | # 33 | # Also, this script generates the KeyError: 'results' but still works due to the try/except/finally block... 34 | # Related to this ESRI bug: 35 | # BUG-000119057 : The Python API 1.5.2 generate_report() method of the arcgis.apps.survey123 module, generates the following error: { KeyError: 'results' } 36 | # 37 | # API docs: https://esri.github.io/arcgis-python-api/apidoc/html/arcgis.apps.survey123.html 38 | # 39 | # -------------------------------------------------------------------------------------------------------------------------- 40 | 41 | def main(): 42 | # Customise the variables below 43 | # --- AGOL information... --- 44 | org = 'https://YOUR-ORGANISATION.maps.arcgis.com' 45 | username = 'ARCGIS ONLINE USERNAME' 46 | password = 'ARCGIS ONLINE PASSWORD' 47 | 48 | 49 | # --- Survey123 variables... --- 50 | surveyID = 'ID OF SURVEY123 FORM' # ID of desired Survey123 form - a unique ID like 4c1b359c4e294c54a02b262b42413f17 51 | output_folder = r'C:\GISWORK\_tmp\Reports' # Output folder WITHOUT trailing slash. This is also where the log file is stored. 52 | 53 | # WHERE_FILTER: Use '1=1' to return all records, or something like {{"where":"=''"} - supports SQL syntax 54 | # Docs for date queries: https://www.esri.com/arcgis-blog/products/api-rest/data-management/querying-feature-services-date-time-queries/ 55 | # In our case below, we filter by records created in the last 1 day. This works for us as the script is run on a daily schedule. 56 | where_filter = '{"where":"CreationDate >= CURRENT_TIMESTAMP - INTERVAL \'1\' DAY"}' 57 | 58 | utc_offset = '+13:00' # UTC Offset for location (+13 is NZST) 59 | report_title = 'Daily_Export' # Title that will show in S123 recent task list 60 | report_template = 1 # ID of the print template in Survey123 that you want to use (0 = ESRI's sample, 1 = first custom report, 2 = second custom report, etc) 61 | 62 | 63 | # --- Email SMTP settings... --- 64 | email_user = 'EMAIL ADDRESS' # Eg. user@gmail.com. Requires a valid SMTP-enabled email account (Eg. a Gmail acct with the SMTP settings below) 65 | email_password = 'EMAIL ACCOUNT PASSWORD' # Password for the email account 66 | smtp_server = 'smtp.gmail.com' 67 | smtp_port = 587 68 | 69 | # -------------------------------------------------------------------------------------------------------------------------- 70 | # Don't edit below this line - unless you know what you are doing :) 71 | # -------------------------------------------------------------------------------------------------------------------------- 72 | 73 | 74 | 75 | log = output_folder+"\daily_export_log.txt" 76 | # Create a log file if it doesn't exist 77 | print('', file=open(log, "a+")) 78 | 79 | # Date variables for later use 80 | today = datetime.datetime.today() 81 | yesterday = today-datetime.timedelta(1) 82 | 83 | # ------------------------------------------------------------- 84 | # REPORT GENERATION AND DOWNLOAD PROCESS 85 | 86 | # Initialise AGOL login by script 87 | print('--------------------------------------------------------------------------------------------------------------------------', file=open(log, "a")) 88 | print('--- STARTING REPORT GENERATION PROCESS ---', today, file=open(log, "a")) 89 | print('') 90 | print('Initialising session in AGOL', file=open(log, "a")) 91 | print('') 92 | agol_login = GIS(org, username, password) 93 | 94 | print('Reading Survey123 information for ID: ',surveyID, file=open(log, "a")) 95 | print('') 96 | surveymgr = SurveyManager(agol_login) 97 | survey = surveymgr.get(surveyID) 98 | # print('Templates available: ',survey.report_templates) # Return all available print templates for the survey 99 | # print('') 100 | 101 | template = survey.report_templates[report_template] 102 | print('Selected template: ',template, file=open(log, "a")) 103 | print('') 104 | 105 | reportCount = 0 106 | # Try/except/finally block to workaround the KeyError: 'results' bug in the generate_report method 107 | # (Waiting on ESRI to fix this bug.) 108 | try: 109 | print('Generating report(s) for submissions from last 24 hours', file=open(log, "a")) 110 | print('') 111 | ## Original Example: survey.generate_report(template, '1=1') #generates report for all features 112 | ## API Docu Example: survey.generate_report(report_template: arcgis.gis.Item, where: str = '1=1', utc_offset: str = '+00:00', report_title: str = None, folder_id: str = None) 113 | ## Our Example: survey.generate_report(template, '1=1', '+13:00', 'Test_Report_Export') 114 | survey.generate_report(template, where_filter, utc_offset, report_title) 115 | except Exception as e: 116 | print('>> ERROR: KeyError: ',e,' (related to ESRI BUG-000119057)', file=open(log, "a")) 117 | print('>> Continuing...', file=open(log, "a")) 118 | print('') 119 | pass 120 | finally: 121 | print('Downloading relevant report(s) to: ',output_folder, file=open(log, "a")) 122 | print('') 123 | # Find all Microsoft Word doc files in AGOL with "Survey 123" in the tags 124 | for x in survey.reports: 125 | # Find the creation date (Unix epoch) and convert to local time 126 | creationdate = datetime.datetime.fromtimestamp(x.created / 1e3) 127 | 128 | # Only find and download AGOL reports created in the last 24 hours 129 | # (this will download reports created manually, as well as ones generated by this script) 130 | if (creationdate > yesterday): 131 | # print('Created epoch ',x.created) # Uncomment datestamps below for testing 132 | # print('Created converted ',creationdate) 133 | # print('Today converted ',today) 134 | # print('Yesterday timedelta ',yesterday) 135 | reportCount += 1 136 | # Only return reports that contain the surveyID in the html code of our description 137 | # This should normally return reports generated with the generate_report() method 138 | if surveyID in x.description: 139 | print('Report desc: ',x.description, file=open(log, "a")) 140 | print('') 141 | id = x.id # Get ID of each Word doc AGOL item 142 | data_item = agol_login.content.get(id) 143 | data_item.download(save_path = output_folder) # Download each Word doc to specified location 144 | data_item.delete() # Delete each Word doc item in AGOL (after download finished/no longer required) 145 | # Finally block end 146 | print('REPORTS GENERATED: ',reportCount, file=open(log, "a")) 147 | print('--- REPORT GENERATION PROCESS - FINISHED ---', file=open(log, "a")) 148 | print('', file=open(log, "a")) 149 | 150 | 151 | 152 | 153 | 154 | 155 | # ------------------------------------------------------------- 156 | # EMAIL REPORT TO USERS PROCESS 157 | 158 | # Optional - email the report documents to specified email address. 159 | # Now we cycle through the new .docx reports in our output_folder, extract the user email address and send the attachment to the email that was collected with S123 160 | 161 | print('--- STARTING EMAIL PROCESS ---', file=open(log, "a")) 162 | sender = email_user 163 | documentCount = 0 164 | 165 | print('') 166 | print('Getting list of Word docx files in: ',output_folder, file=open(log, "a")) 167 | # Add all files ending with .docx to a new list 168 | file_list = glob.glob(output_folder+'\*.docx') 169 | print('Files:', file=open(log, "a")) 170 | for file_name in file_list: 171 | documentCount += 1 172 | print(file_name, file=open(log, "a")) 173 | 174 | print('') 175 | print('Reading raw table data from Word document(s)', file=open(log, "a")) 176 | print('') 177 | for file_name in file_list: 178 | #print(filename) 179 | # Here we read the tabular data from within our docx report - this is based on the report template that you have created... 180 | # In my case, there's a table with 10 rows, and row 8 happens to have the Email address that was collected by Survey123. 181 | docx_data = readDocxTables(file_name) 182 | 183 | # Data[7] happens to be our "Email" row (row 8) in the table within each docx template 184 | #print('Data7: ',data[7]) 185 | recipient = [str(v) for k,v in docx_data[7].items()][0] 186 | print('Sending email with attachment to recipient: ',recipient, file=open(log, "a")) 187 | 188 | # Initialise the email and create the enclosing (outer) message 189 | outer = MIMEMultipart() 190 | outer['Subject'] = 'Survey Report Attached ' + str(today) 191 | outer['To'] = recipient 192 | outer['From'] = sender 193 | outer.preamble = 'You will not see this in a MIME-aware mail reader.\n' 194 | msg_text = 'From ORGANISATION:\n\nPlease find the attached Survey Report from our recent visit or discussion. This is a copy of the information discussed and collected for your records.\n\nKind regards,\nYOUR ORGANISATION (email@org.com)\n\n\n\nNOTE: This is an automated message, please do not reply.' 195 | 196 | # Add the attachment to the message 197 | try: 198 | with open(file_name, 'rb') as fp: 199 | msg = MIMEBase('application', 'octet-stream') 200 | msg.set_payload(fp.read()) 201 | encoders.encode_base64(msg) 202 | msg.add_header('Content-Disposition', 'attachment', filename=os.path.basename(file_name)) 203 | outer.attach(msg) 204 | outer.attach(MIMEText(msg_text, 'plain')) # or 'html' 205 | except: 206 | print('Unable to open one of the attachments. Error: ', sys.exc_info()[0], file=open(log, "a")) 207 | raise 208 | 209 | composed = outer.as_string() 210 | 211 | # Send the email via SMTP - we're using Google SMTP servers below 212 | try: 213 | with smtplib.SMTP(smtp_server, smtp_port) as s: 214 | s.ehlo() 215 | s.starttls() 216 | s.ehlo() 217 | s.login(sender, email_password) 218 | s.sendmail(sender, recipient, composed) 219 | s.close() 220 | # Email sent, now let's remove the file so that it isn't sent again the following day 221 | os.remove(file_name) 222 | print('Email sent to recipient and removed file from download location.', file=open(log, "a")) 223 | print('') 224 | except: 225 | print('Unable to send the email. Error: ', sys.exc_info()[0], file=open(log, "a")) 226 | raise 227 | 228 | print('DOCUMENTS SENT TO RECIPIENTS: ',documentCount, file=open(log, "a")) 229 | print('--- EMAIL PROCESS - FINISHED --- ', file=open(log, "a")) 230 | print('', file=open(log, "a")) 231 | print('', file=open(log, "a")) 232 | 233 | 234 | 235 | 236 | 237 | # ------------------------------------------------------------- 238 | # OTHER FUNCTIONS 239 | 240 | # Function to retrieve raw text only 241 | def readText(filename): 242 | doc = docx.Document(filename) 243 | fullText = [] 244 | for para in doc.paragraphs: 245 | fullText.append(para.text) 246 | return '\n'.join(fullText) 247 | 248 | ## Function to retrieve text from tables 249 | def readDocxTables(filename): 250 | document = docx.Document(filename) 251 | table = document.tables[1] # 0 = logo, 1 = first block 252 | 253 | data = [] 254 | 255 | keys = None 256 | for i, row in enumerate(table.rows): 257 | text = (cell.text for cell in row.cells) 258 | 259 | if i == 0: 260 | keys = tuple(text) 261 | continue 262 | row_data = dict(zip(keys, text)) 263 | data.append(row_data) 264 | return(data) 265 | 266 | 267 | if __name__ == '__main__': 268 | main() 269 | --------------------------------------------------------------------------------