├── requirements.txt ├── app.py ├── pyproject.toml ├── scripts └── cfspflat ├── .gitignore ├── setup.cfg ├── LICENSE.txt ├── cfspflat ├── cli.py ├── __init__.py └── cf_dns.py ├── src └── cfspflat │ ├── cli.py │ ├── __init__.py │ └── cf_dns.py └── README.md /requirements.txt: -------------------------------------------------------------------------------- 1 | cloudflare>=3.1 2 | dnspython>=2.2.0 3 | netaddr>=0.8.0 4 | sender-policy-flattener>=0.3.1 5 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | from cfspflat.cli import main 4 | if __name__ == '__main__': 5 | sys.exit(main()) 6 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools>=45", 4 | "wheel" 5 | ] 6 | build-backend = "setuptools.build_meta" 7 | -------------------------------------------------------------------------------- /scripts/cfspflat: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | from cfspflat.cli import main 4 | if __name__ == '__main__': 5 | sys.exit(main()) 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *.json 3 | junk 4 | pack 5 | dist 6 | .aws 7 | .cloudflare.cfg 8 | *.egg-info 9 | .vscode 10 | spfenv 11 | venv 12 | result.html 13 | __pycache__/ 14 | .env 15 | config -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name=cfspflat 3 | version=24.12.17 4 | author = gunville 5 | author_email = rk13088@yahoo.com 6 | description = Flattens SPF records and updates Cloudflare zones 7 | long_description = file: README.md 8 | long_description_content_type = text/markdown 9 | license_files = LICENSE.txt 10 | url = https://github.com/Glocktober/cfspflat 11 | project_urls = 12 | repo = https://github.com/Glocktober/cfspflat.git 13 | overview = https://github.com/Glocktober/cfspflat/blob/master/README.md 14 | classifiers = 15 | Programming Language :: Python :: 3 16 | License :: OSI Approved :: MIT License 17 | Operating System :: OS Independent 18 | keywords='cloudflare dns spf' 19 | [options] 20 | include_package_data = True 21 | package_dir = 22 | = src 23 | packages = find: 24 | install_requires = 25 | cloudflare>=3.1 26 | sender-policy-flattener==0.3.1 27 | python_requires = >=3.7 28 | scripts = 29 | scripts/cfspflat 30 | [options.data_files] 31 | [options.packages.find] 32 | where = src 33 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Robert Kras 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /cfspflat/cli.py: -------------------------------------------------------------------------------- 1 | """ 2 | A script that crawls and compacts SPF records into IP networks. 3 | This helps to avoid exceeding the DNS lookup limit of the Sender Policy Framework (SPF) 4 | https://tools.ietf.org/html/rfc7208#section-4.6.4 5 | """ 6 | import json 7 | import argparse 8 | 9 | import cfspflat 10 | 11 | # noinspection PyMissingOrEmptyDocstring 12 | def parse_arguments(): 13 | parser = argparse.ArgumentParser(description=__doc__) 14 | parser.add_argument( 15 | "-c", 16 | "--config", 17 | dest="config", 18 | help="Name/path of JSON configuration file (default: spfs.json)", 19 | default='spfs.json', 20 | required=False, 21 | ) 22 | 23 | parser.add_argument( 24 | "-o", 25 | "--output", 26 | dest="output", 27 | help="Name/path of output file (default spf_sums.json)", 28 | default=None, 29 | required=False, 30 | ) 31 | 32 | parser.add_argument( 33 | "--update-records", 34 | dest='update', 35 | help="Update SPF records in CloudFlare", 36 | action="store_true", 37 | default=False, 38 | required=False, 39 | ) 40 | 41 | parser.add_argument( 42 | "--force-update", 43 | help="Force an update of SPF records in Cloudflare", 44 | action="store_true", 45 | dest='force_update', 46 | default=False, 47 | required=False, 48 | ) 49 | 50 | parser.add_argument( 51 | "--no-email", 52 | help="don't send the email", 53 | dest='sendemail', 54 | default=True, 55 | required=False, 56 | action="store_false", 57 | ) 58 | 59 | arguments = parser.parse_args() 60 | 61 | with open(arguments.config) as config: 62 | settings = json.load(config) 63 | arguments.resolvers = settings.get("resolvers",[]) 64 | arguments.toaddr = settings["email"]["to"] 65 | arguments.fromaddr = settings["email"]["from"] 66 | arguments.subject = settings["email"].get("subject", 67 | "[WARNING] SPF Records for {zone} have changed and should be updated.") 68 | arguments.update_subject = settings["email"].get("update_subject", 69 | "[NOTICE] SPF records for {zone} have been updated.") 70 | arguments.mailserver = settings["email"]["server"] 71 | arguments.domains = settings["sending domains"] 72 | if not arguments.output: 73 | arguments.output = settings.get("output", "spf_sums.json") 74 | 75 | if arguments.sendemail: 76 | required_non_config_args = all( 77 | [ 78 | arguments.toaddr, 79 | arguments.fromaddr, 80 | arguments.subject, 81 | arguments.update_subject, 82 | arguments.mailserver, 83 | arguments.domains, 84 | ] 85 | ) 86 | else: 87 | required_non_config_args = all([ 88 | arguments.domains 89 | ]) 90 | if not required_non_config_args: 91 | parser.print_help() 92 | exit() 93 | if "{zone}" not in arguments.subject: 94 | raise ValueError("Subject must contain {zone}") 95 | return arguments 96 | 97 | 98 | def main(): 99 | args = parse_arguments() 100 | cfspflat.main(args) 101 | 102 | 103 | if __name__ == "__main__": 104 | main() 105 | -------------------------------------------------------------------------------- /src/cfspflat/cli.py: -------------------------------------------------------------------------------- 1 | """ 2 | A script that crawls and compacts SPF records into IP networks. 3 | This helps to avoid exceeding the DNS lookup limit of the Sender Policy Framework (SPF) 4 | https://tools.ietf.org/html/rfc7208#section-4.6.4 5 | """ 6 | import json 7 | import argparse 8 | 9 | import cfspflat 10 | 11 | # noinspection PyMissingOrEmptyDocstring 12 | def parse_arguments(): 13 | parser = argparse.ArgumentParser(description=__doc__) 14 | parser.add_argument( 15 | "-c", 16 | "--config", 17 | dest="config", 18 | help="Name/path of JSON configuration file (default: spfs.json)", 19 | default='spfs.json', 20 | required=False, 21 | ) 22 | 23 | parser.add_argument( 24 | "-o", 25 | "--output", 26 | dest="output", 27 | help="Name/path of output file (default spf_sums.json)", 28 | default=None, 29 | required=False, 30 | ) 31 | 32 | parser.add_argument( 33 | "--update-records", 34 | dest='update', 35 | help="Update SPF records in CloudFlare", 36 | action="store_true", 37 | default=False, 38 | required=False, 39 | ) 40 | 41 | parser.add_argument( 42 | "--force-update", 43 | help="Force an update of SPF records in Cloudflare", 44 | action="store_true", 45 | dest='force_update', 46 | default=False, 47 | required=False, 48 | ) 49 | 50 | parser.add_argument( 51 | "--no-email", 52 | help="don't send the email", 53 | dest='sendemail', 54 | default=True, 55 | required=False, 56 | action="store_false", 57 | ) 58 | 59 | arguments = parser.parse_args() 60 | 61 | with open(arguments.config) as config: 62 | settings = json.load(config) 63 | arguments.resolvers = settings.get("resolvers",[]) 64 | arguments.toaddr = settings["email"]["to"] 65 | arguments.fromaddr = settings["email"]["from"] 66 | arguments.subject = settings["email"].get("subject", 67 | "[WARNING] SPF Records for {zone} have changed and should be updated.") 68 | arguments.update_subject = settings["email"].get("update_subject", 69 | "[NOTICE] SPF records for {zone} have been updated.") 70 | arguments.mailserver = settings["email"]["server"] 71 | arguments.domains = settings["sending domains"] 72 | if not arguments.output: 73 | arguments.output = settings.get("output", "spf_sums.json") 74 | 75 | if arguments.sendemail: 76 | required_non_config_args = all( 77 | [ 78 | arguments.toaddr, 79 | arguments.fromaddr, 80 | arguments.subject, 81 | arguments.update_subject, 82 | arguments.mailserver, 83 | arguments.domains, 84 | ] 85 | ) 86 | else: 87 | required_non_config_args = all([ 88 | arguments.domains 89 | ]) 90 | if not required_non_config_args: 91 | parser.print_help() 92 | exit() 93 | if "{zone}" not in arguments.subject: 94 | raise ValueError("Subject must contain {zone}") 95 | return arguments 96 | 97 | 98 | def main(): 99 | args = parse_arguments() 100 | cfspflat.main(args) 101 | 102 | 103 | if __name__ == "__main__": 104 | main() 105 | -------------------------------------------------------------------------------- /cfspflat/__init__.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import json 3 | from dns.resolver import Resolver 4 | from sender_policy_flattener.crawler import spf2ips 5 | from sender_policy_flattener.formatting import sequence_hash 6 | from sender_policy_flattener.email_utils import email_changes 7 | from .cf_dns import TXTrec 8 | 9 | if "FileNotFoundError" not in locals(): 10 | FileNotFoundError = IOError 11 | 12 | 13 | def flatten( 14 | input_records, 15 | dns_servers, 16 | email_server, 17 | email_subject, 18 | update_subject, 19 | fromaddress, 20 | toaddress, 21 | update=False, 22 | email=True, 23 | lastresult=None, 24 | force_update=False 25 | ): 26 | resolver = Resolver() 27 | if dns_servers: 28 | resolver.nameservers = dns_servers 29 | if lastresult is None: 30 | lastresult = dict() 31 | current = dict() 32 | for domain, spf_targets in input_records.items(): 33 | records = spf2ips(spf_targets, domain, resolver) 34 | hashsum = sequence_hash(records) 35 | current[domain] = {"sum": hashsum, "records": records} 36 | if lastresult.get(domain, False) and current.get(domain, False): 37 | previous_sum = lastresult[domain]["sum"] 38 | current_sum = current[domain]["sum"] 39 | mismatch = previous_sum != current_sum 40 | if mismatch: 41 | print(f'\n***WARNING: SPF changes detected for sender domain {domain}\n') 42 | else: 43 | print(f'\nNO SPF changes detected for sender domain {domain}\n') 44 | 45 | if mismatch and email: 46 | print(f'Sending mis-match details email for sender domain {domain}') 47 | if update or force_update: 48 | thesubject = update_subject 49 | else: 50 | thesubject = email_subject 51 | email_changes( 52 | zone=domain, 53 | prev_addrs=lastresult[domain]["records"], 54 | curr_addrs=current[domain]["records"], 55 | subject=thesubject, 56 | server=email_server, 57 | fromaddr=fromaddress, 58 | toaddr=toaddress, 59 | ) 60 | if (mismatch and update) or force_update: 61 | cfzone = TXTrec(domain) 62 | numrecs = len(records) 63 | print(f'\n**** Updating {numrecs} SPF Records for domain {domain}\n') 64 | for i in range(0,numrecs): 65 | recname = f'spf{i}.{domain}' 66 | print(f'===> Updating {recname} TXT record..', end='') 67 | if cfzone.update(recname, records[i],addok=True): 68 | print(f'..Successfully updated\n') 69 | else: 70 | print(f'Failed!\n\n********** WARNING: Update of {recname} TXT record Failed\n') 71 | 72 | 73 | return current if update or force_update or len(lastresult) == 0 else lastresult 74 | 75 | 76 | def main(args): 77 | previous_result = None 78 | try: 79 | with open(args.output) as prev_hashes: 80 | previous_result = json.load(prev_hashes) 81 | except FileNotFoundError as e: 82 | print(repr(e)) 83 | except Exception as e: 84 | print(repr(e)) 85 | finally: 86 | spf = flatten( 87 | input_records=args.domains, 88 | lastresult=previous_result, 89 | dns_servers=args.resolvers, 90 | email_server=args.mailserver, 91 | fromaddress=args.fromaddr, 92 | toaddress=args.toaddr, 93 | email_subject=args.subject, 94 | update_subject=args.update_subject, 95 | update=args.update, 96 | email=args.sendemail, 97 | force_update=args.force_update, 98 | ) 99 | with open(args.output, "w+") as f: 100 | json.dump(spf, f, indent=4, sort_keys=True) 101 | -------------------------------------------------------------------------------- /src/cfspflat/__init__.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import json 3 | from dns.resolver import Resolver 4 | from sender_policy_flattener.crawler import spf2ips 5 | from sender_policy_flattener.formatting import sequence_hash 6 | from sender_policy_flattener.email_utils import email_changes 7 | from .cf_dns import TXTrec 8 | 9 | if "FileNotFoundError" not in locals(): 10 | FileNotFoundError = IOError 11 | 12 | 13 | def flatten( 14 | input_records, 15 | dns_servers, 16 | email_server, 17 | email_subject, 18 | update_subject, 19 | fromaddress, 20 | toaddress, 21 | update=False, 22 | email=True, 23 | lastresult=None, 24 | force_update=False 25 | ): 26 | resolver = Resolver() 27 | if dns_servers: 28 | resolver.nameservers = dns_servers 29 | if lastresult is None: 30 | lastresult = dict() 31 | current = dict() 32 | for domain, spf_targets in input_records.items(): 33 | records = spf2ips(spf_targets, domain, resolver) 34 | hashsum = sequence_hash(records) 35 | current[domain] = {"sum": hashsum, "records": records} 36 | if lastresult.get(domain, False) and current.get(domain, False): 37 | previous_sum = lastresult[domain]["sum"] 38 | current_sum = current[domain]["sum"] 39 | mismatch = previous_sum != current_sum 40 | if mismatch: 41 | print(f'\n***WARNING: SPF changes detected for sender domain {domain}\n') 42 | else: 43 | print(f'\nNO SPF changes detected for sender domain {domain}\n') 44 | 45 | if mismatch and email: 46 | print(f'Sending mis-match details email for sender domain {domain}') 47 | if update or force_update: 48 | thesubject = update_subject 49 | else: 50 | thesubject = email_subject 51 | email_changes( 52 | zone=domain, 53 | prev_addrs=lastresult[domain]["records"], 54 | curr_addrs=current[domain]["records"], 55 | subject=thesubject, 56 | server=email_server, 57 | fromaddr=fromaddress, 58 | toaddr=toaddress, 59 | ) 60 | if (mismatch and update) or force_update: 61 | cfzone = TXTrec(domain) 62 | numrecs = len(records) 63 | print(f'\n**** Updating {numrecs} SPF Records for domain {domain}\n') 64 | for i in range(0,numrecs): 65 | recname = f'spf{i}.{domain}' 66 | print(f'===> Updating {recname} TXT record..', end='') 67 | if cfzone.update(recname, records[i],addok=True): 68 | print(f'..Successfully updated\n') 69 | else: 70 | print(f'Failed!\n\n********** WARNING: Update of {recname} TXT record Failed\n') 71 | 72 | 73 | return current if update or force_update or len(lastresult) == 0 else lastresult 74 | 75 | 76 | def main(args): 77 | previous_result = None 78 | try: 79 | with open(args.output) as prev_hashes: 80 | previous_result = json.load(prev_hashes) 81 | except FileNotFoundError as e: 82 | print(repr(e)) 83 | except Exception as e: 84 | print(repr(e)) 85 | finally: 86 | spf = flatten( 87 | input_records=args.domains, 88 | lastresult=previous_result, 89 | dns_servers=args.resolvers, 90 | email_server=args.mailserver, 91 | fromaddress=args.fromaddr, 92 | toaddress=args.toaddr, 93 | email_subject=args.subject, 94 | update_subject=args.update_subject, 95 | update=args.update, 96 | email=args.sendemail, 97 | force_update=args.force_update, 98 | ) 99 | with open(args.output, "w+") as f: 100 | json.dump(spf, f, indent=4, sort_keys=True) 101 | -------------------------------------------------------------------------------- /cfspflat/cf_dns.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | from cloudflare import Cloudflare 4 | import tomllib 5 | 6 | 7 | class CFzone: 8 | """ CloudFlare dns zone """ 9 | 10 | def __init__(self, cf_domain, ): 11 | 12 | api_email = os.environ.get('CLOUDFLARE_EMAIL') 13 | api_key = os.environ.get('CLOUDFLARE_API_KEY') 14 | api_token = os.environ.get('CLOUDFLARE_API_TOKEN') 15 | cf_file = Path('.cloudflare.cf') 16 | if (not cf_file.exists()): 17 | cf_file = Path.home().joinpath(cf_file) 18 | if (cf_file.exists()): 19 | with open(cf_file, 'rb') as f: 20 | auth_data = tomllib.load(f) 21 | if ('CloudFlare' in auth_data): 22 | api_email = auth_data['CloudFlare'].get('email', None) 23 | api_key = auth_data['CloudFlare'].get('api_key', None) 24 | api_token = auth_data['CloudFlare'].get('api_token', None) 25 | 26 | self._domain = cf_domain 27 | self._cf = Cloudflare(api_email=api_email, api_key=api_key, api_token=api_token) 28 | 29 | zone_info = self.get_zoneid(cf_domain) 30 | if not zone_info: 31 | emes = f'Can\'t Find a CloudFlare zone for {cf_domain}' 32 | raise Exception(emes) 33 | self.zoneid = zone_info.id 34 | self.zonename = zone_info.name 35 | 36 | 37 | def get_zoneid(self,fqdn): 38 | """ From the fqdn find the CloudFlare zone id """ 39 | 40 | fparts = fqdn.split('.') 41 | while(fparts): 42 | r = self._cf.zones.list(match='all', name='.'.join(fparts)) 43 | if r and len(r.result) == 1: 44 | return r.result[0] 45 | fparts = fparts[1:] 46 | return None 47 | 48 | 49 | def create(self,params): 50 | """ Add a record, return the record id """ 51 | 52 | r = self._cf.dns.records.create(zone_id=self.zoneid, **params) 53 | return r.id 54 | 55 | 56 | def get(self,params={}): 57 | """ Get a resouce record """ 58 | 59 | r = self._cf.dns.records.list(zone_id=self.zoneid, **params) 60 | return r 61 | 62 | 63 | def getid(self,params={}): 64 | """ Return a specific resource record values: id, proxied, ttl """ 65 | 66 | r = self.get(params) 67 | recs = 0 if (not r) else len(r.result) 68 | if recs == 1: 69 | rr = r.result[0] 70 | return rr.id, rr.proxied, rr.ttl 71 | elif recs == 0: 72 | return 0, False, 0 73 | else: 74 | raise Exception('Multiple records found') 75 | 76 | 77 | def set(self,rid,params={}): 78 | """ Set (update) a specific record id """ 79 | 80 | r = self._cf.dns.records.update(dns_record_id=rid, zone_id=self.zoneid, **params) 81 | 82 | return r.id 83 | 84 | 85 | def delete(self,rid): 86 | """ Delete a DNS record """ 87 | 88 | r = self._cf.zones.dns_records.delete(dns_record_id=rid, zone_id=self.zoneid) 89 | return r.id 90 | 91 | 92 | 93 | class CFrec: 94 | """ CloudFlare DNS record abstraction """ 95 | 96 | def __init__(self, domain, type='A', ttl=1): 97 | 98 | self.type = type 99 | self.ttl = ttl 100 | self.zone = CFzone(domain) 101 | self.zonename = self.zone.zonename 102 | 103 | 104 | def update(self, name, contents, addok=False): 105 | """ set (update or create) resource record value """ 106 | 107 | fqdn = name if name.endswith(self.zonename) else f'{name}.{self.zonename}' 108 | 109 | rid, proxied, ttl = self.zone.getid({'name':fqdn,'type': self.type, 'match': 'all'}) 110 | if rid: 111 | # update the existing resource record 112 | return self.zone.set(rid, { 113 | 'name': fqdn, 114 | 'type': self.type, 115 | 'proxied': proxied, 116 | 'ttl' : ttl, 117 | 'content': contents 118 | } 119 | ) 120 | elif addok: 121 | # rr doesn't exist be we can create a new record 122 | return self.zone.create({ 123 | 'name': fqdn, 124 | 'type': self.type, 125 | 'ttl' : self.ttl, 126 | 'content': contents 127 | } 128 | ) 129 | else: 130 | # resource record doesn't exist 131 | return None 132 | 133 | 134 | def add(self, name, contents): 135 | """ Add a new record """ 136 | 137 | fqdn = name if name.endswith(self.zonename) else f'{name}.{self.zonename}' 138 | 139 | return self.zone.create({'name': fqdn, 'type': self.type, 'content': contents}) 140 | 141 | 142 | def get(self, name): 143 | """ get contents (value) of a resource record """ 144 | 145 | fqdn = name if name.endswith(self.zonename) else f'{name}.{self.zonename}' 146 | 147 | r = self.zone.get({'name': fqdn, 'type': self.type, 'match': 'all'}) 148 | 149 | recs = len(r) 150 | if recs == 1: 151 | return r[0]['content'] 152 | elif recs == 0: 153 | return None 154 | else: 155 | raise Exception("Multiple records found") 156 | 157 | 158 | def rem(self, name): 159 | """ remove a TXT record """ 160 | 161 | fqdn = name if name.endswith(self.zonename) else f'{name}.{self.zonename}' 162 | 163 | rid, _, _ = self.zone.getid({'name':fqdn,'type': self.type, 'match': 'all'}) 164 | if rid: 165 | # remove the record 166 | return self.zone.delete(rid) 167 | else: 168 | # couldn't find the record 169 | return False 170 | 171 | 172 | class TXTrec(CFrec): 173 | """ CF TXT Record """ 174 | 175 | def __init__(self, domain): 176 | 177 | super().__init__(domain, 'TXT') 178 | 179 | -------------------------------------------------------------------------------- /src/cfspflat/cf_dns.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | from cloudflare import Cloudflare 4 | import tomllib 5 | 6 | 7 | class CFzone: 8 | """ CloudFlare dns zone """ 9 | 10 | def __init__(self, cf_domain, ): 11 | 12 | api_email = os.environ.get('CLOUDFLARE_EMAIL') 13 | api_key = os.environ.get('CLOUDFLARE_API_KEY') 14 | api_token = os.environ.get('CLOUDFLARE_API_TOKEN') 15 | cf_file = Path('.cloudflare.cf') 16 | if (not cf_file.exists()): 17 | cf_file = Path.home().joinpath(cf_file) 18 | if (cf_file.exists()): 19 | with open(cf_file, 'rb') as f: 20 | auth_data = tomllib.load(f) 21 | if ('CloudFlare' in auth_data): 22 | api_email = auth_data['CloudFlare'].get('email', None) 23 | api_key = auth_data['CloudFlare'].get('api_key', None) 24 | api_token = auth_data['CloudFlare'].get('api_token', None) 25 | 26 | self._domain = cf_domain 27 | self._cf = Cloudflare(api_email=api_email, api_key=api_key, api_token=api_token) 28 | 29 | zone_info = self.get_zoneid(cf_domain) 30 | if not zone_info: 31 | emes = f'Can\'t Find a CloudFlare zone for {cf_domain}' 32 | raise Exception(emes) 33 | self.zoneid = zone_info.id 34 | self.zonename = zone_info.name 35 | 36 | 37 | def get_zoneid(self,fqdn): 38 | """ From the fqdn find the CloudFlare zone id """ 39 | 40 | fparts = fqdn.split('.') 41 | while(fparts): 42 | r = self._cf.zones.list(match='all', name='.'.join(fparts)) 43 | if r and len(r.result) == 1: 44 | return r.result[0] 45 | fparts = fparts[1:] 46 | return None 47 | 48 | 49 | def create(self,params): 50 | """ Add a record, return the record id """ 51 | 52 | r = self._cf.dns.records.create(zone_id=self.zoneid, **params) 53 | return r.id 54 | 55 | 56 | def get(self,params={}): 57 | """ Get a resouce record """ 58 | 59 | r = self._cf.dns.records.list(zone_id=self.zoneid, **params) 60 | return r 61 | 62 | 63 | def getid(self,params={}): 64 | """ Return a specific resource record values: id, proxied, ttl """ 65 | 66 | r = self.get(params) 67 | recs = 0 if (not r) else len(r.result) 68 | if recs == 1: 69 | rr = r.result[0] 70 | return rr.id, rr.proxied, rr.ttl 71 | elif recs == 0: 72 | return 0, False, 0 73 | else: 74 | raise Exception('Multiple records found') 75 | 76 | 77 | def set(self,rid,params={}): 78 | """ Set (update) a specific record id """ 79 | 80 | r = self._cf.dns.records.update(dns_record_id=rid, zone_id=self.zoneid, **params) 81 | 82 | return r.id 83 | 84 | 85 | def delete(self,rid): 86 | """ Delete a DNS record """ 87 | 88 | r = self._cf.zones.dns_records.delete(dns_record_id=rid, zone_id=self.zoneid) 89 | return r.id 90 | 91 | 92 | 93 | class CFrec: 94 | """ CloudFlare DNS record abstraction """ 95 | 96 | def __init__(self, domain, type='A', ttl=1): 97 | 98 | self.type = type 99 | self.ttl = ttl 100 | self.zone = CFzone(domain) 101 | self.zonename = self.zone.zonename 102 | 103 | 104 | def update(self, name, contents, addok=False): 105 | """ set (update or create) resource record value """ 106 | 107 | fqdn = name if name.endswith(self.zonename) else f'{name}.{self.zonename}' 108 | 109 | rid, proxied, ttl = self.zone.getid({'name':fqdn,'type': self.type, 'match': 'all'}) 110 | if rid: 111 | # update the existing resource record 112 | return self.zone.set(rid, { 113 | 'name': fqdn, 114 | 'type': self.type, 115 | 'proxied': proxied, 116 | 'ttl' : ttl, 117 | 'content': contents 118 | } 119 | ) 120 | elif addok: 121 | # rr doesn't exist be we can create a new record 122 | return self.zone.create({ 123 | 'name': fqdn, 124 | 'type': self.type, 125 | 'ttl' : self.ttl, 126 | 'content': contents 127 | } 128 | ) 129 | else: 130 | # resource record doesn't exist 131 | return None 132 | 133 | 134 | def add(self, name, contents): 135 | """ Add a new record """ 136 | 137 | fqdn = name if name.endswith(self.zonename) else f'{name}.{self.zonename}' 138 | 139 | return self.zone.create({'name': fqdn, 'type': self.type, 'content': contents}) 140 | 141 | 142 | def get(self, name): 143 | """ get contents (value) of a resource record """ 144 | 145 | fqdn = name if name.endswith(self.zonename) else f'{name}.{self.zonename}' 146 | 147 | r = self.zone.get({'name': fqdn, 'type': self.type, 'match': 'all'}) 148 | 149 | recs = len(r) 150 | if recs == 1: 151 | return r[0]['content'] 152 | elif recs == 0: 153 | return None 154 | else: 155 | raise Exception("Multiple records found") 156 | 157 | 158 | def rem(self, name): 159 | """ remove a TXT record """ 160 | 161 | fqdn = name if name.endswith(self.zonename) else f'{name}.{self.zonename}' 162 | 163 | rid, _, _ = self.zone.getid({'name':fqdn,'type': self.type, 'match': 'all'}) 164 | if rid: 165 | # remove the record 166 | return self.zone.delete(rid) 167 | else: 168 | # couldn't find the record 169 | return False 170 | 171 | 172 | class TXTrec(CFrec): 173 | """ CF TXT Record """ 174 | 175 | def __init__(self, domain): 176 | 177 | super().__init__(domain, 'TXT') 178 | 179 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Cloudflare SPF Flattener 2 | ==================== 3 | 4 | ## Finding sender-policy-flattener (spflat) 5 | 6 | A few weeks ago one of our SaaS vendors added a couple of additional DNS entries to their SPF record (without telling us...). 7 | 8 | We started to get reports of email deliver failures. Looking into it we discovered the number of DNS entries in our SPF record was the problem. The SaaS vendors change put us over the maximum of 10. 9 | 10 | We've decided future SaaS vendors and partners will send from a subdomain, but for existing ones there are too many actors - both internal and external to accomplish this change any time soon. 11 | 12 | Hence we started researching SPF flattening as the best solution, but were concerned about the vendors making changes to the sender records. 13 | 14 | Looking for a solution to that probkem we stumbled upon the [sender-policy-flattener](https://github.com/cetanu/sender_policy_flattener) project. This tool - `spflat` - solved many problems and has some excellent features and characteristics. 15 | * The project is opensource 16 | * It will query your list of approved sendors and flatten these into a list of appropriately formatted and sized spf records composed of `ip4` and `ip6` spf entries. 17 | * Provide these entries as a chain of spf `include` entries. 18 | * Should any of your the senders dns entries change, it sends email indicating the changes and the the corrections required to your spf records. 19 | * `spflat` works reliably and accurately 20 | * Can run entirely from a JSON configuration file. 21 | * Written in Python, and easy to install and configure. 22 | 23 | `sendier-policy-flattner` did ***90%*** of what we desired for flattening and managing our SPF records. 24 | 25 | ## Requirements that led to this project 26 | After using `splat` for a bit we come up with a wish list - the other ***10%*** of what we needed. 27 | 28 | Here is the wishlist of requirements we came up with: 29 | * The ability to run `spflat` in the command line, without the email being sent. (a `--no-email` switch) 30 | * Our public DNS zones are in Cloudflare. We wanted `spflat` to **AUTOMATICALLY** update the necessary resource records in Cloudflare when a change is detected. (a `--update` switch). 31 | * Ability to have `spflat` re-create the SPF zone records in Cloudflare DNS for zone repair (and initial configuration.) (a `--force-update` switch) 32 | * Have the output status file (sums file) update ONLY if the zones are updated. 33 | 34 | * original `spflat` replaces the sums file each time it ran, so in monitoring mode a change was reported only once. 35 | 36 | * `sender-policy-flattener` is a fine tool, so we did not want to reinvent the wheel: 37 | 38 | * Utilized `spflat` by supplementing its functionality. 39 | 40 | * A supplement, not a replacement. 41 | 42 | * Keep the changes opensource: 43 | * Recognize the debt to `sender-policy-flattener` 44 | * Make the changes avialable under the same licenses. 45 | 46 | * To limit confusion, change the command from `spflat` to `cfspflat` 47 | 48 | Hence `cfspflat` - ***CloudFlare Sender Policy Flattener*** was created. 49 | 50 | ## `cfspflat` - Cloudflare Sender Policy Flattener 51 | 52 | Quick overview: 53 | ```bash 54 | % cfspflat -h 55 | usage: cfspflat [-h] [-c CONFIG] [-o OUTPUT] [--update-records] [--force-update] 56 | [--no-email] 57 | 58 | A script that crawls and compacts SPF records into IP networks. This helps to 59 | avoid exceeding the DNS lookup limit of the Sender Policy Framework (SPF) 60 | https://tools.ietf.org/html/rfc7208#section-4.6.4 61 | 62 | optional arguments: 63 | -h, --help show this help message and exit 64 | -c CONFIG, --config CONFIG 65 | Name/path of JSON configuration file (default: 66 | spfs.json) 67 | -o OUTPUT, --output OUTPUT 68 | Name/path of output file (default spf_sums.json) 69 | --update-records Update SPF records in CloudFlare 70 | --force-update Force an update of SPF records in Cloudflare 71 | --no-email don't send the email 72 | ``` 73 | * The `sender-policy-flattener` module is installed as part of `cfspflat` 74 | * The existing core of `spflat` is kept mostly intact, so the basic features are maintained by `cfspflat`. 75 | * The changes to accomodate `cfspflat` were in the parameter handling and adding the Cloudflare updates to the processing look. 76 | * The `cloudflare` library, with some abstraction classes, is used to make the zone updates. 77 | * `cfspflat` eliminates many of the command arguments of spflat in favor of using the json config file. 78 | * Cloudflare TXT records are automatically generated and updated when the configuration changes. 79 | * With `cfspflat` you can completely automate your SPF flattening using cfspflat with the `--update` switch in a cron job, even silently with the `--no-email` switch 80 | 81 | ## Installing and Configuring cfspflat 82 | 83 | ### 1. pip install the cfspflat 84 | ```bash 85 | % pip install cfspflat 86 | ``` 87 | * But it's advisable to do this in its own venv: 88 | ```bash 89 | % python3 -m venv spfenv 90 | % source spfenv/bin/activate 91 | % pip install cfspflat 92 | ``` 93 | * pip will install the prerequisites, including the `sender-policy-flattner` (spflat), `dnspython`, `netaddr`, and `cloudflare` python modules. 94 | * The executable is installed in bin of the venv as `cfspflat` 95 | ### 2. Create an anchor SPF record for the zone apex in Cloudflare 96 | 97 | Create the TXT SPF record on zone apex used (e.g. example.com), At the end of this anchor record include the first SPF record that slpat will write - spf0.example.com 98 | * we also include our own `ip4` and `ip6` entries in this anchor record. 99 | ``` 100 | example.com TXT "v=spf1 mx include:spf0.example.com -all" 101 | ``` 102 | * This anchor record is never changed by `cfspflat`. It's purpose is to link to the first SPF record in the chain that `cfspflat` manages. 103 | 104 | ### 2. Edit the cfspflat configuration file 105 | Create a spfs.json file. Add all the entries required: 106 | * `cfspflat` uses the same configuration and sums file formats as the original `spflat`. 107 | * If you already use spflat you can use thos files as is with cfspflat. 108 | * There is one extension - the "update_subject" entry containing the subject of the email sent when cfspflat has updated your SPF records. This message will contain the same detail spflat provides. 109 | * `spfs.json` is the default name of the config file, but it can be specified with the `--config` switch. 110 | * Here is an example config file: 111 | #### Example spfs.json configuration file: 112 | ```json 113 | { 114 | "sending domains": { 115 | "example.edu": { 116 | "amazonses.com": "txt", 117 | "goodwebsolutions.net": "txt", 118 | .... more sender spf's here .... 119 | "delivery.xyz.com": "txt", 120 | "spf.protection.outlook.com": "txt" 121 | } 122 | }, 123 | "resolvers": [ 124 | "1.1.1.1", "8.8.8.8" 125 | ], 126 | "email": { 127 | "to": "dnsadmins@example.com", 128 | "from": "spf_monitor@example.com", 129 | "subject": "[WARNING] SPF Records for {zone} have changed and should be updated.", 130 | "update_subject" : "[NOTICE] SPF Records for {zone} have been updated.", 131 | "server": "smtp.example.com" 132 | }, 133 | "output": "monitor_sums.json" 134 | } 135 | ``` 136 | #### Config file details 137 | * The `sending domains` section is **required** and contains sending domain entries: this is your sender domain (e.g. example.com for j.smith@example.com, noreply.example.com for deals@noreply.example.com ) There can be multiple sending domains in the config file. 138 | * Each sending domain contains dns spf records for the dns `include` records of your approved senders.These dns names are resolved and flattened: 139 | * These entries are in the key-value pairs of : . 140 | * Record type can be "txt" (other SPF records), "A" and "AAAA" records (for specific hosts). 141 | * The `resolvers` section is **optional** (using the system default DNS resolvers if none are supplied) 142 | * The `email` stanza is **required** and is global (i.e. for all `sending domains`). This section includes: 143 | * `subject` **(optional)** is the email subject if a change was detected but no updates were made. The default message is the one shown in the example. 144 | * `update_subject` **(optional)** is the email subject if a change was detected and the dns records were updated. The default message is shown in the example. 145 | * `to` - is **required** - this is the destination for emails sent by `cfspflat` 146 | * `from` - is **required** - the source email of the messages `cfspflat` sends 147 | * `server` - is **required** - your mail relay. 148 | * `output` is the file that maintains the existing state and checksum of sender records. If this is not specified `spfs_sum.json` is used. 149 | #### Output file details 150 | * The `output` file is a JSON file only updated if it is new (empty) or the records have been updated. 151 | * Likewise the default output file is `spf_sums.json` but can be changed in the config file or by the `--output` switch. 152 | * This contains the list of flattened spf records and a checksum used to assess changes in senders records. 153 | * Because you recieve emails of detected changes or updates, there is little reason to care about the output file. 154 | ### 3. Create a credentials file for Cloudflare 155 | There are a couple of locations and formats for the API credentials Cloudfare requires. 156 | * Consult cloudflare documentation and the dashboard for creating API credentials 157 | * Consult the python-cloudflare site for documentation on passing credentials. 158 | * For simplicity you can use the `.cloudflare.cfg` file in either your home directory or in the directory you will run cfspflat. 159 | ``` 160 | [CloudFlare] 161 | email = "dnsadmin@example.org" 162 | api_key = "1234567890abc...abc" 163 | # or alternatively you can use a API token 164 | api_token = "your-generated..API-token" 165 | ``` 166 | * It should go without saying - protect the API keys and this file. 167 | * It's also possible to pass the credentials as environment variables (`CLOUDFLARE_EMAIL`, `CLOUDFLARE_API_KEY`, `CLOUDFLARE_API_TOKEN`). 168 | 169 | ### 4. Run `cfspflat` to build the sums file and SPF entries 170 | * Run cfspflat twice: 171 | ```bash 172 | % cfspflat --no-email 173 | % cfspflat --force 174 | ``` 175 | * The first time constructs the base records and the second time forces the dns updates. 176 | * With force update the DNS records are created even if a change hasn't been detected. 177 | * A list of the records will be sent to your email. 178 | 179 | ### 5. Automate `cfspflat` to the level you are comfortable with 180 | * You are up and running: 181 | * You can run `cfspflat` in advisory mode (like `spflat`) sending you emails notifying of changes 182 | * Or you can run it with the `--update-records` switch and update your records automatically whenever they change (still giving you notifications of the changes made.) 183 | 184 | Example email format 185 | -------------------- 186 | * Example from `sender-policy-flattener` README: 187 | 188 | example screenshot 189 | --------------------------------------------------------------------------------