├── .dir-locals.el ├── .gitignore ├── .pylintrc ├── .travis.yml ├── LICENSE.TXT ├── README.md ├── RULES.md ├── USAGE.md ├── VALIDATION.md ├── code_legacy ├── PostfixConfigGenerator.py ├── PostfixLogSummary.py └── TestPostfixConfigGenerator.py ├── policy.json ├── policy.json.asc ├── schema └── policy-0.1.schema.json ├── scripts ├── starttls-policy.cron.d ├── update_and_verify.sh └── verify.sh ├── share ├── golden-domains.txt └── google-starttls-domains.csv ├── test_rules ├── bigger_test_config.json └── config.json ├── tests ├── policy_test.py ├── test-requirements.txt └── validate_signature.sh └── tools ├── CheckSTARTTLS.py └── ProcessGoogleSTARTTLSDomains.py /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ;; emacs local configuration settings for starttls source 2 | ((js-mode 3 | (indent-tabs-mode . nil) 4 | (js-indent-level . 2))) 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.orig 2 | *.pyc 3 | *~ 4 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # use as many jobs as there are cores 4 | jobs=0 5 | 6 | # Specify a configuration file. 7 | #rcfile= 8 | 9 | # Python code to execute, usually for sys.path manipulation such as 10 | # pygtk.require(). 11 | #init-hook= 12 | 13 | # Profiled execution. 14 | profile=no 15 | 16 | # Add files or directories to the blacklist. They should be base names, not 17 | # paths. 18 | ignore=CVS 19 | 20 | # Pickle collected data for later comparisons. 21 | persistent=yes 22 | 23 | # Ignore pycurl, since no source access 24 | extension-pkg-whitelist=pycurl 25 | 26 | 27 | [MESSAGES CONTROL] 28 | 29 | # Enable the message, report, category or checker with the given id(s). You can 30 | # either give multiple identifier separated by comma (,) or put this option 31 | # multiple time. See also the "--disable" option for examples. 32 | #enable= 33 | 34 | # Disable the message, report, category or checker with the given id(s). You 35 | # can either give multiple identifiers separated by comma (,) or put this 36 | # option multiple times (only on the command line, not in the configuration 37 | # file where it should appear only once).You can also use "--disable=all" to 38 | # disable everything first and then reenable specific checks. For example, if 39 | # you want to run only the similarities checker, you can use "--disable=all 40 | # --enable=similarities". If you want to run only the classes checker, but have 41 | # no Warning level messages displayed, use"--disable=all --enable=classes 42 | # --disable=W" 43 | disable=fixme,locally-disabled,abstract-class-not-used,abstract-class-little-used,bad-continuation,too-few-public-methods,no-self-use,invalid-name,too-many-instance-attributes,cyclic-import,duplicate-code 44 | # abstract-class-not-used cannot be disabled locally (at least in 45 | # pylint 1.4.1), same for abstract-class-little-used 46 | 47 | 48 | [REPORTS] 49 | 50 | # Set the output format. Available formats are text, parseable, colorized, msvs 51 | # (visual studio) and html. You can also give a reporter class, eg 52 | # mypackage.mymodule.MyReporterClass. 53 | output-format=text 54 | 55 | # Put messages in a separate file for each module / package specified on the 56 | # command line instead of printing them on stdout. Reports (if any) will be 57 | # written in a file name "pylint_global.[txt|html]". 58 | files-output=no 59 | 60 | # Tells whether to display a full report or only the messages 61 | reports=yes 62 | 63 | # Python expression which should return a note less than 10 (10 is the highest 64 | # note). You have access to the variables errors warning, statement which 65 | # respectively contain the number of errors / warnings messages and the total 66 | # number of statements analyzed. This is used by the global evaluation report 67 | # (RP0004). 68 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 69 | 70 | # Add a comment according to your evaluation note. This is used by the global 71 | # evaluation report (RP0004). 72 | comment=no 73 | 74 | # Template used to display messages. This is a python new-style format string 75 | # used to format the message information. See doc for all details 76 | #msg-template= 77 | 78 | 79 | [BASIC] 80 | 81 | # Required attributes for module, separated by a comma 82 | required-attributes= 83 | 84 | # List of builtins function names that should not be used, separated by a comma 85 | bad-functions=map,filter,apply,input,file 86 | 87 | # Good variable names which should always be accepted, separated by a comma 88 | good-names=f,i,j,k,ex,Run,_,fd,logger 89 | 90 | # Bad variable names which should always be refused, separated by a comma 91 | bad-names=foo,bar,baz,toto,tutu,tata 92 | 93 | # Colon-delimited sets of names that determine each other's naming style when 94 | # the name regexes allow several styles. 95 | name-group= 96 | 97 | # Include a hint for the correct naming format with invalid-name 98 | include-naming-hint=no 99 | 100 | # Regular expression matching correct function names 101 | function-rgx=[a-z_][a-z0-9_]{2,40}$ 102 | 103 | # Naming hint for function names 104 | function-name-hint=[a-z_][a-z0-9_]{2,40}$ 105 | 106 | # Regular expression matching correct variable names 107 | variable-rgx=[a-z_][a-z0-9_]{1,30}$ 108 | 109 | # Naming hint for variable names 110 | variable-name-hint=[a-z_][a-z0-9_]{2,30}$ 111 | 112 | # Regular expression matching correct constant names 113 | const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 114 | 115 | # Naming hint for constant names 116 | const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 117 | 118 | # Regular expression matching correct attribute names 119 | attr-rgx=[a-z_][a-z0-9_]{2,30}$ 120 | 121 | # Naming hint for attribute names 122 | attr-name-hint=[a-z_][a-z0-9_]{2,30}$ 123 | 124 | # Regular expression matching correct argument names 125 | argument-rgx=[a-z_][a-z0-9_]{2,30}$ 126 | 127 | # Naming hint for argument names 128 | argument-name-hint=[a-z_][a-z0-9_]{2,30}$ 129 | 130 | # Regular expression matching correct class attribute names 131 | class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 132 | 133 | # Naming hint for class attribute names 134 | class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 135 | 136 | # Regular expression matching correct inline iteration names 137 | inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ 138 | 139 | # Naming hint for inline iteration names 140 | inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ 141 | 142 | # Regular expression matching correct class names 143 | class-rgx=[A-Z_][a-zA-Z0-9]+$ 144 | 145 | # Naming hint for class names 146 | class-name-hint=[A-Z_][a-zA-Z0-9]+$ 147 | 148 | # Regular expression matching correct module names 149 | module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 150 | 151 | # Naming hint for module names 152 | module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 153 | 154 | # Regular expression matching correct method names 155 | method-rgx=[a-z_][a-z0-9_]{2,50}$ 156 | 157 | # Naming hint for method names 158 | method-name-hint=[a-z_][a-z0-9_]{2,50}$ 159 | 160 | # Regular expression which should only match function or class names that do 161 | # not require a docstring. 162 | no-docstring-rgx=(__.*__)|(test_[A-Za-z0-9_]*)|(_.*)|(.*Test$) 163 | 164 | # Minimum line length for functions/classes that require docstrings, shorter 165 | # ones are exempt. 166 | docstring-min-length=-1 167 | 168 | 169 | [MISCELLANEOUS] 170 | 171 | # List of note tags to take in consideration, separated by a comma. 172 | notes=FIXME,XXX,TODO 173 | 174 | 175 | [LOGGING] 176 | 177 | # Logging modules to check that the string format arguments are in logging 178 | # function parameter format 179 | logging-modules=logging,logger 180 | 181 | 182 | [VARIABLES] 183 | 184 | # Tells whether we should check for unused import in __init__ files. 185 | init-import=no 186 | 187 | # A regular expression matching the name of dummy variables (i.e. expectedly 188 | # not used). 189 | dummy-variables-rgx=(unused)?_.*|dummy 190 | 191 | # List of additional names supposed to be defined in builtins. Remember that 192 | # you should avoid to define new builtins when possible. 193 | additional-builtins= 194 | 195 | 196 | [SIMILARITIES] 197 | 198 | # Minimum lines number of a similarity. 199 | min-similarity-lines=6 200 | 201 | # Ignore comments when computing similarities. 202 | ignore-comments=yes 203 | 204 | # Ignore docstrings when computing similarities. 205 | ignore-docstrings=yes 206 | 207 | # Ignore imports when computing similarities. 208 | ignore-imports=yes 209 | 210 | 211 | [FORMAT] 212 | 213 | # Maximum number of characters on a single line. 214 | max-line-length=100 215 | 216 | # Regexp for a line that is allowed to be longer than the limit. 217 | ignore-long-lines=^\s*(# )??$ 218 | 219 | # Allow the body of an if to be on the same line as the test if there is no 220 | # else. 221 | single-line-if-stmt=no 222 | 223 | # List of optional constructs for which whitespace checking is disabled 224 | no-space-check=trailing-comma 225 | 226 | # Maximum number of lines in a module 227 | max-module-lines=1250 228 | 229 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 230 | # tab). 231 | indent-string=' ' 232 | 233 | # Number of spaces of indent required inside a hanging or continued line. 234 | # This does something silly/broken... 235 | #indent-after-paren=4 236 | 237 | 238 | [TYPECHECK] 239 | 240 | # Tells whether missing members accessed in mixin class should be ignored. A 241 | # mixin class is detected if its name ends with "mixin" (case insensitive). 242 | ignore-mixin-members=yes 243 | 244 | # List of module names for which member attributes should not be checked 245 | # (useful for modules/projects where namespaces are manipulated during runtime 246 | # and thus existing member attributes cannot be deduced by static analysis 247 | ignored-modules=pkg_resources,confargparse,argparse,six.moves,six.moves.urllib 248 | # import errors ignored only in 1.4.4 249 | # https://bitbucket.org/logilab/pylint/commits/cd000904c9e2 250 | 251 | # List of classes names for which member attributes should not be checked 252 | # (useful for classes with attributes dynamically set). 253 | ignored-classes=SQLObject 254 | 255 | # When zope mode is activated, add a predefined set of Zope acquired attributes 256 | # to generated-members. 257 | zope=yes 258 | 259 | # List of members which are set dynamically and missed by pylint inference 260 | # system, and so shouldn't trigger E0201 when accessed. Python regular 261 | # expressions are accepted. 262 | generated-members=REQUEST,acl_users,aq_parent 263 | 264 | 265 | [IMPORTS] 266 | 267 | # Deprecated modules which should not be used, separated by a comma 268 | deprecated-modules=regsub,TERMIOS,Bastion,rexec 269 | 270 | # Create a graph of every (i.e. internal and external) dependencies in the 271 | # given file (report RP0402 must not be disabled) 272 | import-graph= 273 | 274 | # Create a graph of external dependencies in the given file (report RP0402 must 275 | # not be disabled) 276 | ext-import-graph= 277 | 278 | # Create a graph of internal dependencies in the given file (report RP0402 must 279 | # not be disabled) 280 | int-import-graph= 281 | 282 | 283 | [CLASSES] 284 | 285 | # List of interface methods to ignore, separated by a comma. This is used for 286 | # instance to not check methods defined in Zope's Interface base class. 287 | ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by,implementedBy,providedBy 288 | 289 | # List of method names used to declare (i.e. assign) instance attributes. 290 | defining-attr-methods=__init__,__new__,setUp 291 | 292 | # List of valid names for the first argument in a class method. 293 | valid-classmethod-first-arg=cls 294 | 295 | # List of valid names for the first argument in a metaclass class method. 296 | valid-metaclass-classmethod-first-arg=mcs 297 | 298 | 299 | [DESIGN] 300 | 301 | # Maximum number of arguments for function / method 302 | max-args=6 303 | 304 | # Argument names that match this expression will be ignored. Default to name 305 | # with leading underscore 306 | ignored-argument-names=_.* 307 | 308 | # Maximum number of locals for function / method body 309 | max-locals=15 310 | 311 | # Maximum number of return / yield for function / method body 312 | max-returns=6 313 | 314 | # Maximum number of branch for function / method body 315 | max-branches=12 316 | 317 | # Maximum number of statements in function / method body 318 | max-statements=50 319 | 320 | # Maximum number of parents for a class (see R0901). 321 | max-parents=12 322 | 323 | # Maximum number of attributes for a class (see R0902). 324 | max-attributes=7 325 | 326 | # Minimum number of public methods for a class (see R0903). 327 | min-public-methods=2 328 | 329 | # Maximum number of public methods for a class (see R0904). 330 | max-public-methods=20 331 | 332 | 333 | [EXCEPTIONS] 334 | 335 | # Exceptions that will emit a warning when being caught. Defaults to 336 | # "Exception" 337 | overgeneral-exceptions=Exception 338 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.6" 4 | install: 5 | - pip install -r tests/test-requirements.txt 6 | script: 7 | - pytest 8 | - ./tests/validate_signature.sh 9 | -------------------------------------------------------------------------------- /LICENSE.TXT: -------------------------------------------------------------------------------- 1 | STARTTLS Everywhere Policy Database and Helper Code 2 | Copyright (c) Electronic Frontier Foundation and others 3 | 4 | This repository is licensed under the Apache License Version 2.0. 5 | 6 | The STARTTLS Everywhere Policy Database file (policy.json) is governed 7 | by the following policy: 8 | 9 | By contributing rules to the STARTTLS Everywhere Policy Database, or 10 | modifying existing rules, you grant the Electronic Frontier Foundation 11 | (EFF) a non-exclusive, transferable, sub-licensable, royalty-free, 12 | worldwide license to use your contribution(s) in any manner that, in 13 | EFF's sole discretion, furthers EFF's mission. 14 | 15 | 16 | 17 | Text of Apache License 18 | ====================== 19 | Apache License 20 | Version 2.0, January 2004 21 | http://www.apache.org/licenses/ 22 | 23 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 24 | 25 | 1. Definitions. 26 | 27 | "License" shall mean the terms and conditions for use, reproduction, 28 | and distribution as defined by Sections 1 through 9 of this document. 29 | 30 | "Licensor" shall mean the copyright owner or entity authorized by 31 | the copyright owner that is granting the License. 32 | 33 | "Legal Entity" shall mean the union of the acting entity and all 34 | other entities that control, are controlled by, or are under common 35 | control with that entity. For the purposes of this definition, 36 | "control" means (i) the power, direct or indirect, to cause the 37 | direction or management of such entity, whether by contract or 38 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 39 | outstanding shares, or (iii) beneficial ownership of such entity. 40 | 41 | "You" (or "Your") shall mean an individual or Legal Entity 42 | exercising permissions granted by this License. 43 | 44 | "Source" form shall mean the preferred form for making modifications, 45 | including but not limited to software source code, documentation 46 | source, and configuration files. 47 | 48 | "Object" form shall mean any form resulting from mechanical 49 | transformation or translation of a Source form, including but 50 | not limited to compiled object code, generated documentation, 51 | and conversions to other media types. 52 | 53 | "Work" shall mean the work of authorship, whether in Source or 54 | Object form, made available under the License, as indicated by a 55 | copyright notice that is included in or attached to the work 56 | (an example is provided in the Appendix below). 57 | 58 | "Derivative Works" shall mean any work, whether in Source or Object 59 | form, that is based on (or derived from) the Work and for which the 60 | editorial revisions, annotations, elaborations, or other modifications 61 | represent, as a whole, an original work of authorship. For the purposes 62 | of this License, Derivative Works shall not include works that remain 63 | separable from, or merely link (or bind by name) to the interfaces of, 64 | the Work and Derivative Works thereof. 65 | 66 | "Contribution" shall mean any work of authorship, including 67 | the original version of the Work and any modifications or additions 68 | to that Work or Derivative Works thereof, that is intentionally 69 | submitted to Licensor for inclusion in the Work by the copyright owner 70 | or by an individual or Legal Entity authorized to submit on behalf of 71 | the copyright owner. For the purposes of this definition, "submitted" 72 | means any form of electronic, verbal, or written communication sent 73 | to the Licensor or its representatives, including but not limited to 74 | communication on electronic mailing lists, source code control systems, 75 | and issue tracking systems that are managed by, or on behalf of, the 76 | Licensor for the purpose of discussing and improving the Work, but 77 | excluding communication that is conspicuously marked or otherwise 78 | designated in writing by the copyright owner as "Not a Contribution." 79 | 80 | "Contributor" shall mean Licensor and any individual or Legal Entity 81 | on behalf of whom a Contribution has been received by Licensor and 82 | subsequently incorporated within the Work. 83 | 84 | 2. Grant of Copyright License. Subject to the terms and conditions of 85 | this License, each Contributor hereby grants to You a perpetual, 86 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 87 | copyright license to reproduce, prepare Derivative Works of, 88 | publicly display, publicly perform, sublicense, and distribute the 89 | Work and such Derivative Works in Source or Object form. 90 | 91 | 3. Grant of Patent License. Subject to the terms and conditions of 92 | this License, each Contributor hereby grants to You a perpetual, 93 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 94 | (except as stated in this section) patent license to make, have made, 95 | use, offer to sell, sell, import, and otherwise transfer the Work, 96 | where such license applies only to those patent claims licensable 97 | by such Contributor that are necessarily infringed by their 98 | Contribution(s) alone or by combination of their Contribution(s) 99 | with the Work to which such Contribution(s) was submitted. If You 100 | institute patent litigation against any entity (including a 101 | cross-claim or counterclaim in a lawsuit) alleging that the Work 102 | or a Contribution incorporated within the Work constitutes direct 103 | or contributory patent infringement, then any patent licenses 104 | granted to You under this License for that Work shall terminate 105 | as of the date such litigation is filed. 106 | 107 | 4. Redistribution. You may reproduce and distribute copies of the 108 | Work or Derivative Works thereof in any medium, with or without 109 | modifications, and in Source or Object form, provided that You 110 | meet the following conditions: 111 | 112 | (a) You must give any other recipients of the Work or 113 | Derivative Works a copy of this License; and 114 | 115 | (b) You must cause any modified files to carry prominent notices 116 | stating that You changed the files; and 117 | 118 | (c) You must retain, in the Source form of any Derivative Works 119 | that You distribute, all copyright, patent, trademark, and 120 | attribution notices from the Source form of the Work, 121 | excluding those notices that do not pertain to any part of 122 | the Derivative Works; and 123 | 124 | (d) If the Work includes a "NOTICE" text file as part of its 125 | distribution, then any Derivative Works that You distribute must 126 | include a readable copy of the attribution notices contained 127 | within such NOTICE file, excluding those notices that do not 128 | pertain to any part of the Derivative Works, in at least one 129 | of the following places: within a NOTICE text file distributed 130 | as part of the Derivative Works; within the Source form or 131 | documentation, if provided along with the Derivative Works; or, 132 | within a display generated by the Derivative Works, if and 133 | wherever such third-party notices normally appear. The contents 134 | of the NOTICE file are for informational purposes only and 135 | do not modify the License. You may add Your own attribution 136 | notices within Derivative Works that You distribute, alongside 137 | or as an addendum to the NOTICE text from the Work, provided 138 | that such additional attribution notices cannot be construed 139 | as modifying the License. 140 | 141 | You may add Your own copyright statement to Your modifications and 142 | may provide additional or different license terms and conditions 143 | for use, reproduction, or distribution of Your modifications, or 144 | for any such Derivative Works as a whole, provided Your use, 145 | reproduction, and distribution of the Work otherwise complies with 146 | the conditions stated in this License. 147 | 148 | 5. Submission of Contributions. Unless You explicitly state otherwise, 149 | any Contribution intentionally submitted for inclusion in the Work 150 | by You to the Licensor shall be under the terms and conditions of 151 | this License, without any additional terms or conditions. 152 | Notwithstanding the above, nothing herein shall supersede or modify 153 | the terms of any separate license agreement you may have executed 154 | with Licensor regarding such Contributions. 155 | 156 | 6. Trademarks. This License does not grant permission to use the trade 157 | names, trademarks, service marks, or product names of the Licensor, 158 | except as required for reasonable and customary use in describing the 159 | origin of the Work and reproducing the content of the NOTICE file. 160 | 161 | 7. Disclaimer of Warranty. Unless required by applicable law or 162 | agreed to in writing, Licensor provides the Work (and each 163 | Contributor provides its Contributions) on an "AS IS" BASIS, 164 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 165 | implied, including, without limitation, any warranties or conditions 166 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 167 | PARTICULAR PURPOSE. You are solely responsible for determining the 168 | appropriateness of using or redistributing the Work and assume any 169 | risks associated with Your exercise of permissions under this License. 170 | 171 | 8. Limitation of Liability. In no event and under no legal theory, 172 | whether in tort (including negligence), contract, or otherwise, 173 | unless required by applicable law (such as deliberate and grossly 174 | negligent acts) or agreed to in writing, shall any Contributor be 175 | liable to You for damages, including any direct, indirect, special, 176 | incidental, or consequential damages of any character arising as a 177 | result of this License or out of the use or inability to use the 178 | Work (including but not limited to damages for loss of goodwill, 179 | work stoppage, computer failure or malfunction, or any and all 180 | other commercial damages or losses), even if such Contributor 181 | has been advised of the possibility of such damages. 182 | 183 | 9. Accepting Warranty or Additional Liability. While redistributing 184 | the Work or Derivative Works thereof, You may choose to offer, 185 | and charge a fee for, acceptance of support, warranty, indemnity, 186 | or other liability obligations and/or rights consistent with this 187 | License. However, in accepting such obligations, You may act only 188 | on Your own behalf and on Your sole responsibility, not on behalf 189 | of any other Contributor, and only if You agree to indemnify, 190 | defend, and hold each Contributor harmless for any liability 191 | incurred by, or claims asserted against, such Contributor by reason 192 | of your accepting any such warranty or additional liability. 193 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # STARTTLS Everywhere 2 | 3 | STARTTLS Everywhere is an initiative for upgrading the security of the email ecosystem. Several sub-projects fit underneath the general umbrella of "STARTTLS Everywhere". The name itself is a bit of a misnomer, since the original idea for the project came about in 2014, when STARTTLS support hovered around 20% across the internet. Since then we've come a long way, with [Gmail's transparency report](https://transparencyreport.google.com/safer-email/overview) citing ~90% of inbound and outbound mail are encrypted with TLS, as of 2018. 4 | 5 | We still have a long way to go. STARTTLS (opportunistic TLS) is vulnerable to [trivial downgrade attacks](https://stomp.colorado.edu/blog/blog/2012/12/31/on-smtp-starttls-and-the-cisco-asa/) that continue to be observed across the Internet. As of 2018, a quick Zmap query for a common STARTTLS-stripping fingerprint (a banner that reads "250 XXXXXXXX" rather than "250 STARTTLS") reveals around 8 thousand hosts. This is likely an under-estimate, since active attackers can perform the stripping in a less detectable way (simply by omitting the banner, for instance, rather than replacing its body with X's). 6 | 7 | In addition, STARTTLS is also vulnerable to TLS man-in-the-middle attacks. Mailservers currently don't validate TLS certificates, since there has only [recently](https://tools.ietf.org/html/rfc8461#section-4.2) been an attempt to standardize the certificate validation rules across the email ecosystem. 8 | 9 | Absent DNSSEC/DANE, STARTTLS by itself thwarts purely passive eavesdroppers. However, as currently deployed, it allows either bulk or semi-targeted attacks that are very unlikely to be detected. We would like to deploy both detection and prevention for such semi-targeted attacks. 10 | 11 | ### Goals 12 | 13 | * Prevent [TLS stripping](https://www.eff.org/deeplinks/2014/11/starttls-downgrade-attacks) from revealing email contents to the network, when in transit between major MTAs that support STARTTLS. 14 | * Prevent active MITM attacks at the DNS, SMTP, TLS, or other layers from revealing contents to the attacker. 15 | * Zero or minimal decrease to deliverability rates unless network attacks are actually occurring. 16 | * Create feedback-loops on targeted attacks and bulk surveilance in an opt-in, anonymized way. 17 | 18 | ### Non-goals 19 | 20 | * Prevent fully-targeted exploits of vulnerabilities on endpoints or on mail hosts. 21 | * Refuse delivery on the recipient side if sender does not negotiate TLS (this may be a future project). 22 | * Develop a fully-decentralized solution. 23 | * Initially we are not engineering to scale to all mail domains on the Internet, though we believe this design can be scaled as required if large numbers of domains publish policies to it. 24 | 25 | ## Solution stacks 26 | 27 | A solution needs the following things: 28 | - [ ] Server can advertise TLS support and MX data 29 | - [ ] In a non-downgrade-able way 30 | - [ ] Minimize deliverability impact 31 | - [ ] Widely deployed 32 | 33 | ### DNSSEC, DANE, and TLSRPT 34 | 35 | With [DNSSEC](https://tools.ietf.org/html/rfc4034) and [DANE](https://tools.ietf.org/html/rfc6698) [for email](https://tools.ietf.org/html/rfc7672), mailservers can essentially publish or pin their public keys via authenticated DNS records. If a domain has DNSSEC-signed their records, the absence/presence of a DANE TLSA record indicates non-support/support for STARTTLS, respectively. 36 | 37 | Our goals can also be accomplished through use of [DNSSEC and DANE](https://tools.ietf.org/html/rfc7672), which is a very scalable solution. DANE adoption has been slow primarily since it is dependent on upstream support for DNSSEC; operators have been very slow to roll out DNSSEC. Making DNSSEC easier to deploy and improving its deployment is, for now, outside the scope of this project, though making DANE easier to deploy may be in-scope. 38 | 39 | The mention of [TLSRPT](https://tools.ietf.org/html/rfc8460) is due to the fact that several operators consistently deploy DNSSEC or DANE incorrectly. We want to close the misconfiguration reporting feedback loop. TLSRPT is an RFC for publishing a "reporting mechanism" to DNS. This endpoint can be an email address or a web endpoint; it is expected that senders will publish to these when failures occur, and that receivers will aggregate these reports and fix their configurations if problems arise. 40 | 41 | - [x] Server can advertise TLS support and MX data *(DANE TLSA records)* 42 | - [x] In a non-downgrade-able way *(NSEC3 for DNSSEC)* 43 | - [x] Minimize deliverability impact *(TLSRPT, ideally)* 44 | 45 | ### MTA-STS, Preloading, and TLSRPT 46 | 47 | MTA-STS is a specification for mailservers to publish their TLS policy and ask senders to cache that policy, absent DNSSEC. The policy can be discovered at a `.well-known` address served over HTTPS at the email domain (for instance, [Gmail's record](https://mta-sts.gmail.com/.well-known/mta-sts.txt)). MTA-STS support is discovered through an initial DNS lookup. 48 | 49 | There is value in deploying an intermediate solution, perhaps through [MTA-STS](https://tools.ietf.org/html/rfc8461), that does not rely on DNSSEC. This will improve the email security situation more quickly. It will also provide operational experience with authenticated SMTP over TLS that will make eventual rollout of DANE-based solutions easier. 50 | 51 | However, MTA-STS, unlike DNSSEC + DANE, is trust-on-first-use. Since MTA-STS assumes no DNSSEC, the initial DNS query to discover MTA-STS support is downgradable. A full solution would include distributing an MTA-STS preload list via our email security database. 52 | 53 | - [x] Server can advertise TLS support and MX data *(MTA-STS)* 54 | - [x] In a non-downgrade-able way *(Preloading)* 55 | - [x] Minimize deliverability impact *(TLSRPT, ideally)* 56 | 57 | ## Project scope 58 | 59 | The project scope is very large, though our development team is extremely small. The following is a list of things that we care about doing, and afterwards is a short-term timeline of the currently prioritized tasks. 60 | 61 | If you are working next to or directly on one or more of these things, feel free to shoot us an email at starttls-everywhere@eff.org. 62 | 63 | ### Email security database (STARTTLS policy list) 64 | 65 | * [Usage guidelines](USAGE.md) (for configuration of sending mailservers) 66 | * [Validation guidelines](VALIDATION.md) (for configuration of receiving mailservers) 67 | * [Detailed spec](RULES.md) of list format. 68 | * [Submission site for adding mail domains](https://starttls-everywhere.org) 69 | * Maintaining and encouraging use of [starttls-policy-cli](https://github.com/EFForg/starttls-policy-cli). 70 | 71 | ### Tracking and encouraging deployment of existing standards. 72 | 73 | * DANE 74 | * Several other folks do a great job of monitoring [DANE deployment](https://mail.sys4.de/pipermail/dane-users/) and [misconfigurations](https://danefail.org/) on the internet. 75 | * MTA-STS 76 | * Encouraging MTA-STS validation support in popular MTA software. 77 | * Encouraging mailservers to publish their policies. 78 | * TLSRPT 79 | * Encouraging reporting support in popular MTA software. 80 | * Encouraging mailservers to host reporting servers/endpoints. 81 | 82 | ### Currently actively maintaining/building 83 | 84 | * Maintaining the [email security database](policy.json) and [submission site](https://starttls-everywhere.org) 85 | * Building out integrations for [starttls-policy-cli](https://github.com/EFForg/starttls-policy-cli/blob/master/README.md) 86 | * Encouraging MTA-STS validation support in other MTA software. 87 | 88 | ### Contributing 89 | 90 | * Announcements mailing list: starttls-everywhere@eff.org, https://lists.eff.org/mailman/listinfo/starttls-everywhere 91 | * Developer mailing list: starttls-everywhere-devs@lists.eff.org, https://lists.eff.org/mailman/listinfo/starttls-everywhere-devs 92 | * IRC/dev channel: TBD 93 | * We host a weekly development call every Thursday at 11AM Pacific Time. Shoot the mailing list an email if you're interested in joining or just sitting in. 94 | 95 | -------------------------------------------------------------------------------- /RULES.md: -------------------------------------------------------------------------------- 1 | # Policy rule configuration format 2 | 3 | The TLS policy file is a `json` file which conforms to the following specification. These fields draw inspiration from the [MTA-STS policy file format](https://tools.ietf.org/html/rfc8461) as well as [Chromium's HSTS Preload List](https://hstspreload.org/). 4 | The basic file format will be JSON. Example: 5 | 6 | ``` 7 | { 8 | "timestamp": "2014-06-06T14:30:16+00:00", 9 | "author": "Electronic Frontier Foundation https://eff.org", 10 | "expires": "2014-06-06T14:30:16+00:00", 11 | "version": "0.1", 12 | "policy-aliases": { 13 | "gmail": { 14 | "mode": "testing", 15 | "mxs": [".mail.google.com"], 16 | } 17 | }, 18 | "policies": { 19 | "yahoo.com": { 20 | "mode": "enforce", 21 | "mxs": [".yahoodns.net"] 22 | }, 23 | "eff.org": { 24 | "mode": "enforce", 25 | "mxs": [".eff.org"] 26 | }, 27 | "gmail.com": { 28 | "policy-alias": "gmail" 29 | }, 30 | "example.com": { 31 | "mode": "testing", 32 | "mxs": ["mail.example.com", ".example.net"] 33 | }, 34 | }, 35 | } 36 | ``` 37 | 38 | At a high level, senders will expect the following for recipient domains: 39 | - 1. Email domain resolves to an MX hostname which matches an entry in `mxs` 40 | - 2. Provides a valid certificate. Validity means: 41 | - a. The CN or DNS entry under subjectAltName matches an appropriate hostname. 42 | - b. The certificate is unexpired. 43 | - c. There is a valid chain from the certificate to a root included in [Mozilla's trust store](https://www.mozilla.org/en-US/about/governance/policies/security-group/certs/included/) (available as [Debian package ca-certificates](https://packages.debian.org/sid/ca-certificates)). 44 | - 3. Successfully negotiates a TLS session (>= TLS 1.2). 45 | 46 | A user of this file format may choose to override individual domain policies. For instance, the EFF might provide an overall configuration covering major mail providers, and another organization might produce an overlay for mail providers in a specific country. 47 | 48 | Before adding a policy to this list, we validate that the email domain conforms to the policy, as above. 49 | 50 | Note that there is no inline signature field. The configuration file should be distributed with authentication using an offline signing key, generated using `gpg --clearsign`. Config-generator should validate the signature against a known GPG public key before extracting. The public key is part of the permanent system configuration, like the fetch URL. 51 | 52 | ### Top-level fields 53 | #### expires 54 | When this configuration file expires, in UTC. Can be in epoch seconds from 00:00:00 UTC on 1 January 1970, or a string `yyyy-MM-dd'T'HH:mm:ssZZZZ`. If the file has ceased being regularly updated for any reason, and the policy file has expired, the MTA should fall-back to opportunistic TLS for e-mail delivery, and the system operator should be alerted. 55 | 56 | #### timestamp 57 | When this configuration file was distributed/fetched, in UTC. Can be in epoch seconds from 00:00:00 UTC on 1 January 1970, or a string `yyyy-MM-dd'T'HH:mm:ssZZZZ`. This field will be monotonically increasing for every update to the policy file. When updating this policy file, should validate that the timestamp is greater than or equal to the existing one. 58 | 59 | #### policy-aliases 60 | A mapping of alias names onto a policy. Domains in the `policies` field can refer to policies defined in this object. 61 | 62 | #### policies 63 | A mapping of mail domains (the part of an address after the "@") onto a "policy". Matching of mail domains is on an exact-match basis. For instance, `eff.org` would be listed separately from `lists.eff.org`. Fields in this policy specify security requirements that should be applied when connecting to any MTA server for a given mail domain. If the `mode` is `testing`, then the sender should not stop mail delivery on policy failures, but should produce logging information. 64 | 65 | #### version 66 | Version of the configuration file. 67 | 68 | ### Policy fields 69 | Every field in `policies` maps to a policy object. A policy object can have the following fields: 70 | 71 | #### policy-alias 72 | 73 | If set, other fields are ignored. This value should be a key in the upper-level "policy-aliases" configuration object. The policy for this domain will be configured as the denoted policy in the "policy-aliases" object. 74 | 75 | #### mode 76 | Default: `testing` (required) 77 | 78 | Either `testing` or `enforce`. If `testing` is set, then the recipient expects any failure in TLS negotiation to be reported via a mechanism such as TLSRPT, but the message is still sent over the insecure communication. 79 | 80 | #### mxs 81 | 82 | A list of the expected MX hostnames for your server. At least one of the names on each mailserver's certificate should match one of these patterns. The pattern can be a suffix, like `.eff.org`, or a fully-qualified domain name, like `mx.eff.org`. Suffixes will only match one subdomain label, so `.eff.org` would match names `*.eff.org` and `mx.eff.org`, but not `good.mx.eff.org` or `*.mx.eff.org`. 83 | 84 | -------------------------------------------------------------------------------- /USAGE.md: -------------------------------------------------------------------------------- 1 | # STARTTLS Policy List Usage Guidelines 2 | 3 | For information about configuration of receiving mailservers and addition of a receiving mailserver to the list, please see [VALIDATION.md](VALIDATION.md) instead. 4 | 5 | ## MTA-STS interoperation for sending mailservers 6 | 7 | The ideal is for the STARTTLS policy list to act as a "preload list" for MTA-STS domains. Although there are many parallels to the web situation with HSTS and the HSTS preload list, this list is not an exact equivalent. There are a couple of edge cases to consider when implementing both MTA-STS and the policy list. This describes the expected behavior of **sending** mailservers that both validate MTA-STS and follow the policy list. 8 | 9 | If a mailserver is able to successfully discover and validate an MTA-STS record that conflicts with a policy on the list, then the mailserver should use the MTA-STS policy. Similarly, if a mailserver has an MTA-STS record cached that conflicts with a policy on the list, then the mailserver should use the MTA-STS policy. 10 | 11 | That is, MTA-STS should take precedence over the STARTTLS policy list. The primary benefit of the STARTTLS policy list with MTA-STS is to secure MTA-STS on first-use. 12 | 13 | 14 | ## Using the list 15 | 16 | To download and verify the most up-to-date version of the STARTTLS policy list: 17 | 18 | ``` 19 | wget https://dl.eff.org/starttls-everywhere/policy.json 20 | wget https://dl.eff.org/starttls-everywhere/policy.json.asc 21 | gpg --recv-key B693F33372E965D76D55368616EEA65D03326C9D 22 | gpg --trusted-key 842AEA40C5BCD6E1 --verify policy.json.asc 23 | ``` 24 | 25 | Our sample [update_and_verify.sh script](https://github.com/EFForg/starttls-everywhere/blob/master/scripts/update_and_verify.sh) does the same. If you are actively using the list, **you must fetch updates at least once every 48 hours**. We provide [a sample cronjob](https://github.com/EFForg/starttls-everywhere/blob/master/scripts/starttls-policy.cron.d) to do this. 26 | 27 | Every policy JSON has an expiry date in the top-level configuration, after which we cannot guarantee deliverability if you are using the expired list. 28 | 29 | #### Behavior 30 | 31 | A domain's policy, `enforce` or `testing`, asks that relays which connect to that domain's MX server and cannot initiate a TLS connection perform different behaviors depending on the policy (e.g. reporting what went wrong to the target domain for `testing`, and additionally aborting the connection for `enforce`). That is the behavior specified by [SMTP MTA Strict Transport Security (MTA-STS)](https://tools.ietf.org/html/rfc8461), an upcoming protocol which this Policy List aims to complement by providing an alternative method for advertising a mail server’s security policy. 32 | 33 | #### Tooling 34 | 35 | Our [starttls-policy](https://github.com/EFForg/starttls-everywhere/tree/master/starttls-policy) Python package can fetch updates to and iterate over the existing list. If you use Postfix, we provide utilities to transform the policy list into configuration parameters that Postfix understands. 36 | 37 | We welcome [contributions](https://github.com/EFForg/starttls-everywhere) for different MTAs! 38 | 39 | -------------------------------------------------------------------------------- /VALIDATION.md: -------------------------------------------------------------------------------- 1 | # STARTTLS Policy List Validation 2 | 3 | For information about configuration of sending mailservers and using the list to validate TLS configurations, please see [USAGE.md](USAGE.md) instead. 4 | 5 | ## MTA-STS 6 | 7 | In order to be preloaded onto the STARTTLS policy list, the easiest way to do and maintain this is to have a valid MTA-STS record up for a max-age of at least half a year (10886400). 8 | 9 | To change your entry on the list, you can serve a valid MTA-STS record with your new expected policy. 10 | 11 | To be removed from the list, you can serve a valid MTA-STS record with `mode: none`. 12 | 13 | ## Manual addition 14 | 15 | If you don't have an MTA-STS file up, For your email domain to be eligible for addition to the STARTTLS policy list, the requirements are that your domain: 16 | 17 | * Supports STARTTLS (as a receiving mailserver) 18 | * Supplied MX hostnames are valid (DNS lookup from this perspective resolves to hostnames that match the given patterns). 19 | * Provides a valid PKI certificate. Validity is defined as in [RFC 3280](https://tools.ietf.org/html/rfc3280#section-6). To clarify: 20 | * The certificate's Common Name or a subjectAltName should match the server's MX hostname. 21 | * There is a valid chain, served by the server, from the certificate to a root included in [Mozilla's trust store](https://wiki.mozilla.org/CA/Included_Certificates) (available as Debian package [ca-certificates](https://packages.debian.org/sid/ca-certificates)). 22 | * The certificate is not expired or revoked. 23 | 24 | (Note: you can obtain a valid certificate for free via [Certbot](https://certbot.eff.org), a client for the [Let's Encrypt](https://letsencrypt.org) certificate authority.) 25 | 26 | Before adding a domain to the list, we continue to perform validation against the mailserver for at least one week. If it fails at any point, **it must be resubmitted.** 27 | 28 | With that in mind, you can [queue your mail domain](https://starttls-everywhere.org/add-domain) for the STARTTLS policy list. Alternatively, you can send an email to [starttls-policy@eff.org](mailto:starttls-policy@eff.org) or [submit a pull request](https://github.com/EFForg/starttls-everywhere) to add your domain, though these other channels may take longer for your domain to be submitted. 29 | 30 | #### Continued requirements 31 | 32 | Failure to continue meeting these requirements could result in deliverability issues to your mailserver, from any mail clients configured to use the STARTTLS policy list. 33 | 34 | We continue to validate all the domains on the list daily. If we notice any oddities, we will notify the contact email associated with the policy submission and urge you to either update or remove your policy. 35 | 36 | ## Manually removing your policy entry from the list 37 | 38 | If you're migrating email hosting, you'll need to update the MX hostnames associated with your domain's policy. Contact us beforehand so we can minimize the deliverability impact and remove your policy from the list-- we may also issue a challenge, solvable over DNS or HTTPS, in order to validate your intentions (more details in the threat modelling section). If we notice that you have migrated domains, we will reach out to you through a contact email that you provide, and the postmaster@ address. 39 | 40 | If you'd like to request removal from the list, or an update to your policy entry (or associated contact email), contact us at [starttls-policy@eff.org](mailto:starttls-policy@eff.org) 41 | 42 | ## Threat modelling and security considerations 43 | 44 | ### DNS forgery 45 | 46 | An attacker with the ability to forge DNS requests to a sending mailserver for a particular receiving mailserver might provide fake MX records for the receiving mailserver, causing the sender to direct its mail towards a malicious server. This is thwarted if the sender validates against the policy list and discovers that the receiving server uses a different set of hostnames. 47 | 48 | This type of attacker may try and induce a removal and re-addition of a policy to the list. To do this, they might need to induce a validating CA to misissue a certificate, or maintain a machine-in-the-middle position on our validation server for several weeks, and against our notification email server, which will send updates to the policy's email contact (also unknown to the attacker, and could be hosted on a different server) about these attempts. 49 | 50 | ### TCP on-path attacker 51 | 52 | Normally, a regular on-path attacker might simply downgrade TLS. A more sophisticated one might perform a certificate man-in-the-middle. In either case, this is thwarted if the sender is validating against the policy list and discovers that the receiving server should support TLS with a valid certificate. 53 | 54 | This type of attacker may try and induce removal of that receiving server from the policy list. To prevent this, we'll issue a DNS challenge to validate any policy list removal. 55 | 56 | ### CA compromise or induced misissuance 57 | 58 | When a valid MTA-STS record is discovered, the STARTTLS policy list cannot necessarily provide a higher degree of security, since we also validate TLS policies from our own network position. Thus, MTA-STS policies should take precedence over the STARTTLS policy list. 59 | 60 | This means that we do assume CA compromise or misissuance is more difficult (in terms of cost, risk, and detectability thanks to Certificate Transparency, or CT) than performing a single DNS on-path attack or forgery, and than performing a TCP on-path attack. However, it is possible, with a temporary DNS machine-in-the-middle position (perhaps due to BGP hijacking) with respect to an automated CA like Let's Encrypt, to induce certificate misissuance. 61 | 62 | Some of these judgments depend on an adversary's relative network perspective to the target mailserver and various automated CAs. Regardless, inducing misissuance is at least detectable if a domain owner is [monitoring CT logs](), and may require extra effort (depending on network perspective) in addition to having a machine-in-the-middle DNS position targeted towards a particular mailserver. Hopefully, with [multi-perspective validation](https://letsencrypt.org/upcoming-features/) on the horizon for Let's Encrypt, the bar could be raised even higher for an attacker to induce misissuance. 63 | 64 | -------------------------------------------------------------------------------- /code_legacy/PostfixConfigGenerator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import logging 4 | import sys 5 | import string 6 | import subprocess 7 | import os, os.path 8 | 9 | 10 | logger = logging.getLogger(__name__) 11 | logger.setLevel(logging.DEBUG) 12 | log_handler = logging.StreamHandler() 13 | log_handler.setLevel(logging.DEBUG) 14 | logger.addHandler(log_handler) 15 | 16 | 17 | def parse_line(line_data): 18 | """ 19 | Return the (line number, left hand side, right hand side) of a stripped 20 | postfix config line. 21 | 22 | Lines are like: 23 | smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache 24 | """ 25 | num,line = line_data 26 | left, sep, right = line.partition("=") 27 | if not sep: 28 | return None 29 | return (num, left.strip(), right.strip()) 30 | 31 | 32 | class ExistingConfigError(ValueError): pass 33 | 34 | 35 | class PostfixConfigGenerator: 36 | def __init__(self, 37 | policy_config, 38 | postfix_dir, 39 | fixup=False, 40 | fopen=open, 41 | version=None): 42 | self.fixup = fixup 43 | self.postfix_dir = postfix_dir 44 | self.policy_config = policy_config 45 | self.policy_file = os.path.join(postfix_dir, 46 | "starttls_everywhere_policy") 47 | self.ca_file = os.path.join(postfix_dir, "starttls_everywhere_CAfile") 48 | 49 | self.additions = [] 50 | self.deletions = [] 51 | self.fn = self.find_postfix_cf() 52 | self.raw_cf = fopen(self.fn).readlines() 53 | self.cf = map(string.strip, self.raw_cf) 54 | #self.cf = [line for line in cf if line and not line.startswith("#")] 55 | self.policy_lines = [] 56 | self.new_cf = "" 57 | 58 | # Set in .prepare() unless running in a test 59 | self.postfix_version = version 60 | 61 | def find_postfix_cf(self): 62 | "Search far and wide for the correct postfix configuration file" 63 | return os.path.join(self.postfix_dir, "main.cf") 64 | 65 | def ensure_cf_var(self, var, ideal, also_acceptable): 66 | """ 67 | Ensure that existing postfix config @var is in the list of @acceptable 68 | values; if not, set it to the ideal value. 69 | """ 70 | acceptable = [ideal] + also_acceptable 71 | 72 | l = [(num,line) for num,line in enumerate(self.cf) 73 | if line.startswith(var)] 74 | if not any(l): 75 | self.additions.append(var + " = " + ideal) 76 | else: 77 | values = map(parse_line, l) 78 | if len(set(values)) > 1: 79 | if self.fixup: 80 | conflicting_lines = [num for num,_var,val in values] 81 | self.deletions.extend(conflicting_lines) 82 | self.additions.append(var + " = " + ideal) 83 | else: 84 | raise ExistingConfigError( 85 | "Conflicting existing config values " + `l` 86 | ) 87 | val = values[0][2] 88 | if val not in acceptable: 89 | if self.fixup: 90 | self.deletions.append(values[0][0]) 91 | self.additions.append(var + " = " + ideal) 92 | else: 93 | raise ExistingConfigError( 94 | "Existing config has %s=%s"%(var,val) 95 | ) 96 | 97 | def wrangle_existing_config(self): 98 | """ 99 | Try to ensure/mutate that the config file is in a sane state. 100 | Fixup means we'll delete existing lines if necessary to get there. 101 | """ 102 | # Check we're currently accepting inbound STARTTLS sensibly 103 | self.ensure_cf_var("smtpd_use_tls", "yes", []) 104 | # Ideally we use it opportunistically in the outbound direction 105 | self.ensure_cf_var("smtp_tls_security_level", "may", ["encrypt","dane"]) 106 | # Maximum verbosity lets us collect failure information 107 | self.ensure_cf_var("smtp_tls_loglevel", "1", []) 108 | # Inject a reference to our per-domain policy map 109 | policy_cf_entry = "texthash:" + self.policy_file 110 | 111 | self.ensure_cf_var("smtp_tls_policy_maps", policy_cf_entry, []) 112 | self.ensure_cf_var("smtp_tls_CAfile", self.ca_file, []) 113 | 114 | # Disable SSLv2 and SSLv3. Syntax for `smtp_tls_protocols` changed 115 | # between Postfix version 2.5 and 2.6, since we only support => 2.11 116 | # we don't use nor support legacy Postfix syntax. 117 | # - Server: 118 | self.ensure_cf_var("smtpd_tls_protocols", "!SSLv2, !SSLv3", []) 119 | self.ensure_cf_var("smtpd_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) 120 | # - Client: 121 | self.ensure_cf_var("smtp_tls_protocols", "!SSLv2, !SSLv3", []) 122 | self.ensure_cf_var("smtp_tls_mandatory_protocols", "!SSLv2, !SSLv3", []) 123 | 124 | def maybe_add_config_lines(self, fopen=open): 125 | if not self.additions: 126 | return 127 | if self.fixup: 128 | logger.info('Deleting lines: {}'.format(self.deletions)) 129 | self.additions[:0]=["#", 130 | "# New config lines added by STARTTLS Everywhere", 131 | "#"] 132 | new_cf_lines = "\n".join(self.additions) + "\n" 133 | logger.info('Adding to {}:'.format(self.fn)) 134 | logger.info(new_cf_lines) 135 | if self.raw_cf[-1][-1] == "\n": sep = "" 136 | else: sep = "\n" 137 | 138 | for num, line in enumerate(self.raw_cf): 139 | if self.fixup and num in self.deletions: 140 | self.new_cf += "# Line removed by STARTTLS Everywhere\n# " + line 141 | else: 142 | self.new_cf += line 143 | self.new_cf += sep + new_cf_lines 144 | 145 | if not os.access(self.fn, os.W_OK): 146 | raise Exception("Can't write to %s, please re-run as root." 147 | % self.fn) 148 | with fopen(self.fn, "w") as f: 149 | f.write(self.new_cf) 150 | 151 | def set_domainwise_tls_policies(self, fopen=open): 152 | all_acceptable_mxs = self.policy_config.acceptable_mxs 153 | for address_domain, properties in all_acceptable_mxs.items(): 154 | mx_list = properties.accept_mx_domains 155 | if len(mx_list) > 1: 156 | logger.warn('Lists of multiple accept-mx-domains not yet ' 157 | 'supported.') 158 | logger.warn('Using MX {} for {}'.format(mx_list[0], 159 | address_domain) 160 | ) 161 | logger.warn('Ignoring: {}'.format(', '.join(mx_list[1:]))) 162 | mx_domain = mx_list[0] 163 | mx_policy = self.policy_config.get_tls_policy(mx_domain) 164 | entry = address_domain + " encrypt" 165 | if mx_policy.min_tls_version.lower() == "tlsv1": 166 | entry += " protocols=!SSLv2:!SSLv3" 167 | elif mx_policy.min_tls_version.lower() == "tlsv1.1": 168 | entry += " protocols=!SSLv2:!SSLv3:!TLSv1" 169 | elif mx_policy.min_tls_version.lower() == "tlsv1.2": 170 | entry += " protocols=!SSLv2:!SSLv3:!TLSv1:!TLSv1.1" 171 | else: 172 | logger.warn('Unknown minimum TLS version: {} '.format( 173 | mx_policy.min_tls_version) 174 | ) 175 | self.policy_lines.append(entry) 176 | 177 | with fopen(self.policy_file, "w") as f: 178 | f.write("\n".join(self.policy_lines) + "\n") 179 | 180 | ### Let's Encrypt client IPlugin ### 181 | # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/plugins/common.py#L35 182 | 183 | def prepare(self): 184 | """Prepare the plugin. 185 | Finish up any additional initialization. 186 | :raises .PluginError: 187 | when full initialization cannot be completed. 188 | :raises .MisconfigurationError: 189 | when full initialization cannot be completed. Plugin will 190 | be displayed on a list of available plugins. 191 | :raises .NoInstallationError: 192 | when the necessary programs/files cannot be located. Plugin 193 | will NOT be displayed on a list of available plugins. 194 | :raises .NotSupportedError: 195 | when the installation is recognized, but the version is not 196 | currently supported. 197 | :rtype tuple: 198 | """ 199 | # XXX ensure we raise the right kinds of exceptions 200 | 201 | if not self.postfix_version: 202 | self.postfix_version = self.get_version() 203 | 204 | if self.postfix_version < (2, 11, 0): 205 | raise Exception( 206 | 'NotSupportedError: Postfix version is too old -- test.' 207 | ) 208 | 209 | # Postfix has changed support for TLS features, supported protocol versions 210 | # KEX methods, ciphers et cetera over the years. We sort out version dependend 211 | # differences here and pass them onto other configuration functions. 212 | # see: 213 | # http://www.postfix.org/TLS_README.html 214 | # http://www.postfix.org/FORWARD_SECRECY_README.html 215 | 216 | # Postfix == 2.2: 217 | # - TLS support introduced via 3rd party patch, see: 218 | # http://www.postfix.org/TLS_LEGACY_README.html 219 | 220 | # Postfix => 2.2: 221 | # - built-in TLS support added 222 | # - Support for PFS introduced 223 | # - Support for (E)DHE params >= 1024bit (need to be generated), default 1k 224 | 225 | # Postfix => 2.5: 226 | # - Syntax to specify mandatory protocol version changes: 227 | # * < 2.5: `smtpd_tls_mandatory_protocols = TLSv1` 228 | # * => 2.5: `smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3` 229 | # - Certificate fingerprint verification added 230 | 231 | # Postfix => 2.6: 232 | # - Support for ECDHE NIST P-256 curve (enable `smtpd_tls_eecdh_grade = strong`) 233 | # - Support for configurable cipher-suites and protocol versions added, pre-2.6 234 | # releases always set EXPORT, options: `smtp_tls_ciphers` and `smtp_tls_protocols` 235 | # - `smtp_tls_eccert_file` and `smtp_tls_eckey_file` config. options added 236 | 237 | # Postfix => 2.8: 238 | # - Override Client suite preference w. `tls_preempt_cipherlist = yes` 239 | # - Elliptic curve crypto. support enabled by default 240 | 241 | # Postfix => 2.9: 242 | # - Public key fingerprint support added 243 | # - `permit_tls_clientcerts`, `permit_tls_all_clientcerts` and 244 | # `check_ccert_access` config. options added 245 | 246 | # Postfix <= 2.9.5: 247 | # - BUG: Public key fingerprint is computed incorrectly 248 | 249 | # Postfix => 3.1: 250 | # - Built-in support for TLS management and DANE added, see: 251 | # http://www.postfix.org/postfix-tls.1.html 252 | 253 | def get_version(self): 254 | """Return the mail version of Postfix. 255 | 256 | Version is returned as a tuple. (e.g. '2.11.3' is (2, 11, 3)) 257 | 258 | :returns: version 259 | :rtype: tuple 260 | 261 | :raises .PluginError: 262 | Unable to find Postfix version. 263 | """ 264 | # Parse Postfix version number (feature support, syntax changes etc.) 265 | cmd = subprocess.Popen(['/usr/sbin/postconf', '-d', 'mail_version'], 266 | stdout=subprocess.PIPE) 267 | stdout, _ = cmd.communicate() 268 | if cmd.returncode != 0: 269 | raise Exception('PluginError: Unable to determine Postfix version.') 270 | 271 | # grabs version component of string like "mail_version = 2.11.3" 272 | mail_version = stdout.split()[2] 273 | postfix_version = tuple([int(i) for i in mail_version.split('.')]) 274 | return postfix_version 275 | 276 | def more_info(self): 277 | """Human-readable string to help the user. 278 | Should describe the steps taken and any relevant info to help the user 279 | decide which plugin to use. 280 | :rtype str: 281 | """ 282 | return ( 283 | "Configures Postfix to try to authenticate mail servers, use " 284 | "installed certificates and disable weak ciphers and protocols.{0}" 285 | "Server root: {root}{0}" 286 | "Version: {version}".format( 287 | os.linesep, 288 | root=self.postfix_dir, 289 | version='.'.join([str(i) for i in self.postfix_version])) 290 | ) 291 | 292 | 293 | ### Let's Encrypt client IInstaller ### 294 | # https://github.com/letsencrypt/letsencrypt/blob/master/letsencrypt/interfaces.py#L232 295 | 296 | def get_all_names(self): 297 | """Returns all names that may be authenticated. 298 | :rtype: `list` of `str` 299 | """ 300 | var_names = ('myhostname', 'mydomain', 'myorigin') 301 | names_found = set() 302 | for num, line in enumerate(self.cf): 303 | num, found_var, found_value = parse_line((num, line)) 304 | if found_var in var_names: 305 | names_found.add(found_value) 306 | name_list = list(names_found) 307 | name_list.sort() 308 | return name_list 309 | 310 | def deploy_cert(self, domain, _cert_path, key_path, _chain_path, fullchain_path): 311 | """Deploy certificate. 312 | :param str domain: domain to deploy certificate file 313 | :param str cert_path: absolute path to the certificate file 314 | :param str key_path: absolute path to the private key file 315 | :param str chain_path: absolute path to the certificate chain file 316 | :param str fullchain_path: absolute path to the certificate fullchain 317 | file (cert plus chain) 318 | :raises .PluginError: when cert cannot be deployed 319 | """ 320 | self.wrangle_existing_config() 321 | self.ensure_cf_var("smtpd_tls_cert_file", fullchain_path, []) 322 | self.ensure_cf_var("smtpd_tls_key_file", key_path, []) 323 | self.set_domainwise_tls_policies() 324 | self.update_CAfile() 325 | 326 | def enhance(self, domain, enhancement, options=None): 327 | """Perform a configuration enhancement. 328 | :param str domain: domain for which to provide enhancement 329 | :param str enhancement: An enhancement as defined in 330 | :const:`~letsencrypt.constants.ENHANCEMENTS` 331 | :param options: Flexible options parameter for enhancement. 332 | Check documentation of 333 | :const:`~letsencrypt.constants.ENHANCEMENTS` 334 | for expected options for each enhancement. 335 | :raises .PluginError: If Enhancement is not supported, or if 336 | an error occurs during the enhancement. 337 | """ 338 | 339 | def supported_enhancements(self): 340 | """Returns a list of supported enhancements. 341 | :returns: supported enhancements which should be a subset of 342 | :const:`~letsencrypt.constants.ENHANCEMENTS` 343 | :rtype: :class:`list` of :class:`str` 344 | """ 345 | 346 | def get_all_certs_keys(self): 347 | """Retrieve all certs and keys set in configuration. 348 | :returns: tuples with form `[(cert, key, path)]`, where: 349 | - `cert` - str path to certificate file 350 | - `key` - str path to associated key file 351 | - `path` - file path to configuration file 352 | :rtype: list 353 | """ 354 | cert_materials = {'smtpd_tls_key_file': None, 355 | 'smtpd_tls_cert_file': None, 356 | } 357 | for num, line in enumerate(self.cf): 358 | num, found_var, found_value = parse_line((num, line)) 359 | if found_var in cert_materials.keys(): 360 | cert_materials[found_var] = found_value 361 | 362 | if not all(cert_materials.values()): 363 | cert_material_tuples = [] 364 | else: 365 | cert_material_tuples = [(cert_materials['smtpd_tls_cert_file'], 366 | cert_materials['smtpd_tls_key_file'], 367 | self.fn),] 368 | return cert_material_tuples 369 | 370 | def save(self, title=None, temporary=False): 371 | """Saves all changes to the configuration files. 372 | Both title and temporary are needed because a save may be 373 | intended to be permanent, but the save is not ready to be a full 374 | checkpoint. If an exception is raised, it is assumed a new 375 | checkpoint was not created. 376 | :param str title: The title of the save. If a title is given, the 377 | configuration will be saved as a new checkpoint and put in a 378 | timestamped directory. `title` has no effect if temporary is true. 379 | :param bool temporary: Indicates whether the changes made will 380 | be quickly reversed in the future (challenges) 381 | :raises .PluginError: when save is unsuccessful 382 | """ 383 | self.maybe_add_config_lines() 384 | 385 | def rollback_checkpoints(self, rollback=1): 386 | """Revert `rollback` number of configuration checkpoints. 387 | :raises .PluginError: when configuration cannot be fully reverted 388 | """ 389 | 390 | def recovery_routine(self): 391 | """Revert configuration to most recent finalized checkpoint. 392 | Remove all changes (temporary and permanent) that have not been 393 | finalized. This is useful to protect against crashes and other 394 | execution interruptions. 395 | :raises .errors.PluginError: If unable to recover the configuration 396 | """ 397 | 398 | def view_config_changes(self): 399 | """Display all of the LE config changes. 400 | :raises .PluginError: when config changes cannot be parsed 401 | """ 402 | 403 | def config_test(self): 404 | """Make sure the configuration is valid. 405 | :raises .MisconfigurationError: when the config is not in a usable state 406 | """ 407 | if os.geteuid() != 0: 408 | rc = os.system('sudo /usr/sbin/postfix check') 409 | else: 410 | rc = os.system('/usr/sbin/postfix check') 411 | if rc != 0: 412 | raise Exception('MisconfigurationError: Postfix failed self-check.') 413 | 414 | def restart(self): 415 | """Restart or refresh the server content. 416 | :raises .PluginError: when server cannot be restarted 417 | """ 418 | logger.info('Reloading postfix config...') 419 | if os.geteuid() != 0: 420 | rc = os.system("sudo service postfix reload") 421 | else: 422 | rc = os.system("service postfix reload") 423 | if rc != 0: 424 | raise Exception('PluginError: cannot restart postfix') 425 | 426 | def update_CAfile(self): 427 | os.system("cat /usr/share/ca-certificates/mozilla/*.crt > " + self.ca_file) 428 | 429 | 430 | def usage(): 431 | print ("Usage: %s starttls-everywhere.json /etc/postfix " 432 | "/etc/letsencrypt/live/example.com/" % sys.argv[0]) 433 | sys.exit(1) 434 | 435 | 436 | if __name__ == "__main__": 437 | import Config as config 438 | if len(sys.argv) != 4: 439 | usage() 440 | c = config.Config() 441 | c.load_from_json_file(sys.argv[1]) 442 | postfix_dir = sys.argv[2] 443 | le_lineage = sys.argv[3] 444 | pieces = [os.path.join(le_lineage, f) for f in ( 445 | "cert.pem", "privkey.pem", "chain.pem", "fullchain.pem")] 446 | if not os.path.isdir(le_lineage) or not all(os.path.isfile(p) for p in pieces) : 447 | print "Let's Encrypt directory", le_lineage, "does not appear to contain a valid lineage" 448 | print 449 | usage() 450 | cert, key, chain, fullchain = pieces 451 | pcgen = PostfixConfigGenerator(c, postfix_dir, fixup=True) 452 | pcgen.prepare() 453 | pcgen.deploy_cert("example.com", cert, key, chain, fullchain) 454 | pcgen.save() 455 | pcgen.restart() 456 | -------------------------------------------------------------------------------- /code_legacy/PostfixLogSummary.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import argparse 3 | import collections 4 | import os 5 | import re 6 | import sys 7 | import time 8 | 9 | import Config 10 | 11 | TIME_FORMAT = "%b %d %H:%M:%S" 12 | 13 | # TODO: There's more to be learned from postfix logs! Here's one sample 14 | # observed during failures from the sender vagrant vm: 15 | 16 | # Jun 6 00:21:31 precise32 postfix/smtpd[3648]: connect from localhost[127.0.0.1] 17 | # Jun 6 00:21:34 precise32 postfix/smtpd[3648]: lost connection after STARTTLS from localhost[127.0.0.1] 18 | # Jun 6 00:21:34 precise32 postfix/smtpd[3648]: disconnect from localhost[127.0.0.1] 19 | # Jun 6 00:21:56 precise32 postfix/master[3001]: reload -- version 2.9.6, configuration /etc/postfix 20 | # Jun 6 00:22:01 precise32 postfix/pickup[3674]: AF3B6480475: uid=0 from= 21 | # Jun 6 00:22:01 precise32 postfix/cleanup[3680]: AF3B6480475: message-id=<20140606002201.AF3B6480475@sender.example.com> 22 | # Jun 6 00:22:01 precise32 postfix/qmgr[3673]: AF3B6480475: from=, size=576, nrcpt=1 (queue active) 23 | # Jun 6 00:22:01 precise32 postfix/smtp[3682]: SSL_connect error to valid-example-recipient.com[192.168.33.7]:25: -1 24 | # Jun 6 00:22:01 precise32 postfix/smtp[3682]: warning: TLS library problem: 3682:error:140740BF:SSL routines:SSL23_CLIENT_HELLO:no protocols available:s23_clnt.c:381: 25 | # Jun 6 00:22:01 precise32 postfix/smtp[3682]: AF3B6480475: to=, relay=valid-example-recipient.com[192.168.33.7]:25, delay=0.06, delays=0.03/0.03/0/0, dsn=4.7.5, status=deferred (Cannot start TLS: handshake failure) 26 | # 27 | # Also: 28 | # Oct 10 19:12:13 sender postfix/smtp[1711]: 62D3F481249: to=, relay=valid-example-recipient.com[192.168.33.7]:25, delay=0.07, delays=0.03/0.01/0.03/0, dsn=4.7.4, status=deferred (TLS is required, but was not offered by host valid-example-recipient.com[192.168.33.7]) 29 | def get_counts(input, config, earliest_timestamp): 30 | seen_trusted = False 31 | 32 | counts = collections.defaultdict(lambda: collections.defaultdict(int)) 33 | tls_deferred = collections.defaultdict(int) 34 | # Typical line looks like: 35 | # Jun 12 06:24:14 sender postfix/smtp[9045]: Untrusted TLS connection established to valid-example-recipient.com[192.168.33.7]:25: TLSv1.1 with cipher AECDH-AES256-SHA (256/256 bits) 36 | # indicate a problem that should be alerted on. 37 | # ([^[]*) <--- any group of characters that is not "[" 38 | # Log lines for when a message is deferred for a TLS-related reason. These 39 | deferred_re = re.compile("relay=([^[ ]*).* status=deferred.*TLS") 40 | # Log lines for when a TLS connection was successfully established. These can 41 | # indicate the difference between Untrusted, Trusted, and Verified certs. 42 | connected_re = re.compile("([A-Za-z]+) TLS connection established to ([^[]*)") 43 | mx_to_domain_mapping = config.get_mx_to_domain_policy_map() 44 | 45 | timestamp = 0 46 | for line in sys.stdin: 47 | timestamp = time.strptime(line[0:15], TIME_FORMAT) 48 | if timestamp < earliest_timestamp: 49 | continue 50 | deferred = deferred_re.search(line) 51 | connected = connected_re.search(line) 52 | if connected: 53 | validation = connected.group(1) 54 | mx_hostname = connected.group(2).lower() 55 | if validation == "Trusted" or validation == "Verified": 56 | seen_trusted = True 57 | address_domains = config.get_address_domains(mx_hostname, mx_to_domain_mapping) 58 | if address_domains: 59 | domains_str = [ a.domain for a in address_domains ] 60 | d = ', '.join(domains_str) 61 | counts[d][validation] += 1 62 | counts[d]["all"] += 1 63 | elif deferred: 64 | mx_hostname = deferred.group(1).lower() 65 | tls_deferred[mx_hostname] += 1 66 | return (counts, tls_deferred, seen_trusted, timestamp) 67 | 68 | def print_summary(counts): 69 | for mx_hostname, validations in counts.items(): 70 | for validation, validation_count in validations.items(): 71 | if validation == "all": 72 | continue 73 | print mx_hostname, validation, validation_count / validations["all"], "of", validations["all"] 74 | 75 | if __name__ == "__main__": 76 | arg_parser = argparse.ArgumentParser(description='Detect delivery problems' 77 | ' in Postfix log files that may be caused by security policies') 78 | arg_parser.add_argument('-c', action="store_true", dest="cron", default=False) 79 | arg_parser.add_argument("policy_file", nargs='?', 80 | default=os.path.join("examples", "starttls-everywhere.json"), 81 | help="STARTTLS Everywhere policy file") 82 | 83 | args = arg_parser.parse_args() 84 | config = Config.Config() 85 | config.load_from_json_file(args.policy_file) 86 | 87 | last_timestamp_processed = 0 88 | timestamp_file = '/tmp/starttls-everywhere-last-timestamp-processed.txt' 89 | if os.path.isfile(timestamp_file): 90 | last_timestamp_processed = time.strptime(open(timestamp_file).read(), TIME_FORMAT) 91 | (counts, tls_deferred, seen_trusted, latest_timestamp) = get_counts(sys.stdin, config, last_timestamp_processed) 92 | with open(timestamp_file, "w") as f: 93 | f.write(time.strftime(TIME_FORMAT, latest_timestamp)) 94 | 95 | # If not running in cron, print an overall summary of log lines seen from known hosts. 96 | if not args.cron: 97 | print_summary(counts) 98 | if not seen_trusted: 99 | print 'No Trusted connections seen! Probably need to install a CAfile.' 100 | 101 | if len(tls_deferred) > 0: 102 | print "Some mail was deferred due to TLS problems:" 103 | for (k, v) in tls_deferred.iteritems(): 104 | print "%s: %s" % (k, v) 105 | -------------------------------------------------------------------------------- /code_legacy/TestPostfixConfigGenerator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from __future__ import absolute_import 5 | from __future__ import division 6 | from __future__ import print_function 7 | from __future__ import unicode_literals 8 | 9 | import io 10 | import logging 11 | import unittest 12 | 13 | import Config 14 | import PostfixConfigGenerator as pcg 15 | 16 | 17 | logger = logging.getLogger(__name__) 18 | logger.addHandler(logging.StreamHandler()) 19 | 20 | 21 | # Fake Postfix Configs 22 | names_only_config = """myhostname = mail.fubard.org 23 | mydomain = fubard.org 24 | myorigin = fubard.org""" 25 | 26 | 27 | certs_only_config = ( 28 | """smtpd_tls_cert_file = /etc/letsencrypt/live/www.fubard.org/fullchain.pem 29 | smtpd_tls_key_file = /etc/letsencrypt/live/www.fubard.org/privkey.pem""") 30 | 31 | 32 | def GetFakeOpen(fake_file_contents): 33 | fake_file = io.StringIO() 34 | # cast this to unicode for py2 35 | fake_file.write(fake_file_contents) 36 | fake_file.seek(0) 37 | 38 | def FakeOpen(_): 39 | return fake_file 40 | 41 | return FakeOpen 42 | 43 | 44 | class TestPostfixConfigGenerator(unittest.TestCase): 45 | 46 | def setUp(self): 47 | self.fopen_names_only_config = GetFakeOpen(names_only_config) 48 | self.fopen_certs_only_config = GetFakeOpen(certs_only_config) 49 | self.fopen_no_certs_only_config = self.fopen_names_only_config 50 | 51 | #self.config = Config.Config() 52 | self.config = None 53 | self.postfix_dir = 'tests/' 54 | 55 | def tearDown(self): 56 | pass 57 | 58 | def testGetAllNames(self): 59 | sorted_names = ['fubard.org', 'mail.fubard.org'] 60 | postfix_config_gen = pcg.PostfixConfigGenerator( 61 | self.config, 62 | self.postfix_dir, 63 | fixup=True, 64 | fopen=self.fopen_names_only_config 65 | ) 66 | self.assertEqual(sorted_names, postfix_config_gen.get_all_names()) 67 | 68 | def testGetAllCertAndKeys(self): 69 | return_vals = [('/etc/letsencrypt/live/www.fubard.org/fullchain.pem', 70 | '/etc/letsencrypt/live/www.fubard.org/privkey.pem', 71 | 'tests/main.cf'),] 72 | postfix_config_gen = pcg.PostfixConfigGenerator( 73 | self.config, 74 | self.postfix_dir, 75 | fixup=True, 76 | fopen=self.fopen_certs_only_config 77 | ) 78 | self.assertEqual(return_vals, postfix_config_gen.get_all_certs_keys()) 79 | 80 | def testGetAllCertsAndKeys_With_None(self): 81 | postfix_config_gen = pcg.PostfixConfigGenerator( 82 | self.config, 83 | self.postfix_dir, 84 | fixup=True, 85 | fopen=self.fopen_no_certs_only_config 86 | ) 87 | self.assertEqual([], postfix_config_gen.get_all_certs_keys()) 88 | 89 | 90 | if __name__ == '__main__': 91 | unittest.main() 92 | -------------------------------------------------------------------------------- /policy.json.asc: -------------------------------------------------------------------------------- 1 | -----BEGIN PGP SIGNATURE----- 2 | 3 | iQFMBAABCgA2FiEELDGd5HBmwgDf3E1rhCrqQMW81uEFAl7FZ/gYHHN0YXJ0dGxz 4 | LXBvbGljeUBlZmYub3JnAAoJEIQq6kDFvNbh97UH/0EgsDI1VPBwhTSKgR4dZi5X 5 | HHS9uwWnhdHh2P9GhOul30n/qL4G3tLhTWXehdnOaRRhtYdrlGGmvffo2Rk+8xSI 6 | L96HvnHR9zL4JS5MhVudsaYm12V87vHWhCoMR495gGSRMbxwHU9IZNRklc/KWvw6 7 | M8LIXQDUM9dcmvg6S1Lak0/h8o6gRrJOul0SwDKYy5aM8oWrOUXpxiSwedjIetnX 8 | G1XCBcr9UJNMrZc2/yOYwAEKjjS8uYy87O0oCfqCBNKT5nMKsnZuBeooL2uLT90Q 9 | uryOxRuV3GZkBVN34HmpAA1D+MJ1dXM+yPVEy3WhHBJ6YOigeP7vrd+Y9leuXVI= 10 | =s+pE 11 | -----END PGP SIGNATURE----- 12 | -------------------------------------------------------------------------------- /schema/policy-0.1.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "$id": "https://raw.githubusercontent.com/EFForg/starttls-everywhere/master/schema/policy-0.1.schema.json", 4 | "title": "STARTTLS-Everywhere policy list", 5 | "description": "A list of email domains who meet a minimum set of security requirements.", 6 | "type": "object", 7 | "properties": { 8 | "timestamp": { 9 | "description": "When this configuration file was distributed/fetched, in UTC. Can be in epoch seconds from 00:00:00 UTC on 1 January 1970, or a string yyyy-MM-dd'T'HH:mm:ssZZZZ. This field will be monotonically increasing for every update to the policy file. When updating this policy file, should validate that the timestamp is greater than or equal to the existing one.", 10 | "type": ["string", "integer"], 11 | "minimum": 0, 12 | "pattern": "^(?:[1-9]\\d{3}-(?:(?:0[1-9]|1[0-2])-(?:0[1-9]|1\\d|2[0-8])|(?:0[13-9]|1[0-2])-(?:29|30)|(?:0[13578]|1[02])-31)|(?:[1-9]\\d(?:0[48]|[2468][048]|[13579][26])|(?:[2468][048]|[13579][26])00)-02-29)T(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(\\.\\d+)?(?:Z|[+-][01]\\d:[0-5]\\d)$" 13 | }, 14 | "expires": { 15 | "description": "When this configuration file expires, in UTC. Can be in epoch seconds from 00:00:00 UTC on 1 January 1970, or a string yyyy-MM-dd'T'HH:mm:ssZZZZ. If the file has ceased being regularly updated for any reason, and the policy file has expired, the MTA should fall-back to opportunistic TLS for e-mail delivery, and the system operator should be alerted.", 16 | "type": ["string", "integer"], 17 | "minimum": 0, 18 | "pattern": "^(?:[1-9]\\d{3}-(?:(?:0[1-9]|1[0-2])-(?:0[1-9]|1\\d|2[0-8])|(?:0[13-9]|1[0-2])-(?:29|30)|(?:0[13578]|1[02])-31)|(?:[1-9]\\d(?:0[48]|[2468][048]|[13579][26])|(?:[2468][048]|[13579][26])00)-02-29)T(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(\\.\\d+)?(?:Z|[+-][01]\\d:[0-5]\\d)$" 19 | }, 20 | "version": { 21 | "description": "Version of the configuration file.", 22 | "type": "string", 23 | "const": "0.1" 24 | }, 25 | "author": { 26 | "description": "Author of policy list.", 27 | "type": "string" 28 | }, 29 | "policy-aliases": { 30 | "description": "A mapping of alias names onto a policy. Domains in the `policies` field can refer to policies defined in this object.", 31 | "type": "object", 32 | "patternProperties": { 33 | ".": { "$ref": "#/definitions/policyDefinition" } 34 | }, 35 | "additionalProperties": false 36 | }, 37 | "policies": { 38 | "description": "A mapping of mail domains (the part of an address after the `@`) onto a policy. Matching of mail domains is on an exact-match basis. For instance, `eff.org` would be listed separately from `lists.eff.org`. Fields in this policy specify security requirements that should be applied when connecting to any MTA server for a given mail domain. If the `mode` is `testing`, then the sender should not stop mail delivery on policy failures, but should produce logging information.", 39 | "type": "object", 40 | "patternProperties": { 41 | "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z0-9]{2,63}$": { 42 | "oneOf": [ 43 | { "$ref": "#/definitions/policyAliasReference" }, 44 | { "$ref": "#/definitions/policyDefinition" } 45 | ] 46 | } 47 | }, 48 | "additionalProperties": false 49 | } 50 | }, 51 | "required": [ "timestamp", "expires", "version", "policies", "policy-aliases" ], 52 | "additionalProperties": false, 53 | "definitions": { 54 | "policyAliasReference": { 55 | "type": "object", 56 | "properties": { 57 | "policy-alias": { 58 | "type": "string", 59 | "description": "If set, other fields are ignored. This value should be a key in the upper-level `policy-aliases` configuration object. The policy for this domain will be configured as the denoted policy in the `policy-aliases` object." 60 | } 61 | }, 62 | "required": [ "policy-alias" ], 63 | "additionalProperties": false 64 | }, 65 | "policyDefinition": { 66 | "type": "object", 67 | "properties": { 68 | "mode": { 69 | "description": "Default: testing (required)\nEither testing or enforce. If testing is set, then any failure in TLS negotiation is logged and reported, but the message is sent over the insecure communication.", 70 | "enum": [ "testing", "enforce" ] 71 | }, 72 | "mxs": { 73 | "type": "array", 74 | "description": "A list of hostnames that the recipient email server's certificates could be valid for. If the server's certificate matches no entry in mxs, the MTA should fail delivery or log an advisory failure, according to mode. Entries in the mxs list can either be a suffix indicated by a leading dot .example.net or a fully qualified domain name mail.example.com. Arbitrarily deep subdomains can match a particular suffix. For instance, mta7.am0.yahoodns.net would match .yahoodns.net.", 75 | "minItems": 1, 76 | "uniqueItems": true, 77 | "items": { 78 | "type": "string", 79 | "pattern": "^\\.?([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z0-9]{2,63}$" 80 | }, 81 | "additionalItems": false 82 | } 83 | }, 84 | "required": [ "mxs", "mode" ], 85 | "additionalProperties": false 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /scripts/starttls-policy.cron.d: -------------------------------------------------------------------------------- 1 | SHELL=/bin/sh 2 | PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin 3 | 4 | 0 */12 * * * root perl -e "sleep int(rand(3600))" && /bin/sh ./update_and_verify.sh 5 | -------------------------------------------------------------------------------- /scripts/update_and_verify.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Depends on wget and gpgv 4 | # Since this tries to modify an /etc/ directory, should be 5 | # run with escalated permissions. 6 | 7 | set -e 8 | 9 | JSON_FILE="policy.json" 10 | SIG_FILE="$JSON_FILE.asc" 11 | 12 | REMOTE_DIR="https://dl.eff.org/starttls-everywhere" 13 | LOCAL_DIR="/etc/starttls-policy" 14 | 15 | TMP_DIR="$(mktemp -d)" 16 | TMP_EXT="tmp" 17 | 18 | clean_and_exit() { 19 | rc=$? 20 | rm -rf "$TMP_DIR" 21 | exit $rc 22 | } 23 | 24 | # traps on regular exit, SIGHUP SIGINT SIGQUIT SIGTERM 25 | trap clean_and_exit 0 1 2 3 15 26 | 27 | # Fetch remote source 28 | wget --quiet "$REMOTE_DIR/$JSON_FILE" -O "$TMP_DIR/$JSON_FILE" 29 | wget --quiet "$REMOTE_DIR/$SIG_FILE" -O "$TMP_DIR/$SIG_FILE" 30 | 31 | "$(dirname $0)/verify.sh" $LOCAL_DIR $TMP_DIR 32 | 33 | # Perform update from tmp => local 34 | mkdir -p $LOCAL_DIR 35 | 36 | cp "$TMP_DIR/$JSON_FILE" "$LOCAL_DIR/$JSON_FILE.$TMP_EXT" 37 | cp "$TMP_DIR/$SIG_FILE" "$LOCAL_DIR/$SIG_FILE.$TMP_EXT" 38 | 39 | mv "$LOCAL_DIR/$JSON_FILE.$TMP_EXT" "$LOCAL_DIR/$JSON_FILE" 40 | mv "$LOCAL_DIR/$SIG_FILE.$TMP_EXT" "$LOCAL_DIR/$SIG_FILE" 41 | 42 | exit 0 43 | -------------------------------------------------------------------------------- /scripts/verify.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Depends on gpgv 4 | # Arguments: 5 | # verify.sh 6 | # Verifies signature on both old and new policy, and ensures 7 | # timestamp on new policy is higher. 8 | 9 | set -e 10 | 11 | JSON_FILE="policy.json" 12 | SIG_FILE="$JSON_FILE.asc" 13 | 14 | OLD_DIR=$1 15 | NEW_DIR=$2 16 | 17 | AUTHORITY_FINGERPRINT="B693F33372E965D76D55368616EEA65D03326C9D" 18 | 19 | TMP_DIR="$(mktemp -d)" 20 | 21 | clean_and_exit() { 22 | rc=$? 23 | rm -rf "$TMP_DIR" 24 | exit $rc 25 | } 26 | 27 | # traps on regular exit, SIGHUP SIGINT SIGQUIT SIGTERM 28 | trap clean_and_exit 0 1 2 3 15 29 | 30 | # This is OpenPGP certificate B693F33372E965D76D55368616EEA65D03326C9D 31 | # in RFC4880#section-11.1 Transferable Public Key format: 32 | base64 -d >"$TMP_DIR/authority.key" <"$TMP_DIR/gpgv.status" --keyring="$TMP_DIR/authority.key" "$NEW_DIR/$SIG_FILE" "$NEW_DIR/$JSON_FILE" 70 | 71 | get_sig_epoch_date() { 72 | awk '($1 == "[GNUPG:]" && $2 == "VALIDSIG" && $12 == "'$AUTHORITY_FINGERPRINT'") { print $5 }' 73 | } 74 | 75 | if [ -r "$OLD_DIR/$JSON_FILE" ] && [ -r "$OLD_DIR/$SIG_FILE" ] ; then 76 | gpgv --status-fd 3 3>"$TMP_DIR/gpgv.old.status" --keyring="$TMP_DIR/authority.key" "$OLD_DIR/$SIG_FILE" "$OLD_DIR/$JSON_FILE" 77 | OLD_DATE=$(get_sig_epoch_date < "$TMP_DIR/gpgv.old.status") 78 | NEW_DATE=$(get_sig_epoch_date < "$TMP_DIR/gpgv.status") 79 | if [ $NEW_DATE -lt $OLD_DATE ] ; then 80 | printf "Rollback detected (old date %d, new date %d)!\n" "$OLD_DATE" "$NEW_DATE" >&2 81 | exit 1 82 | fi 83 | fi 84 | 85 | exit 0 86 | -------------------------------------------------------------------------------- /share/golden-domains.txt: -------------------------------------------------------------------------------- 1 | 163.com 2 | aol.com 3 | bigpond.com 4 | comcast.net 5 | craigslist.org 6 | facebook.com 7 | gmail.com 8 | gmx.de 9 | hotmail.com 10 | icloud.com 11 | live.com 12 | mac.com 13 | me.com 14 | msn.com 15 | naver.com 16 | outlook.com 17 | qq.com 18 | rocketmail.com 19 | rogers.com 20 | salesforce.com 21 | sbcglobal.net 22 | shaw.ca 23 | sympatico.ca 24 | t-online.de 25 | ukr.net 26 | vtext.com 27 | web.de 28 | wp.pl 29 | yahoo.com 30 | yahoogroups.com 31 | yandex.ru 32 | ymail.com 33 | -------------------------------------------------------------------------------- /test_rules/bigger_test_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.1", 3 | "author": "Electronic Frontier Foundation https://eff.org", 4 | "expires": "2015-08-01T12:00:00+08:00", 5 | "timestamp": 1401414363, 6 | "policy-aliases": {}, 7 | "policies": { 8 | "yahoodns.net": { 9 | "mode": "testing", 10 | "mxs": [".yahoodns.net"] 11 | }, 12 | "eff.org": { 13 | "mode": "enforce", 14 | "mxs": [".eff.org"] 15 | }, 16 | "google.com": { 17 | "mode": "testing", 18 | "mxs": [".google.com"] 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /test_rules/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.1", 3 | "author": "Electronic Frontier Foundation", 4 | "expires": 1404677353, 5 | "timestamp": 1401093333, 6 | "policy-aliases": {}, 7 | "policies": { 8 | "valid.example-recipient.com": { 9 | "mode": "enforce", 10 | "mxs": [".valid.example-recipient.com"] 11 | } 12 | } 13 | } 14 | 15 | -------------------------------------------------------------------------------- /tests/policy_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import unittest 4 | import os 5 | import json 6 | 7 | from jsonschema import validate 8 | 9 | ROOT_DIR, _ = os.path.split(os.path.dirname(os.path.abspath(__file__))) 10 | POLICY_FILE = os.path.join(ROOT_DIR, "policy.json") 11 | SCHEMA_FILE = os.path.join(ROOT_DIR, "schema", "policy-0.1.schema.json") 12 | 13 | class TestPolicyList(unittest.TestCase): 14 | def setUp(self): 15 | with open(POLICY_FILE) as pf: 16 | self.policy = json.load(pf) 17 | with open(SCHEMA_FILE) as sf: 18 | self.schema = json.load(sf) 19 | 20 | def test_schema_valid(self): 21 | self.assertIsNone(validate(self.policy, self.schema)) 22 | 23 | def test_links_ok(self): 24 | policy_list = self.policy["policies"] 25 | policy_aliases = self.policy["policy-aliases"] 26 | 27 | for pol_key in policy_list: 28 | pol = policy_list[pol_key] 29 | if "policy-alias" in pol: 30 | self.assertIn(pol["policy-alias"], policy_aliases) 31 | 32 | 33 | if __name__ == '__main__': 34 | unittest.main() 35 | -------------------------------------------------------------------------------- /tests/test-requirements.txt: -------------------------------------------------------------------------------- 1 | jsonschema>=3.0.0 2 | -------------------------------------------------------------------------------- /tests/validate_signature.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Arguments: 4 | # ./validate_signature.sh 5 | 6 | set -e 7 | 8 | JSON_FILE="policy.json" 9 | SIG_FILE="$JSON_FILE.asc" 10 | 11 | REMOTE_DIR="https://dl.eff.org/starttls-everywhere" 12 | 13 | TMP_DIR="$(mktemp -d)" 14 | 15 | clean_and_exit() { 16 | rc=$? 17 | rm -rf "$TMP_DIR" 18 | exit $rc 19 | } 20 | 21 | # traps on regular exit, SIGHUP SIGINT SIGQUIT SIGTERM 22 | trap clean_and_exit 0 1 2 3 15 23 | 24 | # Fetch remote source 25 | wget --quiet "$REMOTE_DIR/$JSON_FILE" -O "$TMP_DIR/$JSON_FILE" 26 | wget --quiet "$REMOTE_DIR/$SIG_FILE" -O "$TMP_DIR/$SIG_FILE" 27 | 28 | ./scripts/verify.sh $TMP_DIR ./ 29 | -------------------------------------------------------------------------------- /tools/CheckSTARTTLS.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import os 4 | import errno 5 | import smtplib 6 | import socket 7 | import subprocess 8 | import re 9 | import json 10 | import collections 11 | 12 | import dns.resolver 13 | from M2Crypto import X509 14 | from publicsuffix import PublicSuffixList 15 | 16 | public_suffix_list = PublicSuffixList() 17 | CERTS_OBSERVED = 'certs-observed' 18 | 19 | def mkdirp(path): 20 | try: 21 | os.makedirs(path) 22 | except OSError as exc: 23 | if exc.errno == errno.EEXIST and os.path.isdir(path): 24 | pass 25 | else: raise 26 | 27 | def extract_names(pem): 28 | """Return a set of DNS subject names from PEM-encoded leaf cert.""" 29 | leaf = X509.load_cert_string(pem, X509.FORMAT_PEM) 30 | 31 | subj = leaf.get_subject() 32 | # Certs have a "subject" identified by a Distingushed Name (DN). 33 | # Host certs should also have a Common Name (CN) with a DNS name. 34 | common_names = subj.get_entries_by_nid(subj.nid['CN']) 35 | common_names = [name.get_data().as_text() for name in common_names] 36 | try: 37 | # The SAN extension allows one cert to cover multiple domains 38 | # and permits DNS wildcards. 39 | # http://www.digicert.com/subject-alternative-name.htm 40 | # The field is a comma delimited list, e.g.: 41 | # >>> twitter_cert.get_ext('subjectAltName').get_value() 42 | # 'DNS:www.twitter.com, DNS:twitter.com' 43 | alt_names = leaf.get_ext('subjectAltName').get_value() 44 | alt_names = alt_names.split(', ') 45 | alt_names = [name.partition(':') for name in alt_names] 46 | alt_names = [name for prot, _, name in alt_names if prot == 'DNS'] 47 | except: 48 | alt_names = [] 49 | return set(common_names + alt_names) 50 | 51 | def tls_connect(mx_host, mail_domain): 52 | """Attempt a STARTTLS connection with openssl and save the output.""" 53 | if supports_starttls(mx_host): 54 | # smtplib doesn't let us access certificate information, 55 | # so shell out to openssl. 56 | try: 57 | output = subprocess.check_output( 58 | """openssl s_client \ 59 | -starttls smtp -connect %s:25 -showcerts /dev/null 61 | """ % mx_host, shell = True) 62 | except subprocess.CalledProcessError: 63 | print "Failed s_client" 64 | return 65 | 66 | # Save a copy of the certificate for later analysis 67 | with open(os.path.join(CERTS_OBSERVED, mail_domain, mx_host), "w") as f: 68 | f.write(output) 69 | 70 | def valid_cert(filename): 71 | """Return true if the certificate is valid. 72 | 73 | Note: CApath must have hashed symlinks to the trust roots. 74 | TODO: Include the -attime flag based on file modification time.""" 75 | 76 | if open(filename).read().find("-----BEGIN CERTIFICATE-----") == -1: 77 | return False 78 | try: 79 | # The file contains both the leaf cert and any intermediates, so we pass it 80 | # as both the cert to validate and as the "untrusted" chain. 81 | output = subprocess.check_output("""openssl verify -CApath /home/jsha/mozilla/ -purpose sslserver \ 82 | -untrusted "%s" \ 83 | "%s" 84 | """ % (filename, filename), shell = True) 85 | return True 86 | except subprocess.CalledProcessError: 87 | return False 88 | 89 | def check_certs(mail_domain): 90 | """ 91 | Return "" if any certs for any mx domains pointed to by mail_domain 92 | were invalid, and a public suffix for one if they were all valid 93 | """ 94 | dir = os.path.join(CERTS_OBSERVED, mail_domain) 95 | if not os.path.exists(dir): 96 | collect(mail_domain) 97 | names = set() 98 | for mx_hostname in os.listdir(dir): 99 | filename = os.path.join(dir, mx_hostname) 100 | if not valid_cert(filename): 101 | return "" 102 | else: 103 | new_names = extract_names_from_openssl_output(filename) 104 | new_names = set(public_suffix_list.get_public_suffix(n) for n in new_names) 105 | names.update(new_names) 106 | if len(names) >= 1: 107 | # Hack: Just pick an arbitrary suffix for now. Do something cleverer later. 108 | return names.pop() 109 | else: 110 | return "" 111 | 112 | def common_suffix(hosts): 113 | num_components = min(len(h.split(".")) for h in hosts) 114 | longest_suffix = "" 115 | for i in range(1, num_components + 1): 116 | suffixes = set(".".join(h.split(".")[-i:]) for h in hosts) 117 | if len(suffixes) == 1: 118 | longest_suffix = suffixes.pop() 119 | else: 120 | return longest_suffix 121 | return longest_suffix 122 | 123 | def extract_names_from_openssl_output(certificates_file): 124 | openssl_output = open(certificates_file, "r").read() 125 | cert = re.findall("-----BEGIN CERTIFICATE-----.*?-----END CERTIFICATE-----", openssl_output, flags = re.DOTALL) 126 | return extract_names(cert[0]) 127 | 128 | def supports_starttls(mx_host): 129 | try: 130 | smtpserver = smtplib.SMTP(mx_host, 25, timeout = 2) 131 | smtpserver.ehlo() 132 | smtpserver.starttls() 133 | return True 134 | print "Success: %s" % mx_host 135 | except socket.error as e: 136 | print "Connection to %s failed: %s" % (mx_host, e.strerror) 137 | return False 138 | except smtplib.SMTPException, e: 139 | # In order to talk to some hosts, you need to run this from a host that has a 140 | # reverse DNS entry. AWS instances all have reverse DNS, as an example. 141 | if e[0] == 554: 142 | print e[1] 143 | else: 144 | print "No STARTTLS support on %s" % mx_host, e[0] 145 | return False 146 | 147 | def min_tls_version(mail_domain): 148 | protocols = [] 149 | for mx_hostname in os.listdir(os.path.join(CERTS_OBSERVED, mail_domain)): 150 | filename = os.path.join(CERTS_OBSERVED, mail_domain, mx_hostname) 151 | contents = open(filename).read() 152 | protocol = re.findall("Protocol : (.*)", contents)[0] 153 | protocols.append(protocol) 154 | return min(protocols) 155 | 156 | def collect(mail_domain): 157 | """ 158 | Attempt to connect to each MX hostname for mail_doman and negotiate STARTTLS. 159 | Store the output in a directory with the same name as mail_domain to make 160 | subsequent analysis faster. 161 | """ 162 | print "Checking domain %s" % mail_domain 163 | mkdirp(os.path.join(CERTS_OBSERVED, mail_domain)) 164 | answers = dns.resolver.query(mail_domain, 'MX') 165 | for rdata in answers: 166 | mx_host = str(rdata.exchange).rstrip(".") 167 | tls_connect(mx_host, mail_domain) 168 | 169 | if __name__ == '__main__': 170 | """Consume a target list of domains and output a configuration file for those domains.""" 171 | if len(sys.argv) < 2: 172 | print("Usage: CheckSTARTTLS.py list-of-domains.txt > output.json") 173 | 174 | config = collections.defaultdict(dict) 175 | 176 | for input in sys.argv[1:]: 177 | for domain in open(input).readlines(): 178 | domain = domain.strip() 179 | suffix = check_certs(domain) 180 | if suffix != "": 181 | min_version = min_tls_version(domain) 182 | suffix_match = "." + suffix 183 | config["acceptable-mxs"][domain] = { 184 | "accept-mx-domains": [suffix_match] 185 | } 186 | config["tls-policies"][suffix_match] = { 187 | "require-tls": True, 188 | "min-tls-version": min_version 189 | } 190 | 191 | print json.dumps(config, indent=2, sort_keys=True) 192 | -------------------------------------------------------------------------------- /tools/ProcessGoogleSTARTTLSDomains.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Process Google's TLS delivery data from 4 | https://www.google.com/transparencyreport/saferemail/data/?hl=en 5 | to look for outbound domains that can negotiate an encrypted 6 | connection >99% of the time. 7 | 8 | Usage: 9 | ./ProcessGoogleSTARTTLSDomains.py google-starttls-domains.csv 10 | """ 11 | import csv 12 | import codecs 13 | import sys 14 | from collections import defaultdict 15 | 16 | csvreader = csv.reader(codecs.open(sys.argv[1], "rU", "utf-8"), delimiter=',', quotechar='"') 17 | d = defaultdict(set) 18 | # Google's report doesn't include gmail.com because it's local delivery, but we 19 | # know they support STARTTLS, so manually include them. 20 | d["gmail.com"] = set([1]) 21 | for (address_suffix, hostname_suffix, direction, region, region_name, fraction_encrypted) in csvreader: 22 | if direction == "outbound": 23 | # Some domains exist in many TLDs and are summarized as, e.g. yahoo.{...}. 24 | # We're tryingto get a solid list of the relevant TLDs, but in the meantime 25 | # just use .com. 26 | address_suffix = address_suffix.replace("{...}", "com") 27 | try: 28 | d[address_suffix].add(float(fraction_encrypted)) 29 | except ValueError: 30 | pass 31 | 32 | for address_suffix, fraction_encrypted in d.iteritems(): 33 | if min(fraction_encrypted) >= 0.99: 34 | print address_suffix 35 | --------------------------------------------------------------------------------