├── .bundler-version
├── .cfignore
├── .codeinventory.yml
├── .gitignore
├── .nvmrc
├── CONTRIBUTING.md
├── Gemfile
├── Gemfile.lock
├── LICENSE.md
├── README.md
├── SECURITY.md
├── Staticfile
├── _config.yml
├── _data
    └── navigation.yml
├── _includes
    ├── banner.html
    ├── footer.html
    ├── head.html
    ├── header.html
    ├── scripts.html
    └── sidenav.html
├── _layouts
    └── base.html
├── assets
    ├── images
    │   ├── mixed-content.png
    │   ├── with-http-headers.png
    │   └── with-https-headers.png
    └── sass
    │   └── main.scss
├── compliance
    ├── bod-18-01
    │   ├── compliance.py
    │   ├── download.py
    │   ├── scratch.py
    │   └── utils.py
    └── m-15-13
    │   ├── compliance.py
    │   ├── data
    │       ├── analytics-domains
    │       │   └── all-domains-30-days-2017-01-01.csv
    │       ├── censys-2016-08-04.csv
    │       ├── censys-2016-12-05.csv
    │       ├── censys-2016-12-31.csv
    │       ├── dap-2016-07-12.csv
    │       ├── dap-2016-12-05.csv
    │       ├── dap-2016-12-31.csv
    │       ├── eot-2012.csv
    │       ├── eot-2016.csv
    │       ├── eot-seeds
    │       │   ├── eot-2012-seeds.csv
    │       │   └── eot-2016-seeds.csv
    │       ├── parents-2015-06-13.csv
    │       ├── parents-2015-06-19.csv
    │       ├── parents-2015-06-26.csv
    │       ├── parents-2015-07-03.csv
    │       ├── parents-2015-07-10.csv
    │       ├── parents-2015-07-17.csv
    │       ├── parents-2015-07-24.csv
    │       ├── parents-2015-07-31.csv
    │       ├── parents-2015-08-07.csv
    │       ├── parents-2015-08-14.csv
    │       ├── parents-2015-08-21.csv
    │       ├── parents-2015-08-28.csv
    │       ├── parents-2015-09-04.csv
    │       ├── parents-2015-09-11.csv
    │       ├── parents-2015-09-18.csv
    │       ├── parents-2015-09-25.csv
    │       ├── parents-2015-10-09.csv
    │       ├── parents-2015-10-16.csv
    │       ├── parents-2015-10-22.csv
    │       ├── parents-2015-10-30.csv
    │       ├── parents-2015-11-06.csv
    │       ├── parents-2015-11-13.csv
    │       ├── parents-2015-11-21.csv
    │       ├── parents-2015-11-28.csv
    │       ├── parents-2015-12-04.csv
    │       ├── parents-2015-12-10.csv
    │       ├── parents-2015-12-18.csv
    │       ├── parents-2015-12-26.csv
    │       ├── parents-2016-01-02.csv
    │       ├── parents-2016-01-08.csv
    │       ├── parents-2016-01-15.csv
    │       ├── parents-2016-01-23.csv
    │       ├── parents-2016-01-29.csv
    │       ├── parents-2016-02-04.csv
    │       ├── parents-2016-02-08.csv
    │       ├── parents-2016-02-17.csv
    │       ├── parents-2016-02-26.csv
    │       ├── parents-2016-03-03.csv
    │       ├── parents-2016-03-13.csv
    │       ├── parents-2016-03-18.csv
    │       ├── parents-2016-03-25.csv
    │       ├── parents-2016-04-01.csv
    │       ├── parents-2016-04-08.csv
    │       ├── parents-2016-04-15.csv
    │       ├── parents-2016-04-22.csv
    │       ├── parents-2016-04-29.csv
    │       ├── parents-2016-05-06.csv
    │       ├── parents-2016-05-13.csv
    │       ├── parents-2016-05-20.csv
    │       ├── parents-2016-05-27.csv
    │       ├── parents-2016-06-03.csv
    │       ├── parents-2016-07-05.csv
    │       ├── parents-2016-07-08.csv
    │       ├── parents-2016-07-15.csv
    │       ├── parents-2016-07-22.csv
    │       ├── parents-2016-07-29.csv
    │       ├── parents-2016-08-05.csv
    │       ├── parents-2016-08-12.csv
    │       ├── parents-2016-08-19.csv
    │       ├── parents-2016-08-26.csv
    │       ├── parents-2016-09-02.csv
    │       ├── parents-2016-09-09.csv
    │       ├── parents-2016-09-16.csv
    │       ├── parents-2016-09-23.csv
    │       ├── parents-2016-10-03.csv
    │       ├── parents-2016-10-07.csv
    │       ├── parents-2016-10-14.csv
    │       ├── parents-2016-10-21.csv
    │       ├── parents-2016-10-28.csv
    │       ├── parents-2016-11-04.csv
    │       ├── parents-2016-11-11.csv
    │       ├── parents-2016-11-18.csv
    │       ├── parents-2016-11-25.csv
    │       ├── parents-2016-12-02.csv
    │       ├── parents-2016-12-05.csv
    │       ├── parents-2016-12-08.csv
    │       ├── parents-2016-12-17.csv
    │       ├── parents-2016-12-21.csv
    │       ├── parents-2016-12-24.csv
    │       ├── parents-2016-12-26.csv
    │       ├── parents-2016-12-28.csv
    │       └── parents-2016-12-31.csv
    │   ├── reports.py
    │   └── utils.py
├── favicon.ico
├── gulpfile.js
├── package-lock.json
├── package.json
├── pages
    ├── apis.md
    ├── certificates.md
    ├── everything.md
    ├── faq.md
    ├── guide.md
    ├── hsts.md
    ├── index.md
    ├── mixed-content.md
    ├── resources.md
    ├── sni.md
    └── technical.md
└── resources
    └── implementing-https-links.md


/.bundler-version:
--------------------------------------------------------------------------------
1 | 2.0.1
2 | 


--------------------------------------------------------------------------------
/.cfignore:
--------------------------------------------------------------------------------
 1 | /compliance
 2 | /resources
 3 | ".ruby-version"
 4 | ".jekyll-metadata"
 5 | ".gitignore"
 6 | node_modules
 7 | todo.txt
 8 | README.md
 9 | CONTRIBUTING.md
10 | LICENSE.md
11 | .nvmrc
12 | 


--------------------------------------------------------------------------------
/.codeinventory.yml:
--------------------------------------------------------------------------------
 1 | name: https.cio.gov
 2 | description: 'Website to support M-15-13 implementation and HTTPS best practices.'
 3 | license: 'https://github.com/GSA/https/blob/master/LICENSE.md'
 4 | openSourceProject: 1
 5 | governmentWideReuseProject: 0
 6 | tags:
 7 |     - https
 8 |     - tls
 9 |     - policy
10 |     - jekyll
11 |     - ssl
12 |     - encryption
13 | contact:
14 |     email: https@cio.gov
15 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | .~lock*
 2 | .sass-cache
 3 | _site
 4 | .DS_Store
 5 | todo.txt
 6 | .jekyll-metadata
 7 | .ruby-version
 8 | *.pyc
 9 | creds.txt
10 | node_modules
11 | /assets/vendor
12 | /assets/css
13 | /vendor
14 | /compliance/*/cache
15 | /compliance/*/data
16 | /compliance/*/*.gz
17 | .bundle
18 | 


--------------------------------------------------------------------------------
/.nvmrc:
--------------------------------------------------------------------------------
1 | 12
2 | 


--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
 1 | ## Public domain
 2 | 
 3 | This project is in the public domain within the United States, and
 4 | copyright and related rights in the work worldwide are waived through
 5 | the [CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/).
 6 | 
 7 | All contributions to this project will be released under the CC0
 8 | dedication. By submitting a pull request, you are agreeing to comply
 9 | with this waiver of copyright interest.
10 | 


--------------------------------------------------------------------------------
/Gemfile:
--------------------------------------------------------------------------------
1 | source "https://rubygems.org"
2 | 
3 | gem 'jekyll'
4 | 
5 | # TODO: remove
6 | gem 'redcarpet'
7 | gem 'rouge'
8 | 


--------------------------------------------------------------------------------
/Gemfile.lock:
--------------------------------------------------------------------------------
 1 | GEM
 2 |   remote: https://rubygems.org/
 3 |   specs:
 4 |     addressable (2.5.2)
 5 |       public_suffix (>= 2.0.2, < 4.0)
 6 |     colorator (1.1.0)
 7 |     concurrent-ruby (1.1.4)
 8 |     em-websocket (0.5.1)
 9 |       eventmachine (>= 0.12.9)
10 |       http_parser.rb (~> 0.6.0)
11 |     eventmachine (1.2.7)
12 |     ffi (1.9.25)
13 |     forwardable-extended (2.6.0)
14 |     http_parser.rb (0.6.0)
15 |     i18n (0.9.5)
16 |       concurrent-ruby (~> 1.0)
17 |     jekyll (3.8.5)
18 |       addressable (~> 2.4)
19 |       colorator (~> 1.0)
20 |       em-websocket (~> 0.5)
21 |       i18n (~> 0.7)
22 |       jekyll-sass-converter (~> 1.0)
23 |       jekyll-watch (~> 2.0)
24 |       kramdown (~> 1.14)
25 |       liquid (~> 4.0)
26 |       mercenary (~> 0.3.3)
27 |       pathutil (~> 0.9)
28 |       rouge (>= 1.7, < 4)
29 |       safe_yaml (~> 1.0)
30 |     jekyll-sass-converter (1.5.2)
31 |       sass (~> 3.4)
32 |     jekyll-watch (2.1.2)
33 |       listen (~> 3.0)
34 |     kramdown (1.17.0)
35 |     liquid (4.0.1)
36 |     listen (3.1.5)
37 |       rb-fsevent (~> 0.9, >= 0.9.4)
38 |       rb-inotify (~> 0.9, >= 0.9.7)
39 |       ruby_dep (~> 1.2)
40 |     mercenary (0.3.6)
41 |     pathutil (0.16.2)
42 |       forwardable-extended (~> 2.6)
43 |     public_suffix (3.0.3)
44 |     rb-fsevent (0.10.3)
45 |     rb-inotify (0.10.0)
46 |       ffi (~> 1.0)
47 |     redcarpet (3.4.0)
48 |     rouge (3.3.0)
49 |     ruby_dep (1.5.0)
50 |     safe_yaml (1.0.4)
51 |     sass (3.7.2)
52 |       sass-listen (~> 4.0.0)
53 |     sass-listen (4.0.0)
54 |       rb-fsevent (~> 0.9, >= 0.9.4)
55 |       rb-inotify (~> 0.9, >= 0.9.7)
56 | 
57 | PLATFORMS
58 |   ruby
59 | 
60 | DEPENDENCIES
61 |   jekyll
62 |   redcarpet
63 |   rouge
64 | 
65 | BUNDLED WITH
66 |    2.0.1
67 | 


--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
 1 | As a work of the United States Government, this project is in the
 2 | public domain within the United States.
 3 | 
 4 | Additionally, we waive copyright and related rights in the work
 5 | worldwide through the CC0 1.0 Universal public domain dedication.
 6 | 
 7 | ## CC0 1.0 Universal Summary
 8 | 
 9 | This is a human-readable summary of the
10 | [Legal Code (read the full text)](https://creativecommons.org/publicdomain/zero/1.0/legalcode).
11 | 
12 | ### No Copyright
13 | 
14 | The person who associated a work with this deed has dedicated the work to
15 | the public domain by waiving all of his or her rights to the work worldwide
16 | under copyright law, including all related and neighboring rights, to the
17 | extent allowed by law.
18 | 
19 | You can copy, modify, distribute and perform the work, even for commercial
20 | purposes, all without asking permission.
21 | 
22 | ### Other Information
23 | 
24 | In no way are the patent or trademark rights of any person affected by CC0,
25 | nor are the rights that other persons may have in the work or in how the
26 | work is used, such as publicity or privacy rights.
27 | 
28 | Unless expressly stated otherwise, the person who associated a work with
29 | this deed makes no warranties about the work, and disclaims liability for
30 | all uses of the work, to the fullest extent permitted by applicable law.
31 | When using or citing the work, you should not imply endorsement by the
32 | author or the affirmer.
33 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | ## HTTPS Everywhere for the U.S. Government
 2 | 
 3 | The American people expect government websites to be secure and their interactions with those websites to be private.
 4 | 
 5 | This site contains a web-friendly version of the White House Office of Management and Budget memorandum [M-15-13](https://obamawhitehouse.archives.gov/sites/default/files/omb/memoranda/2015/m-15-13.pdf), **"A Policy to Require Secure Connections across Federal Websites and Web Services"**, and provides technical guidance and best practices to assist in its implementation.
 6 | 
 7 | **[Read the policy.](https://https.cio.gov)**
 8 | 
 9 | Please [open an issue](https://github.com/gsa/https/issues/new) to leave feedback or suggestions. Pull requests are welcome to pages _other_ than the homepage, which shows the final policy and is not subject to change through GitHub.
10 | 
11 | ### Thank You For Your Feedback
12 | 
13 | This policy was open for public comment before its finalization. It received [numerous comments](https://github.com/GSA/https/issues?utf8=%E2%9C%93&q=label%3A%22Public+Comment%22+) whose thoughtfulness and feedback improved the **[final policy](https://obamawhitehouse.archives.gov/sites/default/files/omb/memoranda/2015/m-15-13.pdf)**.
14 | 
15 | You can see what changed between the proposal and the final policy in [pull request #108](https://github.com/GSA/https/pull/108).
16 | 
17 | The homepage of this site is the final policy. The other pages on [https.cio.gov](https://https.cio.gov) are open for contribution at any time, and are intended to be resources for agencies implementing the HTTPS policy.
18 | 
19 | ### Developing on the site locally
20 | 
21 | If you're using this repository to run the site locally, instructions follow below.
22 | 
23 | Dependencies:
24 | 
25 | * **Node 6+** to install USWDS and dependencies
26 | * **Ruby** and **bundler** to install / run Jekyll
27 | 
28 | ##### First-time setup
29 | 
30 | 1. `npm install` to install the USWDS, and Gulp dependencies.
31 | 2. `npm install -g gulp` to let you use the `gulp` CLI directly.
32 | 3. `bundle install` to install Jekyll.
33 | 
34 | ##### Running the app
35 | 
36 | If you'll be editing the Sass/CSS:
37 | 
38 | * `gulp watch`
39 | 
40 | To run the app:
41 | 
42 | * `bundle exec jekyll serve`
43 | 
44 | ### Public domain
45 | 
46 | This project is in the worldwide [public domain](LICENSE.md). As stated in [CONTRIBUTING](CONTRIBUTING.md):
47 | 
48 | > This project is in the public domain within the United States, and copyright and related rights in the work worldwide are waived through the [CC0 1.0 Universal public domain dedication](https://creativecommons.org/publicdomain/zero/1.0/).
49 | >
50 | > All contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest.
51 | 


--------------------------------------------------------------------------------
/SECURITY.md:
--------------------------------------------------------------------------------
 1 | # Security Policy
 2 | 
 3 | As a U.S. Government agency, the General Services Administration (GSA) takes
 4 | seriously our responsibility to protect the public's information, including
 5 | financial and personal information, from unwarranted disclosure.
 6 | 
 7 | ## Reporting a Vulnerability
 8 | 
 9 | Services operated by the U.S. General Services Administration (GSA)
10 | are covered by the **GSA Vulnerability Disclosure Program (VDP)**.
11 | 
12 | See the [GSA Vulnerability Disclosure Policy](https://gsa.gov/vulnerability-disclosure-policy)
13 | at <https://www.gsa.gov/vulnerability-disclosure-policy> for details including:
14 | 
15 | * How to submit a report if you believe you have discovered a vulnerability.
16 | * Bug bounty scope.
17 | * GSA's coordinated disclosure policy.
18 | * Information on how you may conduct security research on GSA developed
19 |   software and systems.
20 | * Important legal and policy guidance.
21 | 
22 | ## Supported Versions
23 | 
24 | Please note that only certain branches are supported with security updates.
25 | 
26 | | Version (Branch) | Supported          |
27 | | ---------------- | ------------------ |
28 | | master             | :white_check_mark: |
29 | | other            | :x:                |
30 | 
31 | When using this code or reporting vulnerabilities please only use supported
32 | versions.
33 | 


--------------------------------------------------------------------------------
/Staticfile:
--------------------------------------------------------------------------------
1 | root: _site
2 | 


--------------------------------------------------------------------------------
/_config.yml:
--------------------------------------------------------------------------------
 1 | # Site metadata
 2 | title: The HTTPS-Only Standard
 3 | description: "Resources, best practices, and case studies for deploying HTTPS in the federal government."
 4 | url: https://https.cio.gov
 5 | 
 6 | # DAP, GA, and GitHub
 7 | dap_agency: GSA
 8 | google_analytics_ua: UA-48605964-20
 9 | repo: https://github.com/GSA/https
10 | 
11 | # Standard
12 | uswds_path: assets/vendor/uswds
13 | 
14 | # TODO: get rid of this
15 | highlighter: rouge
16 | markdown: redcarpet
17 | redcarpet:
18 |   extensions:
19 |   - smart
20 |   - with_toc_data
21 | 
22 | exclude:
23 | - compliance
24 | - resources
25 | - ".ruby-version"
26 | - ".jekyll-metadata"
27 | - ".gitignore"
28 | - node_modules
29 | - todo.txt
30 | - Staticfile
31 | - Gemfile
32 | - Gemfile.lock
33 | - vendor
34 | - README.md
35 | - CONTRIBUTING.md
36 | - LICENSE.md
37 | - .nvmrc
38 | 


--------------------------------------------------------------------------------
/_data/navigation.yml:
--------------------------------------------------------------------------------
 1 | 
 2 | primary:
 3 | - text: Home
 4 |   href: /
 5 | - text: Why Everything?
 6 |   href: /everything/
 7 | - text: Compliance Guide
 8 |   href: /guide/
 9 | - text: Introduction to HTTPS
10 |   href: /faq/
11 | - text: Certificates
12 |   href: /certificates/
13 | - text: HTTP Strict Transport Security
14 |   href: /hsts/
15 | - text: Migrating APIs
16 |   href: /apis/
17 | - text: Server Name Indication
18 |   href: /sni/
19 | - text: Technical Guidelines
20 |   href: /technical-guidelines/
21 | - text: Mixed Content
22 |   href: /mixed-content/
23 | - text: Resources
24 |   href: /resources/
25 | 
26 | mobile:
27 | - text: Home
28 |   href: /
29 | - text: Why Everything?
30 |   href: /everything/
31 | - text: Compliance Guide
32 |   href: /guide/
33 | - text: Introduction to HTTPS
34 |   href: /faq/
35 | - text: Certificates
36 |   href: /certificates/
37 | - text: HTTP Strict Transport Security
38 |   href: /hsts/
39 | - text: Migrating APIs
40 |   href: /apis/
41 | - text: Server Name Indication
42 |   href: /sni/
43 | - text: Technical Guidelines
44 |   href: /technical-guidelines/
45 | - text: Mixed Content
46 |   href: /mixed-content/
47 | - text: Resources
48 |   href: /resources/
49 | 


--------------------------------------------------------------------------------
/_includes/banner.html:
--------------------------------------------------------------------------------
 1 | <div class="usa-banner">
 2 |   <div class="usa-accordion">
 3 |     <header class="usa-banner-header">
 4 |       <div class="usa-grid usa-banner-inner">
 5 |         <img src="{{ site.baseurl }}/{{ site.uswds_path }}/img/favicons/favicon-57.png" alt="U.S. flag">
 6 |         <p>An official website of the United States government</p>
 7 |       </div>
 8 |     </header>
 9 |   </div>
10 | </div>
11 | 


--------------------------------------------------------------------------------
/_includes/footer.html:
--------------------------------------------------------------------------------
1 | <footer class="usa-footer usa-footer-medium site" role="contentinfo">
2 | 
3 |   <div class="usa-grid usa-footer-return-to-top">
4 |     <a href="#">Return to top</a>
5 |   </div>
6 | 
7 | </footer>
8 | 


--------------------------------------------------------------------------------
/_includes/head.html:
--------------------------------------------------------------------------------
 1 | <meta charset="utf-8">
 2 | <meta http-equiv="X-UA-Compatible" content="IE=edge">
 3 | <meta name="viewport" content="width=device-width, initial-scale=1">
 4 | 
 5 | <title>{% if page.title %}{{ site.title }} - {{ page.title }}{% else %}{{ site.title }}{% endif %}</title>
 6 | <meta property="og:title" content="{% if page.title %}{{ site.title }} - {{ page.title }}{% else %}{{ site.title }}{% endif %}" />
 7 | 
 8 | <meta name="description" content="{% if page.excerpt %}{{ page.excerpt | strip_html | strip_newlines | truncate: 160 }}{% else %}{{ site.description }}{% endif %}" />
 9 | 
10 | <link rel="canonical" href="{{ page.url | replace:'index.html','' | prepend: site.baseurl | prepend: site.url }}">
11 | 
12 | <!-- CSS -->
13 | 
14 | <link rel="stylesheet" href="{{ site.baseurl}}/assets/css/main.css?{{ site.time | date: "%Y%m%j%H%M%S" }}" media="screen" />
15 | 


--------------------------------------------------------------------------------
/_includes/header.html:
--------------------------------------------------------------------------------
 1 | <header class="usa-header site-header" role="banner">
 2 | 
 3 |   {% include banner.html %}
 4 | 
 5 |   <div class="usa-navbar site-header-navbar">
 6 |     <button class="usa-menu-btn">Menu</button>
 7 |     <div class="usa-logo site-logo" id="logo">
 8 |       <em class="usa-logo-text">
 9 |         <a href="{{ site.baseurl }}/" id="logo-text" title="{{ site.title }}">
10 |           {{ site.title }}
11 |         </a>
12 |       </em>
13 |     </div>
14 |   </div>
15 | 
16 | <!-- TBD horizontal nav -->
17 |   <nav role="navigation" class="usa-nav">
18 |     <div class="usa-nav-inner">
19 |       <button class="usa-nav-close">
20 |         <img src="{{ site.baseurl }}/{{ site.uswds_path }}/img/close.svg" alt="close">
21 |       </button>
22 |     </div>
23 |   </nav>
24 | 
25 | </header>
26 | 
27 | <!-- Mobile nav -->
28 | <nav role="navigation" class="usa-nav sidenav-mobile">
29 |   <div class="usa-nav-inner">
30 |     <button class="usa-nav-close">
31 |       <img src="{{ site.baseurl }}/{{ site.uswds_path }}/img/close.svg" alt="close">
32 |     </button>
33 | 
34 |     <ul class="usa-sidenav-list usa-accordion">
35 |       {% for section in site.data.navigation.mobile %}
36 |         <li>
37 |           <a href="{{ section.href }}">{{ section.text }}</a>
38 |         </li>
39 |       {% endfor %}
40 |     </ul>
41 | 
42 |   </div>
43 | </nav>
44 | 


--------------------------------------------------------------------------------
/_includes/scripts.html:
--------------------------------------------------------------------------------
 1 | 
 2 | 
 3 | <script src="{{ site.baseurl }}/{{ site.uswds_path }}/js/uswds.min.js" async></script>
 4 | 
 5 | <!-- Digital Analytics Program roll-up, see https://analytics.usa.gov for data -->
 6 | <script id="_fed_an_ua_tag" src="https://dap.digitalgov.gov/Universal-Federated-Analytics-Min.js?agency={{ site.dap_agency }}"></script>
 7 | 
 8 | <!-- Google Analytics -->
 9 | <script>
10 |   (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
11 |   (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
12 |   m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
13 |   })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
14 |   ga('create', '{{ site.google_analytics_ua }}');
15 |   // anonymize user IPs (chops off the last IP triplet)
16 |   ga('set', 'anonymizeIp', true);
17 |   // forces TLS even if the page were somehow loaded over http://
18 |   ga('set', 'forceSSL', true);
19 |   ga('send', 'pageview');
20 | </script>
21 | 
22 | 


--------------------------------------------------------------------------------
/_includes/sidenav.html:
--------------------------------------------------------------------------------
 1 | <ul class="usa-sidenav-list">
 2 |   {% for link in include.links %}
 3 |     {% assign _href = link.href %}
 4 |     {% assign _current = false %}
 5 |     {% if link.href == page.url or link.href == page.permalink %}
 6 |       {% assign _current = true %}
 7 |     {% endif %}
 8 |   <li>
 9 |     <a href="{{ _href | relative_url }}"{% if _current %}
10 |       class="usa-current"{% endif %}>
11 |       {{ link.text }}
12 |     </a>
13 |     {% if _current and page.subnav %}
14 |     <ul class="usa-sidenav-sub_list">
15 |       {% for link in page.subnav %}
16 |         <li>
17 |           <a href="{{ link.href }}">{{ link.text }}</a>
18 |         </li>
19 |       {% endfor %}
20 |     </ul>
21 |     {% endif %}
22 |   </li>
23 |   {% endfor %}
24 | 
25 |   <p>
26 |     <a href="{{ site.repo }}/edit/master/{{ page.path }}" class="usa-sidenav-edit" target="_blank">Edit this page </a>
27 |   </p>
28 | </ul>
29 | 


--------------------------------------------------------------------------------
/_layouts/base.html:
--------------------------------------------------------------------------------
 1 | ---
 2 | 
 3 | ---
 4 | <!DOCTYPE html>
 5 | <html lang="en-US">
 6 | <head>
 7 |   {% include head.html %}
 8 | </head>
 9 | 
10 | <body class="{{ layout.class }} {{ page.class }}">
11 | 
12 | 
13 | <a class="usa-skipnav" href="#main-content">Skip to main content</a>
14 | 
15 | {% include header.html %}
16 | 
17 | <aside class="sidenav sticky">
18 |   {% include sidenav.html links=site.data.navigation.primary %}
19 | </aside>
20 | 
21 | <main id="main-content" class="main-content">
22 |   <div class="usa-content">
23 |     {% if page.appendix %}
24 |       <h2>{{ page.title }}</h2>
25 |     {% else %}
26 |       <h1>{{ page.title }}</h1>
27 |     {% endif %}
28 | 
29 |     {{ content }}
30 |   </div>
31 | 
32 |   {% include footer.html %}
33 | 
34 | </main>
35 | 
36 | {% include scripts.html %}
37 | 
38 | 
39 | </body>
40 | </html>
41 | 


--------------------------------------------------------------------------------
/assets/images/mixed-content.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GSA/https/8271c90458204ca79220ea266fa88d6ed2c8515c/assets/images/mixed-content.png


--------------------------------------------------------------------------------
/assets/images/with-http-headers.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GSA/https/8271c90458204ca79220ea266fa88d6ed2c8515c/assets/images/with-http-headers.png


--------------------------------------------------------------------------------
/assets/images/with-https-headers.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GSA/https/8271c90458204ca79220ea266fa88d6ed2c8515c/assets/images/with-https-headers.png


--------------------------------------------------------------------------------
/assets/sass/main.scss:
--------------------------------------------------------------------------------
  1 | // USWDS SASS variable overrides go here, before we import USWDS itself.
  2 | 
  3 | $font-path: '../vendor/uswds/fonts';
  4 | $image-path: '../vendor/uswds/img';
  5 | 
  6 | @import 'uswds';
  7 | 
  8 | /***************************************
  9 |   Begin site customizations.
 10 | ****************************************/
 11 | 
 12 | .usa-footer-return-to-top {
 13 |   padding-left: 0;
 14 | }
 15 | 
 16 | .site-header-navbar {
 17 |   border-bottom: none;
 18 | 
 19 |   @include media($nav-width) {
 20 |     height: 8rem;
 21 |     width: 100%;
 22 |     display: block;
 23 |   }
 24 | 
 25 |   // Firefox fix so that navbar buttons to align right
 26 |   @-moz-document url-prefix() {
 27 |     @include media($nav-width) {
 28 |       padding-top: 1px;
 29 |     }
 30 |   }
 31 | }
 32 | 
 33 | .site-header {
 34 | 
 35 |   background-color: #153E56;
 36 | 
 37 |   @include media($nav-width) {
 38 |     border-bottom: none;
 39 |   }
 40 | 
 41 |   .site-logo {
 42 |     margin-left: $site-margins-mobile;
 43 | 
 44 |     @include media($nav-width) {
 45 |       margin-left: $site-margins;
 46 |     }
 47 |   }
 48 | 
 49 |   .usa-banner {
 50 |     padding-bottom: 0;
 51 |   }
 52 | 
 53 |   @media screen and (max-width: 480px) {
 54 |     .usa-banner-header {
 55 |       padding-top: 0;
 56 |       padding-bottom: 0;
 57 | 
 58 |       p {
 59 |         padding-top: 5px;
 60 |       }
 61 |     }
 62 |   }
 63 | 
 64 |   @media screen and (min-width: 481px) {
 65 |     .usa-banner-header {
 66 |       p {
 67 |         margin-top: 1px;
 68 |       }
 69 |     }
 70 |   }
 71 | 
 72 |   .usa-banner-inner {
 73 |     max-width: 100%;
 74 |     padding-top: 3px;
 75 |     padding-bottom: 3px;
 76 |   }
 77 | 
 78 |   .usa-banner-content {
 79 |     margin-left: 0;
 80 |   }
 81 | 
 82 |   .usa-button-list {
 83 |     float: right;
 84 |     display: none;
 85 |     margin-top: -5px;
 86 | 
 87 |     @include media($nav-width) {
 88 |       line-height: 8rem;
 89 |       margin-top: 0;
 90 |       display: block;
 91 |       padding-right: $site-margins;
 92 |     }
 93 | 
 94 |     li {
 95 |       display: inline;
 96 |       font-family: $font-sans;
 97 | 
 98 |       &:last-child .usa-button {
 99 |         margin-right: 0;
100 |       }
101 |     }
102 |   }
103 | }
104 | 
105 | .site-logo {
106 |   @include media($nav-width) {
107 |     margin-top: 0;
108 |   }
109 | 
110 |   a {
111 |     color: $color-white;
112 |   }
113 | 
114 |   em {
115 |     font-family: $font-serif;
116 |     font-size: 1.4rem;
117 |     line-height: 4rem;
118 | 
119 |     @media screen and (min-width: 951px) {
120 |       font-size: 2.6rem !important;
121 |     }
122 | 
123 |     @include media($nav-width) {
124 |       font-size: $h2-font-size;
125 |       line-height: 8rem;
126 |     }
127 |   }
128 | }
129 | 
130 | // Header -------------- //
131 | .sticky {
132 |   position: -webkit-sticky;
133 |   position: sticky;
134 |   top: 0;
135 | 
136 |   &:before,
137 |   &:after {
138 |     content: '';
139 |     display: table;
140 |   }
141 | }
142 | 
143 | 
144 | // Sidebar Nav --------- //
145 | 
146 | .sidenav {
147 |   display: none;
148 |   clear: both;
149 | 
150 |   @include media($nav-width) {
151 |     display: block;
152 |     float: left;
153 |     height: 100vh;
154 |     margin-bottom: 525px;
155 |     margin-top: 0px;
156 |     padding-right: 1rem;
157 |     padding-left: 3rem;
158 |     overflow: auto;
159 |     top: 0rem;
160 |     width: $width-nav-sidebar;
161 |   }
162 | 
163 |   .lt-ie9 & {
164 |     width: 25%;
165 |   }
166 | 
167 |   .usa-navigation-submenu {
168 |     &[aria-hidden="false"] {
169 |       display: block;
170 |     }
171 |   }
172 | }
173 | 
174 | // Main Content --------- //
175 | 
176 | .main-content {
177 |   font-family: $font-serif;
178 | 
179 |   position: absolute;
180 |   right: 0;
181 |   bottom: 0;
182 |   display: inline-block; // starting: not scrolled
183 |   position: relative;
184 |   width: 100%;
185 |   top: 3rem; // height of disclaimer + navbar on mobile
186 | 
187 |   padding-top: 0;
188 |   margin-top: 0;
189 | 
190 |   padding-right: 3rem;
191 |   padding-left: 3rem;
192 | 
193 |   font-size: 1.5rem;
194 | 
195 |   p, li {
196 |     line-height: 1.7;
197 |   }
198 | 
199 |   p, h1, h2, h3, h4, h5, h6, ul, ol {
200 |     max-width: 66ch !important;
201 |   }
202 | 
203 |   @include media($nav-width) {
204 |     width: calc(100% - #{$width-nav-sidebar});
205 |     top: 0;
206 |   }
207 | 
208 |   .lt-ie9 & {
209 |     width: 75%;
210 |   }
211 | 
212 |   @media screen and (max-width: 481px) {
213 |     h1 {font-size: 3rem;}
214 |     h2 {font-size: 2.2rem;}
215 |     h3 {font-size: 1.7rem;}
216 |   }
217 | 
218 |   h1 {
219 |     display: inline-block;
220 |     margin-top: 0;
221 |     margin-bottom: 0.2em;
222 |     padding-bottom: 0px;
223 |   }
224 | 
225 |   h2 {
226 |     margin-top: 1.5em;
227 |     margin-bottom: 0.5em
228 |   }
229 | 
230 |   h2 {
231 |     color: $color-primary-darker;
232 |   }
233 | }
234 | 
235 | @mixin outline-inverse-button {
236 |   // TODO: Fix this with button mixin
237 |   .usa-button.usa-button-outline-inverse {
238 |     box-shadow: inset 0 0 0 2px $color-primary;
239 |     color: $color-primary;
240 | 
241 |     &:hover {
242 |       box-shadow: inset 0 0 0 2px $color-primary-darker;
243 |       color: $color-primary-darker;
244 |     }
245 |   }
246 | }
247 | 
248 | .sidenav-mobile {
249 |   clear: both;
250 | 
251 |   @include outline-inverse-button;
252 | 
253 |   @include media($nav-width) {
254 |     display: none;
255 |   }
256 | 
257 |   .usa-sidenav-sub_list {
258 |     &[aria-hidden=true] {
259 |       display: none;
260 |     }
261 |   }
262 | 
263 |   li > button {
264 |     @include button-unstyled;
265 |     font-weight: $font-normal;
266 |     padding: 1.4rem 1.5rem 1rem 1.8rem;
267 | 
268 |     &:hover {
269 |       background-color: $color-gray-lightest;
270 |       color: $color-primary;
271 |     }
272 | 
273 |     &[aria-expanded=false] {
274 |       background-image: url('#{$image-path}/plus-alt.png');
275 |       background-image: url('#{$image-path}/plus-alt.svg');
276 |       background-repeat: no-repeat;
277 |       background-position: right 0 center;
278 |       background-size: 1rem;
279 |     }
280 | 
281 |     &[aria-expanded=true] {
282 |       background-image: url('#{$image-path}/minus-alt.png');
283 |       background-image: url('#{$image-path}/minus-alt.svg');
284 |       background-repeat: no-repeat;
285 |       background-position: right 0 center;
286 |       background-size: 1rem;
287 |     }
288 |   }
289 | 
290 |   .usa-button-list {
291 |     margin-top: 4rem;
292 |   }
293 | }
294 | 
295 | pre {
296 |   max-width: 80%;
297 |   font-size: 0.875em;
298 |   overflow-y: scroll;
299 |   background-color: #f1f2f2;
300 |   padding: 10px;
301 | 
302 |   white-space: pre-wrap;
303 |   line-height: 14pt;
304 | }
305 | 
306 | p > code, li > code {
307 |   padding: 2px 4px;
308 |   background-color: #eaeaff;
309 |   border: 1px solid #d5d5d5;
310 |   font-size: 10pt;
311 |   font-family: monospace;
312 | }
313 | 
314 | a.usa-sidenav-edit {
315 |     color: #0072ce;
316 |     text-decoration: underline;
317 | }
318 | 


--------------------------------------------------------------------------------
/compliance/bod-18-01/compliance.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | 
  3 | import utils
  4 | import glob
  5 | 
  6 | ## Todo for DHS:
  7 | # * compare data across all dates
  8 | 
  9 | ## Todo for me:
 10 | # * pre-bod path and post-bod path
 11 | # * parents-only data going back to 2015?
 12 | # * cross-reference with DAP?
 13 | 
 14 | # get mapping of domains to agency
 15 | base_domains = utils.domains_to_agencies()
 16 | 
 17 | # get current and pending preloaded domains
 18 | # Downloads live from remote sources. (clear cache/ to re-download)
 19 | preloaded = set(utils.fetch_preloaded()) & set(base_domains)
 20 | 
 21 | # See download.py for which dates are downloaded for each phase.
 22 | 
 23 | # pre-BOD measurements (no 3DES checking, old dir structure)
 24 | pre_bod = [x.split("/")[-1] for x in glob.glob("data/pulse/pre-bod/*")]
 25 | pre_bod.sort()
 26 | 
 27 | # post-BOD measurements (bod_crypto measured, new dir structure)
 28 | post_bod = [x.split("/")[-1] for x in glob.glob("data/pulse/post-bod/*")]
 29 | post_bod.sort()
 30 | 
 31 | 
 32 | # post-bod paths
 33 | def pshtts_and_sslyzes_for(date):
 34 |   pshtts = []
 35 |   sslyzes = []
 36 | 
 37 |   if date in post_bod:
 38 |     pshtts.append("data/pulse/post-bod/%s/parents/results/pshtt.csv" % date)
 39 |     pshtts.append("data/pulse/post-bod/%s/subdomains/scan/results/pshtt.csv" % date)
 40 |     sslyzes.append("data/pulse/post-bod/%s/parents/results/sslyze.csv" % date)
 41 |     sslyzes.append("data/pulse/post-bod/%s/subdomains/scan/results/sslyze.csv" % date)
 42 |   elif date in pre_bod:
 43 |     pshtts.append("data/pulse/pre-bod/%s/scan/pshtt.csv" % date)
 44 |     pshtts.append("data/pulse/pre-bod/%s/subdomains/scan/censys/results/pshtt.csv" % date)
 45 |     pshtts.append("data/pulse/pre-bod/%s/subdomains/scan/url/results/pshtt.csv" % date)
 46 |     sslyzes.append("data/pulse/pre-bod/%s/scan/sslyze.csv" % date)
 47 |     sslyzes.append("data/pulse/pre-bod/%s/subdomains/scan/censys/results/sslyze.csv" % date)
 48 |     sslyzes.append("data/pulse/pre-bod/%s/subdomains/scan/url/results/sslyze.csv" % date)
 49 | 
 50 |   return pshtts, sslyzes
 51 | 
 52 | def pct(num, denom):
 53 |   return round((num / denom) * 100)
 54 | 
 55 | def compliance_stats(name, date, filter):
 56 |   pshtts, sslyzes = pshtts_and_sslyzes_for(date)
 57 |   data = utils.load_pshtt_sslyze(pshtts, sslyzes, base_domains, preloaded, filter=filter)
 58 |   totals = utils.compliance_totals(data)
 59 | 
 60 |   count = len(data.keys())
 61 | 
 62 |   if count == 0:
 63 |     print("No rows returned, no data to display.")
 64 |     return
 65 | 
 66 |   print()
 67 |   print("=====================================================")
 68 |   print(" [%s] %s" % (date, name))
 69 |   print("=====================================================")
 70 |   print()
 71 |   print("Total domains: %i" % count)
 72 |   print()
 73 |   print("== Direct enforcement ==")
 74 |   print("Enforces HTTPS: %i (%i%%)" % (totals['enforces'], pct(totals['enforces'], count)))
 75 |   print("Strong HSTS: %i (%i%%)" % (totals['hsts'], pct(totals['hsts'], count)))
 76 |   print("RC4 support: %i (%i%%)" % (totals['rc4'], pct(totals['rc4'], count)))
 77 | 
 78 |   if date in post_bod:
 79 |     print("3DES support: %i (%i%%)" % (totals['3des'], pct(totals['3des'], count)))
 80 |     print("Free of Known-weak Crypto: %i (%i%%)" % (totals['bod_crypto'], pct(totals['bod_crypto'], count)))
 81 |     print("Compliant with BOD 18-01: %i (%i%%)" % (totals['compliant'], pct(totals['compliant'], count)))
 82 | 
 83 |   elif date in pre_bod:
 84 |     print("Compliant with M-15-13: %i (%i%%)" % (totals['m1513'], pct(totals['m1513'], count)))
 85 | 
 86 |   print()
 87 | 
 88 | def compliance_csv(dates, when, filter, path):
 89 |   if when == "pre":
 90 |     header = [
 91 |       "Date", "Total Hostnames",
 92 |       "Enforces HTTPS", "HSTS",
 93 |       "M-15-13",
 94 |       "RC4",
 95 |       "Enforces HTTPS (%)", "HSTS (%)",
 96 |       "M-15-13 (%)",
 97 |       "RC4 (%)",
 98 |     ]
 99 |   elif when == "post":
100 |     header = [
101 |       "Date", "Total Hostnames",
102 |       "Enforces HTTPS", "HSTS",
103 |       "M-15-13",
104 |       "RC4", "3DES", "Free of SSLv2/SSLv3/RC4/3DES",
105 |       "BOD 18-01",
106 |       "Enforces HTTPS (%)", "HSTS (%)",
107 |       "M-15-13 (%)",
108 |       "RC4 (%)", "3DES (%)", "Free of SSLv2/SSLv3/RC4/3DES (%)",
109 |       "BOD 18-01 (%)",
110 |     ]
111 | 
112 |   rows = []
113 |   for date in dates:
114 |     rows.append(compliance_csv_row(date, filter))
115 | 
116 |   utils.save_csv(header, rows, path)
117 | 
118 | def compliance_csv_row(date, filter):
119 |   print("[%s] Running report for row..." % date)
120 |   pshtts, sslyzes = pshtts_and_sslyzes_for(date)
121 |   data = utils.load_pshtt_sslyze(pshtts, sslyzes, base_domains, preloaded, filter=filter)
122 |   totals = utils.compliance_totals(data)
123 | 
124 |   count = len(data.keys())
125 | 
126 |   if date in pre_bod:
127 |     return [
128 |       date,
129 |       count,
130 |       totals['enforces'],
131 |       totals['hsts'],
132 |       totals['m1513'],
133 |       totals['rc4'],
134 |       pct(totals['enforces'], count),
135 |       pct(totals['hsts'], count),
136 |       pct(totals['m1513'], count),
137 |       pct(totals['rc4'], count),
138 |     ]
139 |   elif date in post_bod:
140 |     return [
141 |       date,
142 |       count,
143 |       totals['enforces'],
144 |       totals['hsts'],
145 |       totals['m1513'],
146 |       totals['rc4'],
147 |       totals['3des'],
148 |       totals['bod_crypto'],
149 |       totals['compliant'],
150 |       pct(totals['enforces'], count),
151 |       pct(totals['hsts'], count),
152 |       pct(totals['m1513'], count),
153 |       pct(totals['rc4'], count),
154 |       pct(totals['3des'], count),
155 |       pct(totals['bod_crypto'], count),
156 |       pct(totals['compliant'], count),
157 |     ]
158 | 
159 | 
160 | # # All executive hostnames pre-BOD.
161 | # compliance_csv(pre_bod, "pre",
162 | #   utils.executive_only,
163 | #   "cache/pre-bod-executive.csv"
164 | # )
165 | # # All executive hostnames post-BOD.
166 | # compliance_csv(post_bod, "post",
167 | #   utils.executive_only,
168 | #   "cache/post-bod-executive.csv"
169 | # )
170 | 
171 | 
172 | # CFO Act
173 | compliance_csv(pre_bod, "pre",
174 |   utils.cfo_act_only,
175 |   "cache/pre-bod-cfo.csv"
176 | )
177 | compliance_csv(post_bod, "post",
178 |   utils.cfo_act_only,
179 |   "cache/post-bod-cfo.csv"
180 | )
181 | 
182 | # Non-CFO Act
183 | compliance_csv(pre_bod, "pre",
184 |   utils.executive_non_cfo_act,
185 |   "cache/pre-bod-non-cfo.csv"
186 | )
187 | compliance_csv(post_bod, "post",
188 |   utils.executive_non_cfo_act,
189 |   "cache/post-bod-non-cfo.csv"
190 | )
191 | 
192 | # # CFO Act (minus DoD)
193 | # compliance_csv(pre_bod, "pre",
194 | #   utils.cfo_act_only_sans_dod,
195 | #   "cache/pre-bod-cfo-no-dod.csv"
196 | # )
197 | # compliance_csv(post_bod, "post",
198 | #   utils.cfo_act_only_sans_dod,
199 | #   "cache/post-bod-cfo-no-dod.csv"
200 | # )
201 | 
202 | # # DoD only
203 | # compliance_csv(pre_bod, "pre",
204 | #   utils.for_agencies(["Department of Defense"]),
205 | #   "cache/pre-bod-dod.csv"
206 | # )
207 | # compliance_csv(post_bod, "post",
208 | #   utils.for_agencies(["Department of Defense"]),
209 | #   "cache/post-bod-dod.csv"
210 | # )
211 | 
212 | 
213 | 


--------------------------------------------------------------------------------
/compliance/bod-18-01/download.py:
--------------------------------------------------------------------------------
 1 | ##
 2 | # Download data from the Pulse S3 bucket to measure scans over time.
 3 | # Must be run from a machine where "aws s3 cp" has credentials for the bucket.
 4 | 
 5 | import os
 6 | import subprocess
 7 | 
 8 | AWS_REGION = "us-gov-west-1"
 9 | BUCKET_NAME = "cg-4adefb86-dadb-4ecf-be3e-f1c7b4f6d084"
10 | DATA_DIR = "data/pulse"
11 | 
12 | # Before BOD 18-01 was issued and before Pulse data was restructured
13 | pre_bod = ["2017-02-10", "2017-02-12", "2017-02-15", "2017-02-17", "2017-02-20", "2017-02-22", "2017-02-24", "2017-02-27", "2017-03-01", "2017-03-03", "2017-03-06", "2017-03-08", "2017-03-10", "2017-03-13", "2017-03-15", "2017-03-17", "2017-03-20", "2017-03-22", "2017-03-24", "2017-03-27", "2017-03-30", "2017-03-31", "2017-04-04", "2017-04-06", "2017-04-08", "2017-04-11", "2017-04-13", "2017-04-20", "2017-04-22", "2017-04-25", "2017-04-27", "2017-04-29", "2017-05-02", "2017-05-04", "2017-05-06", "2017-05-09", "2017-05-11", "2017-05-13", "2017-05-16", "2017-05-18", "2017-05-20", "2017-05-23", "2017-05-25", "2017-05-27", "2017-05-30", "2017-06-01", "2017-06-03", "2017-06-05", "2017-07-02", "2017-07-03", "2017-07-05", "2017-07-07", "2017-07-10", "2017-07-12", "2017-07-14", "2017-07-17", "2017-07-19", "2017-07-21", "2017-07-25", "2017-07-26", "2017-07-28", "2017-07-31", "2017-08-02", "2017-08-07", "2017-08-09", "2017-08-11", "2017-08-14", "2017-08-16", "2017-08-18", "2017-08-21", "2017-08-23", "2017-08-25", "2017-09-28"]
14 | 
15 | # After BOD 18-01 was issued and after Pulse data was restructured
16 | # Exceptions:
17 | #  2018-01-02: no subdomains/scan/results
18 | post_bod = ["2017-11-20", "2017-11-25", "2017-12-11", "2017-12-13", "2017-12-15", "2017-12-16", "2017-12-17", "2017-12-18", "2017-12-19", "2017-12-20", "2017-12-21", "2017-12-22", "2017-12-23", "2017-12-24", "2017-12-25", "2017-12-26", "2017-12-27", "2017-12-28", "2017-12-29", "2017-12-30", "2017-12-31", "2018-01-01", "2018-01-03", "2018-01-04", "2018-01-05", "2018-01-06", "2018-01-07", "2018-01-08", "2018-01-09", "2018-01-10", "2018-01-11", "2018-01-12", "2018-01-13", "2018-01-14", "2018-01-15", "2018-01-16", "2018-01-17", "2018-01-18", "2018-01-19", "2018-01-20", "2018-01-21", "2018-01-22", "2018-01-23", "2018-01-24", "2018-01-25", "2018-01-26", "2018-01-27", "2018-01-28", "2018-01-29", "2018-01-30", "2018-01-31", "2018-02-01", "2018-02-02", "2018-02-03", "2018-02-04", "2018-02-05", "2018-02-06", "2018-02-07", "2018-02-08", "2018-02-09", "2018-02-10", "2018-02-11", "2018-02-12", "2018-02-13", "2018-02-14", "2018-02-15", "2018-02-16", "2018-02-17", "2018-02-18", "2018-02-19", "2018-02-20", "2018-02-21", "2018-02-22", "2018-02-23", "2018-02-24", "2018-02-25", "2018-02-26", "2018-02-27", "2018-02-28", "2018-03-01", "2018-03-02", "2018-03-03", "2018-03-04", "2018-03-05", "2018-03-06", "2018-03-07", "2018-03-08", "2018-03-09", "2018-03-10", "2018-03-11", "2018-03-15", "2018-03-16", "2018-03-17", "2018-03-18", "2018-03-19", "2018-03-20", "2018-03-21", "2018-03-23", "2018-03-24", "2018-03-25", "2018-03-26", "2018-03-27", "2018-03-28", "2018-03-29", "2018-03-30", "2018-03-31", "2018-04-01", "2018-04-02", "2018-04-03", "2018-04-09", "2018-04-10", "2018-04-11", "2018-04-12", "2018-04-13", "2018-04-14", "2018-04-15", "2018-04-16", "2018-04-25", "2018-04-26", "2018-04-27", "2018-04-28"]
19 | 
20 | def shell_out(command, env=None):
21 |     response = subprocess.check_output(command, shell=False, env=env)
22 |     output = str(response, encoding='UTF-8')
23 |     return output
24 | 
25 | def mkdir_p(path):
26 |     try:
27 |         os.makedirs(path)
28 |     except OSError as exc:  # Python >2.5
29 |         if exc.errno == errno.EEXIST:
30 |             pass
31 |         else:
32 |             raise
33 | 
34 | 
35 | # when = "pre" or "post"
36 | def download_date(date, when="post"):
37 | 
38 |   def download(date, file):
39 |     url = "s3://%s/archive/%s/%s" % (BUCKET_NAME, date, file)
40 | 
41 |     # local destinations are relative to data/
42 |     path = os.path.join(DATA_DIR, "%s-bod" % when, date, file)
43 | 
44 |     shell_out(["aws", "s3", "cp", url, path])
45 | 
46 |   try:
47 |     # Newer streamlined directory structure
48 |     if when == "post":
49 |       download(date, "parents/results/pshtt.csv")
50 |       download(date, "parents/results/sslyze.csv")
51 |       download(date, "parents/results/meta.json")
52 |       download(date, "subdomains/scan/results/pshtt.csv")
53 |       download(date, "subdomains/scan/results/sslyze.csv")
54 |       download(date, "subdomains/scan/results/meta.json")
55 | 
56 |     # Older, more laborious directory structure
57 |     elif when == "pre":
58 |       download(date, "scan/pshtt.csv")
59 |       download(date, "scan/sslyze.csv")
60 |       download(date, "scan/meta.json")
61 |       download(date, "subdomains/scan/url/results/pshtt.csv")
62 |       download(date, "subdomains/scan/url/results/sslyze.csv")
63 |       download(date, "subdomains/scan/url/results/meta.json")
64 |       download(date, "subdomains/scan/censys/results/pshtt.csv")
65 |       download(date, "subdomains/scan/censys/results/sslyze.csv")
66 |       download(date, "subdomains/scan/censys/results/meta.json")
67 |   except subprocess.CalledProcessError:
68 |     print("[%s] MISSING SOMETHING" % date)
69 | 
70 | 
71 | # for date in post_bod:
72 | #   download_date(date, "post")
73 | 
74 | # for date in pre_bod:
75 | #   download_date(date, "pre")
76 | 


--------------------------------------------------------------------------------
/compliance/bod-18-01/scratch.py:
--------------------------------------------------------------------------------
  1 | 
  2 | #########################################################
  3 | #
  4 | # Just scratch space, this won't execute or import properly.
  5 | #
  6 | #########################################################
  7 | 
  8 | name = "Pre-BOD (all executive)"
  9 | compliance_stats(name, pre_bod[-1], utils.executive_only)
 10 | 
 11 | name = "Most recent (all executive)"
 12 | compliance_stats(name, post_bod[-1], utils.executive_only)
 13 | 
 14 | name = "Most recent (all legislative)"
 15 | compliance_stats(name, post_bod[-1], utils.legislative_only)
 16 | 
 17 | name = "Most recent (all judicial)"
 18 | compliance_stats(name, post_bod[-1], utils.judicial_only)
 19 | 
 20 | name = "Most recent (CFO Act only)"
 21 | compliance_stats(name, post_bod[-1], utils.cfo_act_only)
 22 | 
 23 | name = "Most recent (CFO Act only, minus DoD)"
 24 | compliance_stats(name, post_bod[-1], utils.cfo_act_only_sans_dod)
 25 | 
 26 | name = "Most recent (DoD only)"
 27 | compliance_stats(name, post_bod[-1], utils.for_agencies(["Department of Defense"]))
 28 | 
 29 | name = "Most recent (non-CFO Act)"
 30 | compliance_stats(name, post_bod[-1], utils.executive_non_cfo_act)
 31 | 
 32 | 
 33 | # All executive hostnames pre-BOD.
 34 | compliance_csv(pre_bod, "pre",
 35 |   utils.executive_only,
 36 |   "cache/pre-bod-executive.csv"
 37 | )
 38 | # All executive hostnames post-BOD.
 39 | compliance_csv(post_bod, "post",
 40 |   utils.executive_only,
 41 |   "cache/post-bod-executive.csv"
 42 | )
 43 | 
 44 | # CFO Act
 45 | compliance_csv(pre_bod, "pre",
 46 |   utils.cfo_act_only,
 47 |   "cache/pre-bod-cfo.csv"
 48 | )
 49 | compliance_csv(post_bod, "post",
 50 |   utils.cfo_act_only,
 51 |   "cache/post-bod-cfo.csv"
 52 | )
 53 | 
 54 | # Non-CFO Act
 55 | compliance_csv(pre_bod, "pre",
 56 |   utils.executive_non_cfo_act,
 57 |   "cache/pre-bod-non-cfo.csv"
 58 | )
 59 | compliance_csv(post_bod, "post",
 60 |   utils.executive_non_cfo_act,
 61 |   "cache/post-bod-non-cfo.csv"
 62 | )
 63 | 
 64 | # CFO Act (minus DoD)
 65 | compliance_csv(pre_bod, "pre",
 66 |   utils.cfo_act_only_sans_dod,
 67 |   "cache/pre-bod-cfo-no-dod.csv"
 68 | )
 69 | compliance_csv(post_bod, "post",
 70 |   utils.cfo_act_only_sans_dod,
 71 |   "cache/post-bod-cfo-no-dod.csv"
 72 | )
 73 | 
 74 | # DoD only
 75 | compliance_csv(pre_bod, "pre",
 76 |   utils.for_agencies(["Department of Defense"]),
 77 |   "cache/pre-bod-dod.csv"
 78 | )
 79 | compliance_csv(post_bod, "post",
 80 |   utils.for_agencies(["Department of Defense"]),
 81 |   "cache/post-bod-dod.csv"
 82 | )
 83 | 
 84 | 
 85 | cfo_act_groups = [
 86 |   "U.S. Department of Agriculture",
 87 |   "Department of Commerce",
 88 |   "Department of Defense",
 89 |   "Department of Education",
 90 |   "Department of Energy",
 91 |   "Department of Health and Human Services",
 92 |   "Department of Homeland Security",
 93 |   "Department of Housing and Urban Development",
 94 |   "Department of the Interior",
 95 |   [
 96 |     "Department of Justice",
 97 |     "Terrorist Screening Center", # (DOJ/FBI) as seen in .gov data
 98 |   ],
 99 |   "Department of Labor",
100 |   [
101 |     "Department of State",
102 |     "Department of State, Office of Inspector General", # seen in .gov data
103 |   ],
104 |   "Department of Transportation",
105 |   "Department of the Treasury",
106 |   "Department of Veterans Affairs",
107 |   "Environmental Protection Agency",
108 |   "National Aeronautics and Space Administration",
109 |   "U.S. Agency for International Development",
110 |   "General Services Administration",
111 |   "National Science Foundation",
112 |   "Nuclear Regulatory Commission",
113 |   "Office of Personnel Management",
114 |   "Small Business Administration",
115 |   "Social Security Administration"
116 | ]
117 | 


--------------------------------------------------------------------------------
/compliance/bod-18-01/utils.py:
--------------------------------------------------------------------------------
  1 | import csv
  2 | import io
  3 | import os
  4 | import errno
  5 | import urllib
  6 | import urllib.request
  7 | import logging
  8 | import requests
  9 | import base64
 10 | import re
 11 | import json
 12 | 
 13 | # == compliance
 14 | 
 15 | # Given a dict of pshtt results for a hostname, what are the
 16 | # Uses, Enforces, and HSTS conclusions?
 17 | #
 18 | # borrows the pulse.cio.gov code for calculating conclusions:
 19 | # https://github.com/18F/pulse/blob/0528773b1d39a664ff8f62d655b8bb7c8979874c/data/processing.py#L408-L509
 20 | def compliance_for(pshtt, sslyze, parent_preloaded=False):
 21 |   report = {}
 22 |   # assumes that HTTPS would be technically present, with or without issues
 23 |   if (pshtt["Downgrades HTTPS"] == "True"):
 24 |     https = 0 # No
 25 |   else:
 26 |     if (pshtt["Valid HTTPS"] == "True"):
 27 |       https = 2 # Yes
 28 |     elif (
 29 |       (pshtt["HTTPS Bad Chain"] == "True") and
 30 |       (pshtt["HTTPS Bad Hostname"] == "False")
 31 |     ):
 32 |       https = 1 # Yes
 33 |     else:
 34 |       https = -1 # No
 35 | 
 36 | 
 37 |   ###
 38 |   # Is HTTPS enforced?
 39 | 
 40 |   if (https <= 0):
 41 |     behavior = 0 # N/A
 42 | 
 43 |   else:
 44 | 
 45 |     # "Yes (Strict)" means HTTP immediately redirects to HTTPS,
 46 |     # *and* that HTTP eventually redirects to HTTPS.
 47 |     #
 48 |     # Since a pure redirector domain can't "default" to HTTPS
 49 |     # for itself, we'll say it "Enforces HTTPS" if it immediately
 50 |     # redirects to an HTTPS URL.
 51 |     if (
 52 |       (pshtt["Strictly Forces HTTPS"] == "True") and
 53 |       (
 54 |         (pshtt["Defaults to HTTPS"] == "True") or
 55 |         (pshtt["Redirect"] == "True")
 56 |       )
 57 |     ):
 58 |       behavior = 3 # Yes (Strict)
 59 | 
 60 |     # "Yes" means HTTP eventually redirects to HTTPS.
 61 |     elif (
 62 |       (pshtt["Strictly Forces HTTPS"] == "False") and
 63 |       (pshtt["Defaults to HTTPS"] == "True")
 64 |     ):
 65 |       behavior = 2 # Yes
 66 | 
 67 |     # Either both are False, or just 'Strict Force' is True,
 68 |     # which doesn't matter on its own.
 69 |     # A "present" is better than a downgrade.
 70 |     else:
 71 |       behavior = 1 # Present (considered 'No')
 72 | 
 73 |   report['enforces'] = behavior
 74 | 
 75 | 
 76 |   ###
 77 |   # Characterize the presence and completeness of HSTS.
 78 | 
 79 |   if pshtt["HSTS Max Age"]:
 80 |     hsts_age = int(pshtt["HSTS Max Age"])
 81 |   else:
 82 |     hsts_age = None
 83 | 
 84 |   # If this is a subdomain, it can be considered as having HSTS, via
 85 |   # the preloading of its parent.
 86 |   if parent_preloaded:
 87 |     hsts = 3 # Yes, via preloading
 88 | 
 89 |   # Otherwise, without HTTPS there can be no HSTS for the domain directly.
 90 |   elif (https <= 0):
 91 |     hsts = -1  # N/A (considered 'No')
 92 | 
 93 |   else:
 94 | 
 95 |     # HSTS is present for the canonical endpoint.
 96 |     if (pshtt["HSTS"] == "True") and hsts_age:
 97 | 
 98 |       # Say No for too-short max-age's, and note in the extended details.
 99 |       if hsts_age >= 31536000:
100 |         hsts = 2  # Yes, directly
101 |       else:
102 |         hsts = 1  # No
103 | 
104 |     else:
105 |       hsts = 0  # No
106 | 
107 |   # Separate preload status from HSTS status:
108 |   #
109 |   # * Domains can be preloaded through manual overrides.
110 |   # * Confusing to mix an endpoint-level decision with a domain-level decision.
111 |   if pshtt["HSTS Preloaded"] == "True":
112 |     preloaded = 2  # Yes
113 |   elif (pshtt["HSTS Preload Ready"] == "True"):
114 |     preloaded = 1  # Ready for submission
115 |   else:
116 |     preloaded = 0  # No
117 | 
118 |   report['hsts'] = hsts
119 |   report['hsts_age'] = hsts_age
120 |   report['preloaded'] = preloaded
121 | 
122 |   ###
123 |   # Get cipher/protocol data via sslyze for a host.
124 | 
125 |   sslv2 = None
126 |   sslv3 = None
127 |   any_rc4 = None
128 |   any_3des = None
129 | 
130 |   # values: unknown or N/A (-1), No (0), Yes (1)
131 |   bod_crypto = None
132 | 
133 |   # N/A if no HTTPS
134 |   if (https <= 0):
135 |     bod_crypto = -1 # N/A
136 | 
137 |   elif sslyze is None:
138 |     # LOGGER.info("[https][%s] No sslyze scan data found." % name)
139 |     bod_crypto = -1 # Unknown
140 | 
141 |   else:
142 |     ###
143 |     # BOD 18-01 (cyber.dhs.gov) cares about SSLv2, SSLv3, RC4, and 3DES.
144 |     any_rc4 = boolean_for(sslyze["Any RC4"])
145 |     # TODO: kill conditional once everything is synced
146 |     if sslyze.get("Any 3DES"):
147 |       any_3des = boolean_for(sslyze["Any 3DES"])
148 |     sslv2 = boolean_for(sslyze["SSLv2"])
149 |     sslv3 = boolean_for(sslyze["SSLv3"])
150 | 
151 |     if any_rc4 or any_3des or sslv2 or sslv3:
152 |       bod_crypto = 0
153 |     else:
154 |       bod_crypto = 1
155 | 
156 |   report['bod_crypto'] = bod_crypto
157 |   report['rc4'] = any_rc4
158 |   report['3des'] = any_3des
159 |   report['sslv2'] = sslv2
160 |   report['sslv3'] = sslv3
161 | 
162 |   # Final calculation: is the service compliant with all of M-15-13
163 |   # (HTTPS+HSTS) and BOD 18-01 (that + RC4/3DES/SSLv2/SSLv3)?
164 | 
165 |   # For M-15-13 compliance, the service has to enforce HTTPS,
166 |   # and has to have strong HSTS in place (can be via preloading).
167 |   m1513 = (behavior >= 2) and (hsts >= 2)
168 | 
169 |   # For BOD compliance, only ding if we have scan data:
170 |   # * If our scanner dropped, give benefit of the doubt.
171 |   # * If they have no HTTPS, this will fix itself once HTTPS comes on.
172 |   bod1801 = m1513 and (bod_crypto != 0)
173 | 
174 |   # Phew!
175 |   report['m1513'] = m1513
176 |   report['compliant'] = bod1801 # equivalent, since BOD is a superset
177 | 
178 |   return report
179 | 
180 | 
181 | # Given a compliance report (e.g. uses=1, enforces=2, hsts=2)
182 | # return a dict with them turned into labels
183 | def compliance_labels(report):
184 |     return {
185 |         'enforces': (report['enforces'] >= 2),
186 |         'hsts': (report['hsts'] >= 2),
187 |         'bod_crypto': (report['bod_crypto'] == 1),
188 |         'compliant': report['compliant'],
189 |         'm1513': report['m1513'],
190 |         'rc4': report['rc4'],
191 |         '3des': report['3des']
192 |     }
193 | 
194 | def boolean_for(string):
195 |   if string == "False":
196 |     return False
197 |   elif string == "True":
198 |     return True
199 |   else:
200 |     return None
201 | 
202 | # Given a set of domains, calculate the compliance totals
203 | # that meet BOD 18-01 requirements.
204 | # Adapted from a snapshot of Pulse code:
205 | # https://github.com/18F/pulse/blob/0528773b1d39a664ff8f62d655b8bb7c8979874c/data/processing.py#L511-L539
206 | 
207 | def compliance_totals(data):
208 |     total_report = {
209 |         'enforces': 0,
210 |         'hsts': 0,
211 |         'bod_crypto': 0,
212 |         'compliant': 0,
213 |         'm1513': 0,
214 |         'rc4': 0,
215 |         '3des': 0
216 |     }
217 | 
218 |     for domain in data.keys():
219 |         # calculated in compliance_for() method using pshtt data
220 |         report = data[domain]['compliance']
221 | 
222 |         # Needs to be Default or Strict to be 'Yes'
223 |         if report['enforces'] >= 2:
224 |             total_report['enforces'] += 1
225 | 
226 |         # Needs to be present with >= 1 year max-age for canonical endpoint
227 |         if report['hsts'] >= 2:
228 |             total_report['hsts'] += 1
229 | 
230 |         if report['bod_crypto'] == 1:
231 |             total_report['bod_crypto'] += 1
232 | 
233 |         if report['compliant']:
234 |             total_report['compliant'] += 1
235 | 
236 |         if report['m1513']:
237 |             total_report['m1513'] += 1
238 | 
239 |         if report['rc4']:
240 |             total_report['rc4'] += 1
241 | 
242 |         if report['3des']:
243 |             total_report['3des'] += 1
244 | 
245 |     return total_report
246 | 
247 | # == filters ==
248 | 
249 | def live_only(pshtt, agency, branch):
250 |     return pshtt["Live"] == "True"
251 | 
252 | # return a dynamic filter function for a given agency
253 | def for_agencies(filter_agencies):
254 |     # render case-insensitive
255 |     filtered = [agency.lower() for agency in filter_agencies]
256 | 
257 |     def the_filter(pshtt, agency, branch):
258 |         live = live_only(pshtt, agency, branch)
259 |         return live and (agency.lower() in filtered)
260 | 
261 |     return the_filter
262 | 
263 | # a specific list of executive branch domains
264 | # used to filter on DAP-tracked domains
265 | def specific_executive(list_of_domains):
266 | 
267 |     def the_filter(pshtt, agency, branch):
268 |         executive = executive_only(pshtt, agency, branch)
269 |         return executive and (pshtt['Domain'] in list_of_domains)
270 | 
271 |     return the_filter
272 | 
273 | def executive_only(pshtt, agency, branch):
274 |     live = live_only(pshtt, agency, branch)
275 |     return live and (branch == "executive")
276 | 
277 | def non_executive(pshtt, agency, branch):
278 |     live = live_only(pshtt, agency, branch)
279 |     return live and (branch != "executive")
280 | 
281 | def legislative_only(pshtt, agency, branch):
282 |     live = live_only(pshtt, agency, branch)
283 |     return live and (branch == "legislative")
284 | 
285 | def judicial_only(pshtt, agency, branch):
286 |     live = live_only(pshtt, agency, branch)
287 |     return live and (branch == "judicial")
288 | 
289 | def cfo_act_only(pshtt, agency, branch):
290 |     live = live_only(pshtt, agency, branch)
291 |     return live and cfo_act(agency)
292 | 
293 | def executive_non_cfo_act(pshtt, agency, branch):
294 |     executive = executive_only(pshtt, agency, branch)
295 |     return executive and (not cfo_act(agency))
296 | 
297 | def cfo_act_only_sans_dod(pshtt, agency, branch):
298 |     live = live_only(pshtt, agency, branch)
299 |     return live and cfo_act_sans_dod(agency)
300 | 
301 | # Return base domain for a subdomain.
302 | # Simplified to not use PSL for the federal use case, while handling .fed.us.
303 | def base_domain_for(subdomain):
304 |     if subdomain.endswith(".fed.us"):
305 |         return str.join(".", subdomain.split(".")[-3:])
306 |     else: # .gov
307 |         return str.join(".", subdomain.split(".")[-2:])
308 | 
309 | # save an array of domain names to a CSV at a given path
310 | def save_domains(domains, path):
311 |     output = io.StringIO()
312 |     writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
313 |     header = ["Domain", "Base Domain"]
314 |     writer.writerow(header)
315 |     for domain in domains:
316 |         writer.writerow([domain, base_domain_for(domain)])
317 |     write(output.getvalue(), path)
318 | 
319 | def save_csv(header, rows, path):
320 |     output = io.StringIO()
321 |     writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
322 |     writer.writerow(header)
323 |     for row in rows:
324 |         writer.writerow(row)
325 |     write(output.getvalue(), path)
326 | 
327 | 
328 | # Load the first column of a CSV into memory as an array of strings.
329 | def load_domains(domain_csv, whole_rows=False):
330 |     domains = []
331 |     with open(domain_csv, newline='') as csvfile:
332 |         for row in csv.reader(csvfile):
333 |             if (not row[0]) or (row[0].lower() == "domain") or (row[0].lower() == "domain name"):
334 |                 continue
335 | 
336 |             row[0] = row[0].lower()
337 | 
338 |             if whole_rows:
339 |                 domains.append(row)
340 |             else:
341 |                 domains.append(row[0])
342 |     return domains
343 | 
344 | # Return true if it's a CFO Act agency
345 | def cfo_act(agency):
346 |     return agency in [
347 |         "U.S. Department of Agriculture",
348 |         "Department of Commerce",
349 |         "Department of Defense",
350 |         "Department of Education",
351 |         "Department of Energy",
352 |         "Department of Health and Human Services",
353 |         "Department of Homeland Security",
354 |         "Department of Housing and Urban Development",
355 |         "Department of the Interior",
356 |         "Department of Justice",
357 |         "Terrorist Screening Center", # (DOJ/FBI) as seen in .gov data
358 |         "Department of Labor",
359 |         "Department of State",
360 |         "Department of State, Office of Inspector General", # seen in .gov data
361 |         "Department of Transportation",
362 |         "Department of the Treasury",
363 |         "Department of Veterans Affairs",
364 |         "Environmental Protection Agency",
365 |         "National Aeronautics and Space Administration",
366 |         "U.S. Agency for International Development",
367 |         "General Services Administration",
368 |         "National Science Foundation",
369 |         "Nuclear Regulatory Commission",
370 |         "Office of Personnel Management",
371 |         "Small Business Administration",
372 |         "Social Security Administration"
373 |     ]
374 | 
375 | # Return true if it's a CFO act agency and not DoD
376 | def cfo_act_sans_dod(agency):
377 |     return cfo_act(agency) and (agency != "Department of Defense")
378 | 
379 | # Taken from a snapshot of Pulse's branch detection code:
380 | # https://github.com/18F/pulse/blob/c5b15bc1d2c9e4ff3b20e352e0a0b4f3d131d70f/data/processing.py#L1165-L1174
381 | def branch_for(domain_type):
382 |   if (not domain_type.startswith("Federal Agency - ")):
383 |     return None
384 | 
385 |   branch = domain_type.replace("Federal Agency - ", "")
386 |   branch = branch.lower().strip()
387 | 
388 |   return branch
389 | 
390 | 
391 | # Download official GSA .gov dataset and create a dict of domains to agencies/branches.
392 | def domains_to_agencies():
393 |     official_csv = "cache/federal-domains.csv"
394 |     if not os.path.exists(official_csv):
395 |         print("Downloading federal domains...")
396 | 
397 |         # Snapshot as of 2018-04-27
398 |         url = "https://github.com/GSA/data/raw/4513ad8ce1548fdb34509e7361d806a5a1e4288b/dotgov-domains/current-federal.csv"
399 |         official_csv = download(url, "cache/federal-domains.csv")
400 | 
401 |     official_data = load_domains(official_csv, whole_rows=True)
402 | 
403 |     # header row for latest .gov data
404 |     headers = [
405 |         "Domain Name", "Domain Type",
406 |         "Agency", "Organization", "City", "State"
407 |     ]
408 | 
409 |     domain_map = {}
410 |     for row in official_data:
411 |         # Turn row headers into a dict.
412 |         dict_row = {}
413 |         for i, cell in enumerate(row):
414 |             dict_row[headers[i]] = cell
415 | 
416 |         domain = dict_row["Domain Name"].lower()
417 |         agency = dict_row["Agency"].strip()
418 |         domain_type = dict_row["Domain Type"]
419 |         branch = branch_for(domain_type)
420 | 
421 |         domain_map[domain] = {
422 |             "agency": agency,
423 |             "branch": branch
424 |         }
425 | 
426 |     return domain_map
427 | 
428 | # Not great at error handling.
429 | def download(url, destination):
430 |     filename, headers = urllib.request.urlretrieve(url, destination)
431 |     return filename
432 | 
433 | # Load one or more pshtt scan files, and turn them into a dict.
434 | # If sending in multiple paths, send from oldest to newest, as
435 | # later scan results for the same domain will overwrite older ones.
436 | def load_pshtt_sslyze(pshtts, sslyzes, base_domains, preloaded, filter=None):
437 |     data = {}
438 | 
439 |     for path in pshtts:
440 |         headers = []
441 |         with open(path, newline='') as csvfile:
442 |             for row in csv.reader(csvfile):
443 |                 if (row[0].lower() == "domain"):
444 |                     headers = row
445 |                     continue
446 | 
447 |                 domain = row[0].lower()
448 |                 base_domain = base_domain_for(domain)
449 |                 if not base_domains.get(base_domain):
450 |                     # print("[load_pshtt] Skipping %s, not a (current) federal domain." % domain)
451 |                     continue
452 | 
453 |                 agency = base_domains[base_domain]["agency"]
454 |                 branch = base_domains[base_domain]["branch"]
455 | 
456 |                 if agency == "Non-Federal Agency":
457 |                     # print("[load_pshtt] Skipping %s, marked as non-federal." % domain)
458 |                     continue
459 | 
460 |                 pshtt = {}
461 |                 for i, cell in enumerate(row):
462 |                     pshtt[headers[i]] = cell
463 | 
464 |                 if filter and (not filter(pshtt, agency, branch)):
465 |                     # print("[load_pshtt] Skipping %s, didn't pass filter." % domain)
466 |                     continue
467 | 
468 |                 # might overwrite a scan result from a previous
469 |                 # CSV file - that's fine, we'll just go with the
470 |                 # latest one, so inputs should use latest last.
471 |                 data[domain] = {
472 |                     'pshtt': pshtt,
473 |                     'base_domain': base_domain,
474 |                     'agency': agency,
475 |                     'branch': branch,
476 |                     'cfo_act': cfo_act(agency)
477 |                 }
478 | 
479 |     for path in sslyzes:
480 |         headers = []
481 |         with open(path, newline='') as csvfile:
482 |             for row in csv.reader(csvfile):
483 |                 if (row[0].lower() == "domain"):
484 |                     headers = row
485 |                     continue
486 | 
487 |                 domain = row[0].lower()
488 |                 base_domain = base_domain_for(domain)
489 |                 if not data.get(domain):
490 |                     # print("[load_sslyze] Skipping %s, filtered out." % domain)
491 |                     continue
492 | 
493 |                 sslyze = {}
494 |                 for i, cell in enumerate(row):
495 |                     sslyze[headers[i]] = cell
496 | 
497 |                 # if no TLSv1.2 value, then no scan was performed
498 |                 if sslyze["TLSv1.2"]:
499 |                     data[domain]['sslyze'] = sslyze
500 | 
501 | 
502 |     # Run each domain through compliance_for, with the pshtt, sslyze,
503 |     # and whether its parent is known to be preloaded
504 |     domains = list(data.keys())
505 |     domains.sort()
506 |     for domain in domains:
507 |         # Debugging:
508 |         # print(domain)
509 | 
510 |         base_domain = base_domain_for(domain)
511 |         if domain == base_domain:
512 |             parent_preloaded = False # Not relevant for base domains themselves.
513 |         else:
514 |             parent_preloaded = (base_domain in preloaded)
515 | 
516 |         data[domain]['compliance'] = compliance_for(data[domain]['pshtt'], data[domain].get('sslyze'), parent_preloaded)
517 | 
518 |     return data
519 | 
520 | def fetch_preloaded():
521 |     preload_json = None
522 | 
523 |     preload_path = "cache/preload.json"
524 | 
525 |     if os.path.exists(preload_path):
526 |         logging.debug("Using cached Chrome preload list.")
527 |         preload_json = json.loads(open(preload_path).read())
528 |     else:
529 |         logging.debug("Fetching Chrome preload list from source...")
530 | 
531 |         # Downloads the chromium preloaded domain list and sets it to a global set
532 |         file_url = 'https://chromium.googlesource.com/chromium/src/net/+/master/http/transport_security_state_static.json?format=TEXT'
533 | 
534 |         # TODO: proper try/except around this network request
535 |         request = requests.get(file_url)
536 |         raw = request.content
537 | 
538 |         # To avoid parsing the contents of the file out of the source tree viewer's
539 |         # HTML, we download it as a raw file. googlesource.com Base64-encodes the
540 |         # file to avoid potential content injection issues, so we need to decode it
541 |         # before using it. https://code.google.com/p/gitiles/issues/detail?id=7
542 |         raw = base64.b64decode(raw).decode('utf-8')
543 | 
544 |         # The .json file contains '//' comments, which are not actually valid JSON,
545 |         # and confuse Python's JSON decoder. Begone, foul comments!
546 |         raw = ''.join([re.sub(r'^\s*//.*$', '', line)
547 |                        for line in raw.splitlines()])
548 | 
549 |         preload_json = json.loads(raw)
550 | 
551 |         logging.debug("Caching preload list at %s" % preload_path)
552 |         write(json_for(preload_json), preload_path)
553 | 
554 |     # For our purposes, we only care about entries that includeSubDomains
555 |     fully_preloaded = []
556 |     for entry in preload_json['entries']:
557 |         if entry.get('include_subdomains', False) is True:
558 |             fully_preloaded.append(entry['name'])
559 | 
560 |     return fully_preloaded
561 | 
562 | def fetch_preload_pending():
563 | 
564 |     pending_path = "cache/preload-pending.json"
565 |     if not os.path.exists(pending_path):
566 |         logging.debug("Fetching Chrome pending preload list...")
567 | 
568 |         pending_url = "https://hstspreload.org/api/v2/pending"
569 |         download(pending_url, pending_path)
570 | 
571 |     raw = open(pending_path).read()
572 |     pending_json = json.loads(raw)
573 | 
574 |     # For our purposes, we only care about entries that include subdomains.
575 |     pending = []
576 |     for entry in pending_json:
577 |         if entry.get('include_subdomains', False) is True:
578 |             pending.append(entry['name'])
579 | 
580 |     return pending
581 | 
582 | def write(content, destination):
583 |     mkdir_p(os.path.dirname(destination))
584 | 
585 |     f = open(destination, 'w', encoding='utf-8')
586 |     f.write(content)
587 |     f.close()
588 | 
589 | def json_for(object):
590 |     return json.dumps(object, sort_keys=True,
591 |                       indent=2, default=format_datetime)
592 | 
593 | def format_datetime(obj):
594 |     if isinstance(obj, datetime.date):
595 |         return obj.isoformat()
596 |     elif isinstance(obj, str):
597 |         return obj
598 |     else:
599 |         return None
600 | 
601 | # mkdir -p in python, from:
602 | # https://stackoverflow.com/questions/600268/mkdir-p-functionality-in-python
603 | def mkdir_p(path):
604 |     try:
605 |         os.makedirs(path)
606 |     except OSError as exc:  # Python >2.5
607 |         if exc.errno == errno.EEXIST:
608 |             pass
609 |         else:
610 |             raise
611 | 


--------------------------------------------------------------------------------
/compliance/m-15-13/compliance.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | import utils
 4 | 
 5 | # get mapping of domains to agency
 6 | base_domains = utils.domains_to_agencies()
 7 | 
 8 | # get current and pending preloaded domains
 9 | # Downloads live from remote sources. (clear cache/ to re-download)
10 | preloaded = set(utils.fetch_preloaded()) & set(base_domains)
11 | preload_pending = set(utils.fetch_preload_pending()) & set(base_domains)
12 | 
13 | # paths to scanned data
14 | parents_path = "data/parents-2016-12-31.csv"
15 | dap_path = "data/dap-2016-12-31.csv"
16 | censys_path = "data/censys-2016-12-31.csv"
17 | eot_2012_path = "data/eot-2012.csv"
18 | eot_2016_path = "data/eot-2016.csv"
19 | 
20 | # past scans
21 | def past(type, date):
22 |   return ("data/%s-%s.csv" % (type, date))
23 | 
24 | 
25 | def pct(num, denom):
26 |   return round((num / denom) * 100)
27 | 
28 | def compliance_stats(name, scans, filter):
29 |   data = utils.load_pshtt(scans, base_domains, filter=filter)
30 |   totals = utils.compliance_totals(data, preload_pending, preloaded)
31 | 
32 |   count = len(data.keys())
33 | 
34 |   print()
35 |   print("=====================================================")
36 |   print("  %s" % name)
37 |   print("=====================================================")
38 |   print()
39 |   print("Total domains: %i" % count)
40 |   print()
41 |   print("== Direct enforcement ==")
42 |   print("Uses HTTPS: %i (%i%%)" % (totals['uses'], pct(totals['uses'], count)))
43 |   print("Enforces HTTPS: %i (%i%%)" % (totals['enforces'], pct(totals['enforces'], count)))
44 |   print("Strong HSTS: %i (%i%%)" % (totals['hsts'], pct(totals['hsts'], count)))
45 |   # print("Strong HSTS (+ preloading): %i (%i%%)" % (totals['hsts_or_preloading'], pct(totals['hsts_or_preloading'], count)))
46 |   # print()
47 |   # print("== Coverage through preloading alone ==")
48 |   # print("Preloaded: %i (%i%%)" % (totals['through_preloading'], pct(totals['through_preloading'], count)))
49 |   # print("Preload pending: %i (%i%%)" % (totals['through_preloading_pending'], pct(totals['through_preloading_pending'], count)))
50 | 
51 |   print()
52 | 
53 | def compliance_csv(reports, path):
54 |   header = ["Type", "Name", "Total Domains", "Uses HTTPS", "Enforces HTTPS", "HSTS", "Uses HTTPS (%)", "Enforces HTTPS (%)", "HSTS (%)"]
55 |   rows = []
56 |   for report in reports:
57 |     rows.append(compliance_csv_row(report))
58 | 
59 |   utils.save_csv(header, rows, path)
60 | 
61 | def compliance_csv_row(report):
62 |   data = utils.load_pshtt(report['files'], base_domains, filter=report['filter'])
63 |   totals = utils.compliance_totals(data, preload_pending, preloaded)
64 | 
65 |   count = len(data.keys())
66 | 
67 |   return [
68 |     report['type'],
69 |     report['name'],
70 |     count,
71 |     totals['uses'],
72 |     totals['enforces'],
73 |     totals['hsts'],
74 |     pct(totals['uses'], count),
75 |     pct(totals['enforces'], count),
76 |     pct(totals['hsts'], count),
77 |   ]
78 | 
79 | # interesting swathes to check for compliance
80 | parents_only = [parents_path]
81 | all_domains = [parents_path, dap_path, censys_path, eot_2012_path, eot_2016_path]
82 | 
83 | # This should reproduce Pulse top-line stats: live second-level domains
84 | # compliance_stats(parents_only, utils.live_only)
85 | 
86 | parent_pasts = ['2015-06-13',  '2015-06-19',  '2015-06-26',  '2015-07-03',  '2015-07-10',  '2015-07-17',  '2015-07-24',  '2015-07-31',  '2015-08-07',  '2015-08-14',  '2015-08-21',  '2015-08-28',  '2015-09-04',  '2015-09-11',  '2015-09-18',  '2015-09-25',  '2015-10-09',  '2015-10-16',  '2015-10-22',  '2015-10-30',  '2015-11-06',  '2015-11-13',  '2015-11-21',  '2015-11-28',  '2015-12-04',  '2015-12-10',  '2015-12-18',  '2015-12-26',  '2016-01-02',  '2016-01-08',  '2016-01-15',  '2016-01-23',  '2016-01-29',  '2016-02-04',  '2016-02-08',  '2016-02-17',  '2016-02-26',  '2016-03-03',  '2016-03-13',  '2016-03-18',  '2016-03-25',  '2016-04-01',  '2016-04-08',  '2016-04-15',  '2016-04-22',  '2016-04-29',  '2016-05-06',  '2016-05-13',  '2016-05-20',  '2016-05-27',  '2016-06-03',  '2016-07-05',  '2016-07-08',  '2016-07-15',  '2016-07-22',  '2016-07-29',  '2016-08-05',  '2016-08-12',  '2016-08-19',  '2016-08-26',  '2016-09-02',  '2016-09-09',  '2016-09-16',  '2016-09-23',  '2016-10-03',  '2016-10-07',  '2016-10-14',  '2016-10-21',  '2016-10-28',  '2016-11-04',  '2016-11-11',  '2016-11-18',  '2016-11-25',  '2016-12-02',  '2016-12-05',  '2016-12-08',  '2016-12-17',  '2016-12-21',  '2016-12-24',  '2016-12-26',  '2016-12-28',  '2016-12-31']
87 | dap_pasts = ["2016-07-12", "2016-12-05", "2016-12-31"]
88 | censys_pasts = ["2016-08-04", "2016-12-05", "2016-12-31"]
89 | 
90 | reports = []
91 | reports.append({
92 |   'type': 'Most subdomains on 12/31 (all branches)',
93 |   'name': "2016-12-31",
94 |   'files': all_domains,
95 |   'filter': utils.executive_only
96 | })
97 | compliance_csv(reports, "cache/all-domains-executive-snapshot.csv")


--------------------------------------------------------------------------------
/compliance/m-15-13/reports.py:
--------------------------------------------------------------------------------
  1 | 
  2 | #########################################################
  3 | #
  4 | # Just scratch space, this won't execute or import properly.
  5 | #
  6 | #########################################################
  7 | 
  8 | name = "All executive parent domains"
  9 | compliance_stats(name, parents_only, utils.executive_only)
 10 | 
 11 | name = "All executive CFO Act parent domains"
 12 | compliance_stats(name, parents_only, utils.cfo_act_only)
 13 | 
 14 | name = "All executive CFO Act (minus DoD) parent domains"
 15 | compliance_stats(name, parents_only, utils.cfo_act_only_sans_dod)
 16 | 
 17 | name = "All executive non-CFO-Act parent domains"
 18 | compliance_stats(name, parents_only, utils.executive_non_cfo_act)
 19 | 
 20 | 
 21 | name = "All subdomains"
 22 | compliance_stats(name, subdomains, utils.live_only)
 23 | 
 24 | name = "All executive subdomains"
 25 | compliance_stats(name, subdomains, utils.executive_only)
 26 | 
 27 | name = "All executive CFO Act subdomains"
 28 | compliance_stats(name, subdomains, utils.cfo_act_only)
 29 | 
 30 | name = "All executive CFO Act (minus DoD) subdomains"
 31 | compliance_stats(name, subdomains, utils.cfo_act_only_sans_dod)
 32 | 
 33 | name = "All executive non-CFO-Act subdomains"
 34 | compliance_stats(name, subdomains, utils.executive_non_cfo_act)
 35 | 
 36 | # agency = "General Services Administration"
 37 | # compliance_stats(agency, plus_dap_and_censys, utils.for_agency(agency))
 38 | 
 39 | for date in parents_pasts:
 40 |   name = "All parent domains (%s)" % date
 41 |   compliance_stats(name, [past("parents", date)], utils.live_only)
 42 | 
 43 | name = "All parent domains (2016-12-28)"
 44 | compliance_stats(name, parents_only, utils.live_only)
 45 | 
 46 | 
 47 | for date in dap_pasts:
 48 |   name = "All DAP subdomains (%s)" % date
 49 |   compliance_stats(name, [past("dap", date)], utils.live_only)
 50 | 
 51 | name = "All DAP subdomains (2016-12-28)"
 52 | compliance_stats(name, [dap_path], utils.live_only)
 53 | 
 54 | 
 55 | for date in censys_pasts:
 56 |   name = "All Censys subdomains (%s)" % date
 57 |   compliance_stats(name, [past("censys", date)], utils.live_only)
 58 | 
 59 | name = "All Censys subdomains (2016-12-28)"
 60 | compliance_stats(name, [censys_path], utils.live_only)
 61 | 
 62 | 
 63 | 
 64 | 
 65 | reports = []
 66 | for date in parent_pasts:
 67 |   reports.append({
 68 |     'type': 'parents',
 69 |     'name': date,
 70 |     'files': [past('parents', date)],
 71 |     'filter': utils.live_only
 72 |   })
 73 | compliance_csv(reports, "cache/parents-history.csv")
 74 | 
 75 | 
 76 | cfo_act_groups = [
 77 |     "Department of Agriculture",
 78 |     "Department of Commerce",
 79 |     "Department of Defense",
 80 |     "Department of Education",
 81 |     "Department of Energy",
 82 |     "Department of Health and Human Services",
 83 |     "Department of Homeland Security",
 84 |     "Department of Housing and Urban Development",
 85 |     "Department of the Interior",
 86 |     [
 87 |         "Department of Justice",
 88 |         "Terrorist Screening Center", # (DOJ/FBI) as seen in .gov data
 89 |     ],
 90 |     "Department of Labor",
 91 |     [
 92 |         "Department of State",
 93 |         "Department of State OIG", # seen in .gov data
 94 |     ],
 95 |     "Department of Transportation",
 96 |     "Department of the Treasury",
 97 |     "Department of Veterans Affairs",
 98 |     "Environmental Protection Agency",
 99 |     "National Aeronautics and Space Administration",
100 |     "U.S. Agency for International Development", # as seen in .gov data
101 |     "General Services Administration",
102 |     "National Science Foundation",
103 |     "Nuclear Regulatory Commission",
104 |     "Office of Personnel Management",
105 |     "Small Business Administration",
106 |     "Social Security Administration"
107 | ]
108 | 
109 | 
110 | reports = []
111 | for group in cfo_act_groups:
112 |   if isinstance(group, str):
113 |     group = [group]
114 | 
115 |   reports.append({
116 |     'type': 'agency parent domains',
117 |     'name': str.join(", ", group),
118 |     'files': parents_only,
119 |     'filter': utils.for_agencies(group)
120 |   })
121 | 
122 | 
123 | compliance_csv(reports, "cache/agencies-parents-latest.csv")
124 | 
125 | 
126 | 
127 | 
128 | # approximation of all subdomains, given data known from 3 sources at 3 junctures
129 | all_pasts = [
130 |   ["data/parents-2016-07-15.csv", "data/dap-2016-07-12.csv", "data/censys-2016-08-04.csv"],
131 |   ["data/parents-2016-12-05.csv", "data/dap-2016-12-05.csv", "data/censys-2016-12-05.csv"],
132 |   ["data/parents-2016-12-31.csv", "data/dap-2016-12-31.csv", "data/censys-2016-12-31.csv"],
133 | ]
134 | 
135 | reports = []
136 | reports.append({
137 |   'type': 'all domains/subdomains',
138 |   'name': "2016-08-01", # approximate
139 |   'files': all_pasts[0],
140 |   'filter': utils.live_only
141 | })
142 | reports.append({
143 |   'type': 'all domains/subdomains',
144 |   'name': "2016-12-05",
145 |   'files': all_pasts[1],
146 |   'filter': utils.live_only
147 | })
148 | reports.append({
149 |   'type': 'all domains/subdomains',
150 |   'name': "2016-12-31",
151 |   'files': all_pasts[2],
152 |   'filter': utils.live_only
153 | })
154 | 
155 | 
156 | compliance_csv(reports, "cache/all-domains-last-6-months.csv")
157 | 
158 | 
159 | 
160 | 
161 | reports = []
162 | for date in parent_pasts:
163 |   reports.append({
164 |     'type': 'parents',
165 |     'name': date,
166 |     'files': [past('parents', date)],
167 |     'filter': utils.cfo_act_only
168 |   })
169 | compliance_csv(reports, "cache/cfo-act-parents-history.csv")
170 | 
171 | 
172 | 
173 | 
174 | reports = []
175 | reports.append({
176 |   'type': 'Most subdomains on 12/31 (all branches)',
177 |   'name': "2016-12-31",
178 |   'files': all_domains,
179 |   'filter': utils.cfo_act_only
180 | })
181 | compliance_csv(reports, "cache/cfo-act-all-domains-snapshot.csv")
182 | 
183 | 
184 | 
185 | 
186 | 
187 | reports = []
188 | reports.append({
189 |   'type': 'Most subdomains on 12/31 (all branches)',
190 |   'name': "2016-12-31",
191 |   'files': all_domains,
192 |   'filter': utils.live_only
193 | })
194 | compliance_csv(reports, "cache/all-domains-snapshot.csv")
195 | 
196 | 
197 | 
198 | 
199 | # Estimating traffic covered by HTTPS, now (for subdomains)
200 | # and over time (for parent domains).
201 | # Load in all DAP-tracked domains with > 1000 visits over the prior 30 days
202 | analytics_rows = utils.load_domains("data/analytics-domains/all-domains-30-days-2017-01-01.csv", whole_rows=True)
203 | analytics = {}
204 | for row in analytics_rows:
205 |   analytics[row[0].lower()] = int(row[1])
206 | 
207 | # load the exec branch domains to evaluate against, filter to those included
208 | data = utils.load_pshtt(
209 |   all_domains, base_domains,
210 |   filter=utils.specific_executive(list(analytics.keys()))
211 | )
212 | 
213 | # Now go through each one and write out a spreadsheet combining the
214 | # results for each domain with their visits as recorded in DAP
215 | 
216 | tracked_domains = list(data.keys())
217 | tracked_domains.sort()
218 | 
219 | headers = ["Domain", "Base Domain", "Visits", "Uses HTTPS", "Enforces HTTPS", "HSTS", "Downgrades HTTPS"]
220 | rows = []
221 | for domain in tracked_domains:
222 |   labels = utils.compliance_labels(data[domain]['compliance'])
223 | 
224 |   rows.append([
225 |     domain,
226 |     data[domain]['base_domain'],
227 |     analytics[domain],
228 |     labels['uses'],
229 |     labels['enforces'],
230 |     labels['hsts'],
231 |     data[domain]['pshtt']['Downgrades HTTPS']
232 |   ])
233 | 
234 | utils.save_csv(headers, rows, "cache/https-by-dap-visits.csv")
235 | 
236 | 


--------------------------------------------------------------------------------
/compliance/m-15-13/utils.py:
--------------------------------------------------------------------------------
  1 | import csv
  2 | import io
  3 | import os
  4 | import errno
  5 | import urllib
  6 | import urllib.request
  7 | import logging
  8 | import requests
  9 | import base64
 10 | import re
 11 | import json
 12 | 
 13 | # == compliance
 14 | 
 15 | # Given a dict of pshtt results for a hostname, what are the
 16 | # Uses, Enforces, and HSTS conclusions?
 17 | #
 18 | # borrows the pulse.cio.gov code for calculating conclusions:
 19 | # https://github.com/18F/pulse/blob/0528773b1d39a664ff8f62d655b8bb7c8979874c/data/processing.py#L408-L509
 20 | def compliance_for(pshtt):
 21 |     report = {}
 22 | 
 23 |     # assumes that HTTPS would be technically present, with or without issues
 24 |     if (pshtt["Downgrades HTTPS"] == "True"):
 25 |       https = 0 # No
 26 |     else:
 27 |       if (pshtt["Valid HTTPS"] == "True"):
 28 |         https = 2 # Yes
 29 |       elif (
 30 |         (pshtt["HTTPS Bad Chain"] == "True") and
 31 |         (pshtt["HTTPS Bad Hostname"] == "False")
 32 |       ):
 33 |         https = 1 # Yes
 34 |       else:
 35 |         https = -1 # No
 36 | 
 37 |     report['uses'] = https
 38 | 
 39 | 
 40 |     ###
 41 |     # Is HTTPS enforced?
 42 | 
 43 |     if (https <= 0):
 44 |       behavior = 0 # N/A
 45 | 
 46 |     else:
 47 | 
 48 |       # "Yes (Strict)" means HTTP immediately redirects to HTTPS,
 49 |       # *and* that HTTP eventually redirects to HTTPS.
 50 |       #
 51 |       # Since a pure redirector domain can't "default" to HTTPS
 52 |       # for itself, we'll say it "Enforces HTTPS" if it immediately
 53 |       # redirects to an HTTPS URL.
 54 |       if (
 55 |         (pshtt["Strictly Forces HTTPS"] == "True") and
 56 |         (
 57 |           (pshtt["Defaults to HTTPS"] == "True") or
 58 |           (pshtt["Redirect"] == "True")
 59 |         )
 60 |       ):
 61 |         behavior = 3 # Yes (Strict)
 62 | 
 63 |       # "Yes" means HTTP eventually redirects to HTTPS.
 64 |       elif (
 65 |         (pshtt["Strictly Forces HTTPS"] == "False") and
 66 |         (pshtt["Defaults to HTTPS"] == "True")
 67 |       ):
 68 |         behavior = 2 # Yes
 69 | 
 70 |       # Either both are False, or just 'Strict Force' is True,
 71 |       # which doesn't matter on its own.
 72 |       # A "present" is better than a downgrade.
 73 |       else:
 74 |         behavior = 1 # Present (considered 'No')
 75 | 
 76 |     report['enforces'] = behavior
 77 | 
 78 | 
 79 |     ###
 80 |     # Characterize the presence and completeness of HSTS.
 81 | 
 82 |     if pshtt["HSTS Max Age"]:
 83 |       hsts_age = int(pshtt["HSTS Max Age"])
 84 |     else:
 85 |       hsts_age = None
 86 | 
 87 |     # Without HTTPS there can be no HSTS.
 88 |     if (https <= 0):
 89 |       hsts = -1  # N/A (considered 'No')
 90 | 
 91 |     else:
 92 | 
 93 |       # HSTS is present for the canonical endpoint.
 94 |       if (pshtt["HSTS"] == "True"):
 95 | 
 96 |         # Say No for too-short max-age's, and note in the extended details.
 97 |         if hsts_age >= 31536000:
 98 |           hsts = 2  # Yes
 99 |         else:
100 |           hsts = 1  # No
101 | 
102 |       else:
103 |         hsts = 0  # No
104 | 
105 |     # Separate preload status from HSTS status:
106 |     #
107 |     # * Domains can be preloaded through manual overrides.
108 |     # * Confusing to mix an endpoint-level decision with a domain-level decision.
109 |     if pshtt.get("HSTS Preloaded") == "True":
110 |       preloaded = 2  # Yes
111 |     elif (pshtt["HSTS Preload Ready"] == "True"):
112 |       preloaded = 1  # Ready for submission
113 |     else:
114 |       preloaded = 0  # No
115 | 
116 |     report['hsts'] = hsts
117 |     report['preloaded'] = preloaded
118 | 
119 |     return report
120 | 
121 | # Given a compliance report (e.g. uses=1, enforces=2, hsts=2)
122 | # return a dict with them turned into labels
123 | def compliance_labels(report):
124 |     return {
125 |         'uses': (report['uses'] >= 1),
126 |         'enforces': (report['enforces'] >= 2),
127 |         'hsts': (report['hsts'] >= 2),
128 |     }
129 | 
130 | # Given a set of domains, calculate the compliance totals
131 | # that meet M-15-13 requirements.
132 | # Adapted from a snapshot of Pulse code:
133 | # https://github.com/18F/pulse/blob/0528773b1d39a664ff8f62d655b8bb7c8979874c/data/processing.py#L511-L539
134 | 
135 | def compliance_totals(data, preload_pending, preloaded):
136 |     total_report = {
137 |         'uses': 0,
138 |         'enforces': 0,
139 |         'hsts': 0,
140 | 
141 |         # includes auto-protected subdomains
142 |         'through_preloading_pending': 0,
143 |         'through_preloading': 0,
144 |         'hsts_or_preloading': 0,
145 |         'hsts_or_preloading_or_pending': 0,
146 |     }
147 | 
148 |     for domain in data.keys():
149 |         # calculated in compliance_for() method using pshtt dat
150 |         report = data[domain]['compliance']
151 | 
152 |         # Needs to be enabled, it's allowed for the chain to have issues
153 |         if report['uses'] >= 1:
154 |             total_report['uses'] += 1
155 | 
156 |         # Needs to be Default or Strict to be 'Yes'
157 |         if report['enforces'] >= 2:
158 |             total_report['enforces'] += 1
159 | 
160 |         # Needs to be present with >= 1 year max-age for canonical endpoint
161 |         if report['hsts'] >= 2:
162 |             total_report['hsts'] += 1
163 | 
164 |         if data[domain]['base_domain'] in preload_pending:
165 |             total_report['through_preloading_pending'] += 1
166 | 
167 |         if data[domain]['base_domain'] in preloaded:
168 |             total_report['through_preloading'] += 1
169 | 
170 |         if (
171 |             (report['hsts'] >= 2) or
172 |             (data[domain]['base_domain'] in preloaded)
173 |         ):
174 |             total_report['hsts_or_preloading'] += 1
175 | 
176 |         if (
177 |             (report['hsts'] >= 2) or
178 |             (data[domain]['base_domain'] in preloaded) or
179 |             (data[domain]['base_domain'] in preload_pending)
180 |         ):
181 |             total_report['hsts_or_preloading_or_pending'] += 1
182 |     return total_report
183 | 
184 | # == filters ==
185 | 
186 | def live_only(pshtt, agency, branch):
187 |     return pshtt["Live"] == "True"
188 | 
189 | # return a dynamic filter function for a given agency
190 | def for_agencies(filter_agencies):
191 |     # render case-insensitive
192 |     filtered = [agency.lower() for agency in filter_agencies]
193 | 
194 |     def the_filter(pshtt, agency, branch):
195 |         live = live_only(pshtt, agency, branch)
196 |         return live and (agency.lower() in filtered)
197 | 
198 |     return the_filter
199 | 
200 | # a specific list of executive branch domains
201 | # used to filter on DAP-tracked domains
202 | def specific_executive(list_of_domains):
203 | 
204 |     def the_filter(pshtt, agency, branch):
205 |         executive = executive_only(pshtt, agency, branch)
206 |         return executive and (pshtt['Domain'] in list_of_domains)
207 | 
208 |     return the_filter
209 | 
210 | def executive_only(pshtt, agency, branch):
211 |     live = live_only(pshtt, agency, branch)
212 |     return live and (branch == "executive")
213 | 
214 | def non_executive(pshtt, agency, branch):
215 |     live = live_only(pshtt, agency, branch)
216 |     return live and (branch != "executive")
217 | 
218 | def legislative_only(pshtt, agency, branch):
219 |     live = live_only(pshtt, agency, branch)
220 |     return live and (branch == "legislative")
221 | 
222 | def judicial_only(pshtt, agency, branch):
223 |     live = live_only(pshtt, agency, branch)
224 |     return live and (branch == "judicial")
225 | 
226 | def cfo_act_only(pshtt, agency, branch):
227 |     live = live_only(pshtt, agency, branch)
228 |     return live and cfo_act(agency)
229 | 
230 | def executive_non_cfo_act(pshtt, agency, branch):
231 |     executive = executive_only(pshtt, agency, branch)
232 |     return executive and (not cfo_act(agency))
233 | 
234 | def cfo_act_only_sans_dod(pshtt, agency, branch):
235 |     live = live_only(pshtt, agency, branch)
236 |     return live and cfo_act_sans_dod(agency)
237 | 
238 | # return base domain for a subdomain
239 | def base_domain_for(subdomain):
240 |     return str.join(".", subdomain.split(".")[-2:])
241 | 
242 | # save an array of domain names to a CSV at a given path
243 | def save_domains(domains, path):
244 |     output = io.StringIO()
245 |     writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
246 |     header = ["Domain", "Base Domain"]
247 |     writer.writerow(header)
248 |     for domain in domains:
249 |         writer.writerow([domain, base_domain_for(domain)])
250 |     write(output.getvalue(), path)
251 | 
252 | def save_csv(header, rows, path):
253 |     output = io.StringIO()
254 |     writer = csv.writer(output, quoting=csv.QUOTE_NONNUMERIC)
255 |     writer.writerow(header)
256 |     for row in rows:
257 |         writer.writerow(row)
258 |     write(output.getvalue(), path)
259 | 
260 | 
261 | # Load the first column of a CSV into memory as an array of strings.
262 | def load_domains(domain_csv, whole_rows=False):
263 |     domains = []
264 |     with open(domain_csv, newline='') as csvfile:
265 |         for row in csv.reader(csvfile):
266 |             if (not row[0]) or (row[0].lower().startswith("domain")):
267 |                 continue
268 | 
269 |             row[0] = row[0].lower()
270 | 
271 |             if whole_rows:
272 |                 domains.append(row)
273 |             else:
274 |                 domains.append(row[0])
275 |     return domains
276 | 
277 | # Return true if it's a CFO Act agency
278 | def cfo_act(agency):
279 |     return agency in [
280 |         "Department of Agriculture",
281 |         "Department of Commerce",
282 |         "Department of Defense",
283 |         "Department of Education",
284 |         "Department of Energy",
285 |         "Department of Health and Human Services",
286 |         "Department of Homeland Security",
287 |         "Department of Housing and Urban Development",
288 |         "Department of the Interior",
289 |         "Department of Justice",
290 |         "Terrorist Screening Center", # (DOJ/FBI) as seen in .gov data
291 |         "Department of Labor",
292 |         "Department of State",
293 |         "Department of State OIG", # seen in .gov data
294 |         "Department of Transportation",
295 |         "Department of the Treasury",
296 |         "Department of Veterans Affairs",
297 |         "Environmental Protection Agency",
298 |         "National Aeronautics and Space Administration",
299 |         "U.S. Agency for International Development", # as seen in .gov data
300 |         "General Services Administration",
301 |         "National Science Foundation",
302 |         "Nuclear Regulatory Commission",
303 |         "Office of Personnel Management",
304 |         "Small Business Administration",
305 |         "Social Security Administration"
306 |     ]
307 | 
308 | # Return true if it's a CFO act agency and not DoD
309 | def cfo_act_sans_dod(agency):
310 |     return cfo_act(agency) and (agency != "Department of Defense")
311 | 
312 | # Taken from a snapshot of Pulse's branch detection code:
313 | # https://github.com/18F/pulse/blob/0528773b1d39a664ff8f62d655b8bb7c8979874c/data/processing.py#L789-L813
314 | def branch_for(agency):
315 |   if agency in [
316 |     "Library of Congress",
317 |     "The Legislative Branch (Congress)",
318 |     "Government Printing Office",
319 |     "Government Publishing Office",
320 |     "Congressional Office of Compliance",
321 |     "Stennis Center for Public Service",
322 |     "U.S. Capitol Police",
323 |     "Architect of the Capitol"
324 |   ]:
325 |     return "legislative"
326 | 
327 |   if agency in [
328 |     "The Judicial Branch (Courts)",
329 |     "The Supreme Court",
330 |     "U.S Courts"
331 |   ]:
332 |     return "judicial"
333 | 
334 |   # I think these:
335 |   #     jusfc.gov, wmatc.gov, and heritageabroad.gov
336 |   # are in the executive branch, in part because of
337 |   # https://www.usa.gov/branches-of-government
338 |   # but to stay consistent with Pulse, and because
339 |   # it's neglible for parent domains (3) and subdomains (~10),
340 |   # I'll continue ignoring them.
341 |   if agency in ["Non-Federal Agency"]:
342 |     return "non-federal"
343 | 
344 |   else:
345 |     return "executive"
346 | 
347 | # Download official GSA .gov dataset and create a dict of domains to agencies.
348 | def domains_to_agencies():
349 |     official_csv = "cache/federal-domains.csv"
350 |     if not os.path.exists(official_csv):
351 |         print("Downloading federal domains...")
352 |         url = "https://github.com/GSA/data/raw/1057ec36a1e97abeaea132d2d9e3e5a31eac5b51/dotgov-domains/current-federal.csv"
353 |         official_csv = download(url, "cache/federal-domains.csv")
354 | 
355 |     official_data = load_domains(official_csv, whole_rows=True)
356 | 
357 |     domain_map = {}
358 |     for row in official_data:
359 |         domain = row[0].lower()
360 |         agency = row[2]
361 |         domain_map[domain] = agency
362 | 
363 |     return domain_map
364 | 
365 | # Not great at error handling.
366 | def download(url, destination):
367 |     filename, headers = urllib.request.urlretrieve(url, destination)
368 |     return filename
369 | 
370 | # Load one or more pshtt scan files, and turn them into a dict.
371 | # If sending in multiple paths, send from oldest to newest, as
372 | # later scan results for the same domain will overwrite older ones.
373 | def load_pshtt(paths, base_domains, filter=None):
374 |     data = {}
375 | 
376 |     for path in paths:
377 |         with open(path, newline='') as csvfile:
378 |             for row in csv.reader(csvfile):
379 |                 if (row[0].lower() == "domain"):
380 |                     headers = row
381 |                     continue
382 | 
383 |                 domain = row[0].lower()
384 |                 base_domain = base_domain_for(domain)
385 |                 if not base_domains.get(base_domain):
386 |                     # print("[load_pshtt] Skipping %s, not a federal domain." % domain)
387 |                     continue
388 | 
389 |                 agency = base_domains[base_domain]
390 |                 branch = branch_for(agency)
391 | 
392 |                 if branch == "non-federal":
393 |                     # print("[load_pshtt] Skipping %s, a non-federal domain." % domain)
394 |                     continue
395 | 
396 |                 pshtt = {}
397 |                 for i, cell in enumerate(row):
398 |                     pshtt[headers[i]] = cell
399 | 
400 |                 # By default, only load in eligible (live) domains.
401 |                 if filter and (not filter(pshtt, agency, branch)):
402 |                     # print("[load_pshtt] Skipping %s, didn't pass filter." % domain)
403 |                     continue
404 | 
405 |                 # might overwrite a scan result from a previous
406 |                 # CSV file - that's fine, we'll just go with the
407 |                 # latest one, so inputs should use latest last.
408 |                 data[domain] = {
409 |                     'pshtt': pshtt,
410 |                     'base_domain': base_domain,
411 |                     'agency': agency,
412 |                     'branch': branch,
413 |                     'cfo_act': cfo_act(agency),
414 |                     'compliance': compliance_for(pshtt)
415 |                 }
416 | 
417 |     return data
418 | 
419 | def fetch_preloaded():
420 |     preload_json = None
421 | 
422 |     preload_path = "cache/preload.json"
423 | 
424 |     if os.path.exists(preload_path):
425 |         logging.debug("Using cached Chrome preload list.")
426 |         preload_json = json.loads(open(preload_path).read())
427 |     else:
428 |         logging.debug("Fetching Chrome preload list from source...")
429 | 
430 |         # Downloads the chromium preloaded domain list and sets it to a global set
431 |         file_url = 'https://chromium.googlesource.com/chromium/src/net/+/master/http/transport_security_state_static.json?format=TEXT'
432 | 
433 |         # TODO: proper try/except around this network request
434 |         request = requests.get(file_url)
435 |         raw = request.content
436 | 
437 |         # To avoid parsing the contents of the file out of the source tree viewer's
438 |         # HTML, we download it as a raw file. googlesource.com Base64-encodes the
439 |         # file to avoid potential content injection issues, so we need to decode it
440 |         # before using it. https://code.google.com/p/gitiles/issues/detail?id=7
441 |         raw = base64.b64decode(raw).decode('utf-8')
442 | 
443 |         # The .json file contains '//' comments, which are not actually valid JSON,
444 |         # and confuse Python's JSON decoder. Begone, foul comments!
445 |         raw = ''.join([re.sub(r'^\s*//.*$', '', line)
446 |                        for line in raw.splitlines()])
447 | 
448 |         preload_json = json.loads(raw)
449 | 
450 |         logging.debug("Caching preload list at %s" % preload_path)
451 |         write(json_for(preload_json), preload_path)
452 | 
453 |     # For our purposes, we only care about entries that includeSubDomains
454 |     fully_preloaded = []
455 |     for entry in preload_json['entries']:
456 |         if entry.get('include_subdomains', False) is True:
457 |             fully_preloaded.append(entry['name'])
458 | 
459 |     return fully_preloaded
460 | 
461 | def fetch_preload_pending():
462 | 
463 |     pending_path = "cache/preload-pending.json"
464 |     if not os.path.exists(pending_path):
465 |         logging.debug("Fetching Chrome pending preload list...")
466 | 
467 |         pending_url = "https://hstspreload.org/api/v2/pending"
468 |         download(pending_url, pending_path)
469 | 
470 |     raw = open(pending_path).read()
471 |     pending_json = json.loads(raw)
472 | 
473 |     # For our purposes, we only care about entries that include subdomains.
474 |     pending = []
475 |     for entry in pending_json:
476 |         if entry.get('include_subdomains', False) is True:
477 |             pending.append(entry['name'])
478 | 
479 |     return pending
480 | 
481 | def write(content, destination):
482 |     mkdir_p(os.path.dirname(destination))
483 | 
484 |     f = open(destination, 'w', encoding='utf-8')
485 |     f.write(content)
486 |     f.close()
487 | 
488 | def json_for(object):
489 |     return json.dumps(object, sort_keys=True,
490 |                       indent=2, default=format_datetime)
491 | 
492 | def format_datetime(obj):
493 |     if isinstance(obj, datetime.date):
494 |         return obj.isoformat()
495 |     elif isinstance(obj, str):
496 |         return obj
497 |     else:
498 |         return None
499 | 
500 | # mkdir -p in python, from:
501 | # https://stackoverflow.com/questions/600268/mkdir-p-functionality-in-python
502 | def mkdir_p(path):
503 |     try:
504 |         os.makedirs(path)
505 |     except OSError as exc:  # Python >2.5
506 |         if exc.errno == errno.EEXIST:
507 |             pass
508 |         else:
509 |             raise
510 | 


--------------------------------------------------------------------------------
/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/GSA/https/8271c90458204ca79220ea266fa88d6ed2c8515c/favicon.ico


--------------------------------------------------------------------------------
/gulpfile.js:
--------------------------------------------------------------------------------
 1 | 'use strict';
 2 | 
 3 | const path = require('path');
 4 | const gulp = require('gulp');
 5 | const sass = require('gulp-sass');
 6 | const sourcemaps = require('gulp-sourcemaps');
 7 | const exec = require('child_process').exec;
 8 | 
 9 | const PORT = process.env.PORT || '8000';
10 | 
11 | const USWDS_DIST = 'node_modules/uswds/dist';
12 | const USWDS_DIST_DIR = path.join(__dirname, ...USWDS_DIST.split('/'));
13 | 
14 | const PROJECT_SASS_SRC = "./assets/sass/**/*.scss";
15 | const PROJECT_CSS_DEST = './assets/css'
16 | 
17 | const PROJECT_ASSETS_SRC = `${USWDS_DIST}/@(js|fonts|img)/**/**`
18 | const PROJECT_ASSETS_DEST = './assets/vendor/uswds'
19 | 
20 | 
21 | gulp.task('copy-uswds-assets', () => {
22 |   return gulp.src(PROJECT_ASSETS_SRC)
23 |     .pipe(gulp.dest(PROJECT_ASSETS_DEST));
24 | });
25 | 
26 | gulp.task('build-sass', () => {
27 |   return gulp.src(PROJECT_SASS_SRC)
28 |     .pipe(sourcemaps.init())
29 |     .pipe(sass({
30 |       includePaths: [
31 |         path.join(USWDS_DIST_DIR, 'scss'),
32 |       ]
33 |     }).on('error', sass.logError))
34 |     .pipe(sourcemaps.write())
35 |     .pipe(gulp.dest(PROJECT_CSS_DEST));
36 | });
37 | 
38 | /** entry points **/
39 | 
40 | // gulp.task('init', gulp.series('copy-uswds-assets'));
41 | 
42 | gulp.task('watch', () => {
43 |   gulp.watch(PROJECT_SASS_SRC, gulp.series('copy-uswds-assets','build-sass'));
44 | });
45 | 
46 | gulp.task('default', gulp.series('copy-uswds-assets', 'build-sass'));
47 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "scripts": {
 3 |     "build": "gulp",
 4 |     "federalist": "npm run build"
 5 |   },
 6 |   "dependencies": {
 7 |     "gulp": "^4.0.2",
 8 |     "gulp-sass": "^4.1.0",
 9 |     "gulp-sourcemaps": "^3.0.0",
10 |     "uswds": "^1.3.0"
11 |   }
12 | }
13 | 


--------------------------------------------------------------------------------
/pages/apis.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | layout: base
 3 | 
 4 | title: Migrating APIs to HTTPS
 5 | permalink: /apis/
 6 | description: "An outline of the process for migrating a web API from HTTP to HTTPS."
 7 | ---
 8 | 
 9 | Examples of federal agency API HTTPS migrations:
10 | 
11 | * NIH posted a **[detailed HTTPS migration plan for their NCBI APIs](https://www.ncbi.nlm.nih.gov/home/develop/https-guidance.shtml)** that involves 301 redirects for GET/HEAD, 403 responses for other methods, helping to patch community client libraries, and a series of staged blackouts.
12 | 
13 | * The [api.data.gov platform](https://api.data.gov), which hosts a number of client APIs that proxy through its service, offered its client APIs a **[variety of HTTPS transition options](https://github.com/18F/api.data.gov/issues/34#issuecomment-87524112)**, and restricted HTTP usage of most APIs right away for new API keys. The api.data.gov platform also implemented certificate issuance/renewal by making an [open source nginx/openresty module in Lua](https://github.com/GUI/lua-resty-auto-ssl) that automates the process.
14 | 
15 | 
16 | ## Introduction
17 | 
18 | All APIs should use and require HTTPS to [help guarantee](/faq/) **confidentiality**, **authenticity**, and **integrity**.
19 | 
20 | HTTPS provides a stronger guarantee that a client is communicating with the real API and receiving back authentic contents. It also enhances privacy for applications and users using the API. For APIs that support cross-origin request sharing (CORS) or JSONP requests, it also ensures the requests are not blocked as [mixed content](/mixed-content/).
21 | 
22 | All new APIs should use and require HTTPS. Rather than issue a redirect when visited over HTTP (redirects within APIs are problematic, as outlined below), the API should likely return an error message (such as the HTTP status code 403 Forbidden).
23 | 
24 | To migrate an existing API that runs over plain HTTP, start by adding HTTPS support.
25 | 
26 | ## Add HTTPS Support
27 | 
28 | Once you've acquired and installed a certificate, you should test the API over HTTPS and make any needed adjustments.
29 | 
30 | Enable [HTTP Strict Transport Security](/hsts/) if possible, but do **not** redirect API traffic from HTTP to HTTPS.
31 | 
32 | You can safely enable HSTS and submit the domain to the HSTS browser preload list. HSTS was primarily [designed](https://tools.ietf.org/html/rfc6797#section-2.1) for web sites and is only supported in web browsers, not clients that would be used for integrating with APIs. As the API will continue to serve content over both HTTP and HTTPS, clients making HTTP requests to the API will continue to work.
33 | 
34 | ## Update Your Documentation
35 | 
36 | Update your documentation to refer only to the HTTPS endpoint. Be sure to update all code examples, client libraries, sample applications, and your own applications (if you are an integrator of your own API). Update any pages, lists, or indexes that reference your API, such as `/data` or [data.gov](https://www.data.gov) (via `data.json`).
37 | 
38 | Mark HTTP as deprecated in your documentation and strongly recommend migrating to the HTTPS endpoint. Publish a dedicated information and resources page to which you can link.
39 | 
40 | ## Measure HTTPS Usage
41 | 
42 | API analytics are key for measuring performance, seeing how others are using your API, and tracking usage over time.
43 | 
44 | You should track usage of the API over HTTP versus HTTPS. If you have a way of identifying individual integrators, such as via an API key, application secret, or other submitted value, you can monitor their usage of HTTP versus HTTPS.
45 | 
46 | ## Announce HTTP Deprecation
47 | 
48 | Announce that you have deprecated the HTTP endpoint of your API. Post this to your developer hub, blog, and/or mailing list. Optionally, include a deadline.
49 | 
50 | If you have a method of contacting specific API integrators, such as an email address when they signed up for an API key, contact them. You may also have a list of known API-driven projects.
51 | 
52 | Search the internet, and GitHub in particular, for references to the domain name or URL of your API, and your API's documentation. Open issues on GitHub repositories about moving to the HTTPS endpoint, or even file a pull request making the change yourself.
53 | 
54 | ## Disable the HTTP Endpoint
55 | 
56 | This entire process could take anywhere from two weeks to six months or longer. You will want to weigh how widely used your API is, how critical it is, and how quickly integrators of your API make adjustments. Questions to ask at this stage:
57 | 
58 | * Does my API work over HTTPS?
59 | * Am I recommending HTTPS for my API?
60 | * Am I collecting useful information from API clients?
61 | 
62 | First, perform a "blackout test" by turning off the HTTP endpoint for a set time period, usually somewhere between a few hours and a few days. Monitor your analytics and see what complaints this sends your way.
63 | 
64 | Refer complainants to your dedicated page in your documentation and encourage them to migrate as soon as possible. It usually takes just a few lines of code or configuration for an API integrator to switch to the HTTPS endpoint of an API, so do not be discouraged by the feedback from eventually disabling the HTTP endpoint.
65 | 
66 | Announce any blackout tests ahead of time. Optional: Pick a deadline for when you will turn off the HTTP API and include this in your announcement.
67 | 
68 | If you had no complaints, try a longer blackout, such as a few days to a week. (This is especially helpful if you have no analytics and are relying on complaints as a metric.)
69 | 
70 | You can conduct multiple blackouts if your analytics show they make a good improvement the number of integrators using HTTPS over HTTP, but be mindful of diminishing returns.
71 | 
72 | Announce a deadline for when you will turn off the HTTP API if you have not yet done so.
73 | 
74 | On the deadline, disable the HTTP endpoint. Make an announcement, referring your previous efforts to ensure a smooth transition and to your documentation, which includes a page on the migration.
75 | 
76 | ## Technical Considerations
77 | 
78 | When turning off an HTTP endpoint, return an error in the same consumable manner with which your API would typically respond. Issue a clear error message, include a hyperlink if possible, and use an appropriate HTTP status code such as 403 Forbidden.
79 | 
80 | **Do not redirect HTTP to HTTPS.** Redirects may break API clients as, unlike browsers, they may not be configured to follow redirects. Even if they followed the redirect, many API clients will change a POST request into a GET request, breaking RESTful APIs.
81 | 
82 | This incorrect behavior is both common and [well-documented](https://tools.ietf.org/html/rfc2616#section-10.3.2). [RFC7231](https://tools.ietf.org/html/rfc7231#section-6.4.7) introduced a new HTTP status code 307 as a temporary redirect that does not allow changing the request method, and [RFC7238](https://tools.ietf.org/html/rfc7238) proposes 308 as a permanent redirect. While most web browsers support 307, and some support 308, these may not be supported by API clients and should not be relied on for migrating an API.
83 | 
84 | This is meant to apply to APIs that are RESTful or equivalent. If your API is actually a public dataset that is downloaded usually through a web browser, a redirect would likely be appropriate. (Note that HSTS could also handle this redirect for you.)
85 | 
86 | **Redirecting CORS Requests:** Your API may support cross-origin resource sharing (CORS) through the use of `Access-Control-Allow-Origin` headers. While we recommend returning an error once an HTTP version of an API is shut down, you can optionally choose to redirect CORS requests, which is widely supported in web browsers. These web browsers will obey HSTS and the HSTS preload list, thus your choices are 1) break non-HTTPS CORS requests or 2) specially handle and redirect CORS requests.
87 | 
88 |  * First determine how widespread CORS requests are, what browsers make them, and ideally, what the referring URLs are (for ease of contacting the integrator).
89 |  * Redirect to the HTTPS version of the API endpoint when the `Origin` HTTP header is present to signify a CORS request. **Note:** CORS headers must be included in the redirect response, as well as the response to which the request is redirected.
90 |  * CORS GET requests will not follow a redirect in Android before version 4.4, all versions of iOS, and all versions of Safari. This also applies to an HSTS "internal" redirect. If mobile browsers are not a major user, then forcing a redirect should go pretty smoothly.
91 | 
92 | **Redirecting JSONP Requests:** Your API may support JSONP requests. Web browsers will follow redirects for JSONP requests, since these are simply `<script>` elements. Enabling HSTS and submitting the domain to the HSTS preload list would be enough for browsers that support the preload list. If you wish to handle all JSONP requests:
93 | 
94 | * First determine how widespread JSONP requests are, what browsers make them, and ideally, what the referring URLs are (for ease of contacting the integrator).
95 | * Redirect to the HTTPS version of the API endpoint when the JSONP callback parameter for your API is present (usually `?jsonp=` or `?callback=`).
96 | * Consider whether to support JSONP going forward, as it is [not secure or performant](https://gist.github.com/tmcw/6244497).
97 | 


--------------------------------------------------------------------------------
/pages/certificates.md:
--------------------------------------------------------------------------------
  1 | ---
  2 | layout: base
  3 | 
  4 | title: Certificates
  5 | permalink: /certificates/
  6 | description: "FAQ on certificates and certificate authorities for agencies migrating to HTTPS."
  7 | ---
  8 | 
  9 | Frequently asked questions and answers about HTTPS certificates and certificate authorities.
 10 | 
 11 | * [What are certificates and certificate authorities?](#what-are-certificates-and-certificate-authorities)
 12 | * [What kind of certificate should I get for my domain?](#what-kind-of-certificate-should-i-get-for-my-domain)
 13 | * [What rules and oversight are certificate authorities subject to?](#what-rules-and-oversight-are-certificate-authorities-subject-to)
 14 | * [Does the US government operate a publicly trusted certificate authority?](#does-the-us-government-operate-a-publicly-trusted-certificate-authority)
 15 | * [Are there federal restrictions on acceptable certificate authorities to use?](#are-there-federal-restrictions-on-acceptable-certificate-authorities-to-use)
 16 | * [Then how can I limit which CAs can issue certificates for a domain?](#then-how-can-i-limit-which-cas-can-issue-certificates-for-a-domain)
 17 | * [How can I find out when any certificate is issued for a domain?](#how-can–i–find-out-when-any-certificate–is-issued-for-a-domain)
 18 | 
 19 | ## What are certificates and certificate authorities?
 20 | 
 21 | Websites use **certificates** to create an HTTPS connection. When signed by a trusted **certificate authority** (CA), certificates give confidence to browsers that they are visiting the "real" website.
 22 | 
 23 | Technically, a certificate is a file that contains:
 24 | 
 25 | * The domain(s) it is authorized to represent.
 26 | * A numeric "public key" that mathematically corresponds to a "private key" held by the website owner.
 27 | * A cryptographic signature by a certificate authority (CA) that vouches for the relationship between the keypair and the authorized domain(s).
 28 | * Other technical information, such as when the certificate expires, what algorithm the CA used to sign it, and how extensively the domain was validated.
 29 | * Optionally, information about a person or organization that owns the domain(s).
 30 | 
 31 | Web browsers are generally set to trust a pre-selected list of certificate authorities (CAs), and the browser can verify that any signature it sees comes from a CA in that list. The list of trusted CAs is set either by the underlying operating system or by the browser itself.
 32 | 
 33 | When a website presents a certificate to a browser during an HTTPS connection, the browser uses the information and signature in the certificate to confirm that a CA it trusts has decided to trust the information in the certificate.
 34 | 
 35 | ## What kind of certificate should I get for my domain?
 36 | 
 37 | There are many kinds of certificates in use in the federal government today, and the right one may depend on a system's technical architecture or an agency's business policies.
 38 | 
 39 | In general:
 40 | 
 41 | * "Domain Validation" (DV) certificates are usually less expensive and more amenable to automation than "Extended Validation" (EV) certificates. **Ordinary DV certificates are completely acceptable for government use.**
 42 | 
 43 | * Certificates can be valid for anywhere from years to days. In general, **shorter-lived certificates offer a better security posture**, since the impact of key compromise is less severe. Automating the issuance and renewal of certificates is an overall best practice, and can make the adoption of shorter-lived certificates more practical.
 44 | 
 45 | * Agencies **[should immediately replace certificates signed with SHA-1](/technical-guidelines/#signature-algorithms)**, as browsers are quickly moving to remove support for the SHA-1 algorithm. Commercial CAs are forbidden from issuing them entirely as of January 1, 2016.
 46 | 
 47 | As a general matter, certificates from any commercial CA will meet the few [NIST technical requirements](https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-52r1.pdf) that relate to certificates.
 48 | 
 49 | ## What rules and oversight are certificate authorities subject to?
 50 | 
 51 | Since 2012, all major browsers and certificate authorities participate in the **[CA/Browser Forum](https://cabforum.org)**. Though self-regulated, the CA/Browser Forum is effectively the governing body for publicly trusted certificate authorities.
 52 | 
 53 | The CA/B Forum produces the **[Baseline Requirements](https://cabforum.org/baseline-requirements-documents/)** (BRs), a set of technical and procedural policies that all CAs must adhere to. These policies are determined through a [formal voting process](https://cabforum.org/ballots/) of browsers and CAs. The BRs are enforced through a combination of technical measures, standard third-party audits, and the overall community's attention to publicly visible certificates.
 54 | 
 55 | The Baseline Requirements only constrain CAs -- they do not constrain browser behavior. Since browser vendors ultimately decide which certificates their browser will trust, they are the enforcers and adjudicators of BR violations. If a CA is found to be in violation of the Baseline Requirements, a browser may penalize or inhibit that CA's ability to issue certificates that that browser will trust, up to and including expulsion from that browser's trust store.
 56 | 
 57 | #### CA / Browser Resources
 58 | 
 59 | * The current [Baseline Requirements](https://cabforum.org/baseline-requirements-documents/)
 60 | * [CA/B Forum voting record](https://cabforum.org/ballots/)
 61 | * [Mozilla revoking an ANSSI intermediate](https://blog.mozilla.org/security/2013/12/09/revoking-trust-in-one-anssi-certificate/) after ANSSI was found to have violated the Baseline Requirements by inappropriately issuing an intermediate certificate for use in network monitoring.
 62 | * [Google requiring Symantec to employ Certificate Transparency](https://security.googleblog.com/2015/10/sustaining-digital-certificate-security.html) after Symantec was found to have violated the Baseline Requirements by misissuing certificates.
 63 | 
 64 | ## Does the US government operate a publicly trusted certificate authority?
 65 | 
 66 | No, not as of early 2016, and this is unlikely to change in the near future.
 67 | 
 68 | The [Federal PKI](https://fpki.idmanagement.gov) root is trusted by some browsers and operating systems, but is not contained in the [Mozilla Trusted Root Program](https://www.mozilla.org/en-US/about/governance/policies/security-group/certs/policy/). The Mozilla Trusted Root Program is used by Firefox, many Android devices, and a variety of other devices and operating systems. This means that the Federal PKI is not able to issue certificates for use in TLS/HTTPS that are trusted widely enough to secure a web service used by the general public.
 69 | 
 70 | The Federal PKI has [cross-certified other commercial CAs](https://fpki.idmanagement.gov/tools/fpkigraph/), which means their certificates will be trusted by clients that trust the Federal PKI. However, even when a publicly trusted commercial CA is cross-certified with the Federal PKI, they are expected to maintain complete separation between their publicly trusted certificates and their Federal PKI cross-certified certificates.
 71 | 
 72 | As a result, there is not currently a viable way to obtain a certificate for use in TLS/HTTPS that is issued or trusted by the Federal PKI, and also trusted by the general public.
 73 | 
 74 | ## Are there federal restrictions on acceptable certificate authorities to use?
 75 | 
 76 | There are no government-wide rules limiting what CAs federal domains can use.
 77 | 
 78 | It is important to understand that, while there may be technical or business reasons for an agency to limit which CAs it uses, **there is no security benefit** to limiting CAs through internal policies alone. Browsers will trust certificates acquired from any publicly trusted CA, and so limiting CA usage internally will not limit the CAs from which an attacker may obtain a forged certificate.
 79 | 
 80 | In practice, federal agencies use a wide variety of publicly trusted commercial CAs and privately trusted enterprise CAs to secure their web services.
 81 | 
 82 | ## Then how can I limit which CAs can issue certificates for a domain?
 83 | 
 84 | There is no simple and 100% effective way to force all browsers to only trust certificates for your domain that have been issued from a certain CA. In general, the strength of HTTPS on today's internet depends on the overall standards, competence, and accountability of the entire CA system.
 85 | 
 86 | However, domain owners can use **DNS Certification Authority Authorization** to publish a list of approved CAs.
 87 | 
 88 | In addition, domain owners can use **Certificate Transparency** (see question below) to monitor and discover certificates issued by any CA.
 89 | 
 90 | **[DNS Certification Authority Authorization](https://en.wikipedia.org/wiki/DNS_Certification_Authority_Authorization)** (CAA) allows domain owners to publish DNS records containing a list of the Certificate Authorities permitted to issue certificates for their domain.
 91 | 
 92 | All major CAs participate in CAA and promise to verify CAA DNS records before issuing certificates. Each CA should refuse to issue certificates for a domain name that publishes a CAA record that excludes the CA.
 93 | 
 94 | This is only a promise, so a non-compliant or compromised CA could still issue certificates for any domain name even in violation of CAA. But such mis-issuance would be more likely to be detected with CAA in place.
 95 | 
 96 | The standard DNS is not secure, so CAA records could be suppressed or spoofed by an attacker in a privileged network position unless DNSSEC is in use by the domain owner and validated by each CA issuer.
 97 | 
 98 | CAA can be paired with Certificate Transparency log monitoring to detect occurrences of mis-issuance.
 99 | 
100 | #### CAA Resources
101 | 
102 | * [Wikipedia entry](https://en.wikipedia.org/wiki/DNS_Certification_Authority_Authorization) for CAA
103 | * [RFC 6844](https://tools.ietf.org/html/rfc6844), the standard for CAA
104 | * [CAA Record Generator](https://sslmate.com/caa/)
105 | 
106 | ## How can I find out when any certificate is issued for a domain?
107 | 
108 | Domain owners can use **Certificate Transparency** to promptly discover any certificates issued for a domain, whether legitimate or fraudulent.
109 | 
110 | **[Certificate Transparency](https://en.wikipedia.org/wiki/Certificate_Transparency)** (CT) allows domain owners to **detect mis-issuance of certificates after the fact**.
111 | 
112 | CT allows CAs to publish some or all of the publicly trusted certificates that they issue to one or more public logs. Multiple organizations run CT logs, and it is possible to automatically monitor the logs for any certificates that are issued for any domains of interest.
113 | 
114 | Comodo has released an [open source](https://github.com/crtsh) Certificate Transparency log viewer that they operate at [crt.sh](https://crt.sh). For example, it is possible to see [all recent certificates for whitehouse.gov](https://crt.sh/?q=whitehouse.gov), and [details of specific certificates](https://crt.sh/?id=7976268).
115 | 
116 | The strength of Certificate Transparency increases as more CAs publish more certificates to public CT logs. [Google Chrome requires Certificate Transparency](https://groups.google.com/a/chromium.org/forum/#!msg/ct-policy/wHILiYf31DE/iMFmpMEkAQAJ) for all new certificates issued after 30 April 2018. [Apple platforms, including Safari, require Certificate Transparency](https://support.apple.com/en-us/HT205280) for all new certificates issued after 15 October 2018. As a result, most CAs now submit new certificates to CT logs by default.
117 | 
118 | However, a CA may still issue new certificates without disclosing them to a CT log. These certificates will not be trusted by Chrome or Safari, but they may be trusted by other browsers.
119 | 
120 | Chrome also exempts private CAs from these transparency rules, so private CAs that do not chain up to any public root may still issue certificates without submitting them to CT logs.
121 | 
122 | #### Certificate Transparency Resources
123 | 
124 | * [Google CT FAQ](https://www.certificate-transparency.org/faq)
125 | * [RFC 6962](https://tools.ietf.org/html/rfc6962), the experimental standard for CT
126 | * [Wikipedia entry](https://en.wikipedia.org/wiki/Certificate_Transparency) for CT
127 | * [Cert Spotter](https://github.com/SSLMate/certspotter), an open source CT log monitor
128 | * [U.S. Federal PKI page on Chrome CT enforcement](https://fpki.idmanagement.gov/announcements/chromect/)
129 | 


--------------------------------------------------------------------------------
/pages/everything.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | layout: base
 3 | 
 4 | title: Why HTTPS for Everything?
 5 | permalink: /everything/
 6 | description: "Today, there is no such thing as non-sensitive web traffic. By simply always using HTTPS, both people and browsers can make safer assumptions about secure connections."
 7 | ---
 8 | 
 9 | HTTP has become central to today's way of life. HTTP is currently the primary protocol for applications used on computers, tablets, smartphones, and many other devices.
10 | 
11 | As our dependency on the internet has grown, the risk to users' privacy and safety has grown along with it.
12 | 
13 | Every unencrypted HTTP request reveals information about a user's behavior, and the interception and tracking of unencrypted browsing has become commonplace.
14 | 
15 | Today, **there is no such thing as non-sensitive web traffic**, and public services should not depend on the benevolence of network operators.
16 | 
17 | When properly configured, HTTPS can provide a fast, secure connection that offers the level of privacy and reliability that users should expect from government web services.
18 | 
19 | ## Privacy and integrity by default
20 | 
21 | By using private connections by default, changed expectations make everyone safer.
22 | 
23 | By always using HTTPS, web services don't have to make a subjective judgment call about what's "sensitive". This leaves less room for error, and makes deployment simpler and more consistent.
24 | 
25 | Widespread use of HTTPS also means that clients can begin assuming HTTPS with more confidence. Attacks designed to track large quantities of unencrypted traffic become less attractive.
26 | 
27 | Web browsers can begin displaying HTTPS connections as normal, and HTTP connections as non-secure. HTTPS validation failures can become more strict, reducing the effectiveness of phishing and user error.
28 | 
29 | These changed expectations improve the security of HTTPS on every website. In other words, **protecting less sensitive sites strengthens the protections of more sensitive sites**.
30 | 
31 | ## HTTPS is the internet's next phase
32 | 
33 | The internet's standards bodies, web browsers, major tech companies, and the internet community of practice have all come to understand that HTTPS should be the baseline for all web traffic.
34 | 
35 | * The W3C's Technical Architecture Group has found that [the web should actively prefer secure connections](https://www.w3.org/2001/tag/doc/web-https) and transition entirely to HTTPS.
36 | * The IETF has said that [pervasive monitoring is an attack](https://datatracker.ietf.org/doc/rfc7258/), and the Internet Architecture Board (the IETF's parent organization) recommends that [new protocols use encryption by default](https://www.iab.org/2014/11/14/iab-statement-on-internet-confidentiality/).
37 | * The Chrome and [Firefox](https://blog.mozilla.org/security/2015/04/30/deprecating-non-secure-http/) security teams are working on [gradually marking plain HTTP as non-secure](https://www.chromium.org/Home/chromium-security/marking-http-as-non-secure).
38 | 
39 | Ultimately, the goal of the internet community is to establish encryption as the norm, and to phase out unencrypted connections.
40 | 
41 | Investing in HTTPS makes it faster, cheaper, and easier for everyone. Many of the advancements of the last several years have come from major institutions and technology companies committing to migrate websites and services, improving the status quo, and contributing their improvements back to the public.
42 | 
43 | The more US government websites and services that join the transition to an encrypted internet, the smoother and faster it will be.
44 | 


--------------------------------------------------------------------------------
/pages/faq.md:
--------------------------------------------------------------------------------
  1 | ---
  2 | layout: base
  3 | 
  4 | title: Introduction to HTTPS
  5 | permalink: /faq/
  6 | description: "An introduction to HTTPS, and frequently asked questions."
  7 | ---
  8 | 
  9 | Below are some frequently asked questions and answers about HTTPS.
 10 | 
 11 | For an in-depth introduction (no technical background required), check out the DigitalGov University presentation, **["An Introduction to HTTPS"](https://www.youtube.com/watch?v=d2GmcPYWm5k)**, to learn what HTTPS is and how it protects web services and users.
 12 | 
 13 | * [What does HTTPS do?](#what-does-https-do)
 14 | * [What information does HTTPS protect?](#what-information-does-https-protect)
 15 | * [What information does HTTPS _not_ protect?](#what-information-does-https-not-protect)
 16 | * [How does HTTPS relate to HTTP/2?](#how-does-https-relate-to-http-2)
 17 | * [How does migrating to HTTPS affect search engine optimization (SEO)?](#how-does-migrating-to-https-affect-search-engine-optimization-seo)
 18 | * [How can an HTTPS site keep sending referrer information to linked HTTP sites?](#how-can-an-https-site-keep-sending-referrer-information-to-linked-http-sites)
 19 | * [How difficult is it to attack an HTTPS connection?](#how-difficult-is-it-to-attack-an-https-connection)
 20 | * [Why are domain names unencrypted over HTTPS today?](#why-are-domain-names-unencrypted-over-https-today)
 21 | * [Why isn't DNSSEC good enough?](#why-isnt-dnssec-good-enough)
 22 | * [How does HTTPS protect against DNS spoofing?](#how-does-https-protect-against-dns-spoofing)
 23 | 
 24 | ### What does HTTPS do?
 25 | 
 26 | When properly configured, an HTTPS connection guarantees three things:
 27 | 
 28 | * **Confidentiality.** The visitor's connection is encrypted, obscuring URLs, cookies, and other sensitive metadata.
 29 | * **Authenticity.** The visitor is talking to the "real" website, and not to an impersonator or through a person-in-the-middle.
 30 | * **Integrity.** The data sent between the visitor and the website has not been tampered with or modified.
 31 | 
 32 | A plain HTTP connection can be easily monitored, modified, and impersonated.
 33 | 
 34 | ### What information does HTTPS protect?
 35 | 
 36 | HTTPS encrypts nearly all information sent between a client and a web service.
 37 | 
 38 | For example, an **unencrypted HTTP** request reveals not just the body of the request, but the full URL, query string, and various HTTP headers about the client and request:
 39 | 
 40 | <img src="/assets/images/with-http-headers.png" title="What you see with HTTP" style="border: 1px solid black" />
 41 | 
 42 | An **encrypted HTTPS** request protects most things:
 43 | 
 44 | <img src="/assets/images/with-https-headers.png" title="What you see with HTTPS" style="border: 1px solid black" />
 45 | 
 46 | This is the same for all HTTP methods (GET, POST, PUT, etc.). The URL path and query string parameters are encrypted, as are POST bodies.
 47 | 
 48 | ### What information does HTTPS _not_ protect?
 49 | 
 50 | While HTTPS encrypts the entire HTTP request and response, the DNS resolution and connection setup can reveal other information, such as the full domain or subdomain and the originating IP address, as shown above.
 51 | 
 52 | Additionally, attackers can still analyze encrypted HTTPS traffic for "side channel" information. This can include the time spent on site, or the relative size of user input.
 53 | 
 54 | ### How does HTTPS relate to HTTP/2?
 55 | 
 56 | HTTP/2 (finalized in [2015](https://tools.ietf.org/html/rfc7540)) is a backwards-compatible update to HTTP/1.1 (finalized in [1999](https://tools.ietf.org/html/rfc2616)) that is optimized for the modern web.
 57 | 
 58 | HTTP/2 includes many features that can drastically speed up website performance, and emerged from the advancements Google demonstrated with [SPDY](https://blog.chromium.org/2009/11/2x-faster-web.html) in 2009.
 59 | 
 60 | While HTTP/2 does not require the use of encryption in its formal spec, every major browser that has implemented HTTP/2 has only implemented support for encrypted connections, and no major browser is working on support for HTTP/2 over unencrypted connections.
 61 | 
 62 | This means that in practice, **the major performance benefits of HTTP/2 first require the use of HTTPS.**
 63 | 
 64 | For more information:
 65 | 
 66 | * [HTTP/2 Working Group FAQ](https://http2.github.io/faq/)
 67 | * [RFC 7540](https://tools.ietf.org/html/rfc7540), the final spec
 68 | * [HTTP/2 Implementation Status](https://www.mnot.net/blog/2015/06/15/http2_implementation_status), by Mark Nottingham (working group chair)
 69 | 
 70 | ### How does migrating to HTTPS affect search engine optimization (SEO)?
 71 | 
 72 | In general, migrating to HTTPS improves a website's own SEO and analytics.
 73 | 
 74 | * As of August 2014, Google [uses HTTPS as a ranking signal](https://security.googleblog.com/2014/08/https-as-ranking-signal_6.html), which can improve search rankings.
 75 | * Migrating to HTTPS will improve analytics about web traffic referred from HTTPS websites, as referrer information [is not passed from HTTPS websites to HTTP websites](https://stackoverflow.com/questions/1361705/is-http-header-referer-sent-when-going-to-a-http-page-from-a-https-page/1361720#1361720).
 76 | 
 77 | To make the migration as smooth as possible, and avoid taking a SEO hit:
 78 | 
 79 | * Use a **proper 301 redirect** to redirect users from `http://` to `https://`. **Do not use a 302 redirect**, as this may negatively impact search rankings.
 80 | * Use the [canonical link element](https://en.wikipedia.org/wiki/Canonical_link_element) (`<link rel="canonical">`) to inform search engines that the "canonical" URL for a website uses `https://`.
 81 | 
 82 | ### How can an HTTPS site keep sending referrer information to linked HTTP sites?
 83 | 
 84 | By default, when a user is on an HTTPS website and clicks a link to an HTTP website, browsers will not send a `Referer` header to the HTTP website. This is [defined in the HTTP 1.1 specification](https://tools.ietf.org/html/rfc2616#section-15.1.3), and is designed to avoid exposing HTTPS URLs that would otherwise have remained protected by the guarantees of HTTPS.
 85 | 
 86 | However, this means that if a website migrates to HTTPS, any HTTP sites it links to will stop seeing referrer data from the HTTPS website. This can be a disincentive to migrate to HTTPS, as it deprives linked HTTP sites of analytics data, and means the HTTPS website won't get "credit" for referring traffic to linked websites.
 87 | 
 88 | Website owners who wish to continue sending outbound referrer information to linked HTTP sites can use **[Referrer Policy](https://www.w3.org/TR/referrer-policy/)** to override browser default behavior, while retaining the privacy of HTTPS URLs.
 89 | 
 90 | To do this, websites **should use** the [`origin-when-cross-origin`](https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin) policy. This will allow supporting browsers to send **only the origin** as the `Referer` header. This limited referral information applies even if both sites use HTTPS.
 91 | 
 92 | For example, if a user is on `https://agency.gov/help/aids.html` and clicks a link to `https://moreinformation.com`, then if `origin-when-cross-origin` is set, the browser will make an HTTP request to `https://moreinformation.com` with a `Referer` header of `https://agency.gov`.
 93 | 
 94 | The simplest way to set this policy is by including a `<meta>` tag in the body of the HTTPS website:
 95 | 
 96 | ```html
 97 | <meta name="referrer" value="origin-when-cross-origin" />
 98 | ```
 99 | 
100 | The `Referrer-Policy` HTTP header may also be used as an [alternate delivery mechanism](https://www.w3.org/TR/referrer-policy/#referrer-policy-delivery), but this is not widely supported in web browsers (as of late 2016).
101 | 
102 | Websites **should not use** the `unsafe-url` policy, as this will cause HTTPS URLs to be exposed on the wire over an HTTP connection, which defeats one of the important privacy and security guarantees of HTTPS.
103 | 
104 | ### How difficult is it to attack an HTTPS connection?
105 | 
106 | Attacks on HTTPS connections generally fall into 3 categories:
107 | 
108 | * Compromising the quality of the HTTPS connection, through cryptanalysis or other protocol weaknesses.
109 | * Compromising the client computer, such as by installing a malicious root certificate into the system or browser trust store.
110 | * Obtaining a "rogue" certificate trusted by major browsers, generally by manipulating or compromising a certificate authority.
111 | 
112 | These are all possible, but for most attackers they are very difficult and require significant expense. Importantly, they are all _targeted_ attacks, and are not feasible to execute against any user connecting to any website.
113 | 
114 | By contrast, plain HTTP connections can be easily intercepted and modified by anyone involved in the network connection, and so attacks can be carried out at large scale and at low cost.
115 | 
116 | ## Why are domain names unencrypted over HTTPS today?
117 | 
118 | This is primarily to support **[Server Name Indication](/sni/)** (SNI), a TLS extension that allows multiple hostnames to be served over HTTPS from one IP address.
119 | 
120 | The SNI extension was introduced in 2003 to allow HTTPS deployment to scale more easily and cheaply, but it does mean that the hostname is sent by browsers to servers "in the clear" so that the receiving IP address knows which certificate to present to the client.
121 | 
122 | When a domain or a subdomain itself reveals sensitive information (e.g. 'contraception.foo.gov' or 'suicide-help.foo.gov'), this can reveal that information to passive eavesdroppers.
123 | 
124 | From a network privacy perspective, DNS also "leaks" hostnames in the clear across the network today (even when DNSSEC is used). There are ongoing efforts in the network standards community to encrypt both the SNI hostname and DNS lookups, but as of late 2015, nothing has been deployed to support these goals.
125 | 
126 | Most clients support SNI today, and site owners are encouraged to [evaluate the feasibility of requiring SNI support](/sni/), to save money and resources. However, whether SNI support is required to access a specific website or not, a website's owner should consider their hostnames to be unencrypted over HTTPS, and account for this when provisioning domains and subdomains.
127 | 
128 | ### Why isn't DNSSEC good enough?
129 | 
130 | [DNSSEC](https://en.wikipedia.org/wiki/Domain_Name_System_Security_Extensions) attempts to guarantee that domain names are resolved to correct IP addresses.
131 | 
132 | However, DNS resolution is just one aspect of securely communicating on the internet. DNSSEC does not fully secure a domain:
133 | 
134 | * Once DNS resolution is complete, DNSSEC does not ensure the confidentiality or integrity of communication between a client and the destination IP.
135 | 
136 | * No major web browsers inform the user when DNSSEC validation fails, limiting its strength and enforceability.
137 | 
138 | HTTPS guarantees the confidentiality and integrity of communication between client and server, and web browsers have rigorous and evolving HTTPS enforcement policies.
139 | 
140 | ### How does HTTPS protect against DNS spoofing?
141 | 
142 | In practice, HTTPS can protect communication with a domain even in the absence of DNSSEC support.
143 | 
144 | A valid HTTPS certificate shows that the server has demonstrated ownership over the domain to a trusted certificate authority at the time of certificate issuance.
145 | 
146 | To ensure that an attacker cannot use DNS spoofing to direct the user to a plain `http://` connection where traffic can be intercepted, websites can use [HTTP Strict Transport Security](/hsts/) (HSTS) to instruct browsers to require an HTTPS connection for their domain at all times.
147 | 
148 | This means that an attacker that successfully spoofs DNS resolution must also create a valid HTTPS connection. This makes DNS spoofing as challenging and expensive as [attacking HTTPS generally](#how-difficult-is-it-to-attack-an-https-connection?).
149 | 
150 | If the attacker spoofs DNS but doesn't compromise HTTPS, users will receive a notable warning message from their browser that will prevent them from visiting the possibly malicious site. If the site uses HSTS, there will be no option for the visitor to disregard and click through the warning.
151 | 
152 | HTTPS and HSTS work together to protect a domain against DNS spoofing.
153 | 


--------------------------------------------------------------------------------
/pages/guide.md:
--------------------------------------------------------------------------------
  1 | ---
  2 | layout: base
  3 | 
  4 | title: Compliance Guide
  5 | permalink: /guide/
  6 | description: "Guide for agencies on implementing the HTTPS transition."
  7 | ---
  8 | 
  9 | [M-15-13](https://obamawhitehouse.archives.gov/sites/default/files/omb/memoranda/2015/m-15-13.pdf) calls for "all publicly accessible Federal websites and web services" to only provide service through a secure connection (HTTPS), and to use [HTTP Strict Transport Security](/hsts/) (HSTS) to ensure this.
 10 | 
 11 | This applies to all public domains and subdomains operated by the federal government, regardless of the domain suffix, as long as they are reachable over HTTP/HTTPS on the public internet.
 12 | 
 13 | This page provides implementation guidance for agencies by the White House Office of Management and Budget, as agencies manage their transition to HTTPS.
 14 | 
 15 | * [Compliance and best practice checklist](#compliance-and-best-practice-checklist)
 16 | * [Options for HSTS compliance](#options-for-hsts-compliance)
 17 | * [Compliance FAQ](#compliance-faq)
 18 |   * [What protocols are covered by M-15-13?](#what-protocols-are-covered-by-m-15-13)
 19 |   * [Do I need to shut off port 80?](#do-i-need-to-shut-off-port-80)
 20 |   * [If plain HTTP is entirely disabled on my server, do I still need HSTS?](#if-plain-http-is-entirely-disabled-on-my-server-do-i-still-need-hsts)
 21 |   * [What about network services that don't actually serve web content?](#what-about-network-services-that-dont-actually-serve-web-content)
 22 |   * [What does "all Federal agency domains or subdomains" include?](#what-does-all-federal-agency-domains-or-subdomains-include)
 23 |   * [What about domains that are only used to redirect visitors to other websites?](#what-about-domains-that-are-only-used-to-redirect-visitors-to-other-websites)
 24 |   * [Do domains that redirect to other external domains need to redirect internally to HTTPS before redirecting externally?](#do-domains-that-redirect-to-other-external-domains-need-to-redirect-internally-to-https-before-redirecting-externally)
 25 |   * [What about domains that are technically public, but in practice are only used internally?](#what-about-domains-that-are-technically-public-but-in-practice-are-only-used-internally)
 26 |   * [What happens to visitors using browsers that don&rsquo;t support HSTS, like older versions of Internet Explorer?](#what-happens-to-visitors-using-browsers-that-dont-support-hsts-like-older-versions-of-internet-explorer)
 27 |   * [This site redirects users to HTTPS -- why is Pulse saying it doesn't enforce HTTPS?](#this-site-redirects-users-to-https-why-is-pulse-saying-it-doesnt-enforce-https)
 28 |   * [Are federally operated certificate revocation services (CRL, OCSP) also required to move to HTTPS?](#are-federally-operated-certificate-revocation-services-crl-ocsp-also-required-to-move-to-https)
 29 |   * [What if I'm using a federally issued certificate -- such as from the Federal PKI or Department of Defense -- for my web service?](#what-if-im-using-a-federally-issued-certificate-such-as-from-the-federal-pki-or-department-of-defense-for-my-web-service)
 30 | 
 31 | ## Compliance and best practice checklist
 32 | 
 33 | Each public website or web service an agency operates **must**:
 34 | 
 35 | * Provide service over HTTPS.
 36 | * Automatically redirect HTTP requests to HTTPS, or disable HTTP entirely.
 37 | * Have an HSTS policy in place, through either of the [two approaches described below](#options-for-hsts-compliance).
 38 | 
 39 | Each public website or web service an agency operates **should**:
 40 | 
 41 | * Follow [technical best practices](/technical-guidelines/) around TLS quality, as [demonstrated by https.cio.gov](https://www.ssllabs.com/ssltest/analyze.html?d=https.cio.gov) and measured by [pulse.cio.gov](https://pulse.cio.gov/).
 42 | * Address any [mixed content issues](/mixed-content/) that arise from the migration process.
 43 | * Evaluate the viability of dropping support for legacy clients and using modern standards like [Server Name Indication](/sni/).
 44 | 
 45 | ## Options for HSTS compliance
 46 | 
 47 | There are a great number of federal websites and web services. To simplify the process of transitioning the federal government to HTTPS, agencies are encouraged to take advantage of **[HSTS preloading](/hsts/#hsts-preloading)**.
 48 | 
 49 | Preloading marks entire domains as HTTPS-only, and allows browsers to enforce this rigorously and automatically for every subdomain. Many `.gov` domains [have already implemented HSTS preloading](https://18f.gsa.gov/2015/02/09/the-first-gov-domains-hardcoded-into-your-browser-as-all-https/), as have a large number of private sector web services.
 50 | 
 51 | Agencies should generally take one of two approaches, on a per-domain basis, to ensure that an HSTS policy is set for all public websites.
 52 | 
 53 | Under either approach, web services used by non-browser clients (e.g. APIs) must still individually enforce HTTPS, as HSTS is not supported by non-browser clients.
 54 | 
 55 | **1. Full HSTS preloading of the parent domain (preferred)**
 56 | 
 57 | * The parent domain (e.g. `https://agency.gov`) has an HSTS policy that includes subdomains and has a max-age of at least 1 year, like this one:
 58 | 
 59 | ```
 60 | Strict-Transport-Security: max-age=31536000; includeSubDomains; preload
 61 | ```
 62 | 
 63 | * The domain has been successfully [submitted and added to the HSTS preload list](https://hstspreload.org).
 64 | * Individual website subdomains are still encouraged to set their own HSTS policies.
 65 | 
 66 | HSTS preloading a parent domain allows agencies to avoid inventorying and configuring an HSTS policy for every individual subdomain. However, this approach also automatically includes **all** subdomains present on this domain -- including intranet subdomains. All subdomains will need to support HTTPS in order to remain reachable for use in major browsers.
 67 | 
 68 | **2. Compliance for each individual subdomain**
 69 | 
 70 | * The parent domain and each of its publicly reachable subdomains must set an HSTS policy with a max-age of at least 1 year, like this one:
 71 | 
 72 | ```
 73 | Strict-Transport-Security: max-age=31536000
 74 | ```
 75 | 
 76 | This approach allows agencies the flexibility to focus only on publicly accessible subdomains, but could entail significantly more work to add an HSTS policy header to each individual website.
 77 | 
 78 | ## Compliance FAQ
 79 | 
 80 | Answers to other common compliance questions appear below.
 81 | 
 82 | ### What protocols are covered by M-15-13?
 83 | 
 84 | M-15-13 requires secure connections for **websites and web services**, which means **only HTTP-based protocols**. This includes all federal websites, as well as federally operated HTTP-based APIs.
 85 | 
 86 | M-15-13 does not address the use of DNS or DNSSEC, FTP or SFTP, or any other non-HTTP network protocol.
 87 | 
 88 | ### Do I need to shut off port 80?
 89 | 
 90 | **No.** [M-15-13 states](https://https.cio.gov/#footnote-3):
 91 | 
 92 | > Allowing HTTP connections for the sole purpose of redirecting clients to HTTPS connections is acceptable and encouraged.
 93 | 
 94 | Agencies may employ port 80 for the sole purpose of redirecting clients to a secure connection.
 95 | 
 96 | HTTP redirects must use a response code in the 300's that can reliably cause HTTP clients to perform redirects to an HTTPS URI, such as 301 or 302.
 97 | 
 98 | The use of error codes in the 400's or 500's **will not** satisfy this requirement.
 99 | 
100 | Note that while connections to port 80 are insecure, even for redirects, the use of [HSTS](/hsts/) will instruct supporting HTTP clients to automatically redirect themselves from port 80 to port 443, without attempting to connect to port 80 over the network.
101 | 
102 | HSTS mitigates the security impact of connections over port 80, while allowing agencies the flexibility to continue redirecting legacy clients or clients which have not yet received an HSTS policy for the target domain.
103 | 
104 | ### If plain HTTP is entirely disabled on my server, do I still need HSTS?
105 | 
106 | **Yes.** Disabling HTTP support is not sufficient to prevent attacks that downgrade web browsers to plain HTTP.
107 | 
108 | During an attack, it doesn't matter whether the "real" server has disabled HTTP. If the client can be coaxed into initiating a plain HTTP connection -- such as when a user clicks an `http://` link, or types a URL into their browser manually -- then a local attacker can respond to that connection attempt from their own server and establish their own connection.
109 | 
110 | HSTS specifically instructs web browsers to never initiate plain HTTP connections. If a user clicks an `http://` link or types in an `http://` URL, HSTS causes the browser to first rewrite the URL to use `https://` before initiating the connection.
111 | 
112 | For this reason, HSTS is necessary to effectively prevent downgrade attacks, even if plain HTTP connections are unsupported on the server.
113 | 
114 | ### What about network services that don't actually serve web content?
115 | 
116 | M-15-13 covers any publicly accessible network service that responds to HTTPS or HTTP requests. This includes network services that don't serve content, but only return HTTP headers, or blank or insubstantial content.
117 | 
118 | This also includes services that respond to HTTPS or HTTP requests on non-standard ports (ports other than 80 or 443), whether or not those services are included in external scans provided to agencies.
119 | 
120 | Network services which do not respond to HTTPS or HTTP requests are not included in the scope of M-15-13.
121 | 
122 | ### What does "all Federal agency domains or subdomains" include?
123 | 
124 | Domains and subdomains, in the context of M-15-13, refer to hostnames that are publicly accessible via HTTP or HTTPS.
125 | 
126 | **Domain** refers to hostnames that are directly registerable. Some examples include `gsa.gov`, `whitehouse.gov`, `dodig.mil`, or `fs.fed.us`.
127 | 
128 | **Subdomain** refers to any hostname that is a child of a registerable domain, and may be of any length. Some examples include `www.gsa.gov`, `planthardiness.ars.usda.gov`, `www.fia.fs.fed.us`, or `www.usar.army.mil`.
129 | 
130 | Federally operated domains do not all end in `.gov`, `.mil`, or `.fed.us`. Some may end in `.com`, `.org`, `.us`, or other suffixes. Any federally operated domain is covered by M-15-13.
131 | 
132 | ### What about domains that are only used to redirect visitors to other websites?
133 | 
134 | These domains must enable port 443 and use properly configured HTTPS.
135 | 
136 | They must follow all the same requirements and guidelines as domains used to host websites and APIs, including HSTS and preloading.
137 | 
138 | ### Do domains that redirect to other external domains need to redirect internally to HTTPS before redirecting externally?
139 | 
140 | Not generally, but it is practically required in order to preload a second-level domain.
141 | 
142 | For example, it is not required by M-15-13 to redirect from `http://example.gov:80` to `https://example.gov:443` before redirecting to `https://another-example.gov:443`. However, doing so enables the connecting client to see and cache the HSTS header on `example.gov`, which it may not otherwise see.
143 | 
144 | However, doing an internal redirect first **is required** [to automatically preload second-level domains](https://hstspreload.org/#submission-requirements), and so this practice is recommended for second-level domains.
145 | 
146 | ### What about domains that are technically public, but in practice are only used internally?
147 | 
148 | M-15-13 includes all domains and subdomains that are publicly reachable over HTTP/HTTPS, regardless of agency operational practices.
149 | 
150 | ### What happens to visitors using browsers that don't support HSTS, like older versions of Internet Explorer?
151 | 
152 | Browsers that don't support HSTS are simply unaffected by HSTS, so there is no harm in enabling it.
153 | 
154 | ### This site redirects users to HTTPS -- why is Pulse saying it doesn't enforce HTTPS?
155 | 
156 | [Pulse](https://pulse.cio.gov) looks for server-side redirects, using an appropriate HTTP response code. Sites that use client-side redirects -- such as a &lt;meta refresh&gt; tag or JavaScript -- will not be seen as redirects. To meet the M-15-13 requirement of enforcing HTTPS, agencies should employ server-side redirects (or alternatively, disable HTTP access altogether).
157 | 
158 | Sites that are reachable on both a root domain (`http://agency.gov`) and their www subdomain (`http://www.agency.gov`) should perform a redirect to HTTPS in both cases. Redirecting one but not the other could also cause Pulse to indicate that a domain does not enforce HTTPS.
159 | 
160 | ### Are federally operated certificate revocation services (CRL, OCSP) also required to move to HTTPS?
161 | 
162 | No. This very narrow class of services, that provide CRL and OCSP information for the purposes of verifying the revocation status of certificates used to make other HTTPS connections, should abide by best practices in the field and their respective specifications.
163 | 
164 | For CRL, [RFC 5280](https://tools.ietf.org/html/rfc5280) says:
165 | 
166 | ```
167 | CAs SHOULD NOT include URIs that specify https, ldaps, or similar schemes in extensions.  CAs that include an https URI in one of these extensions MUST ensure that the server's certificate can be validated without using the information that is pointed to by the URI.  Relying parties that choose to validate the server's certificate when obtaining information pointed to by an https URI in the cRLDistributionPoints, authorityInfoAccess, or subjectInfoAccess extensions MUST be prepared for the possibility that this will result in unbounded recursion.
168 | ```
169 | 
170 | For OCSP, [RFC 6960](https://tools.ietf.org/html/rfc6960) says:
171 | 
172 | ```
173 | Where privacy is a requirement, OCSP transactions exchanged using HTTP MAY be protected using either Transport Layer Security/Secure Socket Layer (TLS/SSL) or some other lower-layer protocol.
174 | ```
175 | 
176 | Agencies are encouraged to operate OCSP and CRL services via hostnames specifically reserved for those services, so that other related information and functionality can be served securely and privately.
177 | 
178 | 
179 | ### What if I'm using a federally issued certificate -- such as from the Federal PKI or Department of Defense -- for my web service?
180 | 
181 | There are [no restrictions on acceptable certificate authorities](/certificates/#are-there-federal-restrictions-on-acceptable-certificate-authorities-to-use) agencies might use to meet the requirements of M-15-13.
182 | 
183 | However, M-15-13 requires agencies to do more than just redirect HTTP traffic to HTTPS. It also requires agencies to enable **[HTTP Strict Transport Security](/hsts/)** (HSTS), as [described above](#options-for-hsts-compliance). HSTS ensures that HTTPS is always used, and protects users from several common vulnerabilities.
184 | 
185 | One important effect of HSTS is that it **disables the ability for users to click through certificate warnings** in supporting browsers. This means that **agencies cannot instruct users to click through certificate warnings** to use their web service while also complying with M-15-13.
186 | 
187 | This is also consistent with security best practices, as instructing users to click through certificate warnings defeats the point of HTTPS, and will subject users to potential network attacks.
188 | 
189 | In practice, to deploy HSTS while using federally issued certificates, an agency will likely need to separate its web services by hostname, based on their expected audience:
190 | 
191 | * Federally issued certificates may be practical for web services whose users can be consistently expected to trust the issuing federal certificate authority (CA). Users whose devices do not trust the issuing CA will experience a connection failure and be unable to use the web service.
192 | * Federally issued certificates will not be practical for web services whose users may not always be expected to trust the issuing federal certificate authority. These web services will likely require the use of a certificate from a publicly trusted (commercial) CA.
193 | 
194 | Whatever strategy an agency employs to manage the use of federally issued certificates, it should allow the practical deployment of [HSTS](/hsts/) across all of its publicly accessible websites and web services.
195 | 


--------------------------------------------------------------------------------
/pages/hsts.md:
--------------------------------------------------------------------------------
  1 | ---
  2 | layout: base
  3 | 
  4 | title: HTTP Strict Transport Security
  5 | permalink: /hsts/
  6 | description: "An overview of HTTP Strict Transport Security (HSTS), a lightweight standard that prevents privacy leaks and downgrade attacks."
  7 | ---
  8 | 
  9 | **[HTTP Strict Transport Security](https://developer.mozilla.org/en-US/docs/Web/Security/HTTP_strict_transport_security)** (HSTS) is a simple and [widely supported](https://caniuse.com/#feat=stricttransportsecurity) standard to protect visitors by ensuring that their browsers _always_ connect to a website over HTTPS. HSTS exists to remove the need for the common, insecure practice of redirecting users from `http://` to `https://` URLs.
 10 | 
 11 | When a browser knows that a domain has enabled HSTS, it does two things:
 12 | 
 13 | * Always uses an `https://` connection, even when clicking on an `http://` link or after typing a domain into the location bar without specifying a protocol.
 14 | * Removes the ability for users to click through warnings about invalid certificates.
 15 | 
 16 | A domain instructs browsers that it has enabled HSTS by returning an HTTP header over an HTTPS connection.
 17 | 
 18 | In its simplest form, the policy tells a browser to enable HSTS for that exact domain or subdomain, and to remember it for a given number of seconds:
 19 | 
 20 | ```
 21 | Strict-Transport-Security: max-age=31536000;
 22 | ```
 23 | 
 24 | In its **strongest and recommended form**, the HSTS policy **includes all subdomains**, and indicates a willingness to be ["preloaded"](#hsts-preloading) into browsers:
 25 | 
 26 | ```
 27 | Strict-Transport-Security: max-age=31536000; includeSubDomains; preload
 28 | ```
 29 | 
 30 | When using this form, bear in mind:
 31 | 
 32 | * The policy should be deployed at `https://domain.gov`, _not_ `https://www.domain.gov`.
 33 | * **All subdomains** associated with the parent domain must support HTTPS. (They do not have to each have their own HSTS policy.)
 34 | 
 35 | See below for examples of **[how to set an HSTS policy in common web servers](#configuration-for-common-web-servers)**.
 36 | 
 37 | ## Background
 38 | 
 39 | Strict Transport Security was [proposed in 2009](https://lists.w3.org/Archives/Public/www-archive/2009Sep/att-0051/draft-hodges-strict-transport-sec-05.plain.html), motivated by [Moxie Marlinspike's demonstration](https://moxie.org/software/sslstrip/) of how a hostile network could downgrade visitor connections and exploit insecure redirects. It was quickly adopted by several major web browsers, and [finalized as RFC 6797 in 2012](https://tools.ietf.org/html/rfc6797).
 40 | 
 41 | The basic problem that HSTS solves is that even after a website turns on HTTPS, visitors may still end up trying to connect over plain HTTP. For example:
 42 | 
 43 | * When a user types "gsa.gov" into the URL bar, browsers default to using `http://`.
 44 | * A user may click on an old link that mistakenly uses an `http://` URL.
 45 | * A user's network may be hostile and actively rewrite `https://` links to `http://`.
 46 | 
 47 | Websites that prefer HTTPS will generally still listen for connections over HTTP in order to redirect the user to the HTTPS URL. For example:
 48 | 
 49 | ```
 50 | $ curl --head http://github.com
 51 | 
 52 | HTTP/1.1 301 Moved Permanently
 53 | Location: https://github.com/
 54 | ```
 55 | 
 56 | **This redirect is insecure** and is an opportunity for an attacker to capture information about the visitor (such as cookies from a previous secure session), or to maliciously redirect the user to a phishing site.
 57 | 
 58 | This can be addressed by returning a `Strict-Transport-Security` header whenever the user connects securely. For example:
 59 | 
 60 | ```
 61 | $ curl --head https://github.com
 62 | 
 63 | HTTP/1.1 200 OK
 64 | Strict-Transport-Security: max-age=31536000; includeSubdomains; preload
 65 | ```
 66 | 
 67 | This enables HSTS for `github.com`. While HSTS is in effect, clicking any links to `http://github.com` will cause the browser to issue a request directly for `https://github.com`.
 68 | 
 69 | In the above example, the browser will remember the HSTS policy for 1 year. The policy is refreshed every time browser sees the header again, so if a user visits `https://github.com` at least once every year, they'll be indefinitely protected by HSTS.
 70 | 
 71 | ## HSTS Preloading
 72 | 
 73 | For a user to take advantage of HSTS, their browser does have to see the HSTS header at least once. This means that users are not protected until after their first successful secure connection to a given domain.
 74 | 
 75 | In addition, in many cases, **there may never be a first visit** to `https://domain.gov`. For example:
 76 | 
 77 | * Many federal websites redirect directly from `http://domain.gov` to `https://www.domain.gov`.
 78 | * Many federal domains that are used solely for redirects will redirect from `http://domain.gov` directly to `https://destination.gov`.
 79 | 
 80 | In either case, `https://domain.gov` is never visited, meaning connecting clients will never see an HSTS policy with an `includeSubDomains` directive that applies to the whole zone.
 81 | 
 82 | To solve this problem, the Chrome security team created an **"HSTS preload list"**: a [list of domains](https://chromium.googlesource.com/chromium/src/+/master/net/http/transport_security_state_static.json) baked into Chrome that get Strict Transport Security enabled automatically, even for the first visit.
 83 | 
 84 | Firefox, Safari, Opera, and Edge also incorporate Chrome's HSTS preload list, making this feature shared across major browsers.
 85 | 
 86 | ## How to preload a domain
 87 | 
 88 | The Chrome security team allows anyone to [submit their domain to the list](https://hstspreload.org), provided it meets the following requirements:
 89 | 
 90 | * HTTPS is enabled on the root domain (e.g. `https://donotcall.gov`), and **all subdomains** (e.g. `https://www.donotcall.gov`) -- especially the `www` subdomain, if a DNS record for it exists. This necessarily includes any subdomains in use solely on intranets.
 91 | * The HSTS policy includes all subdomains, with a long `max-age`, and a `preload` flag to indicate that the domain owner consents to preloading.
 92 | * The website redirects from HTTP to HTTPS, at least on the root domain.
 93 | 
 94 | An example of a valid HSTS header for preloading:
 95 | 
 96 | ```
 97 | Strict-Transport-Security: max-age=31536000; includeSubDomains; preload
 98 | ```
 99 | 
100 | In the long term, as the [web transitions fully to HTTPS](https://www.w3.org/2001/tag/doc/web-https) and browsers can start phasing out plain HTTP and defaulting to HTTPS, the HSTS preload list (and HSTS itself) may eventually become unnecessary.
101 | 
102 | Until that time, the HSTS preload list is a simple, effective mechanism for locking down HTTPS for an entire domain.
103 | 
104 | ## HSTS as a forcing function
105 | 
106 | Strict Transport Security provides meaningful security benefits to visitors, especially visitors on hostile networks.
107 | 
108 | However, it's also highly valuable as an organizational forcing function and compliance mechanism.
109 | 
110 | When a domain owner follows the recommendations in this article and sets an HSTS policy on its base domain with `includeSubDomains` and `preload`, the domain owner is saying _"Every part of our web infrastructure is HTTPS, and always will be."_ &mdash; and is giving browsers permission to vigorously enforce that from then onwards.
111 | 
112 | It's a clear and auditable commitment, and gives anyone overseeing an organization's transition to HTTPS a way of marking domains as "done".
113 | 
114 | Zooming out even further: it's technically possible to preload HSTS for an entire top-level domain (e.g. ".gov"), as [Google first did with `.google`](https://chromium.googlesource.com/chromium/src/+/ea9dfef649a309a05c3b5c112150485836fbfcc7%5E%21/net/http/transport_security_state_static.json). As a relatively small, centrally managed top-level domain, perhaps someday `.gov` can get there.
115 | 
116 | ## Configuration for common web servers
117 | 
118 | On **nginx**, you would apply an `add_header` command to the appropriate virtual host configuration. This website, `https.cio.gov`, is hosted on nginx, and uses [this batch of HTTPS rules](https://github.com/fisma-ready/nginx/blob/master/ssl/ssl.rules) to set this header:
119 | 
120 | ```
121 | add_header Strict-Transport-Security 'max-age=31536000; includeSubDomains; preload;' always;
122 | ```
123 | 
124 | On **Apache**, you would apply a `Header` directive to always set the HSTS header, like so:
125 | 
126 | ```
127 | Header always set Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
128 | ```
129 | 
130 | On **Microsoft systems running IIS** (Internet Information Services), there are no ".htaccess" files to implement custom headers. IIS applications use a central `web.config` file for configuration.
131 | 
132 | For IIS 7.0 and up, the example `web.config` file configuration below will handle secure HTTP to HTTPS redirection with HSTS enabled for HTTPS:
133 | 
134 | ```
135 | <?xml version="1.0" encoding="UTF-8"?>
136 | <configuration>
137 |     <system.webServer>
138 |         <rewrite>
139 |             <rules>
140 |                 <rule name="HTTP to HTTPS redirect" stopProcessing="true">
141 |                     <match url="(.*)" />
142 |                     <conditions>
143 |                         <add input="{HTTPS}" pattern="off" ignoreCase="true" />
144 |                     </conditions>
145 |                     <action type="Redirect" url="https://{HTTP_HOST}/{R:1}"
146 |                         redirectType="Permanent" />
147 |                 </rule>
148 |             </rules>
149 |             <outboundRules>
150 |                 <rule name="Add Strict-Transport-Security when HTTPS" enabled="true">
151 |                     <match serverVariable="RESPONSE_Strict_Transport_Security"
152 |                         pattern=".*" />
153 |                     <conditions>
154 |                         <add input="{HTTPS}" pattern="on" ignoreCase="true" />
155 |                     </conditions>
156 |                     <action type="Rewrite" value="max-age=31536000; includeSubDomains; preload" />
157 |                 </rule>
158 |             </outboundRules>
159 |         </rewrite>
160 |     </system.webServer>
161 | </configuration>
162 | ```
163 | 
164 | Generally, you want to set a custom HTTP header for `Strict-Transport-Security` with the value `max-age=31536000; includeSubDomains; preload` (or some variant).
165 | 
166 | Here are some links to do that with other web servers:
167 | 
168 | * [Caddy](https://caddyserver.com/docs/header)
169 | * [Haproxy](https://www.haproxy.com/doc/aloha/7.0/haproxy/http_rewriting.html#set-a-header-in-the-response)
170 | * [Lighttpd](https://redmine.lighttpd.net/projects/1/wiki/Docs_ModSetEnv)
171 | 
172 | ## Resources
173 | 
174 | * [Browser support for HSTS](https://caniuse.com/#feat=stricttransportsecurity)
175 | * [HSTS web developer documentation](https://developer.mozilla.org/en-US/docs/Web/Security/HTTP_strict_transport_security) maintained by the Mozilla community
176 | * Chrome's [HSTS preload list](https://chromium.googlesource.com/chromium/src/+/master/net/http/transport_security_state_static.json), and their [submission form](https://hstspreload.org).
177 | * ["Upgrading HTTPS in Mid-Air"](https://www.cs.princeton.edu/research/techreps/TR-986-15) - A paper analyzing the current detailed practice of HSTS and [HTTP Public Key Pinning](https://developer.mozilla.org/en-US/docs/Web/Security/Public_Key_Pinning), as of November 2014.
178 | * ["The first .gov domains hardcoded into your browser as all-HTTPS"](https://18f.gsa.gov/2015/02/09/the-first-gov-domains-hardcoded-into-your-browser-as-all-https/), by 18F.
179 | 


--------------------------------------------------------------------------------
/pages/index.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | layout: base
 3 | 
 4 | title: The HTTPS-Only Standard
 5 | permalink: /
 6 | description: "The American people expect government websites to be secure and their interactions with those websites to be private. All browsing activity should be considered private and sensitive. This proposed initiative would require the use of HTTPS on all publicly accessible Federal websites and web services."
 7 | ---
 8 | 
 9 | The American people expect government websites to be secure and their interactions with those websites to be private.
10 | 
11 | This site contains a web-friendly version of the White House Office of Management and Budget memorandum [M-15-13](https://www.whitehouse.gov/sites/whitehouse.gov/files/omb/memoranda/2015/m-15-13.pdf), **"A Policy to Require Secure Connections across Federal Websites and Web Services"**, and provides technical guidance and best practices to assist in its implementation.
12 | 
13 | ## Goal
14 | 
15 | This Memorandum requires that all publicly accessible Federal websites and <a class="footnote" name="footnote-source-1"></a>web services [[1]](#footnote-1) only provide service through a secure connection.  The strongest privacy and integrity protection currently available for public web connections is [Hypertext Transfer Protocol Secure (HTTPS)](/faq/).
16 | 
17 | This Memorandum expands upon the material in prior Office of Management and Budget (OMB) guidance found in [M-05-04](https://www.whitehouse.gov/sites/whitehouse.gov/files/omb/memoranda/2005/m05-04.pdf "Policies for Federal Agency Public Websites") and relates to material in [M-08-23](https://www.whitehouse.gov/sites/whitehouse.gov/files/omb/memoranda/2008/m08-23.pdf "Securing the Federal Government’s Domain Name System Infrastructure."). It provides guidance to agencies for making the transition to HTTPS and a deadline by which agencies must be in compliance.
18 | 
19 | ## Background
20 | 
21 | The unencrypted HTTP protocol does not protect data from interception or alteration, which can subject users to eavesdropping, tracking, and the modification of received data. The majority of Federal websites use HTTP as the primary protocol to communicate over the public internet. Unencrypted HTTP connections create a privacy vulnerability and expose potentially sensitive information about users of unencrypted Federal websites and services. Data sent over HTTP is susceptible to interception, manipulation, and impersonation. This data can include browser identity, website content, search terms, and other user-submitted information.
22 | 
23 | To address these concerns, many commercial organizations have adopted HTTPS or implemented HTTPS-only policies to protect visitors to their websites and services. Users of Federal websites and services deserve the same protection. Private and secure connections are becoming the Internet's baseline, as expressed by the policies of the Internet's [standards](https://www.w3.org/2001/tag/doc/web-https) [bodies](https://www.iab.org/2014/11/14/iab-statement-on-internet-confidentiality/), popular web browsers, and the Internet community of practice.  The Federal government must adapt to this changing landscape, and benefits by beginning the conversion now. Proactive investment at the Federal level will support faster internet-wide adoption and promote better privacy standards for the entire browsing public.
24 | 
25 | **All browsing activity should be considered private and sensitive.**
26 | 
27 | An HTTPS-Only standard will eliminate inconsistent, subjective determinations across agencies regarding which content or browsing activity is sensitive in nature, and create a stronger privacy standard government-wide.
28 | 
29 | Federal websites that do not convert to HTTPS will not keep pace with privacy and security practices used by commercial organizations, and with current and upcoming Internet standards. This leaves Americans vulnerable to known threats, and may reduce their confidence in their government. Although some Federal websites currently use HTTPS, there has not been a consistent policy in this area. An HTTPS-only mandate will provide the public with a consistent, private browsing experience and position the Federal Government as a leader in Internet security.
30 | 
31 | ## What HTTPS Does
32 | 
33 | HTTPS verifies the identity of a website or web service for a connecting client, and encrypts nearly all information sent between the website or service and the user. Protected information includes cookies, user agent details, URL paths, form submissions, and query string parameters. HTTPS is designed to prevent this information from being read or changed while in transit.
34 | 
35 | HTTPS is a combination of HTTP and Transport Layer Security (TLS).  TLS is a network protocol that establishes an encrypted connection to an authenticated peer over an untrusted network.
36 | 
37 | Browsers and other HTTPS clients are configured to trust a set of <a name="footnote-source-2"></a>certificate authorities [[2]](#footnote-2) that can issue cryptographically signed certificates on behalf of web service owners.  These certificates communicate to the client that the web service host demonstrated ownership of the domain to the certificate authority at the time of certificate issuance.  This prevents unknown or untrusted websites from masquerading as a Federal website or service.
38 | 
39 | ## What HTTPS Doesn't Do
40 | 
41 | HTTPS has several important limitations. IP addresses and destination domain names are not encrypted during communication. Even encrypted traffic can reveal some information indirectly, such as time spent on site, or the size of requested resources or submitted information.
42 | 
43 | HTTPS only guarantees the integrity of the connection between two systems, not the systems themselves. It is not designed to protect a web server from being hacked or compromised, or to prevent the web service from exposing user information during its normal operation. Similarly, if a user’s system is compromised by an attacker, that system can be altered so that its future HTTPS connections are under the attacker’s control. The guarantees of HTTPS may also be weakened or eliminated by compromised or malicious certificate authorities.
44 | 
45 | ## Challenges and Considerations
46 | 
47 | **Site Performance:** While encryption adds some computational overhead, modern software and hardware can handle this overhead without substantial deleterious impact on server performance or [latency](https://istlsfastyet.com). Websites with content delivery networks or server software that support the SPDY or HTTP/2 protocols, which require HTTPS in some major browsers, may find their site performance substantially improved as a result of migrating to HTTPS.
48 | 
49 | **[Server Name Indication](/sni/):** The Server Name Indication extension to TLS allows for more efficient use of IP addresses when serving multiple domains. However, these technologies are not supported by some legacy clients. Web service owners should evaluate the feasibility of using this technology to improve performance and efficiency.
50 | 
51 | **[Mixed Content](/mixed-content/):** Websites served over HTTPS need to ensure that all external resources (images, scripts, fonts, iframes, etc.) are also loaded over a secure connection. Modern browsers will refuse to load many insecure resources referenced from within a secure website. When migrating existing websites, this can involve a combination of automated and manual effort to update, replace, or remove references to insecure resources. For some websites, this can be the most time consuming aspect of the migration process.
52 | 
53 | **APIs and Services:** Web services that serve primarily non-browser clients, such as web APIs, may require a [more gradual and hands-on migration strategy](/apis/), as not all clients can be expected to be configured for HTTPS connections or to successfully follow redirects.
54 | 
55 | **Planning for Change:** Protocols and web standards improve regularly, and security vulnerabilities can emerge that require prompt attention. Federal websites and services should deploy HTTPS in a manner that allows for rapid updates to certificates, cipher choices (including [forward secrecy](/technical-guidelines/#forward-secrecy)) protocol versions, and other configuration elements. Agencies should monitor [https.cio.gov](https://https.cio.gov) and other [public resources](/resources/) to keep apprised of current best practices.
56 | 
57 | **[Strict Transport Security](/hsts/):** Websites and services available over HTTPS must enable HTTP Strict Transport Security (HSTS) to instruct compliant browsers to assume HTTPS going forward. This reduces insecure redirects, and protects users against attacks that attempt to downgrade connections to plain HTTP. Once HSTS is in place, domains can be submitted to a ["preload list"](https://hstspreload.org) used by all major browsers to ensure the HSTS policy is in effect at all times.
58 | 
59 | **Domain Name System Security (DNSSEC):** The new policy outlined in this Memorandum does not rescind or conflict with M-08-23, ["Securing the Federal Government's Domain Name System Infrastructure"](https://www.whitehouse.gov/sites/whitehouse.gov/files/omb/memoranda/2008/m08-23.pdf). Once DNS resolution is complete, DNSSEC does not ensure the privacy or integrity of communication between a client and the destination IP. HTTPS provides this additional security.
60 | 
61 | ## Cost Effective Implementation
62 | 
63 | Implementing an HTTPS-only standard does not come without a cost. A significant number of Federal websites have already deployed HTTPS. The goal of this policy is to increase that adoption.
64 | 
65 | The administrative and financial burden of universal HTTPS adoption on all Federal websites includes development time, the financial cost of procuring a certificate and the administrative burden of maintenance over time. The development burden will vary substantially based on the size and technical infrastructure of a site. The compliance timeline, outlined in this Memorandum, provides sufficient flexibility for project planning and resource alignment.
66 | 
67 | OMB affirms that tangible benefits to the American public outweigh the cost to the taxpayer. Even a small number of unofficial or malicious websites claiming to be Federal services, or a small amount of eavesdropping on communication with official U.S. government sites could result in substantial losses to citizens.
68 | 
69 | Technical assistance provided at **[https.cio.gov](https://https.cio.gov)** will aid in the cost-effective implementation of this policy.
70 | 
71 | ## Guidelines
72 | 
73 | In order to promote the efficient and effective deployment of HTTPS, the timeframe for compliance, outlined below, is both reasonable and practical.
74 | 
75 | This Memorandum requires that Federal agencies deploy HTTPS on their domains using the following guidelines.
76 | 
77 | * Newly developed websites and services at [all Federal agency domains or subdomains](/guide/#compliance-faq) must adhere to this policy upon launch.
78 | * For existing websites and services, agencies should prioritize deployment using a risk-based analysis. Web services that involve an exchange of personally identifiable information (PII), where the content is unambiguously sensitive in nature, or where the content receives a high-level of traffic should receive priority and migrate as soon as possible.
79 | * Agencies must make all existing websites and services <a name="footnote-source-3"></a>accessible through a secure connection [[3]](#footnote-3) (HTTPS-only, with HSTS) by December 31, 2016.
80 | * The use of HTTPS is encouraged on <a name="footnote-source-4"></a>intranets [[4]](#footnote-4), but not explicitly required.
81 | 
82 | To monitor agency compliance, a public dashboard has been established at [pulse.cio.gov](https://pulse.cio.gov).
83 | 
84 | ## Footnotes
85 | 
86 | <a name="footnote-1" href="#footnote-source-1">⬑ 1.</a> Publicly-accessible websites and services are defined here as online resources and services available over HTTP or HTTPS over the public internet that are maintained in whole or in part by the Federal Government and operated by an agency, contractor, or other organization on behalf of the agency.  They present government information or provide services to the public or a specific user group and support the performance of an agency’s mission. This definition includes all web interactions, whether a visitor is logged-in or anonymous.
87 | 
88 | <a name="footnote-2" href="#footnote-source-2">⬑ 2.</a> In the context of HTTPS on the web, a certificate authority is a third party organization or company trusted by browsers and operating systems to issue digital certificates on behalf of domain owners.
89 | 
90 | <a name="footnote-3" href="#footnote-source-3">⬑ 3.</a> Allowing HTTP connections for the sole purpose of redirecting clients to HTTPS connections is acceptable and encouraged. HSTS headers must specify a max-age of at least 1 year.
91 | 
92 | <a name="footnote-4" href="#footnote-source-4">⬑ 4.</a> "Intranet" is defined here as a computer network that is not directly reachable over the public internet.
93 | 


--------------------------------------------------------------------------------
/pages/mixed-content.md:
--------------------------------------------------------------------------------
  1 | ---
  2 | layout: base
  3 | 
  4 | title: Mixed Content
  5 | permalink: /mixed-content/
  6 | description: "Strategies for dealing with mixed content when upgrading a website from HTTP to HTTPS."
  7 | ---
  8 | 
  9 | When an HTTPS website references insecure (HTTP) resources, this is called **[mixed content](https://w3c.github.io/webappsec-mixed-content/)**.
 10 | 
 11 | Browsers prevent an HTTPS website from loading most insecure resources, like fonts, scripts, etc. Migrating an existing website from HTTP to HTTPS means identifying and fixing or replacing mixed content.
 12 | 
 13 | [Mixed content](https://w3c.github.io/webappsec-mixed-content/) comes in two varieties:
 14 | 
 15 | **Active** mixed content includes resources that can greatly change the behavior of a website, such as JavaScript, CSS, fonts, and iframes. Browsers refuse to load active mixed content, which often results in affected pages being completely unstyled or broken. Browsers treat these very aggressively because of the consequences if they were compromised. For example, a single compromised Javascript file compromises the entire website, regardless of how other resources are loaded.
 16 | 
 17 | **Passive** mixed content includes resources whose impact on the page's overall behavior is more minimal, such as images, audio, and video. Browsers will load passive mixed content, but will typically change the HTTPS indicator.
 18 | 
 19 | In Chrome, a website indicator for passive mixed content looks like this:
 20 | 
 21 | ![fedramp in chrome](/assets/images/mixed-content.png)
 22 | 
 23 | ## Migration strategy
 24 | 
 25 | Every website's mixed content situation will be different, but the general approach is:
 26 | 
 27 | * Enable `https://` for your website, but don't force a redirect. Continue to present the `http://` version as the canonical URL to search engines.
 28 | * Identify the most obvious and widespread pieces of mixed content by loading your website in a browser over `https://` and observing breakages. Chrome, Opera, and Firefox will log any mixed content warnings to the console, which should point out necessary site-wide changes. Use these to [secure your resource links](#linking-to-resources-securely).
 29 | * After fixing them, tackle the long tail by [scanning your code](#scanning-your-code) and [crawling your website](#crawling-your-website).
 30 | * Finally, force the redirect to HTTPS, [turn on HSTS](/hsts/), and tell search engines that your new URL starts with `https://`.
 31 | 
 32 | Note: the below instructions use tools **optimized for an OS X or Linux environment**. Documentation for Windows-based tools would be a welcome contribution to this guide.
 33 | 
 34 | ## Linking to resources securely
 35 | 
 36 | Most commonly used third party services, such as Google Analytics or AddThis, **will automatically adapt** when migrating to HTTPS.
 37 | 
 38 | Other services may require manual updates, but have an `https://` version ready:
 39 | 
 40 | ```html
 41 | <link href="https://fonts.googleapis.com/css?family=Open+Sans" rel="stylesheet">
 42 | ```
 43 | 
 44 | Generally speaking, for content on your own domain, stick to site-relative URLs wherever possible:
 45 | 
 46 | ```html
 47 | <img src="/media/my-picture.png" />
 48 | ```
 49 | 
 50 | When migrating a site with a lot of user- or staff-submitted content (e.g. a blog), you may find media hotlinked from a third-party domain which doesn't support HTTPS.
 51 | 
 52 | This is a great opportunity to improve your website's privacy and lessen your dependency on third parties, by copying those media files to your own server instead and hosting them yourself.
 53 | 
 54 | ## Scanning your code
 55 | 
 56 | After identifying and fixing the obvious issues, you can scan your website's files for leads. On a Mac or Linux-based system, `grep` is very handy:
 57 | 
 58 | Images and scripts:
 59 | 
 60 |     grep -r "src=\"http:" *
 61 | 
 62 | Stylesheets and fonts:
 63 | 
 64 |     grep -r "href=\"http:" * | grep "<link"
 65 | 
 66 | CSS imports and references:
 67 | 
 68 |     grep -r "url(\"http:"
 69 | 
 70 | Finding links in JavaScript is more challenging, but you can look for all `http:` references and try to exclude hyperlinks in HTML or Markdown:
 71 | 
 72 |     grep -r "http:" | grep -v "href=\"http:"
 73 |     grep -r "http:" | grep -v "](http:"
 74 | 
 75 | 
 76 | ## Crawling your website
 77 | 
 78 | [`mixed-content-scan`](https://github.com/bramus/mixed-content-scan) is a very handy command line tool that can crawl an `http://` or `https://` website to see if it contains any references to insecure resources. This is especially helpful if your content is primarily managed in a CMS.
 79 | 
 80 | `mixed-content-scan` requires PHP, then [installing Composer](https://getcomposer.org/doc/00-intro.md). [Install with Composer](https://github.com/bramus/mixed-content-scan#installation), then use it with your domain:
 81 | 
 82 | ```bash
 83 | mixed-content-scan https://https.cio.gov
 84 | ```
 85 | 
 86 | You should see something like this:
 87 | 
 88 | ```
 89 | [2015-03-15 16:56:48] MCS.NOTICE: Scanning https://https.cio.gov/ [] []
 90 | [2015-03-15 16:56:49] MCS.INFO: 00000 - https://https.cio.gov/ [] []
 91 | [2015-03-15 16:56:49] MCS.INFO: 00001 - https://https.cio.gov/faq/ [] []
 92 | [2015-03-15 16:56:49] MCS.INFO: 00002 - https://https.cio.gov/hsts/ [] []
 93 | [2015-03-15 16:56:49] MCS.INFO: 00003 - https://https.cio.gov/resources/ [] []
 94 | [2015-03-15 16:56:49] MCS.NOTICE: Scanned 4 pages for Mixed Content [] []
 95 | ```
 96 | 
 97 | Any discovered mixed content will be listed as a `WARNING`. You can also get the results as newline-separated JSON objects:
 98 | 
 99 | ```bash
100 | mixed-content-scan https://https.cio.gov --format=json
101 | ```
102 | 
103 | ## Automatically detecting mixed content
104 | 
105 | Site owners also have the option of using a **[Content Security Policy]()** header that will instruct browsers to ping a given URL with information about any observed mixed content warnings.
106 | 
107 | A CSP policy that only specified pinging for mixed content warnings might look like:
108 | 
109 | ```
110 | Content-Security-Policy-Report-Only: default-src https:; report-uri https://example.com/reporting/endpoint
111 | ```
112 | 
113 | CSP reporting, especially for larger services, is an advanced approach that will likely require planning and tuning by a dedicated development team.
114 | 
115 | Some resources:
116 | 
117 | * An engineer [describes Twitter's approach](https://oreoshake.github.io/csp/twitter/2014/07/25/twitters-csp-report-collector-design.html) as of 2014.
118 | * [`report-uri.io`](https://report-uri.io), which offers a hosted reporting URI for CSP violations (or public key pinning violations) and accompanying dashboard.
119 | * A newly developed CSP extension, [Upgrade Insecure Requests](https://w3c.github.io/webappsec/specs/upgrade/), will instruct browsers to automatically upgrade referenced HTTP URLs to HTTPS URLs without triggering mixed content detection. This extension is not finalized, and as of June 2015 is only available in Chrome.
120 | 
121 | ## Why do browsers block mixed content?
122 | 
123 | If mixed content were not blocked, an attacker could control the main website by conducting an attack against any of its active resources.
124 | 
125 | Even with passive content like images, attackers can manipulate what the page looks like, and so the yellow-lock icon is intended to communicate that security has been weakened and user confidence should be reduced. In addition, an attacker will be able to read any cookies for that domain which do have the `Secure` flag, and set cookies.
126 | 
127 | When a website is accessible over `http://`, loading other insecure resources does not generate any sort of warning, and so websites operating over plain HTTP often accumulate many of these sub-resources.
128 | 
129 | ## Security considerations for third party content
130 | 
131 | Incorporating or loading content from third party domains creates an additional attack vector.
132 | 
133 | Even if a page has all page elements loaded over HTTPS, variations in HTTPS configurations could result in security vulnerabilities. For example, if 'foo.gov' loads a page element over HTTPS from 'bar.com' but 'bar.com' is not as fastidious with its HTTPS/TLS configuration, the page element from 'bar.com' may allow injection of malicious software into the page.
134 | 
135 | For example, if 'bar.com' uses a TLS configuration that is known to be weak, a malicious network adversary may be able to modify or replace the page element to inject software that could read the page contents or, potentially, exploit browser vulnerabilities and accomplish more global access to the client device. Accordingly, just as it's important to regularly evaluate the HTTPS/TLS configuration of US government websites, it will be important to also evaluate the configurations of the domains that serve third-party page elements.
136 | 
137 | Note that this is still a strict improvement over incorporating content third party domains over unencrypted HTTP. Attacks on the privacy, integrity, and security of connections to third party domains over unencrypted HTTP are trivial.
138 | 


--------------------------------------------------------------------------------
/pages/resources.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | layout: base
 3 | 
 4 | title: Resources
 5 | permalink: /resources/
 6 | description: "Articles, news, and tools about HTTPS."
 7 | ---
 8 | 
 9 | ### Learning
10 | 
11 | The DigitalGov University has several presentations on HTTPS from the General Services Administration:
12 | 
13 | * **[An Introduction to HTTPS for Beginners](https://www.youtube.com/watch?v=d2GmcPYWm5k)** (June 2015), by Eric Mill and Gray Brooks. This introduction runs a little over an hour, and covers how HTTP and the web work, what HTTPS does to help, and [why we should use it for everything](/everything/).
14 | 
15 | * **[Implementing HTTPS](https://www.youtube.com/watch?v=rnM2qAfEG-M)** (July 2015), by Eric Mill and Gray Brooks. A more detailed explanation of how HTTPS works, how to migrate a website to HTTPS, the [technical concepts](/technical-guidelines/) you should be aware of when implementing HTTPS, and new and upcoming advances in HTTPS.
16 | 
17 | * **[Migrating to HTTPS](https://www.youtube.com/watch?v=X5H8JRULDOo)** (July 2016), by Eric Mill and Timothy Badaczewski. This presentation covers common issues common to federal HTTPS migrations, including: [HTTP Strict Transport Security](/hsts/) (HSTS), [getting certificates](/certificates/), [mixed content](/mixed-content/), and [search engine optimization](/faq/#how-does-migrating-to-https-affect-search-engine-optimization-seo) (SEO).
18 | 
19 | ### Tools
20 | 
21 | * [`crt.sh`](https://crt.sh) - An [open source](https://github.com/crtsh) public viewer for [Certificate Transparency](/certificates/#certificate-transparency) logs. For example, you can view [all publicly logged whitehouse.gov certificates](https://crt.sh/?q=whitehouse.gov).
22 | * [`certspotter`](https://github.com/SSLMate/certspotter) - An open source tool for monitoring issuance of certificates that appear in [Certificate Transparency](/certificates/#certificate-transparency) logs.
23 | * [`certlint`](https://github.com/awslabs/certlint) - An open source tool that reviews x.509 certificates for compliance with CA/Browser Forum requirements and various RFCs.
24 | * [`ssllabs-scan`](https://github.com/ssllabs/ssllabs-scan) - Command line tool for the API for [SSL Labs](https://www.ssllabs.com/ssltest/), a universally referenced HTTPS evaluation and grading tool for public-facing websites.
25 | * [`site-inspector`](https://github.com/benbalter/site-inspector) - Scan a domain for various web/HTTP-related properties, including HTTPS support.
26 | * [`mixed-content-scan`](https://github.com/bramus/mixed-content-scan) - Command line tool for walking over a website and scanning for the use of insecure resources.
27 | 
28 | ### HTTPS in .gov
29 | 
30 | * DigitalGov: [Secure Central Hosting for the Digital Analytics Program](https://www.digitalgov.gov/2015/08/14/secure-central-hosting-for-the-digital-analytics-program/)
31 | * FTC: [FTC.gov is now HTTPS by default](https://www.ftc.gov/news-events/blogs/techftc/2015/03/ftcgov-now-https-default)
32 | * Privacy and Civil Liberties Oversight Board: [PCLOB.gov is now HTTPS by default](https://www.pclob.gov/newsroom/20150318.html)
33 | * CIA: [Statement on CIA Website Enhancement](https://www.cia.gov/news-information/press-releases-statements/press-release-archive-2006/statement-on-cia-website-enhancement.html) from 2006
34 | * 18F: [Why we use HTTPS for every .gov we make](https://18f.gsa.gov/2014/11/13/why-we-use-https-in-every-gov-website-we-make/)
35 | * 18F: [The first .gov domains hardcoded into your browser as all-HTTPS](https://18f.gsa.gov/2015/02/09/the-first-gov-domains-hardcoded-into-your-browser-as-all-https/)
36 | 


--------------------------------------------------------------------------------
/pages/sni.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | layout: base
 3 | 
 4 | title: Server Name Indication
 5 | permalink: /sni/
 6 | description: "An overview of Server Name Indication (SNI), a TLS extension to allow multiple secure hostnames to be served from a single IP address."
 7 | ---
 8 | 
 9 | **[Server Name Indication](https://en.wikipedia.org/wiki/Server_Name_Indication)**, often abbreviated **SNI**, is an [extension to TLS](https://tools.ietf.org/html/rfc6066#page-6) that allows multiple hostnames to be served over HTTPS from the same IP address.
10 | 
11 | A website owner can **require SNI support**, either by allowing their host to do this for them, or by directly consolidating multiple hostnames onto a smaller number of IP addresses. Requiring SNI has the potential to [save significant money](#making-https-cheaper) and resources.
12 | 
13 | However, a few [legacy clients](#client-support) (notably, Internet Explorer on Windows XP) do not support SNI, and will be cut off if SNI is required.
14 | 
15 | Website owners are encouraged to **evaluate whether requiring SNI is feasible**:
16 | 
17 | * For websites accessed primarily by browsers, look at usage by **Internet Explorer on Windows XP**, and **Android 2.3** (and below). If these usage numbers are low, requiring SNI is likely feasible.
18 | 
19 | * For web services accessed by non-browser clients (e.g. APIs), look at usage by Python 2.7.8 and Java 1.6 (and below), and [any other relevant clients](https://en.wikipedia.org/wiki/Server_Name_Indication#Client_side). APIs with heterogeneous clients may wish to do more sophisticated client detection, or staged rollouts.
20 | 
21 | See [`analytics.usa.gov`](https://analytics.usa.gov) for an example of a .gov website which requires support for SNI.
22 | 
23 | ## Making HTTPS cheaper
24 | 
25 | Without SNI, a given IP address is only capable of reliably hosting a single hostname over `https://`. Since [IPv4 addresses are running out](https://en.wikipedia.org/wiki/IPv4_address_exhaustion), IP addresses are expensive to reserve for single domains.
26 | 
27 | For many web hosts and content delivery networks, requiring that clients support SNI allows secure websites to be more efficiently hosted and can greatly reduce costs.
28 | 
29 | ## Client support
30 | 
31 | The most commonly used clients without support for SNI are:
32 | 
33 | * Internet Explorer 8 (and below) on Windows XP
34 | * Some versions of Python 2 (fixed in 2.7.9)
35 | * The default browser in Android 2.3 and earlier
36 | 
37 | Additionally, some enterprise networks may not yet be configured for SNI support. To evaluate support on a network, try visiting [`analytics.usa.gov`](https://analytics.usa.gov) or [`mnot.net`](https://www.mnot.net/blog/2014/05/09/if_you_can_read_this_youre_sniing).
38 | 
39 | ## Resources
40 | 
41 | * See ["If you can read this, you're SNIing"](https://www.mnot.net/blog/2014/05/09/if_you_can_read_this_youre_sniing) for an example of a site that will hard-fail when accessed by a client that does not support SNI.
42 | * [Wikipedia page for SNI](https://en.wikipedia.org/wiki/Server_Name_Indication)
43 | 


--------------------------------------------------------------------------------
/pages/technical.md:
--------------------------------------------------------------------------------
  1 | ---
  2 | layout: base
  3 | 
  4 | title: Technical Guidelines
  5 | permalink: /technical-guidelines/
  6 | description: "Technical guidelines relating to HTTPS strength and quality."
  7 | ---
  8 | 
  9 | This page covers some relevant technical concepts relevant to the **strength and quality** of a server's HTTPS configuration.
 10 | 
 11 | * [SSL and TLS](#ssl-and-tls)
 12 | * [Forward secrecy](#forward-secrecy)
 13 | * [Signature algorithms](#signature-algorithms), such as SHA-1 and SHA-2
 14 | * [Strong ciphersuites](#strong-ciphersuites)
 15 | * [A complete certificate chain](#a-complete-certificate-chain)
 16 | 
 17 | ### SSL and TLS
 18 | 
 19 | HTTPS today uses **Transport Layer Security**, or **TLS**. TLS is a network protocol that establishes an encrypted connection to an authenticated peer over an untrusted network.
 20 | 
 21 | Earlier, less secure versions of this protocol were called **Secure Sockets Layer**, or **SSL)**.
 22 | 
 23 | SSL and TLS perform the same function, and TLS is a direct successor and replacement for SSL. Because of its early ubiquity, "SSL" is frequently used today to generically refer to TLS/HTTPS. However, all versions of SSL as a protocol are now considered insecure for modern use.
 24 | 
 25 | The major versions of SSL/TLS in use today are:
 26 | 
 27 | * **SSLv3:** [Released in 1996.](https://tools.ietf.org/html/rfc6101) **Considered to be insecure** after the [POODLE](https://www.openssl.org/~bodo/ssl-poodle.pdf) attack was published in 2014. Turning off SSLv3 effectively removes support for Internet Explorer 6.
 28 | * **TLSv1.0:** - [Released in 1999.](https://tools.ietf.org/html/rfc2246) Used widely today to support [some older clients](https://www.ssllabs.com/ssltest/clients.html), like IE8 and Android 4.3 and below. [NIST Special Publication 800-52](https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-52r1.pdf) disallows TLSv1.0 for government-facing systems.
 29 | * **TLSv1.1:** - [Released in 2006.](https://tools.ietf.org/html/rfc4346) An improvement over TLSv1.0, but was quickly superseded by TLSv1.2.
 30 | * **TLSv1.2:** - [Released in 2008.](https://tools.ietf.org/html/rfc5246) This is the strongest form of TLS today, and is widely supported by modern browsers.
 31 | 
 32 | Typically, browsers and servers support multiple versions, and will attempt to negotiate the strongest mutually supported version.
 33 | 
 34 | It is possible for an attacker to interfere with the negotiation process and attempt to "downgrade" connections to the oldest mutually supported version.
 35 | 
 36 | A downgrade attack can be prevented by using **[TLS Fallback SCSV](https://tools.ietf.org/html/rfc7507)**, a TLS extension proposed in 2014 and which is enabled by default in newer versions of OpenSSL.
 37 | 
 38 | For more details of NIST recommendations, read [NIST Special Publication 800-52](https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-52r1.pdf).
 39 | 
 40 | * **[The Pulse HTTPS dashboard for .gov domains](https://pulse.cio.gov/https/domains/)** will note when a domain still offers insecure SSLv3, or when a domain does not yet offer TLSv1.2.
 41 | * **[https.cio.gov is configured](https://www.ssllabs.com/ssltest/analyze.html?d=https.cio.gov)** to support TLSv1.0, TLSv1.1, and TLSv1.2, and has TLS Fallback SCSV enabled.
 42 | 
 43 | ### Forward secrecy
 44 | 
 45 | **Forward secrecy** protects information sent over an encrypted HTTPS connection _now_ from being decrypted _later_, even if the server's private key is later compromised.
 46 | 
 47 | In non-forward-secret HTTPS connections, if an attacker records encrypted traffic between a website and its visitors, and later obtains the website's private key, that key can be used to decrypt all past recorded traffic.
 48 | 
 49 | In forward secret connections, the server and client create a temporary key for every new session that gets effectively "thrown away" after the session is complete. This means that even if the server's base private key is compromised, an attacker can't retroactively decrypt information.
 50 | 
 51 | In TLS, forward secrecy is provided by choosing ciphersuites that include the [DHE](https://en.wikipedia.org/wiki/Diffie–Hellman_key_exchange) and [ECDHE](https://en.wikipedia.org/wiki/Elliptic_curve_Diffie%E2%80%93Hellman) key exchanges.
 52 | 
 53 | **Note:** Current drafts of TLS 1.3, the next version of TLS, require new connections to use forward secrecy by [removing support for static RSA and DH key exchange](https://tools.ietf.org/html/draft-ietf-tls-tls13-02#section-1.2).
 54 | 
 55 | * **[The Pulse HTTPS dashboard for .gov domains](https://pulse.cio.gov/https/domains/)** will note when a domain offers little or no forward secrecy.
 56 | * **[https.cio.gov is configured](https://www.ssllabs.com/ssltest/analyze.html?d=https.cio.gov)** to offer robust forward secrecy.
 57 | 
 58 | ### Signature algorithms
 59 | 
 60 | The HTTPS/TLS security model uses "certificates" to guarantee authenticity. These certificates are cryptographically "signed" by a trusted certificate authority.
 61 | 
 62 | The certificate authority's trusted root certificate (which is included with your OS or browser) is used to sign an intermediary certificate, which is used to sign your website's certificate. There may be more than one intermediary certificate in the chain. A part of the signature process is computing a "hash" of the data included in the certificate. This can be done using a standard hashing algorithm, such as [SHA-1](https://en.wikipedia.org/wiki/SHA-1) or [SHA-2](https://en.wikipedia.org/wiki/SHA-2).
 63 | 
 64 | SHA-1 has been shown to have serious weaknesses, and so browser and OS providers like [Google](https://security.googleblog.com/2014/09/gradually-sunsetting-sha-1.html), [Microsoft](https://social.technet.microsoft.com/wiki/contents/articles/32288.windows-enforcement-of-sha1-certificates.aspx), and [Mozilla](https://blog.mozilla.org/security/2014/09/23/phasing-out-certificates-with-sha-1-based-signature-algorithms/) have announced timelines to deprecate SHA-1 in favor of the SHA-2 family of algorithms.
 65 | 
 66 | [NIST has disallowed SHA-1](https://nvlpubs.nist.gov/nistpubs/SpecialPublications/NIST.SP.800-52r1.pdf) for digital signature generation after 2013.
 67 | 
 68 | As of January 2016, commercial CAs are forbidden by most root programs from issuing a SHA-1 certificate. As such, obtaining a publicly trusted SHA-1 certificate is no longer feasible. In addition, site owners with an existing SHA-1 certificate should be aware that many browsers and OSes will be disabling SHA-1 support in early 2017.
 69 | 
 70 | ### Strong ciphersuites
 71 | 
 72 | Each TLS handshake makes use of a set of cryptographic primitives, including ciphers and signature algorithms.
 73 | 
 74 | Which ciphers and algorithms are used in a handshake is a function of client support and preferences, and server support and preferences.
 75 | 
 76 | Federal agencies have no control over the primitives supported by major clients used by the public (such as web browsers, cURL, and other common HTTP clients). However, agencies can control the ciphers and algorithms that are supported by their servers (which can also include proxies, load balancers, or content delivery networks).
 77 | 
 78 | When configuring servers:
 79 | 
 80 | * **Avoid SHA-1 in the TLS handshake.** Though there is no known specific vulnerability in the use of SHA-1 as part of the TLS handshake, SHA-1 has already been shown to be [unacceptably weak for use as a signature algorithm for issued certificates](#signature-algorithms). Beginning [with TLS 1.2](https://tools.ietf.org/html/rfc5246#section-7.4.1.4.1), servers can and should negotiate the use of a signature algorithm other than SHA-1 for the TLS handshake.
 81 | 
 82 | * **Avoid RC4.** [RC4](https://en.wikipedia.org/wiki/RC4) was once a popular cipher, but [in 2013 was found to have a critical flaw](http://www.isg.rhul.ac.uk/tls/). Modern browsers no longer support RC4-based ciphersuites, and servers should no longer need to be configured to support RC4.
 83 | 
 84 | ### A complete certificate chain
 85 | 
 86 | In addition to the certificate itself, you should provide a "chain" of intermediate certificates that give the connecting browser or client enough information to connect the certificate to a trusted root certificate.
 87 | 
 88 | Failing to provide intermediates could prevent various browsers and clients from successfully connecting to your service, especially mobile browsers and non-browser clients (such as cURL, and tools based on libcurl).
 89 | 
 90 | Some browsers will cache intermediates from a previous connection or attempt to automatically download missing intermediates that are presented in a certificate's [Authority Information Access](https://tools.ietf.org/html/rfc5280#section-4.2.2.1) extension, and so it can be easy to miss this problem during initial configuration. Though most browsers have an option to inspect the certificates on a site, they vary in whether they show the exact certificates the server presented or a chain as reconstructed through the fetching of an intermediate listed in the AIA extension.
 91 | 
 92 | In general:
 93 | 
 94 | * You **do not** need to serve the trusted root that the certificate chains to. The client will compare the chain to a local root store, so serving the root will only waste bytes and slow the connection.
 95 | * You **do** need to serve any intermediate certificates that connect your web server certificate to the trusted root. Doing so removes the potential for problems caused by the variation in how clients facilitate trust verification.
 96 | 
 97 | Web servers vary in how they are configured to serve intermediates, but it should generally be straightforward.
 98 | 
 99 | * **[What's My Chain Cert?](https://whatsmychaincert.com/)** can help determine if any intermediate certificates are not being served.
100 | 


--------------------------------------------------------------------------------
/resources/implementing-https-links.md:
--------------------------------------------------------------------------------
 1 | Links used in **[Implementing HTTPS](https://www.digitalgov.gov/event/implementing-https/)**, by Gray Brooks and Eric Mill of GSA.
 2 | 
 3 | See the [recorded presentation on YouTube](https://www.youtube.com/watch?v=rnM2qAfEG-M) (1 hour, 40 minutes).
 4 | 
 5 | #### Introduction
 6 | 
 7 | * [https.cio.gov FAQ on HTTPS](https://https.cio.gov/faq/)
 8 | * [Mozilla blog post on deprecating non-secure HTTP](https://blog.mozilla.org/security/2015/04/30/deprecating-non-secure-http/)
 9 | * [OMB Memorandum M-15-13](https://obamawhitehouse.archives.gov/sites/default/files/omb/memoranda/2015/m-15-13.pdf)
10 | 
11 | #### How HTTPS Works
12 | 
13 | * [Simple explanation for elliptic curve cryptography](https://bithin.wordpress.com/2012/02/22/simple-explanation-for-elliptic-curve-cryptography-ecc/)
14 | * [Prime factorization](http://modular.math.washington.edu/edu/2007/spring/ent/ent-html/node7.html)
15 | * [SSL Labs analysis of https.cio.gov](https://www.ssllabs.com/ssltest/analyze.html?d=https.cio.gov)
16 | * [nginx configuration for https.cio.gov](https://github.com/GSA/https/blob/master/deploy/site.conf)
17 | * [general nginx HTTPS configuration template](https://github.com/fisma-ready/nginx/blob/master/ssl/ssl.rules)
18 | * [Apache configuration for courtlistener.com](https://github.com/freelawproject/courtlistener/blob/master/apache/courtlistener.com.conf)
19 | * [Mozilla CA certificate policy](https://www.mozilla.org/en-US/about/governance/policies/security-group/certs/policy/)
20 | * [Domain name system overview](http://www.itgeared.com/articles/1354-domain-name-system-dns-tutorial-overview/)
21 | 
22 | #### Migrating to HTTPS
23 | 
24 | * [https.cio.gov page on mixed content](https://https.cio.gov/mixed-content/)
25 | * [W3C upgrade-insecure-requests spec](https://w3c.github.io/webappsec/specs/upgrade/)
26 | 
27 | #### HTTP Strict Transport Security
28 | 
29 | * [https.cio.gov page on HSTS](https://https.cio.gov/hsts/)
30 | * [18F blog: The first .gov domains hardcoded into your browser as all-HTTPS](https://18f.gsa.gov/2015/02/09/the-first-gov-domains-hardcoded-into-your-browser-as-all-https/)
31 | * [@HttpSecHeaders tweet about HSTS preloading](https://twitter.com/HttpSecHeaders/status/618546051641909248)
32 | * [Chrome HSTS/HPKP preload list](https://chromium.googlesource.com/chromium/src/+/master/net/http/transport_security_state_static.json)
33 | 
34 | #### Technical Guidelines
35 | 
36 | * [https.cio.gov page on technical guidelines](https://https.cio.gov/technical-guidelines/)
37 | * [Google security blog post on SHA-1 deprecation](http://googleonlinesecurity.blogspot.com/2014/09/gradually-sunsetting-sha-1.html)
38 | * [Pulse page on HTTPS for .gov domains](https://pulse.cio.gov/https/domains/)
39 | * [https.cio.gov page on Server Name Indication](https://https.cio.gov/sni/)
40 | * [Introduction to OCSP](http://www.x500standard.com/index.php?n=X509W.OCSP)
41 | 
42 | #### The Future
43 | 
44 | * [TLS 1.3](https://tlswg.github.io/tls13-spec/)
45 | * [Google blog post on QUIC](http://blog.chromium.org/2015/04/a-quic-update-on-googles-experimental.html)
46 | * [HTTP/2 spec on TLS requirements](http://http2.github.io/http2-spec/#TLSUsage)
47 | * [Digital Analytics Program dashboard](https://analytics.usa.gov)
48 | * [Certificate transparency readings for *.whitehouse.gov](https://crt.sh/?q=%25whitehouse.gov)
49 | * [Certificate Transparency homepage](http://www.certificate-transparency.org/)
50 | * [Mozilla blog post on distrusting CNNIC](https://blog.mozilla.org/security/2015/04/02/distrusting-new-cnnic-certificates/)
51 | * [RFC 7469, HTTP Public Key Pinning](http://tools.ietf.org/html/rfc7469)
52 | * [HTTP Public Key Pinning, Explained](https://timtaubert.de/blog/2014/10/http-public-key-pinning-explained/)
53 | * [CA/Browser Forum Baseline Requiremets](https://cabforum.org/baseline-requirements-documents/)
54 | * [House Energy and Commerce Committee letter to browsers regarding government CAs](https://energycommerce.house.gov/letter/letters-browsers-regarding-government-certificate-authorities)
55 | * [Mozilla's response to the House letter](https://blog.mozilla.org/netpolicy/files/2015/06/Mozilla-Response-to-Congressional-letter-on-CAs-signed.pdf)
56 | * [Let's Encrypt, an upcoming free non-profit certificate authority](https://letsencrypt.org/)
57 | 


--------------------------------------------------------------------------------