├── .github └── workflows │ ├── code-style.yml │ ├── integration-tests.yml │ └── unit-tests.yml ├── .gitignore ├── CONTRIBUTING.md ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── SECURITY.md ├── cloudflare_worker ├── .gitignore ├── .travis.yml ├── Cargo.toml ├── README.md ├── fail_open.png ├── publish.sh ├── src │ └── lib.rs └── worker │ ├── .eslintignore │ ├── .eslintrc.json │ ├── .gitignore │ ├── .prettierrc.js │ ├── build.sh │ ├── package-lock.json │ ├── package.json │ ├── src │ ├── bindings.d.ts │ ├── index.ts │ ├── signer.ts │ ├── storage.ts │ └── wasmFunctions.ts │ └── tsconfig.json ├── credentials ├── .gitignore ├── README.md └── parse_private_key.go ├── distributor ├── .gitignore ├── Cargo.toml ├── README.md ├── example.html ├── instantpage.js.patch └── src │ ├── main.rs │ ├── parse_signature.rs │ ├── parse_sxg.rs │ ├── validate_cert.rs │ └── validate_sxg.rs ├── fastly_compute ├── .gitignore ├── Cargo.toml ├── README.md └── src │ ├── fetcher.rs │ ├── main.rs │ └── storage.rs ├── http_server ├── .gitignore ├── Cargo.toml ├── README.md └── src │ └── main.rs ├── input.example.yaml ├── playground ├── .eslintignore ├── .eslintrc.json ├── .gitignore ├── .prettierrc.js ├── README.md ├── build.sh ├── package-lock.json ├── package.json ├── src │ ├── client │ │ ├── emulationOptions.ts │ │ ├── evaluated.ts │ │ ├── index.ts │ │ ├── statistics.ts │ │ └── templates.ts │ ├── index.ts │ ├── schema.ts │ └── server │ │ ├── cache.ts │ │ ├── credentials.ts │ │ ├── fetcher.ts │ │ ├── index.ts │ │ ├── signer.ts │ │ └── wasmFunctions.ts └── tsconfig.json ├── renovate.json ├── rustfmt.toml ├── sxg_rs ├── Cargo.toml ├── README.md └── src │ ├── acme │ ├── client.rs │ ├── directory.rs │ ├── eab.rs │ ├── jws.rs │ ├── mod.rs │ └── state_machine.rs │ ├── cbor.rs │ ├── config.rs │ ├── crypto.rs │ ├── fetcher │ ├── js_fetcher.rs │ ├── mock_fetcher.rs │ └── mod.rs │ ├── header_integrity.rs │ ├── headers.rs │ ├── http.rs │ ├── http_cache │ ├── js_http_cache.rs │ └── mod.rs │ ├── http_parser │ ├── accept.rs │ ├── base.rs │ ├── cache_control.rs │ ├── link.rs │ ├── media_type.rs │ ├── mod.rs │ └── srcset.rs │ ├── id_headers.rs │ ├── lib.rs │ ├── link.rs │ ├── mice.rs │ ├── ocsp │ └── mod.rs │ ├── process_html.rs │ ├── runtime │ ├── js_runtime.rs │ └── mod.rs │ ├── serde_helpers │ ├── base64.rs │ └── mod.rs │ ├── signature │ ├── js_signer.rs │ ├── mock_signer.rs │ ├── mod.rs │ └── rust_signer.rs │ ├── static │ ├── fallback.html │ ├── image.jpg │ ├── prefetch.html │ ├── success.html │ └── test.html │ ├── storage │ ├── js_storage.rs │ └── mod.rs │ ├── structured_header │ ├── item.rs │ ├── mod.rs │ └── parameterised_list.rs │ ├── sxg.rs │ ├── utils.rs │ └── wasm_worker.rs ├── tools ├── Cargo.toml └── src │ ├── commands │ ├── apply_acme_cert.rs │ ├── gen_config │ │ ├── cloudflare.rs │ │ ├── fastly.rs │ │ ├── http_server.rs │ │ └── mod.rs │ ├── gen_dev_cert.rs │ ├── gen_sxg.rs │ └── mod.rs │ ├── lib.rs │ ├── linux_commands.rs │ ├── main.rs │ └── runtime │ ├── hyper_fetcher.rs │ ├── mod.rs │ └── openssl_signer.rs └── typescript_utilities ├── .gitignore ├── README.md ├── build.sh ├── karma.conf.js ├── package-lock.json ├── package.json ├── spec └── support │ └── jasmine.json └── src ├── processor.ts ├── signer.ts ├── streams.test.ts ├── streams.ts ├── utils.test.ts ├── utils.ts └── wasmFunctions.ts /.github/workflows/code-style.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | name: Code Style 16 | on: 17 | push: 18 | branches: 19 | - main 20 | pull_request: 21 | branches: 22 | - main 23 | jobs: 24 | Rust: 25 | runs-on: ubuntu-latest 26 | steps: 27 | - uses: actions/checkout@v3 28 | - uses: actions/cache@v3 29 | with: 30 | path: | 31 | ~/.cargo/bin/ 32 | ~/.cargo/registry/index/ 33 | ~/.cargo/registry/cache/ 34 | ~/.cargo/git/db/ 35 | target/ 36 | key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} 37 | - uses: actions-rs/cargo@v1 38 | with: 39 | command: fmt 40 | args: -- --check 41 | - run: | 42 | touch credentials/cert.pem 43 | touch credentials/issuer.pem 44 | touch cloudflare_worker/wrangler.toml 45 | touch fastly_compute/fastly.toml 46 | touch fastly_compute/config.yaml 47 | touch http_server/config.yaml 48 | - uses: actions-rs/cargo@v1 49 | with: 50 | command: clippy 51 | args: --workspace --exclude http_server --all-features --all-targets -- -D warnings 52 | - uses: actions-rs/cargo@v1 53 | with: 54 | command: clippy 55 | args: --package http_server --all-targets -- -D warnings 56 | TypeScript: 57 | runs-on: ubuntu-latest 58 | steps: 59 | - uses: actions/checkout@v3 60 | - uses: actions/setup-node@v3 61 | - working-directory: cloudflare_worker/worker 62 | run: | 63 | npm install 64 | npm run lint 65 | - working-directory: playground 66 | run: | 67 | npm install 68 | npm run lint 69 | 70 | -------------------------------------------------------------------------------- /.github/workflows/integration-tests.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | name: Integration tests 16 | on: 17 | push: 18 | branches: 19 | - main 20 | pull_request: 21 | branches: 22 | - main 23 | jobs: 24 | cloudflare_worker: 25 | runs-on: ubuntu-latest 26 | steps: 27 | - uses: actions/checkout@v3 28 | - name: Create self-signed cert 29 | working-directory: credentials 30 | run: | 31 | cargo run -p tools -- gen-dev-cert --domain example.org 32 | - run: npm install -g @cloudflare/wrangler 33 | - name: Gen config 34 | run: | 35 | cargo run -p tools -- gen-config --use-ci-mode --input input.example.yaml --artifact artifact.yaml --platform cloudflare 36 | - name: Build 37 | working-directory: cloudflare_worker 38 | run: | 39 | ./publish.sh build 40 | # TODO: ./publish.sh dev & curl | dump-signedexchange -verify 41 | fastly_compute: 42 | runs-on: ubuntu-latest 43 | steps: 44 | - uses: actions/checkout@v3 45 | - name: Create self-signed cert 46 | working-directory: credentials 47 | run: | 48 | cargo run -p tools -- gen-dev-cert --domain example.org 49 | - name: Install fastly CLI 50 | run: | 51 | wget -nv https://github.com/fastly/cli/releases/download/v0.39.2/fastly_0.39.2_linux_amd64.deb 52 | sudo apt-get install ./fastly_0.39.2_linux_amd64.deb 53 | - uses: actions-rs/toolchain@v1 54 | with: 55 | toolchain: stable 56 | target: wasm32-wasi 57 | - name: Gen config 58 | run: | 59 | cargo run -p tools -- gen-config --use-ci-mode --input input.example.yaml --artifact artifact.yaml --platform fastly 60 | - name: Build and validate 61 | working-directory: fastly_compute 62 | run: | 63 | fastly compute build 64 | fastly compute validate -p pkg/sxg.tar.gz 65 | - name: Serve 66 | working-directory: fastly_compute 67 | run: | 68 | fastly compute serve --skip-build | tee log & 69 | until grep -m1 'Listening on http' log; do sleep 1; done 70 | kill $! 71 | # TODO: curl | dump-signedexchange -verify 72 | # TODO: Get rid of this once dump-signedexchange works on the above two cases. 73 | gen_sxg: 74 | runs-on: ubuntu-latest 75 | steps: 76 | - uses: actions/checkout@v3 77 | - name: Create self-signed cert 78 | working-directory: credentials 79 | run: | 80 | cargo run -p tools -- gen-dev-cert --domain example.org 81 | - uses: actions/setup-go@v3 82 | with: 83 | go-version: '>=1.16' 84 | - run: go install github.com/WICG/webpackage/go/signedexchange/cmd/dump-signedexchange@latest 85 | - uses: actions-rs/toolchain@v1 86 | with: 87 | toolchain: stable 88 | target: wasm32-wasi 89 | - name: Generate and validate 90 | run: | 91 | cargo run -p tools -- gen-config --use-ci-mode --input input.example.yaml --artifact artifact.yaml --platform http-server 92 | cargo run -p tools -- gen-sxg -- \ 93 | http_server/config.yaml credentials/cert.pem credentials/issuer.pem \ 94 | test.cert test.sxg 95 | dump-signedexchange -i test.sxg -cert test.cert -verify 96 | playground: 97 | runs-on: ubuntu-latest 98 | steps: 99 | - uses: actions/checkout@v3 100 | - uses: actions/cache@v3 101 | with: 102 | path: | 103 | ~/.cargo/bin/ 104 | ~/.cargo/registry/index/ 105 | ~/.cargo/registry/cache/ 106 | ~/.cargo/git/db/ 107 | target/ 108 | playground/node_modules 109 | key: ${{ runner.os }}-playground-${{ hashFiles('**/Cargo.lock', 'playground/package.json') }} 110 | - name: Gen Dev certificate 111 | working-directory: credentials 112 | run: | 113 | cargo run -p tools -- gen-dev-cert --domain example.org 114 | - name: Gen config 115 | run: | 116 | cargo run -p tools -- gen-config --use-ci-mode --input input.example.yaml --artifact artifact.yaml --platform cloudflare 117 | - name: Build wasm 118 | working-directory: cloudflare_worker 119 | run: | 120 | npm install -g @cloudflare/wrangler 121 | wrangler build 122 | - name: Build playground 123 | working-directory: playground 124 | run: | 125 | npm install 126 | npm run build 127 | node dist/index.js --single-url https://example.com/ 128 | -------------------------------------------------------------------------------- /.github/workflows/unit-tests.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | name: Unit tests 16 | on: 17 | push: 18 | branches: 19 | - main 20 | pull_request: 21 | branches: 22 | - main 23 | jobs: 24 | Rust: 25 | runs-on: ubuntu-latest 26 | steps: 27 | - uses: actions/checkout@v3 28 | - uses: actions/cache@v3 29 | with: 30 | path: | 31 | ~/.cargo/bin/ 32 | ~/.cargo/registry/index/ 33 | ~/.cargo/registry/cache/ 34 | ~/.cargo/git/db/ 35 | target/ 36 | key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} 37 | - working-directory: credentials 38 | run: | 39 | cargo run -p tools -- gen-dev-cert --domain example.org 40 | - uses: actions-rs/toolchain@v1 41 | with: 42 | toolchain: stable 43 | - uses: actions-rs/cargo@v1 44 | with: 45 | command: test 46 | args: --workspace --exclude http_server --all-features 47 | - uses: actions-rs/cargo@v1 48 | with: 49 | command: test 50 | args: --package http_server 51 | TypeScript: 52 | runs-on: ubuntu-latest 53 | steps: 54 | - uses: actions/checkout@v3 55 | - uses: actions/setup-node@v3 56 | - working-directory: typescript_utilities 57 | run: | 58 | npm install 59 | npm run build 60 | npm run test 61 | - working-directory: cloudflare_worker/worker 62 | run: | 63 | npm install 64 | npm run build 65 | - working-directory: playground 66 | run: | 67 | npm install 68 | mkdir -p ../cloudflare_worker/pkg && touch ../cloudflare_worker/pkg/cloudflare_worker.js 69 | npm run build 70 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | 3 | /input.yaml 4 | /artifact.yaml 5 | 6 | **/*.rs.bk 7 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code Reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Community Guidelines 26 | 27 | This project follows [Google's Open Source Community 28 | Guidelines](https://opensource.google/conduct/). 29 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | members = [ 3 | "cloudflare_worker", 4 | "distributor", 5 | "fastly_compute", 6 | "http_server", 7 | "sxg_rs", 8 | "tools", 9 | ] 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 16 | 17 | # sxg-rs 18 | 19 | sxg-rs is a set of tools for generating [signed 20 | exchanges](https://web.dev/signed-exchanges/) at serve time: 21 | 22 | * [`cloudflare_worker`](cloudflare_worker) runs on [Cloudflare Workers](https://workers.cloudflare.com/). 23 | * [`distributor`](distributor) is an example implementation of privacy-preserving SXG prefetch of outlinks. 24 | * [`fastly_compute`](fastly_compute) runs on [Fastly Compute@Edge](https://www.fastly.com/products/edge-compute/serverless). 25 | * [`http_server`](http_server) runs as an HTTP reverse-proxy on Linux. 26 | * [`playground`](playground) is a CLI for previewing LCP impact of SXG on any site. 27 | * [`sxg_rs`](sxg_rs) is the Rust library that can be used as a basis for other serverless platforms. 28 | 29 | These tools enable sites to be [prefetched from Google 30 | Search](https://developers.google.com/search/docs/advanced/experience/signed-exchange) 31 | in order to improve their [Largest Contentful Paint](https://web.dev/lcp/), one 32 | of the [Core Web Vitals](https://web.dev/vitals/). 33 | 34 | For other technology stacks, see [this list of SXG tools](https://web.dev/signed-exchanges/#tooling). 35 | 36 | ## Next steps 37 | 38 | After installing, take the following steps. 39 | 40 | ### Verify and monitor 41 | 42 | After installing, you may want to 43 | [verify](https://developers.google.com/search/docs/advanced/experience/signed-exchange#verify-sxg-setup) 44 | and 45 | [monitor](https://developers.google.com/search/docs/advanced/experience/signed-exchange#monitor-and-debug-sxg) 46 | the results. 47 | 48 | ### HTML processing 49 | 50 | The worker contains some HTML processors. To activate them, explicitly label the character encoding as UTF-8, either via: 51 | 52 | ```http 53 | Content-Type: text/html;charset=utf-8 54 | ``` 55 | 56 | or via: 57 | 58 | ```html 59 | 60 | ``` 61 | 62 | #### Preload subresources 63 | 64 | LCP can be further improved by instructing Google Search to prefetch 65 | render-critical subresources for the page. 66 | 67 | ##### Same-origin 68 | 69 | Add a preload link tag to the page, such as: 70 | 71 | ``` 72 | 73 | ``` 74 | 75 | sxg-rs will automatically convert these link tags into Link headers as needed for [SXG 76 | subresource 77 | substitution](https://github.com/WICG/webpackage/blob/main/explainers/signed-exchange-subresource-substitution.md). 78 | This uses a form of subresource integrity that includes HTTP headers. sxg-rs 79 | tries to ensure a static integrity value by stripping many noisy HTTP headers 80 | (like Date) for signed subresources, but you may need to list additional ones 81 | in the `strip_response_headers` config param. 82 | 83 | To confirm it is working, run: 84 | 85 | ```bash 86 | $ go install github.com/WICG/webpackage/go/signedexchange/cmd/dump-signedexchange@latest 87 | $ dump-signedexchange -uri "$HTML_URL" -payload=false | grep Link 88 | ``` 89 | 90 | and verify that there is a `rel=allowed-alt-sxg` whose `header-integrity` 91 | matches the output of: 92 | 93 | ```bash 94 | $ dump-signedexchange -uri "$SUBRESOURCE_URL" -headerIntegrity 95 | ``` 96 | 97 | If you have any same-origin preload tags that should not be converted into 98 | headers, add the `data-sxg-no-header` attribute to them. 99 | 100 | ##### Cross-origin 101 | 102 | SXG preloading requires that the subresource is also an SXG. This worker 103 | assumes only same-origin resources are SXG, so its automatic logic is limited 104 | to those. You can manually support cross-origin subresources by adding the 105 | appropriate Link header as 106 | [specified](https://github.com/WICG/webpackage/blob/main/explainers/signed-exchange-subresource-substitution.md). 107 | 108 | #### SXG-only behavior 109 | 110 | There are two syntaxes for behavior that happens only when the page is viewed 111 | as an SXG. If you write: 112 | 113 | ```html 114 | 115 | ``` 116 | 117 | then its inner content will be replaced by `window.isSXG=true` in an SXG. This 118 | could be used as a custom dimension by which to slice web analytics, or as a 119 | cue to fetch a fresh CSRF token. 120 | 121 | If you write: 122 | 123 | ```html 124 | 125 | ``` 126 | 127 | then in an SXG, its inner content will be "unwrapped" out of the template and 128 | thus activated, and when non-SXG it will be deleted. Since SXGs can't Vary by 129 | Cookie, this could be used to add lazy-loaded personalization to the SXG, while 130 | not adding unnecesary bytes to the non-SXG. It could also be used to add 131 | SXG-only subresource preloads. 132 | 133 | ### Preview in Chrome 134 | 135 | Optionally, preview the results in the browser: 136 | 137 | - In development, set Chrome flags to [allow the 138 | certificate](https://github.com/google/webpackager/tree/main/cmd/webpkgserver#testing-with-self-signed--invalid-certificates). 139 | - Use an extension such as 140 | [ModHeader](https://chrome.google.com/webstore/detail/modheader/idgpnmonknjnojddfkpgkljpfnnfcklj) 141 | to set the `Accept` header to 142 | `text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3` 143 | (equivalent to what Googlebot sends). 144 | - Explore the results [in the DevTools Network tab](https://web.dev/signed-exchanges/#debugging). 145 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | To report a security issue, please use http://g.co/vulnz. We use 4 | http://g.co/vulnz for our intake, and do coordination and disclosure here on 5 | GitHub (including using GitHub Security Advisory). The Google Security Team will 6 | respond within 5 working days of your report on g.co/vulnz. 7 | -------------------------------------------------------------------------------- /cloudflare_worker/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /wrangler.toml 3 | **/*.rs.bk 4 | bin/ 5 | pkg/ 6 | wasm-pack.log 7 | worker/generated/ 8 | 9 | -------------------------------------------------------------------------------- /cloudflare_worker/.travis.yml: -------------------------------------------------------------------------------- 1 | language: rust 2 | sudo: false 3 | 4 | cache: cargo 5 | 6 | matrix: 7 | include: 8 | 9 | # Builds with wasm-pack. 10 | - rust: beta 11 | env: RUST_BACKTRACE=1 12 | addons: 13 | firefox: latest 14 | chrome: stable 15 | before_script: 16 | - (test -x $HOME/.cargo/bin/cargo-install-update || cargo install cargo-update) 17 | - (test -x $HOME/.cargo/bin/cargo-generate || cargo install --vers "^0.2" cargo-generate) 18 | - cargo install-update -a 19 | - curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh -s -- -f 20 | script: 21 | - cargo generate --git . --name testing 22 | # Having a broken Cargo.toml (in that it has curlies in fields) anywhere 23 | # in any of our parent dirs is problematic. 24 | - mv Cargo.toml Cargo.toml.tmpl 25 | - cd testing 26 | - wasm-pack build 27 | - wasm-pack test --chrome --firefox --headless 28 | 29 | # Builds on nightly. 30 | - rust: nightly 31 | env: RUST_BACKTRACE=1 32 | before_script: 33 | - (test -x $HOME/.cargo/bin/cargo-install-update || cargo install cargo-update) 34 | - (test -x $HOME/.cargo/bin/cargo-generate || cargo install --vers "^0.2" cargo-generate) 35 | - cargo install-update -a 36 | - rustup target add wasm32-unknown-unknown 37 | script: 38 | - cargo generate --git . --name testing 39 | - mv Cargo.toml Cargo.toml.tmpl 40 | - cd testing 41 | - cargo check 42 | - cargo check --target wasm32-unknown-unknown 43 | - cargo check --no-default-features 44 | - cargo check --target wasm32-unknown-unknown --no-default-features 45 | - cargo check --no-default-features --features console_error_panic_hook 46 | - cargo check --target wasm32-unknown-unknown --no-default-features --features console_error_panic_hook 47 | - cargo check --no-default-features --features "console_error_panic_hook wee_alloc" 48 | - cargo check --target wasm32-unknown-unknown --no-default-features --features "console_error_panic_hook wee_alloc" 49 | 50 | # Builds on beta. 51 | - rust: beta 52 | env: RUST_BACKTRACE=1 53 | before_script: 54 | - (test -x $HOME/.cargo/bin/cargo-install-update || cargo install cargo-update) 55 | - (test -x $HOME/.cargo/bin/cargo-generate || cargo install --vers "^0.2" cargo-generate) 56 | - cargo install-update -a 57 | - rustup target add wasm32-unknown-unknown 58 | script: 59 | - cargo generate --git . --name testing 60 | - mv Cargo.toml Cargo.toml.tmpl 61 | - cd testing 62 | - cargo check 63 | - cargo check --target wasm32-unknown-unknown 64 | - cargo check --no-default-features 65 | - cargo check --target wasm32-unknown-unknown --no-default-features 66 | - cargo check --no-default-features --features console_error_panic_hook 67 | - cargo check --target wasm32-unknown-unknown --no-default-features --features console_error_panic_hook 68 | # Note: no enabling the `wee_alloc` feature here because it requires 69 | # nightly for now. 70 | -------------------------------------------------------------------------------- /cloudflare_worker/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "cloudflare_worker" 17 | version = "0.1.0" 18 | authors = ["9083193+antiphoton@users.noreply.github.com"] 19 | edition = "2018" 20 | 21 | [lib] 22 | crate-type = ["cdylib", "rlib"] 23 | 24 | [dependencies] 25 | console_error_panic_hook = "0.1.7" 26 | sxg_rs = { path = "../sxg_rs", features = ["wasm"] } 27 | wasm-bindgen = "0.2.83" 28 | 29 | [profile.release] 30 | opt-level = 3 31 | 32 | [package.metadata.wasm-pack.profile.release] 33 | wasm-opt = true 34 | 35 | -------------------------------------------------------------------------------- /cloudflare_worker/fail_open.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sxg-rs/06d31585c95cd23c2c6a9f5f5b6fc5eb247bcfdd/cloudflare_worker/fail_open.png -------------------------------------------------------------------------------- /cloudflare_worker/publish.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2021 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -e 18 | 19 | cd worker 20 | npm install 21 | npm run build 22 | cp dist/index.js worker.js 23 | cd .. 24 | 25 | wrangler "${@-publish}" 26 | -------------------------------------------------------------------------------- /cloudflare_worker/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // TODO(antiphoton) No longer allow unused_unit when a new version wasm_bindgen is released with 16 | // https://github.com/rustwasm/wasm-bindgen/pull/2778 17 | #![allow(clippy::unused_unit)] 18 | 19 | use wasm_bindgen::prelude::wasm_bindgen; 20 | 21 | extern crate sxg_rs; 22 | 23 | #[wasm_bindgen(js_name=init)] 24 | pub fn init() { 25 | console_error_panic_hook::set_once() 26 | } 27 | -------------------------------------------------------------------------------- /cloudflare_worker/worker/.eslintignore: -------------------------------------------------------------------------------- 1 | /dist 2 | /generated 3 | /node_modules 4 | /rollup.config.js 5 | /worker.js 6 | -------------------------------------------------------------------------------- /cloudflare_worker/worker/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./node_modules/gts/" 3 | } 4 | -------------------------------------------------------------------------------- /cloudflare_worker/worker/.gitignore: -------------------------------------------------------------------------------- 1 | /dist 2 | /generated 3 | /node_modules 4 | /worker.js 5 | -------------------------------------------------------------------------------- /cloudflare_worker/worker/.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | ...require('gts/.prettierrc.json') 3 | } 4 | -------------------------------------------------------------------------------- /cloudflare_worker/worker/build.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | node_modules/.bin/tsc --noEmit 3 | node_modules/.bin/esbuild src/index.ts --bundle --platform=browser --outfile=dist/index.js 4 | -------------------------------------------------------------------------------- /cloudflare_worker/worker/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "name": "cloudflare_worker", 4 | "version": "0.1.0", 5 | "main": "dist/index.js", 6 | "scripts": { 7 | "build": "./build.sh", 8 | "lint": "gts lint" 9 | }, 10 | "devDependencies": { 11 | "@cloudflare/workers-types": "3.18.0", 12 | "@types/node": "16.18.11", 13 | "esbuild": "0.16.14", 14 | "glob": "8.0.3", 15 | "gts": "3.1.1", 16 | "tslib": "2.4.1", 17 | "typescript": "4.9.4" 18 | }, 19 | "engines": { 20 | "node": ">=16.0.0" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /cloudflare_worker/worker/src/bindings.d.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export {}; 18 | 19 | declare global { 20 | const CERT_ORIGIN: string; 21 | const CERT_PEM: string; 22 | const HTML_HOST: string; 23 | const ISSUER_PEM: string; 24 | const OCSP: KVNamespace; 25 | const PRIVATE_KEY_JWK: string | undefined; 26 | const ACME_PRIVATE_KEY_JWK: string; 27 | const ACME_ACCOUNT: string; 28 | const SXG_CONFIG: string; 29 | } 30 | -------------------------------------------------------------------------------- /cloudflare_worker/worker/src/signer.ts: -------------------------------------------------------------------------------- 1 | ../../../typescript_utilities/src/signer.ts -------------------------------------------------------------------------------- /cloudflare_worker/worker/src/storage.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export async function storageRead(k: string): Promise { 18 | return await OCSP.get(k); 19 | } 20 | 21 | export async function storageWrite(k: string, v: string): Promise { 22 | await OCSP.put(k, v); 23 | } 24 | -------------------------------------------------------------------------------- /cloudflare_worker/worker/src/wasmFunctions.ts: -------------------------------------------------------------------------------- 1 | ../../../typescript_utilities/src/wasmFunctions.ts -------------------------------------------------------------------------------- /cloudflare_worker/worker/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2015", 4 | "module": "ESNext", 5 | "lib": ["DOM", "DOM.Iterable", "ESNext"], 6 | "declaration": true, 7 | "outDir": "./dist", 8 | "strict": true, 9 | "noImplicitAny": true, 10 | "strictNullChecks": true, 11 | "strictFunctionTypes": true, 12 | "strictBindCallApply": true, 13 | "strictPropertyInitialization": true, 14 | "noImplicitThis": true, 15 | "alwaysStrict": true, 16 | "noUnusedLocals": true, 17 | "noUnusedParameters": true, 18 | "noImplicitReturns": true, 19 | "noFallthroughCasesInSwitch": true, 20 | "noUncheckedIndexedAccess": true, 21 | "noImplicitOverride": true, 22 | "noPropertyAccessFromIndexSignature": true, 23 | "moduleResolution": "node", 24 | "types": [ 25 | "@cloudflare/workers-types" 26 | ], 27 | "esModuleInterop": true, 28 | "skipLibCheck": true, 29 | "forceConsistentCasingInFileNames": true 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /credentials/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !/.gitignore 3 | !/README.md 4 | -------------------------------------------------------------------------------- /credentials/parse_private_key.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | import ( 17 | "crypto/ecdsa" 18 | "crypto/x509" 19 | "encoding/base64" 20 | "encoding/json" 21 | "encoding/pem" 22 | "fmt" 23 | "io" 24 | "io/ioutil" 25 | "os" 26 | ) 27 | 28 | func printKey(key *ecdsa.PrivateKey) { 29 | var x = base64.StdEncoding.EncodeToString(key.PublicKey.X.Bytes()) 30 | var y = base64.StdEncoding.EncodeToString(key.PublicKey.Y.Bytes()) 31 | var d = base64.StdEncoding.EncodeToString(key.D.Bytes()) 32 | var jwk, _ = json.Marshal(map[string]string { 33 | "kty": "EC", 34 | "crv": "P-256", 35 | "x": x, 36 | "y": y, 37 | "d": d, 38 | }) 39 | fmt.Printf("Private key in JWK format:\n%s\n", jwk) 40 | fmt.Printf("Private key in base64 format:\n%s\n", d) 41 | } 42 | 43 | func main() { 44 | text, err := ioutil.ReadAll(io.LimitReader(os.Stdin, 4194304)) 45 | if err != nil { 46 | panic("Error reading input: " + err.Error()) 47 | } 48 | for len(text) > 0 { 49 | var block *pem.Block 50 | block, text = pem.Decode(text) 51 | if block == nil { 52 | break 53 | } 54 | key, _ := x509.ParseECPrivateKey(block.Bytes) 55 | if key != nil { 56 | printKey(key) 57 | return 58 | } 59 | } 60 | fmt.Println("No matching private key was found. The PEM file should include the lines:") 61 | fmt.Println("-----BEGIN EC PRIVATE KEY-----") 62 | fmt.Println("and") 63 | fmt.Println("-----END EC PRIVATE KEY-----") 64 | } 65 | -------------------------------------------------------------------------------- /distributor/.gitignore: -------------------------------------------------------------------------------- 1 | instantpage.js 2 | -------------------------------------------------------------------------------- /distributor/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "distributor" 17 | version = "0.1.0" 18 | authors = ["webpackaging-announce@googlegroups.com"] 19 | edition = "2018" 20 | 21 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 22 | 23 | [dependencies] 24 | anyhow = { version = "1.0.66", features = ["backtrace"] } 25 | base64 = "0.13.1" 26 | byte-strings = { version = "0.2.2", features = ["const-friendly"] } 27 | ciborium = "0.2.0" 28 | clap = { version = "3.2.23", features = ["derive"] } 29 | form_urlencoded = "1.1.0" 30 | futures = "0.3.25" 31 | http = "0.2.8" 32 | hyper-rustls = "0.23.2" 33 | hyper-trust-dns = { version = "0.5.0", default-features = false, features = ["rustls-webpki", "rustls-http1", "rustls-tls-12"] } 34 | hyper = { version = "0.14.23", features = ["http1", "http2", "server", "stream", "tcp"] } 35 | lazy_static = "1.4.0" 36 | nom = { version = "7.1.1" } 37 | percent-encoding = "2.2.0" 38 | regex = "1.7.0" 39 | rustls = "0.20.7" 40 | rustls-pemfile = "1.0.1" 41 | sha2 = "0.10.6" 42 | # TODO: Determine if I can remove strip_id_headers because it's default. 43 | sxg_rs = { path = "../sxg_rs", features = ["strip_id_headers", "rust_signer", "srcset"] } 44 | thiserror = "1.0.37" 45 | tls-listener = { version = "0.5.1", features = ["hyper-h1", "hyper-h2", "rustls"] } 46 | tokio = { version = "1.23.0", features = ["rt-multi-thread", "macros", "sync", "time"] } 47 | tokio-rustls = "0.23.4" 48 | url = "2.3.1" 49 | -------------------------------------------------------------------------------- /distributor/README.md: -------------------------------------------------------------------------------- 1 | # SXG distributor 2 | 3 | This is an example SXG distributor that could be used for privacy-preserving 4 | prefetching. If an HTTP caching layer is put in front, the result would be 5 | similar to webpkgcache.com. 6 | 7 | This is early experimental code, useful only as a demo so far. 8 | 9 | It's coded as an https server, but it's stateless so it could easily be ported 10 | to a serverless architecture. 11 | 12 | ## Instructions 13 | 14 | ### Launch the distributor 15 | ```bash 16 | $ openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -sha256 -days 365 -nodes -subj '/CN=localhost' 17 | # Specifying `User-Agent: Googlebot` for now because Cloudflare Automatic Signed Exchanges is only enabled for certain combinations of User-Agent and Accept. Perhaps some others will work. 18 | $ cargo run -p distributor -- --origin https://localhost:8080 --user-agent Googlebot --cert cert.pem --key key.pem & 19 | ``` 20 | 21 | ### Launch the prefetching referrer 22 | ```bash 23 | $ pushd distributor 24 | $ curl -s https://raw.githubusercontent.com/instantpage/instant.page/v5.1.1/instantpage.js >instantpage.js 25 | $ patch instantpage.js instantpage.js.patch 26 | $ python3 -m http.server & 27 | ``` 28 | 29 | ### Launch the test browser 30 | ```bash 31 | $ popd 32 | $ google-chrome --user-data-dir=/tmp/udd --ignore-certificate-errors-spki-list=$(openssl x509 -pubkey -noout -in cert.pem | openssl pkey -pubin -outform der | openssl dgst -sha256 -binary | base64) http://localhost:8000/example.html & 33 | ``` 34 | 35 | Open the DevTools Network tab to see how, on hover, cross-origin links are prefetched from the distributor. 36 | -------------------------------------------------------------------------------- /distributor/example.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |

HTML 4 |

HTML + image 5 |

HTML + lots of stuff 6 |

non-SXG 7 | 85 | 89 | 90 | -------------------------------------------------------------------------------- /distributor/instantpage.js.patch: -------------------------------------------------------------------------------- 1 | 234a235,238 2 | > if (typeof instantUrlModifier === 'function') { 3 | > url = instantUrlModifier(url); 4 | > } 5 | > 6 | 241a246,248 7 | > if (typeof instantLinkModifier === 'function') { 8 | > instantLinkModifier(prefetcher); 9 | > } 10 | -------------------------------------------------------------------------------- /distributor/src/parse_sxg.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{anyhow, Result}; 2 | use futures::stream::{StreamExt, TryStreamExt}; 3 | use hyper::Body; 4 | use nom::{ 5 | bytes::streaming::{tag, take}, 6 | combinator::{map, map_res, verify}, 7 | multi::length_data, 8 | number::streaming::{be_u16, be_u24}, 9 | sequence::{pair, preceded}, 10 | IResult, 11 | }; 12 | 13 | #[derive(Debug)] 14 | pub struct Parts { 15 | pub fallback_url: String, 16 | pub signature: Vec, 17 | pub signed_headers: Vec, 18 | pub payload_body: Body, 19 | } 20 | 21 | fn signature_and_headers(input: &[u8]) -> IResult<&[u8], (&[u8], &[u8])> { 22 | let (input, (sig_length, header_length)) = pair( 23 | verify(be_u24, |len| *len <= 16384), 24 | verify(be_u24, |len| *len <= 524288), 25 | )(input)?; 26 | pair(take(sig_length), take(header_length))(input) 27 | } 28 | 29 | fn parse_impl(sxg: &[u8]) -> IResult<&[u8], Parts> { 30 | preceded( 31 | tag(b"sxg1-b3\0"), 32 | map( 33 | pair( 34 | map_res(length_data(be_u16), |url: &[u8]| { 35 | String::from_utf8(url.to_vec()) 36 | }), 37 | signature_and_headers, 38 | ), 39 | |(fallback_url, (signature, signed_headers))| Parts { 40 | fallback_url, 41 | signature: signature.to_vec(), 42 | signed_headers: signed_headers.to_vec(), 43 | payload_body: Body::empty(), 44 | }, 45 | ), 46 | )(sxg) 47 | } 48 | 49 | // TODO: Add a timeout. 50 | pub async fn parse(mut sxg: Body) -> Result { 51 | let mut body: Vec = vec![]; 52 | while let Some(bytes) = sxg.try_next().await? { 53 | // TODO: Eliminate the duplicate processing that happens when we don't 54 | // yet have the SXG prologue fully buffered and need to await more: 55 | // - Eliminate the copy into body, e.g. by implementing all the 56 | // necessary nom input traits for Vec. 57 | // - Eliminate the duplicate parsing of the SXG prefix, by switching to 58 | // some parsing library (or a hand-rolled parser) that can trampoline 59 | // with an input stream (buffering internally if necessary). 60 | // That said, the performance gain may not be worth the complexity 61 | // cost. SXG prologues are generally <2KB. 62 | body.extend_from_slice(&bytes); 63 | match parse_impl(&body) { 64 | Ok((remaining, parts)) => { 65 | return Ok(Parts { 66 | payload_body: Body::wrap_stream(Body::from(remaining.to_vec()).chain(sxg)), 67 | ..parts 68 | }); 69 | } 70 | Err(nom::Err::Incomplete(_)) => (), 71 | Err(e) => { 72 | return Err(anyhow!(e.to_owned())); 73 | } 74 | } 75 | } 76 | Err(anyhow!("Truncated SXG")) 77 | } 78 | 79 | #[cfg(test)] 80 | mod tests { 81 | use super::*; 82 | use hyper::body::Bytes; 83 | async fn assert_parts(parts: Parts) { 84 | assert_eq!(parts.fallback_url, "https://test.example/"); 85 | assert_eq!(parts.signature, b"abc123"); 86 | assert_eq!(parts.signed_headers, b"\xA2DnameEvalueEname2Fvalue2"); 87 | 88 | let payload_body: Vec = parts.payload_body.try_collect().await.unwrap(); 89 | assert_eq!(payload_body.concat(), b"\0\0\0\0\x04\0\0\0testing"); 90 | } 91 | #[tokio::test] 92 | async fn parse_complete() { 93 | let chunk: &[u8] = b"sxg1-b3\0\0\x15https://test.example/\0\0\x06\0\0\x19abc123\xA2DnameEvalueEname2Fvalue2\0\0\0\0\x04\0\0\0testing"; 94 | let body = Body::wrap_stream(Body::from(chunk)); 95 | 96 | let parts = parse(body).await.unwrap(); 97 | assert_parts(parts).await; 98 | } 99 | #[tokio::test] 100 | async fn parse_split_prologue() { 101 | // The prologue is split across chunks. 102 | let chunk1: &[u8] = b"sxg1-b3\0\0\x15https://test.example/\0\0\x06\0\0"; 103 | let chunk2: &[u8] = b"\x19abc123\xA2DnameEvalueEname2Fvalue2\0\0\0\0\x04\0\0\0testing"; 104 | let body = Body::wrap_stream(Body::from(chunk1).chain(Body::from(chunk2))); 105 | 106 | let parts = parse(body).await.unwrap(); 107 | assert_parts(parts).await; 108 | } 109 | #[tokio::test] 110 | async fn parse_split_body() { 111 | // The body is split across chunks. 112 | let chunk1: &[u8] = b"sxg1-b3\0\0\x15https://test.example/\0\0\x06\0\0\x19abc123\xA2DnameEvalueEname2Fvalue2\0\0\0\0"; 113 | let chunk2: &[u8] = b"\x04\0\0\0testing"; 114 | let body = Body::wrap_stream(Body::from(chunk1).chain(Body::from(chunk2))); 115 | 116 | let parts = parse(body).await.unwrap(); 117 | assert_parts(parts).await; 118 | } 119 | #[tokio::test] 120 | async fn parse_truncated() { 121 | // The prologue is incomplete. 122 | let chunk: &[u8] = 123 | b"sxg1-b3\0\0\x15https://test.example/\0\0\x06\0\0\x19abc123\xA2DnameEvalu"; 124 | let body = Body::wrap_stream(Body::from(chunk)); 125 | 126 | assert_eq!( 127 | format!("{}", parse(body).await.unwrap_err()), 128 | "Truncated SXG" 129 | ); 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /distributor/src/validate_cert.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{anyhow, bail, ensure, Result}; 2 | use ciborium::value::Value; 3 | use futures::TryStreamExt; 4 | use hyper::Body; 5 | use sxg_rs::crypto::HashAlgorithm::Sha256; 6 | 7 | // Verify that the expected_integrity matches, to reduce risk of version skew 8 | // after the origin renews its certificate -- e.g. serving a new certificate 9 | // for an SXG that needs the old and not-yet-expired one -- when cert responses 10 | // from the distributor are cached by an intermediary. Origins using 11 | // content-addressed cert paths won't have this issue, but not all do that. 12 | // TODO: Add a timeout. 13 | pub async fn validate(expected_integrity: &Option<&str>, mut cert: Body) -> Result { 14 | // https://wicg.github.io/webpackage/draft-yasskin-httpbis-origin-signed-exchanges-impl.html#name-loading-a-certificate-chain 15 | let mut body: Vec = vec![]; 16 | while let Some(bytes) = cert.try_next().await? { 17 | body.extend_from_slice(&bytes); 18 | ensure!(body.len() < 10_000); 19 | } 20 | let chain: Vec = ciborium::de::from_reader(body.as_slice())?; 21 | match &chain[..] { 22 | [Value::Text(tag), Value::Map(attrs), ..] if tag == "📜⛓" => { 23 | let cert = match attrs 24 | .iter() 25 | .find(|(name, _)| matches!(name, Value::Text(name) if name == "cert")) 26 | { 27 | Some((_, Value::Bytes(cert))) => cert, 28 | _ => bail!("missing cert attr"), 29 | }; 30 | if let Some(expected_integrity) = expected_integrity { 31 | ensure!( 32 | expected_integrity 33 | == &base64::encode_config(Sha256.digest(cert), base64::URL_SAFE) 34 | .get(..12) 35 | .ok_or_else(|| anyhow!("invalid integrity"))? 36 | ); 37 | } 38 | Ok(Body::from(body)) 39 | } 40 | _ => bail!("invalid cert-chain+cbor"), 41 | } 42 | // TODO: Extract min(forall cert. cert expiry, forall ocsp. ocsp expiry), so the caller can set s-maxage based on it. 43 | } 44 | 45 | // TODO: Test. 46 | -------------------------------------------------------------------------------- /fastly_compute/.gitignore: -------------------------------------------------------------------------------- 1 | /bin 2 | /config.yaml 3 | /fastly.toml 4 | /pkg 5 | -------------------------------------------------------------------------------- /fastly_compute/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "fastly_compute" 17 | version = "0.1.0" 18 | authors = ["9083193+antiphoton@users.noreply.github.com"] 19 | edition = "2018" 20 | publish = false 21 | 22 | [profile.release] 23 | debug = false 24 | 25 | [dependencies] 26 | anyhow = "1.0.66" 27 | async-trait = "0.1.59" 28 | base64 = "0.13.1" 29 | fastly = "^0.8.9" 30 | http = "0.2.8" 31 | log = "0.4.17" 32 | log-fastly = "0.8.9" 33 | pem = "1.1.0" 34 | serde = { version = "1.0.149", features = ["derive"] } 35 | serde_yaml = "0.9.14" 36 | sxg_rs = { path = "../sxg_rs", features = ["rust_signer"] } 37 | tokio = { version = "1.23.0", features = ["rt"] } 38 | url = "2.3.1" 39 | 40 | [features] 41 | # Unsupported, but necessary to make `cargo some-cmd --all-features` happy. 42 | wasm = ["sxg_rs/wasm"] 43 | -------------------------------------------------------------------------------- /fastly_compute/src/fetcher.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use anyhow::{Error, Result}; 16 | use async_trait::async_trait; 17 | use fastly::{Request as FastlyRequest, Response as FastlyResponse}; 18 | use std::convert::TryInto; 19 | use sxg_rs::{ 20 | fetcher::Fetcher, 21 | http::{HttpRequest, HttpResponse}, 22 | }; 23 | 24 | /// A [`Fetcher`] implemented by 25 | /// [Fastly backend](https://developer.fastly.com/reference/api/services/backend/). 26 | pub struct FastlyFetcher { 27 | backend_name: &'static str, 28 | } 29 | 30 | impl FastlyFetcher { 31 | /// Constructs a new `FastlyFetcher` from the backend name. 32 | /// This function does not create the backend in Fastly; 33 | /// the Fastly backend need to be created via Fastly API 34 | /// before calling this function. 35 | pub fn new(backend_name: &'static str) -> Self { 36 | FastlyFetcher { backend_name } 37 | } 38 | } 39 | 40 | #[cfg_attr(feature = "wasm", async_trait(?Send))] 41 | #[cfg_attr(not(feature = "wasm"), async_trait)] 42 | impl Fetcher for FastlyFetcher { 43 | async fn fetch(&self, request: HttpRequest) -> Result { 44 | let request: ::http::request::Request> = request.try_into()?; 45 | let request = request.map(fastly::Body::from); 46 | let request: FastlyRequest = request.try_into()?; 47 | let response: FastlyResponse = request 48 | .send(self.backend_name) 49 | .map_err(|e| Error::new(e).context("Failed to fetch from backend."))?; 50 | 51 | let response: ::http::response::Response = response.into(); 52 | let response = response.map(|body| body.into_bytes()); 53 | response.try_into() 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /fastly_compute/src/storage.rs: -------------------------------------------------------------------------------- 1 | use anyhow::{Error, Result}; 2 | use async_trait::async_trait; 3 | use sxg_rs::storage::Storage; 4 | 5 | /// A [`Storage`] implemented by 6 | /// [Fastly dictionary](https://docs.fastly.com/en/guides/about-dictionaries). 7 | pub struct FastlyStorage { 8 | store: fastly::ConfigStore, 9 | } 10 | 11 | impl FastlyStorage { 12 | /// Constructs a new [`FastlyStorage`] from the dictionary name. 13 | /// This function does not create the dictionary in Fastly; 14 | /// the Fastly dictionary need to be created via Fastly API 15 | /// before calling this function. 16 | pub fn new(name: &str) -> Self { 17 | let store = fastly::ConfigStore::open(name); 18 | FastlyStorage { store } 19 | } 20 | } 21 | 22 | #[cfg_attr(feature = "wasm", async_trait(?Send))] 23 | #[cfg_attr(not(feature = "wasm"), async_trait)] 24 | impl Storage for FastlyStorage { 25 | async fn read(&self, k: &str) -> Result> { 26 | Ok(self.store.get(k)) 27 | } 28 | async fn write(&self, _k: &str, _v: &str) -> Result<()> { 29 | Err(Error::msg( 30 | "Writing to edge dictionary is not allowed by worker.", 31 | )) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /http_server/.gitignore: -------------------------------------------------------------------------------- 1 | /config.yaml 2 | -------------------------------------------------------------------------------- /http_server/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "http_server" 17 | version = "0.1.0" 18 | authors = ["webpackaging-announce@googlegroups.com"] 19 | edition = "2018" 20 | 21 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 22 | 23 | [dependencies] 24 | anyhow = "1.0.66" 25 | async-trait = "0.1.59" 26 | clap = { version = "3.2.23", features = ["derive"] } 27 | fs2 = "0.4.3" 28 | futures = "0.3.25" 29 | http = "0.2.8" 30 | hyper-reverse-proxy = { git = "https://github.com/felipenoris/hyper-reverse-proxy", rev = "96a398de8522fac07a5e15bd0699f6cd7fa84bce" } 31 | hyper-rustls = "0.23.2" 32 | hyper-tls = "0.5.0" 33 | hyper-trust-dns = { version = "0.5.0", default-features = false, features = ["rustls-webpki", "rustls-http1", "rustls-tls-12"] } 34 | hyper = { version = "0.14.23", features = ["http1", "http2", "server", "stream", "tcp"] } 35 | lazy_static = "1.4.0" 36 | lru = "0.8.1" 37 | rand = "0.8.5" 38 | serde_yaml = "0.9.14" 39 | # TODO: Determine if I can remove strip_id_headers because it's default. 40 | sxg_rs = { path = "../sxg_rs", features = ["strip_id_headers", "rust_signer"] } 41 | tokio = { version = "1.23.0", features = ["rt-multi-thread", "macros", "sync", "time"] } 42 | tools = { path = "../tools" } 43 | url = "2.3.1" 44 | 45 | [dev-dependencies] 46 | assert_matches = "1.5.0" 47 | -------------------------------------------------------------------------------- /input.example.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Please replace ALL-CAP values with yours. 16 | --- 17 | sxg_worker: 18 | html_host: YOUR_DOMAIN # example.com 19 | cert_url_dirname: ".well-known/sxg-certs" 20 | forward_request_headers: 21 | - user-agent 22 | - cf-ipcountry 23 | reserved_path: ".sxg" 24 | strip_request_headers: [] 25 | strip_response_headers: 26 | - set-cookie 27 | validity_url_dirname: ".well-known/sxg-validity" 28 | certificates: 29 | !pre_issued 30 | cert_file: credentials/cert.pem 31 | issuer_file: credentials/issuer.pem 32 | # # If this section is uncommented, an ACME account will be created. 33 | # !create_acme_account 34 | # server_url: https://dv-sxg.acme-v02.api.pki.goog/directory 35 | # contact_email: YOUR_EMAIL 36 | # # Read and agree the terms of service before uncommenting next line. 37 | # # agreed_terms_of_service: https://pki.goog/GTS-SA.pdf 38 | # sxg_cert_request_file: credentials/cert.csr 39 | # # To find your eab key, please follow 40 | # # https://cloud.google.com/public-certificate-authority/docs/quickstart#request-key-hmac 41 | # eab: 42 | # base64_mac_key: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXQ 43 | # key_id: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 44 | cloudflare: 45 | account_id: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 46 | zone_id: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 47 | # You can define URL patterns for the SXG worker. See Cloudflare's doc at 48 | # https://developers.cloudflare.com/workers/platform/routing/routes/ 49 | routes: 50 | - https://YOUR_DOMAIN/* 51 | worker_name: sxg 52 | deploy_on_workers_dev_only: false 53 | fastly: 54 | service_name: sxg 55 | sxg_private_key_base64: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= 56 | -------------------------------------------------------------------------------- /playground/.eslintignore: -------------------------------------------------------------------------------- 1 | /dist 2 | /node_modules 3 | /rollup.config.js 4 | -------------------------------------------------------------------------------- /playground/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./node_modules/gts/" 3 | } 4 | -------------------------------------------------------------------------------- /playground/.gitignore: -------------------------------------------------------------------------------- 1 | /dist 2 | /node_modules 3 | -------------------------------------------------------------------------------- /playground/.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | ...require('gts/.prettierrc.json') 3 | } 4 | -------------------------------------------------------------------------------- /playground/README.md: -------------------------------------------------------------------------------- 1 | 16 | 17 | # sxg-rs/playground 18 | 19 | A playground to locally preview Signed Exchanges without needing a certificate. 20 | 21 | ## Build 22 | 23 | 1. Compile sxg-rs to WebAssembly 24 | 25 | ```bash 26 | cd ../cloudflare_worker && wrangler build && cd ../playground 27 | ``` 28 | 29 | 1. Compile playground 30 | 31 | ```bash 32 | npm install 33 | npm run build 34 | ``` 35 | 36 | ## Run 37 | 38 | ```bash 39 | node dist/index.js --single-url https://example.com/ --emulate-network Fast\ 3G --repeat-time 3 40 | ``` 41 | The output will be like below. 42 | ``` 43 | Measuring non-SXG LCP of https://example.com/ 44 | LCP 1 / 3: 720 45 | LCP 2 / 3: 674 46 | LCP 3 / 3: 636 47 | Measuring SXG LCP of https://example.com/ 48 | LCP 1 / 3: 78 49 | LCP 2 / 3: 95 50 | LCP 3 / 3: 69 51 | SXG changes LCP from 677 ± 24 to 80 ± 8 52 | ``` 53 | -------------------------------------------------------------------------------- /playground/build.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | node_modules/.bin/tsc --noEmit 4 | 5 | node_modules/.bin/esbuild src/index.ts --bundle \ 6 | --external:jsdom \ 7 | --external:puppeteer \ 8 | --platform=node --outfile=dist/tmp.js 9 | 10 | cat ../cloudflare_worker/pkg/cloudflare_worker.js dist/tmp.js > dist/index.js 11 | -------------------------------------------------------------------------------- /playground/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "name": "sxg-playground", 4 | "version": "0.1.0", 5 | "main": "dist/index.js", 6 | "scripts": { 7 | "build": "./build.sh", 8 | "lint": "gts lint" 9 | }, 10 | "author": "", 11 | "devDependencies": { 12 | "@types/dompurify": "2.4.0", 13 | "@types/jsdom": "20.0.1", 14 | "@types/node": "17.0.18", 15 | "@types/node-fetch": "2.6.2", 16 | "esbuild": "0.16.14", 17 | "gts": "3.1.1", 18 | "tslib": "2.4.1", 19 | "typescript": "4.9.4" 20 | }, 21 | "dependencies": { 22 | "commander": "9.4.1", 23 | "dompurify": "2.4.2", 24 | "fastify": "4.11.0", 25 | "jsdom": "20.0.3", 26 | "node-fetch": "3.3.0", 27 | "puppeteer": "17.1.3" 28 | }, 29 | "engines": { 30 | "node": ">=16.0.0" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /playground/src/client/emulationOptions.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2022 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export const NOT_EMULATED = 'none'; 18 | 19 | export interface EmulationOptions { 20 | device: string; 21 | networkCondition: 'Fast 3G' | 'Slow 3G' | typeof NOT_EMULATED; 22 | } 23 | -------------------------------------------------------------------------------- /playground/src/client/evaluated.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2022 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export function clickSearchResultLink() { 18 | const p = document.createElement('p'); 19 | p.textContent = 'Puppeteer clicks the link, starting to navigate.'; 20 | document.body.appendChild(p); 21 | document.getElementById('search-result-link')!.click(); 22 | } 23 | 24 | declare global { 25 | interface Window { 26 | lcpResult: number | null; 27 | } 28 | } 29 | 30 | export function setupObserver() { 31 | window.lcpResult = null; 32 | // TODO: Check whether we can use npm package `web-vitals` here. 33 | // The `web-vitals` package not only adds the performance observer, but also 34 | // [handles](https://github.com/GoogleChrome/web-vitals/blob/ed70ed4b56d3f0573da7ca8ec324e630a04beaf2/src/getLCP.ts#L64-L66) 35 | // some user input DOM event. 36 | const observer = new PerformanceObserver(entryList => { 37 | const entries = entryList.getEntries(); 38 | const lastEntry = entries[entries.length - 1]!; 39 | if (lastEntry.entryType === 'largest-contentful-paint') { 40 | window.lcpResult = lastEntry.startTime; 41 | } 42 | }); 43 | observer.observe({type: 'largest-contentful-paint', buffered: true}); 44 | } 45 | 46 | export function getObserverResult() { 47 | return window.lcpResult; 48 | } 49 | -------------------------------------------------------------------------------- /playground/src/client/statistics.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2022 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export interface EstimatedValue { 18 | mean: number; 19 | uncertainty: number; 20 | } 21 | 22 | function sum(values: number[]): number { 23 | return values.reduce((r, x) => r + x, 0); 24 | } 25 | 26 | // Given a set of repeated measurements, calculates mean value and error bar. 27 | export function estimateMeasurements(values: number[]): EstimatedValue { 28 | const mean = sum(values) / values.length; 29 | const stddev = Math.sqrt( 30 | sum(values.map(x => (x - mean) ** 2)) / values.length 31 | ); 32 | return { 33 | mean, 34 | uncertainty: stddev / Math.sqrt(values.length - 1), 35 | }; 36 | } 37 | 38 | export function formatEstimatedValue(x: EstimatedValue): string { 39 | return `${x.mean.toFixed(0)} ± ${x.uncertainty.toFixed(0)}`; 40 | } 41 | -------------------------------------------------------------------------------- /playground/src/client/templates.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2022 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import createDomPurify from 'dompurify'; 18 | import {JSDOM} from 'jsdom'; 19 | 20 | const DomPurify = createDomPurify(new JSDOM('').window as unknown as Window); 21 | 22 | function createLinkFromUntrustedString(href: string, text: string) { 23 | return DomPurify.sanitize( 24 | `${text}`, 25 | {ALLOWED_TAGS: ['a']} 26 | ); 27 | } 28 | 29 | export function createSearchResultPageWithoutSxg( 30 | targetInnerUrl: string 31 | ): string { 32 | return ` 33 |

This is a Search Result Page without using prefetch.

34 | ${createLinkFromUntrustedString(targetInnerUrl, targetInnerUrl)} 35 |

Click the link to load their page.

36 | `; 37 | } 38 | 39 | export function createSearchResultPage( 40 | targetInnerUrl: string, 41 | sxgOuterUrl: string 42 | ): string { 43 | return ` 44 |

This is a Search Result Page using Signed Exchanges

45 | ${createLinkFromUntrustedString(sxgOuterUrl, targetInnerUrl)} 46 | 47 |

48 | Resources are being prefetched. 49 | Open browser's developer tool to check loading state. 50 |

51 | `; 52 | } 53 | -------------------------------------------------------------------------------- /playground/src/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2022 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import fs from 'fs'; 18 | import {program, Option} from 'commander'; 19 | 20 | import {IsolationMode, runBatchClient, runInteractiveClient} from './client/'; 21 | import {NOT_EMULATED} from './client/emulationOptions'; 22 | import {createSelfSignedCredentials} from './server/credentials'; 23 | import {spawnSxgServer} from './server/'; 24 | 25 | async function main() { 26 | program 27 | .addOption( 28 | new Option( 29 | '--crawler-user-agent ', 30 | 'The user-agent request header to be sent to the website server' 31 | // The defalt value is from https://developers.google.com/search/docs/advanced/crawling/overview-google-crawlers#googlebot-smartphone 32 | ).default( 33 | 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' 34 | ) 35 | ) 36 | .addOption( 37 | new Option( 38 | '--emulate-device ', 39 | 'The device that puppeteer emulates' 40 | ) 41 | .choices(['Pixel 5', 'iPhone XR', NOT_EMULATED]) 42 | .default('Pixel 5') 43 | ) 44 | .addOption( 45 | new Option( 46 | '--emulate-network ', 47 | 'The network condition that puppeteer emulates' 48 | ) 49 | .choices(['Slow 3G', 'Fast 3G', NOT_EMULATED]) 50 | .default(NOT_EMULATED) 51 | ) 52 | .addOption( 53 | new Option('--single-url ', 'A single URL to be measured').conflicts( 54 | 'urlList' 55 | ) 56 | ) 57 | .addOption( 58 | new Option( 59 | '--url-list ', 60 | 'A JSON file containing many URLs to be measured' 61 | ).conflicts(['inspect', 'singleUrl']) 62 | ) 63 | .addOption( 64 | new Option( 65 | '--sxg-config ', 66 | 'A YAML file containing the sxg-rs config' 67 | ) 68 | ) 69 | .addOption( 70 | new Option( 71 | '--inspect', 72 | 'open a Chrome window and use ChromeDevTools to preview SXG' 73 | ) 74 | .default(false) 75 | .conflicts(['repeatTime', 'urlList']) 76 | ) 77 | .addOption( 78 | new Option( 79 | '--isolateBrowserContext', 80 | 'create a new browser context when testing each URL' 81 | ).default(false) 82 | ) 83 | .addOption( 84 | new Option('--repeat-time ', 'measure LCP multiple times') 85 | .argParser(x => parseInt(x)) 86 | .default(1) 87 | .conflicts('inspect') 88 | ); 89 | program.parse(); 90 | const opts = program.opts() as { 91 | crawlerUserAgent: string; 92 | emulateDevice: string; 93 | emulateNetwork: 'Fast 3G' | 'Slow 3G' | typeof NOT_EMULATED; 94 | inspect: boolean; 95 | isolateBrowserContext: boolean; 96 | repeatTime: number; 97 | singleUrl?: string; 98 | sxgConfig?: string; 99 | urlList?: string; 100 | }; 101 | let urlList = []; 102 | if (opts.singleUrl) { 103 | urlList.push(opts.singleUrl); 104 | } else if (opts.urlList) { 105 | urlList = JSON.parse(fs.readFileSync(opts.urlList, 'utf8')); 106 | } else { 107 | throw new Error('Please specify either --single-url or --url-list'); 108 | } 109 | const {certificatePem, privateKeyJwk, privateKeyPem, publicKeyHash} = 110 | await createSelfSignedCredentials('example.com'); 111 | const stopSxgServer = await spawnSxgServer({ 112 | certificatePem, 113 | crawlerUserAgent: opts.crawlerUserAgent, 114 | privateKeyJwk, 115 | privateKeyPem, 116 | sxgConfig: opts.sxgConfig && fs.readFileSync(opts.sxgConfig, 'utf8'), 117 | }); 118 | if (opts.inspect) { 119 | await runInteractiveClient({ 120 | url: urlList[0], 121 | certificateSpki: publicKeyHash, 122 | emulationOptions: { 123 | device: opts.emulateDevice, 124 | networkCondition: opts.emulateNetwork, 125 | }, 126 | isolationMode: opts.isolateBrowserContext 127 | ? IsolationMode.IncognitoBrowserContext 128 | : IsolationMode.ClearBrowserCache, 129 | }); 130 | } else { 131 | await runBatchClient({ 132 | urlList, 133 | certificateSpki: publicKeyHash, 134 | emulationOptions: { 135 | device: opts.emulateDevice, 136 | networkCondition: opts.emulateNetwork, 137 | }, 138 | isolationMode: opts.isolateBrowserContext 139 | ? IsolationMode.IncognitoBrowserContext 140 | : IsolationMode.ClearBrowserCache, 141 | repeatTime: opts.repeatTime, 142 | }); 143 | } 144 | await stopSxgServer(); 145 | } 146 | 147 | main().catch(e => console.error(e)); 148 | -------------------------------------------------------------------------------- /playground/src/schema.ts: -------------------------------------------------------------------------------- 1 | export type CreateSignedExchangeRequest = { 2 | innerUrl: string; 3 | }; 4 | 5 | export type CreateSignedExchangeResponse = 6 | | [ 7 | 'Ok', 8 | { 9 | outerUrl: string; 10 | info: { 11 | bodySize: number; 12 | subresourceUrls: string[]; 13 | }; 14 | } 15 | ] 16 | | [ 17 | 'Err', 18 | { 19 | message: string; 20 | } 21 | ]; 22 | -------------------------------------------------------------------------------- /playground/src/server/cache.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2022 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import type {WasmResponse} from './wasmFunctions'; 18 | 19 | // A cache of URL and HTTP response, and a function to get all URLs visited by 20 | // this cache. 21 | interface RecordedCache { 22 | get: (url: string) => Promise; 23 | put: (url: string, response: WasmResponse) => Promise; 24 | visitedUrls: () => Set; 25 | } 26 | 27 | // An in-memory cache to store URLs and their HTTP response. 28 | export class SubresourceCache { 29 | #data = new Map(); 30 | constructor() {} 31 | async #get(url: string): Promise { 32 | const x = this.#data.get(url); 33 | if (x) { 34 | return x; 35 | } else { 36 | return { 37 | body: [], 38 | headers: [], 39 | status: 404, 40 | }; 41 | } 42 | } 43 | async #put(url: string, response: WasmResponse): Promise { 44 | this.#data.set(url, response); 45 | } 46 | // Creates a recorder. All recorders share the same cached HTTP response, but 47 | // each recorder tracks their own list of which URLs have been visited. 48 | createRecorder(): RecordedCache { 49 | const urls = new Set(); 50 | return { 51 | get: async (url: string) => { 52 | urls.add(url); 53 | return await this.#get(url); 54 | }, 55 | put: async (url: string, response: WasmResponse) => { 56 | urls.add(url); 57 | return await this.#put(url, response); 58 | }, 59 | visitedUrls: () => { 60 | return urls; 61 | }, 62 | }; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /playground/src/server/credentials.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2022 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import childProcess from 'child_process'; 18 | import crypto from 'crypto'; 19 | import {promises as fs} from 'fs'; 20 | import os from 'os'; 21 | import path from 'path'; 22 | import util from 'util'; 23 | 24 | const execFile = util.promisify(childProcess.execFile); 25 | 26 | export async function createSelfSignedCredentials(domain: string) { 27 | const certDir = await fs.mkdtemp(path.join(os.tmpdir(), 'sxg-cert-')); 28 | const {stdout: privateKeyPem} = await execFile('openssl', [ 29 | 'ecparam', 30 | '-outform', 31 | 'pem', 32 | '-name', 33 | 'prime256v1', 34 | '-genkey', 35 | ]); 36 | const privateKeyFile = path.join(certDir, 'privkey.pem'); 37 | await fs.writeFile(privateKeyFile, privateKeyPem); 38 | const privateKeyJwk = crypto 39 | .createPrivateKey(privateKeyPem) 40 | .export({format: 'jwk'}); 41 | const certificateRequestFile = path.join(certDir, 'csr.pem'); 42 | const {stdout: certificateRequestPem} = await execFile('openssl', [ 43 | 'req', 44 | '-new', 45 | '-sha256', 46 | '-key', 47 | privateKeyFile, 48 | '-subj', 49 | `/CN=${domain}`, 50 | ]); 51 | await fs.writeFile(certificateRequestFile, certificateRequestPem); 52 | const certificateExtensionFile = path.join(certDir, 'ext.txt'); 53 | await fs.writeFile( 54 | certificateExtensionFile, 55 | `1.3.6.1.4.1.11129.2.1.22 = ASN1:NULL\nsubjectAltName=DNS:${domain}` 56 | ); 57 | const certificateFile = path.join(certDir, 'certificate.pem'); 58 | const {stdout: certificatePem} = await execFile('openssl', [ 59 | 'x509', 60 | '-req', 61 | '-days', 62 | '90', 63 | '-in', 64 | certificateRequestFile, 65 | '-signkey', 66 | privateKeyFile, 67 | '-extfile', 68 | certificateExtensionFile, 69 | ]); 70 | await fs.writeFile(certificateFile, certificatePem); 71 | const {stdout: publicKeyPem} = await execFile('openssl', [ 72 | 'x509', 73 | '-pubkey', 74 | '-noout', 75 | '-in', 76 | certificateFile, 77 | ]); 78 | const publicKeyDer = crypto 79 | .createPublicKey(publicKeyPem) 80 | .export({format: 'der', type: 'spki'}); 81 | const publicKeyHash = crypto 82 | .createHash('sha256') 83 | .update(publicKeyDer) 84 | .digest('base64'); 85 | return { 86 | certificatePem, 87 | privateKeyPem, 88 | privateKeyJwk, 89 | publicKeyHash, 90 | }; 91 | } 92 | -------------------------------------------------------------------------------- /playground/src/server/fetcher.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2022 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import {WasmRequest, WasmResponse} from './wasmFunctions'; 18 | import {RequestInit, Response} from 'node-fetch'; 19 | import fetch from 'node-fetch'; 20 | 21 | export type Fetcher = (request: WasmRequest) => Promise; 22 | 23 | async function wasmFromResponse(response: Response): Promise { 24 | return { 25 | body: Array.from(new Uint8Array(await response.arrayBuffer())), 26 | headers: Array.from(response.headers), 27 | status: response.status, 28 | }; 29 | } 30 | 31 | export async function fetcher(request: WasmRequest) { 32 | const PAYLOAD_SIZE_LIMIT = 8000000; 33 | 34 | const requestInit: RequestInit = { 35 | headers: request.headers, 36 | method: request.method, 37 | }; 38 | if (request.body.length > 0) { 39 | requestInit.body = Buffer.from(request.body); 40 | } 41 | const response = await fetch(request.url, requestInit); 42 | const body = await response.arrayBuffer(); 43 | if (body.byteLength > PAYLOAD_SIZE_LIMIT) { 44 | throw `The size of payload exceeds the limit ${PAYLOAD_SIZE_LIMIT}`; 45 | } 46 | 47 | return await wasmFromResponse( 48 | new Response(Buffer.from(body), { 49 | headers: response.headers, 50 | status: response.status, 51 | }) 52 | ); 53 | } 54 | -------------------------------------------------------------------------------- /playground/src/server/signer.ts: -------------------------------------------------------------------------------- 1 | ../../../typescript_utilities/src/signer.ts -------------------------------------------------------------------------------- /playground/src/server/wasmFunctions.ts: -------------------------------------------------------------------------------- 1 | ../../../typescript_utilities/src/wasmFunctions.ts -------------------------------------------------------------------------------- /playground/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2015", 4 | "module": "ESNext", 5 | "lib": ["DOM", "DOM.Iterable", "ESNext"], 6 | "declaration": true, 7 | "outDir": "./dist", 8 | "strict": true, 9 | "noImplicitAny": true, 10 | "strictNullChecks": true, 11 | "strictFunctionTypes": true, 12 | "strictBindCallApply": true, 13 | "strictPropertyInitialization": true, 14 | "noImplicitThis": true, 15 | "alwaysStrict": true, 16 | "noUnusedLocals": true, 17 | "noUnusedParameters": true, 18 | "noImplicitReturns": true, 19 | "noFallthroughCasesInSwitch": true, 20 | "noUncheckedIndexedAccess": true, 21 | "noImplicitOverride": true, 22 | "noPropertyAccessFromIndexSignature": true, 23 | "moduleResolution": "node", 24 | "types": [ 25 | "@types/node", 26 | ], 27 | "esModuleInterop": true, 28 | "skipLibCheck": true, 29 | "forceConsistentCasingInFileNames": true 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "config:base", 4 | "schedule:weekly" 5 | ], 6 | "postUpdateOptions": [ 7 | "npmDedupe" 8 | ], 9 | "packageRules": [ 10 | { 11 | "matchManagers": ["cargo"], 12 | "groupName": "Rust dependencies" 13 | }, 14 | { 15 | "matchManagers": ["npm"], 16 | "groupName": "TypeScript dependencies" 17 | }, 18 | { 19 | "matchManagers": ["github-actions"], 20 | "groupName": "GitHub Actions" 21 | }, 22 | { 23 | "matchPackageNames": ["wasm-bindgen"], 24 | "allowedVersions": "!/^0\\.2\\.79$/" 25 | } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /rustfmt.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | newline_style = "Unix" 16 | 17 | -------------------------------------------------------------------------------- /sxg_rs/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "sxg_rs" 17 | version = "0.1.0" 18 | authors = ["9083193+antiphoton@users.noreply.github.com"] 19 | edition = "2018" 20 | 21 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 22 | 23 | [features] 24 | default = ["strip_id_headers"] 25 | rust_signer = ["p256"] 26 | srcset = [] 27 | strip_id_headers = [] 28 | wasm = [] 29 | 30 | [lib] 31 | crate-type = ["cdylib", "rlib"] 32 | 33 | [dependencies] 34 | anyhow = "1.0.66" 35 | async-trait = "0.1.59" 36 | base64 = "0.13.1" 37 | chrono = { version = "0.4.23", features = ["serde"] } 38 | der-parser = { version = "8.1.0", features = ["bigint", "serialize"] } 39 | futures = { version = "0.3.25" } 40 | getrandom = { version = "0.2.8", features = ["js"] } 41 | http = "0.2.8" 42 | js-sys = "0.3.60" 43 | lol_html = "0.3.1" 44 | nom = { version = "7.1.1", features = ["alloc"] } 45 | once_cell = "1.16.0" 46 | pem = "1.1.0" 47 | p256 = { version = "0.11.1", features = ["ecdsa"], optional = true } 48 | serde = { version = "1.0.149", features = ["derive"] } 49 | serde-wasm-bindgen = "0.4.5" 50 | serde_json = "1.0.89" 51 | serde_yaml = "0.9.14" 52 | sha1 = "0.10.5" 53 | sha2 = "0.10.6" 54 | tokio = { version = "1.23.0", features = ["macros", "parking_lot", "sync", "time"] } 55 | url = "2.3.1" 56 | wasm-bindgen = "0.2.83" 57 | wasm-bindgen-futures = "0.4.33" 58 | web-sys = { version = "0.3.60", features = ["console"] } 59 | x509-parser = "0.14.0" 60 | 61 | [dev-dependencies] 62 | tokio-test = "0.4.2" 63 | -------------------------------------------------------------------------------- /sxg_rs/README.md: -------------------------------------------------------------------------------- 1 | # sxg-rs 2 | 3 | A Rust library that generate [signed 4 | exchanges](https://web.dev/signed-exchanges/) for given HTTP request/response 5 | pairs. For example usages, see [`cloudflare_worker`](../cloudflare_worker) and 6 | [`fastly_compute`](../fastly_compute). -------------------------------------------------------------------------------- /sxg_rs/src/acme/client.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //! An ACME client handles authentication with the ACME server. 16 | 17 | use super::directory::Directory; 18 | use crate::crypto::EcPublicKey; 19 | use crate::fetcher::Fetcher; 20 | use crate::http::{HttpRequest, HttpResponse, Method}; 21 | use crate::signature::Signer; 22 | use anyhow::{anyhow, Error, Result}; 23 | use serde::{Deserialize, Serialize}; 24 | 25 | pub struct Client<'a> { 26 | pub directory: &'a Directory, 27 | pub auth_method: AuthMethod, 28 | nonce: Option, 29 | } 30 | 31 | pub enum AuthMethod { 32 | JsonWebKey(EcPublicKey), 33 | KeyId(String), 34 | } 35 | 36 | impl<'a> Client<'a> { 37 | pub fn new(directory: &'a Directory, auth_method: AuthMethod, nonce: Option) -> Self { 38 | Client { 39 | directory, 40 | auth_method, 41 | nonce, 42 | } 43 | } 44 | /// Fetches a server resource at given URL using 45 | /// [POST-as-GET](https://datatracker.ietf.org/doc/html/rfc8555#section-6.3) 46 | /// method. `POST-as-GET` is a `POST` request with no request payload. This 47 | /// function is useful because an ACME server always returns error code 48 | /// `405` for `GET` requests, which don't contain request body for 49 | /// authentication. 50 | pub async fn post_as_get( 51 | &mut self, 52 | url: String, 53 | fetcher: &dyn Fetcher, 54 | acme_signer: &dyn Signer, 55 | ) -> Result { 56 | let payload: Option<()> = None; 57 | self.post_impl(url, payload, fetcher, acme_signer).await 58 | } 59 | /// Fetches a server resource at given URL using `POST` method with a 60 | /// request payload. 61 | pub async fn post_with_payload( 62 | &mut self, 63 | url: String, 64 | payload: P, 65 | fetcher: &dyn Fetcher, 66 | acme_signer: &dyn Signer, 67 | ) -> Result { 68 | self.post_impl(url, Some(payload), fetcher, acme_signer) 69 | .await 70 | } 71 | /// Encapsulates the payload in JWS for authentication, connects to the ACME 72 | /// server, saves `nonce` for next request, and returns the server response. 73 | async fn post_impl( 74 | &mut self, 75 | url: String, 76 | payload: Option

, 77 | fetcher: &dyn Fetcher, 78 | acme_signer: &dyn Signer, 79 | ) -> Result { 80 | let nonce = self.take_nonce(fetcher).await?; 81 | let (jwk, key_id) = match &self.auth_method { 82 | AuthMethod::JsonWebKey(public_key) => (Some(public_key), None), 83 | AuthMethod::KeyId(key_id) => (None, Some(key_id.as_str())), 84 | }; 85 | let request_body = 86 | super::jws::create_acme_request_body(jwk, key_id, nonce, &url, payload, acme_signer) 87 | .await?; 88 | let request = HttpRequest { 89 | url: url.clone(), 90 | method: Method::Post, 91 | headers: vec![( 92 | "content-type".to_string(), 93 | "application/jose+json".to_string(), 94 | )], 95 | body: request_body, 96 | }; 97 | let response = fetcher.fetch(request).await?; 98 | if let Ok(nonce) = find_header(&response, "Replay-Nonce") { 99 | let _ = self.nonce.insert(nonce); 100 | } 101 | Ok(response) 102 | } 103 | /// If `self.nonce` exists, deletes and returns it; 104 | /// if there is no `nonce`, fetches a new one and returns it. 105 | async fn take_nonce(&mut self, fetcher: &dyn Fetcher) -> Result { 106 | match self.nonce.take() { 107 | Some(nonce) => Ok(nonce), 108 | None => self.fetch_new_nonce(fetcher).await, 109 | } 110 | } 111 | /// Fetches a new `nonce` from the server. 112 | async fn fetch_new_nonce(&self, fetcher: &dyn Fetcher) -> Result { 113 | let request = HttpRequest { 114 | method: Method::Get, 115 | headers: vec![], 116 | url: self.directory.new_nonce.clone(), 117 | body: vec![], 118 | }; 119 | let response = fetcher.fetch(request).await?; 120 | find_header(&response, "Replay-Nonce") 121 | } 122 | } 123 | 124 | pub fn find_header(response: &HttpResponse, header_name: &str) -> Result { 125 | response 126 | .headers 127 | .iter() 128 | .find_map(|(name, value)| { 129 | if name.eq_ignore_ascii_case(header_name) { 130 | Some(value.to_string()) 131 | } else { 132 | None 133 | } 134 | }) 135 | .ok_or_else(|| anyhow!("The response header does not contain {}", header_name)) 136 | } 137 | 138 | /// Parses response body as JSON of type `T`. 139 | pub fn parse_response_body<'a, T: Deserialize<'a>>(response: &'a HttpResponse) -> Result { 140 | serde_json::from_slice(&response.body).map_err(|e| { 141 | let msg = if let Ok(s) = String::from_utf8(response.body.clone()) { 142 | format!("Body contains text: {}", s) 143 | } else { 144 | format!("Body contains bytes: {:?}", response.body) 145 | }; 146 | Error::new(e) 147 | .context(format!( 148 | "Failed to parse response body into type {}", 149 | std::any::type_name::() 150 | )) 151 | .context(msg) 152 | }) 153 | } 154 | -------------------------------------------------------------------------------- /sxg_rs/src/acme/eab.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //! This module implements External Account Binding (EAB), which is defined in 16 | //! [RFC-8555](https://datatracker.ietf.org/doc/html/rfc8555#section-7.3.4). 17 | 18 | use super::jws::{Algorithm, JsonWebSignature}; 19 | use crate::crypto::EcPublicKey; 20 | use crate::signature::Signer; 21 | use anyhow::Result; 22 | use serde::Serialize; 23 | 24 | /// The protected header which is used for External Account Binding. 25 | #[derive(Serialize)] 26 | struct EabProtectedHeader<'a> { 27 | alg: Algorithm, 28 | /// Key identifier from Certificate Authority. 29 | kid: &'a str, 30 | /// URL of the request. This is usually the new-account URL of ACME server, 31 | /// because only new-account requests need EAB. 32 | url: &'a str, 33 | } 34 | 35 | pub async fn create_external_account_binding( 36 | alg: Algorithm, 37 | kid: &str, 38 | url: &str, 39 | public_key: &EcPublicKey, 40 | hmac_signer: &dyn Signer, 41 | ) -> Result { 42 | let protected_header = EabProtectedHeader { alg, kid, url }; 43 | JsonWebSignature::new( 44 | protected_header, 45 | /*payload=*/ Some(public_key), 46 | hmac_signer, 47 | ) 48 | .await 49 | } 50 | -------------------------------------------------------------------------------- /sxg_rs/src/cbor.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // https://tools.ietf.org/html/rfc7049 16 | 17 | use std::collections::BTreeMap; 18 | 19 | pub enum DataItem<'a> { 20 | #[allow(dead_code)] 21 | UnsignedInteger(u64), 22 | ByteString(&'a [u8]), 23 | TextString(&'a str), 24 | Array(Vec>), 25 | Map(Vec<(DataItem<'a>, DataItem<'a>)>), 26 | } 27 | 28 | impl<'a> DataItem<'a> { 29 | pub fn serialize(&self) -> Vec { 30 | let mut result = Vec::new(); 31 | self.append_binary_to(&mut result); 32 | result 33 | } 34 | fn append_binary_to(&self, output: &mut Vec) { 35 | use DataItem::*; 36 | match self { 37 | UnsignedInteger(x) => append_integer(output, 0, *x), 38 | ByteString(bytes) => { 39 | append_integer(output, 2, bytes.len() as u64); 40 | output.extend_from_slice(bytes); 41 | } 42 | TextString(text) => { 43 | append_integer(output, 3, text.len() as u64); 44 | output.extend_from_slice(text.as_bytes()); 45 | } 46 | Array(items) => { 47 | append_integer(output, 4, items.len() as u64); 48 | for item in items { 49 | item.append_binary_to(output); 50 | } 51 | } 52 | Map(fields) => { 53 | let mut map = BTreeMap::, Vec>::new(); 54 | for (key, value) in fields { 55 | map.insert(key.serialize(), value.serialize()); 56 | } 57 | append_integer(output, 5, map.len() as u64); 58 | for (mut key, mut value) in map.into_iter() { 59 | output.append(&mut key); 60 | output.append(&mut value); 61 | } 62 | } 63 | } 64 | } 65 | } 66 | 67 | fn append_integer(output: &mut Vec, major_type: u8, data: u64) { 68 | let major_type = major_type << 5; 69 | match data { 70 | 0..=23 => { 71 | output.push(major_type | (data as u8)); 72 | } 73 | 24..=0xff => { 74 | output.push(major_type | 24); 75 | output.push(data as u8); 76 | } 77 | 0x100..=0xffff => { 78 | output.push(major_type | 25); 79 | output.extend_from_slice(&(data as u16).to_be_bytes()); 80 | } 81 | 0x10000..=0xffffffff => { 82 | output.push(major_type | 26); 83 | output.extend_from_slice(&(data as u32).to_be_bytes()); 84 | } 85 | 0x100000000..=0xffffffffffffffff => { 86 | output.push(major_type | 27); 87 | output.extend_from_slice(&data.to_be_bytes()); 88 | } 89 | }; 90 | } 91 | 92 | #[cfg(test)] 93 | mod tests { 94 | use super::*; 95 | fn from_hex(input: &str) -> Vec { 96 | (0..input.len()) 97 | .step_by(2) 98 | .map(|i| u8::from_str_radix(&input[i..i + 2], 16).unwrap()) 99 | .collect() 100 | } 101 | #[test] 102 | fn it_works() { 103 | assert_eq!(DataItem::UnsignedInteger(0).serialize(), from_hex("00"),); 104 | assert_eq!(DataItem::UnsignedInteger(23).serialize(), from_hex("17"),); 105 | assert_eq!(DataItem::UnsignedInteger(24).serialize(), from_hex("1818"),); 106 | assert_eq!(DataItem::UnsignedInteger(100).serialize(), from_hex("1864"),); 107 | assert_eq!( 108 | DataItem::UnsignedInteger(1000).serialize(), 109 | from_hex("1903e8"), 110 | ); 111 | assert_eq!( 112 | DataItem::UnsignedInteger(1000000).serialize(), 113 | from_hex("1a000f4240"), 114 | ); 115 | assert_eq!( 116 | DataItem::UnsignedInteger(1000000000000).serialize(), 117 | from_hex("1b000000e8d4a51000"), 118 | ); 119 | assert_eq!( 120 | DataItem::UnsignedInteger(18446744073709551615).serialize(), 121 | from_hex("1bffffffffffffffff"), 122 | ); 123 | assert_eq!( 124 | DataItem::ByteString(&[1, 2, 3, 4]).serialize(), 125 | from_hex("4401020304"), 126 | ); 127 | assert_eq!( 128 | DataItem::TextString("IETF").serialize(), 129 | from_hex("6449455446"), 130 | ); 131 | assert_eq!(DataItem::Map(vec![]).serialize(), from_hex("a0"),); 132 | assert_eq!( 133 | DataItem::Map(vec![ 134 | (DataItem::UnsignedInteger(1), DataItem::UnsignedInteger(2)), 135 | (DataItem::UnsignedInteger(3), DataItem::UnsignedInteger(4)), 136 | ]) 137 | .serialize(), 138 | from_hex("a201020304"), 139 | ); 140 | } 141 | #[test] 142 | fn map_keys_are_sorted() { 143 | use DataItem::*; 144 | assert_eq!( 145 | Map(vec![ 146 | (UnsignedInteger(2), UnsignedInteger(5)), 147 | (UnsignedInteger(1), UnsignedInteger(6)), 148 | ]) 149 | .serialize(), 150 | from_hex("a201060205"), 151 | ); 152 | // Although "AA" is lexicographically less than "B", the CBOR format 153 | // of the string is prefixed with the string length, hence "B" is less 154 | // than "AA". 155 | assert_eq!( 156 | Map(vec![ 157 | (TextString("AA"), UnsignedInteger(6)), 158 | (TextString("B"), UnsignedInteger(5)), 159 | ]) 160 | .serialize(), 161 | from_hex("a261420562414106"), 162 | ); 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /sxg_rs/src/config.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use anyhow::Result; 16 | use serde::{Deserialize, Serialize}; 17 | use std::collections::BTreeSet; 18 | 19 | // This struct is source-of-truth of the sxg config. The user need to create 20 | // a file (like `config.yaml`) to provide this config input. 21 | #[derive(Deserialize, Serialize, Debug, Clone)] 22 | pub struct Config { 23 | pub cert_url_dirname: String, 24 | pub forward_request_headers: BTreeSet, 25 | pub html_host: String, 26 | // This field is only needed by Fastly, because Cloudflare uses secret 27 | // env variables to store private key. 28 | // TODO: check if Fastly edge dictionary is ok to store private key. 29 | pub private_key_base64: Option, 30 | pub reserved_path: String, 31 | pub strip_request_headers: BTreeSet, 32 | pub strip_response_headers: BTreeSet, 33 | pub validity_url_dirname: String, 34 | } 35 | 36 | impl Config { 37 | pub fn normalize(&mut self) { 38 | self.cert_url_dirname = to_url_prefix(&self.cert_url_dirname); 39 | lowercase_all(&mut self.forward_request_headers); 40 | self.reserved_path = to_url_prefix(&self.reserved_path); 41 | lowercase_all(&mut self.strip_request_headers); 42 | lowercase_all(&mut self.strip_response_headers); 43 | self.validity_url_dirname = to_url_prefix(&self.validity_url_dirname); 44 | } 45 | /// Creates config from text 46 | pub fn new(input_yaml: &str) -> Result { 47 | let mut input: Self = serde_yaml::from_str(input_yaml)?; 48 | input.normalize(); 49 | Ok(input) 50 | } 51 | } 52 | 53 | fn lowercase_all(names: &mut BTreeSet) { 54 | let old_names = std::mem::take(names); 55 | *names = old_names 56 | .into_iter() 57 | .map(|h| h.to_ascii_lowercase()) 58 | .collect(); 59 | } 60 | 61 | fn to_url_prefix(dirname: &str) -> String { 62 | format!("/{}/", dirname.trim_matches('/')) 63 | } 64 | 65 | #[cfg(test)] 66 | mod tests { 67 | use super::*; 68 | #[test] 69 | fn processes_input() { 70 | let yaml = r#" 71 | cert_url_dirname: ".well-known/sxg-certs/" 72 | forward_request_headers: 73 | - "cf-IPCOUNTRY" 74 | - "USER-agent" 75 | html_host: my_domain.com 76 | reserved_path: ".sxg" 77 | respond_debug_info: false 78 | strip_request_headers: ["Forwarded"] 79 | strip_response_headers: ["Set-Cookie", "STRICT-TRANSPORT-SECURITY"] 80 | validity_url_dirname: "//.well-known/sxg-validity" 81 | "#; 82 | let config = Config::new(yaml).unwrap(); 83 | assert_eq!(config.cert_url_dirname, "/.well-known/sxg-certs/"); 84 | assert_eq!( 85 | config.forward_request_headers, 86 | ["cf-ipcountry", "user-agent"] 87 | .iter() 88 | .map(|s| s.to_string()) 89 | .collect() 90 | ); 91 | assert_eq!(config.html_host, "my_domain.com".to_string()); 92 | assert_eq!( 93 | config.strip_request_headers, 94 | ["forwarded"].iter().map(|s| s.to_string()).collect() 95 | ); 96 | assert_eq!( 97 | config.strip_response_headers, 98 | ["set-cookie", "strict-transport-security"] 99 | .iter() 100 | .map(|s| s.to_string()) 101 | .collect() 102 | ); 103 | assert_eq!(config.reserved_path, "/.sxg/"); 104 | assert_eq!(config.validity_url_dirname, "/.well-known/sxg-validity/"); 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /sxg_rs/src/fetcher/js_fetcher.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use super::Fetcher; 16 | use crate::http::{HttpRequest, HttpResponse}; 17 | use crate::utils::await_js_promise; 18 | use anyhow::{Error, Result}; 19 | use async_trait::async_trait; 20 | use js_sys::Function as JsFunction; 21 | use wasm_bindgen::JsValue; 22 | 23 | /// A [`Fetcher`] implemented by JavaScript. 24 | pub struct JsFetcher(JsFunction); 25 | 26 | impl JsFetcher { 27 | /// Constructs a new `JsFetcher` with a given JavaScript function, 28 | /// which takes a [`HttpRequest`] and returns a 29 | /// [Promise](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise) 30 | /// of [`HttpResponse`]. 31 | /// ```typescript 32 | /// function js_function(req: HttpRequest): Promise {...} 33 | /// ``` 34 | /// # Panics 35 | /// Panics if `js_function` throws an error. 36 | pub fn new(js_function: JsFunction) -> Self { 37 | JsFetcher(js_function) 38 | } 39 | } 40 | 41 | #[async_trait(?Send)] 42 | impl Fetcher for JsFetcher { 43 | async fn fetch(&self, request: HttpRequest) -> Result { 44 | let request = serde_wasm_bindgen::to_value(&request) 45 | .map_err(|e| Error::msg(e.to_string()).context("Failed to parse request."))?; 46 | let response = await_js_promise(self.0.call1(&JsValue::NULL, &request)).await?; 47 | let response: HttpResponse = serde_wasm_bindgen::from_value(response) 48 | .map_err(|e| Error::msg(e.to_string()).context("Failed to serialize response."))?; 49 | Ok(response) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /sxg_rs/src/fetcher/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #[cfg(feature = "wasm")] 16 | pub mod js_fetcher; 17 | pub mod mock_fetcher; 18 | 19 | use crate::http::{HttpRequest, HttpResponse}; 20 | use crate::utils::{MaybeSend, MaybeSync}; 21 | use anyhow::{anyhow, Result}; 22 | use async_trait::async_trait; 23 | 24 | /// An interface for fetching resources from network. 25 | #[cfg_attr(feature = "wasm", async_trait(?Send))] 26 | #[cfg_attr(not(feature = "wasm"), async_trait)] 27 | pub trait Fetcher: MaybeSend + MaybeSync { 28 | async fn fetch(&self, request: HttpRequest) -> Result; 29 | } 30 | 31 | /// Uses `Get` method and returns response body, 32 | /// iteratively following 301, 302, 303, 307, 308 redirection. 33 | /// - Why this function is not put inside [`Fetcher`] trait? 34 | /// If we declare `Fetcher::get` function with a default implementation, 35 | /// we have to also add a constraint `where Self: Sized` to `Fetcher::get`, 36 | /// because of https://github.com/rust-lang/rust/issues/51443 and in particular 37 | /// https://docs.rs/async-trait/0.1.57/async_trait/#dyn-traits. 38 | /// However, having such constraint `Self: Sized` prevent using `Fetcher::get` method on a 39 | /// `dyn Fetcher` variable, because `dyn Fetcher` is not `Sized`. 40 | pub async fn get(fetcher: &dyn Fetcher, url: impl ToString) -> Result> { 41 | let mut url = url.to_string(); 42 | loop { 43 | let request = HttpRequest { 44 | body: vec![], 45 | headers: vec![], 46 | method: crate::http::Method::Get, 47 | url: url.to_string(), 48 | }; 49 | let response = fetcher.fetch(request).await?; 50 | if matches!(response.status, 301 | 302 | 303 | 307 | 308) { 51 | let location = response.headers.into_iter().find_map(|(name, value)| { 52 | if name.eq_ignore_ascii_case(http::header::LOCATION.as_str()) { 53 | Some(value) 54 | } else { 55 | None 56 | } 57 | }); 58 | if let Some(location) = location { 59 | url = location; 60 | continue; 61 | } 62 | } 63 | return Ok(response.body); 64 | } 65 | } 66 | 67 | pub const NULL_FETCHER: NullFetcher = NullFetcher {}; 68 | 69 | pub struct NullFetcher; 70 | 71 | #[cfg_attr(feature = "wasm", async_trait(?Send))] 72 | #[cfg_attr(not(feature = "wasm"), async_trait)] 73 | impl Fetcher for NullFetcher { 74 | async fn fetch(&self, _request: HttpRequest) -> Result { 75 | Err(anyhow!("Not found")) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /sxg_rs/src/http.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //! Serializable HTTP interfaces. 16 | //! # Conversion 17 | //! For interoperability with other Rust libraries, all structs can be coverted to and from the 18 | //! corresponding types in [`http`] crate. 19 | 20 | use anyhow::{anyhow, Error, Result}; 21 | use serde::{Deserialize, Serialize}; 22 | use std::convert::{Infallible, TryFrom, TryInto}; 23 | 24 | #[derive(Debug, Eq, PartialEq, Serialize, Clone)] 25 | pub struct HttpRequest { 26 | pub body: Vec, 27 | pub headers: HeaderFields, 28 | pub method: Method, 29 | pub url: String, 30 | } 31 | 32 | impl TryFrom<::http::request::Request>> for HttpRequest { 33 | type Error = Error; 34 | fn try_from(input: ::http::request::Request>) -> Result { 35 | let (parts, body) = input.into_parts(); 36 | Ok(HttpRequest { 37 | body, 38 | headers: try_from_header_map(parts.headers)?, 39 | method: parts.method.try_into()?, 40 | url: parts.uri.to_string(), 41 | }) 42 | } 43 | } 44 | 45 | impl TryInto<::http::request::Request>> for HttpRequest { 46 | type Error = Error; 47 | fn try_into(self) -> Result<::http::request::Request>> { 48 | let mut output = ::http::request::Request::new(self.body); 49 | *output.headers_mut() = try_into_header_map(self.headers)?; 50 | *output.method_mut() = self.method.try_into()?; 51 | *output.uri_mut() = self.url.try_into()?; 52 | Ok(output) 53 | } 54 | } 55 | 56 | #[derive(Serialize, Deserialize, Eq, PartialEq, Clone)] 57 | pub struct HttpResponse { 58 | pub body: Vec, 59 | pub headers: HeaderFields, 60 | pub status: u16, 61 | } 62 | 63 | impl TryFrom<::http::response::Response>> for HttpResponse { 64 | type Error = Error; 65 | fn try_from(input: ::http::response::Response>) -> Result { 66 | let (parts, body) = input.into_parts(); 67 | Ok(HttpResponse { 68 | body, 69 | headers: try_from_header_map(parts.headers)?, 70 | status: parts.status.as_u16(), 71 | }) 72 | } 73 | } 74 | 75 | impl TryInto<::http::response::Response>> for HttpResponse { 76 | type Error = Error; 77 | fn try_into(self) -> Result<::http::response::Response>> { 78 | let mut output = ::http::response::Response::new(self.body); 79 | *output.headers_mut() = try_into_header_map(self.headers)?; 80 | *output.status_mut() = self.status.try_into()?; 81 | Ok(output) 82 | } 83 | } 84 | 85 | pub type HeaderFields = Vec<(String, String)>; 86 | 87 | // The more readable way is to write 88 | // ``` 89 | // impl TryFrom<::http::header::HeaderMap> for HeaderFields { ... } 90 | // ``` 91 | // But compiler will throw error E0117 if do that. 92 | // Because `HeaderFields` is a type alias, and we are not the author of `Vec<(String, String)>`. 93 | fn try_from_header_map(input: ::http::header::HeaderMap) -> Result { 94 | let mut output = vec![]; 95 | for (name, value) in input.iter() { 96 | let value = value.to_str().map_err(|e| { 97 | Error::new(e).context(format!("Header {} contains non-ASCII value", name)) 98 | })?; 99 | output.push((name.to_string(), value.to_string())); 100 | } 101 | Ok(output) 102 | } 103 | 104 | // The more readable way is to write 105 | // ``` 106 | // impl TryInto<::http::header::HeaderMap> for HeaderFields { ... } 107 | // ``` 108 | // But compiler will throw error E0117 if do that. 109 | // Because `HeaderFields` is a type alias, and we are not the author of `Vec<(String, String)>`. 110 | fn try_into_header_map(input: HeaderFields) -> Result<::http::header::HeaderMap> { 111 | input 112 | .iter() 113 | .map(|(name, value)| -> Result<_> { 114 | let name = 115 | ::http::header::HeaderName::from_bytes(name.as_bytes()).map_err(Error::new)?; 116 | let value = ::http::header::HeaderValue::from_str(value).map_err(Error::new)?; 117 | Ok((name, value)) 118 | }) 119 | .collect() 120 | } 121 | 122 | #[derive(Debug, Eq, PartialEq, Serialize, Clone)] 123 | pub enum Method { 124 | Get, 125 | Post, 126 | } 127 | 128 | impl TryFrom<::http::Method> for Method { 129 | type Error = Error; 130 | fn try_from(method: ::http::Method) -> Result { 131 | match method { 132 | ::http::Method::GET => Ok(Method::Get), 133 | ::http::Method::POST => Ok(Method::Post), 134 | x => Err(anyhow!("Method {} is not supported", x)), 135 | } 136 | } 137 | } 138 | 139 | impl TryInto<::http::Method> for Method { 140 | type Error = Infallible; 141 | fn try_into(self) -> Result<::http::Method, Self::Error> { 142 | match self { 143 | Method::Get => Ok(::http::Method::GET), 144 | Method::Post => Ok(::http::Method::POST), 145 | } 146 | } 147 | } 148 | 149 | impl std::fmt::Debug for HttpResponse { 150 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 151 | f.debug_struct("HttpResponse") 152 | .field("status", &self.status) 153 | .field("headers", &self.headers) 154 | .field("body", &base64::encode(&self.body)) 155 | .finish() 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /sxg_rs/src/http_cache/js_http_cache.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use super::HttpCache; 16 | use crate::http::HttpResponse; 17 | use anyhow::{anyhow, Error, Result}; 18 | use async_trait::async_trait; 19 | use js_sys::Function as JsFunction; 20 | use wasm_bindgen::JsValue; 21 | 22 | pub struct JsHttpCache { 23 | pub get: JsFunction, 24 | pub put: JsFunction, 25 | } 26 | 27 | #[async_trait(?Send)] 28 | impl HttpCache for JsHttpCache { 29 | async fn get(&self, url: &str) -> Result { 30 | let url = serde_wasm_bindgen::to_value(&url) 31 | .map_err(|e| Error::msg(e.to_string()).context("serializing url to JS"))?; 32 | let this = JsValue::null(); 33 | let response = self 34 | .get 35 | .call1(&this, &url) 36 | .map_err(|_| anyhow!("Error invoking JS get"))?; 37 | let response = wasm_bindgen_futures::JsFuture::from(js_sys::Promise::from(response)); 38 | let response = response 39 | .await 40 | .map_err(|_| anyhow!("Error returned by JS get"))?; 41 | let response = serde_wasm_bindgen::from_value(response) 42 | .map_err(|e| Error::msg(e.to_string()).context("parsing response from JS"))?; 43 | Ok(response) 44 | } 45 | async fn put(&self, url: &str, response: &HttpResponse) -> Result<()> { 46 | let url = serde_wasm_bindgen::to_value(&url) 47 | .map_err(|e| Error::msg(e.to_string()).context("serializing url to JS"))?; 48 | let response = serde_wasm_bindgen::to_value(&response) 49 | .map_err(|e| Error::msg(e.to_string()).context("serializing response to JS"))?; 50 | let this = JsValue::null(); 51 | let ret = self 52 | .put 53 | .call2(&this, &url, &response) 54 | .map_err(|_| anyhow!("Error invoking JS put"))?; 55 | let ret = wasm_bindgen_futures::JsFuture::from(js_sys::Promise::from(ret)); 56 | let ret = ret.await.map_err(|_| anyhow!("Error returned by JS put"))?; 57 | let _ret = serde_wasm_bindgen::from_value(ret) 58 | .map_err(|e| Error::msg(e.to_string()).context("parsing ack from JS"))?; 59 | Ok(()) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /sxg_rs/src/http_cache/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #[cfg(feature = "wasm")] 16 | pub mod js_http_cache; 17 | 18 | use crate::http::HttpResponse; 19 | use crate::utils::{MaybeSend, MaybeSync}; 20 | use anyhow::{anyhow, Result}; 21 | use async_trait::async_trait; 22 | 23 | /// An interface for storing HTTP responses in a cache. 24 | #[cfg_attr(feature = "wasm", async_trait(?Send))] 25 | #[cfg_attr(not(feature = "wasm"), async_trait)] 26 | pub trait HttpCache: MaybeSend + MaybeSync { 27 | async fn get(&self, url: &str) -> Result; 28 | async fn put(&self, url: &str, response: &HttpResponse) -> Result<()>; 29 | } 30 | 31 | pub struct NullCache; 32 | 33 | #[cfg_attr(feature = "wasm", async_trait(?Send))] 34 | #[cfg_attr(not(feature = "wasm"), async_trait)] 35 | impl HttpCache for NullCache { 36 | async fn get(&self, _url: &str) -> Result { 37 | Err(anyhow!("No cache entry found in NullCache")) 38 | } 39 | async fn put(&self, _url: &str, _response: &HttpResponse) -> Result<()> { 40 | Ok(()) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /sxg_rs/src/http_parser/accept.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use super::media_type::{media_type, MediaType, Parameter}; 16 | use nom::{combinator::map, IResult}; 17 | 18 | #[derive(Debug, Eq, PartialEq)] 19 | pub struct Accept<'a> { 20 | pub media_range: MediaType<'a>, 21 | // The q value has at most 3 digits, hence 1000*q must be an integer. 22 | // https://tools.ietf.org/html/rfc7231#section-5.3.1 23 | pub q_millis: u16, 24 | pub extensions: Vec>, 25 | } 26 | 27 | // https://tools.ietf.org/html/rfc7231#section-5.3.2 28 | // The `accept` header has a similar syntax to `media-type`, except a special 29 | // `q` parameter. Parameters before the first `q=...` are media type parameters 30 | // and the parameters after are accept extension parameters. 31 | pub fn accept(input: &str) -> IResult<&str, Accept<'_>> { 32 | map(media_type, |media_range| { 33 | let mut accept = Accept { 34 | media_range, 35 | q_millis: 1000, 36 | extensions: vec![], 37 | }; 38 | let params = &mut accept.media_range.parameters; 39 | for (i, param) in params.iter().enumerate() { 40 | if param.name.eq_ignore_ascii_case("q") { 41 | if let Some(q) = parse_q_millis(¶m.value) { 42 | accept.q_millis = q; 43 | accept.extensions = params.split_off(i + 1); 44 | params.pop(); 45 | break; 46 | } 47 | } 48 | } 49 | accept 50 | })(input) 51 | } 52 | 53 | fn parse_q_millis(s: &str) -> Option { 54 | let x = s.parse::().ok()?; 55 | if (0.0..=1.0).contains(&x) { 56 | Some((x * 1000.0) as u16) 57 | } else { 58 | None 59 | } 60 | } 61 | 62 | #[cfg(test)] 63 | mod tests { 64 | use super::*; 65 | #[test] 66 | fn it_works() { 67 | assert_eq!( 68 | accept("text/html;charset=utf-8;q=0.9").unwrap(), 69 | ( 70 | "", 71 | Accept { 72 | media_range: MediaType { 73 | primary_type: "text", 74 | sub_type: "html", 75 | parameters: vec![Parameter { 76 | name: "charset", 77 | value: "utf-8".to_string(), 78 | }], 79 | }, 80 | q_millis: 900, 81 | extensions: vec![], 82 | } 83 | ) 84 | ); 85 | } 86 | #[test] 87 | fn params_after_q_are_extensions() { 88 | assert_eq!( 89 | accept("a/b;x1=1;x2=2;q=0.9;x3=3;x4=4").unwrap(), 90 | ( 91 | "", 92 | Accept { 93 | media_range: MediaType { 94 | primary_type: "a", 95 | sub_type: "b", 96 | parameters: vec![ 97 | Parameter { 98 | name: "x1", 99 | value: "1".to_string(), 100 | }, 101 | Parameter { 102 | name: "x2", 103 | value: "2".to_string(), 104 | }, 105 | ], 106 | }, 107 | q_millis: 900, 108 | extensions: vec![ 109 | Parameter { 110 | name: "x3", 111 | value: "3".to_string(), 112 | }, 113 | Parameter { 114 | name: "x4", 115 | value: "4".to_string(), 116 | }, 117 | ], 118 | } 119 | ) 120 | ); 121 | } 122 | #[test] 123 | fn default_q() { 124 | assert_eq!( 125 | accept("a/b").unwrap(), 126 | ( 127 | "", 128 | Accept { 129 | media_range: MediaType { 130 | primary_type: "a", 131 | sub_type: "b", 132 | parameters: vec![], 133 | }, 134 | q_millis: 1000, 135 | extensions: vec![], 136 | } 137 | ) 138 | ); 139 | } 140 | #[test] 141 | fn uppercase_q() { 142 | assert_eq!( 143 | accept("a/b;Q=0.5").unwrap(), 144 | ( 145 | "", 146 | Accept { 147 | media_range: MediaType { 148 | primary_type: "a", 149 | sub_type: "b", 150 | parameters: vec![], 151 | }, 152 | q_millis: 500, 153 | extensions: vec![], 154 | } 155 | ) 156 | ); 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /sxg_rs/src/http_parser/base.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use nom::{ 16 | branch::alt, 17 | bytes::complete::{take, take_while, take_while1}, 18 | character::complete::char as char1, 19 | combinator::{map, map_opt}, 20 | multi::many0, 21 | sequence::{delimited, preceded}, 22 | IResult, 23 | }; 24 | 25 | // `token` are defined in 26 | // https://tools.ietf.org/html/rfc7230#section-3.2.6 27 | pub fn token(input: &str) -> IResult<&str, &str> { 28 | take_while1(is_tchar)(input) 29 | } 30 | 31 | pub fn is_tchar(c: char) -> bool { 32 | matches!(c, 33 | '!' | '#' | '$' | '%' | '&' | '\'' | '*' | 34 | '+' | '-' | '.' | '^' | '_' | '`' | '|' | '~' | 35 | '0'..='9' | 'A'..='Z' | 'a'..='z' 36 | ) 37 | } 38 | 39 | fn is_space_or_tab(c: char) -> bool { 40 | c == '\t' || c == ' ' 41 | } 42 | 43 | // `OWS` is defined in 44 | // https://tools.ietf.org/html/rfc7230#section-3.2.3 45 | pub fn ows(input: &str) -> IResult<&str, &str> { 46 | take_while(is_space_or_tab)(input) 47 | } 48 | 49 | // `quoted-string` is defined in 50 | // https://tools.ietf.org/html/rfc7230#section-3.2.6 51 | pub fn quoted_string(input: &str) -> IResult<&str, String> { 52 | map( 53 | delimited(char1('"'), many0(alt((qdtext, quoted_pair))), char1('"')), 54 | |s: Vec| s.into_iter().collect(), 55 | )(input) 56 | } 57 | 58 | fn qdtext(input: &str) -> IResult<&str, char> { 59 | char_if(is_qdtext)(input) 60 | } 61 | 62 | fn is_qdtext(c: char) -> bool { 63 | matches!(c, 64 | '\t' | ' ' | '\x21' | 65 | '\x23'..='\x5B' | '\x5D'..='\x7E' | 66 | '\u{80}'..=std::char::MAX 67 | ) 68 | } 69 | 70 | pub fn is_quoted_pair_payload(c: char) -> bool { 71 | matches!(c, 72 | '\t' | ' ' | 73 | '\x21'..='\x7E' | 74 | '\u{80}'..=std::char::MAX 75 | ) 76 | } 77 | 78 | fn quoted_pair(input: &str) -> IResult<&str, char> { 79 | preceded(char1('\\'), char_if(is_quoted_pair_payload))(input) 80 | } 81 | 82 | fn char_if(predicate: fn(c: char) -> bool) -> impl Fn(&str) -> IResult<&str, char> { 83 | move |input: &str| { 84 | map_opt(take(1usize), |s: &str| { 85 | let c = s.chars().next()?; 86 | if predicate(c) { 87 | Some(c) 88 | } else { 89 | None 90 | } 91 | })(input) 92 | } 93 | } 94 | 95 | pub fn parameter_value(input: &str) -> IResult<&str, String> { 96 | alt((map(token, |s: &str| s.to_string()), quoted_string))(input) 97 | } 98 | 99 | #[cfg(test)] 100 | mod tests { 101 | use super::*; 102 | #[test] 103 | fn incomplete() { 104 | assert!(quoted_string(r#""amp"#).is_err()); 105 | assert!(parameter_value(r#""amp"#).is_err()); 106 | } 107 | #[test] 108 | fn obs_text() { 109 | // `obs-text` are text made by non-ascii bytes (0x80-0xff). 110 | // `obs-test` are not allowed in tokens. 111 | assert_eq!( 112 | token("amp⚡").unwrap(), 113 | ( 114 | "⚡", // unparsed bytes 115 | "amp", // parsed token 116 | ) 117 | ); 118 | // `obs-text` are allowed in quoted-string. 119 | assert_eq!( 120 | quoted_string(r#""amp⚡s""#).unwrap(), 121 | ("", "amp⚡s".to_string(),) 122 | ); 123 | // `obs-text` are allowed as quoted-pair. 124 | assert_eq!( 125 | quoted_string(r#""amp\⚡s""#).unwrap(), 126 | ("", "amp⚡s".to_string(),) 127 | ); 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /sxg_rs/src/http_parser/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | mod accept; 16 | mod base; 17 | pub mod cache_control; 18 | pub mod link; 19 | pub mod media_type; 20 | pub mod srcset; 21 | 22 | use anyhow::{Error, Result}; 23 | use base::ows; 24 | use nom::{ 25 | branch::alt, 26 | bytes::complete::tag, 27 | character::complete::char as char1, 28 | combinator::eof, 29 | multi::separated_list0, 30 | sequence::{separated_pair, terminated}, 31 | IResult, 32 | }; 33 | use std::time::Duration; 34 | 35 | fn format_nom_err(err: nom::Err>) -> Error { 36 | Error::msg(format!("{}", err)) 37 | } 38 | 39 | // Parses a http header which might have multiple values separated by comma. 40 | fn parse_vec<'a, F, T>(input: &'a str, parse_single: F) -> Result> 41 | where 42 | F: Fn(&'a str) -> IResult<&'a str, T>, 43 | { 44 | terminated( 45 | separated_list0(separated_pair(ows, char1(','), ows), parse_single), 46 | eof, 47 | )(input) 48 | .map(|(_, items)| items) 49 | .map_err(format_nom_err) 50 | } 51 | 52 | pub fn parse_accept_header(input: &str) -> Result> { 53 | parse_vec(input, accept::accept) 54 | } 55 | 56 | pub fn parse_cache_control_directives(input: &str) -> Result> { 57 | parse_vec(input, cache_control::directive) 58 | } 59 | 60 | // Returns the freshness lifetime for a shared cache. 61 | pub fn parse_cache_control_header(input: &str) -> Result { 62 | cache_control::freshness_lifetime(parse_cache_control_directives(input)?) 63 | .ok_or_else(|| Error::msg("Freshness lifetime is implicit")) 64 | } 65 | 66 | pub fn parse_content_type_header(input: &str) -> Result { 67 | terminated(media_type::media_type, eof)(input) 68 | .map(|(_, output)| output) 69 | .map_err(format_nom_err) 70 | } 71 | 72 | pub fn parse_link_header(input: &str) -> Result> { 73 | parse_vec(input, link::link) 74 | } 75 | 76 | pub fn parse_token_list(input: &str) -> Result> { 77 | parse_vec(input, base::token) 78 | } 79 | 80 | // https://datatracker.ietf.org/doc/html/rfc7231#section-7.1.4 81 | pub fn parse_vary_header(input: &str) -> Result> { 82 | parse_vec(input, |input| { 83 | alt(( 84 | tag("*"), 85 | // https://datatracker.ietf.org/doc/html/rfc7230#section-3.2 86 | base::token, 87 | ))(input) 88 | }) 89 | } 90 | 91 | #[cfg(test)] 92 | mod tests { 93 | use super::*; 94 | #[test] 95 | fn vary() { 96 | assert_eq!( 97 | parse_vary_header("* , cookie").unwrap(), 98 | vec!["*", "cookie"] 99 | ); 100 | assert!(parse_vary_header("tokens only; no spaces or semicolons allowed").is_err()); 101 | } 102 | #[test] 103 | fn incomplete_is_err() { 104 | assert!(parse_accept_header("application/signed-exchange;v=").is_err()); 105 | assert!(parse_cache_control_header("max-age=\"3600").is_err()); 106 | assert!(parse_content_type_header("application/signed-exchange;v=\"b3").is_err()); 107 | assert!(parse_link_header(r#";bar="baz \""#).is_err()); 108 | assert!(parse_vary_header("incomplete,").is_err()); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /sxg_rs/src/http_parser/srcset.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #[cfg(feature = "srcset")] 16 | use anyhow::bail; 17 | use anyhow::Result; 18 | 19 | // Extracts the URI refs inside the srcset attribute, for instance 'imagesrcset' on , per 20 | // https://html.spec.whatwg.org/multipage/semantics.html#attr-link-imagesrcset. Descriptors are 21 | // discarded; not needed for our use case. 22 | #[cfg(feature = "srcset")] 23 | pub fn parse(mut input: &str) -> Result> { 24 | // Parsing is done imperatively rather than using nom for two reasons: 25 | // 1. It's defined imperatively at 26 | // https://html.spec.whatwg.org/multipage/images.html#parsing-a-srcset-attribute:srcset-attribute, 27 | // so it's easier to compare implementation to spec this way. 28 | // 2. There are things that are difficult to do in nom, like the comma backtracking and the 29 | // splitting on commas except in (non-matched) parens. 30 | let mut candidates = vec![]; 31 | loop { 32 | const WHITESPACE: &[char] = &['\t', '\r', '\x0C', '\n', ' ']; 33 | const WHITESPACE_OR_COMMA: &[char] = &['\t', '\r', '\x0C', '\n', ' ', ',']; 34 | input = input.trim_start_matches(WHITESPACE_OR_COMMA); 35 | if input.is_empty() { 36 | return Ok(candidates); 37 | } 38 | let (mut url, remaining) = input.split_at(input.find(WHITESPACE).unwrap_or(input.len())); 39 | input = remaining; 40 | let len_before_trim = url.len(); 41 | url = url.trim_end_matches(','); 42 | let commas_trimmed = len_before_trim - url.len(); 43 | if commas_trimmed > 0 { 44 | if commas_trimmed > 1 { 45 | bail!("Ambiguous comma at end of URL in srcset"); 46 | } 47 | } else { 48 | input = input.trim_start_matches(WHITESPACE); 49 | // A simplification of step 8, since descriptors are discarded. Look for the first comma not 50 | // inside parens: 51 | let mut in_parens = false; 52 | input = input.trim_start_matches(|c| match c { 53 | '(' => { 54 | in_parens = true; 55 | true 56 | } 57 | ')' => { 58 | in_parens = false; 59 | true 60 | } 61 | ',' => in_parens, // Stop on comma outside of parens. 62 | _ => true, 63 | }); 64 | // The found comma will be trimmed at the beginning of the next iteration. 65 | } 66 | // There are additional steps for validating the descriptors. This is skipped for 67 | // simplicity, so this parser may extract more URI refs than a fully compliant one. 68 | candidates.push(url); 69 | } 70 | } 71 | 72 | #[cfg(not(feature = "srcset"))] 73 | pub fn parse(_input: &str) -> Result> { 74 | Ok(vec![]) 75 | } 76 | 77 | #[cfg(all(test, feature = "srcset"))] 78 | mod tests { 79 | use super::*; 80 | #[test] 81 | fn srcset() { 82 | assert_eq!( 83 | parse("elva-fairy-480w.jpg 480w, elva-fairy-800w.jpg 800w").unwrap(), 84 | vec!["elva-fairy-480w.jpg", "elva-fairy-800w.jpg"] 85 | ); 86 | assert_eq!( 87 | parse("elva-fairy-320w.jpg, elva-fairy-480w.jpg 1.5x, elva-fairy-640w.jpg 2x").unwrap(), 88 | vec![ 89 | "elva-fairy-320w.jpg", 90 | "elva-fairy-480w.jpg", 91 | "elva-fairy-640w.jpg" 92 | ] 93 | ); 94 | assert_eq!(parse("elva-800w.jpg").unwrap(), vec!["elva-800w.jpg"]); 95 | assert_eq!( 96 | parse("url,with,comma.jpg 400w, other,url,with,comma.jpg, third,url.jpg").unwrap(), 97 | vec![ 98 | "url,with,comma.jpg", 99 | "other,url,with,comma.jpg", 100 | "third,url.jpg" 101 | ] 102 | ); 103 | assert_eq!( 104 | parse("hypothetical-comma-in-parens.jpg (400w, 500h), other.jpg").unwrap(), 105 | vec!["hypothetical-comma-in-parens.jpg", "other.jpg"] 106 | ); 107 | assert!(matches!(parse("too,many,trailing,commas,,"), Err(_))); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /sxg_rs/src/mice.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // https://tools.ietf.org/html/draft-thomson-http-mice-03 16 | 17 | use crate::crypto::HashAlgorithm; 18 | use ::sha2::{Digest, Sha256}; 19 | use std::collections::VecDeque; 20 | 21 | pub fn calculate(input: &[u8], record_size: usize) -> (Vec, Vec) { 22 | if input.is_empty() { 23 | return (HashAlgorithm::Sha256.digest(&[0]), vec![]); 24 | } 25 | let record_size = std::cmp::min(record_size, input.len()); 26 | let records: Vec<_> = if record_size > 0 { 27 | input.chunks(record_size).collect() 28 | } else { 29 | vec![input] 30 | }; 31 | let mut proofs: VecDeque> = VecDeque::new(); 32 | for record in records.iter().rev() { 33 | let mut hasher = Sha256::new(); 34 | hasher.update(record); 35 | if let Some(f) = proofs.front() { 36 | hasher.update(f); 37 | hasher.update([1u8]); 38 | } else { 39 | hasher.update([0u8]); 40 | } 41 | proofs.push_front(hasher.finalize().to_vec()); 42 | } 43 | let mut message = Vec::new(); 44 | message.extend_from_slice(&(record_size as u64).to_be_bytes()); 45 | for i in 0..records.len() { 46 | if i > 0 { 47 | message.extend_from_slice(&proofs[i]); 48 | } 49 | message.extend_from_slice(records[i]); 50 | } 51 | let integrity = proofs.pop_front().unwrap(); 52 | (integrity, message) 53 | } 54 | 55 | #[cfg(test)] 56 | mod tests { 57 | use super::*; 58 | #[test] 59 | fn it_works() { 60 | // https://tools.ietf.org/html/draft-thomson-http-mice-03#section-4.1 61 | let input = "When I grow up, I want to be a watermelon".as_bytes(); 62 | assert_eq!( 63 | calculate(input, 1000000), 64 | ( 65 | ::base64::decode("dcRDgR2GM35DluAV13PzgnG6+pvQwPywfFvAu1UeFrs=").unwrap(), 66 | [&0x29_u64.to_be_bytes(), input].concat(), 67 | ), 68 | ); 69 | // https://tools.ietf.org/html/draft-thomson-http-mice-03#section-4.2 70 | assert_eq!( 71 | calculate(input, 16), 72 | ( 73 | ::base64::decode("IVa9shfs0nyKEhHqtB3WVNANJ2Njm5KjQLjRtnbkYJ4=").unwrap(), 74 | [ 75 | &0x10_u64.to_be_bytes(), 76 | &input[0..16], 77 | &::base64::decode("OElbplJlPK+Rv6JNK6p5/515IaoPoZo+2elWL7OQ60A=").unwrap(), 78 | &input[16..32], 79 | &::base64::decode("iPMpmgExHPrbEX3/RvwP4d16fWlK4l++p75PUu/KyN0=").unwrap(), 80 | &input[32..], 81 | ] 82 | .concat(), 83 | ), 84 | ); 85 | } 86 | #[test] 87 | fn empty_payload() { 88 | assert_eq!( 89 | calculate(b"", 16384), 90 | ( 91 | ::base64::decode("bjQLnP+zepicpUTmu3gKLHiQHT+zNzh2hRGjBhevoB0=").unwrap(), 92 | vec![], 93 | ), 94 | ); 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /sxg_rs/src/runtime/js_runtime.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use super::Runtime; 16 | use crate::fetcher::{js_fetcher::JsFetcher, Fetcher, NullFetcher}; 17 | use crate::signature::{js_signer::JsSigner, mock_signer::MockSigner, Signer}; 18 | use crate::storage::{js_storage::JsStorage, Storage}; 19 | use anyhow::{Error, Result}; 20 | use js_sys::Function as JsFunction; 21 | use std::time::{Duration, SystemTime}; 22 | use wasm_bindgen::prelude::*; 23 | 24 | #[wasm_bindgen] 25 | extern "C" { 26 | pub type JsRuntimeInitParams; 27 | #[wasm_bindgen(method, getter, js_name = "nowInSeconds")] 28 | fn now_in_seconds(this: &JsRuntimeInitParams) -> u32; 29 | #[wasm_bindgen(method, getter, js_name = "fetcher")] 30 | fn fetcher(this: &JsRuntimeInitParams) -> Option; 31 | #[wasm_bindgen(method, getter, js_name = "storageRead")] 32 | fn storage_read(this: &JsRuntimeInitParams) -> Option; 33 | #[wasm_bindgen(method, getter, js_name = "storageWrite")] 34 | fn storage_write(this: &JsRuntimeInitParams) -> Option; 35 | #[wasm_bindgen(method, getter, js_name = "sxgAsn1Signer")] 36 | fn sxg_asn1_signer(this: &JsRuntimeInitParams) -> Option; 37 | #[wasm_bindgen(method, getter, js_name = "sxgRawSigner")] 38 | fn sxg_raw_signer(this: &JsRuntimeInitParams) -> Option; 39 | #[wasm_bindgen(method, getter, js_name = "acmeRawSigner")] 40 | fn acme_raw_signer(this: &JsRuntimeInitParams) -> Option; 41 | } 42 | 43 | impl std::convert::TryFrom for Runtime { 44 | type Error = Error; 45 | fn try_from(input: JsRuntimeInitParams) -> Result { 46 | let now = SystemTime::UNIX_EPOCH + Duration::from_secs(input.now_in_seconds() as u64); 47 | let fetcher = input 48 | .fetcher() 49 | .map(|f| Box::new(JsFetcher::new(f)) as Box); 50 | let storage = Box::new(JsStorage::new(input.storage_read(), input.storage_write())) 51 | as Box; 52 | let sxg_asn1_signer = input 53 | .sxg_asn1_signer() 54 | .map(|f| Box::new(JsSigner::from_asn1_signer(f)) as Box); 55 | let sxg_raw_signer = input 56 | .sxg_raw_signer() 57 | .map(|f| Box::new(JsSigner::from_raw_signer(f)) as Box); 58 | let sxg_signer = sxg_asn1_signer.or(sxg_raw_signer); 59 | let acme_signer = input 60 | .acme_raw_signer() 61 | .map(|f| Box::new(JsSigner::from_raw_signer(f)) as Box); 62 | Ok(Runtime { 63 | now, 64 | fetcher: fetcher.unwrap_or_else(|| Box::new(NullFetcher)), 65 | storage, 66 | sxg_signer: sxg_signer.unwrap_or_else(|| Box::new(MockSigner)), 67 | acme_signer: acme_signer.unwrap_or_else(|| Box::new(MockSigner)), 68 | }) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /sxg_rs/src/runtime/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #[cfg(feature = "wasm")] 16 | pub mod js_runtime; 17 | 18 | use crate::fetcher::{Fetcher, NullFetcher}; 19 | use crate::signature::{mock_signer::MockSigner, Signer}; 20 | use crate::storage::{InMemoryStorage, Storage}; 21 | use std::time::SystemTime; 22 | 23 | pub struct Runtime { 24 | pub now: SystemTime, 25 | pub fetcher: Box, 26 | pub storage: Box, 27 | pub sxg_signer: Box, 28 | pub acme_signer: Box, 29 | } 30 | 31 | impl Default for Runtime { 32 | fn default() -> Self { 33 | Runtime { 34 | now: SystemTime::UNIX_EPOCH, 35 | fetcher: Box::new(NullFetcher), 36 | storage: Box::new(InMemoryStorage::default()), 37 | sxg_signer: Box::new(MockSigner), 38 | acme_signer: Box::new(MockSigner), 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /sxg_rs/src/serde_helpers/base64.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //! Module to serialize `Vec` to and from base64 format, 16 | //! using URL-safe charater set without padding. 17 | 18 | use serde::de::Error; 19 | use serde::{Deserialize, Deserializer, Serializer}; 20 | 21 | pub fn serialize(bytes: &[u8], serializer: S) -> Result 22 | where 23 | S: Serializer, 24 | { 25 | serializer.serialize_str(&base64::encode_config(bytes, base64::URL_SAFE_NO_PAD)) 26 | } 27 | 28 | pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> 29 | where 30 | D: Deserializer<'de>, 31 | { 32 | let s = String::deserialize(deserializer)?; 33 | base64::decode_config(s, base64::URL_SAFE_NO_PAD) 34 | .map_err(|_| D::Error::custom("Invalid base64 string")) 35 | } 36 | 37 | #[cfg(test)] 38 | mod tests { 39 | use super::*; 40 | use serde::Serialize; 41 | #[derive(Serialize, Deserialize)] 42 | struct Data { 43 | #[serde(with = "super")] 44 | bytes: Vec, 45 | } 46 | #[test] 47 | fn serialize() { 48 | let x = Data { 49 | bytes: vec![1, 2, 3], 50 | }; 51 | assert_eq!(serde_json::to_string(&x).unwrap(), r#"{"bytes":"AQID"}"#); 52 | } 53 | #[test] 54 | fn deserialize() { 55 | let x: Data = serde_json::from_str(r#"{"bytes":"AQID"}"#).unwrap(); 56 | assert_eq!(x.bytes, vec![1, 2, 3]); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /sxg_rs/src/serde_helpers/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //! Helper functions for [#[serde(with)]](https://serde.rs/field-attrs.html#with). 16 | //! 17 | //! For example, 18 | //! ``` 19 | //! use serde::{Serialize, Deserialize}; 20 | //! #[derive(Serialize, Deserialize)] 21 | //! struct Container { 22 | //! #[serde(with = "sxg_rs::serde_helpers::base64")] 23 | //! value: Vec, 24 | //! } 25 | //! ``` 26 | //! Rust value `Container { value: vec![1, 2, 3] }` will be serialized as JSON `{ "value": "AQID" }`. 27 | 28 | pub mod base64; 29 | -------------------------------------------------------------------------------- /sxg_rs/src/signature/js_signer.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use super::{Format, Signer}; 16 | use crate::utils::await_js_promise; 17 | use anyhow::Result; 18 | use async_trait::async_trait; 19 | use js_sys::{Function as JsFunction, Uint8Array}; 20 | use wasm_bindgen::JsValue; 21 | 22 | /// [JsSigner] allows you to implement [Signer] trait by a JavaScript function. 23 | pub struct JsSigner { 24 | js_function: JsFunction, 25 | js_sig_format: Format, 26 | } 27 | 28 | impl JsSigner { 29 | /// Creates a signer by a JavaScript async function, 30 | /// `js_function` must be of type `(input: Uint8Array) => Promise`. 31 | /// `js_function` should return the signature in `ASN.1` format. 32 | pub fn from_asn1_signer(js_function: JsFunction) -> Self { 33 | JsSigner { 34 | js_function, 35 | js_sig_format: Format::EccAsn1, 36 | } 37 | } 38 | /// Creates a signer by a JavaScript async function. 39 | /// `js_function` must be of type `(input: Uint8Array) => Promise`. 40 | /// `js_function` should return the raw signature, which contains exactly 64 bytes. 41 | /// For example, Web API 42 | /// [SubtleCrypto.sign()](https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/sign) 43 | /// returns the raw signature as 64 bytes. 44 | pub fn from_raw_signer(js_function: JsFunction) -> Self { 45 | JsSigner { 46 | js_function, 47 | js_sig_format: Format::Raw, 48 | } 49 | } 50 | } 51 | 52 | #[async_trait(?Send)] 53 | impl Signer for JsSigner { 54 | async fn sign(&self, message: &[u8], format: Format) -> Result> { 55 | let a = Uint8Array::new_with_length(message.len() as u32); 56 | a.copy_from(message); 57 | let sig = await_js_promise(self.js_function.call1(&JsValue::NULL, &a)).await?; 58 | let sig = Uint8Array::from(sig); 59 | let sig = sig.to_vec(); 60 | match (self.js_sig_format, format) { 61 | (Format::Raw, Format::Raw) => Ok(sig), 62 | (Format::EccAsn1, Format::EccAsn1) => Ok(sig), 63 | (Format::Raw, Format::EccAsn1) => super::raw_sig_to_asn1(sig), 64 | (Format::EccAsn1, Format::Raw) => super::parse_asn1_sig(&sig), 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /sxg_rs/src/signature/mock_signer.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use super::{Format, Signer}; 16 | use anyhow::Result; 17 | use async_trait::async_trait; 18 | 19 | pub struct MockSigner; 20 | 21 | #[cfg_attr(feature = "wasm", async_trait(?Send))] 22 | #[cfg_attr(not(feature = "wasm"), async_trait)] 23 | impl Signer for MockSigner { 24 | async fn sign(&self, _message: &[u8], format: Format) -> Result> { 25 | match format { 26 | Format::EccAsn1 => super::raw_sig_to_asn1([0].repeat(64)), 27 | Format::Raw => Ok([0].repeat(64)), 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /sxg_rs/src/signature/rust_signer.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use super::{Format, Signer}; 16 | use anyhow::Result; 17 | use async_trait::async_trait; 18 | use p256::ecdsa::SigningKey; 19 | 20 | pub struct RustSigner { 21 | private_key: SigningKey, 22 | } 23 | 24 | impl RustSigner { 25 | pub fn new(private_key: &[u8]) -> Result { 26 | let private_key = SigningKey::from_bytes(private_key)?; 27 | Ok(RustSigner { private_key }) 28 | } 29 | } 30 | 31 | #[cfg_attr(feature = "wasm", async_trait(?Send))] 32 | #[cfg_attr(not(feature = "wasm"), async_trait)] 33 | impl Signer for RustSigner { 34 | async fn sign(&self, message: &[u8], format: Format) -> Result> { 35 | use p256::ecdsa::signature::Signer as _; 36 | let sig = self.private_key.try_sign(message)?; 37 | match format { 38 | Format::Raw => Ok(sig.to_vec()), 39 | Format::EccAsn1 => Ok(sig.to_der().as_bytes().to_vec()), 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /sxg_rs/src/static/fallback.html: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 |

No, the SXG fails.

19 | -------------------------------------------------------------------------------- /sxg_rs/src/static/image.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/sxg-rs/06d31585c95cd23c2c6a9f5f5b6fc5eb247bcfdd/sxg_rs/src/static/image.jpg -------------------------------------------------------------------------------- /sxg_rs/src/static/prefetch.html: -------------------------------------------------------------------------------- 1 | 16 | 24 | 25 | 26 |

Loading SXG test page in 2 seconds

27 | 41 | -------------------------------------------------------------------------------- /sxg_rs/src/static/success.html: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 |

Yes, this message comes from a valid SXG.

19 | -------------------------------------------------------------------------------- /sxg_rs/src/static/test.html: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 22 | 23 | -------------------------------------------------------------------------------- /sxg_rs/src/storage/js_storage.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use super::Storage; 16 | use crate::utils::await_js_promise; 17 | use anyhow::{anyhow, Result}; 18 | use async_trait::async_trait; 19 | use js_sys::Function as JsFunction; 20 | use wasm_bindgen::prelude::*; 21 | 22 | #[wasm_bindgen] 23 | pub struct JsStorage { 24 | read: Option, 25 | write: Option, 26 | } 27 | 28 | #[wasm_bindgen] 29 | impl JsStorage { 30 | /// Creates a storage by two JavaScript async functions. 31 | /// `read` must be of type `(key: string) => Promise`; 32 | /// `write` must be of type `(key: string, value: string) => Promise`. 33 | #[wasm_bindgen(constructor)] 34 | pub fn new(read: Option, write: Option) -> Self { 35 | JsStorage { read, write } 36 | } 37 | } 38 | 39 | #[async_trait(?Send)] 40 | impl Storage for JsStorage { 41 | async fn read(&self, k: &str) -> Result> { 42 | if let Some(read) = &self.read { 43 | let k = JsValue::from_str(k); 44 | let v = await_js_promise(read.call1(&JsValue::NULL, &k)).await?; 45 | if v.is_null() { 46 | return Ok(None); 47 | } 48 | let v = v 49 | .as_string() 50 | .ok_or_else(|| anyhow!("Expecting JavaScript function to return a string"))?; 51 | Ok(Some(v)) 52 | } else { 53 | Ok(None) 54 | } 55 | } 56 | async fn write(&self, k: &str, v: &str) -> Result<()> { 57 | if let Some(write) = &self.write { 58 | let k = JsValue::from_str(k); 59 | let v = JsValue::from_str(v); 60 | await_js_promise(write.call2(&JsValue::NULL, &k, &v)).await?; 61 | } 62 | Ok(()) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /sxg_rs/src/storage/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #[cfg(feature = "wasm")] 16 | pub mod js_storage; 17 | 18 | use crate::utils::{MaybeSend, MaybeSync}; 19 | use anyhow::Result; 20 | use async_trait::async_trait; 21 | use std::collections::HashMap; 22 | use std::sync::Arc; 23 | use tokio::sync::RwLock; 24 | 25 | #[cfg_attr(feature = "wasm", async_trait(?Send))] 26 | #[cfg_attr(not(feature = "wasm"), async_trait)] 27 | pub trait Storage: MaybeSend + MaybeSync { 28 | async fn read(&self, k: &str) -> Result>; 29 | async fn write(&self, k: &str, v: &str) -> Result<()>; 30 | } 31 | 32 | pub struct InMemoryStorage(Arc>>); 33 | 34 | impl InMemoryStorage { 35 | pub fn new() -> Self { 36 | InMemoryStorage(Arc::new(RwLock::new(HashMap::new()))) 37 | } 38 | } 39 | 40 | #[cfg_attr(feature = "wasm", async_trait(?Send))] 41 | #[cfg_attr(not(feature = "wasm"), async_trait)] 42 | impl Storage for InMemoryStorage { 43 | async fn read(&self, k: &str) -> Result> { 44 | let guard = self.0.read().await; 45 | Ok(guard.get(k).map(|v| v.to_string())) 46 | } 47 | async fn write(&self, k: &str, v: &str) -> Result<()> { 48 | let mut guard = self.0.write().await; 49 | guard.insert(k.to_string(), v.to_string()); 50 | Ok(()) 51 | } 52 | } 53 | 54 | impl std::default::Default for InMemoryStorage { 55 | fn default() -> Self { 56 | Self::new() 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /sxg_rs/src/structured_header/item.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::borrow::Cow; 16 | use std::fmt; 17 | 18 | #[derive(Debug, PartialEq, Eq)] 19 | pub enum ShItem<'a> { 20 | ByteSequence(Cow<'a, [u8]>), 21 | Integer(i64), 22 | String(Cow<'a, str>), 23 | } 24 | 25 | // should be https://tools.ietf.org/html/draft-ietf-httpbis-header-structure-10#section-4.1.5 26 | impl<'a> fmt::Display for ShItem<'a> { 27 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 28 | match self { 29 | ShItem::ByteSequence(bytes) => { 30 | write!(f, "*{}*", ::base64::encode(bytes)) 31 | } 32 | ShItem::Integer(x) => write!(f, "{}", x), 33 | ShItem::String(x) => { 34 | write!(f, "\"")?; 35 | for c in x.chars() { 36 | match c { 37 | '\\' | '\"' => { 38 | write!(f, "\\{}", c)?; 39 | } 40 | '\u{20}'..='\u{21}' | '\u{23}'..='\u{5b}' | '\u{5d}'..='\u{7e}' => { 41 | write!(f, "{}", c)?; 42 | } 43 | '\u{0}'..='\u{1f}' | '\u{7f}'..='\u{10ffff}' => { 44 | return Err(std::fmt::Error); 45 | } 46 | }; 47 | } 48 | write!(f, "\"") 49 | } 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /sxg_rs/src/structured_header/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // https://tools.ietf.org/html/draft-ietf-httpbis-header-structure-10#section-3.1 16 | 17 | mod item; 18 | mod parameterised_list; 19 | 20 | pub use item::ShItem; 21 | pub use parameterised_list::{ParamItem, ShParamList}; 22 | -------------------------------------------------------------------------------- /sxg_rs/src/structured_header/parameterised_list.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::borrow::Cow; 16 | use std::fmt; 17 | use std::ops::{Deref, DerefMut}; 18 | 19 | use super::ShItem; 20 | 21 | #[derive(Debug)] 22 | pub struct ParamItem<'a> { 23 | pub primary_id: Cow<'a, str>, 24 | pub parameters: Vec<(Cow<'a, str>, Option>)>, 25 | } 26 | 27 | #[derive(Debug)] 28 | pub struct ShParamList<'a>(pub Vec>); 29 | 30 | impl<'a> ParamItem<'a> { 31 | pub fn new(primary_id: &'a str) -> Self { 32 | ParamItem { 33 | primary_id: primary_id.into(), 34 | parameters: Vec::new(), 35 | } 36 | } 37 | } 38 | 39 | impl<'a> ShParamList<'a> { 40 | pub fn new() -> Self { 41 | ShParamList(Vec::new()) 42 | } 43 | } 44 | 45 | impl<'a> Default for ShParamList<'a> { 46 | fn default() -> Self { 47 | Self::new() 48 | } 49 | } 50 | 51 | impl<'a> Deref for ParamItem<'a> { 52 | type Target = Vec<(Cow<'a, str>, Option>)>; 53 | fn deref(&self) -> &Self::Target { 54 | &self.parameters 55 | } 56 | } 57 | 58 | impl<'a> DerefMut for ParamItem<'a> { 59 | fn deref_mut(&mut self) -> &mut Self::Target { 60 | &mut self.parameters 61 | } 62 | } 63 | 64 | impl<'a> Deref for ShParamList<'a> { 65 | type Target = Vec>; 66 | fn deref(&self) -> &Self::Target { 67 | &self.0 68 | } 69 | } 70 | 71 | impl<'a> DerefMut for ShParamList<'a> { 72 | fn deref_mut(&mut self) -> &mut Self::Target { 73 | &mut self.0 74 | } 75 | } 76 | 77 | // https://tools.ietf.org/html/draft-ietf-httpbis-header-structure-10#section-4.1.4 78 | impl<'a> fmt::Display for ShParamList<'a> { 79 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 80 | for (i, mem) in self.0.iter().enumerate() { 81 | write!(f, "{}", mem.primary_id)?; 82 | for (name, value) in mem.parameters.iter() { 83 | write!(f, ";")?; 84 | write!(f, "{}", name)?; 85 | if let Some(value) = value { 86 | write!(f, "={}", value)?; 87 | } 88 | } 89 | if i < self.0.len() - 1 { 90 | write!(f, ", ")?; 91 | } 92 | } 93 | Ok(()) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /sxg_rs/src/sxg.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use anyhow::{Error, Result}; 16 | 17 | // https://wicg.github.io/webpackage/draft-yasskin-httpbis-origin-signed-exchanges-impl.html#application-signed-exchange 18 | pub fn build( 19 | fallback_url: &str, 20 | signature: &[u8], 21 | signed_headers: &[u8], 22 | payload_body: &[u8], 23 | ) -> Result> { 24 | // https://wicg.github.io/webpackage/draft-yasskin-http-origin-signed-responses.html#name-application-signed-exchange 25 | const SIG_LEN_LIMIT: usize = 16384; 26 | let sig_len = signature.len(); 27 | if sig_len > SIG_LEN_LIMIT { 28 | return Err(Error::msg(format!( 29 | "sigLength {} is and larger than the limit {}", 30 | sig_len, SIG_LEN_LIMIT 31 | ))); 32 | } 33 | const HEADER_LEN_LIMIT: usize = 524288; 34 | let header_len = signed_headers.len(); 35 | if header_len > HEADER_LEN_LIMIT { 36 | return Err(Error::msg(format!( 37 | "headerLength {} is larger than the limit {}", 38 | header_len, HEADER_LEN_LIMIT 39 | ))); 40 | } 41 | Ok([ 42 | "sxg1-b3\0".as_bytes(), 43 | &(fallback_url.len() as u16).to_be_bytes(), 44 | fallback_url.as_bytes(), 45 | (sig_len as u32).to_be_bytes().get(1..4).unwrap(), 46 | (header_len as u32).to_be_bytes().get(1..4).unwrap(), 47 | signature, 48 | signed_headers, 49 | payload_body, 50 | ] 51 | .concat()) 52 | } 53 | -------------------------------------------------------------------------------- /tools/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "tools" 17 | version = "0.1.0" 18 | edition = "2018" 19 | 20 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 21 | 22 | [dependencies] 23 | anyhow = "1.0.66" 24 | async-trait = "0.1.59" 25 | base64 = "0.13.1" 26 | clap = { version = "3.2.23", features = ["derive"] } 27 | ctrlc = "3.2.3" 28 | der-parser = { version = "7.0.0", features = ["bigint", "serialize"] } 29 | http = "0.2.8" 30 | hyper = { version = "0.14.23", features = ["client", "http2"]} 31 | hyper-tls = "0.5.0" 32 | pem = "1.1.0" 33 | regex = "1.7.0" 34 | serde = { version = "1.0.149", features = ["derive"] } 35 | serde_json = "1.0.89" 36 | serde_yaml = "0.9.14" 37 | sxg_rs = { path = "../sxg_rs", features = ["rust_signer"] } 38 | toml = "0.5.9" 39 | tokio = { version = "1.23.0", features = ["full"] } 40 | url = "2.3.1" 41 | warp = "0.3.3" 42 | wrangler = "1.19.13" 43 | 44 | [features] 45 | # Unsupported, but necessary to make `cargo some-cmd --all-features` happy. 46 | wasm = ["sxg_rs/wasm"] 47 | -------------------------------------------------------------------------------- /tools/src/commands/apply_acme_cert.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use super::gen_config::read_artifact; 16 | use crate::runtime::hyper_fetcher::HyperFetcher; 17 | use anyhow::Result; 18 | use clap::Parser; 19 | use sxg_rs::acme::state_machine::{ 20 | get_challenge_token_and_answer, update_state as update_acme_state_machine, 21 | }; 22 | use warp::Filter; 23 | 24 | #[derive(Debug, Parser)] 25 | #[clap(allow_hyphen_values = true)] 26 | pub struct Opts { 27 | #[clap(long)] 28 | port: Option, 29 | #[clap(long)] 30 | artifact: String, 31 | /// Puts challenge answer and certificate to Fastly edge dictionary. 32 | #[clap(long)] 33 | use_fastly_dictionary: bool, 34 | } 35 | 36 | fn start_warp_server(port: u16, answer: impl ToString) -> tokio::sync::oneshot::Sender<()> { 37 | let answer = answer.to_string(); 38 | let (tx, rx) = tokio::sync::oneshot::channel(); 39 | let routes = 40 | warp::path!(".well-known" / "acme-challenge" / String).map(move |_name| answer.to_string()); 41 | let (_addr, server) = 42 | warp::serve(routes).bind_with_graceful_shutdown(([127, 0, 0, 1], port), async { 43 | rx.await.ok(); 44 | }); 45 | tokio::spawn(server); 46 | tx 47 | } 48 | 49 | pub async fn main(opts: Opts) -> Result<()> { 50 | let artifact = read_artifact(&opts.artifact)?; 51 | let acme_account = artifact.acme_account.unwrap(); 52 | let acme_private_key = artifact.acme_private_key.unwrap(); 53 | let mut runtime = sxg_rs::runtime::Runtime { 54 | acme_signer: Box::new(acme_private_key.create_signer()?), 55 | fetcher: Box::new(HyperFetcher::new()), 56 | ..Default::default() 57 | }; 58 | let (challenge_token, challenge_answer) = loop { 59 | runtime.now = std::time::SystemTime::now(); 60 | update_acme_state_machine(&runtime, &acme_account).await?; 61 | if let Some((token, answer)) = get_challenge_token_and_answer(&runtime).await? { 62 | break (token, answer); 63 | } 64 | tokio::time::sleep(std::time::Duration::from_secs(1)).await; 65 | }; 66 | 67 | let challenge_url = format!( 68 | "http://{}/.well-known/acme-challenge/{}", 69 | acme_account.domain, challenge_token 70 | ); 71 | let graceful_shutdown = if let Some(port) = opts.port { 72 | println!("Serving ACME challenge answer on local port {}, assuming that this port is binding to http://{}/", port, acme_account.domain); 73 | Some(start_warp_server(port, &challenge_answer)) 74 | } else if opts.use_fastly_dictionary { 75 | println!("Writing ACME challenge answer to Fastly edige dictionary."); 76 | let acme_state = 77 | sxg_rs::acme::state_machine::create_from_challenge(&challenge_token, &challenge_answer); 78 | super::gen_config::fastly::update_dictionary_item( 79 | artifact.fastly_service_id.as_ref().unwrap(), 80 | artifact.fastly_dictionary_id.as_ref().unwrap(), 81 | sxg_rs::acme::state_machine::ACME_STORAGE_KEY, 82 | &serde_json::to_string(&acme_state)?, 83 | )?; 84 | None 85 | } else { 86 | println!( 87 | "\ 88 | Please create a file in your HTTP server to serve the following URL.\n\ 89 | URL:\n\ 90 | {}\n\ 91 | Plain-text content:\n\ 92 | {}\n\ 93 | ", 94 | challenge_url, challenge_answer 95 | ); 96 | None 97 | }; 98 | 99 | println!( 100 | "Waiting for the propagation of ACME challenge answer; checking every 10 seconds from {}.", 101 | challenge_url 102 | ); 103 | loop { 104 | let url = format!( 105 | "http://{}/.well-known/acme-challenge/{}", 106 | acme_account.domain, challenge_token 107 | ); 108 | let actual_response = sxg_rs::fetcher::get(runtime.fetcher.as_ref(), &url).await?; 109 | if let Ok(actual_response) = String::from_utf8(actual_response) { 110 | if actual_response.trim() == challenge_answer { 111 | println!("ACME challenge answer succesfully detected."); 112 | break; 113 | } 114 | } 115 | print!("."); 116 | tokio::time::sleep(std::time::Duration::from_secs(10)).await; 117 | } 118 | 119 | let certificate_pem = loop { 120 | runtime.now = std::time::SystemTime::now(); 121 | update_acme_state_machine(&runtime, &acme_account).await?; 122 | let state = sxg_rs::acme::state_machine::read_current_state(&runtime).await?; 123 | if let Some(cert) = state.certificates.last() { 124 | break cert.clone(); 125 | } 126 | tokio::time::sleep(std::time::Duration::from_secs(1)).await; 127 | }; 128 | if opts.use_fastly_dictionary { 129 | println!("Uploading certificates to Fastly edge dicionary."); 130 | let acme_state = sxg_rs::acme::state_machine::create_from_certificate(certificate_pem); 131 | super::gen_config::fastly::update_dictionary_item( 132 | artifact.fastly_service_id.as_ref().unwrap(), 133 | artifact.fastly_dictionary_id.as_ref().unwrap(), 134 | sxg_rs::acme::state_machine::ACME_STORAGE_KEY, 135 | &serde_json::to_string(&acme_state)?, 136 | )?; 137 | } else { 138 | println!("{}", certificate_pem); 139 | } 140 | if let Some(tx) = graceful_shutdown { 141 | let _ = tx.send(()); 142 | } 143 | Ok(()) 144 | } 145 | -------------------------------------------------------------------------------- /tools/src/commands/gen_config/http_server.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use sxg_rs::config::Config as SxgConfig; 3 | use sxg_rs::crypto::EcPrivateKey; 4 | 5 | const CONFIG_YAML: &str = "http_server/config.yaml"; 6 | const PRIVKEY_PEM: &str = "credentials/privkey.pem"; 7 | 8 | pub(crate) fn main(mut config: SxgConfig) -> Result<()> { 9 | config.private_key_base64 = Some(base64::encode( 10 | EcPrivateKey::from_sec1_pem(&std::fs::read_to_string(PRIVKEY_PEM)?)?.d, 11 | )); 12 | std::fs::write( 13 | CONFIG_YAML, 14 | format!( 15 | "# This file is generated by command \"cargo run -p tools -- gen-config\".\n\ 16 | # Please note that anything you modify won't be preserved\n\ 17 | # at the next time you run \"cargo run -p tools -- -gen-config\".\n\ 18 | {}", 19 | serde_yaml::to_string(&config)?, 20 | ), 21 | )?; 22 | Ok(()) 23 | } 24 | -------------------------------------------------------------------------------- /tools/src/commands/gen_dev_cert.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use anyhow::Result; 16 | use clap::Parser; 17 | use std::fs::write; 18 | 19 | use crate::linux_commands::{ 20 | create_certificate, create_certificate_request_pem, get_certificate_sha256, 21 | read_or_create_private_key_pem, write_new_file, 22 | }; 23 | 24 | #[derive(Parser)] 25 | pub struct Opts { 26 | #[clap(long)] 27 | domain: String, 28 | } 29 | 30 | pub fn main(opts: Opts) -> Result<()> { 31 | const PRIVKEY_FILE: &str = "privkey.pem"; 32 | const CERT_CSR_FILE: &str = "cert.csr"; 33 | const EXT_FILE: &str = "ext.txt"; 34 | const CERT_FILE: &str = "cert.pem"; 35 | const ISSUER_FILE: &str = "issuer.pem"; 36 | const CERT_SHA256_FILE: &str = "cert_sha256.txt"; 37 | read_or_create_private_key_pem(PRIVKEY_FILE)?; 38 | create_certificate_request_pem(&opts.domain, PRIVKEY_FILE, CERT_CSR_FILE)?; 39 | write( 40 | EXT_FILE, 41 | format!( 42 | "1.3.6.1.4.1.11129.2.1.22 = ASN1:NULL\nsubjectAltName=DNS:{}\n", 43 | &opts.domain, 44 | ), 45 | )?; 46 | let cert_pem = create_certificate(PRIVKEY_FILE, CERT_CSR_FILE, EXT_FILE, CERT_FILE)?; 47 | write_new_file(ISSUER_FILE, &cert_pem)?; 48 | write_new_file( 49 | CERT_SHA256_FILE, 50 | base64::encode_config(&get_certificate_sha256(CERT_FILE)?, base64::URL_SAFE_NO_PAD), 51 | )?; 52 | Ok(()) 53 | } 54 | -------------------------------------------------------------------------------- /tools/src/commands/gen_sxg.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use anyhow::Result; 16 | use clap::Parser; 17 | use std::fs; 18 | use sxg_rs::{ 19 | crypto::CertificateChain, fetcher::NULL_FETCHER, http_cache::NullCache, 20 | CreateSignedExchangeParams, SxgWorker, 21 | }; 22 | 23 | // TODO: Make this binary generally useful, by documenting the flags and giving them names. 24 | 25 | #[derive(Parser)] 26 | pub struct Opts { 27 | config_yaml: String, 28 | cert_pem: String, 29 | issuer_pem: String, 30 | out_cert_cbor: String, 31 | out_sxg: String, 32 | } 33 | 34 | pub async fn main(opts: Opts) -> Result<()> { 35 | let mut worker = SxgWorker::new(&fs::read_to_string(opts.config_yaml).unwrap()).unwrap(); 36 | let certificate = CertificateChain::from_pem_files(&[ 37 | &fs::read_to_string(opts.cert_pem).unwrap(), 38 | &fs::read_to_string(opts.issuer_pem).unwrap(), 39 | ])?; 40 | worker.add_certificate(certificate); 41 | fs::write( 42 | opts.out_cert_cbor, 43 | &worker.create_cert_cbor( 44 | worker.latest_certificate_basename().unwrap(), 45 | // TODO: Use a real OCSP 46 | b"ocsp", 47 | ), 48 | )?; 49 | let payload_headers = worker 50 | .transform_payload_headers(vec![("content-type".into(), "text/html".into())]) 51 | .unwrap(); 52 | let runtime = sxg_rs::runtime::Runtime { 53 | now: std::time::SystemTime::now(), 54 | sxg_signer: Box::new(worker.create_rust_signer().unwrap()), 55 | fetcher: Box::new(NULL_FETCHER), 56 | ..Default::default() 57 | }; 58 | let sxg = worker.create_signed_exchange( 59 | &runtime, 60 | CreateSignedExchangeParams { 61 | fallback_url: "https://test.example/", 62 | cert_origin: "https://test.example", 63 | payload_body: b"This is a test.", 64 | payload_headers, 65 | skip_process_link: false, 66 | status_code: 200, 67 | header_integrity_cache: NullCache {}, 68 | }, 69 | ); 70 | let sxg = sxg.await; 71 | fs::write(opts.out_sxg, &sxg.unwrap().body)?; 72 | Ok(()) 73 | } 74 | -------------------------------------------------------------------------------- /tools/src/commands/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | mod apply_acme_cert; 16 | mod gen_config; 17 | mod gen_dev_cert; 18 | mod gen_sxg; 19 | 20 | use super::tokio_block_on as block_on; 21 | use anyhow::Result; 22 | use clap::Parser; 23 | 24 | #[derive(Parser)] 25 | enum SubCommand { 26 | ApplyAcmeCert(apply_acme_cert::Opts), 27 | GenConfig(gen_config::Opts), 28 | GenDevCert(gen_dev_cert::Opts), 29 | GenSxg(gen_sxg::Opts), 30 | } 31 | 32 | #[derive(Parser)] 33 | struct Opts { 34 | #[clap(subcommand)] 35 | sub_command: SubCommand, 36 | } 37 | 38 | pub fn main() -> Result<()> { 39 | match Opts::parse().sub_command { 40 | SubCommand::ApplyAcmeCert(opts) => block_on(apply_acme_cert::main(opts)), 41 | SubCommand::GenConfig(opts) => gen_config::main(opts), 42 | SubCommand::GenSxg(opts) => block_on(gen_sxg::main(opts)), 43 | SubCommand::GenDevCert(opts) => gen_dev_cert::main(opts), 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /tools/src/lib.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | use std::collections::BTreeMap; 3 | use sxg_rs::acme::Account as AcmeAccount; 4 | use sxg_rs::crypto::EcPrivateKey; 5 | 6 | #[derive(Debug, Default, Deserialize, Serialize)] 7 | pub struct Artifact { 8 | pub acme_account: Option, 9 | pub acme_private_key: Option, 10 | pub acme_private_key_instructions: BTreeMap, 11 | pub cloudflare_kv_namespace_id: Option, 12 | pub fastly_service_id: Option, 13 | pub fastly_dictionary_id: Option, 14 | } 15 | -------------------------------------------------------------------------------- /tools/src/linux_commands.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use anyhow::{Error, Result}; 16 | use std::path::Path; 17 | use std::process::Command; 18 | 19 | /// Executes a command, checks the exit code, and returns the stdout as bytes. 20 | pub fn execute(command: &mut Command) -> Result> { 21 | let output = command 22 | .output() 23 | .map_err(|e| Error::new(e).context(format!("Failed to execute command {:?}", command)))?; 24 | if output.status.success() { 25 | Ok(output.stdout) 26 | } else { 27 | let stderr = String::from_utf8_lossy(&output.stderr).to_string(); 28 | Err(Error::msg(stderr).context(format!( 29 | "Command {:?} exited with non-succesful status", 30 | command, 31 | ))) 32 | } 33 | } 34 | 35 | /// Executes a command, and parses the stdout as a string. 36 | pub fn execute_and_parse_stdout(command: &mut Command) -> Result { 37 | let stdout = execute(command)?; 38 | String::from_utf8(stdout) 39 | .map_err(|e| Error::new(e).context("The stdout contains non-utf8 bytes.")) 40 | } 41 | 42 | /// Writes content into a new file. 43 | /// Returns error if a file already exists. 44 | pub fn write_new_file(path: impl AsRef, content: impl AsRef<[u8]>) -> Result<()> { 45 | let path = path.as_ref(); 46 | if path.exists() { 47 | Err(Error::msg(format!( 48 | "Cowardly refuse to overwrite {:?}", 49 | path 50 | ))) 51 | } else { 52 | std::fs::write(path, content)?; 53 | Ok(()) 54 | } 55 | } 56 | 57 | /// Generates a private key, and returns it without writing to any files. 58 | /// Care should be taken to prevent the private key being lost. 59 | pub fn generate_private_key_pem() -> Result { 60 | execute_and_parse_stdout( 61 | Command::new("openssl") 62 | .arg("ecparam") 63 | .arg("-outform") 64 | .arg("pem") 65 | .arg("-name") 66 | .arg("prime256v1") 67 | .arg("-genkey"), 68 | ) 69 | .map_err(|e| e.context("Failed to use openssl to generate private key")) 70 | } 71 | 72 | /// Tries to read the contents of given file; if the file does not exist, 73 | /// generates a private key, and writes PEM to the file, and returns it. 74 | pub fn read_or_create_private_key_pem(file: impl AsRef) -> Result { 75 | if file.as_ref().exists() { 76 | println!("Reading private key from file {:?}", file.as_ref()); 77 | std::fs::read_to_string(file).map_err(Error::new) 78 | } else { 79 | let privkey_pem = generate_private_key_pem()?; 80 | println!( 81 | "Writing private key to file {:?}, please keep it in a safe place.", 82 | file.as_ref() 83 | ); 84 | write_new_file(file, &privkey_pem)?; 85 | Ok(privkey_pem) 86 | } 87 | } 88 | 89 | /// Generates a certificate request, and returns it in PEM format. 90 | /// Writes PEM to `output_file`. 91 | /// Overwrites if `output_file` already exists. 92 | pub fn create_certificate_request_pem( 93 | domain: &str, 94 | private_key_file: impl AsRef, 95 | output_file: impl AsRef, 96 | ) -> Result { 97 | let cert_csr_pem = execute_and_parse_stdout( 98 | Command::new("openssl") 99 | .arg("req") 100 | .arg("-new") 101 | .arg("-sha256") 102 | .arg("-key") 103 | .arg(private_key_file.as_ref().as_os_str()) 104 | .arg("-subj") 105 | .arg(format!("/CN={}/O=Test/C=US", domain)), 106 | )?; 107 | std::fs::write(output_file, &cert_csr_pem)?; 108 | Ok(cert_csr_pem) 109 | } 110 | 111 | /// Create a certificate by signing the certificate request file 112 | /// by the private key, 113 | /// and returns the certificate in PEM format. 114 | /// Writes PEM to `output_file`. 115 | /// Returns error if `output_file` already exists. 116 | pub fn create_certificate( 117 | private_key_file: impl AsRef, 118 | certificiate_request_file: impl AsRef, 119 | ext_file: impl AsRef, 120 | output_file: impl AsRef, 121 | ) -> Result { 122 | let cert_pem = execute_and_parse_stdout( 123 | Command::new("openssl") 124 | .arg("x509") 125 | .arg("-req") 126 | .arg("-days") 127 | .arg("90") 128 | .arg("-in") 129 | .arg(certificiate_request_file.as_ref().as_os_str()) 130 | .arg("-signkey") 131 | .arg(private_key_file.as_ref().as_os_str()) 132 | .arg("-extfile") 133 | .arg(ext_file.as_ref().as_os_str()), 134 | )?; 135 | write_new_file(output_file, &cert_pem)?; 136 | Ok(cert_pem) 137 | } 138 | 139 | pub fn get_certificate_sha256(certificate_file: impl AsRef) -> Result> { 140 | let public_key_pem = execute_and_parse_stdout( 141 | Command::new("openssl") 142 | .arg("x509") 143 | .arg("-pubkey") 144 | .arg("-noout") 145 | .arg("-in") 146 | .arg(certificate_file.as_ref().as_os_str()), 147 | )?; 148 | let public_key_der = sxg_rs::crypto::get_der_from_pem(&public_key_pem, "PUBLIC KEY")?; 149 | Ok(sxg_rs::crypto::HashAlgorithm::Sha256.digest(&public_key_der)) 150 | } 151 | -------------------------------------------------------------------------------- /tools/src/main.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | mod commands; 16 | mod linux_commands; 17 | mod runtime; 18 | 19 | use anyhow::Result; 20 | use std::future::Future; 21 | 22 | fn tokio_block_on(f: F) -> ::Output 23 | where 24 | F: Future, 25 | { 26 | tokio::runtime::Builder::new_current_thread() 27 | .enable_all() 28 | .build() 29 | .unwrap() 30 | .block_on(f) 31 | } 32 | 33 | fn main() -> Result<()> { 34 | commands::main() 35 | } 36 | -------------------------------------------------------------------------------- /tools/src/runtime/hyper_fetcher.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use hyper::{client::connect::HttpConnector, Client}; 16 | use hyper_tls::HttpsConnector; 17 | 18 | use anyhow::{Error, Result}; 19 | use async_trait::async_trait; 20 | use std::convert::TryInto; 21 | use sxg_rs::fetcher::Fetcher; 22 | use sxg_rs::http::{HttpRequest as SxgRsRequest, HttpResponse as SxgRsResponse}; 23 | 24 | /// A [`Fetcher`] implemented by the external `hyper` crate. 25 | pub struct HyperFetcher { 26 | client: Client>, 27 | } 28 | 29 | impl HyperFetcher { 30 | pub fn new() -> Self { 31 | let https = HttpsConnector::new(); 32 | HyperFetcher { 33 | client: Client::builder().build(https), 34 | } 35 | } 36 | } 37 | 38 | impl Default for HyperFetcher { 39 | fn default() -> Self { 40 | Self::new() 41 | } 42 | } 43 | 44 | #[cfg_attr(feature = "wasm", async_trait(?Send))] 45 | #[cfg_attr(not(feature = "wasm"), async_trait)] 46 | impl Fetcher for HyperFetcher { 47 | async fn fetch(&self, request: SxgRsRequest) -> Result { 48 | let request: http::Request> = request 49 | .try_into() 50 | .map_err(|e: Error| e.context("Failed to convert sxg_rs::Request to http::Request"))?; 51 | let request: http::Request = request.map(|body| body.into()); 52 | let response = self 53 | .client 54 | .request(request) 55 | .await 56 | .map_err(|e| Error::new(e).context("Failed to request by Hyper client"))?; 57 | let (response_parts, response_body) = response.into_parts(); 58 | let response_body = hyper::body::to_bytes(response_body) 59 | .await 60 | .map_err(|e| Error::new(e).context("Failed to convert response body to bytes"))?; 61 | let response_body = response_body.into_iter().collect(); 62 | let response = http::Response::from_parts(response_parts, response_body); 63 | response.try_into() 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /tools/src/runtime/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub mod hyper_fetcher; 16 | pub mod openssl_signer; 17 | -------------------------------------------------------------------------------- /tools/src/runtime/openssl_signer.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use crate::linux_commands::{execute, execute_and_parse_stdout}; 16 | use anyhow::{anyhow, Result}; 17 | use async_trait::async_trait; 18 | use std::process::Command; 19 | use sxg_rs::signature::{Format as SignatureFormat, Signer}; 20 | 21 | #[derive(Debug)] 22 | pub enum OpensslSigner<'a> { 23 | Hmac(&'a [u8]), 24 | } 25 | 26 | #[cfg_attr(feature = "wasm", async_trait(?Send))] 27 | #[cfg_attr(not(feature = "wasm"), async_trait)] 28 | impl<'a> Signer for OpensslSigner<'a> { 29 | async fn sign(&self, message: &[u8], format: SignatureFormat) -> Result> { 30 | let tmp_file = execute_and_parse_stdout(&mut Command::new("mktemp"))?; 31 | let tmp_file = tmp_file.trim(); 32 | std::fs::write(tmp_file, message)?; 33 | match self { 34 | OpensslSigner::Hmac(private_key) => { 35 | let hexkey = private_key 36 | .iter() 37 | .map(|x| format!("{:02x}", x)) 38 | .collect::>() 39 | .join(""); 40 | let sig = execute( 41 | Command::new("openssl") 42 | .arg("dgst") 43 | .arg("-sha256") 44 | .arg("-mac") 45 | .arg("HMAC") 46 | .arg("-binary") 47 | .arg("-macopt") 48 | .arg(format!("hexkey:{}", hexkey)) 49 | .arg(tmp_file), 50 | ) 51 | .map_err(|e| e.context("Failed to use openssl to create HMAC"))?; 52 | match format { 53 | SignatureFormat::Raw => Ok(sig), 54 | SignatureFormat::EccAsn1 => { 55 | Err(anyhow!("HMAC signature can't be formatted as EccAsn1.")) 56 | } 57 | } 58 | } 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /typescript_utilities/.gitignore: -------------------------------------------------------------------------------- 1 | /dist 2 | /node_modules 3 | -------------------------------------------------------------------------------- /typescript_utilities/README.md: -------------------------------------------------------------------------------- 1 | This folder contains some glue code to use `sxg-rs` in TypeScript, including 2 | * TypeScript definitions of the worker interface. 3 | * TypeScript code implementing Rust `Signer` trait. 4 | -------------------------------------------------------------------------------- /typescript_utilities/build.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | node_modules/.bin/esbuild src/*.test.ts --bundle --platform=browser --outdir=dist 3 | -------------------------------------------------------------------------------- /typescript_utilities/karma.conf.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | module.exports = function (config) { 18 | config.set({ 19 | basePath: '', 20 | frameworks: ['jasmine'], 21 | files: ['dist/**/*.test.js'], 22 | colors: true, 23 | logLevel: config.LOG_INFO, 24 | autoWatch: false, 25 | browsers: ['ChromeHeadless'], 26 | singleRun: false, 27 | concurrency: Infinity, 28 | }); 29 | }; 30 | -------------------------------------------------------------------------------- /typescript_utilities/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ts_utils_for_sxg_rs", 3 | "version": "0.1.0", 4 | "scripts": { 5 | "build": "./build.sh", 6 | "test": "karma start --single-run", 7 | "lint": "gts lint" 8 | }, 9 | "devDependencies": { 10 | "@types/jasmine": "4.3.1", 11 | "esbuild": "0.16.14", 12 | "gts": "3.1.1", 13 | "jasmine-core": "4.5.0", 14 | "karma": "6.4.1", 15 | "karma-chrome-launcher": "3.1.1", 16 | "karma-jasmine": "5.1.0" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /typescript_utilities/spec/support/jasmine.json: -------------------------------------------------------------------------------- 1 | { 2 | "spec_dir": "dist", 3 | "spec_files": [ 4 | "**/*.test.js" 5 | ], 6 | "stopSpecOnExpectationFailure": false, 7 | "random": true 8 | } 9 | -------------------------------------------------------------------------------- /typescript_utilities/src/signer.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2022 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export type Signer = (message: Uint8Array) => Promise; 18 | 19 | // Uses crypto subtle to create a Signer from a JWK private key. 20 | // TODO: give `subtle` parameter a type that is recognized 21 | // in both NodeJs and Browser. 22 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 23 | export function fromJwk(subtle: any, jwk: object | null): Signer { 24 | if (jwk === null) { 25 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 26 | return async function signer(_message: Uint8Array): Promise { 27 | console.error( 28 | 'Creating an empty signature, because the wrangler secret PRIVATE_KEY_JWK is not set' 29 | ); 30 | return new Uint8Array(64); 31 | }; 32 | } 33 | const privateKeyPromise = (async function initPrivateKey() { 34 | return await subtle.importKey( 35 | 'jwk', 36 | jwk, 37 | { 38 | name: 'ECDSA', 39 | namedCurve: 'P-256', 40 | }, 41 | /*extractable=*/ false, 42 | ['sign'] 43 | ); 44 | })(); 45 | return async function signer(message: Uint8Array): Promise { 46 | const privateKey = await privateKeyPromise; 47 | const signature = await subtle.sign( 48 | { 49 | name: 'ECDSA', 50 | hash: 'SHA-256', 51 | }, 52 | privateKey, 53 | message 54 | ); 55 | return new Uint8Array(signature); 56 | }; 57 | } 58 | -------------------------------------------------------------------------------- /typescript_utilities/src/streams.test.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import {readIntoArray} from './streams'; 18 | 19 | describe('readIntoArray', () => { 20 | it('reads a stream <= maxSize', async () => { 21 | const input = new Response('hello').body; 22 | const output = await readIntoArray(input, 1000); 23 | expect(output).toEqual(new TextEncoder().encode('hello')); 24 | }); 25 | it('reads an empty stream', async () => { 26 | const input = new Response('').body; 27 | const output = await readIntoArray(input, 1000); 28 | expect(output).toEqual(new TextEncoder().encode('')); 29 | }); 30 | it('reads a stream with two chunks', async () => { 31 | const {writable, readable} = new TransformStream(); 32 | const write = (async () => { 33 | const writer = writable.getWriter(); 34 | for (let i = 0; i < 2; i++) { 35 | await writer.write(new TextEncoder().encode('hello')); 36 | } 37 | await writer.close(); 38 | })(); 39 | const output = await readIntoArray(readable, 0x1000); 40 | expect(output).toEqual(new TextEncoder().encode('hellohello')); 41 | await write; 42 | }); 43 | it('errors if stream > maxSize', async () => { 44 | const input = new Response('hello').body; 45 | const output = await readIntoArray(input, 2); 46 | expect(output).toBe(null); 47 | }); 48 | it('errors if second chunk > maxSize', async () => { 49 | const {writable, readable} = new TransformStream(); 50 | const write = (async () => { 51 | const writer = writable.getWriter(); 52 | for (let i = 0; i < 2; i++) { 53 | await writer.write(new TextEncoder().encode('hello')); 54 | } 55 | // Don't close the writer; it'll be cancelled by readIntoArray. 56 | })(); 57 | const output = await readIntoArray(readable, 7); 58 | expect(output).toBe(null); 59 | await write; 60 | }); 61 | }); 62 | -------------------------------------------------------------------------------- /typescript_utilities/src/streams.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // SXGs larger than 8MB are not accepted by 18 | // https://github.com/google/webpackager/blob/main/docs/cache_requirements.md. 19 | export const PAYLOAD_SIZE_LIMIT = 8000000; 20 | 21 | // Calls process for each chunk from inputStream, up to maxSize. The last chunk 22 | // may extend beyond maxSize; process should handle this case. 23 | // 24 | // Returns true if inputStream's total byte length is <= maxSize. After the 25 | // promise resolves, the inputStream is closed and need not be canceled. 26 | // 27 | // (This function could be genericized to all TypedArrays, but no such 28 | // interface exists in TypeScript.) 29 | async function streamFrom( 30 | inputStream: ReadableStream, 31 | maxSize: number, 32 | process?: (currentPos: number, value: Uint8Array) => void 33 | ): Promise { 34 | const reader = inputStream.getReader(); 35 | let receivedSize = 0; 36 | for (;;) { 37 | const {value, done} = await reader.read(); 38 | if (value) { 39 | process?.(receivedSize, value); 40 | receivedSize += value.length; 41 | if (receivedSize > maxSize) { 42 | reader.releaseLock(); 43 | inputStream.cancel(); 44 | return false; 45 | } 46 | } 47 | if (done) { 48 | // This implies closed per 49 | // https://streams.spec.whatwg.org/#default-reader-read. 50 | return true; 51 | } 52 | } 53 | } 54 | 55 | // Consumes the input stream, and returns a byte array containing the first 56 | // size bytes, or null if there aren't enough bytes. 57 | export async function readArrayPrefix( 58 | inputStream: ReadableStream | null, 59 | size: number 60 | ): Promise { 61 | if (inputStream === null) { 62 | return new Uint8Array([]); 63 | } 64 | const received = new Uint8Array(size); 65 | const reachedEOS = await streamFrom( 66 | inputStream, 67 | size, 68 | (currentPos: number, value: Uint8Array) => { 69 | if (currentPos + value.length > size) { 70 | value = value.subarray(0, size - currentPos); 71 | } 72 | // value must be Uint8Array, or else this set() will overflow: 73 | received.set(value, currentPos); 74 | } 75 | ); 76 | return reachedEOS ? null : received; 77 | } 78 | 79 | // Consumes the input stream, and returns a byte array containing the data in 80 | // the input stream. Allocates about 2x the size of the stream (during the 81 | // transfer from discontiguous to contiguous memory). If the input stream 82 | // contains more bytes than maxSize, returns null. 83 | // 84 | // TODO: Consider reducing memory usage at the expense of increased CPU, by 85 | // allocating a contiguous buffer upfront and growing it exponentially as 86 | // necessary. It would be good to do so with a benchmark and an approximate 87 | // distribution of body sizes in the wild (e.g. from HTTP Archive). 88 | export async function readIntoArray( 89 | inputStream: ReadableStream | null, 90 | maxSize: number 91 | ): Promise { 92 | // NOTE: maxSize could be implemented more simply using TransformStream, but 93 | // non-identity TransformStreams don't work on Cloudflare Workers. (See 94 | // https://community.cloudflare.com/t/running-into-unimplemented-functionality/77343.) 95 | // Therefore, we cannot rely on Response.arrayBuffer() and must re-implement 96 | // https://streams.spec.whatwg.org/#readablestreamdefaultreader-read-all-bytes. 97 | // This is a rough port of blink::ScriptPromise::arrayBuffer 98 | // (https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/core/fetch/body.cc;l=186;drc=5539ecff898c79b0771340051d62bf81649e448d), 99 | // but using the variable-length chunks that the Streams API provides, rather 100 | // than 4K segments as defined by WTF::SharedBuffer::kSegmentSize 101 | // (https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/platform/wtf/shared_buffer.h;l=95;drc=5539ecff898c79b0771340051d62bf81649e448d). 102 | if (inputStream === null) { 103 | return new Uint8Array([]); 104 | } 105 | const segments: Uint8Array[] = []; 106 | let size = 0; 107 | const reachedEOS = await streamFrom( 108 | inputStream, 109 | maxSize, 110 | (_currentPos: number, value: Uint8Array) => { 111 | segments.push(value); 112 | size += value.length; 113 | } 114 | ); 115 | // End-of-stream was not reached before maxSize. 116 | if (!reachedEOS) { 117 | return null; 118 | } 119 | // Avoid copying to a new buffer if there's no need to concatenate. 120 | if (segments.length === 1) { 121 | return segments[0] as Uint8Array; 122 | } 123 | // Concatenate segments into a contiguous buffer. 124 | const buffer = new Uint8Array(size); 125 | let bufferPos = 0; 126 | segments.forEach(segment => { 127 | const end = Math.min(segment.length, size - bufferPos); 128 | buffer.set(segment.subarray(0, end), bufferPos); 129 | bufferPos += end; 130 | }); 131 | return buffer; 132 | } 133 | 134 | export function teeResponse(response: Response): [Response, Response] { 135 | const {body, headers, status} = response; 136 | const [body1, body2] = body?.tee() ?? [null, null]; 137 | return [ 138 | new Response(body1, {headers, status}), 139 | new Response(body2, {headers, status}), 140 | ]; 141 | } 142 | -------------------------------------------------------------------------------- /typescript_utilities/src/utils.test.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import {arrayBufferToBase64, escapeLinkParamValue} from './utils'; 18 | 19 | describe('arrayBufferToBase64', () => { 20 | it('works', () => { 21 | const a = new Uint8Array([1, 2, 3]); 22 | expect(arrayBufferToBase64(a.buffer)).toEqual('AQID'); 23 | }); 24 | }); 25 | 26 | describe('escapeLinkParamValue', () => { 27 | it('returns tokens as-is', () => { 28 | expect(escapeLinkParamValue('hello-world')).toEqual('hello-world'); 29 | }); 30 | it('quotes empty string', () => { 31 | expect(escapeLinkParamValue('')).toEqual('""'); 32 | }); 33 | it('quotes non-tokens', () => { 34 | expect(escapeLinkParamValue('hello world')).toEqual('"hello world"'); 35 | }); 36 | it('escapes "', () => { 37 | expect(escapeLinkParamValue('hello, "world"')).toEqual( 38 | String.raw`"hello, \"world\""` 39 | ); 40 | }); 41 | it('escapes \\', () => { 42 | expect(escapeLinkParamValue(String.raw`hello\world`)).toEqual( 43 | String.raw`"hello\\world"` 44 | ); 45 | }); 46 | it('returns null for non-representable strings', () => { 47 | expect(escapeLinkParamValue('👋🌎')).toEqual(null); 48 | }); 49 | }); 50 | -------------------------------------------------------------------------------- /typescript_utilities/src/utils.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // https://datatracker.ietf.org/doc/html/rfc7230#section-3.2.6 18 | export const TOKEN = /^[!#$%&'*+.^_`|~0-9a-zA-Z-]+$/; 19 | 20 | export function arrayBufferToBase64(buffer: ArrayBuffer): string { 21 | const data = Array.from(new Uint8Array(buffer)); 22 | const s = data.map(x => String.fromCharCode(x)).join(''); 23 | return btoa(s); 24 | } 25 | 26 | // Strings representable in a quoted-string, per 27 | // https://datatracker.ietf.org/doc/html/rfc7230#section-3.2.6 (including \x22 28 | // ["] and \x5C [\]).. 29 | const ALLOWED_QUOTED_STRING_VALUE = /^[\t \x21-\x7E]*$/; 30 | 31 | // Escapes a value for use as a Link header param, per 32 | // https://datatracker.ietf.org/doc/html/rfc8288#section-3. 33 | export function escapeLinkParamValue(value: string): string | null { 34 | if (value.match(TOKEN)) { 35 | return value; 36 | } else if (value.match(ALLOWED_QUOTED_STRING_VALUE)) { 37 | return ( 38 | '"' + 39 | [...value] 40 | .map(char => (char === '\\' || char === '"' ? '\\' + char : char)) 41 | .join('') + 42 | '"' 43 | ); 44 | } else { 45 | return null; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /typescript_utilities/src/wasmFunctions.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // `wrangler` uses `wasm-pack build --target no-modules` [^1] to build wasm. 18 | // When the target is `no-modules`, `wasm-bindgen` declares a global variable 19 | // to initialize wasm [^2]. 20 | // The default name of this global variable is `wasm_bindgen` [^3]. 21 | // The example is here [^4]. 22 | // [^1] https://github.com/cloudflare/wrangler/blob/37caf3cb08db3e84fee4c503e1a08f849371c4b8/src/build/mod.rs#L48 23 | // [^2] https://github.com/rustwasm/wasm-bindgen/blob/dc9141e7ccd143e67a282cfa73717bb165049169/crates/cli/src/bin/wasm-bindgen.rs#L27 24 | // [^3] https://github.com/rustwasm/wasm-bindgen/blob/dc9141e7ccd143e67a282cfa73717bb165049169/crates/cli-support/src/lib.rs#L208 25 | // [^4] https://rustwasm.github.io/docs/wasm-bindgen/examples/without-a-bundler.html#using-the-older---target-no-modules 26 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 27 | declare let wasm_bindgen: any; 28 | 29 | type HeaderFields = Array<[string, string]>; 30 | 31 | export type AcceptLevel = 'RejectsSxg' | 'PrefersSxg' | 'AcceptsSxg'; 32 | 33 | export interface WasmRequest { 34 | body: number[]; 35 | headers: HeaderFields; 36 | method: 'Get' | 'Post'; 37 | url: string; 38 | } 39 | 40 | export interface WasmResponse { 41 | body: number[]; 42 | headers: HeaderFields; 43 | status: number; 44 | } 45 | 46 | export type PresetContent = 47 | | ({kind: 'direct'} & WasmResponse) 48 | | { 49 | kind: 'toBeSigned'; 50 | url: string; 51 | payload: WasmResponse; 52 | fallback: WasmResponse; 53 | }; 54 | 55 | export type JsRuntimeInitParams = { 56 | nowInSeconds: number; 57 | fetcher: ((request: WasmRequest) => Promise) | undefined; 58 | storageRead: ((k: string) => Promise) | undefined; 59 | storageWrite: ((k: string, v: string) => Promise) | undefined; 60 | sxgAsn1Signer: ((input: Uint8Array) => Promise) | undefined; 61 | sxgRawSigner: ((input: Uint8Array) => Promise) | undefined; 62 | acmeRawSigner: ((input: Uint8Array) => Promise) | undefined; 63 | }; 64 | 65 | export type CreateSignedExchangedOptions = { 66 | fallbackUrl: string; 67 | certOrigin: string; 68 | statusCode: number; 69 | payloadHeaders: HeaderFields; 70 | payloadBody: Uint8Array; 71 | skipProcessLink: boolean; 72 | headerIntegrityGet: (url: string) => Promise; 73 | headerIntegrityPut: (url: string, response: WasmResponse) => Promise; 74 | }; 75 | 76 | export interface ProcessHtmlOption { 77 | isSxg: boolean; 78 | } 79 | 80 | export interface WasmWorker { 81 | // eslint-disable-next-line @typescript-eslint/no-misused-new 82 | new (configYaml: string, certificatePem: string | undefined): WasmWorker; 83 | addAcmeCertificatesFromStorage(runtime: JsRuntimeInitParams): Promise; 84 | createRequestHeaders( 85 | required_accept_level: AcceptLevel, 86 | fields: HeaderFields 87 | ): Promise; 88 | processHtml( 89 | input: WasmResponse, 90 | option: ProcessHtmlOption 91 | ): Promise; 92 | createSignedExchange( 93 | runtime: JsRuntimeInitParams, 94 | options: CreateSignedExchangedOptions 95 | ): Promise; 96 | updateOcspInStorage(runtime: JsRuntimeInitParams): Promise; 97 | servePresetContent( 98 | runtime: JsRuntimeInitParams, 99 | url: string 100 | ): Promise; 101 | validatePayloadHeaders(fields: HeaderFields): Promise; 102 | updateAcmeStateMachine: ( 103 | runtime: JsRuntimeInitParams, 104 | acmeAccount: string 105 | ) => Promise; 106 | } 107 | 108 | interface WasmFunctions { 109 | init: () => void; 110 | WasmWorker: WasmWorker; 111 | } 112 | 113 | export async function createWorker( 114 | wasmBytes: BufferSource, 115 | configYaml: string, 116 | certificatePems: string[] | undefined 117 | ) { 118 | await wasm_bindgen(wasmBytes); 119 | const {init, WasmWorker} = wasm_bindgen as WasmFunctions; 120 | init(); 121 | if (certificatePems) { 122 | return new WasmWorker(configYaml, certificatePems.join('\n')); 123 | } else { 124 | return new WasmWorker(configYaml, undefined); 125 | } 126 | } 127 | --------------------------------------------------------------------------------