├── webbundle
├── src
│ ├── fs.rs
│ ├── prelude.rs
│ ├── lib.rs
│ ├── builder.rs
│ ├── bundle.rs
│ ├── encoder.rs
│ ├── fs
│ │ └── builder.rs
│ └── decoder.rs
├── tests
│ └── builder
│ │ ├── js
│ │ └── hello.js
│ │ └── index.html
├── Cargo.toml
├── examples
│ └── create-webbundle.rs
└── benches
│ └── fs-build-bench.rs
├── Cargo.toml
├── .gitignore
├── webbundle-bench
├── templates
│ ├── module.html
│ ├── index.html
│ └── benchmark.html
├── Cargo.toml
├── bench.ts
├── run-bench.ts
├── README.md
├── run-webserver.ts
├── Make.zsh
├── run-bench-cache-aware.ts
└── src
│ └── main.rs
├── webbundle-ffi
├── README.md
├── Cargo.toml
├── examples
│ ├── Makefile
│ └── main.c
├── build.rs
└── src
│ └── lib.rs
├── .github
└── workflows
│ ├── weekly.yml
│ ├── build.yml
│ └── release.yml
├── webbundle-cli
├── Cargo.toml
└── src
│ └── main.rs
├── webbundle-server
├── Cargo.toml
└── src
│ └── main.rs
├── contributing.md
├── README.md
└── LICENSE
/webbundle/src/fs.rs:
--------------------------------------------------------------------------------
1 | mod builder;
2 |
--------------------------------------------------------------------------------
/webbundle/tests/builder/js/hello.js:
--------------------------------------------------------------------------------
1 | console.log("Hello World from a.js");
2 |
--------------------------------------------------------------------------------
/Cargo.toml:
--------------------------------------------------------------------------------
1 | [workspace]
2 | members = ["webbundle", "webbundle-bench", "webbundle-cli", "webbundle-server"]
3 |
--------------------------------------------------------------------------------
/webbundle/tests/builder/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Hello World!
5 |
6 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /Cargo.lock
2 | /target
3 | /webbundle-bench/out
4 | **/*.rs.bk
5 | /webbundle/examples/*.wbn
6 | /webbundle/benches/bundle/
7 |
--------------------------------------------------------------------------------
/webbundle-bench/templates/module.html:
--------------------------------------------------------------------------------
1 | {% for import in imports -%}
2 | {{ import|safe }};
3 | {% endfor -%}
4 | {{ function_definition }}
5 |
--------------------------------------------------------------------------------
/webbundle-ffi/README.md:
--------------------------------------------------------------------------------
1 | # webundle-ffi
2 |
3 | This is no longer maintained, in favor of more general solutions, such
4 | aa [cxx](https://cxx.rs/) crate.
5 |
6 | FFI (foreign function interface) for |webbundle| crate.
7 |
8 | At present, webundle-ffi provides a C interface.
9 |
10 | C++ binding will come later.
11 |
--------------------------------------------------------------------------------
/webbundle-bench/templates/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | webbundle-bench
6 |
7 |
8 | Module Loading Benchmarks
9 | Info
10 | {{ info }}
11 |
12 | Variants
13 |
14 | {% for benchmark in benchmarks -%}
15 | - {{ benchmark }}
16 | {% endfor -%}
17 |
18 |
19 |
20 |
--------------------------------------------------------------------------------
/webbundle-bench/Cargo.toml:
--------------------------------------------------------------------------------
1 | [package]
2 | authors = ["Hayato Ito "]
3 | description = "WebBundle Bench"
4 | edition = "2021"
5 | license = "Apache-2.0"
6 | name = "webbundle-bench"
7 | readme = "../README.md"
8 | repository = "https://github.com/google/webbundle"
9 | version = "0.5.1"
10 |
11 | [dependencies]
12 | anyhow = "1.0.57"
13 | askama = "0.11.1"
14 | clap = { version = "4", features = ["derive"] }
15 | env_logger = "0.9.0"
16 | log = "0.4.17"
17 | webbundle = { path = "../webbundle", version = "^0.5.1" }
18 |
--------------------------------------------------------------------------------
/webbundle-ffi/Cargo.toml:
--------------------------------------------------------------------------------
1 | [package]
2 | authors = ["Hayato Ito "]
3 | description = "WebBundle ffi"
4 | edition = "2021"
5 | license = "Apache-2.0"
6 | name = "webbundle-ffi"
7 | readme = "../README.md"
8 | repository = "https://github.com/google/webbundle"
9 | version = "0.3.0"
10 |
11 | [lib]
12 | name = "webbundle_ffi"
13 | crate-type = ["staticlib"]
14 |
15 | [dependencies]
16 | webbundle = { path = "../webbundle", version = "^0.5.0" }
17 | libc = "0.2.69"
18 |
19 | [build-dependencies]
20 | cbindgen = "0.14.1"
21 | anyhow = "1.0.28"
22 |
--------------------------------------------------------------------------------
/.github/workflows/weekly.yml:
--------------------------------------------------------------------------------
1 | name: weekly
2 | on:
3 | schedule:
4 | - cron: 00 3 * * 1
5 | jobs:
6 | build:
7 | strategy:
8 | matrix:
9 | rust: [stable]
10 | runs-on: ubuntu-latest
11 | steps:
12 | - uses: actions/checkout@v3
13 | - run: rustup update ${{ matrix.rust }}
14 | - run: rustup default ${{ matrix.rust }}
15 | - run: rustup component add rustfmt
16 | - run: rustup component add clippy
17 | - run: cargo update
18 | - run: cargo build --all-features
19 | - run: cargo test --all-features
20 | - run: cargo fmt --all -- --check
21 | - run: cargo clippy --all-features --all-targets -- --deny warnings
22 |
--------------------------------------------------------------------------------
/webbundle/src/prelude.rs:
--------------------------------------------------------------------------------
1 | // Copyright 2020 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | pub use anyhow::{bail, ensure, Context as _, Result};
16 |
--------------------------------------------------------------------------------
/webbundle-cli/Cargo.toml:
--------------------------------------------------------------------------------
1 | [package]
2 | authors = ["Hayato Ito "]
3 | description = "WebBundle cli"
4 | edition = "2021"
5 | license = "Apache-2.0"
6 | name = "webbundle-cli"
7 | readme = "../README.md"
8 | repository = "https://github.com/google/webbundle"
9 | version = "0.5.1"
10 |
11 | [dependencies]
12 | clap = { version = "4", features = ["derive"] }
13 | env_logger = "0.9.0"
14 | log = "0.4.17"
15 | chrono = "0.4.19"
16 | serde = { version = "1.0.137", features = ["derive"] }
17 | serde_json = "1.0.81"
18 | webbundle = { path = "../webbundle", version = "^0.5.1", features = ["fs"] }
19 | tokio = { version = "1.18.2", features = ["macros"] }
20 | anyhow = "1.0.57"
21 | url = "2.2.2"
22 |
23 | [[bin]]
24 | name = "webbundle"
25 | path = "src/main.rs"
26 |
--------------------------------------------------------------------------------
/webbundle-ffi/examples/Makefile:
--------------------------------------------------------------------------------
1 | CC = clang
2 | TARGET_DIR = ../../target
3 | BINDGEN_DIR = $(TARGET_DIR)/webbundle-ffi-bindgen
4 | WEBBUNDLE_LIBRARY = $(TARGET_DIR)/debug/libwebbundle_ffi.a
5 | WEBBUNDLE_HEADER = $(BINDGEN_DIR)/webbundle-ffi.h
6 |
7 | ifeq ($(shell uname),Darwin)
8 | LDFLAGS := -Wl,-dead_strip
9 | else
10 | LDFLAGS := -Wl,--gc-sections -lpthread -ldl
11 | endif
12 |
13 | all: $(BINDGEN_DIR)/main
14 |
15 | $(BINDGEN_DIR)/main: $(BINDGEN_DIR)/main.o $(WEBBUNDLE_LIBRARY)
16 | $(CC) -o $@ $^ $(LDFLAGS)
17 |
18 | $(BINDGEN_DIR)/main.o: main.c $(WEBBUNDLE_HEADER)
19 | $(CC) -o $@ -c $< -I $(BINDGEN_DIR)
20 |
21 | $(WEBBUNDLE_HEADER) $(WEBBUNDLE_LIBRARY): ../src/lib.rs ../Cargo.toml
22 | cargo build
23 |
24 | clean:
25 | rm -rf $(BINDGEN_DIR)
26 |
--------------------------------------------------------------------------------
/.github/workflows/build.yml:
--------------------------------------------------------------------------------
1 | name: build
2 | on: [push, pull_request]
3 | jobs:
4 | build:
5 | strategy:
6 | matrix:
7 | os: [ubuntu-latest, macOS-latest, windows-latest]
8 | runs-on: ${{ matrix.os }}
9 | steps:
10 | - uses: actions/checkout@v3
11 | - run: rustup update stable
12 | - uses: actions/cache@v3
13 | with:
14 | path: |
15 | ~/.cargo/bin/
16 | ~/.cargo/registry/index/
17 | ~/.cargo/registry/cache/
18 | ~/.cargo/git/db/
19 | target/
20 | key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
21 | - run: rustup component add rustfmt
22 | - run: rustup component add clippy
23 | - run: cargo build --all-features
24 | - run: cargo test --all-features
25 | - run: cargo fmt --all -- --check
26 | - run: cargo clippy --all-features --all-targets -- --deny warnings
27 |
--------------------------------------------------------------------------------
/webbundle-server/Cargo.toml:
--------------------------------------------------------------------------------
1 | [package]
2 | authors = ["Hayato Ito "]
3 | description = "WebBundle server"
4 | edition = "2021"
5 | license = "Apache-2.0"
6 | name = "webbundle-server"
7 | readme = "../README.md"
8 | repository = "https://github.com/google/webbundle"
9 | version = "0.5.1"
10 |
11 | [dependencies]
12 | anyhow = "1.0.52"
13 | axum = "0.4.4"
14 | axum-extra = "0.1.1"
15 | clap = { version = "4", features = ["derive"] }
16 | headers = "0.3.5"
17 | http = "0.2.6"
18 | mime = "0.3.16"
19 | serde = { version = "1.0.133", features = ["derive"] }
20 | tokio = { version = "1.15.0", features = ["macros"] }
21 | tracing = "0.1.29"
22 | tower-http = { version = "0.2.0", features = ["fs", "trace"] }
23 | tracing-subscriber = { version = "0.3.6", features = ["env-filter"] }
24 | webbundle = { path = "../webbundle", version = "^0.5.1", features = ["fs"] }
25 | futures-util = "0.3.19"
26 | tower = "0.4.11"
27 | url = "2.2.2"
28 |
--------------------------------------------------------------------------------
/webbundle/Cargo.toml:
--------------------------------------------------------------------------------
1 | [package]
2 | authors = ["Hayato Ito "]
3 | description = "WebBundle library"
4 | edition = "2021"
5 | license = "Apache-2.0"
6 | name = "webbundle"
7 | readme = "../README.md"
8 | repository = "https://github.com/google/webbundle"
9 | version = "0.5.1"
10 |
11 | [dependencies]
12 | anyhow = "1.0.52"
13 | log = "0.4.14"
14 | cbor_event = "2.1.3"
15 | http = "0.2.6"
16 | headers = { version = "0.3.5" }
17 | tokio = { version = "1.15.0", features = ["full"], optional = true }
18 | walkdir = { version = "2.3.2", optional = true }
19 | pathdiff = { version = "0.2.1", optional = true }
20 | mime_guess = { version = "2.0.3" }
21 |
22 | [dev-dependencies]
23 | tempfile = "3.3.0"
24 | criterion = { version = "0.4", features = ["html_reports", "async_tokio"] }
25 |
26 | [features]
27 | fs = ["pathdiff", "tokio", "walkdir"]
28 |
29 | [package.metadata."docs.rs"]
30 | all-features = true
31 |
32 | [[bench]]
33 | name = "fs-build-bench"
34 | harness = false
35 |
--------------------------------------------------------------------------------
/webbundle-ffi/examples/main.c:
--------------------------------------------------------------------------------
1 | #include "webbundle-ffi.h"
2 | #include
3 | #include
4 | #include
5 |
6 | // Print primary url of the given bundle.
7 | int main(int argc, char *argv[]) {
8 | if (argc != 2) {
9 | printf( "usage: %s filename", argv[0]);
10 | return 1;
11 | }
12 | FILE *f = fopen(argv[1], "rb");
13 | fseek(f, 0, SEEK_END);
14 | long fsize = ftell(f);
15 | fseek(f, 0, SEEK_SET);
16 | char *bytes = malloc(fsize + 1);
17 | size_t read_size = fread(bytes, 1, fsize, f);
18 | assert(read_size == fsize);
19 |
20 | const WebBundle* bundle = webbundle_parse(bytes, fsize);
21 |
22 | char primary_url[300];
23 | int primary_url_size = webbundle_primary_url(bundle, primary_url, 300 - 1);
24 | assert(primary_url_size >= 0);
25 | assert(primary_url_size < 300);
26 | primary_url[primary_url_size] = 0;
27 |
28 | printf("primary_url: %s\n", primary_url);
29 |
30 | // Closing
31 | fclose(f);
32 | free(bytes);
33 | webbundle_destroy((WebBundle*)(bundle));
34 | return 0;
35 | }
36 |
--------------------------------------------------------------------------------
/contributing.md:
--------------------------------------------------------------------------------
1 | # How to Contribute
2 |
3 | We'd love to accept your patches and contributions to this project. There are
4 | just a few small guidelines you need to follow.
5 |
6 | ## Contributor License Agreement
7 |
8 | Contributions to this project must be accompanied by a Contributor License
9 | Agreement. You (or your employer) retain the copyright to your contribution;
10 | this simply gives us permission to use and redistribute your contributions as
11 | part of the project. Head over to to see
12 | your current agreements on file or to sign a new one.
13 |
14 | You generally only need to submit a CLA once, so if you've already submitted one
15 | (even if it was for a different project), you probably don't need to do it
16 | again.
17 |
18 | ## Code reviews
19 |
20 | All submissions, including submissions by project members, require review. We
21 | use GitHub pull requests for this purpose. Consult
22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
23 | information on using pull requests.
24 |
25 | ## Community Guidelines
26 |
27 | This project follows [Google's Open Source Community
28 | Guidelines](https://opensource.google/conduct/).
29 |
--------------------------------------------------------------------------------
/webbundle/examples/create-webbundle.rs:
--------------------------------------------------------------------------------
1 | use anyhow::Result;
2 | use headers::ContentType;
3 | use std::io::BufWriter;
4 | use webbundle::{Bundle, Exchange, Version};
5 |
6 | // This creates a webbundle which can be used in
7 | // "Navigate-to-WebBundle" feature, explained at
8 | // https://web.dev/web-bundles/.
9 | //
10 | // 1. Run this example: e.g. cargo run --example create-webbundle
11 | // 2. Enable the runtime flag in Chrome: about://flags/#web-bundles
12 | // 3. Drag and drop the generated bundle file (/examples/create-example.wbn) into Chrome.
13 | // 4. You should see "Hello".
14 | fn main() -> Result<()> {
15 | let bundle = Bundle::builder()
16 | .version(Version::VersionB2)
17 | .primary_url("https://example.com/".parse()?)
18 | .exchange(Exchange::from((
19 | "https://example.com/".to_string(),
20 | "Hello".to_string().into_bytes(),
21 | ContentType::html(),
22 | )))
23 | .build()?;
24 |
25 | let out_path =
26 | std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("examples/create-webbundle.wbn");
27 | bundle.write_to(BufWriter::new(std::fs::File::create(out_path)?))?;
28 | Ok(())
29 | }
30 |
--------------------------------------------------------------------------------
/webbundle-bench/bench.ts:
--------------------------------------------------------------------------------
1 | // Experimental benchmark using "deno bench" feature
2 |
3 | import {
4 | Browser,
5 | default as puppeteer,
6 | } from "https://deno.land/x/puppeteer@14.1.1/mod.ts";
7 | import { parse } from "https://deno.land/std@0.141.0/flags/mod.ts";
8 |
9 | const args = parse(Deno.args);
10 |
11 | const executablePath = args.browser;
12 | if (!executablePath) {
13 | console.error("--browser [path] must be specified");
14 | Deno.exit(1);
15 | }
16 |
17 | const launch_options = {
18 | executablePath,
19 | args: (args._ ?? []) as string[],
20 | };
21 | console.log(`browser launch options: ${JSON.stringify(launch_options)}`);
22 | const browser = await puppeteer.launch(launch_options);
23 |
24 | const port = args.port ?? 8080;
25 | for (const name of ["unbundled", "webbundle"]) {
26 | Deno.bench(name, async () => {
27 | await run(browser, `http://localhost:${port}/out/${name}.html`);
28 | });
29 | }
30 |
31 | // browser should not be closed while running Bench.
32 | // TODO: No way to await benchmark's finish?
33 | // browser.close();
34 |
35 | async function run(browser: Browser, url: string) {
36 | const page = await browser.newPage();
37 |
38 | // Disable cache
39 | await page.setCacheEnabled(false);
40 |
41 | await page.goto(url);
42 | await page.waitForSelector("#result");
43 | }
44 |
--------------------------------------------------------------------------------
/webbundle/benches/fs-build-bench.rs:
--------------------------------------------------------------------------------
1 | use criterion::Criterion;
2 | use criterion::*;
3 |
4 | use webbundle::{Bundle, Version};
5 |
6 | /// Benchmarks for fs/builder.rs.
7 | ///
8 | /// You have to prepare benches/bundle directories
9 | /// beforehand.
10 | async fn fs_build_async() -> Bundle {
11 | let mut path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
12 | path.push("benches/bundle");
13 |
14 | Bundle::builder()
15 | .version(Version::VersionB2)
16 | .exchanges_from_dir(path)
17 | .await
18 | .unwrap()
19 | .build()
20 | .unwrap()
21 | }
22 |
23 | fn fs_build_sync() -> Bundle {
24 | let mut path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
25 | path.push("benches/bundle");
26 |
27 | Bundle::builder()
28 | .version(Version::VersionB2)
29 | .exchanges_from_dir_sync(path)
30 | .unwrap()
31 | .build()
32 | .unwrap()
33 | }
34 |
35 | fn fs_build_async_benchmark(c: &mut Criterion) {
36 | let rt = tokio::runtime::Runtime::new().unwrap();
37 | c.bench_function("fs-build-async", |b| b.to_async(&rt).iter(fs_build_async));
38 | }
39 |
40 | fn fs_build_sync_benchmark(c: &mut Criterion) {
41 | c.bench_function("fs-build-sync", |b| b.iter(fs_build_sync));
42 | }
43 |
44 | criterion_group!(benches, fs_build_async_benchmark, fs_build_sync_benchmark,);
45 | criterion_main!(benches);
46 |
--------------------------------------------------------------------------------
/webbundle-ffi/build.rs:
--------------------------------------------------------------------------------
1 | use anyhow::{ensure, Context as _, Result};
2 | use std::env;
3 | use std::fs;
4 | use std::path::{Path, PathBuf};
5 |
6 | fn main() {
7 | let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
8 | let package_name = env::var("CARGO_PKG_NAME").unwrap();
9 | let output_file = target_dir()
10 | .unwrap()
11 | .join(format!("{}-bindgen", package_name))
12 | .join(format!("{}.h", package_name))
13 | .display()
14 | .to_string();
15 |
16 | cbindgen::Builder::new()
17 | .with_crate(crate_dir)
18 | .with_language(cbindgen::Language::C)
19 | .generate()
20 | .expect("Unable to generate bindings")
21 | .write_to_file(&output_file);
22 | }
23 |
24 | // ref.
25 | // https://github.com/dtolnay/cxx/blob/850ca90849e7fb2c045fecdd428f865686e3bb4c/src/paths.rs
26 |
27 | fn out_dir() -> Result {
28 | env::var("OUT_DIR")
29 | .map(PathBuf::from)
30 | .context("OUT_DIR is not set")
31 | }
32 |
33 | fn canonicalize(path: impl AsRef) -> Result {
34 | Ok(fs::canonicalize(path)?)
35 | }
36 |
37 | fn target_dir() -> Result {
38 | let mut dir = out_dir().and_then(canonicalize)?;
39 | // println!("{:?}", dir);
40 | // eprintln!("{:?}", dir);
41 | loop {
42 | if dir.ends_with("target") {
43 | return Ok(dir);
44 | }
45 | ensure!(dir.pop(), "target dir is not found")
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/webbundle-bench/run-bench.ts:
--------------------------------------------------------------------------------
1 | // Usage: deno run --allow-all run-bench.ts --browser ~/src/chrome1/src/out/Default/chrome [--port xxxx]
2 |
3 | import {
4 | Browser,
5 | default as puppeteer,
6 | } from "https://deno.land/x/puppeteer@14.1.1/mod.ts";
7 | import { parse } from "https://deno.land/std@0.141.0/flags/mod.ts";
8 |
9 | const args = parse(Deno.args);
10 |
11 | const executablePath = args.browser;
12 | if (!executablePath) {
13 | console.error("--browser [path] must be specified");
14 | Deno.exit(1);
15 | }
16 |
17 | const launch_options = {
18 | executablePath,
19 | args: (args._ ?? []) as string[],
20 | };
21 |
22 | // console.log(`browser launch options: ${JSON.stringify(launch_options)}`);
23 | const browser = await puppeteer.launch(launch_options);
24 |
25 | const port = args.port ?? 8080;
26 |
27 | async function run(name: string, browser: Browser, url: string) {
28 | console.log(`running ${name} - ${url}`);
29 | const page = await browser.newPage();
30 | // TODO: Support trace.
31 | // await page.tracing.start({ path: `${name}-trace.json` });
32 | await page.goto(url, { waitUntil: "networkidle0" });
33 | // await page.tracing.stop();
34 | const ele = await page.$("#result");
35 | const results = JSON.parse(
36 | await page.evaluate((elm) => elm.textContent, ele),
37 | );
38 | console.log(
39 | name + ": " + (results.importEnd - results.navigationResponseStart),
40 | );
41 | }
42 |
43 | for (const name of ["unbundled", "webbundle"]) {
44 | await run(`${name}`, browser, `http://localhost:${port}/${name}.html`);
45 | }
46 |
47 | await browser.close();
48 |
49 | // [2022-11-16 Wed] deno doesn't finish. Call exit explicitly.
50 | Deno.exit(0);
51 |
--------------------------------------------------------------------------------
/webbundle-bench/README.md:
--------------------------------------------------------------------------------
1 | # Synthetic Module Benchmark with Web Bundles
2 |
3 | ## Basic Usage
4 |
5 | - Install `Rust` and `Deno` (optional) beforehand.
6 | - See [`Make.zsh`](./Make.zsh) for example usages. The following is summary:
7 |
8 | 1. Checkout:
9 |
10 | ```shell
11 | git clone https://github.com/google/webbundle.git`
12 | ```
13 |
14 | 2. Generate the modules and the benchmarks.
15 |
16 | Example:
17 |
18 | ```shell
19 | cd webbundle/webbundle-bench
20 | cargo run --release -- --out out --depth 4 --branches 4
21 | ```
22 |
23 | See `build()` in `Make.zsh`.
24 |
25 | 3. Start webserver.
26 |
27 | Example:
28 |
29 | ```shell
30 | cd ../webbundle-server
31 | cargo build --release
32 | cd ../webbundle-bench
33 | RUST_LOG=error ../target/release/webbundle-server --port 8080
34 | ```
35 |
36 | See `run_webserver()` in `Make.zsh`.
37 |
38 | 4. Open `http://localhost:8080/out/index.html` in your browser, and click each benchmark.
39 |
40 | 5. (Optional) Run the benchmark using puppeteer for automation:
41 |
42 | ```shell
43 | deno run --allow-all ./run-bench.ts --port 8080
44 | ```
45 |
46 | See `bench()` in `Make.zsh`.
47 |
48 | ## What's not implemented
49 |
50 | `webbundle-bench` is inspired by
51 | [`js-modules-benchmark`](https://github.com/GoogleChromeLabs/js-module-benchmark).
52 |
53 | `webbundle-bench` is rewritten with Rust and Deno so we don't depend on Python and Node.js, and supports very minimum features which are necessary to benchmark Web Bundle loading performance.
54 |
55 | `webbundle-bench` is missing many features at this point:
56 |
57 | - [ ] Support `modulepreload`.
58 | - [ ] Rules to generate modules.
59 | - [ ] Specify the size of generated modules.
60 | - [ ] Support other bundlers (e.g. `rollup`)
61 |
--------------------------------------------------------------------------------
/.github/workflows/release.yml:
--------------------------------------------------------------------------------
1 | name: release
2 | on:
3 | push:
4 | tags:
5 | - 'webbundle-cli-v*.*.*'
6 | jobs:
7 | build:
8 | env:
9 | BIN_NAME: webbundle
10 | PACKAGE_NAME: webbundle-cli
11 | strategy:
12 | matrix:
13 | os: [ubuntu-latest, macOS-latest, windows-latest]
14 | include:
15 | - os: ubuntu-latest
16 | target: x86_64-unknown-linux-musl
17 | - os: windows-latest
18 | target: x86_64-pc-windows-msvc
19 | - os: macOS-latest
20 | target: x86_64-apple-darwin
21 | runs-on: ${{ matrix.os }}
22 | steps:
23 | - uses: actions/checkout@v3
24 | - run: rustup update stable
25 | - run: rustup target add ${{ matrix.target }}
26 | - run: cargo build --package webbundle-cli --all-features --release --target ${{ matrix.target }}
27 | - run: cargo test --package webbundle-cli --all-features --release --target ${{ matrix.target }}
28 | - name: Package
29 | if: matrix.os != 'windows-latest'
30 | run: |
31 | strip target/${{ matrix.target }}/release/${{ env.BIN_NAME }}
32 | cd target/${{ matrix.target }}/release
33 | tar czvf ../../../${{ env.PACKAGE_NAME }}-${{ matrix.target }}.tar.gz ${{ env.BIN_NAME }}
34 | cd -
35 | - name: Package (windows)
36 | if: matrix.os == 'windows-latest'
37 | run: |
38 | strip target/${{ matrix.target }}/release/${{ env.BIN_NAME }}.exe
39 | cd target/${{ matrix.target }}/release
40 | 7z a ../../../${{ env.PACKAGE_NAME }}-${{ matrix.target }}.zip ${{ env.BIN_NAME }}.exe
41 | cd -
42 | - name: Release
43 | uses: softprops/action-gh-release@v1
44 | with:
45 | files: '${{ env.PACKAGE_NAME }}-${{ matrix.target }}*'
46 | env:
47 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
48 |
--------------------------------------------------------------------------------
/webbundle-bench/run-webserver.ts:
--------------------------------------------------------------------------------
1 | import { serve } from "https://deno.land/std@0.163.0/http/server.ts";
2 | import { parse } from "https://deno.land/std@0.163.0/flags/mod.ts";
3 |
4 | import { default as staticFiles } from "https://deno.land/x/static_files@1.1.6/mod.ts";
5 |
6 | function setHeaders(headers: Headers, path: string, _stats?: Deno.FileInfo) {
7 | if (path.endsWith(".wbn")) {
8 | headers.set("Content-Type", "application/webbundle");
9 | headers.set("X-Content-Type-Options", "nosniff");
10 | // Vary header is necessary to prevent disk cache.
11 | headers.set("Vary", "bundle-preload");
12 | }
13 | }
14 |
15 | function handler(req: Request): Promise {
16 | console.log(">> ", req.url);
17 |
18 | // TODO: Check headers for bundlepreload.
19 | if (req.url.endsWith(".wbn")) {
20 | console.log("* bundle");
21 | // console.log("req.headers", req.headers);
22 |
23 | const bundle_preload = req.headers.get("bundle-preload");
24 | if (bundle_preload) {
25 | if (args.verbose) {
26 | console.log(
27 | "bunle-preload header:",
28 | bundle_preload,
29 | bundle_preload.split(",").length,
30 | );
31 | }
32 |
33 | if (!bundle_preload.includes('"a0.mjs"')) {
34 | // This should be 2nd visit.
35 | console.log(">>> * 2nd visit");
36 | return staticFiles("cache-aware-2nd", { setHeaders })({
37 | request: req,
38 | respondWith: (r: Response) => r,
39 | });
40 | }
41 | }
42 | }
43 |
44 | return staticFiles(".", { setHeaders })({
45 | request: req,
46 | respondWith: (r: Response) => r,
47 | });
48 | }
49 |
50 | const args = parse(Deno.args);
51 |
52 | console.log("args: %o", args);
53 |
54 | const port = args.port ?? 8080;
55 | const hostname = args.hostname ?? "127.0.0.1";
56 |
57 | await serve(handler, { port, hostname });
58 |
--------------------------------------------------------------------------------
/webbundle-bench/Make.zsh:
--------------------------------------------------------------------------------
1 | # * Example usages of webbundle-bench
2 |
3 | port=8080
4 |
5 | # * Build benchmarks
6 | build() {
7 | cargo run --release -- --out out --depth ${1:-4} --branches ${2:-4}
8 | }
9 |
10 | build_static_cache_aware_bundles() {
11 | cargo run --release -- --out out --depth ${1:-4} --branches ${2:-4} --split
12 | }
13 |
14 | # * Run webserver
15 | run_webserver() {
16 | cd ../webbundle-server && cargo build --release && \
17 | cd ../webbundle-bench/out && RUST_LOG=error ../../target/release/webbundle-server --port ${port}
18 | }
19 |
20 | # Run web server written in Deno, as an alternative of
21 | # `webbundle-server`. Either should work, although `webbundle-server`
22 | # is faster.
23 | run_webserver_deno() {
24 | cd out && deno run --allow-all ../run-webserver.ts --port ${port} "$@"
25 | }
26 |
27 | # * Run Benchmark
28 | bench() {
29 | deno run --allow-all ./run-bench.ts --browser ~/src/chrome1/src/out/Default/chrome \
30 | --port ${port}
31 | echo "bench: finished"
32 | }
33 |
34 | bench_with_flag() {
35 | for arg in "" "--enable-blink-features=SubresourceWebBundlesSameOriginOptimization"; do
36 | echo
37 | echo "Browser flag: $arg"
38 | # Please use your own chrome with --browser option.
39 | deno run --allow-all ./run-bench.ts --browser ~/src/chrome1/src/out/Default/chrome \
40 | --port ${port} -- $arg
41 | done
42 | }
43 |
44 | bench_cache_aware_bundle() {
45 | deno run --allow-all ./run-bench-cache-aware.ts --browser ~/src/chrome1/src/out/Default/chrome \
46 | --port ${port}
47 | echo "bench: finished"
48 | }
49 |
50 | bench_with_deno_bench() {
51 | deno bench --unstable --allow-all ./bench.ts -- --browser ~/src/chrome1/src/out/Default/chrome --port ${port}
52 | echo
53 | deno bench --unstable --allow-all ./bench.ts -- --browser ~/src/chrome1/src/out/Default/chrome --port ${port} -- --enable-blink-features=SubresourceWebBundlesSameOriginOptimization
54 | }
55 |
--------------------------------------------------------------------------------
/webbundle/src/lib.rs:
--------------------------------------------------------------------------------
1 | // Copyright 2020 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | //! # WebBundle library
16 | //!
17 | //! `webbundle` is an experimental library for WebBundle format.
18 | //!
19 | //! # Example
20 | //!
21 | //! ## WebBundle Parsing
22 | //!
23 | //! ```no_run
24 | //! use webbundle::Bundle;
25 | //! use std::io::{Read as _};
26 | //!
27 | //! let mut bytes = Vec::new();
28 | //! std::fs::File::open("example.wbn")?.read_to_end(&mut bytes)?;
29 | //! let bundle = Bundle::from_bytes(bytes)?;
30 | //! println!("Parsed bundle: {:#?}", bundle);
31 | //! # Result::Ok::<(), anyhow::Error>(())
32 | //! ```
33 | //!
34 | //! ## Creating a bundle from files
35 | //!
36 | //! ```no_run
37 | //! # async {
38 | //! use webbundle::{Bundle, Version};
39 | //!
40 | //! // Create an empty bundle. See [`Builder`] for details.
41 | //! let bundle = Bundle::builder()
42 | //! .version(Version::VersionB2)
43 | //! .build()?;
44 | //! println!("Created bundle: {:#?}", bundle);
45 | //! let write = std::io::BufWriter::new(std::fs::File::create("example.wbn")?);
46 | //! bundle.write_to(write)?;
47 | //! # Result::Ok::<(), anyhow::Error>(())
48 | //! # };
49 | //! ```
50 | mod builder;
51 | mod bundle;
52 | mod decoder;
53 | mod encoder;
54 | mod prelude;
55 | pub use builder::Builder;
56 | pub use bundle::{Body, Bundle, Exchange, Request, Response, Uri, Version};
57 | pub use prelude::Result;
58 |
59 | #[cfg(feature = "fs")]
60 | mod fs;
61 |
--------------------------------------------------------------------------------
/webbundle-bench/templates/benchmark.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | webbundle-bench
6 | {{ headers|safe}}
7 |
8 |
9 | Module Loading Benchmark
10 | Info
11 |
12 | {{ info }}
13 |
14 | Results
15 |
16 | {% for link in next_links -%}
17 | {{ link }}
18 | {% endfor -%}
19 |
20 |
21 | {% for module in modules -%}
22 |
23 | {% endfor -%}
24 |
25 |
55 |
56 |
57 |
--------------------------------------------------------------------------------
/webbundle-ffi/src/lib.rs:
--------------------------------------------------------------------------------
1 | use libc::size_t;
2 | // use std::ffi::CStr;
3 | use std::os::raw::{c_char, c_int};
4 | use std::ptr;
5 | use std::slice;
6 | use webbundle::Bundle;
7 |
8 | pub struct WebBundle(Bundle);
9 |
10 | /// Construct a new `WebBundle` from the provided `bytes`.
11 | ///
12 | /// If the bytes passed in isn't a valid WebBundle representation,
13 | /// this will return a null pointer.
14 | ///
15 | /// # Safety
16 | ///
17 | /// Make sure you destroy the WebBundle with [`webbundle_destroy()`] once you are
18 | /// done with it.
19 | ///
20 | /// [`webbundle_destroy()`]: fn.webbundle_destroy.html
21 | #[no_mangle]
22 | pub unsafe extern "C" fn webbundle_parse(bytes: *const c_char, length: size_t) -> *const WebBundle {
23 | let slice = slice::from_raw_parts(bytes as *mut u8, length as usize);
24 | match Bundle::from_bytes(slice) {
25 | Ok(bundle) => Box::into_raw(Box::new(WebBundle(bundle))),
26 | Err(_) => ptr::null(),
27 | }
28 | }
29 |
30 | /// Destroy a `WebBundle` once you are done with it.
31 | ///
32 | /// # Safety
33 | ///
34 | /// The passed `bundle` must be a valid WebBundle created by [`webbundle_parse()`] function.
35 | ///
36 | /// [`webbundle_parse()`]: fn.webbundle_parse.html
37 | #[no_mangle]
38 | pub unsafe extern "C" fn webbundle_destroy(bundle: *mut WebBundle) {
39 | if !bundle.is_null() {
40 | drop(Box::from_raw(bundle));
41 | }
42 | }
43 |
44 | /// Copy the `bundle`'s primary_url into a user-provided `buffer`,
45 | /// returning the number of bytes copied.
46 | ///
47 | /// If there is no primary-url in the bundle, this returns `-1`.
48 | /// If user-provided buffer's length is not enough, this returns `-2`.
49 | ///
50 | /// # Safety
51 | ///
52 | /// - The passed `bundle` must be a valid WebBundle created by [`webbundle_parse()`] function.
53 | /// - The user-provided `buffer` should have `length` length.
54 | #[no_mangle]
55 | pub unsafe extern "C" fn webbundle_primary_url(
56 | bundle: *const WebBundle,
57 | buffer: *mut c_char,
58 | length: size_t,
59 | ) -> c_int {
60 | if bundle.is_null() {
61 | return -1;
62 | }
63 | let bundle: &Bundle = &((*bundle).0);
64 | if let Some(uri) = bundle.primary_url() {
65 | let uri = uri.to_string();
66 |
67 | let buffer: &mut [u8] = slice::from_raw_parts_mut(buffer as *mut u8, length as usize);
68 |
69 | if buffer.len() < uri.len() {
70 | return -1;
71 | }
72 |
73 | ptr::copy_nonoverlapping(uri.as_ptr(), buffer.as_mut_ptr(), uri.len());
74 | uri.len() as c_int
75 | } else {
76 | -1
77 | }
78 | }
79 |
--------------------------------------------------------------------------------
/webbundle-bench/run-bench-cache-aware.ts:
--------------------------------------------------------------------------------
1 | // Usage: deno run --allow-all run-bench.ts --browser ~/src/chrome1/src/out/Default/chrome [--port xxxx]
2 |
3 | import {
4 | default as puppeteer,
5 | } from "https://deno.land/x/puppeteer@14.1.1/mod.ts";
6 | import { parse } from "https://deno.land/std@0.141.0/flags/mod.ts";
7 |
8 | const args = parse(Deno.args);
9 |
10 | const executablePath = args.browser;
11 | if (!executablePath) {
12 | console.error("--browser [path] must be specified");
13 | Deno.exit(1);
14 | }
15 |
16 | const launch_options = {
17 | executablePath,
18 | args: (args._ ?? []) as string[],
19 | };
20 |
21 | const port = args.port ?? 8080;
22 |
23 | async function run(
24 | name: string,
25 | url1: string,
26 | url2: string,
27 | ) {
28 | console.log(`running cache-aware: cache_hit: ${name}`);
29 |
30 | // Launch new browser so that it doesn't have any cache.
31 | const browser = await puppeteer.launch(launch_options);
32 |
33 | {
34 | // 1st visit
35 | const page = await browser.newPage();
36 |
37 | // Connect to Chrome DevTools
38 | const client = await page.target().createCDPSession();
39 |
40 | // Set throttling property
41 | await client.send("Network.emulateNetworkConditions", {
42 | "offline": false,
43 | "downloadThroughput": 100 * 1000,
44 | "uploadThroughput": 100 * 1000,
45 | "latency": 50,
46 | });
47 |
48 | await page.goto(url1, { waitUntil: "networkidle0" });
49 | const ele = await page.$("#result");
50 | const results = JSON.parse(
51 | await page.evaluate((elm) => elm.textContent, ele),
52 | );
53 | console.log(
54 | name + ": 1st: " + (results.importEnd - results.navigationResponseStart),
55 | );
56 | }
57 |
58 | {
59 | // 2nd visit
60 | const page = await browser.newPage();
61 |
62 | // Connect to Chrome DevTools
63 | const client = await page.target().createCDPSession();
64 |
65 | // Set throttling property
66 | await client.send("Network.emulateNetworkConditions", {
67 | "offline": false,
68 | "downloadThroughput": 10 * 1000,
69 | "uploadThroughput": 10 * 1000,
70 | "latency": 50,
71 | });
72 |
73 | await page.goto(url2, { waitUntil: "networkidle0" });
74 | const ele = await page.$("#result");
75 | const results = JSON.parse(
76 | await page.evaluate((elm) => elm.textContent, ele),
77 | );
78 |
79 | console.log(
80 | name + ": 2nd: " + (results.importEnd - results.navigationResponseStart),
81 | );
82 | }
83 |
84 | await browser.close();
85 | }
86 |
87 | for (const name of [10, 20, 30, 40, 50, 60, 70, 80, 90, 100]) {
88 | await run(
89 | `${name}`,
90 | `http://localhost:${port}/webbundle-cache-aware-${name}-1st.html`,
91 | `http://localhost:${port}/webbundle-cache-aware-${name}-2nd.html`,
92 | );
93 | }
94 |
95 | // [2022-11-16 Wed] deno doesn't finish. Call exit explicitly.
96 | Deno.exit(0);
97 |
--------------------------------------------------------------------------------
/webbundle/src/builder.rs:
--------------------------------------------------------------------------------
1 | // Copyright 2020 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | use crate::bundle::{Bundle, Exchange, Uri, Version};
16 | use crate::prelude::*;
17 |
18 | /// A Bundle builder.
19 | #[derive(Default)]
20 | pub struct Builder {
21 | version: Option,
22 | primary_url: Option,
23 | manifest: Option,
24 | pub(crate) exchanges: Vec,
25 | }
26 |
27 | impl Builder {
28 | pub(crate) fn new() -> Self {
29 | Default::default()
30 | }
31 |
32 | /// Sets the version.
33 | pub fn version(mut self, version: Version) -> Self {
34 | self.version = Some(version);
35 | self
36 | }
37 |
38 | /// Sets the primary url.
39 | pub fn primary_url(mut self, primary_url: Uri) -> Self {
40 | self.primary_url = Some(primary_url);
41 | self
42 | }
43 |
44 | /// Sets the manifest url.
45 | pub fn manifest(mut self, manifest: Uri) -> Self {
46 | self.manifest = Some(manifest);
47 | self
48 | }
49 |
50 | /// Adds the exchange.
51 | pub fn exchange(mut self, exchange: Exchange) -> Self {
52 | self.exchanges.push(exchange);
53 | self
54 | }
55 |
56 | /// Builds the bundle.
57 | pub fn build(self) -> Result {
58 | Ok(Bundle {
59 | version: self.version.context("no version")?,
60 | primary_url: self.primary_url,
61 | exchanges: self.exchanges,
62 | })
63 | }
64 | }
65 |
66 | #[cfg(test)]
67 | mod tests {
68 | use super::*;
69 |
70 | #[test]
71 | fn build_invalid_bundle() -> Result<()> {
72 | assert!(Builder::new().build().is_err());
73 | assert!(Builder::new()
74 | .primary_url("https://example.com/".parse()?)
75 | .build()
76 | .is_err());
77 | Ok(())
78 | }
79 |
80 | #[test]
81 | fn build() -> Result<()> {
82 | let bundle = Builder::new()
83 | .version(Version::VersionB2)
84 | .primary_url("https://example.com".parse()?)
85 | .build()?;
86 | assert_eq!(bundle.version, Version::VersionB2);
87 | assert_eq!(
88 | bundle.primary_url,
89 | Some("https://example.com".parse::()?)
90 | );
91 | Ok(())
92 | }
93 |
94 | #[test]
95 | fn build_exchange() -> Result<()> {
96 | let bundle = Builder::new()
97 | .version(Version::VersionB2)
98 | .primary_url("https://example.com/index.html".parse()?)
99 | .exchange(Exchange::from((
100 | "https://example.com/index.html".to_string(),
101 | vec![],
102 | )))
103 | .build()?;
104 | assert_eq!(bundle.exchanges.len(), 1);
105 | assert_eq!(
106 | bundle.exchanges[0].request.url(),
107 | "https://example.com/index.html"
108 | );
109 | Ok(())
110 | }
111 | }
112 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Web bundles
2 |
3 | [](https://github.com/google/webbundle/actions)
4 |
5 | `google/webbundle` is a project which aims to provide a high-performace library
6 | and various tools for handling Web bundles format.
7 |
8 | This is not an officially supported Google product.
9 |
10 | # Specification
11 |
12 | - [Web Bundles (IETF draft)](https://wpack-wg.github.io/bundled-responses/draft-ietf-wpack-bundled-responses.html)
13 | - [Subresource Loading](https://wicg.github.io/webpackage/subresource-loading.html)
14 | ([Explainer](https://github.com/WICG/webpackage/blob/main/explainers/subresource-loading.md)):
15 |
16 | # Crates
17 |
18 | There are several crates in the repository.
19 |
20 | ## [webbundle](https://github.com/google/webbundle/tree/main/webbundle)
21 |
22 | [](https://crates.io/crates/webbundle?label=webbundle)
23 |
24 | The core library. See [the documentation](https://docs.rs/webbundle).
25 |
26 | ## [webbundle-cli](https://github.com/google/webbundle/tree/main/webbundle-cli)
27 |
28 | [](https://crates.io/crates/webbundle-cli)
29 |
30 | The command line tool for packaging resources as Web Bundles.
31 |
32 | ### Installation
33 |
34 | [Archives of precompiled binaries for `webbundle-cli` are available for Windows, macOS and Linux](https://github.com/google/webbundle/releases).
35 |
36 | If you're using Rust, `webbundle-cli` can be installed with `cargo`.
37 |
38 | ```shell
39 | cargo install webbundle-cli
40 | ```
41 |
42 | ### Examples
43 |
44 | The binary name for `webbundle-cli` is `webbundle`.
45 |
46 | #### create
47 |
48 | Create `example.wbn` from the files under `build/dist` directory. This is
49 | similar to `tar cvf example.tar build/dist`.
50 |
51 | ```
52 | $ webbundle create example.wbn build/dist
53 | ```
54 |
55 | #### list
56 |
57 | List the contents of `example.wbn`. This is similar to `tar tvf example.tar`.
58 |
59 | ```
60 | $ webbundle list ./example.wbn
61 | ```
62 |
63 | #### extract
64 |
65 | Extract the contents of `example.wbn`. This is similar to `tar xvf example.tar`.
66 |
67 | ```
68 | $ webbundle extract ./example.wbn
69 | ```
70 |
71 | See `webbundle --help` for detail usage.
72 |
73 | ## [webbundle-server](https://github.com/google/webbundle/tree/main/webbundle-server)
74 |
75 | [](https://crates.io/crates/webbundle-server)
76 |
77 | The experimental web server which dynamically serves Web bundles from underlying resources.
78 |
79 | ## [webbundle-bench](https://github.com/google/webbundle/tree/main/webbundle-bench)
80 |
81 | [](https://crates.io/crates/webbundle-bench)
82 |
83 | The benchmark tool for measuring the browser's loading performance with Web bundles.
84 |
85 | # TODO
86 |
87 | The development is at very early stage. There are many TODO items:
88 |
89 | - [x] Parser
90 | - [x] Support b2 format
91 | - [x] Encoder
92 | - [x] Support b2 format
93 | - [x] Web Bundles Builder
94 | - [x] Create a Web Bundle from a directory structure
95 | - [x] Low-level APIs to create and manipulate Web Bundle file
96 | - [x] Use `http::Request`, `http::Response` and `http::Uri` for better
97 | ergonomics
98 | - [ ] Use async/await to avoid blocking operations
99 | - [ ] More CLI subcommands
100 | - [x] `create`
101 | - [x] `list`
102 | - [x] `extract`
103 | - [ ] Make these subcommands more ergonomics
104 | - [ ] Focus the performance. Avoid copy as much as possible.
105 | - [ ] Split this crate into several crates:
106 | - [x] `webbundle`: Core library
107 | - [x] `webbundle-cli`: CLI, like a `tar` command
108 | - [x] `webbundle-ffi`: Foreign function interface for C or C++ program, like a
109 | chromium.
110 | - [x] `webbundle-server`: Experimental http server which can assemble and
111 | serve a webbundle dynamically, based on request parameters
112 | - [ ] `webbundle-wasm`: WebAssembly binding
113 | - [X] `webbundle-bench`: The benchmark tool
114 |
115 | ## Contributing
116 |
117 | See [contributing.md](contributing.md) for instructions.
118 |
--------------------------------------------------------------------------------
/webbundle-server/src/main.rs:
--------------------------------------------------------------------------------
1 | use axum::{
2 | body::{boxed, Body, BoxBody},
3 | response::{Html, IntoResponse},
4 | routing::{get, get_service},
5 | Router,
6 | };
7 | use axum_extra::middleware::{self, Next};
8 | use clap::Parser;
9 | use headers::{ContentLength, HeaderMapExt as _};
10 | use http::{header, HeaderValue, Request, Response, StatusCode};
11 | use std::fmt::Write as _;
12 | use tower::ServiceBuilder;
13 | use tower_http::{services::ServeDir, trace::TraceLayer};
14 | use webbundle::{Bundle, Version};
15 |
16 | #[derive(Parser, Debug)]
17 | struct Cli {
18 | // TODO: Support https.
19 | // #[arg]
20 | // https: bool,
21 | #[arg(short, long, default_value = "8000")]
22 | port: u16,
23 | #[arg(long)]
24 | /// Bind all interfaces (default: only localhost - "127.0.0.1"),
25 | bind_all: bool,
26 | }
27 |
28 | #[tokio::main]
29 | async fn main() {
30 | // Set the RUST_LOG, if it hasn't been explicitly defined
31 | if std::env::var_os("RUST_LOG").is_none() {
32 | std::env::set_var("RUST_LOG", "my_http_server=debug,tower_http=debug")
33 | }
34 | tracing_subscriber::fmt::init();
35 | let args = Cli::parse();
36 |
37 | let app = Router::new()
38 | .nest("/wbn", get(webbundle_serve))
39 | .fallback(
40 | get_service(ServeDir::new("."))
41 | .handle_error(|error: std::io::Error| async move {
42 | (
43 | StatusCode::INTERNAL_SERVER_ERROR,
44 | format!("Unhandled internal error: {error}"),
45 | )
46 | })
47 | .layer(middleware::from_fn(serve_dir_extra)),
48 | )
49 | .layer(ServiceBuilder::new().layer(TraceLayer::new_for_http()));
50 |
51 | let addr = std::net::SocketAddr::from((
52 | if args.bind_all {
53 | [0, 0, 0, 0]
54 | } else {
55 | [127, 0, 0, 1]
56 | },
57 | args.port,
58 | ));
59 | println!("Listening on http://{addr}/");
60 | axum::Server::bind(&addr)
61 | .serve(app.into_make_service())
62 | .await
63 | .unwrap();
64 | }
65 |
66 | async fn webbundle_serve(req: Request) -> Result, (StatusCode, String)> {
67 | match webbundle_serve_internal(req).await {
68 | Ok(WebBundleServeResponse::Body(body)) => Ok(body),
69 | Ok(WebBundleServeResponse::NotFound) => Err((StatusCode::NOT_FOUND, "".to_string())),
70 | Err(err) => Err((
71 | StatusCode::INTERNAL_SERVER_ERROR,
72 | format!("Unhandled internal error {err}"),
73 | )),
74 | }
75 | }
76 |
77 | enum WebBundleServeResponse {
78 | Body(Response),
79 | NotFound,
80 | }
81 |
82 | async fn webbundle_serve_internal(req: Request) -> anyhow::Result {
83 | let path = req.uri().path();
84 | let mut full_path = std::path::PathBuf::from(".");
85 | for seg in path.trim_start_matches('/').split('/') {
86 | anyhow::ensure!(
87 | !seg.starts_with("..") && !seg.contains('\\'),
88 | "Invalid request"
89 | );
90 | full_path.push(seg);
91 | }
92 | if !is_dir(&full_path).await {
93 | return Ok(WebBundleServeResponse::NotFound);
94 | }
95 |
96 | let bundle = Bundle::builder()
97 | .version(Version::VersionB2)
98 | .exchanges_from_dir(full_path)
99 | .await?
100 | .build()?;
101 |
102 | let bytes = bundle.encode()?;
103 | let content_length = ContentLength(bytes.len() as u64);
104 | let mut response = Response::new(boxed(Body::from(bytes)));
105 | response.headers_mut().typed_insert(content_length);
106 | set_response_webbundle_headers(&mut response);
107 | Ok(WebBundleServeResponse::Body(response))
108 | }
109 |
110 | fn set_response_webbundle_headers(response: &mut Response) {
111 | response.headers_mut().insert(
112 | header::CONTENT_TYPE,
113 | HeaderValue::from_static("application/webbundle"),
114 | );
115 | response.headers_mut().insert(
116 | header::X_CONTENT_TYPE_OPTIONS,
117 | HeaderValue::from_static("nosniff"),
118 | );
119 | }
120 |
121 | async fn is_dir(full_path: &std::path::Path) -> bool {
122 | tokio::fs::metadata(full_path)
123 | .await
124 | .map(|m| m.is_dir())
125 | .unwrap_or(false)
126 | }
127 |
128 | async fn serve_dir_extra(
129 | req: Request,
130 | next: Next,
131 | ) -> Result, (StatusCode, String)> {
132 | serve_dir_extra_internal(req, next).await.map_err(|err| {
133 | (
134 | StatusCode::INTERNAL_SERVER_ERROR,
135 | format!("Unhandled internal error {err}"),
136 | )
137 | })
138 | }
139 |
140 | async fn serve_dir_extra_internal(
141 | req: Request,
142 | next: Next,
143 | ) -> anyhow::Result> {
144 | // Directory listing.
145 | // Ref: https://docs.rs/tower-http/0.1.0/src/tower_http/services/fs/serve_dir.rs.html
146 | let path = req.uri().path();
147 | let mut full_path = std::path::PathBuf::from(".");
148 | for seg in path.trim_start_matches('/').split('/') {
149 | anyhow::ensure!(!seg.starts_with("..") && !seg.contains('\\'));
150 | full_path.push(seg);
151 | }
152 | if is_dir(&full_path).await {
153 | let html = directory_list_files(full_path, path).await?;
154 | return Ok(Html(html).into_response());
155 | }
156 |
157 | if req.uri().path().ends_with(".wbn") {
158 | let mut res = next.run(req).await;
159 | set_response_webbundle_headers(&mut res);
160 | return Ok(res);
161 | }
162 |
163 | // default.
164 | Ok(next.run(req).await)
165 | }
166 |
167 | async fn directory_list_files(
168 | path: impl AsRef,
169 | display_name: &str,
170 | ) -> anyhow::Result {
171 | let path = path.as_ref();
172 |
173 | let mut contents = String::new();
174 | // ReadDir is Stream
175 | let mut read_dir = tokio::fs::read_dir(path).await?;
176 | let mut files = Vec::new();
177 | while let Some(file) = read_dir.next_entry().await? {
178 | files.push(file.path());
179 | }
180 | files.sort();
181 | for p in files {
182 | let link = format!(
183 | "{}{}",
184 | p.file_name().unwrap().to_str().unwrap(),
185 | if is_dir(&p).await { "/" } else { "" }
186 | );
187 | write!(contents, "{link}",)?;
188 | }
189 |
190 | let inline_style = r#"
191 | body {
192 | box-sizing: border-box;
193 | min-width: 200px;
194 | max-width: 980px;
195 | margin: 0 auto;
196 | padding: 45px;
197 | }
198 | "#;
199 |
200 | Ok(format!(
201 | r#"
202 |
203 |
204 | {display_name}
205 |
206 |
209 |
210 |
211 | webbundle-server: Directory listing for {display_name}
212 |
213 | - ..
214 | {contents}
215 |
216 |
217 |
218 |
219 | "#
220 | ))
221 | }
222 |
--------------------------------------------------------------------------------
/webbundle/src/bundle.rs:
--------------------------------------------------------------------------------
1 | // Copyright 2020 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | use crate::builder::Builder;
16 | use crate::decoder;
17 | use crate::encoder;
18 | use crate::prelude::*;
19 | use http::StatusCode;
20 | pub use http::Uri;
21 |
22 | use headers::{ContentLength, ContentType, HeaderMapExt as _};
23 |
24 | use std::convert::TryFrom;
25 | use std::io::Write;
26 | use std::path::Path;
27 |
28 | pub type Body = Vec;
29 | pub type Response = http::Response;
30 | pub type HeaderMap = http::header::HeaderMap;
31 |
32 | /// Represents a HTTP exchange's request.
33 | ///
34 | /// This is different from `http::request::Request` because
35 | /// a resource's URL in Web Bundle can be a relative URL, eg. "./foo.html".
36 | /// `http::request::Request` requires Uri, which can not be a relative URL.
37 | #[derive(Debug, Clone)]
38 | pub struct Request {
39 | url: String,
40 | headers: HeaderMap,
41 | }
42 |
43 | impl Request {
44 | /// Creates a new `Request` with the given url and headers.
45 | pub fn new(url: String, headers: HeaderMap) -> Request {
46 | Request { url, headers }
47 | }
48 |
49 | /// Returns a reference to the associated url.
50 | pub fn url(&self) -> &String {
51 | &self.url
52 | }
53 |
54 | /// Returns a reference to the associated header field map.
55 | pub fn headers(&self) -> &HeaderMap {
56 | &self.headers
57 | }
58 | }
59 |
60 | impl From<(String, HeaderMap)> for Request {
61 | fn from((url, headers): (String, HeaderMap)) -> Self {
62 | Self::new(url, headers)
63 | }
64 | }
65 |
66 | impl From for Request {
67 | fn from(url: String) -> Self {
68 | Self::new(url, HeaderMap::new())
69 | }
70 | }
71 |
72 | // TODO: Use TryFrom?
73 | impl From<&Path> for Request {
74 | fn from(path: &Path) -> Self {
75 | // path.display().to_string() can't be used because
76 | // that may contain a backslash, `\\`, in Windows.
77 | let url = path
78 | .iter()
79 | .map(|s| s.to_str().unwrap())
80 | .collect::>()
81 | .join("/");
82 | Self::new(url, HeaderMap::new())
83 | }
84 | }
85 |
86 | pub const HEADER_MAGIC_BYTES: [u8; 8] = [0xf0, 0x9f, 0x8c, 0x90, 0xf0, 0x9f, 0x93, 0xa6];
87 | pub(crate) const VERSION_BYTES_LEN: usize = 4;
88 | pub(crate) const TOP_ARRAY_LEN: usize = 5;
89 | pub(crate) const KNOWN_SECTION_NAMES: [&str; 4] = ["index", "critical", "responses", "primary"];
90 |
91 | /// Represents the version of WebBundle.
92 | #[derive(Debug, PartialEq, Eq)]
93 | pub enum Version {
94 | /// Version b2, which is used in Google Chrome
95 | VersionB2,
96 | /// Version 1
97 | Version1,
98 | /// Unknown version
99 | Unknown([u8; 4]),
100 | }
101 |
102 | impl Version {
103 | /// Gets the bytes which represents this version.
104 | pub fn bytes(&self) -> &[u8; 4] {
105 | match self {
106 | Version::VersionB2 => &[0x62, 0x32, 0, 0],
107 | Version::Version1 => &[0x31, 0, 0, 0],
108 | Version::Unknown(a) => a,
109 | }
110 | }
111 | }
112 |
113 | /// Represents an HTTP exchange, a pair of a request and a response.
114 | #[derive(Debug)]
115 | pub struct Exchange {
116 | pub request: Request,
117 | pub response: Response,
118 | }
119 |
120 | impl Clone for Exchange {
121 | fn clone(&self) -> Self {
122 | Exchange {
123 | request: self.request.clone(),
124 | response: {
125 | let mut response = Response::new(self.response.body().clone());
126 | *response.status_mut() = self.response.status();
127 | *response.headers_mut() = self.response.headers().clone();
128 | response
129 | },
130 | }
131 | }
132 | }
133 |
134 | impl From<(T, Vec, ContentType)> for Exchange
135 | where
136 | T: Into,
137 | {
138 | fn from((request, body, content_type): (T, Vec, ContentType)) -> Self {
139 | let request: Request = request.into();
140 | let response = {
141 | let content_length = ContentLength(body.len() as u64);
142 | let mut response = Response::new(body);
143 | *response.status_mut() = StatusCode::OK;
144 | response.headers_mut().typed_insert(content_length);
145 | response.headers_mut().typed_insert(content_type);
146 | response
147 | };
148 | Exchange { request, response }
149 | }
150 | }
151 |
152 | impl From<(T, Vec)> for Exchange
153 | where
154 | T: Into,
155 | {
156 | fn from((request, body): (T, Vec)) -> Self {
157 | let request: Request = request.into();
158 | let content_type =
159 | ContentType::from(mime_guess::from_path(&request.url).first_or_octet_stream());
160 | (request, body, content_type).into()
161 | }
162 | }
163 |
164 | /// Represents a WebBundle.
165 | #[derive(Debug)]
166 | pub struct Bundle {
167 | pub(crate) version: Version,
168 | pub(crate) primary_url: Option,
169 | pub(crate) exchanges: Vec,
170 | }
171 |
172 | impl Bundle {
173 | /// Gets the version.
174 | pub fn version(&self) -> &Version {
175 | &self.version
176 | }
177 |
178 | /// Gets the primary url.
179 | pub fn primary_url(&self) -> &Option {
180 | &self.primary_url
181 | }
182 |
183 | /// Gets the exchanges.
184 | pub fn exchanges(&self) -> &[Exchange] {
185 | &self.exchanges
186 | }
187 |
188 | /// Parses the given bytes and returns the parsed Bundle.
189 | pub fn from_bytes(bytes: impl AsRef<[u8]>) -> Result {
190 | decoder::parse(bytes)
191 | }
192 |
193 | /// Encodes this bundle and write the result to the given `write`.
194 | pub fn write_to(&self, write: W) -> Result<()> {
195 | encoder::encode(self, write)
196 | }
197 |
198 | /// Encodes this bundle.
199 | pub fn encode(&self) -> Result> {
200 | encoder::encode_to_vec(self)
201 | }
202 |
203 | /// Returns a new builder.
204 | pub fn builder() -> Builder {
205 | Builder::new()
206 | }
207 | }
208 |
209 | impl<'a> TryFrom<&'a [u8]> for Bundle {
210 | type Error = anyhow::Error;
211 |
212 | fn try_from(bytes: &'a [u8]) -> Result {
213 | Bundle::from_bytes(bytes)
214 | }
215 | }
216 |
217 | #[cfg(test)]
218 | mod tests {
219 | use super::*;
220 | use headers::ContentType;
221 |
222 | #[test]
223 | fn request_from_path() {
224 | let path = Path::new("foo/bar");
225 | let request: Request = path.into();
226 | assert_eq!(request.url(), "foo/bar");
227 |
228 | let path_str = format!("foo{}bar", std::path::MAIN_SEPARATOR);
229 | let path = Path::new(&path_str);
230 | let request: Request = path.into();
231 | assert_eq!(request.url(), "foo/bar");
232 | }
233 |
234 | #[test]
235 | fn exchange_from() {
236 | let exchange = Exchange::from(("index.html".to_string(), "hello".to_string().into_bytes()));
237 | assert_eq!(exchange.request.url(), "index.html");
238 | assert_eq!(exchange.response.body(), b"hello");
239 | assert_eq!(
240 | exchange.response.headers().typed_get::(),
241 | Some(ContentType::html())
242 | );
243 | }
244 |
245 | #[test]
246 | fn exchange_from_with_content_type() {
247 | let exchange = Exchange::from(("./foo/".to_string(), vec![], ContentType::html()));
248 | assert_eq!(exchange.request.url(), "./foo/");
249 | assert_eq!(exchange.response.body(), &[]);
250 | assert_eq!(
251 | exchange.response.headers().typed_get::(),
252 | Some(ContentType::html())
253 | );
254 | }
255 | }
256 |
--------------------------------------------------------------------------------
/webbundle/src/encoder.rs:
--------------------------------------------------------------------------------
1 | // Copyright 2020 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | use crate::bundle::{self, Bundle, Exchange, Response, Uri};
16 | use crate::prelude::*;
17 | use cbor_event::Len;
18 | use std::io::Write;
19 |
20 | use cbor_event::se::Serializer;
21 |
22 | struct CountWrite {
23 | count: usize,
24 | inner: W,
25 | }
26 |
27 | impl CountWrite {
28 | fn new(inner: W) -> Self {
29 | CountWrite { count: 0, inner }
30 | }
31 | }
32 |
33 | impl Write for CountWrite {
34 | fn write(&mut self, buf: &[u8]) -> std::io::Result {
35 | match self.inner.write(buf) {
36 | Ok(n) => {
37 | self.count += n;
38 | Ok(n)
39 | }
40 | Err(e) => Err(e),
41 | }
42 | }
43 | fn flush(&mut self) -> std::io::Result<()> {
44 | self.inner.flush()
45 | }
46 | }
47 |
48 | pub(crate) fn encode(bundle: &Bundle, write: W) -> Result<()> {
49 | Encoder::new(CountWrite::new(write)).encode(bundle)?;
50 | Ok(())
51 | }
52 |
53 | pub(crate) fn encode_to_vec(bundle: &Bundle) -> Result> {
54 | let mut write = Vec::new();
55 | encode(bundle, &mut write)?;
56 | Ok(write)
57 | }
58 |
59 | struct Encoder {
60 | se: Serializer,
61 | }
62 |
63 | trait Count {
64 | fn count(&self) -> usize;
65 | }
66 |
67 | impl Count for Serializer> {
68 | fn count(&self) -> usize {
69 | // Use unsafe because Serializer.0 is private.
70 | // TODO: Avoid to use unsafe.
71 | let se_ptr: *const Serializer> = self;
72 | let count_write_ptr = se_ptr.cast::>();
73 | unsafe { (*count_write_ptr).count }
74 | }
75 | }
76 |
77 | impl Encoder {
78 | fn new(write: W) -> Self {
79 | Encoder {
80 | se: Serializer::new(write),
81 | }
82 | }
83 |
84 | fn write_magic(&mut self) -> Result<()> {
85 | self.se.write_bytes(bundle::HEADER_MAGIC_BYTES)?;
86 | Ok(())
87 | }
88 |
89 | fn write_version(&mut self, version: &bundle::Version) -> Result<()> {
90 | self.se.write_bytes(version.bytes())?;
91 | Ok(())
92 | }
93 | }
94 |
95 | impl Encoder> {
96 | fn encode(&mut self, bundle: &Bundle) -> Result<()> {
97 | self.se
98 | .write_array(Len::Len(bundle::TOP_ARRAY_LEN as u64))?;
99 | self.write_magic()?;
100 | self.write_version(&bundle.version)?;
101 |
102 | let sections = encode_sections(bundle)?;
103 |
104 | let section_length_cbor = encode_section_lengths(§ions)?;
105 | self.se.write_bytes(section_length_cbor)?;
106 |
107 | self.se.write_array(Len::Len(sections.len() as u64))?;
108 | for section in sections {
109 | self.se.write_raw_bytes(§ion.bytes)?;
110 | }
111 |
112 | // Write the length of bytes
113 | // Spec: https://wpack-wg.github.io/bundled-responses/draft-ietf-wpack-bundled-responses.html#name-trailing-length
114 | let bundle_len = self.se.count() as u64 + 8;
115 | self.se.write_raw_bytes(&bundle_len.to_be_bytes())?;
116 | Ok(())
117 | }
118 | }
119 |
120 | struct Section {
121 | name: &'static str,
122 | bytes: Vec,
123 | }
124 |
125 | fn encode_sections(bundle: &Bundle) -> Result> {
126 | let mut sections = Vec::new();
127 |
128 | // primary url
129 | if let Some(uri) = &bundle.primary_url {
130 | let bytes = encode_primary_url_section(uri)?;
131 | sections.push(Section {
132 | name: "primary",
133 | bytes,
134 | });
135 | };
136 |
137 | // responses
138 | let (response_section_bytes, response_locations) = encode_response_section(&bundle.exchanges)?;
139 |
140 | let response_section = Section {
141 | name: "responses",
142 | bytes: response_section_bytes,
143 | };
144 |
145 | // index from responses
146 | let index_section = Section {
147 | name: "index",
148 | bytes: encode_index_section(&response_locations)?,
149 | };
150 |
151 | sections.push(index_section);
152 | sections.push(response_section);
153 | Ok(sections)
154 | }
155 |
156 | fn encode_primary_url_section(url: &Uri) -> Result> {
157 | let mut se = Serializer::new(Vec::new());
158 | se.write_text(url.to_string())?;
159 | Ok(se.finalize().to_vec())
160 | }
161 |
162 | struct ResponseLocation {
163 | url: String,
164 | offset: usize,
165 | length: usize,
166 | }
167 |
168 | fn encode_response_section(exchanges: &[Exchange]) -> Result<(Vec, Vec)> {
169 | let mut se = Serializer::new(CountWrite::new(Vec::new()));
170 |
171 | se.write_array(Len::Len(exchanges.len() as u64))?;
172 |
173 | let mut response_locations = Vec::new();
174 |
175 | for exchange in exchanges {
176 | let offset = se.count();
177 |
178 | se.write_array(Len::Len(2))?;
179 | se.write_bytes(&encode_headers(&exchange.response)?)?;
180 | se.write_bytes(exchange.response.body())?;
181 |
182 | response_locations.push(ResponseLocation {
183 | url: exchange.request.url().clone(),
184 | offset,
185 | length: se.count() - offset,
186 | });
187 | }
188 |
189 | Ok((se.finalize().inner, response_locations))
190 | }
191 |
192 | fn encode_index_section(response_locations: &[ResponseLocation]) -> Result> {
193 | // Map keys must be sorted.
194 | // See [3.9. Canonical CBOR](https://tools.ietf.org/html/rfc7049#section-3.9)
195 | let mut map = std::collections::BTreeMap::, Vec>::new();
196 |
197 | for response_location in response_locations {
198 | let mut key = Serializer::new_vec();
199 | key.write_text(&response_location.url)?;
200 |
201 | let mut value = Serializer::new_vec();
202 | value.write_array(Len::Len(2))?;
203 | value.write_unsigned_integer(response_location.offset as u64)?;
204 | value.write_unsigned_integer(response_location.length as u64)?;
205 |
206 | map.insert(key.finalize(), value.finalize());
207 | }
208 |
209 | let mut se = Serializer::new_vec();
210 | se.write_map(Len::Len(response_locations.len() as u64))?;
211 | for (key, value) in map {
212 | se.write_raw_bytes(&key)?;
213 | se.write_raw_bytes(&value)?;
214 | }
215 | Ok(se.finalize())
216 | }
217 |
218 | fn encode_section_lengths(sections: &[Section]) -> Result> {
219 | let mut se = Serializer::new_vec();
220 |
221 | se.write_array(Len::Len((sections.len() * 2) as u64))?;
222 | for section in sections {
223 | se.write_text(section.name)?;
224 | se.write_unsigned_integer(section.bytes.len() as u64)?;
225 | }
226 | Ok(se.finalize())
227 | }
228 |
229 | fn encode_headers(response: &Response) -> Result> {
230 | // Map keys must be sorted.
231 | // See [3.9. Canonical CBOR](https://tools.ietf.org/html/rfc7049#section-3.9)
232 | let mut map = std::collections::BTreeMap::, Vec>::new();
233 |
234 | // Write status
235 | let mut key = Serializer::new_vec();
236 | key.write_bytes(b":status")?;
237 | let mut value = Serializer::new_vec();
238 | value.write_bytes(response.status().as_u16().to_string().as_bytes())?;
239 | map.insert(key.finalize(), value.finalize());
240 |
241 | // Write headers
242 | for (header_name, header_value) in response.headers() {
243 | let mut key = Serializer::new_vec();
244 | key.write_bytes(header_name.as_str().as_bytes())?;
245 | let mut value = Serializer::new_vec();
246 | value.write_bytes(header_value.to_str()?.as_bytes())?;
247 | map.insert(key.finalize(), value.finalize());
248 | }
249 |
250 | let mut se = Serializer::new_vec();
251 | se.write_map(Len::Len(map.len() as u64))?;
252 | for (key, value) in map {
253 | se.write_raw_bytes(&key)?;
254 | se.write_raw_bytes(&value)?;
255 | }
256 | Ok(se.finalize())
257 | }
258 |
259 | #[cfg(test)]
260 | mod tests {
261 | use super::*;
262 | use crate::bundle::{Bundle, Exchange, Version};
263 |
264 | /// This test uses an external tool, `dump-bundle`.
265 | /// See https://github.com/WICG/webpackage/go/bundle
266 | #[ignore]
267 | #[tokio::test]
268 | async fn encode_and_let_go_dump_bundle_decode_it() -> Result<()> {
269 | let bundle = Bundle::builder()
270 | .version(Version::VersionB2)
271 | .primary_url("https://example.com/index.html".parse()?)
272 | .exchange(Exchange::from((
273 | "https://example.com/index.html".to_string(),
274 | vec![],
275 | )))
276 | .build()?;
277 |
278 | let mut file = tempfile::NamedTempFile::new()?;
279 | bundle.write_to(&mut file)?;
280 |
281 | // Dump the created bundle by `dump-bundle`.
282 | let res = std::process::Command::new("dump-bundle")
283 | .arg("-i")
284 | .arg(file.path())
285 | .output()?;
286 |
287 | assert!(res.status.success(), "dump-bundle should read the bundle");
288 | Ok(())
289 | }
290 | }
291 |
--------------------------------------------------------------------------------
/webbundle-cli/src/main.rs:
--------------------------------------------------------------------------------
1 | // Copyright 2020 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | use anyhow::{ensure, Context as _};
16 | use chrono::Local;
17 | use clap::Parser;
18 | use serde::Serialize;
19 | use std::fs::File;
20 | use std::io::{BufWriter, Read as _, Write as _};
21 | use std::path::{Component, Path, PathBuf};
22 | use url::Url;
23 | use webbundle::{Bundle, Result, Version};
24 |
25 | #[derive(Parser)]
26 | struct Cli {
27 | #[clap(subcommand)]
28 | cmd: Command,
29 | }
30 |
31 | #[derive(Parser, Clone, clap::ValueEnum)]
32 | enum Format {
33 | Plain,
34 | Json,
35 | Debug,
36 | }
37 |
38 | #[derive(Parser)]
39 | enum Command {
40 | /// Example: webbundle create example.wbn foo
41 | Create {
42 | #[arg(short = 'p', long)]
43 | primary_url: Option,
44 | /// File name
45 | file: String,
46 | /// Directory from where resources are read
47 | resources_dir: String,
48 | // TODO: Support version
49 | },
50 | /// List the contents briefly
51 | List {
52 | file: String,
53 | #[arg(long, value_enum)]
54 | format: Option,
55 | },
56 | /// Extract the contents
57 | Extract { file: String },
58 | }
59 |
60 | fn env_logger_init() {
61 | env_logger::builder()
62 | .format(|buf, record| {
63 | writeln!(
64 | buf,
65 | "[{} {:5} {}] ({}:{}) {}",
66 | Local::now().format("%+"),
67 | buf.default_styled_level(record.level()),
68 | record.target(),
69 | record.file().unwrap_or("unknown"),
70 | record.line().unwrap_or(0),
71 | record.args(),
72 | )
73 | })
74 | .init();
75 | }
76 |
77 | fn list(bundle: &Bundle, format: Option) {
78 | match format {
79 | None | Some(Format::Plain) => list_plain(bundle),
80 | Some(Format::Json) => list_json(bundle),
81 | Some(Format::Debug) => list_debug(bundle),
82 | }
83 | }
84 |
85 | fn list_plain(bundle: &Bundle) {
86 | if let Some(primary_url) = bundle.primary_url() {
87 | println!("primary_url: {primary_url}");
88 | }
89 | for exchange in bundle.exchanges() {
90 | let request = &exchange.request;
91 | let response = &exchange.response;
92 | println!(
93 | "{} {} {} bytes",
94 | request.url(),
95 | response.status(),
96 | response.body().len()
97 | );
98 | log::debug!("headers: {:?}", response.headers());
99 | }
100 | }
101 |
102 | fn list_json(bundle: &Bundle) {
103 | #[derive(Serialize)]
104 | struct Request {
105 | uri: String,
106 | }
107 |
108 | #[derive(Serialize)]
109 | struct Response {
110 | status: u16,
111 | size: usize,
112 | body: String,
113 | }
114 |
115 | #[derive(Serialize)]
116 | struct Body {
117 | body: String,
118 | }
119 |
120 | #[derive(Serialize)]
121 | struct Exchange {
122 | request: Request,
123 | response: Response,
124 | }
125 |
126 | #[derive(Serialize)]
127 | struct Bundle<'a> {
128 | version: &'a [u8],
129 | primary_url: &'a Option,
130 | exchanges: Vec,
131 | }
132 |
133 | let bundle = Bundle {
134 | version: bundle.version().bytes(),
135 | primary_url: &bundle.primary_url().as_ref().map(|uri| uri.to_string()),
136 | exchanges: bundle
137 | .exchanges()
138 | .iter()
139 | .map(|exchange| Exchange {
140 | request: Request {
141 | uri: exchange.request.url().to_string(),
142 | },
143 | response: Response {
144 | status: exchange.response.status().as_u16(),
145 | size: exchange.response.body().len(),
146 | body: String::from_utf8_lossy(exchange.response.body()).to_string(),
147 | },
148 | })
149 | .collect(),
150 | };
151 | println!("{}", serde_json::to_string(&bundle).unwrap());
152 | }
153 |
154 | fn list_debug(bundle: &Bundle) {
155 | println!("{bundle:#?}");
156 | }
157 |
158 | fn make_url_path_relative(path: impl AsRef) -> PathBuf {
159 | path.as_ref()
160 | .components()
161 | .fold(PathBuf::new(), |mut result, p| match p {
162 | Component::Normal(x) => {
163 | result.push(x);
164 | result
165 | }
166 | Component::ParentDir => {
167 | log::warn!("path contains: {}", path.as_ref().display());
168 | result.pop();
169 | result
170 | }
171 | _ => result,
172 | })
173 | }
174 |
175 | fn url_to_path(url: &str) -> Result {
176 | let url = "https://example.com/".parse::().unwrap().join(url)?;
177 |
178 | let mut path = PathBuf::new();
179 | path.push(url.scheme());
180 | if let Some(host) = url.host_str() {
181 | path.push(host);
182 | }
183 | if let Some(port) = url.port() {
184 | path.push(port.to_string());
185 | }
186 | let relative = make_url_path_relative(url.path());
187 | // We push `relative` here even if it is empty.
188 | // That makes sure path ends with "/".
189 | path.push(relative);
190 | // TODO: Push query
191 | Ok(path)
192 | }
193 |
194 | #[test]
195 | fn url_to_path_test() -> Result<()> {
196 | assert_eq!(
197 | url_to_path("https://example.com")?,
198 | Path::new("https/example.com/")
199 | );
200 | assert_eq!(
201 | url_to_path("https://example.com/index.html")?,
202 | Path::new("https/example.com/index.html")
203 | );
204 | assert_eq!(
205 | url_to_path("https://example.com/a/")?,
206 | Path::new("https/example.com/a/")
207 | );
208 | assert_eq!(
209 | url_to_path("https://example.com/a/b")?,
210 | Path::new("https/example.com/a/b")
211 | );
212 | assert_eq!(
213 | url_to_path("https://example.com/a/b/")?,
214 | Path::new("https/example.com/a/b/")
215 | );
216 | assert_eq!(url_to_path("")?, Path::new("https/example.com/"));
217 | assert_eq!(url_to_path(".")?, Path::new("https/example.com/"));
218 | assert_eq!(url_to_path("/a")?, Path::new("https/example.com/a"));
219 | assert_eq!(url_to_path("..")?, Path::new("https/example.com/"));
220 | assert_eq!(url_to_path("a/../../b")?, Path::new("https/example.com/b"));
221 | Ok(())
222 | }
223 |
224 | fn extract(bundle: &Bundle) -> Result<()> {
225 | // TODO: Avoid the conflict of file names.
226 | // The current approach is too naive.
227 | for exchange in bundle.exchanges() {
228 | let path = url_to_path(exchange.request.url())?;
229 | ensure!(
230 | path.is_relative(),
231 | format!("path shoould be relative: {}", path.display())
232 | );
233 | if !exchange.response.status().is_success() {
234 | log::info!("Skipping: {:?}", exchange.request.url());
235 | continue;
236 | }
237 | // TODO: "/" should be path::sep in windows?
238 | if path.display().to_string().ends_with('/') {
239 | if !path.exists() {
240 | std::fs::create_dir_all(&path)?;
241 | }
242 | // Use index.html
243 | let index_html = path.join("index.html");
244 | log::info!(
245 | "extract: {} => {}",
246 | exchange.request.url(),
247 | index_html.display()
248 | );
249 | let mut write = BufWriter::new(File::create(&index_html)?);
250 | write.write_all(exchange.response.body())?;
251 | } else {
252 | log::info!("extract: {} => {}", exchange.request.url(), path.display());
253 | let parent = path.parent().context("weired url")?;
254 | if !parent.exists() {
255 | std::fs::create_dir_all(parent)?;
256 | }
257 | let mut write = BufWriter::new(File::create(&path)?);
258 | write.write_all(exchange.response.body())?;
259 | }
260 | }
261 | Ok(())
262 | }
263 |
264 | #[tokio::main]
265 | async fn main() -> Result<()> {
266 | env_logger_init();
267 | let args = Cli::parse();
268 | match args.cmd {
269 | Command::Create {
270 | primary_url,
271 | file,
272 | resources_dir,
273 | } => {
274 | let mut builder = Bundle::builder()
275 | .version(Version::VersionB2)
276 | .exchanges_from_dir(resources_dir)
277 | .await?;
278 | if let Some(primary_url) = primary_url {
279 | builder = builder.primary_url(primary_url.parse()?);
280 | }
281 | let bundle = builder.build()?;
282 | log::debug!("{:#?}", bundle);
283 | let write = BufWriter::new(File::create(&file)?);
284 | bundle.write_to(write)?;
285 | }
286 | Command::List { file, format } => {
287 | let mut buf = Vec::new();
288 | File::open(file)?.read_to_end(&mut buf)?;
289 | let bundle = Bundle::from_bytes(buf)?;
290 | list(&bundle, format);
291 | }
292 | Command::Extract { file } => {
293 | let mut buf = Vec::new();
294 | File::open(file)?.read_to_end(&mut buf)?;
295 | let bundle = Bundle::from_bytes(buf)?;
296 | extract(&bundle)?;
297 | }
298 | }
299 | Ok(())
300 | }
301 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 |
2 | Apache License
3 | Version 2.0, January 2004
4 | http://www.apache.org/licenses/
5 |
6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7 |
8 | 1. Definitions.
9 |
10 | "License" shall mean the terms and conditions for use, reproduction,
11 | and distribution as defined by Sections 1 through 9 of this document.
12 |
13 | "Licensor" shall mean the copyright owner or entity authorized by
14 | the copyright owner that is granting the License.
15 |
16 | "Legal Entity" shall mean the union of the acting entity and all
17 | other entities that control, are controlled by, or are under common
18 | control with that entity. For the purposes of this definition,
19 | "control" means (i) the power, direct or indirect, to cause the
20 | direction or management of such entity, whether by contract or
21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
22 | outstanding shares, or (iii) beneficial ownership of such entity.
23 |
24 | "You" (or "Your") shall mean an individual or Legal Entity
25 | exercising permissions granted by this License.
26 |
27 | "Source" form shall mean the preferred form for making modifications,
28 | including but not limited to software source code, documentation
29 | source, and configuration files.
30 |
31 | "Object" form shall mean any form resulting from mechanical
32 | transformation or translation of a Source form, including but
33 | not limited to compiled object code, generated documentation,
34 | and conversions to other media types.
35 |
36 | "Work" shall mean the work of authorship, whether in Source or
37 | Object form, made available under the License, as indicated by a
38 | copyright notice that is included in or attached to the work
39 | (an example is provided in the Appendix below).
40 |
41 | "Derivative Works" shall mean any work, whether in Source or Object
42 | form, that is based on (or derived from) the Work and for which the
43 | editorial revisions, annotations, elaborations, or other modifications
44 | represent, as a whole, an original work of authorship. For the purposes
45 | of this License, Derivative Works shall not include works that remain
46 | separable from, or merely link (or bind by name) to the interfaces of,
47 | the Work and Derivative Works thereof.
48 |
49 | "Contribution" shall mean any work of authorship, including
50 | the original version of the Work and any modifications or additions
51 | to that Work or Derivative Works thereof, that is intentionally
52 | submitted to Licensor for inclusion in the Work by the copyright owner
53 | or by an individual or Legal Entity authorized to submit on behalf of
54 | the copyright owner. For the purposes of this definition, "submitted"
55 | means any form of electronic, verbal, or written communication sent
56 | to the Licensor or its representatives, including but not limited to
57 | communication on electronic mailing lists, source code control systems,
58 | and issue tracking systems that are managed by, or on behalf of, the
59 | Licensor for the purpose of discussing and improving the Work, but
60 | excluding communication that is conspicuously marked or otherwise
61 | designated in writing by the copyright owner as "Not a Contribution."
62 |
63 | "Contributor" shall mean Licensor and any individual or Legal Entity
64 | on behalf of whom a Contribution has been received by Licensor and
65 | subsequently incorporated within the Work.
66 |
67 | 2. Grant of Copyright License. Subject to the terms and conditions of
68 | this License, each Contributor hereby grants to You a perpetual,
69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70 | copyright license to reproduce, prepare Derivative Works of,
71 | publicly display, publicly perform, sublicense, and distribute the
72 | Work and such Derivative Works in Source or Object form.
73 |
74 | 3. Grant of Patent License. Subject to the terms and conditions of
75 | this License, each Contributor hereby grants to You a perpetual,
76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77 | (except as stated in this section) patent license to make, have made,
78 | use, offer to sell, sell, import, and otherwise transfer the Work,
79 | where such license applies only to those patent claims licensable
80 | by such Contributor that are necessarily infringed by their
81 | Contribution(s) alone or by combination of their Contribution(s)
82 | with the Work to which such Contribution(s) was submitted. If You
83 | institute patent litigation against any entity (including a
84 | cross-claim or counterclaim in a lawsuit) alleging that the Work
85 | or a Contribution incorporated within the Work constitutes direct
86 | or contributory patent infringement, then any patent licenses
87 | granted to You under this License for that Work shall terminate
88 | as of the date such litigation is filed.
89 |
90 | 4. Redistribution. You may reproduce and distribute copies of the
91 | Work or Derivative Works thereof in any medium, with or without
92 | modifications, and in Source or Object form, provided that You
93 | meet the following conditions:
94 |
95 | (a) You must give any other recipients of the Work or
96 | Derivative Works a copy of this License; and
97 |
98 | (b) You must cause any modified files to carry prominent notices
99 | stating that You changed the files; and
100 |
101 | (c) You must retain, in the Source form of any Derivative Works
102 | that You distribute, all copyright, patent, trademark, and
103 | attribution notices from the Source form of the Work,
104 | excluding those notices that do not pertain to any part of
105 | the Derivative Works; and
106 |
107 | (d) If the Work includes a "NOTICE" text file as part of its
108 | distribution, then any Derivative Works that You distribute must
109 | include a readable copy of the attribution notices contained
110 | within such NOTICE file, excluding those notices that do not
111 | pertain to any part of the Derivative Works, in at least one
112 | of the following places: within a NOTICE text file distributed
113 | as part of the Derivative Works; within the Source form or
114 | documentation, if provided along with the Derivative Works; or,
115 | within a display generated by the Derivative Works, if and
116 | wherever such third-party notices normally appear. The contents
117 | of the NOTICE file are for informational purposes only and
118 | do not modify the License. You may add Your own attribution
119 | notices within Derivative Works that You distribute, alongside
120 | or as an addendum to the NOTICE text from the Work, provided
121 | that such additional attribution notices cannot be construed
122 | as modifying the License.
123 |
124 | You may add Your own copyright statement to Your modifications and
125 | may provide additional or different license terms and conditions
126 | for use, reproduction, or distribution of Your modifications, or
127 | for any such Derivative Works as a whole, provided Your use,
128 | reproduction, and distribution of the Work otherwise complies with
129 | the conditions stated in this License.
130 |
131 | 5. Submission of Contributions. Unless You explicitly state otherwise,
132 | any Contribution intentionally submitted for inclusion in the Work
133 | by You to the Licensor shall be under the terms and conditions of
134 | this License, without any additional terms or conditions.
135 | Notwithstanding the above, nothing herein shall supersede or modify
136 | the terms of any separate license agreement you may have executed
137 | with Licensor regarding such Contributions.
138 |
139 | 6. Trademarks. This License does not grant permission to use the trade
140 | names, trademarks, service marks, or product names of the Licensor,
141 | except as required for reasonable and customary use in describing the
142 | origin of the Work and reproducing the content of the NOTICE file.
143 |
144 | 7. Disclaimer of Warranty. Unless required by applicable law or
145 | agreed to in writing, Licensor provides the Work (and each
146 | Contributor provides its Contributions) on an "AS IS" BASIS,
147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 | implied, including, without limitation, any warranties or conditions
149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 | PARTICULAR PURPOSE. You are solely responsible for determining the
151 | appropriateness of using or redistributing the Work and assume any
152 | risks associated with Your exercise of permissions under this License.
153 |
154 | 8. Limitation of Liability. In no event and under no legal theory,
155 | whether in tort (including negligence), contract, or otherwise,
156 | unless required by applicable law (such as deliberate and grossly
157 | negligent acts) or agreed to in writing, shall any Contributor be
158 | liable to You for damages, including any direct, indirect, special,
159 | incidental, or consequential damages of any character arising as a
160 | result of this License or out of the use or inability to use the
161 | Work (including but not limited to damages for loss of goodwill,
162 | work stoppage, computer failure or malfunction, or any and all
163 | other commercial damages or losses), even if such Contributor
164 | has been advised of the possibility of such damages.
165 |
166 | 9. Accepting Warranty or Additional Liability. While redistributing
167 | the Work or Derivative Works thereof, You may choose to offer,
168 | and charge a fee for, acceptance of support, warranty, indemnity,
169 | or other liability obligations and/or rights consistent with this
170 | License. However, in accepting such obligations, You may act only
171 | on Your own behalf and on Your sole responsibility, not on behalf
172 | of any other Contributor, and only if You agree to indemnify,
173 | defend, and hold each Contributor harmless for any liability
174 | incurred by, or claims asserted against, such Contributor by reason
175 | of your accepting any such warranty or additional liability.
176 |
177 | END OF TERMS AND CONDITIONS
178 |
179 | APPENDIX: How to apply the Apache License to your work.
180 |
181 | To apply the Apache License to your work, attach the following
182 | boilerplate notice, with the fields enclosed by brackets "[]"
183 | replaced with your own identifying information. (Don't include
184 | the brackets!) The text should be enclosed in the appropriate
185 | comment syntax for the file format. We also recommend that a
186 | file or class name and description of purpose be included on the
187 | same "printed page" as the copyright notice for easier
188 | identification within third-party archives.
189 |
190 | Copyright [yyyy] [name of copyright owner]
191 |
192 | Licensed under the Apache License, Version 2.0 (the "License");
193 | you may not use this file except in compliance with the License.
194 | You may obtain a copy of the License at
195 |
196 | http://www.apache.org/licenses/LICENSE-2.0
197 |
198 | Unless required by applicable law or agreed to in writing, software
199 | distributed under the License is distributed on an "AS IS" BASIS,
200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 | See the License for the specific language governing permissions and
202 | limitations under the License.
203 |
--------------------------------------------------------------------------------
/webbundle/src/fs/builder.rs:
--------------------------------------------------------------------------------
1 | // Copyright 2021 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | use crate::bundle::{Exchange, Response};
16 | use crate::prelude::*;
17 | use headers::{ContentType, HeaderValue};
18 | use http::StatusCode;
19 | use std::path::{Path, PathBuf};
20 | use tokio::fs;
21 | use tokio::io::AsyncReadExt;
22 | use walkdir::WalkDir;
23 |
24 | impl crate::builder::Builder {
25 | /// Append exchanges from files rooted at the given directory.
26 | ///
27 | /// One exchange is created for each file, however, two exchanges
28 | /// are created for `index.html` file, as follows:
29 | ///
30 | /// 1. The pareent directory **serves** the contents of `index.html` file.
31 | /// 2. The URL for `index.html` file is a redirect to the parent directory
32 | /// (`301` MOVED PERMANENTLY).
33 | ///
34 | /// # Examples
35 | ///
36 | /// ```no_run
37 | /// # async {
38 | /// use webbundle::{Bundle, Version};
39 | /// let bundle = Bundle::builder()
40 | /// .version(Version::VersionB2)
41 | /// .exchanges_from_dir("build").await?
42 | /// .build()?;
43 | /// # std::result::Result::Ok::<_, anyhow::Error>(bundle)
44 | /// # };
45 | /// ```
46 | pub async fn exchanges_from_dir(mut self, dir: impl AsRef) -> Result {
47 | self.exchanges.append(
48 | &mut ExchangeBuilder::new(PathBuf::from(dir.as_ref()))
49 | .walk()
50 | .await?
51 | .build(),
52 | );
53 | Ok(self)
54 | }
55 |
56 | /// Sync version of `exchanges_from_dir`.
57 | pub fn exchanges_from_dir_sync(mut self, dir: impl AsRef) -> Result {
58 | self.exchanges.append(
59 | &mut ExchangeBuilder::new(PathBuf::from(dir.as_ref()))
60 | .walk_sync()?
61 | .build(),
62 | );
63 | Ok(self)
64 | }
65 | }
66 |
67 | pub(crate) struct ExchangeBuilder {
68 | base_dir: PathBuf,
69 | exchanges: Vec,
70 | }
71 |
72 | // TODO: Refactor so that async and sync variants share more code.
73 | impl ExchangeBuilder {
74 | pub fn new(base_dir: PathBuf) -> Self {
75 | ExchangeBuilder {
76 | base_dir,
77 | exchanges: Vec::new(),
78 | }
79 | }
80 |
81 | pub async fn walk(mut self) -> Result {
82 | // TODO: Walkdir is not async.
83 | for entry in WalkDir::new(&self.base_dir) {
84 | let entry = entry?;
85 | log::debug!("visit: {:?}", entry);
86 | let file_type = entry.file_type();
87 | if file_type.is_symlink() {
88 | log::warn!(
89 | "path is symbolink link. Skipping. {}",
90 | entry.path().display()
91 | );
92 | continue;
93 | }
94 | if !file_type.is_file() {
95 | continue;
96 | }
97 | if entry.path().file_name().unwrap() == "index.html" {
98 | let dir = entry.path().parent().unwrap();
99 |
100 | let relative_url = pathdiff::diff_paths(dir, &self.base_dir).unwrap();
101 | let relative_path = pathdiff::diff_paths(entry.path(), &self.base_dir).unwrap();
102 | // for -> Serves the contents of /index.html
103 | self = self.exchange(&relative_url, &relative_path).await?;
104 |
105 | // for /index.html -> redirect to "./"
106 | self = self.exchange_redirect(&relative_path, "./")?;
107 | } else {
108 | let relative_path = pathdiff::diff_paths(entry.path(), &self.base_dir).unwrap();
109 | self = self.exchange(&relative_path, &relative_path).await?;
110 | }
111 | }
112 | Ok(self)
113 | }
114 |
115 | pub fn walk_sync(mut self) -> Result {
116 | for entry in WalkDir::new(&self.base_dir) {
117 | let entry = entry?;
118 | log::debug!("visit: {:?}", entry);
119 | let file_type = entry.file_type();
120 | if file_type.is_symlink() {
121 | log::warn!(
122 | "path is symbolink link. Skipping. {}",
123 | entry.path().display()
124 | );
125 | continue;
126 | }
127 | if !file_type.is_file() {
128 | continue;
129 | }
130 | if entry.path().file_name().unwrap() == "index.html" {
131 | let dir = entry.path().parent().unwrap();
132 |
133 | let relative_url = pathdiff::diff_paths(dir, &self.base_dir).unwrap();
134 | let relative_path = pathdiff::diff_paths(entry.path(), &self.base_dir).unwrap();
135 | // for -> Serves the contents of /index.html
136 | self = self.exchange_sync(relative_url, &relative_path)?;
137 |
138 | // for /index.html -> redirect to "./"
139 | self = self.exchange_redirect(&relative_path, "./")?;
140 | } else {
141 | let relative_path = pathdiff::diff_paths(entry.path(), &self.base_dir).unwrap();
142 | self = self.exchange_sync(&relative_path, &relative_path)?;
143 | }
144 | }
145 | Ok(self)
146 | }
147 |
148 | pub fn build(self) -> Vec {
149 | self.exchanges
150 | }
151 |
152 | pub async fn exchange(
153 | mut self,
154 | relative_url: impl AsRef,
155 | relative_path: impl AsRef,
156 | ) -> Result {
157 | self.exchanges.push(
158 | (
159 | relative_url.as_ref(),
160 | self.read_file(&relative_path).await?,
161 | ContentType::from(mime_guess::from_path(&relative_path).first_or_octet_stream()),
162 | )
163 | .into(),
164 | );
165 | Ok(self)
166 | }
167 |
168 | pub fn exchange_sync(
169 | mut self,
170 | relative_url: impl AsRef,
171 | relative_path: impl AsRef,
172 | ) -> Result {
173 | self.exchanges.push(
174 | (
175 | relative_url.as_ref(),
176 | self.read_file_sync(&relative_path)?,
177 | ContentType::from(mime_guess::from_path(&relative_path).first_or_octet_stream()),
178 | )
179 | .into(),
180 | );
181 | Ok(self)
182 | }
183 |
184 | fn exchange_redirect(mut self, relative_url: &Path, location: &str) -> Result {
185 | self.exchanges.push(Exchange {
186 | request: relative_url.display().to_string().into(),
187 | response: Self::create_redirect(location)?,
188 | });
189 | Ok(self)
190 | }
191 |
192 | fn create_redirect(location: &str) -> Result {
193 | let mut response = Response::new(Vec::new());
194 | *response.status_mut() = StatusCode::MOVED_PERMANENTLY;
195 | response
196 | .headers_mut()
197 | .insert("Location", HeaderValue::from_str(location)?);
198 | Ok(response)
199 | }
200 |
201 | async fn read_file(&self, relative_path: impl AsRef) -> Result> {
202 | ensure!(
203 | relative_path.as_ref().is_relative(),
204 | format!("Path is not relative: {}", relative_path.as_ref().display())
205 | );
206 | let path = self.base_dir.join(relative_path);
207 |
208 | let mut file = tokio::io::BufReader::new(fs::File::open(&path).await?);
209 | let mut body = Vec::new();
210 | file.read_to_end(&mut body).await?;
211 | Ok(body)
212 | }
213 |
214 | fn read_file_sync(&self, relative_path: impl AsRef) -> Result> {
215 | use std::io::Read;
216 |
217 | ensure!(
218 | relative_path.as_ref().is_relative(),
219 | format!("Path is not relative: {}", relative_path.as_ref().display())
220 | );
221 | let path = self.base_dir.join(relative_path);
222 |
223 | let mut file = std::io::BufReader::new(std::fs::File::open(path)?);
224 | let mut body = Vec::new();
225 | file.read_to_end(&mut body)?;
226 | Ok(body)
227 | }
228 | }
229 |
230 | #[cfg(test)]
231 | mod tests {
232 | use super::*;
233 | use crate::bundle::{Bundle, Exchange, Version};
234 |
235 | #[tokio::test]
236 | async fn exchange_builder() -> Result<()> {
237 | let base_dir = {
238 | let mut path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
239 | path.push("tests/builder");
240 | path
241 | };
242 |
243 | let exchanges = ExchangeBuilder::new(base_dir.clone())
244 | .exchange(".", "index.html")
245 | .await?
246 | .build();
247 | assert_eq!(exchanges.len(), 1);
248 | let exchange = &exchanges[0];
249 | assert_eq!(exchange.request.url(), ".");
250 | assert_eq!(exchange.response.status(), StatusCode::OK);
251 | assert_eq!(exchange.response.headers()["content-type"], "text/html");
252 | assert_eq!(
253 | exchange.response.headers()["content-length"],
254 | std::fs::read(base_dir.join("index.html"))?
255 | .len()
256 | .to_string()
257 | );
258 | assert_eq!(
259 | exchange.response.body(),
260 | &std::fs::read(base_dir.join("index.html"))?
261 | );
262 | Ok(())
263 | }
264 |
265 | #[tokio::test]
266 | async fn walk() -> Result<()> {
267 | let base_dir = {
268 | let mut path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
269 | path.push("tests/builder");
270 | path
271 | };
272 |
273 | let exchanges = ExchangeBuilder::new(base_dir).walk().await?.build();
274 | assert_eq!(exchanges.len(), 3);
275 |
276 | let top_dir = find_exchange_by_url(&exchanges, "")?;
277 | assert_eq!(top_dir.response.status(), StatusCode::OK);
278 |
279 | let index_html = find_exchange_by_url(&exchanges, "index.html")?;
280 | assert_eq!(index_html.response.status(), StatusCode::MOVED_PERMANENTLY);
281 |
282 | let a_js = find_exchange_by_url(&exchanges, "js/hello.js")?;
283 | assert_eq!(a_js.response.status(), StatusCode::OK);
284 |
285 | Ok(())
286 | }
287 |
288 | fn find_exchange_by_url<'a>(exchanges: &'a [Exchange], url: &str) -> Result<&'a Exchange> {
289 | exchanges
290 | .iter()
291 | .find(|e| e.request.url() == url)
292 | .context("not fouond")
293 | }
294 |
295 | /// This test uses an external tool, `dump-bundle`.
296 | /// See https://github.com/WICG/webpackage/go/bundle
297 | #[ignore]
298 | #[tokio::test]
299 | async fn encode_and_let_go_dump_bundle_decode_it() -> Result<()> {
300 | // Create a bundle.
301 | let base_dir = {
302 | let mut path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
303 | path.push("tests/builder");
304 | path
305 | };
306 |
307 | let bundle = Bundle::builder()
308 | .version(Version::VersionB2)
309 | .exchanges_from_dir(base_dir)
310 | .await?
311 | .build()?;
312 |
313 | let mut file = tempfile::NamedTempFile::new()?;
314 | bundle.write_to(&mut file)?;
315 |
316 | // Dump the created bundle by `dump-bundle`.
317 | let res = std::process::Command::new("dump-bundle")
318 | .arg("-i")
319 | .arg(file.path())
320 | .output()?;
321 |
322 | assert!(res.status.success(), "dump-bundle should read the bundle");
323 | Ok(())
324 | }
325 | }
326 |
--------------------------------------------------------------------------------
/webbundle-bench/src/main.rs:
--------------------------------------------------------------------------------
1 | use std::path::{Path, PathBuf};
2 |
3 | use anyhow::Result;
4 | use askama::Template;
5 | use clap::Parser;
6 | use webbundle::Bundle;
7 |
8 | #[derive(Parser, Debug)]
9 | struct Cli {
10 | /// The output directory
11 | #[arg(short = 'o', long, default_value = "out")]
12 | out: String,
13 | /// The module tree depth
14 | #[arg(short = 'd', long, default_value = "4")]
15 | depth: u32,
16 | /// The module tree width at each level
17 | #[arg(short = 'b', long, default_value = "4")]
18 | branches: u32,
19 | /// [Experimental] Produce two WebBundle for cache-aware WebBundles static test
20 | #[arg(long)]
21 | split: bool,
22 | }
23 |
24 | struct Module {
25 | children: Vec,
26 | // e.g. "a2_a1_a3"
27 | name: String,
28 | // e.g. "a1"
29 | short_name: String,
30 | // e.g. "a2/a1"
31 | dir: Option,
32 | }
33 |
34 | impl Module {
35 | fn new(name: String, short_name: String, dir: Option) -> Module {
36 | Module {
37 | name,
38 | short_name,
39 | dir,
40 | children: vec![],
41 | }
42 | }
43 |
44 | fn expand_recurse(&mut self, depth: u32, option: &Cli) {
45 | if depth == option.depth {
46 | return;
47 | }
48 | let dir = match &self.dir {
49 | Some(dir) => dir.join(&self.short_name),
50 | None => PathBuf::from(&self.short_name),
51 | };
52 | for index in 0..option.branches {
53 | let short_name = format!("a{index}");
54 | let name = format!("{}_{short_name}", self.name);
55 | let mut module = Module::new(name, short_name, Some(dir.clone()));
56 | module.expand_recurse(depth + 1, option);
57 | self.children.push(module);
58 | }
59 | }
60 |
61 | fn filename(&self) -> String {
62 | format!("{}.mjs", self.name)
63 | }
64 |
65 | fn full_path(&self) -> String {
66 | match &self.dir {
67 | Some(dir) => dir.join(self.filename()).display().to_string(),
68 | None => self.filename(),
69 | }
70 | }
71 |
72 | fn relative_path_from_parent(&self) -> String {
73 | match &self.dir {
74 | Some(dir) => Path::new(dir.file_name().unwrap())
75 | .join(self.filename())
76 | .display()
77 | .to_string(),
78 | None => self.filename(),
79 | }
80 | }
81 |
82 | fn export_function_name(&self) -> String {
83 | format!("f_{}", self.name)
84 | }
85 |
86 | fn function_definition(&self) -> String {
87 | let mut ops = self
88 | .children
89 | .iter()
90 | .map(|child| format!("{}()", child.export_function_name()))
91 | .collect::>();
92 | ops.push("a".to_string());
93 | let res = ops.join(" + ");
94 | format!(
95 | r#"export function {}() {{
96 | let a = 1;
97 | return {res};
98 | }}
99 | "#,
100 | self.export_function_name()
101 | )
102 | }
103 |
104 | fn import_me(&self) -> String {
105 | format!(
106 | r#"import {{ {} }} from "./{}""#,
107 | self.export_function_name(),
108 | self.relative_path_from_parent()
109 | )
110 | }
111 |
112 | fn export(&self, mut builder: webbundle::Builder, option: &Cli) -> Result {
113 | match &self.dir {
114 | Some(dir) => log::debug!("{}", dir.join(self.filename()).display()),
115 | None => log::debug!("{}", self.filename()),
116 | };
117 | let t = ModuleTemplate {
118 | imports: self
119 | .children
120 | .iter()
121 | .map(|child| child.import_me())
122 | .collect(),
123 | function_definition: self.function_definition(),
124 | };
125 |
126 | let output_dir = match &self.dir {
127 | Some(dir) => PathBuf::from(&option.out).join(dir),
128 | None => PathBuf::from(&option.out),
129 | };
130 |
131 | std::fs::create_dir_all(output_dir)?;
132 |
133 | let file = PathBuf::from(&option.out).join(self.full_path());
134 | std::fs::write(file, t.render().unwrap())?;
135 |
136 | builder = builder.exchange((self.full_path(), t.render().unwrap().into_bytes()).into());
137 |
138 | for child in &self.children {
139 | builder = child.export(builder, option)?;
140 | }
141 | Ok(builder)
142 | }
143 | }
144 |
145 | trait Resources {
146 | fn resources(&self) -> Vec;
147 | }
148 |
149 | impl Resources for Bundle {
150 | fn resources(&self) -> Vec {
151 | self.exchanges()
152 | .iter()
153 | .map(|e| format!(r#""{}""#, e.request.url()))
154 | .collect::>()
155 | }
156 | }
157 |
158 | struct Benchmark {
159 | start_module: Module,
160 | }
161 |
162 | const CACHE_HIT: [usize; 11] = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100];
163 |
164 | impl Benchmark {
165 | fn new(option: &Cli) -> Benchmark {
166 | let mut start_module = Module::new("a0".to_string(), "a0".to_string(), None);
167 | start_module.expand_recurse(0, option);
168 | Benchmark { start_module }
169 | }
170 |
171 | fn build(&self, option: &Cli) -> Result<()> {
172 | let bundle = self.build_modules(option)?;
173 | self.build_html(option)?;
174 |
175 | // For cache-aware Web Bundle ad-hoc tests.
176 | if option.split {
177 | for cache_hit in CACHE_HIT {
178 | let (bundle0, bundle1) =
179 | self.build_cache_aware_bundle(option, &bundle, cache_hit)?;
180 | self.build_cache_aware_bundle_html(option, &bundle0, &bundle1, cache_hit)?;
181 | }
182 | }
183 | Ok(())
184 | }
185 |
186 | fn build_modules(&self, option: &Cli) -> Result {
187 | // Build modules
188 | let builder = Bundle::builder().version(webbundle::Version::VersionB2);
189 | let builder = self.start_module.export(builder, option)?;
190 |
191 | // Build webbundle
192 | let bundle = builder.build()?;
193 | println!("Build {} modules", bundle.exchanges().len());
194 | std::fs::create_dir_all(&option.out)?;
195 | let f = std::fs::File::create(PathBuf::from(&option.out).join("webbundle.wbn"))?;
196 | bundle.write_to(f)?;
197 |
198 | Ok(bundle)
199 | }
200 |
201 | fn build_cache_aware_bundle(
202 | &self,
203 | option: &Cli,
204 | bundle: &Bundle,
205 | cache_hit: usize,
206 | ) -> Result<(Bundle, Bundle)> {
207 | let mut builder0 = Bundle::builder().version(webbundle::Version::VersionB2);
208 | let mut builder1 = Bundle::builder().version(webbundle::Version::VersionB2);
209 | let len = bundle.exchanges().len();
210 | for (i, exchange) in bundle.exchanges().iter().enumerate() {
211 | if i * 100 < len * cache_hit {
212 | builder0 = builder0.exchange(exchange.clone());
213 | } else {
214 | builder1 = builder1.exchange(exchange.clone());
215 | }
216 | }
217 |
218 | let bundle0 = builder0.build()?;
219 | let bundle1 = builder1.build()?;
220 |
221 | let f = std::fs::File::create(
222 | PathBuf::from(&option.out).join(format!("webbundle-cache-aware-{cache_hit}.wbn")),
223 | )?;
224 | bundle0.write_to(f)?;
225 |
226 | let dir = PathBuf::from(&option.out).join("cache-aware-2nd");
227 | std::fs::create_dir_all(&dir)?;
228 | let f = std::fs::File::create(dir.join(format!("webbundle-cache-aware-{cache_hit}.wbn")))?;
229 | bundle1.write_to(f)?;
230 |
231 | Ok((bundle0, bundle1))
232 | }
233 |
234 | fn build_html(&self, option: &Cli) -> Result<()> {
235 | self.build_unbundled_html(option)?;
236 | self.build_webbundle_html(option)?;
237 | self.build_index_html(option)
238 | }
239 |
240 | fn build_unbundled_html(&self, option: &Cli) -> Result<()> {
241 | let t = BenchmarkTemplate {
242 | headers: "".to_string(),
243 | info: format!("option: {option:#?}"),
244 | modules: vec![],
245 | start_module: self.start_module.full_path(),
246 | start_func: self.start_module.export_function_name(),
247 | next_links: vec![],
248 | };
249 |
250 | std::fs::create_dir_all(&option.out)?;
251 | let file = PathBuf::from(&option.out).join("unbundled.html");
252 | std::fs::write(file, t.render().unwrap())?;
253 | Ok(())
254 | }
255 |
256 | fn build_webbundle_html(&self, option: &Cli) -> Result<()> {
257 | let t = BenchmarkTemplate {
258 | headers: r#""#.to_string(),
259 | info: format!("option: {option:#?}"),
260 | modules: vec![],
261 | start_module: self.start_module.full_path(),
262 | start_func: self.start_module.export_function_name(),
263 | next_links: vec![],
264 | };
265 |
266 | std::fs::create_dir_all(&option.out)?;
267 | let file = PathBuf::from(&option.out).join("webbundle.html");
268 | std::fs::write(file, t.render().unwrap())?;
269 | Ok(())
270 | }
271 |
272 | fn build_cache_aware_bundle_html(
273 | &self,
274 | option: &Cli,
275 | bundle0: &Bundle,
276 | bundle1: &Bundle,
277 | cache_hit: usize,
278 | ) -> Result<()> {
279 | let bundle_source_name = format!("webbundle-cache-aware-{cache_hit}.wbn");
280 |
281 | // Html for 1st visit.
282 | {
283 | let resources = bundle0.resources().join(", ");
284 |
285 | let t = BenchmarkTemplate {
286 | headers: format!(
287 | r#""#
288 | ),
289 | info: format!("option: {option:#?}"),
290 | modules: vec![],
291 | start_module: self.start_module.full_path(),
292 | start_func: self.start_module.export_function_name(),
293 | next_links: vec![format!("webbundle-cache-aware-{cache_hit}-2nd.html")],
294 | };
295 |
296 | std::fs::create_dir_all(&option.out)?;
297 | let file = PathBuf::from(&option.out)
298 | .join(format!("webbundle-cache-aware-{cache_hit}-1st.html"));
299 | std::fs::write(file, t.render().unwrap())?;
300 | }
301 |
302 | // Html for 2nd visit.
303 | {
304 | let resources = {
305 | let mut resources = bundle0.resources();
306 | resources.append(&mut bundle1.resources());
307 | resources.join(", ")
308 | };
309 |
310 | let t = BenchmarkTemplate {
311 | headers: format!(
312 | r#""#
313 | ),
314 | info: format!("option: {option:#?}"),
315 | modules: vec![],
316 | start_module: self.start_module.full_path(),
317 | start_func: self.start_module.export_function_name(),
318 | next_links: vec![],
319 | };
320 |
321 | std::fs::create_dir_all(&option.out)?;
322 | let file = PathBuf::from(&option.out)
323 | .join(format!("webbundle-cache-aware-{cache_hit}-2nd.html"));
324 | std::fs::write(file, t.render().unwrap())?;
325 | }
326 | Ok(())
327 | }
328 |
329 | fn build_index_html(&self, option: &Cli) -> Result<()> {
330 | let mut benchmarks = vec!["unbundled".to_string(), "webbundle".to_string()];
331 | if option.split {
332 | for cache_hit in CACHE_HIT {
333 | benchmarks.push(format!("webbundle-cache-aware-{cache_hit}-1st"));
334 | }
335 | }
336 | let t = IndexTemplate {
337 | info: format!("option: {option:#?}"),
338 | benchmarks,
339 | };
340 |
341 | std::fs::create_dir_all(&option.out)?;
342 | let file = PathBuf::from(&option.out).join("index.html");
343 | std::fs::write(file, t.render().unwrap())?;
344 | Ok(())
345 | }
346 | }
347 |
348 | #[derive(Template)]
349 | #[template(path = "module.html")]
350 | struct ModuleTemplate {
351 | imports: Vec,
352 | function_definition: String,
353 | }
354 |
355 | #[derive(Template)]
356 | #[template(path = "benchmark.html")]
357 | struct BenchmarkTemplate {
358 | headers: String,
359 | info: String,
360 | modules: Vec,
361 | start_module: String,
362 | start_func: String,
363 | next_links: Vec,
364 | }
365 |
366 | #[derive(Template)]
367 | #[template(path = "index.html")]
368 | struct IndexTemplate {
369 | info: String,
370 | benchmarks: Vec,
371 | }
372 |
373 | fn main() -> Result<()> {
374 | env_logger::init();
375 | let cli = Cli::parse();
376 | let benchmark = Benchmark::new(&cli);
377 | benchmark.build(&cli)
378 | }
379 |
--------------------------------------------------------------------------------
/webbundle/src/decoder.rs:
--------------------------------------------------------------------------------
1 | // Copyright 2020 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | use crate::bundle::{self, Bundle, Exchange, Request, Response, Uri, Version};
16 | use crate::prelude::*;
17 | use cbor_event::Len;
18 | use http::{
19 | header::{HeaderMap, HeaderName, HeaderValue},
20 | StatusCode,
21 | };
22 | use std::collections::HashSet;
23 | use std::convert::TryInto;
24 | use std::io::Cursor;
25 |
26 | pub(crate) fn parse(bytes: impl AsRef<[u8]>) -> Result {
27 | Decoder::new(bytes).decode()
28 | }
29 |
30 | #[derive(Debug)]
31 | struct SectionOffset {
32 | name: String,
33 | offset: u64,
34 | length: u64,
35 | }
36 |
37 | #[derive(Debug)]
38 | struct ResponseLocation {
39 | offset: u64,
40 | length: u64,
41 | }
42 |
43 | impl ResponseLocation {
44 | pub fn new(responses_section_offset: u64, offset: u64, length: u64) -> ResponseLocation {
45 | ResponseLocation {
46 | offset: responses_section_offset + offset,
47 | length,
48 | }
49 | }
50 | }
51 |
52 | #[derive(Debug)]
53 | struct RequestEntry {
54 | request: Request,
55 | response_location: ResponseLocation,
56 | }
57 |
58 | #[derive(Debug)]
59 | struct Metadata {
60 | version: Version,
61 | section_offsets: Vec,
62 | }
63 |
64 | type Deserializer = cbor_event::de::Deserializer;
65 |
66 | struct Decoder {
67 | de: Deserializer>,
68 | }
69 |
70 | impl Decoder {
71 | fn new(buf: T) -> Self {
72 | Decoder {
73 | de: Deserializer::from(Cursor::new(buf)),
74 | }
75 | }
76 | }
77 |
78 | type PrimaryUrl = Uri;
79 |
80 | impl> Decoder {
81 | fn decode(&mut self) -> Result {
82 | let metadata = self.read_metadata()?;
83 | log::debug!("metadata {:?}", metadata);
84 |
85 | let (requests, primary_url) = self.read_sections(&metadata.section_offsets)?;
86 | let exchanges = self.read_responses(requests)?;
87 |
88 | Ok(Bundle {
89 | version: metadata.version,
90 | primary_url,
91 | exchanges,
92 | })
93 | }
94 |
95 | fn read_metadata(&mut self) -> Result {
96 | ensure!(
97 | self.read_array_len()? as usize == bundle::TOP_ARRAY_LEN,
98 | "Invalid header"
99 | );
100 | self.read_magic_bytes()?;
101 | let version = self.read_version()?;
102 | let section_offsets = self.read_section_offsets()?;
103 | Ok(Metadata {
104 | version,
105 | section_offsets,
106 | })
107 | }
108 |
109 | fn read_magic_bytes(&mut self) -> Result<()> {
110 | log::debug!("read_magic_bytes");
111 | let magic: Vec = self.de.bytes().context("Invalid magic bytes")?;
112 | ensure!(magic == bundle::HEADER_MAGIC_BYTES, "Header magic mismatch");
113 | Ok(())
114 | }
115 |
116 | fn read_version(&mut self) -> Result {
117 | log::debug!("read_version");
118 | let bytes: Vec = self.de.bytes().context("Invalid version format")?;
119 | ensure!(
120 | bytes.len() == bundle::VERSION_BYTES_LEN,
121 | "Invalid version format"
122 | );
123 | let version: [u8; bundle::VERSION_BYTES_LEN] =
124 | AsRef::<[u8]>::as_ref(&bytes).try_into().unwrap();
125 | Ok(if &version == bundle::Version::Version1.bytes() {
126 | Version::Version1
127 | } else if &version == bundle::Version::VersionB2.bytes() {
128 | Version::VersionB2
129 | } else {
130 | Version::Unknown(version)
131 | })
132 | }
133 |
134 | fn read_section_offsets(&mut self) -> Result> {
135 | let bytes = self
136 | .de
137 | .bytes()
138 | .context("Failed to read sectionLength byte string")?;
139 | ensure!(
140 | bytes.len() < 8_192,
141 | format!("sectionLengthsLength is too long ({} bytes)", bytes.len())
142 | );
143 | Decoder::new(bytes).read_section_offsets_cbor(self.position())
144 | }
145 |
146 | fn read_array_len(&mut self) -> Result {
147 | match self.de.array()? {
148 | Len::Len(n) => Ok(n),
149 | _ => bail!("bundle: bundle: Failed to decode sectionOffset array header"),
150 | }
151 | }
152 |
153 | fn position(&self) -> u64 {
154 | self.de.as_ref().position()
155 | }
156 |
157 | fn read_section_offsets_cbor(&mut self, mut offset: u64) -> Result> {
158 | let n = self
159 | .read_array_len()
160 | .context("bundle: bundle: Failed to decode sectionOffset array header")?;
161 | let section_num = n / 2;
162 | offset += self.position();
163 | let mut seen_names = HashSet::new();
164 | let mut section_offsets = Vec::with_capacity(section_num as usize);
165 | for _ in 0..section_num {
166 | let name = self.de.text()?;
167 | ensure!(!seen_names.contains(&name), "Duplicate section name");
168 | seen_names.insert(name.clone());
169 | let length = self.de.unsigned_integer()?;
170 | section_offsets.push(SectionOffset {
171 | name,
172 | offset,
173 | length,
174 | });
175 | offset += length;
176 | }
177 | ensure!(!section_offsets.is_empty(), "bundle: section is empty");
178 | ensure!(
179 | section_offsets.last().unwrap().name == "responses",
180 | "bundle: Last section is not \"responses\""
181 | );
182 | Ok(section_offsets)
183 | }
184 |
185 | fn inner_buf(&self) -> &[u8] {
186 | self.de.as_ref().get_ref().as_ref()
187 | }
188 |
189 | fn new_decoder_from_range(&self, start: u64, end: u64) -> Decoder<&[u8]> {
190 | // TODO: Check range, instead of panic
191 | Decoder::new(&self.inner_buf()[start as usize..end as usize])
192 | }
193 |
194 | fn read_sections(
195 | &mut self,
196 | section_offsets: &[SectionOffset],
197 | ) -> Result<(Vec, Option)> {
198 | log::debug!("read_sections");
199 | let n = self
200 | .read_array_len()
201 | .context("Failed to read section header")?;
202 | log::debug!("n: {:?}", n);
203 | ensure!(
204 | n as usize == section_offsets.len(),
205 | format!(
206 | "bundle: Expected {} sections, got {} sections",
207 | section_offsets.len(),
208 | n
209 | )
210 | );
211 |
212 | let responses_section_offset = section_offsets.last().unwrap().offset;
213 |
214 | let mut requests = vec![];
215 | let mut primary_url: Option = None;
216 |
217 | for SectionOffset {
218 | name,
219 | offset,
220 | length,
221 | } in section_offsets
222 | {
223 | if !bundle::KNOWN_SECTION_NAMES.iter().any(|&n| n == name) {
224 | log::warn!("Unknows section name: {}. Skipping", name);
225 | continue;
226 | }
227 | let mut section_decoder = self.new_decoder_from_range(*offset, offset + length);
228 |
229 | // TODO: Support ignoredSections
230 | match name.as_ref() {
231 | "index" => {
232 | requests = section_decoder.read_index(responses_section_offset)?;
233 | }
234 | "responses" => {
235 | // Skip responses section becuase we read responses later.
236 | }
237 | "primary" => {
238 | primary_url = Some(section_decoder.read_primary_url()?);
239 | }
240 | _ => {
241 | log::warn!("Unknown section found: {}", name);
242 | }
243 | }
244 | }
245 | Ok((requests, primary_url))
246 | }
247 |
248 | fn read_primary_url(&mut self) -> Result {
249 | log::debug!("read_primary_url");
250 | self.de
251 | .text()
252 | .context("bundle: Failed to read primary_url string")?
253 | .parse()
254 | .context("Failed to parse primary_url")
255 | }
256 |
257 | fn read_index(&mut self, responses_section_offset: u64) -> Result> {
258 | let index_map_len = match self.de.map()? {
259 | Len::Len(n) => n,
260 | Len::Indefinite => {
261 | bail!("bundle: Failed to decode index section map header");
262 | }
263 | };
264 | // dbg!(index_map_len);
265 |
266 | let mut requests = vec![];
267 | for _ in 0..index_map_len {
268 | // TODO: support relative URL, which can not be Uri.
269 | let url = self.de.text()?;
270 | ensure!(
271 | self.read_array_len()? == 2,
272 | "bundle: Failed to decode index item"
273 | );
274 | let offset = self.de.unsigned_integer()?;
275 | let length = self.de.unsigned_integer()?;
276 | requests.push(RequestEntry {
277 | request: url.into(),
278 | response_location: ResponseLocation::new(responses_section_offset, offset, length),
279 | });
280 | }
281 | Ok(requests)
282 | }
283 |
284 | fn read_responses(&mut self, requests: Vec) -> Result> {
285 | requests
286 | .into_iter()
287 | .map(
288 | |RequestEntry {
289 | request,
290 | response_location: ResponseLocation { offset, length },
291 | }| {
292 | let response = self
293 | .new_decoder_from_range(offset, offset + length)
294 | .read_response()?;
295 | Ok(Exchange { request, response })
296 | },
297 | )
298 | .collect()
299 | }
300 |
301 | fn read_response(&mut self) -> Result {
302 | let responses_array_len = self
303 | .read_array_len()
304 | .context("bundle: Failed to decode responses section array headder")?;
305 | ensure!(
306 | responses_array_len == 2,
307 | "bundle: Failed to decode response entry"
308 | );
309 | log::debug!("read_response: headers byte 1");
310 | let headers = self.de.bytes()?;
311 | log::debug!("read_response: headers byte 2");
312 | let mut nested = Decoder::new(headers);
313 | let (status, headers) = nested.read_headers_cbor()?;
314 | let body = self.de.bytes()?;
315 | let mut response = Response::new(body);
316 | *response.status_mut() = status;
317 | *response.headers_mut() = headers;
318 | Ok(response)
319 | }
320 |
321 | fn read_headers_cbor(&mut self) -> Result<(StatusCode, HeaderMap)> {
322 | let headers_map_len = match self.de.map()? {
323 | Len::Len(n) => n,
324 | Len::Indefinite => {
325 | bail!("bundle: Failed to decode responses headers map headder");
326 | }
327 | };
328 | let mut headers = HeaderMap::new();
329 | let mut status = None;
330 | for _ in 0..headers_map_len {
331 | let name = String::from_utf8(self.de.bytes()?)?;
332 | let value = String::from_utf8(self.de.bytes()?)?;
333 | if name.starts_with(':') {
334 | ensure!(name == ":status", "Unknown pseudo headers");
335 | ensure!(status.is_none(), ":status is duplicated");
336 | status = Some(value.parse()?);
337 | continue;
338 | }
339 | headers.insert(
340 | HeaderName::from_lowercase(name.as_bytes())?,
341 | HeaderValue::from_str(value.as_str())?,
342 | );
343 | }
344 | ensure!(status.is_some(), "no :status header");
345 | Ok((status.unwrap(), headers))
346 | }
347 | }
348 |
349 | #[cfg(test)]
350 | mod tests {
351 | use super::*;
352 | use crate::bundle::{Bundle, Version};
353 |
354 | #[test]
355 | fn encode_and_decode() -> Result<()> {
356 | let bundle = Bundle::builder()
357 | .version(Version::VersionB2)
358 | .primary_url("https://example.com/index.html".parse()?)
359 | .exchange(Exchange::from((
360 | "https://example.com/index.html".to_string(),
361 | vec![],
362 | )))
363 | .build()?;
364 |
365 | let encoded = bundle.encode()?;
366 |
367 | // Decode encoded bundle.
368 | let bundle = Bundle::from_bytes(encoded)?;
369 | assert_eq!(bundle.version(), &Version::VersionB2);
370 | assert_eq!(
371 | bundle.primary_url(),
372 | &Some("https://example.com/index.html".parse()?)
373 | );
374 | assert_eq!(bundle.exchanges().len(), 1);
375 | assert_eq!(
376 | bundle.exchanges()[0].request.url(),
377 | "https://example.com/index.html"
378 | );
379 | assert_eq!(bundle.exchanges()[0].response.body(), &[]);
380 | Ok(())
381 | }
382 |
383 | /// This test uses an external tool, `gen-bundle`.
384 | /// See https://github.com/WICG/webpackage/go/bundle
385 | #[ignore]
386 | #[test]
387 | fn decode_bundle_encoded_by_go_gen_bundle() -> Result<()> {
388 | use std::io::Read;
389 |
390 | let base_dir = {
391 | let mut path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
392 | path.push("tests/builder");
393 | path
394 | };
395 |
396 | let mut file = tempfile::NamedTempFile::new()?;
397 |
398 | // Create a bundle by `gen-bundle`.
399 | let res = std::process::Command::new("gen-bundle")
400 | .arg("--version")
401 | .arg("b2")
402 | .arg("-dir")
403 | .arg(base_dir)
404 | .arg("-baseURL")
405 | .arg("https://example.com/")
406 | .arg("-o")
407 | .arg(file.path())
408 | .output()?;
409 | assert!(res.status.success());
410 |
411 | // Parse the created bundle.
412 | let mut bytes = Vec::new();
413 | file.read_to_end(&mut bytes)?;
414 | let bundle = Bundle::from_bytes(bytes)?;
415 |
416 | assert_eq!(bundle.version, Version::VersionB2);
417 | assert_eq!(bundle.exchanges.len(), 3);
418 | assert_eq!(bundle.exchanges[0].request.url(), "https://example.com/");
419 | assert_eq!(
420 | bundle.exchanges[1].request.url(),
421 | "https://example.com/index.html"
422 | );
423 | assert_eq!(
424 | bundle.exchanges[2].request.url(),
425 | "https://example.com/js/hello.js"
426 | );
427 | Ok(())
428 | }
429 | }
430 |
--------------------------------------------------------------------------------