├── .github └── workflows │ ├── ci.yml │ └── release.yml ├── .gitignore ├── .rustfmt.toml ├── Cargo.lock ├── Cargo.toml ├── LICENCE ├── README.md ├── auth_tokens.ts ├── auth_tokens_test.ts ├── cache.ts ├── cache_test.ts ├── deno.json ├── deno.lock ├── deno_dir.ts ├── deno_dir_test.ts ├── deps_test.ts ├── disk_cache.ts ├── disk_cache_test.ts ├── file_fetcher.ts ├── file_fetcher_test.ts ├── http_cache.ts ├── lib └── deno_cache_dir.generated.d.ts ├── mod.ts ├── rs_lib ├── Cargo.toml ├── clippy.toml ├── src │ ├── cache.rs │ ├── common.rs │ ├── deno_dir.rs │ ├── file_fetcher │ │ ├── auth_tokens.rs │ │ ├── http_util.rs │ │ └── mod.rs │ ├── global │ │ ├── cache_file.rs │ │ └── mod.rs │ ├── lib.rs │ ├── local.rs │ ├── memory.rs │ ├── npm.rs │ └── sync.rs └── tests │ ├── file_fetcher_test.rs │ └── integration_test.rs ├── rust-toolchain.toml ├── test.ts └── util.ts /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | rust: 7 | name: deno_cache_dir-${{ matrix.os }} 8 | if: | 9 | github.event_name == 'push' || 10 | !startsWith(github.event.pull_request.head.label, 'denoland:') 11 | runs-on: ${{ matrix.os }} 12 | timeout-minutes: 30 13 | strategy: 14 | matrix: 15 | os: [macOS-latest, ubuntu-latest, windows-latest] 16 | 17 | steps: 18 | - name: Clone repository 19 | uses: actions/checkout@v4 20 | 21 | - name: Install rust 22 | uses: dsherret/rust-toolchain-file@v1 23 | 24 | - uses: Swatinem/rust-cache@v2 25 | with: 26 | save-if: ${{ github.ref == 'refs/heads/main' }} 27 | 28 | - name: Format 29 | if: contains(matrix.os, 'ubuntu') 30 | run: cargo fmt --check 31 | 32 | - name: Clippy 33 | if: contains(matrix.os, 'ubuntu') 34 | run: cargo clippy 35 | 36 | - name: Build (sync) 37 | run: cargo build --features sync 38 | 39 | - name: Test 40 | run: cargo test 41 | 42 | - name: Cargo publish 43 | if: | 44 | contains(matrix.os, 'ubuntu') && 45 | github.repository == 'denoland/deno_cache_dir' && 46 | startsWith(github.ref, 'refs/tags/') 47 | env: 48 | CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} 49 | run: cargo publish -p deno_cache_dir 50 | 51 | deno: 52 | name: deno_cache_dir-deno 53 | if: | 54 | github.event_name == 'push' || 55 | !startsWith(github.event.pull_request.head.label, 'denoland:') 56 | runs-on: ubuntu-latest 57 | permissions: 58 | contents: read 59 | id-token: write 60 | timeout-minutes: 30 61 | 62 | steps: 63 | - name: Clone repository 64 | uses: actions/checkout@v4 65 | - name: Install rust 66 | uses: dsherret/rust-toolchain-file@v1 67 | - uses: Swatinem/rust-cache@v2 68 | with: 69 | save-if: ${{ github.ref == 'refs/heads/main' }} 70 | - name: Install Deno 71 | uses: denoland/setup-deno@v2 72 | with: 73 | deno-version: canary 74 | 75 | - name: Format 76 | run: deno fmt --check 77 | - name: Build 78 | run: deno task build 79 | - name: Lint 80 | run: deno lint 81 | - name: Test 82 | run: deno task test 83 | 84 | - name: Publish JSR 85 | run: deno run -A jsr:@david/publish-on-tag@0.1.3 86 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | releaseKind: 7 | description: "Kind of release" 8 | default: "minor" 9 | type: choice 10 | options: 11 | - patch 12 | - minor 13 | required: true 14 | 15 | jobs: 16 | rust: 17 | name: release 18 | runs-on: ubuntu-latest 19 | timeout-minutes: 30 20 | 21 | steps: 22 | - name: Clone repository 23 | uses: actions/checkout@v4 24 | with: 25 | token: ${{ secrets.DENOBOT_PAT }} 26 | 27 | - uses: denoland/setup-deno@v1 28 | with: 29 | deno-version: canary 30 | - uses: dsherret/rust-toolchain-file@v1 31 | 32 | - name: Tag and release 33 | env: 34 | GITHUB_TOKEN: ${{ secrets.DENOBOT_PAT }} 35 | GH_WORKFLOW_ACTOR: ${{ github.actor }} 36 | run: | 37 | git config user.email "denobot@users.noreply.github.com" 38 | git config user.name "denobot" 39 | deno run -A https://raw.githubusercontent.com/denoland/automation/0.20.0/tasks/publish_release.ts --${{github.event.inputs.releaseKind}} deno_cache_dir 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | /target 3 | lib/snippets 4 | lib/deno_cache_dir.generated.js 5 | -------------------------------------------------------------------------------- /.rustfmt.toml: -------------------------------------------------------------------------------- 1 | max_width = 80 2 | tab_spaces = 2 3 | edition = "2021" 4 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | members = ["rs_lib"] 4 | 5 | [workspace.dependencies] 6 | sys_traits = "0.1.11" 7 | 8 | [profile.release] 9 | codegen-units = 1 10 | incremental = true 11 | lto = true 12 | opt-level = "z" 13 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018-2025 the Deno authors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # deno_cache_dir 2 | 3 | [![jsr](https://jsr.io/badges/@deno/cache-dir)](https://jsr.io/@deno/cache-dir) 4 | [![](https://img.shields.io/crates/v/deno_cache_dir.svg)](https://crates.io/crates/deno_cache_dir) 5 | 6 | Implementation of the DENO_DIR/cache for the Deno CLI. 7 | 8 | This is designed to provide access to the cache using the same logic that the 9 | Deno CLI accesses the cache, which allows projects like 10 | [`deno_graph`](https://deno.land/x/deno_graph), 11 | [`deno_doc`](https://deno.land/x/deno_doc), [`dnt`](https://deno.land/x/dnt), 12 | and [`emit`](https://deno.land/x/deno_emit) to access and populate the cache in 13 | the same way that the CLI does. 14 | 15 | ## Permissions 16 | 17 | Because of the nature of code, it requires several permissions to be able to 18 | work properly. If the permissions aren't granted at execution, the code will try 19 | to prompt for them, only requesting what is specifically needed to perform the 20 | task. 21 | 22 | - `--allow-env` - The code needs access to several environment variables, 23 | depending on the platform as well, these can include `HOME`, `USERPROFILE`, 24 | `LOCALAPPDATA`, `XDG_CACHE_HOME`, `DENO_DIR`, and `DENO_AUTH_TOKENS`. 25 | - `--allow-read` - In certain cases the code needs to determine the current 26 | working directory, as well as read the cache files, so it requires read 27 | permission. 28 | - `--allow-write` - The code requires write permission to the root of the cache 29 | directory. 30 | - `--allow-net` - The code requires net access to any remote modules that are 31 | not found in the cache. 32 | 33 | This can just be granted on startup to avoid being prompted for them. 34 | 35 | ## Example 36 | 37 | ```shellsession 38 | > deno add @deno/cache-dir 39 | > deno add @deno/graph 40 | ``` 41 | 42 | Using the cache and the file fetcher to provide modules to build a module graph: 43 | 44 | ```ts 45 | import { createCache } from "@deno/cache-dir"; 46 | import { createGraph } from "@deno/graph"; 47 | 48 | // create a cache where the location will be determined environmentally 49 | const cache = createCache(); 50 | // destructuring the load we need to pass to the graph 51 | const { load } = cache; 52 | // create a graph that will use the cache above to load and cache dependencies 53 | const graph = await createGraph("https://deno.land/x/oak@v9.0.1/mod.ts", { 54 | load, 55 | }); 56 | 57 | // log out the console a similar output to `deno info` on the command line. 58 | console.log(graph.toString()); 59 | ``` 60 | -------------------------------------------------------------------------------- /auth_tokens.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | interface BearerAuthToken { 4 | type: "bearer"; 5 | host: string; 6 | token: string; 7 | } 8 | 9 | interface BasicAuthToken { 10 | type: "basic"; 11 | host: string; 12 | username: string; 13 | password: string; 14 | } 15 | 16 | type AuthToken = BearerAuthToken | BasicAuthToken; 17 | 18 | export function splitLast( 19 | value: string, 20 | delimiter: string, 21 | ): [string, string] { 22 | const split = value.split(delimiter); 23 | return [split.slice(0, -1).join(delimiter)].concat(split.slice(-1)) as [ 24 | string, 25 | string, 26 | ]; 27 | } 28 | 29 | function tokenAsValue(authToken: AuthToken): string { 30 | return authToken.type === "basic" 31 | ? `Basic ${btoa(`${authToken.username}:${authToken.password}`)}` 32 | : `Bearer ${authToken.token}`; 33 | } 34 | 35 | export class AuthTokens { 36 | #tokens: AuthToken[]; 37 | constructor(tokensStr = "") { 38 | const tokens: AuthToken[] = []; 39 | for (const tokenStr of tokensStr.split(";").filter((s) => s.length > 0)) { 40 | if (tokensStr.includes("@")) { 41 | const [token, host] = splitLast(tokenStr, "@"); 42 | if (token.includes(":")) { 43 | const [username, password] = splitLast(token, ":"); 44 | tokens.push({ type: "basic", host, username, password }); 45 | } else { 46 | tokens.push({ type: "bearer", host, token }); 47 | } 48 | } else { 49 | // todo(dsherret): feel like this should error? 50 | // deno-lint-ignore no-console 51 | console.error("Badly formed auth token discarded."); 52 | } 53 | } 54 | this.#tokens = tokens; 55 | } 56 | 57 | get(specifier: URL): string | undefined { 58 | for (const token of this.#tokens) { 59 | if (token.host.endsWith(specifier.host)) { 60 | return tokenAsValue(token); 61 | } 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /auth_tokens_test.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | import { AuthTokens } from "./auth_tokens.ts"; 4 | import { assertEquals } from "@std/assert"; 5 | 6 | Deno.test({ 7 | name: "handle undefined token string", 8 | fn() { 9 | const authTokens = new AuthTokens(undefined); 10 | assertEquals(authTokens.get(new URL("http://localhost")), undefined); 11 | }, 12 | }); 13 | 14 | Deno.test({ 15 | name: "find bearer token", 16 | fn() { 17 | const authTokens = new AuthTokens("token1@example.com"); 18 | assertEquals( 19 | authTokens.get(new URL("https://example.com")), 20 | "Bearer token1", 21 | ); 22 | }, 23 | }); 24 | 25 | Deno.test({ 26 | name: "find basic token (base64 encoded)", 27 | fn() { 28 | const authTokens = new AuthTokens("user1:pw1@example.com"); 29 | assertEquals( 30 | authTokens.get(new URL("https://example.com")), 31 | "Basic dXNlcjE6cHcx", 32 | ); 33 | }, 34 | }); 35 | -------------------------------------------------------------------------------- /cache.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | import type { LoadResponse } from "@deno/graph"; 4 | import type { CacheSetting, FileFetcher } from "./file_fetcher.ts"; 5 | 6 | /** Provides an interface to Deno's CLI cache. 7 | * 8 | * It is better to use the {@linkcode createCache} function directly. */ 9 | export class FetchCacher { 10 | #fileFetcher: FileFetcher; 11 | 12 | constructor(fileFetcher: FileFetcher) { 13 | this.#fileFetcher = fileFetcher; 14 | } 15 | 16 | // this should have the same interface as deno_graph's loader 17 | load = ( 18 | specifier: string, 19 | _isDynamic?: boolean, 20 | cacheSetting?: CacheSetting, 21 | checksum?: string, 22 | ): Promise => { 23 | const url = new URL(specifier); 24 | return this.#fileFetcher.fetchOnce(url, { cacheSetting, checksum }) 25 | .catch((e) => { 26 | if (e instanceof Deno.errors.NotFound) { 27 | return undefined; 28 | } 29 | 30 | throw e; 31 | }); 32 | }; 33 | } 34 | -------------------------------------------------------------------------------- /cache_test.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | import { FetchCacher } from "./cache.ts"; 4 | import { DenoDir } from "./deno_dir.ts"; 5 | import { FileFetcher } from "./file_fetcher.ts"; 6 | import { createGraph } from "@deno/graph"; 7 | import { assertEquals } from "@std/assert"; 8 | 9 | async function setup() { 10 | const tempdir = await Deno.makeTempDir({ 11 | prefix: "deno_cache_dir_cache_test", 12 | }); 13 | const denoDir = new DenoDir(tempdir); 14 | const fileFetcher = new FileFetcher( 15 | () => { 16 | return denoDir.createHttpCache(); 17 | }, 18 | "use", 19 | true, 20 | ); 21 | return new FetchCacher(fileFetcher); 22 | } 23 | 24 | Deno.test("FetchCacher#load works with createGraph to deal with a JSR package", async () => { 25 | const fetchCacher = await setup(); 26 | 27 | const graph = await createGraph("jsr:@deno/gfm@0.9.0", { 28 | load: fetchCacher.load, 29 | }); 30 | 31 | assertEquals(graph.roots, ["jsr:@deno/gfm@0.9.0"]); 32 | assertEquals( 33 | graph.redirects["jsr:@deno/gfm@0.9.0"], 34 | "https://jsr.io/@deno/gfm/0.9.0/mod.ts", 35 | ); 36 | }); 37 | 38 | Deno.test("FetchCacher#load works with createGraph to deal with a deno.land/x package", async () => { 39 | const fetchCacher = await setup(); 40 | 41 | const graph = await createGraph("https://deno.land/x/oak@v9.0.1/mod.ts", { 42 | load: fetchCacher.load, 43 | }); 44 | 45 | assertEquals(graph.roots, ["https://deno.land/x/oak@v9.0.1/mod.ts"]); 46 | assertEquals(graph.redirects, {}); 47 | }); 48 | -------------------------------------------------------------------------------- /deno.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@deno/cache-dir", 3 | "version": "0.0.0", 4 | "tasks": { 5 | "test": "deno test --allow-read --allow-write --allow-net --allow-env", 6 | "build": "deno task wasmbuild", 7 | "wasmbuild": "deno run -A jsr:@deno/wasmbuild@0.16.0 --sync --no-default-features --features wasm" 8 | }, 9 | "lint": { 10 | "rules": { 11 | "include": ["no-console"] 12 | } 13 | }, 14 | "publish": { 15 | "exclude": [ 16 | "Cargo.lock", 17 | "rs_lib", 18 | "**/*.toml", 19 | "!lib/snippets/", 20 | "!lib/deno_cache_dir.generated.js" 21 | ] 22 | }, 23 | "exclude": ["target"], 24 | "exports": "./mod.ts", 25 | "imports": { 26 | "@deno/graph": "jsr:@deno/graph@^0.86.0", 27 | "@std/assert": "jsr:@std/assert@^1.0.8", 28 | "@std/fmt": "jsr:@std/fmt@^1.0.3", 29 | "@std/fs": "jsr:@std/fs@^1.0.6", 30 | "@std/io": "jsr:@std/io@^0.225.0", 31 | "@std/path": "jsr:@std/path@^1.0.8" 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /deno.lock: -------------------------------------------------------------------------------- 1 | { 2 | "version": "4", 3 | "specifiers": { 4 | "jsr:@deno/graph@0.86": "0.86.0", 5 | "jsr:@std/assert@^1.0.8": "1.0.8", 6 | "jsr:@std/bytes@^1.0.2": "1.0.4", 7 | "jsr:@std/fmt@^1.0.3": "1.0.3", 8 | "jsr:@std/fs@^1.0.6": "1.0.6", 9 | "jsr:@std/internal@^1.0.5": "1.0.5", 10 | "jsr:@std/io@0.225": "0.225.0", 11 | "jsr:@std/path@^1.0.8": "1.0.8" 12 | }, 13 | "jsr": { 14 | "@deno/graph@0.86.0": { 15 | "integrity": "749bece1db357b1b1e47708b8fd3badea5fdfc3f9e714167c7d985c4d7f6df26" 16 | }, 17 | "@std/assert@1.0.8": { 18 | "integrity": "ebe0bd7eb488ee39686f77003992f389a06c3da1bbd8022184804852b2fa641b", 19 | "dependencies": [ 20 | "jsr:@std/internal" 21 | ] 22 | }, 23 | "@std/bytes@1.0.4": { 24 | "integrity": "11a0debe522707c95c7b7ef89b478c13fb1583a7cfb9a85674cd2cc2e3a28abc" 25 | }, 26 | "@std/fmt@1.0.3": { 27 | "integrity": "97765c16aa32245ff4e2204ecf7d8562496a3cb8592340a80e7e554e0bb9149f" 28 | }, 29 | "@std/fs@1.0.6": { 30 | "integrity": "42b56e1e41b75583a21d5a37f6a6a27de9f510bcd36c0c85791d685ca0b85fa2", 31 | "dependencies": [ 32 | "jsr:@std/path" 33 | ] 34 | }, 35 | "@std/internal@1.0.5": { 36 | "integrity": "54a546004f769c1ac9e025abd15a76b6671ddc9687e2313b67376125650dc7ba" 37 | }, 38 | "@std/io@0.225.0": { 39 | "integrity": "c1db7c5e5a231629b32d64b9a53139445b2ca640d828c26bf23e1c55f8c079b3", 40 | "dependencies": [ 41 | "jsr:@std/bytes" 42 | ] 43 | }, 44 | "@std/path@1.0.8": { 45 | "integrity": "548fa456bb6a04d3c1a1e7477986b6cffbce95102d0bb447c67c4ee70e0364be" 46 | } 47 | }, 48 | "workspace": { 49 | "dependencies": [ 50 | "jsr:@deno/graph@0.86", 51 | "jsr:@std/assert@^1.0.8", 52 | "jsr:@std/fmt@^1.0.3", 53 | "jsr:@std/fs@^1.0.6", 54 | "jsr:@std/io@0.225", 55 | "jsr:@std/path@^1.0.8" 56 | ] 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /deno_dir.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | import { isAbsolute, join, resolve } from "@std/path"; 4 | import { DiskCache } from "./disk_cache.ts"; 5 | import { HttpCache } from "./http_cache.ts"; 6 | import { assert } from "./util.ts"; 7 | import { instantiate } from "./lib/deno_cache_dir.generated.js"; 8 | 9 | export class DenoDir { 10 | readonly root: string; 11 | 12 | constructor(root?: string | URL) { 13 | const resolvedRoot = DenoDir.tryResolveRootPath(root); 14 | assert(resolvedRoot, "Could not set the Deno root directory"); 15 | assert( 16 | isAbsolute(resolvedRoot), 17 | `The root directory "${resolvedRoot}" is not absolute.`, 18 | ); 19 | Deno.permissions.request({ name: "read", path: resolvedRoot }); 20 | this.root = resolvedRoot; 21 | } 22 | 23 | createGenCache(): DiskCache { 24 | return new DiskCache(join(this.root, "gen")); 25 | } 26 | 27 | createHttpCache( 28 | options?: { vendorRoot?: string | URL; readOnly?: boolean }, 29 | ): Promise { 30 | return HttpCache.create({ 31 | root: join(this.root, "remote"), 32 | vendorRoot: options?.vendorRoot == null 33 | ? undefined 34 | : resolvePathOrUrl(options.vendorRoot), 35 | readOnly: options?.readOnly, 36 | }); 37 | } 38 | 39 | static tryResolveRootPath( 40 | root: string | URL | undefined, 41 | ): string | undefined { 42 | if (root) { 43 | return resolvePathOrUrl(root); 44 | } else { 45 | const instance = instantiate(); 46 | return instance.resolve_deno_dir(); 47 | } 48 | } 49 | } 50 | 51 | function resolvePathOrUrl(path: URL | string) { 52 | if (path instanceof URL) { 53 | path = path.toString(); 54 | } 55 | return resolve(path); 56 | } 57 | -------------------------------------------------------------------------------- /deno_dir_test.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | import { assertEquals, assertThrows } from "@std/assert"; 4 | import { DenoDir } from "./deno_dir.ts"; 5 | import { withTempDir } from "./deps_test.ts"; 6 | 7 | Deno.test({ 8 | name: "DenoDir - basic", 9 | async fn() { 10 | const expectedText = 11 | `// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. 12 | // Copyright the Browserify authors. MIT License. 13 | 14 | /** 15 | * Ported mostly from https://github.com/browserify/path-browserify/ 16 | * This module is browser compatible. 17 | * @module 18 | */ 19 | 20 | import { isWindows } from "../_util/os.ts"; 21 | import * as _win32 from "./win32.ts"; 22 | import * as _posix from "./posix.ts"; 23 | 24 | const path = isWindows ? _win32 : _posix; 25 | 26 | export const win32 = _win32; 27 | export const posix = _posix; 28 | export const { 29 | basename, 30 | delimiter, 31 | dirname, 32 | extname, 33 | format, 34 | fromFileUrl, 35 | isAbsolute, 36 | join, 37 | normalize, 38 | parse, 39 | relative, 40 | resolve, 41 | sep, 42 | toFileUrl, 43 | toNamespacedPath, 44 | } = path; 45 | 46 | export * from "./common.ts"; 47 | export { SEP, SEP_PATTERN } from "./separator.ts"; 48 | export * from "./_interface.ts"; 49 | export * from "./glob.ts"; 50 | `; 51 | const denoDir = new DenoDir(); 52 | const url = new URL("https://deno.land/std@0.140.0/path/mod.ts"); 53 | const expectedHeaders = { 54 | "content-type": "application/typescript", 55 | }; 56 | const deps = await denoDir.createHttpCache(); 57 | deps.set(url, expectedHeaders, new TextEncoder().encode(expectedText)); 58 | const headers = deps.getHeaders(url)!; 59 | assertEquals(headers, expectedHeaders); 60 | const cacheEntry = deps.get(url)!; 61 | assertEquals(cacheEntry.headers, expectedHeaders); 62 | const text = new TextDecoder().decode(cacheEntry.content); 63 | assertEquals(text, expectedText); 64 | 65 | // ok 66 | deps.get( 67 | url, 68 | { 69 | checksum: 70 | "d3e68d0abb393fb0bf94a6d07c46ec31dc755b544b13144dee931d8d5f06a52d", 71 | }, 72 | ); 73 | // not ok 74 | assertThrows(() => 75 | deps.get(url, { 76 | checksum: "invalid", 77 | }) 78 | ); 79 | }, 80 | }); 81 | 82 | Deno.test({ 83 | name: "HttpCache - global cache - get", 84 | async fn() { 85 | const denoDir = new DenoDir(); 86 | const url = new URL("https://deno.land/std@0.140.0/path/mod.ts"); 87 | const deps = await denoDir.createHttpCache(); 88 | // disallow will still work because we're using a global cache 89 | // which is not affected by this option 90 | const entry = await deps.get(url); 91 | assertEquals(entry!.content.length, 820); 92 | }, 93 | }); 94 | 95 | Deno.test({ 96 | name: "HttpCache - local cache- allowCopyGlobalToLocal", 97 | async fn() { 98 | await withTempDir(async (tempDir) => { 99 | const denoDir = new DenoDir(); 100 | const url = new URL("https://deno.land/std@0.140.0/path/mod.ts"); 101 | 102 | // disallow copy from global to local because readonly 103 | { 104 | using deps = await denoDir.createHttpCache({ 105 | vendorRoot: tempDir, 106 | readOnly: true, 107 | }); 108 | const text = deps.get(url); 109 | assertEquals(text, undefined); 110 | } 111 | // this should be fine though 112 | { 113 | using deps = await denoDir.createHttpCache({ 114 | vendorRoot: tempDir, 115 | }); 116 | const entry = deps.get(url); 117 | assertEquals(entry!.content.length, 820); 118 | } 119 | }); 120 | }, 121 | }); 122 | -------------------------------------------------------------------------------- /deps_test.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | export { assertEquals, assertRejects } from "@std/assert"; 4 | export { createGraph } from "@deno/graph"; 5 | 6 | export async function withTempDir( 7 | action: (path: string) => Promise | void, 8 | ) { 9 | const tempDir = Deno.makeTempDirSync(); 10 | try { 11 | await action(tempDir); 12 | } finally { 13 | Deno.removeSync(tempDir, { recursive: true }); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /disk_cache.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | import { ensureDir } from "@std/fs/ensure-dir"; 4 | import { dirname, isAbsolute, join } from "@std/path"; 5 | import { readAll, writeAll } from "@std/io"; 6 | import { assert, CACHE_PERM } from "./util.ts"; 7 | import { instantiate } from "./lib/deno_cache_dir.generated.js"; 8 | 9 | export class DiskCache { 10 | location: string; 11 | 12 | constructor(location: string) { 13 | assert(isAbsolute(location)); 14 | this.location = location; 15 | } 16 | 17 | async get(filename: string): Promise { 18 | const path = join(this.location, filename); 19 | const file = await Deno.open(path, { read: true }); 20 | const value = await readAll(file); 21 | file.close(); 22 | return value; 23 | } 24 | 25 | async set(filename: string, data: Uint8Array): Promise { 26 | const path = join(this.location, filename); 27 | const parentFilename = dirname(path); 28 | await ensureDir(parentFilename); 29 | const file = await Deno.open(path, { 30 | write: true, 31 | create: true, 32 | mode: CACHE_PERM, 33 | }); 34 | await writeAll(file, data); 35 | file.close(); 36 | } 37 | 38 | static async getCacheFilename(url: URL): Promise { 39 | const { url_to_filename } = await instantiate(); 40 | return url_to_filename(url.toString()); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /disk_cache_test.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | import { assertEquals, assertRejects } from "@std/assert"; 4 | import { DiskCache } from "./disk_cache.ts"; 5 | 6 | Deno.test({ 7 | name: "DiskCache.getCacheFilename()", 8 | async fn() { 9 | const testCases: [string, string | Error][] = [ 10 | [ 11 | "http://deno.land/std/http/file_server.ts", 12 | "http/deno.land/d8300752800fe3f0beda9505dc1c3b5388beb1ee45afd1f1e2c9fc0866df15cf", 13 | ], 14 | [ 15 | "http://localhost:8000/std/http/file_server.ts", 16 | "http/localhost_PORT8000/d8300752800fe3f0beda9505dc1c3b5388beb1ee45afd1f1e2c9fc0866df15cf", 17 | ], 18 | [ 19 | "https://deno.land/std/http/file_server.ts", 20 | "https/deno.land/d8300752800fe3f0beda9505dc1c3b5388beb1ee45afd1f1e2c9fc0866df15cf", 21 | ], 22 | ["wasm://wasm/d1c677ea", new Error(`Can't convert url`)], 23 | [ 24 | "file://127.0.0.1/d$/a/1/s/format.ts", 25 | new Error(`Can't convert url`), 26 | ], 27 | ]; 28 | 29 | for (const [fixture, expected] of testCases) { 30 | if (expected instanceof Error) { 31 | await assertRejects( 32 | async () => await DiskCache.getCacheFilename(new URL(fixture)), 33 | Error, 34 | expected.message, 35 | ); 36 | continue; 37 | } else { 38 | assertEquals( 39 | await DiskCache.getCacheFilename(new URL(fixture)), 40 | expected, 41 | ); 42 | } 43 | } 44 | }, 45 | }); 46 | -------------------------------------------------------------------------------- /file_fetcher.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | import { AuthTokens } from "./auth_tokens.ts"; 4 | import { fromFileUrl } from "@std/path"; 5 | import * as colors from "@std/fmt/colors"; 6 | import type { LoadResponse } from "@deno/graph"; 7 | import type { HttpCache, HttpCacheGetOptions } from "./http_cache.ts"; 8 | 9 | /** 10 | * A setting that determines how the cache is handled for remote dependencies. 11 | * 12 | * The default is `"use"`. 13 | * 14 | * - `"only"` - only the cache will be re-used, and any remote modules not in 15 | * the cache will error. 16 | * - `"use"` - the cache will be used, meaning existing remote files will not be 17 | * reloaded. 18 | * - `"reload"` - any cached modules will be ignored and their values will be 19 | * fetched. 20 | * - `string[]` - an array of string specifiers, that if they match the start of 21 | * the requested specifier, will be reloaded. 22 | */ 23 | export type CacheSetting = "only" | "reload" | "use" | string[]; 24 | 25 | function shouldUseCache(cacheSetting: CacheSetting, specifier: URL): boolean { 26 | switch (cacheSetting) { 27 | case "only": 28 | case "use": 29 | return true; 30 | // @ts-ignore old setting 31 | case "reloadAll": 32 | case "reload": 33 | return false; 34 | default: { 35 | const specifierStr = specifier.toString(); 36 | for (const value of cacheSetting) { 37 | if (specifierStr.startsWith(value)) { 38 | return false; 39 | } 40 | } 41 | return true; 42 | } 43 | } 44 | } 45 | 46 | const SUPPORTED_SCHEMES = [ 47 | "data:", 48 | "blob:", 49 | "file:", 50 | "http:", 51 | "https:", 52 | ] as const; 53 | 54 | type SupportedSchemes = typeof SUPPORTED_SCHEMES[number]; 55 | 56 | function getValidatedScheme(specifier: URL) { 57 | const scheme = specifier.protocol; 58 | // deno-lint-ignore no-explicit-any 59 | if (!SUPPORTED_SCHEMES.includes(scheme as any)) { 60 | throw new TypeError( 61 | `Unsupported scheme "${scheme}" for module "${specifier.toString()}". Supported schemes: ${ 62 | JSON.stringify(SUPPORTED_SCHEMES) 63 | }.`, 64 | ); 65 | } 66 | return scheme as SupportedSchemes; 67 | } 68 | 69 | function hasHashbang(value: Uint8Array): boolean { 70 | return value[0] === 35 /* # */ && value[1] === 33 /* ! */; 71 | } 72 | 73 | function stripHashbang(value: Uint8Array): string | Uint8Array { 74 | if (hasHashbang(value)) { 75 | const text = new TextDecoder().decode(value); 76 | const lineIndex = text.indexOf("\n"); 77 | if (lineIndex > 0) { 78 | return text.slice(lineIndex + 1); 79 | } else { 80 | return value; 81 | } 82 | } else { 83 | return value; 84 | } 85 | } 86 | 87 | async function fetchLocal(specifier: URL): Promise { 88 | const local = fromFileUrl(specifier); 89 | if (!local) { 90 | throw new TypeError( 91 | `Invalid file path.\n Specifier: "${specifier.toString()}"`, 92 | ); 93 | } 94 | try { 95 | const content = stripHashbang(await Deno.readFile(local)); 96 | return { 97 | kind: "module", 98 | content, 99 | specifier: specifier.toString(), 100 | }; 101 | } catch { 102 | // ignoring errors, we will just return undefined 103 | } 104 | } 105 | 106 | type ResolvedFetchOptions = 107 | & Omit 108 | & Pick, "cacheSetting">; 109 | 110 | interface FetchOptions extends HttpCacheGetOptions { 111 | cacheSetting?: CacheSetting; 112 | } 113 | 114 | export class FileFetcher { 115 | #allowRemote: boolean; 116 | #authTokens: AuthTokens; 117 | #cache = new Map(); 118 | #cacheSetting: CacheSetting; 119 | #httpCache: HttpCache | undefined; 120 | #httpCachePromise: Promise | undefined; 121 | #httpCacheFactory: () => Promise; 122 | 123 | constructor( 124 | httpCacheFactory: () => Promise, 125 | cacheSetting: CacheSetting = "use", 126 | allowRemote = true, 127 | ) { 128 | Deno.permissions.request({ name: "env", variable: "DENO_AUTH_TOKENS" }); 129 | this.#authTokens = new AuthTokens(Deno.env.get("DENO_AUTH_TOKENS")); 130 | this.#allowRemote = allowRemote; 131 | this.#cacheSetting = cacheSetting; 132 | this.#httpCacheFactory = httpCacheFactory; 133 | } 134 | 135 | async #fetchBlobDataUrl( 136 | specifier: URL, 137 | options: ResolvedFetchOptions, 138 | httpCache: HttpCache, 139 | ): Promise { 140 | const cached = this.fetchCachedOnce(specifier, options, httpCache); 141 | if (cached) { 142 | return cached; 143 | } 144 | 145 | if (options.cacheSetting === "only") { 146 | throw new Deno.errors.NotFound( 147 | `Specifier not found in cache: "${specifier.toString()}", --cached-only is specified.`, 148 | ); 149 | } 150 | 151 | const response = await fetchWithRetries(specifier.toString()); 152 | const content = new Uint8Array(await response.arrayBuffer()); 153 | const headers: Record = {}; 154 | for (const [key, value] of response.headers) { 155 | headers[key.toLowerCase()] = value; 156 | } 157 | httpCache.set(specifier, headers, content); 158 | return { 159 | kind: "module", 160 | specifier: specifier.toString(), 161 | headers, 162 | content, 163 | }; 164 | } 165 | 166 | fetchCachedOnce( 167 | specifier: URL, 168 | options: ResolvedFetchOptions, 169 | httpCache: HttpCache, 170 | ): LoadResponse | undefined { 171 | const cacheEntry = httpCache.get(specifier, options); 172 | if (!cacheEntry) { 173 | return undefined; 174 | } 175 | const location = cacheEntry.headers["location"]; 176 | if (location != null && location.length > 0) { 177 | const redirect = new URL(location, specifier); 178 | return { 179 | kind: "redirect", 180 | specifier: redirect.toString(), 181 | }; 182 | } 183 | return { 184 | kind: "module", 185 | specifier: specifier.toString(), 186 | headers: cacheEntry.headers, 187 | content: cacheEntry.content, 188 | }; 189 | } 190 | 191 | async #fetchRemoteOnce( 192 | specifier: URL, 193 | options: ResolvedFetchOptions, 194 | httpCache: HttpCache, 195 | ): Promise { 196 | if (shouldUseCache(options.cacheSetting, specifier)) { 197 | const response = this.fetchCachedOnce( 198 | specifier, 199 | options, 200 | httpCache, 201 | ); 202 | if (response) { 203 | return response; 204 | } 205 | } 206 | 207 | if (options.cacheSetting === "only") { 208 | throw new Deno.errors.NotFound( 209 | `Specifier not found in cache: "${specifier.toString()}", --cached-only is specified.`, 210 | ); 211 | } 212 | 213 | const requestHeaders = new Headers(); 214 | const cachedHeaders = httpCache.getHeaders(specifier); 215 | if (cachedHeaders) { 216 | const etag = cachedHeaders["etag"]; 217 | if (etag != null && etag.length > 0) { 218 | requestHeaders.append("if-none-match", etag); 219 | } 220 | } 221 | const authToken = this.#authTokens.get(specifier); 222 | if (authToken) { 223 | requestHeaders.append("authorization", authToken); 224 | } 225 | // deno-lint-ignore no-console 226 | console.error(`${colors.green("Download")} ${specifier.toString()}`); 227 | const response = await fetchWithRetries(specifier.toString(), { 228 | headers: requestHeaders, 229 | }); 230 | if (!response.ok) { 231 | if (response.status === 404) { 232 | return undefined; 233 | } else { 234 | throw new Deno.errors.Http(`${response.status} ${response.statusText}`); 235 | } 236 | } 237 | // WHATWG fetch follows redirects automatically, so we will try to 238 | // determine if that occurred and cache the value. 239 | if (specifier.toString() !== response.url) { 240 | const headers = { "location": response.url }; 241 | httpCache.set(specifier, headers, new Uint8Array()); 242 | } 243 | const url = new URL(response.url); 244 | const content = new Uint8Array(await response.arrayBuffer()); 245 | const headers: Record = {}; 246 | for (const [key, value] of response.headers) { 247 | headers[key.toLowerCase()] = value; 248 | } 249 | httpCache.set(url, headers, content); 250 | if (options?.checksum != null) { 251 | const digest = await crypto.subtle.digest("SHA-256", content); 252 | const actualChecksum = Array.from(new Uint8Array(digest)) 253 | .map((b) => b.toString(16).padStart(2, "0")) 254 | .join(""); 255 | if (actualChecksum != options.checksum) { 256 | throw new Error( 257 | `Integrity check failed for ${url}\n\nActual: ${actualChecksum}\nExpected: ${options.checksum}`, 258 | ); 259 | } 260 | } 261 | return { 262 | kind: "module", 263 | specifier: response.url, 264 | headers, 265 | content, 266 | }; 267 | } 268 | 269 | async fetch( 270 | specifier: URL, 271 | // Providing a checksum here doesn't make sense because the provided 272 | // checksum will change based on the specifier being requested, which 273 | // could be invalided by a redirect 274 | options?: Omit, 275 | ): Promise { 276 | for (let i = 0; i <= 10; i++) { 277 | const response = await this.fetchOnce(specifier, options); 278 | if (response?.kind !== "redirect") { 279 | return response; 280 | } 281 | specifier = new URL(response.specifier); 282 | } 283 | throw new Deno.errors.Http( 284 | `Too many redirects.\n Specifier: "${specifier.toString()}"`, 285 | ); 286 | } 287 | 288 | async fetchOnce( 289 | specifier: URL, 290 | options?: FetchOptions, 291 | ): Promise { 292 | const scheme = getValidatedScheme(specifier); 293 | if (scheme === "file:") { 294 | return fetchLocal(specifier); 295 | } 296 | const response = this.#cache.get(specifier.toString()); 297 | if (response) { 298 | return response; 299 | } else if (scheme === "data:" || scheme === "blob:") { 300 | const response = await this.#fetchBlobDataUrl( 301 | specifier, 302 | this.#resolveOptions(options), 303 | await this.#resolveHttpCache(), 304 | ); 305 | await this.#cache.set(specifier.toString(), response); 306 | return response; 307 | } else if (!this.#allowRemote) { 308 | throw new Deno.errors.PermissionDenied( 309 | `A remote specifier was requested: "${specifier.toString()}", but --no-remote is specified.`, 310 | ); 311 | } else { 312 | const response = await this.#fetchRemoteOnce( 313 | specifier, 314 | this.#resolveOptions(options), 315 | await this.#resolveHttpCache(), 316 | ); 317 | if (response) { 318 | await this.#cache.set(specifier.toString(), response); 319 | } 320 | return response; 321 | } 322 | } 323 | 324 | #resolveOptions(options?: FetchOptions): ResolvedFetchOptions { 325 | options ??= {}; 326 | options.cacheSetting = options.cacheSetting ?? this.#cacheSetting; 327 | return options as ResolvedFetchOptions; 328 | } 329 | 330 | #resolveHttpCache(): Promise { 331 | if (this.#httpCache != null) { 332 | return Promise.resolve(this.#httpCache); 333 | } 334 | if (!this.#httpCachePromise) { 335 | this.#httpCachePromise = this.#httpCacheFactory().then((cache) => { 336 | this.#httpCache = cache; 337 | this.#httpCachePromise = undefined; 338 | return cache; 339 | }); 340 | } 341 | return this.#httpCachePromise; 342 | } 343 | } 344 | 345 | export async function fetchWithRetries( 346 | url: URL | string, 347 | init?: { headers?: Headers }, 348 | ) { 349 | const maxRetries = 3; 350 | let sleepMs = 250; 351 | let iterationCount = 0; 352 | while (true) { 353 | iterationCount++; 354 | try { 355 | const res = await fetch(url, init); 356 | if ( 357 | res.ok || iterationCount > maxRetries || 358 | res.status >= 400 && res.status < 500 359 | ) { 360 | return res; 361 | } 362 | } catch (err) { 363 | if (iterationCount > maxRetries) { 364 | throw err; 365 | } 366 | } 367 | // deno-lint-ignore no-console 368 | console.warn( 369 | `${ 370 | colors.yellow("WARN") 371 | } Failed fetching ${url}. Retrying in ${sleepMs}ms...`, 372 | ); 373 | await new Promise((resolve) => setTimeout(resolve, sleepMs)); 374 | sleepMs = Math.min(sleepMs * 2, 10_000); 375 | } 376 | } 377 | -------------------------------------------------------------------------------- /file_fetcher_test.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | import { DenoDir } from "./deno_dir.ts"; 4 | import { assertRejects, createGraph } from "./deps_test.ts"; 5 | import { FileFetcher } from "./file_fetcher.ts"; 6 | 7 | Deno.test({ 8 | name: "FileFetcher", 9 | async fn() { 10 | const denoDir = new DenoDir(); 11 | const fileFetcher = new FileFetcher(() => denoDir.createHttpCache()); 12 | const graph = await createGraph("https://deno.land/x/oak@v10.5.1/mod.ts", { 13 | load(specifier) { 14 | return fileFetcher.fetch(new URL(specifier)); 15 | }, 16 | }); 17 | // deno-lint-ignore no-console 18 | console.log(graph); 19 | }, 20 | }); 21 | 22 | Deno.test({ 23 | name: "FileFetcher - bad checksum no cache", 24 | async fn() { 25 | const denoDir = new DenoDir(); 26 | const fileFetcher = new FileFetcher(() => denoDir.createHttpCache()); 27 | { 28 | // should error 29 | await assertRejects(async () => { 30 | await fileFetcher.fetchOnce( 31 | new URL("https://deno.land/x/oak@v10.5.1/mod.ts"), 32 | { 33 | checksum: "bad", 34 | }, 35 | ); 36 | }); 37 | // ok for good checksum 38 | await fileFetcher.fetchOnce( 39 | new URL("https://deno.land/x/oak@v10.5.1/mod.ts"), 40 | { 41 | checksum: 42 | "7a1b5169ef702e96dd994168879dbcbd8af4f639578b6300cbe1c6995d7f3f32", 43 | }, 44 | ); 45 | } 46 | }, 47 | }); 48 | 49 | Deno.test({ 50 | name: "FileFetcher - bad checksum reload", 51 | async fn() { 52 | const denoDir = new DenoDir(); 53 | const fileFetcher = new FileFetcher(() => denoDir.createHttpCache()); 54 | await assertRejects(async () => { 55 | await fileFetcher.fetchOnce( 56 | new URL("https://deno.land/x/oak@v10.5.1/mod.ts"), 57 | { 58 | cacheSetting: "reload", 59 | checksum: "bad", 60 | }, 61 | ); 62 | }); 63 | }, 64 | }); 65 | 66 | Deno.test({ 67 | name: "FileFetcher - good checksum reload", 68 | async fn() { 69 | const denoDir = new DenoDir(); 70 | const fileFetcher = new FileFetcher(() => denoDir.createHttpCache()); 71 | await fileFetcher.fetchOnce( 72 | new URL("https://deno.land/x/oak@v10.5.1/mod.ts"), 73 | { 74 | cacheSetting: "reload", 75 | checksum: 76 | "7a1b5169ef702e96dd994168879dbcbd8af4f639578b6300cbe1c6995d7f3f32", 77 | }, 78 | ); 79 | }, 80 | }); 81 | -------------------------------------------------------------------------------- /http_cache.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | import { isAbsolute } from "@std/path"; 4 | import { assert } from "./util.ts"; 5 | import { 6 | type GlobalHttpCache, 7 | instantiate, 8 | type LocalHttpCache, 9 | } from "./lib/deno_cache_dir.generated.js"; 10 | 11 | export interface HttpCacheCreateOptions { 12 | root: string; 13 | vendorRoot?: string; 14 | readOnly?: boolean; 15 | } 16 | 17 | export interface HttpCacheGetOptions { 18 | /** Checksum to evaluate the file against. This is only evaluated for the 19 | * global cache (DENO_DIR) and not the local cache (vendor folder). 20 | */ 21 | checksum?: string; 22 | } 23 | 24 | export interface HttpCacheEntry { 25 | headers: Record; 26 | content: Uint8Array; 27 | } 28 | 29 | export class HttpCache implements Disposable { 30 | #cache: LocalHttpCache | GlobalHttpCache; 31 | #readOnly: boolean | undefined; 32 | 33 | private constructor( 34 | cache: LocalHttpCache | GlobalHttpCache, 35 | readOnly: boolean | undefined, 36 | ) { 37 | this.#cache = cache; 38 | this.#readOnly = readOnly; 39 | } 40 | 41 | static async create(options: HttpCacheCreateOptions): Promise { 42 | assert(isAbsolute(options.root), "Root must be an absolute path."); 43 | 44 | if (options.vendorRoot != null) { 45 | assert( 46 | isAbsolute(options.vendorRoot), 47 | "Vendor root must be an absolute path.", 48 | ); 49 | } 50 | const { GlobalHttpCache, LocalHttpCache } = await instantiate(); 51 | 52 | let cache: LocalHttpCache | GlobalHttpCache; 53 | if (options.vendorRoot != null) { 54 | cache = LocalHttpCache.new( 55 | options.vendorRoot, 56 | options.root, 57 | /* allow global to local copy */ !options.readOnly, 58 | ); 59 | } else { 60 | cache = GlobalHttpCache.new(options.root); 61 | } 62 | return new HttpCache(cache, options.readOnly); 63 | } 64 | 65 | [Symbol.dispose]() { 66 | this.free(); 67 | } 68 | 69 | free() { 70 | this.#cache?.free(); 71 | } 72 | 73 | getHeaders( 74 | url: URL, 75 | ): Record | undefined { 76 | const map = this.#cache.getHeaders(url.toString()); 77 | return map == null ? undefined : Object.fromEntries(map); 78 | } 79 | 80 | get( 81 | url: URL, 82 | options?: HttpCacheGetOptions, 83 | ): HttpCacheEntry | undefined { 84 | const data = this.#cache.get( 85 | url.toString(), 86 | options?.checksum, 87 | ); 88 | return data == null ? undefined : data; 89 | } 90 | 91 | set( 92 | url: URL, 93 | headers: Record, 94 | content: Uint8Array, 95 | ): void { 96 | if (this.#readOnly === undefined) { 97 | this.#readOnly = 98 | (Deno.permissions.querySync({ name: "write" })).state === "denied" 99 | ? true 100 | : false; 101 | } 102 | if (this.#readOnly) { 103 | return; 104 | } 105 | this.#cache.set( 106 | url.toString(), 107 | headers, 108 | content, 109 | ); 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /lib/deno_cache_dir.generated.d.ts: -------------------------------------------------------------------------------- 1 | // deno-lint-ignore-file 2 | // deno-fmt-ignore-file 3 | 4 | export interface InstantiateResult { 5 | instance: WebAssembly.Instance; 6 | exports: { 7 | url_to_filename: typeof url_to_filename; 8 | resolve_deno_dir: typeof resolve_deno_dir; 9 | GlobalHttpCache : typeof GlobalHttpCache ; 10 | LocalHttpCache : typeof LocalHttpCache 11 | }; 12 | } 13 | 14 | /** Gets if the Wasm module has been instantiated. */ 15 | export function isInstantiated(): boolean; 16 | 17 | 18 | /** Instantiates an instance of the Wasm module returning its functions. 19 | * @remarks It is safe to call this multiple times and once successfully 20 | * loaded it will always return a reference to the same object. */ 21 | export function instantiate(): InstantiateResult["exports"]; 22 | 23 | /** Instantiates an instance of the Wasm module along with its exports. 24 | * @remarks It is safe to call this multiple times and once successfully 25 | * loaded it will always return a reference to the same object. */ 26 | export function instantiateWithInstance(): InstantiateResult; 27 | 28 | /** 29 | * @param {string} url 30 | * @returns {string} 31 | */ 32 | export function url_to_filename(url: string): string; 33 | /** 34 | * @param {string | undefined} [maybe_custom_root] 35 | * @returns {string} 36 | */ 37 | export function resolve_deno_dir(maybe_custom_root?: string): string; 38 | /** 39 | */ 40 | export class GlobalHttpCache { 41 | free(): void; 42 | /** 43 | * @param {string} path 44 | * @returns {GlobalHttpCache} 45 | */ 46 | static new(path: string): GlobalHttpCache; 47 | /** 48 | * @param {string} url 49 | * @returns {any} 50 | */ 51 | getHeaders(url: string): any; 52 | /** 53 | * @param {string} url 54 | * @param {string | undefined} [maybe_checksum] 55 | * @returns {any} 56 | */ 57 | get(url: string, maybe_checksum?: string): any; 58 | /** 59 | * @param {string} url 60 | * @param {any} headers 61 | * @param {Uint8Array} text 62 | */ 63 | set(url: string, headers: any, text: Uint8Array): void; 64 | } 65 | /** 66 | */ 67 | export class LocalHttpCache { 68 | free(): void; 69 | /** 70 | * @param {string} local_path 71 | * @param {string} global_path 72 | * @param {boolean} allow_global_to_local_copy 73 | * @returns {LocalHttpCache} 74 | */ 75 | static new(local_path: string, global_path: string, allow_global_to_local_copy: boolean): LocalHttpCache; 76 | /** 77 | * @param {string} url 78 | * @returns {any} 79 | */ 80 | getHeaders(url: string): any; 81 | /** 82 | * @param {string} url 83 | * @param {string | undefined} [maybe_checksum] 84 | * @returns {any} 85 | */ 86 | get(url: string, maybe_checksum?: string): any; 87 | /** 88 | * @param {string} url 89 | * @param {any} headers 90 | * @param {Uint8Array} text 91 | */ 92 | set(url: string, headers: any, text: Uint8Array): void; 93 | } 94 | -------------------------------------------------------------------------------- /mod.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | /** 4 | * A module which provides a TypeScript implementation of the Deno CLI's cache 5 | * directory logic (`DENO_DIR`). This can be used in combination with other 6 | * modules to provide user loadable APIs that are like the Deno CLI's 7 | * functionality. 8 | * 9 | * This also can provide user read access in Deploy to a Deno CLI's cache when 10 | * the cache is checked into the repository. 11 | * 12 | * ### Example 13 | * 14 | * ```ts 15 | * import { createCache } from "@deno/cache-dir"; 16 | * import { createGraph } from "@deno/graph"; 17 | * 18 | * // create a cache where the location will be determined environmentally 19 | * const cache = createCache(); 20 | * // destructuring the two functions we need to pass to the graph 21 | * const { cacheInfo, load } = cache; 22 | * // create a graph that will use the cache above to load and cache dependencies 23 | * const graph = await createGraph("https://deno.land/x/oak@v9.0.1/mod.ts", { 24 | * cacheInfo, 25 | * load, 26 | * }); 27 | * 28 | * // log out the console a similar output to `deno info` on the command line. 29 | * console.log(graph.toString()); 30 | * ``` 31 | * 32 | * @module 33 | */ 34 | 35 | import { FetchCacher } from "./cache.ts"; 36 | import type { CacheInfo, LoadResponse } from "@deno/graph"; 37 | import { DenoDir } from "./deno_dir.ts"; 38 | import { type CacheSetting, FileFetcher } from "./file_fetcher.ts"; 39 | 40 | export { FetchCacher } from "./cache.ts"; 41 | export { DenoDir } from "./deno_dir.ts"; 42 | export { HttpCache } from "./http_cache.ts"; 43 | export { DiskCache } from "./disk_cache.ts"; 44 | export { type CacheSetting, FileFetcher } from "./file_fetcher.ts"; 45 | 46 | export interface Loader { 47 | /** A function that can be passed to a `deno_graph` building function to 48 | * provide information about the cache to populate the output. 49 | */ 50 | cacheInfo?(specifier: string): CacheInfo; 51 | /** A function that can be passed to a `deno_graph` that will load and cache 52 | * dependencies in the graph in the disk cache. 53 | */ 54 | load( 55 | specifier: string, 56 | isDynamic?: boolean, 57 | cacheSetting?: CacheSetting, 58 | checksum?: string, 59 | ): Promise; 60 | } 61 | 62 | export type { LoadResponse } from "@deno/graph"; 63 | export type { 64 | LoadResponseExternal, 65 | LoadResponseModule, 66 | } from "@deno/graph/types"; 67 | 68 | export interface CacheOptions { 69 | /** Allow remote URLs to be fetched if missing from the cache. This defaults 70 | * to `true`. Setting it to `false` is like passing the `--no-remote` in the 71 | * Deno CLI, meaning that any modules not in cache error. */ 72 | allowRemote?: boolean; 73 | /** Determines how the cache will be used. The default value is `"use"` 74 | * meaning the cache will be used, and any remote module cache misses will 75 | * be fetched and stored in the cache. */ 76 | cacheSetting?: CacheSetting; 77 | /** This forces the cache into a `readOnly` mode, where fetched resources 78 | * will not be stored on disk if `true`. The default is detected from the 79 | * environment, checking to see if `Deno.writeFile` exists. */ 80 | readOnly?: boolean; 81 | /** Specifies a path to the root of the cache. Setting this value overrides 82 | * the detection of location from the environment. */ 83 | root?: string | URL; 84 | /** Specifies a path to the local vendor directory if it exists. */ 85 | vendorRoot?: string | URL; 86 | } 87 | 88 | /** 89 | * Creates a cache object that allows access to the internal `DENO_DIR` cache 90 | * structure for remote dependencies and cached output of emitted modules. 91 | */ 92 | export function createCache({ 93 | root, 94 | cacheSetting = "use", 95 | allowRemote = true, 96 | readOnly, 97 | vendorRoot, 98 | }: CacheOptions = {}): Loader { 99 | const denoDir = new DenoDir(root); 100 | const fileFetcher = new FileFetcher( 101 | () => { 102 | return denoDir.createHttpCache({ 103 | readOnly, 104 | vendorRoot, 105 | }); 106 | }, 107 | cacheSetting, 108 | allowRemote, 109 | ); 110 | return new FetchCacher(fileFetcher); 111 | } 112 | -------------------------------------------------------------------------------- /rs_lib/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "deno_cache_dir" 3 | version = "0.22.2" 4 | edition = "2021" 5 | license = "MIT" 6 | description = "Cache directory logic used in Deno" 7 | repository = "https://github.com/denoland/deno_cache" 8 | 9 | [lib] 10 | crate-type = ["cdylib", "lib"] 11 | 12 | [features] 13 | default = ["file_fetcher"] 14 | file_fetcher = ["async-trait", "base64", "cache_control", "chrono", "data-url", "http"] 15 | wasm = ["console_error_panic_hook", "js-sys", "serde-wasm-bindgen", "wasm-bindgen", "sys_traits/wasm"] 16 | sync = [] 17 | 18 | [dependencies] 19 | async-trait = { version = "0.1.73", optional = true } 20 | base32 = "=0.5.1" 21 | base64 = { version = "0.21.7", optional = true } 22 | boxed_error = "0.2.3" 23 | cache_control = { version = "0.2.0", optional = true } 24 | # Note: Do not use the "clock" feature of chrono, as it links us to CoreFoundation on macOS. 25 | chrono = { version = "0.4", default-features = false, features = ["std"], optional = true } 26 | data-url = { version = "0.3.0", optional = true } 27 | deno_error = { version = "0.6.0", features =["url"] } 28 | deno_media_type = "0.2.2" 29 | deno_path_util = "0.4.0" 30 | sys_traits.workspace = true 31 | http = { version = "1", optional = true } 32 | indexmap = { version = "2.0.0", features = ["serde"] } 33 | log = "0.4.19" 34 | once_cell = "1.18.0" 35 | parking_lot = "0.12.1" 36 | serde = "1.0.183" 37 | serde_json = { version = "1.0.104", features = ["preserve_order"] } 38 | sha2 = "0.10.0" 39 | thiserror = "2" 40 | url = { version = "2.5.1", features = ["serde"] } 41 | 42 | console_error_panic_hook = { version = "0.1.6", optional = true } 43 | js-sys = { version = "=0.3.68", optional = true } 44 | wasm-bindgen = { version = "=0.2.91", optional = true } 45 | serde-wasm-bindgen = { version = "0.6.5", optional = true } 46 | 47 | [dev-dependencies] 48 | pretty_assertions = "1.4.0" 49 | sys_traits = { workspace = true, features = ["memory", "real", "getrandom", "libc", "winapi"] } 50 | tempfile = "3.7.1" 51 | tokio = { version = "1", features = ["rt", "macros"] } 52 | -------------------------------------------------------------------------------- /rs_lib/clippy.toml: -------------------------------------------------------------------------------- 1 | disallowed-methods = [ 2 | { path = "std::env::current_dir", reason = "File system operations should be done using DenoCacheEnv trait" }, 3 | { path = "std::path::Path::canonicalize", reason = "File system operations should be done using DenoCacheEnv trait" }, 4 | { path = "std::path::Path::is_dir", reason = "File system operations should be done using DenoCacheEnv trait" }, 5 | { path = "std::path::Path::is_file", reason = "File system operations should be done using DenoCacheEnv trait" }, 6 | { path = "std::path::Path::is_symlink", reason = "File system operations should be done using DenoCacheEnv trait" }, 7 | { path = "std::path::Path::metadata", reason = "File system operations should be done using DenoCacheEnv trait" }, 8 | { path = "std::path::Path::read_dir", reason = "File system operations should be done using DenoCacheEnv trait" }, 9 | { path = "std::path::Path::read_link", reason = "File system operations should be done using DenoCacheEnv trait" }, 10 | { path = "std::path::Path::symlink_metadata", reason = "File system operations should be done using DenoCacheEnv trait" }, 11 | { path = "std::path::Path::try_exists", reason = "File system operations should be done using DenoCacheEnv trait" }, 12 | { path = "std::path::PathBuf::exists", reason = "File system operations should be done using DenoCacheEnv trait" }, 13 | { path = "std::path::PathBuf::canonicalize", reason = "File system operations should be done using DenoCacheEnv trait" }, 14 | { path = "std::path::PathBuf::is_dir", reason = "File system operations should be done using DenoCacheEnv trait" }, 15 | { path = "std::path::PathBuf::is_file", reason = "File system operations should be done using DenoCacheEnv trait" }, 16 | { path = "std::path::PathBuf::is_symlink", reason = "File system operations should be done using DenoCacheEnv trait" }, 17 | { path = "std::path::PathBuf::metadata", reason = "File system operations should be done using DenoCacheEnv trait" }, 18 | { path = "std::path::PathBuf::read_dir", reason = "File system operations should be done using DenoCacheEnv trait" }, 19 | { path = "std::path::PathBuf::read_link", reason = "File system operations should be done using DenoCacheEnv trait" }, 20 | { path = "std::path::PathBuf::symlink_metadata", reason = "File system operations should be done using DenoCacheEnv trait" }, 21 | { path = "std::path::PathBuf::try_exists", reason = "File system operations should be done using DenoCacheEnv trait" }, 22 | { path = "std::env::set_current_dir", reason = "File system operations should be done using DenoCacheEnv trait" }, 23 | { path = "std::env::temp_dir", reason = "File system operations should be done using DenoCacheEnv trait" }, 24 | { path = "std::fs::canonicalize", reason = "File system operations should be done using DenoCacheEnv trait" }, 25 | { path = "std::fs::copy", reason = "File system operations should be done using DenoCacheEnv trait" }, 26 | { path = "std::fs::create_dir_all", reason = "File system operations should be done using DenoCacheEnv trait" }, 27 | { path = "std::fs::create_dir", reason = "File system operations should be done using DenoCacheEnv trait" }, 28 | { path = "std::fs::DirBuilder::new", reason = "File system operations should be done using DenoCacheEnv trait" }, 29 | { path = "std::fs::hard_link", reason = "File system operations should be done using DenoCacheEnv trait" }, 30 | { path = "std::fs::metadata", reason = "File system operations should be done using DenoCacheEnv trait" }, 31 | { path = "std::fs::OpenOptions::new", reason = "File system operations should be done using DenoCacheEnv trait" }, 32 | { path = "std::fs::read_dir", reason = "File system operations should be done using DenoCacheEnv trait" }, 33 | { path = "std::fs::read_link", reason = "File system operations should be done using DenoCacheEnv trait" }, 34 | { path = "std::fs::read_to_string", reason = "File system operations should be done using DenoCacheEnv trait" }, 35 | { path = "std::fs::read", reason = "File system operations should be done using DenoCacheEnv trait" }, 36 | { path = "std::fs::remove_dir_all", reason = "File system operations should be done using DenoCacheEnv trait" }, 37 | { path = "std::fs::remove_dir", reason = "File system operations should be done using DenoCacheEnv trait" }, 38 | { path = "std::fs::remove_file", reason = "File system operations should be done using DenoCacheEnv trait" }, 39 | { path = "std::fs::rename", reason = "File system operations should be done using DenoCacheEnv trait" }, 40 | { path = "std::fs::set_permissions", reason = "File system operations should be done using DenoCacheEnv trait" }, 41 | { path = "std::fs::symlink_metadata", reason = "File system operations should be done using DenoCacheEnv trait" }, 42 | { path = "std::fs::write", reason = "File system operations should be done using DenoCacheEnv trait" }, 43 | { path = "std::path::Path::canonicalize", reason = "File system operations should be done using DenoCacheEnv trait" }, 44 | { path = "std::path::Path::exists", reason = "File system operations should be done using DenoCacheEnv trait" }, 45 | { path = "std::time::SystemTime::now", reason = "Getting the time should be done using DenoCacheEnv trait" }, 46 | ] 47 | disallowed-types = [ 48 | { path = "std::sync::Arc", reason = "use crate::sync::MaybeArc instead" }, 49 | ] 50 | -------------------------------------------------------------------------------- /rs_lib/src/cache.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | use deno_error::JsError; 4 | use serde::Deserialize; 5 | use serde::Serialize; 6 | use std::borrow::Cow; 7 | use std::io::ErrorKind; 8 | use std::path::PathBuf; 9 | use std::time::SystemTime; 10 | use thiserror::Error; 11 | use url::Url; 12 | 13 | use crate::common::base_url_to_filename_parts; 14 | use crate::common::checksum; 15 | use crate::common::HeadersMap; 16 | use crate::global::GlobalHttpCacheSys; 17 | use crate::local::LocalHttpCacheSys; 18 | use crate::sync::MaybeSend; 19 | use crate::sync::MaybeSync; 20 | use crate::GlobalHttpCacheRc; 21 | use crate::LocalHttpCacheRc; 22 | 23 | #[derive(Debug, Copy, Clone, Eq, PartialEq)] 24 | pub enum GlobalToLocalCopy { 25 | /// When using a local cache (vendor folder), allow the cache to 26 | /// copy from the global cache into the local one. 27 | Allow, 28 | /// Disallow copying from the global to the local cache. This is 29 | /// useful for the LSP because we want to ensure that checksums 30 | /// are evaluated for JSR dependencies, which is difficult to do 31 | /// in the LSP. This could be improved in the future to not require 32 | /// this 33 | Disallow, 34 | } 35 | 36 | impl GlobalToLocalCopy { 37 | pub fn is_true(&self) -> bool { 38 | matches!(self, GlobalToLocalCopy::Allow) 39 | } 40 | } 41 | 42 | #[derive(Debug, Error, JsError)] 43 | #[class(type)] 44 | #[error("Integrity check failed for {}\n\nActual: {}\nExpected: {}", .url, .actual, .expected)] 45 | pub struct ChecksumIntegrityError { 46 | pub url: Url, 47 | pub actual: String, 48 | pub expected: String, 49 | } 50 | 51 | #[derive(Debug, Clone, Copy)] 52 | pub struct Checksum<'a>(&'a str); 53 | 54 | impl<'a> Checksum<'a> { 55 | pub fn new(checksum: &'a str) -> Self { 56 | Self(checksum) 57 | } 58 | 59 | pub fn as_str(&self) -> &str { 60 | self.0 61 | } 62 | 63 | pub fn check( 64 | &self, 65 | url: &Url, 66 | content: &[u8], 67 | ) -> Result<(), Box> { 68 | let actual = checksum(content); 69 | if self.as_str() != actual { 70 | Err(Box::new(ChecksumIntegrityError { 71 | url: url.clone(), 72 | expected: self.as_str().to_string(), 73 | actual, 74 | })) 75 | } else { 76 | Ok(()) 77 | } 78 | } 79 | } 80 | 81 | /// Turn provided `url` into a hashed filename. 82 | /// URLs can contain a lot of characters that cannot be used 83 | /// in filenames (like "?", "#", ":"), so in order to cache 84 | /// them properly they are deterministically hashed into ASCII 85 | /// strings. 86 | pub fn url_to_filename(url: &Url) -> std::io::Result { 87 | // Replaces port part with a special string token (because 88 | // ":" cannot be used in filename on some platforms). 89 | // Ex: $DENO_DIR/remote/https/deno.land/ 90 | let Some(cache_parts) = base_url_to_filename_parts(url, "_PORT") else { 91 | return Err(std::io::Error::new( 92 | ErrorKind::InvalidInput, 93 | format!("Can't convert url (\"{}\") to filename.", url), 94 | )); 95 | }; 96 | 97 | let rest_str = if let Some(query) = url.query() { 98 | let mut rest_str = 99 | String::with_capacity(url.path().len() + 1 + query.len()); 100 | rest_str.push_str(url.path()); 101 | rest_str.push('?'); 102 | rest_str.push_str(query); 103 | Cow::Owned(rest_str) 104 | } else { 105 | Cow::Borrowed(url.path()) 106 | }; 107 | 108 | // NOTE: fragment is omitted on purpose - it's not taken into 109 | // account when caching - it denotes parts of webpage, which 110 | // in case of static resources doesn't make much sense 111 | let hashed_filename = checksum(rest_str.as_bytes()); 112 | let capacity = cache_parts.iter().map(|s| s.len() + 1).sum::() 113 | + 1 114 | + hashed_filename.len(); 115 | let mut cache_filename = PathBuf::with_capacity(capacity); 116 | cache_filename.extend(cache_parts.iter().map(|s| s.as_ref())); 117 | cache_filename.push(hashed_filename); 118 | debug_assert_eq!(cache_filename.capacity(), capacity); 119 | Ok(cache_filename) 120 | } 121 | 122 | #[derive(Debug, Error, JsError)] 123 | pub enum CacheReadFileError { 124 | #[class(inherit)] 125 | #[error(transparent)] 126 | Io(#[from] std::io::Error), 127 | #[class(inherit)] 128 | #[error(transparent)] 129 | ChecksumIntegrity(Box), 130 | } 131 | 132 | #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] 133 | pub struct SerializedCachedUrlMetadata { 134 | pub headers: HeadersMap, 135 | pub url: String, 136 | /// Number of seconds since the UNIX epoch. 137 | #[serde(default)] 138 | pub time: Option, 139 | } 140 | 141 | #[derive(Debug, Clone, PartialEq, Eq)] 142 | pub struct CacheEntry { 143 | pub metadata: SerializedCachedUrlMetadata, 144 | pub content: Cow<'static, [u8]>, 145 | } 146 | 147 | /// Computed cache key, which can help reduce the work of computing the cache key multiple times. 148 | pub struct HttpCacheItemKey<'a> { 149 | // The key is specific to the implementation of HttpCache, 150 | // so keep these private to the module. For example, the 151 | // fact that these may be stored in a file is an implementation 152 | // detail. 153 | #[cfg(debug_assertions)] 154 | pub(super) is_local_key: bool, 155 | pub(super) url: &'a Url, 156 | /// This will be set all the time for the global cache, but it 157 | /// won't ever be set for the local cache because that also needs 158 | /// header information to determine the final path. 159 | pub(super) file_path: Option, 160 | } 161 | 162 | #[allow(clippy::disallowed_types)] 163 | pub type HttpCacheRc = crate::sync::MaybeArc; 164 | 165 | pub trait HttpCache: MaybeSend + MaybeSync + std::fmt::Debug { 166 | /// A pre-computed key for looking up items in the cache. 167 | fn cache_item_key<'a>( 168 | &self, 169 | url: &'a Url, 170 | ) -> std::io::Result>; 171 | 172 | fn contains(&self, url: &Url) -> bool; 173 | fn set( 174 | &self, 175 | url: &Url, 176 | headers: HeadersMap, 177 | content: &[u8], 178 | ) -> std::io::Result<()>; 179 | fn get( 180 | &self, 181 | key: &HttpCacheItemKey, 182 | maybe_checksum: Option, 183 | ) -> Result, CacheReadFileError>; 184 | fn read_modified_time( 185 | &self, 186 | key: &HttpCacheItemKey, 187 | ) -> std::io::Result>; 188 | /// Reads the headers for the cache item. 189 | fn read_headers( 190 | &self, 191 | key: &HttpCacheItemKey, 192 | ) -> std::io::Result>; 193 | /// Reads the time the item was downloaded to the cache. 194 | fn read_download_time( 195 | &self, 196 | key: &HttpCacheItemKey, 197 | ) -> std::io::Result>; 198 | } 199 | 200 | #[derive(Debug, Clone)] 201 | pub enum GlobalOrLocalHttpCache { 202 | Global(GlobalHttpCacheRc), 203 | Local(LocalHttpCacheRc), 204 | } 205 | 206 | impl From> 207 | for GlobalOrLocalHttpCache 208 | { 209 | fn from(global: GlobalHttpCacheRc) -> Self { 210 | Self::Global(global) 211 | } 212 | } 213 | 214 | impl From> 215 | for GlobalOrLocalHttpCache 216 | { 217 | fn from(local: LocalHttpCacheRc) -> Self { 218 | Self::Local(local) 219 | } 220 | } 221 | 222 | impl HttpCache 223 | for GlobalOrLocalHttpCache 224 | { 225 | fn read_headers( 226 | &self, 227 | key: &HttpCacheItemKey, 228 | ) -> std::io::Result> { 229 | match self { 230 | GlobalOrLocalHttpCache::Global(global) => global.read_headers(key), 231 | GlobalOrLocalHttpCache::Local(local) => local.read_headers(key), 232 | } 233 | } 234 | 235 | fn cache_item_key<'a>( 236 | &self, 237 | url: &'a Url, 238 | ) -> std::io::Result> { 239 | match self { 240 | GlobalOrLocalHttpCache::Global(global) => global.cache_item_key(url), 241 | GlobalOrLocalHttpCache::Local(local) => local.cache_item_key(url), 242 | } 243 | } 244 | 245 | fn contains(&self, url: &Url) -> bool { 246 | match self { 247 | GlobalOrLocalHttpCache::Global(global) => global.contains(url), 248 | GlobalOrLocalHttpCache::Local(local) => local.contains(url), 249 | } 250 | } 251 | 252 | fn set( 253 | &self, 254 | url: &Url, 255 | headers: HeadersMap, 256 | content: &[u8], 257 | ) -> std::io::Result<()> { 258 | match self { 259 | GlobalOrLocalHttpCache::Global(global) => { 260 | global.set(url, headers, content) 261 | } 262 | GlobalOrLocalHttpCache::Local(local) => local.set(url, headers, content), 263 | } 264 | } 265 | 266 | fn get( 267 | &self, 268 | key: &HttpCacheItemKey, 269 | maybe_checksum: Option, 270 | ) -> Result, CacheReadFileError> { 271 | match self { 272 | GlobalOrLocalHttpCache::Global(global) => global.get(key, maybe_checksum), 273 | GlobalOrLocalHttpCache::Local(local) => local.get(key, maybe_checksum), 274 | } 275 | } 276 | 277 | fn read_modified_time( 278 | &self, 279 | key: &HttpCacheItemKey, 280 | ) -> std::io::Result> { 281 | match self { 282 | GlobalOrLocalHttpCache::Global(global) => global.read_modified_time(key), 283 | GlobalOrLocalHttpCache::Local(local) => local.read_modified_time(key), 284 | } 285 | } 286 | 287 | fn read_download_time( 288 | &self, 289 | key: &HttpCacheItemKey, 290 | ) -> std::io::Result> { 291 | match self { 292 | GlobalOrLocalHttpCache::Global(global) => global.read_download_time(key), 293 | GlobalOrLocalHttpCache::Local(local) => local.read_download_time(key), 294 | } 295 | } 296 | } 297 | 298 | #[cfg(test)] 299 | mod test { 300 | use super::*; 301 | 302 | #[test] 303 | fn deserialized_no_time() { 304 | let json = r#"{ 305 | "headers": { 306 | "content-type": "application/javascript" 307 | }, 308 | "url": "https://deno.land/std/http/file_server.ts" 309 | }"#; 310 | let data: SerializedCachedUrlMetadata = serde_json::from_str(json).unwrap(); 311 | assert_eq!( 312 | data, 313 | SerializedCachedUrlMetadata { 314 | headers: HeadersMap::from([( 315 | "content-type".to_string(), 316 | "application/javascript".to_string() 317 | )]), 318 | time: None, 319 | url: "https://deno.land/std/http/file_server.ts".to_string(), 320 | } 321 | ); 322 | } 323 | 324 | #[test] 325 | fn serialize_deserialize_time() { 326 | let json = r#"{ 327 | "headers": { 328 | "content-type": "application/javascript" 329 | }, 330 | "url": "https://deno.land/std/http/file_server.ts", 331 | "time": 123456789 332 | }"#; 333 | let data: SerializedCachedUrlMetadata = serde_json::from_str(json).unwrap(); 334 | let expected = SerializedCachedUrlMetadata { 335 | headers: HeadersMap::from([( 336 | "content-type".to_string(), 337 | "application/javascript".to_string(), 338 | )]), 339 | time: Some(123456789), 340 | url: "https://deno.land/std/http/file_server.ts".to_string(), 341 | }; 342 | assert_eq!(data, expected); 343 | } 344 | } 345 | -------------------------------------------------------------------------------- /rs_lib/src/common.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | use std::borrow::Cow; 4 | use std::collections::HashMap; 5 | 6 | use url::Url; 7 | 8 | // TODO(ry) HTTP headers are not unique key, value pairs. There may be more than 9 | // one header line with the same key. This should be changed to something like 10 | // Vec<(String, String)> 11 | pub type HeadersMap = HashMap; 12 | 13 | pub fn base_url_to_filename_parts<'a>( 14 | url: &'a Url, 15 | port_separator: &str, 16 | ) -> Option>> { 17 | let mut out = Vec::with_capacity(2); 18 | 19 | let scheme = url.scheme(); 20 | 21 | match scheme { 22 | "http" | "https" => { 23 | out.push(Cow::Borrowed(scheme)); 24 | 25 | let host = url.host_str().unwrap(); 26 | let host_port = match url.port() { 27 | // underscores are not allowed in domains, so adding one here is fine 28 | Some(port) => Cow::Owned(format!("{host}{port_separator}{port}")), 29 | None => Cow::Borrowed(host), 30 | }; 31 | out.push(host_port); 32 | } 33 | "data" | "blob" => { 34 | out.push(Cow::Borrowed(scheme)); 35 | } 36 | scheme => { 37 | log::debug!("Don't know how to create cache name for scheme: {}", scheme); 38 | return None; 39 | } 40 | }; 41 | 42 | Some(out) 43 | } 44 | 45 | pub fn checksum(v: &[u8]) -> String { 46 | use sha2::Digest; 47 | use sha2::Sha256; 48 | 49 | let mut hasher = Sha256::new(); 50 | hasher.update(v); 51 | format!("{:x}", hasher.finalize()) 52 | } 53 | 54 | #[cfg(test)] 55 | mod tests { 56 | use super::*; 57 | 58 | #[test] 59 | fn test_gen() { 60 | let actual = checksum(b"hello world"); 61 | assert_eq!( 62 | actual, 63 | "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" 64 | ); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /rs_lib/src/deno_dir.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | use std::path::PathBuf; 4 | 5 | use sys_traits::EnvCacheDir; 6 | use sys_traits::EnvCurrentDir; 7 | use sys_traits::EnvHomeDir; 8 | use sys_traits::EnvVar; 9 | use thiserror::Error; 10 | 11 | #[derive(Debug, Error)] 12 | pub enum DenoDirResolutionError { 13 | #[error("Could not resolve global Deno cache directory. Please make sure that either the DENO_DIR environment variable is set or the cache directory is available.")] 14 | NoCacheOrHomeDir, 15 | #[error("Could not resolve global Deno cache directory because the current working directory could not be resolved. Please set the DENO_DIR environment variable and ensure it is pointing at an absolute path.")] 16 | FailedCwd { 17 | #[source] 18 | source: std::io::Error, 19 | }, 20 | } 21 | 22 | #[sys_traits::auto_impl] 23 | pub trait ResolveDenoDirSys: 24 | EnvCacheDir + EnvHomeDir + EnvVar + EnvCurrentDir 25 | { 26 | } 27 | 28 | pub fn resolve_deno_dir( 29 | sys: &Sys, 30 | maybe_custom_root: Option, 31 | ) -> Result { 32 | let maybe_custom_root = 33 | maybe_custom_root.or_else(|| sys.env_var_path("DENO_DIR")); 34 | let root: PathBuf = if let Some(root) = maybe_custom_root { 35 | root 36 | } else if let Some(xdg_cache_dir) = sys.env_var_path("XDG_CACHE_HOME") { 37 | xdg_cache_dir.join("deno") 38 | } else if let Some(cache_dir) = sys.env_cache_dir() { 39 | // We use the OS cache dir because all files deno writes are cache files 40 | // Once that changes we need to start using different roots if DENO_DIR 41 | // is not set, and keep a single one if it is. 42 | cache_dir.join("deno") 43 | } else if let Some(home_dir) = sys.env_home_dir() { 44 | // fallback path 45 | home_dir.join(".deno") 46 | } else { 47 | return Err(DenoDirResolutionError::NoCacheOrHomeDir); 48 | }; 49 | let root = if root.is_absolute() { 50 | root 51 | } else { 52 | sys 53 | .env_current_dir() 54 | .map_err(|source| DenoDirResolutionError::FailedCwd { source })? 55 | .join(root) 56 | }; 57 | Ok(root) 58 | } 59 | -------------------------------------------------------------------------------- /rs_lib/src/file_fetcher/auth_tokens.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | use std::borrow::Cow; 4 | use std::fmt; 5 | use std::net::IpAddr; 6 | use std::net::Ipv4Addr; 7 | use std::net::Ipv6Addr; 8 | use std::net::SocketAddr; 9 | use std::str::FromStr; 10 | 11 | use base64::prelude::BASE64_STANDARD; 12 | use base64::Engine; 13 | use log::debug; 14 | use log::error; 15 | use sys_traits::EnvVar; 16 | use url::Url; 17 | 18 | #[derive(Debug, Clone, PartialEq, Eq)] 19 | pub enum AuthTokenData { 20 | Bearer(String), 21 | Basic { username: String, password: String }, 22 | } 23 | 24 | #[derive(Debug, Clone, PartialEq, Eq)] 25 | pub struct AuthToken { 26 | host: AuthDomain, 27 | token: AuthTokenData, 28 | } 29 | 30 | impl fmt::Display for AuthToken { 31 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 32 | match &self.token { 33 | AuthTokenData::Bearer(token) => write!(f, "Bearer {token}"), 34 | AuthTokenData::Basic { username, password } => { 35 | let credentials = format!("{username}:{password}"); 36 | write!(f, "Basic {}", BASE64_STANDARD.encode(credentials)) 37 | } 38 | } 39 | } 40 | } 41 | 42 | /// A structure which contains bearer tokens that can be used when sending 43 | /// requests to websites, intended to authorize access to private resources 44 | /// such as remote modules. 45 | #[derive(Debug, Clone)] 46 | pub struct AuthTokens(Vec); 47 | 48 | /// An authorization domain, either an exact or suffix match. 49 | #[derive(Debug, Clone, PartialEq, Eq)] 50 | pub enum AuthDomain { 51 | Ip(IpAddr), 52 | IpPort(SocketAddr), 53 | /// Suffix match, no dot. May include a port. 54 | Suffix(Cow<'static, str>), 55 | } 56 | 57 | impl From for AuthDomain { 58 | fn from(value: T) -> Self { 59 | let s = value.to_string().to_lowercase(); 60 | if let Ok(ip) = SocketAddr::from_str(&s) { 61 | return AuthDomain::IpPort(ip); 62 | }; 63 | if s.starts_with('[') && s.ends_with(']') { 64 | if let Ok(ip) = Ipv6Addr::from_str(&s[1..s.len() - 1]) { 65 | return AuthDomain::Ip(ip.into()); 66 | } 67 | } else if let Ok(ip) = Ipv4Addr::from_str(&s) { 68 | return AuthDomain::Ip(ip.into()); 69 | } 70 | if let Some(s) = s.strip_prefix('.') { 71 | AuthDomain::Suffix(Cow::Owned(s.to_owned())) 72 | } else { 73 | AuthDomain::Suffix(Cow::Owned(s)) 74 | } 75 | } 76 | } 77 | 78 | impl AuthDomain { 79 | pub fn matches(&self, specifier: &Url) -> bool { 80 | let Some(host) = specifier.host_str() else { 81 | return false; 82 | }; 83 | match *self { 84 | Self::Ip(ip) => { 85 | let AuthDomain::Ip(parsed) = AuthDomain::from(host) else { 86 | return false; 87 | }; 88 | ip == parsed && specifier.port().is_none() 89 | } 90 | Self::IpPort(ip) => { 91 | let AuthDomain::Ip(parsed) = AuthDomain::from(host) else { 92 | return false; 93 | }; 94 | ip.ip() == parsed && specifier.port() == Some(ip.port()) 95 | } 96 | Self::Suffix(ref suffix) => { 97 | let hostname = if let Some(port) = specifier.port() { 98 | Cow::Owned(format!("{}:{}", host, port)) 99 | } else { 100 | Cow::Borrowed(host) 101 | }; 102 | 103 | if suffix.len() == hostname.len() { 104 | return suffix == &hostname; 105 | } 106 | 107 | // If it's a suffix match, ensure a dot 108 | if hostname.ends_with(suffix.as_ref()) 109 | && hostname.ends_with(&format!(".{suffix}")) 110 | { 111 | return true; 112 | } 113 | 114 | false 115 | } 116 | } 117 | } 118 | } 119 | 120 | impl AuthTokens { 121 | pub fn new_from_sys(sys: &TSys) -> Self { 122 | Self::new(sys.env_var("DENO_AUTH_TOKENS").ok()) 123 | } 124 | 125 | /// Create a new set of tokens based on the provided string. It is intended 126 | /// that the string be the value of an environment variable and the string is 127 | /// parsed for token values. The string is expected to be a semi-colon 128 | /// separated string, where each value is `{token}@{hostname}`. 129 | pub fn new(maybe_tokens_str: Option) -> Self { 130 | let mut tokens = Vec::new(); 131 | if let Some(tokens_str) = maybe_tokens_str { 132 | for token_str in tokens_str.trim().split(';') { 133 | if token_str.contains('@') { 134 | let mut iter = token_str.rsplitn(2, '@'); 135 | let host = AuthDomain::from(iter.next().unwrap()); 136 | let token = iter.next().unwrap(); 137 | if token.contains(':') { 138 | let mut iter = token.rsplitn(2, ':'); 139 | let password = iter.next().unwrap().to_owned(); 140 | let username = iter.next().unwrap().to_owned(); 141 | tokens.push(AuthToken { 142 | host, 143 | token: AuthTokenData::Basic { username, password }, 144 | }); 145 | } else { 146 | tokens.push(AuthToken { 147 | host, 148 | token: AuthTokenData::Bearer(token.to_string()), 149 | }); 150 | } 151 | } else { 152 | error!("Badly formed auth token discarded."); 153 | } 154 | } 155 | debug!("Parsed {} auth token(s).", tokens.len()); 156 | } 157 | 158 | Self(tokens) 159 | } 160 | 161 | /// Attempt to match the provided specifier to the tokens in the set. The 162 | /// matching occurs from the right of the hostname plus port, irrespective of 163 | /// scheme. For example `https://www.deno.land:8080/` would match a token 164 | /// with a host value of `deno.land:8080` but not match `www.deno.land`. The 165 | /// matching is case insensitive. 166 | pub fn get(&self, specifier: &Url) -> Option<&AuthToken> { 167 | self.0.iter().find(|t| t.host.matches(specifier)) 168 | } 169 | } 170 | 171 | #[cfg(test)] 172 | mod tests { 173 | use super::*; 174 | 175 | #[test] 176 | fn test_auth_token() { 177 | let auth_tokens = AuthTokens::new(Some("abc123@deno.land".to_string())); 178 | let fixture = Url::parse("https://deno.land/x/mod.ts").unwrap(); 179 | assert_eq!( 180 | auth_tokens.get(&fixture).unwrap().to_string(), 181 | "Bearer abc123" 182 | ); 183 | let fixture = Url::parse("https://www.deno.land/x/mod.ts").unwrap(); 184 | assert_eq!( 185 | auth_tokens.get(&fixture).unwrap().to_string(), 186 | "Bearer abc123".to_string() 187 | ); 188 | let fixture = Url::parse("http://127.0.0.1:8080/x/mod.ts").unwrap(); 189 | assert_eq!(auth_tokens.get(&fixture), None); 190 | let fixture = Url::parse("https://deno.land.example.com/x/mod.ts").unwrap(); 191 | assert_eq!(auth_tokens.get(&fixture), None); 192 | let fixture = Url::parse("https://deno.land:8080/x/mod.ts").unwrap(); 193 | assert_eq!(auth_tokens.get(&fixture), None); 194 | } 195 | 196 | #[test] 197 | fn test_auth_tokens_multiple() { 198 | let auth_tokens = 199 | AuthTokens::new(Some("abc123@deno.land;def456@example.com".to_string())); 200 | let fixture = Url::parse("https://deno.land/x/mod.ts").unwrap(); 201 | assert_eq!( 202 | auth_tokens.get(&fixture).unwrap().to_string(), 203 | "Bearer abc123".to_string() 204 | ); 205 | let fixture = Url::parse("http://example.com/a/file.ts").unwrap(); 206 | assert_eq!( 207 | auth_tokens.get(&fixture).unwrap().to_string(), 208 | "Bearer def456".to_string() 209 | ); 210 | } 211 | 212 | #[test] 213 | fn test_auth_tokens_space() { 214 | let auth_tokens = AuthTokens::new(Some( 215 | " abc123@deno.land;def456@example.com\t".to_string(), 216 | )); 217 | let fixture = Url::parse("https://deno.land/x/mod.ts").unwrap(); 218 | assert_eq!( 219 | auth_tokens.get(&fixture).unwrap().to_string(), 220 | "Bearer abc123".to_string() 221 | ); 222 | let fixture = Url::parse("http://example.com/a/file.ts").unwrap(); 223 | assert_eq!( 224 | auth_tokens.get(&fixture).unwrap().to_string(), 225 | "Bearer def456".to_string() 226 | ); 227 | } 228 | 229 | #[test] 230 | fn test_auth_tokens_newline() { 231 | let auth_tokens = AuthTokens::new(Some( 232 | "\nabc123@deno.land;def456@example.com\n".to_string(), 233 | )); 234 | let fixture = Url::parse("https://deno.land/x/mod.ts").unwrap(); 235 | assert_eq!( 236 | auth_tokens.get(&fixture).unwrap().to_string(), 237 | "Bearer abc123".to_string() 238 | ); 239 | let fixture = Url::parse("http://example.com/a/file.ts").unwrap(); 240 | assert_eq!( 241 | auth_tokens.get(&fixture).unwrap().to_string(), 242 | "Bearer def456".to_string() 243 | ); 244 | } 245 | 246 | #[test] 247 | fn test_auth_tokens_port() { 248 | let auth_tokens = 249 | AuthTokens::new(Some("abc123@deno.land:8080".to_string())); 250 | let fixture = Url::parse("https://deno.land/x/mod.ts").unwrap(); 251 | assert_eq!(auth_tokens.get(&fixture), None); 252 | let fixture = Url::parse("http://deno.land:8080/x/mod.ts").unwrap(); 253 | assert_eq!( 254 | auth_tokens.get(&fixture).unwrap().to_string(), 255 | "Bearer abc123".to_string() 256 | ); 257 | } 258 | 259 | #[test] 260 | fn test_auth_tokens_contain_at() { 261 | let auth_tokens = AuthTokens::new(Some("abc@123@deno.land".to_string())); 262 | let fixture = Url::parse("https://deno.land/x/mod.ts").unwrap(); 263 | assert_eq!( 264 | auth_tokens.get(&fixture).unwrap().to_string(), 265 | "Bearer abc@123".to_string() 266 | ); 267 | } 268 | 269 | #[test] 270 | fn test_auth_token_basic() { 271 | let auth_tokens = AuthTokens::new(Some("abc:123@deno.land".to_string())); 272 | let fixture = Url::parse("https://deno.land/x/mod.ts").unwrap(); 273 | assert_eq!( 274 | auth_tokens.get(&fixture).unwrap().to_string(), 275 | "Basic YWJjOjEyMw==" 276 | ); 277 | let fixture = Url::parse("https://www.deno.land/x/mod.ts").unwrap(); 278 | assert_eq!( 279 | auth_tokens.get(&fixture).unwrap().to_string(), 280 | "Basic YWJjOjEyMw==".to_string() 281 | ); 282 | let fixture = Url::parse("http://127.0.0.1:8080/x/mod.ts").unwrap(); 283 | assert_eq!(auth_tokens.get(&fixture), None); 284 | let fixture = Url::parse("https://deno.land.example.com/x/mod.ts").unwrap(); 285 | assert_eq!(auth_tokens.get(&fixture), None); 286 | let fixture = Url::parse("https://deno.land:8080/x/mod.ts").unwrap(); 287 | assert_eq!(auth_tokens.get(&fixture), None); 288 | } 289 | 290 | #[test] 291 | fn test_parse_ip() { 292 | let ip = AuthDomain::from("[2001:db8:a::123]"); 293 | assert_eq!("Ip(2001:db8:a::123)", format!("{ip:?}")); 294 | let ip = AuthDomain::from("[2001:db8:a::123]:8080"); 295 | assert_eq!("IpPort([2001:db8:a::123]:8080)", format!("{ip:?}")); 296 | let ip = AuthDomain::from("1.1.1.1"); 297 | assert_eq!("Ip(1.1.1.1)", format!("{ip:?}")); 298 | } 299 | 300 | #[test] 301 | fn test_case_insensitive() { 302 | let domain = AuthDomain::from("EXAMPLE.com"); 303 | assert!(domain.matches(&Url::parse("http://example.com").unwrap())); 304 | assert!(domain.matches(&Url::parse("http://example.COM").unwrap())); 305 | } 306 | 307 | #[test] 308 | fn test_matches() { 309 | let candidates = [ 310 | "example.com", 311 | "www.example.com", 312 | "1.1.1.1", 313 | "[2001:db8:a::123]", 314 | // These will never match 315 | "example.com.evil.com", 316 | "1.1.1.1.evil.com", 317 | "notexample.com", 318 | "www.notexample.com", 319 | ]; 320 | let domains = [ 321 | ("example.com", vec!["example.com", "www.example.com"]), 322 | (".example.com", vec!["example.com", "www.example.com"]), 323 | ("www.example.com", vec!["www.example.com"]), 324 | ("1.1.1.1", vec!["1.1.1.1"]), 325 | ("[2001:db8:a::123]", vec!["[2001:db8:a::123]"]), 326 | ]; 327 | let url = |c: &str| Url::parse(&format!("http://{c}")).unwrap(); 328 | let url_port = |c: &str| Url::parse(&format!("http://{c}:8080")).unwrap(); 329 | 330 | // Generate each candidate with and without a port 331 | let candidates = candidates 332 | .into_iter() 333 | .flat_map(|c| [url(c), url_port(c)]) 334 | .collect::>(); 335 | 336 | for (domain, expected_domain) in domains { 337 | // Test without a port -- all candidates return without a port 338 | let auth_domain = AuthDomain::from(domain); 339 | let actual = candidates 340 | .iter() 341 | .filter(|c| auth_domain.matches(c)) 342 | .cloned() 343 | .collect::>(); 344 | let expected = expected_domain.iter().map(|u| url(u)).collect::>(); 345 | assert_eq!(actual, expected); 346 | 347 | // Test with a port, all candidates return with a port 348 | let auth_domain = AuthDomain::from(&format!("{domain}:8080")); 349 | let actual = candidates 350 | .iter() 351 | .filter(|c| auth_domain.matches(c)) 352 | .cloned() 353 | .collect::>(); 354 | let expected = expected_domain 355 | .iter() 356 | .map(|u| url_port(u)) 357 | .collect::>(); 358 | assert_eq!(actual, expected); 359 | } 360 | } 361 | } 362 | -------------------------------------------------------------------------------- /rs_lib/src/file_fetcher/http_util.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | use std::time::Duration; 4 | use std::time::SystemTime; 5 | 6 | use cache_control::Cachability; 7 | use cache_control::CacheControl; 8 | use chrono::DateTime; 9 | 10 | use crate::common::HeadersMap; 11 | 12 | /// A structure used to determine if a entity in the http cache can be used. 13 | /// 14 | /// This is heavily influenced by 15 | /// which is BSD 16 | /// 2-Clause Licensed and copyright Kornel Lesiński 17 | pub struct CacheSemantics { 18 | cache_control: CacheControl, 19 | cached: SystemTime, 20 | headers: HeadersMap, 21 | now: SystemTime, 22 | } 23 | 24 | impl CacheSemantics { 25 | pub fn new(headers: HeadersMap, cached: SystemTime, now: SystemTime) -> Self { 26 | let cache_control = headers 27 | .get("cache-control") 28 | .map(|v| CacheControl::from_value(v).unwrap_or_default()) 29 | .unwrap_or_default(); 30 | Self { 31 | cache_control, 32 | cached, 33 | headers, 34 | now, 35 | } 36 | } 37 | 38 | fn age(&self) -> Duration { 39 | let mut age = self.age_header_value(); 40 | 41 | if let Ok(resident_time) = self.now.duration_since(self.cached) { 42 | age += resident_time; 43 | } 44 | 45 | age 46 | } 47 | 48 | fn age_header_value(&self) -> Duration { 49 | Duration::from_secs( 50 | self 51 | .headers 52 | .get("age") 53 | .and_then(|v| v.parse().ok()) 54 | .unwrap_or(0), 55 | ) 56 | } 57 | 58 | fn is_stale(&self) -> bool { 59 | self.max_age() <= self.age() 60 | } 61 | 62 | fn max_age(&self) -> Duration { 63 | if self.cache_control.cachability == Some(Cachability::NoCache) { 64 | return Duration::from_secs(0); 65 | } 66 | 67 | if self.headers.get("vary").map(|s| s.trim()) == Some("*") { 68 | return Duration::from_secs(0); 69 | } 70 | 71 | if let Some(max_age) = self.cache_control.max_age { 72 | return max_age; 73 | } 74 | 75 | let default_min_ttl = Duration::from_secs(0); 76 | 77 | let server_date = self.raw_server_date(); 78 | if let Some(expires) = self.headers.get("expires") { 79 | return match DateTime::parse_from_rfc2822(expires) { 80 | Err(_) => Duration::from_secs(0), 81 | Ok(expires) => { 82 | let expires = SystemTime::UNIX_EPOCH 83 | + Duration::from_secs(expires.timestamp().max(0) as _); 84 | return default_min_ttl 85 | .max(expires.duration_since(server_date).unwrap_or_default()); 86 | } 87 | }; 88 | } 89 | 90 | if let Some(last_modified) = self.headers.get("last-modified") { 91 | if let Ok(last_modified) = DateTime::parse_from_rfc2822(last_modified) { 92 | let last_modified = SystemTime::UNIX_EPOCH 93 | + Duration::from_secs(last_modified.timestamp().max(0) as _); 94 | if let Ok(diff) = server_date.duration_since(last_modified) { 95 | let secs_left = diff.as_secs() as f64 * 0.1; 96 | return default_min_ttl.max(Duration::from_secs(secs_left as _)); 97 | } 98 | } 99 | } 100 | 101 | default_min_ttl 102 | } 103 | 104 | fn raw_server_date(&self) -> SystemTime { 105 | self 106 | .headers 107 | .get("date") 108 | .and_then(|d| DateTime::parse_from_rfc2822(d).ok()) 109 | .and_then(|d| { 110 | SystemTime::UNIX_EPOCH 111 | .checked_add(Duration::from_secs(d.timestamp() as _)) 112 | }) 113 | .unwrap_or(self.cached) 114 | } 115 | 116 | /// Returns true if the cached value is "fresh" respecting cached headers, 117 | /// otherwise returns false. 118 | pub fn should_use(&self) -> bool { 119 | if self.cache_control.cachability == Some(Cachability::NoCache) { 120 | return false; 121 | } 122 | 123 | if let Some(max_age) = self.cache_control.max_age { 124 | if self.age() > max_age { 125 | return false; 126 | } 127 | } 128 | 129 | if let Some(min_fresh) = self.cache_control.min_fresh { 130 | if self.time_to_live() < min_fresh { 131 | return false; 132 | } 133 | } 134 | 135 | if self.is_stale() { 136 | let has_max_stale = self.cache_control.max_stale.is_some(); 137 | let allows_stale = has_max_stale 138 | && self 139 | .cache_control 140 | .max_stale 141 | .map(|val| val > self.age() - self.max_age()) 142 | .unwrap_or(true); 143 | if !allows_stale { 144 | return false; 145 | } 146 | } 147 | 148 | true 149 | } 150 | 151 | fn time_to_live(&self) -> Duration { 152 | self.max_age().checked_sub(self.age()).unwrap_or_default() 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /rs_lib/src/file_fetcher/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | use std::borrow::Cow; 4 | use std::collections::HashMap; 5 | use std::io::Read; 6 | use std::path::Path; 7 | use std::path::PathBuf; 8 | use std::time::SystemTime; 9 | 10 | use boxed_error::Boxed; 11 | use data_url::DataUrl; 12 | use deno_error::JsError; 13 | use deno_media_type::MediaType; 14 | use deno_path_util::url_to_file_path; 15 | use http::header; 16 | use http::header::ACCEPT; 17 | use http::header::AUTHORIZATION; 18 | use http::header::IF_NONE_MATCH; 19 | use http::header::LOCATION; 20 | use log::debug; 21 | use sys_traits::FsFileMetadata; 22 | use sys_traits::FsMetadataValue; 23 | use sys_traits::FsOpen; 24 | use sys_traits::OpenOptions; 25 | use sys_traits::SystemTimeNow; 26 | use thiserror::Error; 27 | use url::Url; 28 | 29 | use self::http_util::CacheSemantics; 30 | use crate::cache::HttpCacheRc; 31 | use crate::common::HeadersMap; 32 | use crate::sync::MaybeSend; 33 | use crate::sync::MaybeSync; 34 | use crate::CacheEntry; 35 | use crate::CacheReadFileError; 36 | use crate::Checksum; 37 | use crate::ChecksumIntegrityError; 38 | 39 | mod auth_tokens; 40 | mod http_util; 41 | 42 | pub use auth_tokens::AuthDomain; 43 | pub use auth_tokens::AuthToken; 44 | pub use auth_tokens::AuthTokenData; 45 | pub use auth_tokens::AuthTokens; 46 | pub use http::HeaderMap; 47 | pub use http::HeaderName; 48 | pub use http::HeaderValue; 49 | pub use http::StatusCode; 50 | 51 | /// Indicates how cached source files should be handled. 52 | #[derive(Debug, Clone, Eq, PartialEq)] 53 | pub enum CacheSetting { 54 | /// Only the cached files should be used. Any files not in the cache will 55 | /// error. This is the equivalent of `--cached-only` in the CLI. 56 | Only, 57 | /// No cached source files should be used, and all files should be reloaded. 58 | /// This is the equivalent of `--reload` in the CLI. 59 | ReloadAll, 60 | /// Only some cached resources should be used. This is the equivalent of 61 | /// `--reload=jsr:@std/http/file-server` or 62 | /// `--reload=jsr:@std/http/file-server,jsr:@std/assert/assert-equals`. 63 | ReloadSome(Vec), 64 | /// The usability of a cached value is determined by analyzing the cached 65 | /// headers and other metadata associated with a cached response, reloading 66 | /// any cached "non-fresh" cached responses. 67 | RespectHeaders, 68 | /// The cached source files should be used for local modules. This is the 69 | /// default behavior of the CLI. 70 | Use, 71 | } 72 | 73 | #[derive(Debug, Clone, Eq, PartialEq)] 74 | pub enum FileOrRedirect { 75 | File(File), 76 | Redirect(Url), 77 | } 78 | 79 | impl FileOrRedirect { 80 | fn from_deno_cache_entry( 81 | url: &Url, 82 | cache_entry: CacheEntry, 83 | ) -> Result { 84 | if let Some(redirect_to) = cache_entry.metadata.headers.get("location") { 85 | let redirect = 86 | url 87 | .join(redirect_to) 88 | .map_err(|source| RedirectResolutionError { 89 | url: url.clone(), 90 | location: redirect_to.clone(), 91 | source, 92 | })?; 93 | Ok(FileOrRedirect::Redirect(redirect)) 94 | } else { 95 | Ok(FileOrRedirect::File(File { 96 | url: url.clone(), 97 | mtime: None, 98 | maybe_headers: Some(cache_entry.metadata.headers), 99 | #[allow(clippy::disallowed_types)] // ok for source 100 | source: std::sync::Arc::from(cache_entry.content), 101 | })) 102 | } 103 | } 104 | } 105 | 106 | #[allow(clippy::disallowed_types)] // ok for source 107 | type FileSource = std::sync::Arc<[u8]>; 108 | 109 | /// A structure representing a source file. 110 | #[derive(Debug, Clone, Eq, PartialEq)] 111 | pub struct File { 112 | /// The _final_ specifier for the file. The requested specifier and the final 113 | /// specifier maybe different for remote files that have been redirected. 114 | pub url: Url, 115 | pub mtime: Option, 116 | pub maybe_headers: Option>, 117 | /// The source of the file. 118 | pub source: FileSource, 119 | } 120 | 121 | impl File { 122 | pub fn resolve_media_type_and_charset(&self) -> (MediaType, Option<&str>) { 123 | deno_media_type::resolve_media_type_and_charset_from_content_type( 124 | &self.url, 125 | self 126 | .maybe_headers 127 | .as_ref() 128 | .and_then(|h| h.get("content-type")) 129 | .map(|v| v.as_str()), 130 | ) 131 | } 132 | } 133 | 134 | #[allow(clippy::disallowed_types)] 135 | pub type MemoryFilesRc = crate::sync::MaybeArc; 136 | 137 | pub trait MemoryFiles: std::fmt::Debug + MaybeSend + MaybeSync { 138 | fn get(&self, url: &Url) -> Option; 139 | } 140 | 141 | /// Implementation of `MemoryFiles` that always returns `None`. 142 | #[derive(Debug, Clone, Default)] 143 | pub struct NullMemoryFiles; 144 | 145 | impl MemoryFiles for NullMemoryFiles { 146 | fn get(&self, _url: &Url) -> Option { 147 | None 148 | } 149 | } 150 | 151 | #[derive(Debug, PartialEq, Eq)] 152 | pub enum SendResponse { 153 | NotModified, 154 | Redirect(HeaderMap), 155 | Success(HeaderMap, Vec), 156 | } 157 | 158 | #[derive(Debug)] 159 | pub enum SendError { 160 | Failed(Box), 161 | NotFound, 162 | StatusCode(http::StatusCode), 163 | } 164 | 165 | #[derive(Debug, Error, JsError)] 166 | #[class(inherit)] 167 | #[error("Failed resolving redirect from '{url}' to '{location}'.")] 168 | pub struct RedirectResolutionError { 169 | pub url: Url, 170 | pub location: String, 171 | #[source] 172 | #[inherit] 173 | pub source: url::ParseError, 174 | } 175 | 176 | #[derive(Debug, Error, JsError)] 177 | #[class(inherit)] 178 | #[error("Unable to decode data url.")] 179 | pub struct DataUrlDecodeError { 180 | #[source] 181 | source: DataUrlDecodeSourceError, 182 | } 183 | 184 | #[derive(Debug, Error, JsError)] 185 | #[class(uri)] 186 | pub enum DataUrlDecodeSourceError { 187 | #[error(transparent)] 188 | DataUrl(data_url::DataUrlError), 189 | #[error(transparent)] 190 | InvalidBase64(data_url::forgiving_base64::InvalidBase64), 191 | } 192 | 193 | #[derive(Debug, Error, JsError)] 194 | #[class(inherit)] 195 | #[error("Failed reading cache entry for '{url}'.")] 196 | pub struct CacheReadError { 197 | pub url: Url, 198 | #[source] 199 | #[inherit] 200 | pub source: std::io::Error, 201 | } 202 | 203 | #[derive(Debug, Error, JsError)] 204 | #[class(generic)] 205 | #[error("Failed reading location header for '{}'{}", .request_url, .maybe_location.as_ref().map(|location| format!(" to '{}'", location)).unwrap_or_default())] 206 | pub struct RedirectHeaderParseError { 207 | pub request_url: Url, 208 | pub maybe_location: Option, 209 | #[source] 210 | pub maybe_source: Option>, 211 | } 212 | 213 | #[derive(Debug, Error, JsError)] 214 | #[class(inherit)] 215 | #[error("Import '{url}' failed.")] 216 | pub struct FailedReadingLocalFileError { 217 | pub url: Url, 218 | #[source] 219 | #[inherit] 220 | pub source: std::io::Error, 221 | } 222 | 223 | #[derive(Debug, Error, JsError)] 224 | #[class("Http")] 225 | #[error("Fetch '{0}' failed, too many redirects.")] 226 | pub struct TooManyRedirectsError(pub Url); 227 | 228 | // this message list additional `npm` and `jsr` schemes, but they should actually be handled 229 | // before `file_fetcher.rs` APIs are even hit. 230 | #[derive(Debug, Error, JsError)] 231 | #[class(type)] 232 | #[error("Unsupported scheme \"{scheme}\" for module \"{url}\". Supported schemes:\n - \"blob\"\n - \"data\"\n - \"file\"\n - \"http\"\n - \"https\"\n - \"jsr\"\n - \"npm\"")] 233 | pub struct UnsupportedSchemeError { 234 | pub scheme: String, 235 | pub url: Url, 236 | } 237 | 238 | /// Gets if the provided scheme was valid. 239 | pub fn is_valid_scheme(scheme: &str) -> bool { 240 | matches!( 241 | scheme, 242 | "blob" | "data" | "file" | "http" | "https" | "jsr" | "npm" 243 | ) 244 | } 245 | 246 | #[derive(Debug, Boxed, JsError)] 247 | pub struct FetchNoFollowError(pub Box); 248 | 249 | #[derive(Debug, Error, JsError)] 250 | pub enum FetchNoFollowErrorKind { 251 | #[class(inherit)] 252 | #[error(transparent)] 253 | UrlToFilePath(#[from] deno_path_util::UrlToFilePathError), 254 | #[class("NotFound")] 255 | #[error("Import '{0}' failed, not found.")] 256 | NotFound(Url), 257 | #[class(generic)] 258 | #[error("Import '{url}' failed.")] 259 | ReadingBlobUrl { 260 | url: Url, 261 | #[source] 262 | source: std::io::Error, 263 | }, 264 | #[class(inherit)] 265 | #[error(transparent)] 266 | ReadingFile(#[from] FailedReadingLocalFileError), 267 | #[class(generic)] 268 | #[error("Import '{url}' failed.")] 269 | FetchingRemote { 270 | url: Url, 271 | #[source] 272 | source: Box, 273 | }, 274 | #[class(generic)] 275 | #[error("Import '{url}' failed: {status_code}")] 276 | ClientError { 277 | url: Url, 278 | status_code: http::StatusCode, 279 | }, 280 | #[class("NoRemote")] 281 | #[error( 282 | "A remote specifier was requested: \"{0}\", but --no-remote is specified." 283 | )] 284 | NoRemote(Url), 285 | #[class(inherit)] 286 | #[error(transparent)] 287 | DataUrlDecode(DataUrlDecodeError), 288 | #[class(inherit)] 289 | #[error(transparent)] 290 | RedirectResolution(#[from] RedirectResolutionError), 291 | #[class(inherit)] 292 | #[error(transparent)] 293 | ChecksumIntegrity(#[from] ChecksumIntegrityError), 294 | #[class(inherit)] 295 | #[error(transparent)] 296 | CacheRead(#[from] CacheReadError), 297 | #[class(generic)] 298 | #[error("Failed caching '{url}'.")] 299 | CacheSave { 300 | url: Url, 301 | #[source] 302 | source: std::io::Error, 303 | }, 304 | // this message list additional `npm` and `jsr` schemes, but they should actually be handled 305 | // before `file_fetcher.rs` APIs are even hit. 306 | #[class(inherit)] 307 | #[error(transparent)] 308 | UnsupportedScheme(#[from] UnsupportedSchemeError), 309 | #[class(type)] 310 | #[error(transparent)] 311 | RedirectHeaderParse(#[from] RedirectHeaderParseError), 312 | #[class("NotCached")] 313 | #[error( 314 | "Specifier not found in cache: \"{url}\", --cached-only is specified." 315 | )] 316 | NotCached { url: Url }, 317 | #[class(type)] 318 | #[error("Failed setting header '{name}'.")] 319 | InvalidHeader { 320 | name: &'static str, 321 | #[source] 322 | source: header::InvalidHeaderValue, 323 | }, 324 | } 325 | 326 | #[derive(Debug, Boxed, JsError)] 327 | pub struct FetchCachedError(pub Box); 328 | 329 | #[derive(Debug, Error, JsError)] 330 | pub enum FetchCachedErrorKind { 331 | #[class(inherit)] 332 | #[error(transparent)] 333 | TooManyRedirects(TooManyRedirectsError), 334 | #[class(inherit)] 335 | #[error(transparent)] 336 | ChecksumIntegrity(#[from] ChecksumIntegrityError), 337 | #[class(inherit)] 338 | #[error(transparent)] 339 | CacheRead(#[from] CacheReadError), 340 | #[class(inherit)] 341 | #[error(transparent)] 342 | RedirectResolution(#[from] RedirectResolutionError), 343 | } 344 | 345 | #[derive(Debug, Boxed, JsError)] 346 | pub struct FetchLocalError(pub Box); 347 | 348 | #[derive(Debug, Error, JsError)] 349 | pub enum FetchLocalErrorKind { 350 | #[class(inherit)] 351 | #[error(transparent)] 352 | UrlToFilePath(#[from] deno_path_util::UrlToFilePathError), 353 | #[class(inherit)] 354 | #[error(transparent)] 355 | ReadingFile(#[from] FailedReadingLocalFileError), 356 | } 357 | 358 | impl From for FetchNoFollowError { 359 | fn from(err: FetchLocalError) -> Self { 360 | match err.into_kind() { 361 | FetchLocalErrorKind::UrlToFilePath(err) => err.into(), 362 | FetchLocalErrorKind::ReadingFile(err) => err.into(), 363 | } 364 | } 365 | } 366 | 367 | #[derive(Debug, Boxed, JsError)] 368 | struct FetchCachedNoFollowError(pub Box); 369 | 370 | #[derive(Debug, Error, JsError)] 371 | enum FetchCachedNoFollowErrorKind { 372 | #[class(inherit)] 373 | #[error(transparent)] 374 | ChecksumIntegrity(ChecksumIntegrityError), 375 | #[class(inherit)] 376 | #[error(transparent)] 377 | CacheRead(#[from] CacheReadError), 378 | #[class(inherit)] 379 | #[error(transparent)] 380 | RedirectResolution(#[from] RedirectResolutionError), 381 | } 382 | 383 | impl From for FetchCachedError { 384 | fn from(err: FetchCachedNoFollowError) -> Self { 385 | match err.into_kind() { 386 | FetchCachedNoFollowErrorKind::ChecksumIntegrity(err) => err.into(), 387 | FetchCachedNoFollowErrorKind::CacheRead(err) => err.into(), 388 | FetchCachedNoFollowErrorKind::RedirectResolution(err) => err.into(), 389 | } 390 | } 391 | } 392 | 393 | impl From for FetchNoFollowError { 394 | fn from(err: FetchCachedNoFollowError) -> Self { 395 | match err.into_kind() { 396 | FetchCachedNoFollowErrorKind::ChecksumIntegrity(err) => err.into(), 397 | FetchCachedNoFollowErrorKind::CacheRead(err) => err.into(), 398 | FetchCachedNoFollowErrorKind::RedirectResolution(err) => err.into(), 399 | } 400 | } 401 | } 402 | 403 | #[async_trait::async_trait(?Send)] 404 | pub trait HttpClient: std::fmt::Debug + MaybeSend + MaybeSync { 405 | /// Send a request getting the response. 406 | /// 407 | /// The implementation MUST not follow redirects. Return `SendResponse::Redirect` 408 | /// in that case. 409 | /// 410 | /// The implementation may retry the request on failure. 411 | async fn send_no_follow( 412 | &self, 413 | url: &Url, 414 | headers: HeaderMap, 415 | ) -> Result; 416 | } 417 | 418 | #[derive(Debug, Clone)] 419 | pub struct BlobData { 420 | pub media_type: String, 421 | pub bytes: Vec, 422 | } 423 | 424 | #[derive(Debug, Clone, Default)] 425 | pub struct NullBlobStore; 426 | 427 | #[async_trait::async_trait(?Send)] 428 | impl BlobStore for NullBlobStore { 429 | async fn get(&self, _url: &Url) -> std::io::Result> { 430 | Ok(None) 431 | } 432 | } 433 | 434 | #[async_trait::async_trait(?Send)] 435 | pub trait BlobStore: std::fmt::Debug + MaybeSend + MaybeSync { 436 | async fn get(&self, url: &Url) -> std::io::Result>; 437 | } 438 | 439 | #[derive(Debug, Default)] 440 | pub struct FetchNoFollowOptions<'a> { 441 | pub local: FetchLocalOptions, 442 | pub maybe_auth: Option<(header::HeaderName, header::HeaderValue)>, 443 | pub maybe_checksum: Option>, 444 | pub maybe_accept: Option<&'a str>, 445 | pub maybe_cache_setting: Option<&'a CacheSetting>, 446 | } 447 | 448 | #[derive(Debug, Clone, Default)] 449 | pub struct FetchLocalOptions { 450 | pub include_mtime: bool, 451 | } 452 | 453 | #[derive(Debug)] 454 | pub struct FileFetcherOptions { 455 | pub allow_remote: bool, 456 | pub cache_setting: CacheSetting, 457 | pub auth_tokens: AuthTokens, 458 | } 459 | 460 | #[sys_traits::auto_impl] 461 | pub trait FileFetcherSys: FsOpen + SystemTimeNow {} 462 | 463 | /// A structure for resolving, fetching and caching source files. 464 | #[derive(Debug)] 465 | pub struct FileFetcher< 466 | TBlobStore: BlobStore, 467 | TSys: FileFetcherSys, 468 | THttpClient: HttpClient, 469 | > { 470 | blob_store: TBlobStore, 471 | sys: TSys, 472 | http_cache: HttpCacheRc, 473 | http_client: THttpClient, 474 | memory_files: MemoryFilesRc, 475 | allow_remote: bool, 476 | cache_setting: CacheSetting, 477 | auth_tokens: AuthTokens, 478 | } 479 | 480 | impl 481 | FileFetcher 482 | { 483 | pub fn new( 484 | blob_store: TBlobStore, 485 | sys: TSys, 486 | http_cache: HttpCacheRc, 487 | http_client: THttpClient, 488 | memory_files: MemoryFilesRc, 489 | options: FileFetcherOptions, 490 | ) -> Self { 491 | Self { 492 | blob_store, 493 | sys, 494 | http_cache, 495 | http_client, 496 | memory_files, 497 | allow_remote: options.allow_remote, 498 | auth_tokens: options.auth_tokens, 499 | cache_setting: options.cache_setting, 500 | } 501 | } 502 | 503 | pub fn cache_setting(&self) -> &CacheSetting { 504 | &self.cache_setting 505 | } 506 | 507 | /// Fetch cached remote file. 508 | pub fn fetch_cached( 509 | &self, 510 | url: &Url, 511 | redirect_limit: i64, 512 | ) -> Result, FetchCachedError> { 513 | if !matches!(url.scheme(), "http" | "https") { 514 | return Ok(None); 515 | } 516 | 517 | let mut url = Cow::Borrowed(url); 518 | for _ in 0..=redirect_limit { 519 | match self.fetch_cached_no_follow(&url, None)? { 520 | Some(FileOrRedirect::File(file)) => { 521 | return Ok(Some(file)); 522 | } 523 | Some(FileOrRedirect::Redirect(redirect_url)) => { 524 | url = Cow::Owned(redirect_url); 525 | } 526 | None => { 527 | return Ok(None); 528 | } 529 | } 530 | } 531 | Err( 532 | FetchCachedErrorKind::TooManyRedirects(TooManyRedirectsError( 533 | url.into_owned(), 534 | )) 535 | .into_box(), 536 | ) 537 | } 538 | 539 | /// Fetches without following redirects. 540 | /// 541 | /// You should verify permissions of the specifier before calling this function. 542 | pub async fn fetch_no_follow( 543 | &self, 544 | url: &Url, 545 | options: FetchNoFollowOptions<'_>, 546 | ) -> Result { 547 | // note: this debug output is used by the tests 548 | debug!("FileFetcher::fetch_no_follow - specifier: {}", url); 549 | let scheme = url.scheme(); 550 | if let Some(file) = self.memory_files.get(url) { 551 | Ok(FileOrRedirect::File(file)) 552 | } else if scheme == "file" { 553 | // we do not in memory cache files, as this would prevent files on the 554 | // disk changing effecting things like workers and dynamic imports. 555 | let maybe_file = self.fetch_local(url, &options.local)?; 556 | match maybe_file { 557 | Some(file) => Ok(FileOrRedirect::File(file)), 558 | None => Err(FetchNoFollowErrorKind::NotFound(url.clone()).into_box()), 559 | } 560 | } else if scheme == "data" { 561 | self 562 | .fetch_data_url(url) 563 | .map(FileOrRedirect::File) 564 | .map_err(|e| FetchNoFollowErrorKind::DataUrlDecode(e).into_box()) 565 | } else if scheme == "blob" { 566 | self.fetch_blob_url(url).await.map(FileOrRedirect::File) 567 | } else if scheme == "https" || scheme == "http" { 568 | if !self.allow_remote { 569 | Err(FetchNoFollowErrorKind::NoRemote(url.clone()).into_box()) 570 | } else { 571 | self 572 | .fetch_remote_no_follow( 573 | url, 574 | options.maybe_accept, 575 | options.maybe_cache_setting.unwrap_or(&self.cache_setting), 576 | options.maybe_checksum, 577 | options.maybe_auth, 578 | ) 579 | .await 580 | } 581 | } else { 582 | Err( 583 | FetchNoFollowErrorKind::UnsupportedScheme(UnsupportedSchemeError { 584 | scheme: scheme.to_string(), 585 | url: url.clone(), 586 | }) 587 | .into_box(), 588 | ) 589 | } 590 | } 591 | 592 | fn fetch_cached_no_follow( 593 | &self, 594 | url: &Url, 595 | maybe_checksum: Option>, 596 | ) -> Result, FetchCachedNoFollowError> { 597 | debug!("FileFetcher::fetch_cached_no_follow - specifier: {}", url); 598 | 599 | let cache_key = 600 | self 601 | .http_cache 602 | .cache_item_key(url) 603 | .map_err(|source| CacheReadError { 604 | url: url.clone(), 605 | source, 606 | })?; 607 | match self.http_cache.get(&cache_key, maybe_checksum) { 608 | Ok(Some(entry)) => { 609 | Ok(Some(FileOrRedirect::from_deno_cache_entry(url, entry)?)) 610 | } 611 | Ok(None) => Ok(None), 612 | Err(CacheReadFileError::Io(source)) => Err( 613 | FetchCachedNoFollowErrorKind::CacheRead(CacheReadError { 614 | url: url.clone(), 615 | source, 616 | }) 617 | .into_box(), 618 | ), 619 | Err(CacheReadFileError::ChecksumIntegrity(err)) => { 620 | Err(FetchCachedNoFollowErrorKind::ChecksumIntegrity(*err).into_box()) 621 | } 622 | } 623 | } 624 | 625 | /// Convert a data URL into a file, resulting in an error if the URL is 626 | /// invalid. 627 | fn fetch_data_url(&self, url: &Url) -> Result { 628 | fn parse( 629 | url: &Url, 630 | ) -> Result<(Vec, HashMap), DataUrlDecodeError> { 631 | let url = DataUrl::process(url.as_str()).map_err(|source| { 632 | DataUrlDecodeError { 633 | source: DataUrlDecodeSourceError::DataUrl(source), 634 | } 635 | })?; 636 | let (bytes, _) = 637 | url.decode_to_vec().map_err(|source| DataUrlDecodeError { 638 | source: DataUrlDecodeSourceError::InvalidBase64(source), 639 | })?; 640 | let headers = HashMap::from([( 641 | "content-type".to_string(), 642 | url.mime_type().to_string(), 643 | )]); 644 | Ok((bytes, headers)) 645 | } 646 | 647 | debug!("FileFetcher::fetch_data_url() - specifier: {}", url); 648 | let (bytes, headers) = parse(url)?; 649 | Ok(File { 650 | url: url.clone(), 651 | mtime: None, 652 | maybe_headers: Some(headers), 653 | #[allow(clippy::disallowed_types)] // ok for source 654 | source: std::sync::Arc::from(bytes), 655 | }) 656 | } 657 | 658 | /// Get a blob URL. 659 | async fn fetch_blob_url( 660 | &self, 661 | url: &Url, 662 | ) -> Result { 663 | debug!("FileFetcher::fetch_blob_url() - specifier: {}", url); 664 | let blob = self 665 | .blob_store 666 | .get(url) 667 | .await 668 | .map_err(|err| FetchNoFollowErrorKind::ReadingBlobUrl { 669 | url: url.clone(), 670 | source: err, 671 | })? 672 | .ok_or_else(|| FetchNoFollowErrorKind::NotFound(url.clone()))?; 673 | 674 | let headers = 675 | HashMap::from([("content-type".to_string(), blob.media_type.clone())]); 676 | 677 | Ok(File { 678 | url: url.clone(), 679 | mtime: None, 680 | maybe_headers: Some(headers), 681 | #[allow(clippy::disallowed_types)] // ok for source 682 | source: std::sync::Arc::from(blob.bytes), 683 | }) 684 | } 685 | 686 | async fn fetch_remote_no_follow( 687 | &self, 688 | url: &Url, 689 | maybe_accept: Option<&str>, 690 | cache_setting: &CacheSetting, 691 | maybe_checksum: Option>, 692 | maybe_auth: Option<(header::HeaderName, header::HeaderValue)>, 693 | ) -> Result { 694 | debug!("FileFetcher::fetch_remote_no_follow - specifier: {}", url); 695 | 696 | if self.should_use_cache(url, cache_setting) { 697 | if let Some(file_or_redirect) = 698 | self.fetch_cached_no_follow(url, maybe_checksum)? 699 | { 700 | return Ok(file_or_redirect); 701 | } 702 | } 703 | 704 | if *cache_setting == CacheSetting::Only { 705 | return Err( 706 | FetchNoFollowErrorKind::NotCached { url: url.clone() }.into_box(), 707 | ); 708 | } 709 | 710 | let maybe_etag_cache_entry = self 711 | .http_cache 712 | .cache_item_key(url) 713 | .ok() 714 | .and_then(|key| self.http_cache.get(&key, maybe_checksum).ok().flatten()) 715 | .and_then(|mut cache_entry| { 716 | cache_entry 717 | .metadata 718 | .headers 719 | .remove("etag") 720 | .map(|etag| (cache_entry, etag)) 721 | }); 722 | 723 | let maybe_auth_token = self.auth_tokens.get(url); 724 | match self 725 | .send_request(SendRequestArgs { 726 | url, 727 | maybe_accept, 728 | maybe_auth: maybe_auth.clone(), 729 | maybe_auth_token, 730 | maybe_etag: maybe_etag_cache_entry 731 | .as_ref() 732 | .map(|(_, etag)| etag.as_str()), 733 | }) 734 | .await? 735 | { 736 | SendRequestResponse::NotModified => { 737 | let (cache_entry, _) = maybe_etag_cache_entry.unwrap(); 738 | FileOrRedirect::from_deno_cache_entry(url, cache_entry).map_err(|err| { 739 | FetchNoFollowErrorKind::RedirectResolution(err).into_box() 740 | }) 741 | } 742 | SendRequestResponse::Redirect(redirect_url, headers) => { 743 | self.http_cache.set(url, headers, &[]).map_err(|source| { 744 | FetchNoFollowErrorKind::CacheSave { 745 | url: url.clone(), 746 | source, 747 | } 748 | })?; 749 | Ok(FileOrRedirect::Redirect(redirect_url)) 750 | } 751 | SendRequestResponse::Code(bytes, headers) => { 752 | self.http_cache.set(url, headers.clone(), &bytes).map_err( 753 | |source| FetchNoFollowErrorKind::CacheSave { 754 | url: url.clone(), 755 | source, 756 | }, 757 | )?; 758 | if let Some(checksum) = &maybe_checksum { 759 | checksum 760 | .check(url, &bytes) 761 | .map_err(|err| FetchNoFollowErrorKind::ChecksumIntegrity(*err))?; 762 | } 763 | Ok(FileOrRedirect::File(File { 764 | url: url.clone(), 765 | mtime: None, 766 | maybe_headers: Some(headers), 767 | #[allow(clippy::disallowed_types)] // ok for source 768 | source: std::sync::Arc::from(bytes), 769 | })) 770 | } 771 | } 772 | } 773 | 774 | /// Returns if the cache should be used for a given specifier. 775 | fn should_use_cache(&self, url: &Url, cache_setting: &CacheSetting) -> bool { 776 | match cache_setting { 777 | CacheSetting::ReloadAll => false, 778 | CacheSetting::Use | CacheSetting::Only => true, 779 | CacheSetting::RespectHeaders => { 780 | let Ok(cache_key) = self.http_cache.cache_item_key(url) else { 781 | return false; 782 | }; 783 | let Ok(Some(headers)) = self.http_cache.read_headers(&cache_key) else { 784 | return false; 785 | }; 786 | let Ok(Some(download_time)) = 787 | self.http_cache.read_download_time(&cache_key) 788 | else { 789 | return false; 790 | }; 791 | let cache_semantics = 792 | CacheSemantics::new(headers, download_time, self.sys.sys_time_now()); 793 | cache_semantics.should_use() 794 | } 795 | CacheSetting::ReloadSome(list) => { 796 | let mut url = url.clone(); 797 | url.set_fragment(None); 798 | if list.iter().any(|x| x == url.as_str()) { 799 | return false; 800 | } 801 | url.set_query(None); 802 | let mut path = PathBuf::from(url.as_str()); 803 | loop { 804 | if list.contains(&path.to_str().unwrap().to_string()) { 805 | return false; 806 | } 807 | if !path.pop() { 808 | break; 809 | } 810 | } 811 | true 812 | } 813 | } 814 | } 815 | 816 | /// Asynchronously fetches the given HTTP URL one pass only. 817 | /// If no redirect is present and no error occurs, 818 | /// yields Code(ResultPayload). 819 | /// If redirect occurs, does not follow and 820 | /// yields Redirect(url). 821 | async fn send_request( 822 | &self, 823 | args: SendRequestArgs<'_>, 824 | ) -> Result { 825 | let mut headers = HeaderMap::with_capacity(3); 826 | 827 | if let Some(etag) = args.maybe_etag { 828 | let if_none_match_val = 829 | HeaderValue::from_str(etag).map_err(|source| { 830 | FetchNoFollowErrorKind::InvalidHeader { 831 | name: "etag", 832 | source, 833 | } 834 | })?; 835 | headers.insert(IF_NONE_MATCH, if_none_match_val); 836 | } 837 | if let Some(auth_token) = args.maybe_auth_token { 838 | let authorization_val = HeaderValue::from_str(&auth_token.to_string()) 839 | .map_err(|source| FetchNoFollowErrorKind::InvalidHeader { 840 | name: "authorization", 841 | source, 842 | })?; 843 | headers.insert(AUTHORIZATION, authorization_val); 844 | } else if let Some((header, value)) = args.maybe_auth { 845 | headers.insert(header, value); 846 | } 847 | if let Some(accept) = args.maybe_accept { 848 | let accepts_val = HeaderValue::from_str(accept).map_err(|source| { 849 | FetchNoFollowErrorKind::InvalidHeader { 850 | name: "accept", 851 | source, 852 | } 853 | })?; 854 | headers.insert(ACCEPT, accepts_val); 855 | } 856 | match self.http_client.send_no_follow(args.url, headers).await { 857 | Ok(resp) => match resp { 858 | SendResponse::NotModified => Ok(SendRequestResponse::NotModified), 859 | SendResponse::Redirect(headers) => { 860 | let new_url = resolve_redirect_from_headers(args.url, &headers) 861 | .map_err(|err| { 862 | FetchNoFollowErrorKind::RedirectHeaderParse(*err).into_box() 863 | })?; 864 | Ok(SendRequestResponse::Redirect( 865 | new_url, 866 | response_headers_to_headers_map(headers), 867 | )) 868 | } 869 | SendResponse::Success(headers, body) => Ok(SendRequestResponse::Code( 870 | body, 871 | response_headers_to_headers_map(headers), 872 | )), 873 | }, 874 | Err(err) => match err { 875 | SendError::Failed(err) => Err( 876 | FetchNoFollowErrorKind::FetchingRemote { 877 | url: args.url.clone(), 878 | source: err, 879 | } 880 | .into_box(), 881 | ), 882 | SendError::NotFound => { 883 | Err(FetchNoFollowErrorKind::NotFound(args.url.clone()).into_box()) 884 | } 885 | SendError::StatusCode(status_code) => Err( 886 | FetchNoFollowErrorKind::ClientError { 887 | url: args.url.clone(), 888 | status_code, 889 | } 890 | .into_box(), 891 | ), 892 | }, 893 | } 894 | } 895 | 896 | /// Fetch a source file from the local file system. 897 | pub fn fetch_local( 898 | &self, 899 | url: &Url, 900 | options: &FetchLocalOptions, 901 | ) -> Result, FetchLocalError> { 902 | let local = url_to_file_path(url)?; 903 | match self.fetch_local_inner(url, &local, options) { 904 | Ok(file) => Ok(Some(file)), 905 | Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), 906 | Err(err) => Err( 907 | FetchLocalErrorKind::ReadingFile(FailedReadingLocalFileError { 908 | url: url.clone(), 909 | source: err, 910 | }) 911 | .into_box(), 912 | ), 913 | } 914 | } 915 | 916 | fn fetch_local_inner( 917 | &self, 918 | url: &Url, 919 | path: &Path, 920 | options: &FetchLocalOptions, 921 | ) -> std::io::Result { 922 | let mut file = self.sys.fs_open(path, &OpenOptions::new_read())?; 923 | let mtime = if options.include_mtime { 924 | file.fs_file_metadata().and_then(|m| m.modified()).ok() 925 | } else { 926 | None 927 | }; 928 | let mut bytes = Vec::new(); 929 | file.read_to_end(&mut bytes)?; 930 | // If it doesnt have a extension, we want to treat it as typescript by default 931 | let headers = if path.extension().is_none() { 932 | Some(HashMap::from([( 933 | "content-type".to_string(), 934 | "application/typescript".to_string(), 935 | )])) 936 | } else { 937 | None 938 | }; 939 | Ok(File { 940 | url: url.clone(), 941 | mtime, 942 | maybe_headers: headers, 943 | source: bytes.into(), 944 | }) 945 | } 946 | } 947 | 948 | fn response_headers_to_headers_map(response_headers: HeaderMap) -> HeadersMap { 949 | let mut result_headers = HashMap::with_capacity(response_headers.len()); 950 | // todo(dsherret): change to consume to avoid allocations 951 | for key in response_headers.keys() { 952 | let key_str = key.to_string(); 953 | let values = response_headers.get_all(key); 954 | // todo(dsherret): this seems very strange storing them comma separated 955 | // like this... what happens if a value contains a comma? 956 | let values_str = values 957 | .iter() 958 | .filter_map(|e| Some(e.to_str().ok()?.to_string())) 959 | .collect::>() 960 | .join(","); 961 | result_headers.insert(key_str, values_str); 962 | } 963 | result_headers 964 | } 965 | 966 | pub fn resolve_redirect_from_headers( 967 | request_url: &Url, 968 | headers: &HeaderMap, 969 | ) -> Result> { 970 | if let Some(location) = headers.get(LOCATION) { 971 | let location_string = location.to_str().map_err(|source| { 972 | Box::new(RedirectHeaderParseError { 973 | request_url: request_url.clone(), 974 | maybe_location: None, 975 | maybe_source: Some(source.into()), 976 | }) 977 | })?; 978 | log::debug!("Redirecting to {:?}...", &location_string); 979 | resolve_url_from_location(request_url, location_string).map_err(|source| { 980 | Box::new(RedirectHeaderParseError { 981 | request_url: request_url.clone(), 982 | maybe_location: Some(location_string.to_string()), 983 | maybe_source: Some(source), 984 | }) 985 | }) 986 | } else { 987 | Err(Box::new(RedirectHeaderParseError { 988 | request_url: request_url.clone(), 989 | maybe_location: None, 990 | maybe_source: None, 991 | })) 992 | } 993 | } 994 | 995 | /// Construct the next uri based on base uri and location header fragment 996 | /// See 997 | fn resolve_url_from_location( 998 | base_url: &Url, 999 | location: &str, 1000 | ) -> Result> { 1001 | // todo(dsherret): these shouldn't unwrap 1002 | if location.starts_with("http://") || location.starts_with("https://") { 1003 | // absolute uri 1004 | Ok(Url::parse(location)?) 1005 | } else if location.starts_with("//") { 1006 | // "//" authority path-abempty 1007 | Ok(Url::parse(&format!("{}:{}", base_url.scheme(), location))?) 1008 | } else if location.starts_with('/') { 1009 | // path-absolute 1010 | Ok(base_url.join(location)?) 1011 | } else { 1012 | // assuming path-noscheme | path-empty 1013 | let base_url_path_str = base_url.path().to_owned(); 1014 | // Pop last part or url (after last slash) 1015 | let segs: Vec<&str> = base_url_path_str.rsplitn(2, '/').collect(); 1016 | let new_path = format!("{}/{}", segs.last().unwrap_or(&""), location); 1017 | Ok(base_url.join(&new_path)?) 1018 | } 1019 | } 1020 | 1021 | #[derive(Debug)] 1022 | struct SendRequestArgs<'a> { 1023 | pub url: &'a Url, 1024 | pub maybe_accept: Option<&'a str>, 1025 | pub maybe_etag: Option<&'a str>, 1026 | pub maybe_auth_token: Option<&'a AuthToken>, 1027 | pub maybe_auth: Option<(header::HeaderName, header::HeaderValue)>, 1028 | } 1029 | 1030 | #[derive(Debug, Eq, PartialEq)] 1031 | enum SendRequestResponse { 1032 | Code(Vec, HeadersMap), 1033 | NotModified, 1034 | Redirect(Url, HeadersMap), 1035 | } 1036 | 1037 | #[cfg(test)] 1038 | mod test { 1039 | use url::Url; 1040 | 1041 | use crate::file_fetcher::resolve_url_from_location; 1042 | 1043 | #[test] 1044 | fn test_resolve_url_from_location_full_1() { 1045 | let url = "http://deno.land".parse::().unwrap(); 1046 | let new_uri = resolve_url_from_location(&url, "http://golang.org").unwrap(); 1047 | assert_eq!(new_uri.host_str().unwrap(), "golang.org"); 1048 | } 1049 | 1050 | #[test] 1051 | fn test_resolve_url_from_location_full_2() { 1052 | let url = "https://deno.land".parse::().unwrap(); 1053 | let new_uri = 1054 | resolve_url_from_location(&url, "https://golang.org").unwrap(); 1055 | assert_eq!(new_uri.host_str().unwrap(), "golang.org"); 1056 | } 1057 | 1058 | #[test] 1059 | fn test_resolve_url_from_location_relative_1() { 1060 | let url = "http://deno.land/x".parse::().unwrap(); 1061 | let new_uri = 1062 | resolve_url_from_location(&url, "//rust-lang.org/en-US").unwrap(); 1063 | assert_eq!(new_uri.host_str().unwrap(), "rust-lang.org"); 1064 | assert_eq!(new_uri.path(), "/en-US"); 1065 | } 1066 | 1067 | #[test] 1068 | fn test_resolve_url_from_location_relative_2() { 1069 | let url = "http://deno.land/x".parse::().unwrap(); 1070 | let new_uri = resolve_url_from_location(&url, "/y").unwrap(); 1071 | assert_eq!(new_uri.host_str().unwrap(), "deno.land"); 1072 | assert_eq!(new_uri.path(), "/y"); 1073 | } 1074 | 1075 | #[test] 1076 | fn test_resolve_url_from_location_relative_3() { 1077 | let url = "http://deno.land/x".parse::().unwrap(); 1078 | let new_uri = resolve_url_from_location(&url, "z").unwrap(); 1079 | assert_eq!(new_uri.host_str().unwrap(), "deno.land"); 1080 | assert_eq!(new_uri.path(), "/z"); 1081 | } 1082 | } 1083 | -------------------------------------------------------------------------------- /rs_lib/src/global/cache_file.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | use std::borrow::Cow; 4 | use std::io::ErrorKind; 5 | use std::path::Path; 6 | 7 | use deno_path_util::fs::atomic_write_file_with_retries; 8 | use serde::de::DeserializeOwned; 9 | use sys_traits::FsCreateDirAll; 10 | use sys_traits::FsMetadata; 11 | use sys_traits::FsOpen; 12 | use sys_traits::FsRead; 13 | use sys_traits::FsRemoveFile; 14 | use sys_traits::FsRename; 15 | use sys_traits::SystemRandom; 16 | use sys_traits::ThreadSleep; 17 | 18 | use crate::cache::CacheEntry; 19 | use crate::SerializedCachedUrlMetadata; 20 | use crate::CACHE_PERM; 21 | 22 | // File format: 23 | // \n// denoCacheMetadata= 24 | 25 | const LAST_LINE_PREFIX: &[u8] = b"\n// denoCacheMetadata="; 26 | 27 | pub fn write< 28 | TSys: FsCreateDirAll 29 | + FsMetadata 30 | + FsOpen 31 | + FsRemoveFile 32 | + FsRename 33 | + ThreadSleep 34 | + SystemRandom, 35 | >( 36 | sys: &TSys, 37 | path: &Path, 38 | content: &[u8], 39 | metadata: &SerializedCachedUrlMetadata, 40 | ) -> std::io::Result<()> { 41 | fn estimate_metadata_capacity( 42 | metadata: &SerializedCachedUrlMetadata, 43 | ) -> usize { 44 | metadata 45 | .headers 46 | .iter() 47 | .map(|(k, v)| k.len() + v.len() + 6) 48 | .sum::() 49 | + metadata.url.len() 50 | + metadata.time.as_ref().map(|_| 14).unwrap_or(0) 51 | + 128 // overestimate 52 | } 53 | 54 | let capacity = content.len() 55 | + LAST_LINE_PREFIX.len() 56 | + estimate_metadata_capacity(metadata); 57 | let mut result = Vec::with_capacity(capacity); 58 | result.extend(content); 59 | result.extend(LAST_LINE_PREFIX); 60 | serde_json::to_writer(&mut result, &metadata).unwrap(); 61 | debug_assert!(result.len() < capacity, "{} < {}", result.len(), capacity); 62 | atomic_write_file_with_retries(sys, path, &result, CACHE_PERM)?; 63 | Ok(()) 64 | } 65 | 66 | pub fn read( 67 | sys: &impl FsRead, 68 | path: &Path, 69 | ) -> std::io::Result> { 70 | let original_file_bytes = match sys.fs_read(path) { 71 | Ok(file) => file, 72 | Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None), 73 | Err(err) => return Err(err), 74 | }; 75 | 76 | let Some((content, metadata)) = 77 | read_content_and_metadata(&original_file_bytes) 78 | else { 79 | return Ok(None); 80 | }; 81 | 82 | let content_len = content.len(); 83 | // truncate the original bytes to just the content 84 | let original_file_bytes = match original_file_bytes { 85 | Cow::Borrowed(bytes) => Cow::Borrowed(&bytes[..content_len]), 86 | Cow::Owned(mut bytes) => { 87 | bytes.truncate(content_len); 88 | Cow::Owned(bytes) 89 | } 90 | }; 91 | 92 | Ok(Some(CacheEntry { 93 | metadata, 94 | content: original_file_bytes, 95 | })) 96 | } 97 | 98 | pub fn read_metadata( 99 | sys: &impl FsRead, 100 | path: &Path, 101 | ) -> std::io::Result> { 102 | let file_bytes = match sys.fs_read(path) { 103 | Ok(file) => file, 104 | Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None), 105 | Err(err) => return Err(err), 106 | }; 107 | 108 | let Some((_content_bytes, metadata)) = 109 | read_content_and_metadata::(&file_bytes) 110 | else { 111 | return Ok(None); 112 | }; 113 | 114 | Ok(Some(metadata)) 115 | } 116 | 117 | fn read_content_and_metadata( 118 | file_bytes: &[u8], 119 | ) -> Option<(&[u8], TMetadata)> { 120 | let (file_bytes, metadata_bytes) = split_content_metadata(file_bytes)?; 121 | let serialized_metadata = 122 | serde_json::from_slice::(metadata_bytes).ok()?; 123 | 124 | Some((file_bytes, serialized_metadata)) 125 | } 126 | 127 | fn split_content_metadata(file_bytes: &[u8]) -> Option<(&[u8], &[u8])> { 128 | let last_newline_index = file_bytes.iter().rposition(|&b| b == b'\n')?; 129 | 130 | let (content, trailing_bytes) = file_bytes.split_at(last_newline_index); 131 | let metadata = trailing_bytes.strip_prefix(LAST_LINE_PREFIX)?; 132 | Some((content, metadata)) 133 | } 134 | -------------------------------------------------------------------------------- /rs_lib/src/global/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | use std::path::PathBuf; 4 | use std::time::Duration; 5 | use std::time::SystemTime; 6 | use std::time::UNIX_EPOCH; 7 | 8 | use serde::Deserialize; 9 | use sys_traits::FsCreateDirAll; 10 | use sys_traits::FsMetadata; 11 | use sys_traits::FsMetadataValue; 12 | use sys_traits::FsOpen; 13 | use sys_traits::FsRead; 14 | use sys_traits::FsRemoveFile; 15 | use sys_traits::FsRename; 16 | use sys_traits::SystemRandom; 17 | use sys_traits::SystemTimeNow; 18 | use sys_traits::ThreadSleep; 19 | use url::Url; 20 | 21 | use super::cache::HttpCache; 22 | use super::cache::HttpCacheItemKey; 23 | use crate::cache::url_to_filename; 24 | use crate::cache::CacheEntry; 25 | use crate::cache::CacheReadFileError; 26 | use crate::cache::Checksum; 27 | use crate::cache::SerializedCachedUrlMetadata; 28 | use crate::common::HeadersMap; 29 | use crate::sync::MaybeSend; 30 | use crate::sync::MaybeSync; 31 | 32 | mod cache_file; 33 | 34 | #[sys_traits::auto_impl] 35 | pub trait GlobalHttpCacheSys: 36 | FsCreateDirAll 37 | + FsMetadata 38 | + FsOpen 39 | + FsRead 40 | + FsRemoveFile 41 | + FsRename 42 | + ThreadSleep 43 | + SystemRandom 44 | + SystemTimeNow 45 | + std::fmt::Debug 46 | + MaybeSend 47 | + MaybeSync 48 | + Clone 49 | { 50 | } 51 | 52 | #[allow(clippy::disallowed_types)] 53 | pub type GlobalHttpCacheRc = crate::sync::MaybeArc>; 54 | 55 | #[derive(Debug)] 56 | pub struct GlobalHttpCache { 57 | path: PathBuf, 58 | pub(crate) sys: Sys, 59 | } 60 | 61 | impl GlobalHttpCache { 62 | pub fn new(sys: Sys, path: PathBuf) -> Self { 63 | #[cfg(not(target_arch = "wasm32"))] 64 | assert!(path.is_absolute()); 65 | Self { path, sys } 66 | } 67 | 68 | pub fn dir_path(&self) -> &PathBuf { 69 | &self.path 70 | } 71 | 72 | pub fn local_path_for_url(&self, url: &Url) -> std::io::Result { 73 | Ok(self.path.join(url_to_filename(url)?)) 74 | } 75 | 76 | #[inline] 77 | fn key_file_path<'a>(&self, key: &'a HttpCacheItemKey) -> &'a PathBuf { 78 | // The key file path is always set for the global cache because 79 | // the file will always exist, unlike the local cache, which won't 80 | // have this for redirects. 81 | key.file_path.as_ref().unwrap() 82 | } 83 | } 84 | 85 | impl HttpCache for GlobalHttpCache { 86 | fn cache_item_key<'a>( 87 | &self, 88 | url: &'a Url, 89 | ) -> std::io::Result> { 90 | Ok(HttpCacheItemKey { 91 | #[cfg(debug_assertions)] 92 | is_local_key: false, 93 | url, 94 | file_path: Some(self.local_path_for_url(url)?), 95 | }) 96 | } 97 | 98 | fn contains(&self, url: &Url) -> bool { 99 | let Ok(cache_filepath) = self.local_path_for_url(url) else { 100 | return false; 101 | }; 102 | self.sys.fs_is_file(&cache_filepath).unwrap_or(false) 103 | } 104 | 105 | fn read_modified_time( 106 | &self, 107 | key: &HttpCacheItemKey, 108 | ) -> std::io::Result> { 109 | #[cfg(debug_assertions)] 110 | debug_assert!(!key.is_local_key); 111 | 112 | match self.sys.fs_metadata(self.key_file_path(key)) { 113 | Ok(metadata) => match metadata.modified() { 114 | Ok(time) => Ok(Some(time)), 115 | Err(_) => Ok(Some(self.sys.sys_time_now())), 116 | }, 117 | Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), 118 | Err(err) => Err(err), 119 | } 120 | } 121 | 122 | fn set( 123 | &self, 124 | url: &Url, 125 | headers: HeadersMap, 126 | content: &[u8], 127 | ) -> std::io::Result<()> { 128 | let cache_filepath = self.local_path_for_url(url)?; 129 | cache_file::write( 130 | &self.sys, 131 | &cache_filepath, 132 | content, 133 | &SerializedCachedUrlMetadata { 134 | time: Some( 135 | self 136 | .sys 137 | .sys_time_now() 138 | .duration_since(UNIX_EPOCH) 139 | .unwrap() 140 | .as_secs(), 141 | ), 142 | url: url.to_string(), 143 | headers, 144 | }, 145 | )?; 146 | 147 | Ok(()) 148 | } 149 | 150 | fn get( 151 | &self, 152 | key: &HttpCacheItemKey, 153 | maybe_checksum: Option, 154 | ) -> Result, CacheReadFileError> { 155 | #[cfg(debug_assertions)] 156 | debug_assert!(!key.is_local_key); 157 | 158 | let maybe_file = cache_file::read(&self.sys, self.key_file_path(key))?; 159 | 160 | if let Some(file) = &maybe_file { 161 | if let Some(expected_checksum) = maybe_checksum { 162 | expected_checksum 163 | .check(key.url, &file.content) 164 | .map_err(CacheReadFileError::ChecksumIntegrity)?; 165 | } 166 | } 167 | 168 | Ok(maybe_file) 169 | } 170 | 171 | fn read_headers( 172 | &self, 173 | key: &HttpCacheItemKey, 174 | ) -> std::io::Result> { 175 | // targeted deserialize 176 | #[derive(Deserialize)] 177 | struct SerializedHeaders { 178 | pub headers: HeadersMap, 179 | } 180 | 181 | #[cfg(debug_assertions)] 182 | debug_assert!(!key.is_local_key); 183 | 184 | let maybe_metadata = cache_file::read_metadata::( 185 | &self.sys, 186 | self.key_file_path(key), 187 | )?; 188 | Ok(maybe_metadata.map(|m| m.headers)) 189 | } 190 | 191 | fn read_download_time( 192 | &self, 193 | key: &HttpCacheItemKey, 194 | ) -> std::io::Result> { 195 | // targeted deserialize 196 | #[derive(Deserialize)] 197 | struct SerializedTime { 198 | pub time: Option, 199 | } 200 | 201 | #[cfg(debug_assertions)] 202 | debug_assert!(!key.is_local_key); 203 | let maybe_metadata = cache_file::read_metadata::( 204 | &self.sys, 205 | self.key_file_path(key), 206 | )?; 207 | Ok(maybe_metadata.and_then(|m| { 208 | Some(SystemTime::UNIX_EPOCH + Duration::from_secs(m.time?)) 209 | })) 210 | } 211 | } 212 | 213 | #[cfg(test)] 214 | mod test { 215 | use super::*; 216 | 217 | #[test] 218 | fn test_url_to_filename() { 219 | let test_cases = [ 220 | ("https://deno.land/x/foo.ts", "https/deno.land/2c0a064891b9e3fbe386f5d4a833bce5076543f5404613656042107213a7bbc8"), 221 | ( 222 | "https://deno.land:8080/x/foo.ts", 223 | "https/deno.land_PORT8080/2c0a064891b9e3fbe386f5d4a833bce5076543f5404613656042107213a7bbc8", 224 | ), 225 | ("https://deno.land/", "https/deno.land/8a5edab282632443219e051e4ade2d1d5bbc671c781051bf1437897cbdfea0f1"), 226 | ( 227 | "https://deno.land/?asdf=qwer", 228 | "https/deno.land/e4edd1f433165141015db6a823094e6bd8f24dd16fe33f2abd99d34a0a21a3c0", 229 | ), 230 | // should be the same as case above, fragment (#qwer) is ignored 231 | // when hashing 232 | ( 233 | "https://deno.land/?asdf=qwer#qwer", 234 | "https/deno.land/e4edd1f433165141015db6a823094e6bd8f24dd16fe33f2abd99d34a0a21a3c0", 235 | ), 236 | ( 237 | "data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=", 238 | "data/c21c7fc382b2b0553dc0864aa81a3acacfb7b3d1285ab5ae76da6abec213fb37", 239 | ), 240 | ( 241 | "data:text/plain,Hello%2C%20Deno!", 242 | "data/967374e3561d6741234131e342bf5c6848b70b13758adfe23ee1a813a8131818", 243 | ) 244 | ]; 245 | 246 | for (url, expected) in test_cases.iter() { 247 | let u = Url::parse(url).unwrap(); 248 | let p = url_to_filename(&u).unwrap(); 249 | assert_eq!(p, PathBuf::from(expected)); 250 | } 251 | } 252 | } 253 | -------------------------------------------------------------------------------- /rs_lib/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | mod cache; 4 | mod common; 5 | mod deno_dir; 6 | #[cfg(feature = "file_fetcher")] 7 | pub mod file_fetcher; 8 | mod global; 9 | mod local; 10 | pub mod memory; 11 | pub mod npm; 12 | mod sync; 13 | 14 | /// Permissions used to save a file in the disk caches. 15 | pub const CACHE_PERM: u32 = 0o644; 16 | 17 | pub use cache::url_to_filename; 18 | pub use cache::CacheEntry; 19 | pub use cache::CacheReadFileError; 20 | pub use cache::Checksum; 21 | pub use cache::ChecksumIntegrityError; 22 | pub use cache::GlobalOrLocalHttpCache; 23 | pub use cache::GlobalToLocalCopy; 24 | pub use cache::HttpCache; 25 | pub use cache::HttpCacheItemKey; 26 | pub use cache::HttpCacheRc; 27 | pub use cache::SerializedCachedUrlMetadata; 28 | pub use common::HeadersMap; 29 | pub use deno_dir::resolve_deno_dir; 30 | pub use deno_dir::DenoDirResolutionError; 31 | pub use deno_dir::ResolveDenoDirSys; 32 | pub use global::GlobalHttpCache; 33 | pub use global::GlobalHttpCacheRc; 34 | pub use global::GlobalHttpCacheSys; 35 | pub use local::LocalHttpCache; 36 | pub use local::LocalHttpCacheRc; 37 | pub use local::LocalHttpCacheSys; 38 | pub use local::LocalLspHttpCache; 39 | 40 | #[cfg(feature = "wasm")] 41 | pub mod wasm { 42 | use std::collections::HashMap; 43 | use std::io::ErrorKind; 44 | use std::path::PathBuf; 45 | 46 | use js_sys::Object; 47 | use js_sys::Reflect; 48 | use js_sys::Uint8Array; 49 | use sys_traits::impls::wasm_path_to_str; 50 | use sys_traits::impls::wasm_string_to_path; 51 | use sys_traits::impls::RealSys; 52 | use sys_traits::EnvVar; 53 | use url::Url; 54 | use wasm_bindgen::prelude::*; 55 | 56 | use crate::cache::CacheEntry; 57 | use crate::cache::GlobalToLocalCopy; 58 | use crate::common::HeadersMap; 59 | use crate::deno_dir; 60 | use crate::sync::new_rc; 61 | use crate::CacheReadFileError; 62 | use crate::Checksum; 63 | use crate::HttpCache; 64 | 65 | #[wasm_bindgen] 66 | pub fn url_to_filename(url: &str) -> Result { 67 | console_error_panic_hook::set_once(); 68 | let url = parse_url(url).map_err(as_js_error)?; 69 | crate::cache::url_to_filename(&url) 70 | .map(|s| s.to_string_lossy().to_string()) 71 | .map_err(as_js_error) 72 | } 73 | 74 | #[wasm_bindgen] 75 | pub fn resolve_deno_dir( 76 | maybe_custom_root: Option, 77 | ) -> Result { 78 | console_error_panic_hook::set_once(); 79 | deno_dir::resolve_deno_dir( 80 | &RealSys, 81 | maybe_custom_root.map(wasm_string_to_path), 82 | ) 83 | .map(|path| wasm_path_to_str(&path).into_owned()) 84 | .map_err(|e| JsValue::from(js_sys::Error::new(&e.to_string()))) 85 | } 86 | 87 | #[wasm_bindgen] 88 | pub struct GlobalHttpCache { 89 | cache: crate::GlobalHttpCache, 90 | } 91 | 92 | #[wasm_bindgen] 93 | impl GlobalHttpCache { 94 | pub fn new(path: &str) -> Self { 95 | Self { 96 | cache: crate::GlobalHttpCache::new(RealSys, PathBuf::from(path)), 97 | } 98 | } 99 | 100 | #[wasm_bindgen(js_name = getHeaders)] 101 | pub fn get_headers(&self, url: &str) -> Result { 102 | get_headers(&self.cache, url) 103 | } 104 | 105 | pub fn get( 106 | &self, 107 | url: &str, 108 | maybe_checksum: Option, 109 | ) -> Result { 110 | get_cache_entry(&self.cache, url, maybe_checksum.as_deref()) 111 | } 112 | 113 | pub fn set( 114 | &self, 115 | url: &str, 116 | headers: JsValue, 117 | text: &[u8], 118 | ) -> Result<(), JsValue> { 119 | set(&self.cache, url, headers, text) 120 | } 121 | } 122 | 123 | #[wasm_bindgen] 124 | pub struct LocalHttpCache { 125 | cache: crate::LocalHttpCache, 126 | } 127 | 128 | #[wasm_bindgen] 129 | impl LocalHttpCache { 130 | pub fn new( 131 | local_path: String, 132 | global_path: String, 133 | allow_global_to_local_copy: bool, 134 | ) -> Self { 135 | console_error_panic_hook::set_once(); 136 | let global = 137 | crate::GlobalHttpCache::new(RealSys, wasm_string_to_path(global_path)); 138 | let jsr_url = RealSys 139 | .env_var("JSR_URL") 140 | .ok() 141 | .and_then(|url| { 142 | // ensure there is a trailing slash for the directory 143 | let registry_url = format!("{}/", url.trim_end_matches('/')); 144 | Url::parse(®istry_url).ok() 145 | }) 146 | .unwrap_or_else(|| Url::parse("https://jsr.io/").unwrap()); 147 | let local = crate::LocalHttpCache::new( 148 | wasm_string_to_path(local_path), 149 | new_rc(global), 150 | if allow_global_to_local_copy { 151 | GlobalToLocalCopy::Allow 152 | } else { 153 | GlobalToLocalCopy::Disallow 154 | }, 155 | jsr_url, 156 | ); 157 | Self { cache: local } 158 | } 159 | 160 | #[wasm_bindgen(js_name = getHeaders)] 161 | pub fn get_headers(&self, url: &str) -> Result { 162 | get_headers(&self.cache, url) 163 | } 164 | 165 | pub fn get( 166 | &self, 167 | url: &str, 168 | maybe_checksum: Option, 169 | ) -> Result { 170 | get_cache_entry(&self.cache, url, maybe_checksum.as_deref()) 171 | } 172 | 173 | pub fn set( 174 | &self, 175 | url: &str, 176 | headers: JsValue, 177 | text: &[u8], 178 | ) -> Result<(), JsValue> { 179 | set(&self.cache, url, headers, text) 180 | } 181 | } 182 | 183 | fn get_headers( 184 | cache: &Cache, 185 | url: &str, 186 | ) -> Result { 187 | fn inner( 188 | cache: &Cache, 189 | url: &str, 190 | ) -> std::io::Result> { 191 | let url = parse_url(url)?; 192 | let key = cache.cache_item_key(&url)?; 193 | cache.read_headers(&key) 194 | } 195 | 196 | inner(cache, url) 197 | .map(|headers| match headers { 198 | Some(headers) => serde_wasm_bindgen::to_value(&headers).unwrap(), 199 | None => JsValue::undefined(), 200 | }) 201 | .map_err(as_js_error) 202 | } 203 | 204 | fn get_cache_entry( 205 | cache: &Cache, 206 | url: &str, 207 | maybe_checksum: Option<&str>, 208 | ) -> Result { 209 | fn inner( 210 | cache: &Cache, 211 | url: &str, 212 | maybe_checksum: Option, 213 | ) -> std::io::Result> { 214 | let url = parse_url(url)?; 215 | let key = cache.cache_item_key(&url)?; 216 | match cache.get(&key, maybe_checksum) { 217 | Ok(Some(entry)) => Ok(Some(entry)), 218 | Ok(None) => Ok(None), 219 | Err(err) => match err { 220 | CacheReadFileError::Io(err) => Err(err), 221 | CacheReadFileError::ChecksumIntegrity(err) => { 222 | Err(std::io::Error::new(ErrorKind::InvalidData, err.to_string())) 223 | } 224 | }, 225 | } 226 | } 227 | 228 | inner(cache, url, maybe_checksum.map(Checksum::new)) 229 | .map(|text| match text { 230 | Some(entry) => { 231 | let content = { 232 | let array = Uint8Array::new_with_length(entry.content.len() as u32); 233 | array.copy_from(&entry.content); 234 | JsValue::from(array) 235 | }; 236 | let headers: JsValue = { 237 | // make it an object instead of a Map 238 | let headers_object = Object::new(); 239 | for (key, value) in &entry.metadata.headers { 240 | Reflect::set( 241 | &headers_object, 242 | &JsValue::from_str(key), 243 | &JsValue::from_str(value), 244 | ) 245 | .unwrap(); 246 | } 247 | JsValue::from(headers_object) 248 | }; 249 | let obj = Object::new(); 250 | Reflect::set(&obj, &JsValue::from_str("content"), &content).unwrap(); 251 | Reflect::set(&obj, &JsValue::from_str("headers"), &headers).unwrap(); 252 | JsValue::from(obj) 253 | } 254 | None => JsValue::undefined(), 255 | }) 256 | .map_err(as_js_error) 257 | } 258 | 259 | fn set( 260 | cache: &Cache, 261 | url: &str, 262 | headers: JsValue, 263 | content: &[u8], 264 | ) -> Result<(), JsValue> { 265 | fn inner( 266 | cache: &Cache, 267 | url: &str, 268 | headers: JsValue, 269 | content: &[u8], 270 | ) -> std::io::Result<()> { 271 | let url = parse_url(url)?; 272 | let headers: HashMap = 273 | serde_wasm_bindgen::from_value(headers).map_err(|err| { 274 | std::io::Error::new(ErrorKind::InvalidData, err.to_string()) 275 | })?; 276 | cache.set(&url, headers, content) 277 | } 278 | 279 | inner(cache, url, headers, content).map_err(as_js_error) 280 | } 281 | 282 | fn parse_url(url: &str) -> std::io::Result { 283 | Url::parse(url) 284 | .map_err(|e| std::io::Error::new(ErrorKind::InvalidInput, e.to_string())) 285 | } 286 | 287 | fn as_js_error(e: std::io::Error) -> JsValue { 288 | JsValue::from(js_sys::Error::new(&e.to_string())) 289 | } 290 | } 291 | -------------------------------------------------------------------------------- /rs_lib/src/memory.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | use std::borrow::Cow; 4 | use std::collections::HashMap; 5 | use std::time::UNIX_EPOCH; 6 | 7 | use parking_lot::Mutex; 8 | use sys_traits::SystemTimeNow; 9 | use url::Url; 10 | 11 | use crate::sync::MaybeSend; 12 | use crate::sync::MaybeSync; 13 | use crate::CacheEntry; 14 | use crate::CacheReadFileError; 15 | use crate::Checksum; 16 | use crate::HeadersMap; 17 | use crate::HttpCache; 18 | use crate::HttpCacheItemKey; 19 | use crate::SerializedCachedUrlMetadata; 20 | 21 | #[cfg(not(target_arch = "wasm32"))] 22 | #[derive(Debug)] 23 | pub struct MemoryHttpCacheSystemTimeClock; 24 | 25 | #[cfg(not(target_arch = "wasm32"))] 26 | impl sys_traits::SystemTimeNow for MemoryHttpCacheSystemTimeClock { 27 | fn sys_time_now(&self) -> std::time::SystemTime { 28 | #[allow(clippy::disallowed_methods)] 29 | std::time::SystemTime::now() 30 | } 31 | } 32 | 33 | /// A simple in-memory cache mostly useful for testing. 34 | #[derive(Debug)] 35 | pub struct MemoryHttpCache { 36 | cache: Mutex>, 37 | clock: TSys, 38 | } 39 | 40 | #[cfg(not(target_arch = "wasm32"))] 41 | impl Default for MemoryHttpCache { 42 | fn default() -> Self { 43 | Self::new(MemoryHttpCacheSystemTimeClock) 44 | } 45 | } 46 | 47 | impl 48 | MemoryHttpCache 49 | { 50 | pub fn new(clock: TSys) -> Self { 51 | Self { 52 | cache: Mutex::new(HashMap::new()), 53 | clock, 54 | } 55 | } 56 | } 57 | 58 | impl HttpCache 59 | for MemoryHttpCache 60 | { 61 | fn cache_item_key<'a>( 62 | &self, 63 | url: &'a Url, 64 | ) -> std::io::Result> { 65 | Ok(HttpCacheItemKey { 66 | #[cfg(debug_assertions)] 67 | is_local_key: false, 68 | url, 69 | file_path: None, 70 | }) 71 | } 72 | 73 | fn contains(&self, url: &Url) -> bool { 74 | self.cache.lock().contains_key(url) 75 | } 76 | 77 | fn set( 78 | &self, 79 | url: &Url, 80 | headers: HeadersMap, 81 | content: &[u8], 82 | ) -> std::io::Result<()> { 83 | self.cache.lock().insert( 84 | url.clone(), 85 | CacheEntry { 86 | metadata: SerializedCachedUrlMetadata { 87 | headers, 88 | url: url.to_string(), 89 | time: Some( 90 | self 91 | .clock 92 | .sys_time_now() 93 | .duration_since(UNIX_EPOCH) 94 | .unwrap() 95 | .as_secs(), 96 | ), 97 | }, 98 | content: Cow::Owned(content.to_vec()), 99 | }, 100 | ); 101 | Ok(()) 102 | } 103 | 104 | fn get( 105 | &self, 106 | key: &HttpCacheItemKey, 107 | maybe_checksum: Option, 108 | ) -> Result, CacheReadFileError> { 109 | self 110 | .cache 111 | .lock() 112 | .get(key.url) 113 | .cloned() 114 | .map(|entry| { 115 | if let Some(checksum) = maybe_checksum { 116 | checksum 117 | .check(key.url, &entry.content) 118 | .map_err(CacheReadFileError::ChecksumIntegrity)?; 119 | } 120 | Ok(entry) 121 | }) 122 | .transpose() 123 | } 124 | 125 | fn read_modified_time( 126 | &self, 127 | _key: &HttpCacheItemKey, 128 | ) -> std::io::Result> { 129 | Ok(None) // for now 130 | } 131 | 132 | fn read_headers( 133 | &self, 134 | key: &HttpCacheItemKey, 135 | ) -> std::io::Result> { 136 | Ok( 137 | self 138 | .cache 139 | .lock() 140 | .get(key.url) 141 | .map(|entry| entry.metadata.headers.clone()), 142 | ) 143 | } 144 | 145 | fn read_download_time( 146 | &self, 147 | key: &HttpCacheItemKey, 148 | ) -> std::io::Result> { 149 | Ok(self.cache.lock().get(key.url).and_then(|entry| { 150 | entry 151 | .metadata 152 | .time 153 | .map(|time| UNIX_EPOCH + std::time::Duration::from_secs(time)) 154 | })) 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /rs_lib/src/npm.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | use std::io::ErrorKind; 4 | use std::path::Path; 5 | use std::path::PathBuf; 6 | 7 | use deno_path_util::normalize_path; 8 | use deno_path_util::url_from_directory_path; 9 | use sys_traits::FsCanonicalize; 10 | use sys_traits::FsCreateDirAll; 11 | use url::Url; 12 | 13 | pub struct NpmCacheFolderId { 14 | /// Package name. 15 | pub name: String, 16 | /// Package version. 17 | pub version: String, 18 | /// Package copy index. 19 | pub copy_index: u8, 20 | } 21 | 22 | /// The global cache directory of npm packages. 23 | #[derive(Clone, Debug)] 24 | pub struct NpmCacheDir { 25 | root_dir: PathBuf, 26 | // cached url representation of the root directory 27 | root_dir_url: Url, 28 | // A list of all registry that were discovered via `.npmrc` files 29 | // turned into a safe directory names. 30 | known_registries_dirnames: Vec, 31 | } 32 | 33 | impl NpmCacheDir { 34 | pub fn new( 35 | sys: &Sys, 36 | root_dir: PathBuf, 37 | known_registries_urls: Vec, 38 | ) -> Self { 39 | fn try_get_canonicalized_root_dir( 40 | sys: &Sys, 41 | root_dir: &Path, 42 | ) -> Result { 43 | match sys.fs_canonicalize(root_dir) { 44 | Ok(path) => Ok(path), 45 | Err(err) if err.kind() == ErrorKind::NotFound => { 46 | sys.fs_create_dir_all(root_dir)?; 47 | sys.fs_canonicalize(root_dir) 48 | } 49 | Err(err) => Err(err), 50 | } 51 | } 52 | 53 | // this may fail on readonly file systems, so just ignore if so 54 | let root_dir = normalize_path(root_dir); 55 | let root_dir = 56 | try_get_canonicalized_root_dir(sys, &root_dir).unwrap_or(root_dir); 57 | let root_dir_url = url_from_directory_path(&root_dir).unwrap(); 58 | 59 | let known_registries_dirnames: Vec<_> = known_registries_urls 60 | .into_iter() 61 | .map(|url| { 62 | root_url_to_safe_local_dirname(&url) 63 | .to_string_lossy() 64 | .replace('\\', "/") 65 | }) 66 | .collect(); 67 | 68 | Self { 69 | root_dir, 70 | root_dir_url, 71 | known_registries_dirnames, 72 | } 73 | } 74 | 75 | pub fn root_dir(&self) -> &Path { 76 | &self.root_dir 77 | } 78 | 79 | pub fn root_dir_url(&self) -> &Url { 80 | &self.root_dir_url 81 | } 82 | 83 | pub fn package_folder_for_id( 84 | &self, 85 | package_name: &str, 86 | package_version: &str, 87 | package_copy_index: u8, 88 | registry_url: &Url, 89 | ) -> PathBuf { 90 | if package_copy_index == 0 { 91 | self 92 | .package_name_folder(package_name, registry_url) 93 | .join(package_version) 94 | } else { 95 | self 96 | .package_name_folder(package_name, registry_url) 97 | .join(format!("{}_{}", package_version, package_copy_index)) 98 | } 99 | } 100 | 101 | pub fn package_name_folder(&self, name: &str, registry_url: &Url) -> PathBuf { 102 | let mut dir = self.registry_folder(registry_url); 103 | if name.to_lowercase() != name { 104 | let encoded_name = mixed_case_package_name_encode(name); 105 | // Using the encoded directory may have a collision with an actual package name 106 | // so prefix it with an underscore since npm packages can't start with that 107 | dir.join(format!("_{encoded_name}")) 108 | } else { 109 | // ensure backslashes are used on windows 110 | for part in name.split('/') { 111 | dir = dir.join(part); 112 | } 113 | dir 114 | } 115 | } 116 | 117 | fn registry_folder(&self, registry_url: &Url) -> PathBuf { 118 | self 119 | .root_dir 120 | .join(root_url_to_safe_local_dirname(registry_url)) 121 | } 122 | 123 | pub fn resolve_package_folder_id_from_specifier( 124 | &self, 125 | specifier: &Url, 126 | ) -> Option { 127 | let mut maybe_relative_url = None; 128 | 129 | // Iterate through known registries and try to get a match. 130 | for registry_dirname in &self.known_registries_dirnames { 131 | let registry_root_dir = self 132 | .root_dir_url 133 | .join(&format!("{}/", registry_dirname)) 134 | // this not succeeding indicates a fatal issue, so unwrap 135 | .unwrap(); 136 | 137 | let Some(relative_url) = registry_root_dir.make_relative(specifier) 138 | else { 139 | continue; 140 | }; 141 | 142 | if relative_url.starts_with("../") { 143 | continue; 144 | } 145 | 146 | maybe_relative_url = Some(relative_url); 147 | break; 148 | } 149 | 150 | let mut relative_url = maybe_relative_url?; 151 | 152 | // base32 decode the url if it starts with an underscore 153 | // * Ex. _{base32(package_name)}/ 154 | if let Some(end_url) = relative_url.strip_prefix('_') { 155 | let mut parts = end_url 156 | .split('/') 157 | .map(ToOwned::to_owned) 158 | .collect::>(); 159 | match mixed_case_package_name_decode(&parts[0]) { 160 | Some(part) => { 161 | parts[0] = part; 162 | } 163 | None => return None, 164 | } 165 | relative_url = parts.join("/"); 166 | } 167 | 168 | // examples: 169 | // * chalk/5.0.1/ 170 | // * @types/chalk/5.0.1/ 171 | // * some-package/5.0.1_1/ -- where the `_1` (/_\d+/) is a copy of the folder for peer deps 172 | let is_scoped_package = relative_url.starts_with('@'); 173 | let mut parts = relative_url 174 | .split('/') 175 | .enumerate() 176 | .take(if is_scoped_package { 3 } else { 2 }) 177 | .map(|(_, part)| part) 178 | .collect::>(); 179 | if parts.len() < 2 { 180 | return None; 181 | } 182 | let version_part = parts.pop().unwrap(); 183 | let name = parts.join("/"); 184 | let (version, copy_index) = 185 | if let Some((version, copy_count)) = version_part.split_once('_') { 186 | (version, copy_count.parse::().ok()?) 187 | } else { 188 | (version_part, 0) 189 | }; 190 | Some(NpmCacheFolderId { 191 | name, 192 | version: version.to_string(), 193 | copy_index, 194 | }) 195 | } 196 | 197 | pub fn get_cache_location(&self) -> PathBuf { 198 | self.root_dir.clone() 199 | } 200 | } 201 | 202 | pub fn mixed_case_package_name_encode(name: &str) -> String { 203 | // use base32 encoding because it's reversible and the character set 204 | // only includes the characters within 0-9 and A-Z so it can be lower cased 205 | base32::encode( 206 | base32::Alphabet::Rfc4648Lower { padding: false }, 207 | name.as_bytes(), 208 | ) 209 | .to_lowercase() 210 | } 211 | 212 | pub fn mixed_case_package_name_decode(name: &str) -> Option { 213 | base32::decode(base32::Alphabet::Rfc4648Lower { padding: false }, name) 214 | .and_then(|b| String::from_utf8(b).ok()) 215 | } 216 | 217 | /// Gets a safe local directory name for the provided url. 218 | /// 219 | /// For example: 220 | /// https://deno.land:8080/path -> deno.land_8080/path 221 | fn root_url_to_safe_local_dirname(root: &Url) -> PathBuf { 222 | fn sanitize_segment(text: &str) -> String { 223 | text 224 | .chars() 225 | .map(|c| if is_banned_segment_char(c) { '_' } else { c }) 226 | .collect() 227 | } 228 | 229 | fn is_banned_segment_char(c: char) -> bool { 230 | matches!(c, '/' | '\\') || is_banned_path_char(c) 231 | } 232 | 233 | let mut result = String::new(); 234 | if let Some(domain) = root.domain() { 235 | result.push_str(&sanitize_segment(domain)); 236 | } 237 | if let Some(port) = root.port() { 238 | if !result.is_empty() { 239 | result.push('_'); 240 | } 241 | result.push_str(&port.to_string()); 242 | } 243 | let mut result = PathBuf::from(result); 244 | if let Some(segments) = root.path_segments() { 245 | for segment in segments.filter(|s| !s.is_empty()) { 246 | result = result.join(sanitize_segment(segment)); 247 | } 248 | } 249 | 250 | result 251 | } 252 | 253 | /// Gets if the provided character is not supported on all 254 | /// kinds of file systems. 255 | fn is_banned_path_char(c: char) -> bool { 256 | matches!(c, '<' | '>' | ':' | '"' | '|' | '?' | '*') 257 | } 258 | 259 | #[cfg(test)] 260 | mod test { 261 | use std::path::PathBuf; 262 | 263 | use sys_traits::FsCreateDirAll; 264 | use url::Url; 265 | 266 | use super::NpmCacheDir; 267 | 268 | #[test] 269 | fn should_get_package_folder() { 270 | let sys = sys_traits::impls::InMemorySys::default(); 271 | let root_dir = if cfg!(windows) { 272 | PathBuf::from("C:\\cache") 273 | } else { 274 | PathBuf::from("/cache") 275 | }; 276 | sys.fs_create_dir_all(&root_dir).unwrap(); 277 | let registry_url = Url::parse("https://registry.npmjs.org/").unwrap(); 278 | let cache = 279 | NpmCacheDir::new(&sys, root_dir.clone(), vec![registry_url.clone()]); 280 | 281 | assert_eq!( 282 | cache.package_folder_for_id("json", "1.2.5", 0, ®istry_url,), 283 | root_dir 284 | .join("registry.npmjs.org") 285 | .join("json") 286 | .join("1.2.5"), 287 | ); 288 | 289 | assert_eq!( 290 | cache.package_folder_for_id("json", "1.2.5", 1, ®istry_url,), 291 | root_dir 292 | .join("registry.npmjs.org") 293 | .join("json") 294 | .join("1.2.5_1"), 295 | ); 296 | 297 | assert_eq!( 298 | cache.package_folder_for_id("JSON", "2.1.5", 0, ®istry_url,), 299 | root_dir 300 | .join("registry.npmjs.org") 301 | .join("_jjju6tq") 302 | .join("2.1.5"), 303 | ); 304 | 305 | assert_eq!( 306 | cache.package_folder_for_id("@types/JSON", "2.1.5", 0, ®istry_url,), 307 | root_dir 308 | .join("registry.npmjs.org") 309 | .join("_ib2hs4dfomxuuu2pjy") 310 | .join("2.1.5"), 311 | ); 312 | } 313 | } 314 | -------------------------------------------------------------------------------- /rs_lib/src/sync.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | pub use inner::*; 4 | 5 | #[cfg(feature = "sync")] 6 | mod inner { 7 | #![allow(clippy::disallowed_types)] 8 | 9 | pub use core::marker::Send as MaybeSend; 10 | pub use core::marker::Sync as MaybeSync; 11 | pub use std::sync::Arc as MaybeArc; 12 | } 13 | 14 | #[cfg(not(feature = "sync"))] 15 | mod inner { 16 | pub trait MaybeSync {} 17 | impl MaybeSync for T where T: ?Sized {} 18 | pub trait MaybeSend {} 19 | impl MaybeSend for T where T: ?Sized {} 20 | 21 | pub use std::rc::Rc as MaybeArc; 22 | } 23 | 24 | #[allow(clippy::disallowed_types)] 25 | #[allow(dead_code)] // not used for all features 26 | #[inline] 27 | pub fn new_rc(value: T) -> MaybeArc { 28 | MaybeArc::new(value) 29 | } 30 | -------------------------------------------------------------------------------- /rs_lib/tests/file_fetcher_test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | #![allow(clippy::disallowed_methods)] 4 | 5 | use std::rc::Rc; 6 | use std::time::SystemTime; 7 | 8 | use deno_cache_dir::file_fetcher::AuthTokens; 9 | use deno_cache_dir::file_fetcher::CacheSetting; 10 | use deno_cache_dir::file_fetcher::FetchLocalOptions; 11 | use deno_cache_dir::file_fetcher::FetchNoFollowErrorKind; 12 | use deno_cache_dir::file_fetcher::FetchNoFollowOptions; 13 | use deno_cache_dir::file_fetcher::FileFetcher; 14 | use deno_cache_dir::file_fetcher::FileFetcherOptions; 15 | use deno_cache_dir::file_fetcher::FileOrRedirect; 16 | use deno_cache_dir::file_fetcher::HttpClient; 17 | use deno_cache_dir::file_fetcher::NullBlobStore; 18 | use deno_cache_dir::file_fetcher::NullMemoryFiles; 19 | use deno_cache_dir::file_fetcher::SendError; 20 | use deno_cache_dir::file_fetcher::SendResponse; 21 | use deno_cache_dir::memory::MemoryHttpCache; 22 | use http::HeaderMap; 23 | use sys_traits::impls::InMemorySys; 24 | use sys_traits::FsCreateDirAll; 25 | use sys_traits::FsWrite; 26 | use url::Url; 27 | 28 | #[tokio::test] 29 | async fn test_file_fetcher_redirects() { 30 | #[derive(Debug)] 31 | struct TestHttpClient; 32 | 33 | #[async_trait::async_trait(?Send)] 34 | impl HttpClient for TestHttpClient { 35 | async fn send_no_follow( 36 | &self, 37 | _url: &Url, 38 | _headers: HeaderMap, 39 | ) -> Result { 40 | Ok(SendResponse::Redirect(HeaderMap::new())) 41 | } 42 | } 43 | 44 | let sys = InMemorySys::default(); 45 | let file_fetcher = create_file_fetcher(sys.clone(), TestHttpClient); 46 | let result = file_fetcher 47 | .fetch_no_follow( 48 | &Url::parse("http://localhost/bad_redirect").unwrap(), 49 | FetchNoFollowOptions::default(), 50 | ) 51 | .await; 52 | 53 | match result.unwrap_err().into_kind() { 54 | FetchNoFollowErrorKind::RedirectHeaderParse(err) => { 55 | assert_eq!(err.request_url.as_str(), "http://localhost/bad_redirect"); 56 | } 57 | err => unreachable!("{:?}", err), 58 | } 59 | 60 | let time = SystemTime::now(); 61 | sys.set_time(Some(time)); 62 | sys.fs_create_dir_all("/").unwrap(); 63 | sys.fs_write("/some_path.ts", "text").unwrap(); 64 | 65 | for include_mtime in [true, false] { 66 | let result = file_fetcher 67 | .fetch_no_follow( 68 | &Url::parse("file:///some_path.ts").unwrap(), 69 | FetchNoFollowOptions { 70 | local: FetchLocalOptions { include_mtime }, 71 | ..Default::default() 72 | }, 73 | ) 74 | .await; 75 | match result.unwrap() { 76 | FileOrRedirect::File(file) => { 77 | assert_eq!(file.mtime, if include_mtime { Some(time) } else { None }); 78 | assert_eq!(file.source.to_vec(), b"text"); 79 | } 80 | FileOrRedirect::Redirect(_) => unreachable!(), 81 | } 82 | } 83 | } 84 | 85 | fn create_file_fetcher( 86 | sys: InMemorySys, 87 | client: TClient, 88 | ) -> FileFetcher { 89 | FileFetcher::new( 90 | NullBlobStore, 91 | sys, 92 | Rc::new(MemoryHttpCache::default()), 93 | client, 94 | Rc::new(NullMemoryFiles), 95 | FileFetcherOptions { 96 | allow_remote: true, 97 | cache_setting: CacheSetting::Use, 98 | auth_tokens: AuthTokens::new(None), 99 | }, 100 | ) 101 | } 102 | -------------------------------------------------------------------------------- /rs_lib/tests/integration_test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | #![allow(clippy::disallowed_methods)] 4 | 5 | use std::collections::HashMap; 6 | use std::path::Path; 7 | use std::rc::Rc; 8 | 9 | use deno_cache_dir::CacheReadFileError; 10 | use deno_cache_dir::Checksum; 11 | use deno_cache_dir::GlobalHttpCache; 12 | use deno_cache_dir::GlobalToLocalCopy; 13 | use deno_cache_dir::HttpCache; 14 | use deno_cache_dir::LocalHttpCache; 15 | use deno_cache_dir::LocalLspHttpCache; 16 | use serde_json::json; 17 | use sys_traits::impls::RealSys; 18 | use tempfile::TempDir; 19 | use url::Url; 20 | 21 | fn jsr_url() -> Url { 22 | Url::parse("https://jsr.io/").unwrap() 23 | } 24 | 25 | #[test] 26 | fn test_global_create_cache() { 27 | let dir = TempDir::new().unwrap(); 28 | let cache_path = dir.path().join("foobar"); 29 | // HttpCache should be created lazily on first use: 30 | // when zipping up a local project with no external dependencies 31 | // "$DENO_DIR/remote" is empty. When unzipping such project 32 | // "$DENO_DIR/remote" might not get restored and in situation 33 | // when directory is owned by root we might not be able 34 | // to create that directory. However if it's not needed it 35 | // doesn't make sense to return error in such specific scenarios. 36 | // For more details check issue: 37 | // https://github.com/denoland/deno/issues/5688 38 | let sys = RealSys; 39 | let cache = GlobalHttpCache::new(sys, cache_path.clone()); 40 | assert!(!cache.dir_path().exists()); 41 | let url = Url::parse("http://example.com/foo/bar.js").unwrap(); 42 | cache.set(&url, Default::default(), b"hello world").unwrap(); 43 | assert!(cache_path.is_dir()); 44 | assert!(cache.local_path_for_url(&url).unwrap().is_file()); 45 | } 46 | 47 | #[test] 48 | fn test_global_get_set() { 49 | let dir = TempDir::new().unwrap(); 50 | let sys = RealSys; 51 | let cache = GlobalHttpCache::new(sys, dir.path().to_path_buf()); 52 | let url = Url::parse("https://deno.land/x/welcome.ts").unwrap(); 53 | let mut headers = HashMap::new(); 54 | headers.insert( 55 | "content-type".to_string(), 56 | "application/javascript".to_string(), 57 | ); 58 | headers.insert("etag".to_string(), "as5625rqdsfb".to_string()); 59 | let content = b"Hello world"; 60 | cache.set(&url, headers, content).unwrap(); 61 | let key = cache.cache_item_key(&url).unwrap(); 62 | let content = String::from_utf8( 63 | cache.get(&key, None).unwrap().unwrap().content.into_owned(), 64 | ) 65 | .unwrap(); 66 | let headers = cache.read_headers(&key).unwrap().unwrap(); 67 | assert_eq!(content, "Hello world"); 68 | assert_eq!( 69 | headers.get("content-type").unwrap(), 70 | "application/javascript" 71 | ); 72 | assert_eq!(headers.get("etag").unwrap(), "as5625rqdsfb"); 73 | assert_eq!(headers.get("foobar"), None); 74 | let download_time = cache.read_download_time(&key).unwrap().unwrap(); 75 | let elapsed = download_time.elapsed().unwrap(); 76 | assert!(elapsed.as_secs() < 2, "Elapsed: {:?}", elapsed); 77 | let matching_checksum = 78 | "64ec88ca00b268e5ba1a35678a1b5316d212f4f366b2477232534a8aeca37f3c"; 79 | // reading with checksum that matches 80 | { 81 | let found_content = cache 82 | .get(&key, Some(Checksum::new(matching_checksum))) 83 | .unwrap() 84 | .unwrap() 85 | .content; 86 | assert_eq!(found_content, content.as_bytes()); 87 | } 88 | // reading with a checksum that doesn't match 89 | { 90 | let not_matching_checksum = "1234"; 91 | let err = cache 92 | .get(&key, Some(Checksum::new(not_matching_checksum))) 93 | .err() 94 | .unwrap(); 95 | let err = match err { 96 | CacheReadFileError::ChecksumIntegrity(err) => err, 97 | _ => unreachable!(), 98 | }; 99 | assert_eq!(err.actual, matching_checksum); 100 | assert_eq!(err.expected, not_matching_checksum); 101 | assert_eq!(err.url, url); 102 | } 103 | } 104 | 105 | #[test] 106 | fn test_local_global_cache() { 107 | let temp_dir = TempDir::new().unwrap(); 108 | let global_cache_path = temp_dir.path().join("global"); 109 | let local_cache_path = temp_dir.path().join("local"); 110 | let sys = RealSys; 111 | let global_cache = 112 | Rc::new(GlobalHttpCache::new(sys, global_cache_path.clone())); 113 | let local_cache = LocalHttpCache::new( 114 | local_cache_path.clone(), 115 | global_cache.clone(), 116 | GlobalToLocalCopy::Allow, 117 | jsr_url(), 118 | ); 119 | 120 | let manifest_file_path = local_cache_path.join("manifest.json"); 121 | // mapped url 122 | { 123 | let url = Url::parse("https://deno.land/x/mod.ts").unwrap(); 124 | let content = "export const test = 5;"; 125 | global_cache 126 | .set( 127 | &url, 128 | HashMap::from([( 129 | "content-type".to_string(), 130 | "application/typescript".to_string(), 131 | )]), 132 | content.as_bytes(), 133 | ) 134 | .unwrap(); 135 | let key = local_cache.cache_item_key(&url).unwrap(); 136 | assert_eq!( 137 | String::from_utf8( 138 | local_cache 139 | .get(&key, None) 140 | .unwrap() 141 | .unwrap() 142 | .content 143 | .into_owned() 144 | ) 145 | .unwrap(), 146 | content 147 | ); 148 | let headers = local_cache.read_headers(&key).unwrap().unwrap(); 149 | // won't have any headers because the content-type is derivable from the url 150 | assert_eq!(headers, HashMap::new()); 151 | // no manifest file yet 152 | assert!(!manifest_file_path.exists()); 153 | 154 | // now try deleting the global cache and we should still be able to load it 155 | std::fs::remove_dir_all(&global_cache_path).unwrap(); 156 | assert_eq!( 157 | String::from_utf8( 158 | local_cache 159 | .get(&key, None) 160 | .unwrap() 161 | .unwrap() 162 | .content 163 | .into_owned() 164 | ) 165 | .unwrap(), 166 | content 167 | ); 168 | } 169 | 170 | // file that's directly mappable to a url 171 | { 172 | let content = "export const a = 1;"; 173 | std::fs::write(local_cache_path.join("deno.land").join("main.js"), content) 174 | .unwrap(); 175 | 176 | // now we should be able to read this file because it's directly mappable to a url 177 | let url = Url::parse("https://deno.land/main.js").unwrap(); 178 | let key = local_cache.cache_item_key(&url).unwrap(); 179 | assert_eq!( 180 | String::from_utf8( 181 | local_cache 182 | .get(&key, None) 183 | .unwrap() 184 | .unwrap() 185 | .content 186 | .into_owned() 187 | ) 188 | .unwrap(), 189 | content 190 | ); 191 | let headers = local_cache.read_headers(&key).unwrap().unwrap(); 192 | assert_eq!(headers, HashMap::new()); 193 | } 194 | 195 | // now try a file with a different content-type header 196 | { 197 | let url = 198 | Url::parse("https://deno.land/x/different_content_type.ts").unwrap(); 199 | let content = "export const test = 5;"; 200 | global_cache 201 | .set( 202 | &url, 203 | HashMap::from([( 204 | "content-type".to_string(), 205 | "application/javascript".to_string(), 206 | )]), 207 | content.as_bytes(), 208 | ) 209 | .unwrap(); 210 | let key = local_cache.cache_item_key(&url).unwrap(); 211 | assert_eq!( 212 | String::from_utf8( 213 | local_cache 214 | .get(&key, None) 215 | .unwrap() 216 | .unwrap() 217 | .content 218 | .into_owned() 219 | ) 220 | .unwrap(), 221 | content 222 | ); 223 | let headers = local_cache.read_headers(&key).unwrap().unwrap(); 224 | assert_eq!( 225 | headers, 226 | HashMap::from([( 227 | "content-type".to_string(), 228 | "application/javascript".to_string(), 229 | )]) 230 | ); 231 | assert_eq!( 232 | read_manifest(&manifest_file_path), 233 | json!({ 234 | "modules": { 235 | "https://deno.land/x/different_content_type.ts": { 236 | "headers": { 237 | "content-type": "application/javascript" 238 | } 239 | } 240 | } 241 | }) 242 | ); 243 | // delete the manifest file 244 | std::fs::remove_file(&manifest_file_path).unwrap(); 245 | 246 | // Now try resolving the key again and the content type should still be application/javascript. 247 | // This is maintained because we hash the filename when the headers don't match the extension. 248 | let headers = local_cache.read_headers(&key).unwrap().unwrap(); 249 | assert_eq!( 250 | headers, 251 | HashMap::from([( 252 | "content-type".to_string(), 253 | "application/javascript".to_string(), 254 | )]) 255 | ); 256 | } 257 | 258 | // reset the local cache 259 | std::fs::remove_dir_all(&local_cache_path).unwrap(); 260 | let local_cache = LocalHttpCache::new( 261 | local_cache_path.clone(), 262 | global_cache.clone(), 263 | GlobalToLocalCopy::Allow, 264 | jsr_url(), 265 | ); 266 | 267 | // now try caching a file with many headers 268 | { 269 | let url = Url::parse("https://deno.land/x/my_file.ts").unwrap(); 270 | let content = "export const test = 5;"; 271 | global_cache 272 | .set( 273 | &url, 274 | HashMap::from([ 275 | ( 276 | "content-type".to_string(), 277 | "application/typescript".to_string(), 278 | ), 279 | ("x-typescript-types".to_string(), "./types.d.ts".to_string()), 280 | ("x-deno-warning".to_string(), "Stop right now.".to_string()), 281 | ( 282 | "x-other-header".to_string(), 283 | "Thank you very much.".to_string(), 284 | ), 285 | ]), 286 | content.as_bytes(), 287 | ) 288 | .unwrap(); 289 | let check_output = |local_cache: &LocalHttpCache<_>| { 290 | let key = local_cache.cache_item_key(&url).unwrap(); 291 | assert_eq!( 292 | String::from_utf8( 293 | local_cache 294 | .get(&key, None) 295 | .unwrap() 296 | .unwrap() 297 | .content 298 | .into_owned() 299 | ) 300 | .unwrap(), 301 | content 302 | ); 303 | let headers = local_cache.read_headers(&key).unwrap().unwrap(); 304 | assert_eq!( 305 | headers, 306 | HashMap::from([ 307 | ("x-typescript-types".to_string(), "./types.d.ts".to_string(),), 308 | ("x-deno-warning".to_string(), "Stop right now.".to_string(),) 309 | ]) 310 | ); 311 | assert_eq!( 312 | read_manifest(&manifest_file_path), 313 | json!({ 314 | "modules": { 315 | "https://deno.land/x/my_file.ts": { 316 | "headers": { 317 | "x-deno-warning": "Stop right now.", 318 | "x-typescript-types": "./types.d.ts" 319 | } 320 | } 321 | } 322 | }) 323 | ); 324 | }; 325 | check_output(&local_cache); 326 | // now ensure it's the same when re-creating the cache 327 | check_output(&LocalHttpCache::new( 328 | local_cache_path.to_path_buf(), 329 | global_cache.clone(), 330 | GlobalToLocalCopy::Allow, 331 | jsr_url(), 332 | )); 333 | } 334 | 335 | // reset the local cache 336 | std::fs::remove_dir_all(&local_cache_path).unwrap(); 337 | let local_cache = LocalHttpCache::new( 338 | local_cache_path.clone(), 339 | global_cache.clone(), 340 | GlobalToLocalCopy::Allow, 341 | jsr_url(), 342 | ); 343 | 344 | // try a file that can't be mapped to the file system 345 | { 346 | { 347 | let url = Url::parse("https://deno.land/INVALID/Module.ts?dev").unwrap(); 348 | let content = "export const test = 5;"; 349 | global_cache 350 | .set(&url, HashMap::new(), content.as_bytes()) 351 | .unwrap(); 352 | let key = local_cache.cache_item_key(&url).unwrap(); 353 | assert_eq!( 354 | String::from_utf8( 355 | local_cache 356 | .get(&key, None) 357 | .unwrap() 358 | .unwrap() 359 | .content 360 | .into_owned() 361 | ) 362 | .unwrap(), 363 | content 364 | ); 365 | let headers = local_cache.read_headers(&key).unwrap().unwrap(); 366 | // won't have any headers because the content-type is derivable from the url 367 | assert_eq!(headers, HashMap::new()); 368 | } 369 | 370 | // now try a file in the same directory, but that maps to the local filesystem 371 | { 372 | let url = Url::parse("https://deno.land/INVALID/module2.ts").unwrap(); 373 | let content = "export const test = 4;"; 374 | global_cache 375 | .set(&url, HashMap::new(), content.as_bytes()) 376 | .unwrap(); 377 | let key = local_cache.cache_item_key(&url).unwrap(); 378 | assert_eq!( 379 | String::from_utf8( 380 | local_cache 381 | .get(&key, None) 382 | .unwrap() 383 | .unwrap() 384 | .content 385 | .into_owned() 386 | ) 387 | .unwrap(), 388 | content 389 | ); 390 | assert!(local_cache_path 391 | .join("deno.land/#invalid_1ee01/module2.ts") 392 | .exists()); 393 | 394 | // ensure we can still read this file with a new local cache 395 | let local_cache = LocalHttpCache::new( 396 | local_cache_path.to_path_buf(), 397 | global_cache.clone(), 398 | GlobalToLocalCopy::Allow, 399 | jsr_url(), 400 | ); 401 | assert_eq!( 402 | String::from_utf8( 403 | local_cache 404 | .get(&key, None) 405 | .unwrap() 406 | .unwrap() 407 | .content 408 | .into_owned() 409 | ) 410 | .unwrap(), 411 | content 412 | ); 413 | } 414 | 415 | assert_eq!( 416 | read_manifest(&manifest_file_path), 417 | json!({ 418 | "modules": { 419 | "https://deno.land/INVALID/Module.ts?dev": { 420 | } 421 | }, 422 | "folders": { 423 | "https://deno.land/INVALID/": "deno.land/#invalid_1ee01", 424 | } 425 | }) 426 | ); 427 | } 428 | 429 | // reset the local cache 430 | std::fs::remove_dir_all(&local_cache_path).unwrap(); 431 | let local_cache = LocalHttpCache::new( 432 | local_cache_path.clone(), 433 | global_cache.clone(), 434 | GlobalToLocalCopy::Allow, 435 | jsr_url(), 436 | ); 437 | 438 | // now try a redirect 439 | { 440 | let url = Url::parse("https://deno.land/redirect.ts").unwrap(); 441 | global_cache 442 | .set( 443 | &url, 444 | HashMap::from([("location".to_string(), "./x/mod.ts".to_string())]), 445 | "Redirecting to other url...".as_bytes(), 446 | ) 447 | .unwrap(); 448 | let key = local_cache.cache_item_key(&url).unwrap(); 449 | let headers = local_cache.read_headers(&key).unwrap().unwrap(); 450 | assert_eq!( 451 | headers, 452 | HashMap::from([("location".to_string(), "./x/mod.ts".to_string())]) 453 | ); 454 | assert_eq!( 455 | read_manifest(&manifest_file_path), 456 | json!({ 457 | "modules": { 458 | "https://deno.land/redirect.ts": { 459 | "headers": { 460 | "location": "./x/mod.ts" 461 | } 462 | } 463 | } 464 | }) 465 | ); 466 | } 467 | 468 | // reset the local cache 469 | std::fs::remove_dir_all(&local_cache_path).unwrap(); 470 | let local_cache = LocalHttpCache::new( 471 | local_cache_path.clone(), 472 | global_cache.clone(), 473 | GlobalToLocalCopy::Allow, 474 | jsr_url(), 475 | ); 476 | let url = Url::parse("https://deno.land/x/mod.ts").unwrap(); 477 | let matching_checksum = 478 | "5eadcbe625a8489347fc3b229ab66bdbcbdfecedf229dfe5d0a8a399dae6c005"; 479 | let content = "export const test = 5;"; 480 | global_cache 481 | .set( 482 | &url, 483 | HashMap::from([( 484 | "content-type".to_string(), 485 | "application/typescript".to_string(), 486 | )]), 487 | content.as_bytes(), 488 | ) 489 | .unwrap(); 490 | let key = local_cache.cache_item_key(&url).unwrap(); 491 | // reading with a checksum that doesn't match 492 | // (ensure it doesn't match twice so we know it wasn't copied to the local cache) 493 | for _ in 0..2 { 494 | let not_matching_checksum = "1234"; 495 | let err = local_cache 496 | .get(&key, Some(Checksum::new(not_matching_checksum))) 497 | .err() 498 | .unwrap(); 499 | let err = match err { 500 | CacheReadFileError::ChecksumIntegrity(err) => err, 501 | _ => unreachable!(), 502 | }; 503 | assert_eq!(err.actual, matching_checksum); 504 | assert_eq!(err.expected, not_matching_checksum); 505 | assert_eq!(err.url, url); 506 | } 507 | // reading with checksum that matches 508 | { 509 | let found_content = local_cache 510 | .get(&key, Some(Checksum::new(matching_checksum))) 511 | .unwrap() 512 | .unwrap() 513 | .content; 514 | assert_eq!(found_content, content.as_bytes()); 515 | } 516 | // at this point the file should exist in the local cache and so the checksum will be ignored 517 | { 518 | let found_content = local_cache 519 | .get(&key, Some(Checksum::new("not matching"))) 520 | .unwrap() 521 | .unwrap() 522 | .content; 523 | assert_eq!(found_content, content.as_bytes()); 524 | } 525 | } 526 | 527 | fn read_manifest(path: &Path) -> serde_json::Value { 528 | let manifest = std::fs::read_to_string(path).unwrap(); 529 | serde_json::from_str(&manifest).unwrap() 530 | } 531 | 532 | #[test] 533 | fn test_lsp_local_cache() { 534 | let temp_dir = TempDir::new().unwrap(); 535 | let global_cache_path = temp_dir.path().join("global"); 536 | let local_cache_path = temp_dir.path().join("local"); 537 | let sys = RealSys; 538 | let global_cache = 539 | Rc::new(GlobalHttpCache::new(sys, global_cache_path.to_path_buf())); 540 | let local_cache = LocalHttpCache::new( 541 | local_cache_path.to_path_buf(), 542 | global_cache.clone(), 543 | GlobalToLocalCopy::Allow, 544 | jsr_url(), 545 | ); 546 | let create_readonly_cache = || { 547 | LocalLspHttpCache::new(local_cache_path.to_path_buf(), global_cache.clone()) 548 | }; 549 | 550 | // mapped url 551 | { 552 | let url = Url::parse("https://deno.land/x/mod.ts").unwrap(); 553 | let content = "export const test = 5;"; 554 | global_cache 555 | .set( 556 | &url, 557 | HashMap::from([( 558 | "content-type".to_string(), 559 | "application/typescript".to_string(), 560 | )]), 561 | content.as_bytes(), 562 | ) 563 | .unwrap(); 564 | // will be None because it's readonly 565 | { 566 | let readonly_local_cache = create_readonly_cache(); 567 | let key = readonly_local_cache.cache_item_key(&url).unwrap(); 568 | assert_eq!(readonly_local_cache.get(&key, None).unwrap(), None); 569 | } 570 | // populate it with the non-readonly local cache 571 | { 572 | let key = local_cache.cache_item_key(&url).unwrap(); 573 | assert_eq!( 574 | String::from_utf8( 575 | local_cache 576 | .get(&key, None) 577 | .unwrap() 578 | .unwrap() 579 | .content 580 | .into_owned() 581 | ) 582 | .unwrap(), 583 | content 584 | ); 585 | } 586 | // now the readonly cache will have it 587 | { 588 | let readonly_local_cache = create_readonly_cache(); 589 | let key = readonly_local_cache.cache_item_key(&url).unwrap(); 590 | assert_eq!( 591 | String::from_utf8( 592 | readonly_local_cache 593 | .get(&key, None) 594 | .unwrap() 595 | .unwrap() 596 | .content 597 | .into_owned() 598 | ) 599 | .unwrap(), 600 | content 601 | ); 602 | } 603 | 604 | { 605 | // check getting the file url works 606 | let readonly_local_cache = create_readonly_cache(); 607 | let file_url = readonly_local_cache.get_file_url(&url); 608 | let expected = Url::from_directory_path(&local_cache_path) 609 | .unwrap() 610 | .join("deno.land/x/mod.ts") 611 | .unwrap(); 612 | assert_eq!(file_url, Some(expected)); 613 | 614 | // get the reverse mapping 615 | let mapping = readonly_local_cache.get_remote_url( 616 | local_cache_path 617 | .join("deno.land") 618 | .join("x") 619 | .join("mod.ts") 620 | .as_path(), 621 | ); 622 | assert_eq!(mapping.as_ref(), Some(&url)); 623 | } 624 | } 625 | 626 | // now try a file with a different content-type header 627 | { 628 | let url = 629 | Url::parse("https://deno.land/x/different_content_type.ts").unwrap(); 630 | let content = "export const test = 5;"; 631 | global_cache 632 | .set( 633 | &url, 634 | HashMap::from([( 635 | "content-type".to_string(), 636 | "application/javascript".to_string(), 637 | )]), 638 | content.as_bytes(), 639 | ) 640 | .unwrap(); 641 | // populate it with the non-readonly local cache 642 | { 643 | let key = local_cache.cache_item_key(&url).unwrap(); 644 | assert_eq!( 645 | String::from_utf8( 646 | local_cache 647 | .get(&key, None) 648 | .unwrap() 649 | .unwrap() 650 | .content 651 | .into_owned() 652 | ) 653 | .unwrap(), 654 | content 655 | ); 656 | } 657 | { 658 | let readonly_local_cache = create_readonly_cache(); 659 | let key = readonly_local_cache.cache_item_key(&url).unwrap(); 660 | assert_eq!( 661 | String::from_utf8( 662 | readonly_local_cache 663 | .get(&key, None) 664 | .unwrap() 665 | .unwrap() 666 | .content 667 | .into_owned() 668 | ) 669 | .unwrap(), 670 | content 671 | ); 672 | 673 | let file_url = readonly_local_cache.get_file_url(&url).unwrap(); 674 | let path = file_url.to_file_path().unwrap(); 675 | assert!(path.exists()); 676 | let mapping = readonly_local_cache.get_remote_url(&path); 677 | assert_eq!(mapping.as_ref(), Some(&url)); 678 | } 679 | } 680 | 681 | // try http specifiers that can't be mapped to the file system 682 | { 683 | let urls = [ 684 | "http://deno.land/INVALID/Module.ts?dev", 685 | "http://deno.land/INVALID/SubDir/Module.ts?dev", 686 | ]; 687 | for url in urls { 688 | let url = Url::parse(url).unwrap(); 689 | let content = "export const test = 5;"; 690 | global_cache 691 | .set(&url, HashMap::new(), content.as_bytes()) 692 | .unwrap(); 693 | // populate it with the non-readonly local cache 694 | { 695 | let key = local_cache.cache_item_key(&url).unwrap(); 696 | assert_eq!( 697 | String::from_utf8( 698 | local_cache 699 | .get(&key, None) 700 | .unwrap() 701 | .unwrap() 702 | .content 703 | .into_owned() 704 | ) 705 | .unwrap(), 706 | content 707 | ); 708 | } 709 | { 710 | let readonly_local_cache = create_readonly_cache(); 711 | let key = readonly_local_cache.cache_item_key(&url).unwrap(); 712 | assert_eq!( 713 | String::from_utf8( 714 | readonly_local_cache 715 | .get(&key, None) 716 | .unwrap() 717 | .unwrap() 718 | .content 719 | .into_owned() 720 | ) 721 | .unwrap(), 722 | content 723 | ); 724 | 725 | let file_url = readonly_local_cache.get_file_url(&url).unwrap(); 726 | let path = file_url.to_file_path().unwrap(); 727 | assert!(path.exists()); 728 | let mapping = readonly_local_cache.get_remote_url(&path); 729 | assert_eq!(mapping.as_ref(), Some(&url)); 730 | } 731 | } 732 | 733 | // now try a files in the same and sub directories, that maps to the local filesystem 734 | let urls = [ 735 | "http://deno.land/INVALID/module2.ts", 736 | "http://deno.land/INVALID/SubDir/module3.ts", 737 | "http://deno.land/INVALID/SubDir/sub_dir/module4.ts", 738 | ]; 739 | for url in urls { 740 | let url = Url::parse(url).unwrap(); 741 | let content = "export const test = 4;"; 742 | global_cache 743 | .set(&url, HashMap::new(), content.as_bytes()) 744 | .unwrap(); 745 | // populate it with the non-readonly local cache 746 | { 747 | let key = local_cache.cache_item_key(&url).unwrap(); 748 | assert_eq!( 749 | String::from_utf8( 750 | local_cache 751 | .get(&key, None) 752 | .unwrap() 753 | .unwrap() 754 | .content 755 | .into_owned() 756 | ) 757 | .unwrap(), 758 | content 759 | ); 760 | } 761 | { 762 | let readonly_local_cache = create_readonly_cache(); 763 | let key = readonly_local_cache.cache_item_key(&url).unwrap(); 764 | assert_eq!( 765 | String::from_utf8( 766 | readonly_local_cache 767 | .get(&key, None) 768 | .unwrap() 769 | .unwrap() 770 | .content 771 | .into_owned() 772 | ) 773 | .unwrap(), 774 | content 775 | ); 776 | let file_url = readonly_local_cache.get_file_url(&url).unwrap(); 777 | let path = file_url.to_file_path().unwrap(); 778 | assert!(path.exists()); 779 | let mapping = readonly_local_cache.get_remote_url(&path); 780 | assert_eq!(mapping.as_ref(), Some(&url)); 781 | } 782 | 783 | // ensure we can still get this file with a new local cache 784 | let local_cache = create_readonly_cache(); 785 | let file_url = local_cache.get_file_url(&url).unwrap(); 786 | let path = file_url.to_file_path().unwrap(); 787 | assert!(path.exists()); 788 | let mapping = local_cache.get_remote_url(&path); 789 | assert_eq!(mapping.as_ref(), Some(&url)); 790 | } 791 | } 792 | } 793 | -------------------------------------------------------------------------------- /rust-toolchain.toml: -------------------------------------------------------------------------------- 1 | [toolchain] 2 | channel = "1.85.0" 3 | components = ["clippy", "rustfmt"] 4 | -------------------------------------------------------------------------------- /test.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | import { assertEquals } from "@std/assert"; 4 | import { createGraph } from "@deno/graph"; 5 | import { join } from "@std/path"; 6 | import { createCache } from "./mod.ts"; 7 | import { assert } from "./util.ts"; 8 | 9 | Deno.test({ 10 | name: "createCache()", 11 | async fn() { 12 | const cache = createCache(); 13 | const { load } = cache; 14 | for (let i = 0; i < 2; i++) { 15 | const graph = await createGraph( 16 | "https://deno.land/x/oak@v10.5.1/mod.ts", 17 | { 18 | load, 19 | }, 20 | ); 21 | assertEquals(graph.modules.length, 59); 22 | } 23 | }, 24 | }); 25 | 26 | Deno.test({ 27 | name: "createCache() - local vendor folder", 28 | async fn() { 29 | await withTempDir(async (tempDir) => { 30 | const vendorRoot = join(tempDir, "vendor"); 31 | const cache = createCache({ 32 | vendorRoot, 33 | }); 34 | 35 | for (let i = 0; i < 2; i++) { 36 | const { load } = cache; 37 | const graph = await createGraph( 38 | "https://deno.land/x/oak@v10.5.1/mod.ts", 39 | { 40 | load, 41 | }, 42 | ); 43 | assertEquals(graph.modules.length, 59); 44 | assert(Deno.statSync(vendorRoot).isDirectory); 45 | assert( 46 | Deno.statSync(join(vendorRoot, "deno.land", "x", "oak@v10.5.1")) 47 | .isDirectory, 48 | ); 49 | } 50 | }); 51 | }, 52 | }); 53 | 54 | async function withTempDir(fn: (tempDir: string) => Promise) { 55 | const tempDir = Deno.makeTempDirSync(); 56 | try { 57 | return await fn(tempDir); 58 | } finally { 59 | Deno.removeSync(tempDir, { recursive: true }); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /util.ts: -------------------------------------------------------------------------------- 1 | // Copyright 2018-2025 the Deno authors. MIT license. 2 | 3 | export const CACHE_PERM = 0o644; 4 | 5 | export function assert(cond: unknown, msg = "Assertion failed."): asserts cond { 6 | if (!cond) { 7 | throw new Error(msg); 8 | } 9 | } 10 | 11 | export function isFileSync(filePath: string): boolean { 12 | try { 13 | const stats = Deno.lstatSync(filePath); 14 | return stats.isFile; 15 | } catch (err) { 16 | if (err instanceof Deno.errors.NotFound) { 17 | return false; 18 | } 19 | throw err; 20 | } 21 | } 22 | --------------------------------------------------------------------------------