├── .git-blame-ignore-revs ├── .gitmodules ├── CHANGELOG.md ├── LICENSE.txt ├── README.md ├── default.nix ├── doc ├── data-on-disk.md └── gallery.md ├── extension ├── .gitignore ├── README.amo.html ├── README.md ├── background │ ├── capture-snapshot.js │ ├── capture.js │ ├── display.js │ ├── issue-acc.js │ ├── loggable-dump.js │ ├── main.js │ ├── main.template │ ├── notifier.js │ ├── persistence.js │ ├── reload.js │ ├── scheduler.js │ ├── state-global.js │ ├── state-tab.js │ └── util.js ├── bin │ ├── crx.sh │ └── gen-chromium-key.sh ├── build.sh ├── default.nix ├── gupdate.xml.template ├── icon │ ├── orbitals │ │ ├── archiving.svg │ │ ├── error.svg │ │ ├── idle.svg │ │ ├── main.svg │ │ ├── off.svg │ │ └── tracking.svg │ └── privateer │ │ ├── archiving.svg │ │ ├── bar.svg │ │ ├── bothlimbo.svg │ │ ├── dot.svg │ │ ├── error.svg │ │ ├── failed.svg │ │ ├── idle.svg │ │ ├── in_limbo.svg │ │ ├── limbo.svg │ │ ├── main.svg │ │ ├── neglimbo.svg │ │ ├── off.svg │ │ ├── problematic.svg │ │ ├── tracking.svg │ │ ├── unreplayable.svg │ │ ├── unsnapshottable.svg │ │ └── work_offline.svg ├── inject │ └── snapshot.js ├── lib │ ├── base.js │ ├── caydarsc.js │ ├── cbor-s.js │ ├── compat.js │ ├── idbp.js │ ├── lslot.js │ ├── pako-ext.js │ ├── schedule-timeout.js │ ├── ui.js │ ├── webext-rpc-client.js │ ├── webext-rpc-server.js │ └── webext.js ├── manifest-chromium-mv2.json ├── manifest-common.json ├── manifest-firefox-mv2.json ├── page │ ├── help.js │ ├── help.org │ ├── help.template │ ├── main.css │ ├── minimal.js │ ├── minimal.template │ ├── popup.html │ ├── popup.js │ ├── popup.template │ ├── reqres-ui.css │ ├── reqres-ui.js │ ├── saved.js │ ├── saved.template │ ├── state.js │ └── state.template └── update-readme.sh ├── firefox ├── README.md └── always-give-raw.patch ├── packages.nix ├── simple_server ├── README.md ├── default.nix ├── hoardy_web_sas.py ├── pyproject.toml ├── setup.py └── update-readme.sh ├── source.nix ├── tool ├── README.md ├── default.nix ├── hoardy_web │ ├── __init__.py │ ├── __main__.py │ ├── filter.py │ ├── linst.py │ ├── mime.py │ ├── mitmproxy.py │ ├── output.py │ ├── source.py │ ├── static.py │ ├── tracking.py │ ├── web.py │ ├── wire.py │ └── wrr.py ├── pyproject.toml ├── sanity.sh ├── script │ ├── README.org │ ├── hoardy-web-spd-say │ ├── hoardy-web-view-pandoc │ ├── hoardy-web-view-w3m │ ├── hoardy-web-xdg-open │ └── hoardy-web-xdg-open-mimi ├── setup.py ├── test-tool.sh └── update-readme.sh ├── update-changelog.sh ├── update-docs.sh └── update-readme.sh /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # re-format code with `black` 2 | 3d4af2eba60b47d142e6ddefc23d770a2eed84cf 3 | aec958ed9fc6e898321e55a725862b9f563ac3e4 4 | dc27b78694dfa6d0c70e672be00116ba0c8b19f0 5 | 480262959fc29eba02c23de9841a5f2c5bbad08f 6 | # fix style issues found by `pylint` 7 | 4df5f8ecfbe4afb167015d2b8fd85ac827f84e4f 8 | ebefd024cdb2cfa77f87a9a1d1d01cfb16faefe2 9 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "vendor/kisstdlib"] 2 | path = vendor/kisstdlib 3 | url = https://github.com/oxij/kisstdlib 4 | [submodule "vendor/cbor2"] 5 | path = vendor/cbor2 6 | url = https://github.com/oxij/cbor2 7 | [submodule "vendor/pako"] 8 | path = vendor/pako 9 | url = https://github.com/nodeca/pako 10 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import {} 2 | , developer ? false 3 | }: 4 | 5 | let packages = import ./packages.nix { inherit pkgs developer; }; in 6 | 7 | pkgs.buildEnv { 8 | name = "hoardy-web-env-20250124"; 9 | paths = with packages; [ 10 | simple_server 11 | extension 12 | tool 13 | ]; 14 | } 15 | -------------------------------------------------------------------------------- /doc/gallery.md: -------------------------------------------------------------------------------- 1 | # extension-v1.19.0 2 | 3 | ![Screenshot of Firefox's viewport with extension's popup shown.](https://oxij.org/asset/demo/software/hoardy-web/extension-v1.19.0-popup.png) 4 | 5 | ![Extension's `P&R` tab with it switched to `Submit dumps via 'HTTP'` mode and its `Server URL` pointing to `hoardy-web serve` instance.](https://oxij.org/asset/demo/software/hoardy-web/extension-v1.19.0-pr.png) 6 | 7 | ![Screenshot of Firefox's viewport with a replay of the page from the previous screenshot.](https://oxij.org/asset/demo/software/hoardy-web/extension-v1.19.0-replay.png) 8 | 9 | ![Screenshot of extension's help page under Firefox. The highlighted setting is referenced by the text under the mouse cursor.](https://oxij.org/asset/demo/software/hoardy-web/extension-v1.19.0-help-page.png) 10 | 11 | ![Screenshot of extension's help page under Firefox set to a dark mode theme. The highlighted setting is referenced by the text under the mouse cursor.](https://oxij.org/asset/demo/software/hoardy-web/extension-v1.19.0-help-page-dark.png) 12 | 13 | # extension-v1.18.0 14 | 15 | ![Screenshot of Firefox's viewport with extension's popup shown.](https://oxij.org/asset/demo/software/hoardy-web/extension-v1.18.0-popup.png) 16 | 17 | # extension-v1.13.0 18 | 19 | ![Screenshot of Firefox's viewport with extension's popup shown.](https://oxij.org/asset/demo/software/hoardy-web/extension-v1.13.0-popup.png) 20 | 21 | ![Screenshot of extension's help page under Firefox set to a dark mode theme.](https://oxij.org/asset/demo/software/hoardy-web/extension-v1.13.0-help-page-dark.png) 22 | 23 | # extension-v1.10.0 24 | 25 | ![Screenshot of Firefox's viewport with extension's popup shown.](https://oxij.org/asset/demo/software/hoardy-web/extension-v1.10.0-popup.png) 26 | 27 | ![Screenshot of extension's help page under Firefox set to a dark mode theme. The highlighted setting is referenced by the text under the mouse cursor.](https://oxij.org/asset/demo/software/hoardy-web/extension-v1.10.0-help-page-dark.png) 28 | 29 | ![Screenshot of Chromium's viewport with extension's popup shown.](https://oxij.org/asset/demo/software/hoardy-web/extension-v1.10.0-chromium.png) 30 | 31 | # extension-v1.7.0 32 | 33 | ![Screenshot of browser's viewport with extension's popup shown.](https://oxij.org/asset/demo/software/hoardy-web/extension-v1.7.0-popup.png) 34 | 35 | ![Screenshot of extension's help page. The highlighted setting is referenced by the text under the mouse cursor.](https://oxij.org/asset/demo/software/hoardy-web/extension-v1.7.0-help-page.png) 36 | 37 | # extension-v1.5.0 38 | 39 | ![Screenshot of browser's viewport with extension's popup shown.](https://oxij.org/asset/demo/software/hoardy-web/extension-1.5-popup.png) 40 | 41 | ![Screenshot of extension's help page. The highlighted setting is referenced by the text under the mouse cursor.](https://oxij.org/asset/demo/software/hoardy-web/extension-1.5-help-page.png) 42 | -------------------------------------------------------------------------------- /extension/.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | node_modules/ 3 | package.json 4 | package-lock.json 5 | private/chromium.key.pem 6 | -------------------------------------------------------------------------------- /extension/README.amo.html: -------------------------------------------------------------------------------- 1 | Hoardy-Web passively captures and collects dumps of HTTP requests and responses as you browse the web, and then archives them using one or more of the following methods: 2 | 3 | 13 | 14 | Hoardy-Web produces dumps in a very simple, yet efficient, WRR file format (also on GitHub). 15 | 16 | Moreover, Hoardy-Web implements: 17 | 18 | 26 | 27 | In other words, this extension implements an in-browser half of your own personal private passive Wayback Machine that archives everything you see, including HTTP POST requests and responses (e.g. answer pages of web search engines), as well as most other HTTP-level data (AJAX, JSON RPC, etc). 28 | 29 | For more information see project’s documentation (also on GitHub) and extension’s Help page (also on GitHub) (also distributed with the extension itself, available via the “Help” button from its popup UI), especially the “Frequently Asked Questions” section there (also on GitHub). 30 | 31 | Also, note that: 32 | 33 | 38 | 39 | Hoardy-Web was previously known as “Personal Private Passive Web Archive” aka pWebArc. 40 | 41 | -------------------------------------------------------------------------------- /extension/background/capture-snapshot.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-2025 Jan Malakhovski 3 | * 4 | * This file is a part of `hoardy-web` project. 5 | * 6 | * This program is free software: you can redistribute it and/or modify 7 | * it under the terms of the GNU General Public License as published by 8 | * the Free Software Foundation, either version 3 of the License, or 9 | * (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program. If not, see . 18 | */ 19 | 20 | /* 21 | * Capture of DOM snapshots. 22 | */ 23 | 24 | "use strict"; 25 | 26 | async function snapshotOneTab(tabId, url) { 27 | if (config.debugRuntime) 28 | console.log("taking DOM snapshot of tab", tabId, url); 29 | 30 | let start = Date.now(); 31 | let allErrors = []; 32 | 33 | try { 34 | let allResults = await browser.tabs.executeScript(tabId, { 35 | file: "/inject/snapshot.js", 36 | allFrames: true, 37 | }); 38 | 39 | if (config.debugRuntime) 40 | console.log("snapshot.js returned", allResults); 41 | 42 | let emit = Date.now(); 43 | 44 | for (let data of allResults) { 45 | if (data === undefined) { 46 | allErrors.push("access denied"); 47 | continue; 48 | } 49 | 50 | let [date, documentUrl, originUrl, url, ct, result, errors] = data; 51 | 52 | if (!config.snapshotAny && isBoringOrServerURL(url)) { 53 | // skip stuff like handleBeforeRequest does, again, now for 54 | // sub-frames 55 | if (config.debugRuntime) 56 | console.log("NOT taking DOM snapshot of sub-frame of tab", tabId, url); 57 | continue; 58 | } else if (errors.length > 0) { 59 | allErrors.push(errors.join("; ")); 60 | continue; 61 | } else if (typeof result !== "string") { 62 | allErrors.push(`failed to snapshot a frame with \`${ct}\` content type`); 63 | continue; 64 | } 65 | 66 | let reqres = { 67 | sessionId, 68 | requestId: undefined, 69 | tabId, 70 | fromExtension: false, 71 | 72 | protocol: "SNAPSHOT", 73 | method: "DOM", 74 | url, 75 | 76 | documentUrl, 77 | originUrl, 78 | 79 | errors: [], 80 | 81 | requestTimeStamp: start, 82 | requestHeaders: [], 83 | requestBody: new ChunkedBuffer(), 84 | requestComplete: true, 85 | 86 | submitted: false, 87 | responded: true, 88 | fromCache: false, 89 | 90 | responseTimeStamp: date, 91 | responseHeaders : [ 92 | { name: "Content-Type", value: ct } 93 | ], 94 | responseBody: result, 95 | responseComplete: true, 96 | 97 | statusCode: 200, 98 | reason: "OK", 99 | 100 | emitTimeStamp: emit, 101 | }; 102 | 103 | reqresAlmostDone.push(reqres); 104 | } 105 | } catch (err) { 106 | allErrors.push(errorMessageOf(err)); 107 | } finally { 108 | if (allErrors.length > 0) 109 | await browser.notifications.create(`error-snapshot-${tabId}`, { 110 | title: "Hoardy-Web: ERROR", 111 | message: escapeNotification(config, `While taking DOM snapshot of tab #${tabId} (${url.substr(0, 80)}):\n- ${allErrors.join("\n- ")}`), 112 | iconUrl: iconURL("error", 128), 113 | type: "basic", 114 | }).catch(logError); 115 | } 116 | } 117 | 118 | async function snapshot(tabIdNull) { 119 | let tabs; 120 | if (tabIdNull === null) 121 | tabs = await browser.tabs.query({}); 122 | else { 123 | let tab = await browser.tabs.get(tabIdNull); 124 | tabs = [ tab ]; 125 | } 126 | 127 | for (let tab of tabs) { 128 | let tabId = tab.id; 129 | let tabcfg = getOriginConfig(tabId); 130 | let url = getTabURL(tab); 131 | if (tabIdNull === null && !tabcfg.snapshottable 132 | || !config.snapshotAny && isBoringOrServerURL(url)) { 133 | if (config.debugRuntime) 134 | console.log("NOT taking DOM snapshot of tab", tabId, url); 135 | continue; 136 | } 137 | await snapshotOneTab(tabId, url); 138 | } 139 | 140 | scheduleEndgame(tabIdNull); 141 | } 142 | -------------------------------------------------------------------------------- /extension/background/issue-acc.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-2025 Jan Malakhovski 3 | * 4 | * This file is a part of `hoardy-web` project. 5 | * 6 | * This program is free software: you can redistribute it and/or modify 7 | * it under the terms of the GNU General Public License as published by 8 | * the Free Software Foundation, either version 3 of the License, or 9 | * (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program. If not, see . 18 | */ 19 | 20 | /* 21 | * Issue accumulators. 22 | */ 23 | 24 | "use strict"; 25 | 26 | function newIssueAcc(callback) { 27 | return [new Set(), new Map(), callback]; 28 | } 29 | 30 | function getByReasonMapRecord(byReasonMap, reason) { 31 | return cacheSingleton(byReasonMap, reason, () => { return { 32 | recoverable: true, 33 | queue: [], 34 | size: 0, 35 | }; }); 36 | } 37 | 38 | function pushToByReasonRecord(v, recoverable, archivable) { 39 | v.when = Date.now(); 40 | v.recoverable = v.recoverable && recoverable; 41 | v.queue.push(archivable); 42 | v.size += archivable[0].dumpSize || 0; 43 | } 44 | 45 | function pushManyToSetByReasonRecord(set, v, recoverable, archivables) { 46 | for (let archivable of archivables) { 47 | set.add(archivable); 48 | pushToByReasonRecord(v, recoverable, archivable); 49 | } 50 | } 51 | 52 | function pushToByReasonMap(byReasonMap, reason, recoverable, archivable) { 53 | let v = getByReasonMapRecord(byReasonMap, reason); 54 | pushToByReasonRecord(v, recoverable, archivable); 55 | return v; 56 | } 57 | 58 | function pushToIssueAcc(accumulator, reason, recoverable, archivable) { 59 | accumulator[0].add(archivable); 60 | pushToByReasonMap(accumulator[1], reason, recoverable, archivable); 61 | if (accumulator[2] !== undefined) 62 | accumulator[2](recoverable); 63 | } 64 | -------------------------------------------------------------------------------- /extension/background/main.template: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Hoardy-Web: Background Page 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /extension/background/reload.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-2025 Jan Malakhovski 3 | * 4 | * This file is a part of `hoardy-web` project. 5 | * 6 | * This program is free software: you can redistribute it and/or modify 7 | * it under the terms of the GNU General Public License as published by 8 | * the Free Software Foundation, either version 3 of the License, or 9 | * (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program. If not, see . 18 | */ 19 | 20 | /* 21 | * Self-reload support. 22 | */ 23 | 24 | "use strict"; 25 | 26 | let wantReloadSelf = false; 27 | 28 | async function performReloadSelf() { 29 | if (!wantReloadSelf) 30 | return; 31 | 32 | let notGood 33 | = reqresErroredIssueAcc[0].size 34 | + reqresUnstashedIssueAcc[0].size; 35 | //+ reqresUnarchivedIssueAcc[0].size // these will be caught below 36 | 37 | if (notGood !== 0) { 38 | browser.notifications.create("error-noReload", { 39 | title: "Hoardy-Web: ERROR", 40 | message: escapeNotification(config, `\`Hoardy-Web\` can NOT be reloaded while some \`unstashed\` and/or \`errored\` reqres are present.`), 41 | iconUrl: iconURL("error", 128), 42 | type: "basic", 43 | }).catch(logError); 44 | 45 | wantReloadSelf = false; 46 | return; 47 | } 48 | 49 | let notDoneReqres 50 | = reqresInFlight.size 51 | + debugReqresInFlight.size 52 | + reqresFinishingUp.length 53 | + debugReqresFinishingUp.length 54 | + reqresAlmostDone.length 55 | + reqresBundledAs.size; 56 | 57 | let notDoneTasks 58 | = synchronousClosures.length 59 | + runningActions.size 60 | + scheduledCancelable.size 61 | // scheduledRetry is ignored here 62 | + scheduledDelayed.size 63 | + scheduledSaveState.size 64 | + scheduledInternal.size; 65 | // scheduledHidden is ignored here; 66 | 67 | function isInSyncWithLS(archivable) { 68 | let [loggable, dump] = archivable; 69 | return loggable.inLS !== undefined && !loggable.dirty; 70 | } 71 | 72 | let allInSyncWithLS 73 | = reqresLimbo.every(isInSyncWithLS) 74 | && reqresQueue.every(isInSyncWithLS) 75 | && Array.from(reqresUnarchivedIssueAcc[0]).every(isInSyncWithLS); 76 | 77 | let reloadAllowed 78 | = notDoneReqres === 0 79 | && notDoneTasks === 0 80 | && allInSyncWithLS; 81 | 82 | if (!reloadAllowed) { 83 | let stats = getStats() 84 | console.warn("reload blocked,", 85 | "#reqres", notDoneReqres, 86 | "running", stats.running_actions, 87 | "scheduled", stats.scheduled_actions, 88 | "LS?", allInSyncWithLS); 89 | return; 90 | } 91 | 92 | console.warn("reloading!"); 93 | 94 | let tabs = {}; 95 | let currentTabs = await browser.tabs.query({}); 96 | 97 | for (let tab of currentTabs) { 98 | let tabId = tab.id; 99 | tabs[tabId] = { 100 | url: getTabURL(tab), 101 | tabcfg: tabConfig.get(tabId), 102 | }; 103 | } 104 | 105 | let session = { 106 | id: sessionId, 107 | tabs, 108 | log: reqresLog, 109 | // queue and others are stashed 110 | }; 111 | 112 | await browser.storage.local.set({ session }); 113 | 114 | if (useDebugger && currentTabs.every((tab) => tab.url === "about:blank" || isExtensionURL(tab.url))) 115 | // Chromium will close all such tabs on extension reload, meaning, in 116 | // this case, the whole browser window will close 117 | await browser.tabs.create({ url: "chrome://extensions/" }); 118 | 119 | browser.runtime.reload(); 120 | } 121 | 122 | function reloadSelf() { 123 | wantReloadSelf = true; 124 | syncStashAll(true); 125 | syncRunActions(); 126 | scheduleEndgame(null); 127 | } 128 | 129 | function cancelReloadSelf() { 130 | wantReloadSelf = false; 131 | scheduleEndgame(null); 132 | } 133 | -------------------------------------------------------------------------------- /extension/background/state-tab.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-2025 Jan Malakhovski 3 | * 4 | * This file is a part of `hoardy-web` project. 5 | * 6 | * This program is free software: you can redistribute it and/or modify 7 | * it under the terms of the GNU General Public License as published by 8 | * the Free Software Foundation, either version 3 of the License, or 9 | * (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program. If not, see . 18 | */ 19 | 20 | /* 21 | * Per-tab/origin config and stats. 22 | */ 23 | 24 | "use strict"; 25 | 26 | // per-tab config 27 | let tabConfig = new Map(); 28 | 29 | // per-tab state 30 | let tabStateDefaults = { 31 | problematicTotal: 0, 32 | pickedTotal: 0, 33 | droppedTotal: 0, 34 | inLimboTotal: 0, 35 | inLimboSize: 0, 36 | collectedTotal: 0, 37 | collectedSize: 0, 38 | discardedTotal: 0, 39 | discardedSize: 0, 40 | }; 41 | 42 | // per-source globals.pickedTotal, globals.droppedTotal, etc 43 | let tabState = new Map(); 44 | 45 | function getOriginState(tabId, fromExtension) { 46 | // NB: not tracking extensions separately here, unlike with configs 47 | if (fromExtension) 48 | tabId = -1; 49 | return cacheSingleton(tabState, tabId, () => assignRec({}, tabStateDefaults)); 50 | } 51 | 52 | function prefillChildren(data) { 53 | return assignRec({ 54 | children: assignRec({}, data), 55 | }, data); 56 | } 57 | 58 | function getOriginConfig(tabId, fromExtension) { 59 | if (fromExtension) 60 | return prefillChildren(config.extension); 61 | else if (tabId == -1) 62 | return prefillChildren(config.background); 63 | else if (tabId === null) 64 | return prefillChildren(config.root); 65 | else 66 | return cacheSingleton(tabConfig, tabId, () => prefillChildren(config.root)); 67 | } 68 | 69 | function setTabConfigInternal(tabId, tabcfg) { 70 | if (tabcfg.children !== undefined && !tabcfg.children.bucket) 71 | tabcfg.children.bucket = getFirstOk(tabcfg.bucket, config.root.bucket, configDefaults.root.bucket); 72 | if (!tabcfg.bucket) 73 | tabcfg.bucket = getFirstOk(config.root.bucket, configDefaults.root.bucket); 74 | 75 | tabConfig.set(tabId, tabcfg); 76 | } 77 | 78 | function setTabConfig(tabId, tabcfg) { 79 | setTabConfigInternal(tabId, tabcfg); 80 | 81 | broadcastToPopup("updateTabConfig", tabId, tabcfg); 82 | 83 | if (useDebugger) { 84 | // Chromium does not provide `browser.menus.onShown` event 85 | updateMenu(tabcfg); 86 | syncDebuggersState(); 87 | } 88 | 89 | scheduleUpdateDisplay(false, tabId); 90 | } 91 | 92 | // collect all tabs referenced in not yet archived reqres 93 | function getUsedTabs() { 94 | let usedTabs = new Set(); 95 | for (let [k, v] of reqresInFlight.entries()) 96 | usedTabs.add(v.tabId); 97 | for (let [k, v] of debugReqresInFlight.entries()) 98 | usedTabs.add(v.tabId); 99 | for (let v of reqresFinishingUp) 100 | usedTabs.add(v.tabId); 101 | for (let v of debugReqresFinishingUp) 102 | usedTabs.add(v.tabId); 103 | for (let v of reqresAlmostDone) 104 | usedTabs.add(v.tabId); 105 | for (let [v, _x] of reqresProblematic) 106 | usedTabs.add(v.tabId); 107 | for (let [v, _x] of reqresLimbo) 108 | usedTabs.add(v.tabId); 109 | for (let [v, _x] of reqresQueue) 110 | usedTabs.add(v.tabId); 111 | for (let v of reqresErroredIssueAcc[0]) 112 | usedTabs.add(v[0].tabId); 113 | for (let v of reqresUnstashedIssueAcc[0]) 114 | usedTabs.add(v[0].tabId); 115 | for (let v of reqresUnarchivedIssueAcc[0]) 116 | usedTabs.add(v[0].tabId); 117 | 118 | return usedTabs; 119 | } 120 | 121 | // Free unused `tabConfig` and `tabState` structures. 122 | function cleanupTabs() { 123 | let usedTabs = getUsedTabs(); 124 | 125 | // delete configs of closed and unused tabs 126 | for (let tabId of Array.from(tabConfig.keys())) { 127 | if(tabId === -1 || openTabs.has(tabId) || usedTabs.has(tabId)) 128 | continue; 129 | if (config.debugRuntime) 130 | console.log("removing config of tab", tabId); 131 | tabConfig.delete(tabId); 132 | tabState.delete(tabId); 133 | } 134 | 135 | // delete any stale leftovers from tabState 136 | for (let tabId of Array.from(tabState.keys())) { 137 | if(tabId === -1 || openTabs.has(tabId) || usedTabs.has(tabId)) 138 | continue; 139 | console.warn("removing stale tab state", tabId); 140 | tabState.delete(tabId); 141 | } 142 | } 143 | 144 | // Tracking open tabs and generating their configs. 145 | 146 | let openTabs = new Set(); 147 | let negateConfigFor = new Set(); 148 | let negateOpenerTabIds = []; 149 | 150 | function processNewTab(tabId, openerTabId) { 151 | openTabs.add(tabId); 152 | 153 | if (useDebugger && openerTabId === undefined && negateOpenerTabIds.length > 0) 154 | // On Chromium, `browser.tabs.create` with `openerTabId` specified 155 | // does not pass it into `openerTabId` of `handleTabCreated` (it's a 156 | // bug), so we have to work around it by using `negateOpenerTabIds` 157 | // variable. 158 | openerTabId = negateOpenerTabIds.shift(); 159 | 160 | let openercfg = getOriginConfig(openerTabId !== undefined ? openerTabId : null); 161 | 162 | let children = openercfg.children; 163 | if (openerTabId !== undefined && negateConfigFor.delete(openerTabId)) { 164 | // Negate children.collecting when `openerTabId` is in `negateConfigFor`. 165 | children = assignRec({}, openercfg.children); 166 | children.collecting = !children.collecting; 167 | } 168 | 169 | let tabcfg = prefillChildren(children); 170 | tabConfig.set(tabId, tabcfg); 171 | 172 | scheduleUpdateDisplay(false, tabId); 173 | 174 | return tabcfg; 175 | } 176 | 177 | function processRemoveTab(tabId) { 178 | openTabs.delete(tabId); 179 | 180 | let updatedTabId = stopInFlight(tabId, "capture::EMIT_FORCED::BY_CLOSED_TAB"); 181 | 182 | scheduleCleanupAfterTab(tabId); 183 | 184 | scheduleEndgame(updatedTabId); 185 | } 186 | 187 | function resetTabConfigWorkOffline(tabcfg, url) { 188 | if (config.workOfflineFile && url.startsWith("file:") 189 | || config.workOfflineReplay && isServerURL(url) 190 | || config.workOfflineData && url.startsWith("data:")) { 191 | if (!tabcfg.workOffline) { 192 | tabcfg.workOffline = true; 193 | inheritTabConfigWorkOffline(config, tabcfg, tabcfg.children); 194 | return true; 195 | } 196 | } 197 | return false; 198 | } 199 | -------------------------------------------------------------------------------- /extension/bin/crx.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Purpose: Pack a Chromium extension directory into crx format 3 | # based on https://stackoverflow.com/questions/18693962/pack-chrome-extension-on-server-with-only-command-line-interface 4 | 5 | set -e 6 | 7 | if [[ $# -ne 2 ]]; then 8 | echo "Usage: $0 " 9 | exit 1 10 | fi 11 | 12 | output=$1 13 | key=$2 14 | crx="$output.crx" 15 | pub="$output.pub" 16 | sig="$output.sig" 17 | zip="$output.crxzip" 18 | tosign="$output.presig" 19 | binary_crx_id="$output.crxid" 20 | trap 'rm -f "$pub" "$sig" "$zip" "$tosign" "$binary_crx_id"' EXIT 21 | 22 | # zip up the crx dir 23 | zip -qr -9 -X "$zip" . 24 | 25 | #extract crx id 26 | openssl rsa -in "$key" -pubout -outform der 2>/dev/null | openssl dgst -sha256 -binary -out "$binary_crx_id" 27 | truncate -s 16 "$binary_crx_id" 28 | 29 | #generate file to sign 30 | { 31 | # echo "$crmagic_hex $version_hex $header_length $pub_len_hex $sig_len_hex" 32 | printf "CRX3 SignedData" 33 | echo "00 12 00 00 00 0A 10" | xxd -r -p 34 | cat "$binary_crx_id" "$zip" 35 | } > "$tosign" 36 | 37 | # signature 38 | openssl dgst -sha256 -binary -sign "$key" < "$tosign" > "$sig" 39 | 40 | # public key 41 | openssl rsa -pubout -outform DER < "$key" > "$pub" 2>/dev/null 42 | 43 | crmagic_hex="43 72 32 34" # Cr24 44 | version_hex="03 00 00 00" # 3 45 | header_length="45 02 00 00" 46 | header_chunk_1="12 AC 04 0A A6 02" 47 | header_chunk_2="12 80 02" 48 | header_chunk_3="82 F1 04 12 0A 10" 49 | { 50 | echo "$crmagic_hex $version_hex $header_length $header_chunk_1" | xxd -r -p 51 | cat "$pub" 52 | echo "$header_chunk_2" | xxd -r -p 53 | cat "$sig" 54 | echo "$header_chunk_3" | xxd -r -p 55 | cat "$binary_crx_id" "$zip" 56 | } > "$crx" 57 | -------------------------------------------------------------------------------- /extension/bin/gen-chromium-key.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Purpose: generate Chromium key in PEM format and generate a JSON with its public key 3 | # based on https://stackoverflow.com/questions/37317779/making-a-unique-extension-id-and-key-for-chrome-extension 4 | 5 | set -e 6 | 7 | key=${1:-chromium.key.pem} 8 | output=${2:-manifest-chromium} 9 | 10 | if [[ ! -e "$key" ]]; then 11 | echo "Generating a new Chromium key!" 12 | openssl genrsa 2048 | openssl pkcs8 -topk8 -nocrypt -out "$key" 13 | fi 14 | 15 | { 16 | echo "{" 17 | echo -n '"key": "' 18 | openssl rsa -in "$key" -pubout -outform DER 2>/dev/null | openssl base64 -A 19 | echo '"' 20 | echo "}" 21 | } > "$output-key.json" 22 | 23 | openssl rsa -in "$key" -pubout -outform DER 2>/dev/null | sha256sum | head -c32 | tr 0-9a-f a-p > "$output-id.txt" 24 | -------------------------------------------------------------------------------- /extension/default.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import {} 2 | , lib ? pkgs.lib 3 | , source ? import ../source.nix { inherit pkgs; } 4 | , developer ? false 5 | }: 6 | 7 | with pkgs; 8 | 9 | stdenv.mkDerivation rec { 10 | pname = "hoardy-web-extension"; 11 | version = "1.21.1"; 12 | 13 | inherit (source) src unpackPhase; 14 | sourceRoot = "${src.name}/extension"; 15 | 16 | nativeBuildInputs = [ git jq pandoc zip imagemagick vim.xxd ]; 17 | 18 | buildPhase = '' 19 | ./build.sh clean firefox-mv2 chromium-mv2 20 | ''; 21 | 22 | installPhase = '' 23 | mkdir -p $out 24 | git archive --format tar.gz -o $out/Hoardy-Web-source-v${version}.tar.gz extension-v${version} 25 | cd dist 26 | cp -at $out *.xpi *.zip *.crx 27 | ''; 28 | } 29 | -------------------------------------------------------------------------------- /extension/gupdate.xml.template: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /extension/icon/orbitals/archiving.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 30 | 36 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /extension/icon/orbitals/error.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 30 | 36 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /extension/icon/orbitals/idle.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 30 | 36 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /extension/icon/orbitals/main.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 30 | 36 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /extension/icon/orbitals/off.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 30 | 36 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /extension/icon/orbitals/tracking.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 30 | 36 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /extension/icon/privateer/archiving.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 33 | 39 | 43 | 44 | -------------------------------------------------------------------------------- /extension/icon/privateer/bar.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 34 | 35 | -------------------------------------------------------------------------------- /extension/icon/privateer/bothlimbo.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 33 | 39 | 43 | 49 | 50 | -------------------------------------------------------------------------------- /extension/icon/privateer/dot.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 34 | 35 | -------------------------------------------------------------------------------- /extension/icon/privateer/error.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 33 | 39 | 43 | 44 | -------------------------------------------------------------------------------- /extension/icon/privateer/failed.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 33 | 39 | 40 | -------------------------------------------------------------------------------- /extension/icon/privateer/idle.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 33 | 39 | 45 | 46 | -------------------------------------------------------------------------------- /extension/icon/privateer/in_limbo.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 33 | 39 | 45 | 46 | -------------------------------------------------------------------------------- /extension/icon/privateer/limbo.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 33 | 39 | 45 | 46 | -------------------------------------------------------------------------------- /extension/icon/privateer/neglimbo.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 33 | 39 | 45 | 46 | -------------------------------------------------------------------------------- /extension/icon/privateer/off.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 33 | 39 | 40 | -------------------------------------------------------------------------------- /extension/icon/privateer/problematic.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 31 | 35 | 39 | 43 | 47 | 48 | -------------------------------------------------------------------------------- /extension/icon/privateer/tracking.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 31 | 38 | 42 | 49 | 53 | 57 | 61 | 65 | 69 | 73 | 74 | -------------------------------------------------------------------------------- /extension/icon/privateer/unreplayable.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 33 | 37 | 41 | 45 | 46 | -------------------------------------------------------------------------------- /extension/icon/privateer/unsnapshottable.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 33 | 38 | 42 | 48 | 49 | -------------------------------------------------------------------------------- /extension/icon/privateer/work_offline.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 33 | 39 | 45 | 46 | -------------------------------------------------------------------------------- /extension/inject/snapshot.js: -------------------------------------------------------------------------------- 1 | /* 2 | * An injected script that does HTML DOM snapshotting. 3 | * 4 | * Copyright (c) 2024 Jan Malakhovski 5 | * 6 | * This file is a part of `hoardy-web` project. 7 | * 8 | * This program is free software: you can redistribute it and/or modify 9 | * it under the terms of the GNU General Public License as published by 10 | * the Free Software Foundation, either version 3 of the License, or 11 | * (at your option) any later version. 12 | * 13 | * This program is distributed in the hope that it will be useful, 14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | * GNU General Public License for more details. 17 | * 18 | * You should have received a copy of the GNU General Public License 19 | * along with this program. If not, see . 20 | */ 21 | 22 | "use strict"; 23 | 24 | (() => { 25 | let now = Date.now(); 26 | let errors = []; 27 | let result = null; 28 | let ct = document.contentType; 29 | 30 | if (document instanceof HTMLDocument && ct === "text/html" 31 | || document instanceof XMLDocument && ct === "image/svg+xml") { 32 | ct = `${ct}; charset=${document.characterSet}`; 33 | 34 | let gotDocType = false; 35 | let cres = []; 36 | for (let c of document.childNodes) { 37 | if (c instanceof DocumentType && c.name === "html") { 38 | if (gotDocType) 39 | errors.push("multiple doctypes"); 40 | gotDocType = true; 41 | 42 | cres.push(""); 43 | } else if (c instanceof HTMLHtmlElement) 44 | cres.push(c.outerHTML); 45 | else if (c instanceof SVGSVGElement) { 46 | if (gotDocType) 47 | errors.push("multiple doctypes"); 48 | gotDocType = true; 49 | 50 | cres.push(``) 51 | cres.push(c.outerHTML); 52 | } else 53 | errors.push(`unknown child element type ${c.toString()}`); 54 | } 55 | 56 | result = cres.join("\n"); 57 | } else 58 | errors.push(`snapshotting of frames with \`${ct}\` content type is not implemented`); 59 | 60 | return [now, document.documentURI, document.referrer, document.URL, ct, result, errors]; 61 | })(); 62 | -------------------------------------------------------------------------------- /extension/lib/caydarsc.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-2025 Jan Malakhovski 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to deal 6 | * in the Software without restriction, including without limitation the rights 7 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | * copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | * SOFTWARE. 21 | */ 22 | 23 | /* 24 | * "Chromium, Attach Your Debugger and Atomically Run these Send Command(s)." 25 | * 26 | * Chromium's Debugger API is incredibly hard to use: 27 | * 28 | * - debugger can detach, making all subsequent API calls start failing, even 29 | * while you are configuring it (!), requiring you retry from the beginning; 30 | * 31 | * - double-attachment will not work, so if you want to attach from concurrent 32 | * tasks, you'll have to track your attachment and configuration progress 33 | * yourself, and then generate and join 34 | * attach-and-configure-the-debugger-for-my-debugee `Promise`s. 35 | * 36 | * This tiny wrapper solves all of the above, making Chromium's Debugger API 37 | * actually usable. 38 | * 39 | * Depends on `./base.js`. 40 | */ 41 | 42 | "use strict"; 43 | 44 | // Set to enable debugging. 45 | let DEBUG_CAYDARSC = false; 46 | 47 | async function attachDebuggerWithSendCommandsUnsafe(tabId, version, commands, pre, post) { 48 | let debuggee = { tabId }; 49 | 50 | let lastError = undefined; 51 | let retry = 0; 52 | for (; retry < 10; ++retry) { 53 | if (pre !== undefined) 54 | pre(tabId, retry); 55 | 56 | try { 57 | await chrome.debugger.attach(debuggee, version); 58 | } catch (err) { 59 | lastError = err; 60 | if (typeof err !== "string") 61 | throw err; 62 | else if (err === "Cannot access a chrome:// URL" 63 | || err.startsWith("Cannot access contents of url")) 64 | throw err; 65 | else if (!err.startsWith("Another debugger is already attached to the tab with id:")) 66 | throw err; 67 | // otherwise, continue as normal 68 | } 69 | 70 | try { 71 | for (let args of commands) 72 | await chrome.debugger.sendCommand(debuggee, ...args); 73 | } catch (err) { 74 | // this could happen if the debugger gets detached immediately 75 | // after it gets attached, so we retry again 76 | lastError = err; 77 | await sleep(100); 78 | continue; 79 | } 80 | 81 | lastError = undefined; 82 | break; 83 | } 84 | 85 | if (lastError !== undefined) 86 | throw lastError; 87 | 88 | if (DEBUG_CAYDARSC) 89 | console.debug("CAYDARSC: attached debugger to tab", tabId, "on retry", retry); 90 | 91 | if (post !== undefined) 92 | post(tabId, retry); 93 | } 94 | 95 | // Tabs we are debugging. 96 | let tabsDebugging = new Set(); 97 | // Tabs we are attaching the debugger to. 98 | let tabsAttaching = new Map(); 99 | 100 | function attachDebuggerWithSendCommands(tabId, version, commands, pre, post) { 101 | if (tabsDebugging.has(tabId)) 102 | // nothing to do 103 | return; 104 | 105 | // NB: self-destructing using `tabsAttaching.delete` 106 | return cacheSingleton(tabsAttaching, tabId, 107 | (tabId) => attachDebuggerWithSendCommandsUnsafe(tabId, version, commands, pre, post) 108 | .then(() => tabsDebugging.add(tabId)) 109 | .finally(() => tabsAttaching.delete(tabId))); 110 | } 111 | 112 | function detachDebugger(tabId, post) { 113 | let debuggee = { tabId }; 114 | return chrome.debugger.detach(debuggee).then(() => { 115 | tabsDebugging.delete(tabId); 116 | 117 | if (DEBUG_CAYDARSC) 118 | console.debug("CAYDARSC: detached debugger from tab", tabId); 119 | 120 | if (post !== undefined) 121 | post(tabId); 122 | }); 123 | } 124 | -------------------------------------------------------------------------------- /extension/lib/compat.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-2025 Jan Malakhovski 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to deal 6 | * in the Software without restriction, including without limitation the rights 7 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | * copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | * SOFTWARE. 21 | */ 22 | 23 | /* 24 | * A tiny compatibility layer converting Chromium's Manifest V2 WebExtension 25 | * APIs to those compatible with Firfeox, plus definitions of some constants 26 | * describing available browser features. 27 | * 28 | * (Though, here, both are only done for the parts `Hoardy-Web` uses, to 29 | * minimize deployment of unused code. But, if you want to borrow and reuse 30 | * this code, you can implement other parts by folloing the same patterns 31 | * below.) 32 | * 33 | */ 34 | 35 | "use strict"; 36 | 37 | function parseUA() { 38 | let result = null; 39 | let UA = window.navigator.userAgent; 40 | for (let e of UA.split(" ")) { 41 | if (e.startsWith("Firefox/")) { 42 | result = e; 43 | break; 44 | } else if (e.startsWith("Chrome/")) { 45 | result = e; 46 | break; 47 | } 48 | } 49 | if (result === null) 50 | throw new Error("unknown/unsupported User-Agent: " + UA); 51 | return result; 52 | } 53 | 54 | function makePromiseAPIConst(data) { 55 | return () => { 56 | return new Promise((resolve, reject) => { 57 | resolve(data); 58 | }); 59 | }; 60 | } 61 | 62 | function makePromiseAPI0(old, nthis) { 63 | return () => { 64 | return new Promise((resolve, reject) => { 65 | old.apply(nthis, [(data) => { 66 | if (browser.runtime.lastError === undefined) 67 | resolve(data); 68 | else { 69 | reject(browser.runtime.lastError.message); 70 | } 71 | }]); 72 | }); 73 | }; 74 | } 75 | 76 | function makePromiseAPI(old, nthis) { 77 | return (arg) => { 78 | return new Promise((resolve, reject) => { 79 | old.apply(nthis, [arg, (data) => { 80 | if (browser.runtime.lastError === undefined) 81 | resolve(data); 82 | else { 83 | reject(browser.runtime.lastError.message); 84 | } 85 | }]); 86 | }); 87 | }; 88 | } 89 | 90 | function makePromiseAPI2(old, nthis) { 91 | return (arg1, arg2) => { 92 | return new Promise((resolve, reject) => { 93 | old.apply(nthis, [arg1, arg2, (data) => { 94 | if (browser.runtime.lastError === undefined) 95 | resolve(data); 96 | else { 97 | reject(browser.runtime.lastError.message); 98 | } 99 | }]); 100 | }); 101 | }; 102 | } 103 | 104 | function makePromiseAPI3(old, nthis) { 105 | return (arg1, arg2, arg3) => { 106 | return new Promise((resolve, reject) => { 107 | old.apply(nthis, [arg1, arg2, arg3, (data) => { 108 | if (browser.runtime.lastError === undefined) 109 | resolve(data); 110 | else { 111 | reject(browser.runtime.lastError.message); 112 | } 113 | }]); 114 | }); 115 | }; 116 | } 117 | 118 | function makeFirefoxish(browser) { 119 | // Okay, so the probem here is that with manifest V3 nothing works, but 120 | // with manifest V2 Chromium requires the use of callback-based APIs 121 | // instead of Promise-based ones, which is annoying. 122 | // 123 | // So we wrap the functions we use and turn them into Promise-based ones. 124 | 125 | browser.browserAction.setBadgeText = makePromiseAPI(browser.browserAction.setBadgeText); 126 | browser.browserAction.setIcon = makePromiseAPI(browser.browserAction.setIcon); 127 | browser.browserAction.setTitle = makePromiseAPI(browser.browserAction.setTitle); 128 | browser.browserAction.setBadgeTextColor = makePromiseAPIConst(undefined); 129 | // TODO on V3 do this instead: 130 | //browser.browserAction.setBadgeTextColor = makePromiseAPI(browser.action.setBadgeTextColor); 131 | browser.browserAction.setBadgeBackgroundColor = makePromiseAPI(browser.browserAction.setBadgeBackgroundColor); 132 | browser.commands.getAll = makePromiseAPI0(browser.commands.getAll); 133 | browser.notifications.clear = makePromiseAPI(browser.notifications.clear); 134 | browser.notifications.create = makePromiseAPI2(browser.notifications.create); 135 | browser.notifications.getAll = makePromiseAPI0(browser.notifications.getAll); 136 | browser.runtime.sendMessage = makePromiseAPI(browser.runtime.sendMessage); 137 | browser.tabs.create = makePromiseAPI(browser.tabs.create); 138 | browser.tabs.executeScript = makePromiseAPI2(browser.tabs.executeScript); 139 | browser.tabs.get = makePromiseAPI(browser.tabs.get); 140 | browser.tabs.query = makePromiseAPI(browser.tabs.query); 141 | browser.tabs.update = makePromiseAPI2(browser.tabs.update); 142 | browser.windows.create = makePromiseAPI(browser.windows.create); 143 | 144 | browser.menus = browser.contextMenus; 145 | browser.menus.create = makePromiseAPI(browser.contextMenus.create); 146 | browser.menus.update = makePromiseAPI2(browser.contextMenus.update); 147 | browser.menus.refresh = makePromiseAPI0(browser.contextMenus.refresh); 148 | 149 | let old_local = browser.storage.local; 150 | browser.storage.local = { 151 | clear: makePromiseAPI0(old_local.clear, old_local), 152 | get: makePromiseAPI(old_local.get, old_local), 153 | remove: makePromiseAPI(old_local.remove, old_local), 154 | set: makePromiseAPI(old_local.set, old_local), 155 | }; 156 | 157 | browser.debugger.attach = makePromiseAPI2(browser.debugger.attach); 158 | browser.debugger.detach = makePromiseAPI(browser.debugger.detach); 159 | browser.debugger.sendCommand = makePromiseAPI3(browser.debugger.sendCommand, browser.debugger); 160 | 161 | return browser; 162 | } 163 | 164 | var browser; 165 | if (browser === undefined) { 166 | browser = makeFirefoxish(chrome); 167 | } 168 | browser.nameVersion = parseUA(); 169 | 170 | let manifest = browser.runtime.getManifest(); 171 | let permissions = new Set(manifest.permissions); 172 | let isFirefox = browser.nameVersion.startsWith("Firefox/"); 173 | let useSVGIcons = isFirefox; // are SVG icons supported? 174 | let useDebugger = permissions.has("debugger"); 175 | let useBlocking = permissions.has("webRequestBlocking"); 176 | let isMobile = browser.menus === undefined || browser.commands === undefined; 177 | -------------------------------------------------------------------------------- /extension/lib/idbp.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024-2025 Jan Malakhovski 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to deal 6 | * in the Software without restriction, including without limitation the rights 7 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | * copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | * SOFTWARE. 21 | */ 22 | 23 | /* 24 | * A tiny Promise-based wrapper over `window.indexedDB` API. 25 | */ 26 | 27 | "use strict"; 28 | 29 | function idbRequestAsPromise(request) { 30 | return new Promise((resolve, reject) => { 31 | request.onerror = (event) => reject(event.target.error); 32 | request.onsuccess = (event) => resolve(event.target.result); 33 | }); 34 | } 35 | 36 | function idbFunctionAsPromise(old, nthis) { 37 | return (...args) => { 38 | return idbRequestAsPromise(old.apply(nthis, args)); 39 | }; 40 | } 41 | 42 | function idbStoreProxyPromise(obj) { 43 | return { 44 | count: idbFunctionAsPromise(obj.count, obj), 45 | get: idbFunctionAsPromise(obj.get, obj), 46 | getKey: idbFunctionAsPromise(obj.getKey, obj), 47 | getAll: idbFunctionAsPromise(obj.getAll, obj), 48 | getAllKeys: idbFunctionAsPromise(obj.getAllKeys, obj), 49 | add: idbFunctionAsPromise(obj.add, obj), 50 | put: idbFunctionAsPromise(obj.put, obj), 51 | delete: idbFunctionAsPromise(obj.delete, obj), 52 | 53 | autoIncrement: obj.autoIncrement, 54 | }; 55 | } 56 | 57 | function idbOpen(name, version, upgradeFunc) { 58 | return new Promise((resolve, reject) => { 59 | let request = window.indexedDB.open(name, version); 60 | request.onblocked = (event) => reject("blocked"); 61 | request.onerror = (event) => reject(event.target.error); 62 | request.onsuccess = (event) => resolve(event.target.result); 63 | if (upgradeFunc !== undefined) 64 | request.onupgradeneeded = (event) => { 65 | let db = event.target.result; 66 | try { 67 | upgradeFunc(db, event.oldVersion, event.newVersion); 68 | } catch (err) { 69 | reject(err); 70 | throw new Error("upgradeFunc threw an error, aborting indexdeDB upgrade"); 71 | } 72 | }; 73 | }); 74 | } 75 | 76 | function idbDelete(name) { 77 | return idbRequestAsPromise(window.indexedDB.deleteDatabase(name)); 78 | } 79 | 80 | function idbTransaction(db, mode, objectStoreNames, func) { 81 | return new Promise((resolve, reject) => { 82 | let transaction = db.transaction(objectStoreNames, mode); 83 | let result; 84 | let error; 85 | transaction.onerror = (event) => reject(error !== undefined ? error : event.target.error); 86 | transaction.oncomplete = (event) => { 87 | if (result instanceof Promise) 88 | result.then(resolve, reject); 89 | else 90 | resolve(result); 91 | }; 92 | let args = []; 93 | for (let n of objectStoreNames) 94 | args.push(idbStoreProxyPromise(transaction.objectStore(n))); 95 | try { 96 | result = func(transaction, ...args); 97 | } catch (err) { 98 | error = err; 99 | transaction.abort(); 100 | } 101 | if (result instanceof Promise) 102 | result.catch((err) => { 103 | error = err; 104 | transaction.abort(); 105 | throw err; 106 | }); 107 | }); 108 | } 109 | 110 | async function idbDump(db) { 111 | for (let name of db.objectStoreNames) { 112 | console.log("store", name); 113 | await idbTransaction(db, "readonly", [name], async (transaction, store) => { 114 | let keys = await store.getAllKeys(); 115 | console.log(name); 116 | for (let k of keys) { 117 | let v = await store.get(k); 118 | console.log("object", k, v); 119 | } 120 | }).catch(logError); 121 | } 122 | } 123 | 124 | async function idbExampleTest() { 125 | try { 126 | let db = await idbOpen("test", 1, (db, oldVersion, newVersion) => { 127 | db.createObjectStore("archive", { autoIncrement: true }); 128 | }); 129 | let res = await idbTransaction(db, "readwrite", ["archive"], async (transaction, archiveStore) => { 130 | let one = await archiveStore.add({data: "abc"}); 131 | let two = await archiveStore.add({data: new Uint8Array([1, 2, 3])}); 132 | //transaction.abort(); 133 | //throw new Error("bad"); 134 | return [one, two]; 135 | }); 136 | console.log("ok", res); 137 | db.close(); 138 | } catch(err) { 139 | logError(err); 140 | } finally { 141 | idbDelete("test"); 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /extension/lib/lslot.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024-2025 Jan Malakhovski 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to deal 6 | * in the Software without restriction, including without limitation the rights 7 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | * copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | * SOFTWARE. 21 | */ 22 | 23 | /* 24 | * Same API as `idbp.js`, but using `browser.storage.local` API instead. 25 | */ 26 | 27 | "use strict"; 28 | 29 | async function storageGetOne(storage, id) { 30 | let res = await storage.get([id]); 31 | return res[id]; 32 | } 33 | 34 | function getFromLocalStorage(id) { 35 | return storageGetOne(browser.storage.local, id); 36 | } 37 | 38 | class LSlotTransaction { 39 | constructor(storage) { 40 | this.storage = storage; 41 | this._toPut = {}; // what needs to be written back 42 | this._toDelete = new Set(); // what needs to be deleted 43 | } 44 | 45 | async get(id) { 46 | if (this._toDelete.has(id)) { 47 | console.error("trying to `get`", id, "after a `delete`"); 48 | throw new Error("trying to `get` after a `delete`"); 49 | } 50 | 51 | let res = await storageGetOne(this.storage, id); 52 | return res; 53 | } 54 | 55 | put(id, data) { 56 | this._toDelete.delete(id); 57 | this._toPut[id] = data; 58 | } 59 | 60 | delete(id) { 61 | this._toDelete.add(id); 62 | delete this._toPut[id]; 63 | } 64 | 65 | async commit() { 66 | await this.storage.set(this._toPut); 67 | if (this._toDelete.size > 0) 68 | await this.storage.remove(Array.from(this._toDelete)); 69 | } 70 | } 71 | 72 | function lslotMetaIdOf(prefix) { 73 | return `lsmeta-${prefix}`; 74 | } 75 | 76 | function lslotDataIdOf(prefix, slot) { 77 | return `lsdata-${prefix}-${slot}`; 78 | } 79 | 80 | async function lslotGetMeta(get, prefix) { 81 | let metaid = lslotMetaIdOf(prefix); 82 | let metaobj = await get(metaid); 83 | let meta = updateFromRec({ first: 0, next: 0 }, metaobj); 84 | return [metaid, meta]; 85 | } 86 | 87 | async function lslotGetSlot(get, prefix, slot) { 88 | let sid = lslotDataIdOf(prefix, slot); 89 | let resobj = await get(sid); 90 | return resobj; 91 | } 92 | 93 | async function lslotForEach(get, meta, func) { 94 | let next = meta.next; 95 | let end = next + 100; 96 | 97 | let first = meta.first; 98 | let cur = Math.max(0, first - 100); 99 | let isFirst = true; 100 | 101 | while (cur < end) { 102 | let el = await get(cur); 103 | if (el !== undefined) { 104 | if (isFirst) { 105 | first = cur; 106 | isFirst = false; 107 | } 108 | next = cur + 1; 109 | end = cur + 100; 110 | 111 | try { 112 | let res = func(el, cur); 113 | if (res instanceof Promise) 114 | await res; 115 | } catch(err) { 116 | // widen the range if an error happens 117 | if (first < meta.first) 118 | meta.first = first; 119 | if (next > meta.next) 120 | meta.next = next; 121 | throw err; 122 | } 123 | } 124 | cur += 1; 125 | } 126 | 127 | // widen or narrow the range if everything is ok 128 | meta.first = first; 129 | meta.next = next; 130 | } 131 | 132 | class LSlotObjectStore { 133 | constructor(transaction, name) { 134 | this.transaction = transaction; 135 | this.name = name; 136 | } 137 | 138 | getMeta() { 139 | let transaction = this.transaction; 140 | return lslotGetMeta((id) => transaction.get(id), this.name); 141 | } 142 | 143 | get(slot) { 144 | let transaction = this.transaction; 145 | return lslotGetSlot((id) => transaction.get(id), this.name, slot); 146 | } 147 | 148 | // find first empty slot 149 | async findEmptySlot(start) { 150 | while (true) { 151 | let sid = lslotDataIdOf(this.name, start); 152 | let res = await storageGetOne(this.transaction.storage, sid); 153 | if (res === undefined) 154 | return [start, sid]; 155 | start += 1; 156 | } 157 | //throw new Error("no empty local storage slots are available"); 158 | } 159 | 160 | async put(data, slot) { 161 | let [metaid, meta] = await this.getMeta(); 162 | if (slot === undefined) { 163 | let [id, sid] = await this.findEmptySlot(meta.next); 164 | meta.next = id + 1; 165 | this.transaction.put(metaid, meta); 166 | this.transaction.put(sid, data); 167 | return id; 168 | } else { 169 | let sid = lslotDataIdOf(this.name, slot); 170 | this.transaction.put(sid, data); 171 | return slot; 172 | } 173 | } 174 | 175 | async delete(slot) { 176 | let sid = lslotDataIdOf(this.name, slot); 177 | let [metaid, meta] = await this.getMeta(); 178 | 179 | if (slot === meta.first) { 180 | meta.first += 1; 181 | this.transaction.put(metaid, meta); 182 | } else if (slot === meta.next - 1) { 183 | meta.next -= 1; 184 | this.transaction.put(metaid, meta); 185 | } 186 | 187 | this.transaction.delete(sid); 188 | } 189 | 190 | async forEach(func) { 191 | let [metaid, meta] = await this.getMeta(); 192 | let firstBefore = meta.first; 193 | let nextBefore = meta.next; 194 | 195 | let transaction = this.transaction; 196 | let name = this.name; 197 | 198 | try { 199 | await lslotForEach((slot) => lslotGetSlot((id) => transaction.get(id), name, slot), meta, func); 200 | } finally { 201 | if (firstBefore !== meta.first || nextBefore !== meta.next) 202 | // record the update meta range 203 | this.transaction.put(metaid, meta); 204 | } 205 | } 206 | } 207 | 208 | async function lslotTransaction(storage, /* ignored */ mode, names, func) { 209 | let transaction = new LSlotTransaction(storage); 210 | 211 | let args = []; 212 | for (let n of names) 213 | args.push(new LSlotObjectStore(transaction, n)); 214 | 215 | let res = await func(transaction, ...args); 216 | await transaction.commit(); 217 | return res; 218 | } 219 | 220 | async function lslotDump() { 221 | let res = await browser.storage.local.get(); 222 | for (let [k, v] of Object.entries(res)) { 223 | if (k.startsWith("lsmeta-")) 224 | console.log("meta", k.substr(7), v); 225 | } 226 | for (let [k, v] of Object.entries(res)) { 227 | if (k.startsWith("lsdata-")) 228 | console.log("data", k.substr(7), v); 229 | } 230 | } 231 | -------------------------------------------------------------------------------- /extension/lib/pako-ext.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024-2025 Jan Malakhovski 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to deal 6 | * in the Software without restriction, including without limitation the rights 7 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | * copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | * SOFTWARE. 21 | */ 22 | 23 | /* 24 | * Wrappers over pako. 25 | */ 26 | 27 | // Deflate, and then return the compressed or the original imput, 28 | // depending on which is smaller. 29 | function deflateMaybe(input, options, errorHandler) { 30 | try { 31 | let compressed = pako.deflate(input, options); 32 | if (compressed.byteLength < input.byteLength) 33 | return compressed; 34 | } catch (err) { 35 | if (errorHandler !== undefined) 36 | errorHandler(err); 37 | else 38 | throw err; 39 | } 40 | return input; 41 | } 42 | 43 | // Do the reverse to `deflateMaybe`, checking the input for the GZip header. 44 | function inflateMaybe(input, options, errorHandler) { 45 | if (input[0] === 31 && input[1] === 139 /* GZip header */) { 46 | try { 47 | return pako.inflate(input, options); 48 | } catch (err) { 49 | if (errorHandler !== undefined) 50 | errorHandler(err); 51 | else 52 | throw err; 53 | } 54 | } 55 | return input; 56 | } 57 | 58 | // `pako.Deflate` which does not fattern the compressed chunks and 59 | // tracks the total size of the result. 60 | class DeflateInChunks extends pako.Deflate { 61 | constructor (options) { 62 | super(options); 63 | this.size = 0; 64 | } 65 | 66 | onData(chunk) { 67 | this.chunks.push(chunk); 68 | this.size += chunk.byteLength; 69 | } 70 | 71 | onEnd(status) { 72 | this.err = status; 73 | this.msg = this.strm.msg; 74 | } 75 | } 76 | 77 | // deflateChunks : [Uint8Array] -> DeflateOptions -> [[Uint8Array], int, int] 78 | function deflateChunks(inputChunks, options) { 79 | const deflate = new DeflateInChunks(options); 80 | let inputSize = 0; 81 | 82 | for (let chunk of inputChunks) { 83 | deflate.push(chunk, false); 84 | inputSize += chunk.byteLength; 85 | } 86 | deflate.push(new Uint8Array([]), true); 87 | 88 | if (deflate.err) 89 | throw deflate.msg; 90 | 91 | return [deflate.chunks, deflate.size, inputSize]; 92 | } 93 | 94 | function deflateChunksMaybe(inputChunkss, options, errorHandler) { 95 | try { 96 | let [compressedChunks, compressedSize, inputSize] = deflateChunks(inputChunkss, options); 97 | if (compressedSize < inputSize) 98 | return compressedChunks; 99 | } catch (err) { 100 | if (errorHandler !== undefined) 101 | errorHandler(err); 102 | else 103 | throw err; 104 | } 105 | return inputChunks; 106 | } 107 | -------------------------------------------------------------------------------- /extension/lib/webext-rpc-server.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-2025 Jan Malakhovski 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to deal 6 | * in the Software without restriction, including without limitation the rights 7 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | * copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | * SOFTWARE. 21 | */ 22 | 23 | /* 24 | * A tiny library for WebExtension server-side RPC handling. 25 | * 26 | * Depends on `./webext.js`. 27 | */ 28 | 29 | "use strict"; 30 | 31 | let WEBEXT_RPC_MODE = 0; 32 | 33 | // Set to enable debugging. 34 | let DEBUG_WEBEXT_RPC = false; 35 | 36 | // open client tab ports 37 | let webextRPCOpenPorts = new Map(); 38 | 39 | function webextRPCHandleConnect(port) { 40 | let portId; 41 | let url = normalizedURL(port.sender.url); 42 | if (useDebugger) { 43 | if (port.sender.tab !== undefined) 44 | portId = port.sender.tab.id; 45 | else 46 | portId = url; 47 | } else 48 | portId = port.sender.contextId; 49 | 50 | if (DEBUG_WEBEXT_RPC) 51 | console.debug("WEBEXT_RPC: port opened", portId, url); 52 | 53 | webextRPCOpenPorts.set(portId, {port, name: port.name, url}); 54 | port.onDisconnect.addListener(catchAll(() => { 55 | if (DEBUG_WEBEXT_RPC) 56 | console.debug("WEBEXT_RPC: port disconnected", portId, url); 57 | 58 | webextRPCOpenPorts.delete(portId); 59 | })); 60 | } 61 | 62 | function broadcastToMatching(lazy, predicate, ...args) { 63 | let res = args; 64 | let number = 0; 65 | for (let [portId, info] of webextRPCOpenPorts.entries()) { 66 | if (predicate === undefined || predicate(info)) { 67 | if (lazy) { 68 | res = evalFunctionsAway(res); 69 | lazy = false; 70 | } 71 | info.port.postMessage(res); 72 | number += 1; 73 | } 74 | } 75 | 76 | if (DEBUG_WEBEXT_RPC) 77 | console.debug("WEBEXT_RPC: broadcasted", args, "to", number, "recipients"); 78 | 79 | return [lazy, res]; 80 | } 81 | 82 | function broadcast(lazy, ...args) { 83 | return broadcastToMatching(lazy, undefined, ...args); 84 | } 85 | 86 | function broadcastToURL(lazy, url, ...args) { 87 | return broadcastToMatching(lazy, (info) => info.url === url, ...args); 88 | } 89 | 90 | function broadcastToURLPrefix(lazy, url, ...args) { 91 | return broadcastToMatching(lazy, (info) => info.url.startsWith(url), ...args); 92 | } 93 | 94 | function broadcastToName(lazy, name, ...args) { 95 | return broadcastToMatching(lazy, (info) => info.name === name, ...args); 96 | } 97 | 98 | function broadcastToNamePrefix(lazy, name, ...args) { 99 | return broadcastToMatching(lazy, (info) => info.name.startsWith(name), ...args); 100 | } 101 | 102 | let webextRPCFuncs = { 103 | broadcast, 104 | broadcastToURL, 105 | broadcastToURLPrefix, 106 | broadcastToName, 107 | broadcastToNamePrefix, 108 | }; 109 | 110 | function initWebextRPC(handleMessage) { 111 | browser.runtime.onMessage.addListener(catchAll((request, ...args) => { 112 | if (DEBUG_WEBEXT_RPC) 113 | console.debug("WEBEXT_RPC: message", request); 114 | 115 | let cmd = request[0]; 116 | let func = webextRPCFuncs[cmd]; 117 | if (func !== undefined) { 118 | func(false, ...(request.splice(1))); 119 | return; 120 | } 121 | 122 | handleMessage(request, ...args) 123 | })); 124 | browser.runtime.onConnect.addListener(catchAll(webextRPCHandleConnect)); 125 | } 126 | -------------------------------------------------------------------------------- /extension/lib/webext.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-2025 Jan Malakhovski 3 | * 4 | * Permission is hereby granted, free of charge, to any person obtaining a copy 5 | * of this software and associated documentation files (the "Software"), to deal 6 | * in the Software without restriction, including without limitation the rights 7 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | * copies of the Software, and to permit persons to whom the Software is 9 | * furnished to do so, subject to the following conditions: 10 | * 11 | * The above copyright notice and this permission notice shall be included in all 12 | * copies or substantial portions of the Software. 13 | * 14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 | * SOFTWARE. 21 | */ 22 | 23 | /* 24 | * Utility functions for making in WebExtensions. 25 | * 26 | * Depends on `./base.js`, `./compat.js`, and `./ui.js`. 27 | */ 28 | 29 | "use strict"; 30 | 31 | async function getActiveTab() { 32 | let tabs = await browser.tabs.query({ active: true, currentWindow: true }); 33 | for (let tab of tabs) { 34 | return tab; 35 | } 36 | return null; 37 | } 38 | 39 | function navigateTabTo(tabId, url) { 40 | return browser.tabs.update(tabId, { url }); 41 | } 42 | 43 | function navigateTabToBlank(tabId) { 44 | return navigateTabTo(tabId, "about:blank"); 45 | } 46 | 47 | function getTabURL(tab, def) { 48 | let pendingUrl = tab.pendingUrl; 49 | if (isDefinedURL(pendingUrl)) 50 | return pendingUrl; 51 | let url = tab.url; 52 | if (isDefinedURL(url)) 53 | return url; 54 | return def; 55 | } 56 | 57 | async function getTabURLThenNavigateTabToBlank(tabId) { 58 | let tab = await browser.tabs.get(tabId); 59 | let url = getTabURL(tab); 60 | await navigateTabToBlank(tabId); 61 | return url; 62 | } 63 | 64 | async function getShortcuts() { 65 | if (browser.commands === undefined) 66 | return {}; 67 | 68 | let shortcuts = await browser.commands.getAll(); 69 | let res = {}; 70 | for (let s of shortcuts) 71 | res[s.name] = s.shortcut; 72 | return res; 73 | } 74 | 75 | function macroShortcuts(node, shortcuts, mapShortcutFunc) { 76 | for (let child of node.childNodes) { 77 | if (child.nodeName === "#text" || child.nodeName === "#comment") continue; 78 | macroShortcuts(child, shortcuts, mapShortcutFunc); 79 | } 80 | 81 | let sname = node.getAttribute("data-macro-shortcut"); 82 | if (sname === null) return; 83 | let shortcut = shortcuts[sname]; 84 | node.innerHTML = microMarkdownToHTML(mapShortcutFunc(node.innerText, shortcut, sname)); 85 | } 86 | 87 | // make a DOM node with a given id emit a `browser.runtime.sendMessage` with the same id 88 | function buttonToMessage(id, func) { 89 | if (func === undefined) 90 | return buttonToAction(id, catchAll(() => browser.runtime.sendMessage([id]))); 91 | else 92 | return buttonToAction(id, catchAll(() => browser.runtime.sendMessage(func()))); 93 | } 94 | 95 | // activate a tab with a given document URL if exists, or open new if not 96 | async function spawnOrActivateTab(url, createProperties, currentWindow) { 97 | if (currentWindow === undefined) 98 | currentWindow = true; 99 | 100 | let tabs = await browser.tabs.query({ currentWindow }); 101 | let nurl = normalizedURL(url); 102 | for (let tab of tabs) { 103 | if (normalizedURL(getTabURL(tab, "about:blank")) === nurl) { 104 | // activate that tab instead 105 | await browser.tabs.update(tab.id, { active: true }); 106 | return [tab, false]; 107 | } 108 | } 109 | 110 | // open new tab 111 | let res = await browser.tabs.create(assignRec({ url }, createProperties || {})); 112 | return [res, true]; 113 | } 114 | 115 | async function showInternalPageAtNode(url, id, tabId, spawn, scrollIntoViewOptions) { 116 | let rurl = url + (id ? "#" + id : ""); 117 | if (spawn === false) { 118 | window.location = rurl; 119 | return null; 120 | } else { 121 | let tab, spawned; 122 | try { 123 | [tab, spawned] = await spawnOrActivateTab(rurl, { openerTabId: tabId }); 124 | } catch (e) { 125 | // in case tabId points to a dead tab 126 | [tab, spawned] = await spawnOrActivateTab(rurl); 127 | } 128 | if (!spawned && id !== undefined) 129 | broadcastToURL(false, url, "viewNode", id, scrollIntoViewOptions); 130 | return tab.id; 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /extension/manifest-chromium-mv2.json: -------------------------------------------------------------------------------- 1 | { 2 | "icons": { 3 | "128": "icon/128/main.png" 4 | }, 5 | 6 | "browser_action": { 7 | "default_icon": { 8 | "128": "icon/128/main.png" 9 | } 10 | }, 11 | 12 | "permissions": [ 13 | "", 14 | "contextMenus", 15 | "debugger", 16 | "notifications", 17 | "storage", 18 | "unlimitedStorage", 19 | "tabs", 20 | "webNavigation", 21 | "webRequest", 22 | "webRequestBlocking" 23 | ], 24 | 25 | "update_url": "https://raw.githubusercontent.com/Own-Data-Privateer/hoardy-web/metadata/update-chromium-mv2.xml" 26 | } 27 | -------------------------------------------------------------------------------- /extension/manifest-common.json: -------------------------------------------------------------------------------- 1 | { 2 | "manifest_version": 2, 3 | 4 | "name": "Hoardy-Web", 5 | "version": "1.21.1", 6 | "description": "Passively capture, archive, and hoard your web browsing history, including the contents of the pages you visit, for later offline viewing, replay, mirroring, data scraping, and/or indexing. Low memory footprint, lots of configuration options.", 7 | 8 | "author": "Jan Malakhovski", 9 | "homepage_url": "https://github.com/Own-Data-Privateer/hoardy-web", 10 | 11 | "background": { 12 | "page": "background/main.html" 13 | }, 14 | 15 | "browser_action": { 16 | "default_title": "Hoardy-Web", 17 | "default_popup": "page/popup.html" 18 | }, 19 | 20 | "options_ui": { 21 | "page": "page/popup.html#options" 22 | }, 23 | 24 | "commands": { 25 | "showState": { 26 | "suggested_key": { 27 | "default": "Alt+G" 28 | }, 29 | "description": "Show State: Open the internal state page" 30 | }, 31 | "showLog": { 32 | "description": "Show Log: Open the log of recently collected and discarded reqres" 33 | }, 34 | "showTabState": { 35 | "description": "Show Tab's State: For the currently active tab, open the internal state page" 36 | }, 37 | "showTabLog": { 38 | "description": "Show Tab's Log: For the currently active tab, open the log of recently collected and discarded reqres" 39 | }, 40 | "rearchiveAdjunctSaved": { 41 | "description": "Re-archive adjunct: Re-archive a new batch of reqres using the configured re-archival methods" 42 | }, 43 | "toggleTabConfigWorkOffline": { 44 | "description": "Toggle Tab's Work Offline: For the currently active tab, toggle `Work Offline` setting (then, if impure, set `Track new requests` setting to the opposite value) and reset the related option for the tab's new children" 45 | }, 46 | "toggleTabConfigChildrenWorkOffline": { 47 | "description": "Toggle Children's Work Offline: For currently active tab's new children, toggle `Set 'Work Offline'` setting (then, if impure, set `Set 'Track new requests'` setting to the opposite value)" 48 | }, 49 | "toggleTabConfigTracking": { 50 | "suggested_key": { 51 | "default": "Alt+C" 52 | }, 53 | "description": "Toggle Tab's Tracking: For the currently active tab, toggle `Track new requests` setting and reset the related option for the tab's new children" 54 | }, 55 | "toggleTabConfigChildrenTracking": { 56 | "description": "Toggle Children's Tracking: For currently active tab's new children, toggle `Set 'Track new requests' setting`" 57 | }, 58 | "toggleTabConfigProblematicNotify": { 59 | "description": "Toggle Tab's Notify Problematic: For the currently active tab, toggle `Notify about 'problematic' reqres` setting and reset the related option for the tab's new children" 60 | }, 61 | "toggleTabConfigChildrenProblematicNotify": { 62 | "description": "Toggle Children's Notify Problematic: For currently active tab's new children, toggle `Set 'Notify about 'problematic' reqres'` setting" 63 | }, 64 | "toggleTabConfigLimbo": { 65 | "suggested_key": { 66 | "default": "Alt+L" 67 | }, 68 | "description": "Toggle Tab's Limbo: For the currently active tab, toggle `Pick into limbo` setting and reset the related option for the tab's new children" 69 | }, 70 | "toggleTabConfigChildrenLimbo": { 71 | "description": "Toggle Children's Limbo: For currently active tab's new children, toggle `Set 'Pick into limbo'` setting" 72 | }, 73 | "unmarkAllProblematic": { 74 | "description": "Unmark All Problematic: Unmark all problematic reqres" 75 | }, 76 | "unmarkAllTabProblematic": { 77 | "suggested_key": { 78 | "default": "Alt+U" 79 | }, 80 | "description": "Unmark Tab's Problematic: For the currently active tab, unmark all problematic reqres" 81 | }, 82 | "collectAllInLimbo": { 83 | "description": "Collect All Limbo: Collect all reqres in limbo" 84 | }, 85 | "collectAllTabInLimbo": { 86 | "description": "Collect Tab's Limbo: For the currently active tab, collect all reqres in limbo" 87 | }, 88 | "discardAllInLimbo": { 89 | "description": "Discard All Limbo: Discard all reqres in limbo" 90 | }, 91 | "discardAllTabInLimbo": { 92 | "description": "Discard Tab's Limbo: For the currently active tab, discard all reqres in limbo" 93 | }, 94 | "toggleTabConfigSnapshottable": { 95 | "description": "Toggle Tab's Snapshottable: For the currently active tab, toggle `Include in global snapshots` setting and reset the related option for the tab's new children" 96 | }, 97 | "toggleTabConfigChildrenSnapshottable": { 98 | "description": "Toggle Children's Snapshottable: For currently active tab's new children, toggle `Set 'Include in global snapshots'` setting" 99 | }, 100 | "snapshotAll": { 101 | "description": "Snapshot All: Take DOM snapshots of all frames of all open tabs for which `Include in global snapshots` option is set" 102 | }, 103 | "snapshotTab": { 104 | "description": "Snapshot Tab's: Take DOM snapshots of all frames of the currently active tab" 105 | }, 106 | "toggleTabConfigReplayable": { 107 | "description": "Toggle Tab's Replayable: For the currently active tab, toggle `Include in global replays` setting and reset the related option for the tab's new children" 108 | }, 109 | "toggleTabConfigChildrenReplayable": { 110 | "description": "Toggle Children's Replayable: For currently active tab's new children, toggle `Set 'Include in global replays'` setting" 111 | }, 112 | "replayAll": { 113 | "description": "Replay All: If the archiving server supports replay, re-navigate all tabs that finished loading to their replayed versions" 114 | }, 115 | "replayTabBack": { 116 | "description": "Replay Tab Back: If the archiving server supports replay and the currently active tab has finished loading, re-navigate it to its replayed version" 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /extension/manifest-firefox-mv2.json: -------------------------------------------------------------------------------- 1 | { 2 | "icons": { 3 | "48": "icon/main.svg", 4 | "96": "icon/main.svg", 5 | "128": "icon/main.svg" 6 | }, 7 | 8 | "browser_action": { 9 | "default_icon": { 10 | "48": "icon/main.svg", 11 | "96": "icon/main.svg", 12 | "128": "icon/main.svg" 13 | } 14 | }, 15 | 16 | "permissions": [ 17 | "", 18 | "menus", 19 | "notifications", 20 | "storage", 21 | "unlimitedStorage", 22 | "tabs", 23 | "webNavigation", 24 | "webRequest", 25 | "webRequestBlocking" 26 | ], 27 | 28 | "browser_specific_settings": { 29 | "gecko": { 30 | "id": "pwebarc@oxij.org", 31 | "strict_min_version": "102.0" 32 | }, 33 | "gecko_android": { 34 | "strict_min_version": "113.0" 35 | } 36 | }, 37 | 38 | "commands": { 39 | "_execute_browser_action": { 40 | "suggested_key": { 41 | "default": "Alt+A" 42 | } 43 | }, 44 | "showLog": { 45 | "suggested_key": { 46 | "default": "Alt+Shift+G" 47 | } 48 | }, 49 | "showTabState": { 50 | "suggested_key": { 51 | "default": "Alt+I" 52 | } 53 | }, 54 | "showTabLog": { 55 | "suggested_key": { 56 | "default": "Alt+Shift+I" 57 | } 58 | }, 59 | "toggleTabConfigWorkOffline": { 60 | "suggested_key": { 61 | "default": "Alt+O" 62 | } 63 | }, 64 | "toggleTabConfigChildrenWorkOffline": { 65 | "suggested_key": { 66 | "default": "Alt+Shift+O" 67 | } 68 | }, 69 | "toggleTabConfigChildrenTracking": { 70 | "suggested_key": { 71 | "default": "Alt+Shift+C" 72 | } 73 | }, 74 | "toggleTabConfigChildrenLimbo": { 75 | "suggested_key": { 76 | "default": "Alt+Shift+L" 77 | } 78 | }, 79 | "unmarkAllProblematic": { 80 | "suggested_key": { 81 | "default": "Alt+Shift+U" 82 | } 83 | }, 84 | "collectAllInLimbo": { 85 | "suggested_key": { 86 | "default": "Ctrl+Alt+A" 87 | } 88 | }, 89 | "collectAllTabInLimbo": { 90 | "suggested_key": { 91 | "default": "Ctrl+Alt+C" 92 | } 93 | }, 94 | "discardAllInLimbo": { 95 | "suggested_key": { 96 | "default": "Alt+Shift+D" 97 | } 98 | }, 99 | "discardAllTabInLimbo": { 100 | "suggested_key": { 101 | "default": "Alt+Shift+W" 102 | } 103 | }, 104 | "snapshotTab": { 105 | "suggested_key": { 106 | "default": "Ctrl+Alt+S" 107 | } 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /extension/page/help.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-2025 Jan Malakhovski 3 | * 4 | * This file is a part of `hoardy-web` project. 5 | * 6 | * This program is free software: you can redistribute it and/or modify 7 | * it under the terms of the GNU General Public License as published by 8 | * the Free Software Foundation, either version 3 of the License, or 9 | * (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program. If not, see . 18 | */ 19 | 20 | /* 21 | * The "Help" page. 22 | */ 23 | 24 | "use strict"; 25 | 26 | document.addEventListener("DOMContentLoaded", async () => { 27 | let body = document.getElementById("body"); 28 | let iframe = document.getElementById("iframe"); 29 | let minWidth = 1355; // see ./help.template 30 | let columns = true; 31 | 32 | // allow to un-highlight currently highlighted node 33 | document.body.addEventListener("click", (event) => { 34 | highlightNode(null); 35 | broadcastToPopup("highlightNode", null); 36 | }); 37 | 38 | setupHistoryPopState(); 39 | classifyDocumentLinks(document, [ 40 | ["/page/help.html", "internal"], 41 | ["/page/popup.html", "popup"], 42 | ["/", "local"], 43 | ], (link, info) => { 44 | let klass = info.klass; 45 | if (klass !== undefined) 46 | link.classList.add(klass); 47 | 48 | switch (info.klass) { 49 | case "internal": 50 | link.href = "javascript:void(0)"; 51 | link.onclick = (event) => { 52 | event.cancelBubble = true; 53 | historyFromTo({ id: info.id }, { id: info.target }); 54 | focusNode(info.target); 55 | }; 56 | link.onmouseover = (event) => { 57 | if (columns) 58 | broadcastToPopup("highlightNode", null); 59 | }; 60 | break; 61 | case "popup": 62 | link.href = "javascript:void(0)"; 63 | link.onclick = (event) => { 64 | event.cancelBubble = true; 65 | if (!columns) 66 | historyFromTo({ id: info.id }, info.href); 67 | broadcastToPopup("focusNode", info.target); 68 | }; 69 | link.onmouseover = (event) => { 70 | if (columns) 71 | broadcastToPopup("focusNode", info.target); 72 | }; 73 | break; 74 | case "local": 75 | default: 76 | link.onclick = (event) => { 77 | historyFromTo({ id: info.id }); 78 | }; 79 | link.onmouseover = (event) => { 80 | if (columns) 81 | broadcastToPopup("highlightNode", null); 82 | }; 83 | } 84 | }); 85 | 86 | // Resize elements to window. This switches between `columns` and 87 | // `linear` layouts depending on width. This part is not done via 88 | // `CSS` because we use the `columns` value above too. 89 | function resize() { 90 | let w = window.innerWidth; 91 | let h = window.innerHeight; 92 | 93 | console.log("current viewport:", w, h); 94 | columns = w >= minWidth; 95 | 96 | setConditionalClass(document.body, "columns", columns); 97 | setConditionalClass(document.body, "linear", !columns); 98 | 99 | // Prevent independent scroll in `columns` layout. 100 | let h1 = columns ? `${h - 5}px` : null; 101 | body.style["min-height"] 102 | = body.style["max-height"] 103 | = body.style["height"] 104 | = h1; 105 | 106 | let ib = iframe.contentDocument.body; 107 | if (ib === null) 108 | // not yet loaded 109 | return; 110 | 111 | // Prevent in-iframe scroll in `linear` layout. 112 | let h2 = columns ? h1 : `${ib.scrollHeight + 20}px`; 113 | iframe.style["min-height"] 114 | = iframe.style["max-height"] 115 | = iframe.style["height"] 116 | = h2; 117 | } 118 | 119 | // expand shortcut macros 120 | let shortcuts = await getShortcuts(); 121 | macroShortcuts(body, shortcuts, (inner, shortcut, sname) => { 122 | let sk = manifest.commands[sname]; 123 | let def; 124 | if (sk.suggested_key && sk.suggested_key.default) 125 | def = sk.suggested_key.default; 126 | if (def) { 127 | if (shortcut) { 128 | return (shortcut === def) 129 | ? `currently bound to \`${shortcut}\` (= default)` 130 | : `currently bound to \`${shortcut}\` (default: \`${def}\`)` 131 | } else 132 | return `unbound at the moment (default: \`${def}\`)`; 133 | } else if (shortcut) 134 | return `currently bound to \`${shortcut}\` (default: unbound)` 135 | else 136 | return `unbound at the moment (= default)`; 137 | }); 138 | 139 | async function processUpdate(update) { 140 | let [what, data] = update; 141 | switch (what) { 142 | case "updateConfig": 143 | setRootClasses(data); 144 | break; 145 | case "popupResized": 146 | resize(); 147 | break; 148 | default: 149 | await webextRPCHandleMessageDefault(update); 150 | } 151 | } 152 | 153 | // add default handlers 154 | await subscribeToExtensionSimple("help", catchAll(processUpdate)); 155 | 156 | { 157 | let config = await browser.runtime.sendMessage(["getConfig"]); 158 | setRootClasses(config); 159 | } 160 | 161 | window.onresize = catchAll(resize); 162 | catchAll(resize)(); 163 | 164 | // give it a chance to re-compute the layout 165 | await sleep(1); 166 | 167 | // show UI 168 | document.body.style["visibility"] = null; 169 | 170 | // highlight current target 171 | focusHashNode(); 172 | }); 173 | -------------------------------------------------------------------------------- /extension/page/help.template: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Hoardy-Web: Help 7 | 8 | 9 | 104 | 105 | 106 |
107 |

Table of Contents

108 |
(Click me to see it.) 109 | $table-of-contents$ 110 |
111 | 112 | $body$ 113 |
114 |
115 | 116 |
117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | -------------------------------------------------------------------------------- /extension/page/minimal.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2024-2025 Jan Malakhovski 3 | * 4 | * This file is a part of `hoardy-web` project. 5 | * 6 | * This program is free software: you can redistribute it and/or modify 7 | * it under the terms of the GNU General Public License as published by 8 | * the Free Software Foundation, either version 3 of the License, or 9 | * (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program. If not, see . 18 | */ 19 | 20 | /* 21 | * Minimal page infrastructure. 22 | */ 23 | 24 | "use strict"; 25 | 26 | let pageName = document.location.pathname.substr(6).split(".")[0]; 27 | 28 | document.addEventListener("DOMContentLoaded", async () => { 29 | setupHistoryPopState(); 30 | classifyDocumentLinks(document, [ 31 | [`/page/${pageName}.html`, "internal"], 32 | ["/", "local"], 33 | ]); 34 | 35 | async function processUpdate(update) { 36 | let [what, data] = update; 37 | switch (what) { 38 | case "updateConfig": 39 | setRootClasses(data); 40 | break; 41 | default: 42 | await webextRPCHandleMessageDefault(update); 43 | } 44 | } 45 | 46 | // add default handlers 47 | await subscribeToExtensionSimple(pageName, catchAll(processUpdate)); 48 | 49 | { 50 | let config = await browser.runtime.sendMessage(["getConfig"]); 51 | setRootClasses(config); 52 | } 53 | 54 | // show UI 55 | document.body.style["visibility"] = null; 56 | 57 | // highlight current target 58 | focusHashNode(); 59 | }); 60 | -------------------------------------------------------------------------------- /extension/page/minimal.template: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Hoardy-Web: $title$ 7 | 8 | 9 | 46 | 47 | 48 |
49 | $body$ 50 |
51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /extension/page/popup.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Stub 6 | 7 | 8 |

This is a stub

9 |

The build script will replace this page with a page generated from popup.template.

10 |

If you came here by following a link, then that link is meant to be clicked from within the "Help" page opened from extension's UI.

11 |

See the very top of help.org for more info.

12 | 13 | 14 | -------------------------------------------------------------------------------- /extension/page/reqres-ui.css: -------------------------------------------------------------------------------- 1 | @layer defaults { 2 | :root { 3 | --log-in-flight: #ffffaa; 4 | --log-picked: #eeffee; 5 | --log-dropped: #ffeeee; 6 | --log-collected: #aaffaa; 7 | --log-discarded: #ffaaaa; 8 | } 9 | 10 | .colorblind { 11 | --log-picked: #eeeeff; 12 | --log-collected: #aaaaff; 13 | } 14 | 15 | .dark { 16 | --log-in-flight: #606020; 17 | --log-picked: #446044; 18 | --log-dropped: #604444; 19 | --log-collected: #004000; 20 | --log-discarded: #400000; 21 | } 22 | 23 | .dark.colorblind { 24 | --log-picked: #444460; 25 | --log-collected: #000040; 26 | } 27 | 28 | div.ui.omega input[type="number"]:disabled { 29 | display: none; 30 | } 31 | 32 | h1 { 33 | font-size: 200%; 34 | margin: 5px 0 0 0; 35 | display: flex; 36 | } 37 | 38 | div.controls, 39 | .right { 40 | display: flex; 41 | } 42 | 43 | h1 code, 44 | div.controls code, 45 | .right code { 46 | margin: auto 0.5ch; 47 | } 48 | 49 | div.controls { 50 | margin-top: 10px; 51 | } 52 | 53 | div.controls > span { 54 | margin-right: 10px; 55 | } 56 | 57 | div.controls .ui.number label input { 58 | width: 8em; 59 | } 60 | 61 | .right > span { 62 | margin-left: 10px; 63 | } 64 | 65 | .help-tip { 66 | max-width: 40em; 67 | } 68 | 69 | /* right-most help-tip sticks to the edge */ 70 | .right *:nth-last-child(1) .help-tip, 71 | .right .help-tip-right .help-tip { 72 | right: 0; 73 | } 74 | 75 | table, th, td { 76 | border: 1px solid black; 77 | } 78 | 79 | table { 80 | margin: 10px 0; 81 | width: 100%; 82 | } 83 | 84 | table tr td:nth-child(1) div { 85 | display: flex; 86 | } 87 | 88 | table tr.errors td:nth-child(1) { 89 | border: none; 90 | } 91 | 92 | td.long { 93 | line-break: anywhere; 94 | } 95 | 96 | .in-flight { background: var(--log-in-flight); } 97 | .picked { background: var(--log-picked); } 98 | .dropped { background: var(--log-dropped); } 99 | .collected { background: var(--log-collected); } 100 | .discarded { background: var(--log-discarded); } 101 | } 102 | -------------------------------------------------------------------------------- /extension/page/saved.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-2025 Jan Malakhovski 3 | * 4 | * This file is a part of `hoardy-web` project. 5 | * 6 | * This program is free software: you can redistribute it and/or modify 7 | * it under the terms of the GNU General Public License as published by 8 | * the Free Software Foundation, either version 3 of the License, or 9 | * (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program. If not, see . 18 | */ 19 | 20 | /* 21 | * The "Saved in Local Storage" page. 22 | */ 23 | 24 | "use strict"; 25 | 26 | // TODO: remove this 27 | let tabId = null; 28 | 29 | let dbody = document.body; 30 | 31 | async function stateMain() { 32 | await commonMain(); 33 | 34 | let config; 35 | let rearchive = newRearchiveVars(); 36 | 37 | function updateUI() { 38 | setRootClasses(config); 39 | implySetConditionalOff(dbody, "on-rearchive", !(config.rearchiveExportAs || config.rearchiveSubmitHTTP || rearchive.andRewrite)); 40 | } 41 | 42 | setUIRec(document, "rearchive", rearchive, (newrearchive, path) => { 43 | updateRearchiveVars(newrearchive, path); 44 | updateUI(); 45 | }); 46 | 47 | async function updateConfig(nconfig) { 48 | if (nconfig === undefined) 49 | config = await browser.runtime.sendMessage(["getConfig"]); 50 | else 51 | config = nconfig; 52 | 53 | updateUI(); 54 | } 55 | 56 | let savedFilters; 57 | 58 | async function updateSavedFilters(nsavedFilters) { 59 | if (nsavedFilters === undefined) 60 | savedFilters = await browser.runtime.sendMessage(["getSavedFilters"]); 61 | else 62 | savedFilters = nsavedFilters; 63 | 64 | setUI(document, "rrfilters", savedFilters, (value, path) => { 65 | browser.runtime.sendMessage(["setSavedFilters", value]).catch(logError); 66 | }); 67 | } 68 | 69 | async function processUpdate(update) { 70 | let [what, data] = update; 71 | switch(what) { 72 | case "updateConfig": 73 | await updateConfig(data); 74 | break; 75 | case "setSavedFilters": 76 | await updateSavedFilters(data); 77 | break; 78 | case "resetSaved": 79 | resetDataNode("data", data); 80 | break; 81 | default: 82 | await webextRPCHandleMessageDefault(update); 83 | } 84 | } 85 | 86 | buttonToMessage("rearchiveSaved", () => ["rearchiveSaved", savedFilters, true, rearchive.andDelete, rearchive.andRewrite]); 87 | buttonToAction("deleteSaved", catchAll(() => { 88 | if (!window.confirm("Really?")) 89 | return; 90 | 91 | browser.runtime.sendMessage(["deleteSaved", savedFilters]).catch(logError); 92 | })); 93 | 94 | await subscribeToExtension("saved", catchAll(processUpdate), catchAll(async (willReset) => { 95 | await updateConfig(); 96 | await updateSavedFilters(); 97 | }), () => false, setPageLoading, setPageSettling); 98 | 99 | await browser.runtime.sendMessage(["setSavedFilters", savedFilters]); 100 | 101 | // show UI 102 | setPageLoaded(); 103 | 104 | // force re-scroll 105 | viewHashNode(); 106 | } 107 | 108 | document.addEventListener("DOMContentLoaded", () => stateMain().catch(setPageError), setPageError); 109 | -------------------------------------------------------------------------------- /extension/page/saved.template: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Hoardy-Web: Saved in Local Storage 6 | 7 | 8 | 9 | 10 | 11 |
12 | 13 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /extension/page/state.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023-2025 Jan Malakhovski 3 | * 4 | * This file is a part of `hoardy-web` project. 5 | * 6 | * This program is free software: you can redistribute it and/or modify 7 | * it under the terms of the GNU General Public License as published by 8 | * the Free Software Foundation, either version 3 of the License, or 9 | * (at your option) any later version. 10 | * 11 | * This program is distributed in the hope that it will be useful, 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | * GNU General Public License for more details. 15 | * 16 | * You should have received a copy of the GNU General Public License 17 | * along with this program. If not, see . 18 | */ 19 | 20 | /* 21 | * The "Internal State" page. 22 | */ 23 | 24 | "use strict"; 25 | 26 | let rrfilters = { 27 | problematic: assignRec({}, rrfilterDefaults), 28 | in_limbo: assignRec({}, rrfilterDefaults), 29 | log: assignRec({}, rrfilterDefaults), 30 | queued: assignRec({}, rrfilterDefaults), 31 | unarchived: assignRec({}, rrfilterDefaults), 32 | }; 33 | 34 | let tabId = getMapURLParam(statePageURL, "tab", document.location, toNumber, null, null); 35 | if (tabId !== null) 36 | document.title = `Hoardy-Web: tab ${tabId}: Internal State`; 37 | 38 | function resetInFlight(log_data) { 39 | resetDataNode("data_in_flight", log_data); 40 | } 41 | 42 | function resetProblematic(log_data) { 43 | resetDataNode("data_problematic", log_data, (loggable) => isAcceptedBy(rrfilters.problematic, loggable)); 44 | } 45 | 46 | function resetInLimbo(log_data) { 47 | resetDataNode("data_in_limbo", log_data, (loggable) => isAcceptedBy(rrfilters.in_limbo, loggable)); 48 | } 49 | 50 | function resetLog(log_data) { 51 | resetDataNode("data_log", log_data, (loggable) => isAcceptedBy(rrfilters.log, loggable)); 52 | } 53 | 54 | function resetQueued(log_data) { 55 | resetDataNode("data_queued", log_data, (loggable) => isAcceptedBy(rrfilters.queued, loggable)); 56 | } 57 | 58 | function resetUnarchived(log_data) { 59 | resetDataNode("data_unarchived", log_data, (loggable) => isAcceptedBy(rrfilters.unarchived, loggable)); 60 | } 61 | 62 | async function stateMain() { 63 | await commonMain(); 64 | 65 | buttonToMessage("forgetHistory", () => ["forgetHistory", tabId, rrfilters.log]); 66 | buttonToMessage("rotateOneProblematic", () => ["rotateProblematic", 1, tabId, rrfilters.problematic]); 67 | buttonToMessage("unmarkOneProblematic", () => ["unmarkProblematic", 1, tabId, rrfilters.problematic]); 68 | buttonToMessage("unmarkAllProblematic", () => ["unmarkProblematic", null, tabId, rrfilters.problematic]); 69 | buttonToMessage("rotateOneInLimbo", () => ["rotateInLimbo", 1, tabId, rrfilters.in_limbo]); 70 | buttonToMessage("discardOneInLimbo", () => ["popInLimbo", false, 1, tabId, rrfilters.in_limbo]); 71 | buttonToMessage("discardAllInLimbo", () => ["popInLimbo", false, null, tabId, rrfilters.in_limbo]); 72 | buttonToMessage("collectOneInLimbo", () => ["popInLimbo", true, 1, tabId, rrfilters.in_limbo]); 73 | buttonToMessage("collectAllInLimbo", () => ["popInLimbo", true, null, tabId, rrfilters.in_limbo]); 74 | buttonToMessage("stopAllInFlight", () => ["stopInFlight", tabId]); 75 | 76 | buttonToMessage("retryAllUnarchived"); 77 | 78 | setUI(document, "rrfilters", rrfilters, (value, path) => { 79 | if (path.startsWith("rrfilters.problematic.")) 80 | browser.runtime.sendMessage(["getProblematicLog"]).then(resetProblematic).catch(logError); 81 | else if (path.startsWith("rrfilters.in_limbo.")) 82 | browser.runtime.sendMessage(["getInLimboLog"]).then(resetInLimbo).catch(logError); 83 | else if (path.startsWith("rrfilters.log.")) 84 | browser.runtime.sendMessage(["getLog"]).then(resetLog).catch(logError); 85 | else if (path.startsWith("rrfilters.queued.")) 86 | browser.runtime.sendMessage(["getQueuedLog"]).then(resetQueued).catch(logError); 87 | else if (path.startsWith("rrfilters.unarchived.")) 88 | browser.runtime.sendMessage(["getUnarchivedLog"]).then(resetUnarchived).catch(logError); 89 | else 90 | console.warn("unknown rrfilters update", path, value); 91 | }); 92 | 93 | async function updateConfig(config) { 94 | if (config === undefined) 95 | config = await browser.runtime.sendMessage(["getConfig"]); 96 | setRootClasses(config); 97 | } 98 | 99 | async function processUpdate(update) { 100 | let [what, data] = update; 101 | switch(what) { 102 | case "updateConfig": 103 | await updateConfig(data); 104 | break; 105 | case "resetInFlight": 106 | resetInFlight(data); 107 | break; 108 | case "resetProblematicLog": 109 | resetProblematic(data); 110 | break; 111 | case "resetInLimboLog": 112 | resetInLimbo(data); 113 | break; 114 | case "resetLog": 115 | resetLog(data); 116 | break; 117 | case "resetQueued": 118 | resetQueued(data); 119 | break; 120 | case "resetUnarchived": 121 | resetUnarchived(data); 122 | break; 123 | // incrementally add new rows 124 | case "newInFlight": 125 | appendToLog(document.getElementById("data_in_flight"), data); 126 | break; 127 | case "newProblematic": 128 | appendToLog(document.getElementById("data_problematic"), data, (loggable) => isAcceptedBy(rrfilters.problematic, loggable)); 129 | break; 130 | case "newLimbo": 131 | appendToLog(document.getElementById("data_in_limbo"), data, (loggable) => isAcceptedBy(rrfilters.in_limbo, loggable)); 132 | break; 133 | case "newLog": 134 | appendToLog(document.getElementById("data_log"), data, (loggable) => isAcceptedBy(rrfilters.log, loggable)); 135 | break; 136 | case "newQueued": 137 | appendToLog(document.getElementById("data_queued"), data, (loggable) => isAcceptedBy(rrfilters.queued, loggable)); 138 | break; 139 | default: 140 | await webextRPCHandleMessageDefault(update); 141 | } 142 | } 143 | 144 | await subscribeToExtension("state" + (tabId !== null ? `#${tabId}` : ""), 145 | catchAll(processUpdate), catchAll(async (willReset) => { 146 | await updateConfig(); 147 | let inFlightLog = await browser.runtime.sendMessage(["getInFlightLog"]); 148 | let problematicLog = await browser.runtime.sendMessage(["getProblematicLog"]); 149 | if (willReset()) return; 150 | let inLimboLog = await browser.runtime.sendMessage(["getInLimboLog"]); 151 | if (willReset()) return; 152 | let log = await browser.runtime.sendMessage(["getLog"]); 153 | if (willReset()) return; 154 | let queuedLog = await browser.runtime.sendMessage(["getQueuedLog"]); 155 | if (willReset()) return; 156 | let unarchivedLog = await browser.runtime.sendMessage(["getUnarchivedLog"]); 157 | if (willReset()) return; 158 | 159 | resetInFlight(inFlightLog); 160 | resetProblematic(problematicLog); 161 | resetInLimbo(inLimboLog); 162 | resetLog(log); 163 | resetQueued(queuedLog); 164 | resetUnarchived(unarchivedLog); 165 | }), (event) => { 166 | let cmd = event[0]; 167 | return cmd.startsWith("reset") || cmd.startsWith("new"); 168 | }, setPageLoading, setPageSettling); 169 | 170 | // show UI 171 | setPageLoaded(); 172 | 173 | // force re-scroll 174 | viewHashNode(); 175 | } 176 | 177 | document.addEventListener("DOMContentLoaded", () => stateMain().catch(setPageError), setPageError); 178 | -------------------------------------------------------------------------------- /extension/page/state.template: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Hoardy-Web: Internal State 6 | 7 | 8 | 9 | 10 | 11 |
12 | 13 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | -------------------------------------------------------------------------------- /extension/update-readme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | this_links() { 4 | sed ' 5 | s%\[\([^]]*\)\](\(http[^)]*\))%[\1](\2)%g 6 | t end 7 | s%\[\([^]]*\)\](..\(/[^)]*\))%[\1](https://oxij.org/software/hoardy-web/tree/master\2) (also on [GitHub](https://github.com/Own-Data-Privateer/hoardy-web/tree/master\2))%g 8 | s%\[\([^]]*\)\](.\(/[^)]*\))%[\1](https://oxij.org/software/hoardy-web/tree/master/extension\2) (also on [GitHub](https://github.com/Own-Data-Privateer/hoardy-web/tree/master/extension\2))%g 9 | : end 10 | ' 11 | } 12 | 13 | parent_links() { 14 | sed ' 15 | s%\[\([^]]*\)\](\(http[^)]*\))%[\1](\2)%g 16 | t end 17 | s%\[\([^]]*\)\](\(#[^)]*\))%[\1](https://oxij.org/software/hoardy-web/\2) (also on [GitHub](https://github.com/Own-Data-Privateer/hoardy-web\2))%g 18 | s%\[\([^]]*\)\](.\(/[^)]*\))%[\1](https://oxij.org/software/hoardy-web/tree/master\2) (also on [GitHub](https://github.com/Own-Data-Privateer/hoardy-web/tree/master\2))%g 19 | : end 20 | ' 21 | } 22 | 23 | amo_html() { 24 | pandoc --wrap=none -f markdown -t html | sed ' 25 | s%

%%g 26 | s%

%\n%g 27 | s%\(\)%\1\n%g 28 | ' 29 | } 30 | { 31 | cat ./README.md \ 32 | | sed 's%`Hoardy-Web` is a browser extension (add-on) that%`Hoardy-Web`%' \ 33 | | sed -n '3, /# Screenshots/ p' \ 34 | | head -n -1 | this_links | amo_html 35 | } > README.amo.html 36 | -------------------------------------------------------------------------------- /firefox/README.md: -------------------------------------------------------------------------------- 1 | # What? 2 | 3 | Just a tiny patch for Firefox-based browsers that makes them always give raw `requestBody` (instead of just `formData`) to the [`webRequest.onBeforeRequest` handlers of `WebExtensions` API](https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/webRequest). 4 | -------------------------------------------------------------------------------- /firefox/always-give-raw.patch: -------------------------------------------------------------------------------- 1 | From 314dca490a2b07dd3210be1b5c1afa59f376141a Mon Sep 17 00:00:00 2001 2 | From: Jan Malakhovski 3 | Date: Tue, 15 Aug 2023 07:54:18 +0000 4 | Subject: [PATCH] 5 | toolkit/components/extensions/webrequest/WebRequestUpload.jsm: always give 6 | raw 7 | 8 | --- 9 | .../components/extensions/webrequest/WebRequestUpload.jsm | 5 +++-- 10 | 1 file changed, 3 insertions(+), 2 deletions(-) 11 | 12 | diff --git a/toolkit/components/extensions/webrequest/WebRequestUpload.jsm b/toolkit/components/extensions/webrequest/WebRequestUpload.jsm 13 | index aca855de1277..3ba47dd65bd4 100644 14 | --- a/toolkit/components/extensions/webrequest/WebRequestUpload.jsm 15 | +++ b/toolkit/components/extensions/webrequest/WebRequestUpload.jsm 16 | @@ -538,16 +538,17 @@ WebRequestUpload = { 17 | try { 18 | let stream = channel.uploadStream; 19 | 20 | + let raw = Array.from(getRawDataChunked(stream)); 21 | let formData = createFormData(stream, channel); 22 | if (formData) { 23 | - return { formData }; 24 | + return { raw, formData }; 25 | } 26 | 27 | // If we failed to parse the stream as form data, return it as a 28 | // sequence of raw data chunks, along with a leniently-parsed form 29 | // data object, which ignores encoding errors. 30 | return { 31 | - raw: Array.from(getRawDataChunked(stream)), 32 | + raw, 33 | lenientFormData: createFormData(stream, channel, true), 34 | }; 35 | } catch (e) { 36 | -------------------------------------------------------------------------------- /packages.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import {} 2 | , developer ? false 3 | }: 4 | 5 | let 6 | source = import ./source.nix { inherit pkgs; }; 7 | args = { inherit pkgs source developer; }; 8 | in 9 | 10 | { 11 | simple_server = import ./simple_server args; 12 | extension = import ./extension args; 13 | tool = import ./tool args; 14 | } 15 | -------------------------------------------------------------------------------- /simple_server/README.md: -------------------------------------------------------------------------------- 1 | # What is `hoardy-web-sas`? 2 | 3 | `hoardy-web-sas` is a very simple archiving server for the [`Hoardy-Web` Web Extension browser add-on](https://oxij.org/software/hoardy-web/tree/master/extension/) (also on [GitHub](https://github.com/Own-Data-Privateer/hoardy-web/tree/master/extension/)). 4 | 5 | I.e. this is the thing you run and then paste the URL of into the `Server URL` setting of `Hoardy-Web`. 6 | 7 | This is not the most feature-rich thing for doing that, [`hoardy-web serve`](https://oxij.org/software/hoardy-web/tree/master/tool/) (also on [GitHub](https://github.com/Own-Data-Privateer/hoardy-web/tree/master/tool/)) is much more powerful. 8 | But, `hoardy-web serve` is not at all simple and it depends on quite a lot of things. 9 | Meanwhile, this `hoardy-web-sas` thing is less than 300 lines of pure Python that only uses the Python\'s standard library and nothing else. 10 | You could be running it already. 11 | 12 | # Quickstart 13 | 14 | ## Pre-installation 15 | 16 | - Install `Python 3`: 17 | 18 | - On a Windows system: [Download Python installer from the official website](https://www.python.org/downloads/windows/), run it, **set `Add python.exe to PATH` checkbox**, then `Install` (the default options are fine). 19 | - On a conventional POSIX system like most GNU/Linux distros and MacOS X: Install `python3` via your package manager. Realistically, it probably is installed already. 20 | 21 | ## Installation 22 | 23 | - On a Windows system: 24 | 25 | Open `cmd.exe` (press `Windows+R`, enter `cmd.exe`, press `Enter`), install this tool with 26 | ```bash 27 | python -m pip install hoardy-web-sas 28 | ``` 29 | and run as 30 | ```bash 31 | python -m hoardy_web_sas --help 32 | ``` 33 | 34 | - On a POSIX system or on a Windows system with Python's `/Scripts` added to `PATH`: 35 | 36 | Open a terminal/`cmd.exe`, install with 37 | ```bash 38 | pip install hoardy-web-sas 39 | ``` 40 | and run as 41 | ```bash 42 | hoardy-web-sas --help 43 | ``` 44 | 45 | - Alternatively, run without installing: 46 | 47 | ```bash 48 | python hoardy-web-sas.py --help 49 | # or, on POSIX 50 | ./hoardy-web-sas.py --help 51 | ``` 52 | 53 | - Alternatively, on a system with [Nix package manager](https://nixos.org/nix/) 54 | 55 | ```bash 56 | nix-env -i -f ./default.nix 57 | hoardy-web-sas --help 58 | ``` 59 | 60 | Though, in this case, you'll probably want to do the first command from the parent directory, to install everything all at once. 61 | 62 | ## Start archiving 63 | 64 | ```bash 65 | python -m hoardy_web_sas --archive-to C:\Users\Me\Documents\hoardy-web\raw 66 | # or 67 | hoardy-web-sas --archive-to ~/hoardy-web/raw 68 | ``` 69 | 70 | ## Capture and archive some websites 71 | 72 | See [`Hoardy-Web`'s "Quickstart"](https://oxij.org/software/hoardy-web/tree/master/README.md#quickstart) (also on [GitHub](https://github.com/Own-Data-Privateer/hoardy-web/tree/master/README.md#quickstart)). 73 | 74 | # Usage 75 | 76 | ``` 77 | usage: hoardy-web-sas [-h] [--version] [--host HOST] [--port PORT] [-t ROOT] [--compress | --no-compress] [--default-bucket NAME] [--ignore-buckets] [--no-print] 78 | 79 | A simple archiving server for the `Hoardy-Web` Web Extension browser add-on: listen on given `--host` and `--port` via `HTTP`, dump each `POST`ed `WRR` dump to `<--archive-to>/////_.wrr`. 80 | 81 | options: 82 | -h, --help show this help message and exit 83 | --version show program's version number and exit 84 | --host HOST listen on what host/IP; default: `127.0.0.1` 85 | --port PORT listen on what port; default: `3210` 86 | -t ROOT, --to ROOT, --archive-to ROOT, --root ROOT 87 | path to dump data into; default: `pwebarc-dump` 88 | --compress compress new archivals before dumping them to disk; default 89 | --no-compress, --uncompressed 90 | dump new archivals to disk without compression 91 | --default-bucket NAME, --default-profile NAME 92 | default bucket to use when no `profile` query parameter is supplied by the extension; default: `default` 93 | --ignore-buckets, --ignore-profiles 94 | ignore `profile` query parameter supplied by the extension and use the value of `--default-bucket` instead 95 | --no-print, --no-print-cbors 96 | don't print parsed representations of newly archived CBORs to stdout even if `cbor2` module is available 97 | 98 | ``` 99 | -------------------------------------------------------------------------------- /simple_server/default.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import {} 2 | , lib ? pkgs.lib 3 | , source ? import ../source.nix { inherit pkgs; } 4 | , developer ? false 5 | }: 6 | 7 | with pkgs.python3Packages; 8 | 9 | buildPythonApplication (rec { 10 | pname = "hoardy-web-sas"; 11 | version = "1.9.0"; 12 | format = "pyproject"; 13 | 14 | inherit (source) src unpackPhase; 15 | sourceRoot = "${src.name}/simple_server"; 16 | 17 | propagatedBuildInputs = [ 18 | setuptools 19 | cbor2 20 | ]; 21 | 22 | } // lib.optionalAttrs developer { 23 | nativeBuildInputs = [ 24 | build twine pip mypy pytest black pylint 25 | pkgs.pandoc 26 | ]; 27 | 28 | preBuild = "find . ; black --check . && mypy && pylint ."; 29 | postFixup = "find $out"; 30 | }) 31 | -------------------------------------------------------------------------------- /simple_server/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | [project] 5 | name = "hoardy-web-sas" 6 | version = "1.9.0" 7 | authors = [{ name = "Jan Malakhovski", email = "oxij@oxij.org" }] 8 | description = "A simple archiving server for the `Hoardy-Web` Web Extension browser add-on." 9 | readme = "README.md" 10 | license = { text = "GPL-3.0-or-later" } 11 | classifiers = [ 12 | "Development Status :: 4 - Beta", 13 | "Programming Language :: Python :: 3", 14 | "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", 15 | "Intended Audience :: End Users/Desktop", 16 | "Topic :: Internet :: WWW/HTTP", 17 | "Topic :: System :: Archiving", 18 | "Topic :: System :: Archiving :: Backup", 19 | "Topic :: System :: Archiving :: Mirroring", 20 | "Topic :: System :: Logging", 21 | "Topic :: Internet :: Log Analysis", 22 | "Operating System :: POSIX", 23 | "Environment :: Console", 24 | ] 25 | keywords = [ 26 | "HTTP", "HTTPS", 27 | "WWW", "web", "browser", 28 | "site", "website", 29 | "download", "archive", "mirror", 30 | "wayback", "wayback machine", 31 | ] 32 | dependencies = [ 33 | 'importlib-metadata; python_version<"3.8"', 34 | ] 35 | requires-python = ">=3.7" 36 | [project.optional-dependencies] 37 | cbor = ["cbor2"] 38 | [project.urls] 39 | "Homepage" = "https://oxij.org/software/hoardy-web/" 40 | "GitHub" = "https://github.com/Own-Data-Privateer/hoardy-web" 41 | "Support Development" = "https://oxij.org/#support" 42 | [project.scripts] 43 | hoardy-web-sas = "hoardy_web_sas:main" 44 | pwebarc-dumb-dump-server = "hoardy_web_sas:main" 45 | 46 | [tool.mypy] 47 | python_version = "3.10" 48 | strict = true 49 | implicit_reexport = true 50 | explicit_package_bases = true 51 | files = [ 52 | "*.py", 53 | ] 54 | [[tool.mypy.overrides]] 55 | module = [ 56 | "setuptools", 57 | "cbor2", 58 | ] 59 | ignore_missing_imports = true 60 | 61 | [tool.black] 62 | line-length = 100 63 | 64 | [tool.pylint] 65 | disable = [ 66 | "broad-exception-caught", 67 | "global-statement", 68 | "import-outside-toplevel", 69 | "invalid-name", 70 | "line-too-long", 71 | "missing-function-docstring", 72 | "too-many-branches", 73 | "too-many-locals", 74 | "too-many-statements", 75 | ] 76 | [tool.pylint.format] 77 | max-line-length = "100" 78 | -------------------------------------------------------------------------------- /simple_server/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Setup.""" 3 | from setuptools import setup 4 | 5 | setup() 6 | -------------------------------------------------------------------------------- /simple_server/update-readme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | { 4 | sed -n "0,/# Usage/ p" README.md 5 | echo -e '\n```' 6 | 7 | ./hoardy_web_sas.py --help | sed ' 8 | s/^\(#\+\) /#\1 / 9 | ' 10 | 11 | echo -e '```' 12 | } > README.new 13 | mv README.new README.md 14 | pandoc -f markdown -t html README.md > README.html 15 | -------------------------------------------------------------------------------- /source.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import {} 2 | , lib ? pkgs.lib 3 | }: 4 | 5 | { 6 | 7 | src = lib.cleanSourceWith { 8 | src = ./.; 9 | filter = name: type: let baseName = baseNameOf (toString name); in 10 | (builtins.match ".*.un~" baseName == null) 11 | && (baseName != "dist") 12 | && (baseName != "result") 13 | && (baseName != "results") 14 | && (baseName != "__pycache__") 15 | && (baseName != ".mypy_cache") 16 | && (baseName != ".pytest_cache") 17 | && (builtins.match ".*/simple_server/pwebarc-dump.*" name == null) 18 | ; 19 | }; 20 | 21 | unpackPhase = '' 22 | mkdir home 23 | HOME=$PWD/home 24 | ${pkgs.git}/bin/git config --global --add safe.directory '*' 25 | 26 | ${pkgs.git}/bin/git clone $src source 27 | ${pkgs.git}/bin/git clone $src/vendor/pako source/vendor/pako 28 | ${pkgs.git}/bin/git clone $src/vendor/kisstdlib source/vendor/kisstdlib 29 | cp -a $src/extension/private source/extension || true 30 | patchShebangs source 31 | find source | grep -vF '/.git/' 32 | ''; 33 | 34 | } 35 | -------------------------------------------------------------------------------- /tool/default.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import {} 2 | , lib ? pkgs.lib 3 | , developer ? false 4 | , kisstdlib ? import ../vendor/kisstdlib { inherit pkgs developer; } 5 | , cbor2 ? import ../vendor/cbor2 { inherit pkgs; } 6 | , source ? import ../source.nix { inherit pkgs; } 7 | , mitmproxySupport ? true 8 | }: 9 | 10 | let mycbor2 = cbor2; in 11 | 12 | with pkgs.python3Packages; 13 | 14 | buildPythonApplication (rec { 15 | pname = "hoardy-web"; 16 | version = "0.23.0"; 17 | format = "pyproject"; 18 | 19 | inherit (source) src unpackPhase; 20 | sourceRoot = "${src.name}/tool"; 21 | 22 | propagatedBuildInputs = [ 23 | setuptools 24 | kisstdlib 25 | sortedcontainers 26 | mycbor2 27 | idna 28 | html5lib 29 | tinycss2 30 | bottle 31 | ] 32 | ++ lib.optional mitmproxySupport mitmproxy; 33 | 34 | postInstall = '' 35 | patchShebangs script 36 | install -m 755 -t $out/bin script/hoardy-* 37 | ''; 38 | 39 | } // lib.optionalAttrs developer { 40 | nativeBuildInputs = [ 41 | build twine pip mypy pytest black pylint 42 | pkgs.pandoc 43 | 44 | kisstdlib # for `describe-forest` binary 45 | ]; 46 | 47 | preBuild = "find . ; ./sanity.sh --check"; 48 | postFixup = "find $out"; 49 | }) 50 | -------------------------------------------------------------------------------- /tool/hoardy_web/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Jan Malakhovski 2 | # 3 | # This file is a part of `hoardy-web` project. 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | -------------------------------------------------------------------------------- /tool/hoardy_web/mitmproxy.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-2024 Jan Malakhovski 2 | # 3 | # This file is a part of `hoardy-web` project. 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | """Loading of `mitmproxy` `mitmdump` files into `Reqres` structures.""" 19 | 20 | import io as _io 21 | import os as _os 22 | import struct as _struct 23 | import typing as _t 24 | 25 | import mitmproxy.io 26 | import mitmproxy.http 27 | import mitmproxy.websocket 28 | 29 | from .wrr import * 30 | 31 | 32 | def _hd(x: _t.Any) -> Headers: 33 | res = [] 34 | for k, v in x.fields: 35 | res.append((k.decode("ascii"), v)) 36 | return Headers(res) 37 | 38 | 39 | def mitmproxy_load_flow(flow: mitmproxy.http.HTTPFlow) -> Reqres | None: 40 | rq = flow.request 41 | if rq.scheme == "" and rq.method.upper() == "CONNECT": 42 | # skip mitmproxy CONNECT requests, these are logged my mitmproxy 43 | # when a browser tries to connect to a target host via mitmproxy, 44 | # they don't carry any useful info 45 | return None 46 | 47 | rq.decode() 48 | rq_body = rq.get_content() 49 | rq_complete = True 50 | if rq_body is None: 51 | rq_body = b"" 52 | rq_complete = False 53 | 54 | if rq_complete: 55 | cl = rq.headers.get("content-length", None) 56 | if cl is not None and int(cl) != len(rq_body): 57 | rq_complete = False 58 | 59 | if (rq.scheme == "http" and rq.port == 80) or (rq.scheme == "https" and rq.port == 443): 60 | maybeport = "" 61 | else: 62 | maybeport = ":" + str(rq.port) 63 | url = f"{rq.scheme}://{rq.host}{maybeport}{rq.path}" 64 | 65 | request = Request( 66 | Timestamp(rq.timestamp_start), 67 | rq.method.upper(), 68 | parse_url(url), 69 | _hd(rq.headers), 70 | rq_complete, 71 | rq_body, 72 | ) 73 | 74 | rs = flow.response 75 | if rs is not None: 76 | rs.decode() 77 | rs_body = rs.get_content() 78 | rs_complete = True 79 | if rs_body is None: 80 | rs_body = b"" 81 | rs_complete = False 82 | 83 | if rs_complete: 84 | cl = rs.headers.get("content-length", None) 85 | if cl is not None and int(cl) != len(rs_body): 86 | rs_complete = False 87 | 88 | response = Response( 89 | Timestamp(rs.timestamp_start), 90 | rs.status_code, 91 | rs.reason, 92 | _hd(rs.headers), 93 | rs_complete, 94 | rs_body, 95 | ) 96 | 97 | tend = rs.timestamp_end 98 | if tend is None: 99 | tend = rs.timestamp_start 100 | else: 101 | response = None 102 | tend = rq.timestamp_end 103 | if tend is None: 104 | tend = rq.timestamp_start 105 | 106 | finished_at = Timestamp(tend) 107 | 108 | wsstream = None 109 | if flow.websocket is not None: 110 | ws = flow.websocket 111 | 112 | wsstream = [] 113 | for msg in ws.messages: 114 | if isinstance(msg.content, bytes): 115 | content = msg.content 116 | elif isinstance(msg.content, str): 117 | # even though mitmproxy declares content to be `bytes`, reading 118 | # dump files produced by old mitmproxy can produce `str` 119 | content = msg.content.encode("utf-8") 120 | else: 121 | assert False 122 | wsstream.append( 123 | WebSocketFrame(Timestamp(msg.timestamp), msg.from_client, int(msg.type), content) 124 | ) 125 | 126 | if ws.timestamp_end is not None: 127 | assert ws.closed_by_client is not None 128 | assert ws.close_code is not None 129 | assert ws.close_reason is not None 130 | 131 | # reconstruct the CLOSE frame 132 | wsstream.append( 133 | WebSocketFrame( 134 | Timestamp(ws.timestamp_end), 135 | ws.closed_by_client, 136 | 0x8, 137 | _struct.pack("!H", ws.close_code) + ws.close_reason.encode("utf-8"), 138 | ) 139 | ) 140 | 141 | return Reqres( 142 | 1, "hoardy-mitmproxy/1", rq.http_version, request, response, finished_at, {}, wsstream 143 | ) 144 | 145 | 146 | def rrexprs_mitmproxy_load_fileobj( 147 | fobj: _io.BufferedReader, source: DeferredSourceType 148 | ) -> _t.Iterator[ReqresExpr[StreamElementSource[DeferredSourceType]]]: 149 | stream = mitmproxy.io.FlowReader(fobj).stream() 150 | n = 0 151 | for flow in stream: 152 | if not isinstance(flow, mitmproxy.http.HTTPFlow): 153 | raise ValueError("unknown type", type(flow)) 154 | 155 | reqres = mitmproxy_load_flow(flow) 156 | if reqres is None: 157 | continue 158 | yield ReqresExpr(StreamElementSource(source, n), reqres) 159 | n += 1 160 | 161 | 162 | def rrexprs_mitmproxy_loadf( 163 | path: str | bytes, 164 | ) -> _t.Iterator[ReqresExpr[StreamElementSource[FileSource]]]: 165 | with open(path, "rb") as f: 166 | in_stat = _os.fstat(f.fileno()) 167 | yield from rrexprs_mitmproxy_load_fileobj(f, make_FileSource(path, in_stat)) 168 | -------------------------------------------------------------------------------- /tool/hoardy_web/source.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024 Jan Malakhovski 2 | # 3 | # This file is a part of `hoardy-web` project. 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | """Abstract data sources.""" 19 | 20 | import abc as _abc 21 | import dataclasses as _dc 22 | import io as _io 23 | import os as _os 24 | import typing as _t 25 | 26 | from kisstdlib.failure import * 27 | from kisstdlib.fs import fsdecode as _fsdecode 28 | 29 | 30 | class DeferredSource(metaclass=_abc.ABCMeta): 31 | @_abc.abstractmethod 32 | def approx_size(self) -> int: 33 | raise NotImplementedError() 34 | 35 | @_abc.abstractmethod 36 | def show_source(self) -> str: 37 | raise NotImplementedError() 38 | 39 | @_abc.abstractmethod 40 | def get_fileobj(self) -> _io.BufferedReader: 41 | raise NotImplementedError() 42 | 43 | def get_bytes(self) -> bytes: 44 | with self.get_fileobj() as f: 45 | return f.read() 46 | 47 | def same_as(self, other: _t.Any) -> bool: # pylint: disable=unused-argument 48 | return False 49 | 50 | def replaces(self, other: _t.Any) -> bool: # pylint: disable=unused-argument 51 | return True 52 | 53 | 54 | class UnknownSource(DeferredSource): 55 | def approx_size(self) -> int: 56 | return 8 57 | 58 | def show_source(self) -> str: 59 | return f"" 60 | 61 | def get_fileobj(self) -> _io.BufferedReader: 62 | raise NotImplementedError() 63 | 64 | def get_bytes(self) -> bytes: 65 | raise NotImplementedError() 66 | 67 | 68 | class _BytesIOReader(_io.BytesIO): 69 | def peek(self, size: int = 0) -> bytes: 70 | return self.getvalue()[self.tell() : size] 71 | 72 | 73 | BytesIOReader = _t.cast(_t.Callable[[bytes], _io.BufferedReader], _BytesIOReader) 74 | 75 | 76 | @_dc.dataclass 77 | class BytesSource(DeferredSource): 78 | data: bytes 79 | 80 | def approx_size(self) -> int: 81 | return 16 + len(self.data) 82 | 83 | def show_source(self) -> str: 84 | return f"" 85 | 86 | def get_fileobj(self) -> _io.BufferedReader: 87 | return BytesIOReader(self.data) 88 | 89 | def get_bytes(self) -> bytes: 90 | return self.data 91 | 92 | def replaces(self, other: DeferredSource) -> bool: 93 | if isinstance(other, BytesSource) and self.data == other.data: 94 | return False 95 | return True 96 | 97 | 98 | @_dc.dataclass 99 | class FileSource(DeferredSource): 100 | path: str | bytes 101 | st_mtime_ns: int 102 | st_dev: int 103 | st_ino: int 104 | 105 | def approx_size(self) -> int: 106 | return 40 + len(self.path) 107 | 108 | def show_source(self) -> str: 109 | return _fsdecode(self.path) 110 | 111 | def get_fileobj(self) -> _io.BufferedReader: 112 | fobj = open(self.path, "rb") # pylint: disable=consider-using-with 113 | try: 114 | in_stat = _os.fstat(fobj.fileno()) 115 | if self.st_mtime_ns != in_stat.st_mtime_ns: 116 | raise Failure("`%s` changed between accesses", self.path) 117 | except Exception: 118 | try: 119 | fobj.close() 120 | except Exception: 121 | pass 122 | raise 123 | return fobj 124 | 125 | def same_as(self, other: DeferredSource) -> bool: 126 | if ( 127 | isinstance(other, FileSource) 128 | and self.st_ino != 0 129 | and other.st_ino != 0 130 | and self.st_dev == other.st_dev 131 | and self.st_ino == other.st_ino 132 | ): 133 | # same source file inode 134 | return True 135 | return False 136 | 137 | def replaces(self, other: DeferredSource) -> bool: 138 | if isinstance(other, FileSource) and self.path == other.path: 139 | return False 140 | return True 141 | 142 | 143 | def make_FileSource(path: str | bytes, in_stat: _os.stat_result) -> FileSource: 144 | return FileSource(path, in_stat.st_mtime_ns, in_stat.st_dev, in_stat.st_ino) 145 | 146 | 147 | DeferredSourceType = _t.TypeVar("DeferredSourceType", bound=DeferredSource) 148 | 149 | 150 | @_dc.dataclass 151 | class StreamElementSource(DeferredSource, _t.Generic[DeferredSourceType]): 152 | stream_source: DeferredSourceType 153 | num: int 154 | 155 | def approx_size(self) -> int: 156 | return 24 + self.stream_source.approx_size() 157 | 158 | def show_source(self) -> str: 159 | return self.stream_source.show_source() + "//" + str(self.num) 160 | 161 | def get_fileobj(self) -> _io.BufferedReader: 162 | raise NotImplementedError() 163 | 164 | def get_bytes(self) -> bytes: 165 | raise NotImplementedError() 166 | 167 | def replaces(self, other: DeferredSource) -> bool: 168 | if ( 169 | isinstance(other, StreamElementSource) 170 | and self.stream_source == other.stream_source 171 | and self.num == other.num 172 | ): 173 | return False 174 | return True 175 | -------------------------------------------------------------------------------- /tool/hoardy_web/static.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024 Jan Malakhovski 2 | # 3 | # This file is a part of `hoardy-web` project. 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | """Templates for the generated pages.""" 19 | 20 | style = """ 21 | html { background-color: #eee; font-family: sans-serif; } 22 | body { background-color: #fff; border: 1px solid #ddd; padding: 15px; margin: 15px; } 23 | a, code { overflow-wrap: anywhere; } 24 | pre, code { background-color: #eee; border: 1px solid #ddd; padding: 5px; } 25 | ul { margin: 10px; } 26 | .right { float: right; } 27 | """ 28 | 29 | locate_page_stpl = """ 30 | 31 | 32 | 33 | 34 | %if matching: 35 | hoardy-web: {{visits_total}} visits to {{len(url_visits)}} URLs matching {{selector}} and {{pattern}} 36 | %elif visits_total > 0: 37 | hoardy-web: Not Found, but have {{visits_total}} visits to {{len(url_visits)}} URLs matching {{pattern}} 38 | %else: 39 | hoardy-web: Not Found 40 | %end 41 | 42 | 43 | 44 | %if matching: 45 |

Between {{start}} and {{end}}, matching {{pattern}}

46 | %else: 47 |

Not found {{pretty_net_url}} in the index

48 |

Either it was not archived yet or hoardy-web serve was invoked without indexing a location containing archives of this URL.

49 |

You can try visiting it. That usually helps.

50 |

Similar URLs in the index, matching {{pattern}}

51 | %end 52 | %if visits_total > 0: 53 |
    54 | %for net_url, pretty_net_url, visits in url_visits: 55 |
  • {{pretty_net_url}} [visit it again] 56 |
      57 | %for v in visits: 58 |
    • @[{{v}}]
      • 59 | %end 60 |
    61 |
    • 62 | %end 63 |
64 | %else: 65 |

(Only tumble-weed blown around by winds can be seen here.)

66 | %end 67 | 68 | 69 | """.replace( 70 | "@STYLE@", style 71 | ) 72 | -------------------------------------------------------------------------------- /tool/hoardy_web/tracking.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023-2024 Jan Malakhovski 2 | # 3 | # This file is a part of `hoardy-web` project. 4 | # 5 | # This program is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # This program is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | 18 | """Tracking memory consumption and counting seen strings.""" 19 | 20 | import collections as _c 21 | import dataclasses as _dc 22 | import typing as _t 23 | 24 | 25 | @_dc.dataclass 26 | class Memory: 27 | consumption: int = 0 28 | 29 | 30 | mem = Memory() 31 | 32 | 33 | @_dc.dataclass 34 | class SeenCounter(_t.Generic[_t.AnyStr]): 35 | _state: _c.OrderedDict[_t.AnyStr, int] = _dc.field(default_factory=_c.OrderedDict) 36 | 37 | def __len__(self) -> int: 38 | return len(self._state) 39 | 40 | def count(self, value: _t.AnyStr) -> int: 41 | try: 42 | count = self._state[value] 43 | except KeyError: 44 | self._state[value] = 0 45 | mem.consumption += 16 + len(value) 46 | return 0 47 | count += 1 48 | self._state[value] = count 49 | return count 50 | 51 | def pop(self) -> tuple[_t.AnyStr, int]: 52 | res = self._state.popitem(False) 53 | value, _ = res 54 | mem.consumption -= 16 + len(value) 55 | return res 56 | -------------------------------------------------------------------------------- /tool/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | [tool.setuptools] 5 | packages = ["hoardy_web"] 6 | [project] 7 | name = "hoardy-web" 8 | version = "0.23.0" 9 | authors = [{ name = "Jan Malakhovski", email = "oxij@oxij.org" }] 10 | description = "Inspect, search, organize, programmatically extract values and generate static website mirrors from, archive, view, and replay `HTTP` archives/dumps in `WRR` (\"Web Request+Response\", produced by the `Hoardy-Web` Web Extension browser add-on) and `mitmproxy` (`mitmdump`) file formats." 11 | readme = "README.md" 12 | license = { text = "GPL-3.0-or-later" } 13 | classifiers = [ 14 | "Development Status :: 4 - Beta", 15 | "Programming Language :: Python :: 3", 16 | "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", 17 | "Intended Audience :: End Users/Desktop", 18 | "Topic :: Internet :: WWW/HTTP", 19 | "Topic :: Internet :: WWW/HTTP :: Indexing/Search", 20 | "Topic :: System :: Archiving", 21 | "Topic :: System :: Archiving :: Backup", 22 | "Topic :: System :: Archiving :: Mirroring", 23 | "Topic :: System :: Logging", 24 | "Topic :: Internet :: Log Analysis", 25 | "Operating System :: POSIX", 26 | "Environment :: Console", 27 | ] 28 | keywords = [ 29 | "HTTP", "HTTPS", 30 | "WWW", "web", "browser", 31 | "site", "website", 32 | "download", "archive", "mirror", 33 | "wayback", "wayback machine", 34 | ] 35 | requires-python = ">=3.11" 36 | dependencies = [ 37 | "kisstdlib==0.0.11", 38 | "cbor2", 39 | "idna", 40 | "html5lib", 41 | "tinycss2>=1.3.0", 42 | "bottle", 43 | ] 44 | [project.optional-dependencies] 45 | mitmproxy = [ 46 | "mitmproxy>=5.0", 47 | ] 48 | [project.scripts] 49 | hoardy-web= "hoardy_web.__main__:main" 50 | wrrarms = "hoardy_web.__main__:main" 51 | [project.urls] 52 | "Homepage" = "https://oxij.org/software/hoardy-web/" 53 | "GitHub" = "https://github.com/Own-Data-Privateer/hoardy-web" 54 | "Support Development" = "https://oxij.org/#support" 55 | 56 | [tool.mypy] 57 | python_version = "3.11" 58 | strict = true 59 | implicit_reexport = true 60 | explicit_package_bases = true 61 | files = [ 62 | "*.py", 63 | "hoardy_web/**/*.py" 64 | ] 65 | [[tool.mypy.overrides]] 66 | module = [ 67 | "setuptools", 68 | "cbor2", 69 | "cbor2.*", 70 | "html5lib.*", 71 | "tinycss2.*", 72 | "bottle", 73 | # optional 74 | "mitmproxy", 75 | "mitmproxy.*", 76 | ] 77 | ignore_missing_imports = true 78 | 79 | [tool.pytest.ini_options] 80 | minversion = "6.0" 81 | addopts = "-s -ra -v" 82 | testpaths = [ 83 | "hoardy_web/__main__.py" 84 | ] 85 | 86 | [tool.black] 87 | line-length = 100 88 | 89 | [tool.pylint] 90 | disable = [ 91 | # `mypy` checks these more precisely 92 | "arguments-renamed", 93 | "inconsistent-return-statements", 94 | "no-member", 95 | "possibly-used-before-assignment", 96 | 97 | # `kisstdlib` uses this 98 | "raising-format-tuple", 99 | 100 | # annoying 101 | "dangerous-default-value", 102 | "global-statement", 103 | "import-outside-toplevel", 104 | "invalid-name", 105 | "line-too-long", 106 | "too-few-public-methods", 107 | "too-many-arguments", 108 | "too-many-boolean-expressions", 109 | "too-many-branches", 110 | "too-many-instance-attributes", 111 | "too-many-lines", 112 | "too-many-locals", 113 | "too-many-nested-blocks", 114 | "too-many-positional-arguments", 115 | "too-many-public-methods", 116 | "too-many-return-statements", 117 | "too-many-statements", 118 | 119 | # enable eventually 120 | "broad-exception-caught", 121 | "fixme", 122 | "missing-class-docstring", 123 | "missing-function-docstring", 124 | "unused-wildcard-import", 125 | "wildcard-import", 126 | ] 127 | [tool.pylint.format] 128 | max-line-length = "100" 129 | -------------------------------------------------------------------------------- /tool/sanity.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | black $1 . 4 | mypy 5 | pytest -k 'not slow' 6 | pylint . 7 | ./update-readme.sh 8 | -------------------------------------------------------------------------------- /tool/script/README.org: -------------------------------------------------------------------------------- 1 | #+TITLE: Example scripts using hoardy 2 | 3 | This directory contains a bunch of example scripts built on top of =hoardy-web=. 4 | 5 | * [[./hoardy-web-xdg-open]] 6 | 7 | #+BEGIN_SRC shell :results output :exports both 8 | ./hoardy-web-xdg-open --help 9 | #+END_SRC 10 | 11 | #+RESULTS: 12 | #+begin_example 13 | usage: ./hoardy-web-xdg-open [--help] WRR_FILE 14 | 15 | Open `response.body` of the given WRR file with `xdg-open`. 16 | 17 | This script only works for Debian's `xdg-open`, which waits for the child process to finish. 18 | 19 | # Options: 20 | 21 | --help print this message and exit 22 | 23 | # Example: 24 | 25 | ./hoardy-web-xdg-open path/to/wrr/file.wrr 26 | #+end_example 27 | 28 | * [[./hoardy-web-xdg-open-mimi]] 29 | 30 | #+BEGIN_SRC shell :results output :exports both 31 | ./hoardy-web-xdg-open-mimi --help 32 | #+END_SRC 33 | 34 | #+RESULTS: 35 | #+begin_example 36 | usage: ./hoardy-web-xdg-open-mimi [--help] WRR_FILE 37 | 38 | Open `response.body` of the given WRR file using Mimi's `xdg-open`. 39 | 40 | This will also work for other similar `xdg-open` scripts that do not wait for their child process to finish unless you specify `--wait`. 41 | 42 | # Options: 43 | 44 | --help print this message and exit 45 | 46 | # Example: 47 | 48 | ./hoardy-web-xdg-open-mimi path/to/wrr/file.wrr 49 | #+end_example 50 | 51 | * [[./hoardy-web-view-w3m]] 52 | 53 | #+BEGIN_SRC shell :results output :exports both 54 | ./hoardy-web-view-w3m --help 55 | #+END_SRC 56 | 57 | #+RESULTS: 58 | #+begin_example 59 | usage: ./hoardy-web-view-w3m [--help] WRR_FILE 60 | 61 | Generate a plain text preview of a WRR file containing an HTML document using `w3m`. 62 | 63 | # Options: 64 | 65 | --help print this message and exit 66 | 67 | --raw-url print the raw URL stored in the WRR_FILE 68 | --net-url format the URL using on-the-wire representation 69 | --pretty-url format the URL prettily; default 70 | --normalized-url normalize the URL removing empty query parameters, and then format the URL prettily 71 | 72 | # Example: 73 | 74 | ./hoardy-web-view-w3m path/to/wrr/file/containing/html.wrr 75 | #+end_example 76 | 77 | * [[./hoardy-web-view-pandoc]] 78 | 79 | #+BEGIN_SRC shell :results output :exports both 80 | ./hoardy-web-view-pandoc --help 81 | #+END_SRC 82 | 83 | #+RESULTS: 84 | #+begin_example 85 | usage: ./hoardy-web-view-pandoc [--help] WRR_FILE 86 | 87 | Generate a plain text preview of a WRR file containing an HTML document using `pandoc`. 88 | 89 | # Options: 90 | 91 | --help print this message and exit 92 | 93 | --raw-url print the raw URL stored in the WRR_FILE 94 | --net-url format the URL using on-the-wire representation 95 | --pretty-url format the URL prettily; default 96 | --normalized-url normalize the URL removing empty query parameters, and then format the URL prettily 97 | 98 | -t FORMAT, --to FORMAT 99 | `pandoc` output format: default: `plain` 100 | 101 | # Example: 102 | 103 | ./hoardy-web-view-pandoc path/to/wrr/file/containing/html.wrr 104 | #+end_example 105 | 106 | * [[./hoardy-web-spd-say]] 107 | 108 | #+BEGIN_SRC shell :results output :exports both 109 | ./hoardy-web-spd-say --help 110 | #+END_SRC 111 | 112 | #+RESULTS: 113 | #+begin_example 114 | usage: ./hoardy-web-spd-say [--help] [--dry-run] [-s pattern] [-e pattern] [WRRFILE ...] 115 | 116 | Feed (a part of) an HTML document containted in a given WRR file to a text-to-speech (TTS) engine. 117 | 118 | This depends on `pandoc`, `sed`, and `spd-say` of `speech-dispatcher`. 119 | 120 | The latter of which is a speech server that provides a single common API for whole lot of different TTS engines. 121 | Configuring your `speech-dispatcher` is out of scope of this script, look it up elsewhere. 122 | 123 | # Options: 124 | 125 | --help print this message and exit 126 | 127 | -s PATTER, --start PATTERN 128 | start speaking starting from this PATTERN 129 | 130 | -e PATTER, --end PATTERN 131 | stop speaking at this PATTERN 132 | 133 | --dry-run just print the text that would be fed to the TTS to 134 | stdout, without actually running `spd-say` 135 | 136 | # Note: 137 | 138 | `--start` and `--end` run `sed -n "$start,$ p"` and similar commands internally. 139 | Which is why see `man 1 sed` for more info about PATTERN syntax. 140 | 141 | # Examples: 142 | 143 | - Feed the whole document to the TTS: 144 | 145 | ./hoardy-web-spd-say path/to/wrr/file/containing/html.wrr 146 | 147 | - Skip first 5 lines, then feed the next 100 lines to the TTS: 148 | 149 | ./hoardy-web-spd-say -s 5 -e +100 path/to/wrr/file/containing/html.wrr 150 | 151 | - Start speaking aloud starting from the first `
` element: 152 | 153 | ./hoardy-web-spd-say -s "/^-----/" path/to/wrr/file/containing/html.wrr 154 | 155 | - Feed everything between the first two `
` elements to the TTS: 156 | 157 | ./hoardy-web-spd-say -s "/^-----/" -e "/^-----/" path/to/wrr/file/containing/html.wrr 158 | 159 | - Feed everything between the first "Chapter" header and the following "Next Chapter" link to the TTS: 160 | 161 | ./hoardy-web-spd-say -s "/^Chapter [0-9]/" -e "/^Next Chapter/" path/to/wrr/file/containing/html.wrr 162 | 163 | #+end_example 164 | -------------------------------------------------------------------------------- /tool/script/hoardy-web-spd-say: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | usage() { 4 | cat << EOF 5 | usage: $0 [--help] [--dry-run] [-s pattern] [-e pattern] [WRRFILE ...] 6 | 7 | Feed (a part of) an HTML document containted in a given WRR file to a text-to-speech (TTS) engine. 8 | 9 | This depends on \`pandoc\`, \`sed\`, and \`spd-say\` of \`speech-dispatcher\`. 10 | 11 | The latter of which is a speech server that provides a single common API for whole lot of different TTS engines. 12 | Configuring your \`speech-dispatcher\` is out of scope of this script, look it up elsewhere. 13 | 14 | # Options: 15 | 16 | --help print this message and exit 17 | 18 | -s PATTER, --start PATTERN 19 | start speaking starting from this PATTERN 20 | 21 | -e PATTER, --end PATTERN 22 | stop speaking at this PATTERN 23 | 24 | --dry-run just print the text that would be fed to the TTS to 25 | stdout, without actually running \`spd-say\` 26 | 27 | # Note: 28 | 29 | \`--start\` and \`--end\` run \`sed -n "\$start,$end p"\` and similar commands internally. 30 | Which is why see \`man 1 sed\` for more info about PATTERN syntax. 31 | 32 | # Examples: 33 | 34 | - Feed the whole document to the TTS: 35 | 36 | $0 path/to/wrr/file/containing/html.wrr 37 | 38 | - Skip first 5 lines, then feed the next 100 lines to the TTS: 39 | 40 | $0 -s 5 -e +100 path/to/wrr/file/containing/html.wrr 41 | 42 | - Start speaking aloud starting from the first \`
\` element: 43 | 44 | $0 -s "/^-----/" path/to/wrr/file/containing/html.wrr 45 | 46 | - Feed everything between the first two \`
\` elements to the TTS: 47 | 48 | $0 -s "/^-----/" -e "/^-----/" path/to/wrr/file/containing/html.wrr 49 | 50 | - Feed everything between the first "Chapter" header and the following "Next Chapter" link to the TTS: 51 | 52 | $0 -s "/^Chapter [0-9]/" -e "/^Next Chapter/" path/to/wrr/file/containing/html.wrr 53 | 54 | EOF 55 | } 56 | 57 | start=0 58 | end='$' 59 | dry= 60 | while (($# > 0)); do 61 | case "$1" in 62 | --help) usage; exit 0 ;; 63 | -s|--start) start="$2"; shift ;; 64 | -e|--end) end="$2"; shift ;; 65 | --dry-run) dry=1 ;; 66 | *) break ;; 67 | esac 68 | shift 69 | done 70 | 71 | play() { 72 | { [[ "$start" != '0' ]] && sed -n "$start"',$ p' || cat - ; } | \ 73 | { [[ "$end" != '$' ]] && sed "$end"',$ d' || cat - ; } | \ 74 | { 75 | if [[ -z "$dry" ]]; then 76 | spd-say -ew 77 | else 78 | cat 79 | fi 80 | } 81 | } 82 | 83 | # we need this so that other messages don't get interrupted on success 84 | ok= 85 | # stop talking immediately on interrupt 86 | trap '[[ -z "$ok" ]] && spd-say -S' 0 87 | 88 | for file in "$@"; do 89 | echo "reading $file aloud from $start to $end" 90 | hoardy-web run -- pandoc -f html -t plain "$file" | play 91 | done 92 | ok=1 93 | -------------------------------------------------------------------------------- /tool/script/hoardy-web-view-pandoc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | usage() { 5 | cat << EOF 6 | usage: $0 [--help] WRR_FILE 7 | 8 | Generate a plain text preview of a WRR file containing an HTML document using \`pandoc\`. 9 | 10 | # Options: 11 | 12 | --help print this message and exit 13 | 14 | --raw-url print the raw URL stored in the WRR_FILE 15 | --net-url format the URL using on-the-wire representation 16 | --pretty-url format the URL prettily; default 17 | --normalized-url normalize the URL removing empty query parameters, and then format the URL prettily 18 | 19 | -t FORMAT, --to FORMAT 20 | \`pandoc\` output format: default: \`plain\` 21 | 22 | # Example: 23 | 24 | $0 path/to/wrr/file/containing/html.wrr 25 | EOF 26 | } 27 | 28 | kind=pretty_url 29 | to=plain 30 | while (($# > 0)); do 31 | case "$1" in 32 | --help) usage; exit 0 ;; 33 | --raw-url) kind=raw_url ;; 34 | --net-url) kind=net_url ;; 35 | --pretty-url) kind=pretty_url ;; 36 | --normalized-url) kind=pretty_nurl ;; 37 | --to) to="$2"; shift ;; 38 | *) break ;; 39 | esac 40 | shift 41 | done 42 | 43 | (($# > 0)) || { echo "error: need a WRR_FILE"; echo; usage; exit 1; } >&2 44 | 45 | trap '[[ -n "$tmp" ]] && rm -f "$tmp"' 0 46 | tmp=$(mktemp hoardy-web-view-pandoc.XXXXXXXX.html) 47 | 48 | hoardy-web get -l \ 49 | -e "$kind|add_prefix '# url: '" \ 50 | --expr-fd 3 \ 51 | -e "response.body|eb" \ 52 | -- "$1" 3> "$tmp" 53 | echo 54 | pandoc -f html -t "$to" --wrap=none "$tmp" 55 | -------------------------------------------------------------------------------- /tool/script/hoardy-web-view-w3m: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | usage() { 5 | cat << EOF 6 | usage: $0 [--help] WRR_FILE 7 | 8 | Generate a plain text preview of a WRR file containing an HTML document using \`w3m\`. 9 | 10 | # Options: 11 | 12 | --help print this message and exit 13 | 14 | --raw-url print the raw URL stored in the WRR_FILE 15 | --net-url format the URL using on-the-wire representation 16 | --pretty-url format the URL prettily; default 17 | --normalized-url normalize the URL removing empty query parameters, and then format the URL prettily 18 | 19 | # Example: 20 | 21 | $0 path/to/wrr/file/containing/html.wrr 22 | EOF 23 | } 24 | 25 | kind=pretty_url 26 | while (($# > 0)); do 27 | case "$1" in 28 | --help) usage; exit 0 ;; 29 | --raw-url) kind=raw_url ;; 30 | --net-url) kind=net_url ;; 31 | --pretty-url) kind=pretty_url ;; 32 | --normalized-url) kind=pretty_nurl ;; 33 | *) break ;; 34 | esac 35 | shift 36 | done 37 | 38 | (($# > 0)) || { echo "error: need a WRR_FILE"; echo; usage; exit 1; } >&2 39 | 40 | trap '[[ -n "$tmp" ]] && rm -f "$tmp"' 0 41 | tmp=$(mktemp hoardy-web-view-w3m.XXXXXXXX.html) 42 | 43 | hoardy-web get -l \ 44 | -e "$kind|add_prefix '# url: '" \ 45 | --expr-fd 3 \ 46 | -e "response.body|eb" \ 47 | -- "$1" 3> "$tmp" 48 | echo 49 | w3m -T text/html -dump "$tmp" 50 | -------------------------------------------------------------------------------- /tool/script/hoardy-web-xdg-open: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | usage() { 5 | cat << EOF 6 | usage: $0 [--help] WRR_FILE 7 | 8 | Open \`response.body\` of the given WRR file with \`xdg-open\`. 9 | 10 | This script only works for Debian's \`xdg-open\`, which waits for the child process to finish. 11 | 12 | # Options: 13 | 14 | --help print this message and exit 15 | 16 | # Example: 17 | 18 | $0 path/to/wrr/file.wrr 19 | EOF 20 | } 21 | 22 | while (($# > 0)); do 23 | case "$1" in 24 | --help) usage; exit 0 ;; 25 | *) break ;; 26 | esac 27 | shift 28 | done 29 | 30 | (($# > 0)) || { echo "error: need a WRR_FILE"; echo; usage; exit 1; } >&2 31 | 32 | exec hoardy-web run -- xdg-open "$@" 33 | -------------------------------------------------------------------------------- /tool/script/hoardy-web-xdg-open-mimi: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | usage() { 5 | cat << EOF 6 | usage: $0 [--help] WRR_FILE 7 | 8 | Open \`response.body\` of the given WRR file using Mimi's \`xdg-open\`. 9 | 10 | This will also work for other similar \`xdg-open\` scripts that do not wait for their child process to finish unless you specify \`--wait\`. 11 | 12 | # Options: 13 | 14 | --help print this message and exit 15 | 16 | # Example: 17 | 18 | $0 path/to/wrr/file.wrr 19 | EOF 20 | } 21 | 22 | while (($# > 0)); do 23 | case "$1" in 24 | --help) usage; exit 0 ;; 25 | *) break ;; 26 | esac 27 | shift 28 | done 29 | 30 | (($# > 0)) || { echo "error: need a WRR_FILE"; echo; usage; exit 1; } >&2 31 | 32 | exec hoardy-web run -- xdg-open --wait "$1" 33 | -------------------------------------------------------------------------------- /tool/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Setup.""" 3 | from setuptools import setup 4 | 5 | setup() 6 | -------------------------------------------------------------------------------- /tool/update-readme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | echo '$table-of-contents$' > toc.template 4 | for i in 0 1; do 5 | { 6 | echo "# Table of Contents" 7 | echo "
(Click me to see it.)" 8 | pandoc --wrap=none --toc --template=toc.template --toc-depth=5 -M title=toc -f markdown -t html README.md \ 9 | | sed '/Table of Contents/ d; s%%%' 10 | echo "
" 11 | echo 12 | 13 | sed -n "/# What is/,/# Usage/ p" README.md 14 | echo 15 | 16 | python3 -m hoardy_web --help --markdown | sed ' 17 | s/^\(#\+\) /#\1 / 18 | s/^\(#\+\) \(hoardy-web[^[({]*\) [[({].*/\1 \2/ 19 | ' 20 | 21 | echo 22 | 23 | ./test-tool.sh --help | sed ' 24 | s/^# usage: \(.*\)$/# Development: `\1`/ 25 | ' 26 | } > README.new 27 | mv README.new README.md 28 | done 29 | pandoc -f markdown -t html README.md > README.html 30 | -------------------------------------------------------------------------------- /update-changelog.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | sed -n "0,/^\[/ p" CHANGELOG.md | head -n -1 > CHANGELOG.new 6 | 7 | { 8 | emit() { 9 | echo "## [$1] - $4" >&4 10 | 11 | if [[ -z $3 ]]; then 12 | echo "[$1]: https://github.com/Own-Data-Privateer/hoardy-web/releases/tag/$2" 13 | else 14 | echo "[$1]: https://github.com/Own-Data-Privateer/hoardy-web/compare/$3...$2" 15 | fi 16 | } 17 | 18 | prev_extension= 19 | prev_tool= 20 | prev_simple_server= 21 | git tag --sort=-refname --sort=taggerdate --format '%(taggerdate:short) %(subject) %(refname:short)' | while IFS= read -r -d $'\n' line ; do 22 | refname=${line##* } 23 | date=${line%% *} 24 | title=$line 25 | title=${title#* } 26 | title=${title% *} 27 | title=$(sed 's/ version /-v/' <<< "$title") 28 | case "$refname" in 29 | extension-*) 30 | emit "$title" "$refname" "$prev_extension" "$date" 31 | prev_extension="$refname" 32 | ;; 33 | tool-v0.15.5) 34 | # skip these 35 | continue 36 | ;; 37 | tool-*) 38 | emit "$title" "$refname" "$prev_tool" "$date" 39 | prev_tool="$refname" 40 | ;; 41 | dumb_server-*|simple_server-*) 42 | emit "$title" "$refname" "$prev_simple_server" "$date" 43 | prev_simple_server="$refname" 44 | ;; 45 | esac 46 | done 47 | } 4> CHANGELOG.spine.rnew | tac >> CHANGELOG.new 48 | 49 | { 50 | echo 51 | sed -n "/^# TODO/,$ p" CHANGELOG.md 52 | } >> CHANGELOG.new 53 | 54 | { 55 | echo "# Changelog" 56 | cat CHANGELOG.spine.rnew | tac 57 | } >> CHANGELOG.spine.new 58 | sed -n '/^# TODO/,$ d; /^##\? / p' CHANGELOG.md | sed 's/^\(## [^:]*\): .*/\1/g' > CHANGELOG.spine.old 59 | diff -u CHANGELOG.spine.old CHANGELOG.spine.new || true 60 | rm CHANGELOG.spine.* 61 | 62 | mv CHANGELOG.new CHANGELOG.md 63 | -------------------------------------------------------------------------------- /update-docs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | ./update-changelog.sh 4 | ./update-readme.sh 5 | (cd extension; ./update-readme.sh) 6 | (cd tool; ./update-readme.sh) 7 | (cd simple_server; ./update-readme.sh) 8 | -------------------------------------------------------------------------------- /update-readme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | 3 | echo '$table-of-contents$' > toc.template 4 | for i in 0 1; do 5 | { 6 | echo "# Table of Contents" 7 | echo "
(Click me to see it.)" 8 | pandoc --wrap=none --toc --template=toc.template -M title=toc -f markdown -t html README.md \ 9 | | sed '/Table of Contents/ d; s%%%' 10 | echo "
" 11 | echo 12 | 13 | sed -n "/# What is/,$ p" README.md 14 | } > README.new 15 | mv README.new README.md 16 | done 17 | pandoc -f markdown -t html README.md > README.html 18 | --------------------------------------------------------------------------------