├── .gitignore
└── worker
    └── src
        ├── wrangler.toml
        └── index.js


/.gitignore:
--------------------------------------------------------------------------------
1 | .wrangler
2 | node_modules
3 | 


--------------------------------------------------------------------------------
/worker/src/wrangler.toml:
--------------------------------------------------------------------------------
 1 | name = "indieseas-worker"
 2 | main = "src/index.js"
 3 | compatibility_date = "2025-10-10"
 4 | type = "javascript"
 5 | 
 6 | [dev]
 7 | port = 8787
 8 | 
 9 | [vars]
10 | SERVER_API = "https://api.indieseas.net"
11 | API_TOKEN = "CHANGE_ME6969colon-three"
12 | LOG_LEVEL = "info"
13 | 
14 | [env.dev.vars]
15 | SERVER_API = "http://localhost:3000"
16 | LOG_LEVEL = "debug"
17 | API_TOKEN = "CHANGE_ME6969colon-three"


--------------------------------------------------------------------------------
/worker/src/index.js:
--------------------------------------------------------------------------------
  1 | let cachedUrlList = null;
  2 | let lastCacheTime = 0;
  3 | let DEFAULT_API_KEY = "CHANGE_ME6969colon-three";
  4 | 
  5 | export default {
  6 |   async fetch(req, env) {
  7 |     try {
  8 |       const { searchParams } = new URL(req.url);
  9 | 
 10 |       const authHeader = req.headers.get("Authorization") || "";
 11 |       const apiKey = env.API_KEY || DEFAULT_API_KEY;
 12 | 
 13 |       if (authHeader !== `Bearer ${apiKey}`) {
 14 |         return new Response("Unauthorized: invalid API key", { status: 401 });
 15 |       }
 16 | 
 17 |       const target = searchParams.get("url");
 18 |       if (!target) return new Response("Missing ?url=", { status: 400 });
 19 | 
 20 |       const base = new URL(target);
 21 | 
 22 |       const now = Date.now();
 23 |       if (!cachedUrlList || now - lastCacheTime > 15 * 60 * 1000) {
 24 |         const listRes = await fetch(`${env.SERVER_API}/api/url-list`, {
 25 |           headers: { "Authorization": `Bearer ${env.API_TOKEN}` },
 26 |         });
 27 | 
 28 |         if (!listRes.ok)
 29 |           throw new Error(`Failed to fetch URL list: ${listRes.status}`);
 30 | 
 31 |         cachedUrlList = new Set(await listRes.json());
 32 |         lastCacheTime = now;
 33 |       }
 34 | 
 35 |       const htmlRes = await fetch(base.href, { redirect: "follow" });
 36 |       if (!htmlRes.ok)
 37 |         throw new Error(`Failed to fetch: ${htmlRes.status}`);
 38 |       const html = await htmlRes.text();
 39 | 
 40 |       const assetRegexes = {
 41 |         scripts: /<script[^>]+src=["']([^"']+)["']/gi,
 42 |         links: /<link[^>]+href=["']([^"']+)["']/gi,
 43 |         images: /<img[^>]+src=["']([^"']+)["']/gi,
 44 |         videos: /<video[^>]+src=["']([^"']+)["']/gi,
 45 |         sources: /<source[^>]+src=["']([^"']+)["']/gi,
 46 |         iframes: /<iframe[^>]+src=["']([^"']+)["']/gi,
 47 |       };
 48 | 
 49 |       const urls = new Set();
 50 |       for (const regex of Object.values(assetRegexes)) {
 51 |         let match;
 52 |         while ((match = regex.exec(html))) {
 53 |           try {
 54 |             const abs = new URL(match[1], base).href;
 55 |             urls.add(abs);
 56 |           } catch {}
 57 |         }
 58 |       }
 59 | 
 60 |       const newUrls = [...urls].filter(u => !cachedUrlList.has(u));
 61 | 
 62 |       let forwarded = 0;
 63 |       const failed = [];
 64 | 
 65 |       for (const assetUrl of newUrls) {
 66 |         try {
 67 |           const res = await fetch(assetUrl, { redirect: "follow" });
 68 | 
 69 |           await fetch(`${env.SERVER_API}/api/upload`, {
 70 |             method: "POST",
 71 |             headers: {
 72 |               "Content-Type": res.headers.get("content-type") || "application/octet-stream",
 73 |               "X-Source-Url": assetUrl,
 74 |               "X-Page-Url": base.href,
 75 |               "X-Status-Code": res.status.toString(),
 76 |               "Authorization": `Bearer ${env.API_TOKEN}`,
 77 |             },
 78 |             body: res.body,
 79 |           });
 80 | 
 81 |           if (!res.ok) {
 82 |             failed.push({ url: assetUrl, status: res.status });
 83 |           } else {
 84 |             forwarded++;
 85 |           }
 86 |         } catch (err) {
 87 |           failed.push({ url: assetUrl, error: err.message });
 88 |         }
 89 |       }
 90 | 
 91 |       const anchorRegex = /<a[^>]+href=["']([^"']+)["']/gi;
 92 |       const pages = new Set();
 93 |       let am;
 94 |       const emailLike = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/;
 95 |       while ((am = anchorRegex.exec(html))) {
 96 |         const href = am[1].trim();
 97 |         if (!href ||
 98 |         href.startsWith('#') ||
 99 |         href.startsWith('mailto:') ||
100 |         href.startsWith('tel:') ||
101 |         href.startsWith('javascript:') ||
102 |         href.startsWith('data:')) continue;
103 | 
104 |         if (emailLike.test(href) && !href.includes('://') && !href.startsWith('//')) continue;
105 | 
106 |         try {
107 |           const abs = new URL(href, base).href;
108 |           if (abs === base.href) continue;
109 |           if (urls.has(abs)) continue;
110 |           pages.add(abs);
111 |         } catch {}
112 |       }
113 | 
114 |       return Response.json({
115 |         forwarded,
116 |         skipped: urls.size - newUrls.length,
117 |         totalAssets: urls.size,
118 |         failed,
119 |         newfoundUrls: newUrls,
120 |         otherPages: [...pages],
121 |       });
122 |     } catch (err) {
123 |       return new Response(`Error: ${err.message}`, { status: 500 });
124 |     }
125 |   },
126 | };
127 | 


--------------------------------------------------------------------------------