├── .DS_Store ├── .gitignore ├── LICENSE ├── README.md └── extension ├── background.js ├── content.js ├── html2canvas.min.js ├── icon.png ├── instructions.png ├── lib ├── search-icon-512x512-dxj09ddf.png └── transformers.min.js ├── manifest.json ├── package.json ├── popup.css ├── welcome.html └── welcome.js /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ishandeshpande/x-bookmark-search/34db23d0d6fd05f705dd1845bece79b95b278093/.DS_Store -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | extension/.env 2 | extension.zip -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Ishan Deshpande 4 | Forked from https://github.com/sahil-lalani/bookmarks-wrapped 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # X Bookmark Search 2 | 3 | A Chrome extension that enables semantic search of your Twitter bookmarks using local LangChain embeddings. Find your bookmarked tweets by searching for concepts and meaning, not just keywords. 4 | 5 | ## Installation 6 | 7 | 1. Clone this repository 8 | 2. Open Chrome and navigate to `chrome://extensions/` 9 | 3. Enable "Developer mode" in the top right corner 10 | 4. Click "Load unpacked" and select the `extension` directory 11 | 12 | ## Features 13 | 14 | - Semantic search across your Twitter bookmarks 15 | - Uses LangChain and embeddings for concept-based matching 16 | - Fast local search - no cloud/API dependencies 17 | - Privacy-focused: all processing happens in your browser 18 | - Find tweets based on meaning and concepts, not just exact text matches 19 | 20 | ## How It Works 21 | 22 | 1. The extension processes your Twitter bookmarks locally 23 | 2. Creates embeddings using LangChain to understand the semantic meaning of tweets 24 | 3. When you search, it finds tweets that match the concept you're looking for 25 | 4. Results are ranked by semantic similarity to your query 26 | 27 | ## Usage 28 | 29 | 1. Go to Twitter and open your bookmarks 30 | 2. Use the search bar added by the extension 31 | 3. Enter your search query - try searching for concepts! 32 | 4. See results ranked by semantic relevance 33 | 34 | ## Project Structure 35 | 36 | - `manifest.json`: Extension configuration 37 | - `content.js`: Handles Twitter page integration and UI 38 | - `background.js`: Manages bookmark processing and search 39 | - `popup.html/js/css`: Extension popup interface 40 | 41 | ## Credit to Forked Project 42 | 43 | This project was heavily inspired by and built upon the functionality provided by the [bookmarks-wrapped](https://github.com/sahil-lalani/bookmarks-wrapped) project by [Sahil Lalani](https://github.com/sahil-lalani). Much of the core functionality, including bookmark processing, was adapted from his work. Special thanks to Sahil for laying the foundation. 44 | 45 | ## Development 46 | 47 | ```bash 48 | # Load unpacked extension in Chrome for development 49 | 1. Enable Chrome Developer Mode 50 | 2. Load the /extension directory 51 | ``` 52 | 53 | ## Privacy 54 | 55 | This extension: 56 | - Runs 100% locally in your browser 57 | - Does not send your bookmarks or searches to any server 58 | - Does not require any API keys or cloud services 59 | - Is completely open source 60 | 61 | ## Contributing 62 | 63 | Contributions are welcome! Please feel free to submit a Pull Request. 64 | 65 | ## License 66 | 67 | MIT License - see [LICENSE](LICENSE) for details 68 | -------------------------------------------------------------------------------- /extension/background.js: -------------------------------------------------------------------------------- 1 | chrome.action.onClicked.addListener((tab) => { 2 | chrome.tabs.create({ 3 | url: chrome.runtime.getURL('welcome.html') 4 | }, async (newTab) => { 5 | try { 6 | await waitForRequiredData(); 7 | console.log("Required data received, starting fetch"); 8 | 9 | const tweets = await getBookmarks(); 10 | console.log("Fetched tweets:", tweets?.length); 11 | 12 | if (tweets && tweets.length > 0) { 13 | // Store the tweets and update the UI 14 | chrome.runtime.sendMessage({ 15 | action: "bookmarksImported", 16 | count: tweets.length 17 | }); 18 | } else { 19 | chrome.runtime.sendMessage({ 20 | action: "importError", 21 | error: "No bookmarks found. Try bookmarking some tweets first!" 22 | }); 23 | } 24 | } catch (error) { 25 | console.error("Import error:", error); 26 | chrome.runtime.sendMessage({ 27 | action: "importError", 28 | error: error.message 29 | }); 30 | } 31 | }); 32 | }); 33 | 34 | const getTweetYear = (timestamp) => { 35 | return new Date(timestamp).getFullYear(); 36 | }; 37 | 38 | let isDone = false; 39 | let not2024count = 0; 40 | 41 | const getBookmarks = async (cursor = "", totalImported = 0, allTweets = []) => { 42 | return new Promise((resolve, reject) => { 43 | chrome.storage.local.get( 44 | ["cookie", "csrf", "auth", "bookmarksApiId", "features"], 45 | async (result) => { 46 | try { 47 | if (!result.cookie || !result.csrf || !result.auth || !result.bookmarksApiId || !result.features) { 48 | console.log("Missing required data:", { 49 | cookie: !!result.cookie, 50 | csrf: !!result.csrf, 51 | auth: !!result.auth, 52 | bookmarksApiId: !!result.bookmarksApiId, 53 | features: !!result.features 54 | }); 55 | reject(new Error("Missing required data")); 56 | return; 57 | } 58 | 59 | const headers = new Headers(); 60 | headers.append("Cookie", result.cookie); 61 | headers.append("X-Csrf-token", result.csrf); 62 | headers.append("Authorization", result.auth); 63 | 64 | const variables = { 65 | count: 100, 66 | cursor: cursor, 67 | includePromotedContent: false, 68 | }; 69 | 70 | const API_URL = `https://x.com/i/api/graphql/${ 71 | result.bookmarksApiId 72 | }/Bookmarks?features=${encodeURIComponent( 73 | JSON.stringify(result.features) 74 | )}&variables=${encodeURIComponent(JSON.stringify(variables))}`; 75 | 76 | console.log("Fetching bookmarks from:", API_URL); 77 | 78 | const response = await fetch(API_URL, { 79 | method: "GET", 80 | headers: headers, 81 | credentials: 'include' 82 | }); 83 | 84 | if (!response.ok) { 85 | console.error("API error:", response.status); 86 | if (response.status === 429) { 87 | console.log("Rate limited, waiting 60s before retry..."); 88 | await new Promise(resolve => setTimeout(resolve, 60000)); 89 | const retryResult = await getBookmarks(cursor, totalImported, allTweets); 90 | resolve(retryResult); 91 | return; 92 | } 93 | throw new Error(`HTTP error! status: ${response.status}`); 94 | } 95 | 96 | const data = await response.json(); 97 | console.log("Received data:", data); 98 | 99 | const entries = data.data?.bookmark_timeline_v2?.timeline?.instructions?.[0]?.entries || []; 100 | const tweetEntries = entries.filter(entry => entry.entryId.startsWith("tweet-")); 101 | const parsedTweets = tweetEntries.map(parseTweet); 102 | 103 | allTweets.push(...parsedTweets); 104 | 105 | // Store tweets in chunks to avoid storage limits 106 | const lastUpdateTime = new Date().toISOString(); 107 | const tweetChunks = []; 108 | for (let i = 0; i < allTweets.length; i += 100) { 109 | tweetChunks.push({ 110 | tweets: allTweets.slice(i, i + 100), 111 | chunkIndex: Math.floor(i / 100), 112 | totalChunks: Math.ceil(allTweets.length / 100), 113 | lastUpdated: lastUpdateTime 114 | }); 115 | } 116 | 117 | // Store each chunk 118 | for (const chunk of tweetChunks) { 119 | await new Promise((resolve) => { 120 | chrome.storage.local.set({ 121 | [`bookmarked_tweets_${chunk.chunkIndex}`]: chunk 122 | }, resolve); 123 | }); 124 | } 125 | 126 | // Store metadata 127 | await new Promise((resolve) => { 128 | chrome.storage.local.set({ 129 | bookmarked_tweets_meta: { 130 | totalTweets: allTweets.length, 131 | totalChunks: tweetChunks.length, 132 | lastUpdated: lastUpdateTime 133 | } 134 | }, resolve); 135 | }); 136 | 137 | console.log(`Stored ${allTweets.length} tweets in ${tweetChunks.length} chunks`); 138 | 139 | const nextCursor = getNextCursor(entries); 140 | if (nextCursor && parsedTweets.length > 0) { 141 | // Add delay between requests 142 | await new Promise(resolve => setTimeout(resolve, 1000)); 143 | const result = await getBookmarks(nextCursor, totalImported + parsedTweets.length, allTweets); 144 | resolve(result); 145 | } else { 146 | console.log("Completed fetching bookmarks. Total:", allTweets.length); 147 | resolve(allTweets); 148 | } 149 | } catch (error) { 150 | console.error("Error in getBookmarks:", error); 151 | reject(error); 152 | } 153 | } 154 | ); 155 | }); 156 | }; 157 | 158 | const parseTweet = (entry) => { 159 | const tweet = entry.content?.itemContent?.tweet_results?.result?.tweet || entry.content?.itemContent?.tweet_results?.result; 160 | 161 | // Safely access media, handling potential undefined values 162 | const media = tweet?.legacy?.entities?.media?.[0] || null; 163 | 164 | const getBestVideoVariant = (variants) => { 165 | if (!variants || variants.length === 0) return null; 166 | const mp4Variants = variants.filter(v => v.content_type === "video/mp4"); 167 | return mp4Variants.reduce((best, current) => { 168 | if (!best || (current.bitrate && current.bitrate > best.bitrate)) { 169 | return current; 170 | } 171 | return best; 172 | }, null); 173 | }; 174 | 175 | const getMediaInfo = (media) => { 176 | if (!media) return null; 177 | 178 | if (media.type === 'video' || media.type === 'animated_gif') { 179 | const videoInfo = tweet?.legacy?.extended_entities?.media?.[0]?.video_info; 180 | const bestVariant = getBestVideoVariant(videoInfo?.variants); 181 | return { 182 | type: media.type, 183 | source: bestVariant?.url || media.media_url_https, 184 | }; 185 | } 186 | 187 | return { 188 | type: media.type, 189 | source: media.media_url_https, 190 | }; 191 | }; 192 | 193 | const author = tweet?.core?.user_results?.result?.legacy || {}; 194 | const tweetId = tweet?.legacy?.id_str || entry.entryId.split('-')[1]; 195 | const url = `https://twitter.com/${author.screen_name}/status/${tweetId}`; 196 | 197 | return { 198 | id: entry.entryId, 199 | url: url, 200 | full_text: tweet?.legacy?.full_text, 201 | timestamp: tweet?.legacy?.created_at, 202 | media: getMediaInfo(media), 203 | author: { 204 | name: author.name, 205 | screen_name: author.screen_name, 206 | profile_image_url: author.profile_image_url_https 207 | } 208 | }; 209 | }; 210 | 211 | const getNextCursor = (entries) => { 212 | const cursorEntry = entries.find(entry => entry.entryId.startsWith("cursor-bottom-")); 213 | return cursorEntry ? cursorEntry.content.value : null; 214 | }; 215 | 216 | const waitForRequiredData = () => { 217 | return new Promise((resolve, reject) => { 218 | let retryCount = 0; 219 | const maxRetries = 50; // 5 seconds total (50 * 100ms) 220 | 221 | const checkData = () => { 222 | chrome.storage.local.get(['bookmarksApiId', 'cookie', 'csrf', 'auth', 'features'], (result) => { 223 | console.log('Checking required data:', { 224 | hasBookmarksApiId: !!result.bookmarksApiId, 225 | hasCookie: !!result.cookie, 226 | hasCsrf: !!result.csrf, 227 | hasAuth: !!result.auth, 228 | hasFeatures: !!result.features 229 | }); 230 | 231 | if (result.bookmarksApiId && result.cookie && result.csrf && result.auth && result.features) { 232 | console.log('All required data present:', { 233 | bookmarksApiId: result.bookmarksApiId, 234 | cookieLength: result.cookie.length, 235 | csrfLength: result.csrf.length, 236 | authLength: result.auth.length 237 | }); 238 | resolve(); 239 | } else { 240 | retryCount++; 241 | if (retryCount >= maxRetries) { 242 | console.log('Timed out waiting for required data'); 243 | reject(new Error('Timed out waiting for required data. Please try refreshing the page.')); 244 | return; 245 | } 246 | console.log('Missing required data, retrying in 100ms'); 247 | setTimeout(checkData, 100); 248 | } 249 | }); 250 | }; 251 | checkData(); 252 | }); 253 | }; 254 | 255 | chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { 256 | if (request.action === "exportBookmarks") { 257 | console.log("Starting bookmark export"); 258 | 259 | // Reset global state 260 | isDone = false; 261 | not2024count = 0; 262 | 263 | chrome.tabs.create({ url: "https://x.com/i/bookmarks/all" }, (newTab) => { 264 | setTimeout(async () => { 265 | try { 266 | chrome.tabs.sendMessage(newTab.id, { action: "showLoader" }); 267 | 268 | await waitForRequiredData(); 269 | console.log("Required data received, starting fetch"); 270 | 271 | const tweets = await getBookmarks(); 272 | console.log("Fetched tweets:", tweets?.length); 273 | 274 | chrome.tabs.sendMessage(newTab.id, { action: "hideLoader" }); 275 | 276 | if (tweets && tweets.length > 0) { 277 | // Send tweets to popup and wait for confirmation 278 | chrome.runtime.sendMessage({ 279 | action: "tweetsReady", 280 | tweets: tweets 281 | }, (response) => { 282 | // Only close the tab after popup confirms receipt 283 | if (response && response.status === "received") { 284 | chrome.tabs.remove(newTab.id); 285 | } 286 | }); 287 | } else { 288 | // If no tweets, show error and keep tab open 289 | chrome.tabs.sendMessage(newTab.id, { 290 | action: "hideLoader", 291 | error: "No tweets found from 2024. Try bookmarking some tweets first!" 292 | }); 293 | } 294 | } catch (error) { 295 | console.error("Export error:", error); 296 | chrome.tabs.sendMessage(newTab.id, { 297 | action: "hideLoader", 298 | error: error.message 299 | }); 300 | } 301 | }, 2000); 302 | }); 303 | 304 | return true; 305 | } 306 | 307 | if (request.action === "takeScreenshot") { 308 | 309 | const slideContent = document.querySelector('.slide-content'); 310 | if (!slideContent) { 311 | sendResponse({error: "No slide content found"}); 312 | return; 313 | } 314 | 315 | html2canvas(slideContent, { 316 | backgroundColor: '#1a1f2e', 317 | scale: 2, // Higher resolution 318 | logging: false 319 | }).then(function(canvas) { 320 | const dataURL = canvas.toDataURL("image/png", 1.0); 321 | sendResponse({imageData: dataURL}); 322 | }); 323 | 324 | return true; // Required for async response 325 | } 326 | }); 327 | 328 | chrome.webRequest.onBeforeSendHeaders.addListener( 329 | (details) => { 330 | if (!(details.url.includes("x.com") || details.url.includes("twitter.com"))) { 331 | return; 332 | } 333 | 334 | console.log("Intercepted request:", details.url); 335 | 336 | const authHeader = details.requestHeaders?.find( 337 | (header) => header.name.toLowerCase() === "authorization" 338 | ); 339 | const cookieHeader = details.requestHeaders?.find( 340 | (header) => header.name.toLowerCase() === "cookie" 341 | ); 342 | const csrfHeader = details.requestHeaders?.find( 343 | (header) => header.name.toLowerCase() === "x-csrf-token" 344 | ); 345 | 346 | if (authHeader && cookieHeader && csrfHeader) { 347 | console.log("Found all required headers"); 348 | chrome.storage.local.set({ 349 | auth: authHeader.value, 350 | cookie: cookieHeader.value, 351 | csrf: csrfHeader.value 352 | }, () => { 353 | console.log("Stored auth data in local storage"); 354 | }); 355 | } 356 | 357 | // Extract bookmarksApiId and features from Bookmarks API request 358 | if (details.url.includes("/graphql/") && details.url.includes("/Bookmarks?")) { 359 | try { 360 | // Extract bookmarksApiId from URL 361 | const bookmarksApiId = details.url.split("/graphql/")[1].split("/Bookmarks?")[0]; 362 | 363 | // Extract features from URL 364 | const url = new URL(details.url); 365 | const features = url.searchParams.get("features"); 366 | 367 | if (bookmarksApiId && features) { 368 | console.log("Found Bookmarks API data:", { bookmarksApiId }); 369 | chrome.storage.local.set({ 370 | bookmarksApiId, 371 | features: JSON.parse(decodeURIComponent(features)) 372 | }, () => { 373 | console.log("Stored Bookmarks API data in local storage"); 374 | }); 375 | } 376 | } catch (error) { 377 | console.error("Error extracting Bookmarks API data:", error); 378 | } 379 | } 380 | }, 381 | { urls: ["*://x.com/*", "*://twitter.com/*"] }, 382 | ["requestHeaders", "extraHeaders"] 383 | ); 384 | 385 | // Add onInstalled handler at the top level of the file 386 | chrome.runtime.onInstalled.addListener((details) => { 387 | if (details.reason === 'install') { 388 | chrome.storage.local.get(['wasPopupShown'], (result) => { 389 | if (!result.wasPopupShown) { 390 | chrome.tabs.create({ 391 | url: chrome.runtime.getURL('welcome.html') 392 | }); 393 | chrome.storage.local.set({ wasPopupShown: true }); 394 | } 395 | }); 396 | } 397 | }); 398 | 399 | -------------------------------------------------------------------------------- /extension/content.js: -------------------------------------------------------------------------------- 1 | console.log("Content script loaded"); 2 | 3 | // Listen for messages 4 | chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { 5 | console.log("Content script received message:", message); 6 | 7 | if (message.action === "showLoader") { 8 | console.log("Showing loader"); 9 | createLoadingIndicator(); 10 | sendResponse({ status: "loader shown" }); 11 | } 12 | 13 | if (message.action === "hideLoader") { 14 | console.log("Hiding loader"); 15 | const loader = document.getElementById('bookmarks-wrapped-loader'); 16 | if (loader) { 17 | loader.remove(); 18 | } 19 | sendResponse({ status: "loader hidden" }); 20 | } 21 | 22 | return true; // Keep the message channel open 23 | }); 24 | 25 | function createLoadingIndicator() { 26 | console.log("Creating loading indicator"); 27 | 28 | // Remove existing loader if any 29 | const existingLoader = document.getElementById('bookmarks-wrapped-loader'); 30 | if (existingLoader) { 31 | existingLoader.remove(); 32 | } 33 | 34 | const loadingDiv = document.createElement('div'); 35 | loadingDiv.id = 'bookmarks-wrapped-loader'; 36 | loadingDiv.style.cssText = ` 37 | position: fixed; 38 | top: 20px; 39 | left: 50%; 40 | transform: translateX(-50%); 41 | background: rgba(0, 0, 0, 0.8); 42 | color: white; 43 | padding: 12px 24px; 44 | border-radius: 20px; 45 | z-index: 9999; 46 | font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; 47 | display: flex; 48 | align-items: center; 49 | gap: 10px; 50 | box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15); 51 | `; 52 | 53 | const spinner = document.createElement('div'); 54 | spinner.style.cssText = ` 55 | width: 20px; 56 | height: 20px; 57 | border: 3px solid #ffffff; 58 | border-top: 3px solid transparent; 59 | border-radius: 50%; 60 | animation: spin 1s linear infinite; 61 | `; 62 | 63 | const style = document.createElement('style'); 64 | style.textContent = ` 65 | @keyframes spin { 66 | 0% { transform: rotate(0deg); } 67 | 100% { transform: rotate(360deg); } 68 | } 69 | `; 70 | document.head.appendChild(style); 71 | 72 | const text = document.createElement('span'); 73 | text.textContent = 'fetching your bookmarks (don\'t close this tab)'; 74 | 75 | loadingDiv.appendChild(spinner); 76 | loadingDiv.appendChild(text); 77 | document.body.appendChild(loadingDiv); 78 | 79 | console.log("Loading indicator created and added to page"); 80 | } 81 | 82 | // Add this to verify the script is loaded 83 | document.addEventListener('DOMContentLoaded', () => { 84 | console.log('Content script DOM ready'); 85 | }); 86 | 87 | -------------------------------------------------------------------------------- /extension/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ishandeshpande/x-bookmark-search/34db23d0d6fd05f705dd1845bece79b95b278093/extension/icon.png -------------------------------------------------------------------------------- /extension/instructions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ishandeshpande/x-bookmark-search/34db23d0d6fd05f705dd1845bece79b95b278093/extension/instructions.png -------------------------------------------------------------------------------- /extension/lib/search-icon-512x512-dxj09ddf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ishandeshpande/x-bookmark-search/34db23d0d6fd05f705dd1845bece79b95b278093/extension/lib/search-icon-512x512-dxj09ddf.png -------------------------------------------------------------------------------- /extension/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "manifest_version": 3, 3 | "name": "x bookmark search", 4 | "version": "1.0.2", 5 | "description": "a better way to search your bookmarks.", 6 | "permissions": [ 7 | "storage", 8 | "webRequest", 9 | "activeTab", 10 | "scripting" 11 | ], 12 | "host_permissions": [ 13 | "https://x.com/*", 14 | "https://twitter.com/*", 15 | "https://platform.twitter.com/*", 16 | "https://huggingface.co/*" 17 | ], 18 | "action": { 19 | "default_title": "x bookmark search" 20 | }, 21 | "background": { 22 | "service_worker": "background.js", 23 | "type": "module" 24 | }, 25 | "icons": { 26 | "16": "icon.png", 27 | "32": "icon.png", 28 | "48": "icon.png", 29 | "128": "icon.png" 30 | }, 31 | "web_accessible_resources": [ 32 | { 33 | "resources": [ 34 | "welcome.html", 35 | "welcome.js", 36 | "instructions.png", 37 | "lib/transformers.min.js", 38 | "models/*" 39 | ], 40 | "matches": [""] 41 | } 42 | ], 43 | "content_scripts": [ 44 | { 45 | "matches": ["*://x.com/*", "*://twitter.com/*"], 46 | "js": ["content.js"], 47 | "run_at": "document_end" 48 | } 49 | ], 50 | "content_security_policy": { 51 | "extension_pages": "script-src 'self' 'wasm-unsafe-eval'; object-src 'self'" 52 | } 53 | } -------------------------------------------------------------------------------- /extension/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "module" 3 | } -------------------------------------------------------------------------------- /extension/popup.css: -------------------------------------------------------------------------------- 1 | body { 2 | width: 100%; 3 | min-height: 100vh; 4 | margin: 0; 5 | padding: 20px; 6 | font-family: Arial, sans-serif; 7 | background-color: #1a1f2e; 8 | color: white; 9 | display: flex; 10 | flex-direction: column; 11 | align-items: center; 12 | justify-content: center; 13 | box-sizing: border-box; 14 | } 15 | 16 | #initial-screen { 17 | width: 100%; 18 | text-align: center; 19 | justify-content: center; 20 | flex-direction: column; 21 | align-items: center; 22 | } 23 | 24 | #initial-screen button { 25 | justify-content: center; 26 | margin: 10px auto; 27 | } 28 | 29 | .hidden { 30 | display: none !important; 31 | } 32 | 33 | h1 { 34 | font-size: 48px; 35 | margin-bottom: 10px; 36 | text-align: center; 37 | } 38 | 39 | h2 { 40 | font-size: 24px; 41 | margin-bottom: 40px; 42 | font-weight: normal; 43 | text-align: center; 44 | } 45 | 46 | h3 { 47 | font-size: 18px; 48 | font-weight: normal; 49 | text-align: center; 50 | } 51 | 52 | button { 53 | padding: 15px 30px; 54 | margin: 10px; 55 | font-size: 18px; 56 | border: none; 57 | border-radius: 25px; 58 | display: flex; 59 | align-items: center; 60 | justify-content: center; 61 | cursor: pointer; 62 | gap: 10px; 63 | } 64 | 65 | button:hover { 66 | cursor: pointer; 67 | } 68 | 69 | .primary-button { 70 | background-color: #1da1f2; 71 | color: white; 72 | } 73 | 74 | .support-button { 75 | background-color: #bf5af2; 76 | color: white; 77 | } 78 | 79 | /* Slideshow Styles */ 80 | #slideshow { 81 | width: 100%; 82 | max-width: 1200px; 83 | position: relative; 84 | height: 90vh; 85 | display: flex; 86 | flex-direction: column; 87 | justify-content: center; 88 | align-items: center; 89 | margin-bottom: 40px; 90 | } 91 | 92 | .slide { 93 | position: absolute; 94 | width: 100%; 95 | height: calc(100% - 60px); 96 | display: flex; 97 | justify-content: center; 98 | align-items: center; 99 | pointer-events: none; 100 | visibility: hidden; 101 | opacity: 0; 102 | transform: scale(0.9); 103 | transition: opacity 0.5s ease-out, transform 0.5s ease-out, visibility 0.5s; 104 | } 105 | 106 | .slide.active { 107 | pointer-events: auto; 108 | visibility: visible; 109 | opacity: 1; 110 | transform: scale(1); 111 | } 112 | 113 | .slide-content { 114 | text-align: center; 115 | padding: 20px; 116 | width: 100%; 117 | max-width: 800px; 118 | display: flex; 119 | flex-direction: column; 120 | align-items: center; 121 | justify-content: center; 122 | margin: 0 auto; 123 | } 124 | 125 | .slide-content > * { 126 | opacity: 0; 127 | transform: translateY(20px); 128 | } 129 | 130 | .slide.active .slide-content > * { 131 | opacity: 1; 132 | transform: translateY(0); 133 | transition: opacity 0.5s ease-out, transform 0.5s ease-out; 134 | } 135 | 136 | .slide.active .slide-content > *:nth-child(1) { transition-delay: 0.1s; } 137 | .slide.active .slide-content > *:nth-child(2) { transition-delay: 0.2s; } 138 | .slide.active .slide-content > *:nth-child(3) { transition-delay: 0.3s; } 139 | .slide.active .slide-content > *:nth-child(4) { transition-delay: 0.4s; } 140 | .slide.active .slide-content > *:nth-child(5) { transition-delay: 0.5s; } 141 | 142 | .big-number { 143 | font-size: 96px; 144 | font-weight: bold; 145 | margin: 40px 0; 146 | color: #1da1f2; 147 | } 148 | 149 | .navigation { 150 | position: absolute; 151 | bottom: -35px; 152 | left: 0; 153 | right: 0; 154 | display: flex; 155 | justify-content: center; 156 | align-items: center; 157 | gap: 20px; 158 | background: rgba(26, 31, 46, 0.8); 159 | padding: 10px 0; 160 | z-index: 10; 161 | } 162 | 163 | .nav-button { 164 | background: none; 165 | border: none; 166 | color: white; 167 | font-size: 24px; 168 | cursor: pointer; 169 | padding: 10px; 170 | } 171 | 172 | #slide-dots { 173 | display: flex; 174 | gap: 10px; 175 | } 176 | 177 | .dot { 178 | width: 10px; 179 | height: 10px; 180 | border-radius: 50%; 181 | background-color: rgba(255, 255, 255, 0.3); 182 | } 183 | 184 | .dot:hover { 185 | cursor: pointer; 186 | } 187 | 188 | .dot.active { 189 | background-color: #1da1f2; 190 | } 191 | 192 | #authors-list { 193 | width: 100%; 194 | display: flex; 195 | flex-direction: column; 196 | align-items: center; 197 | gap: 20px; 198 | } 199 | 200 | .author-item { 201 | width: 100%; 202 | max-width: 600px; 203 | display: grid; 204 | grid-template-columns: auto 60px 1fr; 205 | align-items: center; 206 | gap: 20px; 207 | padding: 10px; 208 | background: rgba(255, 255, 255, 0.1); 209 | border-radius: 15px; 210 | } 211 | 212 | .author-info { 213 | display: flex; 214 | flex-direction: column; 215 | gap: 4px; 216 | } 217 | 218 | .author-name { 219 | font-size: 18px; 220 | font-weight: bold; 221 | color: white; 222 | } 223 | 224 | .author-handle { 225 | font-size: 14px; 226 | color: rgba(255, 255, 255, 0.7); 227 | } 228 | 229 | .author-position { 230 | font-size: 24px; 231 | font-weight: bold; 232 | color: #1da1f2; 233 | justify-self: start; 234 | } 235 | 236 | .author-image { 237 | width: 60px; 238 | height: 60px; 239 | border-radius: 50%; 240 | object-fit: cover; 241 | } 242 | 243 | #top-authors .slide-content { 244 | width: 100%; 245 | max-width: 700px; 246 | margin: 0 auto; 247 | padding: 20px; 248 | display: flex; 249 | flex-direction: column; 250 | align-items: center; 251 | text-align: center; 252 | } 253 | 254 | #top-authors .slide-content h2 { 255 | margin-bottom: 30px; 256 | } 257 | 258 | .loader { 259 | width: 48px; 260 | height: 48px; 261 | border: 5px solid #FFF; 262 | border-bottom-color: transparent; 263 | border-radius: 50%; 264 | display: none; 265 | box-sizing: border-box; 266 | animation: rotation 1s linear infinite; 267 | } 268 | 269 | @keyframes rotation { 270 | 0% { transform: rotate(0deg); } 271 | 100% { transform: rotate(360deg); } 272 | } 273 | 274 | #tweetsContainer { 275 | width: 100%; 276 | margin-top: 20px; 277 | display: none; 278 | } 279 | 280 | .tweet { 281 | background: rgba(255, 255, 255, 0.1); 282 | border-radius: 10px; 283 | padding: 15px; 284 | margin-bottom: 15px; 285 | } 286 | 287 | .tweet-text { 288 | margin-bottom: 10px; 289 | } 290 | 291 | .tweet-media { 292 | max-width: 100%; 293 | border-radius: 8px; 294 | } 295 | 296 | .tweet-video { 297 | max-width: 100%; 298 | border-radius: 8px; 299 | } 300 | 301 | .tweet-date { 302 | color: #888; 303 | font-size: 0.9em; 304 | } 305 | 306 | .fun-fact { 307 | font-size: 20px; 308 | color: #888; 309 | margin: 20px 0; 310 | font-style: italic; 311 | } 312 | 313 | .month-count { 314 | font-size: 24px; 315 | margin: 20px 0; 316 | } 317 | 318 | #top-month { 319 | color: #1da1f2; 320 | font-size: 72px; 321 | font-weight: bold; 322 | margin: 20px 0; 323 | } 324 | 325 | #reading-fact a { 326 | text-decoration: none; 327 | color: inherit; 328 | } 329 | 330 | /* Add new spinner style for button */ 331 | .button-spinner { 332 | width: 20px; 333 | height: 20px; 334 | border: 3px solid #FFF; 335 | border-bottom-color: transparent; 336 | border-radius: 50%; 337 | box-sizing: border-box; 338 | animation: rotation 1s linear infinite; 339 | display: inline-block; 340 | margin-right: 8px; 341 | } 342 | 343 | @keyframes rotation { 344 | 0% { transform: rotate(0deg); } 345 | 100% { transform: rotate(360deg); } 346 | } 347 | 348 | .fun-buttons { 349 | display: flex; 350 | gap: 20px; 351 | justify-content: center; 352 | margin-top: 30px; 353 | } 354 | 355 | .fun-choice-button { 356 | background-color: #1da1f2; 357 | color: white; 358 | padding: 15px 40px; 359 | font-size: 20px; 360 | border-radius: 25px; 361 | border: none; 362 | cursor: pointer; 363 | transition: transform 0.2s; 364 | } 365 | 366 | .fun-choice-button:hover { 367 | transform: scale(1.05); 368 | } 369 | 370 | #intro-slide .big-number, 371 | #outro-slide .big-number { 372 | font-size: 72px; 373 | margin: 20px 0; 374 | } 375 | 376 | #outro-slide .fun-fact { 377 | font-size: 20px; 378 | margin-top: 30px; 379 | color: #888; 380 | text-align: center; 381 | width: 100%; 382 | } 383 | 384 | /* Special styling for the final slide with split layout */ 385 | #outro-slide .slide-content { 386 | display: flex; 387 | flex-direction: row; 388 | gap: 60px; 389 | max-width: 1000px; 390 | align-items: center; 391 | padding: 40px; 392 | } 393 | 394 | #outro-slide .content-left { 395 | flex: 0.4; 396 | text-align: center; 397 | display: flex; 398 | flex-direction: column; 399 | align-items: center; 400 | } 401 | 402 | #outro-slide .content-right { 403 | flex: 0.6; 404 | display: flex; 405 | flex-direction: column; 406 | align-items: center; 407 | justify-content: center; 408 | } 409 | 410 | #final-collage { 411 | width: 100%; 412 | display: flex; 413 | justify-content: center; 414 | align-items: center; 415 | margin: 0 auto; 416 | } 417 | 418 | #final-collage img { 419 | width: auto; 420 | max-width: 100%; 421 | height: auto; 422 | border-radius: 15px; 423 | box-shadow: 0 12px 36px rgba(0, 0, 0, 0.2); 424 | animation: shadowPulse 4s ease infinite; 425 | } 426 | 427 | @keyframes shadowPulse { 428 | 0% { 429 | box-shadow: 0 24px 72px rgba(0, 172, 238, 0.4); /* #00acee */ 430 | } 431 | 25% { 432 | box-shadow: 0 24px 72px rgba(29, 161, 242, 0.4); /* #1da1f2 */ 433 | } 434 | 50% { 435 | box-shadow: 0 24px 72px rgba(0, 132, 180, 0.4); /* #0084b4 */ 436 | } 437 | 75% { 438 | box-shadow: 0 24px 72px rgba(0, 172, 237, 0.4); /* #00aced */ 439 | } 440 | 100% { 441 | box-shadow: 0 24px 72px rgba(0, 172, 238, 0.4); /* back to #00acee */ 442 | } 443 | } 444 | 445 | 446 | @keyframes shine { 447 | 0% { 448 | left: -50%; 449 | } 450 | 100% { 451 | left: 150%; 452 | } 453 | } 454 | 455 | @keyframes bounce { 456 | 0%, 100% { 457 | transform: translateY(0); 458 | } 459 | 50% { 460 | transform: translateY(-10px); 461 | } 462 | } 463 | 464 | #status { 465 | margin-top: 20px; 466 | text-align: center; 467 | } 468 | 469 | /* Add medal styling for top 3 */ 470 | .author-image.gold-rank { 471 | box-shadow: 0 0 15px rgba(255, 215, 0, 0.6); 472 | border: 3px solid #FFD700; 473 | } 474 | 475 | .author-image.silver-rank { 476 | box-shadow: 0 0 15px rgba(192, 192, 192, 0.6); 477 | border: 3px solid #C0C0C0; 478 | } 479 | 480 | .author-image.bronze-rank { 481 | box-shadow: 0 0 15px rgba(205, 127, 50, 0.6); 482 | border: 3px solid #CD7F32; 483 | } 484 | 485 | #surfer-icon { 486 | margin-right: 2px; 487 | } 488 | 489 | #surfer-link { 490 | color: #1da1f2; /* Twitter blue */ 491 | } 492 | 493 | #surfer-link:hover { 494 | color: #1991da; /* Slightly darker Twitter blue for hover */ 495 | } 496 | 497 | .surfer-container { 498 | display: flex; 499 | align-items: center; 500 | justify-content: center; 501 | gap: 4px; 502 | } 503 | 504 | #surfer-icon { 505 | margin-right: 6px; /* Remove the previous margin-right */ 506 | } 507 | 508 | /* Preview Slideshow Styles */ 509 | .preview-slideshow { 510 | width: 100%; 511 | position: relative; 512 | margin: 0 auto; 513 | padding: 20px; 514 | background-color: #1a1f2e; 515 | border-radius: 20px; 516 | } 517 | 518 | .preview-image-wrapper { 519 | width: 100%; 520 | height: 400px; 521 | display: flex; 522 | align-items: center; 523 | justify-content: center; 524 | } 525 | 526 | .preview-image { 527 | width: auto; 528 | height: 100%; 529 | max-width: 100%; 530 | object-fit: contain; 531 | border-radius: 10px; 532 | transition: opacity 0.3s ease; 533 | } 534 | 535 | .preview-nav { 536 | position: absolute; 537 | top: 50%; 538 | transform: translateY(-50%); 539 | background: transparent; 540 | border: none; 541 | color: white; 542 | font-size: 24px; 543 | width: 40px; 544 | height: 40px; 545 | border-radius: 50%; 546 | cursor: pointer; 547 | display: flex; 548 | align-items: center; 549 | justify-content: center; 550 | transition: all 0.3s ease; 551 | z-index: 2; 552 | } 553 | 554 | 555 | .preview-nav.prev { 556 | left: 2px; 557 | } 558 | 559 | .preview-nav.next { 560 | right: 2px; 561 | } 562 | 563 | .preview-dots { 564 | position: absolute; 565 | bottom: -20px; 566 | left: 50%; 567 | transform: translateX(-50%); 568 | display: flex; 569 | gap: 8px; 570 | padding: 10px; 571 | } 572 | 573 | .preview-dot { 574 | width: 8px; 575 | height: 8px; 576 | border-radius: 50%; 577 | background: rgba(255, 255, 255, 0.3); 578 | cursor: pointer; 579 | transition: all 0.3s ease; 580 | } 581 | 582 | .preview-dot.active { 583 | background: #1da1f2; 584 | transform: scale(1.2); 585 | } 586 | 587 | .preview-dot:hover { 588 | background: rgba(255, 255, 255, 0.5); 589 | } -------------------------------------------------------------------------------- /extension/welcome.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | x bookmark search 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 450 | 451 | 452 |
453 |
454 |
455 | 456 |
457 |
458 |
459 |
460 |

461 | a better way to search your x bookmarks 462 |

463 |

464 | importing your bookmarks... 465 |

466 |
467 | 468 |
469 |
470 | 471 | 472 | 473 | 474 | 482 |
483 | 484 |
485 |
486 | 487 |
488 |
489 | 490 |
491 |
492 |
493 |
494 | 495 | 498 |
499 |
500 | 501 | 502 | 503 | -------------------------------------------------------------------------------- /extension/welcome.js: -------------------------------------------------------------------------------- 1 | /************************************************************ 2 | * welcome.js -- Local "Semantic-ish" Searching of Bookmarks 3 | * Uses Transformers.js for local embeddings (no API keys). 4 | ************************************************************/ 5 | 6 | import { pipeline } from './lib/transformers.min.js'; 7 | 8 | // --- UI Elements --- 9 | const searchInput = document.getElementById('search-input'); 10 | const searchResults = document.getElementById('search-results').querySelector('.tweet-grid'); 11 | const statusText = document.getElementById('status-text'); 12 | 13 | let allTweets = []; // Will store tweet objects + their embeddings 14 | let embeddingPipeline; // Pipeline for local embeddings from Transformers.js 15 | let isProcessingEmbeddings = false; 16 | const BATCH_SIZE = 10; // Increased batch size 17 | const CACHE_KEY = 'tweet_embeddings_cache'; 18 | 19 | // ---------------------------------------------------------- 20 | // 1) On Load, Check if Tweets Are Imported and Trigger Import 21 | // ---------------------------------------------------------- 22 | chrome.storage.local.get(null, async (result) => { 23 | const meta = result.bookmarked_tweets_meta; 24 | if (meta) { 25 | // Gather all tweets in memory 26 | allTweets = []; 27 | for (let i = 0; i < meta.totalChunks; i++) { 28 | const chunk = result[`bookmarked_tweets_${i}`]; 29 | if (chunk?.tweets) { 30 | allTweets.push(...chunk.tweets); 31 | } 32 | } 33 | 34 | statusText.textContent = `${meta.totalTweets} bookmarks imported`; 35 | searchInput.disabled = false; 36 | 37 | // Try to load cached embeddings first 38 | const cache = await loadEmbeddingCache(); 39 | if (cache) { 40 | // Apply cached embeddings to tweets 41 | let cacheHits = 0; 42 | allTweets.forEach(tweet => { 43 | if (cache[tweet.id]) { 44 | tweet.embedding = cache[tweet.id]; 45 | cacheHits++; 46 | } 47 | }); 48 | console.log(`Loaded ${cacheHits} embeddings from cache`); 49 | 50 | // Only build embeddings for tweets without them 51 | const tweetsNeedingEmbeddings = allTweets.filter(t => !t.embedding); 52 | if (tweetsNeedingEmbeddings.length > 0) { 53 | statusText.textContent = `Building embeddings for ${tweetsNeedingEmbeddings.length} new tweets...`; 54 | await buildAllTweetEmbeddings(tweetsNeedingEmbeddings); 55 | } else { 56 | statusText.textContent = `${allTweets.length} bookmarks ready for search`; 57 | } 58 | } else { 59 | // Build embeddings for all tweets 60 | await buildAllTweetEmbeddings(allTweets); 61 | } 62 | } else { 63 | // Automatically trigger import 64 | statusText.textContent = 'please wait while we import your bookmarks...'; 65 | chrome.runtime.sendMessage({ action: "exportBookmarks" }); 66 | } 67 | }); 68 | 69 | // Cache management functions 70 | async function loadEmbeddingCache() { 71 | try { 72 | const result = await chrome.storage.local.get(CACHE_KEY); 73 | return result[CACHE_KEY] || null; 74 | } catch (error) { 75 | console.warn('Error loading embedding cache:', error); 76 | return null; 77 | } 78 | } 79 | 80 | async function updateEmbeddingCache(tweets) { 81 | try { 82 | const cache = await loadEmbeddingCache() || {}; 83 | tweets.forEach(tweet => { 84 | if (tweet.embedding) { 85 | cache[tweet.id] = tweet.embedding; 86 | } 87 | }); 88 | await chrome.storage.local.set({ [CACHE_KEY]: cache }); 89 | } catch (error) { 90 | console.warn('Error updating embedding cache:', error); 91 | } 92 | } 93 | 94 | // ---------------------------------------------------------- 95 | // 2) Listen for Import Results 96 | // ---------------------------------------------------------- 97 | chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { 98 | if (message.action === "tweetsReady") { 99 | const count = message.tweets ? message.tweets.length : 0; 100 | allTweets = message.tweets || []; 101 | statusText.textContent = `${count} bookmarks imported`; 102 | searchInput.disabled = false; 103 | 104 | // Build embeddings for newly imported tweets 105 | buildAllTweetEmbeddings(allTweets).then(() => { 106 | console.log("All tweet embeddings built!"); 107 | }); 108 | 109 | if (sendResponse) { 110 | sendResponse({ status: "received" }); 111 | } 112 | } else if (message.action === "importError") { 113 | statusText.textContent = message.error; 114 | } 115 | }); 116 | 117 | // ---------------------------------------------------------- 118 | // 3) Search Input Handler (Using Cosine Similarity of Embeddings) 119 | // ---------------------------------------------------------- 120 | let searchTimeout; 121 | searchInput.addEventListener('input', (e) => { 122 | // Clear previous timeout 123 | if (searchTimeout) { 124 | clearTimeout(searchTimeout); 125 | } 126 | 127 | // Debounce search 128 | searchTimeout = setTimeout(async () => { 129 | const query = e.target.value.trim(); 130 | if (!query) { 131 | displayTweets(allTweets.slice(0, 50)); // Show first 50 tweets when no query 132 | statusText.textContent = `${allTweets.length} tweets ready for search`; 133 | return; 134 | } 135 | 136 | try { 137 | // Show loading state 138 | statusText.textContent = 'Searching...'; 139 | 140 | // Embed the query text 141 | const queryEmbedding = await getEmbeddingForText(query); 142 | if (!queryEmbedding) { 143 | displayTweets([]); 144 | return; 145 | } 146 | 147 | // Rank tweets by similarity 148 | const results = allTweets 149 | .map(tweet => ({ 150 | tweet, 151 | score: tweet.embedding ? cosineSimilarity(queryEmbedding, tweet.embedding) : 0 152 | })) 153 | .filter(item => item.score > 0.3) // Only show reasonably good matches 154 | .sort((a, b) => b.score - a.score) 155 | .slice(0, 50) // Limit to top 50 results 156 | .map(item => item.tweet); 157 | 158 | displayTweets(results); 159 | statusText.textContent = `found ${results.length} matches`; 160 | } catch (error) { 161 | console.error('Search error:', error); 162 | statusText.textContent = 'Search error occurred'; 163 | } 164 | }, 300); // Wait 300ms after last keystroke before searching 165 | }); 166 | 167 | // ---------------------------------------------------------- 168 | // 4) Display Tweets Using Custom Implementation 169 | // ---------------------------------------------------------- 170 | function displayTweets(tweets) { 171 | // Clear previous results 172 | searchResults.innerHTML = ''; 173 | 174 | if (tweets.length === 0) { 175 | searchResults.innerHTML = ` 176 |
177 | No tweets found 178 |
179 | `; 180 | return; 181 | } 182 | 183 | // Create custom display for each tweet 184 | tweets.forEach(tweet => { 185 | searchResults.appendChild(createTweetDisplay(tweet)); 186 | }); 187 | } 188 | 189 | // ---------------------------------------------------------- 190 | // 5) Create Tweet Display 191 | // ---------------------------------------------------------- 192 | function createTweetDisplay(tweet) { 193 | const container = document.createElement('div'); 194 | container.className = 'tweet-container'; 195 | 196 | // Format date 197 | const date = new Date(tweet.timestamp); 198 | const formattedDate = date.toLocaleDateString('en-US', { 199 | year: 'numeric', 200 | month: 'short', 201 | day: 'numeric' 202 | }); 203 | 204 | // Only set mediaClass if we actually have media 205 | let mediaSection = ''; 206 | if (tweet.media) { 207 | const mediaCount = Array.isArray(tweet.media) ? tweet.media.length : 1; 208 | const mediaClass = ['single', 'double', 'triple', 'quad'][Math.min(mediaCount - 1, 3)]; 209 | 210 | mediaSection = ` 211 |
212 | ${Array.isArray(tweet.media) ? 213 | tweet.media.map(m => 214 | m.type === 'photo' 215 | ? `` 216 | : `` 217 | ).join('') 218 | : tweet.media.type === 'photo' 219 | ? `` 220 | : `` 221 | } 222 |
223 | `; 224 | } 225 | 226 | container.innerHTML = ` 227 | 232 |
233 | 238 |
239 |
${tweet.author.name}
240 |
@${tweet.author.screen_name}
241 |
242 |
243 | 244 |
${tweet.full_text}
245 | ${mediaSection} 246 |
247 | ${formattedDate} 248 | 252 | View on X → 253 | 254 |
255 | `; 256 | 257 | return container; 258 | } 259 | 260 | // ---------------------------------------------------------- 261 | // 6) Build Embeddings for Tweets (Local Transformers.js) 262 | // ---------------------------------------------------------- 263 | async function buildAllTweetEmbeddings(tweets) { 264 | if (!tweets || tweets.length === 0 || isProcessingEmbeddings) return; 265 | 266 | isProcessingEmbeddings = true; 267 | const startTime = Date.now(); 268 | 269 | try { 270 | // Initialize pipeline if needed 271 | if (!embeddingPipeline) { 272 | statusText.textContent = 'loading model...'; 273 | embeddingPipeline = await pipeline( 274 | 'feature-extraction', 275 | 'Xenova/all-MiniLM-L6-v2', 276 | { 277 | progress_callback: null, 278 | config: { 279 | local: false, 280 | quantized: false, 281 | useWorker: false, 282 | wasmPath: chrome.runtime.getURL('lib/'), 283 | tokenizerId: 'Xenova/all-MiniLM-L6-v2' 284 | } 285 | } 286 | ); 287 | } 288 | 289 | let processedCount = 0; 290 | const updateProgress = () => { 291 | const progress = Math.round((processedCount / tweets.length) * 100); 292 | const timeElapsed = ((Date.now() - startTime) / 1000).toFixed(1); 293 | const tweetsPerSecond = (processedCount / (Date.now() - startTime) * 1000).toFixed(1); 294 | statusText.textContent = `building search index... ${progress}% (${processedCount}/${tweets.length} tweets, ${tweetsPerSecond}/s)`; 295 | }; 296 | 297 | // Process in batches 298 | for (let i = 0; i < tweets.length; i += BATCH_SIZE) { 299 | const batch = tweets.slice(i, i + BATCH_SIZE); 300 | const batchTexts = batch.map(tweet => getTweetText(tweet)); 301 | 302 | try { 303 | // Process batch of tweets 304 | const results = await Promise.all( 305 | batchTexts.map(text => 306 | embeddingPipeline(text, { 307 | pooling: 'mean', 308 | normalize: true 309 | }) 310 | ) 311 | ); 312 | 313 | // Store embeddings 314 | results.forEach((result, idx) => { 315 | batch[idx].embedding = Array.from(result.data); 316 | }); 317 | 318 | processedCount += batch.length; 319 | updateProgress(); 320 | 321 | // Cache embeddings every few batches 322 | if (i % (BATCH_SIZE * 5) === 0) { 323 | await updateEmbeddingCache(batch); 324 | } 325 | } catch (error) { 326 | console.warn('Error processing batch:', error); 327 | // Continue with next batch even if this one failed 328 | } 329 | 330 | // Brief UI pause every few batches 331 | if (i % (BATCH_SIZE * 3) === 0) { 332 | await new Promise(resolve => setTimeout(resolve, 10)); 333 | } 334 | } 335 | 336 | // Final cache update 337 | await updateEmbeddingCache(tweets); 338 | 339 | const totalTime = ((Date.now() - startTime) / 1000).toFixed(1); 340 | statusText.textContent = `${tweets.length} bookmarks ready for search (processed in ${totalTime}s)`; 341 | } catch (error) { 342 | console.error('Error building embeddings:', error); 343 | statusText.textContent = 'Error building search index'; 344 | } finally { 345 | isProcessingEmbeddings = false; 346 | } 347 | } 348 | 349 | /** 350 | * Return the relevant text from tweet for embedding 351 | * Emphasizes the tweet text, author name, and username 352 | */ 353 | function getTweetText(tweet) { 354 | const authorName = tweet.author?.name || ''; 355 | const authorScreenName = tweet.author?.screen_name || ''; 356 | const tweetText = tweet.full_text || ''; 357 | return `${tweetText}\n${authorName}\n${authorScreenName}`.trim(); 358 | } 359 | 360 | /** 361 | * Get an embedding vector from Transformers.js 362 | */ 363 | async function getEmbeddingForText(text) { 364 | try { 365 | if (!embeddingPipeline) { 366 | console.log("loading embedding model..."); 367 | embeddingPipeline = await pipeline( 368 | 'feature-extraction', 369 | 'Xenova/all-MiniLM-L6-v2', 370 | { 371 | config: { 372 | local: false, 373 | quantized: false, 374 | useWorker: false 375 | } 376 | } 377 | ); 378 | } 379 | 380 | // Get embeddings 381 | const result = await embeddingPipeline(text, { 382 | pooling: 'mean', 383 | normalize: true 384 | }); 385 | return Array.from(result.data); 386 | } catch (error) { 387 | console.error('Error generating embedding:', error); 388 | return null; 389 | } 390 | } 391 | 392 | // ---------------------------------------------------------- 393 | // 7) Cosine Similarity for Embedding Arrays 394 | // ---------------------------------------------------------- 395 | function cosineSimilarity(a, b) { 396 | if (!a || !b || !a.length || !b.length || a.length !== b.length) { 397 | return 0; 398 | } 399 | 400 | let dotProduct = 0; 401 | let normA = 0; 402 | let normB = 0; 403 | 404 | for (let i = 0; i < a.length; i++) { 405 | dotProduct += a[i] * b[i]; 406 | normA += a[i] * a[i]; 407 | normB += b[i] * b[i]; 408 | } 409 | 410 | if (normA === 0 || normB === 0) return 0; 411 | 412 | return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)); 413 | } 414 | --------------------------------------------------------------------------------