├── .gitignore ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.MD ├── README.md ├── docs ├── DOCUMENTATION.MD └── images │ └── header.png ├── package.json ├── src ├── background-script │ ├── keep-ping.ts │ ├── purge-on-startup.ts │ └── tab-remove-listeners.ts ├── bcrew-two │ ├── create-jar.ts │ ├── distance.ts │ └── expand-jar.ts ├── cereal │ ├── cereal-index.ts │ ├── cereal-types.ts │ └── cereal-utils.ts ├── constants.ts ├── content-script │ ├── execute-crawl.ts │ ├── queue-crawl.ts │ ├── reset-crawl.ts │ ├── shared-memory.ts │ ├── storage-change-listeners.ts │ ├── test-csp.ts │ └── websocket.ts ├── dnr │ └── dnr-helpers.ts ├── elements │ ├── elements-utils.ts │ ├── generate-links.ts │ └── web-platform.ts ├── get-requests │ └── get-helpers.ts ├── htmlVisualizer │ ├── htmlVisualizer.ts │ └── src │ │ ├── capturer.ts │ │ ├── encoder.ts │ │ ├── index.ts │ │ ├── logger.ts │ │ └── types.ts ├── iframe │ ├── actions.ts │ ├── check-pdf.ts │ ├── contained-visualizer-helpers.ts │ ├── dom-processing.ts │ ├── evaluate-selector.ts │ ├── init-crawl.ts │ ├── message-background.ts │ ├── mutation-observer.ts │ ├── mute-iframe.ts │ ├── safe-render.ts │ └── save │ │ ├── save-crawl.ts │ │ ├── save-with-contained.ts │ │ └── save-with-visualizer.ts ├── index.ts ├── listeners │ ├── listener-helpers-CS.ts │ └── listener-helpers-SW.ts ├── local-rate-limiting │ └── rate-limiter.ts ├── logger │ └── logger.ts ├── meucci │ ├── meucci-save.ts │ └── meucci-utils.ts ├── pascoli │ └── pascoli-utils.ts ├── pdf │ ├── pdf-getter.ts │ └── pdfjs-dist.d.ts ├── permissions │ └── permission-helpers.ts ├── post-requests │ └── post-helpers.ts ├── request-info │ └── request-info-helpers.ts ├── request-message │ └── request-message-helpers.ts ├── storage │ └── storage-helpers.ts ├── switch │ └── check-switch.ts ├── transparency │ └── badge-settings.ts ├── turndown │ └── turndown.ts └── utils │ ├── document-body-observer.ts │ ├── identity-helpers.ts │ ├── iframe-helpers.ts │ ├── measure-connection-speed.ts │ ├── messaging-helpers.ts │ ├── opt-in-out-helpers.ts │ ├── put-to-signed.ts │ ├── start-stop-helpers.ts │ ├── tabs-helpers.ts │ ├── trigger-storage.ts │ ├── triggers-download-helpers.ts │ └── utils.ts ├── tsconfig.json └── tsup.config.ts /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea/ 3 | node_modules/ 4 | dist 5 | package-lock.json 6 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 1.1.2 - 2024-03-03 2 | 3 | - 🚀 Initial release 4 | 5 | ## 1.1.7 - 2024-03-21 6 | 7 | - 📁 Fix PDF crawling (top level await in pdf.js >= 4.0.189) 8 | 9 | ## 1.1.8 - 2024-03-25 10 | 11 | - 🙋 Introduced optional_permissions so developers can release/send an update without requiring users to accept all permissions at installation/update 12 | 13 | ## 1.1.9 - 2024-03-26 14 | 15 | - ⏹️ Introduced local rate limiting (in addition to the server load balancing) to give developers more control and transparency over the rate limiting of their extension 16 | 17 | ## 1.3.0 - 2024-05-20 18 | 19 | - Batch Executions support 20 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | - Demonstrating empathy and kindness toward other people 21 | - Being respectful of differing opinions, viewpoints, and experiences 22 | - Giving and gracefully accepting constructive feedback 23 | - Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | - Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | - The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | - Trolling, insulting or derogatory comments, and personal or political attacks 33 | - Public or private harassment 34 | - Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | - Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at twitter 63 | @asimdotshrestha. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Mellowtel 2 | 3 | Thanks for being interested in contributing to Mellowtel ❤️. 4 | We are extremely open to any and all contributions you might be interested in making. 5 | To contribute to this project, please follow a ["fork and pull request"](https://docs.github.com/en/get-started/quickstart/contributing-to-projects) workflow. 6 | Please do not try to push directly to this repo unless you are a maintainer. 7 | 8 | ## Guidelines 9 | 10 | ### Issues 11 | 12 | The [issues](https://github.com/mellowtel-inc/mellowtel-js/issues) contain all current bugs, improvements, and feature requests. 13 | Please use the corresponding label when creating an issue. 14 | 15 | ### Getting Help 16 | 17 | Contact a maintainer of Mellowtel with any questions or help you might need. 18 | -------------------------------------------------------------------------------- /LICENSE.MD: -------------------------------------------------------------------------------- 1 | # GNU LESSER GENERAL PUBLIC LICENSE 2 | 3 | Version 3, 29 June 2007 4 | 5 | Copyright (C) 2007 Free Software Foundation, Inc. 6 | 7 | 8 | Everyone is permitted to copy and distribute verbatim copies of this 9 | license document, but changing it is not allowed. 10 | 11 | This version of the GNU Lesser General Public License incorporates the 12 | terms and conditions of version 3 of the GNU General Public License, 13 | supplemented by the additional permissions listed below. 14 | 15 | ## 0. Additional Definitions. 16 | 17 | As used herein, "this License" refers to version 3 of the GNU Lesser 18 | General Public License, and the "GNU GPL" refers to version 3 of the 19 | GNU General Public License. 20 | 21 | "The Library" refers to a covered work governed by this License, other 22 | than an Application or a Combined Work as defined below. 23 | 24 | An "Application" is any work that makes use of an interface provided 25 | by the Library, but which is not otherwise based on the Library. 26 | Defining a subclass of a class defined by the Library is deemed a mode 27 | of using an interface provided by the Library. 28 | 29 | A "Combined Work" is a work produced by combining or linking an 30 | Application with the Library. The particular version of the Library 31 | with which the Combined Work was made is also called the "Linked 32 | Version". 33 | 34 | The "Minimal Corresponding Source" for a Combined Work means the 35 | Corresponding Source for the Combined Work, excluding any source code 36 | for portions of the Combined Work that, considered in isolation, are 37 | based on the Application, and not on the Linked Version. 38 | 39 | The "Corresponding Application Code" for a Combined Work means the 40 | object code and/or source code for the Application, including any data 41 | and utility programs needed for reproducing the Combined Work from the 42 | Application, but excluding the System Libraries of the Combined Work. 43 | 44 | ## 1. Exception to Section 3 of the GNU GPL. 45 | 46 | You may convey a covered work under sections 3 and 4 of this License 47 | without being bound by section 3 of the GNU GPL. 48 | 49 | ## 2. Conveying Modified Versions. 50 | 51 | If you modify a copy of the Library, and, in your modifications, a 52 | facility refers to a function or data to be supplied by an Application 53 | that uses the facility (other than as an argument passed when the 54 | facility is invoked), then you may convey a copy of the modified 55 | version: 56 | 57 | - a) under this License, provided that you make a good faith effort 58 | to ensure that, in the event an Application does not supply the 59 | function or data, the facility still operates, and performs 60 | whatever part of its purpose remains meaningful, or 61 | - b) under the GNU GPL, with none of the additional permissions of 62 | this License applicable to that copy. 63 | 64 | ## 3. Object Code Incorporating Material from Library Header Files. 65 | 66 | The object code form of an Application may incorporate material from a 67 | header file that is part of the Library. You may convey such object 68 | code under terms of your choice, provided that, if the incorporated 69 | material is not limited to numerical parameters, data structure 70 | layouts and accessors, or small macros, inline functions and templates 71 | (ten or fewer lines in length), you do both of the following: 72 | 73 | - a) Give prominent notice with each copy of the object code that 74 | the Library is used in it and that the Library and its use are 75 | covered by this License. 76 | - b) Accompany the object code with a copy of the GNU GPL and this 77 | license document. 78 | 79 | ## 4. Combined Works. 80 | 81 | You may convey a Combined Work under terms of your choice that, taken 82 | together, effectively do not restrict modification of the portions of 83 | the Library contained in the Combined Work and reverse engineering for 84 | debugging such modifications, if you also do each of the following: 85 | 86 | - a) Give prominent notice with each copy of the Combined Work that 87 | the Library is used in it and that the Library and its use are 88 | covered by this License. 89 | - b) Accompany the Combined Work with a copy of the GNU GPL and this 90 | license document. 91 | - c) For a Combined Work that displays copyright notices during 92 | execution, include the copyright notice for the Library among 93 | these notices, as well as a reference directing the user to the 94 | copies of the GNU GPL and this license document. 95 | - d) Do one of the following: 96 | - 0. Convey the Minimal Corresponding Source under the terms of 97 | this License, and the Corresponding Application Code in a form 98 | suitable for, and under terms that permit, the user to 99 | recombine or relink the Application with a modified version of 100 | the Linked Version to produce a modified Combined Work, in the 101 | manner specified by section 6 of the GNU GPL for conveying 102 | Corresponding Source. 103 | - 1. Use a suitable shared library mechanism for linking with 104 | the Library. A suitable mechanism is one that (a) uses at run 105 | time a copy of the Library already present on the user's 106 | computer system, and (b) will operate properly with a modified 107 | version of the Library that is interface-compatible with the 108 | Linked Version. 109 | - e) Provide Installation Information, but only if you would 110 | otherwise be required to provide such information under section 6 111 | of the GNU GPL, and only to the extent that such information is 112 | necessary to install and execute a modified version of the 113 | Combined Work produced by recombining or relinking the Application 114 | with a modified version of the Linked Version. (If you use option 115 | 4d0, the Installation Information must accompany the Minimal 116 | Corresponding Source and Corresponding Application Code. If you 117 | use option 4d1, you must provide the Installation Information in 118 | the manner specified by section 6 of the GNU GPL for conveying 119 | Corresponding Source.) 120 | 121 | ## 5. Combined Libraries. 122 | 123 | You may place library facilities that are a work based on the Library 124 | side by side in a single library together with other library 125 | facilities that are not Applications and are not covered by this 126 | License, and convey such a combined library under terms of your 127 | choice, if you do both of the following: 128 | 129 | - a) Accompany the combined library with a copy of the same work 130 | based on the Library, uncombined with any other library 131 | facilities, conveyed under the terms of this License. 132 | - b) Give prominent notice with the combined library that part of it 133 | is a work based on the Library, and explaining where to find the 134 | accompanying uncombined form of the same work. 135 | 136 | ## 6. Revised Versions of the GNU Lesser General Public License. 137 | 138 | The Free Software Foundation may publish revised and/or new versions 139 | of the GNU Lesser General Public License from time to time. Such new 140 | versions will be similar in spirit to the present version, but may 141 | differ in detail to address new problems or concerns. 142 | 143 | Each version is given a distinguishing version number. If the Library 144 | as you received it specifies that a certain numbered version of the 145 | GNU Lesser General Public License "or any later version" applies to 146 | it, you have the option of following the terms and conditions either 147 | of that published version or of any later version published by the 148 | Free Software Foundation. If the Library as you received it does not 149 | specify a version number of the GNU Lesser General Public License, you 150 | may choose any version of the GNU Lesser General Public License ever 151 | published by the Free Software Foundation. 152 | 153 | If the Library as you received it specifies that a proxy can decide 154 | whether future versions of the GNU Lesser General Public License shall 155 | apply, that proxy's public statement of acceptance of any version is 156 | permanent authorization for you to choose that version for the 157 | Library. 158 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Mellowtel cover](https://raw.githubusercontent.com/mellowtel-inc/mellowtel-js/main/docs/images/header.png) 2 | 3 |
Mellowtel
4 |
Monetize your Browser plugin.
Open-Source, Consensual, Transparent.
5 |
6 |
7 | Website 8 | · 9 | GitHub 10 | · 11 | Discord 12 | · 13 | Documentation 14 |
15 | 16 |
17 | 18 |
19 | 20 | ![TypeScript](https://img.shields.io/badge/typescript-%23007ACC.svg?logo=typescript&logoColor=white) 21 | [![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg?style=flat-square)](https://github.com/prettier/prettier) 22 | [![GitHub Repo stars](https://img.shields.io/github/stars/mellowtel-inc/mellowtel-js)](https://github.com/mellowtel-inc/mellowtel-js) 23 | [![Discord](https://img.shields.io/discord/1221455179619106887?label=&logo=discord&logoColor=ffffff&color=7389D8&labelColor=6A7EC2)](https://discord.com/invite/GC8vwpDWC9) 24 | 25 |
26 | 27 | --- 28 | 29 | # Introduction ℹ️ 30 | 31 | With Mellowtel's Open-Source library, your users can decide if they want to support you by sharing a fraction of their unused internet bandwidth. Trusted partners — from startups to non-profits — access the internet to retrieve publicly available data, and you get paid for it. 32 | Mellowtel is supported on all major browsers: Chrome, Firefox and Edge. 33 | 34 | **How?** 35 | 36 | Companies need to retrieve publicly available data from the web. You get a share of the revenue they pay for providing access to the web thanks to users that want to support you and share their unused bandwidth. 37 | 38 | # Key Features 🎯 39 | 40 | - **Easy to use**: Earn from your Browser plugins with a few lines of code. 41 | - **Open-source**: The code is open-source and available for everyone to see. 42 | - **Consensual & Opt-out by default**: Users are opted out by default. If they want to support you they have to explicitly opt-in. They can opt-out and manage their settings at any time. 43 | - **Non-intrusive & Private**: In contrast to ads network, we do not collect, share, or sell personal information (not even anonymized data). The whole business model relies on the fact we don't need to collect or sell data but on using a small portion of unused bandwidth 44 | - **Good user experience**: Mellowtel only requires enough resources to open an additional incognito tab. In order to guarantee a good user experience we only operate when the connection is stable (wifi, ethernet) and there is high bandwidth available. 45 | 46 | # Why❓ 47 | 48 | We believe that extension developers should be able to earn from their hard work without compromising their users' privacy or experience. Most Browser plugins provide a lot of value to users, but they are not willing to pay for them. 49 | 50 | Mellowtel gives your users a way to support your extension without having to directly pay for it. They can instead choose to share a fraction of their unused internet bandwidth. It's a win-win situation for everyone: users pay with a resource they aren't using, and you get paid for your work. 51 | 52 | Hopefully this will lead to fewer extensions being shut down or discontinued due to inability to monetize, fewer personal data being collected and sold, and more transparency in the Browser plugins ecosystem. 53 | 54 | See other "monetization solutions" below for examples of what can go wrong when developers are not given a fair way to earn from their work. 55 | 56 | # What are the current "monetization solutions"? 🧐 57 | 58 | We have been developing Browser plugins and trying to monetize them for years. We, too, received a hefty amount of the "monetization solutions" emails. 59 | 60 | Well, honestly, most attempts to monetize Browser plugins are just shady at best and total scams at worst. They either buy your extension or provide SDKs that collect and sell your users' personal data (credit card information, addresses), spoof their passwords, inject unwanted ads into your extension, inject affiliate links, etc. 61 | 62 | A recommended read: [Temptations of an open-source browser extension developer](https://github.com/extesy/hoverzoom/discussions/670) 63 | 64 | And a highlight from the article 😰: 65 | 66 | > "...we provide several methods of monetizating- from the soft to the hard methods." 67 | 68 | # Getting started 🚀 69 | 70 | We have moved our documentation to a dedicated website: [docs.mellowtel.com](https://docs.mellowtel.com). 71 | 72 | # Quickstart 73 | 74 | [Here](https://docs.mellowtel.com/browser-plugins/quickstart) is a detailed guide on how to get started with Mellowtel. 75 | 76 | # Contributing 🫶 77 | 78 | Mellowtel is an open-source project, and contributions are welcome. If you want to contribute, you can create new features, fix bugs, or improve the infrastructure. Please refer to the [CONTRIBUTING.md](https://github.com/mellowtel-inc/mellowtel-js/blob/main/CONTRIBUTING.md) file in the repository for more information on how to contribute. 79 | 80 | To see how to contribute, visit [Contribution guidelines](https://github.com/mellowtel-inc/mellowtel-js/blob/main/CONTRIBUTING.md) 81 | 82 | # Support 83 | 84 | You can reach out to us on [Discord](https://discord.gg/GC8vwpDWC9) if you have any questions or need help. 85 | 86 | # License 📜 87 | 88 | GNU Lesser General Public License v3.0 89 | 90 | [License](https://github.com/mellowtel-inc/mellowtel-js/blob/main/LICENSE.MD) 91 | -------------------------------------------------------------------------------- /docs/DOCUMENTATION.MD: -------------------------------------------------------------------------------- 1 | # Mellowtel Documentation 2 | 3 | The documentation for the Mellowtel project can be found at [this link](https://docs.mellowtel.com/get-started/welcome). 4 | 5 | A quick start guide is available at [this link](https://docs.mellowtel.com/get-started/quickstart). 6 | -------------------------------------------------------------------------------- /docs/images/header.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mellowtel-inc/mellowtel-js/f9db2cf620302f1e79b5c7dbfef5921c6dc45a4c/docs/images/header.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mellowtel", 3 | "version": "1.6.2", 4 | "description": "", 5 | "main": "./dist/index.js", 6 | "module": "./dist/index.mjs", 7 | "types": "./dist/index.d.ts", 8 | "files": [ 9 | "dist" 10 | ], 11 | "scripts": { 12 | "build": "tsup", 13 | "format": "npx prettier . --write", 14 | "test": "echo \"Error: no test specified\" && exit 1", 15 | "dev": "npm pack --pack-destination ~/Desktop/mellowtel" 16 | }, 17 | "author": "mellowtel-inc", 18 | "license": "LGPL-3.0", 19 | "devDependencies": { 20 | "@types/chrome": "^0.0.256", 21 | "prettier": "3.2.4", 22 | "ts-node": "^10.9.2", 23 | "tsup": "^8.0.1", 24 | "typescript": "^5.3.3" 25 | }, 26 | "dependencies": { 27 | "@cloudflare/speedtest": "^1.3.0", 28 | "isomorphic-ws": "^5.0.0", 29 | "pdfjs-dist": "^3.8.162" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/background-script/keep-ping.ts: -------------------------------------------------------------------------------- 1 | import { getLocalStorage, setLocalStorage } from "../storage/storage-helpers"; 2 | import { Logger } from "../logger/logger"; 3 | import { SW_PING_INTERVAL } from "../constants"; 4 | import { isInSW } from "../utils/utils"; 5 | 6 | const KEEP_PING_STATUS = "keep_ping_status"; 7 | let keepAliveInterval: number | null = null; 8 | 9 | const keepAlive = () => { 10 | chrome.runtime.sendMessage("ping", (response) => { 11 | if (chrome.runtime.lastError) { 12 | // Logger.log("[keepAlive] Error sending ping:", chrome.runtime.lastError.message); 13 | } 14 | }); 15 | }; 16 | 17 | export const startPing = async (): Promise => { 18 | try { 19 | const inServiceWorker = await isInSW(); 20 | 21 | if (inServiceWorker) { 22 | // Clear any existing interval first 23 | await stopPing(); 24 | 25 | // Start new interval 26 | keepAliveInterval = setInterval(keepAlive, SW_PING_INTERVAL); 27 | await setLocalStorage(KEEP_PING_STATUS, true); 28 | Logger.log("[startPing] Keep-alive started in Service Worker"); 29 | } else { 30 | // In content script - send message to SW to start ping 31 | chrome.runtime.sendMessage({ intent: "mllwtl_startPing" }); 32 | Logger.log("[startPing] Requested Service Worker to start keep-alive"); 33 | } 34 | } catch (error) { 35 | Logger.log("[startPing] Error starting keep-alive:", error); 36 | } 37 | }; 38 | 39 | export const stopPing = async (): Promise => { 40 | try { 41 | const inServiceWorker = await isInSW(); 42 | 43 | if (inServiceWorker) { 44 | if (keepAliveInterval) { 45 | clearInterval(keepAliveInterval); 46 | keepAliveInterval = null; 47 | } 48 | await setLocalStorage(KEEP_PING_STATUS, false); 49 | Logger.log("[stopPing] Keep-alive stopped in Service Worker"); 50 | } else { 51 | // In content script - send message to SW to stop ping 52 | chrome.runtime.sendMessage({ intent: "mllwtl_stopPing" }); 53 | Logger.log("[stopPing] Requested Service Worker to stop keep-alive"); 54 | } 55 | } catch (error) { 56 | Logger.log("[stopPing] Error stopping keep-alive:", error); 57 | } 58 | }; 59 | 60 | export const isPingEnabled = async (): Promise => { 61 | try { 62 | const status = await getLocalStorage(KEEP_PING_STATUS, true); 63 | return status === true; 64 | } catch (error) { 65 | Logger.log("[isPingEnabled] Error checking ping status:", error); 66 | return false; 67 | } 68 | }; 69 | -------------------------------------------------------------------------------- /src/background-script/purge-on-startup.ts: -------------------------------------------------------------------------------- 1 | import { deleteLocalStorage } from "../storage/storage-helpers"; 2 | 3 | export async function purgeOnStartup(): Promise { 4 | chrome.runtime.onStartup.addListener(async function () { 5 | const keysToPurge: string[] = [ 6 | "webSocketConnected", 7 | "queue_batch", 8 | "queue", 9 | "already_checked_switch", 10 | "checked_switch_value", 11 | "recordsRequestInfo", 12 | "recordsMessageInfo", 13 | "unfocusedWindowId", 14 | "onlyIfMustArray", 15 | "mllwtl_cereal_frame_tab", 16 | "device_disconnect_session", 17 | ]; 18 | await deleteLocalStorage(keysToPurge); 19 | }); 20 | } 21 | -------------------------------------------------------------------------------- /src/background-script/tab-remove-listeners.ts: -------------------------------------------------------------------------------- 1 | import { sendMessageToContentScript } from "../utils/messaging-helpers"; 2 | import { getLocalStorage, setLocalStorage } from "../storage/storage-helpers"; 3 | import { start } from "../utils/start-stop-helpers"; 4 | import { getOptInStatus, optIn } from "../utils/opt-in-out-helpers"; 5 | import { Logger } from "../logger/logger"; 6 | 7 | export async function setUpOnTabRemoveListeners(): Promise { 8 | chrome.tabs.onRemoved.addListener(async function ( 9 | tabId: number, 10 | removeInfo: chrome.tabs.TabRemoveInfo, 11 | ) { 12 | chrome.storage.local.get( 13 | ["webSocketConnected"], 14 | function (result: { [key: string]: any }) { 15 | let webSocketConnected = result["webSocketConnected"]; 16 | if (webSocketConnected === tabId) { 17 | chrome.tabs.query({}, async function (tabs: chrome.tabs.Tab[]) { 18 | for (let i: number = 0; i < tabs.length; i++) { 19 | let response = await sendMessageToContentScript(tabs[i].id!, { 20 | intent: "startConnectionM", 21 | }); 22 | Logger.log("Response from startConnectionM:", response); 23 | if (response !== null) { 24 | break; 25 | } 26 | } 27 | }); 28 | } 29 | }, 30 | ); 31 | }); 32 | } 33 | -------------------------------------------------------------------------------- /src/bcrew-two/create-jar.ts: -------------------------------------------------------------------------------- 1 | import { WebsiteJar, CookieData } from "./expand-jar"; 2 | import { RULE_ID_START_BCREW } from "../constants"; 3 | import { Logger } from "../logger/logger"; 4 | import { isInSW } from "../utils/utils"; 5 | import { sendMessageToBackground } from "../utils/messaging-helpers"; 6 | 7 | function generateCookieHash(cookie: CookieData): number { 8 | const str = `${cookie.name}${cookie.value}${cookie.domain}${cookie.path}${cookie.isWhisper ? "whisper" : "normal"}`; 9 | let hash = 0; 10 | for (let i = 0; i < str.length; i++) { 11 | const char = str.charCodeAt(i); 12 | hash = (hash << 5) - hash + char; 13 | hash = hash & hash; 14 | } 15 | return Math.abs(hash); 16 | } 17 | 18 | function generateRuleId(cookie: CookieData): number { 19 | const hash = generateCookieHash(cookie); 20 | return RULE_ID_START_BCREW + (hash % 1000000); 21 | } 22 | 23 | export async function removeJarRulesForCookies( 24 | cookies: CookieData[], 25 | ): Promise { 26 | const ruleIds = cookies.map((cookie) => generateRuleId(cookie)); 27 | return new Promise(async (resolve) => { 28 | if (!(await isInSW())) { 29 | let response = await sendMessageToBackground({ 30 | intent: "removeJarRulesForCookies", 31 | cookies: cookies, 32 | }); 33 | resolve(response); 34 | } else { 35 | chrome.declarativeNetRequest.updateSessionRules( 36 | { 37 | removeRuleIds: ruleIds, 38 | addRules: [], 39 | }, 40 | () => { 41 | if (chrome.runtime.lastError) { 42 | Logger.error( 43 | "Error removing specific cookie rules:", 44 | chrome.runtime.lastError, 45 | ); 46 | } else { 47 | Logger.log( 48 | `Successfully removed ${ruleIds.length} specific cookie rules`, 49 | ); 50 | } 51 | resolve(); 52 | }, 53 | ); 54 | } 55 | }); 56 | } 57 | 58 | export async function createJar(jarData: WebsiteJar): Promise { 59 | return new Promise(async (resolve) => { 60 | if (!(await isInSW())) { 61 | Logger.log("[createJar] : Sending message to background"); 62 | let response = await sendMessageToBackground({ 63 | intent: "createJar", 64 | jarData: jarData, 65 | }); 66 | resolve(response); 67 | } else { 68 | let filterHttpOnly = false; 69 | if (jarData.filterHttpOnly) { 70 | filterHttpOnly = jarData.filterHttpOnly; 71 | } 72 | const httpOnlyCookies = filterHttpOnly 73 | ? jarData.cookies.filter((cookie) => !cookie.httpOnly) 74 | : jarData.cookies; 75 | const rules: chrome.declarativeNetRequest.Rule[] = httpOnlyCookies.map( 76 | (cookie, index) => { 77 | let resourceTypes = ["sub_frame"]; 78 | if (cookie.resourceTypes) { 79 | resourceTypes = cookie.resourceTypes; 80 | } 81 | 82 | const ruleId = generateRuleId(cookie); 83 | 84 | let cookieStr = `${cookie.name}=${cookie.value}; Path=${cookie.path}; Domain=${cookie.domain}`; 85 | 86 | if (cookie.secure) cookieStr += "; Secure"; 87 | if (cookie.httpOnly) cookieStr += "; HttpOnly"; 88 | 89 | if (cookie.sameSite && cookie.sameSite !== "unspecified") { 90 | cookieStr += `; SameSite=${cookie.sameSite}`; 91 | } else { 92 | if (cookie.optionalDefaultValue) { 93 | cookieStr += `; ${cookie.optionalDefaultValue}`; 94 | } 95 | if (cookie.secure) { 96 | cookieStr += "; SameSite=None"; 97 | } 98 | } 99 | 100 | if (!cookie.session && cookie.expirationDate) { 101 | const expirationDate = new Date(cookie.expirationDate * 1000); 102 | cookieStr += `; Expires=${expirationDate.toUTCString()}`; 103 | } 104 | 105 | return { 106 | id: ruleId, 107 | priority: 1, 108 | action: { 109 | type: chrome.declarativeNetRequest.RuleActionType.MODIFY_HEADERS, 110 | ...(cookie.isWhisper 111 | ? { 112 | requestHeaders: [ 113 | { 114 | header: "Cookie", 115 | operation: 116 | chrome.declarativeNetRequest.HeaderOperation.SET, 117 | value: cookieStr, 118 | }, 119 | ], 120 | } 121 | : { 122 | responseHeaders: [ 123 | { 124 | header: "Set-Cookie", 125 | operation: 126 | chrome.declarativeNetRequest.HeaderOperation.APPEND, 127 | value: cookieStr, 128 | }, 129 | ], 130 | }), 131 | }, 132 | condition: { 133 | urlFilter: `||${cookie.domain}`, 134 | resourceTypes: resourceTypes, 135 | }, 136 | } as chrome.declarativeNetRequest.Rule; 137 | }, 138 | ); 139 | 140 | // Get rule IDs for removal/tracking 141 | const ruleIds = rules.map((rule) => rule.id); 142 | 143 | // Output the rules for debugging 144 | Logger.log("Creating rules:", JSON.stringify(rules, null, 2)); 145 | 146 | // Update Session rules 147 | chrome.declarativeNetRequest.updateSessionRules( 148 | { 149 | removeRuleIds: ruleIds, 150 | addRules: rules, 151 | }, 152 | () => { 153 | if (chrome.runtime.lastError) { 154 | Logger.error( 155 | "Error setting cookie rules:", 156 | chrome.runtime.lastError, 157 | ); 158 | resolve([]); 159 | } else { 160 | Logger.log("Cookie rules successfully set"); 161 | resolve(ruleIds); 162 | } 163 | }, 164 | ); 165 | } 166 | }); 167 | } 168 | 169 | export function removeJarRules(ruleIds: number[]): void { 170 | chrome.declarativeNetRequest.updateSessionRules( 171 | { 172 | removeRuleIds: ruleIds, 173 | addRules: [], 174 | }, 175 | () => { 176 | if (chrome.runtime.lastError) { 177 | Logger.error("Error removing cookie rules:", chrome.runtime.lastError); 178 | } else { 179 | Logger.log("Cookie rules successfully removed"); 180 | } 181 | }, 182 | ); 183 | } 184 | -------------------------------------------------------------------------------- /src/bcrew-two/distance.ts: -------------------------------------------------------------------------------- 1 | import { WebsiteJar } from "./expand-jar"; 2 | import { Logger } from "../logger/logger"; 3 | import { removeJarRulesForCookies } from "./create-jar"; 4 | 5 | export async function tellToApplyDistance( 6 | jarData: WebsiteJar, 7 | recordID: string, 8 | parsedBCrewObject: any, 9 | originalUrl: string, 10 | ): Promise { 11 | return new Promise((resolve) => { 12 | Logger.log("[tellToApplyDistance] : Sending msg to apply distance"); 13 | let waitBeforeApplyDistance = parsedBCrewObject.waitBeforeDistance || 0; 14 | 15 | setTimeout(() => { 16 | Logger.log( 17 | "[tellToApplyDistance] : Sending msg to apply distance after wait of " + 18 | waitBeforeApplyDistance, 19 | ); 20 | let iframe: HTMLIFrameElement | null = document.getElementById( 21 | recordID, 22 | ) as HTMLIFrameElement | null; 23 | 24 | if (iframe) { 25 | iframe.onload = null; 26 | iframe.onload = async function () { 27 | Logger.log("[tellToApplyDistance] : iframe loaded for second time"); 28 | await removeJarRulesForCookies(jarData.cookies); 29 | resolve(); 30 | }; 31 | iframe.contentWindow?.postMessage( 32 | { 33 | intent: "applyDistance", 34 | jarData: jarData, 35 | recordID: recordID, 36 | parsedBCrewObject: parsedBCrewObject, 37 | originalUrl: originalUrl, 38 | }, 39 | "*", 40 | ); 41 | } else { 42 | resolve(); 43 | } 44 | }, waitBeforeApplyDistance); 45 | }); 46 | } 47 | 48 | export async function applyDistance( 49 | jarData: WebsiteJar, 50 | recordID: string, 51 | parsedBCrewObject: any, 52 | originalUrl: string, 53 | ): Promise { 54 | return new Promise(async (resolve) => { 55 | Logger.log("[applyDistance] : Applying distance"); 56 | let navigationItem = jarData.navigationItem || "set-href"; 57 | const nonHttpOnlyCookies = jarData.cookies.filter( 58 | (cookie) => !cookie.httpOnly, 59 | ); 60 | 61 | nonHttpOnlyCookies.forEach((cookie) => { 62 | let cookieStr = `${cookie.name}=${cookie.value}; path=${cookie.path}`; 63 | 64 | if (cookie.domain) cookieStr += `; domain=${cookie.domain}`; 65 | if (cookie.secure) cookieStr += "; secure"; 66 | 67 | if (!cookie.session && cookie.expirationDate) { 68 | const expirationDate = new Date(cookie.expirationDate * 1000); 69 | cookieStr += `; expires=${expirationDate.toUTCString()}`; 70 | } 71 | 72 | if (cookie.sameSite && cookie.sameSite !== "unspecified") { 73 | cookieStr += `; SameSite=${cookie.sameSite}`; 74 | } else { 75 | if (cookie.optionalDefaultValue) { 76 | cookieStr += `; ${cookie.optionalDefaultValue}`; 77 | } 78 | if (cookie.secure) { 79 | cookieStr += "; SameSite=None"; 80 | } 81 | } 82 | 83 | document.cookie = cookieStr; 84 | Logger.log(`[applyDistance] Set cookie: ${cookieStr}`); 85 | }); 86 | 87 | Object.entries(jarData.localStorage).forEach(([key, value]) => { 88 | try { 89 | localStorage.setItem(key, value); 90 | Logger.log(`[applyDistance] Set localStorage: ${key}`); 91 | } catch (error) { 92 | Logger.error( 93 | `[applyDistance] Error setting localStorage item ${key}:`, 94 | error, 95 | ); 96 | } 97 | }); 98 | 99 | Object.entries(jarData.sessionStorage).forEach(([key, value]) => { 100 | try { 101 | sessionStorage.setItem(key, value); 102 | Logger.log(`[applyDistance] Set sessionStorage: ${key}`); 103 | } catch (error) { 104 | Logger.error( 105 | `[applyDistance] Error setting sessionStorage item ${key}:`, 106 | error, 107 | ); 108 | } 109 | }); 110 | 111 | Logger.log("[applyDistance] Regoing to page to apply changes"); 112 | Logger.log("[applyDistance] Original URL: " + originalUrl); 113 | Logger.log("[applyDistance] Navigation item: " + navigationItem); 114 | if (navigationItem === "set-href") { 115 | window.location.href = originalUrl; 116 | } else { 117 | window.location.reload(); 118 | } 119 | }); 120 | } 121 | -------------------------------------------------------------------------------- /src/bcrew-two/expand-jar.ts: -------------------------------------------------------------------------------- 1 | import { getIdentifier } from "../utils/identity-helpers"; 2 | import { Logger } from "../logger/logger"; 3 | 4 | interface CookieData { 5 | domain: string; 6 | hostOnly: boolean; 7 | httpOnly: boolean; 8 | name: string; 9 | path: string; 10 | sameSite: string; 11 | secure: boolean; 12 | session: boolean; 13 | storeId: string; 14 | value: string; 15 | expirationDate?: number; 16 | optionalDefaultValue?: string; 17 | resourceTypes?: string[]; 18 | isWhisper?: boolean; 19 | } 20 | 21 | interface WebsiteJar { 22 | domain: string; 23 | timestamp: string; 24 | cookies: CookieData[]; 25 | localStorage: Record; 26 | sessionStorage: Record; 27 | filterHttpOnly: boolean; 28 | navigationItem: string; 29 | } 30 | 31 | export async function expandJar( 32 | bcrewTwoId: string, 33 | endpoint: string, 34 | ): Promise { 35 | try { 36 | Logger.log("[expandJar]: bcrowTwoId:", bcrewTwoId); 37 | Logger.log("[expandJar]: endpoint:", endpoint); 38 | const node_id = await getIdentifier(); 39 | Logger.log("Retrieving jar data with node_id:", node_id); 40 | const response = await fetch( 41 | `${endpoint}/?bcrew_two=${bcrewTwoId}&node_id=${node_id}`, 42 | { 43 | method: "GET", 44 | headers: { 45 | "Content-Type": "application/json", 46 | }, 47 | }, 48 | ); 49 | 50 | if (!response.ok) { 51 | throw new Error( 52 | `Failed to retrieve jar data: ${response.status} ${response.statusText}`, 53 | ); 54 | } 55 | 56 | const jarData: WebsiteJar = await response.json(); 57 | Logger.log("[expandJar]: Retrieved jar data:", jarData); 58 | return jarData; 59 | } catch (error) { 60 | Logger.error("[expandJar]: Error retrieving jar data:", error); 61 | throw error; 62 | } 63 | } 64 | 65 | // Export types for use in other files 66 | export type { WebsiteJar, CookieData }; 67 | -------------------------------------------------------------------------------- /src/cereal/cereal-index.ts: -------------------------------------------------------------------------------- 1 | import { getLocalStorage, setLocalStorage } from "../storage/storage-helpers"; 2 | import { Logger } from "../logger/logger"; 3 | import { isInSW } from "../utils/utils"; 4 | import { sendMessageToContentScript } from "../utils/messaging-helpers"; 5 | import { CerealObject } from "./cereal-types"; 6 | 7 | export function cerealMain( 8 | cerealObject: string, 9 | recordID: string, 10 | htmlString: string, 11 | ) { 12 | return new Promise(async (resolve) => { 13 | Logger.log("[cerealMain] => Starting Cereal:", cerealObject); 14 | Logger.log("[cerealMain] => Record ID:", recordID); 15 | const parsedCerealObject: CerealObject = JSON.parse(cerealObject); 16 | const maxTimeout = parsedCerealObject.maxTimeout || 10000; 17 | 18 | // Set timeout to resolve empty object after maxTimeout 19 | const timeoutPromise = new Promise((timeoutResolve) => { 20 | setTimeout(() => timeoutResolve({}), maxTimeout); 21 | }); 22 | 23 | const mainLogic = async () => { 24 | const inSW = await isInSW(); 25 | Logger.log("[cerealMain] => In SW:", inSW); 26 | 27 | if (!inSW) { 28 | Logger.log("[cerealMain] => In Content Script, Delegating to SW"); 29 | const resultReceived = await new Promise((innerResolve) => { 30 | chrome.runtime.sendMessage( 31 | { 32 | intent: "mllwtl_handleCerealRequest", 33 | cerealObject: cerealObject, 34 | recordID: recordID, 35 | htmlString: htmlString, 36 | }, 37 | (response) => { 38 | if (chrome.runtime.lastError) { 39 | Logger.log( 40 | "Hey, error in RUNTIME Error:", 41 | chrome.runtime.lastError, 42 | ); 43 | } 44 | Logger.log( 45 | "[cerealMain] => Response from SW @@@@@@@@@@@@@@:", 46 | response, 47 | ); 48 | innerResolve(response); 49 | }, 50 | ); 51 | }); 52 | 53 | Logger.log("[cerealMain] => ## Result Received ##:", resultReceived); 54 | return resultReceived; 55 | } else { 56 | // The following code only runs in service worker 57 | // Check for existing cereal tab in storage 58 | const storedTab = await getLocalStorage( 59 | "mllwtl_cereal_frame_tab", 60 | true, 61 | ); 62 | Logger.log("[cerealMain] => Stored Tab:", storedTab); 63 | let targetTabId: number | null = storedTab?.tabId ?? null; 64 | Logger.log("[cerealMain] => Target Tab ID:", targetTabId); 65 | 66 | if (targetTabId) { 67 | // Verify tab still exists 68 | Logger.log("[cerealMain] => Verifying Tab:", targetTabId); 69 | try { 70 | const currentTabId = targetTabId; 71 | const tabExists = await new Promise((resolve) => { 72 | chrome.tabs.get(currentTabId, (tab) => { 73 | resolve(!chrome.runtime.lastError && !!tab); // Convert tab to boolean 74 | }); 75 | }); 76 | 77 | Logger.log("[cerealMain] => Tab Exists:", tabExists); 78 | 79 | if (!tabExists) { 80 | targetTabId = null; 81 | } 82 | } catch { 83 | targetTabId = null; 84 | } 85 | } 86 | 87 | if (!targetTabId) { 88 | Logger.log("[cerealMain] => No Stored Tab, Finding New Tab"); 89 | const tabs = await chrome.tabs.query({}); 90 | for (const tab of tabs) { 91 | if (tab.id === undefined) continue; 92 | 93 | try { 94 | const response = await sendMessageToContentScript(tab.id, { 95 | intent: "mllwtl_initCerealFrame", 96 | cerealObject: cerealObject, // Already a string 97 | }); 98 | 99 | if (response?.success) { 100 | targetTabId = tab.id; 101 | Logger.log("[cerealMain] => Found Suitable Tab:", targetTabId); 102 | await setLocalStorage("mllwtl_cereal_frame_tab", { 103 | tabId: targetTabId, 104 | }); 105 | break; 106 | } 107 | } catch { 108 | continue; 109 | } 110 | } 111 | } 112 | 113 | if (!targetTabId) { 114 | Logger.log("[cerealMain] => No Suitable Tab Found"); 115 | return { success: false, message: "not tab found" }; // No suitable tab found 116 | } 117 | 118 | // Send the actual processing message 119 | return await sendMessageToContentScript(targetTabId, { 120 | intent: "mllwtl_processCereal", 121 | cerealObject: cerealObject, // Already a string 122 | recordID: recordID, 123 | htmlString: htmlString, 124 | }); 125 | } 126 | }; 127 | 128 | const result = await Promise.race([timeoutPromise, mainLogic()]); 129 | Logger.log("[cerealMain] => FINAL Result:", result); 130 | resolve(result); 131 | }); 132 | } 133 | 134 | export function refreshCereal() { 135 | return new Promise(async (resolve) => { 136 | // check if the tab with cereal exists. 137 | // if it does, send message to refresh the frame and return success 138 | // if it doesn't, return failure 139 | const storedTab = await getLocalStorage("mllwtl_cereal_frame_tab", true); 140 | Logger.log("[refreshCereal] => Stored Tab:", storedTab); 141 | let targetTabId: number | null = storedTab?.tabId ?? null; 142 | Logger.log("[refreshCereal] => Target Tab ID:", targetTabId); 143 | 144 | if (targetTabId === null) { 145 | Logger.log("[refreshCereal] => No Stored tab found."); 146 | resolve({ success: false, message: "not tab found" }); 147 | } 148 | 149 | // Verify tab still exists 150 | Logger.log("[refreshCereal] => Verifying Tab:", targetTabId); 151 | try { 152 | const tabExists = await new Promise((res) => { 153 | if (targetTabId != null) { 154 | chrome.tabs.get(targetTabId, (tab) => { 155 | res(!chrome.runtime.lastError && !!tab); // Convert tab to boolean 156 | }); 157 | } 158 | }); 159 | 160 | Logger.log("[refreshCereal] => Tab Exists:", tabExists); 161 | 162 | if (!tabExists) { 163 | targetTabId = null; 164 | } 165 | } catch { 166 | targetTabId = null; 167 | } 168 | 169 | if (targetTabId === null) { 170 | Logger.log("[refreshCereal] => Tab does not exist."); 171 | resolve({ success: false, message: "not tab found" }); 172 | } 173 | 174 | // now we know the tab exists, send message to refresh the frame 175 | Logger.log( 176 | "[refreshCereal] => Sending message to refresh tab:", 177 | targetTabId, 178 | ); 179 | if (targetTabId != null) { 180 | await sendMessageToContentScript(targetTabId, { 181 | intent: "mllwtl_refreshCerealFrame", 182 | }); 183 | resolve({ success: true }); 184 | } 185 | }); 186 | } 187 | -------------------------------------------------------------------------------- /src/cereal/cereal-types.ts: -------------------------------------------------------------------------------- 1 | export interface CerealResponse { 2 | success: boolean; 3 | error?: string; 4 | [key: string]: any; 5 | } 6 | 7 | export interface CerealFrameMessage { 8 | type: "PROCESS_DOCUMENT"; 9 | recordID: string; 10 | htmlString: string; 11 | cerealObject: string; 12 | } 13 | 14 | export interface CerealConfig { 15 | cerealURL: string; 16 | [key: string]: any; 17 | } 18 | 19 | export interface CerealObject { 20 | maxTimeout?: number; 21 | [key: string]: any; 22 | } 23 | -------------------------------------------------------------------------------- /src/cereal/cereal-utils.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from "../logger/logger"; 2 | import { CerealConfig, CerealResponse } from "./cereal-types"; 3 | import { CEREAL_FRAME_ID } from "../constants"; 4 | 5 | export function initCerealFrame( 6 | cerealObject: string | CerealConfig, 7 | ): Promise { 8 | return new Promise((resolve) => { 9 | Logger.log("[initCerealFrame] Starting initialization"); 10 | 11 | // Check if frame already exists 12 | let frame = document.getElementById(CEREAL_FRAME_ID) as HTMLIFrameElement; 13 | if (frame) { 14 | Logger.log("[initCerealFrame] Frame already exists"); 15 | resolve({ success: true }); 16 | return; 17 | } 18 | 19 | // Create new frame as HTMLIFrameElement 20 | frame = document.createElement("iframe") as HTMLIFrameElement; 21 | frame.id = CEREAL_FRAME_ID; 22 | 23 | // Parse cereal object if it's a string 24 | const config: CerealConfig = 25 | typeof cerealObject === "string" 26 | ? JSON.parse(cerealObject) 27 | : cerealObject; 28 | 29 | frame.src = config.cerealURL; 30 | frame.style.display = "none"; 31 | 32 | // Setup load handler before setting src 33 | const timeoutId = setTimeout(() => { 34 | Logger.log("[initCerealFrame] Load timeout"); 35 | resolve({ success: false, error: "timeout" }); 36 | }, 8000); // 8 second timeout for loading 37 | 38 | frame.onload = () => { 39 | clearTimeout(timeoutId); 40 | Logger.log("[initCerealFrame] Frame loaded successfully"); 41 | resolve({ success: true }); 42 | }; 43 | 44 | frame.onerror = (error) => { 45 | clearTimeout(timeoutId); 46 | Logger.log("[initCerealFrame] Frame load error:", error); 47 | resolve({ success: false, error: "load_failed" }); 48 | }; 49 | 50 | // Append frame to document 51 | document.body.appendChild(frame); 52 | Logger.log("[initCerealFrame] Frame appended to document"); 53 | }); 54 | } 55 | -------------------------------------------------------------------------------- /src/constants.ts: -------------------------------------------------------------------------------- 1 | export const VERSION: string = "1.6.2"; 2 | export const MAX_PARALLEL_EXECUTIONS: number = 4; 3 | export const MAX_PARALLEL_EXECUTIONS_BATCH: number = 4; 4 | export const MAX_PARALLEL_EXECUTIONS_FETCH: number = 10; 5 | export const MAX_PARALLEL_EXECUTIONS_BATCH_FETCH: number = 10; 6 | export const MAX_QUEUE_SIZE: number = 24; 7 | export const LIFESPAN_IFRAME: number = 1000 * 60 * 1.5; // 1.5 minutes 8 | export const LIFESPAN_TAB: number = 1000 * 10; // 10 seconds 9 | export const DATA_ID_IFRAME: string = "data-mllwtl-frame-crwl-ml"; 10 | export const DATA_ID_IFRAME_BATCH: string = "data-mllwtl-frame-batch-crwl-ml"; 11 | export const DATA_ID_STRING: string = "data-mllwtl-frame"; 12 | export const CEREAL_FRAME_ID: string = "mllwtl-cereal-frame-id"; 13 | export const RULE_ID_XFRAME: number = 80045; 14 | export const RULE_ID_CONTENT_DISPOSITION: number = 80046; 15 | export const RULE_ID_CONTENT_TYPE: number = 80047; 16 | export const RULE_ID_VALUE_TO_MODIFY_CONTENT_TYPE_TO: number = 80048; 17 | export const RULE_ID_POST_REQUEST: number = 80049; 18 | export const RULE_ID_IMAGE_RENDER: number = 80050; 19 | export const RULE_ID_START_BCREW: number = 221022; 20 | export let MAX_DAILY_RATE: number = 1000; 21 | export const BADGE_COLOR: string = "#4CAF50"; 22 | 23 | export const SW_PING_INTERVAL: number = 3 * 1000; // 3 seconds 24 | export const REFRESH_INTERVAL: number = 1000 * 60 * 60 * 24; // 24 hours 25 | export const SPEED_REFRESH_INTERVAL: number = 1000 * 60 * 60 * 12; // 12 hours 26 | -------------------------------------------------------------------------------- /src/content-script/queue-crawl.ts: -------------------------------------------------------------------------------- 1 | import { getLocalStorage, setLocalStorage } from "../storage/storage-helpers"; 2 | import { Logger } from "../logger/logger"; 3 | import { MAX_QUEUE_SIZE } from "../constants"; 4 | 5 | export async function insertInQueue(dataPacket: any, BATCH_execution: boolean) { 6 | return new Promise((resolve) => { 7 | let queueKey = BATCH_execution ? "queue_batch" : "queue"; 8 | getLocalStorage(queueKey).then((result) => { 9 | if (result === undefined || !result.hasOwnProperty(queueKey)) 10 | result = { [queueKey]: [] }; 11 | let queue = result[queueKey]; 12 | if (queue.length > MAX_QUEUE_SIZE && !BATCH_execution) { 13 | // ignore this packet 14 | Logger.log("[🌐] : queue is full. Ignoring this packet"); 15 | resolve(false); 16 | } else { 17 | queue.push(dataPacket); 18 | setLocalStorage(queueKey, queue).then(() => { 19 | resolve(true); 20 | }); 21 | } 22 | }); 23 | }); 24 | } 25 | 26 | // Get last from queue (by shifting. Not optimized, but it's kind of ok because n is small) 27 | export async function getLastFromQueue(BATCH_execution: boolean): Promise<{ 28 | url: string; 29 | recordID: string; 30 | eventData: any; 31 | waitForElement: string; 32 | shouldSandbox: boolean; 33 | sandBoxAttributes: string; 34 | batch_id: string; 35 | triggersDownload: boolean; 36 | skipHeaders: boolean; 37 | hostname: string; 38 | htmlVisualizer: boolean; 39 | htmlContained: boolean; 40 | screenWidth: string; 41 | screenHeight: string; 42 | POST_request: boolean; 43 | GET_request: boolean; 44 | methodEndpoint: string; 45 | methodPayload: string; 46 | methodHeaders: any; 47 | actions: string; 48 | delayBetweenExecutions: number; 49 | openTab: boolean; 50 | openTabOnlyIfMust: boolean; 51 | pascoli: boolean; 52 | refPolicy: string; 53 | rawData: boolean; 54 | cerealObject: string; 55 | bCrewObject: string; 56 | burkeObject: string; 57 | }> { 58 | return new Promise((resolve) => { 59 | let queueKey = BATCH_execution ? "queue_batch" : "queue"; 60 | getLocalStorage(queueKey).then((result) => { 61 | if (result === undefined || !result.hasOwnProperty(queueKey)) 62 | result = { [queueKey]: [] }; 63 | let queue = result[queueKey]; 64 | if (queue.length === 0) 65 | return resolve({ 66 | url: "", 67 | recordID: "0123", 68 | eventData: {}, 69 | waitForElement: "none", 70 | shouldSandbox: false, 71 | sandBoxAttributes: "", 72 | batch_id: "", 73 | triggersDownload: false, 74 | skipHeaders: false, 75 | hostname: "", 76 | htmlVisualizer: false, 77 | htmlContained: false, 78 | screenWidth: "1024px", 79 | screenHeight: "768px", 80 | POST_request: false, 81 | GET_request: false, 82 | methodEndpoint: "", 83 | methodPayload: "no_payload", 84 | methodHeaders: "no_headers", 85 | actions: "[]", 86 | delayBetweenExecutions: 500, 87 | openTab: false, 88 | openTabOnlyIfMust: false, 89 | pascoli: false, 90 | refPolicy: "", 91 | rawData: false, 92 | cerealObject: "", 93 | bCrewObject: "", 94 | burkeObject: "", 95 | }); 96 | let last = queue.shift(); 97 | setLocalStorage(queueKey, queue).then(() => { 98 | resolve(last); 99 | }); 100 | }); 101 | }); 102 | } 103 | -------------------------------------------------------------------------------- /src/content-script/reset-crawl.ts: -------------------------------------------------------------------------------- 1 | import { getLastFromQueue } from "./queue-crawl"; 2 | import { 3 | DATA_ID_IFRAME, 4 | LIFESPAN_IFRAME, 5 | MAX_PARALLEL_EXECUTIONS, 6 | MAX_PARALLEL_EXECUTIONS_BATCH, 7 | } from "../constants"; 8 | import { proceedWithActivation } from "./execute-crawl"; 9 | import { getFrameCount, isInSW } from "../utils/utils"; 10 | import { enableXFrameHeaders } from "../dnr/dnr-helpers"; 11 | import { Logger } from "../logger/logger"; 12 | import { resetTriggersDownload } from "../utils/triggers-download-helpers"; 13 | import { hideBadgeIfShould } from "../transparency/badge-settings"; 14 | import { deleteFromRequestInfoStorage } from "../request-info/request-info-helpers"; 15 | import { deleteFromRequestMessageStorage } from "../request-message/request-message-helpers"; 16 | import { sendMessageToContentScript } from "../utils/messaging-helpers"; 17 | import { waitForResetInterval } from "../utils/trigger-storage"; 18 | 19 | export async function resetAfterCrawl( 20 | recordID: string, 21 | BATCH_execution: boolean, 22 | delayBetweenExecutions: number = 500, 23 | ) { 24 | return new Promise(async (resolve) => { 25 | if (await isInSW()) { 26 | Logger.log( 27 | "[resetAfterCrawl] : In service worker. Sending message to content script", 28 | ); 29 | chrome.tabs.query({}, async function (tabs) { 30 | for (let i = 0; i < tabs.length; i++) { 31 | let response = await sendMessageToContentScript(tabs[i]?.id!, { 32 | intent: "resetAfterCrawl", 33 | recordID: recordID, 34 | BATCH_execution: BATCH_execution, 35 | delayBetweenExecutions: delayBetweenExecutions, 36 | }); 37 | if (response !== null) { 38 | break; 39 | } 40 | } 41 | }); 42 | resolve("done"); 43 | } else { 44 | if ( 45 | delayBetweenExecutions === undefined || 46 | delayBetweenExecutions === null 47 | ) { 48 | Logger.log( 49 | "[resetAfterCrawl] : delayBetweenExecutions is undefined or null. Setting it to 500", 50 | ); 51 | delayBetweenExecutions = 500; 52 | } 53 | await deleteFromRequestInfoStorage(recordID); 54 | await deleteFromRequestMessageStorage(recordID); 55 | let dataPacket = await getLastFromQueue(BATCH_execution); 56 | Logger.log("[resetAfterCrawl] : dataPacket => "); 57 | Logger.log(dataPacket); 58 | Logger.log("##############################"); 59 | if (dataPacket && dataPacket.url !== "") { 60 | let frameCount = getFrameCount(BATCH_execution); 61 | Logger.log("[🌐] : frameCount in cleanUpAfterCrawl => " + frameCount); 62 | let max_parallel_executions = BATCH_execution 63 | ? MAX_PARALLEL_EXECUTIONS_BATCH 64 | : MAX_PARALLEL_EXECUTIONS; 65 | if (frameCount <= max_parallel_executions || BATCH_execution) { 66 | Logger.log("[🌐] getLastFromQueue : dataPacket => "); 67 | Logger.log(dataPacket); 68 | if (BATCH_execution && dataPacket.methodEndpoint !== "") { 69 | Logger.log( 70 | "[🌐] : Waiting for delayBetweenExecutions => " + 71 | delayBetweenExecutions, 72 | ); 73 | // if fetching during batch execution, wait delayBetweenExecutions before proceeding 74 | setTimeout(() => { 75 | proceedWithActivation( 76 | dataPacket.url, 77 | dataPacket.recordID, 78 | dataPacket.eventData, 79 | dataPacket.waitForElement, 80 | dataPacket.shouldSandbox, 81 | dataPacket.sandBoxAttributes, 82 | BATCH_execution, 83 | dataPacket.batch_id, 84 | dataPacket.triggersDownload, 85 | dataPacket.skipHeaders, 86 | dataPacket.hostname, 87 | dataPacket.htmlVisualizer, 88 | dataPacket.htmlContained, 89 | dataPacket.screenWidth, 90 | dataPacket.screenHeight, 91 | dataPacket.POST_request, 92 | dataPacket.GET_request, 93 | dataPacket.methodEndpoint, 94 | dataPacket.methodPayload, 95 | dataPacket.methodHeaders, 96 | dataPacket.actions, 97 | dataPacket.delayBetweenExecutions, 98 | dataPacket.openTab, 99 | dataPacket.openTabOnlyIfMust, 100 | dataPacket.pascoli, 101 | dataPacket.cerealObject, 102 | dataPacket.refPolicy, 103 | dataPacket.bCrewObject, 104 | dataPacket.burkeObject, 105 | ); 106 | resolve("done"); 107 | }, delayBetweenExecutions); 108 | } else { 109 | await proceedWithActivation( 110 | dataPacket.url, 111 | dataPacket.recordID, 112 | dataPacket.eventData, 113 | dataPacket.waitForElement, 114 | dataPacket.shouldSandbox, 115 | dataPacket.sandBoxAttributes, 116 | BATCH_execution, 117 | dataPacket.batch_id, 118 | dataPacket.triggersDownload, 119 | dataPacket.skipHeaders, 120 | dataPacket.hostname, 121 | dataPacket.htmlVisualizer, 122 | dataPacket.htmlContained, 123 | dataPacket.screenWidth, 124 | dataPacket.screenHeight, 125 | dataPacket.POST_request, 126 | dataPacket.GET_request, 127 | dataPacket.methodEndpoint, 128 | dataPacket.methodPayload, 129 | dataPacket.methodHeaders, 130 | dataPacket.actions, 131 | dataPacket.delayBetweenExecutions, 132 | dataPacket.openTab, 133 | dataPacket.openTabOnlyIfMust, 134 | dataPacket.pascoli, 135 | dataPacket.cerealObject, 136 | dataPacket.refPolicy, 137 | dataPacket.bCrewObject, 138 | dataPacket.burkeObject, 139 | ); 140 | resolve("done"); 141 | } 142 | } else { 143 | resolve("done"); 144 | } 145 | } else { 146 | setTimeout(async () => { 147 | let frameCount = getFrameCount(BATCH_execution); 148 | let frameCountOther = getFrameCount(!BATCH_execution); 149 | let frameCountTotal = frameCount + frameCountOther; 150 | Logger.log( 151 | "[🌐] : frameCountTotal in cleanUpAfterCrawl (before resetting headers) => " + 152 | frameCountTotal, 153 | ); 154 | if (frameCountTotal === 0 && !BATCH_execution) { 155 | Logger.log("[🌐] : Resetting headers!"); 156 | enableXFrameHeaders(""); 157 | Logger.log("[🌐] : Waiting for minimum reset interval..."); 158 | await waitForResetInterval(); 159 | Logger.log("[🌐] : Resetting headers!"); 160 | resetTriggersDownload(); 161 | resolve("done"); 162 | } else if (frameCountTotal === 0 && BATCH_execution) { 163 | // wait for 1 minute before resetting headers 164 | setTimeout(async () => { 165 | Logger.log("[🌐] : Resetting headers (BATCH_execution)!"); 166 | enableXFrameHeaders(""); 167 | Logger.log("[🌐] : Waiting for minimum reset interval..."); 168 | await waitForResetInterval(); 169 | Logger.log("[🌐] : Resetting headers!"); 170 | resetTriggersDownload(); 171 | resolve("done"); 172 | }, 60000); 173 | } /* else { 174 | resetAfterCrawl(recordID, BATCH_execution); 175 | }*/ 176 | }, 15000); 177 | } 178 | } 179 | }); 180 | } 181 | 182 | export function setLifespanForIframe( 183 | recordID: string, 184 | waitBeforeScraping: number, 185 | BATCH_execution: boolean, 186 | delayBetweenExecutions: number = 500, 187 | ) { 188 | Logger.log( 189 | "Setting lifespan for iframe => " + 190 | (LIFESPAN_IFRAME + waitBeforeScraping) + 191 | " ms. RecordID => " + 192 | recordID, 193 | ); 194 | setTimeout(async () => { 195 | let iframe = document.getElementById(recordID); 196 | let dataId = iframe?.getAttribute("data-id") || ""; 197 | let divIframe = document.getElementById("div-" + recordID); 198 | if (iframe) iframe.remove(); 199 | if (divIframe) divIframe.remove(); 200 | await resetAfterCrawl(recordID, BATCH_execution, delayBetweenExecutions); 201 | if (dataId === DATA_ID_IFRAME) { 202 | await hideBadgeIfShould(); 203 | } 204 | }, LIFESPAN_IFRAME + waitBeforeScraping); 205 | } 206 | -------------------------------------------------------------------------------- /src/content-script/shared-memory.ts: -------------------------------------------------------------------------------- 1 | import { 2 | sendMessageToBackground, 3 | sendMessageToContentScript, 4 | } from "../utils/messaging-helpers"; 5 | 6 | export function setSharedMemory(key: string, value: any): void { 7 | let hiddenInput: HTMLInputElement = document.createElement("input"); 8 | hiddenInput.setAttribute("type", "hidden"); 9 | hiddenInput.setAttribute("id", key); 10 | hiddenInput.setAttribute("value", value); 11 | document.body.appendChild(hiddenInput); 12 | sendMessageToBackground({ 13 | intent: "setSharedMemoryBCK", 14 | key: key, 15 | value: value, 16 | }).then(); 17 | } 18 | 19 | export function getSharedMemory(key: string): Promise { 20 | return new Promise((resolve) => { 21 | // send message to background.js, that will then broadcast it to all tabs 22 | sendMessageToBackground({ 23 | intent: "getSharedMemoryBCK", 24 | key: key, 25 | }).then(function (response) { 26 | resolve(response); 27 | }); 28 | }); 29 | } 30 | 31 | export function getSharedMemoryDOM(key: string) { 32 | return new Promise((resolve) => { 33 | let hiddenInput: HTMLElement | null = document.getElementById(key); 34 | if (hiddenInput) { 35 | resolve(true); 36 | } else { 37 | resolve(false); 38 | } 39 | }); 40 | } 41 | 42 | export function removeSharedMemory(key: string): void { 43 | let hiddenInput: HTMLElement | null = document.getElementById(key); 44 | if (hiddenInput) hiddenInput.remove(); 45 | } 46 | 47 | export async function setSharedMemoryBCK(key: string, tabId: number) { 48 | return new Promise(function (res) { 49 | chrome.storage.local.set({ [key]: tabId }, function () { 50 | res("done"); 51 | }); 52 | }); 53 | } 54 | 55 | export async function getSharedMemoryBCK(key: string): Promise { 56 | return new Promise(function (res) { 57 | chrome.tabs.query({}, function (tabs) { 58 | let numTabs: number = tabs.length; 59 | let numTabsChecked: number = 0; 60 | let mllwtlFramePresent: boolean = false; 61 | for (let i = 0; i < numTabs; i++) { 62 | /* TODO: FIX THIS 63 | if (tabs[i]?.url?.includes("chrome://")) { 64 | numTabsChecked++; 65 | if (numTabsChecked === numTabs) { 66 | res(mllwtlFramePresent); 67 | } 68 | continue; 69 | }*/ 70 | sendMessageToContentScript(tabs[i].id!, { 71 | intent: "getSharedMemoryDOM", 72 | key: key, 73 | }).then(function (response): void { 74 | numTabsChecked++; 75 | if (response) { 76 | mllwtlFramePresent = true; 77 | } 78 | if (numTabsChecked === numTabs) { 79 | res(mllwtlFramePresent); 80 | } 81 | }); 82 | } 83 | }); 84 | }); 85 | } 86 | -------------------------------------------------------------------------------- /src/content-script/storage-change-listeners.ts: -------------------------------------------------------------------------------- 1 | import { startWebsocket, stopConnection } from "../utils/start-stop-helpers"; 2 | import { Logger } from "../logger/logger"; 3 | 4 | export async function setUpStorageChangeListeners(): Promise { 5 | return new Promise((resolve) => { 6 | chrome.storage.onChanged.addListener(function (changes, namespace) { 7 | for (let key in changes) { 8 | if (key === "mStatus") { 9 | let newValue = changes[key].newValue; 10 | if (newValue === "start") { 11 | Logger.info("Starting to connect..."); 12 | startWebsocket(); 13 | } else if (newValue === "stop") { 14 | Logger.info("Stopping the connection..."); 15 | stopConnection(); 16 | } 17 | } 18 | } 19 | }); 20 | }); 21 | } 22 | -------------------------------------------------------------------------------- /src/content-script/test-csp.ts: -------------------------------------------------------------------------------- 1 | export function testCSP(onload_callback = function () {}): void { 2 | const img: HTMLImageElement = new Image(); 3 | img.src = 4 | "https://mellowtel-bucket.s3.us-east-1.amazonaws.com/lightning-boltrepo-com.svg"; 5 | img.onload = onload_callback; 6 | img.id = "test-csp-image"; 7 | img.style.display = "none"; 8 | document.body.appendChild(img); 9 | } 10 | 11 | export function removeCSPTestImage(): void { 12 | const img: HTMLImageElement | null = document.getElementById( 13 | "test-csp-image", 14 | ) as HTMLImageElement; 15 | if (img) img.remove(); 16 | } 17 | 18 | export function isCSPEnabled(): Promise { 19 | return new Promise((resolve) => { 20 | document.addEventListener("securitypolicyviolation", (e) => { 21 | if ( 22 | e.blockedURI === 23 | "https://m-bucket-light.s3.amazonaws.com/lightning-boltrepo-com.svg" 24 | ) { 25 | removeCSPTestImage(); 26 | resolve(true); 27 | } 28 | }); 29 | testCSP(() => { 30 | removeCSPTestImage(); 31 | resolve(false); 32 | }); 33 | }); 34 | } 35 | -------------------------------------------------------------------------------- /src/dnr/dnr-helpers.ts: -------------------------------------------------------------------------------- 1 | import HeaderOperation = chrome.declarativeNetRequest.HeaderOperation; 2 | import RuleActionType = chrome.declarativeNetRequest.RuleActionType; 3 | import ResourceType = chrome.declarativeNetRequest.ResourceType; 4 | import { sendMessageToBackground } from "../utils/messaging-helpers"; 5 | import { 6 | RULE_ID_IMAGE_RENDER, 7 | RULE_ID_POST_REQUEST, 8 | RULE_ID_XFRAME, 9 | } from "../constants"; 10 | import { Logger } from "../logger/logger"; 11 | import { isInSW } from "../utils/utils"; 12 | 13 | export function disableXFrameHeaders( 14 | hostname: string, 15 | skipHeaders: boolean, 16 | ): Promise { 17 | return new Promise(function (res) { 18 | if (skipHeaders) { 19 | res(false); 20 | } else { 21 | let ruleId = getRuleIdFromHostname(hostname); 22 | shouldDelegateDNR().then((delegate) => { 23 | if (delegate) { 24 | sendMessageToBackground({ 25 | intent: "disableXFrameHeaders", 26 | hostname: hostname, 27 | skipHeaders: skipHeaders, 28 | }).then(() => { 29 | res(true); 30 | }); 31 | } else { 32 | chrome.declarativeNetRequest.updateSessionRules({ 33 | removeRuleIds: [ruleId], 34 | addRules: [ 35 | { 36 | id: ruleId, 37 | priority: 1, 38 | action: { 39 | type: "modifyHeaders" as RuleActionType, 40 | responseHeaders: [ 41 | { 42 | header: "x-frame-options", 43 | operation: "remove" as HeaderOperation, 44 | }, 45 | { 46 | header: "content-security-policy", 47 | operation: "remove" as HeaderOperation, 48 | }, 49 | /*{ 50 | header: "X-Frame-Options", 51 | operation: "remove" as HeaderOperation, 52 | }, 53 | { 54 | header: "Content-Security-Policy", 55 | operation: "remove" as HeaderOperation, 56 | }, 57 | { 58 | header: "Frame-Options", 59 | operation: "remove" as HeaderOperation, 60 | },*/ 61 | { 62 | header: "cross-origin-embedder-policy", 63 | operation: "remove" as HeaderOperation, 64 | }, 65 | { 66 | header: "cross-origin-opener-policy", 67 | operation: "remove" as HeaderOperation, 68 | }, 69 | { 70 | header: "cross-origin-resource-policy", 71 | operation: "remove" as HeaderOperation, 72 | }, 73 | { 74 | header: "content-security-policy-report-only", 75 | operation: "remove" as HeaderOperation, 76 | }, 77 | ], 78 | }, 79 | condition: { 80 | resourceTypes: ["sub_frame" as ResourceType], 81 | urlFilter: "*://*/*", 82 | // `*${hostname}*`, --> specific filter disabled because 83 | // there are internal redirects that need to be handled. 84 | // Need to find a way to handle redirects and disable headers 85 | // for all of them (while leaving them on for other sites). 86 | }, 87 | }, 88 | ], 89 | }); 90 | res(true); 91 | } 92 | }); 93 | } 94 | }); 95 | } 96 | 97 | export function enableXFrameHeaders(hostname: string): Promise { 98 | return new Promise(function (res) { 99 | let ruleId = getRuleIdFromHostname(hostname); 100 | shouldDelegateDNR().then((delegate) => { 101 | if (delegate) { 102 | sendMessageToBackground({ 103 | intent: "enableXFrameHeaders", 104 | hostname: hostname, 105 | }).then(() => { 106 | res(true); 107 | }); 108 | } else { 109 | chrome.declarativeNetRequest.updateSessionRules({ 110 | removeRuleIds: [ruleId], 111 | }); 112 | res(true); 113 | } 114 | }); 115 | }); 116 | } 117 | 118 | export function fixImageRenderHTMLVisualizer(): Promise { 119 | return new Promise(function (res) { 120 | const rule = { 121 | id: RULE_ID_IMAGE_RENDER, 122 | priority: 1, 123 | action: { 124 | type: "modifyHeaders", 125 | requestHeaders: [ 126 | { 127 | header: "Origin", 128 | operation: "remove", 129 | }, 130 | ], 131 | responseHeaders: [ 132 | { 133 | header: "Access-Control-Allow-Origin", 134 | operation: "set", 135 | value: "*", 136 | }, 137 | { 138 | header: "Access-Control-Allow-Methods", 139 | operation: "set", 140 | value: "GET, POST, PUT, DELETE, OPTIONS", 141 | }, 142 | { 143 | header: "Access-Control-Allow-Headers", 144 | operation: "set", 145 | value: "Content-Type", 146 | }, 147 | ], 148 | }, 149 | condition: { 150 | urlFilter: "*://*/*", 151 | resourceTypes: ["image"], 152 | }, 153 | }; 154 | // Add the dynamic rule 155 | chrome.declarativeNetRequest.updateSessionRules( 156 | { 157 | removeRuleIds: [RULE_ID_IMAGE_RENDER], // Clear any existing rules with the same ID to avoid duplicates 158 | addRules: [rule as any], 159 | }, 160 | () => { 161 | if (chrome.runtime.lastError) { 162 | Logger.log("Error adding rule:", chrome.runtime.lastError); 163 | res(false); 164 | } else { 165 | Logger.log("Rule added successfully"); 166 | res(true); 167 | } 168 | }, 169 | ); 170 | }); 171 | } 172 | 173 | export function resetImageRenderHTMLVisualizer(): Promise { 174 | return new Promise(function (res) { 175 | chrome.declarativeNetRequest.updateSessionRules({ 176 | removeRuleIds: [RULE_ID_IMAGE_RENDER], 177 | }); 178 | res(true); 179 | }); 180 | } 181 | 182 | export function disableHeadersForPOST(): Promise { 183 | return new Promise(function (res) { 184 | chrome.declarativeNetRequest.updateSessionRules({ 185 | removeRuleIds: [RULE_ID_POST_REQUEST], 186 | addRules: [ 187 | { 188 | id: RULE_ID_POST_REQUEST, 189 | priority: 1, 190 | action: { 191 | type: "modifyHeaders" as RuleActionType, 192 | requestHeaders: [ 193 | { 194 | header: "Origin", 195 | operation: "remove" as HeaderOperation, 196 | }, 197 | ], 198 | responseHeaders: [ 199 | { 200 | header: "Access-Control-Allow-Origin", 201 | operation: "set" as HeaderOperation, 202 | value: "*", 203 | }, 204 | { 205 | header: "Access-Control-Allow-Methods", 206 | operation: "set" as HeaderOperation, 207 | value: "GET, POST, PUT, DELETE, OPTIONS", 208 | }, 209 | { 210 | header: "Access-Control-Allow-Headers", 211 | operation: "set" as HeaderOperation, 212 | value: "Content-Type", 213 | }, 214 | ], 215 | }, 216 | condition: { 217 | urlFilter: "*://*/*", 218 | resourceTypes: ["xmlhttprequest" as ResourceType], 219 | }, 220 | }, 221 | ], 222 | }); 223 | res(true); 224 | }); 225 | } 226 | 227 | export function enableHeadersForPOST(): Promise { 228 | return new Promise(function (res) { 229 | chrome.declarativeNetRequest.updateSessionRules({ 230 | removeRuleIds: [RULE_ID_POST_REQUEST], 231 | }); 232 | res(true); 233 | }); 234 | } 235 | 236 | export function getRuleIdFromHostname(hostname: string): number { 237 | // Disabled because a hostname can redirect to another hostname. 238 | // We need to disable the headers for the redirected hostname too. 239 | // TODO: Find a way to "map out" the redirects and disable the headers for all of them. 240 | /* 241 | let hashNumber = 0; 242 | for (let i = 0; i < hostname.length; i++) { 243 | hashNumber += hostname.charCodeAt(i); 244 | } 245 | return hashNumber; 246 | */ 247 | return RULE_ID_XFRAME; 248 | } 249 | 250 | export function shouldDelegateDNR(): Promise { 251 | return new Promise((resolve) => { 252 | try { 253 | chrome.declarativeNetRequest.getSessionRules((rules) => { 254 | if (chrome.runtime.lastError) { 255 | resolve(true); 256 | } else { 257 | resolve(false); 258 | } 259 | }); 260 | } catch (error) { 261 | resolve(true); 262 | } 263 | }); 264 | } 265 | 266 | export async function cleaunUpRules() { 267 | const RULES_TO_REMOVE = [RULE_ID_POST_REQUEST]; 268 | 269 | const isInServiceWorker = await isInSW(); 270 | 271 | if (!isInServiceWorker) { 272 | await sendMessageToBackground({ 273 | intent: "cleanUpDNRRules", 274 | }); 275 | } else { 276 | await chrome.declarativeNetRequest.updateSessionRules({ 277 | removeRuleIds: RULES_TO_REMOVE, 278 | }); 279 | } 280 | } 281 | -------------------------------------------------------------------------------- /src/elements/elements-utils.ts: -------------------------------------------------------------------------------- 1 | import { sendMessageToContentScript } from "../utils/messaging-helpers"; 2 | import { DATA_ID_STRING } from "../constants"; 3 | 4 | export async function getIfCurrentlyActiveBCK() { 5 | return new Promise(function (res) { 6 | chrome.tabs.query({}, function (tabs) { 7 | let numTabs: number = tabs.length; 8 | let numTabsChecked: number = 0; 9 | let mllwtlFramePresent: boolean = false; 10 | for (let i = 0; i < numTabs; i++) { 11 | sendMessageToContentScript(tabs[i].id!, { 12 | intent: "getSharedMemoryDOM", 13 | key: "webSocketConnected", 14 | }).then(function (response): void { 15 | numTabsChecked++; 16 | if (response) { 17 | mllwtlFramePresent = true; 18 | } 19 | if (numTabsChecked === numTabs) { 20 | res(mllwtlFramePresent); 21 | } 22 | }); 23 | } 24 | }); 25 | }); 26 | } 27 | 28 | export function getIfCurrentlyActiveDOM() { 29 | return new Promise((resolve) => { 30 | let frame: HTMLIFrameElement | null = document.querySelector( 31 | `[id^=${DATA_ID_STRING}]`, 32 | ); 33 | if (frame) { 34 | resolve(true); 35 | } else { 36 | resolve(false); 37 | } 38 | }); 39 | } 40 | -------------------------------------------------------------------------------- /src/elements/generate-links.ts: -------------------------------------------------------------------------------- 1 | import { 2 | getExtensionIdentifier, 3 | getIdentifier, 4 | } from "../utils/identity-helpers"; 5 | import { setLocalStorage, getLocalStorage } from "../storage/storage-helpers"; 6 | import { shouldDelegateTabsAPI } from "../utils/tabs-helpers"; 7 | import { sendMessageToBackground } from "../utils/messaging-helpers"; 8 | import { 9 | detectBrowser, 10 | getManifestVersion, 11 | openPopupWindow, 12 | } from "../utils/utils"; 13 | import { Logger } from "../logger/logger"; 14 | const BASE_DOMAIN: string = "https://www.mellow.tel/"; 15 | const BASE_LINK_SETTING: string = BASE_DOMAIN + "settings/"; 16 | const BASE_LINK_OPT_IN: string = BASE_DOMAIN + "opt-in/"; 17 | const BASE_LINK_UPDATE: string = BASE_DOMAIN + "update/"; 18 | const BASE_LINK_FEEDBACK: string = BASE_DOMAIN + "uninstall-feedback/"; 19 | 20 | /* 21 | generateAndOpenOptInLink is a convenience function that generates an opt-in link 22 | and opens it in a new tab. It returns a Promise that resolves to the generated link. 23 | It also keeps track to only open it once. 24 | It can be called both from the background script and the content script. 25 | If called from the content script, it will send a message to the background script to open the link. 26 | */ 27 | 28 | const optInOpenedKey: string = "mellowtelOptInOpened"; 29 | const updateOpenedKey: string = "mUpdateOpened"; 30 | 31 | async function setAlreadyOpened( 32 | optInOrUpdate: string = "optIn", 33 | ): Promise { 34 | return new Promise((resolve) => { 35 | let optIn = optInOrUpdate === "optIn"; 36 | setLocalStorage(optIn ? optInOpenedKey : updateOpenedKey, "true").then( 37 | () => { 38 | resolve(true); 39 | }, 40 | ); 41 | }); 42 | } 43 | 44 | async function getAlreadyOpened( 45 | optInOrUpdate: string = "optIn", 46 | ): Promise { 47 | return new Promise((resolve) => { 48 | let optIn = optInOrUpdate === "optIn"; 49 | getLocalStorage(optIn ? optInOpenedKey : updateOpenedKey).then((result) => { 50 | if ( 51 | result === undefined || 52 | !result.hasOwnProperty(optIn ? optInOpenedKey : updateOpenedKey) 53 | ) { 54 | resolve(false); 55 | } else { 56 | let opened = 57 | result[optIn ? optInOpenedKey : updateOpenedKey] 58 | .toString() 59 | .toLowerCase() === "true"; 60 | resolve(opened); 61 | } 62 | }); 63 | }); 64 | } 65 | 66 | export function generateAndOpenOptInLink(): Promise { 67 | return new Promise(async (resolve) => { 68 | // if not access to tabs api, send message to background script to open the link 69 | let shouldDelegate = await shouldDelegateTabsAPI(); 70 | if (shouldDelegate) { 71 | let link = await sendMessageToBackground({ intent: "openOptInLink" }); 72 | resolve(link); 73 | } 74 | let alreadyOpened = await getAlreadyOpened("optIn"); 75 | if (!alreadyOpened) { 76 | let extension_id = await getExtensionIdentifier(); 77 | getIdentifier().then(async (deviceId) => { 78 | let configuration_key = deviceId.split("_")[1]; 79 | let link = `${BASE_LINK_OPT_IN}?extension_id=${encodeURIComponent(extension_id)}&configuration_key=${configuration_key}&browser=${detectBrowser()}`; 80 | await setAlreadyOpened("optIn"); 81 | chrome.tabs.create({ url: link }); 82 | resolve(link); 83 | }); 84 | } else { 85 | resolve(""); 86 | } 87 | }); 88 | } 89 | 90 | export function generateAndOpenUpdateLink(): Promise { 91 | return new Promise(async (resolve) => { 92 | // if not access to tabs api, send message to background script to open the link 93 | let shouldDelegate = await shouldDelegateTabsAPI(); 94 | if (shouldDelegate) { 95 | let link = await sendMessageToBackground({ intent: "openUpdateLink" }); 96 | resolve(link); 97 | } 98 | let alreadyOpened = await getAlreadyOpened("update"); 99 | if (!alreadyOpened) { 100 | let extension_id = await getExtensionIdentifier(); 101 | getIdentifier().then(async (deviceId) => { 102 | let configuration_key = deviceId.split("_")[1]; 103 | let link = `${BASE_LINK_UPDATE}?extension_id=${encodeURIComponent(extension_id)}&configuration_key=${configuration_key}&browser=${detectBrowser()}`; 104 | await setAlreadyOpened("update"); 105 | chrome.tabs.create({ url: link, pinned: true, active: true }, (tab) => { 106 | resolve(link); 107 | }); 108 | }); 109 | } else { 110 | resolve(""); 111 | } 112 | }); 113 | } 114 | 115 | export function generateOptInLink(): Promise { 116 | return new Promise(async (resolve) => { 117 | let extension_id = await getExtensionIdentifier(); 118 | getIdentifier().then((deviceId) => { 119 | let configuration_key = deviceId.split("_")[1]; 120 | resolve( 121 | `${BASE_LINK_OPT_IN}?extension_id=${encodeURIComponent(extension_id)}&configuration_key=${configuration_key}&browser=${detectBrowser()}`, 122 | ); 123 | }); 124 | }); 125 | } 126 | 127 | export function generateUpdateLink(): Promise { 128 | return new Promise(async (resolve) => { 129 | let extension_id = await getExtensionIdentifier(); 130 | getIdentifier().then((deviceId) => { 131 | let configuration_key = deviceId.split("_")[1]; 132 | resolve( 133 | `${BASE_LINK_UPDATE}?extension_id=${encodeURIComponent(extension_id)}&configuration_key=${configuration_key}&browser=${detectBrowser()}`, 134 | ); 135 | }); 136 | }); 137 | } 138 | 139 | export function generateSettingsLink(): Promise { 140 | return new Promise(async (resolve) => { 141 | let extension_id = await getExtensionIdentifier(); 142 | getIdentifier().then((deviceId) => { 143 | let configuration_key = deviceId.split("_")[1]; 144 | resolve( 145 | `${BASE_LINK_SETTING}?extension_id=${encodeURIComponent(extension_id)}&configuration_key=${configuration_key}&browser=${detectBrowser()}`, 146 | ); 147 | }); 148 | }); 149 | } 150 | 151 | export function openUserSettingsInPopupWindow(): Promise { 152 | return new Promise(async (resolve) => { 153 | let userSettingsLink: string = await generateSettingsLink(); 154 | let isInBackgroundScript: boolean = !(await shouldDelegateTabsAPI()); 155 | let manifestVersion = getManifestVersion(); 156 | if (isInBackgroundScript && manifestVersion !== 2) { 157 | Logger.log( 158 | "openUserSettingsInPopupWindow: Method not supported in background script", 159 | ); 160 | resolve(false); 161 | } 162 | await openPopupWindow(userSettingsLink, "Settings", 768, 400); 163 | resolve(true); 164 | }); 165 | } 166 | 167 | export function generateAndOpenFeedbackLink(): Promise { 168 | return new Promise(async (resolve) => { 169 | // if not access to tabs api, send message to background script to open the link 170 | let shouldDelegate = await shouldDelegateTabsAPI(); 171 | if (shouldDelegate) { 172 | let link = await sendMessageToBackground({ intent: "openFeedbackLink" }); 173 | resolve(link); 174 | } 175 | let extension_id = await getExtensionIdentifier(); 176 | getIdentifier().then(async (deviceId) => { 177 | let configuration_key = deviceId.split("_")[1]; 178 | let link = `${BASE_LINK_FEEDBACK}?extension_id=${encodeURIComponent(extension_id)}&configuration_key=${configuration_key}`; 179 | chrome.tabs.create({ url: link }, (tab) => { 180 | resolve(link); 181 | }); 182 | }); 183 | }); 184 | } 185 | 186 | export function generateFeedbackLink(): Promise { 187 | return new Promise(async (resolve) => { 188 | let extension_id = await getExtensionIdentifier(); 189 | getIdentifier().then((deviceId) => { 190 | let configuration_key = deviceId.split("_")[1]; 191 | resolve( 192 | `${BASE_LINK_FEEDBACK}?extension_id=${encodeURIComponent(extension_id)}&configuration_key=${configuration_key}`, 193 | ); 194 | }); 195 | }); 196 | } 197 | -------------------------------------------------------------------------------- /src/get-requests/get-helpers.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from "../logger/logger"; 2 | import { 3 | disableHeadersForPOST, 4 | enableHeadersForPOST, 5 | } from "../dnr/dnr-helpers"; 6 | import { sendMessageToContentScript } from "../utils/messaging-helpers"; 7 | import { saveJSON } from "../post-requests/post-helpers"; 8 | import { addToRequestInfoStorage } from "../request-info/request-info-helpers"; 9 | import { tellToDeleteIframe } from "../iframe/message-background"; 10 | 11 | export function handleGetRequest( 12 | method_endpoint: string, 13 | method_headers: string, 14 | fastLane: boolean, 15 | orgId: string, 16 | recordID: string, 17 | htmlVisualizer: boolean, 18 | htmlContained: boolean, 19 | removeImages: boolean, 20 | removeCSSselectors: string, 21 | classNamesToBeRemoved: string, 22 | htmlTransformer: string, 23 | BATCH_execution: boolean, 24 | batch_id: string, 25 | actions: string, 26 | delayBetweenExecutions: number = 500, 27 | openTab: boolean = false, 28 | openTabOnlyIfMust: boolean = false, 29 | saveHtml: boolean = true, 30 | saveMarkdown: boolean = true, 31 | cerealObject: string = "{}", 32 | refPolicy: string = "", 33 | ) { 34 | return new Promise(async function (res) { 35 | await disableHeadersForPOST(); 36 | // make a fetch/post to the endpoint with the payload (if not empty) 37 | // then save the JSON response to the server 38 | // and return the response to the caller 39 | const requestOptions: { 40 | method: string; 41 | credentials: RequestCredentials; 42 | body?: any; 43 | headers?: any; 44 | } = { 45 | method: "GET", 46 | // we're omitting credentials to avoid leaking cookies & session data 47 | // this is a security measure to protect the user's data 48 | credentials: "omit", 49 | }; 50 | if (method_headers !== "no_headers") { 51 | try { 52 | method_headers = JSON.parse(method_headers); 53 | requestOptions["headers"] = method_headers; 54 | } catch (e) {} 55 | } 56 | let statusCode: number = 1000; 57 | fetch(method_endpoint, requestOptions) 58 | .then((response) => { 59 | statusCode = response.status; 60 | return response.text(); 61 | }) 62 | .then(async (html_or_json: string) => { 63 | // could be json or html 64 | // Logger.log("HTML or JSON:", html_or_json); 65 | try { 66 | JSON.parse(html_or_json); 67 | await saveJSON( 68 | recordID, 69 | JSON.parse(html_or_json), 70 | orgId, 71 | fastLane, 72 | method_endpoint, 73 | BATCH_execution, 74 | batch_id, 75 | statusCode, 76 | ); 77 | await tellToDeleteIframe( 78 | recordID, 79 | BATCH_execution, 80 | delayBetweenExecutions, 81 | ); 82 | } catch (_) { 83 | Logger.log("[handleGetRequest]: Not JSON"); 84 | Logger.log( 85 | "[handleGetRequest]: HTML:", 86 | html_or_json.substring(0, 140), 87 | ); 88 | Logger.log("[handleGetRequest]: BATCH_execution:", BATCH_execution); 89 | Logger.log("[handleGetRequest]: batch_id:", batch_id); 90 | await addToRequestInfoStorage({ 91 | recordID: recordID, 92 | isPDF: false, 93 | isOfficeDoc: false, 94 | statusCode: statusCode, 95 | }); 96 | // not json 97 | // query a tab and send a message 98 | // then save the message to the server 99 | chrome.tabs.query({}, async function (tabs) { 100 | for (let i = 0; i < tabs.length; i++) { 101 | let response = await sendMessageToContentScript(tabs[i].id!, { 102 | intent: "processCrawl", 103 | recordID: recordID, 104 | fastLane: fastLane, 105 | orgId: orgId, 106 | htmlVisualizer: htmlVisualizer, 107 | htmlContained: htmlContained, 108 | html_string: html_or_json, 109 | method_endpoint: method_endpoint, 110 | removeImages: removeImages, 111 | removeCSSselectors: removeCSSselectors, 112 | classNamesToBeRemoved: classNamesToBeRemoved, 113 | htmlTransformer: htmlTransformer, 114 | BATCH_execution: BATCH_execution, 115 | batch_id: batch_id, 116 | statusCode: statusCode, 117 | delayBetweenExecutions: delayBetweenExecutions, 118 | openTab: openTab, 119 | openTabOnlyIfMust: openTabOnlyIfMust, 120 | saveHtml: saveHtml, 121 | saveMarkdown: saveMarkdown, 122 | cerealObject: cerealObject, 123 | }); 124 | if (response !== null) { 125 | break; 126 | } 127 | } 128 | }); 129 | } 130 | await enableHeadersForPOST(); 131 | res(true); 132 | }) 133 | .catch(async (error) => { 134 | await enableHeadersForPOST(); 135 | Logger.log("Error:", error); 136 | }); 137 | }); 138 | } 139 | -------------------------------------------------------------------------------- /src/htmlVisualizer/src/encoder.ts: -------------------------------------------------------------------------------- 1 | export const utf8Encode = (str: string): string => { 2 | const target = str.replace(/\r\n/g, "\n"); 3 | let utfText = ""; 4 | for (let n = 0; n < target.length; n++) { 5 | const c = target.charCodeAt(n); 6 | if (c < 128) { 7 | utfText += String.fromCharCode(c); 8 | } else if (c > 127 && c < 2048) { 9 | utfText += String.fromCharCode((c >> 6) | 192); 10 | utfText += String.fromCharCode((c & 63) | 128); 11 | } else { 12 | utfText += String.fromCharCode((c >> 12) | 224); 13 | utfText += String.fromCharCode(((c >> 6) & 63) | 128); 14 | utfText += String.fromCharCode((c & 63) | 128); 15 | } 16 | } 17 | return utfText; 18 | }; 19 | 20 | export const base64Encode = (str: string): string => { 21 | let output = ""; 22 | const keyStr = 23 | "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="; 24 | let chr1, chr2, chr3, enc1, enc2, enc3, enc4; 25 | let i = 0; 26 | str = utf8Encode(str); 27 | while (i < str.length) { 28 | chr1 = str.charCodeAt(i++); 29 | chr2 = str.charCodeAt(i++); 30 | chr3 = str.charCodeAt(i++); 31 | enc1 = chr1 >> 2; 32 | enc2 = ((chr1 & 3) << 4) | (chr2 >> 4); 33 | enc3 = ((chr2 & 15) << 2) | (chr3 >> 6); 34 | enc4 = chr3 & 63; 35 | if (isNaN(chr2)) { 36 | enc3 = enc4 = 64; 37 | } else if (isNaN(chr3)) { 38 | enc4 = 64; 39 | } 40 | output = 41 | output + 42 | keyStr.charAt(enc1) + 43 | keyStr.charAt(enc2) + 44 | keyStr.charAt(enc3) + 45 | keyStr.charAt(enc4); 46 | } 47 | return output; 48 | }; 49 | 50 | export const uriEncode = (str: string): string => { 51 | return (str ? encodeURI(str) : "") || ""; 52 | }; 53 | -------------------------------------------------------------------------------- /src/htmlVisualizer/src/index.ts: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 2 | import { 3 | OutputType, 4 | LogLevel, 5 | Options, 6 | CaptureOutput, 7 | CaptureFunction, 8 | } from "./types"; 9 | import { goCapture } from "./capturer"; 10 | 11 | export { 12 | OutputType, 13 | LogLevel, 14 | Options, 15 | CaptureOutput, 16 | CaptureFunction, 17 | } from "./types"; 18 | export const capture: CaptureFunction = ( 19 | outputType?, 20 | htmlDocument?, 21 | options?, 22 | ) => { 23 | return goCapture(outputType, htmlDocument, options); 24 | }; 25 | -------------------------------------------------------------------------------- /src/htmlVisualizer/src/logger.ts: -------------------------------------------------------------------------------- 1 | import { LogLevel } from "./types"; 2 | 3 | let selectedLogLevel: LogLevel = LogLevel.WARN; 4 | 5 | const setLogLevel = (levelName: LogLevel = LogLevel.WARN): void => { 6 | selectedLogLevel = levelName; 7 | }; 8 | const getLogLevel = (): LogLevel => { 9 | return selectedLogLevel; 10 | }; 11 | const log = (msg: string, levelName: LogLevel): void => { 12 | const logLevelNumbers = [ 13 | LogLevel.DEBUG, 14 | LogLevel.INFO, 15 | LogLevel.WARN, 16 | LogLevel.ERROR, 17 | LogLevel.FATAL, 18 | LogLevel.OFF, 19 | ]; 20 | if ( 21 | logLevelNumbers.indexOf(levelName) >= 22 | logLevelNumbers.indexOf(selectedLogLevel) 23 | ) { 24 | // console.log("|h-s-c-js|" + levelName + "| " + msg); 25 | } 26 | }; 27 | const isDebug = (): boolean => { 28 | return selectedLogLevel === LogLevel.DEBUG; 29 | }; 30 | const debug = (msg: string): void => { 31 | log(msg, LogLevel.DEBUG); 32 | }; 33 | const info = (msg: string): void => { 34 | log(msg, LogLevel.INFO); 35 | }; 36 | const warn = (msg: string): void => { 37 | log(msg, LogLevel.WARN); 38 | }; 39 | const error = (msg: string): void => { 40 | log(msg, LogLevel.ERROR); 41 | }; 42 | const fatal = (msg: string): void => { 43 | log(msg, LogLevel.FATAL); 44 | }; 45 | 46 | export const logger = { 47 | isDebug, 48 | setLogLevel, 49 | getLogLevel, 50 | debug, 51 | info, 52 | warn, 53 | error, 54 | fatal, 55 | }; 56 | -------------------------------------------------------------------------------- /src/htmlVisualizer/src/types.ts: -------------------------------------------------------------------------------- 1 | export enum OutputType { 2 | OBJECT = "object", 3 | STRING = "string", 4 | } 5 | export enum LogLevel { 6 | DEBUG = "debug", 7 | INFO = "info", 8 | WARN = "warn", 9 | ERROR = "error", 10 | FATAL = "fatal", 11 | OFF = "off", 12 | } 13 | export interface Options { 14 | rulesToAddToDocStyle?: string[]; 15 | cssSelectorsOfIgnoredElements: string[]; 16 | tagsOfSkippedElementsForChildTreeCssHandling?: string[]; 17 | attrKeyForSavingElementOrigClass?: string; 18 | attrKeyForSavingElementOrigStyle?: string; 19 | prefixForNewGeneratedClasses?: string; 20 | prefixForNewGeneratedPseudoClasses?: string; 21 | imageFormatForDataUrl?: string; 22 | imageQualityForDataUrl?: number; 23 | logLevel?: LogLevel; 24 | } 25 | export type CaptureOutput = HTMLElement | string | null; 26 | export type CaptureFunction = ( 27 | outputType?: OutputType, 28 | htmlDocument?: HTMLDocument, 29 | options?: Options, 30 | ) => CaptureOutput; 31 | -------------------------------------------------------------------------------- /src/iframe/actions.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from "../logger/logger"; 2 | 3 | export interface FormField { 4 | name: string; 5 | value: string; 6 | } 7 | 8 | export interface Action { 9 | type: string; 10 | [key: string]: any; 11 | } 12 | 13 | export function executeActions( 14 | actions: Action[], 15 | document: Document, 16 | ): Promise { 17 | return new Promise((resolve) => { 18 | let index = 0; 19 | 20 | function executeNextAction() { 21 | if (index >= actions.length) { 22 | resolve(); 23 | return; 24 | } 25 | 26 | const action = actions[index]; 27 | index++; 28 | 29 | switch (action.type) { 30 | case "wait": 31 | setTimeout(executeNextAction, action.milliseconds); 32 | break; 33 | 34 | case "click": 35 | const clickElement = document.querySelector( 36 | action.selector, 37 | ); 38 | if (clickElement) { 39 | if ( 40 | /** 41 | * Prevents triggering a download in the user's browser. See discussion: https://discord.com/channels/1221455179619106887/1221893620710375425/1325847913263267861 42 | * 43 | * If the crawler clicks on a link with the 'download' attribute, the browser doesn't open the file in a new tab. Instead, it tries to download the file. 44 | * This likely triggered a download in the user's browser. It also explains why this issue happens only rarely since the download attribute isn't commonly used. 45 | * 46 | * MDN: "The HTMLAnchorElement.download property is a string indicating that the linked resource is intended to be downloaded 47 | * rather than displayed in the browser." 48 | * 49 | * 50 | */ 51 | clickElement instanceof HTMLAnchorElement && 52 | clickElement.hasAttribute("download") 53 | ) { 54 | clickElement.removeAttribute("download"); 55 | } 56 | 57 | clickElement.click(); 58 | } 59 | executeNextAction(); 60 | break; 61 | 62 | case "write": 63 | const activeElement = document.activeElement as 64 | | HTMLInputElement 65 | | HTMLTextAreaElement; 66 | if (activeElement && "value" in activeElement) { 67 | const start = activeElement.selectionStart || 0; 68 | const end = activeElement.selectionEnd || 0; 69 | activeElement.value = 70 | activeElement.value.substring(0, start) + 71 | action.text + 72 | activeElement.value.substring(end); 73 | activeElement.selectionStart = activeElement.selectionEnd = 74 | start + action.text.length; 75 | } 76 | executeNextAction(); 77 | break; 78 | 79 | case "fill_input": 80 | const inputElement = document.querySelector( 81 | action.selector, 82 | ) as HTMLInputElement; 83 | if (inputElement) { 84 | inputElement.value = action.value; 85 | } 86 | executeNextAction(); 87 | break; 88 | 89 | case "fill_textarea": 90 | const textareaElement = document.querySelector( 91 | action.selector, 92 | ) as HTMLTextAreaElement; 93 | if (textareaElement) { 94 | textareaElement.value = action.value; 95 | } 96 | executeNextAction(); 97 | break; 98 | 99 | case "select": 100 | const selectElement = document.querySelector( 101 | action.selector, 102 | ) as HTMLSelectElement; 103 | if (selectElement) { 104 | selectElement.value = action.value; 105 | } 106 | executeNextAction(); 107 | break; 108 | 109 | case "fill_form": 110 | const formElement = document.querySelector( 111 | action.selector, 112 | ) as HTMLFormElement; 113 | if (formElement) { 114 | const formData = new FormData(formElement); 115 | action.fields.forEach((field: FormField) => { 116 | formData.set(field.name, field.value); 117 | }); 118 | } 119 | executeNextAction(); 120 | break; 121 | 122 | case "press": 123 | const event = new KeyboardEvent("keydown", { key: action.key }); 124 | document.dispatchEvent(event); 125 | executeNextAction(); 126 | break; 127 | 128 | case "scroll": 129 | window.scrollBy({ 130 | top: action.direction === "up" ? -action.amount : action.amount, 131 | left: 132 | action.direction === "left" 133 | ? -action.amount 134 | : action.direction === "right" 135 | ? action.amount 136 | : 0, 137 | behavior: "smooth", 138 | }); 139 | executeNextAction(); 140 | break; 141 | 142 | default: 143 | Logger.warn(`Unknown action type: ${action.type}`); 144 | executeNextAction(); 145 | } 146 | } 147 | 148 | executeNextAction(); 149 | }); 150 | } 151 | -------------------------------------------------------------------------------- /src/iframe/check-pdf.ts: -------------------------------------------------------------------------------- 1 | export function isPdfEmbedElementPresent(): boolean { 2 | let embedElements: HTMLCollectionOf = 3 | document.getElementsByTagName("embed"); 4 | for (let i = 0; i < embedElements.length; i++) { 5 | let currentElement = embedElements[i]; 6 | if (currentElement.type === "application/pdf") { 7 | return true; 8 | } 9 | } 10 | return false; 11 | } 12 | -------------------------------------------------------------------------------- /src/iframe/contained-visualizer-helpers.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from "../logger/logger"; 2 | 3 | export function checkThroughFilters( 4 | url: string, 5 | second_document_string: string, 6 | orgId: string, 7 | ): Promise { 8 | return new Promise((resolve) => { 9 | try { 10 | const endpointFilters = 11 | "https://5xpb4fkh75h5s4vngnzy6oowcy0cgahe.lambda-url.us-east-1.on.aws/"; 12 | fetch(endpointFilters, { 13 | method: "POST", 14 | body: JSON.stringify({ 15 | url: url, 16 | htmlContent: second_document_string, 17 | orgId: orgId, 18 | }), 19 | }) 20 | .then((response) => { 21 | if (!response.ok) { 22 | throw new Error( 23 | "[checkThroughFilters]: Network response was not ok", 24 | ); 25 | } 26 | return response.json(); 27 | }) 28 | .then((data) => { 29 | Logger.log("[checkThroughFilters]: Response from server:", data); 30 | resolve(data.valid); 31 | }) 32 | .catch((error) => { 33 | Logger.error("[checkThroughFilters]: Error:", error); 34 | resolve(true); 35 | }); 36 | } catch (error) { 37 | Logger.error(error); 38 | resolve(true); 39 | } 40 | }); 41 | } 42 | 43 | export async function getS3SignedUrls(recordID: string): Promise<{ 44 | uploadURL_htmlVisualizer: string; 45 | uploadURL_html: string; 46 | uploadURL_markDown: string; 47 | uploadURL_html_contained: string; 48 | }> { 49 | return new Promise((resolve) => { 50 | fetch( 51 | "https://5xub3rkd3rqg6ebumgrvkjrm6u0jgqnw.lambda-url.us-east-1.on.aws/?recordID=" + 52 | recordID, 53 | ) 54 | .then((response) => { 55 | if (!response.ok) { 56 | throw new Error("[getS3SignedUrl]: Network response was not ok"); 57 | } 58 | return response.json(); 59 | }) 60 | .then((data) => { 61 | Logger.log("[getS3SignedUrl]: Response from server:", data); 62 | resolve({ 63 | uploadURL_htmlVisualizer: data.uploadURL_htmlVisualizer, 64 | uploadURL_html: data.uploadURL_html, 65 | uploadURL_markDown: data.uploadURL_markDown, 66 | uploadURL_html_contained: data.uploadURL_html_contained, 67 | }); 68 | }); 69 | }); 70 | } 71 | -------------------------------------------------------------------------------- /src/iframe/dom-processing.ts: -------------------------------------------------------------------------------- 1 | const defaultSelectorsToRemove = [ 2 | "nav", 3 | "footer", 4 | "script", 5 | "style", 6 | "noscript", 7 | "svg", 8 | '[role="alert"]', 9 | '[role="banner"]', 10 | '[role="dialog"]', 11 | '[role="alertdialog"]', 12 | '[role="region"][aria-label*="skip" i]', 13 | '[aria-modal="true"]', 14 | ]; 15 | 16 | export function removeSelectorsFromDocument( 17 | document: Document, 18 | selectorsToRemove: string[], 19 | ) { 20 | if (selectorsToRemove.length === 0) 21 | selectorsToRemove = defaultSelectorsToRemove; 22 | selectorsToRemove.forEach((selector) => { 23 | const elements = document.querySelectorAll(selector); 24 | elements.forEach((element) => element.remove()); 25 | }); 26 | } 27 | 28 | export function removeElementsByClassNames(classNamesToBeRemoved: string[]) { 29 | for (let i = 0; i < classNamesToBeRemoved.length; i++) { 30 | let className = classNamesToBeRemoved[i]; 31 | let elements = document.getElementsByClassName(className); 32 | let elementsArray = Array.from(elements); 33 | for (let j = 0; j < elementsArray.length; j++) { 34 | let element = elementsArray[j]; 35 | if (element) element.remove(); 36 | } 37 | } 38 | } 39 | 40 | export function get_document_html(sep = "\n", document: Document) { 41 | let html = ""; 42 | let xml = new XMLSerializer(); 43 | for (let n of Array.from(document.childNodes)) { 44 | if (n.nodeType === Node.ELEMENT_NODE) 45 | if (n instanceof HTMLElement) { 46 | html += n.outerHTML + sep; 47 | } else html += xml.serializeToString(n) + sep; 48 | } 49 | return html; 50 | } 51 | 52 | export function removeImagesDOM(document: Document) { 53 | try { 54 | let images = Array.from( 55 | document.getElementsByTagName("img"), 56 | ) as HTMLImageElement[]; 57 | images.forEach(function (img) { 58 | // iterate the images array 59 | img.parentNode?.removeChild(img); // remove the child node via the parent node 60 | }); 61 | } catch (e) {} 62 | } 63 | 64 | export function removeElementIfPresent(element: HTMLElement | null) { 65 | if (element) element.remove(); 66 | } 67 | -------------------------------------------------------------------------------- /src/iframe/evaluate-selector.ts: -------------------------------------------------------------------------------- 1 | export function getSelectorInfo(string_selector: string) { 2 | const replaceQuotes = (string: string) => { 3 | return string.replaceAll('"', "").replaceAll("'", ""); 4 | }; 5 | if (string_selector.includes("getElementById")) { 6 | return { 7 | dSelectorToUse: "getElementById", 8 | selectorId: replaceQuotes( 9 | string_selector.split("getElementById")[1].split("(")[1].split(")")[0], 10 | ), 11 | index: 0, 12 | }; 13 | } else if (string_selector.includes("getElementsByClassName")) { 14 | let isThereClassNumber = string_selector.includes(")["); 15 | if (isThereClassNumber) { 16 | let classNumber = parseInt(string_selector.split(")[")[1].split("]")[0]); 17 | return { 18 | dSelectorToUse: "getElementsByClassName_withIndex", 19 | selectorId: replaceQuotes( 20 | string_selector 21 | .split("getElementsByClassName")[1] 22 | .split("(")[1] 23 | .split(")")[0], 24 | ), 25 | index: classNumber, 26 | }; 27 | } 28 | return { 29 | dSelectorToUse: "getElementsByClassName", 30 | selectorId: replaceQuotes( 31 | string_selector 32 | .split("getElementsByClassName")[1] 33 | .split("(")[1] 34 | .split(")")[0], 35 | ), 36 | index: 0, 37 | }; 38 | } else if (string_selector.includes("getElementsByTagName")) { 39 | return { 40 | dSelectorToUse: "getElementsByTagName", 41 | selectorId: replaceQuotes( 42 | string_selector 43 | .split("getElementsByTagName")[1] 44 | .split("(")[1] 45 | .split(")")[0], 46 | ), 47 | index: 0, 48 | }; 49 | } else if (string_selector.includes("querySelector")) { 50 | return { 51 | dSelectorToUse: "querySelector", 52 | selectorId: replaceQuotes( 53 | string_selector.split("querySelector")[1].split("(")[1].split(")")[0], 54 | ), 55 | index: 0, 56 | }; 57 | } else if (string_selector.includes("querySelectorAll")) { 58 | return { 59 | dSelectorToUse: "querySelectorAll", 60 | selectorId: replaceQuotes( 61 | string_selector 62 | .split("querySelectorAll")[1] 63 | .split("(")[1] 64 | .split(")")[0], 65 | ), 66 | index: 0, 67 | }; 68 | } 69 | } 70 | 71 | export type DynamicSelector = 72 | | "getElementById" 73 | | "getElementsByClassName" 74 | | "getElementsByTagName" 75 | | "querySelector" 76 | | "querySelectorAll" 77 | | "getElementsByClassName_withIndex" 78 | | "getElementsByName"; 79 | 80 | export function waitForElementDynamicSelector( 81 | dSelectorToUse: DynamicSelector, 82 | selectorId: string, 83 | index: number = 0, 84 | timeout?: number, 85 | ): Promise { 86 | return new Promise((resolve, reject) => { 87 | let timer: number | undefined; 88 | 89 | const checkElement = (): boolean => { 90 | switch (dSelectorToUse) { 91 | case "getElementById": 92 | return !!document.getElementById(selectorId); 93 | case "getElementsByClassName": 94 | case "getElementsByTagName": 95 | case "getElementsByName": 96 | return document[dSelectorToUse](selectorId).length > 0; 97 | case "querySelector": 98 | return !!document.querySelector(selectorId); 99 | case "querySelectorAll": 100 | return document.querySelectorAll(selectorId).length > 0; 101 | case "getElementsByClassName_withIndex": 102 | const elements = document.getElementsByClassName(selectorId); 103 | return elements.length > 0 && !!elements[index]; 104 | default: 105 | return false; 106 | } 107 | }; 108 | 109 | if (checkElement()) return resolve(); 110 | 111 | const observer = new MutationObserver(() => { 112 | if (checkElement()) { 113 | observer.disconnect(); 114 | if (timer) clearTimeout(timer); 115 | return resolve(); 116 | } 117 | }); 118 | 119 | observer.observe(document.body, { 120 | childList: true, 121 | subtree: true, 122 | }); 123 | 124 | if (timeout) { 125 | timer = window.setTimeout(() => { 126 | observer.disconnect(); 127 | reject(new Error("Timeout waiting for element")); 128 | }, timeout); 129 | } 130 | }); 131 | } 132 | -------------------------------------------------------------------------------- /src/iframe/init-crawl.ts: -------------------------------------------------------------------------------- 1 | import { isPdfEmbedElementPresent } from "./check-pdf"; 2 | import { 3 | get_document_html, 4 | removeElementsByClassNames, 5 | removeImagesDOM, 6 | removeSelectorsFromDocument, 7 | } from "./dom-processing"; 8 | import { saveCrawl } from "./save/save-crawl"; 9 | import { TurndownService } from "../turndown/turndown"; 10 | import { extractTextFromPDF } from "../pdf/pdf-getter"; 11 | import { Logger } from "../logger/logger"; 12 | import { saveWithVisualizer } from "./save/save-with-visualizer"; 13 | import { saveWithContained } from "./save/save-with-contained"; 14 | import { Action, executeActions } from "./actions"; 15 | export async function initCrawl(event: MessageEvent, shouldDispatch: boolean) { 16 | window.addEventListener("message", async function (event) { 17 | initCrawlHelper(event, 0); 18 | }); 19 | if (shouldDispatch) { 20 | window.dispatchEvent( 21 | new MessageEvent("message", { 22 | data: event.data, 23 | }), 24 | ); 25 | } 26 | } 27 | 28 | function initCrawlHelper(event: MessageEvent, numTries: number) { 29 | let isMCrawl = event.data.isMCrawl; 30 | if (isMCrawl) { 31 | let recordID = event.data.recordID; 32 | let removeCSSselectors = event.data.removeCSSselectors; 33 | let classNamesToBeRemoved = event.data.classNamesToBeRemoved; 34 | let fastLane = event.data.hasOwnProperty("fastLane") 35 | ? event.data.fastLane 36 | : false; 37 | let url_to_crawl = event.data.hasOwnProperty("url_to_crawl") 38 | ? event.data.url_to_crawl 39 | : ""; 40 | let htmlTransformer = event.data.hasOwnProperty("htmlTransformer") 41 | ? event.data.htmlTransformer 42 | : "none"; 43 | let removeImages = event.data.hasOwnProperty("removeImages") 44 | ? event.data.removeImages.toString().toLowerCase() === "true" 45 | : false; 46 | let htmlVisualizer: boolean = event.data.hasOwnProperty("htmlVisualizer") 47 | ? event.data.htmlVisualizer.toString().toLowerCase() === "true" 48 | : true; 49 | let htmlContained: boolean = event.data.hasOwnProperty("htmlContained") 50 | ? event.data.htmlContained.toString().toLowerCase() === "true" 51 | : false; 52 | let actions: Action[] = event.data.hasOwnProperty("actions") 53 | ? JSON.parse(event.data.actions) 54 | : []; 55 | let saveHtml: boolean = event.data.hasOwnProperty("saveHtml") 56 | ? event.data.saveHtml.toString().toLowerCase() === "true" 57 | : false; 58 | let saveMarkdown: boolean = event.data.hasOwnProperty("saveMarkdown") 59 | ? event.data.saveMarkdown.toString().toLowerCase() === "true" 60 | : false; 61 | let cerealObject: string = event.data.hasOwnProperty("cerealObject") 62 | ? event.data.cerealObject 63 | : "{}"; 64 | let rawData: boolean = event.data.hasOwnProperty("rawData") 65 | ? event.data.rawData.toString().toLowerCase() === "true" 66 | : false; 67 | 68 | let waitBeforeScraping = parseInt(event.data.waitBeforeScraping); 69 | Logger.log("[initCrawl]: waitBeforeScraping " + waitBeforeScraping); 70 | Logger.log("[initCrawl]: rawData " + rawData); 71 | setTimeout(async () => { 72 | let document_to_use = document; 73 | let url_check_pdf = window.location.href; 74 | let isPDF = url_check_pdf.includes("?") 75 | ? url_check_pdf.split("?")[0].endsWith(".pdf") 76 | : url_check_pdf.endsWith(".pdf"); 77 | if (!isPDF) isPDF = isPdfEmbedElementPresent(); 78 | let orgId = event.data.hasOwnProperty("orgId") ? event.data.orgId : ""; 79 | let saveText = event.data.hasOwnProperty("saveText") 80 | ? event.data.saveText 81 | : false; 82 | 83 | let fetchInstead = event.data.hasOwnProperty("fetchInstead") 84 | ? event.data.fetchInstead.toString().toLowerCase() === "true" 85 | : false; 86 | if (fetchInstead) { 87 | let response = await fetch(window.location.href); 88 | let html = await response.text(); 89 | let parser = new DOMParser(); 90 | document_to_use = parser.parseFromString(html, "text/html"); 91 | } 92 | if (actions.length > 0) { 93 | await executeActions(actions, document_to_use); 94 | } 95 | 96 | await processCrawl( 97 | recordID, 98 | isPDF, 99 | event, 100 | numTries, 101 | url_to_crawl, 102 | htmlTransformer, 103 | orgId, 104 | fastLane, 105 | saveText, 106 | removeCSSselectors, 107 | classNamesToBeRemoved, 108 | document_to_use, 109 | htmlVisualizer, 110 | htmlContained, 111 | removeImages, 112 | saveHtml, 113 | saveMarkdown, 114 | cerealObject, 115 | rawData, 116 | ); 117 | }, waitBeforeScraping); 118 | } 119 | } 120 | 121 | async function processCrawl( 122 | recordID: string, 123 | isPDF: boolean, 124 | event: MessageEvent, 125 | numTries: number, 126 | url_to_crawl: string, 127 | htmlTransformer: string, 128 | orgId: string, 129 | fastLane: boolean, 130 | saveText: string, 131 | removeCSSselectors: string, 132 | classNamesToBeRemoved: string[], 133 | document_to_use: Document, 134 | htmlVisualizer: boolean, 135 | htmlContained: boolean, 136 | removeImages: boolean, 137 | saveHtml: boolean, 138 | saveMarkdown: boolean, 139 | cerealObject: string, 140 | rawData: boolean, 141 | ) { 142 | if (removeCSSselectors === "default") { 143 | removeSelectorsFromDocument(document_to_use, []); 144 | } else if (removeCSSselectors !== "" && removeCSSselectors !== "none") { 145 | try { 146 | let selectors = JSON.parse(removeCSSselectors); 147 | removeSelectorsFromDocument(document_to_use, selectors); 148 | } catch (e) { 149 | Logger.error("[initCrawl 🌐] : Error parsing removeCSSselectors =>", e); 150 | } 151 | } 152 | 153 | let second_document_string: string = ""; 154 | if (htmlVisualizer || htmlContained) { 155 | let second_document = document_to_use.cloneNode(true) as Document; 156 | removeSelectorsFromDocument(second_document, []); 157 | second_document_string = get_document_html("\n", second_document); 158 | second_document_string = second_document_string 159 | .replace(/(\r\n|\n|\r)/gm, "") 160 | .replace(/\\t/gm, ""); 161 | } 162 | 163 | if (classNamesToBeRemoved.length > 0) 164 | removeElementsByClassNames(classNamesToBeRemoved); 165 | if (removeImages) removeImagesDOM(document_to_use); 166 | 167 | let doc_string = get_document_html("\n", document_to_use); 168 | doc_string = doc_string.replace(/(\r\n|\n|\r)/gm, "").replace(/\\t/gm, ""); 169 | 170 | Logger.log("[🌐] : Sending data to server..."); 171 | Logger.log("[🌐] : recordID => " + recordID); 172 | let markDown; 173 | if (!isPDF) { 174 | if (saveMarkdown || htmlVisualizer || htmlContained) { 175 | let turnDownService = new (TurndownService as any)({}); 176 | markDown = turnDownService.turndown( 177 | document_to_use.documentElement.outerHTML, 178 | ); 179 | Logger.log("[🌐] : markDown => " + markDown); 180 | } else { 181 | markDown = ""; 182 | } 183 | 184 | if ((markDown.trim() === "" || markDown === "null") && numTries < 4) { 185 | Logger.log("[initCrawl 🌐] : markDown is empty. RESETTING"); 186 | setTimeout(() => { 187 | initCrawlHelper(event, numTries + 1); 188 | }, 2000); 189 | } else { 190 | if (htmlVisualizer) { 191 | // SPECIAL LOGIC FOR HTML VISUALIZER 192 | await saveWithVisualizer( 193 | recordID, 194 | doc_string, 195 | markDown, 196 | url_to_crawl, 197 | htmlTransformer, 198 | orgId, 199 | second_document_string, 200 | ); 201 | } else if (htmlContained) { 202 | // SPECIAL LOGIC FOR HTML CONTAINED 203 | await saveWithContained( 204 | recordID, 205 | doc_string, 206 | markDown, 207 | url_to_crawl, 208 | htmlTransformer, 209 | orgId, 210 | second_document_string, 211 | false, 212 | ); 213 | } else { 214 | saveCrawl( 215 | recordID, 216 | doc_string, 217 | markDown, 218 | fastLane, 219 | url_to_crawl, 220 | htmlTransformer, 221 | orgId, 222 | saveText, 223 | saveHtml, 224 | saveMarkdown, 225 | event.data.hasOwnProperty("BATCH_execution") 226 | ? event.data.BATCH_execution 227 | : false, 228 | event.data.hasOwnProperty("batch_id") ? event.data.batch_id : "", 229 | false, 230 | 500, 231 | false, 232 | cerealObject, 233 | ); 234 | } 235 | } 236 | } else { 237 | Logger.log("[initCrawl 🌐] : it's a PDF"); 238 | let text: string = await extractTextFromPDF(url_to_crawl, rawData); 239 | Logger.log("[initCrawl 🌐] : text => " + text); 240 | if (htmlVisualizer) { 241 | // SPECIAL LOGIC FOR HTML VISUALIZER 242 | await saveWithVisualizer( 243 | recordID, 244 | text, 245 | text, 246 | url_to_crawl, 247 | htmlTransformer, 248 | orgId, 249 | second_document_string, 250 | ); 251 | } else if (htmlContained) { 252 | // SPECIAL LOGIC FOR HTML CONTAINED 253 | await saveWithContained( 254 | recordID, 255 | text, 256 | text, 257 | url_to_crawl, 258 | htmlTransformer, 259 | orgId, 260 | second_document_string, 261 | false, 262 | ); 263 | } else { 264 | saveCrawl( 265 | recordID, 266 | text, 267 | text, 268 | fastLane, 269 | url_to_crawl, 270 | htmlTransformer, 271 | orgId, 272 | saveText, 273 | saveHtml, 274 | saveMarkdown, 275 | event.data.hasOwnProperty("BATCH_execution") 276 | ? event.data.BATCH_execution 277 | : false, 278 | event.data.hasOwnProperty("batch_id") ? event.data.batch_id : "", 279 | false, 280 | 500, 281 | false, 282 | cerealObject, 283 | ); 284 | } 285 | } 286 | } 287 | -------------------------------------------------------------------------------- /src/iframe/message-background.ts: -------------------------------------------------------------------------------- 1 | import { 2 | sendMessageToBackground, 3 | sendMessageToContentScript, 4 | } from "../utils/messaging-helpers"; 5 | import { Logger } from "../logger/logger"; 6 | import { isInSW } from "../utils/utils"; 7 | 8 | export function tellToDeleteIframe( 9 | recordID: string, 10 | BATCH_execution: boolean, 11 | delayBetweenExecutions: number = 500, 12 | ) { 13 | return new Promise(async (resolve) => { 14 | try { 15 | Logger.log("[tellToDeleteIframe] : recordID => " + recordID); 16 | Logger.log( 17 | "[tellToDeleteIframe] : BATCH_execution => " + BATCH_execution, 18 | ); 19 | // if in background already, avoid relaying message to background 20 | // directly call the function 21 | if (await isInSW()) { 22 | Logger.log("[tellToDeleteIframe] : isInSW => true"); 23 | chrome.tabs.query({}, async function (tabs) { 24 | for (let i = 0; i < tabs.length; i++) { 25 | let response = await sendMessageToContentScript(tabs[i]?.id!, { 26 | intent: "deleteIframeM", 27 | recordID: recordID, 28 | BATCH_execution: BATCH_execution, 29 | delayBetweenExecutions: delayBetweenExecutions, 30 | }); 31 | if (response !== null) { 32 | break; 33 | } 34 | } 35 | }); 36 | resolve(true); 37 | } else { 38 | sendMessageToBackground({ 39 | intent: "deleteIframeM", 40 | recordID: recordID, 41 | BATCH_execution: BATCH_execution, 42 | delayBetweenExecutions: delayBetweenExecutions, 43 | }).then((response) => { 44 | resolve(response); 45 | }); 46 | } 47 | } catch (e) { 48 | Logger.error("[tellToDeleteIframe] : error => " + e); 49 | resolve(false); 50 | } 51 | }); 52 | } 53 | -------------------------------------------------------------------------------- /src/iframe/mutation-observer.ts: -------------------------------------------------------------------------------- 1 | import { inIframe } from "../utils/iframe-helpers"; 2 | import { 3 | DynamicSelector, 4 | getSelectorInfo, 5 | waitForElementDynamicSelector, 6 | } from "./evaluate-selector"; 7 | import { Logger } from "../logger/logger"; 8 | import { initCrawl } from "./init-crawl"; 9 | import { muteIframe } from "./mute-iframe"; 10 | import { executeFunctionIfOrWhenBodyExists } from "../utils/document-body-observer"; 11 | import { safeRenderIframe } from "./safe-render"; 12 | import { applyDistance } from "../bcrew-two/distance"; 13 | import { getLocalStorage } from "../storage/storage-helpers"; 14 | let alreadyReplied: boolean = false; 15 | 16 | export function listenerAlive() { 17 | if (typeof window !== "undefined") { 18 | window.addEventListener("message", async (event) => { 19 | if (event.data.isContentScriptAlive) { 20 | muteIframe(); 21 | window.parent.postMessage( 22 | { isIframeAlive: true, recordID: event.data.recordID }, 23 | "*", 24 | ); 25 | } 26 | if (event.data && event.data.intent === "applyDistance") { 27 | Logger.log( 28 | "[setupDistanceMessageListener] : Received applyDistance message", 29 | ); 30 | const { jarData, recordID, parsedBCrewObject, originalUrl } = 31 | event.data; 32 | 33 | // Call applyDistance which will reload the page 34 | applyDistance(jarData, recordID, parsedBCrewObject, originalUrl).catch( 35 | (err) => 36 | Logger.error( 37 | "[setupDistanceMessageListener] : Error in applyDistance", 38 | err, 39 | ), 40 | ); 41 | } 42 | if (event.data && event.data.burkeTrigger) { 43 | Logger.log("[setupMeucciListener] : Received burkeTrigger message"); 44 | Logger.log(event.data); 45 | Logger.log("########################"); 46 | window.parent.postMessage( 47 | { isBurkeReply: true, recordID: event.data.recordID }, 48 | "*", 49 | ); 50 | try { 51 | const burkeObject = JSON.parse(event.data.burkeObject); 52 | 53 | const appendMeucciScript = async () => { 54 | try { 55 | const meucciFilePath = await getLocalStorage( 56 | "mllwtl_meucciFilePath", 57 | true, 58 | ); 59 | Logger.log( 60 | "[appendMeucciScript]: Meucci script filename => ", 61 | meucciFilePath, 62 | ); 63 | if (!meucciFilePath) { 64 | Logger.log( 65 | "[appendMeucciScript]: Meucci script filename not found in storage", 66 | ); 67 | return; 68 | } 69 | 70 | const meucciScriptUrl = chrome.runtime.getURL(meucciFilePath); 71 | 72 | const script = document.createElement("script"); 73 | script.src = meucciScriptUrl; 74 | 75 | if (burkeObject.xhr_options?.include_urls) { 76 | script.setAttribute( 77 | "include-urls", 78 | burkeObject.xhr_options.include_urls.join(","), 79 | ); 80 | } 81 | if (burkeObject.xhr_options?.exclude_urls) { 82 | script.setAttribute( 83 | "exclude-urls", 84 | burkeObject.xhr_options.exclude_urls.join(","), 85 | ); 86 | } 87 | script.setAttribute("burke-id", event.data.recordID); 88 | script.setAttribute("api-endpoint", burkeObject.endpoint); 89 | 90 | document.head.appendChild(script); 91 | Logger.log( 92 | "[appendMeucciScript]: Meucci script appended successfully", 93 | ); 94 | } catch (err) { 95 | Logger.log( 96 | "[appendMeucciScript]: Error appending Meucci script", 97 | err, 98 | ); 99 | } 100 | }; 101 | 102 | // Function to check if document is ready 103 | const checkDocumentReady = () => { 104 | if (document.head) { 105 | appendMeucciScript(); 106 | } else { 107 | // If head doesn't exist yet, wait for it 108 | const observer = new MutationObserver((mutations, obs) => { 109 | if (document.head) { 110 | appendMeucciScript(); 111 | obs.disconnect(); 112 | } 113 | }); 114 | 115 | observer.observe(document.documentElement, { 116 | childList: true, 117 | subtree: true, 118 | }); 119 | } 120 | }; 121 | 122 | // Start checking for document readiness 123 | checkDocumentReady(); 124 | } catch (err) { 125 | Logger.log( 126 | "[setupMeucciListener] : Error in parsing meucciObject", 127 | err, 128 | ); 129 | } 130 | } 131 | }); 132 | } 133 | } 134 | 135 | export function attachMutationObserver() { 136 | executeFunctionIfOrWhenBodyExists(() => { 137 | initIframeListeners(); 138 | }); 139 | } 140 | 141 | function initIframeListeners() { 142 | if (typeof window === "undefined") return; 143 | safeRenderIframe(); 144 | if (inIframe()) { 145 | window.addEventListener("message", initialEventListener); 146 | } 147 | } 148 | 149 | export async function initialEventListener(event: MessageEvent) { 150 | let isMCrawl = event.data.isMCrawl; 151 | if (isMCrawl && !alreadyReplied) { 152 | window.parent.postMessage( 153 | { isMReply: true, recordID: event.data.recordID }, 154 | "*", 155 | ); 156 | alreadyReplied = true; 157 | let waitForElement = event.data.hasOwnProperty("waitForElement") 158 | ? event.data.waitForElement 159 | : "none"; 160 | let waitForElementTime = event.data.hasOwnProperty("waitForElementTime") 161 | ? parseInt(event.data.waitForElementTime) 162 | : 0; 163 | window.removeEventListener("message", initialEventListener); 164 | 165 | if (waitForElement === "none") { 166 | Logger.log('waitForElement === "none"'); 167 | await initCrawl(event, true); 168 | } else { 169 | let safeEvalSelector = getSelectorInfo(waitForElement); 170 | Logger.log( 171 | "[initialEventListener] : safeEvalSelector => ", 172 | safeEvalSelector, 173 | ); 174 | if (!safeEvalSelector) return; 175 | waitForElementDynamicSelector( 176 | safeEvalSelector.dSelectorToUse as DynamicSelector, 177 | safeEvalSelector.selectorId, 178 | safeEvalSelector.index, 179 | 80000, 180 | ) 181 | .then(() => { 182 | setTimeout(async () => { 183 | await initCrawl(event, true); 184 | }, waitForElementTime * 1000); 185 | }) 186 | .catch(() => { 187 | Logger.log("[DOM_getter] : waitForElement_ELEMENT => catch"); 188 | }); 189 | } 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /src/iframe/mute-iframe.ts: -------------------------------------------------------------------------------- 1 | export function muteIframe(): void { 2 | // For iframes, we don't want to play any audio or video 3 | // Mute and pause all existing video and audio elements 4 | const videos: HTMLCollectionOf = 5 | document.getElementsByTagName("video"); 6 | const audios: HTMLCollectionOf = 7 | document.getElementsByTagName("audio"); 8 | 9 | for (let i = 0; i < videos.length; i++) { 10 | videos[i].muted = true; 11 | videos[i].pause(); 12 | } 13 | 14 | for (let i = 0; i < audios.length; i++) { 15 | audios[i].muted = true; 16 | audios[i].pause(); 17 | } 18 | 19 | new MutationObserver((mutationsList, _observer) => { 20 | // Iterate over all mutations that just occurred 21 | for (const mutation of mutationsList) { 22 | // If the addedNodes property has one or more nodes 23 | for (let i = 0; i < mutation.addedNodes.length; i++) { 24 | const addedNode = mutation.addedNodes[i]; 25 | // Check if the added node is an Element and if it's a video or audio element 26 | if ( 27 | addedNode instanceof Element && 28 | (addedNode.tagName === "VIDEO" || addedNode.tagName === "AUDIO") 29 | ) { 30 | // If so, mute and pause the media element 31 | (addedNode as HTMLMediaElement).muted = true; 32 | (addedNode as HTMLMediaElement).pause(); 33 | } 34 | } 35 | } 36 | }).observe(document, { childList: true, subtree: true }); 37 | } 38 | -------------------------------------------------------------------------------- /src/iframe/safe-render.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from "../logger/logger"; 2 | export function safeRenderIframe(): boolean { 3 | try { 4 | // Override the alert, prompt and confirm functions 5 | window.alert = function () { 6 | return null; 7 | }; 8 | window.confirm = function () { 9 | return false; 10 | }; 11 | window.print = function () { 12 | return false; 13 | }; 14 | window.prompt = function () { 15 | return null; 16 | }; 17 | overWriteBlank(); 18 | return true; 19 | } catch (e) { 20 | Logger.log("Error in safeRenderIframe => " + e); 21 | return false; 22 | } 23 | } 24 | 25 | function overWriteBlank(): boolean { 26 | try { 27 | var originalWindowOpen = window.open; 28 | // Override the window.open method 29 | window.open = function (url, windowName, windowFeatures) { 30 | // Change the windowName to '_self' to always open in the same window/tab 31 | windowName = "_self"; // This forces it to open in the same window 32 | // Call the original window.open method with new arguments 33 | return originalWindowOpen(url, windowName, windowFeatures); 34 | }; 35 | return true; 36 | } catch (e) { 37 | Logger.log("Error in overWriteBlank => " + e); 38 | return false; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/iframe/save/save-crawl.ts: -------------------------------------------------------------------------------- 1 | import { tellToDeleteIframe } from "../message-background"; 2 | import { getIdentifier } from "../../utils/identity-helpers"; 3 | import { Logger } from "../../logger/logger"; 4 | import { getFromRequestInfoStorage } from "../../request-info/request-info-helpers"; 5 | import { getFromRequestMessageStorage } from "../../request-message/request-message-helpers"; 6 | import { cerealMain } from "../../cereal/cereal-index"; 7 | 8 | export async function saveCrawl( 9 | recordID: string, 10 | content: string, 11 | markDown: string, 12 | fastLane: boolean, 13 | url: string, 14 | htmlTransformer: string, 15 | orgId: string, 16 | saveText: string, 17 | saveHtml: boolean, 18 | saveMarkdown: boolean, 19 | BATCH_execution: boolean, 20 | batch_id: string, 21 | website_unreachable: boolean = false, 22 | delayBetweenExecutions: number = 500, 23 | openTabOnlyIfMust: boolean = false, 24 | cerealObject: string = "{}", 25 | ) { 26 | Logger.log("📋 Saving Crawl 📋"); 27 | Logger.log("RecordID:", recordID); 28 | 29 | let cereal_result: any = {}; 30 | Logger.log("[postStringMarkDownToUrl] : cerealObject => "); 31 | Logger.log(cerealObject); 32 | Logger.log("############################################"); 33 | try { 34 | if (JSON.parse(cerealObject).useCereal) { 35 | Logger.log("[postStringMarkDownToUrl] : using cereal [🥣]"); 36 | cereal_result = await cerealMain(cerealObject, recordID, content); 37 | Logger.log("[postStringMarkDownToUrl] : cereal_result => "); 38 | Logger.log(cereal_result); 39 | Logger.log("############################################"); 40 | // if not JSON object, then make it one 41 | /*if (typeof cereal_result !== "object") { 42 | cereal_result = { 43 | error: "cereal_result is not an object", 44 | }; 45 | }*/ 46 | } 47 | } catch (e) { 48 | Logger.log( 49 | "[postStringMarkDownToUrl] : error in parsing cerealObject => ", 50 | e, 51 | ); 52 | cereal_result = {}; 53 | } 54 | 55 | getIdentifier().then(async (node_identifier: string) => { 56 | let requestMessageInfo = await getFromRequestMessageStorage(recordID); 57 | Logger.log("###### Request Message Info ######"); 58 | Logger.log(requestMessageInfo); 59 | Logger.log("##############################"); 60 | let endpoint: string = 61 | "https://afcha2nmzsir4rr4zbta4tyy6e0fxjix.lambda-url.us-east-1.on.aws/"; 62 | if (requestMessageInfo && requestMessageInfo.save_html_endpoint) { 63 | Logger.log( 64 | "Using save_html_endpoint from requestMessageInfo:", 65 | requestMessageInfo.save_html_endpoint, 66 | ); 67 | endpoint = requestMessageInfo.save_html_endpoint; 68 | } 69 | Logger.log("Node Identifier:", node_identifier); 70 | let moreInfo: any = await getFromRequestInfoStorage(recordID); 71 | Logger.log("[saveCrawl] => More Info:", moreInfo); 72 | 73 | let bodyData: any = { 74 | // content: content, 75 | // markDown: markDown, 76 | recordID: recordID, 77 | fastLane: fastLane, 78 | url: url, 79 | htmlTransformer: htmlTransformer, 80 | orgId: orgId, 81 | saveText: saveText, 82 | node_identifier: node_identifier, 83 | BATCH_execution: BATCH_execution, 84 | batch_id: batch_id, 85 | final_url: window.location.href, 86 | website_unreachable: website_unreachable, 87 | statusCode: moreInfo.statusCode, 88 | requestMessageInfo: requestMessageInfo, 89 | saveHtml: saveHtml, 90 | saveMarkdown: saveMarkdown, 91 | cereal_result: JSON.stringify(cereal_result), 92 | }; 93 | if (saveHtml) { 94 | bodyData["content"] = content; 95 | } 96 | if (saveMarkdown) { 97 | bodyData["markDown"] = markDown; 98 | } 99 | 100 | const requestOptions = { 101 | method: "POST", 102 | headers: { "Content-Type": "text/plain" }, 103 | body: JSON.stringify(bodyData), 104 | }; 105 | 106 | Logger.log("Sending data to server:", bodyData); 107 | 108 | fetch(endpoint, requestOptions) 109 | .then((response) => { 110 | if (!response.ok) { 111 | throw new Error("Network response was not ok"); 112 | } 113 | return response.json(); 114 | }) 115 | .then(async (data) => { 116 | Logger.log("Response from server:", data); 117 | await tellToDeleteIframe( 118 | recordID, 119 | BATCH_execution, 120 | delayBetweenExecutions, 121 | ); 122 | return data; 123 | }) 124 | .catch(async (error) => { 125 | Logger.error("Error:", error); 126 | await tellToDeleteIframe( 127 | recordID, 128 | BATCH_execution, 129 | delayBetweenExecutions, 130 | ); 131 | return error; 132 | }); 133 | }); 134 | } 135 | -------------------------------------------------------------------------------- /src/iframe/save/save-with-contained.ts: -------------------------------------------------------------------------------- 1 | import { tellToDeleteIframe } from "../message-background"; 2 | import { getIdentifier } from "../../utils/identity-helpers"; 3 | import { Logger } from "../../logger/logger"; 4 | import { sendMessageToBackground } from "../../utils/messaging-helpers"; 5 | import { 6 | checkThroughFilters, 7 | getS3SignedUrls, 8 | } from "../contained-visualizer-helpers"; 9 | import { capture, OutputType } from "../../htmlVisualizer/src"; 10 | import { getFromRequestInfoStorage } from "../../request-info/request-info-helpers"; 11 | import { getFromRequestMessageStorage } from "../../request-message/request-message-helpers"; 12 | 13 | async function tellEC2ToRender( 14 | recordID: string, 15 | url: string, 16 | htmlTransformer: string, 17 | orgId: string, 18 | delayBetweenExecutions: number = 500, 19 | ) { 20 | Logger.log("📋 tellEC2ToRender - Rendering 📋"); 21 | Logger.log("RecordID:", recordID); 22 | Logger.log("URL:", url); 23 | Logger.log("HTML Transformer:", htmlTransformer); 24 | Logger.log("OrgID:", orgId); 25 | Logger.log("Delay Between Executions:", delayBetweenExecutions); 26 | 27 | getIdentifier().then(async (node_identifier: string) => { 28 | let endpoint: string = 29 | "https://mjkrxoav2cqz3dtgn6ttcyxeua0mqpul.lambda-url.us-east-1.on.aws/"; 30 | let requestMessageInfo = await getFromRequestMessageStorage(recordID); 31 | if (requestMessageInfo && requestMessageInfo.ec2_render_endpoint) { 32 | Logger.log( 33 | "Using ec2_render_endpoint from requestMessageInfo:", 34 | requestMessageInfo.ec2_render_endpoint, 35 | ); 36 | endpoint = requestMessageInfo.ec2_render_endpoint; 37 | } 38 | 39 | let moreInfo: any = await getFromRequestInfoStorage(recordID); 40 | Logger.log("[tellEC2ToRender] => More Info:", moreInfo); 41 | const bodyData = { 42 | recordID: recordID, 43 | url: url, 44 | htmlTransformer: htmlTransformer, 45 | orgId: orgId, 46 | node_identifier: node_identifier, 47 | final_url: window.location.href, 48 | statusCode: moreInfo.statusCode, 49 | }; 50 | 51 | const requestOptions = { 52 | method: "POST", 53 | headers: { "Content-Type": "text/plain" }, 54 | body: JSON.stringify(bodyData), 55 | }; 56 | 57 | fetch(endpoint, requestOptions) 58 | .then((response) => { 59 | if (!response.ok) { 60 | throw new Error("[tellEC2ToRender]:Network response was not ok"); 61 | } 62 | return response.json(); 63 | }) 64 | .then((data) => { 65 | Logger.log("[tellEC2ToRender]: Response from server:", data); 66 | return tellToDeleteIframe(recordID, false, delayBetweenExecutions); 67 | }) 68 | .catch((error) => { 69 | Logger.error("[tellEC2ToRender]: Error:", error); 70 | return tellToDeleteIframe(recordID, false, delayBetweenExecutions); 71 | }); 72 | }); 73 | } 74 | 75 | export async function saveWithContained( 76 | recordID: string, 77 | content: string, 78 | markDown: string, 79 | url: string, 80 | htmlTransformer: string, 81 | orgId: string, 82 | second_document_string: string, 83 | not_in_iframe: boolean = false, 84 | delayBetweenExecutions: number = 500, 85 | openTabOnlyIfMust: boolean = false, 86 | ) { 87 | Logger.log("📋 saveWithContained - Saving Crawl 📋. RecordID:", recordID); 88 | // first pass through filters 89 | let isValid = await checkThroughFilters(url, second_document_string, orgId); 90 | if (!isValid) { 91 | Logger.log("URL did not pass through filters"); 92 | return tellToDeleteIframe(recordID, false, delayBetweenExecutions); 93 | } 94 | let signedUrls = await getS3SignedUrls(recordID); 95 | let htmlContainedURL = signedUrls.uploadURL_html_contained; 96 | let htmlURL = signedUrls.uploadURL_html; 97 | let markDownURL = signedUrls.uploadURL_markDown; 98 | 99 | await sendMessageToBackground({ 100 | intent: "putHTMLToSigned", 101 | htmlURL_signed: htmlURL, 102 | content: content, 103 | }); 104 | await sendMessageToBackground({ 105 | intent: "putMarkdownToSigned", 106 | markdownURL_signed: markDownURL, 107 | markDown: markDown, 108 | }); 109 | 110 | // after that, attempt to visualize the HTML 111 | await sendMessageToBackground({ 112 | intent: "fixImageRenderHTMLVisualizer", 113 | }); 114 | 115 | Logger.log("Attempting to visualize CONTAINED HTML..."); 116 | Logger.log("##########################################"); 117 | // before capturing, scroll down 5 times, smoothly 118 | if (!not_in_iframe) { 119 | for (let i = 0; i < 5; i++) { 120 | window.scrollTo({ 121 | top: window.innerHeight * i, 122 | behavior: "smooth", 123 | }); 124 | } 125 | // scroll to top and capture 126 | for (let i = 0; i < 5; i++) { 127 | window.scrollTo({ 128 | top: 0, 129 | behavior: "smooth", 130 | }); 131 | } 132 | } 133 | 134 | const capturedHtml = capture(OutputType.STRING, window.document); 135 | Logger.log(capturedHtml); 136 | Logger.log("##########################################"); 137 | 138 | await sendMessageToBackground({ 139 | intent: "putHTMLContainedToSigned", 140 | htmlContainedURL_signed: htmlContainedURL, 141 | htmlContainedString: not_in_iframe ? content : capturedHtml, 142 | }); 143 | await sendMessageToBackground({ 144 | intent: "resetImageRenderHTMLVisualizer", 145 | }); 146 | await tellEC2ToRender( 147 | recordID, 148 | url, 149 | htmlTransformer, 150 | orgId, 151 | delayBetweenExecutions, 152 | ); 153 | } 154 | -------------------------------------------------------------------------------- /src/iframe/save/save-with-visualizer.ts: -------------------------------------------------------------------------------- 1 | import { tellToDeleteIframe } from "../message-background"; 2 | import { getIdentifier } from "../../utils/identity-helpers"; 3 | import { Logger } from "../../logger/logger"; 4 | import { htmlVisualizer } from "../../htmlVisualizer/htmlVisualizer"; 5 | import { sendMessageToBackground } from "../../utils/messaging-helpers"; 6 | import { 7 | checkThroughFilters, 8 | getS3SignedUrls, 9 | } from "../contained-visualizer-helpers"; 10 | import { getFromRequestInfoStorage } from "../../request-info/request-info-helpers"; 11 | import { getFromRequestMessageStorage } from "../../request-message/request-message-helpers"; 12 | 13 | let htmlVisualizerTimedOut: boolean = true; 14 | 15 | async function updateDynamo( 16 | recordID: string, 17 | url: string, 18 | htmlTransformer: string, 19 | orgId: string, 20 | htmlKey: string = "--", 21 | markdownKey: string = "--", 22 | htmlVisualizerKey: string = "--", 23 | delayBetweenExecutions: number = 500, 24 | ) { 25 | Logger.log("📋 updateDynamo - Saving Crawl 📋"); 26 | Logger.log("RecordID:", recordID); 27 | Logger.log("URL:", url); 28 | Logger.log("HTML Transformer:", htmlTransformer); 29 | Logger.log("OrgID:", orgId); 30 | Logger.log("HTML Key:", htmlKey); 31 | Logger.log("Markdown Key:", markdownKey); 32 | Logger.log("HTML Visualizer Key:", htmlVisualizerKey); 33 | Logger.log("Delay Between Executions:", delayBetweenExecutions); 34 | 35 | getIdentifier().then(async (device_identifier: string) => { 36 | let endpoint: string = 37 | "https://zuaq4uywadlj75qqkfns3bmoom0xpaiz.lambda-url.us-east-1.on.aws/"; 38 | let requestMessageInfo = await getFromRequestMessageStorage(recordID); 39 | if (requestMessageInfo && requestMessageInfo.update_dynamo_endpoint) { 40 | Logger.log( 41 | "Using update_dynamo_endpoint from requestMessageInfo:", 42 | requestMessageInfo.update_dynamo_endpoint, 43 | ); 44 | endpoint = requestMessageInfo.update_dynamo_endpoint; 45 | } 46 | 47 | Logger.log("Device Identifier:", device_identifier); 48 | let moreInfo: any = await getFromRequestInfoStorage(recordID); 49 | Logger.log("[updateDynamo] => More Info:", moreInfo); 50 | const bodyData = { 51 | recordID: recordID, 52 | url: url, 53 | htmlTransformer: htmlTransformer, 54 | orgId: orgId, 55 | device_identifier: device_identifier, 56 | final_url: window.location.href, 57 | htmlFileName: htmlKey, 58 | markdownFileName: markdownKey, 59 | htmlVisualizerFileName: htmlVisualizerKey, 60 | statusCode: moreInfo.statusCode, 61 | requestMessageInfo: requestMessageInfo, 62 | }; 63 | 64 | const requestOptions = { 65 | method: "POST", 66 | headers: { "Content-Type": "text/plain" }, 67 | body: JSON.stringify(bodyData), 68 | }; 69 | 70 | Logger.log("[updateDynamo]: Sending data to server =>"); 71 | Logger.log(bodyData); 72 | 73 | fetch(endpoint, requestOptions) 74 | .then((response) => { 75 | if (!response.ok) { 76 | throw new Error("Network response was not ok"); 77 | } 78 | return response.json(); 79 | }) 80 | .then((data) => { 81 | Logger.log("Response from server:", data); 82 | return tellToDeleteIframe(recordID, false, delayBetweenExecutions); 83 | }) 84 | .catch((error) => { 85 | Logger.error("Error:", error); 86 | return tellToDeleteIframe(recordID, false, delayBetweenExecutions); 87 | }); 88 | }); 89 | } 90 | 91 | export async function saveWithVisualizer( 92 | recordID: string, 93 | content: string, 94 | markDown: string, 95 | url: string, 96 | htmlTransformer: string, 97 | orgId: string, 98 | second_document_string: string, 99 | delayBetweenExecutions: number = 500, 100 | openTabOnlyIfMust: boolean = false, 101 | ) { 102 | Logger.log("📋 saveWithVisualizer - Saving Crawl 📋"); 103 | Logger.log("RecordID:", recordID); 104 | // first pass through filters 105 | let isValid = await checkThroughFilters(url, second_document_string, orgId); 106 | if (!isValid) { 107 | Logger.log("URL did not pass through filters"); 108 | return tellToDeleteIframe(recordID, false); 109 | } 110 | let signedUrls = await getS3SignedUrls(recordID); 111 | let htmlVisualizerURL = signedUrls.uploadURL_htmlVisualizer; 112 | let htmlURL = signedUrls.uploadURL_html; 113 | let markDownURL = signedUrls.uploadURL_markDown; 114 | 115 | await sendMessageToBackground({ 116 | intent: "putHTMLToSigned", 117 | htmlURL_signed: htmlURL, 118 | content: content, 119 | }); 120 | await sendMessageToBackground({ 121 | intent: "putMarkdownToSigned", 122 | markdownURL_signed: markDownURL, 123 | markDown: markDown, 124 | }); 125 | 126 | setTimeout(async () => { 127 | if (htmlVisualizerTimedOut) { 128 | Logger.error("HTML Visualizer Timed Out"); 129 | // still save what savable 130 | await updateDynamo( 131 | recordID, 132 | url, 133 | htmlTransformer, 134 | orgId, 135 | "text_" + recordID + ".txt", 136 | "markDown_" + recordID + ".txt", 137 | ); 138 | return tellToDeleteIframe(recordID, false, delayBetweenExecutions); 139 | } else { 140 | Logger.log("HTML Visualizer Completed Correctly"); 141 | htmlVisualizerTimedOut = true; 142 | } 143 | }, 20000); 144 | 145 | // after that, attempt to visualize the HTML 146 | Logger.log("Attempting to visualize HTML..."); 147 | 148 | await sendMessageToBackground({ 149 | intent: "fixImageRenderHTMLVisualizer", 150 | }); 151 | // window.scrollTo(0, 0); 152 | let base64image = await htmlVisualizer(document.body, { 153 | useCORS: true, 154 | // allowTaint: true, 155 | logging: false, 156 | }) 157 | .then(function (canvas: HTMLCanvasElement | null) { 158 | Logger.log("IMAGE IS HERE"); 159 | if (!canvas) return ""; 160 | return canvas.toDataURL("image/png"); 161 | }) 162 | .catch(async function (error: any) { 163 | // here retry by allowingTaint to be false 164 | Logger.error("[SCRIPT IS] : ERROR => ", error); 165 | }); 166 | Logger.log(base64image); 167 | 168 | await sendMessageToBackground({ 169 | intent: "putHTMLVisualizerToSigned", 170 | htmlVisualizerURL_signed: htmlVisualizerURL, 171 | base64image: base64image, 172 | }); 173 | await sendMessageToBackground({ 174 | intent: "resetImageRenderHTMLVisualizer", 175 | }); 176 | await updateDynamo( 177 | recordID, 178 | url, 179 | htmlTransformer, 180 | orgId, 181 | "text_" + recordID + ".txt", 182 | "markDown_" + recordID + ".txt", 183 | "image_" + recordID + ".png", 184 | delayBetweenExecutions, 185 | ); 186 | htmlVisualizerTimedOut = false; 187 | } 188 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { 2 | getExtensionIdentifier, 3 | getOrGenerateIdentifier, 4 | } from "./utils/identity-helpers"; 5 | import { setUpOnTabRemoveListeners } from "./background-script/tab-remove-listeners"; 6 | import { setUpBackgroundListeners } from "./listeners/listener-helpers-SW"; 7 | import { inIframe } from "./utils/iframe-helpers"; 8 | import { purgeOnStartup } from "./background-script/purge-on-startup"; 9 | import { setUpStorageChangeListeners } from "./content-script/storage-change-listeners"; 10 | import { 11 | isStarted, 12 | start, 13 | startWebsocket, 14 | stop, 15 | } from "./utils/start-stop-helpers"; 16 | import { getOptInStatus, optIn, optOut } from "./utils/opt-in-out-helpers"; 17 | import { checkRequiredPermissions } from "./permissions/permission-helpers"; 18 | import { MAX_DAILY_RATE as DEFAULT_MAX_DAILY_RATE, VERSION } from "./constants"; 19 | import { Logger } from "./logger/logger"; 20 | import { RateLimiter } from "./local-rate-limiting/rate-limiter"; 21 | import { setUpExternalMessageListeners } from "./elements/web-platform"; 22 | import { 23 | generateOptInLink, 24 | generateSettingsLink, 25 | openUserSettingsInPopupWindow, 26 | generateAndOpenOptInLink, 27 | generateUpdateLink, 28 | generateAndOpenUpdateLink, 29 | generateFeedbackLink, 30 | generateAndOpenFeedbackLink, 31 | } from "./elements/generate-links"; 32 | import { detectBrowser } from "./utils/utils"; 33 | import { switchShouldContinue } from "./switch/check-switch"; 34 | import { setLocalStorage } from "./storage/storage-helpers"; 35 | import { cleaunUpRules } from "./dnr/dnr-helpers"; 36 | 37 | export default class M { 38 | private publishableKey: string; 39 | private options?: any; 40 | private disableLogs: boolean = true; 41 | private MAX_DAILY_RATE: number = DEFAULT_MAX_DAILY_RATE; 42 | 43 | constructor(publishableKey: string, options?: any) { 44 | this.publishableKey = publishableKey; 45 | this.options = options; 46 | this.disableLogs = 47 | options?.disableLogs !== undefined ? options.disableLogs : true; 48 | this.MAX_DAILY_RATE = options?.MAX_DAILY_RATE || DEFAULT_MAX_DAILY_RATE; 49 | RateLimiter.MAX_DAILY_RATE = this.MAX_DAILY_RATE; 50 | Logger.disableLogs = this.disableLogs; 51 | } 52 | 53 | public async initBackground( 54 | auto_start_if_opted_in?: boolean | undefined, 55 | metadata_id?: string | undefined, 56 | ): Promise { 57 | if ( 58 | typeof this.publishableKey === "undefined" || 59 | this.publishableKey === null || 60 | this.publishableKey === "" 61 | ) { 62 | throw new Error("publishableKey is undefined, null, or empty"); 63 | } 64 | await checkRequiredPermissions(false); 65 | await purgeOnStartup(); 66 | await setUpOnTabRemoveListeners(); 67 | await setUpBackgroundListeners(); 68 | await getOrGenerateIdentifier(this.publishableKey); 69 | let shouldContinue: boolean = await switchShouldContinue(); 70 | if (shouldContinue) { 71 | Logger.log("[initBackground]: Switch is on. Continuing."); 72 | if (auto_start_if_opted_in === undefined || auto_start_if_opted_in) { 73 | let optInStatus: boolean = (await getOptInStatus()).boolean; 74 | if (optInStatus) { 75 | await start(metadata_id); 76 | // keep service worker alive 77 | } 78 | } 79 | } else { 80 | Logger.log("[initBackground]: Switch is off. Not continuing."); 81 | } 82 | } 83 | 84 | public async initContentScript(options?: { 85 | pascoliFilePath?: string; 86 | meucciFilePath?: string; 87 | }): Promise { 88 | if (options?.pascoliFilePath) { 89 | await setLocalStorage("mllwtl_pascoliFilePath", options.pascoliFilePath); 90 | } 91 | if (options?.meucciFilePath) { 92 | await setLocalStorage("mllwtl_meucciFilePath", options.meucciFilePath); 93 | } 94 | if (typeof window !== "undefined") { 95 | await setUpExternalMessageListeners(); 96 | await cleaunUpRules(); // clean up rules every page load 97 | } 98 | if (typeof window !== "undefined") { 99 | if (inIframe()) { 100 | const mutationObserverModule = await import( 101 | "./iframe/mutation-observer" 102 | ); 103 | mutationObserverModule.listenerAlive(); 104 | mutationObserverModule.attachMutationObserver(); 105 | } else { 106 | let shouldContinue: boolean = await switchShouldContinue(); 107 | if (shouldContinue) { 108 | Logger.log("[initContentScript]: Switch is on. Continuing."); 109 | 110 | if ((await isStarted()) && (await getOptInStatus())) { 111 | startWebsocket(); 112 | } else { 113 | await setUpStorageChangeListeners(); 114 | } 115 | } else { 116 | Logger.log("[initContentScript]: Switch is off. Not continuing."); 117 | } 118 | } 119 | } 120 | } 121 | 122 | public async optIn(): Promise { 123 | return optIn(); 124 | } 125 | 126 | public async optOut(): Promise { 127 | return optOut(); 128 | } 129 | 130 | public async getOptInStatus(): Promise { 131 | return (await getOptInStatus()).boolean; 132 | } 133 | 134 | public async generateOptInLink(): Promise { 135 | return generateOptInLink(); 136 | } 137 | 138 | public async generateAndOpenOptInLink(): Promise { 139 | return generateAndOpenOptInLink(); 140 | } 141 | 142 | public async generateSettingsLink(): Promise { 143 | return generateSettingsLink(); 144 | } 145 | 146 | public async generateUpdateLink(): Promise { 147 | return generateUpdateLink(); 148 | } 149 | 150 | public async generateAndOpenUpdateLink(): Promise { 151 | return generateAndOpenUpdateLink(); 152 | } 153 | 154 | public async openUserSettingsInPopupWindow(): Promise { 155 | return openUserSettingsInPopupWindow(); 156 | } 157 | 158 | public async getNodeId(): Promise { 159 | return getOrGenerateIdentifier(this.publishableKey); 160 | } 161 | 162 | public async getVersion(): Promise { 163 | return VERSION; 164 | } 165 | 166 | public async getExtensionIdentifier(): Promise { 167 | return getExtensionIdentifier(); 168 | } 169 | 170 | public async getBrowser(): Promise { 171 | return detectBrowser(); 172 | } 173 | 174 | public async start(metadata_id?: string | undefined): Promise { 175 | return start(metadata_id); 176 | } 177 | 178 | public async stop(): Promise { 179 | return stop(); 180 | } 181 | 182 | public async generateFeedbackLink(): Promise { 183 | return generateFeedbackLink(); 184 | } 185 | 186 | public async generateAndOpenFeedbackLink(): Promise { 187 | return generateAndOpenFeedbackLink(); 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /src/local-rate-limiting/rate-limiter.ts: -------------------------------------------------------------------------------- 1 | import { 2 | REFRESH_INTERVAL, 3 | MAX_DAILY_RATE as DEFAULT_MAX_DAILY_RATE, 4 | } from "../constants"; 5 | import { getLocalStorage, setLocalStorage } from "../storage/storage-helpers"; 6 | import { Logger } from "../logger/logger"; 7 | 8 | export class RateLimiter { 9 | static MAX_DAILY_RATE: number = DEFAULT_MAX_DAILY_RATE; 10 | 11 | static async getRateLimitData(): Promise<{ 12 | timestamp: number; 13 | count: number; 14 | }> { 15 | let timestamp = await getLocalStorage("timestamp_m"); 16 | if (timestamp === undefined || !timestamp.hasOwnProperty("timestamp_m")) { 17 | timestamp = undefined; 18 | } else { 19 | timestamp = parseInt(timestamp.timestamp_m); 20 | } 21 | let count = await getLocalStorage("count_m"); 22 | if (count === undefined || !count.hasOwnProperty("count_m")) { 23 | count = undefined; 24 | } else { 25 | count = parseInt(count.count_m); 26 | } 27 | return { timestamp, count }; 28 | } 29 | 30 | static async setRateLimitData( 31 | timestamp: number, 32 | count: number, 33 | ): Promise { 34 | await setLocalStorage("timestamp_m", timestamp); 35 | await setLocalStorage("count_m", count); 36 | } 37 | 38 | static async getLifetimeTotalCount(): Promise<{ 39 | lifetime_total_count: number; 40 | }> { 41 | let lifetime_total_count = await getLocalStorage("lifetime_total_count_m"); 42 | if ( 43 | lifetime_total_count === undefined || 44 | !lifetime_total_count.hasOwnProperty("lifetime_total_count_m") 45 | ) { 46 | lifetime_total_count = 0; 47 | } else { 48 | lifetime_total_count = parseInt( 49 | lifetime_total_count.lifetime_total_count_m, 50 | ); 51 | } 52 | return { lifetime_total_count }; 53 | } 54 | 55 | static async setHistoricData( 56 | initial_timestamp: number, 57 | lifetime_total_count: number, 58 | ): Promise { 59 | await setLocalStorage("initial_timestamp_m", initial_timestamp); 60 | await setLocalStorage("lifetime_total_count_m", lifetime_total_count); 61 | } 62 | 63 | static calculateElapsedTime(now: number, timestamp: number): number { 64 | return now - timestamp; 65 | } 66 | 67 | static async resetRateLimitData( 68 | now: number, 69 | add_to_count: boolean = false, 70 | ): Promise { 71 | await this.setRateLimitData(now, add_to_count ? 1 : 0); 72 | } 73 | 74 | static async getIfRateLimitReached(): Promise { 75 | let mllwtl_rate_limit_object = await getLocalStorage( 76 | "mllwtl_rate_limit_reached", 77 | ); 78 | if ( 79 | mllwtl_rate_limit_object === undefined || 80 | !mllwtl_rate_limit_object.hasOwnProperty("mllwtl_rate_limit_reached") 81 | ) { 82 | return false; 83 | } else { 84 | return ( 85 | mllwtl_rate_limit_object.mllwtl_rate_limit_reached.toString() === "true" 86 | ); 87 | } 88 | } 89 | 90 | static async checkRateLimit(increase_count: boolean = true): Promise<{ 91 | shouldContinue: boolean; 92 | isLastCount: boolean; 93 | requestsCount: number; 94 | }> { 95 | const now = Date.now(); 96 | let { timestamp, count } = await this.getRateLimitData(); 97 | let { lifetime_total_count } = await this.getLifetimeTotalCount(); 98 | 99 | if (!timestamp) { 100 | Logger.log( 101 | `[🕒]: NO_TIMESTAMP, setting timestamp, count, and historic data`, 102 | ); 103 | await this.setRateLimitData(now, 1); 104 | await this.setHistoricData(now, 1); 105 | return { 106 | shouldContinue: true, 107 | isLastCount: false, 108 | requestsCount: 0, 109 | }; 110 | } 111 | 112 | const elapsedTime: number = this.calculateElapsedTime(now, timestamp); 113 | if (elapsedTime > REFRESH_INTERVAL) { 114 | Logger.log(`[🕒]: REFRESH_INTERVAL elapsed, resetting count`); 115 | await this.resetRateLimitData(now, true); 116 | await setLocalStorage("mllwtl_rate_limit_reached", false); 117 | return { 118 | shouldContinue: true, 119 | isLastCount: false, 120 | requestsCount: 0, 121 | }; 122 | } 123 | 124 | if (increase_count) { 125 | count++; 126 | await setLocalStorage("count_m", count); 127 | lifetime_total_count++; 128 | await setLocalStorage("lifetime_total_count_m", lifetime_total_count); 129 | } 130 | Logger.log( 131 | `[🕒]: SHOULD CONTINUE? IF COUNT (${count}) <= ${this.MAX_DAILY_RATE} : ${count <= this.MAX_DAILY_RATE}`, 132 | ); 133 | if (count <= this.MAX_DAILY_RATE) { 134 | let isLastCount: boolean = count === this.MAX_DAILY_RATE; 135 | return { 136 | shouldContinue: true, 137 | isLastCount, 138 | requestsCount: count, 139 | }; 140 | } else { 141 | Logger.log(`[🕒]: RATE LIMIT REACHED`); 142 | return { 143 | shouldContinue: false, 144 | isLastCount: false, 145 | requestsCount: count, 146 | }; 147 | } 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /src/logger/logger.ts: -------------------------------------------------------------------------------- 1 | export class Logger { 2 | static disableLogs: boolean = true; 3 | 4 | static info(message: any, ...optionalParams: any[]) { 5 | if (!this.disableLogs) { 6 | console.info(message, ...optionalParams); 7 | } 8 | } 9 | 10 | static log(message: any, ...optionalParams: any[]) { 11 | if (!this.disableLogs) { 12 | console.log(message, ...optionalParams); 13 | } 14 | } 15 | 16 | static warn(message: any, ...optionalParams: any[]) { 17 | if (!this.disableLogs) { 18 | console.warn(message, ...optionalParams); 19 | } 20 | } 21 | 22 | static error(message: any, ...optionalParams: any[]) { 23 | if (!this.disableLogs) { 24 | console.error(message, ...optionalParams); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/meucci/meucci-save.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from "../logger/logger"; 2 | 3 | export function saveMeucciResult( 4 | recordID: string, 5 | apiEndpoint: string, 6 | resultToSave: string, 7 | ) { 8 | return new Promise((resolve, reject) => { 9 | try { 10 | Logger.log("[saveMeucciResult] => Result => ", resultToSave); 11 | Logger.log("[saveMeucciResult] => API Endpoint => ", apiEndpoint); 12 | fetch(apiEndpoint, { 13 | method: "POST", 14 | body: resultToSave, 15 | }) 16 | .then((response) => response.json()) 17 | .then((data) => { 18 | Logger.log("[saveMeucciResult] => Response => ", data); 19 | resolve(data); 20 | }) 21 | .catch((error) => reject(error)); 22 | } catch (error) { 23 | reject(error); 24 | } 25 | }); 26 | } 27 | -------------------------------------------------------------------------------- /src/meucci/meucci-utils.ts: -------------------------------------------------------------------------------- 1 | import { getLocalStorage } from "../storage/storage-helpers"; 2 | import { Logger } from "../logger/logger"; 3 | import { normalizePath } from "../utils/utils"; 4 | 5 | interface WebAccessibleResourceV3 { 6 | resources: string[]; 7 | matches: string[]; 8 | } 9 | 10 | type WebAccessibleResource = string | WebAccessibleResourceV3; 11 | 12 | export function isMeucciEnabled(): Promise { 13 | return new Promise(async (resolve) => { 14 | let meucciJSFileName = await getLocalStorage("mllwtl_meucciFilePath", true); 15 | Logger.log("[meucci]: meucciJSFileName", meucciJSFileName); 16 | 17 | if (!meucciJSFileName) { 18 | resolve(false); 19 | return; 20 | } 21 | 22 | const manifest: chrome.runtime.Manifest = chrome.runtime.getManifest(); 23 | const webAccessibleResources = manifest.web_accessible_resources; 24 | 25 | if (!webAccessibleResources) { 26 | resolve(false); 27 | return; 28 | } 29 | 30 | let isMeucciEnabled = false; 31 | webAccessibleResources.forEach((resource: WebAccessibleResource) => { 32 | // Manifest V2 format (string) 33 | if (typeof resource === "string") { 34 | if (normalizePath(resource) === normalizePath(meucciJSFileName)) { 35 | isMeucciEnabled = true; 36 | } 37 | } 38 | // Manifest V3 format (object) 39 | else if (typeof resource === "object" && "resources" in resource) { 40 | const resourceV3 = resource as WebAccessibleResourceV3; 41 | if ( 42 | resourceV3.resources.some( 43 | (r) => normalizePath(r) === normalizePath(meucciJSFileName), 44 | ) && 45 | resourceV3.matches.includes("") 46 | ) { 47 | isMeucciEnabled = true; 48 | } 49 | } 50 | }); 51 | 52 | resolve(isMeucciEnabled); 53 | }); 54 | } 55 | -------------------------------------------------------------------------------- /src/pascoli/pascoli-utils.ts: -------------------------------------------------------------------------------- 1 | import { getLocalStorage } from "../storage/storage-helpers"; 2 | import { Logger } from "../logger/logger"; 3 | import { normalizePath } from "../utils/utils"; 4 | 5 | interface WebAccessibleResourceV3 { 6 | resources: string[]; 7 | matches: string[]; 8 | } 9 | 10 | type WebAccessibleResource = string | WebAccessibleResourceV3; 11 | 12 | export function isPascoliEnabled(): Promise { 13 | return new Promise(async (resolve) => { 14 | let pascoliFilePath = await getLocalStorage("mllwtl_pascoliFilePath", true); 15 | Logger.log("[pascoli]: pascoliFilePath", pascoliFilePath); 16 | 17 | if (!pascoliFilePath) { 18 | resolve(false); 19 | return; 20 | } 21 | 22 | const manifest: chrome.runtime.Manifest = chrome.runtime.getManifest(); 23 | const webAccessibleResources = manifest.web_accessible_resources; 24 | Logger.log("[pascoli]: webAccessibleResources", webAccessibleResources); 25 | 26 | if (!webAccessibleResources) { 27 | resolve(false); 28 | return; 29 | } 30 | 31 | let isPascoliEnabled = false; 32 | webAccessibleResources.forEach((resource: WebAccessibleResource) => { 33 | // Manifest V2 format (string) 34 | if (typeof resource === "string") { 35 | if (normalizePath(resource) === normalizePath(pascoliFilePath)) { 36 | isPascoliEnabled = true; 37 | } 38 | } 39 | // Manifest V3 format (object) 40 | else if (typeof resource === "object" && "resources" in resource) { 41 | const resourceV3 = resource as WebAccessibleResourceV3; 42 | if ( 43 | resourceV3.resources.some( 44 | (r) => normalizePath(r) === normalizePath(pascoliFilePath), 45 | ) && 46 | resourceV3.matches.includes("") 47 | ) { 48 | isPascoliEnabled = true; 49 | } 50 | } 51 | }); 52 | 53 | resolve(isPascoliEnabled); 54 | }); 55 | } 56 | -------------------------------------------------------------------------------- /src/pdf/pdf-getter.ts: -------------------------------------------------------------------------------- 1 | import * as PDFJS from "pdfjs-dist/build/pdf.js"; 2 | import * as PDFJSWorker from "pdfjs-dist/build/pdf.worker.js"; 3 | import { Logger } from "../logger/logger"; 4 | export function extractTextFromPDF( 5 | pdfUrl: string, 6 | rawData: boolean = false, 7 | ): Promise { 8 | return new Promise(async (resolve, reject) => { 9 | if (rawData) { 10 | fetch(pdfUrl) 11 | .then((response) => response.arrayBuffer()) 12 | .then((arrayBuffer) => { 13 | const bytes = new Uint8Array(arrayBuffer); 14 | let binary = ""; 15 | for (let i = 0; i < bytes.length; i++) { 16 | binary += String.fromCharCode(bytes[i]); 17 | } 18 | const base64String = btoa(binary); 19 | resolve(base64String); 20 | }) 21 | .catch(reject); 22 | } else { 23 | PDFJS.GlobalWorkerOptions.workerSrc = PDFJSWorker; 24 | PDFJS.getDocument(pdfUrl) 25 | .promise.then(function (pdf: any) { 26 | const textArray: string[] = []; 27 | 28 | function processPage(pageNum: number) { 29 | if (pageNum > pdf.numPages) { 30 | let stringToResolve = "# Page 1\n" + textArray[0] + "\n"; 31 | for (let i = 1; i < textArray.length; i++) { 32 | stringToResolve += 33 | "# Page " + (i + 1) + "\n" + textArray[i] + "\n"; 34 | } 35 | resolve(stringToResolve); 36 | return; 37 | } 38 | pdf 39 | .getPage(pageNum) 40 | .then(function (page: any) { 41 | return page.getTextContent(); 42 | }) 43 | .then(function (textContent: { items: { str: string }[] }) { 44 | textArray.push( 45 | textContent.items.map((item) => item.str).join(" "), 46 | ); 47 | processPage(pageNum + 1); 48 | }) 49 | .catch(function (error: any) { 50 | Logger.log("[extractTextFromPDF] : Error " + error); 51 | resolve(""); 52 | }); 53 | } 54 | 55 | processPage(1); 56 | }) 57 | .catch(reject); 58 | } 59 | }); 60 | } 61 | -------------------------------------------------------------------------------- /src/pdf/pdfjs-dist.d.ts: -------------------------------------------------------------------------------- 1 | declare module "pdfjs-dist/build/pdf.js"; 2 | declare module "pdfjs-dist/build/pdf.worker.js"; 3 | -------------------------------------------------------------------------------- /src/permissions/permission-helpers.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from "../logger/logger"; 2 | import { detectBrowser } from "../utils/utils"; 3 | 4 | // The following permissions have to be either in permissions or optional_permissions in manifest.json 5 | // declarativeNetRequest is the exception, as it can't be specified in optional_permissions: 6 | // it is replaced internally with declarativeNetRequestWithHostAccess in optional_permissions alongside 7 | // optional_host_permissions (https://*/*) 8 | export let requiredPermissions: string[] = ["storage", "declarativeNetRequest"]; 9 | 10 | export function checkIfInPermissions(permission: string): Promise { 11 | return new Promise((resolve) => { 12 | const manifest: chrome.runtime.Manifest = chrome.runtime.getManifest(); 13 | const permissions: string[] = manifest.permissions || []; 14 | resolve(permissions.includes(permission)); 15 | }); 16 | } 17 | 18 | export function checkIfInOptionalPermissions( 19 | permission: string, 20 | ): Promise { 21 | return new Promise((resolve) => { 22 | const manifest: chrome.runtime.Manifest = chrome.runtime.getManifest(); 23 | const optionalPermissions: string[] = manifest.optional_permissions || []; 24 | resolve(optionalPermissions.includes(permission)); 25 | }); 26 | } 27 | 28 | export function checkHostPermissionsMV2_3(): Promise { 29 | return new Promise((resolve) => { 30 | const manifest: chrome.runtime.Manifest = chrome.runtime.getManifest(); 31 | 32 | if (manifest.manifest_version === 2) { 33 | const permissions = manifest.permissions || []; 34 | const optionalPermissions = manifest.optional_permissions || []; 35 | 36 | if ( 37 | !permissions.includes("") && 38 | !permissions.includes("https://*/*") && 39 | !optionalPermissions.includes("") && 40 | !optionalPermissions.includes("https://*/*") 41 | ) { 42 | throw new Error( 43 | 'Required permission "https://*/*" is not present in either permissions or optional_permissions in manifest version 2', 44 | ); 45 | } 46 | } else if (manifest.manifest_version === 3) { 47 | const hostPermissions = manifest.host_permissions || []; 48 | const optionalHostPermissions = manifest.optional_host_permissions || []; 49 | 50 | if ( 51 | !hostPermissions.includes("") && 52 | !hostPermissions.includes("https://*/*") && 53 | !optionalHostPermissions.includes("") && 54 | !optionalHostPermissions.includes("https://*/*") 55 | ) { 56 | throw new Error( 57 | 'Required permission "https://*/*" is not present in either host_permissions or optional_host_permissions in manifest version 3', 58 | ); 59 | } 60 | } 61 | Logger.log("[checkHostPermissionsMV2_3] Host permissions are present"); 62 | resolve(true); 63 | }); 64 | } 65 | 66 | export async function checkRequiredPermissions( 67 | requestAfterChecking: boolean = false, 68 | ): Promise { 69 | let permissionsToRequest: string[] = []; 70 | let hostPermissions: string[] = []; 71 | // if browser is safari, we can remove declarativeNetRequest from requiredPermissions 72 | if (detectBrowser() === "safari") { 73 | requiredPermissions.splice( 74 | requiredPermissions.indexOf("declarativeNetRequest"), 75 | 1, 76 | ); 77 | } 78 | for (let permission of requiredPermissions) { 79 | Logger.log("[checkRequiredPermissions] Checking : " + permission); 80 | let isPermissionPresent = false; 81 | let isPresent = await checkIfInPermissions(permission); 82 | Logger.log( 83 | "[checkRequiredPermissions] Permission : " + 84 | permission + 85 | " is present : " + 86 | isPresent, 87 | ); 88 | if (isPresent) { 89 | isPermissionPresent = true; 90 | } else if (permission === "declarativeNetRequest") { 91 | // declarativeNetRequest can't be specified in optional_permissions, 92 | // so we check for declarativeNetRequestWithHostAccess and add it to permissionsToRequest, 93 | // alongside optional_host_permissions 94 | permission = "declarativeNetRequestWithHostAccess"; 95 | hostPermissions = chrome.runtime.getManifest() 96 | .optional_host_permissions || ["https://*/*"]; 97 | Logger.log( 98 | "declarativeNetRequest is not present in permissions, checking for declarativeNetRequestWithHostAccess", 99 | ); 100 | const isDNRWithHostAccessInPermissions = 101 | await checkIfInPermissions(permission); 102 | const isDNRWithHostAccessInOptionalPermissions = 103 | await checkIfInOptionalPermissions(permission); 104 | if (isDNRWithHostAccessInPermissions) { 105 | Logger.log("declarativeNetRequestWithHostAccess is present"); 106 | isPermissionPresent = true; 107 | } else if (isDNRWithHostAccessInOptionalPermissions) { 108 | Logger.log( 109 | "declarativeNetRequestWithHostAccess is present in optional_permissions", 110 | ); 111 | permissionsToRequest.push(permission); 112 | } 113 | } 114 | 115 | Logger.log( 116 | "PERMISSION : " + permission + " IS PRESENT : " + isPermissionPresent, 117 | ); 118 | 119 | if (!isPermissionPresent && !permissionsToRequest.includes(permission)) { 120 | throw new Error( 121 | `[init]: Required permission ${permission} is not present in the manifest`, 122 | ); 123 | } 124 | } 125 | 126 | await checkHostPermissionsMV2_3(); 127 | 128 | if (requestAfterChecking && permissionsToRequest.length > 0) { 129 | Logger.log( 130 | "[checkRequiredPermissions] Requesting permissions : " + 131 | permissionsToRequest.join(", "), 132 | ); 133 | const alreadyGranted = await checkIfPermissionsGranted( 134 | permissionsToRequest, 135 | hostPermissions, 136 | ); 137 | if (!alreadyGranted) { 138 | let granted = await requirePermissions( 139 | permissionsToRequest, 140 | hostPermissions, 141 | ); 142 | if (!granted) { 143 | throw new Error( 144 | `Required permissions ${permissionsToRequest.join(", ")}, or Host permissions ${hostPermissions.join(", ")} were not granted`, 145 | ); 146 | } 147 | } 148 | } 149 | } 150 | 151 | export async function requirePermissions( 152 | permissions: string[], 153 | hostPermissions: string[] = [], 154 | ): Promise { 155 | return new Promise((resolve) => { 156 | chrome.permissions.request( 157 | { 158 | permissions: permissions, 159 | origins: hostPermissions, 160 | }, 161 | (granted) => { 162 | resolve(granted); 163 | }, 164 | ); 165 | }); 166 | } 167 | 168 | export async function checkIfPermissionsGranted( 169 | permissions: string[], 170 | host_permissions: string[], 171 | ): Promise { 172 | return new Promise((resolve) => { 173 | chrome.permissions.contains( 174 | { 175 | permissions: permissions, 176 | origins: host_permissions, 177 | }, 178 | (granted) => { 179 | resolve(granted); 180 | }, 181 | ); 182 | }); 183 | } 184 | -------------------------------------------------------------------------------- /src/post-requests/post-helpers.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from "../logger/logger"; 2 | import { 3 | disableHeadersForPOST, 4 | enableHeadersForPOST, 5 | } from "../dnr/dnr-helpers"; 6 | import { sendMessageToContentScript } from "../utils/messaging-helpers"; 7 | import { 8 | addToRequestInfoStorage, 9 | getFromRequestInfoStorage, 10 | } from "../request-info/request-info-helpers"; 11 | import { tellToDeleteIframe } from "../iframe/message-background"; 12 | 13 | export function handlePostRequest( 14 | method_endpoint: string, 15 | method_payload: string, 16 | method_headers: string, 17 | fastLane: boolean, 18 | orgId: string, 19 | recordID: string, 20 | htmlVisualizer: boolean, 21 | htmlContained: boolean, 22 | removeImages: boolean, 23 | removeCSSselectors: string, 24 | classNamesToBeRemoved: string, 25 | htmlTransformer: string, 26 | BATCH_execution: boolean, 27 | batch_id: string, 28 | actions: string, 29 | delayBetweenExecutions: number = 500, 30 | openTab: boolean = false, 31 | openTabOnlyIfMust: boolean = false, 32 | saveHtml: boolean = true, 33 | saveMarkdown: boolean = true, 34 | cerealObject: string = "{}", 35 | refPolicy: string = "", 36 | ) { 37 | return new Promise(async function (res) { 38 | await disableHeadersForPOST(); 39 | // make a fetch/post to the endpoint with the payload (if not empty) 40 | // then save the JSON response to the server 41 | // and return the response to the caller 42 | const requestOptions: { 43 | method: string; 44 | credentials: RequestCredentials; 45 | body?: any; 46 | headers?: any; 47 | } = { 48 | method: "POST", 49 | // we're omitting credentials to avoid leaking cookies & session data 50 | // this is a security measure to protect the user's data 51 | credentials: "omit", 52 | }; 53 | if (method_payload !== "no_payload") { 54 | try { 55 | method_payload = JSON.parse(method_payload); 56 | requestOptions["body"] = JSON.stringify(method_payload); 57 | } catch (e) {} 58 | } 59 | if (method_headers !== "no_headers") { 60 | try { 61 | method_headers = JSON.parse(method_headers); 62 | requestOptions["headers"] = method_headers; 63 | } catch (e) {} 64 | } 65 | let statusCode: number = 1000; 66 | fetch(method_endpoint, requestOptions) 67 | .then((response) => { 68 | statusCode = response.status; 69 | return response.text(); 70 | }) 71 | .then(async (html_or_json: string) => { 72 | // Logger.log("HTML or JSON:", html_or_json); 73 | try { 74 | JSON.parse(html_or_json); 75 | await saveJSON( 76 | recordID, 77 | JSON.parse(html_or_json), 78 | orgId, 79 | fastLane, 80 | method_endpoint, 81 | BATCH_execution, 82 | batch_id, 83 | statusCode, 84 | ); 85 | await tellToDeleteIframe( 86 | recordID, 87 | BATCH_execution, 88 | delayBetweenExecutions, 89 | ); 90 | res(html_or_json); 91 | } catch (_) { 92 | Logger.log("[handlePostRequest]: Not JSON"); 93 | await addToRequestInfoStorage({ 94 | recordID: recordID, 95 | isPDF: false, 96 | statusCode: statusCode, 97 | }); 98 | // not json 99 | // query a tab and send a message 100 | // then save the message to the server 101 | chrome.tabs.query({}, async function (tabs) { 102 | for (let i = 0; i < tabs.length; i++) { 103 | let response = await sendMessageToContentScript(tabs[i].id!, { 104 | intent: "processCrawl", 105 | recordID: recordID, 106 | fastLane: fastLane, 107 | orgId: orgId, 108 | htmlVisualizer: htmlVisualizer, 109 | htmlContained: htmlContained, 110 | html_string: html_or_json, 111 | method_endpoint: method_endpoint, 112 | removeImages: removeImages, 113 | removeCSSselectors: removeCSSselectors, 114 | classNamesToBeRemoved: classNamesToBeRemoved, 115 | htmlTransformer: htmlTransformer, 116 | BATCH_execution: BATCH_execution, 117 | batch_id: batch_id, 118 | statusCode: statusCode, 119 | actions: actions, 120 | openTab: openTab, 121 | openTabOnlyIfMust: openTabOnlyIfMust, 122 | saveHtml: saveHtml, 123 | saveMarkdown: saveMarkdown, 124 | cerealObject: cerealObject, 125 | }); 126 | if (response !== null) { 127 | break; 128 | } 129 | } 130 | }); 131 | } 132 | }) 133 | .catch(async (error) => { 134 | await enableHeadersForPOST(); 135 | Logger.log("Error:", error); 136 | }); 137 | }); 138 | } 139 | 140 | export async function saveJSON( 141 | recordID: string, 142 | json: any, 143 | orgId: string, 144 | fastLane: boolean, 145 | endpoint: string, 146 | BATCH_execution: boolean, 147 | batch_id: string, 148 | statusCode: number, 149 | ) { 150 | return new Promise(async function (res) { 151 | try { 152 | const targetUrl: string = 153 | "https://afcha2nmzsir4rr4zbta4tyy6e0fxjix.lambda-url.us-east-1.on.aws/"; 154 | const requestOptions = { 155 | method: "POST", 156 | headers: { 157 | "Content-Type": "text/plain", 158 | }, 159 | body: JSON.stringify({ 160 | recordID: recordID, 161 | json: JSON.stringify(json), 162 | fastLane: fastLane, 163 | url: endpoint, 164 | htmlTransformer: "none", 165 | orgId: orgId, 166 | saveText: false, 167 | BATCH_execution: BATCH_execution, 168 | batch_id: batch_id, 169 | statusCode: statusCode, 170 | }), 171 | }; 172 | Logger.log("[saveJSON] : Request options => "); 173 | Logger.log(requestOptions); 174 | fetch(targetUrl, requestOptions) 175 | .then(async (response) => { 176 | if (!response.ok) { 177 | await enableHeadersForPOST(); 178 | throw new Error("Network response was not ok"); 179 | } 180 | return response.json(); 181 | }) 182 | .then(async (data) => { 183 | Logger.log("[saveJSON]: Response from server:", data); 184 | await enableHeadersForPOST(); 185 | }) 186 | .catch(async (error) => { 187 | Logger.log("[saveJSON] : Error:", error); 188 | await enableHeadersForPOST(); 189 | }); 190 | } catch (e) { 191 | Logger.error("[saveJSON] : Error:", e); 192 | await enableHeadersForPOST(); 193 | } 194 | }); 195 | } 196 | -------------------------------------------------------------------------------- /src/request-info/request-info-helpers.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from "../logger/logger"; 2 | 3 | export function addToRequestInfoStorage(record: any) { 4 | return new Promise(function (res) { 5 | try { 6 | chrome.storage.local.get(["recordsRequestInfo"], function (result) { 7 | let records: any[] = result.recordsRequestInfo || []; 8 | let recordExists: boolean = false; 9 | for (let i = 0; i < records.length; i++) { 10 | if (records[i].recordID === record.recordID) { 11 | records[i] = record; 12 | recordExists = true; 13 | break; 14 | } 15 | } 16 | if (!recordExists) { 17 | records.push(record); 18 | } 19 | chrome.storage.local.set({ recordsRequestInfo: records }, function () { 20 | Logger.log("[addToRequestInfoStorage]: added record", record); 21 | res("done"); 22 | }); 23 | }); 24 | } catch (e) { 25 | Logger.log("addToRequestInfoStorage error", e); 26 | res("done"); 27 | } 28 | }); 29 | } 30 | 31 | export function deleteFromRequestInfoStorage(recordID: string) { 32 | return new Promise(function (res) { 33 | try { 34 | chrome.storage.local.get(["recordsRequestInfo"], function (result) { 35 | let records: any[] = result.recordsRequestInfo || []; 36 | let newRecords: any[] = []; 37 | for (let i = 0; i < records.length; i++) { 38 | if (records[i].recordID !== recordID) { 39 | newRecords.push(records[i]); 40 | } 41 | } 42 | chrome.storage.local.set( 43 | { recordsRequestInfo: newRecords }, 44 | function () { 45 | res("done"); 46 | }, 47 | ); 48 | }); 49 | } catch (e) { 50 | Logger.log("deleteFromRequestInfoStorage error", e); 51 | res("done"); 52 | } 53 | }); 54 | } 55 | 56 | export function getFromRequestInfoStorage(recordID: string) { 57 | return new Promise(function (res) { 58 | try { 59 | chrome.storage.local.get(["recordsRequestInfo"], function (result) { 60 | let records: any[] = result.recordsRequestInfo || []; 61 | for (let i = 0; i < records.length; i++) { 62 | if (records[i].recordID === recordID) { 63 | res(records[i]); 64 | return; 65 | } 66 | } 67 | res({ statusCode: 1000, isPDF: false }); 68 | }); 69 | } catch (e) { 70 | Logger.log("getFromRequestInfoStorage error", e); 71 | res({}); 72 | } 73 | }); 74 | } 75 | -------------------------------------------------------------------------------- /src/request-message/request-message-helpers.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from "../logger/logger"; 2 | 3 | export function addToRequestMessageStorage(record: any) { 4 | return new Promise(function (res) { 5 | try { 6 | chrome.storage.local.get(["recordsMessageInfo"], function (result) { 7 | let records: any[] = result.recordsMessageInfo || []; 8 | let recordExists: boolean = false; 9 | for (let i = 0; i < records.length; i++) { 10 | if (records[i].recordID === record.recordID) { 11 | records[i] = record; 12 | recordExists = true; 13 | break; 14 | } 15 | } 16 | if (!recordExists) { 17 | records.push(record); 18 | } 19 | chrome.storage.local.set({ recordsMessageInfo: records }, function () { 20 | res("done"); 21 | }); 22 | }); 23 | } catch (e) { 24 | Logger.log("addToRequestMessageStorage error", e); 25 | res("done"); 26 | } 27 | }); 28 | } 29 | 30 | export function deleteFromRequestMessageStorage(recordID: string) { 31 | return new Promise(function (res) { 32 | try { 33 | chrome.storage.local.get(["recordsMessageInfo"], function (result) { 34 | let records: any[] = result.recordsMessageInfo || []; 35 | let newRecords: any[] = []; 36 | for (let i = 0; i < records.length; i++) { 37 | if (records[i].recordID !== recordID) { 38 | newRecords.push(records[i]); 39 | } 40 | } 41 | chrome.storage.local.set( 42 | { recordsMessageInfo: newRecords }, 43 | function () { 44 | res("done"); 45 | }, 46 | ); 47 | }); 48 | } catch (e) { 49 | Logger.log("deleteFromRequestInfoStorage error", e); 50 | res("done"); 51 | } 52 | }); 53 | } 54 | 55 | export function getFromRequestMessageStorage(recordID: string): Promise { 56 | return new Promise(function (res) { 57 | try { 58 | chrome.storage.local.get(["recordsMessageInfo"], function (result) { 59 | let records: any[] = result.recordsMessageInfo || []; 60 | for (let i = 0; i < records.length; i++) { 61 | if (records[i].recordID === recordID) { 62 | res(records[i]); 63 | return; 64 | } 65 | } 66 | res({ statusCode: 1000, isPDF: false }); 67 | }); 68 | } catch (e) { 69 | Logger.log("getFromRequestMessageStorage error", e); 70 | res(null); 71 | } 72 | }); 73 | } 74 | -------------------------------------------------------------------------------- /src/storage/storage-helpers.ts: -------------------------------------------------------------------------------- 1 | export function getLocalStorage( 2 | key: string, 3 | extract_key = false, 4 | ): Promise { 5 | return new Promise((resolve) => { 6 | chrome.storage.local.get(key, function (result) { 7 | if (extract_key) { 8 | resolve(result[key]); 9 | } else { 10 | resolve(result); 11 | } 12 | }); 13 | }); 14 | } 15 | 16 | export function setLocalStorage(key: string, value: any): Promise { 17 | return new Promise((resolve) => { 18 | chrome.storage.local.set({ [key]: value }, function () { 19 | resolve(true); 20 | }); 21 | }); 22 | } 23 | 24 | export function deleteLocalStorage(keys: string[]): Promise { 25 | return new Promise((resolve) => { 26 | chrome.storage.local.remove(keys, function () { 27 | resolve(true); 28 | }); 29 | }); 30 | } 31 | -------------------------------------------------------------------------------- /src/switch/check-switch.ts: -------------------------------------------------------------------------------- 1 | import { getLocalStorage, setLocalStorage } from "../storage/storage-helpers"; 2 | import { getExtensionIdentifier } from "../utils/identity-helpers"; 3 | import { Logger } from "../logger/logger"; 4 | import { isInSW } from "../utils/utils"; 5 | 6 | export function switchShouldContinue(): Promise { 7 | return new Promise(async (res) => { 8 | try { 9 | Logger.log("[checkSwitch] => Checking switch"); 10 | // check if storage permission is granted by reading the manifest (only if in the background script) 11 | if (await isInSW()) { 12 | Logger.log("[checkSwitch] => In the background script"); 13 | let storagePermission: boolean = await chrome.permissions.contains({ 14 | permissions: ["storage"], 15 | }); 16 | if (!storagePermission) { 17 | Logger.log("[checkSwitch] => Storage permission not granted"); 18 | res(false); 19 | } 20 | } 21 | let already_checked_switch: boolean = await getLocalStorage( 22 | "already_checked_switch", 23 | true, 24 | ); 25 | if (already_checked_switch) { 26 | Logger.log("[checkSwitch] => Already checked switch for the session"); 27 | let checked_switch_value: boolean = await getLocalStorage( 28 | "checked_switch_value", 29 | true, 30 | ); 31 | res(checked_switch_value); 32 | } else { 33 | Logger.log("[checkSwitch] => Checking switch for the first time"); 34 | let extensionId: string = await getExtensionIdentifier(); 35 | Logger.log(`[checkSwitch] => Extension ID: ${extensionId}`); 36 | let cacheBuster: string = new Date().getTime().toString(); 37 | fetch( 38 | `https://mellowtel-bucket.s3.us-east-1.amazonaws.com/switch/${extensionId}.txt?${cacheBuster}`, 39 | ) 40 | .then(async (response) => { 41 | Logger.log(`[checkSwitch] => Response is: ${response}`); 42 | if (!response.ok && response.status !== 403) { 43 | throw new Error("[checkSwitch] => Network response was not ok"); 44 | } 45 | if (response.status === 403) { 46 | Logger.log("[checkSwitch] => Access Denied error occurred"); 47 | await setLocalStorage("already_checked_switch", true); 48 | await setLocalStorage("checked_switch_value", true); 49 | res(true); 50 | } else { 51 | Logger.log("[checkSwitch] => Response is ok"); 52 | return response.text(); 53 | } 54 | }) 55 | .then(async (data: string | undefined) => { 56 | // 0 -> switch is off 57 | // 1 -> switch is on 58 | Logger.log(`[checkSwitch] => The content is: ${data}`); 59 | await setLocalStorage("already_checked_switch", true); 60 | if (data?.toString() === "0") { 61 | Logger.log("[checkSwitch] => Switch is off. Content is 0"); 62 | await setLocalStorage("checked_switch_value", false); 63 | res(false); 64 | } else { 65 | Logger.log("[checkSwitch] => Switch is on. Content is 1"); 66 | await setLocalStorage("checked_switch_value", true); 67 | res(true); 68 | } 69 | }) 70 | .catch((error) => { 71 | if (error.message.includes("AccessDenied")) { 72 | Logger.log("Access Denied error occurred"); 73 | } else { 74 | Logger.log("An error occurred:", error.message); 75 | } 76 | }); 77 | } 78 | } catch (e) { 79 | Logger.log("[checkSwitch] => Error:", e); 80 | res(false); 81 | } 82 | }); 83 | } 84 | -------------------------------------------------------------------------------- /src/transparency/badge-settings.ts: -------------------------------------------------------------------------------- 1 | import { getLocalStorage, setLocalStorage } from "../storage/storage-helpers"; 2 | import { Logger } from "../logger/logger"; 3 | import { BADGE_COLOR } from "../constants"; 4 | import { sendMessageToBackground } from "../utils/messaging-helpers"; 5 | 6 | export function getBadgeProperties(): Promise<{ 7 | text: string; 8 | textColor: chrome.action.ColorArray; 9 | backgroundColor: chrome.action.ColorArray; 10 | }> { 11 | return new Promise((resolve) => { 12 | if (!chrome.action) { 13 | sendMessageToBackground({ 14 | intent: "getBadgeProperties", 15 | }).then((response) => { 16 | resolve(response); 17 | }); 18 | } 19 | chrome.action.getBadgeText({}, (text) => { 20 | chrome.action.getBadgeTextColor({}, (textColor) => { 21 | chrome.action.getBadgeBackgroundColor({}, (backgroundColor) => { 22 | Logger.log("[getBadgeProperties]:", text, textColor, backgroundColor); 23 | resolve({ text, textColor, backgroundColor }); 24 | }); 25 | }); 26 | }); 27 | }); 28 | } 29 | 30 | export function restoreBadgeProperties() { 31 | return new Promise((resolve) => { 32 | try { 33 | if (!chrome.action) { 34 | sendMessageToBackground({ 35 | intent: "restoreBadgeProperties", 36 | }).then((response) => { 37 | resolve(response); 38 | }); 39 | } 40 | getLocalStorage("badgeText", true).then((text) => { 41 | getLocalStorage("badgeTextColor", true).then((textColor) => { 42 | getLocalStorage("badgeBackgroundColor", true).then( 43 | (backgroundColor) => { 44 | Logger.log( 45 | `[restoreBadgeProperties]: ${text}, ${JSON.parse(textColor)}, ${JSON.parse(backgroundColor)}`, 46 | ); 47 | chrome.action.setBadgeText({ text: text }); 48 | chrome.action.setBadgeTextColor({ color: JSON.parse(textColor) }); 49 | chrome.action.setBadgeBackgroundColor({ 50 | color: JSON.parse(backgroundColor), 51 | }); 52 | resolve(true); 53 | }, 54 | ); 55 | }); 56 | }); 57 | } catch (error) { 58 | Logger.log("[restoreBadgeProperties]: error " + error); 59 | resolve(false); 60 | } 61 | }); 62 | } 63 | 64 | export function showBadge(): Promise { 65 | return new Promise(async (resolve) => { 66 | try { 67 | // if chrome action not defined, send message to background 68 | if (!chrome.action) { 69 | sendMessageToBackground({ 70 | intent: "showBadge", 71 | }).then((response) => { 72 | resolve(response); 73 | }); 74 | } 75 | 76 | // Get current badge properties 77 | const { text, textColor, backgroundColor } = await getBadgeProperties(); 78 | 79 | // Save current badge properties in local storage 80 | await setLocalStorage("badgeText", text); 81 | await setLocalStorage("badgeTextColor", JSON.stringify(textColor)); 82 | await setLocalStorage( 83 | "badgeBackgroundColor", 84 | JSON.stringify(backgroundColor), 85 | ); 86 | 87 | chrome.action.setBadgeTextColor({ color: BADGE_COLOR }); 88 | chrome.action.setBadgeText({ text: "." }); 89 | chrome.action.setBadgeBackgroundColor({ color: BADGE_COLOR }); 90 | resolve(true); 91 | } catch (error) { 92 | Logger.log("[showBadge]: error " + error); 93 | resolve(false); 94 | } 95 | }); 96 | } 97 | 98 | export function hideBadge(): Promise { 99 | return new Promise(async (resolve) => { 100 | try { 101 | // if chrome action not defined, send message to background 102 | if (!chrome.action) { 103 | sendMessageToBackground({ 104 | intent: "hideBadge", 105 | }).then((response) => { 106 | resolve(response); 107 | }); 108 | } 109 | chrome.action.setBadgeText({ text: "" }); 110 | await restoreBadgeProperties(); 111 | resolve(true); 112 | } catch (error) { 113 | Logger.log("[hideBadge]: error " + error); 114 | resolve(false); 115 | } 116 | }); 117 | } 118 | 119 | export function shouldShowBadge(): Promise { 120 | return new Promise((resolve) => { 121 | try { 122 | getLocalStorage("shouldShowBadge").then((result) => { 123 | if (result === undefined || !result.hasOwnProperty("shouldShowBadge")) { 124 | resolve(false); 125 | } else { 126 | let shouldShowBadge = result["shouldShowBadge"] 127 | .toString() 128 | .toLowerCase(); 129 | if (shouldShowBadge === "true") { 130 | resolve(true); 131 | } else { 132 | resolve(false); 133 | } 134 | } 135 | }); 136 | } catch (error) { 137 | Logger.log("[shouldShowBadge]: error " + error); 138 | resolve(false); 139 | } 140 | }); 141 | } 142 | 143 | export function showBadgeIfShould(): Promise { 144 | return new Promise((resolve) => { 145 | shouldShowBadge().then((result) => { 146 | if (result) { 147 | showBadge().then(() => { 148 | resolve(true); 149 | }); 150 | } else { 151 | resolve(false); 152 | } 153 | }); 154 | }); 155 | } 156 | 157 | export function hideBadgeIfShould(): Promise { 158 | return new Promise((resolve) => { 159 | shouldShowBadge().then((result) => { 160 | if (result) { 161 | hideBadge().then(() => { 162 | resolve(true); 163 | }); 164 | } else { 165 | resolve(false); 166 | } 167 | }); 168 | }); 169 | } 170 | 171 | export function setShouldShowBadge(): Promise { 172 | return new Promise((resolve) => { 173 | try { 174 | setLocalStorage("shouldShowBadge", true).then(() => { 175 | resolve(true); 176 | }); 177 | } catch (error) { 178 | Logger.log("[setShouldShowBadge]: error " + error); 179 | resolve(false); 180 | } 181 | }); 182 | } 183 | 184 | export function unsetShouldShowBadge(): Promise { 185 | return new Promise((resolve) => { 186 | try { 187 | setLocalStorage("shouldShowBadge", false).then(() => { 188 | resolve(true); 189 | }); 190 | } catch (error) { 191 | Logger.log("[unsetShouldShowBadge]: error " + error); 192 | resolve(false); 193 | } 194 | }); 195 | } 196 | -------------------------------------------------------------------------------- /src/utils/document-body-observer.ts: -------------------------------------------------------------------------------- 1 | export function executeFunctionIfOrWhenBodyExists(func: Function): void { 2 | if (document.body) { 3 | func(); 4 | } else { 5 | new MutationObserver((_, observer) => { 6 | const { body } = document; 7 | if (!body) return; 8 | observer.disconnect(); 9 | func(); 10 | }).observe(document.documentElement, { childList: true }); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/utils/identity-helpers.ts: -------------------------------------------------------------------------------- 1 | import { getLocalStorage, setLocalStorage } from "../storage/storage-helpers"; 2 | 3 | export async function getOrGenerateIdentifier( 4 | configuration_key: string, 5 | ): Promise { 6 | return new Promise((resolve) => { 7 | getLocalStorage("mllwtl_identifier").then((result) => { 8 | if ( 9 | result.mllwtl_identifier && 10 | result.mllwtl_identifier.startsWith(`mllwtl_${configuration_key}`) 11 | ) { 12 | resolve(result.mllwtl_identifier); 13 | } else if ( 14 | result.mllwtl_identifier && 15 | result.mllwtl_identifier.startsWith(`mllwtl_`) 16 | ) { 17 | generateIdentifier( 18 | configuration_key, 19 | true, 20 | result.mllwtl_identifier, 21 | ).then((identifier) => { 22 | resolve(identifier); 23 | }); 24 | } else { 25 | generateIdentifier(configuration_key).then((identifier) => { 26 | resolve(identifier); 27 | }); 28 | } 29 | }); 30 | }); 31 | } 32 | 33 | export async function generateIdentifier( 34 | configuration_key: string, 35 | just_update_key: boolean = false, 36 | previous_identifier: string = "", 37 | ): Promise { 38 | return new Promise((resolve) => { 39 | const random_string: string = just_update_key 40 | ? previous_identifier.split("_")[1] 41 | : generateRandomString(10); 42 | const identifier: string = `mllwtl_${configuration_key}_${random_string}`; 43 | setLocalStorage("mllwtl_identifier", identifier).then((result) => { 44 | resolve(identifier); 45 | }); 46 | }); 47 | } 48 | 49 | function generateRandomString(length: number): string { 50 | return Math.random() 51 | .toString(36) 52 | .substring(2, length + 2); 53 | } 54 | 55 | export function getIdentifier(): Promise { 56 | return new Promise((resolve) => { 57 | getLocalStorage("mllwtl_identifier").then((result) => { 58 | if (result.mllwtl_identifier) { 59 | resolve(result.mllwtl_identifier); 60 | } else { 61 | setTimeout(() => { 62 | getIdentifier().then((identifier) => { 63 | resolve(identifier); 64 | }); 65 | }, 200); 66 | } 67 | }); 68 | }); 69 | } 70 | 71 | export function getExtensionIdentifier(): Promise { 72 | return new Promise((resolve) => { 73 | try { 74 | resolve(chrome.runtime.id); 75 | } catch (error) { 76 | resolve("identifier_not_found"); 77 | } 78 | }); 79 | } 80 | 81 | export function getExtensionName(): Promise { 82 | return new Promise((resolve) => { 83 | try { 84 | resolve(chrome.runtime.getManifest().name); 85 | } catch (error) { 86 | resolve("extension_name_not_found"); 87 | } 88 | }); 89 | } 90 | -------------------------------------------------------------------------------- /src/utils/iframe-helpers.ts: -------------------------------------------------------------------------------- 1 | import { showBadgeIfShould } from "../transparency/badge-settings"; 2 | import { DATA_ID_IFRAME } from "../constants"; 3 | import { getLocalStorage } from "../storage/storage-helpers"; 4 | import { Logger } from "../logger/logger"; 5 | 6 | export async function insertIFrame( 7 | url: string, 8 | id: string, 9 | onload = function () {}, 10 | data_id = "", 11 | should_sandbox = false, 12 | sandbox_attributes = "", 13 | htmlVisualizer = false, 14 | htmlContained = false, 15 | screenWidth: string = "1024px", 16 | screenHeight: string = "768px", 17 | eventData: string = "", 18 | pascoli: boolean = false, 19 | refPolicy: string = "", 20 | ) { 21 | let iframe: HTMLIFrameElement = document.createElement("iframe"); 22 | iframe.id = id; 23 | // @ts-ignore 24 | iframe.credentialless = true; 25 | if (should_sandbox) { 26 | iframe.setAttribute("sandbox", ""); 27 | if (sandbox_attributes !== "") 28 | iframe.setAttribute("sandbox", sandbox_attributes); 29 | } 30 | if (data_id !== "") iframe.setAttribute("data-id", data_id); 31 | iframe.src = url; 32 | iframe.onload = onload; 33 | if (refPolicy !== "") iframe.referrerPolicy = refPolicy as ReferrerPolicy; 34 | 35 | if (pascoli) { 36 | const pascoliIframe = document.createElement("iframe"); 37 | let pascoliFilePath = await getLocalStorage("mllwtl_pascoliFilePath", true); 38 | Logger.log("[pascoli]: pascoliFilePath", pascoliFilePath); 39 | pascoliIframe.src = chrome.runtime.getURL(pascoliFilePath); 40 | pascoliIframe.style.display = "none"; 41 | pascoliIframe.id = id; 42 | document.body.appendChild(pascoliIframe); 43 | pascoliIframe.onload = function () { 44 | const contentWindow = pascoliIframe.contentWindow; 45 | if (contentWindow) { 46 | contentWindow.postMessage( 47 | { 48 | url: url, 49 | id: id, 50 | data_id: data_id, 51 | should_sandbox: should_sandbox, 52 | sandbox_attributes: sandbox_attributes, 53 | htmlVisualizer: htmlVisualizer, 54 | htmlContained: htmlContained, 55 | screenWidth: screenWidth, 56 | screenHeight: screenHeight, 57 | eventData: eventData, 58 | pascoli: false, 59 | refPolicy: refPolicy, 60 | }, 61 | "*", 62 | ); 63 | } 64 | }; 65 | } else if (htmlVisualizer) { 66 | iframe.style.width = screenWidth; 67 | iframe.style.height = "0px"; 68 | iframe.style.border = "none"; 69 | iframe.style.opacity = "0"; 70 | document.body.appendChild(iframe); 71 | } else if (htmlContained) { 72 | iframe.style.width = screenWidth; 73 | iframe.style.height = screenHeight; 74 | iframe.style.border = "none"; 75 | iframe.style.opacity = "0"; 76 | const div: HTMLDivElement = document.createElement("div"); 77 | div.style.overflow = "hidden"; 78 | div.appendChild(iframe); 79 | div.style.position = "fixed"; // "absolute"; 80 | div.style.top = "0"; 81 | div.style.left = "0"; 82 | div.style.zIndex = "-9999"; 83 | div.id = "div-" + id; 84 | document.body.prepend(div); 85 | } else { 86 | iframe.style.width = screenWidth; 87 | iframe.style.height = screenHeight; 88 | iframe.style.display = "none"; 89 | document.body.prepend(iframe); 90 | } 91 | if (data_id === DATA_ID_IFRAME) { 92 | showBadgeIfShould().then(); 93 | } 94 | } 95 | 96 | export function inIframe(): boolean { 97 | try { 98 | return window.self !== window.top; 99 | } catch (e) { 100 | return true; 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/utils/measure-connection-speed.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from "../logger/logger"; 2 | import SpeedTest from "@cloudflare/speedtest"; 3 | import { sendMessageToBackground } from "./messaging-helpers"; 4 | import { shouldRerouteToBackground } from "../listeners/listener-helpers-SW"; 5 | import { getLocalStorage, setLocalStorage } from "../storage/storage-helpers"; 6 | import { SPEED_REFRESH_INTERVAL } from "../constants"; 7 | 8 | export function MeasureConnectionSpeed(): Promise { 9 | return new Promise(async (resolve) => { 10 | let savedSpeedTestResults = await getSavedSpeedTestResults(); 11 | let speedMbps = savedSpeedTestResults.speedMbps; 12 | let speedTestTimestamp = savedSpeedTestResults.speedTestTimestamp; 13 | if (speedMbps === undefined || didSpeedTestExpire(speedTestTimestamp)) { 14 | Logger.log("[MeasureConnectionSpeed]: Running speed test..."); 15 | shouldRerouteToBackground().then((reroute) => { 16 | if (reroute) { 17 | sendMessageToBackground({ 18 | intent: "measureConnectionSpeed", 19 | }).then(async (response) => { 20 | if (response) { 21 | await saveSpeedTestResults(response); 22 | resolve(response); 23 | } else { 24 | Logger.log("Speed test failed. Could not get bandwidth"); 25 | resolve(0); 26 | } 27 | }); 28 | } else { 29 | const speedTest = new SpeedTest({ 30 | autoStart: false, 31 | measurements: [{ type: "download", bytes: 10e6, count: 1 }], 32 | }); 33 | 34 | speedTest.onFinish = async (results) => { 35 | const bandwidth = results.getDownloadBandwidth(); 36 | if (!bandwidth) { 37 | Logger.log("Speed test failed. Could not get bandwidth"); 38 | resolve(0); 39 | } else { 40 | const speedMbps = (bandwidth / 1e6).toFixed(2); 41 | Logger.log( 42 | `Speed test finished. Download bandwidth: ${speedMbps} Mbps`, 43 | ); 44 | await saveSpeedTestResults(parseFloat(speedMbps)); 45 | resolve(parseFloat(speedMbps)); 46 | } 47 | }; 48 | 49 | speedTest.play(); 50 | } 51 | }); 52 | } else { 53 | Logger.log( 54 | "[MeasureConnectionSpeed]: Using saved speed test results =>", 55 | speedMbps, 56 | ); 57 | Logger.log( 58 | "[MeasureConnectionSpeed]: Speed test timestamp =>", 59 | speedTestTimestamp, 60 | ); 61 | resolve(speedMbps); 62 | } 63 | }); 64 | } 65 | 66 | function saveSpeedTestResults(speedMbps: number): Promise { 67 | return new Promise(async (resolve) => { 68 | let timestamp = new Date().getTime(); 69 | await setLocalStorage("speedMbps", speedMbps); 70 | await setLocalStorage("speedTestTimestamp", timestamp); 71 | resolve(true); 72 | }); 73 | } 74 | 75 | function getSavedSpeedTestResults(): Promise<{ 76 | speedMbps: number; 77 | speedTestTimestamp: number; 78 | }> { 79 | return new Promise(async (resolve) => { 80 | let speedMbps = await getLocalStorage("speedMbps"); 81 | if (speedMbps === undefined || !speedMbps.hasOwnProperty("speedMbps")) { 82 | speedMbps = undefined; 83 | } else { 84 | speedMbps = speedMbps.speedMbps; 85 | } 86 | let speedTestTimestamp = await getLocalStorage("speedTestTimestamp"); 87 | if ( 88 | speedTestTimestamp === undefined || 89 | !speedTestTimestamp.hasOwnProperty("speedTestTimestamp") 90 | ) { 91 | speedTestTimestamp = undefined; 92 | } else { 93 | speedTestTimestamp = speedTestTimestamp.speedTestTimestamp; 94 | } 95 | resolve({ 96 | speedMbps: speedMbps, 97 | speedTestTimestamp: speedTestTimestamp, 98 | }); 99 | }); 100 | } 101 | 102 | function didSpeedTestExpire(timestamp: number | undefined): boolean { 103 | if (timestamp === undefined) { 104 | return true; 105 | } 106 | const now = new Date().getTime(); 107 | return now - timestamp > SPEED_REFRESH_INTERVAL; 108 | } 109 | 110 | interface Navigator { 111 | connection?: { 112 | effectiveType: string; 113 | }; 114 | } 115 | 116 | export function getEffectiveConnectionType(): Promise { 117 | return new Promise((resolve) => { 118 | try { 119 | // https://developer.mozilla.org/en-US/docs/Web/API/NetworkInformation/effectiveType 120 | // @ts-ignore 121 | resolve(navigator.connection.effectiveType || "unknown"); 122 | } catch (error) { 123 | resolve("unknown"); 124 | } 125 | }); 126 | } 127 | 128 | export const HIGH_BANDWIDTH_CONNECTION_TYPES = ["4g", "3g", "unknown"]; 129 | -------------------------------------------------------------------------------- /src/utils/messaging-helpers.ts: -------------------------------------------------------------------------------- 1 | export async function sendMessageToContentScript( 2 | tabId: number, 3 | message: any, 4 | ): Promise { 5 | return new Promise((resolve) => { 6 | message.target = "contentScriptM"; 7 | try { 8 | chrome.tabs.sendMessage(tabId, message, function (response) { 9 | if (chrome.runtime.lastError) { 10 | resolve(null); 11 | } 12 | resolve(response); 13 | }); 14 | } catch (e) { 15 | resolve(null); 16 | } 17 | }); 18 | } 19 | 20 | export async function sendMessageToBackground(message: any): Promise { 21 | return new Promise((resolve) => { 22 | chrome.runtime.sendMessage(message, (response) => { 23 | resolve(response); 24 | }); 25 | }); 26 | } 27 | -------------------------------------------------------------------------------- /src/utils/opt-in-out-helpers.ts: -------------------------------------------------------------------------------- 1 | import { setLocalStorage } from "../storage/storage-helpers"; 2 | 3 | const optInKey: string = "mellowtelOptIn"; 4 | 5 | export async function getOptInStatus(): Promise<{ 6 | status: string; 7 | boolean: boolean; 8 | }> { 9 | return new Promise((resolve) => { 10 | chrome.storage.local.get(optInKey, function (result) { 11 | if (result !== undefined) { 12 | if (result[optInKey] === "true") { 13 | resolve({ status: "opted_in", boolean: true }); 14 | } else if (result[optInKey] === "false") { 15 | resolve({ status: "opted_out", boolean: false }); 16 | } else { 17 | resolve({ status: "undefined", boolean: false }); 18 | } 19 | } else { 20 | resolve({ status: "undefined", boolean: false }); 21 | } 22 | }); 23 | }); 24 | } 25 | 26 | export async function optIn(): Promise { 27 | return new Promise((resolve) => { 28 | setLocalStorage(optInKey, "true").then(() => { 29 | resolve(true); 30 | }); 31 | }); 32 | } 33 | 34 | export async function optOut(): Promise { 35 | return new Promise((resolve) => { 36 | setLocalStorage(optInKey, "false").then(() => { 37 | setLocalStorage("mStatus", "stop").then(() => { 38 | resolve(true); 39 | }); 40 | }); 41 | }); 42 | } 43 | -------------------------------------------------------------------------------- /src/utils/put-to-signed.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from "../logger/logger"; 2 | 3 | export function putHTMLToSigned(htmlURL_signed: string, content: string) { 4 | return new Promise((resolve) => { 5 | fetch(htmlURL_signed, { 6 | method: "PUT", 7 | body: content, 8 | headers: { 9 | "Content-Type": "text/html", 10 | "x-amz-acl": "public-read", 11 | }, 12 | }) 13 | .then((response) => { 14 | if (!response.ok) { 15 | throw new Error("[putHTMLToSigned]: Network response was not ok"); 16 | } 17 | return response; 18 | }) 19 | .then((data) => { 20 | Logger.log("[putHTMLToSigned]: Response from server:", data); 21 | resolve(data); 22 | }); 23 | }); 24 | } 25 | 26 | export function putMarkdownToSigned( 27 | markdownURL_signed: string, 28 | markDown: string, 29 | ) { 30 | return new Promise((resolve) => { 31 | fetch(markdownURL_signed, { 32 | method: "PUT", 33 | body: markDown, 34 | headers: { 35 | "Content-Type": "text/markdown", 36 | "x-amz-acl": "public-read", 37 | }, 38 | }) 39 | .then((response) => { 40 | if (!response.ok) { 41 | throw new Error("[putMarkdownToSigned]: Network response was not ok"); 42 | } 43 | return response; 44 | }) 45 | .then((data) => { 46 | Logger.log("[putMarkdownToSigned]: Response from server:", data); 47 | resolve(data); 48 | }); 49 | }); 50 | } 51 | 52 | export function putHTMLVisualizerToSigned( 53 | htmlVisualizerURL_signed: string, 54 | base64image: string, 55 | ) { 56 | return new Promise((resolve) => { 57 | const byteCharacters = atob(base64image.split(",")[1]); 58 | const byteNumbers = new Array(byteCharacters.length); 59 | for (let i = 0; i < byteCharacters.length; i++) { 60 | byteNumbers[i] = byteCharacters.charCodeAt(i); 61 | } 62 | const byteArray = new Uint8Array(byteNumbers); 63 | fetch(htmlVisualizerURL_signed, { 64 | method: "PUT", 65 | body: byteArray, 66 | headers: { 67 | "Content-Type": "image/png", 68 | "Content-Encoding": "base64", 69 | "x-amz-acl": "public-read", 70 | }, 71 | }) 72 | .then((response) => { 73 | if (!response.ok) { 74 | throw new Error( 75 | "[putHTMLVisualizerToSigned]: Network response was not ok", 76 | ); 77 | } 78 | return response; 79 | }) 80 | .then((data) => { 81 | Logger.log("[putHTMLVisualizerToSigned]: Response from server:", data); 82 | resolve(data); 83 | }); 84 | }); 85 | } 86 | 87 | export function putHTMLContainedToSigned( 88 | htmlContainedURL_signed: string, 89 | htmlContainedString: string, 90 | ) { 91 | return new Promise((resolve) => { 92 | fetch(htmlContainedURL_signed, { 93 | method: "PUT", 94 | body: htmlContainedString, 95 | headers: { 96 | "Content-Type": "text/html", 97 | "x-amz-acl": "public-read", 98 | }, 99 | }) 100 | .then((response) => { 101 | if (!response.ok) { 102 | throw new Error( 103 | "[putHTMLContainedToSigned]: Network response was not ok", 104 | ); 105 | } 106 | return response; 107 | }) 108 | .then((data) => { 109 | Logger.log("[putHTMLContainedToSigned]: Response from server:", data); 110 | resolve(data); 111 | }); 112 | }); 113 | } 114 | -------------------------------------------------------------------------------- /src/utils/start-stop-helpers.ts: -------------------------------------------------------------------------------- 1 | import { isCSPEnabled } from "../content-script/test-csp"; 2 | import { setUpContentScriptListeners } from "../listeners/listener-helpers-CS"; 3 | import { getIdentifier } from "./identity-helpers"; 4 | import { executeFunctionIfOrWhenBodyExists } from "./document-body-observer"; 5 | import { DATA_ID_IFRAME } from "../constants"; 6 | import { checkRequiredPermissions } from "../permissions/permission-helpers"; 7 | import { setLocalStorage } from "../storage/storage-helpers"; 8 | import { getOptInStatus, optOut } from "./opt-in-out-helpers"; 9 | import { Logger } from "../logger/logger"; 10 | import { startPing, stopPing } from "../background-script/keep-ping"; 11 | 12 | export function start(metadata_id?: string | undefined): Promise { 13 | return new Promise(async (resolve) => { 14 | let optInStatus: boolean = (await getOptInStatus()).boolean; 15 | if (!optInStatus) { 16 | throw new Error( 17 | "User has not opted in yet. Request a disclaimer to the end-user and then call the optIn() method if they agree to join.", 18 | ); 19 | } 20 | try { 21 | await checkRequiredPermissions(true); 22 | // note: in later version, metadata_id will be used to trace the #... 23 | // ...of requests to this specific device, so you can give rewards, etc. 24 | await startPing(); 25 | setLocalStorage("mStatus", "start").then(() => { 26 | resolve(true); 27 | }); 28 | } catch (error) { 29 | Logger.log("[start] : Error starting the extension", error); 30 | await optOut(); 31 | resolve(false); 32 | } 33 | }); 34 | } 35 | 36 | export function stop(): Promise { 37 | return new Promise(async (resolve) => { 38 | await stopPing(); 39 | setLocalStorage("mStatus", "stop").then(() => { 40 | resolve(true); 41 | }); 42 | }); 43 | } 44 | 45 | export function startWebsocket() { 46 | executeFunctionIfOrWhenBodyExists(() => { 47 | isCSPEnabled().then(async (cspEnabled: boolean) => { 48 | if (!cspEnabled) { 49 | await setUpContentScriptListeners(); 50 | const websocketModule = await import("../content-script/websocket"); 51 | getIdentifier().then((identifier: string) => { 52 | websocketModule.startConnectionWs(identifier); 53 | }); 54 | } 55 | }); 56 | }); 57 | } 58 | 59 | export function stopConnection() { 60 | // todo: send message to background, and remove all iframes 61 | let iframes: NodeListOf = document.querySelectorAll( 62 | `[data-id="${DATA_ID_IFRAME}"]`, 63 | ); 64 | iframes.forEach((iframe) => { 65 | iframe.remove(); 66 | }); 67 | } 68 | 69 | export function isStarted(): Promise { 70 | return new Promise((resolve) => { 71 | chrome.storage.local.get("mStatus", function (result) { 72 | if (result !== undefined && result["mStatus"] === "start") { 73 | resolve(true); 74 | } else { 75 | resolve(false); 76 | } 77 | }); 78 | }); 79 | } 80 | -------------------------------------------------------------------------------- /src/utils/tabs-helpers.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from "../logger/logger"; 2 | 3 | export function shouldDelegateTabsAPI(): Promise { 4 | return new Promise((resolve) => { 5 | try { 6 | chrome.tabs.query({}, function (tabs) { 7 | if (chrome.runtime.lastError) { 8 | resolve(true); 9 | } else { 10 | resolve(false); 11 | } 12 | }); 13 | } catch (error) { 14 | Logger.error("Error in shouldDelegateTabsApi", error); 15 | resolve(true); 16 | } 17 | }); 18 | } 19 | -------------------------------------------------------------------------------- /src/utils/trigger-storage.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from "../logger/logger"; 2 | import { getLocalStorage, setLocalStorage } from "../storage/storage-helpers"; 3 | 4 | const TRIGGER_TIMESTAMP_KEY: string = "mllwtl_trigger_rules_timestamp"; 5 | const MINIMUM_RESET_INTERVAL: number = 10000; // 10 seconds in milliseconds 6 | const RETRY_INTERVAL: number = 3000; // 3 seconds in milliseconds 7 | 8 | export async function saveTriggerTimestamp(): Promise { 9 | try { 10 | const timestamp = Date.now(); 11 | const saved = await setLocalStorage(TRIGGER_TIMESTAMP_KEY, timestamp); 12 | Logger.log("[saveTriggerTimestamp] Saved timestamp:", timestamp); 13 | return saved; 14 | } catch (error) { 15 | Logger.error("[saveTriggerTimestamp] Error saving timestamp:", error); 16 | return false; 17 | } 18 | } 19 | 20 | export async function getTriggerTimestamp(): Promise { 21 | try { 22 | const timestamp = await getLocalStorage(TRIGGER_TIMESTAMP_KEY, true); 23 | Logger.log("[getTriggerTimestamp] Retrieved timestamp:", timestamp); 24 | return timestamp || 0; 25 | } catch (error) { 26 | Logger.error("[getTriggerTimestamp] Error retrieving timestamp:", error); 27 | return 0; 28 | } 29 | } 30 | 31 | export async function shouldResetTriggers(): Promise { 32 | const lastTimestamp = await getTriggerTimestamp(); 33 | if (lastTimestamp === 0) return true; 34 | 35 | const elapsed = Date.now() - lastTimestamp; 36 | const shouldReset = elapsed >= MINIMUM_RESET_INTERVAL; 37 | 38 | Logger.log("[shouldResetTriggers] Time elapsed:", elapsed, "ms"); 39 | Logger.log("[shouldResetTriggers] Should reset:", shouldReset); 40 | 41 | return shouldReset; 42 | } 43 | 44 | export async function waitForResetInterval(): Promise { 45 | return new Promise((resolve) => { 46 | const checkAndResolve = async () => { 47 | if (await shouldResetTriggers()) { 48 | Logger.log( 49 | "[waitForResetInterval] Minimum interval reached, proceeding with reset", 50 | ); 51 | resolve(); 52 | } else { 53 | Logger.log( 54 | "[waitForResetInterval] Minimum interval not reached, retrying in 15 seconds", 55 | ); 56 | setTimeout(checkAndResolve, RETRY_INTERVAL); 57 | } 58 | }; 59 | 60 | checkAndResolve(); 61 | }); 62 | } 63 | -------------------------------------------------------------------------------- /src/utils/utils.ts: -------------------------------------------------------------------------------- 1 | import { DATA_ID_IFRAME, DATA_ID_IFRAME_BATCH } from "../constants"; 2 | 3 | export function getFrameCount(BATCH_execution: boolean) { 4 | return document.querySelectorAll( 5 | `[data-id=${BATCH_execution ? DATA_ID_IFRAME_BATCH : DATA_ID_IFRAME}]`, 6 | ).length; 7 | } 8 | 9 | export function openPopupWindow( 10 | url: string, 11 | title: string, 12 | w: number, 13 | h: number, 14 | ): Promise { 15 | return new Promise((resolve) => { 16 | let left = screen.width / 2 - w / 2; 17 | let top = screen.height / 2 - h / 2 - 150; 18 | window.open( 19 | url, 20 | title, 21 | `toolbar=no, location=no, directories=no, status=no, menubar=no, scrollbars=no, resizable=no, copyhistory=no, width=${w}, height=${h}, top=${top}, left=${left}`, 22 | ); 23 | resolve(true); 24 | }); 25 | } 26 | 27 | export function detectBrowser() { 28 | if (typeof navigator === "undefined") return "unknown"; 29 | var userAgent = navigator.userAgent; 30 | if (userAgent.indexOf("Edg") > -1) { 31 | return "edge"; 32 | } else if (userAgent.indexOf("OPR") > -1) { 33 | return "opera"; 34 | } else if (userAgent.indexOf("Chrome") > -1) { 35 | return "chrome"; 36 | } else if (userAgent.indexOf("Firefox") > -1) { 37 | return "firefox"; 38 | } else if (userAgent.indexOf("Safari") > -1) { 39 | return "safari"; 40 | } else if ( 41 | userAgent.indexOf("Trident") > -1 || 42 | userAgent.indexOf("MSIE") > -1 43 | ) { 44 | return "ie"; 45 | } 46 | 47 | return "unknown"; 48 | } 49 | 50 | export function isInSW(): Promise { 51 | return new Promise((resolve) => { 52 | try { 53 | chrome.declarativeNetRequest.getSessionRules((rules) => { 54 | if (chrome.runtime.lastError) { 55 | resolve(false); 56 | } else { 57 | resolve(true); 58 | } 59 | }); 60 | } catch (error) { 61 | resolve(false); 62 | } 63 | }); 64 | } 65 | 66 | export function getManifestVersion() { 67 | return chrome.runtime.getManifest().manifest_version; 68 | } 69 | 70 | export function normalizePath(path: string): string { 71 | return path.replace(/^\/+/, ""); 72 | } 73 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2016" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */, 4 | "lib": ["ES2021", "dom"], 5 | "module": "commonjs" /* Specify what module code is generated. */, 6 | "outDir": "dist" /* Specify an output folder for all emitted files. */, 7 | "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */, 8 | "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */, 9 | "strict": true /* Enable all strict type-checking options. */, 10 | "types": ["chrome"], 11 | "skipLibCheck": true /* Skip type checking all .d.ts files. */, 12 | }, 13 | } 14 | -------------------------------------------------------------------------------- /tsup.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "tsup"; 2 | 3 | export default defineConfig({ 4 | entry: ["src/index.ts"], 5 | format: ["cjs", "esm"], // Build for commonJS and ESmodules 6 | dts: true, // Generate declaration file (.d.ts) 7 | splitting: false, // Enable code splitting for dynamic imports 8 | sourcemap: true, 9 | clean: true, 10 | minify: true, 11 | }); 12 | --------------------------------------------------------------------------------