├── .gitignore ├── .vscode └── settings.json ├── LICENSE ├── README.md ├── build.js ├── docs ├── .nojekyll ├── README.md ├── _sidebar.md ├── add-scraper.md ├── codebase.md ├── css │ └── custom.css ├── devenv.md ├── examples.md ├── examples │ ├── _images │ │ ├── airbnb │ │ │ ├── annotations.png │ │ │ ├── ranked_by_price.png │ │ │ ├── ranked_by_rating.png │ │ │ ├── wildcard_closed.png │ │ │ └── wildcard_open.png │ │ ├── amazon │ │ │ ├── ascended_sort.png │ │ │ ├── cheap_item.png │ │ │ ├── condition_sort_vid.gif │ │ │ ├── condition_sort_vid.mp4 │ │ │ ├── descended_sort.png │ │ │ ├── price_sort_vid.gif │ │ │ ├── price_sort_vid.mp4 │ │ │ ├── select_cell_highlighted.png │ │ │ ├── wildcard_closed.png │ │ │ └── wildcard_open.png │ │ ├── blogger │ │ │ ├── edited_html.png │ │ │ ├── edited_text.png │ │ │ ├── wildcard_closed.png │ │ │ └── wildcard_open.png │ │ ├── hackernews │ │ │ ├── annotation_vid.gif │ │ │ ├── annotation_vid.mp4 │ │ │ ├── annotations.png │ │ │ ├── ranked_by_comments.png │ │ │ ├── ranked_by_points.png │ │ │ ├── wildcard_closed.png │ │ │ └── wildcard_open.png │ │ ├── weather │ │ │ ├── warmest.png │ │ │ ├── wildcard_closed.png │ │ │ └── wildcard_open.png │ │ └── youtube │ │ │ ├── ranked_by_watch_time.png │ │ │ └── wildcard_open.png │ ├── airbnb.md │ ├── amazon.md │ ├── blogger.md │ ├── hackernews.md │ ├── template.md │ ├── ubereats.md │ ├── weather.md │ └── youtube.md ├── index.html └── quickstart.md ├── manifest.json ├── options.html ├── options.js ├── package-lock.json ├── package.json ├── readme-resources └── architecture-v02.png ├── resources └── adapter-configs │ ├── eecscoursecatalog.json │ ├── github.json │ ├── hackernews.json │ ├── mitcoursecatalog.json │ └── youtube.json ├── src ├── core │ ├── actions.ts │ ├── debug.tsx │ ├── getFinalTable.ts │ ├── reducer.ts │ └── types.ts ├── end_user_scraper │ ├── adapterHelpers.ts │ ├── constants.ts │ ├── domHelpers.ts │ ├── eventListeners.ts │ ├── generalizer.ts │ ├── index.ts │ ├── state.ts │ ├── tutorial.ts │ └── utils.ts ├── formula.ts ├── localStorageAdapter.ts ├── marketplace.js ├── site_adapters │ ├── airbnb.ts │ ├── amazon.ts │ ├── blogger.ts │ ├── domScrapingBase.ts │ ├── expedia.ts │ ├── flux.ts │ ├── github.ts │ ├── hackerNews.ts │ ├── harvardbookwarehouse.ts │ ├── index.ts │ ├── instacart.ts │ ├── ubereats.ts │ ├── weatherchannel.ts │ └── youtube.ts ├── tableAdapterMiddleware.ts ├── ui │ ├── AutosuggestInput.tsx │ ├── WcPanel.tsx │ ├── cell_editors │ │ ├── formulaEditor.js │ │ ├── fullCalendarEditor.css │ │ ├── fullCalendarEditor.js │ │ ├── richTextEditor.css │ │ └── richTextEditor.js │ └── overrides.css ├── utils.ts ├── wildcard-ajax.js ├── wildcard-background.ts └── wildcard.tsx ├── tsconfig.json ├── vendor ├── ace │ ├── ace.js │ ├── mode-typescript.js │ └── theme-monokai.js ├── bootstrap │ ├── css │ │ └── bootstrap.min.css │ └── js │ │ └── bootstrap.min.js └── jquery.js └── yarn.lock /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/* 2 | **/.DS_Store 3 | *.sublime-project 4 | *.sublime-workspace 5 | .idea/ 6 | dist/* 7 | .vscode/* 8 | yarn-error.log 9 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Geoffrey Litt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Wildcard 2 | 3 | Wildcard is a platform that empowers anyone to build browser extensions and modify websites to meet their own specific needs. 4 | 5 | Wildcard shows a simplified view of the data in a web page as a familiar table view. People can directly manipulate the table to sort/filter content, add annotations, and even use spreadsheet-style formulas to pull in data from other websites. 6 | 7 | * [Documentation](https://geoffreylitt.github.io/wildcard) 8 | * [Project homepage](https://www.geoffreylitt.com/wildcard/) 9 | * [Research paper](https://www.geoffreylitt.com/wildcard/salon2020/) presented at the Convivial Computing Salon 2020 10 | 11 | ## Install Wildcard 12 | 13 | Be aware: Wildcard is still pre-release software. 14 | 15 | **The current master branch is stable, but missing a few important features 16 | that have yet to be ported over from previous versions of the codebase: mainly filtering the table and formulas.** 17 | 18 | To get an email when a full featured version is ready, [sign up for the email newsletter](https://tinyletter.com/wildcard-extension). 19 | 20 | If you want to install Wildcard today, there are two options: 21 | 22 | To use Wildcard with an existing library of supported websites, follow the [quick start instructions](https://geoffreylitt.github.io/wildcard/#/quickstart). 23 | 24 | ## Contribute 25 | 26 | Follow the [dev env install instructions](https://geoffreylitt.github.io/wildcard/#/devenv). 27 | 28 | To run the build watcher: `yarn run dev` 29 | 30 | To run the extension in a new browser and auto-reload it when you update the code: `yarn run chrome` or `yarn run firefox` -------------------------------------------------------------------------------- /build.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | // 3 | // This example builds a module in both debug and release mode. 4 | // See estrella.d.ts for documentation of available options. 5 | // You can also pass any options for esbuild (as defined in esbuild/lib/main.d.ts). 6 | // 7 | const { build, cliopts } = require("estrella") 8 | const webExt = require("web-ext") 9 | 10 | build({ 11 | entry: "src/wildcard.tsx", 12 | outfile: "dist/wildcard.js", 13 | bundle: true, 14 | sourcemap: true, 15 | define: { 16 | "process.env.NODE_ENV": '"production"' 17 | }, 18 | minify: false 19 | }) 20 | 21 | build({ 22 | entry: "src/wildcard-background.ts", 23 | outfile: "dist/wildcard-background.js", 24 | bundle: true, 25 | sourcemap: true, 26 | define: { 27 | "process.env.NODE_ENV": '"production"' 28 | }, 29 | }) 30 | 31 | 32 | build({ 33 | entry: "src/marketplace.js", 34 | outfile: "dist/marketplace.js", 35 | bundle: true, 36 | sourcemap: true, 37 | define: { 38 | "process.env.NODE_ENV": '"production"' 39 | }, 40 | }) -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/.nojekyll -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Wildcard 2 | 3 | Wildcard is a platform that empowers anyone to build browser extensions and modify websites to meet their own specific needs. 4 | 5 | Wildcard shows a simplified view of the data in a web page as a familiar table view. People can directly manipulate the table to sort/filter content, add annotations, and even use spreadsheet-style formulas to pull in data from other websites. 6 | 7 | For more details, see the [paper](https://www.geoffreylitt.com/wildcard/salon2020/) presented at the Convivial Computing Salon 2020. 8 | 9 | ## Get started 10 | 11 | * Read the [quick start](quickstart.md) for installation instructions 12 | * View the [example gallery](examples.md) to see some things you can do with Wildcard 13 | 14 | ## Customize 15 | 16 | If you know how to program in Javascript, you can write little bits of scraping code to make Wildcard work with any website. See [the instructions](add-scraper.md) for more details. 17 | 18 | ## Stay up to date 19 | 20 | [Sign up for the email newsletter](https://forms.gle/mpn1Hn8Ln7dmPo6T8) to get occasional updates about the project. 21 | -------------------------------------------------------------------------------- /docs/_sidebar.md: -------------------------------------------------------------------------------- 1 | 2 | - [Home](README.md) 3 | - Getting started 4 | 5 | - [Quick start](quickstart.md) 6 | - [Example gallery](examples.md) 7 | - [Amazon](examples/amazon.md) 8 | - [AirBnB](examples/airbnb.md) 9 | - [Blogger](examples/blogger.md) 10 | - [Hackernews](examples/hackernews.md) 11 | - [Weather](examples/weather.md) 12 | - [Youtube](examples/youtube.md) 13 | 14 | 15 | - Customize 16 | - [Set up dev environment](devenv.md) 17 | - [Adding a site adapter](add-scraper.md) 18 | - [Understanding the codebase](codebase.md) 19 | -------------------------------------------------------------------------------- /docs/add-scraper.md: -------------------------------------------------------------------------------- 1 | # Adding a site adapter 2 | 3 | !> These are very preliminary docs. If you get stuck or run into questions/issues, file a Github issue on this repo or [reach out over email](mailto:glitt@mit.edu). 4 | 5 | ## Steps 6 | 7 | To use a new website in Wildcard you need to write a "site adapter", 8 | which is a bit of Javascript that scrapes data from a web page. 9 | 10 | First, install a [development environment](https://geoffreylitt.github.io/wildcard/#/devenv) 11 | and run `yarn run dev` to get changes automatically compiling. 12 | 13 | 1. Copy one of the existing site adapters in `src/site_adapters` as an example template. 14 | 2. Change the name of the adapter to your new adapter name. 15 | 3. Register the new copied adapter in `src/site_adapters/index.ts`. 16 | 4. Make changes to the copied adapter to make it work for your site. See if you can get a table of data to show up. Using the dev tools DOM inspector and adding `console.log` statements to your adapter might help you get through the scraping logic. 17 | 18 | Here's [an example commit](https://github.com/geoffreylitt/wildcard/commit/42fbb748a809aa84b7f6927a9aac02376f5bb926) of adding a site adapter. Your commit should look something like this one. 19 | 20 | ## Video tutorial 21 | 22 | Here's a 30 minute video tutorial of how to make a site adapter, in detail: 23 | 24 |
25 | -------------------------------------------------------------------------------- /docs/codebase.md: -------------------------------------------------------------------------------- 1 | # Understanding the codebase 2 | 3 | !> This doc is only intended for people who want to hack on 4 | the internals of Wildcard. If you just want to add a site adapter 5 | to scrape a specific site, see [Adding a scraper](add-scraper.md) 6 | 7 | ## Background 8 | 9 | It helps to understand the basic ideas of React + Redux to understand this codebase. Some good introductory reads: 10 | 11 | * [Redux core concepts](https://redux.js.org/introduction/core-concepts) 12 | * [Redux data flow](https://redux.js.org/basics/data-flow) 13 | * [Thinking in React](https://reactjs.org/docs/thinking-in-react.html) 14 | 15 | The code is split into three main modules, each with their own directory inside `src`. Here's a quick overview of the contents of each module. 16 | 17 | ![](readme-resources/architecture-v02.png) 18 | 19 | ## Core 20 | 21 | Maintains system state. Defines Redux actions and reducers. 22 | 23 | The root state object looks something like this. 24 | 25 | ```ts 26 | { 27 | // Latest snapshot of data extracted from the original site. 28 | appTable: { 29 | attributes: [...], // columns of the table 30 | records: [...], // rows of the table 31 | }, 32 | // All the user's annotations and formulas associated with this site 33 | userTable: { 34 | attributes: [...], 35 | records: [...], 36 | }, 37 | // Configuration for the current "query" being shown: 38 | // * sorting metadata 39 | // * filtering metadata (not added yet) 40 | query: { 41 | sortConfig: { 42 | attribute: "name", 43 | direction: "asc" 44 | } 45 | } 46 | } 47 | ``` 48 | 49 | ### Assembling query views 50 | 51 | Note how the app's data and user's data are stored independently. 52 | Before we can display it in the table UI, we need to combine it together. 53 | 54 | The core contains logic for executing "queries" that 55 | combine data extracted from the original site with 56 | data maintained by the user, and sort/filter the result. 57 | The query results aren't stored anywhere; they get recomputed 58 | every time the underlying state changes in any way. This makes it trivial 59 | to show correct data in the table UI whenever new data comes in, 60 | either from the site adapter or the user. 61 | 62 | ### Actions 63 | 64 | Per the Redux pattern, system state is only modified through specified actions, 65 | which come from the site adapter or the UI. 66 | 67 | For example, the site adapter can dispatch a "load records" action when new 68 | data is available, or the UI can dispatch a "sort table" action. 69 | 70 | See `src/core/actions.ts` for a full list of actions. 71 | 72 | ## Site adapters 73 | 74 | A Wildcard site adapter connects a specific site/application to Wildcard. 75 | All site adapters must fulfill this abstract interface, which boils down to: 76 | 77 | * Read data from the site, updating when necessary 78 | * When data updates (eg user sorts or adds annotations), reify changes in the site 79 | 80 | ```ts 81 | export interface SiteAdapter { 82 | // ===================== 83 | // Reading data from the site 84 | // ===================== 85 | 86 | /** Return latest data from the site */ 87 | loadRecords():Array; 88 | 89 | /** Register a callback function which will be called with a new table 90 | * of data anytime the data changes. */ 91 | subscribe (callback:(table:Table) => void):void; 92 | 93 | // ===================== 94 | // Modifying the site UI 95 | // ===================== 96 | 97 | /** Apply a new sort order to the UI */ 98 | applySort(finalRecords:Array, sortConfig:SortConfig):void; 99 | 100 | /** Apply a new annotation to the UI */ 101 | annotateRecordInSite(id:id, newValues:any, userAttributes:Array):void; 102 | 103 | // I'm considering replacing the two functions above with a generalized 104 | // version that can apply arbitrary table state to the UI: 105 | 106 | /** Update the UI to match arbitrary table state 107 | * (To implement performantly, probably do a diff inside the adapter 108 | * and only update the UI where necessary) */ 109 | update?(table:Table):void; 110 | } 111 | ``` 112 | 113 | Currently there's one specific type of site adapter used in the system: 114 | DOM Scraping adapters, which scrape the website DOM to fetch data, 115 | and manipulate the DOM to update the page. 116 | 117 | `DomScrapingBaseAdapter` is an abstract base class that new adapters 118 | can inherit from to implement DOM scraping. A concrete DOM scraping adapter 119 | only needs to implement one main function that scrapes data, 120 | and the base class takes care of the rest— 121 | see `src/site_adapters/newHN.ts` for one example of a concrete scraping adapter. 122 | 123 | In the future we anticipate having other categories of site adapters, 124 | which will fulfill the same abstract interface with different techniques: 125 | 126 | * AJAX Scraping: scrape data out of AJAX JSON requests 127 | * Redux adapter: extract data directly out of the application's internal Redux store 128 | 129 | ## UI 130 | 131 | A Wildcard UI has these responsibilities: 132 | 133 | * Display the table resulting from a query 134 | * Dispatch semantic events based on user interactions 135 | 136 | Currently there's just one UI, the `WcPanel` React Component that is a stateless table editor 137 | component built on Handsontable. (`src/ui/WcPanel.tsx`) 138 | 139 | The most important thing to realize is that the UI is **stateless**. 140 | It is a pure function of the Redux state, and displays the "query view" 141 | computed by the Core. All mutation happens by triggering Redux events. 142 | This makes the UI have minimal responsibility, and allows for future 143 | other UIs that view the tabular data in different ways. 144 | 145 | `src/wildcard.ts` is the final file that pulls everything together and 146 | injects Wildcard into the page. 147 | 148 | 149 | -------------------------------------------------------------------------------- /docs/css/custom.css: -------------------------------------------------------------------------------- 1 | .gallery-wrapper{ 2 | display: grid; 3 | grid-template-columns: repeat(2, 1fr); 4 | grid-template-rows: repeat(3, 250px); 5 | grid-gap: .8rem; 6 | } 7 | 8 | .imageTarget{ 9 | width: 350px; 10 | height: 200px; 11 | border: 2px solid #42B983; 12 | } 13 | 14 | 15 | .w3-display-container{ 16 | max-width:800px; 17 | } 18 | 19 | .galleryImages{ 20 | display: none; 21 | width:100%; 22 | } 23 | 24 | figcaption{ 25 | display: none; 26 | text-align: center; 27 | padding: 0.5em; 28 | font-weight: bold; 29 | } 30 | 31 | .w3-left, .w3-right, .w3-badge { 32 | cursor:pointer; 33 | color: #42B983; 34 | } 35 | 36 | .w3-left, .w3-right{ 37 | font-size: 60px; 38 | opacity: .8; 39 | } 40 | 41 | .w3-left:hover, .w3-right:hover{ 42 | color: teal; 43 | } 44 | 45 | .w3-badge { 46 | height:13px; 47 | width:13px; 48 | padding:0; 49 | } -------------------------------------------------------------------------------- /docs/devenv.md: -------------------------------------------------------------------------------- 1 | # Development environment setup 2 | 3 | ## Initial setup 4 | 5 | To install Wildcard in a way that supports adding new site adapters, 6 | or making other changes to the framework, 7 | you'll need to set up the local development environment. Wildcard is built in Typescript and uses yarn and rollup for packages and bundling. 8 | 9 | !> If you don't want to add new scrapers to Wildcard, you can follow the simpler [quick start](quickstart.md) instructions instead. 10 | 11 | Clone the Github repo: `git clone git@github.com:geoffreylitt/wildcard.git` 12 | 13 | Install dependencies : 14 | 15 | * `yarn install` (or `npm install`) 16 | * `yarn global add docsify` 17 | 18 | Build the project: `yarn run dev`. This will compile the Typescript project and create files in the `./dist` folder. 19 | 20 | There are two ways you can install the extension. 21 | 22 | ### Install in your main browser 23 | 24 | Follow [the quick start instructions](quickstart.md) to install the built project directory in your browser. 25 | 26 | This is nice if you want to use Wildcard while you actually browse the web. 27 | 28 | But, it's not great for development, because it's hard to reload the extension automatically when you make changes. For dev, there's a better way: 29 | 30 | ### Run through web-ext in development 31 | 32 | You can run a special browser which will automatically reload the extension when you make changes. Run `yarn run chrome` to start this up. 33 | 34 | To test if you're able to make changes, try adding a `console.log` statement to `src/wildcard.tsx` and see if it shows up in your browser. 35 | 36 | In general while developing you want to run `yarn run dev` and `yarn run chrome` in separate terminal tabs. 37 | 38 | ## Documentation 39 | 40 | This documentation page is built with [Docsify](https://docsify.js.org/). 41 | 42 | To edit docs, edit markdown files in `./docs`. 43 | 44 | To view docs locally, run `yarn run docs`. 45 | 46 | ## To cut a release 47 | 48 | * create a new folder named `wildcard-` 49 | * copy the current dist folder and manifest.json to the new folder 50 | * zip the folder. Create a new release in the Github UI and upload the zip file. 51 | 52 | (In the future we should automate this) 53 | -------------------------------------------------------------------------------- /docs/examples.md: -------------------------------------------------------------------------------- 1 | # Example gallery 2 | 3 | Wildcard provides a wide variety of flexibility and personalization for users. With Wildcard, you can sort elements on a page based on a metric such as price or rating, import your favorite rich text editor, or make annotations on items when shopping. 4 | 5 | Here are some examples of how Wildcard in use on some websites: 6 | 7 |
8 | 9 |
10 | 11 | #### [1. Amazon: Tally and sort prices](examples/amazon.md) 12 |
13 | 14 | 15 | 16 |
17 |
18 | 19 | 20 |
21 | 22 | #### [2. AirBnB: Annotate listings](examples/airbnb.md) 23 |
24 | 25 | 26 | 27 |
28 |
29 | 30 | 31 |
32 | 33 | #### [3. Blogger: Import rich text editors](examples/blogger.md) 34 |
35 | 36 | 37 | 38 |
39 |
40 | 41 | 42 |
43 | 44 | #### [4. Hackernews: Find popular news](examples/hackernews.md) 45 |
46 | 47 | 48 | 49 |
50 |
51 | 52 | 53 |
54 | 55 | #### [5. Weather: Find the best weather](examples/weather.md) 56 |
57 | 58 | 59 | 60 |
61 |
62 | 63 | 64 |
65 | 66 | #### [6. Youtube: Sort videos by watch time](examples/youtube.md) 67 |
68 | 69 | 70 | 71 |
72 |
73 | 74 |
-------------------------------------------------------------------------------- /docs/examples/_images/airbnb/annotations.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/airbnb/annotations.png -------------------------------------------------------------------------------- /docs/examples/_images/airbnb/ranked_by_price.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/airbnb/ranked_by_price.png -------------------------------------------------------------------------------- /docs/examples/_images/airbnb/ranked_by_rating.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/airbnb/ranked_by_rating.png -------------------------------------------------------------------------------- /docs/examples/_images/airbnb/wildcard_closed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/airbnb/wildcard_closed.png -------------------------------------------------------------------------------- /docs/examples/_images/airbnb/wildcard_open.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/airbnb/wildcard_open.png -------------------------------------------------------------------------------- /docs/examples/_images/amazon/ascended_sort.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/amazon/ascended_sort.png -------------------------------------------------------------------------------- /docs/examples/_images/amazon/cheap_item.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/amazon/cheap_item.png -------------------------------------------------------------------------------- /docs/examples/_images/amazon/condition_sort_vid.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/amazon/condition_sort_vid.gif -------------------------------------------------------------------------------- /docs/examples/_images/amazon/condition_sort_vid.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/amazon/condition_sort_vid.mp4 -------------------------------------------------------------------------------- /docs/examples/_images/amazon/descended_sort.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/amazon/descended_sort.png -------------------------------------------------------------------------------- /docs/examples/_images/amazon/price_sort_vid.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/amazon/price_sort_vid.gif -------------------------------------------------------------------------------- /docs/examples/_images/amazon/price_sort_vid.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/amazon/price_sort_vid.mp4 -------------------------------------------------------------------------------- /docs/examples/_images/amazon/select_cell_highlighted.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/amazon/select_cell_highlighted.png -------------------------------------------------------------------------------- /docs/examples/_images/amazon/wildcard_closed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/amazon/wildcard_closed.png -------------------------------------------------------------------------------- /docs/examples/_images/amazon/wildcard_open.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/amazon/wildcard_open.png -------------------------------------------------------------------------------- /docs/examples/_images/blogger/edited_html.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/blogger/edited_html.png -------------------------------------------------------------------------------- /docs/examples/_images/blogger/edited_text.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/blogger/edited_text.png -------------------------------------------------------------------------------- /docs/examples/_images/blogger/wildcard_closed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/blogger/wildcard_closed.png -------------------------------------------------------------------------------- /docs/examples/_images/blogger/wildcard_open.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/blogger/wildcard_open.png -------------------------------------------------------------------------------- /docs/examples/_images/hackernews/annotation_vid.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/hackernews/annotation_vid.gif -------------------------------------------------------------------------------- /docs/examples/_images/hackernews/annotation_vid.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/hackernews/annotation_vid.mp4 -------------------------------------------------------------------------------- /docs/examples/_images/hackernews/annotations.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/hackernews/annotations.png -------------------------------------------------------------------------------- /docs/examples/_images/hackernews/ranked_by_comments.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/hackernews/ranked_by_comments.png -------------------------------------------------------------------------------- /docs/examples/_images/hackernews/ranked_by_points.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/hackernews/ranked_by_points.png -------------------------------------------------------------------------------- /docs/examples/_images/hackernews/wildcard_closed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/hackernews/wildcard_closed.png -------------------------------------------------------------------------------- /docs/examples/_images/hackernews/wildcard_open.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/hackernews/wildcard_open.png -------------------------------------------------------------------------------- /docs/examples/_images/weather/warmest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/weather/warmest.png -------------------------------------------------------------------------------- /docs/examples/_images/weather/wildcard_closed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/weather/wildcard_closed.png -------------------------------------------------------------------------------- /docs/examples/_images/weather/wildcard_open.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/weather/wildcard_open.png -------------------------------------------------------------------------------- /docs/examples/_images/youtube/ranked_by_watch_time.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/youtube/ranked_by_watch_time.png -------------------------------------------------------------------------------- /docs/examples/_images/youtube/wildcard_open.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/docs/examples/_images/youtube/wildcard_open.png -------------------------------------------------------------------------------- /docs/examples/airbnb.md: -------------------------------------------------------------------------------- 1 | # AirBnB Gallery 2 | 3 | When people plan travel, they most likely wanted to find the most budget-friendly option for their needs. Sometimes, they prefer to find the most walkable location. The bottom line is that users have different needs when considering accommodation and Wildcard empowers them to do so. 4 | 5 | 6 |
7 | 8 | AirBnB with Wildcard closed 9 |
AirBnB Default View.
10 | 11 | AirBnB with Wildcard open 12 |
AirBnB Default View with Wildcard Open.
13 | 14 | Users can sort listings by price 15 |
AirBnB with listings sorted by price.
16 | 17 | AirBnB with listings sorted by rating 18 |
Users can also sort listings by rating.
19 | 20 | AirBnB listings with annotations 21 |
One really fascinating thing that you can do with Wildcard is to add annotations/comments to different listings. These annotations persist across page refreshes because they are stored in your browser's local storage.
22 | 23 | 24 |
25 |
❮
26 |
❯
27 | 28 | 29 | 30 | 31 | 32 |
33 | 34 | 35 |
36 | 37 | -------------------------------------------------------------------------------- /docs/examples/amazon.md: -------------------------------------------------------------------------------- 1 | # Amazon Gallery 2 | 3 | When browsing through used products on Amazon, users are not shown the total cost of these product. With Wildcard, they can not just see the total cost but also sort the items in the page by price, rating, condition, delivery date, etc. 4 | 5 |
6 | 7 | Educated by Tara Westover without Wildcard 8 |
Amazon listings before Wildcard is opened
9 | 10 | Educated by Tara Westover with Wildcard opened 11 |
Wildcard opened on the page. Notice how relevant information has been extracted into the spreadsheet.
12 | 13 | Specific row selected 14 |
When the user selects a row, the corresponding element is highlighted in Wildcard.
15 | 16 | Total price sorted in ascending order 17 |
Wildcard computes the total price from the three listed per item, and can be used to sort the total price in ascending order.
18 | 19 | Total price sorted in descending order 20 |
Wildcard used to sort the total price in ascending and descending order.
21 | 22 | Video of book sorted by condition 23 |
Additionally, users can perform nested sorts within the table; after the prices are sorted in ascending order, users can run another sort on the conditions. This will order the offer listings based on both price and condition as shown above.
24 | 25 | Books sorted by condition 26 |
Notice that by sorting the condition of the books, the user will find that the New copy costs only $1.06 more than the used copy.
27 | 28 |
29 |
❮
30 |
❯
31 | 32 | 33 | 34 | 35 | 36 | 37 |
38 | 39 |
40 | -------------------------------------------------------------------------------- /docs/examples/blogger.md: -------------------------------------------------------------------------------- 1 | # Blogger Gallery 2 | 3 | Most websites with text areas do not provide rich text editing functionality. Some applications may have text editors but those lack specific functionality that users are looking for. With Wildcard, users can import their favorite text editors into any website. Below, a rich text editor is imported into the Blogger app. 4 | 5 |
6 | 7 | 8 |
Blogger Default View with Wildcard closed.
9 | 10 | 11 |
Blogger Default View with Wildcard open.
12 | 13 | 14 |
A rich text editor chosen by the user is imported through a Wildcard cell editor and can be used to format text.
15 | 16 | 17 |
The rich text editor can also convert formatting into HTML.
18 | 19 | 20 |
21 |
❮
22 |
❯
23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 |
33 | 34 |
35 | -------------------------------------------------------------------------------- /docs/examples/hackernews.md: -------------------------------------------------------------------------------- 1 | # Hackernews Gallery 2 | 3 | Hackernews is a website that is popular among developers and entrepreneurs. Anybody can create an account and add content which is presented in a timeline format (most recent first). 4 | 5 | But sometimes, users want to see content in a different format. 6 | 7 |
8 | 9 | Hackernews Default View 10 |
Default View in Timeline Format.
11 | 12 | Hackernews Default View with Wildcard Opened 13 |
Default View with Wildcard opened.
14 | 15 | Hackernews with content sorted by points 16 |
Content presented based on descending order of points using Wildcard
17 | 18 | Hackernews with content sorted by points 19 |
Content presented based on highest level of engagement.
20 | 21 | Hackernews with annotations 22 |
Users can also add their own annotations to the articles. These annotations persist across page refreshes.
23 | 24 |
25 |
❮
26 |
❯
27 | 28 | 29 | 30 | 31 | 32 |
33 | 34 |
35 | -------------------------------------------------------------------------------- /docs/examples/template.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 |
FIRST CAPTION
5 | 6 | 7 |
SECOND + OTHER CAPTIONS
8 | 9 | 10 |
11 |
❮
12 |
❯
13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 |
24 | 25 |
26 | -------------------------------------------------------------------------------- /docs/examples/ubereats.md: -------------------------------------------------------------------------------- 1 | # UberEats Gallery 2 | 3 | When most people go on ubereats, they are looking for a wide variety of things - food to be delivered quickly, restaurants with the lowest delivery fees, among many others. Unfortunately, users can neither filter by price nor by estimated delivery time. With Wildcard, users can do both and find restaurants that suit their needs: 4 | 5 | -------------------------------------------------------------------------------- /docs/examples/weather.md: -------------------------------------------------------------------------------- 1 | # Weather Gallery 2 | 3 | Most of us have a weather app that we check daily. It usually has a chart that gives an hourly breakdown of the weather. But what if we wanted **more specific** information, eg. during what hour will it be warmest today? Let's see how Wildcard helps us to accomplish this. 4 | 5 |
6 | 7 | Weather Default View 8 |
Weather Default View.
9 | 10 | Weather Default View 11 |
Default View with Wildcard Open.
12 | 13 | Weather Default View 14 |
Wildcard showing the warmest hours of the day.
15 | 16 |
17 |
❮
18 |
❯
19 | 20 | 21 | 22 |
23 | 24 |
-------------------------------------------------------------------------------- /docs/examples/youtube.md: -------------------------------------------------------------------------------- 1 | # YouTube Gallery 2 | 3 | Youtube has an incredible recommendation system which helps people to find content that they are interested in. There are ways that Wildcard can help with spending your time on the platform (shown below). 4 | 5 |
6 | 7 | YouTube with Wildcard open 8 |
Youtube Default View with Wildcard open.
9 | 10 | YouTube with Wildcard open 11 |
Wildcard can be used to sort the videos based on their length.
12 | 13 | 14 |
15 |
❮
16 |
❯
17 | 18 | 19 |
20 | 21 |
-------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | wildcard - wildcard extension builder 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 |
17 | 66 | 67 | 68 | 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /docs/quickstart.md: -------------------------------------------------------------------------------- 1 | # Quick start 2 | 3 | These instructions are the simplest way to get the Wildcard extension 4 | installed in your browser, in just a few minutes. 5 | 6 | You'll be able to use Wildcard with any website that already has built-in support: Airbnb, Amazon, Hacker News, Instacart, Youtube, and more. See the [Examples Gallery](examples.md) for a full list of supported sites. 7 | 8 | ?> **Note to programmers:** If you know how to program in Javascript, you can write your own bits of scraper code to adapt Wildcard to new websites. If you're interested in doing that in the future, you should skip these steps and follow the slightly more complicated [Dev Install](devenv.md) instructions. (If you're not sure which way to go, you can always try this simple install first and then switch to the dev install later if needed.) 9 | 10 | ## 1. Download latest release 11 | 12 | Download the latest Wildcard release as a zip file from the [releases page](https://github.com/geoffreylitt/wildcard/releases). Unzip it to a folder on your computer. 13 | 14 | ## 2. Install in your browser 15 | 16 | **GOOGLE CHROME**: To install the directory as an unpacked Chrome extension: 17 | 18 | 1. Open the Extension Management page by navigating to **chrome://extensions.** 19 | 2. Enable Developer Mode by clicking the toggle switch next to Developer mode. 20 | 3. Click the LOAD UNPACKED button and select the **manifest.json** directory. 21 | 22 | **MICROSOFT EDGE**: To install the directory as an unpacked Edge extension: 23 | 24 | 1. Click on the three dots at the top of your browser. 25 | 2. Next, choose **Extensions** from the context menu as shown below. 26 | 3. When you are on the **Extensions** page, enable the **Developer mode** by enabling the toggle at the bottom left of the page. 27 | 4. Choose the **Load Unpacked option**. This prompts you for a directory where you have your Extension assets. Open the wildcard directory and select the **manifest.json** extension. 28 | 5. After you install your Extension, you may update it by clicking on the Reload button under your Extension listing. 29 | 30 | **MOZILLA FIREFOX**: To install the manifest.json file as an unpacked Firefox extension: 31 | 32 | 1. Open Firefox. 33 | 2. Enter “about:debugging” in the URL bar. 34 | 3. Click “This Firefox”. 35 | 4. Click “Load Temporary Add-on”. 36 | 5. Open the wildcard directory and select the **manifest.json** file. 37 | 38 | Note: You may need to unblock trackers in your Firefox preferences because Wildcard stores data in your browser’s local storage for persistence of changes across page loads. 39 | 40 | Wildcard is most useful when you enable it for all websites. 41 | It'll only activate if there's actually a site adapter 42 | available for that website. 43 | If you want, you can also change the extension settings in your browser 44 | to only activate it on certain sites, or when you click the icon. 45 | 46 | ## 3. Test it out! 47 | 48 | Navigate to [Hacker News](https://news.ycombinator.com/). 49 | 50 | You should see a button in the bottom right corner 51 | prompting you to "Open Wildcard Table". When you click it, you should see a table showing the data in Hacker News. 52 | 53 | Give it a spin: 54 | 55 | * Try sorting the page by one of the columns. When you refresh, it'll still be sorted. 56 | * Right click a column header and click "insert new column" to add a new column. Type in some text, and the corresponding article will be annotated in the page. 57 | 58 | !> If something doesn't work, it's probably a bug in the beta -- 59 | please [file a Github issue](https://github.com/geoffreylitt/wildcard/issues) or [email me](mailto:glitt@mit.edu). 60 | 61 | ## Explore 62 | 63 | To get inspiration for what to do with Wildcard, 64 | check out some of the other supported sites and use cases 65 | in the [Example gallery](examples.md). 66 | 67 | -------------------------------------------------------------------------------- /manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wildcard", 3 | "version": "0.1", 4 | "description": "Spreadsheet-driven customization of web applications", 5 | "manifest_version": 2, 6 | "content_scripts": [ 7 | { 8 | "js": [ 9 | "dist/wildcard.js", 10 | "vendor/bootstrap/js/bootstrap.min.js" 11 | ], 12 | "matches": [ 13 | "" 14 | ], 15 | "css": [ 16 | "dist/wildcard.css", 17 | "vendor/bootstrap/css/bootstrap.min.css" 18 | ] 19 | }, 20 | { 21 | "js": [ 22 | "dist/marketplace.js" 23 | ], 24 | "matches": [ 25 | "https://wildcard-marketplace.herokuapp.com/adapter.html?aid=*", 26 | "https://wildcard-marketplace.herokuapp.com/upload.html?key=*", 27 | "http://wildcard-marketplace.herokuapp.com/adapter.html?aid=*", 28 | "http://wildcard-marketplace.herokuapp.com/upload.html?key=*", 29 | "http://localhost:3000/adapter.html?aid=*", 30 | "http://localhost:3000/upload.html?key=*" 31 | ] 32 | } 33 | ], 34 | "background": { 35 | "scripts": [ 36 | "dist/wildcard-background.js", 37 | "src/wildcard-ajax.js" 38 | ] 39 | }, 40 | "permissions": [ 41 | "storage", 42 | "*://localhost/*", 43 | "history", 44 | "http://*/*", 45 | "https://*/*", 46 | "webRequest", 47 | "webRequestBlocking", 48 | "", 49 | "contextMenus" 50 | ], 51 | "options_page": "options.html" 52 | } -------------------------------------------------------------------------------- /options.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Wildcard Extension Options 5 | 6 | 7 | 8 | 34 | 35 | 36 |

Wildcard Options

37 |
38 |
39 | 40 | 46 |
47 |
48 | 49 | 50 |
51 |
52 | 53 | 56 |
57 |
58 |
59 |
60 | 61 | 62 |
63 |
64 | 65 | 66 | -------------------------------------------------------------------------------- /options.js: -------------------------------------------------------------------------------- 1 | (function(){ 2 | let LOCAL_ADAPTERS; 3 | const localStorageKey = 'localStorageAdapter'; 4 | const localAdaptersKey = `${localStorageKey}:adapters`; 5 | const adapterActionsSelect = document.getElementById('adapterActions'); 6 | const createAdaptersContainer = document.getElementById('createAdapterContainer'); 7 | const createAdaptersInput = document.getElementById('createAdapter'); 8 | const loadAdaptersContainer = document.getElementById('loadAdaptersContainer'); 9 | const adaptersSelect = document.getElementById('loadAdapters'); 10 | const saveAdapterButton = document.getElementById('saveAdapter'); 11 | const deleteAdapterButton = document.getElementById('deleteAdapter'); 12 | var editor; 13 | 14 | window.onload = function(e){ 15 | editor = ace.edit("adapterEditor"); 16 | editor.session.setMode("ace/mode/typescript"); 17 | editor.setTheme("ace/theme/monokai"); 18 | } 19 | 20 | function statusMessage(msg) { 21 | let status = document.getElementById('status'); 22 | status.textContent = msg; 23 | setTimeout(function() { 24 | status.textContent = ''; 25 | }, 1000); 26 | } 27 | function readFromLocalStorage(key, callback) { 28 | chrome.storage.local.get([key], (results) => { 29 | callback(results[key]) 30 | }); 31 | } 32 | function saveToLocalStorage(key, value, callback) { 33 | if (callback) { 34 | chrome.storage.local.set({ [key]: value }, callback); 35 | } else { 36 | chrome.storage.local.set({ [key]: value }, function() { 37 | // Update status to let user know options were saved. 38 | statusMessage("Adapter saved."); 39 | }); 40 | } 41 | } 42 | function removeFromLocalStorage(key, callback) { 43 | if (callback) { 44 | chrome.storage.local.remove(key, callback); 45 | } else { 46 | chrome.storage.local.remove(key, function() { 47 | // Update status to let user know options were saved. 48 | statusMessage("Options updated."); 49 | }); 50 | } 51 | } 52 | function populateAdapterSelect(localAdapters = []) { 53 | LOCAL_ADAPTERS = localAdapters; 54 | adaptersSelect.innerHTML = ''; 55 | const defaultOption = document.createElement('option'); 56 | defaultOption.text = 'Choose adapter'; 57 | defaultOption.value = ''; 58 | defaultOption.selected = true; 59 | defaultOption.disabled = true; 60 | defaultOption.hidden = true; 61 | adaptersSelect.append(defaultOption); 62 | localAdapters.forEach(adapter => { 63 | const option = document.createElement('option'); 64 | option.value = adapter; 65 | option.text = adapter; 66 | adaptersSelect.append(option); 67 | }); 68 | } 69 | adapterActionsSelect.addEventListener('change', () => { 70 | const action = adapterActionsSelect.value; 71 | loadAdaptersContainer.style.display = 'none'; 72 | createAdaptersContainer.style.display = 'none'; 73 | deleteAdapterButton.style.display = 'none'; 74 | createAdaptersInput.value = ''; 75 | editor.setValue(''); 76 | if (action === 'create') { 77 | createAdaptersContainer.style.display = 'block'; 78 | } else if (action === 'load' || action === 'delete') { 79 | loadAdaptersContainer.style.display = 'block'; 80 | const adapter = adaptersSelect.value; 81 | if (adapter) { 82 | readFromLocalStorage(`${localAdaptersKey}:${adapter}`, (adapterConfig) => { 83 | editor.setValue(adapterConfig); 84 | }); 85 | } 86 | if (action === 'delete') { 87 | deleteAdapterButton.style.display = 'inline'; 88 | } 89 | } 90 | }); 91 | adaptersSelect.addEventListener('change', () => { 92 | const adapter = adaptersSelect.value; 93 | readFromLocalStorage(`${localAdaptersKey}:${adapter}`, (adapterConfig) => { 94 | editor.setValue(adapterConfig); 95 | }); 96 | }); 97 | saveAdapterButton.addEventListener('click', () => { 98 | const adapterConfig = editor.getValue().trim(); 99 | const action = adapterActionsSelect.value; 100 | const adapter = action === 'create' ? createAdaptersInput.value.trim() : adaptersSelect.value; 101 | if (adapter && adapterConfig) { 102 | if (Array.isArray(LOCAL_ADAPTERS) && LOCAL_ADAPTERS.indexOf(adapter) === -1) { 103 | LOCAL_ADAPTERS.push(adapter); 104 | populateAdapterSelect(LOCAL_ADAPTERS); 105 | saveToLocalStorage(localAdaptersKey, LOCAL_ADAPTERS); 106 | } 107 | saveToLocalStorage(`${localAdaptersKey}:${adapter}`, adapterConfig); 108 | } else { 109 | alert('Please select an adapter to save to and make sure the text area has content.') 110 | } 111 | }); 112 | deleteAdapterButton.addEventListener('click', () => { 113 | const adapter = adaptersSelect.value; 114 | if (adapter) { 115 | if (Array.isArray(LOCAL_ADAPTERS) && LOCAL_ADAPTERS.indexOf(adapter) !== -1) { 116 | LOCAL_ADAPTERS.splice(LOCAL_ADAPTERS.indexOf(adapter), 1); 117 | populateAdapterSelect(LOCAL_ADAPTERS); 118 | saveToLocalStorage(localAdaptersKey, LOCAL_ADAPTERS); 119 | } 120 | removeFromLocalStorage(`${localAdaptersKey}:${adapter}`, () => { 121 | editor.setValue(''); 122 | statusMessage("Adapter removed."); 123 | }); 124 | 125 | } 126 | }); 127 | chrome.runtime.getBackgroundPage((backgroundPage) => { 128 | readFromLocalStorage(localAdaptersKey, populateAdapterSelect); 129 | const { state } = backgroundPage; 130 | if (state && state.endUserScraper) { 131 | const { name, config } = state.endUserScraper; 132 | adapterActionsSelect.value = 'create'; 133 | adapterActionsSelect.dispatchEvent(new Event('change')); 134 | createAdaptersInput.value = name; 135 | editor.setValue(config); 136 | delete state.endUserScraper; 137 | } 138 | }); 139 | })(); -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "wildcard", 3 | "version": "1.0.0", 4 | "description": "wildcard extension builder", 5 | "main": "index.js", 6 | "repository": "git@github.com:geoffreylitt/wildcard.git", 7 | "author": "Geoffrey Litt ", 8 | "license": "MIT", 9 | "dependencies": { 10 | "@ckeditor/ckeditor5-build-classic": "^16.0.0", 11 | "@fullcalendar/core": "^4.3.1", 12 | "@fullcalendar/daygrid": "^4.3.0", 13 | "@fullcalendar/google-calendar": "^4.4.0", 14 | "@fullcalendar/interaction": "^4.3.0", 15 | "@fullcalendar/moment": "^4.3.0", 16 | "@handsontable/react": "^3.1.2", 17 | "@types/chrome": "^0.0.95", 18 | "@types/greasemonkey": "^4.0.0", 19 | "@types/jquery": "^3.3.33", 20 | "docsify-cli": "^4.4.0", 21 | "handsontable": "^7.2.2", 22 | "lodash": "^4.17.15", 23 | "moment": "^2.24.0", 24 | "ohm-js": "^0.14.0", 25 | "palettify": "^1.0.3", 26 | "pikaday": "^1.8.0", 27 | "react": "^16.13.1", 28 | "react-ace": "^9.1.4", 29 | "react-autosuggest": "^10.1.0", 30 | "react-dom": "^16.13.1", 31 | "react-redux": "^7.2.0", 32 | "redux": "^4.0.5", 33 | "redux-thunk": "^2.3.0", 34 | "reselect": "^4.0.0", 35 | "rollup-plugin-node-builtins": "^2.1.2", 36 | "string-hash": "^1.1.3", 37 | "styled-components": "^5.1.0", 38 | "unfluff": "^3.2.0", 39 | "url-pattern": "^1.0.3" 40 | }, 41 | "scripts": { 42 | "docs": "docsify serve docs", 43 | "dev": "node ./build.js -watch", 44 | "chrome": "web-ext run --target chromium --keep-profile-changes", 45 | "firefox": "web-ext run --keep-profile-changes" 46 | }, 47 | "devDependencies": { 48 | "@types/lodash": "^4.14.168", 49 | "@types/react": "^16.9.34", 50 | "@types/react-dom": "^16.9.6", 51 | "@types/redux-thunk": "^2.1.0", 52 | "concurrently": "^5.1.0", 53 | "docsify": "^4.11.3", 54 | "esbuild": "^0.8.0", 55 | "estrella": "^1.3.1", 56 | "random-words": "^1.1.1", 57 | "redux-devtools-extension": "^2.13.8", 58 | "socket.io": "^2.3.0", 59 | "typescript": "^3.7.4", 60 | "watch": "^1.0.2", 61 | "web-ext": "^5.5.0" 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /readme-resources/architecture-v02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geoffreylitt/wildcard/dd81d9b4a34729887e86b2e5de0c3e8ecdc94b03/readme-resources/architecture-v02.png -------------------------------------------------------------------------------- /resources/adapter-configs/eecscoursecatalog.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MIT EECS Course Catalog", 3 | "urls": [ 4 | "http://eecs.scripts.mit.edu/eduportal/who_is_teaching_what/F/2020" 5 | ], 6 | "matches": [ 7 | "http://eecs.scripts.mit.edu/eduportal/who_is_teaching_what/F/2020" 8 | ], 9 | "attributes": [ 10 | { 11 | "name": "id", 12 | "type": "text", 13 | "hidden": true 14 | }, 15 | { 16 | "name": "number", 17 | "type": "text" 18 | }, 19 | { 20 | "name": "title", 21 | "type": "text" 22 | }, 23 | { 24 | "name": "mode", 25 | "type": "text" 26 | }, 27 | { 28 | "name": "lecturers", 29 | "type": "text" 30 | }, 31 | { 32 | "name": "recitation instructors", 33 | "type": "text" 34 | } 35 | ], 36 | "scrapePage": "() => {\n return Array.from(document.querySelectorAll('tr[id]')).map(el => {\n let courseData = el.getElementsByTagName('td')\n let courseNumber = el.id\n let courseName = courseData[0].innerText\n\n return {\n id: courseNumber,\n rowElements: [el],\n dataValues: {\n number: courseNumber,\n title: courseName.substring(courseNumber.length + 1),\n mode: courseData[1].innerText,\n lecturers: courseData[2].innerText,\n 'recitation instructors': courseData[3].innerText\n }\n }\n })\n }", 37 | "onRowSelected": "(row) => {\n row.rowElements.forEach(el => {\n if (el.style) {\n el.style['background-color'] = '#c9ebff';\n }\n });\n row.rowElements[0].scrollIntoView({ behavior: 'smooth', block: 'center' });\n }", 38 | "onRowUnselected": "(row) => {\n row.rowElements.forEach(el => {\n if (el.style) {\n el.style['background-color'] = '';\n }\n })\n }" 39 | } -------------------------------------------------------------------------------- /resources/adapter-configs/github.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Github", 3 | "urls": [ 4 | "https://github.com/geoffreylitt?tab=repositories" 5 | ], 6 | "matches": [ 7 | "github.com.*tab=repositories" 8 | ], 9 | "attributes": [ 10 | { 11 | "name": "name", 12 | "type": "text" 13 | }, 14 | { 15 | "name": "stars", 16 | "type": "numeric" 17 | }, 18 | { 19 | "name": "forks", 20 | "type": "text" 21 | }, 22 | { 23 | "name": "updated", 24 | "type": "text" 25 | } 26 | ], 27 | "scrapePage": "() => {\n return Array.from(document.querySelectorAll(\"li.source\")).map(el => {\n let name_el = el.querySelector('a[itemprop=\"name codeRepository\"]')\n let name = name_el.textContent.trim()\n\n let stars_el = el.querySelector('*[href*=\"/stargazers\"')\n let stars = extractNumber(stars_el, 0)\n\n let forks_el = el.querySelector('*[href*=\"/network/members\"]')\n let forks = extractNumber(forks_el, 0)\n\n let lang_el = el.querySelector('*[itemprop=\"programmingLanguage\"]')\n // some repos don't have language set\n let lang = lang_el == null ? null : lang_el.textContent.trim()\n\n let updated_el = el.querySelector('relative-time')\n let updated = updated_el.getAttribute('datetime')\n\n return {\n id: name,\n rowElements: [el],\n dataValues: {\n name: name,\n stars: stars,\n forks: forks,\n updated: updated,\n }\n }\n })\n }", 28 | "onRowSelected": "(row) => {\n row.rowElements.forEach(el => {\n if (el.style) {\n el.style[\"background-color\"] = \"#def3ff\"\n }\n });\n row.rowElements[0].scrollIntoView({ behavior: \"smooth\", block: \"center\" })\n }", 29 | "onRowUnselected": "(row) => {\n row.rowElements.forEach(el => {\n if(el.style) {\n el.style[\"background-color\"] = \"\";\n }\n })\n }" 30 | } -------------------------------------------------------------------------------- /resources/adapter-configs/hackernews.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Hacker News", 3 | "urls": [ 4 | "news.ycombinator.com" 5 | ], 6 | "matches": [ 7 | "news.ycombinator.com", 8 | "news.ycombinator.com/news", 9 | "news.ycombinator.com/newest" 10 | ], 11 | "attributes": [ 12 | { 13 | "name": "id", 14 | "type": "text", 15 | "hidden": true 16 | }, 17 | { 18 | "name": "rank", 19 | "type": "numeric" 20 | }, 21 | { 22 | "name": "title", 23 | "type": "text" 24 | }, 25 | { 26 | "name": "link", 27 | "type": "text" 28 | }, 29 | { 30 | "name": "points", 31 | "type": "numeric" 32 | }, 33 | { 34 | "name": "user", 35 | "type": "text" 36 | }, 37 | { 38 | "name": "comments", 39 | "type": "numeric" 40 | } 41 | ], 42 | "scrapePage": "() => {\n return Array.from(document.querySelectorAll(\"tr.athing\")).map(el => {\n let detailsRow = el.nextElementSibling\n let spacerRow = detailsRow.nextElementSibling\n\n return {\n id: String(el.getAttribute(\"id\")),\n rowElements: [el, detailsRow, spacerRow],\n // todo: Both of these steps should be handled by the framework...\n // .filter(e => e) // Only include if the element is really there\n // .map(e => (e)), // Convert to HTMLElement type\n dataValues: {\n rank: el.querySelector(\"span.rank\"),\n title: el.querySelector(\"a.storylink\"),\n link: el.querySelector(\"a.storylink\").getAttribute(\"href\"),\n // These elements contain text like \"162 points\";\n // Wildcard takes care of extracting a number automatically.\n points: detailsRow.querySelector(\"span.score\"),\n user: detailsRow.querySelector(\"a.hnuser\"),\n comments: extractNumber(Array.from(detailsRow.querySelectorAll(\"a\"))\n .find(e => e.textContent.indexOf(\"comment\") !== -1), 0)\n },\n annotationContainer: detailsRow.querySelector(\"td.subtext\"),\n annotationTemplate: \"| $annotation\"\n }\n })\n }", 43 | "onRowSelected": "(row) => {\n row.rowElements.forEach(el => {\n if (el.style) {\n el.style[\"background-color\"] = \"#def3ff\";\n }\n });\n row.rowElements[0].scrollIntoView({ behavior: \"smooth\", block: \"center\" });\n }", 44 | "onRowUnselected": "(row) => {\n row.rowElements.forEach(el => {\n if(el.style) {\n el.style[\"background-color\"] = \"\";\n }\n })\n }" 45 | } -------------------------------------------------------------------------------- /resources/adapter-configs/mitcoursecatalog.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MIT Course Catalog", 3 | "urls": [ 4 | "http://student.mit.edu/catalog/m6c.html" 5 | ], 6 | "matches": [ 7 | "http://student.mit.edu/catalog" 8 | ], 9 | "attributes": [ 10 | { 11 | "name": "id", 12 | "type": "text", 13 | "hidden": true 14 | }, 15 | { 16 | "name": "code", 17 | "type": "text" 18 | }, 19 | { 20 | "name": "name", 21 | "type": "text" 22 | }, 23 | { 24 | "name": "level", 25 | "type": "text" 26 | } 27 | ], 28 | "scrapePage": "() => {\n return Array.from(document.querySelectorAll('h3'))\n .slice(1)\n .filter((element) => {\n const regEx = new RegExp('[A-Z0-9]+\\\\.[0-9]+(\\\\[[A-Z]\\])?\\\\s.+');\n const matches = regEx.test(element.textContent);\n return matches;\n })\n .map((element) => {\n const regEx = new RegExp('([A-Z0-9]+\\\\.[0-9]+(\\\\[[A-Z]\\\\])?)\\\\s(.+)');\n const matches = regEx.exec(element.textContent.trim());\n const match = matches[0];\n const id = matches[1];\n const symbol = matches[2]; \n const name = matches[3];\n const rowElements = [element];\n let currentElement = element.nextSibling;\n while (currentElement && currentElement.tagName !== \"H3\") {\n rowElements.push(currentElement);\n currentElement = currentElement.nextSibling;\n }\n let level;\n for (var i = 1; i < rowElements.length; i++) {\n var rowElement = rowElements[i];\n if (rowElement.tagName === 'IMG') {\n if (rowElement.title === 'Undergrad') {\n level = 'U';\n break;\n }\n else if (rowElement.title === 'Graduate') {\n level = 'G';\n break;\n }\n }\n }\n const row = {\n id: id,\n rowElements: rowElements,\n dataValues: {\n code: id,\n name: name,\n level: level\n },\n annotationContainer: rowElements[rowElements.length - 1],\n annotationTemplate: '$annotation'\n };\n return row;\n });\n }\n ", 29 | "onRowSelected": "(row) => {\n const rowElement = row.rowElements[0];\n rowElement.style.border = \"solid 2px #b12b28\";\n rowElement.scrollIntoView({ behavior: \"smooth\", block: \"center\" });\n }", 30 | "onRowUnselected": "(row) => {\n const rowElement = row.rowElements[0];\n rowElement.style.border = '';\n }" 31 | } -------------------------------------------------------------------------------- /resources/adapter-configs/youtube.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "YouTube", 3 | "urls": [ 4 | "youtube.com" 5 | ], 6 | "matches": [ 7 | "youtube.com" 8 | ], 9 | "attributes": [ 10 | { 11 | "name": "id", 12 | "type": "text", 13 | "hidden": true 14 | }, 15 | { 16 | "name": "Title", 17 | "type": "text" 18 | }, 19 | { 20 | "name": "Time", 21 | "type": "text" 22 | }, 23 | { 24 | "name": "Uploader", 25 | "type": "text" 26 | }, 27 | { 28 | "name": "% Watched", 29 | "type": "numeric" 30 | } 31 | ], 32 | "scrapePage": "() => {\n function progressToNumber(progress){\n let strippedProgress = progress.slice(0, -1);\n return parseInt(strippedProgress);\n }\n let tableRows = document.querySelector('#contents').children;\n if (tableRows.length == 1) {\n // for use on video listing page e.g. https://www.youtube.com/user/*/videos\n tableRows = document.querySelector('#contents #items').children;\n }\n return Array.from(tableRows).map((el, index) => {\n let elAsHTMLElement = el;\n\n // on /user/*/videos, link is in #thumbnail, not #video-title-link\n if((el.querySelector('#video-title-link') !== null || el.querySelector('#thumbnail') !== null) && el.querySelector('#overlays') != null && el.querySelector('#overlays').children[0] != null){\n\n let overlayChildrenAmount = el.querySelector('#overlays').children.length;\n let timeStampExists = overlayChildrenAmount > 1 && el.querySelector('#overlays').children[overlayChildrenAmount - 2].children[1] !== undefined;\n let timeStamp = timeStampExists\n ? el.querySelector('#overlays').children[overlayChildrenAmount - 2].children[1].textContent.replace(new RegExp(\"|\\\\r\\\\n|\\\\n|\\\\r\", \"gm\"),\"\")\n : \"N/A\";\n let watchedPercentage = el.querySelector('#progress') !== null\n ? progressToNumber((el.querySelector('#progress')).style.width)\n : 0;\n\n return {\n rowElements: [elAsHTMLElement],\n id: (el.querySelector('#video-title-link') || el.querySelector('#thumbnail')).getAttribute(\"href\"),\n dataValues: {\n Title: el.querySelector('#video-title'),\n Time: timeStamp,\n Uploader: el.querySelector('#text').children[0],\n '% Watched': watchedPercentage,\n },\n }\n } else {\n return null;\n }\n }).filter(el => el !== null)\n }", 33 | "addScrapeTriggers": "(reload) => {\n document.addEventListener(\"click\", (e) => { reload() });\n document.addEventListener(\"keydown\", (e) => { reload() });\n document.addEventListener(\"scroll\", debounce((e) => { reload() }, 50));\n }", 34 | "onRowSelected": "(row) => {\n row.rowElements.forEach(el => {\n if (el.style) {\n el.style[\"background-color\"] = \"#c9ebff\";\n }\n });\n row.rowElements[0].scrollIntoView({ behavior: \"smooth\", block: \"center\" });\n }", 35 | "onRowUnselected": "(row) => {\n row.rowElements.forEach(el => {\n if (el.style) {\n el.style[\"background-color\"] = \"\";\n }\n });\n }" 36 | } -------------------------------------------------------------------------------- /src/core/actions.ts: -------------------------------------------------------------------------------- 1 | 2 | // todo: define TS types for these events 3 | // https://redux.js.org/recipes/usage-with-typescript 4 | 5 | // Many of these actions don't directly affect the Redux state, 6 | // instead they ask a TableAdapter to do something async, 7 | // which will update the redux state on completion. 8 | // We use async redux-thunk action creators for this. 9 | 10 | import { Table, TableAdapter, tableId, recordId, RecordEdit, Record, Attribute } from './types' 11 | import groupBy from 'lodash/groupBy' 12 | import forIn from 'lodash/forIn' 13 | import pick from 'lodash/pick' 14 | import { getFinalAttributes, getFinalRecords } from './getFinalTable' 15 | import { evalFormulas } from '../formula' 16 | 17 | export const initializeActions = (TableAdapters:{ [key: string]: TableAdapter }) => { 18 | return { 19 | tableReloaded (table:Table) { 20 | return (dispatch, getState) => { 21 | // load the new data into the UI immediately 22 | dispatch({ type: "TABLE_RELOADED", table }) 23 | 24 | // asynchronously trigger formula re-evaluation 25 | const state = getState() 26 | const finalRecords:Record[] = getFinalRecords(state) 27 | const finalAttributes:Attribute[] = getFinalAttributes(state) 28 | 29 | evalFormulas(finalRecords, finalAttributes, (values) => { 30 | dispatch({ type: "FORMULAS_EVALUATED", values }) 31 | }) 32 | } 33 | }, 34 | 35 | addAttribute (tableId:tableId) { 36 | return (dispatch) => { 37 | dispatch({ 38 | type: "ADD_ATTRIBUTE_REQUESTED" 39 | }) 40 | const TableAdapter = TableAdapters[tableId]; 41 | TableAdapter.addAttribute().then( 42 | // no need to do anything, since we're already subscribed to reloads 43 | (_table) => { }, 44 | (err) => { console.error(err) } 45 | ) 46 | } 47 | }, 48 | 49 | clear (tableId:tableId) { 50 | return (dispatch) => { 51 | const TableAdapter = TableAdapters[tableId]; 52 | TableAdapter.clear() 53 | } 54 | }, 55 | 56 | toggleVisibility (tableId:tableId, colName:string) { 57 | return (dispatch) => { 58 | dispatch({ 59 | type: "HIDE_COL_REQUESTED" 60 | }) 61 | const TableAdapter = TableAdapters[tableId]; 62 | TableAdapter.toggleVisibility(colName); 63 | } 64 | }, 65 | 66 | setFormula (tableId:tableId, attrName:string, formula) { 67 | return (dispatch) => { 68 | const TableAdapter = TableAdapters[tableId]; 69 | TableAdapter.setFormula(attrName, formula); 70 | } 71 | }, 72 | 73 | editRecords(edits) { 74 | return (dispatch) => { 75 | dispatch({ type: "EDIT_RECORDS_REQUESTED", edits }) 76 | 77 | // split up the request edits by table, and ask each table to 78 | // do its part of the edits 79 | 80 | const editsByTable = groupBy(edits, e => e.tableId); 81 | 82 | forIn(editsByTable, (edits, tableId) => { 83 | const TableAdapter = TableAdapters[tableId]; 84 | const editsForTable:Array = edits.map(e => pick(e, "recordId", "attribute", "value")) 85 | TableAdapter.editRecords(editsForTable); 86 | }); 87 | } 88 | }, 89 | 90 | sortRecords (sortConfig) { 91 | return { type: "SORT_RECORDS", sortConfig } 92 | }, 93 | 94 | selectRecord (recordId, attribute) { 95 | return { type: "RECORD_SELECTED", recordId, attribute } 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/core/debug.tsx: -------------------------------------------------------------------------------- 1 | import { getFinalRecords, getFinalAttributes } from './getFinalTable' 2 | 3 | // this middleware sends redux information to the console. 4 | // it would be better to get Redux Dev Tools working, but for some reaosn 5 | // Redux Dev Tools doesn't currently work for this extension, so we roll our own 6 | export const debugMiddleware = ({ getState }) => next => action => { 7 | //console.debug('will dispatch', action) 8 | 9 | // Call the next dispatch method in the middleware chain. 10 | const returnValue = next(action) 11 | 12 | // todo: this is where we're going to update the state in the extension. 13 | // (for now, we don't need to do it because the table lives inside the app) 14 | //console.debug('state after dispatch', getState()) 15 | 16 | return returnValue 17 | } 18 | -------------------------------------------------------------------------------- /src/core/getFinalTable.ts: -------------------------------------------------------------------------------- 1 | // This file contains the logic for assembling a final table to display: 2 | // 3 | // * Join together various tables -- app data, user data 4 | // * Join together attribute lists -- app, user 5 | // * Sort / filter the final output 6 | 7 | // These are implemented as reselect selectors because they're derived state; 8 | // no need to store in the redux store; just a pure function of the various 9 | // tables and attributes. 10 | 11 | // Whatever gets outputted by these selectors is *exactly* what gets displayed 12 | // in the final table view. 13 | 14 | 15 | import { createSelector } from 'reselect'; 16 | import sortBy from 'lodash/sortBy'; 17 | import keyBy from 'lodash/keyBy'; 18 | import { RootState } from './reducer'; 19 | import { Attribute, Record, SortConfig } from './types'; 20 | 21 | const getAppRecords = (state:RootState):Record[] => state.appTable.records 22 | const getAppAttributes = (state:RootState): Attribute[] => state.appTable.attributes 23 | 24 | const getUserRecords = (state:RootState):Record[] => state.userTable.records 25 | const getUserAttributes = (state:RootState):Attribute[] => state.userTable.attributes 26 | 27 | const getSortConfig = (state:RootState):SortConfig => state.query.sortConfig 28 | const getFormulaResults = (state:RootState):any => state.formulaResults 29 | 30 | export const getFinalAttributes = createSelector( 31 | [getAppAttributes, getUserAttributes, getFormulaResults], 32 | (appAttributes, userAttributes, formulaResults) => { 33 | // annotate attrs with a table id 34 | appAttributes = (appAttributes || []).map( a => ({ ...a, tableId: "app" })) 35 | userAttributes = (userAttributes || []).map(a => ({ ...a, tableId: "user" })) 36 | 37 | const attributes = appAttributes.concat(userAttributes) 38 | 39 | // set column type for formulas based on first row 40 | const formulaResultsKeys = Object.keys(formulaResults); 41 | if (formulaResultsKeys.length > 0) { 42 | attributes.forEach(attr => { 43 | if(attr.formula) { 44 | const sampleValue = formulaResults[formulaResultsKeys[0]][attr.name] 45 | if(typeof sampleValue === 'number') { 46 | attr.type = "numeric" 47 | } else if (typeof sampleValue === 'boolean') { 48 | attr.type = "checkbox" 49 | } else if (sampleValue instanceof HTMLElement) { 50 | // hmm.. is this sensible... 51 | attr.type = "element" 52 | } else { 53 | attr.type = "text" 54 | } 55 | } else { 56 | attr.type = "text" 57 | } 58 | }) 59 | } 60 | return attributes 61 | } 62 | ) 63 | 64 | // todo: this selector is just cached on the whole state -- 65 | // probably pointless to use this selector concept here? 66 | export const getFinalRecords = createSelector( 67 | [getAppRecords, getUserRecords, getAppAttributes, getUserAttributes, getSortConfig, getFormulaResults, getFinalAttributes], 68 | (appRecords, userRecords, appAttributes, userAttributes, sortConfig, formulaResults, finalAttributes) => { 69 | 70 | const userRecordsById = keyBy(userRecords, r => r.id); 71 | 72 | let finalRecords = appRecords.slice().map(r => { 73 | // join app records to user records 74 | const finalRecord = { 75 | id: r.id, 76 | values: { 77 | ...r.values, 78 | ...(userRecordsById[r.id] || {}).values 79 | } 80 | } 81 | 82 | // add formula results to the table, where available. 83 | // (any missing results are still in process of being computed, 84 | // and we'll re-run the reducer once they are available) 85 | finalAttributes.filter(attr => attr.formula).forEach(attr => { 86 | const result = formulaResults?.[finalRecord.id]?.[attr.name] 87 | if(result !== undefined) { 88 | finalRecord.values[attr.name] = result 89 | } 90 | }) 91 | 92 | return finalRecord 93 | }) 94 | 95 | // sort 96 | if (sortConfig) { 97 | finalRecords = sortBy(finalRecords, r => r.values[sortConfig.attribute]) 98 | 99 | if (sortConfig.direction === "desc") { 100 | finalRecords = finalRecords.reverse() 101 | } 102 | } 103 | 104 | return finalRecords; 105 | } 106 | ) 107 | -------------------------------------------------------------------------------- /src/core/reducer.ts: -------------------------------------------------------------------------------- 1 | import { Record, Attribute, QueryState, Table} from './types' 2 | import includes from 'lodash/includes' 3 | import pick from 'lodash/pick' 4 | import { combineReducers } from 'redux' 5 | 6 | const createTableReducer = (tableId) => { 7 | const initialTable = { 8 | // todo: init this from the site adapter with a site name? 9 | tableId: tableId, 10 | attributes: [], 11 | records: [], 12 | } 13 | 14 | return (state = initialTable, action):Table => { 15 | if (!action.table || action.table.tableId !== tableId) { 16 | return state; 17 | } 18 | 19 | switch(action.type) { 20 | case "TABLE_RELOADED": 21 | return { 22 | ...state, 23 | attributes: [...action.table.attributes], 24 | records: [...action.table.records], 25 | } 26 | 27 | default: 28 | return state; 29 | } 30 | } 31 | } 32 | 33 | const query = (state = { sortConfig: null }, action):QueryState => { 34 | switch(action.type) { 35 | case "SORT_RECORDS": 36 | return { 37 | ...state, 38 | sortConfig: action.sortConfig 39 | } 40 | 41 | default: 42 | return state; 43 | } 44 | } 45 | 46 | const formulaResults = (state = { }, action) => { 47 | switch(action.type) { 48 | case "FORMULAS_EVALUATED": 49 | return { 50 | ...state, 51 | ...action.values 52 | } 53 | 54 | default: 55 | return state; 56 | } 57 | } 58 | 59 | const rootReducer = combineReducers({ 60 | appTable: createTableReducer("app"), 61 | userTable: createTableReducer("user"), 62 | query, 63 | formulaResults 64 | }) 65 | 66 | export type RootState = ReturnType 67 | 68 | export default rootReducer; 69 | -------------------------------------------------------------------------------- /src/core/types.ts: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import { 4 | ScrapedRow, 5 | ScrapingAdapterConfig 6 | } from '../site_adapters/domScrapingBase' 7 | 8 | // This file defines types used throughout Wildcard 9 | 10 | type id = string; 11 | 12 | export type recordId = id; 13 | export type tableId = id; 14 | 15 | export interface Record { 16 | id: recordId; 17 | values: any; 18 | } 19 | 20 | export interface RecordEdit { 21 | recordId:recordId; 22 | attribute:string; 23 | value:any; 24 | } 25 | 26 | 27 | /** 28 | * Defines the schema for one column of the table being extracted. 29 | */ 30 | export interface Attribute { 31 | /** The name of this data column, to be displayed in the table */ 32 | name: string; 33 | 34 | /** The type of this column. Can be any 35 | * [Handsontable cell type](https://handsontable.com/docs/7.3.0/tutorial-cell-types.html). 36 | * Examples: text, numeric, date, checkbox. */ 37 | type: string; 38 | 39 | /** Allow user to edit this value? Defaults to false. 40 | * Making a column editable requires extracting [[PageValue]]s as Elements.*/ 41 | editable?: boolean; 42 | 43 | /** The formula for computing this cell. (If missing, cell is considered manual data) */ 44 | formula?: string; 45 | 46 | /** Specify a custom [Handsontable editor](https://handsontable.com/docs/7.3.0/tutorial-cell-editor.html) 47 | * as a class (see Expedia adapter for an example) */ 48 | editor?: string; 49 | 50 | /** Specify a custom [Handsontable rendererr](https://handsontable.com/docs/7.3.0/demo-custom-renderers.html) 51 | * as a class (todo: not actually supported yet, but will be soon ) */ 52 | renderer?: string; 53 | 54 | /** Hide this column in the visible table? 55 | Eg, useful for hiding an ID column that's needed for sorting */ 56 | hidden?: boolean; 57 | 58 | // todo: move these into a metadata sub-object or something 59 | timeFormat?: string; 60 | correctFormat?: boolean; 61 | 62 | hideInPage?: boolean; 63 | } 64 | 65 | 66 | export interface Table { 67 | tableId: tableId; 68 | attributes: Array; 69 | records: Array 70 | } 71 | 72 | export interface SortConfig { 73 | attribute: string; 74 | direction: "asc" | "desc"; 75 | } 76 | 77 | export interface QueryState { 78 | sortConfig: SortConfig 79 | } 80 | 81 | export type TableCallback = (table:Table) => void; 82 | 83 | // Generalizing over the site adapters and user data, among others 84 | export interface TableAdapter { 85 | tableId: tableId; 86 | 87 | name:string 88 | 89 | /** return true if this adapter should be enabled for this site */ 90 | enabled():boolean; 91 | 92 | /** start up this TableAdapter */ 93 | initialize?(namespace?:string):void; 94 | 95 | // ===================== 96 | // Reading data 97 | // ===================== 98 | 99 | /** Return latest data */ 100 | loadTable():Table; 101 | 102 | /** Register a callback function which will be called with a new table 103 | * of data anytime the data changes. */ 104 | subscribe (callback:TableCallback):void; 105 | 106 | // ============================================================ 107 | // Requesting to the TableAdapter to modify the table in some way 108 | // ============================================================ 109 | 110 | // todo: should probably update these to return promises 111 | // rather than just void and throwing away any return values 112 | 113 | /** Apply a new sort order to the table */ 114 | applySort(finalRecords:Array, sortConfig:SortConfig):void; 115 | 116 | editRecords(edits:Array):Promise; 117 | 118 | handleRecordSelected(recordId: recordId, attribute: string); 119 | 120 | /** clear the contents of the table */ 121 | clear():void 122 | 123 | addAttribute():Promise
; 124 | 125 | toggleVisibility(colName: string):void; 126 | 127 | setFormula(attrName: string, formula: string); 128 | 129 | 130 | /** Update the UI to match arbitrary table state 131 | * (To implement performantly, probably do a diff inside the adapter 132 | * and only update the UI where necessary) */ 133 | // update?(table:Table):void; 134 | 135 | // ============================================================ 136 | // Notifying the TableAdapter of changes to other tables 137 | // ============================================================ 138 | 139 | handleOtherTableUpdated(table:Table):void; 140 | 141 | updateConfig?(config): void; 142 | 143 | getConfig?(): ScrapingAdapterConfig 144 | 145 | } 146 | -------------------------------------------------------------------------------- /src/end_user_scraper/adapterHelpers.ts: -------------------------------------------------------------------------------- 1 | import { 2 | readFromChromeLocalStorage, 3 | saveToChromeLocalStorage, 4 | removeFromChromeLocalStorage 5 | } from '../utils'; 6 | 7 | import { 8 | ADAPTERS_BASE_KEY 9 | } from './constants'; 10 | import { 11 | indexToAlpha 12 | } from './utils'; 13 | 14 | import { 15 | getCachedActiveAdapter, 16 | setAdapterKey 17 | } from './state'; 18 | 19 | import { 20 | createDomScrapingAdapter 21 | } from '../site_adapters/domScrapingBase'; 22 | 23 | import { compileAdapterJavascript, userStore } from '../localStorageAdapter'; 24 | 25 | export function createAdapterKey() { 26 | return `${ADAPTERS_BASE_KEY}:${_createAdapterId()}` 27 | } 28 | 29 | function _saveAdapter(adapterKey, config, callback?) { 30 | const _config = JSON.stringify(config, null, 2); 31 | if (adapterKey) { 32 | const adapterName = adapterKey.match(/localStorageAdapter\:adapters\:(.*)/)[1] 33 | readFromChromeLocalStorage([ADAPTERS_BASE_KEY]) 34 | .then(results => { 35 | let adapters = results[ADAPTERS_BASE_KEY]; 36 | if (adapters === undefined) { 37 | adapters = [] 38 | } 39 | if (!adapters.includes(adapterName)) { 40 | adapters.push(adapterName); 41 | saveToChromeLocalStorage({ 42 | [ADAPTERS_BASE_KEY]: adapters, 43 | [adapterKey]: _config 44 | }).then(() => { 45 | if (callback) { 46 | callback(); 47 | } 48 | }) 49 | } else { 50 | saveToChromeLocalStorage({ [adapterKey]: _config }) 51 | .then(() => { 52 | if (callback) { 53 | callback(); 54 | } 55 | }); 56 | } 57 | }); 58 | } 59 | } 60 | 61 | export function saveAdapter(adapterKey, config, callback?) { 62 | readFromChromeLocalStorage([adapterKey]) 63 | .then((results) => { 64 | if (results[adapterKey]) { 65 | const currentConfig = JSON.parse(results[adapterKey]); 66 | if (!adaptersAreIdentical(currentConfig, config)) { 67 | _saveAdapter( 68 | adapterKey, 69 | config, 70 | callback 71 | ); 72 | } else if (callback) { 73 | callback(); 74 | } 75 | } else { 76 | _saveAdapter( 77 | adapterKey, 78 | config, 79 | callback 80 | ); 81 | } 82 | }) 83 | } 84 | 85 | export function deleteAdapter(adapterKey, callback) { 86 | if (adapterKey) { 87 | const adapterName = adapterKey.split(':').pop(); 88 | readFromChromeLocalStorage([ADAPTERS_BASE_KEY]) 89 | .then(results => { 90 | const adapters = results[ADAPTERS_BASE_KEY] as Array; 91 | if (Array.isArray(adapters)) { 92 | const adapterIndex = adapters.indexOf(adapterName); 93 | if (adapterIndex !== -1) { 94 | adapters.splice(adapterIndex, 1); 95 | saveToChromeLocalStorage({ [ADAPTERS_BASE_KEY]: adapters }) 96 | .then(() => { 97 | removeFromChromeLocalStorage([adapterKey, `query:${adapterName}`]) 98 | .then(() => { 99 | userStore.clear(); 100 | callback(); 101 | }); 102 | }); 103 | } else { 104 | callback(); 105 | } 106 | } else { 107 | callback(); 108 | } 109 | }); 110 | } else { 111 | callback(); 112 | } 113 | } 114 | 115 | export function generateAdapter(columnSelectors, rowSelector, adapterKey) { 116 | const { attributes, scrapePage } = createAdapterData(rowSelector, columnSelectors); 117 | return { 118 | name: document.title, 119 | urls: [window.location.href], 120 | matches: [`${window.location.origin}${window.location.pathname}`], 121 | attributes, 122 | metadata: { 123 | id: adapterKey, 124 | columnSelectors, 125 | rowSelector 126 | }, 127 | scrapePage 128 | }; 129 | } 130 | 131 | export function createInitialAdapter() { 132 | const config = createInitialAdapterConfig(); 133 | return createDomScrapingAdapter(config as any); 134 | } 135 | 136 | export function createInitialAdapterConfig() { 137 | const adapterKey = createAdapterKey(); 138 | setAdapterKey(adapterKey); 139 | const config = generateAdapter([], '', adapterKey); 140 | compileAdapterJavascript(config); 141 | return config; 142 | } 143 | 144 | export function updateAdapter(adapterKey, columnSelectors, rowSelector) { 145 | const activeAdapter = getCachedActiveAdapter(); 146 | if (activeAdapter) { 147 | const config = generateAdapter(columnSelectors, rowSelector, adapterKey); 148 | const configCopy = {...config}; 149 | compileAdapterJavascript(configCopy); 150 | setTimeout(() => { 151 | activeAdapter.updateConfig(configCopy); 152 | }, 0); 153 | } 154 | } 155 | 156 | function adaptersAreIdentical(adapter1, adapter2) { 157 | if (adapter1.attributes.length !== adapter2.attributes.length) { 158 | return false; 159 | } 160 | if (adapter1.scrapePage !== adapter2.scrapePage) { 161 | return false; 162 | } 163 | for (let i = 0; i < adapter1.attributes.length; i++) { 164 | const adapter1Attribute = adapter1.attributes[i]; 165 | const adapter2Attribute = adapter2.attributes[i]; 166 | if (adapter1Attribute.formula !== adapter2Attribute.formula) { 167 | return false; 168 | } 169 | } 170 | return true; 171 | } 172 | 173 | function createAdapterData(rowSelector, columnSelectors) { 174 | return { 175 | attributes: _createAttributes({ rowSelector, columnSelectors }), 176 | scrapePage: _createScrapPage({ rowSelector }) 177 | } 178 | } 179 | 180 | function _createAttributes({ rowSelector, columnSelectors }) { 181 | const attributes = []; 182 | const domFormulas = ["=QuerySelector", "=GetParent"] 183 | if (rowSelector && columnSelectors && columnSelectors.length) { 184 | // add row element attribute 185 | attributes.push({ 186 | name: "rowElement", 187 | type: "element", 188 | hidden: true 189 | }); 190 | // add remaining attributes 191 | columnSelectors.forEach((columnSelectorList, index) => { 192 | const columnSelector = columnSelectorList[0]; 193 | let type = "element"; 194 | let formula; 195 | if (columnSelector && columnSelector.startsWith("=")) { 196 | formula = columnSelector; 197 | if (!domFormulas.find(v => formula.startsWith(v))) { 198 | type = "text"; 199 | } 200 | } else if (columnSelector) { 201 | formula = `=QuerySelector(rowElement, "${columnSelector}")`; 202 | } else { 203 | formula = ``; 204 | } 205 | attributes.push({ 206 | name: indexToAlpha(index), 207 | type, 208 | formula 209 | }); 210 | }); 211 | } 212 | return attributes; 213 | } 214 | 215 | function _createScrapPage({ rowSelector }) { 216 | return `() => { 217 | const rowElements = ${!!rowSelector} ? document.querySelectorAll("${rowSelector}") : []; 218 | return Array.from(rowElements).map((element, rowIndex) => { 219 | return { 220 | id: String(rowIndex), 221 | index: rowIndex, 222 | dataValues: { 223 | rowElement: element 224 | }, 225 | rowElements: [element] 226 | } 227 | }); 228 | }`; 229 | } 230 | 231 | function _createAdapterId() { 232 | return document.title; 233 | } -------------------------------------------------------------------------------- /src/end_user_scraper/constants.ts: -------------------------------------------------------------------------------- 1 | export const MIN_COLUMNS = 4; 2 | export const MOUSE_MOVE_COLOR = 'rgb(127, 140, 141)'; 3 | export const MOUSE_CLICK_ROW_COLOR = MOUSE_MOVE_COLOR; 4 | export const ADAPTERS_BASE_KEY = 'localStorageAdapter:adapters'; 5 | export const ACTIVE_COLOR = 'rgb(46, 204, 113)'; 6 | export const INACTIVE_COLOR = 'rgb(127, 140, 141)'; -------------------------------------------------------------------------------- /src/end_user_scraper/domHelpers.ts: -------------------------------------------------------------------------------- 1 | function getAllClassCombinations(chars) { 2 | const result = []; 3 | const f = (prefix, chars) => { 4 | for (let i = 0; i < chars.length; i++) { 5 | result.push(`${prefix}.${chars[i]}`); 6 | f(`${prefix}.${chars[i]}`, chars.slice(i + 1)); 7 | } 8 | }; 9 | f('', chars); 10 | return result; 11 | } 12 | 13 | export function generateClassSelector(node, isRow, rowElement?) { 14 | if (node.classList && node.classList.length) { 15 | let selectors = []; 16 | const nodeTagName = node.tagName.toLowerCase(); 17 | const allClassCombinations = getAllClassCombinations(Array.from(node.classList)); 18 | if (isRow) { 19 | const siblings = Array 20 | .from(node.parentNode.children) 21 | .filter((element: HTMLElement) => !element.isSameNode(node)); 22 | allClassCombinations.forEach((selector, i) => { 23 | selector = `${nodeTagName}${selector}`; 24 | selectors[i] = { 25 | selector, 26 | score: 0 27 | } 28 | const selectorClassNames= selector.substring(nodeTagName.length+1).split('.'); 29 | siblings 30 | .filter((sibling: HTMLElement) => sibling.classList && sibling.classList.length) 31 | .map((sibling: HTMLElement) => Array.from(sibling.classList)) 32 | .forEach(classList => { 33 | const allInClasslist = selectorClassNames.every(className => classList.includes(className)); 34 | if (allInClasslist) { 35 | selectors[i].score += 1; 36 | } 37 | }); 38 | }); 39 | } else { 40 | allClassCombinations.forEach((selector, i) => { 41 | selector = `${nodeTagName}${selector}`; 42 | selectors[i] = { 43 | selector, 44 | score: 0 45 | } 46 | const selectorMatchesInRow = rowElement.querySelectorAll(selector); 47 | if (selectorMatchesInRow.length === 1 && selectorMatchesInRow[0].isSameNode(node)) { 48 | selectors[i].score += 1; 49 | } 50 | }); 51 | } 52 | if (selectors.length && selectors.some(({ score }) => score > 0)) { 53 | selectors.sort((a, b) => b.score - a.score); 54 | const highestScore = selectors[0].score; 55 | selectors = selectors.filter(({ score }) => score === highestScore); 56 | selectors.sort((a, b) => a.selector.split('.').length - b.selector.split('.').length); 57 | const shortestLength = selectors[0].selector.split('.').length; 58 | selectors = selectors.filter(({ selector }) => selector.split('.').length === shortestLength); 59 | return selectors.map(s => s.selector); 60 | } 61 | } 62 | return []; 63 | } 64 | 65 | export function generateIndexSelector(node) { 66 | const tag = node.tagName.toLowerCase(); 67 | const index = Array.prototype.indexOf.call(node.parentNode.children, node) + 1; 68 | return `${tag}:nth-child(${index})`; 69 | } 70 | 71 | export function getElementsBySelector(selector) { 72 | return document.querySelectorAll(selector); 73 | } 74 | 75 | export function areAllSiblings(node, selector) { 76 | return Array 77 | .from(getElementsBySelector(selector)) 78 | .every(element => element.parentNode.isSameNode(node.parentNode)); 79 | } 80 | 81 | export function generateClassSelectorFrom(node, from, isRow) { 82 | if (node.isSameNode(from)) { 83 | return null; 84 | } 85 | const selectors = []; 86 | let _node = node; 87 | if (isRow) { 88 | while (!_node.isSameNode(from)) { 89 | const selector = generateClassSelector(_node, isRow, from)[0] || _node.tagName.toLowerCase(); 90 | selectors.unshift(selector); 91 | if (areAllSiblings(_node, selectors.join(' '))) { 92 | return selectors.join(' '); 93 | } 94 | _node = _node.parentNode; 95 | } 96 | return selectors.join(" "); 97 | } 98 | return generateClassSelector(_node, isRow, from)[0] 99 | } 100 | 101 | export function generateIndexSelectorFrom(node, from) { 102 | if (node.isSameNode(from)) { 103 | return null; 104 | } 105 | const selectors = []; 106 | let _node = node; 107 | while (!_node.isSameNode(from)) { 108 | selectors.unshift(generateIndexSelector(_node)); 109 | _node = _node.parentNode; 110 | } 111 | return selectors.join('>'); 112 | } 113 | 114 | export function inSelectorElements({ selector, node }) { 115 | const result = Array 116 | .from(document.querySelectorAll(selector)) 117 | .filter(element => element.contains(node)); 118 | return result.length === 1; 119 | } 120 | -------------------------------------------------------------------------------- /src/end_user_scraper/generalizer.ts: -------------------------------------------------------------------------------- 1 | import { 2 | generateIndexSelectorFrom, 3 | generateIndexSelector, 4 | generateClassSelectorFrom, 5 | getElementsBySelector 6 | } from './domHelpers'; 7 | 8 | export function findRowElement(nodes, lca) { 9 | const candidates = []; 10 | let selectors = nodes.map(node => generateIndexSelectorFrom(node, lca)).filter(selector => selector); 11 | let candidate = lca; 12 | while (candidate && candidate.tagName !== 'BODY') { 13 | const candidateEntry = { 14 | candidate, 15 | score: 0 16 | }; 17 | let nextSibling = candidate.nextElementSibling; 18 | let previousSibling = candidate.previousElementSibling; 19 | while (nextSibling) { 20 | selectors.forEach(selector => { 21 | if (nextSibling.querySelector(selector)) { 22 | candidateEntry.score += 1; 23 | } 24 | }); 25 | nextSibling = nextSibling.nextElementSibling; 26 | } 27 | while (previousSibling) { 28 | selectors.forEach(selector => { 29 | if (previousSibling.querySelector(selector)) { 30 | candidateEntry.score += 1; 31 | } 32 | }); 33 | previousSibling = previousSibling.previousElementSibling; 34 | } 35 | candidates.push(candidateEntry); 36 | if (selectors.length) { 37 | selectors = selectors.map(selector => `${generateIndexSelector(candidate)}>${selector}`); 38 | } else { 39 | selectors = [generateIndexSelector(candidate)]; 40 | } 41 | candidate = candidate.parentNode; 42 | } 43 | if (candidates.length) { 44 | candidates.sort((a, b) => b.score - a.score); 45 | const { candidate } = candidates[0]; 46 | const rowElementSelector = generateClassSelectorFrom(candidate, document.querySelector('body'), true); 47 | return { 48 | rowElement: candidate, 49 | rowElementSelector, 50 | }; 51 | } 52 | return null 53 | } 54 | 55 | export function generateColumnSelectors(rowElementSelector, nodes) { 56 | const selectors = []; 57 | const rowElements = getElementsBySelector(rowElementSelector); 58 | for (let i = 0; i < nodes.length; i++) { 59 | for (let j = 0; j < rowElements.length; j++) { 60 | if (rowElements[j].contains(nodes[i])) { 61 | let selector; 62 | const indexSelector = generateIndexSelectorFrom(nodes[i], rowElements[j]); 63 | const classSelector = generateClassSelectorFrom(nodes[i], rowElements[j], false); 64 | const classSelectorMatches = classSelector ? rowElements[j].querySelectorAll(classSelector) : []; 65 | if (classSelectorMatches.length === 1 && classSelectorMatches[0].isSameNode(nodes[i])) { 66 | selector = classSelector; 67 | } else { 68 | selector = indexSelector; 69 | } 70 | //const selector = generateIndexSelectorFrom(nodes[i], rowElements[j]); 71 | selectors.push(selector); 72 | break; 73 | } 74 | } 75 | } 76 | return selectors; 77 | } -------------------------------------------------------------------------------- /src/end_user_scraper/index.ts: -------------------------------------------------------------------------------- 1 | import { 2 | run 3 | } from '../wildcard'; 4 | 5 | import { 6 | addScrapingListeners, 7 | removeScrapingListeners 8 | } from './eventListeners'; 9 | 10 | import { 11 | deleteAdapter, 12 | createAdapterKey, 13 | saveAdapter, 14 | createInitialAdapterConfig 15 | } from './adapterHelpers'; 16 | 17 | import { 18 | initTutorial, 19 | removeTutorial, 20 | resetTutorial 21 | } from './tutorial'; 22 | 23 | import { 24 | getAdapterKey, 25 | getCachedActiveAdapter, 26 | initState, 27 | resetScraperState, 28 | setCreatingAdapter 29 | } from './state'; 30 | 31 | import { readFromChromeLocalStorage } from '../utils'; 32 | 33 | export function startScrapingListener() { 34 | addScrapingListeners(); 35 | initTutorial(); 36 | setCreatingAdapter(true); 37 | run(); 38 | } 39 | 40 | export function stopScrapingListener({ save }) { 41 | const adapterKey = getAdapterKey(); 42 | removeScrapingListeners(); 43 | if (!save) { 44 | deleteAdapter(adapterKey, () => { 45 | resetScraperState(); 46 | removeTutorial(); 47 | run(); 48 | }); 49 | } else { 50 | const activeAdapter = getCachedActiveAdapter(); 51 | const adapterConfig = activeAdapter.getConfig(); 52 | adapterConfig.attributes.pop(); 53 | adapterConfig.metadata.columnSelectors.pop(); 54 | adapterConfig.scrapePage = adapterConfig.scrapePage.toString(); 55 | saveAdapter(adapterKey, adapterConfig, () => { 56 | resetScraperState(); 57 | removeTutorial(); 58 | run(); 59 | }); 60 | } 61 | } 62 | 63 | export function resetScrapingListener() { 64 | resetScraperState(); 65 | resetTutorial(); 66 | const activeAdapter = getCachedActiveAdapter(); 67 | if (activeAdapter) { 68 | const config = createInitialAdapterConfig() 69 | activeAdapter.updateConfig(config); 70 | } 71 | } 72 | 73 | export function editScraper() { 74 | const cachedActiveAdapter = getCachedActiveAdapter(); 75 | if (cachedActiveAdapter) { 76 | const adapterConfig = cachedActiveAdapter.getConfig(); 77 | const adapterMetadata = adapterConfig.metadata; 78 | initState(adapterMetadata); 79 | addScrapingListeners(); 80 | initTutorial(); 81 | } 82 | // const adapterKey = createAdapterKey(); 83 | // readFromChromeLocalStorage([adapterKey]) 84 | // .then((results) => { 85 | // const adapterConfigString = results[adapterKey]; 86 | // if (adapterConfigString) { 87 | // const adapterConfig = JSON.parse(adapterConfigString); 88 | // const adapterMetadata = adapterConfig.metadata; 89 | // initState(adapterMetadata); 90 | // addScrapingListeners(); 91 | // initTutorial(); 92 | // run({ creatingAdapter: true }); 93 | // } 94 | // }); 95 | } -------------------------------------------------------------------------------- /src/end_user_scraper/state.ts: -------------------------------------------------------------------------------- 1 | import { 2 | copyMap, 3 | mapToArrayOfValues, 4 | randomRGB 5 | } from './utils'; 6 | 7 | import { 8 | MIN_COLUMNS, 9 | ACTIVE_COLOR, 10 | INACTIVE_COLOR 11 | } from './constants'; 12 | 13 | import { 14 | styleColumnElementsOnClick, 15 | styleRowElementsOnClick 16 | } from './eventListeners'; 17 | import { updateAdapter } from './adapterHelpers'; 18 | 19 | const _eventMaps = { 20 | mouseMoveRowElement: new Map(), 21 | mouseMoveColumnElement: new Map(), 22 | mouseClickRowElement: new Map(), 23 | mouseClickColumnElement: new Map(), 24 | defaults: new Map() 25 | }; 26 | let _tempColumnMap; 27 | let _columnColorsList = []; 28 | let _adapterKey; 29 | let _rowElementSelector; 30 | let _rowElement; 31 | let _column = 0; 32 | let _exploring = true; 33 | let _currentColumnSelector; 34 | let _multipleExamples = false; 35 | let _columnMap = new Map(); 36 | let _editing = false; 37 | let _candidateRowElementSelectors = []; 38 | let _candidateColumnElementSelectors = []; 39 | let _activeAdapter; 40 | let _rowElementSelectorCandidates; 41 | let _columnElementSelectorCandidates = []; 42 | let _creatingAdapter = false; 43 | _columnMap.set(_column, []); 44 | 45 | export function initState({ rowSelector, columnSelectors, id }) { 46 | _editing = true; 47 | _exploring = false; 48 | _rowElementSelector = rowSelector; 49 | _rowElement = document.querySelector(rowSelector); 50 | _adapterKey = id; 51 | columnSelectors.forEach((selectors, index) => { 52 | _columnMap.set(index, selectors); 53 | }); 54 | _column = columnSelectors.length; 55 | _columnMap.set(_column, []); 56 | _tempColumnMap = copyMap(_columnMap); 57 | styleColumnElementsOnClick(rowSelector); 58 | styleRowElementsOnClick(); 59 | updateAdapter(id, mapToArrayOfValues(_columnMap), rowSelector); 60 | } 61 | 62 | export function setStyleAndAddToMap({ map, node, styleProperty, styleValue }) { 63 | if (!_eventMaps.defaults.has(node)) { 64 | _eventMaps.defaults.set(node, node.style); 65 | } 66 | map.set(node, { 67 | property: styleProperty, 68 | value: node.style[styleProperty], 69 | set: styleValue 70 | }); 71 | node.style[styleProperty] = styleValue; 72 | } 73 | 74 | export function clearElementMap(map, clear?) { 75 | map.forEach(({ property, value, set }, element) => { 76 | if (element.style) { 77 | if (clear) { 78 | if (_eventMaps.defaults.get(element)) { 79 | element.style = _eventMaps.defaults.get(element); 80 | } else { 81 | delete element.style; 82 | } 83 | } else if (element.style[property] === set) { 84 | element.style[property] = value; 85 | } 86 | } 87 | }); 88 | map.clear(); 89 | } 90 | 91 | export function clearElementMaps() { 92 | Object.keys(_eventMaps) 93 | .filter(mapKey => mapKey !== 'defaults') 94 | .forEach(mapKey => { 95 | clearElementMap(_eventMaps[mapKey], true); 96 | }); 97 | _eventMaps.defaults.clear(); 98 | } 99 | 100 | export function populateColumnColors() { 101 | for (let i = 0; i < MIN_COLUMNS; i++) { 102 | _columnColorsList.push(randomRGB()); 103 | } 104 | } 105 | 106 | export function getAdapterKey() { 107 | return _adapterKey; 108 | } 109 | 110 | export function setAdapterKey(adapterKey) { 111 | _adapterKey = adapterKey; 112 | } 113 | 114 | export function getRowElementSelector() { 115 | return _rowElementSelector; 116 | } 117 | 118 | export function setRowElementSelector(rowElementSelector) { 119 | _rowElementSelector = rowElementSelector; 120 | } 121 | 122 | export function getRowElement() { 123 | return _rowElement; 124 | } 125 | 126 | export function setRowElement(rowElement) { 127 | _rowElement = rowElement; 128 | } 129 | 130 | export function getColumn() { 131 | return _column; 132 | } 133 | 134 | export function setColumn(column) { 135 | _column = column; 136 | } 137 | 138 | export function getColumnMap() { 139 | return _columnMap; 140 | } 141 | 142 | export function setColumnMap(map) { 143 | _columnMap = new Map([...map.entries()].sort()); 144 | } 145 | 146 | export function setTempColumnMap(value) { 147 | _tempColumnMap = value; 148 | } 149 | 150 | export function getTempColumnMap() { 151 | return _tempColumnMap; 152 | } 153 | 154 | export function getColumnColor(i) { 155 | if (!_columnColorsList[i]) { 156 | _columnColorsList[i] = randomRGB(); 157 | } 158 | return _columnColorsList[i]; 159 | 160 | } 161 | 162 | export function getEventMaps() { 163 | return _eventMaps; 164 | } 165 | 166 | export function resetScraperState() { 167 | _editing = false; 168 | _adapterKey = null; 169 | _rowElement = null; 170 | _rowElementSelector = null; 171 | _column = 0; 172 | _columnMap.clear(); 173 | _columnMap.set(_column, []); 174 | clearElementMaps(); 175 | _columnColorsList = []; 176 | _exploring = true; 177 | _currentColumnSelector = null; 178 | _tempColumnMap = null; 179 | _multipleExamples = false; 180 | _candidateRowElementSelectors = []; 181 | _candidateColumnElementSelectors = []; 182 | _rowElementSelectorCandidates = []; 183 | _columnElementSelectorCandidates = []; 184 | _creatingAdapter = false; 185 | } 186 | 187 | export function getMouseClickRowStyleData() { 188 | return { 189 | styleProperty: 'border', 190 | styleValue: `2px solid ${ACTIVE_COLOR}` 191 | } 192 | } 193 | 194 | export function getMouseClickColumnStyleProperty() { 195 | return 'backgroundColor'; 196 | } 197 | 198 | export function getMouseClickColumnStyleValue() { 199 | return INACTIVE_COLOR; 200 | } 201 | 202 | export function getMouseMoveRowStyleData() { 203 | return { 204 | styleProperty: 'border', 205 | styleValue: `2px solid ${ACTIVE_COLOR}` 206 | } 207 | } 208 | 209 | export function getMouseMoveColumnStyleData() { 210 | return { 211 | styleProperty: 'backgroundColor', 212 | styleValue: ACTIVE_COLOR 213 | } 214 | } 215 | 216 | export function getExploring() { 217 | return _exploring; 218 | } 219 | 220 | export function setExploring(exploring) { 221 | _exploring = exploring; 222 | } 223 | 224 | export function getCurrentColumnSelector() { 225 | return _currentColumnSelector; 226 | } 227 | 228 | export function setCurrentColumnSelector(value) { 229 | _currentColumnSelector = value; 230 | } 231 | 232 | export function getMultipleExamples() { 233 | return _multipleExamples; 234 | } 235 | 236 | export function setMultipleExamples(value) { 237 | _multipleExamples = value; 238 | } 239 | 240 | export function getEditing() { 241 | return _editing; 242 | } 243 | 244 | export function setEditing(value) { 245 | _editing = value; 246 | } 247 | 248 | export function getCandidateRowElementSelectors() { 249 | return _candidateRowElementSelectors; 250 | } 251 | 252 | export function setCandidateRowElementSelectors(value) { 253 | _candidateRowElementSelectors = value; 254 | } 255 | 256 | export function getCandidateColumnElementSelectors() { 257 | return _candidateColumnElementSelectors; 258 | } 259 | 260 | export function setCandidateColumnElementSelectors(value) { 261 | _candidateColumnElementSelectors = value; 262 | } 263 | 264 | export function getCachedActiveAdapter() { 265 | return _activeAdapter; 266 | } 267 | 268 | export function setCachedActiveAdapter(value) { 269 | _activeAdapter = value; 270 | } 271 | 272 | export function getRowElementSelectorCandidates() { 273 | return _rowElementSelectorCandidates; 274 | } 275 | 276 | export function setRowElementSelectorCandidates(value) { 277 | _rowElementSelectorCandidates = value; 278 | } 279 | 280 | export function getColumnElementSelectorCandidates() { 281 | return _columnElementSelectorCandidates; 282 | } 283 | 284 | export function setColumnElementSelectorCandidates(value) { 285 | _columnElementSelectorCandidates = value; 286 | } 287 | 288 | export function getCreatingAdapter() { 289 | return _creatingAdapter; 290 | } 291 | 292 | export function setCreatingAdapter(value) { 293 | _creatingAdapter = value; 294 | } -------------------------------------------------------------------------------- /src/end_user_scraper/tutorial.ts: -------------------------------------------------------------------------------- 1 | import { 2 | htmlToElement 3 | } from '../utils'; 4 | 5 | import { 6 | getColumn, 7 | setColumn, 8 | getColumnMap, 9 | setMultipleExamples, 10 | getEventMaps, 11 | clearElementMap, 12 | getEditing 13 | } from './state'; 14 | 15 | import { 16 | ACTIVE_COLOR, 17 | INACTIVE_COLOR 18 | } from './constants'; 19 | 20 | import { indexToAlpha } from './utils'; 21 | 22 | import { 23 | styleRowElementsOnHover 24 | } from './eventListeners'; 25 | 26 | const TUTORIAL_BACKGROUND_COLOR = 'rgb(255, 255, 255)'; 27 | const TUTORIAL_TEXT_COLOR = 'rgb(0, 0, 0)'; 28 | 29 | 30 | 31 | let _tutorialElement; 32 | let _scraperControlsElement; 33 | 34 | function getTutorialHTMLString() { 35 | const editting = getEditing(); 36 | const cancelButtonText = editting ? 'Delete' : 'Cancel'; 37 | const saveButtonText = editting ? 'Save' : 'Done'; 38 | return ` 39 |
40 |
41 | 42 | Alt + click (option instead of alt on Mac) on a field you wish to scrape 43 | 44 |
45 |
46 |
47 |
48 |
49 | 50 | 51 | 52 |
53 |
54 |
55 | `; 56 | } 57 | 58 | function createColumnBoxString({ column, color, index }) { 59 | return ` 60 |
61 | ${column} 62 |
63 | ` 64 | } 65 | 66 | function createColumnBoxElement({ column, color, index }) { 67 | const html = createColumnBoxString({ column, color, index }); 68 | return htmlToElement(html); 69 | } 70 | 71 | function columnBoxListener(event) { 72 | const target = event.target; 73 | if (target.classList.contains('column-box')) { 74 | const columnMap = getColumnMap(); 75 | const column = parseInt(target.id); 76 | setColumn(column); 77 | if (column === columnMap.size - 1) { 78 | setMultipleExamples(false); 79 | } else { 80 | setMultipleExamples(true); 81 | } 82 | renderColumnBoxes(columnMap); 83 | const eventMaps = getEventMaps(); 84 | clearElementMap(eventMaps.mouseMoveRowElement); 85 | styleRowElementsOnHover(); 86 | } 87 | } 88 | 89 | function clearColumnBoxes() { 90 | const columnContainer = document.querySelector('#columnContainer'); 91 | columnContainer.innerHTML = ''; 92 | removeColumnBoxListener(); 93 | } 94 | 95 | function populateColumnBoxes(columnMap, column?) { 96 | const columnContainer = document.querySelector('#columnContainer'); 97 | const columns = columnMap.size; 98 | const activeColumn = Number.isInteger(column) ? column : getColumn(); 99 | if (columns) { 100 | for (let i = 0; i < columns; i++) { 101 | const columnBoxElement = createColumnBoxElement({ 102 | column: indexToAlpha(i), 103 | color: i === activeColumn ? ACTIVE_COLOR : INACTIVE_COLOR, 104 | index: i 105 | }); 106 | columnContainer.appendChild(columnBoxElement); 107 | } 108 | addColumnBoxListener(); 109 | } 110 | } 111 | 112 | function addColumnBoxListener() { 113 | if (_tutorialElement) { 114 | document.querySelector('#columnContainer').addEventListener('click', columnBoxListener); 115 | } 116 | } 117 | 118 | function removeColumnBoxListener(){ 119 | if (_tutorialElement) { 120 | document.querySelector('#columnContainer').addEventListener('click', columnBoxListener); 121 | } 122 | } 123 | 124 | export function renderColumnBoxes(columnMap, column?) { 125 | clearColumnBoxes(); 126 | populateColumnBoxes(columnMap, column); 127 | } 128 | 129 | function createColumnLabel(columnIndex) { 130 | return String.fromCharCode(97 + columnIndex).toUpperCase() 131 | } 132 | 133 | function columnControlListener(event){ 134 | const columnMap = getColumnMap(); 135 | const column = getColumn(); 136 | if (event.target.id === 'prevButton') { 137 | const proposed = column - 1; 138 | if (columnMap.has(proposed)) { 139 | setColumn(proposed); 140 | _scraperControlsElement.querySelector('#columnNumber').textContent = createColumnLabel(proposed); 141 | } else { 142 | alert(`You are trying to switch to an invalid column.`); 143 | } 144 | } else { 145 | const proposed = column + 1; 146 | if (!columnMap.get(column) || (columnMap.get(column) && !columnMap.get(column).length)) { 147 | alert(`Please select fields for column ${createColumnLabel(column)} before moving to column ${createColumnLabel(proposed)}.`); 148 | return; 149 | } 150 | if (!columnMap.has(proposed)) { 151 | columnMap.set(proposed, []); 152 | } 153 | setColumn(proposed); 154 | _scraperControlsElement.querySelector('#columnNumber').textContent = createColumnLabel(proposed); 155 | } 156 | } 157 | 158 | function scraperControlsListener(event) { 159 | switch (event.target.id) { 160 | case 'startOverButton': 161 | chrome.runtime.sendMessage({ command: 'resetAdapter'}) 162 | break; 163 | case 'cancelButton': 164 | chrome.runtime.sendMessage({ command: 'deleteAdapter'}) 165 | break; 166 | case 'saveButton': 167 | chrome.runtime.sendMessage({ command: 'saveAdapter'}) 168 | break; 169 | default: 170 | break; 171 | } 172 | } 173 | 174 | function addColumnControlListeners() { 175 | if (_tutorialElement) { 176 | _tutorialElement.querySelector('#startOverButton').addEventListener('click', scraperControlsListener); 177 | _tutorialElement.querySelector('#cancelButton').addEventListener('click', scraperControlsListener); 178 | _tutorialElement.querySelector('#saveButton').addEventListener('click', scraperControlsListener); 179 | } 180 | } 181 | 182 | function removeColumnControlListeners() { 183 | if (_tutorialElement) { 184 | _tutorialElement.querySelector('#startOverButton').removeEventListener('click', scraperControlsListener); 185 | _tutorialElement.querySelector('#cancelButton').removeEventListener('click', scraperControlsListener); 186 | _tutorialElement.querySelector('#saveButton').removeEventListener('click', scraperControlsListener); 187 | } 188 | } 189 | 190 | export function getTutorialElement() { 191 | return _tutorialElement; 192 | } 193 | 194 | export function initTutorial() { 195 | _tutorialElement = htmlToElement(getTutorialHTMLString()); 196 | document.body.prepend(_tutorialElement); 197 | addColumnControlListeners(); 198 | renderColumnBoxes(getColumnMap()); 199 | } 200 | 201 | export function removeTutorial() { 202 | if (_tutorialElement) { 203 | clearColumnBoxes(); 204 | removeColumnControlListeners(); 205 | _tutorialElement.remove(); 206 | _tutorialElement = null; 207 | } 208 | } 209 | 210 | export function resetTutorial() { 211 | removeTutorial(); 212 | initTutorial(); 213 | } -------------------------------------------------------------------------------- /src/end_user_scraper/utils.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ADAPTERS_BASE_KEY 3 | } from './constants'; 4 | import { getColumnMap } from './state'; 5 | 6 | export function randomRGB() { 7 | const o = Math.round, r = Math.random, s = 255; 8 | return 'rgba(' + o(r()*s) + ',' + o(r()*s) + ',' + o(r()*s) + ')'; 9 | } 10 | 11 | export function getAllCombinations(chars) { 12 | const result = []; 13 | const f = (prefix, chars) => { 14 | for (let i = 0; i < chars.length; i++) { 15 | result.push(`${prefix}.${chars[i]}`); 16 | f(`${prefix}.${chars[i]}`, chars.slice(i + 1)); 17 | } 18 | }; 19 | f('', chars); 20 | return result; 21 | } 22 | 23 | export function mapToArrayOfValues(map) { 24 | const result: Array> = []; 25 | map.forEach((value) => { 26 | result.push(value); 27 | }) 28 | return result; 29 | } 30 | 31 | export function copyMap(map) { 32 | const result = new Map(); 33 | for (let [column, selectors] of map) { 34 | if (Array.isArray(selectors)) { 35 | result.set(column, [...selectors]); 36 | } else { 37 | result.set(column, selectors); 38 | } 39 | } 40 | return result; 41 | } 42 | 43 | export function applyToColumnMap(map) { 44 | const columnMap = getColumnMap(); 45 | for (let [column, selectors] of map) { 46 | if (Array.isArray(selectors)) { 47 | columnMap.set(column, [...selectors]); 48 | } else { 49 | columnMap.set(column, selectors); 50 | } 51 | } 52 | } 53 | 54 | export function generateAdapterKey(id) { 55 | return `${ADAPTERS_BASE_KEY}:${id}` 56 | } 57 | 58 | export function newSelector(selector, columnMap) { 59 | let seen = false; 60 | for (let [column, selectors] of columnMap) { 61 | if (Array.isArray(selectors)) { 62 | if (selectors.includes(selector)) { 63 | seen = true; 64 | break; 65 | } 66 | } 67 | } 68 | return !seen; 69 | } 70 | 71 | export function getColumnForSelector(columnMap, selector) { 72 | for (let [column, selectors] of columnMap) { 73 | if (Array.isArray(selectors)) { 74 | if (selectors.includes(selector)) { 75 | return column; 76 | } 77 | } 78 | } 79 | return null; 80 | } 81 | 82 | export function indexToAlpha(i) { 83 | return String.fromCharCode(97 + i).toUpperCase(); 84 | } 85 | 86 | export function getSelectorFromQueryFormula({ formula }) { 87 | const regex = /\=QuerySelector\(rowElement,\s*"(.+)"\)/; 88 | const matches = formula.match(regex); 89 | if (Array.isArray(matches)){ 90 | return matches[1]; 91 | } 92 | return null; 93 | } 94 | 95 | export function isFormula(value) { 96 | return value.startsWith("="); 97 | } -------------------------------------------------------------------------------- /src/localStorageAdapter.ts: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | declare const browser; 3 | 4 | import { TableAdapter, Record, Attribute, TableCallback, RecordEdit } from './core/types' 5 | import { createDomScrapingAdapter } from './site_adapters/domScrapingBase'; 6 | import { readFromChromeLocalStorage, compileJavascript } from './utils' 7 | 8 | const emptyTable = { 9 | tableId: "user", 10 | attributes: [], 11 | records: [] 12 | } 13 | 14 | let table = emptyTable 15 | 16 | // todo: could we use tableId instead of a separate "namespace" here? 17 | // especially because we might not always want to launch the same user table 18 | // for the same site adapter. we should add some indirection somewhere, 19 | // so a site adapter loads a specific user table by default, but 20 | // you could install someone else's user table, or even have multiple 21 | // possible user tables stored within a single adapter, all joined together 22 | let namespace; 23 | 24 | let subscribers:Array = [] 25 | 26 | const storageKey = () => `localStorageAdapter:${namespace}` 27 | 28 | const loadTable = () => { 29 | chrome.storage.local.set({ [storageKey()]: table }); 30 | for (const callback of subscribers) { callback(table); } 31 | return table; 32 | } 33 | 34 | const editRecords = (edits:Array) => { 35 | for (const { recordId, attribute, value } of edits) { 36 | let newRecords : Array; 37 | 38 | // todo: this does two passes, inefficient 39 | const existingRecord = table.records.find(r => r.id === recordId) 40 | if (existingRecord) { 41 | newRecords = table.records.map(r => { 42 | if (r.id === recordId) { 43 | return { 44 | id: r.id, 45 | values: { ...r.values, [attribute]: value } 46 | } 47 | } 48 | else { return r; } 49 | }) 50 | } else { 51 | newRecords = [...table.records, 52 | { id: recordId, values: { [attribute]: value } } 53 | ] 54 | } 55 | 56 | table = { ...table, records: newRecords } 57 | } 58 | return Promise.resolve(loadTable()); 59 | } 60 | 61 | export const userStore:TableAdapter = { 62 | tableId: "user", 63 | name: "User Local Storage", 64 | initialize: (ns) => { 65 | namespace = ns; 66 | chrome.storage.local.get([storageKey()], (result) => { 67 | const tableFromStorage = result[storageKey()]; 68 | if (tableFromStorage) { table = tableFromStorage; loadTable(); } 69 | }) 70 | }, 71 | enabled: () => true , // user store is always enabled 72 | clear: () => { 73 | table = emptyTable 74 | loadTable() 75 | }, 76 | loadTable: loadTable, 77 | subscribe(callback:TableCallback) { 78 | subscribers = [...subscribers, callback]; 79 | }, 80 | editRecords: editRecords, 81 | addAttribute() { 82 | const newAttribute : Attribute = { 83 | name: "user" + (table.attributes.length + 1), 84 | type: "text", 85 | editable: true, 86 | hideInPage: false 87 | } 88 | 89 | table = { ...table, attributes: [...table.attributes, newAttribute] } 90 | 91 | loadTable(); 92 | 93 | return Promise.resolve(table); 94 | }, 95 | toggleVisibility(colName) { 96 | 97 | var curr = table.attributes.find((attribute) => (attribute.name === colName)); 98 | curr.hideInPage = !curr.hideInPage; 99 | 100 | loadTable(); 101 | 102 | return; 103 | }, 104 | setFormula(attrName, formula) { 105 | table = { ...table, 106 | attributes: table.attributes.map(attr => attr.name === attrName ? { ...attr, formula } : attr )} 107 | 108 | loadTable(); 109 | }, 110 | 111 | // These changes to the table are no-ops 112 | // todo: should these move off the generic table adapter interface? 113 | // should they only apply to dom adapters? 114 | handleRecordSelected() {}, //no-op 115 | applySort() {}, 116 | handleOtherTableUpdated() {}, 117 | } 118 | 119 | export const adapterStore = { 120 | getLocalAdapters: async () => { 121 | const localAdaptersKey = 'localStorageAdapter:adapters'; 122 | const result = []; 123 | try { 124 | const localAdapters = (await readFromChromeLocalStorage([localAdaptersKey]) as Object)[localAdaptersKey] || []; 125 | for (let i = 0; i < localAdapters.length; i++) { 126 | const adapter = localAdapters[i]; 127 | const localAdapterKey = `${localAdaptersKey}:${adapter}`; 128 | const adapterConfigString = (await readFromChromeLocalStorage([localAdapterKey]) as Object)[localAdapterKey]; 129 | // sometimes we can end up with malformed adapters; just ignore and keep going 130 | if(!adapterConfigString) continue; 131 | const adapterConfig = JSON.parse(adapterConfigString); 132 | compileAdapterJavascript(adapterConfig); 133 | const localAdapter = createDomScrapingAdapter(adapterConfig); 134 | result.push(localAdapter); 135 | } 136 | } catch(error){ 137 | console.error('error while retrieving local adapters:', error); 138 | } 139 | return result; 140 | } 141 | } 142 | 143 | export function compileAdapterJavascript(adapterConfig) { 144 | const keysToEvaluate = ['scrapePage', 'onRowSelected', 'onRowUnselected', 'addScrapeTriggers']; 145 | Object.keys(adapterConfig) 146 | .filter(key => keysToEvaluate.includes(key)) 147 | .forEach(key => { 148 | adapterConfig[key] = compileJavascript(adapterConfig[key]); 149 | }); 150 | } 151 | -------------------------------------------------------------------------------- /src/marketplace.js: -------------------------------------------------------------------------------- 1 | const localAdaptersKey = 'localStorageAdapter:adapters'; 2 | let LOCAL_ADAPTERS; 3 | 4 | window.onload = function(e){ 5 | const queryString = window.location.search; 6 | const urlParams = new URLSearchParams(queryString); 7 | const key = urlParams.get('key'); 8 | 9 | // don't do anything if key is not available 10 | if (key == null || key == "") 11 | return; 12 | 13 | // read adapter code from storage 14 | chrome.storage.local.get(key, (results) => { 15 | const code = results[key]; 16 | 17 | // populate fields 18 | document.getElementById("code").textContent = code; 19 | 20 | // get name and url from adapter code 21 | 22 | // const scraper = new Function(`return ${code}`)(); 23 | // document.getElementById("name").value = scraper.name; 24 | // document.getElementById("url").value = scraper.contains; 25 | 26 | const json = JSON.parse(code); 27 | document.getElementById("name").value = json.name; 28 | document.getElementById("url").value = json.urls[0]; 29 | 30 | document.getElementById("upload").removeAttribute('disabled'); 31 | }); 32 | 33 | } 34 | 35 | // listen for custom event to ensure adapter info has been parsed 36 | document.addEventListener('adapterReady', function (e) { 37 | const adapterName = document.getElementById("adapterName").textContent; 38 | const adapterCode = document.getElementById("adapterCode").textContent; 39 | const url = document.getElementById("url").textContent; 40 | 41 | const installBtn = document.getElementById("install"); 42 | const uninstallBtn = document.getElementById("uninstall"); 43 | const status = document.getElementById('status'); 44 | 45 | // check if adapter installed 46 | // load LOCAL_ADAPTERS 47 | chrome.storage.local.get([localAdaptersKey], (results) => { 48 | LOCAL_ADAPTERS = results[localAdaptersKey]; 49 | // initialize LOCAL_ADAPTERS if needed 50 | if (LOCAL_ADAPTERS === undefined) { 51 | LOCAL_ADAPTERS = []; 52 | installBtn.removeAttribute('disabled'); 53 | } else if (Array.isArray(LOCAL_ADAPTERS) && LOCAL_ADAPTERS.indexOf(adapterName) > -1) { 54 | // hide Install button 55 | installBtn.style.display = "none"; 56 | uninstallBtn.style.display = "inline"; 57 | status.textContent = "Adapter already installed."; 58 | } else { 59 | installBtn.removeAttribute('disabled'); 60 | } 61 | }); 62 | 63 | // add onclick to install and uninstall buttons 64 | installBtn.onclick = function () { 65 | // if adapterName not exist, update LOCAL_ADAPTERS and create one 66 | if (Array.isArray(LOCAL_ADAPTERS) && LOCAL_ADAPTERS.indexOf(adapterName) === -1) { 67 | LOCAL_ADAPTERS.push(adapterName); 68 | chrome.storage.local.set({ [localAdaptersKey]: LOCAL_ADAPTERS }, function () { 69 | const key = `${localAdaptersKey}:${adapterName}`; 70 | chrome.storage.local.set({ [key]: adapterCode }, () => { 71 | // Update status to let user know adapter was installed. 72 | status.textContent = "Adapter installed."; 73 | installBtn.style.display = "none"; 74 | uninstallBtn.style.display = "inline"; 75 | 76 | // open a window with the target url with delay 77 | setTimeout(() => { 78 | window.open(url); 79 | }, 1000); 80 | }); 81 | }); 82 | } 83 | } 84 | 85 | uninstallBtn.onclick = function () { 86 | // if adapterName exists, remove it from storage 87 | if (Array.isArray(LOCAL_ADAPTERS) && LOCAL_ADAPTERS.indexOf(adapterName) !== -1) { 88 | LOCAL_ADAPTERS.splice(LOCAL_ADAPTERS.indexOf(adapterName), 1); 89 | chrome.storage.local.set({ [localAdaptersKey]: LOCAL_ADAPTERS }); 90 | 91 | const key = `${localAdaptersKey}:${adapterName}`; 92 | chrome.storage.local.remove(key, () => { 93 | status.textContent = "Adapter removed."; 94 | // hide Uninstall button, show Install button 95 | installBtn.removeAttribute('disabled'); 96 | installBtn.style.display = "inline"; 97 | uninstallBtn.style.display = "none"; 98 | }); 99 | } 100 | } 101 | }); 102 | -------------------------------------------------------------------------------- /src/site_adapters/airbnb.ts: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import { urlContains, extractNumber } from "../utils" 4 | import { createDomScrapingAdapter } from "./domScrapingBase" 5 | 6 | const rowContainerClass = "_fhph4u" 7 | const rowClass = "_8ssblpx" 8 | const titleClass = "_1jbo9b6h" 9 | const titleClassBackup = "_1c2n35az" 10 | const priceClass = "_1p7iugi" 11 | const ratingClass = "_3zgr580" 12 | const ratingClassBackup = "_10fy1f8" 13 | const listingLinkClass = "_i24ijs" 14 | 15 | const AirbnbAdapter = createDomScrapingAdapter({ 16 | name: "Airbnb", 17 | enabled: () => urlContains("airbnb.com/s"), 18 | attributes: [ 19 | { name: "id", type: "text", hidden: true }, 20 | { name: "name", type: "text" }, 21 | { name: "price", type: "numeric" }, 22 | { name: "rating", type: "numeric" }, 23 | {name: "latitude", type: "numeric"}, 24 | {name: "longitude", type: "numeric"} 25 | ], 26 | scrapePage: () => { 27 | return Array.from(document.getElementsByClassName(rowClass)).map(el => { 28 | let path = el.querySelector("." + listingLinkClass).getAttribute('href') 29 | let id = path.match(/\/rooms\/([0-9]*)\?/)[1] 30 | let rating = el.querySelector(`.${ratingClass}`) !== null 31 | ? el.querySelector(`.${ratingClass}`) 32 | : el.querySelector(`.${ratingClassBackup}`); 33 | 34 | return { 35 | id: id, 36 | rowElements: [el], 37 | dataValues: { 38 | name: el.querySelector(`.${titleClass}`) !== null 39 | ? el.querySelector(`.${titleClass}`) 40 | : el.querySelector(`.${titleClassBackup}`), 41 | price: el.querySelector(`.${priceClass}`).textContent.match(/\$([\d]*)/)[1], 42 | rating: extractNumber(rating) 43 | }, 44 | annotationContainer: el.querySelector("div._kqh46o") as HTMLElement, 45 | annotationTemplate: ` · $annotation` 46 | } 47 | }) 48 | }, 49 | scrapeAjax: (request) => { 50 | if(request.url.includes("https://www.airbnb.com/api/v3?")){ 51 | try{ 52 | let listings = request.data.data.dora.exploreV3.sections["1"].items; 53 | return Object.keys(listings).map(key => { 54 | let listing = listings[key].listing; 55 | 56 | return { 57 | id: listing.id, 58 | dataValues: { 59 | latitude: listing.lat, 60 | longitude: listing.lng 61 | } 62 | } 63 | }); 64 | } 65 | catch{ 66 | 67 | } 68 | } 69 | return undefined; 70 | }, 71 | }); 72 | 73 | export default AirbnbAdapter; 74 | 75 | -------------------------------------------------------------------------------- /src/site_adapters/amazon.ts: -------------------------------------------------------------------------------- 1 | import { urlContains } from '../utils' 2 | import { createDomScrapingAdapter } from "./domScrapingBase" 3 | 4 | const rowContainerID = "olpOfferList"; 5 | const rowClass = "a-row a-spacing-mini olpOffer"; 6 | const priceClass = "a-column a-span2 olpPriceColumn"; 7 | const shippingPriceClass = "olpShippingPrice"; 8 | const estTaxClass = "olpEstimatedTaxText"; 9 | const conditionClass = "a-size-medium olpCondition a-text-bold"; 10 | const arrivalClass = "a-expander-content a-expander-partial-collapse-content"; 11 | const sellerClass = "a-column a-span2 olpSellerColumn"; 12 | const sellerName = "a-spacing-none olpSellerName"; 13 | const ratingClass = "a-icon-alt"; 14 | 15 | export const AmazonAdapter = createDomScrapingAdapter({ 16 | name: "Amazon", 17 | enabled: () => urlContains("amazon.com/gp/offer-listing/"), 18 | attributes: [ 19 | { name: "id", type: "text", hidden: true }, 20 | { name: "total_price", editable: true, type: "numeric" }, 21 | { name: "condition", editable: true, type: "text" }, 22 | { name: "delivery_detail", editable: true, type: "text" }, 23 | { name: "rating", editable: true, type: "numeric" } 24 | ], 25 | scrapePage: () => { 26 | 27 | var group = document.getElementById(rowContainerID).getElementsByClassName(rowClass); 28 | 29 | return Array.from(group).map(el => { 30 | var price = 0; 31 | 32 | var price_el = el.getElementsByClassName(priceClass)[0]; 33 | var price_text = price_el.innerText; 34 | 35 | var start_idx = price_text.indexOf("$"); 36 | var end_idx = price_text.indexOf(" "); 37 | 38 | //find every "$" sign and add the number behind it to the total_price 39 | while(start_idx != -1){ 40 | price += parseFloat(price_text.substring(start_idx + 1, end_idx)); 41 | start_idx = price_text.indexOf("$", end_idx); 42 | end_idx = price_text.indexOf(" ", start_idx); 43 | } 44 | 45 | 46 | var delivery_el = el.getElementsByClassName(arrivalClass)[0]; 47 | var delivery_text = ""; 48 | if (delivery_el == undefined){ 49 | delivery_text = "Unavailable"; 50 | } 51 | else { 52 | delivery_text = delivery_el.innerText; 53 | } 54 | 55 | 56 | var rating_el = el.getElementsByClassName(sellerClass)[0].getElementsByClassName(ratingClass)[0]; 57 | var rating_text = ""; 58 | if (rating_el == undefined){ 59 | rating_text = "Unavailable"; 60 | } 61 | else{ 62 | rating_text = rating_el.innerText; 63 | } 64 | 65 | var seller_name = el.getElementsByClassName(sellerName)[0]; 66 | var seller_href = ""; 67 | if (seller_name.querySelector("a") === null){ 68 | seller_href = "Null"; 69 | } 70 | else{ 71 | seller_href = seller_name.querySelector("a").href; 72 | } 73 | 74 | var condition = el.getElementsByClassName(conditionClass)[0]; 75 | console.log(condition); 76 | var cond_text = ""; 77 | if (condition == undefined){ 78 | cond_text = "Unavailable"; 79 | } 80 | else{ 81 | cond_text = condition.innerText; 82 | } 83 | 84 | 85 | return { 86 | id: seller_href, 87 | rowElements: [el], 88 | dataValues: { 89 | total_price: price.toFixed(2), 90 | condition: cond_text, 91 | delivery_detail: delivery_text, 92 | rating: rating_text 93 | } 94 | } 95 | }) 96 | } 97 | }); 98 | 99 | export default AmazonAdapter; 100 | -------------------------------------------------------------------------------- /src/site_adapters/blogger.ts: -------------------------------------------------------------------------------- 1 | import { RichTextEditor } from '../ui/cell_editors/richTextEditor.js' 2 | import { createDomScrapingAdapter } from "./domScrapingBase" 3 | import {urlContains} from "../utils"; 4 | 5 | const BloggerAdapter = createDomScrapingAdapter({ 6 | name: "Blogger", 7 | enabled: () => urlContains("blogger.com"), 8 | attributes: [ 9 | { name: "id", type: "text", hidden: true }, 10 | { name: "document", editable: true, renderer: 'html', type: "text", editor: RichTextEditor }, 11 | { name: "source", editable: true, type: "text", editor: RichTextEditor }, 12 | ], 13 | scrapePage: () => { 14 | let container : HTMLElement = document.getElementById("blogger-app"); 15 | let iframeLoaded = document.querySelectorAll('iframe').length == 3; 16 | let doc = iframeLoaded ? document.querySelectorAll('iframe')[2].contentDocument.body : container.querySelector("#postingComposeBox"); 17 | return [ 18 | { 19 | id: "1", 20 | rowElements: [container], 21 | dataValues: { 22 | document: doc, 23 | source: container.querySelector("#postingHtmlBox"), 24 | } 25 | } 26 | ] 27 | }, 28 | // Reload data anytime there's a click or keypress on the page 29 | addScrapeTriggers: (reload) => { 30 | document.addEventListener("click", (e) => { reload() }); 31 | document.addEventListener("keydown", (e) => { reload() }); 32 | }, 33 | iframe: true 34 | }); 35 | 36 | export default BloggerAdapter 37 | -------------------------------------------------------------------------------- /src/site_adapters/expedia.ts: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | // todo: doesn't make sense to import cell editors here.... should be in UI? 4 | import { FullCalendarEditor } from '../ui/cell_editors/fullCalendarEditor.js' 5 | import { RichTextEditor } from '../ui/cell_editors/richTextEditor.js' 6 | import { urlContains } from '../utils' 7 | import { createDomScrapingAdapter } from "./domScrapingBase" 8 | 9 | const ExpediaAdapter = createDomScrapingAdapter({ 10 | name: "Expedia", 11 | enabled: () => urlContains("expedia.com"), 12 | attributes: [ 13 | { name: "id", type: "text", hidden: true }, 14 | { name: "origin", editable: true, type: "text" }, 15 | { name: "destination", editable: true, type: "text", editor: RichTextEditor }, 16 | { name: "departDate", editable: true, type: "text", editor: FullCalendarEditor }, 17 | { name: "returnDate", editable: true, type: "text", editor: FullCalendarEditor } 18 | ], 19 | scrapePage: () => { 20 | const form = document.getElementById("gcw-packages-form-hp-package") 21 | return [ 22 | { 23 | id: "1", // only one row so we can just hardcode an ID 24 | rowElements: [form], 25 | dataValues: { 26 | origin: form.querySelector("#package-origin-hp-package"), 27 | destination: form.querySelector("#package-destination-hp-package"), 28 | departDate: form.querySelector("#package-departing-hp-package"), 29 | returnDate: form.querySelector("#package-returning-hp-package") 30 | } 31 | } 32 | ] 33 | }, 34 | // Reload data anytime the form changes or there's a click on the page 35 | addScrapeTriggers: (loadTable) => { 36 | document.addEventListener("click", e => loadTable()) 37 | 38 | const form = document.getElementById("gcw-packages-form-hp-package") 39 | form.addEventListener("change", loadTable()) 40 | } 41 | }) 42 | 43 | export default ExpediaAdapter; 44 | 45 | -------------------------------------------------------------------------------- /src/site_adapters/flux.ts: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import { urlExact, urlContains, extractNumber, onDomReady } from "../utils"; 4 | import { createDomScrapingAdapter, ScrapedRow } from "./domScrapingBase" 5 | 6 | const FluxAdapter = createDomScrapingAdapter({ 7 | name: "Flux Valor Bruto Absorbância", 8 | 9 | enabled () { 10 | return urlContains("flux2.luar.dcc.ufmg.br/workflow") 11 | }, 12 | 13 | attributes: [ 14 | { name: "0", type: "numeric", editable: true}, 15 | { name: "1", type: "numeric", editable: true}, 16 | { name: "2", type: "numeric", editable: true}, 17 | { name: "3", type: "numeric", editable: true}, 18 | { name: "4", type: "numeric", editable: true}, 19 | { name: "5", type: "numeric", editable: true}, 20 | { name: "6", type: "numeric", editable: true}, 21 | { name: "7", type: "numeric", editable: true}, 22 | { name: "8", type: "numeric", editable: true}, 23 | { name: "9", type: "numeric", editable: true}, 24 | { name: "10", type: "numeric", editable: true}, 25 | ], 26 | 27 | scrapePage() { 28 | let result = [] 29 | let tbody = document.querySelector("div[xpdlid='valorBrutoAbs'] tbody") 30 | 31 | if (!tbody) { return null; } 32 | 33 | for (let i = 0; i < 8; i++) { 34 | const newScrapedRow:ScrapedRow = 35 | { 36 | id: String(i), 37 | rowElements: [tbody.children[i]], 38 | dataValues: { 39 | "0": document.querySelector(`#valorBrutoAbs--${i}--0`), 40 | "1": document.querySelector(`#valorBrutoAbs--${i}--1`), 41 | "2": document.querySelector(`#valorBrutoAbs--${i}--2`), 42 | "3": document.querySelector(`#valorBrutoAbs--${i}--3`), 43 | "4": document.querySelector(`#valorBrutoAbs--${i}--4`), 44 | "5": document.querySelector(`#valorBrutoAbs--${i}--5`), 45 | "6": document.querySelector(`#valorBrutoAbs--${i}--6`), 46 | "7": document.querySelector(`#valorBrutoAbs--${i}--7`), 47 | "8": document.querySelector(`#valorBrutoAbs--${i}--8`), 48 | "9": document.querySelector(`#valorBrutoAbs--${i}--9`), 49 | "10": document.querySelector(`#valorBrutoAbs--${i}--10`), 50 | } 51 | } 52 | 53 | result.push(newScrapedRow) 54 | } 55 | 56 | return result 57 | }, 58 | 59 | addScrapeTriggers (loadTable) { 60 | // listen for input changes on the table 61 | const tbody = document.querySelector("div[xpdlid='valorBrutoAbs'] tbody") 62 | if (!tbody) return; 63 | tbody.addEventListener("change", (e) => { loadTable() }); 64 | }, 65 | }); 66 | 67 | export default FluxAdapter; 68 | -------------------------------------------------------------------------------- /src/site_adapters/github.ts: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import { urlMatches, extractNumber } from "../utils"; 3 | import { createDomScrapingAdapter } from "./domScrapingBase" 4 | 5 | const GithubAdapter = createDomScrapingAdapter({ 6 | name: "Github", 7 | enabled: () => false, 8 | attributes: [ 9 | { name: "name" , type: "text"}, 10 | { name: "stars" , type: "numeric"}, 11 | { name: "forks" , type: "numeric"}, 12 | { name: "updated", type: "text"}, 13 | // TODO datetime type would be nice? not everything has ISO formatted strings 14 | ], 15 | scrapePage() { 16 | return Array.from(document.querySelectorAll("li.source")).map(el => { 17 | let name_el = el.querySelector('a[itemprop="name codeRepository"]') 18 | let name = name_el.textContent.trim() 19 | 20 | let stars_el = el.querySelector('*[href*="/stargazers"') 21 | let stars = extractNumber(stars_el, 0) 22 | 23 | let forks_el = el.querySelector('*[href*="/network/members"]') 24 | let forks = extractNumber(forks_el, 0) 25 | 26 | let lang_el = el.querySelector('*[itemprop="programmingLanguage"]') 27 | // some repos don't have language set 28 | let lang = lang_el == null ? null : lang_el.textContent.trim() 29 | 30 | let updated_el = el.querySelector('relative-time') 31 | let updated = updated_el.getAttribute('datetime') 32 | 33 | return { 34 | id: name, 35 | rowElements: [el], 36 | dataValues: { 37 | name: name, 38 | stars: stars, 39 | forks: forks, 40 | updated: updated, 41 | }, 42 | } 43 | }) 44 | }, 45 | onRowSelected: (row) => { 46 | row.rowElements.forEach(el => { 47 | if (el.style) { 48 | el.style["background-color"] = "#def3ff" 49 | } 50 | }); 51 | row.rowElements[0].scrollIntoView({ behavior: "smooth", block: "center" }) 52 | }, 53 | onRowUnselected: (row) => { 54 | row.rowElements.forEach(el => { 55 | if(el.style) { 56 | el.style["background-color"] = `` 57 | } 58 | }) 59 | }, 60 | }) 61 | 62 | export default GithubAdapter; 63 | -------------------------------------------------------------------------------- /src/site_adapters/hackerNews.ts: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | // A sample new HN site adapter. 4 | 5 | import { urlExact, urlContains, extractNumber } from "../utils"; 6 | import { createDomScrapingAdapter } from "./domScrapingBase" 7 | 8 | // Configuration options for the Hacker News adapter 9 | const HNAdapter = createDomScrapingAdapter({ 10 | name: "Hacker News", 11 | enabled () { 12 | return urlExact("news.ycombinator.com/") || 13 | urlContains("news.ycombinator.com/news") || 14 | urlContains("news.ycombinator.com/newest") 15 | }, 16 | attributes: [ 17 | { name: "id", type: "text", hidden: true }, 18 | { name: "mainRow", type: "element" }, 19 | { name: "detailsRow", type: "element" }, 20 | { name: "rank", type: "numeric" }, 21 | { name: "title", type: "element", formula: `=QuerySelector(mainRow, "a.storylink")` }, 22 | { name: "link", type: "text", formula: `=GetAttribute(title, "href")` }, 23 | { name: "points", type: "numeric", formula: `=QuerySelector(detailsRow, "span.score")` }, 24 | { name: "user", type: "text" }, 25 | { name: "comments", type: "numeric" } 26 | ], 27 | scrapePage() { 28 | return Array.from(document.querySelectorAll("tr.athing")).map(el => { 29 | let detailsRow = el.nextElementSibling 30 | let spacerRow = detailsRow.nextElementSibling 31 | 32 | return { 33 | id: String(el.getAttribute("id")), 34 | rowElements: [el, detailsRow, spacerRow], 35 | // todo: Both of these steps should be handled by the framework... 36 | // .filter(e => e) // Only include if the element is really there 37 | // .map(e => (e)), // Convert to HTMLElement type 38 | dataValues: { 39 | mainRow: el, 40 | detailsRow: detailsRow, 41 | rank: el.querySelector("span.rank"), 42 | title: el.querySelector("a.storylink"), 43 | link: el.querySelector("a.storylink").getAttribute("href"), 44 | // These elements contain text like "162 points"; 45 | // Wildcard takes care of extracting a number automatically. 46 | points: detailsRow.querySelector("span.score"), 47 | user: detailsRow.querySelector("a.hnuser"), 48 | comments: extractNumber(Array.from(detailsRow.querySelectorAll("a")) 49 | .find(e => e.textContent.indexOf("comment") !== -1), 0) 50 | }, 51 | annotationContainer: detailsRow.querySelector("td.subtext") as HTMLElement, 52 | annotationTemplate: `| $annotation` 53 | } 54 | }) 55 | }, 56 | onRowSelected: (row) => { 57 | row.rowElements.forEach(el => { 58 | if (el.style) { 59 | el.style["background-color"] = "#def3ff" 60 | } 61 | }); 62 | row.rowElements[0].scrollIntoView({ behavior: "smooth", block: "center" }) 63 | }, 64 | onRowUnselected: (row) => { 65 | row.rowElements.forEach(el => { 66 | if(el.style) { 67 | el.style["background-color"] = `` 68 | } 69 | }) 70 | }, 71 | }) 72 | 73 | export default HNAdapter; 74 | -------------------------------------------------------------------------------- /src/site_adapters/harvardbookwarehouse.ts: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import { urlContains } from "../utils"; 4 | import { createDomScrapingAdapter } from "./domScrapingBase" 5 | 6 | // Configuration options for the Hacker News adapter 7 | const HarvardBookWarehouse = createDomScrapingAdapter({ 8 | name: "Harvard Book Warehouse", 9 | enabled() { 10 | return urlContains("https://hbswarehousesale.com/") 11 | }, 12 | attributes: [ 13 | { name: "id", type: "text", hidden: true }, 14 | { name: "title", type: "text" }, 15 | { name: "priceRegular", type: "numeric" }, 16 | { name: "priceSale", type: "numeric" }, 17 | { name: "soldOut", type: "checkbox" }, 18 | ], 19 | scrapePage() { 20 | return Array.from(document.querySelectorAll(".grid__item.grid__item--collection-template")).map(el => { 21 | return { 22 | id: el.querySelector("a.grid-view-item__link").getAttribute('href'), 23 | rowElements: [el], 24 | dataValues: { 25 | title: el.querySelector("a.grid-view-item__link span.visually-hidden"), 26 | priceRegular: el.querySelector(".price__regular .price-item--regular"), 27 | priceSale: el.querySelector(".price__sale .price-item--regular"), 28 | soldOut: el.querySelector(".price__badge--sold-out") && 29 | window.getComputedStyle(el.querySelector(".price__badge--sold-out")).getPropertyValue('display') !== 'none' 30 | }, 31 | } 32 | }) 33 | }, 34 | }) 35 | 36 | export default HarvardBookWarehouse; 37 | -------------------------------------------------------------------------------- /src/site_adapters/index.ts: -------------------------------------------------------------------------------- 1 | // Registry of all the site adapters 2 | 3 | import HNAdapter from './hackerNews' 4 | import FluxAdapter from './flux' 5 | // expedia adapter has a css issue with esbuild 6 | // import ExpediaAdapter from './expedia' 7 | import AirbnbAdapter from './airbnb' 8 | import AmazonAdapter from './amazon' 9 | import InstacartAdapter from './instacart' 10 | import UberEatsAdapter from './ubereats' 11 | import BloggerAdapter from './blogger' 12 | import WeatherChannelAdapter from './weatherchannel' 13 | import YoutubeAdapter from './youtube' 14 | import GithubAdapter from './github' 15 | import HarvardBookWarehouse from './harvardbookwarehouse' 16 | import { adapterStore } from '../localStorageAdapter' 17 | import { TableAdapter } from '../core/types' 18 | import { createInitialAdapter } from '../end_user_scraper/adapterHelpers' 19 | import { getCreatingAdapter } from '../end_user_scraper/state' 20 | 21 | export const siteAdapters = [ 22 | HNAdapter, 23 | FluxAdapter, 24 | // ExpediaAdapter, 25 | AirbnbAdapter, 26 | AmazonAdapter, 27 | InstacartAdapter, 28 | UberEatsAdapter, 29 | BloggerAdapter, 30 | WeatherChannelAdapter, 31 | YoutubeAdapter, 32 | GithubAdapter, 33 | HarvardBookWarehouse 34 | ] 35 | 36 | export async function getActiveAdapter(): Promise { 37 | const creatingAdapter = getCreatingAdapter(); 38 | const localAdapters = await adapterStore.getLocalAdapters(); 39 | const adaptersForPage = [ 40 | ...localAdapters, 41 | ...siteAdapters 42 | ].filter(adapter => adapter.enabled()) 43 | 44 | let activeAdapter; 45 | 46 | if (adaptersForPage.length === 0 && !creatingAdapter) { 47 | return undefined; 48 | } else if (creatingAdapter) { 49 | activeAdapter = createInitialAdapter(); 50 | } else { 51 | activeAdapter = adaptersForPage[0]; 52 | } 53 | 54 | console.log(`Wildcard: activating site adapter: ${activeAdapter.name}`); 55 | 56 | return activeAdapter; 57 | } 58 | -------------------------------------------------------------------------------- /src/site_adapters/instacart.ts: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import {urlContains} from "../utils"; 4 | import { createDomScrapingAdapter } from "./domScrapingBase" 5 | 6 | export const InstacartAdapter = createDomScrapingAdapter({ 7 | name: "Instacart", 8 | enabled: () => { 9 | return urlContains("instacart.com/store/orders") 10 | }, 11 | attributes: [ 12 | // { name: "id", type: "text" }, 13 | { name: "name", type: "text" }, 14 | { name: "price", type: "numeric" }, 15 | { name: "quantity", type: "numeric" }, 16 | ], 17 | scrapePage: () => { 18 | return Array.from(document.querySelectorAll("li.order-status-item")).map (el => { 19 | const itemName = el.querySelector("div.order-status-item-details h5").textContent; 20 | const itemPrice = el.querySelector("div.order-status-item-price p").textContent.substring(1); 21 | 22 | let itemQuantity = null; 23 | let quantityDropdown = el.querySelector("div.icDropdown button span"); 24 | let quantityText = el.querySelector("div.order-status-item-qty p"); 25 | if (quantityDropdown) { 26 | itemQuantity = quantityDropdown.textContent; 27 | } else if (quantityText) { 28 | itemQuantity = quantityText.textContent; 29 | } 30 | 31 | return { 32 | id: itemName, 33 | rowElements: [el], 34 | dataValues: { 35 | name: itemName, 36 | price: itemPrice, 37 | quantity: itemQuantity 38 | } 39 | } 40 | }) 41 | }, 42 | // Reload data anytime the form changes or there's a click on the page 43 | addScrapeTriggers: (loadTable) => { 44 | document.addEventListener("click", e => loadTable()) 45 | } 46 | }); 47 | 48 | export default InstacartAdapter; 49 | -------------------------------------------------------------------------------- /src/site_adapters/ubereats.ts: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import { extractNumber, urlExact, urlContains } from "../utils" 4 | import { createDomScrapingAdapter } from "./domScrapingBase" 5 | 6 | const UberEatsAdapter = createDomScrapingAdapter({ 7 | name: "Uber Eats", 8 | enabled: () => { 9 | return urlContains("ubereats.com") 10 | }, 11 | attributes: [ 12 | { name: "id", type: "text", hidden: true}, 13 | { name: "name", type: "text" }, 14 | { name: "eta", type: "text" }, 15 | { name: "categories", type: "text" }, 16 | { name: "price_bucket", type: "text" }, 17 | { name: "rating", type: "numeric" }, 18 | { name: "fee", type: "numeric" }, 19 | {name: "is_open", type: "text"} 20 | ], 21 | scrapePage: () => { 22 | return Array.from(document.querySelectorAll("a")).map(el => { 23 | var prefix; 24 | 25 | //check that el has restaurant 26 | if (el.getAttribute("href").includes("food-delivery/") == true){ 27 | 28 | var url_elts = el.getAttribute("href").split("/"); 29 | var title = url_elts[url_elts.length-2]; 30 | console.log(title); 31 | 32 | var restaurant_el = el.childNodes[0]; 33 | var restaurant = restaurant_el.textContent; 34 | 35 | return { 36 | id: title.toString(), 37 | rowElements: [el], 38 | dataValues: { 39 | name: restaurant 40 | }, 41 | } 42 | 43 | } 44 | }).filter(row => row != undefined); 45 | }, 46 | 47 | scrapeAjax: (request) => { 48 | if(request.url.includes("https://www.ubereats.com/api/getFeedV1")){ 49 | try{ 50 | let listings = request.data.data.storesMap; 51 | 52 | return Object.keys(listings).map(key => { 53 | let listing = listings[key]; 54 | let l_eta = "Unavailable"; 55 | let l_categories = "Unavailable"; 56 | let l_price_bucket = "Unavailable"; 57 | let l_rating = 0; 58 | let l_fee = 0; 59 | 60 | if (!(listing.etaRange == null)){ 61 | l_eta = listing.etaRange.text; 62 | } 63 | 64 | if (!(listing.meta.categories == null)){ 65 | l_categories = listing.meta.categories; 66 | } 67 | 68 | if (!(listing.meta.priceBucket == null)){ 69 | l_price_bucket = listing.meta.priceBucket; 70 | } 71 | 72 | if (!(listing.meta.deliveryFee == null)){ 73 | l_fee = listing.meta.deliveryFee.text.split(" ")[0]; 74 | } 75 | 76 | if (!(listing.feedback == null)){ 77 | l_rating = listing.feedback.rating; 78 | } 79 | 80 | return { 81 | id: listing.slug.toString(), 82 | dataValues: { 83 | eta: l_eta, 84 | categories: l_categories, 85 | price_bucket: l_price_bucket, 86 | rating: l_rating, 87 | fee: l_fee, 88 | is_open: listing.isOpen.toString() 89 | } 90 | } 91 | }).filter(row => row.dataValues.is_open === "true"); 92 | } 93 | catch(e){ 94 | console.log(e); 95 | } 96 | } 97 | return undefined; 98 | }, 99 | // Reload data anytime there's a click or keypress on the page 100 | addScrapeTriggers: (reload) => { 101 | document.addEventListener("click", (e) => { 102 | console.log("clicked"); 103 | reload() }); 104 | document.addEventListener("keydown", (e) => { reload() }); 105 | } 106 | }); 107 | 108 | export default UberEatsAdapter; 109 | -------------------------------------------------------------------------------- /src/site_adapters/weatherchannel.ts: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import { extractNumber, urlExact, urlContains } from "../utils" 4 | import { createDomScrapingAdapter } from "./domScrapingBase" 5 | import * as moment from 'moment'; 6 | 7 | const WeatherChannelAdapter = createDomScrapingAdapter({ 8 | name: "Weather Channel", 9 | enabled: () => { 10 | return urlContains("https://weather.com/weather/hourbyhour") 11 | }, 12 | attributes: [ 13 | { name: "id", type: "text", hidden: true }, 14 | { name: "Time", type: "time", timeFormat: 'h:mm:ss a', 15 | correctFormat: true }, 16 | { name: "Description", type: "text"}, 17 | { name: "Temp °F", type: "numeric"}, 18 | { name: "Feels °F", type: "numeric"}, 19 | { name: "Precip %", type: "numeric"}, 20 | { name: "Humidity %", type: "numeric"}, 21 | { name: "Wind", type: "text"} 22 | ], 23 | scrapePage: () => { 24 | let tableRows = document.querySelector('.twc-table').querySelectorAll("tr"); 25 | //tableRows includes the heading, so we don't want to include that 26 | let arrayOfRows = Array.from(tableRows); 27 | arrayOfRows.shift(); 28 | return arrayOfRows.map(el => { 29 | return { 30 | rowElements: [el], 31 | id: el.querySelector('.dsx-date').textContent, 32 | dataValues: { 33 | Time: el.querySelector('.dsx-date'), 34 | Description: el.querySelector('.description').children[0], 35 | 'Temp °F': el.querySelector('.temp').children[0], 36 | 'Feels °F': el.querySelector('.feels').children[0], 37 | 'Precip %': el.querySelector('.precip').children[0], 38 | 'Humidity %': el.querySelector('.humidity').children[0], 39 | Wind: el.querySelector('.wind').children[0], 40 | }, 41 | } 42 | }) 43 | }, 44 | // Reload data anytime there's a click or keypress on the page 45 | addScrapeTriggers: (reload) => { 46 | document.addEventListener("click", (e) => { reload() }); 47 | document.addEventListener("keydown", (e) => { reload() }); 48 | } 49 | }); 50 | 51 | export default WeatherChannelAdapter; 52 | 53 | -------------------------------------------------------------------------------- /src/site_adapters/youtube.ts: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | import { extractNumber, urlExact, urlContains } from "../utils" 4 | import { createDomScrapingAdapter } from "./domScrapingBase" 5 | import debounce from 'lodash/debounce' 6 | 7 | 8 | const YoutubeAdapter = createDomScrapingAdapter({ 9 | name: "YouTube", 10 | enabled: () => { 11 | return urlContains("youtube.com") 12 | }, 13 | attributes: [ 14 | { name: "id", type: "text", hidden: true }, 15 | { name: "Title", type: "text" }, 16 | { name: "Time", type: "text"}, 17 | { name: "Uploader", type: "text"}, 18 | { name: "% Watched", type: "numeric"} 19 | ], 20 | scrapePage: () => { 21 | let tableRows = document.querySelector('#contents').children; 22 | if (tableRows.length == 1) { 23 | // for use on video listing page e.g. https://www.youtube.com/user/*/videos 24 | tableRows = document.querySelector('#contents #items').children; 25 | } 26 | return Array.from(tableRows).map((el, index) => { 27 | let elAsHTMLElement : HTMLElement = el; 28 | 29 | // on /user/*/videos, link is in #thumbnail, not #video-title-link 30 | if((el.querySelector('#video-title-link') !== null || el.querySelector('#thumbnail') !== null) && el.querySelector('#overlays') != null && el.querySelector('#overlays').children[0] != null){ 31 | 32 | let overlayChildrenAmount = el.querySelector('#overlays').children.length; 33 | let timeStampExists = overlayChildrenAmount > 1 && el.querySelector('#overlays').children[overlayChildrenAmount - 2].children[1] !== undefined; 34 | let timeStamp = timeStampExists 35 | ? el.querySelector('#overlays').children[overlayChildrenAmount - 2].children[1].textContent.replace((/ |\r\n|\n|\r/gm),"") 36 | : "N/A"; 37 | let watchedPercentage = el.querySelector('#progress') !== null 38 | ? progressToNumber((el.querySelector('#progress') as HTMLElement).style.width) 39 | : 0; 40 | 41 | return { 42 | rowElements: [elAsHTMLElement], 43 | id: (el.querySelector('#video-title-link') || el.querySelector('#thumbnail')).getAttribute("href"), 44 | dataValues: { 45 | Title: el.querySelector('#video-title'), 46 | Time: timeStamp, 47 | Uploader: el.querySelector('#text').children[0], 48 | '% Watched': watchedPercentage, 49 | }, 50 | } 51 | } 52 | else 53 | { 54 | return null; 55 | } 56 | 57 | }).filter(el => el !== null) 58 | }, 59 | // Reload data anytime there's a click or keypress on the page 60 | addScrapeTriggers: (reload) => { 61 | document.addEventListener("click", (e) => { reload() }); 62 | document.addEventListener("keydown", (e) => { reload() }); 63 | document.addEventListener("scroll", debounce((e) => { reload() }, 50)); 64 | }, 65 | onRowSelected: (row) => { 66 | row.rowElements.forEach(el => { 67 | if (el.style) { 68 | el.style["background-color"] = `#c9ebff` 69 | } 70 | }); 71 | row.rowElements[0].scrollIntoView({ behavior: "smooth", block: "center" }) 72 | }, 73 | onRowUnselected: (row) => { 74 | row.rowElements.forEach(el => { 75 | if(el.style) { 76 | el.style["background-color"] = `` 77 | } 78 | }) 79 | }, 80 | }); 81 | 82 | function progressToNumber(progress){ 83 | let strippedProgress = progress.slice(0, -1); 84 | return parseInt(strippedProgress); 85 | } 86 | 87 | export default YoutubeAdapter; 88 | -------------------------------------------------------------------------------- /src/tableAdapterMiddleware.ts: -------------------------------------------------------------------------------- 1 | // When things happen to our Redux state, 2 | // which we want to propagate downstream to the adapter, 3 | // we handle it here. 4 | 5 | // (when adapters are _upstream_ of the state, 6 | // handle it in action creators instead.) 7 | 8 | import { getFinalRecords } from './core/getFinalTable' 9 | import { TableAdapter } from './core/types' 10 | import pick from 'lodash/pick' 11 | 12 | export const TableAdapterMiddleware = (tableAdapter: TableAdapter) => 13 | ({ getState }) => next => action => { 14 | 15 | // Call the next dispatch method in the middleware chain. 16 | const returnValue = next(action) 17 | 18 | const newState = getState(); 19 | 20 | // todo: should we just do this on every state update? 21 | // all we get by being picky is perf improvement; 22 | // we're sending in the whole new state each time anyway? 23 | // actually nvm... maybe we do gain a lot of perf by telling the adapter 24 | // where the edit happened rather than just "something changed" 25 | switch (action.type) { 26 | // Records were sorted; 27 | // apply the sort to this adapter 28 | case "SORT_RECORDS": 29 | tableAdapter.applySort( 30 | getFinalRecords(newState), 31 | newState.sortConfig 32 | ); 33 | break; 34 | 35 | // Notify this table store if another table was reloaded 36 | // (eg: notify the site that user data has changed, 37 | // so we need to update annotations) 38 | case "TABLE_RELOADED": 39 | if (action.table.tableId !== tableAdapter.tableId) { 40 | tableAdapter.handleOtherTableUpdated(action.table) 41 | } 42 | break; 43 | 44 | // update the website with annotations from formula results. 45 | // todo: we're baking in some notions of formulas being only in the user table here... 46 | // should rethink a design where formulas can occur anywhere? 47 | case "FORMULAS_EVALUATED": 48 | const finalRecords = getFinalRecords(newState) 49 | const userAttributeNames = newState.userTable.attributes.map(a => a.name) 50 | const userRecordsWithFormulaResults = 51 | finalRecords.map(record => ({...record, values: pick(record.values, userAttributeNames)})) 52 | 53 | tableAdapter.handleOtherTableUpdated( 54 | {...newState.userTable, 55 | records: userRecordsWithFormulaResults}) 56 | break; 57 | 58 | case "RECORD_SELECTED": 59 | if (action.recordId) { 60 | tableAdapter.handleRecordSelected(action.recordId, action.attribute); 61 | } 62 | 63 | } 64 | 65 | return returnValue 66 | } 67 | 68 | -------------------------------------------------------------------------------- /src/ui/AutosuggestInput.tsx: -------------------------------------------------------------------------------- 1 | import React, {useState} from "react"; 2 | import Autosuggest from 'react-autosuggest'; 3 | import {functions} from '../formula' 4 | 5 | const autosuggestTheme = { 6 | container: { 7 | display: 'inline-block', 8 | position: 'relative', 9 | minWidth: '50%', 10 | marginLeft: '10px', 11 | zIndex: '2500', 12 | color: 'black', 13 | fontFamily: '-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol"', 14 | fontSize: '12px', 15 | }, 16 | input: { 17 | padding: '5px', 18 | border: 'solid thin #ddd', 19 | height: '1.5em', 20 | width: '100%', 21 | fontFamily: '-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol"', 22 | fontSize: '14px', 23 | '&:focus': { 24 | border: 'none' 25 | } 26 | }, 27 | suggestionsContainer: { 28 | display: 'none', 29 | }, 30 | suggestionsContainerOpen: { 31 | display: 'block', 32 | position: 'absolute', 33 | width: '100%', 34 | minHeight: '80px', 35 | maxHeight: '240px', 36 | background: 'rgba(255,255,255,0.8)', 37 | boxShadow: '0px 0px 3px gray', 38 | overflowY: 'auto', 39 | }, 40 | suggestionsList: { 41 | padding: '0px', 42 | margin: '0px', 43 | listStyleType: 'none', 44 | }, 45 | suggestion: { 46 | margin: '0', 47 | padding: '0px 5px', 48 | cursor: 'pointer' 49 | }, 50 | suggestionHighlighted: { 51 | background: 'rgb(200,200,200,0.4)', 52 | }, 53 | sectionContainer: { 54 | padding: '0px 5px', 55 | borderTop: '1px solid #ccc' 56 | }, 57 | sectionTitle: { 58 | fontVariantCaps: 'all-small-caps', 59 | fontSize: '10px', 60 | color: 'gray' 61 | } 62 | }; 63 | 64 | const AutosuggestInput = ({activeCellValue, setActiveCellValue, suggestions, setSuggestions, cellEditorRef, attributes, onCellEditorKeyPress, commitActiveCellValue}) => { 65 | const [prefix, setPrefix] = useState('') 66 | 67 | // This pattern matches every thing after and including any one of these symbols: = ( + - * / , 68 | const regex = /[=(\+\-\*\/,][^=(\+\-\*\/,]*$/; 69 | 70 | // Teach Autosuggest how to calculate suggestions for any given input value. 71 | const attributeNames = attributes.map(attribute => attribute.name); 72 | const mathSymbols = {"Plus": "+", "Minus": "-", "Multiply": "*", "Divide": "/"}; 73 | const mathSuggestions = Object.keys(functions) 74 | .filter(functionName => Object.keys(mathSymbols).indexOf(functionName) !== -1) 75 | .reduce((obj, functionName) => { 76 | const symbol = mathSymbols[functionName]; 77 | obj[symbol] = functions[functionName]; 78 | return obj; 79 | }, {}); 80 | const allFunctionNames = Object.keys(functions) 81 | .sort() 82 | .filter(functionName => Object.keys(mathSymbols).indexOf(functionName) === -1) 83 | const allSuggestions = [ 84 | {title: "Functions", suggestions: allFunctionNames}, 85 | {title: "Columns", suggestions: attributeNames} 86 | ] 87 | 88 | const getSuggestions = value => { 89 | const inputValue = value.trim() 90 | const matchIndex = inputValue.search(regex); 91 | 92 | // save the prefix of the input (everything up to the suggestion) 93 | const curPrefix = value.slice(0, matchIndex+1); 94 | if (curPrefix !== prefix) { 95 | setPrefix(curPrefix); 96 | } 97 | 98 | if (matchIndex === inputValue.length - 1) { 99 | // If at the start of a new expression, include all possible suggestions 100 | return allSuggestions; 101 | } 102 | else if (matchIndex === -1) { 103 | return []; 104 | } 105 | else { 106 | // Use everything after the regex match index (matchValue) as the prefix to determine suggestions 107 | const matchValue = inputValue.slice(matchIndex+1, inputValue.length).toLowerCase().trim(); 108 | let filteredSuggestions = allSuggestions 109 | .map(section => { 110 | return { 111 | title: section.title, 112 | suggestions: section.suggestions.filter(suggestion => suggestion.toLowerCase().slice(0, matchValue.length) === matchValue) 113 | }; 114 | }) 115 | .filter(section => section.suggestions.length > 0); 116 | // Math operations are suggested after a numeric attribute name or after ")" 117 | // const attributeIndex = attributeNames.indexOf(matchValue); 118 | // const isNumericAttribute = attributeIndex != -1 && attributes[attributeIndex].type === "numeric"; 119 | // const lastChar = matchValue[matchValue.length - 1]; 120 | // if (isNumericAttribute || lastChar === ")"){ 121 | // suggestions = suggestions.concat(Object.keys(mathSuggestions)) 122 | // } 123 | return filteredSuggestions; 124 | } 125 | }; 126 | 127 | const getSuggestionValue = function(suggestion) { 128 | return prefix[prefix.length-1] === "," ? prefix + " " + suggestion : prefix + suggestion; 129 | } 130 | 131 | // Determine how individual suggestions are rendered into HTML. 132 | const renderSuggestion = function(suggestion, {query}) { 133 | return(
{suggestion}
) 134 | } 135 | 136 | // Render helper text for functions in the footer 137 | const renderSuggestionsContainer = function({ containerProps, children }) { 138 | let inputValue = activeCellValue.toString().trim() 139 | const lastChar = inputValue[inputValue.length-1] 140 | 141 | if (lastChar === "(" || lastChar === ",") { 142 | inputValue = inputValue.slice(0, inputValue.lastIndexOf("(")) 143 | } 144 | 145 | const matchIndex = inputValue.search(regex); 146 | const matchValue = inputValue.slice(matchIndex+1, inputValue.length).trim(); 147 | const attributeIndex = attributeNames.indexOf(matchValue); 148 | 149 | let footer = undefined; 150 | if (matchValue in functions) { // function 151 | const params = functions[matchValue]["help"]; 152 | footer = ( 153 |
154 | {matchValue + "("} {Object.keys(params).join(", ")} {")"} 155 |
{Object.keys(params).map(key =>
{key}: {params[key]}
)}
156 |
157 | ) 158 | } 159 | else if (lastChar in mathSuggestions) { // math symbol 160 | const helpText = mathSuggestions[lastChar]["help"]; 161 | footer = ( 162 |
163 | numeric1 {lastChar} numeric2 164 |
{helpText}
165 |
166 | ) 167 | } 168 | else if (attributeIndex != -1) { // attribute name 169 | const attribute = attributes[attributeIndex]; 170 | footer = ( 171 |
172 | {attribute.name} is a column with type {attribute.type}. 173 |
174 | ) 175 | } 176 | return ( 177 |
178 | {children} 179 |
180 |
181 |
Documentation
182 | {footer} 183 |
184 |
185 |
186 | ) 187 | } 188 | 189 | const onChange = function(event, { newValue }) { 190 | setActiveCellValue(newValue); 191 | } 192 | 193 | // Autosuggest will call this function every time we need to update suggestions. 194 | const onSuggestionsFetchRequested = function({ value }) { 195 | setSuggestions(getSuggestions(value)); 196 | }; 197 | 198 | // Autosuggest will call this function every time we need to clear suggestions. 199 | const onSuggestionsClearRequested = function() { 200 | setSuggestions([]); 201 | }; 202 | 203 | const renderSectionTitle = function(section) { 204 | return {section.title}; 205 | } 206 | 207 | const getSectionSuggestions = function(section) { 208 | return section.suggestions; 209 | } 210 | 211 | return 230 | } 231 | 232 | export default AutosuggestInput; -------------------------------------------------------------------------------- /src/ui/WcPanel.tsx: -------------------------------------------------------------------------------- 1 | import React, { useRef, useState } from "react"; 2 | import { HotTable } from '@handsontable/react'; 3 | import "handsontable/dist/handsontable.full.css"; 4 | import "./overrides.css"; 5 | import styled from 'styled-components' 6 | import { Record, Attribute } from '../core/types' 7 | import Handsontable from "handsontable"; 8 | import { FormulaEditor } from '../ui/cell_editors/formulaEditor'; 9 | import mapValues from 'lodash/mapValues' 10 | import AutosuggestInput from './AutosuggestInput' 11 | import { getCreatingAdapter, setCreatingAdapter } from "../end_user_scraper/state"; 12 | 13 | const marketplaceUrl = "https://wildcard-marketplace.herokuapp.com"; 14 | 15 | function formatRecordsForHot(records:Array) { 16 | return records.map(record => ({ 17 | id: record.id, 18 | ...mapValues(record.values, v => v instanceof HTMLElement ? v.textContent : v) 19 | })) 20 | } 21 | 22 | function formatAttributesForHot(attributes:Array) { 23 | return attributes.map(attribute => ({ 24 | data: attribute.name, 25 | 26 | // If it's an "element" attribute, just render it as text 27 | type: attribute.type === "element" ? "text" : attribute.type, 28 | readOnly: !attribute.editable, 29 | editor: attribute.editor 30 | })) 31 | } 32 | 33 | const ToggleButton = styled.div` 34 | display: block; 35 | font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol"; 36 | font-size: 14px; 37 | border-radius: 10px; 38 | z-index: 10000; 39 | padding: 10px; 40 | position: fixed; 41 | bottom: ${props => props.hidden ? 20 : 300}px; 42 | right: ${props => props.codeEditorHidden ? 2 : 31}vw; 43 | background-color: white; 44 | box-shadow: 0px 0px 10px -1px #d5d5d5; 45 | border: none; 46 | cursor: pointer; 47 | &:hover { 48 | background-color: #eee; 49 | } 50 | ` 51 | 52 | const Panel = styled.div` 53 | position: fixed; 54 | bottom: 0; 55 | left: 0; 56 | height: ${props => props.hidden ? 0 : 280}px; 57 | width: ${props => props.codeEditorHidden ? 98 : 68.5}vw; 58 | z-index: 2200; 59 | box-shadow: 0px -5px 10px 1px rgba(170,170,170,0.5); 60 | border-top: solid thin #9d9d9d; 61 | overflow: hidden; 62 | background-color: white; 63 | font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol"; 64 | font-size: 14px; 65 | ` 66 | 67 | const ControlBar = styled.div` 68 | height: 30px; 69 | padding: 5px 10px; 70 | ` 71 | 72 | const EditorButton = styled(ToggleButton)` 73 | display: ${props => props.codeEditorHidden ? 'none' : 'block'}; 74 | bottom: 20px; 75 | right: ${props => props.right}; 76 | ` 77 | 78 | const ShareButton = styled(ToggleButton)` 79 | /* right: ${props => props.right}; */ 80 | right: calc(2vw + 165px); 81 | display: ${props => props.hidden || !props.codeEditorHidden ? 'none' : 'block'}; 82 | ` 83 | 84 | const EditButton = styled(ToggleButton)` 85 | /* right: ${props => props.right}; */ 86 | right: calc(2vw + 180px); 87 | display: ${props => props.hidden || !props.codeEditorHidden ? 'none' : 'block'}; 88 | ` 89 | 90 | // Declare our functional React component 91 | 92 | const WcPanel = ({ records = [], attributes, query, actions, adapter }) => { 93 | const creatingAdapter = getCreatingAdapter(); 94 | const hotRef = useRef(null); 95 | const cellEditorRef = useRef(null); 96 | const [hidden, setHidden] = useState(false); 97 | // Declare a new state variable for adapter code 98 | const [adapterCode, setAdapterCode] = useState(""); 99 | const [codeEditorHidden, setCodeEditorHidden] = useState(true); 100 | const _adapterKey = "localStorageAdapter:adapters:" + adapter.name; 101 | 102 | 103 | // Keep track of the currently selected cell 104 | const [activeCell, setActiveCell] = useState(null) 105 | 106 | // The value of the selected cell. 107 | // (Including in-progress updates that we are making in the UI) 108 | const [activeCellValue, setActiveCellValue] = useState('') 109 | 110 | // Autosuggest suggestions 111 | const [suggestions, setSuggestions] = useState([]); 112 | 113 | const onCellEditorKeyPress = (e) => { 114 | const key = e.key 115 | if (key === 'Enter') { 116 | cellEditorRef.current.blur() 117 | } 118 | } 119 | 120 | const commitActiveCellValue = () => { 121 | if(activeCellValue[0] === "=") { 122 | actions.setFormula( 123 | activeCell.attribute.tableId, 124 | activeCell.attribute.name, 125 | activeCellValue 126 | ) 127 | } else { 128 | actions.editRecords([ 129 | { 130 | tableId: activeCell.attribute.tableId, 131 | recordId: activeCell.record.id, 132 | attribute: activeCell.attribute.name, 133 | value: activeCellValue 134 | } 135 | ]) 136 | } 137 | } 138 | 139 | const hotSettings = { 140 | data: formatRecordsForHot(records), 141 | rowHeaders: true, 142 | columns: formatAttributesForHot(attributes), 143 | colHeaders: attributes.map(attr => { 144 | if(attr.formula) { 145 | return `${attr.name}` 146 | } else if(attr.type === "element") { 147 | return `${attr.name}` 148 | } else { 149 | return `${attr.name}` 150 | } 151 | }), 152 | columnSorting: true, 153 | 154 | // Set a low column width, 155 | // then let HOT stretch out the columns to match the width 156 | width: "100%", 157 | colWidths: attributes.map(a => 100), 158 | stretchH: "all" as const, 159 | wordWrap: false, 160 | manualColumnResize: true, 161 | 162 | // todo: parameterize height, make whole panel stretchable 163 | height: 250, 164 | 165 | cells: (row, col, prop) => { 166 | const cellProperties:any = {} 167 | const attr = attributes.find(a => a.name === prop) 168 | if (attr.formula) { 169 | cellProperties.formula = attr.formula 170 | cellProperties.editor = FormulaEditor 171 | cellProperties.placeholder = "loading..." 172 | } 173 | return cellProperties 174 | }, 175 | 176 | hiddenColumns: { 177 | columns: attributes.map((attr, idx) => attr.hidden ? idx : null).filter(e => Number.isInteger(e)) 178 | }, 179 | // contextMenu: { 180 | // items: { 181 | // "insert_user_attribute": { 182 | // name: 'Insert User Column', 183 | // callback: function(key, selection, clickEvent) { 184 | // // TODO: For now, new columns always get added to the user table. 185 | // // Eventually, do we want to allow adding to the main site table? 186 | // // Perhaps that'd be a way of extending scrapers using formulas... 187 | // actions.addAttribute("user"); 188 | // } 189 | // }, 190 | // "rename_user_attribute": { 191 | // // todo: disable this on site columns 192 | // name: 'Rename column', 193 | // callback: function(key, selection, clickEvent) { 194 | // alert('not implemented yet'); 195 | // } 196 | // }, 197 | // "clear_user_table": { 198 | // name: 'Clear user columns', 199 | // callback: function(key, selection, clickEvent) { 200 | // // TODO: For now, new columns always get added to the user table. 201 | // // Eventually, do we want to allow adding to the main site table? 202 | // // Perhaps that'd be a way of extending scrapers using formulas... 203 | // actions.clear("user"); 204 | // } 205 | // }, 206 | // "toggle_column_visibility":{ 207 | // name: 'Show/hide column in page', 208 | // disabled: () => { 209 | // // only allow toggling visibility on user table 210 | // const colIndex = getHotInstance().getSelectedLast()[1] 211 | // const attribute = attributes[colIndex] 212 | 213 | // return attribute.tableId !== "user" 214 | // }, 215 | // callback: function(key, selection, clickEvent) { 216 | // const attribute = attributes[selection[0].start.col]; 217 | 218 | // // NOTE! idx assumes that id is hidden. 219 | // actions.toggleVisibility(attribute.tableId, attribute.name); 220 | // } 221 | // }, 222 | // } 223 | // } 224 | } 225 | 226 | // Get a pointer to the current handsontable instance 227 | const getHotInstance = () => { 228 | if (hotRef && hotRef.current) { return hotRef.current.hotInstance; } 229 | else { return null; } 230 | } 231 | 232 | // make sure the HOT reflects the current sort config 233 | // of the query in our redux state. 234 | // (usually the sort config will be set from within HOT, 235 | // but this is needed e.g. to tell HOT when we load sort state from 236 | // local storage on initial pageload) 237 | const updateHotSortConfig = () => { 238 | if (getHotInstance()) { 239 | const columnSortPlugin = getHotInstance().getPlugin('columnSorting'); 240 | 241 | let newHotSortConfig; 242 | 243 | if (query.sortConfig) { 244 | newHotSortConfig = { 245 | column: attributes.map(a => a.name).indexOf(query.sortConfig.attribute), 246 | sortOrder: query.sortConfig.direction 247 | }; 248 | } else { 249 | newHotSortConfig = undefined; 250 | } 251 | columnSortPlugin.setSortConfig(newHotSortConfig); 252 | } 253 | } 254 | 255 | // todo: don't define these handlers inside the render funciton? 256 | // define outside and parameterize on props? 257 | 258 | // Handle user sorting the table 259 | const onBeforeColumnSort = (_, destinationSortConfigs) => { 260 | const columnSortPlugin = getHotInstance().getPlugin('columnSorting'); 261 | // We suppress HOT's built-in sorting by returning false, 262 | // and manually tell HOT that we've taken care of 263 | // sorting the table ourselves. 264 | // https://handsontable.com/docs/7.4.2/demo-sorting.html#custom-sort-implementation 265 | columnSortPlugin.setSortConfig(destinationSortConfigs); 266 | 267 | // for the moment we only support single column sort 268 | const sortConfig = destinationSortConfigs[0]; 269 | 270 | if (sortConfig) { 271 | actions.sortRecords({ 272 | // Sort config gives us a numerical index; convert to attribute 273 | attribute: attributes[sortConfig.column].name, 274 | direction: sortConfig.sortOrder 275 | }); 276 | } else { 277 | actions.sortRecords(null); 278 | } 279 | 280 | // don't let HOT sort the table 281 | return false; 282 | } 283 | 284 | // Handle user making a change to the table. 285 | // Similar to sorting, we suppress HOT built-in behavior, and 286 | // we handle the edit ourselves by triggering an action and 287 | // eventually rendering a totally fresh table from scratch 288 | const onBeforeChange = (changes, source) => { 289 | const edits = changes.map(([rowIndex, propName, prevValue, nextValue]) => { 290 | const attribute = attributes.find(a => a.name === propName) 291 | 292 | return { 293 | tableId: attribute.tableId, 294 | recordId: records[rowIndex].id, 295 | attribute: attribute.name, 296 | value: nextValue 297 | } 298 | }) 299 | 300 | const dataEdits = edits.filter(e => e.value[0] !== "=") 301 | const formulaEdits = edits.filter(e => e.value[0] === "=") 302 | 303 | console.log({dataEdits, formulaEdits}) 304 | 305 | actions.editRecords(dataEdits); 306 | 307 | for (const formulaEdit of formulaEdits) { 308 | actions.setFormula( 309 | formulaEdit.tableId, 310 | formulaEdit.attribute, 311 | formulaEdit.value 312 | ) 313 | } 314 | 315 | // don't let HOT edit the value 316 | return false; 317 | } 318 | 319 | const onAfterSelection = (rowIndex, prop) => { 320 | const record = records[rowIndex] 321 | const attribute = attributes.find(attr => attr.name === prop) 322 | 323 | actions.selectRecord(record.id, prop) 324 | 325 | setActiveCell({ record, attribute }) 326 | 327 | let activeCellValue 328 | 329 | if (attribute.formula) { 330 | activeCellValue = attribute.formula 331 | } else if (attribute.type === "element") { 332 | activeCellValue = record.values[attribute.name].outerHTML 333 | } else { 334 | activeCellValue = record.values[attribute.name] || "" 335 | } 336 | setActiveCellValue(activeCellValue) 337 | } 338 | 339 | 340 | const loadAdapterCode = function () { 341 | let loaded = false; 342 | // setup listener 343 | chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { 344 | switch (request.command) { 345 | case 'openCodeEditor': 346 | // show code editor 347 | setCodeEditorHidden(false); 348 | sendResponse({ codeEditorHidden: false }); 349 | 350 | // load adapter code 351 | if (!loaded) { 352 | loaded = true; 353 | chrome.storage.local.get(_adapterKey, (results) => { 354 | setAdapterCode(results[_adapterKey]); 355 | console.log("loaded code from storage"); 356 | }); 357 | } 358 | break; 359 | default: 360 | break; 361 | } 362 | }); 363 | } 364 | 365 | const onBlurCodeEditor = function(e, code){ 366 | const data = code.getValue(); 367 | console.log('Editor Data: ',data); 368 | setAdapterCode(data); 369 | } 370 | 371 | const saveAdapterCode = function() { 372 | chrome.storage.local.set({ [_adapterKey]: adapterCode }, function() { 373 | console.log("saved changes"); 374 | }); 375 | } 376 | return <> 377 | {!creatingAdapter && ( 378 | <> 379 | 385 | 389 | 390 | )} 391 | 414 | {saveAdapterCode();}}> Save 416 | 417 | setCodeEditorHidden(true)}> Close 419 | 420 | ; 421 | } 422 | 423 | export default WcPanel; 424 | -------------------------------------------------------------------------------- /src/ui/cell_editors/formulaEditor.js: -------------------------------------------------------------------------------- 1 | import Handsontable from 'handsontable'; 2 | 3 | class FormulaEditor extends Handsontable.editors.TextEditor { 4 | constructor(hotInstance) { 5 | super(hotInstance); 6 | } 7 | 8 | prepare(row, col, prop, td, originalValue, cellProperties) { 9 | super.prepare(row, col, prop, td, originalValue, cellProperties); 10 | if(cellProperties.formula) { 11 | this.formula = cellProperties.formula 12 | } else { 13 | this.formula = null 14 | } 15 | } 16 | 17 | // If cell contains a formula, edit the formula, not the value 18 | beginEditing(newValue, event) { 19 | let valueToEdit = newValue 20 | 21 | if (this.formula) { 22 | valueToEdit = this.formula 23 | } 24 | 25 | super.beginEditing(valueToEdit, event) 26 | } 27 | } 28 | 29 | export { FormulaEditor }; 30 | -------------------------------------------------------------------------------- /src/ui/cell_editors/fullCalendarEditor.css: -------------------------------------------------------------------------------- 1 | #open-apps-calendar-container { 2 | background-color: white; 3 | padding: 20px; 4 | position: fixed; 5 | height: 500px; 6 | width: 700px; 7 | bottom: 100px; 8 | right: 50px; 9 | z-index: 1000; 10 | } -------------------------------------------------------------------------------- /src/ui/cell_editors/fullCalendarEditor.js: -------------------------------------------------------------------------------- 1 | import moment from 'moment'; 2 | import Handsontable from 'handsontable'; 3 | import { Calendar } from '@fullcalendar/core'; 4 | import googleCalendarPlugin from '@fullcalendar/google-calendar' 5 | import dayGridPlugin from '@fullcalendar/daygrid'; 6 | import interactionPlugin from '@fullcalendar/interaction'; 7 | 8 | import '@fullcalendar/core/main.css'; 9 | import '@fullcalendar/daygrid/main.css'; 10 | import '@fullcalendar/timegrid/main.css'; 11 | import '@fullcalendar/list/main.css'; 12 | import './fullCalendarEditor.css' 13 | 14 | // convert HTML to a dom element 15 | function htmlToElement(html) { 16 | var template = document.createElement('template'); 17 | html = html.trim(); // Never return a text node of whitespace as the result 18 | template.innerHTML = html; 19 | return template.content.firstChild; 20 | } 21 | 22 | class FullCalendarEditor extends Handsontable.editors.BaseEditor { 23 | constructor(hotInstance) { 24 | super(hotInstance); 25 | } 26 | 27 | init () { 28 | this.selectedDate = new Date() 29 | 30 | this.calendarDiv = htmlToElement(`
`); 31 | document.body.appendChild(this.calendarDiv); 32 | 33 | this.calendar = new Calendar(document.getElementById('open-apps-calendar'), { 34 | plugins: [ interactionPlugin, dayGridPlugin, googleCalendarPlugin ], 35 | selectable: true, 36 | select: (info) => { 37 | console.log("selected ", info.start, info.end); 38 | this.selectedDate = info.start; 39 | }, 40 | googleCalendarApiKey: 'AIzaSyCpKAQzhc5HOvQ1a7j1QXEKqpIAeEaawLE', 41 | events: { 42 | googleCalendarId: '858lgk6ojl7vio2e3d15gkppv4@group.calendar.google.com' 43 | } 44 | }); 45 | 46 | this.calendar.render(); 47 | this.calendarDiv.style.display = "none" 48 | 49 | this.calendarDiv.addEventListener('mousedown', e => { 50 | event.stopPropagation() 51 | }); 52 | } 53 | 54 | getValue() { 55 | return moment(this.selectedDate).format("M/D/YYYY"); 56 | } 57 | 58 | setValue(newValue) { 59 | let date = moment(newValue, "M/D/YYYY").toDate; 60 | this.calendar.select(date); 61 | } 62 | 63 | open() { 64 | this.calendarDiv.style.display = ''; 65 | } 66 | 67 | close() { 68 | this.calendarDiv.style.display = 'none'; 69 | } 70 | 71 | focus() { 72 | this.calendarDiv.focus(); 73 | } 74 | } 75 | 76 | export { FullCalendarEditor }; 77 | -------------------------------------------------------------------------------- /src/ui/cell_editors/richTextEditor.css: -------------------------------------------------------------------------------- 1 | #open-apps-rich-text-editor-container { 2 | background-color: white; 3 | padding: 20px; 4 | position: fixed; 5 | height: 500px; 6 | width: 700px; 7 | bottom: 100px; 8 | right: 50px; 9 | z-index: 1000; 10 | } 11 | 12 | #wildcard-container strong, .ck strong { 13 | font-weight: bold; 14 | } 15 | 16 | #wildcard-container a:-webkit-any-link, .ck a:-webkit-any-link { 17 | color: -webkit-link; 18 | cursor: pointer; 19 | text-decoration: underline; 20 | } 21 | 22 | #wildcard-container ul, .ck ul { 23 | display: block; 24 | list-style-type: disc; 25 | margin-block-start: 1em; 26 | margin-block-end: 1em; 27 | margin-inline-start: 0px; 28 | margin-inline-end: 0px; 29 | padding-inline-start: 40px; 30 | } 31 | 32 | #wildcard-container ol, .ck ol { 33 | display: block; 34 | list-style-type: decimal; 35 | margin-block-start: 1em; 36 | margin-block-end: 1em; 37 | margin-inline-start: 0px; 38 | margin-inline-end: 0px; 39 | padding-inline-start: 40px; 40 | } -------------------------------------------------------------------------------- /src/ui/overrides.css: -------------------------------------------------------------------------------- 1 | /* In WcPanel.tsx, we set the z-index of our table to 2200 (to put it above original site content) 2 | So, we also need to tell the context menu to be visible above the table. 3 | We do this by overridding the default HoT CSS. */ 4 | .htContextMenu:not(.htGhostTable) { 5 | z-index: 2300; 6 | } 7 | 8 | th span.formula-header:before { 9 | content: "f(x)"; 10 | margin-right: 10px; 11 | margin-left: -10px; 12 | font-style: italic; 13 | font-size: 12px; 14 | color: #888; 15 | } 16 | 17 | th span.element-header:before { 18 | content: ""; 19 | margin-right: 10px; 20 | margin-left: -10px; 21 | font-style: italic; 22 | font-size: 12px; 23 | color: #888; 24 | } 25 | -------------------------------------------------------------------------------- /src/utils.ts: -------------------------------------------------------------------------------- 1 | /** Try to extract the first integer from a text string or HTML element */ 2 | export function extractNumber(input: any, defaultValue?: number, commaIsDecimalSeparator: boolean = false): number { 3 | let text, result; 4 | 5 | if (input instanceof HTMLElement) { 6 | text = input.textContent 7 | } else if (typeof input === "string") { 8 | text = input 9 | } 10 | if (text) { result = text.match(/[^0-9]*([0-9\.\,]*).*/)[1] } 11 | if (result && !commaIsDecimalSeparator) { 12 | // in the US and elsewhere, commas are thousands separator so they can be removed. 13 | // in other countries, the commas is the decimal separator. 14 | result = result.replace(/,/g, '') 15 | } 16 | if (result) { 17 | return Number(result) 18 | } else if (defaultValue !== undefined) { 19 | return defaultValue 20 | } else { 21 | return null 22 | } 23 | } 24 | 25 | /** Returns true if current page URL contains given URL as substring. 26 | * Mainly used in [[SiteAdapterOptions]] enable functions 27 | */ 28 | export function urlContains(fragment: string): boolean { 29 | return String(window.location).indexOf(fragment) !== -1 30 | } 31 | 32 | export function urlExact(url: string): boolean { 33 | return String(window.location) === url || 34 | String(window.location) === "https://" + url 35 | } 36 | 37 | export function urlMatches(regex: RegExp): boolean { 38 | return regex.test(String(window.location)); 39 | } 40 | 41 | export function htmlToElement(html): HTMLElement { 42 | var template = document.createElement('template'); 43 | html = html.trim(); // Never return a text node of whitespace as the result 44 | template.innerHTML = html; 45 | return template.content.firstChild as HTMLElement; 46 | } 47 | 48 | export function onDomReady(fn) { 49 | if (document.readyState != 'loading') fn(); 50 | else document.addEventListener('DOMContentLoaded', fn) 51 | } 52 | 53 | export function readFromChromeLocalStorage(keys) { 54 | return new Promise((resolve, reject) => { 55 | chrome.storage.local.get(keys, (results) => { 56 | resolve(results) 57 | }); 58 | }); 59 | } 60 | 61 | export function saveToChromeLocalStorage(entries) { 62 | return new Promise((resolve, reject) => { 63 | chrome.storage.local.set(entries); 64 | resolve(); 65 | }); 66 | } 67 | 68 | export function removeFromChromeLocalStorage(keys) { 69 | return new Promise((resolve, reject) => { 70 | chrome.storage.local.remove(keys); 71 | resolve(); 72 | }); 73 | } 74 | 75 | export function compileJavascript(code) { 76 | return eval(code); 77 | } 78 | 79 | export function throttleFunction(delay, fn) { 80 | let lastCall = 0; 81 | return function (...args) { 82 | const now = (new Date).getTime(); 83 | if (now - lastCall < delay) { 84 | return; 85 | } 86 | lastCall = now; 87 | return fn(...args); 88 | } 89 | } -------------------------------------------------------------------------------- /src/wildcard-ajax.js: -------------------------------------------------------------------------------- 1 | function onError(error) { 2 | console.error(`Error: ${error}`); 3 | } 4 | 5 | function listener(details) { 6 | if(navigator.userAgent.indexOf("Firefox") != -1 ) 7 | { 8 | let filter = browser.webRequest.filterResponseData(details.requestId); 9 | 10 | let data = []; 11 | filter.ondata = event => 12 | { 13 | data.push(event.data); 14 | filter.write(event.data); 15 | }; 16 | 17 | filter.onstop = async event => 18 | { 19 | let blob = new Blob(data, {type: 'application/json'}); 20 | let bstr = await blob.text(); 21 | let obj = undefined; 22 | try 23 | { 24 | obj = JSON.parse(bstr); 25 | } catch 26 | { 27 | 28 | } 29 | if (obj !== undefined) 30 | { 31 | browser.tabs.sendMessage( 32 | details.tabId, 33 | { 34 | url: details.url, 35 | data: obj 36 | } 37 | ).catch( 38 | onError 39 | ) 40 | } 41 | filter.close(); 42 | }; 43 | } 44 | } 45 | 46 | if(navigator.userAgent.indexOf("Firefox") != -1 ) 47 | { 48 | browser.webRequest.onBeforeRequest.addListener( 49 | listener, 50 | {urls: [""]}, 51 | ["blocking"] 52 | ); 53 | } -------------------------------------------------------------------------------- /src/wildcard-background.ts: -------------------------------------------------------------------------------- 1 | // This is an extension "background script" which access certain Chrome APIs 2 | // and makes them available to content scripts. 3 | // For more info: https://stackoverflow.com/questions/14211771/accessing-chrome-history-from-javascript-in-an-extension-and-changing-the-page-b 4 | 5 | 'use strict'; 6 | 7 | window['state'] = {}; 8 | 9 | // to add functionality only available in background scripts, 10 | // add a message handler to this list 11 | 12 | let fetchWithTimeout: any = (url, options, timeout) => { 13 | return new Promise((resolve, reject) => { 14 | fetch(url, options).then(resolve, reject); 15 | 16 | if (timeout) { 17 | const e = new Error("Connection timed out"); 18 | setTimeout(reject, timeout, e); 19 | } 20 | }); 21 | } 22 | 23 | const getVisits = (request, sender, sendResponse) => { 24 | chrome.history.getVisits({ url: request.url }, (visits) => { 25 | sendResponse({ visits: visits }); 26 | }) 27 | } 28 | 29 | const getReadingTime = (request, sender, sendResponse) => { 30 | const apiUrl = `https://klopets.com/readtime/?url=${request.url}&json` 31 | fetchWithTimeout(apiUrl, {}, 5000) 32 | .then(r => r.json()) 33 | .catch(err => sendResponse({ error: "couldn't fetch read time" })) 34 | .then(result => { 35 | console.log("result", result) 36 | if (result.seconds) { 37 | sendResponse({ seconds: result.seconds }) 38 | } else { 39 | sendResponse({ error: "couldn't fetch read time" }) 40 | } 41 | }) 42 | } 43 | 44 | const forwardToContentScripts = (request, sender, sendResponse) => { 45 | chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => { 46 | if (tabs && tabs.length) { 47 | chrome.tabs.sendMessage(tabs[0].id, request); 48 | } 49 | }); 50 | } 51 | 52 | const handlers = { 53 | getVisits: getVisits, 54 | getReadingTime: getReadingTime, 55 | deleteAdapter: forwardToContentScripts, 56 | saveAdapter: forwardToContentScripts, 57 | resetAdapter: forwardToContentScripts, 58 | editAdapter: forwardToContentScripts 59 | } 60 | 61 | chrome.runtime.onMessage.addListener( 62 | function (request, sender, sendResponse) { 63 | let handler = handlers[request.command] 64 | 65 | if (handler) { 66 | handler.call(this, request, sender, sendResponse) 67 | } 68 | 69 | return true; 70 | }); 71 | 72 | chrome.contextMenus.create({ 73 | title: "Wildcard", 74 | id: "wildcard", 75 | type: "normal", 76 | contexts: ["page"] 77 | }, () => { 78 | chrome.contextMenus.create({ 79 | title: "Create Adapter", 80 | contexts: ["page"], 81 | parentId: "wildcard", 82 | onclick: function () { 83 | chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => { 84 | if (tabs && tabs.length) { 85 | // send message to active tab 86 | chrome.tabs.sendMessage(tabs[0].id, { command: "createAdapter" }, (response) => { 87 | if (response.error) { 88 | alert(response.error); 89 | } 90 | }); 91 | } 92 | }); 93 | } 94 | }); 95 | // chrome.contextMenus.create({ 96 | // title: "Edit Adapter", 97 | // contexts: ["page"], 98 | // parentId: "wildcard", 99 | // onclick: function () { 100 | // chrome.tabs.query({ active: true, currentWindow: true }, (tabs) => { 101 | // if (tabs && tabs.length) { 102 | // // send message to active tab 103 | // chrome.tabs.sendMessage(tabs[0].id, { command: "openCodeEditor" }, (response) => { 104 | // if (response.error) { 105 | // alert(response.error); 106 | // } 107 | // }); 108 | // } 109 | // }); 110 | // } 111 | // }); 112 | // chrome.contextMenus.create({ 113 | // title: "Open Options Page", 114 | // contexts: ["page"], 115 | // parentId: "wildcard", 116 | // onclick: function () { 117 | // chrome.runtime.openOptionsPage(); 118 | // } 119 | // }); 120 | }); -------------------------------------------------------------------------------- /src/wildcard.tsx: -------------------------------------------------------------------------------- 1 | /** This is the output file that the browser runs on each page. 2 | * It compiles the framework and all the site adapters into one file. 3 | */ 4 | 5 | 'use strict'; 6 | 7 | import React from "react"; 8 | import { render } from "react-dom"; 9 | import { createStore, applyMiddleware, bindActionCreators } from "redux"; 10 | import { Provider, connect } from 'react-redux' 11 | import { composeWithDevTools } from 'redux-devtools-extension'; 12 | import reducer from './core/reducer'; 13 | import { debugMiddleware } from './core/debug' 14 | import { htmlToElement } from './utils' 15 | import WcPanel from "./ui/WcPanel"; 16 | import { getActiveAdapter } from "./site_adapters" 17 | import { userStore as userTableAdapter } from "./localStorageAdapter" 18 | import thunk from 'redux-thunk'; 19 | import { initializeActions } from './core/actions' 20 | import { getFinalRecords, getFinalAttributes } from './core/getFinalTable' 21 | import { TableAdapterMiddleware } from './tableAdapterMiddleware' 22 | import { startScrapingListener, stopScrapingListener, resetScrapingListener, editScraper } from './end_user_scraper'; 23 | import { setCachedActiveAdapter } from "./end_user_scraper/state"; 24 | 25 | // todo: move this out of this file 26 | const connectRedux = (component, actions) => { 27 | const mapStateToProps = state => ({ 28 | // todo: when we have non-app records and attributes, 29 | // merge them in the redux state, and pass in merged data here -- 30 | // this panel view isn't responsible for combining them. 31 | // keep this component thin. 32 | records: getFinalRecords(state), 33 | attributes: getFinalAttributes(state), 34 | query: state.query 35 | }) 36 | 37 | const mapDispatchToProps = dispatch => ({ 38 | actions: bindActionCreators(actions, dispatch) 39 | }) 40 | 41 | return connect( 42 | mapStateToProps, 43 | mapDispatchToProps 44 | )(component) 45 | } 46 | 47 | export const run = async function () { 48 | //console.log("Re-running adapter") 49 | const wcRoot = document.getElementById('wc--root'); 50 | if (wcRoot) { 51 | wcRoot.remove(); 52 | } 53 | const activeSiteAdapter = await getActiveAdapter(); 54 | if (!activeSiteAdapter) { return; } 55 | 56 | activeSiteAdapter.initialize(); 57 | 58 | //userTableAdapter.initialize(activeSiteAdapter.name) 59 | 60 | const tables = { app: activeSiteAdapter } 61 | 62 | // pass our TableAdapter objects into action creators, 63 | // so action creator functions can access them. 64 | const actions = initializeActions(tables) 65 | 66 | // stash active adapter if we are in creation mode 67 | setCachedActiveAdapter(activeSiteAdapter); 68 | 69 | // Add extra space to the bottom of the page for the wildcard panel 70 | // todo: move this elsewhere? 71 | document.querySelector("body").style["margin-bottom"] = "300px"; 72 | 73 | // Create our redux store 74 | const store = createStore(reducer, composeWithDevTools( 75 | applyMiddleware(thunk), 76 | applyMiddleware(TableAdapterMiddleware(activeSiteAdapter)), 77 | // applyMiddleware(TableAdapterMiddleware(userTableAdapter)), 78 | applyMiddleware(debugMiddleware), 79 | )); 80 | 81 | // Subscribe to app data updates from the site adapter and user store 82 | activeSiteAdapter.subscribe(table => 83 | store.dispatch(actions.tableReloaded(table)) 84 | ) 85 | 86 | // userTableAdapter.subscribe(table => 87 | // store.dispatch(actions.tableReloaded(table)) 88 | // ) 89 | 90 | // todo: wrap storage stuff in a module 91 | 92 | // Load saved query (including sorting) 93 | chrome.storage.local.get(`query:${activeSiteAdapter.name}`, (result) => { 94 | const query = result[`query:${activeSiteAdapter.name}`] 95 | if (query) { 96 | store.dispatch(actions.sortRecords(query.sortConfig)) 97 | } else { 98 | console.log("no query") 99 | } 100 | }) 101 | 102 | // save the query in local storage when it updates 103 | store.subscribe(() => { 104 | const state = store.getState(); 105 | const queryToStore = { [`query:${activeSiteAdapter.name}`]: state.query } 106 | chrome.storage.local.set(queryToStore) 107 | }) 108 | 109 | // Initialize the container for our view 110 | document.body.appendChild( 111 | htmlToElement(`
`) as HTMLElement); 112 | 113 | // in the future, rather than hardcode WcPanel here, 114 | // could dynamically choose a table editor instrument 115 | const TableEditor = connectRedux(WcPanel, actions) 116 | 117 | render( 118 | 119 | 120 | , 121 | document.getElementById("wc--root") 122 | ); 123 | 124 | } 125 | 126 | chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { 127 | switch(request.command) { 128 | case 'createAdapter': 129 | startScrapingListener(); 130 | break; 131 | case 'saveAdapter': 132 | stopScrapingListener({ save: true }); 133 | break; 134 | case 'deleteAdapter': 135 | stopScrapingListener({ save: false }); 136 | break; 137 | case 'resetAdapter': 138 | resetScrapingListener(); 139 | break; 140 | case 'editAdapter': 141 | editScraper(); 142 | break; 143 | default: 144 | break; 145 | } 146 | }); 147 | 148 | run(); -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "typeRoots": ["node_modules/@types"], 4 | "allowSyntheticDefaultImports": true, 5 | "jsx": "react", 6 | "downlevelIteration": true 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /vendor/ace/theme-monokai.js: -------------------------------------------------------------------------------- 1 | define("ace/theme/monokai",["require","exports","module","ace/lib/dom"],function(e,t,n){t.isDark=!0,t.cssClass="ace-monokai",t.cssText=".ace-monokai .ace_gutter {background: #2F3129;color: #8F908A}.ace-monokai .ace_print-margin {width: 1px;background: #555651}.ace-monokai {background-color: #272822;color: #F8F8F2}.ace-monokai .ace_cursor {color: #F8F8F0}.ace-monokai .ace_marker-layer .ace_selection {background: #49483E}.ace-monokai.ace_multiselect .ace_selection.ace_start {box-shadow: 0 0 3px 0px #272822;}.ace-monokai .ace_marker-layer .ace_step {background: rgb(102, 82, 0)}.ace-monokai .ace_marker-layer .ace_bracket {margin: -1px 0 0 -1px;border: 1px solid #49483E}.ace-monokai .ace_marker-layer .ace_active-line {background: #202020}.ace-monokai .ace_gutter-active-line {background-color: #272727}.ace-monokai .ace_marker-layer .ace_selected-word {border: 1px solid #49483E}.ace-monokai .ace_invisible {color: #52524d}.ace-monokai .ace_entity.ace_name.ace_tag,.ace-monokai .ace_keyword,.ace-monokai .ace_meta.ace_tag,.ace-monokai .ace_storage {color: #F92672}.ace-monokai .ace_punctuation,.ace-monokai .ace_punctuation.ace_tag {color: #fff}.ace-monokai .ace_constant.ace_character,.ace-monokai .ace_constant.ace_language,.ace-monokai .ace_constant.ace_numeric,.ace-monokai .ace_constant.ace_other {color: #AE81FF}.ace-monokai .ace_invalid {color: #F8F8F0;background-color: #F92672}.ace-monokai .ace_invalid.ace_deprecated {color: #F8F8F0;background-color: #AE81FF}.ace-monokai .ace_support.ace_constant,.ace-monokai .ace_support.ace_function {color: #66D9EF}.ace-monokai .ace_fold {background-color: #A6E22E;border-color: #F8F8F2}.ace-monokai .ace_storage.ace_type,.ace-monokai .ace_support.ace_class,.ace-monokai .ace_support.ace_type {font-style: italic;color: #66D9EF}.ace-monokai .ace_entity.ace_name.ace_function,.ace-monokai .ace_entity.ace_other,.ace-monokai .ace_entity.ace_other.ace_attribute-name,.ace-monokai .ace_variable {color: #A6E22E}.ace-monokai .ace_variable.ace_parameter {font-style: italic;color: #FD971F}.ace-monokai .ace_string {color: #E6DB74}.ace-monokai .ace_comment {color: #75715E}.ace-monokai .ace_indent-guide {background: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAACCAYAAACZgbYnAAAAEklEQVQImWPQ0FD0ZXBzd/wPAAjVAoxeSgNeAAAAAElFTkSuQmCC) right repeat-y}";var r=e("../lib/dom");r.importCssString(t.cssText,t.cssClass)}); 2 | (function() { 3 | window.require(["ace/theme/monokai"], function(m) { 4 | if (typeof module == "object" && typeof exports == "object" && module) { 5 | module.exports = m; 6 | } 7 | }); 8 | })(); 9 | --------------------------------------------------------------------------------