├── .github └── workflows │ ├── deploy-mkdocs.yml │ └── nextjs.yml ├── .gitignore ├── .idea ├── .gitignore ├── modules.xml ├── node-crawler.iml └── vcs.xml ├── Dockerfile ├── README.md ├── documentation ├── docs │ ├── extend-project │ │ └── implementing-a-new-node.md │ ├── index.md │ └── stylesheets │ │ └── extra.css └── mkdocs.yml ├── next.config.js ├── nginx.conf ├── package-lock.json ├── package.json ├── postcss.config.js ├── public ├── next.svg ├── node-crawler-overview-canvas.png ├── node-crawler-overview-output.png └── vercel.svg ├── src ├── app │ ├── AppTheme.tsx │ ├── api │ │ └── fetch │ │ │ └── route.ts │ ├── editor │ │ └── page.tsx │ ├── globals.css │ ├── icon.png │ ├── layout.tsx │ └── page.tsx ├── components │ ├── editor │ │ ├── Engine.tsx │ │ ├── headerbar │ │ │ ├── HeaderBar.tsx │ │ │ └── PageNavigation.tsx │ │ └── pages │ │ │ ├── canvas │ │ │ ├── DragAndDropFlow.tsx │ │ │ ├── DragAndDropFlowStyles.css │ │ │ ├── edges │ │ │ │ ├── EdgeGradientStyles.css │ │ │ │ ├── Edges.ts │ │ │ │ └── util │ │ │ │ │ └── EdgeCreators.tsx │ │ │ ├── nodes │ │ │ │ ├── DatabaseTableNode.tsx │ │ │ │ ├── ExtractorNode.tsx │ │ │ │ ├── FetchWebsiteNode.tsx │ │ │ │ ├── HtmlToTextNode.tsx │ │ │ │ ├── SaveNode.tsx │ │ │ │ ├── StartNode.tsx │ │ │ │ ├── ZipNode.tsx │ │ │ │ └── util │ │ │ │ │ ├── Creators.tsx │ │ │ │ │ └── OptionsUtil.ts │ │ │ └── toolbars │ │ │ │ ├── Html.css │ │ │ │ ├── NodesToolbar.tsx │ │ │ │ ├── OnCanvasNodesSelector.tsx │ │ │ │ └── OptionsToolbar.tsx │ │ │ ├── html │ │ │ ├── CssSelectorCard.tsx │ │ │ ├── HtmlSelectorPage.tsx │ │ │ ├── SelectableHtmlPreview.tsx │ │ │ └── util │ │ │ │ └── HtmlUtils.ts │ │ │ └── output │ │ │ ├── Files.tsx │ │ │ ├── Log.tsx │ │ │ ├── OutputPage.tsx │ │ │ └── util │ │ │ └── SmoothScrollUtil.ts │ └── form │ │ ├── CacheTextField.tsx │ │ ├── DraggableOptionsListContainer.tsx │ │ ├── FileNameInputOption.tsx │ │ ├── MultiSelectOption.tsx │ │ ├── OptionsContainer.tsx │ │ ├── RowOptionsContainer.tsx │ │ ├── SelectOption.tsx │ │ └── TextInputOption.tsx ├── config │ ├── ConnectionRules.ts │ ├── NodeType.ts │ ├── NodesMetadata.tsx │ ├── OutputValueType.ts │ └── colors.ts ├── engine │ └── nodes │ │ ├── BasicNode.ts │ │ ├── EngineDatabaseTableNode.ts │ │ ├── EngineExtractorNode.ts │ │ ├── EngineFetchWebsiteNode.ts │ │ ├── EngineHtmlToTextNode.ts │ │ ├── EngineSaveNode.ts │ │ ├── EngineStartNode.ts │ │ └── EngineZipNode.ts ├── model │ ├── CrawlerProjectDto.ts │ ├── NextNodeKey.ts │ ├── NodeData.ts │ └── NodeMap.ts ├── stores │ └── editor │ │ ├── EditorPageStore.tsx │ │ ├── HtmlSelectorStore.ts │ │ ├── PlayStore.ts │ │ └── ReactFlowStore.ts └── util │ ├── IOUtil.ts │ └── NodeMapTransformer.ts ├── tailwind.config.js └── tsconfig.json /.github/workflows/deploy-mkdocs.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | on: 3 | push: 4 | branches: 5 | - master 6 | - main 7 | permissions: 8 | contents: write 9 | jobs: 10 | deploy: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v3 14 | - uses: actions/setup-python@v4 15 | with: 16 | python-version: 3.x 17 | - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV 18 | - uses: actions/cache@v3 19 | with: 20 | key: mkdocs-material-${{ env.cache_id }} 21 | path: .cache 22 | restore-keys: | 23 | mkdocs-material- 24 | - run: pip install mkdocs-material 25 | - name: Navigate to documentation directory and Deploy to GitHub Pages 26 | run: | 27 | cd documentation 28 | mkdocs gh-deploy --force 29 | -------------------------------------------------------------------------------- /.github/workflows/nextjs.yml: -------------------------------------------------------------------------------- 1 | # Sample workflow for building and deploying a Next.js site to GitHub Pages 2 | # 3 | # To get started with Next.js see: https://nextjs.org/docs/getting-started 4 | # 5 | name: Deploy Next.js site to Pages 6 | 7 | on: 8 | # Runs on pushes targeting the default branch 9 | push: 10 | branches: ["master"] 11 | 12 | # Allows you to run this workflow manually from the Actions tab 13 | workflow_dispatch: 14 | 15 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 16 | permissions: 17 | contents: read 18 | pages: write 19 | id-token: write 20 | 21 | # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. 22 | # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. 23 | concurrency: 24 | group: "pages" 25 | cancel-in-progress: false 26 | 27 | jobs: 28 | # Build job 29 | build: 30 | runs-on: ubuntu-latest 31 | steps: 32 | - name: Checkout 33 | uses: actions/checkout@v3 34 | - name: Detect package manager 35 | id: detect-package-manager 36 | run: | 37 | if [ -f "${{ github.workspace }}/yarn.lock" ]; then 38 | echo "manager=yarn" >> $GITHUB_OUTPUT 39 | echo "command=install" >> $GITHUB_OUTPUT 40 | echo "runner=yarn" >> $GITHUB_OUTPUT 41 | exit 0 42 | elif [ -f "${{ github.workspace }}/package.json" ]; then 43 | echo "manager=npm" >> $GITHUB_OUTPUT 44 | echo "command=ci" >> $GITHUB_OUTPUT 45 | echo "runner=npx --no-install" >> $GITHUB_OUTPUT 46 | exit 0 47 | else 48 | echo "Unable to determine package manager" 49 | exit 1 50 | fi 51 | - name: Setup Node 52 | uses: actions/setup-node@v3 53 | with: 54 | node-version: "16" 55 | cache: ${{ steps.detect-package-manager.outputs.manager }} 56 | - name: Setup Pages 57 | uses: actions/configure-pages@v3 58 | with: 59 | # Automatically inject basePath in your Next.js configuration file and disable 60 | # server side image optimization (https://nextjs.org/docs/api-reference/next/image#unoptimized). 61 | # 62 | # You may remove this line if you want to manage the configuration yourself. 63 | static_site_generator: next 64 | - name: Restore cache 65 | uses: actions/cache@v3 66 | with: 67 | path: | 68 | .next/cache 69 | # Generate a new cache whenever packages or source files change. 70 | key: ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json', '**/yarn.lock') }}-${{ hashFiles('**.[jt]s', '**.[jt]sx') }} 71 | # If source files changed but packages didn't, rebuild from a prior cache. 72 | restore-keys: | 73 | ${{ runner.os }}-nextjs-${{ hashFiles('**/package-lock.json', '**/yarn.lock') }}- 74 | - name: Install dependencies 75 | run: ${{ steps.detect-package-manager.outputs.manager }} ${{ steps.detect-package-manager.outputs.command }} 76 | - name: Build with Next.js 77 | run: ${{ steps.detect-package-manager.outputs.runner }} next build 78 | - name: Static HTML export with Next.js 79 | run: ${{ steps.detect-package-manager.outputs.runner }} next export 80 | - name: Upload artifact 81 | uses: actions/upload-pages-artifact@v2 82 | with: 83 | path: ./out 84 | 85 | # Deployment job 86 | deploy: 87 | environment: 88 | name: github-pages 89 | url: ${{ steps.deployment.outputs.page_url }} 90 | runs-on: ubuntu-latest 91 | needs: build 92 | steps: 93 | - name: Deploy to GitHub Pages 94 | id: deployment 95 | uses: actions/deploy-pages@v2 96 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # next.js 12 | /.next/ 13 | /out/ 14 | 15 | # production 16 | /build 17 | 18 | # misc 19 | .DS_Store 20 | *.pem 21 | 22 | # debug 23 | npm-debug.log* 24 | yarn-debug.log* 25 | yarn-error.log* 26 | 27 | # local env files 28 | .env*.local 29 | 30 | # vercel 31 | .vercel 32 | 33 | # typescript 34 | *.tsbuildinfo 35 | next-env.d.ts 36 | /documentation/site/ 37 | -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Editor-based HTTP Client requests 5 | /httpRequests/ 6 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/node-crawler.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Setzen Sie das Basis-Image 2 | FROM node:16-alpine 3 | 4 | # Setzen Sie das Arbeitsverzeichnis 5 | WORKDIR /app 6 | 7 | # Kopieren Sie die Datei package.json (und die dazugehörige package-lock.json, falls vorhanden) 8 | COPY package*.json ./ 9 | 10 | # Installieren Sie alle Abhängigkeiten 11 | RUN npm install 12 | 13 | # Kopieren Sie den Rest des Anwendungsquellcodes 14 | COPY . . 15 | 16 | # Bauen Sie Ihre Next.js-Anwendung 17 | RUN npm install && npm run build 18 | 19 | # Exponieren Sie den Port für den Next.js-Server (standardmäßig 3000) 20 | EXPOSE 3002 21 | 22 | # Führen Sie den Next.js-Server aus 23 | CMD ["npm", "start"] 24 | 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Project Name: Node-Crawler 2 | 3 | [![Next.js](https://img.shields.io/badge/Next.js-13.4.8-000?style=for-the-badge&logo=nextdotjs&logoColor=white)](https://nextjs.org/) 4 | [![ReactFlow](https://img.shields.io/badge/ReactFlow-11.7.4-000?style=for-the-badge&logo=react&logoColor=white)](https://reactflow.dev/) 5 | 6 | ## Overview 7 | 8 | ! Work in progress ! 9 | 10 | See https://mertend.github.io/node-crawler/ for a detailed documentation. 11 | 12 | Node-Crawler is a highly customizable, Node-based web application for creating web crawlers and further processing and transforming the retrieved data. Users can build tailor-made web crawlers and manipulate and transform the collected data as needed. The output data format can be set to a wide range of formats including JSON, CSV, and various database formats. 13 | 14 | The app is developed using [Next.js](https://nextjs.org/) and the [React Flow](https://reactflow.dev/) library provides a framework for creating the node-based editor. 15 | 16 | ![Overview](public/node-crawler-overview-canvas.png) 17 | ![Overview](public/node-crawler-overview-output.png) 18 | 19 | ## Features 20 | 21 | - **Node-based Editing**: Users can create and edit their own crawler workflows by drag-and-dropping nodes. 22 | - **Data Transformation**: The application supports a variety of data manipulation and transformation operations for cleaning and restructuring the gathered data. 23 | - **Data Export**: The transformed data can be output in a variety of formats including JSON, CSV, and various database formats. 24 | 25 | ## Installation 26 | 27 | Make sure you have [Node.js](https://nodejs.org/en) and npm installed on your system before you start. 28 | 29 | 1. Clone the repository: 30 | 31 | ```shell 32 | git clone https://github.com/MertenD/node-crawler.git 33 | ``` 34 | 35 | 2. Navigate into the directory and install the dependencies: 36 | 37 | ```shell 38 | cd node-crawler 39 | npm install 40 | ``` 41 | 42 | 3. Start the development server: 43 | 44 | ```shell 45 | npm run dev 46 | ``` 47 | 48 | Now you should be able to see the web application on `http://localhost:3000` in your browser. 49 | 50 | ## Implementing a New Node 51 | 52 | Follow these steps when you need to create a new Node: 53 | 54 | ### Step 1: Add the Node to the NodeType Enum 55 | 56 | Next, add your newly created Node to the **config/NodeType** enum. 57 | 58 | ### Step 2: Create the New Node 59 | 60 | First, create a new file in the **components/editor/pages/canvas/nodes** directory. In this file, you will define the following elements: 61 | 62 | 1. **Data Interface:** Create a Data interface that stores all data the user can configure. 63 | 64 | 2. **Style Function:** Create a Style function (using the ```createNodeShapeStyle()``` function), where you can customize the Node's appearance. 65 | 66 | 3. **Node Component:** Create a Node component (using the ```createNodeComponent()``` function), which will be the Node on the canvas. 67 | 68 | 4. **Options Component:** Create an Options component (using the ```createOptionsComponent()``` function), where the user can configure the Node's behavior. 69 | 70 | You can use the following template to create a new Node: 71 | 72 | ```tsx 73 | // TODO: Replace [NAME] everywhere 74 | 75 | // --- Data --- 76 | export interface [NAME]NodeData extends NodeData { 77 | // TODO: Add data attributes here 78 | } 79 | 80 | // --- Style --- 81 | export const [NAME]ShapeStyle = createNodeShapeStyle({ 82 | // TODO: Add additional CSS for the node's shape here 83 | }) 84 | 85 | // --- Node --- 86 | export const [NAME]Node = createNodeComponent<[NAME]NodeData>( 87 | NodeType.[NAME]_NODE, 88 | [NAME]ShapeStyle, 89 | (id, selected, data) => { 90 | // TODO: Place the node content here 91 | } 92 | ) 93 | 94 | // --- Options --- 95 | export const [NAME]Options = createOptionsComponent<[NAME]NodeData>("Start", ({ id, data, onDataUpdated }) => { 96 | return // TODO: Place options here 97 | }) 98 | ``` 99 | 100 | ### Step 3: Add Metadata to NodesMetadata.tsx 101 | 102 | Add all metadata of the new Node to the **config/NodesMetadata.tsx** file. 103 | 104 | ### Step 4: Add Connection Rules 105 | 106 | Define the connection rules for the new Node in the **config/ConnectionRules.ts** file. 107 | 108 | ### Step 5: Create a New Node Class for Execution 109 | 110 | You need to create a new class in the **engine/nodes** directory. This class should extend the ```BasicNode``` interface. Below is a basic template for your reference: 111 | 112 | ```ts 113 | // TODO: Replace [NAME] 114 | export class Engine[NAME]Node implements BasicNode { 115 | id: string; 116 | nodeType: NodeType 117 | data: // TODO 118 | 119 | constructor(id: string, data: /* TODO */) { 120 | this.id = id 121 | this.nodeType = // TODO 122 | this.data = data 123 | } 124 | 125 | async run() { 126 | // Optional: Get inputs from previous nodes 127 | const input = usePlayStore.getState().getInput(this.id, "input") 128 | 129 | if (input) { 130 | // TODO Put the logic of the node here 131 | 132 | // Optional: Add downloadable file 133 | usePlayStore.getState().addFile(/* TODO */) 134 | 135 | // Optional: Make outputs accessable for the next node 136 | usePlayStore.getState().addOutgoingPipelines(this.id, /* TODO */) 137 | 138 | // Optional: Write to the log 139 | usePlayStore.getState().writeToLog(/* TODO */) 140 | 141 | // End with calling the next node 142 | usePlayStore.getState().nextNode() 143 | } 144 | } 145 | } 146 | ``` 147 | 148 | ### Step 6: Add Node Transformation Logic 149 | 150 | The final step involves adding the transformation logic for the node. This transformation will convert a [React Flow](https://reactflow.dev/) Node 151 | into an instance of your newly created class from **Step 5**. To do this, navigate to the **util/NodeMapTransformer.ts** file and 152 | add a new case to the ```getNodeFromType()``` method where you create the instance. 153 | -------------------------------------------------------------------------------- /documentation/docs/extend-project/implementing-a-new-node.md: -------------------------------------------------------------------------------- 1 | # Implementing a New Node 2 | 3 | Follow these steps when you need to create a new Node: 4 | 5 | ## Step 1: Add the Node to the NodeType Enum 6 | 7 | Next, add your newly created Node to the **config/NodeType** enum. 8 | 9 | ## Step 2: Create the New Node 10 | 11 | First, create a new file in the **components/editor/pages/canvas/nodes** directory. In this file, you will define the following elements: 12 | 13 | 1. **Data Interface:** Create a Data interface that stores all data the user can configure. 14 | 15 | 2. **Style Function:** Create a Style function (using the ```createNodeShapeStyle()``` function), where you can customize the Node's appearance. 16 | 17 | 3. **Node Component:** Create a Node component (using the ```createNodeComponent()``` function), which will be the Node on the canvas. 18 | 19 | 4. **Options Component:** Create an Options component (using the ```createOptionsComponent()``` function), where the user can configure the Node's behavior. 20 | 21 | You can use the following template to create a new Node: 22 | 23 | ```tsx 24 | // TODO: Replace [NAME] everywhere 25 | 26 | // --- Data --- 27 | export interface [NAME]NodeData extends NodeData { 28 | // TODO: Add data attributes here 29 | } 30 | 31 | // --- Style --- 32 | export const [NAME]ShapeStyle = createNodeShapeStyle({ 33 | // TODO: Add additional CSS for the node's shape here 34 | }) 35 | 36 | // --- Node --- 37 | export const [NAME]Node = createNodeComponent<[NAME]NodeData>( 38 | NodeType.[NAME]_NODE, 39 | [NAME]ShapeStyle, 40 | (id, selected, data) => { 41 | // TODO: Place the node content here 42 | } 43 | ) 44 | 45 | // --- Options --- 46 | export const [NAME]Options = createOptionsComponent<[NAME]NodeData>("Start", ({ id, data, onDataUpdated }) => { 47 | return // TODO: Place options here 48 | }) 49 | ``` 50 | 51 | ## Step 3: Add Metadata to NodesInformation.tsx 52 | 53 | Add all metadata of the new Node to the **config/NodesInformation.tsx** file. 54 | 55 | ## Step 4: Add Connection Rules 56 | 57 | Define the connection rules for the new Node in the **config/ConnectionRules.ts** file. 58 | 59 | ## Step 5: Create a New Node Class for Execution 60 | 61 | You need to create a new class in the **engine/nodes** directory. This class should extend the ```BasicNode``` interface. Below is a basic template for your reference: 62 | 63 | ```ts 64 | // TODO: Replace [NAME] 65 | export class Engine[NAME]Node implements BasicNode { 66 | id: string; 67 | nodeType: NodeType 68 | data: // TODO 69 | 70 | constructor(id: string, data: /* TODO */) { 71 | this.id = id 72 | this.nodeType = // TODO 73 | this.data = data 74 | } 75 | 76 | async run() { 77 | // Optional: Get inputs from previous nodes 78 | const input = usePlayStore.getState().getInput(this.id, "input") 79 | 80 | if (input) { 81 | // TODO Put the logic of the node here 82 | 83 | // Optional: Add downloadable file 84 | usePlayStore.getState().addFile(/* TODO */) 85 | 86 | // Optional: Make outputs accessable for the next node 87 | usePlayStore.getState().addOutgoingPipelines(this.id, /* TODO */) 88 | 89 | // Optional: Write to the log 90 | usePlayStore.getState().writeToLog(/* TODO */) 91 | 92 | // End with calling the next node 93 | usePlayStore.getState().nextNode() 94 | } 95 | } 96 | } 97 | ``` 98 | 99 | ## Step 6: Add Node Transformation Logic 100 | 101 | The final step involves adding the transformation logic for the node. This transformation will convert a [React Flow](https://reactflow.dev/) Node 102 | into an instance of your newly created class from **Step 5**. To do this, navigate to the **util/NodeMapTransformer.ts** file and 103 | add a new case to the ```getNodeFromType()``` method where you create the instance. -------------------------------------------------------------------------------- /documentation/docs/index.md: -------------------------------------------------------------------------------- 1 | # Welcome to Node‐Crawler's Wiki! 2 | 3 | Welcome to the official wiki page of the Node-Crawler repository. This is your go-to resource for understanding and using this tool. Node-Crawler is a powerful, flexible, and easy-to-use web crawler, with a node-based editing system built on top of [Next.js](https://nextjs.org/) and [React Flow](https://reactflow.dev/). 4 | 5 | Here you'll find detailed documentation about each of the functionalities of this application and guides on how to extend the application to help you make the most out of Node-Crawler. 6 | 7 | ## About Node-Crawler 8 | 9 | Node-Crawler is a highly customizable, Node-based web application for creating web crawlers and further processing and transforming the retrieved data. Users can build tailor-made web crawlers and manipulate and transform the collected data as needed. The output data format can be set to a wide range of formats including JSON, CSV, and various database formats. 10 | 11 | ![](https://raw.githubusercontent.com/MertenD/node-crawler/master/public/node-crawler-overview-canvas.png) 12 | ![](https://raw.githubusercontent.com/MertenD/node-crawler/master/public/node-crawler-overview-output.png) 13 | 14 | ## Project Goals and Use Cases 15 | 16 | - **Node-based Editing**: Users can create and edit their own crawler workflows by drag-and-dropping nodes. 17 | - **Data Transformation**: The application supports a variety of data manipulation and transformation operations for cleaning and restructuring the gathered data. 18 | - **Data Export**: The transformed data can be output in a variety of formats including JSON, CSV, and various database formats. 19 | 20 | 21 | ## Installation Guide 22 | 23 | Make sure you have [Node.js](https://nodejs.org/en) and npm installed on your system before you start. 24 | 25 | 1. Clone the repository: 26 | 27 | ```shell 28 | git clone https://github.com/MertenD/node-crawler.git 29 | ``` 30 | 31 | 2. Navigate into the directory and install the dependencies: 32 | 33 | ```shell 34 | cd node-crawler 35 | npm install 36 | ``` 37 | 38 | 3. Start the development server: 39 | 40 | ```shell 41 | npm run dev 42 | ``` 43 | 44 | Now you should be able to see the web application on `http://localhost:3000` in your browser. 45 | -------------------------------------------------------------------------------- /documentation/docs/stylesheets/extra.css: -------------------------------------------------------------------------------- 1 | :root { 2 | --md-default-fg-color: hsla(0, 100%, 100%, 0.9); 3 | --md-default-fg-color--light: hsla(0, 100%, 100%, 0.9); 4 | --md-default-fg-color--lighter: hsla(0, 100%, 100%, 0.9); 5 | --md-default-fg-color--lightest: #222C3C; 6 | --md-default-bg-color: #222C3C; 7 | --md-default-bg-color--light: #222C3C; 8 | --md-default-bg-color--lighter: #222C3C; 9 | --md-default-bg-color--lightest: #222C3C; 10 | 11 | --md-accent-fg-color: #9BA8BD; 12 | 13 | --md-primary-fg-color: #F98E35; 14 | --md-primary-fg-color--dark: #F98E35; 15 | 16 | --md-code-fg-color: hsla(0, 100%, 100%, 0.9); 17 | --md-code-bg-color: #1A202C; 18 | 19 | } -------------------------------------------------------------------------------- /documentation/mkdocs.yml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=https://squidfunk.github.io/mkdocs-material/schema.json 2 | 3 | site_name: Node-Crawler Docs 4 | site_url: https://mertend.github.io/node-crawler/ 5 | site_author: Merten Dieckmann 6 | site_description: "Node-Crawler is a highly customizable, Node-based web application for creating web crawlers and further processing and transforming the retrieved data. " 7 | repo_url: "https://github.com/MertenD/node-crawler" 8 | 9 | theme: 10 | name: material 11 | 12 | extra_css: 13 | - stylesheets/extra.css 14 | 15 | nav: 16 | - Home: index.md 17 | - Extend Project: 18 | - 'Implementing a new Node': 'extend-project/implementing-a-new-node.md' -------------------------------------------------------------------------------- /next.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | const nextConfig = {} 3 | 4 | module.exports = nextConfig 5 | -------------------------------------------------------------------------------- /nginx.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 3002; 3 | server_name localhost; 4 | 5 | location / { 6 | proxy_pass http://localhost:3002; 7 | proxy_http_version 1.1; 8 | proxy_set_header Upgrade $http_upgrade; 9 | proxy_set_header Connection 'upgrade'; 10 | proxy_set_header Host $host; 11 | proxy_cache_bypass $http_upgrade; 12 | } 13 | } 14 | 15 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "node-crawler", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint" 10 | }, 11 | "dependencies": { 12 | "@emotion/react": "^11.11.1", 13 | "@emotion/styled": "^11.11.0", 14 | "@fontsource/roboto": "^5.0.4", 15 | "@mui/icons-material": "^5.13.7", 16 | "@mui/lab": "^5.0.0-alpha.137", 17 | "@mui/material": "^5.13.7", 18 | "@types/node": "20.3.3", 19 | "@types/react": "18.2.14", 20 | "@types/react-beautiful-dnd": "^13.1.4", 21 | "@types/react-dom": "18.2.6", 22 | "autoprefixer": "10.4.14", 23 | "cheerio": "^1.0.0-rc.12", 24 | "css-selector-generator": "^3.6.4", 25 | "dompurify": "^3.0.5", 26 | "get-selector": "^1.0.4", 27 | "next": "13.4.8", 28 | "postcss": "8.4.24", 29 | "react": "18.2.0", 30 | "react-beautiful-dnd": "^13.1.1", 31 | "react-dom": "18.2.0", 32 | "react-flow-renderer": "^10.3.17", 33 | "react-toastify": "^9.1.3", 34 | "reactflow": "^11.7.4", 35 | "tailwindcss": "3.3.2", 36 | "typescript": "5.1.6", 37 | "uuid": "^9.0.0", 38 | "zustand": "4.3.3" 39 | }, 40 | "devDependencies": { 41 | "@types/cheerio": "^0.22.31", 42 | "@types/dompurify": "^3.0.2", 43 | "@types/uuid": "^9.0.2", 44 | "eslint": "^8.45.0" 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /public/next.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /public/node-crawler-overview-canvas.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MertenD/node-crawler/56c0a807be09c770e95336edd5bf9f7464254b49/public/node-crawler-overview-canvas.png -------------------------------------------------------------------------------- /public/node-crawler-overview-output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MertenD/node-crawler/56c0a807be09c770e95336edd5bf9f7464254b49/public/node-crawler-overview-output.png -------------------------------------------------------------------------------- /public/vercel.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/app/AppTheme.tsx: -------------------------------------------------------------------------------- 1 | 'use client' 2 | 3 | import {createTheme, ThemeProvider} from "@mui/material/styles"; 4 | import {ReactNode} from "react"; 5 | import {disabledColor, selectedColor, selectedColorHover, toolbarBackgroundColor} from "@/config/colors"; 6 | 7 | const appTheme = createTheme({ 8 | palette: { 9 | mode: "dark", 10 | text: { 11 | primary: "#fff" 12 | }, 13 | primary: { 14 | main: selectedColor 15 | }, 16 | background: { 17 | default: toolbarBackgroundColor 18 | }, 19 | warning: { 20 | main: selectedColor 21 | } 22 | }, 23 | components: { 24 | MuiButton: { 25 | styleOverrides: { 26 | contained: { 27 | backgroundColor: selectedColor + ' !important', 28 | '&:hover': { 29 | backgroundColor: selectedColorHover + ' !important', 30 | }, 31 | '&:disabled': { 32 | backgroundColor: disabledColor + ' !important', 33 | }, 34 | } 35 | } 36 | }, 37 | MuiMenu: { 38 | styleOverrides: { 39 | paper: { 40 | backgroundColor: toolbarBackgroundColor 41 | } 42 | } 43 | } 44 | } 45 | }); 46 | 47 | interface AppThemeProps { 48 | children: ReactNode; 49 | } 50 | 51 | export default function AppTheme({ children }: AppThemeProps) { 52 | return 53 | { children } 54 | 55 | } -------------------------------------------------------------------------------- /src/app/api/fetch/route.ts: -------------------------------------------------------------------------------- 1 | export async function POST(request: Request) { 2 | 3 | const responseHeaders = { 4 | 'Access-Control-Allow-Origin': '*', 5 | 'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS', 6 | 'Access-Control-Allow-Headers': 'Content-Type, Authorization', 7 | } 8 | 9 | try { 10 | const body = await request.json(); 11 | 12 | const response = await fetch(body.url); 13 | if (!response.ok) { 14 | throw new Error(`Status code was: ${response.status}`); 15 | } 16 | 17 | const data = await response.text(); 18 | 19 | return new Response(data, { 20 | status: 200, 21 | headers: responseHeaders 22 | }); 23 | } catch (error) { 24 | if (error instanceof Error) { 25 | return new Response(error.toString(), { 26 | status: 500, 27 | headers: responseHeaders 28 | }); 29 | } else { 30 | return new Response("An unexpected error occurred", { 31 | status: 500, 32 | headers: responseHeaders 33 | }); 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/app/editor/page.tsx: -------------------------------------------------------------------------------- 1 | 'use client' 2 | 3 | import {ReactFlowProvider} from "reactflow"; 4 | import React, {useEffect} from "react"; 5 | import HeaderBar, {CANVAS_HEIGHT} from "@/components/editor/headerbar/HeaderBar"; 6 | import useEditorPageState from "@/stores/editor/EditorPageStore"; 7 | import {Alert, Snackbar} from "@mui/material"; 8 | import Engine from "@/components/editor/Engine"; 9 | import {selectedColor} from "@/config/colors"; 10 | 11 | export default function Canvas() { 12 | 13 | const selectedPage = useEditorPageState(state => state.selectedPage) 14 | const getPage = useEditorPageState(state => state.getPage) 15 | 16 | const isSnackBarOpen = useEditorPageState(state => state.isSnackBarOpen) 17 | const setIsSnackBarOpen = useEditorPageState(state => state.setIsSnackBarOpen) 18 | const snackBarSeverity = useEditorPageState(state => state.snackBarSeverity) 19 | const snackBarText = useEditorPageState(state => state.snackBarText) 20 | 21 | const handleOnSnackBarClosed = () => { 22 | setIsSnackBarOpen(false) 23 | } 24 | 25 | useEffect(() => { 26 | window.addEventListener('beforeunload', preventReload) 27 | return () => { 28 | window.removeEventListener('beforeunload', preventReload) 29 | } 30 | }, []) 31 | 32 | const preventReload = (e: Event) => { 33 | e.preventDefault() 34 | } 35 | 36 | useEffect(() => { 37 | document.documentElement.style.setProperty('--selected-color', selectedColor); 38 | }, []); 39 | 40 | return
41 | 42 | 43 | 49 | 50 | { snackBarText } 51 | 52 | 53 | 54 |
55 | { getPage(selectedPage)?.child } 56 |
57 | 58 |
59 | } -------------------------------------------------------------------------------- /src/app/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | :root { 6 | --foreground-rgb: 255, 255, 255; 7 | --background-start-rgb: 33, 43, 60; 8 | --background-end-rgb: 33, 43, 60; 9 | } 10 | 11 | @media (prefers-color-scheme: dark) { 12 | :root { 13 | --foreground-rgb: 255, 255, 255; 14 | --background-start-rgb: 33, 43, 60; 15 | --background-end-rgb: 33, 43, 60; 16 | } 17 | } 18 | 19 | body { 20 | color: rgb(var(--foreground-rgb)); 21 | background: linear-gradient( 22 | to bottom, 23 | transparent, 24 | rgb(var(--background-end-rgb)) 25 | ) 26 | rgb(var(--background-start-rgb)); 27 | } 28 | -------------------------------------------------------------------------------- /src/app/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MertenD/node-crawler/56c0a807be09c770e95336edd5bf9f7464254b49/src/app/icon.png -------------------------------------------------------------------------------- /src/app/layout.tsx: -------------------------------------------------------------------------------- 1 | import './globals.css' 2 | import {Inter} from 'next/font/google' 3 | import React from "react"; 4 | import AppTheme from "./AppTheme"; 5 | 6 | const inter = Inter({ subsets: ['latin'] }) 7 | 8 | export const metadata = { 9 | title: 'Node-Crawler', 10 | description: 'Node-Crawler is a highly customizable, Node-based web application for creating web crawlers and further processing and transforming the retrieved data.', 11 | } 12 | 13 | export default function RootLayout({ 14 | children, 15 | }: { 16 | children: React.ReactNode 17 | }) { 18 | return ( 19 | 20 | 21 | 22 | {children} 23 | 24 | 25 | 26 | ) 27 | } 28 | -------------------------------------------------------------------------------- /src/app/page.tsx: -------------------------------------------------------------------------------- 1 | 'use client' 2 | 3 | import Typography from '@mui/material/Typography'; 4 | import {Button} from "@mui/material"; 5 | import Link from "next/link"; 6 | import useEditorPageState from "@/stores/editor/EditorPageStore"; 7 | 8 | // TODO Es wäre geil Verbindungsregeln in einer Config Datei setzten zu können. Vielleicht kann ich dort ja auch die NodeType und co. setzen. Eine einheitliche Struktur wäre auf jeden Fall schön 9 | 10 | export default function Home() { 11 | // noinspection HtmlUnknownTarget 12 | return
20 | 21 | Node-Crawler 22 | 23 | 24 | 31 | 32 |
33 | } 34 | -------------------------------------------------------------------------------- /src/components/editor/Engine.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from "react"; 2 | import { usePlayStore } from "@/stores/editor/PlayStore"; 3 | 4 | export default function Engine() { 5 | 6 | const [nodeContent, setNodeContent] = useState(null); 7 | const currentNode = usePlayStore((state) => state.currentNode); 8 | const isProcessRunning = usePlayStore(state => state.isProcessRunning) 9 | 10 | useEffect(() => { 11 | const runCurrentNode = async () => { 12 | if (currentNode !== null) { 13 | const content = await currentNode.node.run(); 14 | setNodeContent(content); 15 | } 16 | }; 17 | 18 | if (isProcessRunning) { 19 | // noinspection JSIgnoredPromiseFromCall 20 | runCurrentNode(); 21 | } 22 | }, [currentNode]); 23 | 24 | return
{nodeContent || <>}
; 25 | } -------------------------------------------------------------------------------- /src/components/editor/headerbar/HeaderBar.tsx: -------------------------------------------------------------------------------- 1 | import {useReactFlowStore} from "@/stores/editor/ReactFlowStore"; 2 | import React, {useRef} from "react"; 3 | import {IconButton, Tooltip} from "@mui/material"; 4 | import SaveIcon from '@mui/icons-material/Save'; 5 | import UploadIcon from '@mui/icons-material/Upload' 6 | import {loadCrawlerProject, onSave} from "@/util/IOUtil"; 7 | import {CrawlerProjectDto} from "@/model/CrawlerProjectDto"; 8 | import {useReactFlow} from "reactflow"; 9 | import PageNavigation from "@/components/editor/headerbar/PageNavigation"; 10 | import HomeIcon from '@mui/icons-material/Home'; 11 | import Link from "next/link"; 12 | import {usePlayStore} from "@/stores/editor/PlayStore"; 13 | import VisibilityIcon from '@mui/icons-material/Visibility'; 14 | import VisibilityOffIcon from '@mui/icons-material/VisibilityOff'; 15 | import SkipNextIcon from '@mui/icons-material/SkipNext'; 16 | import PlayArrowIcon from '@mui/icons-material/PlayArrow'; 17 | import StopIcon from '@mui/icons-material/Stop'; 18 | import {disabledColor, toolbarBackgroundColor} from "@/config/colors"; 19 | 20 | export const TOOLBAR_HEIGHT = 8 21 | export const CANVAS_HEIGHT = 100 - TOOLBAR_HEIGHT 22 | 23 | export default function HeaderBar() { 24 | 25 | // TODO Ich will einen Schritt für Schritt Debug mode, in dem bei jedem Schritt die Inputs und Outputs angezeigt werden, wöhrend man 26 | // beim aktuellen Knoten die Möglichkeit hat die Optionen anzupassen 27 | // + Live Preview des vorläugfigen Ergebnis was rauskomen würde 28 | 29 | const reactFlowInstance = useReactFlow(); 30 | const inputFile = useRef(null); 31 | 32 | const { nodes, edges } = useReactFlowStore(); 33 | const setup = usePlayStore(state => state.setup) 34 | const stop = usePlayStore(state => state.stop) 35 | const isProcessRunning = usePlayStore(state => state.isProcessRunning) 36 | const isConnectionHighlightingActivated = useReactFlowStore(state => state.isConnectionHighlightingActivated) 37 | const setIsConnectionHighlightingActivated = useReactFlowStore(state => state.setIsConnectionHighlightingActivated) 38 | const { isStepByStep, setIsStepByStep, isNextStepReady, executeNextStep } = usePlayStore() 39 | 40 | return
51 |
58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | { 67 | const data = { 68 | nodes: nodes, 69 | edges: edges 70 | } as CrawlerProjectDto 71 | const dataString = JSON.stringify(data, null, 2) 72 | onSave("crawler-project.ncp", dataString, "saveProject") 73 | }}> 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | { 82 | loadCrawlerProject(event, reactFlowInstance) 83 | }}/> 84 | 85 | 86 |
87 | 88 | { 89 | if (isStepByStep) { 90 | setIsStepByStep(false) 91 | executeNextStep() 92 | } else { 93 | setup() 94 | } 95 | }}> 96 | 97 | 98 | 99 | 100 | { 101 | if (!isStepByStep) { 102 | setIsStepByStep(true) 103 | setup() 104 | } else { 105 | executeNextStep() 106 | } 107 | }}> 108 | 109 | 110 | 111 | 112 | { 113 | usePlayStore.getState().writeToLog("Crawler was stopped manually") 114 | stop() 115 | }}> 116 | 117 | 118 | 119 |
120 | 121 | { 122 | setIsConnectionHighlightingActivated(!isConnectionHighlightingActivated) 123 | }}> 124 | { isConnectionHighlightingActivated ? : } 125 | 126 | 127 |
128 |
129 | 130 |
131 |
134 | 135 |
136 |
137 | } -------------------------------------------------------------------------------- /src/components/editor/headerbar/PageNavigation.tsx: -------------------------------------------------------------------------------- 1 | import {BottomNavigation, BottomNavigationAction} from "@mui/material"; 2 | import useEditorPageState from "@/stores/editor/EditorPageStore"; 3 | import React from "react"; 4 | 5 | export default function PageNavigation() { 6 | 7 | const { pages, selectedPage } = useEditorPageState() 8 | 9 | return { 16 | useEditorPageState.getState().onPageChanged(newSelectedPage) 17 | }} 18 | > 19 | { Array.from(pages, ([value, page]) => { 20 | const {label, icon} = page; 21 | return 26 | }) } 27 | 28 | } -------------------------------------------------------------------------------- /src/components/editor/pages/canvas/DragAndDropFlow.tsx: -------------------------------------------------------------------------------- 1 | 'use client' 2 | 3 | import ReactFlow, { 4 | Background, 5 | BackgroundVariant, 6 | Connection, 7 | Controls, 8 | MiniMap, 9 | Node, 10 | OnConnectStartParams, 11 | Panel, 12 | useReactFlow, 13 | } from 'reactflow'; 14 | import 'reactflow/dist/style.css'; 15 | import shallow from 'zustand/shallow'; 16 | import {useReactFlowStore} from "@/stores/editor/ReactFlowStore"; 17 | import React, {useCallback, useRef, useState} from "react"; 18 | import {v4 as uuidv4} from 'uuid'; 19 | import NodesToolbar from "@/components/editor/pages/canvas/toolbars/NodesToolbar"; 20 | import './DragAndDropFlowStyles.css' 21 | import './edges/EdgeGradientStyles.css' 22 | import OptionsToolbar from "@/components/editor/pages/canvas/toolbars/OptionsToolbar"; 23 | import {usePlayStore} from "@/stores/editor/PlayStore"; 24 | import OnCanvasNodesToolbar from "@/components/editor/pages/canvas/toolbars/OnCanvasNodesSelector"; 25 | import {getConnectionRule} from "@/config/ConnectionRules"; 26 | import {nodesMetadataMap} from "@/config/NodesMetadata"; 27 | import {NodeType} from "@/config/NodeType"; 28 | import {selectedColor, toolbarBackgroundColor} from "@/config/colors"; 29 | 30 | const selector = (state: any) => ({ 31 | nodes: state.nodes, 32 | edges: state.edges, 33 | onNodesChange: state.onNodesChange, 34 | onEdgesChange: state.onEdgesChange, 35 | onConnect: state.onConnect, 36 | nodeTypes: state.nodeTypes, 37 | edgeTypes: state.edgeTypes, 38 | getNodeById: state.getNodeById, 39 | setCurrentConnectionStartNode: state.setCurrentConnectionStartNode 40 | }); 41 | 42 | export default function DragAndDropFlow() { 43 | const { nodes, edges, onNodesChange, onEdgesChange, onConnect, nodeTypes, edgeTypes, getNodeById, setCurrentConnectionStartNode } = useReactFlowStore(selector, shallow) 44 | 45 | const setSelectedNodes = useReactFlowStore((state) => state.setSelectedNodes) 46 | 47 | const connectStartParams = useRef(null) 48 | const reactFlowWrapper = useRef(null) 49 | const reactFlowInstance = useReactFlow() 50 | 51 | const [isOnCanvasNodeSelectorOpen, setIsOnCanvasNodeSelectorOpen] = useState(false) 52 | const [lastEventPosition, setLastEventPosition] = useState<{x: number, y: number}>({x: 0, y: 0}) 53 | 54 | const isProcessRunning = usePlayStore(state => state.isProcessRunning) 55 | const isStepByStep = usePlayStore(state => state.isStepByStep) 56 | const isNextStepReady = usePlayStore(state => state.isNextStepReady) 57 | 58 | const onDragOver = useCallback((event: any) => { 59 | event.preventDefault(); 60 | event.dataTransfer.dropEffect = 'move'; 61 | }, []); 62 | 63 | const onDrop = useCallback((event: any) => { 64 | event.preventDefault(); 65 | 66 | // @ts-ignore 67 | const reactFlowBounds = reactFlowWrapper.current.getBoundingClientRect(); 68 | const { nodeType, nodeData } = JSON.parse(event.dataTransfer.getData('application/reactflow')); 69 | 70 | // check if the dropped element is valid 71 | if (typeof nodeType === 'undefined' || !nodeType) { 72 | return; 73 | } 74 | 75 | const position = reactFlowInstance.project({ 76 | x: event.clientX - reactFlowBounds.left, 77 | y: event.clientY - reactFlowBounds.top, 78 | }); 79 | 80 | addNodeAtPosition(position, nodeType, nodeData) 81 | }, [reactFlowInstance]); 82 | 83 | const onConnectStart = useCallback((event: any, node: OnConnectStartParams) => { 84 | connectStartParams.current = node; 85 | setCurrentConnectionStartNode(getNodeById(node.nodeId)) 86 | }, []); 87 | 88 | const onConnectEnd = useCallback( 89 | (event: any) => { 90 | setCurrentConnectionStartNode(null) 91 | const targetIsPane = event.target.classList.contains('react-flow__pane'); 92 | const targetIsChallengeNode = event.target.parentElement.classList.contains("react-flow__node-challengeNode") 93 | 94 | if ((targetIsPane || targetIsChallengeNode) && connectStartParams.current?.handleType === "source" && reactFlowWrapper.current !== null) { 95 | // @ts-ignore 96 | const { top, left } = reactFlowWrapper.current.getBoundingClientRect(); 97 | setLastEventPosition({ x: event.clientX - left, y: event.clientY - top }) 98 | setIsOnCanvasNodeSelectorOpen(true) 99 | } 100 | }, 101 | [reactFlowInstance.project] 102 | ); 103 | 104 | function addNodeAtPosition(position: {x: number, y: number}, nodeType: NodeType, data: any = {}): string { 105 | let yOffset = 0 106 | let zIndex = 0 107 | 108 | let nodeInfo = nodesMetadataMap[nodeType] 109 | if(nodeInfo && nodeInfo.style.minHeight && typeof nodeInfo.style.minHeight === "number") { 110 | yOffset += nodeInfo.style.minHeight / 2 111 | } 112 | 113 | const id = uuidv4(); 114 | const newNode = { 115 | id, 116 | type: nodeType, 117 | position: { ...position, y: position.y - yOffset }, 118 | zIndex: zIndex, 119 | data: data, 120 | } as Node; 121 | 122 | reactFlowInstance.addNodes(newNode); 123 | 124 | return id 125 | } 126 | 127 | return ( 128 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 165 | { 172 | setIsOnCanvasNodeSelectorOpen(false) 173 | 174 | if (nodeType !== null && connectStartParams.current !== null && connectStartParams.current?.nodeId !== null) { 175 | const id = addNodeAtPosition(reactFlowInstance.project(lastEventPosition), nodeType) 176 | let rule = getConnectionRule(nodeType, id) 177 | if(rule && rule.inputRules && rule.inputRules.length > 0) { 178 | onConnect({ 179 | source: connectStartParams.current?.nodeId, 180 | target: id, 181 | sourceHandle: connectStartParams.current?.handleId, 182 | // TODO Eventuell abändern, sobald ich dynamische nodes in dem onCanvasSelect drin hab 183 | targetHandle: rule.inputRules[0].handleId 184 | } as Connection) 185 | } 186 | } 187 | }} 188 | /> 189 | 190 | ); 191 | } -------------------------------------------------------------------------------- /src/components/editor/pages/canvas/DragAndDropFlowStyles.css: -------------------------------------------------------------------------------- 1 | /*noinspection ALL,CssUnusedSymbol*/ 2 | .react-flow__controls-button { 3 | background: #1A202C; 4 | border-color: #1A202C; 5 | fill: white 6 | } 7 | 8 | /*noinspection ALL,CssUnusedSymbol*/ 9 | .react-flow__controls-button:hover { 10 | background-color: #131820; 11 | } 12 | 13 | .react-flow__minimap svg path { 14 | fill: #131820aa 15 | } 16 | -------------------------------------------------------------------------------- /src/components/editor/pages/canvas/edges/EdgeGradientStyles.css: -------------------------------------------------------------------------------- 1 | /* Incoming Edge */ 2 | /*noinspection ALL,CssUnusedSymbol*/ 3 | .react-flow__edge.incoming .react-flow__edge-path { 4 | stroke: url(#incoming-edge-gradient); 5 | stroke-width: 2; 6 | stroke-opacity: 0.75; 7 | } 8 | 9 | /* Outgoing Edge */ 10 | /*noinspection ALL,CssUnusedSymbol*/ 11 | .react-flow__edge.outgoing .react-flow__edge-path { 12 | stroke: url(#outgoing-edge-gradient); 13 | stroke-width: 2; 14 | stroke-opacity: 0.75; 15 | } 16 | 17 | /* Both Selected Edge */ 18 | /*noinspection ALL,CssUnusedSymbol*/ 19 | .react-flow__edge.bothSelected .react-flow__edge-path { 20 | stroke: url(#bothSelected-edge-gradient); 21 | stroke-width: 2; 22 | stroke-opacity: 0.75; 23 | } 24 | 25 | /* Edge Selected Edge */ 26 | /*noinspection ALL,CssUnusedSymbol*/ 27 | .react-flow__edge.edgeSelected .react-flow__edge-path { 28 | stroke: url(#edgeSelected-edge-gradient); 29 | stroke-width: 2; 30 | stroke-opacity: 0.75; 31 | filter: drop-shadow(0px 0px 5px var(--selected-color)); 32 | } 33 | 34 | /* Default Edge */ 35 | /*noinspection ALL,CssUnusedSymbol*/ 36 | .react-flow__edge.default .react-flow__edge-path { 37 | stroke: url(#default-edge-gradient); 38 | stroke-width: 2; 39 | stroke-opacity: 0.75; 40 | } 41 | 42 | /* HtmlHighlight Edge */ 43 | /*noinspection ALL,CssUnusedSymbol*/ 44 | .react-flow__edge.html .react-flow__edge-path { 45 | stroke: url(#html-edge-gradient); 46 | stroke-width: 2; 47 | stroke-opacity: 0.75; 48 | } 49 | 50 | /* TextHighlight Edge */ 51 | /*noinspection ALL,CssUnusedSymbol*/ 52 | .react-flow__edge.text .react-flow__edge-path { 53 | stroke: url(#text-edge-gradient); 54 | stroke-width: 2; 55 | stroke-opacity: 0.75; 56 | } 57 | 58 | /* JsonHighlight Edge */ 59 | /*noinspection ALL,CssUnusedSymbol*/ 60 | .react-flow__edge.json .react-flow__edge-path { 61 | stroke: url(#json-edge-gradient); 62 | stroke-width: 2; 63 | stroke-opacity: 0.75; 64 | } 65 | 66 | /* NoneHighlight Edge */ 67 | /*noinspection ALL,CssUnusedSymbol*/ 68 | .react-flow__edge.none .react-flow__edge-path { 69 | stroke: url(#none-edge-gradient); 70 | stroke-width: 2; 71 | stroke-opacity: 0.75; 72 | } 73 | 74 | /* DatabaseHighlight Edge */ 75 | /*noinspection ALL,CssUnusedSymbol*/ 76 | .react-flow__edge.database .react-flow__edge-path { 77 | stroke: url(#database-edge-gradient); 78 | stroke-width: 2; 79 | stroke-opacity: 0.75; 80 | } 81 | -------------------------------------------------------------------------------- /src/components/editor/pages/canvas/edges/Edges.ts: -------------------------------------------------------------------------------- 1 | import {createEdge} from "@/components/editor/pages/canvas/edges/util/EdgeCreators"; 2 | import {EdgeProps} from "reactflow"; 3 | import React from "react"; 4 | import {OutputValueType} from "@/config/OutputValueType"; 5 | import './EdgeGradientStyles.css' 6 | import {defaultEdgeColor, selectedColor} from "@/config/colors"; 7 | 8 | export const DefaultEdge = createEdge(defaultEdgeColor, defaultEdgeColor, "default") 9 | export const SelectedIncomingEdge = createEdge(defaultEdgeColor, selectedColor, "incoming") 10 | export const SelectedOutgoingEdge = createEdge(selectedColor, defaultEdgeColor, "outgoing") 11 | export const BothSelectedEdge = createEdge(selectedColor, selectedColor, "bothSelected") 12 | export const EdgeSelectedEdge = createEdge(selectedColor, selectedColor, "edgeSelected") 13 | 14 | export const highlightEdges: HighlightEdgeInfoTypes = { 15 | [OutputValueType.NONE]: createEdge(defaultEdgeColor, defaultEdgeColor, OutputValueType.NONE), 16 | [OutputValueType.TEXT]: createEdge("#7CFC00", "#7CFC00", OutputValueType.TEXT), 17 | [OutputValueType.JSON]: createEdge("#87CEEB", "#87CEEB", OutputValueType.JSON), 18 | [OutputValueType.HTML]: createEdge("#FF00FF", "#FF00FF", OutputValueType.HTML), 19 | [OutputValueType.DATABASE]: createEdge("#FFFF00", "#FFFF00", OutputValueType.DATABASE) 20 | } 21 | 22 | type HighlightEdgeInfoTypes = { 23 | [K in OutputValueType]: EdgeType 24 | } 25 | 26 | export type EdgeType = ({id, sourceX, sourceY, targetX, targetY, sourcePosition, targetPosition, style, markerEnd}: EdgeProps) => React.ReactNode | null 27 | -------------------------------------------------------------------------------- /src/components/editor/pages/canvas/edges/util/EdgeCreators.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import {EdgeProps, getSmoothStepPath} from 'reactflow'; 3 | import "../EdgeGradientStyles.css" 4 | 5 | export function createEdge(startColor: string, endColor: string, edgeId: string) { 6 | 7 | return ({ 8 | id, 9 | sourceX, 10 | sourceY, 11 | targetX, 12 | targetY, 13 | sourcePosition, 14 | targetPosition, 15 | style = {} 16 | }: EdgeProps) => { 17 | 18 | const xEqual = sourceX === targetX; 19 | const yEqual = sourceY === targetY; 20 | 21 | const [edgePath] = getSmoothStepPath({ 22 | sourceX: xEqual ? sourceX + 0.0001 : sourceX, 23 | sourceY: yEqual ? sourceY + 0.0001 : sourceY, 24 | sourcePosition, 25 | targetX, 26 | targetY, 27 | targetPosition, 28 | }); 29 | 30 | return ( 31 | 32 | 33 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 57 | 58 | ); 59 | } 60 | } -------------------------------------------------------------------------------- /src/components/editor/pages/canvas/nodes/DatabaseTableNode.tsx: -------------------------------------------------------------------------------- 1 | // --- Data --- 2 | import { 3 | createDynamicNodeComponent, 4 | createNodeShapeStyle, 5 | createOptionsComponent 6 | } from "@/components/editor/pages/canvas/nodes/util/Creators"; 7 | import {NodeType} from "@/config/NodeType"; 8 | import React, {useEffect} from "react"; 9 | import {NodeMetadata} from "@/config/NodesMetadata"; 10 | import {DynamicNodeData, NodeData} from "@/model/NodeData"; 11 | import {EngineDatabaseTableNode} from "@/engine/nodes/EngineDatabaseTableNode"; 12 | import {Button, IconButton} from "@mui/material"; 13 | import {InputRule} from "@/config/ConnectionRules"; 14 | import {OutputValueType} from "@/config/OutputValueType"; 15 | import TextInputOption from "@/components/form/TextInputOption"; 16 | import StorageIcon from '@mui/icons-material/Storage'; 17 | import RowOptionsContainer from "@/components/form/RowOptionsContainer"; 18 | import DeleteForeverIcon from '@mui/icons-material/DeleteForever'; 19 | import MultiSelectOption from "@/components/form/MultiSelectOption"; 20 | import useReactFlowStore from "@/stores/editor/ReactFlowStore"; 21 | import DraggableOptionsListContainer from "@/components/form/DraggableOptionsListContainer"; 22 | 23 | export interface DatabaseTableNodeData extends DynamicNodeData { 24 | tableName: string 25 | } 26 | 27 | // --- Style --- 28 | export const DatabaseTableShapeStyle = createNodeShapeStyle({ 29 | height: 70, 30 | width: 100, 31 | display: "flex", 32 | flexDirection: "column", 33 | alignItems: "center" 34 | }) 35 | 36 | // --- Node --- 37 | export const DatabaseTableNode = createDynamicNodeComponent( 38 | NodeType.DATABASE_TABLE_NODE, 39 | DatabaseTableShapeStyle, 40 | () => { 41 | return
49 | 50 |
51 | } 52 | ) 53 | 54 | // --- Options --- 55 | export const DatabaseTableOptions = createOptionsComponent("Database Table", ({ id, data, onDataUpdated }) => { 56 | 57 | const [inputs, setInputs] = React.useState(data.connectionRule ? [...data.connectionRule.inputRules] : []) 58 | 59 | function addInput() { 60 | setInputs([...inputs, { 61 | handleId: "", 62 | allowedValueTypes: [ 63 | OutputValueType.HTML 64 | ], 65 | maxConnections: 999 66 | } as InputRule]) 67 | } 68 | 69 | useEffect(() => { 70 | onDataUpdated("connectionRule", { 71 | ...data.connectionRule, 72 | inputRules: inputs, 73 | outputValueType: OutputValueType.DATABASE 74 | }) 75 | }, [inputs]); 76 | 77 | return <> 78 | ) => { 82 | onDataUpdated("tableName", event.target.value) 83 | }} 84 | /> 85 | 86 | items={inputs} 87 | onOrderChanged={(newItems) => setInputs(newItems)} 88 | mapItem={(item: InputRule, index: number) => { 89 | return 90 | ) => { 95 | const cleanedValue = event.target.value.replaceAll(" ", "-") 96 | useReactFlowStore.getState().replaceEdgeAfterHandleRename(id, item.handleId, cleanedValue) 97 | const newInputs = [...inputs] 98 | newInputs[index].handleId = cleanedValue 99 | setInputs(newInputs) 100 | }} 101 | /> 102 | { 106 | const newInputs = [...inputs] 107 | newInputs[index].allowedValueTypes = newSelection.map(selection => selection as OutputValueType) 108 | setInputs(newInputs) 109 | }} 110 | /> 111 | { 113 | const newInputs = [...inputs] 114 | newInputs.splice(index, 1) 115 | setInputs(newInputs) 116 | } 117 | } style={{ width: 40, height: 40 }} > 118 | 119 | 120 | 121 | }} 122 | /> 123 | 124 | 125 | }) 126 | 127 | 128 | // --- Metadata --- 129 | export const DatabaseTableNodeMetadata = { 130 | title: "Database Table", 131 | type: NodeType.DATABASE_TABLE_NODE, 132 | getNodeComponent: DatabaseTableNode, 133 | getOptionsComponent: (id: string) => , 134 | style: DatabaseTableShapeStyle(true), 135 | icon: , 136 | getEngineNode: (id: string, data: NodeData) => { 137 | return new EngineDatabaseTableNode(id, data as DatabaseTableNodeData) 138 | } 139 | } as NodeMetadata -------------------------------------------------------------------------------- /src/components/editor/pages/canvas/nodes/ExtractorNode.tsx: -------------------------------------------------------------------------------- 1 | // --- Data --- 2 | import { 3 | createStaticNodeComponent, 4 | createNodeShapeStyle, 5 | createOptionsComponent 6 | } from "@/components/editor/pages/canvas/nodes/util/Creators"; 7 | import {NodeData} from "@/model/NodeData"; 8 | import {NodeType} from "@/config/NodeType"; 9 | import React from "react"; 10 | import ManageSearchIcon from '@mui/icons-material/ManageSearch'; 11 | import TextInputOption from "@/components/form/TextInputOption"; 12 | import {EngineExtractorNode} from "@/engine/nodes/EngineExtractorNode"; 13 | import {NodeMetadata} from "@/config/NodesMetadata"; 14 | import SelectOption from "@/components/form/SelectOption"; 15 | 16 | export interface ExtractorNodeData extends NodeData { 17 | tag: string 18 | extractionMode: ExtractionMode 19 | attributeToExtract: string 20 | } 21 | 22 | // --- Style --- 23 | export const extractorShapeStyle = createNodeShapeStyle() 24 | 25 | // --- Node --- 26 | export const ExtractorNode = createStaticNodeComponent( 27 | NodeType.EXTRACTOR_NODE, 28 | extractorShapeStyle, 29 | () => { 30 | return
38 | 39 |
40 | } 41 | ) 42 | 43 | // --- Options --- 44 | export const ExtractorOptions = createOptionsComponent("Extractor", ({ id, data, onDataUpdated }) => { 45 | return <> 46 | { 50 | onDataUpdated("tag", event.target.value) 51 | }} 52 | /> 53 | { 57 | onDataUpdated("extractionMode", newSelection) 58 | }} 59 | label={"Select what should be extracted"} 60 | /> 61 | { data.extractionMode === ExtractionMode.ATTRIBUTE && { 65 | onDataUpdated("attributeToExtract", event.target.value) 66 | }} 67 | /> } 68 | 69 | }) 70 | 71 | 72 | // --- Metadata --- 73 | export const extractorNodeMetadata = { 74 | title: "Extractor", 75 | type: NodeType.EXTRACTOR_NODE, 76 | getNodeComponent: ExtractorNode, 77 | getOptionsComponent: (id: string) => , 78 | style: extractorShapeStyle(true), 79 | icon: , 80 | getEngineNode: (id: string, data: NodeData) => { 81 | return new EngineExtractorNode(id, data as ExtractorNodeData) 82 | } 83 | } as NodeMetadata 84 | 85 | 86 | export enum ExtractionMode { 87 | CONTENT = "Content", 88 | ATTRIBUTE = "Attribute" 89 | } -------------------------------------------------------------------------------- /src/components/editor/pages/canvas/nodes/FetchWebsiteNode.tsx: -------------------------------------------------------------------------------- 1 | 'use client' 2 | 3 | import React from "react"; 4 | import TextInputOption from "@/components/form/TextInputOption"; 5 | import {Badge, Button, IconButton, Typography} from "@mui/material"; 6 | import CloudDownloadTwoToneIcon from '@mui/icons-material/CloudDownloadTwoTone'; 7 | import {NodeData} from "@/model/NodeData"; 8 | import { 9 | createNodeShapeStyle, 10 | createOptionsComponent, 11 | createStaticNodeComponent 12 | } from "@/components/editor/pages/canvas/nodes/util/Creators"; 13 | import {EngineFetchWebsiteNode} from "@/engine/nodes/EngineFetchWebsiteNode"; 14 | import {NodeType} from "@/config/NodeType"; 15 | import {NodeMetadata} from "@/config/NodesMetadata"; 16 | import {defaultEdgeColor, selectedColor} from "@/config/colors"; 17 | import DraggableOptionsListContainer from "@/components/form/DraggableOptionsListContainer"; 18 | import DeleteForeverIcon from '@mui/icons-material/DeleteForever'; 19 | 20 | 21 | // --- Data --- 22 | export interface FetchWebsiteNodeData extends NodeData { 23 | name: string 24 | urls: string[] 25 | } 26 | 27 | 28 | // --- Style --- 29 | export const fetchWebsiteShapeStyle = createNodeShapeStyle({ 30 | minWidth: 100 31 | }) 32 | 33 | 34 | // --- Node --- 35 | export const FetchWebsiteNode = createStaticNodeComponent( 36 | NodeType.FETCH_WEBSITE_NODE, 37 | fetchWebsiteShapeStyle, 38 | (id, selected, data) => { 39 | return } anchorOrigin={{ horizontal: "left", vertical: "top" }} > 42 |
50 | 51 | { (data.name && data.name !== "") ? data.name : "Fetch Website" } 52 | 53 |
54 | 55 | } 56 | ) 57 | 58 | 59 | // --- Options --- 60 | export const FetchWebsiteOptions = createOptionsComponent("Fetch Website", ({ id, data, onDataUpdated }) => { 61 | 62 | const [urls, setUrls] = React.useState(data.urls ?? [""]) 63 | 64 | function addValue() { 65 | setUrls([...urls, ""]) 66 | } 67 | 68 | return <> 69 | ) => { 73 | onDataUpdated("name", event.target.value) 74 | }} 75 | /> 76 | setUrls(newUrls)} 79 | mapItem={(url: string, index: number) => { 80 | return 0 ? index + 1 : "")} 83 | value={url} 84 | onChange={(event: React.ChangeEvent) => { 85 | const newUrls = [...urls] 86 | newUrls[index] = event.target.value 87 | setUrls(newUrls) 88 | onDataUpdated("urls", newUrls) 89 | }} 90 | inputProps = {{ 91 | endAdornment: ( 92 | index > 0 ? 93 | { 95 | const newUrls = [...urls] 96 | newUrls.splice(index, 1) 97 | setUrls(newUrls) 98 | onDataUpdated("urls", newUrls) 99 | } 100 | }> 101 | 102 | 103 | : 104 | <> 105 | ) 106 | }} 107 | /> 108 | }} 109 | /> 110 | 111 | 112 | }) 113 | 114 | 115 | // --- Metadata --- 116 | export const fetchWebsiteNodeMetadata = { 117 | title: "Fetch Website", 118 | type: NodeType.FETCH_WEBSITE_NODE, 119 | getNodeComponent: FetchWebsiteNode, 120 | getOptionsComponent: (id: string) => , 121 | style: fetchWebsiteShapeStyle(true), 122 | icon: , 123 | getEngineNode: (id: string, data: NodeData) => { 124 | return new EngineFetchWebsiteNode(id, data as FetchWebsiteNodeData) 125 | } 126 | } as NodeMetadata -------------------------------------------------------------------------------- /src/components/editor/pages/canvas/nodes/HtmlToTextNode.tsx: -------------------------------------------------------------------------------- 1 | 'use client' 2 | 3 | import React from "react"; 4 | import {NodeData} from "@/model/NodeData"; 5 | import { 6 | createStaticNodeComponent, 7 | createNodeShapeStyle, 8 | createOptionsComponent 9 | } from "@/components/editor/pages/canvas/nodes/util/Creators"; 10 | import {NodeType} from "@/config/NodeType"; 11 | import {NodeMetadata} from "@/config/NodesMetadata"; 12 | import SwapHorizIcon from '@mui/icons-material/SwapHoriz'; 13 | import {EngineHtmlToTextNode} from "@/engine/nodes/EngineHtmlToTextNode"; 14 | 15 | 16 | // --- Data --- 17 | export interface HtmlToTextNodeData extends NodeData { 18 | fileName: string 19 | extension: string 20 | separator: string 21 | } 22 | 23 | 24 | // --- Style --- 25 | export const htmlToTextShapeStyle = createNodeShapeStyle() 26 | 27 | 28 | // --- Node --- 29 | export const HtmlToTextNode = createStaticNodeComponent( 30 | NodeType.HTML_TO_TEXT_NODE, 31 | htmlToTextShapeStyle, 32 | () => { 33 | return
41 | 42 |
43 | } 44 | ) 45 | 46 | // --- Options --- 47 | export const HtmlToTextOptions = createOptionsComponent("HtmlToText", ({id, data, onDataUpdated}) => { 48 | return <> 49 | }) 50 | 51 | 52 | // --- Metadata --- 53 | export const htmlToTextNodeMetadata = { 54 | title: "HtmlToText", 55 | type: NodeType.HTML_TO_TEXT_NODE, 56 | getNodeComponent: HtmlToTextNode, 57 | getOptionsComponent: (id: string) => , 58 | style: htmlToTextShapeStyle(true), 59 | icon: , 60 | getEngineNode: (id: string, data: NodeData) => { 61 | return new EngineHtmlToTextNode(id, data as HtmlToTextNodeData) 62 | } 63 | } as NodeMetadata -------------------------------------------------------------------------------- /src/components/editor/pages/canvas/nodes/SaveNode.tsx: -------------------------------------------------------------------------------- 1 | 'use client' 2 | 3 | import React from "react"; 4 | import {NodeData} from "@/model/NodeData"; 5 | import FileNameInputOption from "@/components/form/FileNameInputOption"; 6 | import TextInputOption from "@/components/form/TextInputOption"; 7 | import { 8 | createStaticNodeComponent, 9 | createNodeShapeStyle, 10 | createOptionsComponent 11 | } from "@/components/editor/pages/canvas/nodes/util/Creators"; 12 | import {NodeType} from "@/config/NodeType"; 13 | import SaveIcon from '@mui/icons-material/Save'; 14 | import {EngineSaveNode} from "@/engine/nodes/EngineSaveNode"; 15 | import {NodeMetadata} from "@/config/NodesMetadata"; 16 | 17 | 18 | // --- Data --- 19 | export interface SaveNodeData extends NodeData { 20 | fileName: string 21 | extension: string 22 | separator: string 23 | } 24 | 25 | 26 | // --- Style --- 27 | export const saveShapeStyle = createNodeShapeStyle() 28 | 29 | 30 | // --- Node --- 31 | export const SaveNode = createStaticNodeComponent( 32 | NodeType.SAVE_NODE, 33 | saveShapeStyle, 34 | () => { 35 | return
43 | 44 |
45 | } 46 | ) 47 | 48 | // --- Options --- 49 | export const SaveOptions = createOptionsComponent("Save", ({id, data, onDataUpdated}) => { 50 | return <> 51 | { 54 | onDataUpdated("fileName", event.target.value) 55 | }} 56 | extension={data.extension} 57 | onExtensionChange={(event) => { 58 | onDataUpdated("extension", event.target.value) 59 | }} 60 | /> 61 | { 65 | onDataUpdated("separator", event.target.value) 66 | }} 67 | /> 68 | 69 | }) 70 | 71 | 72 | // --- Metadata --- 73 | export const saveNodeMetadata = { 74 | title: "Save", 75 | type: NodeType.SAVE_NODE, 76 | getNodeComponent: SaveNode, 77 | getOptionsComponent: (id: string) => , 78 | style: saveShapeStyle(true), 79 | icon: , 80 | getEngineNode: (id: string, data: NodeData) => { 81 | return new EngineSaveNode(id, data as SaveNodeData) 82 | } 83 | } as NodeMetadata -------------------------------------------------------------------------------- /src/components/editor/pages/canvas/nodes/StartNode.tsx: -------------------------------------------------------------------------------- 1 | 'use client' 2 | 3 | import React from "react"; 4 | import {NodeData} from "@/model/NodeData"; 5 | import {LoadingButton} from "@mui/lab"; 6 | import PlayCircleFilledIcon from '@mui/icons-material/PlayCircleFilled'; 7 | import {usePlayStore} from "@/stores/editor/PlayStore"; 8 | import { 9 | createStaticNodeComponent, 10 | createNodeShapeStyle, 11 | createOptionsComponent 12 | } from "@/components/editor/pages/canvas/nodes/util/Creators"; 13 | import {NodeType} from "@/config/NodeType"; 14 | import PlayArrowIcon from "@mui/icons-material/PlayArrow"; 15 | import {EngineStartNode} from "@/engine/nodes/EngineStartNode"; 16 | import {NodeMetadata} from "@/config/NodesMetadata"; 17 | 18 | // --- Data --- 19 | export interface StartNodeData extends NodeData { 20 | } 21 | 22 | 23 | // --- Style --- 24 | export const startShapeStyle = createNodeShapeStyle({ 25 | borderRadius: "50%" 26 | }) 27 | 28 | 29 | // --- Node --- 30 | export const StartNode = createStaticNodeComponent( 31 | NodeType.START_NODE, 32 | startShapeStyle, 33 | () => { 34 | return
42 | 43 |
44 | } 45 | ) 46 | 47 | // --- Options --- 48 | export const StartOptions = createOptionsComponent("Start", () => { 49 | return } 52 | variant="contained" 53 | onClick={() => { 54 | usePlayStore.getState().setup() 55 | }} 56 | sx={{ 57 | width: "100%" 58 | }} 59 | > 60 | Start Crawler 61 | 62 | }) 63 | 64 | 65 | // --- Metadata --- 66 | export const startNodeMetadata = { 67 | title: "Start", 68 | type: NodeType.START_NODE, 69 | getNodeComponent: StartNode, 70 | getOptionsComponent: (id: string) => , 71 | style: startShapeStyle(true), 72 | icon: , 73 | getEngineNode: (id: string, data: NodeData) => { 74 | return new EngineStartNode(id, data as StartNodeData) 75 | } 76 | } as NodeMetadata -------------------------------------------------------------------------------- /src/components/editor/pages/canvas/nodes/ZipNode.tsx: -------------------------------------------------------------------------------- 1 | 'use client' 2 | 3 | import React from "react"; 4 | import {NodeData} from "@/model/NodeData"; 5 | import { 6 | createStaticNodeComponent, 7 | createNodeShapeStyle, 8 | createOptionsComponent 9 | } from "@/components/editor/pages/canvas/nodes/util/Creators"; 10 | import {NodeType} from "@/config/NodeType"; 11 | import {NodeMetadata} from "@/config/NodesMetadata"; 12 | import MergeIcon from '@mui/icons-material/Merge'; 13 | import {EngineZipNode} from "@/engine/nodes/EngineZipNode"; 14 | 15 | // TODO beliebig viele Inputs jeweils mit Namen einstellbar in den Optionen. Dafür muss ziemlich sicher wieder etwas an den connectionRules geändert werden 16 | 17 | // --- Data --- 18 | export interface ZipNodeData extends NodeData { 19 | } 20 | 21 | 22 | // --- Style --- 23 | export const zipShapeStyle = createNodeShapeStyle({}) 24 | 25 | 26 | // --- Node --- 27 | export const ZipNode = createStaticNodeComponent( 28 | NodeType.ZIP_NODE, 29 | zipShapeStyle, 30 | () => { 31 | return
39 | 40 |
41 | } 42 | ) 43 | 44 | // --- Options --- 45 | export const ZipOptions = createOptionsComponent("Zip", () => { 46 | return <> 47 | }) 48 | 49 | 50 | // --- Metadata --- 51 | export const zipNodeMetadata = { 52 | title: "Zip", 53 | type: NodeType.ZIP_NODE, 54 | getNodeComponent: ZipNode, 55 | getOptionsComponent: (id: string) => , 56 | style: zipShapeStyle(true), 57 | icon: , 58 | getEngineNode: (id: string, data: NodeData) => { 59 | return new EngineZipNode(id, data as ZipNodeData) 60 | } 61 | } as NodeMetadata -------------------------------------------------------------------------------- /src/components/editor/pages/canvas/nodes/util/Creators.tsx: -------------------------------------------------------------------------------- 1 | import {Handle, Node, NodeProps, Position, useUpdateNodeInternals} from "reactflow"; 2 | import React, {CSSProperties, useEffect, useMemo, useState} from "react"; 3 | import {handleStyle, useReactFlowStore} from "@/stores/editor/ReactFlowStore"; 4 | import {addOrUpdateDynamicRule, getInputRules, getOutputValueType} from "@/config/ConnectionRules"; 5 | import OptionsContainer from "@/components/form/OptionsContainer"; 6 | import {setNodeWithUpdatedDataValue} from "@/components/editor/pages/canvas/nodes/util/OptionsUtil"; 7 | import {usePlayStore} from "@/stores/editor/PlayStore"; 8 | import CacheTextField from "@/components/form/CacheTextField"; 9 | import {Tooltip} from "@mui/material"; 10 | import {NodeType} from "@/config/NodeType"; 11 | import {DynamicNodeData, NodeData} from "@/model/NodeData"; 12 | import {nodeBackgroundColor, nodeShadowColor, selectedColor} from "@/config/colors"; 13 | import Typography from "@mui/material/Typography"; 14 | 15 | export const createNodeShapeStyle = (additionalCSS: CSSProperties = {}): (selected: boolean) => CSSProperties => { 16 | return function(selected) { 17 | return { 18 | minWidth: 60, 19 | minHeight: 60, 20 | backgroundColor: nodeBackgroundColor, 21 | borderRadius: 6, 22 | border: "2px solid", 23 | borderColor: selected ? selectedColor : nodeBackgroundColor, 24 | boxShadow: selected ? `0px 0px 5px 1px ${selectedColor}` : `0px 0px 3px 2px ${nodeShadowColor}`, 25 | ...additionalCSS 26 | } as CSSProperties 27 | } 28 | } 29 | 30 | export function createStaticNodeComponent( 31 | nodeType: NodeType, 32 | shapeStyle: (isSelected: boolean) => CSSProperties, 33 | content: (id: string, selected: boolean, data: DataType) => React.ReactNode 34 | ) { 35 | 36 | return function({ id, selected, data }: NodeProps) { 37 | 38 | const inputRules = getInputRules(nodeType, id) 39 | 40 | const currentConnectionStartNode = useReactFlowStore(state => state.currentConnectionStartNode) 41 | const handleHighlightedMap = useMemo(() => { 42 | let newMap = new Map() 43 | 44 | if (inputRules && currentConnectionStartNode) { 45 | // Find the rule for the currentConnectionStartNodeType once before the loop 46 | const outputValueType = getOutputValueType(currentConnectionStartNode.type as NodeType, currentConnectionStartNode.id) 47 | if (outputValueType) { 48 | inputRules.forEach(rule => { 49 | newMap.set(rule.handleId, rule.allowedValueTypes.includes(outputValueType) && currentConnectionStartNode.id !== id); 50 | }); 51 | } 52 | } 53 | 54 | return newMap; 55 | }, [currentConnectionStartNode]); 56 | 57 | return
60 | { getInputRules(nodeType, id).map(rule => { 61 | return 62 | 66 | 67 | }) } 68 | { getOutputValueType(nodeType, id) && ( 69 | 70 | 71 | 72 | ) } 73 | { content(id, selected, data) } 74 |
75 | } 76 | } 77 | 78 | export function createDynamicNodeComponent( 79 | nodeType: NodeType, 80 | shapeStyle: (isSelected: boolean) => CSSProperties, 81 | content: (id: string, selected: boolean, data: DataType) => React.ReactNode 82 | ) { 83 | 84 | return function({ id, selected, data }: NodeProps) { 85 | 86 | const updateNodeInternals = useUpdateNodeInternals() 87 | 88 | useEffect(() => { 89 | if (data.connectionRule) { 90 | addOrUpdateDynamicRule(nodeType, id, data.connectionRule) 91 | updateNodeInternals(id) 92 | } 93 | }, [data.connectionRule]); 94 | 95 | const currentConnectionStartNode = useReactFlowStore(state => state.currentConnectionStartNode) 96 | const handleHighlightedMap = useMemo(() => { 97 | let newMap = new Map() 98 | 99 | if (data.connectionRule?.inputRules && currentConnectionStartNode) { 100 | // Find the rule for the currentConnectionStartNodeType once before the loop 101 | const outputValueType = getOutputValueType(currentConnectionStartNode.type as NodeType, currentConnectionStartNode.id) 102 | if (outputValueType) { 103 | data.connectionRule.inputRules.forEach(rule => { 104 | newMap.set(rule.handleId, rule.allowedValueTypes.includes(outputValueType) && currentConnectionStartNode.id !== id); 105 | }); 106 | } 107 | } 108 | 109 | return newMap; 110 | }, [currentConnectionStartNode]); 111 | 112 | return
116 | { data.connectionRule?.inputRules.map((rule, index) => { 117 | return 118 | 124 | 125 | }) } 126 | { data.connectionRule?.outputValueType && ( 127 | 128 | 0 ? 131 | Math.max(shapeStyle(selected).height as number || 0, shapeStyle(selected).minHeight as number || 0) + ((data.connectionRule.inputRules.length - 1) * 30) / 2 132 | : undefined, 133 | zIndex: 1 134 | }} type="source" position={Position.Right}/> 135 | 136 | ) } 137 | { content(id, selected, data) } 138 |
147 | { data.connectionRule?.inputRules.map((rule) => { 148 | return
149 | 150 | {rule.handleId} 151 | 152 |
153 | }) } 154 |
155 |
156 | } 157 | } 158 | 159 | export function createOptionsComponent( 160 | optionsTitle: string, 161 | Options: React.ComponentType<{ id: string, data: DataType, onDataUpdated: (attributeName: keyof DataType, newValue: any) => void }> 162 | ) { 163 | return function(props: { id: string }) { 164 | const updateNodeData = useReactFlowStore((state) => state.updateNodeData) 165 | const getNodeById = useReactFlowStore((state) => state.getNodeById) 166 | 167 | const pipelines = usePlayStore(state => state.pipelines) 168 | const [cachedOutput, setCachedOutput] = useState(null) 169 | 170 | useEffect(() => { 171 | const ownOutgoingPipeline = pipelines.find(pipeline => pipeline.from === props.id) 172 | if (ownOutgoingPipeline && ownOutgoingPipeline.value) { 173 | setCachedOutput(ownOutgoingPipeline.value.map(output => JSON.stringify(output.value))) 174 | } else { 175 | setCachedOutput(null) 176 | } 177 | }, [pipelines]) 178 | 179 | const [localNode, setLocalNode] = useState(null) 180 | 181 | useEffect(() => { 182 | const currentNode = getNodeById(props.id); 183 | setLocalNode(currentNode); 184 | }, [props.id, getNodeById]) 185 | 186 | useEffect(() => { 187 | if (localNode !== null) { 188 | updateNodeData(props.id, localNode.data as DataType) 189 | 190 | // Change node data inside of node in nodemap when process is running 191 | // Allows to edit node options while the process runs 192 | if (usePlayStore.getState().isProcessRunning) { 193 | const newNode = usePlayStore.getState().nodeMap.get(props.id) 194 | if (newNode) { 195 | newNode.node.data = localNode.data as DataType 196 | usePlayStore.getState().nodeMap.set(props.id, newNode) 197 | } 198 | } 199 | } 200 | }, [localNode, updateNodeData]) 201 | 202 | return localNode !== null && 203 | { 207 | if(typeof attributeName === "string") { 208 | setNodeWithUpdatedDataValue(setLocalNode, attributeName, newValue) 209 | } 210 | }} 211 | /> 212 | { cachedOutput !== null && 213 | console.log("Change")} 217 | /> 218 | } 219 | 220 | } 221 | } 222 | 223 | -------------------------------------------------------------------------------- /src/components/editor/pages/canvas/nodes/util/OptionsUtil.ts: -------------------------------------------------------------------------------- 1 | import {Node} from "reactflow"; 2 | import React from "react"; 3 | 4 | export function setNodeWithUpdatedDataValue( 5 | setNode: React.Dispatch>, 6 | key: string, 7 | value: string 8 | ): void { 9 | setNode((oldNode) => { 10 | if (oldNode === null) { 11 | return null 12 | } 13 | return { 14 | ...oldNode, 15 | data: { 16 | ...oldNode.data, 17 | [key]: value 18 | } 19 | } 20 | }) 21 | } 22 | -------------------------------------------------------------------------------- /src/components/editor/pages/canvas/toolbars/Html.css: -------------------------------------------------------------------------------- 1 | .highlighted { 2 | background-color: yellow; 3 | } 4 | -------------------------------------------------------------------------------- /src/components/editor/pages/canvas/toolbars/NodesToolbar.tsx: -------------------------------------------------------------------------------- 1 | 'use client' 2 | 3 | import {Tooltip} from "@mui/material"; 4 | import useReactFlowStore from "@/stores/editor/ReactFlowStore"; 5 | import {useEffect, useState} from "react"; 6 | import {NodeType} from "@/config/NodeType"; 7 | import {getAllNodesMetadata, NodeMetadata} from "@/config/NodesMetadata"; 8 | 9 | export default function NodesToolbar() { 10 | 11 | const onDragStart = (event: any, nodeType: String, nodeData: any) => { 12 | event.dataTransfer.setData('application/reactflow', JSON.stringify({ nodeType, nodeData })); 13 | event.dataTransfer.effectAllowed = 'move'; 14 | }; 15 | 16 | const nodes = useReactFlowStore(state => state.nodes) 17 | const [isStartAlreadyPlaced, setIsStartAlreadyPlaced] = useState(false) 18 | 19 | useEffect(() => { 20 | setIsStartAlreadyPlaced(nodes.filter(node => node.type === NodeType.START_NODE).length !== 0) 21 | }, [nodes]) 22 | 23 | return ( 24 | 66 | ); 67 | }; -------------------------------------------------------------------------------- /src/components/editor/pages/canvas/toolbars/OnCanvasNodesSelector.tsx: -------------------------------------------------------------------------------- 1 | 'use Client' 2 | 3 | import * as React from 'react'; 4 | import DialogTitle from '@mui/material/DialogTitle'; 5 | import Dialog from '@mui/material/Dialog'; 6 | import {Tooltip, Typography} from "@mui/material"; 7 | import {NodeType} from "@/config/NodeType"; 8 | import {Node} from "reactflow"; 9 | import {getInputRules, getOutputValueType, isDynamicNodeType} from "@/config/ConnectionRules"; 10 | import {getAllNodesMetadata} from "@/config/NodesMetadata"; 11 | 12 | export interface OnCanvasNodesToolbarProps { 13 | open: boolean; 14 | position: {x: number, y: number} 15 | onClose: (nodeType: NodeType | null) => void; 16 | sourceNode: Node | null 17 | } 18 | 19 | export default function OnCanvasNodesToolbar(props: OnCanvasNodesToolbarProps) { 20 | const { onClose, open, position } = props; 21 | // Change width and height when adding new elements to toolbar 22 | const width = 160 23 | const height = 500 24 | 25 | const [windowDimensions, setWindowDimensions] = React.useState<{ width: number, height: number }>({ 26 | width: typeof window !== "undefined" ? window.innerWidth : 900, // Fall back to 900 if window is not defined 27 | height: typeof window !== "undefined" ? window.innerHeight : 600, // Fall back to 600 if window is not defined 28 | }); 29 | 30 | React.useEffect(() => { 31 | function handleResize() { 32 | setWindowDimensions({ width: window.innerWidth, height: window.innerHeight }); 33 | } 34 | 35 | window.addEventListener('resize', handleResize); 36 | return () => window.removeEventListener('resize', handleResize); 37 | }, []); 38 | 39 | const handleClose = () => { 40 | onClose(null) 41 | } 42 | 43 | const handleNodeSelected = (nodeType: NodeType) => { 44 | onClose(nodeType); 45 | }; 46 | 47 | return ( 48 | 58 | 59 | Select Node 60 | 61 |
73 | { getAllNodesMetadata().filter(nodeInfo => { 74 | // TODO Hier überlegen, wie ich das mit dynamischen Nodes handhabe. Will ichd ie gar nciht in dem Menü anzeigen wie jetzt? Oder Standardwerte? 75 | if (nodeInfo.type === NodeType.START_NODE || isDynamicNodeType(nodeInfo.type)) { 76 | return false 77 | } 78 | if (props.sourceNode === null) { 79 | return true 80 | } 81 | 82 | const sourceOutputValue = getOutputValueType(props.sourceNode.type as NodeType, props.sourceNode.id) 83 | // The nodeInfo nodeType is not a dynamic one (see guards), therefore the "id" is unnecessary and "" is given as the id 84 | const currentNodeInputRules = getInputRules(nodeInfo.type, "") 85 | 86 | // TODO Nochmal weiter überlegen, wie ich das handhabe, sobald ich einen node mit mehr als einem Eingang hab 87 | if ((currentNodeInputRules?.length || 0) > 1 || !sourceOutputValue) { 88 | return false 89 | } 90 | 91 | return (currentNodeInputRules?.[0]?.allowedValueTypes ?? []).includes(sourceOutputValue); 92 | }).map(info => ( 93 | 94 |
105 | handleNodeSelected(info.type) 106 | } > 107 | { info.icon } 108 |
109 | 110 | )) } 111 |
112 | 113 | ); 114 | } -------------------------------------------------------------------------------- /src/components/editor/pages/canvas/toolbars/OptionsToolbar.tsx: -------------------------------------------------------------------------------- 1 | import React, {useEffect, useMemo, useState} from "react"; 2 | import {useReactFlowStore} from "@/stores/editor/ReactFlowStore"; 3 | import {Node,} from 'reactflow'; 4 | import {usePlayStore} from "@/stores/editor/PlayStore"; 5 | import Log from "@/components/editor/pages/output/Log"; 6 | import OptionsContainer from "@/components/form/OptionsContainer"; 7 | import {nodesMetadataMap} from "@/config/NodesMetadata"; 8 | import {BottomNavigation, BottomNavigationAction, Typography} from "@mui/material"; 9 | import TuneIcon from '@mui/icons-material/Tune'; 10 | import DescriptionIcon from '@mui/icons-material/Description'; 11 | import {NodeType} from "@/config/NodeType"; 12 | import {selectedColor, toolbarBackgroundColor} from "@/config/colors"; 13 | 14 | export default function OptionsToolbar() { 15 | 16 | const selectedNodes = useReactFlowStore((state) => state.selectedNodes) 17 | const [options, setOptions] = useState(<>) 18 | const [currentNode, setCurrentNode] = useState(null) 19 | const isProcessRunning = usePlayStore(state => state.isProcessRunning) 20 | 21 | const [selectedPage, setSelectedPage] = useState("options") 22 | const toolbarPages = useMemo(() => { 23 | return new Map([ 24 | ["options", { 25 | icon: , 26 | tooltip: "Options" 27 | }], 28 | ["log", { 29 | icon: , 30 | tooltip: "Log" 31 | }] 32 | ]) 33 | }, [options]) 34 | 35 | useEffect(() => { 36 | if (isProcessRunning) { 37 | setSelectedPage("log") 38 | } 39 | }, [isProcessRunning]) 40 | 41 | useEffect(() => { 42 | if (selectedNodes.length === 1) { 43 | setCurrentNode(selectedNodes[0]) 44 | } else { 45 | setCurrentNode(null) 46 | } 47 | }, [selectedNodes]) 48 | 49 | useEffect(() => { 50 | if (!currentNode) { 51 | setOptions(<>) 52 | return 53 | } 54 | 55 | const nodeInfo = nodesMetadataMap[currentNode.type as NodeType] 56 | 57 | if (nodeInfo) { 58 | setOptions(nodeInfo.getOptionsComponent(currentNode.id)) 59 | } else { 60 | setOptions(<>) 61 | } 62 | }, [currentNode]) 63 | 64 | return ( 65 |
71 | { isProcessRunning &&
83 | 84 | ! Crawler is running ! 85 | 86 | { 96 | setSelectedPage(newSelectedPage) 97 | }} 98 | > 99 | { Array.from(toolbarPages, ([value, page]) => { 100 | const {icon, tooltip} = page; 101 | return 106 | }) } 107 | 108 |
} 109 | { selectedPage === "options" && options } 110 | { selectedPage === "log" && { setSelectedPage("options") } }> 111 | 112 | } 113 |
114 | ) 115 | } -------------------------------------------------------------------------------- /src/components/editor/pages/html/CssSelectorCard.tsx: -------------------------------------------------------------------------------- 1 | import Typography from "@mui/material/Typography"; 2 | import {IconButton, InputLabel, MenuItem, Select, SelectChangeEvent, Tooltip} from "@mui/material"; 3 | import MoveUpIcon from "@mui/icons-material/MoveUp"; 4 | import React, {useEffect, useState} from "react"; 5 | import DeleteForeverIcon from '@mui/icons-material/DeleteForever'; 6 | import {toolbarBackgroundColor} from "@/config/colors"; 7 | 8 | export interface CssSelectorCardProps { 9 | selector: string 10 | index: number, 11 | htmlString: string, 12 | allSelectors: string[], 13 | setAllSelectors: (newSelectors: string[]) => void 14 | isAttributeSelectionVisible: boolean 15 | } 16 | 17 | export default function CssSelectorCard(props: CssSelectorCardProps) { 18 | 19 | const [attributes, setAttributes] = useState([]) 20 | const [selectedAttribute, setSelectedAttribute] = React.useState(''); 21 | 22 | const handleAttributeSelected = (event: SelectChangeEvent) => { 23 | setSelectedAttribute(event.target.value as string); 24 | }; 25 | 26 | useEffect(() => { 27 | 28 | // Create an invisible iFrame 29 | const iframe = document.createElement('iframe') 30 | 31 | iframe.style.display = 'none' 32 | document.body.appendChild(iframe) 33 | 34 | // Put the given html in the iFrame 35 | // @ts-ignore 36 | iframe.contentDocument.open() 37 | // @ts-ignore 38 | iframe.contentDocument.write(props.htmlString) 39 | // @ts-ignore 40 | iframe.contentDocument.close() 41 | 42 | // Search for element in iFrame 43 | // @ts-ignore 44 | let element = iframe.contentDocument.querySelector(props.selector) 45 | 46 | let attributeList = [] 47 | 48 | if (element) { 49 | // @ts-ignore 50 | for (let attr of element.attributes) { 51 | attributeList.push(attr.name) 52 | } 53 | } 54 | 55 | setAttributes(attributeList) 56 | 57 | document.body.removeChild(iframe) 58 | }, [props.selector]) 59 | 60 | return
67 |
74 | 75 | { props.selector } 76 | 77 |
81 | 82 | { 83 | props.allSelectors[props.index] = props.selector.substring(0, props.selector.lastIndexOf(">")) 84 | if (props.allSelectors[props.index] === "") { 85 | props.allSelectors.splice(props.index, 1) 86 | } 87 | props.setAllSelectors([...props.allSelectors]) 88 | }}> 89 | 90 | 91 | 92 | 93 | { 94 | props.allSelectors.splice(props.index, 1) 95 | props.setAllSelectors([...props.allSelectors]) 96 | }}> 97 | 98 | 99 | 100 |
101 |
102 | { props.isAttributeSelectionVisible && 103 | Available attributes to extract 104 | } 105 | { props.isAttributeSelectionVisible && } 118 |
119 | } -------------------------------------------------------------------------------- /src/components/editor/pages/html/HtmlSelectorPage.tsx: -------------------------------------------------------------------------------- 1 | import {SelectableHtmlPreview} from "@/components/editor/pages/html/SelectableHtmlPreview"; 2 | import {IconButton, TextField, ToggleButton, ToggleButtonGroup, Tooltip} from "@mui/material"; 3 | import React, {useState} from "react"; 4 | import {openSuccessSnackBar, openWarningSnackBar} from "@/stores/editor/EditorPageStore"; 5 | import CloudDownloadIcon from '@mui/icons-material/CloudDownload'; 6 | import Typography from "@mui/material/Typography"; 7 | import AddIcon from '@mui/icons-material/Add'; 8 | import DoDisturbIcon from '@mui/icons-material/DoDisturb'; 9 | import ContentCopyIcon from '@mui/icons-material/ContentCopy'; 10 | import useHtmlSelectorStore from "@/stores/editor/HtmlSelectorStore"; 11 | import CssSelectorCard from "@/components/editor/pages/html/CssSelectorCard"; 12 | import DeleteForeverIcon from '@mui/icons-material/DeleteForever'; 13 | 14 | export enum SelectorSelectionModes { 15 | ADD_TO_SELECTION = "Add to selection", 16 | EXCLUDE_FROM_SELECTION = "Exclude from selection", 17 | } 18 | 19 | export default function HtmlSelectorPage() { 20 | 21 | const { url, html, amountOfSelectedElements, setUrl, setHtml, cssSelector, resetSelector, selectedSelector, excludedSelector, setSelectedSelector, setExcludedSelector } = useHtmlSelectorStore() 22 | const [selectionMode, setSelectionMode] = useState(SelectorSelectionModes.ADD_TO_SELECTION) 23 | 24 | const fetchWebsite = async (url: string) => { 25 | await fetch('/api/fetch', { 26 | method: 'POST', 27 | headers: { 28 | 'Content-Type': 'application/json' 29 | }, 30 | body: JSON.stringify({ url: url }) 31 | }) 32 | .then(response => { 33 | if (!response.ok) { 34 | throw new Error(`Response code was: ${response.status}`); 35 | } 36 | return response.text(); 37 | }) 38 | .then(data => { 39 | setHtml(data) 40 | }) 41 | .catch(error => { 42 | console.log(error) 43 | openWarningSnackBar("Website couldn't be loaded") 44 | }); 45 | } 46 | 47 | return
56 |
63 |
68 | 71 |
72 |
80 | { 86 | setUrl(event.target.value) 87 | }} 88 | InputProps={{ 89 | endAdornment: ( 90 | fetchWebsite(url)}> 91 | 92 | 93 | ) 94 | }} 95 | /> 96 |
100 | { 104 | if (newSelectionMode !== null) { 105 | setSelectionMode(newSelectionMode as SelectorSelectionModes) 106 | } 107 | }} 108 | aria-label="selection mode" 109 | > 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | { 123 | resetSelector() 124 | }}> 125 | 126 | 127 | 128 | 129 | { 130 | navigator.clipboard.writeText(cssSelector).then(() => { 131 | openSuccessSnackBar("CSS-Selector copied to clipboard") 132 | }) 133 | }}> 134 | 135 | 136 | 137 |
138 | 139 | Total elements selected on this page: { amountOfSelectedElements } 140 | 141 | 142 | Selected 143 | 144 | { selectedSelector.map((selector, index) => { 145 | return 153 | }) } 154 | 155 | Excluded 156 | 157 | { excludedSelector.map((selector, index) => { 158 | return 166 | }) } 167 |
168 |
169 |
170 | } -------------------------------------------------------------------------------- /src/components/editor/pages/html/SelectableHtmlPreview.tsx: -------------------------------------------------------------------------------- 1 | import React, {useEffect, useRef} from "react"; 2 | import "../canvas/toolbars/Html.css" 3 | import {SelectorSelectionModes} from "@/components/editor/pages/html/HtmlSelectorPage"; 4 | import useHtmlSelectorStore from "@/stores/editor/HtmlSelectorStore"; 5 | import {getUniqueSelector, isValidSelector, removeIDFromSelector} from "@/components/editor/pages/html/util/HtmlUtils"; 6 | 7 | const highlightCss = ".selectedForSelector { background-color: lightgreen; } .excluded { background-color: tomato; }"; 8 | 9 | export interface SelectableHtmlPreviewProps { 10 | selectionMode: SelectorSelectionModes 11 | } 12 | 13 | export function SelectableHtmlPreview(props: SelectableHtmlPreviewProps) { 14 | const { 15 | html, 16 | setAmountOfSelectedElements, 17 | setCssSelector, 18 | selectedSelector, 19 | excludedSelector, 20 | setSelectedSelector, 21 | setExcludedSelector 22 | } = useHtmlSelectorStore() 23 | 24 | const selectedSelectorRef = useRef(selectedSelector); 25 | const excludedSelectorRef = useRef(excludedSelector) 26 | 27 | const selectionModeRef = useRef(props.selectionMode); 28 | const iframeRef = useRef(null) 29 | 30 | useEffect(() => { 31 | selectionModeRef.current = props.selectionMode; 32 | }, [props.selectionMode]); 33 | 34 | useEffect(() => { 35 | // Access the document within the iframe 36 | // @ts-ignore 37 | const iframeDoc = iframeRef.current.contentDocument || iframeRef.current.contentWindow.document; 38 | 39 | // Set the HTML content inside the iframe 40 | iframeDoc.body.innerHTML = html; 41 | 42 | // Add the CSS rules for highlighting 43 | const styleTag = iframeDoc.createElement("style"); 44 | styleTag.innerHTML = highlightCss; 45 | iframeDoc.head.appendChild(styleTag); 46 | 47 | // Add the event listener 48 | iframeDoc.body.addEventListener('click', handleElementClick); 49 | 50 | // Remove the event listener when unmounting 51 | return () => { 52 | iframeDoc.body.removeEventListener('click', handleElementClick); 53 | }; 54 | }, [html]); 55 | 56 | useEffect(() => { 57 | 58 | selectedSelectorRef.current = selectedSelector; 59 | excludedSelectorRef.current = excludedSelector; 60 | 61 | if (excludedSelector.length > 0) { 62 | setCssSelector(selectedSelector.join() + ":not(" + excludedSelector.join() + ")") 63 | } else { 64 | setCssSelector(selectedSelector.join()) 65 | } 66 | 67 | // Get document from iFrame 68 | // @ts-ignore 69 | const iframeDoc = iframeRef.current.contentDocument || iframeRef.current.contentWindow.document; 70 | 71 | iframeDoc.querySelectorAll('.selectedForSelector').forEach((el: { classList: { remove: (arg0: string) => any; }; }) => el.classList.remove('selectedForSelector')); 72 | iframeDoc.querySelectorAll('.excluded').forEach((el: { classList: { remove: (arg0: string) => any; }; }) => el.classList.remove('excluded')); 73 | 74 | if (isValidSelector(iframeDoc, selectedSelector.join())) { 75 | iframeDoc.querySelectorAll(selectedSelector.join()).forEach((el: { classList: { add: (arg0: string) => any; }; }) => el.classList.add('selectedForSelector')); 76 | } 77 | 78 | if (isValidSelector(iframeDoc, excludedSelector.join())) { 79 | iframeDoc.querySelectorAll(excludedSelector.join()).forEach((el: { classList: { add: (arg0: string) => any; }; }) => el.classList.add('excluded')); 80 | } 81 | 82 | setAmountOfSelectedElements(iframeDoc.querySelectorAll('.selectedForSelector').length) 83 | }, [selectedSelector, excludedSelector, html]); 84 | 85 | const handleElementClick = (event: { preventDefault: () => void; target: any; }) => { 86 | event.preventDefault(); 87 | const element = event.target; 88 | 89 | let selectorForClick = removeIDFromSelector(getUniqueSelector(element)); 90 | 91 | switch (selectionModeRef.current) { 92 | case SelectorSelectionModes.ADD_TO_SELECTION: 93 | setExcludedSelector(excludedSelectorRef.current.filter(selector => selector !== selectorForClick)) 94 | setSelectedSelector([...selectedSelectorRef.current.filter(selector => selector !== selectorForClick), selectorForClick]) 95 | break 96 | case SelectorSelectionModes.EXCLUDE_FROM_SELECTION: 97 | setSelectedSelector(selectedSelectorRef.current.filter(selector => selector !== selectorForClick)) 98 | setExcludedSelector([...excludedSelectorRef.current.filter(selector => selector !== selectorForClick), selectorForClick]) 99 | break 100 | } 101 | }; 102 | 103 | return