├── .babelrc ├── .gitignore ├── .nvmrc ├── LICENSE ├── README.md ├── app ├── app.html ├── bundle.css ├── index.html └── main_app.js ├── assets ├── cloneable_large.png └── cloneable_large.svg ├── build ├── icon.icns ├── icon.png ├── mac │ └── bin │ │ └── wget ├── start.js ├── webpack.app.config.js ├── webpack.base.config.js ├── webpack.e2e.config.js ├── webpack.unit.config.js └── win │ └── bin │ └── wget.exe ├── config ├── env_development.json ├── env_production.json └── env_test.json ├── e2e └── utils.js ├── index.html ├── main_app.js ├── package-lock.json ├── package.json ├── postcss.config.js ├── resources ├── icon.icns ├── icon.ico └── icons │ └── 512x512.png ├── src ├── App.jsx ├── binaries.js ├── css │ ├── additional-styles │ │ ├── flatpickr.css │ │ ├── range-slider.css │ │ ├── theme.css │ │ ├── toggle-switch.css │ │ └── utility-patterns.css │ ├── app.css │ ├── dist.css │ ├── style.css │ ├── tailwind.config.js │ └── vendor │ │ ├── accordion.css │ │ └── react-tabs.css ├── db.js ├── favicon.svg ├── get-platform.js ├── helpers │ └── window.js ├── main.js ├── main.jsx ├── menu │ ├── app_menu_template.js │ ├── dev_menu_template.js │ └── edit_menu_template.js ├── pages │ ├── Dashboard.jsx │ ├── Download.jsx │ ├── Downloads.jsx │ ├── Help.jsx │ ├── NewDownload.jsx │ └── Settings.jsx ├── partials │ ├── Header.jsx │ ├── SettingsTabs.jsx │ ├── Sidebar.jsx │ ├── actions │ │ ├── DateSelect.jsx │ │ ├── Datepicker.jsx │ │ └── FilterButton.jsx │ └── header │ │ └── Help.jsx ├── utils │ ├── GetStatusBadge.jsx │ ├── Transition.jsx │ └── Utils.js └── wget │ └── index.js └── vite.config.js /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | "@babel/preset-react", 4 | [ 5 | "@babel/env", 6 | { 7 | "targets": { 8 | "browsers": "last 2 Chrome versions", 9 | "node": "current" 10 | } 11 | } 12 | ] 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .DS_Store 3 | Thumbs.db 4 | *.log 5 | 6 | /dist 7 | /temp 8 | 9 | # ignore everything in 'app' folder what had been generated from 'src' folder 10 | /app/app.js 11 | /app/main.js 12 | /app/**/*.map 13 | 14 | example.com 15 | example.org 16 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | v16.13.1 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cloneable 2 | 3 | [![License](https://img.shields.io/github/license/CloneableApp/Cloneable?label=License&color=brightgreen&cacheSeconds=3600)](./LICENSE) 4 | 5 | > The all-in-one website downloader tool for offline browsing, archiving, backups, and more. 6 | 7 | ## Screenshots 8 | 9 | [![Screenshot](https://cloneable.app/images/dashboard.png)](https://cloneable.app) 10 | [![Screenshot](https://cloneable.app/images/downloads.png)](https://cloneable.app) 11 | [![Screenshot](https://cloneable.app/images/settings.png)](https://cloneable.app) 12 | 13 | ## Downloads 14 | 15 | Downloadable binaries are currently not available, but will be soon. For now, follow the steps in the Installation section. 16 | 17 | ### Requirements 18 | 19 | If you want to use your own version of `wget`, your system must have that installed and you must set that path in your settings. Binaries of `wget` is included with this software but they're not guaranteed to work properly on all systems. 20 | 21 | ## Installation 22 | 23 | To run Cloneable locally in development mode: 24 | 25 | ``` 26 | git clone git@github.com:CloneableApp/Cloneable.git 27 | cd Cloneable 28 | nvm install 29 | nvm use 30 | npm install 31 | npm start 32 | ``` 33 | 34 | To get a working binary created in `dist/`: 35 | 36 | ``` 37 | nvm install 38 | nvm use 39 | npm install 40 | npm postinstall # might be necessary 41 | npm run app:dist 42 | ``` 43 | 44 | ## Information 45 | 46 | Cloneable is a free and open source desktop application that can clone websites to your local computer automatically, with smart handling of links, images, files, stylesheets, and more, so sites load seamlessly for offline browsing. 47 | 48 | It is built with [Electron](https://www.electronjs.org/) and [React](https://reactjs.org). Check the [package.json](./package.json) for a list of all dependencies. 49 | 50 | Behind the scenes, Cloneable relies heavily on [wget](https://www.gnu.org/software/wget/). 51 | 52 | ## TODO 53 | 54 | - Add dark mode 55 | - Add internationalization 56 | - Add pagination to Clones page 57 | - Add different format options to supply cookies (instead of just Netscape cookies.txt format) 58 | - Add ability to stop and continue Clones, saving progress 59 | 60 | ## Getting Help 61 | 62 | Feel free to open an issue here, or email me at [cloneableapp@gmail.com](mailto:cloneableapp@gmail.com). 63 | 64 | ## ❤️ Donating 65 | 66 | If you would like to support development of this project, please consider donating. We currently prefer Ko-fi to accept donations. [Here's our Ko-fi page](https://ko-fi.com/cloneable). 67 | ## License 68 | 69 | Cloneable is released under the [GNU General Public License v3.0](./LICENSE). 70 | 71 | --- 72 | 73 | ## For further information check [cloneable.app](https://cloneable.app) 74 | -------------------------------------------------------------------------------- /app/app.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Cloneable 6 | 7 | 8 | 9 | 16 |
17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /app/index.html: -------------------------------------------------------------------------------- 1 | Webpack App -------------------------------------------------------------------------------- /assets/cloneable_large.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloneableApp/Cloneable/cb4c03801e8d36789ba0c72cec87225d023f68b0/assets/cloneable_large.png -------------------------------------------------------------------------------- /assets/cloneable_large.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /build/icon.icns: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloneableApp/Cloneable/cb4c03801e8d36789ba0c72cec87225d023f68b0/build/icon.icns -------------------------------------------------------------------------------- /build/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloneableApp/Cloneable/cb4c03801e8d36789ba0c72cec87225d023f68b0/build/icon.png -------------------------------------------------------------------------------- /build/mac/bin/wget: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloneableApp/Cloneable/cb4c03801e8d36789ba0c72cec87225d023f68b0/build/mac/bin/wget -------------------------------------------------------------------------------- /build/start.js: -------------------------------------------------------------------------------- 1 | const childProcess = require("child_process"); 2 | const readline = require("readline"); 3 | const electron = require("electron"); 4 | const webpack = require("webpack"); 5 | const config = require("./webpack.app.config"); 6 | 7 | const compiler = webpack(config({ development: true })); 8 | let electronStarted = false; 9 | 10 | const clearTerminal = () => { 11 | if (process.stdout.isTTY) { 12 | const blankLines = "\n".repeat(process.stdout.rows); 13 | console.log(blankLines); 14 | readline.cursorTo(process.stdout, 0, 0); 15 | readline.clearScreenDown(process.stdout); 16 | } 17 | }; 18 | 19 | const watching = compiler.watch({}, (err, stats) => { 20 | if (err != null) { 21 | console.log(err); 22 | } else if (!electronStarted) { 23 | electronStarted = true; 24 | childProcess 25 | .spawn(electron, ["."], { stdio: "inherit" }) 26 | .on("close", () => { 27 | watching.close(); 28 | }); 29 | } 30 | 31 | if (stats != null) { 32 | clearTerminal(); 33 | console.log(stats.toString({ colors: true })); 34 | } 35 | }); 36 | -------------------------------------------------------------------------------- /build/webpack.app.config.js: -------------------------------------------------------------------------------- 1 | const path = require("path"); 2 | const { merge } = require("webpack-merge"); 3 | const base = require("./webpack.base.config"); 4 | const HtmlWebpackPlugin = require('html-webpack-plugin') 5 | 6 | // Any directories you will be adding code/files into, need to be added to this array so webpack will pick them up 7 | const defaultInclude = path.resolve(__dirname, '..', 'src') 8 | console.log(defaultInclude); 9 | module.exports = env => { 10 | return merge(base(env), { 11 | module: { 12 | rules: [ 13 | { 14 | test: /\.jsx?$/, 15 | use: [{ loader: 'babel-loader' }], 16 | include: defaultInclude 17 | }, 18 | { 19 | test: /\.(jpe?g|png|gif)$/, 20 | use: [{ loader: 'file-loader?name=img/[name]__[hash:base64:5].[ext]' }], 21 | include: defaultInclude 22 | }, 23 | { 24 | test: /\.(eot|svg|ttf|woff|woff2)$/, 25 | use: [{ loader: 'file-loader?name=font/[name]__[hash:base64:5].[ext]' }], 26 | include: defaultInclude 27 | } 28 | ], 29 | }, 30 | entry: { 31 | main: "./src/main.js", 32 | main_app: "./src/main.jsx" 33 | }, 34 | 35 | plugins: [ 36 | new HtmlWebpackPlugin(), 37 | ], 38 | 39 | output: { 40 | filename: "[name].js", 41 | path: path.resolve(__dirname, "../app") 42 | }, 43 | 44 | resolve: { 45 | extensions: ["", ".js", ".jsx", ".es6"] 46 | } 47 | }); 48 | }; 49 | -------------------------------------------------------------------------------- /build/webpack.base.config.js: -------------------------------------------------------------------------------- 1 | const path = require("path"); 2 | const nodeExternals = require("webpack-node-externals"); 3 | 4 | const envName = (env) => { 5 | if (env.production) { 6 | return "production"; 7 | } 8 | if (env.test) { 9 | return "test"; 10 | } 11 | return "development"; 12 | }; 13 | 14 | const envToMode = (env) => { 15 | if (env.production) { 16 | process.env.NODE_ENV = "production"; 17 | return "production"; 18 | } 19 | process.env.NODE_ENV = "development"; 20 | return "development"; 21 | }; 22 | 23 | module.exports = env => { 24 | return { 25 | target: "electron-renderer", 26 | mode: envToMode(env), 27 | node: { 28 | __dirname: false, 29 | __filename: false 30 | }, 31 | externals: [nodeExternals()], 32 | resolve: { 33 | alias: { 34 | env: path.resolve(__dirname, `../config/env_${envName(env)}.json`) 35 | } 36 | }, 37 | devtool: "source-map", 38 | module: { 39 | rules: [ 40 | { 41 | test: /\.js$/, 42 | exclude: /node_modules/, 43 | use: ["babel-loader"] 44 | }, 45 | { 46 | test: /\.css$/, 47 | use: ["style-loader", "css-loader"] 48 | } 49 | ] 50 | }, 51 | }; 52 | }; 53 | -------------------------------------------------------------------------------- /build/webpack.e2e.config.js: -------------------------------------------------------------------------------- 1 | const { merge } = require("webpack-merge"); 2 | const jetpack = require("fs-jetpack"); 3 | const base = require("./webpack.base.config"); 4 | 5 | // Test files are scattered through the whole project. Here we're searching 6 | // for them and generating entry file for webpack. 7 | 8 | const e2eDir = jetpack.cwd("e2e"); 9 | const tempDir = jetpack.cwd("temp"); 10 | const entryFilePath = tempDir.path("e2e_entry.js"); 11 | 12 | const entryFileContent = e2eDir 13 | .find({ matching: "*.e2e.js" }) 14 | .reduce((fileContent, path) => { 15 | const normalizedPath = path.replace(/\\/g, "/"); 16 | return `${fileContent}import "../e2e/${normalizedPath}";\n`; 17 | }, ""); 18 | 19 | jetpack.write(entryFilePath, entryFileContent); 20 | 21 | module.exports = env => { 22 | return merge(base(env), { 23 | entry: entryFilePath, 24 | output: { 25 | filename: "e2e.js", 26 | path: tempDir.path() 27 | } 28 | }); 29 | }; 30 | -------------------------------------------------------------------------------- /build/webpack.unit.config.js: -------------------------------------------------------------------------------- 1 | const { merge } = require("webpack-merge"); 2 | const jetpack = require("fs-jetpack"); 3 | const base = require("./webpack.base.config"); 4 | 5 | // Test files are scattered through the whole project. Here we're searching 6 | // for them and generating entry file for webpack. 7 | 8 | const srcDir = jetpack.cwd("src"); 9 | const tempDir = jetpack.cwd("temp"); 10 | const entryFilePath = tempDir.path("specs_entry.js"); 11 | 12 | const entryFileContent = srcDir 13 | .find({ matching: "*.spec.js" }) 14 | .reduce((fileContent, path) => { 15 | const normalizedPath = path.replace(/\\/g, "/"); 16 | return `${fileContent}import "../src/${normalizedPath}";\n`; 17 | }, ""); 18 | 19 | jetpack.write(entryFilePath, entryFileContent); 20 | 21 | module.exports = env => { 22 | return merge(base(env), { 23 | entry: entryFilePath, 24 | output: { 25 | filename: "specs.js", 26 | path: tempDir.path() 27 | } 28 | }); 29 | }; 30 | -------------------------------------------------------------------------------- /build/win/bin/wget.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloneableApp/Cloneable/cb4c03801e8d36789ba0c72cec87225d023f68b0/build/win/bin/wget.exe -------------------------------------------------------------------------------- /config/env_development.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "development", 3 | "description": "Add here any environment specific stuff you like." 4 | } 5 | -------------------------------------------------------------------------------- /config/env_production.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "production", 3 | "description": "Add here any environment specific stuff you like." 4 | } 5 | -------------------------------------------------------------------------------- /config/env_test.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test", 3 | "description": "Add here any environment specific stuff you like." 4 | } 5 | -------------------------------------------------------------------------------- /e2e/utils.js: -------------------------------------------------------------------------------- 1 | import electron from "electron"; 2 | import { Application } from "spectron"; 3 | 4 | const beforeEach = function() { 5 | this.timeout(10000); 6 | 7 | this.app = new Application({ 8 | path: electron, 9 | args: ["."], 10 | chromeDriverArgs: ["remote-debugging-port=9222"] 11 | }); 12 | return this.app.start(); 13 | }; 14 | 15 | const afterEach = function() { 16 | if (this.app && this.app.isRunning()) { 17 | return this.app.stop(); 18 | } 19 | return undefined; 20 | }; 21 | 22 | export default { 23 | beforeEach, 24 | afterEach 25 | } 26 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Cloneable 6 | 7 | 8 | 9 | 16 |
17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /main_app.js: -------------------------------------------------------------------------------- 1 | app/main_app.js -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cloneable", 3 | "productName": "Cloneable", 4 | "description": "The all-in-one website downloader tool for offline browsing, archiving, and backups.", 5 | "version": "0.1.4", 6 | "private": true, 7 | "author": "Cloneable ", 8 | "copyright": "© Cloneable", 9 | "homepage": "https://cloneable.app", 10 | "main": "app/main.js", 11 | "build": { 12 | "appId": "com.cloneable.app", 13 | "mac": { 14 | "identity": null, 15 | "category": "public.app-category.utilities", 16 | "icon": "./assets/cloneable_large.png" 17 | }, 18 | "win": { 19 | "target": "portable", 20 | "icon": "./assets/cloneable_large.png", 21 | "files": [ 22 | "**/*", 23 | "!html/measurements/*" 24 | ] 25 | }, 26 | "files": [ 27 | "app/**/*", 28 | "node_modules/**/*", 29 | "package.json" 30 | ], 31 | "directories": { 32 | "buildResources": "resources" 33 | }, 34 | "publish": [ 35 | { 36 | "private": true, 37 | "provider": "github", 38 | "owner": "CloneableApp", 39 | "repo": "Cloneable" 40 | } 41 | ], 42 | "extraResources": [ 43 | { 44 | "from": "build/mac/bin", 45 | "to": "bin", 46 | "filter": [ 47 | "**/*" 48 | ] 49 | }, 50 | { 51 | "from": "build/win/bin", 52 | "to": "bin", 53 | "filter": [ 54 | "**/*" 55 | ] 56 | } 57 | ] 58 | }, 59 | "scripts": { 60 | "app:dir": "webpack --config=build/webpack.app.config.js --env=production && electron-builder --dir", 61 | "app:dist": "webpack --config=build/webpack.app.config.js --env=production && electron-builder", 62 | "postinstall": "electron-builder install-app-deps", 63 | "preunit": "webpack --config=build/webpack.unit.config.js --env=test", 64 | "unit": "electron-mocha temp/specs.js --renderer --color --require source-map-support/register", 65 | "pree2e": "webpack --config=build/webpack.app.config.js --env=test && webpack --config=build/webpack.e2e.config.js --env=test", 66 | "e2e": "mocha temp/e2e.js --require source-map-support/register", 67 | "test": "npm run unit && npm run e2e", 68 | "start": "vite build && node build/start.js", 69 | "build": "electron-builder --mac --windows", 70 | "release-mac": "npm test && webpack --config=build/webpack.app.config.js --env=production && electron-builder --mac --publish always", 71 | "release-win": "npm test && webpack --config=build/webpack.app.config.js --env=production && electron-builder --windows --publish always" 72 | }, 73 | "dependencies": { 74 | "@babel/preset-react": "^7.18.6", 75 | "@rjsf/bootstrap-4": "^4.2.2", 76 | "@rjsf/core": "^4.2.2", 77 | "@tailwindcss/forms": "^0.5.2", 78 | "better-sqlite3": "^7.6.2", 79 | "chart.js": "^3.8.0", 80 | "chartjs-adapter-moment": "^1.0.0", 81 | "child_process": "^1.0.2", 82 | "electron-root-path": "^1.0.16", 83 | "electron-store": "^8.0.2", 84 | "electron-unhandled": "^4.0.1", 85 | "electron-updater": "^5.0.5", 86 | "file-loader": "^6.2.0", 87 | "fs": "^0.0.1-security", 88 | "fs-jetpack": "^4.1.0", 89 | "html-webpack-plugin": "^5.5.0", 90 | "mini-css-extract-plugin": "^2.6.1", 91 | "moment": "^2.29.4", 92 | "node-dir": "^0.1.17", 93 | "node-html-parser": "^5.3.3", 94 | "parse-url": "^6.0.5", 95 | "postcss-loader": "^7.0.1", 96 | "react": "^18.2.0", 97 | "react-accessible-accordion": "^5.0.0", 98 | "react-bootstrap": "^1.6.5", 99 | "react-dom": "^18.2.0", 100 | "react-flatpickr": "^3.10.13", 101 | "react-router-dom": "^6.3.0", 102 | "react-tabs": "^5.1.0", 103 | "react-toastify": "^9.0.7", 104 | "react-transition-group": "^4.4.2", 105 | "sanitize-filename": "^1.6.3", 106 | "tailwindcss": "^3.1.7", 107 | "tree-kill": "^1.2.2", 108 | "uuid": "^8.3.2" 109 | }, 110 | "devDependencies": { 111 | "@babel/core": "^7.5.5", 112 | "@babel/preset-env": "^7.5.5", 113 | "@vitejs/plugin-react": "^2.0.0", 114 | "autoprefixer": "^10.4.7", 115 | "babel-loader": "^8.2.2", 116 | "chai": "^4.1.0", 117 | "cross-env": "^7.0.3", 118 | "css-loader": "^5.2.7", 119 | "electron": "^13.0.1", 120 | "electron-builder": "^22.5.1", 121 | "electron-mocha": "^10.0.0", 122 | "mocha": "^8.3.2", 123 | "postcss": "^8.4.14", 124 | "source-map-support": "^0.5.6", 125 | "spectron": "^15.0.0", 126 | "style-loader": "^2.0.0", 127 | "vite": "^3.0.2", 128 | "webpack": "^5.30.0", 129 | "webpack-cli": "^4.6.0", 130 | "webpack-merge": "^5.7.3", 131 | "webpack-node-externals": "^3.0.0" 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /postcss.config.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloneableApp/Cloneable/cb4c03801e8d36789ba0c72cec87225d023f68b0/postcss.config.js -------------------------------------------------------------------------------- /resources/icon.icns: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloneableApp/Cloneable/cb4c03801e8d36789ba0c72cec87225d023f68b0/resources/icon.icns -------------------------------------------------------------------------------- /resources/icon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloneableApp/Cloneable/cb4c03801e8d36789ba0c72cec87225d023f68b0/resources/icon.ico -------------------------------------------------------------------------------- /resources/icons/512x512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CloneableApp/Cloneable/cb4c03801e8d36789ba0c72cec87225d023f68b0/resources/icons/512x512.png -------------------------------------------------------------------------------- /src/App.jsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect } from 'react'; 2 | import { 3 | Routes, 4 | Route, 5 | useLocation, 6 | } from 'react-router-dom'; 7 | 8 | import './css/style.css'; 9 | 10 | // Import pages 11 | import Dashboard from './pages/Dashboard'; 12 | import Downloads from './pages/Downloads'; 13 | import Download from './pages/Download'; 14 | import Settings from './pages/Settings'; 15 | import NewDownload from './pages/NewDownload'; 16 | import Help from './pages/Help'; 17 | 18 | const unhandled = require('electron-unhandled'); 19 | unhandled(); 20 | 21 | function App() { 22 | const location = useLocation(); 23 | 24 | useEffect(() => { 25 | document.querySelector('html').style.scrollBehavior = 'auto' 26 | window.scroll({ top: 0 }) 27 | document.querySelector('html').style.scrollBehavior = '' 28 | }, [location.pathname]); // triggered on route change 29 | 30 | return ( 31 |
32 | 33 | } /> 34 | } /> 35 | } /> 36 | } /> 37 | } /> 38 | } /> 39 | } /> 40 | } /> 41 | 42 |
43 | ); 44 | } 45 | 46 | export default App; 47 | -------------------------------------------------------------------------------- /src/binaries.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import path from 'path'; 3 | import { remote, app } from 'electron'; 4 | import { rootPath } from 'electron-root-path'; 5 | import getPlatform from './get-platform'; 6 | const IS_PROD = process.env.NODE_ENV === 'production'; 7 | const root = rootPath; 8 | const isPackaged = 9 | process.mainModule.filename.indexOf('app.asar') !== -1; 10 | const binariesPath = 11 | IS_PROD && isPackaged 12 | ? path.join(path.dirname(app.getAppPath()), '..', './Resources', './bin') 13 | : path.join(root, './build', getPlatform(), './bin'); 14 | export const wgetExecPath = path.resolve(path.join(binariesPath, getPlatform() === 'win' ? './wget.exe' : './wget')); -------------------------------------------------------------------------------- /src/css/additional-styles/flatpickr.css: -------------------------------------------------------------------------------- 1 | @import 'flatpickr/dist/flatpickr.min.css'; 2 | 3 | /* Customise flatpickr */ 4 | * { 5 | --calendarPadding: 24px; 6 | --daySize: 36px; 7 | --daysWidth: calc(var(--daySize)*7); 8 | } 9 | 10 | @keyframes fpFadeInDown { 11 | from { 12 | opacity: 0; 13 | transform: translate3d(0, -8px, 0); 14 | } 15 | to { 16 | opacity: 1; 17 | transform: translate3d(0, 0, 0); 18 | } 19 | } 20 | 21 | .flatpickr-calendar { 22 | border: inherit; 23 | @apply rounded shadow-lg border border-slate-200 left-1/2; 24 | margin-left: calc(calc(var(--daysWidth) + calc(var(--calendarPadding)*2))*0.5*-1); 25 | padding: var(--calendarPadding); 26 | width: calc(var(--daysWidth) + calc(var(--calendarPadding)*2)); 27 | } 28 | 29 | @screen lg { 30 | .flatpickr-calendar { 31 | @apply left-0 right-auto; 32 | margin-left: 0; 33 | } 34 | } 35 | 36 | .flatpickr-right.flatpickr-calendar { 37 | @apply right-0 left-auto; 38 | margin-left: 0; 39 | } 40 | 41 | .flatpickr-calendar.animate.open { 42 | animation: fpFadeInDown 200ms ease-out; 43 | } 44 | 45 | .flatpickr-calendar.static { 46 | position: absolute; 47 | top: calc(100% + 4px); 48 | } 49 | 50 | .flatpickr-calendar.static.open { 51 | z-index: 20; 52 | } 53 | 54 | .flatpickr-days { 55 | width: var(--daysWidth); 56 | } 57 | 58 | .dayContainer { 59 | width: var(--daysWidth); 60 | min-width: var(--daysWidth); 61 | max-width: var(--daysWidth); 62 | } 63 | 64 | .flatpickr-day { 65 | @apply bg-slate-50 text-sm font-medium text-slate-600; 66 | max-width: var(--daySize); 67 | height: var(--daySize); 68 | line-height: var(--daySize); 69 | } 70 | 71 | .flatpickr-day, 72 | .flatpickr-day.prevMonthDay, 73 | .flatpickr-day.nextMonthDay { 74 | border: none; 75 | } 76 | 77 | .flatpickr-day, 78 | .flatpickr-day.prevMonthDay, 79 | .flatpickr-day.nextMonthDay, 80 | .flatpickr-day.selected.startRange, 81 | .flatpickr-day.startRange.startRange, 82 | .flatpickr-day.endRange.startRange, 83 | .flatpickr-day.selected.endRange, 84 | .flatpickr-day.startRange.endRange, 85 | .flatpickr-day.endRange.endRange, 86 | .flatpickr-day.selected.startRange.endRange, 87 | .flatpickr-day.startRange.startRange.endRange, 88 | .flatpickr-day.endRange.startRange.endRange { 89 | border-radius: 0; 90 | } 91 | 92 | .flatpickr-day.flatpickr-disabled, 93 | .flatpickr-day.flatpickr-disabled:hover, 94 | .flatpickr-day.prevMonthDay, 95 | .flatpickr-day.nextMonthDay, 96 | .flatpickr-day.notAllowed, 97 | .flatpickr-day.notAllowed.prevMonthDay, 98 | .flatpickr-day.notAllowed.nextMonthDay { 99 | @apply text-slate-400; 100 | } 101 | 102 | .rangeMode .flatpickr-day { 103 | margin: 0; 104 | } 105 | 106 | .flatpickr-day.selected, 107 | .flatpickr-day.startRange, 108 | .flatpickr-day.endRange, 109 | .flatpickr-day.selected.inRange, 110 | .flatpickr-day.startRange.inRange, 111 | .flatpickr-day.endRange.inRange, 112 | .flatpickr-day.selected:focus, 113 | .flatpickr-day.startRange:focus, 114 | .flatpickr-day.endRange:focus, 115 | .flatpickr-day.selected:hover, 116 | .flatpickr-day.startRange:hover, 117 | .flatpickr-day.endRange:hover, 118 | .flatpickr-day.selected.prevMonthDay, 119 | .flatpickr-day.startRange.prevMonthDay, 120 | .flatpickr-day.endRange.prevMonthDay, 121 | .flatpickr-day.selected.nextMonthDay, 122 | .flatpickr-day.startRange.nextMonthDay, 123 | .flatpickr-day.endRange.nextMonthDay { 124 | @apply bg-indigo-500 text-indigo-50; 125 | } 126 | 127 | .flatpickr-day.inRange, 128 | .flatpickr-day.prevMonthDay.inRange, 129 | .flatpickr-day.nextMonthDay.inRange, 130 | .flatpickr-day.today.inRange, 131 | .flatpickr-day.prevMonthDay.today.inRange, 132 | .flatpickr-day.nextMonthDay.today.inRange, 133 | .flatpickr-day:hover, 134 | .flatpickr-day.prevMonthDay:hover, 135 | .flatpickr-day.nextMonthDay:hover, 136 | .flatpickr-day:focus, 137 | .flatpickr-day.prevMonthDay:focus, 138 | .flatpickr-day.nextMonthDay:focus, 139 | .flatpickr-day.today:hover, 140 | .flatpickr-day.today:focus { 141 | @apply bg-indigo-400 text-indigo-50; 142 | } 143 | 144 | .flatpickr-day.inRange, 145 | .flatpickr-day.selected.startRange + .endRange:not(:nth-child(7n+1)), 146 | .flatpickr-day.startRange.startRange + .endRange:not(:nth-child(7n+1)), 147 | .flatpickr-day.endRange.startRange + .endRange:not(:nth-child(7n+1)) { 148 | box-shadow: none; 149 | } 150 | 151 | .flatpickr-months { 152 | align-items: center; 153 | margin-top: -8px; 154 | margin-bottom: 6px; 155 | } 156 | 157 | .flatpickr-months .flatpickr-prev-month, 158 | .flatpickr-months .flatpickr-next-month { 159 | position: static; 160 | height: auto; 161 | @apply text-slate-600; 162 | } 163 | 164 | .flatpickr-months .flatpickr-prev-month svg, 165 | .flatpickr-months .flatpickr-next-month svg { 166 | width: 7px; 167 | height: 11px; 168 | } 169 | 170 | .flatpickr-months .flatpickr-prev-month:hover, 171 | .flatpickr-months .flatpickr-next-month:hover, 172 | .flatpickr-months .flatpickr-prev-month:hover svg, 173 | .flatpickr-months .flatpickr-next-month:hover svg { 174 | fill: inherit; 175 | @apply text-slate-400; 176 | } 177 | 178 | .flatpickr-months .flatpickr-prev-month { 179 | margin-left: -10px; 180 | } 181 | 182 | .flatpickr-months .flatpickr-next-month { 183 | margin-right: -10px; 184 | } 185 | 186 | .flatpickr-months .flatpickr-month { 187 | @apply text-slate-800; 188 | height: auto; 189 | line-height: inherit; 190 | } 191 | 192 | .flatpickr-current-month { 193 | @apply text-sm font-medium; 194 | position: static; 195 | height: auto; 196 | width: auto; 197 | left: auto; 198 | padding: 0; 199 | } 200 | 201 | .flatpickr-current-month span.cur-month { 202 | @apply font-medium m-0; 203 | } 204 | 205 | .flatpickr-current-month span.cur-month:hover { 206 | background: none; 207 | } 208 | 209 | .flatpickr-current-month input.cur-year { 210 | font-weight: inherit; 211 | box-shadow: none !important; 212 | } 213 | 214 | .numInputWrapper:hover { 215 | background: none; 216 | } 217 | 218 | .numInputWrapper span { 219 | display: none; 220 | } 221 | 222 | span.flatpickr-weekday { 223 | @apply text-slate-400 font-medium text-xs; 224 | } 225 | 226 | .flatpickr-calendar.arrowTop::before, 227 | .flatpickr-calendar.arrowTop::after, 228 | .flatpickr-calendar.arrowBottom::before, 229 | .flatpickr-calendar.arrowBottom::after { 230 | display: none; 231 | } -------------------------------------------------------------------------------- /src/css/additional-styles/range-slider.css: -------------------------------------------------------------------------------- 1 | /* Range slider */ 2 | :root { 3 | --range-thumb-size: 36px; 4 | } 5 | 6 | input[type=range] { 7 | appearance: none; 8 | background: #ccc; 9 | border-radius: 3px; 10 | height: 6px; 11 | margin-top: (--range-thumb-size - 6px) * 0.5; 12 | margin-bottom: (--range-thumb-size - 6px) * 0.5; 13 | --thumb-size: #{--range-thumb-size}; 14 | } 15 | 16 | input[type=range]::-webkit-slider-thumb { 17 | appearance: none; 18 | -webkit-appearance: none; 19 | background-color: #000; 20 | background-image: url("data:image/svg+xml,%3Csvg width='12' height='8' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath d='M8 .5v7L12 4zM0 4l4 3.5v-7z' fill='%23FFF' fill-rule='nonzero'/%3E%3C/svg%3E"); 21 | background-position: center; 22 | background-repeat: no-repeat; 23 | border: 0; 24 | border-radius: 50%; 25 | cursor: pointer; 26 | height: --range-thumb-size; 27 | width: --range-thumb-size; 28 | } 29 | 30 | input[type=range]::-moz-range-thumb { 31 | background-color: #000; 32 | background-image: url("data:image/svg+xml,%3Csvg width='12' height='8' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath d='M8 .5v7L12 4zM0 4l4 3.5v-7z' fill='%23FFF' fill-rule='nonzero'/%3E%3C/svg%3E"); 33 | background-position: center; 34 | background-repeat: no-repeat; 35 | border: 0; 36 | border: none; 37 | border-radius: 50%; 38 | cursor: pointer; 39 | height: --range-thumb-size; 40 | width: --range-thumb-size; 41 | } 42 | 43 | input[type=range]::-ms-thumb { 44 | background-color: #000; 45 | background-image: url("data:image/svg+xml,%3Csvg width='12' height='8' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath d='M8 .5v7L12 4zM0 4l4 3.5v-7z' fill='%23FFF' fill-rule='nonzero'/%3E%3C/svg%3E"); 46 | background-position: center; 47 | background-repeat: no-repeat; 48 | border: 0; 49 | border-radius: 50%; 50 | cursor: pointer; 51 | height: --range-thumb-size; 52 | width: --range-thumb-size; 53 | } 54 | 55 | input[type=range]::-moz-focus-outer { 56 | border: 0; 57 | } -------------------------------------------------------------------------------- /src/css/additional-styles/theme.css: -------------------------------------------------------------------------------- 1 | .form-input:focus, 2 | .form-textarea:focus, 3 | .form-multiselect:focus, 4 | .form-select:focus, 5 | .form-checkbox:focus, 6 | .form-radio:focus { 7 | @apply ring-0; 8 | } 9 | -------------------------------------------------------------------------------- /src/css/additional-styles/toggle-switch.css: -------------------------------------------------------------------------------- 1 | /* Switch element */ 2 | .form-switch { 3 | @apply relative select-none; 4 | width: 44px; 5 | } 6 | 7 | .form-switch label { 8 | @apply block overflow-hidden cursor-pointer h-6 rounded-full; 9 | } 10 | 11 | .form-switch label > span:first-child { 12 | @apply absolute block rounded-full; 13 | width: 20px; 14 | height: 20px; 15 | top: 2px; 16 | left: 2px; 17 | right: 50%; 18 | transition: all .15s ease-out; 19 | } 20 | 21 | .form-switch input[type="checkbox"]:checked + label { 22 | @apply bg-indigo-500; 23 | } 24 | 25 | .form-switch input[type="checkbox"]:checked + label > span:first-child { 26 | left: 22px; 27 | } 28 | 29 | .form-switch input[type="checkbox"]:disabled + label { 30 | @apply cursor-not-allowed bg-slate-100 border border-slate-200; 31 | } 32 | 33 | .form-switch input[type="checkbox"]:disabled + label > span:first-child { 34 | @apply bg-slate-400; 35 | } -------------------------------------------------------------------------------- /src/css/additional-styles/utility-patterns.css: -------------------------------------------------------------------------------- 1 | /* Typography */ 2 | .h1 { 3 | @apply text-4xl font-extrabold tracking-tighter; 4 | } 5 | 6 | .h2 { 7 | @apply text-3xl font-extrabold tracking-tighter; 8 | } 9 | 10 | .h3 { 11 | @apply text-3xl font-extrabold; 12 | } 13 | 14 | .h4 { 15 | @apply text-2xl font-extrabold tracking-tight; 16 | } 17 | 18 | @screen md { 19 | .h1 { 20 | @apply text-5xl; 21 | } 22 | 23 | .h2 { 24 | @apply text-4xl; 25 | } 26 | } 27 | 28 | /* Buttons */ 29 | .btn, 30 | .btn-lg, 31 | .btn-sm, 32 | .btn-xs { 33 | @apply font-medium text-sm inline-flex items-center justify-center border border-transparent rounded leading-5 shadow-sm transition duration-150 ease-in-out; 34 | } 35 | 36 | .btn { 37 | @apply px-3 py-2; 38 | } 39 | 40 | .btn-lg { 41 | @apply px-4 py-3; 42 | } 43 | 44 | .btn-sm { 45 | @apply px-2 py-1; 46 | } 47 | 48 | .btn-xs { 49 | @apply px-2 py-0.5; 50 | } 51 | 52 | /* Forms */ 53 | input[type="search"]::-webkit-search-decoration, 54 | input[type="search"]::-webkit-search-cancel-button, 55 | input[type="search"]::-webkit-search-results-button, 56 | input[type="search"]::-webkit-search-results-decoration { 57 | -webkit-appearance: none; 58 | } 59 | 60 | .form-input, 61 | .form-textarea, 62 | .form-multiselect, 63 | .form-select, 64 | .form-checkbox, 65 | .form-radio { 66 | @apply text-sm text-slate-800 bg-white border; 67 | } 68 | 69 | .form-input, 70 | .form-textarea, 71 | .form-multiselect, 72 | .form-select, 73 | .form-checkbox { 74 | @apply rounded; 75 | } 76 | 77 | .form-input, 78 | .form-textarea, 79 | .form-multiselect, 80 | .form-select { 81 | @apply leading-5 py-2 px-3 border-slate-200 hover:border-slate-300 focus:border-indigo-300 shadow-sm; 82 | } 83 | 84 | .form-input, 85 | .form-textarea { 86 | @apply placeholder-slate-400; 87 | } 88 | 89 | .form-select { 90 | @apply pr-10; 91 | } 92 | 93 | .form-checkbox, 94 | .form-radio { 95 | @apply text-indigo-500 border border-slate-300; 96 | } 97 | 98 | /* Chrome, Safari and Opera */ 99 | .no-scrollbar::-webkit-scrollbar { 100 | display: none; 101 | } 102 | 103 | .no-scrollbar { 104 | -ms-overflow-style: none; /* IE and Edge */ 105 | scrollbar-width: none; /* Firefox */ 106 | } -------------------------------------------------------------------------------- /src/css/app.css: -------------------------------------------------------------------------------- 1 | /* 2 | This is a mess, I know. My frontend-fu is not strong. 3 | */ 4 | 5 | @keyframes fadein { 6 | from { 7 | opacity: 0; 8 | } 9 | to { 10 | opacity: 1; 11 | } 12 | } 13 | 14 | .app { 15 | /* Disable text selection, or your app will feel like a web page */ 16 | 17 | /* Cover the whole window */ 18 | height: 100%; 19 | 20 | /* Smoother startup */ 21 | animation: fadein 0.5s; 22 | } 23 | 24 | .highlight { 25 | color: rgb(108 43 217); 26 | } 27 | .mt-4 { 28 | margin-top: 1rem; 29 | } 30 | 31 | .float-right { 32 | float: right; 33 | } 34 | 35 | .next-step-button { 36 | float: right; 37 | clear: both; 38 | margin-top: -30px; 39 | margin-right: 10px; 40 | } 41 | 42 | .previous-step-button { 43 | float: left; 44 | clear: both; 45 | margin-top: -30px; 46 | margin-left: 10px; 47 | } 48 | 49 | .pb-100px { 50 | padding-bottom: 100px; 51 | } 52 | 53 | .pb-30px { 54 | padding-bottom: 30px; 55 | } 56 | 57 | .status-badge { 58 | display: inline; 59 | } 60 | 61 | button[disabled] { 62 | opacity: 0.5; 63 | cursor: not-allowed; 64 | } 65 | 66 | #root_domains, #root_headers { 67 | min-width: 500px; 68 | } 69 | 70 | #root_acceptFilters, #root_rejectFilters, #root_includeDirs, #root_excludeDirs { 71 | min-width: 500px; 72 | } 73 | 74 | .bg-purple-100 { 75 | background-color: rgb(217, 211, 227); 76 | } 77 | .bg-purple-700 { 78 | background-color: rgb(108 43 217); 79 | } 80 | 81 | a { 82 | color: rgb(108 43 217); 83 | } 84 | a:hover, a:focus { 85 | color: rgb(85 33 181); 86 | } 87 | button a, .button a { 88 | color: white; 89 | } 90 | button:hover, .button:hover { 91 | background-color: rgb(85 33 181); 92 | } 93 | button a:hover, .button a:focus { 94 | color: white; 95 | } 96 | 97 | a strong { 98 | color: rgb(71 85 105); 99 | } 100 | 101 | a strong:hover, a strong:focus { 102 | color: rgb(85 33 181); 103 | } 104 | 105 | .bg-purple-800 { 106 | background-color: rgb(85 33 181); 107 | } 108 | 109 | .float-left-important { 110 | float: left !important; 111 | } 112 | 113 | .btn.btn-info { 114 | background-color: rgb(108 43 217); 115 | color: white; 116 | } 117 | 118 | .rjsf button[type="submit"] { 119 | margin-top: 20px; 120 | visibility: hidden; 121 | } 122 | 123 | .rjsf { 124 | padding-top: 30px; 125 | padding-left: 30px; 126 | padding-right: 30px; 127 | border-radius: 20px; 128 | border-top-left-radius: 0px; 129 | border-top-right-radius: 0px; 130 | } 131 | 132 | label { 133 | font-weight: bold; 134 | } 135 | 136 | .form-group.field { 137 | margin-bottom: 20px; 138 | } 139 | 140 | 141 | .form-label { 142 | margin-right: 15px; 143 | font-weight: bold;; 144 | } 145 | 146 | .form-check-label { 147 | font-weight: bold; 148 | margin-left: 10px; 149 | } 150 | 151 | .form-group .row { 152 | margin-bottom: 30px !important; 153 | } 154 | 155 | [type=integer] { 156 | -webkit-appearance: none; 157 | -moz-appearance: none; 158 | appearance: none; 159 | background-color: #fff; 160 | border-color: #6b7280; 161 | border-width: 1px; 162 | border-radius: 0; 163 | padding: .5rem .75rem; 164 | font-size: 1rem; 165 | line-height: 1.5rem; 166 | --tw-shadow: 0 0 #0000; 167 | } 168 | 169 | h5 { 170 | font-weight: bold;; 171 | } 172 | 173 | .field-description { 174 | margin-bottom: 10px; 175 | } 176 | 177 | .new-folder { 178 | margin-top: -60px; 179 | margin-left: -60px; 180 | position: absolute; 181 | } 182 | 183 | .checkbox { 184 | display: flex; 185 | flex-direction: column-reverse; 186 | } 187 | 188 | .checkbox span { 189 | font-weight: bold; 190 | margin-left: 10px; 191 | } 192 | 193 | .download-info { 194 | margin-top: 10px; 195 | margin-bottom: 10px; 196 | } 197 | 198 | .clone-logs { 199 | max-height: 200px; 200 | overflow-y: scroll; 201 | display: flex; 202 | flex-direction: column-reverse; 203 | } 204 | 205 | #root_basePath, #root_cookiesFilePath { 206 | width: 60%; 207 | display: block; 208 | margin: auto; 209 | } 210 | 211 | .clone-path { 212 | font-weight: bold; 213 | font-family: monospace; 214 | font-size: 1rem; 215 | display: block; 216 | } 217 | 218 | .pb-5 { 219 | padding-bottom: 1.25rem; 220 | } 221 | 222 | .dashboard-card { 223 | max-height: 450px; 224 | overflow-y: scroll; 225 | } 226 | 227 | .status-filters { 228 | margin-bottom: 30px; 229 | } 230 | .status-filter { 231 | cursor: pointer; 232 | display: inline; 233 | margin-right: 10px; 234 | opacity: 0.5; 235 | } 236 | .status-filter.active { 237 | opacity: 1.0 238 | } 239 | 240 | .react-tabs__tab-list { 241 | margin-bottom: 0px; 242 | } -------------------------------------------------------------------------------- /src/css/style.css: -------------------------------------------------------------------------------- 1 | @import url('https://fonts.googleapis.com/css2?family=Roboto:wght@400;500;600;700&display=fallback'); 2 | 3 | @import 'dist'; 4 | 5 | @import 'tailwindcss/base'; 6 | @import 'tailwindcss/components'; 7 | 8 | @import 'additional-styles/utility-patterns.css'; 9 | @import 'additional-styles/range-slider.css'; 10 | @import 'additional-styles/toggle-switch.css'; 11 | @import 'additional-styles/flatpickr.css'; 12 | @import 'additional-styles/theme.css'; 13 | 14 | @import 'vendor/react-tabs.css'; 15 | @import 'vendor/accordion.css'; 16 | 17 | @import 'tailwindcss/utilities'; 18 | 19 | @import 'app'; -------------------------------------------------------------------------------- /src/css/tailwind.config.js: -------------------------------------------------------------------------------- 1 | const plugin = require('tailwindcss/plugin'); 2 | 3 | module.exports = { 4 | content: [ 5 | './index.html', 6 | './app/app.html', 7 | './src/**/*.{html,js,jsx,ts,tsx}', 8 | ], 9 | theme: { 10 | extend: { 11 | boxShadow: { 12 | DEFAULT: '0 1px 3px 0 rgba(0, 0, 0, 0.08), 0 1px 2px 0 rgba(0, 0, 0, 0.02)', 13 | md: '0 4px 6px -1px rgba(0, 0, 0, 0.08), 0 2px 4px -1px rgba(0, 0, 0, 0.02)', 14 | lg: '0 10px 15px -3px rgba(0, 0, 0, 0.08), 0 4px 6px -2px rgba(0, 0, 0, 0.01)', 15 | xl: '0 20px 25px -5px rgba(0, 0, 0, 0.08), 0 10px 10px -5px rgba(0, 0, 0, 0.01)', 16 | }, 17 | outline: { 18 | blue: '2px solid rgba(0, 112, 244, 0.5)', 19 | }, 20 | fontFamily: { 21 | inter: ['Inter', 'sans-serif'], 22 | }, 23 | fontSize: { 24 | xs: ['0.75rem', { lineHeight: '1.5' }], 25 | sm: ['0.875rem', { lineHeight: '1.5715' }], 26 | base: ['1rem', { lineHeight: '1.5', letterSpacing: '-0.01em' }], 27 | lg: ['1.125rem', { lineHeight: '1.5', letterSpacing: '-0.01em' }], 28 | xl: ['1.25rem', { lineHeight: '1.5', letterSpacing: '-0.01em' }], 29 | '2xl': ['1.5rem', { lineHeight: '1.33', letterSpacing: '-0.01em' }], 30 | '3xl': ['1.88rem', { lineHeight: '1.33', letterSpacing: '-0.01em' }], 31 | '4xl': ['2.25rem', { lineHeight: '1.25', letterSpacing: '-0.02em' }], 32 | '5xl': ['3rem', { lineHeight: '1.25', letterSpacing: '-0.02em' }], 33 | '6xl': ['3.75rem', { lineHeight: '1.2', letterSpacing: '-0.02em' }], 34 | }, 35 | screens: { 36 | xs: '480px', 37 | }, 38 | borderWidth: { 39 | 3: '3px', 40 | }, 41 | minWidth: { 42 | 36: '9rem', 43 | 44: '11rem', 44 | 56: '14rem', 45 | 60: '15rem', 46 | 72: '18rem', 47 | 80: '20rem', 48 | }, 49 | maxWidth: { 50 | '8xl': '88rem', 51 | '9xl': '96rem', 52 | }, 53 | zIndex: { 54 | 60: '60', 55 | }, 56 | }, 57 | }, 58 | plugins: [ 59 | // eslint-disable-next-line global-require 60 | require('@tailwindcss/forms'), 61 | // add custom variant for expanding sidebar 62 | plugin(({ addVariant, e }) => { 63 | addVariant('sidebar-expanded', ({ modifySelectors, separator }) => { 64 | modifySelectors(({ className }) => `.sidebar-expanded .${e(`sidebar-expanded${separator}${className}`)}`); 65 | }); 66 | }), 67 | ], 68 | }; 69 | -------------------------------------------------------------------------------- /src/css/vendor/accordion.css: -------------------------------------------------------------------------------- 1 | /** 2 | * ---------------------------------------------- 3 | * Demo styles 4 | * ---------------------------------------------- 5 | **/ 6 | .accordion { 7 | border: 1px solid rgba(0, 0, 0, 0.1); 8 | border-radius: 2px; 9 | } 10 | 11 | .accordion__item + .accordion__item { 12 | border-top: 1px solid rgba(0, 0, 0, 0.1); 13 | } 14 | 15 | .accordion__button { 16 | background-color: white; 17 | color: rgb(30 41 59); 18 | cursor: pointer; 19 | padding: 18px; 20 | width: 100%; 21 | text-align: left; 22 | border: none; 23 | } 24 | 25 | .accordion__button:hover { 26 | background-color: #f7fafc; 27 | } 28 | 29 | .accordion__button:before { 30 | display: inline-block; 31 | content: ''; 32 | height: 10px; 33 | width: 10px; 34 | margin-right: 12px; 35 | border-bottom: 2px solid currentColor; 36 | border-right: 2px solid currentColor; 37 | transform: rotate(-45deg); 38 | } 39 | 40 | .accordion__button[aria-expanded='true']::before, 41 | .accordion__button[aria-selected='true']::before { 42 | transform: rotate(45deg); 43 | } 44 | 45 | [hidden] { 46 | display: none; 47 | } 48 | 49 | .accordion__panel { 50 | padding: 20px; 51 | animation: fadein 0.35s ease-in; 52 | background: white; 53 | } 54 | 55 | /* -------------------------------------------------- */ 56 | /* ---------------- Animation part ------------------ */ 57 | /* -------------------------------------------------- */ 58 | 59 | @keyframes fadein { 60 | 0% { 61 | opacity: 0; 62 | } 63 | 64 | 100% { 65 | opacity: 1; 66 | } 67 | } -------------------------------------------------------------------------------- /src/css/vendor/react-tabs.css: -------------------------------------------------------------------------------- 1 | .react-tabs { 2 | -webkit-tap-highlight-color: transparent; 3 | } 4 | 5 | .react-tabs__tab-list { 6 | border-bottom: 1px solid #aaa; 7 | margin: 0 0 10px; 8 | padding: 0; 9 | } 10 | 11 | .react-tabs__tab { 12 | display: inline-block; 13 | border: 1px solid transparent; 14 | border-bottom: none; 15 | bottom: -1px; 16 | position: relative; 17 | list-style: none; 18 | padding: 6px 12px; 19 | cursor: pointer; 20 | } 21 | 22 | .react-tabs__tab--selected { 23 | background: #fff; 24 | border-color: #aaa; 25 | color: black; 26 | border-radius: 5px 5px 0 0; 27 | } 28 | 29 | .react-tabs__tab--disabled { 30 | color: GrayText; 31 | cursor: default; 32 | } 33 | 34 | .react-tabs__tab:focus { 35 | outline: none; 36 | } 37 | 38 | .react-tabs__tab:focus:after { 39 | content: ''; 40 | position: absolute; 41 | height: 5px; 42 | left: -4px; 43 | right: -4px; 44 | bottom: -5px; 45 | background: #fff; 46 | } 47 | 48 | .react-tabs__tab-panel { 49 | display: none; 50 | } 51 | 52 | .react-tabs__tab-panel--selected { 53 | display: block; 54 | } 55 | -------------------------------------------------------------------------------- /src/db.js: -------------------------------------------------------------------------------- 1 | import { app } from 'electron'; 2 | const path = require('path'); 3 | const db = require('better-sqlite3')(path.join(app.getPath('userData'), "database.db")); 4 | 5 | export const createTables = () => { 6 | try { 7 | db.prepare("CREATE TABLE settings (type TEXT, value TEXT, project_id TEXT DEFAULT NULL, UNIQUE(type, project_id))").run(); 8 | 9 | db.prepare("CREATE TABLE projects \ 10 | (id STRING PRIMARY KEY, \ 11 | name TEXT, \ 12 | url TEXT, \ 13 | status INTEGER, \ 14 | clone_data TEXT, \ 15 | created INTEGER, \ 16 | base_path TEXT, \ 17 | started INTEGER, \ 18 | completed INTEGER, \ 19 | error TEXT, \ 20 | no_directories INTEGER \ 21 | )" 22 | ).run(); 23 | } catch (err) { 24 | 25 | } 26 | } 27 | 28 | export const saveSettings = (settingsKey, settings, projectId) => { 29 | db.prepare( 30 | "INSERT OR IGNORE INTO settings (type, value, project_id) VALUES (?, ?, ?)" 31 | ).run(settingsKey, JSON.stringify(settings), projectId); 32 | db.prepare( 33 | "UPDATE settings SET value=(?) WHERE type=(?) AND project_id=(?)" 34 | ).run(JSON.stringify(settings), settingsKey, projectId); 35 | } 36 | 37 | export const getSettings = (type, projectId, callback) => { 38 | if (!projectId) { 39 | projectId = -1; 40 | } 41 | const row = db.prepare("SELECT * FROM settings WHERE type=(?) AND project_id=(?)").get(type, projectId); 42 | let val = {}; 43 | if (row) { 44 | val = JSON.parse(row.value); 45 | if (type === "auth") { 46 | val.cookiesFile = ""; 47 | } 48 | } 49 | return val; 50 | } 51 | 52 | export const saveNewProject = (id, data) => { 53 | db.prepare( 54 | "INSERT INTO projects (id, name, url, status, clone_data, created, base_path, no_directories) \ 55 | VALUES (?, ?, ?, ?, ?, ?, ?, ?)" 56 | ).run(id, data.name, data.url, 0, "", Date.now(), "", 0); 57 | } 58 | 59 | export const deleteProject = (id) => { 60 | db.prepare("DELETE FROM projects WHERE id=(?)").run(id); 61 | } 62 | 63 | export const saveProjectCloneData = (id, data) => { 64 | db.prepare( 65 | "UPDATE projects SET clone_data=(?) WHERE id=(?)", 66 | ).run(data, id); 67 | } 68 | 69 | export const saveProjectInfo = (id, data) => { 70 | db.prepare( 71 | "UPDATE projects SET name=(?), url=(?) WHERE id=(?)", 72 | ).run(data.name, data.url, id); 73 | } 74 | 75 | export const saveProjectStatus = (id, status) => { 76 | db.prepare( 77 | "UPDATE projects SET status=(?) WHERE id=(?)", 78 | ).run(status, id); 79 | } 80 | 81 | export const getProject = (id) => { 82 | return db.prepare("SELECT * FROM projects WHERE id=(?)").get(id); 83 | } 84 | 85 | export const getProjects = () => { 86 | return db.prepare("SELECT * FROM projects ORDER BY started DESC").all(); 87 | } 88 | 89 | export const getProjectStatusCount = (num) => { 90 | return db.prepare("SELECT count(*) AS count FROM projects WHERE status=(?)").get(num); 91 | } 92 | 93 | export const getRecentProjects = () => { 94 | return db.prepare("SELECT * FROM projects ORDER BY created DESC LIMIT 5").all(); 95 | } 96 | 97 | export const startProjectClone = (id, project) => { 98 | db.prepare( 99 | "UPDATE projects SET started=(?), status=(?), error=(?), \ 100 | clone_data=(?), base_path=(?), no_directories=(?) WHERE id=(?)", 101 | ).run(Date.now(), project.status, project.error, project.clone_data, project.base_path, project.no_directories, id); 102 | } 103 | 104 | export const cancelClone = (id) => { 105 | db.prepare( 106 | "UPDATE projects SET status=(?), completed=(?), error=(?) WHERE id=(?)", 107 | ).run(3, Date.now(), null, id); 108 | } 109 | 110 | export const setProjectError = (id, err) => { 111 | db.prepare( 112 | "UPDATE projects SET completed=(?), status=(?), error=(?) WHERE id=(?)", 113 | ).run(Date.now(), -1, err.toString(), id); 114 | } 115 | 116 | export const setProjectCompleted = (id) => { 117 | db.prepare( 118 | "UPDATE projects SET completed=(?), status=(?) WHERE id=(?)", 119 | ).run(Date.now(), 2, id); 120 | } -------------------------------------------------------------------------------- /src/favicon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /src/get-platform.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | import { platform } from 'os'; 3 | export default () => { 4 | switch (platform()) { 5 | case 'aix': 6 | case 'freebsd': 7 | case 'linux': 8 | case 'openbsd': 9 | case 'android': 10 | return 'linux'; 11 | case 'darwin': 12 | case 'sunos': 13 | return 'mac'; 14 | case 'win32': 15 | return 'win'; 16 | } 17 | }; -------------------------------------------------------------------------------- /src/helpers/window.js: -------------------------------------------------------------------------------- 1 | // This helper remembers the size and position of your windows, and restores 2 | // them in that place after app relaunch. 3 | // Can be used for more than one window, just construct many 4 | // instances of it and give each different name. 5 | 6 | import { app, BrowserWindow, screen } from "electron"; 7 | import jetpack from "fs-jetpack"; 8 | 9 | export default (name, options) => { 10 | const userDataDir = jetpack.cwd(app.getPath("userData")); 11 | const stateStoreFile = `window-state-${name}.json`; 12 | const defaultSize = { 13 | width: options.width, 14 | height: options.height 15 | }; 16 | let state = {}; 17 | let win; 18 | 19 | const restore = () => { 20 | let restoredState = {}; 21 | try { 22 | restoredState = userDataDir.read(stateStoreFile, "json"); 23 | } catch (err) { 24 | // For some reason json can't be read (might be corrupted). 25 | // No worries, we have defaults. 26 | } 27 | return Object.assign({}, defaultSize, restoredState); 28 | }; 29 | 30 | const getCurrentPosition = () => { 31 | const position = win.getPosition(); 32 | const size = win.getSize(); 33 | return { 34 | x: position[0], 35 | y: position[1], 36 | width: size[0], 37 | height: size[1] 38 | }; 39 | }; 40 | 41 | const windowWithinBounds = (windowState, bounds) => { 42 | return ( 43 | windowState.x >= bounds.x && 44 | windowState.y >= bounds.y && 45 | windowState.x + windowState.width <= bounds.x + bounds.width && 46 | windowState.y + windowState.height <= bounds.y + bounds.height 47 | ); 48 | }; 49 | 50 | const resetToDefaults = () => { 51 | const bounds = screen.getPrimaryDisplay().bounds; 52 | return Object.assign({}, defaultSize, { 53 | x: (bounds.width - defaultSize.width) / 2, 54 | y: (bounds.height - defaultSize.height) / 2 55 | }); 56 | }; 57 | 58 | const ensureVisibleOnSomeDisplay = windowState => { 59 | const visible = screen.getAllDisplays().some(display => { 60 | return windowWithinBounds(windowState, display.bounds); 61 | }); 62 | if (!visible) { 63 | // Window is partially or fully not visible now. 64 | // Reset it to safe defaults. 65 | return resetToDefaults(); 66 | } 67 | return windowState; 68 | }; 69 | 70 | const saveState = () => { 71 | if (!win.isMinimized() && !win.isMaximized()) { 72 | Object.assign(state, getCurrentPosition()); 73 | } 74 | userDataDir.write(stateStoreFile, state, { atomic: true }); 75 | }; 76 | 77 | state = ensureVisibleOnSomeDisplay(restore()); 78 | 79 | win = new BrowserWindow(Object.assign({}, options, state)); 80 | 81 | win.on("close", saveState); 82 | 83 | return win; 84 | }; 85 | -------------------------------------------------------------------------------- /src/main.js: -------------------------------------------------------------------------------- 1 | import path from 'path'; 2 | import url from 'url'; 3 | import { app, Menu, ipcMain, shell, dialog, powerSaveBlocker } from "electron"; 4 | import appMenuTemplate from "./menu/app_menu_template"; 5 | import editMenuTemplate from "./menu/edit_menu_template"; 6 | import devMenuTemplate from "./menu/dev_menu_template"; 7 | import createWindow from "./helpers/window"; 8 | import {autoUpdater} from "electron-updater"; 9 | import { v4 as uuidv4 } from 'uuid'; 10 | 11 | const db = require('better-sqlite3')(path.join(app.getPath('userData'), "database.db")); 12 | 13 | let powerSaveBlockerIds = {}; 14 | 15 | let mainWindow = null; 16 | 17 | const unhandled = require('electron-unhandled'); 18 | unhandled(); 19 | 20 | const Store = require('electron-store'); 21 | const store = new Store(); 22 | 23 | // Special module holding environment variables which you declared 24 | // in config/env_xxx.json file. 25 | import env from "env"; 26 | import { getBasePath } from "./utils/Utils"; 27 | import { existsSync } from 'fs'; 28 | import { cancelClone, createTables, deleteProject, getProject, getProjects, getProjectStatusCount, getRecentProjects, getSettings, saveNewProject, saveProjectCloneData, saveProjectInfo, saveProjectStatus, saveSettings, startProjectClone } from './db'; 29 | 30 | // Save userData in separate folders for each environment. 31 | // Thanks to this you can use production and development versions of the app 32 | // on same machine like those are two separate apps. 33 | if (env.name !== "production") { 34 | const userDataPath = app.getPath("userData"); 35 | app.setPath("userData", `${userDataPath} (${env.name})`); 36 | } 37 | 38 | const setApplicationMenu = () => { 39 | const menus = [appMenuTemplate, editMenuTemplate]; 40 | if (env.name !== "production") { 41 | menus.push(devMenuTemplate); 42 | } 43 | Menu.setApplicationMenu(Menu.buildFromTemplate(menus)); 44 | }; 45 | 46 | const settingsTypes = [ 47 | "storage", 48 | "crawl", 49 | "filters", 50 | "browser", 51 | "auth", 52 | "advanced", 53 | ]; 54 | 55 | // We can communicate with our window (the renderer process) via messages. 56 | const initIpc = () => { 57 | ipcMain.on("get-downloads-folder", (event) => { 58 | event.reply("downloads-folder", app.getPath("downloads")); 59 | }); 60 | 61 | ipcMain.on("save-settings-tab", (event, index) => { 62 | store.set("settings-tab", index); 63 | }); 64 | ipcMain.on("get-settings-tab", (event, index) => { 65 | event.reply("settings-tab", store.get("settings-tab")); 66 | }); 67 | 68 | ipcMain.on("save-settings", (event, settingsKey, settings, projectId) => { 69 | if (!projectId) { 70 | projectId = -1; 71 | } 72 | saveSettings(settingsKey, settings, projectId); 73 | }); 74 | ipcMain.on("get-settings", (event, type, projectId) => { 75 | let s = getSettings(type, projectId); 76 | event.reply("settings-" + type, s); 77 | }); 78 | 79 | ipcMain.on("save-new-project", (event, data) => { 80 | let id = uuidv4(); 81 | 82 | saveNewProject(id, data); 83 | 84 | for (let i = 0; i < settingsTypes.length; i++) { 85 | let s = getSettings(settingsTypes[i], null); 86 | saveSettings(settingsTypes[i], s, id); 87 | } 88 | 89 | event.reply("saved-new-project", id); 90 | }); 91 | 92 | ipcMain.on("delete-project", (event, id) => { 93 | deleteProject(id); 94 | event.reply("deleted-project", id); 95 | }); 96 | 97 | ipcMain.on("save-project-clone-data", (event, id, data) => { 98 | saveProjectCloneData(id, data); 99 | event.reply("saved-project-clone-data", id); 100 | }); 101 | 102 | ipcMain.on("save-project-info", (event, id, data) => { 103 | saveProjectInfo(id, data); 104 | event.reply("saved-project-info", id); 105 | }); 106 | 107 | ipcMain.on("save-project-status", (event, id, status) => { 108 | saveProjectStatus(id, status); 109 | event.reply("saved-project-status", id); 110 | }); 111 | 112 | ipcMain.on("get-project", (event, id) => { 113 | const row = getProject(id); 114 | event.reply("project", row); 115 | }); 116 | 117 | ipcMain.on("get-projects", (event) => { 118 | const rows = getProjects(); 119 | event.reply("projects", rows); 120 | }); 121 | 122 | ipcMain.on("get-projects-stats", (event) => { 123 | let statNums = [ 124 | -1, 0, 1, 2, 3 125 | ]; 126 | let stats = { 127 | "-1": 0, 128 | "0": 0, 129 | "1": 0, 130 | "2": 0, 131 | "3": 0, 132 | "total": 0, 133 | }; 134 | for (let i = 0; i < statNums.length; i++) { 135 | const row = getProjectStatusCount(statNums[i]); 136 | stats[statNums[i].toString()] = row.count; 137 | stats["total"] += row.count; 138 | } 139 | event.reply("projects-stats", stats); 140 | }); 141 | 142 | ipcMain.on("get-recent-projects", (event) => { 143 | const rows = getRecentProjects(); 144 | event.reply("recent-projects", rows); 145 | }); 146 | 147 | let timeElapsedIntervalIds = {}; 148 | let wgetPids = {}; 149 | 150 | ipcMain.on("start-clone", (event, id) => { 151 | powerSaveBlockerIds[id] = powerSaveBlocker.start('prevent-app-suspension'); 152 | 153 | let project = getProject(id); 154 | 155 | if (!project) { 156 | return; 157 | } 158 | 159 | project.started = new Date(); 160 | project.status = 1; 161 | project.error = null; 162 | project.clone_data = null; 163 | 164 | mainWindow.webContents.send("time-elapsed", project); 165 | timeElapsedIntervalIds[id] = setInterval(() => { 166 | mainWindow.webContents.send("time-elapsed", project); 167 | }, 1000); 168 | 169 | let options = {}; 170 | for (let i = 0; i < settingsTypes.length; i++) { 171 | let s = getSettings(settingsTypes[i], id) || {}; 172 | options = Object.assign(options, s); 173 | } 174 | 175 | project.base_path = getBasePath(project, options); 176 | project.no_directories = options.no_directories; 177 | 178 | const wget = require('./wget'); 179 | wgetPids[id] = wget(mainWindow, store, project, options, timeElapsedIntervalIds[id], powerSaveBlockerIds[id]); 180 | 181 | startProjectClone(id, project); 182 | 183 | event.reply("started-clone", project); 184 | }); 185 | 186 | ipcMain.on("open-folder", (event, folderPath) => { 187 | if (existsSync(folderPath)) { 188 | shell.openPath(folderPath); 189 | } 190 | else { 191 | dialog.showErrorBox('Not found', 'This folder was not found. Perhaps it was renamed, moved, or deleted.'); 192 | } 193 | }); 194 | 195 | ipcMain.on("cancel-clone", (event, id) => { 196 | if (timeElapsedIntervalIds.hasOwnProperty(id)) { 197 | clearInterval(timeElapsedIntervalIds[id]); 198 | } 199 | 200 | if (powerSaveBlockerIds.hasOwnProperty(id)) { 201 | powerSaveBlocker.stop(powerSaveBlockerIds[id]); 202 | } 203 | 204 | if (wgetPids.hasOwnProperty(id)) { 205 | var kill = require('tree-kill'); 206 | kill(wgetPids[id]); 207 | } 208 | 209 | cancelClone(id); 210 | 211 | const project = getProject(id); 212 | 213 | event.reply("canceled-clone", project); 214 | }); 215 | 216 | ipcMain.on("select-folder", (event) => { 217 | // If the platform is 'win32' or 'Linux' 218 | if (process.platform !== 'darwin') { 219 | // Resolves to a Promise 220 | dialog.showOpenDialog({ 221 | title: 'Select the Folder', 222 | defaultPath: app.getPath('downloads'), 223 | buttonLabel: 'Select', 224 | // Restricting the user to only Text Files. 225 | filters: [], 226 | // Specifying the File Selector Property 227 | properties: ['openDirectory'] 228 | }).then(file => { 229 | // Stating whether dialog operation was 230 | // cancelled or not. 231 | if (!file.canceled) { 232 | // Updating the GLOBAL filepath variable 233 | // to user-selected file. 234 | event.reply('selected-folder', file.filePaths[0].toString()); 235 | return; 236 | } 237 | }).catch(err => { 238 | console.log(err) 239 | }); 240 | } 241 | else { 242 | // If the platform is 'darwin' (macOS) 243 | dialog.showOpenDialog({ 244 | title: 'Select the Folder', 245 | defaultPath: app.getPath('downloads'), 246 | buttonLabel: 'Select', 247 | filters: [], 248 | // Specifying the File Selector and Directory 249 | // Selector Property In macOS 250 | properties: ['openDirectory'] 251 | }).then(file => { 252 | if (!file.canceled) { 253 | event.reply('selected-folder', file.filePaths[0].toString()); 254 | } 255 | }).catch(err => { 256 | console.log(err); 257 | }); 258 | } 259 | }); 260 | ipcMain.on("select-file", (event) => { 261 | // If the platform is 'win32' or 'Linux' 262 | if (process.platform !== 'darwin') { 263 | // Resolves to a Promise 264 | dialog.showOpenDialog({ 265 | title: 'Select the File', 266 | defaultPath: app.getPath('downloads'), 267 | buttonLabel: 'Select', 268 | // Restricting the user to only Text Files. 269 | filters: [], 270 | // Specifying the File Selector Property 271 | properties: ['openFile'] 272 | }).then(file => { 273 | // Stating whether dialog operation was 274 | // cancelled or not. 275 | if (!file.canceled) { 276 | // Updating the GLOBAL filepath variable 277 | // to user-selected file. 278 | event.reply('selected-file', file.filePaths[0].toString()); 279 | return; 280 | } 281 | }).catch(err => { 282 | console.log(err) 283 | }); 284 | } 285 | else { 286 | // If the platform is 'darwin' (macOS) 287 | dialog.showOpenDialog({ 288 | title: 'Select the Folder', 289 | defaultPath: app.getPath('downloads'), 290 | buttonLabel: 'Select', 291 | filters: [], 292 | // Specifying the File Selector and Directory 293 | // Selector Property In macOS 294 | properties: ['openDirectory', 'openFile'] 295 | }).then(file => { 296 | if (!file.canceled) { 297 | event.reply('selected-file', file.filePaths[0].toString()); 298 | } 299 | }).catch(err => { 300 | console.log(err); 301 | }); 302 | } 303 | }); 304 | }; 305 | 306 | app.on("ready", () => { 307 | //autoUpdater.checkForUpdatesAndNotify(); 308 | 309 | createTables(); 310 | 311 | setApplicationMenu(); 312 | initIpc(); 313 | 314 | mainWindow = createWindow("main", { 315 | width: 1000, 316 | height: 600, 317 | webPreferences: { 318 | // Two properties below are here for demo purposes, and are 319 | // security hazard. Make sure you know what you're doing 320 | // in your production app. 321 | nodeIntegration: true, 322 | contextIsolation: false, 323 | // Spectron needs access to remote module 324 | enableRemoteModule: env.name === "test" 325 | } 326 | }); 327 | 328 | mainWindow.setBackgroundColor('#f1f5f9'); 329 | 330 | mainWindow.maximize(); 331 | 332 | mainWindow.loadURL( 333 | url.format({ 334 | pathname: path.join(__dirname, "app.html"), 335 | protocol: "file:", 336 | slashes: true 337 | }) 338 | ); 339 | }); 340 | 341 | app.on("window-all-closed", () => { 342 | for (const id in powerSaveBlockerIds) { 343 | if (powerSaveBlockerIds.hasOwnProperty(id)) { 344 | powerSaveBlocker.stop(powerSaveBlockerIds[id]); 345 | } 346 | } 347 | db.close(); 348 | app.quit(); 349 | }); 350 | -------------------------------------------------------------------------------- /src/main.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom/client'; 3 | import { HashRouter as Router } from 'react-router-dom'; 4 | import App from './App'; 5 | 6 | const unhandled = require('electron-unhandled'); 7 | unhandled(); 8 | 9 | ReactDOM.createRoot(document.getElementById('root')).render( 10 | 11 | 12 | 13 | 14 | 15 | ); 16 | -------------------------------------------------------------------------------- /src/menu/app_menu_template.js: -------------------------------------------------------------------------------- 1 | import { app } from "electron"; 2 | 3 | export default { 4 | label: "App", 5 | submenu: [ 6 | { 7 | label: "Quit", 8 | accelerator: "CmdOrCtrl+Q", 9 | click: () => { 10 | app.quit(); 11 | } 12 | } 13 | ] 14 | }; 15 | -------------------------------------------------------------------------------- /src/menu/dev_menu_template.js: -------------------------------------------------------------------------------- 1 | import { BrowserWindow } from "electron"; 2 | 3 | export default { 4 | label: "Development", 5 | submenu: [ 6 | { 7 | label: "Reload", 8 | accelerator: "CmdOrCtrl+R", 9 | click: () => { 10 | BrowserWindow.getFocusedWindow().webContents.reloadIgnoringCache(); 11 | } 12 | }, 13 | { 14 | label: "Toggle DevTools", 15 | accelerator: "Alt+CmdOrCtrl+I", 16 | click: () => { 17 | BrowserWindow.getFocusedWindow().toggleDevTools(); 18 | } 19 | } 20 | ] 21 | }; 22 | -------------------------------------------------------------------------------- /src/menu/edit_menu_template.js: -------------------------------------------------------------------------------- 1 | export default { 2 | label: "Edit", 3 | submenu: [ 4 | { label: "Undo", accelerator: "CmdOrCtrl+Z", selector: "undo:" }, 5 | { label: "Redo", accelerator: "Shift+CmdOrCtrl+Z", selector: "redo:" }, 6 | { type: "separator" }, 7 | { label: "Cut", accelerator: "CmdOrCtrl+X", selector: "cut:" }, 8 | { label: "Copy", accelerator: "CmdOrCtrl+C", selector: "copy:" }, 9 | { label: "Paste", accelerator: "CmdOrCtrl+V", selector: "paste:" }, 10 | { label: "Select All", accelerator: "CmdOrCtrl+A", selector: "selectAll:" } 11 | ] 12 | }; 13 | -------------------------------------------------------------------------------- /src/pages/Dashboard.jsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from 'react'; 2 | 3 | import Sidebar from '../partials/Sidebar'; 4 | import Header from '../partials/Header'; 5 | import { Link } from 'react-router-dom'; 6 | import GetStatusBadge from '../utils/GetStatusBadge'; 7 | import { ipcRenderer } from 'electron'; 8 | import { getFolderFromProject } from '../utils/Utils'; 9 | 10 | function Dashboard() { 11 | 12 | const [sidebarOpen, setSidebarOpen] = useState(false); 13 | 14 | let [projectsTrs, setProjectsTrs] = useState(null); 15 | 16 | function recentProjectsListener(event, projects) { 17 | setProjectsTrs(projects.map((project, index) => ( 18 | 19 | 20 |
{project.name}
21 | 22 | 23 |
{project.url}
24 | 25 | 26 | 37 | 44 | 45 | 46 | ))); 47 | } 48 | function loadRecentProjects() { 49 | ipcRenderer.send("get-recent-projects"); 50 | ipcRenderer.on("recent-projects", recentProjectsListener); 51 | } 52 | useEffect(() => { 53 | loadRecentProjects(); 54 | return () => { 55 | ipcRenderer.removeListener("recent-projects", recentProjectsListener); 56 | } 57 | }, []); 58 | 59 | let [stats, setStats] = useState(null); 60 | 61 | function statsListener(event, stats) { 62 | setStats(stats); 63 | } 64 | function loadStats() { 65 | ipcRenderer.send("get-projects-stats"); 66 | ipcRenderer.on("projects-stats", statsListener); 67 | } 68 | useEffect(() => { 69 | loadStats(); 70 | return () => { 71 | ipcRenderer.removeListener("projects-stats", statsListener); 72 | } 73 | }, []); 74 | 75 | return ( 76 |
77 | 78 | {/* Sidebar */} 79 | 80 | 81 | {/* Content area */} 82 |
83 | 84 | {/* Site header */} 85 |
86 | 87 |
88 |
89 | 90 | {/* Dashboard actions */} 91 |
92 | 93 | {/* Right: Actions */} 94 |
95 | {/* Add view button */} 96 | 104 |
105 | 106 |
107 | 108 | {/* Cards */} 109 |
110 | 111 |
112 |
113 |

Clone Status

114 |
115 | 116 | {/* Table header */} 117 | 118 | 119 | 122 | 125 | 126 | 127 | {/* Table body */} 128 | 129 | {[0, 1, 2, 3, -1].map((val, index) => { 130 | return ( 131 | 134 | 143 | 146 | 147 | ); 148 | })} 149 | 150 |
120 |
Status
121 | 		123 |
Number
124 |
135 |
136 | 139 | {GetStatusBadge(val)} 140 | 141 |
142 | 		144 | {stats && stats[val.toString()] ? stats[val.toString()] : "0"} 145 |
151 |
152 |
153 |
154 | 155 |
156 |
157 |

158 | Recent Clones 159 |
160 | 167 |
168 |

169 |
170 | 171 | {/* Table header */} 172 | 173 | 174 | 177 | 180 | 183 | 184 | 185 | {/* Table body */} 186 | 187 | {projectsTrs} 188 | 189 |
175 |
Name
176 | 		178 |
URL
179 | 		181 |
Actions
182 |
190 |
191 |
192 | It looks like you don't have any Clones yet.

Start one here, and it will show up in this block. 193 |
194 |
195 |
196 | 197 |
198 | 199 |
200 |
201 |

Need help? Check out the Help Center.

202 |
203 | 204 |
205 |
206 | 207 |
208 |
209 |
210 | 211 |
212 |
213 | ); 214 | } 215 | 216 | export default Dashboard; -------------------------------------------------------------------------------- /src/pages/Download.jsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect, useState } from 'react'; 2 | 3 | import Sidebar from '../partials/Sidebar'; 4 | import Header from '../partials/Header'; 5 | import { ipcRenderer } from "electron"; 6 | const moment = require('moment'); 7 | import { Link, useParams } from "react-router-dom"; 8 | import GetStatusBadge from '../utils/GetStatusBadge'; 9 | import parseUrl from 'parse-url'; 10 | import { getFolderFromProject, getTimeElapsedFunc } from '../utils/Utils'; 11 | 12 | function Download(props) { 13 | 14 | const [sidebarOpen, setSidebarOpen] = useState(false); 15 | 16 | const [project, setProject] = useState(null); 17 | const [projectRender, setProjectRender] = useState((
Loading...
)); 18 | 19 | const params = useParams(); 20 | 21 | function getProjectUrl(project) { 22 | if (!project) { 23 | return ''; 24 | } 25 | let parsed = parseUrl(project.url); 26 | return parsed.resource || parsed.pathname; 27 | } 28 | 29 | function projectListener(event, project) { 30 | setProjectRender(( 31 |
32 |
33 |

34 | {project.name} 35 | 36 | 41 |

42 | 43 |
44 |
45 | 46 | {/* Table */} 47 |
48 | 49 | {/* Table header */} 50 | 51 | 52 | 55 | 58 | 59 | 60 | {/* Table body */} 61 | 62 | {/* 63 | 66 | 69 | */} 70 | 71 | 74 | 77 | 78 | 79 | 82 | 97 | 98 | 99 | 102 | 105 | 106 | 107 | 110 | 113 | 114 | 115 | 118 | 121 | 122 | 123 | 126 | 129 | 130 | 131 | 134 | 137 | 138 | 139 | 142 | 145 | 146 | 147 |
53 |
Field
54 | 		56 |
Value
57 |
64 |
ID
65 | 		67 |
{project.id}
68 |
72 |
Status
73 | 		75 | {GetStatusBadge(project.status)} 76 |
80 |
Open
81 | 		83 |
84 | 95 |
96 |
100 |
Name
101 | 		103 |
{project.name}
104 |
108 |
URL
109 | 		111 |
{project.url}
112 |
116 |
Download path
117 | 		119 |
{project && project.base_path ? project.base_path : ''}/{project ? getProjectUrl(project) : ''}
120 |
124 |
Started
125 | 		127 |
{project.started ? moment(project.started).format('MMMM Do YYYY, h:mma') : ''}
128 |
132 |
Completed
133 | 		135 |
{project.completed ? moment(project.completed).format('MMMM Do YYYY, h:mma') : ''}
136 |
140 |
Total time
141 | 		143 |
{project ? getTimeElapsedFunc(project) : ''}
144 |
148 | 149 |
150 |
151 |
152 | )); 153 | } 154 | function loadProject() { 155 | if (params && params.id) { 156 | ipcRenderer.send("get-project", params.id); 157 | ipcRenderer.on("project", projectListener); 158 | } 159 | } 160 | useEffect(() => { 161 | loadProject(); 162 | 163 | return(() => { 164 | ipcRenderer.removeListener("project", projectListener); 165 | }); 166 | }, []); 167 | 168 | return ( 169 |
170 | 171 | {/* Sidebar */} 172 | 173 | 174 | {/* Content area */} 175 |
176 | 177 | {/* Site header */} 178 |
179 | 180 |
181 |
182 | 183 | {/* Dashboard actions */} 184 |
185 | 186 | {/* Right: Actions */} 187 |
188 | 193 |
194 | 195 |
196 | 197 | {/* Cards */} 198 |
199 |
200 |
201 |
202 | {projectRender} 203 |
204 |
205 |
206 |
207 | 208 |
209 |
210 | 211 |
212 |
213 | ); 214 | } 215 | 216 | export default Download; -------------------------------------------------------------------------------- /src/pages/Downloads.jsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect, useState } from 'react'; 2 | 3 | import Sidebar from '../partials/Sidebar'; 4 | import Header from '../partials/Header'; 5 | import { ipcRenderer } from "electron"; 6 | import { Link, useParams } from 'react-router-dom'; 7 | import GetStatusBadge from '../utils/GetStatusBadge'; 8 | import parseUrl from 'parse-url'; 9 | import { getFolderFromProject } from '../utils/Utils'; 10 | 11 | function Downloads() { 12 | 13 | const [sidebarOpen, setSidebarOpen] = useState(false); 14 | 15 | const params = useParams(); 16 | 17 | let [statusFilters, setStatusFilters] = useState(params.filterId ? [Number.parseInt(params.filterId)] : []); 18 | 19 | let [statusFiltersEls, setStatusFiltersEls] = useState([0, 1, 2, 3, -1].map((val, index) => { 20 | return ( 21 |
{ 24 | toggleStatus(val) 25 | }} 26 | className="status-filter" 27 | > 28 | {GetStatusBadge(val, 'large')} 29 |
30 | ); 31 | })); 32 | let [projects, setProjects] = useState([]); 33 | let [lastSearchValue, setLastSearchValue] = useState(''); 34 | let [allProjects, setAllProjects] = useState([]); 35 | let [filteredProjects, setFilteredProjects] = useState([]); 36 | let [projectsLi, setProjectsLi] = useState(( 37 |
  • 38 | Loading... 39 |
    • 40 | )); 41 | 42 | useEffect(() => { 43 | ipcRenderer.on("deleted-project", loadProjects); 44 | return () => { 45 | ipcRenderer.removeListener("deleted-project", loadProjects); 46 | } 47 | }); 48 | 49 | function projectsListener(event, projects) { 50 | setAllProjects(projects); 51 | } 52 | function setProjectsLiFunc(projects) { 53 | setProjectsLi(projects.map((project) => ( 54 |
  • 55 |
    56 |
    57 |
    58 | {GetStatusBadge(project.status)} {project.name} {/* ({project.id})*/} 59 |
    URL: {project.url}
    60 |
    61 |
    62 | 73 | 80 | { 82 | e.preventDefault(); 83 | e.stopPropagation(); 84 | ipcRenderer.send("delete-project", project.id); 85 | }} 86 | >✖ 87 |
    88 |
    89 |
    90 |
    • 91 | ))); 92 | } 93 | function loadProjects() { 94 | ipcRenderer.send("get-projects"); 95 | ipcRenderer.on("projects", projectsListener); 96 | } 97 | useEffect(() => { 98 | loadProjects(); 99 | return () => { 100 | ipcRenderer.removeListener("projects", projectsListener); 101 | } 102 | }, []); 103 | 104 | function toggleStatus(val) { 105 | let s = statusFilters; 106 | if (typeof val !== 'undefined' ) { 107 | if (statusFilters.indexOf(val) === -1) { 108 | s.push(val); 109 | setStatusFilters(s); 110 | } 111 | else { 112 | const index = statusFilters.indexOf(val); 113 | if (index > -1) { 114 | s.splice(index, 1); 115 | setStatusFilters(s); 116 | } 117 | } 118 | } 119 | 120 | if (s.length > 0) { 121 | let p = allProjects.filter((val) => { 122 | if (val && typeof val.status !== 'undefined' && s.indexOf(val.status) === -1) { 123 | return false; 124 | } 125 | return true; 126 | }); 127 | setFilteredProjects(p); 128 | } 129 | else { 130 | setFilteredProjects(allProjects); 131 | } 132 | 133 | setStatusFiltersElFunc(s); 134 | } 135 | 136 | useEffect(() => { 137 | toggleStatus(); 138 | }, [allProjects]); 139 | 140 | useEffect(() => { 141 | if (lastSearchValue.length > 0) { 142 | let p = filteredProjects.filter((val) => { 143 | if (val && typeof val.name !== 'undefined' && val.name.toLowerCase().indexOf(lastSearchValue.toLowerCase()) !== -1) { 144 | return true; 145 | } 146 | if (val && typeof val.url !== 'undefined' && val.url.toLowerCase().indexOf(lastSearchValue.toLowerCase()) !== -1) { 147 | return true; 148 | } 149 | return false; 150 | }) 151 | setProjectsLiFunc(p); 152 | } 153 | else { 154 | setProjectsLiFunc(filteredProjects); 155 | } 156 | }, [lastSearchValue, filteredProjects]); 157 | 158 | function handleSearchChange(event) { 159 | let search = event.target.value.trim(); 160 | setLastSearchValue(search); 161 | } 162 | 163 | function setStatusFiltersElFunc(arr) { 164 | setStatusFiltersEls([0, 1, 2, 3, -1].map((val, index) => { 165 | return ( 166 |
    { 169 | toggleStatus(val) 170 | }} 171 | className={arr.indexOf(val) === -1 ? "status-filter" : "status-filter active"} 172 | > 173 | {GetStatusBadge(val, 'large')} 174 |
    175 | ); 176 | })); 177 | } 178 | 179 | return ( 180 |
    181 | 182 | {/* Sidebar */} 183 | 184 | 185 | {/* Content area */} 186 |
    187 | 188 | {/* Site header */} 189 |
    190 | 191 |
    192 |
    193 | 194 | {/* Dashboard actions */} 195 |
    196 | 197 | {/* Right: Actions */} 198 |
    199 | {/* Add view button */} 200 | 208 |
    209 | 210 |
    211 | 212 |
    213 |
    214 |

    Filters

    215 |
    216 | {statusFiltersEls} 217 |
    218 |
    219 |
    220 |

    Search

    221 | 222 |
    223 |
    224 | 225 | {/* Cards */} 226 |
    227 |
    228 |
    229 |

    Clones

    230 |
    231 |
    232 |
    233 |
      234 | {projectsLi} 235 |
    236 |
    237 |
    238 |
    239 |
    240 | 241 |
    242 |
    243 | 244 |
    245 |
    246 | ); 247 | } 248 | 249 | export default Downloads; -------------------------------------------------------------------------------- /src/pages/Help.jsx: -------------------------------------------------------------------------------- 1 | import React, { useState } from 'react'; 2 | 3 | import Sidebar from '../partials/Sidebar'; 4 | import Header from '../partials/Header'; 5 | 6 | import { 7 | Accordion, 8 | AccordionItem, 9 | AccordionItemHeading, 10 | AccordionItemButton, 11 | AccordionItemPanel, 12 | } from 'react-accessible-accordion'; 13 | import { Link } from 'react-router-dom'; 14 | 15 | function Help() { 16 | 17 | const [sidebarOpen, setSidebarOpen] = useState(false); 18 | 19 | return ( 20 |
    21 | 22 | {/* Sidebar */} 23 | 24 | 25 | {/* Content area */} 26 |
    27 | 28 | {/* Site header */} 29 |
    30 | 31 |
    32 |
    33 | 34 |

    Help Center

    35 | 36 |

    37 | If you have a question that's not answered here, don't hestitate to contact us at cloneableapp@gmail.com or interact with us on GitHub. 38 |

    39 | 40 |

    General Questions

    41 | 44 | 45 | 46 | 47 | What about [insert missing feature]? 48 | 49 | 50 | 51 |

    52 | If Cloneable is missing a feature that you want, don't hestitate to contact us by emailing us at cloneableapp@gmail.com or contacting us on GitHub. 53 |

    54 | We try to make the software as flexible and useful as possible. 55 |

    56 |     57 |     58 | 59 | 60 | 61 | Who made Cloneable? 62 | 63 | 64 | 65 |

    66 | The founder of Cloneable is web developer who loves making useful tools. Email me at cloneableapp@gmail.com if you want to chat. 67 |

    68 |     69 |     70 |     71 | 72 |

    Filters

    73 | 76 | 77 | 78 | 79 | How do the filter settings work? 80 | 81 | 82 | 83 |

    84 | The filters are a configurable way to control how a website is crawled. 85 |
    86 |
    87 | There are a few different kinds of filters. See the questions below for individual help with each one. 88 |

    89 |     90 |     91 | 92 | 93 | 94 | How does the "Extra URLs / Domains" filter setting work? 95 | 96 | 97 | 98 |

    99 | By default, Cloneable will only crawl the domain of the main URL you give it. But sometimes you'll want to allow Cloneable 100 | to crawl other domains, too. For example, if a website hosts its media files (images, videos) on a different domain, you might want to add that domain 101 | to this setting so that you can download those media files, too. 102 |

    103 |     104 |     105 | 106 | 107 | 108 | How does the "Accept filters" filter setting work? 109 | 110 | 111 | 112 |

    113 | This setting has two different modes, depending on if you include a Regex wildcard character (*, ?, [, ]) or not. 114 |

    115 | If you don't include a wildcard character, this setting will act as a list of suffixes to accept in the crawl. 116 | For example, if you set this to "png", the crawl will only download files ending in "png". 117 |

    118 | If you do include a Regex wildcard character, this setting will act as a list of patterns to match against the filename (not just a suffix). 119 | For example, if you set this to "*mortgage*", the crawl will only download files with "mortgage" in the filename. 120 |

    121 |     122 |     123 | 124 | 125 | 126 | How does the "Reject filters" filter setting work? 127 | 128 | 129 | 130 |

    131 | This setting has two different modes, depending on if you include a Regex wildcard character (*, ?, [, ]) or not. 132 |

    133 | If you don't include a wildcard character, this setting will act as a list of suffixes to reject in the crawl. 134 | For example, if you set this to "png", the crawl will not download files ending in "png". 135 |

    136 | If you do include a Regex wildcard character, this setting will act as a list of patterns to match against the filename (not just a suffix). 137 | For example, if you set this to "*mortgage*", the crawl will not download files with "mortgage" in the filename. 138 |

    139 |     140 |     141 | 142 | 143 | 144 | How does the "Include directories" filter setting work? 145 | 146 | 147 | 148 |

    149 | While "Accept filters" and "Reject filters" work on filenames, this setting works on directory names. For example, 150 | in the URL https://example.com/one/index.html, "one" is a directory name while "index.html" is a filename. 151 |

    152 | If left blank, the crawl will follow all directories (except the ones in "Exclude directories" setting). But if you provide 153 | a value for this setting, the crawl will only follow directories in the given list. 154 |

    155 | For example, if you set this to "one", all of the pages under https://example.com/one/ will be crawled, but not any pages under 156 | https://example.com/two/ or https://example.com/three/ etc. 157 |

    158 |     159 |     160 | 161 | 162 | 163 | How does the "Exclude directories" filter setting work? 164 | 165 | 166 | 167 |

    168 | While "Accept filters" and "Reject filters" work on filenames, this setting works on directory names. For example, 169 | in the URL https://example.com/one/index.html, "one" is a directory name while "index.html" is a filename. 170 |

    171 | If you provide a value for this setting, the crawl will not follow directories in the given list. 172 |

    173 |     174 |     175 |     176 | 177 |

    Errors

    178 | 181 | 182 | 183 | 184 | What does "Error: spawn wget ENOENT" mean? 185 | 186 | 187 | 188 |

    189 | It means the bundled version of wget does not work on your system. Please install wget separately, 190 | and set the path to the binary in your Clone settings under the Advanced tab. 191 |

    192 |     193 |     194 | 195 | 196 | 197 | What does this error code mean? 198 | 199 | 200 | 201 |

    202 | Here are a list of common error codes: 203 |
    204 | 1 - Generic error code.
    205 | 2 - Parse error — for instance, when parsing options.
    206 | 3 - File I/O error.
    207 | 4 - Network failure.
    208 | 5 - SSL verification failure.
    209 | 6 - Username/password authentication failure.
    210 | 7 - Protocol errors.
    211 | 8 - Server issued an error response.
    212 |

    213 |     214 |     215 | 216 | 217 | 218 | I got a different error than the above. 219 | 220 | 221 | 222 |

    223 | Please contact us at cloneableapp@gmail.com with details and we will try to help. Please include the error message and the "Detailed Logs" from the Clone, and any other info you think may be helpful. 224 |

    225 | Or open an issue on our GitHub repo. 226 |

    227 |     228 |     229 |     230 | 231 |

    Cookies

    232 | 235 | 236 | 237 | 238 | What format should the "Cookies file path" file be in? 239 | 240 | 241 | 242 |

    243 | Cloneable expects the file to be in the Netscape cookies format. There are some free tools online for generation and conversion of cookies form your browser to this format. 244 |

    245 |     246 |     247 | 248 | 249 | 250 | If I enable "Save cookies" and/or "Keep session cookies", where are they stored? 251 | 252 | 253 | 254 |

    255 | Your cookies file will be stored in a file called cookies.txt in the root of the chosen working directory of the Clone. 256 |

    257 |     258 |     259 |     260 | 261 |

    Credits

    262 | 265 | 266 | 267 | 268 | Icon8 credit 269 | 270 | 271 | 272 |

    273 | Double Down icon by Icons8 274 |

    275 |     276 |     277 |     278 | 279 |
    280 |
    281 | 282 |
    283 |
    284 | ); 285 | } 286 | 287 | export default Help; -------------------------------------------------------------------------------- /src/pages/NewDownload.jsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useRef, useEffect } from 'react'; 2 | import Sidebar from '../partials/Sidebar'; 3 | import Header from '../partials/Header'; 4 | import { Link, useParams } from 'react-router-dom'; 5 | import SettingsTabs from '../partials/SettingsTabs'; 6 | import { ipcRenderer } from "electron"; 7 | import GetStatusBadge from '../utils/GetStatusBadge'; 8 | import parseUrl from 'parse-url'; 9 | import { getFolderFromProject, getTimeElapsedFunc } from '../utils/Utils'; 10 | const moment = require('moment'); 11 | 12 | function NewDownload(props) { 13 | 14 | const [sidebarOpen, setSidebarOpen] = useState(false); 15 | 16 | const params = useParams(); 17 | 18 | const [step, setStep] = useState(0); 19 | const [projectNameInputValue, setProjectNameInputValue] = useState(''); 20 | const [projectURLInputValue, setProjectURLInputValue] = useState(''); 21 | const [projectId, setProjectId] = useState(params.id ? params.id : props.id ? props.id : ''); 22 | const [project, setProject] = useState(null); 23 | const [timeElapsed, setTimeElapsed] = useState(''); 24 | const [cloneData, setCloneData] = useState(null); 25 | 26 | useEffect(() => { 27 | function projectListener(event, project) { 28 | setProjectNameInputValue(project.name); 29 | setProjectURLInputValue(project.url); 30 | setProject(project); 31 | if (project.clone_data) { 32 | setCloneData(project.clone_data); 33 | } 34 | 35 | if (project.status > 0 && step === 0) { 36 | setStep(4); 37 | } 38 | } 39 | 40 | function timeElapsedListener(event, project) { 41 | setTimeElapsedFunc(project); 42 | } 43 | 44 | if (params.id) { 45 | if (!project) { 46 | ipcRenderer.send("get-project", params.id); 47 | ipcRenderer.on("project", projectListener); 48 | } 49 | ipcRenderer.on("time-elapsed", timeElapsedListener); 50 | } 51 | 52 | return(() => { 53 | ipcRenderer.removeListener("project", projectListener); 54 | ipcRenderer.removeListener("time-elapsed", timeElapsedListener); 55 | }); 56 | }); 57 | 58 | useEffect(() => { 59 | if (step === 4) { 60 | function timeElapsedListener(event, project) { 61 | setTimeElapsedFunc(project); 62 | } 63 | ipcRenderer.on("time-elapsed", timeElapsedListener); 64 | return(() => { 65 | ipcRenderer.removeListener("time-elapsed", timeElapsedListener); 66 | }); 67 | } 68 | }, [step]); 69 | 70 | useEffect(() => { 71 | function cloneFinishedListener(event, project) { 72 | setProject(project); 73 | ipcRenderer.send("save-project-clone-data", project.id, cloneData); 74 | } 75 | ipcRenderer.on("clone-finished", cloneFinishedListener); 76 | return(() => { 77 | ipcRenderer.removeListener("clone-finished", cloneFinishedListener); 78 | }) 79 | }); 80 | 81 | useEffect(() => { 82 | function cloneDataListener(event, projectId, data) { 83 | if (project && projectId !== project.id) { 84 | return; 85 | } 86 | if (cloneData) { 87 | setCloneData(cloneData + "\r\n" + data); 88 | } 89 | else { 90 | setCloneData(data); 91 | } 92 | } 93 | ipcRenderer.on("clone-data", cloneDataListener); 94 | return(() => { 95 | ipcRenderer.removeListener("clone-data", cloneDataListener); 96 | }) 97 | }); 98 | 99 | function setTimeElapsedFunc(project) { 100 | setTimeElapsed(getTimeElapsedFunc(project)); 101 | } 102 | 103 | let stepRendered = ( 104 |
    105 | 106 |
    107 | ); 108 | 109 | function getProjectUrl() { 110 | if (!project) { 111 | return ''; 112 | } 113 | let parsed = parseUrl(project.url); 114 | return parsed.resource || parsed.pathname; 115 | } 116 | 117 | function handleKeyPress(event) { 118 | if(event.key === 'Enter') { 119 | try { 120 | document.getElementById('next-step-button').click(); 121 | } catch (err) { 122 | } 123 | } 124 | } 125 | 126 | if (step === 1 || step === 0) { 127 | stepRendered = ( 128 |
    129 |

    1. Clone info

    130 |
    131 |
    132 | Name: 133 | setProjectNameInputValue(e.target.value)} 139 | defaultValue={projectNameInputValue} 140 | required 141 | onKeyPress={handleKeyPress} 142 | > 143 | 144 |

    145 | This is a helpful name to identify this Clone. It will also be used as the folder name to store the Clone under. 146 |

    147 |
    148 |
    149 | Website address (URL): 150 | setProjectURLInputValue(e.target.value)} 156 | defaultValue={projectURLInputValue} 157 | required 158 | onKeyPress={handleKeyPress} 159 | > 160 | 161 |

    162 | This is the full URL of the website to clone (like https://example.com or https://en.wikipedia.org/wiki/Dog). 163 |

    164 |
    165 |
    166 | 187 |
    188 | ); 189 | } 190 | else if (step === 2) { 191 | stepRendered = ( 192 |
    193 |

    2. Settings

    194 |

    195 | Adjust the settings for this Clone. (You can always adjust these defaults on the global Settings page). 196 |

    197 |
    198 | 201 |
    202 | 209 | 225 |
    226 | ); 227 | } 228 | else if (step === 3) { 229 | stepRendered = ( 230 |
    231 |

    3. Start Clone

    232 |

    233 | When ready, start the Cloning process. Progress will show here. 234 |

    235 | 242 | 258 |
    259 | ); 260 | } 261 | else if (step === 4) { 262 | stepRendered = ( 263 |
    264 |

    3. Clone {project && project.name ? project.name : ''}

    265 |
    266 | {/* Table */} 267 |
    268 | 269 | {/* Table header */} 270 | 271 | 272 | 275 | 278 | 279 | 280 | {/* Table body */} 281 | 282 | 283 | 286 | 289 | 290 | 291 | 294 | 297 | 298 | 299 | 302 | 305 | 306 | 307 | 310 | 313 | 314 | 315 | 318 | 321 | 322 | 323 | 326 | 329 | 330 | 331 | 334 | 337 | 338 | 339 |
    273 |
    Field
    274 |     		276 |
    Value
    277 |
    284 |
    Status
    285 |     		287 | {project ? GetStatusBadge(project.status) : ''} 288 |
    292 |
    Name
    293 |     		295 |
    {project ? project.name : ''}
    296 |
    300 |
    Base URL
    301 |     		303 |
    {project ? project.url : ''}
    304 |
    308 |
    Started
    309 |     		311 |
    {project ? project.started ? moment(project.started).format('MMMM Do YYYY, h:mma') : '' : ''}
    312 |
    316 |
    Completed
    317 |     		319 |
    {project ? project.completed ? moment(project.completed).format('MMMM Do YYYY, h:mma') : '' : ''}
    320 |
    324 |
    Total time
    325 |     		327 |
    {timeElapsed}
    328 |
    332 |
    Total time
    333 |     		335 |
    {project ? getTimeElapsedFunc(project) : ''}
    336 |
    340 | 341 |
    342 | 343 |
    344 | {GetStatusBadge(-1)} {project && project.error ? project.error : ''} 345 |
    346 | 347 |

    Download finished

    348 |
    349 |
    350 | The site is downloaded to your local filesystem at {project && project.base_path ? project.base_path : ''}{project && !project.no_directories ? '/' + (project ? getProjectUrl() : '') : ''} 351 |
    352 | 363 |
    364 | 365 |

    Detailed Logs

    366 | 
    367 |               {cloneData ? cloneData : null}
368 |
    369 |
    370 | 377 | 388 | 400 |
    401 | ); 402 | } 403 | 404 | return ( 405 |
    406 | 407 | {/* Sidebar */} 408 | 409 | 410 | {/* Content area */} 411 |
    412 | 413 | {/* Site header */} 414 |
    415 | 416 |
    417 |
    418 | 419 | {/* Dashboard actions */} 420 |
    421 | 422 | {/* Right: Actions */} 423 |
    424 | 425 | {stepRendered} 426 | 427 |
    428 | 429 |
    430 | 431 | {/* Cards */} 432 |
    433 | 434 |
    435 | 436 |
    437 |
    438 | 439 |
    440 |
    441 | ); 442 | } 443 | 444 | export default NewDownload; -------------------------------------------------------------------------------- /src/pages/Settings.jsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useRef, useEffect } from 'react'; 2 | 3 | import Sidebar from '../partials/Sidebar'; 4 | import Header from '../partials/Header'; 5 | 6 | import SettingsTabs from '../partials/SettingsTabs'; 7 | 8 | function Settings() { 9 | const [sidebarOpen, setSidebarOpen] = useState(false); 10 | 11 | return ( 12 |
    13 | 14 | {/* Sidebar */} 15 | 16 | 17 | {/* Content area */} 18 |
    19 | 20 | {/* Site header */} 21 |
    22 | 23 |
    24 |
    25 |

    Default Settings

    26 |

    27 | These are the global default settings that will prepopulate the settings for each Clone you start. 28 | You can adjust these settings for each Clone before starting. 29 |

    30 |
    31 | 32 |
    33 |
    34 |
    35 | 36 |
    37 |
    38 | ); 39 | } 40 | 41 | export default Settings; -------------------------------------------------------------------------------- /src/partials/Header.jsx: -------------------------------------------------------------------------------- 1 | import React, { useState } from 'react'; 2 | import Help from './header/Help'; 3 | 4 | function Header({ 5 | sidebarOpen, 6 | setSidebarOpen 7 | }) { 8 | 9 | const [searchModalOpen, setSearchModalOpen] = useState(false) 10 | 11 | return ( 12 |
    13 |
    14 |
    15 |

    16 |
    17 | 18 | 19 | 20 | 21 |
    22 | 23 | Cloneable 24 |

    25 | 26 | {/* Header: Left side */} 27 |
    28 | 29 | {/* Hamburger button */} 30 | 43 | 44 |
    45 | 46 | {/* Header: Right side */} 47 |
    48 | 49 | 50 | 51 |
    52 | 53 |
    54 |
    55 |
    56 | ); 57 | } 58 | 59 | export default Header; -------------------------------------------------------------------------------- /src/partials/SettingsTabs.jsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useRef, useEffect } from 'react'; 2 | import Form from "@rjsf/bootstrap-4"; 3 | import { ipcRenderer } from "electron"; 4 | 5 | import { Tab, Tabs, TabList, TabPanel } from 'react-tabs'; 6 | import { ipcMain } from 'electron/main'; 7 | 8 | const os = require('os'); 9 | 10 | const isMac = os.platform() === "darwin"; 11 | const isWindows = os.platform() === "win32"; 12 | const isLinux = os.platform() === "linux"; 13 | 14 | function SettingsTabs(props) { 15 | function getFileNamesModeDefault() { 16 | if (isMac || isLinux) { 17 | return "unix"; 18 | } 19 | if (isWindows) { 20 | return "windows"; 21 | } 22 | } 23 | 24 | function saveTabAndGetSettings(index) { 25 | saveTab(index); 26 | getSettings(); 27 | } 28 | function saveTab(index) { 29 | if (props.projectId) { 30 | return; 31 | } 32 | ipcRenderer.send("save-settings-tab", index); 33 | } 34 | function saveSettings() { 35 | const saveButtons = document.querySelectorAll('button[type="submit"]'); 36 | for (let i = 0; i < saveButtons.length; i++) { 37 | saveButtons[i].click(); 38 | } 39 | } 40 | function settingsTabListener(event, index) { 41 | try { 42 | document.querySelectorAll('.react-tabs__tab:nth-child(' + (index+1) + ')')[0].click(); 43 | } catch (e) { 44 | } 45 | try { 46 | document.querySelectorAll('.react-tabs')[0].style.display = 'block'; 47 | } catch (e) { 48 | } 49 | } 50 | function getSettingsTab() { 51 | if (props.projectId) { 52 | document.querySelectorAll('.react-tabs')[0].style.display = 'block'; 53 | return; 54 | } 55 | ipcRenderer.send("get-settings-tab"); 56 | ipcRenderer.on("settings-tab", settingsTabListener); 57 | } 58 | const [storageForm, setStorageForm] = useState(''); 59 | const [crawlForm, setCrawlForm] = useState(''); 60 | const [filtersForm, setFiltersForm] = useState(''); 61 | const [browserForm, setBrowserForm] = useState(''); 62 | const [authForm, setAuthForm] = useState(''); 63 | const [advancedForm, setAdvancedForm] = useState(''); 64 | const [storageSettings, setStorageSettings] = useState(null); 65 | const [authSettings, setAuthSettings] = useState(null); 66 | const [downloadsFolder, setDownloadsFolder] = useState(""); 67 | 68 | function selectNewFolder() { 69 | ipcRenderer.send("select-folder"); 70 | ipcRenderer.on("selected-folder", newFolderListener); 71 | } 72 | 73 | function selectNewCookiesFile() { 74 | ipcRenderer.send("select-file"); 75 | ipcRenderer.on("selected-file", newCookiesFileListener); 76 | } 77 | 78 | function newFolderListener(event, folder) { 79 | let copySettings = JSON.parse(JSON.stringify(storageSettings || {})); 80 | copySettings.basePath = folder; 81 | ipcRenderer.send("save-settings", "storage", copySettings, props.projectId); 82 | setStorageSettings(copySettings); 83 | setStorageForm( 84 | ( 85 |
    86 |
    { 92 | ipcRenderer.send("save-settings", "storage", formData, props.projectId) 93 | } 94 | } 95 | className="rjsf bg-white shadow-lg rounded-sm border border-slate-200" 96 | /> 97 | 98 |
    99 | ) 100 | ); 101 | } 102 | 103 | function storageListener(event, settings) { 104 | setStorageSettings(settings); 105 | setStorageForm( 106 | ( 107 |
    108 | { 114 | ipcRenderer.send("save-settings", "storage", formData, props.projectId) 115 | } 116 | } 117 | className="rjsf bg-white shadow-lg rounded-sm border border-slate-200" 118 | /> 119 | 120 |
    121 | ) 122 | ); 123 | } 124 | 125 | function crawlListener(event, settings) { 126 | setCrawlForm( 127 | ( 128 |
    129 | { 135 | ipcRenderer.send("save-settings", "crawl", formData, props.projectId) 136 | } 137 | } 138 | className="rjsf bg-white shadow-lg rounded-sm border border-slate-200" 139 | /> 140 |
    141 | ) 142 | ); 143 | } 144 | 145 | function filtersListener(event, settings) { 146 | setFiltersForm( 147 | ( 148 |
    149 | { 155 | ipcRenderer.send("save-settings", "filters", formData, props.projectId) 156 | } 157 | } 158 | className="rjsf rjsf-filters bg-white shadow-lg rounded-sm border border-slate-200" 159 | /> 160 |
    161 | ) 162 | ); 163 | } 164 | 165 | function browserListener(event, settings) { 166 | setBrowserForm( 167 | ( 168 |
    169 | { 175 | ipcRenderer.send("save-settings", "browser", formData, props.projectId) 176 | } 177 | } 178 | className="rjsf bg-white shadow-lg rounded-sm border border-slate-200" 179 | /> 180 |
    181 | ) 182 | ); 183 | } 184 | 185 | function newCookiesFileListener(event, file) { 186 | let copySettings = JSON.parse(JSON.stringify(authSettings || {})); 187 | copySettings.cookiesFilePath = file; 188 | ipcRenderer.send("save-settings", "auth", copySettings, props.projectId); 189 | setAuthSettings(copySettings); 190 | setAuthForm( 191 | ( 192 |
    193 | { 199 | ipcRenderer.send("save-settings", "auth", formData, props.projectId) 200 | } 201 | } 202 | className="rjsf bg-white shadow-lg rounded-sm border border-slate-200" 203 | /> 204 | 205 |
    206 | ) 207 | ); 208 | } 209 | 210 | function authListener(event, settings) { 211 | setAuthSettings(settings); 212 | setAuthForm( 213 | ( 214 |
    215 | { 221 | ipcRenderer.send("save-settings", "auth", formData, props.projectId) 222 | } 223 | } 224 | className="rjsf bg-white shadow-lg rounded-sm border border-slate-200" 225 | /> 226 | 227 |
    228 | ) 229 | ); 230 | } 231 | 232 | function advancedListener(event, settings) { 233 | setAdvancedForm( 234 | ( 235 |
    236 | { 242 | ipcRenderer.send("save-settings", "advanced", formData, props.projectId) 243 | } 244 | } 245 | className="rjsf bg-white shadow-lg rounded-sm border border-slate-200" 246 | /> 247 |
    248 | ) 249 | ); 250 | } 251 | 252 | function getSettings() { 253 | ipcRenderer.send("get-settings", "storage", props.projectId); 254 | ipcRenderer.on("settings-storage", storageListener); 255 | 256 | ipcRenderer.send("get-settings", "crawl", props.projectId); 257 | ipcRenderer.on("settings-crawl", crawlListener); 258 | 259 | ipcRenderer.send("get-settings", "filters", props.projectId); 260 | ipcRenderer.on("settings-filters", filtersListener); 261 | 262 | ipcRenderer.send("get-settings", "browser", props.projectId); 263 | ipcRenderer.on("settings-browser", browserListener); 264 | 265 | ipcRenderer.send("get-settings", "auth", props.projectId); 266 | ipcRenderer.on("settings-auth", authListener); 267 | 268 | ipcRenderer.send("get-settings", "advanced", props.projectId); 269 | ipcRenderer.on("settings-advanced", advancedListener); 270 | } 271 | useEffect(() => { 272 | getSettings(); 273 | getSettingsTab(); 274 | ipcRenderer.send("get-downloads-folder"); 275 | function downloadsFolderListener(event, value) { 276 | setDownloadsFolder(value); 277 | } 278 | ipcRenderer.on("downloads-folder", downloadsFolderListener); 279 | 280 | return(() => { 281 | ipcRenderer.removeListener("settings-storage", storageListener); 282 | ipcRenderer.removeListener("settings-crawl", crawlListener); 283 | ipcRenderer.removeListener("settings-filters", filtersListener); 284 | ipcRenderer.removeListener("settings-browser", browserListener); 285 | ipcRenderer.removeListener("settings-auth", authListener); 286 | ipcRenderer.removeListener("settings-advanced", advancedListener); 287 | 288 | ipcRenderer.removeListener("settings-tab", settingsTabListener); 289 | 290 | ipcRenderer.removeListener("selected-folder", newFolderListener); 291 | 292 | ipcRenderer.removeListener("downloads-folder", downloadsFolderListener); 293 | }); 294 | }, []); 295 | 296 | const uiSchema = { 297 | "ui:submitButtonOptions": { 298 | "submitText": "Save", 299 | }, 300 | acceptFilters: { 301 | "ui:widget": "textarea" 302 | }, 303 | rejectFilters: { 304 | "ui:widget": "textarea" 305 | }, 306 | includeDirs: { 307 | "ui:widget": "textarea" 308 | }, 309 | excludeDirs: { 310 | "ui:widget": "textarea" 311 | }, 312 | domains: { 313 | "ui:widget": "textarea" 314 | }, 315 | headers: { 316 | "ui:widget": "textarea" 317 | }, 318 | }; 319 | 320 | const storageSchema = { 321 | title: "", 322 | type: "object", 323 | required: [ 324 | ], 325 | properties: { 326 | basePath: { 327 | type: "string", 328 | title: "Storage base folder", 329 | description: props.projectId ? "The base folder to store this download's folders in" : "The base folder to store your download folders in", 330 | default: "" //downloadsFolder 331 | }, 332 | } 333 | }; 334 | 335 | const crawlSchema = { 336 | title: "", 337 | type: "object", 338 | required: [ 339 | ], 340 | properties: { 341 | defaultPageName: { 342 | type: "string", 343 | title: "Default page name", 344 | description: "Default file name for pages without a name", 345 | default: "index.html" 346 | }, 347 | noRecursive: { 348 | type: "boolean", 349 | title: "Don't follow recursive links", 350 | description: "", 351 | default: false 352 | }, 353 | recursiveDepthLevel: { 354 | type: "number", 355 | title: "Recursive maximum depth level", 356 | description: "-1 for infinite depth", 357 | default: -1 358 | }, 359 | noDirectories: { 360 | type: "boolean", 361 | title: "Do not create a hierarchy of directories when retrieving recursively", 362 | description: "", 363 | default: false 364 | }, 365 | numRetries: { 366 | type: "number", 367 | title: "# of retries", 368 | description: "0 for infinite retrying", 369 | default: 20 370 | }, 371 | timeout: { 372 | type: "object", 373 | title: "Timeouts (in seconds)", 374 | description: "0 to disable altogether", 375 | properties: { 376 | dnsTimeout: { 377 | type: "number", 378 | title: "DNS timeout", 379 | default: 0, 380 | }, 381 | connectTimeout: { 382 | type: "number", 383 | title: "Connect timeout", 384 | default: 0 385 | }, 386 | readTimeout: { 387 | type: "number", 388 | title: "Read timeout", 389 | default: 900 390 | } 391 | } 392 | }, 393 | limitDownloadSpeed: { 394 | type: "string", 395 | title: "Limit download speed", 396 | description: "Amount may be expressed in bytes, kilobytes with the k suffix, or megabytes with the m suffix. For example, 20k", 397 | default: "" 398 | }, 399 | waitTime: { 400 | type: "number", 401 | title: "Wait time", 402 | description: "Number of seconds to wait between each page request.", 403 | default: 1 404 | }, 405 | randomizeWaitTime: { 406 | type: "boolean", 407 | title: "Randomize wait time", 408 | description: "Slightly randomize the wait time between requests. Helps prevent detection on some servers", 409 | default: false 410 | }, 411 | waitRetryTime: { 412 | type: "number", 413 | title: "Wait time for retries", 414 | description: "Number of seconds to wait between each page request for retries (failures)", 415 | default: 10 416 | }, 417 | downloadQuota: { 418 | type: "string", 419 | title: "Maximum download quota", 420 | description: "Site downloads will stop after they reach this total size. Amount may be expressed in bytes, kilobytes with the k suffix, or megabytes with the m suffix. For example, 20k. Use 0 or inf for unlimited download quota", 421 | default: "inf" 422 | }, 423 | maxRedirects: { 424 | type: "number", 425 | title: "Maximum redirects", 426 | description: "Maximum number of redirections to follow for any given request", 427 | default: 20, 428 | }, 429 | } 430 | }; 431 | 432 | const filtersSchema = { 433 | title: "", 434 | type: "object", 435 | required: [ 436 | ], 437 | properties: { 438 | domains: { 439 | type: "string", 440 | title: "Extra URLs / Domains", 441 | description: "Enter each url or domain on a new line. This is useful if the site you are cloning has media files hosted on a separate domain, for example.", 442 | default: "", 443 | }, 444 | acceptFilters: { 445 | type: "string", 446 | title: "Accept filters", 447 | description: "File name suffixes or patterns to accept (include). Enter each item on a new line. Regex wildcards (*, ?, [, ]) are supported, but the presence of one will make this filter be treated as a pattern, rather than a suffix.", 448 | default: "", 449 | }, 450 | rejectFilters: { 451 | type: "string", 452 | title: "Reject filters", 453 | description: "File name suffixes or patterns to reject (exclude). Enter each item on a new line. Regex wildcards (*, ?, [, ]) are supported, but the presence of one will make this filter be treated as a pattern, rather than a suffix.", 454 | default: "", 455 | }, 456 | includeDirs: { 457 | type: "string", 458 | title: "Include directories", 459 | description: "Directories you wish to follow when downloading. Enter each item on a new line. May contain wildcards. Leave blank for all.", 460 | default: "" 461 | }, 462 | excludeDirs: { 463 | type: "string", 464 | title: "Exclude directories", 465 | description: "Directories you wish to not follow when downloading. Enter each item on a new line. May contain wildcards.", 466 | default: "" 467 | }, 468 | ignoreFiltersCase: { 469 | type: "boolean", 470 | title: "Ignore case in accept/reject filters", 471 | description: "", 472 | default: false, 473 | }, 474 | } 475 | }; 476 | 477 | const browserSchema = { 478 | title: "", 479 | type: "object", 480 | required: [ 481 | ], 482 | properties: { 483 | referer: { 484 | type: "string", 485 | title: "Referer", 486 | description: "URL to send as a Referer HTTP header", 487 | default: "", 488 | }, 489 | userAgent: { 490 | type: "string", 491 | title: "User Agent", 492 | description: "User Agent to send as a User-Agent HTTP header", 493 | default: "", 494 | }, 495 | headers: { 496 | type: "string", 497 | title: "Headers", 498 | description: "Headers to send with each request. Enter each header on a new line", 499 | default: "", 500 | } 501 | } 502 | }; 503 | 504 | const authSchema = { 505 | title: "", 506 | type: "object", 507 | required: [ 508 | ], 509 | properties: { 510 | httpUser: { 511 | type: "string", 512 | title: "HTTP Username", 513 | description: "For HTTP server authentication", 514 | default: "", 515 | }, 516 | httpPassword: { 517 | type: "string", 518 | title: "HTTP Username", 519 | description: "For HTTP server authentication", 520 | default: "", 521 | }, 522 | disableCookies: { 523 | type: "boolean", 524 | title: "Disable cookies", 525 | description: "", 526 | default: false, 527 | }, 528 | saveCookies: { 529 | type: "boolean", 530 | title: "Save cookies", 531 | description: "", 532 | default: false, 533 | }, 534 | saveSessionCookies: { 535 | type: "boolean", 536 | title: "Keep session cookies", 537 | description: "", 538 | default: false, 539 | }, 540 | cookiesFilePath: { 541 | type: "string", 542 | title: "Cookies file path", 543 | description: "Send cookies with each request, from a file", 544 | default: "" 545 | } 546 | } 547 | }; 548 | 549 | const advancedSchema = { 550 | title: "", 551 | type: "object", 552 | required: [ 553 | ], 554 | properties: { 555 | bindAddress: { 556 | type: "string", 557 | title: "Bind Address", 558 | description: "Bind to address (hostname or IP) on the local machine", 559 | default: "" 560 | }, 561 | fileNamesMode: { 562 | type: "string", 563 | title: "File names mode", 564 | description: "", 565 | enum: ["unix", "windows"], 566 | default: getFileNamesModeDefault() 567 | }, 568 | fileNamesForce: { 569 | type: "string", 570 | title: "Force file name format", 571 | description: "", 572 | enum: ["none", "lowercase", "uppercase"], 573 | default: "none" 574 | }, 575 | fileNamesNoControl: { 576 | type: "boolean", 577 | title: "Turn off escaping control characters in file names", 578 | description: "", 579 | default: false 580 | }, 581 | fileNamesAscii: { 582 | type: "boolean", 583 | title: "Escape any bytes outside of ASCII range in file names", 584 | description: "", 585 | default: false 586 | }, 587 | noDNSCache: { 588 | type: "boolean", 589 | title: "Disable DNS lookup cache", 590 | description: "Bind to address (hostname or IP) on the local machine", 591 | default: false 592 | }, 593 | retryConnectionRefused: { 594 | type: "boolean", 595 | title: "Retry when connection refused", 596 | description: "", 597 | default: false 598 | }, 599 | noHttpKeepAlive: { 600 | type: "boolean", 601 | title: "Disable HTTP keep-alive", 602 | description: "", 603 | default: false 604 | }, 605 | noCache: { 606 | type: "boolean", 607 | title: "Disable server-side cache", 608 | description: "", 609 | default: false 610 | }, 611 | ignoreContentLength: { 612 | type: "boolean", 613 | title: "Ignore content length", 614 | description: "", 615 | default: false 616 | }, 617 | noSSL: { 618 | type: "boolean", 619 | title: "Don't check SSL certificates", 620 | description: "", 621 | default: false 622 | }, 623 | ignoreRobots: { 624 | type: "boolean", 625 | title: "Ignore robots.txt", 626 | description: "", 627 | default: false 628 | }, 629 | keepIntegrityAttributes: { 630 | type: "boolean", 631 | title: "Keep integrity attributes (not recommended)", 632 | description: "Enabling this may cause site assets to not load properly in some situations", 633 | default: false 634 | }, 635 | wgetPath: { 636 | type: "string", 637 | title: "Custom path to wget binary", 638 | description: "Use your own version of wget instead of the bundled version", 639 | default: "" 640 | }, 641 | } 642 | }; 643 | 644 | return ( 645 |
    646 | 650 | 651 | Storage 652 | Filters 653 | Crawl 654 | Browser ID 655 | Auth/Cookies 656 | Advanced 657 | 658 | 659 | {/* Storage */} 660 | 661 | {storageForm} 662 | 663 | 664 | {/* Filters */} 665 | 666 | {filtersForm} 667 | 668 | 669 | {/* Crawl */} 670 | 671 | {crawlForm} 672 | 673 | 674 | {/* Browser ID */} 675 | 676 | {browserForm} 677 | 678 | 679 | {/* Auth\Cookies */} 680 | 681 | {authForm} 682 | 683 | 684 | {/* Advanced */} 685 | 686 | {advancedForm} 687 | 688 | 689 |
    690 | ); 691 | } 692 | 693 | export default SettingsTabs; -------------------------------------------------------------------------------- /src/partials/Sidebar.jsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect, useRef } from 'react'; 2 | import { NavLink, useLocation } from 'react-router-dom'; 3 | 4 | function Sidebar({ 5 | sidebarOpen, 6 | setSidebarOpen 7 | }) { 8 | 9 | const location = useLocation(); 10 | const { pathname } = location; 11 | 12 | const trigger = useRef(null); 13 | const sidebar = useRef(null); 14 | 15 | const storedSidebarExpanded = localStorage.getItem('sidebar-expanded'); 16 | const [sidebarExpanded, setSidebarExpanded] = useState(storedSidebarExpanded === null ? true : storedSidebarExpanded === 'true'); 17 | 18 | // close on click outside 19 | useEffect(() => { 20 | const clickHandler = ({ target }) => { 21 | if (!sidebar.current || !trigger.current) return; 22 | if (!sidebarOpen || sidebar.current.contains(target) || trigger.current.contains(target)) return; 23 | setSidebarOpen(false); 24 | }; 25 | document.addEventListener('click', clickHandler); 26 | return () => document.removeEventListener('click', clickHandler); 27 | }); 28 | 29 | // close if the esc key is pressed 30 | useEffect(() => { 31 | const keyHandler = ({ keyCode }) => { 32 | if (!sidebarOpen || keyCode !== 27) return; 33 | setSidebarOpen(false); 34 | }; 35 | document.addEventListener('keydown', keyHandler); 36 | return () => document.removeEventListener('keydown', keyHandler); 37 | }); 38 | 39 | useEffect(() => { 40 | localStorage.setItem('sidebar-expanded', sidebarExpanded); 41 | if (sidebarExpanded) { 42 | document.querySelector('body').classList.add('sidebar-expanded'); 43 | } else { 44 | document.querySelector('body').classList.remove('sidebar-expanded'); 45 | } 46 | }, [sidebarExpanded]); 47 | 48 | return ( 49 |
    50 | {/**/} 51 | 52 | {/* Sidebar backdrop (mobile only) */} 53 | 54 | 55 | {/* Sidebar */} 56 | 162 |
    163 | ); 164 | } 165 | 166 | export default Sidebar; -------------------------------------------------------------------------------- /src/partials/actions/DateSelect.jsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useRef, useEffect } from 'react'; 2 | import Transition from '../../utils/Transition'; 3 | 4 | function DateSelect() { 5 | 6 | const options = [ 7 | { 8 | id: 0, 9 | period: 'Today' 10 | }, 11 | { 12 | id: 1, 13 | period: 'Last 7 Days' 14 | }, 15 | { 16 | id: 2, 17 | period: 'Last Month' 18 | }, 19 | { 20 | id: 3, 21 | period: 'Last 12 Months' 22 | }, 23 | { 24 | id: 4, 25 | period: 'All Time' 26 | } 27 | ]; 28 | 29 | const [dropdownOpen, setDropdownOpen] = useState(false); 30 | const [selected, setSelected] = useState(2); 31 | 32 | const trigger = useRef(null); 33 | const dropdown = useRef(null); 34 | 35 | // close on click outside 36 | useEffect(() => { 37 | const clickHandler = ({ target }) => { 38 | if (!dropdownOpen || dropdown.current.contains(target) || trigger.current.contains(target)) return; 39 | setDropdownOpen(false); 40 | }; 41 | document.addEventListener('click', clickHandler); 42 | return () => document.removeEventListener('click', clickHandler); 43 | }); 44 | 45 | // close if the esc key is pressed 46 | useEffect(() => { 47 | const keyHandler = ({ keyCode }) => { 48 | if (!dropdownOpen || keyCode !== 27) return; 49 | setDropdownOpen(false); 50 | }; 51 | document.addEventListener('keydown', keyHandler); 52 | return () => document.removeEventListener('keydown', keyHandler); 53 | }); 54 | 55 | return ( 56 |
    57 | 75 | 86 |
      setDropdownOpen(true)} 90 | onBlur={() => setDropdownOpen(false)} 91 | > 92 | { 93 | options.map(option => { 94 | return ( 95 | 106 | ) 107 | }) 108 | } 109 |
    110 |     111 |
    112 | ); 113 | } 114 | 115 | export default DateSelect; 116 | -------------------------------------------------------------------------------- /src/partials/actions/Datepicker.jsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Flatpickr from 'react-flatpickr'; 3 | 4 | function Datepicker() { 5 | 6 | const options = { 7 | mode: 'range', 8 | static: true, 9 | monthSelectorType: 'static', 10 | dateFormat: 'M j, Y', 11 | defaultDate: [new Date().setDate(new Date().getDate() - 6), new Date()], 12 | prevArrow: '', 13 | nextArrow: '', 14 | onReady: (selectedDates, dateStr, instance) => { 15 | instance.element.value = dateStr.replace('to', '-'); 16 | }, 17 | onChange: (selectedDates, dateStr, instance) => { 18 | instance.element.value = dateStr.replace('to', '-'); 19 | }, 20 | } 21 | 22 | return ( 23 |
    24 | 25 |
    26 | 27 | 28 | 29 |
    30 |
    31 | ); 32 | } 33 | 34 | export default Datepicker; 35 | -------------------------------------------------------------------------------- /src/partials/actions/FilterButton.jsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useRef, useEffect } from 'react'; 2 | import Transition from '../../utils/Transition'; 3 | 4 | function FilterButton() { 5 | 6 | const [dropdownOpen, setDropdownOpen] = useState(false); 7 | 8 | const trigger = useRef(null); 9 | const dropdown = useRef(null); 10 | 11 | // close on click outside 12 | useEffect(() => { 13 | const clickHandler = ({ target }) => { 14 | if (!dropdownOpen || dropdown.current.contains(target) || trigger.current.contains(target)) return; 15 | setDropdownOpen(false); 16 | }; 17 | document.addEventListener('click', clickHandler); 18 | return () => document.removeEventListener('click', clickHandler); 19 | }); 20 | 21 | // close if the esc key is pressed 22 | useEffect(() => { 23 | const keyHandler = ({ keyCode }) => { 24 | if (!dropdownOpen || keyCode !== 27) return; 25 | setDropdownOpen(false); 26 | }; 27 | document.addEventListener('keydown', keyHandler); 28 | return () => document.removeEventListener('keydown', keyHandler); 29 | }); 30 | 31 | return ( 32 |
    33 | 45 | 56 |
    57 |
    Filters
    58 |
      59 |
    • 60 | 64 |
      • 65 |
    • 66 | 70 |
      • 71 |
    • 72 | 76 |
      • 77 |
    • 78 | 82 |
      • 83 |
    • 84 | 88 |
      • 89 |
    • 90 | 94 |
      • 95 |
    96 |
    97 |
      98 |
    • 99 | 100 |
      • 101 |
    • 102 | 103 |
      • 104 |
    105 |
    106 |
    107 |     108 |
    109 | ); 110 | } 111 | 112 | export default FilterButton; 113 | -------------------------------------------------------------------------------- /src/partials/header/Help.jsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useRef, useEffect } from 'react'; 2 | import { Link } from 'react-router-dom'; 3 | import Transition from '../../utils/Transition'; 4 | 5 | function Help() { 6 | 7 | const [dropdownOpen, setDropdownOpen] = useState(false); 8 | 9 | const trigger = useRef(null); 10 | const dropdown = useRef(null); 11 | 12 | // close on click outside 13 | useEffect(() => { 14 | const clickHandler = ({ target }) => { 15 | if (!dropdownOpen || dropdown.current.contains(target) || trigger.current.contains(target)) return; 16 | setDropdownOpen(false); 17 | }; 18 | document.addEventListener('click', clickHandler); 19 | return () => document.removeEventListener('click', clickHandler); 20 | }); 21 | 22 | // close if the esc key is pressed 23 | useEffect(() => { 24 | const keyHandler = ({ keyCode }) => { 25 | if (!dropdownOpen || keyCode !== 27) return; 26 | setDropdownOpen(false); 27 | }; 28 | document.addEventListener('keydown', keyHandler); 29 | return () => document.removeEventListener('keydown', keyHandler); 30 | }); 31 | 32 | return ( 33 |
    34 | 46 | 47 | 57 |
    setDropdownOpen(true)} 60 | onBlur={() => setDropdownOpen(false)} 61 | > 62 |
    Need help?
    63 |
      64 |
    • 65 | setDropdownOpen(!dropdownOpen)} 69 | > 70 | 71 | 72 | 73 | Help Center 74 | 75 |
      • 76 |
    • 77 | 81 | 82 | 83 | 84 | Email us 85 | 86 |
      setDropdownOpen(!dropdownOpen)} 89 | > 90 | Version 0.1.4 91 |
      92 |
      • 93 |
    94 |
    95 |     96 |
    97 | ) 98 | } 99 | 100 | export default Help; -------------------------------------------------------------------------------- /src/utils/GetStatusBadge.jsx: -------------------------------------------------------------------------------- 1 | import React, { useState } from 'react'; 2 | 3 | function GetStatusBadge( 4 | status, 5 | size = 'small' 6 | ) { 7 | let classes = 'status-badge text-sm font-semibold text-white rounded-full'; 8 | if (size === 'large') { 9 | classes += ' px-3 py-3'; 10 | } 11 | else { 12 | classes += ' px-1.5 py-1.5'; 13 | } 14 | if (status === 0) { 15 | return ( 16 |
    New
    17 | ); 18 | } 19 | 20 | if (status === 1) { 21 | return ( 22 |
    In Progress
    23 | ); 24 | } 25 | 26 | if (status === 2) { 27 | return ( 28 |
    Completed
    29 | ); 30 | } 31 | 32 | if (status === 3) { 33 | return ( 34 |
    Canceled
    35 | ); 36 | } 37 | 38 | if (status === 4) { 39 | return ( 40 |
    Total
    41 | ); 42 | } 43 | 44 | // -1 45 | return ( 46 |
    Error
    47 | ); 48 | } 49 | 50 | export default GetStatusBadge; -------------------------------------------------------------------------------- /src/utils/Transition.jsx: -------------------------------------------------------------------------------- 1 | import React, { useRef, useEffect, useContext } from 'react'; 2 | import { CSSTransition as ReactCSSTransition } from 'react-transition-group'; 3 | 4 | const TransitionContext = React.createContext({ 5 | parent: {}, 6 | }) 7 | 8 | function useIsInitialRender() { 9 | const isInitialRender = useRef(true); 10 | useEffect(() => { 11 | isInitialRender.current = false; 12 | }, []) 13 | return isInitialRender.current; 14 | } 15 | 16 | function CSSTransition({ 17 | show, 18 | enter = '', 19 | enterStart = '', 20 | enterEnd = '', 21 | leave = '', 22 | leaveStart = '', 23 | leaveEnd = '', 24 | appear, 25 | unmountOnExit, 26 | tag = 'div', 27 | children, 28 | ...rest 29 | }) { 30 | const enterClasses = enter.split(' ').filter((s) => s.length); 31 | const enterStartClasses = enterStart.split(' ').filter((s) => s.length); 32 | const enterEndClasses = enterEnd.split(' ').filter((s) => s.length); 33 | const leaveClasses = leave.split(' ').filter((s) => s.length); 34 | const leaveStartClasses = leaveStart.split(' ').filter((s) => s.length); 35 | const leaveEndClasses = leaveEnd.split(' ').filter((s) => s.length); 36 | const removeFromDom = unmountOnExit; 37 | 38 | function addClasses(node, classes) { 39 | classes.length && node.classList.add(...classes); 40 | } 41 | 42 | function removeClasses(node, classes) { 43 | classes.length && node.classList.remove(...classes); 44 | } 45 | 46 | const nodeRef = React.useRef(null); 47 | const Component = tag; 48 | 49 | return ( 50 | { 56 | nodeRef.current.addEventListener('transitionend', done, false) 57 | }} 58 | onEnter={() => { 59 | if (!removeFromDom) nodeRef.current.style.display = null; 60 | addClasses(nodeRef.current, [...enterClasses, ...enterStartClasses]) 61 | }} 62 | onEntering={() => { 63 | removeClasses(nodeRef.current, enterStartClasses) 64 | addClasses(nodeRef.current, enterEndClasses) 65 | }} 66 | onEntered={() => { 67 | removeClasses(nodeRef.current, [...enterEndClasses, ...enterClasses]) 68 | }} 69 | onExit={() => { 70 | addClasses(nodeRef.current, [...leaveClasses, ...leaveStartClasses]) 71 | }} 72 | onExiting={() => { 73 | removeClasses(nodeRef.current, leaveStartClasses) 74 | addClasses(nodeRef.current, leaveEndClasses) 75 | }} 76 | onExited={() => { 77 | removeClasses(nodeRef.current, [...leaveEndClasses, ...leaveClasses]) 78 | if (!removeFromDom) nodeRef.current.style.display = 'none'; 79 | }} 80 | > 81 | {children} 82 | 83 | ) 84 | } 85 | 86 | function Transition({ show, appear, ...rest }) { 87 | const { parent } = useContext(TransitionContext); 88 | const isInitialRender = useIsInitialRender(); 89 | const isChild = show === undefined; 90 | 91 | if (isChild) { 92 | return ( 93 | 98 | ) 99 | } 100 | 101 | return ( 102 | 111 | 112 | 113 | ) 114 | } 115 | 116 | export default Transition; -------------------------------------------------------------------------------- /src/utils/Utils.js: -------------------------------------------------------------------------------- 1 | const moment = require('moment'); 2 | import { app } from "electron"; 3 | import parseUrl from 'parse-url'; 4 | var sanitize = require("sanitize-filename"); 5 | 6 | export const hexToRGB = (h) => { 7 | let r = 0; 8 | let g = 0; 9 | let b = 0; 10 | if (h.length === 4) { 11 | r = `0x${h[1]}${h[1]}`; 12 | g = `0x${h[2]}${h[2]}`; 13 | b = `0x${h[3]}${h[3]}`; 14 | } else if (h.length === 7) { 15 | r = `0x${h[1]}${h[2]}`; 16 | g = `0x${h[3]}${h[4]}`; 17 | b = `0x${h[5]}${h[6]}`; 18 | } 19 | return `${+r},${+g},${+b}`; 20 | }; 21 | 22 | export const formatValue = (value) => Intl.NumberFormat('en-US', { 23 | style: 'currency', 24 | currency: 'USD', 25 | maximumSignificantDigits: 3, 26 | notation: 'compact', 27 | }).format(value); 28 | 29 | export const getTimeElapsedFunc = (project) => { 30 | let endDate = moment(); 31 | if (project.status > 1) { 32 | endDate = moment(project.completed); 33 | } 34 | let timeElapsed = ''; 35 | let unit = 'secs'; 36 | timeElapsed = endDate.diff(project.started, 'seconds'); 37 | if (timeElapsed === 1) { 38 | unit = 'sec'; 39 | } 40 | if (timeElapsed > 59) { 41 | unit = 'mins'; 42 | timeElapsed = endDate.diff(project.started, 'minutes'); 43 | if (timeElapsed === 1) { 44 | unit = 'min'; 45 | } 46 | if (timeElapsed > 59) { 47 | unit = 'hours'; 48 | timeElapsed = endDate.diff(project.started, 'hours'); 49 | if (timeElapsed === 1) { 50 | unit = 'hour'; 51 | } 52 | } 53 | } 54 | timeElapsed = timeElapsed + ' ' + unit; 55 | return timeElapsed; 56 | } 57 | 58 | export const getBasePath = (project, options) => { 59 | let name = sanitize(project.name) || 'default'; 60 | let path = app.getPath('userData') + '/downloads/' + name; 61 | if (options.basePath) { 62 | path = options.basePath + '/' + name; 63 | } 64 | return path; 65 | } 66 | 67 | export const getFolderFromProject = (project) => { 68 | let p = null; 69 | if (project && project.base_path && project.url) { 70 | p = project.base_path; 71 | if (!project.no_directories) { 72 | p += '/'; 73 | let parsed = parseUrl(project.url); 74 | p += parsed.resource || parsed.pathname; 75 | } 76 | } 77 | return p; 78 | } -------------------------------------------------------------------------------- /src/wget/index.js: -------------------------------------------------------------------------------- 1 | const { app, powerSaveBlocker } = require('electron'); 2 | const parseUrl = require("parse-url"); 3 | var spawn = require('child_process').spawn; 4 | var dir = require('node-dir'); 5 | const { default: parse } = require('node-html-parser'); 6 | const fs = require('fs'); 7 | const { getBasePath } = require('../utils/Utils'); 8 | const { setProjectError, getProject, setProjectCompleted } = require('../db'); 9 | const { wgetExecPath } = require('../binaries'); 10 | 11 | function getCommandArgs(project, options) { 12 | let path = project.base_path; 13 | 14 | let args = [ 15 | '--directory-prefix=' + path, 16 | '--adjust-extension', 17 | '--convert-links', 18 | '--page-requisites', 19 | '--no-parent', 20 | '--no-remove-listing', 21 | ]; 22 | 23 | const optionsConfig = { 24 | 'noRecursive': { 25 | 'flag': '--recursive', 26 | }, 27 | 'defaultPageName': { 28 | 'flag': '--default-page', 29 | 'includeValue': true 30 | }, 31 | 'numRetries': { 32 | 'flag': '--tries', 33 | 'includeValue': true, 34 | 'min': 0 35 | }, 36 | 'limitDownloadSpeed': { 37 | 'flag': '--limit-rate', 38 | 'includeValue': true 39 | }, 40 | 'waitTime': { 41 | 'flag': '--wait', 42 | 'includeValue': true, 43 | 'min': 0 44 | }, 45 | 'randomWait': { 46 | 'flag': '--random-wait' 47 | }, 48 | 'waitRetryTime': { 49 | 'flag': '--waitretry', 50 | 'includeValue': true, 51 | 'min': 0 52 | }, 53 | 'downloadQuota': { 54 | 'flag': '--quota', 55 | 'includeValue': true 56 | }, 57 | 'maxRedirects': { 58 | 'flag': '--max-redirect', 59 | 'includeValue': true, 60 | 'min': 0 61 | }, 62 | 'referer': { 63 | 'flag': '--referer', 64 | 'includeValue': true 65 | }, 66 | 'userAgent': { 67 | 'flag': '--user-agent', 68 | 'includeValue': true 69 | }, 70 | 'httpUser': { 71 | 'flag': '--user', 72 | 'includeValue': true 73 | }, 74 | 'httpPassword': { 75 | 'flag': '--password', 76 | 'includeValue': true 77 | }, 78 | 'disableCookies': { 79 | 'flag': '--no-cookies', 80 | 'includeValue': true 81 | }, 82 | 'bindAddress': { 83 | 'flag': '--bind-address', 84 | 'includeValue': true 85 | }, 86 | 'noDNSCache': { 87 | 'flag': '--no-dns-cache' 88 | }, 89 | 'retryConnectionRefused': { 90 | 'flag': '--retry-connrefused' 91 | }, 92 | 'noHttpKeepAlive': { 93 | 'flag': '--no-http-keep-alive' 94 | }, 95 | 'noCache': { 96 | 'flag': '--no-cache' 97 | }, 98 | 'ignoreContentLength': { 99 | 'flag': '--ignore-length' 100 | }, 101 | 'noSSL': { 102 | 'flag': '--no-check-certificate' 103 | }, 104 | 'ignoreCase': { 105 | 'flag': '--ignore-case' 106 | }, 107 | 'noDirectories': { 108 | 'flag': '--no-directories' 109 | } 110 | }; 111 | 112 | // set these flags 113 | 114 | for (var key in optionsConfig) { 115 | if (!optionsConfig.hasOwnProperty(key)) { 116 | continue; 117 | } 118 | if (typeof options[key] !== 'undefined' && options[key]) { 119 | if (optionsConfig[key].includeValue) { 120 | let val = options[key]; 121 | if (typeof optionsConfig[key].min !== 'undefined') { 122 | if (val < optionsConfig[key].min) { 123 | val = optionsConfig[key].min; 124 | } 125 | } 126 | let add = true; 127 | if (typeof val === 'string') { 128 | if (!val.trim()) { 129 | add = false; 130 | } 131 | } 132 | if (add) { 133 | args.push(optionsConfig[key].flag + '=' + val); 134 | } 135 | } 136 | else { 137 | args.push(optionsConfig[key].flag); 138 | } 139 | } 140 | } 141 | 142 | // set the rest of the flags that aren't as simple 143 | 144 | // --level 145 | if (typeof options.recursiveDepthLevel !== 'undefined') { 146 | if (options.recursiveDepthLevel < 0) { 147 | args.push('--level=inf'); 148 | } 149 | else { 150 | args.push('--level=' + options.recursiveDepthLevel); 151 | } 152 | } 153 | 154 | // --save-cookies 155 | if (typeof options.saveCookies !== 'undefined' && options.saveCookies) { 156 | args.push('--save-cookies'); 157 | let saveCookiesPath = getBasePath(project, options); 158 | saveCookiesPath += '/cookies.txt'; 159 | args.push(saveCookiesPath); 160 | } 161 | 162 | // --keep-session-cookies 163 | if (typeof options.saveSessionCookies !== 'undefined' && options.saveSessionCookies) { 164 | args.push('--keep-session-cookies=on'); 165 | } 166 | 167 | // --load-cookies 168 | if (typeof options.cookiesFilePath !== 'undefined' && options.cookiesFilePath) { 169 | args.push('--load-cookies'); 170 | args.push(options.cookiesFilePath); 171 | } 172 | 173 | // --dns-timeout, --connect-timeout, --read-timeout 174 | if (typeof options.timeout !== 'undefined' && options.timeout) { 175 | if (typeof options.timeout.dnsTimeout !== 'undefined') { 176 | args.push('--dns-timeout=' + options.timeout.dnsTimeout); 177 | } 178 | if (typeof options.timeout.connectTimeout !== 'undefined') { 179 | args.push('--connect-timeout=' + options.timeout.connectTimeout); 180 | } 181 | if (typeof options.timeout.readTimeout !== 'undefined') { 182 | args.push('--read-timeout=' + options.timeout.readTimeout); 183 | } 184 | } 185 | 186 | // --restrict-file-names 187 | let fileNameModes = ''; 188 | if (typeof options.fileNamesMode !== 'undefined' && options.fileNamesMode) { 189 | fileNameModes += options.fileNamesMode; 190 | } 191 | if (typeof options.fileNamesForce !== 'undefined' && options.fileNamesForce && options.fileNamesForce !== 'none') { 192 | if (fileNameModes.length) { 193 | fileNameModes += ','; 194 | } 195 | fileNameModes += options.fileNamesForce; 196 | } 197 | if (typeof options.fileNamesNoControl !== 'undefined' && options.fileNamesNoControl) { 198 | if (fileNameModes.length) { 199 | fileNameModes += ','; 200 | } 201 | fileNameModes += "nocontrol"; 202 | } 203 | if (typeof options.fileNamesAscii !== 'undefined' && options.fileNamesAscii) { 204 | if (fileNameModes.length) { 205 | fileNameModes += ','; 206 | } 207 | fileNameModes += "ascii"; 208 | } 209 | if (fileNameModes.length) { 210 | args.push('--restrict-file-names=' + fileNameModes); 211 | } 212 | 213 | // --ignore-robots 214 | if (typeof options.ignoreRobots !== 'undefined' && options.ignoreRobots) { 215 | args.push('-e'); 216 | args.push('robots=off'); 217 | } 218 | 219 | // --accept 220 | if (typeof options.acceptFilters !== 'undefined' && options.acceptFilters) { 221 | let filtersStr = ''; 222 | let filtersArray = options.acceptFilters.split('\n'); 223 | for (let i = 0; i < filtersArray.length; i++) { 224 | let f = filtersArray[i].trim(); 225 | if (!f.length) { 226 | continue; 227 | } 228 | if (filtersStr.length) { 229 | filtersStr += ','; 230 | } 231 | filtersStr += f; 232 | } 233 | if (filtersStr.length) { 234 | args.push('--accept'); 235 | args.push(filtersStr); 236 | } 237 | } 238 | 239 | // --reject 240 | if (typeof options.rejectFilters !== 'undefined' && options.rejectFilters) { 241 | let filtersStr = ''; 242 | let filtersArray = options.rejectFilters.split('\n'); 243 | for (let i = 0; i < filtersArray.length; i++) { 244 | let f = filtersArray[i].trim(); 245 | if (!f.length) { 246 | continue; 247 | } 248 | if (filtersStr.length) { 249 | filtersStr += ','; 250 | } 251 | filtersStr += f; 252 | } 253 | if (filtersStr.length) { 254 | args.push('--reject'); 255 | args.push(filtersStr); 256 | } 257 | } 258 | 259 | // --include-directories 260 | if (typeof options.includeDirs !== 'undefined' && options.includeDirs) { 261 | let filtersStr = ''; 262 | let filtersArray = options.includeDirs.split('\n'); 263 | for (let i = 0; i < filtersArray.length; i++) { 264 | let f = filtersArray[i].trim(); 265 | if (!f.length) { 266 | continue; 267 | } 268 | if (filtersStr.length) { 269 | filtersStr += ','; 270 | } 271 | filtersStr += f; 272 | } 273 | if (filtersStr.length) { 274 | args.push('--include-directories=' + filtersStr); 275 | } 276 | } 277 | 278 | // --exclude-directories 279 | if (typeof options.excludeDirs !== 'undefined' && options.excludeDirs) { 280 | let filtersStr = ''; 281 | let filtersArray = options.excludeDirs.split('\n'); 282 | for (let i = 0; i < filtersArray.length; i++) { 283 | let f = filtersArray[i].trim(); 284 | if (!f.length) { 285 | continue; 286 | } 287 | if (filtersStr.length) { 288 | filtersStr += ','; 289 | } 290 | filtersStr += f; 291 | } 292 | if (filtersStr.length) { 293 | args.push('--exclude-directories=' + filtersStr); 294 | } 295 | } 296 | 297 | function getDomainFromURL(url) { 298 | let parsed = parseUrl(url); 299 | return parsed.resource || parsed.pathname; 300 | } 301 | 302 | // --domains 303 | if (typeof options.domains !== 'undefined' && options.domains) { 304 | let domainsArray = options.domains.split('\n'); 305 | let domainString = getDomainFromURL(project.url); 306 | 307 | for (let i = 0; i < domainsArray.length; i++) { 308 | let d = domainsArray[i].trim(); 309 | if (!d.length) { 310 | continue; 311 | } 312 | domainString = domainString + ','; 313 | domainString += getDomainFromURL(d); 314 | } 315 | 316 | args.push('--domains=' + domainString); 317 | 318 | args.push('--span-hosts'); 319 | } 320 | 321 | // --header 322 | if (typeof options.headers !== 'undefined' && options.headers) { 323 | let headersArray = options.headers.split('\n'); 324 | 325 | for (let i = 0; i < headersArray.length; i++) { 326 | let h = headersArray[i].trim(); 327 | if (!h.length) { 328 | continue; 329 | } 330 | args.push('--header=' + h); 331 | } 332 | } 333 | 334 | // base url 335 | args.push(project.url); 336 | 337 | return args; 338 | } 339 | 340 | function fixFiles(project, options) { 341 | let p = project.base_path; 342 | p += '/'; 343 | let parsed = parseUrl(project.url); 344 | p += parsed.resource || parsed.pathname; 345 | try { 346 | var filenames = dir.files(p, { 347 | sync: true, 348 | recursive: true, 349 | match: /.html?$/, 350 | }); 351 | for (let i = 0; i < filenames.length; i++) { 352 | let filename = filenames[i]; 353 | if (filename.match(/.html?$/gi) === null) { 354 | continue; 355 | } 356 | const data = fs.readFileSync(filename, 'utf8'); 357 | const root = parse(data); 358 | let changed = false; 359 | 360 | // remove 'integrity' attribute 361 | if (!(typeof options.keepIntegrityAttributes !== 'undefined' && options.keepIntegrityAttributes)) { 362 | let integrityEls = root.querySelectorAll('[integrity]'); 363 | for (let j = 0; j < integrityEls.length; j++) { 364 | integrityEls[j].removeAttribute('integrity'); 365 | changed = true; 366 | } 367 | } 368 | 369 | if (changed) { 370 | fs.writeFileSync(filename, root.toString()); 371 | } 372 | } 373 | } catch (err) { 374 | 375 | } 376 | } 377 | 378 | module.exports = (mainWindow, store, project, options, timeElapsedIntervalId, powerSaveBlockerId) => { 379 | function handleError(err) { 380 | setProjectError(project.id, err); 381 | const p = getProject(project.id); 382 | mainWindow.webContents.send("clone-finished", p); 383 | clearInterval(timeElapsedIntervalId); 384 | } 385 | 386 | var cmd; 387 | try { 388 | let wgetPath = wgetExecPath; 389 | if (typeof options.wgetPath !== 'undefined' && options.wgetPath.trim()) { 390 | wgetPath = options.wgetPath.trim(); 391 | } 392 | cmd = spawn(wgetPath, getCommandArgs(project, options)); 393 | } catch(err) { 394 | clearInterval(timeElapsedIntervalId); 395 | handleError(err); 396 | return; 397 | } 398 | 399 | cmd.on('error', function(err) { 400 | clearInterval(timeElapsedIntervalId); 401 | powerSaveBlocker.stop(powerSaveBlockerId); 402 | handleError(err); 403 | }); 404 | 405 | cmd.stdout.on('data', function (data) { 406 | mainWindow.webContents.send("clone-data", project.id, data.toString()); 407 | }); 408 | 409 | cmd.stderr.on('data', function (data) { 410 | mainWindow.webContents.send("clone-data", project.id, data.toString()); 411 | }); 412 | 413 | cmd.on('exit', function (code) { 414 | clearInterval(timeElapsedIntervalId); 415 | powerSaveBlocker.stop(powerSaveBlockerId); 416 | if (code === null) { 417 | return; 418 | } 419 | 420 | if ( 421 | code === 0 || 422 | code === 8 // 8 is WGET_EXIT_SERVER_ERROR but it happens sometimes when there's no error 423 | ) { 424 | fixFiles(project, options); 425 | setProjectCompleted(project.id); 426 | const p = getProject(project.id); 427 | mainWindow.webContents.send("clone-finished", p); 428 | } 429 | else { 430 | handleError(code.toString()); 431 | } 432 | }); 433 | 434 | return cmd.pid; 435 | } -------------------------------------------------------------------------------- /vite.config.js: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite' 2 | import postcss from './postcss.config.js' 3 | import react from '@vitejs/plugin-react' 4 | 5 | // https://vitejs.dev/config/ 6 | export default defineConfig({ 7 | define: { 8 | 'process.env': process.env 9 | }, 10 | css: { 11 | postcss, 12 | }, 13 | plugins: [react()], 14 | resolve: { 15 | alias: [ 16 | { 17 | find: /^~.+/, 18 | replacement: (val) => { 19 | return val.replace(/^~/, ""); 20 | }, 21 | }, 22 | ], 23 | }, 24 | build: { 25 | commonjsOptions: { 26 | transformMixedEsModules: true, 27 | } 28 | } 29 | }) 30 | --------------------------------------------------------------------------------