├── public ├── icon.png ├── favicon.ico ├── fflags.png ├── flagSprite42.png ├── icon-512x512.png ├── manifest.json └── vercel.svg ├── entrypoint.sh ├── postcss.config.js ├── utils ├── client │ ├── helpers.ts │ ├── validators.ts │ ├── IdeasSortFilter.ts │ ├── generateChartData.ts │ ├── exportcsv.ts │ └── SCsortFilter.ts ├── parseKeywords.ts ├── verifyUser.ts └── domains.ts ├── .dockerignore ├── .env.example ├── .sequelizerc ├── next.config.js ├── __tests__ ├── components │ ├── Icon.test.tsx │ ├── Topbar.test.tsx │ ├── Sidebar.test.tsx │ ├── Modal.test.tsx │ ├── DomainItem.test.tsx │ └── Keyword.test.tsx ├── hooks │ └── domains.test.tsx └── pages │ ├── index.test.tsx │ └── domains.test.tsx ├── hooks ├── useIsMobile.tsx ├── useWindowResize.tsx └── useOnKey.tsx ├── services ├── misc.tsx ├── searchConsole.ts ├── settings.ts └── adwords.tsx ├── database ├── config.js ├── database.ts ├── migrations │ ├── 1707233039698-add-domain-searchconsole-field.js │ ├── 1709217223856-add-keyword-volume-field copy.js │ └── 1707068556345-add-new-keyword-fields.js └── models │ ├── domain.ts │ └── keyword.ts ├── .gitignore ├── .stylelintrc.json ├── tailwind.config.js ├── pages ├── _document.tsx ├── _app.tsx ├── api │ ├── logout.ts │ ├── clearfailed.ts │ ├── keyword.ts │ ├── login.ts │ ├── cron.ts │ ├── domain.ts │ ├── dbmigrate.ts │ ├── volume.ts │ ├── searchconsole.ts │ ├── insight.ts │ ├── notify.ts │ ├── ideas.ts │ ├── refresh.ts │ └── adwords.ts ├── index.tsx ├── domain │ └── insight │ │ └── [slug] │ │ └── index.tsx └── login │ └── index.tsx ├── components ├── keywords │ ├── KeywordPosition.tsx │ ├── KeywordTagManager.tsx │ ├── SCKeyword.tsx │ └── AddTags.tsx ├── common │ ├── InputField.tsx │ ├── Chart.tsx │ ├── ToggleField.tsx │ ├── Footer.tsx │ ├── SecretField.tsx │ ├── Modal.tsx │ ├── SidePanel.tsx │ ├── ChartSlim.tsx │ ├── Sidebar.tsx │ ├── TopBar.tsx │ └── SelectField.tsx ├── settings │ ├── SearchConsoleSettings.tsx │ ├── IntegrationSettings.tsx │ ├── Changelog.tsx │ └── NotificationSettings.tsx ├── insight │ ├── InsightItem.tsx │ └── InsightStats.tsx ├── domains │ ├── AddDomain.tsx │ └── DomainItem.tsx └── ideas │ └── KeywordIdea.tsx ├── scrapers ├── index.ts └── services │ ├── scrapingrobot.ts │ ├── scrapingant.ts │ ├── serper.ts │ ├── serpapi.ts │ ├── searchapi.ts │ ├── proxy.ts │ ├── spaceserp.ts │ ├── hasdata.ts │ ├── serply.ts │ └── valueserp.ts ├── tsconfig.json ├── .eslintrc.json ├── jest.config.js ├── jest.setup.js ├── styles └── changelog.css ├── LICENSE ├── __mocks__ ├── utils.tsx └── data.ts ├── Dockerfile ├── package.json └── README.md /public/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/towfiqi/serpbear/HEAD/public/icon.png -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | npx sequelize-cli db:migrate --env production 3 | exec "$@" -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/towfiqi/serpbear/HEAD/public/favicon.ico -------------------------------------------------------------------------------- /public/fflags.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/towfiqi/serpbear/HEAD/public/fflags.png -------------------------------------------------------------------------------- /public/flagSprite42.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/towfiqi/serpbear/HEAD/public/flagSprite42.png -------------------------------------------------------------------------------- /public/icon-512x512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/towfiqi/serpbear/HEAD/public/icon-512x512.png -------------------------------------------------------------------------------- /postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | }; 7 | -------------------------------------------------------------------------------- /utils/client/helpers.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/prefer-default-export */ 2 | export const formattedNum = (num:number) => new Intl.NumberFormat('en-IN', { maximumSignificantDigits: 3 }).format(num); 3 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | 2 | Dockerfile 3 | .dockerignore 4 | node_modules 5 | npm-debug.log 6 | README.md 7 | .next 8 | .git 9 | .vscode 10 | fly.toml 11 | TODO 12 | data/database.sqlite 13 | data/settings.json 14 | data/failed_queue.json -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | USER=admin 2 | PASSWORD=0123456789 3 | SECRET=4715aed3216f7b0a38e6b534a958362654e96d10fbc04700770d572af3dce43625dd 4 | APIKEY=5saedXklbslhnapihe2pihp3pih4fdnakhjwq5 5 | SESSION_DURATION=24 6 | NEXT_PUBLIC_APP_URL=http://localhost:3000 -------------------------------------------------------------------------------- /.sequelizerc: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = { 4 | 'config': path.resolve('database', 'config.js'), 5 | 'models-path': path.resolve('database', 'models'), 6 | 'seeders-path': path.resolve('database', 'seeders'), 7 | 'migrations-path': path.resolve('database', 'migrations') 8 | }; -------------------------------------------------------------------------------- /next.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | const { version } = require('./package.json'); 3 | 4 | const nextConfig = { 5 | reactStrictMode: true, 6 | swcMinify: false, 7 | output: 'standalone', 8 | publicRuntimeConfig: { 9 | version, 10 | }, 11 | }; 12 | 13 | module.exports = nextConfig; 14 | -------------------------------------------------------------------------------- /__tests__/components/Icon.test.tsx: -------------------------------------------------------------------------------- 1 | import { render } from '@testing-library/react'; 2 | import Icon from '../../components/common/Icon'; 3 | 4 | describe('Icon Component', () => { 5 | it('renders without crashing', async () => { 6 | render(); 7 | expect(document.querySelector('svg')).toBeInTheDocument(); 8 | }); 9 | }); 10 | -------------------------------------------------------------------------------- /hooks/useIsMobile.tsx: -------------------------------------------------------------------------------- 1 | import { useEffect, useState } from 'react'; 2 | 3 | const useIsMobile = () => { 4 | const [isMobile, setIsMobile] = useState(false); 5 | useEffect(() => { 6 | setIsMobile(!!(window.matchMedia('only screen and (max-width: 760px)').matches)); 7 | }, []); 8 | 9 | return [isMobile]; 10 | }; 11 | 12 | export default useIsMobile; 13 | -------------------------------------------------------------------------------- /services/misc.tsx: -------------------------------------------------------------------------------- 1 | import { useQuery } from 'react-query'; 2 | 3 | export async function fetchChangelog() { 4 | const res = await fetch('https://api.github.com/repos/towfiqi/serpbear/releases', { method: 'GET' }); 5 | return res.json(); 6 | } 7 | 8 | export function useFetchChangelog() { 9 | return useQuery('changelog', () => fetchChangelog(), { cacheTime: 60 * 60 * 1000 }); 10 | } 11 | -------------------------------------------------------------------------------- /hooks/useWindowResize.tsx: -------------------------------------------------------------------------------- 1 | import { useEffect } from 'react'; 2 | 3 | const useWindowResize = (onResize: () => void) => { 4 | useEffect(() => { 5 | onResize(); 6 | window.addEventListener('resize', onResize); 7 | return () => { 8 | window.removeEventListener('resize', onResize); 9 | }; 10 | }, [onResize]); 11 | }; 12 | 13 | export default useWindowResize; 14 | -------------------------------------------------------------------------------- /database/config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | production: { 3 | username: process.env.USER_NAME ? process.env.USER_NAME : process.env.USER, 4 | password: process.env.PASSWORD, 5 | database: 'sequelize', 6 | host: 'database', 7 | port: 3306, 8 | dialect: 'sqlite', 9 | storage: './data/database.sqlite', 10 | dialectOptions: { 11 | bigNumberStrings: true, 12 | }, 13 | }, 14 | }; 15 | -------------------------------------------------------------------------------- /public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "SerpBear", 3 | "short_name": "SerpBear", 4 | "icons": [ 5 | { 6 | "src": "/icon.png", 7 | "sizes": "192x192", 8 | "type": "image/png" 9 | }, 10 | 11 | { 12 | "src": "/icon-512x512.png", 13 | "sizes": "512x512", 14 | "type": "image/png" 15 | } 16 | ], 17 | "theme_color": "#1d4ed8", 18 | "background_color": "#FFFFFF", 19 | "start_url": "/", 20 | "display": "standalone", 21 | "orientation": "portrait" 22 | } -------------------------------------------------------------------------------- /hooks/useOnKey.tsx: -------------------------------------------------------------------------------- 1 | import { useEffect } from 'react'; 2 | 3 | const useOnKey = (key:string, onPress: Function) => { 4 | useEffect(() => { 5 | const closeModalonEsc = (event:KeyboardEvent) => { 6 | if (event.key === key) { 7 | onPress(); 8 | } 9 | }; 10 | window.addEventListener('keydown', closeModalonEsc, false); 11 | return () => { 12 | window.removeEventListener('keydown', closeModalonEsc, false); 13 | }; 14 | }, [key, onPress]); 15 | }; 16 | 17 | export default useOnKey; 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # dependencies 2 | /node_modules 3 | /.pnp 4 | .pnp.js 5 | 6 | # testing 7 | /coverage 8 | /.swc/ 9 | 10 | # next.js 11 | /.next/ 12 | /out/ 13 | 14 | # production 15 | /build 16 | 17 | # misc 18 | .DS_Store 19 | *.pem 20 | .vscode 21 | fly.toml 22 | 23 | # debug 24 | npm-debug.log* 25 | yarn-debug.log* 26 | yarn-error.log* 27 | .pnpm-debug.log* 28 | 29 | # local env files 30 | .env*.local 31 | 32 | # vercel 33 | .vercel 34 | 35 | # typescript 36 | *.tsbuildinfo 37 | next-env.d.ts 38 | 39 | 40 | #todo 41 | TODO 42 | 43 | #database 44 | data/ -------------------------------------------------------------------------------- /__tests__/components/Topbar.test.tsx: -------------------------------------------------------------------------------- 1 | import { render, screen } from '@testing-library/react'; 2 | import TopBar from '../../components/common/TopBar'; 3 | 4 | jest.mock('next/router', () => ({ 5 | useRouter: () => ({ 6 | pathname: '/', 7 | }), 8 | })); 9 | 10 | describe('TopBar Component', () => { 11 | it('renders without crashing', async () => { 12 | render(); 13 | expect( 14 | await screen.findByText('SerpBear'), 15 | ).toBeInTheDocument(); 16 | }); 17 | }); 18 | -------------------------------------------------------------------------------- /.stylelintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "stylelint-config-standard", 3 | "rules": { 4 | "indentation": 3, 5 | "declaration-block-single-line-max-declarations": 2, 6 | "selector-class-pattern":null, 7 | "at-rule-no-unknown": [ 8 | true, 9 | { 10 | "ignoreAtRules": [ 11 | "tailwind", 12 | "apply", 13 | "variants", 14 | "responsive", 15 | "screen" 16 | ] 17 | } 18 | ], 19 | "declaration-block-trailing-semicolon": null, 20 | "no-descending-specificity": null 21 | } 22 | } -------------------------------------------------------------------------------- /tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | module.exports = { 3 | purge: { 4 | content: [ 5 | './pages/**/*.{js,ts,jsx,tsx}', 6 | './components/**/*.{js,ts,jsx,tsx}', 7 | ], 8 | }, 9 | content: [ 10 | './pages/**/*.{js,ts,jsx,tsx}', 11 | './components/**/*.{js,ts,jsx,tsx}', 12 | ], 13 | safelist: [ 14 | 'max-h-48', 15 | 'w-[150px]', 16 | 'w-[240px]', 17 | 'min-w-[270px]', 18 | 'min-w-[180px]', 19 | 'max-w-[180px]', 20 | ], 21 | theme: { 22 | extend: {}, 23 | }, 24 | plugins: [], 25 | }; 26 | -------------------------------------------------------------------------------- /pages/_document.tsx: -------------------------------------------------------------------------------- 1 | import Document, { Html, Head, Main, NextScript } from 'next/document'; 2 | 3 | class MyDocument extends Document { 4 | // eslint-disable-next-line class-methods-use-this 5 | render() { 6 | return ( 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 |
15 | 16 | 17 | 18 | ); 19 | } 20 | } 21 | 22 | export default MyDocument; 23 | -------------------------------------------------------------------------------- /components/keywords/KeywordPosition.tsx: -------------------------------------------------------------------------------- 1 | import Icon from '../common/Icon'; 2 | 3 | type KeywordPositionProps = { 4 | position: number, 5 | updating?: boolean, 6 | type?: string, 7 | } 8 | 9 | const KeywordPosition = ({ position = 0, type = '', updating = false }:KeywordPositionProps) => { 10 | if (!updating && position === 0) { 11 | return {'>100'}; 12 | } 13 | if (updating && type !== 'sc') { 14 | return ; 15 | } 16 | return <>{Math.round(position)}; 17 | }; 18 | 19 | export default KeywordPosition; 20 | -------------------------------------------------------------------------------- /database/database.ts: -------------------------------------------------------------------------------- 1 | import { Sequelize } from 'sequelize-typescript'; 2 | import sqlite3 from 'sqlite3'; 3 | import Domain from './models/domain'; 4 | import Keyword from './models/keyword'; 5 | 6 | const connection = new Sequelize({ 7 | dialect: 'sqlite', 8 | host: '0.0.0.0', 9 | username: process.env.USER_NAME ? process.env.USER_NAME : process.env.USER, 10 | password: process.env.PASSWORD, 11 | database: 'sequelize', 12 | dialectModule: sqlite3, 13 | pool: { 14 | max: 5, 15 | min: 0, 16 | idle: 10000, 17 | }, 18 | logging: false, 19 | models: [Domain, Keyword], 20 | storage: './data/database.sqlite', 21 | }); 22 | 23 | export default connection; 24 | -------------------------------------------------------------------------------- /scrapers/index.ts: -------------------------------------------------------------------------------- 1 | import scrapingAnt from './services/scrapingant'; 2 | import scrapingRobot from './services/scrapingrobot'; 3 | import serpapi from './services/serpapi'; 4 | import serply from './services/serply'; 5 | import spaceserp from './services/spaceserp'; 6 | import proxy from './services/proxy'; 7 | import searchapi from './services/searchapi'; 8 | import valueSerp from './services/valueserp'; 9 | import serper from './services/serper'; 10 | import hasdata from './services/hasdata'; 11 | 12 | export default [ 13 | scrapingRobot, 14 | scrapingAnt, 15 | serpapi, 16 | serply, 17 | spaceserp, 18 | proxy, 19 | searchapi, 20 | valueSerp, 21 | serper, 22 | hasdata, 23 | ]; 24 | -------------------------------------------------------------------------------- /scrapers/services/scrapingrobot.ts: -------------------------------------------------------------------------------- 1 | const scrapingRobot:ScraperSettings = { 2 | id: 'scrapingrobot', 3 | name: 'Scraping Robot', 4 | website: 'scrapingrobot.com', 5 | scrapeURL: (keyword, settings, countryData) => { 6 | const country = keyword.country || 'US'; 7 | const device = keyword.device === 'mobile' ? '&mobile=true' : ''; 8 | const lang = countryData[country][2]; 9 | const url = encodeURI(`https://www.google.com/search?num=100&hl=${lang}&q=${keyword.keyword}`); 10 | return `https://api.scrapingrobot.com/?token=${settings.scaping_api}&proxyCountry=${country}&render=false${device}&url=${url}`; 11 | }, 12 | resultObjectKey: 'result', 13 | }; 14 | 15 | export default scrapingRobot; 16 | -------------------------------------------------------------------------------- /pages/_app.tsx: -------------------------------------------------------------------------------- 1 | import '../styles/globals.css'; 2 | import React from 'react'; 3 | import type { AppProps } from 'next/app'; 4 | import { QueryClient, QueryClientProvider } from 'react-query'; 5 | import { ReactQueryDevtools } from 'react-query/devtools'; 6 | 7 | function MyApp({ Component, pageProps }: AppProps) { 8 | const [queryClient] = React.useState(() => new QueryClient({ 9 | defaultOptions: { 10 | queries: { 11 | refetchOnWindowFocus: false, 12 | }, 13 | }, 14 | })); 15 | return 16 | 17 | 18 | ; 19 | } 20 | 21 | export default MyApp; 22 | -------------------------------------------------------------------------------- /utils/parseKeywords.ts: -------------------------------------------------------------------------------- 1 | import Keyword from '../database/models/keyword'; 2 | 3 | /** 4 | * Parses the SQL Keyword Model object to frontend cosumable object. 5 | * @param {Keyword[]} allKeywords - Keywords to scrape 6 | * @returns {KeywordType[]} 7 | */ 8 | const parseKeywords = (allKeywords: Keyword[]) : KeywordType[] => { 9 | const parsedItems = allKeywords.map((keywrd:Keyword) => ({ 10 | ...keywrd, 11 | history: JSON.parse(keywrd.history), 12 | tags: JSON.parse(keywrd.tags), 13 | lastResult: JSON.parse(keywrd.lastResult), 14 | lastUpdateError: keywrd.lastUpdateError !== 'false' && keywrd.lastUpdateError.includes('{') ? JSON.parse(keywrd.lastUpdateError) : false, 15 | })); 16 | return parsedItems; 17 | }; 18 | 19 | export default parseKeywords; 20 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "esnext", 4 | "lib": [ 5 | "dom", 6 | "dom.iterable", 7 | "esnext" 8 | ], 9 | "allowJs": true, 10 | "skipLibCheck": true, 11 | "strict": true, 12 | "forceConsistentCasingInFileNames": true, 13 | "noEmit": true, 14 | "esModuleInterop": true, 15 | "module": "esnext", 16 | "moduleResolution": "node", 17 | "resolveJsonModule": true, 18 | "isolatedModules": true, 19 | "jsx": "preserve", 20 | "incremental": true, 21 | "experimentalDecorators": true 22 | }, 23 | "include": [ 24 | "next-env.d.ts", 25 | "**/*.ts", 26 | "**/*.tsx", 27 | "types.d.ts", 28 | "./jest.setup.js" 29 | ], 30 | "exclude": [ 31 | "node_modules" 32 | ] 33 | } -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["next/core-web-vitals", "airbnb-base"], 3 | "rules":{ 4 | "linebreak-style": 0, 5 | "indent":"off", 6 | "no-undef":"off", 7 | "no-console": "off", 8 | "camelcase":"off", 9 | "object-curly-newline":"off", 10 | "no-use-before-define": "off", 11 | "no-restricted-syntax": "off", 12 | "no-await-in-loop": "off", 13 | "arrow-body-style":"off", 14 | "max-len": ["error", {"code": 150, "ignoreComments": true, "ignoreUrls": true}], 15 | "import/no-extraneous-dependencies": "off", 16 | "no-unused-vars": "off", 17 | "implicit-arrow-linebreak": "off", 18 | "function-paren-newline": "off", 19 | "import/extensions": [ 20 | "error", 21 | "ignorePackages", 22 | { 23 | "": "never", 24 | "js": "never", 25 | "jsx": "never", 26 | "ts": "never", 27 | "tsx": "never" 28 | } 29 | ] 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | const nextJest = require('next/jest'); 2 | require('dotenv').config({ path: './.env.local' }); 3 | 4 | const createJestConfig = nextJest({ 5 | // Provide the path to your Next.js app to load next.config.js and .env files in your test environment 6 | dir: './', 7 | }); 8 | 9 | // Add any custom config to be passed to Jest 10 | /** @type {import('jest').Config} */ 11 | const customJestConfig = { 12 | // Add more setup options before each test is run 13 | setupFilesAfterEnv: ['/jest.setup.js'], 14 | // if using TypeScript with a baseUrl set to the root directory then you need the below for alias' to work 15 | moduleDirectories: ['node_modules', '/'], 16 | testEnvironment: 'jest-environment-jsdom', 17 | }; 18 | 19 | // createJestConfig is exported this way to ensure that next/jest can load the Next.js config which is async 20 | 21 | module.exports = createJestConfig(customJestConfig); 22 | -------------------------------------------------------------------------------- /jest.setup.js: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line no-unused-vars 2 | import 'isomorphic-fetch'; 3 | import './styles/globals.css'; 4 | import '@testing-library/jest-dom'; 5 | import { enableFetchMocks } from 'jest-fetch-mock'; 6 | // Optional: configure or set up a testing framework before each test. 7 | // If you delete this file, remove `setupFilesAfterEnv` from `jest.config.js` 8 | 9 | // Used for __tests__/testing-library.js 10 | // Learn more: https://github.com/testing-library/jest-dom 11 | 12 | window.matchMedia = (query) => ({ 13 | matches: false, 14 | media: query, 15 | onchange: null, 16 | addListener: jest.fn(), // deprecated 17 | removeListener: jest.fn(), // deprecated 18 | addEventListener: jest.fn(), 19 | removeEventListener: jest.fn(), 20 | dispatchEvent: jest.fn(), 21 | }); 22 | 23 | global.ResizeObserver = require('resize-observer-polyfill'); 24 | 25 | // Enable Fetch Mocking 26 | enableFetchMocks(); 27 | -------------------------------------------------------------------------------- /pages/api/logout.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from 'next'; 2 | import Cookies from 'cookies'; 3 | import verifyUser from '../../utils/verifyUser'; 4 | 5 | type logoutResponse = { 6 | success?: boolean 7 | error?: string|null, 8 | } 9 | 10 | export default async function handler(req: NextApiRequest, res: NextApiResponse) { 11 | const authorized = verifyUser(req, res); 12 | if (authorized !== 'authorized') { 13 | return res.status(401).json({ error: authorized }); 14 | } 15 | if (req.method === 'POST') { 16 | return logout(req, res); 17 | } 18 | return res.status(401).json({ success: false, error: 'Invalid Method' }); 19 | } 20 | 21 | const logout = async (req: NextApiRequest, res: NextApiResponse) => { 22 | const cookies = new Cookies(req, res); 23 | cookies.set('token', null, { maxAge: new Date().getTime() }); 24 | return res.status(200).json({ success: true, error: null }); 25 | }; 26 | -------------------------------------------------------------------------------- /pages/index.tsx: -------------------------------------------------------------------------------- 1 | import type { NextPage } from 'next'; 2 | import { useEffect } from 'react'; 3 | import Head from 'next/head'; 4 | import { useRouter } from 'next/router'; 5 | import { Toaster } from 'react-hot-toast'; 6 | import Icon from '../components/common/Icon'; 7 | 8 | const Home: NextPage = () => { 9 | const router = useRouter(); 10 | useEffect(() => { 11 | if (router) router.push('/domains'); 12 | }, [router]); 13 | 14 | return ( 15 |
16 | 17 | SerpBear 18 | 19 | 20 | 21 | 22 |
23 | 24 |
25 | 26 |
27 | ); 28 | }; 29 | 30 | export default Home; 31 | -------------------------------------------------------------------------------- /styles/changelog.css: -------------------------------------------------------------------------------- 1 | .changelog-body{ 2 | max-height: calc(100vh - 60px); 3 | } 4 | 5 | .changelog-content h1{ 6 | margin: 1.2rem 0; 7 | font-size: 2rem; 8 | font-weight: 600; 9 | } 10 | 11 | .changelog-content h2{ 12 | margin: 1.2rem 0; 13 | font-size: 1.6rem; 14 | font-weight: bold; 15 | margin-top: 0; 16 | } 17 | 18 | .changelog-content h2:after { 19 | content: "🌟"; 20 | font-size: 1.2rem; 21 | position: relative; 22 | top: -2px; 23 | } 24 | 25 | .changelog-content h3{ 26 | margin: 1.2rem 0; 27 | font-size: 1rem; 28 | font-weight: 600; 29 | margin-top: 0; 30 | } 31 | 32 | .changelog-content h4{ 33 | font-size: 1rem; 34 | font-weight: 600; 35 | } 36 | 37 | 38 | .changelog-content ul{ 39 | margin: 0; 40 | padding: 0; 41 | margin-bottom: 20px; 42 | padding-left: 20px; 43 | list-style-type: disc; 44 | line-height: 1.7em; 45 | } 46 | 47 | .changelog-content a:link{ 48 | color: rgb(75, 75, 241) 49 | } 50 | .changelog-content a:visited{ 51 | color: rgb(116, 116, 121) 52 | } -------------------------------------------------------------------------------- /__tests__/hooks/domains.test.tsx: -------------------------------------------------------------------------------- 1 | import { renderHook, waitFor } from '@testing-library/react'; 2 | import mockRouter from 'next-router-mock'; 3 | 4 | import { useFetchDomains } from '../../services/domains'; 5 | import { createWrapper } from '../../__mocks__/utils'; 6 | import { dummyDomain } from '../../__mocks__/data'; 7 | 8 | jest.mock('next/router', () => jest.requireActual('next-router-mock')); 9 | 10 | fetchMock.mockIf(`${window.location.origin}/api/domains`, async () => { 11 | return new Promise((resolve) => { 12 | resolve({ 13 | body: JSON.stringify({ domains: [dummyDomain] }), 14 | status: 200, 15 | }); 16 | }); 17 | }); 18 | 19 | describe('DomainHooks', () => { 20 | it('useFetchDomains should fetch the Domains', async () => { 21 | const { result } = renderHook(() => useFetchDomains(mockRouter), { wrapper: createWrapper() }); 22 | // const result = { current: { isSuccess: false, data: '' } }; 23 | await waitFor(() => { 24 | return expect(result.current.isLoading).toBe(false); 25 | }); 26 | }); 27 | }); 28 | -------------------------------------------------------------------------------- /public/vercel.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | -------------------------------------------------------------------------------- /components/common/InputField.tsx: -------------------------------------------------------------------------------- 1 | type InputFieldProps = { 2 | label: string; 3 | value: string; 4 | onChange: Function; 5 | placeholder?: string; 6 | classNames?: string; 7 | hasError?: boolean; 8 | } 9 | 10 | const InputField = ({ label = '', value = '', placeholder = '', onChange, hasError = false }: InputFieldProps) => { 11 | const labelStyle = 'mb-2 font-semibold inline-block text-sm text-gray-700 capitalize'; 12 | return ( 13 |
14 | 15 | onChange(event.target.value)} 21 | autoComplete="off" 22 | placeholder={placeholder} 23 | /> 24 |
25 | ); 26 | }; 27 | 28 | export default InputField; 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Amruth Pillai 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /__tests__/components/Sidebar.test.tsx: -------------------------------------------------------------------------------- 1 | import { fireEvent, render, screen } from '@testing-library/react'; 2 | import Sidebar from '../../components/common/Sidebar'; 3 | import { dummyDomain } from '../../__mocks__/data'; 4 | 5 | const addDomainMock = jest.fn(); 6 | jest.mock('next/router', () => jest.requireActual('next-router-mock')); 7 | 8 | describe('Sidebar Component', () => { 9 | it('renders without crashing', async () => { 10 | render(); 11 | expect(screen.getByText('SerpBear')).toBeInTheDocument(); 12 | }); 13 | it('renders domain list', async () => { 14 | render(); 15 | expect(screen.getByText('compressimage.io')).toBeInTheDocument(); 16 | }); 17 | it('calls showAddModal on Add Domain button click', async () => { 18 | render(); 19 | const addDomainBtn = screen.getByTestId('add_domain'); 20 | fireEvent.click(addDomainBtn); 21 | expect(addDomainMock).toHaveBeenCalledWith(true); 22 | }); 23 | }); 24 | -------------------------------------------------------------------------------- /__tests__/pages/index.test.tsx: -------------------------------------------------------------------------------- 1 | import { render, screen } from '@testing-library/react'; 2 | import { QueryClient, QueryClientProvider } from 'react-query'; 3 | import Home from '../../pages/index'; 4 | 5 | const routerPush = jest.fn(); 6 | jest.mock('next/router', () => ({ 7 | useRouter: () => ({ 8 | push: routerPush, 9 | }), 10 | })); 11 | 12 | describe('Home Page', () => { 13 | const queryClient = new QueryClient(); 14 | it('Renders without crashing', async () => { 15 | render( 16 | 17 | 18 | , 19 | ); 20 | // console.log(prettyDOM(renderer.container.firstChild)); 21 | expect(await screen.findByRole('main')).toBeInTheDocument(); 22 | expect(screen.queryByText('Add Domain')).not.toBeInTheDocument(); 23 | }); 24 | it('Should redirect to /domains route.', async () => { 25 | render( 26 | 27 | 28 | , 29 | ); 30 | expect(routerPush).toHaveBeenCalledWith('/domains'); 31 | }); 32 | }); 33 | -------------------------------------------------------------------------------- /pages/api/clearfailed.ts: -------------------------------------------------------------------------------- 1 | import { writeFile } from 'fs/promises'; 2 | import type { NextApiRequest, NextApiResponse } from 'next'; 3 | import verifyUser from '../../utils/verifyUser'; 4 | 5 | type SettingsGetResponse = { 6 | cleared?: boolean, 7 | error?: string, 8 | } 9 | 10 | export default async function handler(req: NextApiRequest, res: NextApiResponse) { 11 | const authorized = verifyUser(req, res); 12 | if (authorized !== 'authorized') { 13 | return res.status(401).json({ error: authorized }); 14 | } 15 | if (req.method === 'PUT') { 16 | return clearFailedQueue(req, res); 17 | } 18 | return res.status(502).json({ error: 'Unrecognized Route.' }); 19 | } 20 | 21 | const clearFailedQueue = async (req: NextApiRequest, res: NextApiResponse) => { 22 | try { 23 | await writeFile(`${process.cwd()}/data/failed_queue.json`, JSON.stringify([]), { encoding: 'utf-8' }); 24 | return res.status(200).json({ cleared: true }); 25 | } catch (error) { 26 | console.log('[ERROR] Cleraring Failed Queue File.', error); 27 | return res.status(200).json({ error: 'Error Cleraring Failed Queue!' }); 28 | } 29 | }; 30 | -------------------------------------------------------------------------------- /scrapers/services/scrapingant.ts: -------------------------------------------------------------------------------- 1 | const scrapingAnt:ScraperSettings = { 2 | id: 'scrapingant', 3 | name: 'ScrapingAnt', 4 | website: 'scrapingant.com', 5 | headers: (keyword) => { 6 | // eslint-disable-next-line max-len 7 | const mobileAgent = 'Mozilla/5.0 (Linux; Android 10; SM-G996U Build/QP1A.190711.020; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Mobile Safari/537.36'; 8 | return keyword && keyword.device === 'mobile' ? { 'Ant-User-Agent': mobileAgent } : {}; 9 | }, 10 | scrapeURL: (keyword, settings, countryData) => { 11 | const scraperCountries = ['AE', 'BR', 'CN', 'DE', 'ES', 'FR', 'GB', 'HK', 'PL', 'IN', 'IT', 'IL', 'JP', 'NL', 'RU', 'SA', 'US', 'CZ']; 12 | const country = scraperCountries.includes(keyword.country.toUpperCase()) ? keyword.country : 'US'; 13 | const lang = countryData[country][2]; 14 | const url = encodeURI(`https://www.google.com/search?num=100&hl=${lang}&q=${keyword.keyword}`); 15 | return `https://api.scrapingant.com/v2/extended?url=${url}&x-api-key=${settings.scaping_api}&proxy_country=${country}&browser=false`; 16 | }, 17 | resultObjectKey: 'result', 18 | }; 19 | 20 | export default scrapingAnt; 21 | -------------------------------------------------------------------------------- /utils/client/validators.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/prefer-default-export */ 2 | export const isValidDomain = (domain:string): boolean => { 3 | if (typeof domain !== 'string') return false; 4 | if (!domain.includes('.')) return false; 5 | let value = domain; 6 | const validHostnameChars = /^[a-zA-Z0-9-.]{1,253}\.?$/g; 7 | if (!validHostnameChars.test(value)) { 8 | return false; 9 | } 10 | 11 | if (value.endsWith('.')) { 12 | value = value.slice(0, value.length - 1); 13 | } 14 | 15 | if (value.length > 253) { 16 | return false; 17 | } 18 | 19 | const labels = value.split('.'); 20 | 21 | const isValid = labels.every((label) => { 22 | const validLabelChars = /^([a-zA-Z0-9-]+)$/g; 23 | 24 | const validLabel = ( 25 | validLabelChars.test(label) 26 | && label.length < 64 27 | && !label.startsWith('-') 28 | && !label.endsWith('-') 29 | ); 30 | 31 | return validLabel; 32 | }); 33 | 34 | return isValid; 35 | }; 36 | 37 | export const isValidUrl = (str: string) => { 38 | let url; 39 | 40 | try { 41 | url = new URL(str); 42 | } catch (e) { 43 | return false; 44 | } 45 | return url.protocol === 'http:' || url.protocol === 'https:'; 46 | }; 47 | -------------------------------------------------------------------------------- /database/migrations/1707233039698-add-domain-searchconsole-field.js: -------------------------------------------------------------------------------- 1 | // Migration: Adds search_console field to domain table to assign search console property type, url and api. 2 | 3 | // CLI Migration 4 | module.exports = { 5 | up: (queryInterface, Sequelize) => { 6 | return queryInterface.sequelize.transaction(async (t) => { 7 | try { 8 | const domainTableDefinition = await queryInterface.describeTable('domain'); 9 | if (domainTableDefinition && !domainTableDefinition.search_console) { 10 | await queryInterface.addColumn('domain', 'search_console', { type: Sequelize.DataTypes.STRING }, { transaction: t }); 11 | } 12 | } catch (error) { 13 | console.log('error :', error); 14 | } 15 | }); 16 | }, 17 | down: (queryInterface) => { 18 | return queryInterface.sequelize.transaction(async (t) => { 19 | try { 20 | const domainTableDefinition = await queryInterface.describeTable('domain'); 21 | if (domainTableDefinition && domainTableDefinition.search_console) { 22 | await queryInterface.removeColumn('domain', 'search_console', { transaction: t }); 23 | } 24 | } catch (error) { 25 | console.log('error :', error); 26 | } 27 | }); 28 | }, 29 | }; 30 | -------------------------------------------------------------------------------- /scrapers/services/serper.ts: -------------------------------------------------------------------------------- 1 | interface SerperResult { 2 | title: string, 3 | link: string, 4 | position: number, 5 | } 6 | 7 | const serper:ScraperSettings = { 8 | id: 'serper', 9 | name: 'Serper.dev', 10 | website: 'serper.dev', 11 | allowsCity: true, 12 | scrapeURL: (keyword, settings, countryData) => { 13 | const country = keyword.country || 'US'; 14 | const lang = countryData[country][2]; 15 | console.log('Serper URL :', `https://google.serper.dev/search?q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`); 16 | return `https://google.serper.dev/search?q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`; 17 | }, 18 | resultObjectKey: 'organic', 19 | serpExtractor: (content) => { 20 | const extractedResult = []; 21 | const results: SerperResult[] = (typeof content === 'string') ? JSON.parse(content) : content as SerperResult[]; 22 | 23 | for (const { link, title, position } of results) { 24 | if (title && link) { 25 | extractedResult.push({ 26 | title, 27 | url: link, 28 | position, 29 | }); 30 | } 31 | } 32 | return extractedResult; 33 | }, 34 | }; 35 | 36 | export default serper; 37 | -------------------------------------------------------------------------------- /__tests__/components/Modal.test.tsx: -------------------------------------------------------------------------------- 1 | import { fireEvent, render, screen } from '@testing-library/react'; 2 | import Modal from '../../components/common/Modal'; 3 | 4 | const closeModalMock = jest.fn(); 5 | describe('Modal Component', () => { 6 | it('Renders without crashing', async () => { 7 | render(
); 8 | expect(document.querySelector('.modal')).toBeInTheDocument(); 9 | }); 10 | it('Displays the Given Content', async () => { 11 | render( 12 |
13 |

Hello Modal!!

14 |
15 | ); 16 | expect(await screen.findByText('Hello Modal!!')).toBeInTheDocument(); 17 | }); 18 | it('Renders Modal Title', async () => { 19 | render(

Some Modal Content

); 20 | expect(await screen.findByText('Sample Modal Title')).toBeInTheDocument(); 21 | }); 22 | it('Closes the modal on close button click', async () => { 23 | const { container } = render(

Some Modal Content

); 24 | const closeBtn = container.querySelector('.modal-close'); 25 | if (closeBtn) fireEvent.click(closeBtn); 26 | expect(closeModalMock).toHaveBeenCalled(); 27 | }); 28 | }); 29 | -------------------------------------------------------------------------------- /database/migrations/1709217223856-add-keyword-volume-field copy.js: -------------------------------------------------------------------------------- 1 | // Migration: Adds volume field to the keyword table. 2 | 3 | // CLI Migration 4 | module.exports = { 5 | up: async (queryInterface, Sequelize) => { 6 | return queryInterface.sequelize.transaction(async (t) => { 7 | try { 8 | const keywordTableDefinition = await queryInterface.describeTable('keyword'); 9 | if (keywordTableDefinition) { 10 | if (!keywordTableDefinition.volume) { 11 | await queryInterface.addColumn('keyword', 'volume', { 12 | type: Sequelize.DataTypes.STRING, allowNull: false, defaultValue: 0, 13 | }, { transaction: t }); 14 | } 15 | } 16 | } catch (error) { 17 | console.log('error :', error); 18 | } 19 | }); 20 | }, 21 | down: (queryInterface) => { 22 | return queryInterface.sequelize.transaction(async (t) => { 23 | try { 24 | const keywordTableDefinition = await queryInterface.describeTable('keyword'); 25 | if (keywordTableDefinition) { 26 | if (keywordTableDefinition.volume) { 27 | await queryInterface.removeColumn('keyword', 'volume', { transaction: t }); 28 | } 29 | } 30 | } catch (error) { 31 | console.log('error :', error); 32 | } 33 | }); 34 | }, 35 | }; 36 | -------------------------------------------------------------------------------- /components/common/Chart.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Chart as ChartJS, CategoryScale, LinearScale, PointElement, LineElement, Title, Tooltip, Legend } from 'chart.js'; 3 | import { Line } from 'react-chartjs-2'; 4 | 5 | ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Title, Tooltip, Legend); 6 | 7 | type ChartProps ={ 8 | labels: string[], 9 | sreies: number[], 10 | reverse? : boolean, 11 | noMaxLimit?: boolean 12 | } 13 | 14 | const Chart = ({ labels, sreies, reverse = true, noMaxLimit = false }:ChartProps) => { 15 | const options = { 16 | responsive: true, 17 | maintainAspectRatio: false, 18 | animation: false as const, 19 | scales: { 20 | y: { 21 | reverse, 22 | min: 1, 23 | max: !noMaxLimit && reverse ? 100 : undefined, 24 | }, 25 | }, 26 | plugins: { 27 | legend: { 28 | display: false, 29 | }, 30 | }, 31 | }; 32 | 33 | return ; 48 | }; 49 | 50 | export default Chart; 51 | -------------------------------------------------------------------------------- /components/common/ToggleField.tsx: -------------------------------------------------------------------------------- 1 | type ToggleFieldProps = { 2 | label: string; 3 | value: boolean; 4 | onChange: (bool:boolean) => void ; 5 | classNames?: string; 6 | } 7 | 8 | const ToggleField = ({ label = '', value = false, onChange, classNames = '' }: ToggleFieldProps) => { 9 | return ( 10 |
11 | 28 |
29 | ); 30 | }; 31 | 32 | export default ToggleField; 33 | -------------------------------------------------------------------------------- /components/common/Footer.tsx: -------------------------------------------------------------------------------- 1 | import { useState } from 'react'; 2 | import { CSSTransition } from 'react-transition-group'; 3 | import { useFetchChangelog } from '../../services/misc'; 4 | import ChangeLog from '../settings/Changelog'; 5 | 6 | interface FooterProps { 7 | currentVersion: string 8 | } 9 | 10 | const Footer = ({ currentVersion = '' }: FooterProps) => { 11 | const [showChangelog, setShowChangelog] = useState(false); 12 | const { data: changeLogs } = useFetchChangelog(); 13 | const latestVersionNum = changeLogs && Array.isArray(changeLogs) && changeLogs[0] ? changeLogs[0].name : ''; 14 | 15 | return ( 16 | 29 | ); 30 | }; 31 | 32 | export default Footer; 33 | -------------------------------------------------------------------------------- /components/common/SecretField.tsx: -------------------------------------------------------------------------------- 1 | import { useState } from 'react'; 2 | import Icon from './Icon'; 3 | 4 | type SecretFieldProps = { 5 | label: string; 6 | value: string; 7 | onChange: Function; 8 | placeholder?: string; 9 | classNames?: string; 10 | hasError?: boolean; 11 | } 12 | 13 | const SecretField = ({ label = '', value = '', placeholder = '', onChange, hasError = false }: SecretFieldProps) => { 14 | const [showValue, setShowValue] = useState(false); 15 | const labelStyle = 'mb-2 font-semibold inline-block text-sm text-gray-700 capitalize'; 16 | return ( 17 |
18 | 19 | setShowValue(!showValue)}> 22 | 23 | 24 | onChange(event.target.value)} 30 | autoComplete="off" 31 | placeholder={placeholder} 32 | /> 33 |
34 | ); 35 | }; 36 | 37 | export default SecretField; 38 | -------------------------------------------------------------------------------- /database/models/domain.ts: -------------------------------------------------------------------------------- 1 | import { Table, Model, Column, DataType, PrimaryKey, Unique } from 'sequelize-typescript'; 2 | 3 | @Table({ 4 | timestamps: false, 5 | tableName: 'domain', 6 | }) 7 | 8 | class Domain extends Model { 9 | @PrimaryKey 10 | @Column({ type: DataType.INTEGER, allowNull: false, primaryKey: true, autoIncrement: true }) 11 | ID!: number; 12 | 13 | @Unique 14 | @Column({ type: DataType.STRING, allowNull: false, defaultValue: true, unique: true }) 15 | domain!: string; 16 | 17 | @Unique 18 | @Column({ type: DataType.STRING, allowNull: false, defaultValue: true, unique: true }) 19 | slug!: string; 20 | 21 | @Column({ type: DataType.INTEGER, allowNull: false, defaultValue: 0 }) 22 | keywordCount!: number; 23 | 24 | @Column({ type: DataType.STRING, allowNull: true }) 25 | lastUpdated!: string; 26 | 27 | @Column({ type: DataType.STRING, allowNull: true }) 28 | added!: string; 29 | 30 | @Column({ type: DataType.STRING, allowNull: true, defaultValue: JSON.stringify([]) }) 31 | tags!: string; 32 | 33 | @Column({ type: DataType.BOOLEAN, allowNull: true, defaultValue: true }) 34 | notification!: boolean; 35 | 36 | @Column({ type: DataType.STRING, allowNull: true, defaultValue: 'daily' }) 37 | notification_interval!: string; 38 | 39 | @Column({ type: DataType.STRING, allowNull: true, defaultValue: '' }) 40 | notification_emails!: string; 41 | 42 | @Column({ type: DataType.STRING, allowNull: true }) 43 | search_console!: string; 44 | } 45 | 46 | export default Domain; 47 | -------------------------------------------------------------------------------- /scrapers/services/serpapi.ts: -------------------------------------------------------------------------------- 1 | import countries from '../../utils/countries'; 2 | 3 | interface SerpApiResult { 4 | title: string, 5 | link: string, 6 | position: number, 7 | } 8 | 9 | const serpapi:ScraperSettings = { 10 | id: 'serpapi', 11 | name: 'SerpApi.com', 12 | website: 'serpapi.com', 13 | allowsCity: true, 14 | headers: (keyword, settings) => { 15 | return { 16 | 'Content-Type': 'application/json', 17 | 'X-API-Key': settings.scaping_api, 18 | }; 19 | }, 20 | scrapeURL: (keyword, settings) => { 21 | const countryName = countries[keyword.country || 'US'][0]; 22 | const location = keyword.city && keyword.country ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : ''; 23 | return `https://serpapi.com/search?q=${encodeURIComponent(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}${location}&api_key=${settings.scaping_api}`; 24 | }, 25 | resultObjectKey: 'organic_results', 26 | serpExtractor: (content) => { 27 | const extractedResult = []; 28 | const results: SerpApiResult[] = (typeof content === 'string') ? JSON.parse(content) : content as SerpApiResult[]; 29 | 30 | for (const { link, title, position } of results) { 31 | if (title && link) { 32 | extractedResult.push({ 33 | title, 34 | url: link, 35 | position, 36 | }); 37 | } 38 | } 39 | return extractedResult; 40 | }, 41 | }; 42 | 43 | export default serpapi; 44 | -------------------------------------------------------------------------------- /scrapers/services/searchapi.ts: -------------------------------------------------------------------------------- 1 | import countries from '../../utils/countries'; 2 | 3 | interface SearchApiResult { 4 | title: string, 5 | link: string, 6 | position: number, 7 | } 8 | 9 | const searchapi:ScraperSettings = { 10 | id: 'searchapi', 11 | name: 'SearchApi.io', 12 | website: 'searchapi.io', 13 | allowsCity: true, 14 | headers: (keyword, settings) => { 15 | return { 16 | 'Content-Type': 'application/json', 17 | Authorization: `Bearer ${settings.scaping_api}`, 18 | }; 19 | }, 20 | scrapeURL: (keyword) => { 21 | const country = keyword.country || 'US'; 22 | const countryName = countries[country][0]; 23 | const location = keyword.city && countryName ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : ''; 24 | return `https://www.searchapi.io/api/v1/search?engine=google&q=${encodeURIComponent(keyword.keyword)}&num=100&gl=${country}&device=${keyword.device}${location}`; 25 | }, 26 | resultObjectKey: 'organic_results', 27 | serpExtractor: (content) => { 28 | const extractedResult = []; 29 | const results: SearchApiResult[] = (typeof content === 'string') ? JSON.parse(content) : content as SearchApiResult[]; 30 | 31 | for (const { link, title, position } of results) { 32 | if (title && link) { 33 | extractedResult.push({ 34 | title, 35 | url: link, 36 | position, 37 | }); 38 | } 39 | } 40 | return extractedResult; 41 | }, 42 | }; 43 | 44 | export default searchapi; 45 | -------------------------------------------------------------------------------- /scrapers/services/proxy.ts: -------------------------------------------------------------------------------- 1 | import * as cheerio from 'cheerio'; 2 | 3 | const proxy:ScraperSettings = { 4 | id: 'proxy', 5 | name: 'Proxy', 6 | website: '', 7 | resultObjectKey: 'data', 8 | headers: () => { 9 | return { Accept: 'gzip,deflate,compress;' }; 10 | }, 11 | scrapeURL: (keyword) => { 12 | return `https://www.google.com/search?num=100&q=${encodeURI(keyword.keyword)}`; 13 | }, 14 | serpExtractor: (content) => { 15 | const extractedResult = []; 16 | 17 | const $ = cheerio.load(content); 18 | let lastPosition = 0; 19 | const hasValidContent = $('body').find('#main'); 20 | if (hasValidContent.length === 0) { 21 | const msg = '[ERROR] Scraped search results from proxy do not adhere to expected format. Unable to parse results'; 22 | console.log(msg); 23 | throw new Error(msg); 24 | } 25 | 26 | const mainContent = $('body').find('#main'); 27 | const children = $(mainContent).find('h3'); 28 | 29 | for (let index = 0; index < children.length; index += 1) { 30 | const title = $(children[index]).text(); 31 | const url = $(children[index]).closest('a').attr('href'); 32 | const cleanedURL = url ? url.replaceAll(/^.+?(?=https:|$)/g, '').replaceAll(/(&).*/g, '') : ''; 33 | if (title && url) { 34 | lastPosition += 1; 35 | extractedResult.push({ title, url: cleanedURL, position: lastPosition }); 36 | } 37 | } 38 | return extractedResult; 39 | }, 40 | }; 41 | 42 | export default proxy; 43 | -------------------------------------------------------------------------------- /pages/api/keyword.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from 'next'; 2 | import db from '../../database/database'; 3 | import Keyword from '../../database/models/keyword'; 4 | import parseKeywords from '../../utils/parseKeywords'; 5 | import verifyUser from '../../utils/verifyUser'; 6 | 7 | type KeywordGetResponse = { 8 | keyword?: KeywordType | null 9 | error?: string|null, 10 | } 11 | 12 | export default async function handler(req: NextApiRequest, res: NextApiResponse) { 13 | const authorized = verifyUser(req, res); 14 | if (authorized === 'authorized' && req.method === 'GET') { 15 | await db.sync(); 16 | return getKeyword(req, res); 17 | } 18 | return res.status(401).json({ error: authorized }); 19 | } 20 | 21 | const getKeyword = async (req: NextApiRequest, res: NextApiResponse) => { 22 | if (!req.query.id && typeof req.query.id !== 'string') { 23 | return res.status(400).json({ error: 'Keyword ID is Required!' }); 24 | } 25 | 26 | try { 27 | const query = { ID: parseInt((req.query.id as string), 10) }; 28 | const foundKeyword:Keyword| null = await Keyword.findOne({ where: query }); 29 | const pareseKeyword = foundKeyword && parseKeywords([foundKeyword.get({ plain: true })]); 30 | const keywords = pareseKeyword && pareseKeyword[0] ? pareseKeyword[0] : null; 31 | return res.status(200).json({ keyword: keywords }); 32 | } catch (error) { 33 | console.log('[ERROR] Getting Keyword: ', error); 34 | return res.status(400).json({ error: 'Error Loading Keyword' }); 35 | } 36 | }; 37 | -------------------------------------------------------------------------------- /components/common/Modal.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Icon from './Icon'; 3 | import useOnKey from '../../hooks/useOnKey'; 4 | 5 | type ModalProps = { 6 | children: React.ReactNode, 7 | width?: string, 8 | title?: string, 9 | verticalCenter?: boolean, 10 | closeModal: Function, 11 | } 12 | 13 | const Modal = ({ children, width = '1/2', closeModal, title, verticalCenter = false }:ModalProps) => { 14 | useOnKey('Escape', closeModal); 15 | 16 | const closeOnBGClick = (e:React.SyntheticEvent) => { 17 | e.stopPropagation(); 18 | e.nativeEvent.stopImmediatePropagation(); 19 | if (e.target === e.currentTarget) { closeModal(); } 20 | }; 21 | 22 | return ( 23 |
24 |
28 | {title &&

{title}

} 29 | 34 |
{children}
35 |
36 |
37 | ); 38 | }; 39 | 40 | export default Modal; 41 | -------------------------------------------------------------------------------- /scrapers/services/spaceserp.ts: -------------------------------------------------------------------------------- 1 | import countries from '../../utils/countries'; 2 | 3 | interface SpaceSerpResult { 4 | title: string, 5 | link: string, 6 | domain: string, 7 | position: number 8 | } 9 | 10 | const spaceSerp:ScraperSettings = { 11 | id: 'spaceSerp', 12 | name: 'Space Serp', 13 | website: 'spaceserp.com', 14 | allowsCity: true, 15 | scrapeURL: (keyword, settings, countryData) => { 16 | const country = keyword.country || 'US'; 17 | const countryName = countries[country][0]; 18 | const location = keyword.city ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : ''; 19 | const device = keyword.device === 'mobile' ? '&device=mobile' : ''; 20 | const lang = countryData[country][2]; 21 | return `https://api.spaceserp.com/google/search?apiKey=${settings.scaping_api}&q=${encodeURIComponent(keyword.keyword)}&pageSize=100&gl=${country}&hl=${lang}${location}${device}&resultBlocks=`; 22 | }, 23 | resultObjectKey: 'organic_results', 24 | serpExtractor: (content) => { 25 | const extractedResult = []; 26 | const results: SpaceSerpResult[] = (typeof content === 'string') ? JSON.parse(content) : content as SpaceSerpResult[]; 27 | for (const result of results) { 28 | if (result.title && result.link) { 29 | extractedResult.push({ 30 | title: result.title, 31 | url: result.link, 32 | position: result.position, 33 | }); 34 | } 35 | } 36 | return extractedResult; 37 | }, 38 | }; 39 | 40 | export default spaceSerp; 41 | -------------------------------------------------------------------------------- /scrapers/services/hasdata.ts: -------------------------------------------------------------------------------- 1 | import countries from '../../utils/countries'; 2 | 3 | interface HasDataResult { 4 | title: string, 5 | link: string, 6 | position: number, 7 | } 8 | 9 | const hasdata:ScraperSettings = { 10 | id: 'hasdata', 11 | name: 'HasData', 12 | website: 'hasdata.com', 13 | allowsCity: true, 14 | headers: (keyword, settings) => { 15 | return { 16 | 'Content-Type': 'application/json', 17 | 'x-api-key': settings.scaping_api, 18 | }; 19 | }, 20 | scrapeURL: (keyword, settings) => { 21 | const country = keyword.country || 'US'; 22 | const countryName = countries[country][0]; 23 | const location = keyword.city && countryName ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : ''; 24 | return `https://api.scrape-it.cloud/scrape/google/serp?q=${encodeURIComponent(keyword.keyword)}${location}&num=100&gl=${country.toLowerCase()}&deviceType=${keyword.device}`; 25 | }, 26 | resultObjectKey: 'organicResults', 27 | serpExtractor: (content) => { 28 | const extractedResult = []; 29 | const results: HasDataResult[] = (typeof content === 'string') ? JSON.parse(content) : content as HasDataResult[]; 30 | 31 | for (const { link, title, position } of results) { 32 | if (title && link) { 33 | extractedResult.push({ 34 | title, 35 | url: link, 36 | position, 37 | }); 38 | } 39 | } 40 | return extractedResult; 41 | }, 42 | }; 43 | 44 | export default hasdata; 45 | -------------------------------------------------------------------------------- /pages/api/login.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from 'next'; 2 | import jwt from 'jsonwebtoken'; 3 | import Cookies from 'cookies'; 4 | 5 | type loginResponse = { 6 | success?: boolean 7 | error?: string|null, 8 | } 9 | 10 | export default async function handler(req: NextApiRequest, res: NextApiResponse) { 11 | if (req.method === 'POST') { 12 | return loginUser(req, res); 13 | } 14 | return res.status(401).json({ success: false, error: 'Invalid Method' }); 15 | } 16 | 17 | const loginUser = async (req: NextApiRequest, res: NextApiResponse) => { 18 | if (!req.body.username || !req.body.password) { 19 | return res.status(401).json({ error: 'Username Password Missing' }); 20 | } 21 | const userName = process.env.USER_NAME ? process.env.USER_NAME : process.env.USER; 22 | 23 | if (req.body.username === userName 24 | && req.body.password === process.env.PASSWORD && process.env.SECRET) { 25 | const token = jwt.sign({ user: userName }, process.env.SECRET); 26 | const cookies = new Cookies(req, res); 27 | const expireDate = new Date(); 28 | const sessDuration = process.env.SESSION_DURATION; 29 | expireDate.setHours((sessDuration && parseInt(sessDuration, 10)) || 24); 30 | cookies.set('token', token, { httpOnly: true, sameSite: 'lax', maxAge: expireDate.getTime() }); 31 | return res.status(200).json({ success: true, error: null }); 32 | } 33 | 34 | const error = req.body.username !== userName ? 'Incorrect Username' : 'Incorrect Password'; 35 | 36 | return res.status(401).json({ success: false, error }); 37 | }; 38 | -------------------------------------------------------------------------------- /components/common/SidePanel.tsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect, useState } from 'react'; 2 | import Icon from './Icon'; 3 | import useOnKey from '../../hooks/useOnKey'; 4 | 5 | type SidePanelProps = { 6 | children: React.ReactNode, 7 | closePanel: Function, 8 | title?: string, 9 | width?: 'large' | 'medium' | 'small', 10 | position?: 'left' | 'right' 11 | } 12 | const SidePanel = ({ children, closePanel, width, position = 'right', title = '' }:SidePanelProps) => { 13 | useOnKey('Escape', closePanel); 14 | const closeOnBGClick = (e:React.SyntheticEvent) => { 15 | e.stopPropagation(); 16 | e.nativeEvent.stopImmediatePropagation(); 17 | if (e.target === e.currentTarget) { closePanel(); } 18 | }; 19 | return ( 20 |
21 |
23 |
24 |

{title}

25 | 30 |
31 |
{children}
32 |
33 |
34 | ); 35 | }; 36 | 37 | export default SidePanel; 38 | -------------------------------------------------------------------------------- /scrapers/services/serply.ts: -------------------------------------------------------------------------------- 1 | interface SerplyResult { 2 | title: string, 3 | link: string, 4 | realPosition: number, 5 | } 6 | const scraperCountries = ['US', 'CA', 'IE', 'GB', 'FR', 'DE', 'SE', 'IN', 'JP', 'KR', 'SG', 'AU', 'BR']; 7 | 8 | const serply:ScraperSettings = { 9 | id: 'serply', 10 | name: 'Serply', 11 | website: 'serply.io', 12 | headers: (keyword, settings) => { 13 | const country = scraperCountries.includes(keyword.country.toUpperCase()) ? keyword.country : 'US'; 14 | return { 15 | 'Content-Type': 'application/json', 16 | 'X-User-Agent': keyword.device === 'mobile' ? 'mobile' : 'desktop', 17 | 'X-Api-Key': settings.scaping_api, 18 | 'X-Proxy-Location': country, 19 | }; 20 | }, 21 | scrapeURL: (keyword) => { 22 | const country = scraperCountries.includes(keyword.country.toUpperCase()) ? keyword.country : 'US'; 23 | return `https://api.serply.io/v1/search/q=${encodeURIComponent(keyword.keyword)}&num=100&hl=${country}`; 24 | }, 25 | resultObjectKey: 'result', 26 | serpExtractor: (content) => { 27 | const extractedResult = []; 28 | const results: SerplyResult[] = (typeof content === 'string') ? JSON.parse(content) : content as SerplyResult[]; 29 | for (const result of results) { 30 | if (result.title && result.link) { 31 | extractedResult.push({ 32 | title: result.title, 33 | url: result.link, 34 | position: result.realPosition, 35 | }); 36 | } 37 | } 38 | return extractedResult; 39 | }, 40 | }; 41 | 42 | export default serply; 43 | -------------------------------------------------------------------------------- /__tests__/components/DomainItem.test.tsx: -------------------------------------------------------------------------------- 1 | import { fireEvent, render } from '@testing-library/react'; 2 | import DomainItem from '../../components/domains/DomainItem'; 3 | import { dummyDomain } from '../../__mocks__/data'; 4 | 5 | const updateThumbMock = jest.fn(); 6 | const domainItemProps = { 7 | domain: dummyDomain, 8 | selected: false, 9 | isConsoleIntegrated: false, 10 | thumb: '', 11 | updateThumb: updateThumbMock, 12 | }; 13 | 14 | describe('DomainItem Component', () => { 15 | it('renders without crashing', async () => { 16 | const { container } = render(); 17 | expect(container.querySelector('.domItem')).toBeInTheDocument(); 18 | }); 19 | it('renders keywords count', async () => { 20 | const { container } = render(); 21 | const domStatskeywords = container.querySelector('.dom_stats div:nth-child(1)'); 22 | expect(domStatskeywords?.textContent).toBe('Keywords10'); 23 | }); 24 | it('renders avg position', async () => { 25 | const { container } = render(); 26 | const domStatsAvg = container.querySelector('.dom_stats div:nth-child(2)'); 27 | expect(domStatsAvg?.textContent).toBe('Avg position24'); 28 | }); 29 | it('updates domain thumbnail on relevant button click', async () => { 30 | const { container } = render(); 31 | const reloadThumbbBtn = container.querySelector('.domain_thumb button'); 32 | if (reloadThumbbBtn) fireEvent.click(reloadThumbbBtn); 33 | expect(updateThumbMock).toHaveBeenCalledWith(dummyDomain.domain); 34 | }); 35 | }); 36 | -------------------------------------------------------------------------------- /services/searchConsole.ts: -------------------------------------------------------------------------------- 1 | import { NextRouter } from 'next/router'; 2 | import { useQuery } from 'react-query'; 3 | 4 | export async function fetchSCKeywords(router: NextRouter) { 5 | // if (!router.query.slug) { throw new Error('Invalid Domain Name'); } 6 | const res = await fetch(`${window.location.origin}/api/searchconsole?domain=${router.query.slug}`, { method: 'GET' }); 7 | if (res.status >= 400 && res.status < 600) { 8 | if (res.status === 401) { 9 | console.log('Unauthorized!!'); 10 | router.push('/login'); 11 | } 12 | throw new Error('Bad response from server'); 13 | } 14 | return res.json(); 15 | } 16 | 17 | export function useFetchSCKeywords(router: NextRouter, domainLoaded: boolean = false) { 18 | // console.log('ROUTER: ', router); 19 | return useQuery('sckeywords', () => router.query.slug && fetchSCKeywords(router), { enabled: domainLoaded }); 20 | } 21 | 22 | export async function fetchSCInsight(router: NextRouter) { 23 | // if (!router.query.slug) { throw new Error('Invalid Domain Name'); } 24 | const res = await fetch(`${window.location.origin}/api/insight?domain=${router.query.slug}`, { method: 'GET' }); 25 | if (res.status >= 400 && res.status < 600) { 26 | if (res.status === 401) { 27 | console.log('Unauthorized!!'); 28 | router.push('/login'); 29 | } 30 | throw new Error('Bad response from server'); 31 | } 32 | return res.json(); 33 | } 34 | 35 | export function useFetchSCInsight(router: NextRouter, domainLoaded: boolean = false) { 36 | // console.log('ROUTER: ', router); 37 | return useQuery('scinsight', () => router.query.slug && fetchSCInsight(router), { enabled: domainLoaded }); 38 | } 39 | -------------------------------------------------------------------------------- /pages/api/cron.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from 'next'; 2 | import db from '../../database/database'; 3 | import Keyword from '../../database/models/keyword'; 4 | import { getAppSettings } from './settings'; 5 | import verifyUser from '../../utils/verifyUser'; 6 | import refreshAndUpdateKeywords from '../../utils/refresh'; 7 | 8 | type CRONRefreshRes = { 9 | started: boolean 10 | error?: string|null, 11 | } 12 | 13 | export default async function handler(req: NextApiRequest, res: NextApiResponse) { 14 | await db.sync(); 15 | const authorized = verifyUser(req, res); 16 | if (authorized !== 'authorized') { 17 | return res.status(401).json({ error: authorized }); 18 | } 19 | if (req.method === 'POST') { 20 | return cronRefreshkeywords(req, res); 21 | } 22 | return res.status(502).json({ error: 'Unrecognized Route.' }); 23 | } 24 | 25 | const cronRefreshkeywords = async (req: NextApiRequest, res: NextApiResponse) => { 26 | try { 27 | const settings = await getAppSettings(); 28 | if (!settings || (settings && settings.scraper_type === 'never')) { 29 | return res.status(400).json({ started: false, error: 'Scraper has not been set up yet.' }); 30 | } 31 | await Keyword.update({ updating: true }, { where: {} }); 32 | const keywordQueries: Keyword[] = await Keyword.findAll(); 33 | 34 | refreshAndUpdateKeywords(keywordQueries, settings); 35 | 36 | return res.status(200).json({ started: true }); 37 | } catch (error) { 38 | console.log('[ERROR] CRON Refreshing Keywords: ', error); 39 | return res.status(400).json({ started: false, error: 'CRON Error refreshing keywords!' }); 40 | } 41 | }; 42 | -------------------------------------------------------------------------------- /__mocks__/utils.tsx: -------------------------------------------------------------------------------- 1 | import { render } from '@testing-library/react'; 2 | import { http } from 'msw'; 3 | import * as React from 'react'; 4 | import { QueryClient, QueryClientProvider } from 'react-query'; 5 | 6 | export const handlers = [ 7 | http.get( 8 | '*/react-query', 9 | ({ request, params }) => { 10 | return new Response( 11 | JSON.stringify({ 12 | name: 'mocked-react-query', 13 | }), 14 | { 15 | status: 200, 16 | headers: { 17 | 'Content-Type': 'application/json', 18 | }, 19 | }, 20 | ); 21 | }, 22 | ), 23 | ]; 24 | const createTestQueryClient = () => new QueryClient({ 25 | defaultOptions: { 26 | queries: { 27 | retry: false, 28 | }, 29 | }, 30 | }); 31 | 32 | export function renderWithClient(ui: React.ReactElement) { 33 | const testQueryClient = createTestQueryClient(); 34 | const { rerender, ...result } = render( 35 | {ui}, 36 | ); 37 | return { 38 | ...result, 39 | rerender: (rerenderUi: React.ReactElement) => rerender( 40 | {rerenderUi}, 41 | ), 42 | }; 43 | } 44 | 45 | export function createWrapper() { 46 | const testQueryClient = createTestQueryClient(); 47 | // eslint-disable-next-line react/display-name 48 | return ({ children }: {children: React.ReactNode}) => ( 49 | {children} 50 | ); 51 | } 52 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:22.11.0-alpine3.20 AS deps 2 | ENV NPM_VERSION=10.3.0 3 | RUN npm install -g npm@"${NPM_VERSION}" 4 | WORKDIR /app 5 | 6 | COPY package.json ./ 7 | RUN npm install 8 | COPY . . 9 | 10 | 11 | FROM node:22.11.0-alpine3.20 AS builder 12 | WORKDIR /app 13 | ENV NPM_VERSION=10.3.0 14 | RUN npm install -g npm@"${NPM_VERSION}" 15 | COPY --from=deps /app ./ 16 | RUN rm -rf /app/data 17 | RUN rm -rf /app/__tests__ 18 | RUN rm -rf /app/__mocks__ 19 | RUN npm run build 20 | 21 | 22 | FROM node:22.11.0-alpine3.20 AS runner 23 | WORKDIR /app 24 | ENV NPM_VERSION=10.3.0 25 | RUN npm install -g npm@"${NPM_VERSION}" 26 | ENV NODE_ENV=production 27 | RUN addgroup --system --gid 1001 nodejs 28 | RUN adduser --system --uid 1001 nextjs 29 | RUN set -xe && mkdir -p /app/data && chown nextjs:nodejs /app/data 30 | COPY --from=builder --chown=nextjs:nodejs /app/public ./public 31 | # COPY --from=builder --chown=nextjs:nodejs /app/data ./data 32 | COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ 33 | COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static 34 | 35 | # setup the cron 36 | COPY --from=builder --chown=nextjs:nodejs /app/cron.js ./ 37 | COPY --from=builder --chown=nextjs:nodejs /app/email ./email 38 | COPY --from=builder --chown=nextjs:nodejs /app/database ./database 39 | COPY --from=builder --chown=nextjs:nodejs /app/.sequelizerc ./.sequelizerc 40 | COPY --from=builder --chown=nextjs:nodejs /app/entrypoint.sh ./entrypoint.sh 41 | RUN rm package.json 42 | RUN npm init -y 43 | RUN npm i cryptr@6.0.3 dotenv@16.0.3 croner@9.0.0 @googleapis/searchconsole@1.0.5 sequelize-cli@6.6.2 @isaacs/ttlcache@1.4.1 44 | RUN npm i -g concurrently 45 | 46 | USER nextjs 47 | 48 | EXPOSE 3000 49 | 50 | ENTRYPOINT ["/app/entrypoint.sh"] 51 | CMD ["concurrently","node server.js", "node cron.js"] -------------------------------------------------------------------------------- /components/common/ChartSlim.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Chart as ChartJS, CategoryScale, LinearScale, PointElement, LineElement, Title, Tooltip, Legend, Filler } from 'chart.js'; 3 | import { Line } from 'react-chartjs-2'; 4 | 5 | ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Filler, Title, Tooltip, Legend); 6 | 7 | type ChartProps ={ 8 | labels: string[], 9 | sreies: number[], 10 | noMaxLimit?: boolean, 11 | reverse?: boolean 12 | } 13 | 14 | const ChartSlim = ({ labels, sreies, noMaxLimit = false, reverse = true }:ChartProps) => { 15 | const options = { 16 | responsive: true, 17 | maintainAspectRatio: false, 18 | animation: false as const, 19 | scales: { 20 | y: { 21 | display: false, 22 | reverse, 23 | min: 1, 24 | max: noMaxLimit ? undefined : 100, 25 | }, 26 | x: { 27 | display: false, 28 | }, 29 | }, 30 | plugins: { 31 | tooltip: { 32 | enabled: false, 33 | }, 34 | legend: { 35 | display: false, 36 | }, 37 | }, 38 | }; 39 | 40 | return
41 | 58 |
; 59 | }; 60 | 61 | export default ChartSlim; 62 | -------------------------------------------------------------------------------- /scrapers/services/valueserp.ts: -------------------------------------------------------------------------------- 1 | import countries from '../../utils/countries'; 2 | 3 | interface ValueSerpResult { 4 | title: string, 5 | link: string, 6 | position: number, 7 | domain: string, 8 | } 9 | 10 | const valueSerp:ScraperSettings = { 11 | id: 'valueserp', 12 | name: 'Value Serp', 13 | website: 'valueserp.com', 14 | allowsCity: true, 15 | scrapeURL: (keyword, settings, countryData) => { 16 | const country = keyword.country || 'US'; 17 | const countryName = countries[country][0]; 18 | const location = keyword.city ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : ''; 19 | const device = keyword.device === 'mobile' ? '&device=mobile' : ''; 20 | const lang = countryData[country][2]; 21 | console.log(`https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`); 22 | return `https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`; 23 | }, 24 | resultObjectKey: 'organic_results', 25 | serpExtractor: (content) => { 26 | const extractedResult = []; 27 | const results: ValueSerpResult[] = (typeof content === 'string') ? JSON.parse(content) : content as ValueSerpResult[]; 28 | for (const result of results) { 29 | if (result.title && result.link) { 30 | extractedResult.push({ 31 | title: result.title, 32 | url: result.link, 33 | position: result.position, 34 | }); 35 | } 36 | } 37 | return extractedResult; 38 | }, 39 | }; 40 | 41 | export default valueSerp; 42 | -------------------------------------------------------------------------------- /pages/api/domain.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from 'next'; 2 | import Cryptr from 'cryptr'; 3 | import db from '../../database/database'; 4 | import Domain from '../../database/models/domain'; 5 | import verifyUser from '../../utils/verifyUser'; 6 | 7 | type DomainGetResponse = { 8 | domain?: DomainType | null 9 | error?: string|null, 10 | } 11 | 12 | export default async function handler(req: NextApiRequest, res: NextApiResponse) { 13 | const authorized = verifyUser(req, res); 14 | if (authorized === 'authorized' && req.method === 'GET') { 15 | await db.sync(); 16 | return getDomain(req, res); 17 | } 18 | return res.status(401).json({ error: authorized }); 19 | } 20 | 21 | const getDomain = async (req: NextApiRequest, res: NextApiResponse) => { 22 | if (!req.query.domain && typeof req.query.domain !== 'string') { 23 | return res.status(400).json({ error: 'Domain Name is Required!' }); 24 | } 25 | 26 | try { 27 | const query = { domain: req.query.domain as string }; 28 | const foundDomain:Domain| null = await Domain.findOne({ where: query }); 29 | const parsedDomain = foundDomain?.get({ plain: true }) || false; 30 | 31 | if (parsedDomain && parsedDomain.search_console) { 32 | try { 33 | const cryptr = new Cryptr(process.env.SECRET as string); 34 | const scData = JSON.parse(parsedDomain.search_console); 35 | scData.client_email = scData.client_email ? cryptr.decrypt(scData.client_email) : ''; 36 | scData.private_key = scData.private_key ? cryptr.decrypt(scData.private_key) : ''; 37 | parsedDomain.search_console = JSON.stringify(scData); 38 | } catch (error) { 39 | console.log('[Error] Parsing Search Console Keys.'); 40 | } 41 | } 42 | 43 | return res.status(200).json({ domain: parsedDomain }); 44 | } catch (error) { 45 | console.log('[ERROR] Getting Domain: ', error); 46 | return res.status(400).json({ error: 'Error Loading Domain' }); 47 | } 48 | }; 49 | -------------------------------------------------------------------------------- /components/settings/SearchConsoleSettings.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import InputField from '../common/InputField'; 3 | 4 | type SearchConsoleSettingsProps = { 5 | settings: SettingsType, 6 | settingsError: null | { 7 | type: string, 8 | msg: string 9 | }, 10 | updateSettings: Function, 11 | } 12 | 13 | const SearchConsoleSettings = ({ settings, settingsError, updateSettings }:SearchConsoleSettingsProps) => { 14 | return ( 15 |
16 |
17 | 18 | {/*
19 | updateSettings('scrape_retry', val)} 23 | /> 24 |
*/} 25 |
26 | updateSettings('search_console_client_email', client_email)} 29 | value={settings.search_console_client_email} 30 | placeholder='myapp@appspot.gserviceaccount.com' 31 | /> 32 |
33 |
34 | 35 | 63 | {newDomainError &&
{newDomainError}
} 64 |
65 | 66 | 69 |
70 |
71 | 72 | ); 73 | }; 74 | 75 | export default AddDomain; 76 | -------------------------------------------------------------------------------- /components/settings/Changelog.tsx: -------------------------------------------------------------------------------- 1 | import React, { useLayoutEffect, useMemo } from 'react'; 2 | import TimeAgo from 'react-timeago'; 3 | import dayjs from 'dayjs'; 4 | import SidePanel from '../common/SidePanel'; 5 | import { useFetchChangelog } from '../../services/misc'; 6 | import Icon from '../common/Icon'; 7 | 8 | const Markdown = React.lazy(() => import('react-markdown')); 9 | 10 | type ChangeLogProps = { 11 | closeChangeLog: Function, 12 | } 13 | 14 | const ChangeLogloader = () => { 15 | return ( 16 |
17 | 18 |
19 | ); 20 | }; 21 | 22 | const ChangeLog = ({ closeChangeLog }: ChangeLogProps) => { 23 | const { data: changeLogData, isLoading } = useFetchChangelog(); 24 | 25 | useLayoutEffect(() => { 26 | document.body.style.overflow = 'hidden'; 27 | // eslint-disable-next-line no-unused-expressions 28 | () => { 29 | console.log('run CleanUp !'); 30 | document.body.style.overflow = 'auto'; 31 | }; 32 | }, []); 33 | 34 | const onClose = () => { 35 | document.body.style.overflow = 'auto'; 36 | closeChangeLog(); 37 | }; 38 | 39 | const changeLogs = useMemo(() => { 40 | if (changeLogData && Array.isArray(changeLogData)) { 41 | return changeLogData.map(({ name = '', body, published_at }:{name: string, body: string, published_at: string}) => ({ 42 | version: name, 43 | major: !!(name.match(/v\d+\.0+\.0/)), 44 | date: published_at, 45 | content: body.replaceAll(/^(##|###) \[([^\]]+)\]\(([^)]+)\) \(([^)]+)\)/g, '') 46 | .replaceAll(/\(\[(.*?)\]\((https:\/\/github\.com\/towfiqi\/serpbear\/commit\/([a-f0-9]{40}))\)\)/g, ''), 47 | })); 48 | } 49 | return []; 50 | }, [changeLogData]); 51 | 52 | return 53 | }> 54 | {!isLoading && changeLogs.length > 0 && ( 55 |
56 | {changeLogs.map(({ version, content, date, major }) => { 57 | return ( 58 |
61 |

62 | 63 | {version} {major && Major} 64 | 65 | 66 | 67 | 68 |

69 |
{content}
70 |
71 | ); 72 | })} 73 |
74 | )} 75 | {isLoading && } 76 | 77 | ; 78 | }; 79 | 80 | export default ChangeLog; 81 | -------------------------------------------------------------------------------- /services/settings.ts: -------------------------------------------------------------------------------- 1 | import toast from 'react-hot-toast'; 2 | import { useMutation, useQuery, useQueryClient } from 'react-query'; 3 | 4 | export async function fetchSettings() { 5 | const res = await fetch(`${window.location.origin}/api/settings`, { method: 'GET' }); 6 | return res.json(); 7 | } 8 | 9 | export function useFetchSettings() { 10 | return useQuery('settings', () => fetchSettings()); 11 | } 12 | 13 | export const useUpdateSettings = (onSuccess:Function|undefined) => { 14 | const queryClient = useQueryClient(); 15 | 16 | return useMutation(async (settings: SettingsType) => { 17 | // console.log('settings: ', JSON.stringify(settings)); 18 | 19 | const headers = new Headers({ 'Content-Type': 'application/json', Accept: 'application/json' }); 20 | const fetchOpts = { method: 'PUT', headers, body: JSON.stringify({ settings }) }; 21 | const res = await fetch(`${window.location.origin}/api/settings`, fetchOpts); 22 | if (res.status >= 400 && res.status < 600) { 23 | throw new Error('Bad response from server'); 24 | } 25 | return res.json(); 26 | }, { 27 | onSuccess: async () => { 28 | if (onSuccess) { 29 | onSuccess(); 30 | } 31 | toast('Settings Updated!', { icon: '✔️' }); 32 | queryClient.invalidateQueries(['settings']); 33 | }, 34 | onError: () => { 35 | console.log('Error Updating App Settings!!!'); 36 | toast('Error Updating App Settings.', { icon: '⚠️' }); 37 | }, 38 | }); 39 | }; 40 | 41 | export function useClearFailedQueue(onSuccess:Function) { 42 | const queryClient = useQueryClient(); 43 | return useMutation(async () => { 44 | const headers = new Headers({ 'Content-Type': 'application/json', Accept: 'application/json' }); 45 | const fetchOpts = { method: 'PUT', headers }; 46 | const res = await fetch(`${window.location.origin}/api/clearfailed`, fetchOpts); 47 | if (res.status >= 400 && res.status < 600) { 48 | throw new Error('Bad response from server'); 49 | } 50 | return res.json(); 51 | }, { 52 | onSuccess: async () => { 53 | onSuccess(); 54 | toast('Failed Queue Cleared', { icon: '✔️' }); 55 | queryClient.invalidateQueries(['settings']); 56 | }, 57 | onError: () => { 58 | console.log('Error Clearing Failed Queue!!!'); 59 | toast('Error Clearing Failed Queue.', { icon: '⚠️' }); 60 | }, 61 | }); 62 | } 63 | 64 | export async function fetchMigrationStatus() { 65 | const res = await fetch(`${window.location.origin}/api/dbmigrate`, { method: 'GET' }); 66 | return res.json(); 67 | } 68 | 69 | export function useCheckMigrationStatus() { 70 | return useQuery('dbmigrate', () => fetchMigrationStatus()); 71 | } 72 | 73 | export const useMigrateDatabase = (onSuccess:Function|undefined) => { 74 | const queryClient = useQueryClient(); 75 | 76 | return useMutation(async () => { 77 | // console.log('settings: ', JSON.stringify(settings)); 78 | const res = await fetch(`${window.location.origin}/api/dbmigrate`, { method: 'POST' }); 79 | if (res.status >= 400 && res.status < 600) { 80 | throw new Error('Bad response from server'); 81 | } 82 | return res.json(); 83 | }, { 84 | onSuccess: async (res) => { 85 | if (onSuccess) { 86 | onSuccess(res); 87 | } 88 | toast('Database Updated!', { icon: '✔️' }); 89 | queryClient.invalidateQueries(['settings']); 90 | }, 91 | onError: () => { 92 | console.log('Error Updating Database!!!'); 93 | toast('Error Updating Database.', { icon: '⚠️' }); 94 | }, 95 | }); 96 | }; 97 | -------------------------------------------------------------------------------- /pages/api/notify.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from 'next'; 2 | import nodeMailer from 'nodemailer'; 3 | import db from '../../database/database'; 4 | import Domain from '../../database/models/domain'; 5 | import Keyword from '../../database/models/keyword'; 6 | import generateEmail from '../../utils/generateEmail'; 7 | import parseKeywords from '../../utils/parseKeywords'; 8 | import { getAppSettings } from './settings'; 9 | 10 | type NotifyResponse = { 11 | success?: boolean 12 | error?: string|null, 13 | } 14 | 15 | export default async function handler(req: NextApiRequest, res: NextApiResponse) { 16 | if (req.method === 'POST') { 17 | await db.sync(); 18 | return notify(req, res); 19 | } 20 | return res.status(401).json({ success: false, error: 'Invalid Method' }); 21 | } 22 | 23 | const notify = async (req: NextApiRequest, res: NextApiResponse) => { 24 | const reqDomain = req?.query?.domain as string || ''; 25 | try { 26 | const settings = await getAppSettings(); 27 | const { smtp_server = '', smtp_port = '', notification_email = '' } = settings; 28 | 29 | if (!smtp_server || !smtp_port || !notification_email) { 30 | return res.status(401).json({ success: false, error: 'SMTP has not been setup properly!' }); 31 | } 32 | 33 | if (reqDomain) { 34 | const theDomain = await Domain.findOne({ where: { domain: reqDomain } }); 35 | if (theDomain) { 36 | await sendNotificationEmail(theDomain, settings); 37 | } 38 | } else { 39 | const allDomains: Domain[] = await Domain.findAll(); 40 | if (allDomains && allDomains.length > 0) { 41 | const domains = allDomains.map((el) => el.get({ plain: true })); 42 | for (const domain of domains) { 43 | if (domain.notification !== false) { 44 | await sendNotificationEmail(domain, settings); 45 | } 46 | } 47 | } 48 | } 49 | 50 | return res.status(200).json({ success: true, error: null }); 51 | } catch (error) { 52 | console.log(error); 53 | return res.status(401).json({ success: false, error: 'Error Sending Notification Email.' }); 54 | } 55 | }; 56 | 57 | const sendNotificationEmail = async (domain: Domain, settings: SettingsType) => { 58 | const { 59 | smtp_server = '', 60 | smtp_port = '', 61 | smtp_username = '', 62 | smtp_password = '', 63 | notification_email = '', 64 | notification_email_from = '', 65 | notification_email_from_name = 'SerpBear', 66 | } = settings; 67 | 68 | const fromEmail = `${notification_email_from_name} <${notification_email_from || 'no-reply@serpbear.com'}>`; 69 | const mailerSettings:any = { host: smtp_server, port: parseInt(smtp_port, 10) }; 70 | if (smtp_username || smtp_password) { 71 | mailerSettings.auth = {}; 72 | if (smtp_username) mailerSettings.auth.user = smtp_username; 73 | if (smtp_password) mailerSettings.auth.pass = smtp_password; 74 | } 75 | const transporter = nodeMailer.createTransport(mailerSettings); 76 | const domainName = domain.domain; 77 | const query = { where: { domain: domainName } }; 78 | const domainKeywords:Keyword[] = await Keyword.findAll(query); 79 | const keywordsArray = domainKeywords.map((el) => el.get({ plain: true })); 80 | const keywords: KeywordType[] = parseKeywords(keywordsArray); 81 | const emailHTML = await generateEmail(domainName, keywords, settings); 82 | await transporter.sendMail({ 83 | from: fromEmail, 84 | to: domain.notification_emails || notification_email, 85 | subject: `[${domainName}] Keyword Positions Update`, 86 | html: emailHTML, 87 | }).catch((err:any) => console.log('[ERROR] Sending Notification Email for', domainName, err?.response || err)); 88 | }; 89 | -------------------------------------------------------------------------------- /components/ideas/KeywordIdea.tsx: -------------------------------------------------------------------------------- 1 | import React, { useMemo } from 'react'; 2 | import Icon from '../common/Icon'; 3 | import countries from '../../utils/countries'; 4 | import { formattedNum } from '../../utils/client/helpers'; 5 | import ChartSlim from '../common/ChartSlim'; 6 | 7 | type KeywordIdeaProps = { 8 | keywordData: IdeaKeyword, 9 | selected: boolean, 10 | lastItem?:boolean, 11 | isFavorite: boolean, 12 | style: Object, 13 | selectKeyword: Function, 14 | favoriteKeyword:Function, 15 | showKeywordDetails: Function 16 | } 17 | 18 | const KeywordIdea = (props: KeywordIdeaProps) => { 19 | const { keywordData, selected, lastItem, selectKeyword, style, isFavorite = false, favoriteKeyword, showKeywordDetails } = props; 20 | const { keyword, uid, position, country, monthlySearchVolumes, avgMonthlySearches, competition, competitionIndex } = keywordData; 21 | 22 | const chartData = useMemo(() => { 23 | const chartDataObj: { labels: string[], sreies: number[] } = { labels: [], sreies: [] }; 24 | Object.keys(monthlySearchVolumes).forEach((dateKey:string) => { 25 | chartDataObj.labels.push(dateKey); 26 | chartDataObj.sreies.push(parseInt(monthlySearchVolumes[dateKey], 10)); 27 | }); 28 | return chartDataObj; 29 | }, [monthlySearchVolumes]); 30 | 31 | return ( 32 |
37 | 38 |
39 | 46 | showKeywordDetails()}> 47 | {keyword} 48 | 49 | 54 |
55 | 56 |
57 | {formattedNum(avgMonthlySearches)}/month 58 |
59 | 60 |
showKeywordDetails()} 62 | className={`keyword_visits text-center hidden mt-4 mr-5 ml-5 cursor-pointer 63 | lg:flex-1 lg:m-0 lg:ml-10 max-w-[70px] lg:max-w-none lg:pr-5 lg:flex justify-center`}> 64 | {chartData.labels.length > 0 && } 65 |
66 | 67 |
68 |
69 | {competitionIndex}/100 70 | {competition} 71 |
72 |
73 |
74 | ); 75 | }; 76 | 77 | export default KeywordIdea; 78 | -------------------------------------------------------------------------------- /utils/client/exportcsv.ts: -------------------------------------------------------------------------------- 1 | import countries from '../countries'; 2 | 3 | /** 4 | * Generates CSV File form the given domain & keywords, and automatically downloads it. 5 | * @param {KeywordType[]} keywords - The keywords of the domain 6 | * @param {string} domain - The domain name. 7 | * @returns {void} 8 | */ 9 | const exportCSV = (keywords: KeywordType[] | SCKeywordType[], domain:string, scDataDuration = 'lastThreeDays') => { 10 | if (!keywords || (keywords && Array.isArray(keywords) && keywords.length === 0)) { return; } 11 | const isSCKeywords = !!(keywords && keywords[0] && keywords[0].uid); 12 | let csvHeader = 'ID,Keyword,Position,URL,Country,City,Device,Updated,Added,Tags\r\n'; 13 | let csvBody = ''; 14 | let fileName = `${domain}-keywords_serp.csv`; 15 | 16 | console.log(keywords[0]); 17 | console.log('isSCKeywords:', isSCKeywords); 18 | 19 | if (isSCKeywords) { 20 | csvHeader = 'ID,Keyword,Position,Impressions,Clicks,CTR,Country,Device\r\n'; 21 | fileName = `${domain}-search-console-${scDataDuration}.csv`; 22 | keywords.forEach((keywordData, index) => { 23 | const { keyword, position, country, device, clicks, impressions, ctr } = keywordData as SCKeywordType; 24 | // eslint-disable-next-line max-len 25 | csvBody += `${index}, ${keyword}, ${position === 0 ? '-' : position}, ${impressions}, ${clicks}, ${ctr}, ${countries[country][0]}, ${device}\r\n`; 26 | }); 27 | } else { 28 | keywords.forEach((keywordData) => { 29 | const { ID, keyword, position, url, country, city, device, lastUpdated, added, tags } = keywordData as KeywordType; 30 | // eslint-disable-next-line max-len 31 | csvBody += `${ID}, ${keyword}, ${position === 0 ? '-' : position}, ${url || '-'}, ${countries[country][0]}, ${city || '-'}, ${device}, ${lastUpdated}, ${added}, ${tags.join(',')}\r\n`; 32 | }); 33 | } 34 | 35 | downloadCSV(csvHeader, csvBody, fileName); 36 | }; 37 | 38 | /** 39 | * Generates CSV File form the given keyword Ideas, and automatically downloads it. 40 | * @param {IdeaKeyword[]} keywords - The keyword Ideas to export 41 | * @param {string} domainName - The domain name. 42 | * @returns {void} 43 | */ 44 | export const exportKeywordIdeas = (keywords: IdeaKeyword[], domainName:string) => { 45 | if (!keywords || (keywords && Array.isArray(keywords) && keywords.length === 0)) { return; } 46 | const csvHeader = 'Keyword,Volume,Competition,CompetitionScore,Country,Added\r\n'; 47 | let csvBody = ''; 48 | const fileName = `${domainName}-keyword_ideas.csv`; 49 | keywords.forEach((keywordData) => { 50 | const { keyword, competition, country, domain, competitionIndex, avgMonthlySearches, added, updated, position } = keywordData; 51 | // eslint-disable-next-line max-len 52 | const addedDate = new Intl.DateTimeFormat('en-US').format(new Date(added)); 53 | csvBody += `${keyword}, ${avgMonthlySearches}, ${competition}, ${competitionIndex}, ${countries[country][0]}, ${addedDate}\r\n`; 54 | }); 55 | downloadCSV(csvHeader, csvBody, fileName); 56 | }; 57 | 58 | /** 59 | * generates a CSV file with a specified header and body content and automatically downloads it. 60 | * @param {string} csvHeader - The `csvHeader` file header. A comma speperated csv header. 61 | * @param {string} csvBody - The content of the csv file. 62 | * @param {string} fileName - The file Name for the downlaoded csv file. 63 | */ 64 | const downloadCSV = (csvHeader:string, csvBody:string, fileName:string) => { 65 | const blob = new Blob([csvHeader + csvBody], { type: 'text/csv;charset=utf-8;' }); 66 | const url = URL.createObjectURL(blob); 67 | const link = document.createElement('a'); 68 | link.setAttribute('href', url); 69 | link.setAttribute('download', fileName); 70 | link.style.visibility = 'hidden'; 71 | document.body.appendChild(link); 72 | link.click(); 73 | document.body.removeChild(link); 74 | }; 75 | 76 | export default exportCSV; 77 | -------------------------------------------------------------------------------- /utils/client/SCsortFilter.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Sorrt Keywords by user's given input. 3 | * @param {SCKeywordType[]} theKeywords - The Keywords to sort. 4 | * @param {string} sortBy - The sort method. 5 | * @returns {SCKeywordType[]} 6 | */ 7 | export const SCsortKeywords = (theKeywords:SCKeywordType[], sortBy:string) : SCKeywordType[] => { 8 | let sortedItems = []; 9 | const keywords = theKeywords.map((k) => ({ ...k, position: k.position === 0 ? 111 : k.position })); 10 | switch (sortBy) { 11 | case 'imp_asc': 12 | sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.impressions > b.impressions ? 1 : -1)); 13 | break; 14 | case 'imp_desc': 15 | sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.impressions > a.impressions ? 1 : -1)); 16 | break; 17 | case 'visits_asc': 18 | sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.clicks > b.clicks ? 1 : -1)); 19 | break; 20 | case 'visits_desc': 21 | sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.clicks > a.clicks ? 1 : -1)); 22 | break; 23 | case 'ctr_asc': 24 | sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => b.ctr - a.ctr); 25 | break; 26 | case 'ctr_desc': 27 | sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => a.ctr - b.ctr); 28 | break; 29 | case 'pos_asc': 30 | sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.position < a.position ? 1 : -1)); 31 | sortedItems = sortedItems.map((k) => ({ ...k, position: k.position === 111 ? 0 : k.position })); 32 | break; 33 | case 'pos_desc': 34 | sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.position < b.position ? 1 : -1)); 35 | sortedItems = sortedItems.map((k) => ({ ...k, position: k.position === 111 ? 0 : k.position })); 36 | break; 37 | case 'alpha_desc': 38 | sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.keyword > a.keyword ? 1 : -1)); 39 | break; 40 | case 'alpha_asc': 41 | sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.keyword > b.keyword ? 1 : -1)); 42 | break; 43 | default: 44 | return theKeywords; 45 | } 46 | 47 | return sortedItems; 48 | }; 49 | 50 | /** 51 | * Filters the Keywords by Device when the Device buttons are switched 52 | * @param {SCKeywordType[]} sortedKeywords - The Sorted Keywords. 53 | * @param {string} device - Device name (desktop or mobile). 54 | * @returns {{desktop: SCKeywordType[], mobile: SCKeywordType[] } } 55 | */ 56 | export const SCkeywordsByDevice = (sortedKeywords: SCKeywordType[], device: string): {[key: string]: SCKeywordType[] } => { 57 | const deviceKeywords: {[key:string] : SCKeywordType[]} = { desktop: [], mobile: [] }; 58 | sortedKeywords.forEach((keyword) => { 59 | if (keyword.device === device) { deviceKeywords[device].push(keyword); } 60 | }); 61 | return deviceKeywords; 62 | }; 63 | 64 | /** 65 | * Filters the keywords by country, search string or tags. 66 | * @param {SCKeywordType[]} keywords - The keywords. 67 | * @param {KeywordFilters} filterParams - The user Selected filter object. 68 | * @returns {SCKeywordType[]} 69 | */ 70 | export const SCfilterKeywords = (keywords: SCKeywordType[], filterParams: KeywordFilters):SCKeywordType[] => { 71 | const filteredItems:SCKeywordType[] = []; 72 | keywords.forEach((keywrd) => { 73 | const countryMatch = filterParams.countries.length === 0 ? true : filterParams.countries && filterParams.countries.includes(keywrd.country); 74 | const searchMatch = !filterParams.search ? true : filterParams.search && keywrd.keyword.includes(filterParams.search); 75 | 76 | if (countryMatch && searchMatch) { 77 | filteredItems.push(keywrd); 78 | } 79 | }); 80 | 81 | return filteredItems; 82 | }; 83 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![SerpBear](https://i.imgur.com/0S2zIH3.png) 2 | 3 | # SerpBear 4 | 5 | ![Codacy Badge](https://app.codacy.com/project/badge/Grade/7e7a0030c3f84c6fb56a3ce6273fbc1d) ![GitHub](https://img.shields.io/github/license/towfiqi/serpbear) ![GitHub package.json version](https://img.shields.io/github/package-json/v/towfiqi/serpbear) ![Docker Pulls](https://img.shields.io/docker/pulls/towfiqi/serpbear) [![StandWithPalestine](https://raw.githubusercontent.com/Safouene1/support-palestine-banner/master/StandWithPalestine.svg)](https://www.youtube.com/watch?v=bjtDsd0g468&rco=1) 6 | 7 | #### [Documentation](https://docs.serpbear.com/) | [Changelog](https://github.com/towfiqi/serpbear/blob/main/CHANGELOG.md) | [Docker Image](https://hub.docker.com/r/towfiqi/serpbear) 8 | 9 | SerpBear is an Open Source Search Engine Position Tracking and Keyword Research App. It allows you to track your website's keyword positions in Google and get notified of their position change. 10 | 11 | ![Easy to Use Search Engine Rank Tracker](https://serpbear.b-cdn.net/serpbear_readme_v2.gif) 12 | 13 | #### Features 14 | 15 | - **Unlimited Keywords:** Add unlimited domains and unlimited keywords to track their SERP. 16 | - **Email Notification:** Get notified of your keyword position changes daily/weekly/monthly through email. 17 | - **SERP API:** SerpBear comes with built-in API that you can use for your marketing & data reporting tools. 18 | - **Keyword Research:** Ability to research keywords and auto-generate keyword ideas from your tracked website's content by integrating your Google Ads test account. 19 | - **Google Search Console Integration:** Get the actual visit count, impressions & more for Each keyword. 20 | - **Mobile App:** Add the PWA app to your mobile for a better mobile experience. 21 | - **Zero Cost to RUN:** Run the App on mogenius.com or Fly.io for free. 22 | 23 | #### How it Works 24 | 25 | The App uses third party website scrapers like ScrapingAnt, ScrapingRobot, SearchApi, SerpApi, HasData or Your given Proxy ips to scrape google search results to see if your domain appears in the search result for the given keyword. 26 | 27 | The Keyword Research and keyword generation feature works by integrating your Google Ads test accounts into SerpBear. You can also view the added keyword's monthly search volume data once you [integrate Google Ads](https://docs.serpbear.com/miscellaneous/integrate-google-ads). 28 | 29 | When you [integrate Google Search Console](https://docs.serpbear.com/miscellaneous/integrate-google-search-console), the app shows actual search visits for each tracked keywords. You can also discover new keywords, and find the most performing keywords, countries, pages.you will be able to view the actual visits count from Google Search for the tracked keywords. 30 | 31 | #### Getting Started 32 | 33 | - **Step 1:** Deploy & Run the App. 34 | - **Step 2:** Access your App and Login. 35 | - **Step 3:** Add your First domain. 36 | - **Step 4:** Get a free API key from ScrapingRobot or select a paid provider (see below) . Skip if you want to use Proxy ips. 37 | - **Step 5:** Setup the Scraping API/Proxy from the App's Settings interface. 38 | - **Step 6:** Add your keywords and start tracking. 39 | - **Step 7:** Optional. From the Settings panel, setup SMTP details to get notified of your keywords positions through email. You can use ElasticEmail and Sendpulse SMTP services that are free. 40 | 41 | #### SerpBear Integrates with popular SERP scraping services 42 | 43 | If you don't want to use proxies, you can use third party Scraping services to scrape Google Search results. 44 | 45 | 46 | | Service | Cost | SERP Lookup | API | 47 | | ----------------- | ------------- | -------------- | --- | 48 | | scrapingrobot.com | Free | 5000/mo | Yes | 49 | | serply.io | $49/mo | 5000/mo | Yes | 50 | | serpapi.com | From $50/mo | From 5,000/mo | Yes | 51 | | spaceserp.com | $59/lifetime | 15,000/mo | Yes | 52 | | SearchApi.io | From $40/mo | From 10,000/mo | Yes | 53 | | valueserp.com | Pay As You Go | $2.50/1000 req | No | 54 | | serper.dev | Pay As You Go | $1.00/1000 req | No | 55 | | hasdata.com | From $29/mo | From 10,000/mo | Yes | 56 | 57 | **Tech Stack** 58 | 59 | - Next.js for Frontend & Backend. 60 | - Sqlite for Database. 61 | -------------------------------------------------------------------------------- /components/keywords/AddTags.tsx: -------------------------------------------------------------------------------- 1 | import { useRef, useState } from 'react'; 2 | import { useUpdateKeywordTags } from '../../services/keywords'; 3 | import Icon from '../common/Icon'; 4 | import Modal from '../common/Modal'; 5 | 6 | type AddTagsProps = { 7 | keywords: KeywordType[], 8 | existingTags: string[], 9 | closeModal: Function 10 | } 11 | 12 | const AddTags = ({ keywords = [], existingTags = [], closeModal }: AddTagsProps) => { 13 | const [tagInput, setTagInput] = useState(() => (keywords.length === 1 ? keywords[0].tags.join(', ') : '')); 14 | const [inputError, setInputError] = useState(''); 15 | const [showSuggestions, setShowSuggestions] = useState(false); 16 | const { mutate: updateMutate } = useUpdateKeywordTags(() => { setTagInput(''); }); 17 | const inputRef = useRef(null); 18 | 19 | const addTag = () => { 20 | if (keywords.length === 0) { return; } 21 | if (!tagInput && keywords.length > 1) { 22 | setInputError('Please Insert a Tag!'); 23 | setTimeout(() => { setInputError(''); }, 3000); 24 | return; 25 | } 26 | 27 | const tagsArray = tagInput.split(',').map((t) => t.trim()); 28 | const tagsPayload:any = {}; 29 | keywords.forEach((keyword:KeywordType) => { 30 | tagsPayload[keyword.ID] = keywords.length === 1 ? tagsArray : [...(new Set([...keyword.tags, ...tagsArray]))]; 31 | }); 32 | updateMutate({ tags: tagsPayload }); 33 | }; 34 | 35 | return ( 36 | { closeModal(false); }} title={`Add New Tags to ${keywords.length} Selected Keyword`}> 37 |
38 | {inputError && {inputError}} 39 | setShowSuggestions(!showSuggestions)}> 40 | 41 | 42 | 43 | setTagInput(e.target.value)} 49 | onKeyDown={(e) => { 50 | if (e.code === 'Enter') { 51 | e.preventDefault(); 52 | addTag(); 53 | } 54 | }} 55 | /> 56 | {showSuggestions && ( 57 |
    59 | {existingTags.length > 0 && existingTags.map((tag, index) => { 60 | return tagInput.split(',').map((t) => t.trim()).includes(tag) === false &&
  • { 64 | // eslint-disable-next-line no-nested-ternary 65 | const tagToInsert = tagInput + (tagInput.trim().slice(-1) === ',' ? '' : (tagInput.trim() ? ', ' : '')) + tag; 66 | setTagInput(tagToInsert); 67 | setShowSuggestions(false); 68 | if (inputRef?.current) (inputRef.current as HTMLInputElement).focus(); 69 | }}> 70 | {tag} 71 |
    • ; 72 | })} 73 | {existingTags.length === 0 &&

    No Existing Tags Found...

    } 74 |
75 | )} 76 | 77 | 82 |
83 | 84 | 85 | ); 86 | }; 87 | 88 | export default AddTags; 89 | -------------------------------------------------------------------------------- /components/common/TopBar.tsx: -------------------------------------------------------------------------------- 1 | import Link from 'next/link'; 2 | import { useRouter } from 'next/router'; 3 | import React, { useState } from 'react'; 4 | import toast from 'react-hot-toast'; 5 | import Icon from './Icon'; 6 | 7 | type TopbarProps = { 8 | showSettings: Function, 9 | showAddModal: Function, 10 | } 11 | 12 | const TopBar = ({ showSettings, showAddModal }:TopbarProps) => { 13 | const [showMobileMenu, setShowMobileMenu] = useState(false); 14 | const router = useRouter(); 15 | const isDomainsPage = router.pathname === '/domains'; 16 | 17 | const logoutUser = async () => { 18 | try { 19 | const fetchOpts = { method: 'POST', headers: new Headers({ 'Content-Type': 'application/json', Accept: 'application/json' }) }; 20 | const res = await fetch(`${window.location.origin}/api/logout`, fetchOpts).then((result) => result.json()); 21 | console.log(res); 22 | if (!res.success) { 23 | toast(res.error, { icon: '⚠️' }); 24 | } else { 25 | router.push('/login'); 26 | } 27 | } catch (fetchError) { 28 | toast('Could not login, Ther Server is not responsive.', { icon: '⚠️' }); 29 | } 30 | }; 31 | 32 | return ( 33 |
35 | 36 |

37 | SerpBear 38 | 39 |

40 | {!isDomainsPage && router.asPath !== '/research' && ( 41 | 42 | 44 | 45 | 46 | 47 | )} 48 |
49 | 52 | 85 |
86 |
87 | ); 88 | }; 89 | 90 | export default TopBar; 91 | -------------------------------------------------------------------------------- /components/domains/DomainItem.tsx: -------------------------------------------------------------------------------- 1 | /* eslint-disable @next/next/no-img-element */ 2 | // import { useRouter } from 'next/router'; 3 | // import { useState } from 'react'; 4 | import TimeAgo from 'react-timeago'; 5 | import dayjs from 'dayjs'; 6 | import Link from 'next/link'; 7 | import Icon from '../common/Icon'; 8 | 9 | type DomainItemProps = { 10 | domain: DomainType, 11 | selected: boolean, 12 | isConsoleIntegrated: boolean, 13 | thumb: string, 14 | updateThumb: Function, 15 | } 16 | 17 | const DomainItem = ({ domain, selected, isConsoleIntegrated = false, thumb, updateThumb }: DomainItemProps) => { 18 | const { keywordsUpdated, slug, keywordCount = 0, avgPosition = 0, scVisits = 0, scImpressions = 0, scPosition = 0 } = domain; 19 | // const router = useRouter(); 20 | return ( 21 | 86 | ); 87 | }; 88 | 89 | export default DomainItem; 90 | -------------------------------------------------------------------------------- /pages/domain/insight/[slug]/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { useMemo, useState } from 'react'; 2 | import type { NextPage } from 'next'; 3 | import Head from 'next/head'; 4 | import { useRouter } from 'next/router'; 5 | // import { useQuery } from 'react-query'; 6 | // import toast from 'react-hot-toast'; 7 | import { CSSTransition } from 'react-transition-group'; 8 | import Sidebar from '../../../../components/common/Sidebar'; 9 | import TopBar from '../../../../components/common/TopBar'; 10 | import DomainHeader from '../../../../components/domains/DomainHeader'; 11 | import AddDomain from '../../../../components/domains/AddDomain'; 12 | import DomainSettings from '../../../../components/domains/DomainSettings'; 13 | import exportCSV from '../../../../utils/client/exportcsv'; 14 | import Settings from '../../../../components/settings/Settings'; 15 | import { useFetchDomains } from '../../../../services/domains'; 16 | import { useFetchSCInsight } from '../../../../services/searchConsole'; 17 | import SCInsight from '../../../../components/insight/Insight'; 18 | import { useFetchSettings } from '../../../../services/settings'; 19 | import Footer from '../../../../components/common/Footer'; 20 | 21 | const InsightPage: NextPage = () => { 22 | const router = useRouter(); 23 | const [showDomainSettings, setShowDomainSettings] = useState(false); 24 | const [showSettings, setShowSettings] = useState(false); 25 | const [showAddDomain, setShowAddDomain] = useState(false); 26 | const [scDateFilter, setSCDateFilter] = useState('thirtyDays'); 27 | const { data: appSettings } = useFetchSettings(); 28 | const { data: domainsData } = useFetchDomains(router); 29 | const scConnected = !!(appSettings && appSettings?.settings?.search_console_integrated); 30 | const { data: insightData } = useFetchSCInsight(router, !!(domainsData?.domains?.length) && scConnected); 31 | 32 | const theDomains: DomainType[] = (domainsData && domainsData.domains) || []; 33 | const theInsight: InsightDataType = insightData && insightData.data ? insightData.data : {}; 34 | 35 | const activDomain: DomainType|null = useMemo(() => { 36 | let active:DomainType|null = null; 37 | if (domainsData?.domains && router.query?.slug) { 38 | active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug) || null; 39 | } 40 | return active; 41 | }, [router.query.slug, domainsData]); 42 | 43 | const domainHasScAPI = useMemo(() => { 44 | const doaminSc = activDomain?.search_console ? JSON.parse(activDomain.search_console) : {}; 45 | return !!(doaminSc?.client_email && doaminSc?.private_key); 46 | }, [activDomain]); 47 | 48 | return ( 49 |
50 | {activDomain && activDomain.domain 51 | && 52 | {`${activDomain.domain} - SerpBear` } 53 | 54 | } 55 | setShowSettings(true)} showAddModal={() => setShowAddDomain(true)} /> 56 |
57 | setShowAddDomain(true)} /> 58 |
59 | {activDomain && activDomain.domain 60 | ? console.log('XXXXX')} 64 | showSettingsModal={setShowDomainSettings} 65 | exportCsv={() => exportCSV([], activDomain.domain, scDateFilter)} 66 | scFilter={scDateFilter} 67 | setScFilter={(item:string) => setSCDateFilter(item)} 68 | /> 69 | :
70 | } 71 | 77 |
78 |
79 | 80 | 81 | setShowAddDomain(false)} domains={domainsData?.domains || []} /> 82 | 83 | 84 | 85 | 89 | 90 | 91 | setShowSettings(false)} /> 92 | 93 |
94 |
95 | ); 96 | }; 97 | 98 | export default InsightPage; 99 | -------------------------------------------------------------------------------- /pages/api/ideas.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from 'next'; 2 | import db from '../../database/database'; 3 | import verifyUser from '../../utils/verifyUser'; 4 | import { 5 | KeywordIdeasDatabase, getAdwordsCredentials, getAdwordsKeywordIdeas, getLocalKeywordIdeas, updateLocalKeywordIdeas, 6 | } from '../../utils/adwords'; 7 | 8 | type keywordsIdeasUpdateResp = { 9 | keywords: IdeaKeyword[], 10 | error?: string|null, 11 | } 12 | 13 | type keywordsIdeasGetResp = { 14 | data: KeywordIdeasDatabase|null, 15 | error?: string|null, 16 | } 17 | 18 | export default async function handler(req: NextApiRequest, res: NextApiResponse) { 19 | await db.sync(); 20 | const authorized = verifyUser(req, res); 21 | if (authorized !== 'authorized') { 22 | return res.status(401).json({ error: authorized }); 23 | } 24 | if (req.method === 'GET') { 25 | return getKeywordIdeas(req, res); 26 | } 27 | if (req.method === 'POST') { 28 | return updateKeywordIdeas(req, res); 29 | } 30 | if (req.method === 'PUT') { 31 | return favoriteKeywords(req, res); 32 | } 33 | return res.status(502).json({ error: 'Unrecognized Route.' }); 34 | } 35 | 36 | const getKeywordIdeas = async (req: NextApiRequest, res: NextApiResponse) => { 37 | try { 38 | const domain = req.query.domain as string; 39 | if (domain) { 40 | const keywordsDatabase = await getLocalKeywordIdeas(domain); 41 | // console.log('keywords :', keywordsDatabase); 42 | if (keywordsDatabase) { 43 | return res.status(200).json({ data: keywordsDatabase }); 44 | } 45 | } 46 | return res.status(400).json({ data: null, error: 'Error Loading Keyword Ideas.' }); 47 | } catch (error) { 48 | console.log('[ERROR] Fetching Keyword Ideas: ', error); 49 | return res.status(400).json({ data: null, error: 'Error Loading Keyword Ideas.' }); 50 | } 51 | }; 52 | 53 | const updateKeywordIdeas = async (req: NextApiRequest, res: NextApiResponse) => { 54 | const errMsg = 'Error Fetching Keywords. Please try again!'; 55 | const { keywords = [], country = 'US', language = '1000', domain = '', seedSCKeywords, seedCurrentKeywords, seedType } = req.body; 56 | 57 | if (!country || !language) { 58 | return res.status(400).json({ keywords: [], error: 'Error Fetching Keywords. Please Provide a Country and Language' }); 59 | } 60 | if (seedType === 'custom' && (keywords.length === 0 && !seedSCKeywords && !seedCurrentKeywords)) { 61 | return res.status(400).json({ keywords: [], error: 'Error Fetching Keywords. Please Provide one of these: keywords, url or domain' }); 62 | } 63 | try { 64 | const adwordsCreds = await getAdwordsCredentials(); 65 | const { client_id, client_secret, developer_token, account_id, refresh_token } = adwordsCreds || {}; 66 | if (adwordsCreds && client_id && client_secret && developer_token && account_id && refresh_token) { 67 | const ideaOptions = { country, language, keywords, domain, seedSCKeywords, seedCurrentKeywords, seedType }; 68 | const keywordIdeas = await getAdwordsKeywordIdeas(adwordsCreds, ideaOptions); 69 | if (keywordIdeas && Array.isArray(keywordIdeas) && keywordIdeas.length > 1) { 70 | return res.status(200).json({ keywords: keywordIdeas }); 71 | } 72 | } 73 | return res.status(400).json({ keywords: [], error: errMsg }); 74 | } catch (error) { 75 | console.log('[ERROR] Fetching Keyword Ideas: ', error); 76 | return res.status(400).json({ keywords: [], error: errMsg }); 77 | } 78 | }; 79 | 80 | const favoriteKeywords = async (req: NextApiRequest, res: NextApiResponse) => { 81 | const errMsg = 'Error Favorating Keyword Idea. Please try again!'; 82 | const { keywordID = '', domain = '' } = req.body; 83 | 84 | if (!keywordID || !domain) { 85 | return res.status(400).json({ keywords: [], error: 'Missing Necessary data. Please provide both keywordID and domain values.' }); 86 | } 87 | 88 | try { 89 | const keywordsDatabase = await getLocalKeywordIdeas(domain); 90 | if (keywordsDatabase && keywordsDatabase.keywords) { 91 | const theKeyword = keywordsDatabase.keywords.find((kw) => kw.uid === keywordID); 92 | const existingKeywords = keywordsDatabase.favorites || []; 93 | const newFavorites = [...existingKeywords]; 94 | const existingKeywordIndex = newFavorites.findIndex((kw) => kw.uid === keywordID); 95 | if (existingKeywordIndex > -1) { 96 | newFavorites.splice(existingKeywordIndex, 1); 97 | } else if (theKeyword) newFavorites.push(theKeyword); 98 | 99 | const updated = await updateLocalKeywordIdeas(domain, { favorites: newFavorites }); 100 | 101 | if (updated) { 102 | return res.status(200).json({ keywords: newFavorites, error: '' }); 103 | } 104 | } 105 | 106 | return res.status(400).json({ keywords: [], error: errMsg }); 107 | } catch (error) { 108 | console.log('[ERROR] Favorating Keyword Idea: ', error); 109 | return res.status(400).json({ keywords: [], error: errMsg }); 110 | } 111 | }; 112 | -------------------------------------------------------------------------------- /pages/api/refresh.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from 'next'; 2 | import { Op } from 'sequelize'; 3 | import db from '../../database/database'; 4 | import Keyword from '../../database/models/keyword'; 5 | import refreshAndUpdateKeywords from '../../utils/refresh'; 6 | import { getAppSettings } from './settings'; 7 | import verifyUser from '../../utils/verifyUser'; 8 | import parseKeywords from '../../utils/parseKeywords'; 9 | import { scrapeKeywordFromGoogle } from '../../utils/scraper'; 10 | 11 | type KeywordsRefreshRes = { 12 | keywords?: KeywordType[] 13 | error?: string|null, 14 | } 15 | 16 | type KeywordSearchResultRes = { 17 | searchResult?: { 18 | results: { title: string, url: string, position: number }[], 19 | keyword: string, 20 | position: number, 21 | country: string, 22 | }, 23 | error?: string|null, 24 | } 25 | 26 | export default async function handler(req: NextApiRequest, res: NextApiResponse) { 27 | await db.sync(); 28 | const authorized = verifyUser(req, res); 29 | if (authorized !== 'authorized') { 30 | return res.status(401).json({ error: authorized }); 31 | } 32 | if (req.method === 'GET') { 33 | return getKeywordSearchResults(req, res); 34 | } 35 | if (req.method === 'POST') { 36 | return refresTheKeywords(req, res); 37 | } 38 | return res.status(502).json({ error: 'Unrecognized Route.' }); 39 | } 40 | 41 | const refresTheKeywords = async (req: NextApiRequest, res: NextApiResponse) => { 42 | if (!req.query.id && typeof req.query.id !== 'string') { 43 | return res.status(400).json({ error: 'keyword ID is Required!' }); 44 | } 45 | if (req.query.id === 'all' && !req.query.domain) { 46 | return res.status(400).json({ error: 'When Refreshing all Keywords of a domian, the Domain name Must be provided.' }); 47 | } 48 | const keywordIDs = req.query.id !== 'all' && (req.query.id as string).split(',').map((item) => parseInt(item, 10)); 49 | const { domain } = req.query || {}; 50 | console.log('keywordIDs: ', keywordIDs); 51 | 52 | try { 53 | const settings = await getAppSettings(); 54 | if (!settings || (settings && settings.scraper_type === 'never')) { 55 | return res.status(400).json({ error: 'Scraper has not been set up yet.' }); 56 | } 57 | const query = req.query.id === 'all' && domain ? { domain } : { ID: { [Op.in]: keywordIDs } }; 58 | await Keyword.update({ updating: true }, { where: query }); 59 | const keywordQueries: Keyword[] = await Keyword.findAll({ where: query }); 60 | 61 | let keywords = []; 62 | 63 | // If Single Keyword wait for the scraping process, 64 | // else, Process the task in background. Do not wait. 65 | if (keywordIDs && keywordIDs.length === 0) { 66 | const refreshed: KeywordType[] = await refreshAndUpdateKeywords(keywordQueries, settings); 67 | keywords = refreshed; 68 | } else { 69 | refreshAndUpdateKeywords(keywordQueries, settings); 70 | keywords = parseKeywords(keywordQueries.map((el) => el.get({ plain: true }))); 71 | } 72 | 73 | return res.status(200).json({ keywords }); 74 | } catch (error) { 75 | console.log('ERROR refresThehKeywords: ', error); 76 | return res.status(400).json({ error: 'Error refreshing keywords!' }); 77 | } 78 | }; 79 | 80 | const getKeywordSearchResults = async (req: NextApiRequest, res: NextApiResponse) => { 81 | if (!req.query.keyword || !req.query.country || !req.query.device) { 82 | return res.status(400).json({ error: 'A Valid keyword, Country Code, and device is Required!' }); 83 | } 84 | try { 85 | const settings = await getAppSettings(); 86 | if (!settings || (settings && settings.scraper_type === 'never')) { 87 | return res.status(400).json({ error: 'Scraper has not been set up yet.' }); 88 | } 89 | const dummyKeyword:KeywordType = { 90 | ID: 99999999999999, 91 | keyword: req.query.keyword as string, 92 | device: 'desktop', 93 | country: req.query.country as string, 94 | domain: '', 95 | lastUpdated: '', 96 | volume: 0, 97 | added: '', 98 | position: 111, 99 | sticky: false, 100 | history: {}, 101 | lastResult: [], 102 | url: '', 103 | tags: [], 104 | updating: false, 105 | lastUpdateError: false, 106 | }; 107 | const scrapeResult = await scrapeKeywordFromGoogle(dummyKeyword, settings); 108 | if (scrapeResult && !scrapeResult.error) { 109 | const searchResult = { 110 | results: scrapeResult.result, 111 | keyword: scrapeResult.keyword, 112 | position: scrapeResult.position !== 111 ? scrapeResult.position : 0, 113 | country: req.query.country as string, 114 | }; 115 | return res.status(200).json({ error: '', searchResult }); 116 | } 117 | return res.status(400).json({ error: 'Error Scraping Search Results for the given keyword!' }); 118 | } catch (error) { 119 | console.log('ERROR refresThehKeywords: ', error); 120 | return res.status(400).json({ error: 'Error refreshing keywords!' }); 121 | } 122 | }; 123 | -------------------------------------------------------------------------------- /pages/login/index.tsx: -------------------------------------------------------------------------------- 1 | import type { NextPage } from 'next'; 2 | import Head from 'next/head'; 3 | import { useRouter } from 'next/router'; 4 | import { useState } from 'react'; 5 | import Icon from '../../components/common/Icon'; 6 | 7 | type LoginError = { 8 | type: string, 9 | msg: string, 10 | } 11 | 12 | const Login: NextPage = () => { 13 | const [error, setError] = useState(null); 14 | const [username, setUsername] = useState(''); 15 | const [password, setPassword] = useState(''); 16 | const router = useRouter(); 17 | 18 | const loginuser = async () => { 19 | let loginError: LoginError |null = null; 20 | if (!username || !password) { 21 | if (!username && !password) { 22 | loginError = { type: 'empty_username_password', msg: 'Please Insert Your App Username & Password to login.' }; 23 | } 24 | if (!username && password) { 25 | loginError = { type: 'empty_username', msg: 'Please Insert Your App Username' }; 26 | } 27 | if (!password && username) { 28 | loginError = { type: 'empty_password', msg: 'Please Insert Your App Password' }; 29 | } 30 | setError(loginError); 31 | setTimeout(() => { setError(null); }, 3000); 32 | } else { 33 | try { 34 | const header = new Headers({ 'Content-Type': 'application/json', Accept: 'application/json' }); 35 | const fetchOpts = { method: 'POST', headers: header, body: JSON.stringify({ username, password }) }; 36 | const fetchRoute = `${window.location.origin}/api/login`; 37 | const res = await fetch(fetchRoute, fetchOpts).then((result) => result.json()); 38 | // console.log(res); 39 | if (!res.success) { 40 | let errorType = ''; 41 | if (res.error && res.error.toLowerCase().includes('username')) { 42 | errorType = 'incorrect_username'; 43 | } 44 | if (res.error && res.error.toLowerCase().includes('password')) { 45 | errorType = 'incorrect_password'; 46 | } 47 | setError({ type: errorType, msg: res.error }); 48 | setTimeout(() => { setError(null); }, 3000); 49 | } else { 50 | router.push('/'); 51 | } 52 | } catch (fetchError) { 53 | setError({ type: 'unknown', msg: 'Could not login, Ther Server is not responsive.' }); 54 | setTimeout(() => { setError(null); }, 3000); 55 | } 56 | } 57 | }; 58 | 59 | const labelStyle = 'mb-2 font-semibold inline-block text-sm text-gray-700'; 60 | // eslint-disable-next-line max-len 61 | const inputStyle = 'w-full p-2 border border-gray-200 rounded mb-3 focus:outline-none focus:border-blue-200'; 62 | const errorBorderStyle = 'border-red-400 focus:border-red-400'; 63 | return ( 64 |
65 | 66 | Login - SerpBear 67 | 68 |
69 |
70 |

71 | 72 | 73 | SerpBear 74 |

75 |
76 |
77 | 78 | setUsername(event.target.value)} 86 | /> 87 |
88 |
89 | 90 | setPassword(event.target.value)} 98 | /> 99 |
100 | 105 | {error && error.msg 106 | &&
108 | {error.msg} 109 |
110 | } 111 |
112 |
113 |
114 | 115 |
116 | ); 117 | }; 118 | 119 | export default Login; 120 | -------------------------------------------------------------------------------- /components/settings/NotificationSettings.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import SelectField from '../common/SelectField'; 3 | import SecretField from '../common/SecretField'; 4 | import InputField from '../common/InputField'; 5 | 6 | type NotificationSettingsProps = { 7 | settings: SettingsType, 8 | settingsError: null | { 9 | type: string, 10 | msg: string 11 | }, 12 | updateSettings: Function, 13 | } 14 | 15 | const NotificationSettings = ({ settings, settingsError, updateSettings }:NotificationSettingsProps) => { 16 | const labelStyle = 'mb-2 font-semibold inline-block text-sm text-gray-700 capitalize'; 17 | 18 | return ( 19 |
20 |
21 |
22 | updated[0] && updateSettings('notification_interval', updated[0])} 34 | rounded='rounded' 35 | maxHeight={48} 36 | minWidth={220} 37 | /> 38 |
39 | {settings.notification_interval !== 'never' && ( 40 | <> 41 |
42 | updateSettings('notification_email', value)} 48 | /> 49 |
50 |
51 | updateSettings('smtp_server', value)} 57 | /> 58 |
59 |
60 | updateSettings('smtp_port', value)} 66 | /> 67 |
68 |
69 | updateSettings('smtp_username', value)} 74 | /> 75 |
76 |
77 | updateSettings('smtp_password', value)} 81 | /> 82 |
83 |
84 | updateSettings('notification_email_from', value)} 90 | /> 91 |
92 |
93 | updateSettings('notification_email_from_name', value)} 99 | /> 100 |
101 | 102 | )} 103 | 104 |
105 | {settingsError?.msg && ( 106 |
107 | {settingsError.msg} 108 |
109 | )} 110 |
111 | ); 112 | }; 113 | 114 | export default NotificationSettings; 115 | -------------------------------------------------------------------------------- /pages/api/adwords.ts: -------------------------------------------------------------------------------- 1 | import type { NextApiRequest, NextApiResponse } from 'next'; 2 | import { OAuth2Client } from 'google-auth-library'; 3 | import { readFile, writeFile } from 'fs/promises'; 4 | import Cryptr from 'cryptr'; 5 | import db from '../../database/database'; 6 | import verifyUser from '../../utils/verifyUser'; 7 | import { getAdwordsCredentials, getAdwordsKeywordIdeas } from '../../utils/adwords'; 8 | 9 | type adwordsValidateResp = { 10 | valid: boolean 11 | error?: string|null, 12 | } 13 | 14 | export default async function handler(req: NextApiRequest, res: NextApiResponse) { 15 | await db.sync(); 16 | const authorized = verifyUser(req, res); 17 | if (authorized !== 'authorized') { 18 | return res.status(401).json({ error: authorized }); 19 | } 20 | if (req.method === 'GET') { 21 | return getAdwordsRefreshToken(req, res); 22 | } 23 | if (req.method === 'POST') { 24 | return validateAdwordsIntegration(req, res); 25 | } 26 | return res.status(502).json({ error: 'Unrecognized Route.' }); 27 | } 28 | 29 | const getAdwordsRefreshToken = async (req: NextApiRequest, res: NextApiResponse) => { 30 | try { 31 | const code = (req.query.code as string); 32 | const https = req.headers.host?.includes('localhost:') ? 'http://' : 'https://'; 33 | const redirectURL = `${https}${req.headers.host}/api/adwords`; 34 | 35 | if (code) { 36 | try { 37 | const settingsRaw = await readFile(`${process.cwd()}/data/settings.json`, { encoding: 'utf-8' }); 38 | const settings: SettingsType = settingsRaw ? JSON.parse(settingsRaw) : {}; 39 | const cryptr = new Cryptr(process.env.SECRET as string); 40 | const adwords_client_id = settings.adwords_client_id ? cryptr.decrypt(settings.adwords_client_id) : ''; 41 | const adwords_client_secret = settings.adwords_client_secret ? cryptr.decrypt(settings.adwords_client_secret) : ''; 42 | const oAuth2Client = new OAuth2Client(adwords_client_id, adwords_client_secret, redirectURL); 43 | const r = await oAuth2Client.getToken(code); 44 | if (r?.tokens?.refresh_token) { 45 | const adwords_refresh_token = cryptr.encrypt(r.tokens.refresh_token); 46 | await writeFile(`${process.cwd()}/data/settings.json`, JSON.stringify({ ...settings, adwords_refresh_token }), { encoding: 'utf-8' }); 47 | return res.status(200).send('Google Ads Intergrated Successfully! You can close this window.'); 48 | } 49 | return res.status(400).send('Error Getting the Google Ads Refresh Token. Please Try Again!'); 50 | } catch (error:any) { 51 | let errorMsg = error?.response?.data?.error; 52 | if (errorMsg.includes('redirect_uri_mismatch')) { 53 | errorMsg += ` Redirected URL: ${redirectURL}`; 54 | } 55 | console.log('[Error] Getting Google Ads Refresh Token! Reason: ', errorMsg); 56 | return res.status(400).send(`Error Saving the Google Ads Refresh Token ${errorMsg ? `. Details: ${errorMsg}` : ''}. Please Try Again!`); 57 | } 58 | } else { 59 | return res.status(400).send('No Code Provided By Google. Please Try Again!'); 60 | } 61 | } catch (error) { 62 | console.log('[ERROR] Getting Google Ads Refresh Token: ', error); 63 | return res.status(400).send('Error Getting Google Ads Refresh Token. Please Try Again!'); 64 | } 65 | }; 66 | 67 | const validateAdwordsIntegration = async (req: NextApiRequest, res: NextApiResponse) => { 68 | const errMsg = 'Error Validating Google Ads Integration. Please make sure your provided data are correct!'; 69 | const { developer_token, account_id } = req.body; 70 | if (!developer_token || !account_id) { 71 | return res.status(400).json({ valid: false, error: 'Please Provide the Google Ads Developer Token and Test Account ID' }); 72 | } 73 | try { 74 | // Save the Adwords Developer Token & Google Ads Test Account ID in App Settings 75 | const settingsRaw = await readFile(`${process.cwd()}/data/settings.json`, { encoding: 'utf-8' }); 76 | const settings: SettingsType = settingsRaw ? JSON.parse(settingsRaw) : {}; 77 | const cryptr = new Cryptr(process.env.SECRET as string); 78 | const adwords_developer_token = cryptr.encrypt(developer_token.trim()); 79 | const adwords_account_id = cryptr.encrypt(account_id.trim()); 80 | const securedSettings = { ...settings, adwords_developer_token, adwords_account_id }; 81 | await writeFile(`${process.cwd()}/data/settings.json`, JSON.stringify(securedSettings), { encoding: 'utf-8' }); 82 | 83 | // Make a test Request to Google Ads 84 | const adwordsCreds = await getAdwordsCredentials(); 85 | const { client_id, client_secret, refresh_token } = adwordsCreds || {}; 86 | if (adwordsCreds && client_id && client_secret && developer_token && account_id && refresh_token) { 87 | const keywords = await getAdwordsKeywordIdeas( 88 | adwordsCreds, 89 | { country: 'US', language: '1000', keywords: ['compress'], seedType: 'custom' }, 90 | true, 91 | ); 92 | if (keywords && Array.isArray(keywords) && keywords.length > 0) { 93 | return res.status(200).json({ valid: true }); 94 | } 95 | } 96 | return res.status(400).json({ valid: false, error: errMsg }); 97 | } catch (error) { 98 | console.log('[ERROR] Validating Google Ads Integration: ', error); 99 | return res.status(400).json({ valid: false, error: errMsg }); 100 | } 101 | }; 102 | -------------------------------------------------------------------------------- /components/insight/InsightStats.tsx: -------------------------------------------------------------------------------- 1 | import React, { useMemo } from 'react'; 2 | import { Chart as ChartJS, CategoryScale, LinearScale, PointElement, LineElement, Title, Tooltip, Legend } from 'chart.js'; 3 | import { Line } from 'react-chartjs-2'; 4 | import { formattedNum } from '../../utils/client/helpers'; 5 | 6 | ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Title, Tooltip, Legend); 7 | 8 | type InsightStatsProps = { 9 | stats: SearchAnalyticsStat[], 10 | totalKeywords: number, 11 | totalCountries: number, 12 | totalPages: number, 13 | } 14 | 15 | const InsightStats = ({ stats = [], totalKeywords = 0, totalPages = 0 }:InsightStatsProps) => { 16 | const totalStat = useMemo(() => { 17 | const totals = stats.reduce((acc, item) => { 18 | return { 19 | impressions: item.impressions + acc.impressions, 20 | clicks: item.clicks + acc.clicks, 21 | position: item.position + acc.position, 22 | }; 23 | }, { impressions: 0, clicks: 0, position: 0 }); 24 | 25 | return { 26 | ...totals, 27 | ctr: totals.impressions > 0 ? (totals.clicks / totals.impressions) * 100 : 0, 28 | }; 29 | }, [stats]); 30 | 31 | const chartData = useMemo(() => { 32 | const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; 33 | const chartSeries: {[key:string]: number[]} = { clicks: [], impressions: [], position: [], ctr: [] }; 34 | stats.forEach((item) => { 35 | chartSeries.clicks.push(item.clicks); 36 | chartSeries.impressions.push(item.impressions); 37 | chartSeries.position.push(item.position); 38 | chartSeries.ctr.push(item.ctr); 39 | }); 40 | return { 41 | labels: stats && stats.length > 0 ? stats.map((item) => `${new Date(item.date).getDate()}-${months[new Date(item.date).getMonth()]}`) : [], 42 | series: chartSeries }; 43 | }, [stats]); 44 | 45 | const renderChart = () => { 46 | // Doc: https://www.chartjs.org/docs/latest/samples/line/multi-axis.html 47 | const chartOptions = { 48 | responsive: true, 49 | maintainAspectRatio: false, 50 | animation: false as const, 51 | interaction: { 52 | mode: 'index' as const, 53 | intersect: false, 54 | }, 55 | scales: { 56 | x: { 57 | grid: { 58 | drawOnChartArea: false, 59 | }, 60 | }, 61 | y1: { 62 | display: true, 63 | position: 'right' as const, 64 | grid: { 65 | drawOnChartArea: false, 66 | }, 67 | }, 68 | }, 69 | plugins: { 70 | legend: { 71 | display: false, 72 | }, 73 | }, 74 | }; 75 | const { clicks, impressions } = chartData.series || {}; 76 | const dataSet = [ 77 | { label: 'Visits', data: clicks, borderColor: 'rgb(117, 50, 205)', backgroundColor: 'rgba(117, 50, 205, 0.5)', yAxisID: 'y' }, 78 | { label: 'Impressions', data: impressions, borderColor: 'rgb(31, 205, 176)', backgroundColor: 'rgba(31, 205, 176, 0.5)', yAxisID: 'y1' }, 79 | ]; 80 | return ; 81 | }; 82 | 83 | return ( 84 |
85 |
86 |
89 | Visits 90 | {new Intl.NumberFormat('en-US', { notation: 'compact', compactDisplay: 'short' }).format(totalStat.clicks || 0).replace('T', 'K')} 91 |
92 |
95 | Impressions 96 | {new Intl.NumberFormat('en-US', { notation: 'compact', compactDisplay: 'short' }).format(totalStat.impressions || 0).replace('T', 'K')} 97 |
98 |
99 | Avg Position 100 | {(totalStat.position ? Math.round(totalStat.position / stats.length) : 0)} 101 |
102 |
103 | Avg CTR 104 | {new Intl.NumberFormat('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 }).format(totalStat.ctr || 0)}% 105 |
106 |
107 | Keywords 108 | {formattedNum(totalKeywords)} 109 |
110 |
111 | Pages 112 | {formattedNum(totalPages)} 113 |
114 |
115 |
116 | {renderChart()} 117 |
118 |
119 | ); 120 | }; 121 | 122 | export default InsightStats; 123 | -------------------------------------------------------------------------------- /services/adwords.tsx: -------------------------------------------------------------------------------- 1 | import { NextRouter } from 'next/router'; 2 | import toast from 'react-hot-toast'; 3 | import { useMutation, useQuery, useQueryClient } from 'react-query'; 4 | 5 | export function useTestAdwordsIntegration(onSuccess?: Function) { 6 | return useMutation(async (payload:{developer_token:string, account_id:string}) => { 7 | const headers = new Headers({ 'Content-Type': 'application/json', Accept: 'application/json' }); 8 | const fetchOpts = { method: 'POST', headers, body: JSON.stringify({ ...payload }) }; 9 | const res = await fetch(`${window.location.origin}/api/adwords`, fetchOpts); 10 | if (res.status >= 400 && res.status < 600) { 11 | throw new Error('Bad response from server'); 12 | } 13 | return res.json(); 14 | }, { 15 | onSuccess: async (data) => { 16 | console.log('Ideas Added:', data); 17 | toast('Google Ads has been integrated successfully!', { icon: '✔️' }); 18 | if (onSuccess) { 19 | onSuccess(false); 20 | } 21 | }, 22 | onError: (error) => { 23 | console.log('Error Loading Keyword Ideas!!!', error); 24 | toast('Failed to connect to Google Ads. Please make sure you have provided the correct API info.', { icon: '⚠️' }); 25 | }, 26 | }); 27 | } 28 | 29 | export async function fetchAdwordsKeywordIdeas(router: NextRouter, domainSlug: string) { 30 | // if (!router.query.slug) { throw new Error('Invalid Domain Name'); } 31 | const res = await fetch(`${window.location.origin}/api/ideas?domain=${domainSlug}`, { method: 'GET' }); 32 | if (res.status >= 400 && res.status < 600) { 33 | if (res.status === 401) { 34 | console.log('Unauthorized!!'); 35 | router.push('/login'); 36 | } 37 | throw new Error('Bad response from server'); 38 | } 39 | return res.json(); 40 | } 41 | 42 | export function useFetchKeywordIdeas(router: NextRouter, adwordsConnected = false) { 43 | const isResearch = router.pathname === '/research'; 44 | const domainSlug = isResearch ? 'research' : (router.query.slug as string); 45 | const enabled = !!(adwordsConnected && domainSlug); 46 | return useQuery(`keywordIdeas-${domainSlug}`, () => domainSlug && fetchAdwordsKeywordIdeas(router, domainSlug), { enabled, retry: false }); 47 | } 48 | 49 | export function useMutateKeywordIdeas(router:NextRouter, onSuccess?: Function) { 50 | const queryClient = useQueryClient(); 51 | const domainSlug = router.pathname === '/research' ? 'research' : router.query.slug as string; 52 | return useMutation(async (data:Record) => { 53 | const headers = new Headers({ 'Content-Type': 'application/json', Accept: 'application/json' }); 54 | const fetchOpts = { method: 'POST', headers, body: JSON.stringify({ ...data }) }; 55 | const res = await fetch(`${window.location.origin}/api/ideas`, fetchOpts); 56 | if (res.status >= 400 && res.status < 600) { 57 | throw new Error('Bad response from server'); 58 | } 59 | return res.json(); 60 | }, { 61 | onSuccess: async (data) => { 62 | console.log('Ideas Added:', data); 63 | toast('Keyword Ideas Loaded Successfully!', { icon: '✔️' }); 64 | if (onSuccess) { 65 | onSuccess(false); 66 | } 67 | queryClient.invalidateQueries([`keywordIdeas-${domainSlug}`]); 68 | }, 69 | onError: (error) => { 70 | console.log('Error Loading Keyword Ideas!!!', error); 71 | toast('Error Loading Keyword Ideas', { icon: '⚠️' }); 72 | }, 73 | }); 74 | } 75 | 76 | export function useMutateFavKeywordIdeas(router:NextRouter, onSuccess?: Function) { 77 | const queryClient = useQueryClient(); 78 | const domainSlug = router.pathname === '/research' ? 'research' : router.query.slug as string; 79 | return useMutation(async (payload:Record) => { 80 | const headers = new Headers({ 'Content-Type': 'application/json', Accept: 'application/json' }); 81 | const fetchOpts = { method: 'PUT', headers, body: JSON.stringify({ ...payload }) }; 82 | const res = await fetch(`${window.location.origin}/api/ideas`, fetchOpts); 83 | if (res.status >= 400 && res.status < 600) { 84 | throw new Error('Bad response from server'); 85 | } 86 | return res.json(); 87 | }, { 88 | onSuccess: async (data) => { 89 | console.log('Ideas Added:', data); 90 | // toast('Keyword Updated!', { icon: '✔️' }); 91 | if (onSuccess) { 92 | onSuccess(false); 93 | } 94 | queryClient.invalidateQueries([`keywordIdeas-${domainSlug}`]); 95 | }, 96 | onError: (error) => { 97 | console.log('Error Favorating Keywords', error); 98 | toast('Error Favorating Keywords', { icon: '⚠️' }); 99 | }, 100 | }); 101 | } 102 | 103 | export function useMutateKeywordsVolume(onSuccess?: Function) { 104 | return useMutation(async (data:Record) => { 105 | const headers = new Headers({ 'Content-Type': 'application/json', Accept: 'application/json' }); 106 | const fetchOpts = { method: 'POST', headers, body: JSON.stringify({ ...data }) }; 107 | const res = await fetch(`${window.location.origin}/api/volume`, fetchOpts); 108 | if (res.status >= 400 && res.status < 600) { 109 | const errorData = await res.json(); 110 | throw new Error(errorData?.error ? errorData.error : 'Bad response from server'); 111 | } 112 | return res.json(); 113 | }, { 114 | onSuccess: async (data) => { 115 | toast('Keyword Volume Data Loaded Successfully! Reloading Page...', { icon: '✔️' }); 116 | if (onSuccess) { 117 | onSuccess(false); 118 | } 119 | setTimeout(() => { 120 | window.location.reload(); 121 | }, 3000); 122 | }, 123 | onError: (error) => { 124 | console.log('Error Loading Keyword Volume Data!!!', error); 125 | toast('Error Loading Keyword Volume Data', { icon: '⚠️' }); 126 | }, 127 | }); 128 | } 129 | -------------------------------------------------------------------------------- /components/common/SelectField.tsx: -------------------------------------------------------------------------------- 1 | import React, { useMemo, useState } from 'react'; 2 | import Icon from './Icon'; 3 | 4 | export type SelectionOption = { 5 | label:string, 6 | value: string 7 | } 8 | type SelectFieldProps = { 9 | defaultLabel: string, 10 | options: SelectionOption[], 11 | selected: string[], 12 | label?: string, 13 | multiple?: boolean, 14 | updateField: Function, 15 | fullWidth?: boolean, 16 | minWidth?: number, 17 | maxHeight?: number|string, 18 | rounded?: string, 19 | flags?: boolean, 20 | inline?: boolean, 21 | emptyMsg?: string 22 | } 23 | const SelectField = (props: SelectFieldProps) => { 24 | const { 25 | options, 26 | selected, 27 | defaultLabel = 'Select an Option', 28 | multiple = true, 29 | updateField, 30 | minWidth = 180, 31 | maxHeight = 96, 32 | fullWidth = false, 33 | rounded = 'rounded-3xl', 34 | inline = false, 35 | flags = false, 36 | label = '', 37 | emptyMsg = '' } = props; 38 | 39 | const [showOptions, setShowOptions] = useState(false); 40 | const [filterInput, setFilterInput] = useState(''); 41 | const [filterdOptions, setFilterdOptions] = useState([]); 42 | 43 | const selectedLabels = useMemo(() => { 44 | return options.reduce((acc:string[], item:SelectionOption) :string[] => { 45 | return selected.includes(item.value) ? [...acc, item.label] : [...acc]; 46 | }, []); 47 | }, [selected, options]); 48 | 49 | const selectItem = (option:SelectionOption) => { 50 | let updatedSelect = [option.value]; 51 | if (multiple && Array.isArray(selected)) { 52 | if (selected.includes(option.value)) { 53 | updatedSelect = selected.filter((x) => x !== option.value); 54 | } else { 55 | updatedSelect = [...selected, option.value]; 56 | } 57 | } 58 | updateField(updatedSelect); 59 | if (!multiple) { setShowOptions(false); } 60 | }; 61 | 62 | const filterOptions = (event:React.FormEvent) => { 63 | setFilterInput(event.currentTarget.value); 64 | const filteredItems:SelectionOption[] = []; 65 | const userVal = event.currentTarget.value.toLowerCase(); 66 | options.forEach((option:SelectionOption) => { 67 | if (flags ? option.label.toLowerCase().startsWith(userVal) : option.label.toLowerCase().includes(userVal)) { 68 | filteredItems.push(option); 69 | } 70 | }); 71 | setFilterdOptions(filteredItems); 72 | }; 73 | 74 | return ( 75 |
76 | {label && } 77 |
setShowOptions(!showOptions)}> 81 | 82 | {selected.length > 0 ? (selectedLabels.slice(0, 2).join(', ')) : defaultLabel} 83 | 84 | {multiple && selected.length > 2 85 | && {(selected.length)}} 86 | 87 |
88 | {showOptions && ( 89 |
92 | {options.length > 20 && ( 93 |
94 | 101 |
102 | )} 103 |
    104 | {(options.length > 20 && filterdOptions.length > 0 && filterInput ? filterdOptions : options).map((opt) => { 105 | const itemActive = selected.includes(opt.value); 106 | return ( 107 |
  • selectItem(opt)} 112 | > 113 | {multiple && ( 114 | 117 | 118 | 119 | )} 120 | {flags && } 121 | {opt.label} 122 |
    • 123 | ); 124 | })} 125 |
126 | {emptyMsg && options.length === 0 &&

{emptyMsg}

} 127 |
128 | )} 129 |
130 | ); 131 | }; 132 | 133 | export default SelectField; 134 | --------------------------------------------------------------------------------