├── .nvmrc ├── src ├── bin.ts ├── shared │ ├── index.ts │ ├── types.ts │ ├── utils.ts │ ├── auth.ts │ ├── sitemap.ts │ └── gsc.ts ├── cli.ts └── index.ts ├── .gitignore ├── output.png ├── .github ├── pull_request_template.md └── workflows │ ├── ci.yml │ └── release.yml ├── tsup.config.ts ├── .changeset └── config.json ├── tsconfig.json ├── CONTRIBUTING.md ├── LICENSE ├── package.json ├── CHANGELOG.md └── README.md /.nvmrc: -------------------------------------------------------------------------------- 1 | 20 2 | -------------------------------------------------------------------------------- /src/bin.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | require("./cli"); 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | service_account.json 2 | .cache 3 | node_modules 4 | .vscode 5 | dist -------------------------------------------------------------------------------- /output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/goenning/google-indexing-script/HEAD/output.png -------------------------------------------------------------------------------- /src/shared/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./auth"; 2 | export * from "./gsc"; 3 | export * from "./sitemap"; 4 | export * from "./types"; 5 | export * from "./utils"; 6 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | **What did I change?** 2 | 3 | 4 | 5 | **Why did I change it?** 6 | 7 | -------------------------------------------------------------------------------- /tsup.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig, Options } from "tsup"; 2 | 3 | const config: Options = { 4 | entry: ["src/**/*.ts"], 5 | splitting: true, 6 | sourcemap: true, 7 | clean: true, 8 | platform: "node", 9 | dts: true, 10 | minify: true, 11 | }; 12 | 13 | export default defineConfig(config); 14 | -------------------------------------------------------------------------------- /.changeset/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://unpkg.com/@changesets/config@3.0.0/schema.json", 3 | "commit": false, 4 | "fixed": [["google-indexing-script"]], 5 | "changelog": [ 6 | "@changesets/changelog-github", 7 | { "repo": "goenning/google-indexing-script" } 8 | ], 9 | "linked": [], 10 | "access": "public", 11 | "baseBranch": "main", 12 | "updateInternalDependencies": "patch" 13 | } 14 | -------------------------------------------------------------------------------- /src/shared/types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Enum representing indexing status of a URL 3 | */ 4 | export enum Status { 5 | SubmittedAndIndexed = "Submitted and indexed", 6 | DuplicateWithoutUserSelectedCanonical = "Duplicate without user-selected canonical", 7 | CrawledCurrentlyNotIndexed = "Crawled - currently not indexed", 8 | DiscoveredCurrentlyNotIndexed = "Discovered - currently not indexed", 9 | PageWithRedirect = "Page with redirect", 10 | URLIsUnknownToGoogle = "URL is unknown to Google", 11 | RateLimited = "RateLimited", 12 | Forbidden = "Forbidden", 13 | Error = "Error", 14 | } 15 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: [push] 3 | 4 | jobs: 5 | build: 6 | name: Build, lint, and test on Node ${{ matrix.node }} and ${{ matrix.os }} 7 | 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | matrix: 11 | node: ["18.x", "20.x"] 12 | os: [ubuntu-latest] 13 | 14 | steps: 15 | - name: Checkout repo 16 | uses: actions/checkout@v4 17 | 18 | - name: Use Node ${{ matrix.node }} 19 | uses: actions/setup-node@v4 20 | with: 21 | node-version: ${{ matrix.node }} 22 | cache: "npm" 23 | 24 | - name: Install Dependencies 25 | run: npm install 26 | 27 | - name: Build 28 | run: npm run build 29 | 30 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "esnext", 4 | "module": "commonjs", 5 | "lib": ["dom", "es6", "es2021", "esnext.asynciterable"], 6 | "skipLibCheck": true, 7 | "sourceMap": true, 8 | "outDir": "./dist", 9 | "moduleResolution": "node", 10 | "removeComments": false, 11 | "noImplicitAny": false, 12 | "strictNullChecks": true, 13 | "strictFunctionTypes": true, 14 | "noImplicitThis": true, 15 | "noUnusedLocals": true, 16 | "noUnusedParameters": true, 17 | "noImplicitReturns": true, 18 | "noFallthroughCasesInSwitch": true, 19 | "allowSyntheticDefaultImports": true, 20 | "esModuleInterop": true, 21 | "emitDecoratorMetadata": true, 22 | "experimentalDecorators": true, 23 | "resolveJsonModule": true 24 | }, 25 | "include": ["**/*.ts"], 26 | "exclude": ["node_modules/", "dist/"] 27 | } 28 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | branches: 5 | - main 6 | 7 | jobs: 8 | release: 9 | if: github.repository == 'goenning/google-indexing-script' 10 | 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - name: Checkout repo 15 | uses: actions/checkout@v4 16 | 17 | - name: Use Node 18 | uses: actions/setup-node@v4 19 | with: 20 | cache: "npm" 21 | 22 | - name: Install Dependencies 23 | run: npm install 24 | 25 | - name: Build 26 | run: npm run build 27 | 28 | - name: Create Release Pull Request or Publish to npm 29 | uses: changesets/action@v1 30 | with: 31 | publish: npm run release 32 | version: npm run version 33 | commit: "release version" 34 | title: "release version" 35 | env: 36 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 37 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 38 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Google Indexing Script 2 | 3 | Before jumping into a PR be sure to search [existing PRs](/goenning/google-indexing-script/pulls) or [issues](/goenning/google-indexing-script/issues) for an open or closed item that relates to your submission. 4 | 5 | # Developing 6 | 7 | All pull requests should be opened against `main`. 8 | 9 | 1. Clone the repository 10 | ```bash 11 | git clone https://github.com/goenning/google-indexing-script.git 12 | ``` 13 | 14 | 2. Install dependencies 15 | ```bash 16 | npm install 17 | ``` 18 | 19 | 3. Install the cli globally 20 | ```bash 21 | npm install -g . 22 | ``` 23 | 24 | 4. Run the development bundle 25 | ```bash 26 | npm run dev 27 | ``` 28 | 29 | 5. See how to [use it](/README.md#installation) and make your changes ! 30 | 31 | # Building 32 | 33 | After making your changes, you can build the project with the following command: 34 | 35 | ```bash 36 | npm run build 37 | ``` 38 | 39 | # Pull Request 40 | 41 | 1. Make sure your code is formatted with `prettier` 42 | 2. Make sure your code passes the tests 43 | 3. Make sure you added the changes with `npm run changeset` 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Guilherme Oenning 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "google-indexing-script", 3 | "description": "Script to get your site indexed on Google in less than 48 hours", 4 | "version": "0.4.0", 5 | "main": "./dist/index.js", 6 | "types": "./dist/index.d.ts", 7 | "bin": { 8 | "google-indexing-script": "./dist/bin.js", 9 | "gis": "./dist/bin.js" 10 | }, 11 | "keywords": [ 12 | "google", 13 | "indexing", 14 | "search-console", 15 | "sitemap", 16 | "seo", 17 | "google-search", 18 | "cli", 19 | "typescript" 20 | ], 21 | "license": "MIT", 22 | "scripts": { 23 | "index": "ts-node ./src/cli.ts", 24 | "build": "tsup", 25 | "dev": "tsup --watch", 26 | "changeset": "changeset", 27 | "version": "changeset version", 28 | "release": "changeset publish" 29 | }, 30 | "dependencies": { 31 | "commander": "^12.1.0", 32 | "googleapis": "131.0.0", 33 | "picocolors": "^1.0.1", 34 | "sitemapper": "3.2.8" 35 | }, 36 | "prettier": { 37 | "printWidth": 120 38 | }, 39 | "devDependencies": { 40 | "@changesets/changelog-github": "^0.5.0", 41 | "@changesets/cli": "^2.27.1", 42 | "ts-node": "^10.9.2", 43 | "tsup": "^8.0.2", 44 | "typescript": "^5.3.3" 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/cli.ts: -------------------------------------------------------------------------------- 1 | import { index } from "."; 2 | import { Command } from "commander"; 3 | import packageJson from "../package.json"; 4 | import { green } from "picocolors"; 5 | 6 | const program = new Command(packageJson.name); 7 | 8 | program 9 | .alias("gis") 10 | .version(packageJson.version, "-v, --version", "Output the current version.") 11 | .description(packageJson.description) 12 | .argument("[input]") 13 | .usage(`${green("[input]")} [options]`) 14 | .helpOption("-h, --help", "Output usage information.") 15 | .option("-c, --client-email ", "The client email for the Google service account.") 16 | .option("-k, --private-key ", "The private key for the Google service account.") 17 | .option("-p, --path ", "The path to the Google service account credentials file.") 18 | .option("-u, --urls ", "A comma-separated list of URLs to index.") 19 | .option("--rpm-retry", "Retry when the rate limit is exceeded.") 20 | .action((input, options) => { 21 | index(input, { 22 | client_email: options.clientEmail, 23 | private_key: options.privateKey, 24 | path: options.path, 25 | urls: options.urls ? options.urls.split(",") : undefined, 26 | quota: { 27 | rpmRetry: options.rpmRetry, 28 | }, 29 | }); 30 | }) 31 | .parse(process.argv); 32 | -------------------------------------------------------------------------------- /src/shared/utils.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Creates an array of chunks from the given array with a specified size. 3 | * @param arr The array to be chunked. 4 | * @param size The size of each chunk. 5 | * @returns An array of chunks. 6 | */ 7 | const createChunks = (arr: any[], size: number) => 8 | Array.from({ length: Math.ceil(arr.length / size) }, (_, i) => arr.slice(i * size, i * size + size)); 9 | 10 | /** 11 | * Executes tasks on items in batches and invokes a callback upon completion of each batch. 12 | * @param task The task function to be executed on each item. 13 | * @param items The array of items on which the task is to be executed. 14 | * @param batchSize The size of each batch. 15 | * @param onBatchComplete The callback function invoked upon completion of each batch. 16 | */ 17 | export async function batch( 18 | task: (url: string) => void, 19 | items: string[], 20 | batchSize: number, 21 | onBatchComplete: (batchIndex: number, batchCount: number) => void 22 | ) { 23 | const chunks = createChunks(items, batchSize); 24 | for (let i = 0; i < chunks.length; i++) { 25 | await Promise.all(chunks[i].map(task)); 26 | onBatchComplete(i, chunks.length); 27 | } 28 | } 29 | 30 | /** 31 | * Fetches a resource from a URL with retry logic. 32 | * @param url The URL of the resource to fetch. 33 | * @param options The options for the fetch request. 34 | * @param retries The number of retry attempts (default is 5). 35 | * @returns A Promise resolving to the fetched response. 36 | * @throws Error when retries are exhausted or server error occurs. 37 | */ 38 | export async function fetchRetry(url: string, options: RequestInit, retries: number = 5) { 39 | try { 40 | const response = await fetch(url, options); 41 | if (response.status >= 500) { 42 | const body = await response.text(); 43 | throw new Error(`Server error code ${response.status}\n${body}`); 44 | } 45 | return response; 46 | } catch (err) { 47 | if (retries <= 0) { 48 | throw err; 49 | } 50 | return fetchRetry(url, options, retries - 1); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/shared/auth.ts: -------------------------------------------------------------------------------- 1 | import { google } from "googleapis"; 2 | import fs from "fs"; 3 | import path from "path"; 4 | import os from "os"; 5 | 6 | /** 7 | * Retrieves an access token for Google APIs using service account credentials. 8 | * @param client_email - The client email of the service account. 9 | * @param private_key - The private key of the service account. 10 | * @param customPath - (Optional) Custom path to the service account JSON file. 11 | * @returns The access token. 12 | */ 13 | export async function getAccessToken(client_email?: string, private_key?: string, customPath?: string) { 14 | if (!client_email && !private_key) { 15 | const filePath = "service_account.json"; 16 | const filePathFromHome = path.join(os.homedir(), ".gis", "service_account.json"); 17 | const isFile = fs.existsSync(filePath); 18 | const isFileFromHome = fs.existsSync(filePathFromHome); 19 | const isCustomFile = !!customPath && fs.existsSync(customPath); 20 | 21 | if (!isFile && !isFileFromHome && !isCustomFile) { 22 | console.error(`❌ ${filePath} not found, please follow the instructions in README.md`); 23 | console.error(""); 24 | process.exit(1); 25 | } 26 | 27 | const key = JSON.parse( 28 | fs.readFileSync(!!customPath && isCustomFile ? customPath : isFile ? filePath : filePathFromHome, "utf8") 29 | ); 30 | client_email = key.client_email; 31 | private_key = key.private_key; 32 | } else { 33 | if (!client_email) { 34 | console.error("❌ Missing client_email in service account credentials."); 35 | console.error(""); 36 | process.exit(1); 37 | } 38 | 39 | if (!private_key) { 40 | console.error("❌ Missing private_key in service account credentials."); 41 | console.error(""); 42 | process.exit(1); 43 | } 44 | } 45 | 46 | const jwtClient = new google.auth.JWT( 47 | client_email, 48 | undefined, 49 | private_key, 50 | ["https://www.googleapis.com/auth/webmasters.readonly", "https://www.googleapis.com/auth/indexing"], 51 | undefined 52 | ); 53 | 54 | const tokens = await jwtClient.authorize(); 55 | return tokens.access_token; 56 | } 57 | -------------------------------------------------------------------------------- /src/shared/sitemap.ts: -------------------------------------------------------------------------------- 1 | import Sitemapper from "sitemapper"; 2 | import { fetchRetry } from "./utils"; 3 | import { webmasters_v3 } from "googleapis"; 4 | 5 | /** 6 | * Retrieves a list of sitemaps associated with the specified site URL from the Google Webmasters API. 7 | * @param accessToken The access token for authentication. 8 | * @param siteUrl The URL of the site for which to retrieve the list of sitemaps. 9 | * @returns An array containing the paths of the sitemaps associated with the site URL. 10 | */ 11 | async function getSitemapsList(accessToken: string, siteUrl: string) { 12 | const url = `https://www.googleapis.com/webmasters/v3/sites/${encodeURIComponent(siteUrl)}/sitemaps`; 13 | 14 | const response = await fetchRetry(url, { 15 | headers: { 16 | "Content-Type": "application/json", 17 | Authorization: `Bearer ${accessToken}`, 18 | }, 19 | }); 20 | 21 | if (response.status === 403) { 22 | console.error(`🔐 This service account doesn't have access to this site.`); 23 | return []; 24 | } 25 | 26 | if (response.status >= 300) { 27 | console.error(`❌ Failed to get list of sitemaps.`); 28 | console.error(`Response was: ${response.status}`); 29 | console.error(await response.text()); 30 | return []; 31 | } 32 | 33 | const body: webmasters_v3.Schema$SitemapsListResponse = await response.json(); 34 | 35 | if (!body.sitemap) { 36 | console.error("❌ No sitemaps found, add them to Google Search Console and try again."); 37 | return []; 38 | } 39 | 40 | return body.sitemap.filter((x) => x.path !== undefined && x.path !== null).map((x) => x.path as string); 41 | } 42 | 43 | /** 44 | * Retrieves a list of pages from all sitemaps associated with the specified site URL. 45 | * @param accessToken The access token for authentication. 46 | * @param siteUrl The URL of the site for which to retrieve the sitemap pages. 47 | * @returns An array containing the list of sitemaps and an array of unique page URLs extracted from those sitemaps. 48 | */ 49 | export async function getSitemapPages(accessToken: string, siteUrl: string) { 50 | const sitemaps = await getSitemapsList(accessToken, siteUrl); 51 | 52 | let pages: string[] = []; 53 | for (const url of sitemaps) { 54 | const Google = new Sitemapper({ 55 | url, 56 | }); 57 | 58 | const { sites } = await Google.fetch(); 59 | pages = [...pages, ...sites]; 60 | } 61 | 62 | return [sitemaps, [...new Set(pages)]]; 63 | } 64 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # google-indexing-script 2 | 3 | ## 0.4.0 4 | 5 | ### Minor Changes 6 | 7 | - [#68](https://github.com/goenning/google-indexing-script/pull/68) [`caa73f7`](https://github.com/goenning/google-indexing-script/commit/caa73f765b5d494d65a894a83bb8faf351e6d8ae) Thanks [@AntoineKM](https://github.com/AntoineKM)! - Improve CLI with commander 8 | 9 | ## 0.3.0 10 | 11 | ### Minor Changes 12 | 13 | - [#65](https://github.com/goenning/google-indexing-script/pull/65) [`e0c31f8`](https://github.com/goenning/google-indexing-script/commit/e0c31f837acfe2083843436050b40f21f3806838) Thanks [@AntoineKM](https://github.com/AntoineKM)! - Add custom URLs option 14 | 15 | ### Patch Changes 16 | 17 | - [#65](https://github.com/goenning/google-indexing-script/pull/65) [`e0c31f8`](https://github.com/goenning/google-indexing-script/commit/e0c31f837acfe2083843436050b40f21f3806838) Thanks [@AntoineKM](https://github.com/AntoineKM)! - Fix siteUrls convertions 18 | 19 | ## 0.2.0 20 | 21 | ### Minor Changes 22 | 23 | - [#62](https://github.com/goenning/google-indexing-script/pull/62) [`93dd956`](https://github.com/goenning/google-indexing-script/commit/93dd956dca4065b97d6076db772560fba57aec50) Thanks [@hasanafzal8485](https://github.com/hasanafzal8485)! - Don't want the same URL use my API limit again until his previous cache limit is completed 24 | 25 | ## 0.1.0 26 | 27 | ### Minor Changes 28 | 29 | - [#55](https://github.com/goenning/google-indexing-script/pull/55) [`908938a`](https://github.com/goenning/google-indexing-script/commit/908938a701d964b75331e322fbea8d77e6db976e) Thanks [@AntoineKM](https://github.com/AntoineKM)! - feat(get-publish-metadata): optional retries if rate limited 30 | 31 | ## 0.0.5 32 | 33 | ### Patch Changes 34 | 35 | - [#44](https://github.com/goenning/google-indexing-script/pull/44) [`77b94ed`](https://github.com/goenning/google-indexing-script/commit/77b94edeef863721c07bd3e12d6d38052723f422) Thanks [@AntoineKM](https://github.com/AntoineKM)! - Add site url checker 36 | 37 | ## 0.0.4 38 | 39 | ### Patch Changes 40 | 41 | - [#40](https://github.com/goenning/google-indexing-script/pull/40) [`074f2c7`](https://github.com/goenning/google-indexing-script/commit/074f2c7ebbafff3a03ebf07baf7b21922a98698d) Thanks [@AntoineKM](https://github.com/AntoineKM)! - Add documentation comments 42 | 43 | ## 0.0.3 44 | 45 | ### Patch Changes 46 | 47 | - [#39](https://github.com/goenning/google-indexing-script/pull/39) [`9467e82`](https://github.com/goenning/google-indexing-script/commit/9467e82496170aeaa42ecd8ab6b8de4ba8f8315f) Thanks [@AntoineKM](https://github.com/AntoineKM)! - Fix index function to handle options passed 48 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Google Indexing Script 2 | 3 | Use this script to get your entire site indexed on Google in less than 48 hours. No tricks, no hacks, just a simple script and a Google API. 4 | 5 | > [!IMPORTANT] 6 | > 7 | > 1. This script uses [Google Indexing API](https://developers.google.com/search/apis/indexing-api/v3/quickstart) and it only works on pages with either `JobPosting` or `BroadcastEvent` structured data. 8 | > 2. Indexing != Ranking. This will not help your page rank on Google, it'll just let Google know about the existence of your pages. 9 | 10 | ## Requirements 11 | 12 | - Install [Node.js](https://nodejs.org/en/download) 13 | - An account on [Google Search Console](https://search.google.com/search-console/about) with the verified sites you want to index 14 | - An account on [Google Cloud](https://console.cloud.google.com/) 15 | 16 | ## Preparation 17 | 18 | 1. Follow this [guide](https://developers.google.com/search/apis/indexing-api/v3/prereqs) from Google. By the end of it, you should have a project on Google Cloud with the Indexing API enabled, a service account with the `Owner` permission on your sites. 19 | 2. Make sure you enable both [`Google Search Console API`](https://console.cloud.google.com/apis/api/searchconsole.googleapis.com) and [`Web Search Indexing API`](https://console.cloud.google.com/apis/api/indexing.googleapis.com) on your [Google Project ➤ API Services ➤ Enabled API & Services](https://console.cloud.google.com/apis/dashboard). 20 | 3. [Download the JSON](https://github.com/goenning/google-indexing-script/issues/2) file with the credentials of your service account and save it in the same folder as the script. The file should be named `service_account.json` 21 | 22 | ## Installation 23 | 24 | ### Using CLI 25 | 26 | Install the cli globally on your machine. 27 | 28 | ```bash 29 | npm i -g google-indexing-script 30 | ``` 31 | 32 | ### Using the repository 33 | 34 | Clone the repository to your machine. 35 | 36 | ```bash 37 | git clone https://github.com/goenning/google-indexing-script.git 38 | cd google-indexing-script 39 | ``` 40 | 41 | Install and build the project. 42 | 43 | ```bash 44 | npm install 45 | npm run build 46 | npm i -g . 47 | ``` 48 | 49 | > [!NOTE] 50 | > Ensure you are using an up-to-date Node.js version, with a preference for v20 or later. Check your current version with `node -v`. 51 | 52 | ## Usage 53 | 54 |
55 | With service_account.json (recommended) 56 | 57 | Create a `.gis` directory in your home folder and move the `service_account.json` file there. 58 | 59 | ```bash 60 | mkdir ~/.gis 61 | mv service_account.json ~/.gis 62 | ``` 63 | 64 | Run the script with the domain or url you want to index. 65 | 66 | ```bash 67 | gis 68 | # example 69 | gis seogets.com 70 | ``` 71 | 72 | Here are some other ways to run the script: 73 | 74 | ```bash 75 | # custom path to service_account.json 76 | gis seogets.com --path /path/to/service_account.json 77 | # long version command 78 | google-indexing-script seogets.com 79 | # cloned repository 80 | npm run index seogets.com 81 | ``` 82 | 83 |
84 | 85 |
86 | With environment variables 87 | 88 | Open `service_account.json` and copy the `client_email` and `private_key` values. 89 | 90 | Run the script with the domain or url you want to index. 91 | 92 | ```bash 93 | GIS_CLIENT_EMAIL=your-client-email GIS_PRIVATE_KEY=your-private-key gis seogets.com 94 | ``` 95 | 96 |
97 | 98 |
99 | With arguments (not recommended) 100 | 101 | Open `service_account.json` and copy the `client_email` and `private_key` values. 102 | 103 | Once you have the values, run the script with the domain or url you want to index, the client email and the private key. 104 | 105 | ```bash 106 | gis seogets.com --client-email your-client-email --private-key your-private-key 107 | ``` 108 | 109 |
110 | 111 |
112 | As a npm module 113 | 114 | You can also use the script as a [npm module](https://www.npmjs.com/package/google-indexing-script) in your own project. 115 | 116 | ```bash 117 | npm i google-indexing-script 118 | ``` 119 | 120 | ```javascript 121 | import { index } from "google-indexing-script"; 122 | import serviceAccount from "./service_account.json"; 123 | 124 | index("seogets.com", { 125 | client_email: serviceAccount.client_email, 126 | private_key: serviceAccount.private_key, 127 | }) 128 | .then(console.log) 129 | .catch(console.error); 130 | ``` 131 | 132 | Read the [API documentation](https://jsdocs.io/package/google-indexing-script) for more details. 133 | 134 |
135 | 136 | Here's an example of what you should expect: 137 | 138 | ![](./output.png) 139 | 140 | > [!IMPORTANT] 141 | > 142 | > - Your site must have 1 or more sitemaps submitted to Google Search Console. Otherwise, the script will not be able to find the pages to index. 143 | > - You can run the script as many times as you want. It will only index the pages that are not already indexed. 144 | > - Sites with a large number of pages might take a while to index, be patient. 145 | 146 | ## Quota 147 | 148 | Depending on your account several quotas are configured for the API (see [docs](https://developers.google.com/search/apis/indexing-api/v3/quota-pricing#quota)). By default the script exits as soon as the rate limit is exceeded. You can configure a retry mechanism for the read requests that apply on a per minute time frame. 149 | 150 |
151 | With environment variables 152 | 153 | ```bash 154 | export GIS_QUOTA_RPM_RETRY=true 155 | ``` 156 | 157 |
158 | 159 |
160 | As a npm module 161 | 162 | ```javascript 163 | import { index } from 'google-indexing-script' 164 | import serviceAccount from './service_account.json' 165 | 166 | index('seogets.com', { 167 | client_email: serviceAccount.client_email, 168 | private_key: serviceAccount.private_key 169 | quota: { 170 | rpmRetry: true 171 | } 172 | }) 173 | .then(console.log) 174 | .catch(console.error) 175 | ``` 176 | 177 |
178 | 179 | ## 📄 License 180 | 181 | MIT License 182 | 183 | ## 💖 Sponsor 184 | 185 | This project is sponsored by [SEO Gets](https://seogets.com) 186 | 187 | ![](https://seogets.com/og.png) 188 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { getAccessToken } from "./shared/auth"; 2 | import { 3 | convertToSiteUrl, 4 | getPublishMetadata, 5 | requestIndexing, 6 | getEmojiForStatus, 7 | getPageIndexingStatus, 8 | convertToFilePath, 9 | checkSiteUrl, 10 | checkCustomUrls, 11 | } from "./shared/gsc"; 12 | import { getSitemapPages } from "./shared/sitemap"; 13 | import { Status } from "./shared/types"; 14 | import { batch } from "./shared/utils"; 15 | import { readFileSync, existsSync, mkdirSync, writeFileSync } from "fs"; 16 | import path from "path"; 17 | 18 | const CACHE_TIMEOUT = 1000 * 60 * 60 * 24 * 14; // 14 days 19 | export const QUOTA = { 20 | rpm: { 21 | retries: 3, 22 | waitingTime: 60000, // 1 minute 23 | }, 24 | }; 25 | 26 | export type IndexOptions = { 27 | client_email?: string; 28 | private_key?: string; 29 | path?: string; 30 | urls?: string[]; 31 | quota?: { 32 | rpmRetry?: boolean; // read requests per minute: retry after waiting time 33 | }; 34 | }; 35 | 36 | /** 37 | * Indexes the specified domain or site URL. 38 | * @param input - The domain or site URL to index. 39 | * @param options - (Optional) Additional options for indexing. 40 | */ 41 | export const index = async (input: string = process.argv[2], options: IndexOptions = {}) => { 42 | if (!input) { 43 | console.error("❌ Please provide a domain or site URL as the first argument."); 44 | console.error(""); 45 | process.exit(1); 46 | } 47 | 48 | if (!options.client_email) { 49 | options.client_email = process.env.GIS_CLIENT_EMAIL; 50 | } 51 | if (!options.private_key) { 52 | options.private_key = process.env.GIS_PRIVATE_KEY; 53 | } 54 | if (!options.path) { 55 | options.path = process.env.GIS_PATH; 56 | } 57 | if (!options.urls) { 58 | options.urls = process.env.GIS_URLS ? process.env.GIS_URLS.split(",") : undefined; 59 | } 60 | if (!options.quota) { 61 | options.quota = { 62 | rpmRetry: process.env.GIS_QUOTA_RPM_RETRY === "true", 63 | }; 64 | } 65 | 66 | const accessToken = await getAccessToken(options.client_email, options.private_key, options.path); 67 | let siteUrl = convertToSiteUrl(input); 68 | console.log(`🔎 Processing site: ${siteUrl}`); 69 | const cachePath = path.join(".cache", `${convertToFilePath(siteUrl)}.json`); 70 | 71 | if (!accessToken) { 72 | console.error("❌ Failed to get access token, check your service account credentials."); 73 | console.error(""); 74 | process.exit(1); 75 | } 76 | 77 | siteUrl = await checkSiteUrl(accessToken, siteUrl); 78 | 79 | let pages = options.urls || []; 80 | if (pages.length === 0) { 81 | console.log(`🔎 Fetching sitemaps and pages...`); 82 | const [sitemaps, pagesFromSitemaps] = await getSitemapPages(accessToken, siteUrl); 83 | 84 | if (sitemaps.length === 0) { 85 | console.error("❌ No sitemaps found, add them to Google Search Console and try again."); 86 | console.error(""); 87 | process.exit(1); 88 | } 89 | 90 | pages = pagesFromSitemaps; 91 | 92 | console.log(`👉 Found ${pages.length} URLs in ${sitemaps.length} sitemap`); 93 | } else { 94 | pages = checkCustomUrls(siteUrl, pages); 95 | console.log(`👉 Found ${pages.length} URLs in the provided list`); 96 | } 97 | 98 | const statusPerUrl: Record = existsSync(cachePath) 99 | ? JSON.parse(readFileSync(cachePath, "utf8")) 100 | : {}; 101 | const pagesPerStatus: Record = { 102 | [Status.SubmittedAndIndexed]: [], 103 | [Status.DuplicateWithoutUserSelectedCanonical]: [], 104 | [Status.CrawledCurrentlyNotIndexed]: [], 105 | [Status.DiscoveredCurrentlyNotIndexed]: [], 106 | [Status.PageWithRedirect]: [], 107 | [Status.URLIsUnknownToGoogle]: [], 108 | [Status.RateLimited]: [], 109 | [Status.Forbidden]: [], 110 | [Status.Error]: [], 111 | }; 112 | 113 | const indexableStatuses = [ 114 | Status.DiscoveredCurrentlyNotIndexed, 115 | Status.CrawledCurrentlyNotIndexed, 116 | Status.URLIsUnknownToGoogle, 117 | Status.Forbidden, 118 | Status.Error, 119 | Status.RateLimited, 120 | ]; 121 | 122 | const shouldRecheck = (status: Status, lastCheckedAt: string) => { 123 | const shouldIndexIt = indexableStatuses.includes(status); 124 | const isOld = new Date(lastCheckedAt) < new Date(Date.now() - CACHE_TIMEOUT); 125 | return shouldIndexIt && isOld; 126 | }; 127 | 128 | await batch( 129 | async (url) => { 130 | let result = statusPerUrl[url]; 131 | if (!result || shouldRecheck(result.status, result.lastCheckedAt)) { 132 | const status = await getPageIndexingStatus(accessToken, siteUrl, url); 133 | result = { status, lastCheckedAt: new Date().toISOString() }; 134 | statusPerUrl[url] = result; 135 | } 136 | 137 | pagesPerStatus[result.status] = pagesPerStatus[result.status] ? [...pagesPerStatus[result.status], url] : [url]; 138 | }, 139 | pages, 140 | 50, 141 | (batchIndex, batchCount) => { 142 | console.log(`📦 Batch ${batchIndex + 1} of ${batchCount} complete`); 143 | } 144 | ); 145 | 146 | console.log(``); 147 | console.log(`👍 Done, here's the status of all ${pages.length} pages:`); 148 | mkdirSync(".cache", { recursive: true }); 149 | writeFileSync(cachePath, JSON.stringify(statusPerUrl, null, 2)); 150 | 151 | for (const status of Object.keys(pagesPerStatus)) { 152 | const pages = pagesPerStatus[status as Status]; 153 | if (pages.length === 0) continue; 154 | console.log(`• ${getEmojiForStatus(status as Status)} ${status}: ${pages.length} pages`); 155 | } 156 | console.log(""); 157 | 158 | const indexablePages = Object.entries(pagesPerStatus).flatMap(([status, pages]) => 159 | indexableStatuses.includes(status as Status) ? pages : [] 160 | ); 161 | 162 | if (indexablePages.length === 0) { 163 | console.log(`✨ There are no pages that can be indexed. Everything is already indexed!`); 164 | } else { 165 | console.log(`✨ Found ${indexablePages.length} pages that can be indexed.`); 166 | indexablePages.forEach((url) => console.log(`• ${url}`)); 167 | } 168 | console.log(``); 169 | 170 | for (const url of indexablePages) { 171 | console.log(`📄 Processing url: ${url}`); 172 | const status = await getPublishMetadata(accessToken, url, { 173 | retriesOnRateLimit: options.quota.rpmRetry ? QUOTA.rpm.retries : 0, 174 | }); 175 | if (status === 404) { 176 | await requestIndexing(accessToken, url); 177 | console.log("🚀 Indexing requested successfully. It may take a few days for Google to process it."); 178 | } else if (status < 400) { 179 | console.log(`🕛 Indexing already requested previously. It may take a few days for Google to process it.`); 180 | } 181 | console.log(``); 182 | } 183 | 184 | console.log(`👍 All done!`); 185 | console.log(`💖 Brought to you by https://seogets.com - SEO Analytics.`); 186 | console.log(``); 187 | }; 188 | 189 | export * from "./shared"; 190 | -------------------------------------------------------------------------------- /src/shared/gsc.ts: -------------------------------------------------------------------------------- 1 | import { webmasters_v3 } from "googleapis"; 2 | import { QUOTA } from ".."; 3 | import { Status } from "./types"; 4 | import { fetchRetry } from "./utils"; 5 | 6 | /** 7 | * Converts a given input string to a valid Google Search Console site URL format. 8 | * @param input - The input string to be converted. 9 | * @returns The converted site URL (domain.com or https://domain.com/) 10 | */ 11 | export function convertToSiteUrl(input: string) { 12 | if (input.startsWith("http://") || input.startsWith("https://")) { 13 | return input.endsWith("/") ? input : `${input}/`; 14 | } 15 | return `sc-domain:${input}`; 16 | } 17 | 18 | /** 19 | * Converts a given file path to a formatted version suitable for use as a file name. 20 | * @param path - The url to be converted as a file name 21 | * @returns The converted file path 22 | */ 23 | export function convertToFilePath(path: string) { 24 | return path.replace("http://", "http_").replace("https://", "https_").replaceAll("/", "_"); 25 | } 26 | 27 | /** 28 | * Converts an HTTP URL to a sc-domain URL format. 29 | * @param httpUrl The HTTP URL to be converted. 30 | * @returns The sc-domain formatted URL. 31 | */ 32 | export function convertToSCDomain(httpUrl: string) { 33 | return `sc-domain:${httpUrl.replace("http://", "").replace("https://", "").replace("/", "")}`; 34 | } 35 | 36 | /** 37 | * Converts a domain to an HTTP URL. 38 | * @param domain The domain to be converted. 39 | * @returns The HTTP URL. 40 | */ 41 | export function convertToHTTP(domain: string) { 42 | return `http://${domain}/`; 43 | } 44 | 45 | /** 46 | * Converts a domain to an HTTPS URL. 47 | * @param domain The domain to be converted. 48 | * @returns The HTTPS URL. 49 | */ 50 | export function convertToHTTPS(domain: string) { 51 | return `https://${domain}/`; 52 | } 53 | 54 | /** 55 | * Retrieves a list of sites associated with the specified service account from the Google Webmasters API. 56 | * @param accessToken - The access token for authentication. 57 | * @returns An array containing the site URLs associated with the service account. 58 | */ 59 | export async function getSites(accessToken: string) { 60 | const sitesResponse = await fetchRetry("https://www.googleapis.com/webmasters/v3/sites", { 61 | headers: { 62 | "Content-Type": "application/json", 63 | Authorization: `Bearer ${accessToken}`, 64 | }, 65 | }); 66 | 67 | if (sitesResponse.status === 403) { 68 | console.error("🔐 This service account doesn't have access to any sites."); 69 | return []; 70 | } 71 | 72 | const sitesBody: webmasters_v3.Schema$SitesListResponse = await sitesResponse.json(); 73 | 74 | if (!sitesBody.siteEntry) { 75 | console.error("❌ No sites found, add them to Google Search Console and try again."); 76 | return []; 77 | } 78 | 79 | return sitesBody.siteEntry.map((x) => x.siteUrl); 80 | } 81 | 82 | /** 83 | * Checks if the site URL is valid and accessible by the service account. 84 | * @param accessToken - The access token for authentication. 85 | * @param siteUrl - The URL of the site to check. 86 | * @returns The corrected URL if found, otherwise the original site URL. 87 | */ 88 | export async function checkSiteUrl(accessToken: string, siteUrl: string) { 89 | const sites = await getSites(accessToken); 90 | let formattedUrls: string[] = []; 91 | 92 | // Convert the site URL into all possible formats 93 | if (siteUrl.startsWith("https://")) { 94 | formattedUrls.push(siteUrl); 95 | formattedUrls.push(convertToHTTP(siteUrl.replace("https://", ""))); 96 | formattedUrls.push(convertToSCDomain(siteUrl)); 97 | } else if (siteUrl.startsWith("http://")) { 98 | formattedUrls.push(siteUrl); 99 | formattedUrls.push(convertToHTTPS(siteUrl.replace("http://", ""))); 100 | formattedUrls.push(convertToSCDomain(siteUrl)); 101 | } else if (siteUrl.startsWith("sc-domain:")) { 102 | formattedUrls.push(siteUrl); 103 | formattedUrls.push(convertToHTTP(siteUrl.replace("sc-domain:", ""))); 104 | formattedUrls.push(convertToHTTPS(siteUrl.replace("sc-domain:", ""))); 105 | } else { 106 | console.error("❌ Unknown site URL format."); 107 | console.error(""); 108 | process.exit(1); 109 | } 110 | 111 | // Check if any of the formatted URLs are accessible 112 | for (const formattedUrl of formattedUrls) { 113 | if (sites.includes(formattedUrl)) { 114 | return formattedUrl; 115 | } 116 | } 117 | 118 | // If none of the formatted URLs are accessible 119 | console.error("❌ This service account doesn't have access to this site."); 120 | console.error(""); 121 | process.exit(1); 122 | } 123 | 124 | /** 125 | * Checks if the given URLs are valid. 126 | * @param siteUrl - The URL of the site. 127 | * @param urls - The URLs to check. 128 | * @returns An array containing the corrected URLs if found, otherwise the original URLs 129 | */ 130 | export function checkCustomUrls(siteUrl: string, urls: string[]) { 131 | const protocol = siteUrl.startsWith("http://") ? "http://" : "https://"; 132 | const domain = siteUrl.replace("https://", "").replace("http://", "").replace("sc-domain:", ""); 133 | const formattedUrls: string[] = urls.map((url) => { 134 | url = url.trim(); 135 | if (url.startsWith("/")) { 136 | // the url is a relative path (e.g. /about) 137 | return `${protocol}${domain}${url}`; 138 | } else if (url.startsWith("http://") || url.startsWith("https://")) { 139 | // the url is already a full url (e.g. https://domain.com/about) 140 | return url; 141 | } else if (url.startsWith(domain)) { 142 | // the url is a full url without the protocol (e.g. domain.com/about) 143 | return `${protocol}${url}`; 144 | } else { 145 | // the url is a relative path without the leading slash (e.g. about) 146 | return `${protocol}${domain}/${url}`; 147 | } 148 | }); 149 | 150 | return formattedUrls; 151 | } 152 | 153 | /** 154 | * Retrieves the indexing status of a page. 155 | * @param accessToken - The access token for authentication. 156 | * @param siteUrl - The URL of the site. 157 | * @param inspectionUrl - The URL of the page to inspect. 158 | * @returns A promise resolving to the status of indexing. 159 | */ 160 | export async function getPageIndexingStatus( 161 | accessToken: string, 162 | siteUrl: string, 163 | inspectionUrl: string 164 | ): Promise { 165 | try { 166 | const response = await fetchRetry(`https://searchconsole.googleapis.com/v1/urlInspection/index:inspect`, { 167 | method: "POST", 168 | headers: { 169 | "Content-Type": "application/json", 170 | Authorization: `Bearer ${accessToken}`, 171 | }, 172 | body: JSON.stringify({ 173 | inspectionUrl, 174 | siteUrl, 175 | }), 176 | }); 177 | 178 | if (response.status === 403) { 179 | console.error(`🔐 This service account doesn't have access to this site.`); 180 | console.error(await response.text()); 181 | 182 | return Status.Forbidden; 183 | } 184 | 185 | if (response.status >= 300) { 186 | if (response.status === 429) { 187 | return Status.RateLimited; 188 | } else { 189 | console.error(`❌ Failed to get indexing status.`); 190 | console.error(`Response was: ${response.status}`); 191 | console.error(await response.text()); 192 | 193 | return Status.Error; 194 | } 195 | } 196 | 197 | const body = await response.json(); 198 | return body.inspectionResult.indexStatusResult.coverageState; 199 | } catch (error) { 200 | console.error(`❌ Failed to get indexing status.`); 201 | console.error(`Error was: ${error}`); 202 | throw error; 203 | } 204 | } 205 | 206 | /** 207 | * Retrieves an emoji representation corresponding to the given status. 208 | * @param status - The status for which to retrieve the emoji. 209 | * @returns The emoji representing the status. 210 | */ 211 | export function getEmojiForStatus(status: Status) { 212 | switch (status) { 213 | case Status.SubmittedAndIndexed: 214 | return "✅"; 215 | case Status.DuplicateWithoutUserSelectedCanonical: 216 | return "😵"; 217 | case Status.CrawledCurrentlyNotIndexed: 218 | case Status.DiscoveredCurrentlyNotIndexed: 219 | return "👀"; 220 | case Status.PageWithRedirect: 221 | return "🔀"; 222 | case Status.URLIsUnknownToGoogle: 223 | return "❓"; 224 | case Status.RateLimited: 225 | return "🚦"; 226 | default: 227 | return "❌"; 228 | } 229 | } 230 | 231 | /** 232 | * Retrieves metadata for publishing from the given URL. 233 | * @param accessToken - The access token for authentication. 234 | * @param url - The URL for which to retrieve metadata. 235 | * @param options - The options for the request. 236 | * @returns The status of the request. 237 | */ 238 | export async function getPublishMetadata(accessToken: string, url: string, options?: { retriesOnRateLimit: number }) { 239 | const response = await fetchRetry( 240 | `https://indexing.googleapis.com/v3/urlNotifications/metadata?url=${encodeURIComponent(url)}`, 241 | { 242 | method: "GET", 243 | headers: { 244 | "Content-Type": "application/json", 245 | Authorization: `Bearer ${accessToken}`, 246 | }, 247 | } 248 | ); 249 | 250 | if (response.status === 403) { 251 | console.error(`🔐 This service account doesn't have access to this site.`); 252 | console.error(`Response was: ${response.status}`); 253 | console.error(await response.text()); 254 | } 255 | 256 | if (response.status === 429) { 257 | if (options?.retriesOnRateLimit && options?.retriesOnRateLimit > 0) { 258 | const RPM_WATING_TIME = (QUOTA.rpm.retries - options.retriesOnRateLimit + 1) * QUOTA.rpm.waitingTime; // increase waiting time for each retry 259 | console.log( 260 | `🚦 Rate limit exceeded for read requests. Retries left: ${options.retriesOnRateLimit}. Waiting for ${ 261 | RPM_WATING_TIME / 1000 262 | }sec.` 263 | ); 264 | await new Promise((resolve) => setTimeout(resolve, RPM_WATING_TIME)); 265 | await getPublishMetadata(accessToken, url, { retriesOnRateLimit: options.retriesOnRateLimit - 1 }); 266 | } else { 267 | console.error("🚦 Rate limit exceeded, try again later."); 268 | console.error(""); 269 | console.error(" Quota: https://developers.google.com/search/apis/indexing-api/v3/quota-pricing#quota"); 270 | console.error(" Usage: https://console.cloud.google.com/apis/enabled"); 271 | console.error(""); 272 | process.exit(1); 273 | } 274 | } 275 | 276 | if (response.status >= 500) { 277 | console.error(`❌ Failed to get publish metadata.`); 278 | console.error(`Response was: ${response.status}`); 279 | console.error(await response.text()); 280 | } 281 | 282 | return response.status; 283 | } 284 | 285 | /** 286 | * Requests indexing for the given URL. 287 | * @param accessToken - The access token for authentication. 288 | * @param url - The URL to be indexed. 289 | */ 290 | export async function requestIndexing(accessToken: string, url: string) { 291 | const response = await fetchRetry("https://indexing.googleapis.com/v3/urlNotifications:publish", { 292 | method: "POST", 293 | headers: { 294 | "Content-Type": "application/json", 295 | Authorization: `Bearer ${accessToken}`, 296 | }, 297 | body: JSON.stringify({ 298 | url: url, 299 | type: "URL_UPDATED", 300 | }), 301 | }); 302 | 303 | if (response.status === 403) { 304 | console.error(`🔐 This service account doesn't have access to this site.`); 305 | console.error(`Response was: ${response.status}`); 306 | } 307 | 308 | if (response.status >= 300) { 309 | if (response.status === 429) { 310 | console.error("🚦 Rate limit exceeded, try again later."); 311 | console.error(""); 312 | console.error(" Quota: https://developers.google.com/search/apis/indexing-api/v3/quota-pricing#quota"); 313 | console.error(" Usage: https://console.cloud.google.com/apis/enabled"); 314 | console.error(""); 315 | process.exit(1); 316 | } else { 317 | console.error(`❌ Failed to request indexing.`); 318 | console.error(`Response was: ${response.status}`); 319 | console.error(await response.text()); 320 | } 321 | } 322 | } 323 | --------------------------------------------------------------------------------