├── .eslintrc.json ├── bunfig.toml ├── bun.lockb ├── app ├── favicon.ico ├── globals.css ├── global-error.tsx ├── sitemap.ts ├── api │ ├── domains │ │ ├── route.ts │ │ └── [domain] │ │ │ └── route.ts │ └── crons │ │ └── refresh │ │ └── route.ts ├── layout.tsx ├── genre │ └── [genre] │ │ └── page.tsx ├── page.tsx ├── affiliation │ └── [identifier] │ │ └── page.tsx ├── domain │ └── [domain] │ │ └── page.tsx └── technology │ └── [identifier] │ ├── and │ └── [subidentifier] │ │ └── page.tsx │ └── page.tsx ├── lib ├── affiliations │ ├── types.ts │ ├── loaders │ │ ├── tranco.ts │ │ ├── osspledge.ts │ │ └── ycombinator.ts │ └── registry.ts ├── loaders │ ├── types.ts │ ├── dns_prefix.ts │ ├── dns.ts │ └── html.ts ├── constants.ts ├── db │ ├── connection.ts │ ├── domains-by-technology.ts │ ├── types.ts │ └── domains.ts ├── parsers │ ├── types.ts │ ├── affiliations.ts │ ├── headers.ts │ ├── dns.ts │ └── html.ts ├── seo.e2e-test.ts ├── utils.ts ├── data.ts ├── data.test.ts └── services.tsx ├── vercel.json ├── .env.sample ├── postcss.config.mjs ├── components ├── SectionHeader.tsx ├── DomainIcon.tsx ├── Header.tsx ├── Grid.tsx ├── TechnologyPill.tsx ├── DomainSearchForm.tsx └── Icon.tsx ├── instrumentation.ts ├── README.md ├── public ├── icons │ └── twitter.svg ├── vercel.svg └── next.svg ├── .github └── workflows │ └── test.yml ├── playwright.config.ts ├── justfile ├── .gitignore ├── tsconfig.json ├── scripts ├── bootstrap.py └── refresh-affiliation.ts ├── tailwind.config.ts ├── sentry.server.config.ts ├── sentry.client.config.ts ├── sentry.edge.config.ts ├── package.json └── next.config.mjs /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "next/core-web-vitals" 3 | } 4 | -------------------------------------------------------------------------------- /bunfig.toml: -------------------------------------------------------------------------------- 1 | [install.scopes] 2 | "@jsr" = "https://npm.jsr.io" 3 | -------------------------------------------------------------------------------- /bun.lockb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buttondown/shovel/HEAD/bun.lockb -------------------------------------------------------------------------------- /app/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buttondown/shovel/HEAD/app/favicon.ico -------------------------------------------------------------------------------- /app/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | -------------------------------------------------------------------------------- /lib/affiliations/types.ts: -------------------------------------------------------------------------------- 1 | export type Affiliation = { 2 | domain: string; 3 | metadata: Record; 4 | }; 5 | -------------------------------------------------------------------------------- /vercel.json: -------------------------------------------------------------------------------- 1 | { 2 | "crons": [ 3 | { 4 | "path": "/api/crons/refresh", 5 | "schedule": "* * * * *" 6 | } 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /.env.sample: -------------------------------------------------------------------------------- 1 | DATABASE_URL=DATABASE_URL 2 | DISABLE_PUPPETEER=true 3 | DISABLE_DATABASE=true 4 | SHOVEL_PRO_URL=https://buy.stripe.com/28o8xEc6ZdpSe9GaEG 5 | DISABLE_ICONHORSE=true 6 | -------------------------------------------------------------------------------- /postcss.config.mjs: -------------------------------------------------------------------------------- 1 | /** @type {import('postcss-load-config').Config} */ 2 | const config = { 3 | plugins: { 4 | tailwindcss: {}, 5 | }, 6 | }; 7 | 8 | export default config; 9 | -------------------------------------------------------------------------------- /components/SectionHeader.tsx: -------------------------------------------------------------------------------- 1 | const SectionHeader = ({ children }: { children: React.ReactNode }) => ( 2 |

{children}

3 | ); 4 | 5 | export default SectionHeader; 6 | -------------------------------------------------------------------------------- /lib/loaders/types.ts: -------------------------------------------------------------------------------- 1 | export type Record = { 2 | value: string; 3 | type: string; 4 | }; 5 | 6 | export type RecordGroup = { 7 | label: string; 8 | data: Record[]; 9 | }; 10 | 11 | export type Loader = (domain: string) => Promise; 12 | -------------------------------------------------------------------------------- /instrumentation.ts: -------------------------------------------------------------------------------- 1 | export async function register() { 2 | if (process.env.NEXT_RUNTIME === 'nodejs') { 3 | await import('./sentry.server.config'); 4 | } 5 | 6 | if (process.env.NEXT_RUNTIME === 'edge') { 7 | await import('./sentry.edge.config'); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Architecture 2 | 3 | There are two types of plugins: `loaders` and `parsers`. 4 | 5 | - `loaders` collect raw data about a given domain (whois, HTML, DNS.) 6 | - `parsers` query that raw data to create `Notes` (this domain uses Mailgun; their HTML contains a reference to Fathom). 7 | -------------------------------------------------------------------------------- /lib/constants.ts: -------------------------------------------------------------------------------- 1 | // The `www` is deliberate; as of this writing, we explicitly redirect from the 2 | // root domain to the `www` subdomain, so a `shovel.report` canonical URL would be 3 | // incorrect (since it would 308 to the `www` subdomain). 4 | export const CANONICAL_URL = "https://www.shovel.report"; 5 | -------------------------------------------------------------------------------- /public/icons/twitter.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | X 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /lib/db/connection.ts: -------------------------------------------------------------------------------- 1 | import { Kysely, PostgresDialect } from "kysely"; 2 | import { Pool } from 'pg'; 3 | import { Database } from "./types"; 4 | 5 | export const db = new Kysely({ 6 | dialect: new PostgresDialect({ 7 | pool: new Pool({ 8 | connectionString: process.env.DATABASE_URL, 9 | }) 10 | }), 11 | }); 12 | -------------------------------------------------------------------------------- /lib/parsers/types.ts: -------------------------------------------------------------------------------- 1 | import type { RecordGroup } from "../loaders/types"; 2 | import type { REGISTRY } from "../services"; 3 | 4 | export type DetectedTechnology = { 5 | identifier: keyof typeof REGISTRY; 6 | metadata: Record; 7 | }; 8 | 9 | export type Parser = ( 10 | domain: string, 11 | data: RecordGroup[], 12 | ) => Promise; 13 | -------------------------------------------------------------------------------- /components/DomainIcon.tsx: -------------------------------------------------------------------------------- 1 | const DISABLE_ICONHORSE = process.env.DISABLE_ICONHORSE === "true"; 2 | 3 | const DomainIcon = ({ domain }: { domain: string }) => { 4 | if (DISABLE_ICONHORSE) { 5 | return null; 6 | } 7 | return {`Icon; 8 | }; 9 | 10 | export default DomainIcon; 11 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: 3 | push: 4 | branches: 5 | - main 6 | jobs: 7 | test: 8 | name: Test 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v4 12 | - uses: oven-sh/setup-bun@v2 13 | - uses: extractions/setup-just@v2 14 | - run: just install 15 | - run: just test 16 | - run: just e2e-test 17 | -------------------------------------------------------------------------------- /playwright.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'playwright/test'; 2 | 3 | export default defineConfig({ 4 | webServer: { 5 | command: 'just server-playwright', 6 | url: 'http://127.0.0.1:3045', 7 | reuseExistingServer: !process.env.CI, 8 | stdout: 'ignore', 9 | stderr: 'pipe', 10 | }, 11 | testMatch: '*.e2e-test.ts', 12 | fullyParallel: true, 13 | }); 14 | -------------------------------------------------------------------------------- /components/Header.tsx: -------------------------------------------------------------------------------- 1 | const Header = ({ 2 | url, 3 | children, 4 | }: { 5 | url: string; 6 | children: React.ReactNode; 7 | }) => { 8 | return ( 9 |

10 | 16 | {children} 17 | 18 |

19 | ); 20 | }; 21 | 22 | export default Header; 23 | -------------------------------------------------------------------------------- /justfile: -------------------------------------------------------------------------------- 1 | server-dev: 2 | bun dev | pino-pretty 3 | 4 | server-playwright: 5 | PINO_LEVEL=silent DISABLE_DATABASE=true DISABLE_PUPPETEER=true DISABLE_ICONHORSE=true bun playwright 6 | 7 | install: 8 | bun i 9 | bunx playwright install 10 | 11 | test *args: 12 | PINO_LEVEL=silent DISABLE_DATABASE=true DISABLE_PUPPETEER=true DISABLE_ICONHORSE=true bun test {{args}} 13 | 14 | e2e-test *args: 15 | PINO_LEVEL=silent DISABLE_DATABASE=true DISABLE_PUPPETEER=true DISABLE_ICONHORSE=true bun e2e-test {{args}} 16 | -------------------------------------------------------------------------------- /lib/seo.e2e-test.ts: -------------------------------------------------------------------------------- 1 | import { expect, test } from "playwright/test"; 2 | 3 | 4 | const REPRESENTATIVE_ROUTES = [ 5 | "/domain/buttondown.com", 6 | "/technology/cloudflare", 7 | "/genre/crm" 8 | ] 9 | 10 | const BASE_URL = "http://127.0.0.1:3045"; 11 | 12 | REPRESENTATIVE_ROUTES.forEach((route) => { 13 | test(`${route} has an h1 element`, async ({ page }) => { 14 | await page.goto(`${BASE_URL}${route}`); 15 | const h1 = await page.$('h1'); 16 | expect(h1).not.toBeNull(); 17 | }); 18 | }); 19 | -------------------------------------------------------------------------------- /lib/affiliations/loaders/tranco.ts: -------------------------------------------------------------------------------- 1 | import type { Affiliation } from "../types"; 2 | 3 | const TRANCO_URL = "https://tranco-list.eu/download/KJ94W/1000000"; 4 | 5 | export default async function* load(): AsyncGenerator { 6 | const response = await fetch(TRANCO_URL); 7 | const data = await response.text(); 8 | const lines = data 9 | .split("\n") 10 | .filter((line) => line.trim() !== "") 11 | .map((line) => line.split(",")); 12 | 13 | for (const [rank, domain] of lines) { 14 | yield { 15 | domain, 16 | metadata: { 17 | rank: rank, 18 | }, 19 | }; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /public/vercel.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | .yarn/install-state.gz 8 | 9 | # testing 10 | /coverage 11 | 12 | # next.js 13 | /.next/ 14 | /out/ 15 | 16 | # production 17 | /build 18 | 19 | # misc 20 | .DS_Store 21 | *.pem 22 | 23 | # debug 24 | npm-debug.log* 25 | yarn-debug.log* 26 | yarn-error.log* 27 | 28 | # local env files 29 | .env*.local 30 | 31 | # vercel 32 | .vercel 33 | 34 | # typescript 35 | *.tsbuildinfo 36 | next-env.d.ts 37 | .env 38 | 39 | # Sentry Config File 40 | .env.sentry-build-plugin 41 | -------------------------------------------------------------------------------- /lib/parsers/affiliations.ts: -------------------------------------------------------------------------------- 1 | import { db } from "@/lib/db/connection"; 2 | import type { RecordGroup } from "../loaders/types"; 3 | import type { Parser } from "./types"; 4 | 5 | const parse: Parser = async (domain: string, data: RecordGroup[]) => { 6 | if (process.env.DISABLE_DATABASE === "true") { 7 | return []; 8 | } 9 | 10 | const affiliations = await db 11 | .selectFrom("affiliations") 12 | .where("domain", "=", domain) 13 | .selectAll() 14 | .execute(); 15 | 16 | return affiliations.map((affiliation) => ({ 17 | identifier: affiliation.identifier, 18 | metadata: affiliation.metadata, 19 | })); 20 | }; 21 | 22 | const exports = { parse }; 23 | export default exports; 24 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["dom", "dom.iterable", "esnext"], 4 | "allowJs": true, 5 | "skipLibCheck": true, 6 | "strict": true, 7 | "noEmit": true, 8 | "esModuleInterop": true, 9 | "module": "esnext", 10 | "moduleResolution": "bundler", 11 | "resolveJsonModule": true, 12 | "isolatedModules": true, 13 | "jsx": "preserve", 14 | "incremental": true, 15 | "plugins": [ 16 | { 17 | "name": "next" 18 | } 19 | ], 20 | "paths": { 21 | "@/*": ["./*"] 22 | } 23 | }, 24 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], 25 | "exclude": ["node_modules"] 26 | } 27 | -------------------------------------------------------------------------------- /scripts/bootstrap.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | import requests 4 | 5 | FILENAME = "data/hn.csv" 6 | NUMBER_OF_THREADS = 8 7 | 8 | domains = open(FILENAME, "r").read().split("\n") 9 | 10 | 11 | def process(domain): 12 | print(f"Requesting {domain}") 13 | requests.get(f"https://shovel.report/{domain}") 14 | 15 | 16 | def chunkify(lst, n): 17 | return [lst[i::n] for i in range(n)] 18 | 19 | 20 | if __name__ == "__main__": 21 | domain_batches = chunkify(domains, NUMBER_OF_THREADS) 22 | threads = [] 23 | for batch in domain_batches: 24 | t = threading.Thread(target=lambda: [process(domain) for domain in batch]) 25 | threads.append(t) 26 | t.start() 27 | -------------------------------------------------------------------------------- /tailwind.config.ts: -------------------------------------------------------------------------------- 1 | import type { Config } from "tailwindcss"; 2 | 3 | const config: Config = { 4 | content: [ 5 | "./pages/**/*.{js,ts,jsx,tsx,mdx}", 6 | "./components/**/*.{js,ts,jsx,tsx,mdx}", 7 | "./app/**/*.{js,ts,jsx,tsx,mdx}", 8 | "./lib/services.tsx", 9 | ], 10 | theme: { 11 | extend: { 12 | backgroundImage: { 13 | "gradient-radial": "radial-gradient(var(--tw-gradient-stops))", 14 | "gradient-conic": 15 | "conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))", 16 | }, 17 | }, 18 | }, 19 | plugins: [], 20 | }; 21 | export default config; 22 | -------------------------------------------------------------------------------- /lib/affiliations/loaders/osspledge.ts: -------------------------------------------------------------------------------- 1 | import { extractDomain } from "../../utils"; 2 | import type { Affiliation } from "../types"; 3 | 4 | const OSSPLEDGE_URL = 5 | "https://raw.githubusercontent.com/opensourcepledge/osspledge.com/main/members.csv"; 6 | 7 | export default async function* load(): AsyncGenerator { 8 | const response = await fetch(OSSPLEDGE_URL); 9 | const data = await response.text(); 10 | const lines = data 11 | .split("\n") 12 | .map((line) => line.split(",")[1]) 13 | .filter((l) => l !== undefined); 14 | 15 | for (const url of lines) { 16 | yield { 17 | domain: extractDomain(url), 18 | metadata: { 19 | url: url, 20 | }, 21 | }; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /lib/db/domains-by-technology.ts: -------------------------------------------------------------------------------- 1 | import { db } from "@/lib/db/connection"; 2 | 3 | const fetchDomainsByTechnology = async (technology: string, limit: number) => { 4 | const data = process.env.DISABLE_DATABASE 5 | ? { data: [], count: 0 } 6 | : await db 7 | .selectFrom("detected_technologies") 8 | .where("technology", "=", technology) 9 | .selectAll() 10 | .distinctOn("domain") 11 | .execute() 12 | .then((results) => { 13 | return { 14 | data: results.slice(0, limit), 15 | count: results.length, 16 | }; 17 | }); 18 | 19 | return data; 20 | }; 21 | 22 | export default fetchDomainsByTechnology; 23 | -------------------------------------------------------------------------------- /app/global-error.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import * as Sentry from "@sentry/nextjs"; 4 | import NextError from "next/error"; 5 | import { useEffect } from "react"; 6 | 7 | export default function GlobalError({ error }: { error: Error & { digest?: string } }) { 8 | useEffect(() => { 9 | Sentry.captureException(error); 10 | }, [error]); 11 | 12 | return ( 13 | 14 | 15 | {/* `NextError` is the default Next.js error page component. Its type 16 | definition requires a `statusCode` prop. However, since the App Router 17 | does not expose status codes for errors, we simply pass 0 to render a 18 | generic error message. */} 19 | 20 | 21 | 22 | ); 23 | } -------------------------------------------------------------------------------- /sentry.server.config.ts: -------------------------------------------------------------------------------- 1 | // This file configures the initialization of Sentry on the server. 2 | // The config you add here will be used whenever the server handles a request. 3 | // https://docs.sentry.io/platforms/javascript/guides/nextjs/ 4 | 5 | import * as Sentry from "@sentry/nextjs"; 6 | 7 | Sentry.init({ 8 | dsn: "https://f177f674bdf028478b1fcdd47477c9d6@o97520.ingest.us.sentry.io/4507885378142208", 9 | 10 | // Define how likely traces are sampled. Adjust this value in production, or use tracesSampler for greater control. 11 | tracesSampleRate: 1, 12 | 13 | // Setting this option to true will print useful information to the console while you're setting up Sentry. 14 | debug: process.env.NODE_ENV !== "production", 15 | enabled: process.env.NODE_ENV === "production", 16 | }); 17 | -------------------------------------------------------------------------------- /sentry.client.config.ts: -------------------------------------------------------------------------------- 1 | // This file configures the initialization of Sentry on the client. 2 | // The config you add here will be used whenever a users loads a page in their browser. 3 | // https://docs.sentry.io/platforms/javascript/guides/nextjs/ 4 | 5 | import * as Sentry from "@sentry/nextjs"; 6 | 7 | Sentry.init({ 8 | dsn: "https://f177f674bdf028478b1fcdd47477c9d6@o97520.ingest.us.sentry.io/4507885378142208", 9 | 10 | // Define how likely traces are sampled. Adjust this value in production, or use tracesSampler for greater control. 11 | tracesSampleRate: 1, 12 | 13 | // Setting this option to true will print useful information to the console while you're setting up Sentry. 14 | debug: process.env.NODE_ENV !== "production", 15 | enabled: process.env.NODE_ENV === "production", 16 | }); 17 | -------------------------------------------------------------------------------- /lib/utils.ts: -------------------------------------------------------------------------------- 1 | const recursivelyStringify = (obj: T): string => { 2 | const allKeys: Set = new Set(); 3 | JSON.stringify(obj, (key, value) => { 4 | allKeys.add(key); 5 | return value; 6 | }); 7 | return JSON.stringify(obj, Array.from(allKeys).sort()); 8 | }; 9 | 10 | export const unique = ( 11 | arr: T[], 12 | keyFn?: (obj: T) => string, 13 | ) => { 14 | // Objects can be complex, so we can't use Set here 15 | return arr.filter( 16 | (v, i, a) => 17 | a.findIndex( 18 | (t) => 19 | recursivelyStringify(keyFn ? keyFn(t) : t) === 20 | recursivelyStringify(keyFn ? keyFn(v) : v), 21 | ) === i, 22 | ); 23 | }; 24 | 25 | export const extractDomain = (url: string) => { 26 | const parsedUrl = new URL(url); 27 | return parsedUrl.hostname.replace(/^www\./, ""); 28 | }; 29 | -------------------------------------------------------------------------------- /lib/affiliations/registry.ts: -------------------------------------------------------------------------------- 1 | import osspledge from "./loaders/osspledge"; 2 | import tranco from "./loaders/tranco"; 3 | import ycombinator from "./loaders/ycombinator"; 4 | import type { Affiliation } from "./types"; 5 | 6 | type RegisteredAffiliation = { 7 | identifier: string; 8 | name: string; 9 | domain?: string; 10 | load: () => AsyncGenerator; 11 | }; 12 | 13 | export const REGISTRY: { [key in string]: RegisteredAffiliation } = { 14 | tranco: { 15 | identifier: "tranco", 16 | name: "Tranco", 17 | load: tranco, 18 | }, 19 | ycombinator: { 20 | identifier: "ycombinator", 21 | name: "Y Combinator", 22 | load: ycombinator, 23 | domain: "ycombinator.com", 24 | }, 25 | osspledge: { 26 | identifier: "osspledge", 27 | name: "OSS Pledge", 28 | load: osspledge, 29 | domain: "osspledge.com", 30 | }, 31 | }; 32 | -------------------------------------------------------------------------------- /lib/parsers/headers.ts: -------------------------------------------------------------------------------- 1 | import type { RecordGroup } from "../loaders/types"; 2 | import { REGISTRY } from "../services"; 3 | import type { Parser } from "./types"; 4 | 5 | const parse: Parser = (domain: string, data: RecordGroup[]) => { 6 | return Promise.resolve( 7 | data 8 | .flatMap((datum) => datum.data) 9 | .flatMap((d) => { 10 | const servicesWithHeaders = Object.values(REGISTRY).filter( 11 | (service) => service.headers, 12 | ); 13 | return servicesWithHeaders.filter( 14 | (service) => 15 | d.type.includes(service.headers?.key || "") && 16 | (service.headers?.value === "*" || 17 | d.value.includes(service.headers?.value || "")), 18 | ); 19 | }) 20 | .map((service) => ({ 21 | identifier: service.identifier, 22 | metadata: { 23 | via: "headers", 24 | }, 25 | })), 26 | ); 27 | }; 28 | 29 | const exports = { parse }; 30 | export default exports; 31 | -------------------------------------------------------------------------------- /sentry.edge.config.ts: -------------------------------------------------------------------------------- 1 | // This file configures the initialization of Sentry for edge features (middleware, edge routes, and so on). 2 | // The config you add here will be used whenever one of the edge features is loaded. 3 | // Note that this config is unrelated to the Vercel Edge Runtime and is also required when running locally. 4 | // https://docs.sentry.io/platforms/javascript/guides/nextjs/ 5 | 6 | import * as Sentry from "@sentry/nextjs"; 7 | 8 | Sentry.init({ 9 | dsn: "https://f177f674bdf028478b1fcdd47477c9d6@o97520.ingest.us.sentry.io/4507885378142208", 10 | 11 | // Define how likely traces are sampled. Adjust this value in production, or use tracesSampler for greater control. 12 | tracesSampleRate: 1, 13 | 14 | // Setting this option to true will print useful information to the console while you're setting up Sentry. 15 | debug: process.env.NODE_ENV !== "production", 16 | enabled: process.env.NODE_ENV === "production", 17 | }); 18 | -------------------------------------------------------------------------------- /lib/db/types.ts: -------------------------------------------------------------------------------- 1 | import type { 2 | ColumnType, 3 | Insertable, 4 | JSONColumnType, 5 | Selectable, 6 | } from "kysely"; 7 | 8 | export interface Database { 9 | detected_technologies: DetectedTechnologyTable; 10 | affiliations: AffiliationTable; 11 | } 12 | 13 | export interface AffiliationTable { 14 | domain: string; 15 | identifier: string; 16 | metadata: JSONColumnType; 17 | creation_date: ColumnType; 18 | } 19 | 20 | export interface DetectedTechnologyTable { 21 | domain: string; 22 | technology: string; 23 | data: JSONColumnType; 24 | creation_date: ColumnType; 25 | } 26 | 27 | export type DetectedTechnology = Selectable; 28 | export type NewDetectedTechnology = Insertable; 29 | 30 | export type Affiliation = Selectable; 31 | export type NewAffiliation = Insertable; 32 | -------------------------------------------------------------------------------- /lib/db/domains.ts: -------------------------------------------------------------------------------- 1 | import type fetch from "@/lib/data"; 2 | import { db } from "@/lib/db/connection"; 3 | 4 | export const reify = async ( 5 | domain: string, 6 | data: Awaited>, 7 | ) => { 8 | const existingTechnologies = await db 9 | .selectFrom("detected_technologies") 10 | .select("technology") 11 | .where("domain", "=", domain) 12 | .execute(); 13 | 14 | const existingTechSet = new Set( 15 | existingTechnologies.map((tech) => tech.technology), 16 | ); 17 | 18 | const newTechnologies = data.detected_technologies 19 | .filter((technology) => !existingTechSet.has(technology.identifier)) 20 | .map((technology) => ({ 21 | domain: domain, 22 | technology: technology.identifier, 23 | data: JSON.stringify(technology.metadata), 24 | creation_date: new Date().toISOString(), 25 | })); 26 | 27 | if (newTechnologies.length > 0) { 28 | await db 29 | .insertInto("detected_technologies") 30 | .values(newTechnologies) 31 | .execute(); 32 | } 33 | }; 34 | -------------------------------------------------------------------------------- /app/sitemap.ts: -------------------------------------------------------------------------------- 1 | import { GENRE_REGISTRY, REGISTRY } from '@/lib/services' 2 | import type { MetadataRoute } from 'next' 3 | 4 | const BASE_URL = "https://shovel.report" 5 | 6 | export default function sitemap(): MetadataRoute.Sitemap { 7 | const technologies = Object.keys(REGISTRY) 8 | return [ 9 | { 10 | url: BASE_URL, 11 | lastModified: new Date(), 12 | changeFrequency: 'daily', 13 | priority: 1, 14 | }, 15 | // @ts-ignore 16 | ...technologies.map(technology => ({ 17 | url: `${BASE_URL}/technology/${technology}`, 18 | lastModified: new Date(), 19 | changeFrequency: 'daily', 20 | priority: 0.8, 21 | })), 22 | // @ts-ignore 23 | ...Object.keys(GENRE_REGISTRY).map(genre => ({ 24 | url: `${BASE_URL}/genre/${genre}`, 25 | lastModified: new Date(), 26 | changeFrequency: 'daily', 27 | priority: 0.8, 28 | })), 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /app/api/domains/route.ts: -------------------------------------------------------------------------------- 1 | import fetchDomainsByTechnology from "@/lib/db/domains-by-technology"; 2 | import { NextRequest } from "next/server"; 3 | 4 | export async function GET( 5 | request: NextRequest, 6 | context: { 7 | }, 8 | ) { 9 | const technology = request.nextUrl.searchParams.get("technology"); 10 | if (!technology) { 11 | return Response.json({ 12 | error: "Missing required parameter: technology", 13 | }, { 14 | status: 400, 15 | }); 16 | } 17 | 18 | const data = await fetchDomainsByTechnology(technology, 100); 19 | return Response.json({ 20 | // We really need to decide on some sort of way to enforce a contract here. 21 | // Maybe zod-openapi? 22 | data: data.data.map((item) => { 23 | return { 24 | domain: item.domain, 25 | creation_date: item.creation_date, 26 | } 27 | }).sort((a, b) => b.creation_date.getTime() - a.creation_date.getTime()), 28 | count: data.count, 29 | }); 30 | } 31 | -------------------------------------------------------------------------------- /lib/loaders/dns_prefix.ts: -------------------------------------------------------------------------------- 1 | import dns from "dns/promises"; 2 | import { REGISTRY } from "../services"; 3 | import { Loader } from "./types"; 4 | 5 | const SERVICES_WITH_DNS_PREFIX = Object.entries(REGISTRY).filter( 6 | ([identifier, service]) => service.dns_prefix 7 | ).map(([identifier, service]) => identifier); 8 | 9 | const load: Loader = async (domain: string) => { 10 | const allRecords = await Promise.all( 11 | SERVICES_WITH_DNS_PREFIX.map(async (identifier) => { 12 | try { 13 | const records = await dns.resolveTxt(`_${identifier}.${domain}`); 14 | return records.map((record: any) => ({ 15 | value: record[0], 16 | type: identifier, 17 | })); 18 | } catch (error) { 19 | return []; 20 | } 21 | }) 22 | ); 23 | return { 24 | label: "SERVICE", 25 | data: allRecords.flat(), 26 | }; 27 | }; 28 | 29 | const exports = { load, name: "dns_prefix" }; 30 | export default exports; 31 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "shovel", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev --port 3044", 7 | "playwright": "next dev --port 3045", 8 | "build": "next build", 9 | "start": "next start", 10 | "lint": "next lint", 11 | "e2e-test": "playwright test" 12 | }, 13 | "dependencies": { 14 | "@radix-ui/react-dialog": "^1.1.1", 15 | "@sentry/nextjs": "^8", 16 | "@vercel/analytics": "^1.3.1", 17 | "fast-xml-parser": "^4.5.0", 18 | "fetch-h2": "^3.0.2", 19 | "kysely": "^0.27.4", 20 | "next": "14.2.3", 21 | "node-html-parser": "^6.1.13", 22 | "pg": "^8.13.0", 23 | "pino": "^9.2.0", 24 | "puppeteer": "^23.2.1", 25 | "react": "^18", 26 | "react-dom": "^18", 27 | "whois": "^2.14.0" 28 | }, 29 | "devDependencies": { 30 | "@types/node": "^20", 31 | "@types/react": "^18", 32 | "@types/react-dom": "^18", 33 | "eslint": "^8", 34 | "eslint-config-next": "14.2.3", 35 | "playwright": "^1.46.1", 36 | "postcss": "^8", 37 | "tailwindcss": "^3.4.1", 38 | "typescript": "^5" 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /app/layout.tsx: -------------------------------------------------------------------------------- 1 | import { CANONICAL_URL } from "@/lib/constants"; 2 | import { Analytics } from "@vercel/analytics/react"; 3 | import { Metadata } from "next"; 4 | import "./globals.css"; 5 | 6 | export const metadata: Metadata = { 7 | metadataBase: new URL(CANONICAL_URL), 8 | alternates: { 9 | canonical: "/", 10 | languages: { 11 | "en-US": "/en-US", 12 | }, 13 | }, 14 | }; 15 | 16 | export default function RootLayout({ 17 | children, 18 | }: Readonly<{ 19 | children: React.ReactNode; 20 | }>) { 21 | return ( 22 | 23 | 24 | 25 |
26 | 33 |
{children}
34 |
35 | © {new Date().getFullYear()}. I hope you have a nice day. 36 |
37 |
38 | 39 | 40 | ); 41 | } 42 | -------------------------------------------------------------------------------- /lib/affiliations/loaders/ycombinator.ts: -------------------------------------------------------------------------------- 1 | import { XMLParser } from "fast-xml-parser"; 2 | import { extractDomain } from "../../utils"; 3 | import type { Affiliation } from "../types"; 4 | 5 | const SITEMAP_URL = "https://www.ycombinator.com/companies/sitemap.xml"; 6 | 7 | export default async function* load(): AsyncGenerator { 8 | const response = await fetch(SITEMAP_URL); 9 | const data = await response.text(); 10 | const parser = new XMLParser(); 11 | const result = parser.parse(data); 12 | const relevantURLs = result.urlset.url.filter( 13 | (url: { loc: string }) => !url.loc.includes("/industry/"), 14 | ); 15 | 16 | for (const url of relevantURLs) { 17 | try { 18 | const companyResponse = await fetch(url.loc); 19 | const companyHtml = await companyResponse.text(); 20 | const hrefMatch = companyHtml.match( 21 | /href="([^"]*)"[^>]*class="[^"]*mb-2[^"]*whitespace-nowrap[^"]*"/, 22 | ); 23 | 24 | if (hrefMatch?.[1]) { 25 | if (hrefMatch[1] === "https://") { 26 | continue; 27 | } 28 | 29 | yield { 30 | domain: extractDomain(hrefMatch[1]), 31 | metadata: { 32 | originalUrl: url.loc, 33 | }, 34 | }; 35 | } 36 | } catch (error) { 37 | console.error(`Error fetching ${url.loc}:`, error); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /components/Grid.tsx: -------------------------------------------------------------------------------- 1 | import DomainIcon from "@/components/DomainIcon"; 2 | import SectionHeader from "@/components/SectionHeader"; 3 | 4 | const Container = ({ 5 | children, 6 | title, 7 | }: { 8 | children: React.ReactNode; 9 | title?: string; 10 | }) => { 11 | return ( 12 |
13 |
14 | {children} 15 |
16 | {title && ( 17 |
18 | {title} 19 |
20 | )} 21 |
22 | ); 23 | }; 24 | 25 | const Item = ({ 26 | children, 27 | url, 28 | domain, 29 | }: { 30 | children: React.ReactNode; 31 | url?: string; 32 | domain?: string; 33 | }) => { 34 | return ( 35 | 39 | {domain && } 40 | {children} 41 | 42 | ); 43 | }; 44 | 45 | export default { Container, Item }; 46 | -------------------------------------------------------------------------------- /public/next.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /components/TechnologyPill.tsx: -------------------------------------------------------------------------------- 1 | import { REGISTRY } from "@/lib/services"; 2 | import DomainIcon from "./DomainIcon"; 3 | 4 | type Props = { 5 | technology: string; 6 | subtitle?: string; 7 | }; 8 | 9 | const ServicePill = ({ service }: { service: string }) => ( 10 |
11 | {REGISTRY[service]?.icon || 12 | (REGISTRY[service]?.url ? ( 13 | 16 | ) : ( 17 | {service} 18 | ))} 19 |
20 | ); 21 | 22 | 23 | const TechnologyPill = ({ technology, subtitle }: Props) => { 24 | return ( 25 | 26 |
  • 29 | 30 |
    {REGISTRY[technology]?.name || technology}
    31 | {subtitle ? ( 32 |
    33 | {subtitle} 34 |
    35 | ) : ( 36 |
    37 | {REGISTRY[technology]?.genre} 38 |
    39 | )} 40 |
    • 41 |     42 | ); 43 | }; 44 | 45 | export default TechnologyPill; 46 | -------------------------------------------------------------------------------- /app/api/domains/[domain]/route.ts: -------------------------------------------------------------------------------- 1 | import fetch from "@/lib/data"; 2 | import { reify } from "@/lib/db/domains"; 3 | 4 | const SOCIAL_MEDIA_SERVICES = [ 5 | "facebook", 6 | "twitter", 7 | "instagram", 8 | "linkedin", 9 | "youtube", 10 | "github", 11 | ]; 12 | 13 | export async function GET( 14 | request: Request, 15 | context: { 16 | params: { 17 | domain: string; 18 | }; 19 | }, 20 | ) { 21 | const rawResponse = await fetch(context.params.domain); 22 | await reify(context.params.domain, rawResponse); 23 | 24 | return Response.json({ 25 | domain: context.params.domain, 26 | records: rawResponse.data 27 | .filter((datum) => datum.label === "DNS") 28 | .flatMap((datum) => datum.data), 29 | ranking: rawResponse.data.find((datum) => datum.label === "Tranco")?.data[0] 30 | ?.value, 31 | services: rawResponse.detected_technologies 32 | .filter((technology) => technology.identifier !== "subdomain") 33 | .map((technology) => technology.identifier) 34 | .sort(), 35 | subdomains: rawResponse.detected_technologies 36 | .filter((technology) => technology.identifier === "subdomain") 37 | .map((technology) => technology.metadata.value) 38 | .sort(), 39 | social_media: Object.fromEntries( 40 | SOCIAL_MEDIA_SERVICES.map((service) => [ 41 | service, 42 | rawResponse.detected_technologies.find( 43 | (note) => note.identifier === service, 44 | )?.metadata.username, 45 | ]), 46 | ), 47 | }); 48 | } 49 | -------------------------------------------------------------------------------- /lib/loaders/dns.ts: -------------------------------------------------------------------------------- 1 | import dns from "dns/promises"; 2 | import { Loader } from "./types"; 3 | 4 | export type DNSRecord = { 5 | value: string; 6 | type: string; 7 | }; 8 | 9 | const TYPE_TO_LOOKUP = { 10 | MX: dns.resolveMx, 11 | A: dns.resolve4, 12 | AAAA: dns.resolve6, 13 | CNAME: dns.resolveCname, 14 | TXT: dns.resolveTxt, 15 | NS: dns.resolveNs, 16 | }; 17 | 18 | const TYPE_TO_MUNGING_FUNCTION: { 19 | [key: string]: (record: any) => string; 20 | } = { 21 | MX: ({ exchange, priority }: { exchange: string; priority: number }) => 22 | `${exchange} (priority: ${priority})`, 23 | TXT: (record: string[]) => record.join(""), 24 | }; 25 | 26 | const load: Loader = async (domain: string) => { 27 | const allRecords = await Promise.all( 28 | Object.entries(TYPE_TO_LOOKUP).map(async ([type, method]) => { 29 | try { 30 | const records = await method(domain); 31 | const mungedRecords = records.map((record: any) => ({ 32 | value: TYPE_TO_MUNGING_FUNCTION[type] 33 | ? TYPE_TO_MUNGING_FUNCTION[type](record) 34 | : record, 35 | type, 36 | })); 37 | return mungedRecords; 38 | } catch (error) { 39 | return []; 40 | } 41 | }) 42 | ); 43 | const sortedRecords = allRecords 44 | .flat() 45 | .sort((a, b) => `${a.type}${a.value}`.localeCompare(`${b.type}${b.value}`)); 46 | return { 47 | label: "DNS", 48 | data: sortedRecords, 49 | }; 50 | }; 51 | 52 | const exports = { load, name: "dns" }; 53 | export default exports; 54 | -------------------------------------------------------------------------------- /lib/loaders/html.ts: -------------------------------------------------------------------------------- 1 | import puppeteer from "puppeteer"; 2 | import type { Loader } from "./types"; 3 | 4 | const load: Loader = async (domain: string) => { 5 | try { 6 | if (process.env.DISABLE_PUPPETEER !== "true") { 7 | const browser = await puppeteer.launch(); 8 | const page = await browser.newPage(); 9 | const response = await page.goto(`https://${domain}`, { 10 | waitUntil: "networkidle0", 11 | }); 12 | const html = await page.content(); 13 | const headers = await response?.headers(); 14 | await browser.close(); 15 | return { 16 | label: "HTML", 17 | data: [ 18 | { 19 | value: html, 20 | type: "text/html", 21 | }, 22 | ...Object.entries(headers || {}).map(([key, value]) => ({ 23 | value: value || "", 24 | type: `text/headers/${key}`, 25 | })), 26 | ], 27 | }; 28 | } 29 | const response = await fetch(`https://${domain}`); 30 | const html = await response.text(); 31 | const headers = response.headers; 32 | return { 33 | label: "HTML", 34 | data: [ 35 | { 36 | value: html, 37 | type: "text/html", 38 | }, 39 | ...Object.entries(headers).map(([key, value]) => ({ 40 | value: value[0] || "", 41 | type: `text/headers/${key}`, 42 | })), 43 | ], 44 | }; 45 | } catch (error) { 46 | return { 47 | label: "HTML", 48 | data: [ 49 | { 50 | value: "Error loading HTML", 51 | type: "text/error", 52 | }, 53 | ], 54 | }; 55 | } 56 | }; 57 | 58 | const exports = { load, name: "html" }; 59 | export default exports; 60 | -------------------------------------------------------------------------------- /app/api/crons/refresh/route.ts: -------------------------------------------------------------------------------- 1 | import fetch from "@/lib/data"; 2 | import { db } from "@/lib/db/connection"; 3 | import { reify } from "@/lib/db/domains"; 4 | import { sql } from "kysely"; 5 | import pino from "pino"; 6 | 7 | const logger = pino({ 8 | name: "cron-refresh", 9 | }); 10 | 11 | // Without this comment, Next.js will cache the response 12 | // and therefore this endpoint does not do anything. (I really wish 13 | // we had a simpler way to trigger and manage these; might be worth looking 14 | // into Trigger at some point.) 15 | export const revalidate = 0; 16 | 17 | // This `sample` is a cute trick to get a random sample of domains without 18 | // having to do a full table scan. 19 | const RAW_QUERY = sql<{ 20 | domain: string; 21 | }>` 22 | select domain from tranco TABLESAMPLE system (0.02) 23 | `; 24 | 25 | const getRandomDomains = async () => { 26 | const domains = await RAW_QUERY.execute(db); 27 | return domains.rows.map((row) => row.domain); 28 | }; 29 | 30 | const MAXIMUM_DOMAINS = 10; 31 | 32 | export async function GET( 33 | request: Request, 34 | context: { 35 | params: { 36 | domain: string; 37 | }; 38 | }, 39 | ) { 40 | const domains = await getRandomDomains(); 41 | const selectedDomains = domains.slice(0, MAXIMUM_DOMAINS); 42 | await Promise.all(selectedDomains.map(async (domain) => { 43 | logger.info({ 44 | message: "refresh.started", 45 | domain: domain, 46 | }); 47 | const rawResponse = await fetch(domain); 48 | await reify(domain, rawResponse); 49 | })); 50 | 51 | return Response.json({ 52 | domains: selectedDomains, 53 | }); 54 | } 55 | -------------------------------------------------------------------------------- /next.config.mjs: -------------------------------------------------------------------------------- 1 | import {withSentryConfig} from '@sentry/nextjs'; 2 | /** @type {import('next').NextConfig} */ 3 | const nextConfig = {}; 4 | 5 | export default withSentryConfig(nextConfig, { 6 | // For all available options, see: 7 | // https://github.com/getsentry/sentry-webpack-plugin#options 8 | 9 | org: "buttondown-email", 10 | project: "shovel", 11 | 12 | // Only print logs for uploading source maps in CI 13 | silent: !process.env.CI, 14 | 15 | // For all available options, see: 16 | // https://docs.sentry.io/platforms/javascript/guides/nextjs/manual-setup/ 17 | 18 | // Upload a larger set of source maps for prettier stack traces (increases build time) 19 | widenClientFileUpload: true, 20 | 21 | // Automatically annotate React components to show their full name in breadcrumbs and session replay 22 | reactComponentAnnotation: { 23 | enabled: true, 24 | }, 25 | 26 | // Route browser requests to Sentry through a Next.js rewrite to circumvent ad-blockers. 27 | // This can increase your server load as well as your hosting bill. 28 | // Note: Check that the configured route will not match with your Next.js middleware, otherwise reporting of client- 29 | // side errors will fail. 30 | tunnelRoute: "/monitoring", 31 | 32 | // Hides source maps from generated client bundles 33 | hideSourceMaps: true, 34 | 35 | // Automatically tree-shake Sentry logger statements to reduce bundle size 36 | disableLogger: true, 37 | 38 | // Enables automatic instrumentation of Vercel Cron Monitors. (Does not yet work with App Router route handlers.) 39 | // See the following for more information: 40 | // https://docs.sentry.io/product/crons/ 41 | // https://vercel.com/docs/cron-jobs 42 | automaticVercelMonitors: true, 43 | }); -------------------------------------------------------------------------------- /components/DomainSearchForm.tsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import { useRouter } from "next/navigation"; 4 | import { FormEvent, useState } from "react"; 5 | 6 | function normalizeDomain(input: string): string | null { 7 | const trimmed = input.trim(); 8 | 9 | if (!trimmed) { 10 | return null; 11 | } 12 | 13 | try { 14 | const candidate = trimmed.includes("://") 15 | ? new URL(trimmed) 16 | : new URL(`https://${trimmed}`); 17 | 18 | return candidate.hostname.replace(/\.$/, ""); 19 | } catch { 20 | const match = trimmed.match( 21 | /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+$/i 22 | ); 23 | return match ? match[0] : null; 24 | } 25 | } 26 | 27 | export default function DomainSearchForm() { 28 | const router = useRouter(); 29 | const [value, setValue] = useState(""); 30 | const [error, setError] = useState(null); 31 | 32 | const handleSubmit = (event: FormEvent) => { 33 | event.preventDefault(); 34 | 35 | const domain = normalizeDomain(value); 36 | 37 | if (!domain) { 38 | setError("Please enter a valid domain or URL."); 39 | return; 40 | } 41 | 42 | setError(null); 43 | router.push(`/domain/${encodeURIComponent(domain)}`); 44 | }; 45 | 46 | return ( 47 |
    48 | setValue(event.target.value)} 52 | placeholder="Enter domain or URL" 53 | className="bg-white/10 p-2 w-80" 54 | /> 55 | 61 | {error &&

    {error}

    } 62 |
    63 | ); 64 | } 65 | -------------------------------------------------------------------------------- /scripts/refresh-affiliation.ts: -------------------------------------------------------------------------------- 1 | import { REGISTRY } from "../lib/affiliations/registry"; 2 | import type { Affiliation } from "../lib/affiliations/types"; 3 | import { db } from "../lib/db/connection"; 4 | 5 | async function refreshAffiliation(identifier: string) { 6 | const affiliation = REGISTRY[identifier]; 7 | 8 | if (!affiliation) { 9 | console.error(`Affiliation "${identifier}" not found in the registry.`); 10 | process.exit(1); 11 | } 12 | 13 | if (!affiliation.load) { 14 | console.error( 15 | `Affiliation "${identifier}" does not have a loader function.`, 16 | ); 17 | process.exit(1); 18 | } 19 | 20 | try { 21 | console.log(`Refreshing affiliation: ${identifier}`); 22 | const generator = affiliation.load(); 23 | let batch: Affiliation[] = []; 24 | 25 | for await (const result of generator) { 26 | batch.push(result); 27 | 28 | if (batch.length === 10) { 29 | await db 30 | .insertInto("affiliations") 31 | .values( 32 | batch.map((item) => ({ 33 | domain: item.domain, 34 | identifier, 35 | metadata: JSON.stringify(item.metadata), 36 | creation_date: new Date().toISOString(), 37 | })), 38 | ) 39 | .execute(); 40 | batch = []; 41 | } 42 | } 43 | 44 | // Insert any remaining items 45 | if (batch.length > 0) { 46 | await db 47 | .insertInto("affiliations") 48 | .values( 49 | batch.map((item) => ({ 50 | domain: item.domain, 51 | identifier, 52 | metadata: JSON.stringify(item.metadata), 53 | creation_date: new Date().toISOString(), 54 | })), 55 | ) 56 | .execute(); 57 | } 58 | } catch (error) { 59 | console.error(`Error refreshing affiliation ${identifier}:`, error); 60 | process.exit(1); 61 | } 62 | } 63 | 64 | // Check if an affiliation name was provided as a command-line argument 65 | const affiliationArg = process.argv[2]; 66 | 67 | if (!affiliationArg) { 68 | console.error("Please provide an affiliation name as an argument."); 69 | process.exit(1); 70 | } 71 | 72 | refreshAffiliation(affiliationArg); 73 | -------------------------------------------------------------------------------- /lib/data.ts: -------------------------------------------------------------------------------- 1 | import dns from "@/lib/loaders/dns"; 2 | import dns_prefix from "@/lib/loaders/dns_prefix"; 3 | import html from "@/lib/loaders/html"; 4 | import affiliations from "@/lib/parsers/affiliations"; 5 | import records from "@/lib/parsers/dns"; 6 | import headers from "@/lib/parsers/headers"; 7 | import htmlRecords from "@/lib/parsers/html"; 8 | import { unique } from "@/lib/utils"; 9 | import pino from "pino"; 10 | import type { Loader, RecordGroup } from "./loaders/types"; 11 | import type { DetectedTechnology } from "./parsers/types"; 12 | 13 | const LOADERS = [dns, html, dns_prefix]; 14 | const PARSERS = [records, htmlRecords, headers, affiliations]; 15 | 16 | const logger = pino({ 17 | level: process.env.PINO_LEVEL || "warn", 18 | }); 19 | 20 | const load = async ( 21 | domain: string, 22 | loader: { 23 | load: Loader; 24 | name: string; 25 | }, 26 | ) => { 27 | logger.info({ message: "loader.started", domain, loader: loader.name }); 28 | const data = await loader.load(domain); 29 | logger.info({ message: "loader.ended", domain, loader: loader.name }); 30 | return data; 31 | }; 32 | 33 | const fetch = async ( 34 | domain: string, 35 | ): Promise<{ 36 | domain: string; 37 | data: RecordGroup[]; 38 | detected_technologies: DetectedTechnology[]; 39 | }> => { 40 | const data = [ 41 | ...(await Promise.all(LOADERS.map((loader) => load(domain, loader)))), 42 | { 43 | label: "URL", 44 | data: [ 45 | { 46 | value: `${domain}`, 47 | type: "text/url", 48 | }, 49 | ], 50 | }, 51 | ]; 52 | 53 | const detected_technologies = ( 54 | await Promise.all(PARSERS.map((parser) => parser.parse(domain, data))) 55 | ).flat(); 56 | 57 | return { 58 | domain, 59 | data: unique(data), 60 | detected_technologies: [ 61 | ...unique(detected_technologies, (n) => 62 | n.identifier === "subdomain" ? n.metadata.value : n.identifier, 63 | ), 64 | ...data 65 | .filter((d) => d.label === "SERVICE") 66 | .flatMap((d) => d.data) 67 | .map((d) => { 68 | return { 69 | identifier: d.type, 70 | metadata: {}, 71 | }; 72 | }), 73 | ], 74 | }; 75 | }; 76 | 77 | export default fetch; 78 | -------------------------------------------------------------------------------- /lib/data.test.ts: -------------------------------------------------------------------------------- 1 | import fetch from "@/lib/data"; 2 | import { describe, expect, test } from "bun:test"; 3 | import type { DetectedTechnology } from "./parsers/types"; 4 | 5 | const DOMAIN_TO_UNEXPECTED_DATA: Record = { 6 | "changelog.com": [ 7 | { 8 | identifier: "subdomain", 9 | metadata: { 10 | value: "op3.dev", 11 | }, 12 | }, 13 | ], 14 | }; 15 | 16 | const DOMAIN_TO_EXPECTED_DATA: Record = { 17 | "lastwatchdog.com": [ 18 | { 19 | identifier: "rss", 20 | metadata: { 21 | url: "https://www.lastwatchdog.com/feed/", 22 | }, 23 | }, 24 | ], 25 | "formkeep.com": [ 26 | { 27 | identifier: "github", 28 | metadata: { 29 | username: "formkeep.js", 30 | }, 31 | }, 32 | { 33 | identifier: "linkedin", 34 | metadata: { username: "formkeep" }, 35 | }, 36 | ], 37 | "savvycal.com": [ 38 | { 39 | identifier: "twitter", 40 | metadata: { username: "savvycal" }, 41 | }, 42 | { 43 | identifier: "rss", 44 | metadata: { url: "https://savvycal.com/feed.xml" }, 45 | }, 46 | { 47 | identifier: "rewardful", 48 | metadata: { value: "rewardful", via: "URL" }, 49 | }, 50 | ], 51 | "buttondown.email": [ 52 | { 53 | identifier: "github", 54 | metadata: { username: "buttondown" }, 55 | }, 56 | ], 57 | "zed.dev": [ 58 | { 59 | identifier: "twitter", 60 | metadata: { username: "zeddotdev" }, 61 | }, 62 | ], 63 | "bytereview.co.uk": [ 64 | { 65 | identifier: "tiktok", 66 | metadata: { username: "@bytereview" }, 67 | }, 68 | { 69 | identifier: "twitter", 70 | metadata: { username: "bytereview" }, 71 | }, 72 | ], 73 | }; 74 | 75 | describe("fetching", () => { 76 | Object.entries(DOMAIN_TO_EXPECTED_DATA).forEach(([domain, expectedData]) => { 77 | expectedData.forEach((data) => { 78 | test(`fetches ${data.identifier} for ${domain}`, async () => { 79 | const { detected_technologies } = await fetch(domain); 80 | expect(detected_technologies).toContainEqual(data); 81 | }); 82 | }); 83 | }); 84 | 85 | Object.entries(DOMAIN_TO_UNEXPECTED_DATA).forEach( 86 | ([domain, unexpectedData]) => { 87 | unexpectedData.forEach((data) => { 88 | test(`does not fetch ${data.identifier} for ${domain}`, async () => { 89 | const { detected_technologies } = await fetch(domain); 90 | expect(detected_technologies).not.toContainEqual(data); 91 | }); 92 | }); 93 | }, 94 | ); 95 | 96 | test("deduping identical records", async () => { 97 | const { detected_technologies } = await fetch("zed.dev"); 98 | expect( 99 | detected_technologies.filter((tech) => tech.identifier === "twitter"), 100 | ).toHaveLength(1); 101 | }); 102 | }); 103 | -------------------------------------------------------------------------------- /app/genre/[genre]/page.tsx: -------------------------------------------------------------------------------- 1 | import Grid from "@/components/Grid"; 2 | import Header from "@/components/Header"; 3 | import { db } from "@/lib/db/connection"; 4 | import { Genre, GENRE_REGISTRY, REGISTRY } from "@/lib/services"; 5 | 6 | import { Metadata, ResolvingMetadata } from "next"; 7 | 8 | type Props = { 9 | params: { genre: string }; 10 | }; 11 | 12 | export async function generateMetadata( 13 | { params }: Props, 14 | parent: ResolvingMetadata 15 | ): Promise { 16 | const genre = GENRE_REGISTRY[params.genre as Genre]; 17 | 18 | return { 19 | title: `${genre.name} - shovel.report`, 20 | description: `Information about domains using ${genre.name}, including DNS records, technologies, social media, and more.`, 21 | }; 22 | } 23 | 24 | export default async function GenrePage({ params }: Props) { 25 | const services = Object.values(REGISTRY) 26 | .filter((service) => service.genre === params.genre) 27 | .sort((a, b) => a.name.localeCompare(b.name)); 28 | 29 | const data = process.env.DISABLE_DATABASE 30 | ? [] 31 | : await db 32 | .selectFrom("detected_technologies") 33 | .where( 34 | "technology", 35 | "in", 36 | services.map((service) => service.identifier) 37 | ) 38 | .select(["technology"]) 39 | .select(db.fn.count("domain").as("count")) 40 | .groupBy("technology") 41 | .execute(); 42 | 43 | const technologyToCount = data.reduce((acc, curr) => { 44 | acc[curr.technology] = Number(curr.count); 45 | return acc; 46 | }, {} as Record); 47 | 48 | return ( 49 |
    50 |
    51 | {GENRE_REGISTRY[params.genre as Genre].name} 52 |
    53 |

    54 | {GENRE_REGISTRY[params.genre as Genre].description} 55 |

    56 | {services.length > 0 ? ( 57 | 58 | {services.map((service) => ( 59 | 64 | {service.name} 65 | 66 | {technologyToCount[service.identifier] ?? 0} domains 67 | 68 | 69 | ))} 70 | 71 | ) : ( 72 |

    73 | No services found for this genre. 74 |

    75 | )} 76 |
    77 | ); 78 | } 79 | 80 | export async function generateStaticParams() { 81 | const genres = Object.keys(GENRE_REGISTRY); 82 | 83 | return genres.map((genre) => ({ 84 | genre, 85 | })); 86 | } 87 | -------------------------------------------------------------------------------- /lib/parsers/dns.ts: -------------------------------------------------------------------------------- 1 | import type { Record } from "../loaders/types"; 2 | import { REGISTRY } from "../services"; 3 | import type { DetectedTechnology, Parser } from "./types"; 4 | 5 | const NAMESERVER_RULE = (record: Record): DetectedTechnology[] => { 6 | if (record.type !== "NS") { 7 | return []; 8 | } 9 | return Object.values(REGISTRY).flatMap((service) => { 10 | if (service.ns_values === undefined) { 11 | return []; 12 | } 13 | if (record.value.includes(service.ns_values[0])) { 14 | return [ 15 | { 16 | identifier: service.identifier, 17 | metadata: { 18 | genre: "nameserver", 19 | }, 20 | }, 21 | ]; 22 | } 23 | return []; 24 | }); 25 | }; 26 | 27 | const TXT_RULE = (record: Record): DetectedTechnology[] => { 28 | if (record.type !== "TXT") { 29 | return []; 30 | } 31 | return Object.values(REGISTRY).flatMap((service) => { 32 | if (service.txt_values === undefined) { 33 | return []; 34 | } 35 | if (service.txt_values.some((value) => record.value.includes(value))) { 36 | return [ 37 | { 38 | identifier: service.identifier, 39 | metadata: { 40 | via: "TXT", 41 | }, 42 | }, 43 | ]; 44 | } 45 | return []; 46 | }); 47 | }; 48 | 49 | const MX_RULE = (record: Record): DetectedTechnology[] => { 50 | if (record.type !== "MX") { 51 | return []; 52 | } 53 | return Object.values(REGISTRY).flatMap((service) => { 54 | if ( 55 | (service.mx_values || []).some((value) => record.value.includes(value)) 56 | ) { 57 | return [ 58 | { 59 | identifier: service.identifier, 60 | metadata: { 61 | genre: "Mailserver", 62 | }, 63 | }, 64 | ]; 65 | } 66 | return []; 67 | }); 68 | }; 69 | 70 | const CNAME_RULE = (record: Record): DetectedTechnology[] => { 71 | if (record.type !== "CNAME") { 72 | return []; 73 | } 74 | return Object.values(REGISTRY).flatMap((service) => { 75 | if ( 76 | (service.cname_values || []).some((value) => record.value.includes(value)) 77 | ) { 78 | return [ 79 | { 80 | identifier: service.identifier, 81 | metadata: { 82 | via: "CNAME", 83 | }, 84 | }, 85 | ]; 86 | } 87 | return []; 88 | }); 89 | }; 90 | 91 | const extractURLsOrIPsFromSPF = (record: string): string[] => { 92 | return record 93 | .split(" ") 94 | .filter((part) => part.includes("include:") || part.includes("ip4:")) 95 | .map((part) => part.split(":")[1]) 96 | .map( 97 | (part) => 98 | Object.values(REGISTRY).find((s) => 99 | s.spf_values?.some((v) => 100 | v.includes("*") ? part.includes(v.replace("*", "")) : v === part, 101 | ), 102 | )?.identifier || part, 103 | ); 104 | }; 105 | 106 | const isIPAddress = (value: string): boolean => { 107 | // Catch both 127.0.0.1 _and_ 127.0.0.1/17. 108 | return ( 109 | value.match(/^(?:[0-9]{1,3}\.){3}[0-9]{1,3}(?:\/[0-9]{1,2})?$/) !== null 110 | ); 111 | }; 112 | 113 | const SPF_RULE = (record: Record): DetectedTechnology[] => { 114 | if (record.type !== "TXT") { 115 | return []; 116 | } 117 | if (record.value.startsWith("v=spf1")) { 118 | return extractURLsOrIPsFromSPF(record.value).flatMap((value) => { 119 | if (isIPAddress(value)) { 120 | return []; 121 | } 122 | return [ 123 | { 124 | identifier: value, 125 | metadata: { 126 | via: "SPF", 127 | }, 128 | }, 129 | ]; 130 | }); 131 | } 132 | return []; 133 | }; 134 | 135 | const RULES = [NAMESERVER_RULE, MX_RULE, SPF_RULE, CNAME_RULE, TXT_RULE]; 136 | 137 | const filterToUnique = (values: DetectedTechnology[]): DetectedTechnology[] => { 138 | const seen = new Set(); 139 | return values.filter((value) => { 140 | const key = JSON.stringify(value); 141 | if (seen.has(key)) { 142 | return false; 143 | } 144 | seen.add(key); 145 | return true; 146 | }); 147 | }; 148 | 149 | const parse: Parser = (domain, data) => { 150 | return Promise.resolve( 151 | filterToUnique( 152 | data 153 | .filter((datum) => datum.label === "DNS") 154 | .flatMap((datum) => RULES.flatMap((rule) => datum.data.flatMap(rule))), 155 | ), 156 | ); 157 | }; 158 | const exports = { parse }; 159 | export default exports; 160 | -------------------------------------------------------------------------------- /app/page.tsx: -------------------------------------------------------------------------------- 1 | import DomainSearchForm from "@/components/DomainSearchForm"; 2 | import Grid from "@/components/Grid"; 3 | import { db } from "@/lib/db/connection"; 4 | import { GENRE_REGISTRY, REGISTRY } from "@/lib/services"; 5 | 6 | export const metadata = { 7 | title: "shovel.report", 8 | description: "A tool to help you dig into the details of a website.", 9 | }; 10 | 11 | export default async function Home() { 12 | const data = process.env.DISABLE_DATABASE 13 | ? [ 14 | { 15 | count: 0, 16 | }, 17 | ] 18 | : await db 19 | .selectFrom("detected_technologies") 20 | .select(db.fn.countAll().as("count")) 21 | .execute(); 22 | 23 | return ( 24 |
    25 | Shovel is a tool to help you dig into 26 | the details of a website. Think of it as `dig` or `nslookup`, but way 27 | better, plus a splash of `BuiltWith`. 28 |
    29 |
    30 | We developed it at Buttondown to help us understand the technical details 31 | of our customers' websites, and we are excited to share it with you. 32 |
    33 |
    34 | 35 |
    36 |
    37 |

    How to use it

    38 |
    39 | You can ask basic questions like "what domain records and 40 | technologies are used by Vercel?": 41 |
    42 |
    43 | 49 | shovel.report/domain/vercel.com 50 | 51 |
    52 |
    53 | You can also retrieve this information programmatically, and get a JSON 54 | response (though at the moment, the API is neither stable nor documented): 55 |
    56 |
    57 | 63 | shovel.report/api/v1/domain/vercel.com 64 | 65 |
    66 |
    67 | Or you can ask more complex questions, like "what domains are using 68 | both Mailgun and Cloudflare?": 69 |
    70 |
    71 | 77 | shovel.report/technology/cloudflare/and/mailgun 78 | 79 |
    80 |
    81 |
    82 |
    83 |

    What we are tracking

    84 |
    85 | We are tracking {Object.keys(REGISTRY).length} technologies across the 86 | following genres: 87 |
    88 |
    89 |
      90 | {Object.entries(GENRE_REGISTRY).map(([genre, { name }]) => ( 91 |
    1. 92 | 96 | {name} 97 | 98 |
      2. 99 | ))} 100 |
    101 |
    102 |
    103 | We've logged {data[0].count} detections. 104 |
    105 |
    106 | 107 | {Object.values(REGISTRY) 108 | .sort((a, b) => a.identifier.localeCompare(b.identifier)) 109 | .map((service) => ( 110 | 115 |
    {service.name}
    116 |     117 | ))} 118 |     119 |
    120 |
    121 | Any questions? Reach out to us on{" "} 122 | 123 | Twitter 124 | {" "} 125 | or{" "} 126 | 127 | GitHub 128 | 129 | . 130 |
    131 | ); 132 | } 133 | -------------------------------------------------------------------------------- /app/affiliation/[identifier]/page.tsx: -------------------------------------------------------------------------------- 1 | import Grid from "@/components/Grid"; 2 | import Header from "@/components/Header"; 3 | import { REGISTRY } from "@/lib/affiliations/registry"; 4 | import { db } from "@/lib/db/connection"; 5 | import * as Dialog from "@radix-ui/react-dialog"; 6 | 7 | const PAGE_SIZE = 101; 8 | 9 | const SHOVEL_PRO_URL = process.env.SHOVEL_PRO_URL; 10 | 11 | import { Metadata, ResolvingMetadata } from "next"; 12 | 13 | type Props = { 14 | params: { identifier: string }; 15 | }; 16 | 17 | export async function generateMetadata( 18 | { params }: Props, 19 | parent: ResolvingMetadata 20 | ): Promise { 21 | const service = REGISTRY[params.identifier]; 22 | 23 | return { 24 | title: `${service.name} - shovel.report`, 25 | description: `Information about domains affiliated with ${service.name}.`, 26 | alternates: { 27 | canonical: `/affiliation/${params.identifier}`, 28 | }, 29 | }; 30 | } 31 | 32 | export default async function AffiliationPage({ 33 | params, 34 | }: { 35 | params: { identifier: string }; 36 | }) { 37 | const service = REGISTRY[params.identifier]; 38 | const data = process.env.DISABLE_DATABASE 39 | ? { data: [], moreCount: 0 } 40 | : await db 41 | .selectFrom("affiliations") 42 | .where("affiliations.identifier", "=", params.identifier) 43 | .selectAll() 44 | .distinctOn("domain") 45 | .execute() 46 | .then((results) => { 47 | if (results.length > PAGE_SIZE) { 48 | const moreCount = results.length - PAGE_SIZE; 49 | return { 50 | data: results.slice(0, PAGE_SIZE), 51 | moreCount, 52 | }; 53 | } 54 | return { 55 | data: results, 56 | moreCount: 0, 57 | }; 58 | }); 59 | 60 | return ( 61 |
    62 | {service ? ( 63 | <> 64 |
    65 | {service.name} 66 |
    67 | 76 | 77 | ) : ( 78 |
    Unknown technology
    79 | )} 80 | 81 | 82 | {data.data.map((item) => ( 83 | 88 |
    {item.domain}
    89 |     90 | ))} 91 | {data.moreCount > 0 && ( 92 | 93 |
    94 | {/* Note: This component should be moved to a client-side component */} 95 | 96 | 97 | 100 | 101 | 102 | 103 | 104 | Upgrade to Pro 105 | 106 | Get the full list for just $299. 107 | 108 |
    109 | 110 | 111 | Upgrade 112 | 113 | 114 |
    115 |     116 |     117 |     118 |
    119 |     120 | )} 121 |     122 |
    123 | ); 124 | } 125 | -------------------------------------------------------------------------------- /app/domain/[domain]/page.tsx: -------------------------------------------------------------------------------- 1 | import DomainIcon from "@/components/DomainIcon"; 2 | import Grid from "@/components/Grid"; 3 | import SectionHeader from "@/components/SectionHeader"; 4 | import { REGISTRY as AFFILIATIONS_REGISTRY } from "@/lib/affiliations/registry"; 5 | import fetch from "@/lib/data"; 6 | import { reify } from "@/lib/db/domains"; 7 | import { GENRE_REGISTRY, REGISTRY } from "@/lib/services"; 8 | import type { Metadata, ResolvingMetadata } from "next"; 9 | 10 | type Props = { 11 | params: { domain: string }; 12 | }; 13 | 14 | const SOCIAL_MEDIA_URL_TEMPLATES: { [key: string]: string } = { 15 | twitter: "https://twitter.com/", 16 | linkedin: "https://linkedin.com/in/", 17 | facebook: "https://facebook.com/", 18 | instagram: "https://instagram.com/", 19 | youtube: "https://youtube.com/", 20 | tiktok: "https://tiktok.com/@", 21 | bluesky: "https://bsky.social/", 22 | github: "https://github.com/", 23 | }; 24 | 25 | const generateURLForSocialMedia = ( 26 | service: string, 27 | username: string, 28 | ): string => { 29 | const template = SOCIAL_MEDIA_URL_TEMPLATES[service]; 30 | return template ? `${template}${username}` : ""; 31 | }; 32 | 33 | export async function generateMetadata( 34 | { params }: Props, 35 | parent: ResolvingMetadata, 36 | ): Promise { 37 | return { 38 | title: `${params.domain} - shovel.report`, 39 | description: `Information about ${params.domain} and its DNS records, technologies, social media and more.`, 40 | alternates: { 41 | canonical: `/domain/${params.domain}`, 42 | }, 43 | }; 44 | } 45 | 46 | function formatJson(json: string) { 47 | try { 48 | return { 49 | valid: true, 50 | value: JSON.stringify(JSON.parse(json || "{}"), null, 2), 51 | }; 52 | } catch { 53 | return { valid: false, value: json }; 54 | } 55 | } 56 | 57 | export default async function Page({ 58 | params, 59 | }: { 60 | params: { 61 | domain: string; 62 | }; 63 | }) { 64 | const data = await fetch(params.domain); 65 | if (!process.env.DISABLE_DATABASE) { 66 | await reify(params.domain, data); 67 | } 68 | 69 | const jsonld = data.detected_technologies.find( 70 | (datum) => datum.identifier === "jsonld", 71 | )?.metadata.value; 72 | const formattedJsonLd = formatJson(jsonld ?? "{}"); 73 | 74 | return ( 75 |
    76 |

    77 | 83 | 84 | {params.domain} 85 | 86 |

    87 | DNS Records 88 | 89 | 90 | {data.data 91 | .filter((datum) => datum.label === "DNS") 92 | .flatMap((datum) => 93 | datum.data.map((record) => ( 94 | 95 | 96 | 97 | 98 | )), 99 | )} 100 | 101 |
    {record.type}{record.value}
    102 | 103 | {data.detected_technologies 104 | .filter((datum) => datum.identifier in AFFILIATIONS_REGISTRY) 105 | .map((affiliation, i) => ( 106 | 111 | {AFFILIATIONS_REGISTRY[affiliation.identifier].name} 112 | 113 | ))} 114 | 115 | 116 | {data.detected_technologies 117 | .filter((datum) => datum.identifier === "subdomain") 118 | .map((note, i) => ( 119 | 123 | {note.metadata.value} 124 | 125 | ))} 126 | 127 | 128 | {data.detected_technologies 129 | .filter((datum) => datum.identifier !== "subdomain") 130 | .filter((note) => REGISTRY[note.identifier]) 131 | .map((note, i) => ( 132 | 137 |
    {REGISTRY[note.identifier]?.name}
    138 |
    139 | {GENRE_REGISTRY[REGISTRY[note.identifier]?.genre].name} 140 |
    141 |     142 | ))} 143 |     144 | 145 | {data.detected_technologies 146 | .filter((note) => REGISTRY[note.identifier]?.genre === "social_media") 147 | .map((note, i) => ( 148 | 156 |
    {note.metadata.username}
    157 |
    158 | {REGISTRY[note.identifier]?.name} 159 |
    160 |     161 | ))} 162 |     163 | {jsonld && ( 164 | <> 165 | JSON+LD 166 | 
    167 | 						{formattedJsonLd.value}
168 |
    169 | {!formattedJsonLd.valid &&

    (this JSON isn't valid)

    } 170 | 171 | )} 172 |
    173 | ); 174 | } 175 | -------------------------------------------------------------------------------- /app/technology/[identifier]/and/[subidentifier]/page.tsx: -------------------------------------------------------------------------------- 1 | import Grid from "@/components/Grid"; 2 | import Header from "@/components/Header"; 3 | import SectionHeader from "@/components/SectionHeader"; 4 | import { db } from "@/lib/db/connection"; 5 | import { REGISTRY } from "@/lib/services"; 6 | import * as Dialog from '@radix-ui/react-dialog'; 7 | 8 | 9 | import { Metadata, ResolvingMetadata } from "next"; 10 | 11 | type Props = { 12 | params: { identifier: string; subidentifier: string }; 13 | }; 14 | 15 | const PAGE_SIZE = 17; 16 | 17 | const SHOVEL_PRO_URL = process.env.SHOVEL_PRO_URL; 18 | 19 | export async function generateMetadata( 20 | { params }: Props, 21 | parent: ResolvingMetadata 22 | ): Promise { 23 | const service1 = REGISTRY[params.identifier]; 24 | const service2 = REGISTRY[params.subidentifier]; 25 | 26 | return { 27 | title: `${service1.name} and ${service2.name} - shovel.report`, 28 | description: `Information about domains using both ${service1.name} and ${service2.name}, including DNS records, technologies, social media, and more.`, 29 | alternates: { 30 | canonical: `/technology/${params.identifier}/and/${params.subidentifier}`, 31 | }, 32 | }; 33 | } 34 | 35 | export default async function TechnologyAndPage({ 36 | params, 37 | }: { 38 | params: { identifier: string; subidentifier: string }; 39 | }) { 40 | const service1 = REGISTRY[params.identifier]; 41 | const service2 = REGISTRY[params.subidentifier]; 42 | const data = await db 43 | .selectFrom("detected_technologies as dt1") 44 | .innerJoin("detected_technologies as dt2", "dt1.domain", "dt2.domain") 45 | .where("dt1.technology", "=", params.identifier) 46 | .where("dt2.technology", "=", params.subidentifier) 47 | .select(["dt1.domain", "dt1.creation_date"]) 48 | .distinct() 49 | .execute() 50 | .then((data) => { 51 | const total = data.length; 52 | const hasMore = total > PAGE_SIZE; 53 | return { data: data.slice(0, PAGE_SIZE), hasMore, total }; 54 | }); 55 | 56 | const technologyCounts = process.env.DISABLE_DATABASE 57 | ? [] 58 | : await db 59 | .selectFrom("detected_technologies") 60 | .where( 61 | "domain", 62 | "in", 63 | db 64 | .selectFrom("detected_technologies as dt1") 65 | .innerJoin("detected_technologies as dt2", "dt1.domain", "dt2.domain") 66 | .where("dt1.technology", "=", params.identifier) 67 | .where("dt2.technology", "=", params.subidentifier) 68 | .select("dt1.domain") 69 | .distinct() 70 | ) 71 | .where("technology", "not in", [params.identifier, params.subidentifier]) 72 | .select(["technology"]) 73 | .select(db.fn.count("domain").as("count")) 74 | .groupBy("technology") 75 | .orderBy("count", "desc") 76 | .execute(); 77 | 78 | return ( 79 |
    80 |
    83 | Domains using both {service1.name} and {service2.name} 84 |
    85 | 86 | 87 | {data.total} domains detected using both{" "} 88 | 92 | {service1.name} 93 | {" "} 94 | and{" "} 95 | 99 | {service2.name} 100 | 101 | : 102 | 103 | 104 | {data.data.map((item) => ( 105 | 110 |
    {item.domain}
    111 |
    112 | {item.creation_date.toLocaleDateString()} 113 |
    114 |     115 | ))} 116 | {data.hasMore && ( 117 | 118 |
    119 | {/* Note: This component should be moved to a client-side component */} 120 | 121 | 122 | 125 | 126 | 127 | 128 | 129 | Upgrade to Pro 130 | 131 | Get the full list for just $299. 132 | 133 |
    134 | 135 | 136 | Upgrade 137 | 138 | 139 |
    140 |     141 |     142 |     143 |
    144 |     145 | )} 146 | {data.data.length === 0 && ( 147 | 148 |
    No examples found
    149 |     150 | )} 151 |     152 | 153 | 154 | Other technologies found on the same domains: 155 | 156 | 157 | {technologyCounts 158 | .filter((item) => item.technology in REGISTRY) 159 | .map((item) => ( 160 | 169 | {item.technology in REGISTRY 170 | ? REGISTRY[item.technology]?.name 171 | : item.technology} 172 |
    {item.count}
    173 |     174 | ))} 175 |     176 |
    177 | ); 178 | } 179 | -------------------------------------------------------------------------------- /lib/parsers/html.ts: -------------------------------------------------------------------------------- 1 | import { parse as parseHTML } from "node-html-parser"; 2 | import { REGISTRY } from "../services"; 3 | import type { DetectedTechnology, Parser } from "./types"; 4 | 5 | const GENERIC_SOCIAL_MEDIA_PROVIDER = (html: string) => { 6 | const socialMediaProviders = Object.values(REGISTRY).filter( 7 | (service) => service.genre === "social_media", 8 | ); 9 | const potentialMatches = socialMediaProviders.filter((provider) => 10 | provider.urlSubstrings?.some((substring) => html.includes(substring)), 11 | ); 12 | return potentialMatches 13 | .flatMap((service) => 14 | service.urlSubstrings?.map((s) => { 15 | return { 16 | identifier: service.identifier, 17 | substring: s, 18 | }; 19 | }), 20 | ) 21 | .flatMap((potentialMatch) => { 22 | const match = html.match( 23 | new RegExp( 24 | `href=["']https?://(www\.)?${potentialMatch?.substring}/([^/"^%]+?)/?["']`, 25 | ), 26 | ); 27 | if (match) { 28 | const username = match[match.length - 1]; 29 | return [ 30 | { 31 | identifier: potentialMatch?.identifier as string, 32 | metadata: { 33 | username: username.split("?")[0], 34 | }, 35 | }, 36 | ]; 37 | } 38 | return []; 39 | }); 40 | }; 41 | 42 | const TWITTER_RULE = (html: string) => { 43 | // Match on ` and pull out the username. 44 | // Make sure to avoid matching on twitter.com/intent. 45 | const match = html.match(/href="https:\/\/twitter.com\/([^\/"]+)"/); 46 | if (match) { 47 | const username = match[1]; 48 | // Also remove query parameters from the username. 49 | const usernameWithoutQuery = username.split("?")[0]; 50 | return [ 51 | { 52 | identifier: "twitter", 53 | metadata: { username: usernameWithoutQuery }, 54 | }, 55 | ]; 56 | } 57 | 58 | // Also check for `rel="me"` links that have twitter in them, like: 59 | // 60 | const match2 = html.match( 61 | / { 78 | // Match on ` and pull out the username. 79 | const match = html.match(/ { 93 | const tag = parseHTML(html).querySelector( 94 | "script[type='application/ld+json']", 95 | ); 96 | if (!tag) { 97 | return []; 98 | } 99 | const text = tag.text; 100 | const baseRule = [ 101 | { 102 | identifier: "jsonld", 103 | metadata: { value: text }, 104 | }, 105 | ]; 106 | 107 | try { 108 | const parsedJson = JSON.parse(text); 109 | const graph = Array.isArray(parsedJson) ? parsedJson : parsedJson["@graph"]; 110 | 111 | if (Array.isArray(graph)) { 112 | const additionalRules = graph 113 | .filter((item) => item && Array.isArray(item.sameAs)) 114 | .flatMap((item) => 115 | item.sameAs 116 | .map((url: string) => { 117 | const service = Object.values(REGISTRY).find((s) => 118 | url.includes(s.urlSubstrings?.[0] || ""), 119 | ); 120 | if (service) { 121 | return { 122 | identifier: service.identifier.split("?")[0] as string, 123 | metadata: { 124 | username: url.split("/").pop() || "", 125 | }, 126 | }; 127 | } 128 | return null; 129 | }) 130 | .filter(Boolean), 131 | ); 132 | 133 | baseRule.push(...additionalRules); 134 | } 135 | } catch (error) { 136 | console.error("Error parsing or processing JSON-LD:", error); 137 | } 138 | return baseRule; 139 | }; 140 | 141 | const RSS_RULE = (html: string): DetectedTechnology[] => { 142 | const tag = parseHTML(html).querySelector("link[type='application/rss+xml']"); 143 | if (tag) { 144 | const href = tag.getAttribute("href") || ""; 145 | return [ 146 | { 147 | identifier: "rss", 148 | metadata: { url: href }, 149 | }, 150 | ]; 151 | } 152 | 153 | const tag2 = parseHTML(html).querySelector("a[href*='feed.xml']"); 154 | if (tag2) { 155 | const href = tag2.getAttribute("href") || ""; 156 | return [ 157 | { 158 | identifier: "rss", 159 | metadata: { url: href }, 160 | }, 161 | ]; 162 | } 163 | 164 | return []; 165 | }; 166 | 167 | const isValidSubdomain = (potentialValue: string, domain: string) => { 168 | if (!potentialValue.startsWith("http")) { 169 | return false; 170 | } 171 | try { 172 | const url = new URL(potentialValue); 173 | return url.hostname.includes(domain) && url.hostname !== `www.${domain}`; 174 | } catch (error) { 175 | return false; 176 | } 177 | }; 178 | 179 | const SUBDOMAIN_RULE = (html: string, domain: string) => { 180 | const subdomains = parseHTML(html) 181 | .querySelectorAll("a") 182 | .map((a) => ({ 183 | value: a.getAttribute("href"), 184 | })) 185 | .filter((v) => isValidSubdomain(v.value || "", domain)) 186 | .map((v) => ({ 187 | value: new URL(v.value || "").hostname, 188 | })) 189 | .filter((v, i, a) => a.findIndex((t) => t.value === v.value) === i) 190 | .filter((v) => v.value !== domain); 191 | return subdomains.map((subdomain) => ({ 192 | // Subdomains aren't a technology, but it's kind of a weird case. We do need 193 | // a better abstraction here, though. 194 | identifier: "subdomain", 195 | metadata: { 196 | value: subdomain.value, 197 | }, 198 | })); 199 | }; 200 | 201 | const RULES: ((html: string, domain: string) => DetectedTechnology[])[] = [ 202 | ...Object.values(REGISTRY).map((service) => { 203 | return (html: string) => { 204 | const potentialMatches = service.substrings?.filter((substring) => 205 | html.includes(substring), 206 | ); 207 | return ( 208 | potentialMatches?.map(() => { 209 | return { 210 | identifier: service.identifier, 211 | metadata: { 212 | value: service.identifier, 213 | via: "URL", 214 | }, 215 | }; 216 | }) || [] 217 | ); 218 | }; 219 | }), 220 | TWITTER_RULE, 221 | GENERIC_SOCIAL_MEDIA_PROVIDER, 222 | EMAIL_ADDRESS_RULE, 223 | RSS_RULE, 224 | JSONLD_RULE, 225 | SUBDOMAIN_RULE, 226 | ]; 227 | 228 | const parse: Parser = (domain, data) => { 229 | const html = data.find((datum) => datum.label === "HTML")?.data[0].value; 230 | if (!domain || !html) { 231 | return Promise.resolve([]); 232 | } 233 | return Promise.resolve(RULES.flatMap((rule) => rule(html, domain))); 234 | }; 235 | const exports = { parse }; 236 | export default exports; 237 | -------------------------------------------------------------------------------- /app/technology/[identifier]/page.tsx: -------------------------------------------------------------------------------- 1 | import Grid from "@/components/Grid"; 2 | import Header from "@/components/Header"; 3 | import { db } from "@/lib/db/connection"; 4 | import fetchDomainsByTechnology from "@/lib/db/domains-by-technology"; 5 | import { GENRE_REGISTRY, REGISTRY } from "@/lib/services"; 6 | import * as Dialog from "@radix-ui/react-dialog"; 7 | 8 | const SHOVEL_PRO_URL = process.env.SHOVEL_PRO_URL; 9 | 10 | const PAGE_SIZE = 100; 11 | 12 | import { Metadata, ResolvingMetadata } from "next"; 13 | 14 | type Props = { 15 | params: { identifier: string }; 16 | }; 17 | 18 | export async function generateMetadata( 19 | { params }: Props, 20 | parent: ResolvingMetadata 21 | ): Promise { 22 | const service = REGISTRY[params.identifier]; 23 | 24 | return { 25 | title: `${service.name} - shovel.report`, 26 | description: `Information about ${service.name}, including domains using this technology, DNS records, social media, and more.`, 27 | alternates: { 28 | canonical: `/technology/${params.identifier}`, 29 | }, 30 | }; 31 | } 32 | 33 | export default async function TechnologyPage({ 34 | params, 35 | }: { 36 | params: { identifier: string }; 37 | }) { 38 | const service = REGISTRY[params.identifier]; 39 | const data = await fetchDomainsByTechnology(params.identifier, PAGE_SIZE); 40 | 41 | const technologyCounts = process.env.DISABLE_DATABASE 42 | ? [] 43 | : await db 44 | .selectFrom("detected_technologies") 45 | .where( 46 | "domain", 47 | "in", 48 | db 49 | .selectFrom("detected_technologies") 50 | .where("technology", "=", params.identifier) 51 | .select("domain") 52 | ) 53 | // Exclude the current technology 54 | .where("technology", "!=", params.identifier) 55 | .select(["technology"]) 56 | .select(db.fn.count("technology").as("count")) 57 | .groupBy("technology") 58 | .orderBy("count", "desc") 59 | .execute(); 60 | 61 | const trancoCount = process.env.DISABLE_DATABASE 62 | ? 0 63 | : await db 64 | .selectFrom("affiliations") 65 | .innerJoin( 66 | "detected_technologies", 67 | "affiliations.domain", 68 | "detected_technologies.domain" 69 | ) 70 | .where("detected_technologies.technology", "=", params.identifier) 71 | .where("affiliations.identifier", "=", "tranco") 72 | .select(db.fn.count("affiliations.domain").as("count")) 73 | .executeTakeFirst() 74 | .then((result) => Number(result?.count || 0)); 75 | 76 | return ( 77 |
    78 | {service ? ( 79 | <> 80 |
    81 | {service.name} 82 |
    83 | {service.description && ( 84 |
    {service.description}
    85 | )} 86 |
    87 | 92 | {service.url} 93 | 94 |
    95 | 99 | {GENRE_REGISTRY[service.genre].name} 100 | /  101 | {trancoCount} notable domains ( 102 | {trancoCount > 0 103 | ? ( 104 | (trancoCount * 100) / 105 | (data.count) 106 | ).toFixed(2) 107 | : "0.00"} 108 | %) / {data.count} total domains 109 |
    110 |
    111 | 112 | ) : ( 113 |
    Unknown technology
    114 | )} 115 | 116 | 117 | {data.data.map((item) => ( 118 | 123 |
    {item.domain}
    124 |     125 | ))} 126 | {data.count > PAGE_SIZE && ( 127 | 128 |
    129 | {/* Note: This component should be moved to a client-side component */} 130 | 131 | 132 | 135 | 136 | 137 | 138 | 139 | Upgrade to Pro 140 | 141 | Get the full list for just $299. 142 | 143 |
    144 | 145 | 146 | Upgrade 147 | 148 | 149 |
    150 |     151 |     152 |     153 |
    154 |     155 | )} 156 |     157 | 158 | 159 | {technologyCounts 160 | .filter((item) => item.technology in REGISTRY) 161 | .map((item) => ( 162 | 171 | {item.technology in REGISTRY 172 | ? REGISTRY[item.technology]?.name 173 | : item.technology} 174 |
    175 | {item.count} ( 176 | {( 177 | (Number(item.count) * 100) / 178 | (data.count) 179 | ).toFixed(2)} 180 | %) 181 |
    182 |     183 | ))} 184 |     185 |
    186 | ); 187 | } 188 | -------------------------------------------------------------------------------- /components/Icon.tsx: -------------------------------------------------------------------------------- 1 | const Twitter = ({ className }: { className: string }) => { 2 | return ( 3 | 10 | X 11 | 12 | 13 | ); 14 | }; 15 | 16 | const Instagram = ({ className }: { className: string }) => { 17 | return ( 18 | 25 | Instagram 26 | 27 | 28 | ); 29 | }; 30 | 31 | const GitHub = ({ className }: { className: string }) => { 32 | return ( 33 | 40 | GitHub 41 | 42 | 43 | ); 44 | }; 45 | 46 | const Facebook = ({ className }: { className: string }) => { 47 | return ( 48 | 55 | Facebook 56 | 57 | 58 | ); 59 | }; 60 | 61 | const LinkedIn = ({ className }: { className: string }) => { 62 | return ( 63 | 70 | LinkedIn 71 | 72 | 73 | ); 74 | }; 75 | 76 | const YouTube = ({ className }: { className: string }) => { 77 | return ( 78 | 85 | YouTube 86 | 87 | 88 | ); 89 | }; 90 | 91 | const TikTok = ({ className }: { className: string }) => { 92 | return ( 93 | 100 | TikTok 101 | 102 | 103 | ); 104 | }; 105 | 106 | const Pinterest = ({ className }: { className: string }) => { 107 | return ( 108 | 115 | Pinterest 116 | 117 | 118 | ); 119 | }; 120 | 121 | export default { 122 | Twitter, 123 | Instagram, 124 | GitHub, 125 | Facebook, 126 | LinkedIn, 127 | YouTube, 128 | TikTok, 129 | Pinterest, 130 | }; 131 | -------------------------------------------------------------------------------- /lib/services.tsx: -------------------------------------------------------------------------------- 1 | import Icon from "@/components/Icon"; 2 | 3 | export type Genre = 4 | | "accounting" 5 | | "ads" 6 | | "analytics" 7 | | "ats" 8 | | "calendar" 9 | | "cdn" 10 | | "cms" 11 | | "crm" 12 | | "design" 13 | | "devtools" 14 | | "dns" 15 | | "storage" 16 | | "documentation" 17 | | "ecommerce" 18 | | "email" 19 | | "everything" 20 | | "form" 21 | | "gdpr" 22 | | "hosting" 23 | | "marketing" 24 | | "monitoring" 25 | | "open_web" 26 | | "payments" 27 | | "podcast" 28 | | "search" 29 | | "security" 30 | | "social_media" 31 | | "static_site_generator" 32 | | "support" 33 | | "url_shortener" 34 | | "videos" 35 | | "web_framework"; 36 | 37 | export const GENRE_REGISTRY: { 38 | [key in Genre]: { 39 | name: string; 40 | description: string; 41 | }; 42 | } = { 43 | url_shortener: { 44 | name: "URL Shortener", 45 | description: "URL shortening services", 46 | }, 47 | static_site_generator: { 48 | name: "Static Site Generator", 49 | description: "Static site generators", 50 | }, 51 | devtools: { 52 | name: "Developer Tools", 53 | description: "Developer tools", 54 | }, 55 | ads: { 56 | name: "Advertising", 57 | description: "Advertising services", 58 | }, 59 | storage: { 60 | name: "Storage", 61 | description: "Storage services", 62 | }, 63 | open_web: { 64 | name: "Open Web", 65 | description: "Open web services", 66 | }, 67 | search: { 68 | name: "Search", 69 | description: "Search services", 70 | }, 71 | accounting: { 72 | name: "Accounting", 73 | description: "Accounting services", 74 | }, 75 | analytics: { 76 | name: "Analytics", 77 | description: "Tools for tracking and analyzing website or app usage", 78 | }, 79 | ats: { 80 | name: "Applicant Tracking System", 81 | description: "Software for managing recruitment processes", 82 | }, 83 | calendar: { 84 | name: "Calendar", 85 | description: "Tools for scheduling and managing events", 86 | }, 87 | cdn: { 88 | name: "Content Delivery Network", 89 | description: "Services for distributing content globally", 90 | }, 91 | cms: { 92 | name: "Content Management System", 93 | description: "Platforms for creating and managing digital content", 94 | }, 95 | crm: { 96 | name: "Customer Relationship Management", 97 | description: "Systems for managing customer interactions and data", 98 | }, 99 | design: { 100 | name: "Design", 101 | description: "Tools for graphic and web design", 102 | }, 103 | dns: { 104 | name: "Domain Name System", 105 | description: "Services for managing domain names and DNS records", 106 | }, 107 | documentation: { 108 | name: "Documentation", 109 | description: "Tools for creating and managing technical documentation", 110 | }, 111 | ecommerce: { 112 | name: "E-commerce", 113 | description: "Platforms for online selling and shopping", 114 | }, 115 | email: { 116 | name: "Email", 117 | description: "Services for email hosting and management", 118 | }, 119 | everything: { 120 | name: "Everything", 121 | description: "Comprehensive platforms covering multiple categories", 122 | }, 123 | form: { 124 | name: "Form", 125 | description: "Tools for creating and managing online forms", 126 | }, 127 | gdpr: { 128 | name: "GDPR Compliance", 129 | description: 130 | "Services for ensuring compliance with data protection regulations", 131 | }, 132 | hosting: { 133 | name: "Hosting", 134 | description: "Services for hosting websites and applications", 135 | }, 136 | marketing: { 137 | name: "Marketing", 138 | description: "Tools for digital marketing and promotion", 139 | }, 140 | monitoring: { 141 | name: "Monitoring", 142 | description: "Services for tracking system performance and uptime", 143 | }, 144 | payments: { 145 | name: "Payments", 146 | description: "Platforms for processing online payments", 147 | }, 148 | podcast: { 149 | name: "Podcast", 150 | description: "Tools for creating and distributing podcasts", 151 | }, 152 | security: { 153 | name: "Security", 154 | description: "Services for enhancing online security and protection", 155 | }, 156 | social_media: { 157 | name: "Social Media", 158 | description: "Platforms for social networking and content sharing", 159 | }, 160 | support: { 161 | name: "Support", 162 | description: "Tools for customer support and helpdesk management", 163 | }, 164 | videos: { 165 | name: "Videos", 166 | description: "Platforms for hosting and streaming video content", 167 | }, 168 | web_framework: { 169 | name: "Web Framework", 170 | description: "Frameworks for building web applications", 171 | }, 172 | }; 173 | 174 | type Service = { 175 | identifier: string; 176 | name: string; 177 | description?: string; 178 | genre: Genre; 179 | url: string; 180 | icon?: React.ReactNode; 181 | cname_values?: string[]; 182 | mx_values?: string[]; 183 | ns_values?: string[]; 184 | txt_values?: string[]; 185 | spf_values?: string[]; 186 | urlSubstrings?: string[]; 187 | substrings?: string[]; 188 | defunct?: boolean; 189 | headers?: { 190 | key: string; 191 | value: string; 192 | }; 193 | dns_prefix?: string; 194 | }; 195 | 196 | export const REGISTRY: { [key in string]: Service } = { 197 | webex: { 198 | identifier: "webex", 199 | name: "Webex", 200 | genre: "hosting", 201 | url: "https://www.webex.com", 202 | txt_values: ["webexdomainverification"], 203 | }, 204 | liveramp: { 205 | identifier: "liveramp", 206 | name: "Liveramp", 207 | genre: "marketing", 208 | url: "https://liveramp.com", 209 | txt_values: ["liveramp-site-verification"], 210 | }, 211 | paloalto: { 212 | identifier: "paloalto", 213 | name: "Palo Alto", 214 | genre: "security", 215 | url: "https://www.paloaltonetworks.com", 216 | txt_values: ["paloaltonetworks-site-verification"], 217 | }, 218 | headway: { 219 | identifier: "headway", 220 | name: "Headway", 221 | genre: "support", 222 | url: "https://headwayapp.co", 223 | cname_values: ["headwayapp.co"], 224 | }, 225 | skysnag: { 226 | identifier: "skysnag", 227 | name: "Skysnag", 228 | genre: "security", 229 | url: "https://skysnag.com", 230 | spf_values: ["_spf.skysnag.com"], 231 | }, 232 | constantcontact: { 233 | identifier: "constantcontact", 234 | name: "Constant Contact", 235 | genre: "marketing", 236 | url: "https://www.constantcontact.com", 237 | substrings: ["constantcontact.com"], 238 | spf_values: ["spf.constantcontact.com"], 239 | }, 240 | rss: { 241 | identifier: "rss", 242 | name: "RSS", 243 | genre: "open_web", 244 | url: "https://rss.com", 245 | }, 246 | readme: { 247 | identifier: "readme", 248 | name: "ReadMe", 249 | genre: "documentation", 250 | url: "https://readme.com", 251 | cname_values: ["ssl.readmessl.com"], 252 | }, 253 | formassembly: { 254 | identifier: "formassembly", 255 | name: "FormAssembly", 256 | genre: "crm", 257 | url: "https://formassembly.com", 258 | spf_values: ["spf1.formassembly.com"], 259 | }, 260 | fireside: { 261 | identifier: "fireside", 262 | name: "Fireside", 263 | genre: "podcast", 264 | url: "https://fireside.fm", 265 | cname_values: ["hosted.fireside.fm"], 266 | }, 267 | geojs: { 268 | identifier: "geojs", 269 | name: "GeoJS", 270 | genre: "analytics", 271 | url: "https://geojs.io", 272 | substrings: ["geojs.io/v1"], 273 | }, 274 | digitalocean: { 275 | identifier: "digitalocean", 276 | name: "DigitalOcean", 277 | genre: "hosting", 278 | url: "https://www.digitalocean.com", 279 | ns_values: ["ns1.digitalocean.com", "ns2.digitalocean.com"], 280 | }, 281 | github_pages: { 282 | identifier: "github_pages", 283 | name: "GitHub Pages", 284 | genre: "hosting", 285 | url: "https://pages.github.com", 286 | cname_values: ["pages.github.com"], 287 | }, 288 | gitbook: { 289 | identifier: "gitbook", 290 | name: "GitBook", 291 | genre: "hosting", 292 | url: "https://www.gitbook.com", 293 | cname_values: ["gitbook.io"], 294 | }, 295 | substack: { 296 | identifier: "substack", 297 | name: "Substack", 298 | url: "https://www.substack.com", 299 | genre: "email", 300 | cname_values: ["substack-custom-domains.com"], 301 | substrings: [".substack.com"], 302 | }, 303 | datadog: { 304 | identifier: "datadog", 305 | name: "Datadog", 306 | url: "https://www.datadoghq.com", 307 | genre: "monitoring", 308 | substrings: ["function(h,o,u,n,d)"], 309 | }, 310 | umami: { 311 | identifier: "umami", 312 | name: "Umami", 313 | url: "https://umami.is", 314 | genre: "analytics", 315 | cname_values: ["umami.sh"], 316 | substrings: ["data-website-id"], 317 | }, 318 | nolt: { 319 | identifier: "nolt", 320 | name: "Nolt", 321 | url: "https://nolt.io", 322 | genre: "support", 323 | cname_values: ["custom-domain.nolt.io"], 324 | }, 325 | improv_mx: { 326 | identifier: "improv_mx", 327 | name: "ImprovMX", 328 | url: "https://www.improvmx.com", 329 | genre: "email", 330 | mx_values: ["mx1.improvmx.com"], 331 | }, 332 | proofpoint: { 333 | identifier: "proofpoint", 334 | name: "Proofpoint", 335 | url: "https://www.proofpoint.com", 336 | genre: "security", 337 | mx_values: ["pphosted.com"], 338 | }, 339 | namecheap: { 340 | identifier: "namecheap", 341 | name: "Namecheap", 342 | url: "https://www.namecheap.com", 343 | genre: "dns", 344 | ns_values: [ 345 | "ns1.namecheap.com", 346 | "ns2.namecheap.com", 347 | "ns3.namecheap.com", 348 | "ns4.namecheap.com", 349 | ], 350 | spf_values: [ 351 | "spf.efwd.registrar-servers.com", 352 | "spf.web-hosting.com", 353 | "privateemail.com", 354 | ], 355 | }, 356 | fastmail: { 357 | identifier: "fastmail", 358 | name: "FastMail", 359 | url: "https://www.fastmail.com", 360 | genre: "email", 361 | mx_values: ["smtp.messagingengine.com"], 362 | spf_values: ["spf.messagingengine.com"], 363 | }, 364 | sailthru: { 365 | identifier: "sailthru", 366 | name: "Sailthru", 367 | url: "https://www.sailthru.com", 368 | genre: "marketing", 369 | spf_values: ["aspmx.sailthru.com"], 370 | }, 371 | simple_analytics: { 372 | identifier: "simple_analytics", 373 | name: "Simple Analytics", 374 | url: "https://simpleanalytics.com", 375 | genre: "analytics", 376 | substrings: ["simpleanalyticscdn.com"], 377 | }, 378 | dnsimple: { 379 | identifier: "dnsimple", 380 | name: "DNSimple", 381 | url: "https://www.dnsimple.com", 382 | genre: "dns", 383 | ns_values: ["dnsimple.com"], 384 | }, 385 | vercel: { 386 | identifier: "vercel", 387 | name: "Vercel", 388 | url: "https://www.vercel.com", 389 | genre: "hosting", 390 | ns_values: ["vercel-dns.com"], 391 | }, 392 | wix: { 393 | identifier: "wix", 394 | name: "Wix", 395 | url: "https://www.wix.com", 396 | genre: "hosting", 397 | ns_values: ["wixdns.net"], 398 | substrings: ["Wix.com Website Builder"], 399 | cname_values: ["wixdns.net"], 400 | }, 401 | zoho_invoices: { 402 | identifier: "zoho_invoices", 403 | name: "Zoho Invoices", 404 | url: "https://www.zoho.com/invoices", 405 | genre: "accounting", 406 | spf_values: ["sender.zohoinvoice.com"], 407 | }, 408 | drip: { 409 | identifier: "drip", 410 | name: "Drip", 411 | url: "https://www.drip.com", 412 | genre: "email", 413 | substrings: ["data-drip"], 414 | }, 415 | zoho_campaigns: { 416 | identifier: "zoho_campaigns", 417 | name: "Zoho Campaigns", 418 | url: "https://www.zoho.com/campaigns", 419 | genre: "marketing", 420 | substrings: ["zcc.zoho.com"], 421 | }, 422 | zoho_mail: { 423 | identifier: "zoho_mail", 424 | name: "Zoho Mail", 425 | url: "https://www.zoho.com/mail", 426 | genre: "email", 427 | mx_values: ["mx.zoho.com"], 428 | spf_values: ["zoho.com", "zeptomail.net"], 429 | }, 430 | elasticemail: { 431 | identifier: "elasticemail", 432 | name: "ElasticEmail", 433 | url: "https://www.elasticemail.com", 434 | genre: "email", 435 | spf_values: ["_spf.elasticemail.com"], 436 | }, 437 | aws_s3: { 438 | identifier: "aws_s3", 439 | name: "AWS S3", 440 | url: "https://aws.amazon.com/s3", 441 | genre: "hosting", 442 | substrings: ["s3.amazonaws.com", "NoSuchBucket"], 443 | }, 444 | aws_elb: { 445 | identifier: "aws_elb", 446 | name: "AWS ELB", 447 | url: "https://aws.amazon.com/elasticloadbalancing", 448 | genre: "hosting", 449 | cname_values: ["elb.amazonaws.com"], 450 | }, 451 | cargo: { 452 | identifier: "cargo", 453 | name: "Cargo", 454 | url: "https://www.cargo.site", 455 | genre: "hosting", 456 | substrings: ["__cargo_context__"], 457 | }, 458 | aws_ses: { 459 | identifier: "aws_ses", 460 | name: "AWS SES", 461 | url: "https://aws.amazon.com/ses", 462 | genre: "email", 463 | spf_values: ["amazonses.com"], 464 | }, 465 | carbon: { 466 | identifier: "carbon", 467 | name: "Carbon", 468 | url: "https://carbonads.net", 469 | genre: "ads", 470 | substrings: ["https://cdn.carbonads.com/carbon.js"], 471 | }, 472 | rocket: { 473 | identifier: "rocket", 474 | name: "Rocket", 475 | url: "https://rocket.net", 476 | genre: "hosting", 477 | cname_values: ["onrocket.site"], 478 | }, 479 | woocommerce: { 480 | identifier: "woocommerce", 481 | name: "WooCommerce", 482 | url: "https://woocommerce.com", 483 | genre: "ecommerce", 484 | substrings: ["woocommerce_params"], 485 | }, 486 | aws_cloudfront: { 487 | identifier: "aws_cloudfront", 488 | name: "AWS CloudFront", 489 | url: "https://aws.amazon.com/cloudfront/", 490 | genre: "cdn", 491 | }, 492 | madmimi: { 493 | identifier: "madmimi", 494 | name: "MadMimi", 495 | url: "https://madmimi.com", 496 | genre: "email", 497 | substrings: ["madmimiwidget"], 498 | }, 499 | google_analytics: { 500 | identifier: "google_analytics", 501 | name: "Google Analytics", 502 | genre: "analytics", 503 | url: "https://google.com/analytics", 504 | substrings: [ 505 | "GoogleAnalyticsObject", 506 | "https://www.googletagmanager.com/gtag/js", 507 | "https://www.google-analytics.com/analytics.js", 508 | ], 509 | }, 510 | google_adsense: { 511 | identifier: "google_adsense", 512 | name: "Google AdSense", 513 | genre: "marketing", 514 | url: "https://www.google.com/adsense", 515 | substrings: ["adsbygoogle.js"], 516 | }, 517 | norton_safe_web: { 518 | identifier: "norton_safe_web", 519 | name: "Norton Safe Web", 520 | genre: "security", 521 | url: "https://safeweb.norton.com", 522 | txt_values: ["norton-safeweb-site-verification"], 523 | }, 524 | cloudflare: { 525 | identifier: "cloudflare", 526 | name: "Cloudflare", 527 | genre: "cdn", 528 | url: "https://www.cloudflare.com", 529 | ns_values: ["cloudflare.com"], 530 | spf_values: ["_spf.mx.cloudflare.net"], 531 | }, 532 | cloudflare_analytics: { 533 | identifier: "cloudflare_analytics", 534 | name: "Cloudflare Analytics", 535 | genre: "analytics", 536 | url: "https://www.cloudflare.com", 537 | substrings: ["data-cf-beacon"], 538 | }, 539 | google_tag_manager: { 540 | identifier: "google_tag_manager", 541 | name: "Google Tag Manager", 542 | genre: "analytics", 543 | url: "https://tagmanager.google.com", 544 | }, 545 | astro: { 546 | identifier: "astro", 547 | name: "Astro", 548 | genre: "web_framework", 549 | url: "https://astro.build", 550 | substrings: ["astro-slot", "/_astro"], 551 | }, 552 | chargebee: { 553 | identifier: "chargebee", 554 | name: "Chargebee", 555 | genre: "payments", 556 | url: "https://www.chargebee.com", 557 | substrings: ["js.chargebee.com"], 558 | }, 559 | mailchannels: { 560 | identifier: "mailchannels", 561 | name: "MailChannels", 562 | url: "https://www.mailchannels.com", 563 | genre: "email", 564 | spf_values: ["relay.mailchannels.net"], 565 | }, 566 | meltwater: { 567 | identifier: "meltwater", 568 | name: "Meltwater", 569 | url: "https://www.meltwater.com", 570 | genre: "marketing", 571 | spf_values: ["meltwater.com"], 572 | }, 573 | mailchimp: { 574 | identifier: "mailchimp", 575 | name: "Mailchimp", 576 | url: "https://mailchimp.com", 577 | genre: "email", 578 | substrings: ["list-manage.com", "mc:edit", "c,h,i,m,p"], 579 | spf_values: ["servers.mcsv.net"], 580 | }, 581 | clearbit: { 582 | identifier: "clearbit", 583 | name: "Clearbit", 584 | url: "https://clearbit.com", 585 | genre: "analytics", 586 | substrings: ["tag.clearbitscripts"], 587 | }, 588 | sparkloop: { 589 | identifier: "sparkloop", 590 | name: "SparkLoop", 591 | url: "https://sparkloop.app", 592 | genre: "email", 593 | substrings: ["sparkloop.app"], 594 | }, 595 | laravel: { 596 | identifier: "laravel", 597 | name: "Laravel", 598 | url: "https://laravel.com", 599 | genre: "web_framework", 600 | substrings: ["LivewireScript"], 601 | headers: { 602 | key: "set-cookie", 603 | value: "laravel_session", 604 | }, 605 | }, 606 | tapfiliate: { 607 | identifier: "tapfiliate", 608 | name: "Tapfiliate", 609 | url: "https://www.tapfiliate.com", 610 | genre: "marketing", 611 | substrings: ["script.tapfiliate.com"], 612 | }, 613 | wistia: { 614 | identifier: "wistia", 615 | name: "Wistia", 616 | url: "https://wistia.com", 617 | genre: "videos", 618 | substrings: ["wistia.com/embed"], 619 | }, 620 | rightmessage: { 621 | identifier: "rightmessage", 622 | name: "RightMessage", 623 | url: "https://www.rightmessage.com", 624 | genre: "email", 625 | substrings: ["rightmessage-id", "tb.rightmessage.com"], 626 | }, 627 | sendowl: { 628 | identifier: "sendowl", 629 | name: "SendOwl", 630 | url: "https://www.sendowl.com", 631 | genre: "ecommerce", 632 | substrings: ["https://transactions.sendowl.com/assets/sendowl.js"], 633 | }, 634 | sendgrid: { 635 | identifier: "sendgrid", 636 | name: "SendGrid", 637 | genre: "email", 638 | url: "https://sendgrid.com", 639 | spf_values: ["sendgrid.net"], 640 | }, 641 | flodesk: { 642 | identifier: "flodesk", 643 | name: "Flodesk", 644 | genre: "email", 645 | url: "https://www.flodesk.com", 646 | substrings: ["view.flodesk.com"], 647 | }, 648 | webmentions: { 649 | identifier: "webmentions", 650 | name: "Webmentions", 651 | genre: "social_media", 652 | url: "https://webmention.io", 653 | substrings: ['rel="webmention"'], 654 | }, 655 | bluehost: { 656 | identifier: "bluehost", 657 | name: "Bluehost", 658 | genre: "hosting", 659 | url: "https://www.bluehost.com", 660 | cname_values: ["bluehost.com"], 661 | }, 662 | bluesky: { 663 | identifier: "bluesky", 664 | name: "BlueSky", 665 | genre: "social_media", 666 | url: "https://bsky.app", 667 | urlSubstrings: ["bsky.app/profile"], 668 | }, 669 | threads: { 670 | identifier: "threads", 671 | name: "Threads", 672 | genre: "social_media", 673 | url: "https://www.threads.net", 674 | urlSubstrings: ["threads.net"], 675 | }, 676 | fresh: { 677 | identifier: "fresh", 678 | name: "Fresh", 679 | genre: "web_framework", 680 | url: "https://fresh.deno.dev", 681 | substrings: ["__frsh"], 682 | }, 683 | jquery: { 684 | identifier: "jquery", 685 | name: "jQuery", 686 | genre: "web_framework", 687 | url: "https://jquery.com", 688 | substrings: ["jquery.com", "jquery.min.js"], 689 | }, 690 | mailerlite: { 691 | identifier: "mailerlite", 692 | name: "MailerLite", 693 | genre: "email", 694 | url: "https://www.mailerlite.com", 695 | spf_values: ["_spf.mlsend.com"], 696 | substrings: ["static.mailerlite.com"], 697 | }, 698 | algolia: { 699 | identifier: "algolia", 700 | name: "Algolia", 701 | genre: "search", 702 | url: "https://www.algolia.com", 703 | substrings: ['algolia.net" crossorigin', "AlgoliaOpts", "docsearch"], 704 | }, 705 | bootstrap: { 706 | identifier: "bootstrap", 707 | name: "Bootstrap", 708 | genre: "web_framework", 709 | url: "https://getbootstrap.com", 710 | substrings: ["bootstrapcdn.com", "navbar-brand", "bootstrap.min.js"], 711 | }, 712 | loom: { 713 | identifier: "loom", 714 | name: "Loom", 715 | genre: "videos", 716 | url: "https://www.loom.com", 717 | txt_values: ["loom-verification", "loom-site-verification"], 718 | }, 719 | vite: { 720 | identifier: "vite", 721 | name: "Vite", 722 | genre: "web_framework", 723 | url: "https://vitejs.dev", 724 | substrings: ["/vite/"], 725 | }, 726 | tailwindcss: { 727 | identifier: "tailwindcss", 728 | name: "Tailwind CSS", 729 | genre: "web_framework", 730 | url: "https://tailwindcss.com", 731 | substrings: ["tailwindcss.com", "mx-auto", "text-white"], 732 | }, 733 | alpinejs: { 734 | identifier: "alpinejs", 735 | name: "Alpine.js", 736 | genre: "web_framework", 737 | url: "https://alpinejs.dev", 738 | substrings: ["x-on:click", "x-text", "x-transition:enter"], 739 | }, 740 | gandi: { 741 | identifier: "gandi", 742 | name: "Gandi", 743 | genre: "dns", 744 | url: "https://www.gandi.net", 745 | ns_values: ["ns.gandi.net"], 746 | spf_values: ["_mailcust.gandi.net"], 747 | }, 748 | gauges: { 749 | identifier: "gauges", 750 | name: "Gauges", 751 | genre: "analytics", 752 | url: "https://www.gaug.es", 753 | substrings: ["gaug.es"], 754 | }, 755 | mailjet: { 756 | identifier: "mailjet", 757 | name: "Mailjet", 758 | genre: "email", 759 | url: "https://www.mailjet.com", 760 | spf_values: ["spf.mailjet.com"], 761 | }, 762 | mailersend: { 763 | identifier: "mailersend", 764 | name: "Mailersend", 765 | genre: "email", 766 | url: "https://www.mailersend.com", 767 | spf_values: ["_spf.mailersend.net"], 768 | }, 769 | facebook_ads: { 770 | identifier: "facebook_ads", 771 | name: "Facebook Ads", 772 | genre: "marketing", 773 | url: "https://www.facebook.com/business/ads", 774 | txt_values: ["facebook-domain-verification"], 775 | }, 776 | uservoice: { 777 | identifier: "uservoice", 778 | name: "UserVoice", 779 | genre: "support", 780 | url: "https://www.uservoice.com", 781 | spf_values: ["smtp1.uservoice.com"], 782 | }, 783 | notion: { 784 | identifier: "notion", 785 | name: "Notion", 786 | genre: "design", 787 | url: "https://www.notion.so", 788 | txt_values: ["notion-domain-verification"], 789 | }, 790 | customer_io: { 791 | identifier: "customer_io", 792 | name: "Customer.io", 793 | genre: "email", 794 | url: "https://www.customer.io", 795 | substrings: ["customerioforms"], 796 | }, 797 | cloudinary: { 798 | identifier: "cloudinary", 799 | name: "Cloudinary", 800 | genre: "hosting", 801 | url: "https://www.cloudinary.com", 802 | substrings: ["res.cloudinary.com"], 803 | }, 804 | oembed: { 805 | identifier: "oembed", 806 | name: "OEmbed", 807 | genre: "open_web", 808 | url: "https://oembed.com", 809 | substrings: ["json+oembed"], 810 | }, 811 | knowbe4: { 812 | identifier: "knowbe4", 813 | name: "KnowBe4", 814 | genre: "security", 815 | url: "https://www.knowbe4.com", 816 | txt_values: ["knowbe4-site-verification"], 817 | }, 818 | box: { 819 | identifier: "box", 820 | name: "Box", 821 | genre: "storage", 822 | url: "https://www.box.com", 823 | txt_values: ["box-domain-verification"], 824 | }, 825 | cookieyes: { 826 | identifier: "cookieyes", 827 | name: "CookieYes", 828 | genre: "gdpr", 829 | url: "https://www.cookieyes.com", 830 | substrings: ["cookieyes.com"], 831 | }, 832 | tumblr: { 833 | identifier: "tumblr", 834 | name: "Tumblr", 835 | genre: "cms", 836 | url: "https://www.tumblr.com", 837 | cname_values: ["domains.tumblr.com"], 838 | }, 839 | statuspage: { 840 | identifier: "statuspage", 841 | name: "Statuspage", 842 | genre: "monitoring", 843 | url: "https://www.statuspage.io", 844 | txt_values: ["stspg-customer.com", "status-page-domain-verification"], 845 | cname_values: ["stspg-customer.com"], 846 | }, 847 | facebook_pixel: { 848 | identifier: "facebook_pixel", 849 | name: "Facebook Pixel", 850 | genre: "analytics", 851 | url: "https://www.facebook.com/business/learn/facebook-ads-pixel", 852 | substrings: ["f.fbq"], 853 | }, 854 | hackerone: { 855 | identifier: "hackerone", 856 | name: "HackerOne", 857 | genre: "security", 858 | url: "https://hackerone.com", 859 | txt_values: ["h1-domain-verification"], 860 | }, 861 | ahrefs: { 862 | identifier: "ahrefs", 863 | name: "Ahrefs", 864 | genre: "marketing", 865 | url: "https://ahrefs.com", 866 | txt_values: ["ahrefs-site-verification"], 867 | }, 868 | google_webmaster_tools: { 869 | identifier: "google_webmaster_tools", 870 | name: "Google Webmaster Tools", 871 | genre: "analytics", 872 | url: "https://www.google.com/webmasters/tools/home", 873 | txt_values: ["google-site-verification"], 874 | }, 875 | senja: { 876 | identifier: "senja", 877 | name: "Senja", 878 | genre: "marketing", 879 | url: "https://www.senja.io", 880 | substrings: ["static.senja"], 881 | }, 882 | crisp: { 883 | identifier: "crisp", 884 | name: "Crisp", 885 | genre: "support", 886 | url: "https://crisp.chat", 887 | substrings: ["CRISP_WEBSITE_ID"], 888 | }, 889 | adestra: { 890 | identifier: "adestra", 891 | name: "Adestra", 892 | url: "https://www.adestra.com", 893 | genre: "email", 894 | spf_values: ["msgfocus.com"], 895 | }, 896 | helpscout: { 897 | identifier: "helpscout", 898 | name: "Help Scout", 899 | genre: "support", 900 | url: "https://www.helpscout.com", 901 | spf_values: ["helpscoutemail.com"], 902 | cname_values: ["helpscoutdocs.com"], 903 | }, 904 | heroku: { 905 | identifier: "heroku", 906 | name: "Heroku", 907 | url: "https://www.heroku.com", 908 | genre: "hosting", 909 | cname_values: ["herokudns.com"], 910 | headers: { 911 | key: "text/headers/nel", 912 | value: "heroku-nel", 913 | }, 914 | }, 915 | ovh: { 916 | identifier: "ovh", 917 | name: "OVH", 918 | url: "https://www.ovh.com", 919 | genre: "hosting", 920 | mx_values: ["mx.ovh.net"], 921 | }, 922 | groove: { 923 | identifier: "groove", 924 | name: "Groove", 925 | url: "https://www.groovehq.com", 926 | genre: "support", 927 | substrings: [], 928 | }, 929 | webflow: { 930 | identifier: "webflow", 931 | name: "Webflow", 932 | url: "https://www.webflow.com", 933 | genre: "web_framework", 934 | substrings: ["/js/webflow"], 935 | cname_values: ["proxy-ssl.webflow.com"], 936 | headers: { 937 | key: "text/headers/vary", 938 | value: "x-wf-forwarded-proto", 939 | }, 940 | }, 941 | loops: { 942 | identifier: "loops", 943 | name: "Loops", 944 | url: "https://www.loops.so", 945 | genre: "email", 946 | substrings: ["app.loops.so"], 947 | }, 948 | netlify: { 949 | identifier: "netlify", 950 | name: "Netlify", 951 | url: "https://www.netlify.com", 952 | genre: "hosting", 953 | headers: { 954 | key: "text/headers/server", 955 | value: "Netlify", 956 | }, 957 | }, 958 | fathom: { 959 | identifier: "fathom", 960 | name: "Fathom", 961 | url: "https://www.usefathom.com", 962 | genre: "analytics", 963 | substrings: ["cdn.usefathom.com"], 964 | }, 965 | tally: { 966 | identifier: "tally", 967 | name: "Tally", 968 | url: "https://www.tally.so", 969 | genre: "form", 970 | substrings: ["tally.so"], 971 | }, 972 | savvycal: { 973 | identifier: "savvycal", 974 | name: "SavvyCal", 975 | url: "https://www.savvycal.com", 976 | genre: "calendar", 977 | substrings: ["savvycal.com"], 978 | }, 979 | podia: { 980 | identifier: "podia", 981 | name: "Podia", 982 | url: "https://www.podia.com", 983 | genre: "podcast", 984 | cname_values: ["podia.com"], 985 | }, 986 | transistor: { 987 | identifier: "transistor", 988 | name: "Transistor", 989 | url: "https://www.transistor.fm", 990 | genre: "podcast", 991 | substrings: [".transistor.fm"], 992 | }, 993 | fastly: { 994 | identifier: "fastly", 995 | name: "Fastly", 996 | url: "https://www.fastly.com", 997 | genre: "hosting", 998 | txt_values: ["fastly-domain-delegation"], 999 | }, 1000 | postman: { 1001 | identifier: "postman", 1002 | name: "Postman", 1003 | url: "https://www.postman.com", 1004 | genre: "devtools", 1005 | txt_values: ["postman-domain-verification"], 1006 | }, 1007 | docker: { 1008 | identifier: "docker", 1009 | name: "Docker", 1010 | url: "https://www.docker.com", 1011 | genre: "hosting", 1012 | txt_values: ["docker-verification"], 1013 | }, 1014 | canva: { 1015 | identifier: "canva", 1016 | name: "Canva", 1017 | url: "https://www.canva.com", 1018 | genre: "design", 1019 | txt_values: ["canva-domain-verification"], 1020 | }, 1021 | rewardful: { 1022 | identifier: "rewardful", 1023 | name: "Rewardful", 1024 | url: "https://www.rewardful.com", 1025 | genre: "marketing", 1026 | substrings: [".getrewardful.com", "data-rewardful", "window,'rewardful'"], 1027 | }, 1028 | php: { 1029 | identifier: "php", 1030 | name: "PHP", 1031 | url: "https://www.php.net", 1032 | genre: "web_framework", 1033 | headers: { 1034 | key: "text/headers/set-cookie", 1035 | value: "PHPSESSID", 1036 | }, 1037 | }, 1038 | globalsign: { 1039 | identifier: "globalsign", 1040 | name: "GlobalSign", 1041 | url: "https://www.globalsign.com", 1042 | genre: "security", 1043 | txt_values: ["globalsign-domain-verification"], 1044 | }, 1045 | zoom: { 1046 | identifier: "zoom", 1047 | name: "Zoom", 1048 | url: "https://www.zoom.us", 1049 | genre: "videos", 1050 | txt_values: ["zoom-domain-verification", "ZOOM_verify"], 1051 | }, 1052 | vimeo: { 1053 | identifier: "vimeo", 1054 | name: "Vimeo", 1055 | url: "https://www.vimeo.com", 1056 | genre: "videos", 1057 | substrings: ["player.vimeo"], 1058 | }, 1059 | shopify: { 1060 | identifier: "shopify", 1061 | name: "Shopify", 1062 | url: "https://www.shopify.com", 1063 | genre: "ecommerce", 1064 | cname_values: ["myshopify.com"], 1065 | substrings: ["shopifycdn.com"], 1066 | spf_values: ["shops.shopify.com"], 1067 | }, 1068 | asp_net: { 1069 | identifier: "asp_net", 1070 | name: "ASP.NET", 1071 | url: "https://www.asp.net", 1072 | genre: "web_framework", 1073 | headers: { 1074 | key: "x-powered-by", 1075 | value: "ASP.NET", 1076 | }, 1077 | }, 1078 | atproto: { 1079 | identifier: "atproto", 1080 | name: "ATPROTO", 1081 | url: "https://www.atproto.com", 1082 | genre: "open_web", 1083 | dns_prefix: "_atproto", 1084 | }, 1085 | bimi: { 1086 | identifier: "bimi", 1087 | name: "BIMI", 1088 | url: "https://www.bimi.com", 1089 | genre: "open_web", 1090 | dns_prefix: "_bimi", 1091 | }, 1092 | dmarc: { 1093 | identifier: "dmarc", 1094 | name: "DMARC", 1095 | url: "https://www.dmarc.com", 1096 | genre: "open_web", 1097 | dns_prefix: "_dmarc", 1098 | }, 1099 | investorflow: { 1100 | identifier: "investorflow", 1101 | name: "InvestorFlow", 1102 | url: "https://www.investorflow.com", 1103 | genre: "marketing", 1104 | txt_values: ["investorflow.com"], 1105 | }, 1106 | rails: { 1107 | identifier: "rails", 1108 | name: "Ruby on Rails", 1109 | url: "https://rubyonrails.org", 1110 | genre: "web_framework", 1111 | substrings: ["data-turbo", "RAILS_ENV"], 1112 | }, 1113 | new_relic: { 1114 | identifier: "new_relic", 1115 | name: "New Relic", 1116 | url: "https://www.newrelic.com", 1117 | genre: "monitoring", 1118 | substrings: ["newrelic.com"], 1119 | }, 1120 | writeas: { 1121 | identifier: "writeas", 1122 | name: "Write.as", 1123 | url: "https://www.write.as", 1124 | genre: "cms", 1125 | substrings: ["cdn.writeas"], 1126 | }, 1127 | nextjs: { 1128 | identifier: "nextjs", 1129 | name: "Next.js", 1130 | url: "https://www.nextjs.org", 1131 | genre: "web_framework", 1132 | substrings: ["_next/static"], 1133 | }, 1134 | klaviyo: { 1135 | identifier: "klaviyo", 1136 | name: "Klaviyo", 1137 | url: "https://www.klaviyo.com", 1138 | genre: "email", 1139 | txt_values: ["klaviyo-site-verification"], 1140 | substrings: ["klaviyo.init"], 1141 | spf_values: ["klaviyomail.com"], 1142 | }, 1143 | "101domain": { 1144 | identifier: "101domain", 1145 | name: "101domain", 1146 | url: "https://www.101domain.com", 1147 | genre: "dns", 1148 | ns_values: ["ns1.101domain.com", "ns2.101domain.com"], 1149 | }, 1150 | apollo: { 1151 | identifier: "apollo", 1152 | name: "Apollo", 1153 | url: "https://www.apollo.io", 1154 | genre: "analytics", 1155 | substrings: ["assets.apollo.io"], 1156 | }, 1157 | brevo: { 1158 | identifier: "brevo", 1159 | name: "Brevo", 1160 | url: "https://www.brevo.com", 1161 | genre: "email", 1162 | txt_values: ["brevo-code", "Sendinblue-code"], 1163 | substrings: ["sib_signup_form"], 1164 | spf_values: ["spf.sendinblue.com"], 1165 | }, 1166 | adobe: { 1167 | identifier: "adobe", 1168 | name: "Adobe", 1169 | url: "https://www.adobe.com", 1170 | genre: "design", 1171 | txt_values: ["adobe-idp"], 1172 | }, 1173 | namebright: { 1174 | identifier: "namebright", 1175 | name: "Namebright", 1176 | url: "https://www.namebright.com", 1177 | genre: "dns", 1178 | spf_values: ["namebrightmail.com"], 1179 | }, 1180 | zendesk: { 1181 | identifier: "zendesk", 1182 | name: "Zendesk", 1183 | url: "https://www.zendesk.com", 1184 | genre: "support", 1185 | txt_values: ["mail.zendesk.com"], 1186 | substrings: ["zdassets.com"], 1187 | spf_values: ["mail.zendesk.com", "_spf.zdsys.com"], 1188 | }, 1189 | paddle: { 1190 | identifier: "paddle", 1191 | name: "Paddle", 1192 | url: "https://www.paddle.com", 1193 | genre: "payments", 1194 | txt_values: ["paddle-verification"], 1195 | }, 1196 | stripe: { 1197 | identifier: "stripe", 1198 | name: "Stripe", 1199 | url: "https://www.stripe.com", 1200 | genre: "payments", 1201 | txt_values: ["stripe-verification"], 1202 | substrings: ["js.stripe.com"], 1203 | }, 1204 | beehiiv: { 1205 | identifier: "beehiiv", 1206 | name: "Beehiiv", 1207 | url: "https://www.beehiiv.com", 1208 | genre: "marketing", 1209 | substrings: ["beehiiv.com"], 1210 | }, 1211 | appsflyer: { 1212 | identifier: "appsflyer", 1213 | name: "AppsFlyer", 1214 | url: "https://www.appsflyer.com", 1215 | genre: "analytics", 1216 | cname_values: ["appsflyer.com"], 1217 | }, 1218 | lemon_squeezy: { 1219 | identifier: "lemon_squeezy", 1220 | name: "Lemon Squeezy", 1221 | url: "https://www.lemonsqueezy.com", 1222 | genre: "payments", 1223 | substrings: ["assets.lemonsqueezy.com"], 1224 | }, 1225 | fly_io: { 1226 | identifier: "fly_io", 1227 | name: "Fly.io", 1228 | url: "https://www.fly.io", 1229 | genre: "hosting", 1230 | cname_values: ["fly.dev"], 1231 | headers: { 1232 | key: "text/headers/fly-request-id", 1233 | value: "*", 1234 | }, 1235 | }, 1236 | buymeacoffee: { 1237 | identifier: "buymeacoffee", 1238 | name: "Buy Me a Coffee", 1239 | url: "https://www.buymeacoffee.com", 1240 | genre: "payments", 1241 | substrings: ["cdnjs.buymeacoffee.com"], 1242 | }, 1243 | icloud_mail: { 1244 | identifier: "icloud_mail", 1245 | name: "iCloud Mail", 1246 | url: "https://www.icloud.com", 1247 | genre: "email", 1248 | mx_values: ["mx01.mail.icloud.com"], 1249 | }, 1250 | blot: { 1251 | identifier: "blot", 1252 | name: "Blot", 1253 | url: "https://www.blot.im", 1254 | genre: "email", 1255 | substrings: ["blot.im"], 1256 | headers: { 1257 | key: "blot-server", 1258 | value: "*", 1259 | }, 1260 | }, 1261 | mimecast: { 1262 | identifier: "mimecast", 1263 | name: "Mimecast", 1264 | url: "https://www.mimecast.com", 1265 | genre: "email", 1266 | spf_values: ["eu._netblocks.mimecast.com"], 1267 | }, 1268 | vwo: { 1269 | identifier: "vwo", 1270 | name: "Visual Website Optimizer", 1271 | url: "https://www.vwo.com", 1272 | genre: "marketing", 1273 | substrings: ["_vwo_code"], 1274 | }, 1275 | squarespace: { 1276 | identifier: "squarespace", 1277 | name: "Squarespace", 1278 | url: "https://www.squarespace.com", 1279 | genre: "cms", 1280 | substrings: ["assets.squarespace.com"], 1281 | }, 1282 | wordpress: { 1283 | identifier: "wordpress", 1284 | name: "WordPress", 1285 | url: "https://www.wordpress.org", 1286 | genre: "cms", 1287 | substrings: ["wp-content/plugins"], 1288 | }, 1289 | blogger: { 1290 | identifier: "blogger", 1291 | name: "Blogger", 1292 | url: "https://www.blogger.com", 1293 | genre: "cms", 1294 | substrings: ['content="blogger" name="generator"'], 1295 | }, 1296 | jekyll: { 1297 | identifier: "jekyll", 1298 | name: "Jekyll", 1299 | url: "https://www.jekyllrb.com", 1300 | genre: "static_site_generator", 1301 | substrings: ['content="Jekyll'], 1302 | }, 1303 | plausible: { 1304 | identifier: "plausible", 1305 | name: "Plausible", 1306 | url: "https://www.plausible.io", 1307 | genre: "analytics", 1308 | substrings: ["plausible.io/js", "click.pageview.click"], 1309 | }, 1310 | whimsical: { 1311 | identifier: "whimsical", 1312 | name: "Whimsical", 1313 | url: "https://www.whimsical.com", 1314 | genre: "design", 1315 | txt_values: ["whimsical="], 1316 | }, 1317 | mediavine: { 1318 | identifier: "mediavine", 1319 | name: "Mediavine", 1320 | url: "https://www.mediavine.com", 1321 | genre: "ads", 1322 | substrings: [".mediavine.com"], 1323 | }, 1324 | gravity_forms: { 1325 | identifier: "gravity_forms", 1326 | name: "Gravity Forms", 1327 | url: "https://www.gravityforms.com", 1328 | genre: "form", 1329 | substrings: ["gravityforms.com", "gform_"], 1330 | }, 1331 | litespeed: { 1332 | identifier: "litespeed", 1333 | name: "LiteSpeed", 1334 | url: "https://www.litespeedtech.com", 1335 | genre: "hosting", 1336 | headers: { 1337 | key: "server", 1338 | value: "LiteSpeed", 1339 | }, 1340 | }, 1341 | pushengage: { 1342 | identifier: "pushengage", 1343 | name: "PushEngage", 1344 | url: "https://www.pushengage.com", 1345 | genre: "marketing", 1346 | substrings: ["pushengage.com"], 1347 | }, 1348 | humanity: { 1349 | identifier: "humanity", 1350 | name: "Humanity", 1351 | url: "https://www.hu-manity.co", 1352 | genre: "gdpr", 1353 | substrings: ["//cdn.hu-manity.co"], 1354 | }, 1355 | clickdimensions: { 1356 | identifier: "clickdimensions", 1357 | name: "ClickDimensions", 1358 | url: "https://www.clickdimensions.com", 1359 | genre: "marketing", 1360 | substrings: ["clickdimensions.com"], 1361 | }, 1362 | mixpanel: { 1363 | identifier: "mixpanel", 1364 | name: "Mixpanel", 1365 | url: "https://www.mixpanel.com", 1366 | genre: "analytics", 1367 | txt_values: ["mixpanel-domain-verify"], 1368 | }, 1369 | microsoft_ads: { 1370 | identifier: "microsoft_ads", 1371 | name: "Microsoft Ads", 1372 | url: "https://www.microsoft.com", 1373 | genre: "analytics", 1374 | substrings: ["w,d,t,r,u"], 1375 | }, 1376 | microsoft_clarity: { 1377 | identifier: "microsoft_clarity", 1378 | name: "Microsoft Clarity", 1379 | url: "https://www.microsoft.com", 1380 | genre: "analytics", 1381 | substrings: ["c,l,a,r,i,t,y"], 1382 | }, 1383 | microsoft_365: { 1384 | identifier: "microsoft_365", 1385 | name: "Microsoft 365", 1386 | url: "https://www.microsoft.com", 1387 | genre: "everything", 1388 | txt_values: ["v=verifydomain MS"], 1389 | }, 1390 | buttondown: { 1391 | identifier: "buttondown", 1392 | name: "Buttondown", 1393 | url: "https://www.buttondown.email", 1394 | genre: "email", 1395 | substrings: ["buttondown.email/", "buttondown.com/"], 1396 | }, 1397 | intercom: { 1398 | identifier: "intercom", 1399 | name: "Intercom", 1400 | url: "https://www.intercom.com", 1401 | genre: "support", 1402 | substrings: ["intercomSettings"], 1403 | spf_values: ["spf.mail.intercom.io"], 1404 | }, 1405 | gatsby: { 1406 | identifier: "gatsby", 1407 | name: "Gatsby", 1408 | url: "https://www.gatsbyjs.com", 1409 | genre: "web_framework", 1410 | substrings: ['name="generator" content="Gatsby'], 1411 | }, 1412 | ghost: { 1413 | identifier: "ghost", 1414 | name: "Ghost", 1415 | url: "https://www.ghost.org", 1416 | genre: "cms", 1417 | substrings: ['name="generator" content="Ghost'], 1418 | }, 1419 | hotjar: { 1420 | identifier: "hotjar", 1421 | name: "Hotjar", 1422 | url: "https://www.hotjar.com", 1423 | genre: "analytics", 1424 | substrings: ["h,o,t,j,a,r", "h, o, t, j, a, r)"], 1425 | }, 1426 | drift: { 1427 | identifier: "drift", 1428 | name: "Drift", 1429 | url: "https://www.drift.com", 1430 | genre: "support", 1431 | txt_values: ["drift-domain-verification"], 1432 | }, 1433 | mailgun: { 1434 | identifier: "mailgun", 1435 | name: "Mailgun", 1436 | url: "https://www.mailgun.com", 1437 | genre: "email", 1438 | spf_values: [ 1439 | "mailgun.org", 1440 | "spf.mandrillapp.com", 1441 | "spf1.mailgun.org", 1442 | "spf2.mailgun.org", 1443 | ], 1444 | }, 1445 | campaign_monitor: { 1446 | identifier: "campaign_monitor", 1447 | name: "Campaign Monitor", 1448 | url: "https://www.campaignmonitor.com", 1449 | genre: "email", 1450 | spf_values: ["_spf.createsend.com", "spf.createsend.com"], 1451 | urlSubstrings: ["confirmsubscription.com"], 1452 | }, 1453 | google: { 1454 | identifier: "google", 1455 | name: "Google", 1456 | url: "https://www.google.com", 1457 | genre: "everything", 1458 | spf_values: ["_spf.google.com"], 1459 | }, 1460 | rollbar: { 1461 | identifier: "rollbar", 1462 | name: "Rollbar", 1463 | url: "https://www.rollbar.com", 1464 | genre: "monitoring", 1465 | substrings: ["_rollbarConfig"], 1466 | }, 1467 | postmark: { 1468 | identifier: "postmark", 1469 | name: "Postmark", 1470 | url: "https://www.postmarkapp.com", 1471 | genre: "email", 1472 | spf_values: ["mtasv.net", "spf.mtasv.net"], 1473 | }, 1474 | posthog: { 1475 | identifier: "posthog", 1476 | name: "PostHog", 1477 | url: "https://www.posthog.com", 1478 | genre: "analytics", 1479 | substrings: ["posthog.init"], 1480 | }, 1481 | amplitude: { 1482 | identifier: "amplitude", 1483 | name: "Amplitude", 1484 | url: "https://www.amplitude.com", 1485 | genre: "analytics", 1486 | substrings: ["amplitude.init", "e.amplitude"], 1487 | }, 1488 | markmonitor: { 1489 | identifier: "markmonitor", 1490 | name: "MarkMonitor", 1491 | url: "https://www.markmonitor.com", 1492 | genre: "monitoring", 1493 | spf_values: ["spfhost.messageprovider.com"], 1494 | }, 1495 | marketo: { 1496 | identifier: "marketo", 1497 | name: "Marketo", 1498 | url: "https://www.marketo.com", 1499 | spf_values: ["mktomail.com"], 1500 | substrings: ["bizible.com"], 1501 | genre: "crm", 1502 | }, 1503 | heymarket: { 1504 | identifier: "heymarket", 1505 | name: "Heymarket", 1506 | url: "https://www.heymarket.com", 1507 | genre: "support", 1508 | substrings: ["heymarket.com"], 1509 | }, 1510 | smtp2go: { 1511 | identifier: "smtp2go", 1512 | name: "SMTP2GO", 1513 | url: "https://www.smtp2go.com", 1514 | genre: "email", 1515 | spf_values: ["spf.smtp2go.com"], 1516 | }, 1517 | dynatrace: { 1518 | identifier: "dynatrace", 1519 | name: "Dynatrace", 1520 | url: "https://www.dynatrace.com", 1521 | genre: "monitoring", 1522 | txt_values: ["Dynatrace-site-verification"], 1523 | }, 1524 | salesforce: { 1525 | identifier: "salesforce", 1526 | name: "Salesforce", 1527 | url: "https://www.salesforce.com", 1528 | genre: "crm", 1529 | spf_values: ["_spf.salesforce.com", "cust-spf.exacttarget.com"], 1530 | }, 1531 | salesforce_marketing_cloud: { 1532 | identifier: "salesforce_marketing_cloud", 1533 | name: "Salesforce Marketing Cloud", 1534 | url: "https://www.salesforce.com", 1535 | genre: "marketing", 1536 | txt_values: ["SMFC-"], 1537 | }, 1538 | happyfox: { 1539 | identifier: "happyfox", 1540 | name: "HappyFox", 1541 | genre: "support", 1542 | spf_values: ["spf.happyfox.com"], 1543 | url: "https://www.happyfox.com", 1544 | substrings: ["HFCHAT_CONFIG"], 1545 | }, 1546 | consider: { 1547 | identifier: "consider", 1548 | name: "Consider", 1549 | url: "https://www.consider.com", 1550 | genre: "ats", 1551 | cname_values: ["boards.consider.com"], 1552 | }, 1553 | svelte: { 1554 | identifier: "svelte", 1555 | name: "Svelte", 1556 | url: "https://www.svelte.dev", 1557 | genre: "web_framework", 1558 | substrings: ["__svelte__", "__sveltekit"], 1559 | }, 1560 | dropbox: { 1561 | identifier: "dropbox", 1562 | name: "Dropbox", 1563 | url: "https://www.dropbox.com", 1564 | genre: "storage", 1565 | txt_values: ["dropbox-domain-verification"], 1566 | }, 1567 | dropcatch: { 1568 | identifier: "dropcatch", 1569 | name: "Dropcatch", 1570 | url: "https://www.dropcatch.com", 1571 | genre: "dns", 1572 | ns_values: ["ns1.dropcatch.com", "ns2.dropcatch.com"], 1573 | }, 1574 | twitter: { 1575 | identifier: "twitter", 1576 | name: "Twitter", 1577 | genre: "social_media", 1578 | url: "https://www.twitter.com", 1579 | icon: , 1580 | urlSubstrings: ["twitter.com", "x.com"], 1581 | }, 1582 | medium: { 1583 | identifier: "medium", 1584 | name: "Medium", 1585 | genre: "social_media", 1586 | url: "https://www.medium.com", 1587 | urlSubstrings: ["medium.com"], 1588 | }, 1589 | email_octopus: { 1590 | identifier: "email_octopus", 1591 | name: "EmailOctopus", 1592 | genre: "email", 1593 | url: "https://www.emailoctopus.com", 1594 | spf_values: ["*.eoidentity.com"], 1595 | }, 1596 | clicky: { 1597 | identifier: "clicky", 1598 | name: "Clicky", 1599 | genre: "analytics", 1600 | url: "https://www.clicky.com", 1601 | substrings: ["getclicky.com"], 1602 | }, 1603 | nuxt: { 1604 | identifier: "nuxt", 1605 | name: "Nuxt", 1606 | genre: "web_framework", 1607 | url: "https://www.nuxtjs.org", 1608 | substrings: ["/_nuxt/"], 1609 | }, 1610 | aweber: { 1611 | identifier: "aweber", 1612 | name: "AWeber", 1613 | genre: "email", 1614 | url: "https://www.aweber.com", 1615 | spf_values: ["send.aweber.com", "fbl.optin.com"], 1616 | }, 1617 | font_awesome: { 1618 | identifier: "font_awesome", 1619 | name: "Font Awesome", 1620 | genre: "web_framework", 1621 | url: "https://www.fontawesome.com", 1622 | substrings: [ 1623 | "cdnjs.cloudflare.com/ajax/libs/font-awesome", 1624 | "kit.fontawesome.com", 1625 | ], 1626 | }, 1627 | moosend: { 1628 | identifier: "moosend", 1629 | name: "Moosend", 1630 | genre: "email", 1631 | url: "https://www.moosend.com", 1632 | spf_values: ["spfa.mailend.com"], 1633 | }, 1634 | uptimerobot: { 1635 | identifier: "uptimerobot", 1636 | name: "UptimeRobot", 1637 | genre: "monitoring", 1638 | url: "https://www.uptimerobot.com", 1639 | cname_values: ["stats.uptimerobot.com"], 1640 | }, 1641 | oneuptime: { 1642 | identifier: "oneuptime", 1643 | name: "OneUptime", 1644 | genre: "monitoring", 1645 | url: "https://www.oneuptime.com", 1646 | txt_values: ["oneuptime-verification"], 1647 | }, 1648 | discord: { 1649 | identifier: "discord", 1650 | name: "Discord", 1651 | genre: "social_media", 1652 | url: "https://www.discord.com", 1653 | urlSubstrings: ["discord.gg/"], 1654 | }, 1655 | tiktok_pixel: { 1656 | identifier: "tiktok_pixel", 1657 | name: "TikTok Pixel", 1658 | genre: "analytics", 1659 | url: "https://www.tiktok.com", 1660 | substrings: ["TiktokAnalyticsObject"], 1661 | }, 1662 | tiktok: { 1663 | identifier: "tiktok", 1664 | name: "TikTok", 1665 | genre: "social_media", 1666 | url: "https://www.tiktok.com", 1667 | icon: , 1668 | urlSubstrings: ["tiktok.com"], 1669 | }, 1670 | greenhouse: { 1671 | identifier: "greenhouse", 1672 | name: "Greenhouse", 1673 | genre: "ats", 1674 | url: "https://www.greenhouse.io", 1675 | substrings: ["greenhouse.io"], 1676 | spf_values: ["mg-spf.greenhouse.io"], 1677 | }, 1678 | netsuite: { 1679 | identifier: "netsuite", 1680 | name: "NetSuite", 1681 | genre: "crm", 1682 | url: "https://www.netsuite.com", 1683 | spf_values: ["mailsenders.netsuite.com"], 1684 | }, 1685 | instagram: { 1686 | identifier: "instagram", 1687 | name: "Instagram", 1688 | genre: "social_media", 1689 | url: "https://www.instagram.com", 1690 | icon: , 1691 | urlSubstrings: ["instagram.com"], 1692 | }, 1693 | facebook: { 1694 | identifier: "facebook", 1695 | name: "Facebook", 1696 | genre: "social_media", 1697 | url: "https://www.facebook.com", 1698 | icon: , 1699 | urlSubstrings: ["facebook.com", "facebook.com/groups"], 1700 | }, 1701 | pardot: { 1702 | identifier: "pardot", 1703 | name: "Pardot", 1704 | genre: "marketing", 1705 | url: "https://www.pardot.com", 1706 | spf_values: ["aspmx.pardot.com"], 1707 | }, 1708 | linear: { 1709 | identifier: "linear", 1710 | name: "Linear", 1711 | genre: "support", 1712 | url: "https://www.linear.app", 1713 | txt_values: ["linea-domain-verification"], 1714 | }, 1715 | linkedin: { 1716 | identifier: "linkedin", 1717 | name: "LinkedIn", 1718 | genre: "social_media", 1719 | url: "https://www.linkedin.com", 1720 | icon: , 1721 | urlSubstrings: [ 1722 | "linkedin.com/company", 1723 | "linkedin.com/school", 1724 | "linkedin.com", 1725 | "linkedin.com/in", 1726 | ], 1727 | }, 1728 | linkedin_ads: { 1729 | identifier: "linkedin_ads", 1730 | name: "LinkedIn Ads", 1731 | genre: "marketing", 1732 | url: "https://www.linkedin.com", 1733 | icon: , 1734 | substrings: ["ads.linkedin.com"], 1735 | }, 1736 | hugo: { 1737 | identifier: "hugo", 1738 | name: "Hugo", 1739 | genre: "static_site_generator", 1740 | url: "https://gohugo.io", 1741 | substrings: ['content="Hugo'], 1742 | }, 1743 | fuse: { 1744 | identifier: "fuse", 1745 | name: "Fuse", 1746 | genre: "search", 1747 | url: "https://www.fusejs.io", 1748 | substrings: ["/fuse.js"], 1749 | }, 1750 | typekit: { 1751 | identifier: "typekit", 1752 | name: "Typekit", 1753 | genre: "web_framework", 1754 | url: "https://typekit.com", 1755 | substrings: ["typekit.net"], 1756 | }, 1757 | youtube: { 1758 | identifier: "youtube", 1759 | name: "YouTube", 1760 | genre: "social_media", 1761 | url: "https://www.youtube.com", 1762 | icon: , 1763 | urlSubstrings: ["youtube.com", "youtube.com/c", "youtube.com/channel"], 1764 | }, 1765 | hashicorp: { 1766 | identifier: "hashicorp", 1767 | name: "HashiCorp", 1768 | genre: "hosting", 1769 | url: "https://www.hashicorp.com", 1770 | txt_values: ["hcp-domain-verification"], 1771 | }, 1772 | afterpay: { 1773 | identifier: "afterpay", 1774 | name: "Afterpay", 1775 | genre: "payments", 1776 | url: "https://www.afterpay.com", 1777 | substrings: ["js.afterpay.com"], 1778 | }, 1779 | stackadapt: { 1780 | identifier: "stackadapt", 1781 | name: "StackAdapt", 1782 | genre: "marketing", 1783 | url: "https://www.stackadapt.com", 1784 | substrings: ["srv.stackadapt.com"], 1785 | }, 1786 | titan_email: { 1787 | identifier: "titan_email", 1788 | name: "Titan Email", 1789 | genre: "email", 1790 | url: "https://titan.email", 1791 | spf_values: ["spf.titan.email"], 1792 | }, 1793 | affwp: { 1794 | identifier: "affwp", 1795 | name: "AffiliateWP", 1796 | genre: "marketing", 1797 | url: "https://affiliatewp.com", 1798 | substrings: ["AFFWP"], 1799 | }, 1800 | jsdelivr: { 1801 | identifier: "jsdelivr", 1802 | name: "JSDelivr", 1803 | genre: "hosting", 1804 | url: "https://www.jsdelivr.com", 1805 | substrings: ["cdn.jsdelivr.net"], 1806 | }, 1807 | google_fonts: { 1808 | identifier: "google_fonts", 1809 | name: "Google Fonts", 1810 | genre: "web_framework", 1811 | url: "https://fonts.google.com", 1812 | substrings: ["fonts.googleapis.com"], 1813 | }, 1814 | chrome_webstore: { 1815 | identifier: "chrome_webstore", 1816 | name: "Chrome Web Store", 1817 | genre: "marketing", 1818 | url: "https://chrome.google.com/webstore", 1819 | substrings: ["chrome-webstore-item"], 1820 | }, 1821 | iubenda: { 1822 | identifier: "iubenda", 1823 | name: "Iubenda", 1824 | genre: "gdpr", 1825 | url: "https://www.iubenda.com", 1826 | substrings: ["cdn.iubenda.com"], 1827 | }, 1828 | foundry: { 1829 | identifier: "foundry", 1830 | name: "Foundry", 1831 | genre: "analytics", 1832 | url: "https://www.foundryco.com", 1833 | substrings: ["triblio.io"], 1834 | }, 1835 | firebase: { 1836 | identifier: "firebase", 1837 | name: "Firebase", 1838 | genre: "web_framework", 1839 | url: "https://firebase.google.com", 1840 | substrings: ["gstatic.com/firebasejs"], 1841 | spf_values: ["_spf.firebasemail.com"], 1842 | }, 1843 | piwik: { 1844 | identifier: "piwik", 1845 | name: "Piwik", 1846 | genre: "analytics", 1847 | url: "https://www.piwik.org", 1848 | substrings: ["piwik.js", "piwik.pro"], 1849 | }, 1850 | genesys: { 1851 | identifier: "genesys", 1852 | name: "Genesys", 1853 | genre: "support", 1854 | url: "https://www.genesys.com", 1855 | substrings: ["Genesys("], 1856 | }, 1857 | cookiebot: { 1858 | identifier: "cookiebot", 1859 | name: "Cookiebot", 1860 | genre: "gdpr", 1861 | url: "https://www.cookiebot.com", 1862 | substrings: ["consent.cookiebot.com"], 1863 | }, 1864 | cookiefirst: { 1865 | identifier: "cookiefirst", 1866 | name: "CookieFirst", 1867 | genre: "gdpr", 1868 | url: "https://www.cookiefirst.com", 1869 | substrings: ["cookiefirst.com"], 1870 | }, 1871 | yottaa: { 1872 | identifier: "yottaa", 1873 | name: "Yottaa", 1874 | genre: "cdn", 1875 | url: "https://www.yottaa.com", 1876 | substrings: ["cdn.yottaa.com"], 1877 | }, 1878 | cookielaw: { 1879 | identifier: "cookielaw", 1880 | name: "Cookie Law", 1881 | genre: "gdpr", 1882 | url: "https://www.cookielaw.org", 1883 | substrings: ["cdn.cookielaw.org"], 1884 | }, 1885 | hubspot: { 1886 | identifier: "hubspot", 1887 | name: "HubSpot", 1888 | genre: "marketing", 1889 | url: "https://www.hubspot.com", 1890 | substrings: ["hs-scripts.com", "hs-banner.com", "hs-script-loader"], 1891 | txt_values: [ 1892 | "hubspot-domain-verification", 1893 | "hubspot-developer-verification", 1894 | ], 1895 | }, 1896 | github: { 1897 | identifier: "github", 1898 | name: "GitHub", 1899 | genre: "social_media", 1900 | url: "https://www.github.com", 1901 | icon: , 1902 | urlSubstrings: ["github.com", "github.com/[a-z]*"], 1903 | }, 1904 | snapchat: { 1905 | identifier: "snapchat", 1906 | name: "Snapchat", 1907 | genre: "social_media", 1908 | url: "https://www.snapchat.com", 1909 | urlSubstrings: ["snapchat.com"], 1910 | }, 1911 | pinterest: { 1912 | identifier: "pinterest", 1913 | name: "Pinterest", 1914 | genre: "social_media", 1915 | url: "https://www.pinterest.com", 1916 | icon: , 1917 | urlSubstrings: ["pinterest.com"], 1918 | }, 1919 | reddit: { 1920 | identifier: "reddit", 1921 | name: "Reddit", 1922 | genre: "social_media", 1923 | url: "https://www.reddit.com", 1924 | urlSubstrings: ["reddit.com"], 1925 | }, 1926 | patreon: { 1927 | identifier: "patreon", 1928 | name: "Patreon", 1929 | genre: "social_media", 1930 | url: "https://www.patreon.com", 1931 | urlSubstrings: ["patreon.com"], 1932 | }, 1933 | outseta: { 1934 | identifier: "outseta", 1935 | name: "Outseta", 1936 | genre: "payments", 1937 | url: "https://www.outseta.com", 1938 | substrings: ["cdn.outseta.com"], 1939 | }, 1940 | segment: { 1941 | identifier: "segment", 1942 | name: "Segment", 1943 | genre: "analytics", 1944 | url: "https://www.segment.com", 1945 | substrings: ["cdn.segment.com"], 1946 | }, 1947 | apple_pay: { 1948 | identifier: "apple_pay", 1949 | name: "Apple Pay", 1950 | genre: "payments", 1951 | url: "https://www.apple.com/apple-pay", 1952 | substrings: ["apple-pay-shop-capabilities"], 1953 | }, 1954 | optinmonster: { 1955 | identifier: "optinmonster", 1956 | name: "OptinMonster", 1957 | genre: "marketing", 1958 | url: "https://www.optinmonster.com", 1959 | substrings: ["cdn.omtrdc.net", "omappapi.com"], 1960 | }, 1961 | chatbase: { 1962 | identifier: "chatbase", 1963 | name: "Chatbase", 1964 | genre: "support", 1965 | url: "https://www.chatbase.com", 1966 | substrings: ["chatbase.co/embed.min.js"], 1967 | }, 1968 | framer: { 1969 | identifier: "framer", 1970 | name: "Framer", 1971 | genre: "hosting", 1972 | url: "https://www.framer.com", 1973 | substrings: ["Built with Framer"], 1974 | }, 1975 | presslabs: { 1976 | identifier: "presslabs", 1977 | name: "Presslabs", 1978 | genre: "hosting", 1979 | url: "https://www.presslabs.com", 1980 | headers: { 1981 | key: "X-Presslab-Stats", 1982 | value: "*" 1983 | } 1984 | }, 1985 | bubble: { 1986 | identifier: "bubble", 1987 | name: "Bubble", 1988 | genre: "hosting", 1989 | url: "https://www.bubble.io", 1990 | substrings: ["bubble_page_load_id"], 1991 | }, 1992 | convertkit: { 1993 | identifier: "convertkit", 1994 | name: "ConvertKit", 1995 | genre: "email", 1996 | url: "https://www.convertkit.com", 1997 | substrings: [ 1998 | "filekitcdn.com", 1999 | "/convertkit/", 2000 | "f.convertkit.com", 2001 | "app.convertkit.com", 2002 | "data-sr-convertkit-subscribe-form", 2003 | ".ck.page", 2004 | ], 2005 | }, 2006 | sparkpost: { 2007 | identifier: "sparkpost", 2008 | name: "SparkPost", 2009 | genre: "email", 2010 | url: "https://www.sparkpost.com", 2011 | spf_values: ["sparkpostmail.com"], 2012 | }, 2013 | formspark: { 2014 | identifier: "formspark", 2015 | name: "Formspark", 2016 | genre: "form", 2017 | url: "https://www.formspark.io", 2018 | substrings: ["submit-form.io"], 2019 | }, 2020 | formkeep: { 2021 | identifier: "formkeep", 2022 | name: "FormKeep", 2023 | genre: "form", 2024 | url: "https://www.formkeep.com", 2025 | substrings: ["formkeep.com/f"], 2026 | }, 2027 | revue: { 2028 | identifier: "revue", 2029 | name: "Revue", 2030 | genre: "email", 2031 | url: "https://www.getrevue.co", 2032 | substrings: ["revue-form"], 2033 | defunct: true, 2034 | }, 2035 | tinylytics: { 2036 | identifier: "tinylytics", 2037 | name: "Tinylytics", 2038 | genre: "analytics", 2039 | url: "https://www.tinylytics.app", 2040 | substrings: ["//tinylytics.app/embed"], 2041 | }, 2042 | judgeme: { 2043 | identifier: "judgeme", 2044 | name: "Judge.me", 2045 | genre: "ecommerce", 2046 | url: "https://www.judge.me", 2047 | substrings: ["judge.me"], 2048 | }, 2049 | pandectes: { 2050 | identifier: "pandectes", 2051 | name: "Pandectes", 2052 | genre: "gdpr", 2053 | url: "https://www.pandectes.io", 2054 | substrings: ["pandectes"], 2055 | }, 2056 | tinyletter: { 2057 | identifier: "tinyletter", 2058 | name: "TinyLetter", 2059 | genre: "email", 2060 | url: "https://www.tinyletter.com", 2061 | substrings: ['action="https://tinyletter.com'], 2062 | defunct: true, 2063 | }, 2064 | cuttly: { 2065 | identifier: "cuttly", 2066 | name: "Cuttly", 2067 | genre: "url_shortener", 2068 | url: "https://www.cutt.ly", 2069 | txt_values: ["cuttly-verification-site"], 2070 | }, 2071 | migadu: { 2072 | identifier: "migadu", 2073 | name: "Migadu", 2074 | genre: "email", 2075 | url: "https://www.migadu.com", 2076 | txt_values: ["hosted-email-verify"], 2077 | spf_values: ["spf.migadu.com"], 2078 | }, 2079 | famewall: { 2080 | identifier: "famewall", 2081 | name: "Famewall", 2082 | genre: "marketing", 2083 | url: "https://www.famewall.com", 2084 | substrings: ["famewall-embed"], 2085 | }, 2086 | buysellads: { 2087 | identifier: "buysellads", 2088 | name: "BuySellAds", 2089 | genre: "ads", 2090 | url: "https://www.buysellads.com", 2091 | substrings: ["servedby-buysellads.com"], 2092 | }, 2093 | rebuy_engine: { 2094 | identifier: "rebuy_engine", 2095 | name: "Rebuy Engine", 2096 | genre: "ecommerce", 2097 | url: "https://www.rebuyengine.com", 2098 | substrings: ["cdn.rebuyengine.com"], 2099 | }, 2100 | omega_commerce: { 2101 | identifier: "omega_commerce", 2102 | name: "Omega Commerce", 2103 | genre: "ecommerce", 2104 | url: "https://www.omegacommerce.com", 2105 | substrings: ["omegacommerce.com"], 2106 | }, 2107 | northbeam: { 2108 | identifier: "northbeam", 2109 | description: "The marketing intelligence platform for profitable growth.", 2110 | name: "Northbeam", 2111 | genre: "analytics", 2112 | url: "https://www.northbeam.io", 2113 | substrings: ["j.northbeam.io"], 2114 | }, 2115 | 506: { 2116 | identifier: "506", 2117 | name: "506", 2118 | genre: "ecommerce", 2119 | url: "https://www.506.io", 2120 | substrings: ["506.io"], 2121 | }, 2122 | protonmail: { 2123 | identifier: "protonmail", 2124 | name: "ProtonMail", 2125 | genre: "email", 2126 | url: "https://www.protonmail.com", 2127 | spf_values: ["_spf.protonmail.ch"], 2128 | txt_values: ["protonmail-verification"], 2129 | }, 2130 | maxio: { 2131 | identifier: "maxio", 2132 | name: "Maxio", 2133 | genre: "accounting", 2134 | url: "https://www.maxio.com", 2135 | spf_values: ["mailer.chargify.com"], 2136 | }, 2137 | atlassian: { 2138 | identifier: "atlassian", 2139 | name: "Atlassian", 2140 | genre: "everything", 2141 | url: "https://www.atlassian.com", 2142 | txt_values: ["atlassian-domain-verification"], 2143 | }, 2144 | onepassword: { 2145 | identifier: "onepassword", 2146 | name: "1Password", 2147 | genre: "security", 2148 | url: "https://www.1password.com", 2149 | txt_values: ["1password-site-verification"], 2150 | }, 2151 | } as const; 2152 | --------------------------------------------------------------------------------