├── pages ├── public │ └── .gitignore ├── .prettierignore ├── .gitignore ├── .prettierrc.json ├── README.md ├── tsconfig.json ├── index.html ├── webpack.dev.config.js ├── src │ ├── hooks │ │ └── useQueryCache.ts │ └── index.tsx ├── webpack.common.js ├── webpack.prod.config.js └── package.json ├── frontend ├── public │ ├── .gitignore │ └── icons │ │ ├── lemmy_64px.png │ │ ├── mbin_64px.png │ │ ├── piefed_64px.png │ │ └── orig │ │ └── MBin_Logo.svg ├── .prettierignore ├── .gitignore ├── tsconfig.jest.json ├── .prettierrc.json ├── src │ ├── store.ts │ ├── theme.ts │ ├── reducers.ts │ ├── components │ │ ├── InstanceView │ │ │ ├── InstanceContentGrowth.tsx │ │ │ ├── InstanceCommunities.tsx │ │ │ ├── InstanceVersions.tsx │ │ │ └── InstanceUserGrowth.tsx │ │ ├── Shared │ │ │ ├── InstanceTypeIcon.tsx │ │ │ ├── CardStatBox.tsx │ │ │ ├── LanguageFilter.tsx │ │ │ ├── InstanceModal.tsx │ │ │ ├── LineGraph.tsx │ │ │ ├── Avatar.tsx │ │ │ ├── MultiDataLineGraph.tsx │ │ │ ├── StatGridCards.tsx │ │ │ └── Link.tsx │ │ ├── GridView │ │ │ ├── Instance.tsx │ │ │ ├── MBin.tsx │ │ │ ├── Piefed.tsx │ │ │ └── Community.tsx │ │ ├── Inspector │ │ │ ├── VersionChart.tsx │ │ │ └── Overview.tsx │ │ └── ListView │ │ │ ├── MBin.tsx │ │ │ └── Piefed.tsx │ ├── pages │ │ ├── Communities.tsx │ │ └── Inspector.tsx │ ├── hooks │ │ ├── useStorage.ts │ │ ├── useQueryCache.ts │ │ └── useCachedMultipart.ts │ ├── index.tsx │ ├── reducers │ │ ├── modalReducer.ts │ │ └── configReducer.ts │ ├── lib │ │ └── storage.ts │ └── App.tsx ├── jest.config.cjs ├── README.md ├── tsconfig.json ├── test │ ├── config │ │ ├── global.setup.ts │ │ └── test.utils.ts │ ├── spec │ │ ├── ui-interactions.spec.ts │ │ ├── query-params.spec.ts │ │ ├── navigation.spec.ts │ │ └── main-screenshots.spec.ts │ └── unit │ │ └── utils.test.ts ├── index.html ├── webpack.dev.config.js ├── webpack.prod.config.js ├── playwright.config.ts ├── webpack.common.js └── package.json ├── cdk ├── .gitignore ├── .prettierrc.json ├── config.example.json ├── README.md ├── package.json ├── lib │ ├── cert-stack.ts │ ├── build-stack.ts │ └── frontend-stack.ts ├── bin │ └── explorer.ts ├── cdk.json └── tsconfig.json ├── crawler ├── .gitignore ├── tsconfig.jest.json ├── .prettierrc.json ├── docker-compose.github.yaml ├── jest.config.cjs ├── tsconfig.json ├── .env.example ├── src │ ├── lib │ │ ├── storage │ │ │ ├── uptime.ts │ │ │ ├── fediseer.ts │ │ │ ├── fediverse.ts │ │ │ ├── instance.ts │ │ │ ├── tracking.ts │ │ │ ├── community.ts │ │ │ ├── mbin.ts │ │ │ └── piefed.ts │ │ ├── error.ts │ │ ├── validator.ts │ │ ├── CrawlClient.ts │ │ └── logging.ts │ ├── queue │ │ ├── community_list.ts │ │ ├── mbin.ts │ │ ├── community_single.ts │ │ ├── piefed.ts │ │ ├── instance.ts │ │ └── BaseQueue.ts │ ├── crawl │ │ └── uptime.ts │ ├── output │ │ ├── sync_s3.ts │ │ └── classifier.ts │ └── bin │ │ └── manual.ts ├── index.ts ├── test │ ├── output │ │ ├── output-classifier.test.ts │ │ └── output-utils.test.ts │ ├── storage │ │ └── mbin.test.ts │ ├── lib │ │ ├── crawlClient.test.ts │ │ ├── keepAlive.test.ts │ │ └── validator.test.ts │ ├── queue │ │ └── instance.test.ts │ └── crawl │ │ ├── instanceCommunity.test.ts │ │ └── mbinInstance.test.ts ├── Dockerfile ├── docker-compose.yaml ├── ecosystem.config.cjs └── package.json ├── docs ├── images │ ├── 0.10.0-communities.png │ └── 0.2.0-communities.png └── data-diagram.drawio ├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE │ ├── feature-request.md │ └── report-issue.md ├── actions │ ├── yarn-install │ │ └── action.yaml │ ├── cdk-deploy │ │ └── action.yaml │ └── start-redis │ │ └── action.yaml └── workflows │ ├── pages-checks.yaml │ ├── aws-deploy-prod.yaml │ ├── aws-deploy-dev.yaml │ ├── crawler-checks.yaml │ └── publish-pages.yaml ├── .vscode ├── extensions.json └── settings.json ├── types ├── basic.ts └── output.ts └── CONTRIBUTING.md /pages/public/.gitignore: -------------------------------------------------------------------------------- 1 | data/ 2 | -------------------------------------------------------------------------------- /frontend/public/.gitignore: -------------------------------------------------------------------------------- 1 | data/ 2 | -------------------------------------------------------------------------------- /pages/.prettierignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | public/ -------------------------------------------------------------------------------- /frontend/.prettierignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | public/ -------------------------------------------------------------------------------- /pages/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | dist/ -------------------------------------------------------------------------------- /cdk/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | cdk.out/ 3 | config.json 4 | *.context.json 5 | -------------------------------------------------------------------------------- /crawler/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | .data/ 3 | logs/ 4 | .env 5 | coverage/ 6 | -------------------------------------------------------------------------------- /frontend/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | dist/ 3 | public/*.zip 4 | .env 5 | output/ 6 | coverage/ 7 | -------------------------------------------------------------------------------- /docs/images/0.10.0-communities.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tgxn/lemmy-explorer/HEAD/docs/images/0.10.0-communities.png -------------------------------------------------------------------------------- /docs/images/0.2.0-communities.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tgxn/lemmy-explorer/HEAD/docs/images/0.2.0-communities.png -------------------------------------------------------------------------------- /frontend/public/icons/lemmy_64px.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tgxn/lemmy-explorer/HEAD/frontend/public/icons/lemmy_64px.png -------------------------------------------------------------------------------- /frontend/public/icons/mbin_64px.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tgxn/lemmy-explorer/HEAD/frontend/public/icons/mbin_64px.png -------------------------------------------------------------------------------- /frontend/public/icons/piefed_64px.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tgxn/lemmy-explorer/HEAD/frontend/public/icons/piefed_64px.png -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [tgxn] 4 | custom: ["https://www.paypal.me/tgxn"] 5 | -------------------------------------------------------------------------------- /cdk/.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": false, 3 | "trailingComma": "all", 4 | "printWidth": 110, 5 | "tabWidth": 2 6 | } 7 | -------------------------------------------------------------------------------- /crawler/tsconfig.jest.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "module": "es2022" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /frontend/tsconfig.jest.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "module": "es2022" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /pages/.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": false, 3 | "trailingComma": "all", 4 | "printWidth": 110, 5 | "tabWidth": 2 6 | } 7 | -------------------------------------------------------------------------------- /crawler/.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": false, 3 | "trailingComma": "all", 4 | "printWidth": 110, 5 | "tabWidth": 2 6 | } 7 | -------------------------------------------------------------------------------- /frontend/.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": false, 3 | "trailingComma": "all", 4 | "printWidth": 110, 5 | "tabWidth": 2 6 | } 7 | -------------------------------------------------------------------------------- /cdk/config.example.json: -------------------------------------------------------------------------------- 1 | { 2 | "account": "ACCOUNT_ID", 3 | "environment": "ENVIRONMENT", 4 | "base_zone": "R53_DOMAIN", 5 | "domain": "SITE_(SUB)DOMAIN" 6 | } 7 | -------------------------------------------------------------------------------- /frontend/src/store.ts: -------------------------------------------------------------------------------- 1 | import { configureStore } from "@reduxjs/toolkit"; 2 | 3 | import combinedReducers from "./reducers"; 4 | 5 | export default configureStore({ 6 | reducer: combinedReducers, 7 | }); 8 | -------------------------------------------------------------------------------- /frontend/src/theme.ts: -------------------------------------------------------------------------------- 1 | import { extendTheme } from "@mui/joy/styles"; 2 | 3 | export default extendTheme({ 4 | fontFamily: { 5 | display: "'Inter', var(--joy-fontFamily-fallback)", 6 | body: "'Inter', var(--joy-fontFamily-fallback)", 7 | }, 8 | }); 9 | -------------------------------------------------------------------------------- /pages/README.md: -------------------------------------------------------------------------------- 1 | # Lemmyverse Data Site 2 | 3 | Hosted on Github Pages: https://data.lemmyverse.net/ 4 | 5 | **Please don't hotlink the files on the public website `https://lemmyverse.net/`** 6 | 7 | ## Info 8 | 9 | data is copied from frontend with 10 | `cp -r ./frontend/public/data/ ./pages/public/` 11 | -------------------------------------------------------------------------------- /frontend/src/reducers.ts: -------------------------------------------------------------------------------- 1 | import { combineReducers } from "redux"; 2 | 3 | import configReducer from "./reducers/configReducer"; 4 | import modalReducer from "./reducers/modalReducer"; 5 | 6 | const reducers = combineReducers({ 7 | configReducer, 8 | modalReducer, 9 | }); 10 | 11 | export default reducers; 12 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "ms-azuretools.vscode-docker", 4 | "ms-vscode.vscode-typescript-next", 5 | "ms-playwright.playwright", 6 | "esbenp.prettier-vscode", 7 | "christian-kohler.npm-intellisense", 8 | "eamodio.gitlens" 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /frontend/src/components/InstanceView/InstanceContentGrowth.tsx: -------------------------------------------------------------------------------- 1 | // import React from "react"; 2 | 3 | // import LineGraph from "../Shared/LineGraph"; 4 | 5 | // export default function InstanceContentGrowth({ userSeries }) { 6 | // console.log("userSeries", userSeries); 7 | 8 | // return ; 9 | // } 10 | -------------------------------------------------------------------------------- /crawler/docker-compose.github.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | redis: 3 | image: redis:6.2-alpine 4 | restart: always 5 | ports: 6 | - "127.0.0.1:6379:6379" 7 | volumes: 8 | - ../.redis:/data 9 | healthcheck: 10 | test: ["CMD", "redis-cli", "ping"] 11 | interval: 30s 12 | timeout: 10s 13 | retries: 3 14 | -------------------------------------------------------------------------------- /crawler/jest.config.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: "ts-jest", 3 | testEnvironment: "node", 4 | extensionsToTreatAsEsm: [".ts"], 5 | transform: { 6 | "^.+\\.ts[x]?$": [ 7 | "ts-jest", 8 | { 9 | useESM: true, 10 | tsconfig: "tsconfig.jest.json", 11 | }, 12 | ], 13 | }, 14 | testMatch: ["**/test/**/*.test.ts"], 15 | }; 16 | -------------------------------------------------------------------------------- /frontend/jest.config.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: "ts-jest", 3 | testEnvironment: "node", 4 | extensionsToTreatAsEsm: [".ts"], 5 | transform: { 6 | "^.+\\.ts[x]?$": [ 7 | "ts-jest", 8 | { 9 | useESM: true, 10 | tsconfig: "tsconfig.jest.json", 11 | }, 12 | ], 13 | }, 14 | testMatch: ["**/test/**/*.test.ts"], 15 | }; 16 | -------------------------------------------------------------------------------- /types/basic.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * A string representing a base URL for a fediverse instance/etc. 3 | * 4 | * Example: `lemmy.example.org` 5 | */ 6 | export type BaseURL = string; 7 | 8 | /** 9 | * A string representing a Lemmy actor ID. 10 | * 11 | * Example: `https://lemmy.example.org/c/example_community`, `https://lemmy.example.org/u/example_user` 12 | */ 13 | export type ActorID = string; 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Request 3 | about: Suggest a feature 4 | title: 'Feature:' 5 | labels: feature 6 | assignees: '' 7 | 8 | --- 9 | 10 | **What do you want to see?** 11 | A clear and concise description of what you would like to see implemented. 12 | 13 | **Additional context** 14 | Add any other context or screenshots about the feature request here. 15 | -------------------------------------------------------------------------------- /frontend/README.md: -------------------------------------------------------------------------------- 1 | # Lemmy Explorer Frontend 2 | 3 | This is a ReactJS SPA that uses the data from the crawler to display and search the instances and communities. 4 | 5 | Data is retrieved using TanStack Query. 6 | 7 | We're a 8 | 9 | ## Development 10 | 11 | 1. `yarn install` 12 | 2. `yarn start` 13 | 14 | > you can run `node index.js --out` in `crawler/` to update the data (it gets dumped into `frontend/dist/data/`) 15 | -------------------------------------------------------------------------------- /cdk/README.md: -------------------------------------------------------------------------------- 1 | # Lemmy Explorer Deployment (Amazon CDK v2) 2 | 3 | The deploy is an Amazon CDK v2 project that deploys the Lemmy Explorer frontend to AWS. 4 | 5 | `config.example.json` has the configuration for the deploy, rename to `config.json` and fill in the values. 6 | 7 | then run `cdk deploy --all` (or `yarn deploy`) to deploy the frontend to AWS. 8 | 9 | ## Deployment 10 | 11 | `yarn install` 12 | 13 | `yarn synth` 14 | 15 | `yarn deploy` 16 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.formatOnSave": true, 3 | "editor.formatOnPaste": false, 4 | "editor.tabSize": 2, 5 | "files.insertFinalNewline": true, 6 | "files.trimTrailingWhitespace": true, 7 | "git.autofetch": true, 8 | "editor.defaultFormatter": "esbenp.prettier-vscode", 9 | "[typescript]": { 10 | "editor.defaultFormatter": "esbenp.prettier-vscode" 11 | }, 12 | "[markdown]": { 13 | "editor.defaultFormatter": "esbenp.prettier-vscode" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /frontend/src/pages/Communities.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from "react"; 2 | 3 | import Container from "@mui/joy/Container"; 4 | 5 | import CommunityList from "../components/Communities"; 6 | 7 | export default function Communities() { 8 | return ( 9 | 16 | 17 | 18 | ); 19 | } 20 | -------------------------------------------------------------------------------- /frontend/src/components/Shared/InstanceTypeIcon.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import Avatar from "@mui/joy/Avatar"; 3 | 4 | const typeIcon: Record = { 5 | lemmy: "/icons/lemmy_64px.png", 6 | mbin: "/icons/mbin_64px.png", 7 | piefed: "/icons/piefed_64px.png", 8 | }; 9 | 10 | export default function InstanceTypeIcon({ type, size = 20 }: { type: string; size?: number }) { 11 | const src = typeIcon[type]; 12 | if (!src) return null; 13 | return ; 14 | } 15 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/report-issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Report Issue 3 | about: Something is broken! 4 | title: 'Bug:' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Additional context** 24 | Add any other context or screenshots about the issue here. 25 | -------------------------------------------------------------------------------- /frontend/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json.schemastore.org/tsconfig", 3 | "compilerOptions": { 4 | "module": "commonjs", 5 | "target": "es6", 6 | 7 | "jsx": "react", 8 | "sourceMap": true, 9 | 10 | "esModuleInterop": true, 11 | "noImplicitAny": false, 12 | 13 | "forceConsistentCasingInFileNames": true, 14 | 15 | "declaration": false, 16 | "declarationMap": false, 17 | 18 | "useUnknownInCatchVariables": false, 19 | "allowSyntheticDefaultImports": true, 20 | 21 | "skipLibCheck": true 22 | }, 23 | "allowJs": true, 24 | "checkJs": true 25 | } 26 | -------------------------------------------------------------------------------- /pages/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json.schemastore.org/tsconfig", 3 | "compilerOptions": { 4 | "module": "commonjs", 5 | "target": "es6", 6 | 7 | "jsx": "react", 8 | "sourceMap": true, 9 | 10 | "esModuleInterop": true, 11 | "noImplicitAny": false, 12 | 13 | "forceConsistentCasingInFileNames": true, 14 | 15 | "declaration": false, 16 | "declarationMap": false, 17 | 18 | "useUnknownInCatchVariables": false, 19 | "allowSyntheticDefaultImports": true, 20 | 21 | "skipLibCheck": true 22 | }, 23 | "allowJs": true, 24 | "checkJs": true 25 | } 26 | -------------------------------------------------------------------------------- /crawler/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json.schemastore.org/tsconfig", 3 | "compilerOptions": { 4 | "module": "es2022", 5 | "moduleResolution": "node", 6 | "target": "es2016", 7 | "esModuleInterop": true, 8 | "forceConsistentCasingInFileNames": true, 9 | "strict": true, 10 | "skipLibCheck": true, 11 | 12 | "declaration": false, 13 | "declarationMap": false, 14 | 15 | "noImplicitAny": false, 16 | 17 | "useUnknownInCatchVariables": false, 18 | "allowSyntheticDefaultImports": true 19 | }, 20 | "exclude": ["node_modules/"], 21 | "allowJs": true, 22 | "checkJs": true 23 | } 24 | -------------------------------------------------------------------------------- /frontend/test/config/global.setup.ts: -------------------------------------------------------------------------------- 1 | import type { FullConfig } from "@playwright/test"; 2 | 3 | import path from "node:path"; 4 | import fs from "node:fs"; 5 | 6 | async function globalSetup(config: FullConfig) { 7 | // delete previous Playwright outputs 8 | 9 | const folderPath = path.join("./output"); 10 | if (fs.existsSync(folderPath)) { 11 | fs.rm(folderPath, { recursive: true, force: true }, (error) => { 12 | if (error) { 13 | console.error(`Error deleting ${"./output"}`, error); 14 | } 15 | }); 16 | console.log(`Deleted folder: ${"./output"}`); 17 | } 18 | } 19 | 20 | export default globalSetup; 21 | -------------------------------------------------------------------------------- /frontend/src/hooks/useStorage.ts: -------------------------------------------------------------------------------- 1 | import { useState, Dispatch, SetStateAction } from "react"; 2 | import storage from "../lib/storage"; 3 | 4 | export default function useStorage(storageKey: string, defaultValue: T): [T, Dispatch>] { 5 | const [storageValue, _setStorageValue] = useState(storage.get(storageKey, defaultValue)); 6 | 7 | const setStorageValue: Dispatch> = (value) => { 8 | const newValue = value instanceof Function ? value(storageValue) : value; 9 | storage.set(storageKey, newValue); 10 | _setStorageValue(newValue); 11 | }; 12 | 13 | return [storageValue, setStorageValue]; 14 | } 15 | -------------------------------------------------------------------------------- /pages/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | Lemmy Explorer :: Data 11 | 12 | 13 | 14 | 15 |
16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /cdk/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lemmy-explorer-cdk", 3 | "version": "2.0.1", 4 | "license": "MIT", 5 | "bin": { 6 | "explorer": "bin/explorer.js" 7 | }, 8 | "scripts": { 9 | "bootstrap": "cdk bootstrap", 10 | "synth": "cdk synth --all", 11 | "diff": "cdk diff --all", 12 | "deploy": "cdk deploy --all", 13 | "deploy:ci": "cdk deploy --all --require-approval never" 14 | }, 15 | "dependencies": { 16 | "aws-cdk": "2.1000.3", 17 | "aws-cdk-lib": "^2.181.0", 18 | "constructs": "^10.2.35" 19 | }, 20 | "devDependencies": { 21 | "@types/node": "^22.13.5", 22 | "aws-cdk": "2.1000.3", 23 | "aws-cdk-lib": "^2.181.0", 24 | "typescript": "^5.7.3" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /frontend/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { createRoot } from "react-dom/client"; 3 | 4 | import { CssVarsProvider } from "@mui/joy/styles"; 5 | 6 | import CssBaseline from "@mui/joy/CssBaseline"; 7 | 8 | import customTheme from "./theme"; 9 | 10 | import App from "./App"; 11 | 12 | export default function Index() { 13 | return ( 14 | 15 | 16 | 17 | 18 | ); 19 | } 20 | 21 | const rootElement = document.getElementById("root"); 22 | const root = createRoot(rootElement); 23 | 24 | root.render( 25 | 26 | 27 | , 28 | ); 29 | -------------------------------------------------------------------------------- /crawler/.env.example: -------------------------------------------------------------------------------- 1 | # Example environment configuration for the crawler 2 | 3 | # Redis Connection String 4 | REDIS_URL=redis://localhost:6379 5 | 6 | # (Optional) Logging level (trace, debug, info, warn, error, fatal) 7 | # LOG_LEVEL=debug 8 | 9 | # (Optional) Automatically upload crawl output to S3 10 | # AUTO_UPLOAD_S3=false 11 | 12 | # (Optional) Redis dump and checkpoint locations 13 | # REDIS_DUMP_FILE=.data/redis/dump.rdb 14 | # CHECKPOINT_DIR=.data/checkpoint/ 15 | 16 | # (Optional) AWS credentials and target bucket 17 | # AWS_REGION=ap-southeast-2 18 | # AWS_ACCESS_KEY_ID= 19 | # AWS_SECRET_ACCESS_KEY= 20 | # PUBLISH_S3_BUCKET=lemmy-build-bucket 21 | 22 | # (Optional) Cron schedule for publishing to S3 23 | # PUBLISH_S3_CRON=0 */4 * * * 24 | -------------------------------------------------------------------------------- /frontend/test/spec/ui-interactions.spec.ts: -------------------------------------------------------------------------------- 1 | import { test, expect } from "@playwright/test"; 2 | import { setupGlobalHooks } from "../config/test.utils"; 3 | setupGlobalHooks(); 4 | 5 | // Test critical UI interactions such as toggling color scheme 6 | 7 | const getColorScheme = async (page) => { 8 | return await page.evaluate(() => document.documentElement.getAttribute("data-joy-color-scheme")); 9 | }; 10 | 11 | test("toggle color scheme", async ({ page }) => { 12 | await page.goto("/", { 13 | waitUntil: "networkidle", 14 | }); 15 | 16 | const initial = await getColorScheme(page); 17 | await page.locator("#toggle-mode").first().click(); 18 | const toggled = await getColorScheme(page); 19 | expect(toggled).not.toBe(initial); 20 | }); 21 | -------------------------------------------------------------------------------- /frontend/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | Lemmy Explorer 11 | 12 | 13 | 14 | 15 | 16 | 17 |
18 | 19 | 20 | -------------------------------------------------------------------------------- /crawler/src/lib/storage/uptime.ts: -------------------------------------------------------------------------------- 1 | import { CrawlStorage } from "../crawlStorage"; 2 | 3 | import { IUptimeNodeData, IFullUptimeData } from "../../../../types/storage"; 4 | 5 | export default class Uptime { 6 | private storage: CrawlStorage; 7 | 8 | constructor(storage: CrawlStorage) { 9 | this.storage = storage; 10 | } 11 | 12 | async getLatest(): Promise { 13 | // records have uptime:timestamp key, extract the latest one 14 | const keys = await this.storage.client.keys(`uptime:*`); 15 | const latestKey = keys.reduce((a, b) => (a > b ? a : b)); 16 | return this.storage.getRedis(latestKey); 17 | } 18 | 19 | async addNew(data: IFullUptimeData) { 20 | return this.storage.putRedis(`uptime:${Date.now()}`, data); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /crawler/index.ts: -------------------------------------------------------------------------------- 1 | // wrapper for cli 2 | 3 | import task from "./src/bin/task"; 4 | import worker from "./src/bin/worker"; 5 | import manual from "./src/bin/manual"; 6 | 7 | function start(args: string[]) { 8 | // run single tasks for --task `--aged` 9 | if (args.length === 1 && args[0].startsWith("--")) { 10 | task(args[0].substring(2)); 11 | } 12 | 13 | // start worker `-w instance`\`-w community`\`-w cron` 14 | else if (args.length === 2 && args[0] == "-w") { 15 | worker(args[1]); 16 | } 17 | 18 | // run manual processing job for one item `-m i ` / `-s single ` 19 | else if (args.length >= 3 && args[0] == "-m") { 20 | manual(args[1], args[2], args[3]); 21 | } 22 | } 23 | 24 | const args = process.argv.slice(2); 25 | start(args); 26 | -------------------------------------------------------------------------------- /crawler/src/queue/community_list.ts: -------------------------------------------------------------------------------- 1 | import BaseQueue, { ISuccessCallback } from "./BaseQueue"; 2 | 3 | import type { ICommunityData } from "../../../types/storage"; 4 | 5 | import { communityListProcessor } from "../crawl/community"; 6 | 7 | export default class CommunityListQueue extends BaseQueue { 8 | constructor(isWorker = false, queueName = "community") { 9 | super(isWorker, queueName, communityListProcessor); 10 | } 11 | 12 | async createJob(instanceBaseUrl: string, onSuccess: ISuccessCallback = null) { 13 | const trimmedUrl = instanceBaseUrl.trim(); 14 | 15 | return await super.createJob( 16 | trimmedUrl, 17 | { 18 | baseUrl: trimmedUrl, 19 | }, 20 | onSuccess, 21 | ); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /crawler/src/lib/storage/fediseer.ts: -------------------------------------------------------------------------------- 1 | import { CrawlStorage } from "../crawlStorage"; 2 | 3 | import { IFediseerInstanceDataTagsObject } from "../../../../types/storage"; 4 | 5 | export default class Fediseer { 6 | private storage: CrawlStorage; 7 | 8 | constructor(storage: CrawlStorage) { 9 | this.storage = storage; 10 | } 11 | 12 | async getLatest(): Promise { 13 | // records have uptime:timestamp key, extract the latest one 14 | const keys = await this.storage.client.keys(`fediseer:*`); 15 | const latestKey = keys.reduce((a, b) => (a > b ? a : b)); 16 | return this.storage.getRedis(latestKey); 17 | } 18 | 19 | async addNew(data: IFediseerInstanceDataTagsObject[]) { 20 | return this.storage.putRedis(`fediseer:${Date.now()}`, data); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /crawler/src/queue/mbin.ts: -------------------------------------------------------------------------------- 1 | import logging from "../lib/logging"; 2 | import storage from "../lib/crawlStorage"; 3 | 4 | import type { IIncomingMagazineData } from "../crawl/mbin"; 5 | 6 | import BaseQueue, { ISuccessCallback } from "./BaseQueue"; 7 | 8 | import { mbinInstanceProcessor } from "../crawl/mbin"; 9 | 10 | export default class MBinQueue extends BaseQueue { 11 | constructor(isWorker = false, queueName = "mbin") { 12 | super(isWorker, queueName, mbinInstanceProcessor); 13 | } 14 | 15 | // use as MBinQueue.createJob({ baseUrl: "https://fedia.io" }); 16 | async createJob( 17 | baseUrl: string, 18 | onSuccess: ISuccessCallback | null = null, 19 | ) { 20 | await super.createJob(baseUrl, { baseUrl }, onSuccess); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /crawler/test/output/output-classifier.test.ts: -------------------------------------------------------------------------------- 1 | import { jest } from "@jest/globals"; 2 | 3 | import OutputClassifier from "../../src/output/classifier"; 4 | 5 | describe("OutputClassifier.findErrorType", () => { 6 | const cases = [ 7 | ["ENOENT something", "contentMissing"], 8 | ["timeout of 100ms", "timeout"], 9 | ["self-signed certificate", "sslException"], 10 | ["baseUrl is not a valid domain", "invalidBaseUrl"], 11 | ["code 404", "httpException"], 12 | ["no diaspora rel in", "httpException"], 13 | ["not a lemmy instance", "notLemmy"], 14 | ["invalid actor id", "invalidActorId"], 15 | ["random message", "unknown"], 16 | ] as const; 17 | 18 | test.each(cases)("%s => %s", (msg, expected) => { 19 | expect(OutputClassifier.findErrorType(msg)).toBe(expected); 20 | }); 21 | }); 22 | -------------------------------------------------------------------------------- /crawler/src/lib/storage/fediverse.ts: -------------------------------------------------------------------------------- 1 | import { CrawlStorage } from "../crawlStorage"; 2 | 3 | import { IFediverseData, IFediverseDataKeyValue } from "../../../../types/storage"; 4 | 5 | export default class Fediverse { 6 | private storage: CrawlStorage; 7 | 8 | constructor(storage: CrawlStorage) { 9 | this.storage = storage; 10 | } 11 | 12 | async getAll(): Promise { 13 | return this.storage.listRedisWithKeys(`fediverse:*`); 14 | } 15 | 16 | async getOne(baseUrl: string): Promise { 17 | return this.storage.getRedis(`fediverse:${baseUrl}`); 18 | } 19 | 20 | async upsert(baseUrl: string, data: IFediverseData) { 21 | const dd = { baseurl: baseUrl, time: Date.now(), ...data }; 22 | return this.storage.putRedis(`fediverse:${baseUrl}`, dd); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Firstly, Thankyou for considering contributing to the project! 4 | 5 | ## Developing 6 | 7 | Check the readme in each module for further dev instructions: 8 | [crawler/](crawler/README.md) 9 | [frontend/](frontend/README.md) 10 | [pages/](pages/README.md) 11 | 12 | ## Merge Requests 13 | 14 | Please create your merge requests against the `develop` branch, which once merged will deploy changes for `develop.lemmyverse.net` before merging with `main`. 15 | 16 | ## Code Style 17 | 18 | All modules of the project use [Prettier](https://prettier.io/) to do code formatting, and there is a `.prettierrc.json` file in most modules which configure their coding style. 19 | 20 | Please run `yarn format:check` in whatever module you're working on before committing :) (or better yet, set up format on save!) 21 | -------------------------------------------------------------------------------- /pages/webpack.dev.config.js: -------------------------------------------------------------------------------- 1 | const webpack = require("webpack"); 2 | const { merge } = require("webpack-merge"); 3 | 4 | const common = require("./webpack.common.js"); 5 | 6 | module.exports = merge(common, { 7 | mode: "development", 8 | devtool: "inline-source-map", 9 | devServer: { 10 | host: "0.0.0.0", 11 | port: 9192, 12 | client: { 13 | overlay: true, 14 | }, 15 | hot: true, 16 | // hotOnly: true, 17 | liveReload: true, 18 | // watchContentBase: true, 19 | historyApiFallback: { index: "/", disableDotRule: true }, 20 | }, 21 | optimization: { 22 | // Instruct webpack not to obfuscate the resulting code 23 | minimize: false, 24 | splitChunks: false, 25 | }, 26 | plugins: [ 27 | new webpack.EnvironmentPlugin({ 28 | NODE_ENV: "development", 29 | }), 30 | ], 31 | }); 32 | -------------------------------------------------------------------------------- /frontend/test/config/test.utils.ts: -------------------------------------------------------------------------------- 1 | import { test } from "@playwright/test"; 2 | 3 | export function setupGlobalHooks() { 4 | // Save a screenshot after each test in a flat directory with a stable name. 5 | // Screenshots contain the test name and are saved in the `output/screens` directory. 6 | test.afterEach(async ({ page }, testInfo) => { 7 | const testName = testInfo.title.replace(/[^a-zA-Z0-9]/g, "_").toLowerCase(); 8 | const path = `output/screens/${testName}.png`; 9 | 10 | // to ensure the page is fully rendered 11 | await Promise.all([ 12 | page.waitForLoadState("networkidle"), 13 | page.waitForFunction(() => document.readyState === "complete"), 14 | ]); 15 | await page.waitForTimeout(2000); 16 | 17 | await page.screenshot({ path }); 18 | console.log("📸 Screenshot saved:", path); 19 | }); 20 | } 21 | -------------------------------------------------------------------------------- /pages/src/hooks/useQueryCache.ts: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | import { useQuery } from "@tanstack/react-query"; 3 | 4 | export default function useQueryCache(queryKey, dataFile) { 5 | const { isSuccess, isLoading, isError, error, data, isFetching } = useQuery({ 6 | queryKey: ["cache", queryKey], // single string key 7 | queryFn: () => 8 | axios 9 | .get(`/data/${dataFile}.json`, { 10 | timeout: 15000, 11 | }) 12 | .then((res) => { 13 | console.log(res.data); 14 | return res.data; 15 | }), 16 | retry: 1, 17 | refetchOnWindowFocus: false, 18 | refetchOnMount: false, 19 | staleTime: Infinity, 20 | // cacheTime: Infinity, 21 | }); 22 | 23 | return { 24 | isLoading, 25 | isSuccess, 26 | isError, 27 | error, 28 | data, 29 | }; 30 | } 31 | -------------------------------------------------------------------------------- /cdk/lib/cert-stack.ts: -------------------------------------------------------------------------------- 1 | import * as acm from "aws-cdk-lib/aws-certificatemanager"; 2 | import * as route53 from "aws-cdk-lib/aws-route53"; 3 | 4 | import { Stack, StackProps } from "aws-cdk-lib"; 5 | import { Construct } from "constructs"; 6 | 7 | import config from "../config.json"; 8 | 9 | export class CertStack extends Stack { 10 | public cert: acm.Certificate; 11 | 12 | constructor(scope: Construct, id: string, props?: StackProps) { 13 | super(scope, id, props); 14 | 15 | // get existing hosted zone and add cname 16 | const myHostedZone = route53.HostedZone.fromLookup(this, "HostedZone", { 17 | domainName: config.base_zone, 18 | }); 19 | 20 | this.cert = new acm.Certificate(this, "Certificate", { 21 | domainName: config.domain, 22 | validation: acm.CertificateValidation.fromDns(myHostedZone), 23 | }); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /crawler/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:22.17-alpine 2 | 3 | RUN apk add --no-cache bash 4 | 5 | # Install Dependencies 6 | RUN yarn global add tsx pm2 7 | 8 | # Create app directory 9 | RUN mkdir -p /usr/src/app 10 | RUN mkdir -p /data/output 11 | RUN mkdir -p /data/checkpoint 12 | 13 | WORKDIR /usr/src/app 14 | 15 | # Install App 16 | COPY index.ts . 17 | COPY package.json . 18 | COPY yarn.lock . 19 | COPY ecosystem.config.cjs . 20 | COPY src src/ 21 | 22 | # Own the app directory and switch to the crawl user 23 | RUN chown -R 1000:1000 /usr/src/app 24 | RUN chown -R 1000:1000 /data/output 25 | RUN chown -R 1000:1000 /data/checkpoint 26 | 27 | # Switch to a non-root user 28 | USER 1000:1000 29 | 30 | RUN yarn install 31 | 32 | # logrotate 33 | RUN pm2 install pm2-logrotate 34 | RUN pm2 set pm2-logrotate:retain 7 35 | 36 | # Run App 37 | CMD [ "pm2-runtime", "--raw", "start", "ecosystem.config.cjs" ] 38 | -------------------------------------------------------------------------------- /crawler/src/queue/community_single.ts: -------------------------------------------------------------------------------- 1 | import BaseQueue, { ISuccessCallback } from "./BaseQueue"; 2 | 3 | import type { ICommunityData } from "../../../types/storage"; 4 | 5 | import { singleCommunityProcessor } from "../crawl/community"; 6 | 7 | export default class SingleCommunityQueue extends BaseQueue { 8 | constructor(isWorker = false, queueName = "one_community") { 9 | super(isWorker, queueName, singleCommunityProcessor); 10 | } 11 | 12 | async createJob( 13 | baseUrl: string, 14 | communityName: string, 15 | onSuccess: ISuccessCallback = null, 16 | ) { 17 | const trimmedUrl = baseUrl.trim(); 18 | 19 | return await super.createJob( 20 | trimmedUrl + ":" + communityName, 21 | { 22 | baseUrl: trimmedUrl, 23 | community: communityName, 24 | }, 25 | onSuccess, 26 | ); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /frontend/test/spec/query-params.spec.ts: -------------------------------------------------------------------------------- 1 | import { test, expect } from "@playwright/test"; 2 | import { setupGlobalHooks } from "../config/test.utils"; 3 | setupGlobalHooks(); 4 | 5 | test("should remain unchanged on initial load with params", async ({ page }) => { 6 | const path = "/?query=foo&order=users&open=true"; 7 | 8 | await page.goto(path, { 9 | waitUntil: "networkidle", 10 | }); 11 | 12 | // wait for potential effect that updates search params 13 | await page.waitForTimeout(600); 14 | 15 | const url = new URL(page.url()); 16 | expect(url.pathname + url.search).toBe(path); 17 | }); 18 | 19 | test("should not add params when none are provided", async ({ page }) => { 20 | await page.goto("/", { 21 | waitUntil: "networkidle", 22 | }); 23 | 24 | await page.waitForTimeout(600); 25 | const url = new URL(page.url()); 26 | expect(url.pathname + url.search).toBe("/"); 27 | }); 28 | -------------------------------------------------------------------------------- /crawler/src/queue/piefed.ts: -------------------------------------------------------------------------------- 1 | import logging from "../lib/logging"; 2 | import storage from "../lib/crawlStorage"; 3 | 4 | import { CrawlTooRecentError } from "../lib/error"; 5 | 6 | import BaseQueue, { ISuccessCallback } from "./BaseQueue"; 7 | import { IIncomingPiefedCommunityData } from "../crawl/piefed"; 8 | 9 | import { piefedInstanceProcessor } from "../crawl/piefed"; 10 | 11 | export default class PiefedQueue extends BaseQueue { 12 | constructor(isWorker = false, queueName = "piefed") { 13 | super(isWorker, queueName, piefedInstanceProcessor); 14 | } 15 | 16 | // use as PiefedQueue.createJob({ baseUrl: "https://piefed.social" }); 17 | async createJob( 18 | baseUrl: string, 19 | onSuccess: ISuccessCallback | null = null, 20 | ) { 21 | await super.createJob(baseUrl, { baseUrl }, onSuccess); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /pages/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | import ReactDOM from "react-dom/client"; 4 | 5 | import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; 6 | import { ReactQueryDevtools } from "@tanstack/react-query-devtools"; 7 | 8 | import { CssVarsProvider } from "@mui/joy/styles"; 9 | 10 | import CssBaseline from "@mui/joy/CssBaseline"; 11 | 12 | import App from "./App"; 13 | 14 | const queryClient = new QueryClient(); 15 | export default function Index() { 16 | return ( 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | ); 25 | } 26 | 27 | const rootElement = document.getElementById("root"); 28 | ReactDOM.createRoot(rootElement).render(); 29 | -------------------------------------------------------------------------------- /crawler/test/storage/mbin.test.ts: -------------------------------------------------------------------------------- 1 | import { jest } from "@jest/globals"; 2 | 3 | import MBinStore from "../../src/lib/storage/mbin"; 4 | 5 | const listRedisWithKeysMock = jest.fn(() => 6 | Promise.resolve({ 7 | "mbin_magazine:example.com:foo": { name: "foo" }, 8 | }), 9 | ); 10 | 11 | const storageMock = { 12 | listRedisWithKeys: listRedisWithKeysMock, 13 | }; 14 | 15 | const store = new MBinStore(storageMock as any); 16 | 17 | describe("MBinStore", () => { 18 | beforeEach(() => { 19 | jest.clearAllMocks(); 20 | listRedisWithKeysMock.mockResolvedValue({ 21 | "mbin_magazine:example.com:foo": { name: "foo" }, 22 | }); 23 | }); 24 | 25 | test("getAll adds baseurl to records", async () => { 26 | const result = await store.getAll(); 27 | expect(listRedisWithKeysMock).toHaveBeenCalledWith("mbin_magazine:*"); 28 | expect(result).toEqual([{ name: "foo", baseurl: "example.com" }]); 29 | }); 30 | }); 31 | -------------------------------------------------------------------------------- /frontend/src/reducers/modalReducer.ts: -------------------------------------------------------------------------------- 1 | export function showInstanceModal(instanceData) { 2 | return { 3 | type: "showInstanceModal", 4 | payload: { instanceData }, 5 | }; 6 | } 7 | export function hideInstanceModal() { 8 | return { 9 | type: "hideInstanceModal", 10 | payload: {}, 11 | }; 12 | } 13 | 14 | const initialState = { 15 | instanceModalOpen: false, 16 | instanceData: {}, 17 | }; 18 | 19 | const modalReducer = (state = initialState, action: any = {}) => { 20 | switch (action.type) { 21 | case "showInstanceModal": 22 | return { 23 | ...state, 24 | instanceModalOpen: true, 25 | instanceData: action.payload.instanceData, 26 | }; 27 | 28 | case "hideInstanceModal": 29 | return { 30 | ...state, 31 | instanceModalOpen: false, 32 | instanceData: {}, 33 | }; 34 | 35 | default: 36 | return state; 37 | } 38 | }; 39 | 40 | export default modalReducer; 41 | -------------------------------------------------------------------------------- /crawler/src/queue/instance.ts: -------------------------------------------------------------------------------- 1 | import BaseQueue, { ISuccessCallback } from "./BaseQueue"; 2 | 3 | import type { IInstanceData } from "../../../types/storage"; 4 | 5 | import { instanceProcessor } from "../crawl/instance"; 6 | import logging from "../lib/logging"; 7 | 8 | export default class InstanceQueue extends BaseQueue { 9 | constructor(isWorker = false, queueName = "instance") { 10 | super(isWorker, queueName, instanceProcessor); 11 | } 12 | 13 | async createJob(instanceBaseUrl: string, onSuccess: ISuccessCallback | null = null) { 14 | // replace http/s with nothing 15 | let trimmedUrl = instanceBaseUrl.replace(/^https?:\/\//, "").trim(); 16 | 17 | // dont create blank jobs 18 | if (trimmedUrl == "") { 19 | logging.warn("createJob: trimmedUrl is blank", instanceBaseUrl); 20 | return; 21 | } 22 | 23 | await super.createJob(trimmedUrl, { baseUrl: trimmedUrl }, onSuccess); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /cdk/bin/explorer.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import * as cdk from "aws-cdk-lib"; 4 | 5 | import { CertStack } from "../lib/cert-stack"; 6 | import { FrontendStack } from "../lib/frontend-stack"; 7 | import { BuildStack } from "../lib/build-stack"; 8 | 9 | import config from "../config.json"; 10 | 11 | const app = new cdk.App(); 12 | 13 | const certStack = new CertStack(app, `LemmyExplorer-Cert-${config.environment}`, { 14 | env: { region: "us-east-1", account: config.account }, 15 | crossRegionReferences: true, 16 | }); 17 | 18 | const buildStack = new BuildStack(app, `LemmyExplorer-Build-${config.environment}`, { 19 | env: { region: "ap-southeast-2", account: config.account }, 20 | }); 21 | 22 | const frontendStack = new FrontendStack(app, `LemmyExplorer-Frontend-${config.environment}`, { 23 | env: { region: "ap-southeast-2", account: config.account }, 24 | cert: certStack.cert, 25 | crossRegionReferences: true, 26 | }); 27 | frontendStack.addDependency(certStack); 28 | -------------------------------------------------------------------------------- /cdk/cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node --prefer-ts-exts bin/explorer.ts", 3 | "watch": { 4 | "include": ["**"], 5 | "exclude": [ 6 | "README.md", 7 | "cdk*.json", 8 | "**/*.d.ts", 9 | "**/*.js", 10 | "tsconfig.json", 11 | "package*.json", 12 | "yarn.lock", 13 | "node_modules", 14 | "test" 15 | ] 16 | }, 17 | "context": { 18 | "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true, 19 | "@aws-cdk/core:stackRelativeExports": true, 20 | "@aws-cdk/aws-rds:lowercaseDbIdentifier": true, 21 | "@aws-cdk/aws-lambda:recognizeVersionProps": true, 22 | "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true, 23 | "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, 24 | "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, 25 | "@aws-cdk/core:target-partitions": ["aws", "aws-cn"], 26 | "@aws-cdk/customresources:installLatestAwsSdkDefault": false 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /cdk/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | // Settings for Node.js >= 12.9.0 4 | "lib": ["es2020"], 5 | "module": "commonjs", 6 | "target": "es2019", 7 | 8 | // Support some older packages 9 | "esModuleInterop": true, 10 | 11 | // Enable super strict setting 12 | "strict": true, 13 | 14 | "resolveJsonModule": true, 15 | 16 | "declaration": true, 17 | 18 | "noImplicitAny": true, 19 | "strictNullChecks": true, 20 | "noImplicitThis": true, 21 | "alwaysStrict": true, 22 | "noUnusedLocals": false, 23 | "noUnusedParameters": false, 24 | "noImplicitReturns": true, 25 | "noFallthroughCasesInSwitch": false, 26 | "experimentalDecorators": true, 27 | "strictPropertyInitialization": false, 28 | 29 | // Function code never has to be compiled outside zeit/ncc and ts-jest 30 | "rootDir": ".", 31 | "noEmit": true 32 | }, 33 | "include": ["**/*.ts"], 34 | "exclude": ["cdk.out", "node_modules"] 35 | } 36 | -------------------------------------------------------------------------------- /.github/actions/yarn-install/action.yaml: -------------------------------------------------------------------------------- 1 | name: Cache and Install Yarn 2 | description: Cache node_modules and install via yarn 3 | 4 | inputs: 5 | working-directory: 6 | description: Base path for the package.json and yarn.lock files 7 | required: true 8 | 9 | runs: 10 | using: "composite" 11 | steps: 12 | # Cache Dependencies 13 | - name: Cache Node Modules (${{ inputs.working-directory }}) 14 | id: cache-yarn 15 | uses: actions/cache@v4 16 | env: 17 | cache-name: cache-crawler-yarn 18 | with: 19 | path: ${{ inputs.working-directory }}/node_modules/ 20 | key: yarn-cache-${{ hashFiles(format('{0}/yarn.lock', inputs.working-directory)) }} 21 | 22 | # Install Dependencies 23 | - name: Install Node Modules (${{ inputs.working-directory }}) 24 | if: steps.cache-yarn.outputs.cache-hit != 'true' 25 | shell: bash 26 | run: yarn --frozen-lockfile 27 | working-directory: ${{ inputs.working-directory }} 28 | -------------------------------------------------------------------------------- /frontend/src/hooks/useQueryCache.ts: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | 3 | import { useQuery } from "@tanstack/react-query"; 4 | 5 | export type IQueryCache = { 6 | isLoading: boolean; 7 | isSuccess: boolean; 8 | isError: boolean; 9 | error: any; 10 | data: any; 11 | }; 12 | 13 | export default function useQueryCache(queryKey: string, dataFile: string): IQueryCache { 14 | const { isSuccess, isLoading, isError, error, data, isFetching } = useQuery({ 15 | queryKey: ["cache", queryKey], // single string key 16 | queryFn: () => 17 | axios 18 | .get(`/data/${dataFile}.json`, { 19 | timeout: 15000, 20 | }) 21 | .then((res) => { 22 | console.log(res.data); 23 | return res.data; 24 | }), 25 | retry: 1, 26 | refetchOnWindowFocus: false, 27 | refetchOnMount: false, 28 | staleTime: Infinity, 29 | cacheTime: Infinity, 30 | }); 31 | 32 | return { 33 | isLoading, 34 | isSuccess, 35 | isError, 36 | error, 37 | data, 38 | }; 39 | } 40 | -------------------------------------------------------------------------------- /pages/webpack.common.js: -------------------------------------------------------------------------------- 1 | const path = require("path"); 2 | 3 | const HtmlWebpackPlugin = require("html-webpack-plugin"); 4 | const CopyWebpackPlugin = require("copy-webpack-plugin"); 5 | 6 | module.exports = { 7 | entry: "./src/index.tsx", 8 | output: { 9 | path: path.join(__dirname, "dist"), 10 | publicPath: "/", 11 | clean: true, 12 | }, 13 | resolve: { 14 | extensions: [".tsx", ".jsx", ".ts", ".js"], 15 | }, 16 | plugins: [ 17 | new HtmlWebpackPlugin({ 18 | title: "Lemmy Explorer - Data Access", 19 | template: "index.html", 20 | }), 21 | new CopyWebpackPlugin({ 22 | patterns: [{ from: "public", to: "" }], 23 | }), 24 | ], 25 | module: { 26 | rules: [ 27 | { 28 | test: /\.tsx?$/, 29 | exclude: /(node_modules)/, 30 | use: { 31 | loader: "babel-loader", 32 | options: { 33 | presets: ["@babel/preset-env", "@babel/preset-react", "@babel/preset-typescript"], 34 | }, 35 | }, 36 | }, 37 | ], 38 | }, 39 | }; 40 | -------------------------------------------------------------------------------- /crawler/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | networks: 2 | crawler: 3 | name: crawler 4 | 5 | services: 6 | redis: 7 | user: 1000:1000 8 | image: redis:6.2-alpine 9 | restart: always 10 | networks: 11 | - crawler 12 | ports: 13 | - "6379:6379" 14 | volumes: 15 | - ./.data/redis:/data 16 | 17 | crawler: 18 | user: 1000:1000 19 | hostname: crawler 20 | build: . 21 | restart: always 22 | depends_on: 23 | - redis 24 | networks: 25 | - crawler 26 | volumes: 27 | - ./.data/logs:/usr/src/app/.data/logs:rw 28 | - ./.data/redis:/redis:ro 29 | environment: 30 | REDIS_URL: redis://redis:6379 31 | 32 | # s3 upload 33 | AUTO_UPLOAD_S3: true 34 | 35 | REDIS_DUMP_FILE: /redis/dump.rdb 36 | CHECKPOINT_DIR: /data/checkpoint 37 | OUTPUT_DIR: /data/output/docker 38 | 39 | AWS_REGION: ${AWS_REGION} 40 | AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} 41 | AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} 42 | PUBLISH_S3_BUCKET: ${PUBLISH_S3_BUCKET} 43 | -------------------------------------------------------------------------------- /frontend/src/components/GridView/Instance.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | import { useMasonry, usePositioner, useContainerPosition, useScroller } from "masonic"; 4 | 5 | import { useWindowSize } from "@react-hook/window-size"; 6 | 7 | import InstanceCard from "./InstanceCard"; 8 | 9 | function InstanceGrid({ items }) { 10 | const containerRef = React.useRef(null); 11 | 12 | const [windowWidth, height] = useWindowSize(); 13 | const { offset, width } = useContainerPosition(containerRef, [windowWidth, height]); 14 | 15 | const positioner = usePositioner({ width, columnGutter: 16, maxColumnCount: 6, columnWidth: 280 }, [items]); 16 | const { scrollTop, isScrolling } = useScroller(offset); 17 | 18 | const CardAsCallback = React.useCallback((props) => , []); 19 | 20 | return useMasonry({ 21 | containerRef, 22 | positioner, 23 | scrollTop, 24 | isScrolling, 25 | height, 26 | items, 27 | overscanBy: 4, 28 | render: CardAsCallback, 29 | }); 30 | } 31 | 32 | export default React.memo(InstanceGrid); 33 | -------------------------------------------------------------------------------- /frontend/webpack.dev.config.js: -------------------------------------------------------------------------------- 1 | const webpack = require("webpack"); 2 | const { merge } = require("webpack-merge"); 3 | 4 | const common = require("./webpack.common.js"); 5 | 6 | module.exports = merge(common, { 7 | mode: "development", 8 | devtool: "inline-source-map", 9 | devServer: { 10 | host: "0.0.0.0", 11 | port: 9191, 12 | client: { 13 | overlay: true, 14 | }, 15 | static: { 16 | watch: false, 17 | }, 18 | hot: true, 19 | liveReload: true, 20 | historyApiFallback: { index: "/", disableDotRule: true }, 21 | }, 22 | optimization: { 23 | // Instruct webpack not to obfuscate the resulting code 24 | minimize: false, 25 | // splitChunks: { 26 | // chunks: "all", 27 | // cacheGroups: { 28 | // vendors: { 29 | // test: /[\\/]node_modules[\\/]/, 30 | // name: "vendors", 31 | // chunks: "all", 32 | // }, 33 | // }, 34 | // }, 35 | }, 36 | plugins: [ 37 | new webpack.EnvironmentPlugin({ 38 | NODE_ENV: "development", 39 | }), 40 | ], 41 | }); 42 | -------------------------------------------------------------------------------- /cdk/lib/build-stack.ts: -------------------------------------------------------------------------------- 1 | import * as s3 from "aws-cdk-lib/aws-s3"; 2 | 3 | import { Stack, StackProps } from "aws-cdk-lib"; 4 | import { Construct } from "constructs"; 5 | 6 | import { AnyPrincipal, Effect, PolicyStatement } from "aws-cdk-lib/aws-iam"; 7 | 8 | export class BuildStack extends Stack { 9 | constructor(scope: Construct, id: string, props?: StackProps) { 10 | super(scope, id, props); 11 | 12 | // Build Bucket 13 | const buildBucket = new s3.Bucket(this, "BuildBucket", { 14 | publicReadAccess: false, 15 | encryption: s3.BucketEncryption.S3_MANAGED, 16 | blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, 17 | }); 18 | 19 | // Policy to deny access to the bucket via unencrypted connections 20 | buildBucket.addToResourcePolicy( 21 | new PolicyStatement({ 22 | effect: Effect.DENY, 23 | principals: [new AnyPrincipal()], 24 | actions: ["s3:*"], 25 | resources: [buildBucket.bucketArn], 26 | conditions: { 27 | Bool: { "aws:SecureTransport": "false" }, 28 | }, 29 | }), 30 | ); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /crawler/src/crawl/uptime.ts: -------------------------------------------------------------------------------- 1 | /** Uptime Crawler 2 | * 3 | * Meant to run max 1/day and get all uptime from api.fediverse.observer 4 | */ 5 | import logging from "../lib/logging"; 6 | 7 | import storage from "../lib/crawlStorage"; 8 | 9 | import CrawlClient from "../lib/CrawlClient"; 10 | 11 | export default class CrawlUptime { 12 | private client: CrawlClient; 13 | 14 | constructor() { 15 | this.client = new CrawlClient(); 16 | } 17 | 18 | async crawl() { 19 | const instances = await this.client.postUrl("https://api.fediverse.observer/", { 20 | query: `query{ 21 | nodes (softwarename: "lemmy") { 22 | domain 23 | latency 24 | countryname 25 | uptime_alltime 26 | date_created 27 | date_updated 28 | date_laststats 29 | score 30 | status 31 | } 32 | }`, 33 | }); 34 | logging.info(instances.data); 35 | 36 | await storage.uptime.addNew({ 37 | timestamp: Date.now(), 38 | nodes: instances.data.data.nodes, 39 | }); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /frontend/src/components/GridView/MBin.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | import { useMasonry, usePositioner, useContainerPosition, useScroller } from "masonic"; 4 | 5 | import { useWindowSize } from "@react-hook/window-size"; 6 | 7 | import NMBinCard from "./MBinCard"; 8 | 9 | type MBinGridProps = { 10 | items: any[]; 11 | }; 12 | 13 | const MBinGrid = React.memo(function ({ items }: MBinGridProps) { 14 | const containerRef = React.useRef(null); 15 | 16 | const [windowWidth, height] = useWindowSize(); 17 | const { offset, width } = useContainerPosition(containerRef, [windowWidth, height]); 18 | 19 | const positioner = usePositioner({ width, columnGutter: 16, maxColumnCount: 6, columnWidth: 280 }, [items]); 20 | 21 | const { scrollTop, isScrolling } = useScroller(offset); 22 | 23 | const CardAsCallback = React.useCallback((props) => , [isScrolling]); 24 | 25 | return useMasonry({ 26 | containerRef, 27 | positioner, 28 | scrollTop, 29 | isScrolling, 30 | height, 31 | items, 32 | overscanBy: 6, 33 | render: CardAsCallback, 34 | }); 35 | }); 36 | export default MBinGrid; 37 | -------------------------------------------------------------------------------- /frontend/src/components/GridView/Piefed.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | import { useMasonry, usePositioner, useContainerPosition, useScroller } from "masonic"; 4 | 5 | import { useWindowSize } from "@react-hook/window-size"; 6 | 7 | import PiefedCard from "./PiefedCard"; 8 | 9 | type PiefedGridProps = { 10 | items: any[]; 11 | }; 12 | 13 | const PiefedGrid = React.memo(function ({ items }: PiefedGridProps) { 14 | const containerRef = React.useRef(null); 15 | 16 | const [windowWidth, height] = useWindowSize(); 17 | const { offset, width } = useContainerPosition(containerRef, [windowWidth, height]); 18 | 19 | const positioner = usePositioner({ width, columnGutter: 16, maxColumnCount: 6, columnWidth: 280 }, [items]); 20 | 21 | const { scrollTop, isScrolling } = useScroller(offset); 22 | 23 | const CardAsCallback = React.useCallback((props) => , [isScrolling]); 24 | 25 | return useMasonry({ 26 | containerRef, 27 | positioner, 28 | scrollTop, 29 | isScrolling, 30 | height, 31 | items, 32 | overscanBy: 6, 33 | render: CardAsCallback, 34 | }); 35 | }); 36 | export default PiefedGrid; 37 | -------------------------------------------------------------------------------- /frontend/src/components/GridView/Community.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | import { useMasonry, usePositioner, useContainerPosition, useScroller } from "masonic"; 4 | 5 | import { useWindowSize } from "@react-hook/window-size"; 6 | 7 | import CommunityCard from "./CommunityCard"; 8 | 9 | type ICommunityGridProps = { 10 | items: any[]; 11 | }; 12 | 13 | function CommunityGrid({ items }: ICommunityGridProps) { 14 | const containerRef = React.useRef(null); 15 | 16 | const [windowWidth, height] = useWindowSize(); 17 | const { offset, width } = useContainerPosition(containerRef, [windowWidth, height]); 18 | 19 | const positioner = usePositioner({ width, columnGutter: 16, maxColumnCount: 6, columnWidth: 280 }, [items]); 20 | 21 | const { scrollTop, isScrolling } = useScroller(offset); 22 | 23 | const CardAsCallback = React.useCallback((props) => , []); 24 | 25 | return useMasonry({ 26 | containerRef, 27 | positioner, 28 | scrollTop, 29 | isScrolling, 30 | height, 31 | items, 32 | overscanBy: 6, 33 | render: CardAsCallback, 34 | }); 35 | } 36 | 37 | export default React.memo(CommunityGrid); 38 | -------------------------------------------------------------------------------- /crawler/test/lib/crawlClient.test.ts: -------------------------------------------------------------------------------- 1 | import { jest } from "@jest/globals"; 2 | 3 | import CrawlClient from "../../src/lib/CrawlClient"; 4 | 5 | import type { AxiosStatic, AxiosInstance } from "axios"; 6 | 7 | const mockedAxiosInstance = { 8 | get: jest.fn(), 9 | post: jest.fn(), 10 | } as unknown as jest.Mocked; 11 | 12 | // Mock axios module 13 | jest.mock("axios", () => { 14 | const actualAxios = jest.requireActual("axios") as AxiosStatic; 15 | return { 16 | __esModule: true, 17 | default: { 18 | ...actualAxios, 19 | create: jest.fn(() => mockedAxiosInstance), 20 | }, 21 | }; 22 | }); 23 | 24 | describe("CrawlClient.getUrlWithRetry", () => { 25 | jest.setTimeout(30000); 26 | 27 | test("retries the expected number of times", async () => { 28 | mockedAxiosInstance.get.mockRejectedValue({ 29 | message: "fail", 30 | config: { url: "https://example.com" }, 31 | }); 32 | 33 | const client = new CrawlClient(); 34 | 35 | await expect(client.getUrlWithRetry("https://example.com", {}, 2)).rejects.toThrow("fail (attempts: 2)"); 36 | 37 | expect(mockedAxiosInstance.get).toHaveBeenCalledTimes(2); 38 | }); 39 | }); 40 | -------------------------------------------------------------------------------- /crawler/test/lib/keepAlive.test.ts: -------------------------------------------------------------------------------- 1 | import http from "node:http"; 2 | import { AddressInfo } from "node:net"; 3 | 4 | import CrawlClient from "../../src/lib/CrawlClient"; 5 | 6 | describe("CrawlClient keep-alive", () => { 7 | test("reuses sockets for sequential requests", async () => { 8 | const connections = new Set(); 9 | const server = http.createServer((_req, res) => { 10 | res.end("ok"); 11 | }); 12 | server.on("connection", (socket) => { 13 | console.log("New connection established"); 14 | connections.add(socket); 15 | }); 16 | await new Promise((resolve) => server.listen(0, resolve)); 17 | 18 | const port = (server.address() as AddressInfo).port; 19 | const client = new CrawlClient(`http://127.0.0.1:${port}`); 20 | 21 | const res1 = await client.getUrl("/"); 22 | await new Promise((r) => setTimeout(r, 50)); 23 | const res2 = await client.getUrl("/"); 24 | 25 | expect(connections.size).toBe(1); 26 | 27 | expect((res1.request as any).reusedSocket).toBe(false); 28 | expect((res2.request as any).reusedSocket).toBe(true); 29 | 30 | await new Promise((resolve) => server.close(resolve)); 31 | }); 32 | }); 33 | -------------------------------------------------------------------------------- /frontend/src/components/InstanceView/InstanceCommunities.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { useSelector } from "react-redux"; 3 | 4 | import useCachedMultipart from "../../hooks/useCachedMultipart"; 5 | import Box from "@mui/joy/Box"; 6 | 7 | import { LinearValueLoader, PageError } from "../Shared/Display"; 8 | 9 | import Communities from "../Communities"; 10 | 11 | export default function InstanceCommunities({ instance }) { 12 | const homeBaseUrl = useSelector((state: any) => state.configReducer.homeBaseUrl); 13 | 14 | const { isLoading, loadingPercent, isSuccess, isError, error, data } = useCachedMultipart( 15 | "communityData", 16 | "community", 17 | ); 18 | 19 | const items = React.useMemo(() => { 20 | if (!isSuccess) return []; 21 | if (!data) return []; 22 | 23 | return data.filter((community) => community.baseurl === instance.baseurl); 24 | }, [data]); 25 | 26 | console.log("items", items); 27 | 28 | return ( 29 | 30 | {isLoading && !isError && } 31 | {isError && } 32 | {isSuccess && } 33 | 34 | ); 35 | } 36 | -------------------------------------------------------------------------------- /crawler/src/lib/error.ts: -------------------------------------------------------------------------------- 1 | export type IHTTPClientError = { 2 | isAxiosError?: boolean; 3 | code?: string; 4 | url?: string; 5 | request?: any; 6 | response?: any; 7 | }; 8 | 9 | export class HTTPError extends Error { 10 | public isAxiosError?: boolean; 11 | public code?: string; 12 | public url?: string; 13 | 14 | public request?: any; 15 | public response?: any; 16 | 17 | constructor(message: string, data: IHTTPClientError) { 18 | super(message); 19 | this.name = "HTTPError"; 20 | 21 | // spread data into this 22 | if (data) Object.assign(this, data); 23 | } 24 | } 25 | 26 | // error record for this domain is stored alongside this failure in the error db 27 | // so we dont hit it again within 24hrs 28 | export class CrawlError extends Error { 29 | constructor(message: string, data: any = false) { 30 | super(message); 31 | this.name = "CrawlError"; 32 | 33 | // spread data into this 34 | if (data) Object.assign(this, data); 35 | } 36 | } 37 | 38 | export class CrawlTooRecentError extends Error { 39 | constructor(message: string, data: any = false) { 40 | super(message); 41 | this.name = "CrawlTooRecentError"; 42 | 43 | // spread data into this 44 | if (data) Object.assign(this, data); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /crawler/ecosystem.config.cjs: -------------------------------------------------------------------------------- 1 | const defaultOptions = { 2 | log_type: "raw", 3 | script: "index.ts", 4 | exec_mode: "fork", 5 | interpreter: "tsx", 6 | }; 7 | 8 | module.exports = { 9 | apps: [ 10 | { 11 | ...defaultOptions, 12 | output: "./.data/logs/scheduler.log", 13 | name: "scheduler", 14 | args: ["-w", "cron"], 15 | instances: 1, 16 | }, 17 | { 18 | ...defaultOptions, 19 | output: "./.data/logs/instance.log", 20 | name: "crawl-instance", 21 | args: ["-w", "instance"], 22 | instances: 10, 23 | }, 24 | { 25 | ...defaultOptions, 26 | output: "./.data/logs/community.log", 27 | name: "crawl-community", 28 | args: ["-w", "community"], 29 | instances: 8, 30 | }, 31 | { 32 | ...defaultOptions, 33 | output: "./.data/logs/single.log", 34 | name: "crawl-one-community", 35 | args: ["-w", "single"], 36 | instances: 6, 37 | }, 38 | { 39 | ...defaultOptions, 40 | output: "./.data/logs/mbin.log", 41 | name: "crawl-mbin", 42 | args: ["-w", "mbin"], 43 | instances: 2, 44 | }, 45 | { 46 | ...defaultOptions, 47 | output: "./.data/logs/piefed.log", 48 | name: "crawl-piefed", 49 | args: ["-w", "piefed"], 50 | instances: 2, 51 | }, 52 | ], 53 | }; 54 | -------------------------------------------------------------------------------- /crawler/test/queue/instance.test.ts: -------------------------------------------------------------------------------- 1 | import { jest } from "@jest/globals"; 2 | 3 | import InstanceQueue from "../../src/queue/instance"; 4 | 5 | // Mock bee-queue to avoid redis dependency 6 | const saveMock = jest.fn<() => Promise>().mockResolvedValue(undefined); 7 | const setIdMock = jest.fn().mockReturnThis(); 8 | const timeoutMock = jest.fn().mockReturnThis(); 9 | const onMock = jest.fn().mockReturnThis(); 10 | 11 | const jobMock = { 12 | on: onMock, 13 | timeout: timeoutMock, 14 | setId: setIdMock, 15 | save: saveMock, 16 | }; 17 | 18 | const createJobMock = jest.fn().mockReturnValue(jobMock); 19 | 20 | class FakeBeeQueue { 21 | createJob = createJobMock; 22 | process = jest.fn(); 23 | on = jest.fn(); 24 | constructor() {} 25 | } 26 | 27 | jest.mock("bee-queue", () => { 28 | return jest.fn().mockImplementation(() => new FakeBeeQueue()); 29 | }); 30 | 31 | describe("InstanceQueue", () => { 32 | beforeEach(() => { 33 | jest.clearAllMocks(); 34 | }); 35 | 36 | test("createJob trims protocol and passes id correctly", async () => { 37 | const q = new InstanceQueue(false); 38 | await q.createJob("https://example.com"); 39 | 40 | expect(createJobMock).toHaveBeenCalledWith({ baseUrl: "example.com" }); 41 | expect(setIdMock).toHaveBeenCalledWith("example.com"); 42 | expect(saveMock).toHaveBeenCalled(); 43 | }); 44 | }); 45 | -------------------------------------------------------------------------------- /crawler/test/lib/validator.test.ts: -------------------------------------------------------------------------------- 1 | import { jest } from "@jest/globals"; 2 | 3 | import { isValidLemmyDomain, getActorBaseUrl, getActorCommunity } from "../../src/lib/validator"; 4 | 5 | describe("isValidLemmyDomain", () => { 6 | test("valid domain", () => { 7 | expect(isValidLemmyDomain("lemmy.ml")).toBe(true); 8 | }); 9 | 10 | test("invalid domain", () => { 11 | expect(isValidLemmyDomain("not_a_domain")).toBe(false); 12 | }); 13 | 14 | test("throws on non-string", () => { 15 | // @ts-expect-error testing runtime 16 | expect(() => isValidLemmyDomain(123)).toThrow(); 17 | }); 18 | }); 19 | 20 | describe("getActorBaseUrl", () => { 21 | test("extracts base url", () => { 22 | expect(getActorBaseUrl("https://example.com/u/user")).toBe("example.com"); 23 | }); 24 | 25 | test("returns null for invalid domain", () => { 26 | expect(getActorBaseUrl("https://invalid/u/user")).toBeNull(); 27 | }); 28 | 29 | test("returns null for malformed actor id", () => { 30 | expect(getActorBaseUrl("badstring")).toBeNull(); 31 | }); 32 | }); 33 | 34 | describe("getActorCommunity", () => { 35 | test("returns community name", () => { 36 | expect(getActorCommunity("https://example.com/c/pics")).toBe("pics"); 37 | }); 38 | 39 | test("returns null when missing", () => { 40 | expect(getActorCommunity("https://example.com/c/")).toBeNull(); 41 | }); 42 | }); 43 | -------------------------------------------------------------------------------- /frontend/src/components/Shared/CardStatBox.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | import Typography from "@mui/joy/Typography"; 4 | import Box from "@mui/joy/Box"; 5 | import Tooltip from "@mui/joy/Tooltip"; 6 | 7 | import PersonIcon from "@mui/icons-material/Person"; 8 | 9 | import { TinyNumber } from "../Shared/Display"; 10 | 11 | export default function CardStatBox({ name, icon = , value }) { 12 | return ( 13 | 14 | 23 | 35 | {icon} 36 | 37 | 49 | 50 | 51 | 52 | 53 | ); 54 | } 55 | -------------------------------------------------------------------------------- /.github/actions/cdk-deploy/action.yaml: -------------------------------------------------------------------------------- 1 | name: cdk-deploy 2 | description: Deploys the AWS CDK stack for the Lemmy Explorer project. 3 | 4 | inputs: 5 | aws-access-key-id: 6 | description: AWS access key 7 | required: true 8 | aws-secret-access-key: 9 | description: AWS secret 10 | required: true 11 | 12 | runs: 13 | using: "composite" 14 | steps: 15 | # Yarn Install ./cdk 16 | - name: Yarn Install ./cdk 17 | uses: ./.github/actions/yarn-install 18 | with: 19 | working-directory: ./cdk 20 | 21 | - name: CDK Bootstrap 22 | shell: bash 23 | run: yarn bootstrap 24 | working-directory: ./cdk 25 | env: 26 | AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }} 27 | AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }} 28 | 29 | - name: CDK Synth 30 | shell: bash 31 | run: yarn synth 32 | working-directory: ./cdk 33 | env: 34 | AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }} 35 | AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }} 36 | 37 | - name: CDK Diff 38 | shell: bash 39 | run: yarn diff 40 | working-directory: ./cdk 41 | env: 42 | AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }} 43 | AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }} 44 | 45 | - name: CDK Deploy 46 | shell: bash 47 | run: yarn deploy:ci 48 | working-directory: ./cdk 49 | env: 50 | AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }} 51 | AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }} 52 | -------------------------------------------------------------------------------- /crawler/src/lib/validator.ts: -------------------------------------------------------------------------------- 1 | import validator from "validator"; 2 | 3 | import logging from "./logging"; 4 | 5 | import type { BaseURL, ActorID } from "../../../types/basic"; 6 | 7 | export function isValidLemmyDomain(domain: BaseURL): boolean { 8 | // if it's not a string 9 | if (typeof domain !== "string") { 10 | logging.error("domain is not a string", domain); 11 | throw new Error("domain is not a string"); 12 | } 13 | 14 | return validator.isFQDN(domain, { allow_numeric_tld: true }); 15 | } 16 | 17 | export function getActorBaseUrl(actorId: ActorID | undefined): string | null { 18 | if (typeof actorId !== "string") { 19 | logging.error("actorId is not a string", actorId); 20 | return null; 21 | } 22 | 23 | const parts = actorId.split("/"); 24 | if (parts.length < 3) { 25 | logging.error("actorId has invalid structure", actorId); 26 | return null; 27 | } 28 | 29 | const actorBaseUrl = parts[2]; 30 | 31 | if (isValidLemmyDomain(actorBaseUrl)) { 32 | return actorBaseUrl; 33 | } 34 | 35 | return null; 36 | } 37 | 38 | export function getActorCommunity(actorId: string | undefined): string | null { 39 | if (typeof actorId !== "string") { 40 | logging.error("actorId is not a string", actorId); 41 | return null; 42 | } 43 | 44 | const parts = actorId.split("/"); 45 | if (parts.length < 5) { 46 | logging.error("actorId has invalid structure", actorId); 47 | return null; 48 | } 49 | 50 | const actorCommunity = parts[4]; 51 | 52 | if (!actorCommunity) { 53 | logging.error("actorId missing community segment", actorId); 54 | return null; 55 | } 56 | 57 | return actorCommunity; 58 | } 59 | -------------------------------------------------------------------------------- /crawler/src/output/sync_s3.ts: -------------------------------------------------------------------------------- 1 | import { join, resolve } from "node:path"; 2 | import { rm, mkdir, readFile, copyFile } from "node:fs/promises"; 3 | 4 | import { S3Client, PutObjectCommand, PutObjectCommandInput } from "@aws-sdk/client-s3"; 5 | 6 | import { AWS_REGION, PUBLISH_S3_BUCKET, CHECKPOINT_DIR, REDIS_DUMP_FILE } from "../lib/const"; 7 | import logging from "../lib/logging"; 8 | 9 | // upload a copy of the file in REDIS_DUMP_FILE to S3 10 | export async function syncCheckpoint() { 11 | try { 12 | const checkpointName = `checkpoint-${Date.now()}.rdb`; 13 | const checkpointPath = join(resolve(CHECKPOINT_DIR), checkpointName); 14 | logging.info("checkpointPath", checkpointPath); 15 | 16 | // copy the file to a save point 17 | await mkdir(CHECKPOINT_DIR, { recursive: true }); 18 | await copyFile(REDIS_DUMP_FILE, checkpointPath); 19 | 20 | const fileData = await readFile(checkpointPath); 21 | 22 | // upload to s3 23 | const client = new S3Client({ region: AWS_REGION }); 24 | if (!PUBLISH_S3_BUCKET) { 25 | throw new Error("PUBLISH_S3_BUCKET is not defined"); 26 | } 27 | 28 | const input: PutObjectCommandInput = { 29 | Body: fileData, 30 | Bucket: PUBLISH_S3_BUCKET, 31 | Key: "checkpoint/dump.rdb", 32 | Metadata: { 33 | checkpoint: checkpointName, 34 | }, 35 | }; 36 | const command = new PutObjectCommand(input); 37 | const response = await client.send(command); 38 | 39 | // delete checkpointPath 40 | await rm(checkpointPath, { force: true }); 41 | 42 | logging.info("syncCheckpoint success", response); 43 | } catch (error) { 44 | logging.error("syncCheckpoint error", error); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /frontend/test/spec/navigation.spec.ts: -------------------------------------------------------------------------------- 1 | import { test, expect } from "@playwright/test"; 2 | import { setupGlobalHooks } from "../config/test.utils"; 3 | setupGlobalHooks(); 4 | 5 | // Helper to check local storage 6 | async function getStorage(page) { 7 | const raw = await page.evaluate(() => localStorage.getItem("explorer_storage")); 8 | return raw ? JSON.parse(raw) : {}; 9 | } 10 | 11 | test.beforeEach(async ({ page }) => { 12 | await page.goto("/", { 13 | waitUntil: "networkidle", 14 | }); 15 | }); 16 | 17 | test("navigate between Instances and Communities", async ({ page }) => { 18 | await expect(page.locator('input[placeholder="Filter Instances"]')).toBeVisible(); 19 | 20 | await page.getByRole("tab", { name: "Communities" }).click(); 21 | await expect(page.locator('input[placeholder="Filter Communities"]')).toBeVisible(); 22 | 23 | await expect(page).toHaveURL("/communities"); 24 | }); 25 | 26 | test("switch instance view type", async ({ page }) => { 27 | await expect(page.locator('input[placeholder="Filter Instances"]')).toBeVisible(); 28 | await page.getByRole("button", { name: "List View" }).click(); 29 | await expect.poll(async () => (await getStorage(page))["instance.viewType"]).toBe("list"); 30 | }); 31 | 32 | test("navigate between main pages", async ({ page }) => { 33 | await page.goto("/"); 34 | 35 | // Instances tab is default 36 | await expect(page).toHaveURL("/"); 37 | 38 | // Click Communities tab 39 | await page.getByRole("tab", { name: /Communities/i }).click(); 40 | await expect(page).toHaveURL("/communities"); 41 | 42 | // Navigate back to instances 43 | await page.getByRole("tab", { name: /Instances/i }).click(); 44 | await expect(page).toHaveURL("/"); 45 | }); 46 | -------------------------------------------------------------------------------- /docs/data-diagram.drawio: -------------------------------------------------------------------------------- 1 | 7Vpdc5s4FP01fmwHIcDk0Xad3Z1NO2nTbJ2njgIKqAXkCuGP/vq92MKAhWMyccHu9CVBRx9I99xzJV0zwJN49Zcg8/A992k0MA1/NcDvBqZpGo4J/3JkvUWQie0tEgjmK6wE7thPqkBDoRnzaVprKDmPJJvXQY8nCfVkDSNC8GW92ROP6m+dk4BqwJ1HIh39wnwZblHXNkr8b8qCsHgzMlRNTIrGCkhD4vNlBcLTAZ4IzuX2KV5NaJRbr7DLtt/1gdrdxARNZJsOgfzX+PFjhhb/ze6/zdyb4MvCeGMqfhYkytSK1WzlujCB4Fni03wUY4DHy5BJejcnXl67BNYBC2UcQQnBoxqOCklXByeKdssHx6E8plKsoYnqMHSVxZTP2EV5WRJgY4WFFeOblgKJIj3YjV3aBR6UaV5gpsIbK1ahPviJKnIhQx7whETTEh3X7Va2ueF8rqz1jUq5Vk5PMsnrtqQrJmeV54d8qLe2Kr1bqZE3hXVRSGC9s2qh0isvlt02paLfdn35op5nDWzAM+HR53xKyZSIgMpn2uFmLxA0IpIt6vM4OaO63/+TpJIksDLTiWDa40cBT0H+9DGjGfB4WBeoG13UZWFhXRaoSRbur1LF1R9RtBYFbikKu09R4GdEYUwEWUa6OArEZ4sSun67hNe9+Z7wZQKlBM4FLHniRVuAas2bRiBzBn8XMKXrFJR1qGvPmjRxC1FaXYoS6Rt4h6oslfhQqTmmylKIDzUdvlaVR9WGDrDbWm6q6y1n8OqdVzhDu/kEUwyxDReq1x7hu2m8YnOz+/UB85yc4Ghotlo6i2n1GZstLTZ/oj5LAbqf+0Q2HFt0hBXRHI/Go7vp1/tPN3qjiKTyq7cJ9njU2KPnmGube+oy9JhrGl3GXEfj5gO/vCjc29nIbinAXvXnahw/0PR8SDZ+F5KHfZJsayRPheBCD3khjx+ztJNwZ5nHj5hul9FueIKdiG7MemB/2W/8+fMN1MP4br5EcDRdd53vQXadFDxsOPfbnZ77rT6D0YXdxlHrC0KvSapimhWpTXgcZwmTLN98zjRR5aC6OByn70wVci5FHKd08rY5J9Rr0gnpWafCydcvTTtVkkZeOcahvmeWS3KNFrLpNpfUa4b3wg64qO0JF/V6xDV7/Snrwm6m7Tl1ew2h53hvuRoeD2ed3lvQKS8u5fZyYTcXhIx6+sxpSJ91fHXRcysvJ6aREDz6MHo/PZLjbKbyNTQ9sSia8Ch3FOiLn1yPeh7gqRT8O63UPLq2ZRsnItbY+25iaPadFzX1o90tBRKM2/yrm30Tw9Jl3Y51eyU8oXvGVRCJWJBA0QNTwfh4nBuSeSQaqYqY+f5mI2sirr65nYCJ4dXxD1icBh7wL+NB//HgmvpUgLTgtmgUP/Ke743SarGXnOpGCcXyo6ztL3Llt214+j8= -------------------------------------------------------------------------------- /frontend/src/components/Shared/LanguageFilter.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | import Autocomplete from "@mui/joy/Autocomplete"; 4 | 5 | import LanguageIcon from "@mui/icons-material/Language"; 6 | 7 | import { BETTER_LANGS_LIST } from "../../lib/const"; 8 | 9 | type ILanguageFilterProps = { 10 | languageCodes: string[]; 11 | setLanguageCodes: (codes: string[]) => void; 12 | }; 13 | 14 | const LanguageFilter = React.memo(({ languageCodes, setLanguageCodes }: ILanguageFilterProps) => { 15 | // console.log("LanguageFilter", DEFAULT_LANGS, languageCodes, setLanguageCodes); 16 | return ( 17 | } 20 | // indicator={} 21 | id="tags-default" 22 | sx={{ 23 | width: { xs: "100%", sm: "auto" }, 24 | flexShrink: 0, 25 | }} 26 | placeholder="Filter Languages" 27 | options={Object.values(BETTER_LANGS_LIST)} 28 | getOptionLabel={(option: any) => `${option.name} (${option.nativeName})`} 29 | // custom search 30 | filterOptions={(options: any, params) => { 31 | const filtered = options.filter((option) => { 32 | if (params.inputValue === "") return true; 33 | if (option.name.toLowerCase().includes(params.inputValue.toLowerCase())) return true; 34 | if (option.code.toLowerCase().includes(params.inputValue.toLowerCase())) return true; 35 | return false; 36 | }); 37 | return filtered; 38 | }} 39 | value={languageCodes} 40 | onChange={(event, newValue: any) => { 41 | console.log("onChange", newValue); 42 | setLanguageCodes(newValue); 43 | }} 44 | /> 45 | ); 46 | }); 47 | export default LanguageFilter; 48 | -------------------------------------------------------------------------------- /frontend/webpack.prod.config.js: -------------------------------------------------------------------------------- 1 | const webpack = require("webpack"); 2 | const { merge } = require("webpack-merge"); 3 | 4 | const TerserPlugin = require("terser-webpack-plugin"); 5 | const CssMinimizerPlugin = require("css-minimizer-webpack-plugin"); 6 | 7 | const common = require("./webpack.common.js"); 8 | 9 | module.exports = merge(common, { 10 | mode: "production", 11 | devtool: false, 12 | output: { 13 | filename: "[name].bundle.[contenthash].js", 14 | chunkFilename: "[name].bundle.[contenthash].js", 15 | }, 16 | plugins: [ 17 | new webpack.EnvironmentPlugin({ 18 | NODE_ENV: "production", 19 | }), 20 | ], 21 | performance: { 22 | hints: false, 23 | maxEntrypointSize: 128000, 24 | maxAssetSize: 0, 25 | }, 26 | 27 | optimization: { 28 | minimize: true, 29 | moduleIds: "deterministic", 30 | // runtimeChunk: "single", 31 | minimizer: [ 32 | new CssMinimizerPlugin(), 33 | new TerserPlugin({ 34 | parallel: true, 35 | terserOptions: { 36 | ecma: 6, 37 | compress: { 38 | drop_console: true, 39 | drop_debugger: true, 40 | }, 41 | format: { 42 | comments: false, 43 | }, 44 | }, 45 | extractComments: false, 46 | }), 47 | ], 48 | // splitChunks: { 49 | // chunks: "all", 50 | // minSize: 50000, 51 | // maxSize: 150000, 52 | // minChunks: 1, 53 | // maxAsyncRequests: 10, 54 | // maxInitialRequests: 5, 55 | // // enforceSizeThreshold: 50000, 56 | // cacheGroups: { 57 | // vendors: { 58 | // test: /[\\/]node_modules[\\/]/, 59 | // name: "vendors", 60 | // chunks: "all", 61 | // }, 62 | // }, 63 | // }, 64 | }, 65 | }); 66 | -------------------------------------------------------------------------------- /crawler/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lemmy-explorer-crawler", 3 | "version": "0.8.2", 4 | "description": "", 5 | "author": "tgxn", 6 | "license": "MIT", 7 | "main": "./index.js", 8 | "type": "module", 9 | "engines": { 10 | "node": "^22.17.0" 11 | }, 12 | "scripts": { 13 | "start": "pm2 start ecosystem.config.cjs", 14 | "logs": "pm2 logs", 15 | "stop": "pm2 stop ecosystem.config.cjs", 16 | "delete": "pm2 delete ecosystem.config.cjs", 17 | "start:dev": "tsx index.ts", 18 | "start:init": "tsx index.ts --init", 19 | "start:dev:instance": "tsx index.ts -q instance", 20 | "start:dev:community": "tsx index.ts -q community", 21 | "health": "tsx index.ts --health", 22 | "out": "tsx index.ts --out", 23 | "output": "tsx index.ts --out", 24 | "format": "prettier --check .", 25 | "format:write": "prettier --write .", 26 | "check:ts": "tsc --noEmit", 27 | "test": "jest --silent --coverage --coverageReporters text", 28 | "test:loud": "jest --coverage --coverageReporters text" 29 | }, 30 | "dependencies": { 31 | "@aws-sdk/client-s3": "3.848.0", 32 | "axios": "1.10.0", 33 | "bee-queue": "1.7.1", 34 | "divinator": "1.0.1", 35 | "dotenv": "17.2.0", 36 | "node-cron": "4.2.1", 37 | "pino": "^9.7.0", 38 | "pino-pretty": "^13.1.1", 39 | "pm2": "6.0.8", 40 | "redis": "5.6.0", 41 | "remove-markdown": "0.6.2", 42 | "tsc": "2.0.4", 43 | "tslib": "2.8.1", 44 | "tsx": "4.20.3", 45 | "typescript": "5.8.3", 46 | "validator": "13.15.15", 47 | "z-score": "1.0.6" 48 | }, 49 | "devDependencies": { 50 | "@types/jest": "30.0.0", 51 | "@types/node": "22.16.5", 52 | "@types/node-cron": "3.0.11", 53 | "@types/validator": "13.15.2", 54 | "jest": "30.0.4", 55 | "prettier": "3.6.2", 56 | "ts-jest": "29.4.0" 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /frontend/test/unit/utils.test.ts: -------------------------------------------------------------------------------- 1 | import { parseVersion, compareVersionStrings } from "../../src/lib/utils"; 2 | 3 | describe("parseVersion", () => { 4 | it("parses standard versions", () => { 5 | expect(parseVersion("1.2.3")).toEqual([[1, 2, 3], ""]); 6 | }); 7 | 8 | it("parses versions with suffix", () => { 9 | expect(parseVersion("1.2.3-alpha")).toEqual([[1, 2, 3], "alpha"]); 10 | }); 11 | 12 | it("parses partial versions", () => { 13 | expect(parseVersion("2")).toEqual([[2, 0, 0], ""]); 14 | }); 15 | 16 | it("returns null for invalid versions", () => { 17 | expect(parseVersion("invalid")).toBeNull(); 18 | }); 19 | }); 20 | 21 | describe("compareVersionStrings", () => { 22 | it("sorts versions descending", () => { 23 | const versions = ["1.2.0", "1.10.0", "1.3.0"]; 24 | const sorted = versions.sort(compareVersionStrings); 25 | expect(sorted).toEqual(["1.10.0", "1.3.0", "1.2.0"]); 26 | }); 27 | 28 | it("places release versions before pre-releases", () => { 29 | const versions = ["1.0.0-alpha", "1.0.0"]; 30 | const sorted = versions.sort(compareVersionStrings); 31 | expect(sorted).toEqual(["1.0.0", "1.0.0-alpha"]); 32 | }); 33 | 34 | it("prioritizes numeric differences over suffix", () => { 35 | const versions = ["1.0.1-alpha", "1.0.0"]; 36 | const sorted = versions.sort(compareVersionStrings); 37 | expect(sorted).toEqual(["1.0.1-alpha", "1.0.0"]); 38 | }); 39 | 40 | it("sorts prerelease identifiers alphabetically", () => { 41 | const versions = ["1.0.0-alpha", "1.0.0-rc", "1.0.0-beta", "1.0.0"]; 42 | const sorted = versions.sort(compareVersionStrings); 43 | expect(sorted).toEqual(["1.0.0", "1.0.0-alpha", "1.0.0-beta", "1.0.0-rc"]); 44 | }); 45 | 46 | it("treats invalid versions as equal", () => { 47 | expect(compareVersionStrings("abc", "1.0.0")).toBe(0); 48 | }); 49 | }); 50 | -------------------------------------------------------------------------------- /crawler/src/lib/storage/instance.ts: -------------------------------------------------------------------------------- 1 | import { CrawlStorage } from "../crawlStorage"; 2 | 3 | import { IInstanceData, IInstanceDataKeyValue } from "../../../../types/storage"; 4 | 5 | /** 6 | * Stores each lemmy instance, keyed on baseUrl as `instance:baseUrl`. 7 | * 8 | * Each instance is stored as a JSON object with the following fields: 9 | */ 10 | 11 | export default class Instance { 12 | private storage: CrawlStorage; 13 | 14 | constructor(storage: CrawlStorage) { 15 | this.storage = storage; 16 | } 17 | 18 | async getAll(): Promise { 19 | return this.storage.listRedis(`instance:*`); 20 | } 21 | 22 | async getAllWithKeys(): Promise { 23 | return this.storage.listRedisWithKeys(`instance:*`); 24 | } 25 | 26 | async getOne(key: string): Promise { 27 | return this.storage.getRedis(`instance:${key}`); 28 | } 29 | 30 | async upsert(baseUrl: string, value: IInstanceData) { 31 | return this.storage.putRedis(`instance:${baseUrl}`, value); 32 | } 33 | 34 | async delete(key: string) { 35 | return this.storage.deleteRedis(`instance:${key}`); 36 | } 37 | 38 | // use these to track instance attributes over time 39 | async setTrackedAttribute(baseUrl: string, attributeName: string, attributeValue: string) { 40 | await this.storage.redisZAdd( 41 | `attributes:instance:${baseUrl}:${attributeName}`, 42 | Date.now(), 43 | attributeValue, 44 | ); 45 | } 46 | 47 | async getAttributeArray(baseUrl: string, attributeName: string) { 48 | const keys = await this.storage.getAttributeArray(baseUrl, attributeName); 49 | 50 | return keys; 51 | } 52 | 53 | async getAttributeWithScores(baseUrl: string, attributeName: string) { 54 | const keys = await this.storage.getAttributesWithScores(baseUrl, attributeName); 55 | 56 | return keys; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /pages/webpack.prod.config.js: -------------------------------------------------------------------------------- 1 | const webpack = require("webpack"); 2 | const { merge } = require("webpack-merge"); 3 | 4 | const TerserPlugin = require("terser-webpack-plugin"); 5 | const CssMinimizerPlugin = require("css-minimizer-webpack-plugin"); 6 | 7 | const common = require("./webpack.common.js"); 8 | 9 | module.exports = merge(common, { 10 | mode: "production", 11 | devtool: false, 12 | output: { 13 | filename: "[name].bundle.[contenthash].js", 14 | chunkFilename: "[name].bundle.[contenthash].js", 15 | publicPath: "./", 16 | }, 17 | plugins: [ 18 | new webpack.EnvironmentPlugin({ 19 | NODE_ENV: "production", 20 | }), 21 | ], 22 | performance: { 23 | hints: false, 24 | maxEntrypointSize: 128000, 25 | maxAssetSize: 0, 26 | }, 27 | 28 | optimization: { 29 | minimize: true, 30 | runtimeChunk: "single", 31 | minimizer: [ 32 | new CssMinimizerPlugin(), 33 | new TerserPlugin({ 34 | parallel: true, 35 | terserOptions: { 36 | ecma: 6, 37 | compress: { 38 | drop_console: true, 39 | drop_debugger: true, 40 | }, 41 | format: { 42 | comments: false, 43 | }, 44 | }, 45 | extractComments: false, 46 | }), 47 | ], 48 | splitChunks: { 49 | chunks: "all", 50 | minSize: 50000, 51 | maxSize: 150000, 52 | minChunks: 1, 53 | maxAsyncRequests: 10, 54 | maxInitialRequests: 5, 55 | // enforceSizeThreshold: 50000, 56 | cacheGroups: { 57 | defaultVendors: { 58 | test: /[\\/]node_modules[\\/]/, 59 | priority: -10, 60 | reuseExistingChunk: true, 61 | }, 62 | default: { 63 | minChunks: 2, 64 | priority: -20, 65 | reuseExistingChunk: true, 66 | }, 67 | }, 68 | }, 69 | }, 70 | }); 71 | -------------------------------------------------------------------------------- /pages/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lemmy-explorer-pages", 3 | "version": "0.4.0", 4 | "description": "", 5 | "author": "tgxn", 6 | "license": "MIT", 7 | "main": "./src/index.tsx", 8 | "scripts": { 9 | "start": "webpack serve --config webpack.dev.config.js", 10 | "build:dev": "webpack --config webpack.dev.config.js", 11 | "build": "webpack --config webpack.prod.config.js", 12 | "fcheck": "prettier --check .", 13 | "fwrite": "prettier --write .", 14 | "format:check": "prettier --check .", 15 | "format:write": "prettier --write .", 16 | "check:ts": "tsc --noEmit" 17 | }, 18 | "dependencies": { 19 | "@emotion/react": "^11.11.1", 20 | "@emotion/styled": "^11.14.1", 21 | "@mui/base": "^5.0.0-beta.70", 22 | "@mui/icons-material": "^7.2.0", 23 | "@mui/joy": "5.0.0-beta.52", 24 | "@mui/material": "^7.2.0", 25 | "@tanstack/react-query": "^5.83.0", 26 | "@tanstack/react-query-devtools": "^5.83.0", 27 | "axios": "^1.10.0", 28 | "moment": "^2.29.4", 29 | "react": "^19.1.0", 30 | "react-dom": "^19.1.0", 31 | "react-moment": "^1.1.3", 32 | "react-number-format": "^5.4.4" 33 | }, 34 | "devDependencies": { 35 | "@babel/core": "^7.28.0", 36 | "@babel/preset-env": "^7.28.0", 37 | "@babel/preset-react": "^7.27.1", 38 | "@babel/register": "^7.27.1", 39 | "@babel/preset-typescript": "^7.26.0", 40 | "@babel/polyfill": "^7.12.1", 41 | "@playwright/test": "^1.54.1", 42 | "@types/react": "^19.1.8", 43 | "@types/react-dom": "^19.1.6", 44 | "babel-loader": "^10.0.0", 45 | "copy-webpack-plugin": "^13.0.0", 46 | "html-webpack-plugin": "^5.5.3", 47 | "css-minimizer-webpack-plugin": "^7.0.2", 48 | "playwright": "^1.54.1", 49 | "prettier": "^3.6.2", 50 | "typescript": "^5.8.3", 51 | "webpack": "^5.100.2", 52 | "webpack-cli": "^6.0.1", 53 | "webpack-dev-server": "^5.2.2" 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /frontend/playwright.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig, devices } from "@playwright/test"; 2 | 3 | export default defineConfig({ 4 | // Look for test files in the "tests" directory, relative to this configuration file. 5 | testDir: "./test", 6 | testMatch: ["**/*.spec.ts"], 7 | testIgnore: ["**/*.test.ts"], 8 | 9 | outputDir: "./output/results", 10 | 11 | // Run all tests in parallel. 12 | fullyParallel: true, 13 | 14 | // path to the global setup files. 15 | globalSetup: require.resolve("./test/config/global.setup.ts"), 16 | 17 | // // path to the global teardown files. 18 | // globalTeardown: require.resolve("./global-teardown"), 19 | 20 | // Each test is given 30 seconds. 21 | timeout: 30000, 22 | 23 | // Fail the build on CI if you accidentally left test.only in the source code. 24 | forbidOnly: !!process.env.CI, 25 | 26 | // Retry on CI only. 27 | retries: process.env.CI ? 2 : 0, 28 | 29 | // Run tests in files in parallel. 30 | workers: process.env.CI ? 10 : undefined, 31 | 32 | // Reporter to use 33 | reporter: [["list"], ["html", { outputFolder: "./output/report" }]], 34 | 35 | // Configure projects for major browsers. 36 | projects: [ 37 | { 38 | name: "chromium", 39 | use: { 40 | permissions: ["clipboard-read", "clipboard-write"], 41 | ...devices["Desktop Chrome"], 42 | }, 43 | }, 44 | ], 45 | // Run your local dev server before starting the tests 46 | webServer: [ 47 | { 48 | command: "yarn run start:test", 49 | url: "http://127.0.0.1:9191", 50 | timeout: 120 * 1000, 51 | reuseExistingServer: true, 52 | stdout: "ignore", 53 | stderr: "pipe", 54 | }, 55 | ], 56 | 57 | use: { 58 | // Base URL to use in actions like `await page.goto('/')`. 59 | baseURL: "http://127.0.0.1:9191", 60 | screenshot: "on", 61 | 62 | // Collect trace when retrying the failed test. 63 | // trace: "on-first-retry", 64 | }, 65 | }); 66 | -------------------------------------------------------------------------------- /frontend/test/spec/main-screenshots.spec.ts: -------------------------------------------------------------------------------- 1 | import { test, expect } from "@playwright/test"; 2 | import { setupGlobalHooks } from "../config/test.utils"; 3 | setupGlobalHooks(); 4 | 5 | test("main: (1080p) instances page screenshot", async ({ page }) => { 6 | await page.setViewportSize({ width: 1920, height: 1080 }); 7 | await page.goto("/", { waitUntil: "networkidle" }); 8 | await expect(page.locator('input[placeholder="Filter Instances"]')).toBeVisible(); 9 | }); 10 | 11 | test("main: (1080p) communities page screenshot", async ({ page }) => { 12 | await page.setViewportSize({ width: 1920, height: 1080 }); 13 | await page.goto("/communities", { waitUntil: "networkidle" }); 14 | await expect(page.locator('input[placeholder="Filter Communities"]')).toBeVisible(); 15 | }); 16 | 17 | test("main: (mobile) instances page screenshot", async ({ page }) => { 18 | await page.setViewportSize({ width: 454, height: 917 }); 19 | await page.goto("/", { waitUntil: "networkidle" }); 20 | await expect(page.locator('input[placeholder="Filter Instances"]')).toBeVisible(); 21 | }); 22 | 23 | test("main: (mobile) communities page screenshot", async ({ page }) => { 24 | await page.setViewportSize({ width: 454, height: 917 }); 25 | await page.goto("/communities", { waitUntil: "networkidle" }); 26 | await expect(page.locator('input[placeholder="Filter Communities"]')).toBeVisible(); 27 | }); 28 | 29 | test("main: instance view overview lemmy.world", async ({ page }) => { 30 | await page.setViewportSize({ width: 1920, height: 1080 }); 31 | await page.goto("/instance/lemmy.world", { waitUntil: "networkidle" }); 32 | await expect(page).toHaveURL(/\/instance\/lemmy\.world/); 33 | }); 34 | 35 | test("main: instance view stats lemmy.world", async ({ page }) => { 36 | await page.setViewportSize({ width: 1920, height: 1080 }); 37 | await page.goto("/instance/lemmy.world/user-growth", { waitUntil: "networkidle" }); 38 | await expect(page).toHaveURL(/\/instance\/lemmy\.world\/user-growth/); 39 | }); 40 | -------------------------------------------------------------------------------- /crawler/src/lib/storage/tracking.ts: -------------------------------------------------------------------------------- 1 | import { CrawlStorage } from "../crawlStorage"; 2 | 3 | import { RECORD_TTL_TIMES_SECONDS } from "../const"; 4 | 5 | import { 6 | IErrorData, 7 | IErrorDataKeyValue, 8 | ILastCrawlData, 9 | ILastCrawlDataKeyValue, 10 | } from "../../../../types/storage"; 11 | 12 | export default class TrackingStore { 13 | private storage: CrawlStorage; 14 | 15 | private failureKey: string; 16 | private historyKey: string; 17 | 18 | constructor(storage: CrawlStorage) { 19 | this.storage = storage; 20 | 21 | this.failureKey = "error"; 22 | this.historyKey = "last_crawl"; 23 | } 24 | 25 | // track errors 26 | async getAllErrors(type: string): Promise { 27 | return this.storage.listRedisWithKeys(`${this.failureKey}:${type}:*`); 28 | } 29 | 30 | async getOneError(type: string, key: string): Promise { 31 | return this.storage.getRedis(`${this.failureKey}:${type}:${key}`); 32 | } 33 | 34 | async upsertError(type: string, baseUrl: string, errorDetail: IErrorData) { 35 | if (!baseUrl) throw new Error("baseUrl is required"); 36 | 37 | return this.storage.putRedisTTL( 38 | `${this.failureKey}:${type}:${baseUrl}`, 39 | errorDetail, 40 | RECORD_TTL_TIMES_SECONDS.ERROR, 41 | ); 42 | } 43 | 44 | // track last scans for instance and communities 45 | async getLastCrawl(type: string, baseUrl: string): Promise { 46 | return await this.storage.getRedis(`${this.historyKey}:${type}:${baseUrl}`); 47 | } 48 | 49 | async listAllLastCrawl(): Promise { 50 | return this.storage.listRedisWithKeys(`${this.historyKey}:*`); 51 | } 52 | 53 | async setLastCrawl(type: string, baseUrl: string, data: Partial) { 54 | if (!baseUrl) throw new Error("baseUrl is required"); 55 | 56 | return this.storage.putRedisTTL( 57 | `${this.historyKey}:${type}:${baseUrl}`, 58 | { time: Date.now(), ...data }, 59 | RECORD_TTL_TIMES_SECONDS.LAST_CRAWL, 60 | ); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /crawler/src/lib/storage/community.ts: -------------------------------------------------------------------------------- 1 | import { CrawlStorage } from "../crawlStorage"; 2 | 3 | import { ICommunityData, ICommunityDataKeyValue } from "../../../../types/storage"; 4 | 5 | export default class Community { 6 | private storage: CrawlStorage; 7 | 8 | constructor(storage: CrawlStorage) { 9 | this.storage = storage; 10 | } 11 | 12 | async getAll(): Promise { 13 | return this.storage.listRedis(`community:*`); 14 | } 15 | 16 | async getAllWithKeys(): Promise { 17 | return this.storage.listRedisWithKeys(`community:*`); 18 | } 19 | 20 | async getOne(baseUrl: string, communityName: string): Promise { 21 | return this.storage.getRedis(`community:${baseUrl}:${communityName.toLowerCase()}`); 22 | } 23 | 24 | async upsert(baseUrl: string, community: ICommunityData) { 25 | const storeData = { 26 | ...community, 27 | lastCrawled: Date.now(), 28 | }; 29 | return this.storage.putRedis(`community:${baseUrl}:${community.community.name.toLowerCase()}`, storeData); 30 | } 31 | 32 | async delete(baseUrl: string, communityName: string, reason: string = "unknown") { 33 | const oldRecord = await this.getOne(baseUrl, communityName); 34 | 35 | await this.storage.putRedis(`deleted:community:${baseUrl}:${communityName.toLowerCase()}`, { 36 | ...oldRecord, 37 | deletedAt: Date.now(), 38 | deleteReason: reason, 39 | }); 40 | 41 | const deletedCommunity = await this.storage.deleteRedis( 42 | `community:${baseUrl}:${communityName.toLowerCase()}`, 43 | ); 44 | 45 | return deletedCommunity; 46 | } 47 | 48 | // use these to track community attributes over time 49 | async setTrackedAttribute( 50 | baseUrl: string, 51 | communityName: string, 52 | attributeName: string, 53 | attributeValue: string, 54 | ) { 55 | return this.storage.redisZAdd( 56 | `attributes:community:${baseUrl}:${communityName}:${attributeName}`, 57 | Date.now(), 58 | attributeValue, 59 | ); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /frontend/src/components/InstanceView/InstanceVersions.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | import Moment from "react-moment"; 4 | 5 | import Alert from "@mui/joy/Alert"; 6 | import Box from "@mui/joy/Box"; 7 | import List from "@mui/joy/List"; 8 | import ListItem from "@mui/joy/ListItem"; 9 | import ListItemContent from "@mui/joy/ListItemContent"; 10 | import ListItemDecorator from "@mui/joy/ListItemDecorator"; 11 | import Typography from "@mui/joy/Typography"; 12 | 13 | import InfoIcon from "@mui/icons-material/Info"; 14 | import TipsAndUpdatesIcon from "@mui/icons-material/TipsAndUpdates"; 15 | 16 | export default function InstanceVersions({ instance, versionSeries }) { 17 | versionSeries = versionSeries.sort((a, b) => { 18 | return new Date(b.time).getTime() - new Date(a.time).getTime(); 19 | }); 20 | 21 | return ( 22 | 23 | } 26 | variant="soft" 27 | color={"primary"} 28 | > 29 | 30 | Current Version: {instance.version} 31 | 32 | 33 | 34 | 39 | {versionSeries.map((version) => ( 40 | 41 | 42 | 43 | 44 | 45 | 46 | 50 | {version.time} 51 | 52 | 53 | 54 | Version Detected: {version.value} 55 | 56 | 57 | 58 | ))} 59 | 60 | 61 | ); 62 | } 63 | -------------------------------------------------------------------------------- /crawler/src/lib/storage/mbin.ts: -------------------------------------------------------------------------------- 1 | import { CrawlStorage } from "../crawlStorage"; 2 | 3 | import { IMagazineData, IMagazineDataKeyValue } from "../../../../types/storage"; 4 | 5 | export default class MBinStore { 6 | private storage: CrawlStorage; 7 | 8 | constructor(storage: CrawlStorage) { 9 | this.storage = storage; 10 | } 11 | 12 | async getAll(): Promise { 13 | const magazineKeyValue = await this.storage.listRedisWithKeys(`mbin_magazine:*`); 14 | 15 | // put baseUrl into the magazine object 16 | for (const key in magazineKeyValue) { 17 | magazineKeyValue[key].baseurl = key.split(":")[1]; 18 | } 19 | 20 | return Object.values(magazineKeyValue); 21 | } 22 | 23 | async getAllWithKeys(): Promise { 24 | return this.storage.listRedisWithKeys(`mbin_magazine:*`); 25 | } 26 | 27 | async getOne(baseUrl: string, magazineName: string) { 28 | return this.storage.getRedis(`mbin_magazine:${baseUrl}:${magazineName}`); 29 | } 30 | 31 | async upsert(baseUrl: string, magazine: IMagazineData) { 32 | const storeData = { 33 | ...magazine, 34 | lastCrawled: Date.now(), 35 | }; 36 | return this.storage.putRedis(`mbin_magazine:${baseUrl}:${magazine.name.toLowerCase()}`, storeData); 37 | } 38 | 39 | async delete(baseUrl: string, magazineName: string, reason = "unknown") { 40 | const oldRecord = await this.getOne(baseUrl, magazineName); 41 | await this.storage.putRedis(`deleted:mbin_magazine:${baseUrl}:${magazineName}`, { 42 | ...oldRecord, 43 | deletedAt: Date.now(), 44 | deleteReason: reason, 45 | }); 46 | 47 | return this.storage.deleteRedis(`mbin_magazine:${baseUrl}:${magazineName}`); 48 | } 49 | 50 | // use these to track magazine attributes over time 51 | async setTrackedAttribute( 52 | baseUrl: string, 53 | magazineName: string, 54 | attributeName: string, 55 | attributeValue: string | number, 56 | ) { 57 | return await this.storage.redisZAdd( 58 | `attributes:mbin_magazine:${baseUrl}:${magazineName}:${attributeName}`, 59 | Date.now(), 60 | attributeValue, 61 | ); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /.github/workflows/pages-checks.yaml: -------------------------------------------------------------------------------- 1 | name: pages-checks 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - .github/workflows/pages-checks.yaml 7 | - pages/** 8 | 9 | # Allows you to run this workflow manually from the Actions tab 10 | workflow_dispatch: 11 | 12 | env: 13 | NODE_VERSION: 22.17.0 14 | 15 | jobs: 16 | pages-checks-style: 17 | runs-on: ubuntu-latest 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | 22 | - name: Use Node.js ${{ env.NODE_VERSION }} 23 | uses: actions/setup-node@v4 24 | with: 25 | node-version: ${{ env.NODE_VERSION }} 26 | 27 | # Install + Cache Frontend Dependencies 28 | - name: Cache Node Modules | Frontend 29 | id: cache-pages-yarn 30 | uses: actions/cache@v4 31 | env: 32 | cache-name: cache-pages-yarn 33 | with: 34 | path: ./pages/node_modules/ 35 | key: cache-pages-yarn-${{ hashFiles('pages/yarn.lock') }} 36 | 37 | - name: Install Node Modules | Frontend 38 | if: steps.cache-pages-yarn.outputs.cache-hit != 'true' 39 | run: yarn --frozen-lockfile 40 | working-directory: ./pages 41 | 42 | - name: Check Style | Frontend 43 | run: yarn format:check 44 | working-directory: ./pages 45 | 46 | pages-checks-types: 47 | runs-on: ubuntu-latest 48 | 49 | steps: 50 | - uses: actions/checkout@v4 51 | 52 | - name: Use Node.js ${{ env.NODE_VERSION }} 53 | uses: actions/setup-node@v4 54 | with: 55 | node-version: ${{ env.NODE_VERSION }} 56 | 57 | # Install + Cache Frontend Dependencies 58 | - name: Cache Node Modules | Frontend 59 | id: cache-pages-yarn 60 | uses: actions/cache@v4 61 | env: 62 | cache-name: cache-pages-yarn 63 | with: 64 | path: ./pages/node_modules/ 65 | key: cache-pages-yarn-${{ hashFiles('pages/yarn.lock') }} 66 | 67 | - name: Install Node Modules | Frontend 68 | if: steps.cache-pages-yarn.outputs.cache-hit != 'true' 69 | run: yarn --frozen-lockfile 70 | working-directory: ./pages 71 | 72 | - name: Check Types | Frontend 73 | run: yarn check:ts 74 | working-directory: ./pages 75 | -------------------------------------------------------------------------------- /crawler/src/lib/storage/piefed.ts: -------------------------------------------------------------------------------- 1 | import { CrawlStorage } from "../crawlStorage"; 2 | 3 | import { IPiefedCommunityData, IPiefedCommunityDataKeyValue } from "../../../../types/storage"; 4 | 5 | export default class PiefedStore { 6 | private storage: CrawlStorage; 7 | 8 | constructor(storage: CrawlStorage) { 9 | this.storage = storage; 10 | } 11 | 12 | async getAll(): Promise { 13 | const communityKeyValue = await this.storage.listRedisWithKeys(`piefed_community:*`); 14 | 15 | // put baseUrl into the magazine object 16 | for (const key in communityKeyValue) { 17 | communityKeyValue[key].baseurl = key.split(":")[1]; 18 | } 19 | 20 | return Object.values(communityKeyValue); 21 | } 22 | 23 | async getAllWithKeys(): Promise { 24 | return this.storage.listRedisWithKeys(`piefed_community:*`); 25 | } 26 | 27 | async getOne(baseUrl: string, communityName: string) { 28 | return this.storage.getRedis(`piefed_community:${baseUrl}:${communityName}`); 29 | } 30 | 31 | async upsert(baseUrl: string, community: IPiefedCommunityData) { 32 | const storeData = { 33 | ...community, 34 | lastCrawled: Date.now(), 35 | }; 36 | return this.storage.putRedis( 37 | `piefed_community:${baseUrl}:${community.community.name.toLowerCase()}`, 38 | storeData, 39 | ); 40 | } 41 | 42 | async delete(baseUrl: string, communityName: string, reason = "unknown") { 43 | const oldRecord = await this.getOne(baseUrl, communityName); 44 | await this.storage.putRedis(`deleted:piefed_community:${baseUrl}:${communityName}`, { 45 | ...oldRecord, 46 | deletedAt: Date.now(), 47 | deleteReason: reason, 48 | }); 49 | 50 | return this.storage.deleteRedis(`piefed_community:${baseUrl}:${communityName}`); 51 | } 52 | 53 | // use these to track community attributes over time 54 | async setTrackedAttribute( 55 | baseUrl: string, 56 | communityName: string, 57 | attributeName: string, 58 | attributeValue: string | number, 59 | ) { 60 | return await this.storage.redisZAdd( 61 | `attributes:piefed_community:${baseUrl}:${communityName}:${attributeName}`, 62 | Date.now(), 63 | attributeValue, 64 | ); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /frontend/src/components/Shared/InstanceModal.tsx: -------------------------------------------------------------------------------- 1 | // // CURRENTLY UNUSED 2 | 3 | // import React from "react"; 4 | // import { useSelector, useDispatch } from "react-redux"; 5 | 6 | // import Button from "@mui/joy/Button"; 7 | // import Modal from "@mui/joy/Modal"; 8 | // import ModalClose from "@mui/joy/ModalClose"; 9 | // import Typography from "@mui/joy/Typography"; 10 | // import Sheet from "@mui/joy/Sheet"; 11 | 12 | // import { showInstanceModal } from "../../reducers/modalReducer"; 13 | 14 | // function InstanceModal() { 15 | // const isOpen = useSelector((state: any) => state.modalReducer.instanceModalOpen); 16 | // const instance = useSelector((state: any) => state.modalReducer.instanceData); 17 | // const dispatch = useDispatch(); 18 | // const setOpen = (instanceData) => dispatch(showInstanceModal(instanceData)); 19 | // return ( 20 | // setOpen(false)} 25 | // // sx={{ display: "flex", justifyContent: "center", alignItems: "center" }} 26 | // > 27 | // 36 | // 46 | // 47 | // This is the modal title 48 | // 49 | // 50 | // Make sure to use aria-labelledby on the modal dialog with an optional{" "} 51 | // aria-describedby attribute. 52 | // 53 | // 54 | // 55 | // ); 56 | // } 57 | 58 | // export default InstanceModal; 59 | -------------------------------------------------------------------------------- /crawler/src/bin/manual.ts: -------------------------------------------------------------------------------- 1 | import logging from "../lib/logging"; 2 | 3 | import InstanceQueue from "../queue/instance"; 4 | import CommunityQueue from "../queue/community_list"; 5 | import SingleCommunityQueue from "../queue/community_single"; 6 | import MBinQueue from "../queue/mbin"; 7 | import PiefedQueue from "../queue/piefed"; 8 | 9 | export default async function runManualWorker(workerName: string, firstParam: string, secondParam: string) { 10 | // scan one instance 11 | if (workerName == "i" || workerName == "instance") { 12 | logging.info(`Running Instance Crawl for ${firstParam}`); 13 | const instanceCrawl = new InstanceQueue(true, "instance_manual"); 14 | await instanceCrawl.createJob(firstParam, (resultData) => { 15 | logging.info("Instance Crawl Complete"); 16 | process.exit(0); 17 | }); 18 | } 19 | 20 | // scan one community 21 | else if (workerName == "c" || workerName == "community") { 22 | logging.info(`Running Community Crawl for ${firstParam}`); 23 | const communityCrawl = new CommunityQueue(true, "community_manual"); 24 | await communityCrawl.createJob(firstParam, (resultData) => { 25 | logging.info("Community Crawl Complete"); 26 | process.exit(0); 27 | }); 28 | } 29 | 30 | // finger one community 31 | else if (workerName == "s" || workerName == "single") { 32 | logging.info(`Running CrawlFinger Crawl for ${firstParam}`); 33 | const crawlOneComm = new SingleCommunityQueue(true, "one_community_manual"); 34 | await crawlOneComm.createJob(firstParam, secondParam, (resultData) => { 35 | logging.info("CrawlFinger Crawl Complete"); 36 | process.exit(0); 37 | }); 38 | } 39 | 40 | // scan one mbin 41 | else if (workerName == "m" || workerName == "mbin") { 42 | logging.info(`Running Single MBin Crawl for ${firstParam}`); 43 | const crawlMBinManual = new MBinQueue(true, "mbin_manual"); 44 | await crawlMBinManual.createJob(firstParam, (resultData) => { 45 | logging.info("MBIN Crawl Complete"); 46 | process.exit(0); 47 | }); 48 | } 49 | 50 | // scan one piefed 51 | else if (workerName == "p" || workerName == "piefed") { 52 | logging.info(`Running Single Piefed Crawl for ${firstParam}`); 53 | const crawlPiefedManual = new PiefedQueue(true, "piefed_manual"); 54 | await crawlPiefedManual.createJob(firstParam, (resultData) => { 55 | logging.info("Piefed Crawl Complete"); 56 | process.exit(0); 57 | }); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /.github/actions/start-redis/action.yaml: -------------------------------------------------------------------------------- 1 | name: start-redis 2 | description: Fetches Redis dump from S3, caches it, and starts Redis via Docker Compose. 3 | 4 | inputs: 5 | aws-access-key-id: 6 | description: AWS access key 7 | required: true 8 | aws-secret-access-key: 9 | description: AWS secret 10 | required: true 11 | aws-region: 12 | description: AWS region 13 | required: false 14 | default: ap-southeast-2 15 | 16 | s3-bucket: 17 | description: S3 bucket to fetch Redis dump from 18 | required: true 19 | checkpoint-s3-path: 20 | description: Path to the Redis dump in S3 21 | required: false 22 | default: "checkpoint/dump.rdb" 23 | 24 | runs: 25 | using: "composite" 26 | steps: 27 | - name: Get current hour for cache busting 28 | id: cache-hour 29 | shell: bash 30 | run: echo "hour=$(date +'%Y-%m-%d-%H')" >>$GITHUB_OUTPUT 31 | 32 | - name: Cache Redis Dump 33 | id: cache-redis 34 | uses: actions/cache@v4 35 | env: 36 | cache-name: cache-redis 37 | with: 38 | path: ./.redis/dump.rdb 39 | key: cache-redis-${{ steps.cache-hour.outputs.hour }} 40 | 41 | - name: Download Redis Dump 42 | if: steps.cache-redis.outputs.cache-hit != 'true' 43 | uses: keithweaver/aws-s3-github-action@v1.0.0 44 | with: 45 | command: cp 46 | source: s3://${{ inputs.s3-bucket }}/${{ inputs.checkpoint-s3-path }} 47 | destination: ./.redis/dump.rdb 48 | aws_access_key_id: ${{ inputs.aws-access-key-id }} 49 | aws_secret_access_key: ${{ inputs.aws-secret-access-key }} 50 | aws_region: ${{ inputs.aws-region }} 51 | 52 | - name: Set COMPOSE_COMMAND 53 | shell: bash 54 | run: echo "COMPOSE_COMMAND=docker compose -f docker-compose.github.yaml" >> $GITHUB_ENV 55 | 56 | - name: Start Redis 57 | working-directory: ./crawler 58 | shell: bash 59 | run: ${{ env.COMPOSE_COMMAND }} up -d redis 60 | 61 | - name: Wait for Redis 62 | working-directory: ./crawler 63 | shell: bash 64 | run: | 65 | until ${{ env.COMPOSE_COMMAND }} exec redis redis-cli ping | grep PONG; do 66 | echo "Waiting for Redis..." 67 | sleep 1 68 | done 69 | 70 | - working-directory: ./crawler 71 | shell: bash 72 | run: docker ps -a 73 | 74 | - working-directory: ./crawler 75 | shell: bash 76 | run: ${{ env.COMPOSE_COMMAND }} logs redis 77 | -------------------------------------------------------------------------------- /frontend/src/components/InstanceView/InstanceUserGrowth.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | import MultiDataLineGraph from "../Shared/MultiDataLineGraph"; 4 | 5 | function InstanceUserGrowth({ metricsData }) { 6 | // console.log("userSeries", metricsData); 7 | 8 | // round to closest 1000 9 | const minUsers = Math.floor(Number(metricsData.users[0].value) / 1000) * 1000; 10 | const maxUsers = 11 | Math.ceil( 12 | metricsData.users.reduce( 13 | (max, metric) => (Number(metric.value) > max ? Number(metric.value) : max), 14 | 0, 15 | ) / 1000, 16 | ) * 1000; 17 | 18 | const minPosts = Math.floor(Number(metricsData.posts[0].value) / 1000) * 1000; 19 | const maxPosts = 20 | Math.ceil( 21 | metricsData.posts.reduce( 22 | (max, metric) => (Number(metric.value) > max ? Number(metric.value) : max), 23 | 0, 24 | ) / 1000, 25 | ) * 1000; 26 | 27 | // merge the arrays, with any that have the same time going into the same object 28 | 29 | const singleStatsArray = []; 30 | 31 | for (const userData of metricsData.users) { 32 | singleStatsArray.push({ 33 | time: userData.time, 34 | users: userData.value, 35 | }); 36 | } 37 | for (const postData of metricsData.posts) { 38 | const time = postData.time; 39 | const posts = postData.value; 40 | 41 | const existing = singleStatsArray.find((i) => i.time === time); 42 | if (existing) { 43 | existing.posts = posts; 44 | } else { 45 | singleStatsArray.push({ 46 | time, 47 | posts, 48 | }); 49 | } 50 | } 51 | 52 | // order by time 53 | singleStatsArray.sort((a, b) => a.time - b.time); 54 | 55 | // console.log("singleStatsArray", singleStatsArray); 56 | 57 | return ( 58 | 79 | ); 80 | } 81 | 82 | export default React.memo(InstanceUserGrowth, (prevProps, nextProps) => { 83 | return JSON.stringify(prevProps.metricsData) === JSON.stringify(nextProps.metricsData); 84 | }); 85 | -------------------------------------------------------------------------------- /frontend/webpack.common.js: -------------------------------------------------------------------------------- 1 | const path = require("path"); 2 | 3 | const HtmlWebpackPlugin = require("html-webpack-plugin"); 4 | const CopyWebpackPlugin = require("copy-webpack-plugin"); 5 | 6 | module.exports = { 7 | entry: "./src/index.tsx", 8 | output: { 9 | path: path.join(__dirname, "dist"), 10 | publicPath: "/", 11 | clean: true, 12 | }, 13 | resolve: { 14 | extensions: [".tsx", ".jsx", ".ts", ".js"], 15 | }, 16 | plugins: [ 17 | new HtmlWebpackPlugin({ 18 | title: "Lemmy Explorer", 19 | template: "index.html", 20 | inject: "body", 21 | scriptLoading: "defer", 22 | hash: true, 23 | }), 24 | new CopyWebpackPlugin({ 25 | patterns: [{ from: "public", to: "" }], 26 | }), 27 | ], 28 | optimization: { 29 | sideEffects: true, 30 | usedExports: true, 31 | splitChunks: { 32 | chunks: "all", 33 | minSize: 50000, 34 | maxSize: 150000, 35 | minChunks: 1, 36 | maxAsyncRequests: 10, 37 | maxInitialRequests: 3, 38 | // enforceSizeThreshold: 50000, 39 | cacheGroups: { 40 | react: { 41 | test: /[\\/]node_modules[\\/](react|react-dom|react-router-dom)[\\/]/, 42 | name: "react", 43 | chunks: "all", 44 | priority: 30, 45 | reuseExistingChunk: true, 46 | }, 47 | tanstack: { 48 | test: /[\\/]node_modules[\\/](@tanstack|react-query|@tanstack|react-table)[\\/]/, 49 | name: "tanstack", 50 | chunks: "all", 51 | priority: 20, 52 | reuseExistingChunk: true, 53 | }, 54 | mui: { 55 | test: /[\\/]node_modules[\\/](@mui|@emotion|tss-react)[\\/]/, 56 | name: "mui", 57 | chunks: "all", 58 | priority: 10, 59 | reuseExistingChunk: true, 60 | }, 61 | vendors: { 62 | test: /[\\/]node_modules[\\/]/, 63 | name: "vendors", 64 | chunks: "all", 65 | priority: 0, 66 | reuseExistingChunk: true, 67 | }, 68 | }, 69 | }, 70 | }, 71 | module: { 72 | rules: [ 73 | { 74 | test: /\.tsx?$/, 75 | exclude: /(node_modules)/, 76 | use: { 77 | loader: "babel-loader", 78 | options: { 79 | presets: [ 80 | ["@babel/preset-env", { modules: false }], 81 | "@babel/preset-react", 82 | "@babel/preset-typescript", 83 | ], 84 | }, 85 | }, 86 | }, 87 | ], 88 | }, 89 | }; 90 | -------------------------------------------------------------------------------- /frontend/src/components/Shared/LineGraph.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { useColorScheme } from "@mui/joy/styles"; 3 | 4 | import Box from "@mui/joy/Box"; 5 | 6 | import moment from "moment"; 7 | import { SimpleNumberFormat } from "../Shared/Display"; 8 | 9 | import { CartesianGrid, ResponsiveContainer, Line, LineChart, Tooltip, XAxis, YAxis } from "recharts"; 10 | 11 | const CustomTooltip = ({ active, payload, label }) => { 12 | console.info(payload); 13 | if (active && payload && payload.length) { 14 | return ( 15 | 16 | {payload.map((i) => ( 17 |

18 | {i.name}: {" "} 19 |

20 | ))} 21 | 22 | ); 23 | } 24 | 25 | return null; 26 | }; 27 | 28 | // export default function LineGraph({ dataSeries }) { 29 | // const { mode } = useColorScheme(); 30 | 31 | // const minUsers = Number(dataSeries[0].value); 32 | // const maxUsers = Number(dataSeries[dataSeries.length - 1].value); 33 | 34 | // // round to closest 1000 35 | // const minUsersRounded = Math.floor(minUsers / 1000) * 1000; 36 | // const maxUsersRounded = Math.ceil(maxUsers / 1000) * 1000; 37 | 38 | // console.log("userSeries", dataSeries); 39 | 40 | // const scale = "linear"; //scaleLog().base(Math.E); 41 | 42 | // return ( 43 | // 44 | // 45 | // 46 | 47 | // {/* Time Axis */} 48 | // moment(unixTime).format("DD-MM-YYYY")} 53 | // type="number" 54 | // padding={{ left: 30, right: 30 }} 55 | // /> 56 | 57 | // 61 | 62 | // {/* Count Axis */} 63 | // unixTime.toLocaleString()} 69 | // /> 70 | 71 | // 78 | // 79 | // 80 | // ); 81 | // } 82 | -------------------------------------------------------------------------------- /.github/workflows/aws-deploy-prod.yaml: -------------------------------------------------------------------------------- 1 | name: deploy-aws-prod 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | - .github/workflows/aws-deploy-prod.yaml 9 | - crawler/** 10 | - frontend/** 11 | - cdk/** 12 | - types/** 13 | 14 | # Allows you to run this workflow manually from the Actions tab 15 | workflow_dispatch: 16 | 17 | # run every 6 hours 18 | schedule: 19 | - cron: "0 */6 * * *" 20 | 21 | concurrency: 22 | group: production 23 | cancel-in-progress: false 24 | 25 | env: 26 | NODE_VERSION: 22.17.0 27 | 28 | jobs: 29 | aws_deploy_prod: 30 | runs-on: ubuntu-latest 31 | 32 | environment: 33 | name: production 34 | url: https://lemmyverse.net 35 | 36 | steps: 37 | # https://github.com/actions/toolkit/issues/946#issuecomment-1590016041 38 | - name: root suid tar 39 | run: sudo chown root:root /bin/tar && sudo chmod u+s /bin/tar 40 | 41 | - uses: actions/checkout@v4 42 | 43 | - name: Use Node.js ${{ env.NODE_VERSION }} 44 | uses: actions/setup-node@v4 45 | with: 46 | node-version: ${{ env.NODE_VERSION }} 47 | 48 | # Download + Start Redis Docker 49 | - name: Get Dump + Start Redis 50 | uses: ./.github/actions/start-redis 51 | with: 52 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} 53 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 54 | s3-bucket: ${{ vars.BUILD_S3_BUCKET }} 55 | 56 | # Yarn Install ./crawler 57 | - name: Yarn Install ./crawler 58 | uses: ./.github/actions/yarn-install 59 | with: 60 | working-directory: ./crawler 61 | 62 | # Run Crawler Output Script 63 | - name: Run Health Script 64 | run: yarn health 65 | working-directory: ./crawler 66 | 67 | - name: Run Output Script 68 | run: yarn output 69 | working-directory: ./crawler 70 | 71 | # Yarn Install ./frontend 72 | - name: Yarn Install ./frontend 73 | uses: ./.github/actions/yarn-install 74 | with: 75 | working-directory: ./frontend 76 | 77 | - name: Build the Frontend 78 | run: yarn build 79 | working-directory: ./frontend 80 | 81 | - name: Create CDK Config JSON 82 | id: create-json 83 | uses: jsdaniell/create-json@v1.2.3 84 | with: 85 | dir: ./cdk 86 | name: "config.json" 87 | json: ${{ vars.CONFIG_JSON }} 88 | 89 | # CDK Deployment 90 | - name: CDK Deployment 91 | uses: ./.github/actions/cdk-deploy 92 | with: 93 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} 94 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 95 | -------------------------------------------------------------------------------- /frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lemmy-explorer-frontend", 3 | "version": "0.11.0", 4 | "description": "", 5 | "author": "tgxn", 6 | "license": "MIT", 7 | "main": "./src/index.tsx", 8 | "scripts": { 9 | "start": "webpack serve --config webpack.dev.config.js", 10 | "start:test": "webpack serve --config webpack.dev.config.js", 11 | "build:dev": "webpack --config webpack.dev.config.js", 12 | "build": "webpack --config webpack.prod.config.js", 13 | "test": "playwright test", 14 | "format:check": "prettier --check .", 15 | "format:write": "prettier --write .", 16 | "check:ts": "tsc --noEmit", 17 | "test:unit": "jest --coverage" 18 | }, 19 | "dependencies": { 20 | "@babel/core": "^7.26.9", 21 | "@babel/preset-env": "^7.26.9", 22 | "@babel/preset-react": "^7.22.5", 23 | "@babel/preset-typescript": "^7.26.0", 24 | "@emotion/react": "^11.11.1", 25 | "@emotion/styled": "^11.11.0", 26 | "@mui/base": "^5.0.0-beta.5", 27 | "@mui/icons-material": "^6.4.6", 28 | "@mui/joy": "5.0.0-alpha.85", 29 | "@mui/lab": "^5.0.0-alpha.134", 30 | "@mui/material": "^6.4.6", 31 | "@react-hook/window-size": "^3.1.1", 32 | "@reduxjs/toolkit": "^1.9.5", 33 | "@tanstack/react-query": "^4.36.1", 34 | "@tanstack/react-query-devtools": "^4.36.1", 35 | "@uidotdev/usehooks": "^2.0.1", 36 | "axios": "^1.8.1", 37 | "d3-scale": "^4.0.2", 38 | "masonic": "^3.7.0", 39 | "moment": "^2.29.4", 40 | "notistack": "^3.0.2", 41 | "react": "^18.2.0", 42 | "react-dom": "^18.2.0", 43 | "react-image": "^4.1.0", 44 | "react-moment": "^1.1.3", 45 | "react-number-format": "^5.2.2", 46 | "react-redux": "^8.1.1", 47 | "react-router-dom": "^6.14.1", 48 | "react-virtualized": "^9.22.6", 49 | "react-window": "^1.8.9", 50 | "recharts": "^2.15.1", 51 | "redux": "^4.2.1", 52 | "sass": "^1.63.6", 53 | "tss-react": "^4.8.8" 54 | }, 55 | "devDependencies": { 56 | "@babel/register": "^7.22.5", 57 | "@playwright/test": "^1.54.2", 58 | "@types/jest": "^30.0.0", 59 | "@types/react": "^18.3.8", 60 | "@types/react-dom": "^18.3.0", 61 | "@types/react-window": "^1.8.8", 62 | "babel-loader": "^9.1.3", 63 | "copy-webpack-plugin": "^12.0.2", 64 | "css-loader": "^7.1.2", 65 | "css-minimizer-webpack-plugin": "^7.0.0", 66 | "file-loader": "^6.2.0", 67 | "html-webpack-plugin": "^5.5.3", 68 | "jest": "^30.0.5", 69 | "playwright": "^1.54.2", 70 | "prettier": "^3.6.2", 71 | "sass-loader": "^16.0.5", 72 | "style-loader": "^4.0.0", 73 | "ts-jest": "29.4.0", 74 | "ts-loader": "^9.2.3", 75 | "typescript": "^5.4.5", 76 | "url-loader": "^4.1.1", 77 | "webpack": "^5.101.0", 78 | "webpack-cli": "^6.0.1", 79 | "webpack-dev-server": "^5.2.2" 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /frontend/src/reducers/configReducer.ts: -------------------------------------------------------------------------------- 1 | import storage from "../lib/storage"; 2 | 3 | export function setHomeInstance(baseUrl, type = "lemmy") { 4 | return { 5 | type: "setHomeInstance", 6 | payload: { baseUrl, type }, 7 | }; 8 | } 9 | 10 | export function changeInstanceType(type) { 11 | return { 12 | type: "changeInstanceType", 13 | payload: { type }, 14 | }; 15 | } 16 | 17 | export function setFilteredInstances(filteredInstances) { 18 | return { 19 | type: "setFilteredInstances", 20 | payload: { filteredInstances }, 21 | }; 22 | } 23 | 24 | export function setFilteredTags(filteredTags) { 25 | console.log("setFilteredTags", filteredTags); 26 | return { 27 | type: "setFilteredTags", 28 | payload: { filteredTags }, 29 | }; 30 | } 31 | 32 | export function setFilterSuspicious(filterSuspicious) { 33 | return { 34 | type: "setFilterSuspicious", 35 | payload: { filterSuspicious }, 36 | }; 37 | } 38 | 39 | const initialState = { 40 | homeBaseUrl: storage.get("instance"), 41 | instanceType: storage.get("type", "lemmy"), 42 | filteredInstances: storage.get("filteredInstances", []), 43 | filteredTags: storage.get("filteredTags", []), 44 | filterSuspicious: storage.get("config.filterSuspicious", true), 45 | }; 46 | 47 | const configReducer = (state = initialState, action: any = {}) => { 48 | switch (action.type) { 49 | case "setHomeInstance": 50 | const baseUrl = action.payload.baseUrl; 51 | if (baseUrl == null) { 52 | storage.remove("instance"); 53 | storage.remove("type"); 54 | } else { 55 | storage.set("instance", action.payload.baseUrl); 56 | storage.set("type", action.payload.type); 57 | } 58 | return { 59 | ...state, 60 | homeBaseUrl: action.payload.baseUrl, 61 | instanceType: action.payload.type, 62 | }; 63 | 64 | case "changeInstanceType": 65 | storage.set("type", action.payload.type); 66 | return { 67 | ...state, 68 | instanceType: action.payload.type, 69 | }; 70 | 71 | case "setFilteredInstances": 72 | storage.set("filteredInstances", action.payload.filteredInstances); 73 | return { 74 | ...state, 75 | filteredInstances: action.payload.filteredInstances, 76 | }; 77 | 78 | case "setFilteredTags": 79 | storage.set("filteredTags", action.payload.filteredTags); 80 | return { 81 | ...state, 82 | filteredTags: action.payload.filteredTags, 83 | }; 84 | 85 | case "setFilterSuspicious": 86 | storage.set("config.filterSuspicious", action.payload.filterSuspicious); 87 | return { 88 | ...state, 89 | filterSuspicious: action.payload.filterSuspicious, 90 | }; 91 | 92 | default: 93 | return state; 94 | } 95 | }; 96 | 97 | export default configReducer; 98 | -------------------------------------------------------------------------------- /frontend/src/components/Shared/Avatar.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { useSelector, useDispatch } from "react-redux"; 3 | 4 | import Avatar from "@mui/joy/Avatar"; 5 | import Tooltip from "@mui/joy/Tooltip"; 6 | import Badge from "@mui/joy/Badge"; 7 | 8 | import HomeIcon from "@mui/icons-material/Home"; 9 | 10 | import { setHomeInstance } from "../../reducers/configReducer"; 11 | 12 | export function IconAvatar({ src, alt }: { src: string; alt: string }) { 13 | let style = { 14 | display: "flex", 15 | borderRadius: 8, 16 | bgcolor: "background.level1", 17 | }; 18 | return ; 19 | } 20 | 21 | export const InstanceAvatar = React.memo(function ({ instance }: { instance: any }) { 22 | const homeBaseUrl = useSelector((state: any) => state.configReducer.homeBaseUrl); 23 | const dispatch = useDispatch(); 24 | 25 | const [isHover, setIsHover] = React.useState(false); 26 | const isHomeUrl = homeBaseUrl == instance.baseurl; 27 | 28 | const style = React.useMemo(() => { 29 | let style: any = { 30 | display: "flex", 31 | borderRadius: 8, 32 | bgcolor: "background.level1", 33 | }; 34 | 35 | if (isHomeUrl) { 36 | style = { 37 | ...style, 38 | outline: "1px solid #0f5d26", 39 | boxShadow: "1px 1px 3px 0px #0f5d26", 40 | }; 41 | } else if (isHover) { 42 | style = { 43 | ...style, 44 | outline: "1px solid #666", 45 | boxShadow: "1px 1px 3px 0px #666", 46 | }; 47 | } 48 | return style; 49 | }, [isHomeUrl, isHover]); 50 | 51 | const setAsHome = () => { 52 | dispatch(setHomeInstance(instance.baseurl, "lemmy")); 53 | }; 54 | 55 | return ( 56 | } 61 | badgeInset="14%" 62 | sx={{ 63 | "--Badge-paddingX": "0px", 64 | // "--Badge-ringSize": "0px", 65 | // "--Badge-borderRadius": "4px", 66 | // "--Badge-ringColor": "success", 67 | // borderRadius: 1, 68 | //remove bg 69 | cursor: "pointer", 70 | "& .MuiBadge-badge": { 71 | // background: "transparent", 72 | // mouse passthru 73 | pointerEvents: "none", 74 | }, 75 | }} 76 | > 77 | 78 | setIsHover(true)} 84 | onMouseLeave={() => setIsHover(false)} 85 | onClick={setAsHome} 86 | sx={style} 87 | /> 88 | 89 | 90 | ); 91 | }); 92 | -------------------------------------------------------------------------------- /frontend/src/lib/storage.ts: -------------------------------------------------------------------------------- 1 | /* global localStorage */ 2 | 3 | class Storage { 4 | private keyName: string; 5 | private store: any; 6 | private writeTimeout?: number; 7 | 8 | constructor() { 9 | this.keyName = "explorer_storage"; 10 | this.store = {}; 11 | 12 | const existing = localStorage.getItem(this.keyName); 13 | if (existing) { 14 | try { 15 | this.store = JSON.parse(existing); 16 | console.log("loaded config", this.store); 17 | } catch (e) { 18 | console.warn("corrupted config in localstorage", e); 19 | this.store = {}; 20 | } 21 | } else { 22 | console.log("no config found in localstorage"); 23 | } 24 | 25 | window.addEventListener("storage", (event: StorageEvent) => { 26 | if (event.key === this.keyName && event.newValue) { 27 | try { 28 | this.store = JSON.parse(event.newValue); 29 | } catch (e) { 30 | console.warn("corrupted config in storage event", e); 31 | this.store = {}; 32 | } 33 | } 34 | }); 35 | } 36 | 37 | /** 38 | * Write the config to localstorage immediately 39 | * 40 | * @returns {void} 41 | */ 42 | writeConfig(): void { 43 | const writeConfigData = JSON.stringify(this.store); 44 | 45 | console.log("wrote config", writeConfigData); 46 | 47 | localStorage.setItem(this.keyName, writeConfigData); 48 | } 49 | 50 | /** 51 | * Debounce writes to localstorage 52 | * 53 | * @returns {void} 54 | */ 55 | private scheduleWrite(): void { 56 | if (this.writeTimeout) { 57 | clearTimeout(this.writeTimeout); 58 | } 59 | 60 | this.writeTimeout = window.setTimeout(() => { 61 | this.writeTimeout = undefined; 62 | this.writeConfig(); 63 | }, 200); 64 | } 65 | 66 | /** 67 | * Get a config value 68 | * 69 | * @param configKey {string} 70 | * @param defaultValue {any} 71 | * @returns {any} 72 | */ 73 | get(configKey: string, defaultValue: any = false): any { 74 | if (this.store.hasOwnProperty(configKey)) { 75 | return this.store[configKey]; 76 | } 77 | 78 | return defaultValue; 79 | } 80 | 81 | /** 82 | * Set a config value 83 | * 84 | * @param configKey {string} 85 | * @param configValue {any} 86 | * @returns {void} 87 | */ 88 | set(configKey: string, configValue: any): void { 89 | this.store[configKey] = configValue; 90 | 91 | return this.scheduleWrite(); 92 | } 93 | 94 | /** 95 | * Remove a config value 96 | * 97 | * @param configKey {string} 98 | * @returns {void} 99 | */ 100 | remove(configKey: string): void { 101 | delete this.store[configKey]; 102 | 103 | return this.scheduleWrite(); 104 | } 105 | } 106 | 107 | const storage = new Storage(); 108 | export default storage; 109 | -------------------------------------------------------------------------------- /frontend/src/App.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; 4 | import { ReactQueryDevtools } from "@tanstack/react-query-devtools"; 5 | 6 | import { Provider } from "react-redux"; 7 | 8 | import { Routes, Route } from "react-router-dom"; 9 | 10 | import { BrowserRouter } from "react-router-dom"; 11 | 12 | import Box from "@mui/joy/Box"; 13 | import Container from "@mui/joy/Container"; 14 | 15 | import Header from "./components/Header/Header"; 16 | 17 | import AppStore from "./store"; 18 | 19 | import Instances from "./pages/Instances"; 20 | import Communities from "./pages/Communities"; 21 | import About from "./pages/About"; 22 | import Join from "./pages/Join"; 23 | import Inspector from "./pages/Inspector"; 24 | import InstanceView from "./pages/InstanceView"; 25 | import MBinMagazines from "./pages/MBinMagazines"; 26 | import PiefedCommunities from "./pages/PiefedCommunities"; 27 | 28 | const queryClient = new QueryClient(); 29 | export default function App() { 30 | return ( 31 | 32 | 33 | 34 | 48 | 49 |
50 | 56 | 57 | {/* } 61 | /> */} 62 | } 66 | /> 67 | } /> 68 | } /> 69 | } /> 70 | } /> 71 | } /> 72 | } /> 73 | 74 | } /> 75 | } /> 76 | {/* } /> */} 77 | 78 | 79 | 80 | 81 | 82 | 83 | ); 84 | } 85 | -------------------------------------------------------------------------------- /crawler/src/lib/CrawlClient.ts: -------------------------------------------------------------------------------- 1 | import logging from "./logging"; 2 | import axios, { AxiosResponse, AxiosError, AxiosInstance, AxiosRequestConfig } from "axios"; 3 | import http from "node:http"; 4 | import https from "node:https"; 5 | 6 | import { HTTPError, CrawlError } from "./error"; 7 | 8 | import { AXIOS_REQUEST_TIMEOUT, CRAWLER_USER_AGENT, CRAWLER_ATTRIB_URL, sleepThreadMs } from "./const"; 9 | 10 | // backoff after failed request 11 | const RETRY_BACKOFF_SECONDS = 2; 12 | export default class CrawlClient { 13 | private axios: AxiosInstance; 14 | 15 | constructor(baseURL: string | undefined = undefined) { 16 | this.axios = axios.create({ 17 | baseURL, 18 | timeout: AXIOS_REQUEST_TIMEOUT, 19 | headers: { 20 | "User-Agent": CRAWLER_USER_AGENT, 21 | "X-Lemmy-SiteUrl": CRAWLER_ATTRIB_URL, 22 | }, 23 | httpAgent: new http.Agent({ keepAlive: true }), 24 | httpsAgent: new https.Agent({ keepAlive: true }), 25 | }); 26 | } 27 | 28 | public async getUrl(url: string, options: AxiosRequestConfig = {}) { 29 | try { 30 | return await this.axios.get(url, options); 31 | } catch (e) { 32 | throw new HTTPError(e.message, { 33 | isAxiosError: true, 34 | code: e.code, 35 | url: e.config.url, 36 | request: e.request || undefined, 37 | response: e.response || undefined, 38 | }); 39 | } 40 | } 41 | 42 | public async postUrl(url: string, data: any = {}, options: AxiosRequestConfig = {}) { 43 | try { 44 | return await this.axios.post(url, data, options); 45 | } catch (e) { 46 | throw new HTTPError(e.message, { 47 | isAxiosError: true, 48 | code: e.code, 49 | url: e.config.url, 50 | request: e.request || null, 51 | response: e.response || null, 52 | }); 53 | } 54 | } 55 | 56 | public async getUrlWithRetry( 57 | url: string, 58 | options: AxiosRequestConfig = {}, 59 | maxRetries: number = 4, 60 | ): Promise { 61 | for (let attempts = 0; attempts < maxRetries; attempts++) { 62 | try { 63 | const axiosResponse: AxiosResponse = await this.axios.get(url, options); 64 | 65 | return axiosResponse; 66 | } catch (e) { 67 | if (attempts < maxRetries - 1) { 68 | const delaySeconds = (attempts + 1) * RETRY_BACKOFF_SECONDS; 69 | 70 | await sleepThreadMs(delaySeconds * 1000); 71 | continue; 72 | } 73 | 74 | logging.error(`getUrlWithRetry: failed to GET ${url} after ${attempts + 1}/${maxRetries} attempts`); 75 | 76 | throw new HTTPError(`${e.message} (attempts: ${attempts + 1})`, { 77 | isAxiosError: true, 78 | code: e.code, 79 | url: e.config.url, 80 | request: e.request || null, 81 | response: e.response || null, 82 | }); 83 | } 84 | } 85 | 86 | throw new CrawlError(`getUrlWithRetry: failed to GET ${url}`, { 87 | url, 88 | options, 89 | maxRetries, 90 | }); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /frontend/src/hooks/useCachedMultipart.ts: -------------------------------------------------------------------------------- 1 | import React, { useState, useMemo } from "react"; 2 | 3 | import axios from "axios"; 4 | import { QueryKey, UseQueryResult, useQuery } from "@tanstack/react-query"; 5 | 6 | import { IMultiPartMetadata } from "../../../types/output"; 7 | 8 | interface CachedDataResult { 9 | isLoading: boolean; 10 | loadingPercent: number; 11 | isSuccess: boolean; 12 | isError: boolean; 13 | error: any; 14 | data: T[] | null; 15 | } 16 | 17 | export default function useCachedMultipart( 18 | queryKey: QueryKey | string, 19 | metadataPath: string, 20 | ): CachedDataResult { 21 | const [loadedChunks, setLoadedChunks] = useState(0); 22 | 23 | const metaQuery: UseQueryResult = useQuery({ 24 | queryKey: [queryKey, metadataPath, "metadata"], 25 | queryFn: () => 26 | axios 27 | .get(`/data/${metadataPath}.json`, { 28 | timeout: 15000, 29 | }) 30 | .then((res) => res.data), 31 | retry: 2, 32 | refetchOnWindowFocus: false, 33 | refetchOnMount: false, 34 | staleTime: Infinity, 35 | cacheTime: Infinity, 36 | }); 37 | 38 | const dataQuery: UseQueryResult = useQuery({ 39 | queryKey: [queryKey, metadataPath, "data"], 40 | enabled: metaQuery.isSuccess, 41 | queryFn: async () => { 42 | const dataChunkLength = metaQuery.data?.count ?? 0; 43 | const requests: Promise[] = Array.from({ length: dataChunkLength }, (_, index) => 44 | axios 45 | .get(`/data/${metadataPath}/${index}.json`, { 46 | timeout: 7500, 47 | }) 48 | .then((res) => { 49 | setLoadedChunks((current: number) => current + 1); 50 | return res.data; 51 | }), 52 | ); 53 | 54 | const results = await Promise.all(requests); 55 | return results.flat(); 56 | }, 57 | retry: 2, 58 | refetchOnWindowFocus: false, 59 | refetchOnMount: false, 60 | staleTime: Infinity, 61 | cacheTime: Infinity, 62 | }); 63 | 64 | if (dataQuery.isSuccess) { 65 | return { 66 | isLoading: false, 67 | loadingPercent: 100, 68 | isSuccess: true, 69 | isError: false, 70 | error: null, 71 | data: dataQuery.data, 72 | }; 73 | } 74 | 75 | // error result 76 | if (dataQuery.isError) { 77 | console.log("some results are error"); 78 | 79 | return { 80 | isLoading: false, 81 | loadingPercent: 100, 82 | isSuccess: false, 83 | isError: true, 84 | error: dataQuery.error, 85 | data: null, 86 | }; 87 | } 88 | 89 | const loadingPercent = metaQuery.data?.count ? (loadedChunks / metaQuery.data.count) * 100 : 0; 90 | console.log("useCachedMultipart loadingPercent", loadingPercent); 91 | 92 | return { 93 | isLoading: true, 94 | loadingPercent, 95 | isSuccess: false, 96 | isError: false, 97 | error: false, 98 | data: null, 99 | }; 100 | } 101 | -------------------------------------------------------------------------------- /frontend/src/pages/Inspector.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | import { Routes, Route, useNavigate } from "react-router-dom"; 4 | 5 | import Tabs from "@mui/joy/Tabs"; 6 | import TabList from "@mui/joy/TabList"; 7 | import Tab from "@mui/joy/Tab"; 8 | import TabPanel from "@mui/joy/TabPanel"; 9 | 10 | import Container from "@mui/joy/Container"; 11 | import Box from "@mui/joy/Box"; 12 | 13 | import Overview from "../components/Inspector/Overview"; 14 | import Versions from "../components/Inspector/Versions"; 15 | import Sus from "../components/Inspector/Sus"; 16 | import VersionChart from "../components/Inspector/VersionChart"; 17 | 18 | export default function Inspector() { 19 | const navigate = useNavigate(); 20 | 21 | const [tabIndex, setTabIndex] = React.useState(0); 22 | 23 | const tabDefs = [ 24 | { 25 | label: "Overview", 26 | nav: "/inspect", 27 | component: , 28 | }, 29 | { 30 | label: "Version Distribution", 31 | nav: "/inspect/versions", 32 | component: , 33 | }, 34 | { 35 | label: "Suspicious Instances", 36 | nav: "/inspect/sus", 37 | component: , 38 | }, 39 | // { 40 | // label: "Instance Debugger", 41 | // nav: "/inspect/debug", 42 | // component: , 43 | // }, 44 | { 45 | label: "Version Chart", 46 | nav: "/inspect/version-chart", 47 | component: , 48 | }, 49 | ]; 50 | 51 | // restore tab 52 | React.useEffect(() => { 53 | const path = window.location.pathname; 54 | 55 | tabDefs.forEach((tab, index) => { 56 | if (path === tab.nav) { 57 | setTabIndex(index); 58 | } 59 | }); 60 | }, []); 61 | 62 | const changeView = (index) => { 63 | setTabIndex(index); 64 | navigate(tabDefs[index].nav); 65 | }; 66 | 67 | return ( 68 | 69 | changeView(value)} 72 | aria-label="tabs" 73 | defaultValue={0} 74 | sx={{ 75 | bgcolor: "background.body", 76 | alignItems: "center", 77 | }} 78 | > 79 | 80 | {tabDefs.map((tab, index) => ( 81 | 82 | {tab.label} 83 | 84 | ))} 85 | 86 | ({ 88 | bgcolor: "background.body", 89 | width: "100%", 90 | p: 2, 91 | })} 92 | > 93 | 94 | {tabDefs.map((tab, index) => ( 95 | {tab.component}} 99 | /> 100 | ))} 101 | 102 | 103 | 104 | 105 | ); 106 | } 107 | -------------------------------------------------------------------------------- /frontend/src/components/Shared/MultiDataLineGraph.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { useColorScheme } from "@mui/joy/styles"; 3 | 4 | import Box from "@mui/joy/Box"; 5 | 6 | import moment from "moment"; 7 | import { SimpleNumberFormat } from "./Display"; 8 | 9 | import { ResponsiveContainer, Line, LineChart, Tooltip, XAxis, YAxis } from "recharts"; 10 | 11 | type IMultiDataLineGraphProps = { 12 | dataSeries: any[]; 13 | dataSeriesInfo: { 14 | yAxisName: string; 15 | yAxisKey: string; 16 | yAxisColor: string; 17 | minValue?: number; 18 | maxValue?: number; 19 | }[]; 20 | }; 21 | 22 | export default function MultiDataLineGraph({ dataSeries, dataSeriesInfo }: IMultiDataLineGraphProps) { 23 | const { mode } = useColorScheme(); 24 | 25 | const scale = "linear"; //scaleLog().base(Math.E); 26 | 27 | return ( 28 | 29 | 30 | {/* Time Axis */} 31 | moment(unixTime).format("DD-MM-YYYY")} 36 | type="number" 37 | padding={{ left: 30, right: 30 }} 38 | /> 39 | 40 | { 43 | if (active && payload && payload.length) { 44 | return ( 45 | 46 |

{moment(label).format("DD-MM-YYYY")}

47 | {payload.map((i) => ( 48 |

49 | {i.name}: 50 |

51 | ))} 52 | 53 | ); 54 | } 55 | return null; 56 | }} 57 | /> 58 | 59 | {/* Count Axis */} 60 | {dataSeriesInfo.map((dataSeries, index) => ( 61 | value.toLocaleString()} 74 | yAxisId={index + 1} 75 | /> 76 | ))} 77 | 78 | {dataSeriesInfo.map((dataSeries, index) => ( 79 | 88 | ))} 89 | 90 | 91 | ); 92 | } 93 | -------------------------------------------------------------------------------- /.github/workflows/aws-deploy-dev.yaml: -------------------------------------------------------------------------------- 1 | name: deploy-aws-dev 2 | 3 | on: 4 | push: 5 | branches: 6 | - develop 7 | paths: 8 | - .github/workflows/aws-deploy-dev.yaml 9 | - crawler/** 10 | - frontend/** 11 | - cdk/** 12 | - types/** 13 | 14 | # # run every 6 hours 15 | # only runs on the base branch, weird, github???? 16 | # schedule: 17 | # - cron: "0 */6 * * *" 18 | 19 | # Allows you to run this workflow manually from the Actions tab 20 | workflow_dispatch: 21 | 22 | concurrency: 23 | group: develop 24 | cancel-in-progress: false 25 | 26 | env: 27 | NODE_VERSION: 22.17.0 28 | 29 | jobs: 30 | aws_deploy_dev: 31 | runs-on: ubuntu-latest 32 | 33 | environment: 34 | name: develop 35 | url: https://develop.lemmyverse.net 36 | 37 | steps: 38 | # https://github.com/actions/toolkit/issues/946#issuecomment-1590016041 39 | - name: root suid tar 40 | run: sudo chown root:root /bin/tar && sudo chmod u+s /bin/tar 41 | 42 | - uses: actions/checkout@v4 43 | 44 | - name: Use Node.js ${{ env.NODE_VERSION }} 45 | uses: actions/setup-node@v4 46 | with: 47 | node-version: ${{ env.NODE_VERSION }} 48 | 49 | # Download + Start Redis Docker 50 | - name: Get Dump + Start Redis 51 | uses: ./.github/actions/start-redis 52 | with: 53 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} 54 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 55 | s3-bucket: ${{ vars.BUILD_S3_BUCKET }} 56 | 57 | # Yarn Install ./crawler 58 | - name: Yarn Install ./crawler 59 | uses: ./.github/actions/yarn-install 60 | with: 61 | working-directory: ./crawler 62 | 63 | # Run Crawler Output Script 64 | - name: Run Health Script 65 | run: yarn health 66 | working-directory: ./crawler 67 | 68 | - name: Run Output Script 69 | run: yarn output 70 | working-directory: ./crawler 71 | 72 | # store the json files as artifacts 73 | - name: archive json artifacts 74 | uses: actions/upload-artifact@v4 75 | with: 76 | name: dist-json-bundle 77 | path: | 78 | ./frontend/public/data/ 79 | 80 | # Yarn Install ./frontend 81 | - name: Yarn Install ./frontend 82 | uses: ./.github/actions/yarn-install 83 | with: 84 | working-directory: ./frontend 85 | 86 | # Build Frontend to ./frontend/dist 87 | - name: Build the Frontend 88 | run: yarn build 89 | working-directory: ./frontend 90 | 91 | - name: Create CDK Config JSON 92 | id: create-json 93 | uses: jsdaniell/create-json@v1.2.3 94 | with: 95 | dir: ./cdk 96 | name: "config.json" 97 | json: ${{ vars.CONFIG_JSON }} 98 | 99 | # CDK Deployment 100 | - name: CDK Deployment 101 | uses: ./.github/actions/cdk-deploy 102 | with: 103 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} 104 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 105 | -------------------------------------------------------------------------------- /.github/workflows/crawler-checks.yaml: -------------------------------------------------------------------------------- 1 | name: crawler-checks 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - .github/workflows/crawler-checks.yaml 7 | - crawler/** 8 | - types/** 9 | 10 | # Allows you to run this workflow manually from the Actions tab 11 | workflow_dispatch: 12 | 13 | env: 14 | NODE_VERSION: 22.17.0 15 | 16 | jobs: 17 | crawler-checks-style: 18 | runs-on: ubuntu-latest 19 | 20 | steps: 21 | - uses: actions/checkout@v4 22 | 23 | - name: Use Node.js ${{ env.NODE_VERSION }} 24 | uses: actions/setup-node@v4 25 | with: 26 | node-version: ${{ env.NODE_VERSION }} 27 | 28 | # Install + Cache Crawler Dependencies 29 | - name: Cache Node Modules | Crawler 30 | id: cache-crawler-yarn 31 | uses: actions/cache@v4 32 | env: 33 | cache-name: cache-crawler-yarn 34 | with: 35 | path: ./crawler/node_modules/ 36 | key: cache-crawler-yarn-${{ hashFiles('crawler/yarn.lock') }} 37 | 38 | - name: Install Node Modules | Crawler 39 | if: steps.cache-crawler-yarn.outputs.cache-hit != 'true' 40 | run: yarn --frozen-lockfile 41 | working-directory: ./crawler 42 | 43 | - name: Check Style | Crawler 44 | run: yarn format 45 | working-directory: ./crawler 46 | 47 | crawler-checks-types: 48 | runs-on: ubuntu-latest 49 | 50 | steps: 51 | - uses: actions/checkout@v4 52 | 53 | - name: Use Node.js ${{ env.NODE_VERSION }} 54 | uses: actions/setup-node@v4 55 | with: 56 | node-version: ${{ env.NODE_VERSION }} 57 | 58 | # Install + Cache Crawler Dependencies 59 | - name: Cache Node Modules | Crawler 60 | id: cache-crawler-yarn 61 | uses: actions/cache@v4 62 | env: 63 | cache-name: cache-crawler-yarn 64 | with: 65 | path: ./crawler/node_modules/ 66 | key: cache-crawler-yarn-${{ hashFiles('crawler/yarn.lock') }} 67 | 68 | - name: Install Node Modules | Crawler 69 | if: steps.cache-crawler-yarn.outputs.cache-hit != 'true' 70 | run: yarn --frozen-lockfile 71 | working-directory: ./crawler 72 | 73 | - name: Check Style | Crawler 74 | run: yarn check:ts 75 | working-directory: ./crawler 76 | 77 | crawler-checks-tests: 78 | runs-on: ubuntu-latest 79 | 80 | steps: 81 | - uses: actions/checkout@v4 82 | 83 | - name: Use Node.js ${{ env.NODE_VERSION }} 84 | uses: actions/setup-node@v4 85 | with: 86 | node-version: ${{ env.NODE_VERSION }} 87 | 88 | - name: Cache Node Modules | Crawler 89 | id: cache-crawler-yarn 90 | uses: actions/cache@v4 91 | env: 92 | cache-name: cache-crawler-yarn 93 | with: 94 | path: ./crawler/node_modules/ 95 | key: cache-crawler-yarn-${{ hashFiles('crawler/yarn.lock') }} 96 | 97 | - name: Install Node Modules | Crawler 98 | if: steps.cache-crawler-yarn.outputs.cache-hit != 'true' 99 | run: yarn --frozen-lockfile 100 | working-directory: ./crawler 101 | 102 | - name: Run Tests | Crawler 103 | run: yarn test 104 | working-directory: ./crawler 105 | -------------------------------------------------------------------------------- /frontend/src/components/Inspector/VersionChart.tsx: -------------------------------------------------------------------------------- 1 | import React, { useMemo } from "react"; 2 | 3 | import useQueryCache from "../../hooks/useQueryCache"; 4 | import moment from "moment"; 5 | 6 | import { useWindowSize } from "@react-hook/window-size"; 7 | 8 | import Box from "@mui/joy/Box"; 9 | 10 | import { AreaChart, Area, XAxis, YAxis, CartesianGrid, Tooltip, ResponsiveContainer } from "recharts"; 11 | 12 | const CustomTooltip = (props) => { 13 | const { active, payload, label } = props; 14 | 15 | console.info("SITEETEte", props, moment(Number(label))); 16 | if (active && payload && payload.length) { 17 | return ( 18 | 26 |

{moment(Number(label)).format("DD-MM-YYYY HH:mm")}

27 | {payload.map((i) => ( 28 |

29 | {i.name}: {i.value} 30 |

31 | ))} 32 | 33 | ); 34 | } 35 | 36 | return null; 37 | }; 38 | 39 | export default function VersionChart() { 40 | const { 41 | isLoading, 42 | isSuccess, 43 | isError, 44 | error, 45 | data: metricsData, 46 | } = useQueryCache("metrics.series", "metrics.series"); 47 | 48 | const [windowWidth, windowHeight] = useWindowSize(); 49 | 50 | const dataset = useMemo(() => { 51 | if (!metricsData) return []; 52 | 53 | const versionsAgg = metricsData.versions; 54 | 55 | return versionsAgg; 56 | }, [metricsData]); 57 | 58 | const datasetSeries = useMemo(() => { 59 | if (!metricsData) return []; 60 | 61 | const versionKeys = metricsData.versionKeys; 62 | 63 | return versionKeys; 64 | }, [metricsData]); 65 | 66 | // console.log(dataset); 67 | console.log("datasetSeries", datasetSeries, dataset); 68 | 69 | if (isLoading) return "Loading..."; 70 | if (isError || !isSuccess) return "An error has occurred: " + error.message; 71 | 72 | return ( 73 | 74 | 75 | 81 | 82 | 83 | moment(unixTime).format("DD-MM-YYYY")} 88 | type="number" 89 | padding={{ left: 30, right: 30 }} 90 | /> 91 | 92 | 93 | 94 | 98 | 99 | {datasetSeries.map((version, index) => { 100 | return ( 101 | 109 | ); 110 | })} 111 | 112 | 113 | 114 | ); 115 | } 116 | -------------------------------------------------------------------------------- /types/output.ts: -------------------------------------------------------------------------------- 1 | import { IUptimeNodeData } from "./storage"; 2 | 3 | export type IMetaDataOutput = { 4 | instances: number; 5 | communities: number; 6 | mbin_instances: number; // @ NEW 7 | magazines: number; 8 | 9 | piefed_instances: number; 10 | piefed_communities: number; 11 | 12 | fediverse: number; 13 | 14 | time: number; 15 | package: string; 16 | version: string; 17 | 18 | linked?: any; 19 | allowed?: any; 20 | blocked?: any; 21 | }; 22 | 23 | export interface IMultiPartMetadata { 24 | count: number; 25 | } 26 | 27 | // -1: Unknown, 0: Closed, 1: RequireApplication, 2: Open 28 | export type IRegistrationMode = -1 | 0 | 1 | 2; 29 | 30 | export type IInstanceDataOutput = { 31 | baseurl: string; 32 | url: string; 33 | name: string; 34 | desc: string; 35 | downvotes: boolean; 36 | nsfw: boolean; 37 | create_admin: boolean; // community creation restricted to admins 38 | private: boolean; 39 | reg_mode: IRegistrationMode; 40 | fed: boolean; 41 | version: string; 42 | open: boolean; 43 | usage: Object; 44 | counts: Object; 45 | icon: string; 46 | banner: string; 47 | langs: string[]; 48 | date: string; 49 | published: number; 50 | time: number; 51 | score: number; 52 | uptime?: IUptimeNodeData; 53 | isSuspicious: boolean; 54 | metrics: Object | null; 55 | tags: string[]; 56 | susReason: string[]; 57 | trust: any; 58 | blocks: { 59 | incoming: number; 60 | outgoing: number; 61 | }; 62 | blocked: string[]; 63 | 64 | admins: string[]; 65 | }; 66 | 67 | export type ICommunityDataOutput = { 68 | baseurl: string; 69 | url: string; 70 | name: string; 71 | title: string; 72 | desc: string; 73 | icon: string | null; 74 | banner: string | null; 75 | nsfw: boolean; 76 | counts: Object; 77 | published: number; 78 | time: number; 79 | isSuspicious: boolean; 80 | score: number; 81 | }; 82 | 83 | export type IMBinInstanceOutput = { 84 | // actor_id: string; 85 | // title: string; 86 | // name: string; 87 | // preferred: string; 88 | // baseurl: string; 89 | // summary: string; 90 | // sensitive: boolean; 91 | // postingRestrictedToMods: boolean; 92 | // icon: string; 93 | // published: string; 94 | // updated: string; 95 | // followers: number; 96 | // time: number; 97 | }; 98 | 99 | export type IMBinMagazineOutput = { 100 | baseurl: string; 101 | magazineId: number; 102 | title: string; 103 | name: string; 104 | description: string; 105 | isAdult: boolean; 106 | postingRestrictedToMods: boolean; 107 | icon: string | null; 108 | subscriptions: number; 109 | posts: number; 110 | time: number; 111 | }; 112 | 113 | export type IPiefedCommunityDataOutput = { 114 | baseurl: string; 115 | // url: string; 116 | name: string; 117 | title: string; 118 | // desc: string; 119 | icon: string | null; 120 | description: string | null; 121 | // banner: string | null; 122 | nsfw: boolean; 123 | // counts: Object; 124 | subscriptions_count: number; 125 | post_count: number; 126 | published: string; 127 | time: number; 128 | // isSuspicious: boolean; 129 | // score: number; 130 | restricted_to_mods: boolean; 131 | }; 132 | 133 | export type IFediverseDataOutput = { 134 | url: string; 135 | software: string; 136 | version: string; 137 | }; 138 | 139 | export type IClassifiedErrorOutput = { 140 | baseurl: string; 141 | time: number; 142 | error: string; 143 | type?: string; 144 | }; 145 | -------------------------------------------------------------------------------- /frontend/src/components/Shared/StatGridCards.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | import { ColorPaletteProp } from "@mui/joy/styles/types/colorSystem"; 4 | 5 | import CardContent from "@mui/joy/CardContent"; 6 | 7 | import Card from "@mui/joy/Card"; 8 | import Typography from "@mui/joy/Typography"; 9 | 10 | import { SimpleNumberFormat } from "../Shared/Display"; 11 | 12 | type IStringStatProps = { 13 | title: string | React.ReactElement; 14 | value: React.ReactNode; 15 | icon?: React.ReactElement; 16 | color?: ColorPaletteProp; 17 | description?: React.ReactElement | string; 18 | sx?: object; 19 | }; 20 | 21 | export function StringStat({ 22 | title, 23 | value, 24 | icon = undefined, 25 | color = "primary", 26 | description = "", 27 | sx = {}, 28 | }: IStringStatProps) { 29 | let iconClone = null; 30 | 31 | if (icon) 32 | iconClone = React.cloneElement(icon, { 33 | sx: { 34 | ...icon.props.sx, 35 | fontSize: "xl5", 36 | color: "#fff", 37 | p: 0, 38 | 39 | // mb: "var(--Card-padding)", 40 | // display: "inline", 41 | }, 42 | }); 43 | 44 | return ( 45 | 67 | {iconClone && ( 68 | 88 | {iconClone} 89 | 90 | )} 91 | 108 | {title} 109 | 110 | {value} 111 | 112 | {description && {description}} 113 | 114 | 115 | ); 116 | } 117 | 118 | type INumberStatProps = { 119 | value: number; 120 | } & IStringStatProps; 121 | 122 | export function NumberStat({ value, ...props }: INumberStatProps) { 123 | return } />; 124 | } 125 | -------------------------------------------------------------------------------- /crawler/src/lib/logging.ts: -------------------------------------------------------------------------------- 1 | import util from "node:util"; 2 | 3 | import pino from "pino"; 4 | import pretty from "pino-pretty"; 5 | 6 | import { LOG_LEVEL } from "./const"; 7 | 8 | const formatDuration = (ms: number) => { 9 | const milliseconds = (ms % 1000).toString().padStart(3, "0"); 10 | 11 | const seconds = Math.floor(ms / 1000); 12 | const minutes = Math.floor(seconds / 60); 13 | const hours = Math.floor(minutes / 60); 14 | const days = Math.floor(hours / 24); 15 | const months = Math.floor(days / 30); 16 | 17 | if (months > 0) { 18 | return `${months}mo+`; 19 | } else if (days > 0) { 20 | return `${days}d ${hours % 24}h ${minutes % 60}m ${seconds % 60}.${milliseconds}s`; 21 | } else if (hours > 0) { 22 | return `${hours}h ${minutes % 60}m ${seconds % 60}.${milliseconds}s`; 23 | } else if (minutes > 0) { 24 | return `${minutes}m ${seconds % 60}.${milliseconds}s`; 25 | } else { 26 | return `${seconds}.${milliseconds}s`; 27 | } 28 | }; 29 | 30 | const levelColours: Record = { 31 | 10: "\x1b[90m", // trace - grey 32 | 20: "\x1b[90m", // debug - grey 33 | 30: "\x1b[32m", // info - green 34 | 40: "\x1b[33m", // warn - yellow 35 | 50: "\x1b[31m", // error - red 36 | 60: "\x1b[1;31m", // fatal - bold red 37 | }; 38 | 39 | const stringifyArg = (arg: any) => { 40 | if (typeof arg === "string") return arg; 41 | try { 42 | const json = JSON.stringify(arg); 43 | return json === undefined ? String(arg) : json; 44 | } catch { 45 | return util.inspect(arg, { depth: null, breakLength: Infinity }); 46 | } 47 | }; 48 | const formatMessage = (log: Record, messageKey: string) => { 49 | const baseMsg = log[messageKey] as string; 50 | const colour = levelColours[log.level] ?? ""; 51 | const reset = colour ? "\x1b[0m" : ""; 52 | 53 | return `${colour}${baseMsg}${reset}`; 54 | }; 55 | const stream = pretty({ 56 | colorize: true, 57 | translateTime: "yyyy-mm-dd HH:MM:ss.l", 58 | ignore: "pid,hostname", 59 | 60 | customLevels: { 61 | trace: 10, 62 | debug: 20, 63 | info: 30, 64 | warn: 40, 65 | error: 50, 66 | fatal: 60, 67 | }, 68 | 69 | messageFormat: (log, messageKey) => formatMessage(log, messageKey), 70 | }); 71 | 72 | const baseLogger = pino({ level: LOG_LEVEL }, stream); 73 | 74 | type ILogFunction = (message: string, ...args: any[]) => void; 75 | 76 | type ILogging = { 77 | trace: ILogFunction; 78 | debug: ILogFunction; 79 | info: ILogFunction; 80 | warn: ILogFunction; 81 | error: ILogFunction; 82 | fatal: ILogFunction; 83 | table: (tableTitle: string, ...data: any) => void; 84 | formatDuration: (ms: number) => string; 85 | }; 86 | 87 | type Level = "trace" | "debug" | "info" | "warn" | "error" | "fatal"; 88 | 89 | const buildLog = (level: Level): ILogFunction => { 90 | return (message: string, ...args: any[]) => { 91 | const extra = args.map(stringifyArg).join(" "); 92 | const finalMessage = extra ? `${message} ${extra}` : message; 93 | (baseLogger as any)[level](finalMessage); 94 | }; 95 | }; 96 | 97 | const logging: ILogging = { 98 | trace: buildLog("trace"), 99 | debug: buildLog("debug"), 100 | info: buildLog("info"), 101 | warn: buildLog("warn"), 102 | error: buildLog("error"), 103 | fatal: buildLog("fatal"), 104 | 105 | table: (tableTitle: string, ...data: any[]) => { 106 | console.log(); 107 | console.info(tableTitle); 108 | console.table(...data); 109 | console.log(); 110 | }, 111 | formatDuration: formatDuration, 112 | }; 113 | 114 | export default logging; 115 | -------------------------------------------------------------------------------- /crawler/src/queue/BaseQueue.ts: -------------------------------------------------------------------------------- 1 | import BeeQueue from "bee-queue"; 2 | 3 | import logging from "../lib/logging"; 4 | import storage from "../lib/crawlStorage"; 5 | 6 | import { CrawlTooRecentError } from "../lib/error"; 7 | import { REDIS_URL, CRAWL_TIMEOUT } from "../lib/const"; 8 | 9 | export type IJobData = { baseUrl?: string; community?: string }; 10 | 11 | export type IJobProcessor = (processorConfig: IJobData) => Promise; 12 | 13 | export type ISuccessCallback = ((resultData: T) => void) | null; 14 | 15 | export default class BaseQueue { 16 | protected queueName: string; 17 | protected logPrefix: string; 18 | 19 | // public to be accessed to get health etc 20 | public queue: BeeQueue; 21 | 22 | protected jobProcessor: IJobProcessor; 23 | 24 | constructor(isWorker: boolean, queueName: string, jobProcessor: IJobProcessor) { 25 | this.queueName = queueName; 26 | 27 | this.queue = new BeeQueue(queueName, { 28 | redis: REDIS_URL, 29 | removeOnSuccess: true, 30 | removeOnFailure: true, 31 | isWorker: isWorker, 32 | getEvents: isWorker, 33 | }); 34 | 35 | this.logPrefix = `[BaseQueue] [${this.queueName}]`; 36 | 37 | this.jobProcessor = jobProcessor; 38 | 39 | // report failures! 40 | this.queue.on("failed", (job, err) => { 41 | logging.error(`${this.logPrefix} job:${job.id} failed with error: ${err.message}`, job, err); 42 | }); 43 | 44 | if (isWorker) this.process(); 45 | } 46 | 47 | async createJob(jobId: string, jobData: any, onSuccess: ISuccessCallback = null) { 48 | const job = this.queue.createJob(jobData); 49 | logging.trace(`${this.logPrefix} createJob`, jobData); 50 | 51 | job.on("succeeded", (result) => { 52 | logging.trace(`${this.logPrefix} ${job.id} succeeded`, jobData); 53 | onSuccess && onSuccess(result); 54 | }); 55 | 56 | await job.timeout(CRAWL_TIMEOUT.DEFAULT).setId(jobId).save(); 57 | } 58 | 59 | process(): void { 60 | this.queue.process(async (job): Promise => { 61 | await storage.connect(); 62 | 63 | try { 64 | logging.info(`${this.logPrefix} [${job.data.baseUrl}] Starting Job Processor`); 65 | 66 | const timeStart = Date.now(); 67 | 68 | const resultData: T | null = await this.jobProcessor(job.data); 69 | 70 | const timeEnd = Date.now(); 71 | const duration = timeEnd - timeStart; 72 | logging.info( 73 | `${this.logPrefix} [${job.data.baseUrl}] Job Processor completed in ${logging.formatDuration(duration)}`, 74 | ); 75 | 76 | if (!resultData) { 77 | logging.debug(`${this.logPrefix} [${job.data.baseUrl}] Processor returned null or undefined`); 78 | return; 79 | } 80 | 81 | return resultData; 82 | } catch (error) { 83 | if (error instanceof CrawlTooRecentError) { 84 | logging.warn(`${this.logPrefix} [${job.data.baseUrl}] CrawlTooRecentError: ${error.message}`); 85 | } else { 86 | // store all other errors 87 | const errorDetail = { 88 | error: error.message, 89 | stack: error.stack, 90 | isAxiosError: error.isAxiosError, 91 | requestUrl: error.isAxiosError ? error.request.url : null, 92 | time: Date.now(), 93 | }; 94 | 95 | // if (error instanceof CrawlError || error instanceof AxiosError) { 96 | await storage.tracking.upsertError(this.queueName, job.data.baseUrl, errorDetail); 97 | 98 | logging.error(`${this.logPrefix} [${job.data.baseUrl}] Error: ${error.message}`, error); 99 | } 100 | 101 | return; 102 | } finally { 103 | await storage.close(); 104 | } 105 | }); 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /crawler/src/output/classifier.ts: -------------------------------------------------------------------------------- 1 | import logging from "../lib/logging"; 2 | 3 | export default class OutputClassifier { 4 | // given an error message from redis, figure out what it relates to.. 5 | static findErrorType(errorMessage: string): string { 6 | // console.log("findErrorType", errorMessage); 7 | 8 | if (!errorMessage || errorMessage.length === 0) { 9 | logging.error("empty error message"); 10 | return "unknown"; 11 | } 12 | 13 | if (errorMessage.includes("EAI_AGAIN")) { 14 | return "dnsFailed"; 15 | } 16 | 17 | if (errorMessage.includes("ENETUNREACH") || errorMessage.includes("EHOSTUNREACH")) { 18 | return "hostUnreachable"; 19 | } 20 | 21 | if (errorMessage.includes("ECONNREFUSED")) { 22 | return "connectRefused"; 23 | } 24 | 25 | if ( 26 | errorMessage.includes("ECONNRESET") || 27 | errorMessage.includes("EPROTO") || 28 | errorMessage.includes("socket hang up") || 29 | errorMessage.includes("Request failed with status code 522") || 30 | errorMessage.includes("Client network socket disconnected") 31 | ) { 32 | return "connectException"; 33 | } 34 | 35 | if ( 36 | errorMessage.includes("ENOENT") || 37 | errorMessage.includes("ENOTFOUND") || 38 | errorMessage.includes("Request failed with status code 401") 39 | ) { 40 | return "contentMissing"; 41 | } 42 | 43 | if (errorMessage.includes("timeout of")) { 44 | return "timeout"; 45 | } 46 | 47 | if ( 48 | errorMessage.includes("self-signed certificate") || 49 | errorMessage.includes("does not match certificate's altnames") || 50 | errorMessage.includes("tlsv1 unrecognized name") || 51 | errorMessage.includes("tlsv1 alert internal error") || 52 | errorMessage.includes("ssl3_get_record:wrong version number") || 53 | errorMessage.includes("unable to verify the first certificate") || 54 | errorMessage.includes("unable to get local issuer certificate") || 55 | errorMessage.includes("certificate has expired") 56 | ) { 57 | return "sslException"; 58 | } 59 | 60 | if (errorMessage.includes("baseUrl is not a valid domain")) { 61 | return "invalidBaseUrl"; 62 | } 63 | 64 | if ( 65 | errorMessage.includes("code 300") || 66 | errorMessage.includes("code 400") || 67 | errorMessage.includes("code 403") || 68 | errorMessage.includes("code 404") || 69 | errorMessage.includes("code 406") || 70 | errorMessage.includes("code 410") || 71 | errorMessage.includes("code 500") || 72 | errorMessage.includes("code 502") || 73 | errorMessage.includes("code 503") || 74 | errorMessage.includes("code 520") || 75 | errorMessage.includes("code 521") || 76 | errorMessage.includes("code 523") || 77 | errorMessage.includes("code 525") || 78 | errorMessage.includes("code 526") || 79 | errorMessage.includes("code 530") || 80 | errorMessage.includes("Maximum number of redirects exceeded") 81 | ) { 82 | return "httpException"; 83 | } 84 | 85 | if ( 86 | errorMessage.includes("no diaspora rel in") || 87 | errorMessage.includes("wellKnownInfo.data.links is not iterable") || 88 | errorMessage.includes("missing /.well-known/nodeinfo links") 89 | ) { 90 | return "httpException"; 91 | } 92 | 93 | if (errorMessage.includes("not a lemmy instance")) { 94 | return "notLemmy"; 95 | } 96 | 97 | if ( 98 | errorMessage.includes("invalid actor id") || 99 | errorMessage.includes("actor id does not match instance domain") 100 | ) { 101 | return "invalidActorId"; 102 | } 103 | 104 | logging.trace("unhandled error", errorMessage); 105 | return "unknown"; 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /frontend/public/icons/orig/MBin_Logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 13 | 15 | 23 | 25 | 29 | 33 | 34 | 42 | 44 | 48 | 52 | 53 | 54 | 57 | 61 | 66 | 71 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /.github/workflows/publish-pages.yaml: -------------------------------------------------------------------------------- 1 | name: publish-pages 2 | 3 | on: 4 | # Runs on pushes targeting the default branch 5 | push: 6 | branches: 7 | - main 8 | paths: 9 | - .github/workflows/publish-pages.yaml 10 | - pages/** 11 | - frontend/** 12 | - crawler/** 13 | - types/** 14 | 15 | # Allows you to run this workflow manually from the Actions tab 16 | workflow_dispatch: 17 | 18 | # run every 6 hours 19 | schedule: 20 | - cron: "0 */6 * * *" 21 | 22 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 23 | permissions: 24 | contents: read 25 | pages: write 26 | id-token: write 27 | 28 | # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. 29 | # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. 30 | concurrency: 31 | group: pages 32 | cancel-in-progress: false 33 | 34 | env: 35 | NODE_VERSION: 22.17.0 36 | 37 | jobs: 38 | # download the redis dump from s3 39 | # create the json files for the redis data 40 | # upload the json files to github artifacts 41 | publish_pages: 42 | runs-on: ubuntu-latest 43 | timeout-minutes: 10 44 | 45 | environment: 46 | name: github-pages 47 | url: ${{ steps.deployment.outputs.page_url }} 48 | 49 | steps: 50 | # https://github.com/actions/toolkit/issues/946#issuecomment-1590016041 51 | - name: root suid tar 52 | run: sudo chown root:root /bin/tar && sudo chmod u+s /bin/tar 53 | 54 | - uses: actions/checkout@v4 55 | 56 | - name: Setup Pages 57 | uses: actions/configure-pages@v5 58 | 59 | - name: Use Node.js ${{ env.NODE_VERSION }} 60 | uses: actions/setup-node@v4 61 | with: 62 | node-version: ${{ env.NODE_VERSION }} 63 | 64 | # Download + Start Redis Docker 65 | - name: Get Dump + Start Redis 66 | uses: ./.github/actions/start-redis 67 | with: 68 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} 69 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 70 | s3-bucket: ${{ vars.BUILD_S3_BUCKET }} 71 | 72 | # Yarn Install ./crawler 73 | - name: Yarn Install ./crawler 74 | uses: ./.github/actions/yarn-install 75 | with: 76 | working-directory: ./crawler 77 | 78 | # Run Crawler Output Script 79 | - name: Run Health Script 80 | run: yarn health 81 | working-directory: ./crawler 82 | 83 | - name: Run Output Script 84 | run: yarn output 85 | working-directory: ./crawler 86 | 87 | # Create ZIP Archive of JSON Files in ./frontend/public/data/ 88 | - name: Create ZIP Archive of JSON Files 89 | working-directory: ./frontend/public/data 90 | run: | 91 | zip -r -9 -q json-bundle.zip ./* 92 | du -sh json-bundle.zip 93 | 94 | - name: copy the json files to github pages 95 | working-directory: ./ 96 | run: cp -r ./frontend/public/data/ ./pages/public/ 97 | 98 | - name: copy the raw db dump file to github pages 99 | working-directory: ./ 100 | run: cp -r ./.redis/dump.rdb ./pages/public/data/lemmyverse.rdb 101 | 102 | # Yarn Install ./pages 103 | - name: Yarn Install ./pages 104 | uses: ./.github/actions/yarn-install 105 | with: 106 | working-directory: ./pages 107 | 108 | - name: Build the Frontend for Pages 109 | run: yarn build 110 | working-directory: ./pages 111 | 112 | # upload pages 113 | - name: upload github pages artifacts 114 | uses: actions/upload-pages-artifact@v3 115 | with: 116 | path: ./pages/dist/ 117 | 118 | - name: Deploy to GitHub Pages 119 | id: deployment 120 | uses: actions/deploy-pages@v4 121 | -------------------------------------------------------------------------------- /crawler/test/crawl/instanceCommunity.test.ts: -------------------------------------------------------------------------------- 1 | import { jest } from "@jest/globals"; 2 | 3 | // in-memory storage mock 4 | const instanceStore: Record = {}; 5 | const communityStore: Record = {}; 6 | 7 | const storageMock = { 8 | connect: jest.fn(), 9 | close: jest.fn(), 10 | instance: { 11 | upsert: jest.fn((key: string, value: any) => { 12 | instanceStore[key] = value; 13 | }), 14 | getOne: jest.fn((key: string) => instanceStore[key] || null), 15 | setTrackedAttribute: jest.fn(), 16 | }, 17 | community: { 18 | upsert: jest.fn((baseUrl: string, community: any) => { 19 | communityStore[`${baseUrl}:${community.community.name}`] = community; 20 | }), 21 | delete: jest.fn(), 22 | setTrackedAttribute: jest.fn(), 23 | }, 24 | tracking: { 25 | getLastCrawl: jest.fn<() => Promise>().mockResolvedValue(null), 26 | getOneError: jest.fn<() => Promise>().mockResolvedValue(null), 27 | setLastCrawl: jest.fn(), 28 | upsertError: jest.fn(), 29 | }, 30 | fediverse: { upsert: jest.fn(), getOne: jest.fn<() => Promise>().mockResolvedValue(null) }, 31 | mbin: { upsert: jest.fn(), setTrackedAttribute: jest.fn() }, 32 | piefed: { upsert: jest.fn(), setTrackedAttribute: jest.fn() }, 33 | }; 34 | 35 | jest.mock("../../src/lib/crawlStorage", () => ({ 36 | __esModule: true, 37 | default: storageMock, 38 | })); 39 | 40 | jest.mock("../../src/queue/community_list", () => ({ 41 | __esModule: true, 42 | default: class { 43 | createJob = jest.fn(); 44 | }, 45 | })); 46 | jest.mock("../../src/queue/instance", () => ({ 47 | __esModule: true, 48 | default: class { 49 | createJob = jest.fn(); 50 | }, 51 | })); 52 | jest.mock("../../src/queue/mbin", () => ({ 53 | __esModule: true, 54 | default: class { 55 | createJob = jest.fn(); 56 | }, 57 | })); 58 | jest.mock("../../src/queue/piefed", () => ({ 59 | __esModule: true, 60 | default: class { 61 | createJob = jest.fn(); 62 | }, 63 | })); 64 | 65 | import { instanceProcessor } from "../../src/crawl/instance"; 66 | import { communityListProcessor, singleCommunityProcessor } from "../../src/crawl/community"; 67 | 68 | describe("crawl jobs live", () => { 69 | jest.setTimeout(30000); 70 | 71 | beforeEach(() => { 72 | jest.clearAllMocks(); 73 | Object.keys(instanceStore).forEach((k) => delete instanceStore[k]); 74 | Object.keys(communityStore).forEach((k) => delete communityStore[k]); 75 | }); 76 | 77 | const instanceBaseUrl = "lemmy.tgxn.net"; 78 | const testCommunity = "lemmyverse"; 79 | 80 | test("instanceProcessor stores instance data", async () => { 81 | const result = await instanceProcessor({ baseUrl: instanceBaseUrl }); 82 | 83 | expect(result).toBeTruthy(); 84 | expect(result).toHaveProperty("nodeData"); 85 | expect(result).toHaveProperty("siteData"); 86 | 87 | expect(storageMock.instance.upsert).toHaveBeenCalledWith( 88 | instanceBaseUrl, 89 | expect.objectContaining({ nodeData: expect.any(Object) }), 90 | ); 91 | }); 92 | 93 | test("communityListProcessor stores communities", async () => { 94 | await instanceProcessor({ baseUrl: instanceBaseUrl }); 95 | 96 | const result = await communityListProcessor({ baseUrl: instanceBaseUrl }); 97 | 98 | expect(Array.isArray(result)).toBe(true); 99 | expect((result as any[]).length).toBeGreaterThan(0); 100 | 101 | expect(storageMock.community.upsert).toHaveBeenCalled(); 102 | }); 103 | 104 | test("singleCommunityProcessor stores one community", async () => { 105 | await instanceProcessor({ baseUrl: instanceBaseUrl }); 106 | 107 | const result = await singleCommunityProcessor({ 108 | baseUrl: instanceBaseUrl, 109 | community: testCommunity, 110 | }); 111 | 112 | expect(result).toBeTruthy(); 113 | expect((result as any).community.name).toBe(testCommunity); 114 | 115 | expect(storageMock.community.upsert).toHaveBeenCalledWith( 116 | instanceBaseUrl, 117 | expect.objectContaining({ community: expect.objectContaining({ name: "lemmyverse" }) }), 118 | ); 119 | }); 120 | }); 121 | -------------------------------------------------------------------------------- /frontend/src/components/ListView/MBin.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | import { Column } from "react-virtualized"; 4 | 5 | import Avatar from "@mui/joy/Avatar"; 6 | import Typography from "@mui/joy/Typography"; 7 | import Box from "@mui/joy/Box"; 8 | 9 | import type { IMBinMagazineOutput } from "../../../../types/output"; 10 | 11 | import VirtualTable from "./VirtualTable"; 12 | import { TinyNumber } from "../Shared/Display"; 13 | import { CopyLink, ExtCommunityLink } from "../Shared/Link"; 14 | 15 | type IMBinListProps = { 16 | items: IMBinMagazineOutput[]; 17 | }; 18 | 19 | function MBinList({ items }: IMBinListProps) { 20 | return ( 21 | 22 | {({ width }) => [ 23 | { 34 | // console.log(rowData); 35 | return ( 36 | 37 | 46 | 53 | 54 | 55 | 62 | 72 | 80 | 81 | 82 | 83 | 93 | 94 | 95 | 96 | ); 97 | }} 98 | />, 99 | 100 | } 109 | />, 110 | } 119 | />, 120 | ]} 121 | 122 | ); 123 | } 124 | 125 | export default React.memo(MBinList); 126 | -------------------------------------------------------------------------------- /frontend/src/components/ListView/Piefed.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | import { Column } from "react-virtualized"; 4 | 5 | import Avatar from "@mui/joy/Avatar"; 6 | import Typography from "@mui/joy/Typography"; 7 | import Box from "@mui/joy/Box"; 8 | 9 | import VirtualTable from "./VirtualTable"; 10 | 11 | import type { IPiefedCommunityDataOutput } from "../../../../types/output"; 12 | 13 | import { TinyNumber } from "../Shared/Display"; 14 | import { CopyLink, ExtCommunityLink } from "../Shared/Link"; 15 | 16 | type IPiefedListProps = { 17 | items: IPiefedCommunityDataOutput[]; 18 | }; 19 | 20 | function PiefedList({ items }: IPiefedListProps) { 21 | return ( 22 | 23 | {({ width }) => [ 24 | { 35 | // console.log(rowData); 36 | return ( 37 | 38 | 47 | 54 | 55 | 56 | 63 | 73 | 81 | 82 | 83 | 84 | 94 | 95 | 96 | 97 | ); 98 | }} 99 | />, 100 | 101 | } 110 | />, 111 | } 120 | />, 121 | ]} 122 | 123 | ); 124 | } 125 | export default React.memo(PiefedList); 126 | -------------------------------------------------------------------------------- /cdk/lib/frontend-stack.ts: -------------------------------------------------------------------------------- 1 | import * as origins from "aws-cdk-lib/aws-cloudfront-origins"; 2 | import * as cloudfront from "aws-cdk-lib/aws-cloudfront"; 3 | import * as s3 from "aws-cdk-lib/aws-s3"; 4 | 5 | import * as acm from "aws-cdk-lib/aws-certificatemanager"; 6 | import * as route53 from "aws-cdk-lib/aws-route53"; 7 | import * as targets from "aws-cdk-lib/aws-route53-targets"; 8 | import { 9 | CachePolicy, 10 | SecurityPolicyProtocol, 11 | ViewerProtocolPolicy, 12 | AllowedMethods, 13 | Distribution, 14 | } from "aws-cdk-lib/aws-cloudfront"; 15 | import { AnyPrincipal, Effect, PolicyStatement } from "aws-cdk-lib/aws-iam"; 16 | 17 | import { Size, CfnOutput, Duration, RemovalPolicy, Stack, StackProps } from "aws-cdk-lib"; 18 | import { Construct } from "constructs"; 19 | 20 | import { BucketDeployment, Source } from "aws-cdk-lib/aws-s3-deployment"; 21 | 22 | import config from "../config.json"; 23 | 24 | type FrontendStackProps = StackProps & { 25 | cert: acm.Certificate; 26 | }; 27 | 28 | export class FrontendStack extends Stack { 29 | constructor(scope: Construct, id: string, props: FrontendStackProps) { 30 | super(scope, id, props); 31 | 32 | // Content Bucket 33 | const siteBucket = new s3.Bucket(this, "SiteBucket", { 34 | publicReadAccess: false, 35 | encryption: s3.BucketEncryption.S3_MANAGED, 36 | blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, 37 | removalPolicy: RemovalPolicy.DESTROY, // NOT recommended for production use 38 | autoDeleteObjects: true, // NOT recommended for production use 39 | }); 40 | new CfnOutput(this, "Bucket", { value: siteBucket.bucketName }); 41 | 42 | // Policy to deny access to the bucket via unencrypted connections 43 | siteBucket.addToResourcePolicy( 44 | new PolicyStatement({ 45 | effect: Effect.DENY, 46 | principals: [new AnyPrincipal()], 47 | actions: ["s3:*"], 48 | resources: [siteBucket.bucketArn], 49 | conditions: { 50 | Bool: { "aws:SecureTransport": "false" }, 51 | }, 52 | }), 53 | ); 54 | 55 | const s3Origin = origins.S3BucketOrigin.withOriginAccessControl(siteBucket, { 56 | originAccessLevels: [cloudfront.AccessLevel.READ], 57 | }); 58 | 59 | const distribution = new Distribution(this, "SiteDistribution", { 60 | defaultBehavior: { 61 | origin: s3Origin, 62 | viewerProtocolPolicy: ViewerProtocolPolicy.REDIRECT_TO_HTTPS, 63 | allowedMethods: AllowedMethods.ALLOW_GET_HEAD_OPTIONS, 64 | cachePolicy: CachePolicy.CACHING_OPTIMIZED, 65 | compress: true, 66 | }, 67 | 68 | // domain name config 69 | domainNames: [config.domain], 70 | certificate: props.cert, 71 | 72 | defaultRootObject: "index.html", 73 | minimumProtocolVersion: SecurityPolicyProtocol.TLS_V1_2_2021, 74 | 75 | // error pages 76 | errorResponses: [ 77 | { 78 | httpStatus: 403, 79 | responseHttpStatus: 200, 80 | responsePagePath: "/index.html", 81 | ttl: Duration.seconds(0), 82 | }, 83 | { 84 | httpStatus: 404, 85 | responseHttpStatus: 200, 86 | responsePagePath: "/index.html", 87 | ttl: Duration.seconds(0), 88 | }, 89 | ], 90 | }); 91 | 92 | new route53.ARecord(this, "SiteAliasRecord", { 93 | zone: route53.HostedZone.fromLookup(this, "Zone", { domainName: config.base_zone }), 94 | recordName: config.domain, 95 | target: route53.RecordTarget.fromAlias(new targets.CloudFrontTarget(distribution)), 96 | }); 97 | 98 | new CfnOutput(this, "DistributionId", { 99 | value: distribution.distributionId, 100 | }); 101 | new CfnOutput(this, "DistributionURL", { 102 | value: distribution.distributionDomainName, 103 | }); 104 | 105 | // Upload the pre-compiled frontend static files 106 | new BucketDeployment(this, `DeployApp-${new Date().toISOString()}`, { 107 | sources: [Source.asset("../frontend/dist/")], 108 | destinationBucket: siteBucket, 109 | distribution: distribution, 110 | distributionPaths: ["/*"], 111 | memoryLimit: 1024, 112 | // useEfs: true, 113 | prune: true, 114 | ephemeralStorageSize: Size.mebibytes(1024), 115 | // retainOnDelete: false, 116 | }); 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /frontend/src/components/Inspector/Overview.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | 3 | import useCachedMultipart from "../../hooks/useCachedMultipart"; 4 | import useQueryCache, { IQueryCache } from "../../hooks/useQueryCache"; 5 | 6 | import Box from "@mui/joy/Box"; 7 | import Grid from "@mui/joy/Grid"; 8 | import Sheet from "@mui/joy/Sheet"; 9 | 10 | import { NumberStat } from "../Shared/StatGridCards"; 11 | 12 | export default function InspectorOverview() { 13 | const { 14 | isLoading: isLoadingSus, 15 | isSuccess: isSuccessSus, 16 | isError: isErrorSus, 17 | error: errorSus, 18 | data: dataSus, 19 | }: IQueryCache = useQueryCache("susData", "sus"); 20 | 21 | const { 22 | isLoading: isLoadingIns, 23 | isSuccess: isSuccessIns, 24 | loadingPercent: loadingPercentIns, 25 | isError: isErrorIns, 26 | error: errorIns, 27 | data: dataIns, 28 | } = useCachedMultipart("instanceData", "instance"); 29 | 30 | const [totalUsers, totalBadUsers] = React.useMemo(() => { 31 | if (!dataSus || !dataIns) return [0, 0, 0]; 32 | 33 | let totalUsers = 0; 34 | let totalBadUsers = 0; 35 | 36 | // instance data 37 | dataIns.forEach((instance) => { 38 | totalUsers += instance.counts.users; 39 | 40 | const susInstance = dataSus.find((susInstance) => susInstance.base === instance.baseurl); 41 | if (susInstance) totalBadUsers += susInstance.users; 42 | }); 43 | 44 | return [totalUsers, totalBadUsers]; 45 | }, [dataIns, dataSus]); 46 | 47 | const [totalComments, totalPosts, totalCommunities] = React.useMemo(() => { 48 | if (!dataSus || !dataIns) return [0, 0]; 49 | 50 | let totalComments = 0; 51 | let totalPosts = 0; 52 | let totalCommunities = 0; 53 | 54 | // instance data 55 | dataIns.forEach((instance) => { 56 | totalComments += instance.counts.comments; 57 | totalPosts += instance.counts.posts; 58 | totalCommunities += instance.counts.communities; 59 | }); 60 | 61 | return [totalComments, totalPosts, totalCommunities]; 62 | }, [dataIns, dataSus]); 63 | 64 | if (isLoadingSus || isLoadingIns) return "Loading..."; 65 | if (isErrorSus) return "An error has occurred: " + errorSus.message; 66 | if (isErrorIns) return "An error has occurred: " + errorIns.message; 67 | 68 | return ( 69 | 70 | 71 | 72 | 78 | 79 | 80 | 85 | 91 | 92 | 93 | 94 | 99 | 105 | 106 | 107 | 108 | 113 | 119 | 120 | 121 | 122 | 127 | 133 | 134 | 135 | 136 | 137 | ); 138 | } 139 | -------------------------------------------------------------------------------- /crawler/test/crawl/mbinInstance.test.ts: -------------------------------------------------------------------------------- 1 | import { jest } from "@jest/globals"; 2 | 3 | // in-memory storage mock 4 | const mbinStore: Record = {}; 5 | 6 | const storageMock = { 7 | connect: jest.fn(), 8 | close: jest.fn(), 9 | mbin: { 10 | upsert: jest.fn((baseUrl: string, magazine: any) => { 11 | mbinStore[`${baseUrl}:${magazine.name}`] = magazine; 12 | }), 13 | setTrackedAttribute: jest.fn(), 14 | }, 15 | tracking: { 16 | getLastCrawl: jest.fn<() => Promise>().mockResolvedValue(null), 17 | getOneError: jest.fn<() => Promise>().mockResolvedValue(null), 18 | setLastCrawl: jest.fn(), 19 | upsertError: jest.fn(), 20 | }, 21 | fediverse: { upsert: jest.fn(), getOne: jest.fn<() => Promise>().mockResolvedValue(null) }, 22 | }; 23 | 24 | const mockMagazine = { 25 | magazineId: 1, 26 | owner: { 27 | magazineId: 1, 28 | userId: 1, 29 | avatar: null, 30 | username: "owner", 31 | apId: null, 32 | }, 33 | icon: null, 34 | name: "testmag", 35 | title: "Test Mag", 36 | description: "desc", 37 | rules: "", 38 | subscriptionsCount: 10, 39 | entryCount: 0, 40 | entryCommentCount: 0, 41 | postCount: 5, 42 | postCommentCount: 3, 43 | isAdult: false, 44 | isUserSubscribed: null, 45 | isBlockedByUser: null, 46 | tags: [], 47 | badges: [], 48 | moderators: [], 49 | apId: null, 50 | apProfileId: "", 51 | serverSoftware: null, 52 | serverSoftwareVersion: null, 53 | isPostingRestrictedToMods: false, 54 | }; 55 | 56 | jest.mock("../../src/lib/crawlStorage", () => ({ 57 | __esModule: true, 58 | default: storageMock, 59 | })); 60 | 61 | jest.mock("../../src/queue/mbin", () => ({ 62 | __esModule: true, 63 | default: class { 64 | createJob = jest.fn(); 65 | }, 66 | })); 67 | 68 | import CrawlMBin, { mbinInstanceProcessor } from "../../src/crawl/mbin"; 69 | 70 | describe("mbinInstanceProcessor", () => { 71 | jest.setTimeout(30000); 72 | 73 | const mbinBaseUrl = "mbin.example"; 74 | 75 | beforeEach(() => { 76 | jest.clearAllMocks(); 77 | Object.keys(mbinStore).forEach((k) => delete mbinStore[k]); 78 | 79 | jest.spyOn(CrawlMBin.prototype, "crawlInstanceData").mockResolvedValue({ site: { name: "Mock MBin" } }); 80 | jest.spyOn(CrawlMBin.prototype, "crawlFederatedInstances").mockResolvedValue(undefined); 81 | jest.spyOn(CrawlMBin.prototype, "crawlMagazinesData").mockImplementation(async (domain: string) => { 82 | await storageMock.mbin.upsert(domain, mockMagazine); 83 | await storageMock.mbin.setTrackedAttribute( 84 | domain, 85 | mockMagazine.name, 86 | "subscriptionsCount", 87 | mockMagazine.subscriptionsCount, 88 | ); 89 | await storageMock.mbin.setTrackedAttribute( 90 | domain, 91 | mockMagazine.name, 92 | "postCount", 93 | mockMagazine.postCount, 94 | ); 95 | await storageMock.mbin.setTrackedAttribute( 96 | domain, 97 | mockMagazine.name, 98 | "postCommentCount", 99 | mockMagazine.postCommentCount, 100 | ); 101 | return [mockMagazine]; 102 | }); 103 | }); 104 | 105 | afterEach(() => { 106 | jest.restoreAllMocks(); 107 | }); 108 | 109 | test("stores magazine data and tracked attributes", async () => { 110 | const result = await mbinInstanceProcessor({ baseUrl: mbinBaseUrl }); 111 | 112 | expect(result).not.toBeNull(); 113 | const res = result as any[]; 114 | expect(Array.isArray(res)).toBe(true); 115 | expect(res[0].name).toBe(mockMagazine.name); 116 | 117 | expect(storageMock.mbin.upsert).toHaveBeenCalledWith( 118 | mbinBaseUrl, 119 | expect.objectContaining({ name: mockMagazine.name }), 120 | ); 121 | expect(mbinStore[`${mbinBaseUrl}:${mockMagazine.name}`]).toHaveProperty( 122 | "subscriptionsCount", 123 | mockMagazine.subscriptionsCount, 124 | ); 125 | expect(storageMock.mbin.setTrackedAttribute).toHaveBeenCalledWith( 126 | mbinBaseUrl, 127 | mockMagazine.name, 128 | "subscriptionsCount", 129 | mockMagazine.subscriptionsCount, 130 | ); 131 | expect(storageMock.mbin.setTrackedAttribute).toHaveBeenCalledWith( 132 | mbinBaseUrl, 133 | mockMagazine.name, 134 | "postCount", 135 | mockMagazine.postCount, 136 | ); 137 | expect(storageMock.mbin.setTrackedAttribute).toHaveBeenCalledWith( 138 | mbinBaseUrl, 139 | mockMagazine.name, 140 | "postCommentCount", 141 | mockMagazine.postCommentCount, 142 | ); 143 | }); 144 | }); 145 | -------------------------------------------------------------------------------- /crawler/test/output/output-utils.test.ts: -------------------------------------------------------------------------------- 1 | import { jest } from "@jest/globals"; 2 | 3 | import OutputUtils from "../../src/output/utils"; 4 | 5 | describe("OutputUtils.safeSplit", () => { 6 | test("splits without breaking words", () => { 7 | const result = OutputUtils.safeSplit("hello world from lemmy", 10); 8 | expect(result).toBe("hello"); 9 | }); 10 | }); 11 | 12 | describe("OutputUtils.stripMarkdownSubStr", () => { 13 | test("strips markdown", () => { 14 | const result = OutputUtils.stripMarkdownSubStr("**hello** _world_!"); 15 | expect(result).toBe("hello world!"); 16 | }); 17 | 18 | test("strips and truncates", () => { 19 | const result = OutputUtils.stripMarkdownSubStr("**hello world**", 8); 20 | expect(result).toBe("hello"); 21 | }); 22 | }); 23 | 24 | describe("OutputUtils.parseLemmyTimeToUnix", () => { 25 | test("parses lemmy timestamp", () => { 26 | const ts = "2024-01-01T12:00:00.123456Z"; 27 | const expected = new Date("2024-01-01T12:00:00Z").getTime(); 28 | expect(OutputUtils.parseLemmyTimeToUnix(ts)).toBe(expected); 29 | }); 30 | }); 31 | 32 | const baseInstance = { baseurl: "example.com" } as any; 33 | const baseCommunity = { 34 | url: "https://lemmy.world/c/lovecraft_mythos", 35 | } as any; 36 | const magazine = { baseurl: "example.com", magazineId: 1 } as any; 37 | const piefedCommunity = { baseurl: "pie.example" } as any; 38 | const fediverse = { url: "https://fediverse" } as any; 39 | const previousRun = { 40 | instances: 1, 41 | communities: 1, 42 | mbin_instances: 1, 43 | magazines: 1, 44 | piefed_instances: 1, 45 | piefed_communities: 1, 46 | fediverse: 1, 47 | }; 48 | 49 | describe("OutputUtils.validateOutput", () => { 50 | test("returns true for valid data", async () => { 51 | await expect( 52 | OutputUtils.validateOutput( 53 | previousRun, 54 | [baseInstance], 55 | [baseCommunity], 56 | ["mbin"], 57 | [magazine], 58 | ["pie"], 59 | [piefedCommunity], 60 | [fediverse], 61 | ), 62 | ).resolves.toBe(true); 63 | }); 64 | 65 | test("throws on empty arrays", async () => { 66 | await expect(OutputUtils.validateOutput(previousRun, [], [], [], [], [], [], [])).rejects.toThrow(); 67 | }); 68 | 69 | test("throws on duplicate instances", async () => { 70 | const prev = { ...previousRun, instances: 2 }; 71 | await expect( 72 | OutputUtils.validateOutput( 73 | prev, 74 | [baseInstance, { baseurl: "example.com" } as any], 75 | [baseCommunity], 76 | ["mbin"], 77 | [magazine], 78 | ["pie"], 79 | [piefedCommunity], 80 | [fediverse], 81 | ), 82 | ).rejects.toThrow(); 83 | }); 84 | 85 | test("throws when lovecraft community missing", async () => { 86 | await expect( 87 | OutputUtils.validateOutput( 88 | previousRun, 89 | [baseInstance], 90 | [{ url: "https://lemmy.world/c/other" } as any], 91 | ["mbin"], 92 | [magazine], 93 | ["pie"], 94 | [piefedCommunity], 95 | [fediverse], 96 | ), 97 | ).rejects.toThrow(); 98 | }); 99 | 100 | test("returns undefined when counts increase", async () => { 101 | const prev = { ...previousRun }; 102 | await expect( 103 | OutputUtils.validateOutput( 104 | prev, 105 | [baseInstance, { baseurl: "two.com" } as any], 106 | [baseCommunity], 107 | ["mbin", "other"], 108 | [magazine], 109 | ["pie"], 110 | [piefedCommunity], 111 | [fediverse, fediverse], 112 | ), 113 | ).resolves.toBeTruthy(); 114 | }); 115 | 116 | test("throws when percent diff too high", async () => { 117 | const prev = { ...previousRun, instances: 100 }; 118 | await expect( 119 | OutputUtils.validateOutput( 120 | prev, 121 | Array(80).fill(baseInstance), 122 | [baseCommunity], 123 | ["mbin"], 124 | [magazine], 125 | ["pie"], 126 | [piefedCommunity], 127 | [fediverse], 128 | ), 129 | ).rejects.toThrow(); 130 | }); 131 | }); 132 | 133 | describe("OutputUtils.validateVersion", () => { 134 | test("cleans valid version", () => { 135 | expect(OutputUtils.validateVersion('"0.19.3"')).toBe("0.19.3"); 136 | }); 137 | 138 | test("returns false for unknown", () => { 139 | expect(OutputUtils.validateVersion("unknown")).toBe(false); 140 | }); 141 | 142 | test("returns false for malformed", () => { 143 | expect(OutputUtils.validateVersion("abc")).toBe(false); 144 | }); 145 | }); 146 | -------------------------------------------------------------------------------- /frontend/src/components/Shared/Link.tsx: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { useSelector } from "react-redux"; 3 | 4 | import Link from "@mui/joy/Link"; 5 | import Tooltip from "@mui/joy/Tooltip"; 6 | 7 | import OpenInNewIcon from "@mui/icons-material/OpenInNew"; 8 | import ContentCopyIcon from "@mui/icons-material/ContentCopy"; 9 | 10 | export function CopyLink({ copyText, linkProps }) { 11 | const [copied, setCopied] = React.useState(false); 12 | React.useEffect(() => { 13 | if (copied) { 14 | // do not re-copy witihin 2 seconds 15 | // this is to give them feedback that it has been copied 16 | setTimeout(() => { 17 | setCopied(false); 18 | }, 2000); 19 | } 20 | }, [copied]); 21 | return ( 22 | 23 | { 35 | e.preventDefault(); 36 | if (navigator.clipboard) { 37 | navigator.clipboard.writeText(copyText); 38 | setCopied(true); 39 | } 40 | }} 41 | > 42 | 48 | {copyText && copyText} 49 | 50 | 51 | ); 52 | } 53 | 54 | type ICommunityLinkProps = { 55 | baseType?: string; 56 | community: { 57 | baseurl: string; 58 | name: string; 59 | title: string; 60 | }; 61 | // homeBaseUrl: string; 62 | // instanceType: string; 63 | [key: string]: any; 64 | }; 65 | 66 | export function ExtCommunityLink({ baseType, community, ...props }: ICommunityLinkProps) { 67 | const homeBaseUrl = useSelector((state: any) => state.configReducer.homeBaseUrl); 68 | const instanceType = useSelector((state: any) => state.configReducer.instanceType); 69 | 70 | const [instanceLink, tooltipTitle] = React.useMemo(() => { 71 | let instanceLink = `https://${community.baseurl}/c/${community.name}`; 72 | let tooltipTitle = `${community.baseurl}/c/${community.name}`; 73 | 74 | if (baseType == "mbin") { 75 | instanceLink = `https://${community.baseurl}/m/${community.name}`; 76 | tooltipTitle = `${community.baseurl}/m/${community.name}`; 77 | } 78 | 79 | // user has a home instance 80 | // note - piefed uses /c/ so does not need it's own selector here 81 | if (homeBaseUrl) { 82 | // if the user is 83 | if (instanceType == "mbin") { 84 | instanceLink = `https://${homeBaseUrl}/m/${community.name}`; 85 | tooltipTitle = `${homeBaseUrl}/m/${community.name}`; 86 | } else { 87 | instanceLink = `https://${homeBaseUrl}/c/${community.name}`; 88 | tooltipTitle = `${homeBaseUrl}/c/${community.name}`; 89 | } 90 | 91 | // if this community isn't on their instance, add the qualifier 92 | if (homeBaseUrl != community.baseurl) { 93 | instanceLink = `${instanceLink}@${community.baseurl}`; 94 | tooltipTitle = `${tooltipTitle}@${community.baseurl}`; 95 | } 96 | } 97 | 98 | return [instanceLink, tooltipTitle]; 99 | }, [community, homeBaseUrl, instanceType]); 100 | 101 | return ( 102 | 103 | 113 | {community.title} 114 | 115 | 116 | 117 | ); 118 | } 119 | 120 | export function ExtLink({ linkName, linkUrl, target = "_blank", ...props }) { 121 | return ( 122 | 136 | {linkName} 137 | 138 | ); 139 | } 140 | 141 | export function ExtInstanceLink({ instance, ...props }) { 142 | return ; 143 | } 144 | --------------------------------------------------------------------------------