├── bin └── cage.cjs ├── .dockerignore ├── .eslintignore ├── src ├── utils │ ├── noop.ts │ ├── isRequired.ts │ ├── sleep.ts │ ├── buildQueryString.ts │ ├── parseCookie.ts │ ├── config.type.ts │ ├── httpError.ts │ ├── index.ts │ ├── getDiff.ts │ ├── config.const.ts │ ├── groupNotifies.ts │ ├── throttle.ts │ ├── cli.ts │ ├── measureTime.ts │ ├── mapBy.ts │ ├── throttle.spec.ts │ ├── types.ts │ ├── fetcher.ts │ ├── logger.spec.ts │ ├── config.ts │ ├── logger.ts │ └── fetcher.spec.ts ├── tasks │ ├── index.spec.ts │ ├── index.ts │ ├── grab-user.ts │ ├── notify.ts │ ├── save.ts │ ├── new-follower.ts │ ├── unfollower.ts │ ├── new-follower.spec.ts │ ├── unfollower.spec.ts │ ├── rename.ts │ ├── grab-user.spec.ts │ ├── save.spec.ts │ ├── notify.spec.ts │ ├── base.spec.ts │ ├── rename.spec.ts │ └── base.ts ├── repositories │ ├── base.ts │ ├── user.ts │ ├── user-log.ts │ └── models │ │ ├── user.ts │ │ └── user-log.ts ├── notifiers │ ├── type.ts │ ├── telegram │ │ ├── constants.ts │ │ ├── types.ts │ │ └── index.ts │ ├── index.ts │ ├── base.ts │ ├── slack.ts │ └── discord.ts ├── watchers │ ├── github │ │ ├── queries.ts │ │ └── index.ts │ ├── base.ts │ ├── mastodon │ │ └── index.ts │ ├── index.ts │ ├── twitter │ │ └── index.ts │ ├── bluesky │ │ └── index.ts │ └── instagram │ │ └── index.ts ├── index.ts └── app.ts ├── tsconfig.build.json ├── .prettierrc ├── .gitignore ├── .graphqlconfig ├── codegen.ts ├── scripts └── generate-schema.ts ├── .eslintrc.json ├── jest.config.ts ├── LICENSE ├── Dockerfile ├── tsconfig.json ├── .releaserc.json ├── .github └── workflows │ ├── ci.yml │ └── docker-deploy.yml ├── package.json ├── README.md └── config.schema.json /bin/cage.cjs: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | require("../dist/src/index"); 4 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | /.idea 3 | /.git 4 | /bin 5 | /scripts 6 | /.github 7 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | /.git 2 | /dist 3 | /node_modules 4 | /schemas 5 | /src/queries.data.ts 6 | -------------------------------------------------------------------------------- /src/utils/noop.ts: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line @typescript-eslint/no-empty-function 2 | export const noop = () => {}; 3 | -------------------------------------------------------------------------------- /src/utils/isRequired.ts: -------------------------------------------------------------------------------- 1 | export function isRequired(value: T | undefined | null): value is T { 2 | return Boolean(value); 3 | } 4 | -------------------------------------------------------------------------------- /src/utils/sleep.ts: -------------------------------------------------------------------------------- 1 | export function sleep(ms: number): Promise { 2 | return new Promise(resolve => { 3 | setTimeout(resolve, ms); 4 | }); 5 | } 6 | -------------------------------------------------------------------------------- /tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "sourceMap": false, 5 | "inlineSourceMap": false 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "tabWidth": 4, 3 | "printWidth": 120, 4 | "singleQuote": false, 5 | "trailingComma": "all", 6 | "endOfLine": "auto", 7 | "arrowParens": "avoid" 8 | } 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | /node_modules 3 | /yarn-error.log 4 | /dist 5 | /coverage 6 | /.env 7 | /followers.json 8 | /dump 9 | /data.sqlite 10 | /config.json 11 | /src/queries.ts 12 | /src/queries.data.ts 13 | -------------------------------------------------------------------------------- /src/utils/buildQueryString.ts: -------------------------------------------------------------------------------- 1 | export function buildQueryString(queries: Record) { 2 | return Object.entries(queries) 3 | .map(([key, value]) => `${key}=${value}`) 4 | .join("&"); 5 | } 6 | -------------------------------------------------------------------------------- /src/tasks/index.spec.ts: -------------------------------------------------------------------------------- 1 | import { DEFAULT_TASKS } from "@tasks/index"; 2 | 3 | describe("Tasks", () => { 4 | it("should provide predefined task array", () => { 5 | expect(DEFAULT_TASKS).toBeDefined(); 6 | expect(DEFAULT_TASKS.length).toBeGreaterThan(0); 7 | }); 8 | }); 9 | -------------------------------------------------------------------------------- /src/utils/parseCookie.ts: -------------------------------------------------------------------------------- 1 | import * as setCookieParser from "set-cookie-parser"; 2 | 3 | export function parseCookie(setCookie: string) { 4 | return setCookieParser 5 | .splitCookiesString(setCookie) 6 | .map(cookie => setCookieParser.parse(cookie)) 7 | .flat(); 8 | } 9 | -------------------------------------------------------------------------------- /.graphqlconfig: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Untitled GraphQL Schema", 3 | "schemaPath": "./schemas/github.graphqls", 4 | "extensions": { 5 | "endpoints": { 6 | "Default GraphQL Endpoint": { 7 | "url": "https://api.github.com/graphql", 8 | "introspect": false 9 | } 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/utils/config.type.ts: -------------------------------------------------------------------------------- 1 | import { WatcherOptionMap } from "@watchers"; 2 | import { NotifierOptionMap } from "@notifiers"; 3 | 4 | export interface ConfigData { 5 | watchInterval: number; 6 | watchers: Partial; 7 | notifiers: Partial; 8 | ignores?: string[]; 9 | } 10 | -------------------------------------------------------------------------------- /src/repositories/base.ts: -------------------------------------------------------------------------------- 1 | import { BaseEntity, Repository } from "typeorm"; 2 | 3 | export abstract class BaseRepository extends Repository { 4 | protected constructor(repository: Repository) { 5 | super(repository.target, repository.manager, repository.queryRunner); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /src/notifiers/type.ts: -------------------------------------------------------------------------------- 1 | import { BaseNotifier } from "@notifiers/base"; 2 | import { UserLog, UserLogType } from "@repositories/models/user-log"; 3 | 4 | export interface BaseNotifierOption { 5 | type: TNotifier extends BaseNotifier ? Lowercase : string; 6 | } 7 | 8 | export type UserLogMap = Record; 9 | -------------------------------------------------------------------------------- /src/repositories/user.ts: -------------------------------------------------------------------------------- 1 | import { Repository } from "typeorm"; 2 | 3 | import { BaseRepository } from "@repositories/base"; 4 | import { User } from "@repositories/models/user"; 5 | 6 | export class UserRepository extends BaseRepository { 7 | public constructor(repository: Repository) { 8 | super(repository); 9 | } 10 | } 11 | 12 | export * from "./models/user"; 13 | -------------------------------------------------------------------------------- /src/utils/httpError.ts: -------------------------------------------------------------------------------- 1 | export class HttpError extends Error { 2 | public readonly statusCode: number; 3 | public readonly statusMessage: string; 4 | 5 | public constructor(statusCode: number, statusMessage: string) { 6 | super(`HTTP Error ${statusCode}: ${statusMessage}`); 7 | 8 | this.statusCode = statusCode; 9 | this.statusMessage = statusMessage; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /codegen.ts: -------------------------------------------------------------------------------- 1 | import type { CodegenConfig } from "@graphql-codegen/cli"; 2 | 3 | const config: CodegenConfig = { 4 | overwrite: true, 5 | schema: "./schemas/github.graphqls", 6 | documents: ["./src/**/queries.ts"], 7 | generates: { 8 | "src/queries.data.ts": { 9 | plugins: ["typescript", "typescript-operations"], 10 | }, 11 | }, 12 | }; 13 | 14 | export default config; 15 | -------------------------------------------------------------------------------- /src/notifiers/telegram/constants.ts: -------------------------------------------------------------------------------- 1 | import { UserLogType } from "@repositories/models/user-log"; 2 | 3 | export const CONTENT_TEMPLATES: Partial> = { 4 | [UserLogType.Follow]: ["**🎉 {}**\n\n{}", "new follower"], 5 | [UserLogType.Unfollow]: ["**❌ {}**\n\n{}", "unfollower"], 6 | [UserLogType.Rename]: ["**✏️ {}**\n\n{}", "rename"], 7 | }; 8 | 9 | export const MAXIMUM_LOG_COUNT = 50; 10 | -------------------------------------------------------------------------------- /src/notifiers/telegram/types.ts: -------------------------------------------------------------------------------- 1 | export interface TelegramNotificationData { 2 | followers?: string; 3 | unfollowers?: string; 4 | renames?: string; 5 | followerCount?: number; 6 | unfollowerCount?: number; 7 | renameCount?: number; 8 | } 9 | 10 | export interface TokenResponse { 11 | token: string; 12 | expires: number; 13 | } 14 | export interface NotifyResponse { 15 | success: boolean; 16 | } 17 | -------------------------------------------------------------------------------- /scripts/generate-schema.ts: -------------------------------------------------------------------------------- 1 | import * as fs from "fs-extra"; 2 | import * as tsj from "ts-json-schema-generator"; 3 | 4 | const config: tsj.Config = { 5 | path: "src/utils/config.type.ts", 6 | tsconfig: "tsconfig.json", 7 | type: "ConfigData", 8 | expose: "none", 9 | }; 10 | 11 | (async () => { 12 | const schema = tsj.createGenerator(config).createSchema(config.type); 13 | const schemaString = JSON.stringify(schema, null, 4); 14 | await fs.writeFile("config.schema.json", schemaString); 15 | })(); 16 | -------------------------------------------------------------------------------- /src/utils/index.ts: -------------------------------------------------------------------------------- 1 | export * from "./buildQueryString"; 2 | export * from "./cli"; 3 | export * from "./config"; 4 | export * from "./config.const"; 5 | export * from "./config.type"; 6 | export * from "./fetcher"; 7 | export * from "./getDiff"; 8 | export * from "./groupNotifies"; 9 | export * from "./httpError"; 10 | export * from "./isRequired"; 11 | export * from "./logger"; 12 | export * from "./mapBy"; 13 | export * from "./measureTime"; 14 | export * from "./noop"; 15 | export * from "./parseCookie"; 16 | export * from "./sleep"; 17 | export * from "./throttle"; 18 | export * from "./types"; 19 | -------------------------------------------------------------------------------- /src/utils/getDiff.ts: -------------------------------------------------------------------------------- 1 | import { mapBy } from "@utils/mapBy"; 2 | import { Values } from "@utils/types"; 3 | 4 | type KeysOnlyString> = Values<{ 5 | [TKey in keyof TData]: TData[TKey] extends string ? TKey : never; 6 | }>; 7 | 8 | export function getDiff>( 9 | newData: TData[], 10 | oldData: TData[], 11 | uniqueKey: KeysOnlyString, 12 | key: keyof TData, 13 | ): TData[] { 14 | const oldDataMap = mapBy(oldData, uniqueKey); 15 | 16 | return newData.filter(item => { 17 | const oldUser = oldDataMap[item[uniqueKey] as string]; 18 | return oldUser && oldUser[key] !== item[key]; 19 | }); 20 | } 21 | -------------------------------------------------------------------------------- /src/tasks/index.ts: -------------------------------------------------------------------------------- 1 | import { TaskClass } from "@tasks/base"; 2 | import { NotifyTask } from "@tasks/notify"; 3 | import { UnfollowerTask } from "@tasks/unfollower"; 4 | import { NewFollowerTask } from "@tasks/new-follower"; 5 | import { GrabUserTask } from "@tasks/grab-user"; 6 | import { SaveTask } from "@tasks/save"; 7 | import { RenameTask } from "@tasks/rename"; 8 | 9 | export const DEFAULT_TASKS: ReadonlyArray = [ 10 | GrabUserTask, 11 | NewFollowerTask, 12 | UnfollowerTask, 13 | RenameTask, 14 | NotifyTask, 15 | SaveTask, 16 | ]; 17 | 18 | export * from "./base"; 19 | export * from "./grab-user"; 20 | export * from "./new-follower"; 21 | export * from "./notify"; 22 | export * from "./unfollower"; 23 | -------------------------------------------------------------------------------- /src/tasks/grab-user.ts: -------------------------------------------------------------------------------- 1 | import { User } from "@repositories/models/user"; 2 | 3 | import { BaseTask, TaskData } from "@tasks/base"; 4 | 5 | export class GrabUserTask extends BaseTask { 6 | public async process(): Promise { 7 | const taskStartedAt = new Date(); 8 | const allUsers: User[] = []; 9 | for (const watcher of this.watchers) { 10 | const users = await watcher.doWatch(taskStartedAt); 11 | allUsers.push(...users); 12 | } 13 | 14 | if (allUsers.length <= 0) { 15 | return this.terminate("No followers found"); 16 | } 17 | 18 | return { 19 | type: "new-users", 20 | data: allUsers, 21 | }; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/tasks/notify.ts: -------------------------------------------------------------------------------- 1 | import { BaseTask, TaskData } from "@tasks/base"; 2 | 3 | import { groupNotifies } from "@utils/groupNotifies"; 4 | 5 | export class NotifyTask extends BaseTask { 6 | public async process(previousData: TaskData[]): Promise { 7 | const newLogs = previousData.filter(BaseTask.isNewLogsData).flatMap(item => item.data); 8 | if (newLogs.length <= 0) { 9 | return this.skip("No new logs to notify was found"); 10 | } 11 | 12 | const logMap = groupNotifies(newLogs); 13 | for (const notifier of this.notifiers) { 14 | await notifier.notify(newLogs, logMap); 15 | } 16 | 17 | return { 18 | type: "notify", 19 | }; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/tasks/save.ts: -------------------------------------------------------------------------------- 1 | import { BaseTask, TaskData } from "@tasks/base"; 2 | 3 | export class SaveTask extends BaseTask { 4 | public async process(previousData: TaskData[]): Promise { 5 | const newLogs = previousData.filter(BaseTask.isNewLogsData).flatMap(item => item.data); 6 | const newUsers = previousData.filter(BaseTask.isNewUsersData).flatMap(item => item.data); 7 | if (newLogs.length <= 0 && newUsers.length <= 0) { 8 | return this.skip("No new logs or users to save was found"); 9 | } 10 | 11 | await this.userRepository.save(newUsers); 12 | await this.userLogRepository.save(newLogs); 13 | 14 | return { 15 | type: "save", 16 | savedCount: newLogs.length + newUsers.length, 17 | }; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/utils/config.const.ts: -------------------------------------------------------------------------------- 1 | import _ from "lodash"; 2 | import Ajv from "ajv"; 3 | 4 | import { ConfigData } from "@utils/config.type"; 5 | 6 | import schema from "@root/../config.schema.json"; 7 | 8 | const ajv = new Ajv({ allErrors: true }); 9 | export const validate = ajv.compile( 10 | _.merge(schema, { 11 | definitions: { 12 | ConfigData: { 13 | type: "object", 14 | properties: { 15 | watchInterval: { 16 | minimum: process.env.NODE_ENV === "development" ? 10000 : 60000, 17 | }, 18 | }, 19 | }, 20 | }, 21 | }), 22 | ); 23 | 24 | export const DEFAULT_CONFIG: ConfigData = { 25 | watchInterval: 60000, 26 | watchers: {}, 27 | notifiers: {}, 28 | }; 29 | -------------------------------------------------------------------------------- /src/utils/groupNotifies.ts: -------------------------------------------------------------------------------- 1 | import _ from "lodash"; 2 | 3 | import { UserLog, UserLogType } from "@repositories/models/user-log"; 4 | 5 | import { UserLogMap } from "@notifiers/type"; 6 | 7 | export function groupNotifies(pairs: UserLog[]): UserLogMap { 8 | const result = _.groupBy(pairs, log => log.type); 9 | 10 | return { 11 | [UserLogType.Follow]: result[UserLogType.Follow] || [], 12 | [UserLogType.Unfollow]: result[UserLogType.Unfollow] || [], 13 | [UserLogType.RenameUserId]: result[UserLogType.RenameUserId] || [], 14 | [UserLogType.RenameDisplayName]: result[UserLogType.RenameDisplayName] || [], 15 | [UserLogType.Rename]: [ 16 | ...(result[UserLogType.RenameDisplayName] || []), 17 | ...(result[UserLogType.RenameUserId] || []), 18 | ], 19 | }; 20 | } 21 | -------------------------------------------------------------------------------- /src/tasks/new-follower.ts: -------------------------------------------------------------------------------- 1 | import { BaseTask, TaskData } from "@tasks/base"; 2 | import { UserLogType } from "@repositories/models/user-log"; 3 | 4 | export class NewFollowerTask extends BaseTask { 5 | public async process(previousData: TaskData[]): Promise { 6 | const followingMap = await this.userLogRepository.getFollowStatusMap(); 7 | const newUsers = previousData.filter(BaseTask.isNewUsersData).flatMap(item => item.data); 8 | const newFollowers = newUsers.filter(p => !followingMap[p.id]); 9 | 10 | return { 11 | type: "new-logs", 12 | data: newFollowers.map(item => { 13 | const log = this.userLogRepository.create(); 14 | log.type = UserLogType.Follow; 15 | log.user = item; 16 | 17 | return log; 18 | }), 19 | }; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/utils/throttle.ts: -------------------------------------------------------------------------------- 1 | import { AsyncFn } from "@utils/types"; 2 | import { sleep } from "@utils/sleep"; 3 | 4 | type Task = Promise | AsyncFn; 5 | 6 | export async function throttle(task: Task, waitFor: number): Promise; 7 | export async function throttle( 8 | task: Task, 9 | waitFor: number, 10 | elapsedTime: boolean, 11 | ): Promise<[TData, number]>; 12 | 13 | export async function throttle( 14 | task: Task, 15 | waitFor: number, 16 | elapsedTime = false, 17 | ): Promise { 18 | const startedAt = Date.now(); 19 | const [result] = await Promise.all([typeof task === "function" ? task() : task, await sleep(waitFor)]); 20 | 21 | if (elapsedTime) { 22 | return [result, Date.now() - startedAt]; 23 | } 24 | 25 | return result; 26 | } 27 | -------------------------------------------------------------------------------- /src/utils/cli.ts: -------------------------------------------------------------------------------- 1 | import chalk from "chalk"; 2 | import stripAnsi from "strip-ansi"; 3 | 4 | export function printLogo(latestVersion: string, currentVersion: any) { 5 | const isOutdated = latestVersion !== currentVersion; 6 | const formatFunction = !isOutdated ? chalk.green : chalk.yellow; 7 | const versionInfo = chalk.italic(formatFunction(`v${currentVersion} ${isOutdated ? "(outdated)" : ""}`)); 8 | 9 | const content = ` _________ _____ ____ 10 | / ___/ __ \`/ __ \`/ _ \\ 11 | / /__/ /_/ / /_/ / __/ 12 | \\___/\\__,_/\\__, /\\___/ 13 | /____/ ${versionInfo}`; 14 | 15 | console.log(chalk.cyan(content)); 16 | 17 | return Math.max( 18 | ...stripAnsi(content) 19 | .split("\n") 20 | .map(l => l.length), 21 | ); 22 | } 23 | 24 | export function drawLine(width = 45) { 25 | console.log("=".repeat(width)); 26 | } 27 | -------------------------------------------------------------------------------- /src/utils/measureTime.ts: -------------------------------------------------------------------------------- 1 | import dayjs from "dayjs"; 2 | 3 | import { Work } from "@utils/types"; 4 | 5 | interface SucceededMeasureTimeResult { 6 | readonly elapsedTime: number; 7 | readonly data: T; 8 | } 9 | 10 | interface FailedMeasureTimeResult { 11 | readonly elapsedTime: number; 12 | readonly exception: Error; 13 | } 14 | 15 | export type MeasureTimeResult = SucceededMeasureTimeResult | FailedMeasureTimeResult; 16 | 17 | export async function measureTime(work: Work): Promise> { 18 | const startedAt = dayjs(); 19 | try { 20 | const result = await work(); 21 | 22 | return { 23 | elapsedTime: dayjs().diff(startedAt), 24 | data: result, 25 | }; 26 | } catch (e) { 27 | return { 28 | elapsedTime: dayjs().diff(startedAt), 29 | exception: e as Error, 30 | }; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/watchers/github/queries.ts: -------------------------------------------------------------------------------- 1 | import gql from "graphql-tag"; 2 | 3 | export const FollowersDocument = gql` 4 | query followers($username: String!, $cursor: String) { 5 | user(login: $username) { 6 | followers(first: 100, after: $cursor) { 7 | totalCount 8 | edges { 9 | node { 10 | id 11 | login 12 | name 13 | } 14 | } 15 | pageInfo { 16 | endCursor 17 | hasNextPage 18 | } 19 | } 20 | } 21 | rateLimit { 22 | limit 23 | cost 24 | remaining 25 | resetAt 26 | } 27 | } 28 | `; 29 | 30 | export const MeDocument = gql` 31 | query me { 32 | viewer { 33 | id 34 | login 35 | } 36 | } 37 | `; 38 | -------------------------------------------------------------------------------- /src/tasks/unfollower.ts: -------------------------------------------------------------------------------- 1 | import { BaseTask, TaskData } from "@tasks/base"; 2 | import { mapBy } from "@utils/mapBy"; 3 | import { UserLogType } from "@repositories/models/user-log"; 4 | 5 | export class UnfollowerTask extends BaseTask { 6 | public async process(previousData: TaskData[]): Promise { 7 | const followingMap = await this.userLogRepository.getFollowStatusMap(); 8 | const oldUsers = await this.userRepository.find(); 9 | const newUsers = previousData.filter(BaseTask.isNewUsersData).flatMap(item => item.data); 10 | const newUserMap = mapBy(newUsers, "id"); 11 | const unfollowers = oldUsers.filter(p => !newUserMap[p.id] && followingMap[p.id]); 12 | 13 | return { 14 | type: "new-logs", 15 | data: unfollowers.map(item => 16 | this.userLogRepository.create({ 17 | type: UserLogType.Unfollow, 18 | user: item, 19 | }), 20 | ), 21 | }; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/repositories/user-log.ts: -------------------------------------------------------------------------------- 1 | import { Repository } from "typeorm"; 2 | 3 | import { BaseRepository } from "@repositories/base"; 4 | import { UserLog, UserLogType } from "@repositories/models/user-log"; 5 | 6 | import { mapBy } from "@utils/mapBy"; 7 | 8 | export class UserLogRepository extends BaseRepository { 9 | public constructor(repository: Repository) { 10 | super(repository); 11 | } 12 | 13 | public async getFollowStatusMap() { 14 | const data = await this.createQueryBuilder() 15 | .select("type") 16 | .addSelect("userId") 17 | .addSelect("MAX(createdAt)", "createdAt") 18 | .where("type IN (:...types)", { types: [UserLogType.Follow, UserLogType.Unfollow] }) 19 | .groupBy("userId") 20 | .getRawMany<{ type: UserLog["type"]; userId: string; createdAt: string }>(); 21 | 22 | return mapBy(data, "userId", item => item.type === UserLogType.Follow); 23 | } 24 | } 25 | 26 | export * from "./models/user-log"; 27 | -------------------------------------------------------------------------------- /src/utils/mapBy.ts: -------------------------------------------------------------------------------- 1 | import { Fn } from "@utils/types"; 2 | 3 | export function mapBy, TKey extends keyof TData>( 4 | data: TData[], 5 | key: TKey | Fn, 6 | ): Record; 7 | export function mapBy, TKey extends keyof TData, TValue = unknown>( 8 | data: TData[], 9 | key: TKey | Fn, 10 | mapFn: Fn, 11 | ): Record; 12 | 13 | export function mapBy, TKey extends keyof TData, TValue = unknown>( 14 | data: TData[], 15 | key: TKey | Fn, 16 | mapFn?: Fn, 17 | ): Record | Record { 18 | return data.reduce((acc, item) => { 19 | const itemKey = typeof key === "function" ? key(item) : item[key]; 20 | acc[itemKey as string] = mapFn ? mapFn(item) : item; 21 | 22 | return acc; 23 | }, {} as Record | Record); 24 | } 25 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "parser": "@typescript-eslint/parser", 3 | "parserOptions": { 4 | "sourceType": "module" 5 | }, 6 | "plugins": ["@typescript-eslint/eslint-plugin"], 7 | "extends": ["plugin:@typescript-eslint/recommended", "plugin:prettier/recommended"], 8 | "root": true, 9 | "env": { 10 | "node": true, 11 | "jest": true 12 | }, 13 | "ignorePatterns": [".eslintrc.js"], 14 | "rules": { 15 | "@typescript-eslint/no-var-requires": "off", 16 | "@typescript-eslint/interface-name-prefix": "off", 17 | "@typescript-eslint/explicit-function-return-type": "off", 18 | "@typescript-eslint/explicit-module-boundary-types": "off", 19 | "@typescript-eslint/no-explicit-any": "off", 20 | "@typescript-eslint/no-unused-vars": [ 21 | "error", 22 | { 23 | "argsIgnorePattern": "^_", 24 | "varsIgnorePattern": "^_", 25 | "caughtErrorsIgnorePattern": "^_" 26 | } 27 | ] 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /jest.config.ts: -------------------------------------------------------------------------------- 1 | import type { JestConfigWithTsJest } from "ts-jest"; 2 | import { pathsToModuleNameMapper } from "ts-jest"; 3 | import { compilerOptions } from "./tsconfig.json"; 4 | 5 | const jestConfig: JestConfigWithTsJest = { 6 | moduleFileExtensions: ["js", "json", "ts"], 7 | rootDir: "./src", 8 | testRegex: ".*\\.spec\\.ts$", 9 | transform: { 10 | "^.+\\.(t|j)s$": "ts-jest", 11 | }, 12 | collectCoverageFrom: ["**/*.ts", "!coverage/**", "!utils/noop.ts", "!index.ts", "!app.ts", "!**/models/*.ts"], 13 | coverageDirectory: "../coverage", 14 | testEnvironment: "node", 15 | roots: [""], 16 | modulePaths: [compilerOptions.baseUrl], // <-- This will be set to 'baseUrl' value 17 | moduleNameMapper: pathsToModuleNameMapper(compilerOptions.paths, { prefix: "/../" }), 18 | collectCoverage: false, 19 | coverageThreshold: { 20 | global: { 21 | lines: 0, 22 | branches: 0, 23 | functions: 0, 24 | statements: 0, 25 | }, 26 | }, 27 | }; 28 | 29 | export = jestConfig; 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Sophia 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/repositories/models/user.ts: -------------------------------------------------------------------------------- 1 | import { BaseEntity, Column, CreateDateColumn, Entity, OneToMany, UpdateDateColumn } from "typeorm"; 2 | 3 | import { UserLog } from "@root/repositories/models/user-log"; 4 | 5 | @Entity({ name: "users" }) 6 | export class User extends BaseEntity { 7 | @Column({ type: "varchar", length: 255, primary: true }) 8 | public id!: string; 9 | 10 | @Column({ type: "varchar", length: 255 }) 11 | public from!: string; 12 | 13 | @Column({ type: "varchar", length: 255 }) 14 | public uniqueId!: string; 15 | 16 | @Column({ type: "varchar", length: 255 }) 17 | public userId!: string; 18 | 19 | @Column({ type: "varchar", length: 255 }) 20 | public displayName!: string; 21 | 22 | @Column({ type: "datetime" }) 23 | public lastlyCheckedAt!: Date; 24 | 25 | @Column({ type: "varchar" }) 26 | public profileUrl!: string; 27 | 28 | @CreateDateColumn() 29 | public createdAt!: Date; 30 | 31 | @UpdateDateColumn() 32 | public updatedAt!: Date; 33 | 34 | @OneToMany(() => UserLog, followerLog => followerLog.user, { cascade: true }) 35 | public userLogs!: UserLog[]; 36 | } 37 | -------------------------------------------------------------------------------- /src/tasks/new-follower.spec.ts: -------------------------------------------------------------------------------- 1 | import { NewFollowerTask } from "@tasks/new-follower"; 2 | import { UserLogRepository, UserLogType } from "@repositories/user-log"; 3 | 4 | describe("NewFollowerTask", () => { 5 | it("should detect new followers between old and new user items", async function () { 6 | const mockUserLogRepository = { 7 | getFollowStatusMap: jest.fn().mockResolvedValue({ 8 | user1: true, 9 | user2: false, 10 | }), 11 | 12 | create: jest.fn().mockReturnValue({}), 13 | } as unknown as UserLogRepository; 14 | 15 | const followers = [{ id: "user1" }, { id: "user2" }]; 16 | const task = new NewFollowerTask([], [], null as any, mockUserLogRepository, NewFollowerTask.name); 17 | const result = await task.process([{ type: "new-users", data: followers as any }]); 18 | 19 | expect(result).toEqual({ 20 | type: "new-logs", 21 | data: [ 22 | { 23 | type: UserLogType.Follow, 24 | user: { id: "user2" }, 25 | }, 26 | ], 27 | }); 28 | }); 29 | }); 30 | -------------------------------------------------------------------------------- /src/tasks/unfollower.spec.ts: -------------------------------------------------------------------------------- 1 | import { UserLogRepository, UserLogType } from "@repositories/user-log"; 2 | import { UserRepository } from "@repositories/user"; 3 | 4 | import { UnfollowerTask } from "@tasks/unfollower"; 5 | 6 | describe("UnfollowerTask", () => { 7 | it("should detect unfollowers between old and new user items", async () => { 8 | const mockUserRepository = { 9 | find: jest.fn().mockResolvedValue([{ id: "user1" }, { id: "user2" }]), 10 | } as unknown as UserRepository; 11 | const mockUserLogRepository = { 12 | getFollowStatusMap: jest.fn().mockResolvedValue({ user1: true, user2: true }), 13 | create: jest.fn().mockImplementation(item => item || {}), 14 | } as unknown as UserLogRepository; 15 | 16 | const task = new UnfollowerTask([], [], mockUserRepository, mockUserLogRepository, UnfollowerTask.name); 17 | const result = await task.process([{ type: "new-users", data: [] }]); 18 | 19 | expect(result).toEqual({ 20 | type: "new-logs", 21 | data: [ 22 | { type: UserLogType.Unfollow, user: { id: "user1" } }, 23 | { type: UserLogType.Unfollow, user: { id: "user2" } }, 24 | ], 25 | }); 26 | }); 27 | }); 28 | -------------------------------------------------------------------------------- /src/utils/throttle.spec.ts: -------------------------------------------------------------------------------- 1 | import { throttle } from "@utils/throttle"; 2 | 3 | describe("throttle() Function", function () { 4 | it("should return the result of the target function", async function () { 5 | const target = async () => 1; 6 | const result = await throttle(target, 0); 7 | 8 | expect(result).toBe(1); 9 | }); 10 | 11 | it("should return the result of the target promise", async function () { 12 | const target = Promise.resolve(1); 13 | const result = await throttle(target, 0); 14 | 15 | expect(result).toBe(1); 16 | }); 17 | 18 | it("should return the result of the target function with the elapsed time", async function () { 19 | const target = async () => 1; 20 | const [result, elapsedTime] = await throttle(target, 1000, true); 21 | 22 | expect(result).toBe(1); 23 | expect(elapsedTime).toBeGreaterThanOrEqual(990); 24 | }); 25 | 26 | it("should return the result of the target promise with the elapsed time", async function () { 27 | const target = Promise.resolve(1); 28 | const [result, elapsedTime] = await throttle(target, 1000, true); 29 | 30 | expect(result).toBe(1); 31 | expect(elapsedTime).toBeGreaterThanOrEqual(990); 32 | }); 33 | }); 34 | -------------------------------------------------------------------------------- /src/repositories/models/user-log.ts: -------------------------------------------------------------------------------- 1 | import { 2 | BaseEntity, 3 | Column, 4 | CreateDateColumn, 5 | Entity, 6 | ManyToOne, 7 | PrimaryGeneratedColumn, 8 | RelationId, 9 | UpdateDateColumn, 10 | } from "typeorm"; 11 | 12 | import { User } from "@repositories/models/user"; 13 | import { Nullable } from "@utils/types"; 14 | 15 | export enum UserLogType { 16 | Follow = "follow", 17 | Unfollow = "unfollow", 18 | RenameDisplayName = "rename-display-name", 19 | RenameUserId = "rename-user-id", 20 | Rename = "rename", 21 | } 22 | 23 | @Entity({ name: "user-logs" }) 24 | export class UserLog extends BaseEntity { 25 | @PrimaryGeneratedColumn() 26 | public id!: number; 27 | 28 | @Column({ type: "varchar", length: 255 }) 29 | public type!: UserLogType; 30 | 31 | @Column({ type: "varchar", length: 255, nullable: true }) 32 | public oldUserId!: Nullable; 33 | 34 | @Column({ type: "varchar", length: 255, nullable: true }) 35 | public oldDisplayName!: Nullable; 36 | 37 | @CreateDateColumn() 38 | public createdAt!: Date; 39 | 40 | @UpdateDateColumn() 41 | public updatedAt!: Date; 42 | 43 | @ManyToOne(() => User, follower => follower.userLogs) 44 | public user!: User; 45 | 46 | @RelationId((userLog: UserLog) => userLog.user) 47 | public userLogId!: string; 48 | } 49 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:20.18-alpine as builder 2 | 3 | RUN apk add --update --no-cache curl git openssh openssl 4 | 5 | USER node 6 | WORKDIR /home/node 7 | 8 | COPY --chown=node:node package*.json ./ 9 | COPY --chown=node:node yarn.lock ./ 10 | RUN yarn install --frozen-lockfile 11 | 12 | COPY --chown=node:node . . 13 | 14 | ARG NODE_ENV=production 15 | ARG APP_ENV=production 16 | 17 | ENV NODE_ENV ${NODE_ENV} 18 | 19 | RUN ["yarn", "build"] 20 | 21 | FROM node:20.18-alpine as prod-deps 22 | 23 | USER node 24 | WORKDIR /home/node 25 | 26 | # copy from build image 27 | COPY --from=builder /home/node/yarn.lock ./yarn.lock 28 | COPY --from=builder /home/node/package.json ./package.json 29 | RUN yarn install --frozen-lockfile --prod 30 | 31 | ARG NODE_ENV=production 32 | ARG APP_ENV=production 33 | 34 | ENV NODE_ENV ${NODE_ENV} 35 | 36 | CMD [ "node", "dist/src/index" ] 37 | 38 | FROM node:20.18-alpine as production 39 | 40 | USER node 41 | WORKDIR /home/node 42 | 43 | # copy from build image 44 | COPY --from=prod-deps /home/node/node_modules ./node_modules 45 | COPY --from=builder /home/node/dist ./dist 46 | COPY --from=builder /home/node/yarn.lock ./yarn.lock 47 | COPY --from=builder /home/node/package.json ./package.json 48 | RUN yarn install --frozen-lockfile --prod 49 | 50 | ARG NODE_ENV=production 51 | ARG APP_ENV=production 52 | 53 | ENV NODE_ENV ${NODE_ENV} 54 | 55 | CMD [ "node", "dist/src/index" ] 56 | -------------------------------------------------------------------------------- /src/utils/types.ts: -------------------------------------------------------------------------------- 1 | import { Logger } from "@utils/logger"; 2 | 3 | export type Resolve = T extends Promise ? U : T; 4 | 5 | export type SelectOnly = { 6 | [Key in keyof Required as Required[Key] extends Type ? Key : never]: Record[Key]; 7 | }; 8 | export type TypeMap = { 9 | [TKey in T["type"]]: TKey extends T["type"] ? Extract : never; 10 | }; 11 | 12 | export type Values = T[keyof T]; 13 | 14 | export type Fn = TArgs extends unknown[] 15 | ? (...arg: TArgs) => TReturn 16 | : TArgs extends void 17 | ? () => TReturn 18 | : (...arg: [TArgs]) => TReturn; 19 | export type AsyncFn = Fn>; 20 | export type Work = Fn | T>; 21 | export type Nullable = T | null | undefined; 22 | 23 | export interface Serializable { 24 | serialize(): Record; 25 | } 26 | export interface Hydratable { 27 | hydrate(data: Record): void; 28 | } 29 | 30 | export abstract class Loggable { 31 | protected readonly logger: Logger; 32 | 33 | protected constructor(public readonly name: TName) { 34 | this.logger = new Logger(name); 35 | } 36 | 37 | public getName(): string { 38 | return this.name; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/watchers/base.ts: -------------------------------------------------------------------------------- 1 | import pluralize from "pluralize"; 2 | import { User } from "@repositories/models/user"; 3 | 4 | import { Loggable } from "@utils/types"; 5 | import { BaseEntity } from "typeorm"; 6 | 7 | export interface BaseWatcherOptions { 8 | type: TWatcher extends BaseWatcher ? Lowercase : string; 9 | } 10 | 11 | export type PartialUser = Omit< 12 | User, 13 | keyof BaseEntity | "from" | "id" | "lastlyCheckedAt" | "createdAt" | "updatedAt" | "userLogs" | "userLogIds" 14 | >; 15 | 16 | export abstract class BaseWatcher extends Loggable { 17 | protected constructor(name: TType) { 18 | super(name); 19 | } 20 | 21 | public abstract initialize(): Promise; 22 | 23 | public async doWatch(startedAt: Date): Promise { 24 | const followers = await this.getFollowers(); 25 | const from = this.getName().toLowerCase(); 26 | 27 | this.logger.info("Successfully crawled {} {}", [followers.length, pluralize("follower", followers.length)]); 28 | 29 | return followers.map(user => 30 | User.create({ 31 | ...user, 32 | id: `${from}:${user.uniqueId}`, 33 | from, 34 | lastlyCheckedAt: startedAt, 35 | }), 36 | ); 37 | } 38 | 39 | protected abstract getFollowers(): Promise; 40 | } 41 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "declaration": false, 4 | "moduleResolution": "Node", 5 | "removeComments": true, 6 | "emitDecoratorMetadata": true, 7 | "experimentalDecorators": true, 8 | "allowSyntheticDefaultImports": true, 9 | "target": "es2017", 10 | "inlineSourceMap": true, 11 | "resolveJsonModule": true, 12 | "outDir": "./dist", 13 | "esModuleInterop": true, 14 | "incremental": true, 15 | "skipLibCheck": true, 16 | "strictNullChecks": true, 17 | "noImplicitAny": false, 18 | "strictBindCallApply": true, 19 | "forceConsistentCasingInFileNames": false, 20 | "noFallthroughCasesInSwitch": false, 21 | "module": "commonjs", 22 | "strict": true, 23 | "strictFunctionTypes": true, 24 | "strictPropertyInitialization": true, 25 | "baseUrl": "./", 26 | "rootDir": ".", 27 | "paths": { 28 | "@root/*": ["./src/*"], 29 | "@utils/*": ["./src/utils/*"], 30 | "@utils": ["./src/utils"], 31 | "@watchers/*": ["./src/watchers/*"], 32 | "@watchers": ["./src/watchers"], 33 | "@followers/*": ["./src/followers/*"], 34 | "@repositories/*": ["./src/repositories/*"], 35 | "@notifiers/*": ["./src/notifiers/*"], 36 | "@notifiers": ["./src/notifiers"], 37 | "@tasks/*": ["./src/tasks/*"], 38 | "@tasks": ["./src/tasks"] 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/notifiers/index.ts: -------------------------------------------------------------------------------- 1 | import { BaseNotifier } from "@notifiers/base"; 2 | import { DiscordNotifier, DiscordNotifierOptions } from "@notifiers/discord"; 3 | import { TelegramNotifier, TelegramNotifierOptions } from "@notifiers/telegram"; 4 | import { SlackNotifier, SlackNotifierOptions } from "@notifiers/slack"; 5 | import { BaseNotifierOption } from "@notifiers/type"; 6 | 7 | import { TypeMap } from "@utils/types"; 8 | 9 | export type NotifierClasses = DiscordNotifier | TelegramNotifier | SlackNotifier; 10 | export type NotifierTypes = Lowercase; 11 | 12 | export type NotifierOptions = DiscordNotifierOptions | TelegramNotifierOptions | SlackNotifierOptions; 13 | export type NotifierOptionMap = TypeMap; 14 | 15 | export type NotifierMap = { 16 | [TKey in NotifierTypes]: TKey extends NotifierTypes ? Extract : never; 17 | }; 18 | export type NotifierFactoryMap = { 19 | [TKey in NotifierTypes]: TKey extends NotifierTypes 20 | ? (options: NotifierOptionMap[TKey]) => Extract }> 21 | : never; 22 | }; 23 | export type NotifierPair = [NotifierTypes, BaseNotifier]; 24 | 25 | const AVAILABLE_NOTIFIERS: Readonly = { 26 | discord: options => new DiscordNotifier(options), 27 | telegram: options => new TelegramNotifier(options), 28 | slack: options => new SlackNotifier(options), 29 | }; 30 | 31 | export const createNotifier = (options: BaseNotifierOption): BaseNotifier => { 32 | const { type } = options; 33 | 34 | return AVAILABLE_NOTIFIERS[type](options); 35 | }; 36 | -------------------------------------------------------------------------------- /src/notifiers/base.ts: -------------------------------------------------------------------------------- 1 | import { capitalCase } from "change-case"; 2 | 3 | import { UserLog, UserLogType } from "@repositories/models/user-log"; 4 | 5 | import { UserLogMap } from "@notifiers/type"; 6 | 7 | import { Logger } from "@utils/logger"; 8 | import { Loggable } from "@utils/types"; 9 | 10 | export abstract class BaseNotifier extends Loggable { 11 | protected constructor(name: TType) { 12 | super(name); 13 | } 14 | 15 | public getName() { 16 | return super.getName().replace("Notifier", ""); 17 | } 18 | 19 | public abstract initialize(): Promise; 20 | 21 | public abstract notify(logs: UserLog[], logMap: UserLogMap): Promise; 22 | 23 | protected formatNotify(log: UserLog): string { 24 | const { user } = log; 25 | 26 | if (log.type === UserLogType.RenameUserId || log.type === UserLogType.RenameDisplayName) { 27 | const tokens = [ 28 | capitalCase(user.from), 29 | log.oldDisplayName || "", 30 | log.oldUserId || "", 31 | user.profileUrl, 32 | log.type === UserLogType.RenameDisplayName ? "" : "@", 33 | log.type === UserLogType.RenameUserId ? user.userId : user.displayName, 34 | ]; 35 | 36 | return Logger.format("[{}] [{} (@{})]({}) → {}{}", ...tokens); 37 | } 38 | 39 | return Logger.format( 40 | "[{}] [{} (@{})]({})", 41 | capitalCase(user.from), 42 | user.displayName, 43 | user.userId, 44 | user.profileUrl, 45 | ); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/tasks/rename.ts: -------------------------------------------------------------------------------- 1 | import { BaseTask, TaskData } from "@tasks/base"; 2 | 3 | import { UserLogType } from "@repositories/models/user-log"; 4 | import { User } from "@repositories/models/user"; 5 | 6 | import { getDiff, isRequired, mapBy } from "@utils"; 7 | 8 | export class RenameTask extends BaseTask { 9 | private createGenerator( 10 | logType: UserLogType.RenameUserId | UserLogType.RenameDisplayName, 11 | oldUsersMap: Record, 12 | ) { 13 | return (item: User) => { 14 | const oldUser = oldUsersMap[item.id]; 15 | return this.userLogRepository.create({ 16 | type: logType, 17 | user: item, 18 | oldDisplayName: oldUser.displayName, 19 | oldUserId: oldUser.userId, 20 | }); 21 | }; 22 | } 23 | 24 | public async process(previousData: TaskData[]): Promise { 25 | const oldUsers = await this.userRepository.find(); 26 | const oldUsersMap = mapBy(oldUsers, "id"); 27 | const newUsers = previousData.filter(BaseTask.isNewUsersData).flatMap(item => item.data); 28 | 29 | // find user renaming their displayName or userId 30 | const displayNameRenamedUsers = getDiff(newUsers, oldUsers, "uniqueId", "displayName"); 31 | const userIdRenamedUsers = getDiff(newUsers, oldUsers, "uniqueId", "userId"); 32 | 33 | return { 34 | type: "new-logs", 35 | data: [ 36 | ...displayNameRenamedUsers.map(this.createGenerator(UserLogType.RenameDisplayName, oldUsersMap)), 37 | ...userIdRenamedUsers.map(this.createGenerator(UserLogType.RenameUserId, oldUsersMap)), 38 | ].filter(isRequired), 39 | }; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/tasks/grab-user.spec.ts: -------------------------------------------------------------------------------- 1 | import { GrabUserTask } from "@tasks/grab-user"; 2 | import { BaseWatcher } from "@watchers/base"; 3 | 4 | describe("GrabUserTask", () => { 5 | it("should grab users from watchers", async function () { 6 | const watchers = [ 7 | { 8 | doWatch: jest.fn().mockResolvedValueOnce([{ id: 1, name: "user1" }]), 9 | }, 10 | { 11 | doWatch: jest.fn().mockResolvedValueOnce([{ id: 2, name: "user2" }]), 12 | }, 13 | ] as any as BaseWatcher[]; 14 | 15 | const task = new GrabUserTask(watchers, [], null as any, null as any, GrabUserTask.name); 16 | const result = await task.process(); 17 | 18 | expect(result).toEqual({ 19 | type: "new-users", 20 | data: [ 21 | { id: 1, name: "user1" }, 22 | { id: 2, name: "user2" }, 23 | ], 24 | }); 25 | expect(watchers[0].doWatch).toBeCalledTimes(1); 26 | expect(watchers[1].doWatch).toBeCalledTimes(1); 27 | }); 28 | 29 | it("should terminate if no users found", async function () { 30 | const watchers = [ 31 | { 32 | doWatch: jest.fn().mockResolvedValueOnce([]), 33 | }, 34 | { 35 | doWatch: jest.fn().mockResolvedValueOnce([]), 36 | }, 37 | ] as any as BaseWatcher[]; 38 | 39 | const task = new GrabUserTask(watchers, [], null as any, null as any, GrabUserTask.name); 40 | const result = await task.process(); 41 | 42 | expect(result).toEqual({ 43 | type: "terminate", 44 | reason: "No followers found", 45 | }); 46 | expect(watchers[0].doWatch).toBeCalledTimes(1); 47 | expect(watchers[1].doWatch).toBeCalledTimes(1); 48 | }); 49 | }); 50 | -------------------------------------------------------------------------------- /src/watchers/mastodon/index.ts: -------------------------------------------------------------------------------- 1 | import * as masto from "masto"; 2 | 3 | import { BaseWatcher, BaseWatcherOptions, PartialUser } from "@watchers/base"; 4 | 5 | export interface MastodonWatcherOptions extends BaseWatcherOptions { 6 | url: string; 7 | accessToken: string; 8 | } 9 | 10 | export class MastodonWatcher extends BaseWatcher<"Mastodon"> { 11 | private client: masto.mastodon.Client | null = null; 12 | private account: masto.mastodon.v1.Account | null = null; 13 | 14 | constructor(private readonly options: MastodonWatcherOptions) { 15 | super("Mastodon"); 16 | } 17 | 18 | public async initialize(): Promise { 19 | this.client = await masto.login({ 20 | url: this.options.url, 21 | accessToken: this.options.accessToken, 22 | }); 23 | 24 | this.account = await this.client.v1.accounts.lookup({ 25 | acct: "@sophia_dev@silicon.moe", 26 | }); 27 | 28 | return Promise.resolve(undefined); 29 | } 30 | 31 | protected async getFollowers(): Promise { 32 | if (!this.client || !this.account) { 33 | throw new Error("Mastodon watcher has not initialized"); 34 | } 35 | 36 | const { id, followersCount } = this.account; 37 | const result: PartialUser[] = []; 38 | for await (const followers of this.client.v1.accounts.listFollowers(id, { limit: 80 })) { 39 | const partialUsers = followers.map(follower => ({ 40 | profileUrl: follower.url, 41 | uniqueId: follower.acct, 42 | displayName: follower.displayName, 43 | userId: follower.id, 44 | })); 45 | 46 | result.push(...partialUsers); 47 | if (result.length >= followersCount) { 48 | break; 49 | } 50 | } 51 | 52 | return result; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/tasks/save.spec.ts: -------------------------------------------------------------------------------- 1 | import { UserRepository } from "@repositories/user"; 2 | import { UserLogRepository } from "@repositories/user-log"; 3 | 4 | import { SaveTask } from "@tasks/save"; 5 | 6 | describe("SaveTask", () => { 7 | it("should save new users and user logs", async () => { 8 | const mockUserRepository = { save: jest.fn() } as unknown as UserRepository; 9 | const mockUserLogRepository = { save: jest.fn() } as unknown as UserLogRepository; 10 | 11 | const mockUsers = [{ id: "user1" }, { id: "user2" }] as any; 12 | const mockUserLogs = [{ id: "log1" }, { id: "log2" }] as any; 13 | const task = new SaveTask([], [], mockUserRepository, mockUserLogRepository, SaveTask.name); 14 | 15 | const result = await task.process([ 16 | { type: "new-users", data: mockUsers }, 17 | { type: "new-logs", data: mockUserLogs }, 18 | ]); 19 | 20 | expect(result).toEqual({ type: "save", savedCount: 4 }); 21 | expect(mockUserRepository.save).toHaveBeenCalledTimes(1); 22 | expect(mockUserRepository.save).toHaveBeenCalledWith(mockUsers); 23 | expect(mockUserLogRepository.save).toHaveBeenCalledTimes(1); 24 | expect(mockUserLogRepository.save).toHaveBeenCalledWith(mockUserLogs); 25 | }); 26 | 27 | it("should skip saving if there is no new users and user logs", async () => { 28 | const mockUserRepository = { save: jest.fn() } as unknown as UserRepository; 29 | const mockUserLogRepository = { save: jest.fn() } as unknown as UserLogRepository; 30 | 31 | const task = new SaveTask([], [], mockUserRepository, mockUserLogRepository, SaveTask.name); 32 | const result = await task.process([]); 33 | 34 | expect(result).toEqual({ type: "skip", reason: "No new logs or users to save was found" }); 35 | expect(mockUserRepository.save).not.toHaveBeenCalled(); 36 | expect(mockUserLogRepository.save).not.toHaveBeenCalled(); 37 | }); 38 | }); 39 | -------------------------------------------------------------------------------- /.releaserc.json: -------------------------------------------------------------------------------- 1 | { 2 | "branches": ["main", { "name": "dev", "prerelease": true }], 3 | "ci": true, 4 | "plugins": [ 5 | "@semantic-release/commit-analyzer", 6 | [ 7 | "@semantic-release/release-notes-generator", 8 | { 9 | "preset": "conventionalCommits", 10 | "parserOpts": { 11 | "noteKeywords": ["BREAKING CHANGE", "BREAKING CHANGES", "BREAKING"] 12 | }, 13 | "presetConfig": { 14 | "types": [ 15 | { 16 | "type": "feat", 17 | "section": "Features ✨" 18 | }, 19 | { 20 | "type": "fix", 21 | "section": "Bug Fixes \uD83D\uDC1E" 22 | }, 23 | { 24 | "type": "chore", 25 | "section": "Internal \uD83E\uDDF0", 26 | "hidden": true 27 | }, 28 | { 29 | "type": "refactor", 30 | "section": "Internal \uD83E\uDDF0", 31 | "hidden": false 32 | }, 33 | { 34 | "type": "perf", 35 | "section": "Internal \uD83E\uDDF0", 36 | "hidden": false 37 | } 38 | ] 39 | } 40 | } 41 | ], 42 | "@semantic-release/npm", 43 | "@semantic-release/github", 44 | [ 45 | "@semantic-release/git", 46 | { 47 | "assets": ["CHANGELOG.md", "package.json"], 48 | "message": "chore(📦): ${nextRelease.version}\n\n${nextRelease.notes}" 49 | } 50 | ] 51 | ] 52 | } 53 | -------------------------------------------------------------------------------- /src/watchers/index.ts: -------------------------------------------------------------------------------- 1 | import { BaseWatcher, BaseWatcherOptions } from "@watchers/base"; 2 | import { TwitterWatcher, TwitterWatcherOptions } from "@watchers/twitter"; 3 | import { GitHubWatcher, GitHubWatcherOptions } from "@watchers/github"; 4 | import { MastodonWatcher, MastodonWatcherOptions } from "@watchers/mastodon"; 5 | import { BlueSkyWatcher, BlueSkyWatcherOptions } from "@watchers/bluesky"; 6 | 7 | import { TypeMap } from "@utils/types"; 8 | import { InstagramWatcher, InstagramWatcherOptions } from "@watchers/instagram"; 9 | 10 | export type WatcherClasses = TwitterWatcher | GitHubWatcher | MastodonWatcher | BlueSkyWatcher | InstagramWatcher; 11 | export type WatcherTypes = Lowercase; 12 | 13 | export type WatcherOptions = 14 | | TwitterWatcherOptions 15 | | GitHubWatcherOptions 16 | | MastodonWatcherOptions 17 | | BlueSkyWatcherOptions 18 | | InstagramWatcherOptions; 19 | 20 | export type WatcherOptionMap = TypeMap; 21 | 22 | export type WatcherMap = { 23 | [TKey in WatcherClasses["name"] as Lowercase]: Extract>; 24 | }; 25 | export type WatcherFactoryMap = { 26 | [TKey in WatcherClasses["name"] as Lowercase]: ( 27 | options: Extract>>, 28 | ) => Extract>; 29 | }; 30 | export type WatcherPair = [WatcherTypes, BaseWatcher]; 31 | 32 | const AVAILABLE_WATCHERS: Readonly = { 33 | twitter: options => new TwitterWatcher(options), 34 | github: options => new GitHubWatcher(options), 35 | mastodon: options => new MastodonWatcher(options), 36 | bluesky: options => new BlueSkyWatcher(options), 37 | instagram: options => new InstagramWatcher(options), 38 | }; 39 | 40 | export const createWatcher = (options: BaseWatcherOptions): BaseWatcher => { 41 | const { type } = options; 42 | 43 | return AVAILABLE_WATCHERS[type](options); 44 | }; 45 | -------------------------------------------------------------------------------- /src/tasks/notify.spec.ts: -------------------------------------------------------------------------------- 1 | import { NotifyTask } from "@tasks/notify"; 2 | import { BaseNotifier } from "@notifiers/base"; 3 | import { UserLogType } from "@repositories/models/user-log"; 4 | 5 | describe("NotifyTask", () => { 6 | it("should notify new logs through notifiers", async () => { 7 | const mockNotifier = { notify: jest.fn() } as unknown as BaseNotifier; 8 | const mockUserLogs = [ 9 | { type: UserLogType.Follow } as any, 10 | { type: UserLogType.Unfollow } as any, 11 | { type: UserLogType.RenameUserId } as any, 12 | { type: UserLogType.RenameDisplayName } as any, 13 | ]; 14 | 15 | const task = new NotifyTask([], [mockNotifier], null as any, null as any, NotifyTask.name); 16 | const result = await task.process([ 17 | { 18 | type: "new-logs", 19 | data: mockUserLogs, 20 | }, 21 | ]); 22 | 23 | expect(result).toEqual({ type: "notify" }); 24 | expect(mockNotifier.notify).toHaveBeenCalledTimes(1); 25 | expect(mockNotifier.notify).toHaveBeenCalledWith(mockUserLogs, { 26 | [UserLogType.Follow]: [mockUserLogs[0]], 27 | [UserLogType.Unfollow]: [mockUserLogs[1]], 28 | [UserLogType.RenameUserId]: [mockUserLogs[2]], 29 | [UserLogType.RenameDisplayName]: [mockUserLogs[3]], 30 | [UserLogType.Rename]: [mockUserLogs[3], mockUserLogs[2]], 31 | }); 32 | }); 33 | 34 | it("should skip notifying if there is no new logs", async () => { 35 | const mockNotifier = { notify: jest.fn() } as unknown as BaseNotifier; 36 | 37 | const task = new NotifyTask([], [mockNotifier], null as any, null as any, NotifyTask.name); 38 | const result = await task.process([]); 39 | 40 | expect(result).toEqual({ type: "skip", reason: "No new logs to notify was found" }); 41 | expect(mockNotifier.notify).not.toHaveBeenCalled(); 42 | }); 43 | }); 44 | -------------------------------------------------------------------------------- /src/watchers/twitter/index.ts: -------------------------------------------------------------------------------- 1 | import { Cursor, Rettiwt, User } from "rettiwt-api"; 2 | 3 | import { BaseWatcher, BaseWatcherOptions, PartialUser } from "@watchers/base"; 4 | 5 | export interface TwitterWatcherOptions extends BaseWatcherOptions { 6 | apiKey: string; 7 | username: string; 8 | } 9 | 10 | export class TwitterWatcher extends BaseWatcher<"Twitter"> { 11 | private readonly twitterClient: Rettiwt; 12 | private readonly currentUserName: string; 13 | 14 | private currentUserId: string | null = null; 15 | 16 | public constructor({ apiKey, username }: TwitterWatcherOptions) { 17 | super("Twitter"); 18 | this.twitterClient = new Rettiwt({ apiKey }); 19 | this.currentUserName = username; 20 | } 21 | 22 | public async initialize() { 23 | const data = await this.twitterClient.user.details(this.currentUserName); 24 | if (!data) { 25 | throw new Error("Failed to get user id"); 26 | } 27 | 28 | this.logger.verbose("Successfully initialized with user id {}", [data.id]); 29 | this.currentUserId = data.id; 30 | } 31 | 32 | protected async getFollowers(): Promise { 33 | if (!this.currentUserId) { 34 | throw new Error("Watcher is not initialized"); 35 | } 36 | 37 | const users: User[] = []; 38 | let cursor: Cursor | null = null; 39 | while (true) { 40 | const followers = await this.twitterClient.user.followers(this.currentUserId, 100, cursor?.value); 41 | if (!followers.next.value || followers.list.length === 0) { 42 | break; 43 | } 44 | 45 | users.push(...followers.list); 46 | cursor = followers.next; 47 | } 48 | 49 | return users.map(user => ({ 50 | uniqueId: user.id, 51 | displayName: user.fullName, 52 | userId: user.userName, 53 | profileUrl: `https://twitter.com/${user.userName}`, 54 | })); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/watchers/bluesky/index.ts: -------------------------------------------------------------------------------- 1 | import { BskyAgent } from "@atproto/api"; 2 | 3 | import { BaseWatcher, BaseWatcherOptions, PartialUser } from "@watchers/base"; 4 | 5 | export interface BlueSkyWatcherOptions extends BaseWatcherOptions { 6 | service?: string; 7 | email: string; 8 | password: string; 9 | } 10 | 11 | export class BlueSkyWatcher extends BaseWatcher<"BlueSky"> { 12 | private agent: BskyAgent | null = null; 13 | private userId: string | null = null; 14 | 15 | constructor(private readonly options: BlueSkyWatcherOptions) { 16 | super("BlueSky"); 17 | } 18 | 19 | public async initialize(): Promise { 20 | this.agent = new BskyAgent({ service: this.options.service || "https://bsky.social" }); 21 | 22 | const response = await this.agent.login({ 23 | identifier: this.options.email, 24 | password: this.options.password, 25 | }); 26 | 27 | this.userId = response.data.did; 28 | } 29 | 30 | protected async getFollowers(): Promise { 31 | if (!this.agent || !this.userId) { 32 | throw new Error("Bluesky watcher has not initialized"); 33 | } 34 | 35 | let cursor: string | undefined = undefined; 36 | const result: PartialUser[] = []; 37 | while (true) { 38 | const { data } = await this.agent.getFollowers({ 39 | actor: this.userId, 40 | limit: 100, 41 | cursor, 42 | }); 43 | 44 | cursor = data.cursor; 45 | for (const follower of data.followers) { 46 | result.push({ 47 | uniqueId: follower.did, 48 | userId: follower.handle, 49 | displayName: follower.displayName || follower.handle, 50 | profileUrl: `https://bsky.app/profile/${follower.handle}`, 51 | }); 52 | } 53 | 54 | if (data.followers.length < 100) { 55 | break; 56 | } 57 | } 58 | 59 | return result; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import updateNotifier from "update-notifier"; 2 | import { Command } from "commander"; 3 | import boxen from "boxen"; 4 | 5 | import { App } from "@root/app"; 6 | 7 | import { Logger } from "@utils/logger"; 8 | import { drawLine, printLogo } from "@utils/cli"; 9 | 10 | import packageJson from "../package.json"; 11 | 12 | interface CLIOptions { 13 | config: string; 14 | verbose: boolean; 15 | dropDatabase: boolean; 16 | database: string; 17 | } 18 | 19 | (async () => { 20 | const program = new Command(); 21 | 22 | program 23 | .name("cage") 24 | .description("(almost) realtime unfollower detection for any social services 🦜⛓️🔒") 25 | .option("-c, --config ", "path to the configuration file", "./config.json") 26 | .option("-d, --database ", "path to the database file", "./data.sqlite") 27 | .option("-p, --drop-database", "delete the old database file") 28 | .option("-v, --verbose", "enable verbose level logging") 29 | .version(packageJson.version) 30 | .parse(process.argv); 31 | 32 | const { config, verbose, dropDatabase, database } = program.opts(); 33 | const { latest, current } = await updateNotifier({ 34 | pkg: packageJson, 35 | distTag: packageJson.version.includes("dev") ? "dev" : "latest", 36 | }).fetchInfo(); 37 | 38 | const logoWidth = printLogo(latest, current); 39 | drawLine(logoWidth); 40 | 41 | if (latest !== current) { 42 | const contents = Logger.format( 43 | "{white}", 44 | [ 45 | Logger.format(`Update available {yellow} → {green}`, current, latest), 46 | Logger.format(`Run {cyan} to update`, `npm i -g ${packageJson.name}@${latest}`), 47 | ].join("\n"), 48 | ); 49 | 50 | console.log( 51 | Logger.format( 52 | "{cyan}", 53 | boxen(contents, { 54 | padding: 1, 55 | margin: 1, 56 | }), 57 | ), 58 | ); 59 | } 60 | 61 | await new App(config, verbose, dropDatabase, database).run(); 62 | })(); 63 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Build and Deploy 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - dev 8 | 9 | jobs: 10 | build-and-deploy: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout 14 | uses: actions/checkout@v2.3.1 15 | with: 16 | persist-credentials: false 17 | 18 | - name: Cache node_modules 19 | id: node-cache 20 | uses: actions/cache@v2 21 | env: 22 | cache-name: cache-node-modules 23 | with: 24 | # npm cache files are stored in `~/.npm` on Linux/macOS 25 | path: node_modules 26 | key: ${{ runner.os }}-node-modules-${{ hashFiles('**/yarn.lock') }} 27 | restore-keys: | 28 | ${{ runner.os }}-node-modules- 29 | 30 | - name: Install and Build 31 | uses: actions/setup-node@v3 32 | with: 33 | node-version: "20.11.1" 34 | 35 | - name: Install yarn 36 | run: | 37 | npm install -g yarn 38 | 39 | - name: Prepare package 40 | run: | 41 | yarn 42 | 43 | - name: Generate codes for GraphQL 44 | run: | 45 | yarn codegen 46 | 47 | - name: Lint 48 | run: | 49 | yarn lint 50 | 51 | - name: Test 52 | env: 53 | FORCE_COLOR: 3 54 | run: | 55 | yarn coverage --ci --verbose --testTimeout=10000 56 | 57 | - name: Codecov 58 | uses: codecov/codecov-action@v3 59 | with: 60 | token: ${{ secrets.CODECOV_TOKEN }} 61 | name: cage 62 | 63 | - name: Build 64 | run: | 65 | yarn build 66 | 67 | - name: Release 68 | env: 69 | GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} 70 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 71 | run: npx semantic-release 72 | 73 | - uses: sarisia/actions-status-discord@v1 74 | if: always() 75 | with: 76 | webhook: ${{ secrets.DISCORD_WEBHOOK }} 77 | -------------------------------------------------------------------------------- /src/tasks/base.spec.ts: -------------------------------------------------------------------------------- 1 | import stripAnsi from "strip-ansi"; 2 | 3 | import { BaseTask, TaskData } from "@tasks/base"; 4 | import { UserLog } from "@repositories/models/user-log"; 5 | import { WorkOptions } from "@utils"; 6 | 7 | describe("BaseTask", () => { 8 | it("should determine given task data type", function () { 9 | const data: TaskData[] = [ 10 | { type: "new-users", data: [] }, 11 | { type: "new-logs", data: [] }, 12 | { type: "notify" }, 13 | { type: "save", savedCount: 0 }, 14 | { type: "terminate", reason: "" }, 15 | { type: "skip", reason: "" }, 16 | ]; 17 | 18 | expect(data.map(BaseTask.isNewUsersData)).toEqual([true, false, false, false, false, false]); 19 | expect(data.map(BaseTask.isNewLogsData)).toEqual([false, true, false, false, false, false]); 20 | expect(data.map(BaseTask.isTerminateData)).toEqual([false, false, false, false, true, false]); 21 | expect(data.map(BaseTask.isSkipData)).toEqual([false, false, false, false, false, true]); 22 | }); 23 | 24 | it("should process task with rich logging", async () => { 25 | const taskDataArray: [TaskData, string][] = [ 26 | [{ type: "new-logs", data: [{} as UserLog] }, "Created 1 new logs"], 27 | [{ type: "skip", reason: "mock-reason" }, "Skipping `Mock` task: mock-reason"], 28 | [{ type: "terminate", reason: "mock-reason" }, "Terminating whole task pipeline: mock-reason"], 29 | ]; 30 | 31 | for (const [item, targetMessage] of taskDataArray) { 32 | class MockTask extends BaseTask { 33 | public async process(): Promise { 34 | return item; 35 | } 36 | } 37 | 38 | const task = new MockTask([], [], null as any, null as any, "MockTask"); 39 | let logMessage = ""; 40 | Object.defineProperty(task, "logger", { 41 | value: { 42 | work: jest.fn().mockImplementation((options: WorkOptions) => options.work()), 43 | info: jest.fn().mockImplementation((message: string) => (logMessage = message)), 44 | }, 45 | }); 46 | 47 | await task.doWork([]); 48 | expect(task["logger"].info).toBeCalledTimes(1); 49 | expect(stripAnsi(logMessage)).toBe(targetMessage); 50 | } 51 | }); 52 | }); 53 | -------------------------------------------------------------------------------- /src/watchers/instagram/index.ts: -------------------------------------------------------------------------------- 1 | import { BaseWatcher, BaseWatcherOptions, PartialUser } from "@watchers/base"; 2 | import { IgApiClient } from "instagram-private-api"; 3 | import { Resolve, sleep } from "@utils"; 4 | 5 | export interface InstagramWatcherOptions extends BaseWatcherOptions { 6 | username: string; 7 | password: string; 8 | targetUserName: string; 9 | requestDelay?: number; 10 | } 11 | 12 | export class InstagramWatcher extends BaseWatcher<"Instagram"> { 13 | private readonly client = new IgApiClient(); 14 | 15 | private readonly username: string; 16 | private readonly password: string; 17 | private readonly targetUserName: string; 18 | private readonly requestDelay: number; 19 | 20 | private loggedInUser: Resolve> | null = null; 21 | 22 | public constructor({ username, password, targetUserName, requestDelay = 1000 }: InstagramWatcherOptions) { 23 | super("Instagram"); 24 | 25 | this.username = username; 26 | this.password = password; 27 | this.targetUserName = targetUserName; 28 | this.requestDelay = requestDelay; 29 | } 30 | 31 | public async initialize() { 32 | this.client.state.generateDevice(this.username); 33 | await this.client.simulate.preLoginFlow(); 34 | 35 | this.loggedInUser = await this.client.account.login(this.username, this.password); 36 | 37 | this.logger.verbose("Successfully initialized with user name {}", [this.loggedInUser.username]); 38 | } 39 | 40 | protected async getFollowers() { 41 | // get followers 42 | const id = await this.client.user.getIdByUsername(this.targetUserName); 43 | if (!id) { 44 | throw new Error("Failed to get user id"); 45 | } 46 | 47 | const followersFeed = this.client.feed.accountFollowers(id); 48 | const followers: PartialUser[] = []; 49 | while (true) { 50 | const items = await followersFeed.items(); 51 | followers.push( 52 | ...items.map(user => ({ 53 | uniqueId: user.pk.toString(), 54 | displayName: user.full_name, 55 | userId: user.username, 56 | profileUrl: `https://instagram.com/${user.username}`, 57 | })), 58 | ); 59 | 60 | if (!followersFeed.isMoreAvailable()) { 61 | break; 62 | } 63 | 64 | await sleep(this.requestDelay); 65 | } 66 | 67 | return followers; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/notifiers/slack.ts: -------------------------------------------------------------------------------- 1 | import { IncomingWebhook, IncomingWebhookSendArguments } from "@slack/webhook"; 2 | 3 | import { BaseNotifier } from "@notifiers/base"; 4 | import { BaseNotifierOption, UserLogMap } from "@notifiers/type"; 5 | 6 | import { Logger } from "@utils/logger"; 7 | import { UserLog } from "@repositories/models/user-log"; 8 | 9 | export interface SlackNotifierOptions extends BaseNotifierOption { 10 | webhookUrl: string; 11 | } 12 | 13 | const MAX_ITEMS_PER_MESSAGE = 25; 14 | 15 | export class SlackNotifier extends BaseNotifier<"Slack"> { 16 | private readonly webhook: IncomingWebhook; 17 | 18 | public constructor(private readonly options: SlackNotifierOptions) { 19 | super("Slack"); 20 | 21 | this.webhook = new IncomingWebhook(this.options.webhookUrl); 22 | } 23 | 24 | public async initialize(): Promise { 25 | return; 26 | } 27 | public async notify(logs: UserLog[], logMap: UserLogMap): Promise { 28 | const { follow, unfollow, rename } = logMap; 29 | const targets: [UserLog[], number, string, string][] = [ 30 | [follow.slice(0, MAX_ITEMS_PER_MESSAGE), follow.length, "🎉 {} new {}", "follower"], 31 | [unfollow.slice(0, MAX_ITEMS_PER_MESSAGE), unfollow.length, "❌ {} {}", "unfollower"], 32 | [rename.slice(0, MAX_ITEMS_PER_MESSAGE), rename.length, "✏️ {} {}", "rename"], 33 | ]; 34 | 35 | const result: IncomingWebhookSendArguments = { 36 | blocks: [{ type: "section", text: { type: "mrkdwn", text: "_*🦜 Cage Report*_" } }], 37 | }; 38 | 39 | let shouldNotify = false; 40 | for (const [logs, count, template, word] of targets) { 41 | if (logs.length <= 0) { 42 | continue; 43 | } 44 | 45 | shouldNotify = true; 46 | 47 | const title = Logger.format(template, count, word); 48 | const userContents = logs.map(this.formatNotify).join("\n"); 49 | let moreText = ""; 50 | if (count > MAX_ITEMS_PER_MESSAGE) { 51 | moreText = Logger.format("... and {} more", count - MAX_ITEMS_PER_MESSAGE); 52 | } 53 | 54 | const content = `*${title}*\n${userContents}\n${moreText}`.trim(); 55 | if (!result.blocks) { 56 | continue; 57 | } 58 | 59 | result.blocks.push({ 60 | type: "section", 61 | text: { 62 | type: "mrkdwn", 63 | text: content, 64 | }, 65 | }); 66 | } 67 | 68 | if (shouldNotify) { 69 | await this.webhook.send(result); 70 | } 71 | } 72 | 73 | protected formatNotify(log: UserLog): string { 74 | return super.formatNotify(log).replace(/ \[(.*? \(@.*?\))\]\((.*?)\)/g, "<$2|$1>"); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/tasks/rename.spec.ts: -------------------------------------------------------------------------------- 1 | import { BaseEntity } from "typeorm"; 2 | import { User, UserRepository } from "@root/repositories/user"; 3 | import { RenameTask } from "@tasks/rename"; 4 | import { UserLogRepository, UserLogType } from "@repositories/user-log"; 5 | 6 | let mockUserId = 0; 7 | function createMockUser(userId: string, displayName: string): Omit { 8 | const id = `user${++mockUserId}`; 9 | 10 | return { 11 | id, 12 | from: "mock", 13 | uniqueId: id, 14 | userId, 15 | displayName, 16 | lastlyCheckedAt: new Date(), 17 | profileUrl: "https://example.com/user1", 18 | createdAt: new Date(), 19 | updatedAt: new Date(), 20 | userLogs: [], 21 | }; 22 | } 23 | 24 | describe("RenameTask", () => { 25 | it("should detect renamed users between old and new user items", async () => { 26 | const mockNewUsers: Omit[] = [ 27 | createMockUser("mock1", "mock1"), 28 | createMockUser("mock2", "mock2"), 29 | ]; 30 | const mockOldUsers: Omit[] = [ 31 | { ...mockNewUsers[0], displayName: "old1" }, 32 | { ...mockNewUsers[1], userId: "old2" }, 33 | ]; 34 | 35 | const task = new RenameTask( 36 | [], 37 | [], 38 | { find: jest.fn().mockResolvedValue(mockOldUsers) } as unknown as UserRepository, 39 | { create: jest.fn().mockImplementation(item => item || {}) } as unknown as UserLogRepository, 40 | RenameTask.name, 41 | ); 42 | const result = await task.process([{ type: "new-users", data: mockNewUsers as unknown as User[] }]); 43 | 44 | expect(result).toMatchObject({ 45 | type: "new-logs", 46 | data: [ 47 | { 48 | type: UserLogType.RenameDisplayName, 49 | user: { id: mockNewUsers[0].id, displayName: "mock1" }, 50 | oldDisplayName: "old1", 51 | }, 52 | { 53 | type: UserLogType.RenameUserId, 54 | user: { id: mockNewUsers[1].id, userId: "mock2" }, 55 | oldUserId: "old2", 56 | }, 57 | ], 58 | }); 59 | }); 60 | 61 | it("should not detect renamed users on new user items", async () => { 62 | const mockNewUsers: Omit[] = [createMockUser("mock1", "mock1")]; 63 | const task = new RenameTask( 64 | [], 65 | [], 66 | { find: jest.fn().mockResolvedValue(mockNewUsers) } as unknown as UserRepository, 67 | { create: jest.fn().mockImplementation(item => item || {}) } as unknown as UserLogRepository, 68 | RenameTask.name, 69 | ); 70 | const result = await task.process([{ type: "new-users", data: [] }]); 71 | 72 | expect(result).toMatchObject({ 73 | type: "new-logs", 74 | data: [], 75 | }); 76 | }); 77 | }); 78 | -------------------------------------------------------------------------------- /.github/workflows/docker-deploy.yml: -------------------------------------------------------------------------------- 1 | name: Docker Image Deploy 2 | 3 | on: 4 | push: 5 | tags: 6 | - "*" 7 | 8 | jobs: 9 | build-and-deploy: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v2.3.1 14 | with: 15 | persist-credentials: false 16 | 17 | - name: Cache node_modules 18 | id: node-cache 19 | uses: actions/cache@v2 20 | env: 21 | cache-name: cache-node-modules 22 | with: 23 | # npm cache files are stored in `~/.npm` on Linux/macOS 24 | path: node_modules 25 | key: ${{ runner.os }}-node-modules-${{ hashFiles('**/yarn.lock') }} 26 | restore-keys: | 27 | ${{ runner.os }}-node-modules- 28 | 29 | - name: Install and Build 30 | uses: actions/setup-node@v3 31 | with: 32 | node-version: "20.11.1" 33 | 34 | - name: Install yarn 35 | run: | 36 | npm install -g yarn 37 | 38 | - name: Prepare package 39 | run: | 40 | yarn 41 | 42 | - name: Generate codes for GraphQL 43 | run: | 44 | yarn codegen 45 | 46 | - name: Lint 47 | run: | 48 | yarn lint 49 | 50 | - name: Test 51 | env: 52 | FORCE_COLOR: 3 53 | run: | 54 | yarn coverage --ci --verbose --testTimeout=10000 55 | 56 | - name: Codecov 57 | uses: codecov/codecov-action@v3 58 | with: 59 | token: ${{ secrets.CODECOV_TOKEN }} 60 | name: cage 61 | 62 | - name: Build 63 | run: | 64 | yarn build 65 | 66 | - name: Login to DockerHub 67 | uses: docker/login-action@v1 68 | with: 69 | username: ${{ secrets.DOCKERHUB_USERNAME }} 70 | password: ${{ secrets.DOCKERHUB_TOKEN }} 71 | 72 | - name: Set Environment Variable for Tagging 73 | run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV 74 | 75 | - name: Build, tag, and push image to Docker Hub 76 | env: 77 | DOCKER_BUILDKIT: 1 78 | GITHUB_OWNER: ${{ github.repository_owner }} 79 | GITHUB_REPO: ${{ github.event.repository.name }} 80 | run: | 81 | docker build -t $GITHUB_OWNER/$GITHUB_REPO:$RELEASE_VERSION . 82 | docker build -t $GITHUB_OWNER/$GITHUB_REPO:latest . 83 | docker push $GITHUB_OWNER/$GITHUB_REPO:$RELEASE_VERSION 84 | docker push $GITHUB_OWNER/$GITHUB_REPO:latest 85 | 86 | - uses: sarisia/actions-status-discord@v1 87 | if: always() 88 | with: 89 | webhook: ${{ secrets.DISCORD_WEBHOOK }} 90 | -------------------------------------------------------------------------------- /src/tasks/base.ts: -------------------------------------------------------------------------------- 1 | import { BaseNotifier } from "@notifiers/base"; 2 | import { BaseWatcher } from "@watchers/base"; 3 | 4 | import { UserRepository } from "@repositories/user"; 5 | import { UserLogRepository } from "@repositories/user-log"; 6 | import { User } from "@repositories/models/user"; 7 | import { UserLog } from "@repositories/models/user-log"; 8 | 9 | import { Loggable, Logger } from "@utils"; 10 | 11 | interface NewUsersTaskData { 12 | type: "new-users"; 13 | data: User[]; 14 | } 15 | interface NewLogsTaskData { 16 | type: "new-logs"; 17 | data: UserLog[]; 18 | } 19 | interface NotifyTaskData { 20 | type: "notify"; 21 | } 22 | interface SaveTaskData { 23 | type: "save"; 24 | savedCount: number; 25 | } 26 | interface TerminateTaskData { 27 | type: "terminate"; 28 | reason: string; 29 | } 30 | interface SkipTaskData { 31 | type: "skip"; 32 | reason: string; 33 | } 34 | 35 | export type TaskData = 36 | | NewUsersTaskData 37 | | NewLogsTaskData 38 | | NotifyTaskData 39 | | SaveTaskData 40 | | TerminateTaskData 41 | | SkipTaskData; 42 | export type TaskClass = new (...args: ConstructorParameters) => BaseTask; 43 | 44 | export abstract class BaseTask extends Loggable { 45 | public constructor( 46 | protected readonly watchers: ReadonlyArray>, 47 | protected readonly notifiers: ReadonlyArray>, 48 | protected readonly userRepository: UserRepository, 49 | protected readonly userLogRepository: UserLogRepository, 50 | name: string, 51 | ) { 52 | super(name.replace(/task$/i, "")); 53 | } 54 | 55 | public static isNewUsersData(data: TaskData): data is NewUsersTaskData { 56 | return data.type === "new-users"; 57 | } 58 | public static isNewLogsData(data: TaskData): data is NewLogsTaskData { 59 | return data.type === "new-logs"; 60 | } 61 | public static isTerminateData(data: TaskData): data is TerminateTaskData { 62 | return data.type === "terminate"; 63 | } 64 | public static isSkipData(data: TaskData): data is SkipTaskData { 65 | return data.type === "skip"; 66 | } 67 | 68 | public async doWork(previousData: TaskData[]): Promise { 69 | const data = await this.logger.work({ 70 | message: Logger.format("processing `{green}` task", this.getName()), 71 | level: "info", 72 | work: () => this.process(previousData), 73 | }); 74 | 75 | this.logData(data); 76 | return data; 77 | } 78 | 79 | protected abstract process(previousData: TaskData[]): Promise; 80 | 81 | protected terminate(reason: string): TerminateTaskData { 82 | return { type: "terminate", reason }; 83 | } 84 | protected skip(reason: string): SkipTaskData { 85 | return { type: "skip", reason }; 86 | } 87 | 88 | private logData(data: TaskData) { 89 | if (BaseTask.isNewLogsData(data) && data.data.length > 0) { 90 | this.logger.info(Logger.format("Created {green} new logs", data.data.length)); 91 | } 92 | 93 | if (BaseTask.isTerminateData(data)) { 94 | this.logger.info(Logger.format("Terminating whole task pipeline: {red}", data.reason)); 95 | } 96 | 97 | if (BaseTask.isSkipData(data)) { 98 | this.logger.info(Logger.format("Skipping `{green}` task: {red}", this.getName(), data.reason)); 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/utils/fetcher.ts: -------------------------------------------------------------------------------- 1 | import nodeFetch, { Headers, Response } from "node-fetch"; 2 | 3 | import { sleep } from "@utils/sleep"; 4 | import { Logger } from "@utils/logger"; 5 | import { parseCookie } from "@utils/parseCookie"; 6 | import { buildQueryString } from "@utils/buildQueryString"; 7 | import { Hydratable, Serializable } from "@utils/types"; 8 | import { HttpError } from "@utils/httpError"; 9 | 10 | interface FetchOption { 11 | url: string; 12 | method?: "GET" | "POST" | "PUT" | "DELETE"; 13 | retryCount?: number; // retry count (default: -1, infinite) 14 | retryDelay?: number; // in ms (default: 1000) 15 | data?: Record; 16 | headers?: Record; 17 | } 18 | 19 | export class Fetcher implements Serializable, Hydratable { 20 | private readonly cookies: Record = {}; 21 | private readonly fetchImpl = nodeFetch; 22 | private readonly logger = new Logger("Fetcher"); 23 | 24 | private getCookieString(): string { 25 | return Object.entries(this.cookies) 26 | .map(([key, value]) => `${key}=${value}`) 27 | .join("; "); 28 | } 29 | private setCookies(setCookie: string | null) { 30 | if (!setCookie) { 31 | return; 32 | } 33 | 34 | const cookies = parseCookie(setCookie); 35 | for (const cookie of cookies) { 36 | if (cookie.name && cookie.value) { 37 | this.cookies[cookie.name] = cookie.value; 38 | } 39 | } 40 | } 41 | public getCookies(): Record { 42 | return { ...this.cookies }; 43 | } 44 | 45 | public async fetchJson(options: FetchOption): Promise { 46 | const response = await this.fetch(options); 47 | return response.json(); 48 | } 49 | public async fetch({ 50 | url, 51 | headers, 52 | data, 53 | method = "GET", 54 | retryCount = 0, 55 | retryDelay = 1000, 56 | }: FetchOption): Promise { 57 | let endpoint = url; 58 | if (method === "GET" && data) { 59 | endpoint = `${endpoint}?${buildQueryString(data)}`; 60 | } 61 | 62 | const fetchHeaders = new Headers({ 63 | cookie: this.getCookieString(), 64 | ...headers, 65 | }); 66 | 67 | if (method !== "GET" && data) { 68 | fetchHeaders.set("Content-Type", "application/json"); 69 | } 70 | 71 | try { 72 | const response = await this.fetchImpl(endpoint, { 73 | method: method, 74 | headers: fetchHeaders, 75 | body: method === "GET" ? undefined : JSON.stringify(data), 76 | }); 77 | 78 | if (!response.ok) { 79 | throw new HttpError(response.status, response.statusText); 80 | } 81 | 82 | this.setCookies(response.headers.get("set-cookie")); 83 | 84 | return response; 85 | } catch (e) { 86 | if (retryCount === 0 || !(e instanceof Error)) { 87 | throw e; 88 | } 89 | 90 | this.logger.error("failed to fetch {}: {}", [url, e.message]); 91 | if (retryDelay > 0) { 92 | this.logger.error("retrying in {}ms ...", [retryDelay]); 93 | } else { 94 | this.logger.error("retrying ..."); 95 | } 96 | 97 | await sleep(retryDelay); 98 | return this.fetch({ 99 | url, 100 | headers, 101 | data, 102 | method, 103 | retryCount: retryCount - 1, 104 | retryDelay, 105 | }); 106 | } 107 | } 108 | 109 | public serialize(): Record { 110 | return { 111 | cookies: this.cookies, 112 | }; 113 | } 114 | public hydrate(data: Record): void { 115 | Object.keys(data.cookies).forEach(key => { 116 | this.cookies[key] = data.cookies[key]; 117 | }); 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/watchers/github/index.ts: -------------------------------------------------------------------------------- 1 | import nodeFetch from "node-fetch"; 2 | import { Client, CombinedError, createClient } from "@urql/core"; 3 | 4 | import { BaseWatcher, BaseWatcherOptions, PartialUser } from "@watchers/base"; 5 | import { FollowersDocument, MeDocument } from "@watchers/github/queries"; 6 | 7 | import { FollowersQuery, FollowersQueryVariables, MeQuery } from "@root/queries.data"; 8 | import { isRequired } from "@utils/isRequired"; 9 | import { Nullable } from "@utils/types"; 10 | 11 | export interface GitHubWatcherOptions extends BaseWatcherOptions { 12 | authToken: string; 13 | } 14 | 15 | const isPartialUser = (user: PartialUser | null): user is PartialUser => Boolean(user); 16 | 17 | export class GitHubWatcher extends BaseWatcher<"GitHub"> { 18 | private client: Client; 19 | 20 | public constructor(private readonly options: GitHubWatcherOptions) { 21 | super("GitHub"); 22 | 23 | this.client = createClient({ 24 | url: "https://api.github.com/graphql", 25 | fetch: nodeFetch as unknown as typeof fetch, 26 | requestPolicy: "network-only", 27 | fetchOptions: () => { 28 | return { 29 | method: "POST", 30 | headers: { 31 | authorization: `token ${this.options.authToken}`, 32 | }, 33 | }; 34 | }, 35 | }); 36 | } 37 | 38 | public async initialize() { 39 | return; 40 | } 41 | 42 | protected async getFollowers() { 43 | try { 44 | const result: PartialUser[] = []; 45 | const currentUserId = await this.getCurrentUserId(); 46 | 47 | let cursor: string | undefined = undefined; 48 | while (true) { 49 | const [followers, nextCursor] = await this.getFollowersFromUserId(currentUserId, cursor); 50 | result.push(...followers); 51 | 52 | if (!nextCursor) { 53 | break; 54 | } 55 | 56 | cursor = nextCursor; 57 | } 58 | 59 | return result; 60 | } catch (e) { 61 | if (e instanceof CombinedError) { 62 | if (e.networkError) { 63 | throw e.networkError; 64 | } else if (e.graphQLErrors && e.graphQLErrors.length > 0) { 65 | throw e.graphQLErrors[0]; 66 | } 67 | } 68 | 69 | throw e; 70 | } 71 | } 72 | 73 | private async getCurrentUserId() { 74 | const { data, error } = await this.client.query(MeDocument, {}).toPromise(); 75 | if (error) { 76 | throw error; 77 | } else if (!data) { 78 | throw new Error("No data returned from Me query"); 79 | } 80 | 81 | return data.viewer.login; 82 | } 83 | private async getFollowersFromUserId( 84 | targetUserId: string, 85 | cursor?: string, 86 | ): Promise<[PartialUser[], Nullable]> { 87 | const { data, error } = await this.client 88 | .query(FollowersDocument, { username: targetUserId, cursor }) 89 | .toPromise(); 90 | 91 | if (error) { 92 | throw error; 93 | } else if (!data?.user?.followers?.edges) { 94 | throw new Error("No followers returned from Followers query"); 95 | } 96 | 97 | return [ 98 | data.user.followers.edges 99 | .map(edge => edge?.node) 100 | .filter(isRequired) 101 | .map(node => ({ 102 | uniqueId: node.id, 103 | userId: node.login, 104 | displayName: node.name || node.login, 105 | profileUrl: `https://github.com/${node.login}`, 106 | })) 107 | .filter(isPartialUser), 108 | data.user.followers.pageInfo.endCursor, 109 | ]; 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /src/notifiers/discord.ts: -------------------------------------------------------------------------------- 1 | import pluralize from "pluralize"; 2 | import dayjs from "dayjs"; 3 | 4 | import { UserLog, UserLogType } from "@repositories/models/user-log"; 5 | 6 | import { BaseNotifier } from "@notifiers/base"; 7 | import { BaseNotifierOption, UserLogMap } from "@notifiers/type"; 8 | 9 | import { Fetcher } from "@utils/fetcher"; 10 | import { Logger } from "@utils/logger"; 11 | 12 | export interface DiscordNotifierOptions extends BaseNotifierOption { 13 | webhookUrl: string; 14 | } 15 | interface DiscordWebhookData { 16 | content: any; 17 | embeds: { 18 | title: string; 19 | color: number; 20 | fields: { 21 | name: string; 22 | value: string; 23 | }[]; 24 | author: { 25 | name: string; 26 | }; 27 | timestamp: string; 28 | }[]; 29 | attachments: []; 30 | } 31 | 32 | export class DiscordNotifier extends BaseNotifier<"Discord"> { 33 | private readonly fetcher = new Fetcher(); 34 | private webhookUrl: string | null = null; 35 | 36 | public constructor(private readonly options: DiscordNotifierOptions) { 37 | super("Discord"); 38 | } 39 | 40 | public async initialize() { 41 | this.webhookUrl = this.options.webhookUrl; 42 | } 43 | public async notify(logs: UserLog[], logMap: UserLogMap) { 44 | if (!this.webhookUrl) { 45 | throw new Error("DiscordNotifier is not initialized"); 46 | } 47 | 48 | if (logs.length <= 0) { 49 | return; 50 | } 51 | 52 | const data: DiscordWebhookData = { 53 | content: null, 54 | embeds: [ 55 | { 56 | title: Logger.format( 57 | "Total {} {} {} found", 58 | logs.length, 59 | pluralize("change", logs.length), 60 | pluralize("was", logs.length), 61 | ), 62 | color: 5814783, 63 | fields: [], 64 | author: { 65 | name: "Cage Report", 66 | }, 67 | // use dayjs to generate timestamp with Z format 68 | timestamp: dayjs().format(), 69 | }, 70 | ], 71 | attachments: [], 72 | }; 73 | 74 | const fields: DiscordWebhookData["embeds"][0]["fields"] = []; 75 | const followerLogs = logMap[UserLogType.Follow]; 76 | const unfollowerLogs = logMap[UserLogType.Unfollow]; 77 | const renameLogs = logMap[UserLogType.Rename]; 78 | if (followerLogs.length > 0) { 79 | fields.push(this.composeLogs(followerLogs, "🎉 {} new {}", "follower")); 80 | } 81 | 82 | if (unfollowerLogs.length > 0) { 83 | fields.push(this.composeLogs(unfollowerLogs, "❌ {} {}", "unfollower")); 84 | } 85 | 86 | if (renameLogs.length > 0) { 87 | fields.push(this.composeLogs(renameLogs, "✏️ {} {}", "rename")); 88 | } 89 | 90 | data.embeds[0].fields.push(...fields); 91 | 92 | await this.fetcher.fetch({ 93 | url: this.webhookUrl, 94 | method: "POST", 95 | data, 96 | }); 97 | } 98 | 99 | private composeLogs( 100 | logs: UserLog[], 101 | messageFormat: string, 102 | word: string, 103 | ): DiscordWebhookData["embeds"][0]["fields"][0] { 104 | const message = Logger.format(messageFormat, logs.length, pluralize(word, logs.length)); 105 | const { name, value } = this.generateEmbedField(logs, message); 106 | const valueLines = [value]; 107 | if (logs.length > 10) { 108 | valueLines.push(`_... and ${logs.length - 10} more_`); 109 | } 110 | 111 | return { 112 | name, 113 | value: valueLines.join("\n"), 114 | }; 115 | } 116 | private generateEmbedField(logs: UserLog[], title: string) { 117 | return { 118 | name: title, 119 | value: logs.slice(0, 10).map(this.formatNotify).join("\n"), 120 | }; 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cage-cli", 3 | "description": "realtime unfollower detection for any social services", 4 | "version": "1.0.0-dev.19", 5 | "license": "MIT", 6 | "publishConfig": { 7 | "access": "public" 8 | }, 9 | "author": { 10 | "name": "Sophia", 11 | "email": "me@sophia-dev.io", 12 | "url": "https://github.com/async3619" 13 | }, 14 | "scripts": { 15 | "build": "tsc --project ./tsconfig.build.json && tsc-alias -p ./tsconfig.build.json", 16 | "dev": "node -r ts-node/register -r dotenv/config -r tsconfig-paths/register ./src/index.ts", 17 | "watch": "node --watch -r ts-node/register -r dotenv/config -r tsconfig-paths/register ./src/index.ts", 18 | "semantic-release": "semantic-release", 19 | "lint": "eslint \"src/**/*.ts\"", 20 | "prepublishOnly": "npm run build", 21 | "test": "jest", 22 | "coverage": "jest --coverage", 23 | "schema": "node -r ts-node/register -r dotenv/config -r tsconfig-paths/register ./scripts/generate-schema.ts", 24 | "codegen": "graphql-codegen --config codegen.ts" 25 | }, 26 | "bin": { 27 | "cage": "./bin/cage.cjs" 28 | }, 29 | "files": [ 30 | "dist", 31 | "yarn.lock", 32 | "bin", 33 | "package.json" 34 | ], 35 | "devDependencies": { 36 | "@graphql-codegen/cli": "2.15.0", 37 | "@graphql-codegen/typescript": "2.8.3", 38 | "@graphql-codegen/typescript-operations": "^2.5.8", 39 | "@semantic-release/commit-analyzer": "^9.0.2", 40 | "@semantic-release/git": "^10.0.1", 41 | "@semantic-release/github": "^8.0.6", 42 | "@semantic-release/npm": "^9.0.1", 43 | "@semantic-release/release-notes-generator": "^10.0.3", 44 | "@types/fs-extra": "^9.0.13", 45 | "@types/jest": "^29.2.4", 46 | "@types/listr": "^0.14.4", 47 | "@types/lodash": "^4.14.191", 48 | "@types/lodash.chunk": "^4.2.7", 49 | "@types/node": "^18.11.10", 50 | "@types/node-cron": "^3.0.6", 51 | "@types/node-fetch": "^2.6.2", 52 | "@types/pluralize": "^0.0.29", 53 | "@types/prompts": "^2.0.14", 54 | "@types/replace-ext": "^2.0.0", 55 | "@types/set-cookie-parser": "^2.4.2", 56 | "@types/update-notifier": "^6.0.1", 57 | "@types/uuid": "^8.3.4", 58 | "@typescript-eslint/eslint-plugin": "^5.38.0", 59 | "@typescript-eslint/parser": "^5.38.0", 60 | "dotenv": "^16.0.3", 61 | "eslint": "^8.24.0", 62 | "eslint-config-prettier": "^8.5.0", 63 | "eslint-plugin-prettier": "^4.2.1", 64 | "jest": "^29.3.1", 65 | "prettier": "^2.7.1", 66 | "rimraf": "^3.0.2", 67 | "semantic-release": "^19.0.5", 68 | "ts-jest": "^29.0.3", 69 | "ts-json-schema-generator": "^1.1.2", 70 | "ts-node": "^10.9.1", 71 | "tsc-alias": "^1.8.1", 72 | "tsconfig-paths": "^4.1.0", 73 | "typescript": "^4.9.3" 74 | }, 75 | "dependencies": { 76 | "@atproto/api": "^0.3.13", 77 | "@octokit/rest": "^19.0.5", 78 | "@slack/webhook": "^6.1.0", 79 | "@urql/core": "^3.0.5", 80 | "ajv": "^8.11.2", 81 | "better-ajv-errors": "^1.2.0", 82 | "boxen": "^5.1.2", 83 | "chalk": "^4.1.2", 84 | "change-case": "^4.1.2", 85 | "commander": "^9.4.1", 86 | "conventional-changelog-conventionalcommits": "^5.0.0", 87 | "cronstrue": "^2.20.0", 88 | "dayjs": "^1.11.6", 89 | "fs-extra": "^11.1.0", 90 | "graphql": "^16.6.0", 91 | "graphql-tag": "^2.12.6", 92 | "instagram-private-api": "^1.46.1", 93 | "lodash": "^4.17.21", 94 | "masto": "^5.11.3", 95 | "node-cron": "^3.0.2", 96 | "node-fetch": "^2.6.7", 97 | "pluralize": "^8.0.0", 98 | "pretty-ms": "^7.0.1", 99 | "reflect-metadata": "^0.1.13", 100 | "rettiwt-api": "^4.1.4", 101 | "set-cookie-parser": "^2.5.1", 102 | "sqlite3": "^5.1.2", 103 | "strip-ansi": "^6.0.1", 104 | "typeorm": "^0.3.10", 105 | "update-notifier": "^5.1.0" 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /src/utils/logger.spec.ts: -------------------------------------------------------------------------------- 1 | import stripAnsi from "strip-ansi"; 2 | 3 | import { Logger, LogLevel } from "@utils/logger"; 4 | import { sleep } from "@utils/sleep"; 5 | import chalk from "chalk"; 6 | 7 | describe("Logger class", () => { 8 | const TARGET_LOG_LEVELS: LogLevel[] = ["info", "warn", "error", "debug", "verbose"]; 9 | let target: Logger; 10 | let buffer: string[]; 11 | let logMockFn: jest.Mock; 12 | 13 | function clearBuffer() { 14 | buffer.length = 0; 15 | } 16 | 17 | beforeEach(() => { 18 | target = new Logger("Test"); 19 | buffer = []; 20 | logMockFn = jest.fn().mockImplementation((data: string) => { 21 | buffer.push(stripAnsi(data)); 22 | }); 23 | 24 | jest.spyOn(process.stdout, "write").mockImplementation(logMockFn); 25 | }); 26 | 27 | it("should format string correctly", () => { 28 | const formattedMessage = Logger.format("{} {} {}", "a", "b", "c"); 29 | 30 | expect(formattedMessage).toBe("a b c"); 31 | }); 32 | 33 | it("should format object correctly", () => { 34 | const formattedMessage = Logger.format("{}", { a: "b", c: "d" }); 35 | 36 | expect(formattedMessage).toBe(JSON.stringify({ a: "b", c: "d" })); 37 | }); 38 | 39 | it("should not format string if no arguments are provided", () => { 40 | const formattedMessage = Logger.format("{} {} {}"); 41 | 42 | expect(formattedMessage).toBe("{} {} {}"); 43 | }); 44 | 45 | it("should style string with format tokens correctly", () => { 46 | const formattedMessage = Logger.format("{bold}", "a"); 47 | 48 | expect(formattedMessage).toBe(chalk.bold("a")); 49 | }); 50 | 51 | it("should provide methods for all log levels", () => { 52 | Logger.verbose = true; 53 | 54 | const mockedLogContent = "test"; 55 | for (const logLevel of TARGET_LOG_LEVELS) { 56 | clearBuffer(); 57 | target[logLevel](mockedLogContent); 58 | 59 | expect(target[logLevel]).toBeDefined(); 60 | expect(buffer).toHaveLength(1); 61 | expect(buffer[0]).toContain(logLevel.toUpperCase()); 62 | expect(buffer[0].trim().endsWith(mockedLogContent)).toBeTruthy(); 63 | } 64 | 65 | Logger.verbose = false; 66 | }); 67 | 68 | it("should not log if verbose mode is disabled", () => { 69 | Logger.verbose = false; 70 | 71 | target.verbose("test"); 72 | expect(buffer).toHaveLength(0); 73 | 74 | Logger.verbose = true; 75 | }); 76 | 77 | it("should provide a method to clear the console", () => { 78 | const clearMock = jest.spyOn(console, "clear"); 79 | target.clear(); 80 | 81 | expect(target.clear).toBeDefined(); 82 | expect(clearMock).toHaveBeenCalled(); 83 | 84 | clearMock.mockClear(); 85 | }); 86 | 87 | it("should provide a method to log a work", async () => { 88 | const mockedWork = async () => { 89 | await sleep(1000); 90 | return 1; 91 | }; 92 | 93 | const mockedWorkResult = await target.work({ 94 | work: mockedWork, 95 | message: "Test", 96 | level: "info", 97 | }); 98 | 99 | expect(target.work).toBeDefined(); 100 | expect(mockedWorkResult).toBe(1); 101 | expect(buffer[0].trim()).toMatch(/Test ...$/); 102 | expect(buffer[1].trim()).toMatch("done."); 103 | }); 104 | 105 | it("should throw an error when work failed", async () => { 106 | const mockedWork = async () => { 107 | await sleep(1000); 108 | throw new Error("Test"); 109 | }; 110 | 111 | await expect( 112 | target.work({ 113 | work: mockedWork, 114 | message: "Test", 115 | level: "info", 116 | }), 117 | ).rejects.toThrow("Test"); 118 | }); 119 | 120 | it("should format the log message correctly", () => { 121 | const mockedLogContent = "test {} {}"; 122 | const mockedArgs = ["1", "2"]; 123 | target.info(mockedLogContent, mockedArgs); 124 | 125 | expect(buffer[0].trim().endsWith("test 1 2")).toBeTruthy(); 126 | }); 127 | 128 | it("should leave the {} in the log message if there are no arguments", () => { 129 | const mockedLogContent = "test {} {}"; 130 | target.info(mockedLogContent, []); 131 | 132 | expect(buffer[0].trim().endsWith("test {} {}")).toBeTruthy(); 133 | }); 134 | }); 135 | -------------------------------------------------------------------------------- /src/notifiers/telegram/index.ts: -------------------------------------------------------------------------------- 1 | import pluralize from "pluralize"; 2 | 3 | import { BaseNotifier } from "@notifiers/base"; 4 | import { BaseNotifierOption, UserLogMap } from "@notifiers/type"; 5 | import { NotifyResponse, TelegramNotificationData, TokenResponse } from "@notifiers/telegram/types"; 6 | import { CONTENT_TEMPLATES, MAXIMUM_LOG_COUNT } from "@notifiers/telegram/constants"; 7 | 8 | import { UserLog, UserLogType } from "@repositories/models/user-log"; 9 | 10 | import { Fetcher } from "@utils/fetcher"; 11 | import { HttpError } from "@utils/httpError"; 12 | import { Logger } from "@utils/logger"; 13 | 14 | export interface TelegramNotifierOptions extends BaseNotifierOption { 15 | token: string; 16 | url?: string; 17 | } 18 | 19 | export class TelegramNotifier extends BaseNotifier<"Telegram"> { 20 | private readonly fetcher = new Fetcher(); 21 | private currentToken: string | null = null; 22 | 23 | public constructor(private readonly options: TelegramNotifierOptions) { 24 | super("Telegram"); 25 | } 26 | 27 | public async initialize(): Promise { 28 | this.currentToken = await this.acquireToken(); 29 | } 30 | public async notify(_: UserLog[], logMap: UserLogMap): Promise { 31 | const reportTargets: UserLogType[] = [UserLogType.Follow, UserLogType.Unfollow, UserLogType.Rename]; 32 | const content: Partial> = {}; 33 | for (const type of reportTargets) { 34 | const logs = logMap[type]; 35 | if (logs.length <= 0) { 36 | continue; 37 | } 38 | 39 | const template = CONTENT_TEMPLATES[type]; 40 | if (!template) { 41 | throw new Error(`There is no message template for log type '${type}'`); 42 | } 43 | 44 | const [title, action] = template; 45 | //TODO: replace this to adjusting the length of the message by content not hard-coded 46 | let messageContent = logs.slice(0, MAXIMUM_LOG_COUNT).map(this.formatNotify).join("\n"); 47 | if (logs.length > MAXIMUM_LOG_COUNT) { 48 | const remainCount = logs.length - MAXIMUM_LOG_COUNT; 49 | messageContent += `\n\n_... and ${remainCount} more_`; 50 | } 51 | 52 | const text = Logger.format(title, pluralize(action, logs.length, true), messageContent); 53 | content[type] = [text, logs.length]; 54 | } 55 | 56 | await this.pushNotify({ 57 | followers: content[UserLogType.Follow]?.[0], 58 | unfollowers: content[UserLogType.Unfollow]?.[0], 59 | renames: content[UserLogType.Rename]?.[0], 60 | followerCount: content[UserLogType.Follow]?.[1], 61 | unfollowerCount: content[UserLogType.Unfollow]?.[1], 62 | renameCount: content[UserLogType.Rename]?.[1], 63 | }); 64 | } 65 | 66 | private getEndpoint(path: string) { 67 | const base = this.options.url || "https://cage-telegram.sophia-dev.io"; 68 | return `${base}${path}`; 69 | } 70 | 71 | private async acquireToken() { 72 | const { token } = await this.fetcher.fetchJson({ 73 | url: this.getEndpoint("/token"), 74 | method: "POST", 75 | headers: { 76 | "Content-Type": "application/json", 77 | }, 78 | retryCount: 5, 79 | retryDelay: 0, 80 | }); 81 | 82 | return token; 83 | } 84 | private async pushNotify(content: TelegramNotificationData) { 85 | while (true) { 86 | try { 87 | await this.fetcher.fetchJson({ 88 | url: this.getEndpoint("/notify"), 89 | method: "POST", 90 | headers: { 91 | "Content-Type": "application/json", 92 | Authorization: `Bearer ${this.currentToken}`, 93 | }, 94 | data: { 95 | token: this.options.token, 96 | ...content, 97 | }, 98 | retryCount: 5, 99 | retryDelay: 0, 100 | }); 101 | 102 | return; 103 | } catch (e) { 104 | if (e instanceof HttpError && e.statusCode === 403) { 105 | this.currentToken = await this.acquireToken(); 106 | continue; 107 | } 108 | 109 | throw e; 110 | } 111 | } 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/utils/config.ts: -------------------------------------------------------------------------------- 1 | import * as path from "path"; 2 | import * as fs from "fs-extra"; 3 | import chalk from "chalk"; 4 | import betterAjvErrors from "better-ajv-errors"; 5 | 6 | import { createWatcher, WatcherMap, WatcherPair, WatcherTypes } from "@watchers"; 7 | import { createNotifier, NotifierMap, NotifierPair, NotifierTypes } from "@notifiers"; 8 | 9 | import { DEFAULT_CONFIG, validate } from "@utils/config.const"; 10 | import { ConfigData } from "@utils/config.type"; 11 | import { Logger } from "@utils/logger"; 12 | 13 | import schema from "@root/../config.schema.json"; 14 | 15 | export class Config { 16 | private static readonly logger = new Logger("Config"); 17 | private static readonly DEFAULT_CONFIG_PATH = path.join(process.cwd(), "./config.json"); 18 | 19 | public static async create(filePath: string = Config.DEFAULT_CONFIG_PATH) { 20 | if (!path.isAbsolute(filePath)) { 21 | filePath = path.join(process.cwd(), filePath); 22 | } 23 | 24 | const pathToken = `\`${chalk.green(filePath)}\``; 25 | 26 | if (!fs.existsSync(filePath)) { 27 | Config.logger.warn( 28 | `config file ${pathToken} does not exist. we will make a new default config file for you.`, 29 | ); 30 | 31 | await fs.writeJSON(filePath, DEFAULT_CONFIG, { 32 | spaces: 4, 33 | }); 34 | 35 | Config.logger.warn(`config file ${pathToken} has been created successfully.`); 36 | } 37 | 38 | try { 39 | return await Config.logger.work({ 40 | level: "info", 41 | message: `loading config file from ${pathToken}`, 42 | work: async () => { 43 | const data: ConfigData = await fs.readJSON(filePath); 44 | const valid = validate(data); 45 | if (!valid && validate.errors) { 46 | const output = betterAjvErrors(schema, data, validate.errors, { 47 | format: "cli", 48 | indent: 4, 49 | }); 50 | 51 | throw new Error(`config file is invalid.\n${output}`); 52 | } 53 | 54 | const watcherTypes = Object.keys(data.watchers) as WatcherTypes[]; 55 | const watchers: WatcherPair[] = []; 56 | const watcherMap: Partial = {}; 57 | for (const type of watcherTypes) { 58 | const configs = data.watchers[type]; 59 | if (!configs) { 60 | throw new Error(`watcher \`${type}\` is not configured.`); 61 | } 62 | 63 | const watcher = createWatcher(configs); 64 | watchers.push([type, watcher]); 65 | watcherMap[type] = watcher as any; 66 | } 67 | 68 | const notifierTypes = Object.keys(data.notifiers) as NotifierTypes[]; 69 | const notifiers: NotifierPair[] = []; 70 | const notifierMap: Partial = {}; 71 | for (const type of notifierTypes) { 72 | const configs = data.notifiers[type]; 73 | if (!configs) { 74 | throw new Error(`notifier \`${type}\` is not configured.`); 75 | } 76 | 77 | const notifier = createNotifier(configs); 78 | notifiers.push([type, notifier]); 79 | notifierMap[type] = notifier as any; 80 | } 81 | 82 | return new Config(data, watcherTypes, watchers, watcherMap, notifierTypes, notifiers, notifierMap); 83 | }, 84 | }); 85 | } catch (e) { 86 | process.exit(-1); 87 | } 88 | } 89 | 90 | get watcherTypes(): ReadonlyArray { 91 | return [...this._watcherTypes]; 92 | } 93 | get watchers(): ReadonlyArray { 94 | return [...this._watchers]; 95 | } 96 | get watcherMap(): Partial { 97 | return { ...this._watcherMap }; 98 | } 99 | 100 | get notifierTypes(): ReadonlyArray { 101 | return [...this._notifierTypes]; 102 | } 103 | get notifiers(): ReadonlyArray { 104 | return [...this._notifiers]; 105 | } 106 | get notifierMap(): Partial { 107 | return { ...this._notifierMap }; 108 | } 109 | 110 | public get ignores(): ReadonlyArray { 111 | return [...(this.rawData.ignores || [])]; 112 | } 113 | public get watchInterval() { 114 | return this.rawData.watchInterval; 115 | } 116 | 117 | private constructor( 118 | private readonly rawData: ConfigData, 119 | private readonly _watcherTypes: ReadonlyArray, 120 | private readonly _watchers: ReadonlyArray, 121 | private readonly _watcherMap: Partial, 122 | private readonly _notifierTypes: NotifierTypes[], 123 | private readonly _notifiers: NotifierPair[], 124 | private readonly _notifierMap: Partial, 125 | ) {} 126 | } 127 | -------------------------------------------------------------------------------- /src/utils/logger.ts: -------------------------------------------------------------------------------- 1 | import dayjs from "dayjs"; 2 | import chalk from "chalk"; 3 | 4 | import { measureTime } from "@utils/measureTime"; 5 | import { noop } from "@utils/noop"; 6 | import { Fn } from "@utils/types"; 7 | 8 | type LoggerFn = (content: string, args?: any[], breakLine?: boolean) => void; 9 | export type LogLevel = "verbose" | "info" | "warn" | "error" | "debug"; 10 | 11 | const LOG_LEVEL_COLOR_MAP: Record> = { 12 | verbose: chalk.blue, 13 | info: chalk.cyan, 14 | warn: chalk.yellow, 15 | error: chalk.red, 16 | debug: chalk.magenta, 17 | }; 18 | 19 | export interface WorkOptions { 20 | level: LogLevel; 21 | message: string; 22 | failedLevel?: LogLevel; 23 | done?: string; 24 | work: () => T | Promise; 25 | } 26 | 27 | const CHALK_FORMAT_NAMES: Array = [ 28 | "black", 29 | "red", 30 | "green", 31 | "yellow", 32 | "blue", 33 | "magenta", 34 | "cyan", 35 | "white", 36 | "gray", 37 | "grey", 38 | "blackBright", 39 | "redBright", 40 | "greenBright", 41 | "yellowBright", 42 | "blueBright", 43 | "magentaBright", 44 | "cyanBright", 45 | "whiteBright", 46 | "bgBlack", 47 | "bgRed", 48 | "bgGreen", 49 | "bgYellow", 50 | "bgBlue", 51 | "bgMagenta", 52 | "bgCyan", 53 | "bgWhite", 54 | "bgGray", 55 | "bgGrey", 56 | "bgBlackBright", 57 | "bgRedBright", 58 | "bgGreenBright", 59 | "bgYellowBright", 60 | "bgBlueBright", 61 | "bgMagentaBright", 62 | "bgCyanBright", 63 | "bgWhiteBright", 64 | "italic", 65 | "bold", 66 | "underline", 67 | "strikethrough", 68 | ]; 69 | 70 | export class Logger implements Record { 71 | public static verbose = false; 72 | 73 | public static format(content: string, ...args: any[]): string { 74 | if (args.length === 0) { 75 | return content; 76 | } 77 | 78 | const replacements = args.map(arg => { 79 | if (typeof arg === "object") { 80 | return JSON.stringify(arg); 81 | } 82 | 83 | return `${arg}`; 84 | }); 85 | 86 | const matches = content.matchAll(/(\{(.*?)\})/g); 87 | if (!matches) { 88 | return content; 89 | } 90 | 91 | for (const [, token, style] of matches) { 92 | let item = replacements.shift(); 93 | if (typeof item === "undefined") { 94 | item = "{}"; 95 | } 96 | 97 | if (style) { 98 | style.split(",").forEach(style => { 99 | if (CHALK_FORMAT_NAMES.includes(style as any)) { 100 | item = chalk[style](item); 101 | } 102 | }); 103 | } 104 | 105 | content = content.replace(token, item); 106 | } 107 | 108 | return content; 109 | } 110 | 111 | private static readonly buffer: string[] = []; 112 | private static isLocked = false; 113 | 114 | private static setLock(lock: boolean) { 115 | if (!lock && Logger.isLocked) { 116 | Logger.buffer.forEach(message => process.stdout.write(message)); 117 | Logger.buffer.length = 0; 118 | } 119 | 120 | Logger.isLocked = lock; 121 | } 122 | 123 | public readonly info: LoggerFn; 124 | public readonly warn: LoggerFn; 125 | public readonly error: LoggerFn; 126 | public readonly debug: LoggerFn; 127 | public readonly verbose: LoggerFn; 128 | 129 | public constructor(private readonly name: string) { 130 | this.info = this.createLoggerFunction("info"); 131 | this.warn = this.createLoggerFunction("warn"); 132 | this.error = this.createLoggerFunction("error"); 133 | this.debug = this.createLoggerFunction("debug"); 134 | this.verbose = this.createLoggerFunction("verbose"); 135 | } 136 | 137 | private createLoggerFunction = (level: LogLevel): LoggerFn => { 138 | const levelString = LOG_LEVEL_COLOR_MAP[level](level.toUpperCase()); 139 | 140 | return (content, args, breakLine = true) => { 141 | if (level === "verbose" && !Logger.verbose) { 142 | return noop; 143 | } 144 | 145 | const tokens = chalk.green( 146 | [chalk.cyan(dayjs().format("HH:mm:ss.SSS")), chalk.yellow(this.name), levelString] 147 | .map(t => `[${t}]`) 148 | .join(""), 149 | ); 150 | 151 | // replace all {} with the corresponding argument 152 | let message = content; 153 | if (args) { 154 | message = Logger.format(content, ...args); 155 | } 156 | 157 | const formattedString = `${tokens} ${message}${breakLine ? "\n" : ""}`; 158 | if (Logger.isLocked) { 159 | Logger.buffer.push(formattedString); 160 | } else { 161 | process.stdout.write(formattedString); 162 | } 163 | }; 164 | }; 165 | 166 | public clear = () => { 167 | console.clear(); 168 | }; 169 | 170 | public work = async ({ 171 | work, 172 | failedLevel = "error", 173 | level, 174 | message, 175 | done = "done.", 176 | }: WorkOptions): Promise => { 177 | this[level](`${message} ... `, [], false); 178 | 179 | Logger.setLock(true); 180 | const measuredData = await measureTime(work); 181 | const time = chalk.gray(`(${measuredData.elapsedTime}ms)`); 182 | 183 | if ("exception" in measuredData) { 184 | process.stdout.write(`failed. ${time}\n`); 185 | this[failedLevel](`${measuredData.exception.message}`); 186 | Logger.setLock(false); 187 | throw measuredData.exception; 188 | } 189 | 190 | process.stdout.write(`${done} ${time}\n`); 191 | Logger.setLock(false); 192 | 193 | return measuredData.data; 194 | }; 195 | } 196 | -------------------------------------------------------------------------------- /src/app.ts: -------------------------------------------------------------------------------- 1 | import * as path from "path"; 2 | import * as fs from "fs-extra"; 3 | import { DataSource } from "typeorm"; 4 | import chalk from "chalk"; 5 | import prettyMilliseconds from "pretty-ms"; 6 | import pluralize from "pluralize"; 7 | 8 | import { UserRepository, User } from "@repositories/user"; 9 | import { UserLogRepository, UserLog } from "@repositories/user-log"; 10 | 11 | import { DEFAULT_TASKS, TaskData } from "@tasks"; 12 | 13 | import { Config, throttle, measureTime, sleep, Logger, Loggable } from "@utils"; 14 | 15 | export class App extends Loggable { 16 | private readonly followerDataSource: DataSource; 17 | private readonly userRepository: UserRepository; 18 | private readonly userLogRepository: UserLogRepository; 19 | private readonly taskClasses = DEFAULT_TASKS; 20 | 21 | private cleaningUp = false; 22 | private config: Config | null = null; 23 | 24 | public constructor( 25 | private readonly configFilePath: string, 26 | private readonly verbose: boolean, 27 | private readonly dropDatabase: boolean, 28 | private readonly databasePath: string, 29 | ) { 30 | super("App"); 31 | Logger.verbose = verbose; 32 | 33 | if (!path.isAbsolute(this.databasePath)) { 34 | this.databasePath = path.join(process.cwd(), this.databasePath); 35 | } 36 | 37 | this.followerDataSource = new DataSource({ 38 | type: "sqlite", 39 | database: this.databasePath, 40 | entities: [User, UserLog], 41 | synchronize: true, 42 | }); 43 | 44 | this.userRepository = new UserRepository(this.followerDataSource.getRepository(User)); 45 | this.userLogRepository = new UserLogRepository(this.followerDataSource.getRepository(UserLog)); 46 | } 47 | 48 | private async doDropDatabase() { 49 | if (!fs.existsSync(this.databasePath)) { 50 | return; 51 | } 52 | 53 | await fs.unlink(this.databasePath); 54 | } 55 | 56 | public async run() { 57 | if (this.dropDatabase) { 58 | await this.logger.work({ 59 | message: "dropping database", 60 | work: () => this.doDropDatabase(), 61 | level: "info", 62 | }); 63 | } 64 | 65 | this.config = await Config.create(this.configFilePath); 66 | if (!this.config) { 67 | throw new Error("config is not loaded."); 68 | } 69 | 70 | const watchers = this.config.watchers; 71 | const notifiers = this.config.notifiers; 72 | 73 | if (!watchers.length) { 74 | this.logger.error("no watchers are configured. exiting..."); 75 | return; 76 | } 77 | 78 | await this.logger.work({ 79 | level: "info", 80 | message: "initialize database", 81 | work: () => this.followerDataSource.initialize(), 82 | }); 83 | 84 | for (const [, watcher] of watchers) { 85 | await this.logger.work({ 86 | level: "info", 87 | message: `initialize \`${chalk.green(watcher.getName())}\` watcher`, 88 | work: () => watcher.initialize(), 89 | }); 90 | } 91 | 92 | for (const [, notifier] of notifiers) { 93 | await this.logger.work({ 94 | level: "info", 95 | message: `initialize \`${chalk.green(notifier.getName())}\` notifier`, 96 | work: () => notifier.initialize(), 97 | }); 98 | } 99 | 100 | const watcherNames = watchers.map(([, p]) => `\`${chalk.green(p.getName())}\``).join(", "); 101 | this.logger.info("start to watch through {} {}.", [watcherNames, pluralize("watcher", watchers.length)]); 102 | 103 | const interval = this.config.watchInterval; 104 | while (true) { 105 | try { 106 | const [, elapsedTime] = await throttle( 107 | async () => { 108 | const result = await measureTime(async () => this.onCycle()); 109 | if ("exception" in result) { 110 | throw result.exception; 111 | } 112 | 113 | const { elapsedTime: time } = result; 114 | 115 | this.logger.info(`last task finished in ${chalk.green("{}")}.`, [ 116 | prettyMilliseconds(time, { verbose: true }), 117 | ]); 118 | 119 | this.logger.info("waiting {green} for next cycle ...", [ 120 | prettyMilliseconds(interval - time, { verbose: true }), 121 | ]); 122 | }, 123 | this.config.watchInterval, 124 | true, 125 | ); 126 | 127 | if (elapsedTime < interval) { 128 | await sleep(interval - elapsedTime); 129 | } 130 | 131 | if (this.cleaningUp) { 132 | break; 133 | } 134 | } catch (e) { 135 | if (!(e instanceof Error)) { 136 | throw e; 137 | } 138 | 139 | this.logger.error("an error occurred while processing scheduled task: {}", [e.message]); 140 | } 141 | } 142 | } 143 | 144 | private onCycle = async () => { 145 | if (!this.config) { 146 | throw new Error("Config is not loaded."); 147 | } 148 | 149 | const taskData: TaskData[] = []; 150 | for (const TaskClass of this.taskClasses) { 151 | const taskInstance = new TaskClass( 152 | this.config.watchers.map(([, watcher]) => watcher), 153 | this.config.notifiers.map(([, notifier]) => notifier), 154 | this.userRepository, 155 | this.userLogRepository, 156 | TaskClass.name, 157 | ); 158 | 159 | const data = await taskInstance.doWork(taskData); 160 | if (data.type === "terminate") { 161 | return; 162 | } 163 | 164 | taskData.push(data); 165 | } 166 | }; 167 | } 168 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 |
3 | 🦜 4 |
5 | Cage 6 | 7 |
8 |
9 | 10 |

11 | 12 |
13 | 14 | Docker Image Version (latest by date) 15 | 16 | 17 | npm (tag) 18 | 19 | 20 | MIT License 21 | 22 | 23 | Codecov 24 | 25 |
26 | (almost) realtime unfollower detection for any social services 27 |
28 |
29 |
30 | 31 | ## Introduction 32 | 33 | Cage is a cli application for detecting unfollowers on any social services. this application will check all of your followers using each watcher and compare them to the previous check. if there is a difference, it will notify you of the changes. 34 | 35 | ## Usage 36 | 37 | ```bash 38 | $ npm install -g cage-cli@dev 39 | 40 | $ cage --help 41 | 42 | Usage: cage [options] 43 | 44 | (almost) realtime unfollower detection for any social services 🦜⛓️🔒 45 | 46 | Options: 47 | -c, --config path to the configuration file (default: "./config.json") 48 | -d, --database path to the database file (default: "./data.sqlite") 49 | -p, --drop-database delete the old database file 50 | -v, --verbose enable verbose level logging 51 | -V, --version output the version number 52 | -h, --help display help for command 53 | ``` 54 | 55 | or you can just deploy with `docker` if you want: 56 | 57 | ```bash 58 | $ docker run -d --name cage async3619/cage -v /path/to/config.json:/home/node/config.json 59 | ``` 60 | 61 | of course, you can also use `docker-compose`: 62 | 63 | ```yaml 64 | version: "3.9" 65 | 66 | services: 67 | cage: 68 | image: async3619/cage 69 | volumes: 70 | - /path/to/config.json:/home/node/config.json 71 | ``` 72 | 73 | ## Watchers and Notifiers 74 | 75 | ### Watchers 76 | 77 | `Watchers` are independent feature that has the ability to watch to check users who follow your account per service. 78 | 79 | #### Supported Watchers 80 | 81 | | Service | Support? | 82 | |-----------|:--------:| 83 | | Twitter | ✅ | 84 | | GitHub | ✅ | 85 | | Mastodon | ✅ | 86 | | Instagram | ✅ | 87 | | TikTok | ❌ | 88 | | YouTube | ❌ | 89 | | Twitch | ❌ | 90 | | Facebook | ❌ | 91 | | Reddit | ❌ | 92 | | Discord | ❌ | 93 | 94 | ### Notifiers 95 | 96 | When we detect unfollowers, new followers, or any other events, Cage will notify you via `Notifiers`. 97 | 98 | #### Supported Notifiers 99 | 100 | | Service | Support? | 101 | |-----------------|:--------:| 102 | | Discord Webhook | ✅ | 103 | | Telegram Bot | ✅ | 104 | | Slack Webhook | ✅ | 105 | | Email | ❌ | 106 | | SMS | ❌ | 107 | 108 | ## Configuration 109 | 110 | this application reads configuration file from `./cage.config.json` by default. 111 | if there's no configuration file to read, this application will create you a default configuration file for you: 112 | 113 | ```json 114 | { 115 | "watchInterval": 60000, 116 | "watchers": {}, 117 | "notifiers": {} 118 | } 119 | ``` 120 | 121 | you can use json schema file `config.schema.json` on this repository. 122 | 123 | ### watchInterval: `number` (required) 124 | 125 | specify watching interval in millisecond format. minimal value is `60000`. 126 | 127 | ### watchers: `Record` (required) 128 | 129 | #### twitter 130 | 131 | Internally, Cage uses [Rettiwt-API](https://github.com/Rishikant181/Rettiwt-API) to get the followers list. so you need to get the API Key with extensions that provided by the author. you can see the instruction [here](https://github.com/Rishikant181/Rettiwt-API#1-using-a-browser-recommended). 132 | 133 | ```json5 134 | { 135 | "type": "twitter", // required 136 | "apiKey": "API Key that retrieved with X Auth Helper extension", // string, required 137 | "username": "your twitter username", // string 138 | } 139 | ``` 140 | 141 | #### github 142 | 143 | watcher configuration for GitHub service. 144 | 145 | ```json5 146 | { 147 | "type": "github", // required 148 | "authToken": "personal access token of your github account" // string, required 149 | } 150 | ``` 151 | 152 | #### instagram 153 | 154 | watcher configuration for Instagram service. 155 | 156 | ```json5 157 | { 158 | "type": "instagram", // required 159 | "username": "your instagram username", // string, required 160 | "password": "your instagram password", // string, required 161 | "targetUserName": "target instagram username you want to crawl followers", // string, required 162 | "requestDelay": 1000 // number, optional, default: 1000 163 | // requestDelay is the delay time between each request to instagram server. 164 | // this is to prevent getting banned from instagram server. 165 | } 166 | ``` 167 | 168 | ### notifiers: `Record` (required) 169 | 170 | #### discord 171 | 172 | notifier configuration for Discord Webhook notifier. 173 | 174 | ```json5 175 | { 176 | "type": "discord", 177 | "webhookUrl": "Discord Webhook url" // string, required 178 | } 179 | ``` 180 | 181 | #### telegram 182 | 183 | notifier configuration for Telegram Bot notifier. 184 | 185 | ```json5 186 | { 187 | "type": "telegram", 188 | "botToken": "token that generated from https://t.me/CageNotifierBot", // string, required 189 | "url": "custom notification relay server url" // string, optional 190 | } 191 | ``` 192 | 193 | In most cases, you will not need `url` property since since Cage already provides hosted version of this server out of the box. but if you want to use custom notification relay server, take a look [this repoitory](https://github.com/async3619/cage-telegram-helper#usage). 194 | 195 | #### slack 196 | 197 | notifier configuration for Slack Webhook notifier. 198 | 199 | ```json5 200 | { 201 | "type": "slack", 202 | "webhookUrl": "Slack Webhook url" // string, required 203 | } 204 | ``` 205 | -------------------------------------------------------------------------------- /config.schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "$ref": "#/definitions/ConfigData", 4 | "definitions": { 5 | "ConfigData": { 6 | "type": "object", 7 | "properties": { 8 | "watchInterval": { 9 | "type": "number" 10 | }, 11 | "watchers": { 12 | "type": "object", 13 | "properties": { 14 | "twitter": { 15 | "type": "object", 16 | "properties": { 17 | "type": { 18 | "type": "string", 19 | "const": "twitter" 20 | }, 21 | "apiKey": { 22 | "type": "string" 23 | }, 24 | "username": { 25 | "type": "string" 26 | } 27 | }, 28 | "required": [ 29 | "apiKey", 30 | "type", 31 | "username" 32 | ], 33 | "additionalProperties": false 34 | }, 35 | "github": { 36 | "type": "object", 37 | "properties": { 38 | "type": { 39 | "type": "string", 40 | "const": "github" 41 | }, 42 | "authToken": { 43 | "type": "string" 44 | } 45 | }, 46 | "required": [ 47 | "authToken", 48 | "type" 49 | ], 50 | "additionalProperties": false 51 | }, 52 | "mastodon": { 53 | "type": "object", 54 | "properties": { 55 | "type": { 56 | "type": "string", 57 | "const": "mastodon" 58 | }, 59 | "url": { 60 | "type": "string" 61 | }, 62 | "accessToken": { 63 | "type": "string" 64 | } 65 | }, 66 | "required": [ 67 | "accessToken", 68 | "type", 69 | "url" 70 | ], 71 | "additionalProperties": false 72 | }, 73 | "bluesky": { 74 | "type": "object", 75 | "properties": { 76 | "type": { 77 | "type": "string", 78 | "const": "bluesky" 79 | }, 80 | "service": { 81 | "type": "string" 82 | }, 83 | "email": { 84 | "type": "string" 85 | }, 86 | "password": { 87 | "type": "string" 88 | } 89 | }, 90 | "required": [ 91 | "email", 92 | "password", 93 | "type" 94 | ], 95 | "additionalProperties": false 96 | }, 97 | "instagram": { 98 | "type": "object", 99 | "properties": { 100 | "type": { 101 | "type": "string", 102 | "const": "instagram" 103 | }, 104 | "username": { 105 | "type": "string" 106 | }, 107 | "password": { 108 | "type": "string" 109 | }, 110 | "targetUserName": { 111 | "type": "string" 112 | }, 113 | "requestDelay": { 114 | "type": "number" 115 | } 116 | }, 117 | "required": [ 118 | "password", 119 | "targetUserName", 120 | "type", 121 | "username" 122 | ], 123 | "additionalProperties": false 124 | } 125 | }, 126 | "additionalProperties": false 127 | }, 128 | "notifiers": { 129 | "type": "object", 130 | "properties": { 131 | "discord": { 132 | "type": "object", 133 | "properties": { 134 | "type": { 135 | "type": "string", 136 | "const": "discord" 137 | }, 138 | "webhookUrl": { 139 | "type": "string" 140 | } 141 | }, 142 | "required": [ 143 | "type", 144 | "webhookUrl" 145 | ], 146 | "additionalProperties": false 147 | }, 148 | "telegram": { 149 | "type": "object", 150 | "properties": { 151 | "type": { 152 | "type": "string", 153 | "const": "telegram" 154 | }, 155 | "token": { 156 | "type": "string" 157 | }, 158 | "url": { 159 | "type": "string" 160 | } 161 | }, 162 | "required": [ 163 | "token", 164 | "type" 165 | ], 166 | "additionalProperties": false 167 | }, 168 | "slack": { 169 | "type": "object", 170 | "properties": { 171 | "type": { 172 | "type": "string", 173 | "const": "slack" 174 | }, 175 | "webhookUrl": { 176 | "type": "string" 177 | } 178 | }, 179 | "required": [ 180 | "type", 181 | "webhookUrl" 182 | ], 183 | "additionalProperties": false 184 | } 185 | }, 186 | "additionalProperties": false 187 | }, 188 | "ignores": { 189 | "type": "array", 190 | "items": { 191 | "type": "string" 192 | } 193 | } 194 | }, 195 | "required": [ 196 | "watchInterval", 197 | "watchers", 198 | "notifiers" 199 | ], 200 | "additionalProperties": false 201 | } 202 | } 203 | } -------------------------------------------------------------------------------- /src/utils/fetcher.spec.ts: -------------------------------------------------------------------------------- 1 | import { Headers, HeadersInit, RequestInit } from "node-fetch"; 2 | 3 | import { Fetcher } from "@utils/fetcher"; 4 | import { throttle } from "@utils/throttle"; 5 | 6 | describe("Fetcher class", function () { 7 | let target: Fetcher; 8 | 9 | beforeEach(() => { 10 | target = new Fetcher(); 11 | 12 | Object.defineProperty(target, "logger", { 13 | value: { 14 | info: jest.fn(), 15 | warn: jest.fn(), 16 | error: jest.fn(), 17 | debug: jest.fn(), 18 | verbose: jest.fn(), 19 | }, 20 | }); 21 | }); 22 | 23 | it("should provide a method to fetch data", async () => { 24 | const res = await target.fetch({ 25 | url: "https://jsonplaceholder.typicode.com/todos/1", 26 | }); 27 | 28 | expect(target.fetch).toBeDefined(); 29 | await expect(res.json()).resolves.toMatchObject({ 30 | userId: 1, 31 | id: 1, 32 | title: "delectus aut autem", 33 | completed: false, 34 | }); 35 | }); 36 | 37 | it("should provide a method to fetch json", async () => { 38 | const res = await target.fetchJson({ 39 | url: "https://jsonplaceholder.typicode.com/todos/1", 40 | }); 41 | 42 | expect(target.fetchJson).toBeDefined(); 43 | expect(res).toMatchObject({ 44 | userId: 1, 45 | id: 1, 46 | title: "delectus aut autem", 47 | completed: false, 48 | }); 49 | }); 50 | 51 | it("should add query params to the url if method is GET and data provided", async () => { 52 | let calledUrl = ""; 53 | Object.defineProperty(target, "fetchImpl", { 54 | value: jest.fn().mockImplementation((url: string) => { 55 | calledUrl = url; 56 | 57 | return Promise.resolve({ 58 | headers: { 59 | get: () => "", 60 | }, 61 | ok: true, 62 | json: () => { 63 | return Promise.resolve({ url }); 64 | }, 65 | }); 66 | }), 67 | }); 68 | 69 | await target.fetch({ 70 | url: "https://jsonplaceholder.typicode.com/todos/1", 71 | method: "GET", 72 | data: { 73 | test: "test", 74 | }, 75 | }); 76 | 77 | expect(calledUrl).toBe("https://jsonplaceholder.typicode.com/todos/1?test=test"); 78 | }); 79 | 80 | it("should add body as json if method is not GET and data provided", async () => { 81 | let calledBody: any = ""; 82 | let calledHeader: HeadersInit | undefined; 83 | Object.defineProperty(target, "fetchImpl", { 84 | value: jest.fn().mockImplementation((url: string, options: RequestInit) => { 85 | calledBody = options.body; 86 | calledHeader = options.headers; 87 | 88 | return Promise.resolve({ 89 | headers: { 90 | get: () => "", 91 | }, 92 | ok: true, 93 | json: () => { 94 | return Promise.resolve({ url }); 95 | }, 96 | }); 97 | }), 98 | }); 99 | 100 | await target.fetch({ 101 | url: "https://jsonplaceholder.typicode.com/todos/1", 102 | method: "POST", 103 | data: { 104 | test: "test", 105 | }, 106 | }); 107 | 108 | expect(calledHeader).toBeDefined(); 109 | if (calledHeader && calledHeader instanceof Headers) { 110 | expect(calledHeader.get("Content-Type")).toBe("application/json"); 111 | } 112 | 113 | expect(calledBody).toBe('{"test":"test"}'); 114 | }); 115 | 116 | it("should add cookies to the request if cookies are set", async () => { 117 | let calledHeader: HeadersInit | undefined; 118 | Object.defineProperty(target, "fetchImpl", { 119 | value: jest.fn().mockImplementation((url: string, options: RequestInit) => { 120 | calledHeader = options.headers; 121 | 122 | return Promise.resolve({ 123 | headers: { 124 | get: () => "", 125 | }, 126 | ok: true, 127 | json: () => { 128 | return Promise.resolve({ url }); 129 | }, 130 | }); 131 | }), 132 | }); 133 | 134 | target.hydrate({ 135 | cookies: { 136 | test: "test", 137 | }, 138 | }); 139 | 140 | await target.fetch({ 141 | url: "https://jsonplaceholder.typicode.com/todos/1", 142 | method: "POST", 143 | data: { 144 | test: "test", 145 | }, 146 | }); 147 | 148 | expect(calledHeader).toBeDefined(); 149 | if (calledHeader && calledHeader instanceof Headers) { 150 | expect(calledHeader.get("cookie")).toBe("test=test"); 151 | } 152 | }); 153 | 154 | it("should retry the request if it fails when retryCount is set", async () => { 155 | let calledCount = 0; 156 | Object.defineProperty(target, "fetchImpl", { 157 | value: jest.fn().mockImplementation(() => { 158 | calledCount++; 159 | 160 | return Promise.resolve({ 161 | headers: { get: () => "" }, 162 | ok: false, 163 | status: 500, 164 | statusText: "Internal Server Error", 165 | }); 166 | }), 167 | }); 168 | 169 | await expect(target.fetch({ url: "", retryCount: 3, retryDelay: 0 })).rejects.toThrow( 170 | "HTTP Error 500: Internal Server Error", 171 | ); 172 | expect(calledCount).toBe(4); 173 | }); 174 | 175 | it("should retry with delay the request if it fails when retryCount & retryDealy is set", async () => { 176 | let calledCount = 0; 177 | Object.defineProperty(target, "fetchImpl", { 178 | value: jest.fn().mockImplementation(() => { 179 | calledCount++; 180 | 181 | return Promise.resolve({ 182 | headers: { get: () => "" }, 183 | ok: false, 184 | status: 500, 185 | statusText: "Internal Server Error", 186 | }); 187 | }), 188 | }); 189 | 190 | const [, elapsedTime] = await throttle( 191 | expect(target.fetch({ url: "", retryCount: 3, retryDelay: 500 })).rejects.toThrow( 192 | "HTTP Error 500: Internal Server Error", 193 | ), 194 | 0, 195 | true, 196 | ); 197 | 198 | expect(calledCount).toBe(4); 199 | expect(elapsedTime).toBeGreaterThanOrEqual(1500); 200 | }); 201 | 202 | it("should store cookies in the cookie jar if set-cookie header is present", async () => { 203 | Object.defineProperty(target, "fetchImpl", { 204 | value: jest.fn().mockImplementation(() => { 205 | return Promise.resolve({ 206 | headers: { 207 | get: (key: string) => { 208 | if (key === "set-cookie") { 209 | return "test=test"; 210 | } 211 | 212 | return ""; 213 | }, 214 | }, 215 | ok: true, 216 | json: () => { 217 | return Promise.resolve({ url: "" }); 218 | }, 219 | }); 220 | }), 221 | }); 222 | 223 | await target.fetch({ 224 | url: "https://jsonplaceholder.typicode.com/todos/1", 225 | method: "POST", 226 | data: { 227 | test: "test", 228 | }, 229 | }); 230 | 231 | expect(target.serialize().cookies).toMatchObject({ test: "test" }); 232 | }); 233 | 234 | it("should able to get the cookies from the cookie jar", async () => { 235 | Object.defineProperty(target, "fetchImpl", { 236 | value: jest.fn().mockImplementation(() => { 237 | return Promise.resolve({ 238 | headers: { 239 | get: (key: string) => { 240 | if (key === "set-cookie") { 241 | return "test=test"; 242 | } 243 | 244 | return ""; 245 | }, 246 | }, 247 | ok: true, 248 | json: () => { 249 | return Promise.resolve({ url: "" }); 250 | }, 251 | }); 252 | }), 253 | }); 254 | 255 | await target.fetch({ 256 | url: "https://jsonplaceholder.typicode.com/todos/1", 257 | method: "POST", 258 | data: { 259 | test: "test", 260 | }, 261 | }); 262 | 263 | expect(target.getCookies()).toMatchObject({ test: "test" }); 264 | }); 265 | }); 266 | --------------------------------------------------------------------------------