├── .dockerignore ├── .gitignore ├── database ├── types │ ├── counters.ts │ ├── videos.ts │ ├── members.ts │ └── channels.ts ├── schemas │ ├── CounterSchema.ts │ ├── subschemas │ │ └── NameSubschema.ts │ ├── MemberSchema.ts │ ├── ChannelSchema.ts │ └── VideoSchema.ts ├── models │ ├── VideoModel.ts │ ├── MemberModel.ts │ ├── CounterModel.ts │ └── ChannelModel.ts ├── middlewares │ ├── ChannelMiddleware.ts │ ├── VideoMiddleware.ts │ └── MemberMiddleware.ts └── index.ts ├── src ├── modules │ ├── index.ts │ ├── logger.ts │ ├── youtube.ts │ ├── cache.ts │ └── types │ │ └── youtube.ts ├── graphql │ ├── typeDefs │ │ ├── index.ts │ │ ├── DatabaseData.ts │ │ ├── ChannelObject.ts │ │ └── VideoObject.ts │ ├── query │ │ ├── index.ts │ │ ├── consts │ │ │ └── index.ts │ │ ├── live.ts │ │ ├── data.ts │ │ ├── videos.ts │ │ └── channels.ts │ ├── root.ts │ └── index.ts └── server │ ├── index.ts │ ├── apis │ └── youtube │ │ ├── channel-updater.ts │ │ ├── types.ts │ │ ├── xml-crawler.ts │ │ └── video-updater.ts │ └── database-managers │ └── youtube.ts ├── tsconfig.json ├── Dockerfile ├── entrypoint.sh ├── .vscode └── member-template.code-snippets ├── channels ├── default │ ├── Megalight.json │ ├── Yuni Create.json │ ├── Kizuna Ai Inc..json │ ├── Iridori.json │ ├── .LIVE.json │ ├── SugarLyric.json │ ├── RumuRumu.json │ ├── Chukorara.json │ ├── VOMS.json │ ├── HoneyStrap.json │ ├── kawaii.json │ ├── Marbl_s.json │ ├── X enc'ount.json │ ├── Eilene Family.json │ ├── Hanayori Joshiryo.json │ ├── VApArt.json │ ├── V Dimension.Creators.json │ ├── Idol-bu.json │ ├── Tsunderia.json │ ├── VShojo.json │ ├── AniMare.json │ ├── Vspo.json │ ├── Nori Pro.json │ ├── ViViD.json │ ├── Atelier Live.json │ ├── ReACT.json │ ├── upd8.json │ └── Independents.json ├── template.json ├── apps │ ├── updaters │ │ └── youtube-updater.ts │ ├── database-manager.ts │ └── scrapers │ │ └── youtube-scraper.ts └── index.ts ├── environment.d.ts ├── docker-compose.yml ├── .env.sample ├── package.json ├── .eslintrc.json ├── README.md └── LICENSE /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | channels/organizations 3 | dist 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | dist 3 | channels/organizations 4 | node_modules 5 | -------------------------------------------------------------------------------- /database/types/counters.ts: -------------------------------------------------------------------------------- 1 | import { Document } from 'mongoose'; 2 | 3 | export interface CounterProps extends Document { 4 | _id: string; 5 | index: number; 6 | } 7 | -------------------------------------------------------------------------------- /src/modules/index.ts: -------------------------------------------------------------------------------- 1 | export * from '../../database'; 2 | export * as memcache from './cache'; 3 | export { default as debug } from './logger'; 4 | export * as youtube from './youtube'; 5 | -------------------------------------------------------------------------------- /database/schemas/CounterSchema.ts: -------------------------------------------------------------------------------- 1 | import { Schema } from 'mongoose'; 2 | 3 | export const CounterSchema = new Schema({ 4 | _id: String, 5 | index: { 6 | type: Number, 7 | default: 0 8 | } 9 | }); 10 | -------------------------------------------------------------------------------- /database/models/VideoModel.ts: -------------------------------------------------------------------------------- 1 | import { model, Model } from 'mongoose'; 2 | import { VideoSchema } from '../schemas/VideoSchema'; 3 | import { VideoProps } from '../types/videos'; 4 | 5 | export const Videos: Model = model('Videos', VideoSchema); 6 | -------------------------------------------------------------------------------- /database/models/MemberModel.ts: -------------------------------------------------------------------------------- 1 | import { model, Model } from 'mongoose'; 2 | import { MemberSchema } from '../schemas/MemberSchema'; 3 | import { MemberProps } from '../types/members'; 4 | 5 | export const Members: Model = model('Members', MemberSchema); 6 | -------------------------------------------------------------------------------- /src/graphql/typeDefs/index.ts: -------------------------------------------------------------------------------- 1 | import { typeDef as channelDefs } from './ChannelObject'; 2 | import { typeDef as videoDefs } from './VideoObject'; 3 | import { typeDef as dataDefs } from './DatabaseData'; 4 | export const typeDefs = [channelDefs, videoDefs, dataDefs]; 5 | -------------------------------------------------------------------------------- /database/middlewares/ChannelMiddleware.ts: -------------------------------------------------------------------------------- 1 | import { ChannelSchema } from '../schemas/ChannelSchema'; 2 | import { ChannelProps } from '../types/channels'; 3 | 4 | ChannelSchema.pre('updateOne', function() { 5 | this.set({ updated_at: new Date() }); 6 | }); 7 | -------------------------------------------------------------------------------- /database/models/CounterModel.ts: -------------------------------------------------------------------------------- 1 | import { model, Model } from 'mongoose'; 2 | import { CounterSchema } from '../schemas/CounterSchema'; 3 | import { CounterProps } from '../types/counters'; 4 | 5 | export const Counter: Model = model('Counter', CounterSchema); 6 | -------------------------------------------------------------------------------- /database/models/ChannelModel.ts: -------------------------------------------------------------------------------- 1 | import { model, Model } from 'mongoose'; 2 | import { ChannelSchema } from '../schemas/ChannelSchema'; 3 | import { ChannelProps } from '../types/channels'; 4 | 5 | export const Channels: Model = model('Channels', ChannelSchema); 6 | -------------------------------------------------------------------------------- /database/schemas/subschemas/NameSubschema.ts: -------------------------------------------------------------------------------- 1 | import { Schema } from 'mongoose'; 2 | 3 | export const NameSchema = new Schema({ 4 | 'en': { 5 | type: String, 6 | required: true 7 | }, 8 | 'jp': String, 9 | 'kr': String, 10 | 'cn': String 11 | }, { _id: false }); 12 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "CommonJS", 4 | "target": "ES2019", 5 | "lib": ["ES2020"], 6 | "moduleResolution": "Node", 7 | "esModuleInterop": true, 8 | "outDir": "dist", 9 | "forceConsistentCasingInFileNames": true, 10 | "noImplicitThis": true 11 | } 12 | } -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:14 2 | EXPOSE 2434 3 | 4 | COPY . /opt/src/ 5 | WORKDIR /opt/src 6 | RUN cp -v .env.sample .env 7 | RUN wget -q https://repo.mongodb.org/apt/debian/dists/stretch/mongodb-org/4.4/main/binary-amd64/mongodb-org-shell_4.4.0_amd64.deb && \ 8 | dpkg -i ./mongodb-org-shell_4.4.0_amd64.deb && \ 9 | npm i 10 | 11 | ENTRYPOINT ["./entrypoint.sh"] -------------------------------------------------------------------------------- /database/middlewares/VideoMiddleware.ts: -------------------------------------------------------------------------------- 1 | import { VideoSchema } from '../schemas/VideoSchema'; 2 | import { VideoProps } from '../types/videos'; 3 | 4 | VideoSchema.pre('updateOne', function() { 5 | this.set({ updated_at: new Date() }); 6 | }); 7 | 8 | VideoSchema.pre('save', async function() { 9 | if (this.isNew) this.updated_at = new Date(); 10 | }); 11 | -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | npm run copy-default 3 | 4 | until channelsCount=`mongo vtapi-mongo/data --quiet --eval="db.channels.count()"`; do 5 | >&2 echo "Mongo is unavailable - sleeping" 6 | sleep 1 7 | done 8 | videosCount=`mongo vtapi-mongo/data --quiet --eval="db.videos.count()"` 9 | 10 | # Run init script only once 11 | if (( channelsCount==0 && videosCount==0 )); then 12 | echo "Run init script" 13 | npm run init 14 | fi 15 | npm start -------------------------------------------------------------------------------- /.vscode/member-template.code-snippets: -------------------------------------------------------------------------------- 1 | { 2 | "Channel Template": { 3 | "scope": "json", 4 | "prefix": "member", 5 | "body": [ 6 | "{", 7 | " \"name\": {", 8 | " \"en\": \"$1\",", 9 | " \"jp\": \"$2\",", 10 | " \"kr\": \"$3\",", 11 | " \"cn\": \"$4\"", 12 | " },", 13 | " \"platform_id\": \"${5|yt,bb,tt|}\",", 14 | " \"channel_id\": \"$6\",", 15 | " \"details\": {", 16 | " \"twitter\": \"$7\"", 17 | " }", 18 | "}" 19 | ] 20 | } 21 | } -------------------------------------------------------------------------------- /src/graphql/typeDefs/DatabaseData.ts: -------------------------------------------------------------------------------- 1 | import { gql } from 'apollo-server'; 2 | 3 | export const typeDef = gql` 4 | type DataObject { 5 | organizations: [String]! 6 | channels: Int! 7 | videos: Int! 8 | } 9 | extend type Query { 10 | data( 11 | channel_id: [ID] 12 | exclude_channel_id: [ID] 13 | organizations: [String] 14 | exclude_organizations: [String] 15 | ): DataObject! 16 | @rateLimit(window: "1s", max: 10, message: "You are doing that too often.") 17 | } 18 | `; 19 | -------------------------------------------------------------------------------- /channels/default/Megalight.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Suzuka Stella", 5 | "jp": "鈴花ステラ" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UChAOCCFuF2hto05Z68xp56A", 9 | "details": { 10 | "twitter": "_suzukastella" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Hien Madoka", 16 | "jp": "火閻まどか" 17 | }, 18 | "platform_id": "yt", 19 | "channel_id": "UCBhhDcVyOAhmUERi1PsQ4Rw", 20 | "details": { 21 | "twitter": "hienmadoka" 22 | } 23 | } 24 | ] -------------------------------------------------------------------------------- /channels/default/Yuni Create.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Mochiduki Himari", 5 | "jp": "餅月ひまり" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UCnZUaScptUZ7vBeV-4Vf4nw", 9 | "details": { 10 | "twitter": "HimariMochimoon" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Akatsuki Yuni", 16 | "jp": "赤月ゆに" 17 | }, 18 | "platform_id": "yt", 19 | "channel_id": "UCsA8nM8e5--IdeRaxZwTWHg", 20 | "details": { 21 | "twitter": "AkatsukiUNI" 22 | } 23 | } 24 | ] -------------------------------------------------------------------------------- /database/schemas/MemberSchema.ts: -------------------------------------------------------------------------------- 1 | import { Schema } from 'mongoose'; 2 | import { NameSchema } from './subschemas/NameSubschema'; 3 | 4 | export const MemberSchema = new Schema({ 5 | '_id': Number, 6 | 'name': { 7 | type: NameSchema, 8 | required: true 9 | }, 10 | 'organization': String, 11 | 'platform_id': { 12 | type: String, 13 | enum: ['yt', 'bb', 'tt'], 14 | required: true 15 | }, 16 | 'channel_id': { 17 | type: String, 18 | required: true, 19 | unique: true 20 | }, 21 | 'details': Schema.Types.Mixed, 22 | 'crawled_at': Date, 23 | 'updated_at': Date 24 | }); 25 | -------------------------------------------------------------------------------- /src/graphql/query/index.ts: -------------------------------------------------------------------------------- 1 | import schedule from 'node-schedule'; 2 | 3 | process.env.CACHE_MINUTE = `${new Date().getMinutes()}`; 4 | const { GQL_CACHE_INVALIDATE } = process.env; 5 | const INVALIDATE_SECOND = isNaN(+GQL_CACHE_INVALIDATE) ? '8' : GQL_CACHE_INVALIDATE; 6 | 7 | schedule.scheduleJob( 8 | 'invalidate-live-cache', 9 | `${INVALIDATE_SECOND} * * * * *`, 10 | () => process.env.CACHE_MINUTE = `${new Date().getMinutes()}` 11 | ); 12 | 13 | import { channels } from './channels'; 14 | import { live } from './live'; 15 | import { videos } from './videos'; 16 | import { data } from './data'; 17 | export const Query = { channels, live, videos, data }; 18 | -------------------------------------------------------------------------------- /environment.d.ts: -------------------------------------------------------------------------------- 1 | export {}; 2 | declare global { 3 | namespace NodeJS { 4 | interface ProcessEnv { 5 | NODE_ENV: 'development'|'production'; 6 | GOOGLE_API_KEY: string; 7 | PORT: string; 8 | LOG_LEVEL: string; 9 | MONGO_HOST: string; 10 | MONGO_PORT: string; 11 | MEMCACHED_HOST: string; 12 | MEMCACHED_PORT: string; 13 | TTL_SHORT: string; 14 | TTL_LONG: string; 15 | TIMINGS_YOUTUBE_CHANNEL_UPDATER: string; 16 | TIMINGS_YOUTUBE_VIDEO_UPDATER: string; 17 | TIMINGS_YOUTUBE_XML_CRAWLER: string; 18 | GQL_CACHE_INVALIDATE: string; 19 | CACHE_MINUTE: string; 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.1' 2 | 3 | services: 4 | mongo: 5 | container_name: vtapi-mongo 6 | image: mongo:4 7 | restart: always 8 | memcached: 9 | container_name: vtapi-memcached 10 | image: memcached:latest 11 | restart: always 12 | vtapi: 13 | depends_on: 14 | - mongo 15 | - memcached 16 | image: vtapi:latest 17 | ports: 18 | - 2434:2434 19 | restart: always 20 | environment: 21 | MONGO_HOST: vtapi-mongo 22 | MEMCACHED_HOST: vtapi-memcached 23 | GOOGLE_API_KEY: # Fill this with your api key 24 | command: # Setup channels, delete if json files copied manually ex. --animare copies animare.json to channels -------------------------------------------------------------------------------- /src/graphql/root.ts: -------------------------------------------------------------------------------- 1 | import { gql } from 'apollo-server'; 2 | import { GraphQLJSON } from 'graphql-type-json'; 3 | import { GraphQLDateTime } from 'graphql-iso-date'; 4 | import { Query } from './query'; 5 | import { typeDefs as types } from './typeDefs'; 6 | 7 | const root = gql` 8 | scalar JSON 9 | scalar DateTime 10 | type Query { 11 | root: String 12 | } 13 | directive @rateLimit( 14 | max: Int, 15 | window: String, 16 | message: String, 17 | identityArgs: [String], 18 | arrayLengthField: String 19 | ) on FIELD_DEFINITION 20 | `; 21 | 22 | export const typeDefs = types.concat(root); 23 | export const resolvers = { 24 | JSON: GraphQLJSON, 25 | DateTime: GraphQLDateTime, 26 | Query 27 | }; 28 | -------------------------------------------------------------------------------- /database/types/videos.ts: -------------------------------------------------------------------------------- 1 | import { Document } from 'mongoose'; 2 | import { PlatformId } from './members'; 3 | import type { VideoId } from '../../src/server/apis/youtube/types'; 4 | 5 | export type VideoStatus = 'live'|'upcoming'|'ended'|'uploaded'|'missing'|'new'; 6 | 7 | export interface VideoObject { 8 | _id: VideoId; 9 | platform_id: PlatformId; 10 | channel_id: string; 11 | organization?: string; 12 | title: string; 13 | time?: { 14 | published?: Date; 15 | scheduled?: Date; 16 | start?: Date; 17 | end?: Date; 18 | duration?: number; 19 | }; 20 | status?: VideoStatus; 21 | viewers?: number; 22 | updated_at?: Date; 23 | } 24 | 25 | export interface VideoProps extends Document, VideoObject { 26 | _id: VideoId; 27 | } 28 | -------------------------------------------------------------------------------- /database/types/members.ts: -------------------------------------------------------------------------------- 1 | import { Document } from 'mongoose'; 2 | 3 | export type PlatformId = 'yt'|'bb'|'tt'; 4 | export interface MemberNames { 5 | en: string; 6 | jp?: string; 7 | kr?: string; 8 | id?: string; 9 | } 10 | 11 | export type YoutubeChannelId = string; 12 | export type BilibiliChannelId = string; 13 | export type TwitterHandle = string; 14 | 15 | export interface MemberObject { 16 | name?: MemberNames; 17 | organization?: string; 18 | platform_id?: PlatformId; 19 | channel_id?: YoutubeChannelId; 20 | details?: { 21 | twitter?: TwitterHandle; 22 | [key: string]: unknown; 23 | }; 24 | crawled_at?: Date; 25 | updated_at?: Date; 26 | } 27 | 28 | export interface MemberProps extends Document, MemberObject { 29 | _id: number; 30 | } 31 | -------------------------------------------------------------------------------- /channels/default/Kizuna Ai Inc..json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Kizuna AI", 5 | "jp": "キズナアイ" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UC4YaOt1yT-ZeyB0OmxHgolA", 9 | "details": { 10 | "twitter": "aichan_nel" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Kizuna AI", 16 | "jp": "キズナアイ" 17 | }, 18 | "platform_id": "yt", 19 | "channel_id": "UCbFwe3COkDrbNsbMyGNCsDg", 20 | "details": { 21 | "twitter": "aichan_nel" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Love-chan", 27 | "jp": "Loveちゃん" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCYm8zALd2uHqyy6C1tb4_zA", 31 | "details": { 32 | "twitter": "lovechan_lp" 33 | } 34 | } 35 | ] -------------------------------------------------------------------------------- /channels/default/Iridori.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Shizuku Fuyumori", 5 | "jp": "冬守しずく" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UCj6ZvZqDJsrwBal-5R_q1ZQ", 9 | "details": { 10 | "twitter": "fuyumori_1616" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Misono Yui", 16 | "jp": "御園結唯" 17 | }, 18 | "platform_id": "yt", 19 | "channel_id": "UCsHILl-8i1H9316RPeTEcfQ", 20 | "details": { 21 | "twitter": "misono_1616" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Fukumaki Yuka", 27 | "jp": "服巻有香" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCpLvJ42sm-edPGF8pKJonOQ", 31 | "details": { 32 | "twitter": "fukumaki_1616" 33 | } 34 | } 35 | ] -------------------------------------------------------------------------------- /channels/default/.LIVE.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Dennou Shojo Shiro", 5 | "jp": "電脳少女シロ" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UCLhUvJ_wO9hOvv_yYENu4fQ", 9 | "details": { 10 | "twitter": "SIROyoutuber" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Dennou Shojo Shiro", 16 | "jp": "電脳少女シロ" 17 | }, 18 | "platform_id": "bb", 19 | "channel_id": "391876799", 20 | "details": { 21 | "twitter": "SIROyoutuber" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Merry Milk", 27 | "jp": "メリーミルク" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCju7v8SkoWUQ5ITCQwmYpYg", 31 | "details": { 32 | "twitter": "milk_merry_" 33 | } 34 | } 35 | ] -------------------------------------------------------------------------------- /channels/default/SugarLyric.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Shisho Chris", 5 | "jp": "獅子王クリス" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UC--A2dwZW7-M2kID0N6_lfA", 9 | "details": { 10 | "twitter": "ChrisShishio" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Ryugasaki Rene", 16 | "jp": "龍ヶ崎リン" 17 | }, 18 | "platform_id": "yt", 19 | "channel_id": "UC2hc-00y-MSR6eYA4eQ4tjQ", 20 | "details": { 21 | "twitter": "Rene_Ryugasaki" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Kojo Anna", 27 | "jp": "虎城アンナ" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCvPPBoTOor5gm8zSlE2tg4w", 31 | "details": { 32 | "twitter": "Anna_Kojo" 33 | } 34 | } 35 | ] -------------------------------------------------------------------------------- /database/types/channels.ts: -------------------------------------------------------------------------------- 1 | import { Document } from 'mongoose'; 2 | import { BilibiliChannelId, MemberNames, PlatformId, TwitterHandle, YoutubeChannelId } from './members'; 3 | 4 | export interface ChannelObject { 5 | _id?: number; 6 | name: MemberNames; 7 | organization: string; 8 | platform_id: PlatformId; 9 | channel_name: string; 10 | channel_id: YoutubeChannelId|BilibiliChannelId; 11 | details?: { 12 | twitter?: TwitterHandle; 13 | [key: string]: unknown; 14 | }; 15 | channel_stats?: { 16 | published_at?: Date; 17 | views?: number; 18 | subscribers?: number; 19 | videos?: number; 20 | }; 21 | description?: string; 22 | thumbnail?: string; 23 | updated_at?: Date; 24 | } 25 | 26 | export interface ChannelProps extends ChannelObject, Document { 27 | _id: number; 28 | } 29 | -------------------------------------------------------------------------------- /database/middlewares/MemberMiddleware.ts: -------------------------------------------------------------------------------- 1 | import { MemberSchema } from '../schemas/MemberSchema'; 2 | import { Counter } from '../models/CounterModel'; 3 | import { MemberProps } from '../types/members'; 4 | 5 | async function getId(increment = true) { 6 | return Counter.findByIdAndUpdate('member_id', 7 | { $inc: { index: increment ? 1 : -1 } }, 8 | { upsert: true } 9 | ).then(counter => counter?.index ?? 0); 10 | } 11 | 12 | MemberSchema.pre('save', async function() { 13 | this.updated_at = new Date(); 14 | if (this.isNew) this._id = await getId(); 15 | }); 16 | 17 | // On duplicate error, decrement id to prevent id jumping. 18 | MemberSchema.post('save', function(err, doc, next) { 19 | if (err.name === 'MongoError' && err.code === 11000) { 20 | return getId(false); 21 | } else { return next(); } 22 | }); 23 | -------------------------------------------------------------------------------- /database/schemas/ChannelSchema.ts: -------------------------------------------------------------------------------- 1 | import { Schema } from 'mongoose'; 2 | import { NameSchema } from './subschemas/NameSubschema'; 3 | 4 | export const ChannelSchema = new Schema({ 5 | '_id': Number, 6 | 'name': NameSchema, 7 | 'organization': { 8 | type: String, 9 | required: true 10 | }, 11 | 'platform_id': { 12 | type: String, 13 | enum: ['yt', 'bb', 'tt'], 14 | required: true 15 | }, 16 | 'channel_name': String, 17 | 'channel_id': { 18 | type: String, 19 | required: true, 20 | unique: true 21 | }, 22 | 'details': Schema.Types.Mixed, 23 | 'channel_stats': new Schema({ 24 | 'published_at': Date, 25 | 'views': Number, 26 | 'subscribers': Number, 27 | 'videos': Number 28 | }, { _id: false }), 29 | 'description': String, 30 | 'thumbnail': String, 31 | 'updated_at': Date 32 | }); 33 | -------------------------------------------------------------------------------- /src/graphql/index.ts: -------------------------------------------------------------------------------- 1 | import { config } from 'dotenv'; 2 | config(); 3 | 4 | import { ApolloServer } from 'apollo-server'; 5 | import { createRateLimitDirective } from 'graphql-rate-limit'; 6 | import debug from '../modules/logger'; 7 | import { resolvers, typeDefs } from './root'; 8 | 9 | const rateLimitDirective = createRateLimitDirective({ identifyContext: (ctx) => ctx.id }); 10 | const { NODE_ENV, PORT } = process.env; 11 | 12 | const logger = debug('app'); 13 | const server = new ApolloServer({ 14 | schemaDirectives: { 15 | rateLimit: rateLimitDirective 16 | }, 17 | typeDefs, 18 | resolvers, 19 | introspection: true, 20 | playground: { 21 | endpoint: '/v1' 22 | }, 23 | tracing: NODE_ENV === 'development' 24 | }); 25 | 26 | server.listen(+PORT || 2434).then(({ url }) => { 27 | logger.info(`Server ready at ${url}`); 28 | }); 29 | -------------------------------------------------------------------------------- /src/server/index.ts: -------------------------------------------------------------------------------- 1 | import { config } from 'dotenv'; 2 | config(); 3 | 4 | import { scheduleJob } from 'node-schedule'; 5 | 6 | export async function youtube() { 7 | const TIMINGS = { 8 | CHANNEL_UPDATER: process.env.TIMINGS_YOUTUBE_CHANNEL_UPDATER ?? '*/15 * * * *', 9 | VIDEO_UPDATER: process.env.TIMINGS_YOUTUBE_VIDEO_UPDATER ?? '5 * * * * *', 10 | XML_CRAWLER: process.env.TIMINGS_YOUTUBE_XML_CRAWLER ?? '1 * * * * *' 11 | }; 12 | const [channel_updater, video_updater] = await Promise.all([ 13 | import('./apis/youtube/channel-updater').then(api => api.default), 14 | import('./apis/youtube/video-updater').then(api => api.default), 15 | import('./apis/youtube/xml-crawler').then(api => api.init(TIMINGS.XML_CRAWLER)), 16 | ]); 17 | scheduleJob('api-youtube-channel-updater', TIMINGS.CHANNEL_UPDATER, channel_updater); 18 | scheduleJob('api-youtube-video_updater', TIMINGS.VIDEO_UPDATER, video_updater); 19 | } 20 | 21 | youtube(); 22 | -------------------------------------------------------------------------------- /database/index.ts: -------------------------------------------------------------------------------- 1 | import { connect, connection } from 'mongoose'; 2 | import debug from '../src/modules/logger'; 3 | const logger = debug('db:mongoose'); 4 | 5 | const URI = `mongodb://${process.env.MONGO_HOST ?? 'localhost'}:${process.env.MONGO_PORT ?? '27017'}/vt-api`; 6 | 7 | const options = { 8 | useNewUrlParser: true, 9 | useUnifiedTopology: true, 10 | useCreateIndex: true, 11 | useFindAndModify: false 12 | }; 13 | 14 | // establish connection and log on status change 15 | connect(URI, options); 16 | connection.on('connected', () => logger.log('Established connection to MongoDB.')); 17 | connection.on('disconnected', () => logger.warn('Lost connection to MongoDB.')); 18 | 19 | // load middlewares 20 | import './middlewares/ChannelMiddleware'; 21 | import './middlewares/MemberMiddleware'; 22 | import './middlewares/VideoMiddleware'; 23 | 24 | // re-export models 25 | export * from './models/ChannelModel'; 26 | export * from './models/CounterModel'; 27 | export * from './models/MemberModel'; 28 | export * from './models/VideoModel'; 29 | -------------------------------------------------------------------------------- /channels/default/RumuRumu.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Amatsuka Kayano", 5 | "jp": "天使かやの" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UCaBNbbikqjXNn8uKBMVj-cQ", 9 | "details": { 10 | "twitter": "AMATSUKA_KAYANO" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Marika Matsurika", 16 | "jp": "マリカ・マツリカ" 17 | }, 18 | "platform_id": "yt", 19 | "channel_id": "UCxcqGAAVl4wL6BlK4Rm4ISA", 20 | "details": { 21 | "twitter": "MarikaMa2rika" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Yurika Matsurika", 27 | "jp": "ユリカ・マツリカ" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCG5NiZERHUgjDiWrDJnaUuw", 31 | "details": { 32 | "twitter": "yurikama2rika" 33 | } 34 | }, 35 | { 36 | "name": { 37 | "en": "Menmo Hanashiro", 38 | "jp": "花城めんも" 39 | }, 40 | "platform_id": "yt", 41 | "channel_id": "UCAb1PTLhKoDFQQOBrInGDQg", 42 | "details": { 43 | "twitter": "hanashiro_menmo" 44 | } 45 | } 46 | ] -------------------------------------------------------------------------------- /channels/default/Chukorara.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Shito Anon", 5 | "jp": "紫桃あのん" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UCUKngXhjnKJ6KCyuC7ejI_w", 9 | "details": { 10 | "twitter": "Shito_ANON" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Hiseki Erio", 16 | "jp": "緋赤エリオ" 17 | }, 18 | "platform_id": "yt", 19 | "channel_id": "UCQ7KYc1-IMQ8_Uz7XTE5DWw", 20 | "details": { 21 | "twitter": "hiseki_erio" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Komori Chiyu", 27 | "jp": "古守ちゆ" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCBIR44irWpj1eTx0ZQFofHg", 31 | "details": { 32 | "twitter": "komori_chiyu" 33 | } 34 | }, 35 | { 36 | "name": { 37 | "en": "Tsudurime Higasa", 38 | "jp": "綴目日傘" 39 | }, 40 | "platform_id": "yt", 41 | "channel_id": "UCwBMSUVMvck6EaBD0f1tHsQ", 42 | "details": { 43 | "twitter": "tsuzurime" 44 | } 45 | } 46 | ] -------------------------------------------------------------------------------- /channels/default/VOMS.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Amano Pikamee", 5 | "jp": "天野ピカミィ" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UCajhBT4nMrg3DLS-bLL2RCg", 9 | "details": { 10 | "twitter": "amanopikamee" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Amano Pikamee", 16 | "jp": "天野ピカミィ" 17 | }, 18 | "platform_id": "tt", 19 | "channel_id": "pikameeamano", 20 | "details": { 21 | "twitter": "amanopikamee" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Hikasa Tomoshika", 27 | "jp": "緋笠トモシカ" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UC3vzVK_N_SUVKqbX69L_X4g", 31 | "details": { 32 | "twitter": "Tomoshika_H" 33 | } 34 | }, 35 | { 36 | "name": { 37 | "en": "Jitomi Monoe", 38 | "jp": "磁富モノエ" 39 | }, 40 | "platform_id": "yt", 41 | "channel_id": "UCaFhsCKSSS821N-EcWmPkUQ", 42 | "details": { 43 | "twitter": "Jitomi_Monoe" 44 | } 45 | } 46 | ] -------------------------------------------------------------------------------- /database/schemas/VideoSchema.ts: -------------------------------------------------------------------------------- 1 | import { Schema } from 'mongoose'; 2 | 3 | export const VideoSchema = new Schema({ 4 | '_id': String, 5 | 'platform_id': { 6 | type: String, 7 | enum: ['yt', 'bb', 'tt'], 8 | required: true 9 | }, 10 | 'channel_id': { 11 | type: String, 12 | required: true 13 | }, 14 | 'organization': { 15 | type: String, 16 | required: true 17 | }, 18 | 'title': { 19 | type: String, 20 | required: true 21 | }, 22 | 'time': new Schema({ 23 | 'published': Date, 24 | 'scheduled': Date, 25 | 'start': Date, 26 | 'end': Date, 27 | 'duration': { 28 | type: Number, 29 | default: function(this: {start: Date; end: Date;}) { 30 | if (this.start && this.end) { 31 | return +this.end - +this.start; 32 | } 33 | } 34 | } 35 | }, { _id: false }), 36 | 'status': { 37 | type: String, 38 | enum: ['live', 'upcoming', 'ended', 'uploaded', 'missing', 'new'], 39 | required: true 40 | }, 41 | 'viewers': Number, 42 | 'updated_at': Date 43 | }); 44 | -------------------------------------------------------------------------------- /src/modules/logger.ts: -------------------------------------------------------------------------------- 1 | import { debug as Debug } from 'debug'; 2 | 3 | let LOG_LEVEL = +process.env.LOG_LEVEL; 4 | LOG_LEVEL = !LOG_LEVEL ? 2 : LOG_LEVEL < 0 ? 0 : LOG_LEVEL > 3 ? 3 : LOG_LEVEL; 5 | 6 | export default function debug(namespace: string) { 7 | const logger = Debug(namespace); 8 | const log = logger.extend('[ LOG ]:'); 9 | log.log = console.log.bind(console); 10 | const info = logger.extend('[ INFO ]:'); 11 | info.log = console.info.bind(console); 12 | info.color = log.color; 13 | const warn = logger.extend('[ WARN ]:'); 14 | warn.log = console.warn.bind(console); 15 | warn.color = '9'; 16 | const error = logger.extend('[ ERROR ]:'); 17 | error.log = console.error.bind(console); 18 | error.color = '196'; 19 | switch (LOG_LEVEL) { 20 | case 0: warn.enabled = false; 21 | case 1: info.enabled = false; 22 | case 2: log.enabled = false; 23 | } 24 | return { extend: extend.bind(logger), log, info, warn, error }; 25 | } 26 | function extend(this: debug.Debugger, namespace: string) { 27 | return debug(`${this.namespace}:${namespace}`); 28 | } 29 | -------------------------------------------------------------------------------- /channels/default/HoneyStrap.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Suou Patra", 5 | "jp": "周防パトラ" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UCeLzT-7b2PBcunJplmWtoDg", 9 | "details": { 10 | "twitter": "Patra_HNST" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Shimamura Charlotte", 16 | "jp": "島村シャルロット" 17 | }, 18 | "platform_id": "yt", 19 | "channel_id": "UCYTz3uIgwVY3ZU-IQJS8r3A", 20 | "details": { 21 | "twitter": "Charlotte_HNST" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Saionji Mary", 27 | "jp": "西園寺メアリ" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCwePpiw1ocZRSNSkpKvVISw", 31 | "details": { 32 | "twitter": "Mary_HNST" 33 | } 34 | }, 35 | { 36 | "name": { 37 | "en": "Sekishiro Mico", 38 | "jp": "堰代ミコ" 39 | }, 40 | "platform_id": "yt", 41 | "channel_id": "UCDh2bWI5EDu7PavqwICkVpA", 42 | "details": { 43 | "twitter": "Mico_HNST" 44 | } 45 | } 46 | ] -------------------------------------------------------------------------------- /src/graphql/query/consts/index.ts: -------------------------------------------------------------------------------- 1 | export type Sort = 'asc'|'desc'; 2 | export const escapeRegex = (text: string) => text.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&'); 3 | export const getNextToken = (text: any) => Buffer.from(text.toString()).toString('base64'); 4 | export const parseToken = (token: string) => Buffer.from(token, 'base64'); 5 | export const getCacheKey = (query: string, invalidate = true) => `${query}:${invalidate ? process.env.CACHE_MINUTE : 0}`.replace(/ |\n/g, ''); 6 | export const parseOrganization = (organizations: string[]) => organizations.length ? '^' + organizations.map(escapeRegex).sort().join('|^') : ''; 7 | export const cutChannelIds = (idList: string[]) => idList.map(id => id.slice(0, 3) + id.slice(-3)); 8 | export const cutGroupString = (groupList: string) => groupList.split('|').map(group => group.slice(1, 3) + group.slice(-2)); 9 | export const minsToMs = (mins: number) => mins * 6e4; 10 | export const firstField = (obj: {[key: string]: any;}): [{[key: string]: any;}, string] => { 11 | const [key, value] = Object.entries(obj)[0]; 12 | return [{ [key]: value }, `${key.slice(0, 2)}-${value[0]}`]; 13 | }; 14 | -------------------------------------------------------------------------------- /src/modules/youtube.ts: -------------------------------------------------------------------------------- 1 | import fetch from 'node-fetch'; 2 | import debug from './logger'; 3 | import { PlaylistParams, SearchParams, YoutubeChannelResponse, YoutubePlaylistItemsResponse, YoutubeVideoResponse } from './types/youtube'; 4 | 5 | const URL = 'https://www.googleapis.com/youtube/v3/'; 6 | const SETTINGS = `key=${process.env.GOOGLE_API_KEY}&accept=application/json&`; 7 | const logger = debug('module:youtube'); 8 | 9 | const parseParams = (params = {}) => SETTINGS + Object.entries(params).map(([k, v]) => k + '=' + v).join('&'); 10 | const youtubeFetch = async (type: string, params: any) => { 11 | logger.log(`Youtube ${type} Endpoint;\npart=${params.part};\nids=${(params?.id ?? params?.playlistId).split(',').length};`); 12 | return fetch(`${URL}${type}?${parseParams(params)}`).then(res => res.json()).then(res => { if (res.error) throw res.error; return res; }); 13 | }; 14 | 15 | export const videos = (params: SearchParams): Promise => youtubeFetch('videos', params); 16 | export const channels = (params: SearchParams): Promise => youtubeFetch('channels', params); 17 | export const playlistItems = (params: PlaylistParams): Promise => youtubeFetch('playlistItems', params); 18 | -------------------------------------------------------------------------------- /channels/default/kawaii.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Reina Sun", 5 | "jp": "レイナ・サン" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UCKJexadCeNo3lu0U20skmNg", 9 | "details": { 10 | "twitter": "reina_kawaiii" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Nene Amano", 16 | "jp": "天野寧々" 17 | }, 18 | "platform_id": "yt", 19 | "channel_id": "UCu1INzefw3R7M9-3QEHYmMQ", 20 | "details": { 21 | "twitter": "amanene_kawaii" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Isla Coleman", 27 | "jp": "アイラ・コールマン" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCuaDidGk4HNLOnQE9wzMMeQ", 31 | "details": { 32 | "twitter": "isla_kawaii" 33 | } 34 | }, 35 | { 36 | "name": { 37 | "en": "Charlotte Suzu", 38 | "jp": "シャーロット・スズ" 39 | }, 40 | "platform_id": "yt", 41 | "channel_id": "UCoAw3SML_09dF-7yhQU8wFQ", 42 | "details": { 43 | "twitter": "charsuzu_kawaii" 44 | } 45 | }, 46 | { 47 | "name": { 48 | "en": "Hana Flores", 49 | "jp": "ハナ・フローレス" 50 | }, 51 | "platform_id": "yt", 52 | "channel_id": "UCkvYfE6_lEmstu6GoqPUOZQ", 53 | "details": { 54 | "twitter": "HanaFlo_Kawaii" 55 | } 56 | } 57 | ] -------------------------------------------------------------------------------- /channels/default/Marbl_s.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Bando", 5 | "jp": "ばんどうりいな" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UCtjQoCilYbnxUXquXcVU3uA", 9 | "details": { 10 | "twitter": "Bando004" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Miwao", 16 | "jp": "那加みわお" 17 | }, 18 | "platform_id": "yt", 19 | "channel_id": "UCv6rts7X60zoL__5JPwdPGA", 20 | "details": { 21 | "twitter": "Miwao_chan" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Kusunoki Sarara", 27 | "jp": "楠木さらら" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCkkdEmzZHMp9xqs_F7v1EQQ", 31 | "details": { 32 | "twitter": "sarara_v" 33 | } 34 | }, 35 | { 36 | "name": { 37 | "en": "Shimada Miharu", 38 | "jp": "島田みはる" 39 | }, 40 | "platform_id": "yt", 41 | "channel_id": "UCfwAKc_vwVTzTcm1wki8oHg", 42 | "details": { 43 | "twitter": "shimada_miharu" 44 | } 45 | }, 46 | { 47 | "name": { 48 | "en": "Yun Chamu", 49 | "jp": "ゆんちゃむ" 50 | }, 51 | "platform_id": "yt", 52 | "channel_id": "UCa9wTL0z_iNN13n4G5VmbJg", 53 | "details": { 54 | "twitter": "yuntyamutyamu" 55 | } 56 | } 57 | ] -------------------------------------------------------------------------------- /channels/default/X enc'ount.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "X'Flare", 5 | "jp": "クロスフレア" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UCaWuiIyX89dLP5H4pA7QAvQ", 9 | "details": { 10 | "twitter": "XFlareVtuber1" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "BΣretta X' Rain", 16 | "jp": "レッタ クロスレイン" 17 | }, 18 | "platform_id": "yt", 19 | "channel_id": "UCJA_J8CpKRT78Sr1bylS1vA", 20 | "details": { 21 | "twitter": "BerettaXRain" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Nero X' Fire", 27 | "jp": "ネロクロスファイア" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCdsHTs3OOKj_ANMKnmbUASw", 31 | "details": { 32 | "twitter": "NeroXFireVtuber" 33 | } 34 | }, 35 | { 36 | "name": { 37 | "en": "Lemuel Ragna X'", 38 | "jp": "レムエル ラグナクロス" 39 | }, 40 | "platform_id": "yt", 41 | "channel_id": "UC9127w5CLmWugWcpdGGuKOA", 42 | "details": { 43 | "twitter": "lemuel_x" 44 | } 45 | }, 46 | { 47 | "name": { 48 | "en": "Lamuril Ragna X'", 49 | "jp": "ラムリル ラグナクロス" 50 | }, 51 | "platform_id": "yt", 52 | "channel_id": "UC1qGKZnAnIc1wgRDOT2IOtg", 53 | "details": { 54 | "twitter": "LamurilX" 55 | } 56 | } 57 | ] -------------------------------------------------------------------------------- /.env.sample: -------------------------------------------------------------------------------- 1 | # Don't put any spaces between the equal sign! 2 | 3 | # DEBUG: Just enables the logger. 4 | # TZ: Sets the time in accordance with JST Timezone so midnight = midnight in Japan. You can set this to whatever you want. 5 | DEBUG=app,db:*,api:*channels:* 6 | TZ=Asia/Tokyo 7 | 8 | # LOG_LEVEL: Set log level. 0 = Error; 1 = Warn; 2 = Info; 3 = Verbose; 9 | LOG_LEVEL=2 10 | 11 | # This is where you put your GCP API key. Get it from [https://console.developers.google.com/apis/credentials] under 'API Keys'. 12 | GOOGLE_API_KEY= 13 | 14 | # Choose any valid port here. Defaults to 2434. 15 | PORT=2434 16 | 17 | # Set localhost as default when not using docker-compose 18 | MONGO_HOST=localhost 19 | MONGO_PORT=27017 20 | MEMCACHED_HOST=localhost 21 | MEMCACHED_PORT=11211 22 | 23 | # Set API timings here. For more info: https://www.npmjs.com/package/node-schedule 24 | TIMINGS_YOUTUBE_CHANNEL_UPDATER=*/15 * * * * 25 | TIMINGS_YOUTUBE_VIDEO_UPDATER=5 * * * * * 26 | TIMINGS_YOUTUBE_XML_CRAWLER=1 * * * * * 27 | 28 | # GQL_CACHE_INVALIDATE: Set which second you want to invalidate cached data. 29 | # Set it a few seconds after TIMINGS_YOUTUBE_VIDEO_UPDATER to prevent caching outdated data. 30 | GQL_CACHE_INVALIDATE=8 31 | 32 | # Set how long you want to store cache 33 | TTL_SHORT=20 34 | TTL_LONG=900 35 | 36 | # Apollo Server stuff. Optional. 37 | APOLLO_KEY= 38 | APOLLO_GRAPH_VARIANT= 39 | APOLLO_SCHEMA_REPORTING= 40 | -------------------------------------------------------------------------------- /src/graphql/typeDefs/ChannelObject.ts: -------------------------------------------------------------------------------- 1 | import { gql } from 'apollo-server'; 2 | 3 | export const typeDef = gql` 4 | type ChannelsResource { 5 | items: [ChannelObject]! 6 | next_page_token: String 7 | page_info: PageInfo! 8 | } 9 | type ChannelObject { 10 | _id: ID! 11 | name: Names! 12 | organization: String! 13 | platform_id: PlatformId! 14 | channel_name: String 15 | channel_id: ID! 16 | details: JSON 17 | channel_stats: ChannelStats 18 | description: String 19 | thumbnail: String 20 | } 21 | type Names { 22 | en: String! 23 | jp: String 24 | kr: String 25 | cn: String 26 | } 27 | type ChannelStats { 28 | published_at: DateTime 29 | views: Float 30 | subscribers: Float 31 | videos: Float 32 | } 33 | input SortChannelsFields { 34 | _id: Sort 35 | published_at: Sort 36 | subscribers: Sort 37 | } 38 | enum Sort { 39 | asc 40 | desc 41 | } 42 | extend type Query { 43 | channels( 44 | _id: [ID] 45 | name: String 46 | organizations: [String] 47 | exclude_organizations: [String] 48 | platforms: [PlatformId] 49 | exclude_channel_id: [ID] 50 | channel_id: [ID] 51 | order_by: SortChannelsFields = { _id: asc } 52 | page_token: String 53 | limit: Int = 25 54 | ): ChannelsResource! 55 | @rateLimit(window: "1s", max: 10, message: "You are doing that too often.") 56 | } 57 | `; 58 | -------------------------------------------------------------------------------- /channels/template.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "", 5 | "jp": "", 6 | "kr": "", 7 | "cn": "" 8 | }, 9 | "platform_id": "", 10 | "channel_id": "", 11 | "details": { 12 | "": "" 13 | } 14 | }, 15 | { 16 | "name": { 17 | "en": "Only the 'en' key is required for the name.", 18 | "jp": "If the following are not applicable", 19 | "kr": "it's safe to leave it out.", 20 | "cn": "" 21 | }, 22 | "platform_id": "yt = youtube channel, bb = bilibili, tt = twitchtv", 23 | "channel_id": "UNIQUE. Channel identifier. Youtube has a 24 long string, Bilibili has numbers, Twitch uses usernames.", 24 | "details": { 25 | "twitter": "Not required, but would be nice to be included in the defaults if you want to submit a pull request.", 26 | "key1": "Any key-value pair here is dynamic, so if you want to save custom", 27 | "key2": "static data put it here. No logic is done here.", 28 | "key3": "If you want to submit a pull request please keep the keys as", 29 | "key4": "consistent as possible." 30 | } 31 | }, 32 | { 33 | "name": { 34 | "en": "Hoshimachi Suisei", 35 | "jp": "星街すいせい", 36 | "cn": "星街彗星" 37 | }, 38 | "platform_id": "yt", 39 | "channel_id": "UC5CwaMl1eIgY8h02uZw7u8A", 40 | "details": { 41 | "twitter": "suisei_hosimati", 42 | "branch": "Hololive JP", 43 | "group": "Gen 0" 44 | } 45 | } 46 | ] -------------------------------------------------------------------------------- /channels/default/Eilene Family.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Yomemi", 5 | "jp": "ヨメミ" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UCy5lOmEQoivK5XK7QCaRKug", 9 | "details": { 10 | "twitter": "APP_Yomemi" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Yomemi", 16 | "jp": "ヨエミ" 17 | }, 18 | "platform_id": "bb", 19 | "channel_id": "292044559", 20 | "details": { 21 | "twitter": "APP_Yomemi" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Natsumi Moe", 27 | "jp": "夏実萌恵" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCBePKUYNhoMcjBi-BRmjarQ", 31 | "details": { 32 | "twitter": "Vtuber_Moe" 33 | } 34 | }, 35 | { 36 | "name": { 37 | "en": "Natsumi Moe", 38 | "jp": "夏実萌恵" 39 | }, 40 | "platform_id": "tt", 41 | "channel_id": "natsumi_moe", 42 | "details": { 43 | "twitter": "Vtuber_Moe" 44 | } 45 | }, 46 | { 47 | "name": { 48 | "en": "Étra", 49 | "jp": "エトラ" 50 | }, 51 | "platform_id": "yt", 52 | "channel_id": "UCAHXqn4nAd2j3LRu1Qyi_JA", 53 | "details": { 54 | "twitter": "etra_ASI" 55 | } 56 | }, 57 | { 58 | "name": { 59 | "en": "Moemi", 60 | "jp": "嫁ノ萌実" 61 | }, 62 | "platform_id": "yt", 63 | "channel_id": "UCIwHOJn_3QjBTwQ_gNj7WRA", 64 | "details": { 65 | "twitter": "Moemi_Yomeno" 66 | } 67 | } 68 | ] -------------------------------------------------------------------------------- /channels/default/Hanayori Joshiryo.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Hanamaru Hareru", 5 | "jp": "花丸はれる" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UCyIcOCH-VWaRKH9IkR8hz7Q", 9 | "details": { 10 | "twitter": "hanamaruhareru" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Hanamaru Hareru", 16 | "jp": "花丸はれる" 17 | }, 18 | "platform_id": "bb", 19 | "channel_id": "441381282", 20 | "details": { 21 | "twitter": "hanamaruhareru" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Kano", 27 | "jp": "鹿乃" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCfuz6xYbYFGsWWBi3SpJI1w", 31 | "details": { 32 | "twitter": "kano_hanayori" 33 | } 34 | }, 35 | { 36 | "name": { 37 | "en": "Kano", 38 | "jp": "鹿乃" 39 | }, 40 | "platform_id": "bb", 41 | "channel_id": "316381099", 42 | "details": { 43 | "twitter": "kano_hanayori" 44 | } 45 | }, 46 | { 47 | "name": { 48 | "en": "Kohigashi Hitona", 49 | "jp": "小東ひとな" 50 | }, 51 | "platform_id": "yt", 52 | "channel_id": "UCV2m2UifDGr3ebjSnDv5rUA", 53 | "details": { 54 | "twitter": "kohigashihitona" 55 | } 56 | }, 57 | { 58 | "name": { 59 | "en": "Kohigashi Hitona", 60 | "jp": "小東ひとな" 61 | }, 62 | "platform_id": "bb", 63 | "channel_id": "441382432", 64 | "details": { 65 | "twitter": "kohigashihitona" 66 | } 67 | } 68 | ] -------------------------------------------------------------------------------- /channels/default/VApArt.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "And Uge", 5 | "jp": "杏戸ゆげ" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UC3EhsuKdEkI99TWZwZgWutg", 9 | "details": { 10 | "twitter": "uge_and" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Camomi Camomi", 16 | "jp": "鴨見カモミ" 17 | }, 18 | "platform_id": "yt", 19 | "channel_id": "UCL-2thbJ7grC9fmGF4OLuTg", 20 | "details": { 21 | "twitter": "camomi_camomi" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Kanade Kanon", 27 | "jp": "花奏かのん" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCmqrvfLMws-GLGHQcB5dasg", 31 | "details": { 32 | "twitter": "_kanade_kanon" 33 | } 34 | }, 35 | { 36 | "name": { 37 | "en": "Kisaki Anko", 38 | "jp": "季咲あんこ" 39 | }, 40 | "platform_id": "yt", 41 | "channel_id": "UChXm-xAYPfygrbyLo2yCASQ", 42 | "details": { 43 | "twitter": "anko_kisaki" 44 | } 45 | }, 46 | { 47 | "name": { 48 | "en": "Komori Met", 49 | "jp": "小森めと" 50 | }, 51 | "platform_id": "yt", 52 | "channel_id": "UCzUNASdzI4PV5SlqtYwAkKQ", 53 | "details": { 54 | "twitter": "met_komori" 55 | } 56 | }, 57 | { 58 | "name": { 59 | "en": "Wat Huma", 60 | "jp": "不磨わっと" 61 | }, 62 | "platform_id": "yt", 63 | "channel_id": "UCV4EoK6BVNl7wxuxpUvvSWA", 64 | "details": { 65 | "twitter": "Wat_Huma" 66 | } 67 | } 68 | ] -------------------------------------------------------------------------------- /src/modules/cache.ts: -------------------------------------------------------------------------------- 1 | import memcached from 'memcached'; 2 | 3 | const { MEMCACHED_HOST = 'localhost', MEMCACHED_PORT = '11211' } = process.env; 4 | const URI = `${MEMCACHED_HOST}:${MEMCACHED_PORT}`; 5 | const OPTIONS = { 6 | timeout: 1000, 7 | retries: 1, 8 | namespace: 'vt' 9 | }; 10 | 11 | const cache = new memcached(URI, OPTIONS); 12 | 13 | /** 14 | * Stores a new key-value pair in memory. 15 | * @param key Key string. 16 | * @param value Value to store. 17 | * @param ttl TTL in seconds. 18 | */ 19 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 20 | export function set(key: string, value: any, ttl = 0) { 21 | return new Promise((res, rej) => { 22 | cache.set(key, value, ttl, (err, data) => { 23 | if (err) rej(err); 24 | res(data); 25 | }); 26 | }); 27 | } 28 | 29 | /** Get Memcached's status */ 30 | export function stats() { 31 | return new Promise((res, rej) => { 32 | cache.stats((err, data) => { 33 | if (err) rej(err); 34 | res(data); 35 | }); 36 | }); 37 | } 38 | 39 | /** 40 | * Retrieves one key-value pair from memory. 41 | * @param key Key to retrieve from memory. 42 | */ 43 | export function get(key: string) { 44 | return new Promise((res, rej) => { 45 | cache.get(key, (err, data) => { 46 | if (err) rej(err); 47 | res(data); 48 | }); 49 | }); 50 | } 51 | 52 | /** 53 | * Retrieves multiple key-value pairs from memory. 54 | * @param keys Keys to retrieve from memory. 55 | */ 56 | export function gets(...keys: string[]): Promise<{[key: string]: unknown;}> { 57 | return new Promise((res, rej) => { 58 | cache.getMulti(keys.flat(), (err, data) => { 59 | if (err) rej(err); 60 | res(data); 61 | }); 62 | }); 63 | } 64 | -------------------------------------------------------------------------------- /src/graphql/query/live.ts: -------------------------------------------------------------------------------- 1 | import { ApolloError, UserInputError } from 'apollo-server'; 2 | import { PlatformId } from '../../../database/types/members'; 3 | import { memcache, Videos } from '../../modules'; 4 | import { cutGroupString, getCacheKey, parseOrganization } from './consts'; 5 | 6 | interface LiveQuery { 7 | organizations: string[]; 8 | platforms: PlatformId[]; 9 | exclude_organizations: string[]; 10 | } 11 | 12 | export async function live(_, query: LiveQuery) { 13 | try { 14 | const { organizations = [], platforms = [], exclude_organizations = [] } = query; 15 | if (organizations.length && exclude_organizations.length) { 16 | return new UserInputError('Setting both organizations and exclude_organizations is redundant. Only choose one.'); 17 | } 18 | const EXCLUDE_ORG = !organizations.length; 19 | const ORGANIZATIONS = parseOrganization(EXCLUDE_ORG ? exclude_organizations : organizations); 20 | const CACHE_KEY = getCacheKey(`LIVE:${+EXCLUDE_ORG}${cutGroupString(ORGANIZATIONS)}${platforms}`); 21 | 22 | const cachedVideos = await memcache.get(CACHE_KEY); 23 | if (cachedVideos) return cachedVideos; 24 | 25 | const uncachedVideos = await Videos.find({ 26 | status: 'live', 27 | ...platforms[0] && { platform_id: { $in: platforms } }, 28 | ...ORGANIZATIONS[0] && { organization: { 29 | ...EXCLUDE_ORG 30 | ? { $not: { $regex: ORGANIZATIONS, $options: 'i' } } 31 | : { $regex: ORGANIZATIONS, $options: 'i' } 32 | } } 33 | }).sort({ 'time.start': 1 }) 34 | .lean() 35 | .exec(); 36 | 37 | memcache.set(CACHE_KEY, uncachedVideos, 60); 38 | return uncachedVideos; 39 | } catch (error) { 40 | throw new ApolloError(error); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/graphql/typeDefs/VideoObject.ts: -------------------------------------------------------------------------------- 1 | import { gql } from 'apollo-server'; 2 | 3 | export const typeDef = gql` 4 | type PageInfo { 5 | total_results: Int! 6 | results_per_page: Int! 7 | } 8 | type VideosResource { 9 | items: [VideoObject]! 10 | next_page_token: String 11 | page_info: PageInfo! 12 | } 13 | type VideoObject { 14 | _id: ID! 15 | platform_id: PlatformId! 16 | channel_id: ID! 17 | organization: String! 18 | title: String! 19 | time: Time 20 | status: VideoStatus! 21 | viewers: Float 22 | } 23 | type Time { 24 | published: DateTime 25 | scheduled: DateTime 26 | start: DateTime 27 | end: DateTime 28 | duration: Float 29 | } 30 | enum PlatformId { 31 | yt 32 | bb 33 | tt 34 | } 35 | enum VideoStatus { 36 | live 37 | upcoming 38 | ended 39 | uploaded 40 | missing 41 | } 42 | input SortVideosFields { 43 | published: Sort 44 | scheduled: Sort 45 | start: Sort 46 | duration: Sort 47 | } 48 | extend type Query { 49 | live( 50 | organizations: [String] 51 | exclude_organizations: [String] 52 | platforms: [PlatformId] 53 | ): [VideoObject]! 54 | @rateLimit(window: "1s", max: 10, message: "You are doing that too often.") 55 | videos( 56 | channel_id: [ID] 57 | status: [VideoStatus] 58 | title: String 59 | organizations: [String] 60 | exclude_organizations: [String] 61 | platforms: [PlatformId] 62 | max_upcoming_mins: Int = 0 63 | order_by: SortVideosFields = { published: desc } 64 | page_token: String 65 | limit: Int = 25 66 | ): VideosResource! 67 | @rateLimit(window: "1s", max: 10, message: "You are doing that too often.") 68 | } 69 | `; 70 | -------------------------------------------------------------------------------- /channels/default/V Dimension.Creators.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Kosumi Ranka", 5 | "jp": "小澄らんか" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UCxmbiuI6Gyyln9Ec7uZ-8wQ", 9 | "details": { 10 | "twitter": "kosumi_ranka" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Nanami Naru", 16 | "jp": "七海なる" 17 | }, 18 | "platform_id": "yt", 19 | "channel_id": "UCE0KFKU_O2FkHljyayB7kXw", 20 | "details": { 21 | "twitter": "_nanaminaru_" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Iris Ayame", 27 | "jp": "アイリス・ヴェール" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCqo0CAZ46l6ic3A_LfhEgHg", 31 | "details": { 32 | "twitter": "Iris_ayame6107" 33 | } 34 | }, 35 | { 36 | "name": { 37 | "en": "Iris Ayame", 38 | "jp": "アイリス・ヴェール" 39 | }, 40 | "platform_id": "tt", 41 | "channel_id": "irisayame6107", 42 | "details": { 43 | "twitter": "Iris_ayame6107" 44 | } 45 | }, 46 | { 47 | "name": { 48 | "en": "Oruka", 49 | "jp": "アクロー/オルカ" 50 | }, 51 | "platform_id": "yt", 52 | "channel_id": "UCheqzZj54e5y_5HEtwG7FSA", 53 | "details": { 54 | "twitter": "アクロー" 55 | } 56 | }, 57 | { 58 | "name": { 59 | "en": "Eto Etora", 60 | "jp": "絵都えとら" 61 | }, 62 | "platform_id": "yt", 63 | "channel_id": "UCwY13RnqWK4FNFeHGrK96Gw", 64 | "details": { 65 | "twitter": "eto_etora" 66 | } 67 | }, 68 | { 69 | "name": { 70 | "en": "Nanase Luna", 71 | "jp": "七星ルナ" 72 | }, 73 | "platform_id": "yt", 74 | "channel_id": "UCvA3f2VI7BDQtTc5FTqjyRg", 75 | "details": { 76 | "twitter": "nanaseruna332" 77 | } 78 | } 79 | ] -------------------------------------------------------------------------------- /src/server/apis/youtube/channel-updater.ts: -------------------------------------------------------------------------------- 1 | import { debug, Members, youtube } from '../../../modules'; 2 | import database from '../../database-managers/youtube'; 3 | import { ChannelResource, YoutubeChannelData, YoutubeChannelId } from './types'; 4 | 5 | const logger = debug('api:youtube:channel-updater'); 6 | 7 | export default async function() { 8 | const memberList = await Members 9 | .find({ platform_id: 'yt' }) 10 | .then(channelList => channelList.map(channel => channel.channel_id)); 11 | const youtubeRequests: Promise[] = []; 12 | while (memberList.length) youtubeRequests.push(fetchYoutubeChannel(memberList.splice(0, 50))); 13 | const updatedChannelData = await Promise.all(youtubeRequests); 14 | database.emit('update-channels', updatedChannelData.flat()); 15 | } 16 | 17 | async function fetchYoutubeChannel(channelIds: YoutubeChannelId[]): Promise { 18 | logger.log(`Requesting channel data from ${channelIds.length} channels from youtube...`); 19 | const results = await youtube.channels({ 20 | part: 'snippet,statistics', 21 | fields: 'items(id,snippet(title,description,thumbnails/high/url,publishedAt),statistics(subscriberCount,videoCount,viewCount))', 22 | id: channelIds.join(',') 23 | }).then(data => data.items.map(parseYoutubeChannelData)) 24 | .catch(err => { 25 | logger.error(err); 26 | return []; 27 | }); 28 | logger.log(`Got ${results.length} channels back from youtube.`); 29 | return results; 30 | } 31 | 32 | const parseYoutubeChannelData = ( 33 | { id, snippet, statistics }: ChannelResource 34 | ): YoutubeChannelData => { 35 | const { title, publishedAt, description, thumbnails } = snippet ?? {}; 36 | const { subscriberCount, videoCount, viewCount } = statistics ?? {}; 37 | return { 38 | channel_id: id, 39 | channel_name: title, 40 | channel_stats: { 41 | published_at: new Date(publishedAt), 42 | subscribers: +subscriberCount || 0, 43 | videos: +videoCount, 44 | views: +viewCount 45 | }, 46 | description, 47 | thumbnail: thumbnails.high.url 48 | }; 49 | }; 50 | -------------------------------------------------------------------------------- /channels/apps/updaters/youtube-updater.ts: -------------------------------------------------------------------------------- 1 | import { ChannelObject } from '../../../database/types/channels'; 2 | import { MemberProps } from '../../../database/types/members'; 3 | import { youtube } from '../../../src/modules'; 4 | import debug from '../../../src/modules/logger'; 5 | import { ChannelResource } from '../../../src/modules/types/youtube'; 6 | import database from '../database-manager'; 7 | 8 | const logger = debug('api:youtube'); 9 | 10 | export default async function(channelData: MemberProps[]) { 11 | const youtubeRequests: Promise[] = []; 12 | while (channelData.length) { 13 | youtubeRequests.push(fetchChannelData(channelData.splice(0, 50))); 14 | } 15 | const newVideos = (await Promise.all(youtubeRequests)).flat(); 16 | database.emit('update-channels', newVideos); 17 | } 18 | 19 | async function fetchChannelData(channels: MemberProps[]) { 20 | logger.info(`Requesting ${channels.length} youtube channel data from youtube...`); 21 | const results = await youtube.channels({ 22 | part: 'snippet,statistics', 23 | fields: 'items(id,snippet(title,publishedAt,description,thumbnails/high/url),statistics(subscriberCount,videoCount,viewCount))', 24 | id: channels.map(channel => channel.channel_id).join(','), 25 | hl: 'ja' 26 | }).then(data => data.items.map(item => { 27 | const memberData = channels.find(channel => channel.channel_id === item.id); 28 | return parseAndMergeChannelData(item, memberData); 29 | })).catch(err => { 30 | logger.error(err); 31 | return [] as ChannelObject[]; 32 | }); 33 | logger.info(`Got ${results.length} results.`); 34 | return results; 35 | } 36 | 37 | const parseAndMergeChannelData = ( 38 | { snippet, statistics }: ChannelResource, 39 | memberData: MemberProps 40 | ): ChannelObject => ({ 41 | _id: memberData._id, 42 | name: memberData.name, 43 | organization: memberData.organization, 44 | platform_id: 'yt', 45 | channel_name: snippet.title, 46 | channel_id: memberData.channel_id, 47 | channel_stats: { 48 | published_at: new Date(snippet.publishedAt), 49 | subscribers: +statistics.subscriberCount || 0, 50 | videos: +statistics.videoCount, 51 | views: +statistics.viewCount 52 | }, 53 | details: memberData.details, 54 | description: snippet.description, 55 | thumbnail: snippet.thumbnails.high.url 56 | }); 57 | -------------------------------------------------------------------------------- /channels/apps/database-manager.ts: -------------------------------------------------------------------------------- 1 | import { EventEmitter } from 'events'; 2 | import * as db from '../../database'; 3 | import { ChannelObject } from '../../database/types/channels'; 4 | import { MemberObject } from '../../database/types/members'; 5 | import { VideoObject } from '../../database/types/videos'; 6 | import debug from '../../src/modules/logger'; 7 | 8 | const logger = debug('channels:database-manager'); 9 | 10 | interface DatabaseEvents { 11 | 'save-videos': (newVideos: VideoObject[]) => void; 12 | 'update-channels': (channels: ChannelObject[]) => void; 13 | 'update-member': (channels: MemberObject) => void; 14 | } 15 | 16 | declare interface DatabaseManager { 17 | on( 18 | event: U, listener: DatabaseEvents[U] 19 | ): this; 20 | emit( 21 | event: U, ...args: Parameters 22 | ): boolean; 23 | } 24 | 25 | class DatabaseManager extends EventEmitter { constructor() { super(); } } 26 | const database = new DatabaseManager(); 27 | export default database; 28 | 29 | database.on('save-videos', async newVideos => { 30 | logger.log(`Saving ${newVideos.length} videos...`); 31 | const result = await db.Videos.create(newVideos).catch(logger.error); 32 | if (result) logger.log(`Finished saving ${result.length} videos.`); 33 | }); 34 | 35 | database.on('update-channels', async channelData => { 36 | logger.log(`Updating ${channelData.length} channels...`); 37 | const results = await Promise.all(channelData 38 | .map(channel => db.Channels.updateOne( 39 | { _id: channel._id }, 40 | { $set: channel }, 41 | { upsert: true } 42 | )) 43 | ).then(writeResults => writeResults.reduce( 44 | (total, result) => total + (result.upserted?.length ?? result.nModified), 0) 45 | ).catch(logger.error); 46 | if (typeof results === 'number') logger.log(`Updated ${results} channels.`); 47 | }); 48 | 49 | database.on('update-member', channelData => { 50 | const { channel_id } = channelData; 51 | logger.log(`Updating member data for ${channel_id}...`); 52 | db.Members.updateOne( 53 | { channel_id }, 54 | { $set: channelData } 55 | ).then(result => logger.log(result.nModified 56 | ? `Updated member: ${channel_id}.` 57 | : `No new data for ${channel_id}.` 58 | )).catch(err => logger.error(err, `channel_id: ${channel_id}`)); 59 | }); 60 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "vt-api", 3 | "version": "1.6.6", 4 | "description": "Simple GraphQL API for getting VTuber livefeeds, channels, and videos.", 5 | "scripts": { 6 | "postinstall": "npm run build", 7 | "build": "tsc", 8 | "watch": "tsc -w", 9 | "dev": "concurrently -r npm:watch npm:dev-*", 10 | "dev-server": "NODE_ENV=development nodemon dist/src/server", 11 | "dev-graphql": "NODE_ENV=development nodemon dist/src/graphql", 12 | "start": "concurrently -r npm:start-*", 13 | "start-server": "NODE_ENV=production node dist/src/server", 14 | "start-graphql": "NODE_ENV=production node dist/src/graphql", 15 | "copy-default": "cp -rT channels/default/ channels/organizations/", 16 | "channel-manager": "node -e \"require('./dist/channels').channelManager();\"", 17 | "init": "node -e \"require('./dist/channels').init();\"" 18 | }, 19 | "keywords": [], 20 | "author": "", 21 | "license": "WTFPL", 22 | "dependencies": { 23 | "apollo-server": "^2.21.0", 24 | "debug": "^4.3.1", 25 | "dotenv": "^8.2.0", 26 | "express": "^4.17.1", 27 | "graphql-iso-date": "^3.6.1", 28 | "graphql-rate-limit": "^2.0.1", 29 | "graphql-type-json": "^0.3.2", 30 | "memcached": "^2.2.2", 31 | "mongoose": "^5.12.4", 32 | "node-fetch": "^2.6.1", 33 | "node-schedule": "^1.3.3", 34 | "xml2js": "^0.4.23" 35 | }, 36 | "devDependencies": { 37 | "@types/debug": "^4.1.5", 38 | "@types/graphql-iso-date": "^3.4.0", 39 | "@types/memcached": "^2.2.6", 40 | "@types/node": "^14.14.28", 41 | "@types/node-fetch": "^2.5.8", 42 | "@types/node-schedule": "^1.3.1", 43 | "@types/xml2js": "^0.4.8", 44 | "@typescript-eslint/eslint-plugin": "^4.15.0", 45 | "@typescript-eslint/parser": "^4.15.0", 46 | "concurrently": "^5.3.0", 47 | "eslint": "^7.20.0", 48 | "nodemon": "^2.0.7", 49 | "typescript": "^4.1.5" 50 | }, 51 | "nodemonConfig": { 52 | "ignore": [ 53 | "channels/default", 54 | "channels/organizations" 55 | ], 56 | "events": { 57 | "start": "clear", 58 | "restart": "clear" 59 | } 60 | }, 61 | "engines": { 62 | "node": ">=12.0.0" 63 | }, 64 | "repository": { 65 | "type": "git", 66 | "url": "git+https://github.com/Choooks22/vt-api.git" 67 | }, 68 | "bugs": { 69 | "url": "https://github.com/Choooks22/vt-api/issues" 70 | }, 71 | "homepage": "https://github.com/Choooks22/vt-api#readme" 72 | } 73 | -------------------------------------------------------------------------------- /channels/default/Idol-bu.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Mokota Mememe", 5 | "jp": "もこ田めめめ" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UCz6Gi81kE6p5cdW1rT0ixqw", 9 | "details": { 10 | "twitter": "mokomeme_ch" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Kitakami Futaba", 16 | "jp": "北上双葉" 17 | }, 18 | "platform_id": "yt", 19 | "channel_id": "UC5nfcGkOAm3JwfPvJvzplHg", 20 | "details": { 21 | "twitter": "KitakamiFutaba" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Kongou Iroha", 27 | "jp": "金剛いろは" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCiGcHHHT3kBB1IGOrv7f3qQ", 31 | "details": { 32 | "twitter": "KongoIroha" 33 | } 34 | }, 35 | { 36 | "name": { 37 | "en": "Yaezawa Natori", 38 | "jp": "八重沢なとり" 39 | }, 40 | "platform_id": "yt", 41 | "channel_id": "UC1519-d1jzGiL1MPTxEdtSA", 42 | "details": { 43 | "twitter": "YaezawaNatori" 44 | } 45 | }, 46 | { 47 | "name": { 48 | "en": "Kakyouin Chieri", 49 | "jp": "花京院ちえり" 50 | }, 51 | "platform_id": "yt", 52 | "channel_id": "UCP9ZgeIJ3Ri9En69R0kJc9Q", 53 | "details": { 54 | "twitter": "chieri_kakyoin" 55 | } 56 | }, 57 | { 58 | "name": { 59 | "en": "Kagura Suzu", 60 | "jp": "神楽すず" 61 | }, 62 | "platform_id": "yt", 63 | "channel_id": "UCUZ5AlC3rTlM-rA2cj5RP6w", 64 | "details": { 65 | "twitter": "kagura_suzu" 66 | } 67 | }, 68 | { 69 | "name": { 70 | "en": "Yamato Iori", 71 | "jp": "ヤマトイオリ" 72 | }, 73 | "platform_id": "yt", 74 | "channel_id": "UCyb-cllCkMREr9de-hoiDrg", 75 | "details": { 76 | "twitter": "YamatoIori" 77 | } 78 | }, 79 | { 80 | "name": { 81 | "en": "Carro Pino", 82 | "jp": "カルロピノ" 83 | }, 84 | "platform_id": "yt", 85 | "channel_id": "UCMzxQ58QL4NNbWghGymtHvw", 86 | "details": { 87 | "twitter": "carro_pino" 88 | } 89 | }, 90 | { 91 | "name": { 92 | "en": "Kiso Azuki", 93 | "jp": "木曽あずき" 94 | }, 95 | "platform_id": "yt", 96 | "channel_id": "UCmM5LprTu6-mSlIiRNkiXYg", 97 | "details": { 98 | "twitter": "KisoAzuki" 99 | } 100 | } 101 | ] -------------------------------------------------------------------------------- /src/server/database-managers/youtube.ts: -------------------------------------------------------------------------------- 1 | import { EventEmitter } from 'events'; 2 | import { YoutubeChannelData, YoutubeVideoObject } from '../apis/youtube/types'; 3 | import { debug } from '../../modules'; 4 | import * as db from '../../../database'; 5 | 6 | const logger = debug('api:youtube:database-manager'); 7 | 8 | interface DatabaseEvents { 9 | 'save-videos': (newVideos: YoutubeVideoObject[]) => void; 10 | 'update-videos': (videos: YoutubeVideoObject[]) => void; 11 | 'update-channels': (channels: YoutubeChannelData[]) => void; 12 | } 13 | 14 | declare interface DatabaseManager { 15 | on( 16 | event: U, listener: DatabaseEvents[U] 17 | ): this; 18 | emit( 19 | event: U, ...args: Parameters 20 | ): boolean; 21 | } 22 | 23 | class DatabaseManager extends EventEmitter { constructor() { super(); } } 24 | const database = new DatabaseManager(); 25 | export default database; 26 | 27 | database.on('save-videos', async newVideos => { 28 | logger.log(`Saving ${newVideos.length} videos...`); 29 | const results = await Promise.all(newVideos 30 | .map(video => db.Videos.updateOne( 31 | { _id: video._id }, 32 | { $setOnInsert: video }, 33 | { upsert: true } 34 | )) 35 | ).then(writeResults => writeResults.reduce( 36 | (total, result) => total + (result.upserted?.length ?? 0), 0) 37 | ).catch(logger.error); 38 | if (typeof results === 'number') logger.log(`Finished saving ${results} videos.`); 39 | }); 40 | 41 | database.on('update-videos', async videos => { 42 | logger.log(`Updating ${videos.length} videos...`); 43 | const results = await Promise.all(videos 44 | .map(video => db.Videos.updateOne( 45 | { _id: video._id }, 46 | { $set: video } 47 | )) 48 | ).then(writeResults => writeResults.reduce( 49 | (total, result) => total + result.nModified, 0) 50 | ).catch(logger.error); 51 | if (results) logger.log(`Updated ${results} videos.`); 52 | }); 53 | 54 | database.on('update-channels', async channels => { 55 | logger.log(`Updating ${channels.length} youtube channels...`); 56 | const results = await Promise.all(channels 57 | .map(channel => db.Channels.updateOne( 58 | { channel_id: channel.channel_id }, 59 | { $set: channel } 60 | )) 61 | ).then(writeResults => writeResults.reduce( 62 | (total, result) => total + result.nModified, 0) 63 | ).catch(logger.error); 64 | if (typeof results === 'number') logger.log(`Updated ${results} channels.`); 65 | }); 66 | -------------------------------------------------------------------------------- /src/server/apis/youtube/types.ts: -------------------------------------------------------------------------------- 1 | import { MemberNames, TwitterHandle, YoutubeChannelId } from '../../../../database/types/members'; 2 | import { VideoObject } from '../../../../database/types/videos'; 3 | 4 | export { ChannelProps } from '../../../../database/types/channels'; 5 | export { YoutubeChannelId } from '../../../../database/types/members'; 6 | export { 7 | ChannelResource, 8 | PlaylistItemsResource, 9 | VideoResource, 10 | YoutubeChannelResponse, 11 | YoutubePlaylistItemsResponse, 12 | YoutubeVideoResponse 13 | } from '../../../modules/types/youtube'; 14 | 15 | export type VideoId = string; 16 | export type DateString = string; 17 | 18 | // #region Youtube Video 19 | export type VideoStatus = 'live'|'upcoming'|'ended'|'uploaded'|'missing'|'new'; 20 | export interface YoutubeVideoObject extends VideoObject { 21 | _id: VideoId; 22 | platform_id: 'yt'; 23 | channel_id: YoutubeChannelId; 24 | organization?: string; 25 | crawled_at?: Date; 26 | } 27 | // #endregion Youtube Video 28 | // #region Youtube Channel 29 | export interface YoutubeChannelData { 30 | channel_name: string; 31 | channel_id: YoutubeChannelId; 32 | channel_stats: { 33 | published_at: Date; 34 | views: number; 35 | subscribers: number; 36 | videos: number; 37 | }; 38 | description: string; 39 | thumbnail: string; 40 | } 41 | export interface BlankYoutubeChannel { 42 | name: MemberNames; 43 | organization: string; 44 | platform_id: 'yt'; 45 | channel_id: YoutubeChannelId; 46 | details: { 47 | twitter: TwitterHandle; 48 | [key: string]: unknown; 49 | }; 50 | } 51 | // #endregion Youtube Channel 52 | // #region Youtube Xml 53 | export interface YoutubeXmlResponse { 54 | feed: { 55 | $: { 56 | 'xmlns:yt': string; 57 | 'xmlns:media': string; 58 | 'xmlns': string; 59 | }; 60 | link: { 61 | $: { 62 | rel: string; 63 | href: string; 64 | }; 65 | }[]; 66 | id: string; 67 | 'yt:channelId': YoutubeChannelId; 68 | title: string; 69 | author: { 70 | name: string; 71 | uri: string; 72 | }; 73 | published: DateString; 74 | entry: VideoXmlEntry[]; 75 | }; 76 | } 77 | 78 | export interface VideoXmlEntry { 79 | id: string; 80 | 'yt:videoId': VideoId; 81 | 'yt:channelId': YoutubeChannelId; 82 | title: string; 83 | link: { $: Record; }; 84 | author: { 85 | name: string; 86 | uri: string; 87 | }; 88 | published: DateString; 89 | updated: DateString; 90 | 'media:group': { 91 | 'media:title': string; 92 | 'media:content': Record; 93 | 'media:thumbnail': Record; 94 | 'media:description': string; 95 | 'media:community': Record; 96 | }; 97 | } 98 | // #endregion Youtube Xml 99 | -------------------------------------------------------------------------------- /channels/default/Tsunderia.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Kirihime Ria", 5 | "jp": "切姫りあ" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UC_FqH-6GpFosu_P0ApjqW5A", 9 | "details": { 10 | "twitter": "kirihimeria" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Yazaki Kallin", 16 | "jp": "夜咲カリン" 17 | }, 18 | "platform_id": "yt", 19 | "channel_id": "UCbTkauz77OIO9KE5YcAKPlg", 20 | "details": { 21 | "twitter": "yazakikallin" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Matsuro Meru", 27 | "jp": "末路める" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCIJ6TGsTcBlYGUj-zbL60EQ", 31 | "details": { 32 | "twitter": "matsuromeru" 33 | } 34 | }, 35 | { 36 | "name": { 37 | "en": "Inukai Purin", 38 | "jp": "犬養プリン" 39 | }, 40 | "platform_id": "yt", 41 | "channel_id": "UCdrQWcX7XLDSUEh9SAxnyBg", 42 | "details": { 43 | "twitter": "inukaipurin" 44 | } 45 | }, 46 | { 47 | "name": { 48 | "en": "Hoshino Char", 49 | "jp": "星乃シャロ" 50 | }, 51 | "platform_id": "yt", 52 | "channel_id": "UCdCczCYPOeP5hTfbx6tzz_A", 53 | "details": { 54 | "twitter": "hoshinochar" 55 | } 56 | }, 57 | { 58 | "name": { 59 | "en": "Kamiko Kana", 60 | "jp": "神狐かな" 61 | }, 62 | "platform_id": "yt", 63 | "channel_id": "UCn1coImC6NMDYCQfH_ILzXw", 64 | "details": { 65 | "twitter": "kamikokana" 66 | } 67 | }, 68 | { 69 | "name": { 70 | "en": "Nini Yuuna", 71 | "jp": "ににゆーな" 72 | }, 73 | "platform_id": "yt", 74 | "channel_id": "UC8CKGSvp9bRfmtWyfV5L9Jw", 75 | "details": { 76 | "twitter": "niniyuuna" 77 | } 78 | }, 79 | { 80 | "name": { 81 | "en": "Umiushi Urara", 82 | "jp": "海牛 うらら " 83 | }, 84 | "platform_id": "yt", 85 | "channel_id": "UCGNxKTqNK3GMjgvgUixJroQ", 86 | "details": { 87 | "twitter": "umiushiurara" 88 | } 89 | }, 90 | { 91 | "name": { 92 | "en": "Amemachi Hanabi", 93 | "jp": "雨街はなび" 94 | }, 95 | "platform_id": "yt", 96 | "channel_id": "UC8yXRB_jKDeapwQak2i30EA", 97 | "details": { 98 | "twitter": "amemachihanabi" 99 | } 100 | }, 101 | { 102 | "name": { 103 | "en": "Orla Gan Ceann" 104 | }, 105 | "platform_id": "yt", 106 | "channel_id": "UCxN7pUZzXmMOuost2oRCHUw", 107 | "details": { 108 | "twitter": "orlaganceann" 109 | } 110 | }, 111 | { 112 | "name": { 113 | "en": "Orla Gan Ceann" 114 | }, 115 | "platform_id": "tt", 116 | "channel_id": "orlaganceann", 117 | "details": { 118 | "twitter": "orlaganceann" 119 | } 120 | } 121 | ] -------------------------------------------------------------------------------- /src/graphql/query/data.ts: -------------------------------------------------------------------------------- 1 | import { ApolloError, UserInputError } from 'apollo-server'; 2 | import { Channels, Videos } from '../../../database'; 3 | import { PlatformId } from '../../../database/types/members'; 4 | import { memcache } from '../../modules'; 5 | import { ChannelId } from '../../modules/types/youtube'; 6 | import { cutChannelIds, cutGroupString, getCacheKey, parseOrganization } from './consts'; 7 | 8 | const CACHE_TTL = +(process.env.TTL_LONG ?? 900); 9 | 10 | interface Query { 11 | organizations: string[]; 12 | exclude_organizations: string[]; 13 | channel_id: ChannelId[]; 14 | exclude_channel_id: ChannelId[]; 15 | platforms: PlatformId[]; 16 | } 17 | 18 | export async function data(_, query: Query) { 19 | try { 20 | const { 21 | organizations = [], 22 | exclude_organizations = [], 23 | channel_id = [], 24 | exclude_channel_id = [], 25 | platforms = [] 26 | } = query; 27 | if (organizations.length && exclude_organizations.length) { 28 | return new UserInputError('Setting both organizations and exclude_organizations is redundant. Only choose one.'); 29 | } 30 | if (channel_id.length && exclude_channel_id.length) { 31 | return new UserInputError('Setting both channel_id and exclude_channel_id is redundant. Only choose one.'); 32 | } 33 | const EXCLUDE_ORG = !organizations.length; 34 | const EXCLUDE_IDS = !channel_id.length; 35 | const ORGANIZATIONS = parseOrganization(EXCLUDE_ORG ? exclude_organizations : organizations); 36 | const CHANNEL_IDS = EXCLUDE_IDS ? exclude_channel_id : channel_id; 37 | const CACHE_KEY = getCacheKey(`CHNLS:${+EXCLUDE_ORG}${cutGroupString(ORGANIZATIONS)}${cutChannelIds(CHANNEL_IDS)}${platforms}`, false); 38 | 39 | const cached = await memcache.get(CACHE_KEY); 40 | if (cached) return cached; 41 | 42 | const QUERY = { 43 | ...ORGANIZATIONS[0] && { organization: { 44 | ...EXCLUDE_ORG 45 | ? { $not: { $regex: ORGANIZATIONS, $options: 'i' } } 46 | : { $regex: ORGANIZATIONS, $options: 'i' } 47 | } }, 48 | ...channel_id[0] && { channel_id: { [EXCLUDE_IDS ? '$nin' : '$in']: CHANNEL_IDS } }, 49 | ...platforms[0] && { platform_id: { $in: platforms } } 50 | }; 51 | 52 | const getChannelList = Channels.find(QUERY).lean().exec(); 53 | const getVideoCount = Videos.countDocuments(QUERY).lean().exec(); 54 | const [channelList, videoCount] = await Promise.all([getChannelList, getVideoCount]); 55 | const groupList = channelList 56 | .map(value => value.organization) 57 | .filter((value, index, array) => array.indexOf(value) === index); 58 | const channelCount = channelList.length; 59 | 60 | const result = { 61 | organizations: groupList, 62 | channels: channelCount, 63 | videos: videoCount 64 | }; 65 | 66 | memcache.set(CACHE_KEY, result, CACHE_TTL); 67 | return result; 68 | } catch(err) { 69 | throw new ApolloError(err); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/server/apis/youtube/xml-crawler.ts: -------------------------------------------------------------------------------- 1 | import fetch from 'node-fetch'; 2 | import schedule from 'node-schedule'; 3 | import { parseStringPromise } from 'xml2js'; 4 | import { Channels, debug, memcache } from '../../../modules'; 5 | import { ChannelId } from '../../../modules/types/youtube'; 6 | import database from '../../database-managers/youtube'; 7 | import { VideoXmlEntry, YoutubeVideoObject } from './types'; 8 | 9 | const logger = debug('api:youtube:xml-crawler'); 10 | const CACHE_TTL = +process.env.TTL_LONG || 900; 11 | 12 | export default init; 13 | export async function init(timer = '1 * * * * *') { 14 | const channelList = await Channels.find({ platform_id: 'yt' }); 15 | channelList.forEach(channel => { 16 | const { channel_id, organization } = channel; 17 | const xmlScraper = new XmlScraper(channel_id, organization); 18 | schedule.scheduleJob(`xml-crawler:${channel_id}`, timer, crawler.bind(xmlScraper)); 19 | }); 20 | logger.info(`Now scraping ${channelList.length} youtube channel xmls.`); 21 | } 22 | 23 | class XmlScraper { 24 | private rawXmlData = null; 25 | private xmlOptions = { explicitArray: false }; 26 | constructor(private channel_id: ChannelId, private organization: string) {} 27 | private get xmlLink() { return `https://www.youtube.com/feeds/videos.xml?channel_id=${this.channel_id}&t=${Date.now()}`; } 28 | get channelId() { return this.channel_id; } 29 | get cacheId() { return `yt-${this.channel_id}`; } 30 | async fetchXml(): Promise { 31 | this.rawXmlData = await fetch(this.xmlLink).then(res => res.text()); 32 | return this.parseXml(); 33 | } 34 | private async parseXml(): Promise { 35 | const parsedString = await parseStringPromise(this.rawXmlData, this.xmlOptions) 36 | .catch(() => logger.warn(`Channel ${this.channelId} doesn\'t exist.`)); 37 | if (!parsedString || !parsedString.feed.entry?.map) return; 38 | return parsedString.feed.entry.map(this.parseEntries.bind(this)).sort(this.videoSorter); 39 | } 40 | private videoSorter(video1: YoutubeVideoObject, video2: YoutubeVideoObject) { 41 | return +video2.crawled_at - +video1.crawled_at; 42 | } 43 | private parseEntries(entry: VideoXmlEntry): YoutubeVideoObject { 44 | return { 45 | _id: entry['yt:videoId'], 46 | platform_id: 'yt', 47 | channel_id: entry['yt:channelId'], 48 | organization: this.organization, 49 | title: entry.title, 50 | status: 'new', 51 | crawled_at: new Date(entry.published) 52 | }; 53 | } 54 | } 55 | 56 | async function crawler(this: XmlScraper) { 57 | logger.log(`Crawling ${this.channelId}...`); 58 | const latestTimestamp = (await memcache.get(this.cacheId)) ?? 0; 59 | const videoList = await this.fetchXml(); 60 | if (!videoList) return logger.warn(`${this.channelId} didn\'t return anything?`); 61 | const newVideos = videoList.filter(video => video.crawled_at > latestTimestamp); 62 | if (!newVideos.length) return logger.log(`${this.channelId} doesn\'t have new videos.`); 63 | logger.info(`Found ${newVideos.length} new videos from ${this.channelId}`); 64 | memcache.set(this.cacheId, newVideos[0].crawled_at, CACHE_TTL); 65 | database.emit('save-videos', newVideos); 66 | } 67 | -------------------------------------------------------------------------------- /src/server/apis/youtube/video-updater.ts: -------------------------------------------------------------------------------- 1 | import { debug, Videos, youtube } from '../../../modules'; 2 | import type { LiveStreamingDetails, VideoResource } from '../../../modules/types/youtube'; 3 | import database from '../../database-managers/youtube'; 4 | import type { VideoId, YoutubeVideoObject } from './types'; 5 | 6 | const ONE_HOUR = 36E5; 7 | const logger = debug('api:youtube:video-updater'); 8 | const db = debug('api:youtube:mongoose'); 9 | 10 | export default async function() { 11 | db.log('Looking for videos to update...'); 12 | const videosToUpdate = await fetchVideosToUpdate(); 13 | if (!videosToUpdate.length) return db.log('No videos to update.'); 14 | db.info(`Found ${videosToUpdate.length} videos to update.`); 15 | logger.log(`Updating ${videosToUpdate.length} videos...`); 16 | const updatedVideos = await fetchYoutubeVideoData(videosToUpdate); 17 | logger.info(`Updated ${updatedVideos.length} videos.`); 18 | database.emit('update-videos', updatedVideos); 19 | } 20 | 21 | const fetchVideosToUpdate = () => Videos 22 | .find({ $or: [ 23 | { status: { $in: ['new', 'live'] } }, 24 | { status: 'upcoming', 'time.scheduled': { $lte: Date.now() + ONE_HOUR } } 25 | ] }) 26 | .sort({ updated_at: 1 }) 27 | .limit(50) 28 | .then(res => res.map(doc => doc._id)); 29 | 30 | async function fetchYoutubeVideoData(ids: VideoId[]) { 31 | logger.log(`Fetching ${ids.length} videos from Youtube...`); 32 | const result = await youtube.videos({ 33 | part: 'snippet,liveStreamingDetails', 34 | fields: 'items(id,snippet,liveStreamingDetails)', 35 | id: ids.join(','), 36 | hl: 'ja' 37 | }).then(res => res.items.map(parseVideo)); 38 | logger.log(`Fetched ${result?.length ?? 0} videos. Status: ${result ? 'OK' : 'ERROR'}`); 39 | if (result.length !== ids.length) result.push(...parseMissingVideos(ids, result) as YoutubeVideoObject[]); 40 | return result; 41 | } 42 | 43 | function parseVideo( 44 | { id, snippet, liveStreamingDetails }: VideoResource 45 | ): YoutubeVideoObject { 46 | const { channelId, title, publishedAt } = snippet ?? {}; 47 | const { scheduledStartTime, actualStartTime, actualEndTime, concurrentViewers } = liveStreamingDetails ?? {}; 48 | return { 49 | _id: id, 50 | platform_id: 'yt', 51 | channel_id: channelId, 52 | title, 53 | time: { 54 | published: new Date(publishedAt), 55 | scheduled: scheduledStartTime ? new Date(scheduledStartTime) : undefined, 56 | start: actualStartTime ? new Date(actualStartTime) : undefined, 57 | end: actualEndTime ? new Date(actualEndTime) : undefined 58 | }, 59 | status: getVideoStatus(liveStreamingDetails), 60 | viewers: +concurrentViewers || null 61 | }; 62 | } 63 | 64 | function parseMissingVideos(idList: VideoId[], videoList: YoutubeVideoObject[]) { 65 | const missingIds = idList.filter(_id => !videoList.find(video => video._id === _id)); 66 | return missingIds.map(_id => ({ _id, status: 'missing' })); 67 | } 68 | 69 | export function getVideoStatus(liveStreamingDetails: LiveStreamingDetails) { 70 | if (!liveStreamingDetails) return 'uploaded'; 71 | const { actualEndTime, actualStartTime } = liveStreamingDetails; 72 | /* eslint-disable indent,no-multi-spaces */ 73 | return actualEndTime ? 'ended' 74 | : actualStartTime ? 'live' 75 | : 'upcoming'; 76 | /* eslint-enable indent,no-multi-spaces */ 77 | } 78 | -------------------------------------------------------------------------------- /channels/default/VShojo.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Nyatasha Nyanners" 5 | }, 6 | "platform_id": "yt", 7 | "channel_id": "UCO5Jvsc_sKuZi3MhnJxrlzQ", 8 | "details": { 9 | "twitter": "NyanNyanners" 10 | } 11 | }, 12 | { 13 | "name": { 14 | "en": "Nyatasha Nyanners" 15 | }, 16 | "platform_id": "tt", 17 | "channel_id": "nyanners", 18 | "details": { 19 | "twitter": "NyanNyanners" 20 | } 21 | }, 22 | { 23 | "name": { 24 | "en": "Projekt Melody", 25 | "jp": "メロディー" 26 | }, 27 | "platform_id": "yt", 28 | "channel_id": "UC1yoRdFoFJaCY-AGfD9W0wQ", 29 | "details": { 30 | "twitter": "projektmelody" 31 | } 32 | }, 33 | { 34 | "name": { 35 | "en": "Projekt Melody", 36 | "jp": "メロディー" 37 | }, 38 | "platform_id": "tt", 39 | "channel_id": "projektmelody", 40 | "details": { 41 | "twitter": "projektmelody" 42 | } 43 | }, 44 | { 45 | "name": { 46 | "en": "Froot" 47 | }, 48 | "platform_id": "yt", 49 | "channel_id": "UCtNeQ8cUwvAAhBbVbwfGWpg", 50 | "details": { 51 | "twitter": "LichVtuber" 52 | } 53 | }, 54 | { 55 | "name": { 56 | "en": "Froot" 57 | }, 58 | "platform_id": "tt", 59 | "channel_id": "bsapricot", 60 | "details": { 61 | "twitter": "LichVtuber" 62 | } 63 | }, 64 | { 65 | "name": { 66 | "en": "Zentraya" 67 | }, 68 | "platform_id": "yt", 69 | "channel_id": "UCVLsPBwDtv7n7FZLiOsvhSQ", 70 | "details": { 71 | "twitter": "zentreya" 72 | } 73 | }, 74 | { 75 | "name": { 76 | "en": "Zentraya" 77 | }, 78 | "platform_id": "tt", 79 | "channel_id": "zentreya", 80 | "details": { 81 | "twitter": "zentreya" 82 | } 83 | }, 84 | { 85 | "name": { 86 | "en": "Silvervale" 87 | }, 88 | "platform_id": "yt", 89 | "channel_id": "UCm8Dj7dQ0oRHXNUXF31kjEw", 90 | "details": { 91 | "twitter": "_silvervale_" 92 | } 93 | }, 94 | { 95 | "name": { 96 | "en": "Silvervale" 97 | }, 98 | "platform_id": "tt", 99 | "channel_id": "silvervale", 100 | "details": { 101 | "twitter": "_silvervale_" 102 | } 103 | }, 104 | { 105 | "name": { 106 | "en": "Ironmouse" 107 | }, 108 | "platform_id": "yt", 109 | "channel_id": "UChgPVLjqugDQpRLWvC7zzig", 110 | "details": { 111 | "twitter": "_silvervale_" 112 | } 113 | }, 114 | { 115 | "name": { 116 | "en": "Ironmouse" 117 | }, 118 | "platform_id": "tt", 119 | "channel_id": "ironmouse", 120 | "details": { 121 | "twitter": "_silvervale_" 122 | } 123 | }, 124 | { 125 | "name": { 126 | "en": "Hime Hajime" 127 | }, 128 | "platform_id": "yt", 129 | "channel_id": "UCRfz2yOn8C7Jg5zq4Xi67-Q", 130 | "details": { 131 | "twitter": "HimeHajime_VSJ" 132 | } 133 | }, 134 | { 135 | "name": { 136 | "en": "Hime Hajime" 137 | }, 138 | "platform_id": "tt", 139 | "channel_id": "hajime", 140 | "details": { 141 | "twitter": "HimeHajime_VSJ" 142 | } 143 | } 144 | ] -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "root": true, 3 | "extends": [ 4 | "plugin:@typescript-eslint/recommended" 5 | ], 6 | "env": { 7 | "commonjs": true, 8 | "node": true, 9 | "es6": true 10 | }, 11 | "parser": "@typescript-eslint/parser", 12 | "plugins": [ 13 | "@typescript-eslint" 14 | ], 15 | "parserOptions": { 16 | "ecmaVersion": 2020 17 | }, 18 | "rules": { 19 | "@typescript-eslint/no-explicit-any": "off", 20 | "@typescript-eslint/member-delimiter-style": [ 21 | "error", 22 | { 23 | "singleline": { 24 | "delimiter": "semi", 25 | "requireLast": true 26 | }, 27 | "multiline": { 28 | "delimiter": "semi", 29 | "requireLast": true 30 | } 31 | } 32 | ], 33 | "@typescript-eslint/explicit-module-boundary-types": "off", 34 | "brace-style": [ 35 | "error", 36 | "1tbs", 37 | { 38 | "allowSingleLine": true 39 | } 40 | ], 41 | "comma-spacing": "error", 42 | "comma-style": "error", 43 | "curly": [ 44 | "error", 45 | "multi-line", 46 | "consistent" 47 | ], 48 | "dot-location": [ 49 | "error", 50 | "property" 51 | ], 52 | "handle-callback-err": "off", 53 | "indent": [ 54 | "error", 55 | 2 56 | ], 57 | "max-nested-callbacks": [ 58 | "error", 59 | { 60 | "max": 4 61 | } 62 | ], 63 | "max-statements-per-line": [ 64 | "error", 65 | { 66 | "max": 3 67 | } 68 | ], 69 | "no-console": "off", 70 | "no-empty-function": [ 71 | "error", 72 | { 73 | "allow": [ 74 | "constructors" 75 | ] 76 | } 77 | ], 78 | "no-floating-decimal": "error", 79 | "no-lonely-if": "error", 80 | "no-multi-spaces": "error", 81 | "no-multiple-empty-lines": [ 82 | "error", 83 | { 84 | "max": 2, 85 | "maxEOF": 1, 86 | "maxBOF": 0 87 | } 88 | ], 89 | "no-shadow": [ 90 | "error", 91 | { 92 | "allow": [ 93 | "err", 94 | "resolve", 95 | "reject" 96 | ] 97 | } 98 | ], 99 | "key-spacing": [ 100 | "error", 101 | { 102 | "afterColon": true 103 | } 104 | ], 105 | "no-trailing-spaces": [ 106 | "error" 107 | ], 108 | "object-curly-spacing": [ 109 | "error", 110 | "always" 111 | ], 112 | "prefer-const": "error", 113 | "quotes": [ 114 | "error", 115 | "single" 116 | ], 117 | "semi": [ 118 | "error", 119 | "always" 120 | ], 121 | "block-spacing": [ 122 | "error", 123 | "always" 124 | ], 125 | "space-before-blocks": "error", 126 | "space-before-function-paren": [ 127 | "error", 128 | { 129 | "anonymous": "never", 130 | "named": "never", 131 | "asyncArrow": "always" 132 | } 133 | ], 134 | "template-curly-spacing": "error", 135 | "space-in-parens": "error", 136 | "space-infix-ops": "error", 137 | "space-unary-ops": "error", 138 | "spaced-comment": "error", 139 | "max-len": [ 140 | "error", 141 | { 142 | "code": 120, 143 | "ignoreStrings": true, 144 | "ignoreTemplateLiterals": true 145 | } 146 | ], 147 | "eol-last": [ 148 | "error", 149 | "always" 150 | ], 151 | "yoda": "error" 152 | } 153 | } -------------------------------------------------------------------------------- /channels/default/AniMare.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Inaba Haneru", 5 | "jp": "因幡はねる" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UC0Owc36U9lOyi9Gx9Ic-4qg", 9 | "details": { 10 | "twitter": "Haneru_Inaba" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Souya Ichika", 16 | "jp": "宗谷 いちか" 17 | }, 18 | "platform_id": "yt", 19 | "channel_id": "UC2kyQhzGOB-JPgcQX9OMgEw", 20 | "details": { 21 | "twitter": "Ichika_Souya" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Hinokuma Ran", 27 | "jp": "日ノ隈らん" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCRvpMpzAXBRKJQuk-8-Sdvg", 31 | "details": { 32 | "twitter": "Ran_Hinokuma" 33 | } 34 | }, 35 | { 36 | "name": { 37 | "en": "Kazami Kuku", 38 | "jp": "風見くく" 39 | }, 40 | "platform_id": "yt", 41 | "channel_id": "UCXp7sNC0F_qkjickvlYkg-Q", 42 | "details": { 43 | "twitter": "Kuku_Kazami" 44 | } 45 | }, 46 | { 47 | "name": { 48 | "en": "Yunohara Izumi", 49 | "jp": "柚原いづみ" 50 | }, 51 | "platform_id": "yt", 52 | "channel_id": "UCW8WKciBixmaqaGqrlTITRQ", 53 | "details": { 54 | "twitter": "Izumi_Yunohara" 55 | } 56 | }, 57 | { 58 | "name": { 59 | "en": "Shiromiya Mimi", 60 | "jp": "白宮みみ" 61 | }, 62 | "platform_id": "yt", 63 | "channel_id": "UCtzCQnCT9E4o6U3mHHSHbQQ", 64 | "details": { 65 | "twitter": "shiromiya_mimi" 66 | } 67 | }, 68 | { 69 | "name": { 70 | "en": "Shiromiya Mimi", 71 | "jp": "白宮みみ" 72 | }, 73 | "platform_id": "tt", 74 | "channel_id": "shiromiyamimi", 75 | "details": { 76 | "twitter": "shiromiya_mimi" 77 | } 78 | }, 79 | { 80 | "name": { 81 | "en": "Hashiba Natsumi", 82 | "jp": "羽柴なつみ" 83 | }, 84 | "platform_id": "yt", 85 | "channel_id": "UC_BlXOQe5OcRC7o0GX8kp8A", 86 | "details": { 87 | "twitter": "Natsumi_Hashiba" 88 | } 89 | }, 90 | { 91 | "name": { 92 | "en": "Seshima Rui", 93 | "jp": "瀬島るい" 94 | }, 95 | "platform_id": "yt", 96 | "channel_id": "UC_WOBIopwUih0rytRnr_1Ag", 97 | "details": { 98 | "twitter": "Rui_Seshima" 99 | } 100 | }, 101 | { 102 | "name": { 103 | "en": "Hira Hikari", 104 | "jp": "飛良ひかり" 105 | }, 106 | "platform_id": "yt", 107 | "channel_id": "UCFsWaTQ7kT76jNNGeGIKNSA", 108 | "details": { 109 | "twitter": "Hikari_Hira" 110 | } 111 | }, 112 | { 113 | "name": { 114 | "en": "Tsukinoki Tirol", 115 | "jp": "月野木ちろる" 116 | }, 117 | "platform_id": "yt", 118 | "channel_id": "UCqskJ0nmw-_eweWfsKvbrzQ", 119 | "details": { 120 | "twitter": "tirol0_0lorit" 121 | } 122 | }, 123 | { 124 | "name": { 125 | "en": "Oura Rukako", 126 | "jp": "大浦るかこ" 127 | }, 128 | "platform_id": "yt", 129 | "channel_id": "UC3xG1XWzAKt5dxSxktJvtxg", 130 | "details": { 131 | "twitter": "Rukako_Oura" 132 | } 133 | }, 134 | { 135 | "name": { 136 | "en": "Konan Mia", 137 | "jp": "湖南みあ" 138 | }, 139 | "platform_id": "yt", 140 | "channel_id": "UC4PrHgUcAtOoj_LKmUL-uLQ", 141 | "details": { 142 | "twitter": "Mia_Konan" 143 | } 144 | } 145 | ] -------------------------------------------------------------------------------- /channels/default/Vspo.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Kogara Toto", 5 | "jp": "小雀とと" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UCgTzsBI0DIRopMylJEDqnog", 9 | "details": { 10 | "twitter": "toto_kogara" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Kaga Sumire", 16 | "jp": "花芽すみれ" 17 | }, 18 | "platform_id": "yt", 19 | "channel_id": "UCyLGcqYs7RsBb3L0SJfzGYA", 20 | "details": { 21 | "twitter": "sumire_kaga" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Kaga Nazuna", 27 | "jp": "花芽なずな" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCiMG6VdScBabPhJ1ZtaVmbw", 31 | "details": { 32 | "twitter": "nazuna_kaga" 33 | } 34 | }, 35 | { 36 | "name": { 37 | "en": "Ichinose Uruha", 38 | "jp": "一ノ瀬うるは" 39 | }, 40 | "platform_id": "yt", 41 | "channel_id": "UC5LyYg6cCA4yHEYvtUsir3g", 42 | "details": { 43 | "twitter": "uruha_ichinose" 44 | } 45 | }, 46 | { 47 | "name": { 48 | "en": "Tachibana Hinano", 49 | "jp": "橘ひなの" 50 | }, 51 | "platform_id": "yt", 52 | "channel_id": "UCvUc0m317LWTTPZoBQV479A", 53 | "details": { 54 | "twitter": "hinano_tachiba7" 55 | } 56 | }, 57 | { 58 | "name": { 59 | "en": "Kurumi Noah", 60 | "jp": "胡桃のあ" 61 | }, 62 | "platform_id": "yt", 63 | "channel_id": "UCIcAj6WkJ8vZ7DeJVgmeqKw", 64 | "details": { 65 | "twitter": "n0ah_kurumi" 66 | } 67 | }, 68 | { 69 | "name": { 70 | "en": "Kisaragi Ren", 71 | "jp": "如月れん" 72 | }, 73 | "platform_id": "yt", 74 | "channel_id": "UCGWa1dMU_sDCaRQjdabsVgg", 75 | "details": { 76 | "twitter": "ren_kisaragi__" 77 | } 78 | }, 79 | { 80 | "name": { 81 | "en": "Hanabusa Lisa", 82 | "jp": "英 リサ" 83 | }, 84 | "platform_id": "yt", 85 | "channel_id": "UCurEA8YoqFwimJcAuSHU0MQ", 86 | "details": { 87 | "twitter": "Lisa_hanabusa" 88 | } 89 | }, 90 | { 91 | "name": { 92 | "en": "Tosaki Mimi" 93 | }, 94 | "platform_id": "yt", 95 | "channel_id": "UCnvVG9RbOW3J6Ifqo-zKLiw", 96 | "details": { 97 | "twitter": "mimi_tosaki" 98 | } 99 | }, 100 | { 101 | "name": { 102 | "en": "Asumi Sena", 103 | "jp": "空澄 セナ" 104 | }, 105 | "platform_id": "yt", 106 | "channel_id": "UCF_U2GCKHvDz52jWdizppIA", 107 | "details": { 108 | "twitter": "sena_asumi" 109 | } 110 | }, 111 | { 112 | "name": { 113 | "en": "Kaminari Qpi", 114 | "jp": "神成きゅぴ" 115 | }, 116 | "platform_id": "yt", 117 | "channel_id": "UCMp55EbT_ZlqiMS3lCj01BQ", 118 | "details": { 119 | "twitter": "xprprQchanx" 120 | } 121 | }, 122 | { 123 | "name": { 124 | "en": "Yakumo Beni", 125 | "jp": "八雲べに" 126 | }, 127 | "platform_id": "yt", 128 | "channel_id": "UCjXBuHmWkieBApgBhDuJMMQ", 129 | "details": { 130 | "twitter": "beni_yakumo" 131 | } 132 | }, 133 | { 134 | "name": { 135 | "en": "Aizawa Ema", 136 | "jp": "藍沢エマ" 137 | }, 138 | "platform_id": "yt", 139 | "channel_id": "UCPkKpOHxEDcwmUAnRpIu-Ng", 140 | "details": { 141 | "twitter": "Ema_Aizawa" 142 | } 143 | }, 144 | { 145 | "name": { 146 | "en": "Shinomiya Runa", 147 | "jp": "紫宮るな" 148 | }, 149 | "platform_id": "yt", 150 | "channel_id": "UCD5W21JqNMv_tV9nfjvF9sw", 151 | "details": { 152 | "twitter": "Runa_shinomiya" 153 | } 154 | } 155 | ] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # VTuber API 2 | A Mongoose / GraphQL based API to serve VTuber information from multiple platforms. 3 | 4 | ## Development 5 | * Prerequisites: 6 | * Have `node` installed. 7 | * Have [MongoDB](https://docs.mongodb.com/manual/installation/) and [Memcached](https://www.howtoforge.com/how-to-install-memcached-on-ubuntu-2004-lts/) installed locally. 8 | * Optional: Download [MongoDB Compass](https://www.mongodb.com/try/download/compass) to access your database with a GUI. 9 | * Have a [Google Cloud Project](https://console.cloud.google.com/apis/credentials) API Key with Youtube API enabled. 10 | * Setup some channels first before starting. 11 | 12 | ## Installation 13 | ``` 14 | # Install dependencies and create your .env copy 15 | $ npm i 16 | $ cp .env.sample .env 17 | # Make sure to adjust your .env file before continuing! 18 | 19 | # Create a directory called organizations inside channels, then move the files from 20 | # channels/default to channels/organizations, or check the template.json file to see how to create your own list. 21 | 22 | # After populating channels/organizations, run: 23 | $ npm run channel-manager 24 | # OR 25 | $ npm run init 26 | 27 | # If all things went well, you can then start the api. 28 | $ npm start 29 | ``` 30 | 31 | ## GraphQL Schema 32 | 33 | ## Types 34 | ```ts 35 | { 36 | VideoId: string 37 | ChannelId: string 38 | PlatformId: "yt"|"bb"|"tt" 39 | VideoStatus: "live"|"upcoming"|"ended"|"uploaded"|"missing" 40 | } 41 | ``` 42 | 43 | ## `VideoObject` Schema 44 | ```ts 45 | { 46 | _id: VideoId! 47 | platform_id: PlatformId! 48 | channel_id: ChannelId! 49 | organization: string! 50 | title: string! 51 | time: { 52 | published: number 53 | scheduled: number 54 | start: number 55 | end: number 56 | duration: number 57 | } 58 | status: VideoStatus! 59 | viewers: number 60 | } 61 | ``` 62 | 63 | ## `ChannelObject` Schema 64 | ```ts 65 | { 66 | _id: number! 67 | name: { 68 | en: string! 69 | jp: string 70 | kr: string 71 | cn: string 72 | }! 73 | organization: string! 74 | platform_id: PlatformId! 75 | channel_name: string 76 | channel_id: ChannelId! 77 | details: { 78 | [key: string]: any 79 | } 80 | channel_stats: { 81 | published_at: number 82 | views: number 83 | subscribers: number 84 | videos: number 85 | } 86 | description: string 87 | thumbnail: string 88 | } 89 | ``` 90 | 91 | ## Videos Resource 92 | ``` 93 | { 94 | items: [VideoObject]! 95 | next_page_token: String 96 | } 97 | ``` 98 | 99 | ## Channels Resource 100 | ``` 101 | { 102 | items: [ChannelObject]! 103 | next_page_token: String 104 | } 105 | ``` 106 | 107 | ## Live Query 108 | ``` 109 | { 110 | live( 111 | organizations: [String] 112 | platforms: [PlatformId] 113 | exclude_organizations: [String] 114 | ): [VideoObject] 115 | } 116 | ``` 117 | 118 | ## Videos Query 119 | ``` 120 | { 121 | videos( 122 | channel_id: [ChannelId] 123 | status: [VideoStatus] 124 | organization: [String] 125 | platforms: [PlatformId] 126 | max_upcoming_mins: Int 127 | order_by: { 128 | published: asc|desc 129 | scheduled: asc|desc 130 | start: asc|desc 131 | } 132 | next_page_token: String 133 | limit: Int // 1-50 134 | ): VideosResource! 135 | } 136 | ``` 137 | 138 | ## Channels Query 139 | ``` 140 | { 141 | channels( 142 | _id: [Int] 143 | name: String 144 | organizations: [String] 145 | platforms: [PlatformId] 146 | channel_id: [ChannelId] 147 | order_by: { 148 | _id: asc|desc 149 | published_at: asc|desc 150 | subscribers: asc|desc 151 | } 152 | next_page_token: String 153 | limit: Int // 1-50 154 | ): ChannelsResource! 155 | } 156 | ``` 157 | 158 | ## 159 | 160 | ## TO-DOs 161 | * Implement twitch and bilibili apis 162 | * Better documentation -------------------------------------------------------------------------------- /channels/default/Nori Pro.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Inuyama Tamaki", 5 | "jp": "犬山たまき", 6 | "cn": "犬山玉姬" 7 | }, 8 | "platform_id": "yt", 9 | "channel_id": "UC8NZiqKx6fsDT3AVcMiVFyA", 10 | "details": { 11 | "twitter": "norioo_", 12 | "group": "Generation 1" 13 | } 14 | }, 15 | { 16 | "name": { 17 | "en": "Inuyama Tamaki", 18 | "jp": "犬山たまき", 19 | "cn": "犬山玉姬" 20 | }, 21 | "platform_id": "bb", 22 | "channel_id": "12362451", 23 | "details": { 24 | "twitter": "norioo_", 25 | "group": "Generation 1" 26 | } 27 | }, 28 | { 29 | "name": { 30 | "en": "Enomiya Milk", 31 | "jp": "愛宮みるく" 32 | }, 33 | "platform_id": "yt", 34 | "channel_id": "UCJCzy0Fyrm0UhIrGQ7tHpjg", 35 | "details": { 36 | "twitter": "Enomiya_MILK", 37 | "group": "Generation 1" 38 | } 39 | }, 40 | { 41 | "name": { 42 | "en": "Enomiya Milk", 43 | "jp": "愛宮みるく" 44 | }, 45 | "platform_id": "bb", 46 | "channel_id": "419429042", 47 | "details": { 48 | "twitter": "Enomiya_MILK", 49 | "group": "Generation 1" 50 | } 51 | }, 52 | { 53 | "name": { 54 | "en": "Shirayuki Mishiro", 55 | "jp": "白雪みしろ", 56 | "cn": "白雪深白" 57 | }, 58 | "platform_id": "yt", 59 | "channel_id": "UCC0i9nECi4Gz7TU63xZwodg", 60 | "details": { 61 | "twitter": "mishiro_seiso", 62 | "group": "Generation 1" 63 | } 64 | }, 65 | { 66 | "name": { 67 | "en": "Shirayuki Mishiro", 68 | "jp": "白雪みしろ", 69 | "cn": "白雪深白" 70 | }, 71 | "platform_id": "bb", 72 | "channel_id": "405981431", 73 | "details": { 74 | "twitter": "mishiro_seiso", 75 | "group": "Generation 1" 76 | } 77 | }, 78 | { 79 | "name": { 80 | "en": "Himesaki Yuzuru", 81 | "jp": "姫咲ゆずる" 82 | }, 83 | "platform_id": "yt", 84 | "channel_id": "UCle1cz6rcyH0a-xoMYwLlAg", 85 | "details": { 86 | "twitter": "Himesaki_yuzuru", 87 | "group": "Generation 1" 88 | } 89 | }, 90 | { 91 | "name": { 92 | "en": "Kumagaya Takuma", 93 | "jp": "熊谷タクマ" 94 | }, 95 | "platform_id": "yt", 96 | "channel_id": "UCCXME7oZmXB2VFHJbz5496A", 97 | "details": { 98 | "twitter": "KUMAgaya_taKUMA", 99 | "group": "Generation 1" 100 | } 101 | }, 102 | { 103 | "name": { 104 | "en": "Hoozuki Warabe", 105 | "jp": "鬼灯わらべ" 106 | }, 107 | "platform_id": "yt", 108 | "channel_id": "UCLyTXfCZtl7dyhta9Jg3pZg", 109 | "details": { 110 | "twitter": "hoozukiwarabe", 111 | "group": "Generation 2" 112 | } 113 | }, 114 | { 115 | "name": { 116 | "en": "Yumeno Lilith", 117 | "jp": "夢乃リリス" 118 | }, 119 | "platform_id": "yt", 120 | "channel_id": "UCH11P1Hq4PXdznyw1Hhr3qw", 121 | "details": { 122 | "twitter": "yumenolilith", 123 | "group": "Generation 2" 124 | } 125 | }, 126 | { 127 | "name": { 128 | "en": "Oma Kirara", 129 | "jp": "逢魔きらら" 130 | }, 131 | "platform_id": "yt", 132 | "channel_id": "UCBAeKqEIugv69Q2GIgcH7oA", 133 | "details": { 134 | "twitter": "omakirara" 135 | } 136 | }, 137 | { 138 | "name": { 139 | "en": "Kurumizawa Momo", 140 | "jp": "胡桃澤もも" 141 | }, 142 | "platform_id": "yt", 143 | "channel_id": "UCxrmkJf_X1Yhte_a4devFzA", 144 | "details": { 145 | "twitter": "kurumizawamomo" 146 | } 147 | }, 148 | { 149 | "name": { 150 | "en": "Mirutani Nia", 151 | "jp": "看谷にぃあ" 152 | }, 153 | "platform_id": "yt", 154 | "channel_id": "UCIRzELGzTVUOARi3Gwf1-yg", 155 | "details": { 156 | "twitter": "mirutani_nia" 157 | } 158 | } 159 | ] -------------------------------------------------------------------------------- /channels/default/ViViD.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Shirayuri Lily", 5 | "jp": "白百合リリィ" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UCl-3q6t6zdZwgIsFZELb7Zg", 9 | "details": { 10 | "twitter": "SRYR_0" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Shirayuri Lily", 16 | "jp": "白百合リリィ" 17 | }, 18 | "platform_id": "bb", 19 | "channel_id": "421347849", 20 | "details": { 21 | "twitter": "SRYR_0" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Yunagi Elena", 27 | "jp": "勇凪エレナ" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCKwATdrcjyzNv9c56PqJbWA", 31 | "details": { 32 | "twitter": "ELENA_YUNAGI" 33 | } 34 | }, 35 | { 36 | "name": { 37 | "en": "Fuwamochi Anko", 38 | "jp": "二和餅あんこ" 39 | }, 40 | "platform_id": "yt", 41 | "channel_id": "UCBKxF1PHQtT78ypcVYmBtBA", 42 | "details": { 43 | "twitter": "ANKOchan_01" 44 | } 45 | }, 46 | { 47 | "name": { 48 | "en": "Utakata Memory", 49 | "jp": "泡沫メモリ" 50 | }, 51 | "platform_id": "yt", 52 | "channel_id": "UClV7gDF9sUzu4jJCK7rCYkQ", 53 | "details": { 54 | "twitter": "Memory_Utakata" 55 | } 56 | }, 57 | { 58 | "name": { 59 | "en": "Nekonogi Bell", 60 | "jp": "猫芒ベル" 61 | }, 62 | "platform_id": "yt", 63 | "channel_id": "UCflNPJUJ4VQh1hGDNK7bsFg", 64 | "details": { 65 | "twitter": "Bell_Nekonogi" 66 | } 67 | }, 68 | { 69 | "name": { 70 | "en": "Nekonogi Bell", 71 | "jp": "猫芒ベル" 72 | }, 73 | "platform_id": "bb", 74 | "channel_id": "487550002", 75 | "details": { 76 | "twitter": "Bell_Nekonogi" 77 | } 78 | }, 79 | { 80 | "name": { 81 | "en": "Chrono Lock", 82 | "jp": "クロノロク" 83 | }, 84 | "platform_id": "yt", 85 | "channel_id": "UCAvgFoTWEisO5Pp7eYVDgEw", 86 | "details": { 87 | "twitter": "chrono_lock" 88 | } 89 | }, 90 | { 91 | "name": { 92 | "en": "Homuragi Pairo", 93 | "jp": "焔機パイロ" 94 | }, 95 | "platform_id": "yt", 96 | "channel_id": "UCx45gypxj0fME6zxsLKsNow", 97 | "details": { 98 | "twitter": "memento_vivi" 99 | } 100 | }, 101 | { 102 | "name": { 103 | "en": "Shinomiya Ruri", 104 | "jp": "紫乃宮るり" 105 | }, 106 | "platform_id": "yt", 107 | "channel_id": "UCvU0iyfiz9zzXSlfzTbbR5w", 108 | "details": { 109 | "twitter": "Shinomiya_Ruri" 110 | } 111 | }, 112 | { 113 | "name": { 114 | "en": "Catena Violetta", 115 | "jp": "カテナ・ヴィオレッタ" 116 | }, 117 | "platform_id": "yt", 118 | "channel_id": "UCn3HVGEh-uMStdkcSL6g7TA", 119 | "details": { 120 | "twitter": "catena_wine" 121 | } 122 | }, 123 | { 124 | "name": { 125 | "en": "Tatsuta Amaki", 126 | "jp": "竜田天姫" 127 | }, 128 | "platform_id": "yt", 129 | "channel_id": "UC-HKrHYwUV1hX6SEn22Su7Q", 130 | "details": { 131 | "twitter": "tatsuta3amaki3" 132 | } 133 | }, 134 | { 135 | "name": { 136 | "en": "Harumi Mika", 137 | "jp": "晴海みか" 138 | }, 139 | "platform_id": "yt", 140 | "channel_id": "UC4lCGN8XFyThIK_E2dlwIow", 141 | "details": { 142 | "twitter": "Harumi_MikaMaru" 143 | } 144 | }, 145 | { 146 | "name": { 147 | "en": "Pinosu Ruri", 148 | "jp": "ぴのするり" 149 | }, 150 | "platform_id": "yt", 151 | "channel_id": "UCqaCCTxZho-MUJlT3c9UyfA", 152 | "details": { 153 | "twitter": "ruri_pino_ruri" 154 | } 155 | }, 156 | { 157 | "name": { 158 | "en": "Hitome Hakuhu", 159 | "jp": "白布ひとめ" 160 | }, 161 | "platform_id": "yt", 162 | "channel_id": "UCY1qEwgCUFAS0mBnHixfPUg", 163 | "details": { 164 | "twitter": "hitome_hakuhu" 165 | } 166 | } 167 | ] -------------------------------------------------------------------------------- /src/graphql/query/videos.ts: -------------------------------------------------------------------------------- 1 | import { ApolloError, UserInputError } from 'apollo-server'; 2 | import { PlatformId } from '../../../database/types/members'; 3 | import { memcache, Videos } from '../../modules'; 4 | import { ChannelId } from '../../modules/types/youtube'; 5 | import { VideoStatus } from '../../server/apis/youtube/types'; 6 | import { cutGroupString, escapeRegex, firstField, getCacheKey, getNextToken, parseOrganization, parseToken, Sort } from './consts'; 7 | 8 | interface SortBy { 9 | published?: Sort; 10 | scheduled?: Sort; 11 | start?: Sort; 12 | } 13 | 14 | interface VideoQuery { 15 | channel_id: ChannelId[]; 16 | status: VideoStatus[]; 17 | title: string; 18 | organizations: string[]; 19 | exclude_organizations: string[]; 20 | platforms: PlatformId[]; 21 | max_upcoming_mins: number; 22 | order_by: SortBy; 23 | page_token: string; 24 | limit: number; 25 | } 26 | 27 | export async function videos(_, query: VideoQuery) { 28 | try { 29 | const { 30 | channel_id = [], 31 | status = [], 32 | title, 33 | organizations = [], 34 | exclude_organizations = [], 35 | platforms = [], 36 | max_upcoming_mins, 37 | page_token = '', 38 | limit 39 | } = query; 40 | 41 | if (limit < 1 || limit > 50) { 42 | return new UserInputError('limit must be between 1-50 inclusive.'); 43 | } 44 | if (max_upcoming_mins < 0 || max_upcoming_mins > 2880) { 45 | return new UserInputError('max_upcoming_mins must be between 0-2880 inclusive.'); 46 | } 47 | if (organizations.length && exclude_organizations.length) { 48 | return new UserInputError('Setting both organizations and exclude_organizations is redundant. Only choose one.'); 49 | } 50 | const EXCLUDE_ORG = !organizations.length; 51 | const MAX_UPCOMING = max_upcoming_mins * 6e4; 52 | const TITLE = title && escapeRegex(title); 53 | const ORGANIZATIONS = parseOrganization(EXCLUDE_ORG ? exclude_organizations : organizations); 54 | const [ORDER_BY, ORDER_BY_KEY] = firstField(query.order_by); 55 | const [ORDER_KEY, ORDER_VALUE] = Object.entries(ORDER_BY)[0]; 56 | const orderBy = { [`time.${ORDER_KEY}`]: ORDER_VALUE }; 57 | const CACHE_KEY = getCacheKey(`VIDS:${+EXCLUDE_ORG}${cutGroupString(ORGANIZATIONS)}${channel_id}${status}${TITLE}${platforms}${max_upcoming_mins}${ORDER_BY_KEY}${limit}${page_token}`); 58 | 59 | const cached = await memcache.get(CACHE_KEY); 60 | if (cached) return cached; 61 | 62 | const QUERY: any = { // any because typescript gets mad for some reason. 63 | status: status[0] ? { $in: status } : { $ne: 'missing' }, 64 | ...channel_id[0] && { channel_id: { $in: channel_id } }, 65 | ...TITLE && { title: { $regex: TITLE, $options: 'i' } }, 66 | ...ORGANIZATIONS[0] && { organization: { 67 | ...EXCLUDE_ORG 68 | ? { $not: { $regex: ORGANIZATIONS, $options: 'i' } } 69 | : { $regex: ORGANIZATIONS, $options: 'i' } 70 | } }, 71 | ...platforms[0] && { platform_id: { $in: platforms } }, 72 | ...max_upcoming_mins && { 'time.scheduled': { $lte: Date.now() + MAX_UPCOMING } } 73 | }; 74 | 75 | const getVideoCount = Videos.countDocuments(QUERY); 76 | const getUncachedVideos = Videos 77 | .find({ 78 | ...QUERY, 79 | ...page_token && { [Object.keys(orderBy)[0]]: { [ORDER_VALUE === 'asc' ? '$gte' : '$lte']: parseToken(page_token) } }, 80 | }) 81 | .sort(orderBy) 82 | .limit(limit + 1) 83 | .lean() 84 | .exec(); 85 | 86 | const [videoCount, uncachedVideos] = await Promise.all([getVideoCount, getUncachedVideos]); 87 | const results = { 88 | items: uncachedVideos, 89 | next_page_token: null, 90 | page_info: { 91 | total_results: videoCount, 92 | results_per_page: limit 93 | } 94 | }; 95 | 96 | const hasNextPage = uncachedVideos.length > limit && results.items.pop(); 97 | if (hasNextPage) { 98 | const token = hasNextPage.time[ORDER_KEY]; 99 | results.next_page_token = getNextToken(token); 100 | } 101 | 102 | memcache.set(CACHE_KEY, results, 60); 103 | return results; 104 | } catch(err) { 105 | throw new ApolloError(err); 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /channels/default/Atelier Live.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Anya Pandaria" 5 | }, 6 | "platform_id": "yt", 7 | "channel_id": "UC_-vS9T3_X52HWMhIBsHouw", 8 | "details": { 9 | "twitter": "anya_pandaria" 10 | } 11 | }, 12 | { 13 | "name": { 14 | "en": "Anya Pandaria" 15 | }, 16 | "platform_id": "tt", 17 | "channel_id": "anyapanpan", 18 | "details": { 19 | "twitter": "anya_pandaria" 20 | } 21 | }, 22 | { 23 | "name": { 24 | "en": "Artemis of the Blue" 25 | }, 26 | "platform_id": "yt", 27 | "channel_id": "UCWImOidHDmm0KK20bkF-rSQ", 28 | "details": { 29 | "twitter": "artemisvtuber" 30 | } 31 | }, 32 | { 33 | "name": { 34 | "en": "Artemis of the Blue" 35 | }, 36 | "platform_id": "tt", 37 | "channel_id": "artemisoftheblue", 38 | "details": { 39 | "twitter": "artemisvtuber" 40 | } 41 | }, 42 | { 43 | "name": { 44 | "en": "Chaikitsu" 45 | }, 46 | "platform_id": "yt", 47 | "channel_id": "UCeZsdcN25H3YmjXAKARorQA", 48 | "details": { 49 | "twitter": "yueko__" 50 | } 51 | }, 52 | { 53 | "name": { 54 | "en": "Chaikitsu" 55 | }, 56 | "platform_id": "tt", 57 | "channel_id": "yueko", 58 | "details": { 59 | "twitter": "yueko__" 60 | } 61 | }, 62 | { 63 | "name": { 64 | "en": "Yamane Chiika", 65 | "jp": "山根ちいか" 66 | }, 67 | "platform_id": "yt", 68 | "channel_id": "UCfIZ5P7BcgIyLpfOj66DwtA", 69 | "details": { 70 | "twitter": "chiikadayo" 71 | } 72 | }, 73 | { 74 | "name": { 75 | "en": "Yamane Chiika", 76 | "jp": "山根ちいか" 77 | }, 78 | "platform_id": "tt", 79 | "channel_id": "yamachiika", 80 | "details": { 81 | "twitter": "chiikadayo" 82 | } 83 | }, 84 | { 85 | "name": { 86 | "en": "Europa" 87 | }, 88 | "platform_id": "yt", 89 | "channel_id": "UC5Ejf_RIWMVDAjA4B-GV5Zg", 90 | "details": { 91 | "twitter": "EuropaYuu" 92 | } 93 | }, 94 | { 95 | "name": { 96 | "en": "Himey" 97 | }, 98 | "platform_id": "tt", 99 | "channel_id": "himeyyy", 100 | "details": { 101 | "twitter": "_hime0" 102 | } 103 | }, 104 | { 105 | "name": { 106 | "en": "Io" 107 | }, 108 | "platform_id": "yt", 109 | "channel_id": "UCHw7uZcEFhbUKyHyz9gOG6A", 110 | "details": { 111 | "twitter": "kuzuryuio" 112 | } 113 | }, 114 | { 115 | "name": { 116 | "en": "Kani Kanizawa" 117 | }, 118 | "platform_id": "yt", 119 | "channel_id": "UC55-Y4cBl6alNaaOVVEYgSA", 120 | "details": { 121 | "twitter": "kanikanizawa" 122 | } 123 | }, 124 | { 125 | "name": { 126 | "en": "Kuri" 127 | }, 128 | "platform_id": "yt", 129 | "channel_id": "UCguTLlIfEP-1KzkogRUcAKw", 130 | "details": { 131 | "twitter": "cloverkuri" 132 | } 133 | }, 134 | { 135 | "name": { 136 | "en": "Kuri" 137 | }, 138 | "platform_id": "tt", 139 | "channel_id": "klaeia", 140 | "details": { 141 | "twitter": "cloverkuri" 142 | } 143 | }, 144 | { 145 | "name": { 146 | "en": "Mari", 147 | "jp": "マリ" 148 | }, 149 | "platform_id": "tt", 150 | "channel_id": "mari_vt", 151 | "details": { 152 | "twitter": "MariTheHybrid" 153 | } 154 | }, 155 | { 156 | "name": { 157 | "en": "Nana Nanatsuki" 158 | }, 159 | "platform_id": "tt", 160 | "channel_id": "exxpulse", 161 | "details": { 162 | "twitter": "nana__nanatsuki" 163 | } 164 | }, 165 | { 166 | "name": { 167 | "en": "Tsubaki" 168 | }, 169 | "platform_id": "tt", 170 | "channel_id": "juwei_", 171 | "details": { 172 | "twitter": "tsubakisinensis", 173 | "instagram": "juwei_" 174 | } 175 | }, 176 | { 177 | "name": { 178 | "en": "Rosuuri" 179 | }, 180 | "platform_id": "yt", 181 | "channel_id": "UC-Z8ZmHmvv_5tinNQSDw0xg", 182 | "details": { 183 | "twitter": "Ganbarosuu" 184 | } 185 | }, 186 | { 187 | "name": { 188 | "en": "Rosuuri" 189 | }, 190 | "platform_id": "tt", 191 | "channel_id": "rosuuri", 192 | "details": { 193 | "twitter": "Ganbarosuu" 194 | } 195 | } 196 | ] -------------------------------------------------------------------------------- /channels/apps/scrapers/youtube-scraper.ts: -------------------------------------------------------------------------------- 1 | import { PlaylistItemsResource, VideoResource, YoutubeVideoObject } from '../../../src/server/apis/youtube/types'; 2 | import { getVideoStatus } from '../../../src/server/apis/youtube/video-updater'; 3 | import { MemberObject } from '../../../database/types/members'; 4 | import { debug, youtube } from '../../../src/modules'; 5 | import database from '../database-manager'; 6 | 7 | const logger = debug('api:youtube'); 8 | const playlistList = logger.extend('playlistList'); 9 | const videoList = logger.extend('videoList'); 10 | 11 | export default async function(channelData: MemberObject): Promise<['FAIL'|'OK', number]> { 12 | const { channel_id, organization } = channelData; 13 | logger.info(`Scraping youtube channel ${channel_id}...`); 14 | const channelVideoList = await scrapeChannel(channel_id, organization); 15 | if (!channelVideoList) { 16 | logger.error(`Failed to scrape youtube channel ${channel_id}. Skipping...`); 17 | return ['FAIL', 0]; 18 | } 19 | logger.info(`Got ${channelVideoList.length} videos from ${channel_id}`); 20 | database.emit('save-videos', channelVideoList); 21 | database.emit('update-member', { channel_id, crawled_at: new Date() }); 22 | return ['OK', channelVideoList.length]; 23 | } 24 | 25 | async function listPlaylistItems( 26 | playlistId: string, pageToken = '' 27 | ): Promise<[PlaylistItemsResource[], string, 'OK']> { 28 | playlistList.info('Fetching playlist items from youtube...'); 29 | const response = await youtube.playlistItems({ 30 | part: 'snippet', 31 | fields: 'nextPageToken,items(snippet(channelId,title,resourceId/videoId))', 32 | playlistId, 33 | pageToken, 34 | maxResults: 50 35 | }).then(data => [ 36 | data.items, 37 | data.nextPageToken, 38 | 'OK' 39 | ]).catch(error => { 40 | playlistList.error(error); 41 | return [[]]; 42 | }); 43 | playlistList.info(`Fetched ${response[0].length} videos from playlist id: ${playlistId}.`); 44 | return response; 45 | } 46 | 47 | async function listVideos( 48 | items: PlaylistItemsResource[], 49 | organization: string 50 | ): Promise { 51 | videoList.info(`Fetching ${items.length} video data from youtube...`); 52 | const results = await youtube.videos({ 53 | part: 'snippet,liveStreamingDetails', 54 | fields: 'items(id,snippet(channelId,title,publishedAt),liveStreamingDetails(scheduledStartTime,actualStartTime,actualEndTime,concurrentViewers))', 55 | id: items.map(item => item.snippet.resourceId.videoId).join(','), 56 | hl: 'ja' 57 | }).then(data => data.items.map(item => parseVideos(item, organization))) 58 | .catch(err => { throw err; }); 59 | videoList.info(`Fetched ${results.length} video data.`); 60 | return results; 61 | } 62 | 63 | function parseVideos( 64 | { id, snippet, liveStreamingDetails }: VideoResource, 65 | organization: string 66 | ): YoutubeVideoObject { 67 | const { channelId, title, publishedAt } = snippet ?? {}; 68 | const { scheduledStartTime, actualStartTime, actualEndTime, concurrentViewers } = liveStreamingDetails ?? {}; 69 | return { 70 | _id: id, 71 | platform_id: 'yt', 72 | channel_id: channelId, 73 | organization, 74 | title, 75 | time: { 76 | published: new Date(publishedAt), 77 | scheduled: scheduledStartTime ? new Date(scheduledStartTime) : undefined, 78 | start: actualStartTime ? new Date(actualStartTime) : undefined, 79 | end: actualEndTime ? new Date(actualEndTime) : undefined 80 | }, 81 | status: getVideoStatus(liveStreamingDetails), 82 | viewers: +concurrentViewers || null, 83 | updated_at: new Date() 84 | }; 85 | } 86 | 87 | async function scrapeChannel(channelId: string, organization: string) { 88 | let playlistVideoList: PlaylistItemsResource[] = [], nextPageToken: string, status: 'OK'; 89 | const youtubeVideos: Promise[] = []; 90 | const playlistId = 'UU' + channelId.slice(2); 91 | const requestPlaylist: (pageToken: string) => Promise<[PlaylistItemsResource[], string, 'OK']> = listPlaylistItems.bind(null, playlistId); 92 | const fetchVideos = async (videos: PlaylistItemsResource[]) => youtubeVideos.push(listVideos(videos, organization)); 93 | 94 | do { 95 | [playlistVideoList = [], nextPageToken, status] = await requestPlaylist(nextPageToken); 96 | fetchVideos(playlistVideoList); 97 | logger.log(`Current video count: ${youtubeVideos.length}`); 98 | } while (nextPageToken && status === 'OK'); 99 | 100 | if (status !== 'OK') { 101 | logger.error('youtube threw an error. skipping.'); 102 | } else { return (await Promise.all(youtubeVideos)).flat(); } 103 | } 104 | -------------------------------------------------------------------------------- /src/graphql/query/channels.ts: -------------------------------------------------------------------------------- 1 | import { ApolloError, UserInputError } from 'apollo-server'; 2 | import { PlatformId } from '../../../database/types/members'; 3 | import { Channels, memcache } from '../../modules'; 4 | import { ChannelId } from '../../modules/types/youtube'; 5 | import { cutChannelIds, cutGroupString, escapeRegex, firstField, getCacheKey, getNextToken, parseOrganization, parseToken, Sort } from './consts'; 6 | 7 | const CACHE_TTL = +(process.env.TTL_LONG ?? 900); 8 | 9 | interface OrderBy { 10 | _id?: Sort; 11 | published_at?: Sort; 12 | subscribers?: Sort; 13 | } 14 | 15 | interface ChannelsQuery { 16 | _id: number[]; 17 | name: string; 18 | organizations: string[]; 19 | exclude_organizations: string[]; 20 | platforms: PlatformId[]; 21 | exclude_channel_id: ChannelId[]; 22 | channel_id: ChannelId[]; 23 | order_by: OrderBy; 24 | page_token: string; 25 | limit: number; 26 | } 27 | 28 | export async function channels(_, query: ChannelsQuery) { 29 | try { 30 | const { 31 | _id = [], 32 | name = '', 33 | organizations = [], 34 | exclude_organizations = [], 35 | exclude_channel_id = [], 36 | channel_id = [], 37 | platforms = [], 38 | page_token = '', 39 | limit 40 | } = query; 41 | if (limit < 1 || limit > 50) { 42 | return new UserInputError('limit must be between 1-50 inclusive.'); 43 | } 44 | if (organizations.length && exclude_organizations.length) { 45 | return new UserInputError('Setting both organizations and exclude_organizations is redundant. Only choose one.'); 46 | } 47 | if (channel_id.length && exclude_channel_id.length) { 48 | return new UserInputError('Setting both channel_id and exclude_channel_id is redundant. Only choose one.'); 49 | } 50 | const EXCLUDE_ORG = !organizations.length; 51 | const EXCLUDE_IDS = !channel_id.length; 52 | const [ORDER_BY, ORDER_BY_KEY] = firstField(query.order_by); 53 | const [ORDER_KEY, ORDER_VALUE] = Object.entries(ORDER_BY)[0]; 54 | const sortById = ORDER_KEY === '_id'; 55 | const sortBy = sortById ? ORDER_BY : { [`channel_stats.${ORDER_KEY}`]: ORDER_VALUE }; 56 | const ORGANIZATIONS = parseOrganization(EXCLUDE_ORG ? exclude_organizations : organizations); 57 | const CHANNEL_IDS = EXCLUDE_IDS ? exclude_channel_id : channel_id; 58 | const CACHE_KEY = getCacheKey(`CHNLS:${+EXCLUDE_ORG}${+EXCLUDE_IDS}${_id}${(name)}${cutGroupString(ORGANIZATIONS)}${cutChannelIds(CHANNEL_IDS)}${platforms}${limit}${ORDER_BY_KEY}${page_token}`, false); 59 | 60 | const cached = await memcache.get(CACHE_KEY); 61 | if (cached) return cached; 62 | 63 | const QUERY = { 64 | _id: { [_id[0] ? '$in' : '$nin']: _id }, 65 | ...name && { $or: getNameQueries(name) }, 66 | ...ORGANIZATIONS[0] && { organization: { 67 | ...EXCLUDE_ORG 68 | ? { $not: { $regex: ORGANIZATIONS, $options: 'i' } } 69 | : { $regex: ORGANIZATIONS, $options: 'i' } 70 | } }, 71 | ...channel_id[0] && { channel_id: { [EXCLUDE_IDS ? '$nin' : '$in']: CHANNEL_IDS } }, 72 | ...platforms[0] && { platform_id: { $in: platforms } } 73 | }; 74 | 75 | const getChannelCount = Channels.countDocuments(QUERY); 76 | const getUncachedChannels = Channels 77 | .find({ 78 | ...QUERY, 79 | ...page_token && { [Object.keys(sortBy)[0]]: { [ORDER_VALUE === 'asc' ? '$gte' : '$lte']: parseToken(page_token) } }, 80 | }) 81 | .sort(sortBy) 82 | .limit(limit + 1) 83 | .lean() 84 | .exec(); 85 | 86 | const [channelCount, uncachedChannels] = await Promise.all([getChannelCount, getUncachedChannels]); 87 | const results = { 88 | items: uncachedChannels, 89 | next_page_token: null, 90 | page_info: { 91 | total_results: channelCount, 92 | results_per_page: limit 93 | } 94 | }; 95 | 96 | const hasNextPage = uncachedChannels.length > limit && results.items.pop(); 97 | if (hasNextPage) { 98 | const token = sortById ? hasNextPage._id : hasNextPage.channel_stats[ORDER_KEY]; 99 | results.next_page_token = getNextToken(token); 100 | } 101 | 102 | memcache.set(CACHE_KEY, results, CACHE_TTL); 103 | return results; 104 | } catch(err) { 105 | throw new ApolloError(err); 106 | } 107 | } 108 | 109 | const getNameQueries = (name: string) => { 110 | const nameRegex = escapeRegex(unescape(name)).split(/ +/g).map(string => `(?=.*${string})`).join(''); 111 | return [ 112 | { 'name.en': { $regex: nameRegex, $options: 'i' } }, 113 | { 'name.jp': { $regex: nameRegex, $options: 'i' } }, 114 | { 'name.kr': { $regex: nameRegex, $options: 'i' } }, 115 | { 'name.cn': { $regex: nameRegex, $options: 'i' } } 116 | ]; 117 | }; 118 | -------------------------------------------------------------------------------- /src/modules/types/youtube.ts: -------------------------------------------------------------------------------- 1 | export type VideoId = string; 2 | export type DateString = string; 3 | export type ChannelId = string; 4 | export type UnsignedLong = string; 5 | export type UnsignedInteger = string; 6 | 7 | interface DefaultParams { 8 | /** 9 | * Available parts: 10 | * - contentDetails 11 | * - fileDetails 12 | * - id 13 | * - liveStreamingDetails 14 | * - localizations 15 | * - player 16 | * - processingDetails 17 | * - recordingDetails 18 | * - snippet 19 | * - statistics 20 | * - status 21 | * - suggestions 22 | * - topicDetails 23 | */ 24 | part: string; 25 | fields?: string; 26 | } 27 | 28 | export interface SearchParams extends DefaultParams { 29 | id: string; 30 | hl?: string; 31 | } 32 | 33 | export interface PlaylistParams extends DefaultParams { 34 | playlistId: string; 35 | pageToken?: string; 36 | maxResults?: number; 37 | } 38 | 39 | export interface YoutubeDefaultResponse { 40 | kind: string; 41 | etag: string; 42 | nextPageToken?: string; 43 | prevPageToken?: string; 44 | pageInfo: { 45 | totalResults: number; 46 | resultsPerPage: number; 47 | }; 48 | items: any[]; 49 | } 50 | 51 | export interface YoutubeVideoResponse extends YoutubeDefaultResponse { 52 | kind: 'youtube#videoListResponse'; 53 | items: VideoResource[]; 54 | } 55 | 56 | export interface LiveStreamingDetails { 57 | actualStartTime: DateString; 58 | actualEndTime: DateString; 59 | scheduledStartTime: DateString; 60 | scheduledEndTime: DateString; 61 | concurrentViewers: UnsignedLong; 62 | activeLiveChatId: string; 63 | } 64 | 65 | export interface VideoResource { 66 | kind: 'youtube#video'; 67 | etag: string; 68 | id: VideoId; 69 | snippet: { 70 | publishedAt: DateString; 71 | channelId: ChannelId; 72 | title: string; 73 | description: string; 74 | thumbnails: VideoThumbnail; 75 | channelTitle: string; 76 | tags: string[]; 77 | categoryId: string; 78 | liveBroadcastContent: string; 79 | defaultLanguage: string; 80 | localized: { 81 | title: string; 82 | description: string; 83 | }; 84 | defaultAudioLanguage: string; 85 | }; 86 | contentDetails: Record; 87 | statistics: { 88 | viewCount: UnsignedLong; 89 | likeCount: UnsignedLong; 90 | dislikeCount: UnsignedLong; 91 | favoriteCount: UnsignedLong; 92 | commentCount: UnsignedLong; 93 | }; 94 | player: Record; 95 | topicDetails: Record; 96 | recordingDetails: { 97 | recordingDate: DateString; 98 | }; 99 | fileDetails: Record; 100 | processingDetails: Record; 101 | suggestions: Record; 102 | liveStreamingDetails: LiveStreamingDetails; 103 | localizations: { 104 | [key: string]: { 105 | title: string; 106 | description: string; 107 | }; 108 | }; 109 | } 110 | 111 | export interface YoutubeChannelResponse extends YoutubeDefaultResponse { 112 | kind: 'youtube#channelListResponse'; 113 | items: ChannelResource[]; 114 | } 115 | 116 | export interface ChannelResource { 117 | kind: 'youtube#channel'; 118 | etag: string; 119 | id: ChannelId; 120 | snippet: { 121 | title: string; 122 | description: string; 123 | customUrl: string; 124 | publishedAt: DateString; 125 | thumbnails: ChannelThumbnail; 126 | defaultLanguage: string; 127 | localized: { 128 | title: string; 129 | description: string; 130 | }; 131 | country: string; 132 | }; 133 | contentDetails: { 134 | relatedPlaylists: { 135 | likes: string; 136 | favorites: string; 137 | uploads: string; 138 | }; 139 | }; 140 | statistics: { 141 | viewCount: UnsignedLong; 142 | subscriberCount: UnsignedLong; 143 | hiddenSubscriberCount: boolean; 144 | videoCount: UnsignedLong; 145 | }; 146 | topicDetails: Record; 147 | status: { 148 | privacyStatus: string; 149 | isLinked: boolean; 150 | longUploadsStatus: string; 151 | madeForKids: boolean; 152 | selfDeclaredMadeForKids: boolean; 153 | }; 154 | brandingSettings: Record; 155 | auditDetails: Record; 156 | contentOwnerDetails: Record; 157 | localizations: { 158 | [key: string]: { 159 | title: string; 160 | description: string; 161 | }; 162 | }; 163 | } 164 | 165 | export interface YoutubePlaylistItemsResponse extends YoutubeDefaultResponse { 166 | kind: 'youtube#playlistItemListResponse'; 167 | items: PlaylistItemsResource[]; 168 | } 169 | 170 | export interface PlaylistItemsResource { 171 | kind: 'youtube#playlistItem'; 172 | etag: string; 173 | id: string; 174 | snippet: { 175 | publishedAt: DateString; 176 | channelId: ChannelId; 177 | title: string; 178 | description: string; 179 | thumbnails: VideoThumbnail; 180 | channelTitle: string; 181 | playlistId: string; 182 | position: UnsignedInteger; 183 | resourceId: { 184 | kind: string; 185 | videoId: VideoId; 186 | }; 187 | }; 188 | contentDetails: { 189 | videoId: VideoId; 190 | startAt: string; 191 | endAt: string; 192 | note: string; 193 | videoPublishedAt: DateString; 194 | }; 195 | status: { 196 | privacyStatus: string; 197 | }; 198 | } 199 | 200 | interface ThumbnailData { 201 | url: string; 202 | width: UnsignedInteger; 203 | height: UnsignedInteger; 204 | } 205 | 206 | type ThumbnailResolution = 'default'|'medium'|'high'; 207 | type VideoThumbnail = { [key in ThumbnailResolution|'standard'|'maxres']: ThumbnailData; } 208 | type ChannelThumbnail = { [key in ThumbnailResolution]: ThumbnailData; } 209 | -------------------------------------------------------------------------------- /channels/default/ReACT.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Kirahoshi Uta", 5 | "jp": "綺羅星ウタ" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UCX2Sm8SGWTUOyCizgrzadXA", 9 | "details": { 10 | "twitter": "uta_kirahoshi", 11 | "group": "1st Gen" 12 | } 13 | }, 14 | { 15 | "name": { 16 | "en": "Inuki Matoi", 17 | "jp": "射貫まとい" 18 | }, 19 | "platform_id": "yt", 20 | "channel_id": "UCFrH3bfn7gDeV29wrHHM80g", 21 | "details": { 22 | "twitter": "matoi_inuki", 23 | "group": "1st Gen" 24 | } 25 | }, 26 | { 27 | "name": { 28 | "en": "Koto Miyuri", 29 | "jp": "琴みゆり" 30 | }, 31 | "platform_id": "yt", 32 | "channel_id": "UCBe_jjkUHhVNAj46bukAbJA", 33 | "details": { 34 | "twitter": "miyuri_koto", 35 | "group": "1st Gen" 36 | } 37 | }, 38 | { 39 | "name": { 40 | "en": "Usami Yuno", 41 | "jp": "宇佐美ユノ" 42 | }, 43 | "platform_id": "yt", 44 | "channel_id": "UCJQMHCFjVZOVRYafR6gY04Q", 45 | "details": { 46 | "twitter": "yuno_usami", 47 | "group": "1st Gen" 48 | } 49 | }, 50 | { 51 | "name": { 52 | "en": "Hanabasami Kyo", 53 | "jp": "花鋏キョウ" 54 | }, 55 | "platform_id": "yt", 56 | "channel_id": "UC4OeUf_KfYRrwksschtRYow", 57 | "details": { 58 | "twitter": "Kyo_Hanabasami", 59 | "group": "2nd Gen" 60 | } 61 | }, 62 | { 63 | "name": { 64 | "en": "Shishigami Leona", 65 | "jp": "獅子神レオナ" 66 | }, 67 | "platform_id": "yt", 68 | "channel_id": "UCB1s_IdO-r0nUkY2mXeti-A", 69 | "details": { 70 | "twitter": "LeonaShishigami", 71 | "group": "2nd Gen" 72 | } 73 | }, 74 | { 75 | "name": { 76 | "en": "Mizugame Mia", 77 | "jp": "水瓶ミア" 78 | }, 79 | "platform_id": "yt", 80 | "channel_id": "UCpPuEfqwYbpn7e2jWdQeWew", 81 | "details": { 82 | "twitter": "Mia_Mizugame", 83 | "group": "2nd Gen" 84 | } 85 | }, 86 | { 87 | "name": { 88 | "en": "Yumekawa Kanau", 89 | "jp": "夢川かなう" 90 | }, 91 | "platform_id": "yt", 92 | "channel_id": "UC8jskpQfW9fn2NLK3PdaGdg", 93 | "details": { 94 | "twitter": "Kanau_Yumekawa", 95 | "group": "2nd Gen" 96 | } 97 | }, 98 | { 99 | "name": { 100 | "en": "Amakawa Hano", 101 | "jp": "天川はの" 102 | }, 103 | "platform_id": "yt", 104 | "channel_id": "UCxJ9SJLG7dA00M7VoEe4ltw", 105 | "details": { 106 | "twitter": "sb_hano_", 107 | "group": "Star!Bear" 108 | } 109 | }, 110 | { 111 | "name": { 112 | "en": "Himekuma Ribon", 113 | "jp": "姫熊 りぼん" 114 | }, 115 | "platform_id": "yt", 116 | "channel_id": "UC6HjtF2rHZO8gAsX5FXF-Kg", 117 | "details": { 118 | "twitter": "sb_ribon_", 119 | "group": "Star!Bear" 120 | } 121 | }, 122 | { 123 | "name": { 124 | "en": "Shirane Yuki", 125 | "jp": "白音ゆき" 126 | }, 127 | "platform_id": "yt", 128 | "channel_id": "UCcD_EdnJcOjJMohDn435D3w", 129 | "details": { 130 | "twitter": "Shirane_Yuki", 131 | "group": "Monokuro Ainos" 132 | } 133 | }, 134 | { 135 | "name": { 136 | "en": "Kurone Yomi", 137 | "jp": "黒音よみ" 138 | }, 139 | "platform_id": "yt", 140 | "channel_id": "UCmssHxtaJAxIvje7LTHoKiw", 141 | "details": { 142 | "twitter": "Kurone__Yomi", 143 | "group": "Monokuro Ainos" 144 | } 145 | }, 146 | { 147 | "name": { 148 | "en": "Inumochi Chiroru", 149 | "jp": "犬望チロル" 150 | }, 151 | "platform_id": "yt", 152 | "channel_id": "UC46Wizhr_xgL9Rm9onVkPQQ", 153 | "details": { 154 | "twitter": "tirol_inumoti", 155 | "group": "Maruchiizu" 156 | } 157 | }, 158 | { 159 | "name": { 160 | "en": "Marumochi Tsukimi", 161 | "jp": "丸餅つきみ" 162 | }, 163 | "platform_id": "yt", 164 | "channel_id": "UCTEZGNJDqv-isWSPRsEhnRA", 165 | "details": { 166 | "twitter": "tukimi_marumoti", 167 | "group": "Maruchiizu" 168 | } 169 | }, 170 | { 171 | "name": { 172 | "en": "Izumo Meguru", 173 | "jp": "出雲めぐる" 174 | }, 175 | "platform_id": "yt", 176 | "channel_id": "UCReYPHLeFKv-0wYgX6F2Zhw", 177 | "details": { 178 | "twitter": "meguru_izumo", 179 | "group": "Maruchiizu" 180 | } 181 | }, 182 | { 183 | "name": { 184 | "en": "Qualia Qu", 185 | "jp": "久檻夜くぅ" 186 | }, 187 | "platform_id": "yt", 188 | "channel_id": "UCXtQTtPJedfjqEPysorbsMg", 189 | "details": { 190 | "twitter": "Qu_Qualia", 191 | "group": "Solo 2020 Debuts" 192 | } 193 | }, 194 | { 195 | "name": { 196 | "en": "Kazami Mikan", 197 | "jp": "風海みかん" 198 | }, 199 | "platform_id": "yt", 200 | "channel_id": "UCa5g-Q_NT2COXiSS55bwXVQ", 201 | "details": { 202 | "twitter": "Kazami_Mikan83", 203 | "group": "Solo 2020 Debuts" 204 | } 205 | }, 206 | { 207 | "name": { 208 | "en": "Hekina Airu", 209 | "jp": "碧那アイル" 210 | }, 211 | "platform_id": "yt", 212 | "channel_id": "UCpBCiBjOoZanFhj-49LgcDg", 213 | "details": { 214 | "twitter": "Ail_Hekina", 215 | "group": "Solo 2020 Debuts" 216 | } 217 | }, 218 | { 219 | "name": { 220 | "en": "Minato Minami", 221 | "jp": "湊音みなみ" 222 | }, 223 | "platform_id": "yt", 224 | "channel_id": "UCyTw66xE5YkeTL2YduA69QQ", 225 | "details": { 226 | "twitter": "_MinatoMinami", 227 | "group": "SuiSay/Sugar Stella" 228 | } 229 | }, 230 | { 231 | "name": { 232 | "en": "Kogami Yukari", 233 | "jp": "皇噛ユカリ" 234 | }, 235 | "platform_id": "yt", 236 | "channel_id": "UCt0F29rrrkOU5Vu2gyYgrBA", 237 | "details": { 238 | "twitter": "yukari_kogami", 239 | "group": "METEO/Sugar Stella" 240 | } 241 | }, 242 | { 243 | "name": { 244 | "en": "Yuiga Kohaku", 245 | "jp": "唯牙コハク" 246 | }, 247 | "platform_id": "yt", 248 | "channel_id": "UCf6J4I7NwMXtuIU8YZKAjCA", 249 | "details": { 250 | "twitter": "kohaku_yuiga", 251 | "group": "METEO/Sugar Stella" 252 | } 253 | } 254 | ] -------------------------------------------------------------------------------- /channels/index.ts: -------------------------------------------------------------------------------- 1 | import { config } from 'dotenv'; 2 | config(); 3 | 4 | process.env.DEBUG += ',-db:*'; 5 | import { readdirSync, readFileSync } from 'fs'; 6 | import { createInterface } from 'readline'; 7 | import { MemberObject, MemberProps, PlatformId } from '../database/types/members'; 8 | import { Counter, debug, Members } from '../src/modules'; 9 | import { ChannelId } from '../src/modules/types/youtube'; 10 | import youtubeChannelScraper from './apps/scrapers/youtube-scraper'; 11 | import updateYoutube from './apps/updaters/youtube-updater'; 12 | 13 | if (!process.env.GOOGLE_API_KEY) throw new Error('GOOGLE_API_KEY is undefined!'); 14 | 15 | export function channelManager() { 16 | console.clear(); 17 | console.log( 18 | '---------------------------- Manage Channels ----------------------------\n' + 19 | ' Make sure you\'ve set up the .json files in channels/organizations directory.\n' + 20 | ' Check templates.json to see how to make custom channels, or move the files\n' + 21 | ' from the default directory to the organizations directory.\n' + 22 | '-----------------------------------------------------------------------------\n' + 23 | ' [1] Initialize (Run Everything)\n' + 24 | ' [2] Validate JSON Files\n' + 25 | ' [3] Save + Update\n' + 26 | ' [4] Save Channels\n' + 27 | ' [5] Update Channels\n' + 28 | ' [6] Scrape Channels\n' + 29 | ' [7] Drop Members and Channels Collection\n' + 30 | ' [8] Drop vt-api Database\n' + 31 | ' [9] Exit\n' 32 | ); 33 | const rl = createInterface({ 34 | input: process.stdin, 35 | output: process.stdout 36 | }); 37 | rl.question('Selection: ', async input => { 38 | process.env.DEBUG = process.env.DEBUG.slice(0, -6); 39 | rl.close(); 40 | switch (input) { 41 | default: 42 | return channelManager(); 43 | case '1': 44 | await init(); 45 | break; 46 | case '2': 47 | validateChannels(); 48 | break; 49 | case '3': 50 | case '4': 51 | await Promise.all(saveChannels({}, true)); 52 | if (input === '4') break; 53 | case '5': 54 | await updateChannels(); 55 | break; 56 | case '6': 57 | await scrapeChannels(); 58 | break; 59 | case '7': 60 | await dropCollections(); 61 | break; 62 | case '8': 63 | await dropDatabase(); 64 | break; 65 | case '9': process.exit(); 66 | } 67 | delayEnd(); 68 | }); 69 | } 70 | 71 | const delayEnd = () => setTimeout(() => { 72 | console.log('Press any key to continue: '); 73 | process.stdin.setRawMode(true); 74 | process.stdin.resume(); 75 | process.stdin.on('data', process.exit.bind(process, 0)); 76 | }, 600); 77 | 78 | const logger = debug('channels'); 79 | const ROOT_DIR = 'channels/organizations'; 80 | 81 | type ChannelPlatform = {[key in PlatformId]: T[]}; 82 | type BasicChannelData = [ChannelId, string, PlatformId]; 83 | 84 | function saveChannel(filename: string, dry = false, save = true, async = false) { 85 | const groupName = filename.slice(0, -5); 86 | const channelList: MemberObject[] = JSON.parse(readFileSync(`${ROOT_DIR}/${filename}`, 'utf-8')); 87 | const parseChannel = (channel: MemberObject): any => { channel.organization = groupName; return channel; }; 88 | const parsedChannels: MemberObject[] = channelList.map(parseChannel); 89 | if (dry) return parsedChannels; 90 | if (save) { 91 | const writeOp = Members 92 | .create(parsedChannels) 93 | .then(() => logger.info(`${filename} OK`)) 94 | .catch(err => logger.error(`${filename} CODE: ${err.code}`, err?.keyValue ?? '')); 95 | if (async) return writeOp; 96 | } 97 | return channelList.map((channel): BasicChannelData => [channel.channel_id, groupName, channel.platform_id]); 98 | } 99 | 100 | function checkChannels(channelList: T[]): T[]|never { 101 | if (!channelList.length) { 102 | throw new Error('No channels found.'); 103 | } else { return channelList; } 104 | } 105 | 106 | function saveChannels( 107 | options: { dry?: T1; save?: boolean; } = { dry: false, save: true }, 108 | async: T2 = false 109 | ): T2 extends true ? Promise[] : T1 extends true ? MemberObject[] : BasicChannelData[] { 110 | return checkChannels(readdirSync(ROOT_DIR) 111 | .filter(file => file.endsWith('.json')) 112 | .flatMap((group): any => saveChannel(group, options.dry, options.save, async)) 113 | ) as T2 extends true ? Promise[] : T1 extends true ? MemberObject[] : BasicChannelData[]; 114 | } 115 | 116 | function validateChannels() { 117 | try { 118 | const channels = saveChannels({ dry: true }); 119 | if (!channels.length) { 120 | logger.error(new Error('No channel jsons found.')); 121 | return; 122 | } 123 | logger.info(`Found ${channels.length} channels.`); 124 | let errorCount = 0; 125 | for (let i = channels.length; i--;) { 126 | const err = new Members(channels[i]).validateSync(); 127 | if (!err) continue; 128 | logger.error({ error: err.message, channel: channels[i] }); 129 | errorCount++; 130 | } 131 | if (errorCount) { 132 | logger.info(`Failed to validate ${errorCount} channels.`); 133 | return false; 134 | } else { 135 | logger.info('All channels validated successfully.'); 136 | return true; 137 | } 138 | } catch(err) { 139 | logger.error(err); 140 | return false; 141 | } 142 | } 143 | 144 | async function scrapeChannels() { 145 | const channelList = await Members 146 | .find({ crawled_at: { $exists: false } }) 147 | .then(groupMemberObject); 148 | if (!Object.values(channelList).flat().length) { 149 | logger.error(new Error('No saved members found.')); 150 | return; 151 | } 152 | const scraper = { 153 | RESULTS: { OK: [], FAIL: [], videoCount: 0 }, 154 | async youtube(channels: MemberObject[]) { 155 | for (let i = channels.length; i--;) { 156 | const currentChannel = channels[i]; 157 | const [STATUS, VIDEO_COUNT] = await youtubeChannelScraper(currentChannel); 158 | this.RESULTS[STATUS].push(currentChannel.channel_id); 159 | this.RESULTS.videoCount += VIDEO_COUNT; 160 | } 161 | }, 162 | // async bilibili(channels: MemberObject[]) { 163 | // }, 164 | // async twitchtv(channels: MemberObject[]) { 165 | // } 166 | }; 167 | await Promise.all([ 168 | scraper.youtube(channelList.yt), 169 | // scraper.bilibili(channelList.bb), 170 | // scraper.twitchtv(channelList.tt) 171 | ]); 172 | logger.info(scraper.RESULTS); 173 | } 174 | 175 | async function updateChannels() { 176 | const CHANNEL_PLATFORMS = await Members.find() 177 | .then(groupMemberObject) as ChannelPlatform; 178 | await Promise.all([ 179 | updateYoutube(CHANNEL_PLATFORMS.yt), 180 | // @TODO: Implement bb and ttv apis 181 | // updateBilibili(CHANNEL_PLATFORMS.bb), 182 | // updateTwitch(CHANNEL_PLATFORMS.tt) 183 | ]); 184 | } 185 | 186 | async function dropCollections() { 187 | const { connection } = await require('mongoose'); 188 | logger.info('Dropping channel related collections...'); 189 | await Promise.all([ 190 | connection.dropCollection('members'), 191 | connection.dropCollection('channels'), 192 | Counter.deleteOne({ _id: 'member_id' }) 193 | ]); 194 | logger.info('Dropped members and channels collection.'); 195 | } 196 | 197 | async function dropDatabase() { 198 | const { connection } = await require('mongoose'); 199 | logger.info('Dropping vt-api database...'); 200 | await connection.dropDatabase(); 201 | logger.info('Dropped vt-api database.'); 202 | } 203 | 204 | function groupMemberObject(memberList: MemberObject[]): ChannelPlatform { 205 | return memberList.reduce( 206 | (platforms, channel) => { 207 | platforms[channel.platform_id].push(channel); 208 | return platforms; 209 | }, { yt: [], bb: [], tt: [] } 210 | ); 211 | } 212 | 213 | export async function init(script = false) { 214 | if(!validateChannels()) return; 215 | await Promise.all(saveChannels({}, true)); 216 | await updateChannels(); 217 | await scrapeChannels(); 218 | if (script) delayEnd(); 219 | } 220 | -------------------------------------------------------------------------------- /channels/default/upd8.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Shibuya HAL", 5 | "jp": "渋谷ハル" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UChLfthKoUV502J7gU9STArg", 9 | "details": { 10 | "twitter": "ShibuyaHAL" 11 | } 12 | }, 13 | { 14 | "name": { 15 | "en": "Shibuya HAL", 16 | "jp": "渋谷ハル" 17 | }, 18 | "platform_id": "bb", 19 | "channel_id": "shibuyahal", 20 | "details": { 21 | "twitter": "ShibuyaHAL" 22 | } 23 | }, 24 | { 25 | "name": { 26 | "en": "Omega Sisters", 27 | "jp": "おめがシスターズ" 28 | }, 29 | "platform_id": "yt", 30 | "channel_id": "UCNjTjd2-PMC8Oo_-dCEss7A", 31 | "details": { 32 | "twitter": "omesis_rio" 33 | } 34 | }, 35 | { 36 | "name": { 37 | "en": "Kimino Miya", 38 | "jp": "キミノミヤ" 39 | }, 40 | "platform_id": "yt", 41 | "channel_id": "UCg8TPE5nLLOqlNnBivmQwww", 42 | "details": { 43 | "twitter": "MIYA_KIMINO" 44 | } 45 | }, 46 | { 47 | "name": { 48 | "en": "Choueki Tarou", 49 | "jp": "懲役太郎" 50 | }, 51 | "platform_id": "yt", 52 | "channel_id": "UC0e6zo8oX6ISSBjPaSZsjeA", 53 | "details": { 54 | "twitter": "choueki_tarou" 55 | } 56 | }, 57 | { 58 | "name": { 59 | "en": "Yuugiri", 60 | "jp": "由宇霧" 61 | }, 62 | "platform_id": "yt", 63 | "channel_id": "UCMxKcUjeTEcgHmC9Zzn3R4w", 64 | "details": { 65 | "twitter": "oiran_yugiri" 66 | } 67 | }, 68 | { 69 | "name": { 70 | "en": "Kerin", 71 | "jp": "ケリン" 72 | }, 73 | "platform_id": "yt", 74 | "channel_id": "UCeAfiVvEuyICYJW-f3GnQjQ", 75 | "details": { 76 | "twitter": "Kerin_Vtuber" 77 | } 78 | }, 79 | { 80 | "name": { 81 | "en": "Utai Meika", 82 | "jp": "歌衣メイカ" 83 | }, 84 | "platform_id": "yt", 85 | "channel_id": "UC7-N7MvN5muVIHqyQx9LFbA", 86 | "details": { 87 | "twitter": "makeagames2018" 88 | } 89 | }, 90 | { 91 | "name": { 92 | "en": "Kashiko Mari", 93 | "jp": "かしこまり" 94 | }, 95 | "platform_id": "yt", 96 | "channel_id": "UCfiK42sBHraMBK6eNWtsy7A", 97 | "details": { 98 | "twitter": "kashikomari_ch" 99 | } 100 | }, 101 | { 102 | "name": { 103 | "en": "Tomari Mari", 104 | "jp": "兎鞠まり" 105 | }, 106 | "platform_id": "yt", 107 | "channel_id": "UCkPIfBOLoO0hVPG-tI2YeGg", 108 | "details": { 109 | "twitter": "tomari_mari" 110 | } 111 | }, 112 | { 113 | "name": { 114 | "en": "MonsterZ MATE" 115 | }, 116 | "platform_id": "yt", 117 | "channel_id": "UCDG8K9g6qo8gX4jCjsqUVzA", 118 | "details": { 119 | "twitter": "monsterzmate" 120 | } 121 | }, 122 | { 123 | "name": { 124 | "en": "Nora Cat", 125 | "jp": "のらきゃっと" 126 | }, 127 | "platform_id": "yt", 128 | "channel_id": "UC3iwL9Yz8LcKkJsnLPevOTQ", 129 | "details": { 130 | "twitter": "VR_Girl_NoraCat" 131 | } 132 | }, 133 | { 134 | "name": { 135 | "en": "Fukuya Master", 136 | "jp": "ふくやマスター" 137 | }, 138 | "platform_id": "yt", 139 | "channel_id": "UCXzcu59Cbu-hkpNNXnNiKDg", 140 | "details": { 141 | "twitter": "lesupo_ru321id" 142 | } 143 | }, 144 | { 145 | "name": { 146 | "en": "Mao Magurona", 147 | "jp": "魔王マグロナ" 148 | }, 149 | "platform_id": "yt", 150 | "channel_id": "UCPf-EnX70UM7jqjKwhDmS8g", 151 | "details": { 152 | "twitter": "ukyo_rst" 153 | } 154 | }, 155 | { 156 | "name": { 157 | "en": "Coco Tsuki", 158 | "jp": "ココツキ" 159 | }, 160 | "platform_id": "yt", 161 | "channel_id": "UCH1B9AR8sfWJVf9zcu9rbGg", 162 | "details": { 163 | "twitter": "CocoTsuki_Mane" 164 | } 165 | }, 166 | { 167 | "name": { 168 | "en": "Utagoe Housoubu", 169 | "jp": "ウタゴエ放送部" 170 | }, 171 | "platform_id": "yt", 172 | "channel_id": "UC84RYorF7UcJzt_iAYXBisA", 173 | "details": { 174 | "twitter": "lara_songs" 175 | } 176 | }, 177 | { 178 | "name": { 179 | "en": "Hibiki Ao", 180 | "jp": "響木アオ" 181 | }, 182 | "platform_id": "yt", 183 | "channel_id": "UCNwo7eikmX5HPs7NedWVBgw", 184 | "details": { 185 | "twitter": "hibiki_ao" 186 | } 187 | }, 188 | { 189 | "name": { 190 | "en": "Nijikawa Laki", 191 | "jp": "虹河ラキ" 192 | }, 193 | "platform_id": "yt", 194 | "channel_id": "UCp77Qho-YHhnklRr-DkjiOw", 195 | "details": { 196 | "twitter": "nijikawa_laki" 197 | } 198 | }, 199 | { 200 | "name": { 201 | "en": "Kasukabe Tsukushi", 202 | "jp": "春日部つくし" 203 | }, 204 | "platform_id": "yt", 205 | "channel_id": "UCHccxX2p_9DB_HPnXSb2omw", 206 | "details": { 207 | "twitter": "kasukaBe_nyoki" 208 | } 209 | }, 210 | { 211 | "name": { 212 | "en": "Sakurazuki Kanon", 213 | "jp": "桜月花音" 214 | }, 215 | "platform_id": "yt", 216 | "channel_id": "UC3-jXrZXv-PQshpaTQpicPQ", 217 | "details": { 218 | "twitter": "SakurazukiKanon" 219 | } 220 | }, 221 | { 222 | "name": { 223 | "en": "Yoruno Neon", 224 | "jp": "夜乃ネオン" 225 | }, 226 | "platform_id": "yt", 227 | "channel_id": "UCLQIMDHNE0ZGAiHIdMbq7dw", 228 | "details": { 229 | "twitter": "NeonNight_exe" 230 | } 231 | }, 232 | { 233 | "name": { 234 | "en": "Virtual Neko", 235 | "jp": "バーチャルねこ" 236 | }, 237 | "platform_id": "yt", 238 | "channel_id": "UCKDHdDpoE9lurDuyAGAuNnw", 239 | "details": { 240 | "twitter": "neko_vtuber" 241 | } 242 | }, 243 | { 244 | "name": { 245 | "en": "Icotsu" 246 | }, 247 | "platform_id": "yt", 248 | "channel_id": "UCGFD_8TRHhlpjfqGhLUSk4g", 249 | "details": { 250 | "twitter": "Vtuber_Icotsu" 251 | } 252 | }, 253 | { 254 | "name": { 255 | "en": "Shiotenshi Rieru", 256 | "jp": "塩天使リエル" 257 | }, 258 | "platform_id": "yt", 259 | "channel_id": "UCE5rWcDxLPaaUFWOxzJFfNg", 260 | "details": { 261 | "twitter": "VAngelf_Riel" 262 | } 263 | }, 264 | { 265 | "name": { 266 | "en": "KON", 267 | "jp": "空" 268 | }, 269 | "platform_id": "yt", 270 | "channel_id": "UCyZZMKRn-mUEkPzaqa9b6bg", 271 | "details": { 272 | "twitter": "kwa_kon" 273 | } 274 | }, 275 | { 276 | "name": { 277 | "en": "Deratohadou", 278 | "jp": "デラとハドウ" 279 | }, 280 | "platform_id": "yt", 281 | "channel_id": "UC0QzO6nK1cEDZvJSt2769Zw", 282 | "details": { 283 | "twitter": "harddeluxe" 284 | } 285 | }, 286 | { 287 | "name": { 288 | "en": "Moscow Mule", 289 | "jp": "モスコミュール" 290 | }, 291 | "platform_id": "yt", 292 | "channel_id": "UCX7WYPXASkYUm2GyomLllIg", 293 | "details": { 294 | "twitter": "Moscow_ojisan" 295 | } 296 | }, 297 | { 298 | "name": { 299 | "en": "upd8" 300 | }, 301 | "platform_id": "yt", 302 | "channel_id": "UCBsrAA8Z_6Ot3my0vM8dlvg", 303 | "details": { 304 | "twitter": "project_upd8" 305 | } 306 | }, 307 | { 308 | "name": { 309 | "en": "Hanagasa Iria", 310 | "jp": "花笠イリヤ" 311 | }, 312 | "platform_id": "yt", 313 | "channel_id": "UCigpSsgPbdCxXtbfGksFIYQ", 314 | "details": { 315 | "twitter": "ilya_cheriko" 316 | } 317 | }, 318 | { 319 | "name": { 320 | "en": "Engine Kazumi", 321 | "jp": "エンジンかずみ" 322 | }, 323 | "platform_id": "yt", 324 | "channel_id": "UCZk6mxfTpjuuxgqxuFBJO_g", 325 | "details": { 326 | "twitter": "Engine_Kazumi" 327 | } 328 | }, 329 | { 330 | "name": { 331 | "en": "Nishikiyama Mutaka", 332 | "jp": "錦山夢鷹" 333 | }, 334 | "platform_id": "yt", 335 | "channel_id": "UCil-nSDmDTgV9n4jjb95znw", 336 | "details": { 337 | "twitter": "mutaka_cheriko" 338 | } 339 | } 340 | ] -------------------------------------------------------------------------------- /channels/default/Independents.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": { 4 | "en": "Shigure Ui", 5 | "jp": "しぐれうい" 6 | }, 7 | "platform_id": "yt", 8 | "channel_id": "UCt30jJgChL8qeT9VPadidSw", 9 | "details": { 10 | "twitter": "ui_shig", 11 | "pixiv": "ui_shig" 12 | } 13 | }, 14 | { 15 | "name": { 16 | "en": "Kagura Nana", 17 | "jp": "カグラナナ", 18 | "cn": "神楽七奈" 19 | }, 20 | "platform_id": "yt", 21 | "channel_id": "UCbfv8uuUXt3RSJGEwxny5Rw", 22 | "details": { 23 | "twitter": "nana_kaguraaa" 24 | } 25 | }, 26 | { 27 | "name": { 28 | "en": "Kagura Nana", 29 | "jp": "カグラナナ", 30 | "cn": "神楽七奈" 31 | }, 32 | "platform_id": "bb", 33 | "channel_id": "386900246", 34 | "details": { 35 | "twitter": "nana_kaguraaa" 36 | } 37 | }, 38 | { 39 | "name": { 40 | "en": "Chimugi", 41 | "jp": "ちむぎ" 42 | }, 43 | "platform_id": "yt", 44 | "channel_id": "UClxrt4pRmaqQnSlDii3EJBQ", 45 | "details": { 46 | "twitter": "chimugixxx" 47 | } 48 | }, 49 | { 50 | "name": { 51 | "en": "Chimugi", 52 | "jp": "ちむぎ" 53 | }, 54 | "platform_id": "tt", 55 | "channel_id": "chimugi", 56 | "details": { 57 | "twitter": "chimugixxx" 58 | } 59 | }, 60 | { 61 | "name": { 62 | "en": "Hinatsu Nono", 63 | "jp": "陽夏のの" 64 | }, 65 | "platform_id": "yt", 66 | "channel_id": "UCPeF0V8dzSUYHwgLjvC86PQ", 67 | "details": { 68 | "twitter": "HinatsuNono" 69 | } 70 | }, 71 | { 72 | "name": { 73 | "en": "Kuroki Natsume", 74 | "jp": "黯希ナツメ" 75 | }, 76 | "platform_id": "yt", 77 | "channel_id": "UCyO7wyh07pYJaZk51OV75bA", 78 | "details": { 79 | "twitter": "96__neko" 80 | } 81 | }, 82 | { 83 | "name": { 84 | "en": "Kagura Mea", 85 | "jp": "神楽めあ" 86 | }, 87 | "platform_id": "yt", 88 | "channel_id": "UCWCc8tO-uUl_7SJXIKJACMw", 89 | "details": { 90 | "twitter": "KaguraMea_VoV" 91 | } 92 | }, 93 | { 94 | "name": { 95 | "en": "Kagura Mea", 96 | "jp": "神楽めあ" 97 | }, 98 | "platform_id": "bb", 99 | "channel_id": "349991143", 100 | "details": { 101 | "twitter": "KaguraMea_VoV" 102 | } 103 | }, 104 | { 105 | "name": { 106 | "en": "Hanazono Serena", 107 | "jp": "花園セレナ", 108 | "cn": "花园Serena" 109 | }, 110 | "platform_id": "yt", 111 | "channel_id": "UCRXBTd80F5IIWWY4HatJ5Ug", 112 | "details": { 113 | "twitter": "hanazono_serena" 114 | } 115 | }, 116 | { 117 | "name": { 118 | "en": "Hanazono Serena", 119 | "jp": "花園セレナ", 120 | "cn": "花园Serena" 121 | }, 122 | "platform_id": "bb", 123 | "channel_id": "380829248", 124 | "details": { 125 | "twitter": "hanazono_serena" 126 | } 127 | }, 128 | { 129 | "name": { 130 | "en": "Kazami Yuika", 131 | "jp": "風見唯花" 132 | }, 133 | "platform_id": "yt", 134 | "channel_id": "UC5Oe-wG53cRHkpV3aHXfLUQ", 135 | "details": { 136 | "twitter": "Yuika_Kazami" 137 | } 138 | }, 139 | { 140 | "name": { 141 | "en": "Hourei Tenten", 142 | "jp": "鳳玲天々", 143 | "cn": "凤玲天天" 144 | }, 145 | "platform_id": "yt", 146 | "channel_id": "UCw-jEa3_788VkvM2zHzrDnw", 147 | "details": { 148 | "twitter": "hourei_1010" 149 | } 150 | }, 151 | { 152 | "name": { 153 | "en": "Hourei Tenten", 154 | "jp": "鳳玲天々", 155 | "cn": "凤玲天天" 156 | }, 157 | "platform_id": "bb", 158 | "channel_id": "623441609", 159 | "details": { 160 | "twitter": "hourei_1010" 161 | } 162 | }, 163 | { 164 | "name": { 165 | "en": "Hanamori Healthy", 166 | "jp": "花守へるし" 167 | }, 168 | "platform_id": "yt", 169 | "channel_id": "UCSH4t_nhsNIoxPza4ooYqaA", 170 | "details": { 171 | "twitter": "hana_healthy" 172 | } 173 | }, 174 | { 175 | "name": { 176 | "en": "Nanase Subaru", 177 | "jp": "七瀬すばる" 178 | }, 179 | "platform_id": "yt", 180 | "channel_id": "UCji4kFR4n5TnEpqga0SHxlQ", 181 | "details": { 182 | "twitter": "nanase_subaru_" 183 | } 184 | }, 185 | { 186 | "name": { 187 | "en": "Maru Iwaka", 188 | "jp": "マールいわっか" 189 | }, 190 | "platform_id": "yt", 191 | "channel_id": "UC_9nmotxi7FAfiJpo1RKy8A", 192 | "details": { 193 | "twitter": "maaru_witch" 194 | } 195 | }, 196 | { 197 | "name": { 198 | "en": "Kotonoha Yukino", 199 | "jp": "琴乃葉雪乃" 200 | }, 201 | "platform_id": "yt", 202 | "channel_id": "UCMAc88lqzqGV0uxgw9JDj0w", 203 | "details": { 204 | "twitter": "kotonoha_yukino" 205 | } 206 | }, 207 | { 208 | "name": { 209 | "en": "Kurumi", 210 | "jp": "くるみ" 211 | }, 212 | "platform_id": "yt", 213 | "channel_id": "UCBJFtEEDnCpz8koPH-nLWUA", 214 | "details": { 215 | "twitter": "kurumi_UoxoU" 216 | } 217 | }, 218 | { 219 | "name": { 220 | "en": "Nanahira", 221 | "jp": "ななひら" 222 | }, 223 | "platform_id": "yt", 224 | "channel_id": "UC_fYA9QRK-aJnFTgvR_4zug", 225 | "details": { 226 | "twitter": "nanahira" 227 | } 228 | }, 229 | { 230 | "name": { 231 | "en": "Yururi Megu", 232 | "jp": "ゆるりめぐ" 233 | }, 234 | "platform_id": "yt", 235 | "channel_id": "UC01gb86Qdlkh23Nqk3A1OLQ", 236 | "details": { 237 | "twitter": "yururimegu" 238 | } 239 | }, 240 | { 241 | "name": { 242 | "en": "Rei Kira", 243 | "jp": "れいきら" 244 | }, 245 | "platform_id": "yt", 246 | "channel_id": "UCSFaf99eT5T4wNN1O2Z3NZg", 247 | "details": { 248 | "twitter": "reikira_yutube" 249 | } 250 | }, 251 | { 252 | "name": { 253 | "en": "Lyrica", 254 | "jp": "リリカ" 255 | }, 256 | "platform_id": "yt", 257 | "channel_id": "UCc5H1WmEARIj2R16mvut0ZA", 258 | "details": { 259 | "twitter": "lyrica_ch" 260 | } 261 | }, 262 | { 263 | "name": { 264 | "en": "Lyrica", 265 | "jp": "リリカ" 266 | }, 267 | "platform_id": "tt", 268 | "channel_id": "lyrica_ch", 269 | "details": { 270 | "twitter": "lyrica_ch" 271 | } 272 | }, 273 | { 274 | "name": { 275 | "en": "Nijino Mahoro", 276 | "jp": "虹乃まほろ" 277 | }, 278 | "platform_id": "yt", 279 | "channel_id": "UCtCXaX0zEvYSgdkp-Po24TQ", 280 | "details": { 281 | "twitter": "nijinomahoro" 282 | } 283 | }, 284 | { 285 | "name": { 286 | "en": "Kawashima Himawari", 287 | "jp": "夏恋ひまわり" 288 | }, 289 | "platform_id": "yt", 290 | "channel_id": "UCPkhI1WcLilaKu44B5p0jZg", 291 | "details": { 292 | "twitter": "HIMA_ASMR" 293 | } 294 | }, 295 | { 296 | "name": { 297 | "en": "Choko", 298 | "jp": "ちょこ" 299 | }, 300 | "platform_id": "yt", 301 | "channel_id": "UCoupL81t75_Z4XRKzOSHLbg", 302 | "details": { 303 | "twitter": "cho_v_ko" 304 | } 305 | }, 306 | { 307 | "name": { 308 | "en": "Mameko", 309 | "jp": "まめこ" 310 | }, 311 | "platform_id": "yt", 312 | "channel_id": "UCxI2sKOL2JPX0SQwwKZFafQ", 313 | "details": { 314 | "twitter": "munimuni_mameko" 315 | } 316 | }, 317 | { 318 | "name": { 319 | "en": "Takakura Muki", 320 | "jp": "高倉むき" 321 | }, 322 | "platform_id": "yt", 323 | "channel_id": "UCEYU-OEI9FnbeC5hna_cbhw", 324 | "details": { 325 | "twitter": "takamukikurage" 326 | } 327 | }, 328 | { 329 | "name": { 330 | "en": "Chiem", 331 | "jp": "チエム" 332 | }, 333 | "platform_id": "yt", 334 | "channel_id": "UCK-J76mePKSWV4Sl6zV1Eiw", 335 | "details": { 336 | "twitter": "_Chiem_" 337 | } 338 | }, 339 | { 340 | "name": { 341 | "en": "Otome Oto", 342 | "jp": "乙女おと" 343 | }, 344 | "platform_id": "yt", 345 | "channel_id": "UCvEX2UICvFAa_T6pqizC20g", 346 | "details": { 347 | "twitter": "0tome0to" 348 | } 349 | }, 350 | { 351 | "name": { 352 | "en": "Otome Oto", 353 | "jp": "乙女おと" 354 | }, 355 | "platform_id": "bb", 356 | "channel_id": "406805563", 357 | "details": { 358 | "twitter": "0tome0to" 359 | } 360 | }, 361 | { 362 | "name": { 363 | "en": "Amaori Hikaru", 364 | "jp": "天居ひかる" 365 | }, 366 | "platform_id": "yt", 367 | "channel_id": "UCORH-VrhMk_S24HzI0jSQEA", 368 | "details": { 369 | "twitter": "HikaruAmaori" 370 | } 371 | }, 372 | { 373 | "name": { 374 | "en": "Tetra", 375 | "jp": "テトラ" 376 | }, 377 | "platform_id": "yt", 378 | "channel_id": "UC5u-VD8k8i8xneMtnWet_9Q", 379 | "details": { 380 | "twitter": "Tetra_VTuber" 381 | } 382 | }, 383 | { 384 | "name": { 385 | "en": "Korone Pochi", 386 | "jp": "ころね ぽち" 387 | }, 388 | "platform_id": "yt", 389 | "channel_id": "UCv2byPgvl60Mh1vQBnXuksA", 390 | "details": { 391 | "twitter": "virtualcat_poch" 392 | } 393 | }, 394 | { 395 | "name": { 396 | "en": "Alice White", 397 | "jp": "アリス・ホワイト" 398 | }, 399 | "platform_id": "yt", 400 | "channel_id": "UC3M6qo4pfbnqDJGEREpf-RA", 401 | "details": { 402 | "twitter": "alice_white_V" 403 | } 404 | }, 405 | { 406 | "name": { 407 | "en": "Fumigami Nanana", 408 | "jp": "風紙七鳴" 409 | }, 410 | "platform_id": "yt", 411 | "channel_id": "UCOozmdnOhOrk3mojTazi-TQ", 412 | "details": { 413 | "twitter": "fugaminanana" 414 | } 415 | }, 416 | { 417 | "name": { 418 | "en": "Ouma Yuu", 419 | "jp": "おうまゆう" 420 | }, 421 | "platform_id": "yt", 422 | "channel_id": "UCw8bMzcXgX4SAQPraOJIVgA", 423 | "details": { 424 | "twitter": "oumayuu" 425 | } 426 | }, 427 | { 428 | "name": { 429 | "en": "Iida Pochi", 430 | "jp": "飯田ぽち" 431 | }, 432 | "platform_id": "yt", 433 | "channel_id": "UC22BVlBsZc6ta3Dqz75NU6Q", 434 | "details": { 435 | "twitter": "lizhi3" 436 | } 437 | }, 438 | { 439 | "name": { 440 | "en": "Yukimaru Sen", 441 | "jp": "雪丸仟" 442 | }, 443 | "platform_id": "yt", 444 | "channel_id": "UCsVUXbx_h0YEvfH41Zwa_5w", 445 | "details": { 446 | "twitter": "yukimarusen" 447 | } 448 | }, 449 | { 450 | "name": { 451 | "en": "Ashieda Lenri", 452 | "jp": "芦枝レンリ" 453 | }, 454 | "platform_id": "yt", 455 | "channel_id": "UCOrjRyXJpMzfQFLWuWzGwlg", 456 | "details": { 457 | "twitter": "" 458 | } 459 | }, 460 | { 461 | "name": { 462 | "en": "Natsume Ulta", 463 | "jp": "夏梅ウルタ" 464 | }, 465 | "platform_id": "yt", 466 | "channel_id": "UCghSSXNbZRmPAv3UaS8g21A", 467 | "details": { 468 | "twitter": "natsume_ulta" 469 | } 470 | }, 471 | { 472 | "name": { 473 | "en": "Annin Miru", 474 | "jp": "杏仁ミル" 475 | }, 476 | "platform_id": "yt", 477 | "channel_id": "UCFahBR2wixu0xOex84bXFvg", 478 | "details": { 479 | "twitter": "AnninMirudayo" 480 | } 481 | }, 482 | { 483 | "name": { 484 | "en": "Shiraishi Yukino", 485 | "jp": "白石ゆきの" 486 | }, 487 | "platform_id": "yt", 488 | "channel_id": "UCoFEbNyIRUS26-hf2AucwFw", 489 | "details": { 490 | "twitter": "shiraishiyukin0" 491 | } 492 | }, 493 | { 494 | "name": { 495 | "en": "Shiraishi Yukino", 496 | "jp": "白石ゆきの" 497 | }, 498 | "platform_id": "tt", 499 | "channel_id": "shiraishiyukin0", 500 | "details": { 501 | "twitter": "shiraishiyukin0" 502 | } 503 | }, 504 | { 505 | "name": { 506 | "en": "Mashiro", 507 | "jp": "ましろ" 508 | }, 509 | "platform_id": "yt", 510 | "channel_id": "UClf0kZBKcBVKVOAjGEGbQ-A" 511 | }, 512 | { 513 | "name": { 514 | "en": "Sephira Su", 515 | "jp": "セフィラ・スゥ" 516 | }, 517 | "platform_id": "yt", 518 | "channel_id": "UCHKJMmZJjLpkjZDPp_miSzg", 519 | "details": { 520 | "twitter": "SephiraSu" 521 | } 522 | }, 523 | { 524 | "name": { 525 | "en": "Asano Ninja Sisters", 526 | "jp": "朝ノ, 朝ノ姉妹ぷろじぇくと" 527 | }, 528 | "platform_id": "yt", 529 | "channel_id": "UCODNLyn3L83wEmC0DLL0cxA", 530 | "details": { 531 | "twitter_1": "asanoruri", 532 | "twitter_2": "asanoakane", 533 | "twitter_3": "asanohikarihaa1" 534 | } 535 | }, 536 | { 537 | "name": { 538 | "en": "Mia Tami", 539 | "jp": "みあたみ" 540 | }, 541 | "platform_id": "yt", 542 | "channel_id": "UC7yqc24BjJwi3PoqhXrx6og", 543 | "details": { 544 | "twitter_1": "narusemia", 545 | "twitter_2": "tammy_now" 546 | } 547 | }, 548 | { 549 | "name": { 550 | "en": "Anezaki Yukimi", 551 | "jp": "姉崎ユキミ" 552 | }, 553 | "platform_id": "yt", 554 | "channel_id": "UCwXOUuxUxCJ1yD0JfogUvMg", 555 | "details": { 556 | "twitter": "az_yukimi" 557 | } 558 | }, 559 | { 560 | "name": { 561 | "en": "Mia Runis", 562 | "jp": "ミア ルーニス" 563 | }, 564 | "platform_id": "yt", 565 | "channel_id": "UCupCAZz1l52vV8m-dvaoBVQ", 566 | "details": { 567 | "twitter": "MiaRunis" 568 | } 569 | }, 570 | { 571 | "name": { 572 | "en": "Asahina Meiro", 573 | "jp": "朝日奈めいろ" 574 | }, 575 | "platform_id": "yt", 576 | "channel_id": "UCFUZ8RD8cYLnOotAPf0qLDg", 577 | "details": { 578 | "twitter": "meiro_emoechi" 579 | } 580 | }, 581 | { 582 | "name": { 583 | "en": "Amatsuka Uto", 584 | "jp": "天使うと" 585 | }, 586 | "platform_id": "yt", 587 | "channel_id": "UCdYR5Oyz8Q4g0ZmB4PkTD7g", 588 | "details": { 589 | "twitter": "amatsukauto" 590 | } 591 | }, 592 | { 593 | "name": { 594 | "en": "Figaro", 595 | "jp": "ふぃがろ" 596 | }, 597 | "platform_id": "yt", 598 | "channel_id": "UC7CtvN04ublW4LGTPksG6mg", 599 | "details": { 600 | "twitter": "figaro_qpt" 601 | } 602 | }, 603 | { 604 | "name": { 605 | "en": "Dizm", 606 | "jp": "ディズム" 607 | }, 608 | "platform_id": "yt", 609 | "channel_id": "UCqJ6QFcqZziMnMLr3L3OWAw", 610 | "details": { 611 | "twitter": "DizmKDC" 612 | } 613 | }, 614 | { 615 | "name": { 616 | "en": "Futakuchi Mana", 617 | "jp": "二口魔菜" 618 | }, 619 | "platform_id": "yt", 620 | "channel_id": "UCqGtqSn0NiOCottKpYwBc4w", 621 | "details": { 622 | "twitter": "futakuchimana" 623 | } 624 | }, 625 | { 626 | "name": { 627 | "en": "Sakura Aoi", 628 | "jp": "桜あおい" 629 | }, 630 | "platform_id": "yt", 631 | "channel_id": "UCbA74nSnIScrPLosct0ZjEA", 632 | "details": { 633 | "twitter": "aooooooi_san" 634 | } 635 | }, 636 | { 637 | "name": { 638 | "en": "Hoshino Meguri", 639 | "jp": "星乃めぐり" 640 | }, 641 | "platform_id": "yt", 642 | "channel_id": "UC0nasgeLGQZYGQdfhu6b-SQ", 643 | "details": { 644 | "twitter": "hoshi_no_meguri" 645 | } 646 | }, 647 | { 648 | "name": { 649 | "en": "Aoi Nabi", 650 | "jp": "蒼彩なび" 651 | }, 652 | "platform_id": "yt", 653 | "channel_id": "UCzKkwB84Y0ql0EvyOWRSkEw", 654 | "details": { 655 | "twitter": "nab0i" 656 | } 657 | }, 658 | { 659 | "name": { 660 | "en": "Lucastre Mavia", 661 | "jp": "ルカスター・マビア" 662 | }, 663 | "platform_id": "yt", 664 | "channel_id": "UC703uUVz3wnbpjVcaTKUdmw", 665 | "details": { 666 | "twitter": "maviavtuber" 667 | } 668 | }, 669 | { 670 | "name": { 671 | "en": "Athena Bambina" 672 | }, 673 | "platform_id": "yt", 674 | "channel_id": "UC987QALs1QENtJyZ1G8EgbQ", 675 | "details": { 676 | "twitter": "" 677 | } 678 | }, 679 | { 680 | "name": { 681 | "en": "Charlotte Van Halen" 682 | }, 683 | "platform_id": "yt", 684 | "channel_id": "UC8Cq5WAkqOVrFrTmUjXckyg", 685 | "details": { 686 | "twitter": "lottevanhalen" 687 | } 688 | }, 689 | { 690 | "name": { 691 | "en": "Kannagi Kurama", 692 | "jp": "神凪くらま" 693 | }, 694 | "platform_id": "yt", 695 | "channel_id": "UCuDIxB9u2VAYp9XLMGVurDg", 696 | "details": { 697 | "twitter": "kannagikurama" 698 | } 699 | }, 700 | { 701 | "name": { 702 | "en": "Gokigen Naname", 703 | "jp": "ごきげんななめ" 704 | }, 705 | "platform_id": "yt", 706 | "channel_id": "UCAE1yQeCWiv1eSxAmzpzD4Q", 707 | "details": { 708 | "twitter": "Naname421" 709 | } 710 | }, 711 | { 712 | "name": { 713 | "en": "Tateno Ito", 714 | "jp": "館乃いと" 715 | }, 716 | "platform_id": "yt", 717 | "channel_id": "UCD8i-h5iqOaysEkDnF1CQ_A", 718 | "details": { 719 | "twitter": "tateno_ito_ch" 720 | } 721 | }, 722 | { 723 | "name": { 724 | "en": "Maisaki Berry", 725 | "jp": "苺咲べりぃ" 726 | }, 727 | "platform_id": "yt", 728 | "channel_id": "UC7A7bGRVdIwo93nqnA3x-OQ", 729 | "details": { 730 | "twitter": "MaisakiBerry" 731 | } 732 | }, 733 | { 734 | "name": { 735 | "en": "Ojiki-chan & Reiny", 736 | "jp": "おじきちゃん & Reiny" 737 | }, 738 | "platform_id": "yt", 739 | "channel_id": "UC_HrzgYmapddmGSfn2UMHTA", 740 | "details": { 741 | "twitter": "dokuganP" 742 | } 743 | }, 744 | { 745 | "name": { 746 | "en": "Manae", 747 | "jp": "まなえ" 748 | }, 749 | "platform_id": "yt", 750 | "channel_id": "UCAPdxmEjYxUdQMf_JaQRl1Q", 751 | "details": { 752 | "twitter": "manae_nme" 753 | } 754 | } 755 | ] 756 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU AFFERO GENERAL PUBLIC LICENSE 2 | Version 3, 19 November 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU Affero General Public License is a free, copyleft license for 11 | software and other kinds of works, specifically designed to ensure 12 | cooperation with the community in the case of network server software. 13 | 14 | The licenses for most software and other practical works are designed 15 | to take away your freedom to share and change the works. By contrast, 16 | our General Public Licenses are intended to guarantee your freedom to 17 | share and change all versions of a program--to make sure it remains free 18 | software for all its users. 19 | 20 | When we speak of free software, we are referring to freedom, not 21 | price. Our General Public Licenses are designed to make sure that you 22 | have the freedom to distribute copies of free software (and charge for 23 | them if you wish), that you receive source code or can get it if you 24 | want it, that you can change the software or use pieces of it in new 25 | free programs, and that you know you can do these things. 26 | 27 | Developers that use our General Public Licenses protect your rights 28 | with two steps: (1) assert copyright on the software, and (2) offer 29 | you this License which gives you legal permission to copy, distribute 30 | and/or modify the software. 31 | 32 | A secondary benefit of defending all users' freedom is that 33 | improvements made in alternate versions of the program, if they 34 | receive widespread use, become available for other developers to 35 | incorporate. Many developers of free software are heartened and 36 | encouraged by the resulting cooperation. However, in the case of 37 | software used on network servers, this result may fail to come about. 38 | The GNU General Public License permits making a modified version and 39 | letting the public access it on a server without ever releasing its 40 | source code to the public. 41 | 42 | The GNU Affero General Public License is designed specifically to 43 | ensure that, in such cases, the modified source code becomes available 44 | to the community. It requires the operator of a network server to 45 | provide the source code of the modified version running there to the 46 | users of that server. Therefore, public use of a modified version, on 47 | a publicly accessible server, gives the public access to the source 48 | code of the modified version. 49 | 50 | An older license, called the Affero General Public License and 51 | published by Affero, was designed to accomplish similar goals. This is 52 | a different license, not a version of the Affero GPL, but Affero has 53 | released a new version of the Affero GPL which permits relicensing under 54 | this license. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | TERMS AND CONDITIONS 60 | 61 | 0. Definitions. 62 | 63 | "This License" refers to version 3 of the GNU Affero General Public License. 64 | 65 | "Copyright" also means copyright-like laws that apply to other kinds of 66 | works, such as semiconductor masks. 67 | 68 | "The Program" refers to any copyrightable work licensed under this 69 | License. Each licensee is addressed as "you". "Licensees" and 70 | "recipients" may be individuals or organizations. 71 | 72 | To "modify" a work means to copy from or adapt all or part of the work 73 | in a fashion requiring copyright permission, other than the making of an 74 | exact copy. The resulting work is called a "modified version" of the 75 | earlier work or a work "based on" the earlier work. 76 | 77 | A "covered work" means either the unmodified Program or a work based 78 | on the Program. 79 | 80 | To "propagate" a work means to do anything with it that, without 81 | permission, would make you directly or secondarily liable for 82 | infringement under applicable copyright law, except executing it on a 83 | computer or modifying a private copy. Propagation includes copying, 84 | distribution (with or without modification), making available to the 85 | public, and in some countries other activities as well. 86 | 87 | To "convey" a work means any kind of propagation that enables other 88 | parties to make or receive copies. Mere interaction with a user through 89 | a computer network, with no transfer of a copy, is not conveying. 90 | 91 | An interactive user interface displays "Appropriate Legal Notices" 92 | to the extent that it includes a convenient and prominently visible 93 | feature that (1) displays an appropriate copyright notice, and (2) 94 | tells the user that there is no warranty for the work (except to the 95 | extent that warranties are provided), that licensees may convey the 96 | work under this License, and how to view a copy of this License. If 97 | the interface presents a list of user commands or options, such as a 98 | menu, a prominent item in the list meets this criterion. 99 | 100 | 1. Source Code. 101 | 102 | The "source code" for a work means the preferred form of the work 103 | for making modifications to it. "Object code" means any non-source 104 | form of a work. 105 | 106 | A "Standard Interface" means an interface that either is an official 107 | standard defined by a recognized standards body, or, in the case of 108 | interfaces specified for a particular programming language, one that 109 | is widely used among developers working in that language. 110 | 111 | The "System Libraries" of an executable work include anything, other 112 | than the work as a whole, that (a) is included in the normal form of 113 | packaging a Major Component, but which is not part of that Major 114 | Component, and (b) serves only to enable use of the work with that 115 | Major Component, or to implement a Standard Interface for which an 116 | implementation is available to the public in source code form. A 117 | "Major Component", in this context, means a major essential component 118 | (kernel, window system, and so on) of the specific operating system 119 | (if any) on which the executable work runs, or a compiler used to 120 | produce the work, or an object code interpreter used to run it. 121 | 122 | The "Corresponding Source" for a work in object code form means all 123 | the source code needed to generate, install, and (for an executable 124 | work) run the object code and to modify the work, including scripts to 125 | control those activities. However, it does not include the work's 126 | System Libraries, or general-purpose tools or generally available free 127 | programs which are used unmodified in performing those activities but 128 | which are not part of the work. For example, Corresponding Source 129 | includes interface definition files associated with source files for 130 | the work, and the source code for shared libraries and dynamically 131 | linked subprograms that the work is specifically designed to require, 132 | such as by intimate data communication or control flow between those 133 | subprograms and other parts of the work. 134 | 135 | The Corresponding Source need not include anything that users 136 | can regenerate automatically from other parts of the Corresponding 137 | Source. 138 | 139 | The Corresponding Source for a work in source code form is that 140 | same work. 141 | 142 | 2. Basic Permissions. 143 | 144 | All rights granted under this License are granted for the term of 145 | copyright on the Program, and are irrevocable provided the stated 146 | conditions are met. This License explicitly affirms your unlimited 147 | permission to run the unmodified Program. The output from running a 148 | covered work is covered by this License only if the output, given its 149 | content, constitutes a covered work. This License acknowledges your 150 | rights of fair use or other equivalent, as provided by copyright law. 151 | 152 | You may make, run and propagate covered works that you do not 153 | convey, without conditions so long as your license otherwise remains 154 | in force. You may convey covered works to others for the sole purpose 155 | of having them make modifications exclusively for you, or provide you 156 | with facilities for running those works, provided that you comply with 157 | the terms of this License in conveying all material for which you do 158 | not control copyright. Those thus making or running the covered works 159 | for you must do so exclusively on your behalf, under your direction 160 | and control, on terms that prohibit them from making any copies of 161 | your copyrighted material outside their relationship with you. 162 | 163 | Conveying under any other circumstances is permitted solely under 164 | the conditions stated below. Sublicensing is not allowed; section 10 165 | makes it unnecessary. 166 | 167 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 168 | 169 | No covered work shall be deemed part of an effective technological 170 | measure under any applicable law fulfilling obligations under article 171 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 172 | similar laws prohibiting or restricting circumvention of such 173 | measures. 174 | 175 | When you convey a covered work, you waive any legal power to forbid 176 | circumvention of technological measures to the extent such circumvention 177 | is effected by exercising rights under this License with respect to 178 | the covered work, and you disclaim any intention to limit operation or 179 | modification of the work as a means of enforcing, against the work's 180 | users, your or third parties' legal rights to forbid circumvention of 181 | technological measures. 182 | 183 | 4. Conveying Verbatim Copies. 184 | 185 | You may convey verbatim copies of the Program's source code as you 186 | receive it, in any medium, provided that you conspicuously and 187 | appropriately publish on each copy an appropriate copyright notice; 188 | keep intact all notices stating that this License and any 189 | non-permissive terms added in accord with section 7 apply to the code; 190 | keep intact all notices of the absence of any warranty; and give all 191 | recipients a copy of this License along with the Program. 192 | 193 | You may charge any price or no price for each copy that you convey, 194 | and you may offer support or warranty protection for a fee. 195 | 196 | 5. Conveying Modified Source Versions. 197 | 198 | You may convey a work based on the Program, or the modifications to 199 | produce it from the Program, in the form of source code under the 200 | terms of section 4, provided that you also meet all of these conditions: 201 | 202 | a) The work must carry prominent notices stating that you modified 203 | it, and giving a relevant date. 204 | 205 | b) The work must carry prominent notices stating that it is 206 | released under this License and any conditions added under section 207 | 7. This requirement modifies the requirement in section 4 to 208 | "keep intact all notices". 209 | 210 | c) You must license the entire work, as a whole, under this 211 | License to anyone who comes into possession of a copy. This 212 | License will therefore apply, along with any applicable section 7 213 | additional terms, to the whole of the work, and all its parts, 214 | regardless of how they are packaged. This License gives no 215 | permission to license the work in any other way, but it does not 216 | invalidate such permission if you have separately received it. 217 | 218 | d) If the work has interactive user interfaces, each must display 219 | Appropriate Legal Notices; however, if the Program has interactive 220 | interfaces that do not display Appropriate Legal Notices, your 221 | work need not make them do so. 222 | 223 | A compilation of a covered work with other separate and independent 224 | works, which are not by their nature extensions of the covered work, 225 | and which are not combined with it such as to form a larger program, 226 | in or on a volume of a storage or distribution medium, is called an 227 | "aggregate" if the compilation and its resulting copyright are not 228 | used to limit the access or legal rights of the compilation's users 229 | beyond what the individual works permit. Inclusion of a covered work 230 | in an aggregate does not cause this License to apply to the other 231 | parts of the aggregate. 232 | 233 | 6. Conveying Non-Source Forms. 234 | 235 | You may convey a covered work in object code form under the terms 236 | of sections 4 and 5, provided that you also convey the 237 | machine-readable Corresponding Source under the terms of this License, 238 | in one of these ways: 239 | 240 | a) Convey the object code in, or embodied in, a physical product 241 | (including a physical distribution medium), accompanied by the 242 | Corresponding Source fixed on a durable physical medium 243 | customarily used for software interchange. 244 | 245 | b) Convey the object code in, or embodied in, a physical product 246 | (including a physical distribution medium), accompanied by a 247 | written offer, valid for at least three years and valid for as 248 | long as you offer spare parts or customer support for that product 249 | model, to give anyone who possesses the object code either (1) a 250 | copy of the Corresponding Source for all the software in the 251 | product that is covered by this License, on a durable physical 252 | medium customarily used for software interchange, for a price no 253 | more than your reasonable cost of physically performing this 254 | conveying of source, or (2) access to copy the 255 | Corresponding Source from a network server at no charge. 256 | 257 | c) Convey individual copies of the object code with a copy of the 258 | written offer to provide the Corresponding Source. This 259 | alternative is allowed only occasionally and noncommercially, and 260 | only if you received the object code with such an offer, in accord 261 | with subsection 6b. 262 | 263 | d) Convey the object code by offering access from a designated 264 | place (gratis or for a charge), and offer equivalent access to the 265 | Corresponding Source in the same way through the same place at no 266 | further charge. You need not require recipients to copy the 267 | Corresponding Source along with the object code. If the place to 268 | copy the object code is a network server, the Corresponding Source 269 | may be on a different server (operated by you or a third party) 270 | that supports equivalent copying facilities, provided you maintain 271 | clear directions next to the object code saying where to find the 272 | Corresponding Source. Regardless of what server hosts the 273 | Corresponding Source, you remain obligated to ensure that it is 274 | available for as long as needed to satisfy these requirements. 275 | 276 | e) Convey the object code using peer-to-peer transmission, provided 277 | you inform other peers where the object code and Corresponding 278 | Source of the work are being offered to the general public at no 279 | charge under subsection 6d. 280 | 281 | A separable portion of the object code, whose source code is excluded 282 | from the Corresponding Source as a System Library, need not be 283 | included in conveying the object code work. 284 | 285 | A "User Product" is either (1) a "consumer product", which means any 286 | tangible personal property which is normally used for personal, family, 287 | or household purposes, or (2) anything designed or sold for incorporation 288 | into a dwelling. In determining whether a product is a consumer product, 289 | doubtful cases shall be resolved in favor of coverage. For a particular 290 | product received by a particular user, "normally used" refers to a 291 | typical or common use of that class of product, regardless of the status 292 | of the particular user or of the way in which the particular user 293 | actually uses, or expects or is expected to use, the product. A product 294 | is a consumer product regardless of whether the product has substantial 295 | commercial, industrial or non-consumer uses, unless such uses represent 296 | the only significant mode of use of the product. 297 | 298 | "Installation Information" for a User Product means any methods, 299 | procedures, authorization keys, or other information required to install 300 | and execute modified versions of a covered work in that User Product from 301 | a modified version of its Corresponding Source. The information must 302 | suffice to ensure that the continued functioning of the modified object 303 | code is in no case prevented or interfered with solely because 304 | modification has been made. 305 | 306 | If you convey an object code work under this section in, or with, or 307 | specifically for use in, a User Product, and the conveying occurs as 308 | part of a transaction in which the right of possession and use of the 309 | User Product is transferred to the recipient in perpetuity or for a 310 | fixed term (regardless of how the transaction is characterized), the 311 | Corresponding Source conveyed under this section must be accompanied 312 | by the Installation Information. But this requirement does not apply 313 | if neither you nor any third party retains the ability to install 314 | modified object code on the User Product (for example, the work has 315 | been installed in ROM). 316 | 317 | The requirement to provide Installation Information does not include a 318 | requirement to continue to provide support service, warranty, or updates 319 | for a work that has been modified or installed by the recipient, or for 320 | the User Product in which it has been modified or installed. Access to a 321 | network may be denied when the modification itself materially and 322 | adversely affects the operation of the network or violates the rules and 323 | protocols for communication across the network. 324 | 325 | Corresponding Source conveyed, and Installation Information provided, 326 | in accord with this section must be in a format that is publicly 327 | documented (and with an implementation available to the public in 328 | source code form), and must require no special password or key for 329 | unpacking, reading or copying. 330 | 331 | 7. Additional Terms. 332 | 333 | "Additional permissions" are terms that supplement the terms of this 334 | License by making exceptions from one or more of its conditions. 335 | Additional permissions that are applicable to the entire Program shall 336 | be treated as though they were included in this License, to the extent 337 | that they are valid under applicable law. If additional permissions 338 | apply only to part of the Program, that part may be used separately 339 | under those permissions, but the entire Program remains governed by 340 | this License without regard to the additional permissions. 341 | 342 | When you convey a copy of a covered work, you may at your option 343 | remove any additional permissions from that copy, or from any part of 344 | it. (Additional permissions may be written to require their own 345 | removal in certain cases when you modify the work.) You may place 346 | additional permissions on material, added by you to a covered work, 347 | for which you have or can give appropriate copyright permission. 348 | 349 | Notwithstanding any other provision of this License, for material you 350 | add to a covered work, you may (if authorized by the copyright holders of 351 | that material) supplement the terms of this License with terms: 352 | 353 | a) Disclaiming warranty or limiting liability differently from the 354 | terms of sections 15 and 16 of this License; or 355 | 356 | b) Requiring preservation of specified reasonable legal notices or 357 | author attributions in that material or in the Appropriate Legal 358 | Notices displayed by works containing it; or 359 | 360 | c) Prohibiting misrepresentation of the origin of that material, or 361 | requiring that modified versions of such material be marked in 362 | reasonable ways as different from the original version; or 363 | 364 | d) Limiting the use for publicity purposes of names of licensors or 365 | authors of the material; or 366 | 367 | e) Declining to grant rights under trademark law for use of some 368 | trade names, trademarks, or service marks; or 369 | 370 | f) Requiring indemnification of licensors and authors of that 371 | material by anyone who conveys the material (or modified versions of 372 | it) with contractual assumptions of liability to the recipient, for 373 | any liability that these contractual assumptions directly impose on 374 | those licensors and authors. 375 | 376 | All other non-permissive additional terms are considered "further 377 | restrictions" within the meaning of section 10. If the Program as you 378 | received it, or any part of it, contains a notice stating that it is 379 | governed by this License along with a term that is a further 380 | restriction, you may remove that term. If a license document contains 381 | a further restriction but permits relicensing or conveying under this 382 | License, you may add to a covered work material governed by the terms 383 | of that license document, provided that the further restriction does 384 | not survive such relicensing or conveying. 385 | 386 | If you add terms to a covered work in accord with this section, you 387 | must place, in the relevant source files, a statement of the 388 | additional terms that apply to those files, or a notice indicating 389 | where to find the applicable terms. 390 | 391 | Additional terms, permissive or non-permissive, may be stated in the 392 | form of a separately written license, or stated as exceptions; 393 | the above requirements apply either way. 394 | 395 | 8. Termination. 396 | 397 | You may not propagate or modify a covered work except as expressly 398 | provided under this License. Any attempt otherwise to propagate or 399 | modify it is void, and will automatically terminate your rights under 400 | this License (including any patent licenses granted under the third 401 | paragraph of section 11). 402 | 403 | However, if you cease all violation of this License, then your 404 | license from a particular copyright holder is reinstated (a) 405 | provisionally, unless and until the copyright holder explicitly and 406 | finally terminates your license, and (b) permanently, if the copyright 407 | holder fails to notify you of the violation by some reasonable means 408 | prior to 60 days after the cessation. 409 | 410 | Moreover, your license from a particular copyright holder is 411 | reinstated permanently if the copyright holder notifies you of the 412 | violation by some reasonable means, this is the first time you have 413 | received notice of violation of this License (for any work) from that 414 | copyright holder, and you cure the violation prior to 30 days after 415 | your receipt of the notice. 416 | 417 | Termination of your rights under this section does not terminate the 418 | licenses of parties who have received copies or rights from you under 419 | this License. If your rights have been terminated and not permanently 420 | reinstated, you do not qualify to receive new licenses for the same 421 | material under section 10. 422 | 423 | 9. Acceptance Not Required for Having Copies. 424 | 425 | You are not required to accept this License in order to receive or 426 | run a copy of the Program. Ancillary propagation of a covered work 427 | occurring solely as a consequence of using peer-to-peer transmission 428 | to receive a copy likewise does not require acceptance. However, 429 | nothing other than this License grants you permission to propagate or 430 | modify any covered work. These actions infringe copyright if you do 431 | not accept this License. Therefore, by modifying or propagating a 432 | covered work, you indicate your acceptance of this License to do so. 433 | 434 | 10. Automatic Licensing of Downstream Recipients. 435 | 436 | Each time you convey a covered work, the recipient automatically 437 | receives a license from the original licensors, to run, modify and 438 | propagate that work, subject to this License. You are not responsible 439 | for enforcing compliance by third parties with this License. 440 | 441 | An "entity transaction" is a transaction transferring control of an 442 | organization, or substantially all assets of one, or subdividing an 443 | organization, or merging organizations. If propagation of a covered 444 | work results from an entity transaction, each party to that 445 | transaction who receives a copy of the work also receives whatever 446 | licenses to the work the party's predecessor in interest had or could 447 | give under the previous paragraph, plus a right to possession of the 448 | Corresponding Source of the work from the predecessor in interest, if 449 | the predecessor has it or can get it with reasonable efforts. 450 | 451 | You may not impose any further restrictions on the exercise of the 452 | rights granted or affirmed under this License. For example, you may 453 | not impose a license fee, royalty, or other charge for exercise of 454 | rights granted under this License, and you may not initiate litigation 455 | (including a cross-claim or counterclaim in a lawsuit) alleging that 456 | any patent claim is infringed by making, using, selling, offering for 457 | sale, or importing the Program or any portion of it. 458 | 459 | 11. Patents. 460 | 461 | A "contributor" is a copyright holder who authorizes use under this 462 | License of the Program or a work on which the Program is based. The 463 | work thus licensed is called the contributor's "contributor version". 464 | 465 | A contributor's "essential patent claims" are all patent claims 466 | owned or controlled by the contributor, whether already acquired or 467 | hereafter acquired, that would be infringed by some manner, permitted 468 | by this License, of making, using, or selling its contributor version, 469 | but do not include claims that would be infringed only as a 470 | consequence of further modification of the contributor version. For 471 | purposes of this definition, "control" includes the right to grant 472 | patent sublicenses in a manner consistent with the requirements of 473 | this License. 474 | 475 | Each contributor grants you a non-exclusive, worldwide, royalty-free 476 | patent license under the contributor's essential patent claims, to 477 | make, use, sell, offer for sale, import and otherwise run, modify and 478 | propagate the contents of its contributor version. 479 | 480 | In the following three paragraphs, a "patent license" is any express 481 | agreement or commitment, however denominated, not to enforce a patent 482 | (such as an express permission to practice a patent or covenant not to 483 | sue for patent infringement). To "grant" such a patent license to a 484 | party means to make such an agreement or commitment not to enforce a 485 | patent against the party. 486 | 487 | If you convey a covered work, knowingly relying on a patent license, 488 | and the Corresponding Source of the work is not available for anyone 489 | to copy, free of charge and under the terms of this License, through a 490 | publicly available network server or other readily accessible means, 491 | then you must either (1) cause the Corresponding Source to be so 492 | available, or (2) arrange to deprive yourself of the benefit of the 493 | patent license for this particular work, or (3) arrange, in a manner 494 | consistent with the requirements of this License, to extend the patent 495 | license to downstream recipients. "Knowingly relying" means you have 496 | actual knowledge that, but for the patent license, your conveying the 497 | covered work in a country, or your recipient's use of the covered work 498 | in a country, would infringe one or more identifiable patents in that 499 | country that you have reason to believe are valid. 500 | 501 | If, pursuant to or in connection with a single transaction or 502 | arrangement, you convey, or propagate by procuring conveyance of, a 503 | covered work, and grant a patent license to some of the parties 504 | receiving the covered work authorizing them to use, propagate, modify 505 | or convey a specific copy of the covered work, then the patent license 506 | you grant is automatically extended to all recipients of the covered 507 | work and works based on it. 508 | 509 | A patent license is "discriminatory" if it does not include within 510 | the scope of its coverage, prohibits the exercise of, or is 511 | conditioned on the non-exercise of one or more of the rights that are 512 | specifically granted under this License. You may not convey a covered 513 | work if you are a party to an arrangement with a third party that is 514 | in the business of distributing software, under which you make payment 515 | to the third party based on the extent of your activity of conveying 516 | the work, and under which the third party grants, to any of the 517 | parties who would receive the covered work from you, a discriminatory 518 | patent license (a) in connection with copies of the covered work 519 | conveyed by you (or copies made from those copies), or (b) primarily 520 | for and in connection with specific products or compilations that 521 | contain the covered work, unless you entered into that arrangement, 522 | or that patent license was granted, prior to 28 March 2007. 523 | 524 | Nothing in this License shall be construed as excluding or limiting 525 | any implied license or other defenses to infringement that may 526 | otherwise be available to you under applicable patent law. 527 | 528 | 12. No Surrender of Others' Freedom. 529 | 530 | If conditions are imposed on you (whether by court order, agreement or 531 | otherwise) that contradict the conditions of this License, they do not 532 | excuse you from the conditions of this License. If you cannot convey a 533 | covered work so as to satisfy simultaneously your obligations under this 534 | License and any other pertinent obligations, then as a consequence you may 535 | not convey it at all. For example, if you agree to terms that obligate you 536 | to collect a royalty for further conveying from those to whom you convey 537 | the Program, the only way you could satisfy both those terms and this 538 | License would be to refrain entirely from conveying the Program. 539 | 540 | 13. Remote Network Interaction; Use with the GNU General Public License. 541 | 542 | Notwithstanding any other provision of this License, if you modify the 543 | Program, your modified version must prominently offer all users 544 | interacting with it remotely through a computer network (if your version 545 | supports such interaction) an opportunity to receive the Corresponding 546 | Source of your version by providing access to the Corresponding Source 547 | from a network server at no charge, through some standard or customary 548 | means of facilitating copying of software. This Corresponding Source 549 | shall include the Corresponding Source for any work covered by version 3 550 | of the GNU General Public License that is incorporated pursuant to the 551 | following paragraph. 552 | 553 | Notwithstanding any other provision of this License, you have 554 | permission to link or combine any covered work with a work licensed 555 | under version 3 of the GNU General Public License into a single 556 | combined work, and to convey the resulting work. The terms of this 557 | License will continue to apply to the part which is the covered work, 558 | but the work with which it is combined will remain governed by version 559 | 3 of the GNU General Public License. 560 | 561 | 14. Revised Versions of this License. 562 | 563 | The Free Software Foundation may publish revised and/or new versions of 564 | the GNU Affero General Public License from time to time. Such new versions 565 | will be similar in spirit to the present version, but may differ in detail to 566 | address new problems or concerns. 567 | 568 | Each version is given a distinguishing version number. If the 569 | Program specifies that a certain numbered version of the GNU Affero General 570 | Public License "or any later version" applies to it, you have the 571 | option of following the terms and conditions either of that numbered 572 | version or of any later version published by the Free Software 573 | Foundation. If the Program does not specify a version number of the 574 | GNU Affero General Public License, you may choose any version ever published 575 | by the Free Software Foundation. 576 | 577 | If the Program specifies that a proxy can decide which future 578 | versions of the GNU Affero General Public License can be used, that proxy's 579 | public statement of acceptance of a version permanently authorizes you 580 | to choose that version for the Program. 581 | 582 | Later license versions may give you additional or different 583 | permissions. However, no additional obligations are imposed on any 584 | author or copyright holder as a result of your choosing to follow a 585 | later version. 586 | 587 | 15. Disclaimer of Warranty. 588 | 589 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 590 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 591 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 592 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 593 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 594 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 595 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 596 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 597 | 598 | 16. Limitation of Liability. 599 | 600 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 601 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 602 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 603 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 604 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 605 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 606 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 607 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 608 | SUCH DAMAGES. 609 | 610 | 17. Interpretation of Sections 15 and 16. 611 | 612 | If the disclaimer of warranty and limitation of liability provided 613 | above cannot be given local legal effect according to their terms, 614 | reviewing courts shall apply local law that most closely approximates 615 | an absolute waiver of all civil liability in connection with the 616 | Program, unless a warranty or assumption of liability accompanies a 617 | copy of the Program in return for a fee. 618 | 619 | END OF TERMS AND CONDITIONS 620 | 621 | How to Apply These Terms to Your New Programs 622 | 623 | If you develop a new program, and you want it to be of the greatest 624 | possible use to the public, the best way to achieve this is to make it 625 | free software which everyone can redistribute and change under these terms. 626 | 627 | To do so, attach the following notices to the program. It is safest 628 | to attach them to the start of each source file to most effectively 629 | state the exclusion of warranty; and each file should have at least 630 | the "copyright" line and a pointer to where the full notice is found. 631 | 632 | 633 | Copyright (C) 634 | 635 | This program is free software: you can redistribute it and/or modify 636 | it under the terms of the GNU Affero General Public License as published 637 | by the Free Software Foundation, either version 3 of the License, or 638 | (at your option) any later version. 639 | 640 | This program is distributed in the hope that it will be useful, 641 | but WITHOUT ANY WARRANTY; without even the implied warranty of 642 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 643 | GNU Affero General Public License for more details. 644 | 645 | You should have received a copy of the GNU Affero General Public License 646 | along with this program. If not, see . 647 | 648 | Also add information on how to contact you by electronic and paper mail. 649 | 650 | If your software can interact with users remotely through a computer 651 | network, you should also make sure that it provides a way for users to 652 | get its source. For example, if your program is a web application, its 653 | interface could display a "Source" link that leads users to an archive 654 | of the code. There are many ways you could offer source, and different 655 | solutions will be better for different programs; see section 13 for the 656 | specific requirements. 657 | 658 | You should also get your employer (if you work as a programmer) or school, 659 | if any, to sign a "copyright disclaimer" for the program, if necessary. 660 | For more information on this, and how to apply and follow the GNU AGPL, see 661 | . --------------------------------------------------------------------------------