├── .env.example ├── .github ├── FUNDING.yml ├── dependabot.yml └── workflows │ └── node.js.yml ├── .gitignore ├── .prettierrc.json ├── LICENSE ├── README.md ├── index.js ├── package-lock.json ├── package.json └── query.graphql /.env.example: -------------------------------------------------------------------------------- 1 | ES_HOST=127.0.0.1 2 | ES_PORT=9200 3 | ES_INDEX=anilist 4 | 5 | DB_HOST=127.0.0.1 6 | DB_PORT=3306 7 | DB_USER=anilist 8 | DB_PASS=anilist 9 | DB_NAME=anilist 10 | DB_TABLE=anilist 11 | 12 | FS_DIR=/tmp/anilist 13 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [soruly] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: soruly 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: https://www.paypal.me/soruly/ 13 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: npm 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | - package-ecosystem: "github-actions" 8 | directory: "/" 9 | schedule: 10 | interval: "daily" 11 | -------------------------------------------------------------------------------- /.github/workflows/node.js.yml: -------------------------------------------------------------------------------- 1 | # This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions 3 | 4 | name: Node.js CI 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | pull_request: 10 | branches: [ master ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | strategy: 18 | matrix: 19 | node-version: [lts/*, current] 20 | # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ 21 | 22 | steps: 23 | - uses: actions/checkout@v4 24 | - name: Use Node.js ${{ matrix.node-version }} 25 | uses: actions/setup-node@v4 26 | with: 27 | node-version: ${{ matrix.node-version }} 28 | - run: npm ci 29 | - run: npm run build --if-present 30 | - run: npm test 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (http://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # Typescript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | config.json 61 | -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 100 3 | } 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 soruly 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # anilist-crawler 2 | 3 | [![License](https://img.shields.io/github/license/soruly/anilist-crawler.svg?style=flat-square)](https://github.com/soruly/anilist-crawler/blob/master/LICENSE) 4 | [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/soruly/anilist-crawler/node.js.yml?style=flat-square)](https://github.com/soruly/anilist-crawler/actions) 5 | 6 | Crawl data from [AniList](https://anilist.co/home) API and store as json in file system, MariaDB, or elasticsearch. 7 | 8 | ## Requirements 9 | 10 | - Node.js >= 20.12 11 | - MariaDB >= 10.5 (optional) 12 | - elasticsearch >= 7.0 (optional) 13 | 14 | ## How to use 15 | 16 | 1. Clone this repository 17 | 18 | 2. `npm install` 19 | 20 | 3. copy `.env.example` and rename to `.env` 21 | 22 | 4. Configure `.env` for your mariaDB, or elasticsearch, leave any of DB_HOST, ES_HOST, FS_DIR empty if you don't need it 23 | 24 | ## Examples 25 | 26 | Fetch anime ID 123 27 | 28 | `node index.js --anime 123` 29 | 30 | Fetch all anime in page 240 31 | 32 | `node index.js --page 240` 33 | 34 | Fetch all anime from page 240 to 244 (inclusive) 35 | 36 | `node index.js --page 240-244` 37 | 38 | Fetch all anime from page 240 to the last page 39 | 40 | `node index.js --page 240-` 41 | 42 | Sometimes anime would be deleted from AniList, but it still exists locally in your database. You can use `--clean` to get a clean copy every time you start crawling. 43 | 44 | `node index.js --clean --page 240-` 45 | 46 | For details of AniList API please visit https://github.com/AniList/ApiV2-GraphQL-Docs/ 47 | 48 | You can try the interactive query tool here. https://anilist.co/graphiql 49 | 50 | ## Notes 51 | 52 | - API request limit exceed (HTTP 429) has not been handled yet. With 60 requests/min per IP, it is unlikely to hit the limit with complex query. 53 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | import path from "node:path"; 2 | import fs from "node:fs/promises"; 3 | import cluster from "node:cluster"; 4 | import Knex from "knex"; 5 | 6 | process.loadEnvFile(); 7 | const { 8 | DB_HOST, 9 | DB_PORT, 10 | DB_USER, 11 | DB_PASS, 12 | DB_NAME, 13 | DB_TABLE, 14 | ES_HOST, 15 | ES_PORT, 16 | ES_INDEX, 17 | FS_DIR, 18 | } = process.env; 19 | 20 | const q = {}; 21 | q.query = await fs.readFile("query.graphql", "utf8"); 22 | 23 | const submitQuery = async (query, variables) => { 24 | query.variables = variables; 25 | try { 26 | const response = await fetch("https://graphql.anilist.co/", { 27 | method: "POST", 28 | body: JSON.stringify(query), 29 | headers: { "Content-Type": "application/json" }, 30 | }).then((res) => res.json()); 31 | if (response.errors) { 32 | console.log(response.errors); 33 | } 34 | return response.data; 35 | } catch (e) { 36 | console.log(e); 37 | return null; 38 | } 39 | }; 40 | 41 | const perPage = 50; 42 | const numOfWorker = 3; 43 | 44 | const knex = DB_HOST 45 | ? Knex({ 46 | client: "mysql", 47 | connection: { 48 | host: DB_HOST, 49 | port: DB_PORT, 50 | user: DB_USER, 51 | password: DB_PASS, 52 | database: DB_NAME, 53 | }, 54 | }) 55 | : null; 56 | 57 | if (cluster.isPrimary) { 58 | const [arg, value] = process.argv.slice(2); 59 | 60 | if (knex) { 61 | if (process.argv.slice(2).includes("--clean")) { 62 | console.log(`Dropping table ${DB_TABLE} if exists`); 63 | await knex.schema.dropTableIfExists(DB_TABLE); 64 | console.log(`Dropped table ${DB_TABLE}`); 65 | } 66 | if (!(await knex.schema.hasTable(DB_TABLE))) { 67 | console.log(`Creating table ${DB_TABLE}`); 68 | await knex.schema.createTable(DB_TABLE, (table) => { 69 | table.integer("id").unsigned().notNullable().primary(); 70 | table.json("json").collate("utf8mb4_unicode_ci"); 71 | }); 72 | console.log(`Created table ${DB_TABLE}`); 73 | } 74 | knex.destroy(); 75 | } 76 | 77 | if (ES_HOST) { 78 | if (process.argv.slice(2).includes("--clean")) { 79 | console.log(`Dropping index ${ES_INDEX} if exists`); 80 | await fetch(`http://${ES_HOST}:${ES_PORT}/${ES_INDEX}`, { method: "DELETE" }); 81 | console.log(`Dropped index ${ES_INDEX}`); 82 | } 83 | if ((await fetch(`http://${ES_HOST}:${ES_PORT}/${ES_INDEX}`)).status === 404) { 84 | console.log(`Creating index ${ES_INDEX}`); 85 | await fetch(`http://${ES_HOST}:${ES_PORT}/${ES_INDEX}`, { 86 | method: "PUT", 87 | body: JSON.stringify({ 88 | settings: { 89 | index: { 90 | number_of_shards: 1, 91 | number_of_replicas: 0, 92 | }, 93 | }, 94 | }), 95 | headers: { "Content-Type": "application/json" }, 96 | }); 97 | console.log(`Created index ${ES_INDEX}`); 98 | } 99 | } 100 | 101 | if (FS_DIR) { 102 | await fs.mkdir(FS_DIR, { recursive: true }); 103 | } 104 | 105 | if (arg === "--anime" && value) { 106 | console.log(`Crawling anime ${value}`); 107 | const anime = (await submitQuery(q, { id: value })).Page.media[0]; 108 | const worker = cluster.fork(); 109 | worker.on("message", (message) => { 110 | console.log(`Completed anime ${anime.id} (${anime.title.native ?? anime.title.romaji})`); 111 | worker.kill(); 112 | }); 113 | await new Promise((resolve) => setTimeout(resolve, 500)); 114 | worker.send(anime); 115 | } else if (arg === "--page" && value) { 116 | const format = /^(\d+)(-)?(\d+)?$/; 117 | const startPage = Number(value.match(format)[1]); 118 | const lastPage = value.match(format)[2] ? Number(value.match(format)[3]) : startPage; 119 | 120 | console.log(`Crawling page ${startPage} to ${lastPage || "end"}`); 121 | 122 | let animeList = []; 123 | let finished = false; 124 | 125 | for (let i = 0; i < numOfWorker; i++) { 126 | cluster.fork(); 127 | } 128 | 129 | cluster.on("message", (worker, anime) => { 130 | console.log(`Completed anime ${anime.id} (${anime.title.native ?? anime.title.romaji})`); 131 | if (animeList.length > 0) { 132 | worker.send(animeList.pop()); 133 | } else if (finished) { 134 | worker.kill(); 135 | } 136 | }); 137 | 138 | let page = startPage; 139 | while (!lastPage || page <= lastPage) { 140 | console.log(`Crawling page ${page}`); 141 | const res = await submitQuery(q, { 142 | page, 143 | perPage, 144 | }); 145 | animeList = animeList.concat(res.Page.media); 146 | for (const id in cluster.workers) { 147 | if (animeList.length > 0) { 148 | cluster.workers[id].send(animeList.pop()); 149 | } 150 | } 151 | if (!res.Page.pageInfo.hasNextPage) break; 152 | page++; 153 | } 154 | finished = true; 155 | console.log("Crawling complete"); 156 | } else { 157 | console.log("Usage: node index.js --anime 1"); 158 | console.log(" node index.js --page 1"); 159 | console.log(" node index.js --page 1-"); 160 | console.log(" node index.js --page 1-2"); 161 | } 162 | } else { 163 | process.on("message", async (anime) => { 164 | if (knex) { 165 | // delete the record from mariadb if already exists 166 | await knex(DB_TABLE).where({ id: anime.id }).del(); 167 | await knex(DB_TABLE).insert({ 168 | id: anime.id, 169 | json: JSON.stringify(anime), 170 | }); 171 | } 172 | 173 | if (ES_HOST) { 174 | const response = await fetch(`http://${ES_HOST}:${ES_PORT}/${ES_INDEX}/anime/${anime.id}`, { 175 | method: "PUT", 176 | body: JSON.stringify(anime), 177 | headers: { "Content-Type": "application/json" }, 178 | }); 179 | } 180 | 181 | if (FS_DIR) { 182 | await fs.writeFile(path.join(FS_DIR, `${anime.id}.json`), JSON.stringify(anime, null, 2)); 183 | } 184 | 185 | process.send(anime); 186 | }); 187 | 188 | process.on("exit", () => { 189 | if (knex) knex.destroy(); 190 | }); 191 | } 192 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "anilist-crawler", 3 | "version": "1.3.0", 4 | "lockfileVersion": 3, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "anilist-crawler", 9 | "version": "1.3.0", 10 | "license": "MIT", 11 | "dependencies": { 12 | "knex": "^3.1.0", 13 | "mysql": "^2.18.1" 14 | }, 15 | "devDependencies": { 16 | "prettier": "^3.5.3" 17 | } 18 | }, 19 | "node_modules/bignumber.js": { 20 | "version": "9.0.0", 21 | "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz", 22 | "integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A==", 23 | "engines": { 24 | "node": "*" 25 | } 26 | }, 27 | "node_modules/colorette": { 28 | "version": "2.0.19", 29 | "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.19.tgz", 30 | "integrity": "sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ==" 31 | }, 32 | "node_modules/commander": { 33 | "version": "10.0.1", 34 | "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", 35 | "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", 36 | "engines": { 37 | "node": ">=14" 38 | } 39 | }, 40 | "node_modules/core-util-is": { 41 | "version": "1.0.3", 42 | "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", 43 | "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" 44 | }, 45 | "node_modules/debug": { 46 | "version": "4.3.4", 47 | "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", 48 | "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", 49 | "dependencies": { 50 | "ms": "2.1.2" 51 | }, 52 | "engines": { 53 | "node": ">=6.0" 54 | }, 55 | "peerDependenciesMeta": { 56 | "supports-color": { 57 | "optional": true 58 | } 59 | } 60 | }, 61 | "node_modules/escalade": { 62 | "version": "3.1.1", 63 | "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", 64 | "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", 65 | "engines": { 66 | "node": ">=6" 67 | } 68 | }, 69 | "node_modules/esm": { 70 | "version": "3.2.25", 71 | "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", 72 | "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==", 73 | "engines": { 74 | "node": ">=6" 75 | } 76 | }, 77 | "node_modules/function-bind": { 78 | "version": "1.1.2", 79 | "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", 80 | "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", 81 | "funding": { 82 | "url": "https://github.com/sponsors/ljharb" 83 | } 84 | }, 85 | "node_modules/get-package-type": { 86 | "version": "0.1.0", 87 | "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", 88 | "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", 89 | "engines": { 90 | "node": ">=8.0.0" 91 | } 92 | }, 93 | "node_modules/getopts": { 94 | "version": "2.3.0", 95 | "resolved": "https://registry.npmjs.org/getopts/-/getopts-2.3.0.tgz", 96 | "integrity": "sha512-5eDf9fuSXwxBL6q5HX+dhDj+dslFGWzU5thZ9kNKUkcPtaPdatmUFKwHFrLb/uf/WpA4BHET+AX3Scl56cAjpA==" 97 | }, 98 | "node_modules/hasown": { 99 | "version": "2.0.0", 100 | "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz", 101 | "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==", 102 | "dependencies": { 103 | "function-bind": "^1.1.2" 104 | }, 105 | "engines": { 106 | "node": ">= 0.4" 107 | } 108 | }, 109 | "node_modules/inherits": { 110 | "version": "2.0.4", 111 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", 112 | "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" 113 | }, 114 | "node_modules/interpret": { 115 | "version": "2.2.0", 116 | "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", 117 | "integrity": "sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==", 118 | "engines": { 119 | "node": ">= 0.10" 120 | } 121 | }, 122 | "node_modules/is-core-module": { 123 | "version": "2.13.1", 124 | "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", 125 | "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==", 126 | "dependencies": { 127 | "hasown": "^2.0.0" 128 | }, 129 | "funding": { 130 | "url": "https://github.com/sponsors/ljharb" 131 | } 132 | }, 133 | "node_modules/isarray": { 134 | "version": "1.0.0", 135 | "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", 136 | "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" 137 | }, 138 | "node_modules/knex": { 139 | "version": "3.1.0", 140 | "resolved": "https://registry.npmjs.org/knex/-/knex-3.1.0.tgz", 141 | "integrity": "sha512-GLoII6hR0c4ti243gMs5/1Rb3B+AjwMOfjYm97pu0FOQa7JH56hgBxYf5WK2525ceSbBY1cjeZ9yk99GPMB6Kw==", 142 | "dependencies": { 143 | "colorette": "2.0.19", 144 | "commander": "^10.0.0", 145 | "debug": "4.3.4", 146 | "escalade": "^3.1.1", 147 | "esm": "^3.2.25", 148 | "get-package-type": "^0.1.0", 149 | "getopts": "2.3.0", 150 | "interpret": "^2.2.0", 151 | "lodash": "^4.17.21", 152 | "pg-connection-string": "2.6.2", 153 | "rechoir": "^0.8.0", 154 | "resolve-from": "^5.0.0", 155 | "tarn": "^3.0.2", 156 | "tildify": "2.0.0" 157 | }, 158 | "bin": { 159 | "knex": "bin/cli.js" 160 | }, 161 | "engines": { 162 | "node": ">=16" 163 | }, 164 | "peerDependenciesMeta": { 165 | "better-sqlite3": { 166 | "optional": true 167 | }, 168 | "mysql": { 169 | "optional": true 170 | }, 171 | "mysql2": { 172 | "optional": true 173 | }, 174 | "pg": { 175 | "optional": true 176 | }, 177 | "pg-native": { 178 | "optional": true 179 | }, 180 | "sqlite3": { 181 | "optional": true 182 | }, 183 | "tedious": { 184 | "optional": true 185 | } 186 | } 187 | }, 188 | "node_modules/lodash": { 189 | "version": "4.17.21", 190 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", 191 | "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" 192 | }, 193 | "node_modules/ms": { 194 | "version": "2.1.2", 195 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", 196 | "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" 197 | }, 198 | "node_modules/mysql": { 199 | "version": "2.18.1", 200 | "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz", 201 | "integrity": "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig==", 202 | "dependencies": { 203 | "bignumber.js": "9.0.0", 204 | "readable-stream": "2.3.7", 205 | "safe-buffer": "5.1.2", 206 | "sqlstring": "2.3.1" 207 | }, 208 | "engines": { 209 | "node": ">= 0.6" 210 | } 211 | }, 212 | "node_modules/path-parse": { 213 | "version": "1.0.7", 214 | "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", 215 | "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" 216 | }, 217 | "node_modules/pg-connection-string": { 218 | "version": "2.6.2", 219 | "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", 220 | "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==" 221 | }, 222 | "node_modules/prettier": { 223 | "version": "3.5.3", 224 | "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.5.3.tgz", 225 | "integrity": "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw==", 226 | "dev": true, 227 | "license": "MIT", 228 | "bin": { 229 | "prettier": "bin/prettier.cjs" 230 | }, 231 | "engines": { 232 | "node": ">=14" 233 | }, 234 | "funding": { 235 | "url": "https://github.com/prettier/prettier?sponsor=1" 236 | } 237 | }, 238 | "node_modules/process-nextick-args": { 239 | "version": "2.0.1", 240 | "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", 241 | "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" 242 | }, 243 | "node_modules/readable-stream": { 244 | "version": "2.3.7", 245 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", 246 | "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", 247 | "dependencies": { 248 | "core-util-is": "~1.0.0", 249 | "inherits": "~2.0.3", 250 | "isarray": "~1.0.0", 251 | "process-nextick-args": "~2.0.0", 252 | "safe-buffer": "~5.1.1", 253 | "string_decoder": "~1.1.1", 254 | "util-deprecate": "~1.0.1" 255 | } 256 | }, 257 | "node_modules/rechoir": { 258 | "version": "0.8.0", 259 | "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", 260 | "integrity": "sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==", 261 | "dependencies": { 262 | "resolve": "^1.20.0" 263 | }, 264 | "engines": { 265 | "node": ">= 10.13.0" 266 | } 267 | }, 268 | "node_modules/resolve": { 269 | "version": "1.22.8", 270 | "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", 271 | "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", 272 | "dependencies": { 273 | "is-core-module": "^2.13.0", 274 | "path-parse": "^1.0.7", 275 | "supports-preserve-symlinks-flag": "^1.0.0" 276 | }, 277 | "bin": { 278 | "resolve": "bin/resolve" 279 | }, 280 | "funding": { 281 | "url": "https://github.com/sponsors/ljharb" 282 | } 283 | }, 284 | "node_modules/resolve-from": { 285 | "version": "5.0.0", 286 | "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", 287 | "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", 288 | "engines": { 289 | "node": ">=8" 290 | } 291 | }, 292 | "node_modules/safe-buffer": { 293 | "version": "5.1.2", 294 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", 295 | "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" 296 | }, 297 | "node_modules/sqlstring": { 298 | "version": "2.3.1", 299 | "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz", 300 | "integrity": "sha512-ooAzh/7dxIG5+uDik1z/Rd1vli0+38izZhGzSa34FwR7IbelPWCCKSNIl8jlL/F7ERvy8CB2jNeM1E9i9mXMAQ==", 301 | "engines": { 302 | "node": ">= 0.6" 303 | } 304 | }, 305 | "node_modules/string_decoder": { 306 | "version": "1.1.1", 307 | "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", 308 | "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", 309 | "dependencies": { 310 | "safe-buffer": "~5.1.0" 311 | } 312 | }, 313 | "node_modules/supports-preserve-symlinks-flag": { 314 | "version": "1.0.0", 315 | "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", 316 | "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", 317 | "engines": { 318 | "node": ">= 0.4" 319 | }, 320 | "funding": { 321 | "url": "https://github.com/sponsors/ljharb" 322 | } 323 | }, 324 | "node_modules/tarn": { 325 | "version": "3.0.2", 326 | "resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.2.tgz", 327 | "integrity": "sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==", 328 | "engines": { 329 | "node": ">=8.0.0" 330 | } 331 | }, 332 | "node_modules/tildify": { 333 | "version": "2.0.0", 334 | "resolved": "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz", 335 | "integrity": "sha512-Cc+OraorugtXNfs50hU9KS369rFXCfgGLpfCfvlc+Ud5u6VWmUQsOAa9HbTvheQdYnrdJqqv1e5oIqXppMYnSw==", 336 | "engines": { 337 | "node": ">=8" 338 | } 339 | }, 340 | "node_modules/util-deprecate": { 341 | "version": "1.0.2", 342 | "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", 343 | "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" 344 | } 345 | } 346 | } 347 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "anilist-crawler", 3 | "version": "1.3.0", 4 | "description": "", 5 | "main": "index.js", 6 | "type": "module", 7 | "scripts": { 8 | "prettier": "prettier", 9 | "format": "prettier --write \"**/*.js\"", 10 | "lint": "prettier --check \"**/*.js\"", 11 | "test": "prettier --check \"**/*.js\"" 12 | }, 13 | "repository": { 14 | "type": "git", 15 | "url": "git+https://github.com/soruly/anilist-crawler.git" 16 | }, 17 | "author": "soruly", 18 | "license": "MIT", 19 | "bugs": { 20 | "url": "https://github.com/soruly/anilist-crawler/issues" 21 | }, 22 | "homepage": "https://github.com/soruly/anilist-crawler#readme", 23 | "dependencies": { 24 | "knex": "^3.1.0", 25 | "mysql": "^2.18.1" 26 | }, 27 | "devDependencies": { 28 | "prettier": "^3.5.3" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /query.graphql: -------------------------------------------------------------------------------- 1 | query ($page: Int = 1, $perPage: Int = 1, $id: Int, $type: MediaType = ANIME) { 2 | Page(page: $page, perPage: $perPage) { 3 | pageInfo { 4 | total 5 | perPage 6 | currentPage 7 | lastPage 8 | hasNextPage 9 | } 10 | media(id: $id, type: $type) { 11 | id 12 | idMal 13 | title { 14 | native 15 | romaji 16 | english 17 | } 18 | type 19 | format 20 | status 21 | description 22 | startDate { 23 | year 24 | month 25 | day 26 | } 27 | endDate { 28 | year 29 | month 30 | day 31 | } 32 | season 33 | seasonYear 34 | seasonInt 35 | episodes 36 | duration 37 | chapters 38 | volumes 39 | countryOfOrigin 40 | isLicensed 41 | source 42 | hashtag 43 | trailer { 44 | id 45 | site 46 | thumbnail 47 | } 48 | updatedAt 49 | coverImage { 50 | extraLarge 51 | large 52 | medium 53 | color 54 | } 55 | bannerImage 56 | genres 57 | synonyms 58 | averageScore 59 | meanScore 60 | popularity 61 | favourites 62 | tags { 63 | id 64 | name 65 | description 66 | category 67 | rank 68 | isGeneralSpoiler 69 | isMediaSpoiler 70 | isAdult 71 | } 72 | relations { 73 | edges { 74 | node { 75 | id 76 | title { 77 | native 78 | } 79 | } 80 | relationType 81 | } 82 | } 83 | characters { 84 | edges { 85 | role 86 | node { 87 | id 88 | name { 89 | first 90 | last 91 | native 92 | alternative 93 | } 94 | image { 95 | large 96 | medium 97 | } 98 | siteUrl 99 | } 100 | voiceActors(language: JAPANESE) { 101 | id 102 | name { 103 | first 104 | last 105 | native 106 | } 107 | language 108 | image { 109 | large 110 | medium 111 | } 112 | siteUrl 113 | } 114 | } 115 | } 116 | staff { 117 | edges { 118 | role 119 | node { 120 | id 121 | name { 122 | first 123 | last 124 | native 125 | } 126 | language 127 | image { 128 | large 129 | medium 130 | } 131 | description 132 | siteUrl 133 | } 134 | } 135 | } 136 | studios { 137 | edges { 138 | isMain 139 | node { 140 | id 141 | name 142 | siteUrl 143 | } 144 | } 145 | } 146 | isAdult 147 | airingSchedule { 148 | edges { 149 | id 150 | node { 151 | id 152 | airingAt 153 | episode 154 | mediaId 155 | media { 156 | id 157 | } 158 | } 159 | } 160 | } 161 | externalLinks { 162 | id 163 | url 164 | site 165 | } 166 | streamingEpisodes { 167 | title 168 | thumbnail 169 | url 170 | site 171 | } 172 | rankings { 173 | id 174 | rank 175 | type 176 | format 177 | year 178 | season 179 | allTime 180 | context 181 | } 182 | rankings { 183 | id 184 | rank 185 | type 186 | format 187 | year 188 | season 189 | allTime 190 | context 191 | } 192 | recommendations { 193 | edges { 194 | node { 195 | id 196 | rating 197 | media { 198 | id 199 | title { 200 | native 201 | } 202 | } 203 | mediaRecommendation { 204 | id 205 | title { 206 | native 207 | } 208 | } 209 | } 210 | } 211 | } 212 | stats { 213 | scoreDistribution { 214 | score 215 | amount 216 | } 217 | statusDistribution { 218 | status 219 | amount 220 | } 221 | } 222 | siteUrl 223 | modNotes 224 | } 225 | } 226 | } 227 | --------------------------------------------------------------------------------