├── .devcontainer ├── aliases ├── history ├── profile ├── inputrc ├── Dockerfile ├── devcontainer.json └── base.Dockerfile ├── .DS_Store ├── extractSources.txt ├── package.json ├── README.md ├── supportedSources.js ├── items-base.js ├── races.js ├── items.js ├── .gitignore ├── spells.js ├── monsters.js └── classes.js /.devcontainer/aliases: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.devcontainer/history: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.devcontainer/profile: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rogueturnip/5eImport/HEAD/.DS_Store -------------------------------------------------------------------------------- /.devcontainer/inputrc: -------------------------------------------------------------------------------- 1 | "\e[A": history-search-backward 2 | "\e[B": history-search-forward -------------------------------------------------------------------------------- /extractSources.txt: -------------------------------------------------------------------------------- 1 | Must use local files from /data/ 2 | Was used to get sources from class files 3 | 4 | cat * | tr -d " \t," |grep \"source\" | sort --unique 5 | -------------------------------------------------------------------------------- /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18, 16, 14, 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster 2 | ARG VARIANT 3 | FROM mcr.microsoft.com/vscode/devcontainers/javascript-node:0-${VARIANT} 4 | 5 | RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ 6 | && apt-get -y install --no-install-recommends git-core bash-completion 7 | 8 | 9 | COPY ./history /home/node/.bash_history 10 | 11 | COPY ./profile /home/node/.bash_profile 12 | 13 | COPY ./inputrc /home/node/.inputrc 14 | 15 | COPY ./aliases /home/node/.bash_aliases 16 | RUN alias 17 | 18 | RUN echo $VARIANT 19 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "5eimport", 3 | "version": "1.0.0", 4 | "description": "Import JSON files from 5eTools to a mongodb for other use. Combines each with their fluff file.", 5 | "main": "index.js", 6 | "type": "module", 7 | "scripts": { 8 | "monsters": "node ./monsters.js", 9 | "items-base": "node ./items-base.js", 10 | "items": "node ./items.js", 11 | "spells": "node ./spells.js", 12 | "races": "node ./races.js", 13 | "classes": "node ./classes.js", 14 | "test": "echo \"Error: no test specified\" && exit 1" 15 | }, 16 | "author": "Tony Mauro (tony@mauro.ca)", 17 | "license": "ISC", 18 | "dependencies": { 19 | "axios": "^0.27.2", 20 | "dotenv": "^16.0.1", 21 | "lodash": "^4.17.21", 22 | "mongodb": "^4.8.1", 23 | "path": "^0.12.7" 24 | }, 25 | "devDependencies": {} 26 | } 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 5eImport 2 | 3 | This tool is designed to import the great JSON files from the 5e.tools data into a MongoDB collection. It combines all the files that would be related to the data being imported so items can be stored as a single document. 4 | It also creates a unique id based on the name-source-page of the item. 5 | 6 | Specific sources can be selected in the import using the arrays in supportedSources.js 7 | 8 | The original JSON has not been modified to allow for easy updates into the database and to allow more generic parsing and rendering to be possible. 9 | 10 | Currently support 11 | 12 | - bestiary 13 | - items and items base 14 | - spells 15 | - races 16 | - classes 17 | 18 | ### .ENV file 19 | 20 | ``` 21 | MONGODB="mongodb+srv://user:password@localhost?retryWrites=true&w=majority" 22 | DBNAME="5e" 23 | ``` 24 | 25 | ### Running imports 26 | 27 | ``` 28 | npm run items 29 | npm run items-base 30 | npm run spells 31 | npm run monsters 32 | npm run races 33 | npm run classes 34 | ``` 35 | 36 | ## Thanks to the 5e Tools team for some great data to work with. https://5e.tools 37 | -------------------------------------------------------------------------------- /supportedSources.js: -------------------------------------------------------------------------------- 1 | export const spellSources = [ 2 | "AI", 3 | "EGW", 4 | "GGR", 5 | "IDRotF", 6 | "LLK", 7 | "PHB", 8 | "TCE", 9 | "XGE", 10 | ]; 11 | 12 | export const classSources = [ 13 | "DMG", 14 | "EGW", 15 | "ERLW", 16 | "GGR", 17 | "MOT", 18 | "PHB", 19 | "PSA", 20 | "PSK", 21 | "SCAG", 22 | "TCE", 23 | "XGE", 24 | ]; 25 | 26 | export const bestiarySources = [ 27 | "AI", 28 | "BGDIA", 29 | "CM", 30 | "CoS", 31 | "DC", 32 | "DIP", 33 | "DMG", 34 | "EGW", 35 | "ERLW", 36 | "ESK", 37 | "GGR", 38 | "GoS", 39 | "HtfT", 40 | "HoL", 41 | "HotDQ", 42 | "IMR", 43 | "KKR", 44 | "LLK", 45 | "LMoP", 46 | "LR", 47 | "Mag", 48 | "MFF", 49 | "MM", 50 | "MOT", 51 | "MTF", 52 | "OotA", 53 | "OoW", 54 | "PSA", 55 | "PSD", 56 | "PSI", 57 | "PSK", 58 | "PSX", 59 | "PSZ", 60 | "PHB", 61 | "PotA", 62 | "RMBRE", 63 | "RoT", 64 | "SADS", 65 | "SDW", 66 | "SKT", 67 | "SLW", 68 | "TCE", 69 | "TTP", 70 | "TftYP", 71 | "ToA", 72 | "VGM", 73 | "VRGR", 74 | "XGE", 75 | "WDH", 76 | "WDMM", 77 | ]; 78 | -------------------------------------------------------------------------------- /items-base.js: -------------------------------------------------------------------------------- 1 | import _ from "lodash"; 2 | import axios from "axios"; 3 | import { default as mongodb } from "mongodb"; 4 | 5 | import dotenv from "dotenv"; 6 | dotenv.config(); 7 | 8 | const uri = process.env.MONGODB; 9 | const client = new mongodb.MongoClient(uri, { 10 | useNewUrlParser: true, 11 | useUnifiedTopology: true, 12 | }); 13 | 14 | const main = async () => { 15 | const responseItems = await axios.get( 16 | "https://5e.tools/data/items-base.json" 17 | ); 18 | try { 19 | const items = responseItems.data.baseitem; 20 | console.log(`count ${items.length}`); 21 | const newItems = await Promise.all( 22 | items.map((item) => { 23 | return { 24 | id: `${item.name.replace(/\W/g, "")}-${item.source}-${ 25 | item.page 26 | }`.toLowerCase(), 27 | base: true, 28 | ...item, 29 | }; 30 | }) 31 | ); 32 | return newItems; 33 | } catch (error) { 34 | throw new Error(error); 35 | } 36 | }; 37 | 38 | (async () => { 39 | try { 40 | await client.connect(); 41 | const dbo = client.db(process.env.DBNAME); 42 | var text = await main(); 43 | await Promise.all( 44 | text.map(async (item) => { 45 | const query = { id: item.id }; 46 | const update = { $set: item }; 47 | const options = { upsert: true }; 48 | await dbo.collection("items").updateOne(query, update, options); 49 | }) 50 | ); 51 | } catch (e) { 52 | // Deal with the fact the chain failed 53 | console.log(e); 54 | } finally { 55 | await client.close(); 56 | } 57 | })(); 58 | -------------------------------------------------------------------------------- /races.js: -------------------------------------------------------------------------------- 1 | import _ from "lodash"; 2 | import axios from "axios"; 3 | import { default as mongodb } from "mongodb"; 4 | 5 | import dotenv from "dotenv"; 6 | dotenv.config(); 7 | 8 | const uri = process.env.MONGODB; 9 | const client = new mongodb.MongoClient(uri, { 10 | useNewUrlParser: true, 11 | useUnifiedTopology: true, 12 | }); 13 | 14 | const main = async () => { 15 | const responseItems = await axios.get("https://5e.tools/data/races.json"); 16 | const responseItemsFluff = await axios.get( 17 | "https://5e.tools/data/fluff-races.json" 18 | ); 19 | try { 20 | const races = responseItems.data.race; 21 | const fluff = responseItemsFluff.data.raceFluff; 22 | console.log(`count items ${races.length} fluff ${fluff.length}`); 23 | const newRaces = await Promise.all( 24 | races.map((item) => { 25 | let itemFluff = {}; 26 | if (item.hasFluff) { 27 | itemFluff = _.find(fluff, { name: item.name, source: item.source }); 28 | } 29 | return { 30 | id: `${item.name.replace(/\W/g, "")}-${item.source}-${ 31 | item.page 32 | }`.toLowerCase(), 33 | ...item, 34 | images: itemFluff.images || [], 35 | entriesFluff: itemFluff.entries || [], 36 | }; 37 | }) 38 | ); 39 | return newRaces; 40 | } catch (error) { 41 | throw new Error(error); 42 | } 43 | }; 44 | 45 | (async () => { 46 | try { 47 | await client.connect(); 48 | const dbo = client.db(process.env.DBNAME); 49 | var text = await main(); 50 | await Promise.all( 51 | text.map(async (item) => { 52 | const query = { id: item.id }; 53 | const update = { $set: item }; 54 | const options = { upsert: true }; 55 | await dbo.collection("races").updateOne(query, update, options); 56 | }) 57 | ); 58 | } catch (e) { 59 | // Deal with the fact the chain failed 60 | console.log(e); 61 | } finally { 62 | await client.close(); 63 | } 64 | })(); 65 | -------------------------------------------------------------------------------- /items.js: -------------------------------------------------------------------------------- 1 | import _ from "lodash"; 2 | import axios from "axios"; 3 | import { default as mongodb } from "mongodb"; 4 | 5 | import dotenv from "dotenv"; 6 | dotenv.config(); 7 | 8 | const uri = process.env.MONGODB; 9 | const client = new mongodb.MongoClient(uri, { 10 | useNewUrlParser: true, 11 | useUnifiedTopology: true, 12 | }); 13 | 14 | const main = async () => { 15 | const responseItems = await axios.get("https://5e.tools/data/items.json"); 16 | const responseItemsFluff = await axios.get( 17 | "https://5e.tools/data/fluff-items.json" 18 | ); 19 | try { 20 | const items = responseItems.data.item; 21 | const fluff = responseItemsFluff.data.itemFluff; 22 | console.log(`count items ${items.length} fluff ${fluff.length}`); 23 | const newItems = await Promise.all( 24 | items.map((item) => { 25 | let itemFluff = {}; 26 | if (item.hasFluffImages) { 27 | itemFluff = _.find(fluff, { name: item.name }); 28 | } 29 | return { 30 | id: `${item.name.replace(/\W/g, "")}-${item.source}-${ 31 | item.page 32 | }`.toLowerCase(), 33 | base: false, 34 | ...item, 35 | images: itemFluff?.images || [], 36 | entriesFluff: itemFluff?.entries || [], 37 | }; 38 | }) 39 | ); 40 | return newItems; 41 | } catch (error) { 42 | throw new Error(error); 43 | } 44 | }; 45 | 46 | (async () => { 47 | try { 48 | await client.connect(); 49 | const dbo = client.db(process.env.DBNAME); 50 | var text = await main(); 51 | await Promise.all( 52 | text.map(async (item) => { 53 | const query = { id: item.id }; 54 | const update = { $set: item }; 55 | const options = { upsert: true }; 56 | await dbo.collection("items").updateOne(query, update, options); 57 | }) 58 | ); 59 | } catch (e) { 60 | // Deal with the fact the chain failed 61 | console.log(e); 62 | } finally { 63 | await client.close(); 64 | } 65 | })(); 66 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | // For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: 2 | // https://github.com/microsoft/vscode-dev-containers/tree/v0.241.1/containers/javascript-node 3 | { 4 | "name": "Node.js", 5 | "build": { 6 | "dockerfile": "Dockerfile", 7 | // Update 'VARIANT' to pick a Node version: 18, 16, 14. 8 | // Append -bullseye or -buster to pin to an OS version. 9 | // Use -bullseye variants on local arm64/Apple Silicon. 10 | "args": { "VARIANT": "18" } 11 | }, 12 | 13 | // Configure tool-specific properties. 14 | "customizations": { 15 | // Configure properties specific to VS Code. 16 | "vscode": { 17 | // Add the IDs of extensions you want installed when the container is created. 18 | "extensions": ["dbaeumer.vscode-eslint", "esbenp.prettier-vscode"] 19 | } 20 | }, 21 | 22 | // Use 'forwardPorts' to make a list of ports inside the container available locally. 23 | // "forwardPorts": [], 24 | 25 | // Use 'postCreateCommand' to run commands after the container is created. 26 | // "postCreateCommand": "yarn install", 27 | "postCreateCommand": "sudo chown node node_modules", 28 | // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. 29 | "remoteUser": "node", 30 | "features": { 31 | "git": "latest", 32 | "aws-cli": "latest" 33 | }, 34 | "hostRequirements": { 35 | "cpus": 2, 36 | "memory": "4gb", 37 | "storage": "16gb" 38 | }, 39 | "mounts": [ 40 | "source=${env:HOME}/.aws,target=/home/node/.aws,type=bind", 41 | "source=${env:HOME}/.gitconfig,target=/home/node/.gitconfig,type=bind", 42 | "source=${env:HOME}/.ssh,target=/home/node/.ssh,type=bind", 43 | "source=try-node-node_modules,target=${containerWorkspaceFolder}/node_modules,type=volume" 44 | // "source=vscode-extensions,target=/home/node/.vscode-server/extensions,type=volume", 45 | // And/or for VS Code Insiders 46 | // "source=vscode-extensions-insider,target=/home/node/.vscode-server-insiders/extensions,type=volume" 47 | ] 48 | } 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | 9 | # Diagnostic reports (https://nodejs.org/api/report.html) 10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 11 | 12 | # Runtime data 13 | pids 14 | *.pid 15 | *.seed 16 | *.pid.lock 17 | 18 | # Directory for instrumented libs generated by jscoverage/JSCover 19 | lib-cov 20 | 21 | # Coverage directory used by tools like istanbul 22 | coverage 23 | *.lcov 24 | 25 | # nyc test coverage 26 | .nyc_output 27 | 28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 29 | .grunt 30 | 31 | # Bower dependency directory (https://bower.io/) 32 | bower_components 33 | 34 | # node-waf configuration 35 | .lock-wscript 36 | 37 | # Compiled binary addons (https://nodejs.org/api/addons.html) 38 | build/Release 39 | 40 | # Dependency directories 41 | node_modules/ 42 | jspm_packages/ 43 | 44 | # TypeScript v1 declaration files 45 | typings/ 46 | 47 | # TypeScript cache 48 | *.tsbuildinfo 49 | 50 | # Optional npm cache directory 51 | .npm 52 | 53 | # Optional eslint cache 54 | .eslintcache 55 | 56 | # Microbundle cache 57 | .rpt2_cache/ 58 | .rts2_cache_cjs/ 59 | .rts2_cache_es/ 60 | .rts2_cache_umd/ 61 | 62 | # Optional REPL history 63 | .node_repl_history 64 | 65 | # Output of 'npm pack' 66 | *.tgz 67 | 68 | # Yarn Integrity file 69 | .yarn-integrity 70 | 71 | # dotenv environment variables file 72 | .env 73 | .env.test 74 | 75 | # parcel-bundler cache (https://parceljs.org/) 76 | .cache 77 | 78 | # Next.js build output 79 | .next 80 | 81 | # Nuxt.js build / generate output 82 | .nuxt 83 | dist 84 | 85 | # Gatsby files 86 | .cache/ 87 | # Comment in the public line in if your project uses Gatsby and *not* Next.js 88 | # https://nextjs.org/blog/next-9-1#public-directory-support 89 | # public 90 | 91 | # vuepress build output 92 | .vuepress/dist 93 | 94 | # Serverless directories 95 | .serverless/ 96 | 97 | # FuseBox cache 98 | .fusebox/ 99 | 100 | # DynamoDB Local files 101 | .dynamodb/ 102 | 103 | # TernJS port file 104 | .tern-port 105 | -------------------------------------------------------------------------------- /spells.js: -------------------------------------------------------------------------------- 1 | import _ from "lodash"; 2 | import axios from "axios"; 3 | import { default as mongodb } from "mongodb"; 4 | import { spellSources } from "./supportedSources.js"; 5 | 6 | import dotenv from "dotenv"; 7 | dotenv.config(); 8 | 9 | const uri = process.env.MONGODB; 10 | const client = new mongodb.MongoClient(uri, { 11 | useNewUrlParser: true, 12 | useUnifiedTopology: true, 13 | }); 14 | 15 | const main = async (filename) => { 16 | const responseSpells = await axios.get( 17 | `https://5e.tools/data/spells/${filename}` 18 | ); 19 | const responseSpellsFluff = await axios.get( 20 | `https://5e.tools/data/spells/fluff-${filename}` 21 | ); 22 | try { 23 | const spells = responseSpells.data.spell; 24 | const fluff = responseSpellsFluff.data.spellFluff; 25 | console.log( 26 | `counts ${filename} spells ${spells.length} fluff ${fluff.length}` 27 | ); 28 | const newSpells = await Promise.all( 29 | spells.map((item) => { 30 | let itemFluff = {}; 31 | if (item.hasFluff || item.hasFluffImages) { 32 | itemFluff = _.find(fluff, { name: item.name }); 33 | } 34 | return { 35 | id: `${item.name.replace(/\W/g, "")}-${item.source}-${ 36 | item.page 37 | }`.toLowerCase(), 38 | ...item, 39 | images: itemFluff.images || [], 40 | entriesFluff: itemFluff.entries || [], 41 | }; 42 | }) 43 | ); 44 | return newSpells; 45 | } catch (error) { 46 | throw new Error(Error); 47 | } 48 | }; 49 | 50 | (async () => { 51 | try { 52 | await client.connect(); 53 | const dbo = client.db(process.env.DBNAME); 54 | const responseIndex = await axios.get( 55 | "https://5e.tools/data/spells/index.json" 56 | ); 57 | await Promise.allSettled( 58 | spellSources.map(async (source) => { 59 | const sourceJson = responseIndex.data[source] || null; 60 | if (sourceJson !== null) { 61 | const result = await main(sourceJson); 62 | await Promise.all( 63 | result.map(async (item) => { 64 | const query = { id: item.id }; 65 | const update = { $set: item }; 66 | const options = { upsert: true }; 67 | await dbo.collection("spells").updateOne(query, update, options); 68 | }) 69 | ); 70 | } 71 | }) 72 | ); 73 | } catch (e) { 74 | // Deal with the fact the chain failed 75 | console.log(e); 76 | } finally { 77 | await client.close(); 78 | } 79 | })(); 80 | -------------------------------------------------------------------------------- /.devcontainer/base.Dockerfile: -------------------------------------------------------------------------------- 1 | # [Choice] Node.js version (use -bullseye variants on local arm64/Apple Silicon): 18-bullseye, 16-bullseye, 14-bullseye, 18-buster, 16-buster, 14-buster 2 | ARG VARIANT=16-bullseye 3 | FROM node:${VARIANT} 4 | 5 | # [Option] Install zsh 6 | ARG INSTALL_ZSH="true" 7 | # [Option] Upgrade OS packages to their latest versions 8 | ARG UPGRADE_PACKAGES="true" 9 | 10 | # Install needed packages, yarn, nvm and setup non-root user. Use a separate RUN statement to add your own dependencies. 11 | ARG USERNAME=node 12 | ARG USER_UID=1000 13 | ARG USER_GID=$USER_UID 14 | ARG NPM_GLOBAL=/usr/local/share/npm-global 15 | ENV NVM_DIR=/usr/local/share/nvm 16 | ENV NVM_SYMLINK_CURRENT=true \ 17 | PATH=${NPM_GLOBAL}/bin:${NVM_DIR}/current/bin:${PATH} 18 | COPY library-scripts/*.sh library-scripts/*.env /tmp/library-scripts/ 19 | RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ 20 | # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 21 | && apt-get purge -y imagemagick imagemagick-6-common \ 22 | # Install common packages, non-root user, update yarn and install nvm 23 | && bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "${USER_UID}" "${USER_GID}" "${UPGRADE_PACKAGES}" "true" "true" \ 24 | # Install yarn, nvm 25 | && rm -rf /opt/yarn-* /usr/local/bin/yarn /usr/local/bin/yarnpkg \ 26 | && bash /tmp/library-scripts/node-debian.sh "${NVM_DIR}" "none" "${USERNAME}" \ 27 | # Configure global npm install location, use group to adapt to UID/GID changes 28 | && if ! cat /etc/group | grep -e "^npm:" > /dev/null 2>&1; then groupadd -r npm; fi \ 29 | && usermod -a -G npm ${USERNAME} \ 30 | && umask 0002 \ 31 | && mkdir -p ${NPM_GLOBAL} \ 32 | && touch /usr/local/etc/npmrc \ 33 | && chown ${USERNAME}:npm ${NPM_GLOBAL} /usr/local/etc/npmrc \ 34 | && chmod g+s ${NPM_GLOBAL} \ 35 | && npm config -g set prefix ${NPM_GLOBAL} \ 36 | && sudo -u ${USERNAME} npm config -g set prefix ${NPM_GLOBAL} \ 37 | # Install eslint 38 | && su ${USERNAME} -c "umask 0002 && npm install -g eslint" \ 39 | && npm cache clean --force > /dev/null 2>&1 \ 40 | # Install python-is-python3 on bullseye to prevent node-gyp regressions 41 | && . /etc/os-release \ 42 | && if [ "${VERSION_CODENAME}" = "bullseye" ]; then apt-get -y install --no-install-recommends python-is-python3; fi \ 43 | # Clean up 44 | && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* /root/.gnupg /tmp/library-scripts 45 | 46 | # [Optional] Uncomment this section to install additional OS packages. 47 | # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ 48 | # && apt-get -y install --no-install-recommends 49 | 50 | # [Optional] Uncomment if you want to install an additional version of node using nvm 51 | # ARG EXTRA_NODE_VERSION=10 52 | # RUN su node -c "source /usr/local/share/nvm/nvm.sh && nvm install ${EXTRA_NODE_VERSION}" 53 | 54 | # [Optional] Uncomment if you want to install more global node modules 55 | # RUN su node -c "npm install -g "" -------------------------------------------------------------------------------- /monsters.js: -------------------------------------------------------------------------------- 1 | import _ from "lodash"; 2 | import axios from "axios"; 3 | import { default as mongodb } from "mongodb"; 4 | import { bestiarySources } from "./supportedSources.js"; 5 | 6 | import dotenv from "dotenv"; 7 | dotenv.config(); 8 | const uri = process.env.MONGODB; 9 | const client = new mongodb.MongoClient(uri, { 10 | useNewUrlParser: true, 11 | useUnifiedTopology: true, 12 | }); 13 | 14 | const main = async (filename) => { 15 | const responseBestiary = await axios.get( 16 | `https://5e.tools/data/bestiary/${filename}` 17 | ); 18 | const responseBestiaryFluff = await axios.get( 19 | `https://5e.tools/data/bestiary/fluff-${filename}` 20 | ); 21 | const responseLegendaryGroup = await axios.get( 22 | `https://5e.tools/data/bestiary/legendarygroups.json` 23 | ); 24 | try { 25 | const monster = responseBestiary.data.monster; 26 | const fluff = responseBestiaryFluff.data.monsterFluff; 27 | const legendaryGroup = responseLegendaryGroup.data.legendaryGroup; 28 | console.log( 29 | `counts ${filename} monster ${monster.length} fluff ${fluff.length} legendaryGroup ${legendaryGroup.length}` 30 | ); 31 | const newMonster = await Promise.all( 32 | monster.map((item) => { 33 | let lairActions = []; 34 | let regionalEffects = []; 35 | let mythicEncounter = []; 36 | let itemFluff = {}; 37 | if (item.hasFluff || item.hasFluffImages) { 38 | itemFluff = _.find(fluff, { name: item.name }); 39 | } 40 | if (!_.isEmpty(item.legendaryGroup)) { 41 | const findGroup = _.find(legendaryGroup, { 42 | name: item.name, 43 | source: item.source, 44 | }); 45 | lairActions = findGroup?.lairActions || []; 46 | regionalEffects = findGroup?.regionalEffects || []; 47 | mythicEncounter = findGroup?.mythicEncounter || []; 48 | } 49 | item["saves"] = item.save || {}; 50 | return { 51 | id: `${item.name.replace(/\W/g, "")}-${item.source}-${ 52 | item.page 53 | }`.toLowerCase(), 54 | ..._.omit(item, ["save"]), 55 | images: itemFluff.images || [], 56 | entriesFluff: itemFluff.entries || [], 57 | lairActions, 58 | regionalEffects, 59 | mythicEncounter, 60 | }; 61 | }) 62 | ); 63 | return newMonster; 64 | } catch (error) { 65 | console.log(error); 66 | throw new Error(Error); 67 | } 68 | }; 69 | 70 | (async () => { 71 | try { 72 | await client.connect(); 73 | const dbo = client.db(process.env.DBNAME); 74 | const responseIndex = await axios.get( 75 | "https://5e.tools/data/bestiary/index.json" 76 | ); 77 | await Promise.allSettled( 78 | bestiarySources.map(async (source) => { 79 | const sourceJson = responseIndex.data[source] || null; 80 | if (sourceJson !== null) { 81 | const result = await main(sourceJson); 82 | await Promise.all( 83 | result.map(async (item) => { 84 | const query = { id: item.id }; 85 | const update = { $set: item }; 86 | const options = { upsert: true }; 87 | await dbo 88 | .collection("monsters") 89 | .updateOne(query, update, options); 90 | }) 91 | ); 92 | } 93 | }) 94 | ); 95 | } catch (e) { 96 | console.log(e); 97 | } finally { 98 | await client.close(); 99 | } 100 | })(); 101 | -------------------------------------------------------------------------------- /classes.js: -------------------------------------------------------------------------------- 1 | import _ from "lodash"; 2 | import axios from "axios"; 3 | import { default as mongodb } from "mongodb"; 4 | import { classSources } from "./supportedSources.js"; 5 | 6 | import dotenv from "dotenv"; 7 | dotenv.config(); 8 | 9 | const uri = process.env.MONGODB; 10 | const client = new mongodb.MongoClient(uri, { 11 | useNewUrlParser: true, 12 | useUnifiedTopology: true, 13 | }); 14 | 15 | const main = async () => { 16 | await client.connect(); 17 | const dbo = client.db(process.env.DBNAME); 18 | const responseIndex = await axios.get( 19 | `https://5e.tools/data/class/index.json` 20 | ); 21 | const classFiles = Object.values(responseIndex.data); 22 | try { 23 | await Promise.all( 24 | classFiles.map(async (classFile) => { 25 | console.log("class file", classFile); 26 | const responseClass = await axios.get( 27 | `https://5e.tools/data/class/${classFile}` 28 | ); 29 | const classItems = responseClass.data.class || []; 30 | const classFeatures = responseClass.data.classFeature || []; 31 | const subclassFeatures = responseClass.data.subclassFeature || []; 32 | 33 | const newClassItems = classItems.map((classItem) => { 34 | // don't process classes from sources not supported 35 | if (!classSources.includes(classItem.source)) return null; 36 | // merge class features into one structure 37 | const newClassFeatures = classItem?.classFeatures.map((feature) => { 38 | if (typeof feature === "string") { 39 | let values = feature.split("|"); 40 | values[2] = _.isEmpty(values[2]) ? "PHB" : values[2]; 41 | values[4] = _.isEmpty(values[4]) ? values[2] : values[4]; 42 | // don't process class features from sources not supported 43 | if ( 44 | !classSources.includes(values[2]) || 45 | !classSources.includes(values[4]) 46 | ) 47 | return null; 48 | return { 49 | gainSubclassFeature: false, 50 | ..._.find(classFeatures, { 51 | name: values[0], 52 | className: values[1], 53 | classSource: values[2], 54 | level: parseInt(values[3]), 55 | }), 56 | }; 57 | } else { 58 | let values = feature.classFeature.split("|"); 59 | values[2] = _.isEmpty(values[2]) ? "PHB" : values[2]; 60 | values[4] = _.isEmpty(values[4]) ? values[2] : values[4]; 61 | // don't process class features from sources not supported 62 | if ( 63 | !classSources.includes(values[2]) || 64 | !classSources.includes(values[4]) 65 | ) 66 | return null; 67 | return { 68 | gainSubclassFeature: feature.gainSubclassFeature, 69 | ..._.find(classFeatures, { 70 | name: values[0], 71 | className: values[1], 72 | classSource: values[2], 73 | level: parseInt(values[3]), 74 | }), 75 | }; 76 | } 77 | }); 78 | // merge subclass features into one structure 79 | const newSubclass = classItem?.subclasses?.map((subclass) => { 80 | // don't process classes from sources not supported 81 | if (!classSources.includes(subclass.source)) return null; 82 | // now loop through subclassFeatures array 83 | const newFeatures = subclass?.subclassFeatures?.map((feature) => { 84 | let values = feature.split("|"); 85 | values[4] = _.isEmpty(values[4]) ? "PHB" : values[4]; 86 | // don't process class features from sources not supported 87 | if (!classSources.includes(values[4])) return null; 88 | return { 89 | ..._.find(subclassFeatures, { 90 | name: values[0], 91 | className: values[1], 92 | subclassSource: values[4], 93 | subclassShortName: values[3], 94 | level: parseInt(values[5]), 95 | }), 96 | }; 97 | }); 98 | return { 99 | ...subclass, 100 | subclassFeatures: _.isEmpty(newFeatures) ? [] : newFeatures, 101 | }; 102 | }); 103 | return { 104 | id: `${classItem.name.replace(/\W/g, "")}-${classItem?.source}-${ 105 | classItem?.page 106 | }`.toLowerCase(), 107 | ...classItem, 108 | classFeatures: !_.isEmpty(newClassFeatures) 109 | ? newClassFeatures.filter((o) => o) 110 | : [], 111 | subclasses: !_.isEmpty(newSubclass) 112 | ? newSubclass.filter((o) => o) 113 | : [], 114 | }; 115 | }); 116 | await Promise.all( 117 | newClassItems 118 | .filter((o) => o) 119 | .map(async (item) => { 120 | console.log(item.id); 121 | const query = { id: item.id }; 122 | const update = { $set: item }; 123 | const options = { upsert: true }; 124 | await dbo.collection("classes").updateOne(query, update, options); 125 | }) 126 | ); 127 | }) 128 | ); 129 | } catch (error) { 130 | throw new Error(error); 131 | } 132 | }; 133 | 134 | (async () => { 135 | try { 136 | await main(); 137 | } catch (e) { 138 | // Deal with the fact the chain failed 139 | console.log(e); 140 | } finally { 141 | await client.close(); 142 | } 143 | })(); 144 | --------------------------------------------------------------------------------