├── .editorconfig ├── .github └── FUNDING.yml ├── .gitignore ├── README.md ├── admin-api-server-cloudbuild.yaml ├── cronjobs ├── account_config_example.csv ├── azurlane_pull.sh ├── data_processor.sh ├── db_config.json ├── discord.sh ├── do_single_process.sh ├── dungeon_scraper.sh ├── export_data.sh ├── game_data │ ├── export_enemy_skills.sh │ ├── export_game_data.sh │ └── export_pad_data.sh ├── media │ ├── update_bgm_files.sh │ ├── update_image_files.sh │ ├── update_orb_styles_files.sh │ ├── update_story_files.sh │ └── update_voice_files.sh ├── pull_data.sh ├── run_loader.sh ├── secrets_example.sh ├── shared.sh ├── shared_root_example.sh └── sync_data.sh ├── docker ├── docker_db_config.json ├── docker_mysql.yml └── start_env.sh ├── etl ├── README.md ├── auto_dungeon_scrape.py ├── dadguide_proto │ ├── README.md │ └── enemy_skills_pb2.py ├── data_processor.py ├── default_db_config.json ├── egg_processor.py ├── export_spawns.py ├── media_copy.py ├── misc │ └── azurlane_image_download.py ├── pad │ ├── __init__.py │ ├── api │ │ ├── __init__.py │ │ ├── pad_api.py │ │ └── player_data.py │ ├── common │ │ ├── dungeon_types.py │ │ ├── icons.py │ │ ├── monster_id_mapping.py │ │ ├── pad_util.py │ │ ├── shared_types.py │ │ └── utils.py │ ├── db │ │ ├── db_util.py │ │ └── sql_item.py │ ├── dungeon │ │ └── wave_converter.py │ ├── raw │ │ ├── __init__.py │ │ ├── bonus.py │ │ ├── card.py │ │ ├── dungeon.py │ │ ├── enemy_skill.py │ │ ├── enemy_skills │ │ │ ├── debug_utils.py │ │ │ ├── enemy_skill_parser.py │ │ │ ├── enemy_skill_proto.py │ │ │ └── enemy_skillset_processor.py │ │ ├── exchange.py │ │ ├── extra_egg_machine.py │ │ ├── purchase.py │ │ ├── skill.py │ │ ├── skills │ │ │ ├── active_behaviors.py │ │ │ ├── active_skill_info.py │ │ │ ├── emoji_en │ │ │ │ ├── enemy_skill_text.py │ │ │ │ └── skill_common.py │ │ │ ├── en │ │ │ │ ├── active_skill_text.py │ │ │ │ ├── enemy_skill_text.py │ │ │ │ ├── leader_skill_text.py │ │ │ │ └── skill_common.py │ │ │ ├── enemy_skill_info.py │ │ │ ├── ja │ │ │ │ ├── active_skill_text.py │ │ │ │ ├── enemy_skill_text.py │ │ │ │ ├── leader_skill_text.py │ │ │ │ └── skill_common.py │ │ │ ├── ko │ │ │ │ ├── active_skill_text.py │ │ │ │ ├── enemy_skill_text.py │ │ │ │ ├── leader_skill_text.py │ │ │ │ └── skill_common.py │ │ │ ├── leader_skill_info.py │ │ │ ├── monster_skill_test.py │ │ │ ├── skill_common.py │ │ │ ├── skill_parser.py │ │ │ └── skill_text_typing.py │ │ └── wave.py │ ├── raw_processor │ │ ├── crossed_data.py │ │ ├── jp_replacements.py │ │ ├── merged_data.py │ │ └── merged_database.py │ ├── storage │ │ ├── awoken_skill.py │ │ ├── dungeon.py │ │ ├── egg_machine.py │ │ ├── egg_machines_monsters.py │ │ ├── encounter.py │ │ ├── enemy_skill.py │ │ ├── exchange.py │ │ ├── latent_skill.py │ │ ├── monster.py │ │ ├── monster_skill.py │ │ ├── purchase.py │ │ ├── rank_reward.py │ │ ├── schedule.py │ │ ├── series.py │ │ ├── skill_tag.py │ │ └── wave.py │ └── storage_processor │ │ ├── awoken_skill.json │ │ ├── awoken_skill_processor.py │ │ ├── dimension_processor.py │ │ ├── dungeon_content_processor.py │ │ ├── dungeon_processor.py │ │ ├── egg_machine_processor.py │ │ ├── enemy_skill.json │ │ ├── enemy_skill_processor.py │ │ ├── exchange_processor.py │ │ ├── latent_skill.json │ │ ├── latent_skill_processor.py │ │ ├── monster_processor.py │ │ ├── purchase_processor.py │ │ ├── purge_data_processor.py │ │ ├── rank_reward.csv │ │ ├── rank_reward_processor.py │ │ ├── schedule_processor.py │ │ ├── series.json │ │ ├── series_processor.py │ │ ├── shared_storage.py │ │ ├── skill_tag_active.json │ │ ├── skill_tag_leader.json │ │ ├── skill_tag_processor.py │ │ ├── timestamp_processor.py │ │ └── wave_processor.py ├── pad_data_pull.py ├── pad_dungeon_pull.py └── rebuild_enemy_skills.py ├── media_pipelines ├── assets │ ├── PADAnimatedGenerator.py │ ├── PADAnimationGenerator.py │ ├── PADHQImageDownload.py │ ├── PADIconGenerator.py │ ├── PADTextureDownload.py │ ├── PADTextureTool.py │ └── README.md └── extras │ ├── PADBGMDownload.py │ ├── PADDMSGParser.py │ ├── PADFullMediaDownload.py │ ├── PADOrbStylesDownload.py │ ├── PADStoryDownload.py │ ├── PADTextureTool.py │ └── PADVoiceDownload.py ├── mobile-api-server-cloudbuild.yaml ├── pit-cloudbuild.yaml ├── proto ├── README.md └── enemy_skills.proto ├── requirements.txt ├── schema ├── README.md ├── mysql.sql └── mysql2sqlite.sh ├── setup.sh ├── utils ├── data_exporter.py ├── download_db_from_prod.sh ├── download_db_with_waves_from_prod.sh ├── refresh_data.sh ├── restore_db_from_prod.sh └── restore_db_with_waves_from_prod.sh └── web ├── Dockerfile ├── README.md ├── admin_api_server.py ├── data └── utils.py ├── mobile_api_server.py ├── requirements.txt ├── serve.php └── serve_dadguide_data.py /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: tsubaki_bot 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: [ ] 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | include/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | output/ 25 | share/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | MANIFEST 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # pyenv 79 | .python-version 80 | 81 | # celery beat schedule file 82 | celerybeat-schedule 83 | 84 | # SageMath parsed files 85 | *.sage.py 86 | 87 | # Environments 88 | .env 89 | .venv 90 | env/ 91 | venv/ 92 | ENV/ 93 | env.bak/ 94 | venv.bak/ 95 | 96 | # Spyder project settings 97 | .spyderproject 98 | .spyproject 99 | 100 | # Rope project settings 101 | .ropeproject 102 | 103 | # mkdocs documentation 104 | /site 105 | 106 | # mypy 107 | .mypy_cache/ 108 | 109 | # Ignore intellij files 110 | .idea 111 | *.iml 112 | 113 | # Ignore data copied by refresh script 114 | pad_data 115 | 116 | # Ignore user-specific db config 117 | etl/db_config.json 118 | 119 | # Ignore secret pad files 120 | etl/pad/api/keygen.py 121 | etl/pad/api/dungeon_encoding.py 122 | cronjobs/secrets.sh 123 | cronjobs/account_config.csv 124 | 125 | # python venv 126 | bin/ 127 | 128 | .DS_Store 129 | 130 | *.orig 131 | 132 | # vim swap files 133 | *.swp 134 | 135 | # For Node 136 | package-lock.json 137 | node_modules/ 138 | 139 | pad-resources/* 140 | pad-game-data-slim/* 141 | /venv3/ 142 | /venv38/ 143 | pyvenv.cfg 144 | cronjobs/shared_root.sh 145 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dadguide-data 2 | 3 | ## Looking for exported data? 4 | 5 | Skip the rest of this and look in the `utils` directory for scripts that will let you access the exports, including the 6 | raw data and processed database. 7 | 8 | ## What's in here 9 | 10 | `docker` : Easier development setup in docker. 11 | 12 | `etl` : Contains the most important stuff, downloading, parsing, and processing the PAD game data. 13 | 14 | `images` : Image files used by DadGuide, latents, awakenings, etc. No monster icons/portraits. 15 | 16 | `media_pipelines` : Code for extracting and processing PAD media, including icons, portraits, voice lines, animations, 17 | etc. 18 | 19 | `proto` : Protocol buffer definitions used by the pipeline and DadGuide (mostly for enemy skills). 20 | 21 | `schema` : Stuff to do with mysql specifically. 22 | 23 | `utils` : Some random scripts for development purposes. If you want to access data exports, look here. 24 | 25 | `web` : The 'admin api' and 'mobile api' sanic servers, plus some stuff that serves the DadGuide API in prod. 26 | 27 | In the root directory are some Google Cloud Build files for building the mobile/api servers, and the requirements file 28 | that you should install if you want to use the ETL pipeline. 29 | 30 | ## Database setup 31 | 32 | MySql is used as a backend on the server, and SQLite for [Tsubaki Bot](https://github.com/TsubakiBotPad/pad-cogs). You can install the server yourself: 33 | 34 | ```bash 35 | sudo apt install mysql-workbench 36 | sudo apt install mysql-server 37 | ``` 38 | 39 | The `utils` folder has some scripts to download the exports and populate the databse. 40 | 41 | Alternatively you can use a docker script to start it up (see below). 42 | 43 | ### Docker based development 44 | 45 | In the `docker` directory are utilities to help you get working faster. You can download the database backup from 46 | production, start a mysql container, and restore the backup with a single command: 47 | 48 | ```bash 49 | docker/start_env.sh 50 | ``` 51 | 52 | ## Pipeline testing 53 | 54 | The utils directory contains a script called `data_exporter.py`. This script is run to generate the 55 | repository: https://github.com/TsubakiBotPad/pad-data-pipeline-export 56 | 57 | The server has a cron job which runs this periodically, commits, and pushes. 58 | 59 | If you're making changes, you should check out a copy of `pad-game-data-slim` at head and run the data exporter locally 60 | to confirm your changes had the effect you expected. 61 | 62 | ## API testing 63 | 64 | Production uses an Apache webserver with PHP that executes a python script. 65 | 66 | Development uses a sanic webserver (requires python 3.6 and sanic\_compress). Eventually production should 67 | probably `mod_proxy` through to this. 68 | 69 | The files used for both are in the `web` directory. 70 | 71 | ### Google Cloud Build pull-tests 72 | 73 | This didn't work well and is currently disabled. Should be re-enabled with just writing the `data_exporter.py` results 74 | and making sure it does not crash, probably. 75 | 76 | ### Future tests 77 | 78 | It would be really great to have a version that either did the mysql data load, or a sqlite one, or maybe just dumps the 79 | computed SQL to a file. The latter is probably not sufficient, since some steps require index lookups and we would have 80 | to fake those. 81 | -------------------------------------------------------------------------------- /admin-api-server-cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: 'gcr.io/cloud-builders/gcloud' 3 | id: 'setup' 4 | entrypoint: 'bash' 5 | args: 6 | - '-c' 7 | - | 8 | mkdir -p output 9 | mkdir -p dadguide-admin 10 | mkdir -p dadguide-flutter 11 | cp -r etl/dadguide_proto output/ 12 | cp -r etl/pad output/ 13 | cp -r web/data output/ 14 | cp web/admin_api_server.py output/ 15 | cp web/requirements.txt output/ 16 | cp web/Dockerfile output/ 17 | 18 | - name: 'gcr.io/cloud-builders/git' 19 | dir: 'dadguide-flutter' 20 | args: [ 'clone', 'https://github.com/nachoapps/dadguide-flutter', '.' ] 21 | 22 | - name: 'gcr.io/${PROJECT_ID}/flutter:beta' 23 | id: 'setup-dadguide-flutter' 24 | dir: 'dadguide-flutter' 25 | entrypoint: 'bash' 26 | args: 27 | - '-c' 28 | - | 29 | flutter pub get 30 | flutter packages pub run build_runner build 31 | 32 | - name: 'gcr.io/cloud-builders/git' 33 | dir: 'dadguide-admin' 34 | args: [ 'clone', 'https://github.com/nachoapps/dadguide-admin', '.' ] 35 | 36 | - name: 'gcr.io/${PROJECT_ID}/flutter:beta' 37 | id: 'setup-dadguide-admin' 38 | dir: 'dadguide-admin' 39 | entrypoint: 'bash' 40 | args: 41 | - '-c' 42 | - | 43 | flutter config --enable-web 44 | flutter pub get 45 | flutter packages pub run build_runner build 46 | 47 | - name: 'gcr.io/${PROJECT_ID}/flutter:beta' 48 | id: 'compile-flutter-web' 49 | dir: 'dadguide-admin' 50 | args: [ 'build', 'web' ] 51 | 52 | - name: 'gcr.io/cloud-builders/gcloud' 53 | id: 'move-web-folder' 54 | entrypoint: 'bash' 55 | args: [ '-c', 'mv dadguide-admin/build/web output/' ] 56 | 57 | - name: 'gcr.io/cloud-builders/gcloud' 58 | id: 'download-db-config' 59 | entrypoint: 'bash' 60 | args: [ '-c', 'gcloud secrets versions access latest --secret=prod_db_config > output/db_config.json' ] 61 | 62 | - name: 'gcr.io/cloud-builders/docker' 63 | dir: 'output' 64 | args: [ 65 | 'build', 66 | '--build-arg', 'SCRIPT_NAME=admin_api_server.py', 67 | '--build-arg', 'PORT=8000', 68 | '--build-arg', 'EXTRA_ARG=--web_dir=/server/web --es_dir=/server/es', 69 | '-t', 'gcr.io/$PROJECT_ID/admin-api-server', 70 | '.' 71 | ] 72 | 73 | images: [ 'gcr.io/$PROJECT_ID/admin-api-server' ] 74 | -------------------------------------------------------------------------------- /cronjobs/account_config_example.csv: -------------------------------------------------------------------------------- 1 | NA,A,D06E98F2-82F7-4D2B-88A9-84FDDB071E72,345184129,GREEN 2 | -------------------------------------------------------------------------------- /cronjobs/azurlane_pull.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | set -x 4 | 5 | cd "$(dirname "$0")" || exit 6 | source ./shared_root.sh 7 | source ./shared.sh 8 | source ./discord.sh 9 | source "${VENV_ROOT}/bin/activate" 10 | 11 | function error_exit() { 12 | hook_error "Azurlane pull failed <@&${NOTIFICATION_DISCORD_ROLE_ID}>" 13 | } 14 | 15 | function success_exit() { 16 | hook_info "Azurlane pull succeeded" 17 | } 18 | 19 | # Enable alerting to discord 20 | trap error_exit ERR 21 | trap success_exit EXIT 22 | 23 | flock -xn /tmp/azurlane.lck python3 "${ETL_DIR}/misc/azurlane_image_download.py" \ 24 | --output_dir="${DADGUIDE_DATA_DIR}/azurlane" 25 | 26 | echo "Syncing" 27 | ./sync_data.sh 28 | -------------------------------------------------------------------------------- /cronjobs/data_processor.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | set -x 4 | 5 | cd "$(dirname "$0")" || exit 6 | source ./shared_root.sh 7 | source ./shared.sh 8 | source ./discord.sh 9 | 10 | function human_fixes_check() { 11 | human_fixes_path="/tmp/dadguide_pipeline_human_fixes.txt" 12 | if [[ -s ${human_fixes_path} ]]; then 13 | echo "Alerting for human fixes" 14 | hook_warn "\`\`\`\n$(cat /tmp/dadguide_pipeline_human_fixes.txt \ 15 | | sed ':a;N;$!ba;s/\n/\\n/g' \ 16 | | head -c 1990)\n\`\`\`" 17 | else 18 | echo "No fixes required" 19 | fi 20 | } 21 | 22 | flock -xn /tmp/dg_processor.lck python3 "${ETL_DIR}/data_processor.py" \ 23 | --input_dir="${RAW_DIR}" \ 24 | --es_dir="${ES_DIR}" \ 25 | --media_dir="${IMG_DIR}" \ 26 | --output_dir="${DADGUIDE_DATA_DIR}/processed" \ 27 | --db_config="${DB_CONFIG}" \ 28 | --server=$1 \ 29 | --doupdates 30 | 31 | human_fixes_check 32 | -------------------------------------------------------------------------------- /cronjobs/db_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "host": "localhost", 3 | "user": "botuser", 4 | "password": "tsubaki", 5 | "charset": "utf8", 6 | "db": "dadguide", 7 | "database": "dadguide" 8 | } 9 | -------------------------------------------------------------------------------- /cronjobs/discord.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | set -x 5 | 6 | function hook_error() { 7 | echo "$1" 8 | data="{\"content\": \"$1\"}" 9 | curl -H "Content-Type: application/json" \ 10 | -X POST \ 11 | -d "$data" $PRIVATE_ERROR_WEBHOOK_URL 12 | } 13 | 14 | function hook_warn() { 15 | echo "$1" 16 | data="{\"content\": \"$1\"}" 17 | curl -H "Content-Type: application/json" \ 18 | -X POST \ 19 | -d "$data" $PRIVATE_WARN_WEBHOOK_URL 20 | } 21 | 22 | function hook_info() { 23 | echo "$1" 24 | data="{\"content\": \"$1\"}" 25 | curl -H "Content-Type: application/json" \ 26 | -X POST \ 27 | -d "$data" $PRIVATE_INFO_WEBHOOK_URL 28 | } 29 | 30 | function hook_file() { 31 | curl -F "data=@$1" $PRIVATE_INFO_WEBHOOK_URL 32 | } 33 | 34 | function public_hook_file() { 35 | curl -F "data=@$1" $PUBLIC_WEBHOOK_URL 36 | } 37 | -------------------------------------------------------------------------------- /cronjobs/do_single_process.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | set -x 4 | 5 | cd "$(dirname "$0")" || exit 6 | source ./shared_root.sh 7 | source ./shared.sh 8 | source ./discord.sh 9 | 10 | flock -xn /tmp/dg_processor.lck python3 "${ETL_DIR}/data_processor.py" \ 11 | --input_dir="${RAW_DIR}" \ 12 | --es_dir="${ES_DIR}" \ 13 | --media_dir="${IMG_DIR}" \ 14 | --output_dir="${DADGUIDE_DATA_DIR}/processed" \ 15 | --db_config="${DB_CONFIG}" \ 16 | --server=$1 \ 17 | --processors=$2 \ 18 | --skipintermediate \ 19 | --doupdates 20 | -------------------------------------------------------------------------------- /cronjobs/dungeon_scraper.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | set -x 5 | 6 | cd "$(dirname "$0")" || exit 7 | source ./shared_root.sh 8 | source ./shared.sh 9 | source ./discord.sh 10 | source "${VENV_ROOT}/bin/activate" 11 | 12 | function error_exit() { 13 | hook_error "Autodungeon failed <@&${NOTIFICATION_DISCORD_ROLE_ID}>" 14 | hook_file "/tmp/dg_scraper_log.txt" 15 | } 16 | 17 | function success_exit() { 18 | echo "Autodungeon finished" 19 | } 20 | 21 | function human_fixes_check() { 22 | processor_fails="/tmp/autodungeon_processor_issues.txt" 23 | if [[ -s ${processor_fails} ]]; then 24 | echo "Alerting for autodungeon processor issues" 25 | hook_file "${processor_fails}" 26 | hook_warn "\`\`\`\n$(cat "${processor_fails}" \ 27 | | sed ':a;N;$!ba;s/\n/\\n/g' \ 28 | | head -c 1990)\n\`\`\`" 29 | else 30 | echo "Autodungeon completely successful" 31 | fi 32 | } 33 | 34 | # Enable alerting to discord 35 | trap error_exit ERR 36 | trap success_exit EXIT 37 | 38 | flock -xn /tmp/dg_scraper_jp.lck python3 ${REPO_ROOT}/etl/auto_dungeon_scrape.py \ 39 | --db_config=${DB_CONFIG} \ 40 | --input_dir=${RAW_DIR} \ 41 | --doupdates \ 42 | --server=jp \ 43 | --user_uuid=${JP_PAD_USER_UUID} \ 44 | --user_intid=${JP_PAD_USER_INTID} 45 | human_fixes_check 46 | 47 | 48 | flock -xn /tmp/dg_scraper_na.lck python3 ${REPO_ROOT}/etl/auto_dungeon_scrape.py \ 49 | --db_config=${DB_CONFIG} \ 50 | --input_dir=${RAW_DIR} \ 51 | --doupdates \ 52 | --server=na \ 53 | --user_uuid=${NA_PAD_USER_UUID} \ 54 | --user_intid=${NA_PAD_USER_INTID} 55 | human_fixes_check 56 | 57 | hook_info "Autodungeon finished" 58 | -------------------------------------------------------------------------------- /cronjobs/export_data.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | set -x 4 | 5 | cd "$(dirname "$0")" || exit 6 | source ./shared_root.sh 7 | source ./shared.sh 8 | source ./discord.sh 9 | 10 | echo "Exporting DG (NO WAVES) to sqlite" 11 | DADGUIDE_DB_FILE=${DADGUIDE_GAME_DB_DIR}/dadguide.sqlite 12 | DADGUIDE_DB_FILE_TMP=${DADGUIDE_DB_FILE}_tmp 13 | rm -f "${DADGUIDE_DB_FILE_TMP}" 14 | ${SCHEMA_TOOLS_DIR}/mysql2sqlite.sh \ 15 | -u ${MYSQL_USER} \ 16 | -p${MYSQL_PASSWORD} \ 17 | dadguide \ 18 | --skip-triggers \ 19 | --ignore-table=dadguide.wave_data | 20 | sqlite3 "${DADGUIDE_DB_FILE_TMP}" 21 | mv "${DADGUIDE_DB_FILE_TMP}" "${DADGUIDE_DB_FILE}" 22 | 23 | echo "Zipping/copying DB dump" 24 | DADGUIDE_ZIP_DB_FILE="${DADGUIDE_GAME_DB_DIR}/dadguide.sqlite.zip" 25 | DADGUIDE_ZIP_DB_FILE_TMP="${DADGUIDE_ZIP_DB_FILE}_tmp" 26 | rm -f "${DADGUIDE_ZIP_DB_FILE_TMP}" 27 | zip -j "${DADGUIDE_ZIP_DB_FILE_TMP}" "${DADGUIDE_DB_FILE}" 28 | mv "${DADGUIDE_ZIP_DB_FILE_TMP}" "${DADGUIDE_ZIP_DB_FILE}" 29 | 30 | echo "Exporting DG (NO WAVES) to mysql" 31 | DADGUIDE_MYSQL_FILE=${DADGUIDE_GAME_DB_DIR}/dadguide.mysql 32 | DADGUIDE_MYSQL_ZIP_FILE=${DADGUIDE_GAME_DB_DIR}/dadguide.mysql.zip 33 | rm -f "${DADGUIDE_MYSQL_FILE}" "${DADGUIDE_MYSQL_ZIP_FILE}" 34 | mysqldump --default-character-set=utf8 \ 35 | -u ${MYSQL_USER} \ 36 | -p${MYSQL_PASSWORD} \ 37 | dadguide \ 38 | --ignore-table=dadguide.wave_data \ 39 | >"${DADGUIDE_MYSQL_FILE}" 40 | zip -j "${DADGUIDE_MYSQL_ZIP_FILE}" "${DADGUIDE_MYSQL_FILE}" 41 | 42 | echo "{\"last_edited\": $(date +%s)}" >${DADGUIDE_GAME_DB_DIR}/version.json 43 | -------------------------------------------------------------------------------- /cronjobs/game_data/export_enemy_skills.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | cd "$(dirname "$0")" || exit 3 | source ../shared_root.sh 4 | source ../shared.sh 5 | source "${VENV_ROOT}/bin/activate" 6 | 7 | cd "${GAME_DATA_DIR}" || exit 8 | git pull --rebase --autostash 9 | 10 | python3 "${ETL_DIR}/rebuild_enemy_skills.py" \ 11 | --input_dir="${RAW_DIR}" \ 12 | --output_dir="${GAME_DATA_DIR}" 13 | 14 | git add behavior_* 15 | git commit -m 'Enemy skill update' || true 16 | git push 17 | -------------------------------------------------------------------------------- /cronjobs/game_data/export_game_data.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | cd "$(dirname "$0")" || exit 4 | source ../shared_root.sh 5 | source ../shared.sh 6 | source ../discord.sh 7 | 8 | echo "Exporting Enemy Skills" 9 | ./export_enemy_skills.sh 10 | 11 | echo "Exporting PAD Data" 12 | ./export_pad_data.sh 13 | 14 | hook_info "Game data export finished" 15 | -------------------------------------------------------------------------------- /cronjobs/game_data/export_pad_data.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | cd "$(dirname "$0")" || exit 3 | source ../shared_root.sh 4 | source ../shared.sh 5 | source "${VENV_ROOT}/bin/activate" 6 | 7 | cd "${GAME_DATA_DIR}" || exit 8 | git add behavior* 9 | git commit -m 'manually approved ES' || true 10 | 11 | git pull --rebase --autostash 12 | 13 | python3 "${UTILS_ETL_DIR}/data_exporter.py" \ 14 | --input_dir="${RAW_DIR}" \ 15 | --output_dir="${GAME_DATA_DIR}" 16 | 17 | git add ./*/assets/ 18 | git add ./*/extras/ 19 | git add *.txt 20 | git commit -m 'Data update' || true 21 | git push 22 | -------------------------------------------------------------------------------- /cronjobs/media/update_bgm_files.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Updates the local cache of voice files and fixes them. 4 | 5 | cd "$(dirname "$0")" || exit 6 | source ../shared_root.sh 7 | source ../shared.sh 8 | source "${VENV_ROOT}/bin/activate" 9 | 10 | RUN_DIR="${MEDIA_ETL_DIR}/extras" 11 | EXTRA_DIR="${PAD_DATA_DIR}/extras" 12 | FINAL_DIR="${DADGUIDE_MEDIA_DIR}/bgm" 13 | 14 | for server in na jp; do 15 | python3 "${RUN_DIR}/PADDSMGParser.py" \ 16 | --db_config="${DB_CONFIG}" \ 17 | --server=$server 18 | 19 | python3 "${RUN_DIR}/PADBGMDownload.py" \ 20 | --extra_dir="${EXTRA_DIR}" \ 21 | --final_dir="${FINAL_DIR}" \ 22 | --server=$server 23 | done 24 | -------------------------------------------------------------------------------- /cronjobs/media/update_image_files.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Updates the local cache of full monster pics / portraits from the JP server. 4 | 5 | cd "$(dirname "$0")" || exit 6 | source ../shared_root.sh 7 | source ../shared.sh 8 | source ../discord.sh 9 | source "${VENV_ROOT}/bin/activate" 10 | 11 | function error_exit() { 12 | hook_error "Image Pipeline failed <@&${NOTIFICATION_DISCORD_ROLE_ID}>" 13 | hook_file "/tmp/dg_image_log.txt" 14 | } 15 | 16 | function success_exit() { 17 | echo "Image pipeline finished" 18 | hook_info "Image pipeline finished" 19 | } 20 | 21 | # Enable alerting to discord 22 | trap error_exit ERR 23 | trap success_exit EXIT 24 | 25 | # Only allow one instance of this script to run at a time 26 | # exec 8>"/tmp/image.lock"; 27 | # flock -nx 8; 28 | 29 | RUN_DIR="${MEDIA_ETL_DIR}/assets" 30 | 31 | # Enable NVM (Spammy) 32 | set +x 33 | export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")" 34 | [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" 35 | nvm use 16 36 | set -x 37 | 38 | mkdir -p "${IMG_DIR}/animated_tombstones" 39 | for SERVER in na jp; do 40 | FILE_DIR="${IMG_DIR}/${SERVER}" 41 | # Make folders (Spammy) 42 | set +x 43 | for FOLDER in raw_data portraits cards icons spine_files animated_portraits; do 44 | mkdir -p "${FILE_DIR}/${FOLDER}" 45 | done 46 | set -x 47 | yarn --cwd=${PAD_RESOURCES_ROOT} update "${FILE_DIR}/raw_data" \ 48 | --new-only --for-tsubaki --server "${SERVER}" --quiet 49 | yarn --cwd=${PAD_RESOURCES_ROOT} extract "${FILE_DIR}/raw_data" \ 50 | --still-dir "${FILE_DIR}/portraits" \ 51 | --card-dir "${FILE_DIR}/cards" \ 52 | --animated-dir "${FILE_DIR}/spine_files" \ 53 | --new-only --for-tsubaki --server "${SERVER}" --quiet 54 | xvfb-run -s "-ac -screen 0 640x388x24" \ 55 | yarn --cwd=${PAD_RESOURCES_ROOT} render "${FILE_DIR}/spine_files" \ 56 | --animated-dir "${FILE_DIR}/animated_portraits" \ 57 | --still-dir "${FILE_DIR}/portraits" \ 58 | --tomb-dir "${IMG_DIR}/animated_tombstones" \ 59 | --new-only --for-tsubaki --server "${SERVER}" --quiet \ 60 | || yarn --cwd=${PAD_RESOURCES_ROOT} render "${FILE_DIR}/spine_files" \ 61 | --animated-dir "${FILE_DIR}/animated_portraits" \ 62 | --still-dir "${FILE_DIR}/portraits" \ 63 | --tomb-dir "${IMG_DIR}/animated_tombstones" \ 64 | --new-only --for-tsubaki --server "${SERVER}" --quiet 65 | 66 | python3 "${RUN_DIR}/PADIconGenerator.py" \ 67 | --card_dir="${FILE_DIR}/cards" \ 68 | --db_config="${DB_CONFIG}" \ 69 | --server=${SERVER} \ 70 | --card_templates_file="${RUN_DIR}/attribute_frames.png" \ 71 | --output_dir="${FILE_DIR}/icons" 72 | done 73 | 74 | # HQ Images are only in JP 75 | mkdir -p "${IMG_DIR}/jp/hq_portraits" 76 | python3 "${RUN_DIR}/PADHQImageDownload.py" \ 77 | --raw_file_dir="${IMG_DIR}/jp/raw_data" \ 78 | --db_config="${DB_CONFIG}" \ 79 | --output_dir="${IMG_DIR}/jp/hq_portraits" 80 | 81 | # Force a sync 82 | ${CRONJOBS_DIR}/sync_data.sh 83 | -------------------------------------------------------------------------------- /cronjobs/media/update_orb_styles_files.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Updates the local cache of orb skin files. 4 | 5 | cd "$(dirname "$0")" || exit 6 | source ../shared_root.sh 7 | source ../shared.sh 8 | source "${VENV_ROOT}/bin/activate" 9 | 10 | RUN_DIR="${MEDIA_ETL_DIR}/extras" 11 | CACHE_DIR="${PAD_DATA_DIR}/orb_styles" 12 | OUTPUT_DIR="${DADGUIDE_MEDIA_DIR}/orb_skins" 13 | 14 | python3 "${RUN_DIR}/PADOrbStylesDownload.py" \ 15 | --cache_dir="${CACHE_DIR}" \ 16 | --output_dir="${OUTPUT_DIR}" \ 17 | --server=na 18 | 19 | python3 "${RUN_DIR}/PADOrbStylesDownload.py" \ 20 | --cache_dir="${CACHE_DIR}" \ 21 | --output_dir="${OUTPUT_DIR}" \ 22 | --server=jp 23 | -------------------------------------------------------------------------------- /cronjobs/media/update_story_files.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Updates the local cache of story files. 4 | 5 | cd "$(dirname "$0")" || exit 6 | source ../shared_root.sh 7 | source ../shared.sh 8 | source "${VENV_ROOT}/bin/activate" 9 | 10 | RUN_DIR="${MEDIA_ETL_DIR}/extras" 11 | TOOL_DIR=${RUN_DIR} 12 | 13 | CACHE_DIR="${PAD_DATA_DIR}/story" 14 | OUTPUT_DIR="${DADGUIDE_MEDIA_DIR}/story" 15 | 16 | python3 "${RUN_DIR}/PADStoryDownload.py" \ 17 | --tool_dir="${TOOL_DIR}" \ 18 | --cache_dir="${CACHE_DIR}" \ 19 | --output_dir="${OUTPUT_DIR}" \ 20 | --server=na 21 | 22 | python3 "${RUN_DIR}/PADStoryDownload.py" \ 23 | --tool_dir="${TOOL_DIR}" \ 24 | --cache_dir="${CACHE_DIR}" \ 25 | --output_dir="${OUTPUT_DIR}" \ 26 | --server=jp 27 | -------------------------------------------------------------------------------- /cronjobs/media/update_voice_files.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Updates the local cache of voice files and fixes them. 4 | 5 | cd "$(dirname "$0")" || exit 6 | source ../shared_root.sh 7 | source ../shared.sh 8 | source "${VENV_ROOT}/bin/activate" 9 | 10 | RUN_DIR="${MEDIA_ETL_DIR}/extras" 11 | CACHE_DIR="${PAD_DATA_DIR}/voices/raw" 12 | FINAL_DIR=/home/bot/dadguide/data/media/voices 13 | 14 | python3 "${RUN_DIR}/PADVoiceDownload.py" \ 15 | --cache_dir="${CACHE_DIR}" \ 16 | --final_dir="${FINAL_DIR}" \ 17 | --server=na 18 | 19 | python3 "${RUN_DIR}/PADVoiceDownload.py" \ 20 | --cache_dir="${CACHE_DIR}" \ 21 | --final_dir="${FINAL_DIR}" \ 22 | --server=jp 23 | -------------------------------------------------------------------------------- /cronjobs/pull_data.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Expects a file called account_config.csv to exist, formatted as: 4 | # <[JP,NA]>,<[A,B,C,D,E]>,,, 5 | # 6 | # Group ID and starter color are not used, just for documentation 7 | 8 | set -e 9 | set -x 10 | 11 | cd "$(dirname "$0")" || exit 12 | source ./shared_root.sh 13 | source ./shared.sh 14 | source ./discord.sh 15 | 16 | IFS="," 17 | 18 | function dl_data() { 19 | # shellcheck disable=SC2034 20 | while read -r server group uuid intid scolor; do 21 | do_only_bonus="" 22 | if [ "${scolor^^}" != "RED" ]; then 23 | do_only_bonus="--only_bonus" 24 | continue # We don't separate by group anymore 25 | fi 26 | 27 | echo "Processing ${server}/${scolor}/${uuid}/${intid} ${do_only_bonus}" 28 | EXIT_CODE=0 29 | python3 "${ETL_DIR}/pad_data_pull.py" \ 30 | --output_dir="${PAD_DATA_DIR}/raw/${server,,}" \ 31 | --server="${server^^}" \ 32 | --user_uuid="${uuid}" \ 33 | --user_intid="${intid}" \ 34 | ${do_only_bonus} || EXIT_CODE=$? 35 | 36 | if [ $EXIT_CODE -ne 0 ]; then 37 | hook_error "Processing ${server}/${scolor} failed with code ${EXIT_CODE}" 38 | fi 39 | done <$1 40 | } 41 | 42 | dl_data "${ACCOUNT_CONFIG}" 43 | -------------------------------------------------------------------------------- /cronjobs/run_loader.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | set -x 5 | 6 | cd "$(dirname "$0")" || exit 7 | source ./shared_root.sh 8 | source ./shared.sh 9 | source ./discord.sh 10 | source "${VENV_ROOT}/bin/activate" 11 | 12 | # This may not work on Mac 13 | options=$(getopt -o '' --long skipdownload,skipupload,server:,processors: -- "$@") 14 | eval set -- "$options" 15 | 16 | # Defaults 17 | SERVER="COMBINED" 18 | PROCESSORS="" 19 | DOWNLOAD=1 20 | UPLOAD=1 21 | 22 | while true; do 23 | case "$1" in 24 | --server) 25 | shift; 26 | SERVER=${1^^} 27 | [[ ! $SERVER =~ JP|NA|KR|COMBINED ]] && { 28 | echo "Server must be JP/NA/KR" 29 | exit 1 30 | } 31 | ;; 32 | --processors) 33 | shift; 34 | PROCESSORS=$1 35 | ;; 36 | --skipdownload) 37 | DOWNLOAD=0 38 | ;; 39 | --skipupload) 40 | UPLOAD=0 41 | ;; 42 | --) 43 | shift 44 | break 45 | ;; 46 | esac 47 | shift 48 | done 49 | 50 | function error_exit() { 51 | hook_error "DadGuide $SERVER Pipeline failed <@&${NOTIFICATION_DISCORD_ROLE_ID}>" 52 | hook_file "/tmp/dg_update_log.txt" 53 | } 54 | 55 | function success_exit() { 56 | echo "Pipeline finished" 57 | } 58 | 59 | # Enable alerting to discord 60 | trap error_exit ERR 61 | trap success_exit EXIT 62 | 63 | if [ $DOWNLOAD -eq 1 ]; then 64 | echo "Pulling Data" 65 | ./pull_data.sh 66 | fi 67 | 68 | echo "Updating DadGuide" 69 | if [ -z "$PROCESSORS" ]; then 70 | ./data_processor.sh $SERVER 71 | else 72 | ./do_single_process.sh "$SERVER" "$PROCESSORS" 73 | fi 74 | 75 | echo "Exporting Data" 76 | ./export_data.sh 77 | 78 | if [ $UPLOAD -eq 1 ]; then 79 | echo "Syncing" 80 | ./sync_data.sh 81 | fi 82 | 83 | hook_info "Pipeline completed successfully!" 84 | -------------------------------------------------------------------------------- /cronjobs/secrets_example.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | echo "Setting secrets..." 5 | set +x 6 | 7 | declare -x AWS_ACCESS_KEY_ID="GetThisFromAWSCred" 8 | declare -x AWS_SECRET_ACCESS_KEY="FromAWSConsole" 9 | declare -x MYSQL_USER="whateverYouMadeYourSqlUserNameToBe" 10 | declare -x MYSQL_PASSWORD="your sql password" 11 | declare -x JP_PAD_USER_UUID="00000000-0000-0000-0000-000000000000" 12 | declare -x JP_PAD_USER_INTID="111111111" 13 | declare -x JP_PAD_USER_COLOR_GROUP="blue" 14 | declare -x NA_PAD_USER_UUID="00000000-0000-0000-0000-000000000000" 15 | declare -x NA_PAD_USER_INTID="111111111" 16 | declare -x NA_PAD_USER_COLOR_GROUP="red" 17 | declare -x PRIVATE_ERROR_WEBHOOK_URL="https://discordapp.com/api/webhooks/THIS/IS-YOUR-WEBHOOK-URL" 18 | declare -x PRIVATE_WARN_WEBHOOK_URL="https://discordapp.com/api/webhooks/THIS/IS-YOUR-WEBHOOK-URL2" 19 | declare -x PRIVATE_INFO_WEBHOOK_URL="https://discordapp.com/api/webhooks/THIS/IS-YOUR-WEBHOOK-URL3" 20 | declare -x PUBLIC_WEBHOOK_URL="https://discordapp.com/api/webhooks/THIS/IS-YOUR-WEBHOOK-URL4" 21 | declare -x NOTIFICATION_DISCORD_ROLE_ID="12398612894698234" 22 | 23 | set -x 24 | -------------------------------------------------------------------------------- /cronjobs/shared.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | set -x 5 | 6 | declare -x REPO_ROOT="${PROJECT_ROOT}/pad-data-pipeline" 7 | declare -x PAD_RESOURCES_ROOT="${PROJECT_ROOT}/pad-visual-media" 8 | declare -x GAME_DATA_DIR="${PROJECT_ROOT}/pad-data-pipeline-export" 9 | 10 | declare -x PAD_DATA_DIR="${REPO_ROOT}/pad_data" 11 | declare -x CRONJOBS_DIR="${REPO_ROOT}/cronjobs" 12 | declare -x RAW_DIR="${PAD_DATA_DIR}/raw" 13 | declare -x IMG_DIR="${PAD_DATA_DIR}/image_data" 14 | declare -x VENV_ROOT="${REPO_ROOT}" 15 | 16 | declare -x ETL_DIR="${REPO_ROOT}/etl" 17 | declare -x MEDIA_ETL_DIR="${REPO_ROOT}/media_pipelines" 18 | declare -x UTILS_ETL_DIR="${REPO_ROOT}/utils" 19 | 20 | declare -x DADGUIDE_DATA_DIR="${REPO_ROOT}/output" 21 | declare -x DADGUIDE_PROCESSED_DATA_DIR="${DADGUIDE_DATA_DIR}/processed" 22 | declare -x DADGUIDE_MEDIA_DIR="${DADGUIDE_DATA_DIR}/media" 23 | declare -x DADGUIDE_EXTRA_DIR="${DADGUIDE_DATA_DIR}/extra" 24 | declare -x DADGUIDE_GAME_DB_DIR="${DADGUIDE_DATA_DIR}/db" 25 | declare -x DADGUIDE_SPINE_DIR="${DADGUIDE_DATA_DIR}/spine" 26 | declare -x DADGUIDE_RAW_DIR="${DADGUIDE_DATA_DIR}/raw" 27 | 28 | declare -x DB_CONFIG="${REPO_ROOT}/cronjobs/db_config.json" 29 | declare -x ACCOUNT_CONFIG="${REPO_ROOT}/cronjobs/account_config.csv" 30 | declare -x SECRETS_CONFIG="${REPO_ROOT}/cronjobs/secrets.sh" 31 | 32 | declare -x SCHEMA_TOOLS_DIR="${REPO_ROOT}/schema" 33 | declare -x ETL_IMAGES_DIR="${REPO_ROOT}/images" 34 | declare -x ES_DIR="${GAME_DATA_DIR}/behavior_data" 35 | 36 | source ${SECRETS_CONFIG} 37 | export PYTHONPATH="${ETL_DIR}" 38 | -------------------------------------------------------------------------------- /cronjobs/shared_root_example.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This is the directory that the pad-data-pipeline, pad-data-pipeline-export, 4 | # and pad-resources directories are located in. 5 | declare -x PROJECT_ROOT="/home/bot" 6 | -------------------------------------------------------------------------------- /cronjobs/sync_data.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copies the portraits/icons from the cache into the DadGuide image dir using 4 | # the DadGuide ID space. 5 | # 6 | # Copies the awakenings, latents, and type icons out of the dadguide-data git 7 | # repo into the DadGuide image dir. 8 | # 9 | # Finally, syncs the raw data files up to GCS, and the DadGuide data into B2. 10 | 11 | set -e 12 | set -x 13 | 14 | cd "$(dirname "$0")" || exit 15 | source ./shared_root.sh 16 | source ./shared.sh 17 | source ./discord.sh 18 | 19 | echo "Copying media" 20 | python3 "${ETL_DIR}/media_copy.py" \ 21 | --base_dir="${IMG_DIR}" \ 22 | --output_dir="${DADGUIDE_MEDIA_DIR}" 23 | 24 | # Spammy commands 25 | echo "Rsyncing from repo to images dir" 26 | set +x 27 | rsync -tr "${ETL_IMAGES_DIR}"/latents/* "${DADGUIDE_MEDIA_DIR}/latents" 28 | rsync -tr "${ETL_IMAGES_DIR}"/awakenings/* "${DADGUIDE_MEDIA_DIR}/awakenings" 29 | rsync -tr "${ETL_IMAGES_DIR}"/badges/* "${DADGUIDE_MEDIA_DIR}/badges" 30 | rsync -tr "${ETL_IMAGES_DIR}"/types/* "${DADGUIDE_MEDIA_DIR}/types" 31 | rsync -tr "${ETL_IMAGES_DIR}"/icons/* "${DADGUIDE_MEDIA_DIR}/icons" 32 | rsync -tr "${PAD_DATA_DIR}"/raw/* "${DADGUIDE_RAW_DIR}" 33 | set -x 34 | 35 | echo "Syncing raw data to AWS s3" 36 | aws s3 sync --acl=private ${DADGUIDE_DATA_DIR} s3://tsubakibotpad 37 | -------------------------------------------------------------------------------- /docker/docker_db_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "host": "localhost", 3 | "user": "root", 4 | "password": "password", 5 | "charset": "utf8", 6 | "db": "dadguide" 7 | } 8 | -------------------------------------------------------------------------------- /docker/docker_mysql.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | services: 3 | db: 4 | image: mysql:5.7 5 | restart: always 6 | environment: 7 | MYSQL_DATABASE: 'dadguide' 8 | MYSQL_ROOT_PASSWORD: 'password' 9 | ports: 10 | # Change this if you have a conflict, but you may have to change other stuff as well. 11 | - '3306:3306' 12 | expose: 13 | - 3306 14 | volumes: 15 | - dadguide-db:/var/lib/mysql 16 | 17 | adminer: 18 | image: adminer 19 | restart: always 20 | ports: 21 | - '8080:8080' 22 | 23 | volumes: 24 | dadguide-db: 25 | -------------------------------------------------------------------------------- /docker/start_env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | cd "$(dirname "$0")" 5 | 6 | container_name="docker_db_1" 7 | 8 | folder="../pad_data/db/" 9 | mkdir -p ${folder} 10 | 11 | echo "Checking for db file" 12 | local_db_file="${folder}/dadguide.mysql" 13 | if [[ ! -f "${local_db_file}" ]]; then 14 | echo "Downloading to ${local_db_file}" 15 | mysql_db="https://d1kpnpud0qoyxf.cloudfront.net/db/dadguide.mysql" 16 | wget -P $folder -N $mysql_db 17 | fi 18 | 19 | echo "Checking if environment has started" 20 | env_up=$(docker ps | grep $container_name) || true 21 | if [[ -z "${env_up}" ]]; then 22 | echo "Starting environment" 23 | docker-compose -f docker_mysql.yml up -d 24 | fi 25 | 26 | echo "Fetching credentials" 27 | user=$(grep user docker_db_config.json | sed -E 's/.*: "(.*)",/\1/') 28 | pword=$(grep password docker_db_config.json | sed -E 's/.*: "(.*)",/\1/') 29 | echo "using credentials: ${user} ${pword}" 30 | 31 | docker exec -i ${container_name} mysql -u ${user} -p${pword} dadguide <${local_db_file} 32 | -------------------------------------------------------------------------------- /etl/README.md: -------------------------------------------------------------------------------- 1 | # PAD Data Processing 2 | 3 | This folder contains the ETL pipeline that downloads raw PAD data (and saves it), converts it to a more useful format ( 4 | and saves it), and then applies updates to the DadGuide database. 5 | 6 | If you're looking to do your own processing, start with `etl/pad/raw`. 7 | 8 | If you're interested in duplicating this whole process, you should probably contact the Tsubaki Team, it's somewhat 9 | involved. 10 | 11 | I use shell scripts triggered via cron to execute the various scripts. 12 | 13 | ## Scripts in this directory 14 | 15 | The scripts here do all the data updating, and also some misc utility stuff. 16 | 17 | ### Primary data pull 18 | 19 | | Script | Purpose | 20 | | --- | --- | 21 | | pad_data_pull.py | Downloads PAD data from server via API | 22 | | data_processor.py | Updates PadGuide database, writes processed files | 23 | | default_db_config.json | Dummy file for DG database connection | 24 | 25 | You don't need to run `pad_data_pull.py`; all the results of calling the API are published, see the `utils` directory 26 | for information on how to access them. 27 | 28 | `data_processor.py` is only neccessary if you want to maintain your own copy of the DadGuide database. The database is 29 | published after each update, see the `utils` directory. 30 | 31 | ### Secondary data pull stuff 32 | 33 | | Script | Purpose | 34 | | --- | --- | 35 | | auto_dungeon_scrape.py | Identifies dungeons with no data and starts loader | 36 | | pad_dungeon_pull.py | Actually pulls the dungen spawns and saves them | 37 | | media_copy.py | Moves image files into place for DadGuide | 38 | 39 | ### Enemy skills 40 | 41 | | Script | Purpose | 42 | | --- | --- | 43 | | rebuild_enemy_skills.py | Rebuilds the enemy skill flat file database | 44 | 45 | # Testing/Utility 46 | 47 | | Script | Purpose | 48 | | --- | --- | 49 | 50 | ## etl 51 | 52 | Contains all the library code that the scripts in this directory rely on. It does the data pulling, processing, 53 | formatting, and database updating. There's a lot of stuff in here. 54 | 55 | ## dadguide_proto 56 | 57 | Contains the Python bindings for the DadGuide protocol buffers. Currently only used for enemy skills. 58 | -------------------------------------------------------------------------------- /etl/dadguide_proto/README.md: -------------------------------------------------------------------------------- 1 | Contains the protocol buffer files compiled to python. See the docs in the proto folder to regenerate. 2 | -------------------------------------------------------------------------------- /etl/default_db_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "host": "localhost", 3 | "user": "root", 4 | "password": "", 5 | "charset": "utf8", 6 | "db": "dadguide" 7 | } -------------------------------------------------------------------------------- /etl/egg_processor.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import ast 3 | import json 4 | import logging 5 | 6 | from pad.db.db_util import DbWrapper 7 | from pad.storage.egg_machines_monsters import EggMachinesMonster 8 | 9 | logging.basicConfig() 10 | logging.getLogger().setLevel(logging.DEBUG) 11 | 12 | logger = logging.getLogger('processor') 13 | logger.setLevel(logging.INFO) 14 | 15 | fail_logger = logging.getLogger('processor_failures') 16 | fail_logger.setLevel(logging.INFO) 17 | 18 | db_logger = logging.getLogger('database') 19 | db_logger.setLevel(logging.INFO) 20 | 21 | human_fix_logger = logging.getLogger('human_fix') 22 | human_fix_logger.setLevel(logging.INFO) 23 | 24 | 25 | def parse_args(): 26 | parser = argparse.ArgumentParser(description="Reads existing egg machines and add its contents.", add_help=False) 27 | 28 | input_group = parser.add_argument_group("Input") 29 | input_group.add_argument("--db_config", required=True, help="JSON database info") 30 | input_group.add_argument("--doupdates", default=False, 31 | action="store_true", help="Enables actions") 32 | 33 | help_group = parser.add_argument_group("Help") 34 | help_group.add_argument("-h", "--help", action="help", 35 | help="Displays this help message and exits.") 36 | return parser.parse_args() 37 | 38 | 39 | def load_data(args): 40 | logger.info('Connecting to database') 41 | with open(args.db_config) as f: 42 | db_config = json.load(f) 43 | dry_run = not args.doupdates 44 | db_wrapper = DbWrapper(dry_run) 45 | db_wrapper.connect(db_config) 46 | data = db_wrapper.fetch_data("SELECT * FROM egg_machines") 47 | for machine_sql in data: 48 | contents = ast.literal_eval(machine_sql['contents']) 49 | for monster_id in contents.keys(): 50 | real_monster_id = int(monster_id.strip("()")) 51 | emm = EggMachinesMonster(monster_id=real_monster_id, 52 | roll_chance=contents.get(monster_id), 53 | machine_row=machine_sql['machine_row'], 54 | machine_type=machine_sql['machine_type'], 55 | server_id=machine_sql['server_id']) 56 | db_wrapper.insert_or_update(emm) 57 | 58 | 59 | if __name__ == '__main__': 60 | load_data(parse_args()) 61 | -------------------------------------------------------------------------------- /etl/export_spawns.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import logging 4 | import os 5 | 6 | from pad.db.db_util import DbWrapper 7 | 8 | logging.basicConfig() 9 | logging.getLogger().setLevel(logging.DEBUG) 10 | 11 | logger = logging.getLogger('processor') 12 | logger.setLevel(logging.INFO) 13 | 14 | fail_logger = logging.getLogger('processor_failures') 15 | fail_logger.setLevel(logging.INFO) 16 | 17 | db_logger = logging.getLogger('database') 18 | db_logger.setLevel(logging.INFO) 19 | 20 | human_fix_logger = logging.getLogger('human_fix') 21 | human_fix_logger.setLevel(logging.INFO) 22 | 23 | 24 | ENCOUNTER_QUERY = """ 25 | SELECT 26 | sub_dungeons.sub_dungeon_id AS sdgid, 27 | sub_dungeons.name_en AS stage_name, 28 | encounters.enemy_id AS enemy_id, 29 | encounters.level AS level, 30 | encounters.stage AS floor 31 | FROM 32 | encounters 33 | LEFT OUTER JOIN sub_dungeons ON encounters.sub_dungeon_id = sub_dungeons.sub_dungeon_id 34 | """ 35 | 36 | 37 | def parse_args(): 38 | parser = argparse.ArgumentParser(description="Takes the spawns and exports them as a nice json.", add_help=False) 39 | 40 | input_group = parser.add_argument_group("Input") 41 | input_group.add_argument("--db_config", required=True, help="JSON database info") 42 | 43 | output_group = parser.add_argument_group("Output") 44 | output_group.add_argument("--output_dir", required=True, help="Directory to save file in.") 45 | 46 | help_group = parser.add_argument_group("Help") 47 | help_group.add_argument("-h", "--help", action="help", 48 | help="Displays this help message and exits.") 49 | return parser.parse_args() 50 | 51 | 52 | def load_data(args): 53 | logger.info('Connecting to database') 54 | with open(args.db_config) as f: 55 | db_config = json.load(f) 56 | db_wrapper = DbWrapper() 57 | db_wrapper.connect(db_config) 58 | data = db_wrapper.fetch_data(ENCOUNTER_QUERY) 59 | output = {} 60 | for encounter in data: 61 | sdgid = encounter['sdgid'] 62 | floor = encounter['floor'] 63 | spawn = {'id': encounter['enemy_id'], 'lv': encounter['level']} 64 | if sdgid not in output: 65 | output[sdgid] = {'name': encounter['stage_name'], 'floors': {}} 66 | if floor not in output[sdgid]['floors']: 67 | output[sdgid]['floors'][floor] = {'spawns': []} 68 | if spawn not in output[sdgid]['floors'][floor]['spawns']: 69 | output[sdgid]['floors'][floor]['spawns'].append(spawn) 70 | with open(os.path.join(args.output_dir, "encounter_data.json"), 'w+') as f: 71 | json.dump(output, f) 72 | 73 | 74 | 75 | if __name__ == '__main__': 76 | args = parse_args() 77 | load_data(args) 78 | -------------------------------------------------------------------------------- /etl/media_copy.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copies PAD media to the expected DadGuide locations. 3 | """ 4 | import argparse 5 | import os 6 | import shutil 7 | from pathlib import Path 8 | 9 | 10 | def parse_args(): 11 | parser = argparse.ArgumentParser( 12 | description="Creates DadGuide image repository.", add_help=False) 13 | input_group = parser.add_argument_group("Input") 14 | input_group.add_argument("--base_dir", required=True, help="Tsubaki image base dir") 15 | 16 | output_group = parser.add_argument_group("Output") 17 | output_group.add_argument("--output_dir", required=True, 18 | help="Dir to write coalesced output data to") 19 | 20 | return parser.parse_args() 21 | 22 | 23 | def do_copy(src_path, dest_path): 24 | for file in os.listdir(src_path): 25 | if not os.path.exists(dest_path / file): 26 | shutil.copy2(src_path / file, dest_path / file) 27 | 28 | 29 | def copy_media(args): 30 | base_dir = args.base_dir 31 | output_dir = args.output_dir 32 | 33 | for server in ('na', 'jp'): 34 | for folder in ('portraits', 'icons', 'spine_files', 'hq_portraits'): 35 | # HQ Portraits don't exist in NA server 36 | if server == 'na' and folder == 'hq_portraits': 37 | continue 38 | from_dir = Path(base_dir, server, folder) 39 | to_dir = Path(output_dir, folder) 40 | to_dir.mkdir(parents=True, exist_ok=True) 41 | do_copy(from_dir, to_dir) 42 | 43 | 44 | if __name__ == '__main__': 45 | copy_media(parse_args()) 46 | -------------------------------------------------------------------------------- /etl/pad/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TsubakiBotPad/pad-data-pipeline/a03870ace9cb40cdc3703dd35317437eede5565a/etl/pad/__init__.py -------------------------------------------------------------------------------- /etl/pad/api/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This package contains modules related to directly accessing the PAD API. 3 | 4 | It requires padkeygen and dungeon_encoding, which are intentionally not checked 5 | in. 6 | """ 7 | -------------------------------------------------------------------------------- /etl/pad/api/player_data.py: -------------------------------------------------------------------------------- 1 | from ..common import pad_util 2 | 3 | 4 | class PlayerDataResponse(pad_util.Printable): 5 | def __init__(self, data): 6 | self.cur_deck_id = data['curDeck'] # int 7 | self.decksb = data['decksb'] 8 | self.cards = [CardEntry(c) for c in data['card']] 9 | self.friends = [FriendEntry(f) for f in data['friends']] 10 | self.cards_by_uuid = {c.card_uuid: c for c in self.cards} 11 | self.egg_data = data['egatya3'] 12 | 13 | gmsg = data['gmsg'].replace('ihttps', 'https') 14 | self.gacha_url = gmsg[:gmsg.rfind('/')] + '/prop.php' 15 | 16 | def get_deck_count(self): 17 | if 'decks' in self.decksb: 18 | return len(self.decksb['decks']) 19 | else: 20 | return len(self.decksb) 21 | 22 | def get_current_deck(self): 23 | if 'decks' in self.decksb: 24 | return self.decksb['decks'][self.cur_deck_id] 25 | else: 26 | cur_deck_str = str(self.cur_deck_id).zfill(2) 27 | cur_decksb_str = 's{}'.format(cur_deck_str) 28 | return self.decksb[cur_decksb_str] 29 | 30 | def get_current_deck_uuids(self): 31 | return self.get_current_deck()[:5] 32 | 33 | def get_deck_and_inherits(self): 34 | deck_and_inherits = [] 35 | for card_uuid in self.get_current_deck_uuids(): 36 | if card_uuid == 0: 37 | deck_and_inherits.append(0) 38 | deck_and_inherits.append(0) 39 | continue 40 | 41 | card = self.cards_by_uuid[card_uuid] 42 | deck_and_inherits.append(card.card_id) 43 | if card.assist_uuid == 0: 44 | deck_and_inherits.append(0) 45 | else: 46 | assist_card = self.cards_by_uuid[card.assist_uuid] 47 | deck_and_inherits.append(assist_card.card_id) 48 | return deck_and_inherits 49 | 50 | def map_card_ids_to_uuids(self, card_ids): 51 | results = [0] * 5 52 | for idx, card_id in enumerate(card_ids): 53 | if card_id == 0: 54 | continue 55 | 56 | for c in self.cards: 57 | # Ensure the card id matches and the UUID isn't already used 58 | if int(c.card_id) == int(card_id) and c.card_uuid not in results: 59 | results[idx] = c.card_uuid 60 | break 61 | if not results[idx]: 62 | raise ValueError('could not find uuid for', card_id) 63 | return results 64 | 65 | 66 | class RecommendedHelpersResponse(pad_util.Printable): 67 | def __init__(self, data): 68 | print(data) 69 | self.helpers = [FriendEntry(f) for f in data['helpers']] 70 | 71 | 72 | class CardEntry(pad_util.Printable): 73 | def __init__(self, row_values): 74 | self.card_uuid = row_values[0] 75 | self.exp = row_values[1] 76 | self.level = row_values[2] 77 | self.skill_level = row_values[3] 78 | self.feed_count = row_values[4] 79 | self.card_id = row_values[5] 80 | self.hp_plus = row_values[6] 81 | self.atk_plus = row_values[7] 82 | self.rcv_plus = row_values[8] 83 | self.awakening_count = row_values[9] 84 | self.latents = row_values[10] 85 | self.assist_uuid = row_values[11] 86 | self.unknown_12 = row_values[12] # 0 in example 87 | self.super_awakening_id = row_values[13] 88 | self.unknown_14 = row_values[14] # 0 in example 89 | 90 | def __str__(self): 91 | return str(self.__dict__) 92 | 93 | 94 | class FriendEntry(pad_util.Printable): 95 | def __init__(self, row_values): 96 | BASE_SIZE = 17 if row_values[0] == 12 else 12 if row_values[0] == 13 else 999 97 | self.base_values = row_values[0:BASE_SIZE] 98 | self.user_intid = row_values[1] 99 | self.leader_1 = FriendLeader( 100 | row_values[BASE_SIZE + 0 * FriendLeader.SIZE: BASE_SIZE + 1 * FriendLeader.SIZE]) 101 | self.leader_2 = FriendLeader( 102 | row_values[BASE_SIZE + 1 * FriendLeader.SIZE: BASE_SIZE + 2 * FriendLeader.SIZE]) 103 | 104 | if len(row_values) > BASE_SIZE + 2 * FriendLeader.SIZE + 2: 105 | self.leader_3 = FriendLeader( 106 | row_values[ 107 | BASE_SIZE + 2 * FriendLeader.SIZE: BASE_SIZE + 3 * FriendLeader.SIZE]) 108 | else: 109 | self.leader_3 = None 110 | 111 | def __str__(self): 112 | return str(self.__dict__) 113 | 114 | 115 | class FriendLeader(pad_util.Printable): 116 | SIZE = 15 117 | 118 | def __init__(self, row_values): 119 | self.monster_id = row_values[0] 120 | self.monster_level = row_values[1] 121 | self.skill_level = row_values[2] 122 | self.hp_plus = row_values[3] 123 | self.atk_plus = row_values[4] 124 | self.rcv_plus = row_values[5] 125 | self.awakening_count = row_values[6] 126 | self.latents = row_values[7] 127 | self.assist_monster_id = row_values[8] 128 | self.assist_unknown_8 = row_values[9] # 1 in examples 129 | self.assist_monster_level = row_values[10] 130 | self.assist_unknown_11 = row_values[11] # 0 in examples 131 | self.assist_awakening_count = row_values[12] 132 | self.unknown_13 = row_values[13] # 0 in examples 133 | self.super_awakening_id = row_values[14] 134 | 135 | def __str__(self): 136 | return str(self.__dict__) 137 | -------------------------------------------------------------------------------- /etl/pad/common/dungeon_types.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class RawDungeonType(Enum): 5 | NORMAL = 0 6 | SPECIAL = 1 7 | TECHNICAL = 2 8 | SOLOSPECIAL = 3 9 | RANKING = 4 10 | DEPRECATED = 5 # Not entirely sure of this one but seems like a safe bet 11 | UNUSED_6 = 6 12 | THREE_PLAYER = 7 13 | UNUSED_8 = 8 14 | STORY = 9 15 | EIGHT_PLAYER = 10 16 | 17 | 18 | class RawRepeatDay(Enum): 19 | NONE = 0 # Doesn't repeat 20 | MONDAY = 1 21 | TUESDAY = 2 22 | WEDNESDAY = 3 23 | THURSDAY = 4 24 | FRIDAY = 5 25 | SATURDAY = 6 26 | SUNDAY = 7 27 | WEEKEND = 8 28 | -------------------------------------------------------------------------------- /etl/pad/common/icons.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | 4 | class SpecialIcons(Enum): 5 | """Special icons used by DadGuide (e.g. icon_99xx.png).""" 6 | TwoPointFiveX = 99910 7 | SixX = 99911 8 | MagicStone = 99981 9 | FiveX = 99982 10 | FourX = 99983 11 | ThreeX = 99984 12 | Point = 99985 13 | TenXMp = 99986 14 | QuestionMark = 99987 15 | Half = 99988 16 | TwoPointFiveXV2 = 99990 17 | Coin = 99991 18 | RedX = 99992 19 | OnePointFiveX = 99993 20 | TwoX = 99994 21 | GoldEgg = 99995 22 | SilverPlusEgg = 99996 23 | StarPlusEgg = 99997 24 | ChimeraPlus = 99998 25 | MultiColoredKing = 99999 26 | -------------------------------------------------------------------------------- /etl/pad/common/monster_id_mapping.py: -------------------------------------------------------------------------------- 1 | # Pad NA/JP don't have the exact same monster IDs for the same monster. 2 | # We use JP ids as the monster number; NA-only cards are adjusted to a new range. 3 | from functools import wraps 4 | from typing import Callable 5 | 6 | from pad.common.shared_types import MonsterId, MonsterNo, Server 7 | 8 | NA_ONLY_OFFSET = 50_000 9 | 10 | 11 | def between(n: int, bottom: int, top: int): 12 | return bottom <= n <= top 13 | 14 | 15 | def adjust(n: MonsterNo, local_bottom: int, remote_bottom: int) -> MonsterId: 16 | return MonsterId(n - local_bottom + remote_bottom) 17 | 18 | 19 | def server_monster_id_fn(server: Server) -> Callable[[MonsterNo], MonsterId]: 20 | if server == Server.jp: 21 | return jp_no_to_monster_id 22 | if server == Server.na: 23 | return na_no_to_monster_id 24 | if server == Server.kr: 25 | return kr_no_to_monster_id 26 | 27 | 28 | def convert_gungho_id(monster_no: MonsterNo) -> MonsterNo: 29 | if monster_no >= 10_000: 30 | monster_no -= 100 31 | return MonsterNo(monster_no) 32 | 33 | 34 | def alt_no_to_monster_id(no_converter: Callable[[MonsterNo], MonsterId]) \ 35 | -> Callable[[MonsterNo], MonsterId]: 36 | @wraps(no_converter) 37 | def convert_alt_no(mno: MonsterNo): 38 | if mno >= 100_000: 39 | sub_id = MonsterNo(mno % 100_000) 40 | mno -= sub_id 41 | mno += no_converter(sub_id) 42 | else: 43 | mno = no_converter(mno) 44 | 45 | return MonsterId(mno) 46 | return convert_alt_no 47 | 48 | 49 | @alt_no_to_monster_id 50 | def jp_no_to_monster_id(jp_no: MonsterNo) -> MonsterId: 51 | # Ghost numbers for coins and other special drops 52 | jp_no = convert_gungho_id(jp_no) 53 | return MonsterId(jp_no) 54 | 55 | 56 | @alt_no_to_monster_id 57 | def na_no_to_monster_id(na_no: MonsterNo) -> MonsterId: 58 | # Ghost numbers for coins and other special drops 59 | jp_no = convert_gungho_id(na_no) 60 | 61 | # Shinra Bansho 1 62 | if between(na_no, 934, 935): 63 | return adjust(na_no, 934, 669) 64 | 65 | # Shinra Bansho 2 66 | if between(na_no, 1049, 1058): 67 | return adjust(na_no, 1049, 671) 68 | 69 | # Batman 1 70 | if between(na_no, 669, 680): 71 | return adjust(na_no, 669, 924) 72 | 73 | # Batman 2 74 | if between(na_no, 924, 933): 75 | return adjust(na_no, 924, 1049) 76 | 77 | # Voltron 78 | if between(na_no, 2601, 2631): 79 | return adjust(na_no, 2601, 2601 + NA_ONLY_OFFSET) 80 | 81 | # Power Rangers 82 | if between(na_no, 4949, 4987): 83 | return adjust(na_no, 4949, 4949 + NA_ONLY_OFFSET) 84 | 85 | # GungHo: Another Story 86 | if between(na_no, 6905, 6992): 87 | return adjust(na_no, 6905, 6905 + NA_ONLY_OFFSET) 88 | 89 | # GungHo: Another Story 2 90 | if between(na_no, 9090, 9130): 91 | return adjust(na_no, 9090, 9090 + NA_ONLY_OFFSET) 92 | 93 | return MonsterId(na_no) 94 | 95 | 96 | @alt_no_to_monster_id 97 | def kr_no_to_monster_id(kr_no: MonsterNo) -> MonsterId: 98 | # Ghost numbers for coins and other special drops 99 | jp_no = convert_gungho_id(kr_no) 100 | 101 | # Shinra Bansho 1 102 | if between(kr_no, 934, 935): 103 | return adjust(kr_no, 934, 669) 104 | 105 | # Shinra Bansho 2 106 | if between(kr_no, 1049, 1058): 107 | return adjust(kr_no, 1049, 671) 108 | 109 | # Batman 1 110 | if between(kr_no, 669, 680): 111 | return adjust(kr_no, 669, 924) 112 | 113 | # Batman 2 114 | if between(kr_no, 924, 933): 115 | return adjust(kr_no, 924, 1049) 116 | 117 | # Voltron 118 | if between(kr_no, 2601, 2631): 119 | return adjust(kr_no, 2601, 2601 + NA_ONLY_OFFSET) 120 | 121 | # GungHo: Another Story 122 | if between(kr_no, 6905, 6992): 123 | return adjust(kr_no, 6905, 6905 + NA_ONLY_OFFSET) 124 | 125 | # GungHo: Another Story 2 126 | if between(kr_no, 9090, 9130): 127 | return adjust(kr_no, 9090, 9090 + NA_ONLY_OFFSET) 128 | 129 | return MonsterId(kr_no) 130 | -------------------------------------------------------------------------------- /etl/pad/common/pad_util.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import json 3 | import os 4 | import re 5 | from typing import Union 6 | 7 | import pytz 8 | 9 | from pad.common.shared_types import JsonType, Printable, Server, dump_helper, ListJsonType 10 | 11 | # Re-exporting these types; should fix imports 12 | Printable = Printable 13 | dump_helper = dump_helper 14 | 15 | 16 | def strip_colors(message: str) -> str: 17 | return re.sub(r'(?i)[$^][a-f0-9]{6}[$^]', '', message) 18 | 19 | 20 | def ghmult(x: int) -> str: 21 | """Normalizes multiplier to a human-readable number.""" 22 | mult = x / 10000 23 | if int(mult) == mult: 24 | mult = int(mult) 25 | return '%sx' % mult 26 | 27 | 28 | def ghmult_plain(x: int) -> str: 29 | """Normalizes multiplier to a human-readable number (without decorations).""" 30 | mult = x / 10000 31 | if int(mult) == mult: 32 | mult = int(mult) 33 | return '{}'.format(mult) 34 | 35 | 36 | def ghchance(x: int) -> str: 37 | """Normalizes percentage to a human-readable number.""" 38 | assert x % 100 == 0 39 | return '%d%%' % (x // 100) 40 | 41 | 42 | def ghchance_plain(x: int) -> str: 43 | """Normalizes percentage to a human-readable number (without decorations).""" 44 | assert x % 100 == 0 45 | return '%d%%' % (x // 100) 46 | 47 | 48 | # TODO: Change this to take Server 49 | def ghtime(time_str: str, server: str) -> datetime.datetime: 50 | """Converts a time string into a datetime.""" 51 | # < 151228000000 52 | # > 2015-12-28 00:00:00 53 | server = server.lower() 54 | server = 'jp' if server == 'ja' else server 55 | tz_offsets = { 56 | 'na': '-0800', 57 | 'jp': '+0900', 58 | 'kr': '+0900', 59 | 'utc': '+0000' 60 | } 61 | timezone_str = '{} {}'.format(time_str, tz_offsets[server]) 62 | return datetime.datetime.strptime(timezone_str, '%y%m%d%H%M%S %z') 63 | 64 | 65 | def gh_to_timestamp_2(time_str: str, server: Server) -> int: 66 | """Converts a time string to a timestamp.""" 67 | dt = ghtime(time_str, server.name) 68 | return int(dt.timestamp()) 69 | 70 | 71 | def datetime_to_gh(dt): 72 | # Assumes timezone is set properly 73 | return dt.strftime('%y%m%d%H%M%S') 74 | 75 | 76 | class NoDstWestern(datetime.tzinfo): 77 | def utcoffset(self, *dt): 78 | return datetime.timedelta(hours=-8) 79 | 80 | def tzname(self, dt): 81 | return "NoDstWestern" 82 | 83 | def dst(self, dt): 84 | return datetime.timedelta(hours=-8) 85 | 86 | 87 | def cur_gh_time(server): 88 | server = server.lower() 89 | server = 'jp' if server == 'ja' else server 90 | tz_offsets = { 91 | 'na': NoDstWestern(), 92 | 'jp': pytz.timezone('Asia/Tokyo'), 93 | 'kr': pytz.timezone('Asia/Tokyo'), 94 | } 95 | return datetime_to_gh(datetime.datetime.now(tz_offsets[server])) 96 | 97 | 98 | def internal_id_to_display_id(i_id: int) -> str: 99 | """Permutes internal PAD ID to the displayed form.""" 100 | i_id = str(i_id).zfill(9) 101 | return ''.join(i_id[x - 1] for x in [1, 5, 9, 6, 3, 8, 2, 4, 7]) 102 | 103 | 104 | def display_id_to_internal_id(d_id: int) -> str: 105 | """Permutes internal PAD ID to the displayed form.""" 106 | i_id = str(d_id).zfill(9) # ########1 2 3 4 5 6 7 8 9 107 | # return ''.join(i_id[x - 1] for x in [1, 5, 9, 6, 3, 8, 2, 4, 7]) 108 | return ''.join(i_id[x - 1] for x in [1, 7, 5, 8, 2, 4, 9, 6, 3]) 109 | 110 | 111 | def display_id_to_group(d_id: str) -> str: 112 | """Converts the display ID into the group name (a,b,c,d,e).""" 113 | return chr(ord('a') + (int(d_id[2]) % 5)) 114 | 115 | 116 | def internal_id_to_group(i_id: str) -> str: 117 | """Converts the internal ID into the group name (a,b,c,d,e).""" 118 | return chr(ord('a') + (int(i_id) % 5)) 119 | 120 | 121 | def identify_server(json_file: str, server: str) -> str: 122 | """Determine the proper server.""" 123 | if server: 124 | return server.lower() 125 | for st in ['na', 'jp', 'kr']: 126 | if '/{}/'.format(st) in json_file or '\\{}\\'.format(st) in json_file: 127 | return st 128 | raise Exception('Server not supplied and not automatically detected from path') 129 | 130 | 131 | def load_raw_json(data_dir: str = None, json_file: str = None, file_name: str = None) -> Union[JsonType, ListJsonType]: 132 | """Load JSON file.""" 133 | if json_file is None: 134 | json_file = os.path.join(data_dir, file_name) 135 | 136 | with open(json_file, encoding='utf-8') as f: 137 | return json.load(f) 138 | 139 | 140 | def json_string_dump(obj, pretty=False): 141 | indent = 4 if pretty else None 142 | return json.dumps(obj, indent=indent, sort_keys=True, default=dump_helper, ensure_ascii=False) 143 | 144 | 145 | def json_file_dump(obj, f, pretty=False): 146 | indent = 4 if pretty else None 147 | json.dump(obj, f, indent=indent, sort_keys=True, default=dump_helper, ensure_ascii=False) 148 | 149 | 150 | def is_bad_name(name): 151 | """Finds names that are currently placeholder data.""" 152 | return any([x in name for x in ['***', '???']]) or any([x == name for x in ['None', '無し', '없음', 'なし']]) 153 | -------------------------------------------------------------------------------- /etl/pad/common/shared_types.py: -------------------------------------------------------------------------------- 1 | import math 2 | from enum import Enum 3 | from typing import NewType, Dict, Any, List, Union 4 | 5 | # Raw data types 6 | AttrId = NewType('AttrId', int) 7 | MonsterNo = NewType('MonsterNo', int) 8 | DungeonId = NewType('DungeonId', int) 9 | SubDungeonId = NewType('SubDungeonId', int) 10 | SkillId = NewType('SkillId', int) 11 | TypeId = NewType('TypeId', int) 12 | 13 | # DadGuide internal types 14 | MonsterId = NewType('MonsterId', int) 15 | 16 | # General purpose types 17 | JsonType = Dict[str, Any] 18 | ListJsonType = List[Dict[str, Any]] 19 | 20 | 21 | class Printable(object): 22 | """Simple way to make an object printable.""" 23 | 24 | def __repr__(self): 25 | return '{}({})'.format(self.__class__.__name__, dump_helper(self)) 26 | 27 | def __str__(self): 28 | return self.__repr__() 29 | 30 | 31 | class Curve(Printable): 32 | """Describes how to scale according to level 1-10.""" 33 | 34 | def __init__(self, 35 | min_value: Union[int, float], 36 | max_value: Union[int, float] = None, 37 | scale: float = 1.0, 38 | max_level: int = 10): 39 | self.min_value = min_value 40 | self.max_value = max_value or min_value * max_level 41 | self.scale = scale 42 | self.max_level = max(max_level, 1) 43 | 44 | def value_at(self, level: int): 45 | f = 1 if self.max_level == 1 else ((level - 1) / (self.max_level - 1)) 46 | return int(round(self.min_value + (self.max_value - self.min_value) * math.pow(f, self.scale))) 47 | 48 | 49 | class Server(Enum): 50 | jp = 0 51 | na = 1 52 | kr = 2 53 | 54 | @staticmethod 55 | def from_str(name: str): 56 | name = str(name).lower() 57 | return Server[name] 58 | 59 | 60 | class StarterGroup(Enum): 61 | red = 'red' 62 | blue = 'blue' 63 | green = 'green' 64 | 65 | 66 | class EvolutionType(Enum): 67 | evo = 1 68 | reversible = 2 69 | non_reversible = 3 70 | 71 | 72 | def dump_helper(x): 73 | if callable(x): 74 | return 'fn_obj' 75 | elif isinstance(x, Enum): 76 | return str(x) 77 | elif hasattr(x, '__dict__'): 78 | return vars(x) 79 | else: 80 | return repr(x) 81 | -------------------------------------------------------------------------------- /etl/pad/common/utils.py: -------------------------------------------------------------------------------- 1 | import unicodedata 2 | from typing import List 3 | 4 | 5 | def remove_diacritics(input: str) -> str: 6 | """ 7 | Return the base character of char, by "removing" any 8 | diacritics like accents or curls and strokes and the like. 9 | """ 10 | output = '' 11 | for c in input: 12 | try: 13 | desc = unicodedata.name(c) 14 | cutoff = desc.find(' WITH ') 15 | if cutoff != -1: 16 | desc = desc[:cutoff] 17 | output += unicodedata.lookup(desc) 18 | except: 19 | output += c 20 | return output 21 | 22 | 23 | def format_int_list(values: List[int]) -> str: 24 | return ','.join(['({})'.format(x) for x in values]) 25 | 26 | 27 | class classproperty(property): 28 | def __get__(self, *args, **kwargs): 29 | return classmethod(self.fget).__get__(None, args[1])() 30 | -------------------------------------------------------------------------------- /etl/pad/raw/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | This package contains utilities for parsing raw PAD endpoint data into usable 3 | data structures. 4 | 5 | It should only depend on items from the common package. 6 | """ 7 | 8 | from . import bonus, card, dungeon, enemy_skill, exchange, purchase, skill 9 | 10 | Bonus = bonus.Bonus 11 | Card = card.Card 12 | Curve = card.Curve 13 | Enemy = card.Enemy 14 | ESRef = card.ESRef 15 | Dungeon = dungeon.Dungeon 16 | SubDungeon = dungeon.SubDungeon 17 | MonsterSkill = skill.MonsterSkill 18 | Exchange = exchange.Exchange 19 | Purchase = purchase.Purchase 20 | EnemySkill = enemy_skill.EnemySkill 21 | -------------------------------------------------------------------------------- /etl/pad/raw/enemy_skill.py: -------------------------------------------------------------------------------- 1 | import csv 2 | from io import StringIO 3 | from typing import List 4 | 5 | from pad.common import pad_util 6 | 7 | FILE_NAME = 'download_enemy_skill_data.json' 8 | 9 | 10 | class EnemySkill(pad_util.Printable): 11 | 12 | def __init__(self, raw: List[str]): 13 | self.enemy_skill_id = int(raw[0]) 14 | self.name = raw[1].replace('\n', ' ') 15 | self.type = int(raw[2]) 16 | self.flags = int(raw[3], 16) # 16bitmap for params 17 | self.params = [None] * 16 18 | offset = 0 19 | p_idx = 4 20 | while offset < self.flags.bit_length(): 21 | if (self.flags >> offset) & 1 != 0: 22 | p_value = raw[p_idx] 23 | self.params[offset] = int(p_value) if p_value.lstrip('-').isdigit() else p_value 24 | p_idx += 1 25 | offset += 1 26 | 27 | 28 | def load_enemy_skill_data(data_dir: str = None, json_file: str = None) -> List[EnemySkill]: 29 | data_json = pad_util.load_raw_json(data_dir, json_file, FILE_NAME) 30 | es = data_json['enemy_skills'] 31 | # Cleanup for the truly atrocious way that GungHo handles CSV/JSON data. 32 | es = es.replace("',", "#,").replace(",'", ",#").replace("'\n", "#\n") 33 | csv_lines = csv.reader(StringIO(es), quotechar="#", delimiter=',') 34 | return [EnemySkill(x) for x in csv_lines if x[0] != 'c'] 35 | -------------------------------------------------------------------------------- /etl/pad/raw/enemy_skills/enemy_skill_parser.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import List, Dict, Optional 3 | 4 | from pad.raw import EnemySkill 5 | from pad.raw.skills.enemy_skill_info import ESBehavior, BEHAVIOR_MAP, ESUnknown, ESSkillSet 6 | 7 | logger = logging.getLogger('processor') 8 | human_fix_logger = logging.getLogger('human_fix') 9 | 10 | 11 | class BehaviorParser(object): 12 | def __init__(self): 13 | self.enemy_behaviors = [] # type: List[ESBehavior] 14 | self.behaviors_by_id = {} # type: Dict[int, ESBehavior] 15 | 16 | def behavior(self, es_id: int) -> Optional[ESBehavior]: 17 | return self.behaviors_by_id.get(es_id, None) 18 | 19 | def parse(self, enemy_skill_list: List[EnemySkill]): 20 | for es in enemy_skill_list: 21 | es_id = es.enemy_skill_id 22 | es_type = es.type 23 | if es_type in BEHAVIOR_MAP: 24 | new_es = BEHAVIOR_MAP[es_type](es) 25 | else: 26 | human_fix_logger.error('Failed to parse enemy skill: %d/%d: %s', es_id, es_type, es.name) 27 | new_es = ESUnknown(es) 28 | 29 | self.enemy_behaviors.append(new_es) 30 | 31 | self.behaviors_by_id = {es.enemy_skill_id: es for es in self.enemy_behaviors} 32 | 33 | for es in self.enemy_behaviors: 34 | if isinstance(es, ESSkillSet): 35 | for sub_es_id in es.skill_ids: 36 | if sub_es_id in self.behaviors_by_id: 37 | es.skills.append(self.behaviors_by_id[sub_es_id]) 38 | else: 39 | human_fix_logger.error('Failed to look up enemy skill: %d', sub_es_id) 40 | 41 | if len(self.enemy_behaviors) != len(self.behaviors_by_id): 42 | human_fix_logger.error('Error, enemy behavior size does not match: %d - %d', 43 | len(self.enemy_behaviors), len(self.behaviors_by_id)) 44 | -------------------------------------------------------------------------------- /etl/pad/raw/exchange.py: -------------------------------------------------------------------------------- 1 | """ 2 | Parses monster exchange data. 3 | """ 4 | 5 | import json 6 | import os 7 | from typing import List 8 | 9 | from pad.common import pad_util 10 | from pad.common.pad_util import Printable 11 | from pad.common.shared_types import Server 12 | # The typical JSON file name for this data. 13 | from pad.common.shared_types import Server 14 | 15 | FILE_NAME = 'mdatadl.json' 16 | 17 | 18 | class Exchange(Printable): 19 | """Exchangeable monsters, options to exhange, and any event text.""" 20 | 21 | def __init__(self, raw: List[str], server: Server): 22 | self.server = server 23 | self.unknown_0 = str(raw[0]) # Seems to always be 'A' (For Array maybe? TODO: Look into this for GH CSV) 24 | 25 | # Seems to be the unique ID for the trade? 26 | self.trade_id = int(raw[1]) 27 | 28 | # Seems to be an order field, with lower values towards the top? 29 | self.display_order = int(raw[2]) 30 | 31 | # 1-indexed menu this appears in 32 | self.menu_idx = int(raw[3]) 33 | 34 | # Trade monster ID 35 | self.monster_id = int(raw[4]) 36 | 37 | # Trade monster info 38 | self.monster_level = int(raw[5]) 39 | monster_flags = int(raw[6]) 40 | 41 | self.monster_max_skill = bool(monster_flags & 1) 42 | self.monster_max_awoken = bool(monster_flags & 2) 43 | 44 | # Trade monster amount 45 | self.monster_amount = int(raw[7]) 46 | 47 | # Trade availability start time string 48 | self.start_time_str = str(raw[8]) 49 | self.start_timestamp = pad_util.gh_to_timestamp_2(self.start_time_str, server) 50 | 51 | # Trade availability end time string 52 | self.end_time_str = str(raw[9]) 53 | self.end_timestamp = pad_util.gh_to_timestamp_2(self.end_time_str, server) 54 | 55 | # Start time string for the announcement text, probably? 56 | self.announcement_start_time_str = str(raw[10]) 57 | self.announcement_start_timestamp = pad_util.gh_to_timestamp_2( 58 | self.announcement_start_time_str, server) if self.announcement_start_time_str else '' 59 | 60 | # End time string for the announcement text, probably? 61 | self.announcement_end_time_str = str(raw[11]) 62 | self.announcement_end_timestamp = pad_util.gh_to_timestamp_2( 63 | self.announcement_end_time_str, server) if self.announcement_end_time_str else '' 64 | 65 | # Optional text that appears above monster name, for limited time events 66 | self.announcement_text = str(raw[12]) 67 | 68 | # Clean version of the announcement text without formatting 69 | self.announcement_text_clean = pad_util.strip_colors(self.announcement_text) 70 | 71 | # Number of required monsters for the trade 72 | self.required_count = int(raw[13]) 73 | 74 | # Flags, e.g. restricted 75 | self.flag_type = int(raw[14]) 76 | self.no_dupes = bool(self.flag_type & 1) 77 | self.restricted = bool(self.flag_type & 2) 78 | self.multi_exchange = bool(self.flag_type & 4) 79 | 80 | # Options for trading the monster 81 | self.required_monsters = list(map(int, raw[15:])) 82 | 83 | def __str__(self): 84 | return 'Exchange({} {} - {} - {}/{})'.format(self.server, self.monster_id, len(self.required_monsters), 85 | self.start_time_str, self.end_time_str) 86 | 87 | 88 | def load_data(server: Server, data_dir: str = None, json_file: str = None) -> List[Exchange]: 89 | """Load Card objects from PAD JSON file.""" 90 | data_json = pad_util.load_raw_json(data_dir, json_file, FILE_NAME) 91 | return [Exchange(item.split(','), server) for item in data_json['d'].split('\n')] 92 | -------------------------------------------------------------------------------- /etl/pad/raw/purchase.py: -------------------------------------------------------------------------------- 1 | """ 2 | Parses monster purchase data. 3 | """ 4 | 5 | import json 6 | import os 7 | from typing import List 8 | 9 | from pad.common import pad_util 10 | from pad.common.pad_util import Printable 11 | from pad.common.shared_types import Server 12 | 13 | FILE_NAME = 'shop_item.json' 14 | 15 | 16 | class Purchase(Printable): 17 | """Buyable monsters.""" 18 | 19 | def __init__(self, raw: List[str], server: Server, tbegin: str, tend: str): 20 | self.server = server 21 | self.start_time_str = tbegin 22 | self.start_timestamp = pad_util.gh_to_timestamp_2(self.start_time_str, server) 23 | self.end_time_str = tend 24 | self.end_timestamp = pad_util.gh_to_timestamp_2(self.end_time_str, server) 25 | self.type = str(raw[0]) # Should be P 26 | 27 | # Trade monster ID 28 | self.monster_id = int(raw[1]) 29 | 30 | # Cost of the monster in MP 31 | self.cost = int(raw[2]) 32 | 33 | # Probably amount. Always 1 34 | self.amount = int(raw[3]) 35 | 36 | # A None and two 0s 37 | self.unknown = raw[4:] 38 | 39 | def __str__(self): 40 | return 'Purchase({} {} - {})'.format(self.server, self.monster_id, self.cost) 41 | 42 | 43 | def load_data(server: Server, data_dir: str = None, json_file: str = None) -> List[Purchase]: 44 | """Load Card objects from PAD JSON file.""" 45 | data_json = pad_util.load_raw_json(data_dir, json_file, FILE_NAME) 46 | start_time, end_time = None, None 47 | mpbuys = [] 48 | for item in filter(None, data_json['d'].split('\n')): 49 | raw = item.split(',') 50 | if raw[0] == 'T': 51 | start_time = raw[1] 52 | end_time = raw[2] 53 | else: 54 | p = Purchase(raw, server, start_time, end_time) 55 | mpbuys.append(p) 56 | return mpbuys # This will have a lot or repeats, but that shouldn't matter 57 | -------------------------------------------------------------------------------- /etl/pad/raw/skill.py: -------------------------------------------------------------------------------- 1 | """ 2 | Parses monster skill (leader/active) data. 3 | """ 4 | 5 | from typing import List 6 | 7 | from pad.common import pad_util 8 | from pad.common.shared_types import SkillId 9 | 10 | # The typical JSON file name for this data. 11 | FILE_NAME = 'download_skill_data.json' 12 | 13 | 14 | class MonsterSkill(pad_util.Printable): 15 | """Leader/active skill info for a player-ownable monster.""" 16 | 17 | def __init__(self, skill_id: int, raw: List[str]): 18 | self.skill_id = SkillId(skill_id) 19 | 20 | # Skill name text. 21 | self.name = raw[0] 22 | 23 | # Skill description text (may include formatting). 24 | self.description = raw[1] 25 | 26 | # Skill description text (no formatting). 27 | self.clean_description = pad_util.strip_colors( 28 | self.description).replace('\n', ' ').replace('^p', '') 29 | 30 | # Encodes the type of skill (requires parsing other_fields). 31 | self.skill_type = int(raw[2]) 32 | 33 | # If an active skill, number of levels to max. 34 | self.levels = int(raw[3]) or None 35 | 36 | # If an active skill, maximum cooldown. 37 | self.cooldown_turns_max = int(raw[4]) if self.levels else None 38 | 39 | # If an active skill, minimum cooldown. 40 | self.cooldown_turns_min = self.cooldown_turns_max - (self.levels - 1) if self.levels else None 41 | 42 | # Unknown field. 43 | self.unknown_005 = raw[5] 44 | 45 | # Fields used in coordination with skill_type. 46 | self.data = raw[6:] 47 | 48 | def __str__(self): 49 | return str(self.__dict__) 50 | 51 | def __repr__(self): 52 | return 'Skill(%s, %r)' % (self.skill_id, self.name) 53 | 54 | 55 | def load_skill_data(data_dir=None, json_file: str = None) -> List[MonsterSkill]: 56 | """Load MonsterSkill objects from the PAD json file.""" 57 | data_json = pad_util.load_raw_json(data_dir, json_file, FILE_NAME) 58 | return [MonsterSkill(i, ms) for i, ms in enumerate(data_json['skill'])] 59 | -------------------------------------------------------------------------------- /etl/pad/raw/skills/active_behaviors.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import Any, List, NamedTuple, TYPE_CHECKING 3 | 4 | 5 | class ASBehavior(NamedTuple): 6 | # This doesn't actually work bc of the way namedtuples are created. 7 | # TODO: Fix this so we can have a nice superclass 8 | _root = True 9 | behavior_type = 'undefined' 10 | 11 | 12 | class ASBOrbChange(ASBehavior): 13 | behavior_type = 'orb_change' 14 | 15 | from_orbs: List[int] 16 | to_orbs: List[int] 17 | amount: int = 999 18 | from_invert: bool = False 19 | 20 | 21 | # Temporary fix 22 | ASBehavior = Any 23 | 24 | 25 | def behavior_to_json(behavior: List[ASBehavior]) -> List[dict]: 26 | return [{'behavior_type': asb.behavior_type, 'properties': asb._asdict()} for asb in behavior] 27 | -------------------------------------------------------------------------------- /etl/pad/raw/skills/ko/active_skill_text.py: -------------------------------------------------------------------------------- 1 | from pad.raw.skills.en.active_skill_text import EnASTextConverter 2 | from pad.raw.skills.ko.skill_common import KoBaseTextConverter 3 | 4 | 5 | class KoASTextConverter(KoBaseTextConverter, EnASTextConverter): 6 | pass 7 | -------------------------------------------------------------------------------- /etl/pad/raw/skills/ko/leader_skill_text.py: -------------------------------------------------------------------------------- 1 | from pad.raw.skills.en.leader_skill_text import EnLSTextConverter 2 | from pad.raw.skills.ko.skill_common import KoBaseTextConverter 3 | 4 | 5 | class KoLSTextConverter(KoBaseTextConverter, EnLSTextConverter): 6 | pass 7 | -------------------------------------------------------------------------------- /etl/pad/raw/skills/skill_parser.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import List, Dict 3 | 4 | from pad.raw.skill import MonsterSkill 5 | from pad.raw.skills import active_skill_info, leader_skill_info 6 | from pad.raw.skills.active_skill_info import ActiveSkill 7 | from pad.raw.skills.leader_skill_info import LeaderSkill 8 | 9 | human_fix_logger = logging.getLogger('human_fix') 10 | 11 | 12 | class SkillParser: 13 | def __init__(self): 14 | self.active_skills = [] # type: List[ActiveSkill] 15 | self.leader_skills = [] # type: List[LeaderSkill] 16 | self.as_by_id = {} # type: Dict[int, ActiveSkill] 17 | self.ls_by_id = {} # type: Dict[int, LeaderSkill] 18 | 19 | def active(self, as_id: int) -> ActiveSkill: 20 | return self.as_by_id.get(as_id, None) 21 | 22 | def leader(self, ls_id: int) -> LeaderSkill: 23 | return self.ls_by_id.get(ls_id, None) 24 | 25 | def parse(self, skill_list: List[MonsterSkill]): 26 | self.active_skills = active_skill_info.convert(skill_list) 27 | self.leader_skills = leader_skill_info.convert(skill_list) 28 | self.as_by_id = {x.skill_id: x for x in self.active_skills} 29 | self.ls_by_id = {x.skill_id: x for x in self.leader_skills} 30 | 31 | for skill in skill_list: 32 | skill_id = skill.skill_id 33 | if skill.skill_type in [0, 89]: # 0 is None, 89 is placeholder 34 | continue 35 | if self.active(skill_id) is None and self.leader(skill_id) is None: 36 | human_fix_logger.error('Skill not parsed into active/leader: %d %d %s', 37 | skill.skill_id, skill.skill_type, skill.data) 38 | 39 | skill.skill_type = -1 40 | active = ActiveSkill(skill) 41 | self.active_skills.append(active) 42 | self.as_by_id[skill_id] = active 43 | 44 | leader = LeaderSkill(-1, skill) 45 | self.leader_skills.append(leader) 46 | self.ls_by_id[skill_id] = leader 47 | return self 48 | -------------------------------------------------------------------------------- /etl/pad/raw/wave.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os 3 | from typing import List 4 | 5 | from pad.common import pad_util 6 | 7 | 8 | class WaveResponse(pad_util.Printable): 9 | def __init__(self, wave_data): 10 | """Converts the raw enemy dungeon wave response into an object.""" 11 | self.floors = [WaveFloor(floor) for floor in wave_data] 12 | 13 | 14 | class WaveFloor(pad_util.Printable): 15 | def __init__(self, floor_data): 16 | """Converts the raw stage response data into an object.""" 17 | self.monsters = [WaveMonster(monster) for monster in floor_data] 18 | 19 | 20 | class WaveMonster(pad_util.Printable): 21 | def __init__(self, monster_data): 22 | """Converts the raw spawn response data into an object.""" 23 | self.spawn_type = monster_data[0] # Dungeon trigger maybe? Mostly 0, last is 1 24 | self.monster_id = monster_data[1] 25 | self.monster_level = monster_data[2] 26 | self.drop_monster_id = monster_data[3] 27 | self.drop_monster_level = monster_data[4] 28 | self.plus_amount = monster_data[5] 29 | 30 | 31 | class WaveSummary(pad_util.Printable): 32 | def __init__(self, 33 | dungeon_id: int = 0, 34 | floor_id: int = 0, 35 | stage: int = 0, 36 | spawn_type: int = 0, 37 | monster_id: int = 0, 38 | monster_level: int = 0, 39 | row_count: int = 0): 40 | self.dungeon_id = dungeon_id 41 | self.floor_id = floor_id 42 | self.sub_dungeon_id = dungeon_id * 1000 + floor_id 43 | self.stage = stage 44 | self.spawn_type = spawn_type 45 | self.monster_id = monster_id 46 | self.monster_level = monster_level 47 | self.row_count = row_count 48 | 49 | 50 | def load_wave_summary(processed_input_dir) -> List[WaveSummary]: 51 | wave_summary_file = os.path.join(processed_input_dir, 'wave_summary.csv') 52 | results = [] 53 | with open(wave_summary_file) as f: 54 | csvreader = csv.reader(f, delimiter=',', quotechar='"') 55 | # Skip header 56 | next(csvreader) 57 | for row in csvreader: 58 | results.append(WaveSummary( 59 | dungeon_id=int(row[0]), 60 | floor_id=int(row[1]), 61 | stage=int(row[2]), 62 | spawn_type=int(row[3]), 63 | monster_id=int(row[4]), 64 | monster_level=int(row[5]), 65 | row_count=int(row[6]))) 66 | return results 67 | -------------------------------------------------------------------------------- /etl/pad/raw_processor/jp_replacements.py: -------------------------------------------------------------------------------- 1 | # A list of JP to EN replacement strings for dungeon/subdungeon names. 2 | _JP_EN_CHUNKS = { 3 | 'ノーコン': 'No Continues', 4 | '回復なし': 'No RCV', 5 | '覚醒無効': 'Awoken Skills Invalid', 6 | '5×4マス': '5x4 Board', 7 | '7×6マス': '7x6 Board', 8 | '全属性必須': 'All Att. Req.', 9 | '覚醒スキル無効': 'Awoken Skills Invalid', 10 | '特殊': 'Special', 11 | '同キャラ禁止': 'No Dupes', 12 | 'スキル使用不可': 'Skills Invalid', 13 | 'アシスト無効': 'Assists Invalid', 14 | 'リーダースキル無効': 'Leader Skills Invalid', 15 | '落ちコンなし': 'No Skyfall Combos', 16 | '★6以下のみ': '★6 or lower', 17 | '固定チーム': 'Fixed Team', 18 | '金曜': 'Mythical', 19 | '木曜': 'Mythical', 20 | '水曜': 'Legend', 21 | '火曜': 'Mythical', 22 | '3色限定': 'Tricolor', 23 | '4体以下編成': 'Teams of 4 or less', 24 | '土日': 'Legend', 25 | '月曜': 'Legend', 26 | 'HP10固定': '10 HP', 27 | '3体以下編成': 'Teams of 3 or less', 28 | '2体以下編成': 'Teams of 2 or less', 29 | 'HP100万固定': '1.000.000 HP', 30 | 'LS無効': 'L. Skills Invalid', 31 | '操作時間': 'Orb move time ', 32 | '秒固定': ' sec', 33 | '制限時間': 'Time limit ', 34 | '分': ' mins', 35 | '日曜': 'Legend', 36 | '土曜': 'Annihilation', 37 | 'リーダー助っ人固定': 'Fixed helper' 38 | } 39 | 40 | 41 | # Finds JP style brackets in a name, and replaces chunks against a list of known translations. 42 | def jp_en_replacements(name: str): 43 | if not ('【' in name and '】' in name): 44 | return name 45 | 46 | # Extract the JP inner part of the brackets 47 | part = name[name.index('【') + 1:name.index('】')] 48 | final_part = part 49 | 50 | # Replace JP bits piece by piece 51 | for k, v in _JP_EN_CHUNKS.items(): 52 | final_part = final_part.replace(k, v) 53 | 54 | # Swap out any bits we could translate 55 | return name.replace(part, final_part) 56 | -------------------------------------------------------------------------------- /etl/pad/raw_processor/merged_data.py: -------------------------------------------------------------------------------- 1 | """ 2 | Data from the different sources in the same server, merged together. 3 | """ 4 | from datetime import datetime 5 | from typing import List 6 | 7 | import pytz 8 | 9 | from pad.common import pad_util 10 | from pad.common.monster_id_mapping import server_monster_id_fn, convert_gungho_id 11 | from pad.common.shared_types import Server, MonsterNo, MonsterId 12 | from pad.raw import Bonus, Card, Dungeon, Enemy 13 | from pad.raw.skills.active_skill_info import ActiveSkill 14 | from pad.raw.skills.enemy_skill_info import ESInstance 15 | from pad.raw.skills.leader_skill_info import LeaderSkill 16 | 17 | 18 | class MergedBonus(pad_util.Printable): 19 | def __init__(self, server: Server, bonus: Bonus, dungeon: Dungeon): 20 | self.server = server 21 | self.bonus = bonus 22 | self.dungeon = dungeon 23 | self.group = None 24 | 25 | def __str__(self): 26 | return 'MergedBonus({} {} - {})'.format( 27 | self.server, self.dungeon, self.bonus) 28 | 29 | def open_duration(self): 30 | open_datetime_utc = datetime.fromtimestamp(self.bonus.start_timestamp, pytz.UTC) 31 | close_datetime_utc = datetime.fromtimestamp(self.bonus.end_timestamp, pytz.UTC) 32 | return close_datetime_utc - open_datetime_utc 33 | 34 | 35 | # class MergedEnemySkill(pad_util.Printable): 36 | # def __init__(self, enemy_skill_ref: ESRef, enemy_skill: EnemySkill): 37 | # self.enemy_skill_ref = enemy_skill_ref 38 | # self.enemy_skill = enemy_skill 39 | 40 | 41 | class MergedEnemy(pad_util.Printable): 42 | def __init__(self, enemy_id: int, enemy: Enemy, enemy_skills: List[ESInstance]): 43 | self.enemy_id = enemy_id 44 | self.enemy = enemy 45 | self.enemy_skills = enemy_skills 46 | 47 | 48 | class MergedCard(pad_util.Printable): 49 | def __init__(self, 50 | server: Server, 51 | card: Card, 52 | active_skill: ActiveSkill, 53 | leader_skill: LeaderSkill, 54 | enemy_skills: List[ESInstance]): 55 | self.server = server 56 | self.gungho_id = card.gungho_id 57 | self.monster_id = self.no_to_id(card.gungho_id) 58 | self.card = card 59 | 60 | self.active_skill_id = active_skill.skill_id if active_skill else None 61 | self.active_skill = active_skill 62 | 63 | self.leader_skill_id = leader_skill.skill_id if leader_skill else None 64 | self.leader_skill = leader_skill 65 | 66 | self.enemy_skills = enemy_skills 67 | 68 | def no_to_id(self, monster_no: MonsterNo) -> MonsterId: 69 | return server_monster_id_fn(self.server)(monster_no) 70 | 71 | def __str__(self): 72 | return 'MergedCard({} - {} - {} [es:{}])'.format( 73 | repr(self.card), repr(self.active_skill), repr(self.leader_skill), len(self.enemy_skills)) 74 | 75 | def __copy__(self): 76 | return MergedCard(self.server, self.card, self.active_skill, self.leader_skill, self.enemy_skills[:]) 77 | -------------------------------------------------------------------------------- /etl/pad/storage/awoken_skill.py: -------------------------------------------------------------------------------- 1 | from pad.db.sql_item import SimpleSqlItem 2 | 3 | 4 | class AwokenSkill(SimpleSqlItem): 5 | """Monster awakening.""" 6 | TABLE = 'awoken_skills' 7 | KEY_COL = 'awoken_skill_id' 8 | 9 | @staticmethod 10 | def from_json(o): 11 | return AwokenSkill(awoken_skill_id=o['pad_awakening_id'], 12 | name_ja=o['name_ja'], 13 | name_en=o['name_en'], 14 | name_ko=o['name_ko'], 15 | desc_ja=o['desc_ja'], 16 | desc_en=o['desc_en'], 17 | desc_ko=o['desc_ko'], 18 | name_ja_official=o['name_ja_official'], 19 | name_en_official=o['name_en_official'], 20 | name_ko_official=o['name_ko_official'], 21 | desc_ja_official=o['desc_ja_official'], 22 | desc_en_official=o['desc_en_official'], 23 | desc_ko_official=o['desc_ko_official'], 24 | adj_hp=o['adj_hp'], 25 | adj_atk=o['adj_atk'], 26 | adj_rcv=o['adj_rcv']) 27 | 28 | def __init__(self, 29 | awoken_skill_id: int = None, 30 | name_ja=None, 31 | name_en: str = None, 32 | name_ko: str = None, 33 | desc_ja: str = None, 34 | desc_en: str = None, 35 | desc_ko: str = None, 36 | name_ja_official: str = None, 37 | name_en_official: str = None, 38 | name_ko_official: str = None, 39 | desc_ja_official: str = None, 40 | desc_en_official: str = None, 41 | desc_ko_official: str = None, 42 | adj_hp: int = None, 43 | adj_atk: int = None, 44 | adj_rcv: int = None, 45 | tstamp: int = None): 46 | self.awoken_skill_id = awoken_skill_id 47 | self.name_ja = name_ja 48 | self.name_en = name_en 49 | self.name_ko = name_ko 50 | self.desc_ja = desc_ja 51 | self.desc_en = desc_en 52 | self.desc_ko = desc_ko 53 | self.name_ja_official = name_ja_official 54 | self.name_en_official = name_en_official 55 | self.name_ko_official = name_ko_official 56 | self.desc_ja_official = desc_ja_official 57 | self.desc_en_official = desc_en_official 58 | self.desc_ko_official = desc_ko_official 59 | self.adj_hp = adj_hp 60 | self.adj_atk = adj_atk 61 | self.adj_rcv = adj_rcv 62 | self.tstamp = tstamp 63 | 64 | def __str__(self): 65 | return 'AwokenSkill({}): {}'.format(self.key_str(), self.name_en) 66 | -------------------------------------------------------------------------------- /etl/pad/storage/egg_machine.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from pad.common.monster_id_mapping import server_monster_id_fn 4 | from pad.common.shared_types import Server 5 | from pad.db.sql_item import SimpleSqlItem, ExistsStrategy 6 | from pad.raw.extra_egg_machine import ExtraEggMachine 7 | 8 | 9 | class EggMachine(SimpleSqlItem): 10 | """A per-server egg machine.""" 11 | TABLE = 'egg_machines' 12 | KEY_COL = {'server_id', 'machine_row', 'machine_type'} 13 | 14 | @staticmethod 15 | def from_eem(eem: ExtraEggMachine, server: Server) -> 'EggMachine': 16 | if eem.egg_machine_type == 1: # REM 17 | egg_machine_type_id = 2 18 | elif eem.egg_machine_type == 2: # PEM 19 | egg_machine_type_id = 1 20 | elif eem.egg_machine_type == 9: # VEM 21 | egg_machine_type_id = 3 22 | else: # Special (collab or other) 23 | egg_machine_type_id = 0 24 | 25 | id_mapper = server_monster_id_fn(server) 26 | content_map = {'({})'.format(id_mapper(k)): v for k, v in eem.contents.items()} 27 | contents = json.dumps(content_map, sort_keys=True) 28 | 29 | return EggMachine( 30 | server_id=server.value, 31 | egg_machine_type_id=egg_machine_type_id, 32 | start_timestamp=eem.start_timestamp, 33 | end_timestamp=eem.end_timestamp, 34 | machine_row=eem.egg_machine_row, 35 | machine_type=eem.egg_machine_type, 36 | name=eem.clean_name, 37 | cost=eem.cost, 38 | contents=contents 39 | ) 40 | 41 | def __init__(self, 42 | server_id: int = None, 43 | egg_machine_type_id: int = None, 44 | start_timestamp: int = None, 45 | end_timestamp: int = None, 46 | machine_row: int = None, 47 | machine_type: int = None, 48 | name: str = None, 49 | cost: int = None, 50 | contents: str = None, 51 | tstamp: int = None): 52 | self.server_id = server_id 53 | self.egg_machine_type_id = egg_machine_type_id 54 | self.start_timestamp = start_timestamp 55 | self.end_timestamp = end_timestamp 56 | self.machine_row = machine_row 57 | self.machine_type = machine_type 58 | self.name = name 59 | self.cost = cost 60 | self.contents = contents 61 | self.tstamp = tstamp 62 | 63 | def __str__(self): 64 | return 'EggMachine ({}): {} [{}]'.format(self.key_str(), 65 | self.name, len(self.contents)) 66 | -------------------------------------------------------------------------------- /etl/pad/storage/egg_machines_monsters.py: -------------------------------------------------------------------------------- 1 | from pad.db.sql_item import SimpleSqlItem 2 | 3 | 4 | class EggMachinesMonster(SimpleSqlItem): 5 | """Monsters that appear in each egg_machine""" 6 | TABLE = 'egg_machines_monsters' 7 | KEY_COL = {'monster_id', 'server_id', 'machine_row', 'machine_type'} 8 | 9 | def __init__(self, 10 | monster_id: int = None, 11 | roll_chance: float = None, 12 | server_id: int = None, 13 | machine_row: int = None, 14 | machine_type: int = None, 15 | ): 16 | self.monster_id = monster_id 17 | self.roll_chance = roll_chance 18 | self.server_id = server_id 19 | self.machine_row = machine_row 20 | self.machine_type = machine_type 21 | 22 | def __str__(self): 23 | return 'EggMachineMonster ({}-{}-{})'.format(self.monster_id, self.roll_chance, self.egg_machine_id) 24 | -------------------------------------------------------------------------------- /etl/pad/storage/encounter.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from pad.common.shared_types import MonsterId 4 | from pad.db.sql_item import SimpleSqlItem, ExistsStrategy 5 | from pad.dungeon.wave_converter import ResultSlot 6 | 7 | 8 | class Encounter(SimpleSqlItem): 9 | """A monster that appears in a dungeon.""" 10 | TABLE = 'encounters' 11 | KEY_COL = 'encounter_id' # ('sub_dungeon_id', 'stage', 'enemy_id', 'order_idx', 'level') 12 | 13 | def __init__(self, 14 | encounter_id: int = None, 15 | dungeon_id: int = None, 16 | sub_dungeon_id: int = None, 17 | enemy_id: int = None, 18 | monster_id: int = None, 19 | stage: int = None, 20 | amount: int = None, 21 | order_idx: int = None, 22 | turns: int = None, 23 | level: int = None, 24 | hp: int = None, 25 | atk: int = None, 26 | defense: int = None, 27 | exp: int = None, 28 | tstamp: int = None): 29 | self.encounter_id = encounter_id 30 | self.dungeon_id = dungeon_id 31 | self.sub_dungeon_id = sub_dungeon_id 32 | self.enemy_id = enemy_id 33 | self.monster_id = monster_id 34 | self.stage = stage 35 | self.amount = amount 36 | self.order_idx = order_idx 37 | self.turns = turns 38 | self.level = level 39 | self.hp = hp 40 | self.atk = atk 41 | self.defense = defense 42 | self.exp = exp 43 | self.tstamp = tstamp 44 | 45 | def exists_strategy(self): 46 | return ExistsStrategy.BY_KEY_IF_SET 47 | 48 | def __str__(self): 49 | return 'Encounter({}): {} -> {} [{}, {}, {}]'.format(self.key_str(), self.sub_dungeon_id, self.enemy_id, 50 | self.stage, self.monster_id, self.level) 51 | 52 | 53 | class Drop(SimpleSqlItem): 54 | """Dungeon monster drop.""" 55 | TABLE = 'drops' 56 | KEY_COL = 'drop_id' 57 | 58 | @staticmethod 59 | def from_slot(o: ResultSlot, e: Encounter) -> List['Drop']: 60 | results = [] 61 | for drop_card in o.drops: 62 | results.append(Drop(encounter_id=e.encounter_id, 63 | monster_id=drop_card.monster_id)) 64 | return results 65 | 66 | def __init__(self, 67 | drop_id: int = None, 68 | encounter_id: int = None, 69 | monster_id: MonsterId = None, 70 | tstamp: int = None): 71 | self.drop_id = drop_id 72 | self.encounter_id = encounter_id 73 | self.monster_id = monster_id 74 | self.tstamp = tstamp 75 | 76 | def exists_strategy(self): 77 | return ExistsStrategy.BY_VALUE 78 | 79 | def _non_auto_insert_cols(self): 80 | return list(self._key()) 81 | 82 | def _non_auto_update_cols(self): 83 | return list(self._key()) 84 | 85 | def __str__(self): 86 | return 'Drop({}): {} -> {}'.format(self.key_str(), self.encounter_id, self.monster_id) 87 | -------------------------------------------------------------------------------- /etl/pad/storage/enemy_skill.py: -------------------------------------------------------------------------------- 1 | from dadguide_proto.enemy_skills_pb2 import MonsterBehavior 2 | from pad.raw.skills.emoji_en.enemy_skill_text import EnEmojiESTextConverter 3 | from pad.raw.skills.en.enemy_skill_text import EnESTextConverter 4 | from pad.raw.skills.ja.enemy_skill_text import JaESTextConverter 5 | from pad.raw.skills.ko.enemy_skill_text import KoESTextConverter 6 | from pad.raw_processor.crossed_data import CrossServerESInstance 7 | from pad.storage_processor.shared_storage import ServerDependentSqlItem 8 | 9 | 10 | class EnemySkill(ServerDependentSqlItem): 11 | """Enemy skill data.""" 12 | KEY_COL = 'enemy_skill_id' 13 | BASE_TABLE = 'enemy_skills' 14 | 15 | @staticmethod 16 | def from_json(o): 17 | return EnemySkill(enemy_skill_id=o['enemy_skill_id'], 18 | name_ja=o['name_ja'], 19 | name_en=o['name_en'], 20 | name_ko=o['name_ko'], 21 | desc_ja=o['desc_ja'], 22 | desc_en=o['desc_en'], 23 | desc_ko=o['desc_ko'], 24 | desc_en_emoji=o['desc_en_emoji'], 25 | min_hits=o['min_hits'], 26 | max_hits=o['max_hits'], 27 | atk_mult=o['atk_mult']) 28 | 29 | @staticmethod 30 | def from_cseb(o: CrossServerESInstance) -> 'EnemySkill': 31 | exemplar = o.cur_skill.behavior 32 | 33 | has_attack = hasattr(exemplar, 'attack') and exemplar.attack 34 | min_hits = exemplar.attack.min_hits if has_attack else 0 35 | max_hits = exemplar.attack.max_hits if has_attack else 0 36 | atk_mult = exemplar.attack.atk_multiplier if has_attack else 0 37 | 38 | desc_ja = exemplar.full_description(JaESTextConverter()) 39 | desc_en = exemplar.full_description(EnESTextConverter()) 40 | desc_ko = exemplar.full_description(KoESTextConverter()) 41 | desc_en_emoji = exemplar.full_description(EnEmojiESTextConverter()) 42 | 43 | return EnemySkill( 44 | enemy_skill_id=o.enemy_skill_id, 45 | name_ja=o.jp_skill.name, 46 | name_en=o.na_skill.name, 47 | name_ko=o.kr_skill.name, 48 | desc_ja=desc_ja, 49 | desc_en=desc_en, 50 | desc_ko=desc_ko, 51 | desc_en_emoji=desc_en_emoji, 52 | min_hits=min_hits, 53 | max_hits=max_hits, 54 | atk_mult=atk_mult) 55 | 56 | def __init__(self, 57 | enemy_skill_id: int = None, 58 | name_ja: str = None, 59 | name_en: str = None, 60 | name_ko: str = None, 61 | desc_ja: str = None, 62 | desc_en: str = None, 63 | desc_ko: str = None, 64 | desc_en_emoji: str = None, 65 | min_hits: int = None, 66 | max_hits: int = None, 67 | atk_mult: int = None, 68 | tstamp: int = None): 69 | self.enemy_skill_id = enemy_skill_id 70 | self.name_ja = name_ja 71 | self.name_en = name_en 72 | self.name_ko = name_ko 73 | self.desc_ja = desc_ja 74 | self.desc_en = desc_en 75 | self.desc_ko = desc_ko 76 | self.desc_en_emoji = desc_en_emoji 77 | self.min_hits = min_hits 78 | self.max_hits = max_hits 79 | self.atk_mult = atk_mult 80 | self.tstamp = tstamp 81 | 82 | def __str__(self): 83 | return 'EnemySkill({}): {} - {}'.format(self.key_str(), self.name_en, self.desc_en) 84 | 85 | 86 | class EnemyData(ServerDependentSqlItem): 87 | """Enemy skill data.""" 88 | KEY_COL = 'enemy_id' 89 | BASE_TABLE = 'enemy_data' 90 | 91 | @staticmethod 92 | def from_mb(o: MonsterBehavior, status: int) -> 'EnemyData': 93 | return EnemyData( 94 | enemy_id=o.monster_id, 95 | status=status, 96 | behavior=o.SerializeToString()) 97 | 98 | def __init__(self, 99 | enemy_id: int = None, 100 | status: int = None, 101 | behavior: str = None, 102 | tstamp: int = None): 103 | self.enemy_id = enemy_id 104 | self.status = status 105 | self.behavior = behavior 106 | self.tstamp = tstamp 107 | 108 | def __str__(self): 109 | return 'EnemyData({}): {} bytes'.format(self.key_str(), len(self.behavior)) 110 | -------------------------------------------------------------------------------- /etl/pad/storage/exchange.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | 3 | from pad.common.monster_id_mapping import server_monster_id_fn 4 | from pad.db.sql_item import SimpleSqlItem 5 | 6 | 7 | class Exchange(SimpleSqlItem): 8 | """Monster exchanges.""" 9 | TABLE = 'exchanges' 10 | KEY_COL = {'trade_id', 'server_id'} 11 | 12 | @staticmethod 13 | def from_raw_exchange(o): 14 | id_mapper = server_monster_id_fn(o.server) 15 | target_monster_id = id_mapper(o.monster_id) 16 | req_monster_csv_str = ','.join(['({})'.format(id_mapper(idx)) for idx in o.required_monsters]) 17 | permanent = int(timedelta(seconds=(o.end_timestamp - o.start_timestamp)) > timedelta(days=60)) 18 | return Exchange(trade_id=o.trade_id, 19 | server_id=o.server.value, 20 | target_monster_id=target_monster_id, 21 | target_monster_amount=o.monster_amount, 22 | required_monster_ids=req_monster_csv_str, 23 | required_count=o.required_count, 24 | start_timestamp=o.start_timestamp, 25 | end_timestamp=o.end_timestamp, 26 | permanent=permanent, 27 | menu_idx=o.menu_idx, 28 | order_idx=o.display_order, 29 | flags=o.flag_type) 30 | 31 | def __init__(self, 32 | trade_id: int = None, 33 | server_id: int = None, 34 | target_monster_id: int = None, 35 | target_monster_amount: int = None, 36 | required_monster_ids: str = None, 37 | required_count: int = None, 38 | start_timestamp: int = None, 39 | end_timestamp: int = None, 40 | permanent: int = None, 41 | menu_idx: int = None, 42 | order_idx: int = None, 43 | flags: int = None, 44 | tstamp: int = None): 45 | self.trade_id = trade_id 46 | self.server_id = server_id 47 | self.target_monster_id = target_monster_id 48 | self.target_monster_amount = target_monster_amount 49 | self.required_monster_ids = required_monster_ids 50 | self.required_count = required_count 51 | self.start_timestamp = start_timestamp 52 | self.end_timestamp = end_timestamp 53 | self.permanent = permanent 54 | self.menu_idx = menu_idx 55 | self.order_idx = order_idx 56 | self.flags = flags 57 | self.tstamp = tstamp 58 | 59 | def __str__(self): 60 | return 'Exchange ({}-{}): {} [{}]'.format(self.trade_id, self.server_id, self.target_monster_id, 61 | self.required_count) 62 | -------------------------------------------------------------------------------- /etl/pad/storage/latent_skill.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from pad.db.sql_item import SimpleSqlItem 4 | from pad.raw_processor.crossed_data import CrossServerCard 5 | 6 | 7 | class LatentSkill(SimpleSqlItem): 8 | """Monster latent.""" 9 | TABLE = 'latent_skills' 10 | KEY_COL = 'latent_skill_id' 11 | 12 | @staticmethod 13 | def from_json(o): 14 | return LatentSkill(latent_skill_id=o['latent_skill_id'], 15 | name_ja=o['name_ja'], 16 | name_en=o['name_en'], 17 | name_ko=o['name_ko'], 18 | desc_ja=o['desc_ja'], 19 | desc_en=o['desc_en'], 20 | desc_ko=o['desc_ko'], 21 | name_ja_official=o['name_ja_official'], 22 | name_en_official=o['name_en_official'], 23 | name_ko_official=o['name_ko_official'], 24 | desc_ja_official=o['desc_ja_official'], 25 | desc_en_official=o['desc_en_official'], 26 | desc_ko_official=o['desc_ko_official'], 27 | slots=o['slots'], 28 | required_awakening=o['required_awakening'], 29 | required_types=o['required_types'], 30 | required_level=o['required_level'], 31 | has_120_boost=o['has_120_boost']) 32 | 33 | def __init__(self, 34 | latent_skill_id: int = None, 35 | name_ja: str = None, 36 | name_en: str = None, 37 | name_ko: str = None, 38 | desc_ja: str = None, 39 | desc_en: str = None, 40 | desc_ko: str = None, 41 | name_ja_official: str = None, 42 | name_en_official: str = None, 43 | name_ko_official: str = None, 44 | desc_ja_official: str = None, 45 | desc_en_official: str = None, 46 | desc_ko_official: str = None, 47 | slots: int = None, 48 | required_awakening: int = None, 49 | required_types: list = None, 50 | required_level: int = None, 51 | has_120_boost: bool = None, 52 | monster_id: int = None, 53 | tstamp: int = None): 54 | self.latent_skill_id = latent_skill_id 55 | self.name_ja = name_ja 56 | self.name_en = name_en 57 | self.name_ko = name_ko 58 | self.desc_ja = desc_ja 59 | self.desc_en = desc_en 60 | self.desc_ko = desc_ko 61 | self.name_ja_official = name_ja_official 62 | self.name_en_official = name_en_official 63 | self.name_ko_official = name_ko_official 64 | self.desc_ja_official = desc_ja_official 65 | self.desc_en_official = desc_en_official 66 | self.desc_ko_official = desc_ko_official 67 | self.slots = slots 68 | self.required_awakening = required_awakening 69 | self.required_types = json.dumps(required_types) 70 | self.required_level = required_level 71 | self.has_120_boost = has_120_boost 72 | self.monster_id = monster_id 73 | self.tstamp = tstamp 74 | 75 | def _non_auto_update_cols(self): 76 | return ['monster_id'] 77 | 78 | def _json_cols(self): 79 | return ['required_types'] 80 | 81 | def __str__(self): 82 | return 'LatentSkill({}): {}'.format(self.key_str(), self.name_en) 83 | -------------------------------------------------------------------------------- /etl/pad/storage/purchase.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | 3 | from pad.common.monster_id_mapping import server_monster_id_fn 4 | from pad.db.sql_item import SimpleSqlItem 5 | 6 | 7 | class Purchase(SimpleSqlItem): 8 | """MP Purchases.""" 9 | TABLE = 'purchases' 10 | KEY_COL = {'server_id', 'target_monster_id', 'start_timestamp', 'end_timestamp'} 11 | 12 | @staticmethod 13 | def from_raw_purchase(o: "Purchase"): 14 | id_mapper = server_monster_id_fn(o.server) 15 | target_monster_id = id_mapper(o.monster_id) 16 | permanent = int(timedelta(seconds=(o.end_timestamp - o.start_timestamp)) > timedelta(days=60)) 17 | return Purchase(server_id=o.server.value, 18 | target_monster_id=target_monster_id, 19 | mp_cost=o.cost, 20 | amount=o.amount, 21 | start_timestamp=o.start_timestamp, 22 | end_timestamp=o.end_timestamp, 23 | permanent=permanent) 24 | 25 | def __init__(self, 26 | server_id: int = None, 27 | target_monster_id: int = None, 28 | mp_cost: int = None, 29 | amount: int = None, 30 | start_timestamp: int = None, 31 | end_timestamp: int = None, 32 | permanent: int = None, 33 | tstamp: int = None): 34 | self.server_id = server_id 35 | self.target_monster_id = target_monster_id 36 | self.mp_cost = mp_cost 37 | self.amount = amount 38 | self.start_timestamp = start_timestamp 39 | self.end_timestamp = end_timestamp 40 | self.permanent = permanent 41 | self.tstamp = tstamp 42 | 43 | def __str__(self): 44 | return 'Purchase ({}): {} - {:,d}MP'.format(self.server_id, self.target_monster_id, self.mp_cost) 45 | -------------------------------------------------------------------------------- /etl/pad/storage/rank_reward.py: -------------------------------------------------------------------------------- 1 | from pad.db.sql_item import SimpleSqlItem 2 | 3 | 4 | class RankReward(SimpleSqlItem): 5 | """Rank reward.""" 6 | TABLE = 'rank_rewards' 7 | KEY_COL = 'rank' 8 | 9 | @staticmethod 10 | def from_csv(o): 11 | return RankReward(rank=int(o[0]), 12 | exp=int(o[1]), 13 | add_cost=int(o[2]), 14 | add_friend=int(o[3]), 15 | add_stamina=int(o[4]), 16 | cost=int(o[5]), 17 | friend=int(o[6]), 18 | stamina=int(o[7])) 19 | 20 | def __init__(self, 21 | rank: int = None, 22 | exp: int = None, 23 | add_cost: int = None, 24 | add_friend: int = None, 25 | add_stamina: int = None, 26 | cost: int = None, 27 | friend: int = None, 28 | stamina: int = None, 29 | tstamp: int = None): 30 | self.rank = rank 31 | self.exp = exp 32 | self.add_cost = add_cost 33 | self.add_friend = add_friend 34 | self.add_stamina = add_stamina 35 | self.cost = cost 36 | self.friend = friend 37 | self.stamina = stamina 38 | self.tstamp = tstamp 39 | -------------------------------------------------------------------------------- /etl/pad/storage/schedule.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from pad.db.sql_item import SimpleSqlItem, ExistsStrategy 4 | from pad.raw_processor.merged_data import MergedBonus 5 | 6 | 7 | class ScheduleEvent(SimpleSqlItem): 8 | """A scheduled event. 9 | 10 | This could indicate a dungeon is open, a dungeon has a modifier active, 11 | or just a line of text + icon that indicates something. 12 | """ 13 | TABLE = 'schedule' 14 | KEY_COL = 'event_id' 15 | 16 | @staticmethod 17 | def from_mb(o: MergedBonus) -> Optional['ScheduleEvent']: 18 | return ScheduleEvent( 19 | event_id=None, # Key that is looked up or inserted 20 | server_id=o.server.value, 21 | event_type_id=o.bonus.bonus_id, 22 | start_timestamp=o.bonus.start_timestamp, 23 | end_timestamp=o.bonus.end_timestamp, 24 | message=o.bonus.message, 25 | url=o.bonus.url, 26 | value=o.bonus.clean_bonus_value, 27 | dungeon_id=o.bonus.dungeon_id, 28 | sub_dungeon_id=o.bonus.sub_dungeon_id, 29 | ) 30 | 31 | def __init__(self, 32 | event_id: int = None, 33 | server_id: int = None, 34 | event_type_id: int = None, 35 | start_timestamp: int = None, 36 | end_timestamp: int = None, 37 | message: str = None, 38 | url: str = None, 39 | value: str = None, 40 | dungeon_id: int = None, 41 | sub_dungeon_id: int = None, 42 | tstamp: int = None): 43 | self.event_id = event_id 44 | self.server_id = server_id 45 | self.event_type_id = event_type_id 46 | self.start_timestamp = start_timestamp 47 | self.end_timestamp = end_timestamp 48 | self.message = message 49 | self.url = url 50 | self.value = value 51 | self.dungeon_id = dungeon_id 52 | self.sub_dungeon_id = sub_dungeon_id 53 | self.tstamp = tstamp 54 | 55 | def exists_strategy(self): 56 | return ExistsStrategy.BY_VALUE 57 | 58 | def _non_auto_insert_cols(self): 59 | return list(self._key()) 60 | 61 | def _non_auto_update_cols(self): 62 | return list(self._key()) 63 | -------------------------------------------------------------------------------- /etl/pad/storage/series.py: -------------------------------------------------------------------------------- 1 | from pad.db import sql_item 2 | from pad.db.sql_item import SimpleSqlItem, ExistsStrategy 3 | 4 | 5 | class Series(SimpleSqlItem): 6 | """A monster series.""" 7 | TABLE = 'series' 8 | KEY_COL = 'series_id' 9 | UNSORTED_SERIES_ID = 0 10 | 11 | @staticmethod 12 | def from_json(o): 13 | return Series(series_id=o['series_id'], 14 | name_ja=o['name_ja'], 15 | name_en=o['name_en'], 16 | name_ko=o['name_ko'], 17 | series_type=o.get('series_type')) 18 | 19 | def __init__(self, 20 | series_id: int = None, 21 | name_ja: str = None, 22 | name_en: str = None, 23 | name_ko: str = None, 24 | series_type: str = None, 25 | tstamp: int = None): 26 | self.series_id = series_id 27 | self.name_ja = name_ja 28 | self.name_en = name_en 29 | self.name_ko = name_ko 30 | self.series_type = series_type 31 | self.tstamp = tstamp 32 | 33 | def __str__(self): 34 | return 'Series ({}): {}'.format(self.key_str(), self.name_en) 35 | 36 | 37 | class MonsterSeries(SimpleSqlItem): 38 | """A monster's association with a series.""" 39 | TABLE = 'monster_series' 40 | KEY_COL = 'monster_series_id' 41 | 42 | def __init__(self, 43 | monster_series_id: int = None, 44 | monster_id: int = None, 45 | series_id: int = None, 46 | tstamp: int = None, 47 | priority: bool = None): 48 | msid = monster_id if priority else monster_id + series_id * 100000 49 | self.monster_series_id = monster_series_id or msid 50 | self.monster_id = monster_id 51 | self.series_id = series_id 52 | self.tstamp = tstamp 53 | self.priority = priority 54 | 55 | def __str__(self): 56 | return 'MonsterSeries({}, {})'.format(self.key_str(), self.series_id) 57 | -------------------------------------------------------------------------------- /etl/pad/storage/skill_tag.py: -------------------------------------------------------------------------------- 1 | from pad.db.sql_item import SimpleSqlItem 2 | 3 | 4 | class ActiveSkillTag(SimpleSqlItem): 5 | """Tags for active skills.""" 6 | TABLE = 'active_skill_tags' 7 | KEY_COL = 'active_skill_tag_id' 8 | 9 | @staticmethod 10 | def from_json(o): 11 | return ActiveSkillTag(active_skill_tag_id=o['active_tag_id'], 12 | name_ja=o['name_ja'], 13 | name_en=o['name_en'], 14 | name_ko=o['name_ko'], 15 | order_idx=o['order_idx']) 16 | 17 | def __init__(self, 18 | active_skill_tag_id: int = None, 19 | name_ja=None, 20 | name_en: str = None, 21 | name_ko: str = None, 22 | order_idx: int = None, 23 | tstamp: int = None): 24 | self.active_skill_tag_id = active_skill_tag_id 25 | self.name_ja = name_ja 26 | self.name_en = name_en 27 | self.name_ko = name_ko 28 | self.order_idx = order_idx 29 | self.tstamp = tstamp 30 | 31 | def __str__(self): 32 | return 'ActiveSkillTag({}): {}'.format(self.key_str(), self.name_en) 33 | 34 | 35 | class LeaderSkillTag(SimpleSqlItem): 36 | """Tags for leader skills.""" 37 | TABLE = 'leader_skill_tags' 38 | KEY_COL = 'leader_skill_tag_id' 39 | 40 | @staticmethod 41 | def from_json(o): 42 | return LeaderSkillTag(leader_skill_tag_id=o['leader_tag_id'], 43 | name_ja=o['name_ja'], 44 | name_en=o['name_en'], 45 | name_ko=o['name_ko'], 46 | order_idx=o['order_idx']) 47 | 48 | def __init__(self, 49 | leader_skill_tag_id: int = None, 50 | name_ja=None, 51 | name_en: str = None, 52 | name_ko: str = None, 53 | order_idx: int = None, 54 | tstamp: int = None): 55 | self.leader_skill_tag_id = leader_skill_tag_id 56 | self.name_ja = name_ja 57 | self.name_en = name_en 58 | self.name_ko = name_ko 59 | self.order_idx = order_idx 60 | self.tstamp = tstamp 61 | 62 | def __str__(self): 63 | return 'LeaderSkillTag({}): {}'.format(self.key_str(), self.name_en) 64 | -------------------------------------------------------------------------------- /etl/pad/storage/wave.py: -------------------------------------------------------------------------------- 1 | from pad.common import monster_id_mapping 2 | from pad.common.monster_id_mapping import server_monster_id_fn 3 | from pad.common.shared_types import Server 4 | from pad.db.sql_item import SqlItem 5 | from pad.raw import wave as wave_data 6 | 7 | 8 | class WaveItem(SqlItem): 9 | DROP_MONSTER_ID_GOLD = 9900 10 | TABLE = 'wave_data' 11 | KEY_COL = 'id' 12 | LIST_COL = 'dungeon_id' 13 | 14 | def __init__(self, 15 | id: int = None, 16 | pull_id: int = None, 17 | entry_id: int = None, 18 | server: str = None, 19 | dungeon_id: int = None, 20 | floor_id: int = None, 21 | stage: int = None, 22 | slot: int = None, 23 | spawn_type: int = None, 24 | monster_id: int = None, 25 | monster_level: int = None, 26 | drop_monster_id: int = None, 27 | drop_monster_level: int = None, 28 | plus_amount: int = None, 29 | monster: wave_data.WaveMonster = None, 30 | pull_time=None, # Ignored 31 | leader_id: int = None, 32 | friend_id: int = None): 33 | self.id = id 34 | self.server = server 35 | self.dungeon_id = dungeon_id 36 | self.floor_id = floor_id # ID starts at 1 for lowest 37 | self.stage = stage # 0-indexed 38 | self.slot = slot # 0-indexed 39 | 40 | self.spawn_type = spawn_type 41 | self.monster_id = monster_id 42 | self.monster_level = monster_level 43 | 44 | # If drop_monster_id == 9900, then drop_monster_level is the bonus gold amount 45 | self.drop_monster_id = drop_monster_id 46 | self.drop_monster_level = drop_monster_level 47 | self.plus_amount = plus_amount 48 | 49 | if monster: 50 | self.spawn_type = monster.spawn_type 51 | # Need to correct the drop/spawn IDs for NA vs JP 52 | mapping_fn = server_monster_id_fn(Server.from_str(self.server)) 53 | self.monster_id = mapping_fn(monster.monster_id) 54 | self.monster_level = monster.monster_level 55 | self.drop_monster_id = mapping_fn(monster.drop_monster_id) 56 | self.drop_monster_level = monster.drop_monster_level 57 | self.plus_amount = monster.plus_amount 58 | 59 | self.pull_id = pull_id 60 | self.entry_id = entry_id 61 | 62 | self.leader_id = leader_id 63 | self.friend_id = friend_id 64 | 65 | def is_invade(self): 66 | return self.spawn_type == 2 67 | 68 | def get_drop(self): 69 | return self.drop_monster_id if self.drop_monster_id > 0 and self.get_coins() == 0 else None 70 | 71 | def get_coins(self): 72 | return self.drop_monster_level if self.drop_monster_id == WaveItem.DROP_MONSTER_ID_GOLD else 0 73 | 74 | def _table(self): 75 | return WaveItem.TABLE 76 | 77 | def _key(self): 78 | return WaveItem.KEY_COL 79 | 80 | def _insert_columns(self): 81 | return self.__dict__.keys() 82 | -------------------------------------------------------------------------------- /etl/pad/storage_processor/awoken_skill_processor.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | from pad.db.db_util import DbWrapper 5 | from pad.storage.awoken_skill import AwokenSkill 6 | 7 | __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) 8 | 9 | 10 | class AwokenSkillProcessor(object): 11 | def __init__(self): 12 | with open(os.path.join(__location__, 'awoken_skill.json')) as f: 13 | self.awoken_skills = json.load(f) 14 | 15 | def process(self, db: DbWrapper): 16 | for raw in self.awoken_skills: 17 | item = AwokenSkill.from_json(raw) 18 | db.insert_or_update(item) 19 | -------------------------------------------------------------------------------- /etl/pad/storage_processor/dimension_processor.py: -------------------------------------------------------------------------------- 1 | """ 2 | This processor sets up dimension tables that are used as foreign keys as other tables. 3 | In general it should do no work; it just simplifies the setup for a new database. 4 | """ 5 | import logging 6 | 7 | from pad.common.shared_types import Server 8 | from pad.db.db_util import DbWrapper 9 | from pad.db.sql_item import SimpleSqlItem 10 | 11 | logger = logging.getLogger('processor') 12 | 13 | 14 | class DimensionItem(SimpleSqlItem): 15 | """Dimension table superclass""" 16 | def __init__(self, dimension_id: int, name: str): 17 | self.__dict__[self.KEY_COL] = dimension_id 18 | self.name = name 19 | 20 | 21 | class DAttribute(DimensionItem): 22 | """Monster color attributes.""" 23 | TABLE = 'd_attributes' 24 | KEY_COL = 'attribute_id' 25 | 26 | 27 | class DType(DimensionItem): 28 | """Monster types.""" 29 | TABLE = 'd_types' 30 | KEY_COL = 'type_id' 31 | 32 | 33 | class DServer(DimensionItem): 34 | """PAD Servers.""" 35 | TABLE = 'd_servers' 36 | KEY_COL = 'server_id' 37 | 38 | 39 | class DEventType(DimensionItem): 40 | """Event types.""" 41 | TABLE = 'd_event_types' 42 | KEY_COL = 'event_type_id' 43 | 44 | 45 | class DEggMachinesType(DimensionItem): 46 | """Egg machine categories.""" 47 | TABLE = 'd_egg_machine_types' 48 | KEY_COL = 'egg_machine_type_id' 49 | 50 | 51 | class DCompoundSkillTypes(DimensionItem): 52 | """Compound active skill group types.""" 53 | TABLE = 'd_compound_skill_types' 54 | KEY_COL = 'compound_skill_type_id' 55 | 56 | 57 | class DFixedSlotType(DimensionItem): 58 | """Compound active skill group types.""" 59 | TABLE = 'd_fixed_slot_type' 60 | KEY_COL = 'fixed_slot_type_id' 61 | 62 | 63 | DIMENSION_OBJECTS = [ 64 | DAttribute(0, 'Fire'), 65 | DAttribute(1, 'Water'), 66 | DAttribute(2, 'Wood'), 67 | DAttribute(3, 'Light'), 68 | DAttribute(4, 'Dark'), 69 | DAttribute(5, 'Unknown'), 70 | DAttribute(6, 'None'), 71 | 72 | DType(0, 'Evolve'), 73 | DType(1, 'Balance'), 74 | DType(2, 'Physical'), 75 | DType(3, 'Healer'), 76 | DType(4, 'Dragon'), 77 | DType(5, 'God'), 78 | DType(6, 'Attacker'), 79 | DType(7, 'Devil'), 80 | DType(8, 'Machine'), 81 | DType(12, 'Awoken'), 82 | DType(14, 'Enhance'), 83 | DType(15, 'Vendor'), 84 | 85 | DServer(Server.jp.value, 'JP'), 86 | DServer(Server.na.value, 'NA'), 87 | DServer(Server.kr.value, 'KR'), 88 | 89 | DEventType(0, 'General'), 90 | DEventType(1, 'EXP Boost'), 91 | DEventType(2, 'Coin Boost'), 92 | DEventType(3, 'Drop Boost'), 93 | DEventType(5, 'Stamina Reduction'), 94 | DEventType(6, 'Dungeon'), 95 | DEventType(8, 'PEM Event'), 96 | DEventType(9, 'REM Event'), 97 | DEventType(10, 'PEM Cost'), 98 | DEventType(11, 'Feed XP Bonus Chance'), 99 | DEventType(12, 'Plus Drop Rate 1'), 100 | DEventType(13, 'Unknown 13'), 101 | DEventType(14, 'Unknown 14'), 102 | DEventType(15, 'Send Egg Roll'), 103 | DEventType(16, 'Plus Drop Rate 2'), 104 | DEventType(17, 'Feed Skillup Bonus Chance'), 105 | DEventType(20, 'Tournament Active'), 106 | DEventType(21, 'Tournament Closed'), 107 | DEventType(22, 'Score Announcement'), 108 | DEventType(23, 'PAD Metadata'), 109 | DEventType(24, 'Gift Dungeon with Reward'), 110 | DEventType(25, 'Dungeon Special Event'), 111 | DEventType(29, 'Multiplayer Announcement'), 112 | DEventType(31, 'Multiplayer Dungeon Text'), 113 | DEventType(32, 'Tournament Text'), 114 | DEventType(33, 'PAD Metadata 2'), 115 | DEventType(36, 'Daily Dragons'), 116 | DEventType(37, 'Monthly Quest Dungeon'), 117 | DEventType(38, 'Exchange Text'), 118 | DEventType(39, 'Dungeon Floor Text'), 119 | DEventType(40, 'Unknown 40'), 120 | DEventType(41, 'Normal Announcement'), 121 | DEventType(42, 'Technical Announcement'), 122 | DEventType(43, 'Dungeon Web Info Link'), 123 | DEventType(44, 'Stone Purchase Text'), 124 | DEventType(47, 'Story Category Text'), 125 | DEventType(50, 'Special Dungeon Info Link'), 126 | DEventType(52, 'Dungeon Unavailable Popup'), 127 | DEventType(53, '8P Reward Table'), 128 | DEventType(54, 'VEM Event'), 129 | 130 | DEggMachinesType(0, 'Special'), 131 | DEggMachinesType(1, 'REM'), 132 | DEggMachinesType(2, 'PEM'), 133 | DEggMachinesType(3, 'VEM'), 134 | 135 | DCompoundSkillTypes(0, 'Normal'), 136 | DCompoundSkillTypes(1, 'Random'), 137 | DCompoundSkillTypes(2, 'Evolving'), 138 | DCompoundSkillTypes(3, 'Looping'), 139 | 140 | DFixedSlotType(0, 'Not Fixed'), 141 | DFixedSlotType(1, 'Fixed Empty'), 142 | DFixedSlotType(2, 'Fixed Monster'), 143 | ] 144 | 145 | 146 | class DimensionProcessor(object): 147 | def __init__(self): 148 | pass 149 | 150 | def process(self, db: DbWrapper): 151 | for item in DIMENSION_OBJECTS: 152 | db.insert_or_update(item) 153 | -------------------------------------------------------------------------------- /etl/pad/storage_processor/dungeon_processor.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from pad.db.db_util import DbWrapper 4 | from pad.raw_processor import crossed_data 5 | from pad.storage.dungeon import Dungeon, FixedTeam, FixedTeamMonster, SubDungeon 6 | 7 | logger = logging.getLogger('processor') 8 | 9 | _ENCOUNTER_VISIBILITY_SQL = """ 10 | UPDATE dungeons 11 | SET visible = true, tstamp = UNIX_TIMESTAMP() 12 | WHERE dungeon_id IN (SELECT dungeon_id FROM encounters GROUP BY 1) 13 | AND visible = false 14 | """ 15 | 16 | 17 | class DungeonProcessor(object): 18 | def __init__(self, data: crossed_data.CrossServerDatabase): 19 | self.data = data 20 | 21 | def process(self, db: DbWrapper): 22 | logger.info('loading dungeon data') 23 | self._process_dungeons(db) 24 | logger.info('done loading dungeon data') 25 | 26 | def post_encounter_process(self, db: DbWrapper): 27 | logger.info('post-encounter processing') 28 | updated_rows = db.update_item(_ENCOUNTER_VISIBILITY_SQL) 29 | logger.info('Updated visibility of %s dungeons', updated_rows) 30 | 31 | def _process_dungeons(self, db: DbWrapper): 32 | for dungeon in self.data.dungeons: 33 | db.insert_or_update(Dungeon.from_csd(dungeon)) 34 | for subdungeon in dungeon.sub_dungeons: 35 | db.insert_or_update(SubDungeon.from_cssd(subdungeon, dungeon.dungeon_id)) 36 | if not subdungeon.cur_sub_dungeon.fixed_monsters: 37 | continue 38 | db.insert_or_update(FixedTeam.from_cssd(subdungeon)) 39 | for fcid in range(6): 40 | fixed = subdungeon.cur_sub_dungeon.fixed_monsters.get(fcid) 41 | db.insert_or_update(FixedTeamMonster.from_fc(fixed, fcid, subdungeon)) 42 | -------------------------------------------------------------------------------- /etl/pad/storage_processor/egg_machine_processor.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from pad.common.monster_id_mapping import server_monster_id_fn 4 | from pad.common.shared_types import Server 5 | from pad.db.db_util import DbWrapper 6 | from pad.raw_processor import crossed_data 7 | from pad.storage.egg_machine import EggMachine 8 | from pad.storage.egg_machines_monsters import EggMachinesMonster 9 | 10 | logger = logging.getLogger('processor') 11 | 12 | 13 | class EggMachineProcessor(object): 14 | def __init__(self, data: crossed_data.CrossServerDatabase): 15 | self.egg_machines = { 16 | Server.jp: data.jp_egg_machines, 17 | Server.na: data.na_egg_machines, 18 | Server.kr: data.kr_egg_machines, 19 | } 20 | 21 | def process(self, db: DbWrapper): 22 | for server, egg_machine_list in self.egg_machines.items(): 23 | logger.debug('Process {} egg machines'.format(server.name.upper())) 24 | for egg_machine in egg_machine_list: 25 | em = EggMachine.from_eem(egg_machine, server) 26 | egg_machine_id = db.insert_or_update(em) 27 | 28 | # process contents of the eggmachines 29 | id_mapper = server_monster_id_fn(server) 30 | monsters = [EggMachinesMonster( 31 | monster_id=id_mapper(k), 32 | roll_chance=v, 33 | machine_row=em.machine_row, 34 | machine_type=em.machine_type, 35 | server_id=em.server_id 36 | ) for k, v in egg_machine.contents.items()] 37 | for emm in monsters: 38 | db.insert_or_update(emm) 39 | -------------------------------------------------------------------------------- /etl/pad/storage_processor/enemy_skill_processor.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | 5 | from dadguide_proto import enemy_skills_pb2 6 | from dadguide_proto.enemy_skills_pb2 import MonsterBehavior 7 | from pad.db.db_util import DbWrapper 8 | from pad.raw.enemy_skills import enemy_skill_proto 9 | from pad.raw.skills.enemy_skill_info import ESLogic 10 | from pad.raw_processor import crossed_data 11 | from pad.storage.enemy_skill import EnemySkill, EnemyData 12 | 13 | __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) 14 | 15 | logger = logging.getLogger('processor') 16 | human_fix_logger = logging.getLogger('human_fix') 17 | 18 | 19 | class EnemySkillProcessor(object): 20 | def __init__(self, db: DbWrapper, data: crossed_data.CrossServerDatabase): 21 | self.db = db 22 | self.data = data 23 | 24 | with open(os.path.join(__location__, 'enemy_skill.json')) as f: 25 | self.static_enemy_skills = json.load(f) 26 | 27 | def load_static(self): 28 | logger.info('loading %d static skills', len(self.static_enemy_skills)) 29 | for raw in self.static_enemy_skills: 30 | item = EnemySkill.from_json(raw) 31 | self.db.insert_or_update(item) 32 | 33 | def load_enemy_skills(self): 34 | used_skills = {} 35 | for csc in self.data.all_cards: 36 | for cseb in csc.enemy_behavior: 37 | # Skip fake skills (loaded via the static import) and logic 38 | if cseb.enemy_skill_id <= 0 or isinstance(cseb.cur_skill.behavior, ESLogic): 39 | continue 40 | 41 | if cseb.enemy_skill_id in used_skills: 42 | if cseb.unique_count() > used_skills[cseb.enemy_skill_id].unique_count(): 43 | # This takes care of a rare issue where multiple monsters can use the 44 | # same skill, but en/kr lag behind jp and we take the cseb that has 45 | # jp values overwritten into the na/kr ones. 46 | # 47 | # Probably we should just stop this from being an issue by 48 | # loading all skills instead of just used skills. 49 | used_skills[cseb.enemy_skill_id] = cseb 50 | else: 51 | used_skills[cseb.enemy_skill_id] = cseb 52 | 53 | logger.info('loading %d enemy skills', len(used_skills)) 54 | for cseb in used_skills.values(): 55 | item = EnemySkill.from_cseb(cseb) 56 | self.db.insert_or_update(item) 57 | 58 | def load_enemy_data(self, base_dir: str): 59 | card_files = [] 60 | logger.info('scanning enemy data for %d cards', len(self.data.all_cards)) 61 | for csc in self.data.all_cards: 62 | card_file = os.path.join(base_dir, '{}.textproto'.format(csc.monster_id)) 63 | if not os.path.exists(card_file): 64 | continue 65 | card_files.append(card_file) 66 | 67 | logger.info('loading enemy data for %d cards', len(card_files)) 68 | count_not_approved = 0 69 | count_needs_reapproval = 0 70 | count_approved = 0 71 | for card_file in card_files: 72 | mbwo = enemy_skill_proto.load_from_file(card_file) 73 | mb = MonsterBehavior() 74 | mb.monster_id = mbwo.monster_id 75 | if mbwo.status == enemy_skills_pb2.MonsterBehaviorWithOverrides.NOT_APPROVED: 76 | mb.levels.extend(mbwo.levels) 77 | mb.approved = False 78 | count_not_approved += 1 79 | else: 80 | mb.levels.extend(mbwo.level_overrides) 81 | mb.approved = True 82 | if mbwo.status == enemy_skills_pb2.MonsterBehaviorWithOverrides.NEEDS_REAPPROVAL: 83 | # human_fix_logger.warning('needs reapproval: %d', mb.monster_id) 84 | pass 85 | else: 86 | count_approved += 1 87 | 88 | item = EnemyData.from_mb(mb, mbwo.status) 89 | self.db.insert_or_update(item) 90 | 91 | logger.info('done, %d approved %d not approved', count_approved, count_not_approved) 92 | -------------------------------------------------------------------------------- /etl/pad/storage_processor/exchange_processor.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | from pad.db.db_util import DbWrapper 5 | from pad.common.shared_types import Server 6 | from pad.storage.exchange import Exchange 7 | from pad.raw_processor import crossed_data 8 | 9 | logger = logging.getLogger('processor') 10 | 11 | 12 | class ExchangeProcessor(object): 13 | def __init__(self, data: crossed_data.CrossServerDatabase): 14 | self.exchange_data = { 15 | Server.jp: data.jp_exchange, 16 | Server.na: data.na_exchange, 17 | Server.kr: data.kr_exchange, 18 | } 19 | 20 | def process(self, db: DbWrapper): 21 | for server, exchange_map in self.exchange_data.items(): 22 | logger.debug('Process {} exchanges'.format(server.name.upper())) 23 | for raw in exchange_map: 24 | logger.debug('Creating exchange: %s', raw) 25 | item = Exchange.from_raw_exchange(raw) 26 | db.insert_or_update(item) 27 | -------------------------------------------------------------------------------- /etl/pad/storage_processor/latent_skill_processor.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | from pad.db.db_util import DbWrapper 5 | from pad.raw_processor.crossed_data import CrossServerDatabase 6 | from pad.storage.latent_skill import LatentSkill 7 | 8 | __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) 9 | 10 | from pad.storage.monster import LatentTamadra 11 | 12 | 13 | class LatentSkillProcessor(object): 14 | def __init__(self, data: CrossServerDatabase): 15 | self.data = data 16 | 17 | with open(os.path.join(__location__, 'latent_skill.json')) as f: 18 | self.latent_skills = json.load(f) 19 | 20 | def process(self, db: DbWrapper): 21 | for raw in self.latent_skills: 22 | item = LatentSkill.from_json(raw) 23 | db.insert_or_update(item) 24 | 25 | for csm in self.data.ownable_cards: 26 | if csm.cur_card.card.latent_on_feed: 27 | item = LatentTamadra.from_csm(csm) 28 | db.insert_or_update(item) 29 | -------------------------------------------------------------------------------- /etl/pad/storage_processor/monster_processor.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from pad.common import pad_util 4 | from pad.common.pad_util import is_bad_name 5 | from pad.db.db_util import DbWrapper 6 | from pad.raw_processor import crossed_data 7 | from pad.storage.monster import AltMonster, Awakening, Evolution, Monster, MonsterWithExtraImageInfo, Transformation 8 | from pad.storage.monster_skill import LeaderSkill, upsert_active_skill_data 9 | 10 | logger = logging.getLogger('processor') 11 | human_fix_logger = logging.getLogger('human_fix') 12 | 13 | 14 | class MonsterProcessor(object): 15 | def __init__(self, data: crossed_data.CrossServerDatabase): 16 | self.data = data 17 | 18 | def process(self, db: DbWrapper): 19 | logger.info('loading monster data') 20 | self._process_skills(db) 21 | self._process_monsters(db) 22 | self._process_monster_images(db) 23 | self._process_awakenings(db) 24 | self._process_evolutions(db) 25 | logger.info('done loading monster data') 26 | 27 | def _process_skills(self, db: DbWrapper): 28 | logger.info('loading skills for %s cards', len(self.data.ownable_cards)) 29 | ls_count = 0 30 | as_count = 0 31 | for csc in self.data.ownable_cards: 32 | if csc.leader_skill: 33 | ls_count += 1 34 | db.insert_or_update(LeaderSkill.from_css(csc.leader_skill)) 35 | if csc.active_skill: 36 | as_count += 1 37 | upsert_active_skill_data(db, csc.active_skill) 38 | logger.info('loaded %s leader skills and %s active skills', ls_count, as_count) 39 | 40 | def _process_monsters(self, db): 41 | logger.info('loading monsters') 42 | for m in self.data.all_cards: 43 | if 0 < m.monster_id < 100_000 and not is_bad_name(m.jp_card.card.name): 44 | db.insert_or_update(Monster.from_csm(m)) 45 | canonical_id = next((cm.monster_id for cm in self.data.ownable_cards 46 | if cm.monster_id == m.monster_id % 100_000), None) 47 | db.insert_or_update(AltMonster.from_csm(m, canonical_id)) 48 | 49 | def _process_monster_images(self, db): 50 | logger.info('monster images, hq_count=%s, anim_count=%s', 51 | len(self.data.hq_image_monster_ids), 52 | len(self.data.animated_monster_ids)) 53 | if not self.data.hq_image_monster_ids or not self.data.animated_monster_ids: 54 | logger.info('skipping image info load') 55 | return 56 | for csm in self.data.ownable_cards: 57 | item = MonsterWithExtraImageInfo(monster_id=csm.monster_id, 58 | has_animation=csm.has_animation, 59 | has_hqimage=csm.has_hqimage) 60 | db.insert_or_update(item) 61 | 62 | def _process_awakenings(self, db): 63 | logger.info('loading awakenings') 64 | for m in self.data.ownable_cards: 65 | items = Awakening.from_csm(m) 66 | for item in items: 67 | try: 68 | db.insert_or_update(item) 69 | except (KeyboardInterrupt, SystemExit): 70 | raise 71 | except Exception: 72 | human_fix_logger.fatal('Failed to insert item (probably new awakening): %s', 73 | pad_util.json_string_dump(item, pretty=True)) 74 | 75 | sql = f'DELETE FROM {Awakening.TABLE} WHERE monster_id = {m.monster_id} AND order_idx >= {len(items)}' 76 | deleted_awos = db.update_item(sql) 77 | if deleted_awos: 78 | logger.info(f"Deleted {deleted_awos} unused awakenings from monster {m.monster_id}") 79 | 80 | def _process_evolutions(self, db): 81 | logger.info('loading evolutions') 82 | for m in self.data.ownable_cards: 83 | if not m.cur_card.card.ancestor_id: 84 | continue 85 | 86 | item = Evolution.from_csm(m) 87 | if item: 88 | db.insert_or_update(item) 89 | 90 | logger.info('loading transforms') 91 | for m in self.data.ownable_cards: 92 | if not (m.cur_card.active_skill and m.cur_card.active_skill.transform_ids): 93 | continue 94 | 95 | denom = sum(val for val in m.cur_card.active_skill.transform_ids.values()) 96 | for tfid, num in m.cur_card.active_skill.transform_ids.items(): 97 | if tfid is not None: 98 | db.insert_or_update(Transformation.from_csm(m, tfid, num, denom)) 99 | -------------------------------------------------------------------------------- /etl/pad/storage_processor/purchase_processor.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from typing import List 4 | 5 | from pad.db.db_util import DbWrapper 6 | from pad.common.shared_types import Server 7 | from pad.storage.purchase import Purchase 8 | from pad.storage.monster import MonsterWithMPValue 9 | from pad.raw_processor import crossed_data 10 | 11 | logger = logging.getLogger('processor') 12 | 13 | 14 | class PurchaseProcessor(object): 15 | def __init__(self, data: crossed_data.CrossServerDatabase): 16 | self.purchase_data = { 17 | Server.jp: data.jp_purchase, 18 | Server.na: data.na_purchase, 19 | Server.kr: data.kr_purchase, 20 | } 21 | 22 | def process(self, db: DbWrapper): 23 | for server, purchase_map in self.purchase_data.items(): 24 | logger.debug('Process {} purchases'.format(server.name.upper())) 25 | for raw in purchase_map: 26 | logger.debug('Creating purchase: %s', raw) 27 | p_item = Purchase.from_raw_purchase(raw) 28 | db.insert_or_update(p_item) 29 | 30 | def purchases_to_map(purchases: List[Purchase]): 31 | return {x.monster_id: x.cost for x in purchases} 32 | 33 | monster_id_to_mp = purchases_to_map(self.purchase_data[Server.kr]) 34 | monster_id_to_mp.update(purchases_to_map(self.purchase_data[Server.na])) 35 | monster_id_to_mp.update(purchases_to_map(self.purchase_data[Server.jp])) 36 | 37 | for monster_id, mp_cost in monster_id_to_mp.items(): 38 | m_item = MonsterWithMPValue(monster_id=monster_id, buy_mp=mp_cost) 39 | db.insert_or_update(m_item) 40 | -------------------------------------------------------------------------------- /etl/pad/storage_processor/purge_data_processor.py: -------------------------------------------------------------------------------- 1 | import logging 2 | # from datetime import datetime, timedelta 3 | 4 | from pad.db.db_util import DbWrapper 5 | 6 | logger = logging.getLogger('processor') 7 | 8 | 9 | def date2tstamp(date): 10 | return int(date.timestamp()) 11 | 12 | 13 | class PurgeDataProcessor: 14 | def process(self, db: DbWrapper): 15 | pass 16 | # print('Starting deletion of old records') 17 | # print('schedule size:', db.get_single_value('select count(*) from schedule')) 18 | # print('deleted_rows size', db.get_single_value('select count(*) from deleted_rows')) 19 | # 20 | # # This is a hint to mysql that we shouldn't insert into deleted_rows 21 | # # while purging. The client should handle deleting old events in bulk. 22 | # db.fetch_data('set @TRIGGER_DISABLED=true') 23 | # 24 | # delete_timestamp = date2tstamp(datetime.now() - timedelta(weeks=4)) 25 | # print('deleting before', delete_timestamp) 26 | # schedule_deletes = db.update_item( 27 | # "DELETE FROM `schedule` WHERE end_timestamp < {}".format(delete_timestamp)) 28 | # deleted_row_deletes = db.update_item( 29 | # "DELETE FROM `deleted_rows` WHERE tstamp < {}".format(delete_timestamp)) 30 | # 31 | # logger.info("purged {} old schedules and {} old deleted_rows".format(schedule_deletes, deleted_row_deletes)) 32 | -------------------------------------------------------------------------------- /etl/pad/storage_processor/rank_reward_processor.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os 3 | 4 | from pad.db.db_util import DbWrapper 5 | from pad.storage.rank_reward import RankReward 6 | 7 | __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) 8 | 9 | 10 | class RankRewardProcessor(object): 11 | def __init__(self): 12 | with open(os.path.join(__location__, 'rank_reward.csv')) as f: 13 | reader = csv.reader(f) 14 | next(reader) 15 | self.rank_rewards = list(reader) 16 | 17 | def process(self, db: DbWrapper): 18 | for row in self.rank_rewards: 19 | item = RankReward.from_csv(row) 20 | db.insert_or_update(item) 21 | -------------------------------------------------------------------------------- /etl/pad/storage_processor/schedule_processor.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | from typing import List 4 | 5 | from datetime import timedelta 6 | 7 | from pad.db.db_util import DbWrapper 8 | from pad.raw.bonus import BonusType 9 | from pad.raw_processor import crossed_data 10 | from pad.raw_processor.merged_data import MergedBonus 11 | from pad.storage.schedule import ScheduleEvent 12 | 13 | logger = logging.getLogger('processor') 14 | human_fix_logger = logging.getLogger('human_fix') 15 | 16 | SUPPORTED_BONUS_TYPES = [ 17 | # Lists dungeons that are open. 18 | BonusType.dungeon, 19 | # Might need this to tag tournaments? 20 | BonusType.tournament_active, 21 | ] 22 | 23 | WARN_BONUS_TYPES = [ 24 | # Nothing should ever be unknown. 25 | BonusType.unknown, 26 | ] 27 | 28 | IGNORED_BONUS_TYPES = [ 29 | # Support these eventually 30 | BonusType.exp_boost, 31 | BonusType.coin_boost, 32 | BonusType.drop_boost, 33 | BonusType.stamina_reduction, 34 | BonusType.plus_drop_rate_1, 35 | BonusType.plus_drop_rate_2, 36 | 37 | # Support these non-dungeon ones eventually 38 | BonusType.feed_xp_bonus_chance, 39 | BonusType.feed_skillup_bonus_chance, 40 | 41 | # Probably never need these 42 | BonusType.send_egg_roll, 43 | BonusType.pem_event, 44 | BonusType.rem_event, 45 | BonusType.pem_cost, # Consider supporting this if PEM starts changing cost again 46 | BonusType.multiplayer_announcement, 47 | BonusType.tournament_text, 48 | BonusType.dungeon_web_info_link, 49 | BonusType.exchange_text, 50 | BonusType.stone_purchase_text, 51 | BonusType.daily_dragons, 52 | BonusType.monthly_quest_dungeon, # This has a dupe dungeon entry. 53 | BonusType.pad_metadata, 54 | BonusType.pad_metadata_2, 55 | BonusType.story_category_text, 56 | BonusType.normal_announcement, 57 | BonusType.technical_announcement, 58 | BonusType.special_dungeon_info_link, 59 | BonusType.dungeon_unavailable_popup, # This might be useful if it becomes more common 60 | BonusType.reward_table_8p, 61 | BonusType.vem_event, 62 | BonusType.vem_rules_1, 63 | BonusType.vem_rules_2, 64 | 65 | # Might need this to tag dungeons as tournaments, even closed ones. 66 | # Probably happens outside this processor though. 67 | BonusType.tournament_closed, 68 | 69 | # Needs research if we can extract scores? 70 | # Probably happens outside this processor though. 71 | BonusType.score_announcement, 72 | 73 | # These should be handled by the Dungeon processor. 74 | # Rewards for clearing the dungeon (and other garbage) 75 | BonusType.dungeon_special_event, 76 | # Rewards for clearing the sub dungeon (and other garbage) 77 | BonusType.dungeon_floor_text, 78 | # Lists multiplayer dungeons with special events active? 79 | BonusType.multiplayer_dungeon_text, 80 | 81 | # This has a variety of stuff, but mostly +X notifications. 82 | BonusType.gift_dungeon_with_reward, 83 | 84 | # Not sure what this does; has no text, no dungeon, 85 | # only a value of 10000, runs for a week, spotted in NA nad JP. 86 | BonusType.unknown_13, 87 | BonusType.unknown_14, 88 | # Not sure what it does, some kind of flag, spotted in JP. 89 | BonusType.unknown_40, 90 | ] 91 | 92 | 93 | class ScheduleProcessor(object): 94 | def __init__(self, data: crossed_data.CrossServerDatabase): 95 | self.data = data 96 | 97 | def process(self, db: DbWrapper): 98 | logger.info('loading JP events') 99 | self._process_schedule(db, self.data.jp_bonuses) 100 | logger.info('loading NA events') 101 | self._process_schedule(db, self.data.na_bonuses) 102 | logger.info('loading KR events') 103 | self._process_schedule(db, self.data.kr_bonuses) 104 | logger.info('done loading schedule data') 105 | 106 | def _process_schedule(self, db: DbWrapper, bonuses: List[MergedBonus]): 107 | for bonus in bonuses: 108 | bonus_type = bonus.bonus.bonus_info.bonus_type 109 | 110 | if bonus_type in WARN_BONUS_TYPES: 111 | human_fix_logger.error('Unexpected bonus: %s\n%s', bonus, bonus.bonus.raw) 112 | continue 113 | 114 | if bonus_type in IGNORED_BONUS_TYPES: 115 | logger.debug('Ignored bonus: %s', bonus) 116 | continue 117 | 118 | if bonus_type not in SUPPORTED_BONUS_TYPES: 119 | human_fix_logger.error('Incorrectly configured bonus: %s', bonus) 120 | continue 121 | 122 | if bonus.open_duration() > timedelta(days=60): 123 | logger.debug('Skipping long bonus: %s', bonus) 124 | continue 125 | 126 | if bonus.dungeon: 127 | logger.debug('Creating event: %s', bonus) 128 | event = ScheduleEvent.from_mb(bonus) 129 | db.insert_or_update(event) 130 | else: 131 | human_fix_logger.error('Dungeon with no dungeon attached: %s', bonus) 132 | -------------------------------------------------------------------------------- /etl/pad/storage_processor/series_processor.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | 5 | from pad.db.db_util import DbWrapper 6 | from pad.raw_processor import crossed_data 7 | from pad.storage.series import Series 8 | 9 | __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) 10 | 11 | logger = logging.getLogger('processor') 12 | 13 | 14 | class SeriesProcessor(object): 15 | def __init__(self, data: crossed_data.CrossServerDatabase): 16 | with open(os.path.join(__location__, 'series.json')) as f: 17 | self.series = json.load(f) 18 | self.data = data 19 | 20 | def process(self, db: DbWrapper): 21 | for raw in self.series: 22 | item = Series.from_json(raw) 23 | db.insert_or_update(item) 24 | -------------------------------------------------------------------------------- /etl/pad/storage_processor/shared_storage.py: -------------------------------------------------------------------------------- 1 | import os 2 | from abc import ABC, abstractmethod 3 | 4 | from pad.common.utils import classproperty 5 | from pad.db.sql_item import SimpleSqlItem 6 | 7 | 8 | class ServerDependentSqlItem(SimpleSqlItem, ABC): 9 | @property 10 | @abstractmethod 11 | def BASE_TABLE(self): 12 | ... 13 | 14 | @classproperty 15 | def TABLE(cls) -> str: 16 | server = os.environ.get("CURRENT_PIPELINE_SERVER") or "" 17 | if server.upper() == "NA": 18 | return cls.BASE_TABLE + '_na' 19 | # elif server.upper() == "JP": 20 | # return cls.BASE_TABLE + '_jp' 21 | # elif server.upper() == "KR": 22 | # return cls.BASE_TABLE + '_kr' 23 | else: 24 | return cls.BASE_TABLE 25 | -------------------------------------------------------------------------------- /etl/pad/storage_processor/skill_tag_leader.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "leader_tag_id": 1, 4 | "name_ja": "自動回復", 5 | "name_ko": "자동 회복", 6 | "name_en": "Auto Heal", 7 | "order_idx": 10 8 | }, 9 | { 10 | "leader_tag_id": 25, 11 | "name_ja": "HP強化", 12 | "name_ko": "HP 강화", 13 | "name_en": "Enhanced HP", 14 | "order_idx": 1 15 | }, 16 | { 17 | "leader_tag_id": 26, 18 | "name_ja": "攻撃強化", 19 | "name_ko": "공격 강화", 20 | "name_en": "Enhanced ATK", 21 | "order_idx": 2 22 | }, 23 | { 24 | "leader_tag_id": 27, 25 | "name_ja": "回復強化", 26 | "name_ko": "회복 강화", 27 | "name_en": "Enhanced RCV", 28 | "order_idx": 3 29 | }, 30 | { 31 | "leader_tag_id": 28, 32 | "name_ja": "HP/攻撃 強化", 33 | "name_ko": "HP/공격 강화", 34 | "name_en": "Enhanced HP/ATK", 35 | "order_idx": 4 36 | }, 37 | { 38 | "leader_tag_id": 29, 39 | "name_ja": "HP/回復 強化", 40 | "name_ko": "HP/회복 강화", 41 | "name_en": "Enhanced HP/RCV", 42 | "order_idx": 5 43 | }, 44 | { 45 | "leader_tag_id": 30, 46 | "name_ja": "攻撃/回復 強化", 47 | "name_ko": "공격/회복 강화", 48 | "name_en": "Enhanced ATK/RCV", 49 | "order_idx": 6 50 | }, 51 | { 52 | "leader_tag_id": 31, 53 | "name_ja": "HP/攻撃/回復 強化", 54 | "name_ko": "HP/공격/회복 강화", 55 | "name_en": "Enhanced HP/ATK/RCV", 56 | "order_idx": 7 57 | }, 58 | { 59 | "leader_tag_id": 32, 60 | "name_ja": "ダメージ減少", 61 | "name_ko": "대미지 경감", 62 | "name_en": "Reduce Damage", 63 | "order_idx": 11 64 | }, 65 | { 66 | "leader_tag_id": 34, 67 | "name_ja": "追加攻撃", 68 | "name_ko": "추가 공격", 69 | "name_en": "Additional Attack", 70 | "order_idx": 13 71 | }, 72 | { 73 | "leader_tag_id": 35, 74 | "name_ja": "反撃", 75 | "name_ko": "반격", 76 | "name_en": "Counterattack", 77 | "order_idx": 14 78 | }, 79 | { 80 | "leader_tag_id": 36, 81 | "name_ja": "根性", 82 | "name_ko": "근성", 83 | "name_en": "Resolve", 84 | "order_idx": 15 85 | }, 86 | { 87 | "leader_tag_id": 37, 88 | "name_ja": "時間延長", 89 | "name_ko": "시간 연장", 90 | "name_en": "Extend Time", 91 | "order_idx": 16 92 | }, 93 | { 94 | "leader_tag_id": 150, 95 | "name_ja": "コイン", 96 | "name_ko": "코인", 97 | "name_en": "Coin", 98 | "order_idx": 100 99 | }, 100 | { 101 | "leader_tag_id": 160, 102 | "name_ja": "Egg Rate+", 103 | "name_ko": "에그 드롭률+", 104 | "name_en": "Egg Rate+", 105 | "order_idx": 110 106 | }, 107 | { 108 | "leader_tag_id": 170, 109 | "name_ja": "経験値", 110 | "name_ko": "경험치", 111 | "name_en": "Exp", 112 | "order_idx": 170 113 | }, 114 | { 115 | "leader_tag_id": 200, 116 | "name_ja": "盤面変化【7x6マス】", 117 | "name_ko": "드롭판 변경 [7x6]", 118 | "name_en": "Board Change [7x6]", 119 | "order_idx": 200 120 | }, 121 | { 122 | "leader_tag_id": 210, 123 | "name_ja": "落ちコンなし", 124 | "name_ko": "낙차 콤보 없음", 125 | "name_en": "No Skyfall Combos", 126 | "order_idx": 210 127 | }, 128 | { 129 | "leader_tag_id": 211, 130 | "name_ja": "Extra Combos", 131 | "name_ko": "Extra Combos", 132 | "name_en": "Extra Combos", 133 | "order_idx": 210 134 | }, 135 | { 136 | "leader_tag_id": 999, 137 | "name_ja": "ドロップ操作音", 138 | "name_ko": "기타", 139 | "name_en": "Orb Sounds", 140 | "order_idx": 999 141 | } 142 | ] 143 | -------------------------------------------------------------------------------- /etl/pad/storage_processor/skill_tag_processor.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | from pad.db.db_util import DbWrapper 5 | from pad.storage.skill_tag import ActiveSkillTag, LeaderSkillTag 6 | 7 | __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) 8 | 9 | 10 | class SkillTagProcessor(object): 11 | def __init__(self): 12 | with open(os.path.join(__location__, 'skill_tag_active.json')) as f: 13 | self.active_skill_tags = json.load(f) 14 | with open(os.path.join(__location__, 'skill_tag_leader.json')) as f: 15 | self.leader_skill_tags = json.load(f) 16 | 17 | def process(self, db: DbWrapper): 18 | for raw in self.active_skill_tags: 19 | item = ActiveSkillTag.from_json(raw) 20 | db.insert_or_update(item) 21 | for raw in self.leader_skill_tags: 22 | item = LeaderSkillTag.from_json(raw) 23 | db.insert_or_update(item) 24 | -------------------------------------------------------------------------------- /etl/pad/storage_processor/timestamp_processor.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import datetime 3 | 4 | from pad.db.db_util import DbWrapper 5 | from pad.storage.awoken_skill import AwokenSkill 6 | from pad.storage.dungeon import Dungeon, SubDungeon 7 | from pad.storage.egg_machine import EggMachine 8 | from pad.storage.encounter import Encounter, Drop 9 | from pad.storage.enemy_skill import EnemySkill, EnemyData 10 | from pad.storage.monster import Monster, Awakening, Evolution 11 | from pad.storage.monster_skill import ActiveSkill, LeaderSkill 12 | from pad.storage.rank_reward import RankReward 13 | from pad.storage.schedule import ScheduleEvent 14 | from pad.storage.series import Series 15 | from pad.storage.skill_tag import ActiveSkillTag, LeaderSkillTag 16 | from pad.storage.exchange import Exchange 17 | from pad.storage.purchase import Purchase 18 | 19 | logger = logging.getLogger('processor') 20 | 21 | _UPDATE_TABLES = [ 22 | ActiveSkill.TABLE, 23 | ActiveSkillTag.TABLE, 24 | Awakening.TABLE, 25 | AwokenSkill.TABLE, 26 | # This is a special table that gets populated when items are deleted from other tables. 27 | 'deleted_rows', 28 | Drop.TABLE, 29 | Dungeon.TABLE, 30 | Encounter.TABLE, 31 | EggMachine.TABLE, 32 | EnemySkill.TABLE, 33 | EnemyData.TABLE, 34 | Evolution.TABLE, 35 | Exchange.TABLE, 36 | Purchase.TABLE, 37 | LeaderSkill.TABLE, 38 | LeaderSkillTag.TABLE, 39 | Monster.TABLE, 40 | RankReward.TABLE, 41 | ScheduleEvent.TABLE, 42 | Series.TABLE, 43 | SubDungeon.TABLE, 44 | ] 45 | 46 | 47 | class TimestampProcessor(object): 48 | def __init__(self): 49 | pass 50 | 51 | def process(self, db: DbWrapper): 52 | logger.info('timestamp update of %s tables', len(_UPDATE_TABLES)) 53 | for table in _UPDATE_TABLES: 54 | max_tstamp_sql = 'SELECT MAX(tstamp) AS tstamp FROM `{}`'.format(table) 55 | tstamp = db.get_single_value(max_tstamp_sql, op=int, fail_on_empty=False) 56 | if tstamp is None: 57 | logger.error('Skipping tstamp update for {}'.format(table)) 58 | continue 59 | update_sql = "INSERT INTO timestamps (name, tstamp) values ('{}', {}) ON DUPLICATE KEY UPDATE tstamp = {}".format( 60 | table, tstamp, tstamp) 61 | rows_updated = db.update_item(update_sql) 62 | if rows_updated: 63 | logger.info('Updated tstamp for {} to {}'.format(table, tstamp)) 64 | logger.info('done updating timestamps') 65 | -------------------------------------------------------------------------------- /etl/pad/storage_processor/wave_processor.py: -------------------------------------------------------------------------------- 1 | from pad.db.db_util import DbWrapper 2 | from pad.dungeon import wave_converter 3 | from pad.raw.dungeon import Dungeon 4 | from pad.raw_processor import crossed_data 5 | 6 | # Finds any dungeon with no encounters set up that has an entry 7 | # in the wave_data table. 8 | from pad.raw_processor.crossed_data import CrossServerDungeon 9 | from pad.storage.wave import WaveItem 10 | 11 | FIND_DUNGEONS_SQL = """ 12 | SELECT dungeon_id 13 | FROM dungeons 14 | INNER JOIN encounters 15 | USING (dungeon_id) 16 | WHERE icon_id IS NULL 17 | AND dungeon_id IN ( 18 | SELECT dungeon_id FROM padguide.wave_data GROUP BY 1 19 | ) 20 | ORDER BY dungeon_id ASC 21 | """ 22 | 23 | 24 | # TODO: Consider renaming subdungeon to floor for consistency 25 | 26 | class WaveProcessor(object): 27 | def __init__(self, data: crossed_data.CrossServerDatabase): 28 | self.data = data 29 | 30 | def process(self, db: DbWrapper): 31 | ready_dungeons = db.fetch_data(FIND_DUNGEONS_SQL) 32 | for row in ready_dungeons: 33 | dungeon_id = row['dungeon_id'] 34 | dungeon = self.data.dungeon_by_id(dungeon_id) 35 | 36 | self._process_dungeon(db, dungeon) 37 | 38 | def _process_dungeon(self, db: DbWrapper, dungeon: CrossServerDungeon): 39 | waves = db.load_multiple_objects(WaveItem, dungeon.dungeon_id) 40 | result_stage = wave_converter.process_waves(self.data, waves) 41 | print('finished computing results for', dungeon.dungeon_id, dungeon.na_dungeon.name) 42 | -------------------------------------------------------------------------------- /etl/pad_data_pull.py: -------------------------------------------------------------------------------- 1 | """ 2 | Pulls data files for specified account/server. 3 | 4 | Requires padkeygen which is not checked in. 5 | """ 6 | import argparse 7 | import os 8 | 9 | from pad.api import pad_api 10 | from pad.common import pad_util 11 | from pad.common.shared_types import Server 12 | from pad.raw import bonus, extra_egg_machine 13 | 14 | parser = argparse.ArgumentParser(description="Extracts PAD API data.", add_help=False) 15 | 16 | inputGroup = parser.add_argument_group("Input") 17 | inputGroup.add_argument("--server", required=True, help="One of [NA, JP, KR]") 18 | inputGroup.add_argument("--user_uuid", required=True, help="Account UUID") 19 | inputGroup.add_argument("--user_intid", required=True, help="Account code") 20 | inputGroup.add_argument("--only_bonus", action='store_true', help="Only populate bonus data") 21 | 22 | outputGroup = parser.add_argument_group("Output") 23 | outputGroup.add_argument("--output_dir", required=True, 24 | help="Path to a folder where output should be saved") 25 | 26 | helpGroup = parser.add_argument_group("Help") 27 | helpGroup.add_argument("-h", "--help", action="help", help="Displays this help message and exits.") 28 | args = parser.parse_args() 29 | 30 | endpoint = None 31 | server = None 32 | if args.server == 'NA': 33 | endpoint = pad_api.ServerEndpoint.NA 34 | server = Server.na 35 | elif args.server == 'JP': 36 | endpoint = pad_api.ServerEndpoint.JA 37 | server = Server.jp 38 | elif args.server == 'KR': 39 | endpoint = pad_api.ServerEndpoint.KR 40 | server = Server.kr 41 | else: 42 | raise Exception('unexpected server:' + args.server) 43 | 44 | api_client = pad_api.PadApiClient(endpoint, args.user_uuid, args.user_intid) 45 | 46 | output_dir = args.output_dir 47 | os.makedirs(output_dir, exist_ok=True) 48 | 49 | api_client.login() 50 | 51 | 52 | def pull_and_write_endpoint(api_client, action): 53 | action_json = api_client.action(action) 54 | 55 | file_name = action.value.name + '.json' 56 | output_file = os.path.join(output_dir, file_name) 57 | print('writing', file_name) 58 | with open(output_file, 'w') as outfile: 59 | pad_util.json_file_dump(action_json, outfile) 60 | 61 | 62 | pull_and_write_endpoint(api_client, pad_api.EndpointAction.DOWNLOAD_LIMITED_BONUS_DATA) 63 | 64 | if args.only_bonus: 65 | print('skipping other downloads') 66 | exit() 67 | 68 | pull_and_write_endpoint(api_client, pad_api.EndpointAction.DOWNLOAD_CARD_DATA) 69 | pull_and_write_endpoint(api_client, pad_api.EndpointAction.DOWNLOAD_DUNGEON_DATA) 70 | pull_and_write_endpoint(api_client, pad_api.EndpointAction.DOWNLOAD_SKILL_DATA) 71 | pull_and_write_endpoint(api_client, pad_api.EndpointAction.DOWNLOAD_ENEMY_SKILL_DATA) 72 | pull_and_write_endpoint(api_client, pad_api.EndpointAction.DOWNLOAD_MONSTER_EXCHANGE) 73 | pull_and_write_endpoint(api_client, pad_api.EndpointAction.SHOP_ITEM) 74 | 75 | api_client.load_player_data() 76 | player_data = api_client.player_data 77 | bonus_data = bonus.load_bonus_data(data_dir=output_dir, 78 | server=server) 79 | 80 | # Egg machine extraction 81 | egg_machines = extra_egg_machine.load_from_player_data(data_json=player_data.egg_data, server=server) 82 | egg_machines.extend(extra_egg_machine.machine_from_bonuses(server, bonus_data, 'rem_event', 'Rare Egg Machine')) 83 | egg_machines.extend(extra_egg_machine.machine_from_bonuses(server, bonus_data, 'pem_event', 'Pal Egg Machine')) 84 | egg_machines.extend(extra_egg_machine.machine_from_bonuses(server, bonus_data, 'vem_event', 'Video Egg Machine')) 85 | 86 | for em in egg_machines: 87 | if not em.is_open(): 88 | # Can only pull rates when the machine is live. 89 | continue 90 | 91 | grow = em.egg_machine_row 92 | gtype = em.egg_machine_type 93 | page = api_client.get_egg_machine_page(gtype, grow) 94 | extra_egg_machine.scrape_machine_contents(page, em) 95 | 96 | output_file = os.path.join(output_dir, extra_egg_machine.FILE_NAME) 97 | with open(output_file, 'w') as outfile: 98 | pad_util.json_file_dump(egg_machines, outfile, pretty=True) 99 | -------------------------------------------------------------------------------- /media_pipelines/assets/PADAnimatedGenerator.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import tempfile 4 | 5 | from PIL import Image 6 | 7 | parser = argparse.ArgumentParser( 8 | description="Generates static frames for animated images.", add_help=False) 9 | 10 | inputGroup = parser.add_argument_group("Input") 11 | inputGroup.add_argument("--raw_dir", required=True, help="Path to input BC files") 12 | inputGroup.add_argument("--working_dir", required=True, help="Path to pad-resources project") 13 | 14 | outputGroup = parser.add_argument_group("Output") 15 | outputGroup.add_argument("--output_dir", required=True, 16 | help="Path to a folder where output should be saved") 17 | 18 | helpGroup = parser.add_argument_group("Help") 19 | helpGroup.add_argument("-h", "--help", action="help", help="Displays this help message and exits.") 20 | args = parser.parse_args() 21 | 22 | # The final padded image size (matches the rest of the miru images) 23 | IMAGE_SIZE = (640, 388) 24 | 25 | # The maximum size of the monster to be positioned within the image 26 | IMAGE_SIZE_NO_PADDING = (640 - 70 * 2, 388 - 35 * 2) 27 | 28 | 29 | def blacken_image(image): 30 | pixel_data = image.load() 31 | for y in range(image.size[1]): 32 | for x in range(image.size[0]): 33 | # Check if it's completely transparent 34 | if pixel_data[x, y][3] == 0: 35 | # Set the color to black 36 | pixel_data[x, y] = (0, 0, 0, 0) 37 | 38 | 39 | def generate_resized_image(source_file, dest_file): 40 | # Resizes the image so it has the correct padding 41 | img = Image.open(source_file) 42 | 43 | # Ensure RGBA 44 | img = img.convert('RGBA') 45 | 46 | # Blacken any fully-transparent pixels 47 | blacken_image(img) 48 | 49 | # Trim transparent edges 50 | img = img.crop(img.getbbox()) 51 | 52 | max_size = IMAGE_SIZE_NO_PADDING[0] if img.size[0] > img.size[1] else IMAGE_SIZE_NO_PADDING[1] 53 | img.thumbnail((max_size, max_size), Image.ANTIALIAS) 54 | 55 | old_size = img.size 56 | 57 | new_img = Image.new("RGBA", IMAGE_SIZE) 58 | new_img.paste(img, 59 | (int((IMAGE_SIZE[0] - old_size[0]) / 2), 60 | int((IMAGE_SIZE[1] - old_size[1]) / 2))) 61 | 62 | new_img.save(dest_file) 63 | 64 | 65 | def process_animated(working_dir, pad_id, file_path): 66 | bin_file = 'mons_{}.bin'.format(pad_id) 67 | bin_path = os.path.join('data', 'HT', 'bin', bin_file) 68 | xvfb_prefix = 'xvfb-run -s "-ac -screen 0 640x640x24"' 69 | yarn_cmd = 'yarn --cwd={} render --bin {} --out {} --nobg'.format( 70 | working_dir, bin_path, file_path) 71 | 72 | full_cmd = '{} {}'.format(xvfb_prefix, yarn_cmd) 73 | print('running', full_cmd) 74 | os.system(full_cmd) 75 | print('done') 76 | 77 | 78 | raw_dir = args.raw_dir 79 | working_dir = args.working_dir 80 | output_dir = args.output_dir 81 | 82 | for file_name in sorted(os.listdir(raw_dir)): 83 | if 'mons' not in file_name or 'isanimated' in file_name: 84 | print('skipping', file_name) 85 | continue 86 | 87 | pad_id = file_name.rstrip('.bc').lstrip('mons_').lstrip('0') 88 | final_image_name = '{}.png'.format(pad_id) 89 | corrected_file_path = os.path.join(output_dir, final_image_name) 90 | 91 | if os.path.exists(corrected_file_path): 92 | print('skipping', corrected_file_path) 93 | continue 94 | 95 | print('processing', corrected_file_path) 96 | with tempfile.TemporaryDirectory() as temp_dir: 97 | try: 98 | tmp_corrected_file_path = os.path.join(temp_dir, final_image_name) 99 | process_animated(working_dir, pad_id, tmp_corrected_file_path) 100 | generate_resized_image(tmp_corrected_file_path, corrected_file_path) 101 | except: 102 | print('error skipping', corrected_file_path) 103 | continue 104 | 105 | print('done') 106 | -------------------------------------------------------------------------------- /media_pipelines/assets/PADAnimationGenerator.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import tempfile 4 | 5 | parser = argparse.ArgumentParser( 6 | description="Generates animated for pad monsters.", add_help=False) 7 | 8 | inputGroup = parser.add_argument_group("Input") 9 | inputGroup.add_argument("--raw_dir", required=True, help="Path to input BC files") 10 | inputGroup.add_argument("--working_dir", required=True, help="Path to pad-resources project") 11 | 12 | outputGroup = parser.add_argument_group("Output") 13 | outputGroup.add_argument("--output_dir", required=True, 14 | help="Path to a folder where output should be saved") 15 | 16 | helpGroup = parser.add_argument_group("Help") 17 | helpGroup.add_argument("-h", "--help", action="help", help="Displays this help message and exits.") 18 | args = parser.parse_args() 19 | 20 | GENERATE_GIF_CMD = """ 21 | ffmpeg -i {} \ 22 | -filter_complex 'scale=400:244:flags=lanczos,split [o1] [o2];[o1] palettegen [p]; [o2] fifo [o3];[o3] [p] paletteuse' \ 23 | -r 10 {} 24 | """ 25 | 26 | OPTIMIZE_GIF_CMD = """ 27 | gifsicle -O3 --lossy=80 {} -o {} 28 | """ 29 | 30 | RESIZE_VIDEO_CMD = """ 31 | ffmpeg -i {} -pix_fmt yuv420p -r 24 -c:v libx264 -filter:v "crop=640:390:0:60" {} 32 | """ 33 | 34 | 35 | def run_cmd(cmd, source_file, dest_file): 36 | final_cmd = cmd.format(source_file, dest_file).strip() 37 | print('running', final_cmd) 38 | os.system(final_cmd) 39 | print('done') 40 | 41 | 42 | def process_animated(working_dir, pad_id, file_path): 43 | bin_file = 'mons_{}.bin'.format(pad_id) 44 | bin_path = os.path.join('data', 'HT', 'bin', bin_file) 45 | xvfb_prefix = 'xvfb-run -s "-ac -screen 0 640x640x24"' 46 | yarn_cmd = 'yarn --cwd={} render --bin {} --out {} --nobg --video'.format( 47 | working_dir, bin_path, file_path) 48 | 49 | full_cmd = '{} {}'.format(xvfb_prefix, yarn_cmd) 50 | print('running', full_cmd) 51 | os.system(full_cmd) 52 | print('done') 53 | 54 | 55 | raw_dir = args.raw_dir 56 | working_dir = args.working_dir 57 | output_dir = args.output_dir 58 | 59 | for file_name in sorted(os.listdir(raw_dir)): 60 | if 'isanimated' not in file_name: 61 | continue 62 | 63 | pad_id = file_name.rstrip('.isanimated').lstrip('mons_').lstrip('0') 64 | 65 | video_name = '{}.mp4'.format(pad_id) 66 | video_path = os.path.join(output_dir, video_name) 67 | 68 | if os.path.exists(video_path): 69 | print('skipping', video_path) 70 | else: 71 | print('processing', video_path) 72 | with tempfile.TemporaryDirectory() as temp_dir: 73 | tmp_video_path = os.path.join(temp_dir, video_name) 74 | process_animated(working_dir, pad_id, tmp_video_path) 75 | run_cmd(RESIZE_VIDEO_CMD, tmp_video_path, video_path) 76 | 77 | gif_name = '{}.gif'.format(pad_id) 78 | gif_path = os.path.join(output_dir, gif_name) 79 | 80 | if os.path.exists(gif_path): 81 | print('skipping', gif_path) 82 | else: 83 | print('processing', gif_path) 84 | with tempfile.TemporaryDirectory() as temp_dir: 85 | tmp_gif_path = os.path.join(temp_dir, gif_name) 86 | run_cmd(GENERATE_GIF_CMD, video_path, tmp_gif_path) 87 | run_cmd(OPTIMIZE_GIF_CMD, tmp_gif_path, gif_path) 88 | 89 | print('done') 90 | -------------------------------------------------------------------------------- /media_pipelines/assets/PADHQImageDownload.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import asyncio 3 | import json 4 | import os 5 | import re 6 | import aiohttp 7 | 8 | import pymysql 9 | 10 | parser = argparse.ArgumentParser( 11 | description="Downloads P&D images from the GungHo site.", add_help=False) 12 | 13 | input_group = parser.add_argument_group("Input") 14 | input_group.add_argument("--raw_file_dir", help="Path to input BC files") 15 | input_group.add_argument("--db_config", help="JSON database info") 16 | 17 | outputGroup = parser.add_argument_group("Output") 18 | outputGroup.add_argument("--output_dir", help="Path to a folder where output should be saved") 19 | 20 | helpGroup = parser.add_argument_group("Help") 21 | helpGroup.add_argument("-h", "--help", action="help", help="Displays this help message and exits.") 22 | 23 | args = parser.parse_args() 24 | bin_dir = args.raw_file_dir 25 | output_dir = args.output_dir 26 | 27 | GUNGHO_TEMPLATE = 'https://pad.gungho.jp/member/img/graphic/illust/{}' 28 | 29 | 30 | async def download_file(url, file_path, monster_id, cursor, semaphore): 31 | async with semaphore: 32 | async with aiohttp.ClientSession() as session: 33 | async with session.get(url, allow_redirects=False) as response: 34 | if response.status == 302: 35 | return 36 | if response.status != 200: 37 | print(f"Invalid response code {response.status} for {monster_id}") 38 | return 39 | image_data = await response.content.read() 40 | 41 | with open(file_path, "wb") as f: 42 | f.write(image_data) 43 | 44 | cursor.execute('''INSERT INTO monster_image_sizes (monster_id, hq_png_size, tstamp) 45 | VALUES (%s, %s, UNIX_TIMESTAMP()) 46 | ON DUPLICATE KEY 47 | UPDATE hq_png_size=%s, tstamp=UNIX_TIMESTAMP();''', 48 | (monster_id, len(image_data), len(image_data))) 49 | print("finished downloading monster", monster_id) 50 | 51 | 52 | async def main(): 53 | with open(args.db_config) as f: 54 | db_config = json.load(f) 55 | 56 | db = pymysql.connect(**db_config, autocommit=True) 57 | cur = db.cursor() 58 | file_downloads = [] 59 | semaphore = asyncio.Semaphore(100) 60 | 61 | for file_name in sorted(os.listdir(bin_dir)): 62 | if not (match := re.match(r'mons_0*(\d+).bin', file_name)): 63 | continue 64 | 65 | monster_id = int(match.group(1)) 66 | final_image_name = f'{monster_id:05d}.png' 67 | corrected_file_path = os.path.join(output_dir, final_image_name) 68 | 69 | if os.path.exists(corrected_file_path): 70 | continue 71 | 72 | gungho_url = GUNGHO_TEMPLATE.format(monster_id) 73 | 74 | try: 75 | file_downloads.append(download_file(gungho_url, corrected_file_path, 76 | int(monster_id), cur, semaphore)) 77 | except Exception as e: 78 | print('Failed to download: ', e) 79 | 80 | await asyncio.gather(*file_downloads) 81 | 82 | cur.close() 83 | db.close() 84 | print('done') 85 | 86 | 87 | if __name__ == "__main__": 88 | asyncio.run(main()) 89 | -------------------------------------------------------------------------------- /media_pipelines/assets/README.md: -------------------------------------------------------------------------------- 1 | For information on image use, see [our GitHub wiki](https://github.com/TsubakiBotPad/pad-data-pipeline/wiki/Image-Use). 2 | -------------------------------------------------------------------------------- /media_pipelines/extras/PADBGMDownload.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import re 4 | import time 5 | import urllib.request 6 | 7 | import padtools 8 | 9 | parser = argparse.ArgumentParser(description="Downloads PAD BGM", add_help=False) 10 | 11 | input_group = parser.add_argument_group("Input") 12 | input_group.add_argument("--server", required=True, help="na or jp") 13 | input_group.add_argument("--db_config", help="JSON database info") 14 | 15 | output_group = parser.add_argument_group("Output") 16 | output_group.add_argument("--extra_dir", help="Path to a folder where raw extras should be saved") 17 | output_group.add_argument("--final_dir", help="Path to a folder where output should be saved") 18 | 19 | settings_group = parser.add_argument_group("Settings") 20 | settings_group.add_argument("--refresh", action="store_true", help="Refresh downloaded data") 21 | 22 | help_group = parser.add_argument_group("Help") 23 | help_group.add_argument("-h", "--help", action="help", help="Displays this help message and exits.") 24 | args = parser.parse_args() 25 | 26 | server = args.server.lower() 27 | 28 | extras = [] 29 | if server == 'na': 30 | extras = padtools.regions.north_america.server.extras # noqa 31 | elif server == 'jp': 32 | extras = padtools.regions.japan.server.extras # noqa 33 | else: 34 | print('invalid server:', server) 35 | exit(1) 36 | 37 | 38 | def download_file(url, file_path): 39 | response_object = urllib.request.urlopen(url) 40 | with response_object as response: 41 | file_data = response.read() 42 | with open(file_path, "wb") as f: 43 | f.write(file_data) 44 | 45 | 46 | print('Found', len(extras), 'extras total') 47 | 48 | raw_dir = args.extra_dir 49 | fixed_dir = args.final_dir 50 | os.makedirs(raw_dir, exist_ok=True) 51 | os.makedirs(fixed_dir, exist_ok=True) 52 | 53 | raw_dir = os.path.join(raw_dir, server) 54 | fixed_dir = os.path.join(fixed_dir, server) 55 | os.makedirs(raw_dir, exist_ok=True) 56 | os.makedirs(fixed_dir, exist_ok=True) 57 | 58 | refresh = args.refresh 59 | for extra in extras: 60 | file_name = extra.file_name 61 | if not (match := re.match(r'bgm_0*(\d+)\.caf', file_name)): 62 | print('skipping', file_name) 63 | continue 64 | 65 | in_file = os.path.join(raw_dir, file_name) 66 | if not refresh and os.path.exists(in_file): 67 | print('file exists', in_file) 68 | continue 69 | 70 | print('downloading', extra.url, 'to', in_file) 71 | download_file(extra.url, in_file) 72 | 73 | bgm_id = match.group(1) 74 | 75 | out_file = os.path.join(fixed_dir, '{}.mp3'.format(file_name.rstrip('.caf'))) 76 | cmd = 'ffmpeg -i {} -hide_banner -loglevel warning -nostats -y -ac 1 {}'.format(in_file, out_file) 77 | print('running', cmd) 78 | out = os.system(cmd) 79 | if out != 0: 80 | print(out) 81 | time.sleep(.1) 82 | 83 | print('done') 84 | -------------------------------------------------------------------------------- /media_pipelines/extras/PADDMSGParser.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import re 4 | import urllib.request 5 | from typing import Optional 6 | 7 | import padtools 8 | import pymysql 9 | from padtexturetool.texture_reader import decrypt_and_decompress_binary_blob 10 | 11 | parser = argparse.ArgumentParser(description="Parses PAD DMSG files", add_help=False) 12 | 13 | input_group = parser.add_argument_group("Input") 14 | input_group.add_argument("--server", required=True, help="na or jp") 15 | input_group.add_argument("--db_config", help="JSON database info") 16 | 17 | help_group = parser.add_argument_group("Help") 18 | help_group.add_argument("-h", "--help", action="help", help="Displays this help message and exits.") 19 | args = parser.parse_args() 20 | 21 | server = args.server.lower() 22 | 23 | extras = [] 24 | if server == 'na': 25 | extras = padtools.regions.north_america.server.extras # noqa 26 | elif server == 'jp': 27 | extras = padtools.regions.japan.server.extras # noqa TODO: Don't require these 28 | else: 29 | print('invalid server:', server) 30 | exit(1) 31 | 32 | 33 | def readint(data, off, sz=4): 34 | return int.from_bytes(data[off:off + sz], 'little') 35 | 36 | 37 | def readstr(data, off, sz=None, encoding='utf8'): 38 | # If size is None, read until \0. 39 | if sz is not None: 40 | return data[off:off + sz].rstrip(b'\0').decode(encoding) 41 | stop = data.find(b'\0', off) 42 | return data[off:stop].rstrip(b'\0').decode(encoding) 43 | 44 | 45 | def parse_dmsg(dmsg: bytes): 46 | assert readstr(dmsg, 0, 4) == 'DMSG' 47 | colct = readint(dmsg, 4, 2) 48 | rowct = readint(dmsg, 6, 2) 49 | table = [] 50 | 51 | def getstr(i, j): 52 | offset = readint(dmsg, 8 + 4 * (i * colct + j)) 53 | if not offset: 54 | return '' 55 | return readstr(dmsg, offset) 56 | 57 | table = [[getstr(i, j) for j in range(colct)] for i in range(rowct)] 58 | return table 59 | 60 | 61 | name_svr = 'name_en' if server == 'na' else 'name_ja' if server == 'jp' else None 62 | 63 | with open(args.db_config) as f: 64 | db_config = json.load(f) 65 | 66 | db = pymysql.connect(**db_config, autocommit=True) 67 | cur = db.cursor() 68 | 69 | 70 | def insert_or_replace_into(data: dict, table: str, id_col: str): 71 | updates = [] 72 | update_repls = () 73 | for key, val in list(data.items()): 74 | if val == '-': 75 | del data[key] 76 | continue 77 | if key != id_col: 78 | updates.append(f'{key}=%s') 79 | update_repls += (val,) 80 | 81 | if updates: 82 | query = (f"INSERT INTO {table} ({', '.join(data)}, tstamp)" 83 | f" VALUES ({', '.join('%s' for _ in data)}, UNIX_TIMESTAMP())" 84 | f" ON DUPLICATE KEY UPDATE {', '.join(updates)};") 85 | cur.execute(query, (*data.values(),) + update_repls) 86 | 87 | 88 | # Skin Data 89 | def bgm_name_to_id(name: str) -> Optional[int]: 90 | if (match := re.match(r'bgm_0*(\d+)', name)): 91 | return int(match.group(1)) 92 | return None 93 | 94 | 95 | skindata = next((e for e in extras if e.file_name == 'skindata.bin')) 96 | 97 | with urllib.request.urlopen(skindata.url) as resp: 98 | skindata = resp.read() 99 | skindata = parse_dmsg(decrypt_and_decompress_binary_blob(skindata)) 100 | 101 | for row in skindata: 102 | if float(row[0]) >= 10000: # GH sucks and sometimes likes to give these as floats 103 | if bgm_name_to_id(row[11]): 104 | insert_or_replace_into({'bgm_id': bgm_name_to_id(row[11]), name_svr: row[10].replace('\n', '')}, 105 | 'bgms', 'bgm_id') 106 | if bgm_name_to_id(row[13]): 107 | insert_or_replace_into({'bgm_id': bgm_name_to_id(row[13]), name_svr: row[12].replace('\n', '')}, 108 | 'bgms', 'bgm_id') 109 | insert_or_replace_into({ 110 | 'bgm_set_id': float(row[0]), 111 | name_svr: row[1].replace('\n', ''), 112 | 'route_bgm_id': bgm_name_to_id(row[11]), 113 | 'boss_bgm_id': bgm_name_to_id(row[13]), 114 | 'unused': float(row[8] or '0') 115 | }, 'bgm_sets', 'bgm_set_id') 116 | 117 | cur.close() 118 | db.close() 119 | -------------------------------------------------------------------------------- /media_pipelines/extras/PADFullMediaDownload.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import urllib.request 4 | 5 | import padtools 6 | 7 | parser = argparse.ArgumentParser( 8 | description="Downloads P&D BGM", add_help=False) 9 | 10 | inputGroup = parser.add_argument_group("Input") 11 | inputGroup.add_argument("--server", required=True, help="na or jp") 12 | 13 | outputGroup = parser.add_argument_group("Output") 14 | outputGroup.add_argument("--cache_dir", help="Path to a folder where output should be saved") 15 | outputGroup.add_argument("--final_dir", help="Path to a folder where output should be saved") 16 | 17 | helpGroup = parser.add_argument_group("Help") 18 | helpGroup.add_argument("-h", "--help", action="help", help="Displays this help message and exits.") 19 | args = parser.parse_args() 20 | 21 | server = args.server.lower() 22 | 23 | extras = [] 24 | if args.server == 'na': 25 | extras = padtools.regions.north_america.server.extras 26 | elif args.server == 'jp': 27 | extras = padtools.regions.japan.server.extras 28 | 29 | 30 | def download_file(url, file_path): 31 | response_object = urllib.request.urlopen(url) 32 | with response_object as response: 33 | file_data = response.read() 34 | with open(file_path, "wb") as f: 35 | f.write(file_data) 36 | 37 | 38 | print('Found', len(extras), 'extras total') 39 | 40 | raw_dir = args.cache_dir 41 | fixed_dir = args.final_dir 42 | os.makedirs(raw_dir, exist_ok=True) 43 | os.makedirs(fixed_dir, exist_ok=True) 44 | 45 | raw_dir = os.path.join(raw_dir, server) 46 | fixed_dir = os.path.join(fixed_dir, server) 47 | os.makedirs(raw_dir, exist_ok=True) 48 | os.makedirs(fixed_dir, exist_ok=True) 49 | 50 | for extra in extras: 51 | raw_file_name = extra.file_name 52 | raw_file_path = os.path.join(raw_dir, raw_file_name) 53 | if os.path.exists(raw_file_path): 54 | print('file exists', raw_file_path) 55 | continue 56 | 57 | print('downloading', extra.url, 'to', raw_file_path) 58 | download_file(extra.url, raw_file_path) 59 | 60 | for file_name in os.listdir(raw_dir): 61 | in_file = os.path.join(raw_dir, file_name) 62 | 63 | if file_name.endswith(b'.wav'): 64 | out_file = os.path.join(fixed_dir, '{}.wav'.format(file_name.rstrip(b'.wav'))) 65 | 66 | cmd = 'sox -t ima -r 44100 -e ima-adpcm -v .5 {} -e signed-integer -b 16 {}'.format(in_file, out_file) 67 | elif file_name.endswith(b'.caf'): 68 | out_file = os.path.join(fixed_dir, '{}.mp3'.format(file_name.rstrip(b'.caf'))) 69 | 70 | cmd = 'ffmpeg -i {} -hide_banner -loglevel warning -nostats -y -ac 1 {}'.format(in_file, out_file) 71 | else: 72 | print(b"skipping unknown type: " + file_name) 73 | continue 74 | print('running', cmd) 75 | out = os.system(cmd) 76 | if out != 0: 77 | print(out) 78 | 79 | print('done') 80 | -------------------------------------------------------------------------------- /media_pipelines/extras/PADOrbStylesDownload.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import shutil 4 | import sys 5 | import urllib.request 6 | 7 | import padtools 8 | from PIL import Image 9 | 10 | parser = argparse.ArgumentParser( 11 | description="Downloads P&D Orb Styles (alternate skins)", add_help=False) 12 | 13 | inputGroup = parser.add_argument_group("Input") 14 | inputGroup.add_argument("--server", required=True, help="na or jp") 15 | 16 | outputGroup = parser.add_argument_group("Output") 17 | inputGroup.add_argument("--cache_dir", required=True, help="Path to a folder where output should be saved") 18 | outputGroup.add_argument("--output_dir", required=True, help="Path to a folder where output should be saved") 19 | 20 | helpGroup = parser.add_argument_group("Help") 21 | helpGroup.add_argument("-h", "--help", action="help", help="Displays this help message and exits.") 22 | args = parser.parse_args() 23 | 24 | server = args.server.lower() 25 | cache_dir = args.cache_dir 26 | output_dir = args.output_dir 27 | 28 | extras = [] 29 | if args.server == 'na': 30 | extras = padtools.regions.north_america.server.extras 31 | elif args.server == 'jp': 32 | extras = padtools.regions.japan.server.extras 33 | 34 | 35 | def download_file(url, file_path): 36 | response_object = urllib.request.urlopen(url) 37 | with response_object as response: 38 | file_data = response.read() 39 | with open(file_path, "wb") as f: 40 | f.write(file_data) 41 | 42 | 43 | print('Found', len(extras), 'extras total') 44 | 45 | cache_dir = os.path.join(cache_dir, server) 46 | cache_raw_dir = os.path.join(cache_dir, 'raw') 47 | cache_extract_dir = os.path.join(cache_dir, 'extract') 48 | output_dir = os.path.join(output_dir, server) 49 | os.makedirs(cache_raw_dir, exist_ok=True) 50 | os.makedirs(cache_extract_dir, exist_ok=True) 51 | os.makedirs(output_dir, exist_ok=True) 52 | 53 | python_exec = sys.executable 54 | cur_file_path = os.path.dirname(os.path.realpath(__file__)) 55 | tool_path = os.path.join(cur_file_path, '..', 'assets', 'PADTextureTool.py') 56 | 57 | should_always_process = False 58 | 59 | for extra in extras: 60 | raw_file_name = extra.file_name 61 | if not raw_file_name.startswith('block') or not raw_file_name.endswith('.btex'): 62 | continue 63 | 64 | raw_file_path = os.path.join(cache_raw_dir, raw_file_name) 65 | if os.path.exists(raw_file_path) and not should_always_process: 66 | print('file exists', raw_file_path) 67 | else: 68 | print('downloading', extra.url, 'to', raw_file_path) 69 | download_file(extra.url, raw_file_path) 70 | 71 | extract_file_name = raw_file_name.upper().replace('BTEX', 'PNG') 72 | extract_file_path = os.path.join(cache_extract_dir, extract_file_name) 73 | 74 | if os.path.exists(extract_file_path) and not should_always_process: 75 | print('skipping existing file', extract_file_path) 76 | else: 77 | print('processing', raw_file_path, 'to', cache_extract_dir, 'with name', extract_file_name) 78 | os.system('{python} {tool} -o={output} {input}'.format( 79 | python=python_exec, 80 | tool=tool_path, 81 | input=raw_file_path, 82 | output=cache_extract_dir)) 83 | 84 | final_file_name = extract_file_name.replace('BLOCK', '').lower() 85 | final_file_path = os.path.join(output_dir, final_file_name) 86 | 87 | if os.path.exists(final_file_path) and not should_always_process: 88 | print('skipping existing file', final_file_path) 89 | else: 90 | print('copying', extract_file_path, 'to', final_file_path) 91 | shutil.copy2(extract_file_path, final_file_path) 92 | 93 | img = Image.open(final_file_path) 94 | orb_width, spacer = 100, 4 95 | 96 | x, y = 0, 0 97 | orb_path = final_file_path.replace('.png', '_00.png') 98 | img.crop(box=(x, y, x + orb_width, y + orb_width)).save(orb_path) 99 | 100 | x += orb_width + spacer 101 | orb_path = final_file_path.replace('.png', '_01.png') 102 | img.crop(box=(x, y, x + orb_width, y + orb_width)).save(orb_path) 103 | 104 | x += orb_width + spacer 105 | orb_path = final_file_path.replace('.png', '_02.png') 106 | img.crop(box=(x, y, x + orb_width, y + orb_width)).save(orb_path) 107 | 108 | x += orb_width + spacer 109 | orb_path = final_file_path.replace('.png', '_03.png') 110 | img.crop(box=(x, y, x + orb_width, y + orb_width)).save(orb_path) 111 | 112 | y += orb_width + spacer 113 | x = 0 114 | orb_path = final_file_path.replace('.png', '_04.png') 115 | img.crop(box=(x, y, x + orb_width, y + orb_width)).save(orb_path) 116 | 117 | print('done') 118 | -------------------------------------------------------------------------------- /media_pipelines/extras/PADStoryDownload.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import urllib.request 4 | 5 | import padtools 6 | 7 | parser = argparse.ArgumentParser( 8 | description="Downloads P&D story files (and decodes them)", add_help=False) 9 | 10 | inputGroup = parser.add_argument_group("Input") 11 | inputGroup.add_argument("--server", required=True, help="na or jp") 12 | inputGroup.add_argument("--tool_dir", required=True, help="Path to decoder tool") 13 | 14 | outputGroup = parser.add_argument_group("Output") 15 | outputGroup.add_argument("--cache_dir", help="Path to a folder where output should be saved") 16 | outputGroup.add_argument("--output_dir", help="Path to a folder where output should be saved") 17 | 18 | helpGroup = parser.add_argument_group("Help") 19 | helpGroup.add_argument("-h", "--help", action="help", help="Displays this help message and exits.") 20 | args = parser.parse_args() 21 | 22 | server = args.server.lower() 23 | cache_dir = args.cache_dir 24 | output_dir = args.output_dir 25 | 26 | extras = [] 27 | if args.server == 'na': 28 | extras = padtools.regions.north_america.server.extras 29 | elif args.server == 'jp': 30 | extras = padtools.regions.japan.server.extras 31 | 32 | 33 | def download_file(url, file_path): 34 | response_object = urllib.request.urlopen(url) 35 | with response_object as response: 36 | file_data = response.read() 37 | with open(file_path, "wb") as f: 38 | f.write(file_data) 39 | 40 | 41 | def decode_file(in_file, out_file): 42 | cmd = 'python3 {}/wPADTextureTool.py {} --outfile {}'.format(args.tool_dir, in_file, out_file) 43 | print('running', cmd) 44 | os.system(cmd) 45 | 46 | 47 | def decode_image(in_file, out_dir): 48 | cmd = 'python3 {}/PADTextureTool.py {} --outdir {}'.format(args.tool_dir, in_file, out_dir) 49 | print('running', cmd) 50 | os.system(cmd) 51 | 52 | 53 | print('Found', len(extras), 'extras total') 54 | 55 | raw_dir = os.path.join(cache_dir, 'raw') 56 | fixed_dir = output_dir 57 | 58 | raw_dir = os.path.join(raw_dir, server) 59 | fixed_dir = os.path.join(fixed_dir, server) 60 | 61 | raw_text_dir = os.path.join(raw_dir, 'text') 62 | raw_image_dir = os.path.join(raw_dir, 'image') 63 | 64 | fixed_text_dir = os.path.join(fixed_dir, 'text') 65 | fixed_image_dir = os.path.join(fixed_dir, 'image') 66 | 67 | os.makedirs(raw_image_dir, exist_ok=True) 68 | os.makedirs(raw_text_dir, exist_ok=True) 69 | os.makedirs(fixed_image_dir, exist_ok=True) 70 | os.makedirs(fixed_text_dir, exist_ok=True) 71 | 72 | for extra in extras: 73 | raw_file_name = extra.file_name 74 | if raw_file_name.startswith('st_') and raw_file_name.endswith('.txt'): 75 | raw_file_path = os.path.join(raw_text_dir, raw_file_name) 76 | fixed_file_path = os.path.join(fixed_text_dir, raw_file_name) 77 | fixed_file_dir = fixed_text_dir 78 | do_decode_file = True 79 | elif raw_file_name.startswith('st_mons') and raw_file_name.endswith('.bin'): 80 | raw_file_path = os.path.join(raw_image_dir, raw_file_name) 81 | fixed_file_path = os.path.join(fixed_image_dir, raw_file_name) 82 | fixed_file_dir = fixed_image_dir 83 | do_decode_file = False 84 | else: 85 | print('skipping', raw_file_name) 86 | continue 87 | 88 | if not os.path.exists(raw_file_path): 89 | print('downloading', extra.url, 'to', raw_file_path) 90 | download_file(extra.url, raw_file_path) 91 | else: 92 | print('raw file exists', raw_file_path) 93 | 94 | if not os.path.exists(fixed_file_path): 95 | print('decoding', raw_file_path, 'to', fixed_file_path) 96 | if do_decode_file: 97 | decode_file(raw_file_path, fixed_file_path) 98 | else: 99 | decode_image(raw_file_path, fixed_file_dir) 100 | else: 101 | print('fixed file exists', fixed_file_path) 102 | 103 | print('done') 104 | -------------------------------------------------------------------------------- /media_pipelines/extras/PADVoiceDownload.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import os 4 | import shutil 5 | import urllib.request 6 | from collections import defaultdict 7 | 8 | import padtools 9 | 10 | parser = argparse.ArgumentParser( 11 | description="Downloads P&D voices (and fixed them)", add_help=False) 12 | 13 | inputGroup = parser.add_argument_group("Input") 14 | inputGroup.add_argument("--server", required=True, help="na or jp") 15 | 16 | outputGroup = parser.add_argument_group("Output") 17 | outputGroup.add_argument("--cache_dir", help="Path to a folder where output should be saved") 18 | outputGroup.add_argument("--final_dir", help="Path to a folder where output should be saved") 19 | 20 | helpGroup = parser.add_argument_group("Help") 21 | helpGroup.add_argument("-h", "--help", action="help", help="Displays this help message and exits.") 22 | args = parser.parse_args() 23 | 24 | server = args.server.lower() 25 | 26 | extras = [] 27 | if args.server == 'na': 28 | extras = padtools.regions.north_america.server.extras 29 | elif args.server == 'jp': 30 | extras = padtools.regions.japan.server.extras 31 | 32 | 33 | def download_file(url, file_path): 34 | response_object = urllib.request.urlopen(url) 35 | with response_object as response: 36 | file_data = response.read() 37 | with open(file_path, "wb") as f: 38 | f.write(file_data) 39 | 40 | 41 | print('Found', len(extras), 'extras total') 42 | 43 | raw_dir = args.cache_dir 44 | fixed_dir = args.final_dir 45 | os.makedirs(raw_dir, exist_ok=True) 46 | os.makedirs(fixed_dir, exist_ok=True) 47 | 48 | raw_dir = os.path.join(raw_dir, server) 49 | fixed_dir = os.path.join(fixed_dir, server) 50 | os.makedirs(raw_dir, exist_ok=True) 51 | os.makedirs(fixed_dir, exist_ok=True) 52 | 53 | for extra in extras: 54 | raw_file_name = extra.file_name 55 | if not raw_file_name.startswith('padv') or not raw_file_name.endswith('.wav'): 56 | print('skipping', raw_file_name) 57 | continue 58 | 59 | raw_file_path = os.path.join(raw_dir, raw_file_name) 60 | if os.path.exists(raw_file_path): 61 | print('file exists', raw_file_path) 62 | continue 63 | 64 | print('downloading', extra.url, 'to', raw_file_path) 65 | download_file(extra.url, raw_file_path) 66 | 67 | for file_name in os.listdir(raw_dir): 68 | in_file = os.path.join(raw_dir, file_name) 69 | 70 | file_id = int(file_name.lstrip('padv').rstrip('.wav')) 71 | padded_file_id = str(file_id).zfill(3) 72 | out_file = os.path.join(fixed_dir, '{}.wav'.format(padded_file_id)) 73 | if os.path.exists(out_file): 74 | continue 75 | 76 | cmd = 'sox -t ima -r 44100 -e ima-adpcm -v .5 {} -e signed-integer -b 16 {}'.format(in_file, out_file) 77 | print('running', cmd) 78 | os.system(cmd) 79 | 80 | print('done') 81 | -------------------------------------------------------------------------------- /mobile-api-server-cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: 'gcr.io/cloud-builders/gcloud' 3 | id: 'setup' 4 | entrypoint: 'bash' 5 | args: 6 | - '-c' 7 | - | 8 | mkdir -p output 9 | cp -r etl/dadguide_proto output/ 10 | cp -r etl/pad output/ 11 | cp -r web/data output/ 12 | cp web/mobile_api_server.py output/ 13 | cp web/requirements.txt output/ 14 | cp web/Dockerfile output/ 15 | 16 | - name: 'gcr.io/cloud-builders/gcloud' 17 | id: 'download-db-config' 18 | entrypoint: 'bash' 19 | args: [ '-c', 'gcloud secrets versions access latest --secret=prod_db_config > output/db_config.json' ] 20 | 21 | - name: 'gcr.io/cloud-builders/docker' 22 | dir: 'output' 23 | args: [ 24 | 'build', 25 | '--build-arg', 'SCRIPT_NAME=mobile_api_server.py', 26 | '--build-arg', 'PORT=8001', 27 | '-t', 'gcr.io/$PROJECT_ID/mobile-api-server', 28 | '.' 29 | ] 30 | 31 | images: [ 'gcr.io/$PROJECT_ID/mobile-api-server' ] 32 | -------------------------------------------------------------------------------- /pit-cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: python:3.6 3 | id: INSTALL 4 | entrypoint: python3 5 | args: [ '-m', 'pip', 'install', '-t', '.', '-r', 'requirements.txt' ] 6 | 7 | - name: alpine 8 | id: MAKE-DIRS 9 | args: [ 'mkdir', '-p', 'pad_data/raw', 'pad_data/integration/golden' ] 10 | 11 | - name: gcr.io/cloud-builders/gsutil 12 | id: COPY-RAW-DATA 13 | args: [ '-m', 'rsync', '-r', '-c', 'gs://dadguide-integration/parser/raw', 'pad_data/raw' ] 14 | 15 | - name: gcr.io/cloud-builders/gsutil 16 | id: COPY-GOLDEN-DATA 17 | args: [ '-m', 'rsync', '-r', '-c', 'gs://dadguide-integration/parser/golden', 'pad_data/integration/golden' ] 18 | 19 | - name: python:3.6-slim-buster 20 | entrypoint: python3 21 | id: RUN-UNIT-TESTS 22 | args: [ 'tests/parser_integration_test.py', '--input_dir', 'pad_data/raw', '--output_dir', 'pad_data/integration' ] 23 | env: [ "PYTHONPATH=etl:." ] 24 | -------------------------------------------------------------------------------- /proto/README.md: -------------------------------------------------------------------------------- 1 | Acquire protoc: 2 | 3 | ``` 4 | wget https://github.com/protocolbuffers/protobuf/releases/download/v3.10.1/protoc-3.10.1-linux-x86_64.zip 5 | unzip protoc-3.10.1-linux-x86_64.zip 6 | ``` 7 | 8 | Install the protoc plugin for dart: 9 | 10 | ``` 11 | flutter pub global activate protoc_plugin 12 | ``` 13 | 14 | You must have 3 items on your path now: 15 | 16 | * protoc 17 | * dart (this will be installed in your flutter/bin/cache/dart-sdk) 18 | * your flutter pub bin (flutter/.pub-cache/bin) 19 | 20 | To compile: 21 | 22 | ``` 23 | protoc -I=. --python_out=../etl/dadguide_proto enemy_skills.proto 24 | # Change this path to match your installation 25 | protoc -I=. --dart_out=../../../AndroidStudioProjects/dadguide2/lib/proto/enemy_skills/ enemy_skills.proto 26 | ``` 27 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | fake-useragent==0.1.13 3 | git+https://github.com/nachoapps/padtools.git 4 | pytz 5 | bs4 6 | jinja2 7 | protobuf 8 | pymysql 9 | pillow 10 | pypng 11 | tqdm 12 | -------------------------------------------------------------------------------- /schema/README.md: -------------------------------------------------------------------------------- 1 | ## Regenerating the schema dump 2 | 3 | This is generally very out of date. It's not actually used for anything. 4 | 5 | ```bash 6 | mysqldump -u root dadguide -p --no-data > mysql.sql 7 | ``` 8 | 9 | ## Dumping a built database to sqlite 10 | 11 | ```bash 12 | rm dadguide.sqlite 13 | ./mysql2sqlite.sh -u root -p dadguide | sqlite3 dadguide.sqlite 14 | ``` 15 | 16 | ## Clearing the local database 17 | 18 | ```sql 19 | delete from schedule where true; 20 | delete from news where true; 21 | 22 | delete from awakenings where true; 23 | delete from awoken_skills where true; 24 | 25 | delete from drops where true; 26 | delete from encounters where true; 27 | delete from sub_dungeons where true; 28 | delete from dungeons where true; 29 | 30 | delete from evolutions where true; 31 | delete from monsters where true; 32 | delete from leader_skills where true; 33 | delete from active_skills where true; 34 | 35 | delete from d_attributes; 36 | delete from d_types; 37 | ``` 38 | 39 | ## Deleting records 40 | 41 | Tables with computed IDs will generally be autocreated and shouldn't be deleted, instead they should have a column added 42 | to hide them from display. 43 | 44 | For tables that are autoincrement ID keyed, we may need to delete records (e.g. for `encounters`). 45 | 46 | This is done by automatically inserting a row into the `deleted_rows` table via a trigger on the table, created like: 47 | 48 | ```sql 49 | delimiter # 50 | CREATE TRIGGER encounters_deleted 51 | AFTER DELETE ON encounters 52 | FOR EACH ROW 53 | BEGIN 54 | INSERT INTO deleted_rows (table_name, table_row_id, tstamp) VALUES ('encounters', OLD.encounter_id, UNIX_TIMESTAMP()); 55 | END# 56 | ``` 57 | -------------------------------------------------------------------------------- /schema/mysql2sqlite.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Converts a mysqldump file into a Sqlite 3 compatible file. It also extracts the MySQL `KEY xxxxx` from the 4 | # CREATE block and create them in separate commands _after_ all the INSERTs. 5 | 6 | # Awk is choosen because it's fast and portable. You can use gawk, original awk or even the lightning fast mawk. 7 | # The mysqldump file is traversed only once. 8 | 9 | # Usage: $ ./mysql2sqlite mysqldump-opts db-name | sqlite3 database.sqlite 10 | # Example: $ ./mysql2sqlite --no-data -u root -pMySecretPassWord myDbase | sqlite3 database.sqlite 11 | 12 | # Thanks to and @artemyk and @gkuenning for their nice tweaks. 13 | 14 | set -e 15 | 16 | mysqldump --hex-blob --default-character-set=utf8 --compatible=ansi --skip-extended-insert --compact "$@" | 17 | # Fix for binary hex blobs 18 | sed -r "s/0x([[:xdigit:]]+)/X'\1'/" | 19 | awk ' 20 | 21 | BEGIN { 22 | FS=",$" 23 | print "PRAGMA synchronous = OFF;" 24 | print "PRAGMA journal_mode = MEMORY;" 25 | print "BEGIN TRANSACTION;" 26 | } 27 | 28 | # CREATE TRIGGER statements have funny commenting. Remember we are in trigger. 29 | /^\/\*.*CREATE.*TRIGGER/ { 30 | gsub( /^.*TRIGGER/, "CREATE TRIGGER" ) 31 | print 32 | inTrigger = 1 33 | next 34 | } 35 | 36 | # The end of CREATE TRIGGER has a stray comment terminator 37 | /END \*\/;;/ { gsub( /\*\//, "" ); print; inTrigger = 0; next } 38 | 39 | # The rest of triggers just get passed through 40 | inTrigger != 0 { print; next } 41 | 42 | # Skip other comments 43 | /^\/\*/ { next } 44 | 45 | # Print all `INSERT` lines. The single quotes are protected by another single quote. 46 | /INSERT/ { 47 | gsub( /\\\047/, "\047\047" ) 48 | gsub(/\\n/, "\n") 49 | gsub(/\\r/, "\r") 50 | gsub(/\\"/, "\"") 51 | gsub(/\\\\/, "\\") 52 | gsub(/\\\032/, "\032") 53 | print 54 | next 55 | } 56 | 57 | # Print the `CREATE` line as is and capture the table name. 58 | /^CREATE/ { 59 | print 60 | if ( match( $0, /\"[^\"]+/ ) ) tableName = substr( $0, RSTART+1, RLENGTH-1 ) 61 | } 62 | 63 | # Replace `FULLTEXT KEY` or any other `XXXXX KEY` except PRIMARY by `KEY` 64 | /^ [^"]+KEY/ && !/^ PRIMARY KEY/ { gsub( /.+KEY/, " KEY" ) } 65 | 66 | # Get rid of field lengths in KEY lines 67 | / KEY/ { gsub(/\([0-9]+\)/, "") } 68 | 69 | # Print all fields definition lines except the `KEY` lines. 70 | /^ / && !/^( KEY|\);)/ { 71 | gsub( /AUTO_INCREMENT|auto_increment/, "" ) 72 | gsub( /(CHARACTER SET|character set) [^ ]+ /, "" ) 73 | gsub( /DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP|default current_timestamp on update current_timestamp/, "" ) 74 | gsub( /(COLLATE|collate) [^ ]+ /, "" ) 75 | gsub(/(ENUM|enum)[^)]+\)/, "text ") 76 | gsub(/(SET|set)\([^)]+\)/, "text ") 77 | gsub(/UNSIGNED|unsigned/, "") 78 | if (prev) print prev "," 79 | prev = $1 80 | } 81 | 82 | # `KEY` lines are extracted from the `CREATE` block and stored in array for later print 83 | # in a separate `CREATE KEY` command. The index name is prefixed by the table name to 84 | # avoid a sqlite error for duplicate index name. 85 | /^( KEY|\);)/ { 86 | if (prev) print prev 87 | prev="" 88 | if ($0 == ");"){ 89 | print 90 | } else { 91 | if ( match( $0, /\"[^"]+/ ) ) indexName = substr( $0, RSTART+1, RLENGTH-1 ) 92 | if ( match( $0, /\([^()]+/ ) ) indexKey = substr( $0, RSTART+1, RLENGTH-1 ) 93 | key[tableName]=key[tableName] "CREATE INDEX \"" tableName "_" indexName "\" ON \"" tableName "\" (" indexKey ");\n" 94 | } 95 | } 96 | 97 | # Print all `KEY` creation lines. 98 | END { 99 | for (table in key) printf key[table] 100 | print "END TRANSACTION;" 101 | } 102 | ' 103 | exit 0 104 | -------------------------------------------------------------------------------- /setup.sh: -------------------------------------------------------------------------------- 1 | sudo apt update 2 | sudo apt install mysql-server 3 | sudo apt install xvfb 4 | sudo apt install zip 5 | sudo apt install sqlite3 6 | sudo apt install virtualenv 7 | sudo apt install pkgconfig 8 | sudo apt install yarn 9 | sudo apt install npm 10 | sudo apt install sox 11 | sudo apt install awscli 12 | sudo apt install ffmpeg 13 | 14 | wget http://launchpadlibrarian.net/434392915/gifsicle_1.92-2_amd64.deb 15 | sudo apt install ./gifsicle_1.92-2_amd64.deb 16 | 17 | git clone https://github.com/TsubakiBotPad/pad-data-pipeline.git 18 | git clone https://github.com/kiootic/pad-resources.git 19 | -------------------------------------------------------------------------------- /utils/download_db_from_prod.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | cd "$(dirname "$0")" 3 | mysql_db="https://d1kpnpud0qoyxf.cloudfront.net/db/dadguide.mysql" 4 | sqlite_db="https://d1kpnpud0qoyxf.cloudfront.net/db/dadguide.sqlite" 5 | folder="../pad_data/db/" 6 | wget -P $folder -N $mysql_db 7 | wget -P $folder -N $sqlite_db 8 | -------------------------------------------------------------------------------- /utils/download_db_with_waves_from_prod.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | cd "$(dirname "$0")" 3 | mysql_db="https://d1kpnpud0qoyxf.cloudfront.net/db/dadguide_wave_data.mysql.zip" 4 | folder="../pad_data/db/" 5 | wget -P $folder -N $mysql_db 6 | unzip "$folder/dadguide_wave_data.mysql.zip" -d $folder 7 | -------------------------------------------------------------------------------- /utils/refresh_data.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | cd "$(dirname "$0")" 3 | data_dir=../pad_data 4 | mkdir ${data_dir} 5 | mkdir ${data_dir}/raw 6 | mkdir ${data_dir}/processed 7 | gsutil -m rsync -r -c gs://mirubot-data/paddata/raw ${data_dir}/raw 8 | -------------------------------------------------------------------------------- /utils/restore_db_from_prod.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | cd "$(dirname "$0")" 3 | user=$(grep user ../etl/db_config.json | sed -r 's/.*: "(.*)",/\1/') 4 | pword=$(grep password ../etl/db_config.json | sed -r 's/.*: "(.*)",/\1/') 5 | input_file="../pad_data/db/dadguide.mysql" 6 | 7 | echo "using credentials: ${user} ${pword}" 8 | mysql -u ${user} -p${pword} dadguide <${input_file} 9 | -------------------------------------------------------------------------------- /utils/restore_db_with_waves_from_prod.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | cd "$(dirname "$0")" 3 | user=$(grep user ../cronjobs/db_config.json | sed -r 's/.*: "(.*)",/\1/') 4 | pword=$(grep password ../cronjobs/db_config.json | sed -r 's/.*: "(.*)",/\1/') 5 | input_file="../pad_data/db/dadguide_wave_data.mysql" 6 | 7 | echo "using credentials: ${user} ${pword}" 8 | mysql -u ${user} -p${pword} dadguide <${input_file} 9 | -------------------------------------------------------------------------------- /web/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6-slim-buster 2 | 3 | ARG SCRIPT_NAME 4 | ARG EXTRA_ARG 5 | ARG PORT 6 | 7 | # slim-buster is 150MB lighter than python:3.6 but does not include git, which we need. 8 | RUN apt-get update && \ 9 | apt-get install -y git 10 | 11 | # The Cloud Build script should have populated the directory properly. 12 | ADD . /server/ 13 | 14 | # Install Python dependencies 15 | RUN python3 -m pip install -r /server/requirements.txt 16 | 17 | # Expose the HTTP port we're listening on 18 | EXPOSE ${PORT} 19 | 20 | # The python deps are installed to this directory. 21 | WORKDIR /server 22 | 23 | # Create a script to pass build-time command line args to python 24 | RUN echo "python3 /server/${SCRIPT_NAME} --port=${PORT} --db_config=/server/db_config.json ${EXTRA_ARG} \$@" > /run_server.sh 25 | 26 | # Default entrypoint that runs the script, which allows for extra args to be passed at run time 27 | ENTRYPOINT ["/bin/bash", "/run_server.sh"] 28 | -------------------------------------------------------------------------------- /web/README.md: -------------------------------------------------------------------------------- 1 | # Web stuff 2 | 3 | ## Legacy serving 4 | 5 | The `server.php` and `serve_dadguide_data.py` stuff are legacy, used to serve the initial iteration of the DadGuide API. 6 | They're symlinked from the Apache web directory. 7 | 8 | Hopefully this will eventually be deprecated and removed. 9 | 10 | ## Containers 11 | 12 | We're trying to move towards running servers/pipelines in containers. This directory two holds Sanic Python webservers 13 | that are containerized. 14 | 15 | The *-api-cloudbuild.yaml files in the project root can be used to build the server containers. 16 | 17 | The containers embed the database configuration; it's imported at build time using Secret Manager, the secret should be 18 | named `prod_db_config` and should contain the standard `db_config.json`. 19 | 20 | ## Mobile server 21 | 22 | Currently this is only used for development purposes, but I'd like to replace the shitty PHP stuff with it. 23 | 24 | Main thing holding me back is: 25 | 26 | 1) The existing stuff seems to work fine 27 | 2) The server version keeps a connection open, as opposed to opening a fresh one (like the script does). I'm not sure 28 | how stable this is. 29 | 3) I have no monitoring set up for the new stuff. None for the old stuff either actually but empirically it seems to 30 | work. 31 | 32 | To run the server on the DG host: 33 | 34 | ```bash 35 | docker run -d --name mobile-api-server --network=host --restart=on-failure gcr.io/rpad-discord/mobile-api-server:latest 36 | ``` 37 | 38 | ## Admin server 39 | 40 | This serves the Monster Admin ES webapp, including the front end stuff and the API queries. Apache is reverse proxying 41 | to this server. 42 | 43 | The admin api server also needs a pointer to the on-disk ES dir, the local clone of the pad-game-data-slim repo. 44 | 45 | Besides the python backend, it also includes a Flutter Web frontend. To run the cloudbuild script, you also need to have 46 | compiled the Flutter Builder: 47 | https://github.com/GoogleCloudPlatform/cloud-builders-community/tree/master/flutter 48 | 49 | This should be updated periodically. 50 | 51 | To run the server on the DG host: 52 | 53 | ```bash 54 | docker run -d --name admin-api-server --network=host --restart=on-failure -v /home/bot/dadguide/pad-game-data:/server/es gcr.io/rpad-discord/admin-api-server:latest 55 | ``` 56 | 57 | I had trouble getting the build args and runtime args to play nicely with the entrypoint, so the ES dir is hardcoded. 58 | The bind mount to `/server/es` is a hard requirement. 59 | 60 | ## Building the containers 61 | 62 | Run these commands from the root of the project. 63 | 64 | ### On Cloud Build 65 | 66 | ```bash 67 | gcloud builds submit . --config=admin-api-server-cloudbuild.yaml 68 | ``` 69 | 70 | ## Local compilation 71 | 72 | After following the instructions here: 73 | https://cloud.google.com/cloud-build/docs/build-debug-locally 74 | 75 | ```bash 76 | cloud-build-local --dryrun=false --config=admin-api-server-cloudbuild.yaml . 77 | ``` 78 | 79 | ## Continuous deployment 80 | 81 | Watchtower is set up to continuously deploy new builds: 82 | 83 | ```bash 84 | docker run -d --name watchtower --restart always -v /home/bot/.docker/config.json:/config.json -v /var/run/docker.sock:/var/run/docker.sock v2tec/watchtower -i 30 85 | ``` 86 | 87 | Watchtower needs to have authorization to GCR. To set up auth: 88 | 89 | Create a service account, and ensure it has Storage Viewer access. Get a json key file and save it into `~/.docker/`: 90 | https://cloud.google.com/container-registry/docs/advanced-authentication#json-key 91 | 92 | Run: 93 | 94 | ```bash 95 | docker login -u _json_key --password-stdin https://gcr.io < ~/.docker/gcloudauth.json 96 | ``` 97 | -------------------------------------------------------------------------------- /web/data/utils.py: -------------------------------------------------------------------------------- 1 | import binascii 2 | import re 3 | from datetime import datetime 4 | from decimal import Decimal 5 | 6 | import pymysql 7 | 8 | 9 | def fix_row(row): 10 | row_data = {} 11 | for col in row: 12 | data = row[col] 13 | if data is None: 14 | fixed_data = None 15 | elif type(data) is Decimal: 16 | fixed_data = float(data) 17 | elif type(data) is datetime: 18 | fixed_data = data.isoformat(' ') 19 | elif type(data) == bytes: 20 | fixed_data = '0x' + binascii.hexlify(bytearray(data)).decode('ascii') 21 | elif type(data) not in [int, float, str]: 22 | fixed_data = str(data) 23 | else: 24 | fixed_data = data 25 | 26 | fixed_col = col 27 | row_data[fixed_col] = fixed_data 28 | return row_data 29 | 30 | 31 | def dump_table(cursor): 32 | result_json = {'items': []} 33 | for row in cursor: 34 | result_json['items'].append(fix_row(row)) 35 | 36 | return result_json 37 | 38 | 39 | def connect(db_config): 40 | return pymysql.connect(host=db_config['host'], 41 | user=db_config['user'], 42 | password=db_config['password'], 43 | db=db_config['db'], 44 | charset=db_config['charset'], 45 | cursorclass=pymysql.cursors.DictCursor) 46 | 47 | 48 | def load_from_db(db_config, table, tstamp): 49 | connection = connect(db_config) 50 | result = load_from_db_connection(connection, table, tstamp) 51 | connection.close() 52 | return result 53 | 54 | 55 | def load_from_db_connection(connection, table, tstamp): 56 | table = table.lower() 57 | # Whitelist alphanum + _ for table names 58 | if re.findall(r'[^\w]', table): 59 | raise ValueError('Bad table name: ' + table) 60 | 61 | sql = 'SELECT * FROM `{}`'.format(table) 62 | 63 | if table == 'timestamps': 64 | pass 65 | else: 66 | sql += ' WHERE tstamp > {}'.format(int(tstamp)) 67 | 68 | if table == 'schedule': 69 | sql += ' AND end_timestamp > UNIX_TIMESTAMP()' 70 | 71 | # Added this to make client updating easier; if the update fails, the lowest-value records will have been inserted, 72 | # and the higher value ones will get inserted on the next run. 73 | sql += ' ORDER BY tstamp ASC' 74 | 75 | with connection.cursor() as cursor: 76 | cursor.execute(sql) 77 | data = dump_table(cursor) 78 | 79 | return data 80 | -------------------------------------------------------------------------------- /web/mobile_api_server.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json as base_json 3 | from typing import Optional 4 | 5 | from sanic import Sanic 6 | from sanic import request 7 | from sanic.exceptions import ServerError 8 | from sanic.response import json 9 | from sanic_compress import Compress 10 | from sanic_cors import CORS 11 | 12 | from data import utils 13 | from data.utils import load_from_db_connection 14 | from pad.db.db_util import DbWrapper 15 | 16 | 17 | def parse_args(): 18 | parser = argparse.ArgumentParser(description="DadGuide mobile backend server", add_help=False) 19 | input_group = parser.add_argument_group("Input") 20 | input_group.add_argument("--db_config", help="JSON database info") 21 | input_group.add_argument("--port", default='8001', help="TCP port to listen on") 22 | return parser.parse_args() 23 | 24 | 25 | app = Sanic() 26 | Compress(app) 27 | CORS(app) 28 | 29 | db_config = None 30 | connection = None 31 | db_wrapper = None # type: Optional[DbWrapper] 32 | 33 | VALID_TABLES = [ 34 | 'active_skills', 35 | 'awakenings', 36 | 'awoken_skills', 37 | 'd_attributes', 38 | 'd_event_types', 39 | 'drops', 40 | 'dungeons', 41 | 'dungeon_type', 42 | 'encounters', 43 | 'enemy_data', 44 | 'evolutions', 45 | 'exchanges', 46 | 'leader_skills', 47 | 'monsters', 48 | 'news', 49 | 'rank_rewards', 50 | 'schedule', 51 | 'series', 52 | 'skill_condition', 53 | 'sub_dungeons', 54 | 'timestamps' 55 | ] 56 | 57 | 58 | @app.route('/dadguide/api/serve') 59 | async def serve_table(request): 60 | table = request.args.get('table') 61 | if table is None: 62 | raise ServerError('table required') 63 | if table not in VALID_TABLES: 64 | raise ServerError('unexpected table') 65 | tstamp = request.args.get('tstamp') 66 | if tstamp is None and table != 'timestamps': 67 | raise ServerError('tstamp required') 68 | if tstamp is not None and not tstamp.isnumeric(): 69 | raise ServerError('tstamp must be a number') 70 | 71 | data = load_from_db_connection(connection, table, tstamp) 72 | return json(data) 73 | 74 | 75 | @app.route('/dadguide/api/v1/purchases', methods={'POST'}) 76 | async def add_purchase(request: request.Request): 77 | print('got add purchase request') 78 | data = request.json 79 | print(data) 80 | device_id = data.get('device_id', 'missing') 81 | if device_id == 'missing': 82 | raise ServerError('device_id required') 83 | 84 | purchase_info = base_json.dumps(data, sort_keys=True, indent=2) 85 | sql = 'INSERT INTO `dadguide_admin`.`purchases` (`device_id`, `purchase_info`) VALUES (%s, %s)' 86 | db_wrapper.insert_item(sql, [device_id, purchase_info]) 87 | 88 | return json({'status': 'ok'}) 89 | 90 | 91 | def main(args): 92 | with open(args.db_config) as f: 93 | global db_config 94 | db_config = base_json.load(f) 95 | 96 | global connection 97 | connection = utils.connect(db_config) 98 | 99 | global db_wrapper 100 | db_wrapper = DbWrapper(False) 101 | db_wrapper.connect(db_config) 102 | 103 | app.run(host='0.0.0.0', port=int(args.port)) 104 | 105 | 106 | if __name__ == '__main__': 107 | main(parse_args()) 108 | -------------------------------------------------------------------------------- /web/requirements.txt: -------------------------------------------------------------------------------- 1 | requests 2 | git+https://github.com/nachoapps/padtools.git 3 | pytz 4 | bs4 5 | protobuf 6 | pymysql 7 | sanic 8 | sanic-cors 9 | sanic_compress 10 | fake_useragent 11 | -------------------------------------------------------------------------------- /web/serve.php: -------------------------------------------------------------------------------- 1 | 21 | -------------------------------------------------------------------------------- /web/serve_dadguide_data.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | 4 | from data.utils import load_from_db 5 | 6 | 7 | def parse_args(): 8 | parser = argparse.ArgumentParser(description="Echos DadGuide database data", add_help=False) 9 | 10 | input_group = parser.add_argument_group("Input") 11 | input_group.add_argument("--db_config", help="JSON database info") 12 | input_group.add_argument("--table", help="Table name") 13 | input_group.add_argument("--tstamp", help="DadGuide tstamp field limit") 14 | input_group.add_argument("--plain", action='store_true', help="Print a more readable output") 15 | 16 | return parser.parse_args() 17 | 18 | 19 | def main(args): 20 | with open(args.db_config) as f: 21 | db_config = json.load(f) 22 | data = load_from_db(db_config, args.table, args.tstamp) 23 | 24 | if args.plain: 25 | print(json.dumps(data, indent=4, sort_keys=True)) 26 | else: 27 | print(json.dumps(data)) 28 | 29 | 30 | if __name__ == "__main__": 31 | args = parse_args() 32 | main(args) 33 | --------------------------------------------------------------------------------