├── requirements.txt ├── LICENSE ├── .github └── workflows │ └── actions.yml ├── README.md ├── reconstruct_historical_data.py ├── .gitignore └── main.py /requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.28.1 2 | GitPython==3.1.31 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Ryan Collingwood 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/workflows/actions.yml: -------------------------------------------------------------------------------- 1 | name: run main.py 2 | 3 | on: 4 | schedule: 5 | - cron: '0 */12 * * *' # every 12 hours 6 | workflow_dispatch: 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | 13 | - name: checkout repo content 14 | uses: actions/checkout@main # checkout the repository content to github runner 15 | 16 | - name: setup python 17 | uses: actions/setup-python@main 18 | with: 19 | python-version: '3.10' # install the python version needed 20 | 21 | - name: install python packages 22 | run: | 23 | python -m pip install --upgrade pip 24 | pip install -r requirements.txt 25 | 26 | - name: execute py script # run main.py 27 | env: 28 | ACCOUNT_ID: ${{ secrets.ACCOUNT_ID }} 29 | run: python main.py 30 | 31 | - name: commit files 32 | run: | 33 | git config --local user.email "action@github.com" 34 | git config --local user.name "GitHub Action" 35 | git add -A 36 | git diff-index --quiet HEAD || (git commit -a -m "updated data" --allow-empty) 37 | 38 | - name: push changes 39 | uses: ad-m/github-push-action@master 40 | with: 41 | github_token: ${{ secrets.GITHUB_TOKEN }} 42 | branch: main 43 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # diablo_4_armory_fetcher 2 | 3 | ## ⚠️ This repo is archived and will not be updated ⚠️ ## 4 | 5 | It seems the unofficial Diablo 4 Armory https://d4armory.io/ is no longer active. 6 | 7 | ## Overview 8 | 9 | Use the unofficial Diablo 4 Armory https://d4armory.io/ to snapshot your characters data, combined with Github actions to fetch and store the data on a schedule 10 | 11 | Using this you can track your characters until/if Blizzard create an official API. 12 | 13 | ## Details 14 | 15 | - Based off of https://github.com/patrickloeber/python-github-action-template for using Github actions to run on a schedule 16 | - By default the script is run every twelve hours as specified in `.github/workflows/actions.yml`. I'd caution against running this too often as it would put a load on the kind folks at d4armory.io 17 | 18 | ## Setup 19 | 20 | ### Find Your Account ID 21 | 22 | To find your account id: 23 | - Open Diablo IV install directory 24 | - Open `FenrisDebug.txt` 25 | - Search for 'account_id' 26 | 27 | ### Register your characters on d4armory.io 28 | - Head to https://d4armory.io/ 29 | - Search for your account using the account_id you fetched from `FenrisDebug.txt` 30 | - Confirm you can see your characters, there may be some latency 31 | 32 | ### Setup your Clone of this Repo 33 | - Fork or Create a copy of this repo using the [Use this template](https://github.com/ryancollingwood/diablo_4_armory_fetcher/generate) button 34 | 35 | #### Setup Repo Secrets 36 | - In your repo on github head to `settings` -> `secrets` -> `actions` 37 | - Click `New repository secret` 38 | - Name your secret: `ACCOUNT_ID` 39 | - Enter the value of the account_id you fetched from `FenrisDebug.txt` 40 | 41 | #### Setup Repo Write Permissions 42 | - In your repo on github head to `settings` -> `actions` 43 | - Under `Workflow permissions` select `Read and write permissions` 44 | - Click `save` 45 | 46 | ## Constructing Data History 47 | - Clone the repo to your local computer 48 | - (Optional, but recommended) Create an python environment and activate it 49 | - Install the requirements - e.g. `pip install requirements.txt` 50 | - Run the following terminal command in the repo folder `python reconstruct_historical_data.py` 51 | - Your characters changes will be written by default as JSON line format to `data_history` directory 52 | 53 | ## TODO: 54 | - Check the status endpoint before proceeding with fetching chars 55 | -------------------------------------------------------------------------------- /reconstruct_historical_data.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | from pathlib import Path 4 | from git import Repo 5 | 6 | 7 | def write_jsonl_changes(output_path, file_stem, revlist): 8 | # produce jsonlines output see: https://jsonlines.org/ 9 | output_file = output_path / f"{file_stem}.jsonl" 10 | output_file.parent.mkdir(parents=True, exist_ok=True) 11 | 12 | with open(output_file, "w", encoding="utf8") as outfile: 13 | for commit, filecontents in revlist: 14 | change_timestamp = int(commit.committed_datetime.timestamp()) 15 | char_data = json.loads(filecontents.decode("utf8")) 16 | data = {"_timestamp": change_timestamp, "_hexsha": commit.hexsha, "data": char_data} 17 | outfile.write(f"{json.dumps(data, ensure_ascii=False, indent=None)}\n") 18 | 19 | def write_individual_changes(output_path, file_stem, file_suffix, revlist): 20 | for commit, filecontents in revlist: 21 | change_timestamp = int(commit.committed_datetime.timestamp()) 22 | output_file = output_path / f"{file_stem}/{change_timestamp}{file_suffix}" 23 | output_file.parent.mkdir(parents=True, exist_ok=True) 24 | 25 | output_file.write_text(filecontents.decode("utf8")) 26 | 27 | if __name__ == "__main__": 28 | repo = Repo() 29 | 30 | parser = argparse.ArgumentParser("Diablo 4 Armory Fetcher - Data Builder") 31 | parser.add_argument("input_dir", help="Directory to source the files - default `data`", type=str, default="data", nargs='?') 32 | parser.add_argument("output_dir", help="Directory to place rebuilt files", type=str, default="data_history", nargs='?') 33 | parser.add_argument("jsonl", help="Write to json line format, if False indivdual files for indivdual changes will be written", type=bool, default=True, nargs='?') 34 | parser.add_argument("glob", help="Glob pattern to get files to retrieve history - relative to input_dir - default `'*/*.json'`", type=str, default="*/*.json", nargs='?') 35 | 36 | 37 | args, unknown = parser.parse_known_args() 38 | 39 | input_dir = args.input_dir 40 | output_dir = args.output_dir 41 | jsonl = args.jsonl 42 | glob_pattern = args.glob 43 | 44 | for json_file in Path(input_dir).glob(glob_pattern): 45 | file_name = json_file.name 46 | if file_name == "_.json": 47 | continue 48 | 49 | file_stem = json_file.stem 50 | file_suffix = json_file.suffix 51 | 52 | file_ext = json_file.suffix 53 | output_path = Path(output_dir) / json_file.parents[0].name 54 | 55 | revlist = ( 56 | (commit, (commit.tree / str(json_file.as_posix())).data_stream.read()) 57 | for commit in repo.iter_commits(paths=str(json_file)) 58 | ) 59 | 60 | if jsonl: 61 | write_jsonl_changes(output_path, file_stem, revlist) 62 | else: 63 | write_individual_changes(output_path, file_stem, file_suffix, revlist) 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | local_settings.py 60 | db.sqlite3 61 | db.sqlite3-journal 62 | 63 | # Flask stuff: 64 | instance/ 65 | .webassets-cache 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | 73 | # PyBuilder 74 | .pybuilder/ 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | # For a library or package, you might want to ignore these files since the code is 86 | # intended to run in multiple environments; otherwise, check them in: 87 | # .python-version 88 | 89 | # pipenv 90 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 91 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 92 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 93 | # install all needed dependencies. 94 | #Pipfile.lock 95 | 96 | # poetry 97 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 98 | # This is especially recommended for binary packages to ensure reproducibility, and is more 99 | # commonly ignored for libraries. 100 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 101 | #poetry.lock 102 | 103 | # pdm 104 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 105 | #pdm.lock 106 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 107 | # in version control. 108 | # https://pdm.fming.dev/#use-with-ide 109 | .pdm.toml 110 | 111 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 112 | __pypackages__/ 113 | 114 | # Celery stuff 115 | celerybeat-schedule 116 | celerybeat.pid 117 | 118 | # SageMath parsed files 119 | *.sage.py 120 | 121 | # Environments 122 | .env 123 | .venv 124 | env/ 125 | venv/ 126 | ENV/ 127 | env.bak/ 128 | venv.bak/ 129 | 130 | # Spyder project settings 131 | .spyderproject 132 | .spyproject 133 | 134 | # Rope project settings 135 | .ropeproject 136 | 137 | # mkdocs documentation 138 | /site 139 | 140 | # mypy 141 | .mypy_cache/ 142 | .dmypy.json 143 | dmypy.json 144 | 145 | # Pyre type checker 146 | .pyre/ 147 | 148 | # pytype static type analyzer 149 | .pytype/ 150 | 151 | # Cython debug symbols 152 | cython_debug/ 153 | 154 | # PyCharm 155 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 156 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 157 | # and can be added to the global gitignore or merged into this file. For a more nuclear 158 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 159 | #.idea/ 160 | 161 | # ignore historical data rebuilds 162 | data_history/ 163 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | import requests 4 | import json 5 | from time import sleep 6 | from pathlib import Path 7 | from typing import Dict, List, Union, Any 8 | import logging 9 | import logging.handlers 10 | 11 | def setup_logger(filename: str): 12 | logger = logging.getLogger(__name__) 13 | logger.setLevel(logging.DEBUG) 14 | logger_file_handler = logging.handlers.RotatingFileHandler( 15 | f"{filename}.log", 16 | maxBytes=1024 * 1024, 17 | backupCount=1, 18 | encoding="utf8", 19 | ) 20 | formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") 21 | logger_file_handler.setFormatter(formatter) 22 | logger.addHandler(logger_file_handler) 23 | 24 | # std out logger 25 | console = logging.StreamHandler() 26 | console.setLevel(logging.INFO) 27 | formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') 28 | console.setFormatter(formatter) 29 | logger.addHandler(console) 30 | 31 | return logger 32 | 33 | class Fetcher(object): 34 | def __init__(self, account_ids: List[str] = None) -> None: 35 | self.base_url = "https://d4armory.io/api/armory" 36 | self.config_valid = False 37 | self.logger = None 38 | 39 | self.account_ids: List[str] = account_ids 40 | if self.account_ids is None: 41 | self.account_ids = self.get_account_ids() 42 | 43 | if self.account_ids is None: 44 | print("No account ids to fetch, pass in value or set environment variable: ACCOUNT_ID") 45 | return 46 | 47 | self.profile_queue_attempts = int(self.get_environ_value("PROFILE_QUEUE_ATTEMPTS", "3")) 48 | self.profile_queue_sleep = float(self.get_environ_value("PROFILE_QUEUE_ATTEMPTS", "5")) 49 | 50 | self.logger: logging.Logger = setup_logger("fetch_data") 51 | self.data_path: Path = self.get_data_path() 52 | 53 | self.config_valid = True 54 | 55 | def get_environ_value(self, key: str, default_value = None) -> str: 56 | logger = self.logger 57 | if logger is not None: 58 | logger.debug(f"getting env var: {key}") 59 | try: 60 | return os.environ[key] 61 | except KeyError: 62 | if logger is not None: 63 | logger.debug(f"couldn't get env var {key} - using default: {default_value}") 64 | return default_value 65 | 66 | def dumps_json(self, obj) -> str: 67 | try: 68 | return json.dumps(obj, ensure_ascii=True) 69 | except Exception as e: 70 | self.logger.exception(e) 71 | raise e 72 | 73 | def get_json(self, url: str) -> Union[List,Dict]: 74 | self.logger.debug(f"fetching json: {url}") 75 | try: 76 | r = requests.get(url) 77 | 78 | if r.status_code == 200: 79 | data = r.json() 80 | 81 | if isinstance(data, dict): 82 | error_message = data.get("error") 83 | 84 | if error_message: 85 | self.logger.warning(f"Error message in 200 response: {error_message}") 86 | 87 | return r.json() 88 | except requests.exceptions.ConnectionError as e: 89 | self.logger.error(f"connection error when accessing: {url}") 90 | raise e 91 | except Exception as e: 92 | self.logger.exception(e) 93 | raise e 94 | 95 | def get_account_ids(self): 96 | logger = self.logger 97 | try: 98 | accounts_ids = self.get_environ_value("ACCOUNT_ID") 99 | if accounts_ids is None or not accounts_ids: 100 | return None 101 | return accounts_ids.split(",") 102 | except Exception as e: 103 | if logger is not None: 104 | logger.exception(e) 105 | raise e 106 | 107 | def get_data_path(self): 108 | try: 109 | data_path: Path = Path(self.get_environ_value("DATA_PATH", "data")) 110 | data_path.mkdir(exist_ok=True) 111 | return data_path 112 | except Exception as e: 113 | self.logger.exception(e) 114 | raise e 115 | 116 | def char_last_login(self, char_data_path: Union[Path, Dict]): 117 | try: 118 | if isinstance(char_data_path, dict): 119 | exisiting_data = char_data_path 120 | elif isinstance(char_data_path, Path): 121 | exisiting_data: Dict = json.loads(char_data_path.read_text()) 122 | last_login_key = "u" 123 | last_login = exisiting_data.get(last_login_key) 124 | self.logger.info(f"last login: {last_login}") 125 | return last_login 126 | except Exception as e: 127 | self.logger.exception(e) 128 | return None 129 | 130 | def process_char(self, char_data: Dict, account_path: Path, attempt_num: int = 0): 131 | try: 132 | char_id = char_data.get("i", None) 133 | char_name = char_data.get("n", None) 134 | 135 | if not char_id or not char_name: 136 | self.logger.error(f"Didn't get character id or name from response - char_id: {char_id} - char_name: {char_name}") 137 | return 138 | 139 | self.logger.info(f"fetching character id: {char_name} - {char_id}") 140 | account_id: str = account_path.name 141 | 142 | output_path: Path = (account_path / f"{char_name}_{char_id}.json") 143 | char_details_data: Dict[str, Any] = self.get_json(f'{self.base_url}/{account_id}/{char_id}.json') 144 | 145 | if attempt_num < self.profile_queue_attempts: 146 | queue: int = char_details_data.get("queue", -1) 147 | if queue > 0: 148 | self.logger.info(f"queue position {queue} - sleeping for {self.profile_queue_attempts} seconds") 149 | sleep(self.profile_queue_attempts) 150 | self.process_char(char_data, account_path, attempt_num + 1) 151 | return 152 | 153 | write_data = True 154 | 155 | write_data = self.has_logged_since_last_check(output_path, char_details_data) 156 | self.logger.info(f"has logged in since last check: {write_data}") 157 | 158 | if write_data: 159 | output_path.write_text(self.dumps_json(char_details_data)) 160 | 161 | except Exception as e: 162 | self.logger.exception(e) 163 | raise e 164 | 165 | def char_data_is_different(self, new_data: Dict, old_data: Dict, ignore_keys: List[str]): 166 | if not new_data or not old_data: 167 | return True 168 | 169 | if new_data.keys() != old_data.keys(): 170 | return True 171 | 172 | for k, new_value in new_data.items(): 173 | if k in ignore_keys: 174 | continue 175 | if old_data.get(k) != new_value: 176 | return True 177 | 178 | return False 179 | 180 | def has_logged_since_last_check(self, output_path: Path, char_details_data: Dict): 181 | result = True 182 | 183 | if output_path.exists(): 184 | old_char_details_data: Dict = json.loads(output_path.read_text()) 185 | last_login = self.char_last_login(old_char_details_data) 186 | last_login_key = "u" 187 | current_login = char_details_data.get(last_login_key) 188 | 189 | if (last_login and current_login): 190 | if (last_login != current_login): 191 | result = True 192 | else: 193 | self.logger.warning("character hasn't logged in since last fetch") 194 | result = False 195 | else: 196 | result = True 197 | 198 | if result: 199 | result = self.char_data_is_different(old_char_details_data, char_details_data, ["u", "au"]) 200 | if not result: 201 | self.logger.info("character may have logged in - but no changes") 202 | 203 | return result 204 | 205 | def process_all_chars(self, all_chars_data: List[Dict], account_path: Path): 206 | try: 207 | if isinstance(all_chars_data, dict): 208 | character_data = all_chars_data.get("characters") 209 | if character_data is None: 210 | return 211 | elif isinstance(all_chars_data, list): 212 | character_data = all_chars_data 213 | 214 | for char_data in character_data: 215 | try: 216 | self.process_char(char_data, account_path) 217 | except Exception as e: 218 | # try next char 219 | continue 220 | 221 | except Exception as e: 222 | self.logger.exception(e) 223 | raise e 224 | 225 | def process_account(self, account_id: str): 226 | self.logger.info(f"processing account: {account_id}") 227 | 228 | data = self.get_json(f'{self.base_url}/{account_id}.json') 229 | 230 | account_path = self.data_path / str(account_id) 231 | account_path.mkdir(exist_ok=True) 232 | 233 | (account_path / f"_.json").write_text(self.dumps_json(data)) 234 | 235 | self.process_all_chars(data, account_path) 236 | 237 | self.logger.info(f"completed processing account: {account_id}") 238 | 239 | def execute(self): 240 | self.logger.info("START EXECUTE") 241 | 242 | for account_id in self.account_ids: 243 | self.process_account(account_id) 244 | 245 | self.logger.info("COMPLETE EXECUTE") 246 | 247 | 248 | if __name__ == "__main__": 249 | parser = argparse.ArgumentParser("Diablo 4 Armory Fetcher") 250 | parser.add_argument("account_id", help="Account ID to fetch", type=str, default=None, nargs='?') 251 | args, unknown = parser.parse_known_args() 252 | 253 | account_ids = None 254 | if args.account_id: 255 | account_ids = [args.account_id] 256 | 257 | fetcher = Fetcher(account_ids = account_ids) 258 | if fetcher.config_valid: 259 | fetcher.execute() 260 | --------------------------------------------------------------------------------