├── .gitignore ├── LICENSE ├── README.md ├── ipynb_examples ├── PlayerPositionPlot-old.gif ├── PlayerPositionPlot.gif └── YAIBA Interactive Player Location Example.ipynb ├── poetry.toml ├── pyproject.toml ├── setup.py ├── yaiba ├── __init__.py ├── constants.py ├── log │ ├── __init__.py │ ├── entries.py │ ├── pseudonymizer.py │ ├── session_log.py │ ├── session_log_csv.py │ ├── session_log_json.py │ ├── test_utils.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_session_log_csv.py │ │ └── test_session_log_json.py │ ├── types.py │ └── vrc │ │ ├── __init__.py │ │ ├── entries │ │ ├── __init__.py │ │ ├── builtin.py │ │ ├── player_position.py │ │ ├── questionnaire.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── test_builtin.py │ │ │ ├── test_player_position.py │ │ │ ├── test_questionnaire.py │ │ │ └── test_yodokoro_tag_marker.py │ │ └── yodokoro_tag_marker.py │ │ ├── parser.py │ │ ├── tests │ │ ├── __init__.py │ │ └── test_parser.py │ │ └── utils.py └── visualization │ ├── __init__.py │ └── vrc │ └── __init__.py ├── yaiba_colab ├── __init__.py └── storage.py └── yaiba_scienceassembly ├── README.md └── __init__.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | poetry.lock -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 VRC Science Assembly 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # YAIBA: Yet Another Interactive Behavior Analysis for VRSNS 2 | 3 | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1fqfbRb6w7VzDEXGho6KWOLFVfQFU0WlS?usp=sharing) 4 | 5 | ## Installation 6 | 7 | ### Prerequisites 8 | 9 | - Python (>=3.9) 10 | - Poetry (for package management) 11 | 12 | ### Setting up YAIBA 13 | 14 | ```bash 15 | # Install dependencies with visualization support 16 | poetry install --all-groups 17 | 18 | # Start Jupyter Lab 19 | poetry run jupyter lab 20 | ``` 21 | 22 | Want to analyze user logs in VRSNS like player location, head angle, questionnaire answers? 23 | This library is for the purpose! 24 | 25 | ![](https://raw.githubusercontent.com/ScienceAssembly/YAIBA/main/ipynb_examples/PlayerPositionPlot.gif) 26 | 27 | ## Features 28 | 29 | * Collecting following informations in VRChat 30 | * Player join / leave event 31 | * PlayerPosition / head angle (install 32 | [unitypackage in this reposlitory](https://github.com/ScienceAssembly/yaiba-vrc)) 33 | * [Yodokoro Tag Marker](https://booth.pm/ja/items/3109716) history 34 | * Questionnaire answers (install [unitypackage in this reposlitory](https://github.com/ScienceAssembly/yaiba-vrc)) 35 | * Pseudonymize user name / user id 36 | * Output as CSV / JSON / Google Drive 37 | 38 | ## Example 39 | 40 | ```python 41 | import yaiba 42 | 43 | # Parse VRChat log to get SessionLog 44 | with open(r"C:\Users\USER_NAME\AppData\Local\VRChat\log_XXXXX.log", "r") as fp: 45 | session_log = yaiba.parse_vrchat_log(fp) 46 | 47 | # You can attach any metadata 48 | session_log.metadata = { 49 | "date": "2028-07-06", 50 | "title": "理系集会 Cyberia", 51 | "instance": "第一インスタンス", 52 | } 53 | 54 | # Save SessionLog to file 55 | with open("XXXX.json", "w") as fp: 56 | # Only stores pseudonymized username 57 | options = yaiba.JsonEncoder.Options.default() 58 | options.output_pseudo_user_name = True 59 | options.output_user_name = False 60 | 61 | # save to file 62 | yaiba.save_session_log(session_log, fp, options) 63 | 64 | # Load SessionLog from file 65 | with open("XXXX.json", "r") as fp: 66 | session_log = yaiba.load_session_log(fp) 67 | 68 | # Do analysis 69 | joined_user_names = set([ 70 | e.pseudo_user_name 71 | for e in session_log.log_entries 72 | if isinstance(e, yaiba.log.vrc.VRCPlayerJoinEntry) 73 | ]) 74 | print(f"The number of unique user joined: {len(joined_user_names)}") 75 | ``` 76 | 77 | ### Questionnaire analysis 78 | https://colab.research.google.com/drive/1GtBARBFPd2Yz4R5BVm63XfKnhBrER4ub 79 | 80 | ### Yodokoro tag marker customization 81 | 82 | ```python 83 | import yaiba 84 | 85 | # Parse VRChat log to get SessionLog 86 | with open(r"C:\Users\USER_NAME\AppData\Local\VRChat\log_XXXXX.log", "r") as fp: 87 | config = yaiba.VRCLogParser.Config() 88 | config.yodokoro_tag_marker_names = [ 89 | "tag name 1", 90 | "tag name 2", 91 | "tag name 3", 92 | ] 93 | session_log = yaiba.parse_vrchat_log(fp, config=config) 94 | ``` 95 | 96 | ### Integration with Google Colab 97 | 98 | This is useful for collaboration. 99 | 100 | Note: Those examples only work in Google Colab. 101 | 102 | ```python 103 | import yaiba 104 | import yaiba_colab 105 | 106 | with open(r"C:\Users\USER_NAME\AppData\Local\VRChat\log_XXXXX.log", "r") as fp: 107 | session_log = yaiba.parse_vrchat_log(fp) 108 | 109 | # drive_id: Find share link in Google Drive, and drive_id contains in the URL 110 | # as follows: https://drive.google.com/drive/folders/${drive_id}?${not_related_parameters} 111 | GOOGLE_DRIVE_ID = "xxxxxxxxx" 112 | 113 | folder = yaiba_colab.GoogleDriveFolder(GOOGLE_DRIVE_ID) 114 | 115 | # Upload log to Google Drive 116 | folder.upload_log(session_log, title="20220722-理系集会-some_title") 117 | 118 | # Get a list of logs stored in Google Drive 119 | folder.get_log_list() 120 | 121 | # Download a session log from Google Drive 122 | # 123 | # Note: `get_log_by_title` returns a list of session logs, because Google Drives allows having same title. 124 | session_logs = folder.get_log_by_title("20220722-理系集会-some_title") 125 | 126 | one_session_log = session_logs[0] 127 | ``` 128 | -------------------------------------------------------------------------------- /ipynb_examples/PlayerPositionPlot-old.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ScienceAssembly/YAIBA/5e58f33a220a6ade05e44c8a800ba762d7d6620f/ipynb_examples/PlayerPositionPlot-old.gif -------------------------------------------------------------------------------- /ipynb_examples/PlayerPositionPlot.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ScienceAssembly/YAIBA/5e58f33a220a6ade05e44c8a800ba762d7d6620f/ipynb_examples/PlayerPositionPlot.gif -------------------------------------------------------------------------------- /ipynb_examples/YAIBA Interactive Player Location Example.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": 1, 13 | "metadata": {}, 14 | "outputs": [ 15 | { 16 | "data": { 17 | "text/plain": [ 18 | "SessionLog(log_entries=[45301 entries], metadata=None)" 19 | ] 20 | }, 21 | "execution_count": 1, 22 | "metadata": {}, 23 | "output_type": "execute_result" 24 | } 25 | ], 26 | "source": [ 27 | "import yaiba\n", 28 | "import os\n", 29 | "from pathlib import Path\n", 30 | "with Path(\"~/tmp/log.txt\").expanduser().open() as fp:\n", 31 | " session_log = yaiba.parse_vrchat_log(fp)\n", 32 | "\n", 33 | "session_log" 34 | ] 35 | }, 36 | { 37 | "cell_type": "code", 38 | "execution_count": 2, 39 | "metadata": { 40 | "pycharm": { 41 | "name": "#%%\n" 42 | } 43 | }, 44 | "outputs": [], 45 | "source": [ 46 | "import yaiba.visualization.vrc\n", 47 | "import importlib\n", 48 | "importlib.reload(yaiba.visualization.vrc)\n", 49 | "None" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": 3, 55 | "metadata": { 56 | "pycharm": { 57 | "name": "#%%\n" 58 | } 59 | }, 60 | "outputs": [ 61 | { 62 | "data": { 63 | "application/vnd.jupyter.widget-view+json": { 64 | "model_id": "ec209f9ee8b04e7eb4819983f6938427", 65 | "version_major": 2, 66 | "version_minor": 0 67 | }, 68 | "text/plain": [ 69 | "VBox(children=(VBox(children=(HBox(children=(Label(value='Entering room:'), Dropdown(options=('2022-07-22T21:2…" 70 | ] 71 | }, 72 | "metadata": {}, 73 | "output_type": "display_data" 74 | } 75 | ], 76 | "source": [ 77 | "plotter = yaiba.visualization.vrc.PlayerLocationPlotter(session_log)\n", 78 | "\n", 79 | "plotter.plot()" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": null, 85 | "metadata": { 86 | "pycharm": { 87 | "name": "#%%\n" 88 | } 89 | }, 90 | "outputs": [], 91 | "source": [] 92 | }, 93 | { 94 | "cell_type": "code", 95 | "execution_count": null, 96 | "metadata": { 97 | "pycharm": { 98 | "name": "#%%\n" 99 | } 100 | }, 101 | "outputs": [], 102 | "source": [] 103 | } 104 | ], 105 | "metadata": { 106 | "kernelspec": { 107 | "display_name": "Python 3 (ipykernel)", 108 | "language": "python", 109 | "name": "python3" 110 | }, 111 | "language_info": { 112 | "codemirror_mode": { 113 | "name": "ipython", 114 | "version": 3 115 | }, 116 | "file_extension": ".py", 117 | "mimetype": "text/x-python", 118 | "name": "python", 119 | "nbconvert_exporter": "python", 120 | "pygments_lexer": "ipython3", 121 | "version": "3.10.5" 122 | } 123 | }, 124 | "nbformat": 4, 125 | "nbformat_minor": 1 126 | } 127 | -------------------------------------------------------------------------------- /poetry.toml: -------------------------------------------------------------------------------- 1 | [virtualenvs] 2 | in-project = true 3 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "YAIBA" 3 | version = "0.9.0" 4 | description = "Yet Another Interactive Behavior Analysis tool for VRSNS." 5 | authors = ["ScienceAssembly"] 6 | license = "MIT" 7 | packages = [ 8 | { include = "yaiba" }, 9 | { include = "yaiba_colab" }, # XXX: Better not to add here, consider to put under a new repo. 10 | { include = "yaiba_scienceassembly" }, # XXX: Better not to add here, consider to put under a new repo. 11 | ] 12 | 13 | [tool.poetry.dependencies] 14 | python = ">=3.9" 15 | 16 | [tool.poetry.group.visualize.dependencies] 17 | ipywidgets = "^7.7.1" 18 | jupyter = "^1.0.0" 19 | plotly = "^5.9.0" 20 | jupyterlab = "^3.4.4" 21 | pandas = "^2.2.3" 22 | 23 | [tool.poetry.group.dev.dependencies] 24 | pytest = "^7.4.4" 25 | 26 | [build-system] 27 | requires = ["poetry-core>=1.0.0"] 28 | build-backend = "poetry.core.masonry.api" 29 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /yaiba/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import TextIO, Union 2 | 3 | from yaiba.log import JsonDecoder, JsonEncoder, SessionLog, VRCLogParser 4 | 5 | try: 6 | from yaiba.visualization.vrc import VRCPlayerLocationPlotter 7 | except ImportError as e: 8 | pass 9 | 10 | 11 | def parse_vrchat_log(fp: Union[TextIO, str], config: VRCLogParser.Config = None) -> SessionLog: 12 | parser = VRCLogParser(config=config) 13 | if isinstance(fp, str): 14 | session_log = parser.parse(fp) 15 | else: 16 | session_log = parser.parse_file(fp) 17 | return session_log 18 | 19 | 20 | def save_session_log(session_log: SessionLog, fp: TextIO, options: JsonEncoder.Options = None): 21 | encoder = JsonEncoder(options=options) 22 | fp.write(encoder.encode(session_log)) 23 | 24 | 25 | def load_session_log(fp: TextIO, options: JsonDecoder.Options = None) -> SessionLog: 26 | decoder = JsonDecoder(options) 27 | return decoder.decode(fp.read()) 28 | 29 | 30 | __all__ = [ 31 | JsonDecoder, JsonEncoder, SessionLog, VRCLogParser, 32 | parse_vrchat_log, 33 | save_session_log, 34 | load_session_log, 35 | ] 36 | -------------------------------------------------------------------------------- /yaiba/constants.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | DEFAULT_TIMEZONE = datetime.timezone.utc 4 | -------------------------------------------------------------------------------- /yaiba/log/__init__.py: -------------------------------------------------------------------------------- 1 | from yaiba.log.session_log import Entry, SessionLog 2 | from yaiba.log.session_log_json import JsonDecoder, JsonEncoder 3 | from yaiba.log.vrc.parser import VRCLogParser 4 | 5 | __all__ = [ 6 | 'VRCLogParser', 7 | 'SessionLog', 8 | 'Entry', 9 | 'JsonDecoder', 10 | 'JsonEncoder', 11 | ] 12 | -------------------------------------------------------------------------------- /yaiba/log/entries.py: -------------------------------------------------------------------------------- 1 | from yaiba.log.vrc.entries.builtin import VRCEnteringRoomEntry, VRCPlayerJoinEntry, VRCPlayerLeftEntry 2 | from yaiba.log.vrc.entries.player_position import VRCYAIBAPlayerPositionEntry, VRCYAIBAPlayerPositionVersionEntry 3 | from yaiba.log.vrc.entries.questionnaire import VRCYAIBAQuestionnaireAnswerEntry 4 | from yaiba.log.vrc.entries.yodokoro_tag_marker import VRCYodokoroTagMarkerEntry 5 | 6 | ALL_ENTRIES = [ 7 | VRCEnteringRoomEntry, 8 | VRCPlayerJoinEntry, 9 | VRCPlayerLeftEntry, 10 | VRCYodokoroTagMarkerEntry, 11 | VRCYAIBAPlayerPositionEntry, 12 | VRCYAIBAPlayerPositionVersionEntry, 13 | VRCYAIBAQuestionnaireAnswerEntry, 14 | ] 15 | 16 | __all__ = ALL_ENTRIES 17 | -------------------------------------------------------------------------------- /yaiba/log/pseudonymizer.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import base64 4 | import hashlib 5 | import secrets 6 | 7 | from yaiba.log.types import PseudoUserName, UserName 8 | 9 | 10 | class Pseudonymizer: 11 | """ 12 | Pseudonymize the input (ex. username). 13 | 14 | `salt` can be any byte strings ( https://en.wikipedia.org/wiki/Salt_(cryptography) ), but should be different 15 | across stakeholders (See guideline from Japan Gov; 16 | https://www.ppc.go.jp/files/pdf/280930_siryou1-5.pdf#page=13). If salt is not applied, salt is generated 17 | randomly. 18 | """ 19 | 20 | def __init__(self, salt: bytes): 21 | self.salt = salt 22 | 23 | @classmethod 24 | def new_random(cls) -> Pseudonymizer: 25 | return cls(secrets.token_bytes(32)) 26 | 27 | def pseudonymize_user_name(self, user_name: UserName) -> PseudoUserName: 28 | hasher = hashlib.sha256() 29 | hasher.update(user_name.encode('utf-8')) 30 | 31 | # salt 32 | hasher.update(b'\0') 33 | hasher.update(self.salt) 34 | 35 | pseudonymized = base64.b64encode(hasher.digest()).decode('utf-8') 36 | return PseudoUserName(pseudonymized) 37 | -------------------------------------------------------------------------------- /yaiba/log/session_log.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from abc import ABC, abstractmethod 4 | from dataclasses import dataclass, field 5 | from typing import Any, Dict, List, Optional 6 | 7 | from yaiba.log.types import FromJson, RawEntry 8 | 9 | 10 | @dataclass(repr=False) 11 | class SessionLog: 12 | log_entries: List[Entry] 13 | metadata: Optional[Any] = field(default=None) 14 | 15 | def __repr__(self): 16 | return f'SessionLog(log_entries=[{len(self.log_entries)} entries], metadata={self.metadata!r})' 17 | 18 | 19 | class Entry(FromJson, ABC): 20 | """ 21 | Base class of all log entry. 22 | 23 | Subclass must be a dataclass. 24 | """ 25 | 26 | @classmethod 27 | @abstractmethod 28 | def type_id(cls): 29 | pass 30 | 31 | @classmethod 32 | @abstractmethod 33 | def from_json(cls, value: Dict[str, Any]) -> Entry: 34 | """ 35 | Note: Should use `value.get(name)` rather than `value[name]`. To avoid key value exception. 36 | """ 37 | pass 38 | 39 | 40 | class EntryParser(ABC): 41 | @abstractmethod 42 | def parse(self, raw_log: RawEntry) -> Optional[Entry]: 43 | pass 44 | -------------------------------------------------------------------------------- /yaiba/log/session_log_csv.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import dataclasses 3 | import io 4 | from dataclasses import is_dataclass, dataclass 5 | from typing import TextIO, Type 6 | 7 | from yaiba.log import Entry, SessionLog 8 | from yaiba.log.types import PseudoUserName, Timestamp, UserName, VRCPlayerId 9 | 10 | 11 | class CsvEncoder: 12 | 13 | @dataclass 14 | class Options: 15 | encode_timestamp: bool = True 16 | encode_pseudo_user_name: bool = True 17 | encode_vrc_player_id: bool = True 18 | encode_user_name: bool = False 19 | 20 | @classmethod 21 | def default(cls): 22 | return cls() 23 | 24 | @classmethod 25 | def export_all(cls): 26 | return cls( 27 | encode_user_name=True, 28 | encode_pseudo_user_name=True, 29 | encode_timestamp=True, 30 | encode_vrc_player_id=True, 31 | ) 32 | 33 | def __init__(self, entry_class: Type[Entry], options: Options = None): 34 | self.entry_class = entry_class 35 | if options is None: 36 | options = CsvEncoder.Options.default() 37 | self.options = options 38 | 39 | def encode(self, session_log: SessionLog) -> str: 40 | fp = io.StringIO() 41 | self.write(session_log, fp) 42 | return fp.getvalue() 43 | 44 | def write(self, session_log: SessionLog, fp: TextIO): 45 | assert is_dataclass(self.entry_class) 46 | fields = dataclasses.fields(self.entry_class) 47 | field_names = [ 48 | f.name for f in fields 49 | if self._is_ok_to_encode(f.type) 50 | ] 51 | 52 | writer = csv.DictWriter(fp, fieldnames=field_names, extrasaction="ignore") 53 | 54 | writer.writeheader() 55 | for entry in session_log.log_entries: 56 | if not isinstance(entry, self.entry_class): 57 | continue 58 | writer.writerow(dataclasses.asdict(entry)) 59 | 60 | def _is_ok_to_encode(self, v_type): 61 | if isinstance(v_type, Timestamp): 62 | return self.options.encode_timestamp 63 | if isinstance(v_type, PseudoUserName): 64 | return self.options.encode_pseudo_user_name 65 | if isinstance(v_type, UserName): 66 | return self.options.encode_user_name 67 | if isinstance(v_type, VRCPlayerId): 68 | return self.options.encode_vrc_player_id 69 | return True -------------------------------------------------------------------------------- /yaiba/log/session_log_json.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import dataclasses 4 | import json 5 | from dataclasses import dataclass 6 | from typing import Any, Dict, Optional, Type 7 | 8 | from yaiba.log.entries import ALL_ENTRIES 9 | from yaiba.log.session_log import Entry, SessionLog 10 | from yaiba.log.types import FromJson, PseudoUserName, Timestamp, UserName, VRCPlayerId 11 | 12 | ENTRY_TYPE_ID_ATTR_NAME = "type_id" 13 | 14 | 15 | class JsonEncoder: 16 | """ 17 | Json serializer. 18 | """ 19 | 20 | @dataclass 21 | class Options: 22 | output_timestamp: bool = True 23 | output_pseudo_user_name: bool = True 24 | output_vrc_player_id: bool = True 25 | output_user_name: bool = True 26 | 27 | @classmethod 28 | def default(cls): 29 | return cls() 30 | 31 | @classmethod 32 | def export_all(cls): 33 | return cls( 34 | output_timestamp=True, 35 | output_pseudo_user_name=True, 36 | output_vrc_player_id=True, 37 | output_user_name=True, 38 | ) 39 | 40 | @classmethod 41 | def pseudonymized(cls): 42 | return cls( 43 | output_timestamp=True, 44 | output_pseudo_user_name=True, 45 | output_vrc_player_id=True, 46 | output_user_name=False, 47 | ) 48 | 49 | def __init__(self, options: Optional[Options] = None): 50 | if options is None: 51 | options = JsonEncoder.Options.default() 52 | self.options = options 53 | 54 | def encode(self, session_log: SessionLog): 55 | encoder = json.JSONEncoder( 56 | default=self._encoder_default, 57 | ) 58 | return encoder.encode(session_log) 59 | 60 | def _encoder_default(self, o): 61 | if isinstance(o, SessionLog): 62 | return self._dataclasses_shadow_asdict(o) 63 | if isinstance(o, Entry): 64 | if dataclasses.is_dataclass(o): 65 | values = self._make_safe_to_store(self._dataclasses_shadow_asdict(o)) 66 | values[ENTRY_TYPE_ID_ATTR_NAME] = o.type_id() 67 | return values 68 | if isinstance(o, Timestamp): 69 | return o.timestamp() 70 | if dataclasses.is_dataclass(o): 71 | return dataclasses.asdict(o) 72 | 73 | @staticmethod 74 | def _dataclasses_shadow_asdict(o): 75 | return { 76 | field.name: getattr(o, field.name) 77 | for field in dataclasses.fields(o) 78 | } 79 | 80 | def _make_safe_to_store(self, o: Dict[str, Any]) -> Dict[str, Any]: 81 | return { 82 | k: v 83 | for k, v in o.items() 84 | if self._is_ok_to_store(k, v) 85 | } 86 | 87 | def _is_ok_to_store(self, k: str, v: Any): 88 | if isinstance(v, UserName): 89 | return self.options.output_user_name 90 | 91 | if isinstance(v, PseudoUserName): 92 | return self.options.output_pseudo_user_name 93 | 94 | if isinstance(v, VRCPlayerId): 95 | return self.options.output_vrc_player_id 96 | 97 | if isinstance(v, Timestamp): 98 | return self.options.output_timestamp 99 | 100 | return True 101 | 102 | @classmethod 103 | def export_all(cls): 104 | return cls(JsonEncoder.Options.export_all()) 105 | 106 | 107 | class JsonDecoder: 108 | """ 109 | Json deserializer. 110 | """ 111 | 112 | @dataclass 113 | class Options: 114 | metadata_class: Optional[Type[FromJson]] = None 115 | 116 | @classmethod 117 | def default(cls): 118 | return cls 119 | 120 | def __init__(self, options: Optional[Options] = None): 121 | if options is None: 122 | options = JsonDecoder.Options.default() 123 | self.options = options 124 | self.entry_class_by_id: Dict[str, Entry] = { 125 | klass.type_id(): klass 126 | for klass in ALL_ENTRIES 127 | } 128 | 129 | def decode(self, session_log_str: str) -> SessionLog: 130 | decoder = json.JSONDecoder() 131 | session_log_dict = decoder.decode(session_log_str) 132 | log_entries_json = session_log_dict.get("log_entries") 133 | metadata_json = session_log_dict.get("metadata") 134 | 135 | log_entries = [] 136 | for entry_json in log_entries_json: 137 | entry_class = self.entry_class_by_id.get(entry_json.get(ENTRY_TYPE_ID_ATTR_NAME)) 138 | entry = entry_class.from_json(entry_json) 139 | log_entries.append(entry) 140 | 141 | if metadata_json is not None: 142 | if self.options.metadata_class is not None: 143 | metadata = self.options.metadata_class.from_json(metadata_json) 144 | else: 145 | metadata = metadata_json 146 | else: 147 | metadata = None 148 | 149 | return SessionLog( 150 | log_entries=log_entries, 151 | metadata=metadata 152 | ) 153 | -------------------------------------------------------------------------------- /yaiba/log/test_utils.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import Mock 2 | 3 | from yaiba.log import Entry, JsonDecoder, JsonEncoder, SessionLog 4 | from yaiba.log.pseudonymizer import Pseudonymizer 5 | from yaiba.log.types import PseudoUserName 6 | 7 | 8 | def new_pseudonymizer(return_value: str = "pseudo_user_name") -> Pseudonymizer: 9 | pseudonymizer: Pseudonymizer = Mock() 10 | pseudonymizer.pseudonymize_user_name.return_value = PseudoUserName(return_value) 11 | return pseudonymizer 12 | 13 | 14 | def encode_and_then_decode(entry: Entry) -> Entry: 15 | encoder = JsonEncoder.export_all() 16 | json = encoder.encode(SessionLog(log_entries=[entry])) 17 | 18 | decoder = JsonDecoder() 19 | session_log = decoder.decode(json) 20 | 21 | assert len(session_log.log_entries) == 1 22 | return session_log.log_entries[0] 23 | -------------------------------------------------------------------------------- /yaiba/log/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ScienceAssembly/YAIBA/5e58f33a220a6ade05e44c8a800ba762d7d6620f/yaiba/log/tests/__init__.py -------------------------------------------------------------------------------- /yaiba/log/tests/test_session_log_csv.py: -------------------------------------------------------------------------------- 1 | from yaiba.log import SessionLog 2 | from yaiba.log.session_log_csv import CsvEncoder 3 | from yaiba.log.types import PseudoUserName, UserName, VRCPlayerId 4 | from yaiba.log.vrc.entries.builtin import VRCPlayerJoinEntry 5 | from yaiba.log.vrc.entries.player_position import VRCYAIBAPlayerPositionEntry 6 | from yaiba.log.vrc.utils import parse_timestamp 7 | 8 | 9 | class TestCsvEncoder: 10 | def test__normal(self): 11 | session_log = SessionLog(log_entries=[ 12 | VRCYAIBAPlayerPositionEntry( 13 | parse_timestamp("2022.03.04 21:50:19"), 14 | 15 | player_id=VRCPlayerId(7), 16 | 17 | user_name=UserName("E.HOBA"), 18 | pseudo_user_name=PseudoUserName('E.HOBA pseudo'), 19 | 20 | location_x=1.0, 21 | location_y=2.0, 22 | location_z=3.0, 23 | rotation_1=4.0, 24 | rotation_2=5.0, 25 | rotation_3=6.0, 26 | 27 | is_vr=True, 28 | ), 29 | VRCYAIBAPlayerPositionEntry( 30 | parse_timestamp("2022.03.04 21:50:19"), 31 | 32 | player_id=VRCPlayerId(7), 33 | 34 | user_name=UserName("A.HOBA"), 35 | pseudo_user_name=PseudoUserName('A.HOBA pseudo'), 36 | 37 | location_x=1.0, 38 | location_y=2.0, 39 | location_z=3.0, 40 | rotation_1=4.0, 41 | rotation_2=5.0, 42 | rotation_3=6.0, 43 | 44 | is_vr=True, 45 | ), 46 | VRCPlayerJoinEntry( 47 | parse_timestamp("2022.03.04 21:50:19"), 48 | 49 | user_name=UserName("E.HOBA"), 50 | pseudo_user_name=PseudoUserName('E.HOBA pseudo'), 51 | ), 52 | VRCYAIBAPlayerPositionEntry( 53 | parse_timestamp("2022.03.04 21:50:19"), 54 | 55 | player_id=VRCPlayerId(7), 56 | 57 | user_name=UserName("B.HOBA"), 58 | pseudo_user_name=PseudoUserName('B.HOBA pseudo'), 59 | 60 | location_x=1.0, 61 | location_y=2.0, 62 | location_z=3.0, 63 | rotation_1=4.0, 64 | rotation_2=5.0, 65 | rotation_3=6.0, 66 | 67 | is_vr=True, 68 | ), 69 | ]) 70 | 71 | encoder = CsvEncoder(VRCYAIBAPlayerPositionEntry) 72 | 73 | assert encoder.encode(session_log) == ( 74 | 'timestamp,player_id,user_name,pseudo_user_name,location_x,location_y,location_z,rotation_1,rotation_2,' 75 | 'rotation_3,is_vr\r\n' 76 | '2022-03-04 21:50:19+00:00,7,E.HOBA,E.HOBA pseudo,1.0,2.0,3.0,4.0,5.0,6.0,True\r\n' 77 | '2022-03-04 21:50:19+00:00,7,A.HOBA,A.HOBA pseudo,1.0,2.0,3.0,4.0,5.0,6.0,True\r\n' 78 | '2022-03-04 21:50:19+00:00,7,B.HOBA,B.HOBA pseudo,1.0,2.0,3.0,4.0,5.0,6.0,True\r\n') 79 | -------------------------------------------------------------------------------- /yaiba/log/tests/test_session_log_json.py: -------------------------------------------------------------------------------- 1 | from yaiba.log.session_log import SessionLog 2 | from yaiba.log.session_log_json import JsonDecoder, JsonEncoder 3 | from yaiba.log.types import PseudoUserName, UserName 4 | from yaiba.log.vrc.entries.builtin import VRCPlayerJoinEntry 5 | from yaiba.log.vrc.utils import parse_timestamp 6 | 7 | 8 | class TestJsonEncoder: 9 | def test__normal(self): 10 | options = JsonEncoder.Options.pseudonymized() 11 | encoder = JsonEncoder(options=options) 12 | 13 | log = SessionLog( 14 | log_entries=[ 15 | VRCPlayerJoinEntry( 16 | timestamp=parse_timestamp("2022.03.04 21:50:19"), 17 | user_name=UserName("E.HOBA"), 18 | pseudo_user_name=PseudoUserName("E.HOBA Pseudo") 19 | ), 20 | VRCPlayerJoinEntry( 21 | timestamp=parse_timestamp("2022.03.04 21:50:19"), 22 | user_name=UserName("E.HOBA"), 23 | pseudo_user_name=PseudoUserName("E.HOBA Pseudo") 24 | ), 25 | VRCPlayerJoinEntry( 26 | timestamp=parse_timestamp("2022.03.04 21:50:19"), 27 | user_name=UserName("E.HOBA"), 28 | pseudo_user_name=PseudoUserName("E.HOBA Pseudo") 29 | ) 30 | ]) 31 | 32 | assert encoder.encode(log) == ( 33 | '{"log_entries": [{"timestamp": 1646430619.0, "pseudo_user_name": "E.HOBA ' 34 | 'Pseudo", "type_id": "vrc/player_join"}, {"timestamp": 1646430619.0, ' 35 | '"pseudo_user_name": "E.HOBA Pseudo", "type_id": "vrc/player_join"}, ' 36 | '{"timestamp": 1646430619.0, "pseudo_user_name": "E.HOBA Pseudo", "type_id": ' 37 | '"vrc/player_join"}], "metadata": null}') 38 | 39 | def test__output_all_personal_info(self): 40 | options = JsonEncoder.Options.default() 41 | options.output_user_name = True 42 | encoder = JsonEncoder(options=options) 43 | 44 | log = SessionLog( 45 | log_entries=[ 46 | VRCPlayerJoinEntry( 47 | timestamp=parse_timestamp("2022.03.04 21:50:19"), 48 | user_name=UserName("E.HOBA"), 49 | pseudo_user_name=PseudoUserName("E.HOBA Pseudo") 50 | ), 51 | VRCPlayerJoinEntry( 52 | timestamp=parse_timestamp("2022.03.04 21:50:19"), 53 | user_name=UserName("E.HOBA"), 54 | pseudo_user_name=PseudoUserName("E.HOBA Pseudo") 55 | ), 56 | VRCPlayerJoinEntry( 57 | timestamp=parse_timestamp("2022.03.04 21:50:19"), 58 | user_name=UserName("E.HOBA"), 59 | pseudo_user_name=PseudoUserName("E.HOBA Pseudo") 60 | ) 61 | ]) 62 | 63 | assert encoder.encode(log) == ( 64 | '{"log_entries": [{"timestamp": 1646430619.0, "user_name": "E.HOBA", ' 65 | '"pseudo_user_name": "E.HOBA Pseudo", "type_id": "vrc/player_join"}, ' 66 | '{"timestamp": 1646430619.0, "user_name": "E.HOBA", "pseudo_user_name": ' 67 | '"E.HOBA Pseudo", "type_id": "vrc/player_join"}, {"timestamp": 1646430619.0, ' 68 | '"user_name": "E.HOBA", "pseudo_user_name": "E.HOBA Pseudo", "type_id": ' 69 | '"vrc/player_join"}], "metadata": null}') 70 | 71 | def test__strict_output_option(self): 72 | options = JsonEncoder.Options.default() 73 | options.output_timestamp = False 74 | options.output_pseudo_user_name = False 75 | options.output_vrc_player_id = False 76 | options.output_user_name = False 77 | encoder = JsonEncoder(options=options) 78 | 79 | log = SessionLog( 80 | log_entries=[ 81 | VRCPlayerJoinEntry( 82 | timestamp=parse_timestamp("2022.03.04 21:50:19"), 83 | user_name=UserName("E.HOBA"), 84 | pseudo_user_name=PseudoUserName("E.HOBA Pseudo") 85 | ), 86 | VRCPlayerJoinEntry( 87 | timestamp=parse_timestamp("2022.03.04 21:50:19"), 88 | user_name=UserName("E.HOBA"), 89 | pseudo_user_name=PseudoUserName("E.HOBA Pseudo") 90 | ), 91 | VRCPlayerJoinEntry( 92 | timestamp=parse_timestamp("2022.03.04 21:50:19"), 93 | user_name=UserName("E.HOBA"), 94 | pseudo_user_name=PseudoUserName("E.HOBA Pseudo") 95 | ) 96 | ]) 97 | 98 | assert encoder.encode(log) == ( 99 | '{"log_entries": [{"type_id": "vrc/player_join"}, {"type_id": ' 100 | '"vrc/player_join"}, {"type_id": "vrc/player_join"}], "metadata": null}') 101 | 102 | 103 | class TestJsonDecoder: 104 | def test__normal(self): 105 | decoder = JsonDecoder() 106 | 107 | output = decoder.decode( 108 | '{"log_entries": [{"timestamp": 1646430619.0, "user_name": "E.HOBA", ' 109 | '"pseudo_user_name": "E.HOBA Pseudo", "type_id": "vrc/player_join"}, ' 110 | '{"timestamp": 1646430619.0, "user_name": "E.HOBA", "pseudo_user_name": ' 111 | '"E.HOBA Pseudo", "type_id": "vrc/player_join"}, {"timestamp": 1646430619.0, ' 112 | '"user_name": "E.HOBA", "pseudo_user_name": "E.HOBA Pseudo", "type_id": ' 113 | '"vrc/player_join"}], "metadata": null}') 114 | 115 | assert output == SessionLog( 116 | log_entries=[ 117 | VRCPlayerJoinEntry( 118 | timestamp=parse_timestamp("2022.03.04 21:50:19"), 119 | user_name=UserName("E.HOBA"), 120 | pseudo_user_name=PseudoUserName("E.HOBA Pseudo") 121 | ), 122 | VRCPlayerJoinEntry( 123 | timestamp=parse_timestamp("2022.03.04 21:50:19"), 124 | user_name=UserName("E.HOBA"), 125 | pseudo_user_name=PseudoUserName("E.HOBA Pseudo") 126 | ), 127 | VRCPlayerJoinEntry( 128 | timestamp=parse_timestamp("2022.03.04 21:50:19"), 129 | user_name=UserName("E.HOBA"), 130 | pseudo_user_name=PseudoUserName("E.HOBA Pseudo") 131 | ) 132 | ]) 133 | -------------------------------------------------------------------------------- /yaiba/log/types.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import datetime 4 | from abc import ABC, abstractmethod 5 | from typing import Any, Dict, TypeVar 6 | 7 | from yaiba.constants import DEFAULT_TIMEZONE 8 | 9 | 10 | class FromJson(ABC): 11 | @classmethod 12 | @abstractmethod 13 | def from_json(cls, value: Dict[str, Any]) -> T: 14 | pass 15 | 16 | 17 | class Timestamp(datetime.datetime, FromJson): 18 | @classmethod 19 | def from_json(cls, timestamp): 20 | return cls.fromtimestamp(timestamp, tz=DEFAULT_TIMEZONE) 21 | 22 | 23 | class UserName(str): 24 | pass 25 | 26 | 27 | class PseudoUserName(str): 28 | """ 29 | Pseudonymized user name. 30 | """ 31 | pass 32 | 33 | 34 | class VRCPlayerId(int): 35 | """ 36 | World internal player id. 37 | https://docs.vrchat.com/docs/getting-players#getplayerbyid 38 | """ 39 | pass 40 | 41 | 42 | class RawEntry(str): 43 | """ 44 | One log entry which is not parsed yet. 45 | 46 | Ex. VRChat:`2022.03.04 21:50:19 Log - [Behaviour] EnteringRoom: Some Room` 47 | """ 48 | pass 49 | 50 | 51 | T = TypeVar("T") 52 | -------------------------------------------------------------------------------- /yaiba/log/vrc/__init__.py: -------------------------------------------------------------------------------- 1 | from yaiba.log.vrc.entries import ( 2 | VRCEnteringRoomEntry, VRCPlayerJoinEntry, VRCPlayerLeftEntry, VRCYAIBAPlayerPositionEntry, 3 | VRCYAIBAPlayerPositionVersionEntry, VRCYAIBAQuestionnaireAnswerEntry, VRCYodokoroTagMarkerEntry 4 | ) 5 | -------------------------------------------------------------------------------- /yaiba/log/vrc/entries/__init__.py: -------------------------------------------------------------------------------- 1 | from yaiba.log.vrc.entries.builtin import VRCEnteringRoomEntry, VRCPlayerJoinEntry, VRCPlayerLeftEntry 2 | from yaiba.log.vrc.entries.player_position import VRCYAIBAPlayerPositionEntry, VRCYAIBAPlayerPositionVersionEntry 3 | from yaiba.log.vrc.entries.questionnaire import VRCYAIBAQuestionnaireAnswerEntry 4 | from yaiba.log.vrc.entries.yodokoro_tag_marker import VRCYodokoroTagMarkerEntry 5 | 6 | ALL_ENTRIES = [ 7 | VRCEnteringRoomEntry, 8 | VRCPlayerJoinEntry, 9 | VRCPlayerLeftEntry, 10 | VRCYodokoroTagMarkerEntry, 11 | VRCYAIBAPlayerPositionEntry, 12 | VRCYAIBAPlayerPositionVersionEntry, 13 | VRCYAIBAQuestionnaireAnswerEntry, 14 | ] 15 | 16 | __all__ = [ 17 | e.__name__ 18 | for e in ALL_ENTRIES 19 | ] 20 | -------------------------------------------------------------------------------- /yaiba/log/vrc/entries/builtin.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import re 4 | from dataclasses import dataclass 5 | from typing import Any, ClassVar, Dict, Optional 6 | 7 | from yaiba.log.pseudonymizer import Pseudonymizer 8 | from yaiba.log.session_log import Entry, EntryParser 9 | from yaiba.log.types import PseudoUserName, RawEntry, Timestamp, UserName 10 | from yaiba.log.vrc.utils import VRC_REGEX_LOG_PREFIX, create_timestamp_from_match 11 | 12 | 13 | @dataclass(frozen=True) 14 | class VRCEnteringRoomEntry(Entry): 15 | timestamp: Timestamp 16 | room_name: str 17 | 18 | @classmethod 19 | def type_id(cls): 20 | return 'vrc/entering_room' 21 | 22 | @classmethod 23 | def from_json(cls, value: Dict[str, Any]) -> Entry: 24 | return cls( 25 | timestamp=Timestamp.from_json(value.get('timestamp')), 26 | room_name=value.get('room_name') 27 | ) 28 | 29 | 30 | @dataclass(frozen=True) 31 | class VRCPlayerJoinEntry(Entry): 32 | timestamp: Timestamp 33 | user_name: Optional[UserName] 34 | pseudo_user_name: PseudoUserName 35 | 36 | @classmethod 37 | def type_id(cls): 38 | return 'vrc/player_join' 39 | 40 | @classmethod 41 | def from_json(cls, value: Dict[str, Any]) -> Entry: 42 | return cls( 43 | timestamp=Timestamp.from_json(value.get('timestamp')), 44 | user_name=value.get('user_name'), 45 | pseudo_user_name=value.get('pseudo_user_name'), 46 | ) 47 | 48 | 49 | @dataclass(frozen=True) 50 | class VRCPlayerLeftEntry(Entry): 51 | timestamp: Timestamp 52 | user_name: Optional[UserName] 53 | pseudo_user_name: PseudoUserName 54 | 55 | @classmethod 56 | def type_id(cls): 57 | return 'vrc/player_left' 58 | 59 | @classmethod 60 | def from_json(cls, value: Dict[str, Any]) -> Entry: 61 | return cls( 62 | timestamp=Timestamp.from_json(value.get('timestamp')), 63 | user_name=value.get('user_name'), 64 | pseudo_user_name=value.get('pseudo_user_name'), 65 | ) 66 | 67 | 68 | class VRCBuiltinEntryParser(EntryParser): 69 | regex_entering_room: ClassVar[re.Pattern] = re.compile( 70 | VRC_REGEX_LOG_PREFIX + 71 | r'\[Behaviour] Entering Room: (?P.+)$' 72 | ) 73 | regex_player_join: ClassVar[re.Pattern] = re.compile( 74 | VRC_REGEX_LOG_PREFIX + 75 | r'\[Behaviour] OnPlayerJoined (?P.+)$' 76 | ) 77 | regex_player_left: ClassVar[re.Pattern] = re.compile( 78 | VRC_REGEX_LOG_PREFIX + 79 | r'\[Behaviour] OnPlayerLeft (?P.+)$' 80 | ) 81 | 82 | def __init__(self, pseudonymizer: Pseudonymizer): 83 | self.pseudonymizer = pseudonymizer 84 | 85 | def parse(self, log_entry: RawEntry) -> Optional[Entry]: 86 | for func in [ 87 | self._try_to_parse_entering_room, 88 | self._try_to_parse_player_join, 89 | self._try_to_parse_player_left, 90 | ]: 91 | value: Optional[Entry] = func(log_entry) 92 | if value is not None: 93 | return value 94 | return None 95 | 96 | def _try_to_parse_entering_room(self, log_entry: RawEntry) -> Optional[VRCEnteringRoomEntry]: 97 | match = self.regex_entering_room.match(log_entry) 98 | if match is None: 99 | return None 100 | return VRCEnteringRoomEntry( 101 | timestamp=create_timestamp_from_match(match), 102 | room_name=match.group('room_name'), 103 | ) 104 | 105 | def _try_to_parse_player_join(self, log_entry: RawEntry) -> Optional[VRCPlayerJoinEntry]: 106 | match = self.regex_player_join.match(log_entry) 107 | if match is None: 108 | return None 109 | user_name = UserName(match.group("user_name")) 110 | return VRCPlayerJoinEntry( 111 | timestamp=create_timestamp_from_match(match), 112 | user_name=user_name, 113 | pseudo_user_name=self.pseudonymizer.pseudonymize_user_name(user_name), 114 | ) 115 | 116 | def _try_to_parse_player_left(self, log_entry: RawEntry) -> Optional[VRCPlayerLeftEntry]: 117 | match = self.regex_player_left.match(log_entry) 118 | if match is None: 119 | return None 120 | user_name = UserName(match.group("user_name")) 121 | return VRCPlayerLeftEntry( 122 | timestamp=create_timestamp_from_match(match), 123 | user_name=user_name, 124 | pseudo_user_name=self.pseudonymizer.pseudonymize_user_name(user_name), 125 | ) 126 | -------------------------------------------------------------------------------- /yaiba/log/vrc/entries/player_position.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import re 3 | from dataclasses import dataclass 4 | from typing import Any, Dict, Optional 5 | 6 | from yaiba.log.pseudonymizer import Pseudonymizer 7 | from yaiba.log.session_log import Entry, EntryParser 8 | from yaiba.log.types import PseudoUserName, RawEntry, Timestamp, UserName, VRCPlayerId 9 | from yaiba.log.vrc.utils import VRC_REGEX_LOG_PREFIX, create_timestamp_from_match 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | @dataclass 15 | class VRCYAIBAPlayerPositionVersionEntry(Entry): 16 | timestamp: Timestamp 17 | 18 | major: int 19 | minor: int 20 | patch: int 21 | 22 | def to_tuple(self): 23 | return ( 24 | self.major, 25 | self.minor, 26 | self.patch, 27 | ) 28 | 29 | @classmethod 30 | def type_id(cls): 31 | return 'yaiba/player_position/version' 32 | 33 | @classmethod 34 | def from_json(cls, value: Dict[str, Any]) -> Entry: 35 | return cls( 36 | timestamp=Timestamp.from_json(value.get('timestamp')), 37 | major=value.get('major'), 38 | minor=value.get('minor'), 39 | patch=value.get('patch'), 40 | ) 41 | 42 | 43 | @dataclass 44 | class VRCYAIBAPlayerPositionEntry(Entry): 45 | timestamp: Timestamp 46 | player_id: VRCPlayerId 47 | 48 | user_name: Optional[UserName] 49 | pseudo_user_name: PseudoUserName 50 | 51 | location_x: float 52 | location_y: Optional[float] 53 | location_z: float # Can be None for V0 (ScienceAssembly internal version; pre-YAIBA) 54 | 55 | rotation_1: float # rotation for x axis, pitch in unity 56 | rotation_2: float # rotation for y axis, yaw in unity 57 | rotation_3: float # rotation for z axis, roll in unity 58 | 59 | is_vr: bool 60 | 61 | @classmethod 62 | def type_id(cls): 63 | return 'yaiba/player_position' 64 | 65 | @classmethod 66 | def from_json(cls, value: Dict[str, Any]) -> Entry: 67 | return cls( 68 | timestamp=Timestamp.from_json(value.get('timestamp')), 69 | player_id=VRCPlayerId(value.get('player_id')), 70 | 71 | user_name=UserName(value.get('user_name')), 72 | pseudo_user_name=PseudoUserName(value.get('pseudo_user_name')), 73 | 74 | location_x=value.get('location_x'), 75 | location_y=value.get('location_y'), 76 | location_z=value.get('location_z'), 77 | rotation_1=value.get('rotation_1'), 78 | rotation_2=value.get('rotation_2'), 79 | rotation_3=value.get('rotation_3'), 80 | 81 | is_vr=value.get('is_vr'), 82 | ) 83 | 84 | 85 | class YAIBAPlayerPositionEntryParser(EntryParser): 86 | """ 87 | Note: This parser may not work for some locales that uses a comma as a decimal point. 88 | """ 89 | 90 | regex_version = re.compile( 91 | VRC_REGEX_LOG_PREFIX + 92 | r'\[Player Position Version]\s*(?P\d+).(?P\d+).(?P\d+)' 93 | ) 94 | 95 | regex_entry_v0 = re.compile( 96 | VRC_REGEX_LOG_PREFIX + 97 | r'\[Player Position](?P\d+),"(?P.+)",' 98 | r'(?P[^,]*),(?P[^,]*),' 99 | r'(?P[^,]*),(?P[^,]*),' 100 | r'(?P[^,]*),' 101 | r'(?P[^,]*)' 102 | ) 103 | 104 | regex_entry_v1_0_0 = re.compile( 105 | VRC_REGEX_LOG_PREFIX + 106 | r'\[Player Position](?P\d+),"(?P.+)",' 107 | r'(?P[^,]*),(?P[^,]*),(?P[^,]*),' 108 | r'(?P[^,]*),(?P[^,]*),' 109 | r'(?P[^,]*),' 110 | r'(?P[^,]*)' 111 | ) 112 | 113 | def __init__(self, pseudonymizer: Pseudonymizer): 114 | self.regex_entry_used = self.regex_entry_v0 115 | self.pseudonymizer = pseudonymizer 116 | 117 | def parse(self, raw_log: RawEntry) -> Optional[Entry]: 118 | # Tries to parse as a position entry 119 | # Since position entry appears more frequently than version entry, checks position before version. 120 | position_entry = self._try_to_parse_entry(raw_log) 121 | if position_entry is not None: 122 | return position_entry 123 | 124 | # Tries to parse as a version 125 | version = self._try_to_parse_version(raw_log) 126 | if version is not None: 127 | self.regex_entry_used = self.regex_entry_v1_0_0 128 | if version.to_tuple() > (1, 0, 0): 129 | logger.warning(f"unexpected version is applied: {version}. Fallback to latest one") 130 | return version 131 | 132 | def _try_to_parse_version(self, log_entry: RawEntry) -> Optional[VRCYAIBAPlayerPositionVersionEntry]: 133 | match = self.regex_version.match(log_entry) 134 | if match is None: 135 | return None 136 | 137 | timestamp = create_timestamp_from_match(match) 138 | major = int(match['major']) 139 | minor = int(match['minor']) 140 | patch = int(match['patch']) 141 | 142 | return VRCYAIBAPlayerPositionVersionEntry( 143 | timestamp=timestamp, 144 | major=major, minor=minor, patch=patch, 145 | ) 146 | 147 | def _try_to_parse_entry(self, log_entry: RawEntry) -> Optional[VRCYAIBAPlayerPositionEntry]: 148 | match = self.regex_entry_used.match(log_entry) 149 | if match is None: 150 | return None 151 | 152 | timestamp = create_timestamp_from_match(match) 153 | player_id = VRCPlayerId(match.group('player_id')) 154 | 155 | user_name = match.group('user_name').replace('""', '"') # Following CSV Escape 156 | user_name = UserName(user_name) 157 | p_user_name = self.pseudonymizer.pseudonymize_user_name(user_name) 158 | 159 | location_x = float(match.group('location_x')) 160 | location_y = match.groupdict().get("location_y", None) # missing in v0. Return None. 161 | if location_y is not None: 162 | location_y = float(location_y) 163 | location_z = float(match.group('location_z')) 164 | rotation_1 = float(match.group('rotation_1')) 165 | rotation_2 = float(match.group('rotation_2')) 166 | rotation_3 = float(match.group('rotation_3')) 167 | is_vr = match.group('is_vr').lower() == "true" 168 | 169 | return VRCYAIBAPlayerPositionEntry( 170 | timestamp=timestamp, 171 | player_id=player_id, 172 | 173 | user_name=user_name, 174 | pseudo_user_name=p_user_name, 175 | 176 | location_x=location_x, 177 | location_y=location_y, 178 | location_z=location_z, 179 | 180 | rotation_1=rotation_1, 181 | rotation_2=rotation_2, 182 | rotation_3=rotation_3, 183 | 184 | is_vr=is_vr, 185 | ) 186 | -------------------------------------------------------------------------------- /yaiba/log/vrc/entries/questionnaire.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import re 3 | from dataclasses import dataclass 4 | from typing import Any, Dict, Optional 5 | 6 | from yaiba.log.session_log import Entry, EntryParser 7 | from yaiba.log.types import RawEntry, Timestamp 8 | from yaiba.log.vrc.utils import VRC_REGEX_LOG_PREFIX, create_timestamp_from_match 9 | 10 | 11 | @dataclass 12 | class VRCYAIBAQuestionnaireAnswerEntry(Entry): 13 | timestamp: Timestamp 14 | answer_for_question: Dict[str, str] = dataclasses.field(default_factory=dict) 15 | 16 | @classmethod 17 | def type_id(cls): 18 | return 'yaiba/questionnaire_answer' 19 | 20 | @classmethod 21 | def from_json(cls, value: Dict[str, Any]) -> Entry: 22 | return VRCYAIBAQuestionnaireAnswerEntry( 23 | timestamp=Timestamp.from_json(value.get('timestamp')), 24 | answer_for_question=value.get('answer_for_question') 25 | ) 26 | 27 | 28 | class YAIBAQuestionnaireAnswerEntryParser(EntryParser): 29 | regex_pattern: re.Pattern = re.compile( 30 | VRC_REGEX_LOG_PREFIX + 31 | r'\[Answer](?P.*)$' 32 | ) 33 | 34 | def parse(self, raw_log: RawEntry) -> Optional[Entry]: 35 | match = self.regex_pattern.match(raw_log) 36 | if match is None: 37 | return None 38 | 39 | timestamp = create_timestamp_from_match(match) 40 | answer_for_question = self._parse_q_and_a(match.group("question_and_answer")) 41 | 42 | return VRCYAIBAQuestionnaireAnswerEntry( 43 | timestamp=timestamp, 44 | answer_for_question=answer_for_question 45 | ) 46 | 47 | def _parse_q_and_a(self, qas_str) -> Dict[str, str]: 48 | """ 49 | :param qas_str: Ex. `"question_1","answer_1","question_2","answer_2"` 50 | """ 51 | if len(qas_str) < 3: 52 | return {} 53 | qas_list = qas_str[1:- 1].split('","') 54 | questions = qas_list[0::2] 55 | answers = qas_list[1::2] 56 | return { 57 | question: answer 58 | for question, answer 59 | in zip(questions, answers) 60 | } 61 | -------------------------------------------------------------------------------- /yaiba/log/vrc/entries/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ScienceAssembly/YAIBA/5e58f33a220a6ade05e44c8a800ba762d7d6620f/yaiba/log/vrc/entries/tests/__init__.py -------------------------------------------------------------------------------- /yaiba/log/vrc/entries/tests/test_builtin.py: -------------------------------------------------------------------------------- 1 | from yaiba.log.test_utils import encode_and_then_decode, new_pseudonymizer 2 | from yaiba.log.types import PseudoUserName, RawEntry, Timestamp, UserName 3 | from yaiba.log.vrc.entries.builtin import VRCBuiltinEntryParser, VRCEnteringRoomEntry, VRCPlayerJoinEntry, \ 4 | VRCPlayerLeftEntry 5 | from yaiba.log.vrc.utils import parse_timestamp 6 | 7 | 8 | class TestVRCEnteringRoomEntry: 9 | def test__regex(self): 10 | output = VRCBuiltinEntryParser.regex_entering_room.match( 11 | '2022.03.04 21:50:19 Log - [Behaviour] Entering Room: VRC理系集会-Science Assembly-') 12 | 13 | assert output is not None 14 | 15 | def test__parse(self): 16 | parser = VRCBuiltinEntryParser(new_pseudonymizer()) 17 | 18 | input = RawEntry("2022.03.04 21:50:19 Log - [Behaviour] Entering Room: HAKOBUNE") 19 | output = parser.parse(input) 20 | 21 | assert isinstance(output, VRCEnteringRoomEntry) 22 | assert isinstance(output.timestamp, Timestamp) 23 | assert output == VRCEnteringRoomEntry( 24 | timestamp=parse_timestamp('2022.03.04 21:50:19'), 25 | room_name="HAKOBUNE", 26 | ) 27 | 28 | def test__from_json(self): 29 | entry = VRCEnteringRoomEntry( 30 | timestamp=parse_timestamp('2022.03.04 21:50:19'), 31 | room_name="HAKOBUNE", 32 | ) 33 | 34 | assert encode_and_then_decode(entry) == entry 35 | 36 | 37 | class TestVRCPlayerJoinEntry: 38 | def test__regex(self): 39 | output = VRCBuiltinEntryParser.regex_player_join.match( 40 | '2022.03.04 21:50:22 Log - [Behaviour] OnPlayerJoined E.HOBA') 41 | 42 | assert output is not None 43 | 44 | def test__parse(self): 45 | parser = VRCBuiltinEntryParser(new_pseudonymizer(return_value="pseudonymized E.HOBA")) 46 | 47 | input = RawEntry('2022.03.04 21:50:22 Log - [Behaviour] OnPlayerJoined E.HOBA') 48 | output = parser.parse(input) 49 | 50 | assert isinstance(output, VRCPlayerJoinEntry) 51 | assert isinstance(output.timestamp, Timestamp) 52 | assert isinstance(output.user_name, UserName) 53 | assert isinstance(output.pseudo_user_name, PseudoUserName) 54 | assert output == VRCPlayerJoinEntry( 55 | timestamp=parse_timestamp('2022.03.04 21:50:22'), 56 | user_name=UserName('E.HOBA'), 57 | pseudo_user_name=PseudoUserName("pseudonymized E.HOBA"), 58 | ) 59 | 60 | def test__from_json(self): 61 | entry = VRCPlayerJoinEntry( 62 | timestamp=parse_timestamp('2022.03.04 21:50:22'), 63 | user_name=UserName('E.HOBA'), 64 | pseudo_user_name=PseudoUserName("pseudonymized E.HOBA"), 65 | ) 66 | 67 | assert encode_and_then_decode(entry) == entry 68 | 69 | 70 | class TestVRCPlayerLeftEntry: 71 | def test__regex(self): 72 | output = VRCBuiltinEntryParser.regex_player_left.match( 73 | '2022.03.04 21:50:22 Log - [Behaviour] OnPlayerLeft E.HOBA') 74 | 75 | assert output is not None 76 | 77 | def test__parse(self): 78 | parser = VRCBuiltinEntryParser(new_pseudonymizer(return_value="pseudonymized E.HOBA")) 79 | 80 | input = RawEntry('2022.03.04 21:50:22 Log - [Behaviour] OnPlayerLeft E.HOBA') 81 | output = parser.parse(input) 82 | 83 | assert isinstance(output, VRCPlayerLeftEntry) 84 | assert isinstance(output.timestamp, Timestamp) 85 | assert isinstance(output.user_name, UserName) 86 | assert isinstance(output.pseudo_user_name, PseudoUserName) 87 | assert output == VRCPlayerLeftEntry( 88 | timestamp=parse_timestamp('2022.03.04 21:50:22'), 89 | user_name=UserName('E.HOBA'), 90 | pseudo_user_name=PseudoUserName("pseudonymized E.HOBA"), 91 | ) 92 | 93 | def test__from_json(self): 94 | entry = VRCPlayerLeftEntry( 95 | timestamp=parse_timestamp('2022.03.04 21:50:22'), 96 | user_name=UserName('E.HOBA'), 97 | pseudo_user_name=PseudoUserName("pseudonymized E.HOBA"), 98 | ) 99 | 100 | assert encode_and_then_decode(entry) == entry 101 | -------------------------------------------------------------------------------- /yaiba/log/vrc/entries/tests/test_player_position.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from yaiba.log.test_utils import encode_and_then_decode, new_pseudonymizer 4 | from yaiba.log.types import PseudoUserName, RawEntry, UserName, VRCPlayerId 5 | from yaiba.log.vrc.entries.player_position import VRCYAIBAPlayerPositionEntry, VRCYAIBAPlayerPositionVersionEntry, \ 6 | YAIBAPlayerPositionEntryParser 7 | from yaiba.log.vrc.utils import parse_timestamp 8 | 9 | 10 | class TestYAIBAPlayerPositionVersionEntry: 11 | def test__regex(self): 12 | output = YAIBAPlayerPositionEntryParser.regex_version.match( 13 | "2022.03.04 21:50:19 Log - [Player Position Version]1.0.0") 14 | 15 | assert output is not None 16 | 17 | def test__parse(self): 18 | parser = YAIBAPlayerPositionEntryParser(new_pseudonymizer()) 19 | 20 | output = parser.parse(RawEntry("2022.03.04 21:50:19 Log - [Player Position Version]1.0.0")) 21 | 22 | assert isinstance(output, VRCYAIBAPlayerPositionVersionEntry) 23 | assert output == VRCYAIBAPlayerPositionVersionEntry( 24 | timestamp=parse_timestamp('2022.03.04 21:50:19'), 25 | major=1, minor=0, patch=0) 26 | 27 | def test__from_json(self): 28 | entry = VRCYAIBAPlayerPositionVersionEntry( 29 | timestamp=parse_timestamp('2022.03.04 21:50:19'), 30 | major=1, minor=0, patch=0) 31 | 32 | assert encode_and_then_decode(entry) == entry 33 | 34 | 35 | class TestYAIBAPlayerPositionEntry: 36 | def test__regex_v0(self): 37 | output = YAIBAPlayerPositionEntryParser.regex_entry_v0.match( 38 | '2022.03.04 21:57:53 Log - [Player Position]13,"E.HOBA",-1.622916,1.637101,230.3723,' 39 | '-3.32147,-2.619154,True' 40 | ) 41 | 42 | assert output is not None 43 | 44 | def test__parse_v0(self): 45 | parser = YAIBAPlayerPositionEntryParser(new_pseudonymizer("pseudo E.HOBA")) 46 | 47 | output = parser.parse(RawEntry( 48 | '2022.03.04 21:57:53 Log - [Player Position]13,"E.HOBA",-1.622916,1.637101,230.3723,' 49 | '-3.32147,-2.619154,True' 50 | )) 51 | 52 | assert isinstance(output, VRCYAIBAPlayerPositionEntry) 53 | assert isinstance(output.user_name, UserName) 54 | assert isinstance(output.player_id, VRCPlayerId) 55 | assert isinstance(output.pseudo_user_name, PseudoUserName) 56 | assert output == VRCYAIBAPlayerPositionEntry( 57 | timestamp=parse_timestamp('2022.03.04 21:57:53'), 58 | player_id=VRCPlayerId(13), 59 | user_name=UserName('E.HOBA'), 60 | pseudo_user_name=PseudoUserName("pseudo E.HOBA"), 61 | location_x=pytest.approx(-1.622916), 62 | location_y=None, 63 | location_z=pytest.approx(1.637101), 64 | rotation_1=pytest.approx(230.3723), 65 | rotation_2=pytest.approx(-3.32147), 66 | rotation_3=pytest.approx(-2.619154), 67 | is_vr=True, 68 | ) 69 | 70 | def test__parse_v0__user_name_contains_double_quotes__unescape(self): 71 | parser = YAIBAPlayerPositionEntryParser(new_pseudonymizer("pseudo E.HOBA")) 72 | 73 | output = parser.parse(RawEntry( 74 | '2022.03.04 21:57:53 Log - [Player Position]13,"E"".""HOBA",-1.622916,1.637101,230.3723,' 75 | '-3.32147,-2.619154,True' 76 | )) 77 | 78 | assert isinstance(output, VRCYAIBAPlayerPositionEntry) 79 | assert isinstance(output.user_name, UserName) 80 | assert isinstance(output.player_id, VRCPlayerId) 81 | assert isinstance(output.pseudo_user_name, PseudoUserName) 82 | assert output == VRCYAIBAPlayerPositionEntry( 83 | timestamp=parse_timestamp('2022.03.04 21:57:53'), 84 | player_id=VRCPlayerId(13), 85 | user_name=UserName('E"."HOBA'), 86 | pseudo_user_name=PseudoUserName("pseudo E.HOBA"), 87 | location_x=pytest.approx(-1.622916), 88 | location_y=None, 89 | location_z=pytest.approx(1.637101), 90 | rotation_1=pytest.approx(230.3723), 91 | rotation_2=pytest.approx(-3.32147), 92 | rotation_3=pytest.approx(-2.619154), 93 | is_vr=True, 94 | ) 95 | 96 | def test__regex_v1_0_0(self): 97 | output = YAIBAPlayerPositionEntryParser.regex_entry_v0.match( 98 | '2022.03.04 21:57:53 Log - [Player Position]13,"E.HOBA",-1.622916,1.637101,1.937101,230.3723,' 99 | '-3.32147,-2.619154,True' 100 | ) 101 | 102 | assert output is not None 103 | 104 | def test__parse_v1_0_0(self): 105 | parser = YAIBAPlayerPositionEntryParser(new_pseudonymizer("pseudo E.HOBA")) 106 | 107 | # Feed version entry to switch the version 108 | parser.parse(RawEntry("2022.03.04 21:50:19 Log - [Player Position Version]1.0.0")) 109 | 110 | output = parser.parse(RawEntry( 111 | '2022.03.04 21:57:53 Log - [Player Position]13,"E.HOBA",-1.622916,1.637101,1.937101,230.3723,' 112 | '-3.32147,-2.619154,True' 113 | )) 114 | 115 | assert isinstance(output, VRCYAIBAPlayerPositionEntry) 116 | assert isinstance(output.user_name, UserName) 117 | assert isinstance(output.player_id, VRCPlayerId) 118 | assert isinstance(output.pseudo_user_name, PseudoUserName) 119 | assert output == VRCYAIBAPlayerPositionEntry( 120 | timestamp=parse_timestamp('2022.03.04 21:57:53'), 121 | player_id=VRCPlayerId(13), 122 | user_name=UserName('E.HOBA'), 123 | pseudo_user_name=PseudoUserName("pseudo E.HOBA"), 124 | location_x=pytest.approx(-1.622916), 125 | location_y=pytest.approx(1.637101), 126 | location_z=pytest.approx(1.937101), 127 | rotation_1=pytest.approx(230.3723), 128 | rotation_2=pytest.approx(-3.32147), 129 | rotation_3=pytest.approx(-2.619154), 130 | is_vr=True, 131 | ) 132 | 133 | def test__parse_v1_0_0__user_name_contains_double_quotes__unescape(self): 134 | parser = YAIBAPlayerPositionEntryParser(new_pseudonymizer("pseudo E.HOBA")) 135 | 136 | # Feed version entry to switch the version 137 | parser.parse(RawEntry("2022.03.04 21:50:19 Log - [Player Position Version]1.0.0")) 138 | 139 | output = parser.parse(RawEntry( 140 | '2022.03.04 21:57:53 Log - [Player Position]13,"E"".""HOBA",-1.622916,1.637101,1.937101,230.3723,' 141 | '-3.32147,-2.619154,True' 142 | )) 143 | 144 | assert isinstance(output, VRCYAIBAPlayerPositionEntry) 145 | assert isinstance(output.user_name, UserName) 146 | assert isinstance(output.player_id, VRCPlayerId) 147 | assert isinstance(output.pseudo_user_name, PseudoUserName) 148 | assert output == VRCYAIBAPlayerPositionEntry( 149 | timestamp=parse_timestamp('2022.03.04 21:57:53'), 150 | player_id=VRCPlayerId(13), 151 | user_name=UserName('E"."HOBA'), 152 | pseudo_user_name=PseudoUserName("pseudo E.HOBA"), 153 | location_x=pytest.approx(-1.622916), 154 | location_y=pytest.approx(1.637101), 155 | location_z=pytest.approx(1.937101), 156 | rotation_1=pytest.approx(230.3723), 157 | rotation_2=pytest.approx(-3.32147), 158 | rotation_3=pytest.approx(-2.619154), 159 | is_vr=True, 160 | ) 161 | 162 | def test__from_json(self): 163 | entry = VRCYAIBAPlayerPositionEntry( 164 | timestamp=parse_timestamp('2022.03.04 21:57:53'), 165 | player_id=VRCPlayerId(13), 166 | user_name=UserName('E"."HOBA'), 167 | pseudo_user_name=PseudoUserName("pseudo E.HOBA"), 168 | location_x=-1.622916, 169 | location_y=1.637101, 170 | location_z=1.937101, 171 | rotation_1=230.3723, 172 | rotation_2=-3.32147, 173 | rotation_3=-2.619154, 174 | is_vr=True, 175 | ) 176 | 177 | assert encode_and_then_decode(entry) == entry 178 | assert isinstance(entry.user_name, UserName) 179 | assert isinstance(entry.player_id, VRCPlayerId) 180 | assert isinstance(entry.pseudo_user_name, PseudoUserName) 181 | -------------------------------------------------------------------------------- /yaiba/log/vrc/entries/tests/test_questionnaire.py: -------------------------------------------------------------------------------- 1 | from yaiba.log.test_utils import encode_and_then_decode 2 | from yaiba.log.types import RawEntry 3 | from yaiba.log.vrc.entries.questionnaire import VRCYAIBAQuestionnaireAnswerEntry, YAIBAQuestionnaireAnswerEntryParser 4 | from yaiba.log.vrc.utils import parse_timestamp 5 | 6 | 7 | class TestYAIBAQuestionnaireAnswerEntry: 8 | def test__regex(self): 9 | output = YAIBAQuestionnaireAnswerEntryParser.regex_pattern.match( 10 | '2022.04.06 22:54:50 Log - [Answer]"question_1","answer_1","question_2","answer_2"') 11 | assert output is not None 12 | 13 | def test__parse(self): 14 | parser = YAIBAQuestionnaireAnswerEntryParser() 15 | 16 | output = parser.parse(RawEntry( 17 | '2022.04.06 22:54:50 Log - [Answer]"question_1","answer_1","question_2","answer_2"')) 18 | 19 | assert output == VRCYAIBAQuestionnaireAnswerEntry( 20 | timestamp=parse_timestamp('2022.04.06 22:54:50'), 21 | answer_for_question={ 22 | "question_1": "answer_1", 23 | "question_2": "answer_2", 24 | }, 25 | ) 26 | 27 | def test__from_json(self): 28 | entry = VRCYAIBAQuestionnaireAnswerEntry( 29 | timestamp=parse_timestamp('2022.04.06 22:54:50'), 30 | answer_for_question={ 31 | "question_1": "answer_1", 32 | "question_2": "answer_2", 33 | }, 34 | ) 35 | assert encode_and_then_decode(entry) == entry 36 | -------------------------------------------------------------------------------- /yaiba/log/vrc/entries/tests/test_yodokoro_tag_marker.py: -------------------------------------------------------------------------------- 1 | from yaiba.log.test_utils import encode_and_then_decode 2 | from yaiba.log.types import RawEntry, VRCPlayerId 3 | from yaiba.log.vrc.entries.yodokoro_tag_marker import TAG_NAMES_USED_IN_SCIENCE_ASSEMBLY, VRCYodokoroTagMarkerEntry, \ 4 | YodokoroTagMarkerEntryParser 5 | from yaiba.log.vrc.utils import parse_timestamp 6 | 7 | 8 | class TestYodokoroTagMarkerEntry: 9 | def test__regex(self): 10 | output = YodokoroTagMarkerEntryParser.regex_pattern.match( 11 | "2022.03.05 00:31:57 Log - [Yodo][Dump][0,-1,00000000],[1,9,00000008],[2,10,00000008],[3,94," 12 | "00000000],[4,91,00004000],[5,23,00008013],[6,44,00000000],[7,-1,00000000],[8,-1,00000000],[9,97," 13 | "00000010],[10,28,00010000],[11,30,00000030],[12,98,00000001],[13,32,00038880],[14,-1,00000000],[15,27," 14 | "00000800],[16,37,00008002],[17,47,00000001],[18,96,00000020],[19,8,00000000],[20,78,00001802],[21,-1," 15 | "00000000],[22,81,00010000],[23,86,00000882],[24,82,00000018],[25,-1,00000000],[26,-1,00000000],[27,-1," 16 | "00000000],[28,-1,00000000],[29,-1,00000000],[30,-1,00000000],[31,-1,00000000],[32,-1,00000000],[33,-1," 17 | "00000000],[34,-1,00000000],[35,-1,00000000],[36,-1,00000000],[37,-1,00000000],[38,-1,00000000],[39,-1," 18 | "00000000],[40,-1,00000000],[41,-1,00000000],[42,-1,00000000],[43,-1,00000000],[44,-1,00000000],[45,-1," 19 | "00000000],[46,-1,00000000],[47,-1,00000000],[48,-1,00000000],[49,-1,00000000],[50,-1,00000000],[51,-1," 20 | "00000000],[52,-1,00000000],[53,-1,00000000],[54,-1,00000000],[55,-1,00000000],[56,-1,00000000],[57,-1," 21 | "00000000],[58,-1,00000000],[59,-1,00000000],[60,-1,00000000],[61,-1,00000000],[62,-1,00000000],[63,-1," 22 | "00000000],[64,-1,00000000],[65,-1,00000000],[66,-1,00000000],[67,-1,00000000],[68,-1,00000000],[69,-1," 23 | "00000000],[70,-1,00000000],[71,-1,00000000],[72,-1,00000000],[73,-1,00000000],[74,-1,00000000],[75,-1," 24 | "00000000],[76,-1,00000000],[77,-1,00000000],[78,-1,00000000],[79,-1,00000000],[80,-1,00000000],[81,-1," 25 | "00000000],") 26 | 27 | assert output is not None 28 | 29 | def test__parse(self): 30 | parser = YodokoroTagMarkerEntryParser(tag_names=TAG_NAMES_USED_IN_SCIENCE_ASSEMBLY) 31 | output = parser.parse(RawEntry( 32 | "2022.03.05 00:31:57 Log - [Yodo][Dump][0,-1,00000000],[1,9,00000008],[2,10,00000008],[3,94," 33 | "00000000],[4,91,00004000],[5,23,00008013],[6,44,00000000],[7,-1,00000000],[8,-1,00000000],[9,97," 34 | "00000010],[10,28,00010000],[11,30,00000030],[12,98,00000001],[13,32,00038880],[14,-1,00000000],[15,27," 35 | "00000800],[16,37,00008002],[17,47,00000001],[18,96,00000020],[19,8,00000000],[20,78,00001802],[21,-1," 36 | "00000000],[22,81,00010000],[23,86,00000882],[24,82,00000018],[25,-1,00000000],[26,-1,00000000],[27,-1," 37 | "00000000],[28,-1,00000000],[29,-1,00000000],[30,-1,00000000],[31,-1,00000000],[32,-1,00000000],[33,-1," 38 | "00000000],[34,-1,00000000],[35,-1,00000000],[36,-1,00000000],[37,-1,00000000],[38,-1,00000000],[39,-1," 39 | "00000000],[40,-1,00000000],[41,-1,00000000],[42,-1,00000000],[43,-1,00000000],[44,-1,00000000],[45,-1," 40 | "00000000],[46,-1,00000000],[47,-1,00000000],[48,-1,00000000],[49,-1,00000000],[50,-1,00000000],[51,-1," 41 | "00000000],[52,-1,00000000],[53,-1,00000000],[54,-1,00000000],[55,-1,00000000],[56,-1,00000000],[57,-1," 42 | "00000000],[58,-1,00000000],[59,-1,00000000],[60,-1,00000000],[61,-1,00000000],[62,-1,00000000],[63,-1," 43 | "00000000],[64,-1,00000000],[65,-1,00000000],[66,-1,00000000],[67,-1,00000000],[68,-1,00000000],[69,-1," 44 | "00000000],[70,-1,00000000],[71,-1,00000000],[72,-1,00000000],[73,-1,00000000],[74,-1,00000000],[75,-1," 45 | "00000000],[76,-1,00000000],[77,-1,00000000],[78,-1,00000000],[79,-1,00000000],[80,-1,00000000],[81,-1," 46 | "00000000]," 47 | )) 48 | assert isinstance(output, VRCYodokoroTagMarkerEntry) 49 | assert all(isinstance(k, VRCPlayerId) for k in output.tag_names_for_player_id.keys()) 50 | assert output == VRCYodokoroTagMarkerEntry( 51 | timestamp=parse_timestamp("2022.03.05 00:31:57"), 52 | tag_names_for_player_id={ 53 | 8: [], 54 | 9: ['物理学'], 55 | 10: ['物理学'], 56 | 23: ['機械工学', 57 | '電気系工学', 58 | '化学', 59 | '製造学'], 60 | 27: ['情報学'], 61 | 28: ['文系'], 62 | 30: ['化学', '生物学'], 63 | 32: ['数学', 64 | '情報学', 65 | '製造学', 66 | '文系', 67 | 'その他'], 68 | 37: ['電気系工学', '製造学'], 69 | 44: [], 70 | 47: ['機械工学'], 71 | 78: ['電気系工学', '情報学', '医学'], 72 | 81: ['文系'], 73 | 82: ['物理学', '化学'], 74 | 86: ['電気系工学', '数学', '情報学'], 75 | 91: ['地学'], 76 | 94: [], 77 | 96: ['生物学'], 78 | 97: ['化学'], 79 | 98: ['機械工学'] 80 | }, 81 | ) 82 | 83 | def test__from_json(self): 84 | entry = VRCYodokoroTagMarkerEntry( 85 | timestamp=parse_timestamp("2022.03.05 00:31:57"), 86 | tag_names_for_player_id={ 87 | 8: [], 88 | 9: ['物理学'], 89 | 10: ['物理学'], 90 | 23: ['機械工学', 91 | '電気系工学', 92 | '化学', 93 | '製造学'], 94 | 27: ['情報学'], 95 | 28: ['文系'], 96 | 30: ['化学', '生物学'], 97 | 32: ['数学', 98 | '情報学', 99 | '製造学', 100 | '文系', 101 | 'その他'], 102 | 37: ['電気系工学', '製造学'], 103 | 44: [], 104 | 47: ['機械工学'], 105 | 78: ['電気系工学', '情報学', '医学'], 106 | 81: ['文系'], 107 | 82: ['物理学', '化学'], 108 | 86: ['電気系工学', '数学', '情報学'], 109 | 91: ['地学'], 110 | 94: [], 111 | 96: ['生物学'], 112 | 97: ['化学'], 113 | 98: ['機械工学'] 114 | }, 115 | ) 116 | output = encode_and_then_decode(entry) 117 | assert output == entry 118 | assert all(isinstance(k, VRCPlayerId) for k in output.tag_names_for_player_id.keys()) 119 | -------------------------------------------------------------------------------- /yaiba/log/vrc/entries/yodokoro_tag_marker.py: -------------------------------------------------------------------------------- 1 | """ 2 | Parse log entries outputted by "ヨドコロちゃんのタグマーカー" (https://booth.pm/ja/items/3109716). 3 | 4 | See https://booth.pm/ja/items/3109716 for detailed format of the log entry. 5 | """ 6 | 7 | import itertools 8 | import re 9 | from dataclasses import dataclass 10 | from typing import Any, Dict, List, Optional 11 | 12 | from yaiba.log.session_log import Entry, EntryParser 13 | from yaiba.log.types import RawEntry, Timestamp, VRCPlayerId 14 | from yaiba.log.vrc.utils import VRC_REGEX_LOG_PREFIX, create_timestamp_from_match 15 | 16 | TAG_NAMES_USED_IN_SCIENCE_ASSEMBLY = [ 17 | "機械工学", 18 | "電気系工学", 19 | "物理工学", 20 | "物理学", 21 | "化学", 22 | "生物学", 23 | "天文学", 24 | "数学", 25 | "農学", 26 | "環境学", 27 | "薬学", 28 | "情報学", 29 | "医学", 30 | "土木工学", 31 | "地学", 32 | "製造学", 33 | "文系", 34 | "その他", 35 | "初めて来ました", 36 | "聞きたい", 37 | "話したい", 38 | "議論したい", 39 | ] 40 | 41 | 42 | @dataclass 43 | class VRCYodokoroTagMarkerEntry(Entry): 44 | timestamp: Timestamp 45 | tag_names_for_player_id: Dict[VRCPlayerId, List[str]] 46 | 47 | @classmethod 48 | def type_id(cls): 49 | return 'yodokoro/tag_marker' 50 | 51 | @classmethod 52 | def from_json(cls, value: Dict[str, Any]) -> Entry: 53 | tag_names_for_player_id = value.get('tag_names_for_player_id') 54 | if isinstance(tag_names_for_player_id, dict): 55 | tag_names_for_player_id = { 56 | VRCPlayerId(k): v 57 | for k, v in 58 | tag_names_for_player_id.items() 59 | } 60 | return cls( 61 | timestamp=Timestamp.from_json(value.get('timestamp')), 62 | tag_names_for_player_id=tag_names_for_player_id, 63 | ) 64 | 65 | 66 | class YodokoroTagMarkerEntryParser(EntryParser): 67 | """ 68 | See https://booth.pm/ja/items/3109716 for detailed format of the log entry. 69 | """ 70 | 71 | regex_pattern = re.compile( 72 | VRC_REGEX_LOG_PREFIX + 73 | r'\[Yodo]\[Dump](?P.+)' 74 | ) 75 | 76 | regex_one_tag = re.compile(r'\[(?P\d+),(?P\d+),(?P\d+)],') 77 | 78 | def __init__(self, tag_names: List[str]): 79 | self.tag_names = tag_names 80 | 81 | def parse(self, raw_log: RawEntry) -> Optional[Entry]: 82 | match = self.regex_pattern.match(raw_log) 83 | if match is None: 84 | return None 85 | 86 | timestamp = create_timestamp_from_match(match) 87 | tag_names_for_player_id = self._parse_tags_for_all_players(match.group("tags_for_all_players")) 88 | 89 | return VRCYodokoroTagMarkerEntry( 90 | timestamp=timestamp, 91 | tag_names_for_player_id=tag_names_for_player_id, 92 | ) 93 | 94 | def _parse_tags_for_all_players(self, value: str) -> Dict[VRCPlayerId, List[str]]: 95 | """ 96 | :param value: Ex. "[0,00000030],[1,00000030],[2,00000030],...[81,00000030]," 97 | :return: parsed value. 98 | """ 99 | tags_for_player_id = {} 100 | for match in self.regex_one_tag.finditer(value): 101 | player_id = VRCPlayerId(match.group('player_id')) 102 | if player_id == -1: 103 | continue 104 | tags = self._parse_tags(match.group('tags_hex')) 105 | tags_for_player_id[player_id] = tags 106 | return tags_for_player_id 107 | 108 | def _parse_tags(self, tags_hex: str) -> List[str]: 109 | tags_bin = bin(int(tags_hex, 16))[2:] # to drop "0b" 110 | return [ 111 | tag_name 112 | for tag_name, has_tag in itertools.zip_longest(self.tag_names, reversed(tags_bin)) 113 | if has_tag == '1' 114 | ] 115 | -------------------------------------------------------------------------------- /yaiba/log/vrc/parser.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import io 4 | import re 5 | import typing 6 | from dataclasses import dataclass, field 7 | from typing import List, Optional 8 | 9 | from yaiba.log.pseudonymizer import Pseudonymizer 10 | from yaiba.log.session_log import Entry, EntryParser, SessionLog 11 | from yaiba.log.types import RawEntry 12 | from yaiba.log.vrc.entries.builtin import VRCBuiltinEntryParser 13 | from yaiba.log.vrc.entries.player_position import YAIBAPlayerPositionEntryParser 14 | from yaiba.log.vrc.entries.questionnaire import YAIBAQuestionnaireAnswerEntryParser 15 | from yaiba.log.vrc.entries.yodokoro_tag_marker import TAG_NAMES_USED_IN_SCIENCE_ASSEMBLY, YodokoroTagMarkerEntryParser 16 | 17 | 18 | class VRCLogParser: 19 | """ 20 | Note: Assumes two empty lines between log entries. 21 | """ 22 | 23 | @dataclass 24 | class Config: 25 | """ 26 | Yodokoro tag names. The order must be the same as the one specified in Unity. 27 | 28 | This field is ignored when parser_list is not none. 29 | """ 30 | yodokoro_tag_marker_names: List[str] = field( 31 | default_factory=lambda: TAG_NAMES_USED_IN_SCIENCE_ASSEMBLY, 32 | ) 33 | pseudonymizer: Pseudonymizer = field( 34 | default_factory=lambda: Pseudonymizer.new_random(), 35 | ) 36 | 37 | """ 38 | The list of parsers. Should be ordered by the frequency of its entry for better performance. 39 | 40 | If None, default parsers are used. 41 | """ 42 | parsers: Optional[List[EntryParser]] = field(default=None) 43 | 44 | @classmethod 45 | def default(cls) -> VRCLogParser.Config: 46 | return VRCLogParser.Config() 47 | 48 | def __init__( 49 | self, 50 | config: Optional[Config] = None, 51 | ): 52 | if config is None: 53 | config = VRCLogParser.Config.default() 54 | if config.parsers is not None: 55 | self.parsers = config.parsers 56 | return 57 | else: 58 | self.parsers = self._create_default_parsers(config) 59 | 60 | @classmethod 61 | def _create_default_parsers(cls, config: Config): 62 | return [ 63 | YAIBAPlayerPositionEntryParser(config.pseudonymizer), 64 | YodokoroTagMarkerEntryParser(config.yodokoro_tag_marker_names), 65 | VRCBuiltinEntryParser(config.pseudonymizer), 66 | YAIBAQuestionnaireAnswerEntryParser(), 67 | ] 68 | 69 | def parse(self, value: str) -> SessionLog: 70 | return self.parse_file(io.StringIO(value)) 71 | 72 | def parse_file(self, fp: typing.TextIO) -> SessionLog: 73 | log_entries: List[Entry] = [] 74 | for raw_entry_str in _iter_per_vrc_log_entry(fp): 75 | entry = self._parse_one_entry(RawEntry(raw_entry_str)) 76 | if entry is not None: 77 | log_entries.append(entry) 78 | return SessionLog(log_entries) 79 | 80 | def _parse_one_entry(self, raw_entry: RawEntry) -> Optional[Entry]: 81 | for parser in self.parsers: 82 | entry = parser.parse(raw_entry) 83 | if entry is not None: 84 | return entry 85 | return None 86 | 87 | 88 | def _iter_per_vrc_log_entry(fp: typing.TextIO): 89 | """ 90 | VRC log entries are separated by two empty lines. 91 | 92 | Note: VRC log entry may contain "\r", and the number of empty lines between entries can be more than three. 93 | 94 | :param fp: 95 | :type fp: 96 | :return: 97 | :rtype: 98 | """ 99 | return filter(lambda line: len(line) > 0, _iter_per_timestamp_line(fp)) 100 | 101 | 102 | VRC_TIMESTAMP_REGEX = re.compile(r"\d{4}\.\d{2}\.\d{2} \d{2}:\d{2}:\d{2} .*") 103 | 104 | 105 | def _iter_per_timestamp_line(fp: typing.TextIO): 106 | log_entry_lines = [] 107 | count_empty_lines = 0 108 | while True: 109 | line = fp.readline().rstrip('\n') 110 | if VRC_TIMESTAMP_REGEX.match(line): 111 | if len(log_entry_lines) > 0: 112 | yield '\n'.join(log_entry_lines).strip('\n') 113 | log_entry_lines = [] 114 | log_entry_lines.append(line) 115 | if line == '': 116 | count_empty_lines += 1 117 | if count_empty_lines >= 100: 118 | break 119 | yield '\n'.join(log_entry_lines).strip('\n') 120 | -------------------------------------------------------------------------------- /yaiba/log/vrc/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ScienceAssembly/YAIBA/5e58f33a220a6ade05e44c8a800ba762d7d6620f/yaiba/log/vrc/tests/__init__.py -------------------------------------------------------------------------------- /yaiba/log/vrc/tests/test_parser.py: -------------------------------------------------------------------------------- 1 | import io 2 | 3 | from yaiba.log.vrc.entries.builtin import VRCEnteringRoomEntry, VRCPlayerJoinEntry, VRCPlayerLeftEntry 4 | from yaiba.log.vrc.entries.player_position import VRCYAIBAPlayerPositionEntry, VRCYAIBAPlayerPositionVersionEntry 5 | from yaiba.log.vrc.parser import VRCLogParser, _iter_per_vrc_log_entry 6 | 7 | 8 | class TestVRCLogParser(): 9 | def test__normal(self): 10 | input_data = '\n'.join([ 11 | # entity 1 12 | '2022.03.04 21:50:19 Log - [Behaviour] Entering Room: FirstRoom', 13 | # entity 2 14 | '2022.03.04 21:50:22 Log - [Behaviour] OnPlayerJoined E.HOBA', 15 | # entity to be ignored 16 | '2022.03.04 21:50:22 Log - [Behaviour] Initialized PlayerAPI "E.HOBA" is remote', 17 | # entity 3 18 | "2022.03.04 21:50:23 Log - [Player Position Version]1.0.0", 19 | # entity 4 20 | '2022.03.04 21:50:31 Log - [Player Position]13,"E.HOBA",-6.329126,-0.3207326,-0.3207326,272.0943,' 21 | '-0.009579957,-0.01711023,True', 22 | # entity 5 23 | '2022.03.04 21:50:41 Log - [Player Position]13,"E.HOBA",-6.336999,-0.3212091,-0.3212091,291.8254,' 24 | '-0.01143897,0.03140759,True', 25 | # entity to be ignored 26 | '2022.03.04 21:50:25 Log - Measure Human Avatar Avatar', 27 | '2022.03.04 21:50:25 Warning - Measure Human Avatar Avatar', 28 | '2022.03.04 21:50:25 Debug - Measure Human Avatar Avatar:', 29 | '{', 30 | ' "Avatar": "E.HOBA",', 31 | ' "AvatarId": 13,', 32 | '}', 33 | # entity 6 34 | '2022.03.05 03:13:50 Log - [Behaviour] OnPlayerLeft E.HOBA', 35 | # entity 7 36 | '2022.03.04 21:50:19 Log - [Behaviour] Entering Room: SecondRoom', 37 | ]) 38 | parser = VRCLogParser() 39 | 40 | session_log = parser.parse(input_data) 41 | 42 | assert isinstance(session_log.log_entries[0], VRCEnteringRoomEntry) 43 | assert isinstance(session_log.log_entries[1], VRCPlayerJoinEntry) 44 | assert isinstance(session_log.log_entries[2], VRCYAIBAPlayerPositionVersionEntry) 45 | assert isinstance(session_log.log_entries[3], VRCYAIBAPlayerPositionEntry) 46 | assert isinstance(session_log.log_entries[4], VRCYAIBAPlayerPositionEntry) 47 | assert isinstance(session_log.log_entries[5], VRCPlayerLeftEntry) 48 | assert isinstance(session_log.log_entries[6], VRCEnteringRoomEntry) 49 | assert len(session_log.log_entries) == 7 50 | 51 | def test__splitter(self): 52 | input_data = '\n'.join([ 53 | # entity 1 54 | '2022.03.04 21:50:19 Log - [Behaviour] Entering Room: FirstRoom', 55 | # entity 2 56 | '2022.03.04 21:50:22 Log - [Behaviour] OnPlayerJoined E.HOBA', 57 | # entity 3 58 | '2022.03.04 21:50:22 Log - [Behaviour] Initialized PlayerAPI "E.HOBA" is remote', 59 | # entity 4 60 | "2022.03.04 21:50:23 Log - [Player Position Version]1.0.0", 61 | # entity 5 62 | '2022.03.04 21:50:31 Log - [Player Position]13,"E.HOBA",-6.329126,-0.3207326,-0.3207326,272.0943,' 63 | '-0.009579957,-0.01711023,True', 64 | # entity 6 65 | '2022.03.04 21:50:41 Log - [Player Position]13,"E.HOBA",-6.336999,-0.3212091,-0.3212091,291.8254,' 66 | '-0.01143897,0.03140759,True', 67 | # entity 7 68 | '2022.03.04 21:50:25 Log - Measure Human Avatar Avatar', 69 | # entity 8 70 | '2022.03.04 21:50:25 Warning - Measure Human Avatar Avatar', 71 | # entity 9 72 | '2022.03.04 21:50:25 Debug - Measure Human Avatar Avatar:', 73 | '{', 74 | ' "Avatar": "E.HOBA",', 75 | ' "AvatarId": 13,', 76 | '}', 77 | # entity 10 78 | '2022.03.05 03:13:50 Log - [Behaviour] OnPlayerLeft E.HOBA', 79 | # entity 11 80 | '2022.03.04 21:50:19 Log - [Behaviour] Entering Room: SecondRoom', 81 | ]) 82 | 83 | values = list(_iter_per_vrc_log_entry(io.StringIO(input_data))) 84 | assert len(values) == 11 85 | 86 | 87 | def test__past_format__double_empty_line(self): 88 | input_data = '\n\n\n'.join([ 89 | # entity 1 90 | '2022.03.04 21:50:19 Log - [Behaviour] Entering Room: FirstRoom', 91 | # entity 2 92 | '2022.03.04 21:50:22 Log - [Behaviour] OnPlayerJoined E.HOBA', 93 | # entity to be ignored 94 | '2022.03.04 21:50:22 Log - [Behaviour] Initialized PlayerAPI "E.HOBA" is remote', 95 | # entity 3 96 | "2022.03.04 21:50:23 Log - [Player Position Version]1.0.0", 97 | # entity 4 98 | '2022.03.04 21:50:31 Log - [Player Position]13,"E.HOBA",-6.329126,-0.3207326,-0.3207326,272.0943,' 99 | '-0.009579957,-0.01711023,True', 100 | # entity 5 101 | '2022.03.04 21:50:41 Log - [Player Position]13,"E.HOBA",-6.336999,-0.3212091,-0.3212091,291.8254,' 102 | '-0.01143897,0.03140759,True', 103 | # entity to be ignored 104 | '2022.03.04 21:50:25 Log - Measure Human Avatar Avatar', 105 | # entity 6 106 | '2022.03.05 03:13:50 Log - [Behaviour] OnPlayerLeft E.HOBA', 107 | # entity 7 108 | '2022.03.04 21:50:19 Log - [Behaviour] Entering Room: SecondRoom', 109 | ]) 110 | parser = VRCLogParser() 111 | 112 | session_log = parser.parse(input_data) 113 | 114 | assert isinstance(session_log.log_entries[0], VRCEnteringRoomEntry) 115 | assert isinstance(session_log.log_entries[1], VRCPlayerJoinEntry) 116 | assert isinstance(session_log.log_entries[2], VRCYAIBAPlayerPositionVersionEntry) 117 | assert isinstance(session_log.log_entries[3], VRCYAIBAPlayerPositionEntry) 118 | assert isinstance(session_log.log_entries[4], VRCYAIBAPlayerPositionEntry) 119 | assert isinstance(session_log.log_entries[5], VRCPlayerLeftEntry) 120 | assert isinstance(session_log.log_entries[6], VRCEnteringRoomEntry) 121 | assert len(session_log.log_entries) == 7 122 | -------------------------------------------------------------------------------- /yaiba/log/vrc/utils.py: -------------------------------------------------------------------------------- 1 | import re 2 | from typing import Union 3 | 4 | from yaiba.constants import DEFAULT_TIMEZONE 5 | from yaiba.log.types import RawEntry, Timestamp 6 | 7 | """ 8 | Matches 9 | "2022.03.04 21:50:19 Log - " 10 | 11 | "2022.03.04 21:50:19 Log - [Behaviour] EnteringRoom: Some Room" 12 | """ 13 | REGEX_LOG_TIMESTAMP = re.compile( 14 | r'^(?P\d{4})\.(?P\d{2}).(?P\d{2})\s+' 15 | r'(?P\d{2}):(?P\d{2}):(?P\d{2})' 16 | ) 17 | 18 | 19 | def parse_timestamp(log_entry: Union[RawEntry, str]) -> Timestamp: 20 | match = REGEX_LOG_TIMESTAMP.match(log_entry) 21 | assert match is not None, f'could not find timestamp, {log_entry:?}' 22 | return create_timestamp_from_match(match) 23 | 24 | 25 | def create_timestamp_from_match(match: re.Match) -> Timestamp: 26 | return Timestamp( 27 | year=int(match.group('year')), 28 | month=int(match.group('month')), 29 | day=int(match.group('day')), 30 | hour=int(match.group('hour')), 31 | minute=int(match.group('minute')), 32 | second=int(match.group('second')), 33 | tzinfo=DEFAULT_TIMEZONE, 34 | ) 35 | 36 | 37 | VRC_REGEX_LOG_PREFIX = ( 38 | # Timestamp 39 | r'^(?P\d{4})\.(?P\d{2}).(?P\d{2})\s+' 40 | r'(?P\d{2}):(?P\d{2}):(?P\d{2})' 41 | # Splitter 42 | r'\s*' 43 | # Log level 44 | r'(?P[A-z]+)' 45 | # Splitter 46 | r'\s*-\s+' 47 | ) 48 | -------------------------------------------------------------------------------- /yaiba/visualization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ScienceAssembly/YAIBA/5e58f33a220a6ade05e44c8a800ba762d7d6620f/yaiba/visualization/__init__.py -------------------------------------------------------------------------------- /yaiba/visualization/vrc/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass 4 | from datetime import timedelta 5 | from typing import List, Optional, Tuple 6 | 7 | import pandas 8 | import pandas as pd 9 | import plotly.graph_objects as go 10 | from ipywidgets import Dropdown, HBox, IntSlider, Label, Play, VBox, jslink 11 | 12 | from yaiba import SessionLog 13 | from yaiba.log.types import PseudoUserName, Timestamp 14 | from yaiba.log.vrc import VRCEnteringRoomEntry, VRCPlayerLeftEntry, VRCYAIBAPlayerPositionEntry 15 | 16 | 17 | @dataclass 18 | class PlayerLocation: 19 | timestamp: Timestamp 20 | pseudo_user_name: PseudoUserName 21 | location_x: Optional[float] 22 | location_z: Optional[float] 23 | 24 | 25 | class VRCPlayerLocationPlotter(): 26 | timestamp_start: Timestamp 27 | timestamp_end: Timestamp 28 | 29 | # timestamp: pseudo_user_name (str), user_id (int), location_x (float), location_z (float) 30 | df: Optional[pd.DataFrame] 31 | world_boundary: Optional[WorldBoundary] 32 | 33 | def __init__(self, session_log: SessionLog): 34 | self.session_log = session_log 35 | 36 | self.room_names = self._get_room_names(session_log) 37 | default_room_name = self.room_names[0] 38 | self.log_idx_slice = self._get_log_entry_slice(session_log, default_room_name) 39 | 40 | # UI components 41 | self.room_dropdown = Dropdown( 42 | options=self.room_names, 43 | value=default_room_name, 44 | ) 45 | self.play = Play(step=10, interval=500) 46 | self.slider = IntSlider() 47 | 48 | self.label_current_timestamp = Label("") 49 | 50 | self.controllers = VBox( 51 | [ 52 | HBox([ 53 | Label("Entering room:"), 54 | self.room_dropdown, 55 | ]), 56 | HBox([ 57 | self.play, 58 | self.slider, 59 | Label("(sec)"), 60 | self.label_current_timestamp 61 | ]) 62 | ] 63 | ) 64 | 65 | self.scatter = go.Scatter( 66 | mode='markers', 67 | ) 68 | self.figure_widget = go.FigureWidget( 69 | data=[self.scatter], 70 | layout=go.Layout( 71 | yaxis=dict( 72 | scaleanchor='x' 73 | ), 74 | height=640, 75 | width=640, 76 | barmode='overlay', 77 | 78 | ) 79 | ) 80 | 81 | jslink((self.play, 'value'), (self.slider, 'value')) 82 | jslink((self.play, 'min'), (self.slider, 'min')) 83 | jslink((self.play, 'max'), (self.slider, 'max')) 84 | self.slider.observe(self._on_changed_slider_value, names='value') 85 | self.room_dropdown.observe(self._on_changed_room_dropdown_value, names='value') 86 | 87 | self.container = VBox([ 88 | self.controllers, 89 | self.figure_widget, 90 | ]) 91 | 92 | self.change_entering_room(default_room_name) 93 | 94 | def plot(self): 95 | return self.container 96 | 97 | def change_entering_room(self, formatted_entering_room: str): 98 | 99 | self.log_idx_slice = self._get_log_entry_slice(self.session_log, formatted_entering_room) 100 | 101 | self.timestamp_start: Timestamp = self.session_log.log_entries[self.log_idx_slice][0].timestamp 102 | self.timestamp_end: Timestamp = self.session_log.log_entries[self.log_idx_slice][-1].timestamp 103 | 104 | duration_sec = int(self.timestamp_end.timestamp()) - int(self.timestamp_start.timestamp()) 105 | 106 | self.play.min = 0 107 | self.play.max = duration_sec 108 | 109 | self.df = self._gen_dataframe(self.session_log, self.log_idx_slice) 110 | self.world_boundary = self._get_world_boundary(self.df) 111 | 112 | if self.world_boundary is not None: 113 | with self.figure_widget.batch_update(): 114 | self.figure_widget.update_layout(xaxis_range=[self.world_boundary.x_min, self.world_boundary.x_max]) 115 | self.figure_widget.update_layout(yaxis_range=[self.world_boundary.z_min, self.world_boundary.z_max]) 116 | 117 | def _on_changed_slider_value(self, values): 118 | sec_diff = values["new"] 119 | timestamp = self.timestamp_start + timedelta(seconds=sec_diff) 120 | self.label_current_timestamp.value = timestamp.isoformat() 121 | 122 | if self.df is None or self.world_boundary is None: 123 | return 124 | 125 | with self.figure_widget.batch_update(): 126 | data_x, data_z, data_username, data_user_id = self._calc_scatter_data(timestamp) 127 | self.figure_widget.data[0].x = data_x 128 | self.figure_widget.data[0].y = data_z 129 | self.figure_widget.data[0].text = data_username 130 | self.figure_widget.data[0].marker.color = data_user_id 131 | 132 | def _calc_scatter_data(self, timestamp: Timestamp) -> Tuple[List[float], List[float], List[str], List[int]]: 133 | one_shot_data = self.df.loc[:timestamp].groupby("pseudo_user_name").tail(1).dropna() 134 | x = one_shot_data["location_x"].tolist() 135 | z = one_shot_data["location_z"].tolist() 136 | pseudo_user_name = one_shot_data["pseudo_user_name"].tolist() 137 | user_id = one_shot_data["user_id"].tolist() 138 | return (x, z, pseudo_user_name, user_id) 139 | 140 | def _on_changed_room_dropdown_value(self, value): 141 | new_room_name: str = value.get("new") 142 | self.change_entering_room(new_room_name) 143 | 144 | @classmethod 145 | def _get_room_names(cls, session_log: SessionLog) -> List[str]: 146 | return [ 147 | cls._format_room_dropdown_item(e) 148 | for e in session_log.log_entries 149 | if isinstance(e, VRCEnteringRoomEntry) 150 | ] 151 | 152 | @classmethod 153 | def _format_room_dropdown_item(cls, entering: VRCEnteringRoomEntry): 154 | return f"{entering.timestamp.isoformat()} : {entering.room_name}" 155 | 156 | @classmethod 157 | def _get_log_entry_slice(cls, session_log: SessionLog, formatted_dropdown_item: str) -> slice: 158 | all_enter_indexes = [ 159 | (idx, e) 160 | for idx, e in enumerate(session_log.log_entries) 161 | if isinstance(e, VRCEnteringRoomEntry) 162 | ] 163 | maybe_enter_index = [ 164 | enter_idx 165 | for enter_idx, (log_idx, e) in enumerate(all_enter_indexes) 166 | if cls._format_room_dropdown_item(e) == formatted_dropdown_item 167 | ] 168 | assert len(maybe_enter_index) == 1, f"{formatted_dropdown_item} should only exist one" 169 | 170 | enter_idx = maybe_enter_index[0] 171 | 172 | log_entry_index_start: int = all_enter_indexes[enter_idx][0] 173 | log_entry_index_end: int = len(session_log.log_entries) \ 174 | if enter_idx + 1 == len(all_enter_indexes) else \ 175 | all_enter_indexes[enter_idx + 1][0] 176 | 177 | return slice( 178 | log_entry_index_start, 179 | log_entry_index_end, 180 | ) 181 | 182 | @classmethod 183 | def _get_world_boundary(cls, df: pandas.DataFrame) -> Optional[WorldBoundary]: 184 | if df is None: 185 | return None 186 | return WorldBoundary( 187 | x_min=df["location_x"].min(), 188 | x_max=df["location_x"].max(), 189 | z_min=df["location_z"].min(), 190 | z_max=df["location_z"].max(), 191 | ) 192 | 193 | @classmethod 194 | def _gen_dataframe(cls, session_log: SessionLog, idx_slice: slice) -> Optional[pd.DataFrame]: 195 | raw_entries = session_log.log_entries[idx_slice] 196 | 197 | has_location_entry = False 198 | 199 | location_entries: List[PlayerLocation] = [] 200 | for entry in raw_entries: 201 | if isinstance(entry, VRCYAIBAPlayerPositionEntry): 202 | has_location_entry = True 203 | location_entries.append( 204 | PlayerLocation( 205 | timestamp=entry.timestamp, 206 | pseudo_user_name=entry.pseudo_user_name, 207 | location_x=entry.location_x, 208 | location_z=entry.location_z, 209 | ) 210 | ) 211 | continue 212 | if isinstance(entry, VRCPlayerLeftEntry): 213 | location_entries.append( 214 | PlayerLocation( 215 | timestamp=entry.timestamp, 216 | pseudo_user_name=entry.pseudo_user_name, 217 | location_x=None, 218 | location_z=None, 219 | ) 220 | ) 221 | continue 222 | 223 | if not has_location_entry: 224 | # No data to be rendered. 225 | return None 226 | 227 | df = pd.DataFrame(location_entries) 228 | df.set_index(["timestamp"], inplace=True) 229 | 230 | ids = sorted(set(df["pseudo_user_name"].tolist())) 231 | 232 | df["user_id"] = df["pseudo_user_name"].apply(lambda t: ids.index(t)) 233 | 234 | return df 235 | 236 | 237 | @dataclass 238 | class WorldBoundary: 239 | x_min: float 240 | x_max: float 241 | z_min: float 242 | z_max: float 243 | -------------------------------------------------------------------------------- /yaiba_colab/__init__.py: -------------------------------------------------------------------------------- 1 | def check_running_in_colab(): 2 | try: 3 | import google.colab 4 | except: 5 | assert False, "This library must be run in Google Colab" 6 | 7 | 8 | check_running_in_colab() 9 | from yaiba_colab.storage import GoogleDriveFolder, GoogleDriveLogFile 10 | -------------------------------------------------------------------------------- /yaiba_colab/storage.py: -------------------------------------------------------------------------------- 1 | import io 2 | from dataclasses import dataclass 3 | from typing import List 4 | 5 | import yaiba 6 | from yaiba import JsonEncoder 7 | from yaiba.log import JsonDecoder, SessionLog 8 | 9 | 10 | def _get_drive_in_colab(): 11 | """ 12 | Only work in colab. 13 | """ 14 | from pydrive.auth import GoogleAuth 15 | from pydrive.drive import GoogleDrive 16 | from google.colab import auth 17 | from oauth2client.client import GoogleCredentials 18 | auth.authenticate_user() 19 | gauth = GoogleAuth() 20 | gauth.credentials = GoogleCredentials.get_application_default() 21 | drive = GoogleDrive(gauth) 22 | return drive 23 | 24 | 25 | def load_session_log(file): 26 | decider = JsonDecoder() 27 | return decider.decode(file.GetContentString()) 28 | 29 | 30 | def get_log_by_gdrive_id(gdrive_id: str) -> SessionLog: 31 | drive = _get_drive_in_colab() 32 | file = drive.CreateFile({"id": gdrive_id}) 33 | return load_session_log(file) 34 | 35 | 36 | @dataclass 37 | class GoogleDriveFolder: 38 | """ 39 | drive_id: Find share link in Google Drive, and drive_id contains in the URL 40 | as follows: https://drive.google.com/drive/folders/${drive_id}?${not_related_parameters} 41 | """ 42 | 43 | def __init__(self, drive_id: str): 44 | self.drive_id = drive_id 45 | 46 | def get_log_list(self) -> List['GoogleDriveLogFile']: 47 | files = self._get_log_files() 48 | return [ 49 | GoogleDriveLogFile.from_gfile(file) 50 | for file in files 51 | ] 52 | 53 | def _get_log_files(self): 54 | drive = _get_drive_in_colab() 55 | return sum(list(drive.ListFile({ 56 | "q": f"'{self.drive_id}' in parents and trashed = false", 57 | "includeItemsFromAllDrives": True, 58 | "supportsAllDrives": True, 59 | })), []) 60 | 61 | def get_log_by_title(self, title: str) -> List[SessionLog]: 62 | drive = _get_drive_in_colab() 63 | files = sum(list(drive.ListFile({ 64 | "q": f"title = '{title}' and '{self.drive_id}' in parents and trashed = false", 65 | "includeItemsFromAllDrives": True, 66 | "supportsAllDrives": True 67 | })), []) 68 | if len(files) == 0: 69 | assert False, f'not found: {files}' 70 | session_logs = [] 71 | for file in files: 72 | session_logs.append( 73 | load_session_log(drive.CreateFile({"id": file['id']})) 74 | ) 75 | return session_logs 76 | 77 | def upload_log(self, log: SessionLog, title: str, options: JsonEncoder.Options = None): 78 | drive = _get_drive_in_colab() 79 | preprocessed_log_file = drive.CreateFile({"parents": [{"id": self.drive_id}]}) 80 | fp = io.StringIO() 81 | yaiba.save_session_log(log, fp, options) 82 | preprocessed_log_file.SetContentString(fp.getvalue()) 83 | preprocessed_log_file['title'] = title 84 | preprocessed_log_file.Upload() 85 | return preprocessed_log_file 86 | 87 | 88 | @dataclass 89 | class GoogleDriveLogFile(): 90 | title: str 91 | id: str 92 | created_time: str 93 | modified_time: str 94 | owner_display_name: str 95 | 96 | @classmethod 97 | def from_gfile(cls, gfile): 98 | # See: https://developers.google.com/drive/api/v3/reference/files 99 | get_owner_display_name = lambda owners: owners[0].get('displayName', 'no name') if len(owners) > 0 else '' 100 | return cls( 101 | title=gfile['title'], 102 | id=gfile['id'], 103 | created_time=gfile['createdDate'], 104 | modified_time=gfile['modifiedDate'], 105 | owner_display_name=get_owner_display_name(gfile['owners']), 106 | ) 107 | 108 | def get_session_log(self) -> SessionLog: 109 | return get_log_by_gdrive_id(self.id) 110 | -------------------------------------------------------------------------------- /yaiba_scienceassembly/README.md: -------------------------------------------------------------------------------- 1 | # This directory is only for Science Assembly (理系集会). Not expected to be used by others. -------------------------------------------------------------------------------- /yaiba_scienceassembly/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass 4 | from typing import Any, Dict 5 | 6 | from yaiba.log.types import FromJson 7 | 8 | 9 | @dataclass 10 | class ScienceAssemblyMetadata(FromJson): 11 | """ 12 | For `SessionLog.metadata` 13 | """ 14 | event_type: str # Ex. "理系集会" 15 | event_date: str # Ex. "2022-04-29" 16 | event_description: str # Ex. "水の汚れを測る方法" 17 | event_instance: str # "main", "sub", "petit" 18 | 19 | @classmethod 20 | def from_json(cls, value: Dict[str, Any]) -> ScienceAssemblyMetadata: 21 | return cls(**value) 22 | --------------------------------------------------------------------------------