├── .gitignore ├── README.md ├── UNLICENSE ├── klbvfs.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | /env 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | this is a proof-of-concept implementation of klab's encrypted sqlite3 vfs. 2 | it can be used to query encrypted databases in your 3 | `/data/data/com.klab.lovelive.allstars/files/files` directory 4 | 5 | it assumes your directory structure is the same as it would be on your 6 | android device to extract your master key from `shared_prefs` . so you 7 | must dump your /data/data directory as is, or run this directly on your 8 | phone 9 | 10 | # usage 11 | you need python3 and pip installed 12 | 13 | if you have python venv: 14 | 15 | ```sh 16 | python3 -m venv env 17 | source env/bin/activate 18 | ``` 19 | 20 | then install dependencies 21 | 22 | ```c 23 | python3 -m pip install -r requirements.txt 24 | ``` 25 | 26 | now you can use it 27 | 28 | ``` 29 | ./klbvfs.py query masterdata.db_* "select sql from sqlite_master;" 30 | ./klbvfs decrypt *.db_*.db 31 | ./klbvfs.py --help 32 | ./klbvfs.py dump [--types [[...]]] [directories [directories ...]] 33 | ``` 34 | 35 | this also registers a python codec for klbvfs which can be used to decrypt 36 | like so 37 | 38 | ```python 39 | key = sqlite_key('encrypted.db') 40 | src = codecs.open('encrypted.db', mode='rb', encoding='klbvfs', errors=key) 41 | dst = open('decrypted.db', 'wb+') 42 | shutil.copyfileobj(src, dst) 43 | ``` 44 | 45 | # future development 46 | I'd like to actually make it dump all the `pkg*` files with correct names 47 | and directory structure. the mapping between virtual paths and pkg dirs 48 | is stored in these db's among other stuff 49 | -------------------------------------------------------------------------------- /UNLICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /klbvfs.py: -------------------------------------------------------------------------------- 1 | #!/bin/env python3 2 | 3 | # This is free and unencumbered software released into the public domain. 4 | # 5 | # Anyone is free to copy, modify, publish, use, compile, sell, or 6 | # distribute this software, either in source code form or as a compiled 7 | # binary, for any purpose, commercial or non-commercial, and by any 8 | # means. 9 | 10 | import apsw 11 | import os.path 12 | import sys 13 | from bs4 import BeautifulSoup 14 | import urllib.parse 15 | import base64 16 | import hmac 17 | import hashlib 18 | import struct 19 | import codecs 20 | import shutil 21 | import re 22 | import multiprocessing as mp 23 | import magic 24 | import mimetypes 25 | import html 26 | 27 | 28 | def i8(x): 29 | return x & 0xFF 30 | 31 | 32 | def i32(x): 33 | return x & 0xFFFFFFFF 34 | 35 | 36 | def hmac_sha1(key, s): 37 | hmacsha1 = hmac.new(key, digestmod=hashlib.sha1) 38 | hmacsha1.update(s) 39 | return hmacsha1.digest() 40 | 41 | 42 | def klbvfs_transform_byte(byte, key): 43 | byte ^= i8(key[0] >> 24) ^ i8(key[1] >> 24) ^ i8(key[2] >> 24) 44 | key[0] = i32(i32(key[0] * 0x343fd) + 0x269ec3) 45 | key[1] = i32(i32(key[1] * 0x343fd) + 0x269ec3) 46 | key[2] = i32(i32(key[2] * 0x343fd) + 0x269ec3) 47 | return byte 48 | 49 | 50 | # this is used for random seeks through encrypted files 51 | # it computes the prng state in log(offset) instead of offset cycles 52 | # https://www.nayuki.io/page/fast-skipping-in-a-linear-congruential-generator 53 | def prng_seek(k, offset, mul, add, mod): 54 | mul1 = mul - 1 55 | modmul = mul1 * mod 56 | y = (pow(mul, offset, modmul) - 1) // mul1 * add 57 | z = pow(mul, offset, mod) * k 58 | return (y + z) % mod 59 | 60 | 61 | def klbvfs_transform(data, key): 62 | return bytes([klbvfs_transform_byte(x, key) for x in data]), len(data) 63 | 64 | 65 | class KLBVFS(apsw.VFS): 66 | def __init__(self, vfsname='klb_vfs', basevfs=''): 67 | self.vfsname = vfsname 68 | self.basevfs = basevfs 69 | apsw.VFS.__init__(self, self.vfsname, self.basevfs) 70 | 71 | def xOpen(self, name, flags): 72 | return KLBVFSFile(self.basevfs, name, flags) 73 | 74 | def xAccess(self, pathname, flags): 75 | actual_path = pathname.split(' ', 2)[1] 76 | return super(KLBVFS, self).xAccess(actual_path, flags) 77 | 78 | def xFullPathname(self, name): 79 | split = name.split(' ', 2) 80 | fullpath = super(KLBVFS, self).xFullPathname(split[1]) 81 | return split[0] + ' ' + fullpath 82 | 83 | 84 | class KLBVFSFile(apsw.VFSFile): 85 | def __init__(self, inheritfromvfsname, filename, flags): 86 | split = filename.filename().split(' ', 2) 87 | keysplit = split[0].split('.') 88 | self.key = [int(x) for x in keysplit] 89 | apsw.VFSFile.__init__(self, inheritfromvfsname, split[1], flags) 90 | 91 | def xRead(self, amount, offset): 92 | encrypted = super(KLBVFSFile, self).xRead(amount, offset) 93 | k = [prng_seek(k, offset, 0x343fd, 0x269ec3, 2**32) for k in self.key] 94 | res, _ = klbvfs_transform(bytearray(encrypted), k) 95 | return res 96 | 97 | 98 | def sqlite_key(dbfile): 99 | abspath = os.path.abspath(dbfile) 100 | base = os.path.dirname(abspath) 101 | base = os.path.dirname(base) 102 | base = os.path.dirname(base) 103 | pkgname = os.path.basename(base) 104 | prefs_path = 'shared_prefs/' + pkgname + '.v2.playerprefs.xml' 105 | prefs = os.path.join(base, prefs_path) 106 | xml = open(prefs, 'r').read() 107 | soup = BeautifulSoup(xml, 'lxml-xml') 108 | sq = urllib.parse.unquote(soup.find('string', {'name': 'SQ'}).getText()) 109 | sq = base64.b64decode(sq) 110 | basename = os.path.basename(dbfile) 111 | sha1 = hmac_sha1(key=sq, s=basename.encode('utf-8')) 112 | return list(struct.unpack('>III', sha1[:12])) 113 | 114 | 115 | class KLBVFSCodec(codecs.Codec): 116 | def encode(self, data, key): 117 | return klbvfs_transform(data, key) 118 | 119 | def decode(self, data, key): 120 | return klbvfs_transform(data, key) 121 | 122 | 123 | class KLBVFSStreamReader(KLBVFSCodec, codecs.StreamReader): 124 | charbuffertype = bytes 125 | 126 | 127 | class KLBVFSStreamWriter(KLBVFSCodec, codecs.StreamWriter): 128 | charbuffertype = bytes 129 | 130 | 131 | def klbvfs_decoder(encoding_name): 132 | t = klbvfs_transform 133 | return codecs.CodecInfo(name='klbvfs', encode=t, decode=t, 134 | streamreader=KLBVFSStreamReader, 135 | streamwriter=KLBVFSStreamWriter, 136 | _is_text_encoding=False) 137 | 138 | 139 | codecs.register(klbvfs_decoder) 140 | 141 | 142 | def vpath(path, key): 143 | return '.'.join([str(i32(x)) for x in key]) + ' ' + path 144 | 145 | 146 | def klb_sqlite(dbfile): 147 | vfs = KLBVFS() 148 | key = sqlite_key(dbfile) 149 | v = vpath(path=dbfile, key=key) 150 | return apsw.Connection(v, flags=apsw.SQLITE_OPEN_READONLY, vfs='klb_vfs') 151 | 152 | 153 | def find_db(name, directory): 154 | pattern = re.compile(name + '.db_[a-z0-9]+.db') 155 | matches = [f for f in os.listdir(directory) if pattern.match(f)] 156 | if len(matches) >= 1: 157 | return os.path.join(directory, matches[0]) 158 | else: 159 | return None 160 | 161 | 162 | def dictionary_get(key, directory): 163 | spl = key.split('.', 2) 164 | if len(spl) < 2: 165 | return key 166 | dbpath = find_db('dictionary_ja_' + spl[0], directory) 167 | if dbpath is None: 168 | dbpath = find_db('dictionary_ko_' + spl[0], directory) 169 | if dbpath is None: 170 | return key 171 | db = klb_sqlite(dbpath).cursor() 172 | sel = 'select message from m_dictionary where id = ?' 173 | rows = db.execute(sel, (spl[1],)) 174 | res = rows.fetchone() 175 | if res is None: 176 | return key 177 | return html.unescape(res[0]) 178 | 179 | 180 | def do_query(args): 181 | for row in klb_sqlite(args.dbfile).cursor().execute(args.sql): 182 | if len(row) == 1: 183 | print(row[0]) 184 | else: 185 | print(row) 186 | 187 | 188 | def decrypt_db(source): 189 | dstpath = '_'.join(source.split('_')[:-1]) 190 | key = sqlite_key(source) 191 | src = codecs.open(source, mode='rb', encoding='klbvfs', errors=key) 192 | dst = open(dstpath, 'wb+') 193 | print('%s -> %s' % (source, dstpath)) 194 | shutil.copyfileobj(src, dst) 195 | src.close() 196 | dst.close() 197 | return dstpath 198 | 199 | 200 | def do_decrypt(args): 201 | for source in args.files: 202 | decrypt_db(source) 203 | 204 | 205 | def decrypt_worker(source, table, pack_name, head, size, key1, key2): 206 | dstdir = os.path.join(source, table) 207 | fpath = os.path.join(dstdir, "%s_%d" % (pack_name, head)) 208 | pkgpath = os.path.join(source, "pkg" + pack_name[:1], pack_name) 209 | key = [key1, key2, 0x3039] 210 | pkg = codecs.open(pkgpath, mode='rb', encoding='klbvfs', errors=key) 211 | pkg.seek(head) 212 | buf = pkg.read(1024) 213 | mime = magic.from_buffer(buf, mime=True) 214 | ext = mimetypes.guess_extension(mime) 215 | if mime == 'application/octet-stream': 216 | if buf.startswith(b'UnityFS'): 217 | mime = "application/unityfs" 218 | ext = ".unity3d" 219 | elif table == 'adv_script': 220 | # proprietary script format, TODO reverse engineer it 221 | mime = "application/advscript" 222 | ext = ".advscript" 223 | key[0] = key1 # hack: reset rng state, codec has reference to this array 224 | key[1] = key2 225 | key[2] = 0x3039 226 | pkg.seek(head) 227 | print("[%s] decrypting to %s (%s)" % (fpath, ext, mime)) 228 | with open(fpath + ext, 'wb+') as dst: 229 | shutil.copyfileobj(pkg, dst, size) 230 | pkg.close() 231 | return fpath 232 | 233 | 234 | def dump_table(dbpath, source, table): 235 | dstdir = os.path.join(source, table) 236 | try: 237 | os.mkdir(dstdir) 238 | except FileExistsError: 239 | pass 240 | db = klb_sqlite(dbpath).cursor() 241 | sel = 'select distinct pack_name, head, size, key1, key2 from ' + table 242 | with mp.Pool() as p: 243 | results = [] 244 | f = decrypt_worker 245 | for (pack_name, head, size, k1, k2) in db.execute(sel): 246 | r = p.apply_async( 247 | f, (source, table, pack_name, head, size, k1, k2)) 248 | results.append(r) 249 | for r in results: 250 | print("[%s] done" % r.get()) 251 | 252 | 253 | def do_dump(args): 254 | for source in args.directories: 255 | dbpath = find_db('asset_a_ja_0' , source) 256 | if dbpath is None: 257 | dbpath = find_db('asset_a_ko' , source) 258 | for table in args.types: 259 | dump_table(dbpath, source, table) 260 | 261 | 262 | def do_dictionary(args): 263 | for word in args.text: 264 | print(dictionary_get(word, args.directory)) 265 | 266 | 267 | def do_tickets(args): 268 | import io 269 | from PIL import Image, ImageFont, ImageDraw 270 | import textwrap 271 | masterdb = klb_sqlite(find_db('masterdata', args.directory)).cursor() 272 | if find_db('asset_a_ja_0', args.directory) is None: 273 | db = klb_sqlite(find_db('asset_a_ko', args.directory)).cursor() 274 | dic = klb_sqlite(find_db('dictionary_ko_k', args.directory)).cursor() 275 | else: 276 | db = klb_sqlite(find_db('asset_a_ja_0', args.directory)).cursor() 277 | dic = klb_sqlite(find_db('dictionary_ja_k', args.directory)).cursor() 278 | mastersel = ''' 279 | select id, name, description, thumbnail_asset_path 280 | from m_gacha_ticket 281 | ''' 282 | i = 0 283 | pics = [] 284 | for (id, name, desc, asset_path) in masterdb.execute(mastersel): 285 | sel = ''' 286 | select pack_name, head, size, key1, key2 287 | from texture 288 | where asset_path = ? 289 | ''' 290 | rows = db.execute(sel, (asset_path,)) 291 | pics.append(rows.fetchone() + (id, name, desc)) 292 | img = None 293 | fnt = None 294 | fonts = ['NotoSerifCJK-Regular.ttc', 'Arial Unicode.ttf'] 295 | for font in fonts: 296 | try: 297 | fnt = ImageFont.truetype(font, 20) 298 | except OSError: 299 | continue 300 | break 301 | if fnt is None: 302 | print('warning: falling back to default font') 303 | for (pakname, head, size, key1, key2, id, name, desc) in pics: 304 | if fnt is not None: 305 | name = dictionary_get(name, args.directory) 306 | desc = dictionary_get(desc, args.directory) 307 | key = [key1, key2, 0x3039] 308 | pkgpath = os.path.join(args.directory, "pkg" + pakname[:1], pakname) 309 | pkg = codecs.open(pkgpath, mode='rb', encoding='klbvfs', errors=key) 310 | pkg.seek(head) 311 | imagedata = pkg.read(size) 312 | mime = magic.from_buffer(imagedata, mime=True) 313 | ext = mimetypes.guess_extension(mime) 314 | thumb = Image.open(io.BytesIO(imagedata)) 315 | if img is None: 316 | (_, height) = thumb.size 317 | h = int(float(height) * 1.1) 318 | x = int(float(height) * 0.1) 319 | img = Image.new('RGBA', (800, x + len(pics) * h), color=(255,) * 3) 320 | d = ImageDraw.Draw(img) 321 | y = x + i * h 322 | img.paste(thumb, (x, y)) 323 | lines = ['%d %s@%d,%d' % (id, pakname, head, size), name] 324 | print('%d -> "texture/%s_%d%s",' % (id, pakname, head, ext)) 325 | for j, l in enumerate(lines + textwrap.wrap(desc, 30)): 326 | d.text((x * 2 + h, y + h / 5 * j), l, fill=(0,) * 3, font=fnt) 327 | i += 1 328 | img.save('tickets.png') 329 | 330 | if __name__ == "__main__": 331 | import argparse 332 | parser = argparse.ArgumentParser(description='klab sqlite vfs utils') 333 | sub = parser.add_subparsers() 334 | 335 | desc = 'run a sql query on the encrypted database' 336 | query = sub.add_parser('query', aliases=['q'], help=desc) 337 | query.add_argument('dbfile') 338 | defsql = "select sql from sqlite_master where type='table'" 339 | query.add_argument('sql', nargs='?', default=defsql) 340 | query.set_defaults(func=do_query) 341 | 342 | desc = 'clone encrypted database to a regular unencrypted sqlite db' 343 | decrypt = sub.add_parser('decrypt', aliases=['de'], help=desc) 344 | decrypt.add_argument('files', nargs='+') 345 | decrypt.set_defaults(func=do_decrypt) 346 | 347 | desc = 'dump encrypted assets from pkg files' 348 | dump = sub.add_parser('dump', aliases=['d'], help=desc) 349 | types = ['texture', 'live2d_sd_model', 'member_model', 'member_sd_model', 350 | 'background', 'shader', 'skill_effect', 'stage', 'stage_effect', 351 | 'skill_timeline', 'skill_wipe', 'adv_script', 352 | 'gacha_performance', 'navi_motion', 'navi_timeline'] 353 | desc = 'types of assets. supported values: ' + ', '.join(types) 354 | dump.add_argument('--types', dest='types', nargs='*', metavar='', 355 | choices=types, default=types, help=desc) 356 | dirdesc = 'directory where the pkg* folders and db files are located. ' 357 | dirdesc += 'usually /data/data/com.klab.lovelive.allstars/files/files' 358 | dump.add_argument('directories', nargs='*', help=dirdesc, default='.') 359 | dump.set_defaults(func=do_dump) 360 | 361 | desc = "look up strings in the game's dictionary" 362 | dictionary = sub.add_parser('dictionary', aliases=['dic'], help=desc) 363 | desc = 'strings to look up. will be returned unchanged if not found' 364 | dictionary.add_argument('--directory', '-d', dest='directory', 365 | help=dirdesc, default='.') 366 | dictionary.add_argument('text', nargs='+', help=desc) 367 | dictionary.set_defaults(func=do_dictionary) 368 | 369 | desc = 'generate tickets.png with all gacha tickets. requires pillow' 370 | tickets = sub.add_parser('tickets', aliases=['tix'], help=desc) 371 | tickets.add_argument('directory', nargs='?', help=dirdesc, default='.') 372 | tickets.set_defaults(func=do_tickets) 373 | 374 | args = parser.parse_args(sys.argv[1:]) 375 | if 'func' not in args: 376 | parser.parse_args(['-h']) 377 | args.func(args) 378 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | apsw==3.9.2.post1 2 | beautifulsoup4==4.8.1 3 | lxml==4.4.1 4 | python-magic==0.4.15 5 | soupsieve==1.9.4 6 | Pillow==7.1.2 7 | --------------------------------------------------------------------------------