├── DataTableOptimizer.lua └── README.md /DataTableOptimizer.lua: -------------------------------------------------------------------------------- 1 | --[[ 2 | How to use: 3 | 4 | Put all lua files into DatabaseRoot then call ExportDatabaseLocalText( tofile = true, newStringBank = false ) at the end of file. ( beware: all your original files will be replaced with optimized files ) 5 | If you want to exlucde some input files in DatabaseRoot, just add theirs names into ExcludedFiles 6 | 7 | 8 | 9 | --]] 10 | 11 | 12 | local Root = "." 13 | if arg and arg[0] then 14 | local s, e = string.find( arg[0], "Client" ) 15 | if e then 16 | Root = string.sub( arg[0], 1, e ) 17 | end 18 | end 19 | Root = string.gsub( Root, '\\', '/' ) 20 | local DatabaseRoot = Root.."/Assets/StreamingAssets/Database" 21 | local LuaRoot = Root.."/Assets/StreamingAssets/LuaRoot" 22 | package.path = package.path..';'..DatabaseRoot..'/?.lua'..';'.. LuaRoot..'/?.lua' 23 | 24 | local EnableDatasetOptimize = true 25 | local EnableDefaultValueOptimize = true 26 | local EnableLocalization = true -- set to false to disable localization process 27 | 28 | local Database = {} 29 | local CSV --= require "std.csv" 30 | local DefaultNumberSerializedFormat = "%.14g" 31 | local NumberSerializedFormat = DefaultNumberSerializedFormat 32 | local DatabaseLocaleTextName = "_LocaleText" 33 | local StringBankOutput = DatabaseRoot.."/"..DatabaseLocaleTextName..".lua" 34 | local StringBankCSVOutput = DatabaseRoot.."/"..DatabaseLocaleTextName..".csv" 35 | local MaxStringBankRedundancy = 100 36 | local MaxStringBankBinSize = 524288 37 | local LocaleTextLeadingTag = '@' 38 | local MaxLocalVariableNum = 160 -- lparser.c #define MAXVARS 200 39 | local RefTableName = "__rt" 40 | local DefaultValueTableName = "__default_values" 41 | local PrintTableRefCount = false 42 | local UnknownName = "___noname___" 43 | 44 | local floor = math.floor 45 | local fmod = math.fmod 46 | 47 | local ExcludedFiles = { 48 | --Add file name to exclude from build 49 | _LocaleText = true, 50 | } 51 | 52 | local UniquifyTables = {} -- hash -> table 53 | local UniquifyTablesIds = {} -- id -> hash 54 | local UniquifyTablesInvIds = {} -- table -> id 55 | local UniquifyTablesRefCounter = {} -- table -> refcount 56 | 57 | local function HashString( v ) 58 | local val = 0 59 | local fmod = fmod 60 | local gmatch = string.gmatch 61 | local byte = string.byte 62 | local MaxStringBankBinSize = MaxStringBankBinSize 63 | local c 64 | for _c in gmatch( v, "." ) do 65 | c = byte( _c ) 66 | val = val + c * 193951 67 | val = fmod( val, MaxStringBankBinSize ) 68 | val = val * 399283 69 | val = fmod( val, MaxStringBankBinSize ) 70 | end 71 | return val 72 | end 73 | 74 | local function AddStringToBank( stringBank, str ) 75 | local meta = getmetatable( stringBank ) 76 | local reversed = nil 77 | local counter = nil 78 | if not meta then 79 | meta = { 80 | __counter = { used = {} }, -- mark used hash value 81 | __reversed = {} -- string -> hash reverse lookup 82 | } 83 | reversed = meta.__reversed 84 | counter = meta.__counter 85 | setmetatable( stringBank, meta ) 86 | local remove = {} 87 | -- lazy initialize reverse lut 88 | for h, s in pairs( stringBank ) do 89 | local _h = reversed[ s ] 90 | assert( _h == nil ) 91 | reversed[ s ] = h 92 | end 93 | end 94 | reversed = reversed or meta.__reversed 95 | counter = counter or meta.__counter 96 | local hash = reversed[ str ] 97 | if hash then 98 | counter.used[ hash ] = true 99 | return hash 100 | end 101 | hash = HashString( str ) 102 | local _v = stringBank[ hash ] 103 | while _v do 104 | hash = hash + 1 105 | hash = fmod( hash, MaxStringBankBinSize ) 106 | _v = stringBank[ hash ] 107 | end 108 | assert( not reversed[ str ] ) 109 | stringBank[ hash ] = str 110 | reversed[ str ] = hash 111 | counter.used[ hash ] = true 112 | return hash 113 | end 114 | 115 | local function OrderedForeach( _table, _func, _filter ) 116 | local _type = type 117 | if _type( _table ) == "table" then 118 | local kv = {} 119 | for k, v in pairs( _table ) do 120 | if not _filter or _filter( k, v ) then 121 | kv[ #kv + 1 ] = { k, v } 122 | end 123 | end 124 | local _tostring = tostring 125 | table.sort( kv, 126 | function( _l, _r ) 127 | local l = _l[ 1 ] 128 | local r = _r[ 1 ] 129 | local lt = _type( l ) 130 | local rt = _type( r ) 131 | if lt == rt and lt ~= "table" then 132 | return l < r 133 | else 134 | return _tostring( l ) < _tostring( r ) 135 | end 136 | end 137 | ) 138 | for _, _v in ipairs( kv ) do 139 | local k = _v[ 1 ] 140 | local v = _v[ 2 ] 141 | _func( k, v ) 142 | end 143 | end 144 | end 145 | 146 | local function OrderedForeachByValue( _table, _func, _filter ) 147 | local _type = type 148 | if _type( _table ) == "table" then 149 | local kv = {} 150 | for k, v in pairs( _table ) do 151 | if not _filter or _filter( k, v ) then 152 | kv[ #kv + 1 ] = { k, v } 153 | end 154 | end 155 | local _tostring = tostring 156 | table.sort( kv, 157 | function( _l, _r ) 158 | local l = _l[ 2 ] 159 | local r = _r[ 2 ] 160 | local lt = _type( l ) 161 | local rt = _type( r ) 162 | if lt == rt and lt ~= "table"then 163 | return l < r 164 | else 165 | return _tostring( l ) < _tostring( r ) 166 | end 167 | end 168 | ) 169 | for _, _v in ipairs( kv ) do 170 | local k = _v[ 1 ] 171 | local v = _v[ 2 ] 172 | if not pcall( _func, k, v ) then 173 | return false 174 | end 175 | end 176 | return true 177 | end 178 | end 179 | 180 | local function EncodeEscapeString( s ) 181 | local buf = {} 182 | buf[#buf + 1] = "\"" 183 | string.gsub( s, ".", 184 | function ( c ) 185 | if c == '\n' then 186 | buf[#buf + 1] = "\\n" 187 | elseif c == '\t' then 188 | buf[#buf + 1] = "\\t" 189 | elseif c == '\r' then 190 | buf[#buf + 1] = "\\r" 191 | elseif c == '\a' then 192 | buf[#buf + 1] = "\\a" 193 | elseif c == '\b' then 194 | buf[#buf + 1] = "\\b" 195 | elseif c == '\\' then 196 | buf[#buf + 1] = "\\\\" 197 | elseif c == '\"' then 198 | buf[#buf + 1] = "\\\"" 199 | elseif c == '\'' then 200 | buf[#buf + 1] = "\\\'" 201 | elseif c == '\v' then 202 | buf[#buf + 1] = "\\\v" 203 | elseif c == '\f' then 204 | buf[#buf + 1] = "\\\f" 205 | else 206 | buf[#buf + 1] = c 207 | end 208 | end 209 | ) 210 | buf[#buf + 1] = "\"" 211 | return table.concat( buf, "" ) 212 | end 213 | 214 | local function StringBuilder() 215 | local sb = {} 216 | local f = function( str ) 217 | if str then 218 | sb[ #sb + 1 ] = str 219 | end 220 | return f, sb 221 | end 222 | return f 223 | end 224 | 225 | local function CreateFileWriter( fileName, mode ) 226 | local file = nil 227 | local indent = 0 228 | if mode and fileName then 229 | local _file, err = io.open( fileName ) 230 | if _file ~= nil then 231 | --print( "remove file "..fileName ) 232 | os.remove( fileName ) 233 | end 234 | file = io.open( fileName, mode ) 235 | end 236 | local ret = nil 237 | if file then 238 | ret = { 239 | write = function( ... ) 240 | if indent > 0 then 241 | for i = 0, indent - 1 do 242 | file:write( "\t" ) 243 | end 244 | end 245 | return file:write( ... ) 246 | end, 247 | close = function( ... ) 248 | return file:close() 249 | end 250 | } 251 | else 252 | ret = { 253 | write = function( ... ) 254 | for i = 0, indent - 1 do 255 | io.write( "\t" ) 256 | end 257 | return io.write( ... ) 258 | end, 259 | close = function( ... ) 260 | end 261 | } 262 | end 263 | ret.indent = function( count ) 264 | count = count or 1 265 | indent = indent + count or 1 266 | end 267 | ret.outdent = function( count ) 268 | count = count or 1 269 | if indent >= count then 270 | indent = indent - count 271 | end 272 | end 273 | return ret 274 | end 275 | 276 | local function SetNumberSerializedFormat( f ) 277 | NumberSerializedFormat = f or DefaultNumberSerializedFormat 278 | if NumberSerializedFormat == "" then 279 | NumberSerializedFormat = DefaultNumberSerializedFormat 280 | end 281 | print( "set NumberSerializedFormat: ".. NumberSerializedFormat ) 282 | end 283 | 284 | local DefaultVisitor = { 285 | recursive = true, 286 | iVisit = function( i, v, curPath ) 287 | print( string.format( "%s[%d] = %s", curPath, i, tostring( v ) ) ) 288 | return true 289 | end, 290 | nVisit = function( n, v, curPath ) 291 | print( string.format( "%s[%g] = %s", curPath, n, tostring( v ) ) ) 292 | return true 293 | end, 294 | sVisit = function( s, v, curPath ) 295 | local _v = tostring( v ) 296 | print( #curPath > 0 and curPath.."."..s.." = ".._v or s.." = ".._v ) 297 | return true 298 | end, 299 | xVisit = function( k, v, curPath ) 300 | local sk = tostring( k ) 301 | local sv = tostring( v ) 302 | print( #curPath > 0 and curPath.."."..sk.." = "..sv or sk.." = "..sv ) 303 | return true 304 | end 305 | } 306 | 307 | local function WalkDataset( t, visitor, parent ) 308 | if not parent then 309 | parent = "" 310 | end 311 | -- all integer key 312 | local continue = true 313 | if visitor.iVisit then 314 | for i, v in ipairs( t ) do 315 | local _t = type( v ) 316 | if _t == "table" and visitor.recursive then 317 | continue = WalkDataset( v, visitor, string.format( "%s[%g]", parent, i ) ) 318 | elseif _t == "string" or _t == "number" then 319 | continue = visitor.iVisit( i, v, parent ) 320 | else 321 | -- not support value type 322 | if visitor.xVisit then 323 | continue = visitor.xVisit( i, v, parent ) 324 | end 325 | end 326 | if not continue then 327 | return continue 328 | end 329 | end 330 | end 331 | 332 | local len = #t 333 | local keys = {} 334 | local idict = {} 335 | for k, v in pairs( t ) do 336 | local _t = type( k ) 337 | if _t == "number" then 338 | local intKey = k == math.floor( k ); 339 | if k > len or k <= 0 or not intKey then 340 | idict[k] = v 341 | end 342 | elseif _t == "string" then 343 | keys[#keys + 1] = k 344 | else 345 | --table, function, ... 346 | --not support data type for key 347 | if visitor.xVisit then 348 | continue = visitor.xVisit( k, v, parent ) 349 | end 350 | end 351 | if not continue then 352 | return continue 353 | end 354 | end 355 | -- for all number keys those are not in array part 356 | -- key must be number 357 | for k, v in pairs( idict ) do 358 | local intKey = k == math.floor( k ); 359 | local _t = type( v ) 360 | if _t ~= "table" then 361 | if _t == "number" or _t == "string" then 362 | if intKey then 363 | if visitor.iVisit then 364 | continue = visitor.iVisit( k, v, parent ) 365 | end 366 | else 367 | if visitor.nVisit then 368 | continue = visitor.nVisit( k, v, parent ) 369 | end 370 | end 371 | else 372 | -- not support value data type 373 | if visitor.xVisit then 374 | continue = visitor.xVisit( k, v, parent ) 375 | end 376 | end 377 | elseif visitor.recursive then 378 | if intKey then 379 | continue = WalkDataset( v, visitor, string.format( "%s[%d]", parent, k ) ) 380 | else 381 | continue = WalkDataset( v, visitor, string.format( "%s[%g]", parent, k ) ) 382 | end 383 | end 384 | if not continue then 385 | return continue 386 | end 387 | end 388 | -- sort all string keys 389 | table.sort( keys ) 390 | -- for all none-table value 391 | local tableValue 392 | for k, v in pairs( keys ) do 393 | local value = t[v] 394 | local _t = type( value ) 395 | if _t == "number" or _t == "string" then 396 | -- print all number or string value here 397 | if visitor.sVisit then 398 | continue = visitor.sVisit( v, value, parent ) 399 | end 400 | elseif _t == "table" then 401 | -- for table value 402 | if not tableValue then 403 | tableValue = {} 404 | end 405 | tableValue[ k ] = v 406 | else 407 | if visitor.xVisit then 408 | continue = visitor.xVisit( v, value, parent ) 409 | end 410 | end 411 | if not continue then 412 | return continue 413 | end 414 | end 415 | if visitor.recursive then 416 | -- for all table value 417 | if tableValue then 418 | for k, v in pairs( tableValue ) do 419 | local value = t[v] 420 | continue = WalkDataset( value, visitor, #parent > 0 and parent.."."..v or v ) 421 | if not continue then 422 | return continue 423 | end 424 | end 425 | end 426 | end 427 | return continue 428 | end 429 | 430 | local function PrintDataset( t, parent ) 431 | if not parent then 432 | parent = "" 433 | end 434 | local string_format = string.format 435 | -- all integer key 436 | for i, v in ipairs( t ) do 437 | local _t = type( v ) 438 | if _t == "table" then 439 | PrintDataset( v, string_format( "%s[%g]", parent, i ) ) 440 | elseif _t == "string" or _t == "number" then 441 | print( string.format( "%s[%d] = %s", parent, i, tostring( v ) ) ) 442 | else 443 | -- not support value type 444 | end 445 | end 446 | local len = #t 447 | local keys = {} 448 | local idict = {} 449 | for k, v in pairs( t ) do 450 | local _t = type( k ) 451 | if _t == "number" then 452 | if k > len or k <= 0 then 453 | idict[k] = v 454 | end 455 | elseif _t == "string" then 456 | keys[#keys + 1] = k 457 | else 458 | --table, function, ... 459 | --not support data type for key 460 | end 461 | end 462 | -- for all number keys those are not in array part 463 | -- key must be number 464 | for k, v in pairs( idict ) do 465 | local intKey = k == math.floor( k ) 466 | local _t = type( v ) 467 | if _t ~= "table" then 468 | if _t == "number" or _t == "string" then 469 | if intKey then 470 | print( string_format( "%s[%d] = %s", parent, k, tostring( v ) ) ) 471 | else 472 | print( string_format( "%s[%g] = %s", parent, k, tostring( v ) ) ) 473 | end 474 | else 475 | -- not support value data type 476 | end 477 | else 478 | if intKey then 479 | PrintDataset( v, string_format( "%s[%d]", parent, k ) ) 480 | else 481 | PrintDataset( v, string_format( "%s[%g]", parent, k ) ) 482 | end 483 | end 484 | end 485 | -- sort all string keys 486 | table.sort( keys ) 487 | -- for all none-table value 488 | local tableValue 489 | for k, v in pairs( keys ) do 490 | local value = t[v] 491 | local _t = type( value ) 492 | if _t ~= "table" then 493 | -- print all number or string value here 494 | local _value = tostring( value ) 495 | print( #parent > 0 and parent.."."..v.." = ".._value or v.." = ".._value ) 496 | else 497 | -- for table value 498 | if not tableValue then 499 | tableValue = {} 500 | end 501 | tableValue[ k ] = v 502 | end 503 | end 504 | -- for all table value 505 | if tableValue then 506 | for k, v in pairs( tableValue ) do 507 | local value = t[v] 508 | PrintDataset( value, #parent > 0 and parent.."."..v or v ) 509 | end 510 | end 511 | end 512 | 513 | local function DeserializeTable( val ) 514 | local loader = loadstring or load -- lua5.2 compat 515 | local chunk = loader( "return " .. val ) 516 | local ok, ret = pcall( chunk ) 517 | if not ok then 518 | ret = nil 519 | print( "DeserializeTable failed!"..val ) 520 | end 521 | return ret 522 | end 523 | 524 | local function _SerializeTable( val, name, skipnewlines, campact, depth, tableRef ) 525 | local valt = type( val ) 526 | depth = depth or 0 527 | campact = campact or false 528 | local append = StringBuilder() 529 | local eqSign = " = " 530 | local tmp = "" 531 | local string_format = string.format 532 | if not campact then 533 | append( string.rep( "\t", depth ) ) 534 | skipnewlines = skipnewlines or false 535 | else 536 | skipnewlines = true 537 | eqSign = "=" 538 | end 539 | if name then 540 | local nt = type( name ) 541 | if nt == "string" then 542 | if name ~= "" then 543 | if string.match( name,'^%d+' ) then 544 | append( "[\"" ) 545 | append( name ) 546 | append( "\"]" ) 547 | else 548 | append( name ) 549 | end 550 | else 551 | append( "[\"\"]" ) 552 | end 553 | append( eqSign ) 554 | elseif nt == "number" then 555 | append( string_format( "[%s]", tostring( name ) ) ) 556 | append( eqSign ) 557 | else 558 | tmp = tmp .. "\"[inserializeable datatype for key:" .. nt .. "]\"" 559 | end 560 | end 561 | local ending = not skipnewlines and "\n" or "" 562 | if tableRef then 563 | local refName = tableRef[ val ] 564 | if refName then 565 | valt = "ref" 566 | val = refName 567 | end 568 | end 569 | if valt == "table" then 570 | append( "{" ) append( ending ) 571 | local array_part = {} 572 | local count = 0 573 | for k, v in ipairs( val ) do 574 | if type( v ) ~= "function" then 575 | array_part[k] = true 576 | if count > 0 then 577 | append( "," ) 578 | append( ending ) 579 | end 580 | append( _SerializeTable( v, nil, skipnewlines, campact, depth + 1, tableRef ) ) 581 | count = count + 1 582 | end 583 | end 584 | local sortedK = {} 585 | for k, v in pairs( val ) do 586 | if type( v ) ~= "function" then 587 | if not array_part[k] then 588 | sortedK[#sortedK + 1] = k 589 | end 590 | end 591 | end 592 | table.sort( sortedK ) 593 | for i, k in ipairs( sortedK ) do 594 | local v = val[k] 595 | if count > 0 then 596 | append( "," ) 597 | append( ending ) 598 | end 599 | append( _SerializeTable( v, k, skipnewlines, campact, depth + 1, tableRef ) ) 600 | count = count + 1 601 | end 602 | if count >= 1 then 603 | append( ending ) 604 | end 605 | if not campact then 606 | append( string.rep( "\t", depth ) ) 607 | end 608 | append( "}" ) 609 | elseif valt == "number" then 610 | if DefaultNumberSerializedFormat == NumberSerializedFormat or math.floor( val ) == val then 611 | append( tostring( val ) ) 612 | else 613 | append( string_format( NumberSerializedFormat, val ) ) 614 | end 615 | elseif valt == "string" then 616 | append( EncodeEscapeString( val ) ) 617 | elseif valt == "boolean" then 618 | append( val and "true" or "false" ) 619 | elseif valt == "ref" then 620 | append( val or "nil" ) 621 | else 622 | tmp = tmp .. "\"[inserializeable datatype:" .. valt .. "]\"" 623 | end 624 | local _, slist = append() 625 | return table.concat( slist, "" ) 626 | end 627 | 628 | local function SerializeTable( val, skipnewlines, campact, tableRef, name ) 629 | getmetatable( "" ).__lt = function( a, b ) return tostring( a ):lower() < tostring( b ):lower() end 630 | local ret = _SerializeTable( val, name, skipnewlines, campact, 0, tableRef ) 631 | getmetatable( "" ).__lt = nil 632 | return ret 633 | end 634 | 635 | local function DumpStringBank( stringBank ) 636 | print( 'dump database local string bank begin...' ) 637 | for k, v in pairs( stringBank ) do 638 | print( string.format( "\t[%g] = %s", k, v ) ) 639 | end 640 | print( 'dump database local string bank end.' ) 641 | end 642 | 643 | local function SaveStringBankToLua( stringBank, tofile ) 644 | if tofile then 645 | local fileName = StringBankOutput 646 | local _file, err = io.open( fileName ) 647 | if _file ~= nil then 648 | _file:close() 649 | os.remove( fileName ) 650 | end 651 | file = io.open( fileName, "w" ) 652 | local fmt = string.format 653 | file:write( fmt( "local %s = {\n", DatabaseLocaleTextName ) ) 654 | for k, v in pairs( stringBank ) do 655 | file:write( fmt( "\t[%g] = %s,\n", k, EncodeEscapeString( v ) ) ) 656 | end 657 | file:write( "}\n" ) 658 | file:write( fmt( "return %s\n--EOF", DatabaseLocaleTextName ) ) 659 | file:close() 660 | else 661 | DumpStringBank( stringBank ) 662 | end 663 | end 664 | 665 | local function SaveStringBankToCSV( stringBank, tofile ) 666 | local _exists = {} 667 | for k, v in pairs( stringBank ) do 668 | assert( not _exists[v] ) 669 | _exists[ v ] = k 670 | end 671 | local csv = CSV 672 | if tofile and csv then 673 | local fileName = StringBankCSVOutput 674 | local _file, err = io.open( fileName ) 675 | if _file ~= nil then 676 | _file:close() 677 | os.remove( fileName ) 678 | end 679 | local t = {} 680 | local count = 1 681 | for k, v in pairs( stringBank ) do 682 | t[count] = { k, v } 683 | count = count + 1 684 | end 685 | table.sort( t, 686 | function( a, b ) 687 | return a[1] < b[1] 688 | end 689 | ) 690 | csv.save( fileName, t, true ) 691 | else 692 | SaveStringBankToLua( stringBank, tofile ) 693 | end 694 | end 695 | 696 | local function LoadStringBankFromLua( info ) 697 | local stringBank = {} 698 | local chunk = loadfile( StringBankOutput ) 699 | if chunk then 700 | print( 'load string bank: '..StringBankOutput ) 701 | local last = chunk() 702 | if last and type( last ) == "table" then 703 | for k, v in pairs( last ) do 704 | stringBank[ k ] = v 705 | end 706 | end 707 | end 708 | return stringBank 709 | end 710 | 711 | local function LoadStringBankFromCSV() 712 | local stringBank = {} 713 | local csv = CSV 714 | if csv then 715 | local fileName = StringBankCSVOutput 716 | local file, err = io.open( fileName ) 717 | if file then 718 | print( "Load StringBank: " .. fileName ) 719 | file:close() 720 | local b = csv.load( fileName, true ) 721 | local allstr = {} 722 | for i = 1, #b do 723 | local key = b[ i ][ 1 ] 724 | local value = b[ i ][ 2 ] 725 | local oldHash = allstr[ value ] 726 | if not oldHash then 727 | stringBank[ key ] = value 728 | allstr[ value ] = key 729 | else 730 | print( string.format( "\"%s\" already exists in StringBank with hash %d", value, oldHash ) ) 731 | end 732 | end 733 | end 734 | else 735 | return LoadStringBankFromLua() 736 | end 737 | return stringBank 738 | end 739 | 740 | local function TrimStringBank( stringBank ) 741 | -- remove useless values 742 | local meta = getmetatable( stringBank ) 743 | if meta then 744 | local counter = meta.__counter 745 | if counter then 746 | local used = counter.used 747 | if used then 748 | local count = 0 749 | for hash, str in pairs( stringBank ) do 750 | count = count + 1 751 | end 752 | local unused = {} 753 | for hash, str in pairs( stringBank ) do 754 | if not used[ hash ] then 755 | unused[#unused + 1] = hash 756 | end 757 | end 758 | if #unused > MaxStringBankRedundancy then 759 | for _, h in ipairs( unused ) do 760 | stringBank[ h ] = nil 761 | end 762 | end 763 | end 764 | end 765 | end 766 | end 767 | 768 | local function GetAllFileNamesAtPath( path ) 769 | path, _ = path:gsub( "/", "\\" ) 770 | local ret = {} 771 | for dir in io.popen( string.format( "dir \"%s\" /S/b", path ) ):lines() do 772 | local s, e, f = dir:find( ".+\\(.+)%.lua$" ) 773 | if f then 774 | table.insert( ret, f ) 775 | end 776 | end 777 | table.sort( ret ) 778 | return ret 779 | end 780 | 781 | local function LoadDataset( name ) 782 | if not Database then 783 | _G["Database"] = {} 784 | Database.loaded = {} 785 | end 786 | local loader = function( name ) 787 | Database.loaded = Database.loaded or {} 788 | local r = Database.loaded[name] 789 | if r then 790 | return r 791 | end 792 | local pname = string.gsub( name, "%.", "/" ) 793 | local split = function( s, p ) 794 | local rt= {} 795 | string.gsub( s, '[^'..p..']+', function( w ) table.insert( rt, w ) end ) 796 | return rt 797 | end 798 | 799 | local curName = pname..".lua" 800 | local fileName = DatabaseRoot.."/"..curName 801 | local checkFileName = function( path, name ) 802 | path, _ = path:gsub( "/", "\\" ) 803 | local _name = string.lower( name ) 804 | for dir in io.popen( string.format( "dir \"%s\" /s/b", path ) ):lines() do 805 | local s, e, f = dir:find( ".+\\(.+%.lua)$" ) 806 | if f then 807 | local _f = string.lower( f ) 808 | if _name == _f then 809 | return name == f, f -- not match, real name 810 | end 811 | end 812 | end 813 | end 814 | 815 | local m, real = checkFileName( DatabaseRoot, curName ) 816 | if not m and real then 817 | local msg = string.format( "filename must be matched by case! realname: \"%s\", you pass: \"%s\"", real, curName ) 818 | print( msg ) 819 | os.execute( "pause" ) 820 | end 821 | 822 | local chunk = loadfile( fileName ) 823 | if not chunk then 824 | fileName = LuaRoot.."/"..pname..".lua" 825 | chunk, err = loadfile( fileName ) 826 | if err then 827 | print( "\n\n" ) 828 | print( "----------------------------------" ) 829 | print( "Load lua failed: "..fileName ) 830 | print( "Error:" ) 831 | print( "\t"..err ) 832 | print( "----------------------------------" ) 833 | print( "\n\n" ) 834 | end 835 | end 836 | print( fileName ) 837 | assert( chunk ) 838 | if not chunk then 839 | os.execute( "pause" ) 840 | end 841 | local rval = chunk() 842 | if rval.__name ~= nil then 843 | os.execute( "pause table's key must not be '__name' which is the reserved keyword." ) 844 | end 845 | if rval.__sourcefile ~= nil then 846 | os.execute( "pause table's key must not be '__sourcefile' which is the reserved keyword." ) 847 | end 848 | rval.__name = name 849 | rval.__sourcefile = fileName 850 | 851 | local namespace = Database 852 | local ns = split( pname, '/' ) 853 | local xname = ns[#ns] -- last one 854 | if #ns > 1 then 855 | for i = 1, #ns - 1 do 856 | local n = ns[i]; 857 | if namespace[n] == nil then 858 | namespace[n] = {} 859 | end 860 | namespace = namespace[n] 861 | end 862 | end 863 | namespace[xname] = rval 864 | print( "dataset: "..name.." has been loaded" ) 865 | Database.loaded[name] = rval 866 | return rval 867 | end 868 | return loader( name ) 869 | end 870 | 871 | local function CheckNotAscii( v ) 872 | if v ~= nil and type( v ) == "string" then 873 | local byte = string.byte 874 | for _c in string.gmatch( v, "." ) do 875 | local c = byte( _c ) 876 | if c < 0 or c > 127 then 877 | return true 878 | end 879 | end 880 | end 881 | return false 882 | end 883 | 884 | local function LocalizeRecord( id, record, genCode, StringBank ) 885 | local localized_fields = nil 886 | local subTable = nil 887 | OrderedForeach( 888 | record, 889 | function( k, v ) 890 | local vt = type( v ) 891 | if vt == "string" then 892 | if CheckNotAscii( v ) then 893 | if #v > 0 and string.sub( v, 1, 1 ) == LocaleTextLeadingTag then 894 | print( string.format( "invalid leading character for localized text! key, value: %s, %s", k, v ) ) 895 | os.execute( "pause" ) 896 | end 897 | if not localized_fields then 898 | localized_fields = {} 899 | end 900 | -- build localized id string with tag 901 | local sid = AddStringToBank( StringBank, v ) 902 | localized_fields[ k ] = string.format( "%s%g", LocaleTextLeadingTag, sid ) 903 | if genCode then 904 | genCode[ #genCode + 1 ] = { 905 | id, 906 | sid, 907 | v 908 | } 909 | end 910 | end 911 | elseif vt == "table" then 912 | if not subTable then 913 | subTable = {} 914 | end 915 | subTable[ #subTable + 1 ] = v 916 | end 917 | end 918 | ) 919 | local localized = false 920 | if localized_fields then 921 | -- override localized string with tag 922 | localized = true 923 | for k, v in pairs( localized_fields ) do 924 | record[ k ] = localized_fields[ k ] 925 | end 926 | end 927 | if subTable then 928 | for _, sub in ipairs( subTable ) do 929 | localized = LocalizeRecord( 0, sub, genCode, StringBank ) or localized 930 | end 931 | end 932 | return localized 933 | end 934 | 935 | local function GetValueTypeNameCS( value ) 936 | local t = type( value ) 937 | if t == "string" then 938 | return "string" 939 | elseif t == "number" then 940 | if value == math.floor( value ) then 941 | return "int" 942 | else 943 | return "float" 944 | end 945 | elseif t == "boolean" then 946 | return "bool" 947 | elseif t == "table" then 948 | return "table" 949 | else 950 | return "void" 951 | end 952 | end 953 | 954 | local function UniquifyTable( t ) 955 | if t == nil or type( t ) ~= "table" then 956 | return nil 957 | end 958 | local hash = SerializeTable( t, true, true ) 959 | local ref = UniquifyTables[ hash ] 960 | if ref then 961 | local refcount = UniquifyTablesRefCounter[ ref ] or 1 962 | UniquifyTablesRefCounter[ ref ] = refcount + 1 963 | return ref 964 | end 965 | 966 | local overwrites = nil 967 | local _type = type 968 | OrderedForeach( 969 | t, 970 | function( k, v ) 971 | overwrites = overwrites or {} 972 | overwrites[k] = UniquifyTable( v ) 973 | end, 974 | function( k, v ) 975 | return _type( v ) == "table" 976 | end 977 | ) 978 | if overwrites then 979 | for k, v in pairs( overwrites ) do 980 | t[ k ] = overwrites[ k ] 981 | end 982 | end 983 | local id = #UniquifyTablesIds + 1 984 | UniquifyTablesIds[ id ] = hash 985 | UniquifyTables[ hash ] = t 986 | UniquifyTablesInvIds[ t ] = id 987 | UniquifyTablesRefCounter[ t ] = 1 988 | return t 989 | end 990 | 991 | local function OptimizeDataset( dataset ) 992 | local ids = {} 993 | local names = {} 994 | local idType = nil 995 | -- choose cs data type 996 | -- for all fields in a record 997 | local typeNameTable = {} 998 | for k, v in pairs( dataset ) do 999 | local _sk = tostring( k ) 1000 | if _sk ~= "__name" and _sk ~= "__sourcefile" then 1001 | if not idType then 1002 | idType = type( k ) 1003 | end 1004 | if idType == type( k ) then 1005 | ids[ #ids + 1 ] = k 1006 | end 1007 | end 1008 | end 1009 | if EnableDefaultValueOptimize then 1010 | -- find bigest table to generate all fields 1011 | local majorItem = 1 1012 | local f = 0 1013 | for k, v in pairs( dataset ) do 1014 | if type( v ) == "table" then 1015 | local num = 0 1016 | for _, _ in pairs( v ) do 1017 | num = num + 1 1018 | end 1019 | if num > f then 1020 | f = num 1021 | majorItem = k 1022 | end 1023 | end 1024 | end 1025 | local v = dataset[ majorItem ] 1026 | if type( v ) == "table" then 1027 | for name, value in pairs( v ) do 1028 | local nt = type( name ) 1029 | if nt == "string" and name == "id" then 1030 | print( "this table already has a field named 'id'" ) 1031 | end 1032 | if nt == "string" then 1033 | names[ #names + 1 ] = name 1034 | end 1035 | end 1036 | table.sort( names, function( a, b ) return a:lower() < b:lower() end ) 1037 | for i, field in ipairs( names ) do 1038 | -- for all record / row 1039 | for r, t in ipairs( ids ) do 1040 | local record = dataset[ t ] 1041 | if record[ field ] ~= nil then 1042 | local v = record[ field ] 1043 | local curType = GetValueTypeNameCS( v ) 1044 | if not typeNameTable[ field ] then 1045 | typeNameTable[ field ] = curType 1046 | elseif typeNameTable[ field ] == "int" and curType == "float" then 1047 | -- overwrite int to float 1048 | typeNameTable[ field ] = curType 1049 | elseif curType == "table" then 1050 | -- overwrite to table 1051 | typeNameTable[ field ] = curType 1052 | end 1053 | end 1054 | end 1055 | -- patching miss fields with default values 1056 | local curType = typeNameTable[ field ] 1057 | for r, t in ipairs( ids ) do 1058 | local record = dataset[ t ] 1059 | local v = record[ field ] 1060 | if v == nil then 1061 | local ft = typeNameTable[ field ] 1062 | if ft == "string" then 1063 | v = "" 1064 | elseif ft == "number" or ft == "int" or ft == "float" then 1065 | v = 0 1066 | elseif ft == "table" then 1067 | v = {} 1068 | elseif ft == "bool" then 1069 | v = false 1070 | end 1071 | record[ field ] = v 1072 | end 1073 | end 1074 | end 1075 | end 1076 | end 1077 | 1078 | ids = {} 1079 | idType = nil 1080 | UniquifyTables = {} 1081 | UniquifyTablesIds = {} 1082 | UniquifyTablesInvIds = {} 1083 | UniquifyTablesRefCounter = {} 1084 | 1085 | local isIntegerKey = true 1086 | local overwrites = nil 1087 | OrderedForeach( 1088 | dataset, 1089 | function( k, v ) 1090 | local _sk = tostring( k ) 1091 | if _sk ~= "__name" and _sk ~= "__sourcefile" then 1092 | if not idType then 1093 | idType = type( k ) 1094 | end 1095 | -- check type 1096 | if idType == type( k ) then 1097 | ids[ #ids + 1 ] = k 1098 | if idType == "number" then 1099 | if isIntegerKey then 1100 | isIntegerKey = k == floor( k ) 1101 | end 1102 | end 1103 | end 1104 | if type( v ) == "table" then 1105 | overwrites = overwrites or {} 1106 | overwrites[ k ] = UniquifyTable( v ) 1107 | end 1108 | end 1109 | end 1110 | ) 1111 | if overwrites then 1112 | for k, v in pairs( overwrites ) do 1113 | dataset[ k ] = overwrites[ k ] 1114 | end 1115 | end 1116 | local returnVal = nil 1117 | if EnableDefaultValueOptimize then 1118 | local defaultValues = nil 1119 | for i, field in ipairs( names ) do 1120 | local curType = typeNameTable[ field ] 1121 | -- for all record/row 1122 | local defaultValueStat = { 1123 | } 1124 | for r, t in ipairs( ids ) do 1125 | local record = dataset[ t ] 1126 | local v = record[ field ] 1127 | if v ~= nil then 1128 | local vcount = defaultValueStat[ v ] or 0 1129 | defaultValueStat[ v ] = vcount + 1 1130 | else 1131 | assert( "default value missing!" ) 1132 | end 1133 | end 1134 | -- find the mostest used as a default value 1135 | local max = -1 1136 | local defaultValue = nil 1137 | local _defaultValue = "{}" 1138 | local result = OrderedForeachByValue( 1139 | defaultValueStat, 1140 | function( value, count ) 1141 | if count >= max then 1142 | if count > max then 1143 | max = count 1144 | defaultValue = value 1145 | _defaultValue = SerializeTable( defaultValue, true, true ) 1146 | else 1147 | if curType == "table" then 1148 | local _value = SerializeTable( value, true, true ) 1149 | if #_value > #_defaultValue then 1150 | defaultValue = value 1151 | _defaultValue = SerializeTable( defaultValue, true, true ) 1152 | end 1153 | else 1154 | local _value = value 1155 | local _defaultValue = defaultValue 1156 | if type( value ) == 'boolean' then 1157 | _value = value and 1 or 0 1158 | _defaultValue = defaultValue and 1 or 0 1159 | end 1160 | if _value < _defaultValue then 1161 | defaultValue = value 1162 | _defaultValue = SerializeTable( defaultValue, true, true ) 1163 | end 1164 | end 1165 | end 1166 | end 1167 | end 1168 | ) 1169 | if not result then 1170 | error( string.format( "create default value for \"%s\" failed. please make sure all the value's types are the same.", field ) ) 1171 | end 1172 | if defaultValue ~= nil then 1173 | defaultValues = defaultValues or {} 1174 | defaultValues[ field ] = defaultValue 1175 | end 1176 | end 1177 | returnVal = defaultValues 1178 | end 1179 | 1180 | -- remove tables whose's ref is 1 and re-mapping id 1181 | local newid = 1 1182 | local newIds = {} 1183 | local newInvIds = {} 1184 | 1185 | OrderedForeach( 1186 | UniquifyTablesIds, 1187 | function( id, hash ) 1188 | local table = UniquifyTables[ hash ] 1189 | local refcount = UniquifyTablesRefCounter[ table ] 1190 | if refcount == 1 then 1191 | UniquifyTables[ hash ] = nil 1192 | else 1193 | newIds[ newid ] = hash 1194 | newInvIds[ table ] = newid 1195 | newid = newid + 1 1196 | end 1197 | end 1198 | ) 1199 | 1200 | UniquifyTablesIds = newIds 1201 | UniquifyTablesInvIds = newInvIds 1202 | return returnVal 1203 | end 1204 | 1205 | local function ToUniqueTableRefName( id ) 1206 | if id <= MaxLocalVariableNum then 1207 | return string.format( RefTableName.."_%d", id ) 1208 | else 1209 | return string.format( RefTableName.."[%d]", id - MaxLocalVariableNum ) 1210 | end 1211 | end 1212 | 1213 | local function SaveDatasetToFile( dataset, tofile, tableRef, name ) 1214 | if tofile then 1215 | outFile = CreateFileWriter( dataset.__sourcefile, "w" ) 1216 | else 1217 | outFile = CreateFileWriter() 1218 | end 1219 | local ptr2ref = nil 1220 | if tableRef and tableRef.ptr2ref then 1221 | ptr2ref = tableRef.ptr2ref 1222 | end 1223 | if tableRef and tableRef.name2value then 1224 | local name2table = tableRef.name2value 1225 | local tables = tableRef.tables 1226 | local tableIds = tableRef.tableIds 1227 | local ptr2ref = tableRef.ptr2ref 1228 | local refcounter = tableRef.refcounter 1229 | local maxLocalVariableNum = tableRef.maxLocalVariableNum or MaxLocalVariableNum 1230 | local refTableName = tableRef.refTableName or "__rt" 1231 | local tableNum = #tableIds 1232 | for id, hash in ipairs( tableIds ) do 1233 | local table = tables[ hash ] 1234 | if table and id <= maxLocalVariableNum then 1235 | local refname = ptr2ref[ table ] 1236 | -- temp comment out top level ref 1237 | ptr2ref[ table ] = nil 1238 | local refcount = refcounter[ table ] 1239 | outFile.write( 1240 | string.format( 1241 | "%slocal %s = %s\n", 1242 | PrintTableRefCount and string.format( "--ref:%d\n", refcount ) or "", 1243 | refname, 1244 | SerializeTable( table, false, false, ptr2ref ) 1245 | ) 1246 | ) 1247 | ptr2ref[ table ] = refname 1248 | else 1249 | break 1250 | end 1251 | end 1252 | if tableNum > maxLocalVariableNum then 1253 | local maxCount = tableNum - maxLocalVariableNum 1254 | outFile.write( string.format( "local %s = createtable and createtable( %d, 0 ) or {}\n", refTableName, maxCount ) ) 1255 | for id = maxLocalVariableNum + 1, tableNum do 1256 | local offset = id - maxLocalVariableNum 1257 | local hash = tableIds[ id ] 1258 | local table = tables[ hash ] 1259 | local refname = ptr2ref[ table ] 1260 | -- temp comment out top level ref 1261 | ptr2ref[ table ] = nil 1262 | local refcount = refcounter[ table ] 1263 | outFile.write( 1264 | string.format( 1265 | "%s%s[%d] = %s\n", 1266 | PrintTableRefCount and string.format( "-- %s, ref:%d\n", refname, refcount ) or "", 1267 | refTableName, offset, 1268 | SerializeTable( table, false, false, ptr2ref ) 1269 | ) 1270 | ) 1271 | ptr2ref[ table ] = refname 1272 | end 1273 | end 1274 | end 1275 | local datasetName = dataset.__name or name 1276 | if not datasetName then 1277 | datasetName = UnknownName 1278 | dataset.__name = datasetName 1279 | end 1280 | outFile.write( string.format( "local %s = \n", datasetName ) ) 1281 | 1282 | -- remove none table value 1283 | local removed = nil 1284 | for k, v in pairs( dataset ) do 1285 | if type( v ) ~= "table" then 1286 | removed = removed or {} 1287 | removed[ #removed + 1 ] = k 1288 | end 1289 | end 1290 | if removed then 1291 | for _, k in ipairs( removed ) do 1292 | dataset[ k ] = nil 1293 | end 1294 | end 1295 | 1296 | outFile.write( SerializeTable( dataset, false, false, ptr2ref ) ) 1297 | outFile.write( "\n" ) 1298 | if tableRef and tableRef.postOutput then 1299 | tableRef.postOutput( outFile ) 1300 | end 1301 | outFile.write( string.format( "\nreturn %s\n", datasetName ) ) 1302 | outFile.close() 1303 | end 1304 | 1305 | local function ExportOptimizedDataset( t, StringBank ) 1306 | local datasetName = t.__name 1307 | if not datasetName then 1308 | datasetName = UnknownName 1309 | t.__name = datasetName 1310 | end 1311 | local localized = false 1312 | local genCode = false 1313 | if EnableLocalization then 1314 | OrderedForeach( 1315 | t, 1316 | function( id, _record ) 1317 | if type( _record ) == "table" then 1318 | localized = LocalizeRecord( id, _record, genCode, StringBank ) or localized 1319 | end 1320 | end 1321 | ) 1322 | end 1323 | local tableRef = nil 1324 | local defaultValues = nil 1325 | if EnableDatasetOptimize then 1326 | defaultValues = OptimizeDataset( t ) 1327 | if defaultValues then 1328 | local removeDefaultValues = function( record ) 1329 | local removes = nil 1330 | local adds = nil 1331 | for field, defaultVal in pairs( defaultValues ) do 1332 | local value = record[ field ] 1333 | local hasValue = true 1334 | if value == nil then 1335 | assert( false, "OptimizeDataset should patch all missing fields!" ) 1336 | hasValue = false 1337 | end 1338 | if value == defaultVal and hasValue then 1339 | removes = removes or {} 1340 | removes[ #removes + 1 ] = field 1341 | else 1342 | adds = adds or {} 1343 | adds[ field ] = value 1344 | end 1345 | end 1346 | -- remove fields with default value 1347 | if removes then 1348 | for _, f in ipairs( removes ) do 1349 | record[ f ] = nil 1350 | end 1351 | end 1352 | -- patch fields with none-default value 1353 | if adds then 1354 | for f, v in pairs( adds ) do 1355 | record[ f ] = v 1356 | end 1357 | end 1358 | end 1359 | local removed = {} 1360 | for _, record in pairs( t ) do 1361 | if type( record ) == "table" then 1362 | if not removed[ record ] then 1363 | removeDefaultValues( record ) 1364 | removed[ record ] = true 1365 | end 1366 | end 1367 | end 1368 | end 1369 | local reftables = nil 1370 | local ptr2ref = nil 1371 | -- create ref table: table -> refname 1372 | for _, hash in pairs( UniquifyTablesIds ) do 1373 | local t = UniquifyTables[ hash ] 1374 | if t then 1375 | local refName = ToUniqueTableRefName( UniquifyTablesInvIds[ t ] ) 1376 | reftables = reftables or {} 1377 | ptr2ref = ptr2ref or {} 1378 | reftables[ refName ] = t 1379 | ptr2ref[ t ] = refName 1380 | end 1381 | end 1382 | tableRef = { 1383 | name2value = reftables, 1384 | tables = UniquifyTables, -- hash -> table 1385 | tableIds = UniquifyTablesIds, -- id -> hash 1386 | ptr2ref = ptr2ref, -- table -> refname 1387 | refcounter = UniquifyTablesRefCounter, -- table -> refcount 1388 | maxLocalVariableNum = MaxLocalVariableNum, 1389 | refTableName = RefTableName, 1390 | postOutput = function( outFile ) 1391 | if defaultValues then 1392 | outFile.write( 1393 | string.format( 1394 | "local %s = %s\n", 1395 | DefaultValueTableName, 1396 | SerializeTable( defaultValues, false, false, ptr2ref ) 1397 | ) 1398 | ) 1399 | outFile.write( "do\n" ) 1400 | outFile.write( string.format( "\tlocal base = { __index = %s, __newindex = function() error( \"Attempt to modify read-only table\" ) end }\n", DefaultValueTableName ) ) 1401 | outFile.write( string.format( "\tfor k, v in pairs( %s ) do\n", datasetName ) ) 1402 | outFile.write( "\t\tsetmetatable( v, base )\n" ) 1403 | outFile.write( "\tend\n" ) 1404 | outFile.write( "\tbase.__metatable = false\n" ) 1405 | outFile.write( "end\n" ) 1406 | end 1407 | end 1408 | } 1409 | end 1410 | return t, tableRef, localized 1411 | end 1412 | 1413 | --tofile: not output to file, just for debug 1414 | --newStringBank: if false, exporter will use existing string hash for increamental building 1415 | local function ExportDatabaseLocalText( tofile, newStringBank ) 1416 | local StringBank = nil 1417 | if newStringBank then 1418 | StringBank = {} 1419 | else 1420 | StringBank = LoadStringBankFromCSV() 1421 | end 1422 | StringBank = StringBank or {} 1423 | local localized_dirty = false 1424 | local files = GetAllFileNamesAtPath( DatabaseRoot ) 1425 | for _, v in ipairs( files ) do 1426 | if not ExcludedFiles[ v ] then 1427 | print( "LoadDataset :"..v ) 1428 | LoadDataset( v ) 1429 | local t = Database[ v ] 1430 | local localized = false 1431 | if t then 1432 | local _t, tableRef, localized = ExportOptimizedDataset( t, StringBank ) 1433 | assert( _t == t ) 1434 | localized_dirty = localized_dirty or localized 1435 | SaveDatasetToFile( Database[ v ], tofile, tableRef ) 1436 | end 1437 | end 1438 | end 1439 | TrimStringBank( StringBank ) 1440 | if localized_dirty then 1441 | SaveStringBankToCSV( StringBank, tofile ) 1442 | else 1443 | print( "\nDatabase LocaleText is up to date.\n" ) 1444 | end 1445 | print( "Database Exporting LocaleText done." ) 1446 | end 1447 | 1448 | --[[ 1449 | local test = { 1450 | { 1451 | 1, 1452 | 2, 1453 | 3, 1454 | a = "123", 1455 | b = "123" 1456 | }, 1457 | { 1458 | 1, 1459 | 2, 1460 | 3, 1461 | a = "123", 1462 | b = "123" 1463 | }, 1464 | { 1465 | 1, 1466 | 2, 1467 | 5, 1468 | a = "123", 1469 | b = "123" 1470 | }, 1471 | [9] = { 1472 | 1, 1473 | 2, 1474 | 5, 1475 | a = "123", 1476 | b = "123" 1477 | }, 1478 | [100] = { 1479 | 1, 1480 | 2, 1481 | 3, 1482 | a = "tttt", 1483 | b = "123" 1484 | }, 1485 | [11] = { 1486 | 1, 1487 | 2, 1488 | 3, 1489 | a = "123", 1490 | b = "123", 1491 | c = { {1}, {1}, {2}, {2} }, 1492 | d = { { a = 1, 1 }, { a = 2, 2 } }, 1493 | e = { { a = 1, 1 }, { a = 2, 2 } }, 1494 | } 1495 | } 1496 | 1497 | EnableDefaultValueOptimize = false 1498 | local _localizedText = {} 1499 | local _src = SerializeTable( test ) 1500 | local _clone = DeserializeTable( _src ) 1501 | print( _src ) 1502 | local t, tableRef = ExportOptimizedDataset( test, _localizedText ) 1503 | assert( t == test ) 1504 | --print( SerializeTable( t ) ) 1505 | SaveDatasetToFile( t, false, tableRef, "test" ) 1506 | local _dst = SerializeTable( t ) 1507 | print( _dst ) 1508 | assert( _src == _dst ) 1509 | 1510 | EnableDefaultValueOptimize = true 1511 | _localizedText = {} 1512 | t, tableRef = ExportOptimizedDataset( _clone, _localizedText ) 1513 | assert( t == _clone ) 1514 | --_clone.__name = "__cloned_test" 1515 | SaveDatasetToFile( _clone, false, tableRef ) 1516 | local _dst = SerializeTable( _clone ) 1517 | print( _dst ) 1518 | print( _src ~= _dst ) 1519 | --]] 1520 | 1521 | --ExportDatabaseLocalText( false ) 1522 | 1523 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LuaTableOptimizer 2 | --- 3 | 4 | Simple readonly lua table optimizer 5 | --- 6 | 7 | Lua Table 通常被用来存储游戏的配置数据，如果配置中有很多冗余重复的数据那么将占用较多的内存，严重影响加载速度 8 | 9 | Lua table commonly use to store configuration data for games. it takes a lot of memory 10 | if it contains many fields with same value. this optimization could improve memory usage 11 | and loading speed. 12 | 13 | ### 功能 14 | * 获取字段中使用最多次数的值作为默认值，并且删除默认值字段 15 | * 使用了非ASCII字符集字符的字段被视为需要做多语言化处理并提取替换成特殊标识符 16 | * 唯一化所有的子table，并且指向同一个引用，以节约内存 17 | 18 | ### Features 19 | * remove default value fields ( store them into metatable ) 20 | * auto localization 21 | * reuse all table values to save memory 22 | 23 | ### Require 24 | * The key of root table must be all string or number type 25 | 26 | 27 | #### Before 28 | ```lua 29 | { 30 | { 31 | 1, 32 | 2, 33 | 3, 34 | a = "123", 35 | b = "123" 36 | }, 37 | { 38 | 1, 39 | 2, 40 | 3, 41 | a = "123", 42 | b = "123" 43 | }, 44 | { 45 | 1, 46 | 2, 47 | 5, 48 | a = "123", 49 | b = "123" 50 | }, 51 | [9] = { 52 | 1, 53 | 2, 54 | 5, 55 | a = "123", 56 | b = "123" 57 | }, 58 | [11] = { 59 | 1, 60 | 2, 61 | 3, 62 | a = "123", 63 | b = "123", 64 | c = { 65 | { 66 | 1 67 | }, 68 | { 69 | 1 70 | }, 71 | { 72 | 2 73 | }, 74 | { 75 | 2 76 | } 77 | }, 78 | d = { 79 | { 80 | 1, 81 | a = 1 82 | }, 83 | { 84 | 2, 85 | a = 2 86 | } 87 | }, 88 | e = { 89 | { 90 | 1, 91 | a = 1 92 | }, 93 | { 94 | 2, 95 | a = 2 96 | } 97 | } 98 | }, 99 | [100] = { 100 | 1, 101 | 2, 102 | 3, 103 | a = "tttt", 104 | b = "123" 105 | } 106 | } 107 | ``` 108 | 109 | #### Optimized 110 | ```lua 111 | local __rt_1 = { 112 | } 113 | local __rt_2 = { 114 | 1, 115 | 2, 116 | 3 117 | } 118 | local __rt_3 = { 119 | 1, 120 | 2, 121 | 5 122 | } 123 | local __rt_4 = { 124 | { 125 | 1, 126 | a = 1 127 | }, 128 | { 129 | 2, 130 | a = 2 131 | } 132 | } 133 | local __rt_5 = { 134 | 1 135 | } 136 | local __rt_6 = { 137 | 2 138 | } 139 | local test = 140 | { 141 | __rt_2, 142 | __rt_2, 143 | __rt_3, 144 | [9] = __rt_3, 145 | [11] = { 146 | 1, 147 | 2, 148 | 3, 149 | c = { 150 | __rt_5, 151 | __rt_5, 152 | __rt_6, 153 | __rt_6 154 | }, 155 | d = __rt_4, 156 | e = __rt_4 157 | }, 158 | [100] = { 159 | 1, 160 | 2, 161 | 3, 162 | a = "tttt" 163 | } 164 | } 165 | local __default_values = { 166 | a = "123", 167 | b = "123", 168 | c = __rt_1, 169 | d = __rt_1, 170 | e = __rt_1 171 | } 172 | do 173 | local base = { __index = __default_values, __newindex = function() error( "Attempt to modify read-only table" ) end } 174 | for k, v in pairs( test ) do 175 | setmetatable( v, base ) 176 | end 177 | base.__metatable = false 178 | end 179 | 180 | return test 181 | ``` 182 | 183 | --------------------------------------------------------------------------------