├── .gitignore ├── README.md └── src ├── .clang-format ├── README.md ├── databus.cpp ├── redo_parse ├── .clang-format ├── .clang_complete ├── .deps │ ├── logical_elems.Po │ ├── physical_elems.Po │ ├── redofile.Po │ ├── worker_test.Po │ └── workers.Po ├── Makefile ├── Makefile.sun ├── applier.cpp ├── applier.h ├── demo.py ├── dump_op.cpp ├── easylogging++.h ├── example.conf ├── logging.conf ├── logical_elems.cpp ├── logical_elems.h ├── metadata.cpp ├── metadata.h ├── mine.sql ├── mine_tab.sql ├── mine_xid.sql ├── monitor.cpp ├── monitor.h ├── opcode.h ├── opcode_ops.cpp ├── opcode_ops.h ├── ora_num_dis.py ├── otlv4.h ├── physical_elems.cpp ├── physical_elems.h ├── pk.sql ├── redofile.cpp ├── redofile.h ├── stream.cpp ├── stream.h ├── stream_error.h ├── t40.sql ├── table.conf ├── tconvert.cpp ├── tconvert.h ├── test_case.sql ├── trans.cpp ├── trans.h ├── worker_test.cpp ├── workers.cpp └── workers.h └── util ├── container.h ├── dassert.h ├── dtypes.h ├── logger.h └── utils.h /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.txt 3 | *.o 4 | *databus 5 | *.log 6 | *.a 7 | *.bin 8 | compile 9 | depcomp 10 | missing 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | DataStream README 2 | 3 | Welcome to DataStream! 4 | 5 | DataStream is designed to caputre recently changed oracle data in almost real time. The purpose is similar with Goldgate. 6 | 7 | DataStream 0.0.1 has been relased at 27/May/15. 8 | 9 | ### Release note of v0.0.1. 10 | 11 | ##### Supports 12 | 13 | * DataStream can parse archive log directly and store the changed PK into another database. 14 | * Data type char, varchar2, date, number as primary key are supported 15 | * Mulit Instances for single source are supported. Datastream is running with single thread in this release, so it has some performance issue. Mining a 2G archive log needs 2 minutes, applying it needs 3 ~ 6 minutes. 16 | 17 | ##### Limitations 18 | 19 | * The table must has primary key. The data type of primary key must be in [NUMBER, CHAR, VARCHAR2, DATE]. Or else Datastream will give warning and ignore this table. 20 | * Support common add/drop columns, but primary key definition change is not supported 21 | * If Adding/Swaping/drop partition on a partitioned table, some data may be lost. Mainly is because of DDL is not tracked. see details 22 | * If the table have more than 250 columns, program will exit. If we run into this in real case, we will verify it more before using it. why 23 | * Performance issue. Normally Single instance can handle 300+ tps, if we set the commit_logging to batch, the performance can increased by 1.5x. 24 | 25 | ### Notes: 26 | * This is an internal/dead project and we do not support it anymore 27 | * Put it here just in case it may helpful for any people 28 | -------------------------------------------------------------------------------- /src/.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | # BasedOnStyle: Google 3 | AccessModifierOffset: -1 4 | ConstructorInitializerIndentWidth: 4 5 | AlignEscapedNewlinesLeft: true 6 | AlignTrailingComments: true 7 | AllowAllParametersOfDeclarationOnNextLine: true 8 | AllowShortIfStatementsOnASingleLine: true 9 | AllowShortLoopsOnASingleLine: true 10 | AlwaysBreakTemplateDeclarations: true 11 | AlwaysBreakBeforeMultilineStrings: true 12 | BreakBeforeBinaryOperators: false 13 | BreakBeforeTernaryOperators: true 14 | BreakConstructorInitializersBeforeComma: false 15 | BinPackParameters: true 16 | ColumnLimit: 80 17 | ConstructorInitializerAllOnOneLineOrOnePerLine: true 18 | DerivePointerBinding: true 19 | ExperimentalAutoDetectBinPacking: false 20 | IndentCaseLabels: true 21 | MaxEmptyLinesToKeep: 1 22 | NamespaceIndentation: None 23 | ObjCSpaceBeforeProtocolList: false 24 | PenaltyBreakBeforeFirstCallParameter: 1 25 | PenaltyBreakComment: 60 26 | PenaltyBreakString: 1000 27 | PenaltyBreakFirstLessLess: 120 28 | PenaltyExcessCharacter: 1000000 29 | PenaltyReturnTypeOnItsOwnLine: 200 30 | PointerBindsToType: true 31 | SpacesBeforeTrailingComments: 2 32 | Cpp11BracedListStyle: true 33 | Standard: Auto 34 | IndentWidth: 2 35 | TabWidth: 8 36 | UseTab: Never 37 | BreakBeforeBraces: Attach 38 | IndentFunctionDeclarationAfterType: true 39 | SpacesInParentheses: false 40 | SpacesInAngles: false 41 | SpaceInEmptyParentheses: false 42 | SpacesInCStyleCastParentheses: false 43 | SpaceAfterControlStatementKeyword: true 44 | SpaceBeforeAssignmentOperators: true 45 | ContinuationIndentWidth: 4 46 | NamespaceIndentation : All 47 | ... 48 | 49 | -------------------------------------------------------------------------------- /src/README.md: -------------------------------------------------------------------------------- 1 | Known Issues: 2 | --------------- 3 | * 4 | -------------------------------------------------------------------------------- /src/databus.cpp: -------------------------------------------------------------------------------- 1 | #include "util/container.h" 2 | #include "redo_parse/logical_elems.h" 3 | #include "redo_parse/redofile.h" 4 | // #include "redo_parse/workers.h" 5 | 6 | #include 7 | #include 8 | 9 | namespace databus { 10 | namespace buffers { 11 | List record_start_positions; 12 | List record_buffer_list; 13 | List transaction_list; 14 | 15 | // Monitor buffer usage every 60 seconds 16 | void ReportBuffUsage() { 17 | while (true) { 18 | ReportList(record_start_positions, "RecordPosisions "); 19 | ReportList(record_buffer_list, "RecordBuffers "); 20 | ReportList(transaction_list, "TranactionList "); 21 | std::this_thread::sleep_for(std::chrono::seconds(5)); 22 | } 23 | } 24 | } 25 | 26 | int main(int argc, char** argv) { 27 | // buffers::ReportBuffUsage(); 28 | RedoFile f(argv[1]); 29 | const char* record = f.firstRecord(); 30 | do { 31 | buffers::record_start_positions.push_back(record); 32 | } while ((record = f.nextRecord(record)) != NULL); 33 | 34 | std::cout << "totol record count " << buffers::record_start_positions.size() 35 | << std::endl; 36 | for (auto i : f.block_record_count_) { 37 | std::cout << "block id " << i.first << "count " << i.second << std::endl; 38 | } 39 | return 0; 40 | } 41 | } 42 | 43 | int main(int argc, char** argv) { return databus::main(argc, argv); } 44 | -------------------------------------------------------------------------------- /src/redo_parse/.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | # BasedOnStyle: Google 3 | AccessModifierOffset: -1 4 | ConstructorInitializerIndentWidth: 4 5 | AlignEscapedNewlinesLeft: true 6 | AlignTrailingComments: true 7 | AllowAllParametersOfDeclarationOnNextLine: true 8 | AllowShortIfStatementsOnASingleLine: true 9 | AllowShortLoopsOnASingleLine: true 10 | AlwaysBreakTemplateDeclarations: true 11 | AlwaysBreakBeforeMultilineStrings: true 12 | BreakBeforeBinaryOperators: false 13 | BreakBeforeTernaryOperators: true 14 | BreakConstructorInitializersBeforeComma: false 15 | BinPackParameters: true 16 | ColumnLimit: 80 17 | ConstructorInitializerAllOnOneLineOrOnePerLine: true 18 | DerivePointerBinding: true 19 | ExperimentalAutoDetectBinPacking: false 20 | IndentCaseLabels: true 21 | MaxEmptyLinesToKeep: 1 22 | NamespaceIndentation: None 23 | ObjCSpaceBeforeProtocolList: false 24 | PenaltyBreakBeforeFirstCallParameter: 1 25 | PenaltyBreakComment: 60 26 | PenaltyBreakString: 1000 27 | PenaltyBreakFirstLessLess: 120 28 | PenaltyExcessCharacter: 1000000 29 | PenaltyReturnTypeOnItsOwnLine: 200 30 | PointerBindsToType: true 31 | SpacesBeforeTrailingComments: 2 32 | Cpp11BracedListStyle: true 33 | Standard: Auto 34 | IndentWidth: 2 35 | TabWidth: 8 36 | UseTab: Never 37 | BreakBeforeBraces: Attach 38 | IndentFunctionDeclarationAfterType: true 39 | SpacesInParentheses: false 40 | SpacesInAngles: false 41 | SpaceInEmptyParentheses: false 42 | SpacesInCStyleCastParentheses: false 43 | SpaceAfterControlStatementKeyword: true 44 | SpaceBeforeAssignmentOperators: true 45 | ContinuationIndentWidth: 4 46 | NamespaceIndentation : All 47 | ... 48 | 49 | -------------------------------------------------------------------------------- /src/redo_parse/.clang_complete: -------------------------------------------------------------------------------- 1 | -I.. 2 | -I. 3 | -I/usr/include/c++/4.2.1 4 | -I/Users/zhifan/Downloads/instantclient_11_2/sdk/include 5 | -------------------------------------------------------------------------------- /src/redo_parse/.deps/logical_elems.Po: -------------------------------------------------------------------------------- 1 | logical_elems.o: logical_elems.cpp logical_elems.h \ 2 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/string \ 3 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__config \ 4 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/iosfwd \ 5 | /usr/include/wchar.h /usr/include/_types.h /usr/include/sys/_types.h \ 6 | /usr/include/sys/cdefs.h /usr/include/sys/_symbol_aliasing.h \ 7 | /usr/include/sys/_posix_availability.h /usr/include/machine/_types.h \ 8 | /usr/include/i386/_types.h /usr/include/Availability.h \ 9 | /usr/include/AvailabilityInternal.h /usr/include/sys/_types/_null.h \ 10 | /usr/include/sys/_types/_size_t.h /usr/include/sys/_types/_mbstate_t.h \ 11 | /usr/include/sys/_types/_ct_rune_t.h /usr/include/sys/_types/_rune_t.h \ 12 | /usr/include/sys/_types/_wchar_t.h \ 13 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/clang/5.1/include/stdarg.h \ 14 | /usr/include/stdio.h /usr/include/sys/_types/_va_list.h \ 15 | /usr/include/sys/_types/_off_t.h /usr/include/sys/_types/_ssize_t.h \ 16 | /usr/include/time.h /usr/include/_structs.h \ 17 | /usr/include/sys/_structs.h /usr/include/sys/_types/_timespec.h \ 18 | /usr/include/sys/_types/_clock_t.h /usr/include/sys/_types/_time_t.h \ 19 | /usr/include/_wctype.h /usr/include/sys/_types/_wint_t.h \ 20 | /usr/include/_types/_wctype_t.h /usr/include/ctype.h \ 21 | /usr/include/runetype.h \ 22 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstring \ 23 | /usr/include/string.h /usr/include/strings.h \ 24 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstdio \ 25 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cwchar \ 26 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cwctype \ 27 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cctype \ 28 | /usr/include/wctype.h /usr/include/_types/_wctrans_t.h \ 29 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/algorithm \ 30 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/initializer_list \ 31 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstddef \ 32 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/clang/5.1/include/stddef.h \ 33 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/type_traits \ 34 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/utility \ 35 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__tuple \ 36 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/memory \ 37 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/typeinfo \ 38 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/exception \ 39 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstdint \ 40 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/clang/5.1/include/stdint.h \ 41 | /usr/include/stdint.h /usr/include/sys/_types/_int8_t.h \ 42 | /usr/include/sys/_types/_int16_t.h /usr/include/sys/_types/_int32_t.h \ 43 | /usr/include/sys/_types/_int64_t.h /usr/include/_types/_uint8_t.h \ 44 | /usr/include/_types/_uint16_t.h /usr/include/_types/_uint32_t.h \ 45 | /usr/include/_types/_uint64_t.h /usr/include/sys/_types/_intptr_t.h \ 46 | /usr/include/sys/_types/_uintptr_t.h /usr/include/_types/_intmax_t.h \ 47 | /usr/include/_types/_uintmax_t.h \ 48 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/new \ 49 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/limits \ 50 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__undef_min_max \ 51 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/iterator \ 52 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__functional_base \ 53 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/tuple \ 54 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/atomic \ 55 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/stdexcept \ 56 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/sstream \ 57 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/ostream \ 58 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/ios \ 59 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__locale \ 60 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/mutex \ 61 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__mutex_base \ 62 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/chrono \ 63 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/ctime \ 64 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/ratio \ 65 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/climits \ 66 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/clang/5.1/include/limits.h \ 67 | /usr/include/limits.h /usr/include/machine/limits.h \ 68 | /usr/include/i386/limits.h /usr/include/i386/_limits.h \ 69 | /usr/include/sys/syslimits.h \ 70 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/system_error \ 71 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cerrno \ 72 | /usr/include/errno.h /usr/include/sys/errno.h /usr/include/pthread.h \ 73 | /usr/include/pthread_impl.h /usr/include/sched.h \ 74 | /usr/include/sys/_types/_pthread_attr_t.h \ 75 | /usr/include/sys/_types/_pthread_cond_t.h \ 76 | /usr/include/sys/_types/_pthread_condattr_t.h \ 77 | /usr/include/sys/_types/_pthread_key_t.h \ 78 | /usr/include/sys/_types/_pthread_mutex_t.h \ 79 | /usr/include/sys/_types/_pthread_mutexattr_t.h \ 80 | /usr/include/sys/_types/_pthread_once_t.h \ 81 | /usr/include/sys/_types/_pthread_rwlock_t.h \ 82 | /usr/include/sys/_types/_pthread_rwlockattr_t.h \ 83 | /usr/include/sys/_types/_pthread_t.h \ 84 | /usr/include/sys/_types/_mach_port_t.h \ 85 | /usr/include/sys/_types/_sigset_t.h \ 86 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/functional \ 87 | /usr/include/locale.h /usr/include/_locale.h /usr/include/xlocale.h \ 88 | /usr/include/_xlocale.h /usr/include/xlocale/_ctype.h \ 89 | /usr/include/xlocale/__wctype.h /usr/include/xlocale/_stdio.h \ 90 | /usr/include/xlocale/_string.h /usr/include/xlocale/_time.h \ 91 | /usr/include/xlocale/_wchar.h /usr/include/xlocale/_wctype.h \ 92 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/streambuf \ 93 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/locale \ 94 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstdlib \ 95 | /usr/include/stdlib.h /usr/include/sys/wait.h \ 96 | /usr/include/sys/_types/_pid_t.h /usr/include/sys/_types/_id_t.h \ 97 | /usr/include/sys/signal.h /usr/include/sys/appleapiopts.h \ 98 | /usr/include/machine/signal.h /usr/include/i386/signal.h \ 99 | /usr/include/machine/_mcontext.h /usr/include/i386/_mcontext.h \ 100 | /usr/include/mach/i386/_structs.h \ 101 | /usr/include/sys/_types/_sigaltstack.h \ 102 | /usr/include/sys/_types/_ucontext.h /usr/include/sys/_types/_uid_t.h \ 103 | /usr/include/sys/resource.h /usr/include/sys/_types/_timeval.h \ 104 | /usr/include/machine/endian.h /usr/include/i386/endian.h \ 105 | /usr/include/sys/_endian.h /usr/include/libkern/_OSByteOrder.h \ 106 | /usr/include/libkern/i386/_OSByteOrder.h /usr/include/alloca.h \ 107 | /usr/include/machine/types.h /usr/include/i386/types.h \ 108 | /usr/include/sys/_types/___offsetof.h /usr/include/sys/_types/_dev_t.h \ 109 | /usr/include/sys/_types/_mode_t.h /usr/include/xlocale/_stdlib.h \ 110 | /usr/include/nl_types.h /usr/include/sys/types.h \ 111 | /usr/include/sys/_types/_blkcnt_t.h \ 112 | /usr/include/sys/_types/_blksize_t.h /usr/include/sys/_types/_gid_t.h \ 113 | /usr/include/sys/_types/_in_addr_t.h \ 114 | /usr/include/sys/_types/_in_port_t.h /usr/include/sys/_types/_ino_t.h \ 115 | /usr/include/sys/_types/_ino64_t.h /usr/include/sys/_types/_key_t.h \ 116 | /usr/include/sys/_types/_nlink_t.h \ 117 | /usr/include/sys/_types/_useconds_t.h \ 118 | /usr/include/sys/_types/_suseconds_t.h \ 119 | /usr/include/sys/_types/_rsize_t.h /usr/include/sys/_types/_errno_t.h \ 120 | /usr/include/sys/_types/_fd_def.h \ 121 | /usr/include/sys/_types/_fd_setsize.h \ 122 | /usr/include/sys/_types/_fd_set.h /usr/include/sys/_types/_fd_clr.h \ 123 | /usr/include/sys/_types/_fd_zero.h /usr/include/sys/_types/_fd_isset.h \ 124 | /usr/include/sys/_types/_fd_copy.h \ 125 | /usr/include/sys/_types/_fsblkcnt_t.h \ 126 | /usr/include/sys/_types/_fsfilcnt_t.h /usr/include/_types/_nl_item.h \ 127 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/bitset \ 128 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__bit_reference \ 129 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/istream \ 130 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/map \ 131 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__tree \ 132 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/list \ 133 | ../util/dtypes.h physical_elems.h \ 134 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/set \ 135 | ../util/dassert.h \ 136 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/iostream 137 | 138 | logical_elems.h: 139 | 140 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/string: 141 | 142 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__config: 143 | 144 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/iosfwd: 145 | 146 | /usr/include/wchar.h: 147 | 148 | /usr/include/_types.h: 149 | 150 | /usr/include/sys/_types.h: 151 | 152 | /usr/include/sys/cdefs.h: 153 | 154 | /usr/include/sys/_symbol_aliasing.h: 155 | 156 | /usr/include/sys/_posix_availability.h: 157 | 158 | /usr/include/machine/_types.h: 159 | 160 | /usr/include/i386/_types.h: 161 | 162 | /usr/include/Availability.h: 163 | 164 | /usr/include/AvailabilityInternal.h: 165 | 166 | /usr/include/sys/_types/_null.h: 167 | 168 | /usr/include/sys/_types/_size_t.h: 169 | 170 | /usr/include/sys/_types/_mbstate_t.h: 171 | 172 | /usr/include/sys/_types/_ct_rune_t.h: 173 | 174 | /usr/include/sys/_types/_rune_t.h: 175 | 176 | /usr/include/sys/_types/_wchar_t.h: 177 | 178 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/clang/5.1/include/stdarg.h: 179 | 180 | /usr/include/stdio.h: 181 | 182 | /usr/include/sys/_types/_va_list.h: 183 | 184 | /usr/include/sys/_types/_off_t.h: 185 | 186 | /usr/include/sys/_types/_ssize_t.h: 187 | 188 | /usr/include/time.h: 189 | 190 | /usr/include/_structs.h: 191 | 192 | /usr/include/sys/_structs.h: 193 | 194 | /usr/include/sys/_types/_timespec.h: 195 | 196 | /usr/include/sys/_types/_clock_t.h: 197 | 198 | /usr/include/sys/_types/_time_t.h: 199 | 200 | /usr/include/_wctype.h: 201 | 202 | /usr/include/sys/_types/_wint_t.h: 203 | 204 | /usr/include/_types/_wctype_t.h: 205 | 206 | /usr/include/ctype.h: 207 | 208 | /usr/include/runetype.h: 209 | 210 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstring: 211 | 212 | /usr/include/string.h: 213 | 214 | /usr/include/strings.h: 215 | 216 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstdio: 217 | 218 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cwchar: 219 | 220 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cwctype: 221 | 222 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cctype: 223 | 224 | /usr/include/wctype.h: 225 | 226 | /usr/include/_types/_wctrans_t.h: 227 | 228 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/algorithm: 229 | 230 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/initializer_list: 231 | 232 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstddef: 233 | 234 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/clang/5.1/include/stddef.h: 235 | 236 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/type_traits: 237 | 238 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/utility: 239 | 240 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__tuple: 241 | 242 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/memory: 243 | 244 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/typeinfo: 245 | 246 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/exception: 247 | 248 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstdint: 249 | 250 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/clang/5.1/include/stdint.h: 251 | 252 | /usr/include/stdint.h: 253 | 254 | /usr/include/sys/_types/_int8_t.h: 255 | 256 | /usr/include/sys/_types/_int16_t.h: 257 | 258 | /usr/include/sys/_types/_int32_t.h: 259 | 260 | /usr/include/sys/_types/_int64_t.h: 261 | 262 | /usr/include/_types/_uint8_t.h: 263 | 264 | /usr/include/_types/_uint16_t.h: 265 | 266 | /usr/include/_types/_uint32_t.h: 267 | 268 | /usr/include/_types/_uint64_t.h: 269 | 270 | /usr/include/sys/_types/_intptr_t.h: 271 | 272 | /usr/include/sys/_types/_uintptr_t.h: 273 | 274 | /usr/include/_types/_intmax_t.h: 275 | 276 | /usr/include/_types/_uintmax_t.h: 277 | 278 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/new: 279 | 280 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/limits: 281 | 282 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__undef_min_max: 283 | 284 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/iterator: 285 | 286 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__functional_base: 287 | 288 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/tuple: 289 | 290 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/atomic: 291 | 292 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/stdexcept: 293 | 294 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/sstream: 295 | 296 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/ostream: 297 | 298 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/ios: 299 | 300 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__locale: 301 | 302 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/mutex: 303 | 304 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__mutex_base: 305 | 306 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/chrono: 307 | 308 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/ctime: 309 | 310 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/ratio: 311 | 312 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/climits: 313 | 314 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/clang/5.1/include/limits.h: 315 | 316 | /usr/include/limits.h: 317 | 318 | /usr/include/machine/limits.h: 319 | 320 | /usr/include/i386/limits.h: 321 | 322 | /usr/include/i386/_limits.h: 323 | 324 | /usr/include/sys/syslimits.h: 325 | 326 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/system_error: 327 | 328 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cerrno: 329 | 330 | /usr/include/errno.h: 331 | 332 | /usr/include/sys/errno.h: 333 | 334 | /usr/include/pthread.h: 335 | 336 | /usr/include/pthread_impl.h: 337 | 338 | /usr/include/sched.h: 339 | 340 | /usr/include/sys/_types/_pthread_attr_t.h: 341 | 342 | /usr/include/sys/_types/_pthread_cond_t.h: 343 | 344 | /usr/include/sys/_types/_pthread_condattr_t.h: 345 | 346 | /usr/include/sys/_types/_pthread_key_t.h: 347 | 348 | /usr/include/sys/_types/_pthread_mutex_t.h: 349 | 350 | /usr/include/sys/_types/_pthread_mutexattr_t.h: 351 | 352 | /usr/include/sys/_types/_pthread_once_t.h: 353 | 354 | /usr/include/sys/_types/_pthread_rwlock_t.h: 355 | 356 | /usr/include/sys/_types/_pthread_rwlockattr_t.h: 357 | 358 | /usr/include/sys/_types/_pthread_t.h: 359 | 360 | /usr/include/sys/_types/_mach_port_t.h: 361 | 362 | /usr/include/sys/_types/_sigset_t.h: 363 | 364 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/functional: 365 | 366 | /usr/include/locale.h: 367 | 368 | /usr/include/_locale.h: 369 | 370 | /usr/include/xlocale.h: 371 | 372 | /usr/include/_xlocale.h: 373 | 374 | /usr/include/xlocale/_ctype.h: 375 | 376 | /usr/include/xlocale/__wctype.h: 377 | 378 | /usr/include/xlocale/_stdio.h: 379 | 380 | /usr/include/xlocale/_string.h: 381 | 382 | /usr/include/xlocale/_time.h: 383 | 384 | /usr/include/xlocale/_wchar.h: 385 | 386 | /usr/include/xlocale/_wctype.h: 387 | 388 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/streambuf: 389 | 390 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/locale: 391 | 392 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstdlib: 393 | 394 | /usr/include/stdlib.h: 395 | 396 | /usr/include/sys/wait.h: 397 | 398 | /usr/include/sys/_types/_pid_t.h: 399 | 400 | /usr/include/sys/_types/_id_t.h: 401 | 402 | /usr/include/sys/signal.h: 403 | 404 | /usr/include/sys/appleapiopts.h: 405 | 406 | /usr/include/machine/signal.h: 407 | 408 | /usr/include/i386/signal.h: 409 | 410 | /usr/include/machine/_mcontext.h: 411 | 412 | /usr/include/i386/_mcontext.h: 413 | 414 | /usr/include/mach/i386/_structs.h: 415 | 416 | /usr/include/sys/_types/_sigaltstack.h: 417 | 418 | /usr/include/sys/_types/_ucontext.h: 419 | 420 | /usr/include/sys/_types/_uid_t.h: 421 | 422 | /usr/include/sys/resource.h: 423 | 424 | /usr/include/sys/_types/_timeval.h: 425 | 426 | /usr/include/machine/endian.h: 427 | 428 | /usr/include/i386/endian.h: 429 | 430 | /usr/include/sys/_endian.h: 431 | 432 | /usr/include/libkern/_OSByteOrder.h: 433 | 434 | /usr/include/libkern/i386/_OSByteOrder.h: 435 | 436 | /usr/include/alloca.h: 437 | 438 | /usr/include/machine/types.h: 439 | 440 | /usr/include/i386/types.h: 441 | 442 | /usr/include/sys/_types/___offsetof.h: 443 | 444 | /usr/include/sys/_types/_dev_t.h: 445 | 446 | /usr/include/sys/_types/_mode_t.h: 447 | 448 | /usr/include/xlocale/_stdlib.h: 449 | 450 | /usr/include/nl_types.h: 451 | 452 | /usr/include/sys/types.h: 453 | 454 | /usr/include/sys/_types/_blkcnt_t.h: 455 | 456 | /usr/include/sys/_types/_blksize_t.h: 457 | 458 | /usr/include/sys/_types/_gid_t.h: 459 | 460 | /usr/include/sys/_types/_in_addr_t.h: 461 | 462 | /usr/include/sys/_types/_in_port_t.h: 463 | 464 | /usr/include/sys/_types/_ino_t.h: 465 | 466 | /usr/include/sys/_types/_ino64_t.h: 467 | 468 | /usr/include/sys/_types/_key_t.h: 469 | 470 | /usr/include/sys/_types/_nlink_t.h: 471 | 472 | /usr/include/sys/_types/_useconds_t.h: 473 | 474 | /usr/include/sys/_types/_suseconds_t.h: 475 | 476 | /usr/include/sys/_types/_rsize_t.h: 477 | 478 | /usr/include/sys/_types/_errno_t.h: 479 | 480 | /usr/include/sys/_types/_fd_def.h: 481 | 482 | /usr/include/sys/_types/_fd_setsize.h: 483 | 484 | /usr/include/sys/_types/_fd_set.h: 485 | 486 | /usr/include/sys/_types/_fd_clr.h: 487 | 488 | /usr/include/sys/_types/_fd_zero.h: 489 | 490 | /usr/include/sys/_types/_fd_isset.h: 491 | 492 | /usr/include/sys/_types/_fd_copy.h: 493 | 494 | /usr/include/sys/_types/_fsblkcnt_t.h: 495 | 496 | /usr/include/sys/_types/_fsfilcnt_t.h: 497 | 498 | /usr/include/_types/_nl_item.h: 499 | 500 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/bitset: 501 | 502 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__bit_reference: 503 | 504 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/istream: 505 | 506 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/map: 507 | 508 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__tree: 509 | 510 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/list: 511 | 512 | ../util/dtypes.h: 513 | 514 | physical_elems.h: 515 | 516 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/set: 517 | 518 | ../util/dassert.h: 519 | 520 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/iostream: 521 | -------------------------------------------------------------------------------- /src/redo_parse/.deps/physical_elems.Po: -------------------------------------------------------------------------------- 1 | physical_elems.o: physical_elems.cpp physical_elems.h \ 2 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/clang/5.1/include/stdint.h \ 3 | /usr/include/stdint.h /usr/include/sys/_types/_int8_t.h \ 4 | /usr/include/sys/_types/_int16_t.h /usr/include/sys/_types/_int32_t.h \ 5 | /usr/include/sys/_types/_int64_t.h /usr/include/_types/_uint8_t.h \ 6 | /usr/include/_types/_uint16_t.h /usr/include/_types/_uint32_t.h \ 7 | /usr/include/_types/_uint64_t.h /usr/include/sys/_types.h \ 8 | /usr/include/sys/cdefs.h /usr/include/sys/_symbol_aliasing.h \ 9 | /usr/include/sys/_posix_availability.h /usr/include/machine/_types.h \ 10 | /usr/include/i386/_types.h /usr/include/sys/_types/_intptr_t.h \ 11 | /usr/include/sys/_types/_uintptr_t.h /usr/include/_types/_intmax_t.h \ 12 | /usr/include/_types/_uintmax_t.h \ 13 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/set \ 14 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__config \ 15 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__tree \ 16 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/iterator \ 17 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/type_traits \ 18 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstddef \ 19 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/clang/5.1/include/stddef.h \ 20 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/iosfwd \ 21 | /usr/include/wchar.h /usr/include/_types.h /usr/include/Availability.h \ 22 | /usr/include/AvailabilityInternal.h /usr/include/sys/_types/_null.h \ 23 | /usr/include/sys/_types/_size_t.h /usr/include/sys/_types/_mbstate_t.h \ 24 | /usr/include/sys/_types/_ct_rune_t.h /usr/include/sys/_types/_rune_t.h \ 25 | /usr/include/sys/_types/_wchar_t.h \ 26 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/clang/5.1/include/stdarg.h \ 27 | /usr/include/stdio.h /usr/include/sys/_types/_va_list.h \ 28 | /usr/include/sys/_types/_off_t.h /usr/include/sys/_types/_ssize_t.h \ 29 | /usr/include/time.h /usr/include/_structs.h \ 30 | /usr/include/sys/_structs.h /usr/include/sys/_types/_timespec.h \ 31 | /usr/include/sys/_types/_clock_t.h /usr/include/sys/_types/_time_t.h \ 32 | /usr/include/_wctype.h /usr/include/sys/_types/_wint_t.h \ 33 | /usr/include/_types/_wctype_t.h /usr/include/ctype.h \ 34 | /usr/include/runetype.h \ 35 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/initializer_list \ 36 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/memory \ 37 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/typeinfo \ 38 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/exception \ 39 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstdint \ 40 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/new \ 41 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/utility \ 42 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__tuple \ 43 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/limits \ 44 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__undef_min_max \ 45 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__functional_base \ 46 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/tuple \ 47 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstring \ 48 | /usr/include/string.h /usr/include/strings.h \ 49 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/atomic \ 50 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/stdexcept \ 51 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/algorithm \ 52 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/functional \ 53 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/string \ 54 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstdio \ 55 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cwchar \ 56 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cwctype \ 57 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cctype \ 58 | /usr/include/wctype.h /usr/include/_types/_wctrans_t.h \ 59 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/sstream \ 60 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/ostream \ 61 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/ios \ 62 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__locale \ 63 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/mutex \ 64 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__mutex_base \ 65 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/chrono \ 66 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/ctime \ 67 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/ratio \ 68 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/climits \ 69 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/clang/5.1/include/limits.h \ 70 | /usr/include/limits.h /usr/include/machine/limits.h \ 71 | /usr/include/i386/limits.h /usr/include/i386/_limits.h \ 72 | /usr/include/sys/syslimits.h \ 73 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/system_error \ 74 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cerrno \ 75 | /usr/include/errno.h /usr/include/sys/errno.h /usr/include/pthread.h \ 76 | /usr/include/pthread_impl.h /usr/include/sched.h \ 77 | /usr/include/sys/_types/_pthread_attr_t.h \ 78 | /usr/include/sys/_types/_pthread_cond_t.h \ 79 | /usr/include/sys/_types/_pthread_condattr_t.h \ 80 | /usr/include/sys/_types/_pthread_key_t.h \ 81 | /usr/include/sys/_types/_pthread_mutex_t.h \ 82 | /usr/include/sys/_types/_pthread_mutexattr_t.h \ 83 | /usr/include/sys/_types/_pthread_once_t.h \ 84 | /usr/include/sys/_types/_pthread_rwlock_t.h \ 85 | /usr/include/sys/_types/_pthread_rwlockattr_t.h \ 86 | /usr/include/sys/_types/_pthread_t.h \ 87 | /usr/include/sys/_types/_mach_port_t.h \ 88 | /usr/include/sys/_types/_sigset_t.h /usr/include/locale.h \ 89 | /usr/include/_locale.h /usr/include/xlocale.h /usr/include/_xlocale.h \ 90 | /usr/include/xlocale/_ctype.h /usr/include/xlocale/__wctype.h \ 91 | /usr/include/xlocale/_stdio.h /usr/include/xlocale/_string.h \ 92 | /usr/include/xlocale/_time.h /usr/include/xlocale/_wchar.h \ 93 | /usr/include/xlocale/_wctype.h \ 94 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/streambuf \ 95 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/locale \ 96 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstdlib \ 97 | /usr/include/stdlib.h /usr/include/sys/wait.h \ 98 | /usr/include/sys/_types/_pid_t.h /usr/include/sys/_types/_id_t.h \ 99 | /usr/include/sys/signal.h /usr/include/sys/appleapiopts.h \ 100 | /usr/include/machine/signal.h /usr/include/i386/signal.h \ 101 | /usr/include/machine/_mcontext.h /usr/include/i386/_mcontext.h \ 102 | /usr/include/mach/i386/_structs.h \ 103 | /usr/include/sys/_types/_sigaltstack.h \ 104 | /usr/include/sys/_types/_ucontext.h /usr/include/sys/_types/_uid_t.h \ 105 | /usr/include/sys/resource.h /usr/include/sys/_types/_timeval.h \ 106 | /usr/include/machine/endian.h /usr/include/i386/endian.h \ 107 | /usr/include/sys/_endian.h /usr/include/libkern/_OSByteOrder.h \ 108 | /usr/include/libkern/i386/_OSByteOrder.h /usr/include/alloca.h \ 109 | /usr/include/machine/types.h /usr/include/i386/types.h \ 110 | /usr/include/sys/_types/___offsetof.h /usr/include/sys/_types/_dev_t.h \ 111 | /usr/include/sys/_types/_mode_t.h /usr/include/xlocale/_stdlib.h \ 112 | /usr/include/nl_types.h /usr/include/sys/types.h \ 113 | /usr/include/sys/_types/_blkcnt_t.h \ 114 | /usr/include/sys/_types/_blksize_t.h /usr/include/sys/_types/_gid_t.h \ 115 | /usr/include/sys/_types/_in_addr_t.h \ 116 | /usr/include/sys/_types/_in_port_t.h /usr/include/sys/_types/_ino_t.h \ 117 | /usr/include/sys/_types/_ino64_t.h /usr/include/sys/_types/_key_t.h \ 118 | /usr/include/sys/_types/_nlink_t.h \ 119 | /usr/include/sys/_types/_useconds_t.h \ 120 | /usr/include/sys/_types/_suseconds_t.h \ 121 | /usr/include/sys/_types/_rsize_t.h /usr/include/sys/_types/_errno_t.h \ 122 | /usr/include/sys/_types/_fd_def.h \ 123 | /usr/include/sys/_types/_fd_setsize.h \ 124 | /usr/include/sys/_types/_fd_set.h /usr/include/sys/_types/_fd_clr.h \ 125 | /usr/include/sys/_types/_fd_zero.h /usr/include/sys/_types/_fd_isset.h \ 126 | /usr/include/sys/_types/_fd_copy.h \ 127 | /usr/include/sys/_types/_fsblkcnt_t.h \ 128 | /usr/include/sys/_types/_fsfilcnt_t.h /usr/include/_types/_nl_item.h \ 129 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/bitset \ 130 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__bit_reference \ 131 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/istream \ 132 | ../util/dassert.h \ 133 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/iostream \ 134 | ../util/dtypes.h logical_elems.h \ 135 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/map \ 136 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/list 137 | 138 | physical_elems.h: 139 | 140 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/clang/5.1/include/stdint.h: 141 | 142 | /usr/include/stdint.h: 143 | 144 | /usr/include/sys/_types/_int8_t.h: 145 | 146 | /usr/include/sys/_types/_int16_t.h: 147 | 148 | /usr/include/sys/_types/_int32_t.h: 149 | 150 | /usr/include/sys/_types/_int64_t.h: 151 | 152 | /usr/include/_types/_uint8_t.h: 153 | 154 | /usr/include/_types/_uint16_t.h: 155 | 156 | /usr/include/_types/_uint32_t.h: 157 | 158 | /usr/include/_types/_uint64_t.h: 159 | 160 | /usr/include/sys/_types.h: 161 | 162 | /usr/include/sys/cdefs.h: 163 | 164 | /usr/include/sys/_symbol_aliasing.h: 165 | 166 | /usr/include/sys/_posix_availability.h: 167 | 168 | /usr/include/machine/_types.h: 169 | 170 | /usr/include/i386/_types.h: 171 | 172 | /usr/include/sys/_types/_intptr_t.h: 173 | 174 | /usr/include/sys/_types/_uintptr_t.h: 175 | 176 | /usr/include/_types/_intmax_t.h: 177 | 178 | /usr/include/_types/_uintmax_t.h: 179 | 180 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/set: 181 | 182 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__config: 183 | 184 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__tree: 185 | 186 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/iterator: 187 | 188 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/type_traits: 189 | 190 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstddef: 191 | 192 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/clang/5.1/include/stddef.h: 193 | 194 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/iosfwd: 195 | 196 | /usr/include/wchar.h: 197 | 198 | /usr/include/_types.h: 199 | 200 | /usr/include/Availability.h: 201 | 202 | /usr/include/AvailabilityInternal.h: 203 | 204 | /usr/include/sys/_types/_null.h: 205 | 206 | /usr/include/sys/_types/_size_t.h: 207 | 208 | /usr/include/sys/_types/_mbstate_t.h: 209 | 210 | /usr/include/sys/_types/_ct_rune_t.h: 211 | 212 | /usr/include/sys/_types/_rune_t.h: 213 | 214 | /usr/include/sys/_types/_wchar_t.h: 215 | 216 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/clang/5.1/include/stdarg.h: 217 | 218 | /usr/include/stdio.h: 219 | 220 | /usr/include/sys/_types/_va_list.h: 221 | 222 | /usr/include/sys/_types/_off_t.h: 223 | 224 | /usr/include/sys/_types/_ssize_t.h: 225 | 226 | /usr/include/time.h: 227 | 228 | /usr/include/_structs.h: 229 | 230 | /usr/include/sys/_structs.h: 231 | 232 | /usr/include/sys/_types/_timespec.h: 233 | 234 | /usr/include/sys/_types/_clock_t.h: 235 | 236 | /usr/include/sys/_types/_time_t.h: 237 | 238 | /usr/include/_wctype.h: 239 | 240 | /usr/include/sys/_types/_wint_t.h: 241 | 242 | /usr/include/_types/_wctype_t.h: 243 | 244 | /usr/include/ctype.h: 245 | 246 | /usr/include/runetype.h: 247 | 248 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/initializer_list: 249 | 250 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/memory: 251 | 252 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/typeinfo: 253 | 254 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/exception: 255 | 256 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstdint: 257 | 258 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/new: 259 | 260 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/utility: 261 | 262 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__tuple: 263 | 264 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/limits: 265 | 266 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__undef_min_max: 267 | 268 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__functional_base: 269 | 270 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/tuple: 271 | 272 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstring: 273 | 274 | /usr/include/string.h: 275 | 276 | /usr/include/strings.h: 277 | 278 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/atomic: 279 | 280 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/stdexcept: 281 | 282 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/algorithm: 283 | 284 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/functional: 285 | 286 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/string: 287 | 288 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstdio: 289 | 290 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cwchar: 291 | 292 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cwctype: 293 | 294 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cctype: 295 | 296 | /usr/include/wctype.h: 297 | 298 | /usr/include/_types/_wctrans_t.h: 299 | 300 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/sstream: 301 | 302 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/ostream: 303 | 304 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/ios: 305 | 306 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__locale: 307 | 308 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/mutex: 309 | 310 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__mutex_base: 311 | 312 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/chrono: 313 | 314 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/ctime: 315 | 316 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/ratio: 317 | 318 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/climits: 319 | 320 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/clang/5.1/include/limits.h: 321 | 322 | /usr/include/limits.h: 323 | 324 | /usr/include/machine/limits.h: 325 | 326 | /usr/include/i386/limits.h: 327 | 328 | /usr/include/i386/_limits.h: 329 | 330 | /usr/include/sys/syslimits.h: 331 | 332 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/system_error: 333 | 334 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cerrno: 335 | 336 | /usr/include/errno.h: 337 | 338 | /usr/include/sys/errno.h: 339 | 340 | /usr/include/pthread.h: 341 | 342 | /usr/include/pthread_impl.h: 343 | 344 | /usr/include/sched.h: 345 | 346 | /usr/include/sys/_types/_pthread_attr_t.h: 347 | 348 | /usr/include/sys/_types/_pthread_cond_t.h: 349 | 350 | /usr/include/sys/_types/_pthread_condattr_t.h: 351 | 352 | /usr/include/sys/_types/_pthread_key_t.h: 353 | 354 | /usr/include/sys/_types/_pthread_mutex_t.h: 355 | 356 | /usr/include/sys/_types/_pthread_mutexattr_t.h: 357 | 358 | /usr/include/sys/_types/_pthread_once_t.h: 359 | 360 | /usr/include/sys/_types/_pthread_rwlock_t.h: 361 | 362 | /usr/include/sys/_types/_pthread_rwlockattr_t.h: 363 | 364 | /usr/include/sys/_types/_pthread_t.h: 365 | 366 | /usr/include/sys/_types/_mach_port_t.h: 367 | 368 | /usr/include/sys/_types/_sigset_t.h: 369 | 370 | /usr/include/locale.h: 371 | 372 | /usr/include/_locale.h: 373 | 374 | /usr/include/xlocale.h: 375 | 376 | /usr/include/_xlocale.h: 377 | 378 | /usr/include/xlocale/_ctype.h: 379 | 380 | /usr/include/xlocale/__wctype.h: 381 | 382 | /usr/include/xlocale/_stdio.h: 383 | 384 | /usr/include/xlocale/_string.h: 385 | 386 | /usr/include/xlocale/_time.h: 387 | 388 | /usr/include/xlocale/_wchar.h: 389 | 390 | /usr/include/xlocale/_wctype.h: 391 | 392 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/streambuf: 393 | 394 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/locale: 395 | 396 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/cstdlib: 397 | 398 | /usr/include/stdlib.h: 399 | 400 | /usr/include/sys/wait.h: 401 | 402 | /usr/include/sys/_types/_pid_t.h: 403 | 404 | /usr/include/sys/_types/_id_t.h: 405 | 406 | /usr/include/sys/signal.h: 407 | 408 | /usr/include/sys/appleapiopts.h: 409 | 410 | /usr/include/machine/signal.h: 411 | 412 | /usr/include/i386/signal.h: 413 | 414 | /usr/include/machine/_mcontext.h: 415 | 416 | /usr/include/i386/_mcontext.h: 417 | 418 | /usr/include/mach/i386/_structs.h: 419 | 420 | /usr/include/sys/_types/_sigaltstack.h: 421 | 422 | /usr/include/sys/_types/_ucontext.h: 423 | 424 | /usr/include/sys/_types/_uid_t.h: 425 | 426 | /usr/include/sys/resource.h: 427 | 428 | /usr/include/sys/_types/_timeval.h: 429 | 430 | /usr/include/machine/endian.h: 431 | 432 | /usr/include/i386/endian.h: 433 | 434 | /usr/include/sys/_endian.h: 435 | 436 | /usr/include/libkern/_OSByteOrder.h: 437 | 438 | /usr/include/libkern/i386/_OSByteOrder.h: 439 | 440 | /usr/include/alloca.h: 441 | 442 | /usr/include/machine/types.h: 443 | 444 | /usr/include/i386/types.h: 445 | 446 | /usr/include/sys/_types/___offsetof.h: 447 | 448 | /usr/include/sys/_types/_dev_t.h: 449 | 450 | /usr/include/sys/_types/_mode_t.h: 451 | 452 | /usr/include/xlocale/_stdlib.h: 453 | 454 | /usr/include/nl_types.h: 455 | 456 | /usr/include/sys/types.h: 457 | 458 | /usr/include/sys/_types/_blkcnt_t.h: 459 | 460 | /usr/include/sys/_types/_blksize_t.h: 461 | 462 | /usr/include/sys/_types/_gid_t.h: 463 | 464 | /usr/include/sys/_types/_in_addr_t.h: 465 | 466 | /usr/include/sys/_types/_in_port_t.h: 467 | 468 | /usr/include/sys/_types/_ino_t.h: 469 | 470 | /usr/include/sys/_types/_ino64_t.h: 471 | 472 | /usr/include/sys/_types/_key_t.h: 473 | 474 | /usr/include/sys/_types/_nlink_t.h: 475 | 476 | /usr/include/sys/_types/_useconds_t.h: 477 | 478 | /usr/include/sys/_types/_suseconds_t.h: 479 | 480 | /usr/include/sys/_types/_rsize_t.h: 481 | 482 | /usr/include/sys/_types/_errno_t.h: 483 | 484 | /usr/include/sys/_types/_fd_def.h: 485 | 486 | /usr/include/sys/_types/_fd_setsize.h: 487 | 488 | /usr/include/sys/_types/_fd_set.h: 489 | 490 | /usr/include/sys/_types/_fd_clr.h: 491 | 492 | /usr/include/sys/_types/_fd_zero.h: 493 | 494 | /usr/include/sys/_types/_fd_isset.h: 495 | 496 | /usr/include/sys/_types/_fd_copy.h: 497 | 498 | /usr/include/sys/_types/_fsblkcnt_t.h: 499 | 500 | /usr/include/sys/_types/_fsfilcnt_t.h: 501 | 502 | /usr/include/_types/_nl_item.h: 503 | 504 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/bitset: 505 | 506 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/__bit_reference: 507 | 508 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/istream: 509 | 510 | ../util/dassert.h: 511 | 512 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/iostream: 513 | 514 | ../util/dtypes.h: 515 | 516 | logical_elems.h: 517 | 518 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/map: 519 | 520 | /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/../lib/c++/v1/list: 521 | -------------------------------------------------------------------------------- /src/redo_parse/Makefile: -------------------------------------------------------------------------------- 1 | CXXFLAGS_BASE=-I. -I.. -std=c++11 -g -O0 -pthread -D__STREAM_DEBUG__ -DELPP_THREAD_SAFE -DOTL_DESTRUCTORS_DO_NOT_THROW 2 | CC=g++ 3 | DEPS=metadata.h \ 4 | logical_elems.h \ 5 | opcode.h \ 6 | opcode_ops.h \ 7 | physical_elems.h \ 8 | redofile.h \ 9 | workers.h \ 10 | stream.h \ 11 | stream_error.h \ 12 | trans.h \ 13 | applier.h \ 14 | monitor.h 15 | 16 | OCI_FLAGS= 17 | 18 | LDFLAGS_BASE=-pthread 19 | OCI_FLAGS=-lclntsh 20 | 21 | PLATFORM=$(shell uname) 22 | 23 | ifeq ($(PLATFORM), Darwin) 24 | OCI_FALGS=-DMAC_OSX -D_GNU_SOURCE -D_REENTRANT -mmacosx-version-min=10.10 25 | CXXFLAGS = ${CXXFLAGS_BASE} -I/Users/zhifan/Downloads/instantclient_11_2/sdk/include/ 26 | LDFLAGS= ${LDFLAGS_BASE} -L/Users/zhifan/Downloads/instantclient_11_2 -lnnz11 27 | endif 28 | 29 | ifeq ($(PLATFORM), Linux) 30 | CXXFLAGS = ${CXXFLAGS_BASE} -I/home/oracle/instantclient_11_2/sdk/include/ -DBOOST_LOG_DYN_LINK 31 | # LDFLAGS=${LDFLAGS_BASE} -L/home/oracle/instantclient_11_2 -L/home/oracle/app/oracle/product/11.2.0/dbhome_1 -lnnz11 -lboost_log -lboost_program_options 32 | LDFLAGS=${LDFLAGS_BASE} -L/home/oracle/instantclient_11_2 -L/home/oracle/app/oracle/product/11.2.0/dbhome_1/lib -lboost_program_options -lnnz11 -latomic 33 | endif 34 | 35 | %.o: %.cpp $(DEPS) 36 | $(CC) -c -o $@ $< $(CXXFLAGS) 37 | 38 | datastream: stream.o logical_elems.o physical_elems.o redofile.o worker_test.o opcode_ops.o stream.o metadata.o tconvert.o trans.o applier.o monitor.o 39 | ${CC} ${CXXFLAGS} -o $@ $^ ${LDFLAGS} $(OCI_FLAGS) 40 | 41 | dumpop: logical_elems.o physical_elems.o redofile.o opcode_ops.o dump_op.o 42 | ${CC} ${CXXFLAGS} -o $@ $^ ${LDFLAGS} 43 | 44 | metadata.o: metadata.cpp $(DEPS) 45 | $(CC) -c -o $@ $< $(CXXFLAGS) $(OCI_FALGS) 46 | 47 | testmeta: metadata.o testmeta.o 48 | ${CC} ${CXXFLAGS} -o $@ $^ ${LDFLAGS} 49 | 50 | myocci: myocci.o 51 | ${CC} ${CXXFLAGS} -o $@ $^ ${LDFLAGS} 52 | 53 | .PHONY: clean 54 | 55 | clean: 56 | @rm -f *.o datastream dumpop 57 | -------------------------------------------------------------------------------- /src/redo_parse/Makefile.sun: -------------------------------------------------------------------------------- 1 | CC=/opt/csw/gcc4/bin/g++ 2 | DEPS=metadata.h \ 3 | logical_elems.h \ 4 | opcode.h \ 5 | opcode_ops.h \ 6 | physical_elems.h \ 7 | redofile.h \ 8 | workers.h \ 9 | stream.h \ 10 | stream_error.h \ 11 | trans.h \ 12 | applier.h\ 13 | monitor.h 14 | 15 | 16 | PLATFORM=$(shell uname) 17 | ifeq ($(PLATFORM),SunOS) 18 | LDFLAGS=-L/oracle/VIKING/home/products/11203/lib -L/export/home/oracle/zhifan/boost_gcc/lib -lboost_program_options -lclntsh -pthread -latomic 19 | CXXFLAGS =-I. -I.. -std=c++11 -O0 -g -m64 -D__STREAM_DEBUG__ -I/oracle/VIKING/home/zhifan/boost_gcc/include -lclntsh -I/oracle/VIKING/home/products/11203/rdbms/public -pthread -DELPP_THREAD_SAFE -DOTL_DESTRUCTORS_DO_NOT_THROW 20 | endif 21 | 22 | %.o: %.cpp $(DEPS) 23 | $(CC) -c -o $@ $< $(CXXFLAGS) 24 | 25 | datastream: stream.o logical_elems.o physical_elems.o redofile.o worker_test.o opcode_ops.o stream.o metadata.o tconvert.o trans.o applier.o monitor.o 26 | ${CC} ${CXXFLAGS} -o $@ $^ ${LDFLAGS} $(OCI_FLAGS) 27 | 28 | datastream2: stream.o logical_elems.o physical_elems.o redofile.o worker_test2.o opcode_ops.o stream.o metadata.o tconvert.o trans.o applier.o 29 | ${CC} ${CXXFLAGS} -o $@ $^ ${LDFLAGS} $(OCI_FLAGS) 30 | 31 | all: datastream datastream2 32 | dumpop: logical_elems.o physical_elems.o redofile.o opcode_ops.o dump_op.o 33 | ${CC} ${CXXFLAGS} -o $@ $^ ${LDFLAGS} 34 | 35 | metadata.o: metadata.cpp $(DEPS) 36 | $(CC) -c -o $@ $< $(CXXFLAGS) $(OCI_FALGS) 37 | 38 | testmeta: metadata.o testmeta.o 39 | ${CC} ${CXXFLAGS} -o $@ $^ ${LDFLAGS} 40 | 41 | myocci: myocci.o 42 | ${CC} ${CXXFLAGS} -o $@ $^ ${LDFLAGS} 43 | 44 | .PHONY: clean 45 | 46 | clean: 47 | @rm -f *.o datastream dumpop 48 | -------------------------------------------------------------------------------- /src/redo_parse/applier.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #define OTL_ORA11G_R2 9 | #define OTL_ORA_UTF8 10 | #include "otlv4.h" 11 | #include "applier.h" 12 | #include "trans.h" 13 | #include "util/logger.h" 14 | #include "util/dassert.h" 15 | #include "metadata.h" 16 | #include "stream.h" 17 | #include "tconvert.h" 18 | 19 | namespace databus { 20 | SimpleApplier::SimpleApplier(const char* conn_str, 21 | const char* primary_conn_str) 22 | : conn_str_(conn_str), 23 | conn_(conn_str), 24 | table_exist_stmt_( 25 | 1, "SELECT 1 FROM USER_TABLES WHERE TABLE_NAME=upper(:x)", 26 | conn_) { 27 | table_exist_stmt_ << "stream_progress"; 28 | if (table_exist_stmt_.eof()) { 29 | otl_cursor::direct_exec(conn_, 30 | "CREATE TABLE STREAM_PROGRESS " 31 | "(INST_ID NUMBER(38) NOT NULL," 32 | "COMMIT_SCN_MAJOR NUMBER(38) NOT NULL," 33 | "COMMIT_SCN_MINOR NUMBER(38) NOT NULL," 34 | "COMMIT_SUBSCN NUMBER(38) NOT NULL," 35 | "COMMIT_OFFSET NUMBER(38) NOT NULL," 36 | "START_SCN_MAJOR NUMBER(38) NOT NULL," 37 | "START_SCN_MINOR NUMBER(38) NOT NULL," 38 | "START_SUBSCN NUMBER(38) NOT NULL," 39 | "START_OFFSET NUMBER(38) NOT NULL," 40 | "CREATION_DATE DATE NOT NULL," 41 | "APPLIED_TIME DATE NOT NULL," 42 | "RESTART_TIME DATE NOT NULL," 43 | "COMMIT_EPOCH NUMBER(38) NOT NULL," 44 | "RESTART_EPOCH NUMBER(38) NOT NULL)"); 45 | otl_cursor::direct_exec( 46 | conn_, "CREATE INDEX STREAM_CD ON STREAM_PROGRESS (CREATION_DATE)"); 47 | } 48 | ApplyStats tp = ApplierHelper::getApplierHelper().getApplyStats(); 49 | if (tp.restart_tp_.empty()) { 50 | LOG(INFO) << "This is the your first time to run stream for this " 51 | "instance, set the current timepoint for you"; 52 | otl_connect primary_conn(primary_conn_str); 53 | otl_stream current_scn_st( 54 | 1, "select to_char(current_scn) from v$database where 1=:n", 55 | primary_conn); 56 | current_scn_st << 1; 57 | char current_scn[50]; 58 | current_scn_st >> current_scn; 59 | primary_conn.logoff(); 60 | otl_stream init_tm( 61 | 1, 62 | "insert into stream_progress " 63 | " (inst_id, " 64 | " commit_scn_major, commit_scn_minor, commit_subscn, commit_offset, " 65 | " start_scn_major, start_scn_minor, start_subscn, start_offset, " 66 | " creation_date, applied_time, restart_time, commit_epoch, " 67 | "restart_epoch) " 68 | "values(1, 0, 0, 0, 0, " 69 | "trunc(to_number(:scn)/power(2,32)), " 70 | "mod(to_number(:scn), power(2,32)), 0, 0, sysdate, " 71 | "sysdate, sysdate, 0, 0)", 72 | conn_); 73 | init_tm << current_scn; 74 | conn_.commit(); 75 | otl_cursor::direct_exec(conn_, "alter system switch logfile"); 76 | sleep(3); 77 | } 78 | } 79 | 80 | void SimpleApplier::addTable(TabDefPtr tab_def, const TabConf& tab_conf, 81 | bool force) { 82 | auto tab_name = tab_def->getTabName(); 83 | if (stmt_dict_.find(tab_name) != stmt_dict_.end() and !force) { 84 | LOG(WARNING) << " statment for " << tab_name << " exists already"; 85 | return; 86 | } 87 | ensureLogTableCreated(tab_def, tab_conf); 88 | auto insert_sql = getInsertStmt(tab_def); 89 | LOG(INFO) << insert_sql; 90 | stmt_dict_[tab_name] = std::shared_ptr( 91 | new otl_stream(1, insert_sql.c_str(), conn_)); 92 | stmt_dict_[tab_name]->set_commit(0); 93 | } 94 | 95 | void SimpleApplier::_apply(RowChangePtr rcp, TabDefPtr tab_def, XID xid, 96 | char offset = 0) { 97 | auto tab_name = tab_def->getTabName(); 98 | otl_stream* this_stream = stmt_dict_[tab_name].get(); 99 | (*this_stream) << std::to_string(xid).c_str(); 100 | (*this_stream) << getOpStr(rcp->op_).c_str(); 101 | if (offset > 0) { 102 | rcp->scn_.noffset_ += 1; 103 | } 104 | (*this_stream) << rcp->scn_.toString().c_str(); 105 | (*this_stream) << epochToTime(rcp->epoch_).c_str(); 106 | switch (rcp->op_) { 107 | case opcode::kInsert: 108 | case opcode::kMultiInsert: 109 | for (auto pk_col : rcp->new_pk_) { 110 | (*this_stream) << colAsStr2(pk_col, tab_def).c_str(); 111 | } 112 | break; 113 | case opcode::kUpdate: 114 | case opcode::kDelete: 115 | for (auto pk_col : rcp->old_pk_) { 116 | (*this_stream) << colAsStr2(pk_col, tab_def).c_str(); 117 | } 118 | break; 119 | } 120 | } 121 | 122 | void SimpleApplier::apply(TransactionPtr tran) { 123 | try { 124 | for (auto rc : tran->changes_) { 125 | if (!rc->completed()) { 126 | LOG(ERROR) << "Transaction ID " << tran->xid_ 127 | << " Incompleted Row Change: SCN " << rc->scn_.toStr(); 128 | std::exit(21); 129 | } 130 | 131 | if (rc->op_ == opcode::kRowChain || rc->op_ == opcode::kLmn) { 132 | rc->op_ = opcode::kUpdate; 133 | } 134 | 135 | auto tab_def = getMetadata().getTabDefFromId(rc->object_id_); 136 | auto tab_name = tab_def->getTabName(); 137 | if (rc->op_ == opcode::kUpdate) { 138 | bool same = true; 139 | for (auto c : rc->old_pk_) { 140 | auto ret = rc->new_pk_.insert(c); 141 | if (!ret.second) { 142 | if (colAsStr2(c, tab_def) 143 | .compare(colAsStr2(*(ret.first), tab_def)) != 0) { 144 | same = false; 145 | } 146 | } 147 | } 148 | if (!same) { 149 | rc->op_ = opcode::kDelete; 150 | _apply(rc, tab_def, tran->xid_); 151 | rc->op_ = opcode::kInsert; 152 | _apply(rc, tab_def, tran->xid_, 1); 153 | continue; 154 | } 155 | } 156 | _apply(rc, tab_def, tran->xid_); 157 | } 158 | conn_.commit(); 159 | } catch (otl_exception& p) { 160 | if (p.code != 1) throw p; 161 | } 162 | Transaction::setTimePointWhenCommit(tran); 163 | } 164 | 165 | void SimpleApplier::ensureLogTableCreated(TabDefPtr tab_def, 166 | const TabConf& tab_conf) { 167 | table_exist_stmt_ << tab_def->name.c_str(); 168 | if (table_exist_stmt_.eof()) { 169 | // create the table 170 | std::stringstream ss; 171 | ss << "create table " << tab_def->name << " ( " 172 | << gen_prefix_cols_string() << gen_pk_string(tab_def) << ")"; 173 | if (!tab_conf.tbs_name.empty()) { 174 | ss << " tablespace " << tab_conf.tbs_name; 175 | } 176 | otl_cursor::direct_exec(conn_, ss.str().c_str()); 177 | 178 | ss.clear(); 179 | ss.str(std::string()); 180 | ss << "alter table " << tab_def->name << " add primary key(stream_scn)"; 181 | if (!tab_conf.tbs_name.empty()) { 182 | ss << "using index tablespace " << tab_conf.tbs_name; 183 | } 184 | otl_cursor::direct_exec(conn_, ss.str().c_str()); 185 | } 186 | } 187 | 188 | std::string SimpleApplier::gen_pk_string(TabDefPtr tab_def) { 189 | std::stringstream ss; 190 | for (auto col_no : tab_def->pk) { 191 | auto col_type = tab_def->col_types[col_no]; 192 | ss << tab_def->col_names[col_no] << " " << col_type; 193 | if (col_type == "NUMBER") { 194 | if (tab_def->col_len[col_no] != 9999) { 195 | ss << "(" << tab_def->col_len[col_no]; 196 | if (tab_def->col_scale[col_no] != 9999) { 197 | ss << "," << tab_def->col_scale[col_no]; 198 | } 199 | ss << ")"; 200 | } 201 | } else if (col_type == "VARCHAR2" || col_type == "CHAR") { 202 | if (tab_def->col_len[col_no] != 9999) { 203 | ss << "(" << tab_def->col_len[col_no] << ")"; 204 | } 205 | } 206 | ss << ","; 207 | } 208 | std::string s = ss.str(); 209 | s.pop_back(); 210 | return s; 211 | } 212 | 213 | std::string SimpleApplier::getInsertStmt(TabDefPtr tab_def) { 214 | // the statement is order by col_no of pk 215 | static auto insert_template = 216 | boost::format("insert into %s(%s) values(%s)"); 217 | 218 | auto pre_cnt = prefix_cols.size(); 219 | auto pk_cnt = tab_def->pk.size(); 220 | std::vector col_name(pk_cnt + pre_cnt); 221 | std::vector col_value(pk_cnt + pre_cnt); 222 | char n = 0; 223 | for (auto p : prefix_cols) { 224 | col_name[n] = p.first; 225 | col_value[n++] = p.second; 226 | } 227 | for (auto col_no : tab_def->pk) { 228 | std::stringstream ss; 229 | if (tab_def->col_types[col_no] == "NUMBER") { 230 | ss << "TO_NUMBER(:" << tab_def->col_names[col_no] << ")"; 231 | } else if (tab_def->col_types[col_no] == "VARCHAR2" || 232 | tab_def->col_types[col_no] == "CHAR") { 233 | ss << ":" << tab_def->col_names[col_no] << "col_len[col_no] + 1 << "]>"; 235 | } else if (tab_def->col_types[col_no] == "DATE") { 236 | ss << "TO_DATE(:" << tab_def->col_names[col_no] 237 | << ", 'yyyy-mm-dd hh24:mi:ss')"; 238 | } else { 239 | LOG(ERROR) << " FIND UNSUPPORT DATA TYPE " 240 | << tab_def->col_types[col_no]; 241 | LOG(ERROR) << " ONLY NUMBER/CHAR/VARCHAR2/DATE SUPPORTED SO FAR " 242 | << tab_def->col_types[col_no]; 243 | std::exit(100); 244 | } 245 | col_name[n] = tab_def->col_names[col_no]; 246 | col_value[n++] = ss.str(); 247 | } 248 | return (insert_template % tab_def->name % boost::join(col_name, ",") % 249 | boost::join(col_value, ",")).str(); 250 | } 251 | 252 | ApplierHelper::ApplierHelper(const char* conn_str, uint32_t inst_id) 253 | : conn_(conn_str), 254 | inst_id_(inst_id), 255 | save_progress_stmt_( 256 | 1, 257 | "INSERT INTO stream_progress " 258 | " (INST_ID, COMMIT_SCN_MAJOR, COMMIT_SCN_MINOR, " 259 | "COMMIT_SUBSCN, COMMIT_OFFSET, APPLIED_TIME, " 260 | " START_SCN_MAJOR, START_SCN_MINOR, START_SUBSCN, " 261 | "START_OFFSET, RESTART_TIME, COMMIT_EPOCH, RESTART_EPOCH, " 262 | "CREATION_DATE) " 263 | "VALUES (:INST_ID, " 264 | ":COMMIT_SCN_MAJOR, " 265 | ":COMMIT_SCN_MINOR, " 266 | ":COMMIT_SUBSCN, " 267 | ":COMMIT_OFFSET, " 268 | "to_date(:APPLIED_TIME, 'yyyy-mm-dd hh24:mi:ss'), " 269 | ":START_SCN_MAJOR, " 270 | ":START_SCN_MINOR, " 271 | ":START_SUBSCN, " 272 | ":START_OFFSET, " 273 | "to_date(:RESTART_TIME, 'yyyy-mm-dd hh24:mi:ss'), " 274 | ":COMMIT_EPOCH, " 275 | ":RESTART_EPOCH," 276 | "SYSDATE)", 277 | conn_), 278 | get_progress_stmt_(1, 279 | "SELECT " 280 | " COMMIT_SCN_MAJOR, " 281 | " COMMIT_SCN_MINOR, " 282 | " COMMIT_SUBSCN, " 283 | " COMMIT_OFFSET, " 284 | " START_SCN_MAJOR, " 285 | " START_SCN_MINOR, " 286 | " START_SUBSCN, " 287 | " START_OFFSET, " 288 | " COMMIT_EPOCH, " 289 | " RESTART_EPOCH" 290 | " FROM STREAM_PROGRESS " 291 | " WHERE INST_ID = :INST_ID " 292 | " AND CREATION_DATE = (SELECT MAX(CREATION_DATE) " 293 | " FROM STREAM_PROGRESS) ", 294 | conn_) { 295 | save_progress_stmt_.set_commit(0); 296 | } 297 | 298 | void ApplierHelper::saveApplyProgress(const TimePoint& commit_tp, 299 | const TimePoint& restart_tp) { 300 | if (restart_tp.scn_ == SCN(-1)) return; 301 | save_progress_stmt_ << inst_id_; 302 | 303 | save_progress_stmt_ << (unsigned)commit_tp.scn_.major_; 304 | save_progress_stmt_ << commit_tp.scn_.minor_; 305 | save_progress_stmt_ << commit_tp.scn_.subscn_; 306 | save_progress_stmt_ << commit_tp.scn_.noffset_; 307 | save_progress_stmt_ << epochToTime(commit_tp.epoch_).c_str(); 308 | 309 | save_progress_stmt_ << (unsigned)restart_tp.scn_.major_; 310 | save_progress_stmt_ << restart_tp.scn_.minor_; 311 | save_progress_stmt_ << restart_tp.scn_.subscn_; 312 | save_progress_stmt_ << restart_tp.scn_.noffset_; 313 | save_progress_stmt_ << epochToTime(restart_tp.epoch_).c_str(); 314 | save_progress_stmt_ << commit_tp.epoch_; 315 | save_progress_stmt_ << restart_tp.epoch_; 316 | 317 | conn_.commit(); 318 | } 319 | 320 | ApplyStats ApplierHelper::getApplyStats() { 321 | get_progress_stmt_ << inst_id_; 322 | if (get_progress_stmt_.eof()) { 323 | return ApplyStats(); 324 | } 325 | TimePoint commit_tp, restart_tp; 326 | unsigned val; 327 | int n = 1; 328 | while (n < 11) { 329 | get_progress_stmt_ >> val; 330 | switch (n) { 331 | case 1: 332 | commit_tp.scn_.major_ = val; 333 | break; 334 | case 2: 335 | commit_tp.scn_.minor_ = val; 336 | break; 337 | case 3: 338 | commit_tp.scn_.subscn_ = val; 339 | break; 340 | case 4: 341 | commit_tp.scn_.noffset_ = val; 342 | break; 343 | case 5: 344 | restart_tp.scn_.major_ = val; 345 | break; 346 | case 6: 347 | restart_tp.scn_.minor_ = val; 348 | break; 349 | case 7: 350 | restart_tp.scn_.subscn_ = val; 351 | break; 352 | case 8: 353 | restart_tp.scn_.noffset_ = val; 354 | break; 355 | case 9: 356 | commit_tp.epoch_ = val; 357 | break; 358 | case 10: 359 | restart_tp.epoch_ = val; 360 | break; 361 | } 362 | n++; 363 | } 364 | return ApplyStats(restart_tp, commit_tp); 365 | } 366 | 367 | ApplierManager::ApplierManager() 368 | : curr_applying_seq_(GlobalStream::getGlobalStream().getAppliedSeq()), 369 | curr_record_(NULL) {} 370 | 371 | void ApplierManager::operator()() { 372 | while ((curr_record_ = getRecordBufList().pop_front()) != NULL) { 373 | if (canApply()) { 374 | applyAllBuf(); 375 | } 376 | addToTransaction(curr_record_); 377 | if (curr_record_->seq_ != curr_applying_seq_) { 378 | GlobalStream::getGlobalStream().setAppliedSeq(++curr_applying_seq_); 379 | } 380 | } 381 | } 382 | 383 | void ApplierManager::applyAllBuf() { 384 | auto n = Transaction::removeUncompletedTrans(); 385 | if (n > 0) 386 | LOG(INFO) << "removed " << n << " incompleted transaction in log seq " 387 | << curr_applying_seq_; 388 | LOG(DEBUG) << "Build Transaction now" << std::endl; 389 | auto tran = Transaction::xid_map_.begin(); 390 | while (tran != Transaction::xid_map_.end()) { 391 | auto it = buildTransaction(tran); 392 | if (it != Transaction::xid_map_.end()) { 393 | tran = it; 394 | } else { 395 | tran++; 396 | } 397 | } 398 | if (!Transaction::start_scn_q_.empty()) { 399 | auto it = Transaction::start_scn_q_.begin(); 400 | Transaction::setRestartTimePoint(it->first, it->second); 401 | } 402 | LOG(DEBUG) << "Apply Transaction now, Total " 403 | << Transaction::commit_trans_.size() << " to apply "; 404 | auto commit_tran = Transaction::commit_trans_.begin(); 405 | while (commit_tran != Transaction::commit_trans_.end()) { 406 | SimpleApplier::getApplier(streamconf->getString("tarConn").c_str(), 407 | streamconf->getString("srcConn").c_str()) 408 | .apply(commit_tran->second); 409 | commit_tran = Transaction::commit_trans_.erase(commit_tran); 410 | } 411 | } 412 | 413 | bool ApplierManager::canApply() { return curr_record_->vld_ == 0x05; } 414 | } 415 | -------------------------------------------------------------------------------- /src/redo_parse/applier.h: -------------------------------------------------------------------------------- 1 | #ifndef APPLIER_INC 2 | #define APPLIER_INC 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #define OTL_ORA11G_R2 10 | #define OTL_ORA_UTF8 11 | #include "otlv4.h" 12 | #include "trans.h" 13 | #include "stream.h" 14 | #include "logical_elems.h" 15 | 16 | namespace databus { 17 | // Please DO read gen_prefix_cols_string before add a new element to 18 | // prefix_cols 19 | const std::list> prefix_cols{ 20 | std::make_pair("STREAM_XID", "to_number(:stream_xid)"), 21 | std::make_pair("STREAM_OP", ":stream_op"), 22 | std::make_pair( 23 | "STREAM_SCN", 24 | "to_number(:stream_scn, 'XXXXXXXXXXXXXXXXXXXXXXXXXXXX')"), 25 | std::make_pair( 26 | "STREAM_TIMESTAMP", 27 | "to_date(:stream_timestamp, 'yyyy-mm-dd hh24:mi:ss')")}; 28 | inline std::string gen_prefix_cols_string() { 29 | std::stringstream ss; 30 | std::string col_type; 31 | for (auto i : prefix_cols) { 32 | if (boost::istarts_with(i.second, "to_number")) { 33 | col_type = "INT"; 34 | } else if (boost::istarts_with(i.second, "to_date")) { 35 | col_type = "DATE"; 36 | } else { 37 | col_type = "VARCHAR2(40)"; 38 | } 39 | ss << i.first << " " << col_type << ","; 40 | } 41 | return ss.str(); 42 | } 43 | class TabDef; 44 | typedef std::shared_ptr TabDefPtr; 45 | class SimpleApplier { 46 | // Only recover PKs 47 | public: 48 | // add table statement into stmt_dict_, 49 | // if false == true, it will overwrite the the previous data 50 | void addTable(TabDefPtr tab_def, const TabConf& tab_conf, 51 | bool force = false); 52 | void apply(TransactionPtr tran_ptr); 53 | static SimpleApplier& getApplier(const char* conn_str, 54 | const char* primary_con_str) { 55 | static SimpleApplier applier(conn_str, primary_con_str); 56 | return applier; 57 | } 58 | 59 | private: 60 | SimpleApplier(const char* conn_str, const char* primary_con_str); 61 | void ensureLogTableCreated(TabDefPtr, const TabConf&); 62 | std::string gen_pk_string(TabDefPtr tab_def); 63 | std::string getInsertStmt(TabDefPtr tab_def); 64 | void _apply(RowChangePtr rcp, TabDefPtr tab_def, XID xid, char offset); 65 | 66 | private: 67 | std::string conn_str_; 68 | otl_connect conn_; 69 | otl_stream table_exist_stmt_; 70 | std::map> stmt_dict_; 71 | }; 72 | 73 | struct ApplyStats { 74 | ApplyStats() {} 75 | ApplyStats(const TimePoint& restart_tp, const TimePoint& commit_tp) 76 | : restart_tp_(restart_tp), commit_tp_(commit_tp) {} 77 | TimePoint restart_tp_; 78 | TimePoint commit_tp_; 79 | }; 80 | 81 | class ApplierHelper { 82 | // record/get the apply progress into/from database 83 | 84 | public: 85 | ApplyStats getApplyStats(); 86 | void saveApplyProgress(const TimePoint& commit_tp, 87 | const TimePoint& restart_tp); 88 | static ApplierHelper& getApplierHelper() { 89 | static ApplierHelper applierHelper( 90 | streamconf->getString("tarConn").c_str(), 91 | streamconf->getUint32("instId")); 92 | return applierHelper; 93 | } 94 | 95 | private: 96 | ApplierHelper(const char* conn_str, uint32_t inst_id); 97 | 98 | private: 99 | uint32_t inst_id_; 100 | otl_connect conn_; 101 | otl_stream save_progress_stmt_; 102 | otl_stream get_progress_stmt_; 103 | }; 104 | 105 | class ApplierManager { 106 | public: 107 | static ApplierManager& getApplierManager() { 108 | static ApplierManager applier_manager; 109 | return applier_manager; 110 | } 111 | 112 | public: 113 | void operator()(); 114 | 115 | private: 116 | ApplierManager(); 117 | bool canApply(); 118 | void applyAllBuf(); 119 | 120 | private: 121 | uint32_t curr_applying_seq_; 122 | RecordBufPtr curr_record_; 123 | }; 124 | } 125 | #endif /* ----- #ifndef APPLIER_INC ----- */ 126 | -------------------------------------------------------------------------------- /src/redo_parse/demo.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | import time 4 | from subprocess import call 5 | 6 | 7 | arch_dir = '/home/oracle/app/oracle/oradata/orcl/arch' 8 | arch_fromat = '/home/oracle/app/oracle/oradata/orcl/arch/1_%d_850903564.dbf' 9 | last_arch = sorted(glob.glob('%s/1_*_850903564.dbf' % arch_dir))[-1] 10 | last_seq = int(last_arch.split('_')[1]) 11 | 12 | while True: 13 | last_seq += 1 14 | next_arch_file = arch_fromat % last_seq 15 | while True: 16 | if os.path.isfile(next_arch_file): 17 | break 18 | time.sleep(0.1) 19 | call(['./worktest', next_arch_file]) 20 | -------------------------------------------------------------------------------- /src/redo_parse/dump_op.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * ===================================================================================== 3 | * 4 | * Filename: dump_op.cpp 5 | * 6 | * Description: Dump Op of a given logfile 7 | * 8 | * Version: 1.0 9 | * Created: 10/14/2014 16:30:27 10 | * Revision: none 11 | * Compiler: gcc 12 | * 13 | * Author: zhifan (Zhihui Fan), zhihuifan@163.com 14 | * Organization: 15 | * 16 | * ===================================================================================== 17 | */ 18 | #include 19 | #include 20 | #include 21 | #include "redofile.h" 22 | #include "logical_elems.h" 23 | 24 | int main(int argc, char** argv) { 25 | std::map opmap; 26 | databus::RedoFile redofile(argv[1]); 27 | databus::RecordBuf* buf = NULL; 28 | while ((buf = redofile.nextRecordBuf()) != NULL) { 29 | for (auto change : buf->change_vectors) { 30 | opmap[change->opCode()] += 1; 31 | } 32 | } 33 | 34 | for (auto i : opmap) { 35 | std::cout << std::hex << "0x" << i.first << "->" << std::dec << i.second 36 | << std::endl; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/redo_parse/example.conf: -------------------------------------------------------------------------------- 1 | srcConn=andy/andy@vm 2 | tarConn=andy2/andy2@vm 3 | tableConf=table.conf 4 | instId=1 5 | -------------------------------------------------------------------------------- /src/redo_parse/logging.conf: -------------------------------------------------------------------------------- 1 | * GLOBAL: 2 | FORMAT = "%datetime %level %msg" 3 | FILENAME = "/tmp/my.log" 4 | ENABLED = true 5 | TO_FILE = true 6 | TO_STANDARD_OUTPUT = true 7 | MILLISECONDS_WIDTH = 6 8 | PERFORMANCE_TRACKING = false 9 | MAX_LOG_FILE_SIZE = 2097152 ## 2MB - Comment starts with two hashes (##) 10 | 11 | * DEBUG: 12 | TO_FILE = true 13 | TO_STANDARD_OUTPUT = false 14 | 15 | * TRACE: 16 | TO_FILE = true 17 | TO_STANDARD_OUTPUT = false 18 | -------------------------------------------------------------------------------- /src/redo_parse/logical_elems.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "logical_elems.h" 6 | #include "physical_elems.h" 7 | #include "opcode_ops.h" 8 | #include "util/dtypes.h" 9 | #include "util/logger.h" 10 | #include "opcode.h" 11 | 12 | namespace databus { 13 | 14 | const std::set kDMLOps{ 15 | opcode::kInsert, opcode::kDelete, opcode::kUpdate, 16 | opcode::kRowChain, opcode::kBeginTrans, opcode::kCommit, 17 | opcode::kMultiInsert, opcode::kLmn, opcode::kBeginTrans, 18 | opcode::kCommit, opcode::kMfc}; 19 | 20 | const std::set kTRANOps{opcode::kUndo}; 21 | 22 | bool SCN::operator<(const SCN& other) const { 23 | uint64_t me = toNum(); 24 | uint64_t you = other.toNum(); 25 | if (me == you) { 26 | if (subscn_ == other.subscn_) 27 | return noffset_ < other.noffset_; 28 | else 29 | return subscn_ < other.subscn_; 30 | } 31 | return me < you; 32 | } 33 | 34 | uint64_t SCN::toNum() const { 35 | uint64_t n = 0; 36 | uint64_t major = major_; 37 | n = n | (major << (sizeof(minor_) * 8)); 38 | n = n | minor_; 39 | return n; 40 | } 41 | 42 | std::string SCN::toStr() const { 43 | std::stringstream ss; 44 | ss << std::hex << major_ << ":" << minor_ << ":" << subscn_ << ":" 45 | << noffset_; 46 | return ss.str(); 47 | } 48 | 49 | std::string SCN::toString() const { 50 | std::stringstream ss; 51 | ss << std::hex << std::setfill('0') << std::setw(4) << major_ 52 | << std::setw(8) << minor_ << std::setw(8) << subscn_ << std::setw(8) 53 | << noffset_; 54 | return ss.str(); 55 | } 56 | RecordBuf::RecordBuf(const SCN& scn, uint32_t len, uint32_t epoch, 57 | char* change_buf, size_t offset, uint32_t seq, Uchar vld, 58 | bool allop) 59 | : scn_(scn), 60 | change_length_(len), 61 | epoch_(epoch), 62 | change_buffers_(change_buf), 63 | op_(0), 64 | seq_(seq), 65 | offset_(offset), 66 | vld_(vld) { 67 | initChangeVectors(allop); 68 | } 69 | 70 | void RecordBuf::initChangeVectors(bool allop) { 71 | // TODO: bypass all the changes whose SEQ=0 72 | ChangeHeader* ch = (ChangeHeader*)(change_buffers_); 73 | uint32_t parsed_change_size = 0; 74 | bool keep = false; 75 | do { 76 | uint32_t change_size = ch->changeSize(); 77 | if (change_size == 0) { 78 | LOG(DEBUG) << "lol is 0, Diag offset ! " << offset_; 79 | return; 80 | } 81 | if (kDMLOps.find(ch->opCode()) != kDMLOps.end()) { 82 | change_vectors.push_back(ch); 83 | keep = true; 84 | op_ = ch->opCode(); 85 | } else if (kTRANOps.find(ch->opCode()) != kTRANOps.end()) { 86 | change_vectors.push_back(ch); 87 | } else if (allop) 88 | change_vectors.push_back(ch); 89 | parsed_change_size += change_size; 90 | // unsigned int seq = ch->seq_; 91 | // std::cout << seq << ":" << std::hex << ch->opCode() << std::endl; 92 | ch = (ChangeHeader*)((char*)ch + change_size); 93 | } while (parsed_change_size != change_length_); 94 | if (!keep) change_vectors.clear(); 95 | } 96 | 97 | void ColumnChange::dump() { 98 | std::cout << "col# " << col_id_ << " " << content_ << "(" << len_ << ")" 99 | << std::endl; 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/redo_parse/logical_elems.h: -------------------------------------------------------------------------------- 1 | #ifndef LOGICAL_ELEMS_H 2 | #define LOGICAL_ELEMS_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "util/dtypes.h" 11 | 12 | namespace databus { 13 | 14 | class ChangeHeader; 15 | class SCN { 16 | public: 17 | SCN() noexcept : major_(0), minor_(0), subscn_(0), noffset_(0) {} 18 | SCN(Ushort maj, uint32_t minor, unsigned int subscn = 0, 19 | uint32_t noffset = 0) noexcept : major_(maj), 20 | minor_(minor), 21 | subscn_(subscn), 22 | noffset_(noffset) {} 23 | SCN(int i) noexcept : major_(0xffff), 24 | minor_(0xffffffff), 25 | subscn_(0xffffffff), 26 | noffset_(0xffffffff) {} 27 | 28 | bool operator<(const SCN& other) const; 29 | bool operator==(const SCN& other) const { 30 | return noffset_ == other.noffset_ && minor_ == other.minor_ && 31 | major_ == other.major_ && subscn_ == other.subscn_; 32 | } 33 | std::string toStr() const; 34 | // To store this data into Database 35 | std::string toString() const; 36 | bool empty() const { 37 | return minor_ == 0 && major_ == 0 && subscn_ == 0 && noffset_ == 0; 38 | } 39 | 40 | Ushort major_; 41 | uint32_t minor_; 42 | unsigned int subscn_; 43 | uint32_t noffset_; 44 | 45 | uint64_t toNum() const; 46 | }; // SCN 47 | 48 | class RecordBuf { 49 | public: 50 | RecordBuf(const SCN& scn, uint32_t len, uint32_t epoch, char* change_buf, 51 | size_t offset, uint32_t seq, Uchar vld, bool allop = false); 52 | 53 | std::list change_vectors; 54 | size_t offset() const { return offset_; } 55 | 56 | ~RecordBuf() { delete[] change_buffers_; } 57 | SCN& scn() { return scn_; } 58 | Ushort op() { return op_; }; 59 | uint32_t epoch() const { return epoch_; } 60 | 61 | private: 62 | // allop: if true, capatural all opocde. or else only captual valid op 63 | void initChangeVectors(bool allop); 64 | 65 | private: 66 | SCN scn_; 67 | uint32_t epoch_; 68 | uint32_t change_length_; 69 | Ushort op_; 70 | char* change_buffers_; 71 | size_t offset_; // debug only 72 | public: 73 | uint32_t seq_; // logfile.seq# 74 | Uchar vld_; 75 | 76 | }; // RecordBuf 77 | 78 | typedef std::shared_ptr RecordBufPtr; 79 | 80 | struct ColumnChange { 81 | ColumnChange(Ushort col_id, Ushort len, char* content) 82 | : col_id_(col_id), len_(len), content_(content) {} 83 | ~ColumnChange() { delete[] content_; } 84 | void dump(); 85 | 86 | // start with 1 87 | Ushort col_id_; 88 | Ushort len_; 89 | char* content_; 90 | }; 91 | typedef std::shared_ptr ColumnChangePtr; 92 | typedef std::list Row; 93 | } // databus 94 | #endif 95 | -------------------------------------------------------------------------------- /src/redo_parse/metadata.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #define OTL_ORA11G_R2 7 | #define OTL_ORA_UTF8 8 | #include "otlv4.h" 9 | #include "util/logger.h" 10 | #include "util/dassert.h" 11 | #include "metadata.h" 12 | 13 | namespace databus { 14 | std::string TabDef::toString() { 15 | std::stringstream ss; 16 | ss << "Table Name : " << name << "\n" 17 | << "Primary Key : " 18 | << "\n"; 19 | for (auto i : pk) ss << i << " "; 20 | ss << std::endl; 21 | for (auto i : col_names) 22 | ss << col_names[i.first] << ":" << col_types[i.first] << std::endl; 23 | return ss.str(); 24 | } 25 | 26 | std::map > MetadataManager::oid2def_; 27 | std::map MetadataManager::poid2goid_; 28 | MetadataManager::MetadataManager(const std::string& conn_str) 29 | : conn_str_(conn_str), 30 | conn_(conn_str.c_str()), 31 | tab2oid_stmt_(10, 32 | "select object_id, nvl(SUBOBJECT_NAME, '__') from " 33 | "dba_objects where " 34 | "owner=upper(:x) and " 35 | "object_name=upper(:y) and object_type in " 36 | "('TABLE', 'TABLE PARTITION')", 37 | conn_), 38 | tab2def_stmt_( 39 | 10, 40 | "select COLUMN_ID, COLUMN_NAME, DATA_TYPE, DATA_LENGTH, " 41 | "NVL(DATA_PRECISION, 9999), NVL(DATA_SCALE, 9999) " 42 | "from dba_tab_cols where " 43 | "owner=upper(:x) and table_name=upper(:y) " 44 | "and column_id is not null", 45 | conn_), 46 | pk_stmt_( 47 | 10, 48 | "select column_id " 49 | " from dba_cons_columns col, dba_constraints con, dba_tab_cols tc " 50 | " where con.owner=col.owner " 51 | " and con.table_name = col.table_name " 52 | " and con.CONSTRAINT_NAME= col.CONSTRAINT_NAME " 53 | " and con.CONSTRAINT_TYPE='P' " 54 | " and con.owner=upper(:x) " 55 | " and con.table_name=upper(:y) " 56 | " and col.column_name = tc.column_name " 57 | " and tc.table_name = con.table_name" 58 | " and tc.owner = con.owner ", 59 | conn_), 60 | objp2g_stmt_(10, 61 | " select object_id from dba_objects" 62 | " where subobject_name is null and" 63 | " (object_name, owner) in (select " 64 | " object_name, owner from dba_objects " 65 | " where object_id = :x)", 66 | conn_), 67 | obj2tab_stmt_(10, 68 | " select owner, object_name from dba_objects" 69 | " where object_id = :x", 70 | conn_) { 71 | // make sure we called otl_connect::otl_initialize() before init this 72 | // class 73 | } 74 | 75 | MetadataManager::~MetadataManager() { conn_.logoff(); } 76 | 77 | uint32_t MetadataManager::getGlobalObjId(uint32_t objid) { 78 | objp2g_stmt_ << objid; 79 | uint32_t gid = 0; 80 | if (!objp2g_stmt_.eof()) { 81 | objp2g_stmt_ >> gid; 82 | } 83 | return gid; 84 | } 85 | 86 | void MetadataManager::initFromId(uint32_t object_id) { 87 | obj2tab_stmt_ << object_id; 88 | if (!obj2tab_stmt_.eof()) { 89 | char owner[31]; 90 | char table[129]; 91 | obj2tab_stmt_ >> owner; 92 | obj2tab_stmt_ >> table; 93 | initTabDefFromName(owner, table); 94 | } 95 | } 96 | 97 | std::shared_ptr MetadataManager::getTabDefFromId(uint32_t object_id, 98 | bool allow_init) { 99 | // may be null 100 | // a). not init b). object_id is a partition obj id 101 | if (haveDef(object_id)) return oid2def_[object_id]; 102 | uint32_t goid = getCachedGlobalId(object_id); 103 | if (goid == 0) { 104 | return NULL; 105 | } 106 | if (haveDef(goid)) return oid2def_[goid]; 107 | if (!allow_init) { 108 | return NULL; 109 | } 110 | initFromId(goid); 111 | // NULL if a) not pk, like some sys.tables 112 | return oid2def_[goid]; 113 | } 114 | 115 | std::shared_ptr MetadataManager::initTabDefFromName( 116 | const char* owner, const char* table) { 117 | 118 | std::shared_ptr tab_def(new TabDef()); 119 | tab_def->owner = std::string(owner); 120 | tab_def->name = std::string(table); 121 | boost::to_upper(tab_def->owner); 122 | boost::to_upper(tab_def->name); 123 | 124 | pk_stmt_ << owner << table; 125 | while (!pk_stmt_.eof()) { 126 | unsigned int col_no; 127 | pk_stmt_ >> col_no; 128 | tab_def->pk.insert(col_no); 129 | } 130 | if (tab_def->pk.empty()) { 131 | LOG(ERROR) << "either " << owner << "." << table 132 | << " not exists or hasn't primary key"; 133 | return NULL; 134 | } 135 | tab2def_stmt_ << owner << table; 136 | unsigned int col_id; 137 | char col_name[129], col_type[129]; 138 | unsigned int len; 139 | while (!tab2def_stmt_.eof()) { 140 | tab2def_stmt_ >> col_id; 141 | tab2def_stmt_ >> col_name; 142 | tab2def_stmt_ >> col_type; 143 | tab_def->col_names[col_id] = std::move(std::string(col_name)); 144 | tab_def->col_types[col_id] = std::move(std::string(col_type)); 145 | if (strcmp(col_type, "VARCHAR2") == 0 || strcmp(col_type, "CHAR") == 0) { 146 | tab2def_stmt_ >> len; 147 | tab_def->col_len[col_id] = len; 148 | tab2def_stmt_ >> len; 149 | tab2def_stmt_ >> len; 150 | } else if (strcmp(col_type, "NUMBER") == 0) { 151 | uint32_t scale; 152 | tab2def_stmt_ >> len; 153 | tab2def_stmt_ >> len; 154 | tab2def_stmt_ >> scale; 155 | tab_def->col_len[col_id] = len; 156 | tab_def->col_scale[col_id] = scale; 157 | } else { 158 | tab2def_stmt_ >> len; 159 | tab2def_stmt_ >> len; 160 | tab2def_stmt_ >> len; 161 | } 162 | } 163 | 164 | tab2oid_stmt_ << owner << table; 165 | uint32_t object_id; 166 | unsigned char object_name[129]; 167 | uint32_t global_object_id = 0; 168 | while (!tab2oid_stmt_.eof()) { 169 | tab2oid_stmt_ >> object_id >> object_name; 170 | if (oid2def_.find(object_id) != oid2def_.end()) { 171 | LOG(WARNING) << "logical error old def exist already" 172 | << oid2def_[object_id]->name << ":" 173 | << oid2def_[object_id]->owner << " new def " 174 | << tab_def->owner << ":" << tab_def->name << std::endl; 175 | } 176 | if (strcmp((const char*)object_name, "__") == 0) { 177 | // global object_id 178 | if (global_object_id > 0) { 179 | util::dassert("Assert Global_object_id Error", 180 | global_object_id == object_id); 181 | } 182 | oid2def_[object_id] = tab_def; 183 | global_object_id = object_id; 184 | } else { 185 | // partition object_id 186 | if (global_object_id == 0) { 187 | global_object_id = getGlobalObjId(object_id); 188 | util::dassert("GlobalIDNotZeroError", global_object_id != 0); 189 | } 190 | poid2goid_[object_id] = global_object_id; 191 | } 192 | } 193 | util::dassert("Can't find out global ID ", global_object_id > 0); 194 | return oid2def_[global_object_id]; 195 | } 196 | 197 | LogManager::LogManager(const char* conn_str) 198 | : conn_str_(conn_str), 199 | conn_(conn_str), 200 | arch_log_stmt_(1, 201 | "select name from v$archived_log where sequence# = " 202 | ":seq and STANDBY_DEST='NO'", 203 | conn_), 204 | online_log_stmt_(1, 205 | "select member from v$logfile lf, v$log l where " 206 | "l.group# = lf.group# and l.sequence# = " 207 | ":seq", 208 | conn_), 209 | log_last_blk_stmt_(1, 210 | "select LAST_REDO_BLOCK from v$thread where " 211 | "LAST_REDO_SEQUENCE# = :seq", 212 | conn_), 213 | log_file_from_scn_stmt_(1, 214 | "select SEQUENCE# from v$archived_log " 215 | "where to_number(:restart_scn) " 216 | "between FIRST_CHANGE# and NEXT_CHANGE# " 217 | "and STANDBY_DEST='NO'", 218 | conn_), 219 | online_log_seq_from_scn_stmt_(1, 220 | "select sequence# from v$log where " 221 | "to_number(:restart_scn) " 222 | "between FIRST_CHANGE# and NEXT_CHANGE#", 223 | conn_) {} 224 | 225 | uint32_t LogManager::getSeqFromScn(const char* restart_scn) { 226 | log_file_from_scn_stmt_ << restart_scn; 227 | uint32_t seq; 228 | if (!log_file_from_scn_stmt_.eof()) { 229 | log_file_from_scn_stmt_ >> seq; 230 | return seq; 231 | } 232 | online_log_seq_from_scn_stmt_ << restart_scn; 233 | if (!online_log_seq_from_scn_stmt_.eof()) { 234 | online_log_seq_from_scn_stmt_ >> seq; 235 | return seq; 236 | } 237 | return 0; 238 | } 239 | 240 | std::string LogManager::getLogfile(uint32_t seq) { 241 | // prefer archive log 242 | tryagain: 243 | arch_log_stmt_ << seq; 244 | char filename[514]; 245 | if (!arch_log_stmt_.eof()) { 246 | arch_log_stmt_ >> filename; 247 | return std::string(filename); 248 | } 249 | online_log_stmt_ << seq; 250 | if (!online_log_stmt_.eof()) { 251 | online_log_stmt_ >> filename; 252 | return std::string(filename); 253 | } 254 | LOG(DEBUG) << "Seems a software bug, request seq " << seq 255 | << " but it is not there, sleep 3 sconds and try again"; 256 | sleep(3); 257 | goto tryagain; 258 | } 259 | 260 | uint32_t LogManager::getOnlineLastBlock(uint32_t seq) { 261 | log_last_blk_stmt_ << seq; 262 | if (!log_last_blk_stmt_.eof()) { 263 | uint32_t last_blk; 264 | log_last_blk_stmt_ >> last_blk; 265 | return last_blk; 266 | } 267 | return 0; 268 | } 269 | 270 | LogManager::~LogManager() { conn_.logoff(); } 271 | } 272 | -------------------------------------------------------------------------------- /src/redo_parse/metadata.h: -------------------------------------------------------------------------------- 1 | #ifndef METADATA_INC 2 | #define METADATA_INC 3 | 4 | #define OTL_ORA11G_R2 5 | #define OTL_ORA_UTF8 6 | #include "otlv4.h" 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include "util/dtypes.h" 18 | 19 | namespace databus { 20 | 21 | class TabDef { 22 | public: 23 | std::string owner; 24 | std::string name; 25 | std::set pk; 26 | std::map col_names; 27 | std::map col_types; // shall I use number? 28 | // we only care the len of char/varchar2/number, the scale of number 29 | std::map col_len; 30 | std::map col_scale; 31 | 32 | public: 33 | std::string toString(); 34 | std::string getTabName() { 35 | std::stringstream ss; 36 | ss << owner << "." << name; 37 | return ss.str(); 38 | } 39 | }; 40 | 41 | typedef std::shared_ptr TabDefPtr; 42 | class MetadataManager { 43 | // the fucntion of MetadataManager should be like this: 44 | // 1. init all the tableDef from a configure file 45 | // 2. add a new table to tracking 46 | // in both cases, we know owner.table at begining 47 | // 3. Get Tabledef from obj# 48 | // 4. We should init TableDef when needed, so modify getTabDefFromId 49 | // TODO: 50 | // a. init TableDef when get a objid effectively 51 | // b. since db connection is not thread-safe, so this function is not 52 | // thread-safe. 53 | // need to verify only 1 thread use db connection, all the others are 54 | // read from static member only, 55 | // c. With the mode stated in b, this read/write mode doesn't need a 56 | // mutex? 57 | // A: on a given time, getTabDefFromId to get a TabDef, the object_id 58 | // must be inited or not before, so when read this id, there is no 59 | // changes 60 | // for this id during the time. so doesn't need a mutex in this case? 61 | // 62 | 63 | public: 64 | MetadataManager(const std::string& conn_str); 65 | ~MetadataManager(); 66 | // re-connect if needed 67 | std::shared_ptr getTabDefFromId(uint32_t object_id, 68 | bool allow_init = true); 69 | std::shared_ptr initTabDefFromName(const char* owner, 70 | const char* table); 71 | 72 | std::string getLogfile(uint32_t seq); 73 | 74 | // return -1 if the given seq is not the current logfile 75 | uint32_t getOnlineLastBlock(uint32_t seq); 76 | 77 | private: 78 | void init(const std::string& username, const std::string& password, 79 | const std::string& db); 80 | uint32_t getGlobalObjId(uint32_t obj); 81 | void initFromId(uint32_t object_id); 82 | uint32_t getCachedGlobalId(uint32_t obj) const { 83 | // I didn't use poid2goid_[obj] directly since 84 | // the way will cause poid2goid_ huge 85 | auto it = poid2goid_.find(obj); 86 | if (it != poid2goid_.end()) { 87 | return it->second; 88 | } 89 | return 0; 90 | } 91 | 92 | public: 93 | static bool haveDef(uint32_t object_id) { 94 | return oid2def_.find(object_id) != oid2def_.end(); 95 | } 96 | 97 | private: 98 | // for re-connect 99 | const std::string conn_str_; 100 | otl_connect conn_; 101 | otl_stream tab2oid_stmt_; 102 | otl_stream tab2def_stmt_; 103 | otl_stream objp2g_stmt_; 104 | otl_stream obj2tab_stmt_; 105 | otl_stream pk_stmt_; 106 | static std::map > oid2def_; 107 | static std::map poid2goid_; 108 | }; 109 | 110 | class LogManager { 111 | public: 112 | LogManager(const char* conn_str); 113 | ~LogManager(); 114 | std::string getLogfile(uint32_t seq); 115 | uint32_t getOnlineLastBlock(uint32_t seq); 116 | uint32_t getSeqFromScn(const char* restart_scn); 117 | 118 | private: 119 | // for re-connect 120 | const std::string conn_str_; 121 | otl_connect conn_; 122 | otl_stream arch_log_stmt_; 123 | otl_stream online_log_stmt_; 124 | otl_stream log_last_blk_stmt_; 125 | otl_stream log_file_from_scn_stmt_; 126 | otl_stream online_log_seq_from_scn_stmt_; 127 | }; 128 | } 129 | #endif 130 | -------------------------------------------------------------------------------- /src/redo_parse/mine.sql: -------------------------------------------------------------------------------- 1 | set echo on 2 | EXECUTE DBMS_LOGMNR.ADD_LOGFILE( LOGFILENAME => '/home/oracle/app/oracle/product/11.2.0/dbhome_1/dbs/arch1_&1._879503312.dbf', OPTIONS => DBMS_LOGMNR.NEW); 3 | EXECUTE DBMS_LOGMNR.START_LOGMNR(OPTIONS => DBMS_LOGMNR.DICT_FROM_ONLINE_CATALOG); 4 | -------------------------------------------------------------------------------- /src/redo_parse/mine_tab.sql: -------------------------------------------------------------------------------- 1 | col sql_redo format a120 2 | set lin 255 3 | select xid as oxid, to_char(XIDUSN,'xxxx')||to_char(XIDSLT,'xxxx') ||to_char(XIDSQN, 'xxxxxxxx') as xid, 4 | to_char(scn, 'xxxxxxxxx') as scn, 5 | to_char(start_scn, 'xxxxxxxxx') as start_scn , 6 | to_char(commit_scn, 'xxxxxxxxx') as commit_scn, 7 | sql_redo 8 | from V$LOGMNR_CONTENTS 9 | where table_name=upper('&1.'); 10 | 11 | -------------------------------------------------------------------------------- /src/redo_parse/mine_xid.sql: -------------------------------------------------------------------------------- 1 | col sql_redo format a150 2 | set lin 255 3 | select to_char(XIDUSN,'xxxx')||to_char(XIDSLT,'xxxx') ||to_char(XIDSQN, 'xxxxxxxx') as xid, 4 | to_char(scn, 'xxxxxxxxx') as scn, 5 | to_char(start_scn, 'xxxxxxxxx') as start_scn , 6 | to_char(commit_scn, 'xxxxxxxxx') as commit_scn, 7 | sql_redo 8 | sql_redo 9 | from V$LOGMNR_CONTENTS 10 | where xid='&1.'; 11 | -------------------------------------------------------------------------------- /src/redo_parse/monitor.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "monitor.h" 4 | #include "applier.h" 5 | #include "trans.h" 6 | #include "stream.h" 7 | #include "easylogging++.h" 8 | 9 | namespace databus { 10 | void Monitor::operator()() { 11 | while (true) { 12 | ApplierHelper& ah = ApplierHelper::getApplierHelper(); 13 | try { 14 | ah.saveApplyProgress(Transaction::getLastCommitTimePoint(), 15 | Transaction::getRestartTimePoint()); 16 | } catch (otl_exception& p) { 17 | LOG(ERROR) << p.msg; 18 | LOG(ERROR) << p.stm_text; 19 | LOG(ERROR) << p.var_info; 20 | } 21 | sleep(3); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/redo_parse/monitor.h: -------------------------------------------------------------------------------- 1 | #ifndef MOITOR_INC 2 | #define MOITOR_INC 3 | #include 4 | 5 | namespace databus { 6 | class Monitor { 7 | public: 8 | void operator()(); 9 | }; 10 | } 11 | 12 | #endif /* ----- #ifndef MOITOR_INC ----- */ 13 | -------------------------------------------------------------------------------- /src/redo_parse/opcode.h: -------------------------------------------------------------------------------- 1 | #ifndef OPCODE_INC 2 | #define OPCODE_INC 3 | #include 4 | #include "util/dtypes.h" 5 | #include "logical_elems.h" 6 | 7 | namespace databus { 8 | 9 | namespace opcode { 10 | const Ushort kOther = 0x0005; 11 | const Ushort kUndo = 0x0501; 12 | const Ushort kBeginTrans = 0x0502; 13 | const Ushort kCommit = 0x0504; 14 | const Ushort kNewSession = 0x0513; 15 | const Ushort kSwitchSession = 0x0514; 16 | // Row Operation 17 | const Ushort kInsert = 0x0b02; 18 | const Ushort kDelete = 0x0b03; 19 | const Ushort kLock = 0x0b04; 20 | const Ushort kUpdate = 0x0b05; 21 | const Ushort kRowChain = 0x0b06; 22 | const Ushort kMfc = 0x0b07; 23 | // CFA means change forwarding address we can ingore 24 | const Ushort kCfa = 0x0b08; 25 | const Ushort kCki = 0x0b09; // Cluster Key Index 26 | const Ushort kSkl = 0x0b0a; // Set cluster key pointer 27 | const Ushort kMultiInsert = 0x0b0b; 28 | // we can ingore this, it is a rollback of MultiInsert 29 | // see https://jirap.corp.ebay.com/browse/DBISTREA-52 30 | const Ushort kMultiDelete = 0x0b0c; 31 | // 0x0b0e should be generated when dropping table first column 32 | const Ushort kLmn = 0x0b10; 33 | const Ushort kDdl = 0x1801; 34 | const std::map kOpMap{{kInsert, "Insert"}, 35 | {kUpdate, "Update"}, 36 | {kDelete, "Delete"}, 37 | {kMultiInsert, "Insert"}, 38 | {kRowChain, "11.06"}, 39 | {kLmn, "11.16"}}; 40 | }; 41 | 42 | inline std::string getOpStr(Ushort op) { 43 | auto it = opcode::kOpMap.find(op); 44 | return it != opcode::kOpMap.end() ? it->second : ""; 45 | } 46 | 47 | struct OpCode0501 { 48 | Ushort size_; 49 | Ushort space_; 50 | Ushort flag_; 51 | Ushort unknown1_; 52 | Ushort xid_high_; 53 | Ushort xid_mid_; 54 | uint32_t xid_low_; 55 | Ushort seq_; 56 | Uchar rec_; 57 | }; 58 | 59 | struct OpCode0501Sec { 60 | uint32_t object_id_; 61 | uint32_t data_object_id_; 62 | uint32_t tsn_; 63 | uint32_t unknown1; 64 | Uchar op_major_; 65 | Uchar op_minor_; 66 | Uchar slt_; 67 | Uchar rci_; 68 | Uchar is_ktubl_; 69 | }; 70 | 71 | struct OpCodeKdo { 72 | uint32_t bdba_; // block address 73 | uint32_t hdba_; // segment header block address 74 | Ushort maxfr_; 75 | Uchar opcode_; 76 | Uchar xtype_; 77 | Uchar itli_; 78 | Uchar ispac_; 79 | }; 80 | 81 | struct OpCodeKtbdir { 82 | char opcode_; 83 | Uchar version_; 84 | Uchar unknown_[3]; 85 | }; 86 | 87 | // insert 88 | struct OpCodeKdoirp { 89 | uint32_t bdba_; 90 | uint32_t hdba_; 91 | Ushort maxfr_; 92 | Uchar opcode_; 93 | Uchar xtype_; 94 | Uchar itli_; 95 | Uchar ispac_; 96 | Ushort unknown1_; 97 | Uchar flag_; 98 | Uchar lb_; 99 | Uchar column_count_; 100 | Uchar cki_; 101 | uint32_t hrid_; 102 | Ushort hrid_minor_; 103 | Ushort unknown2_; 104 | uint32_t nrid_; 105 | Ushort nrid_minor_; 106 | Ushort unknown3_; 107 | uint32_t unknown4_; 108 | Ushort size_; 109 | Ushort slot_; 110 | Uchar tabn_; 111 | Uchar null_bitmap_[1]; 112 | }; 113 | 114 | // cfa 115 | struct OpCodeKdocfa { 116 | uint32_t bdba_; /* 00h */ 117 | uint32_t hdba_; /* 04h */ 118 | Ushort maxfr_; /* 08h */ 119 | Uchar opcode_; /* 0ah */ 120 | Uchar xtype_; /* 0bh */ 121 | Uchar itli_; /* 0ch */ 122 | Uchar ispac_; /* 0dh */ 123 | Ushort unknow_; 124 | uint32_t nrid_; 125 | Ushort nrid_minor_; 126 | Uchar tabn_; 127 | Ushort slot_; 128 | }; 129 | 130 | // delete 131 | struct OpCodeKdodrp { 132 | uint32_t bdba_; 133 | uint32_t hdba_; 134 | Ushort maxfr_; 135 | Uchar opcode_; 136 | Uchar xtype_; 137 | Uchar itli_; 138 | Uchar ispac_; 139 | Ushort unknown1_; 140 | Ushort slot_; 141 | Uchar tabn_; 142 | }; 143 | 144 | // update 145 | struct OpCodeKdourp { // part 4 146 | uint32_t bdba_; /* 00h */ 147 | uint32_t hdba_; /* 04h */ 148 | Ushort maxfr_; /* 08h */ 149 | Uchar opcode_; /* 0ah */ 150 | Uchar xtype_; /* 0bh */ 151 | Uchar itli_; /* 0ch */ 152 | Uchar ispac_; /* 0dh */ 153 | Ushort unknown1_; /* 0eh */ 154 | Uchar flag_; /* 10h */ 155 | Uchar lock_; /* 11h */ 156 | Uchar ckix_; /* 12h */ 157 | Uchar tabn_; /* 13h */ 158 | Ushort slot_; /* 14h */ 159 | Uchar ncol_; /* 16h */ /* total column number */ 160 | Uchar nchanged_; /* 17h */ /* changed columes number */ 161 | short size_; /* 18h */ /* what size (-5) */ 162 | Uchar null_bitmap_[1]; /* 1ah */ 163 | }; 164 | 165 | // mfc 166 | struct OpCodeKdomfc { 167 | uint32_t bdba_; 168 | uint32_t hdba_; 169 | Ushort maxfr_; 170 | Uchar opcode_; 171 | Uchar xtype_; 172 | Uchar itli_; 173 | Uchar ispac_; 174 | Uchar unknown1_[2]; 175 | Ushort slot_; 176 | Uchar tabn_; 177 | }; 178 | 179 | struct OpCodeKdoqm { 180 | uint32_t bdba_; 181 | uint32_t hdba_; 182 | Ushort maxfr_; 183 | Uchar opcode_; 184 | Uchar xtype_; 185 | Uchar itli_; 186 | Uchar ispac_; 187 | Ushort unknown1_; 188 | Uchar tabn_; 189 | Uchar lock_; 190 | Uchar nrow_; 191 | Uchar unknown_; 192 | Ushort slot_[1]; 193 | }; 194 | 195 | struct RedoRid { 196 | uint32_t major; 197 | Ushort minor; 198 | }; 199 | 200 | struct OpCodeSupplemental { 201 | Uchar unknown_; 202 | Uchar flag_; 203 | Ushort total_cols_; 204 | Ushort objv_; 205 | Ushort start_column_; 206 | Ushort start_column2_; 207 | }; 208 | 209 | struct OpCode0502 { 210 | Ushort slot_; 211 | Ushort unknow_; 212 | uint32_t sqn_; 213 | uint32_t uba_high_; 214 | Ushort uba_mid_; 215 | Uchar uba_low_; /* 0x2e */ 216 | Uchar unknown6_[1]; /* 0x2f */ 217 | Ushort flag_; /* 0x30 */ 218 | Ushort size_; /* 0x32 */ 219 | Uchar fbi_; /* 0x34 */ 220 | Uchar unknown1_; 221 | Ushort unknown2_; 222 | Ushort pxid_major_; 223 | Ushort pxid_minor_; 224 | uint32_t pxid_micro_; 225 | }; 226 | 227 | struct OpCode0504_ucm { 228 | Ushort slt_; 229 | Ushort unknown2; 230 | uint32_t sqn_; 231 | Uchar srt; 232 | Uchar unknown1[3]; 233 | uint32_t sta; 234 | Uchar flg_; 235 | }; 236 | struct OpCode0504_ucf { 237 | uint32_t uba_blk_; 238 | Ushort uba_seq_; 239 | Uchar uba_slt_; 240 | Uchar padding_; 241 | Ushort ext_; 242 | Ushort spc_; 243 | Uchar fbi_; 244 | Uchar unknown_[3]; 245 | }; 246 | } 247 | 248 | #endif /* ----- #ifndef OPCODE_INC ----- */ 249 | -------------------------------------------------------------------------------- /src/redo_parse/opcode_ops.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "opcode_ops.h" 9 | #include "opcode.h" 10 | #include "util/dtypes.h" 11 | #include "util/logger.h" 12 | #include "stream_error.h" 13 | 14 | namespace databus { 15 | bool validOp(Ushort op) { 16 | const std::set valid_ops{ 17 | opcode::kOther, opcode::kUndo, opcode::kBeginTrans, 18 | opcode::kCommit, opcode::kNewSession, opcode::kSwitchSession, 19 | opcode::kInsert, opcode::kDelete, opcode::kUpdate, 20 | opcode::kRowChain, opcode::kMfc, opcode::kCfa, 21 | opcode::kMultiInsert, opcode::kMultiDelete, opcode::kLmn, 22 | opcode::kDdl}; 23 | return valid_ops.find(op) != valid_ops.end(); 24 | } 25 | 26 | XID Ops0501::getXID(const ChangeHeader* change0501) { 27 | OpCode0501* op51 = (OpCode0501*)change0501->part(1); 28 | XID xid_high = op51->xid_high_; 29 | XID xid_mid = op51->xid_mid_; 30 | XID xid_low = op51->xid_low_; 31 | return xid_high << ((sizeof(uint32_t) + sizeof(Ushort)) * 8) | 32 | xid_mid << (sizeof(uint32_t) * 8) | xid_low; 33 | } 34 | 35 | uint32_t Ops0501::getObjId(const ChangeHeader* change0501) { 36 | OpCode0501Sec* op51sec = (OpCode0501Sec*)change0501->part(2); 37 | return op51sec->object_id_; 38 | } 39 | 40 | uint32_t Ops0501::getDataObjId(const ChangeHeader* change0501) { 41 | OpCode0501Sec* op51sec = (OpCode0501Sec*)change0501->part(2); 42 | return op51sec->data_object_id_; 43 | } 44 | 45 | std::string rowAsString(const Row& row) { 46 | std::stringstream ss; 47 | ss << "Col_id dump "; 48 | for (auto c : row) { 49 | ss << c->col_id_ << "_"; 50 | } 51 | return ss.str(); 52 | } 53 | 54 | Row _makeUpUncommCols(const char* start, int total_cols) { 55 | Row changes; 56 | for (int i = 0; i < total_cols; ++i) { 57 | start++; 58 | Ushort len = *((Ushort*)start); 59 | start += sizeof(Ushort); 60 | Ushort col_id = i + 1; 61 | char* content = new char[len + 1]; 62 | memcpy(content, start, len); 63 | content[len] = '\0'; 64 | start += len; 65 | ColumnChange* col_change = new ColumnChange(col_id, len, content); 66 | changes.push_back(std::shared_ptr(col_change)); 67 | } 68 | return changes; 69 | } 70 | 71 | Row _makeUpLenPrefixCols(Ushort* col_num, Ushort total_cols, 72 | const ChangeHeader* change, Ushort data_offset) { 73 | // std::cout << "Makeup Len Prefix columns " << std::endl; 74 | Row row; 75 | for (int i = 0; i < total_cols; ++i) { 76 | const char* src = change->part(data_offset + i); 77 | Ushort len = *((Ushort*)src); 78 | char* data = new char[len + 1]; 79 | memcpy(data, src + sizeof(Ushort), len); 80 | data[len] = '\0'; 81 | ColumnChange* col_change = new ColumnChange(*(col_num + i), len, data); 82 | row.push_back(std::shared_ptr(col_change)); 83 | } 84 | return row; 85 | } 86 | 87 | Row _makeUpNoLenPrefixCols(Ushort* col_num, Ushort total_cols, 88 | const ChangeHeader* change, Ushort data_offset, 89 | bool supplemental) { 90 | // std::cout << "Makeup no len in data part columns " << std::endl; 91 | Row row; 92 | Ushort* col_len; 93 | if (supplemental) 94 | col_len = (Ushort*)change->part(data_offset - 1); 95 | else 96 | col_len = change->partLen(data_offset); 97 | for (Ushort i = 0; i < total_cols; ++i) { 98 | const char* src = change->part(data_offset + i); 99 | char* data = new char[*(col_len + i) + 1]; 100 | if (*(change->partLen(data_offset + i)) < *(col_len + i)) { 101 | LOG(DEBUG) << "column_len( " << *(col_len + i) 102 | << ") is bigger than data part length(" 103 | << *(change->partLen(data_offset + i)) << ")" << std::endl; 104 | *(col_len + i) = *(change->partLen(data_offset + i)); 105 | } 106 | memcpy(data, src, *(col_len + i)); 107 | data[*(col_len + i)] = '\0'; 108 | ColumnChange* col_change = NULL; 109 | if (supplemental) 110 | col_change = new ColumnChange(*(col_num + i) - 1, *(col_len + i), data); 111 | else if (col_num == NULL) 112 | col_change = new ColumnChange(i, *(col_len + i), data); 113 | else 114 | col_change = new ColumnChange(*(col_num + i), *(col_len + i), data); 115 | row.push_back(std::shared_ptr(col_change)); 116 | } 117 | return row; 118 | } 119 | 120 | Row makeUpCols(Ushort* col_num, Ushort total_cols, const ChangeHeader* change, 121 | Ushort data_offset, Ushort xtype, bool supplemental) { 122 | if (xtype & 0x80) 123 | return _makeUpLenPrefixCols(col_num, total_cols, change, data_offset); 124 | else 125 | return _makeUpNoLenPrefixCols(col_num, total_cols, change, data_offset, 126 | supplemental); 127 | } 128 | 129 | void Ops0501::printTransBase(const ChangeHeader* change0501) { 130 | std::cout << std::endl << "xid: " << getXID(change0501) 131 | << " object_id: " << getObjId(change0501) 132 | << " data_object_id: " << getDataObjId(change0501) << std::endl; 133 | } 134 | 135 | static void changeColsOffset(Row& row, Ushort start_col) { 136 | if (!row.empty() && start_col > 0) { 137 | Ushort n = row.front()->col_id_; 138 | for (auto& col : row) { 139 | col->col_id_ = start_col + (col->col_id_ - n); 140 | } 141 | } 142 | } 143 | 144 | std::list Ops0501::makeUpUndo(const ChangeHeader* change0501, 145 | RowChangePtr rcp) { 146 | std::list rows; 147 | Row row; 148 | OpCodeKdo* opkdo = (OpCodeKdo*)change0501->part(4); 149 | OpCode0501Sec* sec = (OpCode0501Sec*)(change0501->part(2)); 150 | rcp->uflag_ = ((OpCode0501*)(change0501->part(1)))->flag_; 151 | // printTransBase(change0501); 152 | // if there any exception if opkdo->opcode_ = 0501 153 | switch (opkdo->opcode_ & 0x1f) { 154 | case opcode::kInsert & 0xff: 155 | case opcode::kRowChain & 0xff: { 156 | // common delete will go to here 157 | // LOG(DEBUG) << "Normal Delete " << std::endl; 158 | OpCodeKdoirp* irp = (OpCodeKdoirp*)opkdo; 159 | rcp->cc_ = irp->column_count_; 160 | if (irp->column_count_ != 0) { 161 | Row undo_cols = makeUpCols((Ushort*)NULL, irp->column_count_, 162 | change0501, 5, irp->xtype_, false); 163 | row.splice(row.end(), undo_cols); 164 | } 165 | Ushort part_no; 166 | if (sec->op_major_ == 0x05 && sec->op_minor_ == 0x01) { 167 | part_no = 6 + irp->column_count_; 168 | } else { 169 | part_no = 5 + irp->column_count_; 170 | } 171 | Row suplemental_cols; 172 | if (irp->opcode_ & 0x20) { 173 | OpCodeSupplemental* opsup = 174 | (OpCodeSupplemental*)change0501->part(part_no++); 175 | if (opsup->start_column_ > 0) 176 | rcp->start_col_ = opsup->start_column_ - 1; 177 | suplemental_cols = _makeUpNoLenPrefixCols( 178 | (Ushort*)change0501->part(part_no), opsup->total_cols_, 179 | change0501, part_no + 2, true); 180 | } 181 | LOG(DEBUG) << "Header " << rcp->toString(); 182 | LOG(DEBUG) << "Undo " << rowAsString(row); 183 | LOG(DEBUG) << "Sup " << rowAsString(suplemental_cols); 184 | changeColsOffset(row, rcp->start_col_); 185 | if (!suplemental_cols.empty()) { 186 | row.splice(row.end(), suplemental_cols); 187 | } 188 | } break; 189 | case opcode::kLmn & 0xff: { 190 | OpCodeKdoirp* irp = (OpCodeKdoirp*)opkdo; 191 | if (irp->opcode_ & 0x20) { 192 | OpCodeSupplemental* opsup = (OpCodeSupplemental*)change0501->part(5); 193 | if (opsup->start_column_ > 0) 194 | rcp->start_col_ = opsup->start_column_ - 1; 195 | Row suplemental_cols = 196 | _makeUpNoLenPrefixCols((Ushort*)change0501->part(6), 197 | opsup->total_cols_, change0501, 8, true); 198 | 199 | LOG(DEBUG) << "Header " << rcp->toString(); 200 | LOG(DEBUG) << "Sup " << rowAsString(suplemental_cols); 201 | if (!suplemental_cols.empty()) { 202 | row.splice(row.end(), suplemental_cols); 203 | } 204 | } 205 | } break; 206 | /* 207 | case opcode::kMultiInsert & 0xff: { 208 | LOG(DEBUG) << "seems run into mulit-delete op " << std::endl; 209 | std::exit(100); 210 | } break; 211 | */ 212 | case opcode::kUpdate & 0xff: { 213 | OpCodeKdourp* urp = (OpCodeKdourp*)change0501->part(4); 214 | int total_colums = urp->ncol_; 215 | Ushort total_changes = urp->nchanged_; 216 | row = makeUpCols((Ushort*)change0501->part(5), total_changes, 217 | change0501, 6, urp->xtype_); 218 | 219 | Ushort part_num = 6 + total_changes; 220 | 221 | if (urp->opcode_ & 0x40) ++part_num; 222 | Row suplemental_cols; 223 | if (urp->opcode_ & 0x20) { 224 | OpCodeSupplemental* suplemental_op_header = 225 | (OpCodeSupplemental*)change0501->part(part_num++); 226 | if (suplemental_op_header->start_column_ > 0) 227 | rcp->start_col_ = suplemental_op_header->start_column_ - 1; 228 | suplemental_cols = 229 | _makeUpNoLenPrefixCols((Ushort*)change0501->part(part_num), 230 | suplemental_op_header->total_cols_, 231 | change0501, part_num + 2, true); 232 | LOG(DEBUG) << "Header " << rcp->toString(); 233 | LOG(DEBUG) << "Undo " << rowAsString(row); 234 | LOG(DEBUG) << "Sup " << rowAsString(suplemental_cols); 235 | } 236 | changeColsOffset(row, rcp->start_col_); 237 | row.splice(row.end(), suplemental_cols); 238 | } break; 239 | case opcode::kDelete & 0xff: { 240 | OpCodeKdodrp* drp = (OpCodeKdodrp*)change0501->part(4); 241 | if (drp->opcode_ & 0x20) { 242 | OpCodeSupplemental* sup = (OpCodeSupplemental*)change0501->part(5); 243 | if (sup->start_column2_ > 0) 244 | rcp->start_col_ = sup->start_column2_ - 1; 245 | if (sup->total_cols_ > 0) { 246 | row = _makeUpNoLenPrefixCols((Ushort*)change0501->part(6), 247 | sup->total_cols_, change0501, 8, true); 248 | } 249 | LOG(DEBUG) << "Header " << rcp->toString(); 250 | LOG(DEBUG) << "Sup " << rowAsString(row); 251 | } 252 | } break; 253 | case opcode::kMultiDelete & 0xff: 254 | // mulit_insert will go here, we should be able to find out the pks which 255 | // are inserted 256 | case opcode::kMfc & 0xff: 257 | case opcode::kCfa & 0xff: 258 | default: 259 | return rows; 260 | } 261 | rows.push_back(std::move(row)); 262 | return rows; 263 | } 264 | 265 | std::list OpsDML::makeUpRedoCols(const ChangeHeader* change, 266 | RowChangePtr rcp) { 267 | OpCodeKdo* kdo = (OpCodeKdo*)change->part(2); 268 | std::list redo_rows; 269 | Row redo_row; 270 | switch (change->opCode()) { 271 | case opcode::kRowChain: 272 | case opcode::kInsert: { 273 | OpCodeKdoirp* irp = (OpCodeKdoirp*)kdo; 274 | if (irp->flag_ & 0x80 || irp->flag_ & 0x40) { 275 | LOG(DEBUG) << "Found cluster op, bypass it"; 276 | break; 277 | } 278 | redo_row = 279 | makeUpCols(NULL, irp->column_count_, change, 3, irp->xtype_, false); 280 | rcp->iflag_ = irp->flag_; 281 | } break; 282 | case opcode::kUpdate: { 283 | OpCodeKdourp* urp = (OpCodeKdourp*)kdo; 284 | rcp->iflag_ = urp->flag_; 285 | redo_row = makeUpCols((Ushort*)change->part(3), 286 | ((OpCodeKdourp*)kdo)->nchanged_, change, 4, 287 | kdo->xtype_, false); 288 | } break; 289 | case opcode::kMultiInsert: { 290 | LOG(DEBUG) << "Mulit Insert" << std::endl; 291 | OpCodeKdoqm* qm = (OpCodeKdoqm*)change->part(2); 292 | Uchar* data = (Uchar*)change->part(4); 293 | for (int row = 0; row < qm->nrow_; ++row) { 294 | Uchar flag, col_count = 'x'; 295 | Ushort len; 296 | 297 | flag = *data; 298 | data += 2; 299 | if (!(flag & 0x10) || (flag & 0x40)) { 300 | col_count = *data++; 301 | if (flag & 0x08 && ~flag & 0x20) { 302 | data += sizeof(RedoRid); 303 | } 304 | Row temp_row; 305 | for (int i = 0; i < col_count; i++) { 306 | len = *data++; 307 | 308 | if (len == 255) { 309 | len = 0; 310 | temp_row.push_back(std::shared_ptr( 311 | new ColumnChange(i, len, NULL))); 312 | continue; 313 | } else if (len == 254) { 314 | len = *((Ushort*)data); // TODO: TEST here. the first byte is 315 | // discarded here! 316 | data += 2; 317 | } 318 | 319 | char* col_data = new char[len + 1]; 320 | memcpy(col_data, data, len); 321 | col_data[len] = '\0'; 322 | temp_row.push_back(std::shared_ptr( 323 | new ColumnChange(i, len, col_data))); 324 | data += len; 325 | } 326 | redo_rows.push_back(std::move(temp_row)); 327 | } 328 | } 329 | } break; 330 | default: 331 | break; 332 | } 333 | if (change->opCode() != opcode::kMultiInsert && !redo_row.empty()) { 334 | LOG(DEBUG) << "Redo " << rowAsString(redo_row); 335 | changeColsOffset(redo_row, rcp->start_col_); 336 | redo_rows.push_back(std::move(redo_row)); 337 | } 338 | return redo_rows; 339 | } 340 | } 341 | -------------------------------------------------------------------------------- /src/redo_parse/opcode_ops.h: -------------------------------------------------------------------------------- 1 | #ifndef OPCODE_OPS_INC 2 | #define OPCODE_OPS_INC 3 | 4 | #include 5 | #include 6 | 7 | #include "opcode.h" 8 | #include "physical_elems.h" 9 | #include "logical_elems.h" 10 | #include "util/dassert.h" 11 | #include "util/dtypes.h" 12 | #include "trans.h" 13 | 14 | namespace databus { 15 | using util::dassert; 16 | using util::strange; 17 | 18 | bool validOp(Ushort op); 19 | 20 | class Ops0501 { 21 | // I don't want to build a Ops0501 class per 0501 change, that will cause 22 | // too many construct and Destruction, that will be too ineffecitve 23 | public: 24 | // In c++11, return std::list is effective 25 | static std::list makeUpUndo(const ChangeHeader* change0501, 26 | RowChangePtr rcp); 27 | static bool enabledSuppmentalLog(const ChangeHeader* change0501); 28 | 29 | static XID getXID(const ChangeHeader* change0501); 30 | static uint32_t getObjId(const ChangeHeader* change0501); 31 | static uint32_t getDataObjId(const ChangeHeader* change0501); 32 | 33 | private: 34 | // only for test 35 | static void printTransBase(const ChangeHeader* change0501); 36 | }; 37 | 38 | class OpsDML { 39 | public: 40 | static std::list makeUpRedoCols(const ChangeHeader* change, 41 | RowChangePtr rcp); 42 | }; 43 | 44 | // TODO:when to relase the heap memory ?? 45 | Row _makeUpUncommCols(const char* start, int total_cols); 46 | Row _makeUpNoLenPrefixCols(Ushort* col_num, Ushort total_cols, 47 | const ChangeHeader* change, Ushort data_offset, 48 | bool supplemental = false); 49 | Row _makeUpLenPrefixCols(Ushort* col_num, Ushort total_cols, 50 | const ChangeHeader* change, 51 | OpCodeSupplemental** sup); 52 | Row makeUpCols(Ushort* col_num, Ushort total_cols, const ChangeHeader* change, 53 | Ushort data_offset, Ushort xtype, bool supplemental = false); 54 | } 55 | #endif /* ----- #ifndef OPCODE_OPS_INC ----- */ 56 | -------------------------------------------------------------------------------- /src/redo_parse/ora_num_dis.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | 4 | def positive(l): 5 | num = 0 6 | if l[0] >= 193: 7 | for n, arg1 in enumerate(l[1:]): 8 | num += (arg1 - 1) * (100 ** (l[0] - 193 - n)) 9 | elif l[0] <= 62: 10 | for n, arg2 in enumerate(l[1:-1]): 11 | num += (arg2 - 101) * (100 ** (62 - l[0] - n)) 12 | return num 13 | 14 | if __name__ == '__main__': 15 | print(positive([int(x, 16) for x in sys.argv[1].split()])) 16 | -------------------------------------------------------------------------------- /src/redo_parse/physical_elems.cpp: -------------------------------------------------------------------------------- 1 | #include "physical_elems.h" 2 | 3 | namespace databus { 4 | 5 | uint32_t ChangeHeader::changeSize() const { 6 | int elem_count = partCount(); 7 | if (elem_count == -1) return 0; // lol() == 0 8 | 9 | uint32_t total_length = 0; 10 | 11 | for (int i = 1; i <= elem_count; ++i) { 12 | Ushort len = *(Ushort*)(length_ + i * 2); 13 | total_length += align4(len); 14 | } 15 | 16 | return total_length + headerSize(); 17 | } 18 | 19 | Ushort* ChangeHeader::partLen(int part_no) const { 20 | return (Ushort*)(length_ + part_no * 2); 21 | } 22 | 23 | const char* ChangeHeader::part(int part_no) const { 24 | uint32_t offset = 0; 25 | for (int i = 0; i < part_no; ++i) { 26 | offset += align4(*partLen(i)); 27 | } 28 | return length_ + offset; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/redo_parse/physical_elems.h: -------------------------------------------------------------------------------- 1 | #ifndef PHYSICAL_ELEMS_INC 2 | #define PHYSICAL_ELEMS_INC 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "util/dassert.h" 8 | #include "util/dtypes.h" 9 | #include "logical_elems.h" 10 | 11 | namespace databus { 12 | using util::dassert; 13 | template 14 | T *As(const char *pointer) { 15 | return (T *)pointer; 16 | } 17 | 18 | namespace constants { 19 | const size_t kBlockHeaderSize = 16; 20 | } 21 | 22 | class FileHeaderV9 { 23 | public: 24 | uint32_t getBlockSize() const { return block_size_; } 25 | // if there are 100 block in physical, the last_block_id_ 26 | // here is 99 27 | uint32_t getLastBlockID() const { return last_block_id_; } 28 | 29 | private: 30 | Uchar zero_; 31 | Uchar type_; 32 | Ushort zero1_; 33 | uint32_t block_size_; 34 | uint32_t last_block_id_; 35 | Uchar reserved[500]; 36 | }; 37 | 38 | class FileHeaderV10 { 39 | public: 40 | uint32_t getBlockSize() const { return block_size_; } 41 | uint32_t getLastBlockID() const { return last_block_id_; } 42 | 43 | private: 44 | Uchar unknow_[16]; 45 | Uchar zero_; 46 | Uchar type_; // online? archived? 47 | Ushort zero1_; 48 | uint32_t block_size_; 49 | uint32_t last_block_id_; 50 | Uchar reserved_[484]; 51 | }; 52 | 53 | class BlockHeaderV9 { 54 | public: 55 | uint32_t getSequenceNum() { return sequence_no_; } 56 | uint32_t getBlockId() { return block_id_; } 57 | Ushort getFirstOffset() { return first_record_offset_; } 58 | 59 | private: 60 | uint32_t sequence_no_; 61 | uint32_t block_id_; 62 | uint32_t epoch_; 63 | Ushort first_record_offset_; 64 | Ushort r_; 65 | }; 66 | 67 | class BlockHeaderV10 { 68 | public: 69 | uint32_t getSequenceNum() { return sequence_no_; } 70 | uint32_t getBlockId() { return block_id_; } 71 | Ushort getFirstOffset() { return first_record_offset_ & 0x7FFF; } 72 | 73 | private: 74 | uint32_t block_type_; // unknow 75 | uint32_t block_id_; 76 | uint32_t sequence_no_; 77 | Ushort first_record_offset_; 78 | Ushort r_; 79 | }; 80 | 81 | class RedoHeader { 82 | public: 83 | const char *DbName() { return db_name_; } 84 | uint32_t fileSize() { return file_size_; } 85 | uint32_t blockSize() { return block_size_; } 86 | SCN lowScn() { return SCN(low_scn_major_, low_scn_minor_); } 87 | SCN nextScn() { return SCN(next_scn_major_, next_scn_minor_); } 88 | Uchar getOraVersion() { return compatible_version_ >> 24; } 89 | 90 | uint32_t db_version_; // don't use this one 91 | uint32_t compatible_version_; 92 | uint32_t db_id_; 93 | char db_name_[8]; 94 | uint32_t control_sequence_; 95 | uint32_t file_size_; 96 | uint32_t block_size_; 97 | Ushort file_number_; 98 | Ushort file_type_; 99 | uint32_t activation_id_; 100 | Uchar empty_38h_[0x24]; 101 | char description_[0x40]; 102 | uint32_t nab_; 103 | uint32_t reset_log_count_; 104 | uint32_t scn_minor_; 105 | // The following two fields may be a single 32 bit field, 106 | // this looks like the case on a bigendian dump. 107 | Ushort scn_major_; 108 | Uchar empty_aah_[2]; 109 | uint32_t hws_; 110 | Ushort thread_; 111 | Uchar empty_b2h_[2]; 112 | 113 | uint32_t low_scn_minor_; 114 | Ushort low_scn_major_; 115 | Uchar empty_bah_[2]; 116 | uint32_t low_scn_epoch_; 117 | 118 | uint32_t next_scn_minor_; 119 | Ushort next_scn_major_; 120 | Uchar empty_c6h_[2]; 121 | uint32_t next_scn_epoch_; 122 | 123 | Uchar eot_; 124 | Uchar dis_; 125 | Uchar empty_ceh_[2]; 126 | 127 | uint32_t enabled_scn_minor_; 128 | Ushort enabled_scn_major_; 129 | Uchar empty_d6h_[2]; 130 | uint32_t enabled_scn_epoch_; 131 | 132 | uint32_t thread_closed_scn_minor_; 133 | Ushort thread_closed_scn_major_; 134 | Uchar empty_e2h_[2]; 135 | uint32_t thread_closed_scn_epoch_; 136 | 137 | uint32_t log_format_version_; 138 | uint32_t flags_; 139 | 140 | uint32_t terminal_scn_minor_; 141 | Ushort terminal_scn_major_; 142 | Uchar empty_f6h_[2]; 143 | uint32_t terminal_epoch_; 144 | Uchar unused_[260]; 145 | }; 146 | 147 | class RecordHeaderV9 { 148 | public: 149 | SCN scn() { return SCN(scn_major_, scn_minor_, subscn_); } 150 | 151 | private: 152 | uint32_t record_len_; 153 | Uchar vld; 154 | Uchar subscn_; 155 | Ushort scn_major_; 156 | uint32_t scn_minor_; 157 | }; 158 | 159 | class RecordHeaderMinor { 160 | // total lenght: 22 161 | public: 162 | SCN scn() { return SCN(scn_major_, scn_minor_, subscn_); } 163 | 164 | private: 165 | uint32_t record_length; 166 | Uchar vld_; 167 | Uchar foo_; 168 | Ushort scn_major_; 169 | uint32_t scn_minor_; 170 | Ushort subscn_; 171 | Ushort unknown_1; 172 | Ushort padding_1; 173 | uint32_t padding_2; 174 | }; 175 | 176 | class RecordHeaderMajor { 177 | public: 178 | SCN scn() { return SCN(scn_major_, scn_minor_, subscn_); } 179 | uint32_t getEpoch() { return epoch_; } 180 | 181 | private: 182 | uint32_t record_length_; 183 | Uchar vld_; 184 | Uchar foo1_; 185 | Ushort scn_major_; 186 | uint32_t scn_minor_; 187 | Ushort subscn_; 188 | Ushort unknown_1_; 189 | Ushort foo2_; 190 | int unknown2_[10]; 191 | int thread_; 192 | uint32_t epoch_; 193 | }; 194 | 195 | class ChangeHeader { 196 | public: 197 | Ushort opCode() const { return op_major_ << 8 | op_minor_; }; 198 | Uchar major() const { return op_major_; } 199 | 200 | uint32_t dba() const { return dba_; } 201 | Uchar type() const { return type_; } 202 | 203 | // offset {headerSize()} will be the start of opCode_xxxx 204 | uint32_t headerSize() const { return sizeof(ChangeHeader) + align4(lol()); } 205 | // size of opCode_xxxx 206 | uint32_t changeSize() const; 207 | const SCN scn() const { return SCN(high_scn_, low_scn_, seq_); } 208 | 209 | // return the pointer of the part_n 210 | const char *part(int part_no) const; 211 | Ushort *partLen(int part_no) const; 212 | 213 | private: 214 | Ushort lol() const { return *(Ushort *)length_; } // length of length 215 | uint32_t align4(uint32_t n) const { 216 | if (n % 4 == 0) return n; 217 | return (n / 4 + 1) * 4; 218 | }; 219 | int partCount() const { return lol() / 2 - 1; } 220 | 221 | public: 222 | Uchar op_major_; 223 | Uchar op_minor_; 224 | Ushort block_class_; 225 | uint32_t file_id_; 226 | uint32_t dba_; 227 | uint32_t low_scn_; 228 | Ushort high_scn_; 229 | Uchar zero_[2]; 230 | Uchar seq_; 231 | Uchar type_; 232 | Ushort zero2; 233 | char length_[0]; // array of length 234 | }; // ChangeHeader 235 | 236 | namespace immature { 237 | const std::set major_vlds{0x4, 0x5, 0x6, 0x9, 0xd}; 238 | const std::set minor_vlds{0x1, 0x2}; 239 | 240 | extern inline bool isMajor(Uchar vld) { 241 | return major_vlds.find(vld) != major_vlds.end(); 242 | } 243 | 244 | extern inline bool isMinor(Uchar vld) { 245 | return minor_vlds.find(vld) != minor_vlds.end(); 246 | } 247 | 248 | extern inline bool testedVersion(Uchar ora_version) { 249 | return ora_version > 8 && ora_version < 12; 250 | } 251 | 252 | extern inline void notTestedVersionDie(Uchar ora_version) { 253 | std::stringstream ss; 254 | int v = ora_version; 255 | ss << "Only 9i/10g/11g supported, but found version " << v; 256 | dassert(ss.str().c_str(), false); 257 | } 258 | 259 | extern inline uint32_t recordLength(const char *p_record, 260 | Uchar ora_version) { 261 | if (testedVersion(ora_version)) 262 | return *((uint32_t *)p_record); 263 | else { 264 | notTestedVersionDie(ora_version); 265 | return 0; 266 | } 267 | } 268 | 269 | extern inline Uchar recordVld(const char *p_record, Uchar ora_version) { 270 | if (testedVersion(ora_version)) 271 | return *((Uchar *)(p_record + sizeof(int))); 272 | else { 273 | notTestedVersionDie(ora_version); 274 | return 0; 275 | } 276 | } 277 | 278 | extern inline SCN recordSCN(const char *p_record, Uchar version) { 279 | switch (version) { 280 | case 9: 281 | return As(p_record)->scn(); 282 | case 10: 283 | case 11: 284 | return As(p_record)->scn(); 285 | default: { 286 | notTestedVersionDie(version); 287 | return SCN(0, 0); 288 | } 289 | } 290 | } 291 | 292 | extern inline Uchar oracleVersion(const char *p_logfile) { 293 | return As(p_logfile + 512 + constants::kBlockHeaderSize) 294 | ->getOraVersion(); 295 | }; 296 | } 297 | 298 | namespace constants { 299 | const size_t kMinRecordLen = sizeof(RecordHeaderMinor); 300 | const size_t kMinMajRecordLen = sizeof(RecordHeaderMajor); 301 | const size_t kChangeHeaderSize = sizeof(ChangeHeader); 302 | } 303 | } 304 | 305 | #endif /* ----- #ifndef PHYSICAL_ELEMS_INC ----- */ 306 | -------------------------------------------------------------------------------- /src/redo_parse/pk.sql: -------------------------------------------------------------------------------- 1 | select COLUMN_ID, COLUMN_NAME, DATA_TYPE from dba_tab_cols where 2 | owner=upper('&1') and table_name=upper('&2') 3 | and column_id is not null 4 | / 5 | 6 | select column_id 7 | from dba_cons_columns col, dba_constraints con, dba_tab_cols tc 8 | where con.owner=col.owner 9 | and con.table_name = col.table_name 10 | and con.CONSTRAINT_NAME= col.CONSTRAINT_NAME 11 | and con.CONSTRAINT_TYPE='P' 12 | and con.owner=upper('&1') 13 | and con.table_name=upper('&2') 14 | and col.column_name = tc.column_name 15 | and tc.table_name = upper('&2') 16 | and tc.owner = upper('&1') 17 | / 18 | -------------------------------------------------------------------------------- /src/redo_parse/redofile.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "redofile.h" 14 | #include "physical_elems.h" 15 | #include "util/dassert.h" 16 | #include "opcode_ops.h" 17 | #include "util/logger.h" 18 | 19 | namespace databus { 20 | using util::dassert; 21 | 22 | void RedoFile::init(const char* filename) { 23 | LOG(INFO) << "RedoFile::init " << filename << std::endl; 24 | 25 | if (file_start_pos_ != NULL) munmap(file_start_pos_, length_); 26 | int fd = open(filename, O_RDONLY, 0644); 27 | 28 | dassert(strerror(errno), fd != -1, errno); 29 | 30 | struct stat stats; 31 | dassert(strerror(errno), fstat(fd, &stats) == 0, errno); 32 | off_t l = 4294967296; 33 | dassert("Redo file is larger than 4G, exiting..", stats.st_size <= l); 34 | // we need to change scn offset_ if this happened 35 | length_ = stats.st_size; 36 | file_start_pos_ = (char*)mmap(0, length_, PROT_READ, MAP_PRIVATE, fd, 0); 37 | dassert(strerror(errno), file_start_pos_ != NULL, errno); 38 | close(fd); 39 | 40 | ora_version_ = immature::oracleVersion(file_start_pos_); 41 | 42 | switch (ora_version_) { 43 | case 9: 44 | block_size_ = As(file_start_pos_)->getBlockSize(); 45 | last_block_id_ = As(file_start_pos_)->getLastBlockID(); 46 | p_block_header_ = file_start_pos_ + block_size_; 47 | if (log_sequence_ == -1) { 48 | log_sequence_ = ((BlockHeaderV9*)p_block_header_)->getSequenceNum(); 49 | } 50 | assert(log_sequence_ == 51 | ((BlockHeaderV9*)p_block_header_)->getSequenceNum()); 52 | break; 53 | case 10: 54 | case 11: 55 | block_size_ = As(file_start_pos_)->getBlockSize(); 56 | last_block_id_ = As(file_start_pos_)->getLastBlockID(); 57 | p_block_header_ = file_start_pos_ + block_size_; 58 | if (log_sequence_ == -1) { 59 | log_sequence_ = ((BlockHeaderV10*)p_block_header_)->getSequenceNum(); 60 | } 61 | dassert("", log_sequence_ == 62 | ((BlockHeaderV10*)p_block_header_)->getSequenceNum()); 63 | break; 64 | default: 65 | immature::notTestedVersionDie(ora_version_); 66 | } 67 | 68 | p_redo_header_ = As(file_start_pos_ + block_size_ + 69 | constants::kBlockHeaderSize); 70 | lowscn_ = p_redo_header_->lowScn(); 71 | curr_record_pos_ = firstRecord(); 72 | } 73 | 74 | RedoFile::~RedoFile() { 75 | if (file_start_pos_ != NULL) munmap(file_start_pos_, length_); 76 | } 77 | 78 | void RedoFile::realCopyNBytes(char* from, char* to, uint32_t len) { 79 | uint32_t this_block_space_left = spaceLeft(from); 80 | if (this_block_space_left >= len) { 81 | memcpy(to, from, len); 82 | return; 83 | } 84 | 85 | uint32_t block_data_size = block_size_ - constants::kBlockHeaderSize; 86 | 87 | memcpy(to, from, this_block_space_left); 88 | len -= this_block_space_left; 89 | to += this_block_space_left; 90 | from = from + this_block_space_left + constants::kBlockHeaderSize; 91 | 92 | while (len > block_data_size) { 93 | memcpy(to, from, block_data_size); 94 | len -= block_data_size; 95 | to += block_data_size; 96 | from = from + block_data_size + constants::kBlockHeaderSize; 97 | } 98 | 99 | memcpy(to, from, len); 100 | } 101 | 102 | RecordBufPtr RedoFile::nextRecordBuf() { 103 | again: 104 | if (curr_record_pos_ == NULL) return NULL; // eof 105 | uint32_t record_len = 106 | immature::recordLength(curr_record_pos_, ora_version_); 107 | uint32_t change_length = 0; 108 | char* change_pos = NULL; 109 | SCN record_scn; 110 | Uchar vld; 111 | 112 | if (ora_version_ == 9) { 113 | change_length = record_len - constants::kMinRecordLen; 114 | change_pos = curr_record_pos_ + constants::kMinRecordLen; 115 | record_scn = As(curr_record_pos_)->scn(); 116 | } else { 117 | vld = immature::recordVld(curr_record_pos_, ora_version_); 118 | 119 | if (immature::isMajor(vld)) { 120 | change_length = record_len - constants::kMinMajRecordLen; 121 | change_pos = curr_record_pos_ + constants::kMinMajRecordLen; 122 | record_scn = As(curr_record_pos_)->scn(); 123 | epoch_ = As(curr_record_pos_)->getEpoch(); 124 | } else if (immature::isMinor(vld)) { 125 | change_length = record_len - constants::kMinRecordLen; 126 | change_pos = curr_record_pos_ + constants::kMinRecordLen; 127 | record_scn = As(curr_record_pos_)->scn(); 128 | } else if (vld == 0) { 129 | curr_record_pos_ = nextRecord(curr_record_pos_); 130 | goto again; 131 | } else { 132 | int ivld = vld; 133 | LOG(ERROR) << "unsupport vld " << ivld << " offset " 134 | << curr_record_pos_ - file_start_pos_ << std::endl; 135 | curr_record_pos_ = nextRecord(curr_record_pos_); 136 | goto again; 137 | } 138 | } 139 | 140 | if (change_length == 0 || record_scn < start_scn_) { 141 | curr_record_pos_ = nextRecord(curr_record_pos_); 142 | goto again; 143 | } 144 | 145 | if (spaceLeft(change_pos) == block_size_) 146 | change_pos += constants::kBlockHeaderSize; 147 | size_t offset = curr_record_pos_ - file_start_pos_; 148 | char* change_buf = new char[change_length]; 149 | realCopyNBytes(change_pos, change_buf, change_length); 150 | if (isOverWrite()) { 151 | delete[] change_buf; 152 | init(log_generator_(log_sequence_).c_str()); 153 | curr_record_pos_ = file_start_pos_ + offset; 154 | goto again; 155 | } 156 | 157 | record_scn.noffset_ = offset; 158 | // TODO: remove noffset in record_buf since we can knows this from SCN 159 | RecordBufPtr record_buf(new RecordBuf(record_scn, change_length, epoch_, 160 | change_buf, offset, log_sequence_, 161 | vld, allop_)); 162 | 163 | curr_record_pos_ = nextRecord(curr_record_pos_); // for next round 164 | return record_buf; 165 | } 166 | 167 | bool RedoFile::isValid(char* p_record) { 168 | if (getBlockNo(p_record) > last_block_id_) return false; 169 | 170 | uint32_t left = spaceLeft((char*)p_record); 171 | if (left < constants::kMinRecordLen) { 172 | return false; 173 | } 174 | 175 | if (immature::recordLength(p_record, ora_version_) < 176 | constants::kMinRecordLen) { 177 | return false; 178 | } 179 | 180 | SCN scn = immature::recordSCN(p_record, ora_version_); 181 | if (!(scn < p_redo_header_->nextScn()) || scn < lowscn_) { 182 | LOG(DEBUG) << "Invalid scn " << scn.toStr(); 183 | return false; 184 | } 185 | return true; 186 | } 187 | 188 | Ushort RedoFile::recordOffset(char* p) { 189 | Ushort offset = 0; 190 | if (ora_version_ == 9) 191 | offset = As(p)->getFirstOffset(); 192 | else 193 | offset = As(p)->getFirstOffset(); 194 | return offset; 195 | } 196 | 197 | // changed after 61d61dbd1b016888952a7c7c2a5fa834713b2d4c 198 | char* RedoFile::nextRecord(char* curr_valid_rec) { 199 | bool online_log = false; 200 | char* pos; 201 | tryagain: 202 | pos = realAdvanceNBytes( 203 | curr_valid_rec, immature::recordLength(curr_valid_rec, ora_version_)); 204 | if (isOverWrite()) { 205 | pos = resetPosition(pos); 206 | } 207 | while (!isValid(pos)) { 208 | uint32_t blk_id = getBlockNo(pos); 209 | // std::cout << "Block id " << getBlockNo(pos) << " offset " 210 | // << block_size_ - spaceLeft(pos) << std::endl; 211 | 212 | if (blk_id == last_block_id_) { 213 | // last block already 214 | return NULL; 215 | } 216 | 217 | if (p_redo_header_->next_scn_minor_ == 0xFFFFFFFF && 218 | p_redo_header_->next_scn_major_ == 0xFFFF) { 219 | online_log = true; 220 | if (isOverRead(blk_id)) { 221 | LOG(DEBUG) << "blocking on the last block of online log" 222 | << "blk_id " << blk_id << " latest_blk " << latest_blk_ 223 | << std::endl; 224 | sleep(3); 225 | pos = curr_valid_rec; 226 | goto tryagain; 227 | } 228 | } else { 229 | if (online_log) { 230 | pos = resetPosition(pos); 231 | blk_id = getBlockNo(pos); 232 | online_log = false; 233 | } 234 | } 235 | 236 | if (blk_id > last_block_id_) { 237 | return NULL; 238 | } 239 | 240 | // std::cout << "skip to next block \n"; 241 | char* block_addr = (blk_id + 1) * block_size_ + file_start_pos_; 242 | pos = block_addr + recordOffset(block_addr); 243 | } 244 | return pos; 245 | } 246 | 247 | char* RedoFile::firstRecord() { 248 | char* firstblock = file_start_pos_ + 2 * block_size_; 249 | Ushort offset = recordOffset(firstblock); 250 | return (file_start_pos_ + 2 * block_size_ + offset); 251 | } 252 | 253 | char* RedoFile::_realAdvanceNBytes(char* p_record, size_t bytes_count) { 254 | dassert("p_record must be the begining of a block", 255 | (p_record - file_start_pos_) % block_size_ == 0); 256 | size_t block_data_size = block_size_ - constants::kBlockHeaderSize; 257 | return p_record + (bytes_count / block_data_size) * block_size_ + 258 | bytes_count % block_data_size + constants::kBlockHeaderSize; 259 | } 260 | 261 | char* RedoFile::realAdvanceNBytes(char* p_record, size_t bytes_count) { 262 | size_t left = spaceLeft(p_record); 263 | if (bytes_count < left) return p_record + bytes_count; 264 | 265 | if (bytes_count == left) 266 | return p_record + bytes_count + constants::kBlockHeaderSize; 267 | 268 | return _realAdvanceNBytes(p_record + left, bytes_count - left); 269 | } 270 | 271 | // p_record is a valid record 272 | /* 273 | char* RedoFile::nextRecord(char* p_record) { 274 | char* pos = realAdvanceNBytes( 275 | p_record, immature::recordLength(p_record, ora_version_)); 276 | return nextValid(pos, p_record); 277 | } */ 278 | } 279 | -------------------------------------------------------------------------------- /src/redo_parse/redofile.h: -------------------------------------------------------------------------------- 1 | #ifndef REDOFILE_INC 2 | #define REDOFILE_INC 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "logical_elems.h" 10 | #include "physical_elems.h" 11 | 12 | namespace databus { 13 | 14 | typedef unsigned char Uchar; 15 | 16 | class RedoFile { 17 | public: 18 | // One RedoFile is a represent of a logfile whose 19 | // seq equals log_seq. Since the the same log_seq 20 | // maybe different logfile, so we can't initilize 21 | // RedoFile with a char* filename any more 22 | RedoFile(uint32_t log_seq, 23 | std::function log_generator, 24 | std::function last_blk) 25 | : log_generator_(log_generator), 26 | log_sequence_(log_seq), 27 | online_last_blk_(last_blk), 28 | latest_blk_(0), 29 | file_start_pos_(NULL), 30 | length_(0), 31 | start_scn_(), 32 | allop_(false), 33 | epoch_(0) { 34 | std::string filename = log_generator_(log_seq); 35 | init(filename.c_str()); 36 | }; 37 | 38 | // this is just for some test purposes, works for archivelog only 39 | RedoFile(const char *filename) 40 | : latest_blk_(0xFFFFFFFF), 41 | file_start_pos_(NULL), 42 | length_(0), 43 | log_sequence_(-1), 44 | start_scn_(), 45 | allop_(true), 46 | epoch_(0) { 47 | log_generator_ = [&filename](uint32_t) { return filename; }; 48 | online_last_blk_ = [](uint32_t) { return 0xFFFFFF; }; 49 | init(filename); 50 | } 51 | 52 | // should return NULL if current logfile reach to 53 | // end. 54 | // even if during we read a online logfile, 55 | // the online logfile was re-write(switch logfile) 56 | // just turn to the archive log. Be sure to return 57 | // a valid RecordBuf with this call! 58 | std::shared_ptr nextRecordBuf(); 59 | 60 | void setStartScn(const SCN &scn) { start_scn_ = scn; } 61 | SCN getStartScn() const { return start_scn_; } 62 | SCN getFirstScn() const { return lowscn_; } 63 | 64 | ~RedoFile(); 65 | 66 | private: 67 | void init(const char *filename); 68 | // nextRecord will only return valided records, the validation rules 69 | // are defined here 70 | bool isValid(char *p_record); 71 | Ushort recordOffset(char *p); 72 | 73 | char *resetPosition(char *pos) { 74 | size_t offset = pos - file_start_pos_; 75 | init(log_generator_(log_sequence_).c_str()); 76 | return file_start_pos_ + offset; 77 | } 78 | 79 | char *firstRecord(); 80 | // NULL if the end of file 81 | // switch to archive log file if overwrited 82 | // if no more data to read, just sleep and try again 83 | char *nextRecord(char *cur_record); 84 | 85 | void realCopyNBytes(char *from, char *to, uint32_t len); 86 | 87 | // A Record may take more than 1 blocks, 88 | // every block has a block header which can't store any record content 89 | // so if Record len is n, we need to advance more than n Bytes probably 90 | char *realAdvanceNBytes(char *p_record, size_t byte_count); 91 | 92 | // p_record here must be the exact begining of a block 93 | char *_realAdvanceNBytes(char *p_record, size_t byte_count); 94 | 95 | // given a position, get how many bytes left in that block 96 | uint32_t spaceLeft(char *p) { 97 | return block_size_ - (p - file_start_pos_) % block_size_; 98 | } 99 | 100 | Uchar oraVersion() { return ora_version_; } 101 | 102 | // start from 0 103 | uint32_t getBlockNo(char *p) { 104 | return (p - file_start_pos_) / block_size_; 105 | }; 106 | 107 | bool isOverRead(uint32_t blk) { 108 | if (blk <= latest_blk_) 109 | return false; 110 | else 111 | latest_blk_ = online_last_blk_(log_sequence_); 112 | return latest_blk_ < blk; 113 | } 114 | 115 | bool isOverWrite() { 116 | switch (ora_version_) { 117 | case 9: 118 | return log_sequence_ != 119 | ((BlockHeaderV9 *)p_block_header_)->getSequenceNum(); 120 | case 10: 121 | case 11: 122 | return log_sequence_ != 123 | ((BlockHeaderV10 *)p_block_header_)->getSequenceNum(); 124 | } 125 | // never come here, see "notTestVersionDie(ora_version_)" 126 | return false; 127 | } 128 | 129 | private: 130 | uint32_t log_sequence_; 131 | 132 | std::function log_generator_; 133 | std::function online_last_blk_; 134 | uint32_t latest_blk_; 135 | 136 | char *file_start_pos_; 137 | char *p_block_header_; 138 | RedoHeader *p_redo_header_; 139 | SCN lowscn_; 140 | SCN start_scn_; 141 | size_t length_; 142 | uint32_t block_size_; 143 | uint32_t last_block_id_; 144 | uint32_t epoch_; 145 | Uchar ora_version_; 146 | bool allop_; // log all op in RecordBuf, not only 11.x 147 | 148 | char *curr_record_pos_; 149 | }; 150 | } 151 | #endif /* ----- #ifndef REDOFILE_INC ----- */ 152 | -------------------------------------------------------------------------------- /src/redo_parse/stream.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "util/dassert.h" 10 | #include "util/logger.h" 11 | #include "stream.h" 12 | #include "metadata.h" 13 | #include "applier.h" 14 | #include "monitor.h" 15 | #include "redofile.h" 16 | 17 | namespace databus { 18 | StreamConf* streamconf; 19 | 20 | static void normalFile(std::ifstream& inf, std::stringstream& ss) { 21 | std::string s; 22 | const int max_line_size = 1024; 23 | char line[1024]; 24 | while (inf) { 25 | inf.getline(line, max_line_size); 26 | s = line; 27 | boost::trim(s); 28 | if (boost::starts_with(s, "#")) continue; 29 | if (!s.empty()) ss << s << std::endl; 30 | } 31 | } 32 | 33 | StreamConf::StreamConf(int argc, char** argv) : desc("Supported Options") { 34 | add_options(); 35 | po::store(po::parse_command_line(argc, argv, desc), vm); 36 | if (vm.count("confFile")) { 37 | std::ifstream inf(vm["confFile"].as().c_str()); 38 | util::dassert("Can't open read from config file", inf.is_open()); 39 | std::stringstream ss; 40 | normalFile(inf, ss); 41 | po::store(po::parse_config_file(ss, desc), vm); 42 | } 43 | if (vm.count("help")) { 44 | std::cout << desc << std::endl; 45 | std::exit(1); 46 | } 47 | po::notify(vm); 48 | validParams(); 49 | } 50 | 51 | void StreamConf::add_options() { 52 | desc.add_options()("help,h", "show help message")( 53 | "srcConn", po::value(), 54 | "string for source database, format username/password@vm")( 55 | "tarConn", po::value(), 56 | "string for target database username/password@vm")( 57 | "instId", po::value(), "DataStream instance Id")( 58 | "confFile", po::value(), "configure file for data stream")( 59 | "tableConf", po::value(), "tables to capture changes")( 60 | "startSeq", po::value(), 61 | "the log sequence to start with, will be removed soon"); 62 | } 63 | 64 | void StreamConf::validParams() { 65 | if (vm.count("help")) { 66 | std::cout << desc << std::endl; 67 | std::exit(1); 68 | } 69 | util::dassert("srcConn/tarConn/tableConf are must", 70 | vm.count("srcConn") && vm.count("tarConn") && 71 | vm.count("tableConf") && vm.count("instId")); 72 | } 73 | 74 | int StreamConf::getInt(const char* para, int default_value) { 75 | if (vm.count(para)) return vm[para].as(); 76 | return default_value; 77 | } 78 | 79 | uint32_t StreamConf::getUint32(const char* para) { 80 | return vm[para].as(); 81 | } 82 | 83 | std::string StreamConf::getString(const char* para, 84 | const char* default_value) { 85 | if (vm.count(para)) return vm[para].as(); 86 | return default_value; 87 | } 88 | 89 | bool StreamConf::getBool(const char* para, bool default_value) { 90 | if (vm.count(para)) return vm[para].as(); 91 | return default_value; 92 | } 93 | 94 | static std::list initCaptualTable(const char* filename) { 95 | std::list cap_tables; 96 | std::stringstream ss; 97 | std::ifstream inf(filename); 98 | util::dassert("Can't open read from config file", inf.is_open()); 99 | normalFile(inf, ss); 100 | std::string line; 101 | std::vector v(2); 102 | while (ss) { 103 | std::getline(ss, line); 104 | boost::trim(line); 105 | if (boost::starts_with(line, "#")) { 106 | continue; 107 | } 108 | boost::split(v, line, boost::is_any_of("#")); 109 | switch (v.size()) { 110 | case 1: 111 | cap_tables.push_back(TabConf(boost::trim_copy(v[0]))); 112 | break; 113 | case 2: 114 | cap_tables.push_back( 115 | TabConf(boost::trim_copy(v[0]), boost::trim_copy(v[1]))); 116 | break; 117 | default: 118 | LOG(ERROR) << "bad format of tabconf " << line; 119 | } 120 | } 121 | return cap_tables; 122 | } 123 | 124 | void initStream(int ac, char** av) { 125 | streamconf = new StreamConf(ac, av); 126 | std::stringstream ss; 127 | ss << "logging_" << streamconf->getUint32("instId") << ".conf"; 128 | el::Configurations conf(ss.str().c_str()); 129 | el::Loggers::reconfigureLogger("default", conf); 130 | auto captual_tables = 131 | initCaptualTable(streamconf->getString("tableConf").c_str()); 132 | for (auto table_conf : captual_tables) { 133 | auto first = table_conf.tab_name.find_first_of('.'); 134 | auto last = table_conf.tab_name.find_last_of('.'); 135 | if (first == last && first != table_conf.tab_name.npos) { 136 | std::string owner = table_conf.tab_name.substr(0, first); 137 | std::string tablename = table_conf.tab_name.substr( 138 | first + 1, table_conf.tab_name.npos - first - 1); 139 | LOG(INFO) << "Init table " << table_conf.tab_name; 140 | auto tab_def = 141 | getMetadata().initTabDefFromName(owner.c_str(), tablename.c_str()); 142 | if (tab_def != NULL) { 143 | LOG(DEBUG) << " init tab def " << tab_def->toString(); 144 | SimpleApplier::getApplier(streamconf->getString("tarConn").c_str(), 145 | streamconf->getString("srcConn").c_str()) 146 | .addTable(tab_def, table_conf); 147 | } 148 | // if (tabdef) tabdef->dump(); 149 | } else { 150 | if (!table_conf.tab_name.empty()) 151 | LOG(WARNING) << "Invalid Table format " << table_conf.tab_name 152 | << ", The correct format is owner.table_name"; 153 | } 154 | } 155 | } 156 | 157 | std::string getLogfile(uint32_t seq) { 158 | return getLogManager().getLogfile(seq); 159 | } 160 | 161 | uint32_t getOnlineLastBlock(uint32_t seq) { 162 | return getLogManager().getOnlineLastBlock(seq); 163 | } 164 | 165 | void startMining(uint32_t seq, const TimePoint& tm) { 166 | while (true) { 167 | if (seq - GlobalStream::getGlobalStream().getAppliedSeq() > 2) { 168 | LOG(DEBUG) << "Sleep 3 seconds, applied seq is " 169 | << GlobalStream::getGlobalStream().getAppliedSeq(); 170 | sleep(3); 171 | continue; 172 | } 173 | RedoFile redofile(seq, getLogfile, getOnlineLastBlock); 174 | redofile.setStartScn(tm.scn_); 175 | RecordBufPtr buf; 176 | unsigned long c = 0; 177 | while ((buf = redofile.nextRecordBuf()).get()) { 178 | if (buf->change_vectors.empty()) continue; 179 | getRecordBufList().push_back(buf); 180 | } 181 | ++seq; 182 | } 183 | } 184 | 185 | /* 186 | void applyRecordBuf() { 187 | uint32_t curr_seq = GlobalStream::getGlobalStream().getAppliedSeq(); 188 | LOG(DEBUG) << "Last applied seq " << curr_seq; 189 | while ((curr_record_ = getRecordBufList().pop_front()) != NULL) { 190 | if (curr_seq == curr_record_->seq_) { 191 | addToTransaction(curr_record_); 192 | } else { 193 | LOG(DEBUG) << "Bug seq " << buf->seq_ << " Last applied seq " 194 | << curr_seq; 195 | auto n = Transaction::removeUncompletedTrans(); 196 | if (n > 0) 197 | LOG(WARNING) << "removed " << n 198 | << " incompleted transaction in log seq " << curr_seq; 199 | LOG(INFO) << "Build Transaction now" << std::endl; 200 | auto tran = Transaction::xid_map_.begin(); 201 | while (tran != Transaction::xid_map_.end()) { 202 | auto it = buildTransaction(tran); 203 | if (it != Transaction::xid_map_.end()) { 204 | tran = it; 205 | } else { 206 | tran++; 207 | } 208 | } 209 | 210 | if (!Transaction::start_scn_q_.empty()) { 211 | auto it = Transaction::start_scn_q_.begin(); 212 | Transaction::setRestartTimePoint(it->first, it->second); 213 | } 214 | 215 | LOG(INFO) << "Apply Transaction now, Total " 216 | << Transaction::commit_trans_.size() << " to apply " 217 | << std::endl; 218 | auto commit_tran = Transaction::commit_trans_.begin(); 219 | while (commit_tran != Transaction::commit_trans_.end()) { 220 | SimpleApplier::getApplier(streamconf->getString("tarConn").c_str()) 221 | .apply(commit_tran->second); 222 | commit_tran = Transaction::commit_trans_.erase(commit_tran); 223 | } 224 | GlobalStream::getGlobalStream().setAppliedSeq(++curr_seq); 225 | } 226 | } 227 | } */ 228 | 229 | void startStream(uint32_t seq, const TimePoint& tm) { 230 | std::thread mining{startMining, seq, tm}; 231 | mining.detach(); 232 | std::thread applier{ApplierManager::getApplierManager()}; 233 | applier.detach(); 234 | } 235 | } 236 | -------------------------------------------------------------------------------- /src/redo_parse/stream.h: -------------------------------------------------------------------------------- 1 | #ifndef REDO_PARSE_INC 2 | #define REDO_PARSE_INC 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "util/container.h" 15 | #include "metadata.h" 16 | #include "logical_elems.h" 17 | #include "trans.h" 18 | 19 | namespace databus { 20 | namespace po = boost::program_options; 21 | class StreamConf { 22 | public: 23 | StreamConf(int ac, char** av); 24 | int getInt(const char*, int default_value = -1); 25 | std::string getString(const char*, const char* default_value = ""); 26 | bool getBool(const char*, bool default_value = false); 27 | uint32_t getUint32(const char* para); 28 | 29 | private: 30 | void add_options(); 31 | void validParams(); 32 | void parseConfigFile(std::ifstream& inf, std::stringstream& ss); 33 | 34 | private: 35 | po::options_description desc; 36 | po::variables_map vm; 37 | }; 38 | 39 | struct TabConf { 40 | std::string tab_name; 41 | std::string tbs_name; 42 | TabConf(const std::string& tab, const std::string& tbs = "") 43 | : tab_name(tab), tbs_name(tbs) {} 44 | }; 45 | extern StreamConf* streamconf; 46 | 47 | inline MetadataManager& getMetadata() { 48 | static MetadataManager metadata(streamconf->getString("srcConn").c_str()); 49 | return metadata; 50 | } 51 | 52 | class GlobalStream { 53 | 54 | public: 55 | static GlobalStream& getGlobalStream() { 56 | static GlobalStream global_stats; 57 | return global_stats; 58 | } 59 | uint32_t getAppliedSeq() const { return applied_seq_.load(); } 60 | void setAppliedSeq(uint32_t seq) { applied_seq_ = seq; } 61 | 62 | private: 63 | GlobalStream() : applied_seq_(0) {} 64 | std::atomic_uint applied_seq_; 65 | }; 66 | 67 | inline List& getRecordBufList() { 68 | static List record_buf_list; 69 | return record_buf_list; 70 | } 71 | 72 | inline StreamConf& getStreamConf() { return *streamconf; } 73 | void initStream(int ac, char** av); 74 | void streamMonitor(); 75 | void startStream(uint32_t seq, const TimePoint& tp); 76 | void startMining(uint32_t seq, const TimePoint& tp); 77 | void applyRecordBuf(); 78 | inline std::vector& getGlobalThreads() { 79 | static std::vector thrs; 80 | return thrs; 81 | } 82 | 83 | inline LogManager& getLogManager() { 84 | static LogManager log_manager(streamconf->getString("srcConn").c_str()); 85 | return log_manager; 86 | } 87 | } 88 | #endif /* ----- #ifndef REDO_PARSE_INC ----- */ 89 | -------------------------------------------------------------------------------- /src/redo_parse/stream_error.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | namespace databus { 5 | const std::map errors{{1, "error1"}, {2, "eror2"}}; 6 | class ParseException { 7 | public: 8 | ParseException(const std::string& msg) : msg_(msg) {} 9 | std::string& msg() { return msg_; } 10 | 11 | private: 12 | std::string msg_; 13 | }; 14 | } 15 | -------------------------------------------------------------------------------- /src/redo_parse/t40.sql: -------------------------------------------------------------------------------- 1 | drop table target; 2 | create table target (b varchar2(4000), d varchar2(4000), e varchar2(4000), f varchar2(4000), g varchar2(4000), h varchar2(4000), a number(38,10) , c date, primary key(a, b)); 3 | alter system switch logfile; 4 | 5 | -- Big Insert, PK located in 2 different blocks 6 | insert into target values(lpad('a', 4000, 'a'), lpad('b', 4000, 'b'), lpad('c', 4000, 'c'), lpad('d', 4000, 'd'), lpad('g', 4000, 'g'), lpad('h', 4000, 'h'), -1, sysdate); 7 | commit; 8 | 9 | --- Big Update with touching 1 primary key 10 | update target set b='b', d='d', e='c' where a = -1; 11 | commit; 12 | 13 | --- Big Update with touching 2 primary key 14 | update target set b='b', d='d', e='c', a=1 where a = -1; 15 | commit; 16 | 17 | --- Big delete 18 | delete from target where a=1; 19 | commit; 20 | 21 | -- for Row megiration 22 | -- insert into target values('a', 4000, 'a'), lpad('b', 4000, 'b'), lpad('c', 4000, 'c'), lpad('d', 4000, 'd'), lpad('g', 4000, 'g'), lpad('h', 4000, 'h'), -1, sysdate); 23 | insert into target values('a', 'b', 'c', 'd', 'g', 'h', -1, sysdate); 24 | commit; 25 | 26 | -- Update Row megiration touch 1 PK. We need to parse 11.16 and 11.6 correctly for this case 27 | update target set b=lpad('b', 4000, 'b'), d=lpad('d', 4000, 'd'), f=lpad('f', 4000, 'f') where a = -1; 28 | commit; 29 | 30 | alter system switch logfile; 31 | -------------------------------------------------------------------------------- /src/redo_parse/table.conf: -------------------------------------------------------------------------------- 1 | andy.target#SYSAUX 2 | andy.t1106#SYSAUX 3 | andy.test 4 | -------------------------------------------------------------------------------- /src/redo_parse/tconvert.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "logical_elems.h" 9 | #include "metadata.h" 10 | #include "stream.h" 11 | #include "util/logger.h" 12 | 13 | namespace databus { 14 | // using namespace oracle::occi; 15 | /* 16 | static Number numberAsStr(const char* input, uint32_t len) { 17 | Number num(0); 18 | Number base(100); 19 | Uchar highest = (Uchar)input[0]; 20 | if (highest == 0x80) return num; 21 | if (highest > 192) { // positive 22 | for (int i = 1; i < len; ++i) { 23 | num += Number(((Uchar)input[i] - 1)) * 24 | (base.power((Uchar)input[0] - 192 - i)); 25 | } 26 | } else { // negative 27 | for (int i = 1; i < len - 1; ++i) { 28 | num += Number(((Uchar)input[i] - 101)) * 29 | (base.power(63 - (Uchar)input[0] - i)); 30 | } 31 | } 32 | return num; 33 | } 34 | */ 35 | 36 | static std::string numberAsStr(const char* input, uint32_t len) { 37 | unsigned char sign_bit = *input; // 193, 64 ... 38 | if (len == 1) { 39 | assert(sign_bit == 128); 40 | return std::string("0"); 41 | } 42 | 43 | std::stringstream ss; 44 | unsigned short n; 45 | bool floated = false; 46 | 47 | if (sign_bit > 128) { 48 | int int_bit = sign_bit - 192; 49 | if (int_bit < 0) { 50 | ss << "."; 51 | for (int i = 0; i < abs(int_bit); ++i) { 52 | ss << "00"; 53 | } 54 | } 55 | for (Uchar i = 1; i < len || i <= int_bit; ++i) { 56 | if (i > int_bit && int_bit > 0 && !floated) { 57 | ss << "."; 58 | floated = true; 59 | } 60 | if (i >= len) { 61 | ss << "00"; 62 | } else { 63 | n = input[i] - 1; 64 | if (n < 10) { 65 | ss << 0; 66 | } 67 | ss << n; 68 | } 69 | } 70 | } else { 71 | ss << "-"; 72 | int int_bit = 63 - sign_bit; 73 | for (int i = 1; i < len - 1 || i < abs(int_bit) + 1; ++i) { 74 | // input[len-1] == 102, useless 75 | if (i > int_bit && !floated) { 76 | ss << "."; 77 | floated = true; 78 | if (int_bit <= 0) { 79 | // -0.00000001 80 | for (int i = 0; i < abs(int_bit); ++i) { 81 | ss << "00"; 82 | } 83 | } 84 | } 85 | if (i >= len - 1) { 86 | // handle -10000 87 | ss << "00"; 88 | } else { 89 | n = 101 - input[i]; 90 | if (n < 10) { 91 | ss << "0"; 92 | } 93 | ss << n; 94 | } 95 | } 96 | } 97 | return ss.str(); 98 | } 99 | 100 | struct OracleDate { 101 | unsigned char century_; 102 | unsigned char decade_; 103 | unsigned char month_; 104 | unsigned char day_; 105 | unsigned char hour_; 106 | unsigned char minute_; 107 | unsigned char second_; 108 | unsigned char nanosecond_[4]; 109 | unsigned char zone_hour_; 110 | unsigned char zone_minute_; 111 | }; 112 | 113 | static const std::string dateToStr(const char* input, uint32_t len) { 114 | if (len == 0) return ""; 115 | struct OracleDate* ora_date = (struct OracleDate*)input; 116 | if (len == 7) { // Only handle date today 117 | int year, month, day; 118 | if (ora_date->century_ < 100) 119 | year = (100 - ora_date->century_) * 100; 120 | else 121 | year = (ora_date->century_ - 100) * 100; 122 | 123 | if (ora_date->decade_ < 100) 124 | year += 100 - ora_date->decade_; 125 | else 126 | year += ora_date->decade_ - 100; 127 | 128 | if (ora_date->decade_ < 100 || ora_date->century_ < 100) year = -year; 129 | 130 | std::stringstream ss; 131 | ss << year << "-" << (int)ora_date->month_ << "-" << (int)ora_date->day_ 132 | << " " << (int)ora_date->hour_ - 1 << ":" << (int)ora_date->minute_ - 1 133 | << ":" << (int)ora_date->second_ - 1; 134 | return ss.str(); 135 | } 136 | return NULL; 137 | } 138 | 139 | std::string epochToTime(uint32_t epoch) { 140 | std::stringstream ss; 141 | int yy, mm, dd, hh, mi, sec; 142 | const uint32_t secs_one_year = 12 * 31 * 24 * 3600; 143 | yy = epoch / secs_one_year + 1988; 144 | mm = (epoch % secs_one_year) / (31 * 24 * 3600) + 1; 145 | dd = ((epoch % secs_one_year) % (31 * 24 * 3600)) / (24 * 3600) + 1; 146 | hh = (((epoch % secs_one_year) % (31 * 24 * 3600)) % (24 * 3600)) / 3600; 147 | mi = ((((epoch % secs_one_year) % (31 * 24 * 3600)) % (24 * 3600)) % 3600) / 148 | 60; 149 | sec = 150 | ((((epoch % secs_one_year) % (31 * 24 * 3600)) % (24 * 3600)) % 3600) % 151 | 60; 152 | ss << yy << "-" << mm << "-" << dd << " " << hh << ":" << mi << ":" << sec; 153 | return ss.str(); 154 | } 155 | 156 | std::string convert(const char* input, std::string& type, Ushort len) { 157 | // TODO: why length is 0 A: NULL ? Confirm: Yes, TODO: in real case, we 158 | // need a NULL here 159 | if (len == 0) return ""; 160 | if (type == "VARCHAR2") { 161 | return std::string(input, len); 162 | } 163 | if (type == "NUMBER") { 164 | return numberAsStr(input, len); 165 | } 166 | if (type == "DATE") { 167 | return dateToStr(input, len); 168 | } 169 | return ""; 170 | } 171 | 172 | void tranDump(XID xid, uint32_t object_id, const char* optype, 173 | std::list undos, std::list redos) { 174 | auto table_def = getMetadata().getTabDefFromId(object_id); 175 | if (table_def == NULL) { 176 | LOG(DEBUG) << "Obj (" << object_id << ") doesn't exist or don't have PK" 177 | << std::endl; 178 | 179 | return; 180 | } 181 | LOG(INFO) << "Transaction ID " << xid << std::endl; 182 | LOG(INFO) << optype << " " << table_def->name << std::endl; 183 | if (strncmp(optype, "insert", strlen("insert")) != 0) { 184 | LOG(INFO) << "Primary Keys:" << std::endl; 185 | for (auto undo : undos) { 186 | for (auto col : undo) { 187 | if (col->len_ > 0 && 188 | table_def->pk.find(col->col_id_ + 1) != table_def->pk.end()) { 189 | LOG(INFO) << "\t" << table_def->col_names[col->col_id_ + 1] 190 | << "----" 191 | << convert(col->content_, 192 | table_def->col_types[col->col_id_ + 1], 193 | col->len_) << std::endl; 194 | } 195 | } 196 | } 197 | } 198 | 199 | if (strncmp(optype, "delete", strlen("delete")) != 0) { 200 | LOG(INFO) << "New data: " << std::endl; 201 | for (auto redo : redos) { 202 | for (auto col : redo) { 203 | LOG(INFO) << table_def->col_names[col->col_id_ + 1] << "----" 204 | << convert(col->content_, 205 | table_def->col_types[col->col_id_ + 1], 206 | col->len_) << std::endl; 207 | } 208 | } 209 | } 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /src/redo_parse/tconvert.h: -------------------------------------------------------------------------------- 1 | #ifndef TCONVERT_INC 2 | #define TCONVERT_INC 3 | #include "logical_elems.h" 4 | namespace databus { 5 | void tranDump(XID xid, uint32_t object_id, const char* optype, 6 | std::list undos, std::list redos); 7 | std::string convert(const char* input, std::string& type, Ushort len); 8 | std::string epochToTime(uint32_t epoch); 9 | } 10 | 11 | #endif /* ----- #ifndef TCONVERT_INC ----- */ 12 | -------------------------------------------------------------------------------- /src/redo_parse/test_case.sql: -------------------------------------------------------------------------------- 1 | --- Purpose: Gather all the test cases for log mining 2 | --- Created: 2014/11/29 3 | conn andy/andy 4 | create tablespace tbs_5blocks datafile '/home/oracle/app/oracle/oradata/momo/tbs01.dbf' size 10m uniform size 40k segment space management manual; 5 | drop table target; 6 | drop table t1106; 7 | 8 | -- setup target table 9 | drop table halv.target; 10 | create table halv.target (xid number, scn number, op varchar2(40), b varchar2(4000), a number(38,10), c date); 11 | drop table halv.t1106; 12 | create table halv.t1106(xid number, scn number, op varchar2(40), id int); 13 | 14 | whenever sqlerror exit 15 | set echo on 16 | create table target (b varchar2(4000), d varchar2(4000), e varchar2(4000), f varchar2(4000), g varchar2(4000), h varchar2(4000), a number(38,10) , c date, primary key(a, b, c)); 17 | 18 | create table t1106(id int primary key, c1 varchar2(2000), c2 varchar2(2000), c3 varchar2(2000), c4 varchar2(2000) ) tablespace tbs_5blocks; 19 | truncate table t1106; 20 | begin 21 | for i in 1..63 loop 22 | insert into t1106 values(i,rpad('A',100,'A'),rpad('B',100,'B'),rpad('C',100,'C'),rpad('D',100,'D')); 23 | end loop; 24 | commit; 25 | end; 26 | / 27 | 28 | alter system switch logfile; 29 | 30 | -- Big Insert, PK located in 2 different blocks 31 | insert into target values(lpad('a', 4000, 'a'), lpad('b', 4000, 'b'), lpad('c', 4000, 'c'), lpad('d', 4000, 'd'), lpad('g', 4000, 'g'), lpad('h', 4000, 'h'), -1, sysdate); 32 | commit; 33 | 34 | --- Big Update with touching 1 primary key 35 | update target set b='b', d='d', e='c' where a = -1; 36 | commit; 37 | 38 | --- Big Update with touching 2 primary key 39 | update target set b='b', d='d', e='c', a=1 where a = -1; 40 | commit; 41 | 42 | --- Big delete 43 | delete from target where a=1; 44 | commit; 45 | 46 | -- for Row megiration 47 | -- insert into target values('a', 4000, 'a'), lpad('b', 4000, 'b'), lpad('c', 4000, 'c'), lpad('d', 4000, 'd'), lpad('g', 4000, 'g'), lpad('h', 4000, 'h'), -1, sysdate); 48 | insert into target values('a', 'b', 'c', 'd', 'g', 'h', -1, sysdate); 49 | commit; 50 | 51 | -- Update Row megiration touch 1 PK. We need to parse 11.16 and 11.6 correctly for this case 52 | update target set b=lpad('b', 4000, 'b'), d=lpad('d', 4000, 'd'), f=lpad('f', 4000, 'f') where a = -1; 53 | 54 | -- 11.6 + 11.3 55 | -- Row Megiration 56 | update t1106 set c1=rpad('A', 500,'A'), c2=rpad('B', 500,'B'), c3=rpad('C', 500,'C') where id=20; 57 | commit; 58 | 59 | -- Make room for row 20 60 | delete t1106 where id between 18 and 19; 61 | delete t1106 where id between 21 and 34; 62 | commit; 63 | 64 | -- Let row 20 megirate back 65 | update t1106 set c1=rpad('A', 1000,'A'), c2=rpad('B', 2000,'B'), c3=rpad('C', 2000,'C'), id=-100 where id=20; 66 | commit; 67 | 68 | --- Data Tyep Test 69 | insert into target(a,b,c ) values(0.0001, 'abdef', sysdate); 70 | insert into target(a,b,c ) values(-0.0001, 'abdef', sysdate); 71 | insert into target(a,b,c ) values(-1000, 'abdef', sysdate); 72 | -- for https://jirap.corp.ebay.com/browse/DBISTREA-54 73 | alter system switch logfile; 74 | insert into target(a,b,c ) values(-1000.001, 'abdef', sysdate); 75 | update target set b = 'A' , c= sysdate where a= 0.0001; 76 | commit; 77 | 78 | alter system switch logfile; 79 | -------------------------------------------------------------------------------- /src/redo_parse/trans.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "trans.h" 9 | #include "opcode.h" 10 | #include "opcode_ops.h" 11 | #include "metadata.h" 12 | #include "tconvert.h" 13 | #include "util/logger.h" 14 | #include "util/dassert.h" 15 | #include "stream.h" 16 | #include "applier.h" 17 | 18 | namespace databus { 19 | bool Transaction::operator<(const Transaction& other) const { 20 | return commit_scn_ < other.commit_scn_; 21 | } 22 | 23 | DBAMap Transaction::dba_map_; 24 | XIDMap Transaction::xid_map_; 25 | std::map> Transaction::commit_trans_; 26 | 27 | std::mutex Transaction::restart_mutex_; 28 | std::mutex Transaction::commit_mutex_; 29 | uint32_t Transaction::restart_epoch_; 30 | uint32_t Transaction::last_commit_epoch_; 31 | SCN Transaction::last_commit_scn_(-1); 32 | SCN Transaction::restart_scn_(-1); 33 | 34 | std::map Transaction::start_scn_q_; 35 | 36 | XIDMap::iterator buildTransaction(XIDMap::iterator it) { 37 | if (it->second->still_pending()) { 38 | Transaction::start_scn_q_[it->second->start_scn_] = 39 | it->second->start_epoch_; 40 | return Transaction::xid_map_.end(); 41 | } 42 | if (cflag.find(it->second->cflag_) == cflag.end()) { 43 | LOG(ERROR) << "Unexpected cflag " << (int)it->second->cflag_ << " scn " 44 | << it->second->commit_scn_.toStr(); 45 | std::exit(-22); 46 | } 47 | if (it->second->has_rollback()) { 48 | Transaction::eraseStartScn(it->second->start_scn_); 49 | return Transaction::xid_map_.erase(it); 50 | } else { 51 | if (it->second->commit_scn_ < 52 | Transaction::getLastCommitTimePoint().scn_) { 53 | return Transaction::xid_map_.erase(it); 54 | } 55 | Transaction::start_scn_q_[it->second->start_scn_] = 56 | it->second->start_epoch_; 57 | it->second->tidyChanges(); 58 | Transaction::commit_trans_[it->second->commit_scn_] = 59 | Transaction::xid_map_[it->second->xid_]; 60 | return Transaction::xid_map_.erase(it); 61 | } 62 | } 63 | 64 | bool Transaction::lastCompleted() const { 65 | if (changes_.empty()) return true; 66 | auto it = changes_.end(); 67 | --it; 68 | return (*it)->completed(); 69 | } 70 | 71 | void Transaction::merge(RowChangePtr r) { 72 | if (changes_.empty()) { 73 | changes_.insert(r); 74 | return; 75 | } 76 | if (lastCompleted() && r->op_ != opcode::kRowChain) { 77 | // 11.2 may be completed already, but we still need merge this rowchain to 78 | // it if its cc != 0 79 | changes_.insert(r); 80 | return; 81 | } 82 | if (r->iflag_ == 0x2c) { 83 | if (!lastCompleted()) { 84 | LOG(ERROR) << "Error"; 85 | std::exit(20); 86 | } else { 87 | changes_.insert(r); 88 | return; 89 | } 90 | } 91 | 92 | auto it = changes_.end(); 93 | --it; 94 | std::stringstream ss; 95 | if (r->object_id_ != (*it)->object_id_) { 96 | ss << "Object ID mismatched Previous Object_id " << (*it)->object_id_ 97 | << " Current Object_id " << r->object_id_; 98 | } 99 | 100 | // some pre-check before merge 101 | if ((*it)->op_ == opcode::kInsert) { 102 | if (r->op_ != opcode::kInsert && r->op_ != opcode::kRowChain) { 103 | ss << " opcode miss match, No " << (*it)->op_ 104 | << " can be followed by a " << r->op_; 105 | } 106 | } else if ((*it)->op_ == opcode::kDelete) { 107 | if (r->op_ != opcode::kDelete && r->op_ != opcode::kRowChain) { 108 | ss << " opcode miss match, No " << (*it)->op_ 109 | << " can be followed by a " << r->op_; 110 | } 111 | } else if ((*it)->op_ == opcode::kUpdate) { 112 | if (r->op_ != opcode::kUpdate && r->op_ != opcode::kRowChain && 113 | r->op_ != opcode::kLmn) { 114 | ss << " opcode miss match, No " << (*it)->op_ 115 | << " can be followed by a " << r->op_; 116 | } 117 | } else if ((*it)->op_ == opcode::kRowChain) { 118 | if (r->op_ != opcode::kDelete) { 119 | ss << " opcode miss match, No " << (*it)->op_ 120 | << " can be followed by a " << r->op_; 121 | } 122 | } 123 | 124 | if (!ss.str().empty()) { 125 | ss << " Pevious SCN " << (*it)->scn_.toStr() << " Current SCN " 126 | << r->scn_.toStr(); 127 | LOG(ERROR) << ss.str(); 128 | return; 129 | } 130 | 131 | auto new_pk_cnt = (*it)->new_pk_.size(); 132 | auto old_pk_cnt = (*it)->old_pk_.size(); 133 | (*it)->new_pk_.insert(r->new_pk_.begin(), r->new_pk_.end()); 134 | (*it)->old_pk_.insert(r->old_pk_.begin(), r->old_pk_.end()); 135 | if (new_pk_cnt + r->new_pk_.size() != (*it)->new_pk_.size() || 136 | old_pk_cnt + r->old_pk_.size() != (*it)->old_pk_.size()) { 137 | ss << "Found duplicated PK in the 2 changes"; 138 | } 139 | 140 | if (!ss.str().empty()) { 141 | ss << " Pevious SCN " << (*it)->scn_.toStr() << " Current SCN " 142 | << r->scn_.toStr(); 143 | LOG(ERROR) << ss.str(); 144 | return; 145 | } 146 | 147 | if ((*it)->op_ == opcode::kInsert && r->op_ == opcode::kRowChain) { 148 | (*it)->op_ = opcode::kUpdate; 149 | } 150 | } 151 | 152 | void Transaction::tidyChanges() { 153 | auto temp_changes = std::move(changes_); 154 | for (auto& r : temp_changes) { 155 | switch (r->op_) { 156 | case opcode::kMultiInsert: 157 | changes_.insert(r); 158 | break; 159 | default: 160 | merge(r); 161 | break; 162 | } 163 | } 164 | } 165 | 166 | std::string Transaction::toString() const { 167 | std::stringstream ss; 168 | ss << std::endl << "XID " << std::hex << xid_ 169 | << " start_scn : " << start_scn_.toStr() 170 | << " commit_scn: " << commit_scn_.toStr() 171 | << " commited : " << (int)cflag_ << std::endl; 172 | for (auto& rc : changes_) { 173 | ss << rc->pkToString() << std::endl; 174 | } 175 | return ss.str(); 176 | } 177 | 178 | std::string TimePoint::toString() const { 179 | std::stringstream ss; 180 | ss << " SCN " << scn_.toStr() << " time " << epochToTime(epoch_); 181 | return ss.str(); 182 | } 183 | 184 | static std::string colAsStr(ColumnChangePtr col, TabDefPtr tab_def, 185 | char seperator = ':') { 186 | std::stringstream ss; 187 | ss << tab_def->col_names[col->col_id_ + 1] << seperator 188 | << convert(col->content_, tab_def->col_types[col->col_id_ + 1], 189 | col->len_); 190 | return ss.str(); 191 | } 192 | 193 | std::string colAsStr2(ColumnChangePtr col, TabDefPtr tab_def) { 194 | std::stringstream ss; 195 | ss << convert(col->content_, tab_def->col_types[col->col_id_ + 1], 196 | col->len_); 197 | return ss.str(); 198 | } 199 | 200 | Ushort findPk(std::shared_ptr table_def, const Row& undo, 201 | OrderedPK& pk) { 202 | for (const auto col : undo) { 203 | if (col->col_id_ > 256) { 204 | LOG(WARNING) << "Found column ID " << col->col_id_ 205 | << " probably this is a program bug"; 206 | } 207 | if (col->len_ > 0 && 208 | table_def->pk.find(col->col_id_ + 1) != table_def->pk.end()) { 209 | pk.insert(col); 210 | } 211 | } 212 | return pk.size(); 213 | } 214 | 215 | RowChange::RowChange() 216 | : scn_(), 217 | object_id_(0), 218 | op_(), 219 | uflag_(0), 220 | iflag_(0), 221 | start_col_(0), 222 | cc_(0), 223 | old_pk_{}, 224 | new_pk_{}, 225 | epoch_(0) {} 226 | 227 | std::string RowChange::toString() const { 228 | std::stringstream ss; 229 | ss << " Object_id " << object_id_ << " Op " << op_ << " Start_col " 230 | << start_col_ << " Offset " << scn_.noffset_; 231 | for (auto c : old_pk_) { 232 | ss << " old_pk_c_" << c->col_id_; 233 | } 234 | 235 | for (auto c : new_pk_) { 236 | ss << " new_pk_c_" << c->col_id_; 237 | } 238 | return ss.str(); 239 | } 240 | 241 | bool RowChange::completed() const { 242 | auto tab_def = getMetadata().getTabDefFromId(object_id_); 243 | switch (op_) { 244 | case opcode::kInsert: 245 | case opcode::kMultiInsert: 246 | return new_pk_.size() == tab_def->pk.size(); 247 | case opcode::kUpdate: 248 | case opcode::kDelete: 249 | case opcode::kLmn: 250 | case opcode::kRowChain: 251 | return old_pk_.size() == tab_def->pk.size(); 252 | default: 253 | return false; 254 | } 255 | } 256 | std::string RowChange::pkToString() const { 257 | std::stringstream ss; 258 | TabDefPtr tab_def = getMetadata().getTabDefFromId(object_id_); 259 | ss << tab_def->owner << "." << tab_def->name << " " << getOpStr(op_); 260 | ss << " New PK "; 261 | // for update, delete, row migration, we store pk into pk_ already 262 | for (ColumnChangePtr c : new_pk_) { 263 | ss << colAsStr(c, tab_def); 264 | } 265 | ss << " Old PK "; 266 | for (ColumnChangePtr c : old_pk_) { 267 | ss << colAsStr(c, tab_def); 268 | } 269 | return ss.str(); 270 | } 271 | 272 | std::vector RowChange::getPk() { 273 | // the pk string is order by col_no 274 | TabDefPtr tab_def = getMetadata().getTabDefFromId(object_id_); 275 | // this line is super urgly and error prone, will fix it someday 276 | std::vector pks(new_pk_.size() + old_pk_.size() + 277 | prefix_cols.size() - 1); 278 | int n = 0; 279 | pks[n++] = getOpStr(op_); 280 | pks[n++] = scn_.toString(); 281 | for (ColumnChangePtr c : old_pk_) { 282 | pks[n++] = std::move(colAsStr2(c, tab_def)); 283 | } 284 | for (ColumnChangePtr c : new_pk_) { 285 | pks[n++] = std::move(colAsStr2(c, tab_def)); 286 | } 287 | return pks; 288 | } 289 | 290 | void addToTransaction(RecordBufPtr record) { 291 | DBA dba = 0; 292 | XID xid = 0; 293 | std::list undo, redo; 294 | SCN trans_start_scn; 295 | RowChangePtr rcp(new RowChange()); 296 | rcp->scn_ = record->scn(); 297 | rcp->epoch_ = record->epoch(); 298 | for (auto change : record->change_vectors) { 299 | switch (change->opCode()) { 300 | case opcode::kBeginTrans: 301 | dba = change->dba(); 302 | { 303 | if (((OpCode0502*)change->part(1))->sqn_ > 0) { 304 | // sqn_ == 0 is not a start of transaction 305 | trans_start_scn = rcp->scn_; 306 | } 307 | } 308 | break; 309 | case opcode::kUndo: 310 | xid = Ops0501::getXID(change); 311 | if (xid == 0) return; 312 | if (Transaction::xid_map_.find(xid) == Transaction::xid_map_.end()) { 313 | Transaction::xid_map_[xid] = TransactionPtr(new Transaction()); 314 | Transaction::xid_map_[xid]->xid_ = xid; 315 | } 316 | rcp->object_id_ = Ops0501::getObjId(change); 317 | if (dba > 0) { 318 | Transaction::dba_map_[dba] = 319 | ((OpCode0501*)change->part(1))->xid_high_; 320 | dba = 0; 321 | if (!trans_start_scn.empty()) { 322 | Transaction::xid_map_[xid]->start_scn_ = rcp->scn_; 323 | Transaction::xid_map_[xid]->xid_ = xid; 324 | Transaction::xid_map_[xid]->start_epoch_ = rcp->epoch_; 325 | } 326 | } 327 | if ((record->op() & 0xff00) != 0xb00) return; 328 | { 329 | if (getMetadata().getTabDefFromId(rcp->object_id_, false) == NULL) { 330 | // we don't care about this object id for version 0.1 331 | return; 332 | } 333 | undo = Ops0501::makeUpUndo(change, rcp); 334 | // opsup only set when irp->xtype_ & 0x20 335 | // LOG(INFO) << "Sup start_col_offset " << record->offset() << ":" 336 | // << opsup->start_column_ << ":" << opsup->start_column2_ 337 | // << std::endl; 338 | } 339 | break; 340 | case opcode::kUpdate: 341 | case opcode::kInsert: 342 | case opcode::kRowChain: 343 | case opcode::kMultiInsert: 344 | rcp->op_ = change->opCode(); 345 | redo = OpsDML::makeUpRedoCols(change, rcp); 346 | break; 347 | case opcode::kDelete: 348 | case opcode::kLmn: 349 | rcp->op_ = change->opCode(); 350 | break; 351 | case opcode::kCommit: 352 | dba = change->dba(); 353 | { 354 | auto it = Transaction::dba_map_.find(dba); 355 | XID xid_high = (change->block_class_ - 15) / 2; 356 | if (it != Transaction::dba_map_.end() && it->second != xid_high) { 357 | LOG(ERROR) << "CLS->XID_HIGH Error scn" << rcp->scn_.toStr(); 358 | } 359 | OpCode0504_ucm* ucm = (OpCode0504_ucm*)(change->part(1)); 360 | XID ixid = ((xid_high) << (sizeof(Ushort) + sizeof(uint32_t)) * 8) | 361 | (((XID)ucm->slt_) << sizeof(uint32_t) * 8) | ucm->sqn_; 362 | auto xidit = Transaction::xid_map_.find(ixid); 363 | if (xidit == Transaction::xid_map_.end()) { 364 | Transaction::xid_map_[ixid] = 365 | std::shared_ptr(new Transaction()); 366 | Transaction::xid_map_[ixid]->xid_ = ixid; 367 | } 368 | Transaction::xid_map_[ixid]->commit_scn_ = rcp->scn_; 369 | Transaction::xid_map_[ixid]->cflag_ = ucm->flg_; 370 | Transaction::xid_map_[ixid]->end_epoch_ = rcp->epoch_; 371 | } 372 | break; 373 | default: 374 | rcp->op_ = change->opCode(); 375 | LOG(WARNING) << "Found Un-popular OpCode in our table " 376 | << rcp->toString(); 377 | return; 378 | } // end switch 379 | } 380 | if (rcp->op_ && xid != 0) makeTranRecord(xid, rcp, undo, redo); 381 | } 382 | 383 | void makeTranRecord(XID xid, RowChangePtr rcp, std::list& undos, 384 | std::list& redos) { 385 | XIDMap& xidmap = Transaction::xid_map_; 386 | auto transit = xidmap.find(xid); 387 | if (transit == xidmap.end()) { 388 | LOG(WARNING) << "XID " << xid 389 | << " info was missed when I want to add a change to it" 390 | << std::endl; 391 | return; 392 | } 393 | auto table_def = getMetadata().getTabDefFromId(rcp->object_id_); 394 | if (table_def == NULL) { 395 | LOG(DEBUG) << "Can't get table definition for object_id " 396 | << rcp->object_id_ << " ignore this change " << std::endl; 397 | return; 398 | } 399 | LOG(DEBUG) << getOpStr(rcp->op_) << " " << rcp->scn_.noffset_ 400 | << " start_col " << rcp->start_col_ << " Offset " 401 | << rcp->scn_.noffset_ << std::endl; 402 | 403 | switch (rcp->op_) { 404 | case opcode::kDelete: 405 | case opcode::kLmn: { 406 | for (auto row : undos) { 407 | findPk(table_def, row, rcp->old_pk_); 408 | // even the old_pk_ is null, we still need it to mark a 11.6 completed 409 | transit->second->changes_.insert(std::move(rcp)); 410 | } 411 | } break; 412 | case opcode::kMultiInsert: { 413 | for (auto row : redos) { 414 | findPk(table_def, row, rcp->new_pk_); 415 | if (!rcp->old_pk_.empty()) 416 | transit->second->changes_.insert(std::move(rcp)); 417 | } 418 | } break; 419 | 420 | case opcode::kUpdate: 421 | case opcode::kInsert: 422 | case opcode::kRowChain: { 423 | // this is no mulitUpdate, so there is 1 elems in undo and redo at 424 | // most 425 | OrderedPK pk; 426 | if (!undos.empty()) { 427 | auto undo_iter = undos.begin(); 428 | findPk(table_def, *undo_iter, rcp->old_pk_); 429 | } 430 | if (!redos.empty()) { 431 | auto redo_iter = redos.begin(); 432 | findPk(table_def, *redo_iter, rcp->new_pk_); 433 | } 434 | if (!rcp->old_pk_.empty() || !rcp->new_pk_.empty()) { 435 | transit->second->changes_.insert(std::move(rcp)); 436 | } 437 | } break; 438 | default: 439 | LOG(ERROR) << "Unknown Op " << (int)rcp->op_ << std::endl; 440 | break; 441 | } 442 | } 443 | 444 | bool verifyTrans(TransactionPtr trans_ptr) { 445 | bool dup = false; 446 | #ifdef __STREAM_DEBUG__ 447 | // we use scn to order changes, so we don't want 2 exactly same 448 | // scn in a given transaction 449 | std::map scn_count; 450 | for (auto i : trans_ptr->changes_) { 451 | scn_count[i->scn_]++; 452 | } 453 | for (auto i : scn_count) { 454 | if (i.second > 1) { 455 | dup = true; 456 | break; 457 | } 458 | } 459 | 460 | if (dup) { 461 | LOG(WARNING) << "FOUND duplicated SCN in this transaction" << std::endl; 462 | LOG(WARNING) << trans_ptr->toString() << std::endl; 463 | } 464 | #endif 465 | return ~dup; 466 | } 467 | } 468 | -------------------------------------------------------------------------------- /src/redo_parse/trans.h: -------------------------------------------------------------------------------- 1 | #ifndef TRANS_INC 2 | #define TRANS_INC 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include "easylogging++.h" 13 | #include "logical_elems.h" 14 | #include "opcode.h" 15 | #include "util/dtypes.h" 16 | #include "logical_elems.h" 17 | 18 | namespace databus { 19 | 20 | struct ColumnLess { 21 | bool operator()(const std::shared_ptr& l, 22 | const std::shared_ptr& r) { 23 | return l->col_id_ < r->col_id_; 24 | } 25 | }; 26 | 27 | const std::set cflag{0x00, 0x10, 0x02, 0x12, 0x04, 0x14}; 28 | 29 | struct RowChange; 30 | class TabDef; 31 | typedef std::shared_ptr RowChangePtr; 32 | typedef std::set OrderedPK; 33 | 34 | struct RowChange { 35 | RowChange(); 36 | RowChange(SCN& scn, uint32_t obj_id, Ushort op, Ushort uflag, Ushort iflag, 37 | Row& undo, Row& redo); 38 | bool operator<(const RowChange& other) const { return scn_ < other.scn_; } 39 | std::string toString() const; 40 | std::string pkToString() const; 41 | std::vector getPk(); 42 | bool completed() const; 43 | 44 | SCN scn_; 45 | uint32_t object_id_; 46 | Ushort op_; 47 | Ushort start_col_; 48 | uint32_t epoch_; 49 | // we only care about cc for 11.6 50 | // see https://jirap.corp.ebay.com/browse/DBISTREA-37 51 | Ushort cc_; 52 | Ushort uflag_; 53 | Uchar iflag_; 54 | OrderedPK old_pk_; 55 | OrderedPK new_pk_; 56 | }; 57 | 58 | struct Less { 59 | bool operator()(const std::shared_ptr& l, 60 | const std::shared_ptr& r) { 61 | return l->scn_ < r->scn_; 62 | } 63 | }; 64 | 65 | struct Transaction; 66 | typedef std::map> XIDMap; 67 | typedef std::map DBAMap; 68 | 69 | struct TimePoint { 70 | TimePoint() : epoch_(0), scn_() {} 71 | TimePoint(const SCN& scn, const uint32_t epoch) 72 | : epoch_(epoch), scn_(scn) {} 73 | uint32_t epoch_; 74 | SCN scn_; 75 | std::string toString() const; 76 | bool empty() const { return epoch_ == 0 and scn_.empty(); } 77 | }; 78 | 79 | struct Transaction { 80 | public: 81 | Transaction() 82 | : cflag_(-1), 83 | xid_(0), 84 | start_scn_(), 85 | commit_scn_(), 86 | end_epoch_(0), 87 | start_epoch_(0) {} 88 | XID xid_; 89 | SCN start_scn_; 90 | SCN commit_scn_; 91 | char cflag_; 92 | uint32_t start_epoch_; 93 | uint32_t end_epoch_; 94 | 95 | // orginize changes, for row-chains, row migration 96 | std::set changes_; 97 | std::string toString() const; 98 | 99 | bool operator<(const Transaction& other) const; 100 | bool has_rollback() const { return cflag_ & 4; } 101 | bool has_commited() const { return !(still_pending() || has_rollback()); } 102 | bool still_pending() const { return cflag_ == -1; } 103 | bool empty() const { return changes_.empty(); } 104 | void tidyChanges(); 105 | 106 | static XIDMap xid_map_; 107 | static std::map> commit_trans_; 108 | 109 | private: 110 | void merge(RowChangePtr r); 111 | bool lastCompleted() const; 112 | 113 | public: 114 | static DBAMap dba_map_; 115 | static std::map start_scn_q_; 116 | 117 | public: 118 | static TimePoint getLastCommitTimePoint() { 119 | std::lock_guard lk(commit_mutex_); 120 | return TimePoint(last_commit_scn_, last_commit_epoch_); 121 | } 122 | 123 | static TimePoint getRestartTimePoint() { 124 | std::lock_guard lk(commit_mutex_); 125 | return TimePoint(restart_scn_, restart_epoch_); 126 | } 127 | 128 | static void setRestartTimePoint(const SCN& scn, uint32_t epoch) { 129 | std::lock_guard lk(restart_mutex_); 130 | restart_scn_ = scn; 131 | restart_epoch_ = epoch; 132 | } 133 | 134 | static void setLastCommitTimePoint(const SCN& scn, uint32_t epoch) { 135 | std::lock_guard lk(commit_mutex_); 136 | last_commit_scn_ = scn; 137 | last_commit_epoch_ = epoch; 138 | } 139 | 140 | static void eraseStartScn(const SCN& scn) { 141 | start_scn_q_.erase(scn); 142 | if (scn == restart_scn_) { 143 | if (!start_scn_q_.empty()) { 144 | auto it = start_scn_q_.begin(); 145 | setRestartTimePoint(it->first, it->second); 146 | } 147 | } 148 | } 149 | 150 | static void setTimePointWhenCommit( 151 | const std::shared_ptr tran) { 152 | if (last_commit_scn_ < tran->commit_scn_) { 153 | setLastCommitTimePoint(tran->commit_scn_, tran->end_epoch_); 154 | } 155 | eraseStartScn(tran->start_scn_); 156 | } 157 | 158 | static uint32_t removeUncompletedTrans() { 159 | uint32_t n = 0; 160 | auto it = xid_map_.begin(); 161 | while (it != xid_map_.end()) { 162 | if (it->second->start_scn_.empty()) { 163 | it = xid_map_.erase(it); 164 | n += 1; 165 | } else { 166 | ++it; 167 | } 168 | } 169 | return n; 170 | } 171 | 172 | private: 173 | static std::mutex restart_mutex_; 174 | static std::mutex commit_mutex_; 175 | static uint32_t restart_epoch_; 176 | static uint32_t last_commit_epoch_; 177 | static SCN last_commit_scn_; 178 | static SCN restart_scn_; 179 | }; 180 | 181 | XIDMap::iterator buildTransaction(XIDMap::iterator it); 182 | 183 | typedef std::shared_ptr TransactionPtr; 184 | typedef std::map& SCNTranMap; 185 | 186 | void addToTransaction(RecordBufPtr ptr); 187 | void makeTranRecord(XID xid, RowChangePtr rcp, std::list& undo, 188 | std::list& redo); 189 | 190 | bool verifyTrans(TransactionPtr trans_ptr); 191 | Ushort findPk(std::shared_ptr tab_def, const Row& undo, 192 | OrderedPK& pk); 193 | std::string colAsStr2(ColumnChangePtr col, std::shared_ptr tab_def); 194 | } 195 | #endif /* ----- #ifndef TRANS_INC ----- */ 196 | -------------------------------------------------------------------------------- /src/redo_parse/worker_test.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "easylogging++.h" 9 | #include "util/container.h" 10 | #include "util/utils.h" 11 | #include "stream.h" 12 | #include "metadata.h" 13 | #include "tconvert.h" 14 | #include "redofile.h" 15 | #include "opcode.h" 16 | #include "util/logger.h" 17 | #include "opcode_ops.h" 18 | #include "trans.h" 19 | #include "applier.h" 20 | #include "otlv4.h" 21 | #include "monitor.h" 22 | 23 | INITIALIZE_EASYLOGGINGPP 24 | namespace databus { 25 | 26 | void shutdown(int signum) { 27 | LOG(INFO) << std::endl << "Shutdowning datastream now.. "; 28 | std::exit(0); 29 | } 30 | 31 | int main(int ac, char** av) { 32 | signal(SIGINT, shutdown); 33 | putenv(const_cast("NLS_LANG=.AL32UTF8")); 34 | otl_connect::otl_initialize(1); 35 | uint32_t startSeq; 36 | ApplyStats stats; 37 | try { 38 | initStream(ac, av); 39 | stats = ApplierHelper::getApplierHelper().getApplyStats(); 40 | LOG(INFO) << "Last commit Timepoint " << stats.commit_tp_.toString(); 41 | LOG(INFO) << "Restart Timepoint " << stats.restart_tp_.toString(); 42 | startSeq = getLogManager().getSeqFromScn( 43 | std::to_string(stats.restart_tp_.scn_.toNum()).c_str()); 44 | if (startSeq == 0) { 45 | LOG(ERROR) << "restart scn is " << stats.restart_tp_.scn_.toStr() 46 | << " Can't find out an archived log contains that scn"; 47 | return -10; 48 | } 49 | GlobalStream::getGlobalStream().setAppliedSeq(startSeq); 50 | Transaction::setRestartTimePoint(stats.restart_tp_.scn_, 51 | stats.restart_tp_.epoch_); 52 | Transaction::setLastCommitTimePoint(stats.commit_tp_.scn_, 53 | stats.commit_tp_.epoch_); 54 | } catch (otl_exception& p) { 55 | LOG(ERROR) << p.msg; 56 | LOG(ERROR) << p.stm_text; 57 | LOG(ERROR) << p.var_info; 58 | throw p; 59 | } 60 | Monitor m; 61 | util::guarded_thread t{std::ref(m)}; 62 | startStream(startSeq, stats.restart_tp_); 63 | return 0; 64 | } 65 | } 66 | 67 | int main(int argc, char** argv) { return databus::main(argc, argv); } 68 | -------------------------------------------------------------------------------- /src/redo_parse/workers.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "workers.h" 9 | #include "redofile.h" 10 | #include "util/container.h" 11 | #include "util/dassert.h" 12 | #include "physical_elems.h" 13 | #include "logical_elems.h" 14 | #include "opcode_ops.h" 15 | 16 | namespace databus { 17 | using util::dassert; 18 | 19 | Reader::Reader(SList* record_pos_buf) 20 | : cur_redo_file_(NULL), 21 | record_pos_buf_(record_pos_buf), 22 | curr_done(true) {} 23 | 24 | void Reader::feed(const char* filename) { 25 | file_name_queue_.push_back(filename); 26 | } 27 | 28 | void Reader::parseRedo(const RedoFile* redo_file) { 29 | const char* record = redo_file->firstRecord(); 30 | do { 31 | record_pos_buf_->push_back(record); 32 | } while ((record = redo_file->nextRecord(record)) != NULL); 33 | 34 | record_pos_buf_->push_back(NULL); 35 | } 36 | 37 | void Reader::resetCurLogfile(const char* filename) { 38 | RedoFile* redo_file = new RedoFile(filename); 39 | if (cur_redo_file_ != NULL) delete cur_redo_file_; 40 | cur_redo_file_ = redo_file; 41 | curr_done = false; 42 | } 43 | 44 | void Reader::run() { 45 | while (true) { 46 | const char* file_name = file_name_queue_.pop_front(); 47 | if (curr_done) { 48 | resetCurLogfile(file_name); 49 | parseRedo(cur_redo_file_); 50 | } else { 51 | std::this_thread::sleep_for(std::chrono::seconds(1)); 52 | } 53 | } 54 | } 55 | 56 | void Reader::srun() { 57 | const char* file_name = file_name_queue_.pop_front(); 58 | resetCurLogfile(file_name); 59 | parseRedo(cur_redo_file_); 60 | } 61 | 62 | void RecordBuilder::realCopyNBytes(const char* from, char* to, int32_t len) { 63 | int this_block_space_left = spaceLeft(from); 64 | if (this_block_space_left >= len) { 65 | memcpy(to, from, len); 66 | return; 67 | } 68 | uint32_t block_data_size = 69 | p_reader_->currentRedoLog()->blockSize() - constants::kBlockHeaderSize; 70 | 71 | memcpy(to, from, this_block_space_left); 72 | len -= this_block_space_left; 73 | to += this_block_space_left; 74 | from = from + this_block_space_left + constants::kBlockHeaderSize; 75 | 76 | while (len > block_data_size) { 77 | memcpy(to, from, block_data_size); 78 | len -= block_data_size; 79 | to += block_data_size; 80 | from = from + block_data_size + constants::kBlockHeaderSize; 81 | } 82 | 83 | memcpy(to, from, len); 84 | } 85 | 86 | void RecordBuilder::run() { 87 | while (true) { 88 | const char* record_pos = NULL; 89 | while ((record_pos = fetchARecordPos()) != NULL) { 90 | RecordBuf* record_buf = makeRecord(record_pos); 91 | if (record_buf != NULL) p_record_buf_list_->push_back(record_buf); 92 | } 93 | p_reader_->curr_done = true; 94 | } 95 | } 96 | 97 | void RecordBuilder::srun() { 98 | const char* record_pos = NULL; 99 | while ((record_pos = fetchARecordPos()) != NULL) { 100 | RecordBuf* record_buf = makeRecord(record_pos); 101 | if (record_buf != NULL) p_record_buf_list_->push_back(record_buf); 102 | } 103 | // only for test, add a NULL at last 104 | p_record_buf_list_->push_back(NULL); 105 | } 106 | 107 | RecordBuf* RecordBuilder::makeRecord(const char* record_start_pos) { 108 | Uchar ora_version = oraVersion(); 109 | uint32_t record_len = immature::recordLength(record_start_pos, ora_version); 110 | 111 | int change_length = 0; 112 | const char* change_pos = NULL; 113 | SCN record_scn; 114 | uint32_t epoch = 0; 115 | 116 | if (ora_version == 9) { 117 | change_length = record_len - constants::kMinRecordLen; 118 | change_pos = record_start_pos + constants::kMinRecordLen; 119 | record_scn = As(record_start_pos)->scn(); 120 | } else { 121 | Uchar vld = immature::recordVld(record_start_pos, ora_version); 122 | 123 | if (immature::isMajor(vld)) { 124 | change_length = record_len - constants::kMinMajRecordLen; 125 | change_pos = record_start_pos + constants::kMinMajRecordLen; 126 | record_scn = As(record_start_pos)->scn(); 127 | epoch = As(record_start_pos)->getEpoch(); 128 | } else if (immature::isMinor(vld)) { 129 | change_length = record_len - constants::kMinRecordLen; 130 | change_pos = record_start_pos + constants::kMinRecordLen; 131 | record_scn = As(record_start_pos)->scn(); 132 | } else { 133 | int ivld = vld; 134 | std::stringstream ss; 135 | ss << "unsupport vld " << ivld; 136 | dassert(ss.str().c_str(), false); 137 | } 138 | } 139 | 140 | if (change_length == 0) return NULL; 141 | if (p_reader_->cur_redo_file_->spaceLeft(change_pos) == 142 | p_reader_->cur_redo_file_->blockSize()) 143 | change_pos += constants::kBlockHeaderSize; 144 | 145 | size_t offset = 146 | record_start_pos - p_reader_->cur_redo_file_->fileStartPos(); 147 | 148 | char* change_buf = new char[change_length]; 149 | realCopyNBytes(change_pos, change_buf, change_length); 150 | 151 | RecordBuf* record_buf = 152 | new RecordBuf(record_scn, change_length, epoch, change_buf, offset); 153 | 154 | bool valid = true; 155 | uint32_t obj_id; 156 | for (auto c : record_buf->change_vectors) { 157 | if (c->opCode() == 0x0501) { 158 | obj_id = Ops0501::getObjId(c); 159 | if (!filter(obj_id)) valid = false; 160 | } 161 | } 162 | if (valid) 163 | return record_buf; 164 | else 165 | return NULL; 166 | } 167 | 168 | TransactionBuilder::TransactionBuilder(RecordList* p_record_buf_list, 169 | TransList* p_transaction_list) 170 | : record_buf_list_(p_record_buf_list), 171 | transaction_list_(p_transaction_list), 172 | handler_([](XID xid, uint32_t objid, const char* op, 173 | std::list& undo, std::list& redo) { return; }) {} 174 | 175 | void dumpRow(Row row) { 176 | if (!row.empty()) std::cout << "New Values:" << std::endl; 177 | for (auto i : row) i->dump(); 178 | } 179 | 180 | void TransactionBuilder::srun() { 181 | 182 | RecordBuf* record_buf; 183 | while ((record_buf = record_buf_list_->pop_front()) != NULL) { 184 | XID xid = 0; 185 | std::list undo, redo; 186 | uint32_t object_id; 187 | uint32_t data_object_id; 188 | const char* optype = NULL; 189 | for (auto i : record_buf->change_vectors) { 190 | switch (i->opCode()) { 191 | case opcode::kUndo: { 192 | xid = Ops0501::getXID(i); 193 | object_id = Ops0501::getObjId(i); 194 | data_object_id = Ops0501::getDataObjId(i); 195 | undo = Ops0501::makeUpUndo(i); 196 | } break; 197 | case opcode::kUpdate: 198 | redo = OpsDML::makeUpRedoCols(i); 199 | optype = "update"; 200 | break; 201 | case opcode::kInsert: 202 | case opcode::kMultiInsert: 203 | redo = OpsDML::makeUpRedoCols(i); 204 | optype = "insert"; 205 | break; 206 | case opcode::kDelete: 207 | optype = "delete"; 208 | break; 209 | } // end switch 210 | } 211 | if (optype != NULL) handler_(xid, object_id, optype, undo, redo); 212 | } 213 | } 214 | } // databus 215 | -------------------------------------------------------------------------------- /src/redo_parse/workers.h: -------------------------------------------------------------------------------- 1 | #ifndef _DATABUS_REDOPARSE_WORKERS_H 2 | #define _DATABUS_REDOPARSE_WORKERS_H 1 3 | #include 4 | #include 5 | #include 6 | #include "util/container.h" 7 | #include "util/dtypes.h" 8 | #include "redofile.h" 9 | 10 | namespace databus { 11 | class RecordBuf; 12 | class Transaction; 13 | 14 | typedef List SList; 15 | typedef List RecordList; 16 | typedef List TransList; 17 | 18 | class Reader { 19 | friend class RecordBuilder; 20 | 21 | public: 22 | Reader(SList* record_pos_buf); 23 | const RedoFile* currentRedoLog() const { return cur_redo_file_; } 24 | void feed(const char* filename); 25 | void run(); 26 | 27 | // only for test 28 | void srun(); 29 | 30 | private: 31 | void resetCurLogfile(const char* filename); 32 | void parseRedo(const RedoFile* redo_file); 33 | 34 | SList file_name_queue_; 35 | SList* record_pos_buf_; 36 | RedoFile* cur_redo_file_; 37 | // set it to true when 38 | // 1. the cur_redo_files parse completed 39 | // AND 40 | // 2. builder have build all the record from it 41 | bool curr_done; 42 | }; 43 | 44 | // multi-threads: 45 | // we have have mulit threads to build_records from `records_start_pos` at the 46 | // same time, most of time costing should be copy data from memory 47 | // TODO: when to release resould for records_buf_list 48 | 49 | class RecordBuilder { 50 | public: 51 | RecordBuilder(Reader* reader, RecordList* record_buf_list) 52 | : p_reader_(reader), p_record_buf_list_(record_buf_list) { 53 | filter = [](uint32_t o) { return true; }; 54 | }; 55 | 56 | template 57 | void setFilter(Filter fn) { 58 | filter = fn; 59 | } 60 | 61 | void run(); 62 | // only for test 63 | void srun(); 64 | 65 | private: 66 | uint32_t spaceLeft(const char* p) const { 67 | return p_reader_->currentRedoLog()->spaceLeft(p); 68 | } 69 | 70 | Uchar oraVersion() const { 71 | return p_reader_->currentRedoLog()->oraVersion(); 72 | } 73 | 74 | const char* fetchARecordPos() { 75 | return p_reader_->record_pos_buf_->pop_front(); 76 | } 77 | 78 | void realCopyNBytes(const char* from, char* to, int32_t len); 79 | RecordBuf* makeRecord(const char* record_start_pos); 80 | 81 | private: 82 | Reader* p_reader_; 83 | RecordList* p_record_buf_list_; 84 | std::function filter; 85 | }; 86 | 87 | class TransactionBuilder { 88 | public: 89 | TransactionBuilder(RecordList* record_buf_list, 90 | TransList* transaction_list); 91 | // only for test 92 | void srun(); 93 | template 94 | void setHandler(Handler handler) { 95 | handler_ = handler; 96 | } 97 | 98 | private: 99 | void parseRecord(const RecordBuf* buf); 100 | // init Transaction instance if needed, the result 101 | RowChange* buildRowChange(const ChangeHeader* change_header); 102 | 103 | private: 104 | std::function&, 105 | std::list&)> handler_; 106 | 107 | RecordList* record_buf_list_; 108 | TransList* transaction_list_; 109 | 110 | std::map fba2xid_; 111 | }; 112 | 113 | // Single Threads: 114 | // TODO: make is multi-threadable 115 | // void BuildTransaction(RecordList* records_buf_list, TransList* trans_list); 116 | 117 | // Single Thread: 118 | // It would be interesting to make apply thread is mulit-threadable, 119 | // single thread now 120 | // void ApplyTransaction(TransList* trans_list, MetadataManager&); 121 | } 122 | 123 | #endif 124 | -------------------------------------------------------------------------------- /src/util/container.h: -------------------------------------------------------------------------------- 1 | #ifndef _DATABUS_UTIL_H 2 | #define _DATABUS_UTIL_H 1 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | namespace databus { 10 | template 11 | class List { 12 | public: 13 | List() {} 14 | void push_back(const T& val) { 15 | std::lock_guard lk(mutex_); 16 | list_.push_back(val); 17 | cv_.notify_one(); 18 | } 19 | 20 | T pop_front() { 21 | std::unique_lock lk(mutex_); 22 | cv_.wait(lk, [this] { return !list_.empty(); }); 23 | T it = list_.front(); 24 | list_.pop_front(); 25 | return it; 26 | } 27 | 28 | bool empty() { 29 | std::lock_guard lk(mutex_); 30 | return list_.empty(); 31 | } 32 | 33 | size_t size() { 34 | std::lock_guard lk(mutex_); 35 | return list_.size(); 36 | } 37 | 38 | private: 39 | std::list list_; 40 | std::mutex mutex_; 41 | std::condition_variable cv_; 42 | }; 43 | 44 | template 45 | void ReportList(List& list, const char* name) { 46 | std::cout << name << list.size() << std::endl; 47 | } 48 | } 49 | 50 | #endif 51 | -------------------------------------------------------------------------------- /src/util/dassert.h: -------------------------------------------------------------------------------- 1 | #ifndef DASSERT_INC 2 | #define DASSERT_INC 3 | #include 4 | 5 | namespace util { 6 | 7 | inline void dassert(const char* err, bool expr, int exitno = -1) { 8 | if (!expr) { 9 | std::cerr << err << std::endl; 10 | std::exit(exitno); 11 | } 12 | } 13 | 14 | inline void strange(const char* message) { 15 | std::cout << "[strange] " << message << std::endl; 16 | } 17 | } 18 | #endif /* ----- #ifndef DASSERT_INC ----- */ 19 | -------------------------------------------------------------------------------- /src/util/dtypes.h: -------------------------------------------------------------------------------- 1 | #ifndef DTYPES_INC 2 | #define DTYPES_INC 3 | #include 4 | 5 | namespace databus { 6 | typedef unsigned char Uchar; 7 | typedef unsigned short Ushort; 8 | typedef unsigned int Uint; 9 | typedef uint64_t XID; 10 | typedef uint32_t DBA; 11 | typedef Ushort USN; 12 | } 13 | 14 | #endif /* ----- #ifndef DTYPES_INC ----- */ 15 | -------------------------------------------------------------------------------- /src/util/logger.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "easylogging++.h" 5 | 6 | namespace databus { 7 | inline std::ostream& trace() { return std::cout << "[TRACE] "; } 8 | 9 | inline std::ostream& debug() { return std::cout << "[DEBUG] "; } 10 | 11 | inline std::ostream& info() { return std::cout << "[INFO] "; } 12 | 13 | inline std::ostream& warn() { return std::cout << "[WARN] "; } 14 | 15 | inline std::ostream& error() { return std::cout << "[ERROR] "; } 16 | } 17 | -------------------------------------------------------------------------------- /src/util/utils.h: -------------------------------------------------------------------------------- 1 | #ifndef UTILS_INC 2 | #define UTILS_INC 3 | #include 4 | 5 | namespace util { 6 | struct guarded_thread : std::thread { 7 | using thread::thread; 8 | ~guarded_thread() { 9 | if (joinable()) join(); 10 | } 11 | }; 12 | } 13 | #endif /* ----- #ifndef UTILS_INC ----- */ 14 | --------------------------------------------------------------------------------