├── xlogminer ├── xlogminer.control ├── Makefile ├── xlogminer_contents.h ├── xlogminer--1.0.sql ├── datadictionary.h ├── xlogminer_contents.c ├── pg_logminer.h ├── logminer.h ├── organizsql.c ├── logminer.c ├── xlogreader_logminer.c └── pg_logminer.c ├── LICENSE ├── README.CN.md └── README.md /xlogminer/xlogminer.control: -------------------------------------------------------------------------------- 1 | # xlogminer extension 2 | comment = 'inspect the contents of database pages at a low level' 3 | default_version = '1.0' 4 | module_pathname = '$libdir/xlogminer' 5 | relocatable = true 6 | -------------------------------------------------------------------------------- /xlogminer/Makefile: -------------------------------------------------------------------------------- 1 | ############################################################################ 2 | # 3 | # Authored by lichuancheng@highgo.com ,20170524 4 | # 5 | # Copyright: 6 | # Copyright (c) 2017-2020, HighGo Software Co.,Ltd. All right reserved 7 | # Makefile 8 | ############################################################################ 9 | 10 | MODULE_big = xlogminer 11 | OBJS = pg_logminer.o logminer.o organizsql.o xlogreader_logminer.o datadictionary.o xlogminer_contents.o 12 | 13 | EXTENSION = xlogminer 14 | DATA = xlogminer--1.0.sql 15 | 16 | ifdef USE_PGXS 17 | PG_CONFIG = pg_config 18 | PGXS := $(shell $(PG_CONFIG) --pgxs) 19 | include $(PGXS) 20 | else 21 | subdir = contrib/xlogminer 22 | top_builddir = ../.. 23 | include $(top_builddir)/src/Makefile.global 24 | include $(top_srcdir)/contrib/contrib-global.mk 25 | endif 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017-2020, HighGo Software Co.,Ltd. All right reserved 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /xlogminer/xlogminer_contents.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * Abstract: 4 | * Maintain the display table of analyse result. 5 | * 6 | * Authored by lichuancheng@highgo.com ,20170524 7 | * 8 | * Copyright: 9 | * Copyright (c) 2017-2020, HighGo Software Co.,Ltd. All right reserved 10 | * 11 | * Identification: 12 | * xlogminer_contents.h 13 | * 14 | *------------------------------------------------------------------------- 15 | */ 16 | 17 | #ifndef PG_XLOGMINER_CONTENTS_H 18 | #define PG_XLOGMINER_CONTENTS_H 19 | #include "postgres.h" 20 | 21 | #define PG_XLOGMINER_CONTENTS_SPACE_ADDSTEP 100 22 | #define PG_XLOGMINER_CONTENTS_STEPCOMMIT_STEP 1000 23 | 24 | 25 | typedef struct FormData_xlogminer_contents 26 | { 27 | uint32 sqlno; 28 | uint32 xid; 29 | int virtualxid; 30 | TimestampTz timestamp; 31 | char* record_database; 32 | char* record_user; 33 | char* record_tablespace; 34 | char* record_schema; 35 | char* op_type; 36 | char* op_text; 37 | char* op_undo; 38 | }FormData_xlogminer_contents; 39 | typedef FormData_xlogminer_contents *Form_xlogminer_contents; 40 | 41 | 42 | 43 | #define Natts_xlogminer_contents 10 44 | #define Anum_xlogminer_contents_xid 1 45 | #define Anum_xlogminer_contents_virtualxid 2 46 | #define Anum_xlogminer_contents_timestamp 3 47 | #define Anum_xlogminer_contents_record_database 4 48 | #define Anum_xlogminer_contents_record_user 5 49 | #define Anum_xlogminer_contents_record_tablespace 6 50 | #define Anum_xlogminer_contents_record_schema 7 51 | #define Anum_xlogminer_contents_op_type 8 52 | #define Anum_xlogminer_contents_op_text 9 53 | #define Anum_xlogminer_contents_op_undo 10 54 | 55 | /* 56 | * functions prototype 57 | */ 58 | void InsertXlogContentsTuple(Form_xlogminer_contents fxc); 59 | void UpdateXlogContentsTuple(Form_xlogminer_contents fxc); 60 | void addSQLspace(void); 61 | void cleanSQLspace(void); 62 | void freeSQLspace(void); 63 | 64 | 65 | 66 | #endif 67 | -------------------------------------------------------------------------------- /xlogminer/xlogminer--1.0.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Abstract: 3 | * Create function for xlogminer 4 | * 5 | * Authored by lichuancheng@highgo.com ,20170524 6 | * 7 | * Copyright: 8 | * Copyright (c) 2017-2020, HighGo Software Co.,Ltd. All right reserved 9 | * 10 | * Identification: 11 | * xlogminer--1.0.sql 12 | */ 13 | CREATE OR REPLACE FUNCTION pg_minerXlog(starttime text, endtime text, startxid int, endxid int) 14 | RETURNS text AS 15 | 'MODULE_PATHNAME','pg_minerXlog' 16 | LANGUAGE C CALLED ON NULL INPUT; 17 | 18 | CREATE OR REPLACE FUNCTION xlogminer_temp_table_check() 19 | RETURNS void AS 20 | $BODY$ 21 | DECLARE 22 | rd "varchar"; 23 | checksql "varchar"; 24 | temptablename "varchar"; 25 | tp "varchar"; 26 | BEGIN 27 | temptablename := 'xlogminer_contents'; 28 | tp :='t'; 29 | SELECT * into rd FROM pg_catalog.pg_class WHERE relname = 'xlogminer_contents' AND relpersistence = 't'; 30 | IF FOUND THEN 31 | TRUNCATE TABLE xlogminer_contents; 32 | ELSE 33 | CREATE temp TABLE xlogminer_contents(xid bigint,virtualxid int,timestampTz timestampTz,record_database text, record_user text,record_tablespace text,record_schema text, op_type text,op_text text,op_undo text); 34 | END IF; 35 | END; 36 | $BODY$ 37 | LANGUAGE 'plpgsql' VOLATILE; 38 | 39 | 40 | CREATE OR REPLACE FUNCTION xlogminer_start(starttime text, endtime text, startxid int, endxid int) 41 | RETURNS text AS 42 | $BODY$ 43 | select xlogminer_temp_table_check(); 44 | select pg_minerXlog($1,$2,$3,$4); 45 | $BODY$ 46 | LANGUAGE 'sql'; 47 | 48 | 49 | CREATE OR REPLACE FUNCTION xlogminer_build_dictionary(in path text) 50 | RETURNS text AS 51 | 'MODULE_PATHNAME','xlogminer_build_dictionary' 52 | LANGUAGE C CALLED ON NULL INPUT; 53 | 54 | CREATE OR REPLACE FUNCTION xlogminer_load_dictionary(in path text) 55 | RETURNS text AS 56 | 'MODULE_PATHNAME','xlogminer_load_dictionary' 57 | LANGUAGE C CALLED ON NULL INPUT; 58 | 59 | CREATE OR REPLACE FUNCTION xlogminer_stop() 60 | RETURNS text AS 61 | 'MODULE_PATHNAME','xlogminer_stop' 62 | LANGUAGE C VOLATILE STRICT; 63 | 64 | CREATE OR REPLACE FUNCTION xlogminer_xlogfile_add(in path text) 65 | RETURNS text AS 66 | 'MODULE_PATHNAME','xlogminer_xlogfile_add' 67 | LANGUAGE C CALLED ON NULL INPUT; 68 | 69 | CREATE OR REPLACE FUNCTION xlogminer_xlogfile_remove(in path text) 70 | RETURNS text AS 71 | 'MODULE_PATHNAME','xlogminer_xlogfile_remove' 72 | LANGUAGE C CALLED ON NULL INPUT; 73 | 74 | CREATE OR REPLACE FUNCTION xlogminer_xlogfile_list() 75 | RETURNS setof record AS 76 | 'MODULE_PATHNAME','xlogminer_xlogfile_list' 77 | LANGUAGE C VOLATILE STRICT; 78 | 79 | -------------------------------------------------------------------------------- /xlogminer/datadictionary.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Abstract: 3 | * DataDictionary and XlogFileList function 4 | * 5 | * Authored by lichuancheng@highgo.com ,20170524 6 | * 7 | * Copyright: 8 | * Copyright (c) 2017-2020, HighGo Software Co.,Ltd. All right reserved 9 | * 10 | * Identification: 11 | * datadictionary.h 12 | */ 13 | #ifndef PG_DATADICTIONARY_H 14 | #define PG_DATADICTIONARY_H 15 | #include "postgres.h" 16 | 17 | #include "logminer.h" 18 | 19 | #define PG_LOGMINER_DICTIONARY_DEFAULTNAME "dictionary.d" 20 | #define PG_LOGMINER_DICTIONARY_STOREPATH "pgdictory.store" 21 | #define PG_LOGMINER_DICTIONARY_XLOGFILELIST "xloglist.list" 22 | #define PG_LOGMINER_DICTIONARY_TEMPTABLE "xlogminer_contents" 23 | 24 | #define PG_LOGMINER_DICTIONARY_PATHCHECK_NULL 0 25 | #define PG_LOGMINER_DICTIONARY_PATHCHECK_FILE 1 26 | #define PG_LOGMINER_DICTIONARY_PATHCHECK_DIR 2 27 | #define PG_LOGMINER_DICTIONARY_PATHCHECK_INVALID 3 28 | #define PG_LOGMINER_DICTIONARY_PATHCHECK_SINGLE 4 29 | #define PG_LOGMINER_XLOGFILE_REPEAT_CHECK_DIFFERENT 1 30 | #define PG_LOGMINER_XLOGFILE_REPEAT_CHECK_SAME 2 31 | #define PG_LOGMINER_XLOGFILE_REPEAT_CHECK_SEGSAME 3 32 | 33 | 34 | #define PG_LOGMINER_DICTIONARY_LOADTYPE_SELF 1 35 | #define PG_LOGMINER_DICTIONARY_LOADTYPE_OTHER 2 36 | #define PG_LOGMINER_DICTIONARY_LOADTYPE_NOTHING 3 37 | 38 | 39 | 40 | 41 | #define PG_LOGMINER_DICTIONARY_SYSDATACACHE_SIZE 200 * 1024 42 | extern char* DataDictionaryCache; 43 | extern char* XlogfileListCache; 44 | extern char dictionary_path[MAXPGPATH]; 45 | 46 | typedef struct SysDataCache{ 47 | char* data; 48 | char* curdata; 49 | int64 usesize; 50 | int64 totsize; 51 | int elemnum; 52 | }SysDataCache; 53 | 54 | typedef struct DataDicHead{ 55 | NameData relname; 56 | int elemnum; 57 | }DataDicHead; 58 | 59 | typedef struct PgDataDic{ 60 | uint64 sysid; 61 | Oid dboid; 62 | SysDataCache sdc[PG_LOGMINER_IMPTSYSCLASS_IMPTNUM]; 63 | int dicloadtype; 64 | TimeLineID maxtl; 65 | TimeLineID datadictl; 66 | bool mutitimeline; 67 | }PgDataDic; 68 | 69 | typedef struct XlogFile 70 | { 71 | char filepath[MAXPGPATH]; 72 | struct XlogFile* tail; 73 | struct XlogFile* next; 74 | }XlogFile; 75 | 76 | typedef XlogFile* XlogFileList; 77 | 78 | 79 | char* outputSysTableDictionary(char *path, SysClassLevel *scl, bool self); 80 | void loadSystableDictionary(char *path, SysClassLevel *scl, bool self); 81 | int addxlogfile(char *path); 82 | int removexlogfile(char *path); 83 | void cleanSystableDictionary(void); 84 | void cleanXlogfileList(void); 85 | bool getRelationOidByName(char* relname, Oid* reloid, bool gettemptable); 86 | uint64 getDataDicSysid(void); 87 | int getDatadictionaryLoadType(void); 88 | bool is_xlogfilelist_exist(void); 89 | int getXlogFileNum(); 90 | int getRelationNameByOid(Oid reloid, NameData* relname); 91 | TupleDesc GetDescrByreloid(Oid reloid); 92 | bool tableIftoastrel(Oid reloid); 93 | bool getTypeOutputFuncFromDic(Oid type, Oid *typOutput, bool *typIsVarlena); 94 | Oid getDataDicOid(void); 95 | TimeLineID getDataDictl(void); 96 | bool loadXlogfileList(); 97 | bool loadDicStorePath(char *dicstorepath); 98 | void writeXlogfileList(void); 99 | void writeDicStorePath(char* dicstorepath); 100 | void dropAnalyseFile(void); 101 | char* getNextXlogFile(char *fctx, bool show); 102 | void searchSysClass( SystemClass *sys_class,int *sys_classNum); 103 | char* getdbNameByoid(Oid dboid, bool createdb); 104 | char* gettbsNameByoid(Oid tbsoid); 105 | Oid gettuserOidByReloid(Oid reloid); 106 | char* getuserNameByUseroid(Oid useroid); 107 | Oid getnsoidByReloid(Oid reloid); 108 | char* getnsNameByOid(Oid schoid); 109 | char* getnsNameByReloid(Oid reloid); 110 | Oid getRelationOidByRelfileid(Oid relNodeid); 111 | void freetupdesc(TupleDesc tupdesc); 112 | void checkXlogFileList(void); 113 | int getXlogFileNum(void); 114 | bool loadXlogfileList(void); 115 | 116 | 117 | #endif 118 | -------------------------------------------------------------------------------- /README.CN.md: -------------------------------------------------------------------------------- 1 | XLogMiner 2 | ===== 3 | 4 | # 什么是XLogMiner 5 | XLogMiner是从PostgreSQL的WAL(write ahead logs)日志中解析出执行的SQL语句的工具，并能生成出对应的undo SQL语句。 6 | 7 | # 配置要求 8 | ## 需要将数据库日志级别配置为logical模式, 并将表设置为full模式。例如，下面的语句将表t1设置为full模式: 9 | 10 | ```sql 11 | alter table t1 replica identity FULL; 12 | ``` 13 | 14 | ## PG版本支持 15 | 目前主要是在PostgreSQL 9.5.x上开发测试的，在9.6版本中仅做过简单验证，如果使用过程中发现问题欢迎向我们反馈。 16 | 17 | # 编译安装 18 | 1. 将xlogminer目录放置到编译通过的PG工程的"../contrib/"目录下 19 | 2. 进入xlogminer目录 20 | 3. 执行命令 21 | ```shell 22 | make && make install 23 | ``` 24 | 25 | # 使用方法 26 | ## 场景一：从WAL日志产生的数据库中直接执行解析 27 | ### 1. 创建xlogminer的extension 28 | create extension xlogminer; 29 | 30 | ### 2. Add xlog日志文件 31 | ```sql 32 | -- 增加wal文件： 33 | select xlogminer_xlogfile_add('/opt/test/wal'); 34 | -- 注：参数可以为目录或者文件 35 | ``` 36 | 37 | ### 3. Remove xlog日志文件 38 | ```sql 39 | -- 移除wal文件： 40 | select xlogminer_xlogfile_remove('/opt/test/wal'); 41 | -- 注：参数可以为目录或者文件 42 | ``` 43 | 44 | ### 4. List xlog日志文件 45 | ```sql 46 | -- 列出wal文件： 47 | select xlogminer_xlogfile_list(); 48 | ``` 49 | 50 | ### 5. 执行解析 51 | ```sql 52 | select xlogminer_start(’START_TIMSTAMP’,’STOP_TIMESTAMP’,’START_XID’,’STOP_XID’) 53 | ---如果分析全部日志： 54 | select xlogminer_start('null','null',0,0); 55 | ``` 56 | 57 | * **START_TIMESTAMP**：指定输出结果中最早的记录条目，即从该时间开始输出分析数据；若该参数值为空，则以分析日志列表中最早数据开始输出；若该参数值指定时间没有包含在所分析xlog列表中，即通过分析发现全部早于该参数指定时间，则返回空值。 58 | * **STOP_TIMESTAMP**：指定数据结果中最晚的记录条目，即输出结果如果大于该时间，则停止分析，不需要继续输出；如果该参数值为空，则从**START_TIMESTAMP**开始的所有日志都进行分析和输出。 59 | * **START_XID**：作用与**START_TIMESTAMP**相同，指定开始的**XID**值； 60 | * **STOP_XID**：作用与**STOP_TIMESTAMP**相同，指定结束的**XID**值 61 | 62 | :warning: **两组参数只能有一组为有效输入，否则报错。** 63 | 64 | 65 | 66 | ### 6. 解析结果查看 67 | ```sql 68 | select * from xlogminer_contents; 69 | ``` 70 | 71 | ### 7. 结束xlogminer操作 72 | 该函数作用为释放内存，结束日志分析，该函数没有参数。 73 | ```sql 74 | select xlogminer_stop(); 75 | ``` 76 | 77 | 78 | ## 场景二：从非WAL产生的数据库中执行WAL日志解析 79 | :warning: 要求执行解析的PostgreSQL数据库和被解析的为同一版本 80 | 81 | ### 于生产数据库 82 | 83 | #### 1.创建xlogminer的extension 84 | ```sql 85 | create extension xlogminer; 86 | ``` 87 | 88 | #### 2.生成数据字典 89 | ```sql 90 | select xlogminer_build_dictionary('/opt/proc/store_dictionary'); 91 | -- 注：参数可以为目录或者文件 92 | ``` 93 | 94 | 95 | ### 于测试数据库 96 | 97 | #### 1. 创建xlogminer的extension 98 | ```sql 99 | create extension xlogminer; 100 | ``` 101 | 102 | #### 2. load数据字典 103 | ```sql 104 | select xlogminer_load_dictionary('/opt/test/store_dictionary'); 105 | -- 注：参数可以为目录或者文件 106 | ``` 107 | 108 | #### 3. add xlog日志文件 109 | ```sql 110 | -- 增加wal文件： 111 | select xlogminer_xlogfile_add('/opt/test/wal'); 112 | -- 注：参数可以为目录或者文件 113 | ``` 114 | 115 | #### 4. remove xlog日志文件 116 | ```sql 117 | -- 移除wal文件： 118 | select xlogminer_xlogfile_remove('/opt/test/wal'); 119 | -- 注：参数可以为目录或者文件 120 | ``` 121 | #### 5. list xlog日志文件 122 | ```sql 123 | -- 列出wal文件： 124 | select xlogminer_xlogfile_list(); 125 | -- 注：参数可以为目录或者文件 126 | ``` 127 | 128 | 129 | #### 6. 执行解析 130 | ```sql 131 | select xlogminer_start(’START_TIMSTAMP’,’STOP_TIMESTAMP’,’START_XID’,’STOP_XID’) 132 | ``` 133 | * **START_TIMESTAMP**：指定输出结果中最早的记录条目，即从该时间开始输出分析数据；若该参数值为空，则以分析日志列表中最早数据开始输出；若该参数值指定时间没有包含在所分析xlog列表中，即通过分析发现全部早于该参数指定时间，则返回空值。 134 | * **STOP_TIMESTAMP**：指定数据结果中最晚的记录条目，即输出结果如果大于该时间，则停止分析，不需要继续输出；如果该参数值为空，则从START_TIMESTAMP开始的所有日志都进行分析和输出。 135 | * **START_XID**：作用与START_TIMESTAMP相同，指定开始的XID值； 136 | * **STOP_XID**：作用与STOP_TIMESTAMP相同，指定结束的XID值 137 | 两组参数只能有一组为有效输入，否则报错。 138 | 139 | #### 7. 解析结果查看 140 | ```sql 141 | select * from xlogminer_contents; 142 | ``` 143 | 144 | ### 8.结束xlogminer操作,该函数作用为释放内存，结束日志分析，该函数没有参数。 145 | ```sql 146 | select xlogminer_stop(); 147 | ``` 148 | 149 | :warning: **注意**：xlogminer_contents是xlogminer自动生成的临时表，因此当session断开再重新进入或其他session中解析数据不可见。这么做主要是基于安全考虑。 150 | 如果希望保留解析结果，可利用create xxx as select * from xlogminer_contents;写入普通表中。 151 | 152 | # 使用限制 153 | 1. 本版本只解析DML语句，不处理DDL语句 154 | 2. 执行了删除表、truncate表、更改表的表空间、更改表字段的类型，这样的DDL语句后，发生DDL语句之前的此表相关的DML语句不会再被解析。 155 | 3. 解析结果依赖于最新的数据字典。（举例：创建表t1,所有者为user1，但是中间将所有者改为user2。那解析结果中，所有t1相关操作所有者都将标示为user2） 156 | 4. wal日志如果发生缺失，在缺失的wal日志中发生提交的数据，都不会在解析结果中出现 157 | 5. 解析结果中undo字段的ctid属性是发生变更“当时”的值，如果因为vacuum等操作导致ctid发生变更，这个值将不准确。对于有可能存在重复行的数据，我们需要通过这个值确定undo对应的tuple条数，不代表可以直接执行该undo语句。 158 | 6. 若没有将表设置为full模式，那么update、delete语句将无法被解析。（当然这很影响使用，下一版本就会对这个问题作出改进） 159 | 7. 若没有将数据库日志级别设置为logical，解析结果会有无法预料的语句丢失 160 | 8. 执行了表字段drop的DDL语句后,发生DDL语句之前的这个字段相关的值都会被解析为encode（'AD976BC56F'，hex）的形式，另外自定义类型也会解析为这种形式 161 | 9. 只能解析与数据字典时间线一致的xlog文件 162 | 163 | # 联系我们 164 | 发现bug或者有好的建议可以通过邮箱（opensource@highgo.com）联系我们。 165 | -------------------------------------------------------------------------------- /xlogminer/xlogminer_contents.c: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * Abstract: 4 | * Maintain the display table of analyse result. 5 | * 6 | * Authored by lichuancheng@highgo.com ,20170524 7 | * 8 | * Copyright: 9 | * Copyright (c) 2017-2020, HighGo Software Co.,Ltd. All right reserved 10 | * 11 | * Identification: 12 | * xlogminer_contents.c 13 | *------------------------------------------------------------------------- 14 | */ 15 | #include "logminer.h" 16 | #include "xlogminer_contents.h" 17 | #include "utils/builtins.h" 18 | #include "catalog/indexing.h" 19 | #include "datadictionary.h" 20 | 21 | 22 | 23 | void 24 | addSQLspace() 25 | { 26 | int addstep = PG_XLOGMINER_CONTENTS_SPACE_ADDSTEP; 27 | XlogminerContentsFirst* xcftemp = NULL; 28 | 29 | if(!srctl.xcf) 30 | { 31 | srctl.xcf = (char *)logminer_palloc(addstep * sizeof(XlogminerContentsFirst),0); 32 | srctl.xcftotnum = addstep; 33 | } 34 | else 35 | { 36 | xcftemp = (XlogminerContentsFirst *)logminer_palloc((addstep + srctl.xcftotnum)*sizeof(XlogminerContentsFirst),0); 37 | if(!xcftemp) 38 | ereport(ERROR, 39 | (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), 40 | errmsg("Out of memory"))); 41 | memcpy(xcftemp, srctl.xcf, srctl.xcftotnum * sizeof(XlogminerContentsFirst)); 42 | 43 | logminer_pfree(srctl.xcf,0); 44 | srctl.xcf = (char *)xcftemp; 45 | srctl.xcftotnum += addstep; 46 | } 47 | padNullToXC(); 48 | } 49 | 50 | 51 | void 52 | cleanSQLspace() 53 | { 54 | int loop = 0; 55 | XlogminerContentsFirst* xcftemp = NULL; 56 | 57 | 58 | if(srctl.xcf) 59 | { 60 | xcftemp = (XlogminerContentsFirst*)srctl.xcf; 61 | for(loop = 0;loop < srctl.xcfcurnum; loop++) 62 | { 63 | xcftemp[loop].xid = 0; 64 | cleanSpace(&xcftemp[loop].op_text); 65 | cleanSpace(&xcftemp[loop].op_undo); 66 | cleanSpace(&xcftemp[loop].op_type); 67 | cleanSpace(&xcftemp[loop].record_database); 68 | cleanSpace(&xcftemp[loop].record_schema); 69 | cleanSpace(&xcftemp[loop].record_tablespace); 70 | cleanSpace(&xcftemp[loop].record_user); 71 | } 72 | srctl.xcfcurnum = 0; 73 | } 74 | padNullToXC(); 75 | } 76 | 77 | void 78 | freeSQLspace() 79 | { 80 | int loop = 0; 81 | XlogminerContentsFirst* xcftemp = NULL; 82 | 83 | 84 | if(srctl.xcf) 85 | { 86 | xcftemp = (XlogminerContentsFirst*)srctl.xcf; 87 | for(loop = 0;loop < srctl.xcfcurnum; loop++) 88 | { 89 | freeSpace(&xcftemp[loop].op_text); 90 | freeSpace(&xcftemp[loop].op_undo); 91 | freeSpace(&xcftemp[loop].op_type); 92 | freeSpace(&xcftemp[loop].record_database); 93 | freeSpace(&xcftemp[loop].record_schema); 94 | freeSpace(&xcftemp[loop].record_tablespace); 95 | freeSpace(&xcftemp[loop].record_user); 96 | } 97 | srctl.xcftotnum = 0; 98 | srctl.xcfcurnum = 0; 99 | logminer_pfree(srctl.xcf,0); 100 | srctl.xcf = NULL; 101 | } 102 | } 103 | 104 | void 105 | InsertXlogContentsTuple(Form_xlogminer_contents fxc) 106 | { 107 | Relation xlogminer_contents = NULL; 108 | HeapTuple tup = NULL; 109 | Oid reloid = 0; 110 | text *op_text = NULL; 111 | text *op_undo = NULL; 112 | text *op_type = NULL; 113 | text *record_database = NULL; 114 | text *record_user = NULL; 115 | text *record_tablespace = NULL; 116 | text *record_schema = NULL; 117 | Datum values[Natts_xlogminer_contents]; 118 | bool nulls[Natts_xlogminer_contents]; 119 | 120 | memset(values, 0, sizeof(values)); 121 | memset(nulls, false, sizeof(nulls)); 122 | if(0 == rrctl.logprivate.xlogminer_contents_oid) 123 | if(!getRelationOidByName(PG_LOGMINER_DICTIONARY_TEMPTABLE,&rrctl.logprivate.xlogminer_contents_oid,true)) 124 | ereport(ERROR,(errmsg("It is failed to open temporary table xlogminer_contents.")));/*should not happen*/ 125 | reloid = rrctl.logprivate.xlogminer_contents_oid; 126 | values[Anum_xlogminer_contents_xid - 1] = Int64GetDatum(fxc->xid); 127 | values[Anum_xlogminer_contents_virtualxid - 1] = Int32GetDatum(fxc->virtualxid); 128 | values[Anum_xlogminer_contents_timestamp - 1] = TimestampTzGetDatum(fxc->timestamp); 129 | if(fxc->record_database) 130 | { 131 | record_database = cstring_to_text(fxc->record_database); 132 | values[Anum_xlogminer_contents_record_database - 1] = PointerGetDatum(record_database); 133 | } 134 | else 135 | nulls[Anum_xlogminer_contents_record_database - 1] = true; 136 | 137 | if(fxc->record_user) 138 | { 139 | record_user = cstring_to_text(fxc->record_user); 140 | values[Anum_xlogminer_contents_record_user - 1] = PointerGetDatum(record_user); 141 | } 142 | else 143 | nulls[Anum_xlogminer_contents_record_user - 1] = true; 144 | 145 | if(fxc->record_tablespace) 146 | { 147 | record_tablespace = cstring_to_text(fxc->record_tablespace); 148 | values[Anum_xlogminer_contents_record_tablespace - 1] = PointerGetDatum(record_tablespace); 149 | } 150 | else 151 | nulls[Anum_xlogminer_contents_record_tablespace - 1] = true; 152 | 153 | if(fxc->record_schema) 154 | { 155 | record_schema = cstring_to_text(fxc->record_schema); 156 | values[Anum_xlogminer_contents_record_schema - 1] = PointerGetDatum(record_schema); 157 | } 158 | else 159 | nulls[Anum_xlogminer_contents_record_schema - 1] = true; 160 | 161 | if(fxc->op_type) 162 | { 163 | op_type = cstring_to_text(fxc->op_type); 164 | values[Anum_xlogminer_contents_op_type - 1] = PointerGetDatum(op_type); 165 | } 166 | else 167 | nulls[Anum_xlogminer_contents_op_type - 1] = true; 168 | if(fxc->op_text) 169 | { 170 | op_text = cstring_to_text(fxc->op_text); 171 | values[Anum_xlogminer_contents_op_text - 1] = PointerGetDatum(op_text); 172 | } 173 | else 174 | nulls[Anum_xlogminer_contents_op_text - 1] = true; 175 | if(fxc->op_undo) 176 | { 177 | op_undo = cstring_to_text(fxc->op_undo); 178 | values[Anum_xlogminer_contents_op_undo - 1] = PointerGetDatum(op_undo); 179 | } 180 | else 181 | nulls[Anum_xlogminer_contents_op_undo - 1] = true; 182 | 183 | xlogminer_contents = heap_open(reloid, AccessShareLock); 184 | if(!xlogminer_contents) 185 | ereport(ERROR,(errmsg("It is failed to open temporary table xlogminer_contents."))); 186 | 187 | tup = heap_form_tuple(RelationGetDescr(xlogminer_contents), values, nulls); 188 | simple_heap_insert(xlogminer_contents, tup); 189 | CatalogUpdateIndexes(xlogminer_contents, tup); 190 | logminer_pfree((char *)op_text,0); 191 | op_text = NULL; 192 | logminer_pfree((char *)op_undo,0); 193 | op_undo = NULL; 194 | logminer_pfree((char *)op_type,0); 195 | op_type = NULL; 196 | logminer_pfree((char *)record_database,0); 197 | record_database = NULL; 198 | logminer_pfree((char *)record_user,0); 199 | record_user = NULL; 200 | logminer_pfree((char *)record_tablespace,0); 201 | record_tablespace = NULL; 202 | logminer_pfree((char *)record_schema,0); 203 | record_schema = NULL; 204 | 205 | if(tup) 206 | heap_freetuple(tup); 207 | if(xlogminer_contents) 208 | heap_close(xlogminer_contents, AccessShareLock); 209 | } 210 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | XLogMiner 2 | ===== 3 | an Open-Source SQL miner on PostgreSQL WAL log 4 | 5 | # What is XLogMiner 6 | XLogMiner is used for parsing the SQL statement out from the PostgreSQL WAL (write ahead logs) logs, and it can also generate the corresponding "undo SQL". 7 | 8 | # Configuration requirements 9 | You need configure the WAL level to "logical", and setup the table to "full" mode. For example, the blow statement is setting the table "t1" to "full" mode. 10 | 11 | ```sql 12 | alter table t1 replica identity FULL; 13 | ``` 14 | 15 | # Supported PostgreSQL versions 16 | The project was developed based on PostgreSQL 9.4.x, with only basic verification on PostgreSQL 9.6. 17 | Please let us know if you find any issues during your use. 18 | 19 | # Compile and install 20 | 1. Copy the xlogminer directory into the "../contrib/" directory of the PostgreSQL source code location 21 | 2. Enter "contrib/xlogminer" directory 22 | 3. Execute commands 23 | ```shell 24 | make && make install 25 | ``` 26 | 27 | # User Guide 28 | ## Scenario I: Execute the parsing in the owner database of the WAL files 29 | ### 1. Create extension xlogminer 30 | ```sql 31 | create extension xlogminer; 32 | ``` 33 | 34 | ### 2. Add target WAL files 35 | ```sql 36 | -- Add WAL file or directory 37 | select xlogminer_xlogfile_add('/opt/test/wal'); 38 | -- Note: the parameter can be file name or directory name. 39 | ``` 40 | 41 | ### 3. Remove WAL files 42 | ```sql 43 | -- remove WAL file or directory 44 | select xlogminer_xlogfile_remove('/opt/test/wal'); 45 | -- Note: the parameter can be file name or directory name. 46 | ``` 47 | 48 | ### 4. List WAL files 49 | ```sql 50 | -- List WAL files 51 | select xlogminer_xlogfile_list(); 52 | ``` 53 | 54 | ### 5. Execute the xlogminer 55 | ```sql 56 | select xlogminer_start(’START_TIMSTAMP’,’STOP_TIMESTAMP’,’START_XID’,’STOP_XID’); 57 | -- Run below sql to parse all the WAL logs 58 | select xlogminer_start('null','null',0,0); 59 | ``` 60 | * **START_TIMESTAMP**：Specify the start time condition of the records in the return results, the xlogminer will start parsing from this time value. If this value is NULL, the earlist records will be displayed from the WAL lists. If this time value is not included in the xlog lists, a.k.a all the records are ealier then this value, the NULL will be returned. 61 | * **STOP_TIMESTAMP**：Specify the ending time condition of the records in the results, the xlogminer will stop parsing when the result is later than this time. If this parameter is NULL, then all the records after **START_TIMESTAMP** will parsed and displyed in the WAL logs. 62 | * **START_XID**：Similiar with **START_TIMESTAMP**, specify the starting **XID** value 63 | * **STOP_XID**：Similiar with **STOP_TIMESTAMP**，specify the ending **XID** value 64 | 65 | :warning: **Only one of these two group parameters can be provided, or error will be retured** 66 | 67 | ### 6. Check the parsing result 68 | ```sql 69 | select * from xlogminer_contents; 70 | ``` 71 | 72 | ### 7. Stop the xlogminer 73 | This function is used to free the memory and stop the WAL parsing. No parameters available. 74 | ```sql 75 | select xlogminer_stop(); 76 | ``` 77 | 78 | 79 | ## Scenario II: Parsing the WAL logs from the database which is not the owner of these WAL logs 80 | :warning: The target PostgreSQL database and the source database must have the same version 81 | 82 | ### On production database 83 | 84 | #### 1. Create xlogminer extension 85 | ```sql 86 | create extension xlogminer; 87 | ``` 88 | 89 | #### 2. Build the dictionary 90 | ```sql 91 | select xlogminer_build_dictionary('/opt/proc/store_dictionary'); 92 | -- Note: the parameter can be file name or directory name. 93 | ``` 94 | 95 | ### On testing database 96 | 97 | #### 1. Create xlogminer extension 98 | ```sql 99 | create extension xlogminer; 100 | ``` 101 | 102 | #### 2. Load database dictionary 103 | ```sql 104 | select xlogminer_load_dictionary('/opt/test/store_dictionary'); 105 | -- Note: the parameter can be file name or directory name. 106 | ``` 107 | :bulb: the parameter can be file name or directory name. 108 | 109 | #### 3. Add WAL files 110 | ```sql 111 | -- Add WAL files 112 | select xlogminer_xlogfile_add('/opt/test/wal'); 113 | -- Note: the parameter can be file name or directory name. 114 | ``` 115 | 116 | #### 4. remove xlog WAL files 117 | ```sql 118 | -- Remove WAL files 119 | select xlogminer_xlogfile_remove('/opt/test/wal'); 120 | -- Note:the parameter can be file name or directory name. 121 | ``` 122 | 123 | #### 5. List xlog WAL files 124 | ```sql 125 | -- list WAL files 126 | select xlogminer_xlogfile_list(); 127 | -- Note:the parameter can be file name or directory name. 128 | ``` 129 | 130 | #### 6. Execute the parsing 131 | ```sql 132 | select xlogminer_start(’START_TIMSTAMP’,’STOP_TIMESTAMP’,’START_XID’,’STOP_XID’) 133 | ``` 134 | 135 | * **START_TIMESTAMP**：Specify the start time condition of the records in the return results, the xlogminer will start parsing from this time value. If this value is NULL, the earlist records will be displayed from the WAL lists. If this time value is not included in the xlog lists, a.k.a all the records are ealier then this value, the NULL will be returned. 136 | * **STOP_TIMESTAMP**：Specify the ending time condition of the records in the results, the xlogminer will stop parsing when the result is later than this time. If this parameter is NULL, then all the records after **START_TIMESTAMP** will parsed and displyed in the WAL logs. 137 | * **START_XID**：Similiar with **START_TIMESTAMP**, specify the starting **XID** value 138 | * **STOP_XID**：Similiar with **STOP_TIMESTAMP**，specify the ending **XID** value 139 | 140 | #### 7. Check the parsing result 141 | ```sql 142 | select * from xlogminer_contents; 143 | ``` 144 | 145 | ### 8.Stop the xlogminer 146 | This function is used to free the memory and stop the WAL parsing. No parameters available. 147 | ```sql 148 | select xlogminer_stop(); 149 | ``` 150 | 151 | :warning: **NOTE**：For the security considerations, xlogminer_contents is a temporary table generated by xlogminer automatically, it is not visible when session disconnected and then re-connect, and it is also not visible to other sessions. 152 | If you want to keep the paring result, you can use the below SQL to write the results into a regular table. 153 | ```sql 154 | create xxx as select * from xlogminer_contents; 155 | ``` 156 | 157 | # Limitations 158 | 1. Only DML statements will be parsed in this version, DDL statement not supported. 159 | 2. The DML statemes would **NOT** be parsed out when the below DDL related operations were executed: 160 | Deleting/Truncating table, table space modification and column type modification etcs. 161 | 3. The parsing result is depending on the latest database dictionary. For example, after user1 created table t1, the table owner was modified to user2, then all the parsing results related to table t1 will be marked with user2. 162 | 4. If WAL logs are missed in a time stage, the SQL statements executed in that time stage would **NOT** be parsed out. 163 | 5. The "ctid" attribute is the value of the change "at that time". If there are "ctid" changes due to vacuum or other operations, this value will be **inaccurate**. We need use this value to determine the corresponding undo tuples when the rows of data are duplicate, it does not mean that you can execute such undo statements directly. 164 | 6. If the table is not set to full mode, then the "update" and "delete" statement will not be resolved. (Of course, this affects the use of this software, the next version will make improvements to this problem） 165 | 7. If the database log level is not set to **logical**, there will be unpredictable lost of the SQL statements 166 | 8. If the DDL statement "drop" was executed, all related column value will be decoded as "encode('AD976BC56F',hex)" before this DDL execution. 167 | 168 | # Contact us 169 | Please contact us with opensource@highgo.com if you have any comments or find any bugs, thanks! 170 | -------------------------------------------------------------------------------- /xlogminer/pg_logminer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Abstract: 3 | * Main analyse function of XLogminer 4 | * 5 | * Authored by lichuancheng@highgo.com ,20170524 6 | * 7 | * Copyright: 8 | * Copyright (c) 2017-2020, HighGo Software Co.,Ltd. All right reserved 9 | * 10 | * Identification: 11 | * pg_logminer.h 12 | * 13 | */ 14 | 15 | #ifndef PG_LOGMINER_H 16 | #define PG_LOGMINER_H 17 | #include "postgres.h" 18 | #include "access/xlog.h" 19 | #include "access/xlogreader.h" 20 | #include "access/heapam_xlog.h" 21 | #include "access/xact.h" 22 | #include "utils/relfilenodemap.h" 23 | #include "utils/syscache.h" 24 | #include "commands/dbcommands.h" 25 | #include "commands/tablespace.h" 26 | #include "common/fe_memutils.h" 27 | #include "catalog/pg_class.h" 28 | #include "access/htup_details.h" 29 | #include "utils/rel.h" 30 | #include "funcapi.h" 31 | #include "miscadmin.h" 32 | #include "utils/memutils.h" 33 | #include "xlogminer_contents.h" 34 | 35 | 36 | #define XLogRecGetData_Logminer(record) ((char*) (record) + SizeOfXLogRecord) 37 | 38 | #define PG_LOGMINER_FLAG_FINDHIGHGO -10 39 | #define PG_LOGMINER_FLAG_INITOVER -8 40 | #define PG_LOGMINER_CONTRLKIND_FIND 1 41 | #define PG_LOGMINER_CONTRLKIND_XACT 2 42 | 43 | #define PG_LOGMINER_OID_PGCATALOG 11 44 | #define PG_LOGMINER_INITCHECK_STEP 10 45 | #define PG_LOGMINER_INITCHECK_START 1 46 | #define PG_LOGMINER_INITCHECK_END 10 47 | 48 | #define PG_LOGMINER_SQLKIND_UPDATE 1 49 | #define PG_LOGMINER_SQLKIND_INSERT 2 50 | #define PG_LOGMINER_SQLKIND_DELETE 3 51 | #define PG_LOGMINER_SQLKIND_CREATE 4 52 | #define PG_LOGMINER_SQLKIND_ALTER 5 53 | #define PG_LOGMINER_SQLKIND_XACT 6 54 | #define PG_LOGMINER_SQLKIND_DROP 7 55 | #define PG_LOGMINER_SQLKIND_MAXNUM 7 56 | 57 | #define PG_LOGMINER_TABLEKIND_SYSCLASS 1 58 | #define PG_LOGMINER_TABLEKIND_IMPTSYS 2 59 | #define PG_LOGMINER_TABLEKIND_NORMAL 3 60 | 61 | #define PG_LOGMINER_WALFILE_ERROR_NOFIND -1 /*did not find the wal file*/ 62 | #define PG_LOGMINER_WALFILE_ENDALL -2 /*arrive at num that user refer to*/ 63 | #define PG_LOGMINER_WALFILE_ERROR_COUNT -10 64 | 65 | 66 | #define PG_LOGMINER_XLOG_UNKNOWN 0 67 | #define PG_LOGMINER_XLOG_DBINIT 1 68 | #define PG_LOGMINER_XLOG_NOMAL 2 69 | 70 | #define PG_LOGMINER_SYSCLASS_MAX 80 71 | #define PG_LOGMINER_XLOGMINERSQL_MAXSIZE 100000 72 | 73 | #define PG_LOGMINER_IMPTSYSCLASS_IMPTNUM 18 74 | #define PG_LOGMINER_IMPTSYSCLASS_PGCLASS 0 75 | #define PG_LOGMINER_IMPTSYSCLASS_PGDATABASE 1 76 | #define PG_LOGMINER_IMPTSYSCLASS_PGEXTENSION 2 77 | #define PG_LOGMINER_IMPTSYSCLASS_PGNAMESPACE 3 78 | #define PG_LOGMINER_IMPTSYSCLASS_PGTABLESPACE 4 79 | #define PG_LOGMINER_IMPTSYSCLASS_PGCONSTRAINT 5 80 | #define PG_LOGMINER_IMPTSYSCLASS_PGAUTHID 6 81 | #define PG_LOGMINER_IMPTSYSCLASS_PGPROC 7 82 | #define PG_LOGMINER_IMPTSYSCLASS_PGDEPEND 8 83 | #define PG_LOGMINER_IMPTSYSCLASS_PGINDEX 9 84 | #define PG_LOGMINER_IMPTSYSCLASS_PGATTRIBUTE 10 85 | #define PG_LOGMINER_IMPTSYSCLASS_PGSHDESC 11 86 | #define PG_LOGMINER_IMPTSYSCLASS_PGATTRDEF 12 87 | #define PG_LOGMINER_IMPTSYSCLASS_PGTYPE 13 88 | #define PG_LOGMINER_IMPTSYSCLASS_PGAUTH_MEMBERS 14 89 | #define PG_LOGMINER_IMPTSYSCLASS_PGINHERITS 15 90 | #define PG_LOGMINER_IMPTSYSCLASS_PGTRIGGER 16 91 | #define PG_LOGMINER_IMPTSYSCLASS_PGLANGUAGE 17 92 | 93 | #define SYSLEL_A 1 94 | #define SYSLEL_B 2 95 | #define SYSLEL_C 3 96 | 97 | 98 | #define PG_LOGMINER_DATABASE_POSTGRES "postgres" 99 | #define PG_LOGMINER_DATABASE_HIGHGO "highgo" 100 | #define PG_LOGMINER_TABLE_DATABASE "pg_database" 101 | #define PG_LOGMINER_TABLE_SHDESC "pg_shdescription" 102 | #define PG_LOGMINER_TABLE_DEPEND "pg_depend" 103 | #define PG_LOGMINER_TABLE_ATTRIBUTE "pg_attribute" 104 | #define PG_LOGMINER_TABLE_INDEX "pg_index" 105 | #define PG_LOGMINER_TABLE_CONSTRAINT "pg_constraint" 106 | #define PG_LOGMINER_TABLE_CLASS "pg_class" 107 | 108 | 109 | typedef struct SystemClass{ 110 | NameData classname; 111 | int attnum; 112 | }SystemClass; 113 | 114 | typedef SystemClass *SysClass; 115 | 116 | typedef struct SysClassLevel{ 117 | int tabid; 118 | char *relname; 119 | int datasgsize; 120 | }SysClassLevel; 121 | 122 | typedef struct SQLKind{ 123 | char *sqlhead; 124 | int sqlid; 125 | }SQLKind; 126 | 127 | typedef struct XLogMinerSQL 128 | { 129 | char *sqlStr; 130 | int tot_size; 131 | int rem_size; 132 | int use_size; 133 | }XLogMinerSQL; 134 | 135 | typedef struct XLsqList *XLsqlptr; 136 | typedef struct XLsqList 137 | { 138 | XLogMinerSQL xlsql; 139 | XLsqlptr next; 140 | }XLsqList; 141 | 142 | typedef struct XLsqlhead 143 | { 144 | XLsqlptr head; 145 | XLsqlptr tail; 146 | int xlnum; 147 | }XLsqlhead; 148 | 149 | typedef struct ToastTuple 150 | { 151 | Oid chunk_id; 152 | int chunk_seq; 153 | char *chunk_data; 154 | int datalength; 155 | struct ToastTuple* next; 156 | }ToastTuple; 157 | 158 | 159 | typedef struct XLogMinerPrivate 160 | { 161 | TimeLineID timeline; 162 | XLogRecPtr startptr; 163 | int analynum; 164 | bool staptr_reached; 165 | bool endptr_reached; 166 | bool timecheck; 167 | bool xidcheck; 168 | bool changewal; 169 | bool serialwal; 170 | 171 | 172 | XLogRecPtr limitstartptr; 173 | XLogRecPtr limitendptr; 174 | TimestampTz parser_start_time; 175 | TimestampTz parser_end_time; 176 | TransactionId parser_start_xid; 177 | TransactionId parser_end_xid; 178 | 179 | Oid xlogminer_contents_oid; 180 | }XLogMinerPrivate; 181 | 182 | 183 | typedef struct 184 | { 185 | long offset; 186 | bool hasnextbuff; 187 | bool hasnextxlogfile; 188 | char *xlogfileptr; 189 | 190 | int sendFile; 191 | XLogSegNo sendSegNo; 192 | uint32 sendOff; 193 | }logminer_fctx; 194 | 195 | 196 | typedef struct RecordRecycleCtl 197 | { 198 | XLogReaderState *xlogreader_state; 199 | XLogRecPtr first_record; 200 | XLogMinerPrivate logprivate; 201 | 202 | char *tuplem; 203 | char *tuplem_old; 204 | char *tuplem_bigold; 205 | Oid reloid; 206 | Oid tbsoid; 207 | bool nomalrel; 208 | bool sysrel; 209 | bool imprel; 210 | bool toastrel; 211 | int sqlkind; 212 | uint8 prostatu; 213 | TransactionId recordxid; 214 | 215 | XLogMinerSQL tupinfo; 216 | bool tupinfo_init; 217 | XLogMinerSQL tupinfo_old; 218 | bool tupinfo_old_init; 219 | char *errormsg; 220 | int system_init_record; 221 | int sysstoplocation; 222 | logminer_fctx lfctx; 223 | 224 | Datum *values; 225 | bool *nulls; 226 | Datum *values_old; 227 | bool *nulls_old; 228 | TupleDesc tupdesc; 229 | ToastTuple *tthead; 230 | 231 | MemoryContext oldcxt; 232 | MemoryContext mycxt; 233 | MemoryContext tupledesccxt; 234 | 235 | }RecordRecycleCtl; 236 | 237 | 238 | typedef struct SQLRecycleCtl 239 | { 240 | XLogMinerSQL sql_simple; 241 | XLsqlhead xlhead; 242 | XLsqlptr sqlptr; 243 | 244 | /*for toast index remove when create*/ 245 | Oid toastoid; 246 | 247 | /*for record to xlogminer_contents*/ 248 | XLogMinerSQL sql_undo; 249 | RelFileNode rfnode; 250 | 251 | /*for mutiinsert*/ 252 | bool mutinsert; 253 | int sqlindex; 254 | char *multdata; 255 | /*For store sqls in a xact*/ 256 | char *xcf; 257 | int xcftotnum; 258 | int xcfcurnum; 259 | }SQLRecycleCtl; 260 | 261 | typedef struct XlogResult{ 262 | XLogMinerSQL sql; 263 | char ct[25]; 264 | }XlogResult; 265 | 266 | extern RecordRecycleCtl rrctl; 267 | extern SQLRecycleCtl srctl; 268 | extern uint32 sqlnoser; 269 | 270 | 271 | void appendtoSQL(XLogMinerSQL *sql_simple, char *sqlpara , int spaceKind); 272 | XLogRecord *XLogReadRecord_logminer(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg); 273 | XLogRecPtr XLogFindFirstRecord(XLogReaderState *state, XLogRecPtr RecPtr); 274 | int XLogMinerReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, 275 | XLogRecPtr targetPtr, char *readBuff, TimeLineID *curFileTLI); 276 | int XLogMinerXLogRead(const char *directory, TimeLineID *timeline_id, 277 | XLogRecPtr startptr, char *buf, Size count); 278 | SysClassLevel *getImportantSysClass(); 279 | void processContrl(char* relname, int contrlkind); 280 | void appendtoSQL_simquo(XLogMinerSQL *sql_simple, char* ptr, bool quoset); 281 | void appendtoSQL_doubquo(XLogMinerSQL *sql_simple, char* ptr, bool quoset); 282 | void appendtoSQL_atttyptrans(XLogMinerSQL *sql_simple, Oid typoid); 283 | void appendtoSQL_valuetyptrans(XLogMinerSQL *sql_simple, Oid typoid); 284 | void wipeSQLFromstr(XLogMinerSQL *sql_simple,char *fromstr,char *checkstr); 285 | void appendBlanktoSQL(XLogMinerSQL *sql_simple); 286 | SysClassLevel *getImportantSysClass(void); 287 | 288 | 289 | #endif 290 | -------------------------------------------------------------------------------- /xlogminer/logminer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Abstract: 3 | * Some tool function for Xlogminer 4 | * 5 | * Authored by lichuancheng@highgo.com ,20170524 6 | * 7 | * Copyright: 8 | * Copyright (c) 2017-2020, HighGo Software Co.,Ltd. All right reserved 9 | * 10 | * Identification: 11 | * logminer.h 12 | */ 13 | 14 | #ifndef LOGMINER_H 15 | #define LOGMINER_H 16 | #include "postgres.h" 17 | #include "pg_logminer.h" 18 | #include "utils/timestamp.h" 19 | #include "utils/datetime.h" 20 | #include "access/xlog_internal.h" 21 | 22 | 23 | 24 | #define bbr_Judge(ch) ((PG_LOGMINER_BBRACK_R == ch)?true:false) 25 | #define bbl_Judge(ch) ((PG_LOGMINER_BBRACK_L == ch)?true:false) 26 | #define sbr_Judge(ch) ((PG_LOGMINER_SBRACK_R == ch)?true:false) 27 | #define sbl_Judge(ch) ((PG_LOGMINER_SBRACK_L == ch)?true:false) 28 | 29 | 30 | 31 | #define LOGMINERDIR "xlogminer" 32 | #define LOGMINERDIR_DIC "dic" 33 | #define LOGMINERDIR_LIST "list" 34 | 35 | #define PG_LOGMINER_SPACE ' ' 36 | #define PG_LOGMINER_SBRACK_L '(' 37 | #define PG_LOGMINER_SBRACK_R ')' 38 | #define PG_LOGMINER_COMMA ',' 39 | #define PG_LOGMINER_BBRACK_L '{' 40 | #define PG_LOGMINER_BBRACK_R '}' 41 | #define PG_LOGMINER_ENDSTR '\n' 42 | 43 | #define PG_LOGMINER_RELNULL "NULL" 44 | #define PG_LOMINER_ALLROLEPRIV "arwdDxt" 45 | 46 | #define LOGMINER_PRIVKIND_MAXNUM 15 47 | 48 | #define PG_LOGMINER_SQLPARA_TOTSTEP 10000 49 | #define PG_LOGMINER_SQLPARA_SIMSTEP 100 50 | #define PG_LOGMINER_SQLPARA_TOTLE 1 51 | #define PG_LOGMINER_SQLPARA_SIMPLE 2 52 | #define PG_LOGMINER_SQLPARA_OTHER 3 53 | 54 | #define LOGMINER_INSERTSTEP 4 55 | #define LOGMINER_DELETESTEP 5 56 | 57 | 58 | /*-----------sysclass attribute location-----------*/ 59 | #define LOGMINER_STEP 1 60 | #define LOGMINER_ATTRIBUTE_LOCATION_UPDATE_RELNAME 2 61 | #define LOGMINER_ATTRIBUTE_LOCATION_UPDATE_NEWDATA 5 62 | #define LOGMINER_ATTRIBUTE_LOCATION_UPDATE_OLDDATA 8 63 | #define LOGMINER_ATTRIBUTE_LOCATION_CREATE_KIND 2 64 | #define LOGMINER_ATTRIBUTE_LOCATION_CREATE_NAME 3 65 | #define LOGMINER_INSERT_TABLE_NAME 3 66 | #define LOGMINER_SQL_COMMAND 1 67 | #define LOGMINER_DELETE_TABLE_NAME 3 68 | /*--------------------------------------------*/ 69 | 70 | 71 | #define LOGMINER_RELKINDID_TABLE 1 72 | #define LOGMINER_RELKINDID_INDEX 2 73 | #define LOGMINER_RELKINDID_SEQUENCE 3 74 | #define LOGMINER_RELKINDID_VIEW 4 75 | #define LOGMINER_RELKINDID_TOAST 5 76 | #define LOGMINER_RELKINDID_COMPLEX 6 77 | #define LOGMINER_RELATIONKIND_NUM 7 78 | 79 | #define LOGMINER_CONSTRAINT_CONNUM 6 80 | #define LOGMINER_CONSTRAINT_CHECK 'c' 81 | #define LOGMINER_CONSTRAINT_FOREIGN 'f' 82 | #define LOGMINER_CONSTRAINT_PRIMARY 'p' 83 | #define LOGMINER_CONSTRAINT_UNIQUE 'u' 84 | #define LOGMINER_CONSTRAINT_TRIGGER 't' 85 | #define LOGMINER_CONSTRAINT_EXCLUSION 'x' 86 | 87 | 88 | 89 | #define LOGMINER_SQLGET_DDL_CREATE_TABLE 1 90 | #define LOGMINER_SQLGET_DDL_CREATE_INDEX 2 91 | #define LOGMINER_SQLGET_DDL_CREATE_SEQUENCE 3 92 | #define LOGMINER_SQLGET_DDL_CREATE_VIEW 4 93 | #define LOGMINER_SQLGET_DDL_CREATE_TOAST 5 94 | #define LOGMINER_SQLGET_DDL_CREATE_COMPLEX 6 95 | #define LOGMINER_SQLGET_DDL_CREATE_CONSTRAINT 7 96 | #define LOGMINER_SQLGET_DDL_CREATE_MATERIALIZEDVIEW 8 97 | 98 | #define LOGMINER_SQLGET_DML_INSERT 8 99 | #define LOGMINER_SQLGET_DML_UPDATE 9 100 | #define LOGMINER_SQLGET_DML_DELETE 10 101 | #define LOGMINER_SQLGET_DML_ALTERTABLE_ADDCOLUMN 11 102 | #define LOGMINER_SQLGET_DML_ALTERTABLE_ALTERCOLTYP_SEG1 12 103 | #define LOGMINER_SQLGET_DML_ALTERTABLE_ALTERCOLTYP_SEG2 13 104 | #define LOGMINER_SQLGET_DML_ALTERTABLE_COLUMNDEFAULT 14 105 | #define LOGMINER_SQLGET_DDL_CREATE_DATABASE 15 106 | #define LOGMINER_SQLGET_DDL_CREATE_PARTTABLE 16 107 | 108 | 109 | 110 | 111 | #define LOGMINER_SIMPLESQL_BUFSIZE 1024 112 | 113 | #define LOGMINER_TEMPFILE_SIMPLESQL "simple.sql" 114 | #define LOGMINER_TEMPFILE_DBSQL "db.sql" 115 | #define LOGMINER_PASSKIND_SPECIALLOC_MAX 5 116 | #define LOGMINER_PASSKIND_SPECIALCH_CUT 1 /*a special char as Separator only*/ 117 | #define LOGMINER_PASSKIND_SPECIALCH_AVOID 2 /*a special char escape a Separator*/ 118 | 119 | #define LOGMINER_WARNING_WORD_FULLPAGEWRITE "data missing. \n" 120 | 121 | #define LOGMINER_PROSTATUE_INSERT_MISSING_TUPLEINFO 0x01 122 | #define LOGMINER_PROSTATUE_DELETE_MISSING_TUPLEINFO 0x02 123 | #define LOGMINER_PROSTATUE_UPDATE_MISSING_NEW_TUPLEINFO 0x03 124 | #define LOGMINER_PROSTATUE_UPDATE_MISSING_OLD_TUPLEINFO 0x04 125 | 126 | 127 | 128 | 129 | typedef struct RelationKind{ 130 | int sqlkind; 131 | int relkindid; 132 | char *relname; 133 | char relkind; 134 | bool show; 135 | }RelationKind; 136 | 137 | typedef struct SQLPassOver{ 138 | /*special location*/ 139 | int passloc[LOGMINER_PASSKIND_SPECIALLOC_MAX]; 140 | /*special location num*/ 141 | int passlocnum; 142 | 143 | int passkind; 144 | /*special char*/ 145 | char specialch[2]; 146 | /*special char num*/ 147 | int specialnum; 148 | }SQLPassOver; 149 | 150 | typedef struct OperaPriv{ 151 | char elemname[NAMEDATALEN]; 152 | char privkind[LOGMINER_PRIVKIND_MAXNUM]; 153 | }OperaPriv; 154 | 155 | typedef struct PrivKind{ 156 | char privch; 157 | char *privstr; 158 | }PrivKind; 159 | 160 | 161 | typedef struct XlogminerContentsFirst{ 162 | /*int sqlno*/ 163 | TransactionId xid; 164 | /*uint32 virtualxid;*/ 165 | /*Timestamp timestamp;*/ 166 | XLogMinerSQL record_database; 167 | XLogMinerSQL record_user; 168 | XLogMinerSQL record_tablespace; 169 | XLogMinerSQL record_schema; 170 | XLogMinerSQL op_type; 171 | XLogMinerSQL op_text; 172 | XLogMinerSQL op_undo; 173 | }XlogminerContentsFirst; 174 | 175 | 176 | bool getPhrases(char *sql,int loc, char *term, int ignoresbrackph); 177 | void addSpace(XLogMinerSQL *sql_simple, int spaceKind); 178 | void cleanSpace(XLogMinerSQL *minersql); 179 | void freeSpace(XLogMinerSQL *minersql); 180 | void cleanMentalvalues(void); 181 | void split_path_fname(const char *path, char **dir, char **fname); 182 | RelationKind* getRelKindInfo(); 183 | void xactCommitSQL(char* timestr,XLogMinerSQL *sql_opt,uint8 info); 184 | int xlog_file_open(const char *directory, const char *fname); 185 | bool isEmptStr(char *str); 186 | void xlsql_addtoList(XLsqlhead *xlhead,XLsqList *xlsql); 187 | TupleDesc makeOutputXlogDesc(); 188 | bool inputParaCheck(); 189 | bool curXactCheck(TimestampTz xact_time ,TransactionId xid, bool xactcommit,xl_xact_parsed_commit *parsed_commit); 190 | char* logminer_palloc(int size,int checkflag); 191 | void logminer_pfree(char* ptr,int checkflag); 192 | void logminer_free(char* ptr,int checkflag); 193 | char* logminer_malloc(int size,int checkflag); 194 | void logminer_free(char* ptr,int checkflag); 195 | void logminer_createMemContext(); 196 | void logminer_switchMemContext(); 197 | bool checkLogminerUser(); 198 | bool padingminerXlogconts(char* elemname, TransactionId xid,int loc,long elemoid); 199 | void cleanMentalvalues(); 200 | bool elemNameFind(char* elenname); 201 | void cleanAnalyseInfo(); 202 | void padNullToXC(); 203 | char* getTuplemSpace(int size); 204 | void cleanTuplemSpace(char* tuplem); 205 | bool ifquoneed(Form_pg_attribute attrs); 206 | char* OutputToByte(text* attrpter, int attlen); 207 | ToastTuple* makeToastTuple(int datalength,char* data, Oid id, int seq); 208 | void freeToastTupleHead(); 209 | void toastTupleAddToList(ToastTuple *tt); 210 | text* cstringToTextWithLen(const char *s, int len); 211 | bool getTypeOutputFuncFromDb(Oid type, Oid *typOutput, bool *typIsVarlena); 212 | char* convertAttrToStr(Form_pg_attribute fpa,Oid typoutput, Datum attr); 213 | int strcmp_withlength(char *str1,char *str2,int length); 214 | bool ifQueNeedDelete(Form_pg_attribute attrs); 215 | void checkVarlena(Datum attr,struct varlena** att_return); 216 | void deleteQueFromStr(char* strPara); 217 | void keepDigitFromStr(char* strPara); 218 | void fixPathEnd(char *path); 219 | RelationKind* getRelKindInfo(void); 220 | TupleDesc makeOutputXlogDesc(void); 221 | bool inputParaCheck(text *st, text *et); 222 | void logminer_createMemContext(void); 223 | void logminer_switchMemContext(void); 224 | bool checkLogminerUser(void); 225 | void padNullToXC(void); 226 | void cleanAnalyseInfo(void); 227 | void freeToastTupleHead(void); 228 | 229 | 230 | /* 231 | * organizsql.c funcitons declare here 232 | */ 233 | extern void getInsertSQL(XLogMinerSQL *sql_simple, char *tupleInfo, NameData *relname, char* schname, bool sysrel); 234 | extern void getDeleteSQL(XLogMinerSQL *sql_simple, char *tupleInfo, NameData *relname, char* schname, bool sysrel, bool undo); 235 | extern void getUpdateSQL(XLogMinerSQL *sql_simple, char *tupleInfo, char *tupleInfo_old,NameData *relname, char* schname, bool sysrel); 236 | extern void minerDbCreate(XLogReaderState *record, XLogMinerSQL *sql_simple,uint8 info); 237 | extern void mentalTup_nulldata(int natts, int index, XLogMinerSQL *values_sql,XLogMinerSQL *att_sql, bool valueappend, bool attdroped, bool *firstattget); 238 | extern void mentalTup_valuedata(int natts, int index, XLogMinerSQL *values_sql,XLogMinerSQL *att_sql, bool valueappend, bool attdroped, bool quoset,char* strPara, TupleDesc typeinfo,bool *firstattget); 239 | extern void mentalTup(HeapTuple tuple, TupleDesc typeinfo ,XLogMinerSQL *sql_simple, bool olddata); 240 | extern void reAssembleUpdateSql(XLogMinerSQL *sql_ori, bool undo); 241 | extern void reAssembleDeleteSql(XLogMinerSQL *sql_ori, bool undo); 242 | #endif 243 | 244 | -------------------------------------------------------------------------------- /xlogminer/organizsql.c: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * Abstract: 4 | * Function to spell the SQL. 5 | * 6 | * Authored by lichuancheng@highgo.com ,20170524 7 | * 8 | * Copyright: 9 | * Copyright (c) 2017-2020, HighGo Software Co.,Ltd. All right reserved 10 | * 11 | * Identification: 12 | * organizsql.c 13 | * 14 | *------------------------------------------------------------------------- 15 | */ 16 | 17 | #include "postgres.h" 18 | #include "logminer.h" 19 | #include "datadictionary.h" 20 | #include "catalog/pg_proc.h" 21 | #include "commands/dbcommands_xlog.h" 22 | #include "catalog/pg_extension.h" 23 | #include "catalog/pg_depend.h" 24 | #include "catalog/pg_attrdef.h" 25 | #include "catalog/pg_namespace.h" 26 | #include "catalog/pg_tablespace.h" 27 | #include "catalog/pg_database.h" 28 | #include "catalog/pg_authid.h" 29 | #include "catalog/pg_type.h" 30 | #include "utils/builtins.h" 31 | 32 | void 33 | xactCommitSQL(char* timestr,XLogMinerSQL *sql_opt,uint8 info) 34 | { 35 | if(XLOG_XACT_COMMIT == info) 36 | { 37 | appendtoSQL(sql_opt, "XACT COMMIT(time:", PG_LOGMINER_SQLPARA_SIMPLE); 38 | appendtoSQL(sql_opt, timestr, PG_LOGMINER_SQLPARA_SIMPLE); 39 | appendtoSQL(sql_opt, ")", PG_LOGMINER_SQLPARA_SIMPLE); 40 | } 41 | else if(XLOG_XACT_ABORT == info) 42 | { 43 | appendtoSQL(sql_opt, "XACT ABORT(time:", PG_LOGMINER_SQLPARA_SIMPLE); 44 | appendtoSQL(sql_opt, timestr, PG_LOGMINER_SQLPARA_SIMPLE); 45 | appendtoSQL(sql_opt, ")", PG_LOGMINER_SQLPARA_SIMPLE); 46 | } 47 | appendtoSQL(sql_opt, ";", PG_LOGMINER_SQLPARA_SIMPLE); 48 | } 49 | 50 | 51 | void 52 | getInsertSQL(XLogMinerSQL *sql_simple, char *tupleInfo, NameData *relname, char* schname, bool sysrel) 53 | { 54 | 55 | bool nameFind = false; 56 | appendtoSQL(sql_simple, "INSERT INTO ", PG_LOGMINER_SQLPARA_SIMPLE); 57 | nameFind = (0 == relname->data[0])?false:true; 58 | 59 | if(nameFind) 60 | { 61 | if(sysrel) 62 | appendtoSQL(sql_simple, relname->data, PG_LOGMINER_SQLPARA_SIMPLE); 63 | else 64 | { 65 | appendtoSQL_doubquo(sql_simple, schname, true); 66 | appendtoSQL(sql_simple, ".", PG_LOGMINER_SQLPARA_SIMPLE); 67 | appendtoSQL_doubquo(sql_simple, relname->data, true); 68 | } 69 | } 70 | else 71 | { 72 | appendtoSQL(sql_simple, "(NULL)", PG_LOGMINER_SQLPARA_SIMPLE); 73 | } 74 | 75 | if(tupleInfo) 76 | { 77 | if(sysrel) 78 | appendBlanktoSQL(sql_simple); 79 | appendtoSQL(sql_simple, tupleInfo, PG_LOGMINER_SQLPARA_SIMPLE); 80 | } 81 | else 82 | { 83 | rrctl.prostatu = LOGMINER_PROSTATUE_INSERT_MISSING_TUPLEINFO; 84 | appendtoSQL(sql_simple, " VALUES(NULL) (NOTICE:Miss table info.)", PG_LOGMINER_SQLPARA_SIMPLE); 85 | } 86 | 87 | 88 | appendtoSQL(sql_simple, ";", PG_LOGMINER_SQLPARA_SIMPLE); 89 | 90 | } 91 | 92 | void 93 | getDeleteSQL(XLogMinerSQL *sql_simple, char *tupleInfo, NameData *relname, char* schname, bool sysrel, bool undo) 94 | { 95 | 96 | bool nameFind = false; 97 | char *diviloc = NULL; 98 | 99 | appendtoSQL(sql_simple, "DELETE FROM ", PG_LOGMINER_SQLPARA_SIMPLE); 100 | nameFind = (0 == relname->data[0])?false:true; 101 | if(nameFind) 102 | { 103 | if(sysrel) 104 | appendtoSQL(sql_simple, relname->data, PG_LOGMINER_SQLPARA_SIMPLE); 105 | else 106 | { 107 | appendtoSQL_doubquo(sql_simple, schname, true); 108 | appendtoSQL(sql_simple, ".", PG_LOGMINER_SQLPARA_SIMPLE); 109 | appendtoSQL_doubquo(sql_simple, relname->data, true); 110 | } 111 | } 112 | else 113 | { 114 | appendtoSQL(sql_simple, "(NULL)", PG_LOGMINER_SQLPARA_SIMPLE); 115 | } 116 | 117 | if(tupleInfo) 118 | { 119 | appendtoSQL(sql_simple, " WHERE ", PG_LOGMINER_SQLPARA_SIMPLE); 120 | if(sysrel) 121 | appendtoSQL(sql_simple, tupleInfo, PG_LOGMINER_SQLPARA_SIMPLE); 122 | else 123 | { 124 | diviloc = strstr(tupleInfo,") VALUES(") + 2; 125 | appendtoSQL(sql_simple, diviloc, PG_LOGMINER_SQLPARA_SIMPLE); 126 | } 127 | } 128 | else 129 | { 130 | rrctl.prostatu = LOGMINER_PROSTATUE_INSERT_MISSING_TUPLEINFO; 131 | appendtoSQL(sql_simple, " WHERE VALUES(NULL) (NOTICE:Miss table info.)", PG_LOGMINER_SQLPARA_SIMPLE); 132 | } 133 | 134 | appendtoSQL(sql_simple, ";", PG_LOGMINER_SQLPARA_SIMPLE); 135 | } 136 | 137 | void 138 | getUpdateSQL(XLogMinerSQL *sql_simple, char *tupleInfo, char *tupleInfo_old,NameData *relname, char* schname, bool sysrel) 139 | { 140 | bool nameFind = false; 141 | nameFind = (0 == relname->data[0])?false:true; 142 | if(nameFind) 143 | { 144 | appendtoSQL(sql_simple, "UPDATE ", PG_LOGMINER_SQLPARA_SIMPLE); 145 | if(sysrel) 146 | appendtoSQL(sql_simple, relname->data, PG_LOGMINER_SQLPARA_SIMPLE); 147 | else 148 | { 149 | appendtoSQL_doubquo(sql_simple, schname, true); 150 | appendtoSQL(sql_simple, ".", PG_LOGMINER_SQLPARA_SIMPLE); 151 | appendtoSQL_doubquo(sql_simple, relname->data, true); 152 | } 153 | 154 | } 155 | else 156 | { 157 | appendtoSQL(sql_simple, "UPDATE ", PG_LOGMINER_SQLPARA_SIMPLE); 158 | appendtoSQL(sql_simple, "(NULL)", PG_LOGMINER_SQLPARA_SIMPLE); 159 | } 160 | 161 | if(tupleInfo) 162 | { 163 | appendtoSQL(sql_simple, " SET ", PG_LOGMINER_SQLPARA_SIMPLE); 164 | appendtoSQL(sql_simple, tupleInfo, PG_LOGMINER_SQLPARA_SIMPLE); 165 | } 166 | else 167 | { 168 | rrctl.prostatu = LOGMINER_PROSTATUE_UPDATE_MISSING_NEW_TUPLEINFO; 169 | appendtoSQL(sql_simple, " SET VALUES(NULL) (NOTICE:Miss table info.)", PG_LOGMINER_SQLPARA_SIMPLE); 170 | } 171 | 172 | if(tupleInfo_old) 173 | { 174 | appendtoSQL(sql_simple, " WHERE ", PG_LOGMINER_SQLPARA_SIMPLE); 175 | appendtoSQL(sql_simple, tupleInfo_old, PG_LOGMINER_SQLPARA_SIMPLE); 176 | } 177 | else 178 | rrctl.prostatu |= LOGMINER_PROSTATUE_UPDATE_MISSING_OLD_TUPLEINFO; 179 | 180 | appendtoSQL(sql_simple, ";", PG_LOGMINER_SQLPARA_SIMPLE); 181 | 182 | } 183 | 184 | void 185 | minerDbCreate(XLogReaderState *record, XLogMinerSQL *sql_simple,uint8 info) 186 | { 187 | xl_dbase_create_rec *xlrec = NULL; 188 | char *dbname = NULL; 189 | 190 | xlrec = (xl_dbase_create_rec *) XLogRecGetData(record); 191 | dbname = getdbNameByoid(xlrec->db_id, true); 192 | processContrl(dbname,PG_LOGMINER_CONTRLKIND_FIND); 193 | } 194 | 195 | void 196 | mentalTup_nulldata(int natts, int index, XLogMinerSQL *values_sql,XLogMinerSQL *att_sql, bool valueappend, bool attdroped, bool *firstattget) 197 | { 198 | if(1 == natts) 199 | { 200 | if(valueappend) 201 | { 202 | appendtoSQL(values_sql, "(", PG_LOGMINER_SQLPARA_OTHER); 203 | appendtoSQL(values_sql, ")", PG_LOGMINER_SQLPARA_OTHER); 204 | appendtoSQL(att_sql, "(", PG_LOGMINER_SQLPARA_OTHER); 205 | appendtoSQL(att_sql, ")", PG_LOGMINER_SQLPARA_OTHER); 206 | } 207 | else 208 | { 209 | appendtoSQL(values_sql, "(", PG_LOGMINER_SQLPARA_OTHER); 210 | if(!attdroped) 211 | appendtoSQL(values_sql, PG_LOGMINER_RELNULL, PG_LOGMINER_SQLPARA_OTHER); 212 | else 213 | appendtoSQL(values_sql, "encode(\'\',\'hex\')", PG_LOGMINER_SQLPARA_OTHER); 214 | appendtoSQL(values_sql, ")", PG_LOGMINER_SQLPARA_OTHER); 215 | *firstattget = true; 216 | 217 | } 218 | } 219 | else if(0 == index) 220 | { 221 | if(valueappend) 222 | { 223 | appendtoSQL(values_sql, "(", PG_LOGMINER_SQLPARA_OTHER); 224 | appendtoSQL(att_sql, "(", PG_LOGMINER_SQLPARA_OTHER); 225 | } 226 | else 227 | { 228 | appendtoSQL(values_sql, "(", PG_LOGMINER_SQLPARA_OTHER); 229 | if(!attdroped) 230 | appendtoSQL(values_sql, PG_LOGMINER_RELNULL, PG_LOGMINER_SQLPARA_OTHER); 231 | else 232 | appendtoSQL(values_sql, "encode(\'\',\'hex\')", PG_LOGMINER_SQLPARA_OTHER); 233 | *firstattget = true; 234 | /* appendtoSQL(values_sql, ", ", PG_LOGMINER_SQLPARA_OTHER);*/ 235 | } 236 | } 237 | else if(natts - 1 == index) 238 | { 239 | if(valueappend) 240 | { 241 | appendtoSQL(values_sql, ")", PG_LOGMINER_SQLPARA_OTHER); 242 | appendtoSQL(att_sql, ")", PG_LOGMINER_SQLPARA_OTHER); 243 | } 244 | else 245 | { 246 | if(*firstattget) 247 | appendtoSQL(values_sql, ", ", PG_LOGMINER_SQLPARA_OTHER); 248 | else 249 | *firstattget = true; 250 | if(!attdroped) 251 | appendtoSQL(values_sql, PG_LOGMINER_RELNULL, PG_LOGMINER_SQLPARA_OTHER); 252 | else 253 | appendtoSQL(values_sql, "encode(\'\',\'hex\')", PG_LOGMINER_SQLPARA_OTHER); 254 | appendtoSQL(values_sql, ")", PG_LOGMINER_SQLPARA_OTHER); 255 | } 256 | } 257 | else 258 | { 259 | if(valueappend) 260 | { 261 | } 262 | else 263 | { 264 | if(*firstattget) 265 | appendtoSQL(values_sql, ", ", PG_LOGMINER_SQLPARA_OTHER); 266 | else 267 | *firstattget = true; 268 | if(!attdroped) 269 | appendtoSQL(values_sql, PG_LOGMINER_RELNULL, PG_LOGMINER_SQLPARA_OTHER); 270 | else 271 | appendtoSQL(values_sql, "encode(\'\',\'hex\')", PG_LOGMINER_SQLPARA_OTHER); 272 | /*appendtoSQL(values_sql, ", ", PG_LOGMINER_SQLPARA_OTHER);*/ 273 | } 274 | } 275 | 276 | 277 | } 278 | 279 | void 280 | mentalTup_valuedata(int natts, int index, XLogMinerSQL *values_sql,XLogMinerSQL *att_sql, bool valueappend, bool attdroped, bool quoset,char* strPara, TupleDesc typeinfo,bool *firstattget) 281 | { 282 | char temp_name[NAMEDATALEN + 3] = {0}; 283 | 284 | if(1 == natts) 285 | { 286 | if(valueappend) 287 | { 288 | /*value*/ 289 | appendtoSQL(values_sql, "(", PG_LOGMINER_SQLPARA_OTHER); 290 | appendtoSQL_simquo(values_sql, strPara, quoset); 291 | appendtoSQL(values_sql, ")", PG_LOGMINER_SQLPARA_OTHER); 292 | /*attribute*/ 293 | appendtoSQL(att_sql, "(", PG_LOGMINER_SQLPARA_OTHER); 294 | if(!attdroped) 295 | appendtoSQL_doubquo(att_sql, typeinfo->attrs[index]->attname.data, true); 296 | else 297 | { 298 | memset(temp_name,0,NAMEDATALEN + 3); 299 | sprintf(temp_name, "COL%d", index+1); 300 | appendtoSQL_doubquo(att_sql, temp_name, true); 301 | } 302 | appendtoSQL(att_sql, ")", PG_LOGMINER_SQLPARA_OTHER); 303 | } 304 | else 305 | { 306 | /*value*/ 307 | appendtoSQL(values_sql, "(", PG_LOGMINER_SQLPARA_OTHER); 308 | appendtoSQL_simquo(values_sql, strPara, quoset); 309 | appendtoSQL(values_sql, ")", PG_LOGMINER_SQLPARA_OTHER); 310 | } 311 | *firstattget = true; 312 | } 313 | else if(0 == index) 314 | { 315 | if(valueappend) 316 | { 317 | /*value*/ 318 | appendtoSQL(values_sql, "(", PG_LOGMINER_SQLPARA_OTHER); 319 | appendtoSQL_simquo(values_sql, strPara, quoset); 320 | /*appendtoSQL(values_sql, ", ", PG_LOGMINER_SQLPARA_OTHER);*/ 321 | /*attribute*/ 322 | appendtoSQL(att_sql, "(", PG_LOGMINER_SQLPARA_OTHER); 323 | if(!attdroped) 324 | appendtoSQL_doubquo(att_sql, typeinfo->attrs[index]->attname.data, true); 325 | else 326 | { 327 | memset(temp_name,0,NAMEDATALEN + 3); 328 | sprintf(temp_name, "COL%d", index+1); 329 | appendtoSQL_doubquo(att_sql, temp_name, true); 330 | } 331 | /*appendtoSQL(att_sql, ", ", PG_LOGMINER_SQLPARA_OTHER); */ 332 | } 333 | else 334 | { 335 | /*value*/ 336 | appendtoSQL(values_sql, "(", PG_LOGMINER_SQLPARA_OTHER); 337 | appendtoSQL_simquo(values_sql, strPara, quoset); 338 | /*appendtoSQL(values_sql, ", ", PG_LOGMINER_SQLPARA_OTHER);*/ 339 | } 340 | *firstattget = true; 341 | } 342 | else if(natts - 1 == index) 343 | { 344 | if(*firstattget) 345 | { 346 | appendtoSQL(values_sql, ", ", PG_LOGMINER_SQLPARA_OTHER); 347 | appendtoSQL(att_sql, ", ", PG_LOGMINER_SQLPARA_OTHER); 348 | } 349 | else 350 | *firstattget = true; 351 | if(valueappend) 352 | { 353 | /*value*/ 354 | appendtoSQL_simquo(values_sql, strPara, quoset); 355 | appendtoSQL(values_sql, ")", PG_LOGMINER_SQLPARA_OTHER); 356 | /*attribute*/ 357 | if(!attdroped) 358 | appendtoSQL_doubquo(att_sql, typeinfo->attrs[index]->attname.data, true); 359 | else 360 | { 361 | memset(temp_name,0,NAMEDATALEN + 3); 362 | sprintf(temp_name, "COL%d", index+1); 363 | appendtoSQL_doubquo(att_sql, temp_name, true); 364 | } 365 | appendtoSQL(att_sql, ")", PG_LOGMINER_SQLPARA_OTHER); 366 | } 367 | else 368 | { 369 | /*value*/ 370 | appendtoSQL_simquo(values_sql, strPara, quoset); 371 | appendtoSQL(values_sql, ")", PG_LOGMINER_SQLPARA_OTHER); 372 | } 373 | } 374 | else 375 | { 376 | if(*firstattget) 377 | { 378 | appendtoSQL(values_sql, ", ", PG_LOGMINER_SQLPARA_OTHER); 379 | appendtoSQL(att_sql, ", ", PG_LOGMINER_SQLPARA_OTHER); 380 | } 381 | else 382 | *firstattget = true; 383 | if(valueappend) 384 | { 385 | /*value*/ 386 | appendtoSQL_simquo(values_sql, strPara, quoset); 387 | /* appendtoSQL(values_sql, ", ", PG_LOGMINER_SQLPARA_OTHER);*/ 388 | /*attribute*/ 389 | if(!attdroped) 390 | appendtoSQL_doubquo(att_sql, typeinfo->attrs[index]->attname.data, true); 391 | else 392 | { 393 | memset(temp_name,0,NAMEDATALEN + 3); 394 | sprintf(temp_name, "COL%d", index+1); 395 | appendtoSQL_doubquo(att_sql, temp_name, true); 396 | } 397 | /* appendtoSQL(att_sql, ", ", PG_LOGMINER_SQLPARA_OTHER);*/ 398 | } 399 | else 400 | { 401 | /*value*/ 402 | appendtoSQL_simquo(values_sql, strPara, quoset); 403 | /* appendtoSQL(values_sql, ", ", PG_LOGMINER_SQLPARA_OTHER);*/ 404 | } 405 | } 406 | 407 | } 408 | 409 | 410 | /* 411 | * 412 | * parser tuple info from binary to string. 413 | * 414 | */ 415 | void 416 | mentalTup(HeapTuple tuple, TupleDesc typeinfo ,XLogMinerSQL *sql_simple, bool olddata) 417 | { 418 | int natts; 419 | int i; 420 | Datum attr; 421 | Datum attr1; 422 | char *strPara; 423 | 424 | Oid typoutput; 425 | Oid typoutputfromdb; 426 | Oid typoutputfromdic; 427 | bool typisvarlena; 428 | Datum *values; 429 | bool *nulls; 430 | bool quoset = false; 431 | bool attdroped = false; 432 | bool valueappend = false; 433 | XLogMinerSQL values_sql; 434 | XLogMinerSQL att_sql; 435 | bool firstattget = false; 436 | bool gettype = false; 437 | struct varlena* att_return = NULL; 438 | 439 | memset(&values_sql, 0, sizeof(XLogMinerSQL)); 440 | memset(&att_sql, 0, sizeof(XLogMinerSQL)); 441 | 442 | natts = typeinfo->natts; 443 | valueappend = (rrctl.nomalrel) && (PG_LOGMINER_SQLKIND_INSERT == rrctl.sqlkind || PG_LOGMINER_SQLKIND_DELETE == rrctl.sqlkind); 444 | 445 | if(!olddata) 446 | { 447 | rrctl.values = (Datum *) logminer_palloc(natts * sizeof(Datum),0); 448 | rrctl.nulls = (bool *) logminer_palloc(natts * sizeof(bool),0); 449 | values = rrctl.values; 450 | nulls = rrctl.nulls; 451 | } 452 | else 453 | { 454 | rrctl.values_old = (Datum *) logminer_palloc(natts * sizeof(Datum),0); 455 | rrctl.nulls_old = (bool *) logminer_palloc(natts * sizeof(bool),0); 456 | values = rrctl.values_old; 457 | nulls = rrctl.nulls_old; 458 | } 459 | 460 | heap_deform_tuple(tuple, typeinfo, values ,nulls); 461 | 462 | for (i = 0; i < natts; ++i) 463 | { 464 | attr = values[i]; 465 | 466 | /*If this attribute has been droped*/ 467 | if(0 == typeinfo->attrs[i]->atttypid) 468 | attdroped = true; 469 | else 470 | attdroped = false; 471 | 472 | if (nulls[i]) 473 | { 474 | mentalTup_nulldata(natts, i, &values_sql,&att_sql, valueappend, attdroped,&firstattget); 475 | continue; 476 | } 477 | 478 | if(!attdroped) 479 | { 480 | gettype = getTypeOutputFuncFromDb(typeinfo->attrs[i]->atttypid, 481 | &typoutputfromdb, &typisvarlena); 482 | gettype = gettype && getTypeOutputFuncFromDic(typeinfo->attrs[i]->atttypid, 483 | &typoutputfromdic, &typisvarlena); 484 | gettype = gettype && (typoutputfromdic == typoutputfromdb); 485 | 486 | gettype = gettype && (FirstNormalObjectId > typeinfo->attrs[i]->atttypid); 487 | } 488 | else 489 | gettype = false; 490 | 491 | 492 | if(!attdroped && gettype) 493 | { 494 | /*Attribute is nomal,get data nomal*/ 495 | quoset = ifquoneed(typeinfo->attrs[i]) && rrctl.nomalrel; 496 | typoutput = typoutputfromdic; 497 | if(typisvarlena) 498 | { 499 | checkVarlena(attr,&att_return); 500 | if(!att_return) 501 | /*should not happen*/ 502 | ereport(ERROR,(errmsg("There are some wrong data in record."))); 503 | attr1 = CStringGetDatum(att_return); 504 | strPara = convertAttrToStr(typeinfo->attrs[i], typoutput, attr1); 505 | } 506 | else 507 | strPara = convertAttrToStr(typeinfo->attrs[i], typoutput, attr); 508 | } 509 | else if(attdroped || !gettype) 510 | { 511 | /*Attribute is droped,get data via byte*/ 512 | typisvarlena = (!typeinfo->attrs[i]->attbyval) && (-1 == typeinfo->attrs[i]->attlen); 513 | if(typisvarlena) 514 | { 515 | checkVarlena(attr,&att_return); 516 | attr1 = CStringGetDatum(att_return); 517 | strPara =OutputToByte((text *)attr1, typeinfo->attrs[i]->attlen); 518 | } 519 | else 520 | strPara = OutputToByte((text *)attr, typeinfo->attrs[i]->attlen); 521 | quoset = false; 522 | } 523 | mentalTup_valuedata(natts, i, &values_sql,&att_sql, valueappend, attdroped, quoset, strPara, typeinfo, &firstattget); 524 | 525 | } 526 | if(valueappend) 527 | { 528 | appendtoSQL(sql_simple, att_sql.sqlStr, PG_LOGMINER_SQLPARA_OTHER); 529 | appendBlanktoSQL(sql_simple); 530 | } 531 | appendtoSQL(sql_simple, "VALUES", PG_LOGMINER_SQLPARA_OTHER); 532 | appendtoSQL(sql_simple, values_sql.sqlStr, PG_LOGMINER_SQLPARA_OTHER); 533 | freeSpace(&att_sql); 534 | freeSpace(&values_sql); 535 | } 536 | 537 | void 538 | reAssembleUpdateSql(XLogMinerSQL *sql_ori, bool undo) 539 | { 540 | int natts = 0; 541 | int i = 0; 542 | Datum attr; 543 | Datum attr1; 544 | Datum ctid; 545 | Datum attr_old; 546 | Datum attr_old1; 547 | char *strPara = NULL; 548 | char *strPara_old = NULL; 549 | char *ctid_str = NULL; 550 | char temp_name[NAMEDATALEN + 3]; 551 | 552 | Oid typoutput; 553 | Oid typoutputfromdb; 554 | Oid typoutputfromdic; 555 | bool typisvarlena = false; 556 | TupleDesc typeinfo = NULL; 557 | int count_value = 0; 558 | Datum *values = NULL; 559 | bool *nulls = NULL; 560 | Datum *values_old = NULL; 561 | bool *nulls_old = NULL; 562 | bool quoset = false; 563 | bool attdroped = false; 564 | bool getcondition = false; 565 | bool gettype = false; 566 | bool getchange = false; 567 | struct varlena* att_return = NULL; 568 | HeapTupleHeader htup_old = NULL; 569 | 570 | 571 | if(!rrctl.values || !rrctl.nulls || !rrctl.values_old || !rrctl.nulls_old || !rrctl.tupdesc) 572 | { 573 | /*should not happen*/ 574 | ereport(ERROR,(errmsg("values or nulls or tupdesc is not exist"))); 575 | } 576 | typeinfo = rrctl.tupdesc; 577 | natts = typeinfo->natts; 578 | 579 | if(!undo) 580 | { 581 | values = rrctl.values; 582 | nulls = rrctl.nulls; 583 | values_old = rrctl.values_old; 584 | nulls_old = rrctl.nulls_old; 585 | } 586 | else 587 | { 588 | values = rrctl.values_old; 589 | nulls = rrctl.nulls_old; 590 | values_old = rrctl.values; 591 | nulls_old = rrctl.nulls; 592 | } 593 | 594 | /*set cause*/ 595 | for (i = 0; i < natts; ++i) 596 | { 597 | attr = values[i]; 598 | attr_old = values_old[i]; 599 | if(nulls[i] != nulls_old[i]) 600 | { 601 | ereport(ERROR,(errmsg("new tuple nulls is different from olds"))); 602 | } 603 | if(nulls[i]) 604 | continue; 605 | 606 | if(0 == typeinfo->attrs[i]->atttypid) 607 | attdroped = true; 608 | else 609 | attdroped = false; 610 | 611 | if(!attdroped) 612 | { 613 | gettype = getTypeOutputFuncFromDb(typeinfo->attrs[i]->atttypid, 614 | &typoutputfromdb, &typisvarlena); 615 | gettype = gettype && getTypeOutputFuncFromDic(typeinfo->attrs[i]->atttypid, 616 | &typoutputfromdic, &typisvarlena); 617 | gettype = gettype && (typoutputfromdic == typoutputfromdb); 618 | gettype = gettype && (FirstNormalObjectId > typeinfo->attrs[i]->atttypid); 619 | } 620 | else 621 | gettype = false; 622 | 623 | if(!attdroped && gettype) 624 | { 625 | typoutput = typoutputfromdic; 626 | if(typisvarlena) 627 | { 628 | checkVarlena(attr,&att_return); 629 | if(!att_return) 630 | /*should not happen*/ 631 | ereport(ERROR,(errmsg("wrong varlena data"))); 632 | attr1 = CStringGetDatum(att_return); 633 | strPara = convertAttrToStr(typeinfo->attrs[i], typoutput, attr1); 634 | 635 | att_return = NULL; 636 | checkVarlena(attr_old,&att_return); 637 | if(!att_return) 638 | /*should not happen*/ 639 | ereport(ERROR,(errmsg("wrong varlena data"))); 640 | attr_old1 = CStringGetDatum(att_return); 641 | strPara_old = convertAttrToStr(typeinfo->attrs[i],typoutput, attr_old1); 642 | 643 | } 644 | else 645 | { 646 | strPara = convertAttrToStr(typeinfo->attrs[i], typoutput, attr); 647 | strPara_old = convertAttrToStr(typeinfo->attrs[i], typoutput, attr_old); 648 | } 649 | quoset = ifquoneed(typeinfo->attrs[i]) && rrctl.nomalrel; 650 | } 651 | else 652 | { 653 | typisvarlena = (!typeinfo->attrs[i]->attbyval) && (-1 == typeinfo->attrs[i]->attlen); 654 | if(typisvarlena) 655 | { 656 | checkVarlena(attr,&att_return); 657 | attr1 = CStringGetDatum(att_return); 658 | if(!att_return) 659 | /*should not happen*/ 660 | ereport(ERROR,(errmsg("wrong varlena data"))); 661 | strPara =OutputToByte((text *)attr1, typeinfo->attrs[i]->attlen); 662 | 663 | att_return = NULL; 664 | checkVarlena(attr_old,&att_return); 665 | attr_old1 = CStringGetDatum(att_return); 666 | strPara_old =OutputToByte((text *)attr_old1, typeinfo->attrs[i]->attlen); 667 | } 668 | else 669 | { 670 | strPara = OutputToByte((text *)attr, typeinfo->attrs[i]->attlen); 671 | strPara_old = OutputToByte((text *)attr_old, typeinfo->attrs[i]->attlen); 672 | } 673 | quoset = false; 674 | } 675 | 676 | wipeSQLFromstr(sql_ori, " VALUES", " "); 677 | 678 | if(0 != strcmp(strPara ,strPara_old)) 679 | { 680 | if(!attdroped) 681 | appendtoSQL_doubquo(sql_ori, typeinfo->attrs[i]->attname.data, true); 682 | else 683 | { 684 | memset(temp_name,0,NAMEDATALEN + 3); 685 | sprintf(temp_name, "COL%d", i+1); 686 | appendtoSQL_doubquo(sql_ori, temp_name, true); 687 | } 688 | appendtoSQL(sql_ori, " = ", PG_LOGMINER_SQLPARA_OTHER); 689 | appendtoSQL_simquo(sql_ori, strPara, quoset); 690 | getchange = true; 691 | } 692 | } 693 | /*we get nothing changed,discard the update SQL*/ 694 | if(!getchange) 695 | { 696 | freeSpace(sql_ori); 697 | return; 698 | } 699 | 700 | appendtoSQL(sql_ori, " WHERE ", PG_LOGMINER_SQLPARA_OTHER); 701 | for (i = 0; i < natts; ++i) 702 | { 703 | attr_old = values_old[i]; 704 | if(nulls_old[i]) 705 | { 706 | continue; 707 | } 708 | 709 | if(0 == typeinfo->attrs[i]->atttypid) 710 | attdroped = true; 711 | else 712 | attdroped = false; 713 | 714 | 715 | if(!attdroped) 716 | { 717 | gettype = getTypeOutputFuncFromDb(typeinfo->attrs[i]->atttypid, 718 | &typoutputfromdb, &typisvarlena); 719 | gettype = gettype && getTypeOutputFuncFromDic(typeinfo->attrs[i]->atttypid, 720 | &typoutputfromdic, &typisvarlena); 721 | gettype = gettype && (typoutputfromdic == typoutputfromdb); 722 | gettype = gettype && (FirstNormalObjectId > typeinfo->attrs[i]->atttypid); 723 | } 724 | else 725 | gettype = false; 726 | 727 | 728 | if(!attdroped && gettype) 729 | { 730 | typoutput = typoutputfromdic; 731 | 732 | if(typisvarlena) 733 | { 734 | checkVarlena(attr_old,&att_return); 735 | if(!att_return) 736 | /*should not happen*/ 737 | ereport(ERROR,(errmsg("There are some wrong data in record."))); 738 | attr_old1 = CStringGetDatum(att_return); 739 | strPara_old = convertAttrToStr(typeinfo->attrs[i], typoutput, attr_old1); 740 | } 741 | else 742 | { 743 | strPara_old = convertAttrToStr(typeinfo->attrs[i], typoutput, attr_old); 744 | } 745 | quoset = ifquoneed(typeinfo->attrs[i]) && rrctl.nomalrel; 746 | } 747 | else 748 | { 749 | typisvarlena = (!typeinfo->attrs[i]->attbyval) && (-1 == typeinfo->attrs[i]->attlen); 750 | if(typisvarlena) 751 | { 752 | checkVarlena(attr_old,&att_return); 753 | if(!att_return) 754 | /*should not happen*/ 755 | ereport(ERROR,(errmsg("There are some wrong data in record."))); 756 | attr_old1 = CStringGetDatum(att_return); 757 | strPara_old =OutputToByte((text *)attr_old1, typeinfo->attrs[i]->attlen); 758 | } 759 | else 760 | { 761 | strPara_old = OutputToByte((text *)attr_old, typeinfo->attrs[i]->attlen); 762 | } 763 | } 764 | 765 | if(0 == count_value) 766 | { 767 | if(!attdroped) 768 | { 769 | appendtoSQL_doubquo(sql_ori, typeinfo->attrs[i]->attname.data, true); 770 | appendtoSQL_atttyptrans(sql_ori, typeinfo->attrs[i]->atttypid); 771 | } 772 | else 773 | { 774 | memset(temp_name,0,NAMEDATALEN + 3); 775 | sprintf(temp_name, "COL%d", i+1); 776 | appendtoSQL_doubquo(sql_ori, temp_name, true); 777 | } 778 | appendtoSQL(sql_ori, "=", PG_LOGMINER_SQLPARA_OTHER); 779 | appendtoSQL_simquo(sql_ori, strPara_old, quoset); 780 | appendtoSQL_valuetyptrans(sql_ori, typeinfo->attrs[i]->atttypid); 781 | getcondition = true; 782 | } 783 | else 784 | { 785 | appendtoSQL(sql_ori, " AND ", PG_LOGMINER_SQLPARA_OTHER); 786 | if(!attdroped) 787 | { 788 | appendtoSQL_doubquo(sql_ori, typeinfo->attrs[i]->attname.data, true); 789 | appendtoSQL_atttyptrans(sql_ori, typeinfo->attrs[i]->atttypid); 790 | } 791 | else 792 | { 793 | memset(temp_name,0,NAMEDATALEN + 3); 794 | sprintf(temp_name, "COL%d", i+1); 795 | appendtoSQL_doubquo(sql_ori, temp_name, true); 796 | } 797 | appendtoSQL(sql_ori, "=", PG_LOGMINER_SQLPARA_OTHER); 798 | appendtoSQL_simquo(sql_ori, strPara_old, quoset); 799 | appendtoSQL_valuetyptrans(sql_ori, typeinfo->attrs[i]->atttypid); 800 | getcondition = true; 801 | } 802 | count_value++; 803 | } 804 | if(undo) 805 | { 806 | htup_old = (HeapTupleHeader)rrctl.tuplem; 807 | ctid = (Datum)(&htup_old->t_ctid); 808 | ctid_str = DatumGetCString(DirectFunctionCall1(tidout, ctid)); 809 | if(getcondition) 810 | appendtoSQL(sql_ori, " AND ", PG_LOGMINER_SQLPARA_OTHER); 811 | appendtoSQL(sql_ori, "ctid = \'", PG_LOGMINER_SQLPARA_OTHER); 812 | appendtoSQL(sql_ori, ctid_str, PG_LOGMINER_SQLPARA_OTHER); 813 | appendtoSQL(sql_ori, "\';", PG_LOGMINER_SQLPARA_OTHER); 814 | } 815 | else 816 | appendtoSQL(sql_ori, ";", PG_LOGMINER_SQLPARA_OTHER); 817 | } 818 | 819 | void 820 | reAssembleDeleteSql(XLogMinerSQL *sql_ori, bool undo) 821 | { 822 | int natts = 0; 823 | int i = 0; 824 | Datum attr; 825 | Datum attr1; 826 | Datum ctid; 827 | char *strPara = NULL; 828 | char *ctid_str = NULL; 829 | char temp_name[NAMEDATALEN + 3]; 830 | Oid typoutput; 831 | Oid typoutputfromdb; 832 | Oid typoutputfromdic; 833 | bool typisvarlena = false; 834 | TupleDesc typeinfo = NULL; 835 | Datum *values = NULL; 836 | bool *nulls = NULL; 837 | int count_value = 0; 838 | bool quoset = false; 839 | bool attdroped = false; 840 | bool getcondition = false; 841 | bool gettype = false; 842 | struct varlena* att_return = NULL; 843 | HeapTupleHeader htup = NULL; 844 | 845 | if(!rrctl.values || !rrctl.nulls || !rrctl.tupdesc) 846 | { 847 | /*should not happen*/ 848 | ereport(ERROR,(errmsg("values or nulls or tupdesc is not exist"))); 849 | } 850 | values = rrctl.values; 851 | nulls = rrctl.nulls; 852 | typeinfo = rrctl.tupdesc; 853 | natts = typeinfo->natts; 854 | 855 | wipeSQLFromstr(sql_ori, "WHERE VALUES", "WHERE "); 856 | 857 | for (i = 0; i < natts; ++i) 858 | { 859 | attr = values[i]; 860 | if (nulls[i]) 861 | { 862 | continue; 863 | } 864 | 865 | if(0 == typeinfo->attrs[i]->atttypid) 866 | attdroped = true; 867 | else 868 | attdroped = false; 869 | 870 | 871 | if(!attdroped) 872 | { 873 | gettype = getTypeOutputFuncFromDb(typeinfo->attrs[i]->atttypid, 874 | &typoutputfromdb, &typisvarlena); 875 | gettype = gettype && getTypeOutputFuncFromDic(typeinfo->attrs[i]->atttypid, 876 | &typoutputfromdic, &typisvarlena); 877 | gettype = gettype && (typoutputfromdic == typoutputfromdb); 878 | gettype = gettype && (FirstNormalObjectId > typeinfo->attrs[i]->atttypid); 879 | } 880 | else 881 | gettype = false; 882 | 883 | if(!attdroped && gettype) 884 | { 885 | typoutput = typoutputfromdic; 886 | if(typisvarlena) 887 | { 888 | checkVarlena(attr,&att_return); 889 | if(!att_return) 890 | /*should not happen*/ 891 | ereport(ERROR,(errmsg("There are some wrong data in record."))); 892 | attr1 = CStringGetDatum(att_return); 893 | strPara = convertAttrToStr(typeinfo->attrs[i], typoutput, attr1); 894 | } 895 | else 896 | strPara = convertAttrToStr(typeinfo->attrs[i],typoutput, attr); 897 | quoset = ifquoneed(typeinfo->attrs[i]) && rrctl.nomalrel; 898 | } 899 | else 900 | { 901 | /*Attribute is droped,get data via byte*/ 902 | typisvarlena = (!typeinfo->attrs[i]->attbyval) && (-1 == typeinfo->attrs[i]->attlen); 903 | if(typisvarlena) 904 | { 905 | checkVarlena(attr,&att_return); 906 | if(!att_return) 907 | /*should not happen*/ 908 | ereport(ERROR,(errmsg("There are some wrong data in record."))); 909 | attr1 = CStringGetDatum(att_return); 910 | strPara =OutputToByte((text *)attr1, typeinfo->attrs[i]->attlen); 911 | } 912 | else 913 | strPara = OutputToByte((text *)attr, typeinfo->attrs[i]->attlen); 914 | quoset = false; 915 | } 916 | 917 | if(0 == count_value) 918 | { 919 | if(!attdroped) 920 | { 921 | appendtoSQL_doubquo(sql_ori, typeinfo->attrs[i]->attname.data, true); 922 | appendtoSQL_atttyptrans(sql_ori, typeinfo->attrs[i]->atttypid); 923 | } 924 | else 925 | { 926 | memset(temp_name,0,NAMEDATALEN + 3); 927 | sprintf(temp_name, "COL%d", i+1); 928 | appendtoSQL_doubquo(sql_ori, temp_name, true); 929 | } 930 | appendtoSQL(sql_ori, "=", PG_LOGMINER_SQLPARA_OTHER); 931 | appendtoSQL_simquo(sql_ori, strPara, quoset); 932 | appendtoSQL_valuetyptrans(sql_ori, typeinfo->attrs[i]->atttypid); 933 | getcondition = true; 934 | } 935 | else 936 | { 937 | appendtoSQL(sql_ori, " AND ", PG_LOGMINER_SQLPARA_OTHER); 938 | if(!attdroped) 939 | { 940 | appendtoSQL_doubquo(sql_ori, typeinfo->attrs[i]->attname.data, true); 941 | appendtoSQL_atttyptrans(sql_ori, typeinfo->attrs[i]->atttypid); 942 | } 943 | else 944 | { 945 | memset(temp_name,0,NAMEDATALEN + 3); 946 | sprintf(temp_name, "COL%d", i+1); 947 | appendtoSQL_doubquo(sql_ori, temp_name, true); 948 | } 949 | appendtoSQL(sql_ori, "=", PG_LOGMINER_SQLPARA_OTHER); 950 | appendtoSQL_simquo(sql_ori, strPara, quoset); 951 | appendtoSQL_valuetyptrans(sql_ori, typeinfo->attrs[i]->atttypid); 952 | getcondition = true; 953 | } 954 | count_value++; 955 | } 956 | /*append ctid condition*/ 957 | if(undo) 958 | { 959 | if(rrctl.tuplem_bigold) 960 | htup = (HeapTupleHeader)rrctl.tuplem_bigold; 961 | else 962 | htup = (HeapTupleHeader)rrctl.tuplem; 963 | ctid = (Datum)(&htup->t_ctid); 964 | ctid_str = DatumGetCString(DirectFunctionCall1(tidout, ctid)); 965 | if(getcondition) 966 | appendtoSQL(sql_ori, " AND ", PG_LOGMINER_SQLPARA_OTHER); 967 | appendtoSQL(sql_ori, "ctid = \'", PG_LOGMINER_SQLPARA_OTHER); 968 | appendtoSQL(sql_ori, ctid_str, PG_LOGMINER_SQLPARA_OTHER); 969 | appendtoSQL(sql_ori, "\';", PG_LOGMINER_SQLPARA_OTHER); 970 | } 971 | else 972 | appendtoSQL(sql_ori, ";", PG_LOGMINER_SQLPARA_OTHER); 973 | } 974 | 975 | -------------------------------------------------------------------------------- /xlogminer/logminer.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Abstract: 3 | * Some tool function for Xlogminer 4 | * 5 | * Authored by lichuancheng@highgo.com ,20170524 6 | * 7 | * Copyright: 8 | * Copyright (c) 2017-2020, HighGo Software Co.,Ltd. All right reserved 9 | * 10 | * Identification: 11 | * logminer.c 12 | */ 13 | #include "logminer.h" 14 | #include "datadictionary.h" 15 | #include "catalog/pg_type.h" 16 | #include "catalog/pg_constraint.h" 17 | #include "catalog/pg_language.h" 18 | #include "catalog/pg_namespace.h" 19 | #include "access/tupdesc.h" 20 | #include "access/attnum.h" 21 | #include "access/xlog_internal.h" 22 | #include "xlogminer_contents.h" 23 | #include "utils/bytea.h" 24 | #include "utils/builtins.h" 25 | #include "access/tuptoaster.h" 26 | 27 | 28 | 29 | RelationKind relkind[]={ 30 | {LOGMINER_SQLGET_DDL_CREATE_TABLE,LOGMINER_RELKINDID_TABLE,"TABLE",'r',true}, 31 | {LOGMINER_SQLGET_DDL_CREATE_INDEX,LOGMINER_RELKINDID_INDEX,"INDEX",'i',true}, 32 | {LOGMINER_SQLGET_DDL_CREATE_SEQUENCE,LOGMINER_RELKINDID_SEQUENCE,"SEQUENCE",'s',true}, 33 | {LOGMINER_SQLGET_DDL_CREATE_VIEW,LOGMINER_RELKINDID_VIEW,"VIEW",'v',true}, 34 | {LOGMINER_SQLGET_DDL_CREATE_TOAST,LOGMINER_RELKINDID_TOAST,"TOAST",'t',false}, 35 | {LOGMINER_SQLGET_DDL_CREATE_COMPLEX,LOGMINER_RELKINDID_COMPLEX,"COMPLEX",'c',true}, 36 | {LOGMINER_SQLGET_DDL_CREATE_TABLE,LOGMINER_RELKINDID_TABLE,"TABLE",'P',true} 37 | }; 38 | static bool separateJudge(char ch ,bool ignoresbrack); 39 | static bool passOver(char **sql, bool isspace,bool ignoresbrack); 40 | static int countCharInString(char* str, char ch); 41 | char* addSinglequoteFromStr(char* strPara); 42 | 43 | 44 | char* logminer_palloc(int size,int checkflag) 45 | { 46 | void *result = NULL; 47 | result = (char*)palloc0(size); 48 | return result; 49 | } 50 | 51 | void logminer_pfree(char* ptr,int checkflag) 52 | { 53 | if(ptr) 54 | pfree(ptr); 55 | } 56 | 57 | char* logminer_malloc(int size,int checkflag) 58 | { 59 | char *result = NULL; 60 | result = (char*)malloc(size); 61 | memset(result,0,size); 62 | return result; 63 | } 64 | 65 | void logminer_free(char* ptr,int checkflag) 66 | { 67 | if(ptr) 68 | { 69 | free(ptr); 70 | } 71 | } 72 | 73 | 74 | 75 | static bool 76 | separateJudge(char ch ,bool ignoresbrack) 77 | { 78 | if(!ignoresbrack) 79 | { 80 | if(PG_LOGMINER_SPACE == ch 81 | || PG_LOGMINER_SBRACK_L == ch 82 | || PG_LOGMINER_SBRACK_R == ch 83 | || PG_LOGMINER_COMMA == ch 84 | ) 85 | return true; 86 | } 87 | else 88 | { 89 | if(PG_LOGMINER_SPACE == ch 90 | || PG_LOGMINER_COMMA == ch 91 | ) 92 | return true; 93 | } 94 | return false; 95 | } 96 | 97 | static bool 98 | passOver(char **sql, bool isspace,bool ignoresbrack) 99 | { 100 | int length = 0; 101 | int recCount = 1; 102 | int entflagnum = 0; 103 | if(NULL == *sql) 104 | return false; 105 | length = strlen(*sql); 106 | if(isspace) 107 | { 108 | if(!separateJudge(**sql,ignoresbrack)) 109 | return true; 110 | while(separateJudge(**sql,ignoresbrack)) 111 | { 112 | (*sql)++; 113 | recCount++; 114 | if(recCount > length) 115 | return false; 116 | } 117 | } 118 | else 119 | { 120 | if(separateJudge(**sql,ignoresbrack)) 121 | return true; 122 | if(bbl_Judge(**sql)) 123 | { 124 | entflagnum++; 125 | while(0 < entflagnum) 126 | { 127 | (*sql)++; 128 | if(bbl_Judge(**sql)) 129 | entflagnum++; 130 | else if(bbr_Judge(**sql)) 131 | entflagnum--; 132 | 133 | recCount++; 134 | if(recCount > length) 135 | return false; 136 | } 137 | (*sql)++; 138 | } 139 | else if(sbl_Judge(**sql)) 140 | { 141 | entflagnum++; 142 | while(0 < entflagnum) 143 | { 144 | (*sql)++; 145 | if(sbl_Judge(**sql)) 146 | entflagnum++; 147 | else if(sbr_Judge(**sql)) 148 | entflagnum--; 149 | 150 | recCount++; 151 | if(recCount > length) 152 | return false; 153 | } 154 | (*sql)++; 155 | } 156 | else 157 | { 158 | while(!separateJudge(**sql,ignoresbrack)) 159 | { 160 | (*sql)++; 161 | recCount++; 162 | if(recCount > length) 163 | return false; 164 | } 165 | } 166 | } 167 | return true; 168 | } 169 | 170 | void 171 | fixPathEnd(char *path) 172 | { 173 | int pathLength = 0; 174 | pathLength = strlen(path); 175 | if('/' == path[pathLength - 1]) 176 | path[pathLength - 1] = 0; 177 | } 178 | 179 | bool 180 | isEmptStr(char *str) 181 | { 182 | bool result = false; 183 | char *strptr; 184 | if(!str) 185 | { 186 | result = true; 187 | return result; 188 | } 189 | strptr = str; 190 | 191 | passOver(&strptr,true,false); 192 | if(0 == strcmp("NULL",strptr)) 193 | result = true; 194 | else if(0 == strcmp("null",strptr)) 195 | result = true; 196 | else if(0 == strcmp("\"\"",strptr)) 197 | result = true; 198 | else if(0 == strcmp("",strptr)) 199 | result = true; 200 | return result; 201 | } 202 | 203 | bool 204 | getPhrases(char *sql,int loc, char *term, int ignoresbrackph) 205 | { 206 | char *sql_ptr = NULL; 207 | char *ptr_term_start = NULL; 208 | char *ptr_term_end = NULL; 209 | bool result = true; 210 | int termLength = 0; 211 | int loop; 212 | sql_ptr = sql; 213 | if(!passOver(&sql_ptr,true,false)) 214 | return false; 215 | for(loop = 1; loop < loc; loop++) 216 | { 217 | if(!passOver(&sql_ptr,false,ignoresbrackph == loop)) 218 | return false; 219 | if(!passOver(&sql_ptr,true,(ignoresbrackph == loop + 1))) 220 | return false; 221 | } 222 | ptr_term_start = sql_ptr; 223 | passOver(&sql_ptr,false,ignoresbrackph == loop); 224 | ptr_term_end = sql_ptr; 225 | termLength = ptr_term_end - ptr_term_start; 226 | 227 | memset(term, 0, NAMEDATALEN); 228 | memcpy(term,ptr_term_start,termLength); 229 | 230 | return result; 231 | } 232 | 233 | void 234 | addSpace(XLogMinerSQL *sql_simple, int spaceKind) 235 | { 236 | char *temp = NULL; 237 | int addstep = 0; 238 | 239 | if(!sql_simple) 240 | return; 241 | 242 | if(PG_LOGMINER_SQLPARA_TOTLE == spaceKind) 243 | addstep = PG_LOGMINER_SQLPARA_TOTSTEP; 244 | else 245 | addstep = PG_LOGMINER_SQLPARA_SIMSTEP; 246 | 247 | temp = logminer_palloc(sql_simple->tot_size + addstep,0); 248 | memset(temp,0,sql_simple->tot_size + addstep); 249 | if(sql_simple->sqlStr) 250 | { 251 | memcpy(temp,sql_simple->sqlStr,sql_simple->tot_size); 252 | logminer_pfree(sql_simple->sqlStr,0); 253 | sql_simple->sqlStr = NULL; 254 | } 255 | sql_simple->tot_size += addstep; 256 | sql_simple->rem_size += addstep; 257 | sql_simple->sqlStr = temp; 258 | 259 | } 260 | 261 | void 262 | cleanSpace(XLogMinerSQL *minersql) 263 | { 264 | if(!minersql) 265 | return; 266 | if(!minersql->sqlStr) 267 | return; 268 | memset(minersql->sqlStr, 0, minersql->tot_size); 269 | minersql->rem_size = minersql->tot_size; 270 | minersql->use_size = 0; 271 | } 272 | 273 | void 274 | freeSpace(XLogMinerSQL *minersql) 275 | { 276 | if(!minersql) 277 | return; 278 | if(!minersql->sqlStr) 279 | return; 280 | 281 | logminer_pfree(minersql->sqlStr,0); 282 | memset(minersql , 0, sizeof(XLogMinerSQL)); 283 | } 284 | 285 | 286 | 287 | void 288 | cleanMentalvalues() 289 | { 290 | if(rrctl.values) 291 | logminer_pfree((char *)rrctl.values,0); 292 | if(rrctl.nulls) 293 | logminer_pfree((char *)rrctl.nulls,0); 294 | if(rrctl.values_old) 295 | logminer_pfree((char *)rrctl.values_old,0); 296 | if(rrctl.nulls_old) 297 | logminer_pfree((char *)rrctl.nulls_old,0); 298 | rrctl.values = NULL; 299 | rrctl.nulls = NULL; 300 | rrctl.values_old = NULL; 301 | rrctl.nulls_old = NULL; 302 | } 303 | 304 | bool 305 | elemNameFind(char* elenname) 306 | { 307 | if(!elenname) 308 | return false; 309 | if(isEmptStr(elenname)) 310 | return false; 311 | return true; 312 | } 313 | 314 | void 315 | split_path_fname(const char *path, char **dir, char **fname) 316 | { 317 | char *sep = NULL; 318 | int length_dir = 0; 319 | int length_fname = 0; 320 | 321 | 322 | /* split filepath into directory & filename */ 323 | #ifdef WIN32 324 | sep = strrchr(path, '\\'); 325 | if(NULL == sep) 326 | sep = strrchr(path, '/'); 327 | #else 328 | sep = strrchr(path, '/'); 329 | #endif 330 | 331 | /* directory path */ 332 | if (sep != NULL) 333 | { 334 | length_dir = sep - path; 335 | length_fname = strlen(sep + 1); 336 | 337 | *dir = logminer_palloc(length_dir + 1,0); 338 | memcpy(*dir, path, length_dir); 339 | *fname = logminer_palloc(length_fname + 1,0); 340 | memcpy(*fname, sep + 1, length_fname); 341 | 342 | } 343 | /* local directory */ 344 | else 345 | { 346 | length_dir = strlen(path); 347 | *fname = logminer_palloc(length_dir + 1,0); 348 | *dir = NULL; 349 | memcpy(*fname, path, length_dir); 350 | } 351 | } 352 | 353 | RelationKind* 354 | getRelKindInfo() 355 | { 356 | return relkind; 357 | } 358 | 359 | 360 | int 361 | xlog_file_open(const char *directory, const char *fname) 362 | { 363 | int fd = -1; 364 | char fpath[MAXPGPATH]; 365 | 366 | if (directory == NULL) 367 | { 368 | const char *datadir; 369 | 370 | /* fname */ 371 | fd = open(fname, O_RDONLY | PG_BINARY, 0); 372 | if (fd < 0 && errno != ENOENT) 373 | return -1; 374 | else if (fd >= 0) 375 | return fd; 376 | 377 | /* XLOGDIR / fname */ 378 | snprintf(fpath, MAXPGPATH, "%s/%s", 379 | XLOGDIR, fname); 380 | fd = open(fpath, O_RDONLY | PG_BINARY, 0); 381 | if (fd < 0 && errno != ENOENT) 382 | return -1; 383 | else if (fd >= 0) 384 | return fd; 385 | 386 | datadir = DataDir; 387 | /* $PGDATA / XLOGDIR / fname */ 388 | if (datadir != NULL) 389 | { 390 | snprintf(fpath, MAXPGPATH, "%s/%s/%s", 391 | datadir, XLOGDIR, fname); 392 | fd = open(fpath, O_RDONLY | PG_BINARY, 0); 393 | if (fd < 0 && errno != ENOENT) 394 | return -1; 395 | else if (fd >= 0) 396 | return fd; 397 | } 398 | } 399 | else 400 | { 401 | /* directory / fname */ 402 | snprintf(fpath, MAXPGPATH, "%s/%s", 403 | directory, fname); 404 | fd = open(fpath, O_RDONLY | PG_BINARY, 0); 405 | if (fd < 0 && errno != ENOENT) 406 | return -1; 407 | else if (fd >= 0) 408 | return fd; 409 | 410 | /* directory / XLOGDIR / fname */ 411 | snprintf(fpath, MAXPGPATH, "%s/%s/%s", 412 | directory, XLOGDIR, fname); 413 | fd = open(fpath, O_RDONLY | PG_BINARY, 0); 414 | if (fd < 0 && errno != ENOENT) 415 | return -1; 416 | else if (fd >= 0) 417 | return fd; 418 | } 419 | return -1; 420 | } 421 | 422 | TupleDesc 423 | makeOutputXlogDesc() 424 | { 425 | TupleDesc tupdesc = NULL; 426 | tupdesc = CreateTemplateTupleDesc(1, false); 427 | TupleDescInitEntry(tupdesc, (AttrNumber) 1, "path", TEXTOID, -1, 0); 428 | return tupdesc; 429 | } 430 | 431 | int 432 | strcmp_withlength(char *str1,char *str2,int length) 433 | { 434 | int loop = 0; 435 | for(loop = 0; loop < length; loop++) 436 | { 437 | if(str1[loop] != str2[loop]) 438 | { 439 | return -1; 440 | } 441 | } 442 | return 0; 443 | } 444 | 445 | bool 446 | inputParaCheck(text *st, text *et) 447 | { 448 | char *starttimestamp = NULL, *endtimestamp = NULL; 449 | 450 | if(!st || !et) 451 | return false; 452 | 453 | starttimestamp = text_to_cstring(st); 454 | endtimestamp = text_to_cstring(et); 455 | 456 | /*get begin timestamp and end timestamp. if it din not been limited, will be 0*/ 457 | if(isEmptStr(starttimestamp)) 458 | rrctl.logprivate.parser_start_time = 0; 459 | else 460 | rrctl.logprivate.parser_start_time = DatumGetTimestampTz(DirectFunctionCall3(timestamptz_in, 461 | CStringGetDatum(starttimestamp), 462 | ObjectIdGetDatum(InvalidOid), 463 | Int32GetDatum(-1))); 464 | if(isEmptStr(endtimestamp)) 465 | rrctl.logprivate.parser_end_time = 0; 466 | else 467 | rrctl.logprivate.parser_end_time = DatumGetTimestampTz(DirectFunctionCall3(timestamptz_in, 468 | CStringGetDatum(endtimestamp), 469 | ObjectIdGetDatum(InvalidOid), 470 | Int32GetDatum(-1))); 471 | 472 | /*there must be less than one group valid.*/ 473 | if((0 != rrctl.logprivate.parser_start_time || 0 != rrctl.logprivate.parser_end_time) 474 | &&(0 != rrctl.logprivate.parser_start_xid || 0 != rrctl.logprivate.parser_start_xid)) 475 | ereport(ERROR,(errmsg("Time parameters and XID parameters cannot be provided at the same time."))); 476 | 477 | if(0 != rrctl.logprivate.parser_end_time && rrctl.logprivate.parser_start_time > rrctl.logprivate.parser_end_time) 478 | ereport(ERROR,(errmsg("Start time is greater than end time."))); 479 | if(0 != rrctl.logprivate.parser_end_xid && rrctl.logprivate.parser_start_xid > rrctl.logprivate.parser_end_xid) 480 | ereport(ERROR,(errmsg("Start xid is greater than end xid."))); 481 | 482 | if(0 != rrctl.logprivate.parser_start_time || 0 != rrctl.logprivate.parser_end_time) 483 | rrctl.logprivate.timecheck = true; 484 | if(0 != rrctl.logprivate.parser_start_xid || 0 != rrctl.logprivate.parser_end_xid) 485 | rrctl.logprivate.xidcheck = true; 486 | 487 | if(rrctl.logprivate.timecheck || rrctl.logprivate.xidcheck) 488 | rrctl.logprivate.staptr_reached = false; 489 | else 490 | rrctl.logprivate.staptr_reached = true; 491 | rrctl.logprivate.endptr_reached = false; 492 | rrctl.logprivate.timeline = 1; 493 | rrctl.logprivate.startptr = InvalidXLogRecPtr; 494 | rrctl.logprivate.analynum = getXlogFileNum(); 495 | return true; 496 | } 497 | 498 | /* 499 | When it occurs a commit,we must check if we have some sql to insert into temp table. 500 | */ 501 | bool 502 | curXactCheck(TimestampTz xact_time ,TransactionId xid, bool xactcommit,xl_xact_parsed_commit *parsed_commit) 503 | { 504 | bool result = true; 505 | FormData_xlogminer_contents fxc; 506 | XlogminerContentsFirst *xcf = NULL; 507 | 508 | memset(&fxc,0,sizeof(FormData_xlogminer_contents)); 509 | 510 | if(rrctl.logprivate.timecheck) 511 | { 512 | if(0 != rrctl.logprivate.parser_start_time && xact_time < rrctl.logprivate.parser_start_time) 513 | result = false; 514 | if(0 != rrctl.logprivate.parser_end_time && xact_time > rrctl.logprivate.parser_end_time) 515 | { 516 | rrctl.logprivate.endptr_reached = true; 517 | result = false; 518 | } 519 | } 520 | if((rrctl.logprivate.xidcheck)) 521 | { 522 | if(0 != rrctl.logprivate.parser_start_xid && xid < rrctl.logprivate.parser_start_xid) 523 | result = false; 524 | if(0 != rrctl.logprivate.parser_end_xid && xid > rrctl.logprivate.parser_end_xid) 525 | 526 | { 527 | rrctl.logprivate.endptr_reached = true; 528 | result = false; 529 | } 530 | } 531 | 532 | /* 533 | result is false,means that it did not reach valid record(input limit). 534 | then store the point for a while,just in case,valid next record. 535 | */ 536 | if(!result) 537 | { 538 | rrctl.logprivate.limitstartptr = rrctl.xlogreader_state->ReadRecPtr; 539 | rrctl.logprivate.limitendptr = rrctl.xlogreader_state->EndRecPtr; 540 | } 541 | 542 | if(result && !rrctl.logprivate.staptr_reached) 543 | { 544 | /*first reached the valid(input valid) xlog tuple*/ 545 | rrctl.xlogreader_state->ReadRecPtr = rrctl.logprivate.limitstartptr; 546 | rrctl.xlogreader_state->EndRecPtr = rrctl.logprivate.limitendptr; 547 | rrctl.logprivate.staptr_reached = true; 548 | 549 | } 550 | else if(!xactcommit) 551 | { 552 | /*abord xact,do not show the info,then clean the space*/ 553 | cleanSQLspace(); 554 | } 555 | else if(result) 556 | { 557 | /*reach the nomal valid(input limit) xlog tuple*/ 558 | int loop = 0; 559 | 560 | /*fxc.xid = xid;*/ 561 | fxc.timestamp = xact_time; 562 | fxc.record_database = NULL; 563 | fxc.record_schema = NULL; 564 | fxc.record_tablespace = NULL; 565 | fxc.record_user = NULL; 566 | for(loop = 0; loop < srctl.xcfcurnum; loop++) 567 | { 568 | xcf = (XlogminerContentsFirst *)srctl.xcf; 569 | if(0 == xcf[loop].xid) 570 | fxc.xid = xid; 571 | else if(xid != xcf[loop].xid) 572 | continue; 573 | 574 | fxc.op_text = xcf[loop].op_text.sqlStr; 575 | fxc.op_type = xcf[loop].op_type.sqlStr; 576 | fxc.op_undo = xcf[loop].op_undo.sqlStr; 577 | fxc.virtualxid = loop + 1; 578 | fxc.sqlno = ++sqlnoser; 579 | fxc.record_database = xcf[loop].record_database.sqlStr; 580 | fxc.record_user = xcf[loop].record_user.sqlStr; 581 | fxc.record_tablespace = xcf[loop].record_tablespace.sqlStr; 582 | fxc.record_schema = xcf[loop].record_schema.sqlStr; 583 | fxc.xid = xid; 584 | 585 | 586 | InsertXlogContentsTuple(&fxc); 587 | } 588 | cleanSQLspace(); 589 | } 590 | return result; 591 | } 592 | 593 | void 594 | logminer_createMemContext() 595 | { 596 | rrctl.mycxt = AllocSetContextCreate(CurrentMemoryContext, 597 | "logminer_main", 598 | ALLOCSET_DEFAULT_MINSIZE, 599 | ALLOCSET_DEFAULT_INITSIZE, 600 | ALLOCSET_DEFAULT_MAXSIZE); 601 | rrctl.oldcxt= MemoryContextSwitchTo(rrctl.mycxt); 602 | } 603 | 604 | void 605 | logminer_switchMemContext() 606 | { 607 | if(!rrctl.mycxt || !rrctl.oldcxt) 608 | return; 609 | MemoryContextSwitchTo(rrctl.oldcxt); 610 | MemoryContextDelete(rrctl.mycxt); 611 | } 612 | 613 | bool 614 | checkLogminerUser() 615 | { 616 | bool result = false; 617 | result = superuser(); 618 | if(!result) 619 | ereport(ERROR,(errmsg("Only the superuser execute xlogminer."))); 620 | return result; 621 | } 622 | 623 | bool 624 | padingminerXlogconts(char* elemname, TransactionId xid,int loc, long elemoid) 625 | { 626 | XlogminerContentsFirst *xcf = NULL; 627 | XLogMinerSQL *temp_sql = NULL; 628 | char tempName[NAMEDATALEN] = {0}; 629 | char *appPter = NULL; 630 | 631 | if(srctl.xcfcurnum == srctl.xcftotnum) 632 | addSQLspace(); 633 | 634 | xcf = (XlogminerContentsFirst*)srctl.xcf; 635 | 636 | if(!elemname && -1 != elemoid) 637 | { 638 | if(0 < elemoid) 639 | { 640 | sprintf(tempName,"(%ld)",elemoid); 641 | } 642 | else 643 | { 644 | sprintf(tempName,"(missing data)"); 645 | } 646 | appPter = tempName; 647 | } 648 | else 649 | appPter = elemname; 650 | 651 | 652 | 653 | if(Anum_xlogminer_contents_xid == loc) 654 | { 655 | xcf[srctl.xcfcurnum].xid = xid; 656 | } 657 | else if(Anum_xlogminer_contents_record_database == loc) 658 | { 659 | temp_sql = &xcf[srctl.xcfcurnum].record_database; 660 | cleanSpace(temp_sql); 661 | appendtoSQL(temp_sql, appPter, PG_LOGMINER_SQLPARA_OTHER); 662 | } 663 | else if(Anum_xlogminer_contents_record_user == loc) 664 | { 665 | temp_sql = &xcf[srctl.xcfcurnum].record_user; 666 | cleanSpace(temp_sql); 667 | 668 | appendtoSQL(temp_sql, appPter, PG_LOGMINER_SQLPARA_OTHER); 669 | } 670 | else if(Anum_xlogminer_contents_record_tablespace == loc) 671 | { 672 | temp_sql = &xcf[srctl.xcfcurnum].record_tablespace; 673 | cleanSpace(temp_sql); 674 | appendtoSQL(temp_sql, appPter, PG_LOGMINER_SQLPARA_OTHER); 675 | } 676 | else if(Anum_xlogminer_contents_record_schema == loc) 677 | { 678 | temp_sql = &xcf[srctl.xcfcurnum].record_schema; 679 | cleanSpace(temp_sql); 680 | appendtoSQL(temp_sql, appPter, PG_LOGMINER_SQLPARA_OTHER); 681 | } 682 | else if(Anum_xlogminer_contents_op_type == loc) 683 | { 684 | temp_sql = &xcf[srctl.xcfcurnum].op_type; 685 | cleanSpace(temp_sql); 686 | appendtoSQL(temp_sql, appPter, PG_LOGMINER_SQLPARA_OTHER); 687 | } 688 | else if(Anum_xlogminer_contents_op_text == loc) 689 | { 690 | temp_sql = &xcf[srctl.xcfcurnum].op_text; 691 | cleanSpace(temp_sql); 692 | appendtoSQL(temp_sql, appPter, PG_LOGMINER_SQLPARA_OTHER); 693 | } 694 | else if(Anum_xlogminer_contents_op_undo == loc) 695 | { 696 | temp_sql = &xcf[srctl.xcfcurnum].op_undo; 697 | cleanSpace(temp_sql); 698 | appendtoSQL(temp_sql, appPter, PG_LOGMINER_SQLPARA_OTHER); 699 | } 700 | 701 | return true; 702 | } 703 | 704 | 705 | void 706 | padNullToXC() 707 | { 708 | /*for padding null while did not get any info*/ 709 | padingminerXlogconts(NULL, 0, Anum_xlogminer_contents_xid, -1); 710 | padingminerXlogconts(NULL, 0, Anum_xlogminer_contents_record_database, -1); 711 | padingminerXlogconts(NULL, 0, Anum_xlogminer_contents_record_user,-1); 712 | padingminerXlogconts(NULL, 0, Anum_xlogminer_contents_record_tablespace, -1); 713 | padingminerXlogconts(NULL, 0, Anum_xlogminer_contents_record_schema, -1); 714 | padingminerXlogconts(NULL, 0, Anum_xlogminer_contents_op_type, -1); 715 | padingminerXlogconts(NULL, 0, Anum_xlogminer_contents_op_text, -1); 716 | padingminerXlogconts(NULL, 0, Anum_xlogminer_contents_op_undo, -1); 717 | } 718 | 719 | 720 | void 721 | cleanAnalyseInfo() 722 | { 723 | rrctl.reloid = 0; 724 | rrctl.tbsoid = 0; 725 | rrctl.recordxid = 0; 726 | memset(&srctl.rfnode, 0, sizeof(RelFileNode)); 727 | srctl.toastoid = 0; 728 | cleanSpace(&srctl.sql_undo); 729 | } 730 | 731 | char* 732 | getTuplemSpace(int addsize) 733 | { 734 | int size = 0; 735 | char* result = NULL; 736 | if(0 == addsize) 737 | size = MaxHeapTupleSize + SizeofHeapTupleHeader; 738 | else 739 | size = addsize; 740 | result = logminer_palloc(size, 0); 741 | if(NULL == result) 742 | ereport(ERROR,(errmsg("Out of memory."))); 743 | return result; 744 | } 745 | 746 | 747 | void 748 | cleanTuplemSpace(char* tuplem) 749 | { 750 | int size = 0; 751 | size = MaxHeapTupleSize + SizeofHeapTupleHeader; 752 | memset(tuplem,0,size); 753 | } 754 | 755 | bool 756 | ifquoneed(Form_pg_attribute attrs) 757 | { 758 | Oid quoNeedArray[] = {BPCHAROID,VARCHAROID,TEXTOID,BYTEAOID,BOXOID,CASHOID,TSVECTOROID 759 | ,TSQUERYOID,TIMESTAMPOID,TIMESTAMPTZOID,TIMEOID,TIMETZOID,DATEOID,INTERVALOID,BOOLOID 760 | ,BYTEAOID,POINTOID,CIRCLEOID,CIDROID,INETOID,MACADDROID,BITOID,VARBITOID,UUIDOID,XMLOID 761 | ,JSONBOID,3908,3910,LSEGOID,PATHOID,NAMEOID,POLYGONOID,JSONOID,LINEOID,-1}; 762 | int arrNum = 35; 763 | int loop = 0; 764 | 765 | if(0 < attrs->attndims) 766 | return true; 767 | 768 | for(loop = 0; loop < arrNum; loop++) 769 | { 770 | if(attrs->atttypid == quoNeedArray[loop]) 771 | return true; 772 | } 773 | return false; 774 | 775 | } 776 | 777 | bool 778 | ifQueNeedDelete(Form_pg_attribute attrs) 779 | { 780 | Oid quoNeedDeleteArray[] = {TSVECTOROID,TSQUERYOID}; 781 | int arrNum = 2; 782 | int loop = 0; 783 | 784 | for(loop = 0; loop < arrNum; loop++) 785 | { 786 | if(attrs->atttypid == quoNeedDeleteArray[loop]) 787 | return true; 788 | } 789 | 790 | return false; 791 | } 792 | 793 | ToastTuple* 794 | makeToastTuple(int datalength,char* data, Oid id, int seq) 795 | { 796 | char* ptr = NULL; 797 | ToastTuple* result = NULL; 798 | ptr = logminer_palloc(datalength + sizeof(ToastTuple), 0); 799 | result = (ToastTuple*)ptr; 800 | result->chunk_data = ptr + sizeof(ToastTuple); 801 | 802 | result->chunk_id = id; 803 | result->chunk_seq = seq; 804 | result->datalength = datalength; 805 | result->next = NULL; 806 | memcpy(result->chunk_data, data, datalength); 807 | 808 | return result; 809 | } 810 | 811 | void 812 | freeToastTupleHead() 813 | { 814 | ToastTuple *ttptr = NULL, *ttnext = NULL; 815 | 816 | ttptr = rrctl.tthead; 817 | while(ttptr) 818 | { 819 | ttnext = ttptr->next; 820 | logminer_pfree((char*)ttptr,0); 821 | ttptr = ttnext; 822 | } 823 | rrctl.tthead = NULL; 824 | } 825 | 826 | void 827 | toastTupleAddToList(ToastTuple *tt) 828 | { 829 | ToastTuple *ttptr = NULL; 830 | 831 | if(!rrctl.tthead) 832 | { 833 | rrctl.tthead = tt; 834 | } 835 | else 836 | { 837 | ttptr = rrctl.tthead; 838 | while(ttptr->next) 839 | ttptr = ttptr->next; 840 | ttptr->next = tt; 841 | } 842 | } 843 | 844 | void 845 | checkVarlena(Datum attr,struct varlena** att_return) 846 | { 847 | text *attr_text = NULL; 848 | ToastTuple *ttptr = NULL; 849 | struct varlena *attr_varlena = NULL; 850 | struct varlena *result; 851 | struct varatt_external toast_pointer; 852 | int32 ressize = 0; 853 | 854 | attr_text = (text*)DatumGetPointer(attr); 855 | attr_varlena = (struct varlena *)attr_text; 856 | 857 | if(!VARATT_IS_EXTERNAL_ONDISK(attr_varlena)) 858 | { 859 | *att_return = (struct varlena *)attr; 860 | return; 861 | } 862 | 863 | VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); 864 | ressize = toast_pointer.va_extsize; 865 | result = (struct varlena *) palloc(ressize + VARHDRSZ); 866 | 867 | if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)) 868 | SET_VARSIZE_COMPRESSED(result, ressize + VARHDRSZ); 869 | else 870 | SET_VARSIZE(result, ressize + VARHDRSZ); 871 | 872 | ttptr = rrctl.tthead; 873 | while(ttptr) 874 | { 875 | if(ttptr->chunk_id == toast_pointer.va_valueid) 876 | { 877 | memcpy(VARDATA(result) + ttptr->chunk_seq * TOAST_MAX_CHUNK_SIZE, ttptr->chunk_data, ttptr->datalength); 878 | } 879 | ttptr = ttptr->next; 880 | } 881 | if (VARATT_IS_COMPRESSED(result)) 882 | { 883 | struct varlena *tmp = result; 884 | result = heap_tuple_untoast_attr(tmp); 885 | pfree(tmp); 886 | } 887 | *att_return = result; 888 | } 889 | 890 | text * 891 | cstringToTextWithLen(const char *s, int len) 892 | { 893 | text *result = (text *) palloc(len + VARHDRSZ); 894 | 895 | SET_VARSIZE(result, len + VARHDRSZ); 896 | memcpy(VARDATA(result), s, len); 897 | 898 | return result; 899 | } 900 | 901 | void 902 | deleteQueFromStr(char* strPara) 903 | { 904 | int strlength = 0,loopo = 0,loopt = 0; 905 | char* strtemp = NULL; 906 | 907 | if(!strPara) 908 | return; 909 | strlength = strlen(strPara); 910 | strtemp = (char*)palloc0(strlength + 1); 911 | if(!strtemp) 912 | ereport(ERROR,(errmsg("Out of memory during deleteQueFromStr"))); 913 | 914 | while(loopo != strlength) 915 | { 916 | if((('\'' == strPara[loopo]) && (0 == loopo)) 917 | || (('\'' == strPara[loopo]) && (strlength - 1 == loopo)) 918 | || (('\'' == strPara[loopo]) && (strlength - 1 != loopo && (0 != loopo)) && (' ' == strPara[loopo - 1] || ' ' == strPara[loopo + 1]))) 919 | { 920 | loopo++; 921 | continue; 922 | } 923 | else 924 | { 925 | strtemp[loopt++] = strPara[loopo++]; 926 | } 927 | } 928 | memset(strPara, 0, strlength); 929 | memcpy(strPara, strtemp, strlen(strtemp)); 930 | pfree(strtemp); 931 | } 932 | 933 | void 934 | keepDigitFromStr(char* strPara) 935 | { 936 | int strlength = 0,loopo = 0,loopt = 0; 937 | char* strtemp = NULL; 938 | 939 | if(!strPara) 940 | return; 941 | strlength = strlen(strPara); 942 | strtemp = (char*)palloc0(strlength + 1); 943 | if(!strtemp) 944 | ereport(ERROR,(errmsg("Out of memory during keepDigitFromStr"))); 945 | 946 | while(loopo != strlength) 947 | { 948 | if(('0' <= strPara[loopo] && '9' >= strPara[loopo]) || '.' == strPara[loopo]) 949 | { 950 | strtemp[loopt++] = strPara[loopo++]; 951 | } 952 | else 953 | { 954 | loopo++; 955 | continue; 956 | } 957 | } 958 | memset(strPara, 0, strlength); 959 | memcpy(strPara, strtemp, strlen(strtemp)); 960 | pfree(strtemp); 961 | } 962 | 963 | static int 964 | countCharInString(char* str, char ch) 965 | { 966 | char *strPtr = NULL; 967 | int result = 0; 968 | int strlength = 0; 969 | int loop = 0; 970 | 971 | if(!str) 972 | return result; 973 | 974 | strlength = strlen(str); 975 | strPtr = str; 976 | for(;loop < strlength;loop++) 977 | { 978 | if(*strPtr == ch) 979 | result++; 980 | strPtr++; 981 | } 982 | return result; 983 | } 984 | 985 | 986 | char* 987 | addSinglequoteFromStr(char* strPara) 988 | { 989 | int strlength = 0,loopo = 0,loopt = 0; 990 | char* strtemp = NULL; 991 | int simplequenum = 0; 992 | 993 | if(!strPara) 994 | return NULL; 995 | 996 | simplequenum = countCharInString(strPara,'\''); 997 | if(0 >= simplequenum) 998 | return strPara; 999 | 1000 | strlength = strlen(strPara); 1001 | strtemp = (char*)palloc0(strlength + simplequenum + 1); 1002 | if(!strtemp) 1003 | ereport(ERROR,(errmsg("Out of memory during addSinglequoteFromStr"))); 1004 | 1005 | while(loopo != strlength) 1006 | { 1007 | if('\'' == strPara[loopo]) 1008 | { 1009 | strtemp[loopt++] = '\''; 1010 | } 1011 | strtemp[loopt++] = strPara[loopo++]; 1012 | } 1013 | return strtemp; 1014 | } 1015 | 1016 | 1017 | char* 1018 | convertAttrToStr(Form_pg_attribute fpa,Oid typoutput, Datum attr) 1019 | { 1020 | char *resultstr = NULL; 1021 | resultstr = OidOutputFunctionCall(typoutput, attr); 1022 | 1023 | 1024 | if(ifQueNeedDelete(fpa)) 1025 | { 1026 | deleteQueFromStr(resultstr); 1027 | } 1028 | if(CASHOID == fpa->atttypid) 1029 | { 1030 | keepDigitFromStr(resultstr); 1031 | } 1032 | else if(JSONOID == fpa->atttypid || TEXTOID == fpa->atttypid || BPCHAROID == fpa->atttypid || VARCHAROID == fpa->atttypid 1033 | || XMLOID == fpa->atttypid || NAMEOID == fpa->atttypid || JSONBOID == fpa->atttypid || CHAROID == fpa->atttypid 1034 | || 199 == fpa->atttypid || TEXTARRAYOID == fpa->atttypid || 1014 == fpa->atttypid || 1015 == fpa->atttypid 1035 | || 143 == fpa->atttypid || 1003 == fpa->atttypid || 3807 == fpa->atttypid || 1002 == fpa->atttypid) 1036 | { 1037 | resultstr = addSinglequoteFromStr(resultstr); 1038 | } 1039 | return resultstr; 1040 | } 1041 | 1042 | 1043 | char* 1044 | OutputToByte(text* attrpter, int attlen) 1045 | { 1046 | 1047 | int len = 0; 1048 | char *attstr = NULL; 1049 | bytea *str_byte = NULL; 1050 | char *str = NULL; 1051 | Datum str_datum; 1052 | XLogMinerSQL result; 1053 | 1054 | memset(&result,0,sizeof(XLogMinerSQL)); 1055 | 1056 | 1057 | if(0 < attlen) 1058 | { 1059 | len = attlen; 1060 | attstr = (char*)(&attrpter); 1061 | } 1062 | else if(-1 == attlen) 1063 | { 1064 | len = VARSIZE_ANY(attrpter); 1065 | attstr = VARDATA_ANY(attrpter); 1066 | } 1067 | else if(-2 == attlen) 1068 | { 1069 | len = (strlen((char *) (attrpter)) + 1); 1070 | attstr = (char *)attrpter; 1071 | } 1072 | 1073 | if(-1 ==attlen) 1074 | { 1075 | str = DatumGetCString(DirectFunctionCall1(byteaout, (Datum)attrpter)); 1076 | } 1077 | else 1078 | { 1079 | str_byte = cstringToTextWithLen(attstr, len); 1080 | str_datum = PointerGetDatum(str_byte); 1081 | str = DatumGetCString(DirectFunctionCall1(byteaout, str_datum)); 1082 | } 1083 | appendtoSQL(&result, "encode(\'", PG_LOGMINER_SQLPARA_OTHER); 1084 | appendtoSQL(&result, str, PG_LOGMINER_SQLPARA_OTHER); 1085 | appendtoSQL(&result, "\', \'hex\')", PG_LOGMINER_SQLPARA_OTHER); 1086 | return result.sqlStr; 1087 | 1088 | } 1089 | 1090 | bool 1091 | getTypeOutputFuncFromDb(Oid type, Oid *typOutput, bool *typIsVarlena) 1092 | { 1093 | HeapTuple typeTuple; 1094 | Form_pg_type pt; 1095 | 1096 | typeTuple = SearchSysCache1(TYPEOID, ObjectIdGetDatum(type)); 1097 | if (!HeapTupleIsValid(typeTuple)) 1098 | return false; 1099 | pt = (Form_pg_type) GETSTRUCT(typeTuple); 1100 | 1101 | *typOutput = pt->typoutput; 1102 | *typIsVarlena = (!pt->typbyval) && (pt->typlen == -1); 1103 | 1104 | ReleaseSysCache(typeTuple); 1105 | return true; 1106 | } 1107 | -------------------------------------------------------------------------------- /xlogminer/xlogreader_logminer.c: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * Abstract: 4 | * Code for read wal from disk. 5 | * 6 | * Authored by lichuancheng@highgo.com ,20170524 7 | * 8 | * Copyright: 9 | * Copyright (c) 2017-2020, HighGo Software Co.,Ltd. All right reserved 10 | * 11 | * Identification: 12 | * contrib/xlogminer/xlogreader_logminer.c 13 | * 14 | *------------------------------------------------------------------------- 15 | */ 16 | #include "postgres.h" 17 | 18 | #include "access/transam.h" 19 | #include "access/xlog.h" 20 | #include "access/xlog_internal.h" 21 | #include "access/xlogreader.h" 22 | #include "catalog/pg_control.h" 23 | #include "utils/elog.h" 24 | #include "pg_logminer.h" 25 | #include "datadictionary.h" 26 | 27 | #include 28 | #include 29 | 30 | #define MAX_ERRORMSG_LEN 1000 31 | 32 | static bool allocate_recordbuf_logminer(XLogReaderState *state, uint32 reclength); 33 | static void ResetDecoder_logminer(XLogReaderState *state); 34 | static bool ValidXLogPageHeader_logminer(XLogReaderState *state, XLogRecPtr recptr, 35 | XLogPageHeader hdr); 36 | static bool ValidXLogRecordHeader_logminer(XLogReaderState *state, XLogRecPtr RecPtr, 37 | XLogRecPtr PrevRecPtr, XLogRecord *record, bool randAccess); 38 | 39 | static bool ValidXLogRecord_logminer(XLogReaderState *state, XLogRecord *record, XLogRecPtr recptr); 40 | static int ReadPageInternal_logminer(XLogReaderState *state, XLogRecPtr pageptr,int reqLen); 41 | 42 | static void 43 | ResetDecoder_logminer(XLogReaderState *state) 44 | { 45 | int block_id; 46 | 47 | state->decoded_record = NULL; 48 | 49 | state->main_data_len = 0; 50 | 51 | for (block_id = 0; block_id <= state->max_block_id; block_id++) 52 | { 53 | state->blocks[block_id].in_use = false; 54 | state->blocks[block_id].has_image = false; 55 | state->blocks[block_id].has_data = false; 56 | } 57 | state->max_block_id = -1; 58 | } 59 | 60 | 61 | /* 62 | * Allocate readRecordBuf to fit a record of at least the given length. 63 | * Returns true if successful, false if out of memory. 64 | * 65 | * readRecordBufSize is set to the new buffer size. 66 | * 67 | * To avoid useless small increases, round its size to a multiple of 68 | * XLOG_BLCKSZ, and make sure it's at least 5*Max(BLCKSZ, XLOG_BLCKSZ) to start 69 | * with. (That is enough for all "normal" records, but very large commit or 70 | * abort records might need more space.) 71 | */ 72 | static bool 73 | allocate_recordbuf_logminer(XLogReaderState *state, uint32 reclength) 74 | { 75 | uint32 newSize = reclength; 76 | 77 | newSize += XLOG_BLCKSZ - (newSize % XLOG_BLCKSZ); 78 | newSize = Max(newSize, 5 * Max(BLCKSZ, XLOG_BLCKSZ)); 79 | 80 | if (state->readRecordBuf) 81 | pfree(state->readRecordBuf); 82 | state->readRecordBuf = 83 | (char *) palloc_extended(newSize, MCXT_ALLOC_NO_OOM); 84 | if (state->readRecordBuf == NULL) 85 | { 86 | state->readRecordBufSize = 0; 87 | return false; 88 | } 89 | state->readRecordBufSize = newSize; 90 | return true; 91 | } 92 | 93 | 94 | /* 95 | * Read a single xlog page including at least [pageptr, reqLen] of valid data 96 | * via the read_page() callback. 97 | * 98 | * Returns -1 if the required page cannot be read for some reason; errormsg_buf 99 | * is set in that case (unless the error occurs in the read_page callback). 100 | * 101 | * We fetch the page from a reader-local cache if we know we have the required 102 | * data and if there hasn't been any error since caching the data. 103 | */ 104 | 105 | static int 106 | ReadPageInternal_logminer(XLogReaderState *state, XLogRecPtr pageptr, int reqLen) 107 | { 108 | int readLen; 109 | uint32 targetPageOff; 110 | XLogSegNo targetSegNo; 111 | XLogPageHeader hdr; 112 | 113 | Assert((pageptr % XLOG_BLCKSZ) == 0); 114 | 115 | XLByteToSeg(pageptr, targetSegNo); 116 | targetPageOff = (pageptr % XLogSegSize); 117 | 118 | /* check whether we have all the requested data already */ 119 | if (targetSegNo == state->readSegNo && targetPageOff == state->readOff && 120 | reqLen < state->readLen) 121 | return state->readLen; 122 | 123 | /* 124 | * Data is not in our buffer. 125 | * 126 | * Every time we actually read the page, even if we looked at parts of it 127 | * before, we need to do verification as the read_page callback might now 128 | * be rereading data from a different source. 129 | * 130 | * Whenever switching to a new WAL segment, we read the first page of the 131 | * file and validate its header, even if that's not where the target 132 | * record is. This is so that we can check the additional identification 133 | * info that is present in the first page's "long" header. 134 | */ 135 | if (targetSegNo != state->readSegNo && targetPageOff != 0) 136 | { 137 | XLogPageHeader hdr; 138 | XLogRecPtr targetSegmentPtr = pageptr - targetPageOff; 139 | readLen = state->read_page(state, targetSegmentPtr, XLOG_BLCKSZ, 140 | state->currRecPtr, 141 | state->readBuf, &state->readPageTLI); 142 | if (readLen < 0) 143 | goto err; 144 | 145 | /* we can be sure to have enough WAL available, we scrolled back */ 146 | Assert(readLen == XLOG_BLCKSZ); 147 | 148 | hdr = (XLogPageHeader) state->readBuf; 149 | if(rrctl.logprivate.changewal && pageptr < rrctl.logprivate.startptr) 150 | { 151 | if(!rrctl.logprivate.serialwal) 152 | { 153 | return 0; 154 | } 155 | pageptr = rrctl.logprivate.startptr; 156 | targetSegmentPtr = pageptr; 157 | XLByteToSeg(pageptr, targetSegNo); 158 | targetPageOff = (pageptr % XLogSegSize); 159 | targetSegmentPtr = pageptr - targetPageOff; 160 | } 161 | if (!ValidXLogPageHeader_logminer(state, targetSegmentPtr, hdr)) 162 | goto err; 163 | } 164 | 165 | /* 166 | * First, read the requested data length, but at least a short page header 167 | * so that we can validate it. 168 | */ 169 | 170 | readLen = state->read_page(state, pageptr, Max(reqLen, SizeOfXLogShortPHD), 171 | state->currRecPtr, 172 | state->readBuf, &state->readPageTLI); 173 | if(rrctl.logprivate.changewal && pageptr < rrctl.logprivate.startptr) 174 | { 175 | if(!rrctl.logprivate.serialwal) 176 | { 177 | return 0; 178 | } 179 | pageptr = rrctl.logprivate.startptr; 180 | XLByteToSeg(pageptr, targetSegNo); 181 | targetPageOff = (pageptr % XLogSegSize); 182 | } 183 | 184 | if (readLen < 0) 185 | goto err; 186 | 187 | Assert(readLen <= XLOG_BLCKSZ); 188 | 189 | /* Do we have enough data to check the header length? */ 190 | if (readLen <= SizeOfXLogShortPHD) 191 | goto err; 192 | 193 | Assert(readLen >= reqLen); 194 | 195 | hdr = (XLogPageHeader) state->readBuf; 196 | 197 | /* still not enough */ 198 | if (readLen < XLogPageHeaderSize(hdr)) 199 | { 200 | readLen = state->read_page(state, pageptr, XLogPageHeaderSize(hdr), 201 | state->currRecPtr, 202 | state->readBuf, &state->readPageTLI); 203 | if(rrctl.logprivate.changewal && pageptr < rrctl.logprivate.startptr) 204 | { 205 | if(!rrctl.logprivate.serialwal) 206 | { 207 | return 0; 208 | } 209 | pageptr = rrctl.logprivate.startptr; 210 | XLByteToSeg(pageptr, targetSegNo); 211 | targetPageOff = (pageptr % XLogSegSize); 212 | } 213 | if (readLen < 0) 214 | goto err; 215 | } 216 | 217 | /* 218 | * Now that we know we have the full header, validate it. 219 | */ 220 | if (!ValidXLogPageHeader_logminer(state, pageptr, hdr)) 221 | goto err; 222 | 223 | /* update cache information */ 224 | state->readSegNo = targetSegNo; 225 | state->readOff = targetPageOff; 226 | state->readLen = readLen; 227 | 228 | return readLen; 229 | 230 | err: 231 | state->readSegNo = 0; 232 | state->readOff = 0; 233 | state->readLen = 0; 234 | return -1; 235 | } 236 | 237 | 238 | /* 239 | * Attempt to read an XLOG record. 240 | * 241 | * If RecPtr is valid, try to read a record at that position. Otherwise 242 | * try to read a record just after the last one previously read. 243 | * 244 | * If the read_page callback fails to read the requested data, NULL is 245 | * returned. The callback is expected to have reported the error; errormsg 246 | * is set to NULL. 247 | * 248 | * If the reading fails for some other reason, NULL is also returned, and 249 | * *errormsg is set to a string with details of the failure. 250 | * 251 | * The returned pointer (or *errormsg) points to an internal buffer that's 252 | * valid until the next call to XLogReadRecord. 253 | */ 254 | XLogRecord * 255 | XLogReadRecord_logminer(XLogReaderState *state, XLogRecPtr RecPtr, char **errormsg) 256 | { 257 | XLogRecord *record; 258 | XLogRecPtr targetPagePtr; 259 | bool randAccess = false; 260 | uint32 len, 261 | total_len; 262 | uint32 targetRecOff; 263 | uint32 pageHeaderSize; 264 | bool gotheader; 265 | int readOff; 266 | 267 | /* reset error state */ 268 | *errormsg = NULL; 269 | state->errormsg_buf[0] = '\0'; 270 | 271 | ResetDecoder_logminer(state); 272 | 273 | if (RecPtr == InvalidXLogRecPtr) 274 | { 275 | RecPtr = state->EndRecPtr; 276 | 277 | if (state->ReadRecPtr == InvalidXLogRecPtr) 278 | randAccess = true; 279 | 280 | /* 281 | * RecPtr is pointing to end+1 of the previous WAL record. If we're 282 | * at a page boundary, no more records can fit on the current page. We 283 | * must skip over the page header, but we can't do that until we've 284 | * read in the page, since the header size is variable. 285 | */ 286 | } 287 | else 288 | { 289 | /* 290 | * In this case, the passed-in record pointer should already be 291 | * pointing to a valid record starting position. 292 | */ 293 | Assert(XRecOffIsValid(RecPtr)); 294 | randAccess = true; /* allow readPageTLI to go backwards too */ 295 | } 296 | 297 | state->currRecPtr = RecPtr; 298 | 299 | targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ); 300 | targetRecOff = RecPtr % XLOG_BLCKSZ; 301 | 302 | /* 303 | * Read the page containing the record into state->readBuf. Request enough 304 | * byte to cover the whole record header, or at least the part of it that 305 | * fits on the same page. 306 | */ 307 | readOff = ReadPageInternal_logminer(state, 308 | targetPagePtr, 309 | Min(targetRecOff + SizeOfXLogRecord, XLOG_BLCKSZ)); 310 | 311 | if(rrctl.logprivate.changewal && RecPtr < rrctl.logprivate.startptr) 312 | { 313 | if(!rrctl.logprivate.serialwal) 314 | { 315 | return NULL; 316 | } 317 | RecPtr = rrctl.logprivate.startptr; 318 | state->currRecPtr = RecPtr; 319 | targetPagePtr = RecPtr - (RecPtr % XLOG_BLCKSZ); 320 | targetRecOff = RecPtr % XLOG_BLCKSZ; 321 | rrctl.logprivate.changewal = false; 322 | randAccess = true; 323 | } 324 | 325 | 326 | if (readOff < 0) 327 | goto err; 328 | 329 | /* 330 | * ReadPageInternal_logminer always returns at least the page header, so we can 331 | * examine it now. 332 | */ 333 | pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf); 334 | if (targetRecOff == 0) 335 | { 336 | /* 337 | * At page start, so skip over page header. 338 | */ 339 | RecPtr += pageHeaderSize; 340 | targetRecOff = pageHeaderSize; 341 | } 342 | else if (targetRecOff < pageHeaderSize) 343 | { 344 | ereport(ERROR,(errmsg("invalid record offset at %X/%X",(uint32) (RecPtr >> 32), (uint32) RecPtr))); 345 | } 346 | 347 | if ((((XLogPageHeader) state->readBuf)->xlp_info & XLP_FIRST_IS_CONTRECORD) && 348 | targetRecOff == pageHeaderSize) 349 | { 350 | ereport(ERROR,(errmsg("contrecord is requested by %X/%X",(uint32) (RecPtr >> 32), (uint32) RecPtr))); 351 | } 352 | 353 | /* ReadPageInternal_logminer has verified the page header */ 354 | Assert(pageHeaderSize <= readOff); 355 | 356 | /* 357 | * Read the record length. 358 | * 359 | * NB: Even though we use an XLogRecord pointer here, the whole record 360 | * header might not fit on this page. xl_tot_len is the first field of the 361 | * struct, so it must be on this page (the records are MAXALIGNed), but we 362 | * cannot access any other fields until we've verified that we got the 363 | * whole header. 364 | */ 365 | record = (XLogRecord *) (state->readBuf + RecPtr % XLOG_BLCKSZ); 366 | total_len = record->xl_tot_len; 367 | 368 | /* 369 | * If the whole record header is on this page, validate it immediately. 370 | * Otherwise do just a basic sanity check on xl_tot_len, and validate the 371 | * rest of the header after reading it from the next page. The xl_tot_len 372 | * check is necessary here to ensure that we enter the "Need to reassemble 373 | * record" code path below; otherwise we might fail to apply 374 | * ValidXLogRecordHeader_logminer at all. 375 | */ 376 | if (targetRecOff <= XLOG_BLCKSZ - SizeOfXLogRecord) 377 | { 378 | if(!ValidXLogRecordHeader_logminer(state, RecPtr, state->ReadRecPtr, record, 379 | randAccess)) 380 | goto err; 381 | gotheader = true; 382 | } 383 | else 384 | { 385 | /* XXX: more validation should be done here */ 386 | if (total_len < SizeOfXLogRecord) 387 | { 388 | ereport(NOTICE,(errmsg("It has been loaded the xlog file without the normal end."))); 389 | goto err; 390 | } 391 | gotheader = false; 392 | } 393 | 394 | /* 395 | * Enlarge readRecordBuf as needed. 396 | */ 397 | if (total_len > state->readRecordBufSize && 398 | !allocate_recordbuf_logminer(state, total_len)) 399 | { 400 | /* We treat this as a "bogus data" condition */ 401 | ereport(ERROR,(errmsg("record length %u at %X/%X too long", total_len, (uint32) (RecPtr >> 32), (uint32) RecPtr))); 402 | } 403 | 404 | len = XLOG_BLCKSZ - RecPtr % XLOG_BLCKSZ; 405 | if (total_len > len) 406 | { 407 | /* Need to reassemble record */ 408 | char *contdata; 409 | XLogPageHeader pageHeader; 410 | char *buffer; 411 | uint32 gotlen; 412 | 413 | /* Copy the first fragment of the record from the first page. */ 414 | memcpy(state->readRecordBuf, 415 | state->readBuf + RecPtr % XLOG_BLCKSZ, len); 416 | buffer = state->readRecordBuf + len; 417 | gotlen = len; 418 | 419 | do 420 | { 421 | /* Calculate pointer to beginning of next page */ 422 | targetPagePtr += XLOG_BLCKSZ; 423 | 424 | /* Wait for the next page to become available */ 425 | readOff = ReadPageInternal_logminer(state, targetPagePtr, 426 | Min(total_len - gotlen + SizeOfXLogShortPHD, 427 | XLOG_BLCKSZ)); 428 | if(!rrctl.logprivate.serialwal) 429 | { 430 | return NULL; 431 | } 432 | 433 | if (readOff < 0) 434 | goto err; 435 | 436 | Assert(SizeOfXLogShortPHD <= readOff); 437 | 438 | /* Check that the continuation on next page looks valid */ 439 | pageHeader = (XLogPageHeader) state->readBuf; 440 | if (!(pageHeader->xlp_info & XLP_FIRST_IS_CONTRECORD)) 441 | { 442 | ereport(ERROR,(errmsg("there is no contrecord flag at %X/%X", (uint32)(RecPtr >> 32), (uint32) RecPtr))); 443 | } 444 | 445 | /* 446 | * Cross-check that xlp_rem_len agrees with how much of the record 447 | * we expect there to be left. 448 | */ 449 | if (pageHeader->xlp_rem_len == 0 || 450 | total_len != (pageHeader->xlp_rem_len + gotlen)) 451 | { 452 | ereport(ERROR,(errmsg("invalid contrecord length %u at %X/%X", 453 | pageHeader->xlp_rem_len, (uint32) (RecPtr >> 32), (uint32) RecPtr))); 454 | } 455 | 456 | /* Append the continuation from this page to the buffer */ 457 | pageHeaderSize = XLogPageHeaderSize(pageHeader); 458 | 459 | if (readOff < pageHeaderSize) 460 | readOff = ReadPageInternal_logminer(state, targetPagePtr, 461 | pageHeaderSize); 462 | 463 | Assert(pageHeaderSize <= readOff); 464 | 465 | contdata = (char *) state->readBuf + pageHeaderSize; 466 | len = XLOG_BLCKSZ - pageHeaderSize; 467 | if (pageHeader->xlp_rem_len < len) 468 | len = pageHeader->xlp_rem_len; 469 | 470 | if (readOff < pageHeaderSize + len) 471 | readOff = ReadPageInternal_logminer(state, targetPagePtr, 472 | pageHeaderSize + len); 473 | 474 | memcpy(buffer, (char *) contdata, len); 475 | buffer += len; 476 | gotlen += len; 477 | 478 | /* If we just reassembled the record header, validate it. */ 479 | if (!gotheader) 480 | { 481 | record = (XLogRecord *) state->readRecordBuf; 482 | ValidXLogRecordHeader_logminer(state, RecPtr, state->ReadRecPtr, 483 | record, randAccess); 484 | gotheader = true; 485 | } 486 | } while (gotlen < total_len); 487 | 488 | Assert(gotheader); 489 | 490 | record = (XLogRecord *) state->readRecordBuf; 491 | if (!ValidXLogRecord_logminer(state, record, RecPtr)) 492 | goto err; 493 | 494 | pageHeaderSize = XLogPageHeaderSize((XLogPageHeader) state->readBuf); 495 | state->ReadRecPtr = RecPtr; 496 | state->EndRecPtr = targetPagePtr + pageHeaderSize 497 | + MAXALIGN(pageHeader->xlp_rem_len); 498 | } 499 | else 500 | { 501 | /* Wait for the record data to become available */ 502 | readOff = ReadPageInternal_logminer(state, targetPagePtr, 503 | Min(targetRecOff + total_len, XLOG_BLCKSZ)); 504 | if (readOff < 0) 505 | goto err; 506 | 507 | /* Record does not cross a page boundary */ 508 | if (!ValidXLogRecord_logminer(state, record, RecPtr)) 509 | goto err; 510 | 511 | state->EndRecPtr = RecPtr + MAXALIGN(total_len); 512 | 513 | state->ReadRecPtr = RecPtr; 514 | memcpy(state->readRecordBuf, record, total_len); 515 | } 516 | 517 | /* 518 | * Special processing if it's an XLOG SWITCH record 519 | */ 520 | if (record->xl_rmid == RM_XLOG_ID && record->xl_info == XLOG_SWITCH) 521 | { 522 | /* Pretend it extends to end of segment */ 523 | state->EndRecPtr += XLogSegSize - 1; 524 | state->EndRecPtr -= state->EndRecPtr % XLogSegSize; 525 | } 526 | 527 | /*If return NULL here ,it is reached the end of xlog, 528 | otherwise(other return NULL this func) it is got a boundary record with discontinuous xlogfile*/ 529 | 530 | 531 | rrctl.logprivate.serialwal = true; 532 | if(!record) 533 | rrctl.logprivate.endptr_reached = true; 534 | if (DecodeXLogRecord(state, record, errormsg)) 535 | return record; 536 | else 537 | { 538 | return NULL; 539 | } 540 | 541 | err: 542 | 543 | /* 544 | * Invalidate the xlog page we've cached. We might read from a different 545 | * source after failure. 546 | */ 547 | state->readSegNo = 0; 548 | state->readOff = 0; 549 | state->readLen = 0; 550 | 551 | if (state->errormsg_buf[0] != '\0') 552 | *errormsg = state->errormsg_buf; 553 | rrctl.logprivate.serialwal = true; 554 | rrctl.logprivate.endptr_reached = true; 555 | return NULL; 556 | } 557 | 558 | /* 559 | * Validate an XLOG record header. 560 | * 561 | * This is just a convenience subroutine to avoid duplicated code in 562 | * XLogReadRecord. It's not intended for use from anywhere else. 563 | */ 564 | static bool 565 | ValidXLogRecordHeader_logminer(XLogReaderState *state, XLogRecPtr RecPtr, 566 | XLogRecPtr PrevRecPtr, XLogRecord *record, 567 | bool randAccess) 568 | { 569 | if (record->xl_tot_len < SizeOfXLogRecord) 570 | { 571 | return false; 572 | } 573 | if (record->xl_rmid > RM_MAX_ID) 574 | { 575 | ereport(ERROR,(errmsg("invalid resource manager ID %u at %X/%X", 576 | record->xl_rmid, (uint32) (RecPtr >> 32),(uint32) RecPtr))); 577 | } 578 | if (randAccess) 579 | { 580 | /* 581 | * We can't exactly verify the prev-link, but surely it should be less 582 | * than the record's own address. 583 | */ 584 | if (!(record->xl_prev < RecPtr)) 585 | { 586 | ereport(ERROR,(errmsg("record with incorrect prev-link %X/%X at %X/%X", 587 | (uint32) (record->xl_prev >> 32), 588 | (uint32) record->xl_prev, 589 | (uint32) (RecPtr >> 32), (uint32) RecPtr))); 590 | } 591 | } 592 | else 593 | { 594 | /* 595 | * Record's prev-link should exactly match our previous location. This 596 | * check guards against torn WAL pages where a stale but valid-looking 597 | * WAL record starts on a sector boundary. 598 | */ 599 | if (record->xl_prev != PrevRecPtr) 600 | { 601 | ereport(ERROR,(errmsg("record with incorrect prev-link %X/%X at %X/%X", 602 | (uint32) (record->xl_prev >> 32), 603 | (uint32) record->xl_prev, 604 | (uint32) (RecPtr >> 32), (uint32) RecPtr))); 605 | } 606 | } 607 | 608 | return true; 609 | } 610 | 611 | 612 | 613 | /* 614 | * CRC-check an XLOG record. We do not believe the contents of an XLOG 615 | * record (other than to the minimal extent of computing the amount of 616 | * data to read in) until we've checked the CRCs. 617 | * 618 | * We assume all of the record (that is, xl_tot_len bytes) has been read 619 | * into memory at *record. Also, ValidXLogRecordHeader_logminer() has accepted the 620 | * record's header, which means in particular that xl_tot_len is at least 621 | * SizeOfXlogRecord, so it is safe to fetch xl_len. 622 | */ 623 | static bool 624 | ValidXLogRecord_logminer(XLogReaderState *state, XLogRecord *record, XLogRecPtr recptr) 625 | { 626 | pg_crc32c crc; 627 | 628 | /* Calculate the CRC */ 629 | INIT_CRC32C(crc); 630 | COMP_CRC32C(crc, ((char *) record) + SizeOfXLogRecord, record->xl_tot_len - SizeOfXLogRecord); 631 | /* include the record header last */ 632 | COMP_CRC32C(crc, (char *) record, offsetof(XLogRecord, xl_crc)); 633 | FIN_CRC32C(crc); 634 | 635 | if (!EQ_CRC32C(record->xl_crc, crc)) 636 | { 637 | ereport(ERROR,(errmsg("incorrect resource manager data checksum in record at %X/%X", 638 | (uint32) (recptr >> 32), (uint32) recptr))); 639 | } 640 | 641 | return true; 642 | } 643 | 644 | 645 | /* 646 | * Validate a page header 647 | */ 648 | static bool 649 | ValidXLogPageHeader_logminer(XLogReaderState *state, XLogRecPtr recptr, 650 | XLogPageHeader hdr) 651 | { 652 | XLogRecPtr recaddr; 653 | XLogSegNo segno; 654 | int32 offset; 655 | 656 | Assert((recptr % XLOG_BLCKSZ) == 0); 657 | 658 | XLByteToSeg(recptr, segno); 659 | offset = recptr % XLogSegSize; 660 | 661 | XLogSegNoOffsetToRecPtr(segno, offset, recaddr); 662 | 663 | if (hdr->xlp_magic != XLOG_PAGE_MAGIC) 664 | { 665 | char fname[MAXFNAMELEN]; 666 | 667 | XLogFileName(fname, state->readPageTLI, segno); 668 | return false; 669 | } 670 | 671 | if ((hdr->xlp_info & ~XLP_ALL_FLAGS) != 0) 672 | { 673 | char fname[MAXFNAMELEN]; 674 | 675 | XLogFileName(fname, state->readPageTLI, segno); 676 | ereport(ERROR,(errmsg("invalid info bits %04X in log segment %s, offset %u", 677 | hdr->xlp_info, 678 | fname, 679 | offset))); 680 | return false; 681 | } 682 | 683 | if (hdr->xlp_info & XLP_LONG_HEADER) 684 | { 685 | XLogLongPageHeader longhdr = (XLogLongPageHeader) hdr; 686 | 687 | if (state->system_identifier && 688 | longhdr->xlp_sysid != state->system_identifier) 689 | { 690 | char fhdrident_str[32]; 691 | char sysident_str[32]; 692 | 693 | /* 694 | * Format sysids separately to keep platform-dependent format code 695 | * out of the translatable message string. 696 | */ 697 | snprintf(fhdrident_str, sizeof(fhdrident_str), UINT64_FORMAT, 698 | longhdr->xlp_sysid); 699 | snprintf(sysident_str, sizeof(sysident_str), UINT64_FORMAT, 700 | state->system_identifier); 701 | ereport(ERROR,(errmsg("WAL file is from different database system: WAL file database system identifier is %s, pg_control database system identifier is %s", 702 | fhdrident_str, sysident_str))); 703 | return false; 704 | } 705 | else if (longhdr->xlp_seg_size != XLogSegSize) 706 | { 707 | ereport(ERROR,(errmsg("WAL file is from different database system: incorrect XLOG_SEG_SIZE in page header"))); 708 | return false; 709 | } 710 | else if (longhdr->xlp_xlog_blcksz != XLOG_BLCKSZ) 711 | { 712 | ereport(ERROR,(errmsg("WAL file is from different database system: incorrect XLOG_BLCKSZ in page header"))); 713 | return false; 714 | } 715 | } 716 | else if (offset == 0) 717 | { 718 | char fname[MAXFNAMELEN]; 719 | 720 | XLogFileName(fname, state->readPageTLI, segno); 721 | 722 | /* hmm, first page of file doesn't have a long header? */ 723 | ereport(ERROR,(errmsg( "invalid info bits %04X in log segment %s, offset %u", 724 | hdr->xlp_info, 725 | fname, 726 | offset))); 727 | return false; 728 | } 729 | 730 | if (hdr->xlp_pageaddr != recaddr) 731 | { 732 | char fname[MAXFNAMELEN]; 733 | 734 | XLogFileName(fname, state->readPageTLI, segno); 735 | rrctl.logprivate.endptr_reached = true; 736 | /* 737 | ereport(NOTICE,(errmsg("unexpected pageaddr %X/%X in log segment %s, offset %u", 738 | (uint32) (hdr->xlp_pageaddr >> 32), (uint32) hdr->xlp_pageaddr, 739 | fname, 740 | offset))); 741 | */ 742 | return false; 743 | } 744 | 745 | /* 746 | * Since child timelines are always assigned a TLI greater than their 747 | * immediate parent's TLI, we should never see TLI go backwards across 748 | * successive pages of a consistent WAL sequence. 749 | * 750 | * Sometimes we re-read a segment that's already been (partially) read. So 751 | * we only verify TLIs for pages that are later than the last remembered 752 | * LSN. 753 | */ 754 | if (recptr > state->latestPagePtr) 755 | { 756 | if (hdr->xlp_tli < state->latestPageTLI) 757 | { 758 | char fname[MAXFNAMELEN]; 759 | 760 | XLogFileName(fname, state->readPageTLI, segno); 761 | 762 | ereport(ERROR,(errmsg("out-of-sequence timeline ID %u (after %u) in log segment %s, offset %u", 763 | hdr->xlp_tli, 764 | state->latestPageTLI, 765 | fname, 766 | offset))); 767 | return false; 768 | } 769 | } 770 | state->latestPagePtr = recptr; 771 | state->latestPageTLI = hdr->xlp_tli; 772 | 773 | return true; 774 | } 775 | 776 | 777 | /* 778 | * Functions that are currently not needed in the backend, but are better 779 | * implemented inside xlogreader.c because of the internal facilities available 780 | * here. 781 | */ 782 | 783 | /* 784 | * Find the first record with an lsn >= RecPtr. 785 | * 786 | * Useful for checking whether RecPtr is a valid xlog address for reading, and 787 | * to find the first valid address after some address when dumping records for 788 | * debugging purposes. 789 | */ 790 | XLogRecPtr 791 | XLogFindFirstRecord(XLogReaderState *state, XLogRecPtr RecPtr) 792 | { 793 | XLogReaderState saved_state = *state; 794 | XLogRecPtr targetPagePtr; 795 | XLogRecPtr tmpRecPtr; 796 | int targetRecOff; 797 | XLogRecPtr found = InvalidXLogRecPtr; 798 | uint32 pageHeaderSize; 799 | XLogPageHeader header; 800 | int readLen; 801 | char *errormsg; 802 | 803 | Assert(!XLogRecPtrIsInvalid(RecPtr)); 804 | 805 | targetRecOff = RecPtr % XLOG_BLCKSZ; 806 | 807 | /* scroll back to page boundary */ 808 | targetPagePtr = RecPtr - targetRecOff; 809 | 810 | /* Read the page containing the record */ 811 | readLen = ReadPageInternal_logminer(state, targetPagePtr, targetRecOff); 812 | if (readLen < 0) 813 | goto err; 814 | 815 | header = (XLogPageHeader) state->readBuf; 816 | 817 | pageHeaderSize = XLogPageHeaderSize(header); 818 | 819 | /* make sure we have enough data for the page header */ 820 | readLen = ReadPageInternal_logminer(state, targetPagePtr, pageHeaderSize); 821 | if (readLen < 0) 822 | goto err; 823 | 824 | /* skip over potential continuation data */ 825 | if (header->xlp_info & XLP_FIRST_IS_CONTRECORD) 826 | { 827 | /* record headers are MAXALIGN'ed */ 828 | tmpRecPtr = targetPagePtr + pageHeaderSize 829 | + MAXALIGN(header->xlp_rem_len); 830 | } 831 | else 832 | { 833 | tmpRecPtr = targetPagePtr + pageHeaderSize; 834 | } 835 | 836 | /* 837 | * we know now that tmpRecPtr is an address pointing to a valid XLogRecord 838 | * because either we're at the first record after the beginning of a page 839 | * or we just jumped over the remaining data of a continuation. 840 | */ 841 | rrctl.logprivate.changewal = false; 842 | 843 | while (XLogReadRecord_logminer(state, tmpRecPtr, &errormsg) != NULL) 844 | { 845 | /* continue after the record */ 846 | tmpRecPtr = InvalidXLogRecPtr; 847 | 848 | /* past the record we've found, break out */ 849 | if (RecPtr <= state->ReadRecPtr) 850 | { 851 | found = state->ReadRecPtr; 852 | if (found == InvalidXLogRecPtr) 853 | ereport(ERROR,(errmsg("could not find a valid record after %X/%X", 854 | (uint32) (RecPtr >> 32), 855 | (uint32) RecPtr))); 856 | 857 | if (found != RecPtr && (RecPtr % XLogSegSize) != 0) 858 | printf("first record is after %X/%X, at %X/%X, skipping over %u bytes\n", 859 | (uint32) (RecPtr >> 32), (uint32) RecPtr, 860 | (uint32) (found >> 32), (uint32) found, 861 | (uint32) (found - RecPtr)); 862 | goto out; 863 | } 864 | } 865 | 866 | err: 867 | out: 868 | /* Reset state to what we had before finding the record */ 869 | state->readSegNo = 0; 870 | state->readOff = 0; 871 | state->readLen = 0; 872 | state->ReadRecPtr = saved_state.ReadRecPtr; 873 | state->EndRecPtr = saved_state.EndRecPtr; 874 | 875 | return found; 876 | } 877 | 878 | 879 | static bool 880 | ifSerialWalfile(XLogSegNo xlnPre,XLogSegNo xlnCur) 881 | { 882 | XLogSegNo segIdPre; 883 | XLogSegNo xlogIdPre; 884 | XLogSegNo segIdCur; 885 | XLogSegNo xlogIdCur; 886 | XLogSegNo temp = 1; 887 | 888 | segIdPre = xlnPre % (temp << 32); 889 | xlogIdPre = xlnPre / (temp << 32); 890 | segIdCur = xlnCur % (temp << 32); 891 | xlogIdCur = xlnCur / (temp << 32); 892 | 893 | if(xlogIdPre == xlogIdCur && segIdPre + 1 == segIdCur) 894 | return true; 895 | if(xlogIdPre + 1 == xlogIdCur && 0xff == segIdPre && 0x01 == segIdCur) 896 | return true; 897 | return false; 898 | } 899 | 900 | 901 | int 902 | XLogMinerXLogRead(const char *directory, TimeLineID *timeline_id, 903 | XLogRecPtr startptr, char *buf, Size count) 904 | { 905 | char *p; 906 | XLogRecPtr recptr; 907 | Size nbytes; 908 | XLogSegNo sendSegNo_temp = 0; 909 | 910 | int *sendFile; 911 | XLogSegNo *sendSegNo; 912 | uint32 *sendOff; 913 | 914 | sendFile = &rrctl.lfctx.sendFile; 915 | sendSegNo = &rrctl.lfctx.sendSegNo; 916 | sendOff = &rrctl.lfctx.sendOff; 917 | 918 | p = buf; 919 | recptr = startptr; 920 | nbytes = count; 921 | 922 | while (nbytes > 0) 923 | { 924 | uint32 startoff; 925 | int segbytes; 926 | int readbytes; 927 | char *xlogfilename = NULL; 928 | char *temp_path = NULL; 929 | char *temp_fname = NULL; 930 | TimeLineID tid; 931 | 932 | 933 | startoff = recptr % XLogSegSize; 934 | 935 | if (*sendFile < 0 || !XLByteInSeg(recptr, *sendSegNo)) 936 | { 937 | 938 | /* Switch to another logfile segment */ 939 | if (*sendFile >= 0) 940 | { 941 | close(*sendFile); 942 | *sendFile = -1; 943 | } 944 | xlogfilename = getNextXlogFile((char*)(&rrctl.lfctx),false); 945 | if(xlogfilename) 946 | { 947 | split_path_fname(xlogfilename, &temp_path, &temp_fname); 948 | *sendFile = xlog_file_open(temp_path, temp_fname); 949 | } 950 | else 951 | return PG_LOGMINER_WALFILE_ENDALL; 952 | 953 | if (*sendFile < 0) 954 | { 955 | ereport(NOTICE,(errmsg("could not find file \"%s\": %s",temp_fname, strerror(errno)))); 956 | *sendFile = -1; 957 | *sendSegNo = 0; 958 | *sendOff = 0; 959 | return PG_LOGMINER_WALFILE_ERROR_NOFIND; 960 | } 961 | 962 | XLogFromFileName(temp_fname, &tid, &sendSegNo_temp); 963 | *timeline_id = tid; 964 | XLogSegNoOffsetToRecPtr(sendSegNo_temp, 0, rrctl.logprivate.startptr); 965 | if(!ifSerialWalfile(*sendSegNo,sendSegNo_temp) && 0 != *sendSegNo) 966 | { 967 | rrctl.logprivate.serialwal = false; 968 | rrctl.logprivate.changewal= true; 969 | } 970 | else 971 | rrctl.logprivate.serialwal = true; 972 | *sendSegNo = sendSegNo_temp; 973 | 974 | recptr = rrctl.logprivate.startptr; 975 | rrctl.logprivate.changewal= true; 976 | if(temp_path) 977 | pfree(temp_path); 978 | if(temp_fname) 979 | pfree(temp_fname); 980 | *sendOff = 0; 981 | } 982 | 983 | /* Need to seek in the file? */ 984 | if (*sendOff != startoff) 985 | { 986 | if (lseek(*sendFile, (off_t) startoff, SEEK_SET) < 0) 987 | { 988 | int err = errno; 989 | char fname[MAXPGPATH]; 990 | 991 | XLogFileName(fname, tid, *sendSegNo); 992 | ereport(ERROR,(errmsg("could not seek in log segment %s to offset %u: %s", 993 | fname, startoff, strerror(err)))); 994 | } 995 | *sendOff = startoff; 996 | } 997 | 998 | /* How many bytes are within this segment? */ 999 | if (nbytes > (XLogSegSize - startoff)) 1000 | segbytes = XLogSegSize - startoff; 1001 | else 1002 | segbytes = nbytes; 1003 | 1004 | readbytes = read(*sendFile, p, segbytes); 1005 | if (readbytes <= 0) 1006 | { 1007 | char fname[MAXPGPATH]; 1008 | int err = errno; 1009 | XLogFileName(fname, tid, *sendSegNo); 1010 | ereport(ERROR,(errmsg("could not read from log segment %s, offset %d, length %d: %s", 1011 | fname, *sendOff, segbytes, strerror(err)))); 1012 | } 1013 | 1014 | /* Update state for read */ 1015 | recptr += readbytes; 1016 | 1017 | *sendOff += readbytes; 1018 | nbytes -= readbytes; 1019 | p += readbytes; 1020 | } 1021 | return 0; 1022 | } 1023 | 1024 | /* 1025 | strategy that get data from xlogfile list 1026 | */ 1027 | int 1028 | XLogMinerReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen, 1029 | XLogRecPtr targetPtr, char *readBuff, TimeLineID *curFileTLI) 1030 | { 1031 | XLogMinerPrivate *private = state->private_data; 1032 | int count = XLOG_BLCKSZ; 1033 | int statu = 0; 1034 | 1035 | statu = XLogMinerXLogRead(NULL, &private->timeline, targetPagePtr, 1036 | readBuff, count); 1037 | *curFileTLI = private->timeline; 1038 | if(PG_LOGMINER_WALFILE_ERROR_NOFIND == statu || PG_LOGMINER_WALFILE_ENDALL == statu) 1039 | count = PG_LOGMINER_WALFILE_ERROR_COUNT; 1040 | return count; 1041 | } 1042 | 1043 | 1044 | -------------------------------------------------------------------------------- /xlogminer/pg_logminer.c: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * Abstract: 4 | * Main analyse function of XLogminer 5 | * 6 | * Authored by lichuancheng@highgo.com ,20170524 7 | * 8 | * Copyright: 9 | * Copyright (c) 2017-2020, HighGo Software Co.,Ltd. All right reserved 10 | * 11 | * Identification: 12 | * pg_logminer.c 13 | * 14 | *------------------------------------------------------------------------- 15 | */ 16 | #include "postgres.h" 17 | 18 | #include "utils/builtins.h" 19 | #include 20 | 21 | #include "pg_logminer.h" 22 | #include "logminer.h" 23 | #include "catalog/pg_class.h" 24 | #include "access/heapam.h" 25 | #include "utils/relcache.h" 26 | #include "catalog/pg_proc.h" 27 | #include "catalog/pg_auth_members.h" 28 | #include "access/transam.h" 29 | #include "commands/dbcommands_xlog.h" 30 | #include "datadictionary.h" 31 | #include "xlogminer_contents.h" 32 | #include "catalog/pg_namespace.h" 33 | #include "catalog/pg_extension.h" 34 | #include "catalog/pg_depend.h" 35 | #include "catalog/pg_attrdef.h" 36 | #include "catalog/pg_namespace.h" 37 | #include "catalog/pg_tablespace.h" 38 | #include "catalog/pg_database.h" 39 | #include "catalog/pg_constraint.h" 40 | #include "catalog/pg_type.h" 41 | 42 | 43 | 44 | 45 | RecordRecycleCtl rrctl; 46 | SQLRecycleCtl srctl; 47 | uint32 sqlnoser; 48 | 49 | 50 | static SystemClass sysclass[PG_LOGMINER_SYSCLASS_MAX]; 51 | static int sysclassNum = 0; 52 | RelationKind *relkind_miner; 53 | 54 | SysClassLevel ImportantSysClass[] = { 55 | {PG_LOGMINER_IMPTSYSCLASS_PGCLASS, "pg_class", 0}, 56 | {PG_LOGMINER_IMPTSYSCLASS_PGDATABASE, "pg_database", 0}, 57 | {PG_LOGMINER_IMPTSYSCLASS_PGEXTENSION, "pg_extension", 0}, 58 | {PG_LOGMINER_IMPTSYSCLASS_PGNAMESPACE, "pg_namespace", 0}, 59 | {PG_LOGMINER_IMPTSYSCLASS_PGTABLESPACE, "pg_tablespace", 0}, 60 | {PG_LOGMINER_IMPTSYSCLASS_PGCONSTRAINT, "pg_constraint", 0}, 61 | {PG_LOGMINER_IMPTSYSCLASS_PGAUTHID, "pg_authid", 0}, 62 | {PG_LOGMINER_IMPTSYSCLASS_PGPROC, "pg_proc", 0}, 63 | {PG_LOGMINER_IMPTSYSCLASS_PGDEPEND, "pg_depend", 0}, 64 | {PG_LOGMINER_IMPTSYSCLASS_PGINDEX, "pg_index", 0}, 65 | {PG_LOGMINER_IMPTSYSCLASS_PGATTRIBUTE, "pg_attribute", 0}, 66 | {PG_LOGMINER_IMPTSYSCLASS_PGSHDESC, "pg_shdescription", 0}, 67 | {PG_LOGMINER_IMPTSYSCLASS_PGATTRDEF, "pg_attrdef", 0}, 68 | {PG_LOGMINER_IMPTSYSCLASS_PGTYPE, "pg_type", 0}, 69 | {PG_LOGMINER_IMPTSYSCLASS_PGAUTH_MEMBERS, "pg_auth_members", 0}, 70 | {PG_LOGMINER_IMPTSYSCLASS_PGINHERITS, "pg_inherits", 0}, 71 | {PG_LOGMINER_IMPTSYSCLASS_PGTRIGGER, "pg_trigger", 0}, 72 | {PG_LOGMINER_IMPTSYSCLASS_PGLANGUAGE, "pg_language", 0} 73 | 74 | }; 75 | 76 | static SQLKind sqlkind[] = { 77 | {"UPDATE",PG_LOGMINER_SQLKIND_UPDATE}, 78 | {"INSERT",PG_LOGMINER_SQLKIND_INSERT}, 79 | {"DELETE",PG_LOGMINER_SQLKIND_DELETE}, 80 | {"CREATE",PG_LOGMINER_SQLKIND_CREATE}, 81 | {"ALTER",PG_LOGMINER_SQLKIND_ALTER}, 82 | {"XACT",PG_LOGMINER_SQLKIND_XACT}, 83 | {"DROP",PG_LOGMINER_SQLKIND_DROP} 84 | }; 85 | 86 | 87 | 88 | PG_MODULE_MAGIC; 89 | 90 | /* 91 | PG_FUNCTION_INFO_V1(pg_xlog2sql); 92 | */ 93 | PG_FUNCTION_INFO_V1(xlogminer_build_dictionary); 94 | PG_FUNCTION_INFO_V1(xlogminer_load_dictionary); 95 | PG_FUNCTION_INFO_V1(xlogminer_stop); 96 | PG_FUNCTION_INFO_V1(xlogminer_xlogfile_add); 97 | PG_FUNCTION_INFO_V1(xlogminer_xlogfile_list); 98 | PG_FUNCTION_INFO_V1(xlogminer_xlogfile_remove); 99 | PG_FUNCTION_INFO_V1(pg_minerXlog); 100 | 101 | 102 | static bool tableIfSysclass(char *tablename, Oid reloid); 103 | static bool tableifImpSysclass(char *tablename, Oid reloid); 104 | static void getTupleData_Insert(XLogReaderState *record, char** tuple_info, Oid reloid); 105 | static void getTupleData_Delete(XLogReaderState *record, char** tuple_info, Oid reloid); 106 | static void getTupleData_Update(XLogReaderState *record, char** tuple_info, char** tuple_info_old,Oid reloid); 107 | static bool getTupleInfoByRecord(XLogReaderState *record, uint8 info, NameData* relname,char** schname, char** tuple_info, char** tuple_info_old); 108 | static void minerHeapInsert(XLogReaderState *record, XLogMinerSQL *sql_simple,uint8 info); 109 | static void minerHeapDelete(XLogReaderState *record, XLogMinerSQL *sql_simple,uint8 info); 110 | static void minerHeapUpdate(XLogReaderState *record, XLogMinerSQL *sql_simple, uint8 info); 111 | static void minerHeap2MutiInsert(XLogReaderState *record, XLogMinerSQL *sql_simple, uint8 info) ; 112 | static bool getNextRecord(); 113 | static int getsqlkind(char *sqlheader); 114 | static bool parserInsertSql(XLogMinerSQL *sql_ori, XLogMinerSQL *sql_opt); 115 | static bool parserDeleteSql(XLogMinerSQL *sql_ori, XLogMinerSQL *sql_opt); 116 | static bool parserUpdateSql(XLogMinerSQL *sql_ori, XLogMinerSQL *sql_opt); 117 | static void parserXactSql(XLogMinerSQL *sql_ori, XLogMinerSQL *sql_opt); 118 | static void XLogMinerRecord_heap(XLogReaderState *record, XLogMinerSQL *sql_simple); 119 | static void XLogMinerRecord_heap2(XLogReaderState *record, XLogMinerSQL *sql_simple); 120 | static void XLogMinerRecord_dbase(XLogReaderState *record, XLogMinerSQL *sql_simple); 121 | static void XLogMinerRecord_xact(XLogReaderState *record, XLogMinerSQL *sql_simple, TimestampTz *xacttime); 122 | static bool XLogMinerRecord(XLogReaderState *record, XLogMinerSQL *sql_simple,TimestampTz *xacttime); 123 | static bool sqlParser(XLogReaderState *record, TimestampTz *xacttime); 124 | 125 | 126 | SysClassLevel *getImportantSysClass() 127 | { 128 | return ImportantSysClass; 129 | } 130 | 131 | /* 132 | *append a string to XLogMinerSQL 133 | */ 134 | void 135 | appendtoSQL(XLogMinerSQL *sql_simple, char *sqlpara , int spaceKind) 136 | { 137 | 138 | int addsize; 139 | 140 | if(PG_LOGMINER_SQLPARA_OTHER != spaceKind && PG_LOGMINER_XLOG_DBINIT == rrctl.system_init_record) 141 | return; 142 | 143 | if(NULL == sqlpara || 0 == strcmp("",sqlpara) || 0 == strcmp(" ",sqlpara) ) 144 | sqlpara = "NULL"; 145 | 146 | addsize = strlen(sqlpara); 147 | 148 | while(addsize >= sql_simple->rem_size) 149 | { 150 | addSpace(sql_simple,spaceKind); 151 | } 152 | 153 | memcpy(sql_simple->sqlStr + sql_simple->use_size ,sqlpara ,addsize); 154 | sql_simple->use_size += addsize; 155 | sql_simple->rem_size -= addsize; 156 | } 157 | 158 | void 159 | appendtoSQL_simquo(XLogMinerSQL *sql_simple, char* ptr, bool quoset) 160 | { 161 | if(quoset) 162 | appendtoSQL(sql_simple, "\'", PG_LOGMINER_SQLPARA_OTHER); 163 | appendtoSQL(sql_simple, ptr, PG_LOGMINER_SQLPARA_OTHER); 164 | if(quoset) 165 | appendtoSQL(sql_simple, "\'", PG_LOGMINER_SQLPARA_OTHER); 166 | 167 | } 168 | 169 | void 170 | appendtoSQL_doubquo(XLogMinerSQL *sql_simple, char* ptr, bool quoset) 171 | { 172 | if(quoset) 173 | appendtoSQL(sql_simple, "\"", PG_LOGMINER_SQLPARA_SIMPLE); 174 | appendtoSQL(sql_simple, ptr, PG_LOGMINER_SQLPARA_SIMPLE); 175 | if(quoset) 176 | appendtoSQL(sql_simple, "\"", PG_LOGMINER_SQLPARA_SIMPLE); 177 | 178 | } 179 | 180 | void 181 | appendtoSQL_atttyptrans(XLogMinerSQL *sql_simple, Oid typoid) 182 | { 183 | if(POINTOID == typoid || JSONOID == typoid || (POLYGONOID == typoid) || XMLOID == typoid) 184 | appendtoSQL(sql_simple, "::text", PG_LOGMINER_SQLPARA_SIMPLE); 185 | 186 | } 187 | 188 | void 189 | appendtoSQL_valuetyptrans(XLogMinerSQL *sql_simple, Oid typoid) 190 | { 191 | if(FLOAT4OID == typoid) 192 | appendtoSQL(sql_simple, "::float4", PG_LOGMINER_SQLPARA_SIMPLE); 193 | } 194 | 195 | 196 | /* 197 | * 198 | * Wipe some string from XLogMinerSQL. 199 | * For example 200 | * sql_simple.sqlStr="delete from t1 where values" 201 | * fromstr="where values" 202 | * checkstr="where " 203 | * then sql_simple.sqlStr become "delete from t1 where " 204 | */ 205 | void 206 | wipeSQLFromstr(XLogMinerSQL *sql_simple,char *fromstr,char *checkstr) 207 | { 208 | char *strPtr = NULL; 209 | int length_ptr; 210 | 211 | 212 | if(NULL == sql_simple || NULL == sql_simple->sqlStr ||NULL == fromstr) 213 | return; 214 | strPtr = strstr(sql_simple->sqlStr,fromstr); 215 | if(NULL == strPtr) 216 | return; 217 | strPtr = strPtr + strlen(checkstr); 218 | length_ptr = strlen(strPtr); 219 | memset(strPtr, 0, length_ptr); 220 | sql_simple->use_size -= length_ptr; 221 | sql_simple->rem_size += length_ptr; 222 | } 223 | 224 | /* 225 | * Append a space to XLogMinerSQL 226 | */ 227 | void 228 | appendBlanktoSQL(XLogMinerSQL *sql_simple) 229 | { 230 | 231 | int addsize; 232 | char *sqlpara = " "; 233 | 234 | addsize = strlen(sqlpara); 235 | while(addsize >= sql_simple->rem_size) 236 | { 237 | addSpace(sql_simple,PG_LOGMINER_SQLPARA_OTHER); 238 | } 239 | memcpy(sql_simple->sqlStr + sql_simple->use_size ,sqlpara ,addsize); 240 | sql_simple->use_size += addsize; 241 | sql_simple->rem_size -= addsize; 242 | } 243 | 244 | static bool 245 | tableIfSysclass(char *tablename, Oid reloid) 246 | { 247 | int loop; 248 | if(FirstNormalObjectId < reloid) 249 | return false; 250 | for(loop = 0; loop < sysclassNum; loop++) 251 | { 252 | if(0 == strcmp(sysclass[loop].classname.data,tablename)) 253 | { 254 | return true; 255 | } 256 | } 257 | return false; 258 | } 259 | 260 | static bool 261 | tableifImpSysclass(char *tablename, Oid reloid) 262 | { 263 | int loop; 264 | if(FirstNormalObjectId < reloid) 265 | return false; 266 | for(loop = 0; loop < PG_LOGMINER_IMPTSYSCLASS_IMPTNUM; loop++) 267 | { 268 | if(0 == strcmp(ImportantSysClass[loop].relname,tablename)) 269 | { 270 | return true; 271 | } 272 | } 273 | return false; 274 | } 275 | 276 | /* 277 | * Get useful data from record,and return insert data by tuple_info. 278 | */ 279 | static void 280 | getTupleData_Insert(XLogReaderState *record, char** tuple_info, Oid reloid) 281 | { 282 | HeapTupleData tupledata; 283 | char *tuplem = NULL; 284 | char *data = NULL; 285 | TupleDesc tupdesc = NULL; 286 | Size datalen = 0; 287 | uint32 newlen = 0; 288 | HeapTupleHeader htup; 289 | xl_heap_header xlhdr; 290 | RelFileNode target_node; 291 | BlockNumber blkno; 292 | ItemPointerData target_tid; 293 | xl_heap_insert *xlrec = (xl_heap_insert *) XLogRecGetData(record); 294 | 295 | if(!rrctl.tupinfo_init) 296 | { 297 | memset(&rrctl.tupinfo, 0, sizeof(XLogMinerSQL)); 298 | rrctl.tupinfo_init = true; 299 | } 300 | else 301 | cleanSpace(&rrctl.tupinfo); 302 | memset(&tupledata, 0, sizeof(HeapTupleData)); 303 | XLogRecGetBlockTag(record, 0, &target_node, NULL, &blkno); 304 | ItemPointerSetBlockNumber(&target_tid, blkno); 305 | ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum); 306 | 307 | data = XLogRecGetBlockData(record, 0, &datalen); 308 | if(!data) 309 | return; 310 | 311 | newlen = datalen - SizeOfHeapHeader; 312 | Assert(datalen > SizeOfHeapHeader && newlen <= MaxHeapTupleSize); 313 | memcpy((char *) &xlhdr, data, SizeOfHeapHeader); 314 | data += SizeOfHeapHeader; 315 | 316 | tuplem = rrctl.tuplem; 317 | rrctl.reloid = reloid; 318 | htup = (HeapTupleHeader)tuplem; 319 | memcpy((char *) htup + SizeofHeapTupleHeader, 320 | data,newlen); 321 | newlen += SizeofHeapTupleHeader; 322 | htup->t_infomask2 = xlhdr.t_infomask2; 323 | htup->t_infomask = xlhdr.t_infomask; 324 | htup->t_hoff = xlhdr.t_hoff; 325 | HeapTupleHeaderSetXmin(htup, XLogRecGetXid(record)); 326 | HeapTupleHeaderSetCmin(htup, FirstCommandId); 327 | htup->t_ctid = target_tid; 328 | 329 | if(rrctl.tupdesc) 330 | { 331 | freetupdesc(rrctl.tupdesc); 332 | rrctl.tupdesc = NULL; 333 | } 334 | 335 | rrctl.tupdesc = GetDescrByreloid(reloid); 336 | tupdesc = rrctl.tupdesc; 337 | 338 | if(NULL != htup) 339 | { 340 | tupledata.t_data = htup; 341 | if(rrctl.toastrel) 342 | { 343 | /*if it is insert into a toast table,store it into list*/ 344 | Oid chunk_id; 345 | int chunk_seq; 346 | char *chunk_data; 347 | bool isnull = false; 348 | 349 | chunk_id = DatumGetObjectId(fastgetattr(&tupledata, 1, tupdesc, &isnull)); 350 | chunk_seq =DatumGetInt32(fastgetattr(&tupledata, 2, tupdesc, &isnull)); 351 | chunk_data = DatumGetPointer(fastgetattr(&tupledata, 3, tupdesc, &isnull)); 352 | toastTupleAddToList(makeToastTuple(VARSIZE(chunk_data) - VARHDRSZ, VARDATA(chunk_data), chunk_id, chunk_seq)); 353 | return; 354 | } 355 | else 356 | { 357 | rrctl.sqlkind = PG_LOGMINER_SQLKIND_INSERT; 358 | mentalTup(&tupledata, tupdesc, &rrctl.tupinfo, false); 359 | *tuple_info = rrctl.tupinfo.sqlStr; 360 | rrctl.sqlkind = 0; 361 | } 362 | } 363 | } 364 | 365 | /* 366 | * Get useful data from record,and return delete data by tuple_info. 367 | */ 368 | 369 | static void 370 | getTupleData_Delete(XLogReaderState *record, char** tuple_info, Oid reloid) 371 | { 372 | HeapTupleData tupledata; 373 | char *tuplem = NULL; 374 | char *data = NULL; 375 | TupleDesc tupdesc = NULL; 376 | uint16 newlen = 0; 377 | HeapTupleHeader htup; 378 | xl_heap_header xlhdr; 379 | RelFileNode target_node; 380 | BlockNumber blkno; 381 | ItemPointerData target_tid; 382 | xl_heap_delete *xlrec = (xl_heap_delete *) XLogRecGetData(record); 383 | 384 | if(!rrctl.tupinfo_init) 385 | { 386 | memset(&rrctl.tupinfo, 0, sizeof(XLogMinerSQL)); 387 | rrctl.tupinfo_init = true; 388 | } 389 | else 390 | cleanSpace(&rrctl.tupinfo); 391 | memset(&tupledata, 0, sizeof(HeapTupleData)); 392 | 393 | XLogRecGetBlockTag(record, 0, &target_node, NULL, &blkno); 394 | ItemPointerSetBlockNumber(&target_tid, blkno); 395 | ItemPointerSetOffsetNumber(&target_tid, xlrec->offnum); 396 | 397 | data = (char *) xlrec + SizeOfHeapDelete; 398 | if(!data) 399 | return; 400 | newlen = XLogRecGetDataLen(record) - SizeOfHeapDelete; 401 | if(!(XLH_DELETE_CONTAINS_OLD & xlrec->flags)) 402 | return; 403 | 404 | memcpy((char *) &xlhdr, data, SizeOfHeapHeader); 405 | data += SizeOfHeapHeader; 406 | if(newlen + SizeOfHeapUpdate > MaxHeapTupleSize) 407 | { 408 | rrctl.tuplem_bigold = getTuplemSpace(newlen + SizeOfHeapUpdate + SizeofHeapTupleHeader); 409 | tuplem = rrctl.tuplem_bigold; 410 | } 411 | else 412 | tuplem = rrctl.tuplem; 413 | rrctl.reloid = reloid; 414 | htup = (HeapTupleHeader)tuplem; 415 | memcpy((char *) htup + SizeofHeapTupleHeader,data,newlen); 416 | newlen += SizeofHeapTupleHeader; 417 | htup->t_infomask2 = xlhdr.t_infomask2; 418 | htup->t_infomask = xlhdr.t_infomask; 419 | htup->t_hoff = xlhdr.t_hoff; 420 | HeapTupleHeaderSetXmin(htup, XLogRecGetXid(record)); 421 | HeapTupleHeaderSetCmin(htup, FirstCommandId); 422 | htup->t_ctid = target_tid; 423 | if(rrctl.tupdesc) 424 | { 425 | freetupdesc(rrctl.tupdesc); 426 | rrctl.tupdesc = NULL; 427 | } 428 | 429 | rrctl.tupdesc = GetDescrByreloid(reloid); 430 | tupdesc = rrctl.tupdesc; 431 | if(NULL != htup) 432 | { 433 | tupledata.t_data = htup; 434 | if(rrctl.toastrel) 435 | { 436 | /*if it is insert into a toast table,store it into list*/ 437 | Oid chunk_id; 438 | int chunk_seq; 439 | char *chunk_data; 440 | bool isnull = false; 441 | 442 | chunk_id = DatumGetObjectId(fastgetattr(&tupledata, 1, tupdesc, &isnull)); 443 | chunk_seq =DatumGetInt32(fastgetattr(&tupledata, 2, tupdesc, &isnull)); 444 | chunk_data = DatumGetPointer(fastgetattr(&tupledata, 3, tupdesc, &isnull)); 445 | toastTupleAddToList(makeToastTuple(VARSIZE(chunk_data) - VARHDRSZ, VARDATA(chunk_data), chunk_id, chunk_seq)); 446 | return; 447 | } 448 | else 449 | { 450 | rrctl.sqlkind = PG_LOGMINER_SQLKIND_DELETE; 451 | mentalTup(&tupledata, tupdesc, &rrctl.tupinfo, false); 452 | *tuple_info = rrctl.tupinfo.sqlStr; 453 | rrctl.sqlkind = 0; 454 | } 455 | } 456 | } 457 | 458 | 459 | /* 460 | * Get useful data from record,and return update data by tuple_info and tuple_info_old. 461 | */ 462 | static void 463 | getTupleData_Update(XLogReaderState *record, char** tuple_info, char** tuple_info_old,Oid reloid) 464 | { 465 | xl_heap_update *xlrec = NULL; 466 | xl_heap_header *xlhdr = NULL; 467 | xl_heap_header *xlhdr_old; 468 | uint32 newlen = 0; 469 | 470 | char *tuplem; 471 | char *tuplem_old; 472 | HeapTupleHeader htup; 473 | HeapTupleHeader htup_old; 474 | TupleDesc tupdesc = NULL; 475 | HeapTupleData tupledata; 476 | 477 | Size datalen = 0; 478 | char *recdata = NULL; 479 | char *newp = NULL; 480 | RelFileNode target_node; 481 | 482 | ItemPointerData target_tid; 483 | ItemPointerData target_tid_old; 484 | BlockNumber newblk; 485 | BlockNumber oldblk; 486 | if(!rrctl.tupinfo_init) 487 | { 488 | memset(&rrctl.tupinfo, 0, sizeof(XLogMinerSQL)); 489 | rrctl.tupinfo_init = true; 490 | } 491 | else 492 | cleanSpace(&rrctl.tupinfo); 493 | if(!rrctl.tupinfo_old_init) 494 | { 495 | memset(&rrctl.tupinfo_old, 0, sizeof(XLogMinerSQL)); 496 | rrctl.tupinfo_old_init = true; 497 | } 498 | else 499 | cleanSpace(&rrctl.tupinfo_old); 500 | memset(&tupledata, 0, sizeof(HeapTupleData)); 501 | 502 | xlrec = (xl_heap_update *) XLogRecGetData(record); 503 | XLogRecGetBlockTag(record, 0, &target_node, NULL, &newblk); 504 | if (!XLogRecGetBlockTag(record, 1, NULL, NULL, &oldblk)) 505 | oldblk = newblk; 506 | ItemPointerSet(&target_tid, newblk, xlrec->new_offnum); 507 | ItemPointerSet(&target_tid_old, oldblk, xlrec->old_offnum); 508 | 509 | if(xlrec->flags & XLH_UPDATE_CONTAINS_NEW_TUPLE) 510 | { 511 | recdata = XLogRecGetBlockData(record, 0, &datalen); 512 | 513 | newlen = datalen - SizeOfHeapHeader; 514 | xlhdr = (xl_heap_header *)recdata; 515 | recdata += SizeOfHeapHeader; 516 | tuplem = rrctl.tuplem; 517 | rrctl.reloid = reloid; 518 | htup = (HeapTupleHeader)tuplem; 519 | 520 | newp = (char *) htup + offsetof(HeapTupleHeaderData, t_bits); 521 | memcpy(newp, recdata, newlen); 522 | recdata += newlen; 523 | newp += newlen; 524 | htup->t_infomask2 = xlhdr->t_infomask2; 525 | htup->t_infomask = xlhdr->t_infomask; 526 | 527 | 528 | htup->t_hoff = xlhdr->t_hoff; 529 | HeapTupleHeaderSetXmin(htup, XLogRecGetXid(record)); 530 | HeapTupleHeaderSetCmin(htup, FirstCommandId); 531 | htup->t_ctid = target_tid; 532 | 533 | 534 | if(rrctl.tupdesc) 535 | { 536 | freetupdesc(rrctl.tupdesc); 537 | rrctl.tupdesc = NULL; 538 | } 539 | rrctl.tupdesc = GetDescrByreloid(reloid); 540 | tupdesc = rrctl.tupdesc; 541 | } 542 | else 543 | return; 544 | 545 | if(xlrec->flags & XLH_UPDATE_CONTAINS_OLD) 546 | { 547 | recdata = XLogRecGetData(record) + SizeOfHeapUpdate; 548 | datalen = XLogRecGetDataLen(record) - SizeOfHeapUpdate; 549 | if(datalen + SizeOfHeapUpdate > MaxHeapTupleSize) 550 | { 551 | rrctl.tuplem_bigold = getTuplemSpace(datalen + SizeOfHeapUpdate + SizeofHeapTupleHeader); 552 | tuplem_old = rrctl.tuplem_bigold; 553 | } 554 | else 555 | { 556 | tuplem_old = rrctl.tuplem_old; 557 | } 558 | htup_old = (HeapTupleHeader)tuplem_old; 559 | xlhdr_old = (xl_heap_header *)recdata; 560 | recdata += SizeOfHeapHeader; 561 | newp = (char *) htup_old + offsetof(HeapTupleHeaderData, t_bits); 562 | memcpy(newp, recdata, datalen); 563 | newp += datalen ; 564 | recdata += datalen ; 565 | 566 | htup_old->t_infomask2 = xlhdr_old->t_infomask2; 567 | htup_old->t_infomask = xlhdr_old->t_infomask; 568 | 569 | htup_old->t_hoff = xlhdr_old->t_hoff; 570 | HeapTupleHeaderSetXmin(htup_old, XLogRecGetXid(record)); 571 | HeapTupleHeaderSetCmin(htup_old, FirstCommandId); 572 | htup_old->t_ctid = target_tid_old; 573 | } 574 | else 575 | return; 576 | 577 | 578 | if(NULL != htup) 579 | { 580 | rrctl.sqlkind = PG_LOGMINER_SQLKIND_UPDATE; 581 | tupledata.t_data = htup_old; 582 | mentalTup(&tupledata, tupdesc, &rrctl.tupinfo_old, true); 583 | *tuple_info_old = rrctl.tupinfo_old.sqlStr; 584 | tupledata.t_data = htup; 585 | mentalTup(&tupledata, tupdesc, &rrctl.tupinfo, false); 586 | *tuple_info = rrctl.tupinfo.sqlStr; 587 | rrctl.sqlkind = 0; 588 | } 589 | } 590 | 591 | /*Func control that if we reached the valid record*/ 592 | void 593 | processContrl(char* relname, int contrlkind) 594 | { 595 | 596 | if(PG_LOGMINER_XLOG_NOMAL == rrctl.system_init_record) 597 | return; 598 | if(PG_LOGMINER_CONTRLKIND_FIND == contrlkind && 599 | (0 == strcmp(relname,PG_LOGMINER_DATABASE_HIGHGO) || 0 == strcmp(relname,PG_LOGMINER_DATABASE_POSTGRES))) 600 | { 601 | /*We have got "highgo" or "postgres" db create sql.*/ 602 | rrctl.sysstoplocation = PG_LOGMINER_FLAG_FINDHIGHGO; 603 | } 604 | else if(PG_LOGMINER_CONTRLKIND_XACT == contrlkind) 605 | { 606 | rrctl.sysstoplocation++; 607 | if(PG_LOGMINER_FLAG_INITOVER == rrctl.sysstoplocation) 608 | { 609 | /*We got xact commit just after db create mentioned above*/ 610 | rrctl.system_init_record = PG_LOGMINER_XLOG_NOMAL; 611 | } 612 | } 613 | } 614 | 615 | 616 | static bool 617 | getTupleInfoByRecord(XLogReaderState *record, uint8 info, NameData* relname,char** schname, char** tuple_info, char** tuple_info_old) 618 | { 619 | 620 | RelFileNode *node = NULL; 621 | Oid reloid = 0; 622 | Oid dboid = 0; 623 | uint8 rmid = XLogRecGetRmid(record); 624 | BlockNumber blknum = 0; 625 | 626 | dboid = getDataDicOid(); 627 | cleanMentalvalues(); 628 | 629 | XLogRecGetBlockTag(record, 0, &srctl.rfnode, NULL, &blknum); 630 | node = &srctl.rfnode; 631 | if(dboid != node->dbNode) 632 | return false; 633 | 634 | /*Get which relation it was belonged to*/ 635 | reloid = getRelationOidByRelfileid(node->relNode); 636 | if(0 == reloid) 637 | { 638 | rrctl.reloid = 0; 639 | rrctl.nomalrel = true; 640 | /*ereport(NOTICE,(errmsg("Relfilenode %d can not be handled",node->relNode)));*/ 641 | return false; 642 | } 643 | 644 | if(-1 == getRelationNameByOid(reloid, relname)) 645 | return false; 646 | *schname = getnsNameByReloid(reloid); 647 | rrctl.sysrel = tableIfSysclass(relname->data,reloid); 648 | rrctl.nomalrel = (!rrctl.sysrel) && (!tableIftoastrel(reloid)); 649 | rrctl.imprel = tableifImpSysclass(relname->data,reloid); 650 | rrctl.toastrel = tableIftoastrel(reloid); 651 | rrctl.recordxid = XLogRecGetXid(record); 652 | if(rrctl.nomalrel) 653 | { 654 | rrctl.tbsoid = node->spcNode; 655 | } 656 | /*We does not care unuseful catalog relation 657 | We does not care update toast relation*/ 658 | if(!rrctl.nomalrel && !rrctl.toastrel) 659 | return false; 660 | if(XLOG_HEAP_INSERT == info && RM_HEAP_ID == rmid) 661 | { 662 | getTupleData_Insert(record, tuple_info, reloid); 663 | } 664 | else if((XLOG_HEAP_HOT_UPDATE == info || XLOG_HEAP_UPDATE == info) && RM_HEAP_ID == rmid) 665 | { 666 | getTupleData_Update(record, tuple_info, tuple_info_old, reloid); 667 | } 668 | else if(XLOG_HEAP_DELETE == info && RM_HEAP_ID == rmid) 669 | { 670 | getTupleData_Delete(record, tuple_info, reloid); 671 | } 672 | return true; 673 | } 674 | 675 | static void 676 | minerHeapInsert(XLogReaderState *record, XLogMinerSQL *sql_simple,uint8 info) 677 | { 678 | NameData relname; 679 | char* schname; 680 | bool sqlFind = false; 681 | char *tupleInfo = NULL; 682 | bool nomalrel = false; 683 | bool sysrel = false; 684 | 685 | memset(&relname, 0, sizeof(NameData)); 686 | sqlFind = getTupleInfoByRecord(record, info, &relname, &schname, &tupleInfo ,NULL); 687 | if(!sqlFind) 688 | return; 689 | nomalrel = rrctl.nomalrel; 690 | sysrel = rrctl.sysrel; 691 | /*Assemble "table name","tuple data" and "describe word"*/ 692 | getInsertSQL(sql_simple,tupleInfo,&relname, schname, sysrel); 693 | if(nomalrel && elemNameFind(relname.data) && 0 == rrctl.prostatu) 694 | { 695 | /*Get undo sql*/ 696 | getDeleteSQL(&srctl.sql_undo,tupleInfo,&relname, schname, sysrel, true); 697 | /* 698 | *Format delete sql 699 | *"where values(1,2,3);"-->"where i = 1 AND j = 2 AND k = 3;" 700 | */ 701 | reAssembleDeleteSql(&srctl.sql_undo, true); 702 | } 703 | } 704 | 705 | static void 706 | minerHeapDelete(XLogReaderState *record, XLogMinerSQL *sql_simple,uint8 info) 707 | { 708 | bool sqlFind = false; 709 | NameData relname; 710 | char* schname; 711 | char *tupleInfo = NULL; 712 | bool nomalrel = false; 713 | bool sysrel = false; 714 | 715 | memset(&relname, 0, sizeof(NameData)); 716 | sqlFind = getTupleInfoByRecord(record, info, &relname, &schname, &tupleInfo, NULL); 717 | if(!sqlFind) 718 | return; 719 | nomalrel = rrctl.nomalrel; 720 | sysrel = rrctl.sysrel; 721 | /*Assemble "table name","tuple data" and "describe word"*/ 722 | getDeleteSQL(sql_simple,tupleInfo,&relname,schname,sysrel, false); 723 | if(nomalrel && elemNameFind(relname.data) && 0 == rrctl.prostatu) 724 | { 725 | /*Get undo sql*/ 726 | getInsertSQL(&srctl.sql_undo,tupleInfo,&relname,schname,sysrel); 727 | } 728 | } 729 | 730 | static void 731 | minerHeapUpdate(XLogReaderState *record, XLogMinerSQL *sql_simple, uint8 info) 732 | { 733 | 734 | NameData relname; 735 | char* schname; 736 | bool sqlFind = false; 737 | char *tupleInfo = NULL; 738 | char *tupleInfo_old = NULL; 739 | bool nomalrel = false; 740 | bool sysrel = false; 741 | 742 | memset(&relname, 0, sizeof(NameData)); 743 | sqlFind = getTupleInfoByRecord(record, info, &relname, &schname, &tupleInfo, &tupleInfo_old); 744 | if(!sqlFind) 745 | return; 746 | nomalrel = rrctl.nomalrel; 747 | sysrel = rrctl.sysrel; 748 | /*Assemble "table name","tuple data" and "describe word"*/ 749 | getUpdateSQL(sql_simple, tupleInfo, tupleInfo_old, &relname, schname, sysrel); 750 | 751 | if(nomalrel && elemNameFind(relname.data) && 0 == rrctl.prostatu) 752 | { 753 | /*Get undo sql*/ 754 | getUpdateSQL(&srctl.sql_undo, tupleInfo_old, tupleInfo, &relname, schname, sysrel); 755 | /* 756 | *Format update sql 757 | *"update t1 set values(1,2,4) where values(1,2,3);" 758 | *-->"update t1 set j = 4 where i = 1 AND j = 2 AND k = 3" 759 | */ 760 | reAssembleUpdateSql(&srctl.sql_undo,true); 761 | } 762 | } 763 | 764 | static void 765 | minerHeap2MutiInsert(XLogReaderState *record, XLogMinerSQL *sql_simple, uint8 info) 766 | { 767 | RelFileNode rnode; 768 | HeapTupleData tupledata; 769 | NameData relname; 770 | char *schname = NULL; 771 | char *tuple_info = NULL; 772 | xl_heap_multi_insert *xlrec = NULL; 773 | xl_multi_insert_tuple *xlhdr = NULL; 774 | char *data = NULL; 775 | char *tldata = NULL; 776 | Size tuplelen = 0; 777 | 778 | BlockNumber blkno = 0; 779 | ItemPointerData target_tid; 780 | Oid reloid = 0; 781 | HeapTupleHeader htup = NULL; 782 | int datalen = 0; 783 | 784 | 785 | memset(&rnode, 0, sizeof(RelFileNode)); 786 | memset(&tupledata, 0, sizeof(HeapTupleData)); 787 | memset(&relname, 0, sizeof(NameData)); 788 | 789 | XLogRecGetBlockTag(record, 0, &rnode, NULL, &blkno); 790 | if(getDataDicOid() != rnode.dbNode) 791 | return; 792 | 793 | 794 | if(!rrctl.tupinfo_init) 795 | { 796 | memset(&rrctl.tupinfo, 0, sizeof(XLogMinerSQL)); 797 | rrctl.tupinfo_init = true; 798 | } 799 | else 800 | cleanSpace(&rrctl.tupinfo); 801 | 802 | 803 | 804 | xlrec = (xl_heap_multi_insert *) XLogRecGetData(record); 805 | tldata = XLogRecGetBlockData(record, 0, &tuplelen); 806 | 807 | if(!srctl.mutinsert) 808 | data = tldata; 809 | else 810 | data = srctl.multdata; 811 | 812 | if (xlrec->flags & XLH_INSERT_CONTAINS_NEW_TUPLE) 813 | { 814 | ItemPointerSetBlockNumber(&target_tid, blkno); 815 | ItemPointerSetOffsetNumber(&target_tid, xlrec->offsets[srctl.sqlindex]); 816 | xlhdr = (xl_multi_insert_tuple *) SHORTALIGN(data); 817 | data = ((char *) xlhdr) + SizeOfMultiInsertTuple; 818 | datalen = xlhdr->datalen; 819 | 820 | htup = (HeapTupleHeader)rrctl.tuplem; 821 | 822 | memcpy((char*)htup + SizeofHeapTupleHeader, (char*)data, datalen); 823 | data += datalen; 824 | srctl.multdata = data; 825 | 826 | htup->t_infomask = xlhdr->t_infomask; 827 | htup->t_infomask2 = xlhdr->t_infomask2; 828 | htup->t_hoff = xlhdr->t_hoff; 829 | 830 | HeapTupleHeaderSetXmin(htup, XLogRecGetXid(record)); 831 | HeapTupleHeaderSetCmin(htup, FirstCommandId); 832 | htup->t_ctid = target_tid; 833 | 834 | /*Get which relation it was belonged to*/ 835 | reloid = getRelationOidByRelfileid(rnode.relNode); 836 | if(0 == reloid) 837 | { 838 | rrctl.reloid = 0; 839 | rrctl.nomalrel = true; 840 | getInsertSQL(sql_simple,tuple_info,&relname, schname, rrctl.sysrel); 841 | /* ereport(NOTICE,(errmsg("Relfilenode %d can not be handled",rnode.relNode)));*/ 842 | return; 843 | } 844 | rrctl.reloid = reloid; 845 | if( -1 == getRelationNameByOid(reloid, &relname)) 846 | return; 847 | schname = getnsNameByReloid(reloid); 848 | rrctl.sysrel = tableIfSysclass(relname.data,reloid); 849 | rrctl.nomalrel = (!rrctl.sysrel) && (!tableIftoastrel(reloid)); 850 | rrctl.imprel = tableifImpSysclass(relname.data,reloid); 851 | rrctl.tbsoid = rnode.spcNode; 852 | rrctl.recordxid = XLogRecGetXid(record); 853 | if(rrctl.tupdesc) 854 | { 855 | freetupdesc(rrctl.tupdesc); 856 | rrctl.tupdesc = NULL; 857 | } 858 | rrctl.tupdesc = GetDescrByreloid(reloid); 859 | 860 | if(NULL != htup) 861 | { 862 | rrctl.sqlkind = PG_LOGMINER_SQLKIND_INSERT; 863 | tupledata.t_data = htup; 864 | mentalTup(&tupledata, rrctl.tupdesc, &rrctl.tupinfo, false); 865 | if(!rrctl.tupinfo.sqlStr) 866 | rrctl.prostatu = LOGMINER_PROSTATUE_INSERT_MISSING_TUPLEINFO; 867 | rrctl.sqlkind = 0; 868 | tuple_info = rrctl.tupinfo.sqlStr; 869 | getInsertSQL(sql_simple,tuple_info,&relname, schname, rrctl.sysrel); 870 | if(rrctl.nomalrel && elemNameFind(relname.data) && 0 == rrctl.prostatu) 871 | { 872 | /*Get undo sql*/ 873 | getDeleteSQL(&srctl.sql_undo,tuple_info,&relname, schname, rrctl.sysrel, true); 874 | /* 875 | *Format delete sql 876 | *"where values(1,2,3);"-->"where i = 1 AND j = 2 AND k = 3;" 877 | */ 878 | reAssembleDeleteSql(&srctl.sql_undo, true); 879 | srctl.sqlindex++; 880 | if(srctl.sqlindex >= xlrec->ntuples) 881 | srctl.mutinsert = false; 882 | else 883 | srctl.mutinsert = true; 884 | } 885 | 886 | } 887 | } 888 | } 889 | 890 | /*find next xlog record and store into rrctl.xlogreader_state*/ 891 | static bool 892 | getNextRecord() 893 | { 894 | XLogRecord *record_t; 895 | record_t = XLogReadRecord_logminer(rrctl.xlogreader_state, rrctl.first_record, &rrctl.errormsg); 896 | rrctl.first_record = InvalidXLogRecPtr; 897 | if (!record_t) 898 | { 899 | return false; 900 | } 901 | return true; 902 | } 903 | 904 | 905 | static int 906 | getsqlkind(char *sqlheader) 907 | { 908 | int loop,result = -1; 909 | for(loop = 0 ;loop < PG_LOGMINER_SQLKIND_MAXNUM; loop++) 910 | { 911 | if(0 == strcmp(sqlheader,sqlkind[loop].sqlhead)) 912 | result = sqlkind[loop].sqlid; 913 | } 914 | return result; 915 | } 916 | 917 | 918 | /*Parser all kinds of insert sql, and make decide what sql it will form*/ 919 | static bool 920 | parserInsertSql(XLogMinerSQL *sql_ori, XLogMinerSQL *sql_opt) 921 | { 922 | char tarTable[NAMEDATALEN] = {0}; 923 | 924 | 925 | 926 | getPhrases(sql_ori->sqlStr, LOGMINER_INSERT_TABLE_NAME, tarTable, 0); 927 | 928 | /*It just insert to a user's table*/ 929 | if(rrctl.nomalrel && (elemNameFind(tarTable) || 0 == strcmp("NULL",tarTable))) 930 | { 931 | appendtoSQL(sql_opt,sql_ori->sqlStr,PG_LOGMINER_SQLPARA_SIMSTEP); 932 | /*Here reached,it is not in toat,so try to free tthead*/ 933 | freeToastTupleHead(); 934 | return true; 935 | } 936 | return false; 937 | } 938 | 939 | /*Parser all kinds of delete sql, and make decide what sql it will form*/ 940 | static bool 941 | parserDeleteSql(XLogMinerSQL *sql_ori, XLogMinerSQL *sql_opt) 942 | { 943 | 944 | char tarTable[NAMEDATALEN] = {0}; 945 | 946 | 947 | 948 | getPhrases(sql_ori->sqlStr, LOGMINER_DELETE_TABLE_NAME, tarTable, 0); 949 | if(rrctl.nomalrel && (elemNameFind(tarTable) || 0 == strcmp("NULL",tarTable))) 950 | { 951 | reAssembleDeleteSql(sql_ori, false); 952 | freeToastTupleHead(); 953 | appendtoSQL(sql_opt,sql_ori->sqlStr,PG_LOGMINER_SQLPARA_SIMSTEP); 954 | } 955 | return true; 956 | } 957 | 958 | /*Parser all kinds of update sql, and make decide what sql it will form*/ 959 | static bool 960 | parserUpdateSql(XLogMinerSQL *sql_ori, XLogMinerSQL *sql_opt) 961 | { 962 | char tarTable[NAMEDATALEN] = {0}; 963 | 964 | 965 | 966 | getPhrases(sql_ori->sqlStr, LOGMINER_ATTRIBUTE_LOCATION_UPDATE_RELNAME, tarTable, 0); 967 | if(rrctl.nomalrel && (elemNameFind(tarTable) || 0 == strcmp("NULL",tarTable))) 968 | { 969 | reAssembleUpdateSql(sql_ori, false); 970 | freeToastTupleHead(); 971 | if(sql_ori->sqlStr) 972 | appendtoSQL(sql_opt,sql_ori->sqlStr,PG_LOGMINER_SQLPARA_SIMSTEP); 973 | } 974 | return true; 975 | } 976 | 977 | static void 978 | parserXactSql(XLogMinerSQL *sql_ori, XLogMinerSQL *sql_opt) 979 | { 980 | appendtoSQL(sql_opt,sql_ori->sqlStr,PG_LOGMINER_SQLPARA_SIMSTEP); 981 | } 982 | 983 | 984 | static void 985 | XLogMinerRecord_heap(XLogReaderState *record, XLogMinerSQL *sql_simple) 986 | { 987 | uint8 info; 988 | 989 | 990 | 991 | 992 | 993 | info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; 994 | info &= XLOG_HEAP_OPMASK; 995 | 996 | if (XLOG_HEAP_INSERT == info) 997 | { 998 | minerHeapInsert(record, sql_simple, info); 999 | } 1000 | else if(XLOG_HEAP_DELETE == info) 1001 | { 1002 | minerHeapDelete(record, sql_simple, info); 1003 | } 1004 | else if (XLOG_HEAP_UPDATE == info) 1005 | { 1006 | minerHeapUpdate(record, sql_simple, info); 1007 | } 1008 | else if (XLOG_HEAP_HOT_UPDATE == info) 1009 | { 1010 | minerHeapUpdate(record, sql_simple, info); 1011 | } 1012 | } 1013 | 1014 | static void 1015 | XLogMinerRecord_heap2(XLogReaderState *record, XLogMinerSQL *sql_simple) 1016 | { 1017 | uint8 info = XLogRecGetInfo(record) & XLOG_HEAP_OPMASK; 1018 | 1019 | if(XLOG_HEAP2_MULTI_INSERT == info) 1020 | { 1021 | minerHeap2MutiInsert(record, sql_simple, info); 1022 | } 1023 | 1024 | } 1025 | 1026 | static void 1027 | XLogMinerRecord_dbase(XLogReaderState *record, XLogMinerSQL *sql_simple) 1028 | { 1029 | uint8 info; 1030 | 1031 | 1032 | info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; 1033 | if(XLOG_DBASE_CREATE == info) 1034 | { 1035 | minerDbCreate(record, sql_simple, info); 1036 | } 1037 | } 1038 | 1039 | static void 1040 | XLogMinerRecord_xact(XLogReaderState *record, XLogMinerSQL *sql_simple, TimestampTz *xacttime) 1041 | { 1042 | uint8 info = XLogRecGetInfo(record) & XLOG_XACT_OPMASK; 1043 | xl_xact_parsed_commit parsed_commit; 1044 | char timebuf[MAXDATELEN + 1] = {0}; 1045 | TransactionId xid = 0; 1046 | bool commitxact = false; 1047 | 1048 | if(info == XLOG_XACT_COMMIT) 1049 | { 1050 | xl_xact_commit *xlrec = NULL; 1051 | memset(&parsed_commit, 0, sizeof(xl_xact_parsed_commit)); 1052 | xlrec = (xl_xact_commit *) XLogRecGetData(record); 1053 | ParseCommitRecord(XLogRecGetInfo(record), xlrec, &parsed_commit); 1054 | if (!TransactionIdIsValid(parsed_commit.twophase_xid)) 1055 | xid = XLogRecGetXid(record); 1056 | else 1057 | xid = parsed_commit.twophase_xid; 1058 | 1059 | memcpy(timebuf,timestamptz_to_str(xlrec->xact_time),MAXDATELEN + 1); 1060 | 1061 | *xacttime = xlrec->xact_time; 1062 | commitxact = true; 1063 | } 1064 | else if(info == XLOG_XACT_ABORT) 1065 | { 1066 | xl_xact_abort *xlrec = NULL; 1067 | xl_xact_parsed_abort parsed_abort; 1068 | 1069 | memset(&parsed_abort, 0, sizeof(xl_xact_parsed_abort)); 1070 | xlrec = (xl_xact_abort *) XLogRecGetData(record); 1071 | ParseAbortRecord(XLogRecGetInfo(record), xlrec, &parsed_abort); 1072 | if (!TransactionIdIsValid(parsed_abort.twophase_xid)) 1073 | xid = XLogRecGetXid(record); 1074 | else 1075 | xid = parsed_abort.twophase_xid; 1076 | 1077 | memcpy(timebuf,timestamptz_to_str(xlrec->xact_time),MAXDATELEN + 1); 1078 | *xacttime = xlrec->xact_time; 1079 | } 1080 | 1081 | processContrl(NULL,PG_LOGMINER_CONTRLKIND_XACT); 1082 | if(curXactCheck(*xacttime, xid, commitxact, &parsed_commit)) 1083 | { 1084 | xactCommitSQL(timebuf,sql_simple,info); 1085 | } 1086 | } 1087 | 1088 | static bool 1089 | XLogMinerRecord(XLogReaderState *record, XLogMinerSQL *sql_simple,TimestampTz *xacttime) 1090 | { 1091 | uint8 info; 1092 | bool getxact = false; 1093 | uint8 rmid = 0; 1094 | 1095 | rmid = XLogRecGetRmid(record); 1096 | 1097 | info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; 1098 | info &= XLOG_HEAP_OPMASK; 1099 | 1100 | /*if has not get valid record(input para) and it is not a xact record,parser nothing*/ 1101 | if(PG_LOGMINER_XLOG_NOMAL == rrctl.system_init_record && !rrctl.logprivate.staptr_reached && RM_XACT_ID != rmid) 1102 | return false; 1103 | 1104 | if((RM_DBASE_ID == rmid || PG_LOGMINER_XLOG_DBINIT == rrctl.system_init_record) 1105 | && (0 == rrctl.sysstoplocation || PG_LOGMINER_FLAG_INITOVER == rrctl.sysstoplocation)) 1106 | { 1107 | if(RM_DBASE_ID != rmid) 1108 | return false; 1109 | /*If we find 'postgresql' or 'highgo' db create cade,than it may will reach valid data after a xact commit*/ 1110 | XLogMinerRecord_dbase(record, sql_simple); 1111 | } 1112 | else if(RM_XACT_ID == rmid || PG_LOGMINER_FLAG_INITOVER != rrctl.sysstoplocation) 1113 | { 1114 | if(RM_XACT_ID != rmid) 1115 | return false; 1116 | /* 1117 | if first time reach here,then we just find 'postgresql' or 'highgo' db create code,than it may be will reach valid data 1118 | */ 1119 | XLogMinerRecord_xact(record, sql_simple, xacttime); 1120 | getxact = true; 1121 | } 1122 | else if(RM_HEAP_ID == rmid) 1123 | { 1124 | XLogMinerRecord_heap(record, sql_simple); 1125 | } 1126 | else if(RM_HEAP2_ID == rmid) 1127 | { 1128 | XLogMinerRecord_heap2(record, sql_simple); 1129 | } 1130 | 1131 | return getxact; 1132 | } 1133 | 1134 | 1135 | static bool 1136 | sqlParser(XLogReaderState *record, TimestampTz *xacttime) 1137 | { 1138 | char command_sql[NAMEDATALEN] = {0}; 1139 | 1140 | XLogMinerSQL sql_reass; 1141 | XLogMinerSQL sql; 1142 | int sskind = 0; 1143 | 1144 | bool getxact = false; 1145 | bool getsql = false; 1146 | Oid server_dboid = 0; 1147 | 1148 | memset(&sql_reass, 0, sizeof(XLogMinerSQL)); 1149 | memset(&sql, 0, sizeof(XLogMinerSQL)); 1150 | 1151 | /*parsert data that stored in a record to a simple sql */ 1152 | getxact = XLogMinerRecord(record, &sql, xacttime); 1153 | 1154 | 1155 | /*avoid to parse the initdb record*/ 1156 | if(rrctl.system_init_record != PG_LOGMINER_XLOG_NOMAL) 1157 | return false; 1158 | 1159 | getPhrases(sql.sqlStr, LOGMINER_SQL_COMMAND, command_sql, 0); 1160 | sskind = getsqlkind(command_sql); 1161 | 1162 | if(0 == rrctl.prostatu) 1163 | { 1164 | /*get a sql nomally*/ 1165 | /*Deal with every simple sql*/ 1166 | switch(sskind) 1167 | { 1168 | case PG_LOGMINER_SQLKIND_INSERT: 1169 | getsql = parserInsertSql(&sql, &sql_reass); 1170 | break; 1171 | case PG_LOGMINER_SQLKIND_UPDATE: 1172 | getsql = parserUpdateSql(&sql, &sql_reass); 1173 | break; 1174 | case PG_LOGMINER_SQLKIND_DELETE: 1175 | getsql = parserDeleteSql(&sql, &sql_reass); 1176 | break; 1177 | case PG_LOGMINER_SQLKIND_XACT: 1178 | parserXactSql(&sql, &sql_reass); 1179 | break; 1180 | default: 1181 | break; 1182 | } 1183 | } 1184 | else 1185 | { 1186 | /*Get a unnomal sql,and junk it*/ 1187 | rrctl.prostatu = 0; 1188 | } 1189 | 1190 | if(!isEmptStr(sql_reass.sqlStr) && !getxact) 1191 | { 1192 | /*Now, we get a SQL, it need to be store tempory.*/ 1193 | char *record_schema = NULL; 1194 | char *record_database = NULL; 1195 | char *record_user = NULL; 1196 | char *record_tablespace = NULL; 1197 | Oid useroid = 0, schemaoid = 0; 1198 | 1199 | 1200 | if(getsql) 1201 | { 1202 | /*It is a simple sql,DML*/ 1203 | server_dboid = getDataDicOid(); 1204 | record_database = getdbNameByoid(server_dboid, false); 1205 | record_tablespace = gettbsNameByoid(rrctl.tbsoid); 1206 | if(0 != rrctl.reloid) 1207 | { 1208 | useroid = gettuserOidByReloid(rrctl.reloid); 1209 | if(0 != useroid) 1210 | record_user = getuserNameByUseroid(useroid); 1211 | 1212 | schemaoid = getnsoidByReloid(rrctl.reloid); 1213 | if(0 != schemaoid) 1214 | record_schema = getnsNameByOid(schemaoid); 1215 | } 1216 | padingminerXlogconts(record_schema, 0, Anum_xlogminer_contents_record_schema, schemaoid); 1217 | padingminerXlogconts(record_user, 0, Anum_xlogminer_contents_record_user, useroid); 1218 | padingminerXlogconts(record_database, 0, Anum_xlogminer_contents_record_database, server_dboid); 1219 | padingminerXlogconts(record_tablespace, 0, Anum_xlogminer_contents_record_tablespace, rrctl.tbsoid); 1220 | padingminerXlogconts(srctl.sql_undo.sqlStr, 0, Anum_xlogminer_contents_op_undo, -1); 1221 | } 1222 | else 1223 | { 1224 | /*It is a assemble sql,DDL*/ 1225 | server_dboid = getDataDicOid(); 1226 | record_database = getdbNameByoid(server_dboid,false); 1227 | padingminerXlogconts(record_database, 0, Anum_xlogminer_contents_record_database, server_dboid); 1228 | } 1229 | getPhrases(sql_reass.sqlStr, LOGMINER_SQL_COMMAND, command_sql, 0); 1230 | padingminerXlogconts(sql_reass.sqlStr, 0, Anum_xlogminer_contents_op_text, -1); 1231 | padingminerXlogconts(command_sql, 0, Anum_xlogminer_contents_op_type, -1); 1232 | padingminerXlogconts(NULL,rrctl.recordxid, Anum_xlogminer_contents_xid, -1); 1233 | 1234 | srctl.xcfcurnum++; 1235 | padNullToXC(); 1236 | cleanAnalyseInfo(); 1237 | } 1238 | cleanTuplemSpace(rrctl.tuplem); 1239 | cleanTuplemSpace(rrctl.tuplem_old); 1240 | if(rrctl.tuplem_bigold) 1241 | pfree(rrctl.tuplem_bigold); 1242 | rrctl.tuplem_bigold = NULL; 1243 | rrctl.nomalrel = false; 1244 | rrctl.imprel = false; 1245 | rrctl.sysrel = false; 1246 | rrctl.toastrel = false; 1247 | freeSpace(&sql); 1248 | freeSpace(&sql_reass); 1249 | return getxact; 1250 | } 1251 | 1252 | Datum 1253 | xlogminer_build_dictionary(PG_FUNCTION_ARGS) 1254 | { 1255 | text *dictionary = NULL; 1256 | cleanSystableDictionary(); 1257 | checkLogminerUser(); 1258 | if(!PG_GETARG_DATUM(0)) 1259 | ereport(ERROR,(errmsg("Please enter a file path or directory."))); 1260 | dictionary = PG_GETARG_TEXT_P(0); 1261 | outputSysTableDictionary(text_to_cstring(dictionary), ImportantSysClass,false); 1262 | PG_RETURN_TEXT_P(cstring_to_text("Dictionary build success!")); 1263 | } 1264 | 1265 | Datum 1266 | xlogminer_load_dictionary(PG_FUNCTION_ARGS) 1267 | { 1268 | text *dictionary = NULL; 1269 | cleanSystableDictionary(); 1270 | checkLogminerUser(); 1271 | if(!PG_GETARG_DATUM(0)) 1272 | ereport(ERROR,(errmsg("Please enter a file path or directory."))); 1273 | dictionary = PG_GETARG_TEXT_P(0); 1274 | if(DataDictionaryCache) 1275 | ereport(ERROR,(errmsg("Dictionary has already been loaded."))); 1276 | loadSystableDictionary(text_to_cstring(dictionary), ImportantSysClass, false); 1277 | writeDicStorePath(text_to_cstring(dictionary)); 1278 | cleanSystableDictionary(); 1279 | PG_RETURN_TEXT_P(cstring_to_text("Dictionary load success!")); 1280 | } 1281 | 1282 | Datum 1283 | xlogminer_stop(PG_FUNCTION_ARGS) 1284 | { 1285 | checkLogminerUser(); 1286 | cleanSystableDictionary(); 1287 | cleanXlogfileList(); 1288 | dropAnalyseFile(); 1289 | PG_RETURN_TEXT_P(cstring_to_text("xlogminer stop!")); 1290 | } 1291 | 1292 | Datum 1293 | xlogminer_xlogfile_add(PG_FUNCTION_ARGS) 1294 | { 1295 | text *xlogfile = NULL; 1296 | char backstr[100] = {0}; 1297 | int addnum = 0; 1298 | int dicloadtype = 0; 1299 | char dic_path[MAXPGPATH] = {0}; 1300 | 1301 | if(!PG_GETARG_DATUM(0)) 1302 | ereport(ERROR,(errmsg("Please enter a file path or directory."))); 1303 | xlogfile = PG_GETARG_TEXT_P(0); 1304 | cleanSystableDictionary(); 1305 | checkLogminerUser(); 1306 | loadXlogfileList(); 1307 | 1308 | loadDicStorePath(dic_path); 1309 | if(0 == dic_path[0]) 1310 | { 1311 | dicloadtype = PG_LOGMINER_DICTIONARY_LOADTYPE_NOTHING; 1312 | } 1313 | else 1314 | { 1315 | loadSystableDictionary(dic_path, ImportantSysClass,true); 1316 | dicloadtype = getDatadictionaryLoadType(); 1317 | } 1318 | if(PG_LOGMINER_DICTIONARY_LOADTYPE_NOTHING == dicloadtype) 1319 | { 1320 | char *datadic = NULL; 1321 | datadic = outputSysTableDictionary(NULL, ImportantSysClass, true); 1322 | loadSystableDictionary(NULL, ImportantSysClass,true); 1323 | writeDicStorePath(dictionary_path); 1324 | ereport(NOTICE,(errmsg("Get data dictionary from current database."))); 1325 | } 1326 | addnum = addxlogfile(text_to_cstring(xlogfile)); 1327 | writeXlogfileList(); 1328 | cleanXlogfileList(); 1329 | cleanSystableDictionary(); 1330 | snprintf(backstr, 100, "%d file add success",addnum); 1331 | PG_RETURN_TEXT_P(cstring_to_text(backstr)); 1332 | } 1333 | 1334 | Datum 1335 | xlogminer_xlogfile_remove(PG_FUNCTION_ARGS) 1336 | { 1337 | text *xlogfile = NULL; 1338 | char backstr[100] = {0}; 1339 | int removenum = 0; 1340 | int dicloadtype = 0; 1341 | char dic_path[MAXPGPATH] = {0}; 1342 | 1343 | cleanSystableDictionary(); 1344 | checkLogminerUser(); 1345 | 1346 | if(!PG_GETARG_DATUM(0)) 1347 | ereport(ERROR,(errmsg("Please enter a file path or directory."))); 1348 | xlogfile = PG_GETARG_TEXT_P(0); 1349 | loadXlogfileList(); 1350 | 1351 | loadDicStorePath(dic_path); 1352 | if(0 == dic_path[0]) 1353 | { 1354 | dicloadtype = PG_LOGMINER_DICTIONARY_LOADTYPE_NOTHING; 1355 | } 1356 | else 1357 | { 1358 | loadSystableDictionary(dic_path, ImportantSysClass,true); 1359 | dicloadtype = getDatadictionaryLoadType(); 1360 | } 1361 | 1362 | if(PG_LOGMINER_DICTIONARY_LOADTYPE_NOTHING == getDatadictionaryLoadType()) 1363 | ereport(ERROR,(errmsg("DataDictionary has not been loaded."))); 1364 | removenum = removexlogfile(text_to_cstring(xlogfile)); 1365 | writeXlogfileList(); 1366 | cleanXlogfileList(); 1367 | snprintf(backstr, 100, "%d file remove success",removenum); 1368 | PG_RETURN_TEXT_P(cstring_to_text(backstr)); 1369 | } 1370 | 1371 | Datum 1372 | xlogminer_xlogfile_list(PG_FUNCTION_ARGS) 1373 | { 1374 | FuncCallContext *funcctx = NULL; 1375 | logminer_fctx *temp_fctx = NULL; 1376 | if (SRF_IS_FIRSTCALL()) 1377 | { 1378 | logminer_fctx *fctx = NULL; 1379 | MemoryContext oldcontext = NULL; 1380 | TupleDesc tupdesc = NULL; 1381 | cleanSystableDictionary(); 1382 | checkLogminerUser(); 1383 | loadXlogfileList(); 1384 | if(!is_xlogfilelist_exist()) 1385 | ereport(ERROR,(errmsg("Xlogfilelist has not been loaded or has been removed."))); 1386 | fctx = (logminer_fctx *)logminer_palloc(sizeof(logminer_fctx),0); 1387 | fctx->hasnextxlogfile= true; 1388 | funcctx = SRF_FIRSTCALL_INIT(); 1389 | funcctx->user_fctx = fctx; 1390 | oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); 1391 | tupdesc = makeOutputXlogDesc(); 1392 | funcctx->attinmeta = TupleDescGetAttInMetadata(tupdesc); 1393 | MemoryContextSwitchTo(oldcontext); 1394 | } 1395 | funcctx = SRF_PERCALL_SETUP(); 1396 | temp_fctx = (logminer_fctx*)funcctx->user_fctx; 1397 | while(temp_fctx->hasnextxlogfile) 1398 | { 1399 | HeapTuple tuple; 1400 | char *values[1]; 1401 | char *xlogfile = NULL; 1402 | 1403 | xlogfile = getNextXlogFile(funcctx->user_fctx, true); 1404 | 1405 | values[0] = xlogfile; 1406 | tuple = BuildTupleFromCStrings(funcctx->attinmeta, values); 1407 | SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tuple)); 1408 | } 1409 | cleanXlogfileList(); 1410 | SRF_RETURN_DONE(funcctx); 1411 | } 1412 | 1413 | /* 1414 | xlog analyse begin here 1415 | */ 1416 | Datum pg_minerXlog(PG_FUNCTION_ARGS) 1417 | { 1418 | TimestampTz xacttime = 0; 1419 | bool getrecord = true; 1420 | bool getxact = false; 1421 | text *starttimestamp = NULL; 1422 | text *endtimestamp = NULL; 1423 | int32 startxid = 0; 1424 | int32 endxid = 0; 1425 | XLogSegNo segno; 1426 | char *directory = NULL; 1427 | char *fname = NULL; 1428 | char *firstxlogfile = NULL; 1429 | int dicloadtype = 0; 1430 | char dictionary[MAXPGPATH] = {0}; 1431 | 1432 | memset(&rrctl, 0, sizeof(RecordRecycleCtl)); 1433 | memset(&sysclass, 0, sizeof(SystemClass)*PG_LOGMINER_SYSCLASS_MAX); 1434 | memset(&srctl, 0, sizeof(SQLRecycleCtl)); 1435 | sqlnoser = 0; 1436 | cleanSystableDictionary(); 1437 | checkLogminerUser(); 1438 | logminer_createMemContext(); 1439 | rrctl.tuplem = getTuplemSpace(0); 1440 | rrctl.tuplem_old = getTuplemSpace(0); 1441 | rrctl.lfctx.sendFile = -1; 1442 | 1443 | if(!PG_GETARG_DATUM(0) || !PG_GETARG_DATUM(1)) 1444 | ereport(ERROR,(errmsg("The time parameter can not be null."))); 1445 | starttimestamp = (text *)PG_GETARG_CSTRING(0); 1446 | endtimestamp = (text *)PG_GETARG_CSTRING(1); 1447 | startxid = PG_GETARG_INT32(2); 1448 | endxid = PG_GETARG_INT32(3); 1449 | if(0 > startxid || 0 > endxid) 1450 | ereport(ERROR,(errmsg("The XID parameters cannot be negative."))); 1451 | 1452 | rrctl.logprivate.parser_start_xid = startxid; 1453 | rrctl.logprivate.parser_end_xid = endxid; 1454 | /*parameter check*/ 1455 | inputParaCheck(starttimestamp, endtimestamp); 1456 | 1457 | loadDicStorePath(dictionary); 1458 | if(0 == dictionary[0]) 1459 | ereport(ERROR,(errmsg("Xlogfilelist must be loaded first."))); 1460 | loadSystableDictionary(dictionary, ImportantSysClass,false); 1461 | dicloadtype = getDatadictionaryLoadType(); 1462 | 1463 | if(PG_LOGMINER_DICTIONARY_LOADTYPE_SELF == dicloadtype) 1464 | { 1465 | char *datadic = NULL; 1466 | cleanSystableDictionary(); 1467 | datadic = outputSysTableDictionary(NULL, ImportantSysClass, true); 1468 | loadSystableDictionary(NULL, ImportantSysClass,true); 1469 | if(datadic) 1470 | remove(datadic); 1471 | } 1472 | 1473 | loadXlogfileList(); 1474 | if(!is_xlogfilelist_exist()) 1475 | ereport(ERROR,(errmsg("Xlogfilelist must be loaded first."))); 1476 | checkXlogFileList(); 1477 | 1478 | searchSysClass(sysclass,&sysclassNum); 1479 | relkind_miner = getRelKindInfo(); 1480 | 1481 | 1482 | firstxlogfile = getNextXlogFile((char*)(&rrctl.lfctx),false); 1483 | rrctl.lfctx.xlogfileptr = NULL; 1484 | split_path_fname(firstxlogfile, &directory, &fname); 1485 | XLogFromFileName(fname, &rrctl.logprivate.timeline, &segno); 1486 | if(fname) 1487 | logminer_pfree(fname,0); 1488 | if(directory) 1489 | logminer_pfree(directory,0); 1490 | 1491 | /* if this wal file include catalog relation info*/ 1492 | if(1 == segno) 1493 | rrctl.system_init_record = PG_LOGMINER_XLOG_DBINIT; 1494 | else 1495 | { 1496 | rrctl.system_init_record = PG_LOGMINER_XLOG_NOMAL; 1497 | rrctl.sysstoplocation = PG_LOGMINER_FLAG_INITOVER; 1498 | } 1499 | 1500 | /*configure call back func*/ 1501 | rrctl.xlogreader_state = XLogReaderAllocate(XLogMinerReadPage, &rrctl.logprivate); 1502 | XLogSegNoOffsetToRecPtr(segno, 0, rrctl.logprivate.startptr); 1503 | if(!rrctl.xlogreader_state) 1504 | ereport(ERROR,(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),errmsg("Out of memory"))); 1505 | rrctl.first_record = XLogFindFirstRecord(rrctl.xlogreader_state, rrctl.logprivate.startptr); 1506 | while(!rrctl.logprivate.endptr_reached) 1507 | { 1508 | /*if in a mutiinsert now, avoid to get new record*/ 1509 | if(!srctl.mutinsert) 1510 | getrecord = getNextRecord(); 1511 | if(getrecord) 1512 | getxact = sqlParser(rrctl.xlogreader_state, &xacttime); 1513 | else if(!rrctl.logprivate.serialwal) 1514 | { 1515 | rrctl.logprivate.serialwal = true; 1516 | rrctl.logprivate.changewal = false; 1517 | rrctl.first_record = XLogFindFirstRecord(rrctl.xlogreader_state, rrctl.logprivate.startptr); 1518 | } 1519 | } 1520 | cleanSystableDictionary(); 1521 | cleanXlogfileList(); 1522 | dropAnalyseFile(); 1523 | freeSQLspace(); 1524 | freeSpace(&srctl.sql_simple); 1525 | freeSpace(&srctl.sql_undo); 1526 | XLogReaderFree(rrctl.xlogreader_state); 1527 | pfree(rrctl.tuplem); 1528 | pfree(rrctl.tuplem_old); 1529 | logminer_switchMemContext(); 1530 | PG_RETURN_TEXT_P(cstring_to_text("xlogminer start!")); 1531 | } 1532 | --------------------------------------------------------------------------------