├── README.md ├── c ├── Makefile └── flip_bit_and_checksum.c ├── compare_disk_mem ├── bt_metap_info.sh ├── bt_page_items_info.sh ├── bt_page_stats_info.sh ├── heap_page_items_info.sh ├── heap_page_items_toast_pointer.sh ├── page_header_info.sh └── toast_index_info.sh ├── from_files_only ├── get_multixid_members.sh └── get_xact_status.sh └── sql └── get_infomasks_insights.sql /README.md: -------------------------------------------------------------------------------- 1 | `pg_toolkit` – Some scripts for postgresql 2 | ============================================================= 3 | 4 | compare_disk_mem directory 5 | -------------------------- 6 | 7 | Some scripts to compare pages in memory and on disk (disk stands for outside Postgres shared buffers) 8 | 9 | * `page_header_info.sh`: to inspect page header from memory and from disk 10 | * `heap_page_items_info.sh`: to inspect heap page items from memory and from disk 11 | * `bt_metap_info.sh`: to inspect B-tree index's metapage from memory and from disk 12 | * `bt_page_items_info.sh`: to inspect B-tree index's page items from memory and from disk 13 | * `bt_page_stats_info.sh`: to get summary information about single pages of B-tree indexes from memory and from disk 14 | 15 | Example: 16 | 17 | $ ./bt_page_stats_info.sh 18 | 19 | Usage: 20 | 21 | -b: block to inspect 22 | -p: relation path on disk 23 | -bt: btree name 24 | -i: where to inspect (disk, memory, both) 25 | 26 | $ ./bt_page_stats_info.sh -b 15 -p /usr/local/pgsql12.0/data/base/13593/139931 -bt bdtidx -i both 27 | 28 | BLOCK = 15 29 | PATH = /usr/local/pgsql12.0/data/base/13593/139931 30 | BTREE = bdtidx 31 | INSPECT = both 32 | 33 | from | blkno | type | live_items | dead_items | avg_item_size | free_size | btpo_prev | btpo_next | btpo | btpo_flags 34 | -----+-------+------+------------+------------+---------------+------------+-----------+-----------+------+----------- 35 | dsk | 15 | l | 239 | 0 | 16 | 3368 | 0 | 111 | 0 | 1 36 | mem | 15 | l | 239 | 0 | 16 | 3368 | 0 | 111 | 0 | 1 37 | 38 | $ psql -c "insert into bdt select generate_series(1, 5000),generate_series(1,5000),'bdt'" 39 | INSERT 0 5000 40 | 41 | $ ./bt_page_stats_info.sh -b 15 -p /usr/local/pgsql12.0/data/base/13593/139931 -bt bdtidx -i both 42 | 43 | BLOCK = 15 44 | PATH = /usr/local/pgsql12.0/data/base/13593/139931 45 | BTREE = bdtidx 46 | INSPECT = both 47 | 48 | from | blkno | type | live_items | dead_items | avg_item_size | free_size | btpo_prev | btpo_next | btpo | btpo_flags 49 | -----+-------+------+------------+------------+---------------+------------+-----------+-----------+------+----------- 50 | dsk | 15 | l | 239 | 0 | 16 | 3368 | 0 | 111 | 0 | 1 51 | mem | 15 | l | 273 | 0 | 16 | 2688 | 0 | 111 | 0 | 1 52 | 53 | $ psql -c "checkpoint" 54 | CHECKPOINT 55 | 56 | $ ./bt_page_stats_info.sh -b 15 -p /usr/local/pgsql12.0/data/base/13593/139931 -bt bdtidx -i both 57 | 58 | BLOCK = 15 59 | PATH = /usr/local/pgsql12.0/data/base/13593/139931 60 | BTREE = bdtidx 61 | INSPECT = both 62 | 63 | from | blkno | type | live_items | dead_items | avg_item_size | free_size | btpo_prev | btpo_next | btpo | btpo_flags 64 | -----+-------+------+------------+------------+---------------+------------+-----------+-----------+------+----------- 65 | dsk | 15 | l | 273 | 0 | 16 | 2688 | 0 | 111 | 0 | 1 66 | mem | 15 | l | 273 | 0 | 16 | 2688 | 0 | 111 | 0 | 1 67 | 68 | from_files_only 69 | -------------------------- 70 | 71 | Some scripts to get information from files only 72 | 73 | * `get_multixid_members.sh`: to retrieve multixid members from pg_multixact/offsets and pg_multixact/members 74 | 75 | Example: 76 | 77 | $ ./get_multixid_members.sh -help 78 | 79 | Usage: 80 | 81 | -m: mxid 82 | -d: DATA path 83 | 84 | $ ./get_multixid_members.sh -m 3406700053 -d /usr/local/pgsql11.6/data 85 | 86 | MXID = 3406700053 87 | DATA = /usr/local/pgsql11.6/data 88 | 89 | Members are: 90 | 1191575939 91 | 1191576073 92 | 1191576075 93 | 94 | # Verification from postgres 95 | 96 | #SELECT * FROM pg_get_multixact_members('3406700053'); 97 | 98 | xid | mode 99 | ------------+------- 100 | 1191575939 | keysh 101 | 1191576073 | keysh 102 | 1191576075 | keysh 103 | 104 | * `get_xact_status.sh`: to retrieve xact status from the pg_xact directory 105 | 106 | Example: 107 | 108 | postgres=# select txid_current_snapshot(); 109 | txid_current_snapshot 110 | ----------------------- 111 | 31795295:31795295: 112 | (1 row) 113 | 114 | postgres=# \! ./get_xact_status.sh -x 31795295 -d /usr/local/pgsql12.1-bench/data 115 | 116 | XID = 31795295 117 | DATA = /usr/local/pgsql12.1-bench/data 118 | 119 | xid 31795295 status is: UNKNOWN 120 | postgres=# 121 | postgres=# insert into bdt values (1); 122 | INSERT 0 1 123 | postgres=# checkpoint; 124 | CHECKPOINT 125 | postgres=# \! ./get_xact_status.sh -x 31795295 -d /usr/local/pgsql12.1-bench/data 126 | 127 | XID = 31795295 128 | DATA = /usr/local/pgsql12.1-bench/data 129 | 130 | Reading bits 7,8 in byte 2583 in page 10 of file 001E 131 | 132 | xid 31795295 status is: COMMITTED 133 | postgres=# 134 | postgres=# begin; 135 | BEGIN 136 | postgres=# insert into bdt values (1); 137 | INSERT 0 1 138 | postgres=# rollback; 139 | ROLLBACK 140 | postgres=# checkpoint; 141 | CHECKPOINT 142 | postgres=# \! ./get_xact_status.sh -x 31795296 -d /usr/local/pgsql12.1-bench/data 143 | 144 | XID = 31795296 145 | DATA = /usr/local/pgsql12.1-bench/data 146 | 147 | Reading bits 1,2 in byte 2584 in page 10 of file 001E 148 | 149 | xid 31795296 status is: ABORTED 150 | 151 | c 152 | -------------------------- 153 | 154 | Some utilities written in c 155 | 156 | * `flip_bit_and_checksum.bin`: flip one bit one by one and look for a checksum 157 | 158 | Example: 159 | 160 | say you got: 161 | 162 | postgres=# select * from bdt; 163 | WARNING: page verification failed, calculated checksum 34999 but expected 26532 164 | ERROR: invalid page in block 37 of relation base/13580/32812 165 | 166 | copy the block: 167 | 168 | postgres=# select pg_relation_filepath('bdt'); 169 | pg_relation_filepath 170 | ---------------------- 171 | base/13580/32812 172 | 173 | $ dd status=none bs=8192 count=1 if=/usr/local/pgsql13.0-fresh/data/base/13580/32812 skip=37 of=./for_bit_flip_investigation 174 | 175 | launch the utility to look for the expected checksum: 176 | 177 | $ ./flip_bit_and_checksum.bin 178 | 179 | ./flip_bit_and_checksum.bin: 180 | Flip one bit one by one and compute the checksum. 181 | The bit that has been flipped is displayed if the computed checksum matches the one in argument. 182 | 183 | Usage: 184 | ./flip_bit_and_checksum.bin -c checksum -b blockno [-d] 185 | -c, --checksum to look for 186 | -b, --blockno block offset from relation (as a result of segmentno * RELSEG_SIZE + blockoffset) 187 | 188 | $ ./flip_bit_and_checksum.bin ./for_bit_flip_investigation -c 26532 -b 37 189 | Warning: Keep in mind that numbering starts from 0 for both bit and byte 190 | checksum 67a4 (26532) found while flipping bit 1926 (bit 6 in byte 240) 191 | Dumping block with flipped bit to: ./for_bit_flip_investigation_with_bit_1926_flipped 192 | 193 | so by flipping bit 1926 the expected checksum is returned. It's an indication that the corruption might be due to a bit flip at that position. 194 | There is only one bit different from the original block at any time. 195 | -------------------------------------------------------------------------------- /c/Makefile: -------------------------------------------------------------------------------- 1 | PG_CONFIG = pg_config 2 | PGXS := $(shell $(PG_CONFIG) --pgxs) 3 | include $(PGXS) 4 | 5 | BIT_OBJS = flip_bit_and_checksum.o 6 | 7 | all: flip_bit_and_checksum.bin 8 | 9 | flip_bit_and_checksum.bin: $(BIT_OBJS) 10 | $(CC) $^ -o $@ 11 | 12 | clean distclean maintainer-clean: 13 | rm -f flip_bit_and_checksum.bin $(BIT_OBJS) $(COMP_OBJS) 14 | -------------------------------------------------------------------------------- /c/flip_bit_and_checksum.c: -------------------------------------------------------------------------------- 1 | /* 2 | * flip_bit_and_checksum.c 3 | * flip one bit one by one and compute the checksum 4 | * only one bit is different compare to the original page 5 | * we are looking for a bit flip that could generate the checksum 6 | * 7 | * Bertrand Drouvot, 2020-08-01 8 | */ 9 | 10 | #define FRONTEND 1 11 | #include 12 | #include 13 | #include 14 | 15 | #include "c.h" 16 | #include "pg_config.h" 17 | #include "storage/checksum.h" 18 | #include "storage/checksum_impl.h" 19 | #include "getopt.h" 20 | 21 | #define HANDLE_ERR(a) \ 22 | if ((a) == -1) { \ 23 | perror(argv[argc - 1]); \ 24 | exit(errno); \ 25 | } 26 | 27 | #ifdef printf 28 | #undef printf 29 | #endif 30 | #ifdef snprintf 31 | #undef snprintf 32 | #endif 33 | #ifdef fprintf 34 | #undef fprintf 35 | #endif 36 | #ifdef strerror 37 | #undef strerror 38 | #endif 39 | 40 | uint16 compute_checksum(const char *page, BlockNumber blockno); 41 | void open_file_flip_and_compute(const char *filepath, int bit, BlockNumber blockno, int checksum); 42 | static const char *progname; 43 | int isPowerOfTwo(unsigned n); 44 | int findBitSetPosition(unsigned n); 45 | int flippdupper(const char *filepath); 46 | 47 | static void 48 | usage(void) 49 | { 50 | printf("\n"); 51 | printf("%s:\n", progname); 52 | printf("Flip one bit one by one and compute the checksum.\n"); 53 | printf("The bit that has been flipped is displayed if the computed checksum matches the one in argument.\n\n"); 54 | printf("Usage:\n"); 55 | printf(" %s -c checksum -b blockno [-d] \n", progname); 56 | printf(" -c, --checksum to look for\n"); 57 | printf(" -b, --blockno block offset from relation (as a result of segmentno * RELSEG_SIZE + blockoffset) \n"); 58 | } 59 | 60 | int isPowerOfTwo(unsigned n) 61 | { 62 | return n && (!(n & (n - 1))); 63 | } 64 | 65 | int findBitSetPosition(unsigned n) 66 | { 67 | unsigned i; 68 | unsigned pos; 69 | 70 | if (!isPowerOfTwo(n)) 71 | return -1; 72 | 73 | i = pos = 1; 74 | 75 | // Iterate through bits of n till we find a set bit 76 | // i&n will be non-zero only when 'i' and 'n' have a set bit 77 | // at same position 78 | while (!(i & n)) { 79 | // Unset current bit and set the next bit in 'i' 80 | i = i << 1; 81 | // increment position 82 | ++pos; 83 | } 84 | return pos; 85 | } 86 | 87 | int 88 | flippdupper(const char *filepath) 89 | { 90 | 91 | int fd; 92 | unsigned char pd_upper[2]; 93 | uint16 p_upper; 94 | unsigned pos; 95 | 96 | fd = open(filepath, O_RDONLY); 97 | 98 | if (fd <= 0) 99 | { 100 | fprintf(stderr, "%s: %s\n", strerror(errno), filepath); 101 | exit(2); 102 | } 103 | 104 | // pd_upper starts at bit 112 105 | lseek(fd, 112 / 8, SEEK_SET); 106 | read(fd, &pd_upper, 2); 107 | close(fd); 108 | 109 | p_upper = *(uint16*) pd_upper; 110 | 111 | pos = findBitSetPosition(p_upper); 112 | 113 | if (pos != -1) 114 | { 115 | pos = pos + 111; 116 | printf("bit %d will not be flipped to avoid current pd_upper (%u) to become 0\n", pos, *(uint16*) pd_upper); 117 | printf("as that would trigger an assert on PageIsNew (aka pd_upper == 0)\n"); 118 | printf("\n"); 119 | } 120 | 121 | return pos; 122 | } 123 | 124 | uint16 125 | compute_checksum(const char *page, BlockNumber blockno) 126 | { 127 | PageHeader phdr = (PageHeader) page; 128 | uint16 checksum; 129 | 130 | checksum = pg_checksum_page((char *)page, blockno); 131 | 132 | /* 133 | * In 9.2 or lower, pd_checksum is 1 since data checksums are not supported. 134 | */ 135 | if (phdr->pd_checksum == 1) 136 | { 137 | printf("Data checksums are not supported. (9.2 or lower)\n"); 138 | exit(2); 139 | } 140 | 141 | /* 142 | * pd_checksum is 0 if data checksums are disabled. 143 | */ 144 | if (phdr->pd_checksum == 0) 145 | { 146 | printf("Data checksums are disabled.\n"); 147 | exit(2); 148 | } 149 | 150 | return checksum; 151 | } 152 | 153 | void 154 | open_file_flip_and_compute(const char *filepath, int bit, BlockNumber blockno, int checksum) 155 | { 156 | int fd; 157 | char page[BLCKSZ]; 158 | char mask; 159 | char byte; 160 | 161 | fd = open(filepath, O_RDONLY); 162 | 163 | if (fd <= 0) 164 | { 165 | fprintf(stderr, "%s: %s\n", strerror(errno), filepath); 166 | exit(2); 167 | } 168 | 169 | read(fd, page, BLCKSZ); 170 | close(fd); 171 | mask = 0x01 << (bit % 8); 172 | byte = page [bit / 8]; 173 | byte ^= mask; 174 | memset(page + (bit / 8), byte, 1); 175 | 176 | if (compute_checksum(page, blockno) == checksum) 177 | { 178 | char fpath[MAXPGPATH]; 179 | FILE *file; 180 | 181 | printf("Warning: Keep in mind that numbering starts from 0 for both bit and byte\n"); 182 | printf("checksum %x (%d) found while flipping bit %d (bit %d in byte %d)\n", checksum, checksum, bit, bit%8, bit/8); 183 | 184 | snprintf(fpath, MAXPGPATH, "%s_with_bit_%d_flipped", filepath, bit); 185 | printf("Dumping block with flipped bit to: %s\n", fpath); 186 | file = fopen(fpath, "wb"); 187 | fwrite(page, BLCKSZ, 1, file); 188 | fclose(file); 189 | } 190 | } 191 | 192 | int 193 | main(int argc, char *argv[]) 194 | { 195 | char *file = NULL; 196 | struct stat st; 197 | int i; 198 | int option; 199 | int optindex = 0; 200 | int checksum = 0; 201 | uint32 blockno = 0; 202 | int bit_pos; 203 | 204 | static struct option long_options[] = { 205 | {"help", no_argument, NULL, '?'}, 206 | {"checksum", required_argument, NULL, 'c'}, 207 | {"blockno", required_argument, NULL, 'b'}, 208 | {NULL, 0, NULL, 0} 209 | }; 210 | 211 | progname = argv[0]; 212 | 213 | if (argc <= 4) 214 | { 215 | usage(); 216 | exit(0); 217 | } 218 | 219 | if (argc > 1) 220 | { 221 | if (strcmp(argv[optind], "--help") == 0 || strcmp(argv[optind], "-?") == 0) 222 | { 223 | usage(); 224 | exit(0); 225 | } 226 | } 227 | 228 | while ((option = getopt_long(argc, argv, "c:b:", 229 | long_options, &optindex)) != -1) 230 | { 231 | switch (option) 232 | { 233 | case 'c': 234 | checksum = atoi(optarg); 235 | break; 236 | case 'b': 237 | blockno = atoi(optarg); 238 | break; 239 | default: 240 | usage(); 241 | exit(0); 242 | } 243 | } 244 | 245 | if (optind >= argc) { 246 | printf ("Block path has not been provided\n"); 247 | exit (1); 248 | } 249 | 250 | errno = 0; 251 | HANDLE_ERR(stat(argv[argc - 1], &st)); 252 | 253 | if (!S_ISREG(st.st_mode)) 254 | { 255 | fprintf(stderr, "The block must be a regular file\n"); 256 | exit(1); 257 | } 258 | 259 | if (st.st_size != BLCKSZ) 260 | { 261 | fprintf(stderr, "The file size must be %d\n",BLCKSZ); 262 | exit(1); 263 | } 264 | 265 | file = argv[argc - 1]; 266 | 267 | // check if pd_upper has only one bit set 268 | // get its position if that's the case 269 | // -1 if not the case 270 | // to avoid the assert on PageIsNew (aka pd_upper == 0) 271 | // in checksum_impl.h 272 | bit_pos = flippdupper (file); 273 | 274 | for (i=0; i < 8 * BLCKSZ; i++) 275 | { 276 | if (i == bit_pos) 277 | continue; 278 | 279 | open_file_flip_and_compute(file, i, blockno, checksum); 280 | } 281 | return 0; 282 | } 283 | -------------------------------------------------------------------------------- /compare_disk_mem/bt_metap_info.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$1" = "--help" -o $# -eq 0 ] ; then 4 | echo "" 5 | echo "Usage: " 6 | echo "" 7 | echo "-p : relation path on disk" 8 | echo "-bt: btree name" 9 | echo "-i : where to inspect (disk, memory, both)" 10 | echo "" 11 | exit 0 12 | fi 13 | 14 | POSITIONAL=() 15 | while [[ $# -gt 0 ]] 16 | do 17 | key="$1" 18 | 19 | case $key in 20 | -p|--path) 21 | relpath="$2" 22 | shift # past argument 23 | shift # past value 24 | ;; 25 | -bt|--btree) 26 | bt="$2" 27 | shift # past argument 28 | shift # past value 29 | ;; 30 | -i|--inspect) 31 | inspect="$2" 32 | shift # past argument 33 | shift # past value 34 | ;; 35 | *) # unknown option 36 | POSITIONAL+=("$1") # save it in an array for later 37 | shift # past argument 38 | ;; 39 | esac 40 | done 41 | set -- "${POSITIONAL[@]}" 42 | 43 | echo "" 44 | echo "PATH = ${relpath}" 45 | echo "BTREE = ${bt}" 46 | echo "INSPECT = ${inspect}" 47 | echo "" 48 | 49 | echo "from | magic | version | root | level | fastroot | fastlevel | oldest_xact | last_cleanup_num_tuples" 50 | echo "-----+----------+---------+------+-------+----------+-----------+-------------+------------------------" 51 | 52 | if [ "$inspect" = "disk" -o "$inspect" = "both" ] ; then 53 | blk=0 54 | skipbytes=$(( 24 + 0 )) 55 | dsk_magic=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u4 -j $skipbytes -N 4` 56 | skipbytes=$(( 24 + 4 )) 57 | dsk_version=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u4 -j $skipbytes -N 4` 58 | skipbytes=$(( 24 + 8 )) 59 | dsk_root=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u4 -j $skipbytes -N 4` 60 | skipbytes=$(( 24 + 12 )) 61 | dsk_level=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u4 -j $skipbytes -N 4` 62 | skipbytes=$(( 24 + 16 )) 63 | dsk_fastroot=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u4 -j $skipbytes -N 4` 64 | skipbytes=$(( 24 + 20 )) 65 | dsk_fastlevel=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u4 -j $skipbytes -N 4` 66 | skipbytes=$(( 24 + 24 )) 67 | dsk_oldest_xact=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u4 -j $skipbytes -N 4` 68 | skipbytes=$(( 24 + 28 )) 69 | dsk_last_cleanup_num_tuples=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t f4 -j $skipbytes -N 4` 70 | 71 | BTREE_NOVAC_VERSION=3 72 | 73 | if [ "$dsk_version" -ge "$BTREE_NOVAC_VERSION" ] 74 | then 75 | dsk_oldest_xact=0 76 | dsk_last_cleanup_num_tuples=-1 77 | fi 78 | 79 | printf "%-4s %1s %-8s %1s %-7s %1s %-4s %1s %-5s %1s %-8s %1s %-9s %1s %-11s %1s %-2s \n" 'disk' '|' $dsk_magic '|' $dsk_version '|' $dsk_root '|' $dsk_level '|' $dsk_fastroot '|' $dsk_fastlevel '|' $dsk_oldest_xact '|' $dsk_last_cleanup_num_tuples 80 | lp=$(( $lp + 1 )) 81 | fi 82 | 83 | if [ "$inspect" = "mem" -o "$inspect" = "both" ] ; then 84 | from_mem=`psql -tA -c "SELECT * FROM bt_metap('$bt')"` 85 | 86 | mem_magic=`echo $from_mem | cut -f1 -d "|"` 87 | mem_version=`echo $from_mem | cut -f2 -d "|"` 88 | mem_root=`echo $from_mem | cut -f3 -d "|"` 89 | mem_level=`echo $from_mem | cut -f4 -d "|"` 90 | mem_fastroot=`echo $from_mem | cut -f5 -d "|"` 91 | mem_fastlevel=`echo $from_mem | cut -f6 -d "|"` 92 | mem_oldest_xact=`echo $from_mem | cut -f7 -d "|"` 93 | mem_last_cleanup_num_tuples=`echo $from_mem | cut -f8 -d "|"` 94 | printf "%-4s %1s %-8s %1s %-7s %1s %-4s %1s %-5s %1s %-8s %1s %-9s %1s %-11s %1s %-2s \n" 'mem' '|' $mem_magic '|' $mem_version '|' $mem_root '|' $mem_level '|' $mem_fastroot '|' $mem_fastlevel '|' $mem_oldest_xact '|' $mem_last_cleanup_num_tuples 95 | fi 96 | -------------------------------------------------------------------------------- /compare_disk_mem/bt_page_items_info.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$1" = "--help" -o $# -eq 0 ] ; then 4 | echo "" 5 | echo "Usage: " 6 | echo "" 7 | echo "-b: block to inspect" 8 | echo "-p: relation path on disk" 9 | echo "-bt: btree name" 10 | echo "-i: where to inspect (disk, memory, both)" 11 | echo "" 12 | exit 0 13 | fi 14 | 15 | POSITIONAL=() 16 | while [[ $# -gt 0 ]] 17 | do 18 | key="$1" 19 | 20 | case $key in 21 | -b|--block) 22 | blk="$2" 23 | shift # past argument 24 | shift # past value 25 | ;; 26 | -p|--path) 27 | relpath="$2" 28 | shift # past argument 29 | shift # past value 30 | ;; 31 | -bt|--btree) 32 | bt="$2" 33 | shift # past argument 34 | shift # past value 35 | ;; 36 | -i|--inspect) 37 | inspect="$2" 38 | shift # past argument 39 | shift # past value 40 | ;; 41 | *) # unknown option 42 | POSITIONAL+=("$1") # save it in an array for later 43 | shift # past argument 44 | ;; 45 | esac 46 | done 47 | set -- "${POSITIONAL[@]}" 48 | 49 | echo "" 50 | echo "BLOCK = ${blk}" 51 | echo "PATH = ${relpath}" 52 | echo "BTREE = ${bt}" 53 | echo "INSPECT = ${inspect}" 54 | echo "" 55 | 56 | echo "from | ofs | t_ctid | len | nu | va | data " 57 | echo "-----+-----+---------------------+-----+----+----+------------------------------------------------------------------------" 58 | 59 | if [ "$inspect" = "disk" -o "$inspect" = "both" ] ; then 60 | 61 | filenumber=`expr $blk / 131072` 62 | blktoread=0 63 | if (( $filenumber >= 1 )) 64 | then 65 | relpath=$relpath.$filenumber 66 | blktoread=`expr 131072 \\* $filenumber` 67 | fi 68 | blkmem=$blk 69 | blk=`expr $blk - $blktoread` 70 | 71 | dsk_lower=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t d -j 12 -N 2` 72 | nbentries=`expr $dsk_lower - 24` 73 | nbentries=`expr $nbentries / 4` 74 | ent=1 75 | while [ $ent -le $nbentries ] 76 | do 77 | skipbytes=$(( 4 * $ent - 4 )) 78 | skipbytes=$(( 24 + $skipbytes )) 79 | dsk_item_off=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x2 -j $skipbytes -N 2 | sed 's/^ *//'` 80 | dsk_item_off=`echo $((0x$dsk_item_off & ~$((1<<15))))` 81 | skipbytes=$(( 2 + $skipbytes )) 82 | dsk_item_len=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x2 -j $skipbytes -N 2 | sed 's/^ *//'` 83 | dsk_item_len=`echo $((0x$dsk_item_len >> 1))` 84 | toread=$(( 0 + $dsk_item_off )) 85 | dsk_ctid_block_number=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x2 -j $toread -N 4 | sed 's/ //g'` 86 | dsk_ctid_block_number=`echo $dsk_ctid_block_number | tr '[:lower:]' '[:upper:]'` 87 | dsk_ctid_block_number=`echo "ibase=16;$dsk_ctid_block_number"|bc` 88 | toread=$(( 4 + $dsk_item_off )) 89 | dsk_ctid_tuple_id=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u2 -j $toread -N 2` 90 | toread=$(( 6 + $dsk_item_off )) 91 | dsk_t_info=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u2 -j $toread -N 2` 92 | dsk_t_info_b=`echo "obase=2;$dsk_t_info" | bc | rev` 93 | # typedef struct IndexTupleData 94 | # { 95 | # ItemPointerData t_tid; /* reference TID to heap tuple */ 96 | # /* --------------- 97 | # * t_info is laid out in the following fashion: 98 | # * 99 | # * 15th (high) bit: has nulls 100 | # * 14th bit: has var-width attributes 101 | # * 13th bit: unused 102 | # * 12-0 bit: size of tuple 103 | # * --------------- 104 | # */ 105 | # unsigned short t_info; /* various info about tuple */ 106 | # } IndexTupleData; 107 | has_var=`echo $dsk_t_info_b | cut -c15` #0 to 15 so need to ad one here 108 | if [ ! -z "$has_var" ] && (( $has_var == 1 )) 109 | then 110 | item_has_var="t" 111 | else 112 | item_has_var="f" 113 | fi 114 | 115 | has_null=`echo $dsk_t_info_b | cut -c16` #0 to 15 so need to ad one here 116 | if [ ! -z "$has_null" ] && (( $has_null == 1 )) 117 | then 118 | item_has_null="t" 119 | else 120 | item_has_null="f" 121 | fi 122 | 123 | toread=$(( 8 + $dsk_item_off )) 124 | n_toread=$(( $dsk_item_len - 8 )) 125 | dsk_t_data=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x1 -j $toread -N $n_toread` 126 | printf "%-4s %1s %-3s %1s %1s %-8s %1s %-4s %1s %1s %-3s %1s %-2s %1s %-2s %1s %-60s \n" 'disk' '|' $ent '|' '(' $dsk_ctid_block_number ',' $dsk_ctid_tuple_id ')' '|' $dsk_item_len '|' $item_has_null '|' $item_has_var '|' "$dsk_t_data" 127 | ent=$(( $ent + 1 )) 128 | done 129 | fi 130 | 131 | if [ "$inspect" = "mem" -o "$inspect" = "both" ] ; then 132 | from_mem=`psql -tA -c "SELECT * FROM bt_page_items('$bt', $blkmem)"` 133 | 134 | IFS=$'\n' 135 | for item in `echo "$from_mem"` 136 | do 137 | mem_offset=`echo $item | cut -f1 -d "|"` 138 | mem_ctid=`echo $item | cut -f2 -d "|"` 139 | mem_length=`echo $item | cut -f3 -d "|"` 140 | mem_has_null=`echo $item | cut -f4 -d "|"` 141 | mem_has_var=`echo $item | cut -f5 -d "|"` 142 | mem_item_data=`echo $item | cut -f6 -d "|"` 143 | printf "%-4s %1s %-3s %1s %-19s %1s %-3s %1s %-2s %1s %-2s %1s %-60s \n" 'mem' '|' $mem_offset '|' $mem_ctid '|' $mem_length '|' $mem_has_null '|' $mem_has_var '| ' $mem_item_data 144 | done 145 | fi 146 | -------------------------------------------------------------------------------- /compare_disk_mem/bt_page_stats_info.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$1" = "--help" -o $# -eq 0 ] ; then 4 | echo "" 5 | echo "Usage: " 6 | echo "" 7 | echo "-b: block to inspect" 8 | echo "-p: relation path on disk" 9 | echo "-bt: btree name" 10 | echo "-i: where to inspect (disk, memory, both)" 11 | echo "" 12 | exit 0 13 | fi 14 | 15 | POSITIONAL=() 16 | while [[ $# -gt 0 ]] 17 | do 18 | key="$1" 19 | 20 | case $key in 21 | -b|--block) 22 | blk="$2" 23 | shift # past argument 24 | shift # past value 25 | ;; 26 | -p|--path) 27 | relpath="$2" 28 | shift # past argument 29 | shift # past value 30 | ;; 31 | -bt|--btree) 32 | bt="$2" 33 | shift # past argument 34 | shift # past value 35 | ;; 36 | -i|--inspect) 37 | inspect="$2" 38 | shift # past argument 39 | shift # past value 40 | ;; 41 | *) # unknown option 42 | POSITIONAL+=("$1") # save it in an array for later 43 | shift # past argument 44 | ;; 45 | esac 46 | done 47 | set -- "${POSITIONAL[@]}" 48 | 49 | echo "" 50 | echo "BLOCK = ${blk}" 51 | echo "PATH = ${relpath}" 52 | echo "BTREE = ${bt}" 53 | echo "INSPECT = ${inspect}" 54 | echo "" 55 | 56 | BTP_LEAF=0x1 57 | BTP_ROOT=0x2 58 | BTP_DELETED=0x4 59 | BTP_META=0x8 60 | BTP_HALF_DEAD=0x10 61 | BTP_SPLIT_END=0x20 62 | BTP_HAS_GARBAGE=0x40 63 | BTP_INCOMPLETE_SPLIT=0x80 64 | 65 | search_type () { 66 | 67 | btpo_flags_param=0x$1 68 | 69 | P_ISDELETED=`echo $(( $btpo_flags_param & $BTP_DELETED ))` 70 | P_IGNORE=`echo $(( $btpo_flags_param & ($BTP_DELETED|$BTP_HALF_DEAD) ))` 71 | P_ISLEAF=`echo $(( $btpo_flags_param & $BTP_LEAF ))` 72 | P_ISROOT=`echo $(( $btpo_flags_param & $BTP_ROOT ))` 73 | 74 | if (( $P_ISDELETED != 0 )) 75 | then 76 | page_type="d" 77 | elif (( $P_IGNORE != 0 )) 78 | then 79 | page_type="e" 80 | elif (( $P_ISLEAF != 0 )) 81 | then 82 | page_type="l" 83 | elif (( $P_ISROOT != 0 )) 84 | then 85 | page_type="r" 86 | else 87 | page_type="i" 88 | fi 89 | 90 | echo "$page_type" 91 | } 92 | 93 | echo "from | blkno | type | live_items | dead_items | avg_item_size | free_size | btpo_prev | btpo_next | btpo | btpo_flags" 94 | echo "-----+-------+------+------------+------------+---------------+------------+-----------+-----------+------+-----------" 95 | 96 | if [ "$inspect" = "disk" -o "$inspect" = "both" ] ; then 97 | 98 | filenumber=`expr $blk / 131072` 99 | blktoread=0 100 | if (( $filenumber >= 1 )) 101 | then 102 | relpath=$relpath.$filenumber 103 | blktoread=`expr 131072 \\* $filenumber` 104 | fi 105 | blkmem=$blk 106 | blk=`expr $blk - $blktoread` 107 | 108 | dsk_lower=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t d -j 12 -N 2` 109 | dsk_upper=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t d -j 14 -N 2` 110 | # where is opaque? 111 | dsk_special=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t d -j 16 -N 2` 112 | # go read opaque 113 | toread=$(( 0 + $dsk_special )) 114 | dsk_btpo_prev=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u4 -j $toread -N 4` 115 | toread=$(( 4 + $dsk_special )) 116 | dsk_btpo_next=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u4 -j $toread -N 4` 117 | toread=$(( 8 + $dsk_special )) 118 | dsk_btpo=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u4 -j $toread -N 4` 119 | toread=$(( 12 + $dsk_special )) 120 | dsk_btpo_flags=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u2 -j $toread -N 2` 121 | dsk_btpo_flags_hexa=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x2 -j $toread -N 2` 122 | toread=$(( 14 + $dsk_special )) 123 | dsk_BTCycleId=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u2 -j $toread -N 2` 124 | dsk_p_type=$(search_type $dsk_btpo_flags_hexa) 125 | dsk_free_size=$(( $dsk_upper - $dsk_lower )) 126 | dsk_free_size=$(( $dsk_free_size - 4 )) 127 | 128 | if [ "$dsk_p_type" = "d" ] ; then 129 | dsk_live_items=0 130 | dsk_dead_items=0 131 | dsk_dead_items=0 132 | dsk_free_size=0 133 | dsk_btpo_prev=-1 134 | dsk_btpo_next=-1 135 | dsk_btpo_flags=0 136 | 137 | else 138 | 139 | # how many entries? 140 | nbentries=`expr $dsk_lower - 24` 141 | nbentries=`expr $nbentries / 4` 142 | ent=1 143 | dsk_items_size=0 144 | dsk_live_items=0 145 | dsk_dead_items=0 146 | LP_DEAD=3 147 | while [ $ent -le $nbentries ] 148 | do 149 | skipbytes=$(( 4 * $ent - 4 )) 150 | skipbytes=$(( 24 + $skipbytes )) 151 | dsk_item_off=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x2 -j $skipbytes -N 2 | sed 's/^ *//'` 152 | dsk_item_off=`echo $((0x$dsk_item_off & ~$((1<<15))))` 153 | dsk_lp_flags=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x4 -j $skipbytes -N 4 | sed 's/^ *//'` 154 | dsk_lp_flags=`echo $dsk_lp_flags | tr '[:lower:]' '[:upper:]'` 155 | # bits 16 and 17 156 | dsk_lp_flags="$((2#`echo "ibase=16;obase=2;${dsk_lp_flags}" | bc | rev | cut -c16,17 | rev`))" 157 | if (( $dsk_lp_flags != $LP_DEAD )) 158 | then 159 | dsk_live_items=$(( $dsk_live_items + 1 )) 160 | else 161 | dsk_dead_items=$(( $dsk_dead_items + 1 )) 162 | fi 163 | skipbytes=$(( 2 + $skipbytes )) 164 | dsk_item_len=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x2 -j $skipbytes -N 2 | sed 's/^ *//'` 165 | dsk_item_len=`echo $((0x$dsk_item_len >> 1))` 166 | dsk_items_size=$(( $dsk_items_size + $dsk_item_len )) 167 | ent=$(( $ent + 1 )) 168 | done 169 | fi 170 | 171 | dsk_total_items=$(( $dsk_live_items + $dsk_dead_items )) 172 | if (( $dsk_total_items != 0 )) 173 | then 174 | dsk_avg_item_size=$(( $dsk_items_size / $dsk_total_items )) 175 | else 176 | dsk_avg_item_size=0 177 | fi 178 | fi 179 | 180 | printf "%-4s %1s %-5s %1s %-4s %1s %-10s %1s %-10s %1s %-13s %1s %-9s %1s %-9s %1s %-9s %1s %-4s %1s %-10s \n" 'dsk' '|' $blkmem '|' $dsk_p_type '|' $dsk_live_items '|' $dsk_dead_items '|' $dsk_avg_item_size '| ' "$dsk_free_size" '|' $dsk_btpo_prev '|' $dsk_btpo_next '|' $dsk_btpo '|' $dsk_btpo_flags 181 | 182 | 183 | if [ "$inspect" = "mem" -o "$inspect" = "both" ] ; then 184 | from_mem=`psql -tA -c "SELECT * FROM bt_page_stats('$bt', $blkmem)"` 185 | 186 | for item in `echo "$from_mem"` 187 | do 188 | mem_blk=`echo $item | cut -f1 -d "|"` 189 | mem_p_type=`echo $item | cut -f2 -d "|"` 190 | mem_live_items=`echo $item | cut -f3 -d "|"` 191 | mem_dead_items=`echo $item | cut -f4 -d "|"` 192 | mem_avg_item_size=`echo $item | cut -f5 -d "|"` 193 | mem_free_size=`echo $item | cut -f7 -d "|"` 194 | mem_btpo_prev=`echo $item | cut -f8 -d "|"` 195 | mem_btpo_next=`echo $item | cut -f9 -d "|"` 196 | mem_btpo=`echo $item | cut -f10 -d "|"` 197 | mem_btpo_flags=`echo $item | cut -f11 -d "|"` 198 | 199 | printf "%-4s %1s %-5s %1s %-4s %1s %-10s %1s %-10s %1s %-13s %1s %-9s %1s %-9s %1s %-9s %1s %-4s %1s %-10s \n" 'mem' '|' $mem_blk '|' $mem_p_type '|' $mem_live_items '|' $mem_dead_items '|' $mem_avg_item_size '| ' "$mem_free_size" '|' $mem_btpo_prev '|' $mem_btpo_next '|' $mem_btpo '|' $mem_btpo_flags 200 | done 201 | fi 202 | -------------------------------------------------------------------------------- /compare_disk_mem/heap_page_items_info.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$1" = "--help" -o $# -eq 0 ] ; then 4 | echo "" 5 | echo "Usage: " 6 | echo "" 7 | echo "-b: block to inspect" 8 | echo "-p: relation path on disk" 9 | echo "-t: table name" 10 | echo "-i: where to inspect (disk, memory, both)" 11 | echo "" 12 | exit 0 13 | fi 14 | 15 | POSITIONAL=() 16 | while [[ $# -gt 0 ]] 17 | do 18 | key="$1" 19 | 20 | case $key in 21 | -b|--block) 22 | blk="$2" 23 | shift # past argument 24 | shift # past value 25 | ;; 26 | -p|--path) 27 | relpath="$2" 28 | shift # past argument 29 | shift # past value 30 | ;; 31 | -t|--table) 32 | tb="$2" 33 | shift # past argument 34 | shift # past value 35 | ;; 36 | -i|--inspect) 37 | inspect="$2" 38 | shift # past argument 39 | shift # past value 40 | ;; 41 | *) # unknown option 42 | POSITIONAL+=("$1") # save it in an array for later 43 | shift # past argument 44 | ;; 45 | esac 46 | done 47 | set -- "${POSITIONAL[@]}" 48 | 49 | echo "" 50 | echo "BLOCK = ${blk}" 51 | echo "PATH = ${relpath}" 52 | echo "TABLE = ${tb}" 53 | echo "INSPECT = ${inspect}" 54 | echo "" 55 | 56 | echo "from | lp | lp_off | lp_flags | lp_len | t_xmin | t_xmax | t_field3 | t_ctid | t_infomask2 | t_infomask | t_hoff" 57 | echo "-----+----+--------+----------+--------+--------+--------+----------+---------------------+-------------+------------+--------" 58 | 59 | if [ "$inspect" = "disk" -o "$inspect" = "both" ] ; then 60 | filenumber=`expr $blk / 131072` 61 | blktoread=0 62 | if (( $filenumber >= 1 )) 63 | then 64 | relpath=$relpath.$filenumber 65 | blktoread=`expr 131072 \\* $filenumber` 66 | fi 67 | blkmem=$blk 68 | blk=`expr $blk - $blktoread` 69 | #echo "Reading block $blk in file $relpath....." 70 | 71 | dsk_lower=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t d -j 12 -N 2` 72 | nbtuples=`expr $dsk_lower - 24` 73 | nbtuples=`expr $nbtuples / 4` 74 | lp=1 75 | while [ $lp -le $nbtuples ] 76 | do 77 | skipbytes=$(( 4 * $lp - 4 )) 78 | skipbytes=$(( 24 + $skipbytes )) 79 | dsk_lp_off=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x2 -j $skipbytes -N 2 | sed 's/^ *//'` 80 | dsk_lp_off=`echo $((0x$dsk_lp_off & ~$((1<<15))))` 81 | dsk_lp_flags=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x4 -j $skipbytes -N 4 | sed 's/^ *//'` 82 | dsk_lp_flags=`echo $dsk_lp_flags | tr '[:lower:]' '[:upper:]'` 83 | skipbytes=$(( 2 + $skipbytes )) 84 | dsk_lp_len=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x2 -j $skipbytes -N 2 | sed 's/^ *//'` 85 | dsk_lp_len=`echo $((0x$dsk_lp_len >> 1))` 86 | dsk_lp_flags="$((2#`echo "ibase=16;obase=2;${dsk_lp_flags}" | bc | rev | cut -c16,17 | rev`))" 87 | dsk_xmin=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u4 -j $dsk_lp_off -N 4` 88 | toread=$(( 4 + $dsk_lp_off )) 89 | dsk_xmax=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u4 -j $toread -N 4` 90 | toread=$(( 8 + $dsk_lp_off )) 91 | dsk_t_field3=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u4 -j $toread -N 4` 92 | toread=$(( 12 + $dsk_lp_off )) 93 | dsk_ctid_block_number=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x2 -j $toread -N 4 | sed 's/ //g'` 94 | dsk_ctid_block_number=`echo $dsk_ctid_block_number | tr '[:lower:]' '[:upper:]'` 95 | dsk_ctid_block_number=`echo "ibase=16;$dsk_ctid_block_number"|bc` 96 | toread=$(( 16 + $dsk_lp_off )) 97 | dsk_ctid_tuple_id=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u2 -j $toread -N 2` 98 | toread=$(( 18 + $dsk_lp_off )) 99 | dsk_t_infomask2=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u2 -j $toread -N 2` 100 | toread=$(( 20 + $dsk_lp_off )) 101 | dsk_t_infomask=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u2 -j $toread -N 2` 102 | toread=$(( 22 + $dsk_lp_off )) 103 | dsk_t_hoff=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u1 -j $toread -N 1` 104 | 105 | if [ "${dsk_lp_len}" != "0" ] 106 | then 107 | printf "%-4s %1s %-2s %1s %-6s %1s %-8s %1s %-6s %1s %-6s %1s %-6s %1s %-8s %1s %1s %-8s %1s %-4s %1s %1s %-11s %1s %-10s %1s %-6s \n" 'disk' '|' $lp '|' $dsk_lp_off '|' $dsk_lp_flags '|' $dsk_lp_len '|' $dsk_xmin '|' $dsk_xmax '|' $dsk_t_field3 '|' '(' $dsk_ctid_block_number ',' $dsk_ctid_tuple_id ')' '|' $dsk_t_infomask2 '|' $dsk_t_infomask '|' $dsk_t_hoff 108 | else 109 | printf "%-4s %1s %-2s %1s %-6s %1s %-8s %1s %-6s %1s %-6s %1s %-6s %1s %-8s %1s %1s %-8s %1s %-4s %1s %1s %-11s %1s %-10s %1s %-6s \n" 'disk' '|' $lp '|' $dsk_lp_off '|' $dsk_lp_flags '|' $dsk_lp_len '|' ' ' '|' ' ' '|' ' ' '|' ' ' '|' ' ' '|' ' ' '|' ' ' 110 | fi 111 | lp=$(( $lp + 1 )) 112 | done 113 | fi 114 | 115 | if [ "$inspect" = "mem" -o "$inspect" = "both" ] ; then 116 | from_mem=`psql -tA -c "SELECT * FROM heap_page_items(get_raw_page('$tb', $blkmem))"` 117 | 118 | for lp in `echo "$from_mem"` 119 | do 120 | #echo $lp 121 | mem_lp=`echo $lp | cut -f1 -d "|"` 122 | mem_lp_off=`echo $lp | cut -f2 -d "|"` 123 | mem_lp_flags=`echo $lp | cut -f3 -d "|"` 124 | mem_lp_len=`echo $lp | cut -f4 -d "|"` 125 | mem_t_xmin=`echo $lp | cut -f5 -d "|"` 126 | mem_t_max=`echo $lp | cut -f6 -d "|"` 127 | mem_t_field3=`echo $lp | cut -f7 -d "|"` 128 | mem_t_ctid=`echo $lp | cut -f8 -d "|"` 129 | mem_t_infomask2=`echo $lp | cut -f9 -d "|"` 130 | mem_t_infomask=`echo $lp | cut -f10 -d "|"` 131 | mem_t_hoff=`echo $lp | cut -f11 -d "|"` 132 | mem_t_bits=`echo $lp | cut -f12 -d "|"` 133 | mem_t_oid=`echo $lp | cut -f13 -d "|"` 134 | printf "%-4s %1s %-2s %1s %-6s %1s %-8s %1s %-6s %1s %-6s %1s %-6s %1s %-8s %1s %-19s %1s %-11s %1s %-10s %1s %-6s \n" 'mem' '|' $mem_lp '|' $mem_lp_off '|' $mem_lp_flags '|' $mem_lp_len '|' $mem_t_xmin '|' $mem_t_max '|' $mem_t_field3 '|' $mem_t_ctid '|' $mem_t_infomask2 '|' $mem_t_infomask '|' $mem_t_hoff 135 | done 136 | fi 137 | -------------------------------------------------------------------------------- /compare_disk_mem/heap_page_items_toast_pointer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$1" = "--help" -o $# -eq 0 ] ; then 4 | echo "" 5 | echo "Usage: " 6 | echo "" 7 | echo "-b: block to inspect" 8 | echo "-p: relation path on disk" 9 | echo "-t: table name" 10 | echo "-i: where to inspect (disk, memory, both)" 11 | echo "" 12 | exit 0 13 | fi 14 | 15 | POSITIONAL=() 16 | while [[ $# -gt 0 ]] 17 | do 18 | key="$1" 19 | 20 | case $key in 21 | -b|--block) 22 | blk="$2" 23 | shift # past argument 24 | shift # past value 25 | ;; 26 | -p|--path) 27 | relpath="$2" 28 | shift # past argument 29 | shift # past value 30 | ;; 31 | -t|--table) 32 | tb="$2" 33 | shift # past argument 34 | shift # past value 35 | ;; 36 | -i|--inspect) 37 | inspect="$2" 38 | shift # past argument 39 | shift # past value 40 | ;; 41 | *) # unknown option 42 | POSITIONAL+=("$1") # save it in an array for later 43 | shift # past argument 44 | ;; 45 | esac 46 | done 47 | set -- "${POSITIONAL[@]}" 48 | 49 | echo "" 50 | echo "BLOCK = ${blk}" 51 | echo "PATH = ${relpath}" 52 | echo "TABLE = ${tb}" 53 | echo "INSPECT = ${inspect}" 54 | echo "" 55 | 56 | echo "from | lp | lp_off | lp_flags | lp_len | t_xmin | t_xmax | t_field3 | t_ctid | t_infomask2 | t_infomask | t_hoff" 57 | echo "-----+----+--------+----------+--------+--------+--------+----------+---------------------+-------------+------------+--------" 58 | 59 | if [ "$inspect" = "disk" -o "$inspect" = "both" ] ; then 60 | filenumber=`expr $blk / 131072` 61 | blktoread=0 62 | if (( $filenumber >= 1 )) 63 | then 64 | relpath=$relpath.$filenumber 65 | blktoread=`expr 131072 \\* $filenumber` 66 | fi 67 | blkmem=$blk 68 | blk=`expr $blk - $blktoread` 69 | #echo "Reading block $blk in file $relpath....." 70 | 71 | dsk_lower=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t d -j 12 -N 2` 72 | nbtuples=`expr $dsk_lower - 24` 73 | nbtuples=`expr $nbtuples / 4` 74 | lp=1 75 | while [ $lp -le $nbtuples ] 76 | do 77 | skipbytes=$(( 4 * $lp - 4 )) 78 | skipbytes=$(( 24 + $skipbytes )) 79 | dsk_lp_off=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x2 -j $skipbytes -N 2 | sed 's/^ *//'` 80 | dsk_lp_off=`echo $((0x$dsk_lp_off & ~$((1<<15))))` 81 | dsk_lp_flags=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x4 -j $skipbytes -N 4 | sed 's/^ *//'` 82 | dsk_lp_flags=`echo $dsk_lp_flags | tr '[:lower:]' '[:upper:]'` 83 | skipbytes=$(( 2 + $skipbytes )) 84 | dsk_lp_len=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x2 -j $skipbytes -N 2 | sed 's/^ *//'` 85 | dsk_lp_len=`echo $((0x$dsk_lp_len >> 1))` 86 | dsk_lp_flags="$((2#`echo "ibase=16;obase=2;${dsk_lp_flags}" | bc | rev | cut -c16,17 | rev`))" 87 | dsk_xmin=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u4 -j $dsk_lp_off -N 4` 88 | toread=$(( 4 + $dsk_lp_off )) 89 | dsk_xmax=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u4 -j $toread -N 4` 90 | toread=$(( 8 + $dsk_lp_off )) 91 | dsk_t_field3=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u4 -j $toread -N 4` 92 | toread=$(( 12 + $dsk_lp_off )) 93 | dsk_ctid_block_number=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x2 -j $toread -N 4 | sed 's/ //g'` 94 | dsk_ctid_block_number=`echo $dsk_ctid_block_number | tr '[:lower:]' '[:upper:]'` 95 | dsk_ctid_block_number=`echo "ibase=16;$dsk_ctid_block_number"|bc` 96 | toread=$(( 16 + $dsk_lp_off )) 97 | dsk_ctid_tuple_id=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u2 -j $toread -N 2` 98 | toread=$(( 18 + $dsk_lp_off )) 99 | dsk_t_infomask2=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u2 -j $toread -N 2` 100 | toread=$(( 20 + $dsk_lp_off )) 101 | dsk_t_infomask=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u2 -j $toread -N 2` 102 | toread=$(( 22 + $dsk_lp_off )) 103 | dsk_t_hoff=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u1 -j $toread -N 1` 104 | toread=$(( $dsk_t_hoff + $dsk_lp_off )) 105 | bdt=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x1 -j $toread -N 8` 106 | echo $bdt 107 | 108 | 109 | printf "%-4s %1s %-2s %1s %-6s %1s %-8s %1s %-6s %1s %-6s %1s %-6s %1s %-8s %1s %1s %-8s %1s %-4s %1s %1s %-11s %1s %-10s %1s %-6s \n" 'disk' '|' $lp '|' $dsk_lp_off '|' $dsk_lp_flags '|' $dsk_lp_len '|' $dsk_xmin '|' $dsk_xmax '|' $dsk_t_field3 '|' '(' $dsk_ctid_block_number ',' $dsk_ctid_tuple_id ')' '|' $dsk_t_infomask2 '|' $dsk_t_infomask '|' $dsk_t_hoff 110 | lp=$(( $lp + 1 )) 111 | done 112 | fi 113 | 114 | if [ "$inspect" = "mem" -o "$inspect" = "both" ] ; then 115 | from_mem=`psql -tA -c "SELECT * FROM heap_page_items(get_raw_page('$tb', $blkmem))"` 116 | 117 | for lp in `echo "$from_mem"` 118 | do 119 | #echo $lp 120 | mem_lp=`echo $lp | cut -f1 -d "|"` 121 | mem_lp_off=`echo $lp | cut -f2 -d "|"` 122 | mem_lp_flags=`echo $lp | cut -f3 -d "|"` 123 | mem_lp_len=`echo $lp | cut -f4 -d "|"` 124 | mem_t_xmin=`echo $lp | cut -f5 -d "|"` 125 | mem_t_max=`echo $lp | cut -f6 -d "|"` 126 | mem_t_field3=`echo $lp | cut -f7 -d "|"` 127 | mem_t_ctid=`echo $lp | cut -f8 -d "|"` 128 | mem_t_infomask2=`echo $lp | cut -f9 -d "|"` 129 | mem_t_infomask=`echo $lp | cut -f10 -d "|"` 130 | mem_t_hoff=`echo $lp | cut -f11 -d "|"` 131 | mem_t_bits=`echo $lp | cut -f12 -d "|"` 132 | mem_t_oid=`echo $lp | cut -f13 -d "|"` 133 | printf "%-4s %1s %-2s %1s %-6s %1s %-8s %1s %-6s %1s %-6s %1s %-6s %1s %-8s %1s %-19s %1s %-11s %1s %-10s %1s %-6s \n" 'mem' '|' $mem_lp '|' $mem_lp_off '|' $mem_lp_flags '|' $mem_lp_len '|' $mem_t_xmin '|' $mem_t_max '|' $mem_t_field3 '|' $mem_t_ctid '|' $mem_t_infomask2 '|' $mem_t_infomask '|' $mem_t_hoff 134 | done 135 | fi 136 | -------------------------------------------------------------------------------- /compare_disk_mem/page_header_info.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$1" = "--help" -o $# -eq 0 ] ; then 4 | echo "" 5 | echo "Usage: " 6 | echo "" 7 | echo "-b: block to inspect" 8 | echo "-p: relation path on disk" 9 | echo "-t: table name" 10 | echo "-i: where to inspect (disk, memory, both)" 11 | echo "" 12 | exit 0 13 | fi 14 | 15 | POSITIONAL=() 16 | while [[ $# -gt 0 ]] 17 | do 18 | key="$1" 19 | 20 | case $key in 21 | -b|--block) 22 | blk="$2" 23 | shift # past argument 24 | shift # past value 25 | ;; 26 | -p|--path) 27 | relpath="$2" 28 | shift # past argument 29 | shift # past value 30 | ;; 31 | -t|--table) 32 | tb="$2" 33 | shift # past argument 34 | shift # past value 35 | ;; 36 | -i|--inspect) 37 | inspect="$2" 38 | shift # past argument 39 | shift # past value 40 | ;; 41 | *) # unknown option 42 | POSITIONAL+=("$1") # save it in an array for later 43 | shift # past argument 44 | ;; 45 | esac 46 | done 47 | set -- "${POSITIONAL[@]}" 48 | 49 | echo "" 50 | echo "BLOCK = ${blk}" 51 | echo "PATH = ${relpath}" 52 | echo "TABLE = ${tb}" 53 | echo "INSPECT = ${inspect}" 54 | echo "" 55 | 56 | echo " from | lsn | checksum | flags | lower | upper | special | prune_xid " 57 | echo "---------+-------------------+-------------+--------------+--------------+--------------+--------------+-----------" 58 | if [ "$inspect" = "disk" -o "$inspect" = "both" ] ; then 59 | 60 | filenumber=`expr $blk / 131072` 61 | blktoread=0 62 | if (( $filenumber >= 1 )) 63 | then 64 | relpath=${relpath}.${filenumber} 65 | blktoread=`expr 131072 \\* $filenumber` 66 | fi 67 | blkmem=$blk 68 | blk=`expr $blk - $blktoread` 69 | 70 | dsk_lsn1=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x4 -N 4` 71 | dsk_lsn1=`echo $dsk_lsn1 | tr '[:lower:]' '[:upper:]'` 72 | dsk_lsn2=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x4 -j 4 -N 4` 73 | dsk_lsn2=`echo $dsk_lsn2 | tr '[:lower:]' '[:upper:]'` 74 | dsk_checksum=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t d -j 8 -N 2` 75 | dsk_flags=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t d -j 10 -N 2` 76 | dsk_lower=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t d -j 12 -N 2` 77 | dsk_upper=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t d -j 14 -N 2` 78 | dsk_special=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t d -j 16 -N 2` 79 | dsk_pagesize=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t d -j 18 -N 2` 80 | dsk_prune=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t d -j 20 -N 4` 81 | 82 | printf "%-8s %1s %-16s %1s %-11s %1s %-12s %1s %-12s %1s %-12s %1s %-12s %1s %-10s \n" 'disk' '|' $dsk_lsn1/$dsk_lsn2 '|' $dsk_checksum '|' $dsk_flags '|' $dsk_lower '|' $dsk_upper '|' $dsk_special '|' $dsk_prune 83 | fi 84 | 85 | if [ "$inspect" = "mem" -o "$inspect" = "both" ] ; then 86 | from_mem=`psql -tA -c "SELECT * FROM page_header(get_raw_page('$tb', $blkmem))"` 87 | 88 | mem_lsn=`echo $from_mem | cut -f1 -d "|"` 89 | mem_checksum=`echo $from_mem | cut -f2 -d "|"` 90 | mem_flags=`echo $from_mem | cut -f3 -d "|"` 91 | mem_lower=`echo $from_mem | cut -f4 -d "|"` 92 | mem_upper=`echo $from_mem | cut -f5 -d "|"` 93 | mem_special=`echo $from_mem | cut -f6 -d "|"` 94 | mem_prune=`echo $from_mem | cut -f9 -d "|"` 95 | printf "%-8s %1s %-17s %1s %-11s %1s %-12s %1s %-12s %1s %-12s %1s %-12s %1s %-10s \n" 'mem' '|' $mem_lsn '|' $mem_checksum '|' $mem_flags '|' $mem_lower '|' $mem_upper '|' $mem_special '|' $mem_prune 96 | fi 97 | -------------------------------------------------------------------------------- /compare_disk_mem/toast_index_info.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Same as SELECT * FROM bt_page_items('pg_toast.pg_toast_174243_index', 1); 4 | # but display the data as chunk_id and chunk_seq 5 | # ./toast_index_info.sh -b 1 -p /usr/local/pgsql12.0/data/base/174236/174248 6 | # 7 | #from | ofs | t_ctid | len | nu | va | chunk_id | chunk_seq 8 | #-----+-----+---------------------+-----+----+----+-------------------|------------------- 9 | #disk | 1 | ( 0 , 1 ) | 16 | f | f | 174249 | 0 10 | #disk | 2 | ( 0 , 2 ) | 16 | f | f | 174249 | 1 11 | #disk | 3 | ( 0 , 3 ) | 16 | f | f | 174249 | 2 12 | #disk | 4 | ( 0 , 4 ) | 16 | f | f | 174249 | 3 13 | #disk | 5 | ( 1 , 1 ) | 16 | f | f | 174249 | 4 14 | #disk | 6 | ( 1 , 2 ) | 16 | f | f | 174249 | 5 15 | #disk | 7 | ( 1 , 3 ) | 16 | f | f | 174250 | 0 16 | #disk | 8 | ( 1 , 4 ) | 16 | f | f | 174250 | 1 17 | 18 | # means 174250 , seq 0 is located at block 1 and lp 4 in the toast relation 19 | 20 | # to get the path of the index: 21 | 22 | #select 23 | #pg_relation_filepath(c.reltoastrelid) as toast_table_path, 24 | #pg_relation_filepath(i.indexrelid) as toast_index_path 25 | #from pg_class c 26 | #left outer join pg_index i on c.reltoastrelid=i.indrelid 27 | #where c.relname = 'messages'; 28 | 29 | if [ "$1" = "--help" -o $# -eq 0 ] ; then 30 | echo "" 31 | echo "Usage: " 32 | echo "" 33 | echo "-b: block to inspect" 34 | echo "-p: relation path on disk" 35 | echo "" 36 | exit 0 37 | fi 38 | 39 | POSITIONAL=() 40 | while [[ $# -gt 0 ]] 41 | do 42 | key="$1" 43 | 44 | case $key in 45 | -b|--block) 46 | blk="$2" 47 | shift # past argument 48 | shift # past value 49 | ;; 50 | -p|--path) 51 | relpath="$2" 52 | shift # past argument 53 | shift # past value 54 | ;; 55 | *) # unknown option 56 | POSITIONAL+=("$1") # save it in an array for later 57 | shift # past argument 58 | ;; 59 | esac 60 | done 61 | set -- "${POSITIONAL[@]}" 62 | 63 | echo "" 64 | echo "BLOCK = ${blk}" 65 | echo "PATH = ${relpath}" 66 | echo "" 67 | 68 | echo "from | ofs | t_ctid | len | nu | va | chunk_id | chunk_seq " 69 | echo "-----+-----+---------------------+-----+----+----+-------------------|-------------------" 70 | 71 | filenumber=`expr $blk / 131072` 72 | blktoread=0 73 | if (( $filenumber >= 1 )) 74 | then 75 | relpath=$relpath.$filenumber 76 | blktoread=`expr 131072 \\* $filenumber` 77 | fi 78 | blkmem=$blk 79 | blk=`expr $blk - $blktoread` 80 | 81 | dsk_lower=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t d -j 12 -N 2` 82 | nbentries=`expr $dsk_lower - 24` 83 | nbentries=`expr $nbentries / 4` 84 | ent=1 85 | while [ $ent -le $nbentries ] 86 | do 87 | skipbytes=$(( 4 * $ent - 4 )) 88 | skipbytes=$(( 24 + $skipbytes )) 89 | dsk_item_off=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x2 -j $skipbytes -N 2 | sed 's/^ *//'` 90 | dsk_item_off=`echo $((0x$dsk_item_off & ~$((1<<15))))` 91 | skipbytes=$(( 2 + $skipbytes )) 92 | dsk_item_len=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x2 -j $skipbytes -N 2 | sed 's/^ *//'` 93 | dsk_item_len=`echo $((0x$dsk_item_len >> 1))` 94 | toread=$(( 0 + $dsk_item_off )) 95 | dsk_ctid_block_number=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x2 -j $toread -N 4 | sed 's/ //g'` 96 | dsk_ctid_block_number=`echo $dsk_ctid_block_number | tr '[:lower:]' '[:upper:]'` 97 | dsk_ctid_block_number=`echo "ibase=16;$dsk_ctid_block_number"|bc` 98 | toread=$(( 4 + $dsk_item_off )) 99 | dsk_ctid_tuple_id=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u2 -j $toread -N 2` 100 | toread=$(( 6 + $dsk_item_off )) 101 | dsk_t_info=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u2 -j $toread -N 2` 102 | dsk_t_info_b=`echo "obase=2;$dsk_t_info" | bc | rev` 103 | # typedef struct IndexTupleData 104 | # { 105 | # ItemPointerData t_tid; /* reference TID to heap tuple */ 106 | # /* --------------- 107 | # * t_info is laid out in the following fashion: 108 | # * 109 | # * 15th (high) bit: has nulls 110 | # * 14th bit: has var-width attributes 111 | # * 13th bit: unused 112 | # * 12-0 bit: size of tuple 113 | # * --------------- 114 | # */ 115 | # unsigned short t_info; /* various info about tuple */ 116 | # } IndexTupleData; 117 | has_var=`echo $dsk_t_info_b | cut -c15` #0 to 15 so need to ad one here 118 | if [ ! -z "$has_var" ] && (( $has_var == 1 )) 119 | then 120 | item_has_var="t" 121 | else 122 | item_has_var="f" 123 | fi 124 | 125 | has_null=`echo $dsk_t_info_b | cut -c16` #0 to 15 so need to ad one here 126 | if [ ! -z "$has_null" ] && (( $has_null == 1 )) 127 | then 128 | item_has_null="t" 129 | else 130 | item_has_null="f" 131 | fi 132 | 133 | toread=$(( 8 + $dsk_item_off )) 134 | toread_seq=$(( 4 + $toread )) 135 | n_toread=$(( $dsk_item_len - 8 )) 136 | #dsk_t_data=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t x4 -j $toread -N $n_toread` 137 | dsk_t_chunkid=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u4 -j $toread -N 4` 138 | dsk_t_chunkseq=`dd status=none bs=8192 count=1 if=$relpath skip=$blk | od -A n -t u4 -j $toread_seq -N 4` 139 | printf "%-4s %1s %-3s %1s %1s %-8s %1s %-4s %1s %1s %-3s %1s %-2s %1s %-2s %1s %-17s %1s %-17s\n" 'disk' '|' $ent '|' '(' $dsk_ctid_block_number ',' $dsk_ctid_tuple_id ')' '|' $dsk_item_len '|' $item_has_null '|' $item_has_var '|' $dsk_t_chunkid '|' $dsk_t_chunkseq 140 | ent=$(( $ent + 1 )) 141 | done 142 | -------------------------------------------------------------------------------- /from_files_only/get_multixid_members.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$1" = "--help" ] || [ $# -lt 4 ] ; then 4 | echo "" 5 | echo "Usage: " 6 | echo "" 7 | echo "-m: mxid (mandatory)" 8 | echo "-d: DATA path (mandatory)" 9 | echo "-b: block size (default 8192)" 10 | echo "" 11 | echo "example: ./get_multixid_members.sh -m 2 -d /usr/local/pgsql11.6/data/" 12 | exit 0 13 | fi 14 | 15 | POSITIONAL=() 16 | while [[ $# -gt 0 ]] 17 | do 18 | key="$1" 19 | 20 | case $key in 21 | -m|--mxid) 22 | mxid="$2" 23 | shift # past argument 24 | shift # past value 25 | ;; 26 | -d|--data) 27 | pgdata="$2" 28 | shift # past argument 29 | shift # past value 30 | ;; 31 | -b|--blcks) 32 | blcks="$2" 33 | shift # past argument 34 | shift # past value 35 | ;; 36 | *) # unknown option 37 | POSITIONAL+=("$1") # save it in an array for later 38 | shift # past argument 39 | ;; 40 | esac 41 | done 42 | 43 | set -- "${POSITIONAL[@]}" 44 | 45 | blcks=${blcks:-8192} 46 | 47 | echo "" 48 | echo "MXID = ${mxid}" 49 | echo "DATA = ${pgdata}" 50 | echo "BLCKS = ${blcks}" 51 | echo "" 52 | 53 | #See multixact.c: 54 | #MULTIXACT_OFFSETS_PER_PAGE is 2048 55 | #SLRU_PAGES_PER_SEGMENT is 32 56 | #MULTIXACT_MEMBERS_PER_MEMBERGROUP is 4 57 | #MULTIXACT_MEMBERGROUP_SIZE is 20 58 | #MULTIXACT_MEMBERGROUPS_PER_PAGE is 409 59 | #MULTIXACT_MEMBERS_PER_PAGE is 1636 60 | #MAX_MEMBERS_IN_LAST_MEMBERS_PAGE is 1036 61 | #sizeof(TransactionId) is 4 62 | 63 | OFFLOC="${pgdata}/pg_multixact/offsets/" 64 | MEMBLOC="${pgdata}/pg_multixact/members/" 65 | 66 | #SLRU_PAGES_PER_SEGMENT is 32 67 | slru_pages_per_segment=32 68 | 69 | #define MULTIXACT_MEMBERGROUPS_PER_PAGE (BLCKSZ / MULTIXACT_MEMBERGROUP_SIZE) 70 | multixact_membergroups_per_page=`expr ${blcks} / 20` 71 | 72 | #define MULTIXACT_MEMBERS_PER_PAGE (MULTIXACT_MEMBERGROUPS_PER_PAGE * MULTIXACT_MEMBERS_PER_MEMBERGROUP) 73 | multixact_members_per_page=`expr ${multixact_membergroups_per_page} \* 4` 74 | 75 | #define MULTIXACT_OFFSETS_PER_PAGE (BLCKSZ / sizeof(MultiXactOffset)) 76 | multixact_offsets_per_page=`expr ${blcks} / 4` 77 | 78 | #define MultiXactIdToOffsetPage(xid) \ ((xid) / (MultiXactOffset) MULTIXACT_OFFSETS_PER_PAGE) 79 | pagenumber=`expr ${mxid} / ${multixact_offsets_per_page}` 80 | pageinfile=`expr ${pagenumber} % ${slru_pages_per_segment}` 81 | 82 | #define MultiXactIdToOffsetSegment(xid) (MultiXactIdToOffsetPage(xid) / SLRU_PAGES_PER_SEGMENT) 83 | hexfic=`expr ${pagenumber} / ${slru_pages_per_segment}` 84 | hexfic=`echo "ibase=10;obase=16;${hexfic}" | bc` 85 | hexfic=`printf "%04x" "0x${hexfic}" | tr '[:lower:]' '[:upper:]'` 86 | 87 | #define MultiXactIdToOffsetEntry(xid) \ ((xid) % (MultiXactOffset) MULTIXACT_OFFSETS_PER_PAGE) 88 | offsetid=`expr ${mxid} % ${multixact_offsets_per_page}` 89 | 90 | # Same computation with next mxid 91 | nextmxid=`expr ${mxid} + 1` 92 | nextpagenumber=`expr ${nextmxid} / ${multixact_offsets_per_page}` 93 | nextpageinfile=`expr ${nextpagenumber} % ${slru_pages_per_segment}` 94 | nexthexfic=`expr ${nextpagenumber} / ${slru_pages_per_segment}` 95 | nexthexfic=`echo "ibase=10;obase=16;${nexthexfic}" | bc` 96 | nexthexfic=`printf "%04x" "0x${nexthexfic}" | tr '[:lower:]' '[:upper:]'` 97 | nextoffsetid=`expr ${nextmxid} % ${multixact_offsets_per_page}` 98 | 99 | #echo "Looking for offset $offsetid in page $pagenumber" 100 | #echo "Page $pagenumber is page $pageinfile in file ${OFFLOC}${hexfic}" 101 | #echo "" 102 | 103 | # read with dd 104 | skipread=`expr ${offsetid} \* 4` 105 | next_skipread=`expr ${nextoffsetid} \* 4` 106 | 107 | offset_members=`dd status=none bs=${blcks} count=1 if=${OFFLOC}${hexfic} skip=$pageinfile | od -A n -t u4 -j $skipread -N 4 | sed 's/^ *//'` 108 | next_offset_members=`dd status=none bs=${blcks} count=1 if=${OFFLOC}${nexthexfic} skip=$nextpageinfile | od -A n -t u4 -j $next_skipread -N 4 | sed 's/^ *//'` 109 | #echo "Offset value is: ${offset_members}" 110 | #echo "NextOffset value is: ${next_offset_members}" 111 | 112 | [ ${offset_members} -eq 0 ] && echo "No members to look for, exiting...." && exit 0 113 | 114 | # Now look at the members 115 | echo "" 116 | echo "Members are:" 117 | off=${offset_members} 118 | 119 | if [ ! ${next_offset_members} -eq 0 ] 120 | then 121 | 122 | while [ $off -lt $next_offset_members ] 123 | do 124 | #define MXOffsetToMemberPage(xid) ((xid) / (TransactionId) MULTIXACT_MEMBERS_PER_PAGE) 125 | memberpage=`expr ${off} / ${multixact_members_per_page}` 126 | memberpageinfile=`expr ${memberpage} % ${slru_pages_per_segment}` 127 | 128 | #define MXOffsetToMemberSegment(xid) (MXOffsetToMemberPage(xid) / SLRU_PAGES_PER_SEGMENT) 129 | membersegment=`expr ${memberpage} / ${slru_pages_per_segment}` 130 | membersegment=`echo "ibase=10;obase=16;${membersegment}" | bc` 131 | membersegment=`printf "%04x" "0x${membersegment}" | tr '[:lower:]' '[:upper:]'` 132 | 133 | #define MXOffsetToFlagsOffset(xid) \ ((((xid) / (TransactionId) MULTIXACT_MEMBERS_PER_MEMBERGROUP) % (TransactionId) MULTIXACT_MEMBERGROUPS_PER_PAGE) * (TransactionId) MULTIXACT_MEMBERGROUP_SIZE) 134 | flagtooffset=`echo "$(( (${off} / 4 ) % ${multixact_membergroups_per_page} * 20 ))"` 135 | 136 | #define MXOffsetToMemberOffset(xid) \ (MXOffsetToFlagsOffset(xid) + MULTIXACT_FLAGBYTES_PER_GROUP + ((xid) % MULTIXACT_MEMBERS_PER_MEMBERGROUP) * sizeof(TransactionId)) 137 | memberoffset=`echo "$(( ${flagtooffset} + 4 + ( ${off} % 4) * 4 ))"` 138 | 139 | #echo "Reading offset $memberoffset in page $memberpageinfile in file ${MEMBLOC}${membersegment}" 140 | dd status=none bs=${blcks} count=1 if=${MEMBLOC}${membersegment} skip=$memberpageinfile | od -A n -t u4 -j $memberoffset -N 4 | sed 's/^ *//' 141 | off=$(( $off + 1 )) 142 | done 143 | 144 | else 145 | nextone=true 146 | while $nextone 147 | do 148 | #define MXOffsetToMemberPage(xid) ((xid) / (TransactionId) MULTIXACT_MEMBERS_PER_PAGE) 149 | memberpage=`expr ${off} / ${multixact_members_per_page}` 150 | memberpageinfile=`expr ${memberpage} % ${slru_pages_per_segment}` 151 | 152 | #define MXOffsetToMemberSegment(xid) (MXOffsetToMemberPage(xid) / SLRU_PAGES_PER_SEGMENT) 153 | membersegment=`expr ${memberpage} / ${slru_pages_per_segment}` 154 | membersegment=`echo "ibase=10;obase=16;${membersegment}" | bc` 155 | membersegment=`printf "%04x" "0x${membersegment}" | tr '[:lower:]' '[:upper:]'` 156 | 157 | #define MXOffsetToFlagsOffset(xid) \ ((((xid) / (TransactionId) MULTIXACT_MEMBERS_PER_MEMBERGROUP) % (TransactionId) MULTIXACT_MEMBERGROUPS_PER_PAGE) * (TransactionId) MULTIXACT_MEMBERGROUP_SIZE) 158 | flagtooffset=`echo "$(( (${off} / 4 ) % ${multixact_membergroups_per_page} * 20 ))"` 159 | 160 | #define MXOffsetToMemberOffset(xid) \ (MXOffsetToFlagsOffset(xid) + MULTIXACT_FLAGBYTES_PER_GROUP + ((xid) % MULTIXACT_MEMBERS_PER_MEMBERGROUP) * sizeof(TransactionId)) 161 | memberoffset=`echo "$(( ${flagtooffset} + 4 + ( ${off} % 4) * 4 ))"` 162 | 163 | #echo "Reading offset $memberoffset in page $memberpageinfile in file ${MEMBLOC}${membersegment}" 164 | THISMEMB=`dd status=none bs=${blcks} count=1 if=${MEMBLOC}${membersegment} skip=$memberpageinfile | od -A n -t u4 -j $memberoffset -N 4 | sed 's/^ *//'` 165 | if [ $THISMEMB -ne 0 ] 166 | then 167 | echo "${THISMEMB}" 168 | off=$(( $off + 1 )) 169 | else 170 | nextone=false 171 | fi 172 | done 173 | 174 | fi 175 | -------------------------------------------------------------------------------- /from_files_only/get_xact_status.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$1" = "--help" ] || [ $# -lt 4 ] ; then 4 | echo "" 5 | echo "Usage: " 6 | echo "" 7 | echo "-x: xid" 8 | echo "-d: DATA path" 9 | echo "-b: block size (default 8192)" 10 | echo "" 11 | echo "example: $0 -x 31795287 -d /usr/local/pgsql11.6/data/" 12 | exit 0 13 | fi 14 | 15 | POSITIONAL=() 16 | while [[ $# -gt 0 ]] 17 | do 18 | key="$1" 19 | 20 | case $key in 21 | -x|--xid) 22 | xid="$2" 23 | shift # past argument 24 | shift # past value 25 | ;; 26 | -d|--data) 27 | pgdata="$2" 28 | shift # past argument 29 | shift # past value 30 | ;; 31 | -b|--blcks) 32 | blcks="$2" 33 | shift # past argument 34 | shift # past value 35 | ;; 36 | *) # unknown option 37 | POSITIONAL+=("$1") # save it in an array for later 38 | shift # past argument 39 | ;; 40 | esac 41 | done 42 | 43 | set -- "${POSITIONAL[@]}" 44 | 45 | blcks=${blcks:-8192} 46 | 47 | echo "" 48 | echo "XID = ${xid}" 49 | echo "DATA = ${pgdata}" 50 | echo "BLCKS = ${blcks}" 51 | echo "" 52 | 53 | #See clog.c: 54 | #define CLOG_BITS_PER_XACT 2 55 | #define CLOG_XACTS_PER_BYTE 4 56 | #define CLOG_XACTS_PER_PAGE (BLCKSZ * CLOG_XACTS_PER_BYTE) 57 | 58 | PGCLOG="${pgdata}/pg_xact/" 59 | 60 | #SLRU_PAGES_PER_SEGMENT is 32 61 | slru_pages_per_segment=32 62 | 63 | #define CLOG_XACTS_PER_PAGE (BLCKSZ * CLOG_XACTS_PER_BYTE) 64 | clog_xacts_per_page=`expr ${blcks} \* 4` 65 | 66 | #define TransactionIdToPage(xid) ((xid) / (TransactionId) CLOG_XACTS_PER_PAGE) 67 | pagenumber=`expr ${xid} / ${clog_xacts_per_page}` 68 | pageinfile=`expr ${pagenumber} % ${slru_pages_per_segment}` 69 | 70 | hexfic=`expr ${pagenumber} / ${slru_pages_per_segment}` 71 | hexfic=`echo "ibase=10;obase=16;${hexfic}" | bc` 72 | hexfic=`printf "%04x" "0x${hexfic}" | tr '[:lower:]' '[:upper:]'` 73 | 74 | #define TransactionIdToPgIndex(xid) ((xid) % (TransactionId) CLOG_XACTS_PER_PAGE) 75 | offsetid=`expr ${xid} % ${clog_xacts_per_page}` 76 | 77 | #define TransactionIdToByte(xid) (TransactionIdToPgIndex(xid) / CLOG_XACTS_PER_BYTE) 78 | xidtobyte=`expr ${offsetid} / 4` 79 | 80 | #define TransactionIdToBIndex(xid) ((xid) % (TransactionId) CLOG_XACTS_PER_BYTE) 81 | xidtobindex=`expr ${xid} % 4` 82 | 83 | BYREAD=`dd status=none bs=${blcks} count=1 if=${PGCLOG}${hexfic} skip=$pageinfile | od -A n -t x1 -j $xidtobyte -N 1| sed 's/^ *//'` 84 | BYREAD=`echo $BYREAD | tr '[:lower:]' '[:upper:]'` 85 | ALLBITS=`echo "ibase=16;obase=2;${BYREAD}" | bc` 86 | NBBITS=`echo -n "$ALLBITS" | wc -c` 87 | 88 | # get the right 2 bits 89 | HEAD=`expr ${xidtobindex} + 1` 90 | HEAD=`expr ${HEAD} \* 2` 91 | HEADMIN1=`expr ${HEAD} - 1` 92 | 93 | # if we want a bit position greater than we want to 94 | # aka xid not written in pg_xact yet 95 | if (( $HEADMIN1 > $NBBITS)) 96 | then 97 | echo "xid $xid status is: UNKNOWN" 98 | exit 99 | fi 100 | 101 | # Read the 2 bits of interest 102 | CSTATUS=`echo "$ALLBITS" | rev | head -c $HEAD | tail -c 2 | rev` 103 | echo "Reading bits $HEADMIN1,$HEAD in byte $xidtobyte in page $pageinfile of file ${hexfic}" 104 | 105 | #define TRANSACTION_STATUS_IN_PROGRESS 0x00 106 | #define TRANSACTION_STATUS_COMMITTED 0x01 107 | #define TRANSACTION_STATUS_ABORTED 0x02 108 | #define TRANSACTION_STATUS_SUB_COMMITTED 0x03 109 | 110 | echo "" 111 | case $CSTATUS in 112 | 0|00) 113 | echo "xid $xid status is: IN PROGRESS" 114 | ;; 115 | 1|01) 116 | echo "xid $xid status is: COMMITTED" 117 | ;; 118 | 10) 119 | echo "xid $xid status is: ABORTED" 120 | ;; 121 | 11) 122 | echo "xid $xid status is: SUB COMMITED" 123 | ;; 124 | *) 125 | echo "xid $xid status is: UNKNOWN" 126 | ;; 127 | esac 128 | -------------------------------------------------------------------------------- /sql/get_infomasks_insights.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | -- See htup_details.h 3 | t_ctid AS ctid, 4 | t_xmin AS xmin, 5 | t_xmax AS xmax, 6 | t_infomask, 7 | (t_infomask & 16)::boolean AS xmax_kshr_lock, 8 | (t_infomask & 64)::boolean AS xmax_excl_lock, 9 | (t_infomask & 128)::boolean AS xmax_lock_only, 10 | (t_infomask & 1024)::boolean AS xmax_committed, 11 | (t_infomask & 2048)::boolean AS xmax_invalid, 12 | (t_infomask & 4096)::boolean AS xmax_multixact, 13 | (t_infomask & 1)::boolean AS has_null, 14 | (t_infomask & 2)::boolean AS has_varwidth, 15 | (t_infomask & 4)::boolean AS has_external, 16 | (t_infomask & 8)::boolean AS has_oid, 17 | (t_infomask & 32)::boolean AS combo_cid, 18 | (t_infomask & 256)::boolean AS xmin_commited, 19 | (t_infomask & 512)::boolean AS xmin_invalid, 20 | (t_infomask::bit(16) & x'0300' = x'0300'::bit(16))::boolean AS xmin_frozen, 21 | (t_infomask & 8192)::boolean AS updated, 22 | (t_infomask & 16384)::boolean AS moved_off, 23 | (t_infomask & 32768)::boolean AS moved_in, 24 | ((((((t_infomask & ~16) & ~64) & ~128) & ~1024) & ~2048) & ~4096) | 2048 as infomask_with_xmax_invalid, 25 | t_infomask2, 26 | (t_infomask2 & 8192)::boolean AS keys_updated, 27 | (t_infomask2 & 16384)::boolean AS hot_updated, 28 | (t_infomask2 & 32768)::boolean AS only_tuple 29 | FROM heap_page_items(get_raw_page('parent', 0)) ; 30 | 31 | select lp,t_ctid, 32 | CASE WHEN t_infomask::bit(16) & x'0300' = x'0300'::bit(16) THEN 'HEAP_XMIN_FROZEN|' 33 | ELSE '|' 34 | END 35 | || CASE WHEN t_infomask::bit(16) & x'8000' = x'8000'::bit(16) THEN 'MOVED_IN|' 36 | ELSE '|' 37 | END 38 | || CASE WHEN t_infomask::bit(16) & x'4000' = x'4000'::bit(16) THEN 'MOVED_OFF|' 39 | ELSE '|' 40 | END 41 | || CASE WHEN t_infomask::bit(16) & x'2000' = x'2000'::bit(16) THEN 'UPDATED|' 42 | ELSE '|' 43 | END 44 | || CASE WHEN t_infomask::bit(16) & x'1000' = x'1000'::bit(16) THEN 'XMAX_IS_MULTI|' 45 | ELSE '|' 46 | END 47 | || CASE WHEN t_infomask::bit(16) & x'0800' = x'0800'::bit(16) THEN 'XMAX_INVALID|' 48 | ELSE '|' 49 | END 50 | || CASE WHEN t_infomask::bit(16) & x'0400' = x'0400'::bit(16) THEN 'XMAX_COMMITTED|' 51 | ELSE '|' 52 | END 53 | || CASE WHEN t_infomask::bit(16) & x'0200' = x'0200'::bit(16) THEN 'XMIN_INVALID|' 54 | ELSE '|' 55 | END 56 | || CASE WHEN t_infomask::bit(16) & x'0080' = x'0080'::bit(16) THEN 'XMAX_LOCK_ONLY|' 57 | ELSE '|' 58 | END 59 | || CASE WHEN t_infomask::bit(16) & x'0040' = x'0040'::bit(16) THEN 'EXCL_LOCK|' 60 | ELSE '|' 61 | END 62 | || CASE WHEN t_infomask::bit(16) & x'0020' = x'0020'::bit(16) THEN 'COMBOCID|' 63 | ELSE '|' 64 | END 65 | || CASE WHEN t_infomask::bit(16) & x'0010' = x'0010'::bit(16) THEN 'XMAX_KEYSHR_LOCK|' 66 | ELSE '|' 67 | END 68 | || CASE WHEN t_infomask::bit(16) & x'0008' = x'0008'::bit(16) THEN 'HASOID|' 69 | ELSE '|' 70 | END 71 | || CASE WHEN t_infomask::bit(16) & x'0004' = x'0004'::bit(16) THEN 'HASEXTERNAL|' 72 | ELSE '|' 73 | END 74 | || CASE WHEN t_infomask::bit(16) & x'0002' = x'0002'::bit(16) THEN 'HASVARWIDTH|' 75 | ELSE '|' 76 | END 77 | || CASE WHEN t_infomask::bit(16) & x'0001' = x'0001'::bit(16) THEN 'HASNULL|' 78 | ELSE '|' 79 | END 80 | || CASE WHEN t_infomask::bit(16) & x'0100' = x'0100'::bit(16) THEN 'XMIN_COMMITTED|' 81 | ELSE '|' 82 | END as t_infomask_info, 83 | CASE WHEN t_infomask2::bit(16) & x'2000' = x'2000'::bit(16) THEN 'HEAP_KEYS_UPDATED|' 84 | ELSE '|' 85 | END 86 | || CASE WHEN t_infomask2::bit(16) & x'4000' = x'4000'::bit(16) THEN 'HEAP_HOT_UPDATED|' 87 | ELSE '|' 88 | END 89 | || CASE WHEN t_infomask2::bit(16) & x'8000' = x'8000'::bit(16) THEN 'HEAP_ONLY_TUPLE|' 90 | ELSE '|' 91 | END as t_infomask2_info 92 | FROM heap_page_items(get_raw_page('parent', 0)); 93 | --------------------------------------------------------------------------------