Re: relfrozenxid may disagree with row XIDs after 1ccc1e05ae

Hi,

On Thu, May 16, 2024 at 11:33 AM Andres Freund <andres@anarazel.de> wrote:

> Hi,
>
> On 2024-05-16 10:21:59 +0800, Bowen Shi wrote:
> > Yes,  I found the vacuum process hanging in a retry loop on PG16. I think
> > it was the problem discussed in this thread. Here is some information:
>
> Very interesting.
>
>
> Do you have the page contents that triggered the issue?


Yes, I have preserved the coredump file of vacuum and several other related
processes. It has been at least 10 minutes stuck in the retry loop since I
dumped this core file.

(gdb) p *(BufferDesc *) (&BufferDescriptors[(buffer)].bufferdesc)
$8 = {tag = {spcOid = 1663, dbOid = 16425, relNumber = 61755, forkNum =
MAIN_FORKNUM,
    blockNum = 50002}, buf_id = 67521, state = {value = 3548643328},
  wait_backend_pgprocno = 2147483647, freeNext = -2, content_lock =
{tranche = 78, state = {
      value = 536870912}, waiters = {head = 2147483647, tail = 2147483647}}}

And in lazy_scan_prune() :
(gdb) p maxoff
$11 = 32
(gdb) p offnum
$12 = 1

Item info for offsets 1 to 32 using loop:
(gdb) define print_itemids
> set $i = 1
> while $i <= 32
> printf "offset %d: ", $i
> print ((PageHeader)(page))->pd_linp[$i - 1]
> set $i = $i + 1
> end
> end
(gdb) print_itemids

offset 1: $1 = {lp_off = 7944, lp_flags = 1, lp_len = 248}
offset 2: $2 = {lp_off = 7696, lp_flags = 1, lp_len = 248}
offset 3: $3 = {lp_off = 7448, lp_flags = 1, lp_len = 248}
offset 4: $4 = {lp_off = 7200, lp_flags = 1, lp_len = 248}
offset 5: $5 = {lp_off = 6952, lp_flags = 1, lp_len = 248}
offset 6: $6 = {lp_off = 6704, lp_flags = 1, lp_len = 248}
offset 7: $7 = {lp_off = 6456, lp_flags = 1, lp_len = 248}
offset 8: $8 = {lp_off = 6208, lp_flags = 1, lp_len = 248}
offset 9: $9 = {lp_off = 5960, lp_flags = 1, lp_len = 248}
offset 10: $10 = {lp_off = 5712, lp_flags = 1, lp_len = 248}
offset 11: $11 = {lp_off = 5464, lp_flags = 1, lp_len = 248}
offset 12: $12 = {lp_off = 5216, lp_flags = 1, lp_len = 248}
offset 13: $13 = {lp_off = 4968, lp_flags = 1, lp_len = 248}
offset 14: $14 = {lp_off = 4720, lp_flags = 1, lp_len = 248}
offset 15: $15 = {lp_off = 4472, lp_flags = 1, lp_len = 248}
offset 16: $16 = {lp_off = 4224, lp_flags = 1, lp_len = 248}
offset 17: $17 = {lp_off = 3976, lp_flags = 1, lp_len = 248}
offset 18: $18 = {lp_off = 3728, lp_flags = 1, lp_len = 248}
offset 19: $19 = {lp_off = 3480, lp_flags = 1, lp_len = 248}
offset 20: $20 = {lp_off = 3232, lp_flags = 1, lp_len = 248}
offset 21: $21 = {lp_off = 2984, lp_flags = 1, lp_len = 248}
offset 22: $22 = {lp_off = 2736, lp_flags = 1, lp_len = 248}
offset 23: $23 = {lp_off = 2488, lp_flags = 1, lp_len = 248}
offset 24: $24 = {lp_off = 2240, lp_flags = 1, lp_len = 248}
offset 25: $25 = {lp_off = 1992, lp_flags = 1, lp_len = 248}
offset 26: $26 = {lp_off = 1744, lp_flags = 1, lp_len = 248}
offset 27: $27 = {lp_off = 1496, lp_flags = 1, lp_len = 248}
offset 28: $28 = {lp_off = 1248, lp_flags = 1, lp_len = 248}
offset 29: $29 = {lp_off = 1000, lp_flags = 1, lp_len = 248}
offset 30: $30 = {lp_off = 752, lp_flags = 1, lp_len = 248}
offset 31: $31 = {lp_off = 504, lp_flags = 1, lp_len = 248}
offset 32: $32 = {lp_off = 256, lp_flags = 1, lp_len = 248}

Using following function to print HeapTupleHeader info:
(gdb) define print_all_heaptupleheaders
>set $i = 1
>while $i <= 32
 >set $tup = (HeapTupleHeader) ((char *) (page) + ((PageHeader)
(page))->pd_linp[$i - 1].lp_off)
 >print *$tup
 >set $i = $i + 1
 >end
>end
(gdb) print_all_heaptupleheaders

offset 1: $1 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 7},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b729f1f ""}
offset 2: $2 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 8},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b729e27 ""}
offset 3: $3 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 9},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b729d2f ""}
offset 4: $4 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 10},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b729c37 ""}
offset 5: $5 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 11},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b729b3f ""}
offset 6: $6 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 12},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b729a47 ""}
offset 7: $7 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 13},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b72994f ""}
offset 8: $8 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
---Type <return> to continue, or q <return> to quit---
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 14},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b729857 ""}
offset 9: $9 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 15},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b72975f ""}
offset 10: $10 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 16},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b729667 ""}
offset 11: $11 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 17},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b72956f ""}
offset 12: $12 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 18},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b729477 ""}
offset 13: $13 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 19},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b72937f ""}
offset 14: $14 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 20},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b729287 ""}
offset 15: $15 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 21},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b72918f ""}
---Type <return> to continue, or q <return> to quit---
offset 16: $16 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 22},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b729097 ""}
offset 17: $17 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 23},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b728f9f ""}
offset 18: $18 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 24},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b728ea7 ""}
offset 19: $19 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 25},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b728daf ""}
offset 20: $20 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 26},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b728cb7 ""}
offset 21: $21 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 27},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b728bbf ""}
offset 22: $22 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 28},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b728ac7 ""}
offset 23: $23 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
---Type <return> to continue, or q <return> to quit---
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 29},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b7289cf ""}
offset 24: $24 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 30},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b7288d7 ""}
offset 25: $25 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 31},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b7287df ""}
offset 26: $26 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54842}, ip_posid = 32},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b7286e7 ""}
offset 27: $27 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54843}, ip_posid = 1},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b7285ef ""}
offset 28: $28 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54843}, ip_posid = 2},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b7284f7 ""}
offset 29: $29 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54843}, ip_posid = 3},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b7283ff ""}
offset 30: $30 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54843}, ip_posid = 4},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b728307 ""}
---Type <return> to continue, or q <return> to quit---
offset 31: $31 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54843}, ip_posid = 5},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b72820f ""}
offset 32: $32 = {t_choice = {t_heap = {t_xmin = 1094, t_xmax = 1094,
t_field3 = {t_cid = 5986,
        t_xvac = 5986}}, t_datum = {datum_len_ = 1094, datum_typmod = 1094,
      datum_typeid = 5986}}, t_ctid = {ip_blkid = {bi_hi = 6, bi_lo =
54843}, ip_posid = 6},
  t_infomask2 = 24, t_infomask = 9506, t_hoff = 24 '\030', t_bits =
0x7fa25b728117 ""}

Do you have any other variables? E.g. RecentXmin,
> ComputeXidHorizonsResultLastXmin?


RecentXmin = 1079
ComputeXidHorizonsResultLastXmin = 1079


-- 
Regards
Bowen Shi