pglz_wal_update_v1.patch
application/octet-stream
Filename: pglz_wal_update_v1.patch
Type: application/octet-stream
Part: 1
Patch
Same data as JSON:
GET /api/v1/attachments/:id/patch
the parsed metadata as JSON — format, series position, per-file stats; never the diff bytes.
API reference →
Format: context
Series: patch v1
| File | + | − |
|---|---|---|
| src/backend/access/heap/heapam.c | 93 | 0 |
| src/backend/utils/adt/pg_lzcompress.c | 104 | 0 |
| src/include/access/heapam_xlog.h | 7 | 0 |
| src/include/utils/pg_lzcompress.h | 6 | 0 |
*** a/src/backend/access/heap/heapam.c
--- b/src/backend/access/heap/heapam.c
***************
*** 70,75 ****
--- 70,76 ----
#include "utils/snapmgr.h"
#include "utils/syscache.h"
#include "utils/tqual.h"
+ #include "utils/pg_lzcompress.h"
/* GUC variable */
***************
*** 85,90 **** static HeapTuple heap_prepare_insert(Relation relation, HeapTuple tup,
--- 86,92 ----
TransactionId xid, CommandId cid, int options);
static XLogRecPtr log_heap_update(Relation reln, Buffer oldbuf,
ItemPointerData from, Buffer newbuf, HeapTuple newtup,
+ HeapTuple oldtup,
bool all_visible_cleared, bool new_all_visible_cleared);
static bool HeapSatisfiesHOTUpdate(Relation relation, Bitmapset *hot_attrs,
HeapTuple oldtup, HeapTuple newtup);
***************
*** 3195,3204 **** l2:
/* XLOG stuff */
if (RelationNeedsWAL(relation))
{
! XLogRecPtr recptr = log_heap_update(relation, buffer, oldtup.t_self,
! newbuf, heaptup,
! all_visible_cleared,
! all_visible_cleared_new);
if (newbuf != buffer)
{
--- 3197,3208 ----
/* XLOG stuff */
if (RelationNeedsWAL(relation))
{
! XLogRecPtr recptr;
!
! recptr = log_heap_update(relation, buffer, oldtup.t_self,
! newbuf, heaptup, &oldtup,
! all_visible_cleared,
! all_visible_cleared_new);
if (newbuf != buffer)
{
***************
*** 4428,4434 **** log_heap_visible(RelFileNode rnode, BlockNumber block, Buffer vm_buffer,
*/
static XLogRecPtr
log_heap_update(Relation reln, Buffer oldbuf, ItemPointerData from,
! Buffer newbuf, HeapTuple newtup,
bool all_visible_cleared, bool new_all_visible_cleared)
{
xl_heap_update xlrec;
--- 4432,4438 ----
*/
static XLogRecPtr
log_heap_update(Relation reln, Buffer oldbuf, ItemPointerData from,
! Buffer newbuf, HeapTuple newtup, HeapTuple oldtup,
bool all_visible_cleared, bool new_all_visible_cleared)
{
xl_heap_update xlrec;
***************
*** 4437,4442 **** log_heap_update(Relation reln, Buffer oldbuf, ItemPointerData from,
--- 4441,4456 ----
XLogRecPtr recptr;
XLogRecData rdata[4];
Page page = BufferGetPage(newbuf);
+ union
+ {
+ PGLZ_Header pglzheader;
+ char buf[BLCKSZ];
+ } buf;
+ char *newtupdata;
+ int newtuplen;
+ char *oldtupdata;
+ int oldtuplen;
+ bool compressed = false;
/* Caller should not call me on a non-WAL-logged relation */
Assert(RelationNeedsWAL(reln));
***************
*** 4446,4456 **** log_heap_update(Relation reln, Buffer oldbuf, ItemPointerData from,
else
info = XLOG_HEAP_UPDATE;
xlrec.target.node = reln->rd_node;
xlrec.target.tid = from;
! xlrec.all_visible_cleared = all_visible_cleared;
xlrec.newtid = newtup->t_self;
! xlrec.new_all_visible_cleared = new_all_visible_cleared;
rdata[0].data = (char *) &xlrec;
rdata[0].len = SizeOfHeapUpdate;
--- 4460,4503 ----
else
info = XLOG_HEAP_UPDATE;
+ newtupdata = ((char *) newtup->t_data) + offsetof(HeapTupleHeaderData, t_bits);
+ newtuplen = newtup->t_len - offsetof(HeapTupleHeaderData, t_bits);
+ oldtupdata = ((char *) oldtup->t_data) + offsetof(HeapTupleHeaderData, t_bits);
+ oldtuplen = oldtup->t_len - offsetof(HeapTupleHeaderData, t_bits);
+
+ /* Is the update is going to the same page? */
+ if (oldbuf == newbuf)
+ {
+ /*
+ * enable this if you only want to compress the new tuple as is,
+ * without taking advantage of the old tuple.
+ */
+ #ifdef COMPRESS_ONLY
+ oldtuplen = 0;
+ #endif
+
+ /* Delta-encode the new tuple using the old tuple */
+ /* XXX: assert that the output buffer is large enough (PGLZ_MAX_OUTPUT) */
+ if (pglz_compress_with_history(newtupdata, newtuplen,
+ oldtupdata, oldtuplen,
+ (PGLZ_Header *) &buf.pglzheader, NULL))
+ {
+ compressed = true;
+ newtupdata = (char *) &buf.pglzheader;
+ newtuplen = VARSIZE(&buf.pglzheader);
+ }
+ }
+
+ xlrec.flags = 0;
xlrec.target.node = reln->rd_node;
xlrec.target.tid = from;
! if (all_visible_cleared)
! xlrec.flags |= XL_HEAP_UPDATE_ALL_VISIBLE_CLEARED;
xlrec.newtid = newtup->t_self;
! if (new_all_visible_cleared)
! xlrec.flags |= XL_HEAP_UPDATE_NEW_ALL_VISIBLE_CLEARED;
! if (compressed)
! xlrec.flags |= XL_HEAP_UPDATE_DELTA_ENCODED;
rdata[0].data = (char *) &xlrec;
rdata[0].len = SizeOfHeapUpdate;
***************
*** 4478,4489 **** log_heap_update(Relation reln, Buffer oldbuf, ItemPointerData from,
rdata[2].next = &(rdata[3]);
/* PG73FORMAT: write bitmap [+ padding] [+ oid] + data */
! rdata[3].data = (char *) newtup->t_data + offsetof(HeapTupleHeaderData, t_bits);
! rdata[3].len = newtup->t_len - offsetof(HeapTupleHeaderData, t_bits);
rdata[3].buffer = newbuf;
rdata[3].buffer_std = true;
rdata[3].next = NULL;
/* If new tuple is the single and first tuple on page... */
if (ItemPointerGetOffsetNumber(&(newtup->t_self)) == FirstOffsetNumber &&
PageGetMaxOffsetNumber(page) == FirstOffsetNumber)
--- 4525,4537 ----
rdata[2].next = &(rdata[3]);
/* PG73FORMAT: write bitmap [+ padding] [+ oid] + data */
! rdata[3].data = newtupdata;
! rdata[3].len = newtuplen;
rdata[3].buffer = newbuf;
rdata[3].buffer_std = true;
rdata[3].next = NULL;
+
/* If new tuple is the single and first tuple on page... */
if (ItemPointerGetOffsetNumber(&(newtup->t_self)) == FirstOffsetNumber &&
PageGetMaxOffsetNumber(page) == FirstOffsetNumber)
***************
*** 5232,5237 **** heap_xlog_update(XLogRecPtr lsn, XLogRecord *record, bool hot_update)
--- 5280,5287 ----
OffsetNumber offnum;
ItemId lp = NULL;
HeapTupleHeader htup;
+ HeapTupleHeader oldtup = NULL;
+ uint32 old_tup_len = 0;
struct
{
HeapTupleHeaderData hdr;
***************
*** 5246,5252 **** heap_xlog_update(XLogRecPtr lsn, XLogRecord *record, bool hot_update)
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
*/
! if (xlrec->all_visible_cleared)
{
Relation reln = CreateFakeRelcacheEntry(xlrec->target.node);
BlockNumber block = ItemPointerGetBlockNumber(&xlrec->target.tid);
--- 5296,5302 ----
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
*/
! if (xlrec->flags & XL_HEAP_UPDATE_ALL_VISIBLE_CLEARED)
{
Relation reln = CreateFakeRelcacheEntry(xlrec->target.node);
BlockNumber block = ItemPointerGetBlockNumber(&xlrec->target.tid);
***************
*** 5289,5295 **** heap_xlog_update(XLogRecPtr lsn, XLogRecord *record, bool hot_update)
if (PageGetMaxOffsetNumber(page) < offnum || !ItemIdIsNormal(lp))
elog(PANIC, "heap_update_redo: invalid lp");
! htup = (HeapTupleHeader) PageGetItem(page, lp);
htup->t_infomask &= ~(HEAP_XMAX_COMMITTED |
HEAP_XMAX_INVALID |
--- 5339,5346 ----
if (PageGetMaxOffsetNumber(page) < offnum || !ItemIdIsNormal(lp))
elog(PANIC, "heap_update_redo: invalid lp");
! oldtup = htup = (HeapTupleHeader) PageGetItem(page, lp);
! old_tup_len = ItemIdGetLength(lp);
htup->t_infomask &= ~(HEAP_XMAX_COMMITTED |
HEAP_XMAX_INVALID |
***************
*** 5308,5314 **** heap_xlog_update(XLogRecPtr lsn, XLogRecord *record, bool hot_update)
/* Mark the page as a candidate for pruning */
PageSetPrunable(page, record->xl_xid);
! if (xlrec->all_visible_cleared)
PageClearAllVisible(page);
/*
--- 5359,5365 ----
/* Mark the page as a candidate for pruning */
PageSetPrunable(page, record->xl_xid);
! if (xlrec->flags & XL_HEAP_UPDATE_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
/*
***************
*** 5330,5336 **** newt:;
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
*/
! if (xlrec->new_all_visible_cleared)
{
Relation reln = CreateFakeRelcacheEntry(xlrec->target.node);
BlockNumber block = ItemPointerGetBlockNumber(&xlrec->newtid);
--- 5381,5387 ----
* The visibility map may need to be fixed even if the heap page is
* already up-to-date.
*/
! if (xlrec->flags & XL_HEAP_UPDATE_NEW_ALL_VISIBLE_CLEARED)
{
Relation reln = CreateFakeRelcacheEntry(xlrec->target.node);
BlockNumber block = ItemPointerGetBlockNumber(&xlrec->newtid);
***************
*** 5380,5395 **** newsame:;
hsize = SizeOfHeapUpdate + SizeOfHeapHeader;
newlen = record->xl_len - hsize;
Assert(newlen <= MaxHeapTupleSize);
memcpy((char *) &xlhdr,
(char *) xlrec + SizeOfHeapUpdate,
SizeOfHeapHeader);
htup = &tbuf.hdr;
MemSet((char *) htup, 0, sizeof(HeapTupleHeaderData));
! /* PG73FORMAT: get bitmap [+ padding] [+ oid] + data */
! memcpy((char *) htup + offsetof(HeapTupleHeaderData, t_bits),
! (char *) xlrec + hsize,
! newlen);
newlen += offsetof(HeapTupleHeaderData, t_bits);
htup->t_infomask2 = xlhdr.t_infomask2;
htup->t_infomask = xlhdr.t_infomask;
--- 5431,5470 ----
hsize = SizeOfHeapUpdate + SizeOfHeapHeader;
newlen = record->xl_len - hsize;
+
Assert(newlen <= MaxHeapTupleSize);
memcpy((char *) &xlhdr,
(char *) xlrec + SizeOfHeapUpdate,
SizeOfHeapHeader);
htup = &tbuf.hdr;
MemSet((char *) htup, 0, sizeof(HeapTupleHeaderData));
!
! /*
! * If the new tuple was delta-encoded, decode it.
! */
! if (xlrec->flags & XL_HEAP_UPDATE_DELTA_ENCODED)
! {
! PGLZ_Header *encoded_data = (PGLZ_Header *) (((char *) xlrec) + hsize);
!
! /*
! * FIXME: this won't work on architectures with strict alignment,
! * because encoded_data might not be aligned and pglz_decompress
! * assumes that the PGLZ_Header is correctly aligned. XXX: also add
! * some sanity checks with PGLZ_RAW_SIZE here.
! */
! pglz_decompress_with_history(encoded_data,
! ((char *) htup) + offsetof(HeapTupleHeaderData, t_bits),
! ((char *) oldtup) + offsetof(HeapTupleHeaderData, t_bits),
! old_tup_len - offsetof(HeapTupleHeaderData, t_bits));
! newlen = encoded_data->rawsize;
! }
! else
! {
! /* PG73FORMAT: get bitmap [+ padding] [+ oid] + data */
! memcpy((char *) htup + offsetof(HeapTupleHeaderData, t_bits),
! (char *) xlrec + hsize,
! newlen);
! }
newlen += offsetof(HeapTupleHeaderData, t_bits);
htup->t_infomask2 = xlhdr.t_infomask2;
htup->t_infomask = xlhdr.t_infomask;
***************
*** 5404,5410 **** newsame:;
if (offnum == InvalidOffsetNumber)
elog(PANIC, "heap_update_redo: failed to add tuple");
! if (xlrec->new_all_visible_cleared)
PageClearAllVisible(page);
freespace = PageGetHeapFreeSpace(page); /* needed to update FSM below */
--- 5479,5485 ----
if (offnum == InvalidOffsetNumber)
elog(PANIC, "heap_update_redo: failed to add tuple");
! if (xlrec->flags & XL_HEAP_UPDATE_NEW_ALL_VISIBLE_CLEARED)
PageClearAllVisible(page);
freespace = PageGetHeapFreeSpace(page); /* needed to update FSM below */
*** a/src/backend/utils/adt/pg_lzcompress.c
--- b/src/backend/utils/adt/pg_lzcompress.c
***************
*** 373,379 **** do { \
*/
static inline int
pglz_find_match(PGLZ_HistEntry **hstart, const char *input, const char *end,
! int *lenp, int *offp, int good_match, int good_drop)
{
PGLZ_HistEntry *hent;
int32 len = 0;
--- 373,380 ----
*/
static inline int
pglz_find_match(PGLZ_HistEntry **hstart, const char *input, const char *end,
! const char *hend, int *lenp, int *offp, int good_match,
! int good_drop)
{
PGLZ_HistEntry *hent;
int32 len = 0;
***************
*** 391,399 **** pglz_find_match(PGLZ_HistEntry **hstart, const char *input, const char *end,
int32 thislen;
/*
* Stop if the offset does not fit into our tag anymore.
*/
- thisoff = ip - hp;
if (thisoff >= 0x0fff)
break;
--- 392,408 ----
int32 thislen;
/*
+ * Check If the history presents,
+ * calculate the offset from history end instead of input
+ */
+ if (NULL == hend)
+ thisoff = ip - hp;
+ else
+ thisoff = hend - hp;
+
+ /*
* Stop if the offset does not fit into our tag anymore.
*/
if (thisoff >= 0x0fff)
break;
***************
*** 482,487 **** bool
--- 491,510 ----
pglz_compress(const char *source, int32 slen, PGLZ_Header *dest,
const PGLZ_Strategy *strategy)
{
+ return pglz_compress_with_history(source, slen, NULL, 0, dest, strategy);
+ }
+
+ /*
+ * Like pglz_compress, but uses another piece of data to initialize the
+ * history table. When decompressing, you must pass the same history data
+ * to pglz_decompress_with_history(). This makes it possible to do simple
+ * delta compression.
+ */
+ bool
+ pglz_compress_with_history(const char *source, int32 slen,
+ const char *history, int32 hlen,
+ PGLZ_Header *dest, const PGLZ_Strategy *strategy)
+ {
unsigned char *bp = ((unsigned char *) dest) + sizeof(PGLZ_Header);
unsigned char *bstart = bp;
int hist_next = 0;
***************
*** 500,505 **** pglz_compress(const char *source, int32 slen, PGLZ_Header *dest,
--- 523,530 ----
int32 result_size;
int32 result_max;
int32 need_rate;
+ const char *hp = NULL;
+ const char *hend = NULL;
/*
* Our fallback strategy is the default.
***************
*** 560,565 **** pglz_compress(const char *source, int32 slen, PGLZ_Header *dest,
--- 585,608 ----
* hist_entries[] array; its entries are initialized as they are used.
*/
memset(hist_start, 0, sizeof(hist_start));
+ if (hlen > 0)
+ {
+ hp = history;
+ hend = history + hlen;
+ while (hp < hend)
+ {
+ /*
+ * XXX: I think this doesn't handle the last few bytes of the
+ * history correctly, or at least not in the most efficient way.
+ * Logically, we should behave like the history and the source
+ * strings are concatenated, but we use 'hend' here.
+ */
+ pglz_hist_add(hist_start, hist_entries,
+ hist_next, hist_recycle,
+ hp, hend);
+ hp++; /* Do not do this ++ in the line above! */
+ }
+ }
/*
* Compress the source directly into the output buffer.
***************
*** 588,594 **** pglz_compress(const char *source, int32 slen, PGLZ_Header *dest,
/*
* Try to find a match in the history
*/
! if (pglz_find_match(hist_start, dp, dend, &match_len,
&match_off, good_match, good_drop))
{
/*
--- 631,637 ----
/*
* Try to find a match in the history
*/
! if (pglz_find_match(hist_start, dp, dend, hend, &match_len,
&match_off, good_match, good_drop))
{
/*
***************
*** 596,609 **** pglz_compress(const char *source, int32 slen, PGLZ_Header *dest,
* characters.
*/
pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off);
! while (match_len--)
{
! pglz_hist_add(hist_start, hist_entries,
! hist_next, hist_recycle,
! dp, dend);
! dp++; /* Do not do this ++ in the line above! */
! /* The macro would do it four times - Jan. */
}
found_match = true;
}
else
--- 639,670 ----
* characters.
*/
pglz_out_tag(ctrlp, ctrlb, ctrl, bp, match_len, match_off);
!
! /*
! * Incase of histor is passed as a separate buffer then don't add
! * source data further to the history. This is required as we need
! * to calculate the offset in the history buffer.
! */
! if (NULL == hend)
{
! while (match_len--)
! {
! pglz_hist_add(hist_start, hist_entries,
! hist_next, hist_recycle,
! dp, dend);
! dp++; /* Do not do this ++ in the line above! */
! /* The macro would do it four times - Jan. */
! }
}
+ else
+ {
+ /*
+ * Increment the source pointer with the match len directly
+ * because source data is not adding to the history.
+ */
+ dp += match_len;
+ }
+
found_match = true;
}
else
***************
*** 612,620 **** pglz_compress(const char *source, int32 slen, PGLZ_Header *dest,
* No match found. Copy one literal byte.
*/
pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp);
! pglz_hist_add(hist_start, hist_entries,
! hist_next, hist_recycle,
! dp, dend);
dp++; /* Do not do this ++ in the line above! */
/* The macro would do it four times - Jan. */
}
--- 673,687 ----
* No match found. Copy one literal byte.
*/
pglz_out_literal(ctrlp, ctrlb, ctrl, bp, *dp);
!
! /*
! * Incase of history is passed as a separate buffer, don't add any
! * unmatched input data to the history.
! */
! if (NULL == hend)
! pglz_hist_add(hist_start, hist_entries,
! hist_next, hist_recycle,
! dp, dend);
dp++; /* Do not do this ++ in the line above! */
/* The macro would do it four times - Jan. */
}
***************
*** 647,656 **** pglz_compress(const char *source, int32 slen, PGLZ_Header *dest,
--- 714,734 ----
void
pglz_decompress(const PGLZ_Header *source, char *dest)
{
+ pglz_decompress_with_history(source, dest, NULL, 0);
+ }
+
+ void
+ pglz_decompress_with_history(const PGLZ_Header *source, char *dest,
+ const char *history, int32 hlen)
+ {
const unsigned char *sp;
const unsigned char *srcend;
unsigned char *dp;
unsigned char *destend;
+ unsigned char *hend = NULL;
+
+ if (hlen > 0)
+ hend = (unsigned char *) history + hlen;
sp = ((const unsigned char *) source) + sizeof(PGLZ_Header);
srcend = ((const unsigned char *) source) + VARSIZE(source);
***************
*** 679,684 **** pglz_decompress(const PGLZ_Header *source, char *dest)
--- 757,763 ----
*/
int32 len;
int32 off;
+ int32 hoff;
len = (sp[0] & 0x0f) + 3;
off = ((sp[0] & 0xf0) << 4) | sp[1];
***************
*** 705,713 **** pglz_decompress(const PGLZ_Header *source, char *dest)
* memcpy() here, because the copied areas could overlap
* extremely!
*/
while (len--)
{
! *dp = dp[-off];
dp++;
}
}
--- 784,804 ----
* memcpy() here, because the copied areas could overlap
* extremely!
*/
+ hoff = off;
while (len--)
{
! if (NULL == hend)
! *dp = dp[-off];
! else
! {
! /*
! * hoff provides the offset in the history buffer from
! * the history end
! */
! Assert(hoff < hlen);
! *dp = hend[-hoff];
! hoff--;
! }
dp++;
}
}
*** a/src/include/access/heapam_xlog.h
--- b/src/include/access/heapam_xlog.h
***************
*** 142,153 **** typedef struct xl_heap_update
{
xl_heaptid target; /* deleted tuple id */
ItemPointerData newtid; /* new inserted tuple id */
! bool all_visible_cleared; /* PD_ALL_VISIBLE was cleared */
! bool new_all_visible_cleared; /* same for the page of newtid */
/* NEW TUPLE xl_heap_header AND TUPLE DATA FOLLOWS AT END OF STRUCT */
} xl_heap_update;
! #define SizeOfHeapUpdate (offsetof(xl_heap_update, new_all_visible_cleared) + sizeof(bool))
/*
* This is what we need to know about vacuum page cleanup/redirect
--- 142,157 ----
{
xl_heaptid target; /* deleted tuple id */
ItemPointerData newtid; /* new inserted tuple id */
! char flags;
!
/* NEW TUPLE xl_heap_header AND TUPLE DATA FOLLOWS AT END OF STRUCT */
} xl_heap_update;
! #define XL_HEAP_UPDATE_ALL_VISIBLE_CLEARED 0x01
! #define XL_HEAP_UPDATE_NEW_ALL_VISIBLE_CLEARED 0x02
! #define XL_HEAP_UPDATE_DELTA_ENCODED 0x04
!
! #define SizeOfHeapUpdate (offsetof(xl_heap_update, flags) + sizeof(char))
/*
* This is what we need to know about vacuum page cleanup/redirect
*** a/src/include/utils/pg_lzcompress.h
--- b/src/include/utils/pg_lzcompress.h
***************
*** 107,112 **** extern const PGLZ_Strategy *const PGLZ_strategy_always;
--- 107,118 ----
*/
extern bool pglz_compress(const char *source, int32 slen, PGLZ_Header *dest,
const PGLZ_Strategy *strategy);
+ extern bool pglz_compress_with_history(const char *source, int32 slen,
+ const char *history, int32 hlen,
+ PGLZ_Header *dest,
+ const PGLZ_Strategy *strategy);
extern void pglz_decompress(const PGLZ_Header *source, char *dest);
+ extern void pglz_decompress_with_history(const PGLZ_Header *source, char *dest,
+ const char *history, int32 hlen);
#endif /* _PG_LZCOMPRESS_H_ */