v1-0003-Add-RMGR-and-WAL-logging-API-for-pgstats.patch

From d260487585c9effec3b22994825bec9ea7a734dc Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@paquier.xyz>
Date: Fri, 27 Dec 2024 12:03:34 +0900
Subject: [PATCH v1 3/4] Add RMGR and WAL-logging API for pgstats

This commit adds a new facility in the backend that offers to pgstats
kinds the possibility to WAL-log data that need to be persisted across
restarts:
- A new callback called redo_cb can be defined by a pgstats_kind.
- pgstat_xlog.c includes one API able to register some payload data and
its size.  Stats kinds doing WAL-logging require a definition of
redo_cb.
- pg_waldump is able to show the data logged, as hexadedimal data.

This facility has applications for in-core and custom stats kinds, with
one primary case being the possibility to WAL-log some statistics
related to relations so as these are not lost post-crash.  This is left
as future work.
---
 src/include/access/rmgrlist.h            |  1 +
 src/include/utils/pgstat_internal.h      |  8 +++
 src/include/utils/pgstat_xlog.h          | 41 +++++++++++++
 src/backend/access/rmgrdesc/Makefile     |  1 +
 src/backend/access/rmgrdesc/meson.build  |  1 +
 src/backend/access/rmgrdesc/pgstatdesc.c | 48 +++++++++++++++
 src/backend/access/transam/rmgr.c        |  1 +
 src/backend/utils/activity/Makefile      |  1 +
 src/backend/utils/activity/meson.build   |  1 +
 src/backend/utils/activity/pgstat_xlog.c | 77 ++++++++++++++++++++++++
 src/bin/pg_waldump/.gitignore            |  1 +
 src/bin/pg_waldump/rmgrdesc.c            |  1 +
 src/bin/pg_waldump/t/001_basic.pl        |  3 +-
 src/tools/pgindent/typedefs.list         |  1 +
 14 files changed, 185 insertions(+), 1 deletion(-)
 create mode 100644 src/include/utils/pgstat_xlog.h
 create mode 100644 src/backend/access/rmgrdesc/pgstatdesc.c
 create mode 100644 src/backend/utils/activity/pgstat_xlog.c

diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index 78e6b908c6..abf9393bbe 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i
 PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL, NULL)
 PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask, NULL)
 PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL, logicalmsg_decode)
+PG_RMGR(RM_PGSTAT_ID, "PgStat", pgstat_redo, pgstat_desc, pgstat_identify, NULL, NULL, NULL, NULL)
diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h
index 811ed9b005..47d7f8988e 100644
--- a/src/include/utils/pgstat_internal.h
+++ b/src/include/utils/pgstat_internal.h
@@ -258,6 +258,14 @@ typedef struct PgStat_KindInfo
 	 */
 	void		(*init_backend_cb) (void);
 
+	/*
+	 * Perform custom actions when replaying WAL related to a statistics kind.
+	 *
+	 * "data" is a pointer to the stats information that can be used by the
+	 * stats kind at redo.
+	 */
+	void		(*redo_cb) (void *data, size_t data_size);
+
 	/*
 	 * For variable-numbered stats: flush pending stats. Required if pending
 	 * data is used.  See flush_fixed_cb for fixed-numbered stats.
diff --git a/src/include/utils/pgstat_xlog.h b/src/include/utils/pgstat_xlog.h
new file mode 100644
index 0000000000..25b7a78dd5
--- /dev/null
+++ b/src/include/utils/pgstat_xlog.h
@@ -0,0 +1,41 @@
+/* ----------
+ * pgstat_xlog.h
+ *	  Exports from utils/activity/pgstat_xlog.c
+ *
+ * Copyright (c) 2001-2024, PostgreSQL Global Development Group
+ *
+ * src/include/utils/pgstat_xlog.h
+ * ----------
+ */
+#ifndef PGSTAT_XLOG_H
+#define PGSTAT_XLOG_H
+
+#include "access/xlog.h"
+#include "access/xlogdefs.h"
+#include "access/xlogreader.h"
+#include "utils/pgstat_kind.h"
+
+/*
+ * Generic WAL record for pgstat data.
+ */
+typedef struct xl_pgstat_data
+{
+	PgStat_Kind stats_kind;
+	size_t		data_size;		/* size of the data */
+	/* data payload, worth data_size */
+	char		data[FLEXIBLE_ARRAY_MEMBER];
+} xl_pgstat_data;
+
+#define SizeOfPgStatData    (offsetof(xl_pgstat_data, data))
+
+extern XLogRecPtr pgstat_xlog_data(PgStat_Kind stats_kind,
+								   const void *data,
+								   size_t data_size);
+
+/* RMGR API */
+#define XLOG_PGSTAT_DATA	0x00
+extern void pgstat_redo(XLogReaderState *record);
+extern void pgstat_desc(StringInfo buf, XLogReaderState *record);
+extern const char *pgstat_identify(uint8 info);
+
+#endif							/* PGSTAT_XLOG_H */
diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile
index cd95eec37f..3979d22946 100644
--- a/src/backend/access/rmgrdesc/Makefile
+++ b/src/backend/access/rmgrdesc/Makefile
@@ -21,6 +21,7 @@ OBJS = \
 	logicalmsgdesc.o \
 	mxactdesc.o \
 	nbtdesc.o \
+	pgstatdesc.o \
 	relmapdesc.o \
 	replorigindesc.o \
 	rmgrdesc_utils.o \
diff --git a/src/backend/access/rmgrdesc/meson.build b/src/backend/access/rmgrdesc/meson.build
index e8b7a65fc7..951dacbd72 100644
--- a/src/backend/access/rmgrdesc/meson.build
+++ b/src/backend/access/rmgrdesc/meson.build
@@ -14,6 +14,7 @@ rmgr_desc_sources = files(
   'logicalmsgdesc.c',
   'mxactdesc.c',
   'nbtdesc.c',
+  'pgstatdesc.c',
   'relmapdesc.c',
   'replorigindesc.c',
   'rmgrdesc_utils.c',
diff --git a/src/backend/access/rmgrdesc/pgstatdesc.c b/src/backend/access/rmgrdesc/pgstatdesc.c
new file mode 100644
index 0000000000..0fc19d1e56
--- /dev/null
+++ b/src/backend/access/rmgrdesc/pgstatdesc.c
@@ -0,0 +1,48 @@
+/*-------------------------------------------------------------------------
+ *
+ * pgstatdesc.c
+ *	  rmgr descriptor routines for utils/activity/pgstat_xlog.c
+ *
+ * Portions Copyright (c) 2001-2024, PostgreSQL Global Development Group
+ *
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/rmgrdesc/pgstatdesc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "utils/pgstat_xlog.h"
+
+void
+pgstat_desc(StringInfo buf, XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_PGSTAT_DATA)
+	{
+		xl_pgstat_data *xlrec = (xl_pgstat_data *) rec;
+		char	   *sep = "";
+
+		appendStringInfo(buf, "stats kind \"%u\"; with data (%zu bytes): ",
+						 xlrec->stats_kind, xlrec->data_size);
+		/* Write data payload as a series of hex bytes */
+		for (int cnt = 0; cnt < xlrec->data_size; cnt++)
+		{
+			appendStringInfo(buf, "%s%02X", sep, (unsigned char) xlrec->data[cnt]);
+			sep = " ";
+		}
+	}
+}
+
+const char *
+pgstat_identify(uint8 info)
+{
+	if ((info & ~XLR_INFO_MASK) == XLOG_PGSTAT_DATA)
+		return "PGSTAT_DATA";
+
+	return NULL;
+}
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 1b7499726e..945843a78e 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -39,6 +39,7 @@
 #include "replication/message.h"
 #include "replication/origin.h"
 #include "storage/standby.h"
+#include "utils/pgstat_xlog.h"
 #include "utils/relmapper.h"
 /* IWYU pragma: end_keep */
 
diff --git a/src/backend/utils/activity/Makefile b/src/backend/utils/activity/Makefile
index 24b64a2742..186108c60a 100644
--- a/src/backend/utils/activity/Makefile
+++ b/src/backend/utils/activity/Makefile
@@ -33,6 +33,7 @@ OBJS = \
 	pgstat_subscription.o \
 	pgstat_wal.o \
 	pgstat_xact.o \
+	pgstat_xlog.o \
 	wait_event.o \
 	wait_event_funcs.o
 
diff --git a/src/backend/utils/activity/meson.build b/src/backend/utils/activity/meson.build
index 380d3dd70c..a79e2d3731 100644
--- a/src/backend/utils/activity/meson.build
+++ b/src/backend/utils/activity/meson.build
@@ -18,6 +18,7 @@ backend_sources += files(
   'pgstat_subscription.c',
   'pgstat_wal.c',
   'pgstat_xact.c',
+  'pgstat_xlog.c',
 )
 
 # this includes a .c file with contents generated in ../../../include/activity,
diff --git a/src/backend/utils/activity/pgstat_xlog.c b/src/backend/utils/activity/pgstat_xlog.c
new file mode 100644
index 0000000000..397a49a33a
--- /dev/null
+++ b/src/backend/utils/activity/pgstat_xlog.c
@@ -0,0 +1,77 @@
+/* ----------
+ * pgstat_xlog.c
+ *	   WAL replay logic for cumulative statistics.
+ *
+ *
+ * Copyright (c) 2001-2024, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/utils/activity/pgstat_xlog.c
+ * ----------
+ */
+
+#include "postgres.h"
+
+#include <unistd.h>
+
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "utils/pgstat_internal.h"
+#include "utils/pgstat_xlog.h"
+
+/*
+ * Write pgstats record data into WAL.
+ */
+XLogRecPtr
+pgstat_xlog_data(PgStat_Kind stats_kind, const void *data,
+				 size_t data_size)
+{
+	xl_pgstat_data xlrec;
+	XLogRecPtr	lsn;
+
+	xlrec.stats_kind = stats_kind;
+	xlrec.data_size = data_size;
+
+	XLogBeginInsert();
+	XLogRegisterData((char *) &xlrec, SizeOfPgStatData);
+	XLogRegisterData(data, data_size);
+
+	lsn = XLogInsert(RM_PGSTAT_ID, XLOG_PGSTAT_DATA);
+	return lsn;
+}
+
+/*
+ * Redo just passes down to the stats kind expected by this record
+ * data included in it.
+ */
+void
+pgstat_redo(XLogReaderState *record)
+{
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+	xl_pgstat_data *xlrec;
+	PgStat_Kind stats_kind;
+	size_t		data_size;
+	void	   *data;
+	const PgStat_KindInfo *kind_info;
+
+	if (info != XLOG_PGSTAT_DATA)
+		elog(PANIC, "pgstat_redo: unknown op code %u", info);
+
+	xlrec = (xl_pgstat_data *) XLogRecGetData(record);
+
+	stats_kind = xlrec->stats_kind;
+	data_size = xlrec->data_size;
+	data = xlrec->data;
+
+	kind_info = pgstat_get_kind_info(stats_kind);
+
+	if (kind_info == NULL)
+		elog(FATAL, "pgstat_redo: invalid stats data found: kind=%u",
+			 stats_kind);
+
+	if (kind_info->redo_cb == NULL)
+		elog(FATAL, "pgstat_redo: no redo callback found: kind=%s",
+			 kind_info->name);
+
+	kind_info->redo_cb(data, data_size);
+}
diff --git a/src/bin/pg_waldump/.gitignore b/src/bin/pg_waldump/.gitignore
index ec51f41c76..e5089b322d 100644
--- a/src/bin/pg_waldump/.gitignore
+++ b/src/bin/pg_waldump/.gitignore
@@ -13,6 +13,7 @@
 /logicalmsgdesc.c
 /mxactdesc.c
 /nbtdesc.c
+/pgstatdesc.c
 /relmapdesc.c
 /replorigindesc.c
 /rmgrdesc_utils.c
diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c
index 6b8c17bb4c..05b4f2acd1 100644
--- a/src/bin/pg_waldump/rmgrdesc.c
+++ b/src/bin/pg_waldump/rmgrdesc.c
@@ -30,6 +30,7 @@
 #include "replication/origin.h"
 #include "rmgrdesc.h"
 #include "storage/standbydefs.h"
+#include "utils/pgstat_xlog.h"
 #include "utils/relmapper.h"
 
 #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask,decode) \
diff --git a/src/bin/pg_waldump/t/001_basic.pl b/src/bin/pg_waldump/t/001_basic.pl
index 578e473139..367ed6ab8c 100644
--- a/src/bin/pg_waldump/t/001_basic.pl
+++ b/src/bin/pg_waldump/t/001_basic.pl
@@ -73,7 +73,8 @@ BRIN
 CommitTs
 ReplicationOrigin
 Generic
-LogicalMessage$/,
+LogicalMessage
+PgStat$/,
 	'rmgr list');
 
 
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index e1c4f913f8..a5b1a2c998 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -4134,6 +4134,7 @@ xl_multixact_create
 xl_multixact_truncate
 xl_overwrite_contrecord
 xl_parameter_change
+xl_pgstat_data
 xl_relmap_update
 xl_replorigin_drop
 xl_replorigin_set
-- 
2.45.2