*** a/contrib/Makefile --- b/contrib/Makefile *************** *** 31,36 **** SUBDIRS = \ --- 31,37 ---- passwordcheck \ pg_archivecleanup \ pg_buffercache \ + pg_computemaxlsn \ pg_freespacemap \ pg_standby \ pg_stat_statements \ *** /dev/null --- b/contrib/pg_computemaxlsn/Makefile *************** *** 0 **** --- 1,22 ---- + # contrib/pg_computemaxlsn/Makefile + + PGFILEDESC = "pg_computemaxlsn - an utility to find max LSN from data pages" + PGAPPICON = win32 + + PROGRAM = pg_computemaxlsn + OBJS = pg_computemaxlsn.o $(WIN32RES) + + PG_CPPFLAGS = -I$(srcdir) + PG_LIBS = $(libpq_pgport) + + + ifdef USE_PGXS + PG_CONFIG = pg_config + PGXS := $(shell $(PG_CONFIG) --pgxs) + include $(PGXS) + else + subdir = contrib/pg_computemaxlsn + top_builddir = ../.. + include $(top_builddir)/src/Makefile.global + include $(top_srcdir)/contrib/contrib-global.mk + endif *** /dev/null --- b/contrib/pg_computemaxlsn/pg_computemaxlsn.c *************** *** 0 **** --- 1,430 ---- + /*------------------------------------------------------------------------- + * + * pg_computemaxlsn.c + * A utility to compute the maximum LSN in data pages + * + * Portions Copyright (c) 1996-2012, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * contrib/pg_computemaxlsn/pg_computemaxlsn.c + * + *------------------------------------------------------------------------- + */ + + /* + * We have to use postgres.h not postgres_fe.h here, because there's so much + * backend-only stuff for reading data files we need. But we need a + * frontend-ish environment otherwise. Hence this ugly hack. + */ + #define FRONTEND 1 + + #include "postgres.h" + + #include + #include + #include + #include + #include + #include + #include + + #include "getopt_long.h" + + #include "access/xlog_internal.h" + #include "catalog/catalog.h" + #include "storage/bufpage.h" + #include "storage/fd.h" + + /* Page header size */ + #define PAGEHDRSZ (sizeof(PageHeaderData)) + + extern int optind; + extern char *optarg; + static const char *progname; + static int quiet = false; + + static bool FindMaxLSNinFile(char *filename, XLogRecPtr *maxlsn); + static bool FindMaxLSNinDir(char *path, XLogRecPtr *maxlsn); + static bool FindMaxLSNinPgData(char *pgdatapath, XLogRecPtr *maxlsn); + static void usage(void); + + /* + * This function validates the given cluster directory - we search for a + * small set of subdirectories that we expect to find in a valid data directory. + * directory. If any of the subdirectories are missing, then it treats as a + * normal directory. + */ + static bool + check_data_dir(const char *pg_data) + { + char subDirName[MAXPGPATH]; + int dnum; + + /* start check with top-most directory */ + const char *requiredSubdirs[] = {"", "base", "global", "pg_tblspc", + "pg_multixact", "pg_subtrans", "pg_clog", "pg_twophase", + "pg_xlog"}; + + for (dnum = 0; dnum < lengthof(requiredSubdirs); ++dnum) + { + struct stat statBuf; + + snprintf(subDirName, sizeof(subDirName), "%s%s%s", pg_data, + /* Win32 can't stat() a directory with a trailing slash. */ + *requiredSubdirs[dnum] ? "/" : "", + requiredSubdirs[dnum]); + + if (stat(subDirName, &statBuf) != 0 + || (!S_ISDIR(statBuf.st_mode))) + return false; + } + + return true; + } + + /* + * relfilenode name validation. + * Format with_ext == true [0-9]+[ \w | _vm | _fsm | _init ][\.][0-9]* + * with_ext == false [0-9]+ + */ + static bool + validateRelfilenodename(char *name) + { + int pos = 0; + + while ((name[pos] >= '0') && (name[pos] <= '9')) + pos++; + + if (name[pos] == '_') + { + pos++; + while ((name[pos] >= 'a') && (name[pos] <= 'z')) + pos++; + } + if (name[pos] == '.') + { + pos++; + while ((name[pos] >= '0') && (name[pos] <= '9')) + pos++; + } + + if (name[pos] == 0) + return true; + + return false; + } + + int + main(int argc, char *argv[]) + { + static struct option long_options[] = { + {"quiet", no_argument, NULL, 'q'}, + {NULL, 0, NULL, 0} + }; + int optindex; + int c; + char *LsnSearchPath = NULL; + XLogRecPtr maxLSN = 0; + XLogSegNo logSegNo = 0; + bool result = true; + struct stat statbuf; + + set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_computemaxlsn")); + + progname = get_progname(argv[0]); + + if (argc > 1) + { + if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0) + { + usage(); + exit(0); + } + if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0) + { + puts("pg_computemaxlsn (PostgreSQL) " PG_VERSION); + exit(0); + } + } + + while ((c = getopt_long(argc, argv, "q", long_options, &optindex)) != -1) + { + switch (c) + { + case 'q': + quiet = true; + break; + default: + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); + } + } + + /* Loop over all input arguements */ + do + { + if (optind < argc) + LsnSearchPath = argv[optind]; + else + LsnSearchPath = getenv("PGDATA"); + + if (LsnSearchPath == NULL) + { + fprintf(stderr, _("%s: no file/directory specified\n"), progname); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); + } + + maxLSN = 0; + + /* Check for the provided search path is a data directory? */ + if (check_data_dir(LsnSearchPath)) + { + result = FindMaxLSNinPgData(LsnSearchPath, &maxLSN); + } + else + { + if (lstat(LsnSearchPath, &statbuf) < 0) + { + if (errno == ENOENT) + { + fprintf(stderr, _("%s: file or directory \"%s\" does not exists.\n"), + progname, LsnSearchPath); + } + else + { + fprintf(stderr, _("%s: could not stat file or directory \"%s\": %s\n"), + progname, LsnSearchPath, strerror(errno)); + } + exit(1); + } + + if (S_ISREG(statbuf.st_mode)) + result = FindMaxLSNinFile(LsnSearchPath, &maxLSN); + else + result = FindMaxLSNinDir(LsnSearchPath, &maxLSN); + } + + if (!result) + { + /* Message already provided, simply exit */ + exit(1); + } + + XLByteToSeg(maxLSN, logSegNo); + + printf("Maximum LSN found in \"%s\" is: %X/%X \n" + "WAL segment file name (fileid,seg): %X/%X\n", + LsnSearchPath, (uint32) (maxLSN >> 32), (uint32) maxLSN, + (uint32) (logSegNo >> 32), (uint32) (logSegNo)); + } while (++optind < argc); + + return 0; + } + + + /* + * Read the maximum LSN number in the one of data file (relnode file). + */ + static bool + FindMaxLSNinFile(char *filename, XLogRecPtr *maxlsn) + { + XLogRecPtr pagelsn; + off_t len, + seekpos; + uint32 nblocks, + blocknum; + char buffer[PAGEHDRSZ]; + int nbytes; + int fd; + XLogRecPtr filemaxlsn = 0; + uint32 filemaxlsn_block = 0; + + if ((fd = open(filename, O_RDONLY | PG_BINARY, 0)) < 0) + { + /* + * If file does not exist or we can't read it. give error + */ + fprintf(stderr, _("%s: could not open file \"%s\" for reading: %s\n"), + progname, filename, strerror(errno)); + return false; + } + + /* Calculate the number of pages in file */ + len = lseek(fd, 0L, SEEK_END); + if (len < 0) + { + close(fd); + fprintf(stderr, _("%s: file \"%s\" for seeking: %s\n"), + progname, filename, strerror(errno)); + return false; + } + + nblocks = (len / BLCKSZ); + if (nblocks > RELSEG_SIZE) + { + /* + * In one relfilenode file length can't be more that RELSEG_SIZE + */ + close(fd); + fprintf(stderr, _("%s: file \"%s\" length is more than segment size: %d.\n"), + progname, filename, RELSEG_SIZE); + return false; + } + + /* + * Read the only page header and find the LSN of the page and continue to + * next page. + */ + seekpos = 0; + for (blocknum = 0; blocknum < nblocks; blocknum++) + { + len = lseek(fd, seekpos, SEEK_SET); + if (len != seekpos) + { + close(fd); + fprintf(stderr, _("%s: could not seek to offset %u in file \"%s\": %s\n"), + progname, blocknum, filename, strerror(errno)); + return false; + } + + nbytes = read(fd, buffer, PAGEHDRSZ); + if (nbytes < 0) + { + close(fd); + fprintf(stderr, _("%s: could not read file \"%s\" at offset %u: %s\n"), + progname, filename, blocknum, strerror(errno)); + return false; + } + + /* + * Remember this lsn as the highest (if it is) + */ + pagelsn = PageGetLSN(buffer); + if (filemaxlsn < pagelsn) + { + filemaxlsn = pagelsn; + filemaxlsn_block = blocknum; + } + + seekpos += (off_t) BLCKSZ; + } + + close(fd); + + if (*maxlsn < filemaxlsn) + *maxlsn = filemaxlsn; + + if (!quiet) + printf("Highest LSN for file:\"%s\" is: %X/%X in block %u\n", + filename, (uint32) (filemaxlsn >> 32), (uint32) filemaxlsn, + filemaxlsn_block); + + return true; + } + + /* + * Read the maximum LSN number in current directory; including sub directories + * and links. + */ + static bool + FindMaxLSNinDir(char *path, XLogRecPtr *maxlsn) + { + DIR *dir; + struct dirent *de; + char pathbuf[MAXPGPATH]; + struct stat statbuf; + bool result = true; + + dir = opendir(path); + if (NULL == dir) + { + fprintf(stderr, _("%s: could not open directory \"%s\": %s\n"), + progname, path, strerror(errno)); + return false; + } + + while ((de = readdir(dir)) != NULL) + { + /* Skip special stuff */ + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + + /* Skip temporary files */ + if (strncmp(de->d_name, + PG_TEMP_FILE_PREFIX, + strlen(PG_TEMP_FILE_PREFIX)) == 0) + continue; + + /* + * Skip all invalid files, and read remaining relfilenode files + */ + if (!validateRelfilenodename(de->d_name)) + continue; + + snprintf(pathbuf, MAXPGPATH, "%s/%s", path, de->d_name); + + if (lstat(pathbuf, &statbuf) != 0) + { + if (errno != ENOENT) + fprintf(stderr, _("%s: could not stat file or directory \"%s\": %s\n"), + progname, pathbuf, strerror(errno)); + + /* If the file went away while scanning, it's no error. */ + continue; + } + + if (S_ISREG(statbuf.st_mode)) + { + if (!FindMaxLSNinFile(pathbuf, maxlsn)) + result = false; + } + else + { + if (!FindMaxLSNinDir(pathbuf, maxlsn)) + result = false; + } + } + + closedir(dir); + return result; + } + + + /* + * Read the maximum LSN number in the DATA directory. + */ + static bool + FindMaxLSNinPgData(char *pgdatapath, XLogRecPtr *maxlsn) + { + char pathbuf[MAXPGPATH]; + bool result = true; + + /* scan all the relfilenodes in data directory */ + snprintf(pathbuf, MAXPGPATH, "%s/global", pgdatapath); + if (!FindMaxLSNinDir(pathbuf, maxlsn)) + result = false; + + snprintf(pathbuf, MAXPGPATH, "%s/base", pgdatapath); + if (!FindMaxLSNinDir(pathbuf, maxlsn)) + result = false; + + snprintf(pathbuf, MAXPGPATH, "%s/pg_tblspc", pgdatapath); + if (!FindMaxLSNinDir(pathbuf, maxlsn)) + result = false; + + return result; + } + + static void + usage(void) + { + printf(_("%s compute the maximum LSN in PostgreSQL data pages.\n\n"), progname); + printf(_("Usage:\n %s [OPTION] [FILE/DIRECTORY...]\n\n"), progname); + printf(_("Options:\n")); + printf(_(" -q, --quiet stops the output of highest LSN in a file\n")); + printf(_(" -V, --version output version information, then exit\n")); + printf(_(" -?, --help show this help, then exit\n")); + printf(_("\nReport bugs to .\n")); + } *** a/doc/src/sgml/contrib.sgml --- b/doc/src/sgml/contrib.sgml *************** *** 208,212 **** pages. --- 208,213 ---- &pgtesttiming; &pgupgrade; &pgxlogdump; + &pgcomputemaxlsn; *** a/doc/src/sgml/filelist.sgml --- b/doc/src/sgml/filelist.sgml *************** *** 124,129 **** --- 124,130 ---- + *** /dev/null --- b/doc/src/sgml/pg_computemaxlsn.sgml *************** *** 0 **** --- 1,100 ---- + + + + + pg_computemaxlsn + 1 + Application + + + + pg_computemaxlsn + computes the maximum LSN in database of a PostgreSQL database cluster + + + + pg_computemaxlsn + + + + + pg_computemaxlsn + option + file or directory + + + + + Description + + pg_computemaxlsn computes maximun LSN from database pages + in the specified list of files or directories. + + + + If user doesn't provide the file or directory to find the max lsn then + pg_computemaxlsn use the environment variable PGDATA + if exists otherwise reports an error. + + + + The + + + The + + + + use cases + + This utility can be used to avoid more data corruption by finding out the + maximum lsn in the data directory. which is required by the pg_resetxlog + to provide the new WAL segment file name. + + + + This utility can also be used to decide whether backup is required or not when the data page + in old-master precedes the last applied LSN in old-standby (i.e., new-master) at the + moment of the failover. + + + + + Examples + + pg_computemaxlsn> ./pg_computemaxlsn -q ../../../data/ + Maximum LSN found in "../../../data/" is: 0/181B090 + WAL segment file name (fileid,seg): 0/1 + + + + pg_computemaxlsn> ./pg_computemaxlsn -q ../../../data/base/126* + Maximum LSN found in "../../../data/base/12625" is: 0/17D5360 + WAL segment file name (fileid,seg): 0/1 + Maximum LSN found in "../../../data/base/12630" is: 0/181B090 + WAL segment file name (fileid,seg): 0/1 + + + + + Notes + + This utility just parse the header of the page and provides the LSN, please be sure + when using this tool on directory which is not a data directory. + + + + If any error occurs during the finding of max LSN in the provided file or directory, + the max lsn is not printed. Once the reported problem solves need to run the tool again + for proper result. + + + *** a/doc/src/sgml/ref/pg_resetxlog.sgml --- b/doc/src/sgml/ref/pg_resetxlog.sgml *************** *** 139,144 **** PostgreSQL documentation --- 139,153 ---- largest entry in pg_xlog, use -l 00000001000000320000004B or higher. + + If pg_resetxlog complains that it cannot determine + valid data for pg_control, and if you do not have or corrupted + WAL segment files in the directory pg_xlog under the data directory, + then to identify larger WAL segment file from data files we can use utility pg_computemaxlsn + by specifing file or folder or data directory. Once larger WAL segment file is found use + pg_resetxlog itself looks at the files in *** a/src/tools/msvc/Mkvcbuild.pm --- b/src/tools/msvc/Mkvcbuild.pm *************** *** 43,49 **** my @contrib_uselibpgcommon = ( 'pg_standby', 'pg_archivecleanup', 'pg_test_fsync', 'pg_test_timing', 'pg_upgrade', 'pg_xlogdump', ! 'vacuumlo'); my $contrib_extralibs = { 'pgbench' => ['wsock32.lib'] }; my $contrib_extraincludes = { 'tsearch2' => ['contrib/tsearch2'], 'dblink' => ['src/backend'] }; --- 43,49 ---- 'pg_standby', 'pg_archivecleanup', 'pg_test_fsync', 'pg_test_timing', 'pg_upgrade', 'pg_xlogdump', ! 'vacuumlo', 'pg_computemaxlsn'); my $contrib_extralibs = { 'pgbench' => ['wsock32.lib'] }; my $contrib_extraincludes = { 'tsearch2' => ['contrib/tsearch2'], 'dblink' => ['src/backend'] };