pg_dump-directory.diff

diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml
index 8242b53..2278147 100644
*** a/doc/src/sgml/ref/pg_dump.sgml
--- b/doc/src/sgml/ref/pg_dump.sgml
*************** PostgreSQL documentation
*** 194,201 ****
        <term><option>--file=<replaceable class="parameter">file</replaceable></option></term>
        <listitem>
         <para>
!         Send output to the specified file.  If this is omitted, the
!         standard output is used.
         </para>
        </listitem>
       </varlistentry>
--- 194,203 ----
        <term><option>--file=<replaceable class="parameter">file</replaceable></option></term>
        <listitem>
         <para>
! 		Send output to the specified file.  If this is omitted for the file
!         based output formats, then the standard output is used. This parameter must
! 		be given for the directory output format where it specifies the target
!         directory instead of a file.
         </para>
        </listitem>
       </varlistentry>
*************** PostgreSQL documentation
*** 226,234 ****
            <para>
             Output a custom-format archive suitable for input into
             <application>pg_restore</application>.
!            This is the most flexible output format in that it allows manual
!            selection and reordering of archived items during restore.
!            This format is also compressed by default.
            </para>
           </listitem>
          </varlistentry>
--- 228,251 ----
            <para>
             Output a custom-format archive suitable for input into
             <application>pg_restore</application>.
!            Together with the directory output format, this is the most flexible
!            output format in that it allows manual selection and reordering of
!            archived items during restore. This format is also compressed by
!            default.
!           </para>
!          </listitem>
!         </varlistentry>
! 
!         <varlistentry>
!          <term><literal>d</></term>
!          <term><literal>directory</></term>
!          <listitem>
!           <para>
!            Output a directory-format archive suitable for input into
!            <application>pg_restore</application>. This will create a directory
!            instead of a file and this directory will contain one file for each
! 		   table and BLOB of the database that is being dumped. This format is
!            compressed by default.
            </para>
           </listitem>
          </varlistentry>
*************** PostgreSQL documentation
*** 267,272 ****
--- 284,300 ----
       </varlistentry>
  
       <varlistentry>
+       <term><option>-k</></term>
+       <listitem>
+        <para>
+ 		Check a directory archive. This will walk through all files of a
+ 		directory-format archive and check that all required files exist and
+         that they are from the same dump.
+        </para>
+       </listitem>
+      </varlistentry>
+ 
+      <varlistentry>
        <term><option>-n <replaceable class="parameter">schema</replaceable></option></term>
        <term><option>--schema=<replaceable class="parameter">schema</replaceable></option></term>
        <listitem>
*************** CREATE DATABASE foo WITH TEMPLATE templa
*** 937,942 ****
--- 965,978 ----
    </para>
  
    <para>
+    To dump a database into a directory-format archive:
+ 
+ <screen>
+ <prompt>$</prompt> <userinput>pg_dump -Fd mydb -f dumpdir</userinput>
+ </screen>
+   </para>
+ 
+   <para>
     To reload an archive file into a (freshly created) database named
     <literal>newdb</>:
  
diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index efb031a..9a1c025 100644
*** a/src/bin/pg_dump/Makefile
--- b/src/bin/pg_dump/Makefile
*************** override CPPFLAGS := -I$(libpq_srcdir) $
*** 20,26 ****
  
  OBJS=	pg_backup_archiver.o pg_backup_db.o pg_backup_custom.o \
  	pg_backup_files.o pg_backup_null.o pg_backup_tar.o \
! 	dumputils.o compress_io.o $(WIN32RES)
  
  KEYWRDOBJS = keywords.o kwlookup.o
  
--- 20,26 ----
  
  OBJS=	pg_backup_archiver.o pg_backup_db.o pg_backup_custom.o \
  	pg_backup_files.o pg_backup_null.o pg_backup_tar.o \
! 	dumputils.o pg_backup_directory.o compress_io.o $(WIN32RES)
  
  KEYWRDOBJS = keywords.o kwlookup.o
  
diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h
index 8fa9a57..08fb910 100644
*** a/src/bin/pg_dump/pg_backup.h
--- b/src/bin/pg_dump/pg_backup.h
*************** typedef enum _archiveFormat
*** 48,56 ****
  {
  	archUnknown = 0,
  	archCustom = 1,
! 	archFiles = 2,
! 	archTar = 3,
! 	archNull = 4
  } ArchiveFormat;
  
  typedef enum _archiveMode
--- 48,58 ----
  {
  	archUnknown = 0,
  	archCustom = 1,
! 	archDirectory = 2,
! 	archFiles = 3,
! 	archTar = 4,
! 	archNull = 5,
! 	archNullAppend = 6
  } ArchiveFormat;
  
  typedef enum _archiveMode
*************** typedef struct _restoreOptions
*** 112,117 ****
--- 114,120 ----
  	int			schemaOnly;
  	int			verbose;
  	int			aclsSkip;
+ 	int			checkArchive;
  	int			tocSummary;
  	char	   *tocFile;
  	int			format;
*************** extern Archive *CreateArchive(const char
*** 195,200 ****
--- 198,206 ----
  /* The --list option */
  extern void PrintTOCSummary(Archive *AH, RestoreOptions *ropt);
  
+ /* Check an existing archive */
+ extern bool CheckArchive(Archive *AH, RestoreOptions *ropt);
+ 
  extern RestoreOptions *NewRestoreOptions(void);
  
  /* Rearrange and filter TOC entries */
diff --git a/src/bin/pg_dump/pg_backup_archiver.c b/src/bin/pg_dump/pg_backup_archiver.c
index a28956b..75a4b6d 100644
*** a/src/bin/pg_dump/pg_backup_archiver.c
--- b/src/bin/pg_dump/pg_backup_archiver.c
***************
*** 26,31 ****
--- 26,32 ----
  
  #include <ctype.h>
  #include <unistd.h>
+ #include <sys/stat.h>
  #include <sys/types.h>
  #include <sys/wait.h>
  
*************** static int	_discoverArchiveFormat(Archiv
*** 109,114 ****
--- 110,116 ----
  static void dump_lo_buf(ArchiveHandle *AH);
  static void _write_msg(const char *modulename, const char *fmt, va_list ap);
  static void _die_horribly(ArchiveHandle *AH, const char *modulename, const char *fmt, va_list ap);
+ static void outputSummaryHeaderText(Archive *AHX);
  
  static void dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim);
  static OutputContext SetOutput(ArchiveHandle *AH, char *filename, int compression);
*************** PrintTOCSummary(Archive *AHX, RestoreOpt
*** 779,818 ****
  	ArchiveHandle *AH = (ArchiveHandle *) AHX;
  	TocEntry   *te;
  	OutputContext sav;
- 	char	   *fmtName;
  
  	if (ropt->filename)
  		sav = SetOutput(AH, ropt->filename, 0 /* no compression */ );
  
! 	ahprintf(AH, ";\n; Archive created at %s", ctime(&AH->createDate));
! 	ahprintf(AH, ";     dbname: %s\n;     TOC Entries: %d\n;     Compression: %d\n",
! 			 AH->archdbname, AH->tocCount, AH->compression);
! 
! 	switch (AH->format)
! 	{
! 		case archFiles:
! 			fmtName = "FILES";
! 			break;
! 		case archCustom:
! 			fmtName = "CUSTOM";
! 			break;
! 		case archTar:
! 			fmtName = "TAR";
! 			break;
! 		default:
! 			fmtName = "UNKNOWN";
! 	}
! 
! 	ahprintf(AH, ";     Dump Version: %d.%d-%d\n", AH->vmaj, AH->vmin, AH->vrev);
! 	ahprintf(AH, ";     Format: %s\n", fmtName);
! 	ahprintf(AH, ";     Integer: %d bytes\n", (int) AH->intSize);
! 	ahprintf(AH, ";     Offset: %d bytes\n", (int) AH->offSize);
! 	if (AH->archiveRemoteVersion)
! 		ahprintf(AH, ";     Dumped from database version: %s\n",
! 				 AH->archiveRemoteVersion);
! 	if (AH->archiveDumpVersion)
! 		ahprintf(AH, ";     Dumped by pg_dump version: %s\n",
! 				 AH->archiveDumpVersion);
  
  	ahprintf(AH, ";\n;\n; Selected TOC Entries:\n;\n");
  
--- 781,791 ----
  	ArchiveHandle *AH = (ArchiveHandle *) AHX;
  	TocEntry   *te;
  	OutputContext sav;
  
  	if (ropt->filename)
  		sav = SetOutput(AH, ropt->filename, 0 /* no compression */ );
  
! 	outputSummaryHeaderText(AHX);
  
  	ahprintf(AH, ";\n;\n; Selected TOC Entries:\n;\n");
  
*************** PrintTOCSummary(Archive *AHX, RestoreOpt
*** 841,846 ****
--- 814,856 ----
  		ResetOutput(AH, sav);
  }
  
+ bool
+ CheckArchive(Archive *AHX, RestoreOptions *ropt)
+ {
+ 	ArchiveHandle  *AH = (ArchiveHandle *) AHX;
+ 	TocEntry	   *te;
+ 	teReqs			reqs;
+ 	bool			checkOK;
+ 
+ 	outputSummaryHeaderText(AHX);
+ 
+ 	checkOK = (*AH->StartCheckArchivePtr)(AH);
+ 
+ 	/* this gets only called from the commandline so we write to stdout as
+ 	 * usual */
+ 	printf(";\n; Performing Checks...\n;\n");
+ 
+ 	for (te = AH->toc->next; te != AH->toc; te = te->next)
+ 	{
+ 		if (!(reqs = _tocEntryRequired(te, ropt, true)))
+ 			continue;
+ 
+ 		if (!(*AH->CheckTocEntryPtr)(AH, te, reqs))
+ 			checkOK = false;
+ 
+ 		/* do not dump the contents but only the errors */
+ 	}
+ 
+ 	if (!(*AH->EndCheckArchivePtr)(AH))
+ 		checkOK = false;
+ 
+ 	printf("; Check result: %s\n", checkOK ? "OK" : "FAILED");
+ 
+ 	return checkOK;
+ }
+ 
+ 
+ 
  /***********
   * BLOB Archival
   ***********/
*************** archprintf(Archive *AH, const char *fmt,
*** 1116,1121 ****
--- 1126,1174 ----
   * Stuff below here should be 'private' to the archiver routines
   *******************************/
  
+ static void
+ outputSummaryHeaderText(Archive *AHX)
+ {
+ 	ArchiveHandle  *AH = (ArchiveHandle *) AHX;
+ 	const char	   *fmtName;
+ 
+ 	ahprintf(AH, ";\n; Archive created at %s", ctime(&AH->createDate));
+ 	ahprintf(AH, ";     dbname: %s\n;     TOC Entries: %d\n;     Compression: %d\n",
+ 			 AH->archdbname, AH->tocCount, AH->compression);
+ 
+ 	switch (AH->format)
+ 	{
+ 		case archCustom:
+ 			fmtName = "CUSTOM";
+ 			break;
+ 		case archDirectory:
+ 			fmtName = "DIRECTORY";
+ 			break;
+ 		case archFiles:
+ 			fmtName = "FILES";
+ 			break;
+ 		case archTar:
+ 			fmtName = "TAR";
+ 			break;
+ 		default:
+ 			fmtName = "UNKNOWN";
+ 	}
+ 
+ 	ahprintf(AH, ";     Dump Version: %d.%d-%d\n", AH->vmaj, AH->vmin, AH->vrev);
+ 	ahprintf(AH, ";     Format: %s\n", fmtName);
+ 	ahprintf(AH, ";     Integer: %d bytes\n", (int) AH->intSize);
+ 	ahprintf(AH, ";     Offset: %d bytes\n", (int) AH->offSize);
+ 	if (AH->archiveRemoteVersion)
+ 		ahprintf(AH, ";     Dumped from database version: %s\n",
+ 				 AH->archiveRemoteVersion);
+ 	if (AH->archiveDumpVersion)
+ 		ahprintf(AH, ";     Dumped by pg_dump version: %s\n",
+ 				 AH->archiveDumpVersion);
+ 
+ 	if (AH->PrintExtraTocSummaryPtr != NULL)
+ 		(*AH->PrintExtraTocSummaryPtr) (AH);
+ }
+ 
  static OutputContext
  SetOutput(ArchiveHandle *AH, char *filename, int compression)
  {
*************** _discoverArchiveFormat(ArchiveHandle *AH
*** 1721,1726 ****
--- 1774,1781 ----
  	char		sig[6];			/* More than enough */
  	size_t		cnt;
  	int			wantClose = 0;
+ 	char		buf[MAXPGPATH];
+ 	struct stat	st;
  
  #if 0
  	write_msg(modulename, "attempting to ascertain archive format\n");
*************** _discoverArchiveFormat(ArchiveHandle *AH
*** 1737,1743 ****
  	if (AH->fSpec)
  	{
  		wantClose = 1;
! 		fh = fopen(AH->fSpec, PG_BINARY_R);
  		if (!fh)
  			die_horribly(AH, modulename, "could not open input file \"%s\": %s\n",
  						 AH->fSpec, strerror(errno));
--- 1792,1813 ----
  	if (AH->fSpec)
  	{
  		wantClose = 1;
! 		/*
! 		 * Check if the specified archive is a directory actually. If so, we open
! 		 * the TOC file instead.
! 		 */
! 		buf[0] = '\0';
! 		if (stat(AH->fSpec, &st) == 0 && S_ISDIR(st.st_mode))
! 		{
! 			if (snprintf(buf, MAXPGPATH, "%s/%s", AH->fSpec, "TOC") >= MAXPGPATH)
! 				die_horribly(AH, modulename, "directory name too long: \"%s\"\n",
! 							 AH->fSpec);
! 		}
! 
! 		if (strlen(buf) == 0)
! 			strcpy(buf, AH->fSpec);
! 
! 		fh = fopen(buf, PG_BINARY_R);
  		if (!fh)
  			die_horribly(AH, modulename, "could not open input file \"%s\": %s\n",
  						 AH->fSpec, strerror(errno));
*************** _allocAH(const char *FileSpec, const Arc
*** 1950,1955 ****
--- 2020,2029 ----
  			InitArchiveFmt_Custom(AH);
  			break;
  
+ 		case archDirectory:
+ 			InitArchiveFmt_Directory(AH);
+ 			break;
+ 
  		case archFiles:
  			InitArchiveFmt_Files(AH);
  			break;
*************** WriteHead(ArchiveHandle *AH)
*** 2975,2985 ****
  	(*AH->WriteBytePtr) (AH, AH->format);
  
  #ifndef HAVE_LIBZ
! 	if (AH->compression != 0)
  		write_msg(modulename, "WARNING: requested compression not available in this "
  				  "installation -- archive will be uncompressed\n");
  
! 	AH->compression = 0;
  #endif
  
  	WriteInt(AH, AH->compression);
--- 3049,3061 ----
  	(*AH->WriteBytePtr) (AH, AH->format);
  
  #ifndef HAVE_LIBZ
! 	if (AH->compression > 0 && AH->compression <= 9)
! 	{
  		write_msg(modulename, "WARNING: requested compression not available in this "
  				  "installation -- archive will be uncompressed\n");
  
! 		AH->compression = 0;
! 	}
  #endif
  
  	WriteInt(AH, AH->compression);
*************** ReadHead(ArchiveHandle *AH)
*** 3063,3069 ****
  		AH->compression = Z_DEFAULT_COMPRESSION;
  
  #ifndef HAVE_LIBZ
! 	if (AH->compression != 0)
  		write_msg(modulename, "WARNING: archive is compressed, but this installation does not support compression -- no data will be available\n");
  #endif
  
--- 3139,3145 ----
  		AH->compression = Z_DEFAULT_COMPRESSION;
  
  #ifndef HAVE_LIBZ
! 	if (AH->compression > 0 && AH->compression <= 9)
  		write_msg(modulename, "WARNING: archive is compressed, but this installation does not support compression -- no data will be available\n");
  #endif
  
*************** checkSeek(FILE *fp)
*** 3128,3155 ****
  	return true;
  }
  
! 
! /*
!  * dumpTimestamp
!  */
! static void
! dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim)
  {
- 	char		buf[256];
- 
  	/*
  	 * We don't print the timezone on Win32, because the names are long and
  	 * localized, which means they may contain characters in various random
  	 * encodings; this has been seen to cause encoding errors when reading the
  	 * dump script.
  	 */
! 	if (strftime(buf, sizeof(buf),
  #ifndef WIN32
  				 "%Y-%m-%d %H:%M:%S %Z",
  #else
  				 "%Y-%m-%d %H:%M:%S",
  #endif
! 				 localtime(&tim)) != 0)
  		ahprintf(AH, "-- %s %s\n\n", msg, buf);
  }
  
--- 3204,3235 ----
  	return true;
  }
  
! bool
! getTimestampString(char *buf, size_t buflen, time_t tim)
  {
  	/*
  	 * We don't print the timezone on Win32, because the names are long and
  	 * localized, which means they may contain characters in various random
  	 * encodings; this has been seen to cause encoding errors when reading the
  	 * dump script.
  	 */
! 	return strftime(buf, buflen,
  #ifndef WIN32
  				 "%Y-%m-%d %H:%M:%S %Z",
  #else
  				 "%Y-%m-%d %H:%M:%S",
  #endif
! 				 localtime(&tim)) != 0;
! }
! 
! /*
!  * dumpTimestamp
!  */
! static void
! dumpTimestamp(ArchiveHandle *AH, const char *msg, time_t tim)
! {
! 	char	buf[256];
! 	if (getTimestampString(buf, sizeof(buf), tim))
  		ahprintf(AH, "-- %s %s\n\n", msg, buf);
  }
  
diff --git a/src/bin/pg_dump/pg_backup_archiver.h b/src/bin/pg_dump/pg_backup_archiver.h
index 705b2e4..9795eac 100644
*** a/src/bin/pg_dump/pg_backup_archiver.h
--- b/src/bin/pg_dump/pg_backup_archiver.h
*************** typedef z_stream *z_streamp;
*** 112,117 ****
--- 112,118 ----
  struct _archiveHandle;
  struct _tocEntry;
  struct _restoreList;
+ enum _teReqs;
  
  typedef void (*ClosePtr) (struct _archiveHandle * AH);
  typedef void (*ReopenPtr) (struct _archiveHandle * AH);
*************** typedef void (*WriteExtraTocPtr) (struct
*** 135,144 ****
--- 136,151 ----
  typedef void (*ReadExtraTocPtr) (struct _archiveHandle * AH, struct _tocEntry * te);
  typedef void (*PrintExtraTocPtr) (struct _archiveHandle * AH, struct _tocEntry * te);
  typedef void (*PrintTocDataPtr) (struct _archiveHandle * AH, struct _tocEntry * te, RestoreOptions *ropt);
+ typedef void (*PrintExtraTocSummaryPtr) (struct _archiveHandle * AH);
  
  typedef void (*ClonePtr) (struct _archiveHandle * AH);
  typedef void (*DeClonePtr) (struct _archiveHandle * AH);
  
+ typedef bool (*StartCheckArchivePtr)(struct _archiveHandle * AH);
+ typedef bool (*CheckTocEntryPtr)(struct _archiveHandle * AH, struct _tocEntry * te,
+ 								 enum _teReqs reqs);
+ typedef bool (*EndCheckArchivePtr)(struct _archiveHandle * AH);
+ 
  typedef size_t (*CustomOutPtr) (struct _archiveHandle * AH, const void *buf, size_t len);
  
  typedef struct _outputContext
*************** typedef enum
*** 177,183 ****
  	STAGE_FINALIZING
  } ArchiverStage;
  
! typedef enum
  {
  	REQ_SCHEMA = 1,
  	REQ_DATA = 2,
--- 184,190 ----
  	STAGE_FINALIZING
  } ArchiverStage;
  
! typedef enum _teReqs
  {
  	REQ_SCHEMA = 1,
  	REQ_DATA = 2,
*************** typedef struct _archiveHandle
*** 239,244 ****
--- 246,252 ----
  										 * archie format */
  	PrintExtraTocPtr PrintExtraTocPtr;	/* Extra TOC info for format */
  	PrintTocDataPtr PrintTocDataPtr;
+ 	PrintExtraTocSummaryPtr PrintExtraTocSummaryPtr;
  
  	StartBlobsPtr StartBlobsPtr;
  	EndBlobsPtr EndBlobsPtr;
*************** typedef struct _archiveHandle
*** 248,253 ****
--- 256,265 ----
  	ClonePtr ClonePtr;			/* Clone format-specific fields */
  	DeClonePtr DeClonePtr;		/* Clean up cloned fields */
  
+ 	StartCheckArchivePtr StartCheckArchivePtr;
+ 	CheckTocEntryPtr CheckTocEntryPtr;
+ 	EndCheckArchivePtr EndCheckArchivePtr;
+ 
  	CustomOutPtr CustomOutPtr;	/* Alternative script output routine */
  
  	/* Stuff for direct DB connection */
*************** extern void EndRestoreBlob(ArchiveHandle
*** 383,388 ****
--- 395,401 ----
  extern void EndRestoreBlobs(ArchiveHandle *AH);
  
  extern void InitArchiveFmt_Custom(ArchiveHandle *AH);
+ extern void InitArchiveFmt_Directory(ArchiveHandle *AH);
  extern void InitArchiveFmt_Files(ArchiveHandle *AH);
  extern void InitArchiveFmt_Null(ArchiveHandle *AH);
  extern void InitArchiveFmt_Tar(ArchiveHandle *AH);
*************** int			ahprintf(ArchiveHandle *AH, const 
*** 397,402 ****
--- 410,417 ----
  
  void		ahlog(ArchiveHandle *AH, int level, const char *fmt,...) __attribute__((format(printf, 3, 4)));
  
+ extern bool getTimestampString(char *buf, size_t buflen, time_t tim);
+ 
  #ifdef USE_ASSERT_CHECKING
  #define Assert(condition) \
  	if (!(condition)) \
diff --git a/src/bin/pg_dump/pg_backup_custom.c b/src/bin/pg_dump/pg_backup_custom.c
index f3f41b5..e40a57c 100644
*** a/src/bin/pg_dump/pg_backup_custom.c
--- b/src/bin/pg_dump/pg_backup_custom.c
*************** InitArchiveFmt_Custom(ArchiveHandle *AH)
*** 118,123 ****
--- 118,124 ----
  	AH->ReadExtraTocPtr = _ReadExtraToc;
  	AH->WriteExtraTocPtr = _WriteExtraToc;
  	AH->PrintExtraTocPtr = _PrintExtraToc;
+ 	AH->PrintExtraTocSummaryPtr = NULL;
  
  	AH->StartBlobsPtr = _StartBlobs;
  	AH->StartBlobPtr = _StartBlob;
*************** InitArchiveFmt_Custom(ArchiveHandle *AH)
*** 126,131 ****
--- 127,136 ----
  	AH->ClonePtr = _Clone;
  	AH->DeClonePtr = _DeClone;
  
+ 	AH->StartCheckArchivePtr = NULL;
+ 	AH->CheckTocEntryPtr = NULL;
+ 	AH->EndCheckArchivePtr = NULL;
+ 
  	/*
  	 * Set up some special context used in compressing data.
  	 */
diff --git a/src/bin/pg_dump/pg_backup_directory.c b/src/bin/pg_dump/pg_backup_directory.c
index ...d8c550a .
*** a/src/bin/pg_dump/pg_backup_directory.c
--- b/src/bin/pg_dump/pg_backup_directory.c
***************
*** 0 ****
--- 1,1469 ----
+ /*-------------------------------------------------------------------------
+  *
+  * pg_backup_directory.c
+  *
+  *	This file is copied from the 'files' format file and dumps data into
+  *	separate files in a directory.
+  *
+  *	See the headers to pg_backup_files & pg_restore for more details.
+  *
+  * Copyright (c) 2000, Philip Warner
+  *		Rights are granted to use this software in any way so long
+  *		as this notice is not removed.
+  *
+  *	The author is not responsible for loss or damages that may
+  *	result from it's use.
+  *
+  *
+  * IDENTIFICATION
+  * 	XXX
+  *
+  *-------------------------------------------------------------------------
+  */
+ 
+ #include <dirent.h>
+ #include <sys/stat.h>
+ 
+ #include "compress_io.h"
+ #include "pg_backup_archiver.h"
+ #include "utils/pg_crc.h"
+ 
+ #define TOC_FH_ACTIVE \
+ 		(ctx->dataFH == NULL && ctx->blobsTocFH == NULL && AH->FH != NULL)
+ #define BLOBS_TOC_FH_ACTIVE \
+ 		(ctx->dataFH == NULL && ctx->blobsTocFH != NULL)
+ #define DATA_FH_ACTIVE \
+ 		(ctx->dataFH != NULL)
+ 
+ struct _lclFileHeader;
+ struct _lclContext;
+ 
+ static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
+ static void _StartData(ArchiveHandle *AH, TocEntry *te);
+ static void _EndData(ArchiveHandle *AH, TocEntry *te);
+ static size_t _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
+ static int	_WriteByte(ArchiveHandle *AH, const int i);
+ static int	_ReadByte(ArchiveHandle *);
+ static size_t _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
+ static size_t _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
+ static void _CloseArchive(ArchiveHandle *AH);
+ static void _PrintTocData(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt);
+ 
+ static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
+ static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
+ static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
+ static void _PrintExtraTocSummary(ArchiveHandle *AH);
+ 
+ static void _WriteExtraHead(ArchiveHandle *AH);
+ static void _ReadExtraHead(ArchiveHandle *AH);
+ 
+ static void WriteFileHeader(ArchiveHandle *AH, int type);
+ static int ReadFileHeader(ArchiveHandle *AH, struct _lclFileHeader *fileHeader);
+ 
+ static void _StartBlobs(ArchiveHandle *AH, TocEntry *te);
+ static void _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
+ static void _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
+ static void _EndBlobs(ArchiveHandle *AH, TocEntry *te);
+ static void _LoadBlobs(ArchiveHandle *AH, RestoreOptions *ropt);
+ 
+ static size_t _DirectoryReadFunction(ArchiveHandle *AH, void **buf,
+ 									 size_t sizeHint);
+ 
+ static void _checkOutput(ArchiveHandle *AH, bool isChecking, const char *fmt, ...);
+ static bool _StartCheckArchive(ArchiveHandle *AH);
+ static bool _CheckTocEntry(ArchiveHandle *AH, TocEntry *te, teReqs reqs);
+ static bool _CheckFileContents(ArchiveHandle *AH, const char *fname,
+ 							   bool dieOnError);
+ static bool _CheckFileSize(ArchiveHandle *AH, const char *fname, pgoff_t pgSize,
+ 						   bool isChecking);
+ static bool _CheckBlob(ArchiveHandle *AH, Oid oid, pgoff_t size);
+ static bool _CheckBlobs(ArchiveHandle *AH, TocEntry *te, teReqs reqs);
+ static bool _EndCheckArchive(ArchiveHandle *AH);
+ 
+ static char *prependDirectory(ArchiveHandle *AH, const char *relativeFilename);
+ static char *prependBlobsDirectory(ArchiveHandle *AH, Oid oid);
+ static void createDirectory(const char *dir, const char *subdir);
+ 
+ static bool isDirectory(const char *fname);
+ static bool isRegularFile(const char *fname);
+ 
+ #define K_STD_BUF_SIZE	1024
+ #define FILE_SUFFIX		".dat"
+ 
+ typedef struct _lclContext
+ {
+ 	/*
+ 	 * Our archive location. This is basically what the user specified as his
+ 	 * backup file but of course here it is a directory.
+ 	 */
+ 	char			   *directory;
+ 
+ 	/*
+ 	 * To prevent that we accidentially mix the different files of a backup
+ 	 * set, we keep an id in each of the files. If pg_restore finds out that it
+ 	 * should restore files from different backup sets, it issues a warning.
+ 	 */
+ 	char				idStr[256];
+ 
+ 	/*
+ 	 * In the directory archive format we have three file handles:
+ 	 *
+ 	 * AH->FH           points to the TOC
+ 	 * ctx->blobsTocFH  points to the TOC for the BLOBs
+ 	 * ctx->dataFH      points to data files (both BLOBs and regular)
+ 	 *
+ 	 * Instead of specifying where each I/O operation should go (which would
+ 	 * require own prototypes anyway and wouldn't be that straightforward
+ 	 * either), we rely on a hierarchy among the file descriptors.
+ 	 *
+ 	 * As a matter of fact we never access any of the TOCs when we are writing
+ 	 * to a data file, only before or after that. Similarly we never access the
+ 	 * general TOC when we have opened the TOC for BLOBs. Given these facts we
+ 	 * can just write our I/O routines such that they access:
+ 	 *
+ 	 * if      defined(ctx->dataFH)      => access ctx->dataFH
+ 	 * else if defined(ctx->blobsTocFH)  => access ctx->blobsTocFH
+ 	 * else                              => access AH->FH
+ 	 *
+ 	 * To make it more transparent what is going on, we use assertions like
+ 	 *
+ 	 *      Assert(DATA_FH_ACTIVE); ...
+ 	 *
+ 	 */
+ 	FILE			   *dataFH;				/* file position for a data file */
+ 	pgoff_t				dataFilePos;
+ 	FILE			   *blobsTocFH;			/* file position for BLOBS.TOC */
+ 	pgoff_t				blobsTocFilePos;
+ 	pgoff_t				tocFilePos;			/* file position for AH->FH */
+ 
+ 	/* these are used for checking a directory archive */
+ 	DumpId			   *chkList;
+ 	int					chkListSize;
+ 
+ 	CompressorState	   *cs;
+ } lclContext;
+ 
+ typedef struct
+ {
+ 	char	   *filename;		/* filename excluding the directory (basename) */
+ 	pgoff_t		fileSize;
+ } lclTocEntry;
+ 
+ typedef struct _lclFileHeader
+ {
+ 	int			version;
+ 	int			type;			/* BLK_DATA or BLK_BLOB */
+ 	char	   *idStr;
+ 	int			compression;
+ } lclFileHeader;
+ 
+ static const char *modulename = gettext_noop("directory archiver");
+ 
+ /*
+  *	Init routine required by ALL formats. This is a global routine
+  *	and should be declared in pg_backup_archiver.h
+  *
+  *	It's task is to create any extra archive context (using AH->formatData),
+  *	and to initialize the supported function pointers.
+  *
+  *	It should also prepare whatever it's input source is for reading/writing,
+  *	and in the case of a read mode connection, it should load the Header & TOC.
+  */
+ void
+ InitArchiveFmt_Directory(ArchiveHandle *AH)
+ {
+ 	lclContext *ctx;
+ 
+ 	/* Assuming static functions, this can be copied for each format. */
+ 	AH->ArchiveEntryPtr = _ArchiveEntry;
+ 	AH->StartDataPtr = _StartData;
+ 	AH->WriteDataPtr = _WriteData;
+ 	AH->EndDataPtr = _EndData;
+ 	AH->WriteBytePtr = _WriteByte;
+ 	AH->ReadBytePtr = _ReadByte;
+ 	AH->WriteBufPtr = _WriteBuf;
+ 	AH->ReadBufPtr = _ReadBuf;
+ 	AH->ClosePtr = _CloseArchive;
+ 	AH->ReopenPtr = NULL;
+ 	AH->PrintTocDataPtr = _PrintTocData;
+ 	AH->ReadExtraTocPtr = _ReadExtraToc;
+ 	AH->WriteExtraTocPtr = _WriteExtraToc;
+ 	AH->PrintExtraTocPtr = _PrintExtraToc;
+ 	AH->PrintExtraTocSummaryPtr = _PrintExtraTocSummary;
+ 
+ 	AH->StartBlobsPtr = _StartBlobs;
+ 	AH->StartBlobPtr = _StartBlob;
+ 	AH->EndBlobPtr = _EndBlob;
+ 	AH->EndBlobsPtr = _EndBlobs;
+ 
+ 	AH->ClonePtr = NULL;
+ 	AH->DeClonePtr = NULL;
+ 
+ 	AH->StartCheckArchivePtr = _StartCheckArchive;
+ 	AH->CheckTocEntryPtr = _CheckTocEntry;
+ 	AH->EndCheckArchivePtr = _EndCheckArchive;
+ 
+ 	/*
+ 	 * Set up some special context used in compressing data.
+ 	 */
+ 	ctx = (lclContext *) calloc(1, sizeof(lclContext));
+ 	if (ctx == NULL)
+ 		die_horribly(AH, modulename, "out of memory\n");
+ 	AH->formatData = (void *) ctx;
+ 
+ 	ctx->dataFH = NULL;
+ 	ctx->blobsTocFH = NULL;
+ 	ctx->cs = NULL;
+ 
+ 	/* Initialize LO buffering */
+ 	AH->lo_buf_size = LOBBUFSIZE;
+ 	AH->lo_buf = (void *) malloc(LOBBUFSIZE);
+ 	if (AH->lo_buf == NULL)
+ 		die_horribly(AH, modulename, "out of memory\n");
+ 
+ 	/*
+ 	 * Now open the TOC file
+ 	 */
+ 
+ 	if (!AH->fSpec || strcmp(AH->fSpec, "") == 0)
+ 		die_horribly(AH, modulename, "no directory specified\n");
+ 
+ 	ctx->directory = AH->fSpec;
+ 
+ 	if (AH->mode == archModeWrite)
+ 	{
+ 		char   *fname = prependDirectory(AH, "TOC");
+ 
+ 		/*
+ 		 * Create the ID string, basically the current timestamp that prevents
+ 		 * that we accidently mix files from different backups.
+ 		 */
+ 		if (!getTimestampString(ctx->idStr, sizeof(ctx->idStr), time(NULL)))
+ 			die_horribly(AH, modulename, "could not get timestamp\n");
+ 
+ 		/* Create the directory, errors are caught there */
+ 		createDirectory(ctx->directory, NULL);
+ 
+ 		ctx->cs = AllocateCompressorState(COMPRESSOR_DEFLATE);
+ 
+ 		AH->FH = fopen(fname, PG_BINARY_W);
+ 		if (AH->FH == NULL)
+ 			die_horribly(AH, modulename, "could not open output file \"%s\": %s\n",
+ 						 fname, strerror(errno));
+ 	}
+ 	else
+ 	{							/* Read Mode */
+ 		char	   *fname;
+ 
+ 		fname = prependDirectory(AH, "TOC");
+ 
+ 		AH->FH = fopen(fname, PG_BINARY_R);
+ 		if (AH->FH == NULL)
+ 			die_horribly(AH, modulename,
+ 						 "could not open input file \"%s\": %s\n",
+ 						 fname, strerror(errno));
+ 
+ 		Assert(TOC_FH_ACTIVE);
+ 
+ 		ReadHead(AH);
+ 		_ReadExtraHead(AH);
+ 		ReadToc(AH);
+ 
+ 		/*
+ 		 * We get the compression information from the TOC, hence no need to
+ 		 * initialize the compressor earlier.  Also, remember that the TOC file
+ 		 * is always uncompressed. Compression is only used for the data files.
+ 		 */
+ 		ctx->cs = AllocateCompressorState(COMPRESSOR_INFLATE);
+ 
+ 		/* Nothing else in the file, so close it again... */
+ 
+ 		if (fclose(AH->FH) != 0)
+ 			die_horribly(AH, modulename, "could not close TOC file: %s\n",
+ 						 strerror(errno));
+ 	}
+ }
+ 
+ /*
+  * Called by the Archiver when the dumper creates a new TOC entry.
+  *
+  * Optional.
+  *
+  * Set up extrac format-related TOC data.
+ */
+ static void
+ _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
+ {
+ 	lclTocEntry	   *tctx;
+ 	char			fn[MAXPGPATH];
+ 
+ 	tctx = (lclTocEntry *) calloc(1, sizeof(lclTocEntry));
+ 	if (te->dataDumper)
+ 	{
+ 		sprintf(fn, "%d"FILE_SUFFIX, te->dumpId);
+ 		tctx->filename = strdup(fn);
+ 	}
+ 	else if (strcmp(te->desc, "BLOBS") == 0)
+ 	{
+ 		tctx->filename = strdup("BLOBS.TOC");
+ 	}
+ 	else
+ 		tctx->filename = NULL;
+ 
+ 	tctx->fileSize = 0;
+ 	te->formatData = (void *) tctx;
+ }
+ 
+ /*
+  * Called by the Archiver to save any extra format-related TOC entry
+  * data.
+  *
+  * Optional.
+  *
+  * Use the Archiver routines to write data - they are non-endian, and
+  * maintain other important file information.
+  */
+ static void
+ _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
+ {
+ 	lclTocEntry *tctx = (lclTocEntry *) te->formatData;
+ 
+ 	/*
+ 	 * A dumpable object has set tctx->filename, any other object hasnt.
+ 	 * (see _ArchiveEntry).
+ 	 */
+ 	if (tctx->filename)
+ 	{
+ 		WriteStr(AH, tctx->filename);
+ 		WriteOffset(AH, tctx->fileSize, K_OFFSET_POS_SET);
+ 	}
+ 	else
+ 		WriteStr(AH, "");
+ }
+ 
+ /*
+  * Called by the Archiver to read any extra format-related TOC data.
+  *
+  * Optional.
+  *
+  * Needs to match the order defined in _WriteExtraToc, and sould also
+  * use the Archiver input routines.
+  */
+ static void
+ _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
+ {
+ 	lclTocEntry *tctx = (lclTocEntry *) te->formatData;
+ 
+ 	if (tctx == NULL)
+ 	{
+ 		tctx = (lclTocEntry *) calloc(1, sizeof(lclTocEntry));
+ 		te->formatData = (void *) tctx;
+ 	}
+ 
+ 	tctx->filename = ReadStr(AH);
+ 	if (strlen(tctx->filename) == 0)
+ 	{
+ 		free(tctx->filename);
+ 		tctx->filename = NULL;
+ 	}
+ 	else
+ 		ReadOffset(AH, &(tctx->fileSize));
+ }
+ 
+ /*
+  * Called by the Archiver when restoring an archive to output a comment
+  * that includes useful information about the TOC entry.
+  *
+  * Optional.
+  *
+  */
+ static void
+ _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
+ {
+ 	lclTocEntry *tctx = (lclTocEntry *) te->formatData;
+ 
+ 	if (AH->public.verbose && tctx->filename)
+ 		ahprintf(AH, "-- File: %s\n", tctx->filename);
+ }
+ 
+ /*
+  * Called by the Archiver when listing the contents of an archive to output a
+  * comment that includes useful information about the archive.
+  *
+  * Optional.
+  *
+  */
+ static void
+ _PrintExtraTocSummary(ArchiveHandle *AH)
+ {
+ 	lclContext *ctx = (lclContext *) AH->formatData;
+ 	ahprintf(AH, ";     ID: %s\n", ctx->idStr);
+ }
+ 
+ 
+ /*
+  * Called by the archiver when saving TABLE DATA (not schema). This routine
+  * should save whatever format-specific information is needed to read
+  * the archive back.
+  *
+  * It is called just prior to the dumper's 'DataDumper' routine being called.
+  *
+  * Optional, but strongly recommended.
+  *
+  */
+ static void
+ _StartData(ArchiveHandle *AH, TocEntry *te)
+ {
+ 	lclTocEntry	   *tctx = (lclTocEntry *) te->formatData;
+ 	lclContext	   *ctx = (lclContext *) AH->formatData;
+ 	char		   *fname;
+ 
+ 	fname = prependDirectory(AH, tctx->filename);
+ 
+ 	ctx->dataFH = (FILE *) fopen(fname, PG_BINARY_W);
+ 	if (ctx->dataFH == NULL)
+ 		die_horribly(AH, modulename, "could not open output file \"%s\": %s\n",
+ 					 fname, strerror(errno));
+ 
+ 	Assert(DATA_FH_ACTIVE);
+ 
+ 	ctx->dataFilePos = 0;
+ 
+ 	WriteFileHeader(AH, BLK_DATA);
+ 
+ 	InitCompressorState(ctx->cs, AH->compression);
+ }
+ 
+ static void
+ WriteFileHeader(ArchiveHandle *AH, int type)
+ {
+ 	lclContext *ctx = (lclContext *) AH->formatData;
+ 	int compression = AH->compression;
+ 
+ 	/*
+ 	 * We always write the header uncompressed. If any compression is active,
+ 	 * switch it off for a moment and restore it after writing the header.
+ 	 */
+ 	AH->compression = 0;
+ 	(*AH->WriteBufPtr) (AH, "PGDMP", 5);		/* Magic code */
+ 	(*AH->WriteBytePtr) (AH, AH->vmaj);
+ 	(*AH->WriteBytePtr) (AH, AH->vmin);
+ 	(*AH->WriteBytePtr) (AH, AH->vrev);
+ 
+ 	_WriteByte(AH, type);
+ 	WriteInt(AH, compression);
+ 	WriteStr(AH, ctx->idStr);
+ 
+ 	AH->compression = compression;
+ }
+ 
+ static int
+ ReadFileHeader(ArchiveHandle *AH, lclFileHeader *fileHeader)
+ {
+ 	char		tmpMag[7];
+ 	int			vmaj, vmin, vrev;
+ 	int			compression = AH->compression;
+ 	bool		err = false;
+ 
+ #ifdef USE_ASSERT_CHECKING
+ 	lclContext *ctx = (lclContext *) AH->formatData;
+ #endif
+ 	Assert(ftell(ctx->dataFH ? ctx->dataFH :
+ 				 ctx->blobsTocFH ? ctx->blobsTocFH :
+ 				 AH->FH) == 0);
+ 
+ 	/* Read with compression switched off. See WriteFileHeader() */
+ 	AH->compression = 0;
+ 	if ((*AH->ReadBufPtr) (AH, tmpMag, 5) != 5)
+ 		die_horribly(AH, modulename, "unexpected end of file\n");
+ 
+ 	vmaj = (*AH->ReadBytePtr) (AH);
+ 	vmin = (*AH->ReadBytePtr) (AH);
+ 	vrev = (*AH->ReadBytePtr) (AH);
+ 
+ 	/* Make a convenient integer <maj><min><rev>00 */
+ 	fileHeader->version = ((vmaj * 256 + vmin) * 256 + vrev) * 256 + 0;
+ 	fileHeader->type = _ReadByte(AH);
+ 	if (fileHeader->type != BLK_BLOBS && fileHeader->type != BLK_DATA)
+ 		err = true;
+ 	if (!err)
+ 		fileHeader->compression = ReadInt(AH);
+ 	if (!err)
+ 	{
+ 		fileHeader->idStr = ReadStr(AH);
+ 		if (fileHeader->idStr == NULL)
+ 			err = true;
+ 	}
+ 
+ 	/* we do not check fileHeader->idStr against ctx->idStr, this is left to
+ 	 * the caller. */
+ 
+ 	AH->compression = compression;
+ 
+ 	return err ? -1 : 0;
+ }
+ 
+ /*
+  * Called by archiver when dumper calls WriteData. This routine is
+  * called for both BLOB and TABLE data; it is the responsibility of
+  * the format to manage each kind of data using StartBlob/StartData.
+  *
+  * It should only be called from within a DataDumper routine.
+  *
+  * Mandatory.
+  */
+ static size_t
+ _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
+ {
+ 	lclContext		   *ctx = (lclContext *) AH->formatData;
+ 
+ 	return WriteDataToArchive(AH, ctx->cs, _WriteBuf, data, dLen);
+ }
+ 
+ /*
+  * Called by the archiver when a dumper's 'DataDumper' routine has
+  * finished.
+  *
+  * Optional.
+  *
+  */
+ static void
+ _EndData(ArchiveHandle *AH, TocEntry *te)
+ {
+ 	lclTocEntry	   *tctx = (lclTocEntry *) te->formatData;
+ 	lclContext	   *ctx = (lclContext *) AH->formatData;
+ 
+ 	FlushCompressorState(AH, ctx->cs, _WriteBuf);
+ 
+ 	Assert(DATA_FH_ACTIVE);
+ 
+ 	/* Close the file */
+ 	fclose(ctx->dataFH);
+ 
+ 	/* the file won't grow anymore. Record the size. */
+ 	tctx->fileSize = ctx->dataFilePos;
+ 
+ 	ctx->dataFH = NULL;
+ }
+ 
+ /*
+  * Print data for a given file (can be a BLOB as well)
+  */
+ static void
+ _PrintFileData(ArchiveHandle *AH, char *filename, pgoff_t expectedSize,
+ 			   RestoreOptions *ropt)
+ {
+ 	lclContext		   *ctx = (lclContext *) AH->formatData;
+ 	lclFileHeader		fileHeader;
+ 
+ 	if (!filename)
+ 		return;
+ 
+ 	_CheckFileContents(AH, filename, false);
+ 	_CheckFileSize(AH, filename, expectedSize, false);
+ 
+ 	ctx->dataFH = fopen(filename, PG_BINARY_R);
+ 	if (!ctx->dataFH)
+ 		die_horribly(AH, modulename, "could not open input file \"%s\": %s\n",
+ 					 filename, strerror(errno));
+ 
+ 	if (ReadFileHeader(AH, &fileHeader) != 0)
+ 		die_horribly(AH, modulename,
+ 					 "could not read valid file header from file \"%s\"\n",
+ 					 filename);
+ 
+ 	Assert(DATA_FH_ACTIVE);
+ 
+ 	InitCompressorState(ctx->cs, fileHeader.compression);
+ 	ReadDataFromArchive(AH, ctx->cs, _DirectoryReadFunction);
+ 
+ 	ctx->dataFH = NULL;
+ }
+ 
+ 
+ /*
+  * Print data for a given TOC entry
+ */
+ static void
+ _PrintTocData(ArchiveHandle *AH, TocEntry *te, RestoreOptions *ropt)
+ {
+ 	lclTocEntry *tctx = (lclTocEntry *) te->formatData;
+ 
+ 	if (!tctx->filename)
+ 		return;
+ 
+ 	if (strcmp(te->desc, "BLOBS") == 0)
+ 		_LoadBlobs(AH, ropt);
+ 	else
+ 	{
+ 		char   *fname = prependDirectory(AH, tctx->filename);
+ 		_PrintFileData(AH, fname, tctx->fileSize, ropt);
+ 	}
+ }
+ 
+ static void
+ _LoadBlobs(ArchiveHandle *AH, RestoreOptions *ropt)
+ {
+ 	Oid				oid;
+ 	lclContext	   *ctx = (lclContext *) AH->formatData;
+ 	lclFileHeader	fileHeader;
+ 	char		   *fname;
+ 
+ 	StartRestoreBlobs(AH);
+ 
+ 	fname = prependDirectory(AH, "BLOBS.TOC");
+ 
+ 	ctx->blobsTocFH = fopen(fname, "rb");
+ 
+ 	if (ctx->blobsTocFH == NULL)
+ 		die_horribly(AH, modulename, "could not open large object TOC file \"%s\" for input: %s\n",
+ 					 fname, strerror(errno));
+ 
+ 	ReadFileHeader(AH, &fileHeader);
+ 
+ 	/* we cannot test for feof() since EOF only shows up in the low
+ 	 * level read functions. But they would die_horribly() anyway. */
+ 	while (1)
+ 	{
+ 		char			   *blobFname;
+ 		pgoff_t				blobSize;
+ 
+ 		oid = ReadInt(AH);
+ 		/* oid == 0 is our end marker */
+ 		if (oid == 0)
+ 			break;
+ 		ReadOffset(AH, &blobSize);
+ 
+ 		StartRestoreBlob(AH, oid, ropt->dropSchema);
+ 		blobFname = prependBlobsDirectory(AH, oid);
+ 		_PrintFileData(AH, blobFname, blobSize, ropt);
+ 		EndRestoreBlob(AH, oid);
+ 	}
+ 
+ 	if (fclose(ctx->blobsTocFH) != 0)
+ 		die_horribly(AH, modulename, "could not close large object TOC file \"%s\": %s\n",
+ 					 fname, strerror(errno));
+ 
+ 	ctx->blobsTocFH = NULL;
+ 
+ 	EndRestoreBlobs(AH);
+ }
+ 
+ 
+ /*
+  * Write a byte of data to the archive.
+  *
+  * Mandatory.
+  *
+  * Called by the archiver to do integer & byte output to the archive.
+  * These routines are only used to read & write headers & TOC.
+  *
+  */
+ static int
+ _WriteByte(ArchiveHandle *AH, const int i)
+ {
+ 	lclContext *ctx = (lclContext *) AH->formatData;
+ 	pgoff_t	   *filePos = &ctx->tocFilePos;
+ 	FILE	   *stream = AH->FH;
+ 
+ 	if (ctx->dataFH)
+ 	{
+ 		stream = ctx->dataFH;
+ 		filePos = &ctx->dataFilePos;
+ 	}
+ 	else if (ctx->blobsTocFH)
+ 	{
+ 		stream = ctx->blobsTocFH;
+ 		filePos = &ctx->blobsTocFilePos;
+ 	}
+ 
+ 	if (fputc(i, stream) == EOF)
+ 		die_horribly(AH, modulename, "could not write byte\n");
+ 
+ 	*filePos += 1;
+ 
+ 	return 1;
+ }
+ 
+ /*
+  * Read a byte of data from the archive.
+  *
+  * Mandatory
+  *
+  * Called by the archiver to read bytes & integers from the archive.
+  * These routines are only used to read & write headers & TOC.
+  * EOF should be treated as a fatal error.
+  */
+ static int
+ _ReadByte(ArchiveHandle *AH)
+ {
+ 	lclContext *ctx = (lclContext *) AH->formatData;
+ 	pgoff_t	   *filePos = &ctx->tocFilePos;
+ 	int			res;
+ 	FILE	   *stream = AH->FH;
+ 
+ 	if (ctx->dataFH)
+ 	{
+ 		stream = ctx->dataFH;
+ 		filePos = &ctx->dataFilePos;
+ 	}
+ 	else if (ctx->blobsTocFH)
+ 	{
+ 		stream = ctx->blobsTocFH;
+ 		filePos = &ctx->blobsTocFilePos;
+ 	}
+ 
+ 	res = getc(stream);
+ 	if (res == EOF)
+ 		die_horribly(AH, modulename, "unexpected end of file\n");
+ 
+ 	*filePos += 1;
+ 
+ 	return res;
+ }
+ 
+ /*
+  * Write a buffer of data to the archive.
+  *
+  * Mandatory.
+  *
+  * Called by the archiver to write a block of bytes to the TOC and by the
+  * compressor to write compressed data to the data files.
+  *
+  */
+ static size_t
+ _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
+ {
+ 	lclContext *ctx = (lclContext *) AH->formatData;
+ 	pgoff_t	   *filePos = &ctx->tocFilePos;
+ 	size_t		res;
+ 	FILE	   *stream = AH->FH;
+ 
+ 	if (ctx->dataFH)
+ 	{
+ 		stream = ctx->dataFH;
+ 		filePos = &ctx->dataFilePos;
+ 	}
+ 	else if (ctx->blobsTocFH)
+ 	{
+ 		stream = ctx->blobsTocFH;
+ 		filePos = &ctx->blobsTocFilePos;
+ 	}
+ 
+ 	res = fwrite(buf, 1, len, stream);
+ 	if (res != len)
+ 		die_horribly(AH, modulename, "could not write to output file: %s\n",
+ 					 strerror(errno));
+ 
+ 	*filePos += res;
+ 
+ 	return res;
+ }
+ 
+ /*
+  * Read a block of bytes from the archive.
+  *
+  * Mandatory.
+  *
+  * Called by the archiver to read a block of bytes from the archive
+  *
+  */
+ static size_t
+ _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
+ {
+ 	lclContext *ctx = (lclContext *) AH->formatData;
+ 	pgoff_t	   *filePos = &ctx->tocFilePos;
+ 	size_t		res;
+ 	FILE	   *stream = AH->FH;
+ 
+ 	if (ctx->dataFH)
+ 	{
+ 		stream = ctx->dataFH;
+ 		filePos = &ctx->dataFilePos;
+ 	}
+ 	else if (ctx->blobsTocFH)
+ 	{
+ 		stream = ctx->blobsTocFH;
+ 		filePos = &ctx->blobsTocFilePos;
+ 	}
+ 
+ 	res = fread(buf, 1, len, stream);
+ 
+ 	*filePos += res;
+ 
+ 	return res;
+ }
+ 
+ /*
+  * Close the archive.
+  *
+  * Mandatory.
+  *
+  * When writing the archive, this is the routine that actually starts
+  * the process of saving it to files. No data should be written prior
+  * to this point, since the user could sort the TOC after creating it.
+  *
+  * If an archive is to be written, this routine must call:
+  *		WriteHead			to save the archive header
+  *		WriteToc			to save the TOC entries
+  *		WriteDataChunks		to save all DATA & BLOBs.
+  *
+  */
+ static void
+ _CloseArchive(ArchiveHandle *AH)
+ {
+ 	if (AH->mode == archModeWrite)
+ 	{
+ #ifdef USE_ASSERT_CHECKING
+ 		lclContext	   *ctx = (lclContext *) AH->formatData;
+ #endif
+ 
+ 		WriteDataChunks(AH);
+ 
+ 		Assert(TOC_FH_ACTIVE);
+ 
+ 		WriteHead(AH);
+ 		_WriteExtraHead(AH);
+ 		WriteToc(AH);
+ 
+ 		if (fclose(AH->FH) != 0)
+ 			die_horribly(AH, modulename, "could not close TOC file: %s\n",
+ 						 strerror(errno));
+ 	}
+ 	AH->FH = NULL;
+ }
+ 
+ 
+ 
+ /*
+  * BLOB support
+  */
+ 
+ /*
+  * Called by the archiver when starting to save all BLOB DATA (not schema).
+  * This routine should save whatever format-specific information is needed
+  * to read the BLOBs back into memory.
+  *
+  * It is called just prior to the dumper's DataDumper routine.
+  *
+  * Optional, but strongly recommended.
+  */
+ static void
+ _StartBlobs(ArchiveHandle *AH, TocEntry *te)
+ {
+ 	lclContext	   *ctx = (lclContext *) AH->formatData;
+ 	char		   *fname;
+ 
+ 	fname = prependDirectory(AH, "BLOBS.TOC");
+ 	createDirectory(ctx->directory, "blobs");
+ 
+ 	ctx->blobsTocFH = fopen(fname, "ab");
+ 	if (ctx->blobsTocFH == NULL)
+ 		die_horribly(AH, modulename, "could not open output file \"%s\": %s\n",
+ 					 fname, strerror(errno));
+ 
+ 	Assert(BLOBS_TOC_FH_ACTIVE);
+ 
+ 	ctx->blobsTocFilePos = 0;
+ 
+ 	WriteFileHeader(AH, BLK_BLOBS);
+ }
+ 
+ /*
+  * Called by the archiver when the dumper calls StartBlob.
+  *
+  * Mandatory.
+  *
+  * Must save the passed OID for retrieval at restore-time.
+  */
+ static void
+ _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
+ {
+ 	lclContext	   *ctx = (lclContext *) AH->formatData;
+ 	char		   *fname;
+ 
+ 	fname = prependBlobsDirectory(AH, oid);
+ 	ctx->dataFH = (FILE *) fopen(fname, PG_BINARY_W);
+ 
+ 	if (ctx->dataFH == NULL)
+ 		die_horribly(AH, modulename, "could not open output file \"%s\": %s\n",
+ 					 fname, strerror(errno));
+ 
+ 	Assert(DATA_FH_ACTIVE);
+ 
+ 	ctx->dataFilePos = 0;
+ 
+ 	WriteFileHeader(AH, BLK_BLOBS);
+ 
+ 	InitCompressorState(ctx->cs, AH->compression);
+ }
+ 
+ /*
+  * Called by the archiver when the dumper calls EndBlob.
+  *
+  * Optional.
+  */
+ static void
+ _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
+ {
+ 	lclContext	   *ctx = (lclContext *) AH->formatData;
+ 	pgoff_t			save_filePos;
+ 
+ 	FlushCompressorState(AH, ctx->cs, _WriteBuf);
+ 
+ 	Assert(DATA_FH_ACTIVE);
+ 
+ 	save_filePos = ctx->dataFilePos;
+ 
+ 	/* Close the BLOB data file itself */
+ 	fclose(ctx->dataFH);
+ 	ctx->dataFH = NULL;
+ 
+ 	Assert(BLOBS_TOC_FH_ACTIVE);
+ 
+ 	/* register the BLOB data file to BLOBS.TOC */
+ 	WriteInt(AH, oid);
+ 	WriteOffset(AH, save_filePos, K_OFFSET_POS_NOT_SET);
+ }
+ 
+ /*
+  * Called by the archiver when finishing saving all BLOB DATA.
+  *
+  * Optional.
+  */
+ static void
+ _EndBlobs(ArchiveHandle *AH, TocEntry *te)
+ {
+ 	lclContext *ctx = (lclContext *) AH->formatData;
+ 	lclTocEntry *tctx = (lclTocEntry *) te->formatData;
+ 
+ 	Assert(BLOBS_TOC_FH_ACTIVE);
+ 
+ 	WriteInt(AH, 0);
+ 
+ 	fclose(ctx->blobsTocFH);
+ 	ctx->blobsTocFH = NULL;
+ 
+ 	tctx->fileSize = ctx->blobsTocFilePos;
+ }
+ 
+ static void
+ _checkOutput(ArchiveHandle *AH, bool isChecking, const char *fmt, ...)
+ {
+ 	va_list	ap;
+ 	/*
+ 	 * We use isChecking to find out where our output should go to.
+ 	 * If it is set, then we are running from an explicit check routine and
+ 	 * write to stdout. If we are not checking explicitly, we write any issues
+ 	 * to stderr.
+ 	 */
+ 
+ 	va_start(ap, fmt);
+ 	if (isChecking)
+ 		vprintf(fmt, ap);
+ 	else
+ 	{
+ 		char	buf[512];
+ 		vsnprintf(buf, sizeof(buf), fmt, ap);
+ 		warn_or_die_horribly(AH, modulename, "%s", buf);
+ 	}
+ 	va_end(ap);
+ }
+ 
+ /*
+  * The idea for the directory check is as follows: First we do a list of every
+  * file that we find in the directory. We reject filenames that don't fit our
+  * pattern outright. So at this stage we only accept all kinds of TOC data and
+  * our data files.
+  *
+  * If a filename looks good (like nnnnn.dat), we save its dumpId to
+  * ctx->chkList.
+  *
+  * Other checks then walk through the TOC and for every file they make sure
+  * that the file is what it is pretending to be. Once it passes the checks we
+  * take out its entry in chkList, i.e. replace its dumpId by InvalidDumpId.
+  *
+  * At the end what is left in chkList must be files that are not referenced
+  * from the TOC.
+  */
+ static bool
+ _StartCheckArchive(ArchiveHandle *AH)
+ {
+ 	bool			checkOK = true;
+ 	lclContext	   *ctx = (lclContext *) AH->formatData;
+ 	DIR			   *dir;
+ 	char		   *dname = ctx->directory;
+ 	struct dirent  *entry;
+ 	int				idx = 0;
+ 	char		   *suffix;
+ 	bool			tocSeen = false;
+ 
+ 	dir = opendir(dname);
+ 	if (!dir)
+ 	{
+ 		_checkOutput(AH, true, "Could not open directory \"%s\": %s\n",
+ 					 dname, strerror(errno));
+ 		return false;
+ 	}
+ 
+ 	/*
+ 	 * Actually we are just avoiding a linked list here by getting an upper
+ 	 * limit of the number of elements in the directory.
+ 	 */
+ 	while ((entry = readdir(dir)))
+ 		idx++;
+ 
+ 	ctx->chkListSize = idx;
+ 	ctx->chkList = (DumpId *) malloc(ctx->chkListSize * sizeof(DumpId));
+ 
+ 	/* seems that Windows doesn't have a rewinddir() equivalent */
+ 	closedir(dir);
+ 	dir = opendir(dname);
+ 	if (!dir)
+ 	{
+ 		_checkOutput(AH, true, "Could not open directory \"%s\": %s\n",
+ 					 dname, strerror(errno));
+ 		return false;
+ 	}
+ 
+ 	idx = 0;
+ 
+ 	for (;;)
+ 	{
+ 		errno = 0;
+ 		entry = readdir(dir);
+ 		if (!entry && errno == 0)
+ 			/* end of directory entries reached */
+ 			break;
+ 		if (!entry && errno)
+ 		{
+ 			_checkOutput(AH, true, "Error reading directory %s: %s\n",
+ 						 entry->d_name, strerror(errno));
+ 			checkOK = false;
+ 			break;
+ 		}
+ 
+ 		if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
+ 			continue;
+ 		if (strcmp(entry->d_name, "blobs") == 0 &&
+ 						isDirectory(prependDirectory(AH, entry->d_name)))
+ 			continue;
+ 		if (strcmp(entry->d_name, "BLOBS.TOC") == 0 &&
+ 						isRegularFile(prependDirectory(AH, entry->d_name)))
+ 			continue;
+ 		if (strcmp(entry->d_name, "TOC") == 0 &&
+ 						isRegularFile(prependDirectory(AH, entry->d_name)))
+ 		{
+ 			tocSeen = true;
+ 			continue;
+ 		}
+ 		/* besides the above we only expect nnnn.dat, with nnnn being our
+ 		 * numerical dumpID */
+ 		if ((suffix = strstr(entry->d_name, FILE_SUFFIX)) == NULL)
+ 		{
+ 			_checkOutput(AH, true, "Unexpected file \"%s\" in directory \"%s\"\n",
+ 						 entry->d_name, dname);
+ 			checkOK = false;
+ 			continue;
+ 		}
+ 		else
+ 		{
+ 			/* suffix now points into entry->d_name */
+ 			int dumpId;
+ 			int scBytes, scItems;
+ 
+ 			/* check if FILE_SUFFIX is really a suffix instead of just a
+ 			 * substring. */
+ 			if (strlen(suffix) != strlen(FILE_SUFFIX))
+ 			{
+ 				_checkOutput(AH, true, "Unexpected file \"%s\" in directory \"%s\"\n",
+ 							 entry->d_name, dname);
+ 				checkOK = false;
+ 				continue;
+ 			}
+ 
+ 			/* cut off the suffix, now entry->d_name contains the null
+ 			 * terminated dumpId, and we parse it back. */
+ 			*suffix = '\0';
+ 			scItems = sscanf(entry->d_name, "%d%n", &dumpId, &scBytes);
+ 			if (scItems != 1 || scBytes != strlen(entry->d_name))
+ 			{
+ 				_checkOutput(AH, true, "Unexpected file \"%s\" in directory \"%s\"\n",
+ 							 entry->d_name, dname);
+ 				checkOK = false;
+ 				continue;
+ 			}
+ 
+ 			/* Still here so this entry is good. Add the dumpId to our list. */
+ 			ctx->chkList[idx++] = (DumpId) dumpId;
+ 		}
+ 	}
+ 	closedir(dir);
+ 
+ 	/* we probably counted a few entries too much, just ignore them. */
+ 	while (idx < ctx->chkListSize)
+ 		ctx->chkList[idx++] = InvalidDumpId;
+ 
+ 	/* also return false if we haven't seen the TOC file */
+ 	return checkOK && tocSeen;
+ }
+ 
+ static bool
+ _CheckFileSize(ArchiveHandle *AH, const char *fname, pgoff_t pgSize,
+ 			   bool isChecking)
+ {
+ 	bool			checkOK = true;
+ 	unsigned long	size = (unsigned long) pgSize;
+ 	struct stat		st;
+ 
+ 	if (!fname || fname[0] == '\0')
+ 	{
+ 		_checkOutput(AH, isChecking, "Invalid (empty) filename\n");
+ 		checkOK = false;
+ 	}
+ 	else if (stat(fname, &st) != 0)
+ 	{
+ 		_checkOutput(AH, isChecking, "File not found: \"%s\"\n", fname);
+ 		checkOK = false;
+ 	}
+ 	else if (st.st_size != (off_t) pgSize)
+ 	{
+ 		_checkOutput(AH, isChecking, "Size mismatch for file \"%s\" "
+ 					 "(expected: %lu bytes, actual %lu bytes)\n",
+ 					 fname, size, (unsigned long) st.st_size);
+ 		checkOK = false;
+ 	}
+ 
+ 	return checkOK;
+ }
+ 
+ static bool
+ _CheckFileContents(ArchiveHandle *AH, const char *fname, bool isChecking)
+ {
+ 	bool			checkOK = true;
+ 	lclContext	   *ctx = (lclContext *) AH->formatData;
+ 	FILE		   *file;
+ 	lclFileHeader	fileHeader;
+ 
+ 	Assert(ctx->dataFH == NULL);
+ 
+ 	/* see comment at _CheckFileSize() */
+ 
+ 	if (!fname || fname[0] == '\0')
+ 	{
+ 		_checkOutput(AH, isChecking, "Invalid (empty) filename\n");
+ 		return false;
+ 	}
+ 
+ 	if (!(file = fopen(fname, PG_BINARY_R)))
+ 	{
+ 		_checkOutput(AH, isChecking, "Could not open file \"%s\": %s\n",
+ 					 fname, strerror(errno));
+ 		return false;
+ 	}
+ 
+ 	ctx->dataFH = file;
+ 	if (ReadFileHeader(AH, &fileHeader) != 0)
+ 	{
+ 		_checkOutput(AH, isChecking, "Could not read valid file header from file \"%s\"\n",
+ 					 fname);
+ 		checkOK = false;
+ 	}
+ 
+ 	if (file)
+ 		fclose(file);
+ 	ctx->dataFH = NULL;
+ 
+ 	if (!checkOK && !isChecking)
+ 	{
+ 		/* this error is serious enough that we cannot go on */
+ 		if (AH->connection)
+ 			PQfinish(AH->connection);
+ 		exit(1);
+ 	}
+ 
+ 	if (strcmp(fileHeader.idStr, ctx->idStr) != 0)
+ 	{
+ 		_checkOutput(AH, isChecking, "File \"%s\" belongs to a different backup "
+ 					 "(expected id: %s, actual id: %s)\n",
+ 					 fname, ctx->idStr, fileHeader.idStr);
+ 		checkOK = false;
+ 	}
+ 
+ 	return checkOK;
+ }
+ 
+ static bool
+ _CheckBlob(ArchiveHandle *AH, Oid oid, pgoff_t size)
+ {
+ 	char		   *fname = prependBlobsDirectory(AH, oid);
+ 	bool			checkOK = true;
+ 
+ 	if (!_CheckFileSize(AH, fname, size, true))
+ 		checkOK = false;
+ 	else if (!_CheckFileContents(AH, fname, true))
+ 		checkOK = false;
+ 
+ 	return checkOK;
+ }
+ 
+ static bool
+ _CheckBlobs(ArchiveHandle *AH, TocEntry *te, teReqs reqs)
+ {
+ 	lclTocEntry	   *tctx = (lclTocEntry *) te->formatData;
+ 	lclContext	   *ctx = (lclContext *) AH->formatData;
+ 	char		   *fname;
+ 	bool			checkOK = true;
+ 	lclFileHeader	fileHeader;
+ 	pgoff_t			size;
+ 	Oid				oid;
+ 
+ 	/* check the BLOBS.TOC first */
+ 	fname = prependDirectory(AH, "BLOBS.TOC");
+ 
+ 	if (!fname)
+ 	{
+ 		_checkOutput(AH, true, "Could not find BLOBS.TOC. Check the archive!\n");
+ 		return false;
+ 	}
+ 
+ 	if (!_CheckFileSize(AH, fname, tctx->fileSize, true))
+ 		checkOK = false;
+ 	else if (!_CheckFileContents(AH, fname, true))
+ 		checkOK = false;
+ 
+ 	/* now check every single BLOB object */
+ 	ctx->blobsTocFH = fopen(fname, "rb");
+ 	if (ctx->blobsTocFH == NULL)
+ 	{
+ 		_checkOutput(AH, true, "could not open large object TOC for input: %s\n",
+ 					 strerror(errno));
+ 		return false;
+ 	}
+ 	ReadFileHeader(AH, &fileHeader);
+ 
+ 	/* we cannot test for feof() since EOF only shows up in the low
+ 	 * level read functions. But they would die_horribly() anyway. */
+ 	while ((oid = ReadInt(AH)))
+ 	{
+ 		Assert(BLOBS_TOC_FH_ACTIVE);
+ 
+ 		ReadOffset(AH, &size);
+ 
+ 		if (!_CheckBlob(AH, oid, size))
+ 			checkOK = false;
+ 
+ 		Assert(BLOBS_TOC_FH_ACTIVE);
+ 	}
+ 
+ 	Assert(BLOBS_TOC_FH_ACTIVE);
+ 
+ 	if (fclose(ctx->blobsTocFH) != 0)
+ 	{
+ 		_checkOutput(AH, true, "could not close large object TOC file: %s\n",
+ 					 strerror(errno));
+ 		checkOK = false;
+ 	}
+ 
+ 	return checkOK;
+ }
+ 
+ 
+ static bool
+ _CheckTocEntry(ArchiveHandle *AH, TocEntry *te, teReqs reqs)
+ {
+ 	lclContext	   *ctx = (lclContext *) AH->formatData;
+ 	lclTocEntry	   *tctx = (lclTocEntry *) te->formatData;
+ 	int				idx;
+ 	bool			checkOK = true;
+ 
+ 	/* take out files from chkList as we see them */
+ 	for (idx = 0; idx < ctx->chkListSize; idx++)
+ 	{
+ 		if (ctx->chkList[idx] == te->dumpId && te->section == SECTION_DATA)
+ 		{
+ 			ctx->chkList[idx] = InvalidDumpId;
+ 			break;
+ 		}
+ 	}
+ 
+ 	/* see comment in _tocEntryRequired() for the special case of SEQUENCE SET */
+ 	if (reqs & REQ_DATA && strcmp(te->desc, "BLOBS") == 0)
+ 	{
+ 		if (!_CheckBlobs(AH, te, reqs))
+ 			checkOK = false;
+ 	}
+ 	else if (reqs & REQ_DATA && strcmp(te->desc, "SEQUENCE SET") != 0
+ 							 && strcmp(te->desc, "BLOB") != 0
+ 							 && strcmp(te->desc, "COMMENT") != 0)
+ 	{
+ 		char		   *fname;
+ 
+ 		fname = prependDirectory(AH, tctx->filename);
+ 		if (!fname)
+ 		{
+ 			printf("Could not find file %s\n", tctx->filename);
+ 			checkOK = false;
+ 		}
+ 		else if (!_CheckFileContents(AH, fname, true))
+ 			checkOK = false;
+ 		else if (!_CheckFileSize(AH, fname, tctx->fileSize, true))
+ 			checkOK = false;
+ 	}
+ 
+ 	return checkOK;
+ }
+ 
+ static bool
+ _EndCheckArchive(ArchiveHandle *AH)
+ {
+ 	/* check left over files */
+ 	lclContext	   *ctx = (lclContext *) AH->formatData;
+ 	int				idx;
+ 	bool			checkOK = true;
+ 
+ 	for (idx = 0; idx < ctx->chkListSize; idx++)
+ 	{
+ 		if (ctx->chkList[idx] != InvalidDumpId)
+ 		{
+ 			printf("Unexpected file: %d"FILE_SUFFIX"\n", ctx->chkList[idx]);
+ 			checkOK = false;
+ 		}
+ 	}
+ 
+ 	return checkOK;
+ }
+ 
+ 
+ static void
+ createDirectory(const char *dir, const char *subdir)
+ {
+ 	struct stat		st;
+ 	char			dirname[MAXPGPATH];
+ 
+ 	/* the directory must not yet exist, first check if it is existing */
+ 	if (subdir && strlen(dir) + 1 + strlen(subdir) + 1 > MAXPGPATH)
+ 		die_horribly(NULL, modulename, "directory name %s too long", dir);
+ 
+ 	strcpy(dirname, dir);
+ 
+ 	if (subdir)
+ 	{
+ 		strcat(dirname, "/");
+ 		strcat(dirname, subdir);
+ 	}
+ 
+ 	if (stat(dirname, &st) == 0)
+ 	{
+ 		if (S_ISDIR(st.st_mode))
+ 			die_horribly(NULL, modulename,
+ 						 "Cannot create directory %s, it exists already\n",
+ 						 dirname);
+ 		else
+ 			die_horribly(NULL, modulename,
+ 						 "Cannot create directory %s, a file with this name "
+ 						 "exists already\n", dirname);
+ 	}
+ 
+ 	/*
+ 	 * Now we create the directory. Note that for some race condition we could
+ 	 * also run into the situation that the directory has been created just
+ 	 * between our two calls.
+ 	 */
+ 	if (mkdir(dirname, 0700) < 0)
+ 		die_horribly(NULL, modulename, "Could not create directory %s: %s",
+ 					 dirname, strerror(errno));
+ }
+ 
+ 
+ static char *
+ prependDirectory(ArchiveHandle *AH, const char *relativeFilename)
+ {
+ 	lclContext	   *ctx = (lclContext *) AH->formatData;
+ 	static char		buf[MAXPGPATH];
+ 	char		   *dname;
+ 
+ 	dname = ctx->directory;
+ 
+ 	if (strlen(dname) + 1 + strlen(relativeFilename) + 1 > MAXPGPATH)
+ 			die_horribly(AH, modulename, "path name too long: %s", dname);
+ 
+ 	strcpy(buf, dname);
+ 	strcat(buf, "/");
+ 	strcat(buf, relativeFilename);
+ 
+ 	return buf;
+ }
+ 
+ static char *
+ prependBlobsDirectory(ArchiveHandle *AH, Oid oid)
+ {
+ 	static char		buf[MAXPGPATH];
+ 	char		   *dname;
+ 	lclContext	   *ctx = (lclContext *) AH->formatData;
+ 	int				r;
+ 
+ 	dname = ctx->directory;
+ 
+ 	r = snprintf(buf, MAXPGPATH, "%s/blobs/%d%s",
+ 				 dname, oid, FILE_SUFFIX);
+ 
+ 	if (r < 0 || r >= MAXPGPATH)
+ 		die_horribly(AH, modulename, "path name too long: %s", dname);
+ 
+ 	return buf;
+ }
+ 
+ static size_t
+ _DirectoryReadFunction(ArchiveHandle *AH, void **buf, size_t sizeHint)
+ {
+ 	lclContext		   *ctx = (lclContext *) AH->formatData;
+ 	CompressorState	   *cs = ctx->cs;
+ 
+ 	Assert(cs->comprInSize >= comprInInitSize);
+ 
+ 	if (sizeHint == 0)
+ 		sizeHint = comprInInitSize;
+ 
+ 	*buf = cs->comprIn;
+ 	return _ReadBuf(AH, cs->comprIn, sizeHint);
+ }
+ 
+ static void
+ _WriteExtraHead(ArchiveHandle *AH)
+ {
+ 	lclContext	   *ctx = (lclContext *) AH->formatData;
+ 	WriteStr(AH, ctx->idStr);
+ }
+ 
+ static void
+ _ReadExtraHead(ArchiveHandle *AH)
+ {
+ 	lclContext *ctx = (lclContext *) AH->formatData;
+ 	char	   *str = ReadStr(AH);
+ 
+ 	strcpy(ctx->idStr, str);
+ }
+ 
+ static bool
+ isDirectory(const char *fname)
+ {
+ 	struct stat st;
+ 
+ 	if (stat(fname, &st))
+ 		return false;
+ 
+ 	return S_ISDIR(st.st_mode);
+ }
+ 
+ static bool
+ isRegularFile(const char *fname)
+ {
+ 	struct stat st;
+ 
+ 	if (stat(fname, &st))
+ 		return false;
+ 
+ 	return S_ISREG(st.st_mode);
+ }
+ 
diff --git a/src/bin/pg_dump/pg_backup_files.c b/src/bin/pg_dump/pg_backup_files.c
index abc93b1..825c473 100644
*** a/src/bin/pg_dump/pg_backup_files.c
--- b/src/bin/pg_dump/pg_backup_files.c
*************** InitArchiveFmt_Files(ArchiveHandle *AH)
*** 92,97 ****
--- 92,98 ----
  	AH->ReadExtraTocPtr = _ReadExtraToc;
  	AH->WriteExtraTocPtr = _WriteExtraToc;
  	AH->PrintExtraTocPtr = _PrintExtraToc;
+ 	AH->PrintExtraTocSummaryPtr = NULL;
  
  	AH->StartBlobsPtr = _StartBlobs;
  	AH->StartBlobPtr = _StartBlob;
*************** InitArchiveFmt_Files(ArchiveHandle *AH)
*** 100,105 ****
--- 101,110 ----
  	AH->ClonePtr = NULL;
  	AH->DeClonePtr = NULL;
  
+ 	AH->StartCheckArchivePtr = NULL;
+ 	AH->CheckTocEntryPtr = NULL;
+ 	AH->EndCheckArchivePtr = NULL;
+ 
  	/*
  	 * Set up some special context used in compressing data.
  	 */
diff --git a/src/bin/pg_dump/pg_backup_tar.c b/src/bin/pg_dump/pg_backup_tar.c
index 006f7da..dcc13ee 100644
*** a/src/bin/pg_dump/pg_backup_tar.c
--- b/src/bin/pg_dump/pg_backup_tar.c
*************** InitArchiveFmt_Tar(ArchiveHandle *AH)
*** 144,149 ****
--- 144,150 ----
  	AH->ReadExtraTocPtr = _ReadExtraToc;
  	AH->WriteExtraTocPtr = _WriteExtraToc;
  	AH->PrintExtraTocPtr = _PrintExtraToc;
+ 	AH->PrintExtraTocSummaryPtr = NULL;
  
  	AH->StartBlobsPtr = _StartBlobs;
  	AH->StartBlobPtr = _StartBlob;
*************** InitArchiveFmt_Tar(ArchiveHandle *AH)
*** 152,157 ****
--- 153,162 ----
  	AH->ClonePtr = NULL;
  	AH->DeClonePtr = NULL;
  
+ 	AH->StartCheckArchivePtr = NULL;
+ 	AH->CheckTocEntryPtr = NULL;
+ 	AH->EndCheckArchivePtr = NULL;
+ 
  	/*
  	 * Set up some special context used in compressing data.
  	 */
diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c
index 8a71b99..dcd77da 100644
*** a/src/bin/pg_dump/pg_dump.c
--- b/src/bin/pg_dump/pg_dump.c
*************** static int	no_security_label = 0;
*** 138,143 ****
--- 138,144 ----
  
  
  static void help(const char *progname);
+ static ArchiveFormat parseArchiveFormat(const char *format);
  static void expand_schema_name_patterns(SimpleStringList *patterns,
  							SimpleOidList *oids);
  static void expand_table_name_patterns(SimpleStringList *patterns,
*************** main(int argc, char **argv)
*** 267,272 ****
--- 268,274 ----
  	int			my_version;
  	int			optindex;
  	RestoreOptions *ropt;
+ 	ArchiveFormat   archiveFormat = archUnknown;
  
  	static int	disable_triggers = 0;
  	static int	outputNoTablespaces = 0;
*************** main(int argc, char **argv)
*** 542,575 ****
  	if (compressLevel == COMPRESSION_UNKNOWN)
  		compressLevel = Z_DEFAULT_COMPRESSION;
  
! 	/* open the output file */
! 	if (pg_strcasecmp(format, "a") == 0 || pg_strcasecmp(format, "append") == 0)
! 	{
! 		/* This is used by pg_dumpall, and is not documented */
  		plainText = 1;
! 		g_fout = CreateArchive(filename, archNull, 0, archModeAppend);
! 	}
! 	else if (pg_strcasecmp(format, "c") == 0 || pg_strcasecmp(format, "custom") == 0)
! 		g_fout = CreateArchive(filename, archCustom, compressLevel, archModeWrite);
! 	else if (pg_strcasecmp(format, "f") == 0 || pg_strcasecmp(format, "file") == 0)
! 	{
! 		/*
! 		 * Dump files into the current directory; for demonstration only, not
! 		 * documented.
! 		 */
! 		g_fout = CreateArchive(filename, archFiles, compressLevel, archModeWrite);
! 	}
! 	else if (pg_strcasecmp(format, "p") == 0 || pg_strcasecmp(format, "plain") == 0)
  	{
! 		plainText = 1;
! 		g_fout = CreateArchive(filename, archNull, 0, archModeWrite);
  	}
! 	else if (pg_strcasecmp(format, "t") == 0 || pg_strcasecmp(format, "tar") == 0)
! 		g_fout = CreateArchive(filename, archTar, compressLevel, archModeWrite);
! 	else
  	{
! 		write_msg(NULL, "invalid output format \"%s\" specified\n", format);
! 		exit(1);
  	}
  
  	if (g_fout == NULL)
--- 544,607 ----
  	if (compressLevel == COMPRESSION_UNKNOWN)
  		compressLevel = Z_DEFAULT_COMPRESSION;
  
! 	archiveFormat = parseArchiveFormat(format);
! 
! 	/* archiveFormat specific setup */
! 	if (archiveFormat == archNull || archiveFormat == archNullAppend)
  		plainText = 1;
! 
! 	/*
! 	 * If AH->compression == UNKNOWN_COMPRESSION then it has not been set to some
! 	 * value explicitly.
! 	 *
! 	 * Fall back to default:
! 	 *
! 	 * zlib with Z_DEFAULT_COMPRESSION for those formats that support it.
! 	 * If either one is not available: use no compression at all.
! 	 */
! 
! 	if (compressLevel == COMPRESSION_UNKNOWN)
  	{
! #ifdef HAVE_LIBZ
! 		if (archiveFormat == archCustom || archiveFormat == archDirectory)
! 			compressLevel = Z_DEFAULT_COMPRESSION;
! 		else
! 			compressLevel = 0;
! #else
! 		compressLevel = 0;
! #endif
  	}
! 
! 	/* open the output file */
! 	switch(archiveFormat)
  	{
! 		case archCustom:
! 			g_fout = CreateArchive(filename, archCustom, compressLevel,
! 								   archModeWrite);
! 			break;
! 		case archDirectory:
! 			g_fout = CreateArchive(filename, archDirectory, compressLevel,
! 								   archModeWrite);
! 			break;
! 		case archFiles:
! 			g_fout = CreateArchive(filename, archFiles, compressLevel,
! 								   archModeWrite);
! 			break;
! 		case archNull:
! 			g_fout = CreateArchive(filename, archNull, 0, archModeWrite);
! 			break;
! 		case archNullAppend:
! 			g_fout = CreateArchive(filename, archNull, 0, archModeAppend);
! 			break;
! 		case archTar:
! 			g_fout = CreateArchive(filename, archTar, compressLevel,
! 								   archModeWrite);
! 			break;
! 
! 		default:
! 			/* we never reach here, because we check in parseArchiveFormat()
!  			 * already. */
! 			break;
  	}
  
  	if (g_fout == NULL)
*************** help(const char *progname)
*** 839,847 ****
  
  	printf(_("\nGeneral options:\n"));
  	printf(_("  -f, --file=FILENAME         output file name\n"));
! 	printf(_("  -F, --format=c|t|p          output file format (custom, tar, plain text)\n"));
  	printf(_("  -v, --verbose               verbose mode\n"));
! 	printf(_("  -Z, --compress=0-9          compression level for compressed formats\n"));
  	printf(_("  --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n"));
  	printf(_("  --help                      show this help, then exit\n"));
  	printf(_("  --version                   output version information, then exit\n"));
--- 871,879 ----
  
  	printf(_("\nGeneral options:\n"));
  	printf(_("  -f, --file=FILENAME         output file name\n"));
! 	printf(_("  -F, --format=c|d|t|p        output file format (custom, directory, tar, plain text)\n"));
  	printf(_("  -v, --verbose               verbose mode\n"));
! 	printf(_("  -Z, --compress=0-9          compression level of libz for compressed formats\n"));
  	printf(_("  --lock-wait-timeout=TIMEOUT fail after waiting TIMEOUT for a table lock\n"));
  	printf(_("  --help                      show this help, then exit\n"));
  	printf(_("  --version                   output version information, then exit\n"));
*************** exit_nicely(void)
*** 896,901 ****
--- 928,971 ----
  	exit(1);
  }
  
+ static ArchiveFormat
+ parseArchiveFormat(const char *format)
+ {
+ 	ArchiveFormat archiveFormat;
+ 
+ 	if (pg_strcasecmp(format, "a") == 0 || pg_strcasecmp(format, "append") == 0)
+ 		/* This is used by pg_dumpall, and is not documented */
+ 		archiveFormat = archNullAppend;
+ 	else if (pg_strcasecmp(format, "c") == 0)
+ 		archiveFormat = archCustom;
+ 	else if (pg_strcasecmp(format, "custom") == 0)
+ 		archiveFormat = archCustom;
+ 	else if (pg_strcasecmp(format, "d") == 0)
+ 		archiveFormat = archDirectory;
+ 	else if (pg_strcasecmp(format, "directory") == 0)
+ 		archiveFormat = archDirectory;
+ 	else if (pg_strcasecmp(format, "f") == 0 || pg_strcasecmp(format, "file") == 0)
+ 		/*
+ 		 * Dump files into the current directory; for demonstration only, not
+ 		 * documented.
+ 		 */
+ 		archiveFormat = archFiles;
+ 	else if (pg_strcasecmp(format, "p") == 0)
+ 		archiveFormat = archNull;
+ 	else if (pg_strcasecmp(format, "plain") == 0)
+ 		archiveFormat = archNull;
+ 	else if (pg_strcasecmp(format, "t") == 0)
+ 		archiveFormat = archTar;
+ 	else if (pg_strcasecmp(format, "tar") == 0)
+ 		archiveFormat = archTar;
+ 	else
+ 	{
+ 		write_msg(NULL, "invalid output format \"%s\" specified\n", format);
+ 		exit(1);
+ 	}
+ 	return archiveFormat;
+ }
+ 
  /*
   * Find the OIDs of all schemas matching the given list of patterns,
   * and append them to the given OID list.
diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h
index 7885535..0f643b9 100644
*** a/src/bin/pg_dump/pg_dump.h
--- b/src/bin/pg_dump/pg_dump.h
*************** typedef struct
*** 39,44 ****
--- 39,45 ----
  } CatalogId;
  
  typedef int DumpId;
+ #define InvalidDumpId		(-1)
  
  /*
   * Data structures for simple lists of OIDs and strings.  The support for
diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c
index 1ddba72..3fbe264 100644
*** a/src/bin/pg_dump/pg_restore.c
--- b/src/bin/pg_dump/pg_restore.c
*************** main(int argc, char **argv)
*** 79,84 ****
--- 79,85 ----
  	static int	skip_seclabel = 0;
  
  	struct option cmdopts[] = {
+ 		{"check", 0, NULL, 'k'},
  		{"clean", 0, NULL, 'c'},
  		{"create", 0, NULL, 'C'},
  		{"data-only", 0, NULL, 'a'},
*************** main(int argc, char **argv)
*** 144,150 ****
  		}
  	}
  
! 	while ((c = getopt_long(argc, argv, "acCd:ef:F:h:iI:j:lL:n:Op:P:RsS:t:T:U:vwWxX:1",
  							cmdopts, NULL)) != -1)
  	{
  		switch (c)
--- 145,151 ----
  		}
  	}
  
! 	while ((c = getopt_long(argc, argv, "acCd:ef:F:h:iI:j:klL:n:Op:P:RsS:t:T:U:vwWxX:1",
  							cmdopts, NULL)) != -1)
  	{
  		switch (c)
*************** main(int argc, char **argv)
*** 182,188 ****
  			case 'j':			/* number of restore jobs */
  				opts->number_of_jobs = atoi(optarg);
  				break;
! 
  			case 'l':			/* Dump the TOC summary */
  				opts->tocSummary = 1;
  				break;
--- 183,191 ----
  			case 'j':			/* number of restore jobs */
  				opts->number_of_jobs = atoi(optarg);
  				break;
! 			case 'k':			/* check the archive */
! 				opts->checkArchive = 1;
! 				break;
  			case 'l':			/* Dump the TOC summary */
  				opts->tocSummary = 1;
  				break;
*************** main(int argc, char **argv)
*** 352,357 ****
--- 355,365 ----
  				opts->format = archCustom;
  				break;
  
+ 			case 'd':
+ 			case 'D':
+ 				opts->format = archDirectory;
+ 				break;
+ 
  			case 'f':
  			case 'F':
  				opts->format = archFiles;
*************** main(int argc, char **argv)
*** 363,369 ****
  				break;
  
  			default:
! 				write_msg(NULL, "unrecognized archive format \"%s\"; please specify \"c\" or \"t\"\n",
  						  opts->formatName);
  				exit(1);
  		}
--- 371,377 ----
  				break;
  
  			default:
! 				write_msg(NULL, "unrecognized archive format \"%s\"; please specify \"c\", \"d\" or \"t\"\n",
  						  opts->formatName);
  				exit(1);
  		}
*************** main(int argc, char **argv)
*** 392,397 ****
--- 400,413 ----
  
  	if (opts->tocSummary)
  		PrintTOCSummary(AH, opts);
+ 	else if (opts->checkArchive)
+ 	{
+ 		bool    checkOK;
+ 		checkOK = CheckArchive(AH, opts);
+ 		CloseArchive(AH);
+ 		if (!checkOK)
+ 			exit(1);
+ 	}
  	else
  		RestoreArchive(AH, opts);
  
*************** usage(const char *progname)
*** 418,425 ****
  	printf(_("\nGeneral options:\n"));
  	printf(_("  -d, --dbname=NAME        connect to database name\n"));
  	printf(_("  -f, --file=FILENAME      output file name\n"));
! 	printf(_("  -F, --format=c|t         backup file format (should be automatic)\n"));
  	printf(_("  -l, --list               print summarized TOC of the archive\n"));
  	printf(_("  -v, --verbose            verbose mode\n"));
  	printf(_("  --help                   show this help, then exit\n"));
  	printf(_("  --version                output version information, then exit\n"));
--- 434,442 ----
  	printf(_("\nGeneral options:\n"));
  	printf(_("  -d, --dbname=NAME        connect to database name\n"));
  	printf(_("  -f, --file=FILENAME      output file name\n"));
! 	printf(_("  -F, --format=c|d|t       backup file format (should be automatic)\n"));
  	printf(_("  -l, --list               print summarized TOC of the archive\n"));
+ 	printf(_("  -k                       check the directory archive\n"));
  	printf(_("  -v, --verbose            verbose mode\n"));
  	printf(_("  --help                   show this help, then exit\n"));
  	printf(_("  --version                output version information, then exit\n"));