skip_hidden_files_v5.diff

application/octet-stream

Filename: skip_hidden_files_v5.diff
Type: application/octet-stream
Part: 0
Message: Re: pg_basebackup: errors on macOS on directories with ".DS_Store" files

Patch

Same data as JSON: GET /api/v1/attachments/:id/patch the parsed metadata as JSON — format, series position, per-file stats; never the diff bytes. API reference →
Format: unified
Series: patch v5
File+
doc/src/sgml/protocol.sgml 1 1
doc/src/sgml/ref/pg_basebackup.sgml 1 1
doc/src/sgml/ref/pg_rewind.sgml 3 2
src/backend/backup/basebackup.c 7 4
src/bin/pg_basebackup/t/010_pg_basebackup.pl 2 2
src/bin/pg_checksums/pg_checksums.c 5 8
src/bin/pg_checksums/t/002_actions.pl 3 0
src/bin/pg_rewind/file_ops.c 4 0
src/bin/pg_rewind/t/003_extrafiles.pl 4 0
diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index b11d9a6ba3..08f740f6ca 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -3048,7 +3048,7 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
         <listitem>
          <para>
           Files other than regular files and directories, such as symbolic
-          links (other than for the directories listed above) and special
+          links (other than for the directories listed above), hidden files and special
           device files, are skipped.  (Symbolic links
           in <filename>pg_tblspc</filename> are maintained.)
          </para>
diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml
index 79d3e657c3..a7c8ed7370 100644
--- a/doc/src/sgml/ref/pg_basebackup.sgml
+++ b/doc/src/sgml/ref/pg_basebackup.sgml
@@ -909,7 +909,7 @@ PostgreSQL documentation
    The backup will include all files in the data directory and tablespaces,
    including the configuration files and any additional files placed in the
    directory by third parties, except certain temporary files managed by
-   PostgreSQL.  But only regular files and directories are copied, except that
+   PostgreSQL and hidden files.  But only regular files and directories are copied, except that
    symbolic links used for tablespaces are preserved.  Symbolic links pointing
    to certain directories known to PostgreSQL are copied as empty directories.
    Other symbolic links and special device files are skipped.
diff --git a/doc/src/sgml/ref/pg_rewind.sgml b/doc/src/sgml/ref/pg_rewind.sgml
index 9118f05bf2..005fa83b9e 100644
--- a/doc/src/sgml/ref/pg_rewind.sgml
+++ b/doc/src/sgml/ref/pg_rewind.sgml
@@ -381,8 +381,9 @@ GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text, bigint, bigint, b
       <filename>backup_label</filename>,
       <filename>tablespace_map</filename>,
       <filename>pg_internal.init</filename>,
-      <filename>postmaster.opts</filename>, and
-      <filename>postmaster.pid</filename>, as well as any file or directory
+      <filename>postmaster.opts</filename>,
+      <filename>postmaster.pid</filename>, and
+      hidden files as well as any file or directory
       beginning with <filename>pgsql_tmp</filename>, are omitted.
      </para>
     </step>
diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c
index 5baea7535b..e3d041c35d 100644
--- a/src/backend/backup/basebackup.c
+++ b/src/backend/backup/basebackup.c
@@ -1196,10 +1196,13 @@ sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
 		if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
 			continue;
 
-		/* Skip temporary files */
-		if (strncmp(de->d_name,
-					PG_TEMP_FILE_PREFIX,
-					strlen(PG_TEMP_FILE_PREFIX)) == 0)
+		/* Skip temporary files and folders */
+		if (strstr(de->d_name, PG_TEMP_FILE_PREFIX) != NULL ||
+			strstr(de->d_name, PG_TEMP_FILES_DIR "/") != NULL)
+			continue;
+
+		/* Skip hidden files */
+		if (de->d_name[0] == '.')
 			continue;
 
 		/*
diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
index 4d130a7f94..95dbcf7338 100644
--- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl
+++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
@@ -173,7 +173,7 @@ SKIP:
 # Write some files to test that they are not copied.
 foreach my $filename (
 	qw(backup_label tablespace_map postgresql.auto.conf.tmp
-	current_logfiles.tmp global/pg_internal.init.123))
+	current_logfiles.tmp global/pg_internal.init.123 pgsql_tmp.donotcopy .donotcopy))
 {
 	open my $file, '>>', "$pgdata/$filename";
 	print $file "DONOTCOPY";
@@ -244,7 +244,7 @@ foreach my $dirname (
 # These files should not be copied.
 foreach my $filename (
 	qw(postgresql.auto.conf.tmp postmaster.opts postmaster.pid tablespace_map current_logfiles.tmp
-	global/pg_internal.init global/pg_internal.init.123))
+	global/pg_internal.init global/pg_internal.init.123 pgsql_tmp.donotcopy .donotcopy))
 {
 	ok(!-f "$tempdir/backup/$filename", "$filename not copied");
 }
diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c
index 19eb67e485..68c255877f 100644
--- a/src/bin/pg_checksums/pg_checksums.c
+++ b/src/bin/pg_checksums/pg_checksums.c
@@ -325,16 +325,13 @@ scan_directory(const char *basedir, const char *subdir, bool sizeonly)
 			strcmp(de->d_name, "..") == 0)
 			continue;
 
-		/* Skip temporary files */
-		if (strncmp(de->d_name,
-					PG_TEMP_FILE_PREFIX,
-					strlen(PG_TEMP_FILE_PREFIX)) == 0)
+		/* Skip temporary files and folders */
+		if (strstr(de->d_name, PG_TEMP_FILE_PREFIX) != NULL ||
+			strstr(de->d_name, PG_TEMP_FILES_DIR "/") != NULL)
 			continue;
 
-		/* Skip temporary folders */
-		if (strncmp(de->d_name,
-					PG_TEMP_FILES_DIR,
-					strlen(PG_TEMP_FILES_DIR)) == 0)
+		/* Skip hidden files */
+		if (de->d_name[0] == '.')
 			continue;
 
 		snprintf(fn, sizeof(fn), "%s/%s", path, de->d_name);
diff --git a/src/bin/pg_checksums/t/002_actions.pl b/src/bin/pg_checksums/t/002_actions.pl
index 2316f611b2..ff975e5601 100644
--- a/src/bin/pg_checksums/t/002_actions.pl
+++ b/src/bin/pg_checksums/t/002_actions.pl
@@ -114,6 +114,9 @@ append_to_file "$pgdata/global/pgsql_tmp/1.1",        "foo";
 append_to_file "$pgdata/global/pg_internal.init",     "foo";
 append_to_file "$pgdata/global/pg_internal.init.123", "foo";
 
+# These are non-postgres files, which should be ignored by the scan
+append_to_file "$pgdata/global/.donotscan", "foo";
+
 # Enable checksums.
 command_ok([ 'pg_checksums', '--enable', '--no-sync', '-D', $pgdata ],
 	"checksums successfully enabled in cluster");
diff --git a/src/bin/pg_rewind/file_ops.c b/src/bin/pg_rewind/file_ops.c
index 25996b4da4..77ff8ac5d2 100644
--- a/src/bin/pg_rewind/file_ops.c
+++ b/src/bin/pg_rewind/file_ops.c
@@ -398,6 +398,10 @@ recurse_dir(const char *datadir, const char *parentpath,
 			strcmp(xlde->d_name, "..") == 0)
 			continue;
 
+		/* Skip hidden files */
+		if (xlde->d_name[0] == '.')
+			continue;
+
 		snprintf(fullpath, sizeof(fullpath), "%s/%s", fullparentpath, xlde->d_name);
 
 		if (lstat(fullpath, &fst) < 0)
diff --git a/src/bin/pg_rewind/t/003_extrafiles.pl b/src/bin/pg_rewind/t/003_extrafiles.pl
index fd2bee5d20..95ae15f946 100644
--- a/src/bin/pg_rewind/t/003_extrafiles.pl
+++ b/src/bin/pg_rewind/t/003_extrafiles.pl
@@ -35,6 +35,9 @@ sub run_test
 	append_to_file
 	  "$test_primary_datadir/tst_both_dir/both_subdir/both_file3",
 	  "in both3";
+	append_to_file
+	  "$test_primary_datadir/tst_both_dir/.donotcopy",
+	  "in both4";
 
 	RewindTest::create_standby($test_mode);
 
@@ -82,6 +85,7 @@ sub run_test
 		\@paths,
 		[
 			"$test_primary_datadir/tst_both_dir",
+			"$test_primary_datadir/tst_both_dir/.donotcopy",
 			"$test_primary_datadir/tst_both_dir/both_file1",
 			"$test_primary_datadir/tst_both_dir/both_file2",
 			"$test_primary_datadir/tst_both_dir/both_subdir",