Horton

Thread

Commits

  1. Doc: improve directions for building on macOS.

  2. Further tweaking of PG_SYSROOT heuristics for macOS.

  1. [PATCH 1/1] Fix detection of pwritev support for OSX.

    James Hilliard <james.hilliard1@gmail.com> — 2021-01-19T11:16:25Z

    Fixes:
gcc -Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Werror=vla -Wendif-labels -Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv -Wno-unused-command-line-argument -O2 -I../../../../src/include  -isysroot /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk    -c -o fd.o fd.c
fd.c:3661:10: warning: 'pwritev' is only available on macOS 11.0 or newer [-Wunguarded-availability-new]
                part = pg_pwritev(fd, iov, iovcnt, offset);
                       ^~~~~~~~~~
../../../../src/include/port/pg_iovec.h:49:20: note: expanded from macro 'pg_pwritev'
                   ^~~~~~~
/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk/usr/include/sys/uio.h:104:9: note: 'pwritev' has been marked as being introduced in macOS 11.0
      here, but the deployment target is macOS 10.15.0
ssize_t pwritev(int, const struct iovec *, int, off_t) __DARWIN_NOCANCEL(pwritev) __API_AVAILABLE(macos(11.0), ios(14.0), watchos(7.0), tvos(14.0));
        ^
fd.c:3661:10: note: enclose 'pwritev' in a __builtin_available check to silence this warning
                part = pg_pwritev(fd, iov, iovcnt, offset);
                       ^~~~~~~~~~
../../../../src/include/port/pg_iovec.h:49:20: note: expanded from macro 'pg_pwritev'
                   ^~~~~~~
1 warning generated.

This results in a runtime error:
running bootstrap script ... dyld: lazy symbol binding failed: Symbol not found: _pwritev
  Referenced from: /usr/local/pgsql/bin/postgres
  Expected in: /usr/lib/libSystem.B.dylib

dyld: Symbol not found: _pwritev
  Referenced from: /usr/local/pgsql/bin/postgres
  Expected in: /usr/lib/libSystem.B.dylib

child process was terminated by signal 6: Abort trap: 6

To fix this we set -Werror=unguarded-availability-new so that a compile
test for pwritev will fail if the symbol is unavailable on the requested
SDK version.
---
 configure    | 88 ++++++++++++++++++++++++++++++++++++++++++++--------
 configure.ac | 19 +++++++++++-
 2 files changed, 93 insertions(+), 14 deletions(-)

diff --git a/configure b/configure
index 8af4b99021..503b0d27e6 100755
--- a/configure
+++ b/configure
@@ -5373,6 +5373,47 @@ if test x"$pgac_cv_prog_CC_cflags__Werror_vla" = x"yes"; then
 fi
 
 
+  # Prevent usage of symbols marked as newer than our target.
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ${CC} supports -Werror=unguarded-availability-new, for CFLAGS" >&5
+$as_echo_n "checking whether ${CC} supports -Werror=unguarded-availability-new, for CFLAGS... " >&6; }
+if ${pgac_cv_prog_CC_cflags__Werror_unguarded_availability_new+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  pgac_save_CFLAGS=$CFLAGS
+pgac_save_CC=$CC
+CC=${CC}
+CFLAGS="${CFLAGS} -Werror=unguarded-availability-new"
+ac_save_c_werror_flag=$ac_c_werror_flag
+ac_c_werror_flag=yes
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+  pgac_cv_prog_CC_cflags__Werror_unguarded_availability_new=yes
+else
+  pgac_cv_prog_CC_cflags__Werror_unguarded_availability_new=no
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+ac_c_werror_flag=$ac_save_c_werror_flag
+CFLAGS="$pgac_save_CFLAGS"
+CC="$pgac_save_CC"
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_prog_CC_cflags__Werror_unguarded_availability_new" >&5
+$as_echo "$pgac_cv_prog_CC_cflags__Werror_unguarded_availability_new" >&6; }
+if test x"$pgac_cv_prog_CC_cflags__Werror_unguarded_availability_new" = x"yes"; then
+  CFLAGS="${CFLAGS} -Werror=unguarded-availability-new"
+fi
+
+
   # -Wvla is not applicable for C++
 
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether ${CC} supports -Wendif-labels, for CFLAGS" >&5
@@ -15715,6 +15756,40 @@ $as_echo "#define HAVE_PS_STRINGS 1" >>confdefs.h
 
 fi
 
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for pwritev" >&5
+$as_echo_n "checking for pwritev... " >&6; }
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_UIO_H
+#include <sys/uio.h>
+#endif
+int
+main ()
+{
+struct iovec *iov;
+off_t offset;
+offset = 0;
+pwritev(0, iov, 0, offset);
+
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+$as_echo "#define HAVE_PWRITEV 1" >>confdefs.h
+
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+
 ac_fn_c_check_func "$LINENO" "dlopen" "ac_cv_func_dlopen"
 if test "x$ac_cv_func_dlopen" = xyes; then :
   $as_echo "#define HAVE_DLOPEN 1" >>confdefs.h
@@ -15871,19 +15946,6 @@ esac
 
 fi
 
-ac_fn_c_check_func "$LINENO" "pwritev" "ac_cv_func_pwritev"
-if test "x$ac_cv_func_pwritev" = xyes; then :
-  $as_echo "#define HAVE_PWRITEV 1" >>confdefs.h
-
-else
-  case " $LIBOBJS " in
-  *" pwritev.$ac_objext "* ) ;;
-  *) LIBOBJS="$LIBOBJS pwritev.$ac_objext"
- ;;
-esac
-
-fi
-
 ac_fn_c_check_func "$LINENO" "random" "ac_cv_func_random"
 if test "x$ac_cv_func_random" = xyes; then :
   $as_echo "#define HAVE_RANDOM 1" >>confdefs.h
diff --git a/configure.ac b/configure.ac
index 868a94c9ba..30fa39c859 100644
--- a/configure.ac
+++ b/configure.ac
@@ -494,6 +494,8 @@ if test "$GCC" = yes -a "$ICC" = no; then
   AC_SUBST(PERMIT_DECLARATION_AFTER_STATEMENT)
   # Really don't want VLAs to be used in our dialect of C
   PGAC_PROG_CC_CFLAGS_OPT([-Werror=vla])
+  # Prevent usage of symbols marked as newer than our target.
+  PGAC_PROG_CC_CFLAGS_OPT([-Werror=unguarded-availability-new])
   # -Wvla is not applicable for C++
   PGAC_PROG_CC_CFLAGS_OPT([-Wendif-labels])
   PGAC_PROG_CXX_CFLAGS_OPT([-Wendif-labels])
@@ -1726,6 +1728,22 @@ if test "$pgac_cv_var_PS_STRINGS" = yes ; then
   AC_DEFINE([HAVE_PS_STRINGS], 1, [Define to 1 if the PS_STRINGS thing exists.])
 fi
 
+AC_MSG_CHECKING([for pwritev])
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
+[#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_UIO_H
+#include <sys/uio.h>
+#endif],
+[struct iovec *iov;
+off_t offset;
+offset = 0;
+pwritev(0, iov, 0, offset);
+])], [AC_MSG_RESULT(yes)
+AC_DEFINE([HAVE_PWRITEV], 1, [Define to 1 if you have the `pwritev' function.])],
+[AC_MSG_RESULT(no)])
+
 AC_REPLACE_FUNCS(m4_normalize([
 	dlopen
 	explicit_bzero
@@ -1739,7 +1757,6 @@ AC_REPLACE_FUNCS(m4_normalize([
 	pread
 	preadv
 	pwrite
-	pwritev
 	random
 	srandom
 	strlcat
-- 
2.30.0

  2. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    Tom Lane <tgl@sss.pgh.pa.us> — 2021-01-19T15:27:21Z

    James Hilliard <james.hilliard1@gmail.com> writes:
> Fixes:
> gcc -Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Werror=vla -Wendif-labels -Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv -Wno-unused-command-line-argument -O2 -I../../../../src/include  -isysroot /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk    -c -o fd.o fd.c
> fd.c:3661:10: warning: 'pwritev' is only available on macOS 11.0 or newer [-Wunguarded-availability-new]

We already dealt with that by not selecting an SDK newer than the
underlying OS (see 4823621db).  I do not believe that your proposal
is more reliable than that approach, and it's surely uglier.  Are
we really going to abandon Autoconf's built-in checking method every
time Apple adds an API they should have had ten years ago?  If so,
you forgot preadv ...

			regards, tom lane

  3. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    James Hilliard <james.hilliard1@gmail.com> — 2021-01-19T15:36:49Z

    On Tue, Jan 19, 2021 at 8:27 AM Tom Lane <tgl@sss.pgh.pa.us> wrote:
>
> James Hilliard <james.hilliard1@gmail.com> writes:
> > Fixes:
> > gcc -Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Werror=vla -Wendif-labels -Wmissing-format-attribute -Wformat-security -fno-strict-aliasing -fwrapv -Wno-unused-command-line-argument -O2 -I../../../../src/include  -isysroot /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk    -c -o fd.o fd.c
> > fd.c:3661:10: warning: 'pwritev' is only available on macOS 11.0 or newer [-Wunguarded-availability-new]
>
> We already dealt with that by not selecting an SDK newer than the
> underlying OS (see 4823621db).
Tried that, doesn't work, not even sure how it could possibly fix this
issue at all,
this can not be fixed properly by selecting a specific SDK version
alone, it's the
symbols valid for a specific target deployment version that matters here.
> I do not believe that your proposal
> is more reliable than that approach, and it's surely uglier.  Are
> we really going to abandon Autoconf's built-in checking method every
> time Apple adds an API they should have had ten years ago?  If so,
> you forgot preadv ...
I didn't run into an issue there for some reason...but this was the cleanest fix
I could come up with that seemed to work.
>
>                         regards, tom lane

  4. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    Tom Lane <tgl@sss.pgh.pa.us> — 2021-01-19T15:57:25Z

    James Hilliard <james.hilliard1@gmail.com> writes:
> On Tue, Jan 19, 2021 at 8:27 AM Tom Lane <tgl@sss.pgh.pa.us> wrote:
>> We already dealt with that by not selecting an SDK newer than the
>> underlying OS (see 4823621db).

> Tried that, doesn't work, not even sure how it could possibly fix this
> issue at all,

It worked for me and for Sergey, so we need to figure out what's different
about your setup.  What do you get from "xcrun --show-sdk-path" and
"xcrun --sdk macosx --show-sdk-path"?  What have you got under
/Library/Developer/CommandLineTools/SDKs ?

> this can not be fixed properly by selecting a specific SDK version
> alone, it's the symbols valid for a specific target deployment version
> that matters here.

I don't think I believe that argument.  As a counterexample, supposing
that somebody were intentionally cross-compiling on an older OSX platform
but using a newer SDK, shouldn't they get an executable suited to the
SDK's target version?

(I realize that Apple thinks we ought to handle that through run-time
not compile-time adaptation, but I have no interest in going there.)

			regards, tom lane

  5. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    James Hilliard <james.hilliard1@gmail.com> — 2021-01-19T16:49:58Z

    On Tue, Jan 19, 2021 at 8:57 AM Tom Lane <tgl@sss.pgh.pa.us> wrote:
>
> James Hilliard <james.hilliard1@gmail.com> writes:
> > On Tue, Jan 19, 2021 at 8:27 AM Tom Lane <tgl@sss.pgh.pa.us> wrote:
> >> We already dealt with that by not selecting an SDK newer than the
> >> underlying OS (see 4823621db).
>
> > Tried that, doesn't work, not even sure how it could possibly fix this
> > issue at all,
>
> It worked for me and for Sergey, so we need to figure out what's different
> about your setup.  What do you get from "xcrun --show-sdk-path" and
> "xcrun --sdk macosx --show-sdk-path"?  What have you got under
> /Library/Developer/CommandLineTools/SDKs ?
$ xcrun --show-sdk-path
/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk
$ xcrun --sdk macosx --show-sdk-path
/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk
$ ls -laht /Library/Developer/CommandLineTools/SDKs
total 0
drwxr-xr-x  5 root  wheel   160B Jan 14  2020 .
drwxr-xr-x  8 root  wheel   256B Jan 14  2020 MacOSX10.15.sdk
drwxr-xr-x  7 root  wheel   224B Jan 14  2020 MacOSX10.14.sdk
lrwxr-xr-x  1 root  wheel    15B Jan 14  2020 MacOSX.sdk -> MacOSX10.15.sdk
>
> > this can not be fixed properly by selecting a specific SDK version
> > alone, it's the symbols valid for a specific target deployment version
> > that matters here.
>
> I don't think I believe that argument.  As a counterexample, supposing
> that somebody were intentionally cross-compiling on an older OSX platform
> but using a newer SDK, shouldn't they get an executable suited to the
> SDK's target version?
Yep, that's exactly what this should fix:

MACOSX_DEPLOYMENT_TARGET=11.0 ./configure
checking for pwritev... yes

Which fails at runtime on 10.15:
dyld: lazy symbol binding failed: Symbol not found: _pwritev
  Referenced from: /usr/local/pgsql/bin/postgres (which was built for
Mac OS X 11.0)
  Expected in: /usr/lib/libSystem.B.dylib

dyld: Symbol not found: _pwritev
  Referenced from: /usr/local/pgsql/bin/postgres (which was built for
Mac OS X 11.0)
  Expected in: /usr/lib/libSystem.B.dylib

child process was terminated by signal 6: Abort trap: 6

MACOSX_DEPLOYMENT_TARGET=10.15 ./configure
checking for pwritev... no

Noticed a couple small issues, I'll send a v2.
>
> (I realize that Apple thinks we ought to handle that through run-time
> not compile-time adaptation, but I have no interest in going there.)
>
>                         regards, tom lane

  6. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    Tom Lane <tgl@sss.pgh.pa.us> — 2021-01-19T17:17:16Z

    James Hilliard <james.hilliard1@gmail.com> writes:
> On Tue, Jan 19, 2021 at 8:57 AM Tom Lane <tgl@sss.pgh.pa.us> wrote:
>> It worked for me and for Sergey, so we need to figure out what's different
>> about your setup.  What do you get from "xcrun --show-sdk-path" and
>> "xcrun --sdk macosx --show-sdk-path"?  What have you got under
>> /Library/Developer/CommandLineTools/SDKs ?

> $ xcrun --show-sdk-path
> /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk
> $ xcrun --sdk macosx --show-sdk-path
> /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk
> $ ls -laht /Library/Developer/CommandLineTools/SDKs
> total 0
> drwxr-xr-x  5 root  wheel   160B Jan 14  2020 .
> drwxr-xr-x  8 root  wheel   256B Jan 14  2020 MacOSX10.15.sdk
> drwxr-xr-x  7 root  wheel   224B Jan 14  2020 MacOSX10.14.sdk
> lrwxr-xr-x  1 root  wheel    15B Jan 14  2020 MacOSX.sdk -> MacOSX10.15.sdk

Ah, got it.  So "xcrun --show-sdk-path" tells us the right thing (that
is, it *does* give us a symlink to a 10.15 SDK) but by refusing to
believe we've got the right thing, we end up picking MacOSX11.1.sdk.
Drat.  I suppose we could drop the heuristic about wanting a version
number in the SDK path, but I really don't want to do that.  Now I'm
thinking about trying to dereference the symlink after the first step.

BTW, it's curious that you get a reference to the MacOSX.sdk symlink
where both Sergey and I got references to the actual directory.
Do you happen to recall the order in which you installed/upgraded
Xcode and its command line tools?

>> I don't think I believe that argument.  As a counterexample, supposing
>> that somebody were intentionally cross-compiling on an older OSX platform
>> but using a newer SDK, shouldn't they get an executable suited to the
>> SDK's target version?

> Yep, that's exactly what this should fix:
> MACOSX_DEPLOYMENT_TARGET=11.0 ./configure
> checking for pwritev... yes
> Which fails at runtime on 10.15:

Well yeah, exactly.  It should fail at run-time, because you
cross-compiled an executable that's not built for the machine
you're on.  What we need is to prevent configure from setting up
a cross-compile situation by default.

			regards, tom lane

  7. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    James Hilliard <james.hilliard1@gmail.com> — 2021-01-19T17:42:07Z

    On Tue, Jan 19, 2021 at 10:17 AM Tom Lane <tgl@sss.pgh.pa.us> wrote:
>
> James Hilliard <james.hilliard1@gmail.com> writes:
> > On Tue, Jan 19, 2021 at 8:57 AM Tom Lane <tgl@sss.pgh.pa.us> wrote:
> >> It worked for me and for Sergey, so we need to figure out what's different
> >> about your setup.  What do you get from "xcrun --show-sdk-path" and
> >> "xcrun --sdk macosx --show-sdk-path"?  What have you got under
> >> /Library/Developer/CommandLineTools/SDKs ?
>
> > $ xcrun --show-sdk-path
> > /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk
> > $ xcrun --sdk macosx --show-sdk-path
> > /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk
> > $ ls -laht /Library/Developer/CommandLineTools/SDKs
> > total 0
> > drwxr-xr-x  5 root  wheel   160B Jan 14  2020 .
> > drwxr-xr-x  8 root  wheel   256B Jan 14  2020 MacOSX10.15.sdk
> > drwxr-xr-x  7 root  wheel   224B Jan 14  2020 MacOSX10.14.sdk
> > lrwxr-xr-x  1 root  wheel    15B Jan 14  2020 MacOSX.sdk -> MacOSX10.15.sdk
>
> Ah, got it.  So "xcrun --show-sdk-path" tells us the right thing (that
> is, it *does* give us a symlink to a 10.15 SDK) but by refusing to
> believe we've got the right thing, we end up picking MacOSX11.1.sdk.
> Drat.  I suppose we could drop the heuristic about wanting a version
> number in the SDK path, but I really don't want to do that.  Now I'm
> thinking about trying to dereference the symlink after the first step.
The MacOSX11.1.sdk can build for a 10.15 target just fine when passed
an appropriate MACOSX_DEPLOYMENT_TARGET, so that SDK should be
fine.
>
> BTW, it's curious that you get a reference to the MacOSX.sdk symlink
> where both Sergey and I got references to the actual directory.
> Do you happen to recall the order in which you installed/upgraded
> Xcode and its command line tools?
I generally just upgrade to the latest as it becomes available.
>
> >> I don't think I believe that argument.  As a counterexample, supposing
> >> that somebody were intentionally cross-compiling on an older OSX platform
> >> but using a newer SDK, shouldn't they get an executable suited to the
> >> SDK's target version?
>
> > Yep, that's exactly what this should fix:
> > MACOSX_DEPLOYMENT_TARGET=11.0 ./configure
> > checking for pwritev... yes
> > Which fails at runtime on 10.15:
>
> Well yeah, exactly.  It should fail at run-time, because you
> cross-compiled an executable that's not built for the machine
> you're on.  What we need is to prevent configure from setting up
> a cross-compile situation by default.
The toolchain already selects the correct deployment target by default, the
issue is just that the configure test for pwritev was being done in a way that
ignored the deployment target version, I fixed that.
>
>                         regards, tom lane

  8. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    Tom Lane <tgl@sss.pgh.pa.us> — 2021-01-19T20:54:38Z

    James Hilliard <james.hilliard1@gmail.com> writes:
> On Tue, Jan 19, 2021 at 10:17 AM Tom Lane <tgl@sss.pgh.pa.us> wrote:
>> Ah, got it.  So "xcrun --show-sdk-path" tells us the right thing (that
>> is, it *does* give us a symlink to a 10.15 SDK) but by refusing to
>> believe we've got the right thing, we end up picking MacOSX11.1.sdk.
>> Drat.  I suppose we could drop the heuristic about wanting a version
>> number in the SDK path, but I really don't want to do that.  Now I'm
>> thinking about trying to dereference the symlink after the first step.

> The MacOSX11.1.sdk can build for a 10.15 target just fine when passed
> an appropriate MACOSX_DEPLOYMENT_TARGET, so that SDK should be
> fine.

But our out-of-the-box default should be to build for the current
platform; we don't want users to have to set MACOSX_DEPLOYMENT_TARGET
for that case.  Besides, the problem we're having is exactly that Apple's
definition of "builds for a 10.15 target just fine" is different from
ours.  They think you should use a run-time test not a compile-time test
to discover whether preadv is available, and we don't want to do that.

In almost all of the cases I've seen so far, Apple's compiler actually
does default to using an SDK matching the platform.  The problem we
have is that we try to name the SDK explicitly, and the current
method is failing to pick the right one in your case.  There are
several reasons for using an explicit -isysroot rather than just
letting the compiler default:

* We have seen cases in which the compiler acts as though it has
*no* default sysroot, and we have to help it out.

* The explicit root reduces version-skew build hazards for extensions
that are not built at the same time as the core system.

* There are a few tests in configure itself that need to know the
sysroot path to check for files there.

Anyway, the behavior you're seeing shows that 4823621db is still a
bit shy of a load.  I'm thinking about the attached as a further
fix --- can you verify it helps for you?

			regards, tom lane

  9. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    James Hilliard <james.hilliard1@gmail.com> — 2021-01-19T22:47:50Z

    On Tue, Jan 19, 2021 at 1:54 PM Tom Lane <tgl@sss.pgh.pa.us> wrote:
>
> James Hilliard <james.hilliard1@gmail.com> writes:
> > On Tue, Jan 19, 2021 at 10:17 AM Tom Lane <tgl@sss.pgh.pa.us> wrote:
> >> Ah, got it.  So "xcrun --show-sdk-path" tells us the right thing (that
> >> is, it *does* give us a symlink to a 10.15 SDK) but by refusing to
> >> believe we've got the right thing, we end up picking MacOSX11.1.sdk.
> >> Drat.  I suppose we could drop the heuristic about wanting a version
> >> number in the SDK path, but I really don't want to do that.  Now I'm
> >> thinking about trying to dereference the symlink after the first step.
>
> > The MacOSX11.1.sdk can build for a 10.15 target just fine when passed
> > an appropriate MACOSX_DEPLOYMENT_TARGET, so that SDK should be
> > fine.
>
> But our out-of-the-box default should be to build for the current
> platform; we don't want users to have to set MACOSX_DEPLOYMENT_TARGET
> for that case.  Besides, the problem we're having is exactly that Apple's
> definition of "builds for a 10.15 target just fine" is different from
> ours.  They think you should use a run-time test not a compile-time test
> to discover whether preadv is available, and we don't want to do that.
The default for MACOSX_DEPLOYMENT_TARGET is always the current
running OS version from my understanding. So if I build with MacOSX11.1.sdk
on 10.15 with default settings the binaries will work fine because the
MACOSX_DEPLOYMENT_TARGET gets set to 10.15 automatically even
if the same SDK is capable of producing incompatible binaries if you set
MACOSX_DEPLOYMENT_TARGET to 11.0.
>
> In almost all of the cases I've seen so far, Apple's compiler actually
> does default to using an SDK matching the platform.  The problem we
> have is that we try to name the SDK explicitly, and the current
> method is failing to pick the right one in your case.  There are
> several reasons for using an explicit -isysroot rather than just
> letting the compiler default:
No, it's only the MACOSX_DEPLOYMENT_TARGET that matches the
platform, SDK can be arbitrary more or less, but it will work fine because
the autoselected MACOSX_DEPLOYMENT_TARGET will force compatibility
no matter what SDK version you use. This is always how it has worked
from what I've seen.
>
> * We have seen cases in which the compiler acts as though it has
> *no* default sysroot, and we have to help it out.
>
> * The explicit root reduces version-skew build hazards for extensions
> that are not built at the same time as the core system.
The deployment target is effectively entirely separate from SDK version,
so it really shouldn't make a difference unless the SDK is significantly
older or newer than the running version from what I can tell.
>
> * There are a few tests in configure itself that need to know the
> sysroot path to check for files there.
>
> Anyway, the behavior you're seeing shows that 4823621db is still a
> bit shy of a load.  I'm thinking about the attached as a further
> fix --- can you verify it helps for you?
Best I can tell it provides no change for me(this patch is tested on top of it)
because it does not provide any MACOSX_DEPLOYMENT_TARGET
based feature detection for pwritev at all.
>
>                         regards, tom lane
>

  10. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    James Hilliard <james.hilliard1@gmail.com> — 2021-01-19T23:07:26Z

    On Tue, Jan 19, 2021 at 3:47 PM James Hilliard
<james.hilliard1@gmail.com> wrote:
>
> On Tue, Jan 19, 2021 at 1:54 PM Tom Lane <tgl@sss.pgh.pa.us> wrote:
> >
> > James Hilliard <james.hilliard1@gmail.com> writes:
> > > On Tue, Jan 19, 2021 at 10:17 AM Tom Lane <tgl@sss.pgh.pa.us> wrote:
> > >> Ah, got it.  So "xcrun --show-sdk-path" tells us the right thing (that
> > >> is, it *does* give us a symlink to a 10.15 SDK) but by refusing to
> > >> believe we've got the right thing, we end up picking MacOSX11.1.sdk.
> > >> Drat.  I suppose we could drop the heuristic about wanting a version
> > >> number in the SDK path, but I really don't want to do that.  Now I'm
> > >> thinking about trying to dereference the symlink after the first step.
> >
> > > The MacOSX11.1.sdk can build for a 10.15 target just fine when passed
> > > an appropriate MACOSX_DEPLOYMENT_TARGET, so that SDK should be
> > > fine.
> >
> > But our out-of-the-box default should be to build for the current
> > platform; we don't want users to have to set MACOSX_DEPLOYMENT_TARGET
> > for that case.  Besides, the problem we're having is exactly that Apple's
> > definition of "builds for a 10.15 target just fine" is different from
> > ours.  They think you should use a run-time test not a compile-time test
> > to discover whether preadv is available, and we don't want to do that.
> The default for MACOSX_DEPLOYMENT_TARGET is always the current
> running OS version from my understanding. So if I build with MacOSX11.1.sdk
> on 10.15 with default settings the binaries will work fine because the
> MACOSX_DEPLOYMENT_TARGET gets set to 10.15 automatically even
> if the same SDK is capable of producing incompatible binaries if you set
> MACOSX_DEPLOYMENT_TARGET to 11.0.
> >
> > In almost all of the cases I've seen so far, Apple's compiler actually
> > does default to using an SDK matching the platform.  The problem we
> > have is that we try to name the SDK explicitly, and the current
> > method is failing to pick the right one in your case.  There are
> > several reasons for using an explicit -isysroot rather than just
> > letting the compiler default:
> No, it's only the MACOSX_DEPLOYMENT_TARGET that matches the
> platform, SDK can be arbitrary more or less, but it will work fine because
> the autoselected MACOSX_DEPLOYMENT_TARGET will force compatibility
> no matter what SDK version you use. This is always how it has worked
> from what I've seen.
> >
> > * We have seen cases in which the compiler acts as though it has
> > *no* default sysroot, and we have to help it out.
> >
> > * The explicit root reduces version-skew build hazards for extensions
> > that are not built at the same time as the core system.
> The deployment target is effectively entirely separate from SDK version,
> so it really shouldn't make a difference unless the SDK is significantly
> older or newer than the running version from what I can tell.
> >
> > * There are a few tests in configure itself that need to know the
> > sysroot path to check for files there.
> >
> > Anyway, the behavior you're seeing shows that 4823621db is still a
> > bit shy of a load.  I'm thinking about the attached as a further
> > fix --- can you verify it helps for you?
> Best I can tell it provides no change for me(this patch is tested on top of it)
> because it does not provide any MACOSX_DEPLOYMENT_TARGET
> based feature detection for pwritev at all.
Actually, this looks path looks wrong in general, the value for
"xcrun --sdk macosx --show-sdk-path" should take precedence over
"xcrun --show-sdk-path" as the latter may be used for IOS potentially.
On my system "xcodebuild -version -sdk macosx Path" and
"xcrun --sdk macosx --show-sdk-path" both point to the
correct latest MacOSX11.1.sdk SDK while "xcrun --show-sdk-path"
points to the older one.
> >
> >                         regards, tom lane
> >

  11. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    Tom Lane <tgl@sss.pgh.pa.us> — 2021-01-20T01:37:08Z

    James Hilliard <james.hilliard1@gmail.com> writes:
> Actually, this looks path looks wrong in general, the value for
> "xcrun --sdk macosx --show-sdk-path" should take precedence over
> "xcrun --show-sdk-path" as the latter may be used for IOS potentially.

What is "potentially"?  I've found no direct means to control the
SDK path at all, but so far it appears that "xcrun --show-sdk-path"
agrees with the compiler's default -isysroot path as seen in the
compiler's -v output.  I suspect that this isn't coincidental,
but reflects xcrun actually being used in the compiler launch
process.  If it were to flip over to using a IOS SDK, that would
mean that bare "cc" would generate nonfunctional executables,
which just about any onlooker would agree is broken.

I'm really not excited about trying to make the build work with
a non-native SDK as you are proposing.  I think that's just going
to lead to a continuing stream of problems, because of Apple's
opinions about how cross-version compatibility should work.
It also seems like unnecessary complexity, because there is always
(AFAICS) a native SDK version available.  We just need to find it.

			regards, tom lane

  12. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    James Hilliard <james.hilliard1@gmail.com> — 2021-01-20T21:52:19Z

    On Tue, Jan 19, 2021 at 6:37 PM Tom Lane <tgl@sss.pgh.pa.us> wrote:
>
> James Hilliard <james.hilliard1@gmail.com> writes:
> > Actually, this looks path looks wrong in general, the value for
> > "xcrun --sdk macosx --show-sdk-path" should take precedence over
> > "xcrun --show-sdk-path" as the latter may be used for IOS potentially.
>
> What is "potentially"?

Well I'm not sure the SDK parameter always defaults to macos although
I guess it probably does as I couldn't figure out a way to change it:
$ xcodebuild -showsdks
iOS SDKs:
    iOS 14.3                          -sdk iphoneos14.3
iOS Simulator SDKs:
    Simulator - iOS 14.3              -sdk iphonesimulator14.3
macOS SDKs:
    DriverKit 20.2                    -sdk driverkit.macosx20.2
    macOS 11.1                        -sdk macosx11.1
tvOS SDKs:
    tvOS 14.3                         -sdk appletvos14.3
tvOS Simulator SDKs:
    Simulator - tvOS 14.3             -sdk appletvsimulator14.3
watchOS SDKs:
    watchOS 7.2                       -sdk watchos7.2
watchOS Simulator SDKs:
    Simulator - watchOS 7.2           -sdk watchsimulator7.2

> I've found no direct means to control the
> SDK path at all, but so far it appears that "xcrun --show-sdk-path"
> agrees with the compiler's default -isysroot path as seen in the
> compiler's -v output.  I suspect that this isn't coincidental,
> but reflects xcrun actually being used in the compiler launch
> process.  If it were to flip over to using a IOS SDK, that would
> mean that bare "cc" would generate nonfunctional executables,
> which just about any onlooker would agree is broken.

So there's some more weirdness involved here, whether or not you
have the command line install seems to affect the output of the
"xcrun --show-sdk-path" command, but not the
"xcrun --sdk macosx --show-sdk-path" command.

This is what I get without the command line tools:
$ xcrun --show-sdk-path
/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk
$ xcrun --sdk macosx --show-sdk-path
/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk
this last one is just a symlink to the other path.

With command line tools this is different however:
$ xcrun --show-sdk-path
/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk
$ xcrun --sdk macosx --show-sdk-path
/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk

Note that the /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk
is different from the normal SDK and doesn't seem to be able to generate
binaries that target a 11.0 deployment target on my 10.15 system, however
I am unsure if this behavior can be relied upon.

So in terms of what works best, the newer normal SDK has the most flexibility
as it can produce both 10.15 target binaries and 11.0 target binaries
depending on the MACOSX_DEPLOYMENT_TARGET while the command
line tools SDK can only produce 10.15 target binaries it would appear.

Note that with my patch the binaries will always be compatible with the
host system by default, even if the SDK is capable of producing binaries
that are incompatible so building postgres works with and without the
command line tools SDK.

So I think "xcrun --sdk macosx --show-sdk-path" is probably preferable
but either should work as long as we can properly detect deployment
target symbol availability, regardless this SDK sysroot selection issue is
effectively an entirely different issue from the feature detection not properly
respecting the configured deployment target.

>
> I'm really not excited about trying to make the build work with
> a non-native SDK as you are proposing.  I think that's just going
> to lead to a continuing stream of problems, because of Apple's
> opinions about how cross-version compatibility should work.

Well the minimum required target version is pretty much strictly based on
MACOSX_DEPLOYMENT_TARGET so our feature detection still needs
to use that, otherwise cross target compilation for newer or older targets will
not work correctly.

From my understanding the reason AC_REPLACE_FUNCS does not
throw an error for deployment target incompatible functions is that it only
checks if the function exists and not if it is actually useable, this is
why I had to add an explicit AC_LANG_PROGRAM compile test to
properly trigger a compile failure if the function is not usable for a
particular deployment target version, merely checking if the function
exists in the header is not sufficient.

> It also seems like unnecessary complexity, because there is always
> (AFAICS) a native SDK version available.  We just need to find it.

Best I can tell this is not true, it is some(most?) of the time but
it's not something
we can rely upon as systems may only contain a newer SDK, but this newer SDK
is still capable of producing binaries that can run on the build host system so
this shouldn't be an issue as long as we can do target feature
detection properly.

>
>                         regards, tom lane

  13. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    Tom Lane <tgl@sss.pgh.pa.us> — 2021-01-20T23:07:58Z

    James Hilliard <james.hilliard1@gmail.com> writes:
> On Tue, Jan 19, 2021 at 6:37 PM Tom Lane <tgl@sss.pgh.pa.us> wrote:
>> I've found no direct means to control the
>> SDK path at all, but so far it appears that "xcrun --show-sdk-path"
>> agrees with the compiler's default -isysroot path as seen in the
>> compiler's -v output.  I suspect that this isn't coincidental,
>> but reflects xcrun actually being used in the compiler launch
>> process.  If it were to flip over to using a IOS SDK, that would
>> mean that bare "cc" would generate nonfunctional executables,
>> which just about any onlooker would agree is broken.

> So there's some more weirdness involved here, whether or not you
> have the command line install seems to affect the output of the
> "xcrun --show-sdk-path" command, but not the
> "xcrun --sdk macosx --show-sdk-path" command.

Yeah, that's what we discovered in the other thread.  It seems that
with "--sdk macosx" you'll always get a pointer to the (solitary)
SDK under /Applications/Xcode.app, but with the short "xcrun
--show-sdk-path" command you might get either that or a pointer to
something under /Library/Developer/CommandLineTools.

I now believe what is actually happening with the short command is
that it's iterating through the available SDKs (according to some not
very clear search path) and picking the first one it finds that
matches the host system version.  That matches the ktrace evidence
that shows it reading the SDKSettings.plist file in each SDK
directory.  The fact that it can seize on either an actual directory
or an equivalent symlink might be due to chance ordering of directory
entries.  (It'd be interesting to see "ls -f" output for your
/Library/Developer/CommandLineTools/SDKs directory ... though if
you've been experimenting with deinstall/reinstall, there's no
reason to suppose the entry order is still the same.)

I'm not sure that the case of not having the "command line tools"
installed is interesting for our purposes.  AFAIK you have to have
that in order to have access to required tools like bison and gmake.
(That reminds me, I was intending to add something to our docs
about how-to-build-from-source to say that you need to install those.)

> Note that with my patch the binaries will always be compatible with the
> host system by default, even if the SDK is capable of producing binaries
> that are incompatible so building postgres works with and without the
> command line tools SDK.

Yeah.  I don't see that as a benefit actually.  Adding the
-no_weak_imports linker switch (or the other one you're suggesting)
means that you *cannot* cross-compile for a newer macOS version,
even if you set PG_SYSROOT and/or MACOSX_DEPLOYMENT_TARGET with the
intention of doing so.  You'll still get a build that reflects the set
of kernel calls available on the host system.  Admittedly, this is a
case that's not likely to be of interest to very many people, but
I don't see why a method with that restriction is superior to picking
a default SDK that matches the host system (and can be overridden).

> So I think "xcrun --sdk macosx --show-sdk-path" is probably preferable
> but either should work as long as we can properly detect deployment
> target symbol availability, regardless this SDK sysroot selection issue is
> effectively an entirely different issue from the feature detection not properly
> respecting the configured deployment target.

No, I think it's pretty much equivalent.  If we pick the right SDK
then we'll get the build we want.

			regards, tom lane

  14. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    James Hilliard <james.hilliard1@gmail.com> — 2021-01-20T23:49:50Z

    On Wed, Jan 20, 2021 at 4:07 PM Tom Lane <tgl@sss.pgh.pa.us> wrote:
>
> James Hilliard <james.hilliard1@gmail.com> writes:
> > On Tue, Jan 19, 2021 at 6:37 PM Tom Lane <tgl@sss.pgh.pa.us> wrote:
> >> I've found no direct means to control the
> >> SDK path at all, but so far it appears that "xcrun --show-sdk-path"
> >> agrees with the compiler's default -isysroot path as seen in the
> >> compiler's -v output.  I suspect that this isn't coincidental,
> >> but reflects xcrun actually being used in the compiler launch
> >> process.  If it were to flip over to using a IOS SDK, that would
> >> mean that bare "cc" would generate nonfunctional executables,
> >> which just about any onlooker would agree is broken.
>
> > So there's some more weirdness involved here, whether or not you
> > have the command line install seems to affect the output of the
> > "xcrun --show-sdk-path" command, but not the
> > "xcrun --sdk macosx --show-sdk-path" command.
>
> Yeah, that's what we discovered in the other thread.  It seems that
> with "--sdk macosx" you'll always get a pointer to the (solitary)
> SDK under /Applications/Xcode.app, but with the short "xcrun
> --show-sdk-path" command you might get either that or a pointer to
> something under /Library/Developer/CommandLineTools.
>
> I now believe what is actually happening with the short command is
> that it's iterating through the available SDKs (according to some not
> very clear search path) and picking the first one it finds that
> matches the host system version.  That matches the ktrace evidence
> that shows it reading the SDKSettings.plist file in each SDK
> directory.  The fact that it can seize on either an actual directory
> or an equivalent symlink might be due to chance ordering of directory
> entries.  (It'd be interesting to see "ls -f" output for your
> /Library/Developer/CommandLineTools/SDKs directory ... though if

Well at the moment I completely deleted that directory...and the build
works fine with my patch still.

> you've been experimenting with deinstall/reinstall, there's no
> reason to suppose the entry order is still the same.)
>
> I'm not sure that the case of not having the "command line tools"
> installed is interesting for our purposes.  AFAIK you have to have
> that in order to have access to required tools like bison and gmake.
> (That reminds me, I was intending to add something to our docs
> about how-to-build-from-source to say that you need to install those.)

Yeah, not 100% sure but I was able to build just fine after deleting my
command line tools. I think it just switched to using the normal SDK
toolchain, I guess that's the fallback logic doing that.

It would be pretty annoying to have to install an outdated SDK just to
build postgres for no other reason than the autoconf feature detection
being broken.

>
> > Note that with my patch the binaries will always be compatible with the
> > host system by default, even if the SDK is capable of producing binaries
> > that are incompatible so building postgres works with and without the
> > command line tools SDK.
>
> Yeah.  I don't see that as a benefit actually.  Adding the
> -no_weak_imports linker switch (or the other one you're suggesting)
> means that you *cannot* cross-compile for a newer macOS version,
> even if you set PG_SYSROOT and/or MACOSX_DEPLOYMENT_TARGET with the
> intention of doing so.

Best I can tell this isn't true, I was able to cross compile for a newer
MACOSX_DEPLOYMENT_TARGET than my build host just fine. The
binary fails with a "Symbol not found: _pwritev" error when I try
to run it on the system that built it.

In regards to the -no_weak_imports switch...that is something different
from my understanding as it just strips the weak imports forcing the
fallback code paths to be taken instead, essentially functioning as if
the weak symbols are never available. It's largely separate from the
deployment target from my understanding as weak symbols are feature
that lets you use newer syscalls while still providing backwards
compatible fallbacks for older systems.

> You'll still get a build that reflects the set
> of kernel calls available on the host system.  Admittedly, this is a
> case that's not likely to be of interest to very many people, but
> I don't see why a method with that restriction is superior to picking
> a default SDK that matches the host system (and can be overridden).

But to fix the build when using a newer SDK overriding the SDK location
does not help, you would have to override the broken feature detection.

>
> > So I think "xcrun --sdk macosx --show-sdk-path" is probably preferable
> > but either should work as long as we can properly detect deployment
> > target symbol availability, regardless this SDK sysroot selection issue is
> > effectively an entirely different issue from the feature detection not properly
> > respecting the configured deployment target.
>
> No, I think it's pretty much equivalent.  If we pick the right SDK
> then we'll get the build we want.

Generally any recent SDK installed should work as long as the feature detection
in autoconf isn't broken. I'm not really sure what's the most correct option in
regards to picking a SDK version, however the feature detection should be
fixed regardless IMO.

>
>                         regards, tom lane

  15. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    Sergey Shinderuk <s.shinderuk@postgrespro.ru> — 2021-01-21T08:39:48Z

    On 21.01.2021 02:07, Tom Lane wrote:
> I now believe what is actually happening with the short command is
> that it's iterating through the available SDKs (according to some not
> very clear search path) and picking the first one it finds that
> matches the host system version.  That matches the ktrace evidence
> that shows it reading the SDKSettings.plist file in each SDK
> directory.

Yes, you are right. After some more digging...

It searches the DEVELOPER_DIR first and then 
/Library/Developer/CommandLineTools, which is hardcoded.

My DEVELOPER_DIR is
% xcode-select -p
/Applications/Xcode.app/Contents/Developer

(For more detail try "otool -tV /usr/lib/libxcselect.dylib -p 
_xcselect_get_developer_dir_path".)

It reads ProductVersion from 
/System/Library/CoreServices/SystemVersion.plist

% plutil -p /System/Library/CoreServices/SystemVersion.plist | grep 
ProductVersion
   "ProductVersion" => "10.15.7"

Strips anything after the second dot, and prepends "macosx" to it, which 
gives "macosx10.15".

Then it scans through SDK dirs looking up CanonicalName from 
SDKSettings.plist until it finds a match with "macosx10.15".


The overall callstack:

% sudo dtrace -n 'syscall::getdirentries64:entry { ustack() }' -c 'xcrun 
--show-sdk-path'
dtrace: description 'syscall::getdirentries64:entry ' matched 1 probe
/Library/Developer/CommandLineTools/SDKs/MacOSX10.15.sdk
dtrace: pid 20183 has exited
CPU     ID                    FUNCTION:NAME
   0    846            getdirentries64:entry
               libsystem_kernel.dylib`__getdirentries64+0xa
               libsystem_c.dylib`readdir$INODE64+0x23
               libsystem_c.dylib`scandir$INODE64+0x6c
               libxcrun.dylib`cltools_lookup_sdk_by_key+0x5f
               libxcrun.dylib`cltools_lookup_boot_system_sdk+0xda
               libxcrun.dylib`xcinfocache_resolve_sdkroot+0xc0
               libxcrun.dylib`xcrun_main2+0x57a
               libxcrun.dylib`xcrun_main+0x9
               libxcselect.dylib`xcselect_invoke_xcrun_via_library+0xc8
               libxcselect.dylib`xcselect_invoke_xcrun+0x25a
               xcrun`DYLD-STUB$$getprogname
               libdyld.dylib`start+0x1
               xcrun`0x2

   0    846            getdirentries64:entry
               libsystem_kernel.dylib`__getdirentries64+0xa
               libsystem_c.dylib`readdir$INODE64+0x23
               libsystem_c.dylib`scandir$INODE64+0x6c
               libxcrun.dylib`cltools_lookup_sdk_by_key+0x5f
               libxcrun.dylib`cltools_lookup_boot_system_sdk+0xf3
               libxcrun.dylib`xcinfocache_resolve_sdkroot+0xc0
               libxcrun.dylib`xcrun_main2+0x57a
               libxcrun.dylib`xcrun_main+0x9
               libxcselect.dylib`xcselect_invoke_xcrun_via_library+0xc8
               libxcselect.dylib`xcselect_invoke_xcrun+0x25a
               xcrun`DYLD-STUB$$getprogname
               libdyld.dylib`start+0x1
               xcrun`0x2


The SDK search path:

% sudo dtrace -n 'pid$target:::entry 
/probefunc=="cltools_lookup_sdk_by_key"/ { trace(copyinstr(arg0)); 
trace(copyinstr(arg1)) }' -c 'xcrun --show-sdk-path'
dtrace: description 'pid$target:::entry ' matched 17293 probes
/Library/Developer/CommandLineTools/SDKs/MacOSX10.15.sdk
dtrace: pid 20191 has exited
CPU     ID                    FUNCTION:NAME
   8 398290  cltools_lookup_sdk_by_key:entry 
/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer 
  macosx10.15
   9 398290  cltools_lookup_sdk_by_key:entry 
/Library/Developer/CommandLineTools  macosx10.15


The properties read from SDKSettings.plist:

% sudo dtrace -n 'pid$target:::entry 
/probefunc=="_cltools_lookup_property_in_path"/ { 
trace(copyinstr(arg0)); trace(copyinstr(arg1)); trace(copyinstr(arg2)) 
}' -c 'xcrun --show-sdk-path'
dtrace: description 'pid$target:::entry ' matched 17293 probes
/Library/Developer/CommandLineTools/SDKs/MacOSX10.15.sdk
dtrace: pid 20195 has exited
CPU     ID                    FUNCTION:NAME
   8 398288 _cltools_lookup_property_in_path:entry   / 
                 System/Library/CoreServices/SystemVersion.plist 
ProductVersion
   8 398288 _cltools_lookup_property_in_path:entry 
/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/DriverKit20.2.sdk 
  SDKSettings.plist                  IsBaseSDK
   8 398288 _cltools_lookup_property_in_path:entry 
/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/DriverKit20.2.sdk 
  SDKSettings.plist                  CanonicalName
   4 398288 _cltools_lookup_property_in_path:entry 
/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/DriverKit20.2.sdk 
  SDKSettings.plist                  CanonicalNameForBuildSettings
   4 398288 _cltools_lookup_property_in_path:entry 
/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk 
  SDKSettings.plist                  IsBaseSDK
   4 398288 _cltools_lookup_property_in_path:entry 
/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk 
  SDKSettings.plist                  CanonicalName
   4 398288 _cltools_lookup_property_in_path:entry 
/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk 
  SDKSettings.plist                  CanonicalNameForBuildSettings
   4 398288 _cltools_lookup_property_in_path:entry 
/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk 
  SDKSettings.plist                  PLATFORM_NAME
   4 398288 _cltools_lookup_property_in_path:entry 
/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk 
  SDKSettings.plist                  IsBaseSDK
   2 398288 _cltools_lookup_property_in_path:entry 
/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk 
  SDKSettings.plist                  CanonicalName
   2 398288 _cltools_lookup_property_in_path:entry 
/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk 
  SDKSettings.plist                  CanonicalNameForBuildSettings
   2 398288 _cltools_lookup_property_in_path:entry 
/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX11.1.sdk 
  SDKSettings.plist                  PLATFORM_NAME
   2 398288 _cltools_lookup_property_in_path:entry 
/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk  SDKSettings.plist 
                 IsBaseSDK
   2 398288 _cltools_lookup_property_in_path:entry 
/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk  SDKSettings.plist 
                 CanonicalName
   2 398288 _cltools_lookup_property_in_path:entry 
/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk  SDKSettings.plist 
                 CanonicalNameForBuildSettings
   0 398288 _cltools_lookup_property_in_path:entry 
/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk  SDKSettings.plist 
                 PLATFORM_NAME
   0 398288 _cltools_lookup_property_in_path:entry 
/Library/Developer/CommandLineTools/SDKs/MacOSX10.15.sdk 
SDKSettings.plist                  IsBaseSDK
   0 398288 _cltools_lookup_property_in_path:entry 
/Library/Developer/CommandLineTools/SDKs/MacOSX10.15.sdk 
SDKSettings.plist                  CanonicalName


BTW, on my machine /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk 
is skipped because it points to 11.0:

% ls -l /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk
lrwxr-xr-x  1 root  wheel  14 Nov 17 02:21 
/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk -> MacOSX11.0.sdk

For more detail try
% otool -tV 
/Applications/Xcode.app/Contents/Developer/usr/lib/libxcrun.dylib -p 
_cltools_lookup_boot_system_sdk

  16. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    Tom Lane <tgl@sss.pgh.pa.us> — 2021-01-21T18:38:44Z

    James Hilliard <james.hilliard1@gmail.com> writes:
> On Wed, Jan 20, 2021 at 4:07 PM Tom Lane <tgl@sss.pgh.pa.us> wrote:
>> I'm not sure that the case of not having the "command line tools"
>> installed is interesting for our purposes.  AFAIK you have to have
>> that in order to have access to required tools like bison and gmake.
>> (That reminds me, I was intending to add something to our docs
>> about how-to-build-from-source to say that you need to install those.)

> Yeah, not 100% sure but I was able to build just fine after deleting my
> command line tools.

Hm.  I've never been totally clear on what's included in the "command line
tools", although it's now apparent that one thing that gets installed is
an SDK matching the host OS version.  However, Apple's description at [1]
says

    Command Line Tools

    Download the macOS SDK, headers, and build tools such as the Apple
    LLVM compiler and Make. These tools make it easy to install open
    source software or develop on UNIX within Terminal. macOS can
    automatically download these tools the first time you try to build
    software, and they are available on the downloads page.

which certainly strongly implies that gmake is not there otherwise.
At this point I lack any "bare" macOS system to check it on.  I wonder
whether you have a copy of make available from MacPorts or Homebrew.
Or maybe uninstalling the command line tools doesn't really remove
everything?

> It would be pretty annoying to have to install an outdated SDK just to
> build postgres for no other reason than the autoconf feature detection
> being broken.

It's only as "outdated" as your host system ;-).  Besides, it doesn't
look like Apple's really giving you a choice not to.

The long and short of this is that I'm unwilling to buy into maintaining
our own substitutes for standard autoconf probes in order to make it
possible to use the wrong SDK version.  The preadv/pwritev case is already
messy enough, and I fear that trying to support such scenarios is going to
lead to more and more pain in the future.

			regards, tom lane

[1] https://developer.apple.com/xcode/features/

  17. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    James Hilliard <james.hilliard1@gmail.com> — 2021-01-21T22:17:29Z

    On Thu, Jan 21, 2021 at 11:38 AM Tom Lane <tgl@sss.pgh.pa.us> wrote:
>
> James Hilliard <james.hilliard1@gmail.com> writes:
> > On Wed, Jan 20, 2021 at 4:07 PM Tom Lane <tgl@sss.pgh.pa.us> wrote:
> >> I'm not sure that the case of not having the "command line tools"
> >> installed is interesting for our purposes.  AFAIK you have to have
> >> that in order to have access to required tools like bison and gmake.
> >> (That reminds me, I was intending to add something to our docs
> >> about how-to-build-from-source to say that you need to install those.)
>
> > Yeah, not 100% sure but I was able to build just fine after deleting my
> > command line tools.
>
> Hm.  I've never been totally clear on what's included in the "command line
> tools", although it's now apparent that one thing that gets installed is
> an SDK matching the host OS version.  However, Apple's description at [1]
> says
>
>     Command Line Tools
>
>     Download the macOS SDK, headers, and build tools such as the Apple
>     LLVM compiler and Make. These tools make it easy to install open
>     source software or develop on UNIX within Terminal. macOS can
>     automatically download these tools the first time you try to build
>     software, and they are available on the downloads page.
>
> which certainly strongly implies that gmake is not there otherwise.
> At this point I lack any "bare" macOS system to check it on.  I wonder
> whether you have a copy of make available from MacPorts or Homebrew.
> Or maybe uninstalling the command line tools doesn't really remove
> everything?
Yeah, not entirely sure there but I do use homebrew.
>
> > It would be pretty annoying to have to install an outdated SDK just to
> > build postgres for no other reason than the autoconf feature detection
> > being broken.
>
> It's only as "outdated" as your host system ;-).  Besides, it doesn't
> look like Apple's really giving you a choice not to.
The newer SDK should work fine as long as long as the autoconf feature
detection is fixed somehow.
>
> The long and short of this is that I'm unwilling to buy into maintaining
> our own substitutes for standard autoconf probes in order to make it
> possible to use the wrong SDK version.  The preadv/pwritev case is already
> messy enough, and I fear that trying to support such scenarios is going to
> lead to more and more pain in the future.
Well it's actually a larger issue, if it isn't fixed then the ability
to change the
MACOSX_DEPLOYMENT_TARGET doesn't work properly, not only for
the case of having a newer SDK on an older host but it would also prevent
MACOSX_DEPLOYMENT_TARGET from working in general such as for
building with support for older targets from newer hosts, I'll see if there's
maybe a better way to fix the feature detection that's less of a maintenance
issue.
>
>                         regards, tom lane
>
> [1] https://developer.apple.com/xcode/features/

  18. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    Sergey Shinderuk <s.shinderuk@postgrespro.ru> — 2021-01-21T23:32:46Z

    On 22.01.2021 01:17, James Hilliard wrote:
> On Thu, Jan 21, 2021 at 11:38 AM Tom Lane <tgl@sss.pgh.pa.us> wrote:
>>
>> James Hilliard <james.hilliard1@gmail.com> writes:
>>> On Wed, Jan 20, 2021 at 4:07 PM Tom Lane <tgl@sss.pgh.pa.us> wrote:
>>>> I'm not sure that the case of not having the "command line tools"
>>>> installed is interesting for our purposes.  AFAIK you have to have
>>>> that in order to have access to required tools like bison and gmake.
>>>> (That reminds me, I was intending to add something to our docs
>>>> about how-to-build-from-source to say that you need to install those.)
>>
>>> Yeah, not 100% sure but I was able to build just fine after deleting my
>>> command line tools.
>>
>> Hm.  I've never been totally clear on what's included in the "command line
>> tools", although it's now apparent that one thing that gets installed is
>> an SDK matching the host OS version.  However, Apple's description at [1]
>> says
>>
>>      Command Line Tools
>>
>>      Download the macOS SDK, headers, and build tools such as the Apple
>>      LLVM compiler and Make. These tools make it easy to install open
>>      source software or develop on UNIX within Terminal. macOS can
>>      automatically download these tools the first time you try to build
>>      software, and they are available on the downloads page.
>>
>> which certainly strongly implies that gmake is not there otherwise.
>> At this point I lack any "bare" macOS system to check it on.  I wonder
>> whether you have a copy of make available from MacPorts or Homebrew.
>> Or maybe uninstalling the command line tools doesn't really remove
>> everything?
> Yeah, not entirely sure there but I do use homebrew.


FWIW, I tested with a clean install of Catalina. Before I install 
anything at all, I already have xcode-select, xcrun and all the shims in 
/usr/bin for developer tools, including cc, make, git, xcodebuild... 
Just about everything listed in the FILES section of "man xcode-select".

When I run any tool (except xcode-select), a GUI dialog pops up offering 
to install the Command Line Tools. So apparently those shims are not 
functional yet. I rejected the installation.

Instead I downloaded Xcode12.1.xip via [1], the latest version with 
macosx10.15 SDK. I unpacked it and installed by dragging Xcode.app to 
/Applications. It seems to me there is no magic behind the scenes, just 
moving the directory. I selectively checked that the shims in /usr/bin 
didn't change after that.

Now, running "cc" tells me that I have to accept the Xcode license 
agreement. After accepting it, all the shims in /usr/bin start to work, 
forwarding to the real tools inside Xcode.app.

If I run the Homebrew installer, it says that it's going to install the 
Command Line Tools. I don't know why it needs them, all the tools are 
there already. I thought that CLT is a lighter-weight option when you 
don't want the full Xcode installation, but Homebrew requires them anyway.

I rejected to install CLT and abandoned Homebrew. Then I just cloned and 
built Postgres successfully. So it looks like Xcode is really enough, at 
least on a recent macOS version.


[1] https://xcodereleases.com

-- 
Sergey Shinderuk
Postgres Professional: http://www.postgrespro.com
The Russian Postgres Company

  19. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    Tom Lane <tgl@sss.pgh.pa.us> — 2021-01-22T17:12:11Z

    Sergey Shinderuk <s.shinderuk@postgrespro.ru> writes:
> I rejected to install CLT and abandoned Homebrew. Then I just cloned and 
> built Postgres successfully. So it looks like Xcode is really enough, at 
> least on a recent macOS version.

Hm.  I seem to recall having had to install CLT as well as Xcode back
in the day, but maybe Apple improved that.  On the other side of the
coin, it also seems to be possible to build PG with only CLT and not
Xcode.  I didn't try to verify that with a scorched-earth test, but
I did trash Xcode (and empty trash) on my wife's Mac, and I could
still build and "make check" with only the CLT in place.

[ pokes more carefully... ]  Ah-hah, I see why I needed the CLT.
I bet you'll find that you can't build from "git clean -dfx" state
with only Xcode, because comparing the contents of
/Applications/Xcode.app/Contents/Developer/usr/bin and
/Library/Developer/CommandLineTools/usr/bin on my own Mac,
I observe that only the CLT provides bison and flex.  I also see
install_name_tool only in the CLT; we don't depend on that today,
but may soon (see the latest thread about coping with SIP).

On the whole it looks like we should recommend installing the CLT
and not bothering with Xcode, which is about 10X the size:

$ du -hs /Library/Developer/CommandLineTools
1.1G    /Library/Developer/CommandLineTools
$ du -hs /Applications/Xcode.app
 15G    /Applications/Xcode.app

			regards, tom lane

  20. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    Tom Lane <tgl@sss.pgh.pa.us> — 2021-01-22T18:38:17Z

    Sergey Shinderuk <s.shinderuk@postgrespro.ru> writes:
> If I run the Homebrew installer, it says that it's going to install the 
> Command Line Tools. I don't know why it needs them, all the tools are 
> there already. I thought that CLT is a lighter-weight option when you 
> don't want the full Xcode installation, but Homebrew requires them anyway.

BTW, reading [1] I see

    You can install Xcode, the CLT, or both; Homebrew supports all three
    configurations.

So I'm not sure why you got that prompt, unless you were using a formula
that knew you were going to need bison.

			regards, tom lane

[1] https://docs.brew.sh/Installation#3

  21. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    James Hilliard <james.hilliard1@gmail.com> — 2021-01-22T20:53:31Z

    On Thu, Jan 21, 2021 at 11:38 AM Tom Lane <tgl@sss.pgh.pa.us> wrote:
>
> James Hilliard <james.hilliard1@gmail.com> writes:
> > On Wed, Jan 20, 2021 at 4:07 PM Tom Lane <tgl@sss.pgh.pa.us> wrote:
> >> I'm not sure that the case of not having the "command line tools"
> >> installed is interesting for our purposes.  AFAIK you have to have
> >> that in order to have access to required tools like bison and gmake.
> >> (That reminds me, I was intending to add something to our docs
> >> about how-to-build-from-source to say that you need to install those.)
>
> > Yeah, not 100% sure but I was able to build just fine after deleting my
> > command line tools.
>
> Hm.  I've never been totally clear on what's included in the "command line
> tools", although it's now apparent that one thing that gets installed is
> an SDK matching the host OS version.  However, Apple's description at [1]
> says
>
>     Command Line Tools
>
>     Download the macOS SDK, headers, and build tools such as the Apple
>     LLVM compiler and Make. These tools make it easy to install open
>     source software or develop on UNIX within Terminal. macOS can
>     automatically download these tools the first time you try to build
>     software, and they are available on the downloads page.
>
> which certainly strongly implies that gmake is not there otherwise.
> At this point I lack any "bare" macOS system to check it on.  I wonder
> whether you have a copy of make available from MacPorts or Homebrew.
> Or maybe uninstalling the command line tools doesn't really remove
> everything?
>
> > It would be pretty annoying to have to install an outdated SDK just to
> > build postgres for no other reason than the autoconf feature detection
> > being broken.
>
> It's only as "outdated" as your host system ;-).  Besides, it doesn't
> look like Apple's really giving you a choice not to.
>
> The long and short of this is that I'm unwilling to buy into maintaining
> our own substitutes for standard autoconf probes in order to make it
> possible to use the wrong SDK version.  The preadv/pwritev case is already
> messy enough, and I fear that trying to support such scenarios is going to
> lead to more and more pain in the future.

I found a cleaner alternative to the compile test that appears to work:
https://postgr.es/m/20210122193230.25295-1-james.hilliard1%40gmail.com

Best I can tell the target deployment version check logic requires that the
<sys/uio.h> header be included in order for the check to function properly.

It seems we just need to avoid AC_REPLACE_FUNCS for these cases since
it doesn't allow for passing headers.

>
>                         regards, tom lane
>
> [1] https://developer.apple.com/xcode/features/

  22. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    Sergey Shinderuk <s.shinderuk@postgrespro.ru> — 2021-01-23T05:02:01Z

    On 22.01.2021 20:12, Tom Lane wrote:
> [ pokes more carefully... ]  Ah-hah, I see why I needed the CLT.
> I bet you'll find that you can't build from "git clean -dfx" state
> with only Xcode, because comparing the contents of
> /Applications/Xcode.app/Contents/Developer/usr/bin and
> /Library/Developer/CommandLineTools/usr/bin on my own Mac,
> I observe that only the CLT provides bison and flex.  I also see
> install_name_tool only in the CLT; we don't depend on that today,
> but may soon (see the latest thread about coping with SIP).
> 

I did git clone from scratch. Xcode really has all the tools.

configure:9519: checking for bison
configure:9537: found /usr/bin/bison
configure:9549: result: /usr/bin/bison
configure:9571: using bison (GNU Bison) 2.3
configure:9609: checking for flex
configure:9654: result: /usr/bin/flex
configure:9674: using flex 2.5.35 Apple(flex-32)

% xcrun --find bison
/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/bison

% xcrun --find install_name_tool
/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/install_name_tool


> On the whole it looks like we should recommend installing the CLT
> and not bothering with Xcode, which is about 10X the size:
> 
> $ du -hs /Library/Developer/CommandLineTools
> 1.1G    /Library/Developer/CommandLineTools
> $ du -hs /Applications/Xcode.app
>   15G    /Applications/Xcode.app
> 

Fair.


> BTW, reading [1] I see
> 
>     You can install Xcode, the CLT, or both; Homebrew supports all three
>     configurations.
> 
> So I'm not sure why you got that prompt, unless you were using a formula
> that knew you were going to need bison.
> 
> [1] https://docs.brew.sh/Installation#3

Apparently, this documentation is wrong. I’m not installing any 
particular formula, just running the Homebrew installer script.

% /bin/bash -c "$(curl -fsSL 
https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
Password:
==> This script will install:
[...]
==> The following new directories will be created:
[...]
==> The Xcode Command Line Tools will be installed.

Press RETURN to continue or any other key to abort

==> Installing Command Line Tools for Xcode-12.3
==> /usr/bin/sudo /usr/sbin/softwareupdate -i Command\ Line\ Tools\ for\ 
Xcode-12.3
Software Update Tool

Downloading Command Line Tools for Xcode
[...]

I checked the script [1], and it really requires the CLT. Here is the 
explanation [2] for this:

	There is actually no such requirement. However, there are
	formulae that will be forced to build from source if you do not
	have the CLT. They can still be built from source with Xcode
	only, but because the pre-built bottles are compiled in an
	environment that has both Xcode and the CLT installed, there are
	some cases where the bottles end up having a hard dependency on
	the CLT. A major example is gcc. So installing the CLT may help
	you avoid some lengthy source builds.

	We ensure that all Homebrew formulae can be built with Xcode.app
	alone. Most formulae can be built with just the CLT, and those
	that require the full Xcode.app have an explicit depends_on
	:xcode => :build. Some users would prefer to use only the CLT
	because it's a much smaller download and takes less time to
	install and upgrade than Xcode.


[1] https://github.com/Homebrew/install/blob/master/install.sh#L191
[2] https://github.com/Homebrew/brew/issues/1613


Regards.

-- 
Sergey Shinderuk
Postgres Professional: http://www.postgrespro.com
The Russian Postgres Company

  23. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    Sergey Shinderuk <s.shinderuk@postgrespro.ru> — 2021-01-23T05:12:16Z

    On 23.01.2021 08:02, Sergey Shinderuk wrote:
> I checked the script [1], and it really requires the CLT. Here is the 
> explanation [2] for this:
> 
>      There is actually no such requirement. However, there are
>      formulae that will be forced to build from source if you do not
>      have the CLT. They can still be built from source with Xcode
>      only, but because the pre-built bottles are compiled in an
>      environment that has both Xcode and the CLT installed, there are
>      some cases where the bottles end up having a hard dependency on
>      the CLT. A major example is gcc. So installing the CLT may help
>      you avoid some lengthy source builds.
> 
>      We ensure that all Homebrew formulae can be built with Xcode.app
>      alone. Most formulae can be built with just the CLT, and those
>      that require the full Xcode.app have an explicit depends_on
>      :xcode => :build. Some users would prefer to use only the CLT
>      because it's a much smaller download and takes less time to
>      install and upgrade than Xcode.


In the gcc formula [1]:

   # The bottles are built on systems with the CLT installed, and do not 
work
   # out of the box on Xcode-only systems due to an incorrect sysroot.
   pour_bottle? do
     reason "The bottle needs the Xcode CLT to be installed."
     satisfy { MacOS::CLT.installed? }
   end


I guess this is the "xcrun --show-sdk-path" thing we've alredy disccussed :)


[1] https://github.com/Homebrew/homebrew-core/blob/master/Formula/gcc.rb#L36

-- 
Sergey Shinderuk
Postgres Professional: http://www.postgrespro.com
The Russian Postgres Company

  24. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    Sergey Shinderuk <s.shinderuk@postgrespro.ru> — 2021-01-23T08:27:16Z

    On 23.01.2021 08:02, Sergey Shinderuk wrote:
>> On the whole it looks like we should recommend installing the CLT
>> and not bothering with Xcode, which is about 10X the size:
>>
>> $ du -hs /Library/Developer/CommandLineTools
>> 1.1G    /Library/Developer/CommandLineTools
>> $ du -hs /Applications/Xcode.app
>>   15G    /Applications/Xcode.app
>>
> 
> Fair.

BTW, Homebrew prefers the CLT SDK:
https://github.com/Homebrew/brew/blob/master/Library/Homebrew/os/mac.rb#L138

       # Prefer CLT SDK when both Xcode and the CLT are installed.
       # Expected results:
       # 1. On Xcode-only systems, return the Xcode SDK.
       # 2. On Xcode-and-CLT systems where headers are provided by the 
system, return nil.
       # 3. On CLT-only systems with no CLT SDK, return nil.
       # 4. On CLT-only systems with a CLT SDK, where headers are 
provided by the system, return nil.
       # 5. On CLT-only systems with a CLT SDK, where headers are not 
provided by the system, return the CLT SDK.


Here is the relevant discussion:
https://github.com/Homebrew/brew/pull/7134

I like the example of Git compiled against the wrong 
LIBCURL_VERSION_NUM. Clearly, there are other issues with 
cross-compiling to a newer SDK, besides autoconf probes and weak imports.

-- 
Sergey Shinderuk
Postgres Professional: http://www.postgrespro.com
The Russian Postgres Company

  25. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    James Hilliard <james.hilliard1@gmail.com> — 2021-01-23T09:22:23Z

    On Sat, Jan 23, 2021 at 1:27 AM Sergey Shinderuk
<s.shinderuk@postgrespro.ru> wrote:
>
> On 23.01.2021 08:02, Sergey Shinderuk wrote:
> >> On the whole it looks like we should recommend installing the CLT
> >> and not bothering with Xcode, which is about 10X the size:
> >>
> >> $ du -hs /Library/Developer/CommandLineTools
> >> 1.1G    /Library/Developer/CommandLineTools
> >> $ du -hs /Applications/Xcode.app
> >>   15G    /Applications/Xcode.app
> >>
> >
> > Fair.
>
> BTW, Homebrew prefers the CLT SDK:
> https://github.com/Homebrew/brew/blob/master/Library/Homebrew/os/mac.rb#L138
>
>        # Prefer CLT SDK when both Xcode and the CLT are installed.
>        # Expected results:
>        # 1. On Xcode-only systems, return the Xcode SDK.
>        # 2. On Xcode-and-CLT systems where headers are provided by the
> system, return nil.
>        # 3. On CLT-only systems with no CLT SDK, return nil.
>        # 4. On CLT-only systems with a CLT SDK, where headers are
> provided by the system, return nil.
>        # 5. On CLT-only systems with a CLT SDK, where headers are not
> provided by the system, return the CLT SDK.
>
>
> Here is the relevant discussion:
> https://github.com/Homebrew/brew/pull/7134
>
> I like the example of Git compiled against the wrong
> LIBCURL_VERSION_NUM. Clearly, there are other issues with
> cross-compiling to a newer SDK, besides autoconf probes and weak imports.

From my understanding homebrew considers supporting deployment targets
other than that of the build host entirely out of scope, due to its
nature of being
a meta build system homebrew probably needs to sidestep package target
deployment bugs in this way as they are likely to be quite common. Homebrew
also has to ensure compatibility with the binary bottles. Homebrew handles
backwards compatibility largely by using host OS specific binaries, which is not
the typical way package binaries are distributed on OSX.

So it appears that their reasoning for doing this may not be directly applicable
to the situation we have with postgres as they have additional concerns. In
theory we should generally not have to worry about this much as long as target
deployment feature detection is functional as either SDK would generally work
for producing binaries that can run on the build host.

>
> --
> Sergey Shinderuk
> Postgres Professional: http://www.postgrespro.com
> The Russian Postgres Company

  26. Re: [PATCH 1/1] Fix detection of pwritev support for OSX.

    James Hilliard <james.hilliard1@gmail.com> — 2021-02-06T03:53:09Z

    On Fri, Jan 22, 2021 at 1:53 PM James Hilliard
<james.hilliard1@gmail.com> wrote:
>
> On Thu, Jan 21, 2021 at 11:38 AM Tom Lane <tgl@sss.pgh.pa.us> wrote:
> >
> > James Hilliard <james.hilliard1@gmail.com> writes:
> > > On Wed, Jan 20, 2021 at 4:07 PM Tom Lane <tgl@sss.pgh.pa.us> wrote:
> > >> I'm not sure that the case of not having the "command line tools"
> > >> installed is interesting for our purposes.  AFAIK you have to have
> > >> that in order to have access to required tools like bison and gmake.
> > >> (That reminds me, I was intending to add something to our docs
> > >> about how-to-build-from-source to say that you need to install those.)
> >
> > > Yeah, not 100% sure but I was able to build just fine after deleting my
> > > command line tools.
> >
> > Hm.  I've never been totally clear on what's included in the "command line
> > tools", although it's now apparent that one thing that gets installed is
> > an SDK matching the host OS version.  However, Apple's description at [1]
> > says
> >
> >     Command Line Tools
> >
> >     Download the macOS SDK, headers, and build tools such as the Apple
> >     LLVM compiler and Make. These tools make it easy to install open
> >     source software or develop on UNIX within Terminal. macOS can
> >     automatically download these tools the first time you try to build
> >     software, and they are available on the downloads page.
> >
> > which certainly strongly implies that gmake is not there otherwise.
> > At this point I lack any "bare" macOS system to check it on.  I wonder
> > whether you have a copy of make available from MacPorts or Homebrew.
> > Or maybe uninstalling the command line tools doesn't really remove
> > everything?
> >
> > > It would be pretty annoying to have to install an outdated SDK just to
> > > build postgres for no other reason than the autoconf feature detection
> > > being broken.
> >
> > It's only as "outdated" as your host system ;-).  Besides, it doesn't
> > look like Apple's really giving you a choice not to.
> >
> > The long and short of this is that I'm unwilling to buy into maintaining
> > our own substitutes for standard autoconf probes in order to make it
> > possible to use the wrong SDK version.  The preadv/pwritev case is already
> > messy enough, and I fear that trying to support such scenarios is going to
> > lead to more and more pain in the future.
>
> I found a cleaner alternative to the compile test that appears to work:
> https://postgr.es/m/20210122193230.25295-1-james.hilliard1%40gmail.com
>
> Best I can tell the target deployment version check logic requires that the
> <sys/uio.h> header be included in order for the check to function properly.
>
> It seems we just need to avoid AC_REPLACE_FUNCS for these cases since
> it doesn't allow for passing headers.

I did manage to verify that AC_REPLACE_FUNCS generates an incorrect conftest.c
for OSX which is why it is incorrectly detecting the availability of pwritev.

conftest.c:
/* confdefs.h */
#define PACKAGE_NAME "PostgreSQL"
#define PACKAGE_TARNAME "postgresql"
#define PACKAGE_VERSION "14devel"
#define PACKAGE_STRING "PostgreSQL 14devel"
#define PACKAGE_BUGREPORT "pgsql-bugs@lists.postgresql.org"
#define PACKAGE_URL "https://www.postgresql.org/"
#define CONFIGURE_ARGS ""
#define PG_MAJORVERSION "14"
#define PG_MAJORVERSION_NUM 14
#define PG_MINORVERSION_NUM 0
#define PG_VERSION "14devel"
#define DEF_PGPORT 5432
#define DEF_PGPORT_STR "5432"
#define BLCKSZ 8192
#define RELSEG_SIZE 131072
#define XLOG_BLCKSZ 8192
#define ENABLE_THREAD_SAFETY 1
#define PG_KRB_SRVNAM "postgres"
#define STDC_HEADERS 1
#define HAVE_SYS_TYPES_H 1
#define HAVE_SYS_STAT_H 1
#define HAVE_STDLIB_H 1
#define HAVE_STRING_H 1
#define HAVE_MEMORY_H 1
#define HAVE_STRINGS_H 1
#define HAVE_INTTYPES_H 1
#define HAVE_STDINT_H 1
#define HAVE_UNISTD_H 1
#define HAVE_PTHREAD_PRIO_INHERIT 1
#define HAVE_PTHREAD 1
#define HAVE_STRERROR_R 1
#define HAVE_GETPWUID_R 1
#define STRERROR_R_INT 1
#define HAVE_LIBM 1
#define HAVE_LIBREADLINE 1
#define HAVE_LIBZ 1
#define HAVE_SPINLOCKS 1
#define HAVE_ATOMICS 1
#define HAVE__BOOL 1
#define HAVE_STDBOOL_H 1
#define HAVE_COPYFILE_H 1
#define HAVE_EXECINFO_H 1
#define HAVE_GETOPT_H 1
#define HAVE_IFADDRS_H 1
#define HAVE_LANGINFO_H 1
#define HAVE_POLL_H 1
#define HAVE_SYS_EVENT_H 1
#define HAVE_SYS_IPC_H 1
#define HAVE_SYS_RESOURCE_H 1
#define HAVE_SYS_SELECT_H 1
#define HAVE_SYS_SEM_H 1
#define HAVE_SYS_SHM_H 1
#define HAVE_SYS_SOCKIO_H 1
#define HAVE_SYS_UIO_H 1
#define HAVE_SYS_UN_H 1
#define HAVE_TERMIOS_H 1
#define HAVE_WCTYPE_H 1
#define HAVE_NET_IF_H 1
#define HAVE_SYS_UCRED_H 1
#define HAVE_NETINET_TCP_H 1
#define HAVE_READLINE_READLINE_H 1
#define HAVE_READLINE_HISTORY_H 1
#define PG_PRINTF_ATTRIBUTE printf
#define HAVE_FUNCNAME__FUNC 1
#define HAVE__STATIC_ASSERT 1
#define HAVE_TYPEOF 1
#define HAVE__BUILTIN_TYPES_COMPATIBLE_P 1
#define HAVE__BUILTIN_CONSTANT_P 1
#define HAVE__BUILTIN_UNREACHABLE 1
#define HAVE_COMPUTED_GOTO 1
#define HAVE_STRUCT_TM_TM_ZONE 1
#define HAVE_UNION_SEMUN 1
#define HAVE_STRUCT_SOCKADDR_UN 1
#define HAVE_STRUCT_SOCKADDR_STORAGE 1
#define HAVE_STRUCT_SOCKADDR_STORAGE_SS_FAMILY 1
#define HAVE_STRUCT_SOCKADDR_STORAGE_SS_LEN 1
#define HAVE_STRUCT_SOCKADDR_SA_LEN 1
#define HAVE_STRUCT_ADDRINFO 1
#define HAVE_LOCALE_T 1
#define LOCALE_T_IN_XLOCALE 1
#define restrict __restrict
#define pg_restrict __restrict
#define HAVE_STRUCT_OPTION 1
#define HAVE_X86_64_POPCNTQ 1
#define SIZEOF_OFF_T 8
#define SIZEOF_BOOL 1
#define PG_USE_STDBOOL 1
#define HAVE_INT_TIMEZONE 1
#define ACCEPT_TYPE_RETURN int
#define ACCEPT_TYPE_ARG1 int
#define ACCEPT_TYPE_ARG2 struct sockaddr *
#define ACCEPT_TYPE_ARG3 socklen_t
#define WCSTOMBS_L_IN_XLOCALE 1
#define HAVE_BACKTRACE_SYMBOLS 1
#define HAVE_CLOCK_GETTIME 1
#define HAVE_COPYFILE 1
#define HAVE_FDATASYNC 1
#define HAVE_GETIFADDRS 1
#define HAVE_GETRLIMIT 1
#define HAVE_KQUEUE 1
#define HAVE_MBSTOWCS_L 1
#define HAVE_MEMSET_S 1
#define HAVE_POLL 1
#define HAVE_PTHREAD_IS_THREADED_NP 1
#define HAVE_READLINK 1
#define HAVE_READV 1
#define HAVE_SETSID 1
#define HAVE_SHM_OPEN 1
#define HAVE_STRSIGNAL 1
#define HAVE_SYMLINK 1
#define HAVE_USELOCALE 1
#define HAVE_WCSTOMBS_L 1
#define HAVE_WRITEV 1
#define HAVE__BUILTIN_BSWAP16 1
#define HAVE__BUILTIN_BSWAP32 1
#define HAVE__BUILTIN_BSWAP64 1
#define HAVE__BUILTIN_CLZ 1
#define HAVE__BUILTIN_CTZ 1
#define HAVE__BUILTIN_POPCOUNT 1
#define HAVE_FSEEKO 1
#define HAVE_DECL_POSIX_FADVISE 0
#define HAVE_DECL_FDATASYNC 0
#define HAVE_DECL_STRLCAT 1
#define HAVE_DECL_STRLCPY 1
#define HAVE_DECL_STRNLEN 1
#define HAVE_DECL_F_FULLFSYNC 1
#define HAVE_DECL_RTLD_GLOBAL 1
#define HAVE_DECL_RTLD_NOW 1
#define HAVE_IPV6 1
#define HAVE_DLOPEN 1
#define HAVE_FLS 1
#define HAVE_GETOPT 1
#define HAVE_GETPEEREID 1
#define HAVE_GETRUSAGE 1
#define HAVE_INET_ATON 1
#define HAVE_LINK 1
#define HAVE_MKDTEMP 1
#define HAVE_PREAD 1
#define HAVE_PREADV 1
#define HAVE_PWRITE 1
/* end confdefs.h.  */
/* Define pwritev to an innocuous variant, in case <limits.h> declares pwritev.
   For example, HP-UX 11i <limits.h> declares gettimeofday.  */
#define pwritev innocuous_pwritev

/* System header to define __stub macros and hopefully few prototypes,
    which can conflict with char pwritev (); below.
    Prefer <limits.h> to <assert.h> if __STDC__ is defined, since
    <limits.h> exists even on freestanding compilers.  */

#ifdef __STDC__
# include <limits.h>
#else
# include <assert.h>
#endif

#undef pwritev

/* Override any GCC internal prototype to avoid an error.
   Use char because int might match the return type of a GCC
   builtin and then its argument prototype would still apply.  */
#ifdef __cplusplus
extern "C"
#endif
char pwritev ();
/* The GNU C library defines this for functions which it implements
    to always fail with ENOSYS.  Some functions are actually named
    something starting with __ and the normal name is an alias.  */
#if defined __stub_pwritev || defined __stub___pwritev
choke me
#endif

int
main ()
{
return pwritev ();
  ;
  return 0;
}

The correct declaration for pwritev on OSX is:
ssize_t pwritev(int, const struct iovec *, int, off_t)
__DARWIN_NOCANCEL(pwritev) __API_AVAILABLE(macos(11.0), ios(14.0),
watchos(7.0), tvos(14.0));
while the conftest.c generated by AC_REPLACE_FUNCS declares:
char pwritev ();
which results in a broken conftest binary.

On OSX if the declaration is missing __API_AVAILABLE then the target
deployment version will not be checked properly and you might end up
with a broken binary.

>
> >
> >                         regards, tom lane
> >
> > [1] https://developer.apple.com/xcode/features/