0001-improve-hex-functions-handling.patch

diff --git a/src/backend/replication/backup_manifest.c b/src/backend/replication/backup_manifest.c
index 8882444025..1a3f06b3fe 100644
--- a/src/backend/replication/backup_manifest.c
+++ b/src/backend/replication/backup_manifest.c
@@ -150,7 +150,7 @@ AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid,
 	}
 	else
 	{
-		uint64		dstlen = pg_hex_enc_len(pathlen);
+		size_t		dstlen = pg_hex_enc_len(pathlen);
 
 		appendStringInfoString(&buf, "{ \"Encoded-Path\": \"");
 		enlargeStringInfo(&buf, dstlen);
diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c
index acb8741734..5989eee651 100644
--- a/src/backend/utils/adt/varlena.c
+++ b/src/backend/utils/adt/varlena.c
@@ -305,7 +305,7 @@ byteain(PG_FUNCTION_ARGS)
 	if (inputText[0] == '\\' && inputText[1] == 'x')
 	{
 		size_t		len = strlen(inputText);
-		uint64		dstlen = pg_hex_dec_len(len - 2);
+		size_t		dstlen = pg_hex_dec_len(len - 2);
 
 		bc = dstlen + VARHDRSZ; /* maximum possible length */
 		result = palloc(bc);
@@ -399,7 +399,7 @@ byteaout(PG_FUNCTION_ARGS)
 
 	if (bytea_output == BYTEA_OUTPUT_HEX)
 	{
-		uint64		dstlen = pg_hex_enc_len(VARSIZE_ANY_EXHDR(vlena));
+		size_t		dstlen = pg_hex_enc_len(VARSIZE_ANY_EXHDR(vlena));
 
 		/* Print hex format */
 		rp = result = palloc(dstlen + 2 + 1);
@@ -413,8 +413,8 @@ byteaout(PG_FUNCTION_ARGS)
 	{
 		/* Print traditional escaped format */
 		char	   *vp;
-		uint64		len;
-		int			i;
+		size_t		len;
+		size_t		i;
 
 		len = 1;				/* empty string has 1 char */
 		vp = VARDATA_ANY(vlena);
diff --git a/src/common/hex.c b/src/common/hex.c
index 3b1bc8fa75..0aae74fc10 100644
--- a/src/common/hex.c
+++ b/src/common/hex.c
@@ -70,30 +70,30 @@ get_hex(const char *cp)
  * Encode into hex the given string.  Returns the length of the encoded
  * string.
  */
-uint64
+size_t
 pg_hex_encode(const char *src, size_t srclen, char *dst, size_t dstlen)
 {
-	const char *end = src + srclen;
+	const char *end;
 	char	   *p;
 
-	p = dst;
-
-	while (src < end)
+	/*
+	 * Leave if there is an overflow in the area allocated for the encoded
+	 * string.
+	 */
+	if (dstlen < pg_hex_enc_len(srclen))
 	{
-		/*
-		 * Leave if there is an overflow in the area allocated for the encoded
-		 * string.
-		 */
-		if ((p - dst + 2) > dstlen)
-		{
 #ifdef FRONTEND
-			pg_log_fatal("overflow of destination buffer in hex encoding");
-			exit(EXIT_FAILURE);
+		pg_log_fatal("overflow of destination buffer in hex encoding");
+		exit(EXIT_FAILURE);
 #else
-			elog(ERROR, "overflow of destination buffer in hex encoding");
+		elog(ERROR, "overflow of destination buffer in hex encoding");
 #endif
-		}
+	}
 
+	end = src + srclen;
+	p = dst;
+	while (src < end)
+	{
 		*p++ = hextbl[(*src >> 4) & 0xF];
 		*p++ = hextbl[*src & 0xF];
 		src++;
@@ -108,7 +108,7 @@ pg_hex_encode(const char *src, size_t srclen, char *dst, size_t dstlen)
  *
  * Decode the given hex string.  Returns the length of the decoded string.
  */
-uint64
+size_t
 pg_hex_decode(const char *src, size_t srclen, char *dst, size_t dstlen)
 {
 	const char *s,
@@ -117,6 +117,20 @@ pg_hex_decode(const char *src, size_t srclen, char *dst, size_t dstlen)
 				v2,
 			   *p;
 
+	/*
+	 * Leave if there is an overflow in the area allocated for the encoded
+	 * string.
+	 */
+	if (dstlen < pg_hex_dec_len(srclen))
+	{
+#ifdef FRONTEND
+		pg_log_fatal("overflow of destination buffer in hex encoding");
+		exit(EXIT_FAILURE);
+#else
+		elog(ERROR, "overflow of destination buffer in hex encoding");
+#endif
+	}
+
 	srcend = src + srclen;
 	s = src;
 	p = dst;
@@ -130,7 +144,7 @@ pg_hex_decode(const char *src, size_t srclen, char *dst, size_t dstlen)
 		v1 = get_hex(s) << 4;
 		s++;
 
-		if (s >= srcend)
+		if (s == srcend)
 		{
 #ifdef FRONTEND
 			pg_log_fatal("invalid hexadecimal data: odd number of digits");
@@ -138,24 +152,12 @@ pg_hex_decode(const char *src, size_t srclen, char *dst, size_t dstlen)
 #else
 			ereport(ERROR,
 					(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
-					 errmsg("invalid hexadecimal data: odd number of digits")));
+					errmsg("invalid hexadecimal data: odd number of digits")));
 #endif
 		}
-
 		v2 = get_hex(s);
 		s++;
 
-		/* overflow check */
-		if ((p - dst + 1) > dstlen)
-		{
-#ifdef FRONTEND
-			pg_log_fatal("overflow of destination buffer in hex decoding");
-			exit(EXIT_FAILURE);
-#else
-			elog(ERROR, "overflow of destination buffer in hex decoding");
-#endif
-		}
-
 		*p++ = v1 | v2;
 	}
 
@@ -171,10 +173,10 @@ pg_hex_decode(const char *src, size_t srclen, char *dst, size_t dstlen)
  * how large a buffer allocation needs to be done before doing the actual
  * encoding.
  */
-uint64
+size_t
 pg_hex_enc_len(size_t srclen)
 {
-	return (uint64) srclen << 1;
+	return srclen << 1;
 }
 
 /*
@@ -185,8 +187,8 @@ pg_hex_enc_len(size_t srclen)
  * estimate how large a buffer allocation needs to be done before doing
  * the actual decoding.
  */
-uint64
+size_t
 pg_hex_dec_len(size_t srclen)
 {
-	return (uint64) srclen >> 1;
+	return srclen >> 1;
 }
diff --git a/src/include/common/hex.h b/src/include/common/hex.h
index 150771a14d..e515f58b8f 100644
--- a/src/include/common/hex.h
+++ b/src/include/common/hex.h
@@ -15,11 +15,11 @@
 #ifndef COMMON_HEX_H
 #define COMMON_HEX_H
 
-extern uint64 pg_hex_decode(const char *src, size_t srclen,
+extern size_t pg_hex_decode(const char *src, size_t srclen,
 							char *dst, size_t dstlen);
-extern uint64 pg_hex_encode(const char *src, size_t srclen,
+extern size_t pg_hex_encode(const char *src, size_t srclen,
 							char *dst, size_t dstlen);
-extern uint64 pg_hex_enc_len(size_t srclen);
-extern uint64 pg_hex_dec_len(size_t srclen);
+extern size_t pg_hex_enc_len(size_t srclen);
+extern size_t pg_hex_dec_len(size_t srclen);
 
 #endif							/* COMMON_HEX_H */