libpq_altstore_20120117.patch

text/x-patch

Filename: libpq_altstore_20120117.patch
Type: text/x-patch
Part: 0
Message: Re: Speed dblink using alternate libpq tuple storage

Patch

Same data as JSON: GET /api/v1/attachments/:id/patch the parsed metadata as JSON — format, series position, per-file stats; never the diff bytes. API reference →
Format: unified
File+
src/interfaces/libpq/exports.txt 3 0
src/interfaces/libpq/fe-connect.c 8 0
src/interfaces/libpq/fe-exec.c 73 4
src/interfaces/libpq/fe-protocol2.c 28 10
src/interfaces/libpq/fe-protocol3.c 27 11
src/interfaces/libpq/libpq-fe.h 67 0
src/interfaces/libpq/libpq-int.h 12 7
diff --git a/src/interfaces/libpq/exports.txt b/src/interfaces/libpq/exports.txt
index 1af8df6..83525e1 100644
--- a/src/interfaces/libpq/exports.txt
+++ b/src/interfaces/libpq/exports.txt
@@ -160,3 +160,6 @@ PQconnectStartParams      157
 PQping                    158
 PQpingParams              159
 PQlibVersion              160
+PQregisterStoreHandler	  161
+PQgetStoreHandlerParam	  163
+PQsetStoreHandlerErrMes	  164
diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c
index d454538..5559f0b 100644
--- a/src/interfaces/libpq/fe-connect.c
+++ b/src/interfaces/libpq/fe-connect.c
@@ -2692,6 +2692,7 @@ makeEmptyPGconn(void)
 	conn->allow_ssl_try = true;
 	conn->wait_ssl_try = false;
 #endif
+	conn->storeHandler = NULL;
 
 	/*
 	 * We try to send at least 8K at a time, which is the usual size of pipe
@@ -5076,3 +5077,10 @@ PQregisterThreadLock(pgthreadlock_t newhandler)
 
 	return prev;
 }
+
+void
+PQregisterStoreHandler(PGconn *conn, StoreHandler func, void *param)
+{
+	conn->storeHandler = func;
+	conn->storeHandlerParam = param;
+}
diff --git a/src/interfaces/libpq/fe-exec.c b/src/interfaces/libpq/fe-exec.c
index b743566..96e5974 100644
--- a/src/interfaces/libpq/fe-exec.c
+++ b/src/interfaces/libpq/fe-exec.c
@@ -67,6 +67,10 @@ static int PQsendDescribe(PGconn *conn, char desc_type,
 			   const char *desc_target);
 static int	check_field_number(const PGresult *res, int field_num);
 
+static void *pqDefaultStoreHandler(PGresult *res, PQStoreFunc func,
+								   int id, size_t len);
+static void *pqAddTuple(PGresult *res, PGresAttValue *tup);
+
 
 /* ----------------
  * Space management for PGresult.
@@ -160,6 +164,9 @@ PQmakeEmptyPGresult(PGconn *conn, ExecStatusType status)
 	result->curBlock = NULL;
 	result->curOffset = 0;
 	result->spaceLeft = 0;
+	result->storeHandler = pqDefaultStoreHandler;
+	result->storeHandlerParam = NULL;
+	result->storeHandlerErrMes = NULL;
 
 	if (conn)
 	{
@@ -194,6 +201,12 @@ PQmakeEmptyPGresult(PGconn *conn, ExecStatusType status)
 			}
 			result->nEvents = conn->nEvents;
 		}
+
+		if (conn->storeHandler)
+		{
+			result->storeHandler = conn->storeHandler;
+			result->storeHandlerParam = conn->storeHandlerParam;
+		}
 	}
 	else
 	{
@@ -487,6 +500,33 @@ PQresultAlloc(PGresult *res, size_t nBytes)
 	return pqResultAlloc(res, nBytes, TRUE);
 }
 
+void *
+pqDefaultStoreHandler(PGresult *res, PQStoreFunc func, int id, size_t len)
+{
+	void *p;
+
+	switch (func)
+	{
+		case PQSF_ALLOC_TEXT:
+			return pqResultAlloc(res, len, TRUE);
+
+		case PQSF_ALLOC_BINARY:
+			p = pqResultAlloc(res, len, FALSE);
+
+			if (id == -1)
+				res->storeHandlerParam = p;
+
+			return p;
+
+		case PQSF_ADD_TUPLE:
+			return pqAddTuple(res, res->storeHandlerParam);
+
+		default:
+			/* Ignore */
+			break;
+	}
+	return NULL;
+}
 /*
  * pqResultAlloc -
  *		Allocate subsidiary storage for a PGresult.
@@ -830,9 +870,9 @@ pqInternalNotice(const PGNoticeHooks *hooks, const char *fmt,...)
 /*
  * pqAddTuple
  *	  add a row pointer to the PGresult structure, growing it if necessary
- *	  Returns TRUE if OK, FALSE if not enough memory to add the row
+ *	  Returns tup if OK, NULL if not enough memory to add the row.
  */
-int
+static void *
 pqAddTuple(PGresult *res, PGresAttValue *tup)
 {
 	if (res->ntups >= res->tupArrSize)
@@ -858,13 +898,13 @@ pqAddTuple(PGresult *res, PGresAttValue *tup)
 			newTuples = (PGresAttValue **)
 				realloc(res->tuples, newSize * sizeof(PGresAttValue *));
 		if (!newTuples)
-			return FALSE;		/* malloc or realloc failed */
+			return NULL;		/* malloc or realloc failed */
 		res->tupArrSize = newSize;
 		res->tuples = newTuples;
 	}
 	res->tuples[res->ntups] = tup;
 	res->ntups++;
-	return TRUE;
+	return tup;
 }
 
 /*
@@ -2822,6 +2862,35 @@ PQgetisnull(const PGresult *res, int tup_num, int field_num)
 		return 0;
 }
 
+/* PQgetAddStoreHandlerParam
+ *	Get the pointer to the contextual parameter from PGresult which is
+ *	registered to PGconn by PQregisterStoreHandler
+ */
+void *
+PQgetStoreHandlerParam(const PGresult *res)
+{
+	if (!res)
+		return NULL;
+	return res->storeHandlerParam;
+}
+
+/* PQsetStorHandlerErrMes
+ *	Set the error message pass back to the caller of StoreHandler.
+ *
+ *  mes must be a malloc'ed memory block and it will be released by
+ *  the caller of StoreHandler.  You can replace the previous message
+ *  by alternative mes, or clear it with NULL. The previous one will
+ *  be freed internally.
+ */
+void
+PQsetStoreHandlerErrMes(PGresult *res, char *mes)
+{
+	/* Free existing message */
+	if (res->storeHandlerErrMes)
+		free(res->storeHandlerErrMes);
+	res->storeHandlerErrMes = mes;
+}
+
 /* PQnparams:
  *	returns the number of input parameters of a prepared statement.
  */
diff --git a/src/interfaces/libpq/fe-protocol2.c b/src/interfaces/libpq/fe-protocol2.c
index a7c3899..205502b 100644
--- a/src/interfaces/libpq/fe-protocol2.c
+++ b/src/interfaces/libpq/fe-protocol2.c
@@ -733,9 +733,10 @@ getAnotherTuple(PGconn *conn, bool binary)
 	if (conn->curTuple == NULL)
 	{
 		conn->curTuple = (PGresAttValue *)
-			pqResultAlloc(result, nfields * sizeof(PGresAttValue), TRUE);
+			result->storeHandler(result, PQSF_ALLOC_BINARY, -1,
+								 nfields * sizeof(PGresAttValue));
 		if (conn->curTuple == NULL)
-			goto outOfMemory;
+			goto addTupleError;
 		MemSet(conn->curTuple, 0, nfields * sizeof(PGresAttValue));
 
 		/*
@@ -757,7 +758,7 @@ getAnotherTuple(PGconn *conn, bool binary)
 	{
 		bitmap = (char *) malloc(nbytes);
 		if (!bitmap)
-			goto outOfMemory;
+			goto addTupleError;
 	}
 
 	if (pqGetnchar(bitmap, nbytes, conn))
@@ -787,9 +788,12 @@ getAnotherTuple(PGconn *conn, bool binary)
 				vlen = 0;
 			if (tup[i].value == NULL)
 			{
-				tup[i].value = (char *) pqResultAlloc(result, vlen + 1, binary);
+				PQStoreFunc func = 
+					(binary ? PQSF_ALLOC_BINARY : PQSF_ALLOC_TEXT);
+				tup[i].value =
+					(char *) result->storeHandler(result, func, i, vlen + 1);
 				if (tup[i].value == NULL)
-					goto outOfMemory;
+					goto addTupleError;
 			}
 			tup[i].len = vlen;
 			/* read in the value */
@@ -812,8 +816,9 @@ getAnotherTuple(PGconn *conn, bool binary)
 	}
 
 	/* Success!  Store the completed tuple in the result */
-	if (!pqAddTuple(result, tup))
-		goto outOfMemory;
+	if (!result->storeHandler(result, PQSF_ADD_TUPLE, 0, 0))
+		goto addTupleError;
+
 	/* and reset for a new message */
 	conn->curTuple = NULL;
 
@@ -821,7 +826,7 @@ getAnotherTuple(PGconn *conn, bool binary)
 		free(bitmap);
 	return 0;
 
-outOfMemory:
+addTupleError:
 	/* Replace partially constructed result with an error result */
 
 	/*
@@ -829,8 +834,21 @@ outOfMemory:
 	 * there's not enough memory to concatenate messages...
 	 */
 	pqClearAsyncResult(conn);
-	printfPQExpBuffer(&conn->errorMessage,
-					  libpq_gettext("out of memory for query result\n"));
+	resetPQExpBuffer(&conn->errorMessage);
+
+	/*
+	 * If error message is passed from addTupleFunc, set it into
+	 * PGconn, assume out of memory if not.
+	 */
+	appendPQExpBufferStr(&conn->errorMessage,
+						 libpq_gettext(result->storeHandlerErrMes ?
+									   result->storeHandlerErrMes :
+									   "out of memory for query result\n"));
+	if (result->storeHandlerErrMes)
+	{
+		free(result->storeHandlerErrMes);
+		result->storeHandlerErrMes = NULL;
+	}
 
 	/*
 	 * XXX: if PQmakeEmptyPGresult() fails, there's probably not much we can
diff --git a/src/interfaces/libpq/fe-protocol3.c b/src/interfaces/libpq/fe-protocol3.c
index 892dcbc..117c38a 100644
--- a/src/interfaces/libpq/fe-protocol3.c
+++ b/src/interfaces/libpq/fe-protocol3.c
@@ -634,9 +634,10 @@ getAnotherTuple(PGconn *conn, int msgLength)
 	if (conn->curTuple == NULL)
 	{
 		conn->curTuple = (PGresAttValue *)
-			pqResultAlloc(result, nfields * sizeof(PGresAttValue), TRUE);
+			result->storeHandler(result, PQSF_ALLOC_BINARY, -1,
+								 nfields * sizeof(PGresAttValue));
 		if (conn->curTuple == NULL)
-			goto outOfMemory;
+			goto addTupleError;
 		MemSet(conn->curTuple, 0, nfields * sizeof(PGresAttValue));
 	}
 	tup = conn->curTuple;
@@ -673,11 +674,12 @@ getAnotherTuple(PGconn *conn, int msgLength)
 			vlen = 0;
 		if (tup[i].value == NULL)
 		{
-			bool		isbinary = (result->attDescs[i].format != 0);
-
-			tup[i].value = (char *) pqResultAlloc(result, vlen + 1, isbinary);
+			PQStoreFunc func = (result->attDescs[i].format != 0 ?
+								PQSF_ALLOC_BINARY : PQSF_ALLOC_TEXT);
+			tup[i].value =
+				(char *) result->storeHandler(result, func, i, vlen + 1);
 			if (tup[i].value == NULL)
-				goto outOfMemory;
+				goto addTupleError;
 		}
 		tup[i].len = vlen;
 		/* read in the value */
@@ -689,22 +691,36 @@ getAnotherTuple(PGconn *conn, int msgLength)
 	}
 
 	/* Success!  Store the completed tuple in the result */
-	if (!pqAddTuple(result, tup))
-		goto outOfMemory;
+	if (!result->storeHandler(result, PQSF_ADD_TUPLE, 0, 0))
+		goto addTupleError;
+	
 	/* and reset for a new message */
 	conn->curTuple = NULL;
 
 	return 0;
 
-outOfMemory:
+addTupleError:
 
 	/*
 	 * Replace partially constructed result with an error result. First
 	 * discard the old result to try to win back some memory.
 	 */
 	pqClearAsyncResult(conn);
-	printfPQExpBuffer(&conn->errorMessage,
-					  libpq_gettext("out of memory for query result\n"));
+	resetPQExpBuffer(&conn->errorMessage);
+
+	/*
+	 * If error message is passed from addTupleFunc, set it into
+	 * PGconn, assume out of memory if not.
+	 */
+	appendPQExpBufferStr(&conn->errorMessage,
+						 libpq_gettext(result->storeHandlerErrMes ?
+									   result->storeHandlerErrMes : 
+									   "out of memory for query result\n"));
+	if (result->storeHandlerErrMes)
+	{
+		free(result->storeHandlerErrMes);
+		result->storeHandlerErrMes = NULL;
+	}
 	pqSaveErrorResult(conn);
 
 	/* Discard the failed message by pretending we read it */
diff --git a/src/interfaces/libpq/libpq-fe.h b/src/interfaces/libpq/libpq-fe.h
index ef26ab9..6d86fa0 100644
--- a/src/interfaces/libpq/libpq-fe.h
+++ b/src/interfaces/libpq/libpq-fe.h
@@ -116,6 +116,16 @@ typedef enum
 	PQPING_NO_ATTEMPT			/* connection not attempted (bad params) */
 } PGPing;
 
+/* PQStoreFunc is the enum for one of the parameters of storeHandler
+ * that decides what to do. See the typedef StoreHandler for
+ * details */
+typedef enum 
+{
+	PQSF_ALLOC_TEXT,          /* Requested non-aligned memory for text value */
+	PQSF_ALLOC_BINARY,        /* Requested aligned memory for binary value */
+	PQSF_ADD_TUPLE            /* Requested to add tuple data into store */
+} PQStoreFunc;
+
 /* PGconn encapsulates a connection to the backend.
  * The contents of this struct are not supposed to be known to applications.
  */
@@ -149,6 +159,15 @@ typedef struct pgNotify
 	struct pgNotify *next;		/* list link */
 } PGnotify;
 
+/* PGresAttValue represents a value of one tuple field in string form.
+   NULL is represented as len < 0. Otherwise value points to a null
+   terminated C string with the length of len. */
+typedef struct pgresAttValue
+{
+	int			len;			/* length in bytes of the value */
+	char	   *value;			/* actual value, plus terminating zero byte */
+} PGresAttValue;
+
 /* Function types for notice-handling callbacks */
 typedef void (*PQnoticeReceiver) (void *arg, const PGresult *res);
 typedef void (*PQnoticeProcessor) (void *arg, const char *message);
@@ -416,6 +435,52 @@ extern PGPing PQping(const char *conninfo);
 extern PGPing PQpingParams(const char *const * keywords,
 			 const char *const * values, int expand_dbname);
 
+/*
+ * Typedef for alternative result store handler.
+ *
+ * This function pointer is used for alternative result store handler
+ * callback in PGresult and PGconn.
+ *
+ * StoreHandler is called for three functions designated by the enum
+ * PQStoreFunc.
+ *
+ * id is the identifier for allocated memory block. The caller sets -1
+ * for PGresAttValue array, and 0 to number of cols - 1 for each
+ * column.
+ *
+ * PQSF_ALLOC_TEXT requests the size bytes memory block for a text
+ * value which may not be alingned to the word boundary.
+ *
+ * PQSF_ALLOC_BINARY requests the size bytes memory block for a binary
+ * value which is aligned to the word boundary.
+ *
+ * PQSF_ADD_TUPLE requests to add tuple data into the result store,
+ * and free the memory blocks allocated by this function if necessary.
+ * id and size are to be ignored for this function.
+ *
+ * This function must return non-NULL value for success and must
+ * return NULL for failure and may set error message by
+ * PQsetStoreHandlerErrMes. It is assumed by caller as out of memory
+ * when the error message is NULL on failure. This function is assumed
+ * not to throw any exception.
+ */
+typedef void *(*StoreHandler)(PGresult *res, PQStoreFunc func,
+							  int id, size_t size);
+
+/*
+ * Register alternative result store function to PGconn.
+ * 
+ * By registering this function, pg_result disables its own result
+ * store and calls it to append rows one by one.
+ *
+ * func is tuple store function. See the typedef StoreHandler.
+ * 
+ * storeHandlerParam is the contextual variable that can be get with
+ * PQgetStoreHandlerParam in StoreHandler.
+ */
+extern void PQregisterStoreHandler(PGconn *conn, StoreHandler func,
+								   void *storeHandlerParam);
+
 /* Force the write buffer to be written (or at least try) */
 extern int	PQflush(PGconn *conn);
 
@@ -454,6 +519,8 @@ extern char *PQcmdTuples(PGresult *res);
 extern char *PQgetvalue(const PGresult *res, int tup_num, int field_num);
 extern int	PQgetlength(const PGresult *res, int tup_num, int field_num);
 extern int	PQgetisnull(const PGresult *res, int tup_num, int field_num);
+extern void *PQgetStoreHandlerParam(const PGresult *res);
+extern void	PQsetStoreHandlerErrMes(PGresult *res, char *mes);
 extern int	PQnparams(const PGresult *res);
 extern Oid	PQparamtype(const PGresult *res, int param_num);
 
diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h
index d967d60..e28e712 100644
--- a/src/interfaces/libpq/libpq-int.h
+++ b/src/interfaces/libpq/libpq-int.h
@@ -134,12 +134,6 @@ typedef struct pgresParamDesc
 
 #define NULL_LEN		(-1)	/* pg_result len for NULL value */
 
-typedef struct pgresAttValue
-{
-	int			len;			/* length in bytes of the value */
-	char	   *value;			/* actual value, plus terminating zero byte */
-} PGresAttValue;
-
 /* Typedef for message-field list entries */
 typedef struct pgMessageField
 {
@@ -209,6 +203,11 @@ struct pg_result
 	PGresult_data *curBlock;	/* most recently allocated block */
 	int			curOffset;		/* start offset of free space in block */
 	int			spaceLeft;		/* number of free bytes remaining in block */
+
+	StoreHandler storeHandler;  /* Result store handler. See
+								 * StoreHandler for details. */
+	void *storeHandlerParam;    /* Contextual parameter for storeHandler */
+	char *storeHandlerErrMes;   /* Error message from storeHandler */
 };
 
 /* PGAsyncStatusType defines the state of the query-execution state machine */
@@ -443,6 +442,13 @@ struct pg_conn
 
 	/* Buffer for receiving various parts of messages */
 	PQExpBufferData workBuffer; /* expansible string */
+
+    /* Tuple store handler. The two fields below is copied to newly
+	 * created PGresult if tupStoreHandler is not NULL. Use default
+	 * function if NULL. */
+	StoreHandler storeHandler;   /* Result store handler. See
+								  * StoreHandler for details. */
+	void *storeHandlerParam;  /* Contextual parameter for storeHandler */
 };
 
 /* PGcancel stores all data necessary to cancel a connection. A copy of this
@@ -507,7 +513,6 @@ extern void
 pqInternalNotice(const PGNoticeHooks *hooks, const char *fmt,...)
 /* This lets gcc check the format string for consistency. */
 __attribute__((format(PG_PRINTF_ATTRIBUTE, 2, 3)));
-extern int	pqAddTuple(PGresult *res, PGresAttValue *tup);
 extern void pqSaveMessageField(PGresult *res, char code,
 				   const char *value);
 extern void pqSaveParameterStatus(PGconn *conn, const char *name,