fast_copy_V14_8.1.patch
application/octet-stream
Filename: fast_copy_V14_8.1.patch
Type: application/octet-stream
Part: 0
Patch
Format: context
| File | + | − |
|---|---|---|
| src/backend/commands/copy.c | 1592 | 305 |
Index: src/backend/commands/copy.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/backend/commands/copy.c,v
retrieving revision 1.245
diff -c -r1.245 copy.c
*** src/backend/commands/copy.c 2 Jun 2005 01:21:22 -0000 1.245
--- src/backend/commands/copy.c 28 Jun 2005 04:35:09 -0000
***************
*** 51,56 ****
--- 51,57 ----
#define ISOCTAL(c) (((c) >= '0') && ((c) <= '7'))
#define OCTVALUE(c) ((c) - '0')
+ #define COPY_BUF_SIZE 65536
/*
* Represents the different source/dest cases we need to worry about at
***************
*** 61,67 ****
COPY_FILE, /* to/from file */
COPY_OLD_FE, /* to/from frontend (2.0 protocol) */
COPY_NEW_FE /* to/from frontend (3.0 protocol) */
! } CopyDest;
/*
* State indicator showing what stopped CopyReadAttribute()
--- 62,68 ----
COPY_FILE, /* to/from file */
COPY_OLD_FE, /* to/from frontend (2.0 protocol) */
COPY_NEW_FE /* to/from frontend (3.0 protocol) */
! } CopyDest;
/*
* State indicator showing what stopped CopyReadAttribute()
***************
*** 82,89 ****
EOL_NL,
EOL_CR,
EOL_CRNL
! } EolType;
!
static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0";
--- 83,89 ----
EOL_NL,
EOL_CR,
EOL_CRNL
! } EolType;
static const char BinarySignature[11] = "PGCOPY\n\377\r\n\0";
***************
*** 92,97 ****
--- 92,101 ----
* never been reentrant...
*/
static CopyDest copy_dest;
+ static int eol_ch[2]; /* The byte values of the 1 or 2 eol bytes */
+ static char escapec; /* escape char for delimited data format. */
+ static bool client_encoding_only; /* true if client encoding is a non
+ * supported server encoding */
static FILE *copy_file; /* used if copy_dest == COPY_FILE */
static StringInfo copy_msgbuf; /* used if copy_dest == COPY_NEW_FE */
static bool fe_eof; /* true if detected end of copy data */
***************
*** 105,144 ****
static int copy_lineno; /* line number for error messages */
static const char *copy_attname; /* current att for error messages */
/*
* These static variables are used to avoid incurring overhead for each
! * attribute processed. attribute_buf is reused on each CopyReadAttribute
* call to hold the string being read in. Under normal use it will soon
* grow to a suitable size, and then we will avoid palloc/pfree overhead
* for subsequent attributes. Note that CopyReadAttribute returns a pointer
! * to attribute_buf's data buffer!
*/
! static StringInfoData attribute_buf;
/*
* Similarly, line_buf holds the whole input line being processed (its
* cursor field points to the next character to be read by CopyReadAttribute).
* The input cycle is first to read the whole line into line_buf, convert it
* to server encoding, and then extract individual attribute fields into
! * attribute_buf. (We used to have CopyReadAttribute read the input source
* directly, but that caused a lot of encoding issues and unnecessary logic
* complexity.)
*/
! static StringInfoData line_buf;
static bool line_buf_converted;
/* non-export function prototypes */
static void DoCopyTo(Relation rel, List *attnumlist, bool binary, bool oids,
char *delim, char *null_print, bool csv_mode, char *quote,
! char *escape, List *force_quote_atts, bool header_line, bool fe_copy);
static void CopyTo(Relation rel, List *attnumlist, bool binary, bool oids,
! char *delim, char *null_print, bool csv_mode, char *quote, char *escape,
List *force_quote_atts, bool header_line);
static void CopyFrom(Relation rel, List *attnumlist, bool binary, bool oids,
! char *delim, char *null_print, bool csv_mode, char *quote, char *escape,
List *force_notnull_atts, bool header_line);
! static bool CopyReadLine(char * quote, char * escape);
static char *CopyReadAttribute(const char *delim, const char *null_print,
CopyReadResult *result, bool *isnull);
static char *CopyReadAttributeCSV(const char *delim, const char *null_print,
--- 109,162 ----
static int copy_lineno; /* line number for error messages */
static const char *copy_attname; /* current att for error messages */
+ /* Static variables for buffered input parsing */
+ /* leave room for extra null (to enable use of string functions) */
+ static char input_buf[COPY_BUF_SIZE + 1];
+ static char *begloc;
+ static char *endloc;
+ static bool esc_in_prevbuf; /* backslash was last character of the data
+ * input buffer */
+ static bool cr_in_prevbuf; /* CR was last character of the data input
+ * buffer */
+ static bool line_done; /* finished processing the whole line or
+ * stopped in the middle */
+ static bool buf_done; /* finished processing the current buffer */
+ static int buffer_index; /* input buffer index */
+
/*
* These static variables are used to avoid incurring overhead for each
! * attribute processed. attr_buf is reused on each CopyReadAttribute
* call to hold the string being read in. Under normal use it will soon
* grow to a suitable size, and then we will avoid palloc/pfree overhead
* for subsequent attributes. Note that CopyReadAttribute returns a pointer
! * to attr_buf's data buffer!
*/
! static StringInfoData attr_buf; /* still used in CopyFrom() */
/*
* Similarly, line_buf holds the whole input line being processed (its
* cursor field points to the next character to be read by CopyReadAttribute).
* The input cycle is first to read the whole line into line_buf, convert it
* to server encoding, and then extract individual attribute fields into
! * attr_buf. (We used to have CopyReadAttribute read the input source
* directly, but that caused a lot of encoding issues and unnecessary logic
* complexity.)
*/
! static StringInfoData line_buf; /* still used in CopyFromBinary/CSV() */
static bool line_buf_converted;
/* non-export function prototypes */
static void DoCopyTo(Relation rel, List *attnumlist, bool binary, bool oids,
char *delim, char *null_print, bool csv_mode, char *quote,
! char *escape, List *force_quote_atts, bool header_line, bool fe_copy);
static void CopyTo(Relation rel, List *attnumlist, bool binary, bool oids,
! char *delim, char *null_print, bool csv_mode, char *quote, char *escape,
List *force_quote_atts, bool header_line);
static void CopyFrom(Relation rel, List *attnumlist, bool binary, bool oids,
! char *delim, char *null_print, bool csv_mode, char *quote, char *escape,
List *force_notnull_atts, bool header_line);
! static bool CopyReadLine(char *quote, char *escape);
static char *CopyReadAttribute(const char *delim, const char *null_print,
CopyReadResult *result, bool *isnull);
static char *CopyReadAttributeCSV(const char *delim, const char *null_print,
***************
*** 151,156 ****
--- 169,187 ----
char *escape, bool force_quote);
static List *CopyGetAttnums(Relation rel, List *attnamelist);
static void limit_printout_length(StringInfo buf);
+ static bool CopyReadLineBuffered(size_t bytesread);
+ static void CopyFromDelimited(Relation rel, List *attnumlist, bool binary, bool oids,
+ char *delim, char *null_print, bool csv_mode, char *quote, char *escape,
+ List *force_notnull_atts);
+ static char *CopyReadAttribute(const char *delim, const char *null_print,
+ CopyReadResult *result, bool *isnull);
+ static void CopyReadAllAttrs(const char *delim, const char *null_print, int null_print_len,
+ char *nulls, List *attnumlist, int *attr_offsets, int num_phys_attrs,
+ Form_pg_attribute *attr);
+ static char *CopyReadOidAttr(const char *delim, const char *null_print, int null_print_len,
+ CopyReadResult *result, bool *isnull);
+ static bool DetectLineEnd(size_t bytesread);
+
/* Internal communications functions */
static void SendCopyBegin(bool binary, int natts);
***************
*** 160,166 ****
static void CopySendString(const char *str);
static void CopySendChar(char c);
static void CopySendEndOfRow(bool binary);
! static void CopyGetData(void *databuf, int datasize);
static int CopyGetChar(void);
#define CopyGetEof() (fe_eof)
--- 191,197 ----
static void CopySendString(const char *str);
static void CopySendChar(char c);
static void CopySendEndOfRow(bool binary);
! static int CopyGetData(void *databuf, int datasize);
static int CopyGetChar(void);
#define CopyGetEof() (fe_eof)
***************
*** 171,176 ****
--- 202,210 ----
static void CopySendInt16(int16 val);
static int16 CopyGetInt16(void);
+ /* new parsing utils */
+ static char *strchrlen(const char *s, int c, size_t len);
+ static char *str2chrlen(const char *s, int c1, int c2, size_t len, char *c_found);
/*
* Send copy start/stop messages for frontend copies. These have changed
***************
*** 201,207 ****
if (binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
! errmsg("COPY BINARY is not supported to stdout or from stdin")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
--- 235,241 ----
if (binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
! errmsg("COPY BINARY is not supported to stdout or from stdin")));
pq_putemptymessage('H');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
***************
*** 213,219 ****
if (binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
! errmsg("COPY BINARY is not supported to stdout or from stdin")));
pq_putemptymessage('B');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
--- 247,253 ----
if (binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
! errmsg("COPY BINARY is not supported to stdout or from stdin")));
pq_putemptymessage('B');
/* grottiness needed for old COPY OUT protocol */
pq_startcopyout();
***************
*** 246,252 ****
if (binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
! errmsg("COPY BINARY is not supported to stdout or from stdin")));
pq_putemptymessage('G');
copy_dest = COPY_OLD_FE;
}
--- 280,286 ----
if (binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
! errmsg("COPY BINARY is not supported to stdout or from stdin")));
pq_putemptymessage('G');
copy_dest = COPY_OLD_FE;
}
***************
*** 256,262 ****
if (binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
! errmsg("COPY BINARY is not supported to stdout or from stdin")));
pq_putemptymessage('D');
copy_dest = COPY_OLD_FE;
}
--- 290,296 ----
if (binary)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
! errmsg("COPY BINARY is not supported to stdout or from stdin")));
pq_putemptymessage('D');
copy_dest = COPY_OLD_FE;
}
***************
*** 317,323 ****
/* no hope of recovering connection sync, so FATAL */
ereport(FATAL,
(errcode(ERRCODE_CONNECTION_FAILURE),
! errmsg("connection lost during COPY to stdout")));
}
break;
case COPY_NEW_FE:
--- 351,357 ----
/* no hope of recovering connection sync, so FATAL */
ereport(FATAL,
(errcode(ERRCODE_CONNECTION_FAILURE),
! errmsg("connection lost during COPY to stdout")));
}
break;
case COPY_NEW_FE:
***************
*** 383,396 ****
* It seems unwise to allow the COPY IN to complete normally in that case.
*
* NB: no data conversion is applied by these functions
*/
! static void
CopyGetData(void *databuf, int datasize)
{
switch (copy_dest)
{
case COPY_FILE:
! fread(databuf, datasize, 1, copy_file);
if (feof(copy_file))
fe_eof = true;
break;
--- 417,435 ----
* It seems unwise to allow the COPY IN to complete normally in that case.
*
* NB: no data conversion is applied by these functions
+ *
+ * Returns: the number of bytes that were successfully read
+ * into the data buffer.
*/
! static int
CopyGetData(void *databuf, int datasize)
{
+ size_t bytesread = 0;
+
switch (copy_dest)
{
case COPY_FILE:
! bytesread = fread(databuf, 1, datasize, copy_file);
if (feof(copy_file))
fe_eof = true;
break;
***************
*** 402,407 ****
--- 441,448 ----
(errcode(ERRCODE_CONNECTION_FAILURE),
errmsg("unexpected EOF on client connection")));
}
+ bytesread += datasize; /* update the count of bytes that were
+ * read so far */
break;
case COPY_NEW_FE:
while (datasize > 0 && !fe_eof)
***************
*** 418,428 ****
if (mtype == EOF)
ereport(ERROR,
(errcode(ERRCODE_CONNECTION_FAILURE),
! errmsg("unexpected EOF on client connection")));
if (pq_getmessage(copy_msgbuf, 0))
ereport(ERROR,
(errcode(ERRCODE_CONNECTION_FAILURE),
! errmsg("unexpected EOF on client connection")));
switch (mtype)
{
case 'd': /* CopyData */
--- 459,469 ----
if (mtype == EOF)
ereport(ERROR,
(errcode(ERRCODE_CONNECTION_FAILURE),
! errmsg("unexpected EOF on client connection")));
if (pq_getmessage(copy_msgbuf, 0))
ereport(ERROR,
(errcode(ERRCODE_CONNECTION_FAILURE),
! errmsg("unexpected EOF on client connection")));
switch (mtype)
{
case 'd': /* CopyData */
***************
*** 430,441 ****
case 'c': /* CopyDone */
/* COPY IN correctly terminated by frontend */
fe_eof = true;
! return;
case 'f': /* CopyFail */
ereport(ERROR,
(errcode(ERRCODE_QUERY_CANCELED),
errmsg("COPY from stdin failed: %s",
! pq_getmsgstring(copy_msgbuf))));
break;
case 'H': /* Flush */
case 'S': /* Sync */
--- 471,482 ----
case 'c': /* CopyDone */
/* COPY IN correctly terminated by frontend */
fe_eof = true;
! return bytesread;
case 'f': /* CopyFail */
ereport(ERROR,
(errcode(ERRCODE_QUERY_CANCELED),
errmsg("COPY from stdin failed: %s",
! pq_getmsgstring(copy_msgbuf))));
break;
case 'H': /* Flush */
case 'S': /* Sync */
***************
*** 460,469 ****
--- 501,514 ----
avail = datasize;
pq_copymsgbytes(copy_msgbuf, databuf, avail);
databuf = (void *) ((char *) databuf + avail);
+ bytesread += avail; /* update the count of bytes that were
+ * read so far */
datasize -= avail;
}
break;
}
+
+ return bytesread;
}
static int
***************
*** 695,701 ****
bool binary = false;
bool oids = false;
bool csv_mode = false;
! bool header_line = false;
char *delim = NULL;
char *quote = NULL;
char *escape = NULL;
--- 740,746 ----
bool binary = false;
bool oids = false;
bool csv_mode = false;
! bool header_line = false;
char *delim = NULL;
char *quote = NULL;
char *escape = NULL;
***************
*** 828,840 ****
escape = quote;
}
/* Only single-character delimiter strings are supported. */
if (strlen(delim) != 1)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("COPY delimiter must be a single character")));
! /* Check header */
if (!csv_mode && header_line)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
--- 873,887 ----
escape = quote;
}
+ escapec = '\\'; /* will be configurable in the future */
+
/* Only single-character delimiter strings are supported. */
if (strlen(delim) != 1)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("COPY delimiter must be a single character")));
! /* Check header */
if (!csv_mode && header_line)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
***************
*** 870,892 ****
if (force_quote != NIL && is_from)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
! errmsg("COPY force quote only available using COPY TO")));
/* Check force_notnull */
if (!csv_mode && force_notnull != NIL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
! errmsg("COPY force not null available only in CSV mode")));
if (force_notnull != NIL && !is_from)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
! errmsg("COPY force not null only available using COPY FROM")));
/* Don't allow the delimiter to appear in the null string. */
if (strchr(null_print, delim[0]) != NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
! errmsg("COPY delimiter must not appear in the NULL specification")));
/* Don't allow the csv quote char to appear in the null string. */
if (csv_mode && strchr(null_print, quote[0]) != NULL)
--- 917,939 ----
if (force_quote != NIL && is_from)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
! errmsg("COPY force quote only available using COPY TO")));
/* Check force_notnull */
if (!csv_mode && force_notnull != NIL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
! errmsg("COPY force not null available only in CSV mode")));
if (force_notnull != NIL && !is_from)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
! errmsg("COPY force not null only available using COPY FROM")));
/* Don't allow the delimiter to appear in the null string. */
if (strchr(null_print, delim[0]) != NULL)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
! errmsg("COPY delimiter must not appear in the NULL specification")));
/* Don't allow the csv quote char to appear in the null string. */
if (csv_mode && strchr(null_print, quote[0]) != NULL)
***************
*** 914,920 ****
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
errmsg("must be superuser to COPY to or from a file"),
errhint("Anyone can COPY to stdout or from stdin. "
! "psql's \\copy command also works for anyone.")));
/* Don't allow COPY w/ OIDs to or from a table without them */
if (oids && !rel->rd_rel->relhasoids)
--- 961,967 ----
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
errmsg("must be superuser to COPY to or from a file"),
errhint("Anyone can COPY to stdout or from stdin. "
! "psql's \\copy command also works for anyone.")));
/* Don't allow COPY w/ OIDs to or from a table without them */
if (oids && !rel->rd_rel->relhasoids)
***************
*** 942,949 ****
if (!list_member_int(attnumlist, attnum))
ereport(ERROR,
(errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
! errmsg("FORCE QUOTE column \"%s\" not referenced by COPY",
! NameStr(attr[attnum - 1]->attname))));
}
}
--- 989,996 ----
if (!list_member_int(attnumlist, attnum))
ereport(ERROR,
(errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
! errmsg("FORCE QUOTE column \"%s\" not referenced by COPY",
! NameStr(attr[attnum - 1]->attname))));
}
}
***************
*** 963,981 ****
if (!list_member_int(attnumlist, attnum))
ereport(ERROR,
(errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
! errmsg("FORCE NOT NULL column \"%s\" not referenced by COPY",
! NameStr(attr[attnum - 1]->attname))));
}
}
/* Set up variables to avoid per-attribute overhead. */
! initStringInfo(&attribute_buf);
initStringInfo(&line_buf);
line_buf_converted = false;
client_encoding = pg_get_client_encoding();
server_encoding = GetDatabaseEncoding();
copy_dest = COPY_FILE; /* default */
copy_file = NULL;
copy_msgbuf = NULL;
--- 1010,1046 ----
if (!list_member_int(attnumlist, attnum))
ereport(ERROR,
(errcode(ERRCODE_INVALID_COLUMN_REFERENCE),
! errmsg("FORCE NOT NULL column \"%s\" not referenced by COPY",
! NameStr(attr[attnum - 1]->attname))));
}
}
/* Set up variables to avoid per-attribute overhead. */
! initStringInfo(&attr_buf);
initStringInfo(&line_buf);
line_buf_converted = false;
client_encoding = pg_get_client_encoding();
server_encoding = GetDatabaseEncoding();
+ /*
+ * check if the client encoding is one of the 5 encodings
+ * that are not supported as a server encodings.
+ */
+ switch (client_encoding)
+ {
+ case PG_SJIS:
+ case PG_BIG5:
+ case PG_GBK:
+ case PG_UHC:
+ case PG_GB18030:
+ client_encoding_only = true;
+ break;
+ default:
+ client_encoding_only = false;
+ }
+
+
copy_dest = COPY_FILE; /* default */
copy_file = NULL;
copy_msgbuf = NULL;
***************
*** 998,1005 ****
else
ereport(ERROR,
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
! errmsg("cannot copy to non-table relation \"%s\"",
! RelationGetRelationName(rel))));
}
if (pipe)
{
--- 1063,1070 ----
else
ereport(ERROR,
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
! errmsg("cannot copy to non-table relation \"%s\"",
! RelationGetRelationName(rel))));
}
if (pipe)
{
***************
*** 1017,1024 ****
if (copy_file == NULL)
ereport(ERROR,
(errcode_for_file_access(),
! errmsg("could not open file \"%s\" for reading: %m",
! filename)));
fstat(fileno(copy_file), &st);
if (S_ISDIR(st.st_mode))
--- 1082,1089 ----
if (copy_file == NULL)
ereport(ERROR,
(errcode_for_file_access(),
! errmsg("could not open file \"%s\" for reading: %m",
! filename)));
fstat(fileno(copy_file), &st);
if (S_ISDIR(st.st_mode))
***************
*** 1029,1036 ****
errmsg("\"%s\" is a directory", filename)));
}
}
! CopyFrom(rel, attnumlist, binary, oids, delim, null_print, csv_mode,
! quote, escape, force_notnull_atts, header_line);
}
else
{ /* copy from database to file */
--- 1094,1112 ----
errmsg("\"%s\" is a directory", filename)));
}
}
!
! if (csv_mode || binary) /* old path */
! CopyFrom(rel, attnumlist, binary, oids, delim, null_print, csv_mode,
! quote, escape, force_notnull_atts, header_line);
!
! /*
! * new path for improved performance (only for delimited format for
! * now)
! */
! else
! CopyFromDelimited(rel, attnumlist, binary, oids, delim, null_print, csv_mode,
! quote, escape, force_notnull_atts);
!
}
else
{ /* copy from database to file */
***************
*** 1049,1056 ****
else
ereport(ERROR,
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
! errmsg("cannot copy from non-table relation \"%s\"",
! RelationGetRelationName(rel))));
}
if (pipe)
{
--- 1125,1132 ----
else
ereport(ERROR,
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
! errmsg("cannot copy from non-table relation \"%s\"",
! RelationGetRelationName(rel))));
}
if (pipe)
{
***************
*** 1071,1077 ****
if (!is_absolute_path(filename))
ereport(ERROR,
(errcode(ERRCODE_INVALID_NAME),
! errmsg("relative path not allowed for COPY to file")));
oumask = umask((mode_t) 022);
copy_file = AllocateFile(filename, PG_BINARY_W);
--- 1147,1153 ----
if (!is_absolute_path(filename))
ereport(ERROR,
(errcode(ERRCODE_INVALID_NAME),
! errmsg("relative path not allowed for COPY to file")));
oumask = umask((mode_t) 022);
copy_file = AllocateFile(filename, PG_BINARY_W);
***************
*** 1080,1087 ****
if (copy_file == NULL)
ereport(ERROR,
(errcode_for_file_access(),
! errmsg("could not open file \"%s\" for writing: %m",
! filename)));
fstat(fileno(copy_file), &st);
if (S_ISDIR(st.st_mode))
--- 1156,1163 ----
if (copy_file == NULL)
ereport(ERROR,
(errcode_for_file_access(),
! errmsg("could not open file \"%s\" for writing: %m",
! filename)));
fstat(fileno(copy_file), &st);
if (S_ISDIR(st.st_mode))
***************
*** 1106,1114 ****
errmsg("could not write to file \"%s\": %m",
filename)));
}
! pfree(attribute_buf.data);
pfree(line_buf.data);
/*
* Close the relation. If reading, we can release the AccessShareLock
* we got; if writing, we should hold the lock until end of
--- 1182,1191 ----
errmsg("could not write to file \"%s\": %m",
filename)));
}
! pfree(attr_buf.data);
pfree(line_buf.data);
+
/*
* Close the relation. If reading, we can release the AccessShareLock
* we got; if writing, we should hold the lock until end of
***************
*** 1245,1253 ****
/* if a header has been requested send the line */
if (header_line)
{
! bool hdr_delim = false;
! char *colname;
!
foreach(cur, attnumlist)
{
int attnum = lfirst_int(cur);
--- 1322,1330 ----
/* if a header has been requested send the line */
if (header_line)
{
! bool hdr_delim = false;
! char *colname;
!
foreach(cur, attnumlist)
{
int attnum = lfirst_int(cur);
***************
*** 1298,1304 ****
if (oids)
{
string = DatumGetCString(DirectFunctionCall1(oidout,
! ObjectIdGetDatum(HeapTupleGetOid(tuple))));
CopySendString(string);
need_delim = true;
}
--- 1375,1381 ----
if (oids)
{
string = DatumGetCString(DirectFunctionCall1(oidout,
! ObjectIdGetDatum(HeapTupleGetOid(tuple))));
CopySendString(string);
need_delim = true;
}
***************
*** 1335,1342 ****
if (csv_mode)
{
CopyAttributeOutCSV(string, delim, quote, escape,
! (strcmp(string, null_print) == 0 ||
! force_quote[attnum - 1]));
}
else
CopyAttributeOut(string, delim);
--- 1412,1419 ----
if (csv_mode)
{
CopyAttributeOutCSV(string, delim, quote, escape,
! (strcmp(string, null_print) == 0 ||
! force_quote[attnum - 1]));
}
else
CopyAttributeOut(string, delim);
***************
*** 1346,1353 ****
{
bytea *outputbytes;
! outputbytes = DatumGetByteaP(FunctionCall1(&out_functions[attnum - 1],
! value));
/* We assume the result will not have been toasted */
CopySendInt32(VARSIZE(outputbytes) - VARHDRSZ);
CopySendData(VARDATA(outputbytes),
--- 1423,1431 ----
{
bytea *outputbytes;
! outputbytes =
! DatumGetByteaP(FunctionCall1(&out_functions[attnum - 1],
! value));
/* We assume the result will not have been toasted */
CopySendInt32(VARSIZE(outputbytes) - VARHDRSZ);
CopySendData(VARDATA(outputbytes),
***************
*** 1396,1405 ****
if (copy_attname)
{
/* error is relevant to a particular column */
! limit_printout_length(&attribute_buf);
errcontext("COPY %s, line %d, column %s: \"%s\"",
copy_relname, copy_lineno, copy_attname,
! attribute_buf.data);
}
else
{
--- 1474,1483 ----
if (copy_attname)
{
/* error is relevant to a particular column */
! limit_printout_length(&attr_buf);
errcontext("COPY %s, line %d, column %s: \"%s\"",
copy_relname, copy_lineno, copy_attname,
! attr_buf.data);
}
else
{
***************
*** 1407,1412 ****
--- 1485,1492 ----
if (line_buf_converted ||
client_encoding == server_encoding)
{
+ /* Strip off the newline */
+ *(line_buf.data + line_buf.len - 1) = '\0';
limit_printout_length(&line_buf);
errcontext("COPY %s, line %d: \"%s\"",
copy_relname, copy_lineno,
***************
*** 1417,1423 ****
/*
* Here, the line buffer is still in a foreign encoding,
* and indeed it's quite likely that the error is precisely
! * a failure to do encoding conversion (ie, bad data). We
* dare not try to convert it, and at present there's no way
* to regurgitate it without conversion. So we have to punt
* and just report the line number.
--- 1497,1503 ----
/*
* Here, the line buffer is still in a foreign encoding,
* and indeed it's quite likely that the error is precisely
! * a failure to do encoding conversion (ie, bad data). We
* dare not try to convert it, and at present there's no way
* to regurgitate it without conversion. So we have to punt
* and just report the line number.
***************
*** 1460,1465 ****
--- 1540,1546 ----
appendStringInfoString(buf, "...");
}
+
/*
* Copy FROM file to relation.
*/
***************
*** 1551,1557 ****
/* Fetch the input function and typioparam info */
if (binary)
getTypeBinaryInputInfo(attr[attnum - 1]->atttypid,
! &in_func_oid, &typioparams[attnum - 1]);
else
getTypeInputInfo(attr[attnum - 1]->atttypid,
&in_func_oid, &typioparams[attnum - 1]);
--- 1632,1638 ----
/* Fetch the input function and typioparam info */
if (binary)
getTypeBinaryInputInfo(attr[attnum - 1]->atttypid,
! &in_func_oid, &typioparams[attnum - 1]);
else
getTypeInputInfo(attr[attnum - 1]->atttypid,
&in_func_oid, &typioparams[attnum - 1]);
***************
*** 1645,1657 ****
if ((tmp >> 16) != 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unrecognized critical flags in COPY file header")));
/* Header extension length */
tmp = CopyGetInt32();
if (CopyGetEof() || tmp < 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("invalid COPY file header (missing length)")));
/* Skip extension header, if present */
while (tmp-- > 0)
{
--- 1726,1738 ----
if ((tmp >> 16) != 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unrecognized critical flags in COPY file header")));
/* Header extension length */
tmp = CopyGetInt32();
if (CopyGetEof() || tmp < 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("invalid COPY file header (missing length)")));
/* Skip extension header, if present */
while (tmp-- > 0)
{
***************
*** 1659,1665 ****
if (CopyGetEof())
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("invalid COPY file header (wrong length)")));
}
}
--- 1740,1746 ----
if (CopyGetEof())
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("invalid COPY file header (wrong length)")));
}
}
***************
*** 1696,1702 ****
if (header_line)
{
copy_lineno++;
! done = CopyReadLine(quote, escape) ;
}
while (!done)
--- 1777,1783 ----
if (header_line)
{
copy_lineno++;
! done = CopyReadLine(quote, escape);
}
while (!done)
***************
*** 1725,1731 ****
ListCell *cur;
/* Actually read the line into memory here */
! done = csv_mode ?
CopyReadLine(quote, escape) : CopyReadLine(NULL, NULL);
/*
--- 1806,1812 ----
ListCell *cur;
/* Actually read the line into memory here */
! done = csv_mode ?
CopyReadLine(quote, escape) : CopyReadLine(NULL, NULL);
/*
***************
*** 1751,1757 ****
{
copy_attname = "oid";
loaded_oid = DatumGetObjectId(DirectFunctionCall1(oidin,
! CStringGetDatum(string)));
if (loaded_oid == InvalidOid)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
--- 1832,1838 ----
{
copy_attname = "oid";
loaded_oid = DatumGetObjectId(DirectFunctionCall1(oidin,
! CStringGetDatum(string)));
if (loaded_oid == InvalidOid)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
***************
*** 1779,1789 ****
if (csv_mode)
{
string = CopyReadAttributeCSV(delim, null_print, quote,
! escape, &result, &isnull);
if (result == UNTERMINATED_FIELD)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unterminated CSV quoted field")));
}
else
string = CopyReadAttribute(delim, null_print,
--- 1860,1870 ----
if (csv_mode)
{
string = CopyReadAttributeCSV(delim, null_print, quote,
! escape, &result, &isnull);
if (result == UNTERMINATED_FIELD)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unterminated CSV quoted field")));
}
else
string = CopyReadAttribute(delim, null_print,
***************
*** 1801,1808 ****
copy_attname = NameStr(attr[m]->attname);
values[m] = FunctionCall3(&in_functions[m],
CStringGetDatum(string),
! ObjectIdGetDatum(typioparams[m]),
! Int32GetDatum(attr[m]->atttypmod));
nulls[m] = ' ';
copy_attname = NULL;
}
--- 1882,1889 ----
copy_attname = NameStr(attr[m]->attname);
values[m] = FunctionCall3(&in_functions[m],
CStringGetDatum(string),
! ObjectIdGetDatum(typioparams[m]),
! Int32GetDatum(attr[m]->atttypmod));
nulls[m] = ' ';
copy_attname = NULL;
}
***************
*** 1818,1824 ****
if (result == NORMAL_ATTR && line_buf.len != 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("extra data after last expected column")));
}
else
{
--- 1899,1905 ----
if (result == NORMAL_ATTR && line_buf.len != 0)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("extra data after last expected column")));
}
else
{
***************
*** 1844,1851 ****
copy_attname = "oid";
loaded_oid =
DatumGetObjectId(CopyReadBinaryAttribute(0,
! &oid_in_function,
! oid_typioparam,
&isnull));
if (isnull || loaded_oid == InvalidOid)
ereport(ERROR,
--- 1925,1932 ----
copy_attname = "oid";
loaded_oid =
DatumGetObjectId(CopyReadBinaryAttribute(0,
! &oid_in_function,
! oid_typioparam,
&isnull));
if (isnull || loaded_oid == InvalidOid)
ereport(ERROR,
***************
*** 1924,1930 ****
/* BEFORE ROW INSERT Triggers */
if (resultRelInfo->ri_TrigDesc &&
! resultRelInfo->ri_TrigDesc->n_before_row[TRIGGER_EVENT_INSERT] > 0)
{
HeapTuple newtuple;
--- 2005,2011 ----
/* BEFORE ROW INSERT Triggers */
if (resultRelInfo->ri_TrigDesc &&
! resultRelInfo->ri_TrigDesc->n_before_row[TRIGGER_EVENT_INSERT] > 0)
{
HeapTuple newtuple;
***************
*** 1987,2173 ****
FreeExecutorState(estate);
}
-
/*
! * Read the next input line and stash it in line_buf, with conversion to
! * server encoding.
! *
! * Result is true if read was terminated by EOF, false if terminated
! * by newline.
*/
! static bool
! CopyReadLine(char * quote, char * escape)
{
! bool result;
! bool change_encoding = (client_encoding != server_encoding);
! int c;
! int mblen;
! int j;
! unsigned char s[2];
! char *cvt;
! bool in_quote = false, last_was_esc = false, csv_mode = false;
! char quotec = '\0', escapec = '\0';
! if (quote)
! {
! csv_mode = true;
! quotec = quote[0];
! escapec = escape[0];
! /* ignore special escape processing if it's the same as quotec */
! if (quotec == escapec)
! escapec = '\0';
! }
! s[1] = 0;
! /* reset line_buf to empty */
! line_buf.len = 0;
! line_buf.data[0] = '\0';
! line_buf.cursor = 0;
! /* mark that encoding conversion hasn't occurred yet */
! line_buf_converted = false;
! /* set default status */
! result = false;
/*
! * In this loop we only care for detecting newlines (\r and/or \n) and
! * the end-of-copy marker (\.).
! *
! * In Text mode, for backwards compatibility we allow
! * backslashes to escape newline characters. Backslashes other than
! * the end marker get put into the line_buf, since CopyReadAttribute
! * does its own escape processing.
! *
! * In CSV mode, CR and NL inside q quoted field are just part of the
! * data value and are put in line_buf. We keep just enough state
! * to know if we are currently in a quoted field or not.
! *
! * These four characters, and only these four, are assumed the same in
! * frontend and backend encodings.
! *
! * We do not assume that second and later bytes of a frontend
! * multibyte character couldn't look like ASCII characters.
*/
! for (;;)
{
! c = CopyGetChar();
! if (c == EOF)
! {
! result = true;
! break;
! }
! if (csv_mode)
! {
! /*
! * Dealing with quotes and escapes here is mildly tricky. If the
! * quote char is also the escape char, there's no problem - we
! * just use the char as a toggle. If they are different, we need
! * to ensure that we only take account of an escape inside a quoted
! * field and immediately preceding a quote char, and not the
! * second in a escape-escape sequence.
! */
! if (in_quote && c == escapec)
! last_was_esc = ! last_was_esc;
! if (c == quotec && ! last_was_esc)
! in_quote = ! in_quote;
! if (c != escapec)
! last_was_esc = false;
! /*
! * updating the line count for embedded CR and/or LF chars is
! * necessarily a little fragile - this test is probably about
! * the best we can do.
! */
! if (in_quote && c == (eol_type == EOL_CR ? '\r' : '\n'))
! copy_lineno++;
! }
! if (!in_quote && c == '\r')
{
! if (eol_type == EOL_NL)
! {
! if (! csv_mode)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("literal carriage return found in data"),
! errhint("Use \"\\r\" to represent carriage return.")));
! else
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unquoted carriage return found in CSV data"),
! errhint("Use quoted CSV field to represent carriage return.")));
! }
! /* Check for \r\n on first line, _and_ handle \r\n. */
! if (eol_type == EOL_UNKNOWN || eol_type == EOL_CRNL)
! {
! int c2 = CopyPeekChar();
!
! if (c2 == '\n')
! {
! CopyDonePeek(c2, true); /* eat newline */
! eol_type = EOL_CRNL;
! }
! else
! {
! /* found \r, but no \n */
! if (eol_type == EOL_CRNL)
! {
! if (!csv_mode)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("literal carriage return found in data"),
! errhint("Use \"\\r\" to represent carriage return.")));
! else
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unquoted carriage return found in data"),
! errhint("Use quoted CSV field to represent carriage return.")));
!
! }
! /*
! * if we got here, it is the first line and we didn't
! * get \n, so put it back
! */
! CopyDonePeek(c2, false);
! eol_type = EOL_CR;
! }
! }
! break;
! }
! if (!in_quote && c == '\n')
! {
! if (eol_type == EOL_CR || eol_type == EOL_CRNL)
{
! if (!csv_mode)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("literal newline found in data"),
! errhint("Use \"\\n\" to represent newline.")));
! else
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unquoted newline found in data"),
! errhint("Use quoted CSV field to represent newline.")));
!
}
- eol_type = EOL_NL;
- break;
}
! if ((line_buf.len == 0 || !csv_mode) && c == '\\')
{
! int c2;
!
! if (csv_mode)
! c2 = CopyPeekChar();
! else
! c2 = c = CopyGetChar();
if (c2 == EOF)
{
--- 2068,2710 ----
FreeExecutorState(estate);
}
/*
! * Copy FROM file to relation with faster processing.
*/
! static void
! CopyFromDelimited(Relation rel, List *attnumlist, bool binary, bool oids,
! char *delim, char *null_print, bool csv_mode, char *quote,
! char *escape, List *force_notnull_atts)
{
! HeapTuple tuple;
! TupleDesc tupDesc;
! Form_pg_attribute *attr;
! AttrNumber num_phys_attrs,
! attr_count,
! num_defaults;
! FmgrInfo *in_functions;
! Oid *typioparams;
! ExprState **constraintexprs;
! bool *force_notnull;
! bool hasConstraints = false;
! int attnum;
! int i;
! Oid in_func_oid;
! Datum *values;
! char *nulls;
! /* an array of offsets into attr_buf that point to beginning of attributes */
! int *attr_offsets;
! bool isnull;
! int null_print_len; /* length of null print */
! ResultRelInfo *resultRelInfo;
! EState *estate = CreateExecutorState(); /* for ExecConstraints() */
! TupleTableSlot *slot;
! bool file_has_oids;
! int *defmap;
! ExprState **defexprs; /* array of default att expressions */
! ExprContext *econtext; /* used for ExecEvalExpr for default atts */
! MemoryContext oldcontext = CurrentMemoryContext;
! ErrorContextCallback errcontext;
! bool no_more_data;
! CopyReadResult result;
! ListCell *cur;
! tupDesc = RelationGetDescr(rel);
! attr = tupDesc->attrs;
! num_phys_attrs = tupDesc->natts;
! attr_count = list_length(attnumlist);
! num_defaults = 0;
! /*
! * We need a ResultRelInfo so we can use the regular executor's
! * index-entry-making machinery. (There used to be a huge amount of
! * code here that basically duplicated execUtils.c ...)
! */
! resultRelInfo = makeNode(ResultRelInfo);
! resultRelInfo->ri_RangeTableIndex = 1; /* dummy */
! resultRelInfo->ri_RelationDesc = rel;
! resultRelInfo->ri_TrigDesc = CopyTriggerDesc(rel->trigdesc);
! if (resultRelInfo->ri_TrigDesc)
! resultRelInfo->ri_TrigFunctions = (FmgrInfo *)
! palloc0(resultRelInfo->ri_TrigDesc->numtriggers * sizeof(FmgrInfo));
! resultRelInfo->ri_TrigInstrument = NULL;
! ExecOpenIndices(resultRelInfo);
! estate->es_result_relations = resultRelInfo;
! estate->es_num_result_relations = 1;
! estate->es_result_relation_info = resultRelInfo;
!
! /* Set up a tuple slot too */
! slot = MakeSingleTupleTableSlot(tupDesc);
!
! econtext = GetPerTupleExprContext(estate);
/*
! * Pick up the required catalog information for each attribute in the
! * relation, including the input function, the element type (to pass
! * to the input function), and info about defaults and constraints.
! * (Which input function we use depends on text/binary format choice.)
*/
! in_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
! typioparams = (Oid *) palloc(num_phys_attrs * sizeof(Oid));
! defmap = (int *) palloc(num_phys_attrs * sizeof(int));
! defexprs = (ExprState **) palloc(num_phys_attrs * sizeof(ExprState *));
! constraintexprs = (ExprState **) palloc0(num_phys_attrs * sizeof(ExprState *));
! force_notnull = (bool *) palloc(num_phys_attrs * sizeof(bool));
!
! for (attnum = 1; attnum <= num_phys_attrs; attnum++)
{
! /* We don't need info for dropped attributes */
! if (attr[attnum - 1]->attisdropped)
! continue;
! getTypeInputInfo(attr[attnum - 1]->atttypid,
! &in_func_oid, &typioparams[attnum - 1]);
! fmgr_info(in_func_oid, &in_functions[attnum - 1]);
! if (list_member_int(force_notnull_atts, attnum))
! force_notnull[attnum - 1] = true;
! else
! force_notnull[attnum - 1] = false;
! /* Get default info if needed */
! if (!list_member_int(attnumlist, attnum))
{
! /* attribute is NOT to be copied from input */
! /* use default value if one exists */
! Node *defexpr = build_column_default(rel, attnum);
! if (defexpr != NULL)
{
! defexprs[num_defaults] = ExecPrepareExpr((Expr *) defexpr,
! estate);
! defmap[num_defaults] = attnum - 1;
! num_defaults++;
}
}
! /* If it's a domain type, set up to check domain constraints */
! if (get_typtype(attr[attnum - 1]->atttypid) == 'd')
{
! Param *prm;
! Node *node;
!
! /*
! * Easiest way to do this is to use parse_coerce.c to set up
! * an expression that checks the constraints. (At present,
! * the expression might contain a length-coercion-function
! * call and/or CoerceToDomain nodes.) The bottom of the
! * expression is a Param node so that we can fill in the
! * actual datum during the data input loop.
! */
! prm = makeNode(Param);
! prm->paramkind = PARAM_EXEC;
! prm->paramid = 0;
! prm->paramtype = getBaseType(attr[attnum - 1]->atttypid);
!
! node = coerce_to_domain((Node *) prm,
! prm->paramtype,
! attr[attnum - 1]->atttypid,
! COERCE_IMPLICIT_CAST, false, false);
!
! constraintexprs[attnum - 1] = ExecPrepareExpr((Expr *) node, estate);
! hasConstraints = true;
! }
! }
!
! /*
! * Prepare to catch AFTER triggers.
! */
! AfterTriggerBeginQuery();
!
! /*
! * Check BEFORE STATEMENT insertion triggers. It's debateable whether
! * we should do this for COPY, since it's not really an "INSERT"
! * statement as such. However, executing these triggers maintains
! * consistency with the EACH ROW triggers that we already fire on
! * COPY.
! */
! ExecBSInsertTriggers(estate, resultRelInfo);
!
! file_has_oids = oids; /* must rely on user to tell us this... */
!
! values = (Datum *) palloc(num_phys_attrs * sizeof(Datum));
! nulls = (char *) palloc(num_phys_attrs * sizeof(char));
! attr_offsets = (int *) palloc(num_phys_attrs * sizeof(int));
!
! /* Make room for a PARAM_EXEC value for domain constraint checks */
! if (hasConstraints)
! econtext->ecxt_param_exec_vals = (ParamExecData *)
! palloc0(sizeof(ParamExecData));
!
! /* Initialize static variables */
! fe_eof = false;
! eol_type = EOL_UNKNOWN;
! copy_binary = binary;
! copy_relname = RelationGetRelationName(rel);
! copy_lineno = 0;
! copy_attname = NULL;
! line_buf_converted = false;
!
! /* it is MUCH faster to do this once here than in the attribute parse loop */
! null_print_len = strlen(null_print);
!
! /* Set up callback to identify error line number */
! errcontext.callback = copy_in_error_callback;
! errcontext.arg = NULL;
! errcontext.previous = error_context_stack;
! error_context_stack = &errcontext;
!
! /* Set up data buffer to hold a chunk of data */
! MemSet(input_buf, ' ', COPY_BUF_SIZE * sizeof(char));
! input_buf[COPY_BUF_SIZE] = '\0';
!
! no_more_data = false; /* no more input data to read from file or FE */
! line_done = true;
!
! do
! {
! /* read a chunk of data into the buffer */
! size_t bytesread = CopyGetData(input_buf, COPY_BUF_SIZE);
!
! buf_done = false;
!
! /* set buffer pointers to beginning of the buffer */
! begloc = input_buf;
! buffer_index = 0;
!
! /*
! * continue if some bytes were read or if we didn't reach EOF. if we
! * both * reached EOF _and_ no bytes were read, quit the loop we are
! * done
! */
! if (bytesread > 0 || !fe_eof)
! {
!
! while (!buf_done)
! {
! bool skip_tuple;
! Oid loaded_oid = InvalidOid;
!
! CHECK_FOR_INTERRUPTS();
!
! /* Reset the per-tuple exprcontext */
! ResetPerTupleExprContext(estate);
!
! /* Switch into its memory context */
! MemoryContextSwitchTo(GetPerTupleMemoryContext(estate));
!
! /* Initialize all values for row to NULL */
! MemSet(values, 0, num_phys_attrs * sizeof(Datum));
! MemSet(nulls, 'n', num_phys_attrs * sizeof(char));
! /* reset attribute pointers */
! MemSet(attr_offsets, 0, num_phys_attrs * sizeof(int));
!
! result = NORMAL_ATTR;
!
! /* Actually read the line into memory here */
! line_done = CopyReadLineBuffered(bytesread);
! copy_lineno++;
!
! /*
! * if didn't finish processing data line, skip att parsing and
! * read more data, unless there is no more data to read...
! * (which means that the original last data line is missing
! * attrs and we want to catch that error)
! */
! if (!line_done)
! {
! copy_lineno--;
! if (!fe_eof || buf_done)
! break;
! }
!
! if (file_has_oids)
! {
! char *oid_string;
!
! /* can't be in CSV mode here */
! oid_string = CopyReadOidAttr(delim, null_print, null_print_len,
! &result, &isnull);
!
! if (isnull)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("null OID in COPY data")));
! else
! {
! copy_attname = "oid";
! loaded_oid = DatumGetObjectId(DirectFunctionCall1(oidin,
! CStringGetDatum(oid_string)));
! if (loaded_oid == InvalidOid)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("invalid OID in COPY data")));
! copy_attname = NULL;
! }
! }
!
! /* parse all the attribute in the data line */
! CopyReadAllAttrs(delim, null_print, null_print_len,
! nulls, attnumlist, attr_offsets, num_phys_attrs, attr);
!
! /*
! * Loop to read the user attributes on the line.
! */
! foreach(cur, attnumlist)
! {
! int attnum = lfirst_int(cur);
! int m = attnum - 1;
!
! if (nulls[m] == ' ')
! isnull = false;
! else
! isnull = true;
!
! /* we read an SQL NULL, no need to do anything */
! if (!isnull)
! {
! copy_attname = NameStr(attr[m]->attname);
! values[m] = FunctionCall3(&in_functions[m],
! CStringGetDatum(attr_buf.data + attr_offsets[m]),
! ObjectIdGetDatum(typioparams[m]),
! Int32GetDatum(attr[m]->atttypmod));
! copy_attname = NULL;
! }
! }
!
! /*
! * Now compute and insert any defaults available for the columns
! * not provided by the input data. Anything not processed here or
! * above will remain NULL.
! */
! for (i = 0; i < num_defaults; i++)
! {
! values[defmap[i]] = ExecEvalExpr(defexprs[i], econtext, &isnull, NULL);
! if (!isnull)
! nulls[defmap[i]] = ' ';
! }
!
! /*
! * Next apply any domain constraints
! */
! if (hasConstraints)
! {
! ParamExecData *prmdata = &econtext->ecxt_param_exec_vals[0];
!
! for (i = 0; i < num_phys_attrs; i++)
! {
! ExprState *exprstate = constraintexprs[i];
!
! if (exprstate == NULL)
! continue; /* no constraint for this attr */
!
! /* Insert current row's value into the Param value */
! prmdata->value = values[i];
! prmdata->isnull = (nulls[i] == 'n');
!
! /*
! * Execute the constraint expression. Allow the
! * expression to replace the value (consider e.g. a
! * timestamp precision restriction).
! */
! values[i] = ExecEvalExpr(exprstate, econtext,
! &isnull, NULL);
! nulls[i] = isnull ? 'n' : ' ';
! }
! }
!
! /*
! * And now we can form the input tuple.
! */
! tuple = heap_formtuple(tupDesc, values, nulls);
!
! if (oids && file_has_oids)
! HeapTupleSetOid(tuple, loaded_oid);
!
! /*
! * Triggers and stuff need to be invoked in query context.
! */
! MemoryContextSwitchTo(oldcontext);
!
! skip_tuple = false;
!
! /* BEFORE ROW INSERT Triggers */
! if (resultRelInfo->ri_TrigDesc &&
! resultRelInfo->ri_TrigDesc->n_before_row[TRIGGER_EVENT_INSERT] > 0)
! {
! HeapTuple newtuple;
!
! newtuple = ExecBRInsertTriggers(estate, resultRelInfo, tuple);
!
! if (newtuple == NULL) /* "do nothing" */
! skip_tuple = true;
! else if (newtuple != tuple) /* modified by Trigger(s) */
! {
! heap_freetuple(tuple);
! tuple = newtuple;
! }
! }
!
! if (!skip_tuple)
! {
! /* Place tuple in tuple slot */
! ExecStoreTuple(tuple, slot, InvalidBuffer, false);
!
! /*
! * Check the constraints of the tuple
! */
! if (rel->rd_att->constr)
! ExecConstraints(resultRelInfo, slot, estate);
!
! /*
! * OK, store the tuple and create index entries for it
! */
! simple_heap_insert(rel, tuple);
!
! if (resultRelInfo->ri_NumIndices > 0)
! ExecInsertIndexTuples(slot, &(tuple->t_self), estate, false);
!
! /* AFTER ROW INSERT Triggers */
! ExecARInsertTriggers(estate, resultRelInfo, tuple);
! }
!
! line_buf.len = 0; /* we can reset line buffer now. */
! line_buf.data[0] = '\0';
! line_buf.cursor = 0;
! } /* end while(!buf_done) */
! } /* end if (bytesread > 0 || !fe_eof) */
! else
! /* no bytes read, end of data */
! {
! no_more_data = TRUE;
! }
! } while (!no_more_data);
!
! /*
! * Done, clean up
! */
! error_context_stack = errcontext.previous;
!
! MemoryContextSwitchTo(oldcontext);
!
! /*
! * Execute AFTER STATEMENT insertion triggers
! */
! ExecASInsertTriggers(estate, resultRelInfo);
!
! /*
! * Handle queued AFTER triggers
! */
! AfterTriggerEndQuery(estate);
!
! pfree(values);
! pfree(nulls);
!
! pfree(attr_offsets);
!
! pfree(in_functions);
! pfree(typioparams);
! pfree(defmap);
! pfree(defexprs);
! pfree(constraintexprs);
! pfree(force_notnull);
!
! ExecDropSingleTupleTableSlot(slot);
!
! ExecCloseIndices(resultRelInfo);
!
! FreeExecutorState(estate);
! }
!
!
! /*
! * Read the next input line and stash it in line_buf, with conversion to
! * server encoding.
! *
! * Result is true if read was terminated by EOF, false if terminated
! * by newline.
! */
! static bool
! CopyReadLine(char *quote, char *escape)
! {
! bool result;
! bool change_encoding = (client_encoding != server_encoding);
! int c;
! int mblen;
! int j;
! unsigned char s[2];
! char *cvt;
! bool in_quote = false,
! last_was_esc = false,
! csv_mode = false;
! char quotec = '\0',
! escapec = '\0';
!
! if (quote)
! {
! csv_mode = true;
! quotec = quote[0];
! escapec = escape[0];
! /* ignore special escape processing if it's the same as quotec */
! if (quotec == escapec)
! escapec = '\0';
! }
!
!
! s[1] = 0;
!
! /* reset line_buf to empty */
! line_buf.len = 0;
! line_buf.data[0] = '\0';
! line_buf.cursor = 0;
!
! /* mark that encoding conversion hasn't occurred yet */
! line_buf_converted = false;
!
! /* set default status */
! result = false;
!
! /*
! * In this loop we only care for detecting newlines (\r and/or \n) and
! * the end-of-copy marker (\.).
! *
! * In Text mode, for backwards compatibility we allow
! * backslashes to escape newline characters. Backslashes other than
! * the end marker get put into the line_buf, since CopyReadAttribute
! * does its own escape processing.
! *
! * In CSV mode, CR and NL inside q quoted field are just part of the
! * data value and are put in line_buf. We keep just enough state
! * to know if we are currently in a quoted field or not.
! *
! * These four characters, and only these four, are assumed the same in
! * frontend and backend encodings.
! *
! * We do not assume that second and later bytes of a frontend
! * multibyte character couldn't look like ASCII characters.
! */
! for (;;)
! {
! c = CopyGetChar();
! if (c == EOF)
! {
! result = true;
! break;
! }
!
! if (csv_mode)
! {
! /*
! * Dealing with quotes and escapes here is mildly tricky. If the
! * quote char is also the escape char, there's no problem - we
! * just use the char as a toggle. If they are different, we need
! * to ensure that we only take account of an escape inside a quoted
! * field and immediately preceding a quote char, and not the
! * second in a escape-escape sequence.
! */
!
! if (in_quote && c == escapec)
! last_was_esc = !last_was_esc;
! if (c == quotec && !last_was_esc)
! in_quote = !in_quote;
! if (c != escapec)
! last_was_esc = false;
!
! /*
! * updating the line count for embedded CR and/or LF chars is
! * necessarily a little fragile - this test is probably about
! * the best we can do.
! */
! if (in_quote && c == (eol_type == EOL_CR ? '\r' : '\n'))
! copy_lineno++;
! }
!
! if (!in_quote && c == '\r')
! {
! if (eol_type == EOL_NL)
! {
! if (!csv_mode)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("literal carriage return found in data"),
! errhint("Use \"\\r\" to represent carriage return.")));
! else
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unquoted carriage return found in CSV data"),
! errhint("Use quoted CSV field to represent carriage return.")));
! }
! /* Check for \r\n on first line, _and_ handle \r\n. */
! if (eol_type == EOL_UNKNOWN || eol_type == EOL_CRNL)
! {
! int c2 = CopyPeekChar();
!
! if (c2 == '\n')
! {
! CopyDonePeek(c2, true); /* eat newline */
! eol_type = EOL_CRNL;
! }
! else
! {
! /* found \r, but no \n */
! if (eol_type == EOL_CRNL)
! {
! if (!csv_mode)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("literal carriage return found in data"),
! errhint("Use \"\\r\" to represent carriage return.")));
! else
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unquoted carriage return found in data"),
! errhint("Use quoted CSV field to represent carriage return.")));
!
! }
!
! /*
! * if we got here, it is the first line and we didn't
! * get \n, so put it back
! */
! CopyDonePeek(c2, false);
! eol_type = EOL_CR;
! }
! }
! break;
! }
! if (!in_quote && c == '\n')
! {
! if (eol_type == EOL_CR || eol_type == EOL_CRNL)
! {
! if (!csv_mode)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("literal newline found in data"),
! errhint("Use \"\\n\" to represent newline.")));
! else
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unquoted newline found in data"),
! errhint("Use quoted CSV field to represent newline.")));
!
! }
! eol_type = EOL_NL;
! break;
! }
!
! if ((line_buf.len == 0 || !csv_mode) && c == '\\')
! {
! int c2;
!
! if (csv_mode)
! c2 = CopyPeekChar();
! else
! c2 = c = CopyGetChar();
if (c2 == EOF)
{
***************
*** 2179,2185 ****
if (c2 == '.')
{
if (csv_mode)
! CopyDonePeek(c2, true); /* allow keep calling GetChar() */
if (eol_type == EOL_CRNL)
{
--- 2716,2723 ----
if (c2 == '.')
{
if (csv_mode)
! CopyDonePeek(c2, true); /* allow keep calling
! * GetChar() */
if (eol_type == EOL_CRNL)
{
***************
*** 2206,2261 ****
errmsg("end-of-copy marker does not match previous newline style")));
/*
! * In protocol version 3, we should ignore anything after
! * \. up to the protocol end of copy data. (XXX maybe
! * better not to treat \. as special?)
*/
! if (copy_dest == COPY_NEW_FE)
{
! while (c != EOF)
! c = CopyGetChar();
}
- result = true; /* report EOF */
- break;
}
-
- if (csv_mode)
- CopyDonePeek(c2, false); /* not a dot, so put it back */
- else
- /* not EOF mark, so emit \ and following char literally */
- appendStringInfoCharMacro(&line_buf, '\\');
- }
! appendStringInfoCharMacro(&line_buf, c);
! /*
! * When client encoding != server, must be careful to read the
! * extra bytes of a multibyte character exactly, since the
! * encoding might not ensure they don't look like ASCII. When the
! * encodings are the same, we need not do this, since no server
! * encoding we use has ASCII-like following bytes.
! */
! if (change_encoding)
{
! s[0] = c;
! mblen = pg_encoding_mblen(client_encoding, s);
! for (j = 1; j < mblen; j++)
{
! c = CopyGetChar();
! if (c == EOF)
{
! result = true;
! break;
}
- appendStringInfoCharMacro(&line_buf, c);
}
! if (result)
! break; /* out of outer loop */
}
! } /* end of outer loop */
! /* Done reading the line. Convert it to server encoding. */
! if (change_encoding)
{
cvt = (char *) pg_client_to_server((unsigned char *) line_buf.data,
line_buf.len);
--- 2744,3100 ----
errmsg("end-of-copy marker does not match previous newline style")));
/*
! * In protocol version 3, we should ignore anything after
! * \. up to the protocol end of copy data. (XXX maybe
! * better not to treat \. as special?)
! */
! if (copy_dest == COPY_NEW_FE)
! {
! while (c != EOF)
! c = CopyGetChar();
! }
! result = true; /* report EOF */
! break;
! }
!
! if (csv_mode)
! CopyDonePeek(c2, false); /* not a dot, so put it back */
! else
! /* not EOF mark, so emit \ and following char literally */
! appendStringInfoCharMacro(&line_buf, '\\');
! }
!
! appendStringInfoCharMacro(&line_buf, c);
!
! /*
! * When client encoding != server, must be careful to read the
! * extra bytes of a multibyte character exactly, since the
! * encoding might not ensure they don't look like ASCII. When the
! * encodings are the same, we need not do this, since no server
! * encoding we use has ASCII-like following bytes.
! */
! if (change_encoding)
! {
! s[0] = c;
! mblen = pg_encoding_mblen(client_encoding, s);
! for (j = 1; j < mblen; j++)
! {
! c = CopyGetChar();
! if (c == EOF)
! {
! result = true;
! break;
! }
! appendStringInfoCharMacro(&line_buf, c);
! }
! if (result)
! break; /* out of outer loop */
! }
! } /* end of outer loop */
!
! /* Done reading the line. Convert it to server encoding. */
! if (change_encoding)
! {
! cvt = (char *) pg_client_to_server((unsigned char *) line_buf.data,
! line_buf.len);
! if (cvt != line_buf.data)
! {
! /* transfer converted data back to line_buf */
! line_buf.len = 0;
! line_buf.data[0] = '\0';
! line_buf.cursor = 0;
! appendBinaryStringInfo(&line_buf, cvt, strlen(cvt));
! }
! }
!
! /* Now it's safe to use the buffer in error messages */
! line_buf_converted = true;
!
! return result;
! }
!
! /*
! * Return decimal value for a hexadecimal digit
! */
! static
! int
! GetDecimalFromHex(char hex)
! {
! if (isdigit(hex))
! return hex - '0';
! else
! return tolower(hex) - 'a' + 10;
! }
!
! /*
! * Detected the eol type by looking at the first data row.
! * Possible eol types are NL, CR, or CRNL. If eol type was
! * detected, it is set and a boolean true is returned to
! * indicated detection was successful. If the first data row
! * is longer than the input buffer, we return false and will
! * try again in the next buffer.
! */
! static bool
! DetectLineEnd(size_t bytesread)
! {
! int bytes_remaining = COPY_BUF_SIZE;
! char ch_found; /* the character found in the scan ('\n' or
! * '\r' or escapec) */
! char *start = input_buf;
! char *end;
!
! while (true)
! {
! bytes_remaining = COPY_BUF_SIZE - (input_buf - start);
!
! if (bytes_remaining <= 0)
! return false;
!
! if ((end = str2chrlen(start, '\n', '\r', bytes_remaining, &ch_found)) == NULL)
! {
! return false;
! }
! else
! {
! if (ch_found == '\n')
! {
! eol_type = EOL_NL;
! eol_ch[0] = '\n';
! eol_ch[1] = '\0';
!
! return true;
! }
! if (ch_found == '\r')
! {
! if (*(end + 1) == '\n')
! {
! eol_type = EOL_CRNL;
! eol_ch[0] = '\r';
! eol_ch[1] = '\n';
! }
! else
! {
! eol_type = EOL_CR;
! eol_ch[0] = '\r';
! eol_ch[1] = '\0';
! }
!
! return true;
! }
! }
! }
!
! }
!
!
! /*
! * Finds the next data line that is in the input buffer and loads it into line_buf.
! * Returns an indication if the line that was read is complete (if an unescaped line-end was
! * encountered). If we reached the end of buffer before the whole line was written into the
! * line buffer then returns false.
! */
! static bool
! CopyReadLineBuffered(size_t bytesread)
! {
! int linesize;
! bool transcode = (client_encoding != server_encoding);
! char *cvt;
! bool end_marker;
!
! /* mark that encoding conversion hasn't occurred yet */
! line_buf_converted = false;
!
! /*
! * Detect end of line type if not already detected.
! */
! if (eol_type == EOL_UNKNOWN)
! {
! bool eol_detected = DetectLineEnd(bytesread);
!
! if (!eol_detected)
! {
! /* load entire input buffer into line buf, and quit */
! appendBinaryStringInfo(&line_buf, input_buf, COPY_BUF_SIZE);
! line_done = false;
! buf_done = true;
!
! return line_done;
! }
! }
!
! /*
! * Special case: eol is CRNL, last byte of previous buffer was an
! * unescaped CR and 1st byte of current buffer is NL. We check for
! * that here.
! */
! if (eol_type == EOL_CRNL)
! {
! /* if we started scanning from the 1st byte of the buffer */
! if (begloc == input_buf)
! {
! /* and had a CR in last byte of prev buf */
! if (cr_in_prevbuf)
! {
! /*
! * if this 1st byte in buffer is 2nd byte of line end sequence
! * (linefeed)
*/
! if (*begloc == eol_ch[1])
{
! /*
! * load that one linefeed byte and indicate we are done
! * with the data line
! */
! appendBinaryStringInfo(&line_buf, begloc, 1);
! buffer_index++;
! begloc++;
!
! line_done = true;
! esc_in_prevbuf = false;
! cr_in_prevbuf = false;
!
! return line_done;
}
}
! cr_in_prevbuf = false;
! }
! }
! /*
! * (we need a loop so that if eol_ch is found, but prev ch is backslash,
! * we can search for the next eol_ch)
! */
! while (true)
! {
! /* reached end of buffer */
! if ((endloc = strchrlen(begloc, eol_ch[0], bytesread - buffer_index)) == NULL)
{
! linesize = COPY_BUF_SIZE - (begloc - input_buf);
! appendBinaryStringInfo(&line_buf, begloc, linesize);
!
! if (line_buf.len > 1)
{
! char *last_ch = line_buf.data + line_buf.len - 1; /* before terminating \0 */
!
! if (*last_ch == escapec)
{
! esc_in_prevbuf = true;
!
! if (line_buf.len > 2)
! {
! last_ch--;
! if (*last_ch == escapec)
! esc_in_prevbuf = false;
! }
! }
! else if (*last_ch == '\r')
! {
! if (eol_type == EOL_CRNL)
! cr_in_prevbuf = true;
}
}
!
! line_done = false;
! buf_done = true;
! break;
}
! else
! /* found the 1st eol ch in input_buf. */
! {
! bool eol_found = true;
! bool eol_escaped = true;
! /*
! * Load that piece of data (potentially a data line) into the line buffer,
! * and update the pointers for the next scan.
! */
! linesize = endloc - begloc + 1;
! appendBinaryStringInfo(&line_buf, begloc, linesize);
! buffer_index += linesize;
! begloc = endloc + 1;
!
! if (eol_type == EOL_CRNL)
! {
! /* check if there is a '\n' after the '\r' */
! if (*(endloc + 1) == '\n')
! {
! /* this is a line end */
! appendBinaryStringInfo(&line_buf, begloc, 1); /* load that '\n' */
! buffer_index++;
! begloc++;
! }
! else
! /* just a CR, not a line end */
! eol_found = false;
! }
!
! /*
! * in some cases, this end of line char happens to be the
! * last character in the buffer. we need to catch that.
! */
! if (buffer_index >= bytesread)
! buf_done = true;
!
! /*
! * Check if the 1st end of line ch is escaped.
! */
! if (endloc != input_buf) /* can we look 1 char back? */
! {
! if (*(endloc - 1) != escapec) /* prev char is not an escape */
! eol_escaped = false;
! else
! /* prev char is an escape */
! {
! if (endloc != (input_buf + 1)) /* can we look another
! * char back? */
! {
! /* it's a double escape char, so it's not an escape */
! if (*(endloc - 2) == escapec)
! eol_escaped = false;
! /* else it's a single escape char, so EOL is ascaped */
! }
! else
! {
! /* we need to check in the last buffer */
! if (esc_in_prevbuf) /* double escape char, so not
! * an escape */
! eol_escaped = false;
! }
! }
! }
! else
!
! /*
! * this eol ch is first ch in buffer, check for escape in prev
! * buf
! */
! {
! if (!esc_in_prevbuf)
! eol_escaped = false;
! }
!
! esc_in_prevbuf = false; /* reset variable */
!
! /*
! * if eol was found, and it isn't escaped, line is done
! */
! if ((eol_escaped == false) && eol_found)
! {
! line_done = true;
! break;
! }
! else
! /* stay in the loop and process some more data. */
! line_done = false;
!
! } /* end of found eol_ch */
! }
!
! /*
! * Done reading the line. Convert it to server encoding.
! */
! if (transcode)
{
cvt = (char *) pg_client_to_server((unsigned char *) line_buf.data,
line_buf.len);
***************
*** 2264,2291 ****
/* transfer converted data back to line_buf */
line_buf.len = 0;
line_buf.data[0] = '\0';
appendBinaryStringInfo(&line_buf, cvt, strlen(cvt));
}
}
! /* Now it's safe to use the buffer in error messages */
line_buf_converted = true;
! return result;
! }
! /*
! * Return decimal value for a hexadecimal digit
! */
! static
! int GetDecimalFromHex(char hex)
! {
! if (isdigit(hex))
! return hex - '0';
! else
! return tolower(hex) - 'a' + 10;
}
/*----------
* Read the value of a single attribute, performing de-escaping as needed.
*
--- 3103,3151 ----
/* transfer converted data back to line_buf */
line_buf.len = 0;
line_buf.data[0] = '\0';
+ line_buf.cursor = 0;
appendBinaryStringInfo(&line_buf, cvt, strlen(cvt));
}
}
! /* indicate that conversion had occured */
line_buf_converted = true;
! /*
! * check if this line is an end marker -- "\."
! */
! end_marker = false;
! switch (eol_type)
! {
! case EOL_NL:
! if (!strcmp(line_buf.data, "\\.\n"))
! end_marker = true;
! break;
! case EOL_CR:
! if (!strcmp(line_buf.data, "\\.\r"))
! end_marker = true;
! break;
! case EOL_CRNL:
! if (!strcmp(line_buf.data, "\\.\r\n"))
! end_marker = true;
! break;
! case EOL_UNKNOWN:
! break;
! }
!
! if (end_marker)
! {
! fe_eof = true;
! /* we don't want to process a \. as data line, want to quit. */
! line_done = false;
! buf_done = true;
! }
!
! return line_done;
}
+
/*----------
* Read the value of a single attribute, performing de-escaping as needed.
*
***************
*** 2314,2322 ****
int end_cursor;
int input_len;
! /* reset attribute_buf to empty */
! attribute_buf.len = 0;
! attribute_buf.data[0] = '\0';
/* set default status */
*result = END_OF_LINE;
--- 3174,3183 ----
int end_cursor;
int input_len;
! /* reset attr_buf to empty */
! attr_buf.len = 0;
! attr_buf.data[0] = '\0';
! attr_buf.cursor = 0;
/* set default status */
*result = END_OF_LINE;
***************
*** 2377,2387 ****
/* Handle \x3F */
if (line_buf.cursor < line_buf.len)
{
! char hexchar = line_buf.data[line_buf.cursor];
if (isxdigit(hexchar))
{
! int val = GetDecimalFromHex(hexchar);
line_buf.cursor++;
if (line_buf.cursor < line_buf.len)
--- 3238,3248 ----
/* Handle \x3F */
if (line_buf.cursor < line_buf.len)
{
! char hexchar = line_buf.data[line_buf.cursor];
if (isxdigit(hexchar))
{
! int val = GetDecimalFromHex(hexchar);
line_buf.cursor++;
if (line_buf.cursor < line_buf.len)
***************
*** 2422,2428 ****
*/
}
}
! appendStringInfoCharMacro(&attribute_buf, c);
}
/* check whether raw input matched null marker */
--- 3283,3289 ----
*/
}
}
! appendStringInfoCharMacro(&attr_buf, c);
}
/* check whether raw input matched null marker */
***************
*** 2433,2439 ****
else
*isnull = false;
! return attribute_buf.data;
}
--- 3294,3662 ----
else
*isnull = false;
! return attr_buf.data;
! }
!
! /*
! * Read the first attribute. This is mainly used to maintain support
! * for an OID column. All the rest of the columns will be read at once with
! * CopyReadAllAttrs().
! */
! static char *
! CopyReadOidAttr(const char *delim, const char *null_print, int null_print_len,
! CopyReadResult *result, bool *isnull)
! {
! char delimc = delim[0];
! char *start_loc = line_buf.data + line_buf.cursor;
! char *end_loc;
! int attr_len = 0;
! int bytes_remaining;
!
! /* reset attribute buf to empty */
! attr_buf.len = 0;
! attr_buf.data[0] = '\0';
! attr_buf.cursor = 0;
!
! /* set default status */
! *result = END_OF_LINE;
!
! /* # of bytes that were not yet processed in this line */
! bytes_remaining = line_buf.len - line_buf.cursor;
!
! /* got to end of line */
! if ((end_loc = strchrlen(start_loc, delimc, bytes_remaining)) == NULL)
! {
! attr_len = bytes_remaining - 1; /* don't count '\n' in len calculation */
! appendBinaryStringInfo(&attr_buf, start_loc, attr_len);
! line_buf.cursor += attr_len + 2; /* skip '\n' and '\0' */
!
! *result = END_OF_LINE;
! }
! else
! /* found a delimiter */
! {
! /*
! * (we don't care if delim was preceded with a backslash, because it's
! * an invalid OID anyway)
! */
!
! attr_len = end_loc - start_loc; /* we don't include the delimiter ch */
!
! appendBinaryStringInfo(&attr_buf, start_loc, attr_len);
! line_buf.cursor += attr_len + 1;
!
! *result = NORMAL_ATTR;
! }
!
!
! /* check whether raw input matched null marker */
! if (attr_len == null_print_len && strncmp(start_loc, null_print, attr_len) == 0)
! *isnull = true;
! else
! *isnull = false;
!
! return attr_buf.data;
! }
!
!
! /*
! * Read all attributes. Attributes are parsed from line_buf and
! * inserted (all at once) to attr_buf, while saving pointers to
! * each attribute's starting position.
! *
! * When this routine finishes execution both the nulls array and
! * the attr_offsets array are updated. The attr_offsets will include
! * the offset from the beginning of the attribute array of which
! * each attribute begins. If a specific attribute is not used for this
! * COPY command (ommitted from the column list), a value of 0 will be assigned.
! * For example: for table foo(a,b,c,d,e) and COPY foo(a,b,e)
! * attr_offsets may look something like this after this routine
! * returns: [0,20,0,0,55]. That means that column "a" value starts
! * at byte offset 0, "b" in 20 and "e" in 55, in attr_buf.
! *
! * In the attribute buffer (attr_buf) each attribute
! * is terminated with a '\0', and therefore by using the attr_offsets
! * array we could point to a beginning of an attribute and have it
! * behave as a C string, much like previously done in COPY.
! *
! * Another aspect to improving performance is reducing the frequency
! * of data load into buffers. The original COPY read attribute code
! * loaded a character at a time. In here we try to load a chunk of data
! * at a time. Usually a chunk will include a full data row
! * (unless we have an escaped delim). That effectively reduces the number of
! * loads by a factor of number of bytes per row. This improves performance
! * greatly, unfortunately it add more complexity to the code.
! *
! * Global participants in parsing logic:
! *
! * line_buf.cursor -- an offset from beginning of the line buffer
! * that indicates where we are about to begin the next scan. Note that
! * if we have WITH OIDS this cursor is already shifted past the first
! * OID attribute.
! *
! * attr_buf.cursor -- an offset from the beginning of the
! * attribute buffer that indicates where the current attribute begins.
! */
! static void
! CopyReadAllAttrs(const char *delim, const char *null_print, int null_print_len,
! char *nulls, List *attnumlist, int *attr_offsets,
! int num_phys_attrs, Form_pg_attribute *attr)
! {
! char delimc = delim[0]; /* delimiter character */
! char *scan_start; /* pointer to line buffer where scan should
! * start. */
! char *scan_end; /* pointer to line buffer where char was found */
! char ch_found; /* the character found in the scan (delimc or
! * escape) */
! int attr_pre_len; /* current attr raw len, before processing
! * escapes */
! int attr_post_len; /* current attr len after escaping */
! int m; /* attribute index being parsed */
! int bytes_remaining;/* num of bytes remaining to be scanned in
! * line buf */
! int chunk_start; /* offset to beginning of line chunk to load */
! int chunk_len; /* length of chunk of data to load to attr buf */
! int oct_val; /* byte value for octal escapes */
! int attnum; /* attribute number being parsed */
! ListCell *cur; /* cursor to attribute list used for this COPY */
! int attribute;
!
! /*
! * init variables for attribute scan
! */
! attr_buf.len = 0;
! attr_buf.data[0] = '\0';
! attr_buf.cursor = 0;
! /* cursor is now > 0 if we copy WITH OIDS */
! scan_start = line_buf.data + line_buf.cursor;
! cur = list_head(attnumlist);
! attnum = lfirst_int(cur);
! m = attnum - 1;
! chunk_start = line_buf.cursor;
! chunk_len = 0;
! attr_pre_len = 0;
! attr_post_len = 0;
!
! /*
! * Scan through the line buffer to read all attributes data
! */
! while (line_buf.cursor < line_buf.len)
! {
! bytes_remaining = line_buf.len - line_buf.cursor;
! ch_found = '\0';
!
! if ((scan_end = str2chrlen(scan_start, delimc, escapec, bytes_remaining, &ch_found))
! == NULL)
! {
! /* GOT TO END OF LINE BUFFER */
!
! if (cur == NULL)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("extra data after last expected column")));
!
! attnum = lfirst_int(cur);
! m = attnum - 1;
!
! /* don't count eol char(s) in attr and chunk len calculation */
! if (eol_type == EOL_CRNL)
! {
! attr_pre_len += bytes_remaining - 2;
! chunk_len = line_buf.len - chunk_start - 2;
! }
! else
! {
! attr_pre_len += bytes_remaining - 1;
! chunk_len = line_buf.len - chunk_start - 1;
! }
!
! /* check if this is a NULL value or data value (assumed NULL) */
! if (attr_pre_len == null_print_len
! &&
! strncmp(line_buf.data + line_buf.len - attr_pre_len - 1, null_print, attr_pre_len)
! == 0)
! nulls[m] = 'n';
! else
! nulls[m] = ' ';
!
! attr_offsets[m] = attr_buf.cursor;
!
!
! /* load the last chunk, the whole buffer in most cases */
! appendBinaryStringInfo(&attr_buf, line_buf.data + chunk_start, chunk_len);
!
! line_buf.cursor += attr_pre_len + 2; /* skip eol char and
! * '\0' to exit loop */
!
! if (lnext(cur) != NULL)
! {
! /*
! * For an empty data line, the previous COPY code will
! * fail it during the conversion stage. We can fail it here
! * already, but then we will fail the regression tests b/c
! * of a different error message. that's why we return so we
! * can get the same error message that regress expects. ahh...
! * this conditional is unnecessary and should be removed soon.
! */
! if (line_buf.len > 1)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("missing data for column \"%s\"",
! NameStr(attr[m + 1]->attname))));
! else
! return;
! }
! }
! else
! /* FOUND A DELIMITER OR ESCAPE */
! {
! if (cur == NULL)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("extra data after last expected column")));
!
! if (ch_found == delimc) /* found a delimiter */
! {
! attnum = lfirst_int(cur);
! m = attnum - 1;
!
! /* (we don't include the delimiter ch in length) */
! attr_pre_len += scan_end - scan_start;
! /* (we don't include the delimiter ch in length) */
! attr_post_len += scan_end - scan_start;
!
! /* check if this is a null print or data (assumed NULL) */
! if (attr_pre_len == null_print_len
! &&
! strncmp(scan_end - attr_pre_len, null_print, attr_pre_len)
! == 0)
! nulls[m] = 'n';
! else
! nulls[m] = ' ';
!
! /* set the pointer to next attribute position */
! attr_offsets[m] = attr_buf.cursor;
!
! /*
! * update buffer cursors to our current location, +1 to skip
! * the delimc
! */
! line_buf.cursor = scan_end - line_buf.data + 1;
! attr_buf.cursor += attr_post_len + 1;
!
! /* prepare scan for next attr */
! scan_start = line_buf.data + line_buf.cursor;
! cur = lnext(cur);
! attr_pre_len = 0;
! attr_post_len = 0;
! }
! else
! /* found an escape character */
! {
! char nextc = *(scan_end + 1);
! char newc;
! int skip = 2;
!
! chunk_len = (scan_end - line_buf.data) - chunk_start + 1;
!
! /* load a chunk of data */
! appendBinaryStringInfo(&attr_buf, line_buf.data + chunk_start, chunk_len);
!
! switch (nextc)
! {
! case '0':
! case '1':
! case '2':
! case '3':
! case '4':
! case '5':
! case '6':
! case '7':
! oct_val = OCTVALUE(nextc);
! nextc = *(scan_end + 2);
!
! /*
! * (no need for out bad access check since line if
! * buffered)
! */
! if (ISOCTAL(nextc))
! {
! skip++;
! oct_val = (oct_val << 3) + OCTVALUE(nextc);
! nextc = *(scan_end + 3);
! if (ISOCTAL(nextc))
! {
! skip++;
! oct_val = (oct_val << 3) + OCTVALUE(nextc);
! }
! }
! newc = oct_val & 0377; /* the escaped byte value */
! break;
! case 'b':
! newc = '\b';
! break;
! case 'f':
! newc = '\f';
! break;
! case 'n':
! newc = '\n';
! break;
! case 'r':
! newc = '\r';
! break;
! case 't':
! newc = '\t';
! break;
! case 'v':
! newc = '\v';
! break;
! default:
! if (nextc == delimc)
! newc = delimc;
! else if (nextc == escapec)
! newc = escapec;
! else
! /* no escape sequence, take next char literaly */
! newc = nextc;
! break;
! }
!
! /* update to current length, add escape and escaped chars */
! attr_pre_len += scan_end - scan_start + 2;
! /* update to current length, escaped char */
! attr_post_len += scan_end - scan_start + 1;
!
! /*
! * Need to get rid of the escape character. This is done by
! * loading the chunk up to including the escape character
! * into the attribute buffer. Then overwritting the backslash
! * with the escaped sequence or char, and continuing to scan
! * from *after* the char than is after the escape in line buf.
! */
! *(attr_buf.data + attr_buf.len - 1) = newc;
! line_buf.cursor = scan_end - line_buf.data + skip;
! scan_start = scan_end + skip;
! chunk_start = line_buf.cursor;
! chunk_len = 0;
! }
!
! } /* end delimiter/backslash */
!
! } /* end line buffer scan. */
!
! /*
! * Replace all delimiters with NULL for string termination.
! * NOTE: only delimiters (NOT necessarily all delimc) are replaced.
! * Example (delimc = '|'):
! * - Before: f 1 | f \| 2 | f 3
! * - After : f 1 \0 f | 2 \0 f 3
! */
! for (attribute = 1; attribute < num_phys_attrs; attribute++)
! {
! if (attr_offsets[attribute] != 0)
! *(attr_buf.data + attr_offsets[attribute] - 1) = '\0';
! }
!
}
***************
*** 2477,2485 ****
bool in_quote = false;
bool saw_quote = false;
! /* reset attribute_buf to empty */
! attribute_buf.len = 0;
! attribute_buf.data[0] = '\0';
/* set default status */
*result = END_OF_LINE;
--- 3700,3709 ----
bool in_quote = false;
bool saw_quote = false;
! /* reset attr_buf to empty */
! attr_buf.len = 0;
! attr_buf.data[0] = '\0';
! attr_buf.cursor = 0;
/* set default status */
*result = END_OF_LINE;
***************
*** 2519,2525 ****
if (nextc == escapec || nextc == quotec)
{
! appendStringInfoCharMacro(&attribute_buf, nextc);
line_buf.cursor++;
continue;
}
--- 3743,3749 ----
if (nextc == escapec || nextc == quotec)
{
! appendStringInfoCharMacro(&attr_buf, nextc);
line_buf.cursor++;
continue;
}
***************
*** 2536,2542 ****
in_quote = false;
continue;
}
! appendStringInfoCharMacro(&attribute_buf, c);
}
if (in_quote)
--- 3760,3766 ----
in_quote = false;
continue;
}
! appendStringInfoCharMacro(&attr_buf, c);
}
if (in_quote)
***************
*** 2550,2556 ****
else
*isnull = false;
! return attribute_buf.data;
}
/*
--- 3774,3780 ----
else
*isnull = false;
! return attr_buf.data;
}
/*
***************
*** 2578,2606 ****
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("invalid field size")));
! /* reset attribute_buf to empty, and load raw data in it */
! attribute_buf.len = 0;
! attribute_buf.data[0] = '\0';
! attribute_buf.cursor = 0;
! enlargeStringInfo(&attribute_buf, fld_size);
! CopyGetData(attribute_buf.data, fld_size);
if (CopyGetEof())
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("unexpected EOF in COPY data")));
! attribute_buf.len = fld_size;
! attribute_buf.data[fld_size] = '\0';
/* Call the column type's binary input converter */
result = FunctionCall2(flinfo,
! PointerGetDatum(&attribute_buf),
ObjectIdGetDatum(typioparam));
/* Trouble if it didn't eat the whole buffer */
! if (attribute_buf.cursor != attribute_buf.len)
ereport(ERROR,
(errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
errmsg("incorrect binary data format")));
--- 3802,3830 ----
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("invalid field size")));
! /* reset attr_buf to empty, and load raw data in it */
! attr_buf.len = 0;
! attr_buf.data[0] = '\0';
! attr_buf.cursor = 0;
! enlargeStringInfo(&attr_buf, fld_size);
! CopyGetData(attr_buf.data, fld_size);
if (CopyGetEof())
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("unexpected EOF in COPY data")));
! attr_buf.len = fld_size;
! attr_buf.data[fld_size] = '\0';
/* Call the column type's binary input converter */
result = FunctionCall2(flinfo,
! PointerGetDatum(&attr_buf),
ObjectIdGetDatum(typioparam));
/* Trouble if it didn't eat the whole buffer */
! if (attr_buf.cursor != attr_buf.len)
ereport(ERROR,
(errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
errmsg("incorrect binary data format")));
***************
*** 2797,2799 ****
--- 4021,4086 ----
return attnums;
}
+
+ /******************************************************
+ * utility functions, should be placed elsewhere in src
+ ******************************************************/
+ /*
+ * These are custom versions of the string function strchr().
+ * As opposed to the original strchr which searches through
+ * a string until the target character is found, or a NULL is
+ * found, this version will not return when a NULL is found.
+ * Instead it will search through a pre-defined length of
+ * bytes and will return only if the target character(s) is reached.
+ *
+ * If our client encoding is not a supported server encoding, we
+ * know that it is not safe to look at each character as trailing
+ * byte in a multibyte character may be a 7-bit ASCII equivalent.
+ * Therefore we use pg_encoding_mblen to skip to the end of the
+ * character.
+ *
+ * parameters:
+ * s - string being searched.
+ * c(n) - char we are searching for.
+ * len - maximum # of bytes to search.
+ *
+ * returns:
+ * pointer to c - if c is located within the string.
+ * NULL - if c was not found in specified length of search. Note:
+ * this DOESN'T mean that a '\0' was reached.
+ */
+ char *
+ strchrlen(const char *s, int c, size_t len)
+ {
+ const char *start;
+
+ if (client_encoding_only)
+ {
+ int mblen = pg_encoding_mblen(client_encoding, s);
+
+ for (start = s; *s != c && s < start + len; s += mblen)
+ mblen = pg_encoding_mblen(client_encoding, s);
+ }
+ else
+ /* safe to scroll byte by byte */
+ {
+ for (start = s; *s != c && s < start + len; s++)
+ ;
+ }
+
+ return ((*s == c) ? (char *) s : NULL);
+ }
+
+ char *
+ str2chrlen(const char *s, int c1, int c2, size_t len, char *c_found)
+ {
+ const char *start;
+
+ *c_found = '\0';
+
+ for (start = s; *s != c1 && *s != c2 && s < start + len; s++)
+ ;
+
+ *c_found = *s;
+ return (*c_found != '\0' ? (char *) s : NULL);
+ }