copy_parse_improvements_V16.patch
application/octet-stream
Filename: copy_parse_improvements_V16.patch
Type: application/octet-stream
Part: 0
Patch
Format: context
| File | + | − |
|---|---|---|
| src/backend/commands/copy.c | 1234 | 600 |
Index: src/backend/commands/copy.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/backend/commands/copy.c,v
retrieving revision 1.247
diff -c -r1.247 copy.c
*** src/backend/commands/copy.c 10 Jul 2005 21:13:58 -0000 1.247
--- src/backend/commands/copy.c 2 Aug 2005 14:38:37 -0000
***************
*** 50,55 ****
--- 50,56 ----
#define ISOCTAL(c) (((c) >= '0') && ((c) <= '7'))
#define OCTVALUE(c) ((c) - '0')
+ #define COPY_BUF_SIZE 65536
/*
* Represents the different source/dest cases we need to worry about at
***************
*** 63,78 ****
} CopyDest;
/*
- * State indicator showing what stopped CopyReadAttribute()
- */
- typedef enum CopyReadResult
- {
- NORMAL_ATTR,
- END_OF_LINE,
- UNTERMINATED_FIELD
- } CopyReadResult;
-
- /*
* Represents the end-of-line terminator type of the input
*/
typedef enum EolType
--- 64,69 ----
***************
*** 97,102 ****
--- 88,96 ----
static EolType eol_type; /* EOL type of input */
static int client_encoding; /* remote side's character encoding */
static int server_encoding; /* local encoding */
+ static char eol_ch[2]; /* The byte values of the 1 or 2 eol bytes */
+ static bool client_encoding_only; /* true if client encoding is a non
+ * supported server encoding */
/* these are just for error messages, see copy_in_error_callback */
static bool copy_binary; /* is it a binary copy? */
***************
*** 104,126 ****
static int copy_lineno; /* line number for error messages */
static const char *copy_attname; /* current att for error messages */
/*
* These static variables are used to avoid incurring overhead for each
! * attribute processed. attribute_buf is reused on each CopyReadAttribute
* call to hold the string being read in. Under normal use it will soon
* grow to a suitable size, and then we will avoid palloc/pfree overhead
* for subsequent attributes. Note that CopyReadAttribute returns a pointer
! * to attribute_buf's data buffer!
*/
! static StringInfoData attribute_buf;
/*
* Similarly, line_buf holds the whole input line being processed (its
* cursor field points to the next character to be read by CopyReadAttribute).
* The input cycle is first to read the whole line into line_buf, convert it
* to server encoding, and then extract individual attribute fields into
! * attribute_buf. (We used to have CopyReadAttribute read the input source
* directly, but that caused a lot of encoding issues and unnecessary logic
* complexity.)
*/
--- 98,141 ----
static int copy_lineno; /* line number for error messages */
static const char *copy_attname; /* current att for error messages */
+ /*
+ * Static variables for buffered input parsing
+ */
+ static char input_buf[COPY_BUF_SIZE + 1]; /* extra byte for '\0' */
+ static int buffer_index; /* input buffer index */
+ static bool end_marker;
+ static char *begloc;
+ static char *endloc;
+ static bool buf_done; /* finished processing the current buffer */
+ static bool line_done; /* finished processing the whole line or
+ * stopped in the middle */
+ /* these are for CSV format */
+ static bool in_quote;
+ static bool last_was_esc;
+
+ /* these are for TEXT format */
+ static bool esc_in_prevbuf; /* escape was last character of the data
+ * input buffer */
+ static bool cr_in_prevbuf; /* CR was last character of the data input
+ * buffer */
+
/*
* These static variables are used to avoid incurring overhead for each
! * attribute processed. attr_buf is reused on each CopyReadAttribute
* call to hold the string being read in. Under normal use it will soon
* grow to a suitable size, and then we will avoid palloc/pfree overhead
* for subsequent attributes. Note that CopyReadAttribute returns a pointer
! * to attr_buf's data buffer!
*/
! static StringInfoData attr_buf;
/*
* Similarly, line_buf holds the whole input line being processed (its
* cursor field points to the next character to be read by CopyReadAttribute).
* The input cycle is first to read the whole line into line_buf, convert it
* to server encoding, and then extract individual attribute fields into
! * attr_buf. (We used to have CopyReadAttribute read the input source
* directly, but that caused a lot of encoding issues and unnecessary logic
* complexity.)
*/
***************
*** 137,155 ****
static void CopyFrom(Relation rel, List *attnumlist, bool binary, bool oids,
char *delim, char *null_print, bool csv_mode, char *quote, char *escape,
List *force_notnull_atts, bool header_line);
! static bool CopyReadLine(char * quote, char * escape);
! static char *CopyReadAttribute(const char *delim, const char *null_print,
! CopyReadResult *result, bool *isnull);
! static char *CopyReadAttributeCSV(const char *delim, const char *null_print,
! char *quote, char *escape,
! CopyReadResult *result, bool *isnull);
static Datum CopyReadBinaryAttribute(int column_no, FmgrInfo *flinfo,
Oid typioparam, int32 typmod, bool *isnull);
static void CopyAttributeOut(char *string, char *delim);
static void CopyAttributeOutCSV(char *string, char *delim, char *quote,
char *escape, bool force_quote);
static List *CopyGetAttnums(Relation rel, List *attnamelist);
static void limit_printout_length(StringInfo buf);
/* Internal communications functions */
static void SendCopyBegin(bool binary, int natts);
--- 152,176 ----
static void CopyFrom(Relation rel, List *attnumlist, bool binary, bool oids,
char *delim, char *null_print, bool csv_mode, char *quote, char *escape,
List *force_notnull_atts, bool header_line);
! static bool CopyReadLineText(size_t bytesread, char *escape);
! static bool CopyReadLineCSV(size_t bytesread, char *quote, char *escape);
! static void CopyReadAttributesText(const char *delim, const char *escape, const char *null_print,
! int null_print_len, char *nulls, List *attnumlist,
! int *attr_offsets, int num_phys_attrs, Form_pg_attribute *attr);
! static void CopyReadAttributesCSV(const char *delim, const char *null_print, char *quote,
! char *escape, int null_print_len, char *nulls, List *attnumlist,
! int *attr_offsets, int num_phys_attrs, Form_pg_attribute *attr);
static Datum CopyReadBinaryAttribute(int column_no, FmgrInfo *flinfo,
Oid typioparam, int32 typmod, bool *isnull);
+ static char *CopyReadOidAttr(const char *delim, const char *null_print, int null_print_len,
+ bool *isnull);
static void CopyAttributeOut(char *string, char *delim);
static void CopyAttributeOutCSV(char *string, char *delim, char *quote,
char *escape, bool force_quote);
static List *CopyGetAttnums(Relation rel, List *attnamelist);
static void limit_printout_length(StringInfo buf);
+ static bool DetectLineEnd(size_t bytesread, char *quote, char *escape);
+
/* Internal communications functions */
static void SendCopyBegin(bool binary, int natts);
***************
*** 159,175 ****
static void CopySendString(const char *str);
static void CopySendChar(char c);
static void CopySendEndOfRow(bool binary);
! static void CopyGetData(void *databuf, int datasize);
! static int CopyGetChar(void);
#define CopyGetEof() (fe_eof)
- static int CopyPeekChar(void);
- static void CopyDonePeek(int c, bool pickup);
static void CopySendInt32(int32 val);
static int32 CopyGetInt32(void);
static void CopySendInt16(int16 val);
static int16 CopyGetInt16(void);
/*
* Send copy start/stop messages for frontend copies. These have changed
--- 180,197 ----
static void CopySendString(const char *str);
static void CopySendChar(char c);
static void CopySendEndOfRow(bool binary);
! static int CopyGetData(void *databuf, int datasize);
#define CopyGetEof() (fe_eof)
static void CopySendInt32(int32 val);
static int32 CopyGetInt32(void);
static void CopySendInt16(int16 val);
static int16 CopyGetInt16(void);
+ /* byte scaning utils */
+ static char *scanTextLine(const char *s, char c, size_t len);
+ static char *scanCSVLine(const char *s, char c1, char c2, char c3, size_t len);
+ static char *scanTextAttr(const char *s, char c1, char c2, size_t len);
/*
* Send copy start/stop messages for frontend copies. These have changed
***************
*** 382,395 ****
* It seems unwise to allow the COPY IN to complete normally in that case.
*
* NB: no data conversion is applied by these functions
*/
! static void
CopyGetData(void *databuf, int datasize)
{
switch (copy_dest)
{
case COPY_FILE:
! fread(databuf, datasize, 1, copy_file);
if (feof(copy_file))
fe_eof = true;
break;
--- 404,422 ----
* It seems unwise to allow the COPY IN to complete normally in that case.
*
* NB: no data conversion is applied by these functions
+ *
+ * Returns: the number of bytes that were successfully read
+ * into the data buffer.
*/
! static int
CopyGetData(void *databuf, int datasize)
{
+ size_t bytesread = 0;
+
switch (copy_dest)
{
case COPY_FILE:
! bytesread = fread(databuf, 1, datasize, copy_file);
if (feof(copy_file))
fe_eof = true;
break;
***************
*** 401,406 ****
--- 428,435 ----
(errcode(ERRCODE_CONNECTION_FAILURE),
errmsg("unexpected EOF on client connection")));
}
+ bytesread += datasize; /* update the count of bytes that were
+ * read so far */
break;
case COPY_NEW_FE:
while (datasize > 0 && !fe_eof)
***************
*** 429,435 ****
case 'c': /* CopyDone */
/* COPY IN correctly terminated by frontend */
fe_eof = true;
! return;
case 'f': /* CopyFail */
ereport(ERROR,
(errcode(ERRCODE_QUERY_CANCELED),
--- 458,464 ----
case 'c': /* CopyDone */
/* COPY IN correctly terminated by frontend */
fe_eof = true;
! return bytesread;
case 'f': /* CopyFail */
ereport(ERROR,
(errcode(ERRCODE_QUERY_CANCELED),
***************
*** 459,598 ****
avail = datasize;
pq_copymsgbytes(copy_msgbuf, databuf, avail);
databuf = (void *) ((char *) databuf + avail);
datasize -= avail;
}
break;
}
- }
-
- static int
- CopyGetChar(void)
- {
- int ch;
-
- switch (copy_dest)
- {
- case COPY_FILE:
- ch = getc(copy_file);
- break;
- case COPY_OLD_FE:
- ch = pq_getbyte();
- if (ch == EOF)
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection")));
- }
- break;
- case COPY_NEW_FE:
- {
- unsigned char cc;
-
- CopyGetData(&cc, 1);
- if (fe_eof)
- ch = EOF;
- else
- ch = cc;
- break;
- }
- default:
- ch = EOF;
- break;
- }
- if (ch == EOF)
- fe_eof = true;
- return ch;
- }
-
- /*
- * CopyPeekChar reads a byte in "peekable" mode.
- *
- * after each call to CopyPeekChar, a call to CopyDonePeek _must_
- * follow, unless EOF was returned.
- *
- * CopyDonePeek will either take the peeked char off the stream
- * (if pickup is true) or leave it on the stream (if pickup is false).
- */
- static int
- CopyPeekChar(void)
- {
- int ch;
-
- switch (copy_dest)
- {
- case COPY_FILE:
- ch = getc(copy_file);
- break;
- case COPY_OLD_FE:
- ch = pq_peekbyte();
- if (ch == EOF)
- {
- /* Only a \. terminator is legal EOF in old protocol */
- ereport(ERROR,
- (errcode(ERRCODE_CONNECTION_FAILURE),
- errmsg("unexpected EOF on client connection")));
- }
- break;
- case COPY_NEW_FE:
- {
- unsigned char cc;
-
- CopyGetData(&cc, 1);
- if (fe_eof)
- ch = EOF;
- else
- ch = cc;
- break;
- }
- default:
- ch = EOF;
- break;
- }
- if (ch == EOF)
- fe_eof = true;
- return ch;
- }
-
- static void
- CopyDonePeek(int c, bool pickup)
- {
- if (fe_eof)
- return; /* can't unget an EOF */
- switch (copy_dest)
- {
- case COPY_FILE:
- if (!pickup)
- {
- /* We don't want to pick it up - so put it back in there */
- ungetc(c, copy_file);
- }
- /* If we wanted to pick it up, it's already done */
- break;
- case COPY_OLD_FE:
- if (pickup)
- {
- /* We want to pick it up */
- (void) pq_getbyte();
- }
! /*
! * If we didn't want to pick it up, just leave it where it
! * sits
! */
! break;
! case COPY_NEW_FE:
! if (!pickup)
! {
! /* We don't want to pick it up - so put it back in there */
! copy_msgbuf->cursor--;
! }
! /* If we wanted to pick it up, it's already done */
! break;
! }
}
-
/*
* These functions do apply some data conversion
*/
--- 488,503 ----
avail = datasize;
pq_copymsgbytes(copy_msgbuf, databuf, avail);
databuf = (void *) ((char *) databuf + avail);
+ bytesread += avail; /* update the count of bytes that were
+ * read so far */
datasize -= avail;
}
break;
}
! return bytesread;
}
/*
* These functions do apply some data conversion
*/
***************
*** 968,980 ****
}
/* Set up variables to avoid per-attribute overhead. */
! initStringInfo(&attribute_buf);
initStringInfo(&line_buf);
line_buf_converted = false;
client_encoding = pg_get_client_encoding();
server_encoding = GetDatabaseEncoding();
copy_dest = COPY_FILE; /* default */
copy_file = NULL;
copy_msgbuf = NULL;
--- 873,905 ----
}
/* Set up variables to avoid per-attribute overhead. */
! initStringInfo(&attr_buf);
initStringInfo(&line_buf);
line_buf_converted = false;
client_encoding = pg_get_client_encoding();
server_encoding = GetDatabaseEncoding();
+ /*
+ * check if the client encoding is one of the 5 encodings
+ * that are not supported as a server encodings.
+ */
+ switch (client_encoding)
+ {
+ case PG_SJIS:
+ case PG_BIG5:
+ case PG_GBK:
+ case PG_UHC:
+ case PG_GB18030:
+ client_encoding_only = true;
+ break;
+ default:
+ client_encoding_only = false;
+ }
+
+ if(!csv_mode)
+ escape = "\\"; /* default for text format */
+
copy_dest = COPY_FILE; /* default */
copy_file = NULL;
copy_msgbuf = NULL;
***************
*** 1105,1113 ****
errmsg("could not write to file \"%s\": %m",
filename)));
}
! pfree(attribute_buf.data);
pfree(line_buf.data);
/*
* Close the relation. If reading, we can release the AccessShareLock
* we got; if writing, we should hold the lock until end of
--- 1030,1039 ----
errmsg("could not write to file \"%s\": %m",
filename)));
}
! pfree(attr_buf.data);
pfree(line_buf.data);
+
/*
* Close the relation. If reading, we can release the AccessShareLock
* we got; if writing, we should hold the lock until end of
***************
*** 1345,1351 ****
{
bytea *outputbytes;
! outputbytes = DatumGetByteaP(FunctionCall1(&out_functions[attnum - 1],
value));
/* We assume the result will not have been toasted */
CopySendInt32(VARSIZE(outputbytes) - VARHDRSZ);
--- 1271,1278 ----
{
bytea *outputbytes;
! outputbytes =
! DatumGetByteaP(FunctionCall1(&out_functions[attnum - 1],
value));
/* We assume the result will not have been toasted */
CopySendInt32(VARSIZE(outputbytes) - VARHDRSZ);
***************
*** 1395,1404 ****
if (copy_attname)
{
/* error is relevant to a particular column */
! limit_printout_length(&attribute_buf);
errcontext("COPY %s, line %d, column %s: \"%s\"",
copy_relname, copy_lineno, copy_attname,
! attribute_buf.data);
}
else
{
--- 1322,1331 ----
if (copy_attname)
{
/* error is relevant to a particular column */
! limit_printout_length(&attr_buf);
errcontext("COPY %s, line %d, column %s: \"%s\"",
copy_relname, copy_lineno, copy_attname,
! attr_buf.data);
}
else
{
***************
*** 1406,1411 ****
--- 1333,1340 ----
if (line_buf_converted ||
client_encoding == server_encoding)
{
+ /* Strip off the newline */
+ *(line_buf.data + line_buf.len - 1) = '\0';
limit_printout_length(&line_buf);
errcontext("COPY %s, line %d: \"%s\"",
copy_relname, copy_lineno,
***************
*** 1485,1491 ****
Oid in_func_oid;
Datum *values;
char *nulls;
- bool done = false;
bool isnull;
ResultRelInfo *resultRelInfo;
EState *estate = CreateExecutorState(); /* for ExecConstraints() */
--- 1414,1419 ----
***************
*** 1496,1501 ****
--- 1424,1434 ----
ExprContext *econtext; /* used for ExecEvalExpr for default atts */
MemoryContext oldcontext = CurrentMemoryContext;
ErrorContextCallback errcontext;
+ int *attr_offsets;
+ int null_print_len; /* length of null print */
+ bool no_more_data;
+ ListCell *cur;
+
tupDesc = RelationGetDescr(rel);
attr = tupDesc->attrs;
***************
*** 1607,1613 ****
}
}
! /* Prepare to catch AFTER triggers. */
AfterTriggerBeginQuery();
/*
--- 1540,1548 ----
}
}
! /*
! * Prepare to catch AFTER triggers.
! */
AfterTriggerBeginQuery();
/*
***************
*** 1671,1676 ****
--- 1606,1612 ----
values = (Datum *) palloc(num_phys_attrs * sizeof(Datum));
nulls = (char *) palloc(num_phys_attrs * sizeof(char));
+ attr_offsets = (int *) palloc(num_phys_attrs * sizeof(int));
/* Make room for a PARAM_EXEC value for domain constraint checks */
if (hasConstraints)
***************
*** 1691,1712 ****
errcontext.previous = error_context_stack;
error_context_stack = &errcontext;
! /* on input just throw the header line away */
! if (header_line)
{
! copy_lineno++;
! done = CopyReadLine(quote, escape) ;
}
! while (!done)
{
bool skip_tuple;
Oid loaded_oid = InvalidOid;
CHECK_FOR_INTERRUPTS();
- copy_lineno++;
-
/* Reset the per-tuple exprcontext */
ResetPerTupleExprContext(estate);
--- 1627,1681 ----
errcontext.previous = error_context_stack;
error_context_stack = &errcontext;
! /*
! * initialize buffered scan variables.
! */
! if(csv_mode)
! {
! in_quote = false;
! last_was_esc = false;
! }
!
! null_print_len = strlen(null_print);
!
! /* Set up data buffer to hold a chunk of data */
! MemSet(input_buf, ' ', COPY_BUF_SIZE * sizeof(char));
! input_buf[COPY_BUF_SIZE] = '\0';
!
! no_more_data = false; /* no more input data to read from file or FE */
! line_done = true;
! buf_done = false;
!
! do
{
! size_t bytesread = 0;
!
! /* read a chunk of data into the buffer */
! if (!binary)
! {
! bytesread = CopyGetData(input_buf, COPY_BUF_SIZE);
! buf_done = false;
!
! /* set buffer pointers to beginning of the buffer */
! begloc = input_buf;
! buffer_index = 0;
}
! /*
! * continue if some bytes were read or if we didn't reach EOF. if we
! * both reached EOF _and_ no bytes were read, quit the loop we are
! * done
! */
! if (bytesread > 0 || !fe_eof || binary)
! {
!
! while (!buf_done)
{
bool skip_tuple;
Oid loaded_oid = InvalidOid;
CHECK_FOR_INTERRUPTS();
/* Reset the per-tuple exprcontext */
ResetPerTupleExprContext(estate);
***************
*** 1716,1746 ****
/* Initialize all values for row to NULL */
MemSet(values, 0, num_phys_attrs * sizeof(Datum));
MemSet(nulls, 'n', num_phys_attrs * sizeof(char));
if (!binary)
{
- CopyReadResult result = NORMAL_ATTR;
- char *string;
- ListCell *cur;
/* Actually read the line into memory here */
! done = csv_mode ?
! CopyReadLine(quote, escape) : CopyReadLine(NULL, NULL);
/*
! * EOF at start of line means we're done. If we see EOF after
! * some characters, we act as though it was newline followed
! * by EOF, ie, process the line and then exit loop on next
! * iteration.
*/
! if (done && line_buf.len == 0)
break;
if (file_has_oids)
{
/* can't be in CSV mode here */
! string = CopyReadAttribute(delim, null_print,
! &result, &isnull);
if (isnull)
ereport(ERROR,
--- 1685,1729 ----
/* Initialize all values for row to NULL */
MemSet(values, 0, num_phys_attrs * sizeof(Datum));
MemSet(nulls, 'n', num_phys_attrs * sizeof(char));
+ /* reset attribute pointers */
+ MemSet(attr_offsets, 0, num_phys_attrs * sizeof(int));
if (!binary)
{
/* Actually read the line into memory here */
! line_done = csv_mode ?
! CopyReadLineCSV(bytesread, quote, escape) :
! CopyReadLineText(bytesread, escape);
/*
! * if finished processing data line - increment line count.
! * Otherwise, if eof is not yet reached, we skip att parsing
! * and read more data. But if eof _was_ reached it means
! * that the original last data line is defective and
! * we want to catch that error later on.
*/
! if (line_done)
! copy_lineno++;
! else if (!fe_eof || end_marker )
break;
+ if (header_line)
+ {
+ line_buf.len = 0; /* we can reset line buffer now. */
+ line_buf.data[0] = '\0';
+ line_buf.cursor = 0;
+ header_line = false;
+ continue;
+ }
+
if (file_has_oids)
{
+ char *oid_string;
+
/* can't be in CSV mode here */
! oid_string = CopyReadOidAttr(delim, null_print, null_print_len,
! &isnull);
if (isnull)
ereport(ERROR,
***************
*** 1750,1756 ****
{
copy_attname = "oid";
loaded_oid = DatumGetObjectId(DirectFunctionCall1(oidin,
! CStringGetDatum(string)));
if (loaded_oid == InvalidOid)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
--- 1733,1739 ----
{
copy_attname = "oid";
loaded_oid = DatumGetObjectId(DirectFunctionCall1(oidin,
! CStringGetDatum(oid_string)));
if (loaded_oid == InvalidOid)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
***************
*** 1759,1792 ****
}
}
! /* Loop to read the user attributes on the line. */
foreach(cur, attnumlist)
{
int attnum = lfirst_int(cur);
int m = attnum - 1;
! /*
! * If prior attr on this line was ended by newline,
! * complain.
! */
! if (result != NORMAL_ATTR)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("missing data for column \"%s\"",
! NameStr(attr[m]->attname))));
! if (csv_mode)
! {
! string = CopyReadAttributeCSV(delim, null_print, quote,
! escape, &result, &isnull);
! if (result == UNTERMINATED_FIELD)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unterminated CSV quoted field")));
! }
else
! string = CopyReadAttribute(delim, null_print,
! &result, &isnull);
if (csv_mode && isnull && force_notnull[m])
{
--- 1742,1770 ----
}
}
! /* parse all the attribute in the data line */
! if(csv_mode)
! CopyReadAttributesCSV(delim, null_print, quote, escape, null_print_len,
! nulls, attnumlist, attr_offsets, num_phys_attrs, attr);
! else
! CopyReadAttributesText(delim, escape, null_print, null_print_len,
! nulls, attnumlist, attr_offsets, num_phys_attrs, attr);
!
! /*
! * Loop to read the user attributes on the line.
! */
foreach(cur, attnumlist)
{
int attnum = lfirst_int(cur);
int m = attnum - 1;
+ char *string;
! string = attr_buf.data + attr_offsets[m];
! if (nulls[m] == ' ')
! isnull = false;
else
! isnull = true;
if (csv_mode && isnull && force_notnull[m])
{
***************
*** 1806,1823 ****
copy_attname = NULL;
}
}
-
- /*
- * Complain if there are more fields on the input line.
- *
- * Special case: if we're reading a zero-column table, we won't
- * yet have called CopyReadAttribute() at all; so no error if
- * line is empty.
- */
- if (result == NORMAL_ATTR && line_buf.len != 0)
- ereport(ERROR,
- (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
- errmsg("extra data after last expected column")));
}
else
{
--- 1784,1789 ----
***************
*** 1828,1834 ****
fld_count = CopyGetInt16();
if (CopyGetEof() || fld_count == -1)
{
! done = true;
break;
}
--- 1794,1801 ----
fld_count = CopyGetInt16();
if (CopyGetEof() || fld_count == -1)
{
! buf_done = true;
! no_more_data = true;
break;
}
***************
*** 1885,1891 ****
nulls[defmap[i]] = ' ';
}
! /* Next apply any domain constraints */
if (hasConstraints)
{
ParamExecData *prmdata = &econtext->ecxt_param_exec_vals[0];
--- 1852,1860 ----
nulls[defmap[i]] = ' ';
}
! /*
! * Next apply any domain constraints
! */
if (hasConstraints)
{
ParamExecData *prmdata = &econtext->ecxt_param_exec_vals[0];
***************
*** 1912,1924 ****
}
}
! /* And now we can form the input tuple. */
tuple = heap_formtuple(tupDesc, values, nulls);
if (oids && file_has_oids)
HeapTupleSetOid(tuple, loaded_oid);
! /* Triggers and stuff need to be invoked in query context. */
MemoryContextSwitchTo(oldcontext);
skip_tuple = false;
--- 1881,1897 ----
}
}
! /*
! * And now we can form the input tuple.
! */
tuple = heap_formtuple(tupDesc, values, nulls);
if (oids && file_has_oids)
HeapTupleSetOid(tuple, loaded_oid);
! /*
! * Triggers and stuff need to be invoked in query context.
! */
MemoryContextSwitchTo(oldcontext);
skip_tuple = false;
***************
*** 1945,1955 ****
/* Place tuple in tuple slot */
ExecStoreTuple(tuple, slot, InvalidBuffer, false);
! /* Check the constraints of the tuple */
if (rel->rd_att->constr)
ExecConstraints(resultRelInfo, slot, estate);
! /* OK, store the tuple and create index entries for it */
simple_heap_insert(rel, tuple);
if (resultRelInfo->ri_NumIndices > 0)
--- 1918,1932 ----
/* Place tuple in tuple slot */
ExecStoreTuple(tuple, slot, InvalidBuffer, false);
! /*
! * Check the constraints of the tuple
! */
if (rel->rd_att->constr)
ExecConstraints(resultRelInfo, slot, estate);
! /*
! * OK, store the tuple and create index entries for it
! */
simple_heap_insert(rel, tuple);
if (resultRelInfo->ri_NumIndices > 0)
***************
*** 1958,1976 ****
/* AFTER ROW INSERT Triggers */
ExecARInsertTriggers(estate, resultRelInfo, tuple);
}
}
! /* Done, clean up */
error_context_stack = errcontext.previous;
MemoryContextSwitchTo(oldcontext);
! /* Execute AFTER STATEMENT insertion triggers */
ExecASInsertTriggers(estate, resultRelInfo);
! /* Handle queued AFTER triggers */
AfterTriggerEndQuery(estate);
pfree(values);
pfree(nulls);
--- 1935,1972 ----
/* AFTER ROW INSERT Triggers */
ExecARInsertTriggers(estate, resultRelInfo, tuple);
}
+
+ line_buf.len = 0; /* we can reset line buffer now. */
+ line_buf.data[0] = '\0';
+ line_buf.cursor = 0;
+ } /* end while(!buf_done) */
+ } /* end if (bytesread > 0 || !fe_eof) */
+ else
+ /* no bytes read, end of data */
+ {
+ no_more_data = true;
}
+ } while (!no_more_data);
! /*
! * Done, clean up
! */
error_context_stack = errcontext.previous;
MemoryContextSwitchTo(oldcontext);
! /*
! * Execute AFTER STATEMENT insertion triggers
! */
ExecASInsertTriggers(estate, resultRelInfo);
! /*
! * Handle queued AFTER triggers
! */
AfterTriggerEndQuery(estate);
+ pfree(attr_offsets);
+
pfree(values);
pfree(nulls);
***************
*** 1990,2262 ****
/*
! * Read the next input line and stash it in line_buf, with conversion to
! * server encoding.
! *
! * Result is true if read was terminated by EOF, false if terminated
! * by newline.
*/
static bool
! CopyReadLine(char * quote, char * escape)
{
! bool result;
! bool change_encoding = (client_encoding != server_encoding);
! int c;
! int mblen;
! int j;
! unsigned char s[2];
char *cvt;
! bool in_quote = false, last_was_esc = false, csv_mode = false;
! char quotec = '\0', escapec = '\0';
!
! if (quote)
! {
! csv_mode = true;
! quotec = quote[0];
! escapec = escape[0];
! /* ignore special escape processing if it's the same as quotec */
! if (quotec == escapec)
! escapec = '\0';
! }
!
!
! s[1] = 0;
!
! /* reset line_buf to empty */
! line_buf.len = 0;
! line_buf.data[0] = '\0';
! line_buf.cursor = 0;
/* mark that encoding conversion hasn't occurred yet */
line_buf_converted = false;
! /* set default status */
! result = false;
/*
! * In this loop we only care for detecting newlines (\r and/or \n) and
! * the end-of-copy marker (\.).
! *
! * In Text mode, for backwards compatibility we allow
! * backslashes to escape newline characters. Backslashes other than
! * the end marker get put into the line_buf, since CopyReadAttribute
! * does its own escape processing.
! *
! * In CSV mode, CR and NL inside q quoted field are just part of the
! * data value and are put in line_buf. We keep just enough state
! * to know if we are currently in a quoted field or not.
! *
! * These four characters, and only these four, are assumed the same in
! * frontend and backend encodings.
! *
! * We do not assume that second and later bytes of a frontend
! * multibyte character couldn't look like ASCII characters.
*/
! for (;;)
{
! c = CopyGetChar();
! if (c == EOF)
{
! result = true;
! break;
}
! if (csv_mode)
{
/*
! * Dealing with quotes and escapes here is mildly tricky. If the
! * quote char is also the escape char, there's no problem - we
! * just use the char as a toggle. If they are different, we need
! * to ensure that we only take account of an escape inside a quoted
! * field and immediately preceding a quote char, and not the
! * second in a escape-escape sequence.
*/
! if (in_quote && c == escapec)
! last_was_esc = ! last_was_esc;
! if (c == quotec && ! last_was_esc)
! in_quote = ! in_quote;
! if (c != escapec)
! last_was_esc = false;
/*
! * updating the line count for embedded CR and/or LF chars is
! * necessarily a little fragile - this test is probably about
! * the best we can do.
*/
! if (in_quote && c == (eol_type == EOL_CR ? '\r' : '\n'))
! copy_lineno++;
! }
! if (!in_quote && c == '\r')
{
! if (eol_type == EOL_NL)
{
! if (! csv_mode)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("literal carriage return found in data"),
! errhint("Use \"\\r\" to represent carriage return.")));
! else
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unquoted carriage return found in CSV data"),
! errhint("Use quoted CSV field to represent carriage return.")));
}
! /* Check for \r\n on first line, _and_ handle \r\n. */
! if (eol_type == EOL_UNKNOWN || eol_type == EOL_CRNL)
{
! int c2 = CopyPeekChar();
! if (c2 == '\n')
! {
! CopyDonePeek(c2, true); /* eat newline */
! eol_type = EOL_CRNL;
}
else
{
! /* found \r, but no \n */
if (eol_type == EOL_CRNL)
{
! if (!csv_mode)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("literal carriage return found in data"),
! errhint("Use \"\\r\" to represent carriage return.")));
else
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unquoted carriage return found in data"),
! errhint("Use quoted CSV field to represent carriage return.")));
!
}
/*
! * if we got here, it is the first line and we didn't
! * get \n, so put it back
*/
! CopyDonePeek(c2, false);
! eol_type = EOL_CR;
}
}
- break;
}
! if (!in_quote && c == '\n')
{
! if (eol_type == EOL_CR || eol_type == EOL_CRNL)
! {
! if (!csv_mode)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("literal newline found in data"),
! errhint("Use \"\\n\" to represent newline.")));
! else
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unquoted newline found in data"),
! errhint("Use quoted CSV field to represent newline.")));
!
}
- eol_type = EOL_NL;
- break;
- }
! if ((line_buf.len == 0 || !csv_mode) && c == '\\')
! {
! int c2;
!
! if (csv_mode)
! c2 = CopyPeekChar();
! else
! c2 = c = CopyGetChar();
! if (c2 == EOF)
{
! result = true;
! if (csv_mode)
! CopyDonePeek(c2, true);
break;
}
! if (c2 == '.')
! {
! if (csv_mode)
! CopyDonePeek(c2, true); /* allow keep calling GetChar() */
! if (eol_type == EOL_CRNL)
! {
! c = CopyGetChar();
! if (c == '\n')
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("end-of-copy marker does not match previous newline style")));
! if (c != '\r')
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("end-of-copy marker corrupt")));
}
- c = CopyGetChar();
- if (c != '\r' && c != '\n')
- ereport(ERROR,
- (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
- errmsg("end-of-copy marker corrupt")));
- if ((eol_type == EOL_NL && c != '\n') ||
- (eol_type == EOL_CRNL && c != '\n') ||
- (eol_type == EOL_CR && c != '\r'))
- ereport(ERROR,
- (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
- errmsg("end-of-copy marker does not match previous newline style")));
/*
! * In protocol version 3, we should ignore anything after
! * \. up to the protocol end of copy data. (XXX maybe
! * better not to treat \. as special?)
*/
! if (copy_dest == COPY_NEW_FE)
{
! while (c != EOF)
! c = CopyGetChar();
}
! result = true; /* report EOF */
break;
}
! if (csv_mode)
! CopyDonePeek(c2, false); /* not a dot, so put it back */
else
! /* not EOF mark, so emit \ and following char literally */
! appendStringInfoCharMacro(&line_buf, '\\');
}
! appendStringInfoCharMacro(&line_buf, c);
/*
! * When client encoding != server, must be careful to read the
! * extra bytes of a multibyte character exactly, since the
! * encoding might not ensure they don't look like ASCII. When the
! * encodings are the same, we need not do this, since no server
! * encoding we use has ASCII-like following bytes.
*/
! if (change_encoding)
! {
! s[0] = c;
! mblen = pg_encoding_mblen(client_encoding, s);
! for (j = 1; j < mblen; j++)
! {
! c = CopyGetChar();
! if (c == EOF)
{
! result = true;
break;
}
- appendStringInfoCharMacro(&line_buf, c);
}
! if (result)
! break; /* out of outer loop */
}
- } /* end of outer loop */
! /* Done reading the line. Convert it to server encoding. */
! if (change_encoding)
{
cvt = (char *) pg_client_to_server((unsigned char *) line_buf.data,
line_buf.len);
--- 1986,2424 ----
/*
! * Finds the next TEXT line that is in the input buffer and loads
! * it into line_buf. Returns an indication if the line that was read
! * is complete (if an unescaped line-end was encountered). If we
! * reached the end of buffer before the whole line was written into the
! * line buffer then returns false.
*/
static bool
! CopyReadLineText(size_t bytesread, char *escape)
{
! int linesize;
! bool transcode = (client_encoding != server_encoding);
char *cvt;
! char escapec = '\0';
/* mark that encoding conversion hasn't occurred yet */
line_buf_converted = false;
! /*
! * set the escape char for text format ('\\' by default).
! */
! escapec = escape[0];
/*
! * Detect end of line type if not already detected.
*/
! if (eol_type == EOL_UNKNOWN)
{
! bool eol_detected = DetectLineEnd(bytesread, NULL, escape);
!
! if (!eol_detected)
{
! /* load entire input buffer into line buf, and quit */
! appendBinaryStringInfo(&line_buf, input_buf, COPY_BUF_SIZE);
! buf_done = true;
! return false;
! }
}
! /*
! * Special case: eol is CRNL, last byte of previous buffer was an
! * unescaped CR and 1st byte of current buffer is NL. We check for
! * that here.
! */
! if (eol_type == EOL_CRNL)
! {
! /* if we started scanning from the 1st byte of the buffer */
! if (begloc == input_buf)
! {
! /* and had a CR in last byte of prev buf */
! if (cr_in_prevbuf)
! {
! /*
! * if this 1st byte in buffer is 2nd byte of line end sequence
! * (linefeed)
! */
! if (*begloc == eol_ch[1])
{
/*
! * load that one linefeed byte and indicate we are done
! * with the data line
*/
+ appendBinaryStringInfo(&line_buf, begloc, 1);
+ buffer_index++;
+ begloc++;
+ esc_in_prevbuf = false;
+ cr_in_prevbuf = false;
+ return true;
+ }
+ }
! cr_in_prevbuf = false;
! }
! }
/*
! * (we need a loop so that if eol_ch is found, but prev ch is backslash,
! * we can search for the next eol_ch)
*/
! while (true)
! {
! /* reached end of buffer */
! if ((endloc = scanTextLine(begloc, eol_ch[0], bytesread - buffer_index)) == NULL)
! {
! linesize = COPY_BUF_SIZE - (begloc - input_buf);
! appendBinaryStringInfo(&line_buf, begloc, linesize);
! if (line_buf.len > 1)
{
! char *last_ch = line_buf.data + line_buf.len - 1; /* before terminating \0 */
!
! if (*last_ch == escapec)
{
! esc_in_prevbuf = true;
!
! if (line_buf.len > 2)
! {
! last_ch--;
! if (*last_ch == escapec)
! esc_in_prevbuf = false;
}
! }
! else if (*last_ch == '\r')
{
! if (eol_type == EOL_CRNL)
! cr_in_prevbuf = true;
! }
! }
! line_done = false;
! buf_done = true;
! break;
}
else
+ /* found the 1st eol ch in input_buf. */
{
! bool eol_found = true;
! bool eol_escaped = true;
!
! /*
! * Load that piece of data (potentially a data line) into the line buffer,
! * and update the pointers for the next scan.
! */
! linesize = endloc - begloc + 1;
! appendBinaryStringInfo(&line_buf, begloc, linesize);
! buffer_index += linesize;
! begloc = endloc + 1;
!
if (eol_type == EOL_CRNL)
{
! /* check if there is a '\n' after the '\r' */
! if (*(endloc + 1) == '\n')
! {
! /* this is a line end */
! appendBinaryStringInfo(&line_buf, begloc, 1); /* load that '\n' */
! buffer_index++;
! begloc++;
! }
else
! /* just a CR, not a line end */
! eol_found = false;
}
/*
! * in some cases, this end of line char happens to be the
! * last character in the buffer. we need to catch that.
*/
! if (buffer_index >= bytesread)
! buf_done = true;
!
! /*
! * Check if the 1st end of line ch is escaped.
! */
! if (endloc != input_buf) /* can we look 1 char back? */
! {
! if (*(endloc - 1) != escapec) /* prev char is not an escape */
! eol_escaped = false;
! else
! /* prev char is an escape */
! {
! if (endloc != (input_buf + 1)) /* can we look another
! * char back? */
! {
! /* it's a double escape char, so it's not an escape */
! if (*(endloc - 2) == escapec)
! eol_escaped = false;
! /* else it's a single escape char, so EOL is ascaped */
! }
! else
! {
! /* we need to check in the last buffer */
! if (esc_in_prevbuf)
! eol_escaped = false;
}
}
}
! else
! /*
! * this eol ch is first ch in buffer, check for escape in prev buf
! */
{
! if (!esc_in_prevbuf)
! eol_escaped = false;
}
! esc_in_prevbuf = false; /* reset variable */
! /*
! * if eol was found, and it isn't escaped, line is done
! */
! if ((eol_escaped == false) && eol_found)
{
! line_done = true;
break;
}
! else
! /* stay in the loop and process some more data. */
! line_done = false;
!
! } /* end of found eol_ch */
! }
! /*
! * Done reading the line. Convert it to server encoding.
! */
! if (transcode)
! {
! cvt = (char *) pg_client_to_server((unsigned char *) line_buf.data,
! line_buf.len);
! if (cvt != line_buf.data)
! {
! /* transfer converted data back to line_buf */
! line_buf.len = 0;
! line_buf.data[0] = '\0';
! line_buf.cursor = 0;
! appendBinaryStringInfo(&line_buf, cvt, strlen(cvt));
! }
! }
!
! /* indicate that conversion had occured */
! line_buf_converted = true;
!
! /*
! * check if this line is an end marker -- "\."
! */
! end_marker = false;
!
! switch (eol_type)
! {
! case EOL_NL:
! if (!strcmp(line_buf.data, "\\.\n"))
! end_marker = true;
! break;
! case EOL_CR:
! if (!strcmp(line_buf.data, "\\.\r"))
! end_marker = true;
! break;
! case EOL_CRNL:
! if (!strcmp(line_buf.data, "\\.\r\n"))
! end_marker = true;
! break;
! case EOL_UNKNOWN:
! break;
! }
!
! if (end_marker)
! {
! fe_eof = true;
! /* we don't want to process a \. as data line, want to quit. */
! line_done = false;
! buf_done = true;
! }
!
! return line_done;
! }
!
! /*
! * Finds the next CSV line that is in the input buffer and loads
! * it into line_buf. Returns an indication if the line that was read
! * is complete (if an unescaped line-end was encountered). If we
! * reached the end of buffer before the whole line was written into the
! * line buffer then returns false.
! */
! static bool
! CopyReadLineCSV(size_t bytesread, char *quote, char *escape)
! {
! int linesize;
! char *cvt;
! char quotec = '\0',
! escapec = '\0';
! bool transcode = (client_encoding != server_encoding);
!
! /* mark that encoding conversion hasn't occurred yet */
! line_buf_converted = false;
!
! escapec = escape[0];
! quotec = quote[0];
!
! /* ignore special escape processing if it's the same as quotec */
! if (quotec == escapec)
! escapec = '\0';
!
! /*
! * Detect end of line type if not already detected.
! */
! if (eol_type == EOL_UNKNOWN)
! {
! bool eol_detected = DetectLineEnd(bytesread, quote, escape);
!
! if (!eol_detected)
! {
! /* load entire input buffer into line buf, and quit */
! appendBinaryStringInfo(&line_buf, input_buf, COPY_BUF_SIZE);
! line_done = false;
! buf_done = true;
!
! return line_done;
! }
! }
!
! /*
! * Special case: eol is CRNL, last byte of previous buffer was an
! * unescaped CR and 1st byte of current buffer is NL. We check for
! * that here.
! */
! if (eol_type == EOL_CRNL)
! {
! /* if we started scanning from the 1st byte of the buffer */
! if (begloc == input_buf)
! {
! /* and had a CR in last byte of prev buf */
! if (cr_in_prevbuf)
! {
! /*
! * if this 1st byte in buffer is 2nd byte of line end sequence
! * (linefeed)
! */
! if (*begloc == eol_ch[1])
! {
! /*
! * load that one linefeed byte and indicate we are done
! * with the data line
! */
! appendBinaryStringInfo(&line_buf, begloc, 1);
! buffer_index++;
! begloc++;
!
! line_done = true;
! esc_in_prevbuf = false;
! cr_in_prevbuf = false;
!
! return line_done;
! }
! }
!
! cr_in_prevbuf = false;
! }
}
/*
! * (we need a loop so that if eol_ch is found, but we are in quotes,
! * we can search for the next eol_ch)
*/
! while (true)
! {
! /* reached end of buffer */
! if ((endloc = scanCSVLine(begloc, eol_ch[0], escapec, quotec, bytesread - buffer_index)) == NULL)
! {
! linesize = COPY_BUF_SIZE - (begloc - input_buf);
! appendBinaryStringInfo(&line_buf, begloc, linesize);
!
! if (last_was_esc)
! esc_in_prevbuf = true;
!
! if (line_buf.len > 1)
! {
! char *last_ch = line_buf.data + line_buf.len - 1; /* before terminating \0 */
!
! if (*last_ch == '\r')
{
! if (eol_type == EOL_CRNL)
! cr_in_prevbuf = true;
}
! }
!
! line_done = false;
! buf_done = true;
break;
}
+ else
+ /* found 1st eol char in input_buf. */
+ {
+ bool eol_found = true;
! /*
! * Load that piece of data (potentially a data line) into the line buffer,
! * and update the pointers for the next scan.
! */
! linesize = endloc - begloc + 1;
! appendBinaryStringInfo(&line_buf, begloc, linesize);
! buffer_index += linesize;
! begloc = endloc + 1;
!
! /* end of line only if not in quotes */
! if(in_quote)
! {
! line_done = false;
! /* buf done, but still in quote */
! if (buffer_index >= bytesread)
! {
! buf_done = true;
! break;
! }
! }
! else
! {
! /* if dos eol, check for '\n' after the '\r' */
! if (eol_type == EOL_CRNL)
! {
! if (*(endloc + 1) == '\n')
! {
! /* this is a line end */
! appendBinaryStringInfo(&line_buf, begloc, 1); /* load that '\n' */
! buffer_index++;
! begloc++;
! }
else
! /* just a CR, not a line end */
! eol_found = false;
}
! /*
! * in some cases, this end of line char happens to be the
! * last character in the buffer. we need to catch that.
! */
! if (buffer_index >= bytesread)
! buf_done = true;
/*
! * if eol was found line is done
*/
! if (eol_found)
{
! line_done = true;
break;
}
}
! } /* end of found eol_ch */
}
! /*
! * Done reading the line. Convert it to server encoding.
! */
! if (transcode)
{
cvt = (char *) pg_client_to_server((unsigned char *) line_buf.data,
line_buf.len);
***************
*** 2265,2285 ****
/* transfer converted data back to line_buf */
line_buf.len = 0;
line_buf.data[0] = '\0';
appendBinaryStringInfo(&line_buf, cvt, strlen(cvt));
}
}
! /* Now it's safe to use the buffer in error messages */
line_buf_converted = true;
! return result;
}
/*
* Return decimal value for a hexadecimal digit
*/
static
! int GetDecimalFromHex(char hex)
{
if (isdigit(hex))
return hex - '0';
--- 2427,2480 ----
/* transfer converted data back to line_buf */
line_buf.len = 0;
line_buf.data[0] = '\0';
+ line_buf.cursor = 0;
appendBinaryStringInfo(&line_buf, cvt, strlen(cvt));
}
}
! /* indicate that conversion had occured */
line_buf_converted = true;
! /*
! * check if this line is an end marker -- "\."
! */
! end_marker = false;
!
! switch (eol_type)
! {
! case EOL_NL:
! if (!strcmp(line_buf.data, "\\.\n"))
! end_marker = true;
! break;
! case EOL_CR:
! if (!strcmp(line_buf.data, "\\.\r"))
! end_marker = true;
! break;
! case EOL_CRNL:
! if (!strcmp(line_buf.data, "\\.\r\n"))
! end_marker = true;
! break;
! case EOL_UNKNOWN:
! break;
! }
!
! if (end_marker)
! {
! fe_eof = true;
! /* we don't want to process a \. as data line, want to quit. */
! line_done = false;
! buf_done = true;
! }
!
! return line_done;
}
/*
* Return decimal value for a hexadecimal digit
*/
static
! int
! GetDecimalFromHex(char hex)
{
if (isdigit(hex))
return hex - '0';
***************
*** 2287,2344 ****
return tolower(hex) - 'a' + 10;
}
! /*----------
! * Read the value of a single attribute, performing de-escaping as needed.
! *
! * delim is the column delimiter string (must be just one byte for now).
! * null_print is the null marker string. Note that this is compared to
! * the pre-de-escaped input string.
! *
! * *result is set to indicate what terminated the read:
! * NORMAL_ATTR: column delimiter
! * END_OF_LINE: end of line
! * In either case, the string read up to the terminator is returned.
! *
! * *isnull is set true or false depending on whether the input matched
! * the null marker. Note that the caller cannot check this since the
! * returned string will be the post-de-escaping equivalent, which may
! * look the same as some valid data string.
! *----------
*/
! static char *
! CopyReadAttribute(const char *delim, const char *null_print,
! CopyReadResult *result, bool *isnull)
{
char c;
! char delimc = delim[0];
! int start_cursor = line_buf.cursor;
! int end_cursor;
! int input_len;
! /* reset attribute_buf to empty */
! attribute_buf.len = 0;
! attribute_buf.data[0] = '\0';
! /* set default status */
! *result = END_OF_LINE;
! for (;;)
{
! end_cursor = line_buf.cursor;
! if (line_buf.cursor >= line_buf.len)
! break;
! c = line_buf.data[line_buf.cursor++];
! if (c == delimc)
{
! *result = NORMAL_ATTR;
! break;
}
! if (c == '\\')
{
! if (line_buf.cursor >= line_buf.len)
! break;
! c = line_buf.data[line_buf.cursor++];
! switch (c)
{
case '0':
case '1':
--- 2482,2771 ----
return tolower(hex) - 'a' + 10;
}
! /*
! * Detected the eol type by looking at the first data row.
! * Possible eol types are NL, CR, or CRNL. If eol type was
! * detected, it is set and a boolean true is returned to
! * indicated detection was successful. If the first data row
! * is longer than the input buffer, we return false and will
! * try again in the next buffer.
*/
! static bool
! DetectLineEnd(size_t bytesread, char *quote, char *escape)
{
+ int index = 0;
char c;
! char quotec = '\0',
! escapec = '\0';
! bool csv = false;
! if (quote) /* CSV format */
! {
! csv = true;
! quotec = quote[0];
! escapec = escape[0];
! /* ignore special escape processing if it's the same as quotec */
! if (quotec == escapec)
! escapec = '\0';
! }
! while (index < COPY_BUF_SIZE)
! {
! c = input_buf[index];
! if(csv)
{
! if (in_quote && c == escapec)
! last_was_esc = !last_was_esc;
! if (c == quotec && !last_was_esc)
! in_quote = !in_quote;
! if (c != escapec)
! last_was_esc = false;
! }
!
! if (c == '\n')
{
! if(!csv || (csv && !in_quote))
! {
! eol_type = EOL_NL;
! eol_ch[0] = '\n';
! eol_ch[1] = '\0';
!
! in_quote = false;
! last_was_esc = false;
! return true;
! }
}
! if (c == '\r')
{
! if(!csv || (csv && !in_quote))
! {
! if (input_buf[index + 1] == '\n') /* always safe */
! {
! eol_type = EOL_CRNL;
! eol_ch[0] = '\r';
! eol_ch[1] = '\n';
! }
! else
! {
! eol_type = EOL_CR;
! eol_ch[0] = '\r';
! eol_ch[1] = '\0';
! }
!
! in_quote = false;
! last_was_esc = false;
! return true;
! }
! }
!
! index++;
! }
!
! return false;
! }
!
! /*
! * Read all TEXT attributes. Attributes are parsed from line_buf and
! * inserted (all at once) to attr_buf, while saving pointers to
! * each attribute's starting position.
! *
! * When this routine finishes execution both the nulls array and
! * the attr_offsets array are updated. The attr_offsets will include
! * the offset from the beginning of the attribute array of which
! * each attribute begins. If a specific attribute is not used for this
! * COPY command (ommitted from the column list), a value of 0 will be assigned.
! * For example: for table foo(a,b,c,d,e) and COPY foo(a,b,e)
! * attr_offsets may look something like this after this routine
! * returns: [0,20,0,0,55]. That means that column "a" value starts
! * at byte offset 0, "b" in 20 and "e" in 55, in attr_buf.
! *
! * In the attribute buffer (attr_buf) each attribute
! * is terminated with a '\0', and therefore by using the attr_offsets
! * array we could point to a beginning of an attribute and have it
! * behave as a C string, much like previously done in COPY.
! *
! * Another aspect to improving performance is reducing the frequency
! * of data load into buffers. The original COPY read attribute code
! * loaded a character at a time. In here we try to load a chunk of data
! * at a time. Usually a chunk will include a full data row
! * (unless we have an escaped delim). That effectively reduces the number of
! * loads by a factor of number of bytes per row. This improves performance
! * greatly, unfortunately it add more complexity to the code.
! *
! * Global participants in parsing logic:
! *
! * line_buf.cursor -- an offset from beginning of the line buffer
! * that indicates where we are about to begin the next scan. Note that
! * if we have WITH OIDS this cursor is already shifted past the first
! * OID attribute.
! *
! * attr_buf.cursor -- an offset from the beginning of the
! * attribute buffer that indicates where the current attribute begins.
! */
! static void
! CopyReadAttributesText(const char *delim, const char *escape, const char *null_print,
! int null_print_len, char *nulls, List *attnumlist,
! int *attr_offsets, int num_phys_attrs, Form_pg_attribute *attr)
! {
! char delimc = delim[0]; /* delimiter character */
! char escapec = escape[0]; /* escape character */
! char *scan_start; /* pointer to line buffer for scan start. */
! char *scan_end; /* pointer to line buffer where char was found */
! int attr_pre_len; /* attr raw len, before processing escapes */
! int attr_post_len; /* current attr len after escaping */
! int m; /* attribute index being parsed */
! int bytes_remaining;/* num bytes remaining to be scanned in line buf */
! int chunk_start; /* offset to beginning of line chunk to load */
! int chunk_len; /* length of chunk of data to load to attr buf */
! int oct_val; /* byte value for octal escapes */
! int hex_val; /* byte value for hexadecimal escapes */
! char hexchar; /* char that appears after \x for hex values */
! int attnum; /* attribute number being parsed */
! ListCell *cur; /* cursor to attribute list used for this COPY */
! int attribute;
!
! /*
! * init variables for attribute scan
! */
! attr_buf.len = 0;
! attr_buf.data[0] = '\0';
! attr_buf.cursor = 0;
! /* cursor is now > 0 if we copy WITH OIDS */
! scan_start = line_buf.data + line_buf.cursor;
! cur = list_head(attnumlist);
! attnum = lfirst_int(cur);
! m = attnum - 1;
! chunk_start = line_buf.cursor;
! chunk_len = 0;
! attr_pre_len = 0;
! attr_post_len = 0;
!
! /*
! * Scan through the line buffer to read all attributes data
! */
! while (line_buf.cursor < line_buf.len)
! {
! bytes_remaining = line_buf.len - line_buf.cursor;
!
! if ((scan_end = scanTextAttr(scan_start, delimc, escapec, bytes_remaining))
! == NULL)
! {
! /* GOT TO END OF LINE BUFFER */
!
! if (cur == NULL)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("extra data after last expected column")));
!
! attnum = lfirst_int(cur);
! m = attnum - 1;
!
! /* don't count eol char(s) in attr and chunk len calculation */
! if (eol_type == EOL_CRNL)
! {
! attr_pre_len += bytes_remaining - 2;
! chunk_len = line_buf.len - chunk_start - 2;
! }
! else
! {
! attr_pre_len += bytes_remaining - 1;
! chunk_len = line_buf.len - chunk_start - 1;
! }
!
! /* check if this is a NULL value or data value (assumed NULL) */
! if (attr_pre_len == null_print_len
! &&
! strncmp(line_buf.data + line_buf.len - attr_pre_len - 1, null_print, attr_pre_len)
! == 0)
! nulls[m] = 'n';
! else
! nulls[m] = ' ';
!
! attr_offsets[m] = attr_buf.cursor;
!
!
! /* load the last chunk, the whole buffer in most cases */
! appendBinaryStringInfo(&attr_buf, line_buf.data + chunk_start, chunk_len);
!
! line_buf.cursor += attr_pre_len + 2; /* skip eol char and
! * '\0' to exit loop */
!
! if (lnext(cur) != NULL)
! {
! /*
! * For an empty data line, the previous COPY code will
! * fail it during the conversion stage. We can fail it here
! * already, but then we will fail the regression tests b/c
! * of a different error message. that's why we return so we
! * can get the same error message that regress expects. ahh...
! * this conditional is unnecessary and should be removed soon.
! */
! if (line_buf.len > 1)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("missing data for column \"%s\"",
! NameStr(attr[m + 1]->attname))));
! else
! return;
! }
! }
! else
! /* FOUND A DELIMITER OR ESCAPE */
! {
! if (cur == NULL)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("extra data after last expected column")));
!
! if (*scan_end == delimc) /* found a delimiter */
! {
! attnum = lfirst_int(cur);
! m = attnum - 1;
!
! /* (we don't include the delimiter ch in length) */
! attr_pre_len += scan_end - scan_start;
! /* (we don't include the delimiter ch in length) */
! attr_post_len += scan_end - scan_start;
!
! /* check if this is a null print or data (assumed NULL) */
! if (attr_pre_len == null_print_len
! &&
! strncmp(scan_end - attr_pre_len, null_print, attr_pre_len)
! == 0)
! nulls[m] = 'n';
! else
! nulls[m] = ' ';
!
! /* set the pointer to next attribute position */
! attr_offsets[m] = attr_buf.cursor;
!
! /*
! * update buffer cursors to our current location, +1 to skip
! * the delimc
! */
! line_buf.cursor = scan_end - line_buf.data + 1;
! attr_buf.cursor += attr_post_len + 1;
!
! /* prepare scan for next attr */
! scan_start = line_buf.data + line_buf.cursor;
! cur = lnext(cur);
! attr_pre_len = 0;
! attr_post_len = 0;
! }
! else
! /* found an escape character */
! {
! char nextc = *(scan_end + 1);
! char newc;
! int skip = 2;
!
! chunk_len = (scan_end - line_buf.data) - chunk_start + 1;
!
! /* load a chunk of data */
! appendBinaryStringInfo(&attr_buf, line_buf.data + chunk_start, chunk_len);
!
! switch (nextc)
{
case '0':
case '1':
***************
*** 2348,2445 ****
case '5':
case '6':
case '7':
! /* handle \013 */
! {
! int val;
! val = OCTVALUE(c);
! if (line_buf.cursor < line_buf.len)
! {
! c = line_buf.data[line_buf.cursor];
! if (ISOCTAL(c))
! {
! line_buf.cursor++;
! val = (val << 3) + OCTVALUE(c);
! if (line_buf.cursor < line_buf.len)
{
! c = line_buf.data[line_buf.cursor];
! if (ISOCTAL(c))
{
! line_buf.cursor++;
! val = (val << 3) + OCTVALUE(c);
}
}
! }
! }
! c = val & 0377;
! }
break;
case 'x':
/* Handle \x3F */
! if (line_buf.cursor < line_buf.len)
! {
! char hexchar = line_buf.data[line_buf.cursor];
if (isxdigit(hexchar))
{
! int val = GetDecimalFromHex(hexchar);
! line_buf.cursor++;
! if (line_buf.cursor < line_buf.len)
! {
! hexchar = line_buf.data[line_buf.cursor];
if (isxdigit(hexchar))
{
! line_buf.cursor++;
! val = (val << 4) + GetDecimalFromHex(hexchar);
! }
! }
! c = val & 0xff;
}
}
break;
case 'b':
! c = '\b';
break;
case 'f':
! c = '\f';
break;
case 'n':
! c = '\n';
break;
case 'r':
! c = '\r';
break;
case 't':
! c = '\t';
break;
case 'v':
! c = '\v';
break;
/*
! * in all other cases, take the char after '\'
! * literally
*/
}
! }
! appendStringInfoCharMacro(&attribute_buf, c);
}
! /* check whether raw input matched null marker */
! input_len = end_cursor - start_cursor;
! if (input_len == strlen(null_print) &&
! strncmp(&line_buf.data[start_cursor], null_print, input_len) == 0)
*isnull = true;
! else
! *isnull = false;
! return attribute_buf.data;
! }
/*
! * Read the value of a single attribute in CSV mode,
* performing de-escaping as needed. Escaping does not follow the normal
* PostgreSQL text mode, but instead "standard" (i.e. common) CSV usage.
*
--- 2775,2948 ----
case '5':
case '6':
case '7':
! oct_val = OCTVALUE(nextc);
! nextc = *(scan_end + 2);
! /*
! * (no need for out bad access check since line if
! * buffered)
! */
! if (ISOCTAL(nextc))
{
! skip++;
! oct_val = (oct_val << 3) + OCTVALUE(nextc);
! nextc = *(scan_end + 3);
! if (ISOCTAL(nextc))
{
! skip++;
! oct_val = (oct_val << 3) + OCTVALUE(nextc);
}
}
! newc = oct_val & 0377; /* the escaped byte value */
break;
case 'x':
/* Handle \x3F */
! hexchar = *(scan_end + 2);
if (isxdigit(hexchar))
{
! skip++;
! hex_val = GetDecimalFromHex(hexchar);
! hexchar = *(scan_end + 3);
if (isxdigit(hexchar))
{
! skip++;
! hex_val = (hex_val << 4) + GetDecimalFromHex(hexchar);
}
+ newc = hex_val & 0xff;
}
+ else /* "\x" with no hex value */
+ newc = nextc;
break;
case 'b':
! newc = '\b';
break;
case 'f':
! newc = '\f';
break;
case 'n':
! newc = '\n';
break;
case 'r':
! newc = '\r';
break;
case 't':
! newc = '\t';
break;
case 'v':
! newc = '\v';
! break;
! default:
! if (nextc == delimc)
! newc = delimc;
! else if (nextc == escapec)
! newc = escapec;
! else
! /* no escape sequence, take next char literaly */
! newc = nextc;
break;
+ }
+
+ /* update to current length, add escape and escaped chars */
+ attr_pre_len += scan_end - scan_start + 2;
+ /* update to current length, escaped char */
+ attr_post_len += scan_end - scan_start + 1;
/*
! * Need to get rid of the escape character. This is done by
! * loading the chunk up to including the escape character
! * into the attribute buffer. Then overwritting the backslash
! * with the escaped sequence or char, and continuing to scan
! * from *after* the char than is after the escape in line buf.
*/
+ *(attr_buf.data + attr_buf.len - 1) = newc;
+ line_buf.cursor = scan_end - line_buf.data + skip;
+ scan_start = scan_end + skip;
+ chunk_start = line_buf.cursor;
+ chunk_len = 0;
}
!
! } /* end delimiter/backslash */
!
! } /* end line buffer scan. */
!
! /*
! * Replace all delimiters with NULL for string termination.
! * NOTE: only delimiters (NOT necessarily all delimc) are replaced.
! * Example (delimc = '|'):
! * - Before: f 1 | f \| 2 | f 3
! * - After : f 1 \0 f | 2 \0 f 3
! */
! for (attribute = 1; attribute < num_phys_attrs; attribute++)
! {
! if (attr_offsets[attribute] != 0)
! *(attr_buf.data + attr_offsets[attribute] - 1) = '\0';
}
! }
!
!
! /*
! * Read a binary attribute
! */
! static Datum
! CopyReadBinaryAttribute(int column_no, FmgrInfo *flinfo,
! Oid typioparam, int32 typmod,
! bool *isnull)
! {
! int32 fld_size;
! Datum result;
!
! fld_size = CopyGetInt32();
! if (CopyGetEof())
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unexpected EOF in COPY data")));
! if (fld_size == -1)
! {
*isnull = true;
! return (Datum) 0;
! }
! if (fld_size < 0)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("invalid field size")));
! /* reset attr_buf to empty, and load raw data in it */
! attr_buf.len = 0;
! attr_buf.data[0] = '\0';
! attr_buf.cursor = 0;
!
! enlargeStringInfo(&attr_buf, fld_size);
!
! CopyGetData(attr_buf.data, fld_size);
! if (CopyGetEof())
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unexpected EOF in COPY data")));
+ attr_buf.len = fld_size;
+ attr_buf.data[fld_size] = '\0';
+
+ /* Call the column type's binary input converter */
+ result = FunctionCall3(flinfo,
+ PointerGetDatum(&attr_buf),
+ ObjectIdGetDatum(typioparam),
+ Int32GetDatum(typmod));
+
+ /* Trouble if it didn't eat the whole buffer */
+ if (attr_buf.cursor != attr_buf.len)
+ ereport(ERROR,
+ (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
+ errmsg("incorrect binary data format")));
+
+ *isnull = false;
+ return result;
+ }
/*
! * Read all the attributes of the data line in CSV mode,
* performing de-escaping as needed. Escaping does not follow the normal
* PostgreSQL text mode, but instead "standard" (i.e. common) CSV usage.
*
***************
*** 2448,2472 ****
*
* null_print is the null marker string. Note that this is compared to
* the pre-de-escaped input string (thus if it is quoted it is not a NULL).
- *
- * *result is set to indicate what terminated the read:
- * NORMAL_ATTR: column delimiter
- * END_OF_LINE: end of line
- * UNTERMINATED_FIELD no quote detected at end of a quoted field
- *
- * In any case, the string read up to the terminator (or end of file)
- * is returned.
- *
- * *isnull is set true or false depending on whether the input matched
- * the null marker. Note that the caller cannot check this since the
- * returned string will be the post-de-escaping equivalent, which may
- * look the same as some valid data string.
*----------
*/
!
! static char *
! CopyReadAttributeCSV(const char *delim, const char *null_print, char *quote,
! char *escape, CopyReadResult *result, bool *isnull)
{
char delimc = delim[0];
char quotec = quote[0];
--- 2951,2962 ----
*
* null_print is the null marker string. Note that this is compared to
* the pre-de-escaped input string (thus if it is quoted it is not a NULL).
*----------
*/
! static void
! CopyReadAttributesCSV(const char *delim, const char *null_print, char *quote,
! char *escape, int null_print_len, char *nulls, List *attnumlist,
! int *attr_offsets, int num_phys_attrs, Form_pg_attribute *attr)
{
char delimc = delim[0];
char quotec = quote[0];
***************
*** 2477,2502 ****
int input_len;
bool in_quote = false;
bool saw_quote = false;
! /* reset attribute_buf to empty */
! attribute_buf.len = 0;
! attribute_buf.data[0] = '\0';
- /* set default status */
- *result = END_OF_LINE;
for (;;)
{
end_cursor = line_buf.cursor;
if (line_buf.cursor >= line_buf.len)
break;
c = line_buf.data[line_buf.cursor++];
/* unquoted field delimiter */
if (!in_quote && c == delimc)
{
! *result = NORMAL_ATTR;
! break;
}
/* start of quoted field (or part of field) */
--- 2967,3047 ----
int input_len;
bool in_quote = false;
bool saw_quote = false;
+ int attnum; /* attribute number being parsed */
+ int m; /* attribute index being parsed */
+ ListCell *cur; /* cursor to attribute list used for this COPY */
! /*
! * init variables for attribute scan
! */
! attr_buf.len = 0;
! attr_buf.data[0] = '\0';
! attr_buf.cursor = 0;
!
! cur = list_head(attnumlist);
! attnum = lfirst_int(cur);
! m = attnum - 1;
! input_len = 0;
for (;;)
{
end_cursor = line_buf.cursor;
+
+ /* finished processing attributes in line */
if (line_buf.cursor >= line_buf.len)
+ {
+ /* check whether raw input matched null marker */
+ input_len = end_cursor - start_cursor;
+ if (!saw_quote && input_len == null_print_len &&
+ strncmp(&line_buf.data[start_cursor], null_print, input_len) == 0)
+ nulls[m] = 'n';
+ else
+ nulls[m] = ' ';
+
+ if (in_quote)
+ ereport(ERROR,
+ (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
+ errmsg("unterminated CSV quoted field")));
+
+ if (lnext(cur) != NULL)
+ ereport(ERROR,
+ (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
+ errmsg("missing data for column \"%s\"",
+ NameStr(attr[m + 1]->attname))));
break;
+ }
+
c = line_buf.data[line_buf.cursor++];
/* unquoted field delimiter */
if (!in_quote && c == delimc)
{
! /* check whether raw input matched null marker */
! input_len = end_cursor - start_cursor;
! if (!saw_quote && input_len == null_print_len &&
! strncmp(&line_buf.data[start_cursor], null_print, input_len) == 0)
! nulls[m] = 'n';
! else
! nulls[m] = ' ';
!
! /* terminate attr string with '\0' */
! appendStringInfoCharMacro(&attr_buf, '\0');
! attr_buf.cursor++;
!
! /* setup next attribute scan */
! cur = lnext(cur);
!
! if (cur == NULL)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("extra data after last expected column")));
!
! attnum = lfirst_int(cur);
! m = attnum - 1;
! attr_offsets[m] = attr_buf.cursor;
! start_cursor = line_buf.cursor;
! continue;
}
/* start of quoted field (or part of field) */
***************
*** 2520,2527 ****
if (nextc == escapec || nextc == quotec)
{
! appendStringInfoCharMacro(&attribute_buf, nextc);
line_buf.cursor++;
continue;
}
}
--- 3065,3073 ----
if (nextc == escapec || nextc == quotec)
{
! appendStringInfoCharMacro(&attr_buf, nextc);
line_buf.cursor++;
+ attr_buf.cursor++;
continue;
}
}
***************
*** 2537,2616 ****
in_quote = false;
continue;
}
! appendStringInfoCharMacro(&attribute_buf, c);
}
- if (in_quote)
- *result = UNTERMINATED_FIELD;
-
- /* check whether raw input matched null marker */
- input_len = end_cursor - start_cursor;
- if (!saw_quote && input_len == strlen(null_print) &&
- strncmp(&line_buf.data[start_cursor], null_print, input_len) == 0)
- *isnull = true;
- else
- *isnull = false;
-
- return attribute_buf.data;
}
/*
! * Read a binary attribute
*/
! static Datum
! CopyReadBinaryAttribute(int column_no, FmgrInfo *flinfo,
! Oid typioparam, int32 typmod,
bool *isnull)
{
! int32 fld_size;
! Datum result;
!
! fld_size = CopyGetInt32();
! if (CopyGetEof())
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unexpected EOF in COPY data")));
! if (fld_size == -1)
! {
! *isnull = true;
! return (Datum) 0;
}
! if (fld_size < 0)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("invalid field size")));
! /* reset attribute_buf to empty, and load raw data in it */
! attribute_buf.len = 0;
! attribute_buf.data[0] = '\0';
! attribute_buf.cursor = 0;
! enlargeStringInfo(&attribute_buf, fld_size);
- CopyGetData(attribute_buf.data, fld_size);
- if (CopyGetEof())
- ereport(ERROR,
- (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
- errmsg("unexpected EOF in COPY data")));
! attribute_buf.len = fld_size;
! attribute_buf.data[fld_size] = '\0';
! /* Call the column type's binary input converter */
! result = FunctionCall3(flinfo,
! PointerGetDatum(&attribute_buf),
! ObjectIdGetDatum(typioparam),
! Int32GetDatum(typmod));
- /* Trouble if it didn't eat the whole buffer */
- if (attribute_buf.cursor != attribute_buf.len)
- ereport(ERROR,
- (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
- errmsg("incorrect binary data format")));
- *isnull = false;
- return result;
- }
/*
* Send text representation of one attribute, with conversion and escaping
--- 3083,3149 ----
in_quote = false;
continue;
}
! appendStringInfoCharMacro(&attr_buf, c);
! attr_buf.cursor++;
}
}
/*
! * Read the first attribute. This is mainly used to maintain support
! * for an OID column. All the rest of the columns will be read at once with
! * CopyReadAttributesText.
*/
! static char *
! CopyReadOidAttr(const char *delim, const char *null_print, int null_print_len,
bool *isnull)
{
! char delimc = delim[0];
! char *start_loc = line_buf.data + line_buf.cursor;
! char *end_loc;
! int attr_len = 0;
! int bytes_remaining;
!
! /* reset attribute buf to empty */
! attr_buf.len = 0;
! attr_buf.data[0] = '\0';
! attr_buf.cursor = 0;
!
! /* # of bytes that were not yet processed in this line */
! bytes_remaining = line_buf.len - line_buf.cursor;
!
! /* got to end of line */
! if ((end_loc = scanTextLine(start_loc, delimc, bytes_remaining)) == NULL)
! {
! attr_len = bytes_remaining - 1; /* don't count '\n' in len calculation */
! appendBinaryStringInfo(&attr_buf, start_loc, attr_len);
! line_buf.cursor += attr_len + 2; /* skip '\n' and '\0' */
}
! else
! /* found a delimiter */
! {
! /*
! * (we don't care if delim was preceded with a backslash, because it's
! * an invalid OID anyway)
! */
! attr_len = end_loc - start_loc; /* we don't include the delimiter ch */
! appendBinaryStringInfo(&attr_buf, start_loc, attr_len);
! line_buf.cursor += attr_len + 1;
! }
! /* check whether raw input matched null marker */
! if (attr_len == null_print_len && strncmp(start_loc, null_print, attr_len) == 0)
! *isnull = true;
! else
! *isnull = false;
! return attr_buf.data;
! }
/*
* Send text representation of one attribute, with conversion and escaping
***************
*** 2800,2802 ****
--- 3333,3436 ----
return attnums;
}
+
+ /*
+ * The following are custom versions of the string function strchr().
+ * As opposed to the original strchr which searches through
+ * a string until the target character is found, or a NULL is
+ * found, this version will not return when a NULL is found.
+ * Instead it will search through a pre-defined length of
+ * bytes and will return only if the target character(s) is reached.
+ *
+ * If our client encoding is not a supported server encoding, we
+ * know that it is not safe to look at each character as trailing
+ * byte in a multibyte character may be a 7-bit ASCII equivalent.
+ * Therefore we use pg_encoding_mblen to skip to the end of the
+ * character.
+ *
+ * returns:
+ * pointer to c - if c is located within the string.
+ * NULL - if c was not found in specified length of search. Note:
+ * this DOESN'T mean that a '\0' was reached.
+ */
+ char *
+ scanTextLine(const char *s, char eol, size_t len)
+ {
+ const char *start;
+
+ if (client_encoding_only && !line_buf_converted)
+ {
+ int mblen = pg_encoding_mblen(client_encoding, (unsigned char*)s);
+
+ for (start = s; *s != eol && s < start + len; s += mblen)
+ mblen = pg_encoding_mblen(client_encoding, (unsigned char*)s);
+
+ return ((*s == eol) ? (char *) s : NULL);
+ }
+ else
+ return memchr(s,eol,len);
+ }
+
+
+ char *
+ scanCSVLine(const char *s, char eol, char escapec, char quotec, size_t len)
+ {
+ const char *start;
+
+ if (client_encoding_only && !line_buf_converted)
+ {
+ int mblen = pg_encoding_mblen(client_encoding, (unsigned char*)s);
+
+ for (start = s; *s != eol && s < start + len; s += mblen)
+ {
+ if (in_quote && *s == escapec)
+ last_was_esc = !last_was_esc;
+ if (*s == quotec && !last_was_esc)
+ in_quote = !in_quote;
+ if (*s != escapec)
+ last_was_esc = false;
+
+ mblen = pg_encoding_mblen(client_encoding, (unsigned char*)s);
+ }
+ }
+ else
+ /* safe to scroll byte by byte */
+ {
+ for (start = s; *s != eol && s < start + len; s++)
+ {
+ if (in_quote && *s == escapec)
+ last_was_esc = !last_was_esc;
+ if (*s == quotec && !last_was_esc)
+ in_quote = !in_quote;
+ if (*s != escapec)
+ last_was_esc = false;
+ }
+
+ }
+
+ if(*s != escapec) /* found eol_ch */
+ last_was_esc = false;
+
+ return ((*s == eol) ? (char *) s : NULL);
+ }
+
+ /*
+ * Scan for 1 of 2 characters necessary for attribute parsing. No need
+ * for checking for multi-byte characters since conversion already
+ * happened.
+ */
+ char *
+ scanTextAttr(const char *s, char c1, char c2, size_t len)
+ {
+ const char *start;
+
+ for (start = s; *s != c1 && *s != c2 && s < start + len; s++)
+ ;
+
+ return (*s != '\0' ? (char *) s : NULL);
+ }
+
+
+
+
+