copyreadline-optimization-1.patch
text/x-diff
Filename: copyreadline-optimization-1.patch
Type: text/x-diff
Part: 0
Patch
Same data as JSON:
GET /api/v1/attachments/:id/patch
the parsed metadata as JSON — format, series position, per-file stats; never the diff bytes.
API reference →
Format: context
| File | + | − |
|---|---|---|
| src/backend/commands/copy.c | 415 | 0 |
*** a/src/backend/commands/copy.c
--- b/src/backend/commands/copy.c
***************
*** 143,167 **** typedef struct CopyStateData
/*
* Similarly, line_buf holds the whole input line being processed. The
! * input cycle is first to read the whole line into line_buf, convert it
! * to server encoding there, and then extract the individual attribute
! * fields into attribute_buf. line_buf is preserved unmodified so that we
! * can display it in error messages if appropriate.
*/
StringInfoData line_buf;
- bool line_buf_converted; /* converted to server encoding? */
/*
! * Finally, raw_buf holds raw data read from the data source (file or
! * client connection). CopyReadLine parses this data sufficiently to
! * locate line boundaries, then transfers the data to line_buf and
! * converts it. Note: we guarantee that there is a \0 at
! * raw_buf[raw_buf_len].
*/
#define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */
char *raw_buf;
int raw_buf_index; /* next byte to process */
int raw_buf_len; /* total # of bytes stored */
} CopyStateData;
typedef CopyStateData *CopyState;
--- 143,175 ----
/*
* Similarly, line_buf holds the whole input line being processed. The
! * input cycle is first to convert the input to server encoding in
! * raw_buf, then read the whole line into line_buf, and then extract the
! * individual attribute fields into attribute_buf. line_buf is preserved
! * unmodified so that we can display it in error messages if appropriate.
*/
StringInfoData line_buf;
/*
! * raw_buf holds raw data read from the data source (file or client
! * connection), converted to server encoding if necessary. CopyReadLine
! * parses this data sufficiently to locate line boundaries, then
! * transfers the data to line_buf. Note: we guarantee that there is
! * a \0 at raw_buf[raw_buf_len].
*/
#define RAW_BUF_SIZE 65536 /* we palloc RAW_BUF_SIZE+1 bytes */
char *raw_buf;
int raw_buf_index; /* next byte to process */
int raw_buf_len; /* total # of bytes stored */
+
+ /*
+ * Finally, unconverted_buf holds residual raw data read from data source
+ * that could not be converted to server encoding yet, because we don't
+ * have the whole multi-byte character yet.
+ */
+ #define MAX_CONVERSION_GROWTH 4 /* from mbutils.c */
+ char unconverted_buf[MAX_CONVERSION_GROWTH];
+ int unconverted_buf_len; /* total # of bytes stored */
} CopyStateData;
typedef CopyStateData *CopyState;
***************
*** 250,255 **** static void CopyOneRowTo(CopyState cstate, Oid tupleOid,
--- 258,264 ----
static void CopyFrom(CopyState cstate);
static bool CopyReadLine(CopyState cstate);
static bool CopyReadLineText(CopyState cstate);
+ static bool CopyReadLineCSV(CopyState cstate);
static int CopyReadAttributesText(CopyState cstate, int maxfields,
char **fieldvals);
static int CopyReadAttributesCSV(CopyState cstate, int maxfields,
***************
*** 650,656 **** CopyGetInt16(CopyState cstate, int16 *val)
/*
! * CopyLoadRawBuf loads some more data into raw_buf
*
* Returns TRUE if able to obtain at least one more byte, else FALSE.
*
--- 659,666 ----
/*
! * CopyLoadRawBuf loads some more data into raw_buf, converting to
! * server encoding if necessary.
*
* Returns TRUE if able to obtain at least one more byte, else FALSE.
*
***************
*** 664,669 **** CopyLoadRawBuf(CopyState cstate)
--- 674,683 ----
{
int nbytes;
int inbytes;
+ bool needmore;
+
+ getmore:
+ needmore = false;
if (cstate->raw_buf_index < cstate->raw_buf_len)
{
***************
*** 675,686 **** CopyLoadRawBuf(CopyState cstate)
else
nbytes = 0; /* no data need be saved */
! inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
! 1, RAW_BUF_SIZE - nbytes);
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
return (inbytes > 0);
}
--- 689,827 ----
else
nbytes = 0; /* no data need be saved */
! if (!cstate->need_transcoding)
! {
! /* Read data directly to raw_buf */
! inbytes = CopyGetData(cstate, cstate->raw_buf + nbytes,
! 1, RAW_BUF_SIZE - nbytes);
! }
! else
! {
! /*
! * Read data and convert to server encoding.
! */
! char *cvt;
! int convertable_bytes;
! char *raw;
!
! /*
! * Read data directly to raw_buf. That way, if pg_client_to_server
! * doesn't need to do any conversion after all, we avoid one memcpy.
! * Prepend any unconverted bytes from the last batch first.
! */
! raw = cstate->raw_buf + nbytes;
! memcpy(raw, cstate->unconverted_buf, cstate->unconverted_buf_len);
! inbytes = CopyGetData(cstate, raw + cstate->unconverted_buf_len, 1,
! (RAW_BUF_SIZE - nbytes - cstate->unconverted_buf_len) / MAX_CONVERSION_GROWTH); /* XXX: Is dividing with that right? */
! if (inbytes == 0 && cstate->unconverted_buf_len > 0)
! {
! /*
! * Oops, we hit EOF, but we still had a partial multi-byte
! * character stashed in unconverted_buf. Let pg_client_to_server()
! * throw an appropriate error on it.
! */
! (void) pg_client_to_server(raw, inbytes);
! /* pg_client_to_server should've errored already */
! elog(ERROR, "invalid multi-byte sequence at end of input");
! }
! inbytes += cstate->unconverted_buf_len;
! if (inbytes > 0)
! {
! /*
! * Determine the number of bytes that can be converted in this
! * batch. If we the batch ends with the a partial multi-byte
! * character, we must stop the conversion before that and stash
! * away the partial character to be processed on the next round.
! */
! if (cstate->client_encoding == PG_UTF8)
! {
! /*
! * In UTF-8, you can distinguish whether a byte belongs to
! * multi-byte characters, and if it's the first or non-first
! * byte in such a character, just by looking at the byte.
! * The first byte of a multibyte character always has 10 in
! * the two high bits, and a non-first byte has 11.
! *
! * Knowing that, start from the last byte read, and backtrack
! * to the beginning of the last character.
! *
! * XXX: Do other encodings have similar properties? This
! * should be generalized and pushed to wchar.c.
! */
! convertable_bytes = inbytes;
! while((raw[convertable_bytes - 1] & 0xC0) == 0x80 &&
! (inbytes - convertable_bytes) < MAX_CONVERSION_GROWTH)
! convertable_bytes--;
! }
! else if (pg_encoding_max_length(cstate->client_encoding) > 1)
! {
! /*
! * Otherwise we have to do it the hard way, starting from
! * the beginning and traversing with pg_encoding_mblen() to
! * the end.
! */
! convertable_bytes = 0;
! for (;;)
! {
! int n = pg_encoding_mblen(cstate->client_encoding,
! &raw[convertable_bytes]);
! if (n + convertable_bytes > inbytes)
! break;
! convertable_bytes += n;
! }
! }
! else
! convertable_bytes = inbytes;
!
! /*
! * Any multi-byte character should fit in MAX_CONVERSION_GROWTH,
! * so if we have more residual bytes than that, there's something
! * wrong with the input. Let pg_client_to_server() report the
! * error on it.
! */
! if (inbytes - convertable_bytes > MAX_CONVERSION_GROWTH)
! {
! cvt = pg_client_to_server(raw, inbytes);
! /* pg_client_to_server should've errored already */
! elog(ERROR, "unexpected multi-byte sequence");
! }
!
! /*
! * Stash away any unconverted bytes at the end for the next round
! */
! memcpy(cstate->unconverted_buf,
! raw + convertable_bytes,
! inbytes - convertable_bytes);
!
! /*
! * Perform conversion. If we didn't read enough to have a whole
! * character, try to read more, rather than be totally useless
! * to the caller.
! */
! if (convertable_bytes > 0)
! {
! cvt = pg_client_to_server(raw, convertable_bytes);
! if (cvt != raw)
! {
! inbytes = strlen(cvt);
! memcpy(cstate->raw_buf + nbytes, cvt, inbytes);
! pfree(cvt);
! }
! else
! inbytes = convertable_bytes;
! }
! else
! needmore = true;
! }
! }
nbytes += inbytes;
cstate->raw_buf[nbytes] = '\0';
cstate->raw_buf_index = 0;
cstate->raw_buf_len = nbytes;
+
+ if (needmore)
+ goto getmore;
+
return (inbytes > 0);
}
***************
*** 1160,1168 **** DoCopy(const CopyStmt *stmt, const char *queryString)
/* Set up variables to avoid per-attribute overhead. */
initStringInfo(&cstate->attribute_buf);
initStringInfo(&cstate->line_buf);
- cstate->line_buf_converted = false;
cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1);
cstate->raw_buf_index = cstate->raw_buf_len = 0;
cstate->processed = 0;
/*
--- 1301,1309 ----
/* Set up variables to avoid per-attribute overhead. */
initStringInfo(&cstate->attribute_buf);
initStringInfo(&cstate->line_buf);
cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1);
cstate->raw_buf_index = cstate->raw_buf_len = 0;
+ cstate->unconverted_buf_len = 0;
cstate->processed = 0;
/*
***************
*** 1596,1623 **** copy_in_error_callback(void *arg)
else
{
/* error is relevant to a particular line */
! if (cstate->line_buf_converted || !cstate->need_transcoding)
! {
! char *lineval;
! lineval = limit_printout_length(cstate->line_buf.data);
! errcontext("COPY %s, line %d: \"%s\"",
! cstate->cur_relname, cstate->cur_lineno, lineval);
! pfree(lineval);
! }
! else
! {
! /*
! * Here, the line buffer is still in a foreign encoding, and
! * indeed it's quite likely that the error is precisely a
! * failure to do encoding conversion (ie, bad data). We dare
! * not try to convert it, and at present there's no way to
! * regurgitate it without conversion. So we have to punt and
! * just report the line number.
! */
! errcontext("COPY %s, line %d",
! cstate->cur_relname, cstate->cur_lineno);
! }
}
}
}
--- 1737,1748 ----
else
{
/* error is relevant to a particular line */
! char *lineval;
! lineval = limit_printout_length(cstate->line_buf.data);
! errcontext("COPY %s, line %d: \"%s\"",
! cstate->cur_relname, cstate->cur_lineno, lineval);
! pfree(lineval);
}
}
}
***************
*** 2239,2249 **** CopyReadLine(CopyState cstate)
resetStringInfo(&cstate->line_buf);
- /* Mark that encoding conversion hasn't occurred yet */
- cstate->line_buf_converted = false;
-
/* Parse data and transfer into line_buf */
! result = CopyReadLineText(cstate);
if (result)
{
--- 2364,2374 ----
resetStringInfo(&cstate->line_buf);
/* Parse data and transfer into line_buf */
! if (cstate->csv_mode)
! result = CopyReadLineCSV(cstate);
! else
! result = CopyReadLineText(cstate);
if (result)
{
***************
*** 2294,2326 **** CopyReadLine(CopyState cstate)
}
}
! /* Done reading the line. Convert it to server encoding. */
! if (cstate->need_transcoding)
{
! char *cvt;
! cvt = pg_client_to_server(cstate->line_buf.data,
! cstate->line_buf.len);
! if (cvt != cstate->line_buf.data)
{
! /* transfer converted data back to line_buf */
! resetStringInfo(&cstate->line_buf);
! appendBinaryStringInfo(&cstate->line_buf, cvt, strlen(cvt));
! pfree(cvt);
}
- }
! /* Now it's safe to use the buffer in error messages */
! cstate->line_buf_converted = true;
return result;
}
/*
! * CopyReadLineText - inner loop of CopyReadLine for text mode
*/
static bool
! CopyReadLineText(CopyState cstate)
{
char *copy_raw_buf;
int raw_buf_ptr;
--- 2419,2676 ----
}
}
! return result;
! }
!
! /*
! * CopyReadLineText - inner loop of CopyReadLine for text mode
! */
! static bool
! CopyReadLineText(CopyState cstate)
! {
! char *copy_raw_buf;
! int raw_buf_ptr;
! int prev_raw_ptr;
! int copy_buf_len;
! bool need_data = false;
! bool hit_eof = false;
! bool result = false;
!
! /*
! * The objective of this loop is to transfer the entire next input line
! * into line_buf. Hence, we only care for detecting newlines (\r and/or
! * \n) and the end-of-copy marker (\.).
! *
! * For speed, we try to move data from raw_buf to line_buf in chunks
! * rather than one character at a time. raw_buf_ptr points to the next
! * character to examine; any characters from raw_buf_index to raw_buf_ptr
! * have been determined to be part of the line, but not yet transferred to
! * line_buf.
! *
! * For a little extra speed within the loop, we copy raw_buf and
! * raw_buf_len into local variables.
! */
! copy_raw_buf = cstate->raw_buf;
! raw_buf_ptr = cstate->raw_buf_index;
! copy_buf_len = cstate->raw_buf_len;
!
! for (;;)
{
! char c;
! char *cc;
! /*
! * Load more data if needed. Ideally we would just force four bytes
! * of read-ahead and avoid the many calls to
! * IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(), but the COPY_OLD_FE protocol
! * does not allow us to read too far ahead or we might read into the
! * next data, so we read-ahead only as far we know we can. One
! * optimization would be to read-ahead four byte here if
! * cstate->copy_dest != COPY_OLD_FE, but it hardly seems worth it,
! * considering the size of the buffer.
! */
! if (raw_buf_ptr >= copy_buf_len || need_data)
{
! REFILL_LINEBUF;
!
! /*
! * Try to read some more data. This will certainly reset
! * raw_buf_index to zero, and raw_buf_ptr must go with it.
! */
! if (!CopyLoadRawBuf(cstate))
! hit_eof = true;
! raw_buf_ptr = 0;
! copy_buf_len = cstate->raw_buf_len;
!
! /*
! * If we are completely out of data, break out of the loop,
! * reporting EOF.
! */
! if (copy_buf_len <= 0)
! {
! result = true;
! break;
! }
! need_data = false;
!
! /*
! * If line terminator is NL, check that there is no CRs in the
! * input, and vice versa. We do this here, rather than
! * line-at-a-time, for efficiency. Presumably the raw input
! * buffer is larger than an average line.
! *
! * CRNL mode is more complicated and is handled line-at-a-time.
! */
! if (cstate->eol_type == EOL_NL)
! {
! if (memchr(©_raw_buf[raw_buf_ptr], '\r',
! copy_buf_len - raw_buf_ptr) != NULL)
! {
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("literal carriage return found in data"),
! errhint("Use \"\\r\" to represent carriage return.")));
!
! }
! }
! if (cstate->eol_type == EOL_CR)
! {
! if (memchr(©_raw_buf[raw_buf_ptr], '\n',
! copy_buf_len - raw_buf_ptr) != NULL)
! {
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("literal newline found in data"),
! errhint("Use \"\\n\" to represent newline.")));
!
! }
! }
}
! /* Look for the first CR or LF */
! if (cstate->eol_type == EOL_UNKNOWN)
! {
! char *nl,
! *cr;
!
! nl = memchr(©_raw_buf[raw_buf_ptr], '\n',
! copy_buf_len - raw_buf_ptr);
! if (nl == NULL)
! nl = ©_raw_buf[copy_buf_len];
!
! cr = memchr(©_raw_buf[raw_buf_ptr], '\r',
! copy_buf_len - raw_buf_ptr);
! if (cr == NULL)
! cr = ©_raw_buf[copy_buf_len];
!
! if (nl < cr)
! {
! raw_buf_ptr = nl - copy_raw_buf + 1;
!
! cstate->eol_type = EOL_NL;
!
! /* If reach here, we have found the line terminator */
! break;
! }
! if (cr < nl)
! {
! prev_raw_ptr = cr - copy_raw_buf;
! raw_buf_ptr = prev_raw_ptr + 1;
!
! /*
! * If need more data, go back to loop top to load it.
! *
! * Note that if we are at EOF, c will wind up as '\0' because
! * of the guaranteed pad of raw_buf.
! */
! IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
!
! /* get next char */
! c = copy_raw_buf[raw_buf_ptr];
!
! if (c == '\n')
! {
! raw_buf_ptr++; /* eat newline */
! cstate->eol_type = EOL_CRNL; /* in case not set yet */
! }
! else
! {
! /* found \r, but no \n */
! /*
! * if we got here, it is the first line and we didn't find
! * \n, so don't consume the peeked character
! */
! cstate->eol_type = EOL_CR;
! }
! }
! /* If reach here, we have found the line terminator */
! break;
! }
!
! /* Search for the next line terminator */
! Assert(cstate->eol_type != EOL_UNKNOWN);
! if (cstate->eol_type == EOL_NL)
! {
! cc = memchr(©_raw_buf[raw_buf_ptr], '\n',
! copy_buf_len - raw_buf_ptr);
! if (cc == NULL)
! {
! /* No newline. Read more data */
! raw_buf_ptr = copy_buf_len;
! continue;
! }
! else
! {
! /* Found line terminator. */
! raw_buf_ptr = cc - copy_raw_buf + 1;
! break;
! }
! }
! else
! {
! cc = memchr(©_raw_buf[raw_buf_ptr], '\r',
! copy_buf_len - raw_buf_ptr);
! if (cc == NULL)
! {
! raw_buf_ptr = copy_buf_len;
! continue;
! }
! prev_raw_ptr = cc - copy_raw_buf;
! raw_buf_ptr = prev_raw_ptr + 1;
!
! if (cstate->eol_type == EOL_CRNL)
! {
! /*
! * If need more data, go back to loop top to load it.
! *
! * Note that if we are at EOF, c will wind up as '\0' because
! * of the guaranteed pad of raw_buf.
! */
! IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(0);
!
! /* ensure that there's no stray newlines before the CR */
! if (memchr(©_raw_buf[cstate->raw_buf_index], '\n',
! copy_buf_len - cstate->raw_buf_index) != NULL)
! {
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("literal newline found in data"),
! errhint("Use \"\\n\" to represent newline.")));
!
! }
!
! /* get next char */
! c = copy_raw_buf[raw_buf_ptr];
!
! if (c != '\n')
! {
! /* found \r, but no \n */
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("literal carriage return found in data"),
! errhint("Use \"\\r\" to represent carriage return.")));
! }
!
! raw_buf_ptr++; /* eat newline */
! }
! /* If reach here, we have found the line terminator */
! break;
! }
! } /* end of outer loop */
!
! /*
! * Transfer any still-uncopied data to line_buf.
! */
! REFILL_LINEBUF;
return result;
}
/*
! * CopyReadLineCSV - inner loop of CopyReadLine for CSV mode
*/
static bool
! CopyReadLineCSV(CopyState cstate)
{
char *copy_raw_buf;
int raw_buf_ptr;
***************
*** 2328,2334 **** CopyReadLineText(CopyState cstate)
bool need_data = false;
bool hit_eof = false;
bool result = false;
- char mblen_str[2];
/* CSV variables */
bool first_char_in_line = true;
--- 2678,2683 ----
***************
*** 2337,2343 **** CopyReadLineText(CopyState cstate)
char quotec = '\0';
char escapec = '\0';
- if (cstate->csv_mode)
{
quotec = cstate->quote[0];
escapec = cstate->escape[0];
--- 2686,2691 ----
***************
*** 2346,2353 **** CopyReadLineText(CopyState cstate)
escapec = '\0';
}
- mblen_str[1] = '\0';
-
/*
* The objective of this loop is to transfer the entire next input line
* into line_buf. Hence, we only care for detecting newlines (\r and/or
--- 2694,2699 ----
***************
*** 2417,2423 **** CopyReadLineText(CopyState cstate)
prev_raw_ptr = raw_buf_ptr;
c = copy_raw_buf[raw_buf_ptr++];
- if (cstate->csv_mode)
{
/*
* If character is '\\' or '\r', we may need to look ahead below.
--- 2763,2768 ----
***************
*** 2459,2465 **** CopyReadLineText(CopyState cstate)
}
/* Process \r */
! if (c == '\r' && (!cstate->csv_mode || !in_quote))
{
/* Check for \r\n on first line, _and_ handle \r\n. */
if (cstate->eol_type == EOL_UNKNOWN ||
--- 2804,2810 ----
}
/* Process \r */
! if (c == '\r' && !in_quote)
{
/* Check for \r\n on first line, _and_ handle \r\n. */
if (cstate->eol_type == EOL_UNKNOWN ||
***************
*** 2487,2497 **** CopyReadLineText(CopyState cstate)
if (cstate->eol_type == EOL_CRNL)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! !cstate->csv_mode ?
! errmsg("literal carriage return found in data") :
! errmsg("unquoted carriage return found in data"),
! !cstate->csv_mode ?
! errhint("Use \"\\r\" to represent carriage return.") :
errhint("Use quoted CSV field to represent carriage return.")));
/*
--- 2832,2838 ----
if (cstate->eol_type == EOL_CRNL)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("unquoted carriage return found in data"),
errhint("Use quoted CSV field to represent carriage return.")));
/*
***************
*** 2504,2530 **** CopyReadLineText(CopyState cstate)
else if (cstate->eol_type == EOL_NL)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
- !cstate->csv_mode ?
- errmsg("literal carriage return found in data") :
errmsg("unquoted carriage return found in data"),
- !cstate->csv_mode ?
- errhint("Use \"\\r\" to represent carriage return.") :
errhint("Use quoted CSV field to represent carriage return.")));
/* If reach here, we have found the line terminator */
break;
}
/* Process \n */
! if (c == '\n' && (!cstate->csv_mode || !in_quote))
{
if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
- !cstate->csv_mode ?
- errmsg("literal newline found in data") :
errmsg("unquoted newline found in data"),
- !cstate->csv_mode ?
- errhint("Use \"\\n\" to represent newline.") :
errhint("Use quoted CSV field to represent newline.")));
cstate->eol_type = EOL_NL; /* in case not set yet */
/* If reach here, we have found the line terminator */
--- 2845,2863 ----
else if (cstate->eol_type == EOL_NL)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("unquoted carriage return found in data"),
errhint("Use quoted CSV field to represent carriage return.")));
/* If reach here, we have found the line terminator */
break;
}
/* Process \n */
! if (c == '\n' && !in_quote)
{
if (cstate->eol_type == EOL_CR || cstate->eol_type == EOL_CRNL)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("unquoted newline found in data"),
errhint("Use quoted CSV field to represent newline.")));
cstate->eol_type = EOL_NL; /* in case not set yet */
/* If reach here, we have found the line terminator */
***************
*** 2535,2541 **** CopyReadLineText(CopyState cstate)
* In CSV mode, we only recognize \. alone on a line. This is because
* \. is a valid CSV data value.
*/
! if (c == '\\' && (!cstate->csv_mode || first_char_in_line))
{
char c2;
--- 2868,2874 ----
* In CSV mode, we only recognize \. alone on a line. This is because
* \. is a valid CSV data value.
*/
! if (c == '\\' && first_char_in_line)
{
char c2;
***************
*** 2568,2588 **** CopyReadLineText(CopyState cstate)
if (c2 == '\n')
{
! if (!cstate->csv_mode)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("end-of-copy marker does not match previous newline style")));
! else
! NO_END_OF_COPY_GOTO;
}
else if (c2 != '\r')
{
! if (!cstate->csv_mode)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("end-of-copy marker corrupt")));
! else
! NO_END_OF_COPY_GOTO;
}
}
--- 2901,2911 ----
if (c2 == '\n')
{
! NO_END_OF_COPY_GOTO;
}
else if (c2 != '\r')
{
! NO_END_OF_COPY_GOTO;
}
}
***************
*** 2593,2604 **** CopyReadLineText(CopyState cstate)
if (c2 != '\r' && c2 != '\n')
{
! if (!cstate->csv_mode)
! ereport(ERROR,
! (errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! errmsg("end-of-copy marker corrupt")));
! else
! NO_END_OF_COPY_GOTO;
}
if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
--- 2916,2922 ----
if (c2 != '\r' && c2 != '\n')
{
! NO_END_OF_COPY_GOTO;
}
if ((cstate->eol_type == EOL_NL && c2 != '\n') ||
***************
*** 2622,2641 **** CopyReadLineText(CopyState cstate)
result = true; /* report EOF */
break;
}
! else if (!cstate->csv_mode)
!
/*
* If we are here, it means we found a backslash followed by
! * something other than a period. In non-CSV mode, anything
! * after a backslash is special, so we skip over that second
! * character too. If we didn't do that \\. would be
! * considered an eof-of copy, while in non-CVS mode it is a
! * literal backslash followed by a period. In CSV mode,
* backslashes are not special, so we want to process the
* character after the backslash just like a normal character,
* so we don't increment in those cases.
*/
! raw_buf_ptr++;
}
/*
--- 2940,2955 ----
result = true; /* report EOF */
break;
}
! else
! {
/*
* If we are here, it means we found a backslash followed by
! * something other than a period. In CSV mode,
* backslashes are not special, so we want to process the
* character after the backslash just like a normal character,
* so we don't increment in those cases.
*/
! }
}
/*
***************
*** 2645,2669 **** CopyReadLineText(CopyState cstate)
* value, while in non-CSV mode, \. cannot be a data value.
*/
not_end_of_copy:
-
- /*
- * Process all bytes of a multi-byte character as a group.
- *
- * We only support multi-byte sequences where the first byte has the
- * high-bit set, so as an optimization we can avoid this block
- * entirely if it is not set.
- */
- if (cstate->encoding_embeds_ascii && IS_HIGHBIT_SET(c))
- {
- int mblen;
-
- mblen_str[0] = c;
- /* All our encodings only read the first byte to get the length */
- mblen = pg_encoding_mblen(cstate->client_encoding, mblen_str);
- IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1);
- IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1);
- raw_buf_ptr += mblen - 1;
- }
first_char_in_line = false;
} /* end of outer loop */
--- 2959,2964 ----